build.sh

   1 #!/usr/bin/env bash
   2
   3 # Work around Jenkins CI + msbuild problem: Jenkins sometimes creates very large environment
   4 # variables, and msbuild can't handle environment blocks with such large variables. So clear
   5 # out the variables that might be too large.
   6 export ghprbCommentBody=
   7
   8 # resolve python-version to use
   9 if [ "$PYTHON" == "" ] ; then
  10     if ! PYTHON=$(command -v python3 || command -v python2 || command -v python || command -v py)
  11     then
  12        echo "Unable to locate build-dependency python!" 1>&2
  13        exit 1
  14     fi
  15 fi
  16 # validate python-dependency
  17 # useful in case of explicitly set option.
  18 if ! command -v $PYTHON > /dev/null
  19 then
  20    echo "Unable to locate build-dependency python ($PYTHON)!" 1>&2
  21    exit 1
  22 fi
  23
  24 export PYTHON
  25
  26 usage()
  27 {
  28     echo "Usage: $0 [BuildArch] [BuildType] [-verbose] [-coverage] [-cross] [-gccx.y] [-clangx.y] [-ninja] [-configureonly] [-skipconfigure] [-skipnative] [-skipcrossarchnative] [-skipmanaged] [-skipmscorlib] [-skiptests] [-stripsymbols] [-ignorewarnings] [-cmakeargs] [-bindir]"
  29     echo "BuildArch can be: -x64, -x86, -arm, -armel, -arm64"
  30     echo "BuildType can be: -debug, -checked, -release"
  31     echo "-coverage - optional argument to enable code coverage build (currently supported only for Linux and OSX)."
  32     echo "-ninja - target ninja instead of GNU make"
  33     echo "-gccx.y - optional argument to build using gcc version x.y."
  34     echo "-clangx.y - optional argument to build using clang version x.y."
  35     echo "-cross - optional argument to signify cross compilation,"
  36     echo "       - will use ROOTFS_DIR environment variable if set."
  37     echo "-nopgooptimize - do not use profile guided optimizations."
  38     echo "-pgoinstrument - generate instrumented code for profile guided optimization enabled binaries."
  39     echo "-ibcinstrument - generate IBC-tuning-enabled native images when invoking crossgen."
  40     echo "-configureonly - do not perform any builds; just configure the build."
  41     echo "-skipconfigure - skip build configuration."
  42     echo "-skipnative - do not build native components."
  43     echo "-skipcrossarchnative - do not build cross-architecture native components."
  44     echo "-skipmanaged - do not build managed components."
  45     echo "-skipmscorlib - do not build mscorlib.dll."
  46     echo "-skiptests - skip the tests in the 'tests' subdirectory."
  47     echo "-skipnuget - skip building nuget packages."
  48     echo "-skiprestoreoptdata - skip restoring optimization data used by profile-based optimizations."
  49     echo "-skipcrossgen - skip native image generation"
  50     echo "-crossgenonly - only run native image generation"
  51     echo "-partialngen - build CoreLib as PartialNGen"
  52     echo "-verbose - optional argument to enable verbose build output."
  53     echo "-skiprestore: skip restoring packages ^(default: packages are restored during build^)."
  54     echo "-disableoss: Disable Open Source Signing for System.Private.CoreLib."
  55     echo "-officialbuildid=^<ID^>: specify the official build ID to be used by this build."
  56     echo "-stripSymbols - Optional argument to strip native symbols during the build."
  57     echo "-skipgenerateversion - disable version generation even if MSBuild is supported."
  58     echo "-ignorewarnings - do not treat warnings as errors"
  59     echo "-cmakeargs - user-settable additional arguments passed to CMake."
  60     echo "-bindir - output directory (defaults to $__ProjectRoot/bin)"
  61     echo "-msbuildonunsupportedplatform - build managed binaries even if distro is not officially supported."
  62     echo "-numproc - set the number of build processes."
  63     echo "-portablebuild - pass -portablebuild=false to force a non-portable build."
  64     echo "-staticanalyzer - build with clang static analyzer enabled."
  65     exit 1
  66 }
  67
  68 initTargetDistroRid()
  69 {
  70     source init-distro-rid.sh
  71
  72     local passedRootfsDir=""
  73
  74     # Only pass ROOTFS_DIR if cross is specified.
  75     if (( ${__CrossBuild} == 1 )); then
  76         passedRootfsDir=${ROOTFS_DIR}
  77     elif [ "${__BuildArch}" != "${__HostArch}" ]; then
  78         echo "Error, you are building a cross scenario without passing -cross."
  79         exit 1
  80     fi
  81
  82     initDistroRidGlobal ${__BuildOS} ${__BuildArch} ${__PortableBuild} ${passedRootfsDir}
  83 }
  84
  85 setup_dirs()
  86 {
  87     echo Setting up directories for build
  88
  89     mkdir -p "$__RootBinDir"
  90     mkdir -p "$__BinDir"
  91     mkdir -p "$__LogsDir"
  92     mkdir -p "$__MsbuildDebugLogsDir"
  93     mkdir -p "$__IntermediatesDir"
  94
  95     if [ $__CrossBuild == 1 ]; then
  96         mkdir -p "$__CrossComponentBinDir"
  97     fi
  98 }
  99
 100 # Check the system to ensure the right prereqs are in place
 101
 102 check_prereqs()
 103 {
 104     echo "Checking prerequisites..."
 105
 106     # Check presence of CMake on the path
 107     hash cmake 2>/dev/null || { echo >&2 "Please install cmake before running this script"; exit 1; }
 108
 109
 110     # Minimum required version of clang is version 4.0 for arm/armel cross build
 111     if [[ $__CrossBuild == 1 && $__GccBuild == 0 &&  ("$__BuildArch" == "arm" || "$__BuildArch" == "armel") ]]; then
 112         if ! [[ "$__ClangMajorVersion" -ge "4" ]]; then
 113             echo "Please install clang4.0 or latest for arm/armel cross build"; exit 1;
 114         fi
 115     fi
 116
 117     # Check for clang
 118     if [[ $__GccBuild == 0 ]]; then
 119         __ClangCombinedDottedVersion=$__ClangMajorVersion;
 120         if [[ "$__ClangMinorVersion" != "" ]]; then
 121             __ClangCombinedDottedVersion=$__ClangCombinedDottedVersion.$__ClangMinorVersion
 122         fi
 123         hash clang-$__ClangCombinedDottedVersion 2>/dev/null ||  hash clang$__ClangMajorVersion$__ClangMinorVersion 2>/dev/null || hash clang 2>/dev/null || { echo >&2 "Please install clang-$__ClangMajorVersion.$__ClangMinorVersion before running this script"; exit 1; }
 124     else
 125         __GccCombinedDottedVersion=$__GccMajorVersion;
 126         if [[ "$__GccMinorVersion" != "" ]]; then
 127             __GccCombinedDottedVersion=$__GccCombinedDottedVersion.$__GccMinorVersion
 128         fi
 129         hash gcc-$__GccCombinedDottedVersion 2>/dev/null ||  hash gcc$__GccMajorVersion$__GccMinorVersion 2>/dev/null || hash gcc 2>/dev/null || { echo >&2 "Please install gcc-$__GccMajorVersion.$__GccMinorVersion before running this script"; exit 1; }
 130     fi
 131
 132 }
 133
 134 restore_optdata()
 135 {
 136     # we only need optdata on a Release build
 137     if [[ "$__BuildType" != "Release" ]]; then __SkipRestoreOptData=1; fi
 138
 139     local OptDataProjectFilePath="$__ProjectRoot/src/.nuget/optdata/optdata.csproj"
 140     if [[ ( $__SkipRestoreOptData == 0 ) && ( $__isMSBuildOnNETCoreSupported == 1 ) ]]; then
 141         echo "Restoring the OptimizationData package"
 142         "$__ProjectRoot/dotnet.sh" restore /nologo /verbosity:minimal /clp:Summary /m \
 143                                    $OptDataProjectFilePath \
 144                                    $__CommonMSBuildArgs $__UnprocessedBuildArgs
 145         if [ $? != 0 ]; then
 146             echo "Failed to restore the optimization data package."
 147             exit 1
 148         fi
 149     fi
 150
 151     if [ $__isMSBuildOnNETCoreSupported == 1 ]; then
 152         # Parse the optdata package versions out of msbuild so that we can pass them on to CMake
 153
 154         # Writes into ${__IntermediatesDir}/optdataversion.txt
 155         ${__ProjectDir}/dotnet.sh msbuild $OptDataProjectFilePath /t:DumpPgoDataPackageVersion ${__CommonMSBuildArgs} /nologo 2>&1 >/dev/null
 156         if [ ! -f "${__IntermediatesDir}/optdataversion.txt" ]; then
 157             echo "Failed to get PGO data package version."
 158             exit $?
 159         fi
 160
 161         __PgoOptDataVersion=$(<"${__IntermediatesDir}/optdataversion.txt")
 162
 163         # Writes into ${__IntermediatesDir}/ibcoptdataversion.txt
 164         ${__ProjectDir}/dotnet.sh msbuild $OptDataProjectFilePath /t:DumpIbcDataPackageVersion ${__CommonMSBuildArgs} /nologo 2>&1 >/dev/null
 165         if [ ! -f "${__IntermediatesDir}/ibcoptdataversion.txt" ]; then
 166             echo "Failed to get IBC data package version."
 167             exit $?
 168         fi
 169
 170         __IbcOptDataVersion=$(<"${__IntermediatesDir}/ibcoptdataversion.txt")
 171     fi
 172 }
 173
 174 generate_event_logging_sources()
 175 {
 176     __OutputDir=$1
 177     __OutputEventingDir="$__OutputDir/Eventing"
 178
 179     __PythonWarningFlags="-Wall"
 180     if [[ $__IgnoreWarnings == 0 ]]; then
 181         __PythonWarningFlags="$__PythonWarningFlags -Werror"
 182     fi
 183
 184     echo "Laying out dynamically generated EventSource classes"
 185     $PYTHON -B $__PythonWarningFlags "$__ProjectRoot/src/scripts/genRuntimeEventSources.py" --man "$__ProjectRoot/src/vm/ClrEtwAll.man" --intermediate "$__OutputEventingDir"
 186 }
 187
 188 generate_event_logging()
 189 {
 190     # Event Logging Infrastructure
 191     if [[ $__SkipMSCorLib == 0 ]]; then
 192         generate_event_logging_sources "$__IntermediatesDir"
 193     fi
 194 }
 195
 196 build_native()
 197 {
 198     skipCondition=$1
 199     platformArch="$2"
 200     intermediatesForBuild="$3"
 201     extraCmakeArguments="$4"
 202     message="$5"
 203
 204     if [ $skipCondition == 1 ]; then
 205         echo "Skipping $message build."
 206         return
 207     fi
 208
 209     # All set to commence the build
 210     echo "Commencing build of $message for $__BuildOS.$__BuildArch.$__BuildType in $intermediatesForBuild"
 211
 212     generator=""
 213     buildFile="Makefile"
 214     buildTool="make"
 215     if [ $__UseNinja == 1 ]; then
 216         generator="ninja"
 217         buildFile="build.ninja"
 218         if ! buildTool=$(command -v ninja || command -v ninja-build); then
 219            echo "Unable to locate ninja!" 1>&2
 220            exit 1
 221         fi
 222     fi
 223
 224     if [ $__SkipConfigure == 0 ]; then
 225         # if msbuild is not supported, then set __SkipGenerateVersion to 1
 226         if [ $__isMSBuildOnNETCoreSupported == 0 ]; then __SkipGenerateVersion=1; fi
 227         # Drop version.c file
 228         __versionSourceFile="$intermediatesForBuild/version.c"
 229         if [ $__SkipGenerateVersion == 0 ]; then
 230             pwd
 231             "$__ProjectRoot/eng/common/msbuild.sh" $__ProjectRoot/eng/empty.csproj \
 232                                         /p:NativeVersionFile=$__versionSourceFile \
 233                                         /p:ArcadeBuild=true /t:GenerateNativeVersionFile /restore \
 234                                         $__CommonMSBuildArgs $__UnprocessedBuildArgs
 235             if [ $? -ne 0 ]; then
 236                 echo "Failed to generate native version file."
 237                 exit $?
 238             fi
 239         else
 240             # Generate the dummy version.c, but only if it didn't exist to make sure we don't trigger unnecessary rebuild
 241             __versionSourceLine="static char sccsid[] __attribute__((used)) = \"@(#)No version information produced\";"
 242             if [ -e $__versionSourceFile ]; then
 243                 read existingVersionSourceLine < $__versionSourceFile
 244             fi
 245             if [ "$__versionSourceLine" != "$existingVersionSourceLine" ]; then
 246                 echo $__versionSourceLine > $__versionSourceFile
 247             fi
 248         fi
 249
 250
 251         pushd "$intermediatesForBuild"
 252         # Regenerate the CMake solution
 253
 254         scriptDir="$__ProjectRoot/src/pal/tools"
 255         if [[ $__GccBuild == 0 ]]; then
 256             scan_build=
 257             if [[ $__StaticAnalyzer == 1 ]]; then
 258                 scan_build=scan-build
 259             fi
 260             echo "Invoking \"$scriptDir/gen-buildsys-clang.sh\" \"$__ProjectRoot\" $__ClangMajorVersion \"$__ClangMinorVersion\" $platformArch "$scriptDir" $__BuildType $__CodeCoverage $scan_build $generator $extraCmakeArguments $__cmakeargs"
 261             source "$scriptDir/gen-buildsys-clang.sh" "$__ProjectRoot" $__ClangMajorVersion "$__ClangMinorVersion" $platformArch "$scriptDir" $__BuildType $__CodeCoverage $scan_build $generator "$extraCmakeArguments" "$__cmakeargs"
 262         else
 263             echo "Invoking \"$scriptDir/gen-buildsys-gcc.sh\" \"$__ProjectRoot\" $__GccMajorVersion \"$__GccMinorVersion\" $platformArch "$scriptDir" $__BuildType $__CodeCoverage $generator $extraCmakeArguments $__cmakeargs"
 264             source "$scriptDir/gen-buildsys-gcc.sh" "$__ProjectRoot" "$__GccMajorVersion" "$__CGccMinorVersion" $platformArch "$scriptDir" $__BuildType $__CodeCoverage $generator "$extraCmakeArguments" "$__cmakeargs"
 265         fi
 266         popd
 267     fi
 268
 269     if [ ! -f "$intermediatesForBuild/$buildFile" ]; then
 270         echo "Failed to generate $message build project!"
 271         exit 1
 272     fi
 273
 274     # Build
 275     if [ $__ConfigureOnly == 1 ]; then
 276         echo "Finish configuration & skipping $message build."
 277         return
 278     fi
 279
 280     # Check that the makefiles were created.
 281     pushd "$intermediatesForBuild"
 282
 283     if [ $__StaticAnalyzer == 1 ]; then
 284         buildTool="$SCAN_BUILD_COMMAND $buildTool"
 285     fi
 286
 287     echo "Executing $buildTool install -j $__NumProc"
 288
 289     $buildTool install -j $__NumProc
 290     if [ $? != 0 ]; then
 291         echo "Failed to build $message."
 292         exit 1
 293     fi
 294
 295     popd
 296 }
 297
 298 build_cross_architecture_components()
 299 {
 300     local intermediatesForBuild="$__IntermediatesDir/Host$__CrossArch/crossgen"
 301     local crossArchBinDir="$__BinDir/$__CrossArch"
 302
 303     mkdir -p "$intermediatesForBuild"
 304     mkdir -p "$crossArchBinDir"
 305
 306     generate_event_logging_sources "$intermediatesForBuild" "the crossarch build system"
 307
 308     __SkipCrossArchBuild=1
 309     # check supported cross-architecture components host(__HostArch)/target(__BuildArch) pair
 310     if [[ ("$__BuildArch" == "arm" || "$__BuildArch" == "armel") && ("$__CrossArch" == "x86" || "$__CrossArch" == "x64") ]]; then
 311         __SkipCrossArchBuild=0
 312     elif [[ "$__BuildArch" == "arm64" && "$__CrossArch" == "x64" ]]; then
 313         __SkipCrossArchBuild=0
 314     else
 315         # not supported
 316         return
 317     fi
 318
 319     export __CMakeBinDir="$crossArchBinDir"
 320     export CROSSCOMPILE=0
 321
 322     __ExtraCmakeArgs="-DCLR_CMAKE_TARGET_ARCH=$__BuildArch -DCLR_CMAKE_TARGET_OS=$__BuildOS -DCLR_CMAKE_PACKAGES_DIR=$__PackagesDir -DCLR_CMAKE_PGO_INSTRUMENT=$__PgoInstrument -DCLR_CMAKE_OPTDATA_VERSION=$__PgoOptDataVersion -DCLR_CMAKE_PGO_OPTIMIZE=$__PgoOptimize -DCLR_CROSS_COMPONENTS_BUILD=1"
 323     build_native $__SkipCrossArchBuild "$__CrossArch" "$intermediatesForBuild" "$__ExtraCmakeArgs" "cross-architecture components"
 324
 325     export CROSSCOMPILE=1
 326 }
 327
 328 isMSBuildOnNETCoreSupported()
 329 {
 330     __isMSBuildOnNETCoreSupported=$__msbuildonunsupportedplatform
 331
 332     if [ $__isMSBuildOnNETCoreSupported == 1 ]; then
 333         return
 334     fi
 335
 336     if [ $__SkipManaged == 1 ]; then
 337         __isMSBuildOnNETCoreSupported=0
 338         return
 339     fi
 340
 341     if [ "$__HostArch" == "x64" ]; then
 342         if [ "$__HostOS" == "Linux" ]; then
 343             __isMSBuildOnNETCoreSupported=1
 344             # note: the RIDs below can use globbing patterns
 345             UNSUPPORTED_RIDS=("ubuntu.17.04-x64")
 346             for UNSUPPORTED_RID in "${UNSUPPORTED_RIDS[@]}"
 347             do
 348                 if [[ ${__DistroRid} == $UNSUPPORTED_RID ]]; then
 349                     __isMSBuildOnNETCoreSupported=0
 350                     break
 351                 fi
 352             done
 353         elif [ "$__HostOS" == "OSX" ]; then
 354             __isMSBuildOnNETCoreSupported=1
 355         elif [ "$__HostOS" == "FreeBSD" ]; then
 356             __isMSBuildOnNETCoreSupported=1
 357         fi
 358     fi
 359 }
 360
 361
 362 build_CoreLib_ni()
 363 {
 364     local __CrossGenExec=$1
 365     local __CoreLibILDir=$2
 366
 367     if [ $__PartialNgen == 1 ]; then
 368         export COMPlus_PartialNGen=1
 369     fi
 370
 371     if [ -e $__CrossGenCoreLibLog ]; then
 372         rm $__CrossGenCoreLibLog
 373     fi
 374     echo "Generating native image of System.Private.CoreLib.dll for $__BuildOS.$__BuildArch.$__BuildType. Logging to \"$__CrossGenCoreLibLog\"."
 375     echo "$__CrossGenExec /Platform_Assemblies_Paths $__CoreLibILDir $__IbcTuning /out $__BinDir/System.Private.CoreLib.dll $__CoreLibILDir/System.Private.CoreLib.dll"
 376     $__CrossGenExec /Platform_Assemblies_Paths $__CoreLibILDir $__IbcTuning /out $__BinDir/System.Private.CoreLib.dll $__CoreLibILDir/System.Private.CoreLib.dll >> $__CrossGenCoreLibLog 2>&1
 377     if [ $? -ne 0 ]; then
 378         echo "Failed to generate native image for System.Private.CoreLib. Refer to $__CrossGenCoreLibLog"
 379         exit 1
 380     fi
 381
 382     if [ "$__BuildOS" == "Linux" ]; then
 383         echo "Generating symbol file for System.Private.CoreLib.dll"
 384         echo "$__CrossGenExec /Platform_Assemblies_Paths $__BinDir /CreatePerfMap $__BinDir $__BinDir/System.Private.CoreLib.dll"
 385         $__CrossGenExec /Platform_Assemblies_Paths $__BinDir /CreatePerfMap $__BinDir $__BinDir/System.Private.CoreLib.dll >> $__CrossGenCoreLibLog 2>&1
 386         if [ $? -ne 0 ]; then
 387             echo "Failed to generate symbol file for System.Private.CoreLib. Refer to $__CrossGenCoreLibLog"
 388             exit 1
 389         fi
 390     fi
 391 }
 392
 393 build_CoreLib()
 394 {
 395     if [ $__isMSBuildOnNETCoreSupported == 0 ]; then
 396         echo "System.Private.CoreLib.dll build unsupported."
 397         return
 398     fi
 399
 400     if [ $__SkipMSCorLib == 1 ]; then
 401        echo "Skipping building System.Private.CoreLib."
 402        return
 403     fi
 404
 405     echo "Commencing build of managed components for $__BuildOS.$__BuildArch.$__BuildType"
 406
 407     # Invoke MSBuild
 408     __ExtraBuildArgs=""
 409     if [[ "$__IbcTuning" == "" ]]; then
 410         __ExtraBuildArgs="$__ExtraBuildArgs /p:OptimizationDataDir=\"$__PackagesDir/optimization.$__BuildOS-$__BuildArch.IBC.CoreCLR/$__IbcOptDataVersion/data\""
 411         __ExtraBuildArgs="$__ExtraBuildArgs /p:EnableProfileGuidedOptimization=true"
 412     fi
 413
 414     if [[ "$__BuildManagedTools" -eq "1" ]]; then
 415         __ExtraBuildArgs="$__ExtraBuildArgs /p:BuildManagedTools=true"
 416     fi
 417
 418     $__ProjectRoot/dotnet.sh restore /nologo /verbosity:minimal /clp:Summary \
 419                              /p:PortableBuild=true /maxcpucount /p:IncludeRestoreOnlyProjects=true /p:ArcadeBuild=true\
 420                              $__ProjectDir/src/build.proj \
 421                              /flp:Verbosity=normal\;LogFile=$__LogsDir/System.Private.CoreLib_$__BuildOS__$__BuildArch__$__BuildType.log \
 422                              /p:__IntermediatesDir=$__IntermediatesDir /p:__RootBinDir=$__RootBinDir \
 423                              $__CommonMSBuildArgs $__ExtraBuildArgs $__UnprocessedBuildArgs
 424
 425     if [ $? -ne 0 ]; then
 426         echo "Failed to restore managed components."
 427         exit 1
 428     fi
 429
 430     $__ProjectRoot/dotnet.sh msbuild /nologo /verbosity:minimal /clp:Summary \
 431                              /p:PortableBuild=true /maxcpucount /p:ArcadeBuild=true\
 432                              $__ProjectDir/src/build.proj \
 433                              /flp:Verbosity=normal\;LogFile=$__LogsDir/System.Private.CoreLib_$__BuildOS__$__BuildArch__$__BuildType.log \
 434                              /p:__IntermediatesDir=$__IntermediatesDir /p:__RootBinDir=$__RootBinDir \
 435                              $__CommonMSBuildArgs $__ExtraBuildArgs $__UnprocessedBuildArgs
 436
 437     if [ $? -ne 0 ]; then
 438         echo "Failed to build managed components."
 439         exit 1
 440     fi
 441
 442     local __CoreLibILDir=$__BinDir/IL
 443
 444     if [ $__SkipCrossgen == 1 ]; then
 445         echo "Skipping generating native image"
 446
 447         if [ $__CrossBuild == 1 ]; then
 448             # Crossgen not performed, so treat the IL version as the final version
 449             cp $__CoreLibILDir/System.Private.CoreLib.dll $__BinDir/System.Private.CoreLib.dll
 450         fi
 451
 452         return
 453     fi
 454
 455     # The cross build generates a crossgen with the target architecture.
 456     if [ $__CrossBuild == 0 ]; then
 457        if [ $__SkipCoreCLR == 1 ]; then
 458            return
 459        fi
 460
 461        # The architecture of host pc must be same architecture with target.
 462        if [[ ( "$__HostArch" == "$__BuildArch" ) ]]; then
 463            build_CoreLib_ni "$__BinDir/crossgen" $__CoreLibILDir
 464        elif [[ ( "$__HostArch" == "x64" ) && ( "$__BuildArch" == "x86" ) ]]; then
 465            build_CoreLib_ni "$__BinDir/crossgen" $__CoreLibILDir
 466        elif [[ ( "$__HostArch" == "arm64" ) && ( "$__BuildArch" == "arm" ) ]]; then
 467            build_CoreLib_ni "$__BinDir/crossgen" $__CoreLibILDir
 468        else
 469            exit 1
 470        fi
 471     else
 472        if [[ ( "$__CrossArch" == "x86" ) && ( "$__BuildArch" == "arm" ) ]]; then
 473            build_CoreLib_ni "$__CrossComponentBinDir/crossgen" $__CoreLibILDir
 474        elif [[ ( "$__CrossArch" == "x64" ) && ( "$__BuildArch" == "arm" ) ]]; then
 475            build_CoreLib_ni "$__CrossComponentBinDir/crossgen" $__CoreLibILDir
 476        elif [[ ( "$__HostArch" == "x64" ) && ( "$__BuildArch" == "arm64" ) ]]; then
 477            build_CoreLib_ni "$__CrossComponentBinDir/crossgen" $__CoreLibILDir
 478        else
 479            # Crossgen not performed, so treat the IL version as the final version
 480            cp $__CoreLibILDir/System.Private.CoreLib.dll $__BinDir/System.Private.CoreLib.dll
 481        fi
 482     fi
 483 }
 484
 485 generate_NugetPackages()
 486 {
 487     # We can only generate nuget package if we also support building mscorlib as part of this build.
 488     if [ $__isMSBuildOnNETCoreSupported == 0 ]; then
 489         echo "Nuget package generation unsupported."
 490         return
 491     fi
 492
 493     # Since we can build mscorlib for this OS, did we build the native components as well?
 494     if [[ $__SkipCoreCLR == 1 && $__CrossgenOnly == 0 ]]; then
 495         echo "Unable to generate nuget packages since native components were not built."
 496         return
 497     fi
 498
 499     echo "Generating nuget packages for "$__BuildOS
 500     echo "DistroRid is "$__DistroRid
 501     echo "ROOTFS_DIR is "$ROOTFS_DIR
 502     # Build the packages
 503     # Package build uses the Arcade system and scripts, relying on it to restore required toolsets as part of build
 504     $__ProjectRoot/eng/common/build.sh -r -b -projects $__SourceDir/.nuget/packages.builds \
 505                                        -verbosity minimal -bl:$__LogsDir/Nuget_$__BuildOS__$__BuildArch__$__BuildType.binlog \
 506                                        /p:PortableBuild=true /p:ArcadeBuild=true \
 507                                        /p:__IntermediatesDir=$__IntermediatesDir /p:__RootBinDir=$__RootBinDir /p:__DoCrossArchBuild=$__CrossBuild \
 508                                        $__CommonMSBuildArgs $__UnprocessedBuildArgs
 509
 510     if [ $? -ne 0 ]; then
 511         echo "Failed to generate Nuget packages."
 512         exit 1
 513     fi
 514 }
 515
 516 echo "Commencing CoreCLR Repo build"
 517
 518 # Argument types supported by this script:
 519 #
 520 # Build architecture - valid values are: x64, ARM.
 521 # Build Type         - valid values are: Debug, Checked, Release
 522 #
 523 # Set the default arguments for build
 524
 525 # Obtain the location of the bash script to figure out where the root of the repo is.
 526 __ProjectRoot="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
 527
 528 # Use uname to determine what the CPU is.
 529 CPUName=$(uname -p)
 530 # Some Linux platforms report unknown for platform, but the arch for machine.
 531 if [ "$CPUName" == "unknown" ]; then
 532     CPUName=$(uname -m)
 533 fi
 534
 535 case $CPUName in
 536     i686)
 537         echo "Unsupported CPU $CPUName detected, build might not succeed!"
 538         __BuildArch=x86
 539         __HostArch=x86
 540         ;;
 541
 542     x86_64)
 543         __BuildArch=x64
 544         __HostArch=x64
 545         ;;
 546
 547     armv7l)
 548         echo "Unsupported CPU $CPUName detected, build might not succeed!"
 549         __BuildArch=arm
 550         __HostArch=arm
 551         ;;
 552
 553     aarch64)
 554         __BuildArch=arm64
 555         __HostArch=arm64
 556         ;;
 557
 558     amd64)
 559         __BuildArch=x64
 560         __HostArch=x64
 561         ;;
 562     *)
 563         echo "Unknown CPU $CPUName detected, configuring as if for x64"
 564         __BuildArch=x64
 565         __HostArch=x64
 566         ;;
 567 esac
 568
 569 # Use uname to determine what the OS is.
 570 OSName=$(uname -s)
 571 case $OSName in
 572     Linux)
 573         __BuildOS=Linux
 574         __HostOS=Linux
 575         ;;
 576
 577     Darwin)
 578         __BuildOS=OSX
 579         __HostOS=OSX
 580         ;;
 581
 582     FreeBSD)
 583         __BuildOS=FreeBSD
 584         __HostOS=FreeBSD
 585         ;;
 586
 587     OpenBSD)
 588         __BuildOS=OpenBSD
 589         __HostOS=OpenBSD
 590         ;;
 591
 592     NetBSD)
 593         __BuildOS=NetBSD
 594         __HostOS=NetBSD
 595         ;;
 596
 597     SunOS)
 598         __BuildOS=SunOS
 599         __HostOS=SunOS
 600         ;;
 601
 602     *)
 603         echo "Unsupported OS $OSName detected, configuring as if for Linux"
 604         __BuildOS=Linux
 605         __HostOS=Linux
 606         ;;
 607 esac
 608
 609 __BuildType=Debug
 610 __CodeCoverage=
 611 __IgnoreWarnings=0
 612
 613 # Set the various build properties here so that CMake and MSBuild can pick them up
 614 __ProjectDir="$__ProjectRoot"
 615 __SourceDir="$__ProjectDir/src"
 616 __PackagesDir="${DotNetRestorePackagesPath:-${__ProjectDir}/.packages}"
 617 __RootBinDir="$__ProjectDir/bin"
 618 __UnprocessedBuildArgs=
 619 __CommonMSBuildArgs=
 620 __MSBCleanBuildArgs=
 621 __UseNinja=0
 622 __VerboseBuild=0
 623 __PgoInstrument=0
 624 __PgoOptimize=1
 625 __IbcTuning=""
 626 __ConfigureOnly=0
 627 __SkipConfigure=0
 628 __SkipManaged=0
 629 __SkipRestore=""
 630 __SkipNuget=0
 631 __SkipCoreCLR=0
 632 __SkipCrossArchNative=0
 633 __SkipMSCorLib=0
 634 __SkipRestoreOptData=0
 635 __SkipCrossgen=0
 636 __CrossgenOnly=0
 637 __PartialNgen=0
 638 __SkipTests=0
 639 __CrossBuild=0
 640 __ClangMajorVersion=0
 641 __ClangMinorVersion=0
 642 __GccBuild=0
 643 __GccMajorVersion=0
 644 __GccMinorVersion=0
 645 __NuGetPath="$__PackagesDir/NuGet.exe"
 646 __DistroRid=""
 647 __cmakeargs=""
 648 __SkipGenerateVersion=0
 649 __PortableBuild=1
 650 __msbuildonunsupportedplatform=0
 651 __PgoOptDataVersion=""
 652 __IbcOptDataVersion=""
 653 __BuildManagedTools=1
 654 __SkipRestoreArg="/p:RestoreDuringBuild=true"
 655 __SignTypeArg=""
 656 __OfficialBuildIdArg=""
 657 __StaticAnalyzer=0
 658
 659 # Get the number of processors available to the scheduler
 660 # Other techniques such as `nproc` only get the number of
 661 # processors available to a single process.
 662 if [ `uname` = "FreeBSD" ]; then
 663   __NumProc=`sysctl hw.ncpu | awk '{ print $2+1 }'`
 664 elif [ `uname` = "NetBSD" ]; then
 665   __NumProc=$(($(getconf NPROCESSORS_ONLN)+1))
 666 elif [ `uname` = "Darwin" ]; then
 667   __NumProc=$(($(getconf _NPROCESSORS_ONLN)+1))
 668 else
 669   __NumProc=$(nproc --all)
 670 fi
 671
 672 while :; do
 673     if [ $# -le 0 ]; then
 674         break
 675     fi
 676
 677     lowerI="$(echo $1 | awk '{print tolower($0)}')"
 678     case $lowerI in
 679         -\?|-h|--help)
 680             usage
 681             exit 1
 682             ;;
 683
 684         x86|-x86)
 685             __BuildArch=x86
 686             ;;
 687
 688         x64|-x64)
 689             __BuildArch=x64
 690             ;;
 691
 692         arm|-arm)
 693             __BuildArch=arm
 694             ;;
 695
 696         armel|-armel)
 697             __BuildArch=armel
 698             ;;
 699
 700         arm64|-arm64)
 701             __BuildArch=arm64
 702             ;;
 703
 704         debug|-debug)
 705             __BuildType=Debug
 706             ;;
 707
 708         checked|-checked)
 709             __BuildType=Checked
 710             ;;
 711
 712         release|-release)
 713             __BuildType=Release
 714             ;;
 715
 716         coverage|-coverage)
 717             __CodeCoverage=Coverage
 718             ;;
 719
 720         cross|-cross)
 721             __CrossBuild=1
 722             ;;
 723
 724         -portablebuild=false)
 725             __PortableBuild=0
 726             ;;
 727
 728         verbose|-verbose)
 729             __VerboseBuild=1
 730             ;;
 731
 732         stripsymbols|-stripsymbols)
 733             __cmakeargs="$__cmakeargs -DSTRIP_SYMBOLS=true"
 734             ;;
 735
 736         clang3.5|-clang3.5)
 737             __ClangMajorVersion=3
 738             __ClangMinorVersion=5
 739             ;;
 740
 741         clang3.6|-clang3.6)
 742             __ClangMajorVersion=3
 743             __ClangMinorVersion=6
 744             ;;
 745
 746         clang3.7|-clang3.7)
 747             __ClangMajorVersion=3
 748             __ClangMinorVersion=7
 749             ;;
 750
 751         clang3.8|-clang3.8)
 752             __ClangMajorVersion=3
 753             __ClangMinorVersion=8
 754             ;;
 755
 756         clang3.9|-clang3.9)
 757             __ClangMajorVersion=3
 758             __ClangMinorVersion=9
 759             ;;
 760
 761         clang4.0|-clang4.0)
 762             __ClangMajorVersion=4
 763             __ClangMinorVersion=0
 764             ;;
 765
 766         clang5.0|-clang5.0)
 767             __ClangMajorVersion=5
 768             __ClangMinorVersion=0
 769             ;;
 770
 771         clang6.0|-clang6.0)
 772             __ClangMajorVersion=6
 773             __ClangMinorVersion=0
 774             ;;
 775
 776         clang7|-clang7)
 777             __ClangMajorVersion=7
 778             __ClangMinorVersion=
 779             ;;
 780
 781         gcc5|-gcc5)
 782             __GccMajorVersion=5
 783             __GccMinorVersion=
 784             __GccBuild=1
 785             ;;
 786
 787         gcc6|-gcc6)
 788             __GccMajorVersion=6
 789             __GccMinorVersion=
 790             __GccBuild=1
 791             ;;
 792
 793         gcc7|-gcc7)
 794             __GccMajorVersion=7
 795             __GccMinorVersion=
 796             __GccBuild=1
 797             ;;
 798
 799         gcc8|-gcc8)
 800             __GccMajorVersion=8
 801             __GccMinorVersion=
 802             __GccBuild=1
 803             ;;
 804
 805         gcc|-gcc)
 806             __GccMajorVersion=
 807             __GccMinorVersion=
 808             __GccBuild=1
 809             ;;
 810
 811         ninja|-ninja)
 812             __UseNinja=1
 813             ;;
 814
 815         pgoinstrument|-pgoinstrument)
 816             __PgoInstrument=1
 817             ;;
 818
 819         nopgooptimize|-nopgooptimize)
 820             __PgoOptimize=0
 821             __SkipRestoreOptData=1
 822             ;;
 823
 824         ibcinstrument|-ibcinstrument)
 825             __IbcTuning="/Tuning"
 826             ;;
 827
 828         configureonly|-configureonly)
 829             __ConfigureOnly=1
 830             __SkipMSCorLib=1
 831             __SkipNuget=1
 832             ;;
 833
 834         skipconfigure|-skipconfigure)
 835             __SkipConfigure=1
 836             ;;
 837
 838         skipnative|-skipnative)
 839             # Use "skipnative" to use the same option name as build.cmd.
 840             __SkipCoreCLR=1
 841             ;;
 842
 843         skipcoreclr|-skipcoreclr)
 844             # Accept "skipcoreclr" for backwards-compatibility.
 845             __SkipCoreCLR=1
 846             ;;
 847
 848         skipcrossarchnative|-skipcrossarchnative)
 849             __SkipCrossArchNative=1
 850             ;;
 851
 852         skipmanaged|-skipmanaged)
 853             __SkipManaged=1
 854             ;;
 855
 856         skipmscorlib|-skipmscorlib)
 857             __SkipMSCorLib=1
 858             ;;
 859
 860         skipgenerateversion|-skipgenerateversion)
 861             __SkipGenerateVersion=1
 862             ;;
 863
 864         skiprestoreoptdata|-skiprestoreoptdata)
 865             __SkipRestoreOptData=1
 866             ;;
 867
 868         skipcrossgen|-skipcrossgen)
 869             __SkipCrossgen=1
 870             ;;
 871
 872         crossgenonly|-crossgenonly)
 873             __SkipMSCorLib=1
 874             __SkipCoreCLR=1
 875             __CrossgenOnly=1
 876             ;;
 877         partialngen|-partialngen)
 878             __PartialNgen=1
 879             ;;
 880
 881         skiptests|-skiptests)
 882             __SkipTests=1
 883             ;;
 884
 885         skipnuget|-skipnuget|skipbuildpackages|-skipbuildpackages)
 886             __SkipNuget=1
 887             ;;
 888
 889         ignorewarnings|-ignorewarnings)
 890             __IgnoreWarnings=1
 891             __cmakeargs="$__cmakeargs -DCLR_CMAKE_WARNINGS_ARE_ERRORS=OFF"
 892             ;;
 893
 894         cmakeargs|-cmakeargs)
 895             if [ -n "$2" ]; then
 896                 __cmakeargs="$__cmakeargs $2"
 897                 shift
 898             else
 899                 echo "ERROR: 'cmakeargs' requires a non-empty option argument"
 900                 exit 1
 901             fi
 902             ;;
 903
 904         bindir|-bindir)
 905             if [ -n "$2" ]; then
 906                 __RootBinDir="$2"
 907                 if [ ! -d $__RootBinDir ]; then
 908                     mkdir $__RootBinDir
 909                 fi
 910                 __RootBinParent=$(dirname $__RootBinDir)
 911                 __RootBinName=${__RootBinDir##*/}
 912                 __RootBinDir="$(cd $__RootBinParent &>/dev/null && printf %s/%s $PWD $__RootBinName)"
 913                 shift
 914             else
 915                 echo "ERROR: 'bindir' requires a non-empty option argument"
 916                 exit 1
 917             fi
 918             ;;
 919         msbuildonunsupportedplatform|-msbuildonunsupportedplatform)
 920             __msbuildonunsupportedplatform=1
 921             ;;
 922         numproc|-numproc)
 923             if [ -n "$2" ]; then
 924               __NumProc="$2"
 925               shift
 926             else
 927               echo "ERROR: 'numproc' requires a non-empty option argument"
 928               exit 1
 929             fi
 930             ;;
 931         osgroup|-osgroup)
 932             if [ -n "$2" ]; then
 933               __BuildOS="$2"
 934               shift
 935             else
 936               echo "ERROR: 'osgroup' requires a non-empty option argument"
 937               exit 1
 938             fi
 939             ;;
 940         rebuild|-rebuild)
 941             echo "ERROR: 'Rebuild' is not supported.  Please remove it."
 942             exit 1
 943             ;;
 944
 945         -skiprestore)
 946             __SkipRestoreArg="/p:RestoreDuringBuild=false"
 947             ;;
 948
 949         -disableoss)
 950             __SignTypeArg="/p:SignType=real"
 951             ;;
 952
 953         -officialbuildid=*)
 954             __Id=$(echo $1| cut -d'=' -f 2)
 955             __OfficialBuildIdArg="/p:OfficialBuildId=$__Id"
 956             ;;
 957
 958         -staticanalyzer)
 959             __StaticAnalyzer=1
 960             ;;
 961
 962         --)
 963             # Skip -Option=Value style argument passing
 964             ;;
 965
 966         *)
 967             __UnprocessedBuildArgs="$__UnprocessedBuildArgs $1"
 968             ;;
 969     esac
 970
 971     shift
 972 done
 973
 974 __CommonMSBuildArgs="/p:__BuildArch=$__BuildArch /p:__BuildType=$__BuildType /p:__BuildOS=$__BuildOS $__OfficialBuildIdArg $__SignTypeArg $__SkipRestoreArg"
 975
 976 # Configure environment if we are doing a verbose build
 977 if [ $__VerboseBuild == 1 ]; then
 978     export VERBOSE=1
 979     __CommonMSBuildArgs="$__CommonMSBuildArgs /v:detailed"
 980 fi
 981
 982 # Set default clang version
 983 if [[ $__ClangMajorVersion == 0 && $__ClangMinorVersion == 0 ]]; then
 984     if [[ "$__BuildArch" == "arm" || "$__BuildArch" == "armel" ]]; then
 985         __ClangMajorVersion=5
 986         __ClangMinorVersion=0
 987     else
 988         __ClangMajorVersion=3
 989         __ClangMinorVersion=9
 990     fi
 991 fi
 992
 993 # Set dependent variables
 994 __LogsDir="$__RootBinDir/Logs"
 995 __MsbuildDebugLogsDir="$__LogsDir/MsbuildDebugLogs"
 996
 997 # Set the remaining variables based upon the determined build configuration
 998 __BinDir="$__RootBinDir/Product/$__BuildOS.$__BuildArch.$__BuildType"
 999 __PackagesBinDir="$__BinDir/.nuget"
1000 export __IntermediatesDir="$__RootBinDir/obj/$__BuildOS.$__BuildArch.$__BuildType"
1001 __isMSBuildOnNETCoreSupported=0
1002 __CrossComponentBinDir="$__BinDir"
1003
1004 __CrossArch="$__HostArch"
1005 if [ $__CrossBuild == 1 ]; then
1006     __CrossComponentBinDir="$__CrossComponentBinDir/$__CrossArch"
1007 fi
1008 __CrossGenCoreLibLog="$__LogsDir/CrossgenCoreLib_$__BuildOS.$__BuildArch.$__BuildType.log"
1009
1010 # Configure environment if we are doing a cross compile.
1011 if [ $__CrossBuild == 1 ]; then
1012     export CROSSCOMPILE=1
1013     if ! [[ -n "$ROOTFS_DIR" ]]; then
1014         export ROOTFS_DIR="$__ProjectRoot/cross/rootfs/$__BuildArch"
1015     fi
1016 fi
1017
1018 # init the target distro name
1019 initTargetDistroRid
1020
1021 if [ $__PortableBuild == 0 ]; then
1022     __CommonMSBuildArgs="$__CommonMSBuildArgs /p:PortableBuild=false"
1023 fi
1024
1025 # Init if MSBuild for .NET Core is supported for this platform
1026 isMSBuildOnNETCoreSupported
1027
1028 # CI_SPECIFIC - On CI machines, $HOME may not be set. In such a case, create a subfolder and set the variable to set.
1029 # This is needed by CLI to function.
1030 if [ -z "$HOME" ]; then
1031     if [ ! -d "$__ProjectDir/temp_home" ]; then
1032         mkdir temp_home
1033     fi
1034     export HOME=$__ProjectDir/temp_home
1035     echo "HOME not defined; setting it to $HOME"
1036 fi
1037
1038 # Specify path to be set for CMAKE_INSTALL_PREFIX.
1039 # This is where all built CoreClr libraries will copied to.
1040 export __CMakeBinDir="$__BinDir"
1041
1042 # Make the directories necessary for build if they don't exist
1043 setup_dirs
1044
1045 # Set up the directory for MSBuild debug logs.
1046 export MSBUILDDEBUGPATH="${__MsbuildDebugLogsDir}"
1047
1048 # Check prereqs.
1049 check_prereqs
1050
1051 # Restore the package containing profile counts for profile-guided optimizations
1052 restore_optdata
1053
1054 # Generate event logging infrastructure sources
1055 generate_event_logging
1056
1057 # Build the coreclr (native) components.
1058 __ExtraCmakeArgs="-DCLR_CMAKE_TARGET_OS=$__BuildOS -DCLR_CMAKE_PACKAGES_DIR=$__PackagesDir -DCLR_CMAKE_PGO_INSTRUMENT=$__PgoInstrument -DCLR_CMAKE_OPTDATA_VERSION=$__PgoOptDataVersion -DCLR_CMAKE_PGO_OPTIMIZE=$__PgoOptimize"
1059
1060 # [TODO] Remove this when the `build-test.sh` script properly builds and deploys test assets.
1061 if [ $__SkipTests != 1 ]; then
1062     echo "Adding CMake flags to build native tests for $__BuildOS.$__BuildArch.$__BuildType"
1063     __ExtraCmakeArgs="$__ExtraCmakeArgs -DCLR_CMAKE_BUILD_TESTS=ON"
1064 fi
1065
1066 build_native $__SkipCoreCLR "$__BuildArch" "$__IntermediatesDir" "$__ExtraCmakeArgs" "CoreCLR component"
1067
1068 # Build cross-architecture components
1069 if [ $__SkipCrossArchNative != 1 ]; then
1070     if [[ $__CrossBuild == 1 ]]; then
1071         build_cross_architecture_components
1072     fi
1073 fi
1074
1075 # Build System.Private.CoreLib.
1076
1077 build_CoreLib
1078
1079 if [ $__CrossgenOnly == 1 ]; then
1080     build_CoreLib_ni "$__BinDir/crossgen"
1081 fi
1082
1083 # Generate nuget packages
1084 if [ $__SkipNuget != 1 ]; then
1085     generate_NugetPackages
1086 fi
1087
1088
1089 # Build complete
1090
1091 echo "Repo successfully built."
1092 echo "Product binaries are available at $__BinDir"
1093 exit 0