build.sh

   1 #!/usr/bin/env bash
   2
   3 # Work around Jenkins CI + msbuild problem: Jenkins sometimes creates very large environment
   4 # variables, and msbuild can't handle environment blocks with such large variables. So clear
   5 # out the variables that might be too large.
   6 export ghprbCommentBody=
   7
   8 # resolve python-version to use
   9 if [ "$PYTHON" == "" ] ; then
  10     if ! PYTHON=$(command -v python3 || command -v python2 || command -v python)
  11     then
  12        echo "Unable to locate build-dependency python!" 1>&2
  13        exit 1
  14     fi
  15 fi
  16 # validate python-dependency
  17 # useful in case of explicitly set option.
  18 if ! command -v $PYTHON > /dev/null
  19 then
  20    echo "Unable to locate build-dependency python ($PYTHON)!" 1>&2
  21    exit 1
  22 fi
  23
  24 export PYTHON
  25
  26 usage()
  27 {
  28     echo "Usage: $0 [BuildArch] [BuildType] [-verbose] [-coverage] [-cross] [-clangx.y] [-ninja] [-configureonly] [-skipconfigure] [-skipnative] [-skipmanaged] [-skipmscorlib] [-skiptests] [-stripsymbols] [-ignorewarnings] [-cmakeargs] [-bindir]"
  29     echo "BuildArch can be: -x64, -x86, -arm, -armel, -arm64"
  30     echo "BuildType can be: -debug, -checked, -release"
  31     echo "-coverage - optional argument to enable code coverage build (currently supported only for Linux and OSX)."
  32     echo "-ninja - target ninja instead of GNU make"
  33     echo "-clangx.y - optional argument to build using clang version x.y."
  34     echo "-cross - optional argument to signify cross compilation,"
  35     echo "       - will use ROOTFS_DIR environment variable if set."
  36     echo "-nopgooptimize - do not use profile guided optimizations."
  37     echo "-pgoinstrument - generate instrumented code for profile guided optimization enabled binaries."
  38     echo "-ibcinstrument - generate IBC-tuning-enabled native images when invoking crossgen."
  39     echo "-configureonly - do not perform any builds; just configure the build."
  40     echo "-skipconfigure - skip build configuration."
  41     echo "-skipnative - do not build native components."
  42     echo "-skipmanaged - do not build managed components."
  43     echo "-skipmscorlib - do not build mscorlib.dll."
  44     echo "-skiptests - skip the tests in the 'tests' subdirectory."
  45     echo "-skipnuget - skip building nuget packages."
  46     echo "-skiprestoreoptdata - skip restoring optimization data used by profile-based optimizations."
  47     echo "-skipcrossgen - skip native image generation"
  48     echo "-crossgenonly - only run native image generation"
  49     echo "-partialngen - build CoreLib as PartialNGen"
  50     echo "-verbose - optional argument to enable verbose build output."
  51     echo "-skiprestore: skip restoring packages ^(default: packages are restored during build^)."
  52     echo "-disableoss: Disable Open Source Signing for System.Private.CoreLib."
  53     echo "-officialbuildid=^<ID^>: specify the official build ID to be used by this build."
  54     echo "-stripSymbols - Optional argument to strip native symbols during the build."
  55     echo "-skipgenerateversion - disable version generation even if MSBuild is supported."
  56     echo "-ignorewarnings - do not treat warnings as errors"
  57     echo "-cmakeargs - user-settable additional arguments passed to CMake."
  58     echo "-bindir - output directory (defaults to $__ProjectRoot/bin)"
  59     echo "-msbuildonunsupportedplatform - build managed binaries even if distro is not officially supported."
  60     echo "-numproc - set the number of build processes."
  61     exit 1
  62 }
  63
  64 initHostDistroRid()
  65 {
  66     __HostDistroRid=""
  67     if [ "$__HostOS" == "Linux" ]; then
  68         if [ -e /etc/os-release ]; then
  69             source /etc/os-release
  70             if [[ $ID == "rhel" ]]; then
  71                 # remove the last version digit
  72                 VERSION_ID=${VERSION_ID%.*}
  73             fi
  74             __HostDistroRid="$ID.$VERSION_ID-$__HostArch"
  75             if [[ $ID == "alpine" ]]; then
  76                 __HostDistroRid="linux-musl-$__HostArch"
  77             fi
  78         elif [ -e /etc/redhat-release ]; then
  79             local redhatRelease=$(</etc/redhat-release)
  80             if [[ $redhatRelease == "CentOS release 6."* || $redhatRelease == "Red Hat Enterprise Linux Server release 6."* ]]; then
  81                __HostDistroRid="rhel.6-$__HostArch"
  82             fi
  83             if [[ $redhatRelease == "CentOS Linux release 7."* ]]; then
  84                 __HostDistroRid="rhel.7-$__Arch"
  85             fi
  86         fi
  87     fi
  88     if [ "$__HostOS" == "FreeBSD" ]; then
  89         __freebsd_version=`sysctl -n kern.osrelease | cut -f1 -d'.'`
  90         __HostDistroRid="freebsd.$__freebsd_version-$__HostArch"
  91     fi
  92
  93     if [ "$__HostDistroRid" == "" ]; then
  94         echo "WARNING: Can not determine runtime id for current distro."
  95     fi
  96 }
  97
  98 initTargetDistroRid()
  99 {
 100     if [ $__CrossBuild == 1 ]; then
 101         if [ "$__BuildOS" == "Linux" ]; then
 102             if [ ! -e $ROOTFS_DIR/etc/os-release ]; then
 103                 if [ -e $ROOTFS_DIR/android_platform ]; then
 104                     source $ROOTFS_DIR/android_platform
 105                     export __DistroRid="$RID"
 106                 else
 107                     echo "WARNING: Can not determine runtime id for current distro."
 108                     export __DistroRid=""
 109                 fi
 110             else
 111                 source $ROOTFS_DIR/etc/os-release
 112                 export __DistroRid="$ID.$VERSION_ID-$__BuildArch"
 113             fi
 114         fi
 115     else
 116         export __DistroRid="$__HostDistroRid"
 117     fi
 118
 119     if [ "$__BuildOS" == "OSX" ]; then
 120         __PortableBuild=1
 121     fi
 122
 123     # Portable builds target the base RID
 124     if [ $__PortableBuild == 1 ]; then
 125         if [ "$__BuildOS" == "Linux" ]; then
 126             export __DistroRid="linux-$__BuildArch"
 127         elif [ "$__BuildOS" == "OSX" ]; then
 128             export __DistroRid="osx-$__BuildArch"
 129         elif [ "$__BuildOS" == "FreeBSD" ]; then
 130             export __DistroRid="freebsd-$__BuildArch"
 131         fi
 132     fi
 133 }
 134
 135 setup_dirs()
 136 {
 137     echo Setting up directories for build
 138
 139     mkdir -p "$__RootBinDir"
 140     mkdir -p "$__BinDir"
 141     mkdir -p "$__LogsDir"
 142     mkdir -p "$__IntermediatesDir"
 143
 144     if [ $__CrossBuild == 1 ]; then
 145         mkdir -p "$__CrossComponentBinDir"
 146     fi
 147 }
 148
 149 # Check the system to ensure the right prereqs are in place
 150
 151 check_prereqs()
 152 {
 153     echo "Checking prerequisites..."
 154
 155     # Check presence of CMake on the path
 156     hash cmake 2>/dev/null || { echo >&2 "Please install cmake before running this script"; exit 1; }
 157
 158
 159     # Minimum required version of clang is version 4.0 for arm/armel cross build
 160     if [[ $__CrossBuild == 1 && ("$__BuildArch" == "arm" || "$__BuildArch" == "armel") ]]; then
 161         if ! [[ "$__ClangMajorVersion" -ge "4" ]]; then
 162             echo "Please install clang4.0 or latest for arm/armel cross build"; exit 1;
 163         fi
 164     fi
 165
 166     # Check for clang
 167     __ClangCombinedDottedVersion=$__ClangMajorVersion;
 168     if [[ "$__ClangMinorVersion" != "" ]]; then
 169         __ClangCombinedDottedVersion=$__ClangCombinedDottedVersion.$__ClangMinorVersion
 170     fi
 171     hash clang-$__ClangCombinedDottedVersion 2>/dev/null ||  hash clang$__ClangMajorVersion$__ClangMinorVersion 2>/dev/null || hash clang 2>/dev/null || { echo >&2 "Please install clang-$__ClangMajorVersion.$__ClangMinorVersion before running this script"; exit 1; }
 172
 173 }
 174
 175 restore_optdata()
 176 {
 177     # we only need optdata on a Release build
 178     if [[ "$__BuildType" != "Release" ]]; then __SkipRestoreOptData=1; fi
 179
 180     if [[ ( $__SkipRestoreOptData == 0 ) && ( $__isMSBuildOnNETCoreSupported == 1 ) ]]; then
 181         echo "Restoring the OptimizationData package"
 182         "$__ProjectRoot/dotnet.sh" msbuild /nologo /verbosity:minimal /clp:Summary \
 183                                    /p:RestoreDefaultOptimizationDataPackage=false /p:PortableBuild=true \
 184                                    /p:UsePartialNGENOptimization=false /maxcpucount \
 185                                    /t:RestoreOptData ./build.proj \
 186                                    $__CommonMSBuildArgs $__UnprocessedBuildArgs
 187         if [ $? != 0 ]; then
 188             echo "Failed to restore the optimization data package."
 189             exit 1
 190         fi
 191     fi
 192
 193     if [ $__isMSBuildOnNETCoreSupported == 1 ]; then
 194         # Parse the optdata package versions out of msbuild so that we can pass them on to CMake
 195         local DotNetCli="$__ProjectRoot/Tools/dotnetcli/dotnet"
 196         if [ ! -f $DotNetCli ]; then
 197             source "$__ProjectRoot/init-tools.sh"
 198             if [ $? != 0 ]; then
 199                 echo "Failed to restore buildtools."
 200                 exit 1
 201             fi
 202         fi
 203         local OptDataProjectFilePath="$__ProjectRoot/src/.nuget/optdata/optdata.csproj"
 204         __PgoOptDataVersion=$(DOTNET_SKIP_FIRST_TIME_EXPERIENCE=1 $DotNetCli msbuild $OptDataProjectFilePath /t:DumpPgoDataPackageVersion /nologo | sed 's/^\s*//')
 205         __IbcOptDataVersion=$(DOTNET_SKIP_FIRST_TIME_EXPERIENCE=1 $DotNetCli msbuild $OptDataProjectFilePath /t:DumpIbcDataPackageVersion /nologo | sed 's/^[[:blank:]]*//')
 206     fi
 207 }
 208
 209 generate_event_logging_sources()
 210 {
 211     __OutputDir=$1
 212     __ConsumingBuildSystem=$2
 213
 214     __OutputIncDir="$__OutputDir/src/inc"
 215     __OutputEventingDir="$__OutputDir/eventing"
 216     __OutputEventProviderDir="$__OutputEventingDir/eventprovider"
 217
 218     echo "Laying out dynamically generated files consumed by $__ConsumingBuildSystem"
 219     echo "Laying out dynamically generated Event test files, etmdummy stub functions, and external linkages"
 220
 221     __PythonWarningFlags="-Wall"
 222     if [[ $__IgnoreWarnings == 0 ]]; then
 223         __PythonWarningFlags="$__PythonWarningFlags -Werror"
 224     fi
 225
 226     $PYTHON -B $__PythonWarningFlags "$__ProjectRoot/src/scripts/genEventing.py" --inc $__OutputIncDir --dummy $__OutputIncDir/etmdummy.h --man "$__ProjectRoot/src/vm/ClrEtwAll.man" --testdir "$__OutputEventProviderDir/tests"
 227     if [[ $? != 0 ]]; then
 228         exit 1
 229     fi
 230
 231     echo "Laying out dynamically generated EventPipe Implementation"
 232     $PYTHON -B $__PythonWarningFlags "$__ProjectRoot/src/scripts/genEventPipe.py" --man "$__ProjectRoot/src/vm/ClrEtwAll.man" --intermediate "$__OutputEventingDir/eventpipe"
 233
 234     echo "Laying out dynamically generated EventSource classes"
 235     $PYTHON -B $__PythonWarningFlags "$__ProjectRoot/src/scripts/genRuntimeEventSources.py" --man "$__ProjectRoot/src/vm/ClrEtwAll.man" --intermediate "$__OutputEventingDir"
 236
 237     # determine the logging system
 238     case $__BuildOS in
 239         Linux|FreeBSD)
 240             echo "Laying out dynamically generated Event Logging Implementation of Lttng"
 241             $PYTHON -B $__PythonWarningFlags "$__ProjectRoot/src/scripts/genLttngProvider.py" --man "$__ProjectRoot/src/vm/ClrEtwAll.man" --intermediate "$__OutputEventProviderDir"
 242             if [[ $? != 0 ]]; then
 243                 exit 1
 244             fi
 245             ;;
 246         *)
 247             echo "Laying out dummy event logging provider"
 248             $PYTHON -B $__PythonWarningFlags "$__ProjectRoot/src/scripts/genDummyProvider.py" --man "$__ProjectRoot/src/vm/ClrEtwAll.man" --intermediate "$__OutputEventProviderDir"
 249             if [[ $? != 0 ]]; then
 250                 exit 1
 251             fi
 252             ;;
 253     esac
 254 }
 255
 256 generate_event_logging()
 257 {
 258     # Event Logging Infrastructure
 259     if [[ $__SkipCoreCLR == 0 || $__SkipMSCorLib == 0 || $__ConfigureOnly == 1 ]]; then
 260         generate_event_logging_sources "$__IntermediatesDir" "the native build system"
 261     fi
 262 }
 263
 264 build_native()
 265 {
 266     skipCondition=$1
 267     platformArch="$2"
 268     intermediatesForBuild="$3"
 269     extraCmakeArguments="$4"
 270     message="$5"
 271
 272     if [ $skipCondition == 1 ]; then
 273         echo "Skipping $message build."
 274         return
 275     fi
 276
 277     # All set to commence the build
 278     echo "Commencing build of $message for $__BuildOS.$__BuildArch.$__BuildType in $intermediatesForBuild"
 279
 280     generator=""
 281     buildFile="Makefile"
 282     buildTool="make"
 283     if [ $__UseNinja == 1 ]; then
 284         generator="ninja"
 285         buildFile="build.ninja"
 286         if ! buildTool=$(command -v ninja || command -v ninja-build); then
 287            echo "Unable to locate ninja!" 1>&2
 288            exit 1
 289         fi
 290     fi
 291
 292     if [ $__SkipConfigure == 0 ]; then
 293         # if msbuild is not supported, then set __SkipGenerateVersion to 1
 294         if [ $__isMSBuildOnNETCoreSupported == 0 ]; then __SkipGenerateVersion=1; fi
 295         # Drop version.cpp file
 296         __versionSourceFile="$intermediatesForBuild/version.cpp"
 297         if [ $__SkipGenerateVersion == 0 ]; then
 298             pwd
 299             "$__ProjectRoot/dotnet.sh" msbuild /nologo /verbosity:minimal /clp:Summary \
 300                                        /l:BinClashLogger,Tools/Microsoft.DotNet.Build.Tasks.dll\;LogFile=binclash.log \
 301                                        /p:RestoreDefaultOptimizationDataPackage=false /p:PortableBuild=true \
 302                                        /p:UsePartialNGENOptimization=false /maxcpucount \
 303                                        "$__ProjectDir/build.proj" /p:GenerateVersionSourceFile=true /t:GenerateVersionSourceFile /p:NativeVersionSourceFile=$__versionSourceFile \
 304                                        $__CommonMSBuildArgs $__UnprocessedBuildArgs
 305         else
 306             # Generate the dummy version.cpp, but only if it didn't exist to make sure we don't trigger unnecessary rebuild
 307             __versionSourceLine="static char sccsid[] __attribute__((used)) = \"@(#)No version information produced\";"
 308             if [ -e $__versionSourceFile ]; then
 309                 read existingVersionSourceLine < $__versionSourceFile
 310             fi
 311             if [ "$__versionSourceLine" != "$existingVersionSourceLine" ]; then
 312                 echo $__versionSourceLine > $__versionSourceFile
 313             fi
 314         fi
 315
 316
 317         pushd "$intermediatesForBuild"
 318         # Regenerate the CMake solution
 319         echo "Invoking \"$__ProjectRoot/src/pal/tools/gen-buildsys-clang.sh\" \"$__ProjectRoot\" $__ClangMajorVersion \"$__ClangMinorVersion\" $platformArch $__BuildType $__CodeCoverage $generator $extraCmakeArguments $__cmakeargs"
 320         "$__ProjectRoot/src/pal/tools/gen-buildsys-clang.sh" "$__ProjectRoot" $__ClangMajorVersion "$__ClangMinorVersion" $platformArch $__BuildType $__CodeCoverage $generator "$extraCmakeArguments" "$__cmakeargs"
 321         popd
 322     fi
 323
 324     if [ ! -f "$intermediatesForBuild/$buildFile" ]; then
 325         echo "Failed to generate $message build project!"
 326         exit 1
 327     fi
 328
 329     # Build
 330     if [ $__ConfigureOnly == 1 ]; then
 331         echo "Finish configuration & skipping $message build."
 332         return
 333     fi
 334
 335     # Check that the makefiles were created.
 336     pushd "$intermediatesForBuild"
 337
 338     echo "Executing $buildTool install -j $__NumProc"
 339
 340     $buildTool install -j $__NumProc
 341     if [ $? != 0 ]; then
 342         echo "Failed to build $message."
 343         exit 1
 344     fi
 345
 346     popd
 347 }
 348
 349 build_cross_architecture_components()
 350 {
 351     local intermediatesForBuild="$__IntermediatesDir/Host$__CrossArch/crossgen"
 352     local crossArchBinDir="$__BinDir/$__CrossArch"
 353
 354     mkdir -p "$intermediatesForBuild"
 355     mkdir -p "$crossArchBinDir"
 356
 357     generate_event_logging_sources "$intermediatesForBuild" "the crossarch build system"
 358
 359     __SkipCrossArchBuild=1
 360     # check supported cross-architecture components host(__HostArch)/target(__BuildArch) pair
 361     if [[ ("$__BuildArch" == "arm" || "$__BuildArch" == "armel") && ("$__CrossArch" == "x86" || "$__CrossArch" == "x64") ]]; then
 362         __SkipCrossArchBuild=0
 363     elif [[ "$__BuildArch" == "arm64" && "$__CrossArch" == "x64" ]]; then
 364         __SkipCrossArchBuild=0
 365     else
 366         # not supported
 367         return
 368     fi
 369
 370     export __CMakeBinDir="$crossArchBinDir"
 371     export CROSSCOMPILE=0
 372
 373     __ExtraCmakeArgs="-DCLR_CMAKE_TARGET_ARCH=$__BuildArch -DCLR_CMAKE_TARGET_OS=$__BuildOS -DCLR_CMAKE_PACKAGES_DIR=$__PackagesDir -DCLR_CMAKE_PGO_INSTRUMENT=$__PgoInstrument -DCLR_CMAKE_OPTDATA_VERSION=$__PgoOptDataVersion -DCLR_CMAKE_PGO_OPTIMIZE=$__PgoOptimize -DCLR_CROSS_COMPONENTS_BUILD=1"
 374     build_native $__SkipCrossArchBuild "$__CrossArch" "$intermediatesForBuild" "$__ExtraCmakeArgs" "cross-architecture components"
 375
 376     export CROSSCOMPILE=1
 377 }
 378
 379 isMSBuildOnNETCoreSupported()
 380 {
 381     __isMSBuildOnNETCoreSupported=$__msbuildonunsupportedplatform
 382
 383     if [ $__isMSBuildOnNETCoreSupported == 1 ]; then
 384         return
 385     fi
 386
 387     if [ $__SkipManaged == 1 ]; then
 388         __isMSBuildOnNETCoreSupported=0
 389         return
 390     fi
 391
 392     if [ "$__HostArch" == "x64" ]; then
 393         if [ "$__HostOS" == "Linux" ]; then
 394             __isMSBuildOnNETCoreSupported=1
 395             # note: the RIDs below can use globbing patterns
 396             UNSUPPORTED_RIDS=("debian.9-x64" "ubuntu.17.04-x64")
 397             for UNSUPPORTED_RID in "${UNSUPPORTED_RIDS[@]}"
 398             do
 399                 if [[ $__HostDistroRid == $UNSUPPORTED_RID ]]; then
 400                     __isMSBuildOnNETCoreSupported=0
 401                     break
 402                 fi
 403             done
 404         elif [ "$__HostOS" == "OSX" ]; then
 405             __isMSBuildOnNETCoreSupported=1
 406         elif [ "$__HostOS" == "FreeBSD" ]; then
 407             __isMSBuildOnNETCoreSupported=1
 408         fi
 409     fi
 410 }
 411
 412
 413 build_CoreLib_ni()
 414 {
 415     local __CrossGenExec=$1
 416
 417     if [ $__PartialNgen == 1 ]; then
 418         export COMPlus_PartialNGen=1
 419     fi
 420
 421     if [ -e $__CrossGenCoreLibLog ]; then
 422         rm $__CrossGenCoreLibLog
 423     fi
 424     echo "Generating native image of System.Private.CoreLib.dll for $__BuildOS.$__BuildArch.$__BuildType. Logging to \"$__CrossGenCoreLibLog\"."
 425     echo "$__CrossGenExec /Platform_Assemblies_Paths $__BinDir/IL $__IbcTuning /out $__BinDir/System.Private.CoreLib.dll $__BinDir/IL/System.Private.CoreLib.dll"
 426     $__CrossGenExec /Platform_Assemblies_Paths $__BinDir/IL $__IbcTuning /out $__BinDir/System.Private.CoreLib.dll $__BinDir/IL/System.Private.CoreLib.dll >> $__CrossGenCoreLibLog 2>&1
 427     if [ $? -ne 0 ]; then
 428         echo "Failed to generate native image for System.Private.CoreLib. Refer to $__CrossGenCoreLibLog"
 429         exit 1
 430     fi
 431
 432     if [ "$__BuildOS" == "Linux" ]; then
 433         echo "Generating symbol file for System.Private.CoreLib.dll"
 434         echo "$__CrossGenExec /Platform_Assemblies_Paths $__BinDir /CreatePerfMap $__BinDir $__BinDir/System.Private.CoreLib.dll"
 435         $__CrossGenExec /Platform_Assemblies_Paths $__BinDir /CreatePerfMap $__BinDir $__BinDir/System.Private.CoreLib.dll >> $__CrossGenCoreLibLog 2>&1
 436         if [ $? -ne 0 ]; then
 437             echo "Failed to generate symbol file for System.Private.CoreLib. Refer to $__CrossGenCoreLibLog"
 438             exit 1
 439         fi
 440     fi
 441 }
 442
 443 build_CoreLib()
 444 {
 445     if [ $__isMSBuildOnNETCoreSupported == 0 ]; then
 446         echo "System.Private.CoreLib.dll build unsupported."
 447         return
 448     fi
 449
 450     if [ $__SkipMSCorLib == 1 ]; then
 451        echo "Skipping building System.Private.CoreLib."
 452        return
 453     fi
 454
 455     echo "Commencing build of managed components for $__BuildOS.$__BuildArch.$__BuildType"
 456
 457     # Invoke MSBuild
 458     __ExtraBuildArgs=""
 459     if [[ "$__IbcTuning" == "" ]]; then
 460         __ExtraBuildArgs="$__ExtraBuildArgs /p:OptimizationDataDir=\"$__PackagesDir/optimization.$__BuildOS-$__BuildArch.IBC.CoreCLR/$__IbcOptDataVersion/data\""
 461         __ExtraBuildArgs="$__ExtraBuildArgs /p:EnableProfileGuidedOptimization=true"
 462     fi
 463
 464     if [[ "$__BuildManagedTools" -eq "1" ]]; then
 465         __ExtraBuildArgs="$__ExtraBuildArgs /p:BuildManagedTools=true"
 466     fi
 467
 468     $__ProjectRoot/dotnet.sh msbuild /nologo /verbosity:minimal /clp:Summary \
 469                              /l:BinClashLogger,Tools/Microsoft.DotNet.Build.Tasks.dll\;LogFile=binclash.log \
 470                              /p:RestoreDefaultOptimizationDataPackage=false /p:PortableBuild=true \
 471                              /p:UsePartialNGENOptimization=false /maxcpucount \
 472                              $__ProjectDir/build.proj \
 473                              /flp:Verbosity=normal\;LogFile=$__LogsDir/System.Private.CoreLib_$__BuildOS__$__BuildArch__$__BuildType.log \
 474                              /p:__IntermediatesDir=$__IntermediatesDir /p:__RootBinDir=$__RootBinDir /p:BuildNugetPackage=false /p:UseSharedCompilation=false \
 475                              $__CommonMSBuildArgs $__ExtraBuildArgs $__UnprocessedBuildArgs
 476
 477     if [ $? -ne 0 ]; then
 478         echo "Failed to build managed components."
 479         exit 1
 480     fi
 481
 482     if [ $__SkipCrossgen == 1 ]; then
 483         echo "Skipping generating native image"
 484         return
 485     fi
 486
 487     # The cross build generates a crossgen with the target architecture.
 488     if [ $__CrossBuild == 0 ]; then
 489        if [ $__SkipCoreCLR == 1 ]; then
 490            return
 491        fi
 492
 493        # The architecture of host pc must be same architecture with target.
 494        if [[ ( "$__HostArch" == "$__BuildArch" ) ]]; then
 495            build_CoreLib_ni "$__BinDir/crossgen"
 496        elif [[ ( "$__HostArch" == "x64" ) && ( "$__BuildArch" == "x86" ) ]]; then
 497            build_CoreLib_ni "$__BinDir/crossgen"
 498        elif [[ ( "$__HostArch" == "arm64" ) && ( "$__BuildArch" == "arm" ) ]]; then
 499            build_CoreLib_ni "$__BinDir/crossgen"
 500        else
 501            exit 1
 502        fi
 503     else
 504        if [[ ( "$__CrossArch" == "x86" ) && ( "$__BuildArch" == "arm" ) ]]; then
 505            build_CoreLib_ni "$__CrossComponentBinDir/crossgen"
 506        elif [[ ( "$__CrossArch" == "x64" ) && ( "$__BuildArch" == "arm" ) ]]; then
 507            build_CoreLib_ni "$__CrossComponentBinDir/crossgen"
 508        elif [[ ( "$__HostArch" == "x64" ) && ( "$__BuildArch" == "arm64" ) ]]; then
 509            build_CoreLib_ni "$__CrossComponentBinDir/crossgen"
 510        fi
 511     fi
 512 }
 513
 514 generate_NugetPackages()
 515 {
 516     # We can only generate nuget package if we also support building mscorlib as part of this build.
 517     if [ $__isMSBuildOnNETCoreSupported == 0 ]; then
 518         echo "Nuget package generation unsupported."
 519         return
 520     fi
 521
 522     # Since we can build mscorlib for this OS, did we build the native components as well?
 523     if [[ $__SkipCoreCLR == 1 && $__CrossgenOnly == 0 ]]; then
 524         echo "Unable to generate nuget packages since native components were not built."
 525         return
 526     fi
 527
 528     echo "Generating nuget packages for "$__BuildOS
 529     echo "DistroRid is "$__DistroRid
 530     echo "ROOTFS_DIR is "$ROOTFS_DIR
 531     # Build the packages
 532     $__ProjectRoot/dotnet.sh msbuild /nologo /verbosity:minimal /clp:Summary \
 533                              /l:BinClashLogger,Tools/Microsoft.DotNet.Build.Tasks.dll\;LogFile=binclash.log \
 534                              /p:RestoreDefaultOptimizationDataPackage=false /p:PortableBuild=true \
 535                              /p:UsePartialNGENOptimization=false /maxcpucount \
 536                              $__SourceDir/.nuget/packages.builds \
 537                              /flp:Verbosity=normal\;LogFile=$__LogsDir/Nuget_$__BuildOS__$__BuildArch__$__BuildType.log \
 538                              /p:__IntermediatesDir=$__IntermediatesDir /p:__RootBinDir=$__RootBinDir /p:BuildNugetPackages=false /p:UseSharedCompilation=false /p:__DoCrossArchBuild=$__CrossBuild \
 539                              $__CommonMSBuildArgs $__UnprocessedBuildArgs
 540
 541     if [ $? -ne 0 ]; then
 542         echo "Failed to generate Nuget packages."
 543         exit 1
 544     fi
 545 }
 546
 547 echo "Commencing CoreCLR Repo build"
 548
 549 # Argument types supported by this script:
 550 #
 551 # Build architecture - valid values are: x64, ARM.
 552 # Build Type         - valid values are: Debug, Checked, Release
 553 #
 554 # Set the default arguments for build
 555
 556 # Obtain the location of the bash script to figure out where the root of the repo is.
 557 __ProjectRoot="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
 558
 559 # Use uname to determine what the CPU is.
 560 CPUName=$(uname -p)
 561 # Some Linux platforms report unknown for platform, but the arch for machine.
 562 if [ "$CPUName" == "unknown" ]; then
 563     CPUName=$(uname -m)
 564 fi
 565
 566 case $CPUName in
 567     i686)
 568         echo "Unsupported CPU $CPUName detected, build might not succeed!"
 569         __BuildArch=x86
 570         __HostArch=x86
 571         ;;
 572
 573     x86_64)
 574         __BuildArch=x64
 575         __HostArch=x64
 576         ;;
 577
 578     armv7l)
 579         echo "Unsupported CPU $CPUName detected, build might not succeed!"
 580         __BuildArch=arm
 581         __HostArch=arm
 582         ;;
 583
 584     aarch64)
 585         __BuildArch=arm64
 586         __HostArch=arm64
 587         ;;
 588
 589     amd64)
 590         __BuildArch=x64
 591         __HostArch=x64
 592         ;;
 593     *)
 594         echo "Unknown CPU $CPUName detected, configuring as if for x64"
 595         __BuildArch=x64
 596         __HostArch=x64
 597         ;;
 598 esac
 599
 600 # Use uname to determine what the OS is.
 601 OSName=$(uname -s)
 602 case $OSName in
 603     Linux)
 604         __BuildOS=Linux
 605         __HostOS=Linux
 606         ;;
 607
 608     Darwin)
 609         __BuildOS=OSX
 610         __HostOS=OSX
 611         ;;
 612
 613     FreeBSD)
 614         __BuildOS=FreeBSD
 615         __HostOS=FreeBSD
 616         ;;
 617
 618     OpenBSD)
 619         __BuildOS=OpenBSD
 620         __HostOS=OpenBSD
 621         ;;
 622
 623     NetBSD)
 624         __BuildOS=NetBSD
 625         __HostOS=NetBSD
 626         ;;
 627
 628     SunOS)
 629         __BuildOS=SunOS
 630         __HostOS=SunOS
 631         ;;
 632
 633     *)
 634         echo "Unsupported OS $OSName detected, configuring as if for Linux"
 635         __BuildOS=Linux
 636         __HostOS=Linux
 637         ;;
 638 esac
 639
 640 __BuildType=Debug
 641 __CodeCoverage=
 642 __IgnoreWarnings=0
 643
 644 # Set the various build properties here so that CMake and MSBuild can pick them up
 645 __ProjectDir="$__ProjectRoot"
 646 __SourceDir="$__ProjectDir/src"
 647 __PackagesDir="${DotNetRestorePackagesPath:-${__ProjectDir}/packages}"
 648 __RootBinDir="$__ProjectDir/bin"
 649 __UnprocessedBuildArgs=
 650 __CommonMSBuildArgs=
 651 __MSBCleanBuildArgs=
 652 __UseNinja=0
 653 __VerboseBuild=0
 654 __PgoInstrument=0
 655 __PgoOptimize=1
 656 __IbcTuning=""
 657 __ConfigureOnly=0
 658 __SkipConfigure=0
 659 __SkipManaged=0
 660 __SkipRestore=""
 661 __SkipNuget=0
 662 __SkipCoreCLR=0
 663 __SkipMSCorLib=0
 664 __SkipRestoreOptData=0
 665 __SkipCrossgen=0
 666 __CrossgenOnly=0
 667 __PartialNgen=0
 668 __SkipTests=0
 669 __CrossBuild=0
 670 __ClangMajorVersion=0
 671 __ClangMinorVersion=0
 672 __NuGetPath="$__PackagesDir/NuGet.exe"
 673 __HostDistroRid=""
 674 __DistroRid=""
 675 __cmakeargs=""
 676 __SkipGenerateVersion=0
 677 __PortableBuild=1
 678 __msbuildonunsupportedplatform=0
 679 __PgoOptDataVersion=""
 680 __IbcOptDataVersion=""
 681 __BuildManagedTools=1
 682 __SkipRestoreArg=""
 683 __SignTypeArg=""
 684 __OfficialBuildIdArg=""
 685
 686 # Get the number of processors available to the scheduler
 687 # Other techniques such as `nproc` only get the number of
 688 # processors available to a single process.
 689 if [ `uname` = "FreeBSD" ]; then
 690   __NumProc=`sysctl hw.ncpu | awk '{ print $2+1 }'`
 691 elif [ `uname` = "NetBSD" ]; then
 692   __NumProc=$(($(getconf NPROCESSORS_ONLN)+1))
 693 elif [ `uname` = "Darwin" ]; then
 694   __NumProc=$(($(getconf _NPROCESSORS_ONLN)+1))
 695 else
 696   __NumProc=$(nproc --all)
 697 fi
 698
 699 while :; do
 700     if [ $# -le 0 ]; then
 701         break
 702     fi
 703
 704     lowerI="$(echo $1 | awk '{print tolower($0)}')"
 705     case $lowerI in
 706         -\?|-h|--help)
 707             usage
 708             exit 1
 709             ;;
 710
 711         x86|-x86)
 712             __BuildArch=x86
 713             ;;
 714
 715         x64|-x64)
 716             __BuildArch=x64
 717             ;;
 718
 719         arm|-arm)
 720             __BuildArch=arm
 721             ;;
 722
 723         armel|-armel)
 724             __BuildArch=armel
 725             ;;
 726
 727         arm64|-arm64)
 728             __BuildArch=arm64
 729             ;;
 730
 731         debug|-debug)
 732             __BuildType=Debug
 733             ;;
 734
 735         checked|-checked)
 736             __BuildType=Checked
 737             ;;
 738
 739         release|-release)
 740             __BuildType=Release
 741             ;;
 742
 743         coverage|-coverage)
 744             __CodeCoverage=Coverage
 745             ;;
 746
 747         cross|-cross)
 748             __CrossBuild=1
 749             ;;
 750
 751         -portablebuild=false)
 752             __PortableBuild=0
 753             ;;
 754
 755         verbose|-verbose)
 756             __VerboseBuild=1
 757             ;;
 758
 759         stripsymbols|-stripsymbols)
 760             __cmakeargs="$__cmakeargs -DSTRIP_SYMBOLS=true"
 761             ;;
 762
 763         clang3.5|-clang3.5)
 764             __ClangMajorVersion=3
 765             __ClangMinorVersion=5
 766             ;;
 767
 768         clang3.6|-clang3.6)
 769             __ClangMajorVersion=3
 770             __ClangMinorVersion=6
 771             ;;
 772
 773         clang3.7|-clang3.7)
 774             __ClangMajorVersion=3
 775             __ClangMinorVersion=7
 776             ;;
 777
 778         clang3.8|-clang3.8)
 779             __ClangMajorVersion=3
 780             __ClangMinorVersion=8
 781             ;;
 782
 783         clang3.9|-clang3.9)
 784             __ClangMajorVersion=3
 785             __ClangMinorVersion=9
 786             ;;
 787
 788         clang4.0|-clang4.0)
 789             __ClangMajorVersion=4
 790             __ClangMinorVersion=0
 791             ;;
 792
 793         clang5.0|-clang5.0)
 794             __ClangMajorVersion=5
 795             __ClangMinorVersion=0
 796             ;;
 797
 798         clang6.0|-clang6.0)
 799             __ClangMajorVersion=6
 800             __ClangMinorVersion=0
 801             ;;
 802
 803         clang7|-clang7)
 804             __ClangMajorVersion=7
 805             __ClangMinorVersion=
 806             ;;
 807
 808         ninja|-ninja)
 809             __UseNinja=1
 810             ;;
 811
 812         pgoinstrument|-pgoinstrument)
 813             __PgoInstrument=1
 814             ;;
 815
 816         nopgooptimize|-nopgooptimize)
 817             __PgoOptimize=0
 818             __SkipRestoreOptData=1
 819             ;;
 820
 821         ibcinstrument|-ibcinstrument)
 822             __IbcTuning="/Tuning"
 823             ;;
 824
 825         configureonly|-configureonly)
 826             __ConfigureOnly=1
 827             __SkipMSCorLib=1
 828             __SkipNuget=1
 829             ;;
 830
 831         skipconfigure|-skipconfigure)
 832             __SkipConfigure=1
 833             ;;
 834
 835         skipnative|-skipnative)
 836             # Use "skipnative" to use the same option name as build.cmd.
 837             __SkipCoreCLR=1
 838             ;;
 839
 840         skipcoreclr|-skipcoreclr)
 841             # Accept "skipcoreclr" for backwards-compatibility.
 842             __SkipCoreCLR=1
 843             ;;
 844
 845         skipmanaged|-skipmanaged)
 846             __SkipManaged=1
 847             ;;
 848
 849         skipmscorlib|-skipmscorlib)
 850             __SkipMSCorLib=1
 851             ;;
 852
 853         skipgenerateversion|-skipgenerateversion)
 854             __SkipGenerateVersion=1
 855             ;;
 856
 857         skiprestoreoptdata|-skiprestoreoptdata)
 858             __SkipRestoreOptData=1
 859             ;;
 860
 861         skipcrossgen|-skipcrossgen)
 862             __SkipCrossgen=1
 863             ;;
 864
 865         crossgenonly|-crossgenonly)
 866             __SkipMSCorLib=1
 867             __SkipCoreCLR=1
 868             __CrossgenOnly=1
 869             ;;
 870         partialngen|-partialngen)
 871             __PartialNgen=1
 872             ;;
 873
 874         skiptests|-skiptests)
 875             __SkipTests=1
 876             ;;
 877
 878         skipnuget|-skipnuget|skipbuildpackages|-skipbuildpackages)
 879             __SkipNuget=1
 880             ;;
 881
 882         ignorewarnings|-ignorewarnings)
 883             __IgnoreWarnings=1
 884             __cmakeargs="$__cmakeargs -DCLR_CMAKE_WARNINGS_ARE_ERRORS=OFF"
 885             ;;
 886
 887         cmakeargs|-cmakeargs)
 888             if [ -n "$2" ]; then
 889                 __cmakeargs="$__cmakeargs $2"
 890                 shift
 891             else
 892                 echo "ERROR: 'cmakeargs' requires a non-empty option argument"
 893                 exit 1
 894             fi
 895             ;;
 896
 897         bindir|-bindir)
 898             if [ -n "$2" ]; then
 899                 __RootBinDir="$2"
 900                 if [ ! -d $__RootBinDir ]; then
 901                     mkdir $__RootBinDir
 902                 fi
 903                 __RootBinParent=$(dirname $__RootBinDir)
 904                 __RootBinName=${__RootBinDir##*/}
 905                 __RootBinDir="$(cd $__RootBinParent &>/dev/null && printf %s/%s $PWD $__RootBinName)"
 906                 shift
 907             else
 908                 echo "ERROR: 'bindir' requires a non-empty option argument"
 909                 exit 1
 910             fi
 911             ;;
 912         msbuildonunsupportedplatform|-msbuildonunsupportedplatform)
 913             __msbuildonunsupportedplatform=1
 914             ;;
 915         numproc|-numproc)
 916             if [ -n "$2" ]; then
 917               __NumProc="$2"
 918               shift
 919             else
 920               echo "ERROR: 'numproc' requires a non-empty option argument"
 921               exit 1
 922             fi
 923             ;;
 924         osgroup|-osgroup)
 925             if [ -n "$2" ]; then
 926               __BuildOS="$2"
 927               shift
 928             else
 929               echo "ERROR: 'osgroup' requires a non-empty option argument"
 930               exit 1
 931             fi
 932             ;;
 933         rebuild|-rebuild)
 934             echo "ERROR: 'Rebuild' is not supported.  Please remove it."
 935             exit 1
 936             ;;
 937
 938         -skiprestore)
 939             __SkipRestoreArg="/p:RestoreDuringBuild=false"
 940             ;;
 941
 942         -disableoss)
 943             __SignTypeArg="/p:SignType=real"
 944             ;;
 945
 946         -officialbuildid=*)
 947             __Id=$(echo $1| cut -d'=' -f 2)
 948             __OfficialBuildIdArg="/p:OfficialBuildId=$__Id"
 949             ;;
 950
 951         --)
 952             # Skip -Option=Value style argument passing
 953             ;;
 954
 955         *)
 956             __UnprocessedBuildArgs="$__UnprocessedBuildArgs $1"
 957             ;;
 958     esac
 959
 960     shift
 961 done
 962
 963 __CommonMSBuildArgs="/p:__BuildArch=$__BuildArch /p:__BuildType=$__BuildType /p:__BuildOS=$__BuildOS $__OfficialBuildIdArg $__SignTypeArg $__SkipRestoreArg"
 964
 965 # Configure environment if we are doing a verbose build
 966 if [ $__VerboseBuild == 1 ]; then
 967     export VERBOSE=1
 968     __CommonMSBuildArgs="$__CommonMSBuildArgs /v:detailed"
 969 fi
 970
 971 # Set default clang version
 972 if [[ $__ClangMajorVersion == 0 && $__ClangMinorVersion == 0 ]]; then
 973     if [[ "$__BuildArch" == "arm" || "$__BuildArch" == "armel" ]]; then
 974         __ClangMajorVersion=5
 975         __ClangMinorVersion=0
 976     else
 977         __ClangMajorVersion=3
 978         __ClangMinorVersion=9
 979     fi
 980 fi
 981
 982 if [[ "$__BuildArch" == "armel" ]]; then
 983     # Armel cross build is Tizen specific and does not support Portable RID build
 984     __PortableBuild=0
 985 fi
 986
 987 if [ $__PortableBuild == 0 ]; then
 988     __CommonMSBuildArgs="$__CommonMSBuildArgs /p:PortableBuild=false"
 989 fi
 990
 991 # Set dependent variables
 992 __LogsDir="$__RootBinDir/Logs"
 993
 994 # init the host distro name
 995 initHostDistroRid
 996
 997 # Set the remaining variables based upon the determined build configuration
 998 __BinDir="$__RootBinDir/Product/$__BuildOS.$__BuildArch.$__BuildType"
 999 __PackagesBinDir="$__BinDir/.nuget"
1000 __ToolsDir="$__RootBinDir/tools"
1001 __TestWorkingDir="$__RootBinDir/tests/$__BuildOS.$__BuildArch.$__BuildType"
1002 export __IntermediatesDir="$__RootBinDir/obj/$__BuildOS.$__BuildArch.$__BuildType"
1003 __TestIntermediatesDir="$__RootBinDir/tests/obj/$__BuildOS.$__BuildArch.$__BuildType"
1004 __isMSBuildOnNETCoreSupported=0
1005 __CrossComponentBinDir="$__BinDir"
1006
1007 __CrossArch="$__HostArch"
1008 if [ $__CrossBuild == 1 ]; then
1009     __CrossComponentBinDir="$__CrossComponentBinDir/$__CrossArch"
1010 fi
1011 __CrossGenCoreLibLog="$__LogsDir/CrossgenCoreLib_$__BuildOS.$__BuildArch.$__BuildType.log"
1012
1013 # Init if MSBuild for .NET Core is supported for this platform
1014 isMSBuildOnNETCoreSupported
1015
1016 # CI_SPECIFIC - On CI machines, $HOME may not be set. In such a case, create a subfolder and set the variable to set.
1017 # This is needed by CLI to function.
1018 if [ -z "$HOME" ]; then
1019     if [ ! -d "$__ProjectDir/temp_home" ]; then
1020         mkdir temp_home
1021     fi
1022     export HOME=$__ProjectDir/temp_home
1023     echo "HOME not defined; setting it to $HOME"
1024 fi
1025
1026 # Specify path to be set for CMAKE_INSTALL_PREFIX.
1027 # This is where all built CoreClr libraries will copied to.
1028 export __CMakeBinDir="$__BinDir"
1029
1030 # Configure environment if we are doing a cross compile.
1031 if [ $__CrossBuild == 1 ]; then
1032     export CROSSCOMPILE=1
1033     if ! [[ -n "$ROOTFS_DIR" ]]; then
1034         export ROOTFS_DIR="$__ProjectRoot/cross/rootfs/$__BuildArch"
1035     fi
1036 fi
1037
1038 # init the target distro name
1039 initTargetDistroRid
1040
1041 # Make the directories necessary for build if they don't exist
1042 setup_dirs
1043
1044 # Check prereqs.
1045 check_prereqs
1046
1047 # Restore the package containing profile counts for profile-guided optimizations
1048 restore_optdata
1049
1050 # Generate event logging infrastructure sources
1051 generate_event_logging
1052
1053 # Build the coreclr (native) components.
1054 __ExtraCmakeArgs="-DCLR_CMAKE_TARGET_OS=$__BuildOS -DCLR_CMAKE_PACKAGES_DIR=$__PackagesDir -DCLR_CMAKE_PGO_INSTRUMENT=$__PgoInstrument -DCLR_CMAKE_OPTDATA_VERSION=$__PgoOptDataVersion -DCLR_CMAKE_PGO_OPTIMIZE=$__PgoOptimize"
1055
1056 # [TODO] Remove this when the `build-test.sh` script properly builds and deploys test assets.
1057 if [ $__SkipTests != 1 ]; then
1058     echo "Adding CMake flags to build native tests for $__BuildOS.$__BuildArch.$__BuildType"
1059     __ExtraCmakeArgs="$__ExtraCmakeArgs -DCLR_CMAKE_BUILD_TESTS=ON"
1060 fi
1061
1062 build_native $__SkipCoreCLR "$__BuildArch" "$__IntermediatesDir" "$__ExtraCmakeArgs" "CoreCLR component"
1063
1064 # Build cross-architecture components
1065 if [[ $__CrossBuild == 1 ]]; then
1066     build_cross_architecture_components
1067 fi
1068
1069 # Build System.Private.CoreLib.
1070
1071 build_CoreLib
1072
1073 if [ $__CrossgenOnly == 1 ]; then
1074     build_CoreLib_ni "$__BinDir/crossgen"
1075 fi
1076
1077 # Generate nuget packages
1078 if [ $__SkipNuget != 1 ]; then
1079     generate_NugetPackages
1080 fi
1081
1082
1083 # Build complete
1084
1085 echo "Repo successfully built."
1086 echo "Product binaries are available at $__BinDir"
1087 exit 0