build.sh

   1 #!/usr/bin/env bash
   2
   3 # Work around Jenkins CI + msbuild problem: Jenkins sometimes creates very large environment
   4 # variables, and msbuild can't handle environment blocks with such large variables. So clear
   5 # out the variables that might be too large.
   6 export ghprbCommentBody=
   7
   8 # resolve python-version to use
   9 if [ "$PYTHON" == "" ] ; then
  10     if ! PYTHON=$(command -v python3 || command -v python2 || command -v python)
  11     then
  12        echo "Unable to locate build-dependency python!" 1>&2
  13        exit 1
  14     fi
  15 fi
  16 # validate python-dependency
  17 # useful in case of explicitly set option.
  18 if ! command -v $PYTHON > /dev/null
  19 then
  20    echo "Unable to locate build-dependency python ($PYTHON)!" 1>&2
  21    exit 1
  22 fi
  23
  24 export PYTHON
  25
  26 usage()
  27 {
  28     echo "Usage: $0 [BuildArch] [BuildType] [-verbose] [-coverage] [-cross] [-clangx.y] [-ninja] [-configureonly] [-skipconfigure] [-skipnative] [-skipmanaged] [-skipmscorlib] [-skiptests] [-stripsymbols] [-ignorewarnings] [-cmakeargs] [-bindir]"
  29     echo "BuildArch can be: -x64, -x86, -arm, -armel, -arm64"
  30     echo "BuildType can be: -debug, -checked, -release"
  31     echo "-coverage - optional argument to enable code coverage build (currently supported only for Linux and OSX)."
  32     echo "-ninja - target ninja instead of GNU make"
  33     echo "-clangx.y - optional argument to build using clang version x.y."
  34     echo "-cross - optional argument to signify cross compilation,"
  35     echo "       - will use ROOTFS_DIR environment variable if set."
  36     echo "-nopgooptimize - do not use profile guided optimizations."
  37     echo "-pgoinstrument - generate instrumented code for profile guided optimization enabled binaries."
  38     echo "-ibcinstrument - generate IBC-tuning-enabled native images when invoking crossgen."
  39     echo "-configureonly - do not perform any builds; just configure the build."
  40     echo "-skipconfigure - skip build configuration."
  41     echo "-skipnative - do not build native components."
  42     echo "-skipmanaged - do not build managed components."
  43     echo "-skipmscorlib - do not build mscorlib.dll."
  44     echo "-skiptests - skip the tests in the 'tests' subdirectory."
  45     echo "-skipnuget - skip building nuget packages."
  46     echo "-skiprestoreoptdata - skip restoring optimization data used by profile-based optimizations."
  47     echo "-skipcrossgen - skip native image generation"
  48     echo "-crossgenonly - only run native image generation"
  49     echo "-partialngen - build CoreLib as PartialNGen"
  50     echo "-verbose - optional argument to enable verbose build output."
  51     echo "-skiprestore: skip restoring packages ^(default: packages are restored during build^)."
  52     echo "-disableoss: Disable Open Source Signing for System.Private.CoreLib."
  53     echo "-officialbuildid=^<ID^>: specify the official build ID to be used by this build."
  54     echo "-stripSymbols - Optional argument to strip native symbols during the build."
  55     echo "-skipgenerateversion - disable version generation even if MSBuild is supported."
  56     echo "-ignorewarnings - do not treat warnings as errors"
  57     echo "-cmakeargs - user-settable additional arguments passed to CMake."
  58     echo "-bindir - output directory (defaults to $__ProjectRoot/bin)"
  59     echo "-msbuildonunsupportedplatform - build managed binaries even if distro is not officially supported."
  60     echo "-numproc - set the number of build processes."
  61     exit 1
  62 }
  63
  64 initHostDistroRid()
  65 {
  66     __HostDistroRid=""
  67     if [ "$__HostOS" == "Linux" ]; then
  68         if [ -e /etc/os-release ]; then
  69             source /etc/os-release
  70             if [[ $ID == "rhel" ]]; then
  71                 # remove the last version digit
  72                 VERSION_ID=${VERSION_ID%.*}
  73             fi
  74             __HostDistroRid="$ID.$VERSION_ID-$__HostArch"
  75             if [[ $ID == "alpine" ]]; then
  76                 __HostDistroRid="linux-musl-$__HostArch"
  77             fi
  78         elif [ -e /etc/redhat-release ]; then
  79             local redhatRelease=$(</etc/redhat-release)
  80             if [[ $redhatRelease == "CentOS release 6."* || $redhatRelease == "Red Hat Enterprise Linux Server release 6."* ]]; then
  81                __HostDistroRid="rhel.6-$__HostArch"
  82             fi
  83             if [[ $redhatRelease == "CentOS Linux release 7."* ]]; then
  84                 __HostDistroRid="rhel.7-$__Arch"
  85             fi
  86         fi
  87     fi
  88     if [ "$__HostOS" == "FreeBSD" ]; then
  89         __freebsd_version=`sysctl -n kern.osrelease | cut -f1 -d'.'`
  90         __HostDistroRid="freebsd.$__freebsd_version-$__HostArch"
  91     fi
  92
  93     if [ "$__HostDistroRid" == "" ]; then
  94         echo "WARNING: Can not determine runtime id for current distro."
  95     fi
  96 }
  97
  98 initTargetDistroRid()
  99 {
 100     if [ $__CrossBuild == 1 ]; then
 101         if [ "$__BuildOS" == "Linux" ]; then
 102             if [ ! -e $ROOTFS_DIR/etc/os-release ]; then
 103                 if [ -e $ROOTFS_DIR/android_platform ]; then
 104                     source $ROOTFS_DIR/android_platform
 105                     export __DistroRid="$RID"
 106                 else
 107                     echo "WARNING: Can not determine runtime id for current distro."
 108                     export __DistroRid=""
 109                 fi
 110             else
 111                 source $ROOTFS_DIR/etc/os-release
 112                 export __DistroRid="$ID.$VERSION_ID-$__BuildArch"
 113             fi
 114         fi
 115     else
 116         export __DistroRid="$__HostDistroRid"
 117     fi
 118
 119     if [ "$__BuildOS" == "OSX" ]; then
 120         __PortableBuild=1
 121     fi
 122
 123     # Portable builds target the base RID
 124     if [ $__PortableBuild == 1 ]; then
 125         if [ "$__BuildOS" == "Linux" ]; then
 126             export __DistroRid="linux-$__BuildArch"
 127         elif [ "$__BuildOS" == "OSX" ]; then
 128             export __DistroRid="osx-$__BuildArch"
 129         elif [ "$__BuildOS" == "FreeBSD" ]; then
 130             export __DistroRid="freebsd-$__BuildArch"
 131         fi
 132     fi
 133 }
 134
 135 setup_dirs()
 136 {
 137     echo Setting up directories for build
 138
 139     mkdir -p "$__RootBinDir"
 140     mkdir -p "$__BinDir"
 141     mkdir -p "$__LogsDir"
 142     mkdir -p "$__IntermediatesDir"
 143
 144     if [ $__CrossBuild == 1 ]; then
 145         mkdir -p "$__CrossComponentBinDir"
 146     fi
 147 }
 148
 149 # Check the system to ensure the right prereqs are in place
 150
 151 check_prereqs()
 152 {
 153     echo "Checking prerequisites..."
 154
 155     # Check presence of CMake on the path
 156     hash cmake 2>/dev/null || { echo >&2 "Please install cmake before running this script"; exit 1; }
 157
 158
 159     # Minimum required version of clang is version 4.0 for arm/armel cross build
 160     if [[ $__CrossBuild == 1 && ("$__BuildArch" == "arm" || "$__BuildArch" == "armel") ]]; then
 161         if ! [[ "$__ClangMajorVersion" -ge "4" ]]; then
 162             echo "Please install clang4.0 or latest for arm/armel cross build"; exit 1;
 163         fi
 164     fi
 165
 166     # Check for clang
 167     __ClangCombinedDottedVersion=$__ClangMajorVersion;
 168     if [[ "$__ClangMinorVersion" != "" ]]; then
 169         __ClangCombinedDottedVersion=$__ClangCombinedDottedVersion.$__ClangMinorVersion
 170     fi
 171     hash clang-$__ClangCombinedDottedVersion 2>/dev/null ||  hash clang$__ClangMajorVersion$__ClangMinorVersion 2>/dev/null || hash clang 2>/dev/null || { echo >&2 "Please install clang-$__ClangMajorVersion.$__ClangMinorVersion before running this script"; exit 1; }
 172
 173 }
 174
 175 restore_optdata()
 176 {
 177     # we only need optdata on a Release build
 178     if [[ "$__BuildType" != "Release" ]]; then __SkipRestoreOptData=1; fi
 179
 180     if [[ ( $__SkipRestoreOptData == 0 ) && ( $__isMSBuildOnNETCoreSupported == 1 ) ]]; then
 181         echo "Restoring the OptimizationData package"
 182         "$__ProjectRoot/run.sh" build -optdata $__RunArgs $__UnprocessedBuildArgs
 183         if [ $? != 0 ]; then
 184             echo "Failed to restore the optimization data package."
 185             exit 1
 186         fi
 187     fi
 188
 189     if [ $__isMSBuildOnNETCoreSupported == 1 ]; then
 190         # Parse the optdata package versions out of msbuild so that we can pass them on to CMake
 191         local DotNetCli="$__ProjectRoot/Tools/dotnetcli/dotnet"
 192         if [ ! -f $DotNetCli ]; then
 193             source "$__ProjectRoot/init-tools.sh"
 194             if [ $? != 0 ]; then
 195                 echo "Failed to restore buildtools."
 196                 exit 1
 197             fi
 198         fi
 199         local OptDataProjectFilePath="$__ProjectRoot/src/.nuget/optdata/optdata.csproj"
 200         __PgoOptDataVersion=$(DOTNET_SKIP_FIRST_TIME_EXPERIENCE=1 $DotNetCli msbuild $OptDataProjectFilePath /t:DumpPgoDataPackageVersion /nologo | sed 's/^\s*//')
 201         __IbcOptDataVersion=$(DOTNET_SKIP_FIRST_TIME_EXPERIENCE=1 $DotNetCli msbuild $OptDataProjectFilePath /t:DumpIbcDataPackageVersion /nologo | sed 's/^[[:blank:]]*//')
 202     fi
 203 }
 204
 205 generate_event_logging_sources()
 206 {
 207     __OutputDir=$1
 208     __ConsumingBuildSystem=$2
 209
 210     __OutputIncDir="$__OutputDir/src/inc"
 211     __OutputEventingDir="$__OutputDir/eventing"
 212     __OutputEventProviderDir="$__OutputEventingDir/eventprovider"
 213
 214     echo "Laying out dynamically generated files consumed by $__ConsumingBuildSystem"
 215     echo "Laying out dynamically generated Event test files, etmdummy stub functions, and external linkages"
 216
 217     __PythonWarningFlags="-Wall"
 218     if [[ $__IgnoreWarnings == 0 ]]; then
 219         __PythonWarningFlags="$__PythonWarningFlags -Werror"
 220     fi
 221
 222     $PYTHON -B $__PythonWarningFlags "$__ProjectRoot/src/scripts/genEventing.py" --inc $__OutputIncDir --dummy $__OutputIncDir/etmdummy.h --man "$__ProjectRoot/src/vm/ClrEtwAll.man" --testdir "$__OutputEventProviderDir/tests"
 223     if [[ $? != 0 ]]; then
 224         exit 1
 225     fi
 226
 227     echo "Laying out dynamically generated EventPipe Implementation"
 228     $PYTHON -B $__PythonWarningFlags "$__ProjectRoot/src/scripts/genEventPipe.py" --man "$__ProjectRoot/src/vm/ClrEtwAll.man" --intermediate "$__OutputEventingDir/eventpipe"
 229
 230     echo "Laying out dynamically generated EventSource classes"
 231     $PYTHON -B $__PythonWarningFlags "$__ProjectRoot/src/scripts/genRuntimeEventSources.py" --man "$__ProjectRoot/src/vm/ClrEtwAll.man" --intermediate "$__OutputEventingDir"
 232
 233     # determine the logging system
 234     case $__BuildOS in
 235         Linux|FreeBSD)
 236             echo "Laying out dynamically generated Event Logging Implementation of Lttng"
 237             $PYTHON -B $__PythonWarningFlags "$__ProjectRoot/src/scripts/genLttngProvider.py" --man "$__ProjectRoot/src/vm/ClrEtwAll.man" --intermediate "$__OutputEventProviderDir"
 238             if [[ $? != 0 ]]; then
 239                 exit 1
 240             fi
 241             ;;
 242         *)
 243             echo "Laying out dummy event logging provider"
 244             $PYTHON -B $__PythonWarningFlags "$__ProjectRoot/src/scripts/genDummyProvider.py" --man "$__ProjectRoot/src/vm/ClrEtwAll.man" --intermediate "$__OutputEventProviderDir"
 245             if [[ $? != 0 ]]; then
 246                 exit 1
 247             fi
 248             ;;
 249     esac
 250 }
 251
 252 generate_event_logging()
 253 {
 254     # Event Logging Infrastructure
 255     if [[ $__SkipCoreCLR == 0 || $__SkipMSCorLib == 0 || $__ConfigureOnly == 1 ]]; then
 256         generate_event_logging_sources "$__IntermediatesDir" "the native build system"
 257     fi
 258 }
 259
 260 build_native()
 261 {
 262     skipCondition=$1
 263     platformArch="$2"
 264     intermediatesForBuild="$3"
 265     extraCmakeArguments="$4"
 266     message="$5"
 267
 268     if [ $skipCondition == 1 ]; then
 269         echo "Skipping $message build."
 270         return
 271     fi
 272
 273     # All set to commence the build
 274     echo "Commencing build of $message for $__BuildOS.$__BuildArch.$__BuildType in $intermediatesForBuild"
 275
 276     generator=""
 277     buildFile="Makefile"
 278     buildTool="make"
 279     if [ $__UseNinja == 1 ]; then
 280         generator="ninja"
 281         buildFile="build.ninja"
 282         if ! buildTool=$(command -v ninja || command -v ninja-build); then
 283            echo "Unable to locate ninja!" 1>&2
 284            exit 1
 285         fi
 286     fi
 287
 288     if [ $__SkipConfigure == 0 ]; then
 289         # if msbuild is not supported, then set __SkipGenerateVersion to 1
 290         if [ $__isMSBuildOnNETCoreSupported == 0 ]; then __SkipGenerateVersion=1; fi
 291         # Drop version.cpp file
 292         __versionSourceFile="$intermediatesForBuild/version.cpp"
 293         if [ $__SkipGenerateVersion == 0 ]; then
 294             pwd
 295             "$__ProjectRoot/run.sh" build -Project=$__ProjectDir/build.proj -generateHeaderUnix -NativeVersionSourceFile=$__versionSourceFile -MsBuildEventLogging="/l:BinClashLogger,Tools/Microsoft.DotNet.Build.Tasks.dll;LogFile=binclash.log"  $__RunArgs $__UnprocessedBuildArgs
 296         else
 297             # Generate the dummy version.cpp, but only if it didn't exist to make sure we don't trigger unnecessary rebuild
 298             __versionSourceLine="static char sccsid[] __attribute__((used)) = \"@(#)No version information produced\";"
 299             if [ -e $__versionSourceFile ]; then
 300                 read existingVersionSourceLine < $__versionSourceFile
 301             fi
 302             if [ "$__versionSourceLine" != "$existingVersionSourceLine" ]; then
 303                 echo $__versionSourceLine > $__versionSourceFile
 304             fi
 305         fi
 306
 307
 308         pushd "$intermediatesForBuild"
 309         # Regenerate the CMake solution
 310         echo "Invoking \"$__ProjectRoot/src/pal/tools/gen-buildsys-clang.sh\" \"$__ProjectRoot\" $__ClangMajorVersion \"$__ClangMinorVersion\" $platformArch $__BuildType $__CodeCoverage $generator $extraCmakeArguments $__cmakeargs"
 311         "$__ProjectRoot/src/pal/tools/gen-buildsys-clang.sh" "$__ProjectRoot" $__ClangMajorVersion "$__ClangMinorVersion" $platformArch $__BuildType $__CodeCoverage $generator "$extraCmakeArguments" "$__cmakeargs"
 312         popd
 313     fi
 314
 315     if [ ! -f "$intermediatesForBuild/$buildFile" ]; then
 316         echo "Failed to generate $message build project!"
 317         exit 1
 318     fi
 319
 320     # Build
 321     if [ $__ConfigureOnly == 1 ]; then
 322         echo "Finish configuration & skipping $message build."
 323         return
 324     fi
 325
 326     # Check that the makefiles were created.
 327     pushd "$intermediatesForBuild"
 328
 329     echo "Executing $buildTool install -j $__NumProc"
 330
 331     $buildTool install -j $__NumProc
 332     if [ $? != 0 ]; then
 333         echo "Failed to build $message."
 334         exit 1
 335     fi
 336
 337     popd
 338 }
 339
 340 build_cross_architecture_components()
 341 {
 342     local intermediatesForBuild="$__IntermediatesDir/Host$__CrossArch/crossgen"
 343     local crossArchBinDir="$__BinDir/$__CrossArch"
 344
 345     mkdir -p "$intermediatesForBuild"
 346     mkdir -p "$crossArchBinDir"
 347
 348     generate_event_logging_sources "$intermediatesForBuild" "the crossarch build system"
 349
 350     __SkipCrossArchBuild=1
 351     # check supported cross-architecture components host(__HostArch)/target(__BuildArch) pair
 352     if [[ ("$__BuildArch" == "arm" || "$__BuildArch" == "armel") && ("$__CrossArch" == "x86" || "$__CrossArch" == "x64") ]]; then
 353         __SkipCrossArchBuild=0
 354     elif [[ "$__BuildArch" == "arm64" && "$__CrossArch" == "x64" ]]; then
 355         __SkipCrossArchBuild=0
 356     else
 357         # not supported
 358         return
 359     fi
 360
 361     export __CMakeBinDir="$crossArchBinDir"
 362     export CROSSCOMPILE=0
 363
 364     __ExtraCmakeArgs="-DCLR_CMAKE_TARGET_ARCH=$__BuildArch -DCLR_CMAKE_TARGET_OS=$__BuildOS -DCLR_CMAKE_PACKAGES_DIR=$__PackagesDir -DCLR_CMAKE_PGO_INSTRUMENT=$__PgoInstrument -DCLR_CMAKE_OPTDATA_VERSION=$__PgoOptDataVersion -DCLR_CMAKE_PGO_OPTIMIZE=$__PgoOptimize -DCLR_CROSS_COMPONENTS_BUILD=1"
 365     build_native $__SkipCrossArchBuild "$__CrossArch" "$intermediatesForBuild" "$__ExtraCmakeArgs" "cross-architecture components"
 366
 367     export CROSSCOMPILE=1
 368 }
 369
 370 isMSBuildOnNETCoreSupported()
 371 {
 372     __isMSBuildOnNETCoreSupported=$__msbuildonunsupportedplatform
 373
 374     if [ $__isMSBuildOnNETCoreSupported == 1 ]; then
 375         return
 376     fi
 377
 378     if [ $__SkipManaged == 1 ]; then
 379         __isMSBuildOnNETCoreSupported=0
 380         return
 381     fi
 382
 383     if [ "$__HostArch" == "x64" ]; then
 384         if [ "$__HostOS" == "Linux" ]; then
 385             __isMSBuildOnNETCoreSupported=1
 386             # note: the RIDs below can use globbing patterns
 387             UNSUPPORTED_RIDS=("debian.9-x64" "ubuntu.17.04-x64")
 388             for UNSUPPORTED_RID in "${UNSUPPORTED_RIDS[@]}"
 389             do
 390                 if [[ $__HostDistroRid == $UNSUPPORTED_RID ]]; then
 391                     __isMSBuildOnNETCoreSupported=0
 392                     break
 393                 fi
 394             done
 395         elif [ "$__HostOS" == "OSX" ]; then
 396             __isMSBuildOnNETCoreSupported=1
 397         elif [ "$__HostOS" == "FreeBSD" ]; then
 398             __isMSBuildOnNETCoreSupported=1
 399         fi
 400     fi
 401 }
 402
 403
 404 build_CoreLib_ni()
 405 {
 406     local __CrossGenExec=$1
 407
 408     if [ $__PartialNgen == 1 ]; then
 409         export COMPlus_PartialNGen=1
 410     fi
 411
 412     if [ -e $__CrossGenCoreLibLog ]; then
 413         rm $__CrossGenCoreLibLog
 414     fi
 415     echo "Generating native image of System.Private.CoreLib.dll for $__BuildOS.$__BuildArch.$__BuildType. Logging to \"$__CrossGenCoreLibLog\"."
 416     echo "$__CrossGenExec /Platform_Assemblies_Paths $__BinDir/IL $__IbcTuning /out $__BinDir/System.Private.CoreLib.dll $__BinDir/IL/System.Private.CoreLib.dll"
 417     $__CrossGenExec /Platform_Assemblies_Paths $__BinDir/IL $__IbcTuning /out $__BinDir/System.Private.CoreLib.dll $__BinDir/IL/System.Private.CoreLib.dll >> $__CrossGenCoreLibLog 2>&1
 418     if [ $? -ne 0 ]; then
 419         echo "Failed to generate native image for System.Private.CoreLib. Refer to $__CrossGenCoreLibLog"
 420         exit 1
 421     fi
 422
 423     if [ "$__BuildOS" == "Linux" ]; then
 424         echo "Generating symbol file for System.Private.CoreLib.dll"
 425         echo "$__CrossGenExec /Platform_Assemblies_Paths $__BinDir /CreatePerfMap $__BinDir $__BinDir/System.Private.CoreLib.dll"
 426         $__CrossGenExec /Platform_Assemblies_Paths $__BinDir /CreatePerfMap $__BinDir $__BinDir/System.Private.CoreLib.dll >> $__CrossGenCoreLibLog 2>&1
 427         if [ $? -ne 0 ]; then
 428             echo "Failed to generate symbol file for System.Private.CoreLib. Refer to $__CrossGenCoreLibLog"
 429             exit 1
 430         fi
 431     fi
 432 }
 433
 434 build_CoreLib()
 435 {
 436     if [ $__isMSBuildOnNETCoreSupported == 0 ]; then
 437         echo "System.Private.CoreLib.dll build unsupported."
 438         return
 439     fi
 440
 441     if [ $__SkipMSCorLib == 1 ]; then
 442        echo "Skipping building System.Private.CoreLib."
 443        return
 444     fi
 445
 446     echo "Commencing build of managed components for $__BuildOS.$__BuildArch.$__BuildType"
 447
 448     # Invoke MSBuild
 449     __ExtraBuildArgs=""
 450     if [[ "$__IbcTuning" == "" ]]; then
 451         __ExtraBuildArgs="$__ExtraBuildArgs -OptimizationDataDir=\"$__PackagesDir/optimization.$__BuildOS-$__BuildArch.IBC.CoreCLR/$__IbcOptDataVersion/data/\""
 452         __ExtraBuildArgs="$__ExtraBuildArgs -EnableProfileGuidedOptimization=true"
 453     fi
 454
 455     if [[ "$__BuildManagedTools" -eq "1" ]]; then
 456         __ExtraBuildArgs="$__ExtraBuildArgs -BuildManagedTools=true"
 457     fi
 458
 459     $__ProjectRoot/run.sh build -Project=$__ProjectDir/build.proj -MsBuildEventLogging="/l:BinClashLogger,Tools/Microsoft.DotNet.Build.Tasks.dll;LogFile=binclash.log" -MsBuildLog="/flp:Verbosity=normal;LogFile=$__LogsDir/System.Private.CoreLib_$__BuildOS__$__BuildArch__$__BuildType.log" -BuildTarget -__IntermediatesDir=$__IntermediatesDir -__RootBinDir=$__RootBinDir -BuildNugetPackage=false -UseSharedCompilation=false $__RunArgs $__ExtraBuildArgs $__UnprocessedBuildArgs
 460
 461     if [ $? -ne 0 ]; then
 462         echo "Failed to build managed components."
 463         exit 1
 464     fi
 465
 466     if [ $__SkipCrossgen == 1 ]; then
 467         echo "Skipping generating native image"
 468         return
 469     fi
 470
 471     # The cross build generates a crossgen with the target architecture.
 472     if [ $__CrossBuild == 0 ]; then
 473        if [ $__SkipCoreCLR == 1 ]; then
 474            return
 475        fi
 476
 477        # The architecture of host pc must be same architecture with target.
 478        if [[ ( "$__HostArch" == "$__BuildArch" ) ]]; then
 479            build_CoreLib_ni "$__BinDir/crossgen"
 480        elif [[ ( "$__HostArch" == "x64" ) && ( "$__BuildArch" == "x86" ) ]]; then
 481            build_CoreLib_ni "$__BinDir/crossgen"
 482        elif [[ ( "$__HostArch" == "arm64" ) && ( "$__BuildArch" == "arm" ) ]]; then
 483            build_CoreLib_ni "$__BinDir/crossgen"
 484        else
 485            exit 1
 486        fi
 487     else
 488        if [[ ( "$__CrossArch" == "x86" ) && ( "$__BuildArch" == "arm" ) ]]; then
 489            build_CoreLib_ni "$__CrossComponentBinDir/crossgen"
 490        elif [[ ( "$__CrossArch" == "x64" ) && ( "$__BuildArch" == "arm" ) ]]; then
 491            build_CoreLib_ni "$__CrossComponentBinDir/crossgen"
 492        elif [[ ( "$__HostArch" == "x64" ) && ( "$__BuildArch" == "arm64" ) ]]; then
 493            build_CoreLib_ni "$__CrossComponentBinDir/crossgen"
 494        fi
 495     fi
 496 }
 497
 498 generate_NugetPackages()
 499 {
 500     # We can only generate nuget package if we also support building mscorlib as part of this build.
 501     if [ $__isMSBuildOnNETCoreSupported == 0 ]; then
 502         echo "Nuget package generation unsupported."
 503         return
 504     fi
 505
 506     # Since we can build mscorlib for this OS, did we build the native components as well?
 507     if [[ $__SkipCoreCLR == 1 && $__CrossgenOnly == 0 ]]; then
 508         echo "Unable to generate nuget packages since native components were not built."
 509         return
 510     fi
 511
 512     echo "Generating nuget packages for "$__BuildOS
 513     echo "DistroRid is "$__DistroRid
 514     echo "ROOTFS_DIR is "$ROOTFS_DIR
 515     # Build the packages
 516     $__ProjectRoot/run.sh build -Project=$__SourceDir/.nuget/packages.builds -MsBuildEventLogging="/l:BinClashLogger,Tools/Microsoft.DotNet.Build.Tasks.dll;LogFile=binclash.log" -MsBuildLog="/flp:Verbosity=normal;LogFile=$__LogsDir/Nuget_$__BuildOS__$__BuildArch__$__BuildType.log" -BuildTarget -__IntermediatesDir=$__IntermediatesDir -__RootBinDir=$__RootBinDir -BuildNugetPackage=false -UseSharedCompilation=false -__DoCrossArchBuild=$__CrossBuild $__RunArgs $__UnprocessedBuildArgs
 517
 518     if [ $? -ne 0 ]; then
 519         echo "Failed to generate Nuget packages."
 520         exit 1
 521     fi
 522 }
 523
 524 echo "Commencing CoreCLR Repo build"
 525
 526 # Argument types supported by this script:
 527 #
 528 # Build architecture - valid values are: x64, ARM.
 529 # Build Type         - valid values are: Debug, Checked, Release
 530 #
 531 # Set the default arguments for build
 532
 533 # Obtain the location of the bash script to figure out where the root of the repo is.
 534 __ProjectRoot="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
 535
 536 # Use uname to determine what the CPU is.
 537 CPUName=$(uname -p)
 538 # Some Linux platforms report unknown for platform, but the arch for machine.
 539 if [ "$CPUName" == "unknown" ]; then
 540     CPUName=$(uname -m)
 541 fi
 542
 543 case $CPUName in
 544     i686)
 545         echo "Unsupported CPU $CPUName detected, build might not succeed!"
 546         __BuildArch=x86
 547         __HostArch=x86
 548         ;;
 549
 550     x86_64)
 551         __BuildArch=x64
 552         __HostArch=x64
 553         ;;
 554
 555     armv7l)
 556         echo "Unsupported CPU $CPUName detected, build might not succeed!"
 557         __BuildArch=arm
 558         __HostArch=arm
 559         ;;
 560
 561     aarch64)
 562         __BuildArch=arm64
 563         __HostArch=arm64
 564         ;;
 565
 566     amd64)
 567         __BuildArch=x64
 568         __HostArch=x64
 569         ;;
 570     *)
 571         echo "Unknown CPU $CPUName detected, configuring as if for x64"
 572         __BuildArch=x64
 573         __HostArch=x64
 574         ;;
 575 esac
 576
 577 # Use uname to determine what the OS is.
 578 OSName=$(uname -s)
 579 case $OSName in
 580     Linux)
 581         __BuildOS=Linux
 582         __HostOS=Linux
 583         ;;
 584
 585     Darwin)
 586         __BuildOS=OSX
 587         __HostOS=OSX
 588         ;;
 589
 590     FreeBSD)
 591         __BuildOS=FreeBSD
 592         __HostOS=FreeBSD
 593         ;;
 594
 595     OpenBSD)
 596         __BuildOS=OpenBSD
 597         __HostOS=OpenBSD
 598         ;;
 599
 600     NetBSD)
 601         __BuildOS=NetBSD
 602         __HostOS=NetBSD
 603         ;;
 604
 605     SunOS)
 606         __BuildOS=SunOS
 607         __HostOS=SunOS
 608         ;;
 609
 610     *)
 611         echo "Unsupported OS $OSName detected, configuring as if for Linux"
 612         __BuildOS=Linux
 613         __HostOS=Linux
 614         ;;
 615 esac
 616
 617 __BuildType=Debug
 618 __CodeCoverage=
 619 __IgnoreWarnings=0
 620
 621 # Set the various build properties here so that CMake and MSBuild can pick them up
 622 __ProjectDir="$__ProjectRoot"
 623 __SourceDir="$__ProjectDir/src"
 624 __PackagesDir="${DotNetRestorePackagesPath:-${__ProjectDir}/packages}"
 625 __RootBinDir="$__ProjectDir/bin"
 626 __UnprocessedBuildArgs=
 627 __RunArgs=
 628 __MSBCleanBuildArgs=
 629 __UseNinja=0
 630 __VerboseBuild=0
 631 __PgoInstrument=0
 632 __PgoOptimize=1
 633 __IbcTuning=""
 634 __ConfigureOnly=0
 635 __SkipConfigure=0
 636 __SkipManaged=0
 637 __SkipRestore=""
 638 __SkipNuget=0
 639 __SkipCoreCLR=0
 640 __SkipMSCorLib=0
 641 __SkipRestoreOptData=0
 642 __SkipCrossgen=0
 643 __CrossgenOnly=0
 644 __PartialNgen=0
 645 __SkipTests=0
 646 __CrossBuild=0
 647 __ClangMajorVersion=0
 648 __ClangMinorVersion=0
 649 __NuGetPath="$__PackagesDir/NuGet.exe"
 650 __HostDistroRid=""
 651 __DistroRid=""
 652 __cmakeargs=""
 653 __SkipGenerateVersion=0
 654 __PortableBuild=1
 655 __msbuildonunsupportedplatform=0
 656 __PgoOptDataVersion=""
 657 __IbcOptDataVersion=""
 658 __BuildManagedTools=1
 659
 660 # Get the number of processors available to the scheduler
 661 # Other techniques such as `nproc` only get the number of
 662 # processors available to a single process.
 663 if [ `uname` = "FreeBSD" ]; then
 664   __NumProc=`sysctl hw.ncpu | awk '{ print $2+1 }'`
 665 elif [ `uname` = "NetBSD" ]; then
 666   __NumProc=$(($(getconf NPROCESSORS_ONLN)+1))
 667 elif [ `uname` = "Darwin" ]; then
 668   __NumProc=$(($(getconf _NPROCESSORS_ONLN)+1))
 669 else
 670   __NumProc=$(nproc --all)
 671 fi
 672
 673 while :; do
 674     if [ $# -le 0 ]; then
 675         break
 676     fi
 677
 678     lowerI="$(echo $1 | awk '{print tolower($0)}')"
 679     case $lowerI in
 680         -\?|-h|--help)
 681             usage
 682             exit 1
 683             ;;
 684
 685         x86|-x86)
 686             __BuildArch=x86
 687             ;;
 688
 689         x64|-x64)
 690             __BuildArch=x64
 691             ;;
 692
 693         arm|-arm)
 694             __BuildArch=arm
 695             ;;
 696
 697         armel|-armel)
 698             __BuildArch=armel
 699             ;;
 700
 701         arm64|-arm64)
 702             __BuildArch=arm64
 703             ;;
 704
 705         debug|-debug)
 706             __BuildType=Debug
 707             ;;
 708
 709         checked|-checked)
 710             __BuildType=Checked
 711             ;;
 712
 713         release|-release)
 714             __BuildType=Release
 715             ;;
 716
 717         coverage|-coverage)
 718             __CodeCoverage=Coverage
 719             ;;
 720
 721         cross|-cross)
 722             __CrossBuild=1
 723             ;;
 724
 725         -portablebuild=false)
 726             __PortableBuild=0
 727             ;;
 728
 729         verbose|-verbose)
 730             __VerboseBuild=1
 731             ;;
 732
 733         stripsymbols|-stripsymbols)
 734             __cmakeargs="$__cmakeargs -DSTRIP_SYMBOLS=true"
 735             ;;
 736
 737         clang3.5|-clang3.5)
 738             __ClangMajorVersion=3
 739             __ClangMinorVersion=5
 740             ;;
 741
 742         clang3.6|-clang3.6)
 743             __ClangMajorVersion=3
 744             __ClangMinorVersion=6
 745             ;;
 746
 747         clang3.7|-clang3.7)
 748             __ClangMajorVersion=3
 749             __ClangMinorVersion=7
 750             ;;
 751
 752         clang3.8|-clang3.8)
 753             __ClangMajorVersion=3
 754             __ClangMinorVersion=8
 755             ;;
 756
 757         clang3.9|-clang3.9)
 758             __ClangMajorVersion=3
 759             __ClangMinorVersion=9
 760             ;;
 761
 762         clang4.0|-clang4.0)
 763             __ClangMajorVersion=4
 764             __ClangMinorVersion=0
 765             ;;
 766
 767         clang5.0|-clang5.0)
 768             __ClangMajorVersion=5
 769             __ClangMinorVersion=0
 770             ;;
 771
 772         clang6.0|-clang6.0)
 773             __ClangMajorVersion=6
 774             __ClangMinorVersion=0
 775             ;;
 776
 777         clang7|-clang7)
 778             __ClangMajorVersion=7
 779             __ClangMinorVersion=
 780             ;;
 781
 782         ninja|-ninja)
 783             __UseNinja=1
 784             ;;
 785
 786         pgoinstrument|-pgoinstrument)
 787             __PgoInstrument=1
 788             ;;
 789
 790         nopgooptimize|-nopgooptimize)
 791             __PgoOptimize=0
 792             __SkipRestoreOptData=1
 793             ;;
 794
 795         ibcinstrument|-ibcinstrument)
 796             __IbcTuning="/Tuning"
 797             ;;
 798
 799         configureonly|-configureonly)
 800             __ConfigureOnly=1
 801             __SkipMSCorLib=1
 802             __SkipNuget=1
 803             ;;
 804
 805         skipconfigure|-skipconfigure)
 806             __SkipConfigure=1
 807             ;;
 808
 809         skipnative|-skipnative)
 810             # Use "skipnative" to use the same option name as build.cmd.
 811             __SkipCoreCLR=1
 812             ;;
 813
 814         skipcoreclr|-skipcoreclr)
 815             # Accept "skipcoreclr" for backwards-compatibility.
 816             __SkipCoreCLR=1
 817             ;;
 818
 819         skipmanaged|-skipmanaged)
 820             __SkipManaged=1
 821             ;;
 822
 823         skipmscorlib|-skipmscorlib)
 824             __SkipMSCorLib=1
 825             ;;
 826
 827         skipgenerateversion|-skipgenerateversion)
 828             __SkipGenerateVersion=1
 829             ;;
 830
 831         skiprestoreoptdata|-skiprestoreoptdata)
 832             __SkipRestoreOptData=1
 833             ;;
 834
 835         skipcrossgen|-skipcrossgen)
 836             __SkipCrossgen=1
 837             ;;
 838
 839         crossgenonly|-crossgenonly)
 840             __SkipMSCorLib=1
 841             __SkipCoreCLR=1
 842             __CrossgenOnly=1
 843             ;;
 844         partialngen|-partialngen)
 845             __PartialNgen=1
 846             ;;
 847
 848         skiptests|-skiptests)
 849             __SkipTests=1
 850             ;;
 851
 852         skipnuget|-skipnuget|skipbuildpackages|-skipbuildpackages)
 853             __SkipNuget=1
 854             ;;
 855
 856         ignorewarnings|-ignorewarnings)
 857             __IgnoreWarnings=1
 858             __cmakeargs="$__cmakeargs -DCLR_CMAKE_WARNINGS_ARE_ERRORS=OFF"
 859             ;;
 860
 861         cmakeargs|-cmakeargs)
 862             if [ -n "$2" ]; then
 863                 __cmakeargs="$__cmakeargs $2"
 864                 shift
 865             else
 866                 echo "ERROR: 'cmakeargs' requires a non-empty option argument"
 867                 exit 1
 868             fi
 869             ;;
 870
 871         bindir|-bindir)
 872             if [ -n "$2" ]; then
 873                 __RootBinDir="$2"
 874                 if [ ! -d $__RootBinDir ]; then
 875                     mkdir $__RootBinDir
 876                 fi
 877                 __RootBinParent=$(dirname $__RootBinDir)
 878                 __RootBinName=${__RootBinDir##*/}
 879                 __RootBinDir="$(cd $__RootBinParent &>/dev/null && printf %s/%s $PWD $__RootBinName)"
 880                 shift
 881             else
 882                 echo "ERROR: 'bindir' requires a non-empty option argument"
 883                 exit 1
 884             fi
 885             ;;
 886         msbuildonunsupportedplatform|-msbuildonunsupportedplatform)
 887             __msbuildonunsupportedplatform=1
 888             ;;
 889         numproc|-numproc)
 890             if [ -n "$2" ]; then
 891               __NumProc="$2"
 892               shift
 893             else
 894               echo "ERROR: 'numproc' requires a non-empty option argument"
 895               exit 1
 896             fi
 897             ;;
 898         osgroup|-osgroup)
 899             if [ -n "$2" ]; then
 900               __BuildOS="$2"
 901               shift
 902             else
 903               echo "ERROR: 'osgroup' requires a non-empty option argument"
 904               exit 1
 905             fi
 906             ;;
 907         rebuild|-rebuild)
 908             echo "ERROR: 'Rebuild' is not supported.  Please remove it."
 909             exit 1
 910             ;;
 911
 912         *)
 913             __UnprocessedBuildArgs="$__UnprocessedBuildArgs $1"
 914             ;;
 915     esac
 916
 917     shift
 918 done
 919
 920 __RunArgs="-BuildArch=$__BuildArch -BuildType=$__BuildType -BuildOS=$__BuildOS"
 921
 922 # Configure environment if we are doing a verbose build
 923 if [ $__VerboseBuild == 1 ]; then
 924     export VERBOSE=1
 925         __RunArgs="$__RunArgs -verbose"
 926 fi
 927
 928 # Set default clang version
 929 if [[ $__ClangMajorVersion == 0 && $__ClangMinorVersion == 0 ]]; then
 930         if [[ "$__BuildArch" == "arm" || "$__BuildArch" == "armel" ]]; then
 931                 __ClangMajorVersion=5
 932                 __ClangMinorVersion=0
 933         else
 934                 __ClangMajorVersion=3
 935                 __ClangMinorVersion=9
 936         fi
 937 fi
 938
 939 if [[ "$__BuildArch" == "armel" ]]; then
 940     # Armel cross build is Tizen specific and does not support Portable RID build
 941     __PortableBuild=0
 942 fi
 943
 944 if [ $__PortableBuild == 0 ]; then
 945         __RunArgs="$__RunArgs -PortableBuild=false"
 946 fi
 947
 948 # Set dependent variables
 949 __LogsDir="$__RootBinDir/Logs"
 950
 951 # init the host distro name
 952 initHostDistroRid
 953
 954 # Set the remaining variables based upon the determined build configuration
 955 __BinDir="$__RootBinDir/Product/$__BuildOS.$__BuildArch.$__BuildType"
 956 __PackagesBinDir="$__BinDir/.nuget"
 957 __ToolsDir="$__RootBinDir/tools"
 958 __TestWorkingDir="$__RootBinDir/tests/$__BuildOS.$__BuildArch.$__BuildType"
 959 export __IntermediatesDir="$__RootBinDir/obj/$__BuildOS.$__BuildArch.$__BuildType"
 960 __TestIntermediatesDir="$__RootBinDir/tests/obj/$__BuildOS.$__BuildArch.$__BuildType"
 961 __isMSBuildOnNETCoreSupported=0
 962 __CrossComponentBinDir="$__BinDir"
 963
 964 __CrossArch="$__HostArch"
 965 if [ $__CrossBuild == 1 ]; then
 966     __CrossComponentBinDir="$__CrossComponentBinDir/$__CrossArch"
 967 fi
 968 __CrossGenCoreLibLog="$__LogsDir/CrossgenCoreLib_$__BuildOS.$__BuildArch.$__BuildType.log"
 969
 970 # Init if MSBuild for .NET Core is supported for this platform
 971 isMSBuildOnNETCoreSupported
 972
 973 # CI_SPECIFIC - On CI machines, $HOME may not be set. In such a case, create a subfolder and set the variable to set.
 974 # This is needed by CLI to function.
 975 if [ -z "$HOME" ]; then
 976     if [ ! -d "$__ProjectDir/temp_home" ]; then
 977         mkdir temp_home
 978     fi
 979     export HOME=$__ProjectDir/temp_home
 980     echo "HOME not defined; setting it to $HOME"
 981 fi
 982
 983 # Specify path to be set for CMAKE_INSTALL_PREFIX.
 984 # This is where all built CoreClr libraries will copied to.
 985 export __CMakeBinDir="$__BinDir"
 986
 987 # Configure environment if we are doing a cross compile.
 988 if [ $__CrossBuild == 1 ]; then
 989     export CROSSCOMPILE=1
 990     if ! [[ -n "$ROOTFS_DIR" ]]; then
 991         export ROOTFS_DIR="$__ProjectRoot/cross/rootfs/$__BuildArch"
 992     fi
 993 fi
 994
 995 # init the target distro name
 996 initTargetDistroRid
 997
 998 # Make the directories necessary for build if they don't exist
 999 setup_dirs
1000
1001 # Check prereqs.
1002 check_prereqs
1003
1004 # Restore the package containing profile counts for profile-guided optimizations
1005 restore_optdata
1006
1007 # Generate event logging infrastructure sources
1008 generate_event_logging
1009
1010 # Build the coreclr (native) components.
1011 __ExtraCmakeArgs="-DCLR_CMAKE_TARGET_OS=$__BuildOS -DCLR_CMAKE_PACKAGES_DIR=$__PackagesDir -DCLR_CMAKE_PGO_INSTRUMENT=$__PgoInstrument -DCLR_CMAKE_OPTDATA_VERSION=$__PgoOptDataVersion -DCLR_CMAKE_PGO_OPTIMIZE=$__PgoOptimize"
1012
1013 # [TODO] Remove this when the `build-test.sh` script properly builds and deploys test assets.
1014 if [ $__SkipTests != 1 ]; then
1015     echo "Adding CMake flags to build native tests for $__BuildOS.$__BuildArch.$__BuildType"
1016     __ExtraCmakeArgs="$__ExtraCmakeArgs -DCLR_CMAKE_BUILD_TESTS=ON"
1017 fi
1018
1019 build_native $__SkipCoreCLR "$__BuildArch" "$__IntermediatesDir" "$__ExtraCmakeArgs" "CoreCLR component"
1020
1021 # Build cross-architecture components
1022 if [[ $__CrossBuild == 1 ]]; then
1023     build_cross_architecture_components
1024 fi
1025
1026 # Build System.Private.CoreLib.
1027
1028 build_CoreLib
1029
1030 if [ $__CrossgenOnly == 1 ]; then
1031     build_CoreLib_ni "$__BinDir/crossgen"
1032 fi
1033
1034 # Generate nuget packages
1035 if [ $__SkipNuget != 1 ]; then
1036     generate_NugetPackages
1037 fi
1038
1039
1040 # Build complete
1041
1042 echo "Repo successfully built."
1043 echo "Product binaries are available at $__BinDir"
1044 exit 0