build.sh

   1 #!/usr/bin/env bash
   2
   3 # Work around Jenkins CI + msbuild problem: Jenkins sometimes creates very large environment
   4 # variables, and msbuild can't handle environment blocks with such large variables. So clear
   5 # out the variables that might be too large.
   6 export ghprbCommentBody=
   7
   8 # resolve python-version to use
   9 if [ "$PYTHON" == "" ] ; then
  10     if ! PYTHON=$(command -v python2.7 || command -v python2 || command -v python)
  11     then
  12        echo "Unable to locate build-dependency python2.x!" 1>&2
  13        exit 1
  14     fi
  15 fi
  16
  17 # validate python-dependency
  18 # useful in case of explicitly set option.
  19 if ! command -v $PYTHON > /dev/null
  20 then
  21    echo "Unable to locate build-dependency python2.x ($PYTHON)!" 1>&2
  22    exit 1
  23 fi
  24
  25 export PYTHON
  26
  27 usage()
  28 {
  29     echo "Usage: $0 [BuildArch] [BuildType] [-verbose] [-coverage] [-cross] [-clangx.y] [-ninja] [-configureonly] [-skipconfigure] [-skipnative] [-skipmanaged] [-skipmscorlib] [-skiptests] [-stripsymbols] [-ignorewarnings] [-cmakeargs] [-bindir]"
  30     echo "BuildArch can be: -x64, -x86, -arm, -armel, -arm64"
  31     echo "BuildType can be: -debug, -checked, -release"
  32     echo "-coverage - optional argument to enable code coverage build (currently supported only for Linux and OSX)."
  33     echo "-ninja - target ninja instead of GNU make"
  34     echo "-clangx.y - optional argument to build using clang version x.y."
  35     echo "-cross - optional argument to signify cross compilation,"
  36     echo "       - will use ROOTFS_DIR environment variable if set."
  37     echo "-nopgooptimize - do not use profile guided optimizations."
  38     echo "-pgoinstrument - generate instrumented code for profile guided optimization enabled binaries."
  39     echo "-ibcinstrument - generate IBC-tuning-enabled native images when invoking crossgen."
  40     echo "-configureonly - do not perform any builds; just configure the build."
  41     echo "-skipconfigure - skip build configuration."
  42     echo "-skipnative - do not build native components."
  43     echo "-skipmanaged - do not build managed components."
  44     echo "-skipmscorlib - do not build mscorlib.dll."
  45     echo "-skiptests - skip the tests in the 'tests' subdirectory."
  46     echo "-skipnuget - skip building nuget packages."
  47     echo "-skiprestoreoptdata - skip restoring optimization data used by profile-based optimizations."
  48     echo "-skipcrossgen - skip native image generation"
  49     echo "-crossgenonly - only run native image generation"
  50     echo "-partialngen - build CoreLib as PartialNGen"
  51     echo "-verbose - optional argument to enable verbose build output."
  52     echo "-skiprestore: skip restoring packages ^(default: packages are restored during build^)."
  53     echo "-disableoss: Disable Open Source Signing for System.Private.CoreLib."
  54     echo "-officialbuildid=^<ID^>: specify the official build ID to be used by this build."
  55     echo "-stripSymbols - Optional argument to strip native symbols during the build."
  56     echo "-skipgenerateversion - disable version generation even if MSBuild is supported."
  57     echo "-ignorewarnings - do not treat warnings as errors"
  58     echo "-cmakeargs - user-settable additional arguments passed to CMake."
  59     echo "-bindir - output directory (defaults to $__ProjectRoot/bin)"
  60     echo "-msbuildonunsupportedplatform - build managed binaries even if distro is not officially supported."
  61     echo "-numproc - set the number of build processes."
  62     exit 1
  63 }
  64
  65 initHostDistroRid()
  66 {
  67     __HostDistroRid=""
  68     if [ "$__HostOS" == "Linux" ]; then
  69         if [ -e /etc/os-release ]; then
  70             source /etc/os-release
  71             if [[ $ID == "rhel" ]]; then
  72                 # remove the last version digit
  73                 VERSION_ID=${VERSION_ID%.*}
  74             fi
  75             __HostDistroRid="$ID.$VERSION_ID-$__HostArch"
  76             if [[ $ID == "alpine" ]]; then
  77                 __HostDistroRid="linux-musl-$__HostArch"
  78             fi
  79         elif [ -e /etc/redhat-release ]; then
  80             local redhatRelease=$(</etc/redhat-release)
  81             if [[ $redhatRelease == "CentOS release 6."* || $redhatRelease == "Red Hat Enterprise Linux Server release 6."* ]]; then
  82                __HostDistroRid="rhel.6-$__HostArch"
  83             fi
  84         fi
  85     fi
  86     if [ "$__HostOS" == "FreeBSD" ]; then
  87         __freebsd_version=`sysctl -n kern.osrelease | cut -f1 -d'.'`
  88         __HostDistroRid="freebsd.$__freebsd_version-$__HostArch"
  89     fi
  90
  91     if [ "$__HostDistroRid" == "" ]; then
  92         echo "WARNING: Can not determine runtime id for current distro."
  93     fi
  94 }
  95
  96 initTargetDistroRid()
  97 {
  98     if [ $__CrossBuild == 1 ]; then
  99         if [ "$__BuildOS" == "Linux" ]; then
 100             if [ ! -e $ROOTFS_DIR/etc/os-release ]; then
 101                 if [ -e $ROOTFS_DIR/android_platform ]; then
 102                     source $ROOTFS_DIR/android_platform
 103                     export __DistroRid="$RID"
 104                 else
 105                     echo "WARNING: Can not determine runtime id for current distro."
 106                     export __DistroRid=""
 107                 fi
 108             else
 109                 source $ROOTFS_DIR/etc/os-release
 110                 export __DistroRid="$ID.$VERSION_ID-$__BuildArch"
 111             fi
 112         fi
 113     else
 114         export __DistroRid="$__HostDistroRid"
 115     fi
 116
 117     if [ "$__BuildOS" == "OSX" ]; then
 118         __PortableBuild=1
 119     fi
 120
 121     # Portable builds target the base RID
 122     if [ $__PortableBuild == 1 ]; then
 123         if [ "$__BuildOS" == "Linux" ]; then
 124             export __DistroRid="linux-$__BuildArch"
 125         elif [ "$__BuildOS" == "OSX" ]; then
 126             export __DistroRid="osx-$__BuildArch"
 127         elif [ "$__BuildOS" == "FreeBSD" ]; then
 128             export __DistroRid="freebsd-$__BuildArch"
 129         fi
 130     fi
 131 }
 132
 133 setup_dirs()
 134 {
 135     echo Setting up directories for build
 136
 137     mkdir -p "$__RootBinDir"
 138     mkdir -p "$__BinDir"
 139     mkdir -p "$__LogsDir"
 140     mkdir -p "$__IntermediatesDir"
 141
 142     if [ $__CrossBuild == 1 ]; then
 143         mkdir -p "$__CrossComponentBinDir"
 144     fi
 145 }
 146
 147 # Check the system to ensure the right prereqs are in place
 148
 149 check_prereqs()
 150 {
 151     echo "Checking prerequisites..."
 152
 153     # Check presence of CMake on the path
 154     hash cmake 2>/dev/null || { echo >&2 "Please install cmake before running this script"; exit 1; }
 155
 156
 157     # Minimum required version of clang is version 4.0 for arm/armel cross build
 158     if [[ $__CrossBuild == 1 && ("$__BuildArch" == "arm" || "$__BuildArch" == "armel") ]]; then
 159         if ! [[ "$__ClangMajorVersion" -ge "4" ]]; then
 160             echo "Please install clang4.0 or latest for arm/armel cross build"; exit 1;
 161         fi
 162     fi
 163
 164     # Check for clang
 165     __ClangCombinedDottedVersion=$__ClangMajorVersion;
 166     if [[ "$__ClangMinorVersion" != "" ]]; then
 167         __ClangCombinedDottedVersion=$__ClangCombinedDottedVersion.$__ClangMinorVersion
 168     fi
 169     hash clang-$__ClangCombinedDottedVersion 2>/dev/null ||  hash clang$__ClangMajorVersion$__ClangMinorVersion 2>/dev/null || hash clang 2>/dev/null || { echo >&2 "Please install clang-$__ClangMajorVersion.$__ClangMinorVersion before running this script"; exit 1; }
 170
 171 }
 172
 173 restore_optdata()
 174 {
 175     # we only need optdata on a Release build
 176     if [[ "$__BuildType" != "Release" ]]; then __SkipRestoreOptData=1; fi
 177
 178     if [[ ( $__SkipRestoreOptData == 0 ) && ( $__isMSBuildOnNETCoreSupported == 1 ) ]]; then
 179         echo "Restoring the OptimizationData package"
 180         "$__ProjectRoot/run.sh" build -optdata $__RunArgs $__UnprocessedBuildArgs
 181         if [ $? != 0 ]; then
 182             echo "Failed to restore the optimization data package."
 183             exit 1
 184         fi
 185     fi
 186
 187     if [ $__isMSBuildOnNETCoreSupported == 1 ]; then
 188         # Parse the optdata package versions out of msbuild so that we can pass them on to CMake
 189         local DotNetCli="$__ProjectRoot/Tools/dotnetcli/dotnet"
 190         if [ ! -f $DotNetCli ]; then
 191             source "$__ProjectRoot/init-tools.sh"
 192             if [ $? != 0 ]; then
 193                 echo "Failed to restore buildtools."
 194                 exit 1
 195             fi
 196         fi
 197         local OptDataProjectFilePath="$__ProjectRoot/src/.nuget/optdata/optdata.csproj"
 198         __PgoOptDataVersion=$(DOTNET_SKIP_FIRST_TIME_EXPERIENCE=1 $DotNetCli msbuild $OptDataProjectFilePath /t:DumpPgoDataPackageVersion /nologo | sed 's/^\s*//')
 199         __IbcOptDataVersion=$(DOTNET_SKIP_FIRST_TIME_EXPERIENCE=1 $DotNetCli msbuild $OptDataProjectFilePath /t:DumpIbcDataPackageVersion /nologo | sed 's/^[[:blank:]]*//')
 200     fi
 201 }
 202
 203 generate_event_logging_sources()
 204 {
 205     __OutputDir=$1
 206     __ConsumingBuildSystem=$2
 207
 208     __OutputIncDir="$__OutputDir/src/inc"
 209     __OutputEventingDir="$__OutputDir/eventing"
 210     __OutputEventProviderDir="$__OutputEventingDir/eventprovider"
 211
 212     echo "Laying out dynamically generated files consumed by $__ConsumingBuildSystem"
 213     echo "Laying out dynamically generated Event test files, etmdummy stub functions, and external linkages"
 214
 215     __PythonWarningFlags="-Wall"
 216     if [[ $__IgnoreWarnings == 0 ]]; then
 217         __PythonWarningFlags="$__PythonWarningFlags -Werror"
 218     fi
 219
 220     $PYTHON -B $__PythonWarningFlags "$__ProjectRoot/src/scripts/genEventing.py" --inc $__OutputIncDir --dummy $__OutputIncDir/etmdummy.h --man "$__ProjectRoot/src/vm/ClrEtwAll.man" --testdir "$__OutputEventProviderDir/tests"
 221     if [[ $? != 0 ]]; then
 222         exit 1
 223     fi
 224
 225     echo "Laying out dynamically generated EventPipe Implementation"
 226     $PYTHON -B $__PythonWarningFlags "$__ProjectRoot/src/scripts/genEventPipe.py" --man "$__ProjectRoot/src/vm/ClrEtwAll.man" --intermediate "$__OutputEventingDir/eventpipe"
 227
 228     echo "Laying out dynamically generated EventSource classes"
 229     $PYTHON -B $__PythonWarningFlags "$__ProjectRoot/src/scripts/genRuntimeEventSources.py" --man "$__ProjectRoot/src/vm/ClrEtwAll.man" --intermediate "$__OutputEventingDir"
 230
 231     # determine the logging system
 232     case $__BuildOS in
 233         Linux|FreeBSD)
 234             echo "Laying out dynamically generated Event Logging Implementation of Lttng"
 235             $PYTHON -B $__PythonWarningFlags "$__ProjectRoot/src/scripts/genLttngProvider.py" --man "$__ProjectRoot/src/vm/ClrEtwAll.man" --intermediate "$__OutputEventProviderDir"
 236             if [[ $? != 0 ]]; then
 237                 exit 1
 238             fi
 239             ;;
 240         *)
 241             echo "Laying out dummy event logging provider"
 242             $PYTHON -B $__PythonWarningFlags "$__ProjectRoot/src/scripts/genDummyProvider.py" --man "$__ProjectRoot/src/vm/ClrEtwAll.man" --intermediate "$__OutputEventProviderDir"
 243             if [[ $? != 0 ]]; then
 244                 exit 1
 245             fi
 246             ;;
 247     esac
 248 }
 249
 250 generate_event_logging()
 251 {
 252     # Event Logging Infrastructure
 253     if [[ $__SkipCoreCLR == 0 || $__SkipMSCorLib == 0 || $__ConfigureOnly == 1 ]]; then
 254         generate_event_logging_sources "$__IntermediatesDir" "the native build system"
 255     fi
 256 }
 257
 258 build_native()
 259 {
 260     skipCondition=$1
 261     platformArch="$2"
 262     intermediatesForBuild="$3"
 263     extraCmakeArguments="$4"
 264     message="$5"
 265
 266     if [ $skipCondition == 1 ]; then
 267         echo "Skipping $message build."
 268         return
 269     fi
 270
 271     # All set to commence the build
 272     echo "Commencing build of $message for $__BuildOS.$__BuildArch.$__BuildType in $intermediatesForBuild"
 273
 274     generator=""
 275     buildFile="Makefile"
 276     buildTool="make"
 277     if [ $__UseNinja == 1 ]; then
 278         generator="ninja"
 279         buildFile="build.ninja"
 280         if ! buildTool=$(command -v ninja || command -v ninja-build); then
 281            echo "Unable to locate ninja!" 1>&2
 282            exit 1
 283         fi
 284     fi
 285
 286     if [ $__SkipConfigure == 0 ]; then
 287         # if msbuild is not supported, then set __SkipGenerateVersion to 1
 288         if [ $__isMSBuildOnNETCoreSupported == 0 ]; then __SkipGenerateVersion=1; fi
 289         # Drop version.cpp file
 290         __versionSourceFile="$intermediatesForBuild/version.cpp"
 291         if [ $__SkipGenerateVersion == 0 ]; then
 292             pwd
 293             "$__ProjectRoot/run.sh" build -Project=$__ProjectDir/build.proj -generateHeaderUnix -NativeVersionSourceFile=$__versionSourceFile -MsBuildEventLogging="/l:BinClashLogger,Tools/Microsoft.DotNet.Build.Tasks.dll;LogFile=binclash.log"  $__RunArgs $__UnprocessedBuildArgs
 294         else
 295             # Generate the dummy version.cpp, but only if it didn't exist to make sure we don't trigger unnecessary rebuild
 296             __versionSourceLine="static char sccsid[] __attribute__((used)) = \"@(#)No version information produced\";"
 297             if [ -e $__versionSourceFile ]; then
 298                 read existingVersionSourceLine < $__versionSourceFile
 299             fi
 300             if [ "$__versionSourceLine" != "$existingVersionSourceLine" ]; then
 301                 echo $__versionSourceLine > $__versionSourceFile
 302             fi
 303         fi
 304
 305
 306         pushd "$intermediatesForBuild"
 307         # Regenerate the CMake solution
 308         echo "Invoking \"$__ProjectRoot/src/pal/tools/gen-buildsys-clang.sh\" \"$__ProjectRoot\" $__ClangMajorVersion \"$__ClangMinorVersion\" $platformArch $__BuildType $__CodeCoverage $generator $extraCmakeArguments $__cmakeargs"
 309         "$__ProjectRoot/src/pal/tools/gen-buildsys-clang.sh" "$__ProjectRoot" $__ClangMajorVersion "$__ClangMinorVersion" $platformArch $__BuildType $__CodeCoverage $generator "$extraCmakeArguments" "$__cmakeargs"
 310         popd
 311     fi
 312
 313     if [ ! -f "$intermediatesForBuild/$buildFile" ]; then
 314         echo "Failed to generate $message build project!"
 315         exit 1
 316     fi
 317
 318     # Build
 319     if [ $__ConfigureOnly == 1 ]; then
 320         echo "Finish configuration & skipping $message build."
 321         return
 322     fi
 323
 324     # Check that the makefiles were created.
 325     pushd "$intermediatesForBuild"
 326
 327     echo "Executing $buildTool install -j $__NumProc"
 328
 329     $buildTool install -j $__NumProc
 330     if [ $? != 0 ]; then
 331         echo "Failed to build $message."
 332         exit 1
 333     fi
 334
 335     popd
 336 }
 337
 338 build_cross_architecture_components()
 339 {
 340     local crossArch="$1"
 341
 342     local intermediatesForBuild="$__IntermediatesDir/Host$crossArch/crossgen"
 343     local crossArchBinDir="$__BinDir/$crossArch"
 344
 345     mkdir -p "$intermediatesForBuild"
 346     mkdir -p "$crossArchBinDir"
 347
 348     generate_event_logging_sources "$intermediatesForBuild" "the crossarch build system"
 349
 350     __SkipCrossArchBuild=1
 351     TARGET_ROOTFS=""
 352     # check supported cross-architecture components host(__HostArch)/target(__BuildArch) pair
 353     if [[ ("$__BuildArch" == "arm" || "$__BuildArch" == "armel") && "$crossArch" == "x86" ]]; then
 354         export CROSSCOMPILE=0
 355         __SkipCrossArchBuild=0
 356
 357         # building x64-host/arm-target cross-architecture component need to use cross toolchain of x86
 358         if [ "$__HostArch" == "x64" ]; then
 359             export CROSSCOMPILE=1
 360         fi
 361     elif [[ ("$__BuildArch" == "arm64") && "$crossArch" == "x64" ]]; then
 362         export CROSSCOMPILE=0
 363         __SkipCrossArchBuild=0
 364     elif [[ ("$__BuildArch" == "arm" || "$__BuildArch" == "armel") && "$crossArch" == "x64" ]]; then
 365         export CROSSCOMPILE=0
 366         __SkipCrossArchBuild=0
 367     else
 368         # not supported
 369         return
 370     fi
 371
 372     export __CMakeBinDir="$crossArchBinDir"
 373     export CROSSCOMPONENT=1
 374
 375     if [ $CROSSCOMPILE == 1 ]; then
 376         TARGET_ROOTFS="$ROOTFS_DIR"
 377         if [ -n "$CAC_ROOTFS_DIR" ]; then
 378             export ROOTFS_DIR="$CAC_ROOTFS_DIR"
 379         else
 380             export ROOTFS_DIR="$__ProjectRoot/cross/rootfs/$__CrossArch"
 381         fi
 382     fi
 383
 384     __ExtraCmakeArgs="-DCLR_CMAKE_TARGET_ARCH=$__BuildArch -DCLR_CMAKE_TARGET_OS=$__BuildOS -DCLR_CMAKE_PACKAGES_DIR=$__PackagesDir -DCLR_CMAKE_PGO_INSTRUMENT=$__PgoInstrument -DCLR_CMAKE_OPTDATA_VERSION=$__PgoOptDataVersion -DCLR_CMAKE_PGO_OPTIMIZE=$__PgoOptimize -DCLR_CROSS_COMPONENTS_BUILD=1"
 385     build_native $__SkipCrossArchBuild "$crossArch" "$intermediatesForBuild" "$__ExtraCmakeArgs" "cross-architecture components"
 386
 387     # restore ROOTFS_DIR and CROSSCOMPILE
 388     if [ -n "$TARGET_ROOTFS" ]; then
 389         export ROOTFS_DIR="$TARGET_ROOTFS"
 390     fi
 391     export CROSSCOMPILE=1
 392 }
 393
 394 isMSBuildOnNETCoreSupported()
 395 {
 396     __isMSBuildOnNETCoreSupported=$__msbuildonunsupportedplatform
 397
 398     if [ $__isMSBuildOnNETCoreSupported == 1 ]; then
 399         return
 400     fi
 401
 402     if [ $__SkipManaged == 1 ]; then
 403         __isMSBuildOnNETCoreSupported=0
 404         return
 405     fi
 406
 407     if [ "$__HostArch" == "x64" ]; then
 408         if [ "$__HostOS" == "Linux" ]; then
 409             __isMSBuildOnNETCoreSupported=1
 410             # note: the RIDs below can use globbing patterns
 411             UNSUPPORTED_RIDS=("debian.9-x64" "ubuntu.17.04-x64")
 412             for UNSUPPORTED_RID in "${UNSUPPORTED_RIDS[@]}"
 413             do
 414                 if [[ $__HostDistroRid == $UNSUPPORTED_RID ]]; then
 415                     __isMSBuildOnNETCoreSupported=0
 416                     break
 417                 fi
 418             done
 419         elif [ "$__HostOS" == "OSX" ]; then
 420             __isMSBuildOnNETCoreSupported=1
 421         elif [ "$__HostOS" == "FreeBSD" ]; then
 422             __isMSBuildOnNETCoreSupported=1
 423         fi
 424     fi
 425 }
 426
 427
 428 build_CoreLib_ni()
 429 {
 430     local __CrossGenExec=$1
 431
 432     if [ $__PartialNgen == 1 ]; then
 433         export COMPlus_PartialNGen=1
 434     fi
 435
 436     if [ -e $__CrossGenCoreLibLog ]; then
 437         rm $__CrossGenCoreLibLog
 438     fi
 439     echo "Generating native image of System.Private.CoreLib.dll for $__BuildOS.$__BuildArch.$__BuildType. Logging to \"$__CrossGenCoreLibLog\"."
 440     echo "$__CrossGenExec /Platform_Assemblies_Paths $__BinDir/IL $__IbcTuning /out $__BinDir/System.Private.CoreLib.dll $__BinDir/IL/System.Private.CoreLib.dll"
 441     $__CrossGenExec /Platform_Assemblies_Paths $__BinDir/IL $__IbcTuning /out $__BinDir/System.Private.CoreLib.dll $__BinDir/IL/System.Private.CoreLib.dll >> $__CrossGenCoreLibLog 2>&1
 442     if [ $? -ne 0 ]; then
 443         echo "Failed to generate native image for System.Private.CoreLib. Refer to $__CrossGenCoreLibLog"
 444         exit 1
 445     fi
 446
 447     if [ "$__BuildOS" == "Linux" ]; then
 448         echo "Generating symbol file for System.Private.CoreLib.dll"
 449         echo "$__CrossGenExec /Platform_Assemblies_Paths $__BinDir /CreatePerfMap $__BinDir $__BinDir/System.Private.CoreLib.dll"
 450         $__CrossGenExec /Platform_Assemblies_Paths $__BinDir /CreatePerfMap $__BinDir $__BinDir/System.Private.CoreLib.dll >> $__CrossGenCoreLibLog 2>&1
 451         if [ $? -ne 0 ]; then
 452             echo "Failed to generate symbol file for System.Private.CoreLib. Refer to $__CrossGenCoreLibLog"
 453             exit 1
 454         fi
 455     fi
 456 }
 457
 458 build_CoreLib()
 459 {
 460     if [ $__isMSBuildOnNETCoreSupported == 0 ]; then
 461         echo "System.Private.CoreLib.dll build unsupported."
 462         return
 463     fi
 464
 465     if [ $__SkipMSCorLib == 1 ]; then
 466        echo "Skipping building System.Private.CoreLib."
 467        return
 468     fi
 469
 470     echo "Commencing build of managed components for $__BuildOS.$__BuildArch.$__BuildType"
 471
 472     # Invoke MSBuild
 473     __ExtraBuildArgs=""
 474     if [[ "$__IbcTuning" == "" ]]; then
 475         __ExtraBuildArgs="$__ExtraBuildArgs -OptimizationDataDir=\"$__PackagesDir/optimization.$__BuildOS-$__BuildArch.IBC.CoreCLR/$__IbcOptDataVersion/data/\""
 476         __ExtraBuildArgs="$__ExtraBuildArgs -EnableProfileGuidedOptimization=true"
 477     fi
 478
 479     if [[ "$__BuildManagedTools" -eq "1" ]]; then
 480         __ExtraBuildArgs="$__ExtraBuildArgs -BuildManagedTools=true"
 481     fi
 482
 483     $__ProjectRoot/run.sh build -Project=$__ProjectDir/build.proj -MsBuildEventLogging="/l:BinClashLogger,Tools/Microsoft.DotNet.Build.Tasks.dll;LogFile=binclash.log" -MsBuildLog="/flp:Verbosity=normal;LogFile=$__LogsDir/System.Private.CoreLib_$__BuildOS__$__BuildArch__$__BuildType.log" -BuildTarget -__IntermediatesDir=$__IntermediatesDir -__RootBinDir=$__RootBinDir -BuildNugetPackage=false -UseSharedCompilation=false $__RunArgs $__ExtraBuildArgs $__UnprocessedBuildArgs
 484
 485     if [ $? -ne 0 ]; then
 486         echo "Failed to build managed components."
 487         exit 1
 488     fi
 489
 490     if [ $__SkipCrossgen == 1 ]; then
 491         echo "Skipping generating native image"
 492         return
 493     fi
 494
 495     # The cross build generates a crossgen with the target architecture.
 496     if [ $__CrossBuild == 0 ]; then
 497        if [ $__SkipCoreCLR == 1 ]; then
 498            return
 499        fi
 500
 501        # The architecture of host pc must be same architecture with target.
 502        if [[ ( "$__HostArch" == "$__BuildArch" ) ]]; then
 503            build_CoreLib_ni "$__BinDir/crossgen"
 504        elif [[ ( "$__HostArch" == "x64" ) && ( "$__BuildArch" == "x86" ) ]]; then
 505            build_CoreLib_ni "$__BinDir/crossgen"
 506        elif [[ ( "$__HostArch" == "arm64" ) && ( "$__BuildArch" == "arm" ) ]]; then
 507            build_CoreLib_ni "$__BinDir/crossgen"
 508        else
 509            exit 1
 510        fi
 511     else
 512        if [[ ( "$__CrossArch" == "x86" ) && ( "$__BuildArch" == "arm" ) ]]; then
 513            build_CoreLib_ni "$__CrossComponentBinDir/crossgen"
 514        elif [[ ( "$__CrossArch" == "x64" ) && ( "$__BuildArch" == "arm" ) ]]; then
 515            build_CoreLib_ni "$__CrossComponentBinDir/crossgen"
 516        elif [[ ( "$__HostArch" == "x64" ) && ( "$__BuildArch" == "arm64" ) ]]; then
 517            build_CoreLib_ni "$__CrossComponentBinDir/crossgen"
 518        fi
 519     fi
 520 }
 521
 522 generate_NugetPackages()
 523 {
 524     # We can only generate nuget package if we also support building mscorlib as part of this build.
 525     if [ $__isMSBuildOnNETCoreSupported == 0 ]; then
 526         echo "Nuget package generation unsupported."
 527         return
 528     fi
 529
 530     # Since we can build mscorlib for this OS, did we build the native components as well?
 531     if [[ $__SkipCoreCLR == 1 && $__CrossgenOnly == 0 ]]; then
 532         echo "Unable to generate nuget packages since native components were not built."
 533         return
 534     fi
 535
 536     echo "Generating nuget packages for "$__BuildOS
 537     echo "DistroRid is "$__DistroRid
 538     echo "ROOTFS_DIR is "$ROOTFS_DIR
 539     # Build the packages
 540     $__ProjectRoot/run.sh build -Project=$__SourceDir/.nuget/packages.builds -MsBuildEventLogging="/l:BinClashLogger,Tools/Microsoft.DotNet.Build.Tasks.dll;LogFile=binclash.log" -MsBuildLog="/flp:Verbosity=normal;LogFile=$__LogsDir/Nuget_$__BuildOS__$__BuildArch__$__BuildType.log" -BuildTarget -__IntermediatesDir=$__IntermediatesDir -__RootBinDir=$__RootBinDir -BuildNugetPackage=false -UseSharedCompilation=false -__DoCrossArchBuild=$__CrossBuild $__RunArgs $__UnprocessedBuildArgs
 541
 542     if [ $? -ne 0 ]; then
 543         echo "Failed to generate Nuget packages."
 544         exit 1
 545     fi
 546 }
 547
 548 echo "Commencing CoreCLR Repo build"
 549
 550 # Argument types supported by this script:
 551 #
 552 # Build architecture - valid values are: x64, ARM.
 553 # Build Type         - valid values are: Debug, Checked, Release
 554 #
 555 # Set the default arguments for build
 556
 557 # Obtain the location of the bash script to figure out where the root of the repo is.
 558 __ProjectRoot="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
 559
 560 # Use uname to determine what the CPU is.
 561 CPUName=$(uname -p)
 562 # Some Linux platforms report unknown for platform, but the arch for machine.
 563 if [ "$CPUName" == "unknown" ]; then
 564     CPUName=$(uname -m)
 565 fi
 566
 567 case $CPUName in
 568     i686)
 569         echo "Unsupported CPU $CPUName detected, build might not succeed!"
 570         __BuildArch=x86
 571         __HostArch=x86
 572         ;;
 573
 574     x86_64)
 575         __BuildArch=x64
 576         __HostArch=x64
 577         ;;
 578
 579     armv7l)
 580         echo "Unsupported CPU $CPUName detected, build might not succeed!"
 581         __BuildArch=arm
 582         __HostArch=arm
 583         ;;
 584
 585     aarch64)
 586         __BuildArch=arm64
 587         __HostArch=arm64
 588         ;;
 589
 590     amd64)
 591         __BuildArch=x64
 592         __HostArch=x64
 593         ;;
 594     *)
 595         echo "Unknown CPU $CPUName detected, configuring as if for x64"
 596         __BuildArch=x64
 597         __HostArch=x64
 598         ;;
 599 esac
 600
 601 # Use uname to determine what the OS is.
 602 OSName=$(uname -s)
 603 case $OSName in
 604     Linux)
 605         __BuildOS=Linux
 606         __HostOS=Linux
 607         ;;
 608
 609     Darwin)
 610         __BuildOS=OSX
 611         __HostOS=OSX
 612         ;;
 613
 614     FreeBSD)
 615         __BuildOS=FreeBSD
 616         __HostOS=FreeBSD
 617         ;;
 618
 619     OpenBSD)
 620         __BuildOS=OpenBSD
 621         __HostOS=OpenBSD
 622         ;;
 623
 624     NetBSD)
 625         __BuildOS=NetBSD
 626         __HostOS=NetBSD
 627         ;;
 628
 629     SunOS)
 630         __BuildOS=SunOS
 631         __HostOS=SunOS
 632         ;;
 633
 634     *)
 635         echo "Unsupported OS $OSName detected, configuring as if for Linux"
 636         __BuildOS=Linux
 637         __HostOS=Linux
 638         ;;
 639 esac
 640
 641 __BuildType=Debug
 642 __CodeCoverage=
 643 __IgnoreWarnings=0
 644
 645 # Set the various build properties here so that CMake and MSBuild can pick them up
 646 __ProjectDir="$__ProjectRoot"
 647 __SourceDir="$__ProjectDir/src"
 648 __PackagesDir="${DotNetRestorePackagesPath:-${__ProjectDir}/packages}"
 649 __RootBinDir="$__ProjectDir/bin"
 650 __UnprocessedBuildArgs=
 651 __RunArgs=
 652 __MSBCleanBuildArgs=
 653 __UseNinja=0
 654 __VerboseBuild=0
 655 __PgoInstrument=0
 656 __PgoOptimize=1
 657 __IbcTuning=""
 658 __ConfigureOnly=0
 659 __SkipConfigure=0
 660 __SkipManaged=0
 661 __SkipRestore=""
 662 __SkipNuget=0
 663 __SkipCoreCLR=0
 664 __SkipMSCorLib=0
 665 __SkipRestoreOptData=0
 666 __SkipCrossgen=0
 667 __CrossgenOnly=0
 668 __PartialNgen=0
 669 __SkipTests=0
 670 __CrossBuild=0
 671 __ClangMajorVersion=0
 672 __ClangMinorVersion=0
 673 __NuGetPath="$__PackagesDir/NuGet.exe"
 674 __HostDistroRid=""
 675 __DistroRid=""
 676 __cmakeargs=""
 677 __SkipGenerateVersion=0
 678 __PortableBuild=1
 679 __msbuildonunsupportedplatform=0
 680 __PgoOptDataVersion=""
 681 __IbcOptDataVersion=""
 682 __BuildManagedTools=1
 683
 684 # Get the number of processors available to the scheduler
 685 # Other techniques such as `nproc` only get the number of
 686 # processors available to a single process.
 687 if [ `uname` = "FreeBSD" ]; then
 688   __NumProc=`sysctl hw.ncpu | awk '{ print $2+1 }'`
 689 elif [ `uname` = "NetBSD" ]; then
 690   __NumProc=$(($(getconf NPROCESSORS_ONLN)+1))
 691 elif [ `uname` = "Darwin" ]; then
 692   __NumProc=$(($(getconf _NPROCESSORS_ONLN)+1))
 693 else
 694   __NumProc=$(nproc --all)
 695 fi
 696
 697 while :; do
 698     if [ $# -le 0 ]; then
 699         break
 700     fi
 701
 702     lowerI="$(echo $1 | awk '{print tolower($0)}')"
 703     case $lowerI in
 704         -\?|-h|--help)
 705             usage
 706             exit 1
 707             ;;
 708
 709         x86|-x86)
 710             __BuildArch=x86
 711             ;;
 712
 713         x64|-x64)
 714             __BuildArch=x64
 715             ;;
 716
 717         arm|-arm)
 718             __BuildArch=arm
 719             ;;
 720
 721         armel|-armel)
 722             __BuildArch=armel
 723             ;;
 724
 725         arm64|-arm64)
 726             __BuildArch=arm64
 727             ;;
 728
 729         debug|-debug)
 730             __BuildType=Debug
 731             ;;
 732
 733         checked|-checked)
 734             __BuildType=Checked
 735             ;;
 736
 737         release|-release)
 738             __BuildType=Release
 739             ;;
 740
 741         coverage|-coverage)
 742             __CodeCoverage=Coverage
 743             ;;
 744
 745         cross|-cross)
 746             __CrossBuild=1
 747             ;;
 748
 749         -portablebuild=false)
 750             __PortableBuild=0
 751             ;;
 752
 753         verbose|-verbose)
 754             __VerboseBuild=1
 755             ;;
 756
 757         stripsymbols|-stripsymbols)
 758             __cmakeargs="$__cmakeargs -DSTRIP_SYMBOLS=true"
 759             ;;
 760
 761         clang3.5|-clang3.5)
 762             __ClangMajorVersion=3
 763             __ClangMinorVersion=5
 764             ;;
 765
 766         clang3.6|-clang3.6)
 767             __ClangMajorVersion=3
 768             __ClangMinorVersion=6
 769             ;;
 770
 771         clang3.7|-clang3.7)
 772             __ClangMajorVersion=3
 773             __ClangMinorVersion=7
 774             ;;
 775
 776         clang3.8|-clang3.8)
 777             __ClangMajorVersion=3
 778             __ClangMinorVersion=8
 779             ;;
 780
 781         clang3.9|-clang3.9)
 782             __ClangMajorVersion=3
 783             __ClangMinorVersion=9
 784             ;;
 785
 786         clang4.0|-clang4.0)
 787             __ClangMajorVersion=4
 788             __ClangMinorVersion=0
 789             ;;
 790
 791         clang5.0|-clang5.0)
 792             __ClangMajorVersion=5
 793             __ClangMinorVersion=0
 794             ;;
 795
 796         clang6.0|-clang6.0)
 797             __ClangMajorVersion=6
 798             __ClangMinorVersion=0
 799             ;;
 800
 801         clang7|-clang7)
 802             __ClangMajorVersion=7
 803             __ClangMinorVersion=
 804             ;;
 805
 806         ninja|-ninja)
 807             __UseNinja=1
 808             ;;
 809
 810         pgoinstrument|-pgoinstrument)
 811             __PgoInstrument=1
 812             ;;
 813
 814         nopgooptimize|-nopgooptimize)
 815             __PgoOptimize=0
 816             __SkipRestoreOptData=1
 817             ;;
 818
 819         ibcinstrument|-ibcinstrument)
 820             __IbcTuning="/Tuning"
 821             ;;
 822
 823         configureonly|-configureonly)
 824             __ConfigureOnly=1
 825             __SkipMSCorLib=1
 826             __SkipNuget=1
 827             ;;
 828
 829         skipconfigure|-skipconfigure)
 830             __SkipConfigure=1
 831             ;;
 832
 833         skipnative|-skipnative)
 834             # Use "skipnative" to use the same option name as build.cmd.
 835             __SkipCoreCLR=1
 836             ;;
 837
 838         skipcoreclr|-skipcoreclr)
 839             # Accept "skipcoreclr" for backwards-compatibility.
 840             __SkipCoreCLR=1
 841             ;;
 842
 843         crosscomponent|-crosscomponent)
 844             # Accept "crosscomponent" for backward-compatibility but ignore it.
 845             echo "WARNING: 'crosscomponent' is obsolete and should not be used"
 846             ;;
 847
 848         skipmanaged|-skipmanaged)
 849             __SkipManaged=1
 850             ;;
 851
 852         skipmscorlib|-skipmscorlib)
 853             __SkipMSCorLib=1
 854             ;;
 855
 856         skipgenerateversion|-skipgenerateversion)
 857             __SkipGenerateVersion=1
 858             ;;
 859
 860         skiprestoreoptdata|-skiprestoreoptdata)
 861             __SkipRestoreOptData=1
 862             ;;
 863
 864         skipcrossgen|-skipcrossgen)
 865             __SkipCrossgen=1
 866             ;;
 867
 868         crossgenonly|-crossgenonly)
 869             __SkipMSCorLib=1
 870             __SkipCoreCLR=1
 871             __CrossgenOnly=1
 872             ;;
 873         partialngen|-partialngen)
 874             __PartialNgen=1
 875             ;;
 876
 877         skiptests|-skiptests)
 878             __SkipTests=1
 879             ;;
 880
 881         skipnuget|-skipnuget|skipbuildpackages|-skipbuildpackages)
 882             __SkipNuget=1
 883             ;;
 884
 885         ignorewarnings|-ignorewarnings)
 886             __IgnoreWarnings=1
 887             __cmakeargs="$__cmakeargs -DCLR_CMAKE_WARNINGS_ARE_ERRORS=OFF"
 888             ;;
 889
 890         cmakeargs|-cmakeargs)
 891             if [ -n "$2" ]; then
 892                 __cmakeargs="$__cmakeargs $2"
 893                 shift
 894             else
 895                 echo "ERROR: 'cmakeargs' requires a non-empty option argument"
 896                 exit 1
 897             fi
 898             ;;
 899
 900         bindir|-bindir)
 901             if [ -n "$2" ]; then
 902                 __RootBinDir="$2"
 903                 if [ ! -d $__RootBinDir ]; then
 904                     mkdir $__RootBinDir
 905                 fi
 906                 __RootBinParent=$(dirname $__RootBinDir)
 907                 __RootBinName=${__RootBinDir##*/}
 908                 __RootBinDir="$(cd $__RootBinParent &>/dev/null && printf %s/%s $PWD $__RootBinName)"
 909                 shift
 910             else
 911                 echo "ERROR: 'bindir' requires a non-empty option argument"
 912                 exit 1
 913             fi
 914             ;;
 915         msbuildonunsupportedplatform|-msbuildonunsupportedplatform)
 916             __msbuildonunsupportedplatform=1
 917             ;;
 918         numproc|-numproc)
 919             if [ -n "$2" ]; then
 920               __NumProc="$2"
 921               shift
 922             else
 923               echo "ERROR: 'numproc' requires a non-empty option argument"
 924               exit 1
 925             fi
 926             ;;
 927         osgroup|-osgroup)
 928             if [ -n "$2" ]; then
 929               __BuildOS="$2"
 930               shift
 931             else
 932               echo "ERROR: 'osgroup' requires a non-empty option argument"
 933               exit 1
 934             fi
 935             ;;
 936         rebuild|-rebuild)
 937             echo "ERROR: 'Rebuild' is not supported.  Please remove it."
 938             exit 1
 939             ;;
 940
 941         *)
 942             __UnprocessedBuildArgs="$__UnprocessedBuildArgs $1"
 943             ;;
 944     esac
 945
 946     shift
 947 done
 948
 949 __RunArgs="-BuildArch=$__BuildArch -BuildType=$__BuildType -BuildOS=$__BuildOS"
 950
 951 # Configure environment if we are doing a verbose build
 952 if [ $__VerboseBuild == 1 ]; then
 953     export VERBOSE=1
 954         __RunArgs="$__RunArgs -verbose"
 955 fi
 956
 957 # Set default clang version
 958 if [[ $__ClangMajorVersion == 0 && $__ClangMinorVersion == 0 ]]; then
 959         if [[ "$__BuildArch" == "arm" || "$__BuildArch" == "armel" ]]; then
 960                 __ClangMajorVersion=5
 961                 __ClangMinorVersion=0
 962         else
 963                 __ClangMajorVersion=3
 964                 __ClangMinorVersion=9
 965         fi
 966 fi
 967
 968 if [[ "$__BuildArch" == "armel" ]]; then
 969     # Armel cross build is Tizen specific and does not support Portable RID build
 970     __PortableBuild=0
 971 fi
 972
 973 if [ $__PortableBuild == 0 ]; then
 974         __RunArgs="$__RunArgs -PortableBuild=false"
 975 fi
 976
 977 # Set dependent variables
 978 __LogsDir="$__RootBinDir/Logs"
 979
 980 # init the host distro name
 981 initHostDistroRid
 982
 983 # Set the remaining variables based upon the determined build configuration
 984 __BinDir="$__RootBinDir/Product/$__BuildOS.$__BuildArch.$__BuildType"
 985 __PackagesBinDir="$__BinDir/.nuget"
 986 __ToolsDir="$__RootBinDir/tools"
 987 __TestWorkingDir="$__RootBinDir/tests/$__BuildOS.$__BuildArch.$__BuildType"
 988 export __IntermediatesDir="$__RootBinDir/obj/$__BuildOS.$__BuildArch.$__BuildType"
 989 __TestIntermediatesDir="$__RootBinDir/tests/obj/$__BuildOS.$__BuildArch.$__BuildType"
 990 __isMSBuildOnNETCoreSupported=0
 991 __CrossComponentBinDir="$__BinDir"
 992
 993 __CrossArch="$__HostArch"
 994 if [ $__CrossBuild == 1 ]; then
 995     __CrossComponentBinDir="$__CrossComponentBinDir/$__CrossArch"
 996 fi
 997 __CrossGenCoreLibLog="$__LogsDir/CrossgenCoreLib_$__BuildOS.$__BuildArch.$__BuildType.log"
 998
 999 # Init if MSBuild for .NET Core is supported for this platform
1000 isMSBuildOnNETCoreSupported
1001
1002 # CI_SPECIFIC - On CI machines, $HOME may not be set. In such a case, create a subfolder and set the variable to set.
1003 # This is needed by CLI to function.
1004 if [ -z "$HOME" ]; then
1005     if [ ! -d "$__ProjectDir/temp_home" ]; then
1006         mkdir temp_home
1007     fi
1008     export HOME=$__ProjectDir/temp_home
1009     echo "HOME not defined; setting it to $HOME"
1010 fi
1011
1012 # Specify path to be set for CMAKE_INSTALL_PREFIX.
1013 # This is where all built CoreClr libraries will copied to.
1014 export __CMakeBinDir="$__BinDir"
1015
1016 # Configure environment if we are doing a cross compile.
1017 if [ $__CrossBuild == 1 ]; then
1018     export CROSSCOMPILE=1
1019     if ! [[ -n "$ROOTFS_DIR" ]]; then
1020         export ROOTFS_DIR="$__ProjectRoot/cross/rootfs/$__BuildArch"
1021     fi
1022 fi
1023
1024 # init the target distro name
1025 initTargetDistroRid
1026
1027 # Make the directories necessary for build if they don't exist
1028 setup_dirs
1029
1030 # Check prereqs.
1031 check_prereqs
1032
1033 # Restore the package containing profile counts for profile-guided optimizations
1034 restore_optdata
1035
1036 # Generate event logging infrastructure sources
1037 generate_event_logging
1038
1039 # Build the coreclr (native) components.
1040 __ExtraCmakeArgs="-DCLR_CMAKE_TARGET_OS=$__BuildOS -DCLR_CMAKE_PACKAGES_DIR=$__PackagesDir -DCLR_CMAKE_PGO_INSTRUMENT=$__PgoInstrument -DCLR_CMAKE_OPTDATA_VERSION=$__PgoOptDataVersion -DCLR_CMAKE_PGO_OPTIMIZE=$__PgoOptimize"
1041
1042 # [TODO] Remove this when the `build-test.sh` script properly builds and deploys test assets.
1043 if [ $__SkipTests != 1 ]; then
1044     echo "Adding CMake flags to build native tests for $__BuildOS.$__BuildArch.$__BuildType"
1045     __ExtraCmakeArgs="$__ExtraCmakeArgs -DCLR_CMAKE_BUILD_TESTS=ON"
1046 fi
1047
1048 build_native $__SkipCoreCLR "$__BuildArch" "$__IntermediatesDir" "$__ExtraCmakeArgs" "CoreCLR component"
1049
1050 # Build cross-architecture components
1051 if [[ $__CrossBuild == 1 ]]; then
1052     build_cross_architecture_components "$__CrossArch"
1053
1054     # For now, continue building Hostx86/arm crossgen
1055     if [[ "$__HostArch" == "x64" && "$__BuildArch" == "arm" ]]; then
1056         build_cross_architecture_components "x86"
1057     fi
1058 fi
1059
1060 # Build System.Private.CoreLib.
1061
1062 build_CoreLib
1063
1064 if [ $__CrossgenOnly == 1 ]; then
1065     build_CoreLib_ni "$__BinDir/crossgen"
1066 fi
1067
1068 # Generate nuget packages
1069 if [ $__SkipNuget != 1 ]; then
1070     generate_NugetPackages
1071 fi
1072
1073
1074 # Build complete
1075
1076 echo "Repo successfully built."
1077 echo "Product binaries are available at $__BinDir"
1078 exit 0