build.sh

   1 #!/usr/bin/env bash
   2
   3 # Work around Jenkins CI + msbuild problem: Jenkins sometimes creates very large environment
   4 # variables, and msbuild can't handle environment blocks with such large variables. So clear
   5 # out the variables that might be too large.
   6 export ghprbCommentBody=
   7
   8 # resolve python-version to use
   9 if [ "$PYTHON" == "" ] ; then
  10     if ! PYTHON=$(command -v python3 || command -v python2 || command -v python)
  11     then
  12        echo "Unable to locate build-dependency python!" 1>&2
  13        exit 1
  14     fi
  15 fi
  16 # validate python-dependency
  17 # useful in case of explicitly set option.
  18 if ! command -v $PYTHON > /dev/null
  19 then
  20    echo "Unable to locate build-dependency python ($PYTHON)!" 1>&2
  21    exit 1
  22 fi
  23
  24 export PYTHON
  25
  26 usage()
  27 {
  28     echo "Usage: $0 [BuildArch] [BuildType] [-verbose] [-coverage] [-cross] [-clangx.y] [-ninja] [-configureonly] [-skipconfigure] [-skipnative] [-skipcrossarchnative] [-skipmanaged] [-skipmscorlib] [-skiptests] [-stripsymbols] [-ignorewarnings] [-cmakeargs] [-bindir]"
  29     echo "BuildArch can be: -x64, -x86, -arm, -armel, -arm64"
  30     echo "BuildType can be: -debug, -checked, -release"
  31     echo "-coverage - optional argument to enable code coverage build (currently supported only for Linux and OSX)."
  32     echo "-ninja - target ninja instead of GNU make"
  33     echo "-clangx.y - optional argument to build using clang version x.y."
  34     echo "-cross - optional argument to signify cross compilation,"
  35     echo "       - will use ROOTFS_DIR environment variable if set."
  36     echo "-nopgooptimize - do not use profile guided optimizations."
  37     echo "-pgoinstrument - generate instrumented code for profile guided optimization enabled binaries."
  38     echo "-ibcinstrument - generate IBC-tuning-enabled native images when invoking crossgen."
  39     echo "-configureonly - do not perform any builds; just configure the build."
  40     echo "-skipconfigure - skip build configuration."
  41     echo "-skipnative - do not build native components."
  42     echo "-skipcrossarchnative - do not build cross-architecture native components."
  43     echo "-skipmanaged - do not build managed components."
  44     echo "-skipmscorlib - do not build mscorlib.dll."
  45     echo "-skiptests - skip the tests in the 'tests' subdirectory."
  46     echo "-skipnuget - skip building nuget packages."
  47     echo "-skiprestoreoptdata - skip restoring optimization data used by profile-based optimizations."
  48     echo "-skipcrossgen - skip native image generation"
  49     echo "-crossgenonly - only run native image generation"
  50     echo "-partialngen - build CoreLib as PartialNGen"
  51     echo "-verbose - optional argument to enable verbose build output."
  52     echo "-skiprestore: skip restoring packages ^(default: packages are restored during build^)."
  53     echo "-disableoss: Disable Open Source Signing for System.Private.CoreLib."
  54     echo "-officialbuildid=^<ID^>: specify the official build ID to be used by this build."
  55     echo "-stripSymbols - Optional argument to strip native symbols during the build."
  56     echo "-skipgenerateversion - disable version generation even if MSBuild is supported."
  57     echo "-ignorewarnings - do not treat warnings as errors"
  58     echo "-cmakeargs - user-settable additional arguments passed to CMake."
  59     echo "-bindir - output directory (defaults to $__ProjectRoot/bin)"
  60     echo "-msbuildonunsupportedplatform - build managed binaries even if distro is not officially supported."
  61     echo "-numproc - set the number of build processes."
  62     exit 1
  63 }
  64
  65 initHostDistroRid()
  66 {
  67     __HostDistroRid=""
  68     if [ "$__HostOS" == "Linux" ]; then
  69         if [ -e /etc/os-release ]; then
  70             source /etc/os-release
  71             if [[ $ID == "rhel" ]]; then
  72                 # remove the last version digit
  73                 VERSION_ID=${VERSION_ID%.*}
  74             fi
  75             __HostDistroRid="$ID.$VERSION_ID-$__HostArch"
  76             if [[ $ID == "alpine" ]]; then
  77                 __HostDistroRid="linux-musl-$__HostArch"
  78             fi
  79         elif [ -e /etc/redhat-release ]; then
  80             local redhatRelease=$(</etc/redhat-release)
  81             if [[ $redhatRelease == "CentOS release 6."* || $redhatRelease == "Red Hat Enterprise Linux Server release 6."* ]]; then
  82                __HostDistroRid="rhel.6-$__HostArch"
  83             fi
  84             if [[ $redhatRelease == "CentOS Linux release 7."* ]]; then
  85                 __HostDistroRid="rhel.7-$__Arch"
  86             fi
  87         fi
  88     fi
  89     if [ "$__HostOS" == "FreeBSD" ]; then
  90         __freebsd_version=`sysctl -n kern.osrelease | cut -f1 -d'.'`
  91         __HostDistroRid="freebsd.$__freebsd_version-$__HostArch"
  92     fi
  93
  94     if [ "$__HostDistroRid" == "" ]; then
  95         echo "WARNING: Can not determine runtime id for current distro."
  96     fi
  97 }
  98
  99 initTargetDistroRid()
 100 {
 101     if [ $__CrossBuild == 1 ]; then
 102         if [ "$__BuildOS" == "Linux" ]; then
 103             if [ ! -e $ROOTFS_DIR/etc/os-release ]; then
 104                 if [ -e $ROOTFS_DIR/android_platform ]; then
 105                     source $ROOTFS_DIR/android_platform
 106                     export __DistroRid="$RID"
 107                 else
 108                     echo "WARNING: Can not determine runtime id for current distro."
 109                     export __DistroRid=""
 110                 fi
 111             else
 112                 source $ROOTFS_DIR/etc/os-release
 113                 export __DistroRid="$ID.$VERSION_ID-$__BuildArch"
 114             fi
 115         fi
 116     else
 117         export __DistroRid="$__HostDistroRid"
 118     fi
 119
 120     if [ "$__BuildOS" == "OSX" ]; then
 121         __PortableBuild=1
 122     fi
 123
 124     # Portable builds target the base RID
 125     if [ $__PortableBuild == 1 ]; then
 126         if [ "$__BuildOS" == "Linux" ]; then
 127             export __DistroRid="linux-$__BuildArch"
 128         elif [ "$__BuildOS" == "OSX" ]; then
 129             export __DistroRid="osx-$__BuildArch"
 130         elif [ "$__BuildOS" == "FreeBSD" ]; then
 131             export __DistroRid="freebsd-$__BuildArch"
 132         fi
 133     fi
 134 }
 135
 136 setup_dirs()
 137 {
 138     echo Setting up directories for build
 139
 140     mkdir -p "$__RootBinDir"
 141     mkdir -p "$__BinDir"
 142     mkdir -p "$__LogsDir"
 143     mkdir -p "$__IntermediatesDir"
 144
 145     if [ $__CrossBuild == 1 ]; then
 146         mkdir -p "$__CrossComponentBinDir"
 147     fi
 148 }
 149
 150 # Check the system to ensure the right prereqs are in place
 151
 152 check_prereqs()
 153 {
 154     echo "Checking prerequisites..."
 155
 156     # Check presence of CMake on the path
 157     hash cmake 2>/dev/null || { echo >&2 "Please install cmake before running this script"; exit 1; }
 158
 159
 160     # Minimum required version of clang is version 4.0 for arm/armel cross build
 161     if [[ $__CrossBuild == 1 && ("$__BuildArch" == "arm" || "$__BuildArch" == "armel") ]]; then
 162         if ! [[ "$__ClangMajorVersion" -ge "4" ]]; then
 163             echo "Please install clang4.0 or latest for arm/armel cross build"; exit 1;
 164         fi
 165     fi
 166
 167     # Check for clang
 168     __ClangCombinedDottedVersion=$__ClangMajorVersion;
 169     if [[ "$__ClangMinorVersion" != "" ]]; then
 170         __ClangCombinedDottedVersion=$__ClangCombinedDottedVersion.$__ClangMinorVersion
 171     fi
 172     hash clang-$__ClangCombinedDottedVersion 2>/dev/null ||  hash clang$__ClangMajorVersion$__ClangMinorVersion 2>/dev/null || hash clang 2>/dev/null || { echo >&2 "Please install clang-$__ClangMajorVersion.$__ClangMinorVersion before running this script"; exit 1; }
 173
 174 }
 175
 176 restore_optdata()
 177 {
 178     # we only need optdata on a Release build
 179     if [[ "$__BuildType" != "Release" ]]; then __SkipRestoreOptData=1; fi
 180
 181     if [[ ( $__SkipRestoreOptData == 0 ) && ( $__isMSBuildOnNETCoreSupported == 1 ) ]]; then
 182         echo "Restoring the OptimizationData package"
 183         "$__ProjectRoot/dotnet.sh" msbuild /nologo /verbosity:minimal /clp:Summary \
 184                                    /p:RestoreDefaultOptimizationDataPackage=false /p:PortableBuild=true \
 185                                    /p:UsePartialNGENOptimization=false /maxcpucount \
 186                                    /t:RestoreOptData ./build.proj \
 187                                    $__CommonMSBuildArgs $__UnprocessedBuildArgs
 188         if [ $? != 0 ]; then
 189             echo "Failed to restore the optimization data package."
 190             exit 1
 191         fi
 192     fi
 193
 194     if [ $__isMSBuildOnNETCoreSupported == 1 ]; then
 195         # Parse the optdata package versions out of msbuild so that we can pass them on to CMake
 196         local DotNetCli="$__ProjectRoot/Tools/dotnetcli/dotnet"
 197         if [ ! -f $DotNetCli ]; then
 198             source "$__ProjectRoot/init-tools.sh"
 199             if [ $? != 0 ]; then
 200                 echo "Failed to restore buildtools."
 201                 exit 1
 202             fi
 203         fi
 204         local OptDataProjectFilePath="$__ProjectRoot/src/.nuget/optdata/optdata.csproj"
 205         __PgoOptDataVersion=$(DOTNET_SKIP_FIRST_TIME_EXPERIENCE=1 $DotNetCli msbuild $OptDataProjectFilePath /t:DumpPgoDataPackageVersion /nologo | sed 's/^\s*//')
 206         __IbcOptDataVersion=$(DOTNET_SKIP_FIRST_TIME_EXPERIENCE=1 $DotNetCli msbuild $OptDataProjectFilePath /t:DumpIbcDataPackageVersion /nologo | sed 's/^[[:blank:]]*//')
 207     fi
 208 }
 209
 210 generate_event_logging_sources()
 211 {
 212     __OutputDir=$1
 213     __ConsumingBuildSystem=$2
 214
 215     __OutputIncDir="$__OutputDir/src/inc"
 216     __OutputEventingDir="$__OutputDir/eventing"
 217     __OutputEventProviderDir="$__OutputEventingDir/eventprovider"
 218
 219     echo "Laying out dynamically generated files consumed by $__ConsumingBuildSystem"
 220     echo "Laying out dynamically generated Event test files, etmdummy stub functions, and external linkages"
 221
 222     __PythonWarningFlags="-Wall"
 223     if [[ $__IgnoreWarnings == 0 ]]; then
 224         __PythonWarningFlags="$__PythonWarningFlags -Werror"
 225     fi
 226
 227     $PYTHON -B $__PythonWarningFlags "$__ProjectRoot/src/scripts/genEventing.py" --inc $__OutputIncDir --dummy $__OutputIncDir/etmdummy.h --man "$__ProjectRoot/src/vm/ClrEtwAll.man" --testdir "$__OutputEventProviderDir/tests"
 228     if [[ $? != 0 ]]; then
 229         exit 1
 230     fi
 231
 232     echo "Laying out dynamically generated EventPipe Implementation"
 233     $PYTHON -B $__PythonWarningFlags "$__ProjectRoot/src/scripts/genEventPipe.py" --man "$__ProjectRoot/src/vm/ClrEtwAll.man" --intermediate "$__OutputEventingDir/eventpipe"
 234
 235     echo "Laying out dynamically generated EventSource classes"
 236     $PYTHON -B $__PythonWarningFlags "$__ProjectRoot/src/scripts/genRuntimeEventSources.py" --man "$__ProjectRoot/src/vm/ClrEtwAll.man" --intermediate "$__OutputEventingDir"
 237
 238     # determine the logging system
 239     case $__BuildOS in
 240         Linux|FreeBSD)
 241             echo "Laying out dynamically generated Event Logging Implementation of Lttng"
 242             $PYTHON -B $__PythonWarningFlags "$__ProjectRoot/src/scripts/genLttngProvider.py" --man "$__ProjectRoot/src/vm/ClrEtwAll.man" --intermediate "$__OutputEventProviderDir"
 243             if [[ $? != 0 ]]; then
 244                 exit 1
 245             fi
 246             ;;
 247         *)
 248             echo "Laying out dummy event logging provider"
 249             $PYTHON -B $__PythonWarningFlags "$__ProjectRoot/src/scripts/genDummyProvider.py" --man "$__ProjectRoot/src/vm/ClrEtwAll.man" --intermediate "$__OutputEventProviderDir"
 250             if [[ $? != 0 ]]; then
 251                 exit 1
 252             fi
 253             ;;
 254     esac
 255 }
 256
 257 generate_event_logging()
 258 {
 259     # Event Logging Infrastructure
 260     if [[ $__SkipCoreCLR == 0 || $__SkipMSCorLib == 0 || $__ConfigureOnly == 1 ]]; then
 261         generate_event_logging_sources "$__IntermediatesDir" "the native build system"
 262     fi
 263 }
 264
 265 build_native()
 266 {
 267     skipCondition=$1
 268     platformArch="$2"
 269     intermediatesForBuild="$3"
 270     extraCmakeArguments="$4"
 271     message="$5"
 272
 273     if [ $skipCondition == 1 ]; then
 274         echo "Skipping $message build."
 275         return
 276     fi
 277
 278     # All set to commence the build
 279     echo "Commencing build of $message for $__BuildOS.$__BuildArch.$__BuildType in $intermediatesForBuild"
 280
 281     generator=""
 282     buildFile="Makefile"
 283     buildTool="make"
 284     if [ $__UseNinja == 1 ]; then
 285         generator="ninja"
 286         buildFile="build.ninja"
 287         if ! buildTool=$(command -v ninja || command -v ninja-build); then
 288            echo "Unable to locate ninja!" 1>&2
 289            exit 1
 290         fi
 291     fi
 292
 293     if [ $__SkipConfigure == 0 ]; then
 294         # if msbuild is not supported, then set __SkipGenerateVersion to 1
 295         if [ $__isMSBuildOnNETCoreSupported == 0 ]; then __SkipGenerateVersion=1; fi
 296         # Drop version.cpp file
 297         __versionSourceFile="$intermediatesForBuild/version.cpp"
 298         if [ $__SkipGenerateVersion == 0 ]; then
 299             pwd
 300             "$__ProjectRoot/dotnet.sh" msbuild /nologo /verbosity:minimal /clp:Summary \
 301                                        /l:BinClashLogger,Tools/Microsoft.DotNet.Build.Tasks.dll\;LogFile=binclash.log \
 302                                        /p:RestoreDefaultOptimizationDataPackage=false /p:PortableBuild=true \
 303                                        /p:UsePartialNGENOptimization=false /maxcpucount \
 304                                        "$__ProjectDir/build.proj" /p:GenerateVersionSourceFile=true /t:GenerateVersionSourceFile /p:NativeVersionSourceFile=$__versionSourceFile \
 305                                        $__CommonMSBuildArgs $__UnprocessedBuildArgs
 306         else
 307             # Generate the dummy version.cpp, but only if it didn't exist to make sure we don't trigger unnecessary rebuild
 308             __versionSourceLine="static char sccsid[] __attribute__((used)) = \"@(#)No version information produced\";"
 309             if [ -e $__versionSourceFile ]; then
 310                 read existingVersionSourceLine < $__versionSourceFile
 311             fi
 312             if [ "$__versionSourceLine" != "$existingVersionSourceLine" ]; then
 313                 echo $__versionSourceLine > $__versionSourceFile
 314             fi
 315         fi
 316
 317
 318         pushd "$intermediatesForBuild"
 319         # Regenerate the CMake solution
 320         echo "Invoking \"$__ProjectRoot/src/pal/tools/gen-buildsys-clang.sh\" \"$__ProjectRoot\" $__ClangMajorVersion \"$__ClangMinorVersion\" $platformArch $__BuildType $__CodeCoverage $generator $extraCmakeArguments $__cmakeargs"
 321         "$__ProjectRoot/src/pal/tools/gen-buildsys-clang.sh" "$__ProjectRoot" $__ClangMajorVersion "$__ClangMinorVersion" $platformArch $__BuildType $__CodeCoverage $generator "$extraCmakeArguments" "$__cmakeargs"
 322         popd
 323     fi
 324
 325     if [ ! -f "$intermediatesForBuild/$buildFile" ]; then
 326         echo "Failed to generate $message build project!"
 327         exit 1
 328     fi
 329
 330     # Build
 331     if [ $__ConfigureOnly == 1 ]; then
 332         echo "Finish configuration & skipping $message build."
 333         return
 334     fi
 335
 336     # Check that the makefiles were created.
 337     pushd "$intermediatesForBuild"
 338
 339     echo "Executing $buildTool install -j $__NumProc"
 340
 341     $buildTool install -j $__NumProc
 342     if [ $? != 0 ]; then
 343         echo "Failed to build $message."
 344         exit 1
 345     fi
 346
 347     popd
 348 }
 349
 350 build_cross_architecture_components()
 351 {
 352     local intermediatesForBuild="$__IntermediatesDir/Host$__CrossArch/crossgen"
 353     local crossArchBinDir="$__BinDir/$__CrossArch"
 354
 355     mkdir -p "$intermediatesForBuild"
 356     mkdir -p "$crossArchBinDir"
 357
 358     generate_event_logging_sources "$intermediatesForBuild" "the crossarch build system"
 359
 360     __SkipCrossArchBuild=1
 361     # check supported cross-architecture components host(__HostArch)/target(__BuildArch) pair
 362     if [[ ("$__BuildArch" == "arm" || "$__BuildArch" == "armel") && ("$__CrossArch" == "x86" || "$__CrossArch" == "x64") ]]; then
 363         __SkipCrossArchBuild=0
 364     elif [[ "$__BuildArch" == "arm64" && "$__CrossArch" == "x64" ]]; then
 365         __SkipCrossArchBuild=0
 366     else
 367         # not supported
 368         return
 369     fi
 370
 371     export __CMakeBinDir="$crossArchBinDir"
 372     export CROSSCOMPILE=0
 373
 374     __ExtraCmakeArgs="-DCLR_CMAKE_TARGET_ARCH=$__BuildArch -DCLR_CMAKE_TARGET_OS=$__BuildOS -DCLR_CMAKE_PACKAGES_DIR=$__PackagesDir -DCLR_CMAKE_PGO_INSTRUMENT=$__PgoInstrument -DCLR_CMAKE_OPTDATA_VERSION=$__PgoOptDataVersion -DCLR_CMAKE_PGO_OPTIMIZE=$__PgoOptimize -DCLR_CROSS_COMPONENTS_BUILD=1"
 375     build_native $__SkipCrossArchBuild "$__CrossArch" "$intermediatesForBuild" "$__ExtraCmakeArgs" "cross-architecture components"
 376
 377     export CROSSCOMPILE=1
 378 }
 379
 380 isMSBuildOnNETCoreSupported()
 381 {
 382     __isMSBuildOnNETCoreSupported=$__msbuildonunsupportedplatform
 383
 384     if [ $__isMSBuildOnNETCoreSupported == 1 ]; then
 385         return
 386     fi
 387
 388     if [ $__SkipManaged == 1 ]; then
 389         __isMSBuildOnNETCoreSupported=0
 390         return
 391     fi
 392
 393     if [ "$__HostArch" == "x64" ]; then
 394         if [ "$__HostOS" == "Linux" ]; then
 395             __isMSBuildOnNETCoreSupported=1
 396             # note: the RIDs below can use globbing patterns
 397             UNSUPPORTED_RIDS=("debian.9-x64" "ubuntu.17.04-x64")
 398             for UNSUPPORTED_RID in "${UNSUPPORTED_RIDS[@]}"
 399             do
 400                 if [[ $__HostDistroRid == $UNSUPPORTED_RID ]]; then
 401                     __isMSBuildOnNETCoreSupported=0
 402                     break
 403                 fi
 404             done
 405         elif [ "$__HostOS" == "OSX" ]; then
 406             __isMSBuildOnNETCoreSupported=1
 407         elif [ "$__HostOS" == "FreeBSD" ]; then
 408             __isMSBuildOnNETCoreSupported=1
 409         fi
 410     fi
 411 }
 412
 413
 414 build_CoreLib_ni()
 415 {
 416     local __CrossGenExec=$1
 417
 418     if [ $__PartialNgen == 1 ]; then
 419         export COMPlus_PartialNGen=1
 420     fi
 421
 422     if [ -e $__CrossGenCoreLibLog ]; then
 423         rm $__CrossGenCoreLibLog
 424     fi
 425     echo "Generating native image of System.Private.CoreLib.dll for $__BuildOS.$__BuildArch.$__BuildType. Logging to \"$__CrossGenCoreLibLog\"."
 426     echo "$__CrossGenExec /Platform_Assemblies_Paths $__BinDir/IL $__IbcTuning /out $__BinDir/System.Private.CoreLib.dll $__BinDir/IL/System.Private.CoreLib.dll"
 427     $__CrossGenExec /Platform_Assemblies_Paths $__BinDir/IL $__IbcTuning /out $__BinDir/System.Private.CoreLib.dll $__BinDir/IL/System.Private.CoreLib.dll >> $__CrossGenCoreLibLog 2>&1
 428     if [ $? -ne 0 ]; then
 429         echo "Failed to generate native image for System.Private.CoreLib. Refer to $__CrossGenCoreLibLog"
 430         exit 1
 431     fi
 432
 433     if [ "$__BuildOS" == "Linux" ]; then
 434         echo "Generating symbol file for System.Private.CoreLib.dll"
 435         echo "$__CrossGenExec /Platform_Assemblies_Paths $__BinDir /CreatePerfMap $__BinDir $__BinDir/System.Private.CoreLib.dll"
 436         $__CrossGenExec /Platform_Assemblies_Paths $__BinDir /CreatePerfMap $__BinDir $__BinDir/System.Private.CoreLib.dll >> $__CrossGenCoreLibLog 2>&1
 437         if [ $? -ne 0 ]; then
 438             echo "Failed to generate symbol file for System.Private.CoreLib. Refer to $__CrossGenCoreLibLog"
 439             exit 1
 440         fi
 441     fi
 442 }
 443
 444 build_CoreLib()
 445 {
 446     if [ $__isMSBuildOnNETCoreSupported == 0 ]; then
 447         echo "System.Private.CoreLib.dll build unsupported."
 448         return
 449     fi
 450
 451     if [ $__SkipMSCorLib == 1 ]; then
 452        echo "Skipping building System.Private.CoreLib."
 453        return
 454     fi
 455
 456     echo "Commencing build of managed components for $__BuildOS.$__BuildArch.$__BuildType"
 457
 458     # Invoke MSBuild
 459     __ExtraBuildArgs=""
 460     if [[ "$__IbcTuning" == "" ]]; then
 461         __ExtraBuildArgs="$__ExtraBuildArgs /p:OptimizationDataDir=\"$__PackagesDir/optimization.$__BuildOS-$__BuildArch.IBC.CoreCLR/$__IbcOptDataVersion/data\""
 462         __ExtraBuildArgs="$__ExtraBuildArgs /p:EnableProfileGuidedOptimization=true"
 463     fi
 464
 465     if [[ "$__BuildManagedTools" -eq "1" ]]; then
 466         __ExtraBuildArgs="$__ExtraBuildArgs /p:BuildManagedTools=true"
 467     fi
 468
 469     $__ProjectRoot/dotnet.sh msbuild /nologo /verbosity:minimal /clp:Summary \
 470                              /l:BinClashLogger,Tools/Microsoft.DotNet.Build.Tasks.dll\;LogFile=binclash.log \
 471                              /p:RestoreDefaultOptimizationDataPackage=false /p:PortableBuild=true \
 472                              /p:UsePartialNGENOptimization=false /maxcpucount \
 473                              $__ProjectDir/build.proj \
 474                              /flp:Verbosity=normal\;LogFile=$__LogsDir/System.Private.CoreLib_$__BuildOS__$__BuildArch__$__BuildType.log \
 475                              /p:__IntermediatesDir=$__IntermediatesDir /p:__RootBinDir=$__RootBinDir /p:BuildNugetPackage=false /p:UseSharedCompilation=false \
 476                              $__CommonMSBuildArgs $__ExtraBuildArgs $__UnprocessedBuildArgs
 477
 478     if [ $? -ne 0 ]; then
 479         echo "Failed to build managed components."
 480         exit 1
 481     fi
 482
 483     if [ $__SkipCrossgen == 1 ]; then
 484         echo "Skipping generating native image"
 485         return
 486     fi
 487
 488     # The cross build generates a crossgen with the target architecture.
 489     if [ $__CrossBuild == 0 ]; then
 490        if [ $__SkipCoreCLR == 1 ]; then
 491            return
 492        fi
 493
 494        # The architecture of host pc must be same architecture with target.
 495        if [[ ( "$__HostArch" == "$__BuildArch" ) ]]; then
 496            build_CoreLib_ni "$__BinDir/crossgen"
 497        elif [[ ( "$__HostArch" == "x64" ) && ( "$__BuildArch" == "x86" ) ]]; then
 498            build_CoreLib_ni "$__BinDir/crossgen"
 499        elif [[ ( "$__HostArch" == "arm64" ) && ( "$__BuildArch" == "arm" ) ]]; then
 500            build_CoreLib_ni "$__BinDir/crossgen"
 501        else
 502            exit 1
 503        fi
 504     else
 505        if [[ ( "$__CrossArch" == "x86" ) && ( "$__BuildArch" == "arm" ) ]]; then
 506            build_CoreLib_ni "$__CrossComponentBinDir/crossgen"
 507        elif [[ ( "$__CrossArch" == "x64" ) && ( "$__BuildArch" == "arm" ) ]]; then
 508            build_CoreLib_ni "$__CrossComponentBinDir/crossgen"
 509        elif [[ ( "$__HostArch" == "x64" ) && ( "$__BuildArch" == "arm64" ) ]]; then
 510            build_CoreLib_ni "$__CrossComponentBinDir/crossgen"
 511        fi
 512     fi
 513 }
 514
 515 generate_NugetPackages()
 516 {
 517     # We can only generate nuget package if we also support building mscorlib as part of this build.
 518     if [ $__isMSBuildOnNETCoreSupported == 0 ]; then
 519         echo "Nuget package generation unsupported."
 520         return
 521     fi
 522
 523     # Since we can build mscorlib for this OS, did we build the native components as well?
 524     if [[ $__SkipCoreCLR == 1 && $__CrossgenOnly == 0 ]]; then
 525         echo "Unable to generate nuget packages since native components were not built."
 526         return
 527     fi
 528
 529     echo "Generating nuget packages for "$__BuildOS
 530     echo "DistroRid is "$__DistroRid
 531     echo "ROOTFS_DIR is "$ROOTFS_DIR
 532     # Build the packages
 533     $__ProjectRoot/dotnet.sh msbuild /nologo /verbosity:minimal /clp:Summary \
 534                              /l:BinClashLogger,Tools/Microsoft.DotNet.Build.Tasks.dll\;LogFile=binclash.log \
 535                              /p:RestoreDefaultOptimizationDataPackage=false /p:PortableBuild=true \
 536                              /p:UsePartialNGENOptimization=false /maxcpucount \
 537                              $__SourceDir/.nuget/packages.builds \
 538                              /flp:Verbosity=normal\;LogFile=$__LogsDir/Nuget_$__BuildOS__$__BuildArch__$__BuildType.log \
 539                              /p:__IntermediatesDir=$__IntermediatesDir /p:__RootBinDir=$__RootBinDir /p:BuildNugetPackages=false /p:UseSharedCompilation=false /p:__DoCrossArchBuild=$__CrossBuild \
 540                              $__CommonMSBuildArgs $__UnprocessedBuildArgs
 541
 542     if [ $? -ne 0 ]; then
 543         echo "Failed to generate Nuget packages."
 544         exit 1
 545     fi
 546 }
 547
 548 echo "Commencing CoreCLR Repo build"
 549
 550 # Argument types supported by this script:
 551 #
 552 # Build architecture - valid values are: x64, ARM.
 553 # Build Type         - valid values are: Debug, Checked, Release
 554 #
 555 # Set the default arguments for build
 556
 557 # Obtain the location of the bash script to figure out where the root of the repo is.
 558 __ProjectRoot="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
 559
 560 # Use uname to determine what the CPU is.
 561 CPUName=$(uname -p)
 562 # Some Linux platforms report unknown for platform, but the arch for machine.
 563 if [ "$CPUName" == "unknown" ]; then
 564     CPUName=$(uname -m)
 565 fi
 566
 567 case $CPUName in
 568     i686)
 569         echo "Unsupported CPU $CPUName detected, build might not succeed!"
 570         __BuildArch=x86
 571         __HostArch=x86
 572         ;;
 573
 574     x86_64)
 575         __BuildArch=x64
 576         __HostArch=x64
 577         ;;
 578
 579     armv7l)
 580         echo "Unsupported CPU $CPUName detected, build might not succeed!"
 581         __BuildArch=arm
 582         __HostArch=arm
 583         ;;
 584
 585     aarch64)
 586         __BuildArch=arm64
 587         __HostArch=arm64
 588         ;;
 589
 590     amd64)
 591         __BuildArch=x64
 592         __HostArch=x64
 593         ;;
 594     *)
 595         echo "Unknown CPU $CPUName detected, configuring as if for x64"
 596         __BuildArch=x64
 597         __HostArch=x64
 598         ;;
 599 esac
 600
 601 # Use uname to determine what the OS is.
 602 OSName=$(uname -s)
 603 case $OSName in
 604     Linux)
 605         __BuildOS=Linux
 606         __HostOS=Linux
 607         ;;
 608
 609     Darwin)
 610         __BuildOS=OSX
 611         __HostOS=OSX
 612         ;;
 613
 614     FreeBSD)
 615         __BuildOS=FreeBSD
 616         __HostOS=FreeBSD
 617         ;;
 618
 619     OpenBSD)
 620         __BuildOS=OpenBSD
 621         __HostOS=OpenBSD
 622         ;;
 623
 624     NetBSD)
 625         __BuildOS=NetBSD
 626         __HostOS=NetBSD
 627         ;;
 628
 629     SunOS)
 630         __BuildOS=SunOS
 631         __HostOS=SunOS
 632         ;;
 633
 634     *)
 635         echo "Unsupported OS $OSName detected, configuring as if for Linux"
 636         __BuildOS=Linux
 637         __HostOS=Linux
 638         ;;
 639 esac
 640
 641 __BuildType=Debug
 642 __CodeCoverage=
 643 __IgnoreWarnings=0
 644
 645 # Set the various build properties here so that CMake and MSBuild can pick them up
 646 __ProjectDir="$__ProjectRoot"
 647 __SourceDir="$__ProjectDir/src"
 648 __PackagesDir="${DotNetRestorePackagesPath:-${__ProjectDir}/packages}"
 649 __RootBinDir="$__ProjectDir/bin"
 650 __UnprocessedBuildArgs=
 651 __CommonMSBuildArgs=
 652 __MSBCleanBuildArgs=
 653 __UseNinja=0
 654 __VerboseBuild=0
 655 __PgoInstrument=0
 656 __PgoOptimize=1
 657 __IbcTuning=""
 658 __ConfigureOnly=0
 659 __SkipConfigure=0
 660 __SkipManaged=0
 661 __SkipRestore=""
 662 __SkipNuget=0
 663 __SkipCoreCLR=0
 664 __SkipCrossArchNative=0
 665 __SkipMSCorLib=0
 666 __SkipRestoreOptData=0
 667 __SkipCrossgen=0
 668 __CrossgenOnly=0
 669 __PartialNgen=0
 670 __SkipTests=0
 671 __CrossBuild=0
 672 __ClangMajorVersion=0
 673 __ClangMinorVersion=0
 674 __NuGetPath="$__PackagesDir/NuGet.exe"
 675 __HostDistroRid=""
 676 __DistroRid=""
 677 __cmakeargs=""
 678 __SkipGenerateVersion=0
 679 __PortableBuild=1
 680 __msbuildonunsupportedplatform=0
 681 __PgoOptDataVersion=""
 682 __IbcOptDataVersion=""
 683 __BuildManagedTools=1
 684 __SkipRestoreArg=""
 685 __SignTypeArg=""
 686 __OfficialBuildIdArg=""
 687
 688 # Get the number of processors available to the scheduler
 689 # Other techniques such as `nproc` only get the number of
 690 # processors available to a single process.
 691 if [ `uname` = "FreeBSD" ]; then
 692   __NumProc=`sysctl hw.ncpu | awk '{ print $2+1 }'`
 693 elif [ `uname` = "NetBSD" ]; then
 694   __NumProc=$(($(getconf NPROCESSORS_ONLN)+1))
 695 elif [ `uname` = "Darwin" ]; then
 696   __NumProc=$(($(getconf _NPROCESSORS_ONLN)+1))
 697 else
 698   __NumProc=$(nproc --all)
 699 fi
 700
 701 while :; do
 702     if [ $# -le 0 ]; then
 703         break
 704     fi
 705
 706     lowerI="$(echo $1 | awk '{print tolower($0)}')"
 707     case $lowerI in
 708         -\?|-h|--help)
 709             usage
 710             exit 1
 711             ;;
 712
 713         x86|-x86)
 714             __BuildArch=x86
 715             ;;
 716
 717         x64|-x64)
 718             __BuildArch=x64
 719             ;;
 720
 721         arm|-arm)
 722             __BuildArch=arm
 723             ;;
 724
 725         armel|-armel)
 726             __BuildArch=armel
 727             ;;
 728
 729         arm64|-arm64)
 730             __BuildArch=arm64
 731             ;;
 732
 733         debug|-debug)
 734             __BuildType=Debug
 735             ;;
 736
 737         checked|-checked)
 738             __BuildType=Checked
 739             ;;
 740
 741         release|-release)
 742             __BuildType=Release
 743             ;;
 744
 745         coverage|-coverage)
 746             __CodeCoverage=Coverage
 747             ;;
 748
 749         cross|-cross)
 750             __CrossBuild=1
 751             ;;
 752
 753         -portablebuild=false)
 754             __PortableBuild=0
 755             ;;
 756
 757         verbose|-verbose)
 758             __VerboseBuild=1
 759             ;;
 760
 761         stripsymbols|-stripsymbols)
 762             __cmakeargs="$__cmakeargs -DSTRIP_SYMBOLS=true"
 763             ;;
 764
 765         clang3.5|-clang3.5)
 766             __ClangMajorVersion=3
 767             __ClangMinorVersion=5
 768             ;;
 769
 770         clang3.6|-clang3.6)
 771             __ClangMajorVersion=3
 772             __ClangMinorVersion=6
 773             ;;
 774
 775         clang3.7|-clang3.7)
 776             __ClangMajorVersion=3
 777             __ClangMinorVersion=7
 778             ;;
 779
 780         clang3.8|-clang3.8)
 781             __ClangMajorVersion=3
 782             __ClangMinorVersion=8
 783             ;;
 784
 785         clang3.9|-clang3.9)
 786             __ClangMajorVersion=3
 787             __ClangMinorVersion=9
 788             ;;
 789
 790         clang4.0|-clang4.0)
 791             __ClangMajorVersion=4
 792             __ClangMinorVersion=0
 793             ;;
 794
 795         clang5.0|-clang5.0)
 796             __ClangMajorVersion=5
 797             __ClangMinorVersion=0
 798             ;;
 799
 800         clang6.0|-clang6.0)
 801             __ClangMajorVersion=6
 802             __ClangMinorVersion=0
 803             ;;
 804
 805         clang7|-clang7)
 806             __ClangMajorVersion=7
 807             __ClangMinorVersion=
 808             ;;
 809
 810         ninja|-ninja)
 811             __UseNinja=1
 812             ;;
 813
 814         pgoinstrument|-pgoinstrument)
 815             __PgoInstrument=1
 816             ;;
 817
 818         nopgooptimize|-nopgooptimize)
 819             __PgoOptimize=0
 820             __SkipRestoreOptData=1
 821             ;;
 822
 823         ibcinstrument|-ibcinstrument)
 824             __IbcTuning="/Tuning"
 825             ;;
 826
 827         configureonly|-configureonly)
 828             __ConfigureOnly=1
 829             __SkipMSCorLib=1
 830             __SkipNuget=1
 831             ;;
 832
 833         skipconfigure|-skipconfigure)
 834             __SkipConfigure=1
 835             ;;
 836
 837         skipnative|-skipnative)
 838             # Use "skipnative" to use the same option name as build.cmd.
 839             __SkipCoreCLR=1
 840             ;;
 841
 842         skipcoreclr|-skipcoreclr)
 843             # Accept "skipcoreclr" for backwards-compatibility.
 844             __SkipCoreCLR=1
 845             ;;
 846
 847         skipcrossarchnative|-skipcrossarchnative)
 848             __SkipCrossArchNative=1
 849             ;;
 850
 851         skipmanaged|-skipmanaged)
 852             __SkipManaged=1
 853             ;;
 854
 855         skipmscorlib|-skipmscorlib)
 856             __SkipMSCorLib=1
 857             ;;
 858
 859         skipgenerateversion|-skipgenerateversion)
 860             __SkipGenerateVersion=1
 861             ;;
 862
 863         skiprestoreoptdata|-skiprestoreoptdata)
 864             __SkipRestoreOptData=1
 865             ;;
 866
 867         skipcrossgen|-skipcrossgen)
 868             __SkipCrossgen=1
 869             ;;
 870
 871         crossgenonly|-crossgenonly)
 872             __SkipMSCorLib=1
 873             __SkipCoreCLR=1
 874             __CrossgenOnly=1
 875             ;;
 876         partialngen|-partialngen)
 877             __PartialNgen=1
 878             ;;
 879
 880         skiptests|-skiptests)
 881             __SkipTests=1
 882             ;;
 883
 884         skipnuget|-skipnuget|skipbuildpackages|-skipbuildpackages)
 885             __SkipNuget=1
 886             ;;
 887
 888         ignorewarnings|-ignorewarnings)
 889             __IgnoreWarnings=1
 890             __cmakeargs="$__cmakeargs -DCLR_CMAKE_WARNINGS_ARE_ERRORS=OFF"
 891             ;;
 892
 893         cmakeargs|-cmakeargs)
 894             if [ -n "$2" ]; then
 895                 __cmakeargs="$__cmakeargs $2"
 896                 shift
 897             else
 898                 echo "ERROR: 'cmakeargs' requires a non-empty option argument"
 899                 exit 1
 900             fi
 901             ;;
 902
 903         bindir|-bindir)
 904             if [ -n "$2" ]; then
 905                 __RootBinDir="$2"
 906                 if [ ! -d $__RootBinDir ]; then
 907                     mkdir $__RootBinDir
 908                 fi
 909                 __RootBinParent=$(dirname $__RootBinDir)
 910                 __RootBinName=${__RootBinDir##*/}
 911                 __RootBinDir="$(cd $__RootBinParent &>/dev/null && printf %s/%s $PWD $__RootBinName)"
 912                 shift
 913             else
 914                 echo "ERROR: 'bindir' requires a non-empty option argument"
 915                 exit 1
 916             fi
 917             ;;
 918         msbuildonunsupportedplatform|-msbuildonunsupportedplatform)
 919             __msbuildonunsupportedplatform=1
 920             ;;
 921         numproc|-numproc)
 922             if [ -n "$2" ]; then
 923               __NumProc="$2"
 924               shift
 925             else
 926               echo "ERROR: 'numproc' requires a non-empty option argument"
 927               exit 1
 928             fi
 929             ;;
 930         osgroup|-osgroup)
 931             if [ -n "$2" ]; then
 932               __BuildOS="$2"
 933               shift
 934             else
 935               echo "ERROR: 'osgroup' requires a non-empty option argument"
 936               exit 1
 937             fi
 938             ;;
 939         rebuild|-rebuild)
 940             echo "ERROR: 'Rebuild' is not supported.  Please remove it."
 941             exit 1
 942             ;;
 943
 944         -skiprestore)
 945             __SkipRestoreArg="/p:RestoreDuringBuild=false"
 946             ;;
 947
 948         -disableoss)
 949             __SignTypeArg="/p:SignType=real"
 950             ;;
 951
 952         -officialbuildid=*)
 953             __Id=$(echo $1| cut -d'=' -f 2)
 954             __OfficialBuildIdArg="/p:OfficialBuildId=$__Id"
 955             ;;
 956
 957         --)
 958             # Skip -Option=Value style argument passing
 959             ;;
 960
 961         *)
 962             __UnprocessedBuildArgs="$__UnprocessedBuildArgs $1"
 963             ;;
 964     esac
 965
 966     shift
 967 done
 968
 969 __CommonMSBuildArgs="/p:__BuildArch=$__BuildArch /p:__BuildType=$__BuildType /p:__BuildOS=$__BuildOS $__OfficialBuildIdArg $__SignTypeArg $__SkipRestoreArg"
 970
 971 # Configure environment if we are doing a verbose build
 972 if [ $__VerboseBuild == 1 ]; then
 973     export VERBOSE=1
 974     __CommonMSBuildArgs="$__CommonMSBuildArgs /v:detailed"
 975 fi
 976
 977 # Set default clang version
 978 if [[ $__ClangMajorVersion == 0 && $__ClangMinorVersion == 0 ]]; then
 979     if [[ "$__BuildArch" == "arm" || "$__BuildArch" == "armel" ]]; then
 980         __ClangMajorVersion=5
 981         __ClangMinorVersion=0
 982     else
 983         __ClangMajorVersion=3
 984         __ClangMinorVersion=9
 985     fi
 986 fi
 987
 988 if [[ "$__BuildArch" == "armel" ]]; then
 989     # Armel cross build is Tizen specific and does not support Portable RID build
 990     __PortableBuild=0
 991 fi
 992
 993 if [ $__PortableBuild == 0 ]; then
 994     __CommonMSBuildArgs="$__CommonMSBuildArgs /p:PortableBuild=false"
 995 fi
 996
 997 # Set dependent variables
 998 __LogsDir="$__RootBinDir/Logs"
 999
1000 # init the host distro name
1001 initHostDistroRid
1002
1003 # Set the remaining variables based upon the determined build configuration
1004 __BinDir="$__RootBinDir/Product/$__BuildOS.$__BuildArch.$__BuildType"
1005 __PackagesBinDir="$__BinDir/.nuget"
1006 __ToolsDir="$__RootBinDir/tools"
1007 __TestWorkingDir="$__RootBinDir/tests/$__BuildOS.$__BuildArch.$__BuildType"
1008 export __IntermediatesDir="$__RootBinDir/obj/$__BuildOS.$__BuildArch.$__BuildType"
1009 __TestIntermediatesDir="$__RootBinDir/tests/obj/$__BuildOS.$__BuildArch.$__BuildType"
1010 __isMSBuildOnNETCoreSupported=0
1011 __CrossComponentBinDir="$__BinDir"
1012
1013 __CrossArch="$__HostArch"
1014 if [ $__CrossBuild == 1 ]; then
1015     __CrossComponentBinDir="$__CrossComponentBinDir/$__CrossArch"
1016 fi
1017 __CrossGenCoreLibLog="$__LogsDir/CrossgenCoreLib_$__BuildOS.$__BuildArch.$__BuildType.log"
1018
1019 # Init if MSBuild for .NET Core is supported for this platform
1020 isMSBuildOnNETCoreSupported
1021
1022 # CI_SPECIFIC - On CI machines, $HOME may not be set. In such a case, create a subfolder and set the variable to set.
1023 # This is needed by CLI to function.
1024 if [ -z "$HOME" ]; then
1025     if [ ! -d "$__ProjectDir/temp_home" ]; then
1026         mkdir temp_home
1027     fi
1028     export HOME=$__ProjectDir/temp_home
1029     echo "HOME not defined; setting it to $HOME"
1030 fi
1031
1032 # Specify path to be set for CMAKE_INSTALL_PREFIX.
1033 # This is where all built CoreClr libraries will copied to.
1034 export __CMakeBinDir="$__BinDir"
1035
1036 # Configure environment if we are doing a cross compile.
1037 if [ $__CrossBuild == 1 ]; then
1038     export CROSSCOMPILE=1
1039     if ! [[ -n "$ROOTFS_DIR" ]]; then
1040         export ROOTFS_DIR="$__ProjectRoot/cross/rootfs/$__BuildArch"
1041     fi
1042 fi
1043
1044 # init the target distro name
1045 initTargetDistroRid
1046
1047 # Make the directories necessary for build if they don't exist
1048 setup_dirs
1049
1050 # Check prereqs.
1051 check_prereqs
1052
1053 # Restore the package containing profile counts for profile-guided optimizations
1054 restore_optdata
1055
1056 # Generate event logging infrastructure sources
1057 generate_event_logging
1058
1059 # Build the coreclr (native) components.
1060 __ExtraCmakeArgs="-DCLR_CMAKE_TARGET_OS=$__BuildOS -DCLR_CMAKE_PACKAGES_DIR=$__PackagesDir -DCLR_CMAKE_PGO_INSTRUMENT=$__PgoInstrument -DCLR_CMAKE_OPTDATA_VERSION=$__PgoOptDataVersion -DCLR_CMAKE_PGO_OPTIMIZE=$__PgoOptimize"
1061
1062 # [TODO] Remove this when the `build-test.sh` script properly builds and deploys test assets.
1063 if [ $__SkipTests != 1 ]; then
1064     echo "Adding CMake flags to build native tests for $__BuildOS.$__BuildArch.$__BuildType"
1065     __ExtraCmakeArgs="$__ExtraCmakeArgs -DCLR_CMAKE_BUILD_TESTS=ON"
1066 fi
1067
1068 build_native $__SkipCoreCLR "$__BuildArch" "$__IntermediatesDir" "$__ExtraCmakeArgs" "CoreCLR component"
1069
1070 # Build cross-architecture components
1071 if [ $__SkipCrossArchNative != 1 ]; then
1072     if [[ $__CrossBuild == 1 ]]; then
1073         build_cross_architecture_components
1074     fi
1075 fi
1076
1077 # Build System.Private.CoreLib.
1078
1079 build_CoreLib
1080
1081 if [ $__CrossgenOnly == 1 ]; then
1082     build_CoreLib_ni "$__BinDir/crossgen"
1083 fi
1084
1085 # Generate nuget packages
1086 if [ $__SkipNuget != 1 ]; then
1087     generate_NugetPackages
1088 fi
1089
1090
1091 # Build complete
1092
1093 echo "Repo successfully built."
1094 echo "Product binaries are available at $__BinDir"
1095 exit 0