build.sh

   1 #!/usr/bin/env bash
   2
   3 # Work around Jenkins CI + msbuild problem: Jenkins sometimes creates very large environment
   4 # variables, and msbuild can't handle environment blocks with such large variables. So clear
   5 # out the variables that might be too large.
   6 export ghprbCommentBody=
   7
   8 # resolve python-version to use
   9 if [ "$PYTHON" == "" ] ; then
  10     if ! PYTHON=$(command -v python3 || command -v python2 || command -v python || command -v py)
  11     then
  12        echo "Unable to locate build-dependency python!" 1>&2
  13        exit 1
  14     fi
  15 fi
  16 # validate python-dependency
  17 # useful in case of explicitly set option.
  18 if ! command -v $PYTHON > /dev/null
  19 then
  20    echo "Unable to locate build-dependency python ($PYTHON)!" 1>&2
  21    exit 1
  22 fi
  23
  24 export PYTHON
  25
  26 usage()
  27 {
  28     echo "Usage: $0 [BuildArch] [BuildType] [-verbose] [-coverage] [-cross] [-gccx.y] [-clangx.y] [-ninja] [-configureonly] [-skipconfigure] [-skipnative] [-skipcrossarchnative] [-skipmanaged] [-skipmscorlib] [-skiptests] [-stripsymbols] [-ignorewarnings] [-cmakeargs] [-bindir]"
  29     echo "BuildArch can be: -x64, -x86, -arm, -armel, -arm64"
  30     echo "BuildType can be: -debug, -checked, -release"
  31     echo "-coverage - optional argument to enable code coverage build (currently supported only for Linux and OSX)."
  32     echo "-ninja - target ninja instead of GNU make"
  33     echo "-gccx.y - optional argument to build using gcc version x.y."
  34     echo "-clangx.y - optional argument to build using clang version x.y."
  35     echo "-cross - optional argument to signify cross compilation,"
  36     echo "       - will use ROOTFS_DIR environment variable if set."
  37     echo "-nopgooptimize - do not use profile guided optimizations."
  38     echo "-pgoinstrument - generate instrumented code for profile guided optimization enabled binaries."
  39     echo "-ibcinstrument - generate IBC-tuning-enabled native images when invoking crossgen."
  40     echo "-configureonly - do not perform any builds; just configure the build."
  41     echo "-skipconfigure - skip build configuration."
  42     echo "-skipnative - do not build native components."
  43     echo "-skipcrossarchnative - do not build cross-architecture native components."
  44     echo "-skipmanaged - do not build managed components."
  45     echo "-skipmscorlib - do not build mscorlib.dll."
  46     echo "-skiptests - skip the tests in the 'tests' subdirectory."
  47     echo "-skipnuget - skip building nuget packages."
  48     echo "-skiprestoreoptdata - skip restoring optimization data used by profile-based optimizations."
  49     echo "-skipcrossgen - skip native image generation"
  50     echo "-crossgenonly - only run native image generation"
  51     echo "-partialngen - build CoreLib as PartialNGen"
  52     echo "-verbose - optional argument to enable verbose build output."
  53     echo "-skiprestore: skip restoring packages ^(default: packages are restored during build^)."
  54     echo "-disableoss: Disable Open Source Signing for System.Private.CoreLib."
  55     echo "-officialbuildid=^<ID^>: specify the official build ID to be used by this build."
  56     echo "-stripSymbols - Optional argument to strip native symbols during the build."
  57     echo "-skipgenerateversion - disable version generation even if MSBuild is supported."
  58     echo "-ignorewarnings - do not treat warnings as errors"
  59     echo "-cmakeargs - user-settable additional arguments passed to CMake."
  60     echo "-bindir - output directory (defaults to $__ProjectRoot/bin)"
  61     echo "-msbuildonunsupportedplatform - build managed binaries even if distro is not officially supported."
  62     echo "-numproc - set the number of build processes."
  63     exit 1
  64 }
  65
  66 initHostDistroRid()
  67 {
  68     __HostDistroRid=""
  69     if [ "$__HostOS" == "Linux" ]; then
  70         if [ -e /etc/os-release ]; then
  71             source /etc/os-release
  72             if [[ $ID == "rhel" ]]; then
  73                 # remove the last version digit
  74                 VERSION_ID=${VERSION_ID%.*}
  75             fi
  76             __HostDistroRid="$ID.$VERSION_ID-$__HostArch"
  77             if [[ $ID == "alpine" ]]; then
  78                 __HostDistroRid="linux-musl-$__HostArch"
  79             fi
  80         elif [ -e /etc/redhat-release ]; then
  81             local redhatRelease=$(</etc/redhat-release)
  82             if [[ $redhatRelease == "CentOS release 6."* || $redhatRelease == "Red Hat Enterprise Linux Server release 6."* ]]; then
  83                __HostDistroRid="rhel.6-$__HostArch"
  84             fi
  85             if [[ $redhatRelease == "CentOS Linux release 7."* ]]; then
  86                 __HostDistroRid="rhel.7-$__Arch"
  87             fi
  88         fi
  89     fi
  90     if [ "$__HostOS" == "FreeBSD" ]; then
  91         __freebsd_version=`sysctl -n kern.osrelease | cut -f1 -d'.'`
  92         __HostDistroRid="freebsd.$__freebsd_version-$__HostArch"
  93     fi
  94
  95     if [ "$__HostDistroRid" == "" ]; then
  96         echo "WARNING: Can not determine runtime id for current distro."
  97     fi
  98 }
  99
 100 initTargetDistroRid()
 101 {
 102     if [ $__CrossBuild == 1 ]; then
 103         if [ "$__BuildOS" == "Linux" ]; then
 104             if [ ! -e $ROOTFS_DIR/etc/os-release ]; then
 105                 if [ -e $ROOTFS_DIR/android_platform ]; then
 106                     source $ROOTFS_DIR/android_platform
 107                     export __DistroRid="$RID"
 108                 else
 109                     echo "WARNING: Can not determine runtime id for current distro."
 110                     export __DistroRid=""
 111                 fi
 112             else
 113                 source $ROOTFS_DIR/etc/os-release
 114                 export __DistroRid="$ID.$VERSION_ID-$__BuildArch"
 115             fi
 116         fi
 117     else
 118         export __DistroRid="$__HostDistroRid"
 119     fi
 120
 121     if [ "$__BuildOS" == "OSX" ]; then
 122         __PortableBuild=1
 123     fi
 124
 125     # Portable builds target the base RID
 126     if [ $__PortableBuild == 1 ]; then
 127         if [ "$__BuildOS" == "Linux" ]; then
 128             export __DistroRid="linux-$__BuildArch"
 129         elif [ "$__BuildOS" == "OSX" ]; then
 130             export __DistroRid="osx-$__BuildArch"
 131         elif [ "$__BuildOS" == "FreeBSD" ]; then
 132             export __DistroRid="freebsd-$__BuildArch"
 133         fi
 134     fi
 135 }
 136
 137 setup_dirs()
 138 {
 139     echo Setting up directories for build
 140
 141     mkdir -p "$__RootBinDir"
 142     mkdir -p "$__BinDir"
 143     mkdir -p "$__LogsDir"
 144     mkdir -p "$__MsbuildDebugLogsDir"
 145     mkdir -p "$__IntermediatesDir"
 146
 147     if [ $__CrossBuild == 1 ]; then
 148         mkdir -p "$__CrossComponentBinDir"
 149     fi
 150 }
 151
 152 # Check the system to ensure the right prereqs are in place
 153
 154 check_prereqs()
 155 {
 156     echo "Checking prerequisites..."
 157
 158     # Check presence of CMake on the path
 159     hash cmake 2>/dev/null || { echo >&2 "Please install cmake before running this script"; exit 1; }
 160
 161
 162     # Minimum required version of clang is version 4.0 for arm/armel cross build
 163     if [[ $__CrossBuild == 1 && $__GccBuild == 0 &&  ("$__BuildArch" == "arm" || "$__BuildArch" == "armel") ]]; then
 164         if ! [[ "$__ClangMajorVersion" -ge "4" ]]; then
 165             echo "Please install clang4.0 or latest for arm/armel cross build"; exit 1;
 166         fi
 167     fi
 168
 169     # Check for clang
 170     if [[ $__GccBuild == 0 ]]; then
 171         __ClangCombinedDottedVersion=$__ClangMajorVersion;
 172         if [[ "$__ClangMinorVersion" != "" ]]; then
 173             __ClangCombinedDottedVersion=$__ClangCombinedDottedVersion.$__ClangMinorVersion
 174         fi
 175         hash clang-$__ClangCombinedDottedVersion 2>/dev/null ||  hash clang$__ClangMajorVersion$__ClangMinorVersion 2>/dev/null || hash clang 2>/dev/null || { echo >&2 "Please install clang-$__ClangMajorVersion.$__ClangMinorVersion before running this script"; exit 1; }
 176     else
 177         __GccCombinedDottedVersion=$__GccMajorVersion;
 178         if [[ "$__GccMinorVersion" != "" ]]; then
 179             __GccCombinedDottedVersion=$__GccCombinedDottedVersion.$__GccMinorVersion
 180         fi
 181         hash gcc-$__GccCombinedDottedVersion 2>/dev/null ||  hash gcc$__GccMajorVersion$__GccMinorVersion 2>/dev/null || hash gcc 2>/dev/null || { echo >&2 "Please install gcc-$__GccMajorVersion.$__GccMinorVersion before running this script"; exit 1; }
 182     fi
 183
 184 }
 185
 186 restore_optdata()
 187 {
 188     # we only need optdata on a Release build
 189     if [[ "$__BuildType" != "Release" ]]; then __SkipRestoreOptData=1; fi
 190
 191     if [[ ( $__SkipRestoreOptData == 0 ) && ( $__isMSBuildOnNETCoreSupported == 1 ) ]]; then
 192         echo "Restoring the OptimizationData package"
 193         "$__ProjectRoot/dotnet.sh" msbuild /nologo /verbosity:minimal /clp:Summary \
 194                                    /p:RestoreDefaultOptimizationDataPackage=false /p:PortableBuild=true \
 195                                    /p:UsePartialNGENOptimization=false /maxcpucount \
 196                                    /t:RestoreOptData ./build.proj \
 197                                    $__CommonMSBuildArgs $__UnprocessedBuildArgs
 198         if [ $? != 0 ]; then
 199             echo "Failed to restore the optimization data package."
 200             exit 1
 201         fi
 202     fi
 203
 204     if [ $__isMSBuildOnNETCoreSupported == 1 ]; then
 205         # Parse the optdata package versions out of msbuild so that we can pass them on to CMake
 206         local DotNetCli="$__ProjectRoot/Tools/dotnetcli/dotnet"
 207         if [ ! -f $DotNetCli ]; then
 208             source "$__ProjectRoot/init-tools.sh"
 209             if [ $? != 0 ]; then
 210                 echo "Failed to restore buildtools."
 211                 exit 1
 212             fi
 213         fi
 214         local OptDataProjectFilePath="$__ProjectRoot/src/.nuget/optdata/optdata.csproj"
 215         __PgoOptDataVersion=$(DOTNET_SKIP_FIRST_TIME_EXPERIENCE=1 $DotNetCli msbuild $OptDataProjectFilePath /t:DumpPgoDataPackageVersion /nologo | sed 's/^\s*//')
 216         __IbcOptDataVersion=$(DOTNET_SKIP_FIRST_TIME_EXPERIENCE=1 $DotNetCli msbuild $OptDataProjectFilePath /t:DumpIbcDataPackageVersion /nologo | sed 's/^[[:blank:]]*//')
 217     fi
 218 }
 219
 220 generate_event_logging_sources()
 221 {
 222     __OutputDir=$1
 223     __ConsumingBuildSystem=$2
 224
 225     __OutputIncDir="$__OutputDir/src/inc"
 226     __OutputEventingDir="$__OutputDir/Eventing"
 227     __OutputEventProviderDir="$__OutputEventingDir/eventprovider"
 228
 229     echo "Laying out dynamically generated files consumed by $__ConsumingBuildSystem"
 230     echo "Laying out dynamically generated Event test files, etmdummy stub functions, and external linkages"
 231
 232     __PythonWarningFlags="-Wall"
 233     if [[ $__IgnoreWarnings == 0 ]]; then
 234         __PythonWarningFlags="$__PythonWarningFlags -Werror"
 235     fi
 236
 237     $PYTHON -B $__PythonWarningFlags "$__ProjectRoot/src/scripts/genEventing.py" --inc $__OutputIncDir --dummy $__OutputIncDir/etmdummy.h --man "$__ProjectRoot/src/vm/ClrEtwAll.man" --testdir "$__OutputEventProviderDir/tests"
 238     if [[ $? != 0 ]]; then
 239         exit 1
 240     fi
 241
 242     echo "Laying out dynamically generated EventPipe Implementation"
 243     $PYTHON -B $__PythonWarningFlags "$__ProjectRoot/src/scripts/genEventPipe.py" --man "$__ProjectRoot/src/vm/ClrEtwAll.man" --intermediate "$__OutputEventingDir/eventpipe"
 244
 245     echo "Laying out dynamically generated EventSource classes"
 246     $PYTHON -B $__PythonWarningFlags "$__ProjectRoot/src/scripts/genRuntimeEventSources.py" --man "$__ProjectRoot/src/vm/ClrEtwAll.man" --intermediate "$__OutputEventingDir"
 247
 248     # determine the logging system
 249     case $__BuildOS in
 250         Linux|FreeBSD)
 251             echo "Laying out dynamically generated Event Logging Implementation of Lttng"
 252             $PYTHON -B $__PythonWarningFlags "$__ProjectRoot/src/scripts/genLttngProvider.py" --man "$__ProjectRoot/src/vm/ClrEtwAll.man" --intermediate "$__OutputEventProviderDir"
 253             if [[ $? != 0 ]]; then
 254                 exit 1
 255             fi
 256             ;;
 257         *)
 258             echo "Laying out dummy event logging provider"
 259             $PYTHON -B $__PythonWarningFlags "$__ProjectRoot/src/scripts/genDummyProvider.py" --man "$__ProjectRoot/src/vm/ClrEtwAll.man" --intermediate "$__OutputEventProviderDir"
 260             if [[ $? != 0 ]]; then
 261                 exit 1
 262             fi
 263             ;;
 264     esac
 265 }
 266
 267 generate_event_logging()
 268 {
 269     # Event Logging Infrastructure
 270     if [[ $__SkipCoreCLR == 0 || $__SkipMSCorLib == 0 || $__ConfigureOnly == 1 ]]; then
 271         generate_event_logging_sources "$__IntermediatesDir" "the native build system"
 272     fi
 273 }
 274
 275 build_native()
 276 {
 277     skipCondition=$1
 278     platformArch="$2"
 279     intermediatesForBuild="$3"
 280     extraCmakeArguments="$4"
 281     message="$5"
 282
 283     if [ $skipCondition == 1 ]; then
 284         echo "Skipping $message build."
 285         return
 286     fi
 287
 288     # All set to commence the build
 289     echo "Commencing build of $message for $__BuildOS.$__BuildArch.$__BuildType in $intermediatesForBuild"
 290
 291     generator=""
 292     buildFile="Makefile"
 293     buildTool="make"
 294     if [ $__UseNinja == 1 ]; then
 295         generator="ninja"
 296         buildFile="build.ninja"
 297         if ! buildTool=$(command -v ninja || command -v ninja-build); then
 298            echo "Unable to locate ninja!" 1>&2
 299            exit 1
 300         fi
 301     fi
 302
 303     if [ $__SkipConfigure == 0 ]; then
 304         # if msbuild is not supported, then set __SkipGenerateVersion to 1
 305         if [ $__isMSBuildOnNETCoreSupported == 0 ]; then __SkipGenerateVersion=1; fi
 306         # Drop version.c file
 307         __versionSourceFile="$intermediatesForBuild/version.c"
 308         if [ $__SkipGenerateVersion == 0 ]; then
 309             pwd
 310             "$__ProjectRoot/dotnet.sh" msbuild /nologo /verbosity:minimal /clp:Summary \
 311                                        /l:BinClashLogger,Tools/Microsoft.DotNet.Build.Tasks.dll\;LogFile=binclash.log \
 312                                        /p:RestoreDefaultOptimizationDataPackage=false /p:PortableBuild=true \
 313                                        /p:UsePartialNGENOptimization=false /maxcpucount \
 314                                        "$__ProjectDir/build.proj" /p:GenerateVersionSourceFile=true /t:GenerateVersionSourceFile /p:NativeVersionSourceFile=$__versionSourceFile \
 315                                        $__CommonMSBuildArgs $__UnprocessedBuildArgs
 316         else
 317             # Generate the dummy version.c, but only if it didn't exist to make sure we don't trigger unnecessary rebuild
 318             __versionSourceLine="static char sccsid[] __attribute__((used)) = \"@(#)No version information produced\";"
 319             if [ -e $__versionSourceFile ]; then
 320                 read existingVersionSourceLine < $__versionSourceFile
 321             fi
 322             if [ "$__versionSourceLine" != "$existingVersionSourceLine" ]; then
 323                 echo $__versionSourceLine > $__versionSourceFile
 324             fi
 325         fi
 326
 327
 328         pushd "$intermediatesForBuild"
 329         # Regenerate the CMake solution
 330
 331         if [[ $__GccBuild == 0 ]]; then
 332             echo "Invoking \"$__ProjectRoot/src/pal/tools/gen-buildsys-clang.sh\" \"$__ProjectRoot\" $__ClangMajorVersion \"$__ClangMinorVersion\" $platformArch $__BuildType $__CodeCoverage $generator $extraCmakeArguments $__cmakeargs"
 333             "$__ProjectRoot/src/pal/tools/gen-buildsys-clang.sh" "$__ProjectRoot" $__ClangMajorVersion "$__ClangMinorVersion" $platformArch $__BuildType $__CodeCoverage $generator "$extraCmakeArguments" "$__cmakeargs"
 334         else
 335             echo "Invoking \"$__ProjectRoot/src/pal/tools/gen-buildsys-gcc.sh\" \"$__ProjectRoot\" $__GccMajorVersion \"$__GccMinorVersion\" $platformArch $__BuildType $__CodeCoverage $generator $extraCmakeArguments $__cmakeargs"
 336             "$__ProjectRoot/src/pal/tools/gen-buildsys-gcc.sh" "$__ProjectRoot" "$__GccMajorVersion" "$__CGccMinorVersion" $platformArch $__BuildType $__CodeCoverage $generator "$extraCmakeArguments" "$__cmakeargs"
 337         fi
 338         popd
 339     fi
 340
 341     if [ ! -f "$intermediatesForBuild/$buildFile" ]; then
 342         echo "Failed to generate $message build project!"
 343         exit 1
 344     fi
 345
 346     # Build
 347     if [ $__ConfigureOnly == 1 ]; then
 348         echo "Finish configuration & skipping $message build."
 349         return
 350     fi
 351
 352     # Check that the makefiles were created.
 353     pushd "$intermediatesForBuild"
 354
 355     echo "Executing $buildTool install -j $__NumProc"
 356
 357     $buildTool install -j $__NumProc
 358     if [ $? != 0 ]; then
 359         echo "Failed to build $message."
 360         exit 1
 361     fi
 362
 363     popd
 364 }
 365
 366 build_cross_architecture_components()
 367 {
 368     local intermediatesForBuild="$__IntermediatesDir/Host$__CrossArch/crossgen"
 369     local crossArchBinDir="$__BinDir/$__CrossArch"
 370
 371     mkdir -p "$intermediatesForBuild"
 372     mkdir -p "$crossArchBinDir"
 373
 374     generate_event_logging_sources "$intermediatesForBuild" "the crossarch build system"
 375
 376     __SkipCrossArchBuild=1
 377     # check supported cross-architecture components host(__HostArch)/target(__BuildArch) pair
 378     if [[ ("$__BuildArch" == "arm" || "$__BuildArch" == "armel") && ("$__CrossArch" == "x86" || "$__CrossArch" == "x64") ]]; then
 379         __SkipCrossArchBuild=0
 380     elif [[ "$__BuildArch" == "arm64" && "$__CrossArch" == "x64" ]]; then
 381         __SkipCrossArchBuild=0
 382     else
 383         # not supported
 384         return
 385     fi
 386
 387     export __CMakeBinDir="$crossArchBinDir"
 388     export CROSSCOMPILE=0
 389
 390     __ExtraCmakeArgs="-DCLR_CMAKE_TARGET_ARCH=$__BuildArch -DCLR_CMAKE_TARGET_OS=$__BuildOS -DCLR_CMAKE_PACKAGES_DIR=$__PackagesDir -DCLR_CMAKE_PGO_INSTRUMENT=$__PgoInstrument -DCLR_CMAKE_OPTDATA_VERSION=$__PgoOptDataVersion -DCLR_CMAKE_PGO_OPTIMIZE=$__PgoOptimize -DCLR_CROSS_COMPONENTS_BUILD=1"
 391     build_native $__SkipCrossArchBuild "$__CrossArch" "$intermediatesForBuild" "$__ExtraCmakeArgs" "cross-architecture components"
 392
 393     export CROSSCOMPILE=1
 394 }
 395
 396 isMSBuildOnNETCoreSupported()
 397 {
 398     __isMSBuildOnNETCoreSupported=$__msbuildonunsupportedplatform
 399
 400     if [ $__isMSBuildOnNETCoreSupported == 1 ]; then
 401         return
 402     fi
 403
 404     if [ $__SkipManaged == 1 ]; then
 405         __isMSBuildOnNETCoreSupported=0
 406         return
 407     fi
 408
 409     if [ "$__HostArch" == "x64" ]; then
 410         if [ "$__HostOS" == "Linux" ]; then
 411             __isMSBuildOnNETCoreSupported=1
 412             # note: the RIDs below can use globbing patterns
 413             UNSUPPORTED_RIDS=("debian.9-x64" "ubuntu.17.04-x64")
 414             for UNSUPPORTED_RID in "${UNSUPPORTED_RIDS[@]}"
 415             do
 416                 if [[ $__HostDistroRid == $UNSUPPORTED_RID ]]; then
 417                     __isMSBuildOnNETCoreSupported=0
 418                     break
 419                 fi
 420             done
 421         elif [ "$__HostOS" == "OSX" ]; then
 422             __isMSBuildOnNETCoreSupported=1
 423         elif [ "$__HostOS" == "FreeBSD" ]; then
 424             __isMSBuildOnNETCoreSupported=1
 425         fi
 426     fi
 427 }
 428
 429
 430 build_CoreLib_ni()
 431 {
 432     local __CrossGenExec=$1
 433
 434     if [ $__PartialNgen == 1 ]; then
 435         export COMPlus_PartialNGen=1
 436     fi
 437
 438     if [ -e $__CrossGenCoreLibLog ]; then
 439         rm $__CrossGenCoreLibLog
 440     fi
 441     echo "Generating native image of System.Private.CoreLib.dll for $__BuildOS.$__BuildArch.$__BuildType. Logging to \"$__CrossGenCoreLibLog\"."
 442     echo "$__CrossGenExec /Platform_Assemblies_Paths $__BinDir/IL $__IbcTuning /out $__BinDir/System.Private.CoreLib.dll $__BinDir/IL/System.Private.CoreLib.dll"
 443     $__CrossGenExec /Platform_Assemblies_Paths $__BinDir/IL $__IbcTuning /out $__BinDir/System.Private.CoreLib.dll $__BinDir/IL/System.Private.CoreLib.dll >> $__CrossGenCoreLibLog 2>&1
 444     if [ $? -ne 0 ]; then
 445         echo "Failed to generate native image for System.Private.CoreLib. Refer to $__CrossGenCoreLibLog"
 446         exit 1
 447     fi
 448
 449     if [ "$__BuildOS" == "Linux" ]; then
 450         echo "Generating symbol file for System.Private.CoreLib.dll"
 451         echo "$__CrossGenExec /Platform_Assemblies_Paths $__BinDir /CreatePerfMap $__BinDir $__BinDir/System.Private.CoreLib.dll"
 452         $__CrossGenExec /Platform_Assemblies_Paths $__BinDir /CreatePerfMap $__BinDir $__BinDir/System.Private.CoreLib.dll >> $__CrossGenCoreLibLog 2>&1
 453         if [ $? -ne 0 ]; then
 454             echo "Failed to generate symbol file for System.Private.CoreLib. Refer to $__CrossGenCoreLibLog"
 455             exit 1
 456         fi
 457     fi
 458 }
 459
 460 build_CoreLib()
 461 {
 462     if [ $__isMSBuildOnNETCoreSupported == 0 ]; then
 463         echo "System.Private.CoreLib.dll build unsupported."
 464         return
 465     fi
 466
 467     if [ $__SkipMSCorLib == 1 ]; then
 468        echo "Skipping building System.Private.CoreLib."
 469        return
 470     fi
 471
 472     echo "Commencing build of managed components for $__BuildOS.$__BuildArch.$__BuildType"
 473
 474     # Invoke MSBuild
 475     __ExtraBuildArgs=""
 476     if [[ "$__IbcTuning" == "" ]]; then
 477         __ExtraBuildArgs="$__ExtraBuildArgs /p:OptimizationDataDir=\"$__PackagesDir/optimization.$__BuildOS-$__BuildArch.IBC.CoreCLR/$__IbcOptDataVersion/data\""
 478         __ExtraBuildArgs="$__ExtraBuildArgs /p:EnableProfileGuidedOptimization=true"
 479     fi
 480
 481     if [[ "$__BuildManagedTools" -eq "1" ]]; then
 482         __ExtraBuildArgs="$__ExtraBuildArgs /p:BuildManagedTools=true"
 483     fi
 484
 485     $__ProjectRoot/dotnet.sh msbuild /nologo /verbosity:minimal /clp:Summary \
 486                              /l:BinClashLogger,Tools/Microsoft.DotNet.Build.Tasks.dll\;LogFile=binclash.log \
 487                              /p:RestoreDefaultOptimizationDataPackage=false /p:PortableBuild=true \
 488                              /p:UsePartialNGENOptimization=false /maxcpucount \
 489                              $__ProjectDir/build.proj \
 490                              /flp:Verbosity=normal\;LogFile=$__LogsDir/System.Private.CoreLib_$__BuildOS__$__BuildArch__$__BuildType.log \
 491                              /p:__IntermediatesDir=$__IntermediatesDir /p:__RootBinDir=$__RootBinDir /p:BuildNugetPackage=false /p:UseSharedCompilation=false \
 492                              $__CommonMSBuildArgs $__ExtraBuildArgs $__UnprocessedBuildArgs
 493
 494     if [ $? -ne 0 ]; then
 495         echo "Failed to build managed components."
 496         exit 1
 497     fi
 498
 499     if [ $__SkipCrossgen == 1 ]; then
 500         echo "Skipping generating native image"
 501         return
 502     fi
 503
 504     # The cross build generates a crossgen with the target architecture.
 505     if [ $__CrossBuild == 0 ]; then
 506        if [ $__SkipCoreCLR == 1 ]; then
 507            return
 508        fi
 509
 510        # The architecture of host pc must be same architecture with target.
 511        if [[ ( "$__HostArch" == "$__BuildArch" ) ]]; then
 512            build_CoreLib_ni "$__BinDir/crossgen"
 513        elif [[ ( "$__HostArch" == "x64" ) && ( "$__BuildArch" == "x86" ) ]]; then
 514            build_CoreLib_ni "$__BinDir/crossgen"
 515        elif [[ ( "$__HostArch" == "arm64" ) && ( "$__BuildArch" == "arm" ) ]]; then
 516            build_CoreLib_ni "$__BinDir/crossgen"
 517        else
 518            exit 1
 519        fi
 520     else
 521        if [[ ( "$__CrossArch" == "x86" ) && ( "$__BuildArch" == "arm" ) ]]; then
 522            build_CoreLib_ni "$__CrossComponentBinDir/crossgen"
 523        elif [[ ( "$__CrossArch" == "x64" ) && ( "$__BuildArch" == "arm" ) ]]; then
 524            build_CoreLib_ni "$__CrossComponentBinDir/crossgen"
 525        elif [[ ( "$__HostArch" == "x64" ) && ( "$__BuildArch" == "arm64" ) ]]; then
 526            build_CoreLib_ni "$__CrossComponentBinDir/crossgen"
 527        fi
 528     fi
 529 }
 530
 531 generate_NugetPackages()
 532 {
 533     # We can only generate nuget package if we also support building mscorlib as part of this build.
 534     if [ $__isMSBuildOnNETCoreSupported == 0 ]; then
 535         echo "Nuget package generation unsupported."
 536         return
 537     fi
 538
 539     # Since we can build mscorlib for this OS, did we build the native components as well?
 540     if [[ $__SkipCoreCLR == 1 && $__CrossgenOnly == 0 ]]; then
 541         echo "Unable to generate nuget packages since native components were not built."
 542         return
 543     fi
 544
 545     echo "Generating nuget packages for "$__BuildOS
 546     echo "DistroRid is "$__DistroRid
 547     echo "ROOTFS_DIR is "$ROOTFS_DIR
 548     # Build the packages
 549     $__ProjectRoot/dotnet.sh msbuild /nologo /verbosity:minimal /clp:Summary \
 550                              /l:BinClashLogger,Tools/Microsoft.DotNet.Build.Tasks.dll\;LogFile=binclash.log \
 551                              /p:RestoreDefaultOptimizationDataPackage=false /p:PortableBuild=true \
 552                              /p:UsePartialNGENOptimization=false /maxcpucount \
 553                              $__SourceDir/.nuget/packages.builds \
 554                              /flp:Verbosity=normal\;LogFile=$__LogsDir/Nuget_$__BuildOS__$__BuildArch__$__BuildType.log \
 555                              /p:__IntermediatesDir=$__IntermediatesDir /p:__RootBinDir=$__RootBinDir /p:BuildNugetPackages=false /p:UseSharedCompilation=false /p:__DoCrossArchBuild=$__CrossBuild \
 556                              $__CommonMSBuildArgs $__UnprocessedBuildArgs
 557
 558     if [ $? -ne 0 ]; then
 559         echo "Failed to generate Nuget packages."
 560         exit 1
 561     fi
 562 }
 563
 564 echo "Commencing CoreCLR Repo build"
 565
 566 # Argument types supported by this script:
 567 #
 568 # Build architecture - valid values are: x64, ARM.
 569 # Build Type         - valid values are: Debug, Checked, Release
 570 #
 571 # Set the default arguments for build
 572
 573 # Obtain the location of the bash script to figure out where the root of the repo is.
 574 __ProjectRoot="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
 575
 576 # Use uname to determine what the CPU is.
 577 CPUName=$(uname -p)
 578 # Some Linux platforms report unknown for platform, but the arch for machine.
 579 if [ "$CPUName" == "unknown" ]; then
 580     CPUName=$(uname -m)
 581 fi
 582
 583 case $CPUName in
 584     i686)
 585         echo "Unsupported CPU $CPUName detected, build might not succeed!"
 586         __BuildArch=x86
 587         __HostArch=x86
 588         ;;
 589
 590     x86_64)
 591         __BuildArch=x64
 592         __HostArch=x64
 593         ;;
 594
 595     armv7l)
 596         echo "Unsupported CPU $CPUName detected, build might not succeed!"
 597         __BuildArch=arm
 598         __HostArch=arm
 599         ;;
 600
 601     aarch64)
 602         __BuildArch=arm64
 603         __HostArch=arm64
 604         ;;
 605
 606     amd64)
 607         __BuildArch=x64
 608         __HostArch=x64
 609         ;;
 610     *)
 611         echo "Unknown CPU $CPUName detected, configuring as if for x64"
 612         __BuildArch=x64
 613         __HostArch=x64
 614         ;;
 615 esac
 616
 617 # Use uname to determine what the OS is.
 618 OSName=$(uname -s)
 619 case $OSName in
 620     Linux)
 621         __BuildOS=Linux
 622         __HostOS=Linux
 623         ;;
 624
 625     Darwin)
 626         __BuildOS=OSX
 627         __HostOS=OSX
 628         ;;
 629
 630     FreeBSD)
 631         __BuildOS=FreeBSD
 632         __HostOS=FreeBSD
 633         ;;
 634
 635     OpenBSD)
 636         __BuildOS=OpenBSD
 637         __HostOS=OpenBSD
 638         ;;
 639
 640     NetBSD)
 641         __BuildOS=NetBSD
 642         __HostOS=NetBSD
 643         ;;
 644
 645     SunOS)
 646         __BuildOS=SunOS
 647         __HostOS=SunOS
 648         ;;
 649
 650     *)
 651         echo "Unsupported OS $OSName detected, configuring as if for Linux"
 652         __BuildOS=Linux
 653         __HostOS=Linux
 654         ;;
 655 esac
 656
 657 __BuildType=Debug
 658 __CodeCoverage=
 659 __IgnoreWarnings=0
 660
 661 # Set the various build properties here so that CMake and MSBuild can pick them up
 662 __ProjectDir="$__ProjectRoot"
 663 __SourceDir="$__ProjectDir/src"
 664 __PackagesDir="${DotNetRestorePackagesPath:-${__ProjectDir}/packages}"
 665 __RootBinDir="$__ProjectDir/bin"
 666 __UnprocessedBuildArgs=
 667 __CommonMSBuildArgs=
 668 __MSBCleanBuildArgs=
 669 __UseNinja=0
 670 __VerboseBuild=0
 671 __PgoInstrument=0
 672 __PgoOptimize=1
 673 __IbcTuning=""
 674 __ConfigureOnly=0
 675 __SkipConfigure=0
 676 __SkipManaged=0
 677 __SkipRestore=""
 678 __SkipNuget=0
 679 __SkipCoreCLR=0
 680 __SkipCrossArchNative=0
 681 __SkipMSCorLib=0
 682 __SkipRestoreOptData=0
 683 __SkipCrossgen=0
 684 __CrossgenOnly=0
 685 __PartialNgen=0
 686 __SkipTests=0
 687 __CrossBuild=0
 688 __ClangMajorVersion=0
 689 __ClangMinorVersion=0
 690 __GccBuild=0
 691 __GccMajorVersion=0
 692 __GccMinorVersion=0
 693 __NuGetPath="$__PackagesDir/NuGet.exe"
 694 __HostDistroRid=""
 695 __DistroRid=""
 696 __cmakeargs=""
 697 __SkipGenerateVersion=0
 698 __PortableBuild=1
 699 __msbuildonunsupportedplatform=0
 700 __PgoOptDataVersion=""
 701 __IbcOptDataVersion=""
 702 __BuildManagedTools=1
 703 __SkipRestoreArg=""
 704 __SignTypeArg=""
 705 __OfficialBuildIdArg=""
 706
 707 # Get the number of processors available to the scheduler
 708 # Other techniques such as `nproc` only get the number of
 709 # processors available to a single process.
 710 if [ `uname` = "FreeBSD" ]; then
 711   __NumProc=`sysctl hw.ncpu | awk '{ print $2+1 }'`
 712 elif [ `uname` = "NetBSD" ]; then
 713   __NumProc=$(($(getconf NPROCESSORS_ONLN)+1))
 714 elif [ `uname` = "Darwin" ]; then
 715   __NumProc=$(($(getconf _NPROCESSORS_ONLN)+1))
 716 else
 717   __NumProc=$(nproc --all)
 718 fi
 719
 720 while :; do
 721     if [ $# -le 0 ]; then
 722         break
 723     fi
 724
 725     lowerI="$(echo $1 | awk '{print tolower($0)}')"
 726     case $lowerI in
 727         -\?|-h|--help)
 728             usage
 729             exit 1
 730             ;;
 731
 732         x86|-x86)
 733             __BuildArch=x86
 734             ;;
 735
 736         x64|-x64)
 737             __BuildArch=x64
 738             ;;
 739
 740         arm|-arm)
 741             __BuildArch=arm
 742             ;;
 743
 744         armel|-armel)
 745             __BuildArch=armel
 746             ;;
 747
 748         arm64|-arm64)
 749             __BuildArch=arm64
 750             ;;
 751
 752         debug|-debug)
 753             __BuildType=Debug
 754             ;;
 755
 756         checked|-checked)
 757             __BuildType=Checked
 758             ;;
 759
 760         release|-release)
 761             __BuildType=Release
 762             ;;
 763
 764         coverage|-coverage)
 765             __CodeCoverage=Coverage
 766             ;;
 767
 768         cross|-cross)
 769             __CrossBuild=1
 770             ;;
 771
 772         -portablebuild=false)
 773             __PortableBuild=0
 774             ;;
 775
 776         verbose|-verbose)
 777             __VerboseBuild=1
 778             ;;
 779
 780         stripsymbols|-stripsymbols)
 781             __cmakeargs="$__cmakeargs -DSTRIP_SYMBOLS=true"
 782             ;;
 783
 784         clang3.5|-clang3.5)
 785             __ClangMajorVersion=3
 786             __ClangMinorVersion=5
 787             ;;
 788
 789         clang3.6|-clang3.6)
 790             __ClangMajorVersion=3
 791             __ClangMinorVersion=6
 792             ;;
 793
 794         clang3.7|-clang3.7)
 795             __ClangMajorVersion=3
 796             __ClangMinorVersion=7
 797             ;;
 798
 799         clang3.8|-clang3.8)
 800             __ClangMajorVersion=3
 801             __ClangMinorVersion=8
 802             ;;
 803
 804         clang3.9|-clang3.9)
 805             __ClangMajorVersion=3
 806             __ClangMinorVersion=9
 807             ;;
 808
 809         clang4.0|-clang4.0)
 810             __ClangMajorVersion=4
 811             __ClangMinorVersion=0
 812             ;;
 813
 814         clang5.0|-clang5.0)
 815             __ClangMajorVersion=5
 816             __ClangMinorVersion=0
 817             ;;
 818
 819         clang6.0|-clang6.0)
 820             __ClangMajorVersion=6
 821             __ClangMinorVersion=0
 822             ;;
 823
 824         clang7|-clang7)
 825             __ClangMajorVersion=7
 826             __ClangMinorVersion=
 827             ;;
 828
 829         gcc5|-gcc5)
 830             __GccMajorVersion=5
 831             __GccMinorVersion=
 832             __GccBuild=1
 833             ;;
 834
 835         gcc7|-gcc7)
 836             __GccMajorVersion=7
 837             __GccMinorVersion=
 838             __GccBuild=1
 839             ;;
 840
 841         gcc|-gcc)
 842             __GccMajorVersion=
 843             __GccMinorVersion=
 844             __GccBuild=1
 845             ;;
 846
 847         ninja|-ninja)
 848             __UseNinja=1
 849             ;;
 850
 851         pgoinstrument|-pgoinstrument)
 852             __PgoInstrument=1
 853             ;;
 854
 855         nopgooptimize|-nopgooptimize)
 856             __PgoOptimize=0
 857             __SkipRestoreOptData=1
 858             ;;
 859
 860         ibcinstrument|-ibcinstrument)
 861             __IbcTuning="/Tuning"
 862             ;;
 863
 864         configureonly|-configureonly)
 865             __ConfigureOnly=1
 866             __SkipMSCorLib=1
 867             __SkipNuget=1
 868             ;;
 869
 870         skipconfigure|-skipconfigure)
 871             __SkipConfigure=1
 872             ;;
 873
 874         skipnative|-skipnative)
 875             # Use "skipnative" to use the same option name as build.cmd.
 876             __SkipCoreCLR=1
 877             ;;
 878
 879         skipcoreclr|-skipcoreclr)
 880             # Accept "skipcoreclr" for backwards-compatibility.
 881             __SkipCoreCLR=1
 882             ;;
 883
 884         skipcrossarchnative|-skipcrossarchnative)
 885             __SkipCrossArchNative=1
 886             ;;
 887
 888         skipmanaged|-skipmanaged)
 889             __SkipManaged=1
 890             ;;
 891
 892         skipmscorlib|-skipmscorlib)
 893             __SkipMSCorLib=1
 894             ;;
 895
 896         skipgenerateversion|-skipgenerateversion)
 897             __SkipGenerateVersion=1
 898             ;;
 899
 900         skiprestoreoptdata|-skiprestoreoptdata)
 901             __SkipRestoreOptData=1
 902             ;;
 903
 904         skipcrossgen|-skipcrossgen)
 905             __SkipCrossgen=1
 906             ;;
 907
 908         crossgenonly|-crossgenonly)
 909             __SkipMSCorLib=1
 910             __SkipCoreCLR=1
 911             __CrossgenOnly=1
 912             ;;
 913         partialngen|-partialngen)
 914             __PartialNgen=1
 915             ;;
 916
 917         skiptests|-skiptests)
 918             __SkipTests=1
 919             ;;
 920
 921         skipnuget|-skipnuget|skipbuildpackages|-skipbuildpackages)
 922             __SkipNuget=1
 923             ;;
 924
 925         ignorewarnings|-ignorewarnings)
 926             __IgnoreWarnings=1
 927             __cmakeargs="$__cmakeargs -DCLR_CMAKE_WARNINGS_ARE_ERRORS=OFF"
 928             ;;
 929
 930         cmakeargs|-cmakeargs)
 931             if [ -n "$2" ]; then
 932                 __cmakeargs="$__cmakeargs $2"
 933                 shift
 934             else
 935                 echo "ERROR: 'cmakeargs' requires a non-empty option argument"
 936                 exit 1
 937             fi
 938             ;;
 939
 940         bindir|-bindir)
 941             if [ -n "$2" ]; then
 942                 __RootBinDir="$2"
 943                 if [ ! -d $__RootBinDir ]; then
 944                     mkdir $__RootBinDir
 945                 fi
 946                 __RootBinParent=$(dirname $__RootBinDir)
 947                 __RootBinName=${__RootBinDir##*/}
 948                 __RootBinDir="$(cd $__RootBinParent &>/dev/null && printf %s/%s $PWD $__RootBinName)"
 949                 shift
 950             else
 951                 echo "ERROR: 'bindir' requires a non-empty option argument"
 952                 exit 1
 953             fi
 954             ;;
 955         msbuildonunsupportedplatform|-msbuildonunsupportedplatform)
 956             __msbuildonunsupportedplatform=1
 957             ;;
 958         numproc|-numproc)
 959             if [ -n "$2" ]; then
 960               __NumProc="$2"
 961               shift
 962             else
 963               echo "ERROR: 'numproc' requires a non-empty option argument"
 964               exit 1
 965             fi
 966             ;;
 967         osgroup|-osgroup)
 968             if [ -n "$2" ]; then
 969               __BuildOS="$2"
 970               shift
 971             else
 972               echo "ERROR: 'osgroup' requires a non-empty option argument"
 973               exit 1
 974             fi
 975             ;;
 976         rebuild|-rebuild)
 977             echo "ERROR: 'Rebuild' is not supported.  Please remove it."
 978             exit 1
 979             ;;
 980
 981         -skiprestore)
 982             __SkipRestoreArg="/p:RestoreDuringBuild=false"
 983             ;;
 984
 985         -disableoss)
 986             __SignTypeArg="/p:SignType=real"
 987             ;;
 988
 989         -officialbuildid=*)
 990             __Id=$(echo $1| cut -d'=' -f 2)
 991             __OfficialBuildIdArg="/p:OfficialBuildId=$__Id"
 992             ;;
 993
 994         --)
 995             # Skip -Option=Value style argument passing
 996             ;;
 997
 998         *)
 999             __UnprocessedBuildArgs="$__UnprocessedBuildArgs $1"
1000             ;;
1001     esac
1002
1003     shift
1004 done
1005
1006 __CommonMSBuildArgs="/p:__BuildArch=$__BuildArch /p:__BuildType=$__BuildType /p:__BuildOS=$__BuildOS $__OfficialBuildIdArg $__SignTypeArg $__SkipRestoreArg"
1007
1008 # Configure environment if we are doing a verbose build
1009 if [ $__VerboseBuild == 1 ]; then
1010     export VERBOSE=1
1011     __CommonMSBuildArgs="$__CommonMSBuildArgs /v:detailed"
1012 fi
1013
1014 # Set default clang version
1015 if [[ $__ClangMajorVersion == 0 && $__ClangMinorVersion == 0 ]]; then
1016     if [[ "$__BuildArch" == "arm" || "$__BuildArch" == "armel" ]]; then
1017         __ClangMajorVersion=5
1018         __ClangMinorVersion=0
1019     else
1020         __ClangMajorVersion=3
1021         __ClangMinorVersion=9
1022     fi
1023 fi
1024
1025 if [[ "$__BuildArch" == "armel" ]]; then
1026     # Armel cross build is Tizen specific and does not support Portable RID build
1027     __PortableBuild=0
1028 fi
1029
1030 if [ $__PortableBuild == 0 ]; then
1031     __CommonMSBuildArgs="$__CommonMSBuildArgs /p:PortableBuild=false"
1032 fi
1033
1034 # Set dependent variables
1035 __LogsDir="$__RootBinDir/Logs"
1036 __MsbuildDebugLogsDir="$__LogsDir/MsbuildDebugLogs"
1037
1038 # init the host distro name
1039 initHostDistroRid
1040
1041 # Set the remaining variables based upon the determined build configuration
1042 __BinDir="$__RootBinDir/Product/$__BuildOS.$__BuildArch.$__BuildType"
1043 __PackagesBinDir="$__BinDir/.nuget"
1044 __ToolsDir="$__RootBinDir/tools"
1045 __TestWorkingDir="$__RootBinDir/tests/$__BuildOS.$__BuildArch.$__BuildType"
1046 export __IntermediatesDir="$__RootBinDir/obj/$__BuildOS.$__BuildArch.$__BuildType"
1047 __TestIntermediatesDir="$__RootBinDir/tests/obj/$__BuildOS.$__BuildArch.$__BuildType"
1048 __isMSBuildOnNETCoreSupported=0
1049 __CrossComponentBinDir="$__BinDir"
1050
1051 __CrossArch="$__HostArch"
1052 if [ $__CrossBuild == 1 ]; then
1053     __CrossComponentBinDir="$__CrossComponentBinDir/$__CrossArch"
1054 fi
1055 __CrossGenCoreLibLog="$__LogsDir/CrossgenCoreLib_$__BuildOS.$__BuildArch.$__BuildType.log"
1056
1057 # Init if MSBuild for .NET Core is supported for this platform
1058 isMSBuildOnNETCoreSupported
1059
1060 # CI_SPECIFIC - On CI machines, $HOME may not be set. In such a case, create a subfolder and set the variable to set.
1061 # This is needed by CLI to function.
1062 if [ -z "$HOME" ]; then
1063     if [ ! -d "$__ProjectDir/temp_home" ]; then
1064         mkdir temp_home
1065     fi
1066     export HOME=$__ProjectDir/temp_home
1067     echo "HOME not defined; setting it to $HOME"
1068 fi
1069
1070 # Specify path to be set for CMAKE_INSTALL_PREFIX.
1071 # This is where all built CoreClr libraries will copied to.
1072 export __CMakeBinDir="$__BinDir"
1073
1074 # Configure environment if we are doing a cross compile.
1075 if [ $__CrossBuild == 1 ]; then
1076     export CROSSCOMPILE=1
1077     if ! [[ -n "$ROOTFS_DIR" ]]; then
1078         export ROOTFS_DIR="$__ProjectRoot/cross/rootfs/$__BuildArch"
1079     fi
1080 fi
1081
1082 # init the target distro name
1083 initTargetDistroRid
1084
1085 # Make the directories necessary for build if they don't exist
1086 setup_dirs
1087
1088 # Set up the directory for MSBuild debug logs.
1089 export MSBUILDDEBUGPATH="${__MsbuildDebugLogsDir}"
1090
1091 # Check prereqs.
1092 check_prereqs
1093
1094 # Restore the package containing profile counts for profile-guided optimizations
1095 restore_optdata
1096
1097 # Generate event logging infrastructure sources
1098 generate_event_logging
1099
1100 # Build the coreclr (native) components.
1101 __ExtraCmakeArgs="-DCLR_CMAKE_TARGET_OS=$__BuildOS -DCLR_CMAKE_PACKAGES_DIR=$__PackagesDir -DCLR_CMAKE_PGO_INSTRUMENT=$__PgoInstrument -DCLR_CMAKE_OPTDATA_VERSION=$__PgoOptDataVersion -DCLR_CMAKE_PGO_OPTIMIZE=$__PgoOptimize"
1102
1103 # [TODO] Remove this when the `build-test.sh` script properly builds and deploys test assets.
1104 if [ $__SkipTests != 1 ]; then
1105     echo "Adding CMake flags to build native tests for $__BuildOS.$__BuildArch.$__BuildType"
1106     __ExtraCmakeArgs="$__ExtraCmakeArgs -DCLR_CMAKE_BUILD_TESTS=ON"
1107 fi
1108
1109 build_native $__SkipCoreCLR "$__BuildArch" "$__IntermediatesDir" "$__ExtraCmakeArgs" "CoreCLR component"
1110
1111 # Build cross-architecture components
1112 if [ $__SkipCrossArchNative != 1 ]; then
1113     if [[ $__CrossBuild == 1 ]]; then
1114         build_cross_architecture_components
1115     fi
1116 fi
1117
1118 # Build System.Private.CoreLib.
1119
1120 build_CoreLib
1121
1122 if [ $__CrossgenOnly == 1 ]; then
1123     build_CoreLib_ni "$__BinDir/crossgen"
1124 fi
1125
1126 # Generate nuget packages
1127 if [ $__SkipNuget != 1 ]; then
1128     generate_NugetPackages
1129 fi
1130
1131
1132 # Build complete
1133
1134 echo "Repo successfully built."
1135 echo "Product binaries are available at $__BinDir"
1136 exit 0