build.sh

   1 #!/usr/bin/env bash
   2
   3 # Work around Jenkins CI + msbuild problem: Jenkins sometimes creates very large environment
   4 # variables, and msbuild can't handle environment blocks with such large variables. So clear
   5 # out the variables that might be too large.
   6 export ghprbCommentBody=
   7
   8 # resolve python-version to use
   9 if [ "$PYTHON" == "" ] ; then
  10     if ! PYTHON=$(command -v python3 || command -v python2 || command -v python || command -v py)
  11     then
  12        echo "Unable to locate build-dependency python!" 1>&2
  13        exit 1
  14     fi
  15 fi
  16 # validate python-dependency
  17 # useful in case of explicitly set option.
  18 if ! command -v $PYTHON > /dev/null
  19 then
  20    echo "Unable to locate build-dependency python ($PYTHON)!" 1>&2
  21    exit 1
  22 fi
  23
  24 export PYTHON
  25
  26 usage()
  27 {
  28     echo "Usage: $0 [BuildArch] [BuildType] [-verbose] [-coverage] [-cross] [-gccx.y] [-clangx.y] [-ninja] [-configureonly] [-skipconfigure] [-skipnative] [-skipcrossarchnative] [-skipmanaged] [-skipmscorlib] [-skiptests] [-stripsymbols] [-ignorewarnings] [-cmakeargs] [-bindir]"
  29     echo "BuildArch can be: -x64, -x86, -arm, -armel, -arm64"
  30     echo "BuildType can be: -debug, -checked, -release"
  31     echo "-coverage - optional argument to enable code coverage build (currently supported only for Linux and OSX)."
  32     echo "-ninja - target ninja instead of GNU make"
  33     echo "-gccx.y - optional argument to build using gcc version x.y."
  34     echo "-clangx.y - optional argument to build using clang version x.y."
  35     echo "-cross - optional argument to signify cross compilation,"
  36     echo "       - will use ROOTFS_DIR environment variable if set."
  37     echo "-nopgooptimize - do not use profile guided optimizations."
  38     echo "-pgoinstrument - generate instrumented code for profile guided optimization enabled binaries."
  39     echo "-ibcinstrument - generate IBC-tuning-enabled native images when invoking crossgen."
  40     echo "-configureonly - do not perform any builds; just configure the build."
  41     echo "-skipconfigure - skip build configuration."
  42     echo "-skipnative - do not build native components."
  43     echo "-skipcrossarchnative - do not build cross-architecture native components."
  44     echo "-skipmanaged - do not build managed components."
  45     echo "-skipmscorlib - do not build mscorlib.dll."
  46     echo "-skiptests - skip the tests in the 'tests' subdirectory."
  47     echo "-skipnuget - skip building nuget packages."
  48     echo "-skiprestoreoptdata - skip restoring optimization data used by profile-based optimizations."
  49     echo "-skipcrossgen - skip native image generation"
  50     echo "-skipmanagedtools -- skip build tools such as R2Rdump and RunInContext"
  51     echo "-crossgenonly - only run native image generation"
  52     echo "-partialngen - build CoreLib as PartialNGen"
  53     echo "-verbose - optional argument to enable verbose build output."
  54     echo "-skiprestore: skip restoring packages ^(default: packages are restored during build^)."
  55     echo "-disableoss: Disable Open Source Signing for System.Private.CoreLib."
  56     echo "-officialbuildid=^<ID^>: specify the official build ID to be used by this build."
  57     echo "-stripSymbols - Optional argument to strip native symbols during the build."
  58     echo "-skipgenerateversion - disable version generation even if MSBuild is supported."
  59     echo "-ignorewarnings - do not treat warnings as errors"
  60     echo "-cmakeargs - user-settable additional arguments passed to CMake."
  61     echo "-bindir - output directory (defaults to $__ProjectRoot/bin)"
  62     echo "-msbuildonunsupportedplatform - build managed binaries even if distro is not officially supported."
  63     echo "-numproc - set the number of build processes."
  64     echo "-portablebuild - pass -portablebuild=false to force a non-portable build."
  65     echo "-staticanalyzer - build with clang static analyzer enabled."
  66     exit 1
  67 }
  68
  69 initTargetDistroRid()
  70 {
  71     source init-distro-rid.sh
  72
  73     local passedRootfsDir=""
  74
  75     # Only pass ROOTFS_DIR if cross is specified.
  76     if (( ${__CrossBuild} == 1 )); then
  77         passedRootfsDir=${ROOTFS_DIR}
  78     #elif [ "${__BuildArch}" != "${__HostArch}" ]; then
  79     #    echo "Error, you are building a cross scenario without passing -cross."
  80     #    exit 1
  81     fi
  82
  83     initDistroRidGlobal ${__BuildOS} ${__BuildArch} ${__PortableBuild} ${passedRootfsDir}
  84 }
  85
  86 setup_dirs()
  87 {
  88     echo Setting up directories for build
  89
  90     mkdir -p "$__RootBinDir"
  91     mkdir -p "$__BinDir"
  92     mkdir -p "$__LogsDir"
  93     mkdir -p "$__MsbuildDebugLogsDir"
  94     mkdir -p "$__IntermediatesDir"
  95
  96     if [ $__CrossBuild == 1 ]; then
  97         mkdir -p "$__CrossComponentBinDir"
  98     fi
  99 }
 100
 101 # Check the system to ensure the right prereqs are in place
 102
 103 check_prereqs()
 104 {
 105     echo "Checking prerequisites..."
 106
 107     # Check presence of CMake on the path
 108     hash cmake 2>/dev/null || { echo >&2 "Please install cmake before running this script"; exit 1; }
 109
 110
 111     # Minimum required version of clang is version 4.0 for arm/armel cross build
 112     if [[ $__CrossBuild == 1 && $__GccBuild == 0 &&  ("$__BuildArch" == "arm" || "$__BuildArch" == "armel") ]]; then
 113         if ! [[ "$__ClangMajorVersion" -ge "4" ]]; then
 114             echo "Please install clang4.0 or latest for arm/armel cross build"; exit 1;
 115         fi
 116     fi
 117
 118     # Check for clang
 119     if [[ $__GccBuild == 0 ]]; then
 120         __ClangCombinedDottedVersion=$__ClangMajorVersion;
 121         if [[ "$__ClangMinorVersion" != "" ]]; then
 122             __ClangCombinedDottedVersion=$__ClangCombinedDottedVersion.$__ClangMinorVersion
 123         fi
 124         hash clang-$__ClangCombinedDottedVersion 2>/dev/null ||  hash clang$__ClangMajorVersion$__ClangMinorVersion 2>/dev/null || hash clang 2>/dev/null || { echo >&2 "Please install clang-$__ClangMajorVersion.$__ClangMinorVersion before running this script"; exit 1; }
 125     else
 126         __GccCombinedDottedVersion=$__GccMajorVersion;
 127         if [[ "$__GccMinorVersion" != "" ]]; then
 128             __GccCombinedDottedVersion=$__GccCombinedDottedVersion.$__GccMinorVersion
 129         fi
 130         hash gcc-$__GccCombinedDottedVersion 2>/dev/null ||  hash gcc$__GccMajorVersion$__GccMinorVersion 2>/dev/null || hash gcc 2>/dev/null || { echo >&2 "Please install gcc-$__GccMajorVersion.$__GccMinorVersion before running this script"; exit 1; }
 131     fi
 132
 133 }
 134
 135 restore_optdata()
 136 {
 137     # we only need optdata on a Release build
 138     if [[ "$__BuildType" != "Release" ]]; then __SkipRestoreOptData=1; fi
 139
 140     local OptDataProjectFilePath="$__ProjectRoot/src/.nuget/optdata/optdata.csproj"
 141     if [[ ( $__SkipRestoreOptData == 0 ) && ( $__isMSBuildOnNETCoreSupported == 1 ) ]]; then
 142         echo "Restoring the OptimizationData package"
 143         "$__ProjectRoot/dotnet.sh" restore /nologo /verbosity:minimal /clp:Summary /m \
 144                                    $OptDataProjectFilePath \
 145                                    $__CommonMSBuildArgs $__UnprocessedBuildArgs
 146         if [ $? != 0 ]; then
 147             echo "Failed to restore the optimization data package."
 148             exit 1
 149         fi
 150     fi
 151
 152     if [ $__isMSBuildOnNETCoreSupported == 1 ]; then
 153         # Parse the optdata package versions out of msbuild so that we can pass them on to CMake
 154
 155         local PgoDataPackageVersionOutputFile="${__IntermediatesDir}/optdataversion.txt"
 156         local IbcDataPackageVersionOutputFile="${__IntermediatesDir}/ibcoptdataversion.txt"
 157
 158         # Writes into ${PgoDataPackageVersionOutputFile}
 159         ${__ProjectDir}/dotnet.sh msbuild $OptDataProjectFilePath /t:DumpPgoDataPackageVersion ${__CommonMSBuildArgs} /p:PgoDataPackageVersionOutputFile=${PgoDataPackageVersionOutputFile} /nologo 2>&1 > /dev/null
 160         if [ $? != 0 ] || [ ! -f "${PgoDataPackageVersionOutputFile}" ]; then
 161             echo "Failed to get PGO data package version."
 162             exit $?
 163         fi
 164
 165         __PgoOptDataVersion=$(<"${PgoDataPackageVersionOutputFile}")
 166
 167         # Writes into ${IbcDataPackageVersionOutputFile}
 168         ${__ProjectDir}/dotnet.sh msbuild $OptDataProjectFilePath /t:DumpIbcDataPackageVersion ${__CommonMSBuildArgs} /p:IbcDataPackageVersionOutputFile=${IbcDataPackageVersionOutputFile} /nologo 2>&1 > /dev/null
 169         if [ $? != 0 ] || [ ! -f "${IbcDataPackageVersionOutputFile}" ]; then
 170             echo "Failed to get IBC data package version."
 171             exit $?
 172         fi
 173
 174         __IbcOptDataVersion=$(<"${IbcDataPackageVersionOutputFile}")
 175     fi
 176 }
 177
 178 generate_event_logging_sources()
 179 {
 180     __OutputDir=$1
 181     __OutputEventingDir="$__OutputDir/Eventing"
 182
 183     __PythonWarningFlags="-Wall"
 184     if [[ $__IgnoreWarnings == 0 ]]; then
 185         __PythonWarningFlags="$__PythonWarningFlags -Werror"
 186     fi
 187
 188     echo "Laying out dynamically generated EventSource classes"
 189     $PYTHON -B $__PythonWarningFlags "$__ProjectRoot/src/scripts/genRuntimeEventSources.py" --man "$__ProjectRoot/src/vm/ClrEtwAll.man" --intermediate "$__OutputEventingDir"
 190 }
 191
 192 generate_event_logging()
 193 {
 194     # Event Logging Infrastructure
 195     if [[ $__SkipMSCorLib == 0 ]]; then
 196         generate_event_logging_sources "$__IntermediatesDir"
 197     fi
 198 }
 199
 200 build_native()
 201 {
 202     skipCondition=$1
 203     platformArch="$2"
 204     intermediatesForBuild="$3"
 205     extraCmakeArguments="$4"
 206     message="$5"
 207
 208     if [ $skipCondition == 1 ]; then
 209         echo "Skipping $message build."
 210         return
 211     fi
 212
 213     # All set to commence the build
 214     echo "Commencing build of $message for $__BuildOS.$__BuildArch.$__BuildType in $intermediatesForBuild"
 215
 216     generator=""
 217     buildFile="Makefile"
 218     buildTool="make"
 219     if [ $__UseNinja == 1 ]; then
 220         generator="ninja"
 221         buildFile="build.ninja"
 222         if ! buildTool=$(command -v ninja || command -v ninja-build); then
 223            echo "Unable to locate ninja!" 1>&2
 224            exit 1
 225         fi
 226     fi
 227
 228     if [ $__SkipConfigure == 0 ]; then
 229         # if msbuild is not supported, then set __SkipGenerateVersion to 1
 230         if [ $__isMSBuildOnNETCoreSupported == 0 ]; then __SkipGenerateVersion=1; fi
 231         # Drop version.c file
 232         __versionSourceFile="$intermediatesForBuild/version.c"
 233         if [ $__SkipGenerateVersion == 0 ]; then
 234             pwd
 235             "$__ProjectRoot/eng/common/msbuild.sh" $__ProjectRoot/eng/empty.csproj \
 236                                         /p:NativeVersionFile=$__versionSourceFile \
 237                                         /p:ArcadeBuild=true /t:GenerateNativeVersionFile /restore \
 238                                         $__CommonMSBuildArgs $__UnprocessedBuildArgs
 239             if [ $? -ne 0 ]; then
 240                 echo "Failed to generate native version file."
 241                 exit $?
 242             fi
 243         else
 244             # Generate the dummy version.c, but only if it didn't exist to make sure we don't trigger unnecessary rebuild
 245             __versionSourceLine="static char sccsid[] __attribute__((used)) = \"@(#)No version information produced\";"
 246             if [ -e $__versionSourceFile ]; then
 247                 read existingVersionSourceLine < $__versionSourceFile
 248             fi
 249             if [ "$__versionSourceLine" != "$existingVersionSourceLine" ]; then
 250                 echo $__versionSourceLine > $__versionSourceFile
 251             fi
 252         fi
 253
 254
 255         pushd "$intermediatesForBuild"
 256         # Regenerate the CMake solution
 257
 258         scriptDir="$__ProjectRoot/src/pal/tools"
 259         if [[ $__GccBuild == 0 ]]; then
 260             scan_build=
 261             if [[ $__StaticAnalyzer == 1 ]]; then
 262                 scan_build=scan-build
 263             fi
 264             echo "Invoking \"$scriptDir/gen-buildsys-clang.sh\" \"$__ProjectRoot\" $__ClangMajorVersion \"$__ClangMinorVersion\" $platformArch "$scriptDir" $__BuildType $__CodeCoverage $scan_build $generator $extraCmakeArguments $__cmakeargs"
 265             source "$scriptDir/gen-buildsys-clang.sh" "$__ProjectRoot" $__ClangMajorVersion "$__ClangMinorVersion" $platformArch "$scriptDir" $__BuildType $__CodeCoverage $scan_build $generator "$extraCmakeArguments" "$__cmakeargs"
 266         else
 267             echo "Invoking \"$scriptDir/gen-buildsys-gcc.sh\" \"$__ProjectRoot\" $__GccMajorVersion \"$__GccMinorVersion\" $platformArch "$scriptDir" $__BuildType $__CodeCoverage $generator $extraCmakeArguments $__cmakeargs"
 268             source "$scriptDir/gen-buildsys-gcc.sh" "$__ProjectRoot" "$__GccMajorVersion" "$__CGccMinorVersion" $platformArch "$scriptDir" $__BuildType $__CodeCoverage $generator "$extraCmakeArguments" "$__cmakeargs"
 269         fi
 270         popd
 271     fi
 272
 273     if [ ! -f "$intermediatesForBuild/$buildFile" ]; then
 274         echo "Failed to generate $message build project!"
 275         exit 1
 276     fi
 277
 278     # Build
 279     if [ $__ConfigureOnly == 1 ]; then
 280         echo "Finish configuration & skipping $message build."
 281         return
 282     fi
 283
 284     # Check that the makefiles were created.
 285     pushd "$intermediatesForBuild"
 286
 287     if [ $__StaticAnalyzer == 1 ]; then
 288         buildTool="$SCAN_BUILD_COMMAND $buildTool"
 289     fi
 290
 291     echo "Executing $buildTool install -j $__NumProc"
 292
 293     $buildTool install -j $__NumProc
 294     if [ $? != 0 ]; then
 295         echo "Failed to build $message."
 296         exit 1
 297     fi
 298
 299     popd
 300 }
 301
 302 build_cross_architecture_components()
 303 {
 304     local intermediatesForBuild="$__IntermediatesDir/Host$__CrossArch/crossgen"
 305     local crossArchBinDir="$__BinDir/$__CrossArch"
 306
 307     mkdir -p "$intermediatesForBuild"
 308     mkdir -p "$crossArchBinDir"
 309
 310     generate_event_logging_sources "$intermediatesForBuild" "the crossarch build system"
 311
 312     __SkipCrossArchBuild=1
 313     # check supported cross-architecture components host(__HostArch)/target(__BuildArch) pair
 314     if [[ ("$__BuildArch" == "arm" || "$__BuildArch" == "armel") && ("$__CrossArch" == "x86" || "$__CrossArch" == "x64") ]]; then
 315         __SkipCrossArchBuild=0
 316     elif [[ "$__BuildArch" == "arm64" && "$__CrossArch" == "x64" ]]; then
 317         __SkipCrossArchBuild=0
 318     else
 319         # not supported
 320         return
 321     fi
 322
 323     export __CMakeBinDir="$crossArchBinDir"
 324     export CROSSCOMPILE=0
 325
 326     __ExtraCmakeArgs="-DCLR_CMAKE_TARGET_ARCH=$__BuildArch -DCLR_CMAKE_TARGET_OS=$__BuildOS -DCLR_CMAKE_PACKAGES_DIR=$__PackagesDir -DCLR_CMAKE_PGO_INSTRUMENT=$__PgoInstrument -DCLR_CMAKE_OPTDATA_VERSION=$__PgoOptDataVersion -DCLR_CMAKE_PGO_OPTIMIZE=$__PgoOptimize -DCLR_CROSS_COMPONENTS_BUILD=1"
 327     build_native $__SkipCrossArchBuild "$__CrossArch" "$intermediatesForBuild" "$__ExtraCmakeArgs" "cross-architecture components"
 328
 329     export CROSSCOMPILE=1
 330 }
 331
 332 isMSBuildOnNETCoreSupported()
 333 {
 334     __isMSBuildOnNETCoreSupported=$__msbuildonunsupportedplatform
 335
 336     if [ $__isMSBuildOnNETCoreSupported == 1 ]; then
 337         return
 338     fi
 339
 340     if [ $__SkipManaged == 1 ]; then
 341         __isMSBuildOnNETCoreSupported=0
 342         return
 343     fi
 344
 345     if [ "$__HostArch" == "x64" ]; then
 346         if [ "$__HostOS" == "Linux" ]; then
 347             __isMSBuildOnNETCoreSupported=1
 348             # note: the RIDs below can use globbing patterns
 349             UNSUPPORTED_RIDS=("ubuntu.17.04-x64")
 350             for UNSUPPORTED_RID in "${UNSUPPORTED_RIDS[@]}"
 351             do
 352                 if [[ ${__DistroRid} == $UNSUPPORTED_RID ]]; then
 353                     __isMSBuildOnNETCoreSupported=0
 354                     break
 355                 fi
 356             done
 357         elif [ "$__HostOS" == "OSX" ]; then
 358             __isMSBuildOnNETCoreSupported=1
 359         elif [ "$__HostOS" == "FreeBSD" ]; then
 360             __isMSBuildOnNETCoreSupported=1
 361         fi
 362     fi
 363 }
 364
 365
 366 build_CoreLib_ni()
 367 {
 368     local __CrossGenExec=$1
 369     local __CoreLibILDir=$2
 370
 371     if [ $__PartialNgen == 1 ]; then
 372         export COMPlus_PartialNGen=1
 373     fi
 374
 375     if [ -e $__CrossGenCoreLibLog ]; then
 376         rm $__CrossGenCoreLibLog
 377     fi
 378     echo "Generating native image of System.Private.CoreLib.dll for $__BuildOS.$__BuildArch.$__BuildType. Logging to \"$__CrossGenCoreLibLog\"."
 379     echo "$__CrossGenExec /Platform_Assemblies_Paths $__CoreLibILDir $__IbcTuning /out $__BinDir/System.Private.CoreLib.dll $__CoreLibILDir/System.Private.CoreLib.dll"
 380     $__CrossGenExec /Platform_Assemblies_Paths $__CoreLibILDir $__IbcTuning /out $__BinDir/System.Private.CoreLib.dll $__CoreLibILDir/System.Private.CoreLib.dll >> $__CrossGenCoreLibLog 2>&1
 381     if [ $? -ne 0 ]; then
 382         echo "Failed to generate native image for System.Private.CoreLib. Refer to $__CrossGenCoreLibLog"
 383         exit 1
 384     fi
 385
 386     if [ "$__BuildOS" == "Linux" ]; then
 387         echo "Generating symbol file for System.Private.CoreLib.dll"
 388         echo "$__CrossGenExec /Platform_Assemblies_Paths $__BinDir /CreatePerfMap $__BinDir $__BinDir/System.Private.CoreLib.dll"
 389         $__CrossGenExec /Platform_Assemblies_Paths $__BinDir /CreatePerfMap $__BinDir $__BinDir/System.Private.CoreLib.dll >> $__CrossGenCoreLibLog 2>&1
 390         if [ $? -ne 0 ]; then
 391             echo "Failed to generate symbol file for System.Private.CoreLib. Refer to $__CrossGenCoreLibLog"
 392             exit 1
 393         fi
 394     fi
 395 }
 396
 397 build_CoreLib()
 398 {
 399     if [ $__isMSBuildOnNETCoreSupported == 0 ]; then
 400         echo "System.Private.CoreLib.dll build unsupported."
 401         return
 402     fi
 403
 404     if [ $__SkipMSCorLib == 1 ]; then
 405        echo "Skipping building System.Private.CoreLib."
 406        return
 407     fi
 408
 409     echo "Commencing build of managed components for $__BuildOS.$__BuildArch.$__BuildType"
 410
 411     # Invoke MSBuild
 412     __ExtraBuildArgs=""
 413     if [[ "$__IbcTuning" == "" ]]; then
 414         __ExtraBuildArgs="$__ExtraBuildArgs /p:OptimizationDataDir=\"$__PackagesDir/optimization.$__BuildOS-$__BuildArch.IBC.CoreCLR/$__IbcOptDataVersion/data\""
 415         __ExtraBuildArgs="$__ExtraBuildArgs /p:EnableProfileGuidedOptimization=true"
 416     fi
 417
 418     if [[ "$__BuildManagedTools" -eq "1" ]]; then
 419         __ExtraBuildArgs="$__ExtraBuildArgs /p:BuildManagedTools=true"
 420     fi
 421
 422     $__ProjectRoot/dotnet.sh restore /nologo /verbosity:minimal /clp:Summary \
 423                              /p:PortableBuild=true /maxcpucount /p:IncludeRestoreOnlyProjects=true /p:ArcadeBuild=true\
 424                              $__ProjectDir/src/build.proj \
 425                              /flp:Verbosity=normal\;LogFile=$__LogsDir/System.Private.CoreLib_$__BuildOS__$__BuildArch__$__BuildType.log \
 426                              /p:__IntermediatesDir=$__IntermediatesDir /p:__RootBinDir=$__RootBinDir \
 427                              $__CommonMSBuildArgs $__ExtraBuildArgs $__UnprocessedBuildArgs
 428
 429     if [ $? -ne 0 ]; then
 430         echo "Failed to restore managed components."
 431         exit 1
 432     fi
 433
 434     $__ProjectRoot/dotnet.sh msbuild /nologo /verbosity:minimal /clp:Summary \
 435                              /p:PortableBuild=true /maxcpucount /p:ArcadeBuild=true\
 436                              $__ProjectDir/src/build.proj \
 437                              /flp:Verbosity=normal\;LogFile=$__LogsDir/System.Private.CoreLib_$__BuildOS__$__BuildArch__$__BuildType.log \
 438                              /p:__IntermediatesDir=$__IntermediatesDir /p:__RootBinDir=$__RootBinDir \
 439                              $__CommonMSBuildArgs $__ExtraBuildArgs $__UnprocessedBuildArgs
 440
 441     if [ $? -ne 0 ]; then
 442         echo "Failed to build managed components."
 443         exit 1
 444     fi
 445
 446     local __CoreLibILDir=$__BinDir/IL
 447
 448     if [ $__SkipCrossgen == 1 ]; then
 449         echo "Skipping generating native image"
 450
 451         #if [ $__CrossBuild == 1 ]; then
 452             # Crossgen not performed, so treat the IL version as the final version
 453             cp $__CoreLibILDir/System.Private.CoreLib.dll $__BinDir/System.Private.CoreLib.dll
 454         #fi
 455
 456         return
 457     fi
 458
 459     # The cross build generates a crossgen with the target architecture.
 460     if [ $__CrossBuild == 0 ]; then
 461        if [ $__SkipCoreCLR == 1 ]; then
 462            return
 463        fi
 464
 465        # The architecture of host pc must be same architecture with target.
 466        if [[ ( "$__HostArch" == "$__BuildArch" ) ]]; then
 467            build_CoreLib_ni "$__BinDir/crossgen" $__CoreLibILDir
 468        elif [[ ( "$__HostArch" == "x64" ) && ( "$__BuildArch" == "x86" ) ]]; then
 469            build_CoreLib_ni "$__BinDir/crossgen" $__CoreLibILDir
 470        elif [[ ( "$__HostArch" == "arm64" ) && ( "$__BuildArch" == "arm" ) ]]; then
 471            build_CoreLib_ni "$__BinDir/crossgen" $__CoreLibILDir
 472        else
 473            exit 1
 474        fi
 475     else
 476        if [[ ( "$__CrossArch" == "x86" ) && ( "$__BuildArch" == "arm" ) ]]; then
 477            build_CoreLib_ni "$__CrossComponentBinDir/crossgen" $__CoreLibILDir
 478        elif [[ ( "$__CrossArch" == "x64" ) && ( "$__BuildArch" == "arm" ) ]]; then
 479            build_CoreLib_ni "$__CrossComponentBinDir/crossgen" $__CoreLibILDir
 480        elif [[ ( "$__HostArch" == "x64" ) && ( "$__BuildArch" == "arm64" ) ]]; then
 481            build_CoreLib_ni "$__CrossComponentBinDir/crossgen" $__CoreLibILDir
 482        else
 483            # Crossgen not performed, so treat the IL version as the final version
 484            cp $__CoreLibILDir/System.Private.CoreLib.dll $__BinDir/System.Private.CoreLib.dll
 485        fi
 486     fi
 487 }
 488
 489 generate_NugetPackages()
 490 {
 491     # We can only generate nuget package if we also support building mscorlib as part of this build.
 492     if [ $__isMSBuildOnNETCoreSupported == 0 ]; then
 493         echo "Nuget package generation unsupported."
 494         return
 495     fi
 496
 497     # Since we can build mscorlib for this OS, did we build the native components as well?
 498     if [[ $__SkipCoreCLR == 1 && $__CrossgenOnly == 0 ]]; then
 499         echo "Unable to generate nuget packages since native components were not built."
 500         return
 501     fi
 502
 503     echo "Generating nuget packages for "$__BuildOS
 504     echo "DistroRid is "$__DistroRid
 505     echo "ROOTFS_DIR is "$ROOTFS_DIR
 506     # Build the packages
 507     # Package build uses the Arcade system and scripts, relying on it to restore required toolsets as part of build
 508     $__ProjectRoot/eng/common/build.sh -r -b -projects $__SourceDir/.nuget/packages.builds \
 509                                        -verbosity minimal -bl:$__LogsDir/Nuget_$__BuildOS__$__BuildArch__$__BuildType.binlog \
 510                                        /p:PortableBuild=true /p:ArcadeBuild=true \
 511                                        /p:__IntermediatesDir=$__IntermediatesDir /p:__RootBinDir=$__RootBinDir /p:__DoCrossArchBuild=$__CrossBuild \
 512                                        $__CommonMSBuildArgs $__UnprocessedBuildArgs
 513
 514     if [ $? -ne 0 ]; then
 515         echo "Failed to generate Nuget packages."
 516         exit 1
 517     fi
 518 }
 519
 520 echo "Commencing CoreCLR Repo build"
 521
 522 # Argument types supported by this script:
 523 #
 524 # Build architecture - valid values are: x64, ARM.
 525 # Build Type         - valid values are: Debug, Checked, Release
 526 #
 527 # Set the default arguments for build
 528
 529 # Obtain the location of the bash script to figure out where the root of the repo is.
 530 __ProjectRoot="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
 531
 532 # Use uname to determine what the CPU is.
 533 CPUName=$(uname -p)
 534 # Some Linux platforms report unknown for platform, but the arch for machine.
 535 if [ "$CPUName" == "unknown" ]; then
 536     CPUName=$(uname -m)
 537 fi
 538
 539 case $CPUName in
 540     i686)
 541         echo "Unsupported CPU $CPUName detected, build might not succeed!"
 542         __BuildArch=x86
 543         __HostArch=x86
 544         ;;
 545
 546     x86_64)
 547         __BuildArch=x64
 548         __HostArch=x64
 549         ;;
 550
 551     armv7l)
 552         echo "Unsupported CPU $CPUName detected, build might not succeed!"
 553         __BuildArch=arm
 554         __HostArch=arm
 555         ;;
 556
 557     aarch64)
 558         __BuildArch=arm64
 559         __HostArch=arm64
 560         ;;
 561
 562     amd64)
 563         __BuildArch=x64
 564         __HostArch=x64
 565         ;;
 566     *)
 567         echo "Unknown CPU $CPUName detected, configuring as if for x64"
 568         __BuildArch=x64
 569         __HostArch=x64
 570         ;;
 571 esac
 572
 573 # Use uname to determine what the OS is.
 574 OSName=$(uname -s)
 575 case $OSName in
 576     Linux)
 577         __BuildOS=Linux
 578         __HostOS=Linux
 579         ;;
 580
 581     Darwin)
 582         __BuildOS=OSX
 583         __HostOS=OSX
 584         ;;
 585
 586     FreeBSD)
 587         __BuildOS=FreeBSD
 588         __HostOS=FreeBSD
 589         ;;
 590
 591     OpenBSD)
 592         __BuildOS=OpenBSD
 593         __HostOS=OpenBSD
 594         ;;
 595
 596     NetBSD)
 597         __BuildOS=NetBSD
 598         __HostOS=NetBSD
 599         ;;
 600
 601     SunOS)
 602         __BuildOS=SunOS
 603         __HostOS=SunOS
 604         ;;
 605
 606     *)
 607         echo "Unsupported OS $OSName detected, configuring as if for Linux"
 608         __BuildOS=Linux
 609         __HostOS=Linux
 610         ;;
 611 esac
 612
 613 __BuildType=Debug
 614 __CodeCoverage=
 615 __IgnoreWarnings=0
 616
 617 # Set the various build properties here so that CMake and MSBuild can pick them up
 618 __ProjectDir="$__ProjectRoot"
 619 __SourceDir="$__ProjectDir/src"
 620 __PackagesDir="${DotNetRestorePackagesPath:-${__ProjectDir}/.packages}"
 621 __RootBinDir="$__ProjectDir/bin"
 622 __UnprocessedBuildArgs=
 623 __CommonMSBuildArgs=
 624 __MSBCleanBuildArgs=
 625 __UseNinja=0
 626 __VerboseBuild=0
 627 __PgoInstrument=0
 628 __PgoOptimize=1
 629 __IbcTuning=""
 630 __ConfigureOnly=0
 631 __SkipConfigure=0
 632 __SkipManaged=0
 633 __SkipRestore=""
 634 __SkipNuget=0
 635 __SkipCoreCLR=0
 636 __SkipCrossArchNative=0
 637 __SkipMSCorLib=0
 638 __SkipRestoreOptData=0
 639 __SkipCrossgen=0
 640 __CrossgenOnly=0
 641 __PartialNgen=0
 642 __SkipTests=0
 643 __CrossBuild=0
 644 __ClangMajorVersion=0
 645 __ClangMinorVersion=0
 646 __GccBuild=0
 647 __GccMajorVersion=0
 648 __GccMinorVersion=0
 649 __NuGetPath="$__PackagesDir/NuGet.exe"
 650 __DistroRid=""
 651 __cmakeargs=""
 652 __SkipGenerateVersion=0
 653 __PortableBuild=1
 654 __msbuildonunsupportedplatform=0
 655 __PgoOptDataVersion=""
 656 __IbcOptDataVersion=""
 657 __BuildManagedTools=1
 658 __SkipRestoreArg="/p:RestoreDuringBuild=true"
 659 __SignTypeArg=""
 660 __OfficialBuildIdArg=""
 661 __StaticAnalyzer=0
 662
 663 # Get the number of processors available to the scheduler
 664 # Other techniques such as `nproc` only get the number of
 665 # processors available to a single process.
 666 if [ `uname` = "FreeBSD" ]; then
 667   __NumProc=`sysctl hw.ncpu | awk '{ print $2+1 }'`
 668 elif [ `uname` = "NetBSD" ]; then
 669   __NumProc=$(($(getconf NPROCESSORS_ONLN)+1))
 670 elif [ `uname` = "Darwin" ]; then
 671   __NumProc=$(($(getconf _NPROCESSORS_ONLN)+1))
 672 else
 673   __NumProc=$(nproc --all)
 674 fi
 675
 676 while :; do
 677     if [ $# -le 0 ]; then
 678         break
 679     fi
 680
 681     lowerI="$(echo $1 | awk '{print tolower($0)}')"
 682     case $lowerI in
 683         -\?|-h|--help)
 684             usage
 685             exit 1
 686             ;;
 687
 688         x86|-x86)
 689             __BuildArch=x86
 690             ;;
 691
 692         x64|-x64)
 693             __BuildArch=x64
 694             ;;
 695
 696         arm|-arm)
 697             __BuildArch=arm
 698             ;;
 699
 700         armel|-armel)
 701             __BuildArch=armel
 702             ;;
 703
 704         arm64|-arm64)
 705             __BuildArch=arm64
 706             ;;
 707
 708         debug|-debug)
 709             __BuildType=Debug
 710             ;;
 711
 712         checked|-checked)
 713             __BuildType=Checked
 714             ;;
 715
 716         release|-release)
 717             __BuildType=Release
 718             ;;
 719
 720         coverage|-coverage)
 721             __CodeCoverage=Coverage
 722             ;;
 723
 724         cross|-cross)
 725             __CrossBuild=1
 726             ;;
 727
 728         -portablebuild=false)
 729             __PortableBuild=0
 730             ;;
 731
 732         verbose|-verbose)
 733             __VerboseBuild=1
 734             ;;
 735
 736         stripsymbols|-stripsymbols)
 737             __cmakeargs="$__cmakeargs -DSTRIP_SYMBOLS=true"
 738             ;;
 739
 740         clang3.5|-clang3.5)
 741             __ClangMajorVersion=3
 742             __ClangMinorVersion=5
 743             ;;
 744
 745         clang3.6|-clang3.6)
 746             __ClangMajorVersion=3
 747             __ClangMinorVersion=6
 748             ;;
 749
 750         clang3.7|-clang3.7)
 751             __ClangMajorVersion=3
 752             __ClangMinorVersion=7
 753             ;;
 754
 755         clang3.8|-clang3.8)
 756             __ClangMajorVersion=3
 757             __ClangMinorVersion=8
 758             ;;
 759
 760         clang3.9|-clang3.9)
 761             __ClangMajorVersion=3
 762             __ClangMinorVersion=9
 763             ;;
 764
 765         clang4.0|-clang4.0)
 766             __ClangMajorVersion=4
 767             __ClangMinorVersion=0
 768             ;;
 769
 770         clang5.0|-clang5.0)
 771             __ClangMajorVersion=5
 772             __ClangMinorVersion=0
 773             ;;
 774
 775         clang6.0|-clang6.0)
 776             __ClangMajorVersion=6
 777             __ClangMinorVersion=0
 778             ;;
 779
 780         clang7|-clang7)
 781             __ClangMajorVersion=7
 782             __ClangMinorVersion=
 783             ;;
 784
 785         gcc5|-gcc5)
 786             __GccMajorVersion=5
 787             __GccMinorVersion=
 788             __GccBuild=1
 789             ;;
 790
 791         gcc6|-gcc6)
 792             __GccMajorVersion=6
 793             __GccMinorVersion=
 794             __GccBuild=1
 795             ;;
 796
 797         gcc7|-gcc7)
 798             __GccMajorVersion=7
 799             __GccMinorVersion=
 800             __GccBuild=1
 801             ;;
 802
 803         gcc8|-gcc8)
 804             __GccMajorVersion=8
 805             __GccMinorVersion=
 806             __GccBuild=1
 807             ;;
 808
 809         gcc|-gcc)
 810             __GccMajorVersion=
 811             __GccMinorVersion=
 812             __GccBuild=1
 813             ;;
 814
 815         ninja|-ninja)
 816             __UseNinja=1
 817             ;;
 818
 819         pgoinstrument|-pgoinstrument)
 820             __PgoInstrument=1
 821             ;;
 822
 823         nopgooptimize|-nopgooptimize)
 824             __PgoOptimize=0
 825             __SkipRestoreOptData=1
 826             ;;
 827
 828         ibcinstrument|-ibcinstrument)
 829             __IbcTuning="/Tuning"
 830             ;;
 831
 832         configureonly|-configureonly)
 833             __ConfigureOnly=1
 834             __SkipMSCorLib=1
 835             __SkipNuget=1
 836             ;;
 837
 838         skipconfigure|-skipconfigure)
 839             __SkipConfigure=1
 840             ;;
 841
 842         skipnative|-skipnative)
 843             # Use "skipnative" to use the same option name as build.cmd.
 844             __SkipCoreCLR=1
 845             ;;
 846
 847         skipcoreclr|-skipcoreclr)
 848             # Accept "skipcoreclr" for backwards-compatibility.
 849             __SkipCoreCLR=1
 850             ;;
 851
 852         skipcrossarchnative|-skipcrossarchnative)
 853             __SkipCrossArchNative=1
 854             ;;
 855
 856         skipmanaged|-skipmanaged)
 857             __SkipManaged=1
 858             ;;
 859
 860         skipmscorlib|-skipmscorlib)
 861             __SkipMSCorLib=1
 862             ;;
 863
 864         skipgenerateversion|-skipgenerateversion)
 865             __SkipGenerateVersion=1
 866             ;;
 867
 868         skiprestoreoptdata|-skiprestoreoptdata)
 869             __SkipRestoreOptData=1
 870             ;;
 871
 872         skipcrossgen|-skipcrossgen)
 873             __SkipCrossgen=1
 874             ;;
 875
 876         skipmanagedtools | -skipmanagedtools)
 877             __BuildManagedTools=0
 878             ;;
 879
 880         crossgenonly|-crossgenonly)
 881             __SkipMSCorLib=1
 882             __SkipCoreCLR=1
 883             __CrossgenOnly=1
 884             ;;
 885         partialngen|-partialngen)
 886             __PartialNgen=1
 887             ;;
 888
 889         skiptests|-skiptests)
 890             __SkipTests=1
 891             ;;
 892
 893         skipnuget|-skipnuget|skipbuildpackages|-skipbuildpackages)
 894             __SkipNuget=1
 895             ;;
 896
 897         ignorewarnings|-ignorewarnings)
 898             __IgnoreWarnings=1
 899             __cmakeargs="$__cmakeargs -DCLR_CMAKE_WARNINGS_ARE_ERRORS=OFF"
 900             ;;
 901
 902         cmakeargs|-cmakeargs)
 903             if [ -n "$2" ]; then
 904                 __cmakeargs="$__cmakeargs $2"
 905                 shift
 906             else
 907                 echo "ERROR: 'cmakeargs' requires a non-empty option argument"
 908                 exit 1
 909             fi
 910             ;;
 911
 912         bindir|-bindir)
 913             if [ -n "$2" ]; then
 914                 __RootBinDir="$2"
 915                 if [ ! -d $__RootBinDir ]; then
 916                     mkdir $__RootBinDir
 917                 fi
 918                 __RootBinParent=$(dirname $__RootBinDir)
 919                 __RootBinName=${__RootBinDir##*/}
 920                 __RootBinDir="$(cd $__RootBinParent &>/dev/null && printf %s/%s $PWD $__RootBinName)"
 921                 shift
 922             else
 923                 echo "ERROR: 'bindir' requires a non-empty option argument"
 924                 exit 1
 925             fi
 926             ;;
 927         msbuildonunsupportedplatform|-msbuildonunsupportedplatform)
 928             __msbuildonunsupportedplatform=1
 929             ;;
 930         numproc|-numproc)
 931             if [ -n "$2" ]; then
 932               __NumProc="$2"
 933               shift
 934             else
 935               echo "ERROR: 'numproc' requires a non-empty option argument"
 936               exit 1
 937             fi
 938             ;;
 939         osgroup|-osgroup)
 940             if [ -n "$2" ]; then
 941               __BuildOS="$2"
 942               shift
 943             else
 944               echo "ERROR: 'osgroup' requires a non-empty option argument"
 945               exit 1
 946             fi
 947             ;;
 948         rebuild|-rebuild)
 949             echo "ERROR: 'Rebuild' is not supported.  Please remove it."
 950             exit 1
 951             ;;
 952
 953         -skiprestore)
 954             __SkipRestoreArg="/p:RestoreDuringBuild=false"
 955             ;;
 956
 957         -disableoss)
 958             __SignTypeArg="/p:SignType=real"
 959             ;;
 960
 961         -officialbuildid=*)
 962             __Id=$(echo $1| cut -d'=' -f 2)
 963             __OfficialBuildIdArg="/p:OfficialBuildId=$__Id"
 964             ;;
 965
 966         -staticanalyzer)
 967             __StaticAnalyzer=1
 968             ;;
 969
 970         --)
 971             # Skip -Option=Value style argument passing
 972             ;;
 973
 974         *)
 975             __UnprocessedBuildArgs="$__UnprocessedBuildArgs $1"
 976             ;;
 977     esac
 978
 979     shift
 980 done
 981
 982 __CommonMSBuildArgs="/p:__BuildArch=$__BuildArch /p:__BuildType=$__BuildType /p:__BuildOS=$__BuildOS /nodeReuse:false $__OfficialBuildIdArg $__SignTypeArg $__SkipRestoreArg"
 983
 984 # Configure environment if we are doing a verbose build
 985 if [ $__VerboseBuild == 1 ]; then
 986     export VERBOSE=1
 987     __CommonMSBuildArgs="$__CommonMSBuildArgs /v:detailed"
 988 fi
 989
 990 # Set default clang version
 991 if [[ $__ClangMajorVersion == 0 && $__ClangMinorVersion == 0 ]]; then
 992     if [[ "$__BuildArch" == "arm" || "$__BuildArch" == "armel" ]]; then
 993         __ClangMajorVersion=5
 994         __ClangMinorVersion=0
 995     else
 996         __ClangMajorVersion=3
 997         __ClangMinorVersion=9
 998     fi
 999 fi
1000
1001 # Set dependent variables
1002 __LogsDir="$__RootBinDir/Logs"
1003 __MsbuildDebugLogsDir="$__LogsDir/MsbuildDebugLogs"
1004
1005 # Set the remaining variables based upon the determined build configuration
1006 __BinDir="$__RootBinDir/Product/$__BuildOS.$__BuildArch.$__BuildType"
1007 __PackagesBinDir="$__BinDir/.nuget"
1008 export __IntermediatesDir="$__RootBinDir/obj/$__BuildOS.$__BuildArch.$__BuildType"
1009 __isMSBuildOnNETCoreSupported=0
1010 __CrossComponentBinDir="$__BinDir"
1011
1012 __CrossArch="$__HostArch"
1013 if [ $__CrossBuild == 1 ]; then
1014     __CrossComponentBinDir="$__CrossComponentBinDir/$__CrossArch"
1015 fi
1016 __CrossGenCoreLibLog="$__LogsDir/CrossgenCoreLib_$__BuildOS.$__BuildArch.$__BuildType.log"
1017
1018 # Configure environment if we are doing a cross compile.
1019 if [ $__CrossBuild == 1 ]; then
1020     export CROSSCOMPILE=1
1021     if ! [[ -n "$ROOTFS_DIR" ]]; then
1022         export ROOTFS_DIR="$__ProjectRoot/cross/rootfs/$__BuildArch"
1023     fi
1024 fi
1025
1026 # init the target distro name
1027 initTargetDistroRid
1028
1029 if [ $__PortableBuild == 0 ]; then
1030     __CommonMSBuildArgs="$__CommonMSBuildArgs /p:PortableBuild=false"
1031 fi
1032
1033 # Init if MSBuild for .NET Core is supported for this platform
1034 isMSBuildOnNETCoreSupported
1035
1036 # CI_SPECIFIC - On CI machines, $HOME may not be set. In such a case, create a subfolder and set the variable to set.
1037 # This is needed by CLI to function.
1038 if [ -z "$HOME" ]; then
1039     if [ ! -d "$__ProjectDir/temp_home" ]; then
1040         mkdir temp_home
1041     fi
1042     export HOME=$__ProjectDir/temp_home
1043     echo "HOME not defined; setting it to $HOME"
1044 fi
1045
1046 # Specify path to be set for CMAKE_INSTALL_PREFIX.
1047 # This is where all built CoreClr libraries will copied to.
1048 export __CMakeBinDir="$__BinDir"
1049
1050 # Make the directories necessary for build if they don't exist
1051 setup_dirs
1052
1053 # Set up the directory for MSBuild debug logs.
1054 export MSBUILDDEBUGPATH="${__MsbuildDebugLogsDir}"
1055
1056 # Check prereqs.
1057 check_prereqs
1058
1059 # Restore the package containing profile counts for profile-guided optimizations
1060 restore_optdata
1061
1062 # Generate event logging infrastructure sources
1063 generate_event_logging
1064
1065 # Build the coreclr (native) components.
1066 __ExtraCmakeArgs="-DCLR_CMAKE_TARGET_OS=$__BuildOS -DCLR_CMAKE_PACKAGES_DIR=$__PackagesDir -DCLR_CMAKE_PGO_INSTRUMENT=$__PgoInstrument -DCLR_CMAKE_OPTDATA_VERSION=$__PgoOptDataVersion -DCLR_CMAKE_PGO_OPTIMIZE=$__PgoOptimize"
1067
1068 # [TODO] Remove this when the `build-test.sh` script properly builds and deploys test assets.
1069 if [ $__SkipTests != 1 ]; then
1070     echo "Adding CMake flags to build native tests for $__BuildOS.$__BuildArch.$__BuildType"
1071     __ExtraCmakeArgs="$__ExtraCmakeArgs -DCLR_CMAKE_BUILD_TESTS=ON"
1072 fi
1073
1074 build_native $__SkipCoreCLR "$__BuildArch" "$__IntermediatesDir" "$__ExtraCmakeArgs" "CoreCLR component"
1075
1076 # Build cross-architecture components
1077 if [ $__SkipCrossArchNative != 1 ]; then
1078     if [[ $__CrossBuild == 1 ]]; then
1079         build_cross_architecture_components
1080     fi
1081 fi
1082
1083 # Build System.Private.CoreLib.
1084
1085 build_CoreLib
1086
1087 if [ $__CrossgenOnly == 1 ]; then
1088     build_CoreLib_ni "$__BinDir/crossgen"
1089 fi
1090
1091 # Generate nuget packages
1092 if [ $__SkipNuget != 1 ]; then
1093     generate_NugetPackages
1094 fi
1095
1096
1097 # Build complete
1098
1099 echo "Repo successfully built."
1100 echo "Product binaries are available at $__BinDir"
1101 exit 0