build-test.sh

   1 #!/usr/bin/env bash
   2
   3 initHostDistroRid()
   4 {
   5     __HostDistroRid=""
   6
   7     # Some OS groups should default to use the portable packages
   8     if [ "$__BuildOS" == "OSX" ]; then
   9         __PortableBuild=1
  10     fi
  11
  12     if [ "$__HostOS" == "Linux" ]; then
  13         if [ -e /etc/redhat-release ]; then
  14             local redhatRelease=$(</etc/redhat-release)
  15             if [[ $redhatRelease == "CentOS release 6."* || $redhatRelease == "Red Hat Enterprise Linux Server release 6."* ]]; then
  16                 __HostDistroRid="rhel.6-$__HostArch"
  17             else
  18                 __PortableBuild=1
  19             fi
  20         elif [ -e /etc/os-release ]; then
  21             source /etc/os-release
  22             if [[ $ID == "alpine" ]]; then
  23                 __HostDistroRid="linux-musl-$__HostArch"
  24             else
  25                 __PortableBuild=1
  26                 __HostDistroRid="$ID.$VERSION_ID-$__HostArch"
  27             fi
  28         fi
  29     elif [ "$__HostOS" == "FreeBSD" ]; then
  30         __freebsd_version=`sysctl -n kern.osrelease | cut -f1 -d'.'`
  31         __HostDistroRid="freebsd.$__freebsd_version-$__HostArch"
  32     fi
  33
  34     # Portable builds target the base RID
  35     if [ "$__PortableBuild" == 1 ]; then
  36         if [ "$__BuildOS" == "OSX" ]; then
  37             export __HostDistroRid="osx-$__BuildArch"
  38         elif [ "$__BuildOS" == "Linux" ]; then
  39             export __HostDistroRid="linux-$__BuildArch"
  40         fi
  41     fi
  42
  43     if [ "$__HostDistroRid" == "" ]; then
  44         echo "WARNING: Cannot determine runtime id for current distro."
  45     fi
  46
  47     echo "Setting __HostDistroRid to $__HostDistroRid"
  48 }
  49
  50 initTargetDistroRid()
  51 {
  52     if [ $__CrossBuild == 1 ]; then
  53         if [ "$__BuildOS" == "Linux" ]; then
  54             if [ ! -e $ROOTFS_DIR/etc/os-release ]; then
  55                 if [ -e $ROOTFS_DIR/android_platform ]; then
  56                     source $ROOTFS_DIR/android_platform
  57                     export __DistroRid="$RID"
  58                 else
  59                     echo "WARNING: Cannot determine runtime id for current distro."
  60                     export __DistroRid=""
  61                 fi
  62             else
  63                 source $ROOTFS_DIR/etc/os-release
  64                 export __DistroRid="$ID.$VERSION_ID-$__BuildArch"
  65             fi
  66         fi
  67     else
  68         export __DistroRid="$__HostDistroRid"
  69     fi
  70
  71     if [ "$ID.$VERSION_ID" == "ubuntu.16.04" ]; then
  72      export __DistroRid="ubuntu.14.04-$__BuildArch"
  73     fi
  74
  75     # Portable builds target the base RID
  76     if [ "$__PortableBuild" == 1 ]; then
  77         if [ "$__BuildOS" == "Linux" ]; then
  78             export __DistroRid="linux-$__BuildArch"
  79             export __RuntimeId="linux-$__BuildArch"
  80         elif [ "$__BuildOS" == "OSX" ]; then
  81             export __DistroRid="osx-$__BuildArch"
  82             export __RuntimeId="osx-$__BuildArch"
  83         fi
  84     fi
  85
  86     echo "__DistroRid: " $__DistroRid
  87 }
  88
  89 isMSBuildOnNETCoreSupported()
  90 {
  91     __isMSBuildOnNETCoreSupported=$__msbuildonunsupportedplatform
  92
  93     if [ $__isMSBuildOnNETCoreSupported == 1 ]; then
  94         return
  95     fi
  96
  97     if [ "$__HostArch" == "x64" ]; then
  98         if [ "$__HostOS" == "Linux" ]; then
  99             __isMSBuildOnNETCoreSupported=1
 100             UNSUPPORTED_RIDS=("debian.9-x64" "ubuntu.17.04-x64")
 101             for UNSUPPORTED_RID in "${UNSUPPORTED_RIDS[@]}"
 102             do
 103                 if [ "$__HostDistroRid" == "$UNSUPPORTED_RID" ]; then
 104                     __isMSBuildOnNETCoreSupported=0
 105                     break
 106                 fi
 107             done
 108         elif [ "$__HostOS" == "OSX" ]; then
 109             __isMSBuildOnNETCoreSupported=1
 110         fi
 111     fi
 112 }
 113
 114 generate_layout()
 115 {
 116     __TestDir=$__ProjectDir/tests
 117     __ProjectFilesDir=$__TestDir
 118     __TestBinDir=$__TestWorkingDir
 119
 120     if [ $__RebuildTests -ne 0 ]; then
 121         if [ -d "${__TestBinDir}" ]; then
 122             echo "Removing tests build dir: ${__TestBinDir}"
 123             rm -rf $__TestBinDir
 124         fi
 125     fi
 126
 127     __CMakeBinDir="${__TestBinDir}"
 128
 129     if [ -z "$__TestIntermediateDir" ]; then
 130         __TestIntermediateDir="tests/obj/${__BuildOS}.${__BuildArch}.${__BuildType}"
 131     fi
 132
 133     echo "__BuildOS: ${__BuildOS}"
 134     echo "__BuildArch: ${__BuildArch}"
 135     echo "__BuildType: ${__BuildType}"
 136     echo "__TestIntermediateDir: ${__TestIntermediateDir}"
 137
 138     if [ ! -f "$__TestBinDir" ]; then
 139         echo "Creating TestBinDir: ${__TestBinDir}"
 140         mkdir -p $__TestBinDir
 141     fi
 142     if [ ! -f "$__LogsDir" ]; then
 143         echo "Creating LogsDir: ${__LogsDir}"
 144         mkdir -p $__LogsDir
 145     fi
 146
 147     __BuildProperties="-p:OSGroup=${__BuildOS} -p:BuildOS=${__BuildOS} -p:BuildArch=${__BuildArch} -p:BuildType=${__BuildType}"
 148
 149     # =========================================================================================
 150     # ===
 151     # === Restore product binaries from packages
 152     # ===
 153     # =========================================================================================
 154
 155     build_MSBuild_projects "Restore_Packages" "${__ProjectDir}/tests/build.proj" "Restore product binaries (build tests)" "-BatchRestorePackages"
 156
 157     if [ -n "$__UpdateInvalidPackagesArg" ]; then
 158         __up=-updateinvalidpackageversion
 159     fi
 160
 161     echo "${__MsgPrefix}Creating test overlay..."
 162
 163     if [ -z "$xUnitTestBinBase" ]; then
 164       xUnitTestBinBase=$__TestWorkingDir
 165     fi
 166
 167     export CORE_ROOT=$xUnitTestBinBase/Tests/Core_Root
 168
 169     if [ -d "${CORE_ROOT}" ]; then
 170       rm -rf $CORE_ROOT
 171     fi
 172
 173     mkdir -p $CORE_ROOT
 174
 175     build_MSBuild_projects "Tests_Overlay_Managed" "${__ProjectDir}/tests/runtest.proj" "Creating test overlay" "-testOverlay"
 176
 177     chmod +x $__BinDir/corerun
 178     chmod +x $__BinDir/crossgen
 179
 180     # Make sure to copy over the pulled down packages
 181     cp -r $__BinDir/* $CORE_ROOT/ > /dev/null
 182
 183 }
 184
 185 generate_testhost()
 186 {
 187     export TEST_HOST=$xUnitTestBinBase/testhost
 188
 189     if [ -d "${TEST_HOST}" ]; then
 190         rm -rf $TEST_HOST
 191     fi
 192
 193     echo "${__MsgPrefix}Creating test overlay..."
 194     mkdir -p $TEST_HOST
 195
 196     build_MSBuild_projects "Tests_Generate_TestHost" "${__ProjectDir}/tests/runtest.proj" "Creating test host" "-testHost"
 197 }
 198
 199
 200 build_Tests()
 201 {
 202     __TestDir=$__ProjectDir/tests
 203     __ProjectFilesDir=$__TestDir
 204     __TestBinDir=$__TestWorkingDir
 205
 206     if [ -f  "${__TestWorkingDir}/build_info.json" ]; then
 207         rm  "${__TestWorkingDir}/build_info.json"
 208     fi
 209
 210     if [ $__RebuildTests -ne 0 ]; then
 211         if [ -d "${__TestBinDir}" ]; then
 212             echo "Removing tests build dir: ${__TestBinDir}"
 213             rm -rf $__TestBinDir
 214         fi
 215     fi
 216
 217     export __CMakeBinDir="${__TestBinDir}"
 218     if [ ! -d "${__TestIntermediatesDir}" ]; then
 219         mkdir -p ${__TestIntermediatesDir}
 220     fi
 221
 222     __NativeTestIntermediatesDir="${__TestIntermediatesDir}/Native"
 223     if [  ! -d "${__NativeTestIntermediatesDir}" ]; then
 224         mkdir -p ${__NativeTestIntermediatesDir}
 225     fi
 226
 227     __ManagedTestIntermediatesDir="${__TestIntermediatesDir}/Managed"
 228     if [ ! -d "${__ManagedTestIntermediatesDir}" ]; then
 229         mkdir -p ${__ManagedTestIntermediatesDir}
 230     fi
 231
 232     echo "__BuildOS: ${__BuildOS}"
 233     echo "__BuildArch: ${__BuildArch}"
 234     echo "__BuildType: ${__BuildType}"
 235     echo "__TestIntermediatesDir: ${__TestIntermediatesDir}"
 236     echo "__NativeTestIntermediatesDir: ${__NativeTestIntermediatesDir}"
 237     echo "__ManagedTestIntermediatesDir: ${__ManagedTestIntermediatesDir}"
 238
 239     if [ ! -f "$__TestBinDir" ]; then
 240         echo "Creating TestBinDir: ${__TestBinDir}"
 241         mkdir -p $__TestBinDir
 242     fi
 243     if [ ! -f "$__LogsDir" ]; then
 244         echo "Creating LogsDir: ${__LogsDir}"
 245         mkdir -p $__LogsDir
 246     fi
 247
 248     __BuildProperties="-p:OSGroup=${__BuildOS} -p:BuildOS=${__BuildOS} -p:BuildArch=${__BuildArch} -p:BuildType=${__BuildType}"
 249
 250     # =========================================================================================
 251     # ===
 252     # === Restore product binaries from packages
 253     # ===
 254     # =========================================================================================
 255
 256     if [ ${__SkipRestorePackages} != 1 ]; then
 257         build_MSBuild_projects "Restore_Product" "${__ProjectDir}/tests/build.proj" "Restore product binaries (build tests)" "-BatchRestorePackages"
 258     fi
 259
 260     if [ -n "$__BuildAgainstPackagesArg" ]; then
 261         build_MSBuild_projects "Tests_GenerateRuntimeLayout" "${__ProjectDir}/tests/runtest.proj" "Restore product binaries (run tests)" "-BinPlaceRef" "-BinPlaceProduct" "-CopyCrossgenToProduct"
 262     fi
 263
 264     if [ $__SkipNative != 1 ]; then
 265         build_native_projects "$__BuildArch" "${__NativeTestIntermediatesDir}"
 266
 267         if [ $? -ne 0 ]; then
 268             echo "${__MsgPrefix}Error: build failed. Refer to the build log files for details (above)"
 269             exit 1
 270         fi
 271     fi
 272
 273     echo "Starting the Managed Tests Build..."
 274
 275     build_MSBuild_projects "Tests_Managed" "$__ProjectDir/tests/build.proj" "Managed tests build (build tests)" "$__up"
 276
 277     if [ $? -ne 0 ]; then
 278         echo "${__MsgPrefix}Error: build failed. Refer to the build log files for details (above)"
 279         exit 1
 280     else
 281         echo "Checking the Managed Tests Build..."
 282
 283         build_MSBuild_projects "Check_Test_Build" "${__ProjectDir}/tests/runtest.proj" "Check Test Build" "-ExtraParameters:/t:CheckTestBuild"
 284
 285         if [ $? -ne 0 ]; then
 286             echo "${__MsgPrefix}Error: Check Test Build failed."
 287             exit 1
 288         fi
 289
 290         echo "Managed tests build success!"
 291     fi
 292
 293     if [ $__BuildTestWrappers -ne -0 ]; then
 294         echo "${__MsgPrefix}Creating test wrappers..."
 295
 296         export __Exclude="${__ProjectDir}/tests/issues.targets"
 297         export __BuildLogRootName="Tests_XunitWrapper"
 298
 299         # Set up directories and file names
 300         __BuildLogRootName=$subDirectoryName
 301         __BuildLog="$__LogsDir/${__BuildLogRootName}.${__BuildOS}.${__BuildArch}.${__BuildType}.log"
 302         __BuildWrn="$__LogsDir/${__BuildLogRootName}.${__BuildOS}.${__BuildArch}.${__BuildType}.wrn"
 303         __BuildErr="$__LogsDir/${__BuildLogRootName}.${__BuildOS}.${__BuildArch}.${__BuildType}.err"
 304
 305         buildVerbosity="Summary"
 306
 307         if [ $__VerboseBuild == 1 ]; then
 308             buildVerbosity="Diag"
 309         fi
 310
 311         echo "${__DotNetCli}" msbuild "${__ProjectDir}/tests/runtest.proj" /p:RestoreAdditionalProjectSources=https://dotnet.myget.org/F/dotnet-core/ /p:BuildWrappers=true /p:TargetsWindows=false /fileloggerparameters:"\"Verbosity=normal;LogFile=${__BuildLog}\"" /fileloggerparameters1:"\"WarningsOnly;LogFile=${__BuildWrn}\"" /fileloggerparameters2:"\"ErrorsOnly;LogFile=${__BuildErr}\"" /consoleloggerparameters:$buildVerbosity /p:__BuildOS=$__BuildOS /p:__BuildType=$__BuildType /p:__BuildArch=$__BuildArch
 312         "${__DotNetCli}" msbuild "${__ProjectDir}/tests/runtest.proj" /p:RestoreAdditionalProjectSources=https://dotnet.myget.org/F/dotnet-core/ /p:BuildWrappers=true /p:TargetsWindows=false /fileloggerparameters:"\"Verbosity=normal;LogFile=${__BuildLog}\"" /fileloggerparameters1:"\"WarningsOnly;LogFile=${__BuildWrn}\"" /fileloggerparameters2:"\"ErrorsOnly;LogFile=${__BuildErr}\"" /consoleloggerparameters:$buildVerbosity /p:__BuildOS=$__BuildOS /p:__BuildType=$__BuildType /p:__BuildArch=$__BuildArch
 313
 314         if [ $? -ne 0 ]; then
 315             echo "${__MsgPrefix}Error: build failed. Refer to the build log files for details (above)"
 316             exit 1
 317         else
 318             echo "XUnit Wrappers have been built."
 319             echo { "\"build_os\"": "\"${__BuildOS}\"", "\"build_arch\"": "\"${__BuildArch}\"", "\"build_type\"": "\"${__BuildType}\"" } > "${__TestWorkingDir}/build_info.json"
 320
 321         fi
 322     fi
 323
 324     if [ -n "$__UpdateInvalidPackagesArg" ]; then
 325         __up=-updateinvalidpackageversion
 326     fi
 327
 328     echo "${__MsgPrefix}Creating test overlay..."
 329
 330     generate_layout
 331
 332     if [ $__ZipTests -ne 0 ]; then
 333         echo "${__MsgPrefix}ZIP tests packages..."
 334         build_MSBuild_projects "Helix_Prep" "$__ProjectDir/tests/helixprep.proj" "Prep test binaries for Helix publishing" " "
 335     fi
 336 }
 337
 338 build_MSBuild_projects()
 339 {
 340     subDirectoryName=$1
 341     shift
 342     projectName=$1
 343     shift
 344     stepName="$1"
 345     shift
 346     extraBuildParameters=("$@")
 347
 348     # Set up directories and file names
 349     __BuildLogRootName=$subDirectoryName
 350     __BuildLog="$__LogsDir/${__BuildLogRootName}.${__BuildOS}.${__BuildArch}.${__BuildType}.log"
 351     __BuildWrn="$__LogsDir/${__BuildLogRootName}.${__BuildOS}.${__BuildArch}.${__BuildType}.wrn"
 352     __BuildErr="$__LogsDir/${__BuildLogRootName}.${__BuildOS}.${__BuildArch}.${__BuildType}.err"
 353
 354     # Use binclashlogger by default if no other logger is specified
 355     if [[ "${extraBuildParameters[*]}" == *"-MsBuildEventLogging"* ]]; then
 356         msbuildEventLogging=""
 357     else
 358         msbuildEventLogging="-MsBuildEventLogging=\"/l:BinClashLogger,Tools/Microsoft.DotNet.Build.Tasks.dll;LogFile=binclash.log\""
 359     fi
 360
 361     if [[ "$subDirectoryName" == "Tests_Managed" ]]; then
 362         # Execute msbuild managed test build in stages - workaround for excessive data retention in MSBuild ConfigCache
 363         # See https://github.com/Microsoft/msbuild/issues/2993
 364
 365         # __SkipPackageRestore and __SkipTargetingPackBuild used  to control build by tests/src/dirs.proj
 366         export __SkipPackageRestore=false
 367         export __SkipTargetingPackBuild=false
 368         export __BuildLoopCount=2
 369         export __TestGroupToBuild=1
 370         __AppendToLog=false
 371
 372         if [ -n "$__priority1" ]; then
 373             export __BuildLoopCount=16
 374             export __TestGroupToBuild=2
 375         fi
 376
 377         for (( slice=1 ; slice <= __BuildLoopCount; slice = slice + 1 ))
 378         do
 379             __msbuildLog="\"/flp:Verbosity=normal;LogFile=${__BuildLog};Append=${__AppendToLog}\""
 380             __msbuildWrn="\"/flp1:WarningsOnly;LogFile=${__BuildWrn};Append=${__AppendToLog}\""
 381             __msbuildErr="\"/flp2:ErrorsOnly;LogFile=${__BuildErr};Append=${__AppendToLog}\""
 382
 383             export TestBuildSlice=$slice
 384
 385             # Generate build command
 386             buildArgs=("-Project=$projectName" "-MsBuildLog=${__msbuildLog}" "-MsBuildWrn=${__msbuildWrn}" "-MsBuildErr=${__msbuildErr}")
 387             buildArgs+=("$msbuildEventLogging")
 388             buildArgs+=("${extraBuildParameters[@]}")
 389             buildArgs+=("${__RunArgs[@]}")
 390             buildArgs+=("${__UnprocessedBuildArgs[@]}")
 391
 392             echo "Building step '$stepName' slice=$slice via $buildCommand"
 393
 394             # Invoke MSBuild
 395             "$__ProjectRoot/run.sh" build "${buildArgs[@]}"
 396
 397             # Make sure everything is OK
 398             if [ $? -ne 0 ]; then
 399                 echo "${__MsgPrefix}Failed to build $stepName. See the build logs:"
 400                 echo "    $__BuildLog"
 401                 echo "    $__BuildWrn"
 402                 echo "    $__BuildErr"
 403                 exit 1
 404             fi
 405             export __SkipPackageRestore=true
 406             export __SkipTargetingPackBuild=true
 407             __AppendToLog=true
 408         done
 409     else
 410         __msbuildLog="\"/flp:Verbosity=normal;LogFile=${__BuildLog}\""
 411         __msbuildWrn="\"/flp1:WarningsOnly;LogFile=${__BuildWrn}\""
 412         __msbuildErr="\"/flp2:ErrorsOnly;LogFile=${__BuildErr}\""
 413
 414         # Generate build command
 415         buildArgs=("-Project=$projectName" "-MsBuildLog=${__msbuildLog}" "-MsBuildWrn=${__msbuildWrn}" "-MsBuildErr=${__msbuildErr}")
 416         buildArgs+=("$msbuildEventLogging")
 417         buildArgs+=("${extraBuildParameters[@]}")
 418         buildArgs+=("${__RunArgs[@]}")
 419         buildArgs+=("${__UnprocessedBuildArgs[@]}")
 420
 421         echo "Building step '$stepName' via $buildCommand"
 422
 423         # Invoke MSBuild
 424         "$__ProjectRoot/run.sh" build "${buildArgs[@]}"
 425
 426         # Make sure everything is OK
 427         if [ $? -ne 0 ]; then
 428             echo "${__MsgPrefix}Failed to build $stepName. See the build logs:"
 429             echo "    $__BuildLog"
 430             echo "    $__BuildWrn"
 431             echo "    $__BuildErr"
 432             exit 1
 433         fi
 434     fi
 435 }
 436
 437 build_native_projects()
 438 {
 439     platformArch="$1"
 440     intermediatesForBuild="$2"
 441
 442     extraCmakeArguments="-DCLR_CMAKE_TARGET_OS=${__BuildOS} -DCLR_CMAKE_HOST_ARCH=${platformArch}"
 443     message="native tests assets"
 444
 445     # All set to commence the build
 446     echo "Commencing build of $message for $__BuildOS.$__BuildArch.$__BuildType in $intermediatesForBuild"
 447
 448     generator=""
 449     buildFile="Makefile"
 450     buildTool="make"
 451     if [ $__UseNinja == 1 ]; then
 452         generator="ninja"
 453         buildFile="build.ninja"
 454         if ! buildTool=$(command -v ninja || command -v ninja-build); then
 455            echo "Unable to locate ninja!" 1>&2
 456            exit 1
 457         fi
 458     fi
 459
 460     if [ $__SkipConfigure == 0 ]; then
 461         # if msbuild is not supported, then set __SkipGenerateVersion to 1
 462         if [ $__isMSBuildOnNETCoreSupported == 0 ]; then __SkipGenerateVersion=1; fi
 463         # Drop version.cpp file
 464         __versionSourceFile="$intermediatesForBuild/version.cpp"
 465         if [ $__SkipGenerateVersion == 0 ]; then
 466             pwd
 467             "$__ProjectRoot/run.sh" build -Project=$__ProjectDir/build.proj -generateHeaderUnix -NativeVersionSourceFile=$__versionSourceFile -MsBuildEventLogging="/l:BinClashLogger,Tools/Microsoft.DotNet.Build.Tasks.dll;LogFile=binclash.log"  $__RunArgs $__UnprocessedBuildArgs
 468         else
 469             # Generate the dummy version.cpp, but only if it didn't exist to make sure we don't trigger unnecessary rebuild
 470             __versionSourceLine="static char sccsid[] __attribute__((used)) = \"@(#)No version information produced\";"
 471             if [ -e $__versionSourceFile ]; then
 472                 read existingVersionSourceLine < $__versionSourceFile
 473             fi
 474             if [ "$__versionSourceLine" != "$existingVersionSourceLine" ]; then
 475                 echo $__versionSourceLine > $__versionSourceFile
 476             fi
 477         fi
 478
 479         pushd "$intermediatesForBuild"
 480         # Regenerate the CMake solution
 481         echo "Invoking \"$__ProjectRoot/src/pal/tools/gen-buildsys-clang.sh\" \"$__TestDir\" $__ClangMajorVersion $__ClangMinorVersion $platformArch $__BuildType $__CodeCoverage $generator $extraCmakeArguments $__cmakeargs"
 482         "$__ProjectRoot/src/pal/tools/gen-buildsys-clang.sh" "$__TestDir" $__ClangMajorVersion $__ClangMinorVersion $platformArch $__BuildType $__CodeCoverage $generator "$extraCmakeArguments" "$__cmakeargs"
 483         popd
 484     fi
 485
 486     if [ ! -f "$intermediatesForBuild/$buildFile" ]; then
 487         echo "Failed to generate $message build project!"
 488         exit 1
 489     fi
 490
 491     # Build
 492     if [ $__ConfigureOnly == 1 ]; then
 493         echo "Finish configuration & skipping $message build."
 494         return
 495     fi
 496
 497     pushd "$intermediatesForBuild"
 498
 499     echo "Executing $buildTool install -j $__NumProc"
 500
 501     $buildTool install -j $__NumProc
 502     if [ $? != 0 ]; then
 503         echo "Failed to build $message."
 504         exit 1
 505     fi
 506
 507     popd
 508     echo "Native tests build success!"
 509 }
 510
 511 usage()
 512 {
 513     echo "Usage: $0 [BuildArch] [BuildType] [verbose] [coverage] [cross] [clangx.y] [ninja] [runtests] [bindir]"
 514     echo "BuildArch can be: x64, x86, arm, armel, arm64"
 515     echo "BuildType can be: debug, checked, release"
 516     echo "coverage - optional argument to enable code coverage build (currently supported only for Linux and OSX)."
 517     echo "ninja - target ninja instead of GNU make"
 518     echo "clangx.y - optional argument to build using clang version x.y - supported version 3.5 - 6.0"
 519     echo "cross - optional argument to signify cross compilation,"
 520     echo "      - will use ROOTFS_DIR environment variable if set."
 521     echo "crosscomponent - optional argument to build cross-architecture component,"
 522     echo "               - will use CAC_ROOTFS_DIR environment variable if set."
 523     echo "portableLinux - build for Portable Linux Distribution"
 524     echo "portablebuild - Use portable build."
 525     echo "verbose - optional argument to enable verbose build output."
 526     echo "rebuild - if tests have already been built - rebuild them"
 527     echo "skipnative: skip the native tests build"
 528     echo "generatelayoutonly - only pull down dependencies and build coreroot"
 529     echo "generatetesthostonly - only pull down dependencies and build coreroot and the CoreFX testhost"
 530     echo "buildagainstpackages - pull down and build using packages."
 531     echo "skiprestorepackages - skip package restore"
 532     echo "runtests - run tests after building them"
 533     echo "ziptests - zips CoreCLR tests & Core_Root for a Helix run"
 534     echo "bindir - output directory (defaults to $__ProjectRoot/bin)"
 535     echo "msbuildonunsupportedplatform - build managed binaries even if distro is not officially supported."
 536     echo "priority1 - include priority=1 tests in the build"
 537     exit 1
 538 }
 539
 540
 541 # Obtain the location of the bash script to figure out where the root of the repo is.
 542 __ProjectRoot="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
 543
 544 # Use uname to determine what the CPU is.
 545 CPUName=$(uname -p)
 546
 547 # Some Linux platforms report unknown for platform, but the arch for machine.
 548 if [ "$CPUName" == "unknown" ]; then
 549     CPUName=$(uname -m)
 550 fi
 551
 552 case $CPUName in
 553     i686)
 554         echo "Unsupported CPU $CPUName detected, build might not succeed!"
 555         __BuildArch=x86
 556         __HostArch=x86
 557         ;;
 558
 559     x86_64)
 560         __BuildArch=x64
 561         __HostArch=x64
 562         ;;
 563
 564     armv7l)
 565         echo "Unsupported CPU $CPUName detected, build might not succeed!"
 566         __BuildArch=arm
 567         __HostArch=arm
 568         ;;
 569
 570     aarch64)
 571         __BuildArch=arm64
 572         __HostArch=arm64
 573         ;;
 574
 575     *)
 576         echo "Unknown CPU $CPUName detected, configuring as if for x64"
 577         __BuildArch=x64
 578         __HostArch=x64
 579         ;;
 580 esac
 581
 582 # Use uname to determine what the OS is.
 583 OSName=$(uname -s)
 584 case $OSName in
 585     Linux)
 586         __BuildOS=Linux
 587         __HostOS=Linux
 588         ;;
 589
 590     Darwin)
 591         __BuildOS=OSX
 592         __HostOS=OSX
 593         ;;
 594
 595     FreeBSD)
 596         __BuildOS=FreeBSD
 597         __HostOS=FreeBSD
 598         ;;
 599
 600     OpenBSD)
 601         __BuildOS=OpenBSD
 602         __HostOS=OpenBSD
 603         ;;
 604
 605     NetBSD)
 606         __BuildOS=NetBSD
 607         __HostOS=NetBSD
 608         ;;
 609
 610     SunOS)
 611         __BuildOS=SunOS
 612         __HostOS=SunOS
 613         ;;
 614
 615     *)
 616         echo "Unsupported OS $OSName detected, configuring as if for Linux"
 617         __BuildOS=Linux
 618         __HostOS=Linux
 619         ;;
 620 esac
 621
 622 __BuildType=Debug
 623 __CodeCoverage=
 624 __IncludeTests=INCLUDE_TESTS
 625
 626 # Set the various build properties here so that CMake and MSBuild can pick them up
 627 export __ProjectDir="$__ProjectRoot"
 628 __SourceDir="$__ProjectDir/src"
 629 __PackagesDir="$__ProjectDir/packages"
 630 __RootBinDir="$__ProjectDir/bin"
 631 __BuildToolsDir="$__ProjectDir/Tools"
 632 __DotNetCli="${__BuildToolsDir}/dotnetcli/dotnet"
 633 __UnprocessedBuildArgs=
 634 __RunArgs=
 635 __MSBCleanBuildArgs=
 636 __UseNinja=0
 637 __VerboseBuild=0
 638 __SkipRestore=""
 639 __SkipNative=1 # [REMOVE] Temporarily default to skip native
 640 __SkipConfigure=0
 641 __SkipGenerateVersion=0
 642 __ConfigureOnly=0
 643 __CrossBuild=0
 644 __ClangMajorVersion=0
 645 __ClangMinorVersion=0
 646 __NuGetPath="$__PackagesDir/NuGet.exe"
 647 __HostDistroRid=""
 648 __BuildAgainstPackagesArg=
 649 __SkipRestorePackages=0
 650 __DistroRid=""
 651 __cmakeargs=""
 652 __PortableLinux=0
 653 __msbuildonunsupportedplatform=0
 654 __ZipTests=0
 655 __NativeTestIntermediatesDir=
 656 __RunTests=0
 657 __RebuildTests=0
 658 __BuildTestWrappers=1
 659 __GenerateLayoutOnly=
 660 __GenerateTestHostOnly=
 661 __priority1=
 662 CORE_ROOT=
 663
 664 while :; do
 665     if [ $# -le 0 ]; then
 666         break
 667     fi
 668
 669     lowerI="$(echo $1 | awk '{print tolower($0)}')"
 670     case $lowerI in
 671         -\?|-h|--help)
 672             usage
 673             exit 1
 674             ;;
 675
 676         x86)
 677             __BuildArch=x86
 678             ;;
 679
 680         x64)
 681             __BuildArch=x64
 682             ;;
 683
 684         arm)
 685             __BuildArch=arm
 686             ;;
 687
 688         armel)
 689             __BuildArch=armel
 690             ;;
 691
 692         arm64)
 693             __BuildArch=arm64
 694             ;;
 695
 696         debug)
 697             __BuildType=Debug
 698             ;;
 699
 700         checked)
 701             __BuildType=Checked
 702             ;;
 703
 704         release)
 705             __BuildType=Release
 706             ;;
 707
 708         coverage)
 709             __CodeCoverage=Coverage
 710             ;;
 711
 712         cross)
 713             __CrossBuild=1
 714             ;;
 715
 716         portableBuild)
 717             __PortableBuild=1
 718             ;;
 719
 720         portablelinux)
 721             if [ "$__BuildOS" == "Linux" ]; then
 722                 __PortableLinux=1
 723             else
 724                 echo "ERROR: portableLinux not supported for non-Linux platforms."
 725                 exit 1
 726             fi
 727             ;;
 728
 729         verbose)
 730         __VerboseBuild=1
 731         ;;
 732
 733         clang3.5|-clang3.5)
 734             __ClangMajorVersion=3
 735             __ClangMinorVersion=5
 736             ;;
 737
 738         clang3.6|-clang3.6)
 739             __ClangMajorVersion=3
 740             __ClangMinorVersion=6
 741             ;;
 742
 743         clang3.7|-clang3.7)
 744             __ClangMajorVersion=3
 745             __ClangMinorVersion=7
 746             ;;
 747
 748         clang3.8|-clang3.8)
 749             __ClangMajorVersion=3
 750             __ClangMinorVersion=8
 751             ;;
 752
 753         clang3.9|-clang3.9)
 754             __ClangMajorVersion=3
 755             __ClangMinorVersion=9
 756             ;;
 757
 758         clang4.0|-clang4.0)
 759             __ClangMajorVersion=4
 760             __ClangMinorVersion=0
 761             ;;
 762
 763         clang5.0|-clang5.0)
 764             __ClangMajorVersion=5
 765             __ClangMinorVersion=0
 766             ;;
 767
 768         clang6.0|-clang6.0)
 769             __ClangMajorVersion=6
 770             __ClangMinorVersion=0
 771             ;;
 772
 773         ninja)
 774             __UseNinja=1
 775             ;;
 776
 777         runtests)
 778             __RunTests=1
 779             ;;
 780
 781         rebuild)
 782             __RebuildTests=1
 783             ;;
 784
 785         skipnative|-skipnative)
 786             __SkipNative=1
 787             ;;
 788
 789         # [REMOVE] Enable native build - the temporary default is to skip native
 790         --skipnative)
 791             __SkipNative=0
 792             ;;
 793
 794         ziptests)
 795             __ZipTests=1
 796             ;;
 797
 798         generatelayoutonly)
 799             __GenerateLayoutOnly=1
 800             ;;
 801         generatetesthostonly)
 802             __GenerateTestHostOnly=1
 803             ;;
 804         buildagainstpackages)
 805             __BuildAgainstPackagesArg=1
 806             ;;
 807         skiprestorepackages)
 808             __SkipRestorePackages=1
 809             ;;
 810
 811         bindir)
 812             if [ -n "$2" ]; then
 813                 __RootBinDir="$2"
 814                 if [ ! -d $__RootBinDir ]; then
 815                     mkdir $__RootBinDir
 816                 fi
 817                 __RootBinParent=$(dirname $__RootBinDir)
 818                 __RootBinName=${__RootBinDir##*/}
 819                 __RootBinDir="$(cd $__RootBinParent &>/dev/null && printf %s/%s $PWD $__RootBinName)"
 820                 shift
 821             else
 822                 echo "ERROR: 'bindir' requires a non-empty option argument"
 823                 exit 1
 824             fi
 825             ;;
 826
 827         msbuildonunsupportedplatform)
 828             __msbuildonunsupportedplatform=1
 829             ;;
 830         priority1)
 831             __priority1=1
 832             __UnprocessedBuildArgs+=("-priority=1")
 833             ;;
 834         *)
 835             __UnprocessedBuildArgs+=("$1")
 836             ;;
 837     esac
 838
 839     shift
 840 done
 841
 842 # Get the number of processors available to the scheduler
 843 # Other techniques such as `nproc` only get the number of
 844 # processors available to a single process.
 845 if [ `uname` = "FreeBSD" ]; then
 846   __NumProc=`sysctl hw.ncpu | awk '{ print $2+1 }'`
 847 elif [ `uname` = "NetBSD" ]; then
 848   __NumProc=$(($(getconf NPROCESSORS_ONLN)+1))
 849 elif [ `uname` = "Darwin" ]; then
 850   __NumProc=$(($(getconf _NPROCESSORS_ONLN)+1))
 851 else
 852   __NumProc=$(nproc --all)
 853 fi
 854
 855 __RunArgs=("-BuildArch=$__BuildArch" "-BuildType=$__BuildType" "-BuildOS=$__BuildOS")
 856
 857 # Configure environment if we are doing a verbose build
 858 if [ $__VerboseBuild == 1 ]; then
 859     export VERBOSE=1
 860     __RunArgs+=("-verbose")
 861 fi
 862
 863 # Set default clang version
 864 if [[ $__ClangMajorVersion == 0 && $__ClangMinorVersion == 0 ]]; then
 865     if [ $__CrossBuild == 1 ]; then
 866         __ClangMajorVersion=3
 867         __ClangMinorVersion=6
 868     else
 869         __ClangMajorVersion=3
 870         __ClangMinorVersion=5
 871     fi
 872 fi
 873
 874 # Set dependent variables
 875 __LogsDir="$__RootBinDir/Logs"
 876
 877 # init the host distro name
 878 initHostDistroRid
 879
 880 # Set the remaining variables based upon the determined build configuration
 881 __BinDir="$__RootBinDir/Product/$__BuildOS.$__BuildArch.$__BuildType"
 882 __PackagesBinDir="$__BinDir/.nuget"
 883 __TestDir="$__ProjectDir/tests"
 884 __TestWorkingDir="$__RootBinDir/tests/$__BuildOS.$__BuildArch.$__BuildType"
 885 __IntermediatesDir="$__RootBinDir/obj/$__BuildOS.$__BuildArch.$__BuildType"
 886 __TestIntermediatesDir="$__RootBinDir/tests/obj/$__BuildOS.$__BuildArch.$__BuildType"
 887 __isMSBuildOnNETCoreSupported=0
 888 __CrossComponentBinDir="$__BinDir"
 889 __CrossCompIntermediatesDir="$__IntermediatesDir/crossgen"
 890
 891 __CrossArch="$__HostArch"
 892 if [[ "$__HostArch" == "x64" && "$__BuildArch" == "arm" ]]; then
 893     __CrossArch="x86"
 894 fi
 895 if [ $__CrossBuild == 1 ]; then
 896     __CrossComponentBinDir="$__CrossComponentBinDir/$__CrossArch"
 897 fi
 898 __CrossgenCoreLibLog="$__LogsDir/CrossgenCoreLib_$__BuildOS.$BuildArch.$__BuildType.log"
 899 __CrossgenExe="$__CrossComponentBinDir/crossgen"
 900
 901 isMSBuildOnNETCoreSupported
 902
 903 # CI_SPECIFIC - On CI machines, $HOME may not be set. In such a case, create a subfolder and set the variable to it.
 904 # This is needed by CLI to function.
 905 if [ -z "$HOME" ]; then
 906     if [ ! -d "$__ProjectDir/temp_home" ]; then
 907         mkdir temp_home
 908     fi
 909     export HOME=$__ProjectDir/temp_home
 910     echo "HOME not defined; setting it to $HOME"
 911 fi
 912
 913 # Configure environment if we are doing a cross compile.
 914 if [ $__CrossBuild == 1 ]; then
 915     export CROSSCOMPILE=1
 916     if ! [[ -n "$ROOTFS_DIR" ]]; then
 917         export ROOTFS_DIR="$__ProjectRoot/cross/rootfs/$__BuildArch"
 918     fi
 919 fi
 920
 921 # init the target distro name
 922 initTargetDistroRid
 923
 924 # Override tool directory
 925
 926 __CoreClrVersion=1.1.0
 927 __sharedFxDir=$__BuildToolsDir/dotnetcli/shared/Microsoft.NETCore.App/$__CoreClrVersion/
 928
 929 if [[ (-z "$__GenerateLayoutOnly") && (-z "$__GenerateTestHostOnly") ]]; then
 930     echo "Building Tests..."
 931     build_Tests
 932 else
 933     echo "Generating test layout..."
 934     generate_layout
 935     if [ ! -z "$__GenerateTestHostOnly" ]; then
 936         echo "Generating test host..."
 937         generate_testhost
 938     fi
 939 fi
 940
 941 if [ $? -ne 0 ]; then
 942     echo "Failed to build tests"
 943     exit 1
 944 fi
 945
 946 echo "${__MsgPrefix}Test build successful."
 947 echo "${__MsgPrefix}Test binaries are available at ${__TestBinDir}"
 948
 949 __testNativeBinDir=$__IntermediatesDir/tests
 950
 951 if [ $__RunTests -ne 0 ]; then
 952
 953     echo "Run Tests..."
 954
 955     echo "${__TestDir}/runtest.sh --testRootDir=$__TestBinDir --coreClrBinDir=$__BinDir --coreFxBinDir=$__sharedFxDir --testNativeBinDir=$__testNativeBinDir"
 956
 957     $__TestDir/runtest.sh --testRootDir=$__TestBinDir --coreClrBinDir=$__BinDir --coreFxBinDir=$CORE_ROOT --testNativeBinDir=$__testNativeBinDir
 958
 959     echo "Tests run successful."
 960 else
 961     echo "To run all tests use 'tests/runtests.sh' where:"
 962     echo "    testRootDir      = $__TestBinDir"
 963     echo "    coreClrBinDir    = $__BinDir"
 964     echo "    coreFxBinDir     = $CORE_ROOT"
 965     echo "    testNativeBinDir = $__testNativeBinDir"
 966     echo " -------------------------------------------------- "
 967     echo " Example runtest.sh command"
 968     echo ""
 969     echo " ./tests/runtest.sh --coreOverlayDir=$CORE_ROOT --testNativeBinDir=$__testNativeBinDir --testRootDir=$__TestBinDir --copyNativeTestBin"
 970     echo " -------------------------------------------------- "
 971     echo "To run single test use the following command:"
 972     echo "    bash ${__TestBinDir}/__TEST_PATH__/__TEST_NAME__.sh -coreroot=${CORE_ROOT}"
 973 fi
 974