build-test.sh

   1 #!/usr/bin/env bash
   2
   3 initHostDistroRid()
   4 {
   5     __HostDistroRid=""
   6
   7     # Some OS groups should default to use the portable packages
   8     if [ "$__BuildOS" == "OSX" ]; then
   9         __PortableBuild=1
  10     fi
  11
  12     if [ "$__HostOS" == "Linux" ]; then
  13         if [ -e /etc/redhat-release ]; then
  14             __PortableBuild=1
  15         elif [ -e /etc/os-release ]; then
  16             source /etc/os-release
  17             if [[ $ID == "alpine" ]]; then
  18                 __HostDistroRid="linux-musl-$__HostArch"
  19             else
  20                 __PortableBuild=1
  21                 __HostDistroRid="$ID.$VERSION_ID-$__HostArch"
  22             fi
  23         fi
  24     elif [ "$__HostOS" == "FreeBSD" ]; then
  25         __freebsd_version=`sysctl -n kern.osrelease | cut -f1 -d'.'`
  26         __HostDistroRid="freebsd.$__freebsd_version-$__HostArch"
  27     fi
  28
  29     # Portable builds target the base RID
  30     if [ "$__PortableBuild" == 1 ]; then
  31         if [ "$__BuildOS" == "OSX" ]; then
  32             export __HostDistroRid="osx-$__BuildArch"
  33         elif [ "$__BuildOS" == "Linux" ]; then
  34             export __HostDistroRid="linux-$__BuildArch"
  35         fi
  36     fi
  37
  38     if [ "$__HostDistroRid" == "" ]; then
  39         echo "WARNING: Cannot determine runtime id for current distro."
  40     fi
  41
  42     echo "Setting __HostDistroRid to $__HostDistroRid"
  43 }
  44
  45 initTargetDistroRid()
  46 {
  47     if [ $__CrossBuild == 1 ]; then
  48         if [ "$__BuildOS" == "Linux" ]; then
  49             if [ ! -e $ROOTFS_DIR/etc/os-release ]; then
  50                 if [ -e $ROOTFS_DIR/android_platform ]; then
  51                     source $ROOTFS_DIR/android_platform
  52                     export __DistroRid="$RID"
  53                 else
  54                     echo "WARNING: Cannot determine runtime id for current distro."
  55                     export __DistroRid=""
  56                 fi
  57             else
  58                 source $ROOTFS_DIR/etc/os-release
  59                 export __DistroRid="$ID.$VERSION_ID-$__BuildArch"
  60             fi
  61         fi
  62     else
  63         export __DistroRid="$__HostDistroRid"
  64         export __RuntimeId="$__HostDistroRid"
  65     fi
  66
  67     if [ "$ID.$VERSION_ID" == "ubuntu.16.04" ]; then
  68      export __DistroRid="ubuntu.14.04-$__BuildArch"
  69     fi
  70
  71     # Portable builds target the base RID
  72     if [ "$__PortableBuild" == 1 ]; then
  73         if [ "$__BuildOS" == "Linux" ]; then
  74             export __DistroRid="linux-$__BuildArch"
  75             export __RuntimeId="linux-$__BuildArch"
  76         elif [ "$__BuildOS" == "OSX" ]; then
  77             export __DistroRid="osx-$__BuildArch"
  78             export __RuntimeId="osx-$__BuildArch"
  79         fi
  80     fi
  81
  82     echo "__DistroRid: " $__DistroRid
  83 }
  84
  85 isMSBuildOnNETCoreSupported()
  86 {
  87     __isMSBuildOnNETCoreSupported=$__msbuildonunsupportedplatform
  88
  89     if [ $__isMSBuildOnNETCoreSupported == 1 ]; then
  90         return
  91     fi
  92
  93     if [ "$__HostArch" == "x64" ]; then
  94         if [ "$__HostOS" == "Linux" ]; then
  95             __isMSBuildOnNETCoreSupported=1
  96             UNSUPPORTED_RIDS=("debian.9-x64" "ubuntu.17.04-x64")
  97             for UNSUPPORTED_RID in "${UNSUPPORTED_RIDS[@]}"
  98             do
  99                 if [ "$__HostDistroRid" == "$UNSUPPORTED_RID" ]; then
 100                     __isMSBuildOnNETCoreSupported=0
 101                     break
 102                 fi
 103             done
 104         elif [ "$__HostOS" == "OSX" ]; then
 105             __isMSBuildOnNETCoreSupported=1
 106         fi
 107     fi
 108 }
 109
 110 build_test_wrappers()
 111 {
 112     if [ $__BuildTestWrappers -ne -0 ]; then
 113         echo "${__MsgPrefix}Creating test wrappers..."
 114
 115         export __Exclude="${__ProjectDir}/tests/issues.targets"
 116         export __BuildLogRootName="Tests_XunitWrapper"
 117
 118         buildVerbosity="Summary"
 119
 120         if [ $__VerboseBuild == 1 ]; then
 121             buildVerbosity="Diag"
 122         fi
 123
 124         # Set up directories and file names
 125         __BuildLogRootName=$subDirectoryName
 126         __BuildLog="$__LogsDir/${__BuildLogRootName}.${__BuildOS}.${__BuildArch}.${__BuildType}.log"
 127         __BuildWrn="$__LogsDir/${__BuildLogRootName}.${__BuildOS}.${__BuildArch}.${__BuildType}.wrn"
 128         __BuildErr="$__LogsDir/${__BuildLogRootName}.${__BuildOS}.${__BuildArch}.${__BuildType}.err"
 129         __MsbuildLog="/fileloggerparameters:\"Verbosity=normal;LogFile=${__BuildLog}\""
 130         __MsbuildWrn="/fileloggerparameters1:\"WarningsOnly;LogFile=${__BuildWrn}\""
 131         __MsbuildErr="/fileloggerparameters2:\"ErrorsOnly;LogFile=${__BuildErr}\""
 132         __Logging="$__MsbuildLog $__MsbuildWrn $__MsbuildErr /consoleloggerparameters:$buildVerbosity"
 133
 134         nextCommand="\"${__DotNetCli}\" msbuild \"${__ProjectDir}/tests/runtest.proj\" /p:RestoreAdditionalProjectSources=https://dotnet.myget.org/F/dotnet-core/ /p:BuildWrappers=true /p:TargetsWindows=false $__Logging /p:__BuildOS=$__BuildOS /p:__BuildType=$__BuildType /p:__BuildArch=$__BuildArch"
 135         echo "$nextCommand"
 136         eval $nextCommand
 137
 138         if [ $? -ne 0 ]; then
 139             echo "${__MsgPrefix}Error: build failed. Refer to the build log files for details (above)"
 140             exit 1
 141         else
 142             echo "XUnit Wrappers have been built."
 143             echo { "\"build_os\"": "\"${__BuildOS}\"", "\"build_arch\"": "\"${__BuildArch}\"", "\"build_type\"": "\"${__BuildType}\"" } > "${__TestWorkingDir}/build_info.json"
 144
 145         fi
 146     fi
 147 }
 148
 149 generate_layout()
 150 {
 151     echo "${__MsgPrefix}Creating test overlay..."
 152
 153     __TestDir=$__ProjectDir/tests
 154     __ProjectFilesDir=$__TestDir
 155     __TestBinDir=$__TestWorkingDir
 156
 157     if [ $__RebuildTests -ne 0 ]; then
 158         if [ -d "${__TestBinDir}" ]; then
 159             echo "Removing tests build dir: ${__TestBinDir}"
 160             rm -rf $__TestBinDir
 161         fi
 162     fi
 163
 164     __CMakeBinDir="${__TestBinDir}"
 165
 166     if [ -z "$__TestIntermediateDir" ]; then
 167         __TestIntermediateDir="tests/obj/${__BuildOS}.${__BuildArch}.${__BuildType}"
 168     fi
 169
 170     echo "__BuildOS: ${__BuildOS}"
 171     echo "__BuildArch: ${__BuildArch}"
 172     echo "__BuildType: ${__BuildType}"
 173     echo "__TestIntermediateDir: ${__TestIntermediateDir}"
 174
 175     if [ ! -f "$__TestBinDir" ]; then
 176         echo "Creating TestBinDir: ${__TestBinDir}"
 177         mkdir -p $__TestBinDir
 178     fi
 179     if [ ! -f "$__LogsDir" ]; then
 180         echo "Creating LogsDir: ${__LogsDir}"
 181         mkdir -p $__LogsDir
 182     fi
 183     if [ ! -f "$__MsbuildDebugLogsDir" ]; then
 184         echo "Creating MsbuildDebugLogsDir: ${__MsbuildDebugLogsDir}"
 185         mkdir -p $__MsbuildDebugLogsDir
 186     fi
 187
 188     # Set up the directory for MSBuild debug logs.
 189     export MSBUILDDEBUGPATH="${__MsbuildDebugLogsDir}"
 190
 191     __BuildProperties="-p:OSGroup=${__BuildOS} -p:BuildOS=${__BuildOS} -p:BuildArch=${__BuildArch} -p:BuildType=${__BuildType}"
 192
 193     # =========================================================================================
 194     # ===
 195     # === Restore product binaries from packages
 196     # ===
 197     # =========================================================================================
 198
 199     build_MSBuild_projects "Restore_Packages" "${__ProjectDir}/tests/build.proj" "Restore product binaries (build tests)" "/t:BatchRestorePackages"
 200
 201     if [ -n "$__UpdateInvalidPackagesArg" ]; then
 202         __up="/t:UpdateInvalidPackageVersions"
 203     fi
 204
 205     echo "${__MsgPrefix}Creating test overlay..."
 206
 207     if [ -z "$xUnitTestBinBase" ]; then
 208         xUnitTestBinBase=$__TestWorkingDir
 209     fi
 210
 211     export CORE_ROOT=$xUnitTestBinBase/Tests/Core_Root
 212
 213     if [ -d "${CORE_ROOT}" ]; then
 214         rm -rf $CORE_ROOT
 215     fi
 216
 217     mkdir -p $CORE_ROOT
 218
 219     build_MSBuild_projects "Tests_Overlay_Managed" "${__ProjectDir}/tests/runtest.proj" "Creating test overlay" "/t:CreateTestOverlay"
 220
 221     chmod +x $__BinDir/corerun
 222     chmod +x $__BinDir/crossgen
 223
 224     # Make sure to copy over the pulled down packages
 225     cp -r $__BinDir/* $CORE_ROOT/ > /dev/null
 226 }
 227
 228 generate_testhost()
 229 {
 230     echo "${__MsgPrefix}Generating test host..."
 231
 232     export TEST_HOST=$xUnitTestBinBase/testhost
 233
 234     if [ -d "${TEST_HOST}" ]; then
 235         rm -rf $TEST_HOST
 236     fi
 237
 238     mkdir -p $TEST_HOST
 239
 240     build_MSBuild_projects "Tests_Generate_TestHost" "${__ProjectDir}/tests/runtest.proj" "Creating test host" "/t:CreateTestHost"
 241 }
 242
 243
 244 build_Tests()
 245 {
 246     echo "${__MsgPrefix}Building Tests..."
 247
 248     __TestDir=$__ProjectDir/tests
 249     __ProjectFilesDir=$__TestDir
 250     __TestBinDir=$__TestWorkingDir
 251
 252     if [ -f  "${__TestWorkingDir}/build_info.json" ]; then
 253         rm  "${__TestWorkingDir}/build_info.json"
 254     fi
 255
 256     if [ $__RebuildTests -ne 0 ]; then
 257         if [ -d "${__TestBinDir}" ]; then
 258             echo "Removing tests build dir: ${__TestBinDir}"
 259             rm -rf $__TestBinDir
 260         fi
 261     fi
 262
 263     export __CMakeBinDir="${__TestBinDir}"
 264     if [ ! -d "${__TestIntermediatesDir}" ]; then
 265         mkdir -p ${__TestIntermediatesDir}
 266     fi
 267
 268     __NativeTestIntermediatesDir="${__TestIntermediatesDir}/Native"
 269     if [  ! -d "${__NativeTestIntermediatesDir}" ]; then
 270         mkdir -p ${__NativeTestIntermediatesDir}
 271     fi
 272
 273     __ManagedTestIntermediatesDir="${__TestIntermediatesDir}/Managed"
 274     if [ ! -d "${__ManagedTestIntermediatesDir}" ]; then
 275         mkdir -p ${__ManagedTestIntermediatesDir}
 276     fi
 277
 278     echo "__BuildOS: ${__BuildOS}"
 279     echo "__BuildArch: ${__BuildArch}"
 280     echo "__BuildType: ${__BuildType}"
 281     echo "__TestIntermediatesDir: ${__TestIntermediatesDir}"
 282     echo "__NativeTestIntermediatesDir: ${__NativeTestIntermediatesDir}"
 283     echo "__ManagedTestIntermediatesDir: ${__ManagedTestIntermediatesDir}"
 284
 285     if [ ! -f "$__TestBinDir" ]; then
 286         echo "Creating TestBinDir: ${__TestBinDir}"
 287         mkdir -p $__TestBinDir
 288     fi
 289     if [ ! -f "$__LogsDir" ]; then
 290         echo "Creating LogsDir: ${__LogsDir}"
 291         mkdir -p $__LogsDir
 292     fi
 293     if [ ! -f "$__MsbuildDebugLogsDir" ]; then
 294         echo "Creating MsbuildDebugLogsDir: ${__MsbuildDebugLogsDir}"
 295         mkdir -p $__MsbuildDebugLogsDir
 296     fi
 297
 298     # Set up the directory for MSBuild debug logs.
 299     export MSBUILDDEBUGPATH="${__MsbuildDebugLogsDir}"
 300
 301     __BuildProperties="-p:OSGroup=${__BuildOS} -p:BuildOS=${__BuildOS} -p:BuildArch=${__BuildArch} -p:BuildType=${__BuildType}"
 302
 303     # =========================================================================================
 304     # ===
 305     # === Restore product binaries from packages
 306     # ===
 307     # =========================================================================================
 308
 309     if [ ${__SkipRestorePackages} != 1 ]; then
 310         build_MSBuild_projects "Restore_Product" "${__ProjectDir}/tests/build.proj" "Restore product binaries (build tests)" "/t:BatchRestorePackages"
 311     fi
 312
 313     if [ $__SkipNative != 1 ]; then
 314         build_native_projects "$__BuildArch" "${__NativeTestIntermediatesDir}"
 315
 316         if [ $? -ne 0 ]; then
 317             echo "${__MsgPrefix}Error: build failed. Refer to the build log files for details (above)"
 318             exit 1
 319         fi
 320     fi
 321
 322     if [ $__SkipManaged != 1 ]; then
 323         echo "Starting the Managed Tests Build..."
 324
 325         build_MSBuild_projects "Tests_Managed" "$__ProjectDir/tests/build.proj" "Managed tests build (build tests)" "$__up"
 326
 327         if [ $? -ne 0 ]; then
 328             echo "${__MsgPrefix}Error: build failed. Refer to the build log files for details (above)"
 329             exit 1
 330         else
 331             echo "Checking the Managed Tests Build..."
 332
 333             build_MSBuild_projects "Check_Test_Build" "${__ProjectDir}/tests/runtest.proj" "Check Test Build" "/t:CheckTestBuild"
 334
 335             if [ $? -ne 0 ]; then
 336                 echo "${__MsgPrefix}Error: Check Test Build failed."
 337                 exit 1
 338             fi
 339         fi
 340
 341         echo "Managed tests build success!"
 342     fi
 343
 344     build_test_wrappers
 345
 346     if [ -n "$__UpdateInvalidPackagesArg" ]; then
 347         __up="/t:UpdateInvalidPackageVersions"
 348     fi
 349
 350     generate_layout
 351
 352     if [ $__ZipTests -ne 0 ]; then
 353         echo "${__MsgPrefix}ZIP tests packages..."
 354         build_MSBuild_projects "Helix_Prep" "$__ProjectDir/tests/helixprep.proj" "Prep test binaries for Helix publishing" " "
 355     fi
 356 }
 357
 358 build_MSBuild_projects()
 359 {
 360     subDirectoryName=$1
 361     shift
 362     projectName=$1
 363     shift
 364     stepName="$1"
 365     shift
 366     extraBuildParameters=("$@")
 367
 368     # Set up directories and file names
 369     __BuildLogRootName=$subDirectoryName
 370     __BuildLog="$__LogsDir/${__BuildLogRootName}.${__BuildOS}.${__BuildArch}.${__BuildType}.log"
 371     __BuildWrn="$__LogsDir/${__BuildLogRootName}.${__BuildOS}.${__BuildArch}.${__BuildType}.wrn"
 372     __BuildErr="$__LogsDir/${__BuildLogRootName}.${__BuildOS}.${__BuildArch}.${__BuildType}.err"
 373
 374     # Use binclashlogger by default if no other logger is specified
 375     if [[ "${extraBuildParameters[*]}" == *"/l:"* ]]; then
 376         __msbuildEventLogging=
 377     else
 378         __msbuildEventLogging="/l:BinClashLogger,Tools/Microsoft.DotNet.Build.Tasks.dll\;LogFile=binclash.log"
 379     fi
 380
 381     if [[ "$subDirectoryName" == "Tests_Managed" ]]; then
 382         # Execute msbuild managed test build in stages - workaround for excessive data retention in MSBuild ConfigCache
 383         # See https://github.com/Microsoft/msbuild/issues/2993
 384
 385         # __SkipPackageRestore and __SkipTargetingPackBuild used  to control build by tests/src/dirs.proj
 386         export __SkipPackageRestore=false
 387         export __SkipTargetingPackBuild=false
 388         export __BuildLoopCount=2
 389         export __TestGroupToBuild=1
 390         __AppendToLog=false
 391
 392         if [ -n "$__priority1" ]; then
 393             export __BuildLoopCount=16
 394             export __TestGroupToBuild=2
 395         fi
 396
 397         for (( slice=1 ; slice <= __BuildLoopCount; slice = slice + 1 ))
 398         do
 399             __msbuildLog="\"/flp:Verbosity=normal;LogFile=${__BuildLog};Append=${__AppendToLog}\""
 400             __msbuildWrn="\"/flp1:WarningsOnly;LogFile=${__BuildWrn};Append=${__AppendToLog}\""
 401             __msbuildErr="\"/flp2:ErrorsOnly;LogFile=${__BuildErr};Append=${__AppendToLog}\""
 402
 403             export TestBuildSlice=$slice
 404
 405             # Generate build command
 406             buildArgs=("/nologo" "/verbosity:minimal" "/clp:Summary")
 407             buildArgs+=("/p:RestoreDefaultOptimizationDataPackage=false" "/p:PortableBuild=true")
 408             buildArgs+=("/p:UsePartialNGENOptimization=false" "/maxcpucount")
 409
 410             buildArgs+=("$projectName" "${__msbuildLog}" "${__msbuildWrn}" "${__msbuildErr}")
 411             buildArgs+=("$__msbuildEventLogging")
 412             buildArgs+=("${extraBuildParameters[@]}")
 413             buildArgs+=("${__CommonMSBuildArgs[@]}")
 414             buildArgs+=("${__UnprocessedBuildArgs[@]}")
 415
 416             nextCommand="\"$__ProjectRoot/dotnet.sh\" msbuild ${buildArgs[@]}"
 417             echo "Building step '$stepName' slice=$slice via $nextCommand"
 418             eval $nextCommand
 419
 420             # Make sure everything is OK
 421             if [ $? -ne 0 ]; then
 422                 echo "${__MsgPrefix}Failed to build $stepName. See the build logs:"
 423                 echo "    $__BuildLog"
 424                 echo "    $__BuildWrn"
 425                 echo "    $__BuildErr"
 426                 exit 1
 427             fi
 428             export __SkipPackageRestore=true
 429             export __SkipTargetingPackBuild=true
 430             __AppendToLog=true
 431         done
 432     else
 433         __msbuildLog="\"/flp:Verbosity=normal;LogFile=${__BuildLog}\""
 434         __msbuildWrn="\"/flp1:WarningsOnly;LogFile=${__BuildWrn}\""
 435         __msbuildErr="\"/flp2:ErrorsOnly;LogFile=${__BuildErr}\""
 436
 437         # Generate build command
 438         buildArgs=("/nologo" "/verbosity:minimal" "/clp:Summary")
 439         buildArgs+=("/p:RestoreDefaultOptimizationDataPackage=false" "/p:PortableBuild=true")
 440         buildArgs+=("/p:UsePartialNGENOptimization=false" "/maxcpucount")
 441
 442         buildArgs+=("$projectName" "${__msbuildLog}" "${__msbuildWrn}" "${__msbuildErr}")
 443         buildArgs+=("$__msbuildEventLogging")
 444         buildArgs+=("${extraBuildParameters[@]}")
 445         buildArgs+=("${__CommonMSBuildArgs[@]}")
 446         buildArgs+=("${__UnprocessedBuildArgs[@]}")
 447
 448         nextCommand="\"$__ProjectRoot/dotnet.sh\" msbuild ${buildArgs[@]}"
 449         echo "Building step '$stepName' via $nextCommand"
 450         eval $nextCommand
 451
 452         # Make sure everything is OK
 453         if [ $? -ne 0 ]; then
 454             echo "${__MsgPrefix}Failed to build $stepName. See the build logs:"
 455             echo "    $__BuildLog"
 456             echo "    $__BuildWrn"
 457             echo "    $__BuildErr"
 458             exit 1
 459         fi
 460     fi
 461 }
 462
 463 build_native_projects()
 464 {
 465     platformArch="$1"
 466     intermediatesForBuild="$2"
 467
 468     extraCmakeArguments="-DCLR_CMAKE_TARGET_OS=${__BuildOS} -DCLR_CMAKE_HOST_ARCH=${platformArch}"
 469     message="native tests assets"
 470
 471     # All set to commence the build
 472     echo "Commencing build of $message for $__BuildOS.$__BuildArch.$__BuildType in $intermediatesForBuild"
 473
 474     generator=""
 475     buildFile="Makefile"
 476     buildTool="make"
 477     if [ $__UseNinja == 1 ]; then
 478         generator="ninja"
 479         buildFile="build.ninja"
 480         if ! buildTool=$(command -v ninja || command -v ninja-build); then
 481            echo "Unable to locate ninja!" 1>&2
 482            exit 1
 483         fi
 484     fi
 485
 486     if [ $__SkipConfigure == 0 ]; then
 487         # if msbuild is not supported, then set __SkipGenerateVersion to 1
 488         if [ $__isMSBuildOnNETCoreSupported == 0 ]; then __SkipGenerateVersion=1; fi
 489         # Drop version.c file
 490         __versionSourceFile="$intermediatesForBuild/version.c"
 491         if [ $__SkipGenerateVersion == 0 ]; then
 492             pwd
 493             $__ProjectRoot/dotnet.sh msbuild /nologo /verbosity:minimal /clp:Summary \
 494                                      /p:RestoreDefaultOptimizationDataPackage=false /p:PortableBuild=true \
 495                                      /p:UsePartialNGENOptimization=false /maxcpucount \
 496                                      $__ProjectDir/build.proj /t:GenerateVersionHeader \
 497                                      /p:GenerateVersionHeader=true /p:NativeVersionSourceFile=$__versionSourceFile \
 498                                      /l:BinClashLogger,Tools/Microsoft.DotNet.Build.Tasks.dll\;LogFile=binclash.log \
 499                                      $__CommonMSBuildArgs $__UnprocessedBuildArgs
 500         else
 501             # Generate the dummy version.c, but only if it didn't exist to make sure we don't trigger unnecessary rebuild
 502             __versionSourceLine="static char sccsid[] __attribute__((used)) = \"@(#)No version information produced\";"
 503             if [ -e $__versionSourceFile ]; then
 504                 read existingVersionSourceLine < $__versionSourceFile
 505             fi
 506             if [ "$__versionSourceLine" != "$existingVersionSourceLine" ]; then
 507                 echo $__versionSourceLine > $__versionSourceFile
 508             fi
 509         fi
 510
 511         pushd "$intermediatesForBuild"
 512         # Regenerate the CMake solution
 513         # Force cross dir to point to project root cross dir, in case there is a cross build.
 514         nextCommand="CONFIG_DIR=\"$__ProjectRoot/cross\" \"$__ProjectRoot/src/pal/tools/gen-buildsys-clang.sh\" \"$__TestDir\" $__ClangMajorVersion $__ClangMinorVersion $platformArch $__BuildType $__CodeCoverage $generator $extraCmakeArguments $__cmakeargs"
 515         echo "Invoking $nextCommand"
 516         eval $nextCommand
 517         popd
 518     fi
 519
 520     if [ ! -f "$intermediatesForBuild/$buildFile" ]; then
 521         echo "Failed to generate $message build project!"
 522         exit 1
 523     fi
 524
 525     # Build
 526     if [ $__ConfigureOnly == 1 ]; then
 527         echo "Finish configuration & skipping $message build."
 528         return
 529     fi
 530
 531     pushd "$intermediatesForBuild"
 532
 533     echo "Executing $buildTool install -j $__NumProc"
 534
 535     $buildTool install -j $__NumProc
 536     if [ $? != 0 ]; then
 537         echo "Failed to build $message."
 538         exit 1
 539     fi
 540
 541     popd
 542     echo "Native tests build success!"
 543 }
 544
 545 usage()
 546 {
 547     echo "Usage: $0 [BuildArch] [BuildType] [verbose] [coverage] [cross] [clangx.y] [ninja] [runtests] [bindir]"
 548     echo "BuildArch can be: x64, x86, arm, armel, arm64"
 549     echo "BuildType can be: debug, checked, release"
 550     echo "coverage - optional argument to enable code coverage build (currently supported only for Linux and OSX)."
 551     echo "ninja - target ninja instead of GNU make"
 552     echo "clangx.y - optional argument to build using clang version x.y - supported version 3.5 - 6.0"
 553     echo "cross - optional argument to signify cross compilation,"
 554     echo "      - will use ROOTFS_DIR environment variable if set."
 555     echo "portableLinux - build for Portable Linux Distribution"
 556     echo "portablebuild - Use portable build."
 557     echo "verbose - optional argument to enable verbose build output."
 558     echo "rebuild - if tests have already been built - rebuild them"
 559     echo "skipnative: skip the native tests build"
 560     echo "skipmanaged: skip the managed section of the test build"
 561     echo "buildtestwrappersonly - only build the test wrappers"
 562     echo "generatelayoutonly - only pull down dependencies and build coreroot"
 563     echo "generatetesthostonly - only pull down dependencies and build coreroot and the CoreFX testhost"
 564     echo "skiprestorepackages - skip package restore"
 565     echo "runtests - run tests after building them"
 566     echo "ziptests - zips CoreCLR tests & Core_Root for a Helix run"
 567     echo "bindir - output directory (defaults to $__ProjectRoot/bin)"
 568     echo "msbuildonunsupportedplatform - build managed binaries even if distro is not officially supported."
 569     echo "priority1 - include priority=1 tests in the build"
 570     exit 1
 571 }
 572
 573
 574 # Obtain the location of the bash script to figure out where the root of the repo is.
 575 __ProjectRoot="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
 576
 577 # Use uname to determine what the CPU is.
 578 CPUName=$(uname -p)
 579
 580 # Some Linux platforms report unknown for platform, but the arch for machine.
 581 if [ "$CPUName" == "unknown" ]; then
 582     CPUName=$(uname -m)
 583 fi
 584
 585 case $CPUName in
 586     i686)
 587         echo "Unsupported CPU $CPUName detected, build might not succeed!"
 588         __BuildArch=x86
 589         __HostArch=x86
 590         ;;
 591
 592     x86_64)
 593         __BuildArch=x64
 594         __HostArch=x64
 595         ;;
 596
 597     armv7l)
 598         echo "Unsupported CPU $CPUName detected, build might not succeed!"
 599         __BuildArch=arm
 600         __HostArch=arm
 601         ;;
 602
 603     aarch64)
 604         __BuildArch=arm64
 605         __HostArch=arm64
 606         ;;
 607
 608     *)
 609         echo "Unknown CPU $CPUName detected, configuring as if for x64"
 610         __BuildArch=x64
 611         __HostArch=x64
 612         ;;
 613 esac
 614
 615 # Use uname to determine what the OS is.
 616 OSName=$(uname -s)
 617 case $OSName in
 618     Linux)
 619         __BuildOS=Linux
 620         __HostOS=Linux
 621         ;;
 622
 623     Darwin)
 624         __BuildOS=OSX
 625         __HostOS=OSX
 626         ;;
 627
 628     FreeBSD)
 629         __BuildOS=FreeBSD
 630         __HostOS=FreeBSD
 631         ;;
 632
 633     OpenBSD)
 634         __BuildOS=OpenBSD
 635         __HostOS=OpenBSD
 636         ;;
 637
 638     NetBSD)
 639         __BuildOS=NetBSD
 640         __HostOS=NetBSD
 641         ;;
 642
 643     SunOS)
 644         __BuildOS=SunOS
 645         __HostOS=SunOS
 646         ;;
 647
 648     *)
 649         echo "Unsupported OS $OSName detected, configuring as if for Linux"
 650         __BuildOS=Linux
 651         __HostOS=Linux
 652         ;;
 653 esac
 654
 655 __BuildType=Debug
 656 __CodeCoverage=
 657 __IncludeTests=INCLUDE_TESTS
 658
 659 # Set the various build properties here so that CMake and MSBuild can pick them up
 660 export __ProjectDir="$__ProjectRoot"
 661 __SourceDir="$__ProjectDir/src"
 662 __PackagesDir="$__ProjectDir/packages"
 663 __RootBinDir="$__ProjectDir/bin"
 664 __BuildToolsDir="$__ProjectDir/Tools"
 665 __DotNetCli="${__BuildToolsDir}/dotnetcli/dotnet"
 666 __UnprocessedBuildArgs=
 667 __CommonMSBuildArgs=
 668 __MSBCleanBuildArgs=
 669 __UseNinja=0
 670 __VerboseBuild=0
 671 __SkipRestore=""
 672 __SkipNative=0
 673 __SkipManaged=0
 674 __SkipConfigure=0
 675 __SkipGenerateVersion=0
 676 __ConfigureOnly=0
 677 __CrossBuild=0
 678 __ClangMajorVersion=0
 679 __ClangMinorVersion=0
 680 __NuGetPath="$__PackagesDir/NuGet.exe"
 681 __HostDistroRid=""
 682 __SkipRestorePackages=0
 683 __DistroRid=""
 684 __cmakeargs=""
 685 __PortableLinux=0
 686 __msbuildonunsupportedplatform=0
 687 __ZipTests=0
 688 __NativeTestIntermediatesDir=
 689 __RunTests=0
 690 __RebuildTests=0
 691 __BuildTestWrappers=1
 692 __GenerateLayoutOnly=
 693 __GenerateTestHostOnly=
 694 __priority1=
 695 __BuildTestWrappersOnly=
 696 CORE_ROOT=
 697
 698 while :; do
 699     if [ $# -le 0 ]; then
 700         break
 701     fi
 702
 703     lowerI="$(echo $1 | awk '{print tolower($0)}')"
 704     case $lowerI in
 705         -\?|-h|--help)
 706             usage
 707             exit 1
 708             ;;
 709
 710         x86)
 711             __BuildArch=x86
 712             ;;
 713
 714         x64)
 715             __BuildArch=x64
 716             ;;
 717
 718         arm)
 719             __BuildArch=arm
 720             ;;
 721
 722         armel)
 723             __BuildArch=armel
 724             ;;
 725
 726         arm64)
 727             __BuildArch=arm64
 728             ;;
 729
 730         debug)
 731             __BuildType=Debug
 732             ;;
 733
 734         checked)
 735             __BuildType=Checked
 736             ;;
 737
 738         release)
 739             __BuildType=Release
 740             ;;
 741
 742         coverage)
 743             __CodeCoverage=Coverage
 744             ;;
 745
 746         cross)
 747             __CrossBuild=1
 748             ;;
 749
 750         portableBuild)
 751             __PortableBuild=1
 752             ;;
 753
 754         portablelinux)
 755             if [ "$__BuildOS" == "Linux" ]; then
 756                 __PortableLinux=1
 757             else
 758                 echo "ERROR: portableLinux not supported for non-Linux platforms."
 759                 exit 1
 760             fi
 761             ;;
 762
 763         verbose)
 764             __VerboseBuild=1
 765             ;;
 766
 767         clang3.5|-clang3.5)
 768             __ClangMajorVersion=3
 769             __ClangMinorVersion=5
 770             ;;
 771
 772         clang3.6|-clang3.6)
 773             __ClangMajorVersion=3
 774             __ClangMinorVersion=6
 775             ;;
 776
 777         clang3.7|-clang3.7)
 778             __ClangMajorVersion=3
 779             __ClangMinorVersion=7
 780             ;;
 781
 782         clang3.8|-clang3.8)
 783             __ClangMajorVersion=3
 784             __ClangMinorVersion=8
 785             ;;
 786
 787         clang3.9|-clang3.9)
 788             __ClangMajorVersion=3
 789             __ClangMinorVersion=9
 790             ;;
 791
 792         clang4.0|-clang4.0)
 793             __ClangMajorVersion=4
 794             __ClangMinorVersion=0
 795             ;;
 796
 797         clang5.0|-clang5.0)
 798             __ClangMajorVersion=5
 799             __ClangMinorVersion=0
 800             ;;
 801
 802         clang6.0|-clang6.0)
 803             __ClangMajorVersion=6
 804             __ClangMinorVersion=0
 805             ;;
 806
 807         ninja)
 808             __UseNinja=1
 809             ;;
 810
 811         runtests)
 812             __RunTests=1
 813             ;;
 814
 815         rebuild)
 816             __RebuildTests=1
 817             ;;
 818
 819         skipnative|-skipnative)
 820             __SkipNative=1
 821             ;;
 822
 823         skipmanaged|-skipmanaged)
 824             __SkipManaged=1
 825             __BuildTestWrappers=0
 826             ;;
 827
 828         ziptests)
 829             __ZipTests=1
 830             ;;
 831
 832         buildtestwrappersonly)
 833             __BuildTestWrappersOnly=1
 834             ;;
 835
 836         generatelayoutonly)
 837             __GenerateLayoutOnly=1
 838             ;;
 839
 840         generatetesthostonly)
 841             __GenerateTestHostOnly=1
 842             ;;
 843
 844         skiprestorepackages)
 845             __SkipRestorePackages=1
 846             ;;
 847
 848         bindir)
 849             if [ -n "$2" ]; then
 850                 __RootBinDir="$2"
 851                 if [ ! -d $__RootBinDir ]; then
 852                     mkdir $__RootBinDir
 853                 fi
 854                 __RootBinParent=$(dirname $__RootBinDir)
 855                 __RootBinName=${__RootBinDir##*/}
 856                 __RootBinDir="$(cd $__RootBinParent &>/dev/null && printf %s/%s $PWD $__RootBinName)"
 857                 shift
 858             else
 859                 echo "ERROR: 'bindir' requires a non-empty option argument"
 860                 exit 1
 861             fi
 862             ;;
 863
 864         msbuildonunsupportedplatform)
 865             __msbuildonunsupportedplatform=1
 866             ;;
 867
 868         priority1)
 869             __priority1=1
 870             __UnprocessedBuildArgs+=("/p:CLRTestPriorityToBuild=1")
 871             ;;
 872
 873         *)
 874             __UnprocessedBuildArgs+=("$1")
 875             ;;
 876     esac
 877
 878     shift
 879 done
 880
 881 # Get the number of processors available to the scheduler
 882 # Other techniques such as `nproc` only get the number of
 883 # processors available to a single process.
 884 if [ `uname` = "FreeBSD" ]; then
 885   __NumProc=`sysctl hw.ncpu | awk '{ print $2+1 }'`
 886 elif [ `uname` = "NetBSD" ]; then
 887   __NumProc=$(($(getconf NPROCESSORS_ONLN)+1))
 888 elif [ `uname` = "Darwin" ]; then
 889   __NumProc=$(($(getconf _NPROCESSORS_ONLN)+1))
 890 else
 891   __NumProc=$(nproc --all)
 892 fi
 893
 894 __CommonMSBuildArgs=("/p:__BuildArch=$__BuildArch" "/p:__BuildType=$__BuildType" "/p:__BuildOS=$__BuildOS")
 895
 896 # Configure environment if we are doing a verbose build
 897 if [ $__VerboseBuild == 1 ]; then
 898     export VERBOSE=1
 899     __CommonMSBuildArgs+=("/v:detailed")
 900 fi
 901
 902 # Set default clang version
 903 if [[ $__ClangMajorVersion == 0 && $__ClangMinorVersion == 0 ]]; then
 904     if [[ "$__BuildArch" == "arm" || "$__BuildArch" == "armel" ]]; then
 905         __ClangMajorVersion=5
 906         __ClangMinorVersion=0
 907     else
 908         __ClangMajorVersion=3
 909         __ClangMinorVersion=9
 910     fi
 911 fi
 912
 913 # Set dependent variables
 914 __LogsDir="$__RootBinDir/Logs"
 915 __MsbuildDebugLogsDir="$__LogsDir/MsbuildDebugLogs"
 916
 917 # init the host distro name
 918 initHostDistroRid
 919
 920 # Set the remaining variables based upon the determined build configuration
 921 __BinDir="$__RootBinDir/Product/$__BuildOS.$__BuildArch.$__BuildType"
 922 __PackagesBinDir="$__BinDir/.nuget"
 923 __TestDir="$__ProjectDir/tests"
 924 __TestWorkingDir="$__RootBinDir/tests/$__BuildOS.$__BuildArch.$__BuildType"
 925 __IntermediatesDir="$__RootBinDir/obj/$__BuildOS.$__BuildArch.$__BuildType"
 926 __TestIntermediatesDir="$__RootBinDir/tests/obj/$__BuildOS.$__BuildArch.$__BuildType"
 927 __isMSBuildOnNETCoreSupported=0
 928 __CrossComponentBinDir="$__BinDir"
 929 __CrossCompIntermediatesDir="$__IntermediatesDir/crossgen"
 930
 931 __CrossArch="$__HostArch"
 932 if [[ "$__HostArch" == "x64" && "$__BuildArch" == "arm" ]]; then
 933     __CrossArch="x86"
 934 fi
 935 if [ $__CrossBuild == 1 ]; then
 936     __CrossComponentBinDir="$__CrossComponentBinDir/$__CrossArch"
 937 fi
 938 __CrossgenCoreLibLog="$__LogsDir/CrossgenCoreLib_$__BuildOS.$BuildArch.$__BuildType.log"
 939 __CrossgenExe="$__CrossComponentBinDir/crossgen"
 940
 941 isMSBuildOnNETCoreSupported
 942
 943 # CI_SPECIFIC - On CI machines, $HOME may not be set. In such a case, create a subfolder and set the variable to it.
 944 # This is needed by CLI to function.
 945 if [ -z "$HOME" ]; then
 946     if [ ! -d "$__ProjectDir/temp_home" ]; then
 947         mkdir temp_home
 948     fi
 949     export HOME=$__ProjectDir/temp_home
 950     echo "HOME not defined; setting it to $HOME"
 951 fi
 952
 953 # Configure environment if we are doing a cross compile.
 954 if [ $__CrossBuild == 1 ]; then
 955     export CROSSCOMPILE=1
 956     if ! [[ -n "$ROOTFS_DIR" ]]; then
 957         export ROOTFS_DIR="$__ProjectRoot/cross/rootfs/$__BuildArch"
 958     fi
 959 fi
 960
 961 # init the target distro name
 962 initTargetDistroRid
 963
 964 # Override tool directory
 965
 966 __CoreClrVersion=1.1.0
 967 __sharedFxDir=$__BuildToolsDir/dotnetcli/shared/Microsoft.NETCore.App/$__CoreClrVersion/
 968
 969 if [[ (-z "$__GenerateLayoutOnly") && (-z "$__GenerateTestHostOnly") && (-z "$__BuildTestWrappersOnly") ]]; then
 970     build_Tests
 971 elif [ ! -z "$__BuildTestWrappersOnly" ]; then
 972     build_test_wrappers
 973 else
 974     generate_layout
 975     if [ ! -z "$__GenerateTestHostOnly" ]; then
 976         generate_testhost
 977     fi
 978 fi
 979
 980 if [ $? -ne 0 ]; then
 981     echo "Failed to build tests"
 982     exit 1
 983 fi
 984
 985 echo "${__MsgPrefix}Test build successful."
 986 echo "${__MsgPrefix}Test binaries are available at ${__TestBinDir}"
 987
 988 __testNativeBinDir=$__IntermediatesDir/tests
 989
 990 if [ $__RunTests -ne 0 ]; then
 991
 992     echo "Run Tests..."
 993
 994     nextCommand="$__TestDir/runtest.sh --testRootDir=$__TestBinDir --coreClrBinDir=$__BinDir --coreFxBinDir=$CORE_ROOT --testNativeBinDir=$__testNativeBinDir"
 995     echo "$nextCommand"
 996     eval $nextCommand
 997
 998     echo "Tests run successful."
 999 else
1000     echo "To run all tests use 'tests/runtests.sh' where:"
1001     echo "    testRootDir      = $__TestBinDir"
1002     echo "    coreClrBinDir    = $__BinDir"
1003     echo "    coreFxBinDir     = $CORE_ROOT"
1004     echo "    testNativeBinDir = $__testNativeBinDir"
1005     echo " -------------------------------------------------- "
1006     echo " Example runtest.sh command"
1007     echo ""
1008     echo " ./tests/runtest.sh --coreOverlayDir=$CORE_ROOT --testNativeBinDir=$__testNativeBinDir --testRootDir=$__TestBinDir --copyNativeTestBin"
1009     echo " -------------------------------------------------- "
1010     echo "To run single test use the following command:"
1011     echo "    bash ${__TestBinDir}/__TEST_PATH__/__TEST_NAME__.sh -coreroot=${CORE_ROOT}"
1012 fi
1013