build-test.sh

   1 #!/usr/bin/env bash
   2
   3 initHostDistroRid()
   4 {
   5     __HostDistroRid=""
   6
   7     # Some OS groups should default to use the portable packages
   8     if [ "$__BuildOS" == "OSX" ]; then
   9         __PortableBuild=1
  10     fi
  11
  12     if [ "$__HostOS" == "Linux" ]; then
  13         if [ -e /etc/redhat-release ]; then
  14             __PortableBuild=1
  15         elif [ -e /etc/os-release ]; then
  16             source /etc/os-release
  17             if [[ $ID == "alpine" ]]; then
  18                 __HostDistroRid="linux-musl-$__HostArch"
  19             else
  20                 __PortableBuild=1
  21                 __HostDistroRid="$ID.$VERSION_ID-$__HostArch"
  22             fi
  23         fi
  24     elif [ "$__HostOS" == "FreeBSD" ]; then
  25         __freebsd_version=`sysctl -n kern.osrelease | cut -f1 -d'.'`
  26         __HostDistroRid="freebsd.$__freebsd_version-$__HostArch"
  27     fi
  28
  29     # Portable builds target the base RID
  30     if [ "$__PortableBuild" == 1 ]; then
  31         if [ "$__BuildOS" == "OSX" ]; then
  32             export __HostDistroRid="osx-$__BuildArch"
  33         elif [ "$__BuildOS" == "Linux" ]; then
  34             export __HostDistroRid="linux-$__BuildArch"
  35         fi
  36     fi
  37
  38     if [ "$__HostDistroRid" == "" ]; then
  39         echo "WARNING: Cannot determine runtime id for current distro."
  40     fi
  41
  42     echo "Setting __HostDistroRid to $__HostDistroRid"
  43 }
  44
  45 initTargetDistroRid()
  46 {
  47     if [ $__CrossBuild == 1 ]; then
  48         if [ "$__BuildOS" == "Linux" ]; then
  49             if [ ! -e $ROOTFS_DIR/etc/os-release ]; then
  50                 if [ -e $ROOTFS_DIR/android_platform ]; then
  51                     source $ROOTFS_DIR/android_platform
  52                     export __DistroRid="$RID"
  53                 else
  54                     echo "WARNING: Cannot determine runtime id for current distro."
  55                     export __DistroRid=""
  56                 fi
  57             else
  58                 source $ROOTFS_DIR/etc/os-release
  59                 export __DistroRid="$ID.$VERSION_ID-$__BuildArch"
  60             fi
  61         fi
  62     else
  63         export __DistroRid="$__HostDistroRid"
  64         export __RuntimeId="$__HostDistroRid"
  65     fi
  66
  67     if [ "$ID.$VERSION_ID" == "ubuntu.16.04" ]; then
  68      export __DistroRid="ubuntu.14.04-$__BuildArch"
  69     fi
  70
  71     # Portable builds target the base RID
  72     if [ "$__PortableBuild" == 1 ]; then
  73         if [ "$__BuildOS" == "Linux" ]; then
  74             export __DistroRid="linux-$__BuildArch"
  75             export __RuntimeId="linux-$__BuildArch"
  76         elif [ "$__BuildOS" == "OSX" ]; then
  77             export __DistroRid="osx-$__BuildArch"
  78             export __RuntimeId="osx-$__BuildArch"
  79         fi
  80     fi
  81
  82     echo "__DistroRid: " $__DistroRid
  83 }
  84
  85 isMSBuildOnNETCoreSupported()
  86 {
  87     __isMSBuildOnNETCoreSupported=$__msbuildonunsupportedplatform
  88
  89     if [ $__isMSBuildOnNETCoreSupported == 1 ]; then
  90         return
  91     fi
  92
  93     if [ "$__HostArch" == "x64" ]; then
  94         if [ "$__HostOS" == "Linux" ]; then
  95             __isMSBuildOnNETCoreSupported=1
  96             UNSUPPORTED_RIDS=("debian.9-x64" "ubuntu.17.04-x64")
  97             for UNSUPPORTED_RID in "${UNSUPPORTED_RIDS[@]}"
  98             do
  99                 if [ "$__HostDistroRid" == "$UNSUPPORTED_RID" ]; then
 100                     __isMSBuildOnNETCoreSupported=0
 101                     break
 102                 fi
 103             done
 104         elif [ "$__HostOS" == "OSX" ]; then
 105             __isMSBuildOnNETCoreSupported=1
 106         fi
 107     fi
 108 }
 109
 110 build_test_wrappers()
 111 {
 112     if [ $__BuildTestWrappers -ne -0 ]; then
 113         echo "${__MsgPrefix}Creating test wrappers..."
 114
 115         export __Exclude="${__ProjectDir}/tests/issues.targets"
 116         export __BuildLogRootName="Tests_XunitWrapper"
 117
 118         buildVerbosity="Summary"
 119
 120         if [ $__VerboseBuild == 1 ]; then
 121             buildVerbosity="Diag"
 122         fi
 123
 124         # Set up directories and file names
 125         __BuildLogRootName=$subDirectoryName
 126         __BuildLog="$__LogsDir/${__BuildLogRootName}.${__BuildOS}.${__BuildArch}.${__BuildType}.log"
 127         __BuildWrn="$__LogsDir/${__BuildLogRootName}.${__BuildOS}.${__BuildArch}.${__BuildType}.wrn"
 128         __BuildErr="$__LogsDir/${__BuildLogRootName}.${__BuildOS}.${__BuildArch}.${__BuildType}.err"
 129         __MsbuildLog="/fileloggerparameters:\"Verbosity=normal;LogFile=${__BuildLog}\""
 130         __MsbuildWrn="/fileloggerparameters1:\"WarningsOnly;LogFile=${__BuildWrn}\""
 131         __MsbuildErr="/fileloggerparameters2:\"ErrorsOnly;LogFile=${__BuildErr}\""
 132         __Logging="$__MsbuildLog $__MsbuildWrn $__MsbuildErr /consoleloggerparameters:$buildVerbosity"
 133
 134         nextCommand="\"${__DotNetCli}\" msbuild \"${__ProjectDir}/tests/runtest.proj\" /p:RestoreAdditionalProjectSources=https://dotnet.myget.org/F/dotnet-core/ /p:BuildWrappers=true /p:TargetsWindows=false $__Logging /p:__BuildOS=$__BuildOS /p:__BuildType=$__BuildType /p:__BuildArch=$__BuildArch"
 135         echo "$nextCommand"
 136         eval $nextCommand
 137
 138         if [ $? -ne 0 ]; then
 139             echo "${__MsgPrefix}Error: build failed. Refer to the build log files for details (above)"
 140             exit 1
 141         else
 142             echo "XUnit Wrappers have been built."
 143             echo { "\"build_os\"": "\"${__BuildOS}\"", "\"build_arch\"": "\"${__BuildArch}\"", "\"build_type\"": "\"${__BuildType}\"" } > "${__TestWorkingDir}/build_info.json"
 144
 145         fi
 146     fi
 147 }
 148
 149 generate_layout()
 150 {
 151     echo "${__MsgPrefix}Creating test overlay..."
 152
 153     __TestDir=$__ProjectDir/tests
 154     __ProjectFilesDir=$__TestDir
 155     __TestBinDir=$__TestWorkingDir
 156
 157     if [ $__RebuildTests -ne 0 ]; then
 158         if [ -d "${__TestBinDir}" ]; then
 159             echo "Removing tests build dir: ${__TestBinDir}"
 160             rm -rf $__TestBinDir
 161         fi
 162     fi
 163
 164     __CMakeBinDir="${__TestBinDir}"
 165
 166     if [ -z "$__TestIntermediateDir" ]; then
 167         __TestIntermediateDir="tests/obj/${__BuildOS}.${__BuildArch}.${__BuildType}"
 168     fi
 169
 170     echo "__BuildOS: ${__BuildOS}"
 171     echo "__BuildArch: ${__BuildArch}"
 172     echo "__BuildType: ${__BuildType}"
 173     echo "__TestIntermediateDir: ${__TestIntermediateDir}"
 174
 175     if [ ! -f "$__TestBinDir" ]; then
 176         echo "Creating TestBinDir: ${__TestBinDir}"
 177         mkdir -p $__TestBinDir
 178     fi
 179     if [ ! -f "$__LogsDir" ]; then
 180         echo "Creating LogsDir: ${__LogsDir}"
 181         mkdir -p $__LogsDir
 182     fi
 183
 184     __BuildProperties="-p:OSGroup=${__BuildOS} -p:BuildOS=${__BuildOS} -p:BuildArch=${__BuildArch} -p:BuildType=${__BuildType}"
 185
 186     # =========================================================================================
 187     # ===
 188     # === Restore product binaries from packages
 189     # ===
 190     # =========================================================================================
 191
 192     build_MSBuild_projects "Restore_Packages" "${__ProjectDir}/tests/build.proj" "Restore product binaries (build tests)" "-BatchRestorePackages"
 193
 194     if [ -n "$__UpdateInvalidPackagesArg" ]; then
 195         __up=-updateinvalidpackageversion
 196     fi
 197
 198     echo "${__MsgPrefix}Creating test overlay..."
 199
 200     if [ -z "$xUnitTestBinBase" ]; then
 201         xUnitTestBinBase=$__TestWorkingDir
 202     fi
 203
 204     export CORE_ROOT=$xUnitTestBinBase/Tests/Core_Root
 205
 206     if [ -d "${CORE_ROOT}" ]; then
 207         rm -rf $CORE_ROOT
 208     fi
 209
 210     mkdir -p $CORE_ROOT
 211
 212     build_MSBuild_projects "Tests_Overlay_Managed" "${__ProjectDir}/tests/runtest.proj" "Creating test overlay" "-testOverlay"
 213
 214     chmod +x $__BinDir/corerun
 215     chmod +x $__BinDir/crossgen
 216
 217     # Make sure to copy over the pulled down packages
 218     cp -r $__BinDir/* $CORE_ROOT/ > /dev/null
 219 }
 220
 221 generate_testhost()
 222 {
 223     echo "${__MsgPrefix}Generating test host..."
 224
 225     export TEST_HOST=$xUnitTestBinBase/testhost
 226
 227     if [ -d "${TEST_HOST}" ]; then
 228         rm -rf $TEST_HOST
 229     fi
 230
 231     mkdir -p $TEST_HOST
 232
 233     build_MSBuild_projects "Tests_Generate_TestHost" "${__ProjectDir}/tests/runtest.proj" "Creating test host" "-testHost"
 234 }
 235
 236
 237 build_Tests()
 238 {
 239     echo "${__MsgPrefix}Building Tests..."
 240
 241     __TestDir=$__ProjectDir/tests
 242     __ProjectFilesDir=$__TestDir
 243     __TestBinDir=$__TestWorkingDir
 244
 245     if [ -f  "${__TestWorkingDir}/build_info.json" ]; then
 246         rm  "${__TestWorkingDir}/build_info.json"
 247     fi
 248
 249     if [ $__RebuildTests -ne 0 ]; then
 250         if [ -d "${__TestBinDir}" ]; then
 251             echo "Removing tests build dir: ${__TestBinDir}"
 252             rm -rf $__TestBinDir
 253         fi
 254     fi
 255
 256     export __CMakeBinDir="${__TestBinDir}"
 257     if [ ! -d "${__TestIntermediatesDir}" ]; then
 258         mkdir -p ${__TestIntermediatesDir}
 259     fi
 260
 261     __NativeTestIntermediatesDir="${__TestIntermediatesDir}/Native"
 262     if [  ! -d "${__NativeTestIntermediatesDir}" ]; then
 263         mkdir -p ${__NativeTestIntermediatesDir}
 264     fi
 265
 266     __ManagedTestIntermediatesDir="${__TestIntermediatesDir}/Managed"
 267     if [ ! -d "${__ManagedTestIntermediatesDir}" ]; then
 268         mkdir -p ${__ManagedTestIntermediatesDir}
 269     fi
 270
 271     echo "__BuildOS: ${__BuildOS}"
 272     echo "__BuildArch: ${__BuildArch}"
 273     echo "__BuildType: ${__BuildType}"
 274     echo "__TestIntermediatesDir: ${__TestIntermediatesDir}"
 275     echo "__NativeTestIntermediatesDir: ${__NativeTestIntermediatesDir}"
 276     echo "__ManagedTestIntermediatesDir: ${__ManagedTestIntermediatesDir}"
 277
 278     if [ ! -f "$__TestBinDir" ]; then
 279         echo "Creating TestBinDir: ${__TestBinDir}"
 280         mkdir -p $__TestBinDir
 281     fi
 282     if [ ! -f "$__LogsDir" ]; then
 283         echo "Creating LogsDir: ${__LogsDir}"
 284         mkdir -p $__LogsDir
 285     fi
 286
 287     __BuildProperties="-p:OSGroup=${__BuildOS} -p:BuildOS=${__BuildOS} -p:BuildArch=${__BuildArch} -p:BuildType=${__BuildType}"
 288
 289     # =========================================================================================
 290     # ===
 291     # === Restore product binaries from packages
 292     # ===
 293     # =========================================================================================
 294
 295     if [ ${__SkipRestorePackages} != 1 ]; then
 296         build_MSBuild_projects "Restore_Product" "${__ProjectDir}/tests/build.proj" "Restore product binaries (build tests)" "-BatchRestorePackages"
 297     fi
 298
 299     if [ $__SkipNative != 1 ]; then
 300         build_native_projects "$__BuildArch" "${__NativeTestIntermediatesDir}"
 301
 302         if [ $? -ne 0 ]; then
 303             echo "${__MsgPrefix}Error: build failed. Refer to the build log files for details (above)"
 304             exit 1
 305         fi
 306     fi
 307
 308     if [ $__SkipManaged != 1 ]; then
 309         echo "Starting the Managed Tests Build..."
 310
 311         build_MSBuild_projects "Tests_Managed" "$__ProjectDir/tests/build.proj" "Managed tests build (build tests)" "$__up"
 312
 313         if [ $? -ne 0 ]; then
 314             echo "${__MsgPrefix}Error: build failed. Refer to the build log files for details (above)"
 315             exit 1
 316         else
 317             echo "Checking the Managed Tests Build..."
 318
 319             build_MSBuild_projects "Check_Test_Build" "${__ProjectDir}/tests/runtest.proj" "Check Test Build" "-ExtraParameters:/t:CheckTestBuild"
 320
 321             if [ $? -ne 0 ]; then
 322                 echo "${__MsgPrefix}Error: Check Test Build failed."
 323                 exit 1
 324             fi
 325         fi
 326
 327         echo "Managed tests build success!"
 328     fi
 329
 330     build_test_wrappers
 331
 332     if [ -n "$__UpdateInvalidPackagesArg" ]; then
 333         __up=-updateinvalidpackageversion
 334     fi
 335
 336     generate_layout
 337
 338     if [ $__ZipTests -ne 0 ]; then
 339         echo "${__MsgPrefix}ZIP tests packages..."
 340         build_MSBuild_projects "Helix_Prep" "$__ProjectDir/tests/helixprep.proj" "Prep test binaries for Helix publishing" " "
 341     fi
 342 }
 343
 344 build_MSBuild_projects()
 345 {
 346     subDirectoryName=$1
 347     shift
 348     projectName=$1
 349     shift
 350     stepName="$1"
 351     shift
 352     extraBuildParameters=("$@")
 353
 354     # Set up directories and file names
 355     __BuildLogRootName=$subDirectoryName
 356     __BuildLog="$__LogsDir/${__BuildLogRootName}.${__BuildOS}.${__BuildArch}.${__BuildType}.log"
 357     __BuildWrn="$__LogsDir/${__BuildLogRootName}.${__BuildOS}.${__BuildArch}.${__BuildType}.wrn"
 358     __BuildErr="$__LogsDir/${__BuildLogRootName}.${__BuildOS}.${__BuildArch}.${__BuildType}.err"
 359
 360     # Use binclashlogger by default if no other logger is specified
 361     if [[ "${extraBuildParameters[*]}" == *"-MsBuildEventLogging"* ]]; then
 362         msbuildEventLogging=""
 363     else
 364         msbuildEventLogging="-MsBuildEventLogging=\"/l:BinClashLogger,Tools/Microsoft.DotNet.Build.Tasks.dll;LogFile=binclash.log\""
 365     fi
 366
 367     if [[ "$subDirectoryName" == "Tests_Managed" ]]; then
 368         # Execute msbuild managed test build in stages - workaround for excessive data retention in MSBuild ConfigCache
 369         # See https://github.com/Microsoft/msbuild/issues/2993
 370
 371         # __SkipPackageRestore and __SkipTargetingPackBuild used  to control build by tests/src/dirs.proj
 372         export __SkipPackageRestore=false
 373         export __SkipTargetingPackBuild=false
 374         export __BuildLoopCount=2
 375         export __TestGroupToBuild=1
 376         __AppendToLog=false
 377
 378         if [ -n "$__priority1" ]; then
 379             export __BuildLoopCount=16
 380             export __TestGroupToBuild=2
 381         fi
 382
 383         for (( slice=1 ; slice <= __BuildLoopCount; slice = slice + 1 ))
 384         do
 385             __msbuildLog="\"/flp:Verbosity=normal;LogFile=${__BuildLog};Append=${__AppendToLog}\""
 386             __msbuildWrn="\"/flp1:WarningsOnly;LogFile=${__BuildWrn};Append=${__AppendToLog}\""
 387             __msbuildErr="\"/flp2:ErrorsOnly;LogFile=${__BuildErr};Append=${__AppendToLog}\""
 388
 389             export TestBuildSlice=$slice
 390
 391             # Generate build command
 392             buildArgs=("-Project=$projectName" "-MsBuildLog=${__msbuildLog}" "-MsBuildWrn=${__msbuildWrn}" "-MsBuildErr=${__msbuildErr}")
 393             buildArgs+=("$msbuildEventLogging")
 394             buildArgs+=("${extraBuildParameters[@]}")
 395             buildArgs+=("${__RunArgs[@]}")
 396             buildArgs+=("${__UnprocessedBuildArgs[@]}")
 397
 398             nextCommand="\"$__ProjectRoot/run.sh\" build ${buildArgs[@]}"
 399             echo "Building step '$stepName' slice=$slice via $nextCommand"
 400             eval $nextCommand
 401
 402             # Make sure everything is OK
 403             if [ $? -ne 0 ]; then
 404                 echo "${__MsgPrefix}Failed to build $stepName. See the build logs:"
 405                 echo "    $__BuildLog"
 406                 echo "    $__BuildWrn"
 407                 echo "    $__BuildErr"
 408                 exit 1
 409             fi
 410             export __SkipPackageRestore=true
 411             export __SkipTargetingPackBuild=true
 412             __AppendToLog=true
 413         done
 414     else
 415         __msbuildLog="\"/flp:Verbosity=normal;LogFile=${__BuildLog}\""
 416         __msbuildWrn="\"/flp1:WarningsOnly;LogFile=${__BuildWrn}\""
 417         __msbuildErr="\"/flp2:ErrorsOnly;LogFile=${__BuildErr}\""
 418
 419         # Generate build command
 420         buildArgs=("-Project=$projectName" "-MsBuildLog=${__msbuildLog}" "-MsBuildWrn=${__msbuildWrn}" "-MsBuildErr=${__msbuildErr}")
 421         buildArgs+=("$msbuildEventLogging")
 422         buildArgs+=("${extraBuildParameters[@]}")
 423         buildArgs+=("${__RunArgs[@]}")
 424         buildArgs+=("${__UnprocessedBuildArgs[@]}")
 425
 426         nextCommand="\"$__ProjectRoot/run.sh\" build ${buildArgs[@]}"
 427         echo "Building step '$stepName' via $nextCommand"
 428         eval $nextCommand
 429
 430         # Make sure everything is OK
 431         if [ $? -ne 0 ]; then
 432             echo "${__MsgPrefix}Failed to build $stepName. See the build logs:"
 433             echo "    $__BuildLog"
 434             echo "    $__BuildWrn"
 435             echo "    $__BuildErr"
 436             exit 1
 437         fi
 438     fi
 439 }
 440
 441 build_native_projects()
 442 {
 443     platformArch="$1"
 444     intermediatesForBuild="$2"
 445
 446     extraCmakeArguments="-DCLR_CMAKE_TARGET_OS=${__BuildOS} -DCLR_CMAKE_HOST_ARCH=${platformArch}"
 447     message="native tests assets"
 448
 449     # All set to commence the build
 450     echo "Commencing build of $message for $__BuildOS.$__BuildArch.$__BuildType in $intermediatesForBuild"
 451
 452     generator=""
 453     buildFile="Makefile"
 454     buildTool="make"
 455     if [ $__UseNinja == 1 ]; then
 456         generator="ninja"
 457         buildFile="build.ninja"
 458         if ! buildTool=$(command -v ninja || command -v ninja-build); then
 459            echo "Unable to locate ninja!" 1>&2
 460            exit 1
 461         fi
 462     fi
 463
 464     if [ $__SkipConfigure == 0 ]; then
 465         # if msbuild is not supported, then set __SkipGenerateVersion to 1
 466         if [ $__isMSBuildOnNETCoreSupported == 0 ]; then __SkipGenerateVersion=1; fi
 467         # Drop version.cpp file
 468         __versionSourceFile="$intermediatesForBuild/version.cpp"
 469         if [ $__SkipGenerateVersion == 0 ]; then
 470             pwd
 471             "$__ProjectRoot/run.sh" build -Project=$__ProjectDir/build.proj -generateHeaderUnix -NativeVersionSourceFile=$__versionSourceFile -MsBuildEventLogging="/l:BinClashLogger,Tools/Microsoft.DotNet.Build.Tasks.dll;LogFile=binclash.log"  $__RunArgs $__UnprocessedBuildArgs
 472         else
 473             # Generate the dummy version.cpp, but only if it didn't exist to make sure we don't trigger unnecessary rebuild
 474             __versionSourceLine="static char sccsid[] __attribute__((used)) = \"@(#)No version information produced\";"
 475             if [ -e $__versionSourceFile ]; then
 476                 read existingVersionSourceLine < $__versionSourceFile
 477             fi
 478             if [ "$__versionSourceLine" != "$existingVersionSourceLine" ]; then
 479                 echo $__versionSourceLine > $__versionSourceFile
 480             fi
 481         fi
 482
 483         pushd "$intermediatesForBuild"
 484         # Regenerate the CMake solution
 485         # Force cross dir to point to project root cross dir, in case there is a cross build.
 486         nextCommand="CONFIG_DIR=\"$__ProjectRoot/cross\" \"$__ProjectRoot/src/pal/tools/gen-buildsys-clang.sh\" \"$__TestDir\" $__ClangMajorVersion $__ClangMinorVersion $platformArch $__BuildType $__CodeCoverage $generator $extraCmakeArguments $__cmakeargs"
 487         echo "Invoking $nextCommand"
 488         eval $nextCommand
 489         popd
 490     fi
 491
 492     if [ ! -f "$intermediatesForBuild/$buildFile" ]; then
 493         echo "Failed to generate $message build project!"
 494         exit 1
 495     fi
 496
 497     # Build
 498     if [ $__ConfigureOnly == 1 ]; then
 499         echo "Finish configuration & skipping $message build."
 500         return
 501     fi
 502
 503     pushd "$intermediatesForBuild"
 504
 505     echo "Executing $buildTool install -j $__NumProc"
 506
 507     $buildTool install -j $__NumProc
 508     if [ $? != 0 ]; then
 509         echo "Failed to build $message."
 510         exit 1
 511     fi
 512
 513     popd
 514     echo "Native tests build success!"
 515 }
 516
 517 usage()
 518 {
 519     echo "Usage: $0 [BuildArch] [BuildType] [verbose] [coverage] [cross] [clangx.y] [ninja] [runtests] [bindir]"
 520     echo "BuildArch can be: x64, x86, arm, armel, arm64"
 521     echo "BuildType can be: debug, checked, release"
 522     echo "coverage - optional argument to enable code coverage build (currently supported only for Linux and OSX)."
 523     echo "ninja - target ninja instead of GNU make"
 524     echo "clangx.y - optional argument to build using clang version x.y - supported version 3.5 - 6.0"
 525     echo "cross - optional argument to signify cross compilation,"
 526     echo "      - will use ROOTFS_DIR environment variable if set."
 527     echo "portableLinux - build for Portable Linux Distribution"
 528     echo "portablebuild - Use portable build."
 529     echo "verbose - optional argument to enable verbose build output."
 530     echo "rebuild - if tests have already been built - rebuild them"
 531     echo "skipnative: skip the native tests build"
 532     echo "skipmanaged: skip the managed section of the test build"
 533     echo "buildtestwrappersonly - only build the test wrappers"
 534     echo "generatelayoutonly - only pull down dependencies and build coreroot"
 535     echo "generatetesthostonly - only pull down dependencies and build coreroot and the CoreFX testhost"
 536     echo "skiprestorepackages - skip package restore"
 537     echo "runtests - run tests after building them"
 538     echo "ziptests - zips CoreCLR tests & Core_Root for a Helix run"
 539     echo "bindir - output directory (defaults to $__ProjectRoot/bin)"
 540     echo "msbuildonunsupportedplatform - build managed binaries even if distro is not officially supported."
 541     echo "priority1 - include priority=1 tests in the build"
 542     exit 1
 543 }
 544
 545
 546 # Obtain the location of the bash script to figure out where the root of the repo is.
 547 __ProjectRoot="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
 548
 549 # Use uname to determine what the CPU is.
 550 CPUName=$(uname -p)
 551
 552 # Some Linux platforms report unknown for platform, but the arch for machine.
 553 if [ "$CPUName" == "unknown" ]; then
 554     CPUName=$(uname -m)
 555 fi
 556
 557 case $CPUName in
 558     i686)
 559         echo "Unsupported CPU $CPUName detected, build might not succeed!"
 560         __BuildArch=x86
 561         __HostArch=x86
 562         ;;
 563
 564     x86_64)
 565         __BuildArch=x64
 566         __HostArch=x64
 567         ;;
 568
 569     armv7l)
 570         echo "Unsupported CPU $CPUName detected, build might not succeed!"
 571         __BuildArch=arm
 572         __HostArch=arm
 573         ;;
 574
 575     aarch64)
 576         __BuildArch=arm64
 577         __HostArch=arm64
 578         ;;
 579
 580     *)
 581         echo "Unknown CPU $CPUName detected, configuring as if for x64"
 582         __BuildArch=x64
 583         __HostArch=x64
 584         ;;
 585 esac
 586
 587 # Use uname to determine what the OS is.
 588 OSName=$(uname -s)
 589 case $OSName in
 590     Linux)
 591         __BuildOS=Linux
 592         __HostOS=Linux
 593         ;;
 594
 595     Darwin)
 596         __BuildOS=OSX
 597         __HostOS=OSX
 598         ;;
 599
 600     FreeBSD)
 601         __BuildOS=FreeBSD
 602         __HostOS=FreeBSD
 603         ;;
 604
 605     OpenBSD)
 606         __BuildOS=OpenBSD
 607         __HostOS=OpenBSD
 608         ;;
 609
 610     NetBSD)
 611         __BuildOS=NetBSD
 612         __HostOS=NetBSD
 613         ;;
 614
 615     SunOS)
 616         __BuildOS=SunOS
 617         __HostOS=SunOS
 618         ;;
 619
 620     *)
 621         echo "Unsupported OS $OSName detected, configuring as if for Linux"
 622         __BuildOS=Linux
 623         __HostOS=Linux
 624         ;;
 625 esac
 626
 627 __BuildType=Debug
 628 __CodeCoverage=
 629 __IncludeTests=INCLUDE_TESTS
 630
 631 # Set the various build properties here so that CMake and MSBuild can pick them up
 632 export __ProjectDir="$__ProjectRoot"
 633 __SourceDir="$__ProjectDir/src"
 634 __PackagesDir="$__ProjectDir/packages"
 635 __RootBinDir="$__ProjectDir/bin"
 636 __BuildToolsDir="$__ProjectDir/Tools"
 637 __DotNetCli="${__BuildToolsDir}/dotnetcli/dotnet"
 638 __UnprocessedBuildArgs=
 639 __RunArgs=
 640 __MSBCleanBuildArgs=
 641 __UseNinja=0
 642 __VerboseBuild=0
 643 __SkipRestore=""
 644 __SkipNative=0
 645 __SkipManaged=0
 646 __SkipConfigure=0
 647 __SkipGenerateVersion=0
 648 __ConfigureOnly=0
 649 __CrossBuild=0
 650 __ClangMajorVersion=0
 651 __ClangMinorVersion=0
 652 __NuGetPath="$__PackagesDir/NuGet.exe"
 653 __HostDistroRid=""
 654 __SkipRestorePackages=0
 655 __DistroRid=""
 656 __cmakeargs=""
 657 __PortableLinux=0
 658 __msbuildonunsupportedplatform=0
 659 __ZipTests=0
 660 __NativeTestIntermediatesDir=
 661 __RunTests=0
 662 __RebuildTests=0
 663 __BuildTestWrappers=1
 664 __GenerateLayoutOnly=
 665 __GenerateTestHostOnly=
 666 __priority1=
 667 __BuildTestWrappersOnly=
 668 CORE_ROOT=
 669
 670 while :; do
 671     if [ $# -le 0 ]; then
 672         break
 673     fi
 674
 675     lowerI="$(echo $1 | awk '{print tolower($0)}')"
 676     case $lowerI in
 677         -\?|-h|--help)
 678             usage
 679             exit 1
 680             ;;
 681
 682         x86)
 683             __BuildArch=x86
 684             ;;
 685
 686         x64)
 687             __BuildArch=x64
 688             ;;
 689
 690         arm)
 691             __BuildArch=arm
 692             ;;
 693
 694         armel)
 695             __BuildArch=armel
 696             ;;
 697
 698         arm64)
 699             __BuildArch=arm64
 700             ;;
 701
 702         debug)
 703             __BuildType=Debug
 704             ;;
 705
 706         checked)
 707             __BuildType=Checked
 708             ;;
 709
 710         release)
 711             __BuildType=Release
 712             ;;
 713
 714         coverage)
 715             __CodeCoverage=Coverage
 716             ;;
 717
 718         cross)
 719             __CrossBuild=1
 720             ;;
 721
 722         portableBuild)
 723             __PortableBuild=1
 724             ;;
 725
 726         portablelinux)
 727             if [ "$__BuildOS" == "Linux" ]; then
 728                 __PortableLinux=1
 729             else
 730                 echo "ERROR: portableLinux not supported for non-Linux platforms."
 731                 exit 1
 732             fi
 733             ;;
 734
 735         verbose)
 736             __VerboseBuild=1
 737             ;;
 738
 739         clang3.5|-clang3.5)
 740             __ClangMajorVersion=3
 741             __ClangMinorVersion=5
 742             ;;
 743
 744         clang3.6|-clang3.6)
 745             __ClangMajorVersion=3
 746             __ClangMinorVersion=6
 747             ;;
 748
 749         clang3.7|-clang3.7)
 750             __ClangMajorVersion=3
 751             __ClangMinorVersion=7
 752             ;;
 753
 754         clang3.8|-clang3.8)
 755             __ClangMajorVersion=3
 756             __ClangMinorVersion=8
 757             ;;
 758
 759         clang3.9|-clang3.9)
 760             __ClangMajorVersion=3
 761             __ClangMinorVersion=9
 762             ;;
 763
 764         clang4.0|-clang4.0)
 765             __ClangMajorVersion=4
 766             __ClangMinorVersion=0
 767             ;;
 768
 769         clang5.0|-clang5.0)
 770             __ClangMajorVersion=5
 771             __ClangMinorVersion=0
 772             ;;
 773
 774         clang6.0|-clang6.0)
 775             __ClangMajorVersion=6
 776             __ClangMinorVersion=0
 777             ;;
 778
 779         ninja)
 780             __UseNinja=1
 781             ;;
 782
 783         runtests)
 784             __RunTests=1
 785             ;;
 786
 787         rebuild)
 788             __RebuildTests=1
 789             ;;
 790
 791         skipnative|-skipnative)
 792             __SkipNative=1
 793             ;;
 794
 795         skipmanaged|-skipmanaged)
 796             __SkipManaged=1
 797             __BuildTestWrappers=0
 798             ;;
 799
 800         ziptests)
 801             __ZipTests=1
 802             ;;
 803
 804         buildtestwrappersonly)
 805             __BuildTestWrappersOnly=1
 806             ;;
 807
 808         generatelayoutonly)
 809             __GenerateLayoutOnly=1
 810             ;;
 811
 812         generatetesthostonly)
 813             __GenerateTestHostOnly=1
 814             ;;
 815
 816         skiprestorepackages)
 817             __SkipRestorePackages=1
 818             ;;
 819
 820         bindir)
 821             if [ -n "$2" ]; then
 822                 __RootBinDir="$2"
 823                 if [ ! -d $__RootBinDir ]; then
 824                     mkdir $__RootBinDir
 825                 fi
 826                 __RootBinParent=$(dirname $__RootBinDir)
 827                 __RootBinName=${__RootBinDir##*/}
 828                 __RootBinDir="$(cd $__RootBinParent &>/dev/null && printf %s/%s $PWD $__RootBinName)"
 829                 shift
 830             else
 831                 echo "ERROR: 'bindir' requires a non-empty option argument"
 832                 exit 1
 833             fi
 834             ;;
 835
 836         msbuildonunsupportedplatform)
 837             __msbuildonunsupportedplatform=1
 838             ;;
 839
 840         priority1)
 841             __priority1=1
 842             __UnprocessedBuildArgs+=("-priority=1")
 843             ;;
 844
 845         *)
 846             __UnprocessedBuildArgs+=("$1")
 847             ;;
 848     esac
 849
 850     shift
 851 done
 852
 853 # Get the number of processors available to the scheduler
 854 # Other techniques such as `nproc` only get the number of
 855 # processors available to a single process.
 856 if [ `uname` = "FreeBSD" ]; then
 857   __NumProc=`sysctl hw.ncpu | awk '{ print $2+1 }'`
 858 elif [ `uname` = "NetBSD" ]; then
 859   __NumProc=$(($(getconf NPROCESSORS_ONLN)+1))
 860 elif [ `uname` = "Darwin" ]; then
 861   __NumProc=$(($(getconf _NPROCESSORS_ONLN)+1))
 862 else
 863   __NumProc=$(nproc --all)
 864 fi
 865
 866 __RunArgs=("-BuildArch=$__BuildArch" "-BuildType=$__BuildType" "-BuildOS=$__BuildOS")
 867
 868 # Configure environment if we are doing a verbose build
 869 if [ $__VerboseBuild == 1 ]; then
 870     export VERBOSE=1
 871     __RunArgs+=("-verbose")
 872 fi
 873
 874 # Set default clang version
 875 if [[ $__ClangMajorVersion == 0 && $__ClangMinorVersion == 0 ]]; then
 876     if [[ "$__BuildArch" == "arm" || "$__BuildArch" == "armel" ]]; then
 877         __ClangMajorVersion=5
 878         __ClangMinorVersion=0
 879     else
 880         __ClangMajorVersion=3
 881         __ClangMinorVersion=9
 882     fi
 883 fi
 884
 885 # Set dependent variables
 886 __LogsDir="$__RootBinDir/Logs"
 887
 888 # init the host distro name
 889 initHostDistroRid
 890
 891 # Set the remaining variables based upon the determined build configuration
 892 __BinDir="$__RootBinDir/Product/$__BuildOS.$__BuildArch.$__BuildType"
 893 __PackagesBinDir="$__BinDir/.nuget"
 894 __TestDir="$__ProjectDir/tests"
 895 __TestWorkingDir="$__RootBinDir/tests/$__BuildOS.$__BuildArch.$__BuildType"
 896 __IntermediatesDir="$__RootBinDir/obj/$__BuildOS.$__BuildArch.$__BuildType"
 897 __TestIntermediatesDir="$__RootBinDir/tests/obj/$__BuildOS.$__BuildArch.$__BuildType"
 898 __isMSBuildOnNETCoreSupported=0
 899 __CrossComponentBinDir="$__BinDir"
 900 __CrossCompIntermediatesDir="$__IntermediatesDir/crossgen"
 901
 902 __CrossArch="$__HostArch"
 903 if [[ "$__HostArch" == "x64" && "$__BuildArch" == "arm" ]]; then
 904     __CrossArch="x86"
 905 fi
 906 if [ $__CrossBuild == 1 ]; then
 907     __CrossComponentBinDir="$__CrossComponentBinDir/$__CrossArch"
 908 fi
 909 __CrossgenCoreLibLog="$__LogsDir/CrossgenCoreLib_$__BuildOS.$BuildArch.$__BuildType.log"
 910 __CrossgenExe="$__CrossComponentBinDir/crossgen"
 911
 912 isMSBuildOnNETCoreSupported
 913
 914 # CI_SPECIFIC - On CI machines, $HOME may not be set. In such a case, create a subfolder and set the variable to it.
 915 # This is needed by CLI to function.
 916 if [ -z "$HOME" ]; then
 917     if [ ! -d "$__ProjectDir/temp_home" ]; then
 918         mkdir temp_home
 919     fi
 920     export HOME=$__ProjectDir/temp_home
 921     echo "HOME not defined; setting it to $HOME"
 922 fi
 923
 924 # Configure environment if we are doing a cross compile.
 925 if [ $__CrossBuild == 1 ]; then
 926     export CROSSCOMPILE=1
 927     if ! [[ -n "$ROOTFS_DIR" ]]; then
 928         export ROOTFS_DIR="$__ProjectRoot/cross/rootfs/$__BuildArch"
 929     fi
 930 fi
 931
 932 # init the target distro name
 933 initTargetDistroRid
 934
 935 # Override tool directory
 936
 937 __CoreClrVersion=1.1.0
 938 __sharedFxDir=$__BuildToolsDir/dotnetcli/shared/Microsoft.NETCore.App/$__CoreClrVersion/
 939
 940 if [[ (-z "$__GenerateLayoutOnly") && (-z "$__GenerateTestHostOnly") && (-z "$__BuildTestWrappersOnly") ]]; then
 941     build_Tests
 942 elif [ ! -z "$__BuildTestWrappersOnly" ]; then
 943     build_test_wrappers
 944 else
 945     generate_layout
 946     if [ ! -z "$__GenerateTestHostOnly" ]; then
 947         generate_testhost
 948     fi
 949 fi
 950
 951 if [ $? -ne 0 ]; then
 952     echo "Failed to build tests"
 953     exit 1
 954 fi
 955
 956 echo "${__MsgPrefix}Test build successful."
 957 echo "${__MsgPrefix}Test binaries are available at ${__TestBinDir}"
 958
 959 __testNativeBinDir=$__IntermediatesDir/tests
 960
 961 if [ $__RunTests -ne 0 ]; then
 962
 963     echo "Run Tests..."
 964
 965     nextCommand="$__TestDir/runtest.sh --testRootDir=$__TestBinDir --coreClrBinDir=$__BinDir --coreFxBinDir=$CORE_ROOT --testNativeBinDir=$__testNativeBinDir"
 966     echo "$nextCommand"
 967     eval $nextCommand
 968
 969     echo "Tests run successful."
 970 else
 971     echo "To run all tests use 'tests/runtests.sh' where:"
 972     echo "    testRootDir      = $__TestBinDir"
 973     echo "    coreClrBinDir    = $__BinDir"
 974     echo "    coreFxBinDir     = $CORE_ROOT"
 975     echo "    testNativeBinDir = $__testNativeBinDir"
 976     echo " -------------------------------------------------- "
 977     echo " Example runtest.sh command"
 978     echo ""
 979     echo " ./tests/runtest.sh --coreOverlayDir=$CORE_ROOT --testNativeBinDir=$__testNativeBinDir --testRootDir=$__TestBinDir --copyNativeTestBin"
 980     echo " -------------------------------------------------- "
 981     echo "To run single test use the following command:"
 982     echo "    bash ${__TestBinDir}/__TEST_PATH__/__TEST_NAME__.sh -coreroot=${CORE_ROOT}"
 983 fi
 984