build-test.sh

   1 #!/usr/bin/env bash
   2
   3 initHostDistroRid()
   4 {
   5     __HostDistroRid=""
   6
   7     # Some OS groups should default to use the portable packages
   8     if [ "$__BuildOS" == "OSX" ]; then
   9         __PortableBuild=1
  10     fi
  11
  12     if [ "$__HostOS" == "Linux" ]; then
  13         if [ -e /etc/redhat-release ]; then
  14             local redhatRelease=$(</etc/redhat-release)
  15             if [[ $redhatRelease == "CentOS release 6."* || $redhatRelease == "Red Hat Enterprise Linux Server release 6."* ]]; then
  16                 __HostDistroRid="rhel.6-$__HostArch"
  17             else
  18                 __PortableBuild=1
  19             fi
  20         elif [ -e /etc/os-release ]; then
  21             source /etc/os-release
  22             if [[ $ID == "alpine" ]]; then
  23                 __HostDistroRid="linux-musl-$__HostArch"
  24             else
  25                 __PortableBuild=1
  26                 __HostDistroRid="$ID.$VERSION_ID-$__HostArch"
  27             fi
  28         fi
  29     elif [ "$__HostOS" == "FreeBSD" ]; then
  30         __freebsd_version=`sysctl -n kern.osrelease | cut -f1 -d'.'`
  31         __HostDistroRid="freebsd.$__freebsd_version-$__HostArch"
  32     fi
  33
  34     # Portable builds target the base RID
  35     if [ "$__PortableBuild" == 1 ]; then
  36         if [ "$__BuildOS" == "OSX" ]; then
  37             export __HostDistroRid="osx-$__BuildArch"
  38         elif [ "$__BuildOS" == "Linux" ]; then
  39             export __HostDistroRid="linux-$__BuildArch"
  40         fi
  41     fi
  42
  43     if [ "$__HostDistroRid" == "" ]; then
  44         echo "WARNING: Cannot determine runtime id for current distro."
  45     fi
  46
  47     echo "Setting __HostDistroRid to $__HostDistroRid"
  48 }
  49
  50 initTargetDistroRid()
  51 {
  52     if [ $__CrossBuild == 1 ]; then
  53         if [ "$__BuildOS" == "Linux" ]; then
  54             if [ ! -e $ROOTFS_DIR/etc/os-release ]; then
  55                 if [ -e $ROOTFS_DIR/android_platform ]; then
  56                     source $ROOTFS_DIR/android_platform
  57                     export __DistroRid="$RID"
  58                 else
  59                     echo "WARNING: Cannot determine runtime id for current distro."
  60                     export __DistroRid=""
  61                 fi
  62             else
  63                 source $ROOTFS_DIR/etc/os-release
  64                 export __DistroRid="$ID.$VERSION_ID-$__BuildArch"
  65             fi
  66         fi
  67     else
  68         export __DistroRid="$__HostDistroRid"
  69     fi
  70
  71     if [ "$ID.$VERSION_ID" == "ubuntu.16.04" ]; then
  72      export __DistroRid="ubuntu.14.04-$__BuildArch"
  73     fi
  74
  75     # Portable builds target the base RID
  76     if [ "$__PortableBuild" == 1 ]; then
  77         if [ "$__BuildOS" == "Linux" ]; then
  78             export __DistroRid="linux-$__BuildArch"
  79             export __RuntimeId="linux-$__BuildArch"
  80         elif [ "$__BuildOS" == "OSX" ]; then
  81             export __DistroRid="osx-$__BuildArch"
  82             export __RuntimeId="osx-$__BuildArch"
  83         fi
  84     fi
  85
  86     echo "__DistroRid: " $__DistroRid
  87 }
  88
  89 isMSBuildOnNETCoreSupported()
  90 {
  91     __isMSBuildOnNETCoreSupported=$__msbuildonunsupportedplatform
  92
  93     if [ $__isMSBuildOnNETCoreSupported == 1 ]; then
  94         return
  95     fi
  96
  97     if [ "$__HostArch" == "x64" ]; then
  98         if [ "$__HostOS" == "Linux" ]; then
  99             __isMSBuildOnNETCoreSupported=1
 100             UNSUPPORTED_RIDS=("debian.9-x64" "ubuntu.17.04-x64")
 101             for UNSUPPORTED_RID in "${UNSUPPORTED_RIDS[@]}"
 102             do
 103                 if [ "$__HostDistroRid" == "$UNSUPPORTED_RID" ]; then
 104                     __isMSBuildOnNETCoreSupported=0
 105                     break
 106                 fi
 107             done
 108         elif [ "$__HostOS" == "OSX" ]; then
 109             __isMSBuildOnNETCoreSupported=1
 110         fi
 111     fi
 112 }
 113
 114 build_test_wrappers()
 115 {
 116     if [ $__BuildTestWrappers -ne -0 ]; then
 117         echo "${__MsgPrefix}Creating test wrappers..."
 118
 119         export __Exclude="${__ProjectDir}/tests/issues.targets"
 120         export __BuildLogRootName="Tests_XunitWrapper"
 121
 122         buildVerbosity="Summary"
 123
 124         if [ $__VerboseBuild == 1 ]; then
 125             buildVerbosity="Diag"
 126         fi
 127
 128         # Set up directories and file names
 129         __BuildLogRootName=$subDirectoryName
 130         __BuildLog="$__LogsDir/${__BuildLogRootName}.${__BuildOS}.${__BuildArch}.${__BuildType}.log"
 131         __BuildWrn="$__LogsDir/${__BuildLogRootName}.${__BuildOS}.${__BuildArch}.${__BuildType}.wrn"
 132         __BuildErr="$__LogsDir/${__BuildLogRootName}.${__BuildOS}.${__BuildArch}.${__BuildType}.err"
 133         __MsbuildLog="/fileloggerparameters:\"Verbosity=normal;LogFile=${__BuildLog}\""
 134         __MsbuildWrn="/fileloggerparameters1:\"WarningsOnly;LogFile=${__BuildWrn}\""
 135         __MsbuildErr="/fileloggerparameters2:\"ErrorsOnly;LogFile=${__BuildErr}\""
 136         __Logging="$__MsbuildLog $__MsbuildWrn $__MsbuildErr /consoleloggerparameters:$buildVerbosity"
 137
 138         nextCommand="\"${__DotNetCli}\" msbuild \"${__ProjectDir}/tests/runtest.proj\" /p:RestoreAdditionalProjectSources=https://dotnet.myget.org/F/dotnet-core/ /p:BuildWrappers=true /p:TargetsWindows=false $__Logging /p:__BuildOS=$__BuildOS /p:__BuildType=$__BuildType /p:__BuildArch=$__BuildArch"
 139         echo "$nextCommand"
 140         eval $nextCommand
 141
 142         if [ $? -ne 0 ]; then
 143             echo "${__MsgPrefix}Error: build failed. Refer to the build log files for details (above)"
 144             exit 1
 145         else
 146             echo "XUnit Wrappers have been built."
 147             echo { "\"build_os\"": "\"${__BuildOS}\"", "\"build_arch\"": "\"${__BuildArch}\"", "\"build_type\"": "\"${__BuildType}\"" } > "${__TestWorkingDir}/build_info.json"
 148
 149         fi
 150     fi
 151 }
 152
 153 generate_layout()
 154 {
 155     echo "${__MsgPrefix}Creating test overlay..."
 156
 157     __TestDir=$__ProjectDir/tests
 158     __ProjectFilesDir=$__TestDir
 159     __TestBinDir=$__TestWorkingDir
 160
 161     if [ $__RebuildTests -ne 0 ]; then
 162         if [ -d "${__TestBinDir}" ]; then
 163             echo "Removing tests build dir: ${__TestBinDir}"
 164             rm -rf $__TestBinDir
 165         fi
 166     fi
 167
 168     __CMakeBinDir="${__TestBinDir}"
 169
 170     if [ -z "$__TestIntermediateDir" ]; then
 171         __TestIntermediateDir="tests/obj/${__BuildOS}.${__BuildArch}.${__BuildType}"
 172     fi
 173
 174     echo "__BuildOS: ${__BuildOS}"
 175     echo "__BuildArch: ${__BuildArch}"
 176     echo "__BuildType: ${__BuildType}"
 177     echo "__TestIntermediateDir: ${__TestIntermediateDir}"
 178
 179     if [ ! -f "$__TestBinDir" ]; then
 180         echo "Creating TestBinDir: ${__TestBinDir}"
 181         mkdir -p $__TestBinDir
 182     fi
 183     if [ ! -f "$__LogsDir" ]; then
 184         echo "Creating LogsDir: ${__LogsDir}"
 185         mkdir -p $__LogsDir
 186     fi
 187
 188     __BuildProperties="-p:OSGroup=${__BuildOS} -p:BuildOS=${__BuildOS} -p:BuildArch=${__BuildArch} -p:BuildType=${__BuildType}"
 189
 190     # =========================================================================================
 191     # ===
 192     # === Restore product binaries from packages
 193     # ===
 194     # =========================================================================================
 195
 196     build_MSBuild_projects "Restore_Packages" "${__ProjectDir}/tests/build.proj" "Restore product binaries (build tests)" "-BatchRestorePackages"
 197
 198     if [ -n "$__UpdateInvalidPackagesArg" ]; then
 199         __up=-updateinvalidpackageversion
 200     fi
 201
 202     echo "${__MsgPrefix}Creating test overlay..."
 203
 204     if [ -z "$xUnitTestBinBase" ]; then
 205         xUnitTestBinBase=$__TestWorkingDir
 206     fi
 207
 208     export CORE_ROOT=$xUnitTestBinBase/Tests/Core_Root
 209
 210     if [ -d "${CORE_ROOT}" ]; then
 211         rm -rf $CORE_ROOT
 212     fi
 213
 214     mkdir -p $CORE_ROOT
 215
 216     build_MSBuild_projects "Tests_Overlay_Managed" "${__ProjectDir}/tests/runtest.proj" "Creating test overlay" "-testOverlay"
 217
 218     chmod +x $__BinDir/corerun
 219     chmod +x $__BinDir/crossgen
 220
 221     # Make sure to copy over the pulled down packages
 222     cp -r $__BinDir/* $CORE_ROOT/ > /dev/null
 223 }
 224
 225 generate_testhost()
 226 {
 227     echo "${__MsgPrefix}Generating test host..."
 228
 229     export TEST_HOST=$xUnitTestBinBase/testhost
 230
 231     if [ -d "${TEST_HOST}" ]; then
 232         rm -rf $TEST_HOST
 233     fi
 234
 235     mkdir -p $TEST_HOST
 236
 237     build_MSBuild_projects "Tests_Generate_TestHost" "${__ProjectDir}/tests/runtest.proj" "Creating test host" "-testHost"
 238 }
 239
 240
 241 build_Tests()
 242 {
 243     echo "${__MsgPrefix}Building Tests..."
 244
 245     __TestDir=$__ProjectDir/tests
 246     __ProjectFilesDir=$__TestDir
 247     __TestBinDir=$__TestWorkingDir
 248
 249     if [ -f  "${__TestWorkingDir}/build_info.json" ]; then
 250         rm  "${__TestWorkingDir}/build_info.json"
 251     fi
 252
 253     if [ $__RebuildTests -ne 0 ]; then
 254         if [ -d "${__TestBinDir}" ]; then
 255             echo "Removing tests build dir: ${__TestBinDir}"
 256             rm -rf $__TestBinDir
 257         fi
 258     fi
 259
 260     export __CMakeBinDir="${__TestBinDir}"
 261     if [ ! -d "${__TestIntermediatesDir}" ]; then
 262         mkdir -p ${__TestIntermediatesDir}
 263     fi
 264
 265     __NativeTestIntermediatesDir="${__TestIntermediatesDir}/Native"
 266     if [  ! -d "${__NativeTestIntermediatesDir}" ]; then
 267         mkdir -p ${__NativeTestIntermediatesDir}
 268     fi
 269
 270     __ManagedTestIntermediatesDir="${__TestIntermediatesDir}/Managed"
 271     if [ ! -d "${__ManagedTestIntermediatesDir}" ]; then
 272         mkdir -p ${__ManagedTestIntermediatesDir}
 273     fi
 274
 275     echo "__BuildOS: ${__BuildOS}"
 276     echo "__BuildArch: ${__BuildArch}"
 277     echo "__BuildType: ${__BuildType}"
 278     echo "__TestIntermediatesDir: ${__TestIntermediatesDir}"
 279     echo "__NativeTestIntermediatesDir: ${__NativeTestIntermediatesDir}"
 280     echo "__ManagedTestIntermediatesDir: ${__ManagedTestIntermediatesDir}"
 281
 282     if [ ! -f "$__TestBinDir" ]; then
 283         echo "Creating TestBinDir: ${__TestBinDir}"
 284         mkdir -p $__TestBinDir
 285     fi
 286     if [ ! -f "$__LogsDir" ]; then
 287         echo "Creating LogsDir: ${__LogsDir}"
 288         mkdir -p $__LogsDir
 289     fi
 290
 291     __BuildProperties="-p:OSGroup=${__BuildOS} -p:BuildOS=${__BuildOS} -p:BuildArch=${__BuildArch} -p:BuildType=${__BuildType}"
 292
 293     # =========================================================================================
 294     # ===
 295     # === Restore product binaries from packages
 296     # ===
 297     # =========================================================================================
 298
 299     if [ ${__SkipRestorePackages} != 1 ]; then
 300         build_MSBuild_projects "Restore_Product" "${__ProjectDir}/tests/build.proj" "Restore product binaries (build tests)" "-BatchRestorePackages"
 301     fi
 302
 303     if [ $__SkipNative != 1 ]; then
 304         build_native_projects "$__BuildArch" "${__NativeTestIntermediatesDir}"
 305
 306         if [ $? -ne 0 ]; then
 307             echo "${__MsgPrefix}Error: build failed. Refer to the build log files for details (above)"
 308             exit 1
 309         fi
 310     fi
 311
 312     if [ $__SkipManaged != 1 ]; then
 313         echo "Starting the Managed Tests Build..."
 314
 315         build_MSBuild_projects "Tests_Managed" "$__ProjectDir/tests/build.proj" "Managed tests build (build tests)" "$__up"
 316
 317         if [ $? -ne 0 ]; then
 318             echo "${__MsgPrefix}Error: build failed. Refer to the build log files for details (above)"
 319             exit 1
 320         else
 321             echo "Checking the Managed Tests Build..."
 322
 323             build_MSBuild_projects "Check_Test_Build" "${__ProjectDir}/tests/runtest.proj" "Check Test Build" "-ExtraParameters:/t:CheckTestBuild"
 324
 325             if [ $? -ne 0 ]; then
 326                 echo "${__MsgPrefix}Error: Check Test Build failed."
 327                 exit 1
 328             fi
 329         fi
 330
 331         echo "Managed tests build success!"
 332     fi
 333
 334     build_test_wrappers
 335
 336     if [ -n "$__UpdateInvalidPackagesArg" ]; then
 337         __up=-updateinvalidpackageversion
 338     fi
 339
 340     generate_layout
 341
 342     if [ $__ZipTests -ne 0 ]; then
 343         echo "${__MsgPrefix}ZIP tests packages..."
 344         build_MSBuild_projects "Helix_Prep" "$__ProjectDir/tests/helixprep.proj" "Prep test binaries for Helix publishing" " "
 345     fi
 346 }
 347
 348 build_MSBuild_projects()
 349 {
 350     subDirectoryName=$1
 351     shift
 352     projectName=$1
 353     shift
 354     stepName="$1"
 355     shift
 356     extraBuildParameters=("$@")
 357
 358     # Set up directories and file names
 359     __BuildLogRootName=$subDirectoryName
 360     __BuildLog="$__LogsDir/${__BuildLogRootName}.${__BuildOS}.${__BuildArch}.${__BuildType}.log"
 361     __BuildWrn="$__LogsDir/${__BuildLogRootName}.${__BuildOS}.${__BuildArch}.${__BuildType}.wrn"
 362     __BuildErr="$__LogsDir/${__BuildLogRootName}.${__BuildOS}.${__BuildArch}.${__BuildType}.err"
 363
 364     # Use binclashlogger by default if no other logger is specified
 365     if [[ "${extraBuildParameters[*]}" == *"-MsBuildEventLogging"* ]]; then
 366         msbuildEventLogging=""
 367     else
 368         msbuildEventLogging="-MsBuildEventLogging=\"/l:BinClashLogger,Tools/Microsoft.DotNet.Build.Tasks.dll;LogFile=binclash.log\""
 369     fi
 370
 371     if [[ "$subDirectoryName" == "Tests_Managed" ]]; then
 372         # Execute msbuild managed test build in stages - workaround for excessive data retention in MSBuild ConfigCache
 373         # See https://github.com/Microsoft/msbuild/issues/2993
 374
 375         # __SkipPackageRestore and __SkipTargetingPackBuild used  to control build by tests/src/dirs.proj
 376         export __SkipPackageRestore=false
 377         export __SkipTargetingPackBuild=false
 378         export __BuildLoopCount=2
 379         export __TestGroupToBuild=1
 380         __AppendToLog=false
 381
 382         if [ -n "$__priority1" ]; then
 383             export __BuildLoopCount=16
 384             export __TestGroupToBuild=2
 385         fi
 386
 387         for (( slice=1 ; slice <= __BuildLoopCount; slice = slice + 1 ))
 388         do
 389             __msbuildLog="\"/flp:Verbosity=normal;LogFile=${__BuildLog};Append=${__AppendToLog}\""
 390             __msbuildWrn="\"/flp1:WarningsOnly;LogFile=${__BuildWrn};Append=${__AppendToLog}\""
 391             __msbuildErr="\"/flp2:ErrorsOnly;LogFile=${__BuildErr};Append=${__AppendToLog}\""
 392
 393             export TestBuildSlice=$slice
 394
 395             # Generate build command
 396             buildArgs=("-Project=$projectName" "-MsBuildLog=${__msbuildLog}" "-MsBuildWrn=${__msbuildWrn}" "-MsBuildErr=${__msbuildErr}")
 397             buildArgs+=("$msbuildEventLogging")
 398             buildArgs+=("${extraBuildParameters[@]}")
 399             buildArgs+=("${__RunArgs[@]}")
 400             buildArgs+=("${__UnprocessedBuildArgs[@]}")
 401
 402             nextCommand="\"$__ProjectRoot/run.sh\" build ${buildArgs[@]}"
 403             echo "Building step '$stepName' slice=$slice via $nextCommand"
 404             eval $nextCommand
 405
 406             # Make sure everything is OK
 407             if [ $? -ne 0 ]; then
 408                 echo "${__MsgPrefix}Failed to build $stepName. See the build logs:"
 409                 echo "    $__BuildLog"
 410                 echo "    $__BuildWrn"
 411                 echo "    $__BuildErr"
 412                 exit 1
 413             fi
 414             export __SkipPackageRestore=true
 415             export __SkipTargetingPackBuild=true
 416             __AppendToLog=true
 417         done
 418     else
 419         __msbuildLog="\"/flp:Verbosity=normal;LogFile=${__BuildLog}\""
 420         __msbuildWrn="\"/flp1:WarningsOnly;LogFile=${__BuildWrn}\""
 421         __msbuildErr="\"/flp2:ErrorsOnly;LogFile=${__BuildErr}\""
 422
 423         # Generate build command
 424         buildArgs=("-Project=$projectName" "-MsBuildLog=${__msbuildLog}" "-MsBuildWrn=${__msbuildWrn}" "-MsBuildErr=${__msbuildErr}")
 425         buildArgs+=("$msbuildEventLogging")
 426         buildArgs+=("${extraBuildParameters[@]}")
 427         buildArgs+=("${__RunArgs[@]}")
 428         buildArgs+=("${__UnprocessedBuildArgs[@]}")
 429
 430         nextCommand="\"$__ProjectRoot/run.sh\" build ${buildArgs[@]}"
 431         echo "Building step '$stepName' via $nextCommand"
 432         eval $nextCommand
 433
 434         # Make sure everything is OK
 435         if [ $? -ne 0 ]; then
 436             echo "${__MsgPrefix}Failed to build $stepName. See the build logs:"
 437             echo "    $__BuildLog"
 438             echo "    $__BuildWrn"
 439             echo "    $__BuildErr"
 440             exit 1
 441         fi
 442     fi
 443 }
 444
 445 build_native_projects()
 446 {
 447     platformArch="$1"
 448     intermediatesForBuild="$2"
 449
 450     extraCmakeArguments="-DCLR_CMAKE_TARGET_OS=${__BuildOS} -DCLR_CMAKE_HOST_ARCH=${platformArch}"
 451     message="native tests assets"
 452
 453     # All set to commence the build
 454     echo "Commencing build of $message for $__BuildOS.$__BuildArch.$__BuildType in $intermediatesForBuild"
 455
 456     generator=""
 457     buildFile="Makefile"
 458     buildTool="make"
 459     if [ $__UseNinja == 1 ]; then
 460         generator="ninja"
 461         buildFile="build.ninja"
 462         if ! buildTool=$(command -v ninja || command -v ninja-build); then
 463            echo "Unable to locate ninja!" 1>&2
 464            exit 1
 465         fi
 466     fi
 467
 468     if [ $__SkipConfigure == 0 ]; then
 469         # if msbuild is not supported, then set __SkipGenerateVersion to 1
 470         if [ $__isMSBuildOnNETCoreSupported == 0 ]; then __SkipGenerateVersion=1; fi
 471         # Drop version.cpp file
 472         __versionSourceFile="$intermediatesForBuild/version.cpp"
 473         if [ $__SkipGenerateVersion == 0 ]; then
 474             pwd
 475             "$__ProjectRoot/run.sh" build -Project=$__ProjectDir/build.proj -generateHeaderUnix -NativeVersionSourceFile=$__versionSourceFile -MsBuildEventLogging="/l:BinClashLogger,Tools/Microsoft.DotNet.Build.Tasks.dll;LogFile=binclash.log"  $__RunArgs $__UnprocessedBuildArgs
 476         else
 477             # Generate the dummy version.cpp, but only if it didn't exist to make sure we don't trigger unnecessary rebuild
 478             __versionSourceLine="static char sccsid[] __attribute__((used)) = \"@(#)No version information produced\";"
 479             if [ -e $__versionSourceFile ]; then
 480                 read existingVersionSourceLine < $__versionSourceFile
 481             fi
 482             if [ "$__versionSourceLine" != "$existingVersionSourceLine" ]; then
 483                 echo $__versionSourceLine > $__versionSourceFile
 484             fi
 485         fi
 486
 487         pushd "$intermediatesForBuild"
 488         # Regenerate the CMake solution
 489         # Force cross dir to point to project root cross dir, in case there is a cross build.
 490         nextCommand="CONFIG_DIR=\"$__ProjectRoot/cross\" \"$__ProjectRoot/src/pal/tools/gen-buildsys-clang.sh\" \"$__TestDir\" $__ClangMajorVersion $__ClangMinorVersion $platformArch $__BuildType $__CodeCoverage $generator $extraCmakeArguments $__cmakeargs"
 491         echo "Invoking $nextCommand"
 492         eval $nextCommand
 493         popd
 494     fi
 495
 496     if [ ! -f "$intermediatesForBuild/$buildFile" ]; then
 497         echo "Failed to generate $message build project!"
 498         exit 1
 499     fi
 500
 501     # Build
 502     if [ $__ConfigureOnly == 1 ]; then
 503         echo "Finish configuration & skipping $message build."
 504         return
 505     fi
 506
 507     pushd "$intermediatesForBuild"
 508
 509     echo "Executing $buildTool install -j $__NumProc"
 510
 511     $buildTool install -j $__NumProc
 512     if [ $? != 0 ]; then
 513         echo "Failed to build $message."
 514         exit 1
 515     fi
 516
 517     popd
 518     echo "Native tests build success!"
 519 }
 520
 521 usage()
 522 {
 523     echo "Usage: $0 [BuildArch] [BuildType] [verbose] [coverage] [cross] [clangx.y] [ninja] [runtests] [bindir]"
 524     echo "BuildArch can be: x64, x86, arm, armel, arm64"
 525     echo "BuildType can be: debug, checked, release"
 526     echo "coverage - optional argument to enable code coverage build (currently supported only for Linux and OSX)."
 527     echo "ninja - target ninja instead of GNU make"
 528     echo "clangx.y - optional argument to build using clang version x.y - supported version 3.5 - 6.0"
 529     echo "cross - optional argument to signify cross compilation,"
 530     echo "      - will use ROOTFS_DIR environment variable if set."
 531     echo "crosscomponent - optional argument to build cross-architecture component,"
 532     echo "               - will use CAC_ROOTFS_DIR environment variable if set."
 533     echo "portableLinux - build for Portable Linux Distribution"
 534     echo "portablebuild - Use portable build."
 535     echo "verbose - optional argument to enable verbose build output."
 536     echo "rebuild - if tests have already been built - rebuild them"
 537     echo "skipnative: skip the native tests build"
 538     echo "skipmanaged: skip the managed section of the test build"
 539     echo "buildtestwrappersonly - only build the test wrappers"
 540     echo "generatelayoutonly - only pull down dependencies and build coreroot"
 541     echo "generatetesthostonly - only pull down dependencies and build coreroot and the CoreFX testhost"
 542     echo "skiprestorepackages - skip package restore"
 543     echo "runtests - run tests after building them"
 544     echo "ziptests - zips CoreCLR tests & Core_Root for a Helix run"
 545     echo "bindir - output directory (defaults to $__ProjectRoot/bin)"
 546     echo "msbuildonunsupportedplatform - build managed binaries even if distro is not officially supported."
 547     echo "priority1 - include priority=1 tests in the build"
 548     exit 1
 549 }
 550
 551
 552 # Obtain the location of the bash script to figure out where the root of the repo is.
 553 __ProjectRoot="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
 554
 555 # Use uname to determine what the CPU is.
 556 CPUName=$(uname -p)
 557
 558 # Some Linux platforms report unknown for platform, but the arch for machine.
 559 if [ "$CPUName" == "unknown" ]; then
 560     CPUName=$(uname -m)
 561 fi
 562
 563 case $CPUName in
 564     i686)
 565         echo "Unsupported CPU $CPUName detected, build might not succeed!"
 566         __BuildArch=x86
 567         __HostArch=x86
 568         ;;
 569
 570     x86_64)
 571         __BuildArch=x64
 572         __HostArch=x64
 573         ;;
 574
 575     armv7l)
 576         echo "Unsupported CPU $CPUName detected, build might not succeed!"
 577         __BuildArch=arm
 578         __HostArch=arm
 579         ;;
 580
 581     aarch64)
 582         __BuildArch=arm64
 583         __HostArch=arm64
 584         ;;
 585
 586     *)
 587         echo "Unknown CPU $CPUName detected, configuring as if for x64"
 588         __BuildArch=x64
 589         __HostArch=x64
 590         ;;
 591 esac
 592
 593 # Use uname to determine what the OS is.
 594 OSName=$(uname -s)
 595 case $OSName in
 596     Linux)
 597         __BuildOS=Linux
 598         __HostOS=Linux
 599         ;;
 600
 601     Darwin)
 602         __BuildOS=OSX
 603         __HostOS=OSX
 604         ;;
 605
 606     FreeBSD)
 607         __BuildOS=FreeBSD
 608         __HostOS=FreeBSD
 609         ;;
 610
 611     OpenBSD)
 612         __BuildOS=OpenBSD
 613         __HostOS=OpenBSD
 614         ;;
 615
 616     NetBSD)
 617         __BuildOS=NetBSD
 618         __HostOS=NetBSD
 619         ;;
 620
 621     SunOS)
 622         __BuildOS=SunOS
 623         __HostOS=SunOS
 624         ;;
 625
 626     *)
 627         echo "Unsupported OS $OSName detected, configuring as if for Linux"
 628         __BuildOS=Linux
 629         __HostOS=Linux
 630         ;;
 631 esac
 632
 633 __BuildType=Debug
 634 __CodeCoverage=
 635 __IncludeTests=INCLUDE_TESTS
 636
 637 # Set the various build properties here so that CMake and MSBuild can pick them up
 638 export __ProjectDir="$__ProjectRoot"
 639 __SourceDir="$__ProjectDir/src"
 640 __PackagesDir="$__ProjectDir/packages"
 641 __RootBinDir="$__ProjectDir/bin"
 642 __BuildToolsDir="$__ProjectDir/Tools"
 643 __DotNetCli="${__BuildToolsDir}/dotnetcli/dotnet"
 644 __UnprocessedBuildArgs=
 645 __RunArgs=
 646 __MSBCleanBuildArgs=
 647 __UseNinja=0
 648 __VerboseBuild=0
 649 __SkipRestore=""
 650 __SkipNative=0
 651 __SkipManaged=0
 652 __SkipConfigure=0
 653 __SkipGenerateVersion=0
 654 __ConfigureOnly=0
 655 __CrossBuild=0
 656 __ClangMajorVersion=0
 657 __ClangMinorVersion=0
 658 __NuGetPath="$__PackagesDir/NuGet.exe"
 659 __HostDistroRid=""
 660 __SkipRestorePackages=0
 661 __DistroRid=""
 662 __cmakeargs=""
 663 __PortableLinux=0
 664 __msbuildonunsupportedplatform=0
 665 __ZipTests=0
 666 __NativeTestIntermediatesDir=
 667 __RunTests=0
 668 __RebuildTests=0
 669 __BuildTestWrappers=1
 670 __GenerateLayoutOnly=
 671 __GenerateTestHostOnly=
 672 __priority1=
 673 __BuildTestWrappersOnly=
 674 CORE_ROOT=
 675
 676 while :; do
 677     if [ $# -le 0 ]; then
 678         break
 679     fi
 680
 681     lowerI="$(echo $1 | awk '{print tolower($0)}')"
 682     case $lowerI in
 683         -\?|-h|--help)
 684             usage
 685             exit 1
 686             ;;
 687
 688         x86)
 689             __BuildArch=x86
 690             ;;
 691
 692         x64)
 693             __BuildArch=x64
 694             ;;
 695
 696         arm)
 697             __BuildArch=arm
 698             ;;
 699
 700         armel)
 701             __BuildArch=armel
 702             ;;
 703
 704         arm64)
 705             __BuildArch=arm64
 706             ;;
 707
 708         debug)
 709             __BuildType=Debug
 710             ;;
 711
 712         checked)
 713             __BuildType=Checked
 714             ;;
 715
 716         release)
 717             __BuildType=Release
 718             ;;
 719
 720         coverage)
 721             __CodeCoverage=Coverage
 722             ;;
 723
 724         cross)
 725             __CrossBuild=1
 726             ;;
 727
 728         portableBuild)
 729             __PortableBuild=1
 730             ;;
 731
 732         portablelinux)
 733             if [ "$__BuildOS" == "Linux" ]; then
 734                 __PortableLinux=1
 735             else
 736                 echo "ERROR: portableLinux not supported for non-Linux platforms."
 737                 exit 1
 738             fi
 739             ;;
 740
 741         verbose)
 742             __VerboseBuild=1
 743             ;;
 744
 745         clang3.5|-clang3.5)
 746             __ClangMajorVersion=3
 747             __ClangMinorVersion=5
 748             ;;
 749
 750         clang3.6|-clang3.6)
 751             __ClangMajorVersion=3
 752             __ClangMinorVersion=6
 753             ;;
 754
 755         clang3.7|-clang3.7)
 756             __ClangMajorVersion=3
 757             __ClangMinorVersion=7
 758             ;;
 759
 760         clang3.8|-clang3.8)
 761             __ClangMajorVersion=3
 762             __ClangMinorVersion=8
 763             ;;
 764
 765         clang3.9|-clang3.9)
 766             __ClangMajorVersion=3
 767             __ClangMinorVersion=9
 768             ;;
 769
 770         clang4.0|-clang4.0)
 771             __ClangMajorVersion=4
 772             __ClangMinorVersion=0
 773             ;;
 774
 775         clang5.0|-clang5.0)
 776             __ClangMajorVersion=5
 777             __ClangMinorVersion=0
 778             ;;
 779
 780         clang6.0|-clang6.0)
 781             __ClangMajorVersion=6
 782             __ClangMinorVersion=0
 783             ;;
 784
 785         ninja)
 786             __UseNinja=1
 787             ;;
 788
 789         runtests)
 790             __RunTests=1
 791             ;;
 792
 793         rebuild)
 794             __RebuildTests=1
 795             ;;
 796
 797         skipnative|-skipnative)
 798             __SkipNative=1
 799             ;;
 800
 801         skipmanaged|-skipmanaged)
 802             __SkipManaged=1
 803             __BuildTestWrappers=0
 804             ;;
 805
 806         ziptests)
 807             __ZipTests=1
 808             ;;
 809
 810         buildtestwrappersonly)
 811             __BuildTestWrappersOnly=1
 812             ;;
 813
 814         generatelayoutonly)
 815             __GenerateLayoutOnly=1
 816             ;;
 817
 818         generatetesthostonly)
 819             __GenerateTestHostOnly=1
 820             ;;
 821
 822         skiprestorepackages)
 823             __SkipRestorePackages=1
 824             ;;
 825
 826         bindir)
 827             if [ -n "$2" ]; then
 828                 __RootBinDir="$2"
 829                 if [ ! -d $__RootBinDir ]; then
 830                     mkdir $__RootBinDir
 831                 fi
 832                 __RootBinParent=$(dirname $__RootBinDir)
 833                 __RootBinName=${__RootBinDir##*/}
 834                 __RootBinDir="$(cd $__RootBinParent &>/dev/null && printf %s/%s $PWD $__RootBinName)"
 835                 shift
 836             else
 837                 echo "ERROR: 'bindir' requires a non-empty option argument"
 838                 exit 1
 839             fi
 840             ;;
 841
 842         msbuildonunsupportedplatform)
 843             __msbuildonunsupportedplatform=1
 844             ;;
 845
 846         priority1)
 847             __priority1=1
 848             __UnprocessedBuildArgs+=("-priority=1")
 849             ;;
 850
 851         *)
 852             __UnprocessedBuildArgs+=("$1")
 853             ;;
 854     esac
 855
 856     shift
 857 done
 858
 859 # Get the number of processors available to the scheduler
 860 # Other techniques such as `nproc` only get the number of
 861 # processors available to a single process.
 862 if [ `uname` = "FreeBSD" ]; then
 863   __NumProc=`sysctl hw.ncpu | awk '{ print $2+1 }'`
 864 elif [ `uname` = "NetBSD" ]; then
 865   __NumProc=$(($(getconf NPROCESSORS_ONLN)+1))
 866 elif [ `uname` = "Darwin" ]; then
 867   __NumProc=$(($(getconf _NPROCESSORS_ONLN)+1))
 868 else
 869   __NumProc=$(nproc --all)
 870 fi
 871
 872 __RunArgs=("-BuildArch=$__BuildArch" "-BuildType=$__BuildType" "-BuildOS=$__BuildOS")
 873
 874 # Configure environment if we are doing a verbose build
 875 if [ $__VerboseBuild == 1 ]; then
 876     export VERBOSE=1
 877     __RunArgs+=("-verbose")
 878 fi
 879
 880 # Set default clang version
 881 if [[ $__ClangMajorVersion == 0 && $__ClangMinorVersion == 0 ]]; then
 882     if [[ "$__BuildArch" == "arm" || "$__BuildArch" == "armel" ]]; then
 883         __ClangMajorVersion=5
 884         __ClangMinorVersion=0
 885     else
 886         __ClangMajorVersion=3
 887         __ClangMinorVersion=9
 888     fi
 889 fi
 890
 891 # Set dependent variables
 892 __LogsDir="$__RootBinDir/Logs"
 893
 894 # init the host distro name
 895 initHostDistroRid
 896
 897 # Set the remaining variables based upon the determined build configuration
 898 __BinDir="$__RootBinDir/Product/$__BuildOS.$__BuildArch.$__BuildType"
 899 __PackagesBinDir="$__BinDir/.nuget"
 900 __TestDir="$__ProjectDir/tests"
 901 __TestWorkingDir="$__RootBinDir/tests/$__BuildOS.$__BuildArch.$__BuildType"
 902 __IntermediatesDir="$__RootBinDir/obj/$__BuildOS.$__BuildArch.$__BuildType"
 903 __TestIntermediatesDir="$__RootBinDir/tests/obj/$__BuildOS.$__BuildArch.$__BuildType"
 904 __isMSBuildOnNETCoreSupported=0
 905 __CrossComponentBinDir="$__BinDir"
 906 __CrossCompIntermediatesDir="$__IntermediatesDir/crossgen"
 907
 908 __CrossArch="$__HostArch"
 909 if [[ "$__HostArch" == "x64" && "$__BuildArch" == "arm" ]]; then
 910     __CrossArch="x86"
 911 fi
 912 if [ $__CrossBuild == 1 ]; then
 913     __CrossComponentBinDir="$__CrossComponentBinDir/$__CrossArch"
 914 fi
 915 __CrossgenCoreLibLog="$__LogsDir/CrossgenCoreLib_$__BuildOS.$BuildArch.$__BuildType.log"
 916 __CrossgenExe="$__CrossComponentBinDir/crossgen"
 917
 918 isMSBuildOnNETCoreSupported
 919
 920 # CI_SPECIFIC - On CI machines, $HOME may not be set. In such a case, create a subfolder and set the variable to it.
 921 # This is needed by CLI to function.
 922 if [ -z "$HOME" ]; then
 923     if [ ! -d "$__ProjectDir/temp_home" ]; then
 924         mkdir temp_home
 925     fi
 926     export HOME=$__ProjectDir/temp_home
 927     echo "HOME not defined; setting it to $HOME"
 928 fi
 929
 930 # Configure environment if we are doing a cross compile.
 931 if [ $__CrossBuild == 1 ]; then
 932     export CROSSCOMPILE=1
 933     if ! [[ -n "$ROOTFS_DIR" ]]; then
 934         export ROOTFS_DIR="$__ProjectRoot/cross/rootfs/$__BuildArch"
 935     fi
 936 fi
 937
 938 # init the target distro name
 939 initTargetDistroRid
 940
 941 # Override tool directory
 942
 943 __CoreClrVersion=1.1.0
 944 __sharedFxDir=$__BuildToolsDir/dotnetcli/shared/Microsoft.NETCore.App/$__CoreClrVersion/
 945
 946 if [[ (-z "$__GenerateLayoutOnly") && (-z "$__GenerateTestHostOnly") && (-z "$__BuildTestWrappersOnly") ]]; then
 947     build_Tests
 948 elif [ ! -z "$__BuildTestWrappersOnly" ]; then
 949     build_test_wrappers
 950 else
 951     generate_layout
 952     if [ ! -z "$__GenerateTestHostOnly" ]; then
 953         generate_testhost
 954     fi
 955 fi
 956
 957 if [ $? -ne 0 ]; then
 958     echo "Failed to build tests"
 959     exit 1
 960 fi
 961
 962 echo "${__MsgPrefix}Test build successful."
 963 echo "${__MsgPrefix}Test binaries are available at ${__TestBinDir}"
 964
 965 __testNativeBinDir=$__IntermediatesDir/tests
 966
 967 if [ $__RunTests -ne 0 ]; then
 968
 969     echo "Run Tests..."
 970
 971     nextCommand="$__TestDir/runtest.sh --testRootDir=$__TestBinDir --coreClrBinDir=$__BinDir --coreFxBinDir=$CORE_ROOT --testNativeBinDir=$__testNativeBinDir"
 972     echo "$nextCommand"
 973     eval $nextCommand
 974
 975     echo "Tests run successful."
 976 else
 977     echo "To run all tests use 'tests/runtests.sh' where:"
 978     echo "    testRootDir      = $__TestBinDir"
 979     echo "    coreClrBinDir    = $__BinDir"
 980     echo "    coreFxBinDir     = $CORE_ROOT"
 981     echo "    testNativeBinDir = $__testNativeBinDir"
 982     echo " -------------------------------------------------- "
 983     echo " Example runtest.sh command"
 984     echo ""
 985     echo " ./tests/runtest.sh --coreOverlayDir=$CORE_ROOT --testNativeBinDir=$__testNativeBinDir --testRootDir=$__TestBinDir --copyNativeTestBin"
 986     echo " -------------------------------------------------- "
 987     echo "To run single test use the following command:"
 988     echo "    bash ${__TestBinDir}/__TEST_PATH__/__TEST_NAME__.sh -coreroot=${CORE_ROOT}"
 989 fi
 990