build-test.sh

   1 #!/usr/bin/env bash
   2
   3 initHostDistroRid()
   4 {
   5     __HostDistroRid=""
   6
   7     # Some OS groups should default to use the portable packages
   8     if [ "$__BuildOS" == "OSX" ]; then
   9         __PortableBuild=1
  10     fi
  11
  12     if [ "$__HostOS" == "Linux" ]; then
  13         if [ -e /etc/redhat-release ]; then
  14             __PortableBuild=1
  15         elif [ -e /etc/os-release ]; then
  16             source /etc/os-release
  17             if [[ $ID == "alpine" ]]; then
  18                 __HostDistroRid="linux-musl-$__HostArch"
  19             else
  20                 __PortableBuild=1
  21                 __HostDistroRid="$ID.$VERSION_ID-$__HostArch"
  22             fi
  23         fi
  24     elif [ "$__HostOS" == "FreeBSD" ]; then
  25         __freebsd_version=`sysctl -n kern.osrelease | cut -f1 -d'.'`
  26         __HostDistroRid="freebsd.$__freebsd_version-$__HostArch"
  27     fi
  28
  29     # Portable builds target the base RID
  30     if [ "$__PortableBuild" == 1 ]; then
  31         if [ "$__BuildOS" == "OSX" ]; then
  32             export __HostDistroRid="osx-$__BuildArch"
  33         elif [ "$__BuildOS" == "Linux" ]; then
  34             export __HostDistroRid="linux-$__BuildArch"
  35         fi
  36     fi
  37
  38     if [ "$__HostDistroRid" == "" ]; then
  39         echo "WARNING: Cannot determine runtime id for current distro."
  40     fi
  41
  42     echo "Setting __HostDistroRid to $__HostDistroRid"
  43 }
  44
  45 initTargetDistroRid()
  46 {
  47     if [ $__CrossBuild == 1 ]; then
  48         if [ "$__BuildOS" == "Linux" ]; then
  49             if [ ! -e $ROOTFS_DIR/etc/os-release ]; then
  50                 if [ -e $ROOTFS_DIR/android_platform ]; then
  51                     source $ROOTFS_DIR/android_platform
  52                     export __DistroRid="$RID"
  53                 else
  54                     echo "WARNING: Cannot determine runtime id for current distro."
  55                     export __DistroRid=""
  56                 fi
  57             else
  58                 source $ROOTFS_DIR/etc/os-release
  59                 export __DistroRid="$ID.$VERSION_ID-$__BuildArch"
  60             fi
  61         fi
  62     else
  63         export __DistroRid="$__HostDistroRid"
  64     fi
  65
  66     if [ "$ID.$VERSION_ID" == "ubuntu.16.04" ]; then
  67      export __DistroRid="ubuntu.14.04-$__BuildArch"
  68     fi
  69
  70     # Portable builds target the base RID
  71     if [ "$__PortableBuild" == 1 ]; then
  72         if [ "$__BuildOS" == "Linux" ]; then
  73             export __DistroRid="linux-$__BuildArch"
  74             export __RuntimeId="linux-$__BuildArch"
  75         elif [ "$__BuildOS" == "OSX" ]; then
  76             export __DistroRid="osx-$__BuildArch"
  77             export __RuntimeId="osx-$__BuildArch"
  78         fi
  79     fi
  80
  81     echo "__DistroRid: " $__DistroRid
  82 }
  83
  84 isMSBuildOnNETCoreSupported()
  85 {
  86     __isMSBuildOnNETCoreSupported=$__msbuildonunsupportedplatform
  87
  88     if [ $__isMSBuildOnNETCoreSupported == 1 ]; then
  89         return
  90     fi
  91
  92     if [ "$__HostArch" == "x64" ]; then
  93         if [ "$__HostOS" == "Linux" ]; then
  94             __isMSBuildOnNETCoreSupported=1
  95             UNSUPPORTED_RIDS=("debian.9-x64" "ubuntu.17.04-x64")
  96             for UNSUPPORTED_RID in "${UNSUPPORTED_RIDS[@]}"
  97             do
  98                 if [ "$__HostDistroRid" == "$UNSUPPORTED_RID" ]; then
  99                     __isMSBuildOnNETCoreSupported=0
 100                     break
 101                 fi
 102             done
 103         elif [ "$__HostOS" == "OSX" ]; then
 104             __isMSBuildOnNETCoreSupported=1
 105         fi
 106     fi
 107 }
 108
 109 build_test_wrappers()
 110 {
 111     if [ $__BuildTestWrappers -ne -0 ]; then
 112         echo "${__MsgPrefix}Creating test wrappers..."
 113
 114         export __Exclude="${__ProjectDir}/tests/issues.targets"
 115         export __BuildLogRootName="Tests_XunitWrapper"
 116
 117         buildVerbosity="Summary"
 118
 119         if [ $__VerboseBuild == 1 ]; then
 120             buildVerbosity="Diag"
 121         fi
 122
 123         # Set up directories and file names
 124         __BuildLogRootName=$subDirectoryName
 125         __BuildLog="$__LogsDir/${__BuildLogRootName}.${__BuildOS}.${__BuildArch}.${__BuildType}.log"
 126         __BuildWrn="$__LogsDir/${__BuildLogRootName}.${__BuildOS}.${__BuildArch}.${__BuildType}.wrn"
 127         __BuildErr="$__LogsDir/${__BuildLogRootName}.${__BuildOS}.${__BuildArch}.${__BuildType}.err"
 128         __MsbuildLog="/fileloggerparameters:\"Verbosity=normal;LogFile=${__BuildLog}\""
 129         __MsbuildWrn="/fileloggerparameters1:\"WarningsOnly;LogFile=${__BuildWrn}\""
 130         __MsbuildErr="/fileloggerparameters2:\"ErrorsOnly;LogFile=${__BuildErr}\""
 131         __Logging="$__MsbuildLog $__MsbuildWrn $__MsbuildErr /consoleloggerparameters:$buildVerbosity"
 132
 133         nextCommand="\"${__DotNetCli}\" msbuild \"${__ProjectDir}/tests/runtest.proj\" /p:RestoreAdditionalProjectSources=https://dotnet.myget.org/F/dotnet-core/ /p:BuildWrappers=true /p:TargetsWindows=false $__Logging /p:__BuildOS=$__BuildOS /p:__BuildType=$__BuildType /p:__BuildArch=$__BuildArch"
 134         echo "$nextCommand"
 135         eval $nextCommand
 136
 137         if [ $? -ne 0 ]; then
 138             echo "${__MsgPrefix}Error: build failed. Refer to the build log files for details (above)"
 139             exit 1
 140         else
 141             echo "XUnit Wrappers have been built."
 142             echo { "\"build_os\"": "\"${__BuildOS}\"", "\"build_arch\"": "\"${__BuildArch}\"", "\"build_type\"": "\"${__BuildType}\"" } > "${__TestWorkingDir}/build_info.json"
 143
 144         fi
 145     fi
 146 }
 147
 148 generate_layout()
 149 {
 150     echo "${__MsgPrefix}Creating test overlay..."
 151
 152     __TestDir=$__ProjectDir/tests
 153     __ProjectFilesDir=$__TestDir
 154     __TestBinDir=$__TestWorkingDir
 155
 156     if [ $__RebuildTests -ne 0 ]; then
 157         if [ -d "${__TestBinDir}" ]; then
 158             echo "Removing tests build dir: ${__TestBinDir}"
 159             rm -rf $__TestBinDir
 160         fi
 161     fi
 162
 163     __CMakeBinDir="${__TestBinDir}"
 164
 165     if [ -z "$__TestIntermediateDir" ]; then
 166         __TestIntermediateDir="tests/obj/${__BuildOS}.${__BuildArch}.${__BuildType}"
 167     fi
 168
 169     echo "__BuildOS: ${__BuildOS}"
 170     echo "__BuildArch: ${__BuildArch}"
 171     echo "__BuildType: ${__BuildType}"
 172     echo "__TestIntermediateDir: ${__TestIntermediateDir}"
 173
 174     if [ ! -f "$__TestBinDir" ]; then
 175         echo "Creating TestBinDir: ${__TestBinDir}"
 176         mkdir -p $__TestBinDir
 177     fi
 178     if [ ! -f "$__LogsDir" ]; then
 179         echo "Creating LogsDir: ${__LogsDir}"
 180         mkdir -p $__LogsDir
 181     fi
 182
 183     __BuildProperties="-p:OSGroup=${__BuildOS} -p:BuildOS=${__BuildOS} -p:BuildArch=${__BuildArch} -p:BuildType=${__BuildType}"
 184
 185     # =========================================================================================
 186     # ===
 187     # === Restore product binaries from packages
 188     # ===
 189     # =========================================================================================
 190
 191     build_MSBuild_projects "Restore_Packages" "${__ProjectDir}/tests/build.proj" "Restore product binaries (build tests)" "-BatchRestorePackages"
 192
 193     if [ -n "$__UpdateInvalidPackagesArg" ]; then
 194         __up=-updateinvalidpackageversion
 195     fi
 196
 197     echo "${__MsgPrefix}Creating test overlay..."
 198
 199     if [ -z "$xUnitTestBinBase" ]; then
 200         xUnitTestBinBase=$__TestWorkingDir
 201     fi
 202
 203     export CORE_ROOT=$xUnitTestBinBase/Tests/Core_Root
 204
 205     if [ -d "${CORE_ROOT}" ]; then
 206         rm -rf $CORE_ROOT
 207     fi
 208
 209     mkdir -p $CORE_ROOT
 210
 211     build_MSBuild_projects "Tests_Overlay_Managed" "${__ProjectDir}/tests/runtest.proj" "Creating test overlay" "-testOverlay"
 212
 213     chmod +x $__BinDir/corerun
 214     chmod +x $__BinDir/crossgen
 215
 216     # Make sure to copy over the pulled down packages
 217     cp -r $__BinDir/* $CORE_ROOT/ > /dev/null
 218 }
 219
 220 generate_testhost()
 221 {
 222     echo "${__MsgPrefix}Generating test host..."
 223
 224     export TEST_HOST=$xUnitTestBinBase/testhost
 225
 226     if [ -d "${TEST_HOST}" ]; then
 227         rm -rf $TEST_HOST
 228     fi
 229
 230     mkdir -p $TEST_HOST
 231
 232     build_MSBuild_projects "Tests_Generate_TestHost" "${__ProjectDir}/tests/runtest.proj" "Creating test host" "-testHost"
 233 }
 234
 235
 236 build_Tests()
 237 {
 238     echo "${__MsgPrefix}Building Tests..."
 239
 240     __TestDir=$__ProjectDir/tests
 241     __ProjectFilesDir=$__TestDir
 242     __TestBinDir=$__TestWorkingDir
 243
 244     if [ -f  "${__TestWorkingDir}/build_info.json" ]; then
 245         rm  "${__TestWorkingDir}/build_info.json"
 246     fi
 247
 248     if [ $__RebuildTests -ne 0 ]; then
 249         if [ -d "${__TestBinDir}" ]; then
 250             echo "Removing tests build dir: ${__TestBinDir}"
 251             rm -rf $__TestBinDir
 252         fi
 253     fi
 254
 255     export __CMakeBinDir="${__TestBinDir}"
 256     if [ ! -d "${__TestIntermediatesDir}" ]; then
 257         mkdir -p ${__TestIntermediatesDir}
 258     fi
 259
 260     __NativeTestIntermediatesDir="${__TestIntermediatesDir}/Native"
 261     if [  ! -d "${__NativeTestIntermediatesDir}" ]; then
 262         mkdir -p ${__NativeTestIntermediatesDir}
 263     fi
 264
 265     __ManagedTestIntermediatesDir="${__TestIntermediatesDir}/Managed"
 266     if [ ! -d "${__ManagedTestIntermediatesDir}" ]; then
 267         mkdir -p ${__ManagedTestIntermediatesDir}
 268     fi
 269
 270     echo "__BuildOS: ${__BuildOS}"
 271     echo "__BuildArch: ${__BuildArch}"
 272     echo "__BuildType: ${__BuildType}"
 273     echo "__TestIntermediatesDir: ${__TestIntermediatesDir}"
 274     echo "__NativeTestIntermediatesDir: ${__NativeTestIntermediatesDir}"
 275     echo "__ManagedTestIntermediatesDir: ${__ManagedTestIntermediatesDir}"
 276
 277     if [ ! -f "$__TestBinDir" ]; then
 278         echo "Creating TestBinDir: ${__TestBinDir}"
 279         mkdir -p $__TestBinDir
 280     fi
 281     if [ ! -f "$__LogsDir" ]; then
 282         echo "Creating LogsDir: ${__LogsDir}"
 283         mkdir -p $__LogsDir
 284     fi
 285
 286     __BuildProperties="-p:OSGroup=${__BuildOS} -p:BuildOS=${__BuildOS} -p:BuildArch=${__BuildArch} -p:BuildType=${__BuildType}"
 287
 288     # =========================================================================================
 289     # ===
 290     # === Restore product binaries from packages
 291     # ===
 292     # =========================================================================================
 293
 294     if [ ${__SkipRestorePackages} != 1 ]; then
 295         build_MSBuild_projects "Restore_Product" "${__ProjectDir}/tests/build.proj" "Restore product binaries (build tests)" "-BatchRestorePackages"
 296     fi
 297
 298     if [ $__SkipNative != 1 ]; then
 299         build_native_projects "$__BuildArch" "${__NativeTestIntermediatesDir}"
 300
 301         if [ $? -ne 0 ]; then
 302             echo "${__MsgPrefix}Error: build failed. Refer to the build log files for details (above)"
 303             exit 1
 304         fi
 305     fi
 306
 307     if [ $__SkipManaged != 1 ]; then
 308         echo "Starting the Managed Tests Build..."
 309
 310         build_MSBuild_projects "Tests_Managed" "$__ProjectDir/tests/build.proj" "Managed tests build (build tests)" "$__up"
 311
 312         if [ $? -ne 0 ]; then
 313             echo "${__MsgPrefix}Error: build failed. Refer to the build log files for details (above)"
 314             exit 1
 315         else
 316             echo "Checking the Managed Tests Build..."
 317
 318             build_MSBuild_projects "Check_Test_Build" "${__ProjectDir}/tests/runtest.proj" "Check Test Build" "-ExtraParameters:/t:CheckTestBuild"
 319
 320             if [ $? -ne 0 ]; then
 321                 echo "${__MsgPrefix}Error: Check Test Build failed."
 322                 exit 1
 323             fi
 324         fi
 325
 326         echo "Managed tests build success!"
 327     fi
 328
 329     build_test_wrappers
 330
 331     if [ -n "$__UpdateInvalidPackagesArg" ]; then
 332         __up=-updateinvalidpackageversion
 333     fi
 334
 335     generate_layout
 336
 337     if [ $__ZipTests -ne 0 ]; then
 338         echo "${__MsgPrefix}ZIP tests packages..."
 339         build_MSBuild_projects "Helix_Prep" "$__ProjectDir/tests/helixprep.proj" "Prep test binaries for Helix publishing" " "
 340     fi
 341 }
 342
 343 build_MSBuild_projects()
 344 {
 345     subDirectoryName=$1
 346     shift
 347     projectName=$1
 348     shift
 349     stepName="$1"
 350     shift
 351     extraBuildParameters=("$@")
 352
 353     # Set up directories and file names
 354     __BuildLogRootName=$subDirectoryName
 355     __BuildLog="$__LogsDir/${__BuildLogRootName}.${__BuildOS}.${__BuildArch}.${__BuildType}.log"
 356     __BuildWrn="$__LogsDir/${__BuildLogRootName}.${__BuildOS}.${__BuildArch}.${__BuildType}.wrn"
 357     __BuildErr="$__LogsDir/${__BuildLogRootName}.${__BuildOS}.${__BuildArch}.${__BuildType}.err"
 358
 359     # Use binclashlogger by default if no other logger is specified
 360     if [[ "${extraBuildParameters[*]}" == *"-MsBuildEventLogging"* ]]; then
 361         msbuildEventLogging=""
 362     else
 363         msbuildEventLogging="-MsBuildEventLogging=\"/l:BinClashLogger,Tools/Microsoft.DotNet.Build.Tasks.dll;LogFile=binclash.log\""
 364     fi
 365
 366     if [[ "$subDirectoryName" == "Tests_Managed" ]]; then
 367         # Execute msbuild managed test build in stages - workaround for excessive data retention in MSBuild ConfigCache
 368         # See https://github.com/Microsoft/msbuild/issues/2993
 369
 370         # __SkipPackageRestore and __SkipTargetingPackBuild used  to control build by tests/src/dirs.proj
 371         export __SkipPackageRestore=false
 372         export __SkipTargetingPackBuild=false
 373         export __BuildLoopCount=2
 374         export __TestGroupToBuild=1
 375         __AppendToLog=false
 376
 377         if [ -n "$__priority1" ]; then
 378             export __BuildLoopCount=16
 379             export __TestGroupToBuild=2
 380         fi
 381
 382         for (( slice=1 ; slice <= __BuildLoopCount; slice = slice + 1 ))
 383         do
 384             __msbuildLog="\"/flp:Verbosity=normal;LogFile=${__BuildLog};Append=${__AppendToLog}\""
 385             __msbuildWrn="\"/flp1:WarningsOnly;LogFile=${__BuildWrn};Append=${__AppendToLog}\""
 386             __msbuildErr="\"/flp2:ErrorsOnly;LogFile=${__BuildErr};Append=${__AppendToLog}\""
 387
 388             export TestBuildSlice=$slice
 389
 390             # Generate build command
 391             buildArgs=("-Project=$projectName" "-MsBuildLog=${__msbuildLog}" "-MsBuildWrn=${__msbuildWrn}" "-MsBuildErr=${__msbuildErr}")
 392             buildArgs+=("$msbuildEventLogging")
 393             buildArgs+=("${extraBuildParameters[@]}")
 394             buildArgs+=("${__RunArgs[@]}")
 395             buildArgs+=("${__UnprocessedBuildArgs[@]}")
 396
 397             nextCommand="\"$__ProjectRoot/run.sh\" build ${buildArgs[@]}"
 398             echo "Building step '$stepName' slice=$slice via $nextCommand"
 399             eval $nextCommand
 400
 401             # Make sure everything is OK
 402             if [ $? -ne 0 ]; then
 403                 echo "${__MsgPrefix}Failed to build $stepName. See the build logs:"
 404                 echo "    $__BuildLog"
 405                 echo "    $__BuildWrn"
 406                 echo "    $__BuildErr"
 407                 exit 1
 408             fi
 409             export __SkipPackageRestore=true
 410             export __SkipTargetingPackBuild=true
 411             __AppendToLog=true
 412         done
 413     else
 414         __msbuildLog="\"/flp:Verbosity=normal;LogFile=${__BuildLog}\""
 415         __msbuildWrn="\"/flp1:WarningsOnly;LogFile=${__BuildWrn}\""
 416         __msbuildErr="\"/flp2:ErrorsOnly;LogFile=${__BuildErr}\""
 417
 418         # Generate build command
 419         buildArgs=("-Project=$projectName" "-MsBuildLog=${__msbuildLog}" "-MsBuildWrn=${__msbuildWrn}" "-MsBuildErr=${__msbuildErr}")
 420         buildArgs+=("$msbuildEventLogging")
 421         buildArgs+=("${extraBuildParameters[@]}")
 422         buildArgs+=("${__RunArgs[@]}")
 423         buildArgs+=("${__UnprocessedBuildArgs[@]}")
 424
 425         nextCommand="\"$__ProjectRoot/run.sh\" build ${buildArgs[@]}"
 426         echo "Building step '$stepName' via $nextCommand"
 427         eval $nextCommand
 428
 429         # Make sure everything is OK
 430         if [ $? -ne 0 ]; then
 431             echo "${__MsgPrefix}Failed to build $stepName. See the build logs:"
 432             echo "    $__BuildLog"
 433             echo "    $__BuildWrn"
 434             echo "    $__BuildErr"
 435             exit 1
 436         fi
 437     fi
 438 }
 439
 440 build_native_projects()
 441 {
 442     platformArch="$1"
 443     intermediatesForBuild="$2"
 444
 445     extraCmakeArguments="-DCLR_CMAKE_TARGET_OS=${__BuildOS} -DCLR_CMAKE_HOST_ARCH=${platformArch}"
 446     message="native tests assets"
 447
 448     # All set to commence the build
 449     echo "Commencing build of $message for $__BuildOS.$__BuildArch.$__BuildType in $intermediatesForBuild"
 450
 451     generator=""
 452     buildFile="Makefile"
 453     buildTool="make"
 454     if [ $__UseNinja == 1 ]; then
 455         generator="ninja"
 456         buildFile="build.ninja"
 457         if ! buildTool=$(command -v ninja || command -v ninja-build); then
 458            echo "Unable to locate ninja!" 1>&2
 459            exit 1
 460         fi
 461     fi
 462
 463     if [ $__SkipConfigure == 0 ]; then
 464         # if msbuild is not supported, then set __SkipGenerateVersion to 1
 465         if [ $__isMSBuildOnNETCoreSupported == 0 ]; then __SkipGenerateVersion=1; fi
 466         # Drop version.cpp file
 467         __versionSourceFile="$intermediatesForBuild/version.cpp"
 468         if [ $__SkipGenerateVersion == 0 ]; then
 469             pwd
 470             "$__ProjectRoot/run.sh" build -Project=$__ProjectDir/build.proj -generateHeaderUnix -NativeVersionSourceFile=$__versionSourceFile -MsBuildEventLogging="/l:BinClashLogger,Tools/Microsoft.DotNet.Build.Tasks.dll;LogFile=binclash.log"  $__RunArgs $__UnprocessedBuildArgs
 471         else
 472             # Generate the dummy version.cpp, but only if it didn't exist to make sure we don't trigger unnecessary rebuild
 473             __versionSourceLine="static char sccsid[] __attribute__((used)) = \"@(#)No version information produced\";"
 474             if [ -e $__versionSourceFile ]; then
 475                 read existingVersionSourceLine < $__versionSourceFile
 476             fi
 477             if [ "$__versionSourceLine" != "$existingVersionSourceLine" ]; then
 478                 echo $__versionSourceLine > $__versionSourceFile
 479             fi
 480         fi
 481
 482         pushd "$intermediatesForBuild"
 483         # Regenerate the CMake solution
 484         # Force cross dir to point to project root cross dir, in case there is a cross build.
 485         nextCommand="CONFIG_DIR=\"$__ProjectRoot/cross\" \"$__ProjectRoot/src/pal/tools/gen-buildsys-clang.sh\" \"$__TestDir\" $__ClangMajorVersion $__ClangMinorVersion $platformArch $__BuildType $__CodeCoverage $generator $extraCmakeArguments $__cmakeargs"
 486         echo "Invoking $nextCommand"
 487         eval $nextCommand
 488         popd
 489     fi
 490
 491     if [ ! -f "$intermediatesForBuild/$buildFile" ]; then
 492         echo "Failed to generate $message build project!"
 493         exit 1
 494     fi
 495
 496     # Build
 497     if [ $__ConfigureOnly == 1 ]; then
 498         echo "Finish configuration & skipping $message build."
 499         return
 500     fi
 501
 502     pushd "$intermediatesForBuild"
 503
 504     echo "Executing $buildTool install -j $__NumProc"
 505
 506     $buildTool install -j $__NumProc
 507     if [ $? != 0 ]; then
 508         echo "Failed to build $message."
 509         exit 1
 510     fi
 511
 512     popd
 513     echo "Native tests build success!"
 514 }
 515
 516 usage()
 517 {
 518     echo "Usage: $0 [BuildArch] [BuildType] [verbose] [coverage] [cross] [clangx.y] [ninja] [runtests] [bindir]"
 519     echo "BuildArch can be: x64, x86, arm, armel, arm64"
 520     echo "BuildType can be: debug, checked, release"
 521     echo "coverage - optional argument to enable code coverage build (currently supported only for Linux and OSX)."
 522     echo "ninja - target ninja instead of GNU make"
 523     echo "clangx.y - optional argument to build using clang version x.y - supported version 3.5 - 6.0"
 524     echo "cross - optional argument to signify cross compilation,"
 525     echo "      - will use ROOTFS_DIR environment variable if set."
 526     echo "portableLinux - build for Portable Linux Distribution"
 527     echo "portablebuild - Use portable build."
 528     echo "verbose - optional argument to enable verbose build output."
 529     echo "rebuild - if tests have already been built - rebuild them"
 530     echo "skipnative: skip the native tests build"
 531     echo "skipmanaged: skip the managed section of the test build"
 532     echo "buildtestwrappersonly - only build the test wrappers"
 533     echo "generatelayoutonly - only pull down dependencies and build coreroot"
 534     echo "generatetesthostonly - only pull down dependencies and build coreroot and the CoreFX testhost"
 535     echo "skiprestorepackages - skip package restore"
 536     echo "runtests - run tests after building them"
 537     echo "ziptests - zips CoreCLR tests & Core_Root for a Helix run"
 538     echo "bindir - output directory (defaults to $__ProjectRoot/bin)"
 539     echo "msbuildonunsupportedplatform - build managed binaries even if distro is not officially supported."
 540     echo "priority1 - include priority=1 tests in the build"
 541     exit 1
 542 }
 543
 544
 545 # Obtain the location of the bash script to figure out where the root of the repo is.
 546 __ProjectRoot="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
 547
 548 # Use uname to determine what the CPU is.
 549 CPUName=$(uname -p)
 550
 551 # Some Linux platforms report unknown for platform, but the arch for machine.
 552 if [ "$CPUName" == "unknown" ]; then
 553     CPUName=$(uname -m)
 554 fi
 555
 556 case $CPUName in
 557     i686)
 558         echo "Unsupported CPU $CPUName detected, build might not succeed!"
 559         __BuildArch=x86
 560         __HostArch=x86
 561         ;;
 562
 563     x86_64)
 564         __BuildArch=x64
 565         __HostArch=x64
 566         ;;
 567
 568     armv7l)
 569         echo "Unsupported CPU $CPUName detected, build might not succeed!"
 570         __BuildArch=arm
 571         __HostArch=arm
 572         ;;
 573
 574     aarch64)
 575         __BuildArch=arm64
 576         __HostArch=arm64
 577         ;;
 578
 579     *)
 580         echo "Unknown CPU $CPUName detected, configuring as if for x64"
 581         __BuildArch=x64
 582         __HostArch=x64
 583         ;;
 584 esac
 585
 586 # Use uname to determine what the OS is.
 587 OSName=$(uname -s)
 588 case $OSName in
 589     Linux)
 590         __BuildOS=Linux
 591         __HostOS=Linux
 592         ;;
 593
 594     Darwin)
 595         __BuildOS=OSX
 596         __HostOS=OSX
 597         ;;
 598
 599     FreeBSD)
 600         __BuildOS=FreeBSD
 601         __HostOS=FreeBSD
 602         ;;
 603
 604     OpenBSD)
 605         __BuildOS=OpenBSD
 606         __HostOS=OpenBSD
 607         ;;
 608
 609     NetBSD)
 610         __BuildOS=NetBSD
 611         __HostOS=NetBSD
 612         ;;
 613
 614     SunOS)
 615         __BuildOS=SunOS
 616         __HostOS=SunOS
 617         ;;
 618
 619     *)
 620         echo "Unsupported OS $OSName detected, configuring as if for Linux"
 621         __BuildOS=Linux
 622         __HostOS=Linux
 623         ;;
 624 esac
 625
 626 __BuildType=Debug
 627 __CodeCoverage=
 628 __IncludeTests=INCLUDE_TESTS
 629
 630 # Set the various build properties here so that CMake and MSBuild can pick them up
 631 export __ProjectDir="$__ProjectRoot"
 632 __SourceDir="$__ProjectDir/src"
 633 __PackagesDir="$__ProjectDir/packages"
 634 __RootBinDir="$__ProjectDir/bin"
 635 __BuildToolsDir="$__ProjectDir/Tools"
 636 __DotNetCli="${__BuildToolsDir}/dotnetcli/dotnet"
 637 __UnprocessedBuildArgs=
 638 __RunArgs=
 639 __MSBCleanBuildArgs=
 640 __UseNinja=0
 641 __VerboseBuild=0
 642 __SkipRestore=""
 643 __SkipNative=0
 644 __SkipManaged=0
 645 __SkipConfigure=0
 646 __SkipGenerateVersion=0
 647 __ConfigureOnly=0
 648 __CrossBuild=0
 649 __ClangMajorVersion=0
 650 __ClangMinorVersion=0
 651 __NuGetPath="$__PackagesDir/NuGet.exe"
 652 __HostDistroRid=""
 653 __SkipRestorePackages=0
 654 __DistroRid=""
 655 __cmakeargs=""
 656 __PortableLinux=0
 657 __msbuildonunsupportedplatform=0
 658 __ZipTests=0
 659 __NativeTestIntermediatesDir=
 660 __RunTests=0
 661 __RebuildTests=0
 662 __BuildTestWrappers=1
 663 __GenerateLayoutOnly=
 664 __GenerateTestHostOnly=
 665 __priority1=
 666 __BuildTestWrappersOnly=
 667 CORE_ROOT=
 668
 669 while :; do
 670     if [ $# -le 0 ]; then
 671         break
 672     fi
 673
 674     lowerI="$(echo $1 | awk '{print tolower($0)}')"
 675     case $lowerI in
 676         -\?|-h|--help)
 677             usage
 678             exit 1
 679             ;;
 680
 681         x86)
 682             __BuildArch=x86
 683             ;;
 684
 685         x64)
 686             __BuildArch=x64
 687             ;;
 688
 689         arm)
 690             __BuildArch=arm
 691             ;;
 692
 693         armel)
 694             __BuildArch=armel
 695             ;;
 696
 697         arm64)
 698             __BuildArch=arm64
 699             ;;
 700
 701         debug)
 702             __BuildType=Debug
 703             ;;
 704
 705         checked)
 706             __BuildType=Checked
 707             ;;
 708
 709         release)
 710             __BuildType=Release
 711             ;;
 712
 713         coverage)
 714             __CodeCoverage=Coverage
 715             ;;
 716
 717         cross)
 718             __CrossBuild=1
 719             ;;
 720
 721         portableBuild)
 722             __PortableBuild=1
 723             ;;
 724
 725         portablelinux)
 726             if [ "$__BuildOS" == "Linux" ]; then
 727                 __PortableLinux=1
 728             else
 729                 echo "ERROR: portableLinux not supported for non-Linux platforms."
 730                 exit 1
 731             fi
 732             ;;
 733
 734         verbose)
 735             __VerboseBuild=1
 736             ;;
 737
 738         clang3.5|-clang3.5)
 739             __ClangMajorVersion=3
 740             __ClangMinorVersion=5
 741             ;;
 742
 743         clang3.6|-clang3.6)
 744             __ClangMajorVersion=3
 745             __ClangMinorVersion=6
 746             ;;
 747
 748         clang3.7|-clang3.7)
 749             __ClangMajorVersion=3
 750             __ClangMinorVersion=7
 751             ;;
 752
 753         clang3.8|-clang3.8)
 754             __ClangMajorVersion=3
 755             __ClangMinorVersion=8
 756             ;;
 757
 758         clang3.9|-clang3.9)
 759             __ClangMajorVersion=3
 760             __ClangMinorVersion=9
 761             ;;
 762
 763         clang4.0|-clang4.0)
 764             __ClangMajorVersion=4
 765             __ClangMinorVersion=0
 766             ;;
 767
 768         clang5.0|-clang5.0)
 769             __ClangMajorVersion=5
 770             __ClangMinorVersion=0
 771             ;;
 772
 773         clang6.0|-clang6.0)
 774             __ClangMajorVersion=6
 775             __ClangMinorVersion=0
 776             ;;
 777
 778         ninja)
 779             __UseNinja=1
 780             ;;
 781
 782         runtests)
 783             __RunTests=1
 784             ;;
 785
 786         rebuild)
 787             __RebuildTests=1
 788             ;;
 789
 790         skipnative|-skipnative)
 791             __SkipNative=1
 792             ;;
 793
 794         skipmanaged|-skipmanaged)
 795             __SkipManaged=1
 796             __BuildTestWrappers=0
 797             ;;
 798
 799         ziptests)
 800             __ZipTests=1
 801             ;;
 802
 803         buildtestwrappersonly)
 804             __BuildTestWrappersOnly=1
 805             ;;
 806
 807         generatelayoutonly)
 808             __GenerateLayoutOnly=1
 809             ;;
 810
 811         generatetesthostonly)
 812             __GenerateTestHostOnly=1
 813             ;;
 814
 815         skiprestorepackages)
 816             __SkipRestorePackages=1
 817             ;;
 818
 819         bindir)
 820             if [ -n "$2" ]; then
 821                 __RootBinDir="$2"
 822                 if [ ! -d $__RootBinDir ]; then
 823                     mkdir $__RootBinDir
 824                 fi
 825                 __RootBinParent=$(dirname $__RootBinDir)
 826                 __RootBinName=${__RootBinDir##*/}
 827                 __RootBinDir="$(cd $__RootBinParent &>/dev/null && printf %s/%s $PWD $__RootBinName)"
 828                 shift
 829             else
 830                 echo "ERROR: 'bindir' requires a non-empty option argument"
 831                 exit 1
 832             fi
 833             ;;
 834
 835         msbuildonunsupportedplatform)
 836             __msbuildonunsupportedplatform=1
 837             ;;
 838
 839         priority1)
 840             __priority1=1
 841             __UnprocessedBuildArgs+=("-priority=1")
 842             ;;
 843
 844         *)
 845             __UnprocessedBuildArgs+=("$1")
 846             ;;
 847     esac
 848
 849     shift
 850 done
 851
 852 # Get the number of processors available to the scheduler
 853 # Other techniques such as `nproc` only get the number of
 854 # processors available to a single process.
 855 if [ `uname` = "FreeBSD" ]; then
 856   __NumProc=`sysctl hw.ncpu | awk '{ print $2+1 }'`
 857 elif [ `uname` = "NetBSD" ]; then
 858   __NumProc=$(($(getconf NPROCESSORS_ONLN)+1))
 859 elif [ `uname` = "Darwin" ]; then
 860   __NumProc=$(($(getconf _NPROCESSORS_ONLN)+1))
 861 else
 862   __NumProc=$(nproc --all)
 863 fi
 864
 865 __RunArgs=("-BuildArch=$__BuildArch" "-BuildType=$__BuildType" "-BuildOS=$__BuildOS")
 866
 867 # Configure environment if we are doing a verbose build
 868 if [ $__VerboseBuild == 1 ]; then
 869     export VERBOSE=1
 870     __RunArgs+=("-verbose")
 871 fi
 872
 873 # Set default clang version
 874 if [[ $__ClangMajorVersion == 0 && $__ClangMinorVersion == 0 ]]; then
 875     if [[ "$__BuildArch" == "arm" || "$__BuildArch" == "armel" ]]; then
 876         __ClangMajorVersion=5
 877         __ClangMinorVersion=0
 878     else
 879         __ClangMajorVersion=3
 880         __ClangMinorVersion=9
 881     fi
 882 fi
 883
 884 # Set dependent variables
 885 __LogsDir="$__RootBinDir/Logs"
 886
 887 # init the host distro name
 888 initHostDistroRid
 889
 890 # Set the remaining variables based upon the determined build configuration
 891 __BinDir="$__RootBinDir/Product/$__BuildOS.$__BuildArch.$__BuildType"
 892 __PackagesBinDir="$__BinDir/.nuget"
 893 __TestDir="$__ProjectDir/tests"
 894 __TestWorkingDir="$__RootBinDir/tests/$__BuildOS.$__BuildArch.$__BuildType"
 895 __IntermediatesDir="$__RootBinDir/obj/$__BuildOS.$__BuildArch.$__BuildType"
 896 __TestIntermediatesDir="$__RootBinDir/tests/obj/$__BuildOS.$__BuildArch.$__BuildType"
 897 __isMSBuildOnNETCoreSupported=0
 898 __CrossComponentBinDir="$__BinDir"
 899 __CrossCompIntermediatesDir="$__IntermediatesDir/crossgen"
 900
 901 __CrossArch="$__HostArch"
 902 if [[ "$__HostArch" == "x64" && "$__BuildArch" == "arm" ]]; then
 903     __CrossArch="x86"
 904 fi
 905 if [ $__CrossBuild == 1 ]; then
 906     __CrossComponentBinDir="$__CrossComponentBinDir/$__CrossArch"
 907 fi
 908 __CrossgenCoreLibLog="$__LogsDir/CrossgenCoreLib_$__BuildOS.$BuildArch.$__BuildType.log"
 909 __CrossgenExe="$__CrossComponentBinDir/crossgen"
 910
 911 isMSBuildOnNETCoreSupported
 912
 913 # CI_SPECIFIC - On CI machines, $HOME may not be set. In such a case, create a subfolder and set the variable to it.
 914 # This is needed by CLI to function.
 915 if [ -z "$HOME" ]; then
 916     if [ ! -d "$__ProjectDir/temp_home" ]; then
 917         mkdir temp_home
 918     fi
 919     export HOME=$__ProjectDir/temp_home
 920     echo "HOME not defined; setting it to $HOME"
 921 fi
 922
 923 # Configure environment if we are doing a cross compile.
 924 if [ $__CrossBuild == 1 ]; then
 925     export CROSSCOMPILE=1
 926     if ! [[ -n "$ROOTFS_DIR" ]]; then
 927         export ROOTFS_DIR="$__ProjectRoot/cross/rootfs/$__BuildArch"
 928     fi
 929 fi
 930
 931 # init the target distro name
 932 initTargetDistroRid
 933
 934 # Override tool directory
 935
 936 __CoreClrVersion=1.1.0
 937 __sharedFxDir=$__BuildToolsDir/dotnetcli/shared/Microsoft.NETCore.App/$__CoreClrVersion/
 938
 939 if [[ (-z "$__GenerateLayoutOnly") && (-z "$__GenerateTestHostOnly") && (-z "$__BuildTestWrappersOnly") ]]; then
 940     build_Tests
 941 elif [ ! -z "$__BuildTestWrappersOnly" ]; then
 942     build_test_wrappers
 943 else
 944     generate_layout
 945     if [ ! -z "$__GenerateTestHostOnly" ]; then
 946         generate_testhost
 947     fi
 948 fi
 949
 950 if [ $? -ne 0 ]; then
 951     echo "Failed to build tests"
 952     exit 1
 953 fi
 954
 955 echo "${__MsgPrefix}Test build successful."
 956 echo "${__MsgPrefix}Test binaries are available at ${__TestBinDir}"
 957
 958 __testNativeBinDir=$__IntermediatesDir/tests
 959
 960 if [ $__RunTests -ne 0 ]; then
 961
 962     echo "Run Tests..."
 963
 964     nextCommand="$__TestDir/runtest.sh --testRootDir=$__TestBinDir --coreClrBinDir=$__BinDir --coreFxBinDir=$CORE_ROOT --testNativeBinDir=$__testNativeBinDir"
 965     echo "$nextCommand"
 966     eval $nextCommand
 967
 968     echo "Tests run successful."
 969 else
 970     echo "To run all tests use 'tests/runtests.sh' where:"
 971     echo "    testRootDir      = $__TestBinDir"
 972     echo "    coreClrBinDir    = $__BinDir"
 973     echo "    coreFxBinDir     = $CORE_ROOT"
 974     echo "    testNativeBinDir = $__testNativeBinDir"
 975     echo " -------------------------------------------------- "
 976     echo " Example runtest.sh command"
 977     echo ""
 978     echo " ./tests/runtest.sh --coreOverlayDir=$CORE_ROOT --testNativeBinDir=$__testNativeBinDir --testRootDir=$__TestBinDir --copyNativeTestBin"
 979     echo " -------------------------------------------------- "
 980     echo "To run single test use the following command:"
 981     echo "    bash ${__TestBinDir}/__TEST_PATH__/__TEST_NAME__.sh -coreroot=${CORE_ROOT}"
 982 fi
 983