build-test.sh

   1 #!/usr/bin/env bash
   2
   3 initHostDistroRid()
   4 {
   5     __HostDistroRid=""
   6
   7     # Some OS groups should default to use the portable packages
   8     if [ "$__BuildOS" == "OSX" ]; then
   9         __PortableBuild=1
  10     fi
  11
  12     if [ "$__HostOS" == "Linux" ]; then
  13         if [ -e /etc/redhat-release ]; then
  14             __PortableBuild=1
  15         elif [ -e /etc/os-release ]; then
  16             source /etc/os-release
  17             if [[ $ID == "alpine" ]]; then
  18                 __HostDistroRid="linux-musl-$__HostArch"
  19             else
  20                 __PortableBuild=1
  21                 __HostDistroRid="$ID.$VERSION_ID-$__HostArch"
  22             fi
  23         fi
  24     elif [ "$__HostOS" == "FreeBSD" ]; then
  25         __freebsd_version=`sysctl -n kern.osrelease | cut -f1 -d'.'`
  26         __HostDistroRid="freebsd.$__freebsd_version-$__HostArch"
  27     fi
  28
  29     # Portable builds target the base RID
  30     if [ "$__PortableBuild" == 1 ]; then
  31         if [ "$__BuildOS" == "OSX" ]; then
  32             export __HostDistroRid="osx-$__BuildArch"
  33         elif [ "$__BuildOS" == "Linux" ]; then
  34             export __HostDistroRid="linux-$__BuildArch"
  35         fi
  36     fi
  37
  38     if [ "$__HostDistroRid" == "" ]; then
  39         echo "WARNING: Cannot determine runtime id for current distro."
  40     fi
  41
  42     echo "Setting __HostDistroRid to $__HostDistroRid"
  43 }
  44
  45 initTargetDistroRid()
  46 {
  47     if [ $__CrossBuild == 1 ]; then
  48         if [ "$__BuildOS" == "Linux" ]; then
  49             if [ ! -e $ROOTFS_DIR/etc/os-release ]; then
  50                 if [ -e $ROOTFS_DIR/android_platform ]; then
  51                     source $ROOTFS_DIR/android_platform
  52                     export __DistroRid="$RID"
  53                 else
  54                     echo "WARNING: Cannot determine runtime id for current distro."
  55                     export __DistroRid=""
  56                 fi
  57             else
  58                 source $ROOTFS_DIR/etc/os-release
  59                 export __DistroRid="$ID.$VERSION_ID-$__BuildArch"
  60             fi
  61         fi
  62     else
  63         export __DistroRid="$__HostDistroRid"
  64         export __RuntimeId="$__HostDistroRid"
  65     fi
  66
  67     if [ "$ID.$VERSION_ID" == "ubuntu.16.04" ]; then
  68      export __DistroRid="ubuntu.14.04-$__BuildArch"
  69     fi
  70
  71     # Portable builds target the base RID
  72     if [ "$__PortableBuild" == 1 ]; then
  73         if [ "$__BuildOS" == "Linux" ]; then
  74             export __DistroRid="linux-$__BuildArch"
  75             export __RuntimeId="linux-$__BuildArch"
  76         elif [ "$__BuildOS" == "OSX" ]; then
  77             export __DistroRid="osx-$__BuildArch"
  78             export __RuntimeId="osx-$__BuildArch"
  79         fi
  80     fi
  81
  82     echo "__DistroRid: " $__DistroRid
  83 }
  84
  85 isMSBuildOnNETCoreSupported()
  86 {
  87     __isMSBuildOnNETCoreSupported=$__msbuildonunsupportedplatform
  88
  89     if [ $__isMSBuildOnNETCoreSupported == 1 ]; then
  90         return
  91     fi
  92
  93     if [ "$__HostArch" == "x64" ]; then
  94         if [ "$__HostOS" == "Linux" ]; then
  95             __isMSBuildOnNETCoreSupported=1
  96             UNSUPPORTED_RIDS=("debian.9-x64" "ubuntu.17.04-x64")
  97             for UNSUPPORTED_RID in "${UNSUPPORTED_RIDS[@]}"
  98             do
  99                 if [ "$__HostDistroRid" == "$UNSUPPORTED_RID" ]; then
 100                     __isMSBuildOnNETCoreSupported=0
 101                     break
 102                 fi
 103             done
 104         elif [ "$__HostOS" == "OSX" ]; then
 105             __isMSBuildOnNETCoreSupported=1
 106         fi
 107     fi
 108 }
 109
 110 build_test_wrappers()
 111 {
 112     if [ $__BuildTestWrappers -ne -0 ]; then
 113         echo "${__MsgPrefix}Creating test wrappers..."
 114
 115         export __Exclude="${__ProjectDir}/tests/issues.targets"
 116         export __BuildLogRootName="Tests_XunitWrapper"
 117
 118         buildVerbosity="Summary"
 119
 120         if [ $__VerboseBuild == 1 ]; then
 121             buildVerbosity="Diag"
 122         fi
 123
 124         # Set up directories and file names
 125         __BuildLogRootName=$subDirectoryName
 126         __BuildLog="$__LogsDir/${__BuildLogRootName}.${__BuildOS}.${__BuildArch}.${__BuildType}.log"
 127         __BuildWrn="$__LogsDir/${__BuildLogRootName}.${__BuildOS}.${__BuildArch}.${__BuildType}.wrn"
 128         __BuildErr="$__LogsDir/${__BuildLogRootName}.${__BuildOS}.${__BuildArch}.${__BuildType}.err"
 129         __MsbuildLog="/fileloggerparameters:\"Verbosity=normal;LogFile=${__BuildLog}\""
 130         __MsbuildWrn="/fileloggerparameters1:\"WarningsOnly;LogFile=${__BuildWrn}\""
 131         __MsbuildErr="/fileloggerparameters2:\"ErrorsOnly;LogFile=${__BuildErr}\""
 132         __Logging="$__MsbuildLog $__MsbuildWrn $__MsbuildErr /consoleloggerparameters:$buildVerbosity"
 133
 134         nextCommand="\"${__DotNetCli}\" msbuild \"${__ProjectDir}/tests/runtest.proj\" /p:RestoreAdditionalProjectSources=https://dotnet.myget.org/F/dotnet-core/ /p:BuildWrappers=true /p:TargetsWindows=false $__Logging /p:__BuildOS=$__BuildOS /p:__BuildType=$__BuildType /p:__BuildArch=$__BuildArch"
 135         echo "$nextCommand"
 136         eval $nextCommand
 137
 138         if [ $? -ne 0 ]; then
 139             echo "${__MsgPrefix}Error: build failed. Refer to the build log files for details (above)"
 140             exit 1
 141         else
 142             echo "XUnit Wrappers have been built."
 143             echo { "\"build_os\"": "\"${__BuildOS}\"", "\"build_arch\"": "\"${__BuildArch}\"", "\"build_type\"": "\"${__BuildType}\"" } > "${__TestWorkingDir}/build_info.json"
 144
 145         fi
 146     fi
 147 }
 148
 149 generate_layout()
 150 {
 151     echo "${__MsgPrefix}Creating test overlay..."
 152
 153     __TestDir=$__ProjectDir/tests
 154     __ProjectFilesDir=$__TestDir
 155     __TestBinDir=$__TestWorkingDir
 156
 157     if [ $__RebuildTests -ne 0 ]; then
 158         if [ -d "${__TestBinDir}" ]; then
 159             echo "Removing tests build dir: ${__TestBinDir}"
 160             rm -rf $__TestBinDir
 161         fi
 162     fi
 163
 164     __CMakeBinDir="${__TestBinDir}"
 165
 166     if [ -z "$__TestIntermediateDir" ]; then
 167         __TestIntermediateDir="tests/obj/${__BuildOS}.${__BuildArch}.${__BuildType}"
 168     fi
 169
 170     echo "__BuildOS: ${__BuildOS}"
 171     echo "__BuildArch: ${__BuildArch}"
 172     echo "__BuildType: ${__BuildType}"
 173     echo "__TestIntermediateDir: ${__TestIntermediateDir}"
 174
 175     if [ ! -f "$__TestBinDir" ]; then
 176         echo "Creating TestBinDir: ${__TestBinDir}"
 177         mkdir -p $__TestBinDir
 178     fi
 179     if [ ! -f "$__LogsDir" ]; then
 180         echo "Creating LogsDir: ${__LogsDir}"
 181         mkdir -p $__LogsDir
 182     fi
 183
 184     __BuildProperties="-p:OSGroup=${__BuildOS} -p:BuildOS=${__BuildOS} -p:BuildArch=${__BuildArch} -p:BuildType=${__BuildType}"
 185
 186     # =========================================================================================
 187     # ===
 188     # === Restore product binaries from packages
 189     # ===
 190     # =========================================================================================
 191
 192     build_MSBuild_projects "Restore_Packages" "${__ProjectDir}/tests/build.proj" "Restore product binaries (build tests)" "/t:BatchRestorePackages"
 193
 194     if [ -n "$__UpdateInvalidPackagesArg" ]; then
 195         __up="/t:UpdateInvalidPackageVersions"
 196     fi
 197
 198     echo "${__MsgPrefix}Creating test overlay..."
 199
 200     if [ -z "$xUnitTestBinBase" ]; then
 201         xUnitTestBinBase=$__TestWorkingDir
 202     fi
 203
 204     export CORE_ROOT=$xUnitTestBinBase/Tests/Core_Root
 205
 206     if [ -d "${CORE_ROOT}" ]; then
 207         rm -rf $CORE_ROOT
 208     fi
 209
 210     mkdir -p $CORE_ROOT
 211
 212     build_MSBuild_projects "Tests_Overlay_Managed" "${__ProjectDir}/tests/runtest.proj" "Creating test overlay" "/t:CreateTestOverlay"
 213
 214     chmod +x $__BinDir/corerun
 215     chmod +x $__BinDir/crossgen
 216
 217     # Make sure to copy over the pulled down packages
 218     cp -r $__BinDir/* $CORE_ROOT/ > /dev/null
 219 }
 220
 221 generate_testhost()
 222 {
 223     echo "${__MsgPrefix}Generating test host..."
 224
 225     export TEST_HOST=$xUnitTestBinBase/testhost
 226
 227     if [ -d "${TEST_HOST}" ]; then
 228         rm -rf $TEST_HOST
 229     fi
 230
 231     mkdir -p $TEST_HOST
 232
 233     build_MSBuild_projects "Tests_Generate_TestHost" "${__ProjectDir}/tests/runtest.proj" "Creating test host" "/t:CreateTestHost"
 234 }
 235
 236
 237 build_Tests()
 238 {
 239     echo "${__MsgPrefix}Building Tests..."
 240
 241     __TestDir=$__ProjectDir/tests
 242     __ProjectFilesDir=$__TestDir
 243     __TestBinDir=$__TestWorkingDir
 244
 245     if [ -f  "${__TestWorkingDir}/build_info.json" ]; then
 246         rm  "${__TestWorkingDir}/build_info.json"
 247     fi
 248
 249     if [ $__RebuildTests -ne 0 ]; then
 250         if [ -d "${__TestBinDir}" ]; then
 251             echo "Removing tests build dir: ${__TestBinDir}"
 252             rm -rf $__TestBinDir
 253         fi
 254     fi
 255
 256     export __CMakeBinDir="${__TestBinDir}"
 257     if [ ! -d "${__TestIntermediatesDir}" ]; then
 258         mkdir -p ${__TestIntermediatesDir}
 259     fi
 260
 261     __NativeTestIntermediatesDir="${__TestIntermediatesDir}/Native"
 262     if [  ! -d "${__NativeTestIntermediatesDir}" ]; then
 263         mkdir -p ${__NativeTestIntermediatesDir}
 264     fi
 265
 266     __ManagedTestIntermediatesDir="${__TestIntermediatesDir}/Managed"
 267     if [ ! -d "${__ManagedTestIntermediatesDir}" ]; then
 268         mkdir -p ${__ManagedTestIntermediatesDir}
 269     fi
 270
 271     echo "__BuildOS: ${__BuildOS}"
 272     echo "__BuildArch: ${__BuildArch}"
 273     echo "__BuildType: ${__BuildType}"
 274     echo "__TestIntermediatesDir: ${__TestIntermediatesDir}"
 275     echo "__NativeTestIntermediatesDir: ${__NativeTestIntermediatesDir}"
 276     echo "__ManagedTestIntermediatesDir: ${__ManagedTestIntermediatesDir}"
 277
 278     if [ ! -f "$__TestBinDir" ]; then
 279         echo "Creating TestBinDir: ${__TestBinDir}"
 280         mkdir -p $__TestBinDir
 281     fi
 282     if [ ! -f "$__LogsDir" ]; then
 283         echo "Creating LogsDir: ${__LogsDir}"
 284         mkdir -p $__LogsDir
 285     fi
 286
 287     __BuildProperties="-p:OSGroup=${__BuildOS} -p:BuildOS=${__BuildOS} -p:BuildArch=${__BuildArch} -p:BuildType=${__BuildType}"
 288
 289     # =========================================================================================
 290     # ===
 291     # === Restore product binaries from packages
 292     # ===
 293     # =========================================================================================
 294
 295     if [ ${__SkipRestorePackages} != 1 ]; then
 296         build_MSBuild_projects "Restore_Product" "${__ProjectDir}/tests/build.proj" "Restore product binaries (build tests)" "/t:BatchRestorePackages"
 297     fi
 298
 299     if [ $__SkipNative != 1 ]; then
 300         build_native_projects "$__BuildArch" "${__NativeTestIntermediatesDir}"
 301
 302         if [ $? -ne 0 ]; then
 303             echo "${__MsgPrefix}Error: build failed. Refer to the build log files for details (above)"
 304             exit 1
 305         fi
 306     fi
 307
 308     if [ $__SkipManaged != 1 ]; then
 309         echo "Starting the Managed Tests Build..."
 310
 311         build_MSBuild_projects "Tests_Managed" "$__ProjectDir/tests/build.proj" "Managed tests build (build tests)" "$__up"
 312
 313         if [ $? -ne 0 ]; then
 314             echo "${__MsgPrefix}Error: build failed. Refer to the build log files for details (above)"
 315             exit 1
 316         else
 317             echo "Checking the Managed Tests Build..."
 318
 319             build_MSBuild_projects "Check_Test_Build" "${__ProjectDir}/tests/runtest.proj" "Check Test Build" "/t:CheckTestBuild"
 320
 321             if [ $? -ne 0 ]; then
 322                 echo "${__MsgPrefix}Error: Check Test Build failed."
 323                 exit 1
 324             fi
 325         fi
 326
 327         echo "Managed tests build success!"
 328     fi
 329
 330     build_test_wrappers
 331
 332     if [ -n "$__UpdateInvalidPackagesArg" ]; then
 333         __up="/t:UpdateInvalidPackageVersions"
 334     fi
 335
 336     generate_layout
 337
 338     if [ $__ZipTests -ne 0 ]; then
 339         echo "${__MsgPrefix}ZIP tests packages..."
 340         build_MSBuild_projects "Helix_Prep" "$__ProjectDir/tests/helixprep.proj" "Prep test binaries for Helix publishing" " "
 341     fi
 342 }
 343
 344 build_MSBuild_projects()
 345 {
 346     subDirectoryName=$1
 347     shift
 348     projectName=$1
 349     shift
 350     stepName="$1"
 351     shift
 352     extraBuildParameters=("$@")
 353
 354     # Set up directories and file names
 355     __BuildLogRootName=$subDirectoryName
 356     __BuildLog="$__LogsDir/${__BuildLogRootName}.${__BuildOS}.${__BuildArch}.${__BuildType}.log"
 357     __BuildWrn="$__LogsDir/${__BuildLogRootName}.${__BuildOS}.${__BuildArch}.${__BuildType}.wrn"
 358     __BuildErr="$__LogsDir/${__BuildLogRootName}.${__BuildOS}.${__BuildArch}.${__BuildType}.err"
 359
 360     # Use binclashlogger by default if no other logger is specified
 361     if [[ "${extraBuildParameters[*]}" == *"/l:"* ]]; then
 362         __msbuildEventLogging=
 363     else
 364         __msbuildEventLogging="/l:BinClashLogger,Tools/Microsoft.DotNet.Build.Tasks.dll\;LogFile=binclash.log"
 365     fi
 366
 367     if [[ "$subDirectoryName" == "Tests_Managed" ]]; then
 368         # Execute msbuild managed test build in stages - workaround for excessive data retention in MSBuild ConfigCache
 369         # See https://github.com/Microsoft/msbuild/issues/2993
 370
 371         # __SkipPackageRestore and __SkipTargetingPackBuild used  to control build by tests/src/dirs.proj
 372         export __SkipPackageRestore=false
 373         export __SkipTargetingPackBuild=false
 374         export __BuildLoopCount=2
 375         export __TestGroupToBuild=1
 376         __AppendToLog=false
 377
 378         if [ -n "$__priority1" ]; then
 379             export __BuildLoopCount=16
 380             export __TestGroupToBuild=2
 381         fi
 382
 383         for (( slice=1 ; slice <= __BuildLoopCount; slice = slice + 1 ))
 384         do
 385             __msbuildLog="\"/flp:Verbosity=normal;LogFile=${__BuildLog};Append=${__AppendToLog}\""
 386             __msbuildWrn="\"/flp1:WarningsOnly;LogFile=${__BuildWrn};Append=${__AppendToLog}\""
 387             __msbuildErr="\"/flp2:ErrorsOnly;LogFile=${__BuildErr};Append=${__AppendToLog}\""
 388
 389             export TestBuildSlice=$slice
 390
 391             # Generate build command
 392             buildArgs=("/nologo" "/verbosity:minimal" "/clp:Summary")
 393             buildArgs+=("/p:RestoreDefaultOptimizationDataPackage=false" "/p:PortableBuild=true")
 394             buildArgs+=("/p:UsePartialNGENOptimization=false" "/maxcpucount")
 395
 396             buildArgs+=("$projectName" "${__msbuildLog}" "${__msbuildWrn}" "${__msbuildErr}")
 397             buildArgs+=("$__msbuildEventLogging")
 398             buildArgs+=("${extraBuildParameters[@]}")
 399             buildArgs+=("${__CommonMSBuildArgs[@]}")
 400             buildArgs+=("${__UnprocessedBuildArgs[@]}")
 401
 402             nextCommand="\"$__ProjectRoot/dotnet.sh\" msbuild ${buildArgs[@]}"
 403             echo "Building step '$stepName' slice=$slice via $nextCommand"
 404             eval $nextCommand
 405
 406             # Make sure everything is OK
 407             if [ $? -ne 0 ]; then
 408                 echo "${__MsgPrefix}Failed to build $stepName. See the build logs:"
 409                 echo "    $__BuildLog"
 410                 echo "    $__BuildWrn"
 411                 echo "    $__BuildErr"
 412                 exit 1
 413             fi
 414             export __SkipPackageRestore=true
 415             export __SkipTargetingPackBuild=true
 416             __AppendToLog=true
 417         done
 418     else
 419         __msbuildLog="\"/flp:Verbosity=normal;LogFile=${__BuildLog}\""
 420         __msbuildWrn="\"/flp1:WarningsOnly;LogFile=${__BuildWrn}\""
 421         __msbuildErr="\"/flp2:ErrorsOnly;LogFile=${__BuildErr}\""
 422
 423         # Generate build command
 424         buildArgs=("/nologo" "/verbosity:minimal" "/clp:Summary")
 425         buildArgs+=("/p:RestoreDefaultOptimizationDataPackage=false" "/p:PortableBuild=true")
 426         buildArgs+=("/p:UsePartialNGENOptimization=false" "/maxcpucount")
 427
 428         buildArgs+=("$projectName" "${__msbuildLog}" "${__msbuildWrn}" "${__msbuildErr}")
 429         buildArgs+=("$__msbuildEventLogging")
 430         buildArgs+=("${extraBuildParameters[@]}")
 431         buildArgs+=("${__CommonMSBuildArgs[@]}")
 432         buildArgs+=("${__UnprocessedBuildArgs[@]}")
 433
 434         nextCommand="\"$__ProjectRoot/dotnet.sh\" msbuild ${buildArgs[@]}"
 435         echo "Building step '$stepName' via $nextCommand"
 436         eval $nextCommand
 437
 438         # Make sure everything is OK
 439         if [ $? -ne 0 ]; then
 440             echo "${__MsgPrefix}Failed to build $stepName. See the build logs:"
 441             echo "    $__BuildLog"
 442             echo "    $__BuildWrn"
 443             echo "    $__BuildErr"
 444             exit 1
 445         fi
 446     fi
 447 }
 448
 449 build_native_projects()
 450 {
 451     platformArch="$1"
 452     intermediatesForBuild="$2"
 453
 454     extraCmakeArguments="-DCLR_CMAKE_TARGET_OS=${__BuildOS} -DCLR_CMAKE_HOST_ARCH=${platformArch}"
 455     message="native tests assets"
 456
 457     # All set to commence the build
 458     echo "Commencing build of $message for $__BuildOS.$__BuildArch.$__BuildType in $intermediatesForBuild"
 459
 460     generator=""
 461     buildFile="Makefile"
 462     buildTool="make"
 463     if [ $__UseNinja == 1 ]; then
 464         generator="ninja"
 465         buildFile="build.ninja"
 466         if ! buildTool=$(command -v ninja || command -v ninja-build); then
 467            echo "Unable to locate ninja!" 1>&2
 468            exit 1
 469         fi
 470     fi
 471
 472     if [ $__SkipConfigure == 0 ]; then
 473         # if msbuild is not supported, then set __SkipGenerateVersion to 1
 474         if [ $__isMSBuildOnNETCoreSupported == 0 ]; then __SkipGenerateVersion=1; fi
 475         # Drop version.c file
 476         __versionSourceFile="$intermediatesForBuild/version.c"
 477         if [ $__SkipGenerateVersion == 0 ]; then
 478             pwd
 479             $__ProjectRoot/dotnet.sh msbuild /nologo /verbosity:minimal /clp:Summary \
 480                                      /p:RestoreDefaultOptimizationDataPackage=false /p:PortableBuild=true \
 481                                      /p:UsePartialNGENOptimization=false /maxcpucount \
 482                                      $__ProjectDir/build.proj /t:GenerateVersionHeader \
 483                                      /p:GenerateVersionHeader=true /p:NativeVersionSourceFile=$__versionSourceFile \
 484                                      /l:BinClashLogger,Tools/Microsoft.DotNet.Build.Tasks.dll\;LogFile=binclash.log \
 485                                      $__CommonMSBuildArgs $__UnprocessedBuildArgs
 486         else
 487             # Generate the dummy version.c, but only if it didn't exist to make sure we don't trigger unnecessary rebuild
 488             __versionSourceLine="static char sccsid[] __attribute__((used)) = \"@(#)No version information produced\";"
 489             if [ -e $__versionSourceFile ]; then
 490                 read existingVersionSourceLine < $__versionSourceFile
 491             fi
 492             if [ "$__versionSourceLine" != "$existingVersionSourceLine" ]; then
 493                 echo $__versionSourceLine > $__versionSourceFile
 494             fi
 495         fi
 496
 497         pushd "$intermediatesForBuild"
 498         # Regenerate the CMake solution
 499         # Force cross dir to point to project root cross dir, in case there is a cross build.
 500         nextCommand="CONFIG_DIR=\"$__ProjectRoot/cross\" \"$__ProjectRoot/src/pal/tools/gen-buildsys-clang.sh\" \"$__TestDir\" $__ClangMajorVersion $__ClangMinorVersion $platformArch $__BuildType $__CodeCoverage $generator $extraCmakeArguments $__cmakeargs"
 501         echo "Invoking $nextCommand"
 502         eval $nextCommand
 503         popd
 504     fi
 505
 506     if [ ! -f "$intermediatesForBuild/$buildFile" ]; then
 507         echo "Failed to generate $message build project!"
 508         exit 1
 509     fi
 510
 511     # Build
 512     if [ $__ConfigureOnly == 1 ]; then
 513         echo "Finish configuration & skipping $message build."
 514         return
 515     fi
 516
 517     pushd "$intermediatesForBuild"
 518
 519     echo "Executing $buildTool install -j $__NumProc"
 520
 521     $buildTool install -j $__NumProc
 522     if [ $? != 0 ]; then
 523         echo "Failed to build $message."
 524         exit 1
 525     fi
 526
 527     popd
 528     echo "Native tests build success!"
 529 }
 530
 531 usage()
 532 {
 533     echo "Usage: $0 [BuildArch] [BuildType] [verbose] [coverage] [cross] [clangx.y] [ninja] [runtests] [bindir]"
 534     echo "BuildArch can be: x64, x86, arm, armel, arm64"
 535     echo "BuildType can be: debug, checked, release"
 536     echo "coverage - optional argument to enable code coverage build (currently supported only for Linux and OSX)."
 537     echo "ninja - target ninja instead of GNU make"
 538     echo "clangx.y - optional argument to build using clang version x.y - supported version 3.5 - 6.0"
 539     echo "cross - optional argument to signify cross compilation,"
 540     echo "      - will use ROOTFS_DIR environment variable if set."
 541     echo "portableLinux - build for Portable Linux Distribution"
 542     echo "portablebuild - Use portable build."
 543     echo "verbose - optional argument to enable verbose build output."
 544     echo "rebuild - if tests have already been built - rebuild them"
 545     echo "skipnative: skip the native tests build"
 546     echo "skipmanaged: skip the managed section of the test build"
 547     echo "buildtestwrappersonly - only build the test wrappers"
 548     echo "generatelayoutonly - only pull down dependencies and build coreroot"
 549     echo "generatetesthostonly - only pull down dependencies and build coreroot and the CoreFX testhost"
 550     echo "skiprestorepackages - skip package restore"
 551     echo "runtests - run tests after building them"
 552     echo "ziptests - zips CoreCLR tests & Core_Root for a Helix run"
 553     echo "bindir - output directory (defaults to $__ProjectRoot/bin)"
 554     echo "msbuildonunsupportedplatform - build managed binaries even if distro is not officially supported."
 555     echo "priority1 - include priority=1 tests in the build"
 556     exit 1
 557 }
 558
 559
 560 # Obtain the location of the bash script to figure out where the root of the repo is.
 561 __ProjectRoot="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
 562
 563 # Use uname to determine what the CPU is.
 564 CPUName=$(uname -p)
 565
 566 # Some Linux platforms report unknown for platform, but the arch for machine.
 567 if [ "$CPUName" == "unknown" ]; then
 568     CPUName=$(uname -m)
 569 fi
 570
 571 case $CPUName in
 572     i686)
 573         echo "Unsupported CPU $CPUName detected, build might not succeed!"
 574         __BuildArch=x86
 575         __HostArch=x86
 576         ;;
 577
 578     x86_64)
 579         __BuildArch=x64
 580         __HostArch=x64
 581         ;;
 582
 583     armv7l)
 584         echo "Unsupported CPU $CPUName detected, build might not succeed!"
 585         __BuildArch=arm
 586         __HostArch=arm
 587         ;;
 588
 589     aarch64)
 590         __BuildArch=arm64
 591         __HostArch=arm64
 592         ;;
 593
 594     *)
 595         echo "Unknown CPU $CPUName detected, configuring as if for x64"
 596         __BuildArch=x64
 597         __HostArch=x64
 598         ;;
 599 esac
 600
 601 # Use uname to determine what the OS is.
 602 OSName=$(uname -s)
 603 case $OSName in
 604     Linux)
 605         __BuildOS=Linux
 606         __HostOS=Linux
 607         ;;
 608
 609     Darwin)
 610         __BuildOS=OSX
 611         __HostOS=OSX
 612         ;;
 613
 614     FreeBSD)
 615         __BuildOS=FreeBSD
 616         __HostOS=FreeBSD
 617         ;;
 618
 619     OpenBSD)
 620         __BuildOS=OpenBSD
 621         __HostOS=OpenBSD
 622         ;;
 623
 624     NetBSD)
 625         __BuildOS=NetBSD
 626         __HostOS=NetBSD
 627         ;;
 628
 629     SunOS)
 630         __BuildOS=SunOS
 631         __HostOS=SunOS
 632         ;;
 633
 634     *)
 635         echo "Unsupported OS $OSName detected, configuring as if for Linux"
 636         __BuildOS=Linux
 637         __HostOS=Linux
 638         ;;
 639 esac
 640
 641 __BuildType=Debug
 642 __CodeCoverage=
 643 __IncludeTests=INCLUDE_TESTS
 644
 645 # Set the various build properties here so that CMake and MSBuild can pick them up
 646 export __ProjectDir="$__ProjectRoot"
 647 __SourceDir="$__ProjectDir/src"
 648 __PackagesDir="$__ProjectDir/packages"
 649 __RootBinDir="$__ProjectDir/bin"
 650 __BuildToolsDir="$__ProjectDir/Tools"
 651 __DotNetCli="${__BuildToolsDir}/dotnetcli/dotnet"
 652 __UnprocessedBuildArgs=
 653 __CommonMSBuildArgs=
 654 __MSBCleanBuildArgs=
 655 __UseNinja=0
 656 __VerboseBuild=0
 657 __SkipRestore=""
 658 __SkipNative=0
 659 __SkipManaged=0
 660 __SkipConfigure=0
 661 __SkipGenerateVersion=0
 662 __ConfigureOnly=0
 663 __CrossBuild=0
 664 __ClangMajorVersion=0
 665 __ClangMinorVersion=0
 666 __NuGetPath="$__PackagesDir/NuGet.exe"
 667 __HostDistroRid=""
 668 __SkipRestorePackages=0
 669 __DistroRid=""
 670 __cmakeargs=""
 671 __PortableLinux=0
 672 __msbuildonunsupportedplatform=0
 673 __ZipTests=0
 674 __NativeTestIntermediatesDir=
 675 __RunTests=0
 676 __RebuildTests=0
 677 __BuildTestWrappers=1
 678 __GenerateLayoutOnly=
 679 __GenerateTestHostOnly=
 680 __priority1=
 681 __BuildTestWrappersOnly=
 682 CORE_ROOT=
 683
 684 while :; do
 685     if [ $# -le 0 ]; then
 686         break
 687     fi
 688
 689     lowerI="$(echo $1 | awk '{print tolower($0)}')"
 690     case $lowerI in
 691         -\?|-h|--help)
 692             usage
 693             exit 1
 694             ;;
 695
 696         x86)
 697             __BuildArch=x86
 698             ;;
 699
 700         x64)
 701             __BuildArch=x64
 702             ;;
 703
 704         arm)
 705             __BuildArch=arm
 706             ;;
 707
 708         armel)
 709             __BuildArch=armel
 710             ;;
 711
 712         arm64)
 713             __BuildArch=arm64
 714             ;;
 715
 716         debug)
 717             __BuildType=Debug
 718             ;;
 719
 720         checked)
 721             __BuildType=Checked
 722             ;;
 723
 724         release)
 725             __BuildType=Release
 726             ;;
 727
 728         coverage)
 729             __CodeCoverage=Coverage
 730             ;;
 731
 732         cross)
 733             __CrossBuild=1
 734             ;;
 735
 736         portableBuild)
 737             __PortableBuild=1
 738             ;;
 739
 740         portablelinux)
 741             if [ "$__BuildOS" == "Linux" ]; then
 742                 __PortableLinux=1
 743             else
 744                 echo "ERROR: portableLinux not supported for non-Linux platforms."
 745                 exit 1
 746             fi
 747             ;;
 748
 749         verbose)
 750             __VerboseBuild=1
 751             ;;
 752
 753         clang3.5|-clang3.5)
 754             __ClangMajorVersion=3
 755             __ClangMinorVersion=5
 756             ;;
 757
 758         clang3.6|-clang3.6)
 759             __ClangMajorVersion=3
 760             __ClangMinorVersion=6
 761             ;;
 762
 763         clang3.7|-clang3.7)
 764             __ClangMajorVersion=3
 765             __ClangMinorVersion=7
 766             ;;
 767
 768         clang3.8|-clang3.8)
 769             __ClangMajorVersion=3
 770             __ClangMinorVersion=8
 771             ;;
 772
 773         clang3.9|-clang3.9)
 774             __ClangMajorVersion=3
 775             __ClangMinorVersion=9
 776             ;;
 777
 778         clang4.0|-clang4.0)
 779             __ClangMajorVersion=4
 780             __ClangMinorVersion=0
 781             ;;
 782
 783         clang5.0|-clang5.0)
 784             __ClangMajorVersion=5
 785             __ClangMinorVersion=0
 786             ;;
 787
 788         clang6.0|-clang6.0)
 789             __ClangMajorVersion=6
 790             __ClangMinorVersion=0
 791             ;;
 792
 793         ninja)
 794             __UseNinja=1
 795             ;;
 796
 797         runtests)
 798             __RunTests=1
 799             ;;
 800
 801         rebuild)
 802             __RebuildTests=1
 803             ;;
 804
 805         skipnative|-skipnative)
 806             __SkipNative=1
 807             ;;
 808
 809         skipmanaged|-skipmanaged)
 810             __SkipManaged=1
 811             __BuildTestWrappers=0
 812             ;;
 813
 814         ziptests)
 815             __ZipTests=1
 816             ;;
 817
 818         buildtestwrappersonly)
 819             __BuildTestWrappersOnly=1
 820             ;;
 821
 822         generatelayoutonly)
 823             __GenerateLayoutOnly=1
 824             ;;
 825
 826         generatetesthostonly)
 827             __GenerateTestHostOnly=1
 828             ;;
 829
 830         skiprestorepackages)
 831             __SkipRestorePackages=1
 832             ;;
 833
 834         bindir)
 835             if [ -n "$2" ]; then
 836                 __RootBinDir="$2"
 837                 if [ ! -d $__RootBinDir ]; then
 838                     mkdir $__RootBinDir
 839                 fi
 840                 __RootBinParent=$(dirname $__RootBinDir)
 841                 __RootBinName=${__RootBinDir##*/}
 842                 __RootBinDir="$(cd $__RootBinParent &>/dev/null && printf %s/%s $PWD $__RootBinName)"
 843                 shift
 844             else
 845                 echo "ERROR: 'bindir' requires a non-empty option argument"
 846                 exit 1
 847             fi
 848             ;;
 849
 850         msbuildonunsupportedplatform)
 851             __msbuildonunsupportedplatform=1
 852             ;;
 853
 854         priority1)
 855             __priority1=1
 856             __UnprocessedBuildArgs+=("/p:CLRTestPriorityToBuild=1")
 857             ;;
 858
 859         *)
 860             __UnprocessedBuildArgs+=("$1")
 861             ;;
 862     esac
 863
 864     shift
 865 done
 866
 867 # Get the number of processors available to the scheduler
 868 # Other techniques such as `nproc` only get the number of
 869 # processors available to a single process.
 870 if [ `uname` = "FreeBSD" ]; then
 871   __NumProc=`sysctl hw.ncpu | awk '{ print $2+1 }'`
 872 elif [ `uname` = "NetBSD" ]; then
 873   __NumProc=$(($(getconf NPROCESSORS_ONLN)+1))
 874 elif [ `uname` = "Darwin" ]; then
 875   __NumProc=$(($(getconf _NPROCESSORS_ONLN)+1))
 876 else
 877   __NumProc=$(nproc --all)
 878 fi
 879
 880 __CommonMSBuildArgs=("/p:__BuildArch=$__BuildArch" "/p:__BuildType=$__BuildType" "/p:__BuildOS=$__BuildOS")
 881
 882 # Configure environment if we are doing a verbose build
 883 if [ $__VerboseBuild == 1 ]; then
 884     export VERBOSE=1
 885     __CommonMSBuildArgs+=("/v:detailed")
 886 fi
 887
 888 # Set default clang version
 889 if [[ $__ClangMajorVersion == 0 && $__ClangMinorVersion == 0 ]]; then
 890     if [[ "$__BuildArch" == "arm" || "$__BuildArch" == "armel" ]]; then
 891         __ClangMajorVersion=5
 892         __ClangMinorVersion=0
 893     else
 894         __ClangMajorVersion=3
 895         __ClangMinorVersion=9
 896     fi
 897 fi
 898
 899 # Set dependent variables
 900 __LogsDir="$__RootBinDir/Logs"
 901
 902 # init the host distro name
 903 initHostDistroRid
 904
 905 # Set the remaining variables based upon the determined build configuration
 906 __BinDir="$__RootBinDir/Product/$__BuildOS.$__BuildArch.$__BuildType"
 907 __PackagesBinDir="$__BinDir/.nuget"
 908 __TestDir="$__ProjectDir/tests"
 909 __TestWorkingDir="$__RootBinDir/tests/$__BuildOS.$__BuildArch.$__BuildType"
 910 __IntermediatesDir="$__RootBinDir/obj/$__BuildOS.$__BuildArch.$__BuildType"
 911 __TestIntermediatesDir="$__RootBinDir/tests/obj/$__BuildOS.$__BuildArch.$__BuildType"
 912 __isMSBuildOnNETCoreSupported=0
 913 __CrossComponentBinDir="$__BinDir"
 914 __CrossCompIntermediatesDir="$__IntermediatesDir/crossgen"
 915
 916 __CrossArch="$__HostArch"
 917 if [[ "$__HostArch" == "x64" && "$__BuildArch" == "arm" ]]; then
 918     __CrossArch="x86"
 919 fi
 920 if [ $__CrossBuild == 1 ]; then
 921     __CrossComponentBinDir="$__CrossComponentBinDir/$__CrossArch"
 922 fi
 923 __CrossgenCoreLibLog="$__LogsDir/CrossgenCoreLib_$__BuildOS.$BuildArch.$__BuildType.log"
 924 __CrossgenExe="$__CrossComponentBinDir/crossgen"
 925
 926 isMSBuildOnNETCoreSupported
 927
 928 # CI_SPECIFIC - On CI machines, $HOME may not be set. In such a case, create a subfolder and set the variable to it.
 929 # This is needed by CLI to function.
 930 if [ -z "$HOME" ]; then
 931     if [ ! -d "$__ProjectDir/temp_home" ]; then
 932         mkdir temp_home
 933     fi
 934     export HOME=$__ProjectDir/temp_home
 935     echo "HOME not defined; setting it to $HOME"
 936 fi
 937
 938 # Configure environment if we are doing a cross compile.
 939 if [ $__CrossBuild == 1 ]; then
 940     export CROSSCOMPILE=1
 941     if ! [[ -n "$ROOTFS_DIR" ]]; then
 942         export ROOTFS_DIR="$__ProjectRoot/cross/rootfs/$__BuildArch"
 943     fi
 944 fi
 945
 946 # init the target distro name
 947 initTargetDistroRid
 948
 949 # Override tool directory
 950
 951 __CoreClrVersion=1.1.0
 952 __sharedFxDir=$__BuildToolsDir/dotnetcli/shared/Microsoft.NETCore.App/$__CoreClrVersion/
 953
 954 if [[ (-z "$__GenerateLayoutOnly") && (-z "$__GenerateTestHostOnly") && (-z "$__BuildTestWrappersOnly") ]]; then
 955     build_Tests
 956 elif [ ! -z "$__BuildTestWrappersOnly" ]; then
 957     build_test_wrappers
 958 else
 959     generate_layout
 960     if [ ! -z "$__GenerateTestHostOnly" ]; then
 961         generate_testhost
 962     fi
 963 fi
 964
 965 if [ $? -ne 0 ]; then
 966     echo "Failed to build tests"
 967     exit 1
 968 fi
 969
 970 echo "${__MsgPrefix}Test build successful."
 971 echo "${__MsgPrefix}Test binaries are available at ${__TestBinDir}"
 972
 973 __testNativeBinDir=$__IntermediatesDir/tests
 974
 975 if [ $__RunTests -ne 0 ]; then
 976
 977     echo "Run Tests..."
 978
 979     nextCommand="$__TestDir/runtest.sh --testRootDir=$__TestBinDir --coreClrBinDir=$__BinDir --coreFxBinDir=$CORE_ROOT --testNativeBinDir=$__testNativeBinDir"
 980     echo "$nextCommand"
 981     eval $nextCommand
 982
 983     echo "Tests run successful."
 984 else
 985     echo "To run all tests use 'tests/runtests.sh' where:"
 986     echo "    testRootDir      = $__TestBinDir"
 987     echo "    coreClrBinDir    = $__BinDir"
 988     echo "    coreFxBinDir     = $CORE_ROOT"
 989     echo "    testNativeBinDir = $__testNativeBinDir"
 990     echo " -------------------------------------------------- "
 991     echo " Example runtest.sh command"
 992     echo ""
 993     echo " ./tests/runtest.sh --coreOverlayDir=$CORE_ROOT --testNativeBinDir=$__testNativeBinDir --testRootDir=$__TestBinDir --copyNativeTestBin"
 994     echo " -------------------------------------------------- "
 995     echo "To run single test use the following command:"
 996     echo "    bash ${__TestBinDir}/__TEST_PATH__/__TEST_NAME__.sh -coreroot=${CORE_ROOT}"
 997 fi
 998