build-test.sh

   1 #!/usr/bin/env bash
   2
   3 __PortableBuild=1
   4
   5 initTargetDistroRid()
   6 {
   7     source init-distro-rid.sh
   8
   9     # Only pass ROOTFS_DIR if cross is specified.
  10     if (( ${__CrossBuild} == 1 )); then
  11         passedRootfsDir=${ROOTFS_DIR}
  12     fi
  13
  14     initDistroRidGlobal ${__BuildOS} ${__BuildArch} ${__PortableBuild} ${passedRootfsDir}
  15 }
  16
  17 isMSBuildOnNETCoreSupported()
  18 {
  19     __isMSBuildOnNETCoreSupported=$__msbuildonunsupportedplatform
  20
  21     if [ $__isMSBuildOnNETCoreSupported == 1 ]; then
  22         return
  23     fi
  24
  25     if [ "$__HostArch" == "x64" ]; then
  26         if [ "$__HostOS" == "Linux" ]; then
  27             __isMSBuildOnNETCoreSupported=1
  28             UNSUPPORTED_RIDS=("debian.9-x64" "ubuntu.17.04-x64")
  29             for UNSUPPORTED_RID in "${UNSUPPORTED_RIDS[@]}"
  30             do
  31                 if [ "${__DistroRid}" == "$UNSUPPORTED_RID" ]; then
  32                     __isMSBuildOnNETCoreSupported=0
  33                     break
  34                 fi
  35             done
  36         elif [ "$__HostOS" == "OSX" ]; then
  37             __isMSBuildOnNETCoreSupported=1
  38         fi
  39     fi
  40 }
  41
  42 build_test_wrappers()
  43 {
  44     if [ $__BuildTestWrappers -ne -0 ]; then
  45         echo "${__MsgPrefix}Creating test wrappers..."
  46
  47         export __Exclude="${__ProjectDir}/tests/issues.targets"
  48         export __BuildLogRootName="Tests_XunitWrapper"
  49
  50         buildVerbosity="Summary"
  51
  52         if [ $__VerboseBuild == 1 ]; then
  53             buildVerbosity="Diag"
  54         fi
  55
  56         # Set up directories and file names
  57         __BuildLogRootName=$subDirectoryName
  58         __BuildLog="$__LogsDir/${__BuildLogRootName}.${__BuildOS}.${__BuildArch}.${__BuildType}.log"
  59         __BuildWrn="$__LogsDir/${__BuildLogRootName}.${__BuildOS}.${__BuildArch}.${__BuildType}.wrn"
  60         __BuildErr="$__LogsDir/${__BuildLogRootName}.${__BuildOS}.${__BuildArch}.${__BuildType}.err"
  61         __MsbuildLog="/fileloggerparameters:\"Verbosity=normal;LogFile=${__BuildLog}\""
  62         __MsbuildWrn="/fileloggerparameters1:\"WarningsOnly;LogFile=${__BuildWrn}\""
  63         __MsbuildErr="/fileloggerparameters2:\"ErrorsOnly;LogFile=${__BuildErr}\""
  64         __Logging="$__MsbuildLog $__MsbuildWrn $__MsbuildErr /consoleloggerparameters:$buildVerbosity"
  65
  66         nextCommand="\"${__DotNetCli}\" msbuild \"${__ProjectDir}/tests/runtest.proj\" /p:RestoreAdditionalProjectSources=https://dotnet.myget.org/F/dotnet-core/ /p:BuildWrappers=true /p:TargetsWindows=false $__Logging /p:__BuildOS=$__BuildOS /p:__BuildType=$__BuildType /p:__BuildArch=$__BuildArch"
  67         eval $nextCommand
  68
  69         if [ $? -ne 0 ]; then
  70             echo "${__MsgPrefix}Error: build failed. Refer to the build log files for details (above)"
  71             exit 1
  72         else
  73             echo "XUnit Wrappers have been built."
  74             echo { "\"build_os\"": "\"${__BuildOS}\"", "\"build_arch\"": "\"${__BuildArch}\"", "\"build_type\"": "\"${__BuildType}\"" } > "${__TestWorkingDir}/build_info.json"
  75
  76         fi
  77     fi
  78 }
  79
  80 generate_layout()
  81 {
  82     echo "${__MsgPrefix}Creating test overlay..."
  83
  84     __TestDir=$__ProjectDir/tests
  85     __ProjectFilesDir=$__TestDir
  86     __TestBinDir=$__TestWorkingDir
  87
  88     if [ $__RebuildTests -ne 0 ]; then
  89         if [ -d "${__TestBinDir}" ]; then
  90             echo "Removing tests build dir: ${__TestBinDir}"
  91             rm -rf $__TestBinDir
  92         fi
  93     fi
  94
  95     __CMakeBinDir="${__TestBinDir}"
  96
  97     if [ -z "$__TestIntermediateDir" ]; then
  98         __TestIntermediateDir="tests/obj/${__BuildOS}.${__BuildArch}.${__BuildType}"
  99     fi
 100
 101     echo "__BuildOS: ${__BuildOS}"
 102     echo "__BuildArch: ${__BuildArch}"
 103     echo "__BuildType: ${__BuildType}"
 104     echo "__TestIntermediateDir: ${__TestIntermediateDir}"
 105
 106     if [ ! -f "$__TestBinDir" ]; then
 107         echo "Creating TestBinDir: ${__TestBinDir}"
 108         mkdir -p $__TestBinDir
 109     fi
 110     if [ ! -f "$__LogsDir" ]; then
 111         echo "Creating LogsDir: ${__LogsDir}"
 112         mkdir -p $__LogsDir
 113     fi
 114     if [ ! -f "$__MsbuildDebugLogsDir" ]; then
 115         echo "Creating MsbuildDebugLogsDir: ${__MsbuildDebugLogsDir}"
 116         mkdir -p $__MsbuildDebugLogsDir
 117     fi
 118
 119     # Set up the directory for MSBuild debug logs.
 120     export MSBUILDDEBUGPATH="${__MsbuildDebugLogsDir}"
 121
 122     __BuildProperties="-p:OSGroup=${__BuildOS} -p:BuildOS=${__BuildOS} -p:BuildArch=${__BuildArch} -p:BuildType=${__BuildType}"
 123
 124     # =========================================================================================
 125     # ===
 126     # === Restore product binaries from packages
 127     # ===
 128     # =========================================================================================
 129
 130     build_MSBuild_projects "Restore_Packages" "${__ProjectDir}/tests/build.proj" "Restore product binaries (build tests)" "/t:BatchRestorePackages"
 131
 132     if [ -n "$__UpdateInvalidPackagesArg" ]; then
 133         __up="/t:UpdateInvalidPackageVersions"
 134     fi
 135
 136     echo "${__MsgPrefix}Creating test overlay..."
 137
 138     if [ -z "$xUnitTestBinBase" ]; then
 139         xUnitTestBinBase=$__TestWorkingDir
 140     fi
 141
 142     export CORE_ROOT=$xUnitTestBinBase/Tests/Core_Root
 143
 144     if [ -d "${CORE_ROOT}" ]; then
 145         rm -rf $CORE_ROOT
 146     fi
 147
 148     mkdir -p $CORE_ROOT
 149
 150     build_MSBuild_projects "Tests_Overlay_Managed" "${__ProjectDir}/tests/runtest.proj" "Creating test overlay" "/t:CreateTestOverlay"
 151
 152     chmod +x $__BinDir/corerun
 153     chmod +x $__BinDir/crossgen
 154
 155     # Make sure to copy over the pulled down packages
 156     cp -r $__BinDir/* $CORE_ROOT/ > /dev/null
 157
 158     if [ "$__BuildOS" != "OSX" ]; then
 159         nextCommand="\"$__TestDir/setup-stress-dependencies.sh\" --outputDir=$CORE_ROOT"
 160         echo "Resolve runtime dependences via $nextCommand"
 161         eval $nextCommand
 162     fi
 163
 164     # Precompile framework assemblies with crossgen if required
 165     if [ $__DoCrossgen -ne 0 ]; then
 166         precompile_coreroot_fx
 167     fi
 168 }
 169
 170 precompile_coreroot_fx()
 171 {
 172     echo "${__MsgPrefix}Running crossgen on framework assemblies in CORE_ROOT: '${CORE_ROOT}'"
 173
 174     # Read the exclusion file for this platform
 175     skipCrossGenFiles=($(read_array "$(dirname "$0")/tests/skipCrossGenFiles.${__BuildArch}.txt"))
 176
 177     local overlayDir=$CORE_ROOT
 178
 179     # TODO: Remove System.Xml.XDocument.dll exclusion - it was added because "custom" xunit.console.dll fails when running from Core_Root with crossgen-d framework.
 180     filesToPrecompile=$(find -L $overlayDir -iname \*.dll -not -iname \*.ni.dll -not -iname \*-ms-win-\* -not -iname xunit.\* -not -name System.Xml.XDocument.dll -type f)
 181     for fileToPrecompile in ${filesToPrecompile}
 182     do
 183         local filename=${fileToPrecompile}
 184         if is_skip_crossgen_test "$(basename $filename)"; then
 185                 continue
 186         fi
 187         echo Precompiling $filename
 188         $overlayDir/crossgen /Platform_Assemblies_Paths $overlayDir $filename 1> $filename.stdout 2>$filename.stderr
 189         local exitCode=$?
 190         if [[ $exitCode != 0 ]]; then
 191             if grep -q -e '0x80131018' $filename.stderr; then
 192                 printf "\n\t$filename is not a managed assembly.\n\n"
 193             else
 194                 echo Unable to precompile $filename.
 195                 cat $filename.stdout
 196                 cat $filename.stderr
 197                 exit $exitCode
 198             fi
 199         else
 200             rm $filename.{stdout,stderr}
 201         fi
 202     done
 203 }
 204
 205 declare -a skipCrossGenFiles
 206
 207 function is_skip_crossgen_test {
 208     for skip in "${skipCrossGenFiles[@]}"; do
 209         if [ "$1" == "$skip" ]; then
 210             return 0
 211         fi
 212     done
 213     return 1
 214 }
 215
 216 # Get an array of items by reading the specified file line by line.
 217 function read_array {
 218     local theArray=()
 219
 220     if [ ! -f "$1" ]; then
 221         return
 222     fi
 223
 224     # bash in Mac OS X doesn't support 'readarray', so using alternate way instead.
 225     # readarray -t theArray < "$1"
 226     # Any line that starts with '#' is ignored.
 227     while IFS='' read -r line || [ -n "$line" ]; do
 228         if [[ $line != "#"* ]]; then
 229             theArray[${#theArray[@]}]=$line
 230         fi
 231     done < "$1"
 232     echo ${theArray[@]}
 233 }
 234
 235 generate_testhost()
 236 {
 237     echo "${__MsgPrefix}Generating test host..."
 238
 239     export TEST_HOST=$xUnitTestBinBase/testhost
 240
 241     if [ -d "${TEST_HOST}" ]; then
 242         rm -rf $TEST_HOST
 243     fi
 244
 245     mkdir -p $TEST_HOST
 246
 247     build_MSBuild_projects "Tests_Generate_TestHost" "${__ProjectDir}/tests/runtest.proj" "Creating test host" "/t:CreateTestHost"
 248 }
 249
 250
 251 build_Tests()
 252 {
 253     echo "${__MsgPrefix}Building Tests..."
 254
 255     __TestDir=$__ProjectDir/tests
 256     __ProjectFilesDir=$__TestDir
 257     __TestBinDir=$__TestWorkingDir
 258
 259     if [ -f  "${__TestWorkingDir}/build_info.json" ]; then
 260         rm  "${__TestWorkingDir}/build_info.json"
 261     fi
 262
 263     if [ $__RebuildTests -ne 0 ]; then
 264         if [ -d "${__TestBinDir}" ]; then
 265             echo "Removing tests build dir: ${__TestBinDir}"
 266             rm -rf $__TestBinDir
 267         fi
 268     fi
 269
 270     export __CMakeBinDir="${__TestBinDir}"
 271     if [ ! -d "${__TestIntermediatesDir}" ]; then
 272         mkdir -p ${__TestIntermediatesDir}
 273     fi
 274
 275     __NativeTestIntermediatesDir="${__TestIntermediatesDir}/Native"
 276     if [  ! -d "${__NativeTestIntermediatesDir}" ]; then
 277         mkdir -p ${__NativeTestIntermediatesDir}
 278     fi
 279
 280     __ManagedTestIntermediatesDir="${__TestIntermediatesDir}/Managed"
 281     if [ ! -d "${__ManagedTestIntermediatesDir}" ]; then
 282         mkdir -p ${__ManagedTestIntermediatesDir}
 283     fi
 284
 285     echo "__BuildOS: ${__BuildOS}"
 286     echo "__BuildArch: ${__BuildArch}"
 287     echo "__BuildType: ${__BuildType}"
 288     echo "__TestIntermediatesDir: ${__TestIntermediatesDir}"
 289     echo "__NativeTestIntermediatesDir: ${__NativeTestIntermediatesDir}"
 290     echo "__ManagedTestIntermediatesDir: ${__ManagedTestIntermediatesDir}"
 291
 292     if [ ! -f "$__TestBinDir" ]; then
 293         echo "Creating TestBinDir: ${__TestBinDir}"
 294         mkdir -p $__TestBinDir
 295     fi
 296     if [ ! -f "$__LogsDir" ]; then
 297         echo "Creating LogsDir: ${__LogsDir}"
 298         mkdir -p $__LogsDir
 299     fi
 300     if [ ! -f "$__MsbuildDebugLogsDir" ]; then
 301         echo "Creating MsbuildDebugLogsDir: ${__MsbuildDebugLogsDir}"
 302         mkdir -p $__MsbuildDebugLogsDir
 303     fi
 304
 305     # Set up the directory for MSBuild debug logs.
 306     export MSBUILDDEBUGPATH="${__MsbuildDebugLogsDir}"
 307
 308     __BuildProperties="-p:OSGroup=${__BuildOS} -p:BuildOS=${__BuildOS} -p:BuildArch=${__BuildArch} -p:BuildType=${__BuildType}"
 309
 310     # =========================================================================================
 311     # ===
 312     # === Restore product binaries from packages
 313     # ===
 314     # =========================================================================================
 315
 316     if [ ${__SkipRestorePackages} != 1 ]; then
 317         build_MSBuild_projects "Restore_Product" "${__ProjectDir}/tests/build.proj" "Restore product binaries (build tests)" "/t:BatchRestorePackages"
 318     fi
 319
 320     if [ $__SkipNative != 1 ]; then
 321         build_native_projects "$__BuildArch" "${__NativeTestIntermediatesDir}"
 322
 323         if [ $? -ne 0 ]; then
 324             echo "${__MsgPrefix}Error: build failed. Refer to the build log files for details (above)"
 325             exit 1
 326         fi
 327     fi
 328
 329     if [ $__SkipManaged != 1 ]; then
 330         echo "Starting the Managed Tests Build..."
 331
 332         build_MSBuild_projects "Tests_Managed" "$__ProjectDir/tests/build.proj" "Managed tests build (build tests)" "$__up"
 333
 334         if [ $? -ne 0 ]; then
 335             echo "${__MsgPrefix}Error: build failed. Refer to the build log files for details (above)"
 336             exit 1
 337         else
 338             echo "Checking the Managed Tests Build..."
 339
 340             build_MSBuild_projects "Check_Test_Build" "${__ProjectDir}/tests/runtest.proj" "Check Test Build" "/t:CheckTestBuild"
 341
 342             if [ $? -ne 0 ]; then
 343                 echo "${__MsgPrefix}Error: Check Test Build failed."
 344                 exit 1
 345             fi
 346         fi
 347
 348         echo "Managed tests build success!"
 349     fi
 350
 351     build_test_wrappers
 352
 353     if [ -n "$__UpdateInvalidPackagesArg" ]; then
 354         __up="/t:UpdateInvalidPackageVersions"
 355     fi
 356
 357     generate_layout
 358 }
 359
 360 build_MSBuild_projects()
 361 {
 362     subDirectoryName=$1
 363     shift
 364     projectName=$1
 365     shift
 366     stepName="$1"
 367     shift
 368     extraBuildParameters=("$@")
 369
 370     # Set up directories and file names
 371     __BuildLogRootName=$subDirectoryName
 372     __BuildLog="$__LogsDir/${__BuildLogRootName}.${__BuildOS}.${__BuildArch}.${__BuildType}.log"
 373     __BuildWrn="$__LogsDir/${__BuildLogRootName}.${__BuildOS}.${__BuildArch}.${__BuildType}.wrn"
 374     __BuildErr="$__LogsDir/${__BuildLogRootName}.${__BuildOS}.${__BuildArch}.${__BuildType}.err"
 375
 376     # Use binclashlogger by default if no other logger is specified
 377     if [[ "${extraBuildParameters[*]}" == *"/l:"* ]]; then
 378         __msbuildEventLogging=
 379     else
 380         __msbuildEventLogging="/l:BinClashLogger,Tools/Microsoft.DotNet.Build.Tasks.dll\;LogFile=binclash.log"
 381     fi
 382
 383     if [[ "$subDirectoryName" == "Tests_Managed" ]]; then
 384         # Execute msbuild managed test build in stages - workaround for excessive data retention in MSBuild ConfigCache
 385         # See https://github.com/Microsoft/msbuild/issues/2993
 386
 387         # __SkipPackageRestore and __SkipTargetingPackBuild used  to control build by tests/src/dirs.proj
 388         export __SkipPackageRestore=false
 389         export __SkipTargetingPackBuild=false
 390         export __NumberOfTestGroups=3
 391
 392         __AppendToLog=false
 393
 394         if [ -n "$__priority1" ]; then
 395             export __NumberOfTestGroups=10
 396         fi
 397
 398         for (( testGroupToBuild=1 ; testGroupToBuild <= __NumberOfTestGroups; testGroupToBuild = testGroupToBuild + 1 ))
 399         do
 400             __msbuildLog="\"/flp:Verbosity=normal;LogFile=${__BuildLog};Append=${__AppendToLog}\""
 401             __msbuildWrn="\"/flp1:WarningsOnly;LogFile=${__BuildWrn};Append=${__AppendToLog}\""
 402             __msbuildErr="\"/flp2:ErrorsOnly;LogFile=${__BuildErr};Append=${__AppendToLog}\""
 403
 404             export __TestGroupToBuild=$testGroupToBuild
 405
 406             # Generate build command
 407             buildArgs=("/nologo" "/verbosity:minimal" "/clp:Summary")
 408             buildArgs+=("/p:RestoreDefaultOptimizationDataPackage=false" "/p:PortableBuild=true")
 409             buildArgs+=("/p:UsePartialNGENOptimization=false" "/maxcpucount")
 410
 411             buildArgs+=("$projectName" "${__msbuildLog}" "${__msbuildWrn}" "${__msbuildErr}")
 412             buildArgs+=("$__msbuildEventLogging")
 413             buildArgs+=("${extraBuildParameters[@]}")
 414             buildArgs+=("${__CommonMSBuildArgs[@]}")
 415             buildArgs+=("${__UnprocessedBuildArgs[@]}")
 416
 417             nextCommand="\"$__ProjectRoot/dotnet.sh\" msbuild ${buildArgs[@]}"
 418             echo "Building step '$stepName' testGroupToBuild=$testGroupToBuild via $nextCommand"
 419             eval $nextCommand
 420
 421             # Make sure everything is OK
 422             if [ $? -ne 0 ]; then
 423                 echo "${__MsgPrefix}Failed to build $stepName. See the build logs:"
 424                 echo "    $__BuildLog"
 425                 echo "    $__BuildWrn"
 426                 echo "    $__BuildErr"
 427                 exit 1
 428             fi
 429             export __SkipPackageRestore=true
 430             export __SkipTargetingPackBuild=true
 431             __AppendToLog=true
 432         done
 433     else
 434         __msbuildLog="\"/flp:Verbosity=normal;LogFile=${__BuildLog}\""
 435         __msbuildWrn="\"/flp1:WarningsOnly;LogFile=${__BuildWrn}\""
 436         __msbuildErr="\"/flp2:ErrorsOnly;LogFile=${__BuildErr}\""
 437
 438         # Generate build command
 439         buildArgs=("/nologo" "/verbosity:minimal" "/clp:Summary")
 440         buildArgs+=("/p:RestoreDefaultOptimizationDataPackage=false" "/p:PortableBuild=true")
 441         buildArgs+=("/p:UsePartialNGENOptimization=false" "/maxcpucount")
 442
 443         buildArgs+=("$projectName" "${__msbuildLog}" "${__msbuildWrn}" "${__msbuildErr}")
 444         buildArgs+=("$__msbuildEventLogging")
 445         buildArgs+=("${extraBuildParameters[@]}")
 446         buildArgs+=("${__CommonMSBuildArgs[@]}")
 447         buildArgs+=("${__UnprocessedBuildArgs[@]}")
 448
 449         nextCommand="\"$__ProjectRoot/dotnet.sh\" msbuild ${buildArgs[@]}"
 450         echo "Building step '$stepName' via $nextCommand"
 451         eval $nextCommand
 452
 453         # Make sure everything is OK
 454         if [ $? -ne 0 ]; then
 455             echo "${__MsgPrefix}Failed to build $stepName. See the build logs:"
 456             echo "    $__BuildLog"
 457             echo "    $__BuildWrn"
 458             echo "    $__BuildErr"
 459             exit 1
 460         fi
 461     fi
 462 }
 463
 464 build_native_projects()
 465 {
 466     platformArch="$1"
 467     intermediatesForBuild="$2"
 468
 469     extraCmakeArguments="-DCLR_CMAKE_TARGET_OS=${__BuildOS} -DCLR_CMAKE_HOST_ARCH=${platformArch}"
 470     message="native tests assets"
 471
 472     # All set to commence the build
 473     echo "Commencing build of $message for $__BuildOS.$__BuildArch.$__BuildType in $intermediatesForBuild"
 474
 475     generator=""
 476     buildFile="Makefile"
 477     buildTool="make"
 478     if [ $__UseNinja == 1 ]; then
 479         generator="ninja"
 480         buildFile="build.ninja"
 481         if ! buildTool=$(command -v ninja || command -v ninja-build); then
 482            echo "Unable to locate ninja!" 1>&2
 483            exit 1
 484         fi
 485     fi
 486
 487     if [ $__SkipConfigure == 0 ]; then
 488         # if msbuild is not supported, then set __SkipGenerateVersion to 1
 489         if [ $__isMSBuildOnNETCoreSupported == 0 ]; then __SkipGenerateVersion=1; fi
 490         # Drop version.c file
 491         __versionSourceFile="$intermediatesForBuild/version.c"
 492         if [ $__SkipGenerateVersion == 0 ]; then
 493             pwd
 494             $__ProjectRoot/dotnet.sh msbuild /nologo /verbosity:minimal /clp:Summary \
 495                                      /p:RestoreDefaultOptimizationDataPackage=false /p:PortableBuild=true \
 496                                      /p:UsePartialNGENOptimization=false /maxcpucount \
 497                                      $__ProjectDir/build.proj /t:GenerateVersionHeader \
 498                                      /p:GenerateVersionHeader=true /p:NativeVersionSourceFile=$__versionSourceFile \
 499                                      /l:BinClashLogger,Tools/Microsoft.DotNet.Build.Tasks.dll\;LogFile=binclash.log \
 500                                      $__CommonMSBuildArgs $__UnprocessedBuildArgs
 501         else
 502             # Generate the dummy version.c, but only if it didn't exist to make sure we don't trigger unnecessary rebuild
 503             __versionSourceLine="static char sccsid[] __attribute__((used)) = \"@(#)No version information produced\";"
 504             if [ -e $__versionSourceFile ]; then
 505                 read existingVersionSourceLine < $__versionSourceFile
 506             fi
 507             if [ "$__versionSourceLine" != "$existingVersionSourceLine" ]; then
 508                 echo $__versionSourceLine > $__versionSourceFile
 509             fi
 510         fi
 511
 512         pushd "$intermediatesForBuild"
 513         # Regenerate the CMake solution
 514         # Force cross dir to point to project root cross dir, in case there is a cross build.
 515         scriptDir="$__ProjectRoot/src/pal/tools"
 516         if [[ $__GccBuild == 0 ]]; then
 517             nextCommand="CONFIG_DIR=\"$__ProjectRoot/cross\" \"$scriptDir/gen-buildsys-clang.sh\" \"$__TestDir\" $__ClangMajorVersion $__ClangMinorVersion $platformArch $scriptDir $__BuildType $__CodeCoverage $generator $extraCmakeArguments $__cmakeargs"
 518         else
 519             nextCommand="CONFIG_DIR=\"$__ProjectRoot/cross\" \"$scriptDir/gen-buildsys-gcc.sh\" \"$__TestDir\" \"$__GccMajorVersion\" \"$__GccMinorVersion\" $platformArch $scriptDir $__BuildType $__CodeCoverage $generator $extraCmakeArguments $__cmakeargs"
 520         fi
 521         echo "Invoking $nextCommand"
 522         eval $nextCommand
 523         popd
 524     fi
 525
 526     if [ ! -f "$intermediatesForBuild/$buildFile" ]; then
 527         echo "Failed to generate $message build project!"
 528         exit 1
 529     fi
 530
 531     # Build
 532     if [ $__ConfigureOnly == 1 ]; then
 533         echo "Finish configuration & skipping $message build."
 534         return
 535     fi
 536
 537     pushd "$intermediatesForBuild"
 538
 539     echo "Executing $buildTool install -j $__NumProc"
 540
 541     $buildTool install -j $__NumProc
 542     if [ $? != 0 ]; then
 543         echo "Failed to build $message."
 544         exit 1
 545     fi
 546
 547     popd
 548     echo "Native tests build success!"
 549 }
 550
 551 usage()
 552 {
 553     echo "Usage: $0 [BuildArch] [BuildType] [verbose] [coverage] [cross] [clangx.y] [ninja] [runtests] [bindir]"
 554     echo "BuildArch can be: x64, x86, arm, armel, arm64"
 555     echo "BuildType can be: debug, checked, release"
 556     echo "coverage - optional argument to enable code coverage build (currently supported only for Linux and OSX)."
 557     echo "ninja - target ninja instead of GNU make"
 558     echo "clangx.y - optional argument to build using clang version x.y - supported version 3.5 - 6.0"
 559     echo "gccx.y - optional argument to build using gcc version x.y."
 560     echo "cross - optional argument to signify cross compilation,"
 561     echo "      - will use ROOTFS_DIR environment variable if set."
 562     echo "portableLinux - build for Portable Linux Distribution"
 563     echo "portablebuild - Use portable build."
 564     echo "verbose - optional argument to enable verbose build output."
 565     echo "rebuild - if tests have already been built - rebuild them"
 566     echo "skipnative: skip the native tests build"
 567     echo "skipmanaged: skip the managed section of the test build"
 568     echo "buildtestwrappersonly - only build the test wrappers"
 569     echo "generatelayoutonly - only pull down dependencies and build coreroot"
 570     echo "generatetesthostonly - only pull down dependencies and build coreroot and the CoreFX testhost"
 571     echo "skiprestorepackages - skip package restore"
 572     echo "crossgen - Precompiles the framework managed assemblies in coreroot"
 573     echo "runtests - run tests after building them"
 574     echo "bindir - output directory (defaults to $__ProjectRoot/bin)"
 575     echo "msbuildonunsupportedplatform - build managed binaries even if distro is not officially supported."
 576     echo "priority1 - include priority=1 tests in the build"
 577     exit 1
 578 }
 579
 580
 581 # Obtain the location of the bash script to figure out where the root of the repo is.
 582 __ProjectRoot="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
 583
 584 # Use uname to determine what the CPU is.
 585 CPUName=$(uname -p)
 586
 587 # Some Linux platforms report unknown for platform, but the arch for machine.
 588 if [ "$CPUName" == "unknown" ]; then
 589     CPUName=$(uname -m)
 590 fi
 591
 592 case $CPUName in
 593     i686)
 594         echo "Unsupported CPU $CPUName detected, build might not succeed!"
 595         __BuildArch=x86
 596         __HostArch=x86
 597         ;;
 598
 599     x86_64)
 600         __BuildArch=x64
 601         __HostArch=x64
 602         ;;
 603
 604     armv7l)
 605         echo "Unsupported CPU $CPUName detected, build might not succeed!"
 606         __BuildArch=arm
 607         __HostArch=arm
 608         ;;
 609
 610     aarch64)
 611         __BuildArch=arm64
 612         __HostArch=arm64
 613         ;;
 614
 615     *)
 616         echo "Unknown CPU $CPUName detected, configuring as if for x64"
 617         __BuildArch=x64
 618         __HostArch=x64
 619         ;;
 620 esac
 621
 622 # Use uname to determine what the OS is.
 623 OSName=$(uname -s)
 624 case $OSName in
 625     Linux)
 626         __BuildOS=Linux
 627         __HostOS=Linux
 628         ;;
 629
 630     Darwin)
 631         __BuildOS=OSX
 632         __HostOS=OSX
 633         ;;
 634
 635     FreeBSD)
 636         __BuildOS=FreeBSD
 637         __HostOS=FreeBSD
 638         ;;
 639
 640     OpenBSD)
 641         __BuildOS=OpenBSD
 642         __HostOS=OpenBSD
 643         ;;
 644
 645     NetBSD)
 646         __BuildOS=NetBSD
 647         __HostOS=NetBSD
 648         ;;
 649
 650     SunOS)
 651         __BuildOS=SunOS
 652         __HostOS=SunOS
 653         ;;
 654
 655     *)
 656         echo "Unsupported OS $OSName detected, configuring as if for Linux"
 657         __BuildOS=Linux
 658         __HostOS=Linux
 659         ;;
 660 esac
 661
 662 __BuildType=Debug
 663 __CodeCoverage=
 664 __IncludeTests=INCLUDE_TESTS
 665
 666 # Set the various build properties here so that CMake and MSBuild can pick them up
 667 export __ProjectDir="$__ProjectRoot"
 668 __SourceDir="$__ProjectDir/src"
 669 __PackagesDir="$__ProjectDir/packages"
 670 __RootBinDir="$__ProjectDir/bin"
 671 __BuildToolsDir="$__ProjectDir/Tools"
 672 __DotNetCli="$__ProjectDir/dotnet.sh"
 673 __UnprocessedBuildArgs=
 674 __CommonMSBuildArgs=
 675 __MSBCleanBuildArgs=
 676 __UseNinja=0
 677 __VerboseBuild=0
 678 __SkipRestore=""
 679 __SkipNative=0
 680 __SkipManaged=0
 681 __SkipConfigure=0
 682 __SkipGenerateVersion=0
 683 __ConfigureOnly=0
 684 __CrossBuild=0
 685 __ClangMajorVersion=0
 686 __ClangMinorVersion=0
 687 __GccBuild=0
 688 __GccMajorVersion=0
 689 __GccMinorVersion=0
 690 __NuGetPath="$__PackagesDir/NuGet.exe"
 691 __SkipRestorePackages=0
 692 __DistroRid=""
 693 __cmakeargs=""
 694 __PortableLinux=0
 695 __msbuildonunsupportedplatform=0
 696 __NativeTestIntermediatesDir=
 697 __RunTests=0
 698 __RebuildTests=0
 699 __BuildTestWrappers=1
 700 __GenerateLayoutOnly=
 701 __GenerateTestHostOnly=
 702 __priority1=
 703 __BuildTestWrappersOnly=
 704 __DoCrossgen=0
 705 CORE_ROOT=
 706
 707 while :; do
 708     if [ $# -le 0 ]; then
 709         break
 710     fi
 711
 712     lowerI="$(echo $1 | awk '{print tolower($0)}')"
 713     case $lowerI in
 714         -\?|-h|--help)
 715             usage
 716             exit 1
 717             ;;
 718
 719         x86)
 720             __BuildArch=x86
 721             ;;
 722
 723         x64)
 724             __BuildArch=x64
 725             ;;
 726
 727         arm)
 728             __BuildArch=arm
 729             ;;
 730
 731         armel)
 732             __BuildArch=armel
 733             ;;
 734
 735         arm64)
 736             __BuildArch=arm64
 737             ;;
 738
 739         debug)
 740             __BuildType=Debug
 741             ;;
 742
 743         checked)
 744             __BuildType=Checked
 745             ;;
 746
 747         release)
 748             __BuildType=Release
 749             ;;
 750
 751         coverage)
 752             __CodeCoverage=Coverage
 753             ;;
 754
 755         cross)
 756             __CrossBuild=1
 757             ;;
 758
 759         portablebuild=false)
 760             __PortableBuild=0
 761             ;;
 762
 763         portablelinux)
 764             if [ "$__BuildOS" == "Linux" ]; then
 765                 __PortableLinux=1
 766             else
 767                 echo "ERROR: portableLinux not supported for non-Linux platforms."
 768                 exit 1
 769             fi
 770             ;;
 771
 772         verbose)
 773             __VerboseBuild=1
 774             ;;
 775
 776         clang3.5|-clang3.5)
 777             __ClangMajorVersion=3
 778             __ClangMinorVersion=5
 779             ;;
 780
 781         clang3.6|-clang3.6)
 782             __ClangMajorVersion=3
 783             __ClangMinorVersion=6
 784             ;;
 785
 786         clang3.7|-clang3.7)
 787             __ClangMajorVersion=3
 788             __ClangMinorVersion=7
 789             ;;
 790
 791         clang3.8|-clang3.8)
 792             __ClangMajorVersion=3
 793             __ClangMinorVersion=8
 794             ;;
 795
 796         clang3.9|-clang3.9)
 797             __ClangMajorVersion=3
 798             __ClangMinorVersion=9
 799             ;;
 800
 801         clang4.0|-clang4.0)
 802             __ClangMajorVersion=4
 803             __ClangMinorVersion=0
 804             ;;
 805
 806         clang5.0|-clang5.0)
 807             __ClangMajorVersion=5
 808             __ClangMinorVersion=0
 809             ;;
 810
 811         clang6.0|-clang6.0)
 812             __ClangMajorVersion=6
 813             __ClangMinorVersion=0
 814             ;;
 815
 816         gcc5|-gcc5)
 817             __GccMajorVersion=5
 818             __GccMinorVersion=
 819             __GccBuild=1
 820             ;;
 821
 822         gcc6|-gcc6)
 823             __GccMajorVersion=6
 824             __GccMinorVersion=
 825             __GccBuild=1
 826             ;;
 827
 828         gcc7|-gcc7)
 829             __GccMajorVersion=7
 830             __GccMinorVersion=
 831             __GccBuild=1
 832             ;;
 833
 834         gcc8|-gcc8)
 835             __GccMajorVersion=8
 836             __GccMinorVersion=
 837             __GccBuild=1
 838             ;;
 839
 840         gcc|-gcc)
 841             __GccMajorVersion=
 842             __GccMinorVersion=
 843             __GccBuild=1
 844             ;;
 845
 846         ninja)
 847             __UseNinja=1
 848             ;;
 849
 850         runtests)
 851             __RunTests=1
 852             ;;
 853
 854         rebuild)
 855             __RebuildTests=1
 856             ;;
 857
 858         skipnative|-skipnative)
 859             __SkipNative=1
 860             ;;
 861
 862         skipmanaged|-skipmanaged)
 863             __SkipManaged=1
 864             __BuildTestWrappers=0
 865             ;;
 866
 867         buildtestwrappersonly)
 868             __BuildTestWrappersOnly=1
 869             ;;
 870
 871         generatelayoutonly)
 872             __GenerateLayoutOnly=1
 873             ;;
 874
 875         generatetesthostonly)
 876             __GenerateTestHostOnly=1
 877             ;;
 878
 879         skiprestorepackages)
 880             __SkipRestorePackages=1
 881             ;;
 882
 883         crossgen)
 884             __DoCrossgen=1
 885             ;;
 886
 887         bindir)
 888             if [ -n "$2" ]; then
 889                 __RootBinDir="$2"
 890                 if [ ! -d $__RootBinDir ]; then
 891                     mkdir $__RootBinDir
 892                 fi
 893                 __RootBinParent=$(dirname $__RootBinDir)
 894                 __RootBinName=${__RootBinDir##*/}
 895                 __RootBinDir="$(cd $__RootBinParent &>/dev/null && printf %s/%s $PWD $__RootBinName)"
 896                 shift
 897             else
 898                 echo "ERROR: 'bindir' requires a non-empty option argument"
 899                 exit 1
 900             fi
 901             ;;
 902
 903         msbuildonunsupportedplatform)
 904             __msbuildonunsupportedplatform=1
 905             ;;
 906
 907         priority1)
 908             __priority1=1
 909             __UnprocessedBuildArgs+=("/p:CLRTestPriorityToBuild=1")
 910             ;;
 911
 912         *)
 913             __UnprocessedBuildArgs+=("$1")
 914             ;;
 915     esac
 916
 917     shift
 918 done
 919
 920 # Get the number of processors available to the scheduler
 921 # Other techniques such as `nproc` only get the number of
 922 # processors available to a single process.
 923 if [ `uname` = "FreeBSD" ]; then
 924   __NumProc=`sysctl hw.ncpu | awk '{ print $2+1 }'`
 925 elif [ `uname` = "NetBSD" ]; then
 926   __NumProc=$(($(getconf NPROCESSORS_ONLN)+1))
 927 elif [ `uname` = "Darwin" ]; then
 928   __NumProc=$(($(getconf _NPROCESSORS_ONLN)+1))
 929 else
 930   __NumProc=$(nproc --all)
 931 fi
 932
 933 __CommonMSBuildArgs=("/p:__BuildArch=$__BuildArch" "/p:__BuildType=$__BuildType" "/p:__BuildOS=$__BuildOS")
 934
 935 # Configure environment if we are doing a verbose build
 936 if [ $__VerboseBuild == 1 ]; then
 937     export VERBOSE=1
 938     __CommonMSBuildArgs+=("/v:detailed")
 939 fi
 940
 941 # Set default clang version
 942 if [[ $__ClangMajorVersion == 0 && $__ClangMinorVersion == 0 ]]; then
 943     if [[ "$__BuildArch" == "arm" || "$__BuildArch" == "armel" ]]; then
 944         __ClangMajorVersion=5
 945         __ClangMinorVersion=0
 946     else
 947         __ClangMajorVersion=3
 948         __ClangMinorVersion=9
 949     fi
 950 fi
 951
 952 # Set dependent variables
 953 __LogsDir="$__RootBinDir/Logs"
 954 __MsbuildDebugLogsDir="$__LogsDir/MsbuildDebugLogs"
 955
 956 # Set the remaining variables based upon the determined build configuration
 957 __BinDir="$__RootBinDir/Product/$__BuildOS.$__BuildArch.$__BuildType"
 958 __PackagesBinDir="$__BinDir/.nuget"
 959 __TestDir="$__ProjectDir/tests"
 960 __TestWorkingDir="$__RootBinDir/tests/$__BuildOS.$__BuildArch.$__BuildType"
 961 __IntermediatesDir="$__RootBinDir/obj/$__BuildOS.$__BuildArch.$__BuildType"
 962 __TestIntermediatesDir="$__RootBinDir/tests/obj/$__BuildOS.$__BuildArch.$__BuildType"
 963 __isMSBuildOnNETCoreSupported=0
 964 __CrossComponentBinDir="$__BinDir"
 965 __CrossCompIntermediatesDir="$__IntermediatesDir/crossgen"
 966
 967 __CrossArch="$__HostArch"
 968 if [[ "$__HostArch" == "x64" && "$__BuildArch" == "arm" ]]; then
 969     __CrossArch="x86"
 970 fi
 971 if [ $__CrossBuild == 1 ]; then
 972     __CrossComponentBinDir="$__CrossComponentBinDir/$__CrossArch"
 973 fi
 974 __CrossgenCoreLibLog="$__LogsDir/CrossgenCoreLib_$__BuildOS.$BuildArch.$__BuildType.log"
 975 __CrossgenExe="$__CrossComponentBinDir/crossgen"
 976
 977 isMSBuildOnNETCoreSupported
 978
 979 # CI_SPECIFIC - On CI machines, $HOME may not be set. In such a case, create a subfolder and set the variable to it.
 980 # This is needed by CLI to function.
 981 if [ -z "$HOME" ]; then
 982     if [ ! -d "$__ProjectDir/temp_home" ]; then
 983         mkdir temp_home
 984     fi
 985     export HOME=$__ProjectDir/temp_home
 986     echo "HOME not defined; setting it to $HOME"
 987 fi
 988
 989 # Configure environment if we are doing a cross compile.
 990 if [ $__CrossBuild == 1 ]; then
 991     export CROSSCOMPILE=1
 992     if ! [[ -n "$ROOTFS_DIR" ]]; then
 993         export ROOTFS_DIR="$__ProjectRoot/cross/rootfs/$__BuildArch"
 994     fi
 995 fi
 996
 997 # init the target distro name
 998 initTargetDistroRid
 999
1000 # Override tool directory
1001
1002 if [[ (-z "$__GenerateLayoutOnly") && (-z "$__GenerateTestHostOnly") && (-z "$__BuildTestWrappersOnly") ]]; then
1003     build_Tests
1004 elif [ ! -z "$__BuildTestWrappersOnly" ]; then
1005     build_test_wrappers
1006 else
1007     generate_layout
1008     if [ ! -z "$__GenerateTestHostOnly" ]; then
1009         generate_testhost
1010     fi
1011 fi
1012
1013 if [ $? -ne 0 ]; then
1014     echo "Failed to build tests"
1015     exit 1
1016 fi
1017
1018 echo "${__MsgPrefix}Test build successful."
1019 echo "${__MsgPrefix}Test binaries are available at ${__TestBinDir}"
1020
1021 __testNativeBinDir=$__IntermediatesDir/tests
1022
1023 if [ $__RunTests -ne 0 ]; then
1024
1025     echo "Run Tests..."
1026
1027     nextCommand="$__TestDir/runtest.sh --testRootDir=$__TestBinDir --coreClrBinDir=$__BinDir --coreFxBinDir=$CORE_ROOT --testNativeBinDir=$__testNativeBinDir"
1028     echo "$nextCommand"
1029     eval $nextCommand
1030
1031     echo "Tests run successful."
1032 else
1033     echo "To run all tests use 'tests/runtests.sh' where:"
1034     echo "    testRootDir      = $__TestBinDir"
1035     echo "    coreClrBinDir    = $__BinDir"
1036     echo "    coreFxBinDir     = $CORE_ROOT"
1037     echo "    testNativeBinDir = $__testNativeBinDir"
1038     echo " -------------------------------------------------- "
1039     echo " Example runtest.sh command"
1040     echo ""
1041     echo " ./tests/runtest.sh --coreOverlayDir=$CORE_ROOT --testNativeBinDir=$__testNativeBinDir --testRootDir=$__TestBinDir --copyNativeTestBin"
1042     echo " -------------------------------------------------- "
1043     echo "To run single test use the following command:"
1044     echo "    bash ${__TestBinDir}/__TEST_PATH__/__TEST_NAME__.sh -coreroot=${CORE_ROOT}"
1045 fi
1046