build-test.sh

   1 #!/usr/bin/env bash
   2
   3 __PortableBuild=1
   4
   5 initTargetDistroRid()
   6 {
   7     source init-distro-rid.sh
   8
   9     # Only pass ROOTFS_DIR if cross is specified.
  10     if (( ${__CrossBuild} == 1 )); then
  11         passedRootfsDir=${ROOTFS_DIR}
  12     fi
  13
  14     initDistroRidGlobal ${__BuildOS} ${__BuildArch} ${__PortableBuild} ${passedRootfsDir}
  15 }
  16
  17 isMSBuildOnNETCoreSupported()
  18 {
  19     __isMSBuildOnNETCoreSupported=$__msbuildonunsupportedplatform
  20
  21     if [ $__isMSBuildOnNETCoreSupported == 1 ]; then
  22         return
  23     fi
  24
  25     if [ "$__HostArch" == "x64" ]; then
  26         if [ "$__HostOS" == "Linux" ]; then
  27             __isMSBuildOnNETCoreSupported=1
  28             UNSUPPORTED_RIDS=("debian.9-x64" "ubuntu.17.04-x64")
  29             for UNSUPPORTED_RID in "${UNSUPPORTED_RIDS[@]}"
  30             do
  31                 if [ "${__DistroRid}" == "$UNSUPPORTED_RID" ]; then
  32                     __isMSBuildOnNETCoreSupported=0
  33                     break
  34                 fi
  35             done
  36         elif [ "$__HostOS" == "OSX" ]; then
  37             __isMSBuildOnNETCoreSupported=1
  38         fi
  39     fi
  40 }
  41
  42 build_test_wrappers()
  43 {
  44     if [ $__BuildTestWrappers -ne -0 ]; then
  45         echo "${__MsgPrefix}Creating test wrappers..."
  46
  47         export __Exclude="${__ProjectDir}/tests/issues.targets"
  48         export __BuildLogRootName="Tests_XunitWrapper"
  49
  50         buildVerbosity="Summary"
  51
  52         if [ $__VerboseBuild == 1 ]; then
  53             buildVerbosity="Diag"
  54         fi
  55
  56         # Set up directories and file names
  57         __BuildLogRootName=$subDirectoryName
  58         __BuildLog="$__LogsDir/${__BuildLogRootName}.${__BuildOS}.${__BuildArch}.${__BuildType}.log"
  59         __BuildWrn="$__LogsDir/${__BuildLogRootName}.${__BuildOS}.${__BuildArch}.${__BuildType}.wrn"
  60         __BuildErr="$__LogsDir/${__BuildLogRootName}.${__BuildOS}.${__BuildArch}.${__BuildType}.err"
  61         __MsbuildLog="/fileloggerparameters:\"Verbosity=normal;LogFile=${__BuildLog}\""
  62         __MsbuildWrn="/fileloggerparameters1:\"WarningsOnly;LogFile=${__BuildWrn}\""
  63         __MsbuildErr="/fileloggerparameters2:\"ErrorsOnly;LogFile=${__BuildErr}\""
  64         __Logging="$__MsbuildLog $__MsbuildWrn $__MsbuildErr /consoleloggerparameters:$buildVerbosity"
  65
  66         nextCommand="\"${__DotNetCli}\" msbuild \"${__ProjectDir}/tests/runtest.proj\" /p:RestoreAdditionalProjectSources=https://dotnet.myget.org/F/dotnet-core/ /p:BuildWrappers=true /p:TargetsWindows=false $__Logging /p:__BuildOS=$__BuildOS /p:__BuildType=$__BuildType /p:__BuildArch=$__BuildArch"
  67         eval $nextCommand
  68
  69         if [ $? -ne 0 ]; then
  70             echo "${__MsgPrefix}Error: build failed. Refer to the build log files for details (above)"
  71             exit 1
  72         else
  73             echo "XUnit Wrappers have been built."
  74             echo { "\"build_os\"": "\"${__BuildOS}\"", "\"build_arch\"": "\"${__BuildArch}\"", "\"build_type\"": "\"${__BuildType}\"" } > "${__TestWorkingDir}/build_info.json"
  75
  76         fi
  77     fi
  78 }
  79
  80 generate_layout()
  81 {
  82     echo "${__MsgPrefix}Creating test overlay..."
  83
  84     __TestDir=$__ProjectDir/tests
  85     __ProjectFilesDir=$__TestDir
  86     __TestBinDir=$__TestWorkingDir
  87
  88     if [ $__RebuildTests -ne 0 ]; then
  89         if [ -d "${__TestBinDir}" ]; then
  90             echo "Removing tests build dir: ${__TestBinDir}"
  91             rm -rf $__TestBinDir
  92         fi
  93     fi
  94
  95     __CMakeBinDir="${__TestBinDir}"
  96
  97     if [ -z "$__TestIntermediateDir" ]; then
  98         __TestIntermediateDir="tests/obj/${__BuildOS}.${__BuildArch}.${__BuildType}"
  99     fi
 100
 101     echo "__BuildOS: ${__BuildOS}"
 102     echo "__BuildArch: ${__BuildArch}"
 103     echo "__BuildType: ${__BuildType}"
 104     echo "__TestIntermediateDir: ${__TestIntermediateDir}"
 105
 106     if [ ! -f "$__TestBinDir" ]; then
 107         echo "Creating TestBinDir: ${__TestBinDir}"
 108         mkdir -p $__TestBinDir
 109     fi
 110     if [ ! -f "$__LogsDir" ]; then
 111         echo "Creating LogsDir: ${__LogsDir}"
 112         mkdir -p $__LogsDir
 113     fi
 114     if [ ! -f "$__MsbuildDebugLogsDir" ]; then
 115         echo "Creating MsbuildDebugLogsDir: ${__MsbuildDebugLogsDir}"
 116         mkdir -p $__MsbuildDebugLogsDir
 117     fi
 118
 119     # Set up the directory for MSBuild debug logs.
 120     export MSBUILDDEBUGPATH="${__MsbuildDebugLogsDir}"
 121
 122     __BuildProperties="-p:OSGroup=${__BuildOS} -p:BuildOS=${__BuildOS} -p:BuildArch=${__BuildArch} -p:BuildType=${__BuildType}"
 123
 124     # =========================================================================================
 125     # ===
 126     # === Restore product binaries from packages
 127     # ===
 128     # =========================================================================================
 129
 130     build_MSBuild_projects "Restore_Packages" "${__ProjectDir}/tests/build.proj" "Restore product binaries (build tests)" "/t:BatchRestorePackages"
 131
 132     if [ -n "$__UpdateInvalidPackagesArg" ]; then
 133         __up="/t:UpdateInvalidPackageVersions"
 134     fi
 135
 136     echo "${__MsgPrefix}Creating test overlay..."
 137
 138     if [ -z "$xUnitTestBinBase" ]; then
 139         xUnitTestBinBase=$__TestWorkingDir
 140     fi
 141
 142     export CORE_ROOT=$xUnitTestBinBase/Tests/Core_Root
 143
 144     if [ -d "${CORE_ROOT}" ]; then
 145         rm -rf $CORE_ROOT
 146     fi
 147
 148     mkdir -p $CORE_ROOT
 149
 150     build_MSBuild_projects "Tests_Overlay_Managed" "${__ProjectDir}/tests/runtest.proj" "Creating test overlay" "/t:CreateTestOverlay"
 151
 152     chmod +x $__BinDir/corerun
 153
 154     # Make sure to copy over the pulled down packages
 155     cp -r $__BinDir/* $CORE_ROOT/ > /dev/null
 156
 157     if [ $__SkipStressDependencies == 0 ]; then
 158         if [ "$__BuildOS" != "OSX" ]; then
 159             nextCommand="\"$__TestDir/setup-stress-dependencies.sh\" --arch=$__BuildArch --outputDir=$CORE_ROOT"
 160             echo "Resolve runtime dependences via $nextCommand"
 161             eval $nextCommand
 162             if [ $? != 0 ]; then
 163                 echo "${__MsgPrefix}Error: setup-stress-dependencies failed."
 164                 exit 1
 165             fi
 166         fi
 167     fi
 168
 169     # Precompile framework assemblies with crossgen if required
 170     if [ $__DoCrossgen -ne 0 ]; then
 171         chmod +x $__CrossgenExe
 172         precompile_coreroot_fx
 173     fi
 174 }
 175
 176 precompile_coreroot_fx()
 177 {
 178     echo "${__MsgPrefix}Running crossgen on framework assemblies in CORE_ROOT: '${CORE_ROOT}'"
 179
 180     # Read the exclusion file for this platform
 181     skipCrossGenFiles=($(read_array "$(dirname "$0")/tests/skipCrossGenFiles.${__BuildArch}.txt"))
 182
 183     local overlayDir=$CORE_ROOT
 184
 185     filesToPrecompile=$(find -L $overlayDir -iname \*.dll -not -iname \*.ni.dll -not -iname \*-ms-win-\* -not -iname xunit.\* -type f)
 186     for fileToPrecompile in ${filesToPrecompile}
 187     do
 188         local filename=${fileToPrecompile}
 189         if is_skip_crossgen_test "$(basename $filename)"; then
 190                 continue
 191         fi
 192         echo Precompiling $filename
 193         $__CrossgenExe /Platform_Assemblies_Paths $overlayDir $filename 1> $filename.stdout 2>$filename.stderr
 194         local exitCode=$?
 195         if [[ $exitCode != 0 ]]; then
 196             if grep -q -e '0x80131018' $filename.stderr; then
 197                 printf "\n\t$filename is not a managed assembly.\n\n"
 198             else
 199                 echo Unable to precompile $filename.
 200                 cat $filename.stdout
 201                 cat $filename.stderr
 202                 exit $exitCode
 203             fi
 204         else
 205             rm $filename.{stdout,stderr}
 206         fi
 207     done
 208 }
 209
 210 declare -a skipCrossGenFiles
 211
 212 function is_skip_crossgen_test {
 213     for skip in "${skipCrossGenFiles[@]}"; do
 214         if [ "$1" == "$skip" ]; then
 215             return 0
 216         fi
 217     done
 218     return 1
 219 }
 220
 221 # Get an array of items by reading the specified file line by line.
 222 function read_array {
 223     local theArray=()
 224
 225     if [ ! -f "$1" ]; then
 226         return
 227     fi
 228
 229     # bash in Mac OS X doesn't support 'readarray', so using alternate way instead.
 230     # readarray -t theArray < "$1"
 231     # Any line that starts with '#' is ignored.
 232     while IFS='' read -r line || [ -n "$line" ]; do
 233         if [[ $line != "#"* ]]; then
 234             theArray[${#theArray[@]}]=$line
 235         fi
 236     done < "$1"
 237     echo ${theArray[@]}
 238 }
 239
 240 generate_testhost()
 241 {
 242     echo "${__MsgPrefix}Generating test host..."
 243
 244     export TEST_HOST=$xUnitTestBinBase/testhost
 245
 246     if [ -d "${TEST_HOST}" ]; then
 247         rm -rf $TEST_HOST
 248     fi
 249
 250     mkdir -p $TEST_HOST
 251
 252     build_MSBuild_projects "Tests_Generate_TestHost" "${__ProjectDir}/tests/runtest.proj" "Creating test host" "/t:CreateTestHost"
 253 }
 254
 255
 256 build_Tests()
 257 {
 258     echo "${__MsgPrefix}Building Tests..."
 259
 260     __TestDir=$__ProjectDir/tests
 261     __ProjectFilesDir=$__TestDir
 262     __TestBinDir=$__TestWorkingDir
 263
 264     if [ -f  "${__TestWorkingDir}/build_info.json" ]; then
 265         rm  "${__TestWorkingDir}/build_info.json"
 266     fi
 267
 268     if [ $__RebuildTests -ne 0 ]; then
 269         if [ -d "${__TestBinDir}" ]; then
 270             echo "Removing tests build dir: ${__TestBinDir}"
 271             rm -rf $__TestBinDir
 272         fi
 273     fi
 274
 275     export __CMakeBinDir="${__TestBinDir}"
 276     if [ ! -d "${__TestIntermediatesDir}" ]; then
 277         mkdir -p ${__TestIntermediatesDir}
 278     fi
 279
 280     __NativeTestIntermediatesDir="${__TestIntermediatesDir}/Native"
 281     if [  ! -d "${__NativeTestIntermediatesDir}" ]; then
 282         mkdir -p ${__NativeTestIntermediatesDir}
 283     fi
 284
 285     __ManagedTestIntermediatesDir="${__TestIntermediatesDir}/Managed"
 286     if [ ! -d "${__ManagedTestIntermediatesDir}" ]; then
 287         mkdir -p ${__ManagedTestIntermediatesDir}
 288     fi
 289
 290     echo "__BuildOS: ${__BuildOS}"
 291     echo "__BuildArch: ${__BuildArch}"
 292     echo "__BuildType: ${__BuildType}"
 293     echo "__TestIntermediatesDir: ${__TestIntermediatesDir}"
 294     echo "__NativeTestIntermediatesDir: ${__NativeTestIntermediatesDir}"
 295     echo "__ManagedTestIntermediatesDir: ${__ManagedTestIntermediatesDir}"
 296
 297     if [ ! -f "$__TestBinDir" ]; then
 298         echo "Creating TestBinDir: ${__TestBinDir}"
 299         mkdir -p $__TestBinDir
 300     fi
 301     if [ ! -f "$__LogsDir" ]; then
 302         echo "Creating LogsDir: ${__LogsDir}"
 303         mkdir -p $__LogsDir
 304     fi
 305     if [ ! -f "$__MsbuildDebugLogsDir" ]; then
 306         echo "Creating MsbuildDebugLogsDir: ${__MsbuildDebugLogsDir}"
 307         mkdir -p $__MsbuildDebugLogsDir
 308     fi
 309
 310     # Set up the directory for MSBuild debug logs.
 311     export MSBUILDDEBUGPATH="${__MsbuildDebugLogsDir}"
 312
 313     __BuildProperties="-p:OSGroup=${__BuildOS} -p:BuildOS=${__BuildOS} -p:BuildArch=${__BuildArch} -p:BuildType=${__BuildType}"
 314
 315     # =========================================================================================
 316     # ===
 317     # === Restore product binaries from packages
 318     # ===
 319     # =========================================================================================
 320
 321     if [ ${__SkipRestorePackages} != 1 ]; then
 322         build_MSBuild_projects "Restore_Product" "${__ProjectDir}/tests/build.proj" "Restore product binaries (build tests)" "/t:BatchRestorePackages"
 323
 324         if [ $? -ne 0 ]; then
 325             echo "${__ErrMsgPrefix}${__MsgPrefix}Error: package restoration failed. Refer to the build log files for details (above)"
 326             exit 1
 327         fi
 328     fi
 329
 330     if [ $__SkipNative != 1 ]; then
 331         build_native_projects "$__BuildArch" "${__NativeTestIntermediatesDir}"
 332
 333         if [ $? -ne 0 ]; then
 334             echo "${__MsgPrefix}Error: build failed. Refer to the build log files for details (above)"
 335             exit 1
 336         fi
 337     fi
 338
 339     if [ $__SkipManaged != 1 ]; then
 340         echo "Starting the Managed Tests Build..."
 341
 342         build_MSBuild_projects "Tests_Managed" "$__ProjectDir/tests/build.proj" "Managed tests build (build tests)" "$__up"
 343
 344         if [ $? -ne 0 ]; then
 345             echo "${__MsgPrefix}Error: build failed. Refer to the build log files for details (above)"
 346             exit 1
 347         else
 348             echo "Checking the Managed Tests Build..."
 349
 350             build_MSBuild_projects "Check_Test_Build" "${__ProjectDir}/tests/runtest.proj" "Check Test Build" "/t:CheckTestBuild"
 351
 352             if [ $? -ne 0 ]; then
 353                 echo "${__MsgPrefix}Error: Check Test Build failed."
 354                 exit 1
 355             fi
 356         fi
 357
 358         echo "Managed tests build success!"
 359     fi
 360
 361     build_test_wrappers
 362
 363     if [ -n "$__UpdateInvalidPackagesArg" ]; then
 364         __up="/t:UpdateInvalidPackageVersions"
 365     fi
 366
 367     if [[ "$__SkipGenerateLayout" != 1 ]]; then
 368         generate_layout
 369     fi
 370 }
 371
 372 build_MSBuild_projects()
 373 {
 374     subDirectoryName=$1
 375     shift
 376     projectName=$1
 377     shift
 378     stepName="$1"
 379     shift
 380     extraBuildParameters=("$@")
 381
 382     # Set up directories and file names
 383     __BuildLogRootName=$subDirectoryName
 384     __BuildLog="$__LogsDir/${__BuildLogRootName}.${__BuildOS}.${__BuildArch}.${__BuildType}.log"
 385     __BuildWrn="$__LogsDir/${__BuildLogRootName}.${__BuildOS}.${__BuildArch}.${__BuildType}.wrn"
 386     __BuildErr="$__LogsDir/${__BuildLogRootName}.${__BuildOS}.${__BuildArch}.${__BuildType}.err"
 387
 388     # Use binclashlogger by default if no other logger is specified
 389     if [[ "${extraBuildParameters[*]}" == *"/l:"* ]]; then
 390         __msbuildEventLogging=
 391     else
 392         __msbuildEventLogging="/l:BinClashLogger,Tools/Microsoft.DotNet.Build.Tasks.dll\;LogFile=binclash.log"
 393     fi
 394
 395     if [[ "$subDirectoryName" == "Tests_Managed" ]]; then
 396         # Execute msbuild managed test build in stages - workaround for excessive data retention in MSBuild ConfigCache
 397         # See https://github.com/Microsoft/msbuild/issues/2993
 398
 399         # __SkipPackageRestore and __SkipTargetingPackBuild used  to control build by tests/src/dirs.proj
 400         export __SkipPackageRestore=false
 401         export __SkipTargetingPackBuild=false
 402         export __NumberOfTestGroups=3
 403
 404         __AppendToLog=false
 405
 406         if [ -n "$__priority1" ]; then
 407             export __NumberOfTestGroups=10
 408         fi
 409
 410         for (( testGroupToBuild=1 ; testGroupToBuild <= __NumberOfTestGroups; testGroupToBuild = testGroupToBuild + 1 ))
 411         do
 412             __msbuildLog="\"/flp:Verbosity=normal;LogFile=${__BuildLog};Append=${__AppendToLog}\""
 413             __msbuildWrn="\"/flp1:WarningsOnly;LogFile=${__BuildWrn};Append=${__AppendToLog}\""
 414             __msbuildErr="\"/flp2:ErrorsOnly;LogFile=${__BuildErr};Append=${__AppendToLog}\""
 415
 416             export __TestGroupToBuild=$testGroupToBuild
 417
 418             # Generate build command
 419             buildArgs=("/nologo" "/verbosity:minimal" "/clp:Summary")
 420             buildArgs+=("/p:RestoreDefaultOptimizationDataPackage=false" "/p:PortableBuild=true")
 421             buildArgs+=("/p:UsePartialNGENOptimization=false" "/maxcpucount")
 422
 423             buildArgs+=("$projectName" "${__msbuildLog}" "${__msbuildWrn}" "${__msbuildErr}")
 424             buildArgs+=("$__msbuildEventLogging")
 425             buildArgs+=("${extraBuildParameters[@]}")
 426             buildArgs+=("${__CommonMSBuildArgs[@]}")
 427             buildArgs+=("${__UnprocessedBuildArgs[@]}")
 428             buildArgs+=("/p:__SkipPackageRestore=true");
 429
 430             nextCommand="\"$__ProjectRoot/dotnet.sh\" msbuild ${buildArgs[@]}"
 431             echo "Building step '$stepName' testGroupToBuild=$testGroupToBuild via $nextCommand"
 432             eval $nextCommand
 433
 434             # Make sure everything is OK
 435             if [ $? -ne 0 ]; then
 436                 echo "${__MsgPrefix}Failed to build $stepName. See the build logs:"
 437                 echo "    $__BuildLog"
 438                 echo "    $__BuildWrn"
 439                 echo "    $__BuildErr"
 440                 exit 1
 441             fi
 442             export __SkipPackageRestore=true
 443             export __SkipTargetingPackBuild=true
 444             __AppendToLog=true
 445         done
 446     else
 447         __msbuildLog="\"/flp:Verbosity=normal;LogFile=${__BuildLog}\""
 448         __msbuildWrn="\"/flp1:WarningsOnly;LogFile=${__BuildWrn}\""
 449         __msbuildErr="\"/flp2:ErrorsOnly;LogFile=${__BuildErr}\""
 450
 451         # Generate build command
 452         buildArgs=("/nologo" "/verbosity:minimal" "/clp:Summary")
 453         buildArgs+=("/p:RestoreDefaultOptimizationDataPackage=false" "/p:PortableBuild=true")
 454         buildArgs+=("/p:UsePartialNGENOptimization=false" "/maxcpucount")
 455
 456         buildArgs+=("$projectName" "${__msbuildLog}" "${__msbuildWrn}" "${__msbuildErr}")
 457         buildArgs+=("$__msbuildEventLogging")
 458         buildArgs+=("${extraBuildParameters[@]}")
 459         buildArgs+=("${__CommonMSBuildArgs[@]}")
 460         buildArgs+=("${__UnprocessedBuildArgs[@]}")
 461
 462         nextCommand="\"$__ProjectRoot/dotnet.sh\" msbuild ${buildArgs[@]}"
 463         echo "Building step '$stepName' via $nextCommand"
 464         eval $nextCommand
 465
 466         # Make sure everything is OK
 467         if [ $? -ne 0 ]; then
 468             echo "${__MsgPrefix}Failed to build $stepName. See the build logs:"
 469             echo "    $__BuildLog"
 470             echo "    $__BuildWrn"
 471             echo "    $__BuildErr"
 472             exit 1
 473         fi
 474     fi
 475 }
 476
 477 build_native_projects()
 478 {
 479     platformArch="$1"
 480     intermediatesForBuild="$2"
 481
 482     extraCmakeArguments="-DCLR_CMAKE_TARGET_OS=${__BuildOS} -DCLR_CMAKE_HOST_ARCH=${platformArch}"
 483     message="native tests assets"
 484
 485     # All set to commence the build
 486     echo "Commencing build of $message for $__BuildOS.$__BuildArch.$__BuildType in $intermediatesForBuild"
 487
 488     generator=""
 489     buildFile="Makefile"
 490     buildTool="make"
 491     if [ $__UseNinja == 1 ]; then
 492         generator="ninja"
 493         buildFile="build.ninja"
 494         if ! buildTool=$(command -v ninja || command -v ninja-build); then
 495            echo "Unable to locate ninja!" 1>&2
 496            exit 1
 497         fi
 498     fi
 499
 500     if [ $__SkipConfigure == 0 ]; then
 501         # if msbuild is not supported, then set __SkipGenerateVersion to 1
 502         if [ $__isMSBuildOnNETCoreSupported == 0 ]; then __SkipGenerateVersion=1; fi
 503         # Drop version.c file
 504         __versionSourceFile="$intermediatesForBuild/version.c"
 505         if [ $__SkipGenerateVersion == 0 ]; then
 506             pwd
 507             $__ProjectRoot/eng/common/msbuild.sh $__ProjectRoot/eng/empty.csproj \
 508                                                  /p:NativeVersionFile=$__versionSourceFile \
 509                                                  /p:ArcadeBuild=true /t:GenerateNativeVersionFile /restore \
 510                                                  $__CommonMSBuildArgs $__UnprocessedBuildArgs
 511             if [ $? -ne 0 ]; then
 512                 echo "Failed to generate native version file."
 513                 exit $?
 514             fi
 515         else
 516             # Generate the dummy version.c, but only if it didn't exist to make sure we don't trigger unnecessary rebuild
 517             __versionSourceLine="static char sccsid[] __attribute__((used)) = \"@(#)No version information produced\";"
 518             if [ -e $__versionSourceFile ]; then
 519                 read existingVersionSourceLine < $__versionSourceFile
 520             fi
 521             if [ "$__versionSourceLine" != "$existingVersionSourceLine" ]; then
 522                 echo $__versionSourceLine > $__versionSourceFile
 523             fi
 524         fi
 525
 526         pushd "$intermediatesForBuild"
 527         # Regenerate the CMake solution
 528         # Force cross dir to point to project root cross dir, in case there is a cross build.
 529         scriptDir="$__ProjectRoot/src/pal/tools"
 530         if [[ $__GccBuild == 0 ]]; then
 531             nextCommand="CONFIG_DIR=\"$__ProjectRoot/cross\" \"$scriptDir/gen-buildsys-clang.sh\" \"$__TestDir\" $__ClangMajorVersion \"$__ClangMinorVersion\" $platformArch $scriptDir $__BuildType $__CodeCoverage $generator $extraCmakeArguments $__cmakeargs"
 532         else
 533             nextCommand="CONFIG_DIR=\"$__ProjectRoot/cross\" \"$scriptDir/gen-buildsys-gcc.sh\" \"$__TestDir\" \"$__GccMajorVersion\" \"$__GccMinorVersion\" $platformArch $scriptDir $__BuildType $__CodeCoverage $generator $extraCmakeArguments $__cmakeargs"
 534         fi
 535         echo "Invoking $nextCommand"
 536         eval $nextCommand
 537         popd
 538     fi
 539
 540     if [ ! -f "$intermediatesForBuild/$buildFile" ]; then
 541         echo "Failed to generate $message build project!"
 542         exit 1
 543     fi
 544
 545     # Build
 546     if [ $__ConfigureOnly == 1 ]; then
 547         echo "Finish configuration & skipping $message build."
 548         return
 549     fi
 550
 551     pushd "$intermediatesForBuild"
 552
 553     echo "Executing $buildTool install -j $__NumProc"
 554
 555     $buildTool install -j $__NumProc
 556     if [ $? != 0 ]; then
 557         echo "Failed to build $message."
 558         exit 1
 559     fi
 560
 561     popd
 562     echo "Native tests build success!"
 563 }
 564
 565 usage()
 566 {
 567     echo "Usage: $0 [BuildArch] [BuildType] [verbose] [coverage] [cross] [clangx.y] [ninja] [runtests] [bindir]"
 568     echo "BuildArch can be: x64, x86, arm, armel, arm64"
 569     echo "BuildType can be: debug, checked, release"
 570     echo "coverage - optional argument to enable code coverage build (currently supported only for Linux and OSX)."
 571     echo "ninja - target ninja instead of GNU make"
 572     echo "clangx.y - optional argument to build using clang version x.y - supported version 3.5 - 6.0"
 573     echo "gccx.y - optional argument to build using gcc version x.y."
 574     echo "cross - optional argument to signify cross compilation,"
 575     echo "      - will use ROOTFS_DIR environment variable if set."
 576     echo "portableLinux - build for Portable Linux Distribution"
 577     echo "portablebuild - Use portable build."
 578     echo "verbose - optional argument to enable verbose build output."
 579     echo "rebuild - if tests have already been built - rebuild them"
 580     echo "skipnative: skip the native tests build"
 581     echo "skipmanaged: skip the managed section of the test build"
 582     echo "buildtestwrappersonly - only build the test wrappers"
 583     echo "generatelayoutonly - only pull down dependencies and build coreroot"
 584     echo "generatetesthostonly - only pull down dependencies and build coreroot and the CoreFX testhost"
 585     echo "skiprestorepackages - skip package restore"
 586     echo "crossgen - Precompiles the framework managed assemblies in coreroot"
 587     echo "runtests - run tests after building them"
 588     echo "bindir - output directory (defaults to $__ProjectRoot/bin)"
 589     echo "msbuildonunsupportedplatform - build managed binaries even if distro is not officially supported."
 590     echo "priority1 - include priority=1 tests in the build"
 591     echo "skipgenerateversion - disable version generation even if MSBuild is supported."
 592     echo "skipstressdependencies - disable build of libcoredistools.so for x64."
 593     echo "skipgeneratelayout - Do not generate the Core_Root layout."
 594     exit 1
 595 }
 596
 597
 598 # Obtain the location of the bash script to figure out where the root of the repo is.
 599 __ProjectRoot="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
 600
 601 # Use uname to determine what the CPU is.
 602 CPUName=$(uname -p)
 603
 604 # Some Linux platforms report unknown for platform, but the arch for machine.
 605 if [ "$CPUName" == "unknown" ]; then
 606     CPUName=$(uname -m)
 607 fi
 608
 609 case $CPUName in
 610     i686)
 611         echo "Unsupported CPU $CPUName detected, build might not succeed!"
 612         __BuildArch=x86
 613         __HostArch=x86
 614         ;;
 615
 616     x86_64)
 617         __BuildArch=x64
 618         __HostArch=x64
 619         ;;
 620
 621     armv7l)
 622         echo "Unsupported CPU $CPUName detected, build might not succeed!"
 623         __BuildArch=arm
 624         __HostArch=arm
 625         ;;
 626
 627     aarch64)
 628         __BuildArch=arm64
 629         __HostArch=arm64
 630         ;;
 631
 632     *)
 633         echo "Unknown CPU $CPUName detected, configuring as if for x64"
 634         __BuildArch=x64
 635         __HostArch=x64
 636         ;;
 637 esac
 638
 639 # Use uname to determine what the OS is.
 640 OSName=$(uname -s)
 641 case $OSName in
 642     Linux)
 643         __BuildOS=Linux
 644         __HostOS=Linux
 645         ;;
 646
 647     Darwin)
 648         __BuildOS=OSX
 649         __HostOS=OSX
 650         ;;
 651
 652     FreeBSD)
 653         __BuildOS=FreeBSD
 654         __HostOS=FreeBSD
 655         ;;
 656
 657     OpenBSD)
 658         __BuildOS=OpenBSD
 659         __HostOS=OpenBSD
 660         ;;
 661
 662     NetBSD)
 663         __BuildOS=NetBSD
 664         __HostOS=NetBSD
 665         ;;
 666
 667     SunOS)
 668         __BuildOS=SunOS
 669         __HostOS=SunOS
 670         ;;
 671
 672     *)
 673         echo "Unsupported OS $OSName detected, configuring as if for Linux"
 674         __BuildOS=Linux
 675         __HostOS=Linux
 676         ;;
 677 esac
 678
 679 __BuildType=Debug
 680 __CodeCoverage=
 681 __IncludeTests=INCLUDE_TESTS
 682
 683 # Set the various build properties here so that CMake and MSBuild can pick them up
 684 export __ProjectDir="$__ProjectRoot"
 685 __SourceDir="$__ProjectDir/src"
 686 __PackagesDir="$__ProjectDir/.packages"
 687 __RootBinDir="$__ProjectDir/bin"
 688 __BuildToolsDir="$__ProjectDir/Tools"
 689 __DotNetCli="$__ProjectDir/dotnet.sh"
 690 __UnprocessedBuildArgs=
 691 __CommonMSBuildArgs=
 692 __MSBCleanBuildArgs=
 693 __UseNinja=0
 694 __VerboseBuild=0
 695 __SkipRestore=""
 696 __SkipNative=0
 697 __SkipManaged=0
 698 __SkipConfigure=0
 699 __SkipGenerateVersion=0
 700 __SkipStressDependencies=0
 701 __SkipGenerateLayout=0
 702 __ConfigureOnly=0
 703 __CrossBuild=0
 704 __ClangMajorVersion=0
 705 __ClangMinorVersion=0
 706 __GccBuild=0
 707 __GccMajorVersion=0
 708 __GccMinorVersion=0
 709 __NuGetPath="$__PackagesDir/NuGet.exe"
 710 __SkipRestorePackages=0
 711 __DistroRid=""
 712 __cmakeargs=""
 713 __PortableLinux=0
 714 __msbuildonunsupportedplatform=0
 715 __NativeTestIntermediatesDir=
 716 __RunTests=0
 717 __RebuildTests=0
 718 __BuildTestWrappers=1
 719 __GenerateLayoutOnly=
 720 __GenerateTestHostOnly=
 721 __priority1=
 722 __BuildTestWrappersOnly=
 723 __DoCrossgen=0
 724 CORE_ROOT=
 725
 726 while :; do
 727     if [ $# -le 0 ]; then
 728         break
 729     fi
 730
 731     lowerI="$(echo $1 | awk '{print tolower($0)}')"
 732     case $lowerI in
 733         -\?|-h|--help)
 734             usage
 735             exit 1
 736             ;;
 737
 738         x86)
 739             __BuildArch=x86
 740             ;;
 741
 742         x64)
 743             __BuildArch=x64
 744             ;;
 745
 746         arm)
 747             __BuildArch=arm
 748             ;;
 749
 750         armel)
 751             __BuildArch=armel
 752             ;;
 753
 754         arm64)
 755             __BuildArch=arm64
 756             ;;
 757
 758         debug)
 759             __BuildType=Debug
 760             ;;
 761
 762         checked)
 763             __BuildType=Checked
 764             ;;
 765
 766         release)
 767             __BuildType=Release
 768             ;;
 769
 770         coverage)
 771             __CodeCoverage=Coverage
 772             ;;
 773
 774         cross)
 775             __CrossBuild=1
 776             ;;
 777
 778         portablebuild=false)
 779             __PortableBuild=0
 780             ;;
 781
 782         portablelinux)
 783             if [ "$__BuildOS" == "Linux" ]; then
 784                 __PortableLinux=1
 785             else
 786                 echo "ERROR: portableLinux not supported for non-Linux platforms."
 787                 exit 1
 788             fi
 789             ;;
 790
 791         verbose)
 792             __VerboseBuild=1
 793             ;;
 794
 795         clang3.5|-clang3.5)
 796             __ClangMajorVersion=3
 797             __ClangMinorVersion=5
 798             ;;
 799
 800         clang3.6|-clang3.6)
 801             __ClangMajorVersion=3
 802             __ClangMinorVersion=6
 803             ;;
 804
 805         clang3.7|-clang3.7)
 806             __ClangMajorVersion=3
 807             __ClangMinorVersion=7
 808             ;;
 809
 810         clang3.8|-clang3.8)
 811             __ClangMajorVersion=3
 812             __ClangMinorVersion=8
 813             ;;
 814
 815         clang3.9|-clang3.9)
 816             __ClangMajorVersion=3
 817             __ClangMinorVersion=9
 818             ;;
 819
 820         clang4.0|-clang4.0)
 821             __ClangMajorVersion=4
 822             __ClangMinorVersion=0
 823             ;;
 824
 825         clang5.0|-clang5.0)
 826             __ClangMajorVersion=5
 827             __ClangMinorVersion=0
 828             ;;
 829
 830         clang6.0|-clang6.0)
 831             __ClangMajorVersion=6
 832             __ClangMinorVersion=0
 833             ;;
 834
 835         clang7|-clang7)
 836             __ClangMajorVersion=7
 837             __ClangMinorVersion=
 838             ;;
 839
 840         clang8|-clang8)
 841             __ClangMajorVersion=8
 842             __ClangMinorVersion=
 843             ;;
 844
 845         clang9|-clang9)
 846             __ClangMajorVersion=9
 847             __ClangMinorVersion=
 848             ;;
 849
 850         clang10|-clang10)
 851             __ClangMajorVersion=10
 852             __ClangMinorVersion=
 853             ;;
 854
 855         gcc5|-gcc5)
 856             __GccMajorVersion=5
 857             __GccMinorVersion=
 858             __GccBuild=1
 859             ;;
 860
 861         gcc6|-gcc6)
 862             __GccMajorVersion=6
 863             __GccMinorVersion=
 864             __GccBuild=1
 865             ;;
 866
 867         gcc7|-gcc7)
 868             __GccMajorVersion=7
 869             __GccMinorVersion=
 870             __GccBuild=1
 871             ;;
 872
 873         gcc8|-gcc8)
 874             __GccMajorVersion=8
 875             __GccMinorVersion=
 876             __GccBuild=1
 877             ;;
 878
 879         gcc|-gcc)
 880             __GccMajorVersion=
 881             __GccMinorVersion=
 882             __GccBuild=1
 883             ;;
 884
 885         ninja)
 886             __UseNinja=1
 887             ;;
 888
 889         runtests)
 890             __RunTests=1
 891             ;;
 892
 893         rebuild)
 894             __RebuildTests=1
 895             ;;
 896
 897         skipnative|-skipnative)
 898             __SkipNative=1
 899             ;;
 900
 901         skipmanaged|-skipmanaged)
 902             __SkipManaged=1
 903             __BuildTestWrappers=0
 904             ;;
 905
 906         buildtestwrappersonly)
 907             __BuildTestWrappersOnly=1
 908             ;;
 909
 910         generatelayoutonly)
 911             __GenerateLayoutOnly=1
 912             ;;
 913
 914         generatetesthostonly)
 915             __GenerateTestHostOnly=1
 916             ;;
 917
 918         skiprestorepackages)
 919             __SkipRestorePackages=1
 920             ;;
 921
 922         crossgen)
 923             __DoCrossgen=1
 924             ;;
 925
 926         bindir)
 927             if [ -n "$2" ]; then
 928                 __RootBinDir="$2"
 929                 if [ ! -d $__RootBinDir ]; then
 930                     mkdir $__RootBinDir
 931                 fi
 932                 __RootBinParent=$(dirname $__RootBinDir)
 933                 __RootBinName=${__RootBinDir##*/}
 934                 __RootBinDir="$(cd $__RootBinParent &>/dev/null && printf %s/%s $PWD $__RootBinName)"
 935                 shift
 936             else
 937                 echo "ERROR: 'bindir' requires a non-empty option argument"
 938                 exit 1
 939             fi
 940             ;;
 941
 942         msbuildonunsupportedplatform)
 943             __msbuildonunsupportedplatform=1
 944             ;;
 945
 946         priority1)
 947             __priority1=1
 948             __UnprocessedBuildArgs+=("/p:CLRTestPriorityToBuild=1")
 949             ;;
 950
 951         skipgenerateversion)
 952             __SkipGenerateVersion=1
 953             ;;
 954
 955         skipstressdependencies)
 956             __SkipStressDependencies=1
 957             ;;
 958
 959         skipgeneratelayout)
 960             __SkipGenerateLayout=1
 961             ;;
 962
 963         *)
 964             __UnprocessedBuildArgs+=("$1")
 965             ;;
 966     esac
 967
 968     shift
 969 done
 970
 971 # Get the number of processors available to the scheduler
 972 # Other techniques such as `nproc` only get the number of
 973 # processors available to a single process.
 974 if [ `uname` = "FreeBSD" ]; then
 975   __NumProc=`sysctl hw.ncpu | awk '{ print $2+1 }'`
 976 elif [ `uname` = "NetBSD" ]; then
 977   __NumProc=$(($(getconf NPROCESSORS_ONLN)+1))
 978 elif [ `uname` = "Darwin" ]; then
 979   __NumProc=$(($(getconf _NPROCESSORS_ONLN)+1))
 980 else
 981   __NumProc=$(nproc --all)
 982 fi
 983
 984 __CommonMSBuildArgs=("/p:__BuildArch=$__BuildArch" "/p:__BuildType=$__BuildType" "/p:__BuildOS=$__BuildOS" "/nodeReuse:false")
 985
 986 # Configure environment if we are doing a verbose build
 987 if [ $__VerboseBuild == 1 ]; then
 988     export VERBOSE=1
 989     __CommonMSBuildArgs+=("/v:detailed")
 990 fi
 991
 992 # Set default clang version
 993 if [[ $__ClangMajorVersion == 0 && $__ClangMinorVersion == 0 ]]; then
 994     if [[ "$__BuildArch" == "arm" || "$__BuildArch" == "armel" ]]; then
 995         __ClangMajorVersion=5
 996         __ClangMinorVersion=0
 997     else
 998         __ClangMajorVersion=3
 999         __ClangMinorVersion=9
1000     fi
1001 fi
1002
1003 # Set dependent variables
1004 __LogsDir="$__RootBinDir/Logs"
1005 __MsbuildDebugLogsDir="$__LogsDir/MsbuildDebugLogs"
1006
1007 # Set the remaining variables based upon the determined build configuration
1008 __BinDir="$__RootBinDir/Product/$__BuildOS.$__BuildArch.$__BuildType"
1009 __PackagesBinDir="$__BinDir/.nuget"
1010 __TestDir="$__ProjectDir/tests"
1011 __TestWorkingDir="$__RootBinDir/tests/$__BuildOS.$__BuildArch.$__BuildType"
1012 __IntermediatesDir="$__RootBinDir/obj/$__BuildOS.$__BuildArch.$__BuildType"
1013 __TestIntermediatesDir="$__RootBinDir/tests/obj/$__BuildOS.$__BuildArch.$__BuildType"
1014 __isMSBuildOnNETCoreSupported=0
1015 __CrossComponentBinDir="$__BinDir"
1016 __CrossCompIntermediatesDir="$__IntermediatesDir/crossgen"
1017
1018 __CrossArch="$__HostArch"
1019 if [ $__CrossBuild == 1 ]; then
1020     __CrossComponentBinDir="$__CrossComponentBinDir/$__CrossArch"
1021 fi
1022 __CrossgenCoreLibLog="$__LogsDir/CrossgenCoreLib_$__BuildOS.$BuildArch.$__BuildType.log"
1023 __CrossgenExe="$__CrossComponentBinDir/crossgen"
1024
1025 isMSBuildOnNETCoreSupported
1026
1027 # CI_SPECIFIC - On CI machines, $HOME may not be set. In such a case, create a subfolder and set the variable to it.
1028 # This is needed by CLI to function.
1029 if [ -z "$HOME" ]; then
1030     if [ ! -d "$__ProjectDir/temp_home" ]; then
1031         mkdir temp_home
1032     fi
1033     export HOME=$__ProjectDir/temp_home
1034     echo "HOME not defined; setting it to $HOME"
1035 fi
1036
1037 # Configure environment if we are doing a cross compile.
1038 if [ $__CrossBuild == 1 ]; then
1039     export CROSSCOMPILE=1
1040     if ! [[ -n "$ROOTFS_DIR" ]]; then
1041         export ROOTFS_DIR="$__ProjectRoot/cross/rootfs/$__BuildArch"
1042     fi
1043 fi
1044
1045 # init the target distro name
1046 initTargetDistroRid
1047
1048 if [ $__PortableBuild == 0 ]; then
1049     __CommonMSBuildArgs="$__CommonMSBuildArgs /p:PortableBuild=false"
1050 fi
1051
1052 # Restore Build Tools
1053 source $__ProjectRoot/init-tools.sh
1054
1055 if [[ (-z "$__GenerateLayoutOnly") && (-z "$__GenerateTestHostOnly") && (-z "$__BuildTestWrappersOnly") ]]; then
1056     build_Tests
1057 elif [ ! -z "$__BuildTestWrappersOnly" ]; then
1058     build_test_wrappers
1059 else
1060     generate_layout
1061     if [ ! -z "$__GenerateTestHostOnly" ]; then
1062         generate_testhost
1063     fi
1064 fi
1065
1066 if [ $? -ne 0 ]; then
1067     echo "Failed to build tests"
1068     exit 1
1069 fi
1070
1071 echo "${__MsgPrefix}Test build successful."
1072 echo "${__MsgPrefix}Test binaries are available at ${__TestBinDir}"
1073
1074 __testNativeBinDir=$__IntermediatesDir/tests
1075
1076 if [ $__RunTests -ne 0 ]; then
1077
1078     echo "Run Tests..."
1079
1080     nextCommand="$__TestDir/runtest.sh --testRootDir=$__TestBinDir --coreClrBinDir=$__BinDir --coreFxBinDir=$CORE_ROOT --testNativeBinDir=$__testNativeBinDir"
1081     echo "$nextCommand"
1082     eval $nextCommand
1083
1084     echo "Tests run successful."
1085 else
1086     echo "To run all tests use 'tests/runtests.sh' where:"
1087     echo "    testRootDir      = $__TestBinDir"
1088     echo "    coreClrBinDir    = $__BinDir"
1089     echo "    coreFxBinDir     = $CORE_ROOT"
1090     echo "    testNativeBinDir = $__testNativeBinDir"
1091     echo " -------------------------------------------------- "
1092     echo " Example runtest.sh command"
1093     echo ""
1094     echo " ./tests/runtest.sh --coreOverlayDir=$CORE_ROOT --testNativeBinDir=$__testNativeBinDir --testRootDir=$__TestBinDir --copyNativeTestBin"
1095     echo " -------------------------------------------------- "
1096     echo "To run single test use the following command:"
1097     echo "    bash ${__TestBinDir}/__TEST_PATH__/__TEST_NAME__.sh -coreroot=${CORE_ROOT}"
1098 fi
1099