build-test.sh

   1 #!/usr/bin/env bash
   2
   3 initHostDistroRid()
   4 {
   5     if [ "$__HostOS" == "Linux" ]; then
   6         if [ ! -e /etc/os-release ]; then
   7             echo "WARNING: Can not determine runtime id for current distro."
   8             __HostDistroRid=""
   9         else
  10             source /etc/os-release
  11             __HostDistroRid="$ID.$VERSION_ID-$__HostArch"
  12         fi
  13     fi
  14 }
  15
  16 initTargetDistroRid()
  17 {
  18     if [ $__CrossBuild == 1 ]; then
  19         if [ "$__BuildOS" == "Linux" ]; then
  20             if [ ! -e $ROOTFS_DIR/etc/os-release ]; then
  21                 echo "WARNING: Can not determine runtime id for current distro."
  22                 export __DistroRid=""
  23             else
  24                 source $ROOTFS_DIR/etc/os-release
  25                 export __DistroRid="$ID.$VERSION_ID-$__BuildArch"
  26             fi
  27         fi
  28     else
  29         export __DistroRid="$__HostDistroRid"
  30     fi
  31
  32     if [ "$__BuildOS" == "OSX" ]; then
  33         __PortableBuild=1
  34     fi
  35
  36     # Portable builds target the base RID
  37     if [ "$__PortableBuild" == 1 ]; then
  38         if [ "$__BuildOS" == "Linux" ]; then
  39             export __DistroRid="linux-$__BuildArch"
  40         elif [ "$__BuildOS" == "OSX" ]; then
  41             export __DistroRid="osx-$__BuildArch"
  42         fi
  43     fi
  44
  45    if [ "$ID.$VERSION_ID" == "ubuntu.16.04" ]; then
  46      export __DistroRid="ubuntu.14.04-$__BuildArch"
  47    fi
  48
  49    echo "__DistroRid: " $__DistroRid
  50 }
  51
  52 isMSBuildOnNETCoreSupported()
  53 {
  54     __isMSBuildOnNETCoreSupported=$__msbuildonunsupportedplatform
  55
  56     if [ $__isMSBuildOnNETCoreSupported == 1 ]; then
  57         return
  58     fi
  59
  60     if [ "$__HostArch" == "x64" ]; then
  61         if [ "$__HostOS" == "Linux" ]; then
  62             __isMSBuildOnNETCoreSupported=1
  63             UNSUPPORTED_RIDS=("debian.9-x64" "ubuntu.17.04-x64")
  64             for UNSUPPORTED_RID in "${UNSUPPORTED_RIDS[@]}"
  65             do
  66                 if [ "$__HostDistroRid" == "$UNSUPPORTED_RID" ]; then
  67                     __isMSBuildOnNETCoreSupported=0
  68                     break
  69                 fi
  70             done
  71         elif [ "$__HostOS" == "OSX" ]; then
  72             __isMSBuildOnNETCoreSupported=1
  73         fi
  74     fi
  75 }
  76
  77 build_Tests()
  78 {
  79     __TestDir=$__ProjectDir/tests
  80     __ProjectFilesDir=$__TestDir
  81     __TestBinDir=$__TestWorkingDir
  82
  83     if [ $__RebuildTests -ne 0 ]; then
  84         if [ -d "${__TestBinDir}" ]; then
  85             echo "Removing tests build dir: ${__TestBinDir}"
  86             rm -rf $__TestBinDir
  87         fi
  88     fi
  89
  90     __CMakeBinDir="${__TestBinDir}"
  91
  92     if [ -z "$__TestIntermediateDir" ]; then
  93         __TestIntermediateDir="tests/obj/${__BuildOS}.${__BuildArch}.${__BuildType}"
  94     fi
  95
  96         echo "__BuildOS: ${__BuildOS}"
  97         echo "__BuildArch: ${__BuildArch}"
  98         echo "__BuildType: ${__BuildType}"
  99         echo "__TestIntermediateDir: ${__TestIntermediateDir}"
 100
 101     if [ ! -f "$__TestBinDir" ]; then
 102         echo "Creating TestBinDir: ${__TestBinDir}"
 103         mkdir -p $__TestBinDir
 104     fi
 105     if [ ! -f "$__LogsDir" ]; then
 106         echo "Creating LogsDir: ${__LogsDir}"
 107         mkdir -p $__LogsDir
 108     fi
 109
 110     __BuildProperties="-p:OSGroup=${__BuildOS} -p:BuildOS=${__BuildOS} -p:BuildArch=${__BuildArch} -p:BuildType=${__BuildType}"
 111
 112     # =========================================================================================
 113     # ===
 114     # === Restore product binaries from packages
 115     # ===
 116     # =========================================================================================
 117
 118     build_Tests_internal "Restore_Product" "${__ProjectDir}/tests/build.proj" " -BatchRestorePackages" "Restore product binaries (build tests)"
 119
 120     build_Tests_internal "Tests_GenerateRuntimeLayout" "${__ProjectDir}/tests/runtest.proj" "-BinPlaceRef -BinPlaceProduct -CopyCrossgenToProduct" "Restore product binaries (run tests)"
 121
 122     if [ -n "$__UpdateInvalidPackagesArg" ]; then
 123         __up=-updateinvalidpackageversion
 124     fi
 125
 126     # Work hardcoded path around
 127     if [ ! -f "${__BuildToolsDir}/Microsoft.CSharp.Core.Targets" ]; then
 128         ln -s "${__BuildToolsDir}/Microsoft.CSharp.Core.targets" "${__BuildToolsDir}/Microsoft.CSharp.Core.Targets"
 129     fi
 130     if [ ! -f "${__BuildToolsDir}/Microsoft.CSharp.targets" ]; then
 131         ln -s "${__BuildToolsDir}/Microsoft.CSharp.Targets" "${__BuildToolsDir}/Microsoft.CSharp.targets"
 132     fi
 133
 134     echo "Starting the Managed Tests Build..."
 135
 136     __ManagedTestBuiltMarker=${__TestBinDir}/managed_test_build
 137
 138     if [ ! -f $__ManagedTestBuiltMarker ]; then
 139
 140             build_Tests_internal "Tests_Managed" "$__ProjectDir/tests/build.proj" "$__up" "Managed tests build (build tests)"
 141
 142         if [ $? -ne 0 ]; then
 143             echo "${__MsgPrefix}Error: build failed. Refer to the build log files for details (above)"
 144             exit 1
 145         else
 146             echo "Tests have been built."
 147             echo "Create marker \"${__ManagedTestBuiltMarker}\""
 148             touch $__ManagedTestBuiltMarker
 149         fi
 150     else
 151         echo "Managed Tests had been built before."
 152     fi
 153
 154     if [ $__BuildTestWrappers -ne -0 ]; then
 155         echo "${__MsgPrefix}Creating test wrappers..."
 156
 157         __XUnitWrapperBuiltMarker=${__TestBinDir}/xunit_wrapper_build
 158
 159         if [ ! -f $__XUnitWrapperBuiltMarker ]; then
 160
 161             build_Tests_internal "Tests_XunitWrapper" "$__ProjectDir/tests/runtest.proj" "-BuildWrappers -MsBuildEventLogging=\" \" " "Test Xunit Wrapper"
 162
 163             if [ $? -ne 0 ]; then
 164                 echo "${__MsgPrefix}Error: build failed. Refer to the build log files for details (above)"
 165                 exit 1
 166             else
 167                 echo "XUnit Wrappers have been built."
 168                 echo "Create marker \"${__XUnitWrapperBuiltMarker}\""
 169                 touch $__XUnitWrapperBuiltMarker
 170             fi
 171         else
 172             echo "XUnit Wrappers had been built before."
 173         fi
 174     fi
 175
 176     echo "${__MsgPrefix}Creating test overlay..."
 177
 178     if [ -z "$XuintTestBinBase" ]; then
 179       XuintTestBinBase=$__TestWorkingDir
 180     fi
 181
 182     export CORE_ROOT=$XuintTestBinBase/Tests/Core_Root
 183
 184     if [ ! -f "${CORE_ROOT}" ]; then
 185       mkdir -p $CORE_ROOT
 186     else
 187       rm -rf $CORE_ROOT/*
 188     fi
 189
 190     cp -r $__BinDir/* $CORE_ROOT/ > /dev/null
 191
 192     build_Tests_internal "Tests_Overlay_Managed" "$__ProjectDir/tests/runtest.proj" "-testOverlay" "Creating test overlay"
 193
 194     if [ $__ZipTests -ne 0 ]; then
 195         echo "${__MsgPrefix}ZIP tests packages..."
 196         build_Tests_internal "Helix_Prep" "$__ProjectDir/tests/helixprep.proj" " " "Prep test binaries for Helix publishing"
 197     fi
 198 }
 199
 200 build_Tests_internal()
 201 {
 202         subDirectoryName=$1
 203         projectName=$2
 204         extraBuildParameters=$3
 205         stepName="$4"
 206
 207         # Set up directories and file names
 208         __BuildLogRootName=$subDirectoryName
 209     __BuildLog="$__LogsDir/${__BuildLogRootName}.${__BuildOS}.${__BuildArch}.${__BuildType}.log"
 210     __BuildWrn="$__LogsDir/${__BuildLogRootName}.${__BuildOS}.${__BuildArch}.${__BuildType}.wrn"
 211     __BuildErr="$__LogsDir/${__BuildLogRootName}.${__BuildOS}.${__BuildArch}.${__BuildType}.err"
 212     __msbuildLog="\"/flp:Verbosity=normal;LogFile=${__BuildLog}\""
 213     __msbuildWrn="\"/flp1:WarningsOnly;LogFile=${__BuildWrn}\""
 214     __msbuildErr="\"/flp2:ErrorsOnly;LogFile=${__BuildErr}\""
 215
 216     # Generate build command
 217     buildCommand="$__ProjectRoot/run.sh build -Project=$projectName -MsBuildLog=${__msbuildLog} -MsBuildWrn=${__msbuildWrn} -MsBuildErr=${__msbuildErr} $extraBuildParameters $__RunArgs $__UnprocessedBuildArgs"
 218
 219     echo "Building step '$stepName' via $buildCommand"
 220
 221     # Invoke MSBuild
 222     eval $buildCommand
 223
 224     # Invoke MSBuild
 225     # $__ProjectRoot/run.sh build -Project=$projectName -MsBuildLog="$__msbuildLog" -MsBuildWrn="$__msbuildWrn" -MsBuildErr="$__msbuildErr" $extraBuildParameters $__RunArgs $__UnprocessedBuildArgs
 226
 227     # Make sure everything is OK
 228     if [ $? -ne 0 ]; then
 229         echo "${__MsgPrefix}Failed to build $stepName. See the build logs:"
 230         echo "    $__BuildLog"
 231         echo "    $__BuildWrn"
 232         echo "    $__BuildErr"
 233         exit 1
 234     fi
 235 }
 236
 237 usage()
 238 {
 239     echo "Usage: $0 [BuildArch] [BuildType] [verbose] [coverage] [cross] [clangx.y] [ninja] [runtests] [bindir]"
 240     echo "BuildArch can be: x64, x86, arm, armel, arm64"
 241     echo "BuildType can be: debug, checked, release"
 242     echo "coverage - optional argument to enable code coverage build (currently supported only for Linux and OSX)."
 243     echo "ninja - target ninja instead of GNU make"
 244     echo "clangx.y - optional argument to build using clang version x.y."
 245     echo "cross - optional argument to signify cross compilation,"
 246     echo "      - will use ROOTFS_DIR environment variable if set."
 247     echo "crosscomponent - optional argument to build cross-architecture component,"
 248     echo "               - will use CAC_ROOTFS_DIR environment variable if set."
 249     echo "portableLinux - build for Portable Linux Distribution"
 250     echo "verbose - optional argument to enable verbose build output."
 251     echo "rebuild - if tests have already been built - rebuild them"
 252     echo "runtests - run tests after building them"
 253     echo "ziptests - zips CoreCLR tests & Core_Root for a Helix run"
 254     echo "bindir - output directory (defaults to $__ProjectRoot/bin)"
 255     echo "msbuildonunsupportedplatform - build managed binaries even if distro is not officially supported."
 256     exit 1
 257 }
 258
 259
 260 # Obtain the location of the bash script to figure out where the root of the repo is.
 261 __ProjectRoot="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
 262
 263 # $__ProjectRoot/build.sh $1 $2
 264
 265 # Use uname to determine what the CPU is.
 266 CPUName=$(uname -p)
 267
 268 # Some Linux platforms report unknown for platform, but the arch for machine.
 269 if [ "$CPUName" == "unknown" ]; then
 270     CPUName=$(uname -m)
 271 fi
 272
 273 case $CPUName in
 274     i686)
 275         echo "Unsupported CPU $CPUName detected, build might not succeed!"
 276         __BuildArch=x86
 277         __HostArch=x86
 278         ;;
 279
 280     x86_64)
 281         __BuildArch=x64
 282         __HostArch=x64
 283         ;;
 284
 285     armv7l)
 286         echo "Unsupported CPU $CPUName detected, build might not succeed!"
 287         __BuildArch=arm
 288         __HostArch=arm
 289         ;;
 290
 291     aarch64)
 292         __BuildArch=arm64
 293         __HostArch=arm64
 294         ;;
 295
 296     *)
 297         echo "Unknown CPU $CPUName detected, configuring as if for x64"
 298         __BuildArch=x64
 299         __HostArch=x64
 300         ;;
 301 esac
 302
 303 # Use uname to determine what the OS is.
 304 OSName=$(uname -s)
 305 case $OSName in
 306     Linux)
 307         __BuildOS=Linux
 308         __HostOS=Linux
 309         ;;
 310
 311     Darwin)
 312         __BuildOS=OSX
 313         __HostOS=OSX
 314         ;;
 315
 316     FreeBSD)
 317         __BuildOS=FreeBSD
 318         __HostOS=FreeBSD
 319         ;;
 320
 321     OpenBSD)
 322         __BuildOS=OpenBSD
 323         __HostOS=OpenBSD
 324         ;;
 325
 326     NetBSD)
 327         __BuildOS=NetBSD
 328         __HostOS=NetBSD
 329         ;;
 330
 331     SunOS)
 332         __BuildOS=SunOS
 333         __HostOS=SunOS
 334         ;;
 335
 336     *)
 337         echo "Unsupported OS $OSName detected, configuring as if for Linux"
 338         __BuildOS=Linux
 339         __HostOS=Linux
 340         ;;
 341 esac
 342
 343 __BuildType=Debug
 344 __CodeCoverage=
 345 __IncludeTests=INCLUDE_TESTS
 346
 347 # Set the various build properties here so that CMake and MSBuild can pick them up
 348 export __ProjectDir="$__ProjectRoot"
 349 __SourceDir="$__ProjectDir/src"
 350 __PackagesDir="$__ProjectDir/packages"
 351 __RootBinDir="$__ProjectDir/bin"
 352 __BuildToolsDir="$__ProjectDir/Tools"
 353 __UnprocessedBuildArgs=
 354 __RunArgs=
 355 __MSBCleanBuildArgs=
 356 __UseNinja=0
 357 __VerboseBuild=0
 358 __SkipRestore=""
 359 __CrossBuild=0
 360 __ClangMajorVersion=0
 361 __ClangMinorVersion=0
 362 __NuGetPath="$__PackagesDir/NuGet.exe"
 363 __HostDistroRid=""
 364 __DistroRid=""
 365 __cmakeargs=""
 366 __PortableLinux=0
 367 __msbuildonunsupportedplatform=0
 368 __ZipTests=0
 369 __NativeTestIntermediatesDir=
 370 __RunTests=0
 371 __RebuildTests=0
 372 __BuildTestWrappers=0
 373 CORE_ROOT=
 374
 375
 376 while :; do
 377     if [ $# -le 0 ]; then
 378         break
 379     fi
 380
 381     lowerI="$(echo $1 | awk '{print tolower($0)}')"
 382     case $lowerI in
 383         -\?|-h|--help)
 384             usage
 385             exit 1
 386             ;;
 387
 388         x86)
 389             __BuildArch=x86
 390             ;;
 391
 392         x64)
 393             __BuildArch=x64
 394             ;;
 395
 396         arm)
 397             __BuildArch=arm
 398             ;;
 399
 400         armel)
 401             __BuildArch=armel
 402             ;;
 403
 404         arm64)
 405             __BuildArch=arm64
 406             ;;
 407
 408         debug)
 409             __BuildType=Debug
 410             ;;
 411
 412         checked)
 413             __BuildType=Checked
 414             ;;
 415
 416         release)
 417             __BuildType=Release
 418             ;;
 419
 420         coverage)
 421             __CodeCoverage=Coverage
 422             ;;
 423
 424         cross)
 425             __CrossBuild=1
 426             ;;
 427
 428         portablelinux)
 429             if [ "$__BuildOS" == "Linux" ]; then
 430                 __PortableLinux=1
 431             else
 432                 echo "ERROR: portableLinux not supported for non-Linux platforms."
 433                 exit 1
 434             fi
 435             ;;
 436
 437         verbose)
 438         __VerboseBuild=1
 439         ;;
 440
 441         clang3.5)
 442             __ClangMajorVersion=3
 443             __ClangMinorVersion=5
 444             ;;
 445
 446         clang3.6)
 447             __ClangMajorVersion=3
 448             __ClangMinorVersion=6
 449             ;;
 450
 451         clang3.7)
 452             __ClangMajorVersion=3
 453             __ClangMinorVersion=7
 454             ;;
 455
 456         clang3.8)
 457             __ClangMajorVersion=3
 458             __ClangMinorVersion=8
 459             ;;
 460
 461         clang3.9)
 462             __ClangMajorVersion=3
 463             __ClangMinorVersion=9
 464             ;;
 465
 466         ninja)
 467             __UseNinja=1
 468             ;;
 469
 470         runtests)
 471             __RunTests=1
 472             ;;
 473
 474         rebuild)
 475             __RebuildTests=1
 476             ;;
 477
 478         ziptests)
 479             __ZipTests=1
 480             ;;
 481
 482         bindir)
 483             if [ -n "$2" ]; then
 484                 __RootBinDir="$2"
 485                 if [ ! -d $__RootBinDir ]; then
 486                     mkdir $__RootBinDir
 487                 fi
 488                 __RootBinParent=$(dirname $__RootBinDir)
 489                 __RootBinName=${__RootBinDir##*/}
 490                 __RootBinDir="$(cd $__RootBinParent &>/dev/null && printf %s/%s $PWD $__RootBinName)"
 491                 shift
 492             else
 493                 echo "ERROR: 'bindir' requires a non-empty option argument"
 494                 exit 1
 495             fi
 496             ;;
 497
 498         msbuildonunsupportedplatform)
 499             __msbuildonunsupportedplatform=1
 500             ;;
 501         *)
 502             __UnprocessedBuildArgs="$__UnprocessedBuildArgs $1"
 503             ;;
 504     esac
 505
 506     shift
 507 done
 508
 509
 510 __RunArgs="-BuildArch=$__BuildArch -BuildType=$__BuildType -BuildOS=$__BuildOS"
 511
 512 # Configure environment if we are doing a verbose build
 513 if [ $__VerboseBuild == 1 ]; then
 514     export VERBOSE=1
 515         __RunArgs="$__RunArgs -verbose"
 516 fi
 517
 518 # Set default clang version
 519 if [[ $__ClangMajorVersion == 0 && $__ClangMinorVersion == 0 ]]; then
 520     if [ $__CrossBuild == 1 ]; then
 521         __ClangMajorVersion=3
 522         __ClangMinorVersion=6
 523     else
 524         __ClangMajorVersion=3
 525         __ClangMinorVersion=5
 526     fi
 527 fi
 528
 529
 530 # Set dependent variables
 531 __LogsDir="$__RootBinDir/Logs"
 532
 533 # init the host distro name
 534 initHostDistroRid
 535
 536 # Set the remaining variables based upon the determined build configuration
 537 __BinDir="$__RootBinDir/Product/$__BuildOS.$__BuildArch.$__BuildType"
 538 __PackagesBinDir="$__BinDir/.nuget"
 539 __ToolsDir="$__RootBinDir/tools"
 540 __TestDir="$__ProjectDir/tests"
 541 __TestWorkingDir="$__RootBinDir/tests/$__BuildOS.$__BuildArch.$__BuildType"
 542 __IntermediatesDir="$__RootBinDir/obj/$__BuildOS.$__BuildArch.$__BuildType"
 543 __TestIntermediatesDir="$__RootBinDir/tests/obj/$__BuildOS.$__BuildArch.$__BuildType"
 544 __isMSBuildOnNETCoreSupported=0
 545 __CrossComponentBinDir="$__BinDir"
 546 __CrossCompIntermediatesDir="$__IntermediatesDir/crossgen"
 547
 548 __CrossArch="$__HostArch"
 549 if [[ "$__HostArch" == "x64" && "$__BuildArch" == "arm" ]]; then
 550     __CrossArch="x86"
 551 fi
 552 if [ $__CrossBuild == 1 ]; then
 553     __CrossComponentBinDir="$__CrossComponentBinDir/$__CrossArch"
 554 fi
 555 __CrossgenCoreLibLog="$__LogsDir/CrossgenCoreLib_$__BuildOS.$BuildArch.$__BuildType.log"
 556 __CrossgenExe="$__CrossComponentBinDir/crossgen"
 557
 558 isMSBuildOnNETCoreSupported
 559
 560 # CI_SPECIFIC - On CI machines, $HOME may not be set. In such a case, create a subfolder and set the variable to set.
 561 # This is needed by CLI to function.
 562 if [ -z "$HOME" ]; then
 563     if [ ! -d "$__ProjectDir/temp_home" ]; then
 564         mkdir temp_home
 565     fi
 566     export HOME=$__ProjectDir/temp_home
 567     echo "HOME not defined; setting it to $HOME"
 568 fi
 569
 570 # Specify path to be set for CMAKE_INSTALL_PREFIX.
 571 # This is where all built CoreClr libraries will copied to.
 572 export __CMakeBinDir="$__BinDir"
 573
 574 if [ ! -d "$__BinDir" ] || [ ! -d "$__BinDir/bin" ]; then
 575
 576     echo "Has not been found built CoreCLR instance"
 577     echo "Please build it before tests using './build.sh $__BuildArch $__BuildType'"
 578     exit 1
 579 fi
 580
 581 # Configure environment if we are doing a cross compile.
 582 if [ $__CrossBuild == 1 ]; then
 583     export CROSSCOMPILE=1
 584     if ! [[ -n "$ROOTFS_DIR" ]]; then
 585         export ROOTFS_DIR="$__ProjectRoot/cross/rootfs/$__BuildArch"
 586     fi
 587 fi
 588
 589 # init the target distro name
 590 initTargetDistroRid
 591
 592 # Override tool directory
 593
 594 __CoreClrVersion=1.1.0
 595 __sharedFxDir=$__BuildToolsDir/dotnetcli/shared/Microsoft.NETCore.App/$__CoreClrVersion/
 596
 597 echo "Building Tests..."
 598
 599 build_Tests
 600
 601 if [ $? -ne 0 ]; then
 602     echo "Failed to build tests"
 603     exit 1
 604 fi
 605
 606 echo "${__MsgPrefix}Test build successful."
 607 echo "${__MsgPrefix}Test binaries are available at ${__TestBinDir}"
 608
 609 __testNativeBinDir=$__IntermediatesDir/tests
 610
 611 if [ $__RunTests -ne 0 ]; then
 612
 613     echo "Run Tests..."
 614
 615     echo "${__TestDir}/runtest.sh --testRootDir=$__TestBinDir --coreClrBinDir=$__BinDir --coreFxBinDir=$__sharedFxDir --testNativeBinDir=$__testNativeBinDir"
 616
 617     $__TestDir/runtest.sh --testRootDir=$__TestBinDir --coreClrBinDir=$__BinDir --coreFxBinDir=$CORE_ROOT --testNativeBinDir=$__testNativeBinDir
 618
 619     echo "Tests run successful."
 620 else
 621     echo "To run all tests use 'tests/runtests.sh' where:"
 622     echo "    testRootDir      = $__TestBinDir"
 623     echo "    coreClrBinDir    = $__BinDir"
 624     echo "    coreFxBinDir     = $CORE_ROOT"
 625     echo "    testNativeBinDir = $__testNativeBinDir"
 626     echo " -------------------------------------------------- "
 627     echo "To run single test use the following command:"
 628     echo "    bash ${__TestBinDir}/__TEST_PATH__/__TEST_NAME__.sh -coreroot=${CORE_ROOT}"
 629 fi
 630