build-test.sh

   1 #!/usr/bin/env bash
   2
   3 initHostDistroRid()
   4 {
   5     if [ "$__HostOS" == "Linux" ]; then
   6         if [ ! -e /etc/os-release ]; then
   7             echo "WARNING: Can not determine runtime id for current distro."
   8             __HostDistroRid=""
   9         else
  10             source /etc/os-release
  11             __HostDistroRid="$ID.$VERSION_ID-$__HostArch"
  12         fi
  13     fi
  14 }
  15
  16 initTargetDistroRid()
  17 {
  18     if [ $__CrossBuild == 1 ]; then
  19         if [ "$__BuildOS" == "Linux" ]; then
  20             if [ ! -e $ROOTFS_DIR/etc/os-release ]; then
  21                 echo "WARNING: Can not determine runtime id for current distro."
  22                 export __DistroRid=""
  23             else
  24                 source $ROOTFS_DIR/etc/os-release
  25                 export __DistroRid="$ID.$VERSION_ID-$__BuildArch"
  26             fi
  27         fi
  28     else
  29         export __DistroRid="$__HostDistroRid"
  30     fi
  31
  32     # Portable builds target the base RID
  33     if [ "$__PortableBuild" == 1 ]; then
  34         if [ "$__BuildOS" == "Linux" ]; then
  35             export __DistroRid="linux-$__BuildArch"
  36         elif [ "$__BuildOS" == "OSX" ]; then
  37             export __DistroRid="osx-$__BuildArch"
  38         fi
  39     fi
  40
  41    if [ "$ID.$VERSION_ID" == "ubuntu.16.04" ]; then
  42      export __DistroRid="ubuntu.14.04-$__BuildArch"
  43    fi
  44 }
  45
  46 isMSBuildOnNETCoreSupported()
  47 {
  48     __isMSBuildOnNETCoreSupported=$__msbuildonunsupportedplatform
  49
  50     if [ $__isMSBuildOnNETCoreSupported == 1 ]; then
  51         return
  52     fi
  53
  54     if [ "$__HostArch" == "x64" ]; then
  55         if [ "$__HostOS" == "Linux" ]; then
  56             __isMSBuildOnNETCoreSupported=1
  57             UNSUPPORTED_RIDS=("debian.9-x64" "ubuntu.17.04-x64")
  58             for UNSUPPORTED_RID in "${UNSUPPORTED_RIDS[@]}"
  59             do
  60                 if [ "$__HostDistroRid" == "$UNSUPPORTED_RID" ]; then
  61                     __isMSBuildOnNETCoreSupported=0
  62                     break
  63                 fi
  64             done
  65         elif [ "$__HostOS" == "OSX" ]; then
  66             __isMSBuildOnNETCoreSupported=1
  67         fi
  68     fi
  69 }
  70
  71 build_Tests()
  72 {
  73     __TestDir=$__ProjectDir/tests
  74     __ProjectFilesDir=$__TestDir
  75     __TestBinDir=$__TestWorkingDir
  76
  77     if [ $__RebuildTests -ne 0 ]; then
  78         if [ -d "${__TestBinDir}" ]; then
  79             echo "Removing tests build dir: ${__TestBinDir}"
  80             rm -rf $__TestBinDir
  81         fi
  82     fi
  83
  84     __CMakeBinDir="${__TestBinDir}"
  85
  86     if [ -z "$__TestIntermediateDir" ]; then
  87         __TestIntermediateDir="tests/obj/${__BuildOS}.${__BuildArch}.${__BuildType}"
  88     fi
  89
  90         echo "__BuildOS: ${__BuildOS}"
  91         echo "__BuildArch: ${__BuildArch}"
  92         echo "__BuildType: ${__BuildType}"
  93         echo "__TestIntermediateDir: ${__TestIntermediateDir}"
  94
  95     if [ ! -f "$__TestBinDir" ]; then
  96         echo "Creating TestBinDir: ${__TestBinDir}"
  97         mkdir -p $__TestBinDir
  98     fi
  99     if [ ! -f "$__LogsDir" ]; then
 100         echo "Creating LogsDir: ${__LogsDir}"
 101         mkdir -p $__LogsDir
 102     fi
 103
 104     __BuildProperties="-p:OSGroup=${__BuildOS} -p:BuildOS=${__BuildOS} -p:BuildArch=${__BuildArch} -p:BuildType=${__BuildType}"
 105
 106     # =========================================================================================
 107     # ===
 108     # === Restore product binaries from packages
 109     # ===
 110     # =========================================================================================
 111
 112     build_Tests_internal "Restore_Product" "${__ProjectDir}/tests/build.proj" " -BatchRestorePackages" "Restore product binaries (build tests)"
 113
 114     build_Tests_internal "Tests_GenerateRuntimeLayout" "${__ProjectDir}/tests/runtest.proj" "-BinPlaceRef -BinPlaceProduct -CopyCrossgenToProduct" "Restore product binaries (run tests)"
 115
 116     if [ -n "$__UpdateInvalidPackagesArg" ]; then
 117         __up=-updateinvalidpackageversion
 118     fi
 119
 120     # Work hardcoded path around
 121     if [ ! -f "${__BuildToolsDir}/Microsoft.CSharp.Core.Targets" ]; then
 122         ln -s "${__BuildToolsDir}/Microsoft.CSharp.Core.targets" "${__BuildToolsDir}/Microsoft.CSharp.Core.Targets"
 123     fi
 124     if [ ! -f "${__BuildToolsDir}/Microsoft.CSharp.targets" ]; then
 125         ln -s "${__BuildToolsDir}/Microsoft.CSharp.Targets" "${__BuildToolsDir}/Microsoft.CSharp.targets"
 126     fi
 127
 128     echo "Starting the Managed Tests Build..."
 129
 130     __ManagedTestBuiltMarker=${__TestBinDir}/managed_test_build
 131
 132     if [ ! -f $__ManagedTestBuiltMarker ]; then
 133
 134             build_Tests_internal "Tests_Managed" "$__ProjectDir/tests/build.proj" "$__up" "Managed tests build (build tests)"
 135
 136         if [ $? -ne 0 ]; then
 137             echo "${__MsgPrefix}Error: build failed. Refer to the build log files for details (above)"
 138             exit 1
 139         else
 140             echo "Tests have been built."
 141             echo "Create marker \"${__ManagedTestBuiltMarker}\""
 142             touch $__ManagedTestBuiltMarker
 143         fi
 144     else
 145         echo "Managed Tests had been built before."
 146     fi
 147
 148     if [ $__BuildTestWrappers -ne -0 ]; then
 149         echo "${__MsgPrefix}Creating test wrappers..."
 150
 151         __XUnitWrapperBuiltMarker=${__TestBinDir}/xunit_wrapper_build
 152
 153         if [ ! -f $__XUnitWrapperBuiltMarker ]; then
 154
 155             build_Tests_internal "Tests_XunitWrapper" "$__ProjectDir/tests/runtest.proj" "-BuildWrappers -MsBuildEventLogging=\" \" " "Test Xunit Wrapper"
 156
 157             if [ $? -ne 0 ]; then
 158                 echo "${__MsgPrefix}Error: build failed. Refer to the build log files for details (above)"
 159                 exit 1
 160             else
 161                 echo "XUnit Wrappers have been built."
 162                 echo "Create marker \"${__XUnitWrapperBuiltMarker}\""
 163                 touch $__XUnitWrapperBuiltMarker
 164             fi
 165         else
 166             echo "XUnit Wrappers had been built before."
 167         fi
 168     fi
 169
 170     echo "${__MsgPrefix}Creating test overlay..."
 171
 172     if [ -z "$XuintTestBinBase" ]; then
 173       XuintTestBinBase=$__TestWorkingDir
 174     fi
 175
 176     export CORE_ROOT=$XuintTestBinBase/Tests/Core_Root
 177
 178     if [ ! -f "${CORE_ROOT}" ]; then
 179       mkdir -p $CORE_ROOT
 180     else
 181       rm -rf $CORE_ROOT/*
 182     fi
 183
 184     cp -r $__BinDir/* $CORE_ROOT/ > /dev/null
 185
 186     build_Tests_internal "Tests_Overlay_Managed" "$__ProjectDir/tests/runtest.proj" "-testOverlay" "Creating test overlay"
 187
 188     if [ $__ZipTests -ne 0 ]; then
 189         echo "${__MsgPrefix}ZIP tests packages..."
 190         build_Tests_internal "Helix_Prep" "$__ProjectDir/tests/helixprep.proj" " " "Prep test binaries for Helix publishing"
 191     fi
 192 }
 193
 194 build_Tests_internal()
 195 {
 196         subDirectoryName=$1
 197         projectName=$2
 198         extraBuildParameters=$3
 199         stepName="$4"
 200
 201         # Set up directories and file names
 202         __BuildLogRootName=$subDirectoryName
 203     __BuildLog="$__LogsDir/${__BuildLogRootName}.${__BuildOS}.${__BuildArch}.${__BuildType}.log"
 204     __BuildWrn="$__LogsDir/${__BuildLogRootName}.${__BuildOS}.${__BuildArch}.${__BuildType}.wrn"
 205     __BuildErr="$__LogsDir/${__BuildLogRootName}.${__BuildOS}.${__BuildArch}.${__BuildType}.err"
 206     __msbuildLog="\"/flp:Verbosity=normal;LogFile=${__BuildLog}\""
 207     __msbuildWrn="\"/flp1:WarningsOnly;LogFile=${__BuildWrn}\""
 208     __msbuildErr="\"/flp2:ErrorsOnly;LogFile=${__BuildErr}\""
 209
 210     # Generate build command
 211     buildCommand="$__ProjectRoot/run.sh build -Project=$projectName -MsBuildLog=${__msbuildLog} -MsBuildWrn=${__msbuildWrn} -MsBuildErr=${__msbuildErr} $extraBuildParameters $__RunArgs $__UnprocessedBuildArgs"
 212
 213     echo "Building step '$stepName' via $buildCommand"
 214
 215     # Invoke MSBuild
 216     eval $buildCommand
 217
 218     # Invoke MSBuild
 219     # $__ProjectRoot/run.sh build -Project=$projectName -MsBuildLog="$__msbuildLog" -MsBuildWrn="$__msbuildWrn" -MsBuildErr="$__msbuildErr" $extraBuildParameters $__RunArgs $__UnprocessedBuildArgs
 220
 221     # Make sure everything is OK
 222     if [ $? -ne 0 ]; then
 223         echo "${__MsgPrefix}Failed to build $stepName. See the build logs:"
 224         echo "    $__BuildLog"
 225         echo "    $__BuildWrn"
 226         echo "    $__BuildErr"
 227         exit 1
 228     fi
 229 }
 230
 231 usage()
 232 {
 233     echo "Usage: $0 [BuildArch] [BuildType] [verbose] [coverage] [cross] [clangx.y] [ninja] [runtests] [bindir]"
 234     echo "BuildArch can be: x64, x86, arm, armel, arm64"
 235     echo "BuildType can be: debug, checked, release"
 236     echo "coverage - optional argument to enable code coverage build (currently supported only for Linux and OSX)."
 237     echo "ninja - target ninja instead of GNU make"
 238     echo "clangx.y - optional argument to build using clang version x.y."
 239     echo "cross - optional argument to signify cross compilation,"
 240     echo "      - will use ROOTFS_DIR environment variable if set."
 241     echo "crosscomponent - optional argument to build cross-architecture component,"
 242     echo "               - will use CAC_ROOTFS_DIR environment variable if set."
 243     echo "portableLinux - build for Portable Linux Distribution"
 244     echo "verbose - optional argument to enable verbose build output."
 245     echo "rebuild - if tests have already been built - rebuild them"
 246     echo "runtests - run tests after building them"
 247     echo "ziptests - zips CoreCLR tests & Core_Root for a Helix run"
 248     echo "bindir - output directory (defaults to $__ProjectRoot/bin)"
 249     echo "msbuildonunsupportedplatform - build managed binaries even if distro is not officially supported."
 250     exit 1
 251 }
 252
 253
 254 # Obtain the location of the bash script to figure out where the root of the repo is.
 255 __ProjectRoot="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
 256
 257 # $__ProjectRoot/build.sh $1 $2
 258
 259 # Use uname to determine what the CPU is.
 260 CPUName=$(uname -p)
 261
 262 # Some Linux platforms report unknown for platform, but the arch for machine.
 263 if [ "$CPUName" == "unknown" ]; then
 264     CPUName=$(uname -m)
 265 fi
 266
 267 case $CPUName in
 268     i686)
 269         echo "Unsupported CPU $CPUName detected, build might not succeed!"
 270         __BuildArch=x86
 271         __HostArch=x86
 272         ;;
 273
 274     x86_64)
 275         __BuildArch=x64
 276         __HostArch=x64
 277         ;;
 278
 279     armv7l)
 280         echo "Unsupported CPU $CPUName detected, build might not succeed!"
 281         __BuildArch=arm
 282         __HostArch=arm
 283         ;;
 284
 285     aarch64)
 286         __BuildArch=arm64
 287         __HostArch=arm64
 288         ;;
 289
 290     *)
 291         echo "Unknown CPU $CPUName detected, configuring as if for x64"
 292         __BuildArch=x64
 293         __HostArch=x64
 294         ;;
 295 esac
 296
 297 # Use uname to determine what the OS is.
 298 OSName=$(uname -s)
 299 case $OSName in
 300     Linux)
 301         __BuildOS=Linux
 302         __HostOS=Linux
 303         ;;
 304
 305     Darwin)
 306         __BuildOS=OSX
 307         __HostOS=OSX
 308         ;;
 309
 310     FreeBSD)
 311         __BuildOS=FreeBSD
 312         __HostOS=FreeBSD
 313         ;;
 314
 315     OpenBSD)
 316         __BuildOS=OpenBSD
 317         __HostOS=OpenBSD
 318         ;;
 319
 320     NetBSD)
 321         __BuildOS=NetBSD
 322         __HostOS=NetBSD
 323         ;;
 324
 325     SunOS)
 326         __BuildOS=SunOS
 327         __HostOS=SunOS
 328         ;;
 329
 330     *)
 331         echo "Unsupported OS $OSName detected, configuring as if for Linux"
 332         __BuildOS=Linux
 333         __HostOS=Linux
 334         ;;
 335 esac
 336
 337 __BuildType=Debug
 338 __CodeCoverage=
 339 __IncludeTests=INCLUDE_TESTS
 340
 341 # Set the various build properties here so that CMake and MSBuild can pick them up
 342 export __ProjectDir="$__ProjectRoot"
 343 __SourceDir="$__ProjectDir/src"
 344 __PackagesDir="$__ProjectDir/packages"
 345 __RootBinDir="$__ProjectDir/bin"
 346 __BuildToolsDir="$__ProjectDir/Tools"
 347 __UnprocessedBuildArgs=
 348 __RunArgs=
 349 __MSBCleanBuildArgs=
 350 __UseNinja=0
 351 __VerboseBuild=0
 352 __SkipRestore=""
 353 __CrossBuild=0
 354 __ClangMajorVersion=0
 355 __ClangMinorVersion=0
 356 __NuGetPath="$__PackagesDir/NuGet.exe"
 357 __HostDistroRid=""
 358 __DistroRid=""
 359 __cmakeargs=""
 360 __PortableLinux=0
 361 __msbuildonunsupportedplatform=0
 362 __ZipTests=0
 363 __NativeTestIntermediatesDir=
 364 __RunTests=0
 365 __RebuildTests=0
 366 __BuildTestWrappers=0
 367 CORE_ROOT=
 368
 369
 370 while :; do
 371     if [ $# -le 0 ]; then
 372         break
 373     fi
 374
 375     lowerI="$(echo $1 | awk '{print tolower($0)}')"
 376     case $lowerI in
 377         -\?|-h|--help)
 378             usage
 379             exit 1
 380             ;;
 381
 382         x86)
 383             __BuildArch=x86
 384             ;;
 385
 386         x64)
 387             __BuildArch=x64
 388             ;;
 389
 390         arm)
 391             __BuildArch=arm
 392             ;;
 393
 394         armel)
 395             __BuildArch=armel
 396             ;;
 397
 398         arm64)
 399             __BuildArch=arm64
 400             ;;
 401
 402         debug)
 403             __BuildType=Debug
 404             ;;
 405
 406         checked)
 407             __BuildType=Checked
 408             ;;
 409
 410         release)
 411             __BuildType=Release
 412             ;;
 413
 414         coverage)
 415             __CodeCoverage=Coverage
 416             ;;
 417
 418         cross)
 419             __CrossBuild=1
 420             ;;
 421
 422         portablelinux)
 423             if [ "$__BuildOS" == "Linux" ]; then
 424                 __PortableLinux=1
 425             else
 426                 echo "ERROR: portableLinux not supported for non-Linux platforms."
 427                 exit 1
 428             fi
 429             ;;
 430
 431         verbose)
 432         __VerboseBuild=1
 433         ;;
 434
 435         clang3.5)
 436             __ClangMajorVersion=3
 437             __ClangMinorVersion=5
 438             ;;
 439
 440         clang3.6)
 441             __ClangMajorVersion=3
 442             __ClangMinorVersion=6
 443             ;;
 444
 445         clang3.7)
 446             __ClangMajorVersion=3
 447             __ClangMinorVersion=7
 448             ;;
 449
 450         clang3.8)
 451             __ClangMajorVersion=3
 452             __ClangMinorVersion=8
 453             ;;
 454
 455         clang3.9)
 456             __ClangMajorVersion=3
 457             __ClangMinorVersion=9
 458             ;;
 459
 460         ninja)
 461             __UseNinja=1
 462             ;;
 463
 464         runtests)
 465             __RunTests=1
 466             ;;
 467
 468         rebuild)
 469             __RebuildTests=1
 470             ;;
 471
 472         ziptests)
 473             __ZipTests=1
 474             ;;
 475
 476         bindir)
 477             if [ -n "$2" ]; then
 478                 __RootBinDir="$2"
 479                 if [ ! -d $__RootBinDir ]; then
 480                     mkdir $__RootBinDir
 481                 fi
 482                 __RootBinParent=$(dirname $__RootBinDir)
 483                 __RootBinName=${__RootBinDir##*/}
 484                 __RootBinDir="$(cd $__RootBinParent &>/dev/null && printf %s/%s $PWD $__RootBinName)"
 485                 shift
 486             else
 487                 echo "ERROR: 'bindir' requires a non-empty option argument"
 488                 exit 1
 489             fi
 490             ;;
 491
 492         msbuildonunsupportedplatform)
 493             __msbuildonunsupportedplatform=1
 494             ;;
 495         *)
 496             __UnprocessedBuildArgs="$__UnprocessedBuildArgs $1"
 497             ;;
 498     esac
 499
 500     shift
 501 done
 502
 503
 504 __RunArgs="-BuildArch=$__BuildArch -BuildType=$__BuildType -BuildOS=$__BuildOS"
 505
 506 # Configure environment if we are doing a verbose build
 507 if [ $__VerboseBuild == 1 ]; then
 508     export VERBOSE=1
 509         __RunArgs="$__RunArgs -verbose"
 510 fi
 511
 512 # Set default clang version
 513 if [[ $__ClangMajorVersion == 0 && $__ClangMinorVersion == 0 ]]; then
 514     if [ $__CrossBuild == 1 ]; then
 515         __ClangMajorVersion=3
 516         __ClangMinorVersion=6
 517     else
 518         __ClangMajorVersion=3
 519         __ClangMinorVersion=5
 520     fi
 521 fi
 522
 523
 524 # Set dependent variables
 525 __LogsDir="$__RootBinDir/Logs"
 526
 527 # init the host distro name
 528 initHostDistroRid
 529
 530 # Set the remaining variables based upon the determined build configuration
 531 __BinDir="$__RootBinDir/Product/$__BuildOS.$__BuildArch.$__BuildType"
 532 __PackagesBinDir="$__BinDir/.nuget"
 533 __ToolsDir="$__RootBinDir/tools"
 534 __TestDir="$__ProjectDir/tests"
 535 __TestWorkingDir="$__RootBinDir/tests/$__BuildOS.$__BuildArch.$__BuildType"
 536 __IntermediatesDir="$__RootBinDir/obj/$__BuildOS.$__BuildArch.$__BuildType"
 537 __TestIntermediatesDir="$__RootBinDir/tests/obj/$__BuildOS.$__BuildArch.$__BuildType"
 538 __isMSBuildOnNETCoreSupported=0
 539 __CrossComponentBinDir="$__BinDir"
 540 __CrossCompIntermediatesDir="$__IntermediatesDir/crossgen"
 541
 542 __CrossArch="$__HostArch"
 543 if [[ "$__HostArch" == "x64" && "$__BuildArch" == "arm" ]]; then
 544     __CrossArch="x86"
 545 fi
 546 if [ $__CrossBuild == 1 ]; then
 547     __CrossComponentBinDir="$__CrossComponentBinDir/$__CrossArch"
 548 fi
 549 __CrossgenCoreLibLog="$__LogsDir/CrossgenCoreLib_$__BuildOS.$BuildArch.$__BuildType.log"
 550 __CrossgenExe="$__CrossComponentBinDir/crossgen"
 551
 552 isMSBuildOnNETCoreSupported
 553
 554 # CI_SPECIFIC - On CI machines, $HOME may not be set. In such a case, create a subfolder and set the variable to set.
 555 # This is needed by CLI to function.
 556 if [ -z "$HOME" ]; then
 557     if [ ! -d "$__ProjectDir/temp_home" ]; then
 558         mkdir temp_home
 559     fi
 560     export HOME=$__ProjectDir/temp_home
 561     echo "HOME not defined; setting it to $HOME"
 562 fi
 563
 564 # Specify path to be set for CMAKE_INSTALL_PREFIX.
 565 # This is where all built CoreClr libraries will copied to.
 566 export __CMakeBinDir="$__BinDir"
 567
 568 if [ ! -d "$__BinDir" ] || [ ! -d "$__BinDir/bin" ]; then
 569
 570     echo "Has not been found built CoreCLR instance"
 571     echo "Please build it before tests using './build.sh $__BuildArch $__BuildType'"
 572     exit 1
 573 fi
 574
 575 # Configure environment if we are doing a cross compile.
 576 if [ $__CrossBuild == 1 ]; then
 577     export CROSSCOMPILE=1
 578     if ! [[ -n "$ROOTFS_DIR" ]]; then
 579         export ROOTFS_DIR="$__ProjectRoot/cross/rootfs/$__BuildArch"
 580     fi
 581 fi
 582
 583 # init the target distro name
 584 initTargetDistroRid
 585
 586 # Override tool directory
 587
 588 __CoreClrVersion=1.1.0
 589 __sharedFxDir=$__BuildToolsDir/dotnetcli/shared/Microsoft.NETCore.App/$__CoreClrVersion/
 590
 591 echo "Building Tests..."
 592
 593 build_Tests
 594
 595 if [ $? -ne 0 ]; then
 596     echo "Failed to build tests"
 597     exit 1
 598 fi
 599
 600 echo "${__MsgPrefix}Test build successful."
 601 echo "${__MsgPrefix}Test binaries are available at ${__TestBinDir}"
 602
 603 __testNativeBinDir=$__IntermediatesDir/tests
 604
 605 if [ $__RunTests -ne 0 ]; then
 606
 607     echo "Run Tests..."
 608
 609     echo "${__TestDir}/runtest.sh --testRootDir=$__TestBinDir --coreClrBinDir=$__BinDir --coreFxBinDir=$__sharedFxDir --testNativeBinDir=$__testNativeBinDir"
 610
 611     $__TestDir/runtest.sh --testRootDir=$__TestBinDir --coreClrBinDir=$__BinDir --coreFxBinDir=$CORE_ROOT --testNativeBinDir=$__testNativeBinDir
 612
 613     echo "Tests run successful."
 614 else
 615     echo "To run all tests use 'tests/runtests.sh' where:"
 616     echo "    testRootDir      = $__TestBinDir"
 617     echo "    coreClrBinDir    = $__BinDir"
 618     echo "    coreFxBinDir     = $CORE_ROOT"
 619     echo "    testNativeBinDir = $__testNativeBinDir"
 620     echo " -------------------------------------------------- "
 621     echo "To run single test use the following command:"
 622     echo "    bash ${__TestBinDir}/__TEST_PATH__/__TEST_NAME__.sh -coreroot=${CORE_ROOT}"
 623 fi
 624