build-test.sh

   1 #!/usr/bin/env bash
   2
   3 initHostDistroRid()
   4 {
   5     if [ "$__HostOS" == "Linux" ]; then
   6         if [ ! -e /etc/os-release ]; then
   7             echo "WARNING: Can not determine runtime id for current distro."
   8             __HostDistroRid=""
   9         else
  10             source /etc/os-release
  11             __HostDistroRid="$ID.$VERSION_ID-$__HostArch"
  12         fi
  13     fi
  14 }
  15
  16 initTargetDistroRid()
  17 {
  18     if [ $__CrossBuild == 1 ]; then
  19         if [ "$__BuildOS" == "Linux" ]; then
  20             if [ ! -e $ROOTFS_DIR/etc/os-release ]; then
  21                 echo "WARNING: Can not determine runtime id for current distro."
  22                 export __DistroRid=""
  23             else
  24                 source $ROOTFS_DIR/etc/os-release
  25                 export __DistroRid="$ID.$VERSION_ID-$__BuildArch"
  26             fi
  27         fi
  28     else
  29         export __DistroRid="$__HostDistroRid"
  30     fi
  31
  32     # Portable builds target the base RID
  33     if [ "$__PortableBuild" == 1 ]; then
  34         if [ "$__BuildOS" == "Linux" ]; then
  35             export __DistroRid="linux-$__BuildArch"
  36         elif [ "$__BuildOS" == "OSX" ]; then
  37             export __DistroRid="osx-$__BuildArch"
  38         fi
  39     fi
  40 }
  41
  42 isMSBuildOnNETCoreSupported()
  43 {
  44     __isMSBuildOnNETCoreSupported=$__msbuildonunsupportedplatform
  45
  46     if [ $__isMSBuildOnNETCoreSupported == 1 ]; then
  47         return
  48     fi
  49
  50     if [ "$__HostArch" == "x64" ]; then
  51         if [ "$__HostOS" == "Linux" ]; then
  52             __isMSBuildOnNETCoreSupported=1
  53             UNSUPPORTED_RIDS=("debian.9-x64" "ubuntu.17.04-x64")
  54             for UNSUPPORTED_RID in "${UNSUPPORTED_RIDS[@]}"
  55             do
  56                 if [ "$__HostDistroRid" == "$UNSUPPORTED_RID" ]; then
  57                     __isMSBuildOnNETCoreSupported=0
  58                     break
  59                 fi
  60             done
  61         elif [ "$__HostOS" == "OSX" ]; then
  62             __isMSBuildOnNETCoreSupported=1
  63         fi
  64     fi
  65 }
  66
  67 build_Tests()
  68 {
  69     __TestDir=$__ProjectDir/tests
  70     __ProjectFilesDir=$__TestDir
  71     __TestBinDir=$__TestWorkingDir
  72
  73     if [ $__RebuildTests -ne 0 ]; then
  74         if [ -d "${__TestBinDir}" ]; then
  75             echo "Removing tests build dir: ${__TestBinDir}"
  76             rm -rf $__TestBinDir
  77         fi
  78     fi
  79
  80     __CMakeBinDir="${__TestBinDir}"
  81
  82     if [ -z "$__TestIntermediateDir" ]; then
  83         __TestIntermediateDir="tests/obj/${__BuildOS}.${__BuildArch}.${__BuildType}"
  84     fi
  85
  86         echo "__BuildOS: ${__BuildOS}"
  87         echo "__BuildArch: ${__BuildArch}"
  88         echo "__BuildType: ${__BuildType}"
  89         echo "__TestIntermediateDir: ${__TestIntermediateDir}"
  90
  91     if [ ! -f "$__TestBinDir" ]; then
  92         echo "Creating TestBinDir: ${__TestBinDir}"
  93         mkdir -p $__TestBinDir
  94     fi
  95     if [ ! -f "$__LogsDir" ]; then
  96         echo "Creating LogsDir: ${__LogsDir}"
  97         mkdir -p $__LogsDir
  98     fi
  99
 100     __BuildProperties="-p:OSGroup=${__BuildOS} -p:BuildOS=${__BuildOS} -p:BuildArch=${__BuildArch} -p:BuildType=${__BuildType}"
 101
 102     # =========================================================================================
 103     # ===
 104     # === Restore product binaries from packages
 105     # ===
 106     # =========================================================================================
 107
 108     build_Tests_internal "Restore_Product" "${__ProjectDir}/tests/build.proj" " -BatchRestorePackages" "Restore product binaries (build tests)"
 109
 110     build_Tests_internal "Tests_GenerateRuntimeLayout" "${__ProjectDir}/tests/runtest.proj" "-BinPlaceRef -BinPlaceProduct -CopyCrossgenToProduct" "Restore product binaries (run tests)"
 111
 112     if [ -n "$__UpdateInvalidPackagesArg" ]; then
 113         __up=-updateinvalidpackageversion
 114     fi
 115
 116     # Work hardcoded path around
 117     if [ ! -f "${__BuildToolsDir}/Microsoft.CSharp.Core.Targets" ]; then
 118         ln -s "${__BuildToolsDir}/Microsoft.CSharp.Core.targets" "${__BuildToolsDir}/Microsoft.CSharp.Core.Targets"
 119     fi
 120     if [ ! -f "${__BuildToolsDir}/Microsoft.CSharp.targets" ]; then
 121         ln -s "${__BuildToolsDir}/Microsoft.CSharp.Targets" "${__BuildToolsDir}/Microsoft.CSharp.targets"
 122     fi
 123
 124     echo "Starting the Managed Tests Build..."
 125
 126     __ManagedTestBuiltMarker=${__TestBinDir}/managed_test_build
 127
 128     if [ ! -f $__ManagedTestBuiltMarker ]; then
 129
 130             build_Tests_internal "Tests_Managed" "$__ProjectDir/tests/build.proj" "$__up" "Managed tests build (build tests)"
 131
 132         if [ $? -ne 0 ]; then
 133             echo "${__MsgPrefix}Error: build failed. Refer to the build log files for details (above)"
 134             exit 1
 135         else
 136             echo "Tests have been built."
 137             echo "Create marker \"${__ManagedTestBuiltMarker}\""
 138             touch $__ManagedTestBuiltMarker
 139         fi
 140     else
 141         echo "Managed Tests had been built before."
 142     fi
 143
 144     if [ $__BuildTestWrappers -ne -0 ]; then
 145         echo "${__MsgPrefix}Creating test wrappers..."
 146
 147         __XUnitWrapperBuiltMarker=${__TestBinDir}/xunit_wrapper_build
 148
 149         if [ ! -f $__XUnitWrapperBuiltMarker ]; then
 150
 151             build_Tests_internal "Tests_XunitWrapper" "$__ProjectDir/tests/runtest.proj" "-BuildWrappers -MsBuildEventLogging=\" \" " "Test Xunit Wrapper"
 152
 153             if [ $? -ne 0 ]; then
 154                 echo "${__MsgPrefix}Error: build failed. Refer to the build log files for details (above)"
 155                 exit 1
 156             else
 157                 echo "XUnit Wrappers have been built."
 158                 echo "Create marker \"${__XUnitWrapperBuiltMarker}\""
 159                 touch $__XUnitWrapperBuiltMarker
 160             fi
 161         else
 162             echo "XUnit Wrappers had been built before."
 163         fi
 164     fi
 165
 166     echo "${__MsgPrefix}Creating test overlay..."
 167
 168     if [ -z "$XuintTestBinBase" ]; then
 169       XuintTestBinBase=$__TestWorkingDir
 170     fi
 171
 172     export CORE_ROOT=$XuintTestBinBase/Tests/Core_Root
 173
 174     if [ ! -f "${CORE_ROOT}" ]; then
 175       mkdir -p $CORE_ROOT
 176     else
 177       rm -rf $CORE_ROOT/*
 178     fi
 179
 180     cp -r $__BinDir/* $CORE_ROOT/ > /dev/null
 181
 182     build_Tests_internal "Tests_Overlay_Managed" "$__ProjectDir/tests/runtest.proj" "-testOverlay" "Creating test overlay"
 183
 184     if [ $__ZipTests -ne 0 ]; then
 185         echo "${__MsgPrefix}ZIP tests packages..."
 186         build_Tests_internal "Helix_Prep" "$__ProjectDir/tests/helixprep.proj" " " "Prep test binaries for Helix publishing"
 187     fi
 188 }
 189
 190 build_Tests_internal()
 191 {
 192         subDirectoryName=$1
 193         projectName=$2
 194         extraBuildParameters=$3
 195         stepName="$4"
 196
 197         # Set up directories and file names
 198         __BuildLogRootName=$subDirectoryName
 199     __BuildLog="$__LogsDir/${__BuildLogRootName}.${__BuildOS}.${__BuildArch}.${__BuildType}.log"
 200     __BuildWrn="$__LogsDir/${__BuildLogRootName}.${__BuildOS}.${__BuildArch}.${__BuildType}.wrn"
 201     __BuildErr="$__LogsDir/${__BuildLogRootName}.${__BuildOS}.${__BuildArch}.${__BuildType}.err"
 202     __msbuildLog="\"/flp:Verbosity=normal;LogFile=${__BuildLog}\""
 203     __msbuildWrn="\"/flp1:WarningsOnly;LogFile=${__BuildWrn}\""
 204     __msbuildErr="\"/flp2:ErrorsOnly;LogFile=${__BuildErr}\""
 205
 206     # Generate build command
 207     buildCommand="$__ProjectRoot/run.sh build -Project=$projectName -MsBuildLog=${__msbuildLog} -MsBuildWrn=${__msbuildWrn} -MsBuildErr=${__msbuildErr} $extraBuildParameters $__RunArgs $__UnprocessedBuildArgs"
 208
 209     echo "Building step '$stepName' via $buildCommand"
 210
 211     # Invoke MSBuild
 212     eval $buildCommand
 213
 214     # Invoke MSBuild
 215     # $__ProjectRoot/run.sh build -Project=$projectName -MsBuildLog="$__msbuildLog" -MsBuildWrn="$__msbuildWrn" -MsBuildErr="$__msbuildErr" $extraBuildParameters $__RunArgs $__UnprocessedBuildArgs
 216
 217     # Make sure everything is OK
 218     if [ $? -ne 0 ]; then
 219         echo "${__MsgPrefix}Failed to build $stepName. See the build logs:"
 220         echo "    $__BuildLog"
 221         echo "    $__BuildWrn"
 222         echo "    $__BuildErr"
 223         exit 1
 224     fi
 225 }
 226
 227 usage()
 228 {
 229     echo "Usage: $0 [BuildArch] [BuildType] [verbose] [coverage] [cross] [clangx.y] [ninja] [runtests] [bindir]"
 230     echo "BuildArch can be: x64, x86, arm, armel, arm64"
 231     echo "BuildType can be: debug, checked, release"
 232     echo "coverage - optional argument to enable code coverage build (currently supported only for Linux and OSX)."
 233     echo "ninja - target ninja instead of GNU make"
 234     echo "clangx.y - optional argument to build using clang version x.y."
 235     echo "cross - optional argument to signify cross compilation,"
 236     echo "      - will use ROOTFS_DIR environment variable if set."
 237     echo "crosscomponent - optional argument to build cross-architecture component,"
 238     echo "               - will use CAC_ROOTFS_DIR environment variable if set."
 239     echo "portableLinux - build for Portable Linux Distribution"
 240     echo "verbose - optional argument to enable verbose build output."
 241     echo "rebuild - if tests have already been built - rebuild them"
 242     echo "runtests - run tests after building them"
 243     echo "ziptests - zips CoreCLR tests & Core_Root for a Helix run"
 244     echo "bindir - output directory (defaults to $__ProjectRoot/bin)"
 245     echo "msbuildonunsupportedplatform - build managed binaries even if distro is not officially supported."
 246     exit 1
 247 }
 248
 249
 250 # Obtain the location of the bash script to figure out where the root of the repo is.
 251 __ProjectRoot="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
 252
 253 # $__ProjectRoot/build.sh $1 $2
 254
 255 # Use uname to determine what the CPU is.
 256 CPUName=$(uname -p)
 257
 258 # Some Linux platforms report unknown for platform, but the arch for machine.
 259 if [ "$CPUName" == "unknown" ]; then
 260     CPUName=$(uname -m)
 261 fi
 262
 263 case $CPUName in
 264     i686)
 265         echo "Unsupported CPU $CPUName detected, build might not succeed!"
 266         __BuildArch=x86
 267         __HostArch=x86
 268         ;;
 269
 270     x86_64)
 271         __BuildArch=x64
 272         __HostArch=x64
 273         ;;
 274
 275     armv7l)
 276         echo "Unsupported CPU $CPUName detected, build might not succeed!"
 277         __BuildArch=arm
 278         __HostArch=arm
 279         ;;
 280
 281     aarch64)
 282         __BuildArch=arm64
 283         __HostArch=arm64
 284         ;;
 285
 286     *)
 287         echo "Unknown CPU $CPUName detected, configuring as if for x64"
 288         __BuildArch=x64
 289         __HostArch=x64
 290         ;;
 291 esac
 292
 293 # Use uname to determine what the OS is.
 294 OSName=$(uname -s)
 295 case $OSName in
 296     Linux)
 297         __BuildOS=Linux
 298         __HostOS=Linux
 299         ;;
 300
 301     Darwin)
 302         __BuildOS=OSX
 303         __HostOS=OSX
 304         ;;
 305
 306     FreeBSD)
 307         __BuildOS=FreeBSD
 308         __HostOS=FreeBSD
 309         ;;
 310
 311     OpenBSD)
 312         __BuildOS=OpenBSD
 313         __HostOS=OpenBSD
 314         ;;
 315
 316     NetBSD)
 317         __BuildOS=NetBSD
 318         __HostOS=NetBSD
 319         ;;
 320
 321     SunOS)
 322         __BuildOS=SunOS
 323         __HostOS=SunOS
 324         ;;
 325
 326     *)
 327         echo "Unsupported OS $OSName detected, configuring as if for Linux"
 328         __BuildOS=Linux
 329         __HostOS=Linux
 330         ;;
 331 esac
 332
 333 __BuildType=Debug
 334 __CodeCoverage=
 335 __IncludeTests=INCLUDE_TESTS
 336
 337 # Set the various build properties here so that CMake and MSBuild can pick them up
 338 export __ProjectDir="$__ProjectRoot"
 339 __SourceDir="$__ProjectDir/src"
 340 __PackagesDir="$__ProjectDir/packages"
 341 __RootBinDir="$__ProjectDir/bin"
 342 __BuildToolsDir="$__ProjectDir/Tools"
 343 __UnprocessedBuildArgs=
 344 __RunArgs=
 345 __MSBCleanBuildArgs=
 346 __UseNinja=0
 347 __VerboseBuild=0
 348 __SkipRestore=""
 349 __CrossBuild=0
 350 __ClangMajorVersion=0
 351 __ClangMinorVersion=0
 352 __NuGetPath="$__PackagesDir/NuGet.exe"
 353 __HostDistroRid=""
 354 __DistroRid=""
 355 __cmakeargs=""
 356 __PortableLinux=0
 357 __msbuildonunsupportedplatform=0
 358 __ZipTests=0
 359 __NativeTestIntermediatesDir=
 360 __RunTests=0
 361 __RebuildTests=0
 362 __BuildTestWrappers=0
 363 CORE_ROOT=
 364
 365
 366 while :; do
 367     if [ $# -le 0 ]; then
 368         break
 369     fi
 370
 371     lowerI="$(echo $1 | awk '{print tolower($0)}')"
 372     case $lowerI in
 373         -\?|-h|--help)
 374             usage
 375             exit 1
 376             ;;
 377
 378         x86)
 379             __BuildArch=x86
 380             ;;
 381
 382         x64)
 383             __BuildArch=x64
 384             ;;
 385
 386         arm)
 387             __BuildArch=arm
 388             ;;
 389
 390         armel)
 391             __BuildArch=armel
 392             ;;
 393
 394         arm64)
 395             __BuildArch=arm64
 396             ;;
 397
 398         debug)
 399             __BuildType=Debug
 400             ;;
 401
 402         checked)
 403             __BuildType=Checked
 404             ;;
 405
 406         release)
 407             __BuildType=Release
 408             ;;
 409
 410         coverage)
 411             __CodeCoverage=Coverage
 412             ;;
 413
 414         cross)
 415             __CrossBuild=1
 416             ;;
 417
 418         portablelinux)
 419             if [ "$__BuildOS" == "Linux" ]; then
 420                 __PortableLinux=1
 421             else
 422                 echo "ERROR: portableLinux not supported for non-Linux platforms."
 423                 exit 1
 424             fi
 425             ;;
 426
 427         verbose)
 428         __VerboseBuild=1
 429         ;;
 430
 431         clang3.5)
 432             __ClangMajorVersion=3
 433             __ClangMinorVersion=5
 434             ;;
 435
 436         clang3.6)
 437             __ClangMajorVersion=3
 438             __ClangMinorVersion=6
 439             ;;
 440
 441         clang3.7)
 442             __ClangMajorVersion=3
 443             __ClangMinorVersion=7
 444             ;;
 445
 446         clang3.8)
 447             __ClangMajorVersion=3
 448             __ClangMinorVersion=8
 449             ;;
 450
 451         clang3.9)
 452             __ClangMajorVersion=3
 453             __ClangMinorVersion=9
 454             ;;
 455
 456         ninja)
 457             __UseNinja=1
 458             ;;
 459
 460         runtests)
 461             __RunTests=1
 462             ;;
 463
 464         rebuild)
 465             __RebuildTests=1
 466             ;;
 467
 468         ziptests)
 469             __ZipTests=1
 470             ;;
 471
 472         bindir)
 473             if [ -n "$2" ]; then
 474                 __RootBinDir="$2"
 475                 if [ ! -d $__RootBinDir ]; then
 476                     mkdir $__RootBinDir
 477                 fi
 478                 __RootBinParent=$(dirname $__RootBinDir)
 479                 __RootBinName=${__RootBinDir##*/}
 480                 __RootBinDir="$(cd $__RootBinParent &>/dev/null && printf %s/%s $PWD $__RootBinName)"
 481                 shift
 482             else
 483                 echo "ERROR: 'bindir' requires a non-empty option argument"
 484                 exit 1
 485             fi
 486             ;;
 487
 488         msbuildonunsupportedplatform)
 489             __msbuildonunsupportedplatform=1
 490             ;;
 491         *)
 492             __UnprocessedBuildArgs="$__UnprocessedBuildArgs $1"
 493             ;;
 494     esac
 495
 496     shift
 497 done
 498
 499
 500 __RunArgs="-BuildArch=$__BuildArch -BuildType=$__BuildType -BuildOS=$__BuildOS"
 501
 502 # Configure environment if we are doing a verbose build
 503 if [ $__VerboseBuild == 1 ]; then
 504     export VERBOSE=1
 505         __RunArgs="$__RunArgs -verbose"
 506 fi
 507
 508 # Set default clang version
 509 if [[ $__ClangMajorVersion == 0 && $__ClangMinorVersion == 0 ]]; then
 510     if [ $__CrossBuild == 1 ]; then
 511         __ClangMajorVersion=3
 512         __ClangMinorVersion=6
 513     else
 514         __ClangMajorVersion=3
 515         __ClangMinorVersion=5
 516     fi
 517 fi
 518
 519
 520 # Set dependent variables
 521 __LogsDir="$__RootBinDir/Logs"
 522
 523 # init the host distro name
 524 initHostDistroRid
 525
 526 # Set the remaining variables based upon the determined build configuration
 527 __BinDir="$__RootBinDir/Product/$__BuildOS.$__BuildArch.$__BuildType"
 528 __PackagesBinDir="$__BinDir/.nuget"
 529 __ToolsDir="$__RootBinDir/tools"
 530 __TestDir="$__ProjectDir/tests"
 531 __TestWorkingDir="$__RootBinDir/tests/$__BuildOS.$__BuildArch.$__BuildType"
 532 __IntermediatesDir="$__RootBinDir/obj/$__BuildOS.$__BuildArch.$__BuildType"
 533 __TestIntermediatesDir="$__RootBinDir/tests/obj/$__BuildOS.$__BuildArch.$__BuildType"
 534 __isMSBuildOnNETCoreSupported=0
 535 __CrossComponentBinDir="$__BinDir"
 536 __CrossCompIntermediatesDir="$__IntermediatesDir/crossgen"
 537
 538 __CrossArch="$__HostArch"
 539 if [[ "$__HostArch" == "x64" && "$__BuildArch" == "arm" ]]; then
 540     __CrossArch="x86"
 541 fi
 542 if [ $__CrossBuild == 1 ]; then
 543     __CrossComponentBinDir="$__CrossComponentBinDir/$__CrossArch"
 544 fi
 545 __CrossgenCoreLibLog="$__LogsDir/CrossgenCoreLib_$__BuildOS.$BuildArch.$__BuildType.log"
 546 __CrossgenExe="$__CrossComponentBinDir/crossgen"
 547
 548 isMSBuildOnNETCoreSupported
 549
 550 # CI_SPECIFIC - On CI machines, $HOME may not be set. In such a case, create a subfolder and set the variable to set.
 551 # This is needed by CLI to function.
 552 if [ -z "$HOME" ]; then
 553     if [ ! -d "$__ProjectDir/temp_home" ]; then
 554         mkdir temp_home
 555     fi
 556     export HOME=$__ProjectDir/temp_home
 557     echo "HOME not defined; setting it to $HOME"
 558 fi
 559
 560 # Specify path to be set for CMAKE_INSTALL_PREFIX.
 561 # This is where all built CoreClr libraries will copied to.
 562 export __CMakeBinDir="$__BinDir"
 563
 564 if [ ! -d "$__BinDir" ] || [ ! -d "$__BinDir/bin" ]; then
 565
 566     echo "Has not been found built CoreCLR instance"
 567     echo "Please build it before tests using './build.sh $__BuildArch $__BuildType'"
 568     exit 1
 569 fi
 570
 571 # Configure environment if we are doing a cross compile.
 572 if [ $__CrossBuild == 1 ]; then
 573     export CROSSCOMPILE=1
 574     if ! [[ -n "$ROOTFS_DIR" ]]; then
 575         export ROOTFS_DIR="$__ProjectRoot/cross/rootfs/$__BuildArch"
 576     fi
 577 fi
 578
 579 # init the target distro name
 580 initTargetDistroRid
 581
 582 # Override tool directory
 583
 584 __CoreClrVersion=1.1.0
 585 __sharedFxDir=$__BuildToolsDir/dotnetcli/shared/Microsoft.NETCore.App/$__CoreClrVersion/
 586
 587 echo "Building Tests..."
 588
 589 build_Tests
 590
 591 if [ $? -ne 0 ]; then
 592     echo "Failed to build tests"
 593     exit 1
 594 fi
 595
 596 echo "${__MsgPrefix}Test build successful."
 597 echo "${__MsgPrefix}Test binaries are available at ${__TestBinDir}"
 598
 599 __testNativeBinDir=$__IntermediatesDir/tests
 600
 601 if [ $__RunTests -ne 0 ]; then
 602
 603     echo "Run Tests..."
 604
 605     echo "${__TestDir}/runtest.sh --testRootDir=$__TestBinDir --coreClrBinDir=$__BinDir --coreFxBinDir=$__sharedFxDir --testNativeBinDir=$__testNativeBinDir"
 606
 607     $__TestDir/runtest.sh --testRootDir=$__TestBinDir --coreClrBinDir=$__BinDir --coreFxBinDir=$CORE_ROOT --testNativeBinDir=$__testNativeBinDir
 608
 609     echo "Tests run successful."
 610 else
 611     echo "To run all tests use 'tests/runtests.sh' where:"
 612     echo "    testRootDir      = $__TestBinDir"
 613     echo "    coreClrBinDir    = $__BinDir"
 614     echo "    coreFxBinDir     = $CORE_ROOT"
 615     echo "    testNativeBinDir = $__testNativeBinDir"
 616     echo " -------------------------------------------------- "
 617     echo "To run single test use the following command:"
 618     echo "    bash ${__TestBinDir}/__TEST_PATH__/__TEST_NAME__.sh -coreroot=${CORE_ROOT}"
 619 fi
 620