build.sh

   1 #!/usr/bin/env bash
   2
   3 PYTHON=${PYTHON:-python}
   4
   5 usage()
   6 {
   7     echo "Usage: $0 [BuildArch] [BuildType] [clean] [verbose] [coverage] [cross] [clangx.y] [ninja] [configureonly] [skipconfigure] [skipnative] [skipmscorlib] [skiptests] [cmakeargs]"
   8     echo "BuildArch can be: x64, x86, arm, arm-softfp, arm64"
   9     echo "BuildType can be: debug, checked, release"
  10     echo "clean - optional argument to force a clean build."
  11     echo "verbose - optional argument to enable verbose build output."
  12     echo "coverage - optional argument to enable code coverage build (currently supported only for Linux and OSX)."
  13     echo "ninja - target ninja instead of GNU make"
  14     echo "clangx.y - optional argument to build using clang version x.y."
  15     echo "cross - optional argument to signify cross compilation,"
  16     echo "      - will use ROOTFS_DIR environment variable if set."
  17     echo "configureonly - do not perform any builds; just configure the build."
  18     echo "skipconfigure - skip build configuration."
  19     echo "skipnative - do not build native components."
  20     echo "skipmscorlib - do not build mscorlib.dll."
  21     echo "skiptests - skip the tests in the 'tests' subdirectory."
  22     echo "disableoss - Disable Open Source Signing for mscorlib."
  23     echo "cmakeargs - user-settable additional arguments passed to CMake."
  24
  25     exit 1
  26 }
  27
  28 initDistroName()
  29 {
  30     if [ "$__BuildOS" == "Linux" ]; then
  31         # Detect Distro
  32         if [ "$(cat /etc/*-release | grep -cim1 ubuntu)" -eq 1 ]; then
  33             export __DistroName=ubuntu
  34         elif [ "$(cat /etc/*-release | grep -cim1 centos)" -eq 1 ]; then
  35             export __DistroName=rhel
  36         elif [ "$(cat /etc/*-release | grep -cim1 rhel)" -eq 1 ]; then
  37             export __DistroName=rhel
  38         elif [ "$(cat /etc/*-release | grep -cim1 debian)" -eq 1 ]; then
  39             export __DistroName=debian
  40         else
  41             export __DistroName=""
  42         fi
  43     fi
  44 }
  45
  46 setup_dirs()
  47 {
  48     echo Setting up directories for build
  49
  50     mkdir -p "$__RootBinDir"
  51     mkdir -p "$__BinDir"
  52     mkdir -p "$__LogsDir"
  53     mkdir -p "$__IntermediatesDir"
  54 }
  55
  56 # Performs "clean build" type actions (deleting and remaking directories)
  57
  58 clean()
  59 {
  60     echo Cleaning previous output for the selected configuration
  61     rm -rf "$__BinDir"
  62     rm -rf "$__IntermediatesDir"
  63
  64     rm -rf "$__TestWorkingDir"
  65     rm -rf "$__TestIntermediatesDir"
  66
  67     rm -rf "$__LogsDir/*_$__BuildOS__$__BuildArch__$__BuildType.*"
  68     rm -rf "$__ProjectRoot/Tools"
  69 }
  70
  71 # Check the system to ensure the right prereqs are in place
  72
  73 check_prereqs()
  74 {
  75     echo "Checking prerequisites..."
  76
  77     # Check presence of CMake on the path
  78     hash cmake 2>/dev/null || { echo >&2 "Please install cmake before running this script"; exit 1; }
  79
  80     # Check for clang
  81     hash clang-$__ClangMajorVersion.$__ClangMinorVersion 2>/dev/null ||  hash clang$__ClangMajorVersion$__ClangMinorVersion 2>/dev/null ||  hash clang 2>/dev/null || { echo >&2 "Please install clang before running this script"; exit 1; }
  82
  83 }
  84
  85 build_coreclr()
  86 {
  87
  88 # Event Logging Infrastructure
  89    __GeneratedIntermediate="$__IntermediatesDir/Generated"
  90    __GeneratedIntermediateEventProvider="$__GeneratedIntermediate/eventprovider_new"
  91     if [[ -d "$__GeneratedIntermediateEventProvider" ]]; then
  92         rm -rf  "$__GeneratedIntermediateEventProvider"
  93     fi
  94
  95     if [[ ! -d "$__GeneratedIntermediate/eventprovider" ]]; then
  96         mkdir -p "$__GeneratedIntermediate/eventprovider"
  97     fi
  98
  99     mkdir -p "$__GeneratedIntermediateEventProvider"
 100     if [[ $__SkipCoreCLR == 0 || $__ConfigureOnly == 1 ]]; then
 101         echo "Laying out dynamically generated files consumed by the build system "
 102         echo "Laying out dynamically generated Event Logging Test files"
 103         $PYTHON -B -Wall -Werror "$__ProjectRoot/src/scripts/genXplatEventing.py" --man "$__ProjectRoot/src/vm/ClrEtwAll.man" --exc "$__ProjectRoot/src/vm/ClrEtwAllMeta.lst" --testdir "$__GeneratedIntermediateEventProvider/tests"
 104
 105         if  [[ $? != 0 ]]; then
 106             exit
 107         fi
 108
 109         #determine the logging system
 110         case $__BuildOS in
 111             Linux)
 112                 echo "Laying out dynamically generated Event Logging Implementation of Lttng"
 113                 $PYTHON -B -Wall -Werror "$__ProjectRoot/src/scripts/genXplatLttng.py" --man "$__ProjectRoot/src/vm/ClrEtwAll.man" --intermediate "$__GeneratedIntermediateEventProvider"
 114                 if  [[ $? != 0 ]]; then
 115                     exit
 116                 fi
 117                 ;;
 118             *)
 119                 ;;
 120         esac
 121     fi
 122
 123     echo "Cleaning the temp folder of dynamically generated Event Logging files"
 124     $PYTHON -B -Wall -Werror -c "import sys;sys.path.insert(0,\"$__ProjectRoot/src/scripts\"); from Utilities import *;UpdateDirectory(\"$__GeneratedIntermediate/eventprovider\",\"$__GeneratedIntermediateEventProvider\")"
 125     if  [[ $? != 0 ]]; then
 126         exit
 127     fi
 128
 129     rm -rf "$__GeneratedIntermediateEventProvider"
 130
 131     # All set to commence the build
 132
 133     echo "Commencing build of native components for $__BuildOS.$__BuildArch.$__BuildType in $__IntermediatesDir"
 134
 135     cd "$__IntermediatesDir"
 136
 137     generator=""
 138     buildFile="Makefile"
 139     buildTool="make"
 140     if [ $__UseNinja == 1 ]; then
 141         generator="ninja"
 142         buildFile="build.ninja"
 143         buildTool="ninja"
 144     fi
 145
 146     if [ $__SkipConfigure == 0 ]; then
 147         # Regenerate the CMake solution
 148         echo "Invoking \"$__ProjectRoot/src/pal/tools/gen-buildsys-clang.sh\" \"$__ProjectRoot\" $__ClangMajorVersion $__ClangMinorVersion $__BuildArch $__BuildType $__CodeCoverage $__IncludeTests $generator $__cmakeargs"
 149         "$__ProjectRoot/src/pal/tools/gen-buildsys-clang.sh" "$__ProjectRoot" $__ClangMajorVersion $__ClangMinorVersion $__BuildArch $__BuildType $__CodeCoverage $__IncludeTests $generator "$__cmakeargs"
 150     fi
 151
 152     if [ $__SkipCoreCLR == 1 ]; then
 153         echo "Skipping CoreCLR build."
 154         return
 155     fi
 156
 157     # Check that the makefiles were created.
 158
 159     if [ ! -f "$__IntermediatesDir/$buildFile" ]; then
 160         echo "Failed to generate native component build project!"
 161         exit 1
 162     fi
 163
 164     # Get the number of processors available to the scheduler
 165     # Other techniques such as `nproc` only get the number of
 166     # processors available to a single process.
 167     if [ `uname` = "FreeBSD" ]; then
 168         NumProc=`sysctl hw.ncpu | awk '{ print $2+1 }'`
 169     elif [ `uname` = "NetBSD" ]; then
 170         NumProc=$(($(getconf NPROCESSORS_ONLN)+1))
 171     else
 172         NumProc=$(($(getconf _NPROCESSORS_ONLN)+1))
 173     fi
 174
 175     # Build CoreCLR
 176
 177     echo "Executing $buildTool install -j $NumProc $__UnprocessedBuildArgs"
 178
 179     $buildTool install -j $NumProc $__UnprocessedBuildArgs
 180     if [ $? != 0 ]; then
 181         echo "Failed to build coreclr components."
 182         exit 1
 183     fi
 184 }
 185
 186 restoreBuildTools()
 187 {
 188     echo "Restoring BuildTools..."
 189     $__ProjectRoot/init-tools.sh
 190     if [ $? -ne 0 ]; then
 191         echo "Failed to restore BuildTools."
 192         exit 1
 193     fi
 194 }
 195
 196 isMSBuildOnNETCoreSupported()
 197 {
 198     # This needs to be updated alongwith corresponding changes to netci.groovy.
 199     __isMSBuildOnNETCoreSupported=0
 200
 201     if [ "$__BuildArch" == "x64" ]; then
 202         if [ "$__BuildOS" == "Linux" ]; then
 203             if [ "$__DistroName" == "ubuntu" ]; then
 204                 __OSVersion=$(lsb_release -rs)
 205                 if [ "$__OSVersion" == "14.04" ]; then
 206                     __isMSBuildOnNETCoreSupported=1
 207                 fi
 208             elif [ "$__DistroName" == "rhel" ]; then
 209                 __isMSBuildOnNETCoreSupported=1
 210             elif [ "$__DistroName" == "debian" ]; then
 211                 __isMSBuildOnNETCoreSupported=1
 212             fi
 213         elif [ "$__BuildOS" == "OSX" ]; then
 214             __isMSBuildOnNETCoreSupported=1
 215         fi
 216     elif [ "$__BuildArch" == "arm" ] || [ "$__BuildArch" == "arm64" ] ; then
 217         if [ "$__BuildOS" == "Linux" ]; then
 218             if [ "$__DistroName" == "ubuntu" ]; then
 219                 __isMSBuildOnNETCoreSupported=1
 220             fi
 221         fi
 222
 223     fi
 224 }
 225
 226 build_mscorlib_ni()
 227 {
 228     if [ $__SkipCoreCLR == 0 -a -e $__BinDir/crossgen ]; then
 229         echo "Generating native image for mscorlib."
 230         $__BinDir/crossgen $__BinDir/mscorlib.dll
 231         if [ $? -ne 0 ]; then
 232             echo "Failed to generate native image for mscorlib."
 233             exit 1
 234         fi
 235     fi
 236 }
 237
 238 build_mscorlib()
 239 {
 240
 241     if [ $__isMSBuildOnNETCoreSupported == 0 ]; then
 242         echo "Mscorlib.dll build unsupported."
 243         return
 244     fi
 245
 246     if [ $__SkipMSCorLib == 1 ]; then
 247        echo "Skipping building mscorlib."
 248        return
 249     fi
 250
 251     # Restore buildTools
 252
 253     restoreBuildTools
 254
 255     echo "Commencing build of mscorlib components for $__BuildOS.$__BuildArch.$__BuildType"
 256
 257     # Invoke MSBuild
 258     $__ProjectRoot/Tools/corerun "$__MSBuildPath" /nologo "$__ProjectRoot/build.proj" /verbosity:minimal "/fileloggerparameters:Verbosity=normal;LogFile=$__LogsDir/MSCorLib_$__BuildOS__$__BuildArch__$__BuildType.log" /t:Build /p:__BuildOS=$__BuildOS /p:__BuildArch=$__BuildArch /p:__BuildType=$__BuildType /p:__IntermediatesDir=$__IntermediatesDir /p:BuildNugetPackage=false /p:UseSharedCompilation=false ${__SignTypeReal}
 259
 260     if [ $? -ne 0 ]; then
 261         echo "Failed to build mscorlib."
 262         exit 1
 263     fi
 264
 265     # The cross build generates a crossgen with the target architecture.
 266     if [ $__CrossBuild != 1 ]; then
 267        # The architecture of host pc must be same architecture with target.
 268        if [[ ( "$__HostArch" == "$__BuildArch" ) ]]; then
 269            build_mscorlib_ni
 270        elif [[ ( "$__HostArch" == "x64" ) && ( "$__BuildArch" == "x86" ) ]]; then
 271            build_mscorlib_ni
 272        elif [[ ( "$__HostArch" == "arm64" ) && ( "$__BuildArch" == "arm" ) ]]; then
 273            build_mscorlib_ni
 274        else
 275            exit 1
 276        fi
 277     fi
 278 }
 279
 280
 281
 282 generate_NugetPackages()
 283 {
 284     # We can only generate nuget package if we also support building mscorlib as part of this build.
 285     if [ $__isMSBuildOnNETCoreSupported == 0 ]; then
 286         echo "Microsoft.NETCore.Runtime.CoreCLR nuget package generation unsupported."
 287         return
 288     fi
 289
 290     # Since we can build mscorlib for this OS, did we build the native components as well?
 291     if [ $__SkipCoreCLR == 1 ]; then
 292         echo "Unable to generate Microsoft.NETCore.Runtime.CoreCLR nuget package since native components were not built."
 293         return
 294     fi
 295
 296     if [ $__SkipMSCorLib == 1 ]; then
 297        echo "Unable to generate Microsoft.NETCore.Runtime.CoreCLR nuget package since mscorlib was not built."
 298        return
 299     fi
 300
 301     echo "Generating nuget packages for "$__BuildOS
 302
 303     # Invoke MSBuild
 304     $__ProjectRoot/Tools/corerun "$__MSBuildPath" /nologo "$__ProjectRoot/src/.nuget/Microsoft.NETCore.Runtime.CoreCLR/Microsoft.NETCore.Runtime.CoreCLR.builds" /verbosity:minimal "/fileloggerparameters:Verbosity=normal;LogFile=$__LogsDir/Nuget_$__BuildOS__$__BuildArch__$__BuildType.log" /t:Build /p:__BuildOS=$__BuildOS /p:__BuildArch=$__BuildArch /p:__BuildType=$__BuildType /p:__IntermediatesDir=$__IntermediatesDir /p:BuildNugetPackage=false /p:UseSharedCompilation=false
 305
 306     if [ $? -ne 0 ]; then
 307         echo "Failed to generate Nuget packages."
 308         exit 1
 309     fi
 310 }
 311
 312 echo "Commencing CoreCLR Repo build"
 313
 314 # Argument types supported by this script:
 315 #
 316 # Build architecture - valid values are: x64, ARM.
 317 # Build Type         - valid values are: Debug, Checked, Release
 318 #
 319 # Set the default arguments for build
 320
 321 # Obtain the location of the bash script to figure out where the root of the repo is.
 322 __ProjectRoot="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
 323
 324 # Use uname to determine what the CPU is.
 325 CPUName=$(uname -p)
 326 # Some Linux platforms report unknown for platform, but the arch for machine.
 327 if [ $CPUName == "unknown" ]; then
 328     CPUName=$(uname -m)
 329 fi
 330
 331 case $CPUName in
 332     i686)
 333         echo "Unsupported CPU $CPUName detected, build might not succeed!"
 334         __BuildArch=x86
 335         __HostArch=x86
 336         ;;
 337
 338     x86_64)
 339         __BuildArch=x64
 340         __HostArch=x64
 341         ;;
 342
 343     armv7l)
 344         echo "Unsupported CPU $CPUName detected, build might not succeed!"
 345         __BuildArch=arm
 346         __HostArch=arm
 347         ;;
 348
 349     aarch64)
 350         echo "Unsupported CPU $CPUName detected, build might not succeed!"
 351         __BuildArch=arm64
 352         __HostArch=arm64
 353         ;;
 354
 355     *)
 356         echo "Unknown CPU $CPUName detected, configuring as if for x64"
 357         __BuildArch=x64
 358         __HostArch=x64
 359         ;;
 360 esac
 361
 362 # Use uname to determine what the OS is.
 363 OSName=$(uname -s)
 364 case $OSName in
 365     Linux)
 366         __BuildOS=Linux
 367         ;;
 368
 369     Darwin)
 370         __BuildOS=OSX
 371         ;;
 372
 373     FreeBSD)
 374         __BuildOS=FreeBSD
 375         ;;
 376
 377     OpenBSD)
 378         __BuildOS=OpenBSD
 379         ;;
 380
 381     NetBSD)
 382         __BuildOS=NetBSD
 383         ;;
 384
 385     SunOS)
 386         __BuildOS=SunOS
 387         ;;
 388
 389     *)
 390         echo "Unsupported OS $OSName detected, configuring as if for Linux"
 391         __BuildOS=Linux
 392         ;;
 393 esac
 394
 395 __BuildType=Debug
 396 __CodeCoverage=
 397 __IncludeTests=Include_Tests
 398
 399 # Set the various build properties here so that CMake and MSBuild can pick them up
 400 __ProjectDir="$__ProjectRoot"
 401 __SourceDir="$__ProjectDir/src"
 402 __PackagesDir="$__ProjectDir/packages"
 403 __RootBinDir="$__ProjectDir/bin"
 404 __LogsDir="$__RootBinDir/Logs"
 405 __UnprocessedBuildArgs=
 406 __MSBCleanBuildArgs=
 407 __UseNinja=0
 408 __ConfigureOnly=0
 409 __SkipConfigure=0
 410 __SkipCoreCLR=0
 411 __SkipMSCorLib=0
 412 __CleanBuild=0
 413 __VerboseBuild=0
 414 __SignTypeReal=""
 415 __CrossBuild=0
 416 __ClangMajorVersion=3
 417 __ClangMinorVersion=5
 418 __MSBuildPath=$__ProjectRoot/Tools/MSBuild.exe
 419 __NuGetPath="$__PackagesDir/NuGet.exe"
 420 __DistroName=""
 421 __cmakeargs=""
 422
 423 while :; do
 424     if [ $# -le 0 ]; then
 425         break
 426     fi
 427
 428     lowerI="$(echo $1 | awk '{print tolower($0)}')"
 429     case $lowerI in
 430         -\?|-h|--help)
 431             usage
 432             exit 1
 433             ;;
 434
 435         x86)
 436             __BuildArch=x86
 437             ;;
 438
 439         x64)
 440             __BuildArch=x64
 441             ;;
 442
 443         arm)
 444             __BuildArch=arm
 445             ;;
 446
 447         arm-softfp)
 448             __BuildArch=arm-softfp
 449             ;;
 450
 451         arm64)
 452             __BuildArch=arm64
 453             ;;
 454
 455         debug)
 456             __BuildType=Debug
 457             ;;
 458
 459         checked)
 460             __BuildType=Checked
 461             ;;
 462
 463         release)
 464             __BuildType=Release
 465             ;;
 466
 467         coverage)
 468             __CodeCoverage=Coverage
 469             ;;
 470
 471         clean)
 472             __CleanBuild=1
 473             ;;
 474
 475         verbose)
 476             __VerboseBuild=1
 477             ;;
 478
 479         cross)
 480             __CrossBuild=1
 481             ;;
 482
 483         clang3.5)
 484             __ClangMajorVersion=3
 485             __ClangMinorVersion=5
 486             ;;
 487
 488         clang3.6)
 489             __ClangMajorVersion=3
 490             __ClangMinorVersion=6
 491             ;;
 492
 493         clang3.7)
 494             __ClangMajorVersion=3
 495             __ClangMinorVersion=7
 496             ;;
 497
 498         clang3.8)
 499             __ClangMajorVersion=3
 500             __ClangMinorVersion=8
 501             ;;
 502
 503         ninja)
 504             __UseNinja=1
 505             ;;
 506
 507         configureonly)
 508             __ConfigureOnly=1
 509             __SkipCoreCLR=1
 510             __SkipMSCorLib=1
 511             __IncludeTests=
 512             ;;
 513
 514         skipconfigure)
 515             __SkipConfigure=1
 516             ;;
 517
 518         skipnative)
 519             # Use "skipnative" to use the same option name as build.cmd.
 520             __SkipCoreCLR=1
 521             ;;
 522
 523         skipcoreclr)
 524             # Accept "skipcoreclr" for backwards-compatibility.
 525             __SkipCoreCLR=1
 526             ;;
 527
 528         skipmscorlib)
 529             __SkipMSCorLib=1
 530             ;;
 531
 532         includetests)
 533             ;;
 534
 535         skiptests)
 536             __IncludeTests=
 537             ;;
 538
 539         disableoss)
 540             __SignTypeReal="/p:SignType=real"
 541             ;;
 542
 543         cmakeargs)
 544             if [ -n "$2" ]; then
 545                 __cmakeargs="$2"
 546                 shift
 547             else
 548                 echo "ERROR: 'cmakeargs' requires a non-empty option argument"
 549                 exit 1
 550             fi
 551             ;;
 552
 553         *)
 554             __UnprocessedBuildArgs="$__UnprocessedBuildArgs $1"
 555             ;;
 556     esac
 557
 558     shift
 559 done
 560
 561 if [[ $__ConfigureOnly == 1 && $__SkipConfigure == 1 ]]; then
 562     echo "configureonly and skipconfigure are mutually exclusive!"
 563     exit 1
 564 fi
 565
 566 # init the distro name
 567 initDistroName
 568
 569 # Set the remaining variables based upon the determined build configuration
 570 __BinDir="$__RootBinDir/Product/$__BuildOS.$__BuildArch.$__BuildType"
 571 __PackagesBinDir="$__BinDir/.nuget"
 572 __ToolsDir="$__RootBinDir/tools"
 573 __TestWorkingDir="$__RootBinDir/tests/$__BuildOS.$__BuildArch.$__BuildType"
 574 export __IntermediatesDir="$__RootBinDir/obj/$__BuildOS.$__BuildArch.$__BuildType"
 575 __TestIntermediatesDir="$__RootBinDir/tests/obj/$__BuildOS.$__BuildArch.$__BuildType"
 576 __isMSBuildOnNETCoreSupported=0
 577
 578 # Init if MSBuild for .NET Core is supported for this platform
 579 isMSBuildOnNETCoreSupported
 580
 581 # CI_SPECIFIC - On CI machines, $HOME may not be set. In such a case, create a subfolder and set the variable to set.
 582 # This is needed by CLI to function.
 583 if [ -z "$HOME" ]; then
 584     if [ ! -d "$__ProjectDir/temp_home" ]; then
 585         mkdir temp_home
 586     fi
 587     export HOME=$__ProjectDir/temp_home
 588     echo "HOME not defined; setting it to $HOME"
 589 fi
 590
 591 # Specify path to be set for CMAKE_INSTALL_PREFIX.
 592 # This is where all built CoreClr libraries will copied to.
 593 export __CMakeBinDir="$__BinDir"
 594
 595 # Configure environment if we are doing a clean build.
 596 if [ $__CleanBuild == 1 ]; then
 597     clean
 598 fi
 599
 600 # Configure environment if we are doing a verbose build
 601 if [ $__VerboseBuild == 1 ]; then
 602     export VERBOSE=1
 603 fi
 604
 605 # Configure environment if we are doing a cross compile.
 606 if [ $__CrossBuild == 1 ]; then
 607     export CROSSCOMPILE=1
 608     if ! [[ -n "$ROOTFS_DIR" ]]; then
 609         export ROOTFS_DIR="$__ProjectRoot/cross/rootfs/$__BuildArch"
 610     fi
 611 fi
 612
 613 # Make the directories necessary for build if they don't exist
 614
 615 setup_dirs
 616
 617 # Check prereqs.
 618
 619 check_prereqs
 620
 621 # Build the coreclr (native) components.
 622
 623 build_coreclr
 624
 625 # Build mscorlib.
 626
 627 build_mscorlib
 628
 629 # Generate nuget packages
 630
 631 generate_NugetPackages
 632
 633
 634 # Build complete
 635
 636 echo "Repo successfully built."
 637 echo "Product binaries are available at $__BinDir"
 638 exit 0