build.sh

   1 #!/usr/bin/env bash
   2
   3 PYTHON=${PYTHON:-python}
   4
   5 usage()
   6 {
   7     echo "Usage: $0 [BuildArch] [BuildType] [clean] [verbose] [coverage] [cross] [clangx.y] [ninja] [configureonly] [skipconfigure] [skipnative] [skipmscorlib] [skiptests] [cmakeargs]"
   8     echo "BuildArch can be: x64, x86, arm, arm64"
   9     echo "BuildType can be: Debug, Checked, Release"
  10     echo "clean - optional argument to force a clean build."
  11     echo "verbose - optional argument to enable verbose build output."
  12     echo "coverage - optional argument to enable code coverage build (currently supported only for Linux and OSX)."
  13     echo "ninja - target ninja instead of GNU make"
  14     echo "clangx.y - optional argument to build using clang version x.y."
  15     echo "cross - optional argument to signify cross compilation,"
  16     echo "      - will use ROOTFS_DIR environment variable if set."
  17     echo "configureonly - do not perform any builds; just configure the build."
  18     echo "skipconfigure - skip build configuration."
  19     echo "skipnative - do not build native components."
  20     echo "skipmscorlib - do not build mscorlib.dll."
  21     echo "skiptests - skip the tests in the 'tests' subdirectory."
  22     echo "disableoss - Disable Open Source Signing for mscorlib."
  23     echo "cmakeargs - user-settable additional arguments passed to CMake."
  24
  25     exit 1
  26 }
  27
  28 initDistroName()
  29 {
  30     if [ "$__BuildOS" == "Linux" ]; then
  31         # Detect Distro
  32         if [ "$(cat /etc/*-release | grep -cim1 ubuntu)" -eq 1 ]; then
  33             export __DistroName=ubuntu
  34         elif [ "$(cat /etc/*-release | grep -cim1 centos)" -eq 1 ]; then
  35             export __DistroName=rhel
  36         elif [ "$(cat /etc/*-release | grep -cim1 rhel)" -eq 1 ]; then
  37             export __DistroName=rhel
  38         elif [ "$(cat /etc/*-release | grep -cim1 debian)" -eq 1 ]; then
  39             export __DistroName=debian
  40         else
  41             export __DistroName=""
  42         fi
  43     fi
  44 }
  45
  46 setup_dirs()
  47 {
  48     echo Setting up directories for build
  49
  50     mkdir -p "$__RootBinDir"
  51     mkdir -p "$__BinDir"
  52     mkdir -p "$__LogsDir"
  53     mkdir -p "$__IntermediatesDir"
  54 }
  55
  56 # Performs "clean build" type actions (deleting and remaking directories)
  57
  58 clean()
  59 {
  60     echo Cleaning previous output for the selected configuration
  61     rm -rf "$__BinDir"
  62     rm -rf "$__IntermediatesDir"
  63
  64     rm -rf "$__TestWorkingDir"
  65     rm -rf "$__TestIntermediatesDir"
  66
  67     rm -rf "$__LogsDir/*_$__BuildOS__$__BuildArch__$__BuildType.*"
  68 }
  69
  70 # Check the system to ensure the right prereqs are in place
  71
  72 check_prereqs()
  73 {
  74     echo "Checking prerequisites..."
  75
  76     # Check presence of CMake on the path
  77     hash cmake 2>/dev/null || { echo >&2 "Please install cmake before running this script"; exit 1; }
  78
  79     # Check for clang
  80     hash clang-$__ClangMajorVersion.$__ClangMinorVersion 2>/dev/null ||  hash clang$__ClangMajorVersion$__ClangMinorVersion 2>/dev/null ||  hash clang 2>/dev/null || { echo >&2 "Please install clang before running this script"; exit 1; }
  81
  82 }
  83
  84 build_coreclr()
  85 {
  86
  87 # Event Logging Infrastructure
  88    __GeneratedIntermediate="$__IntermediatesDir/Generated"
  89    __GeneratedIntermediateEventProvider="$__GeneratedIntermediate/eventprovider_new"
  90     if [[ -d "$__GeneratedIntermediateEventProvider" ]]; then
  91         rm -rf  "$__GeneratedIntermediateEventProvider"
  92     fi
  93
  94     if [[ ! -d "$__GeneratedIntermediate/eventprovider" ]]; then
  95         mkdir -p "$__GeneratedIntermediate/eventprovider"
  96     fi
  97
  98     mkdir -p "$__GeneratedIntermediateEventProvider"
  99     if [[ $__SkipCoreCLR == 0 || $__ConfigureOnly == 1 ]]; then
 100         echo "Laying out dynamically generated files consumed by the build system "
 101         echo "Laying out dynamically generated Event Logging Test files"
 102         $PYTHON -B -Wall -Werror "$__ProjectRoot/src/scripts/genXplatEventing.py" --man "$__ProjectRoot/src/vm/ClrEtwAll.man" --exc "$__ProjectRoot/src/vm/ClrEtwAllMeta.lst" --testdir "$__GeneratedIntermediateEventProvider/tests"
 103
 104         if  [[ $? != 0 ]]; then
 105             exit
 106         fi
 107
 108         #determine the logging system
 109         case $__BuildOS in
 110             Linux)
 111                 echo "Laying out dynamically generated Event Logging Implementation of Lttng"
 112                 $PYTHON -B -Wall -Werror "$__ProjectRoot/src/scripts/genXplatLttng.py" --man "$__ProjectRoot/src/vm/ClrEtwAll.man" --intermediate "$__GeneratedIntermediateEventProvider"
 113                 if  [[ $? != 0 ]]; then
 114                     exit
 115                 fi
 116                 ;;
 117             *)
 118                 ;;
 119         esac
 120     fi
 121
 122     echo "Cleaning the temp folder of dynamically generated Event Logging files"
 123     $PYTHON -B -Wall -Werror -c "import sys;sys.path.insert(0,\"$__ProjectRoot/src/scripts\"); from Utilities import *;UpdateDirectory(\"$__GeneratedIntermediate/eventprovider\",\"$__GeneratedIntermediateEventProvider\")"
 124     if  [[ $? != 0 ]]; then
 125         exit
 126     fi
 127
 128     rm -rf "$__GeneratedIntermediateEventProvider"
 129
 130     # All set to commence the build
 131
 132     echo "Commencing build of native components for $__BuildOS.$__BuildArch.$__BuildType in $__IntermediatesDir"
 133
 134     cd "$__IntermediatesDir"
 135
 136     generator=""
 137     buildFile="Makefile"
 138     buildTool="make"
 139     if [ $__UseNinja == 1 ]; then
 140         generator="ninja"
 141         buildFile="build.ninja"
 142         buildTool="ninja"
 143     fi
 144
 145     if [ $__SkipConfigure == 0 ]; then
 146         # Regenerate the CMake solution
 147         echo "Invoking \"$__ProjectRoot/src/pal/tools/gen-buildsys-clang.sh\" \"$__ProjectRoot\" $__ClangMajorVersion $__ClangMinorVersion $__BuildArch $__BuildType $__CodeCoverage $__IncludeTests $generator $__cmakeargs"
 148         "$__ProjectRoot/src/pal/tools/gen-buildsys-clang.sh" "$__ProjectRoot" $__ClangMajorVersion $__ClangMinorVersion $__BuildArch $__BuildType $__CodeCoverage $__IncludeTests $generator "$__cmakeargs"
 149     fi
 150
 151     if [ $__SkipCoreCLR == 1 ]; then
 152         echo "Skipping CoreCLR build."
 153         return
 154     fi
 155
 156     # Check that the makefiles were created.
 157
 158     if [ ! -f "$__IntermediatesDir/$buildFile" ]; then
 159         echo "Failed to generate native component build project!"
 160         exit 1
 161     fi
 162
 163     # Get the number of processors available to the scheduler
 164     # Other techniques such as `nproc` only get the number of
 165     # processors available to a single process.
 166     if [ `uname` = "FreeBSD" ]; then
 167         NumProc=`sysctl hw.ncpu | awk '{ print $2+1 }'`
 168     elif [ `uname` = "NetBSD" ]; then
 169         NumProc=$(($(getconf NPROCESSORS_ONLN)+1))
 170     else
 171         NumProc=$(($(getconf _NPROCESSORS_ONLN)+1))
 172     fi
 173
 174     # Build CoreCLR
 175
 176     echo "Executing $buildTool install -j $NumProc $__UnprocessedBuildArgs"
 177
 178     $buildTool install -j $NumProc $__UnprocessedBuildArgs
 179     if [ $? != 0 ]; then
 180         echo "Failed to build coreclr components."
 181         exit 1
 182     fi
 183 }
 184
 185 restoreBuildTools()
 186 {
 187     echo "Restoring BuildTools..."
 188     $__ProjectRoot/init-tools.sh
 189     if [ $? -ne 0 ]; then
 190         echo "Failed to restore BuildTools."
 191         exit 1
 192     fi
 193 }
 194
 195 isMSBuildOnNETCoreSupported()
 196 {
 197     # This needs to be updated alongwith corresponding changes to netci.groovy.
 198     __isMSBuildOnNETCoreSupported=0
 199
 200     if [ "$__BuildArch" == "x64" ]; then
 201         if [ "$__BuildOS" == "Linux" ]; then
 202             if [ "$__DistroName" == "ubuntu" ]; then
 203                 __OSVersion=$(lsb_release -rs)
 204                 if [ "$__OSVersion" == "14.04" ]; then
 205                     __isMSBuildOnNETCoreSupported=1
 206                 fi
 207             elif [ "$__DistroName" == "rhel" ]; then
 208                 __isMSBuildOnNETCoreSupported=1
 209             elif [ "$__DistroName" == "debian" ]; then
 210                 __isMSBuildOnNETCoreSupported=1
 211             fi
 212         elif [ "$__BuildOS" == "OSX" ]; then
 213             __isMSBuildOnNETCoreSupported=1
 214         fi
 215     elif [ "$__BuildArch" == "arm" ] || [ "$__BuildArch" == "arm64" ] ; then
 216         if [ "$__BuildOS" == "Linux" ]; then
 217             if [ "$__DistroName" == "ubuntu" ]; then
 218                 __isMSBuildOnNETCoreSupported=1
 219             fi
 220         fi
 221
 222     fi
 223 }
 224
 225 build_mscorlib_ni()
 226 {
 227     if [ $__SkipCoreCLR == 0 -a -e $__BinDir/crossgen ]; then
 228         echo "Generating native image for mscorlib."
 229         $__BinDir/crossgen $__BinDir/mscorlib.dll
 230         if [ $? -ne 0 ]; then
 231             echo "Failed to generate native image for mscorlib."
 232             exit 1
 233         fi
 234     fi
 235 }
 236
 237 build_mscorlib()
 238 {
 239
 240     if [ $__isMSBuildOnNETCoreSupported == 0 ]; then
 241         echo "Mscorlib.dll build unsupported."
 242         return
 243     fi
 244
 245     if [ $__SkipMSCorLib == 1 ]; then
 246        echo "Skipping building mscorlib."
 247        return
 248     fi
 249
 250     # Restore buildTools
 251
 252     restoreBuildTools
 253
 254     echo "Commencing build of mscorlib components for $__BuildOS.$__BuildArch.$__BuildType"
 255
 256     # Invoke MSBuild
 257     $__ProjectRoot/Tools/corerun "$__MSBuildPath" /nologo "$__ProjectRoot/build.proj" /verbosity:minimal "/fileloggerparameters:Verbosity=normal;LogFile=$__LogsDir/MSCorLib_$__BuildOS__$__BuildArch__$__BuildType.log" /t:Build /p:__BuildOS=$__BuildOS /p:__BuildArch=$__BuildArch /p:__BuildType=$__BuildType /p:__IntermediatesDir=$__IntermediatesDir /p:UseRoslynCompiler=true /p:BuildNugetPackage=false /p:UseSharedCompilation=false ${__SignTypeReal}
 258
 259     if [ $? -ne 0 ]; then
 260         echo "Failed to build mscorlib."
 261         exit 1
 262     fi
 263
 264     # The cross build generates a crossgen with the target architecture.
 265     if [ $__CrossBuild != 1 ]; then
 266        # The architecture of host pc must be same architecture with target.
 267        if [[ ( "$__HostArch" == "$__BuildArch" ) ]]; then
 268            build_mscorlib_ni
 269        elif [[ ( "$__HostArch" == "x64" ) && ( "$__BuildArch" == "x86" ) ]]; then
 270            build_mscorlib_ni
 271        elif [[ ( "$__HostArch" == "arm64" ) && ( "$__BuildArch" == "arm" ) ]]; then
 272            build_mscorlib_ni
 273        else
 274            exit 1
 275        fi
 276     fi
 277 }
 278
 279
 280
 281 generate_NugetPackages()
 282 {
 283     # We can only generate nuget package if we also support building mscorlib as part of this build.
 284     if [ $__isMSBuildOnNETCoreSupported == 0 ]; then
 285         echo "Microsoft.NETCore.Runtime.CoreCLR nuget package generation unsupported."
 286         return
 287     fi
 288
 289     # Since we can build mscorlib for this OS, did we build the native components as well?
 290     if [ $__SkipCoreCLR == 1 ]; then
 291         echo "Unable to generate Microsoft.NETCore.Runtime.CoreCLR nuget package since native components were not built."
 292         return
 293     fi
 294
 295     if [ $__SkipMSCorLib == 1 ]; then
 296        echo "Unable to generate Microsoft.NETCore.Runtime.CoreCLR nuget package since mscorlib was not built."
 297        return
 298     fi
 299
 300     echo "Generating nuget packages for "$__BuildOS
 301
 302     # Invoke MSBuild
 303     $__ProjectRoot/Tools/corerun "$__MSBuildPath" /nologo "$__ProjectRoot/src/.nuget/Microsoft.NETCore.Runtime.CoreCLR/Microsoft.NETCore.Runtime.CoreCLR.builds" /verbosity:minimal "/fileloggerparameters:Verbosity=normal;LogFile=$__LogsDir/Nuget_$__BuildOS__$__BuildArch__$__BuildType.log" /t:Build /p:__BuildOS=$__BuildOS /p:__BuildArch=$__BuildArch /p:__BuildType=$__BuildType /p:__IntermediatesDir=$__IntermediatesDir /p:UseRoslynCompiler=true /p:BuildNugetPackage=false /p:UseSharedCompilation=false
 304
 305     if [ $? -ne 0 ]; then
 306         echo "Failed to generate Nuget packages."
 307         exit 1
 308     fi
 309 }
 310
 311 echo "Commencing CoreCLR Repo build"
 312
 313 # Argument types supported by this script:
 314 #
 315 # Build architecture - valid values are: x64, ARM.
 316 # Build Type         - valid values are: Debug, Checked, Release
 317 #
 318 # Set the default arguments for build
 319
 320 # Obtain the location of the bash script to figure out where the root of the repo is.
 321 __ProjectRoot="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
 322
 323 # Use uname to determine what the CPU is.
 324 CPUName=$(uname -p)
 325 # Some Linux platforms report unknown for platform, but the arch for machine.
 326 if [ $CPUName == "unknown" ]; then
 327     CPUName=$(uname -m)
 328 fi
 329
 330 case $CPUName in
 331     i686)
 332         echo "Unsupported CPU $CPUName detected, build might not succeed!"
 333         __BuildArch=x86
 334         __HostArch=x86
 335         ;;
 336
 337     x86_64)
 338         __BuildArch=x64
 339         __HostArch=x64
 340         ;;
 341
 342     armv7l)
 343         echo "Unsupported CPU $CPUName detected, build might not succeed!"
 344         __BuildArch=arm
 345         __HostArch=arm
 346         ;;
 347
 348     aarch64)
 349         echo "Unsupported CPU $CPUName detected, build might not succeed!"
 350         __BuildArch=arm64
 351         __HostArch=arm64
 352         ;;
 353
 354     *)
 355         echo "Unknown CPU $CPUName detected, configuring as if for x64"
 356         __BuildArch=x64
 357         __HostArch=x64
 358         ;;
 359 esac
 360
 361 # Use uname to determine what the OS is.
 362 OSName=$(uname -s)
 363 case $OSName in
 364     Linux)
 365         __BuildOS=Linux
 366         ;;
 367
 368     Darwin)
 369         __BuildOS=OSX
 370         ;;
 371
 372     FreeBSD)
 373         __BuildOS=FreeBSD
 374         ;;
 375
 376     OpenBSD)
 377         __BuildOS=OpenBSD
 378         ;;
 379
 380     NetBSD)
 381         __BuildOS=NetBSD
 382         ;;
 383
 384     SunOS)
 385         __BuildOS=SunOS
 386         ;;
 387
 388     *)
 389         echo "Unsupported OS $OSName detected, configuring as if for Linux"
 390         __BuildOS=Linux
 391         ;;
 392 esac
 393
 394 __BuildType=Debug
 395 __CodeCoverage=
 396 __IncludeTests=Include_Tests
 397
 398 # Set the various build properties here so that CMake and MSBuild can pick them up
 399 __ProjectDir="$__ProjectRoot"
 400 __SourceDir="$__ProjectDir/src"
 401 __PackagesDir="$__ProjectDir/packages"
 402 __RootBinDir="$__ProjectDir/bin"
 403 __LogsDir="$__RootBinDir/Logs"
 404 __UnprocessedBuildArgs=
 405 __MSBCleanBuildArgs=
 406 __UseNinja=0
 407 __ConfigureOnly=0
 408 __SkipConfigure=0
 409 __SkipCoreCLR=0
 410 __SkipMSCorLib=0
 411 __CleanBuild=0
 412 __VerboseBuild=0
 413 __SignTypeReal=""
 414 __CrossBuild=0
 415 __ClangMajorVersion=3
 416 __ClangMinorVersion=5
 417 __MSBuildPath=$__ProjectRoot/Tools/MSBuild.exe
 418 __NuGetPath="$__PackagesDir/NuGet.exe"
 419 __DistroName=""
 420 __cmakeargs=""
 421
 422 while :; do
 423     if [ $# -le 0 ]; then
 424         break
 425     fi
 426
 427     lowerI="$(echo $1 | awk '{print tolower($0)}')"
 428     case $lowerI in
 429         -\?|-h|--help)
 430             usage
 431             exit 1
 432             ;;
 433
 434         x86)
 435             __BuildArch=x86
 436             ;;
 437
 438         x64)
 439             __BuildArch=x64
 440             ;;
 441
 442         arm)
 443             __BuildArch=arm
 444             ;;
 445
 446         arm64)
 447             __BuildArch=arm64
 448             ;;
 449
 450         debug)
 451             __BuildType=Debug
 452             ;;
 453
 454         checked)
 455             __BuildType=Checked
 456             ;;
 457
 458         release)
 459             __BuildType=Release
 460             ;;
 461
 462         coverage)
 463             __CodeCoverage=Coverage
 464             ;;
 465
 466         clean)
 467             __CleanBuild=1
 468             ;;
 469
 470         verbose)
 471             __VerboseBuild=1
 472             ;;
 473
 474         cross)
 475             __CrossBuild=1
 476             ;;
 477
 478         clang3.5)
 479             __ClangMajorVersion=3
 480             __ClangMinorVersion=5
 481             ;;
 482
 483         clang3.6)
 484             __ClangMajorVersion=3
 485             __ClangMinorVersion=6
 486             ;;
 487
 488         clang3.7)
 489             __ClangMajorVersion=3
 490             __ClangMinorVersion=7
 491             ;;
 492
 493         ninja)
 494             __UseNinja=1
 495             ;;
 496
 497         configureonly)
 498             __ConfigureOnly=1
 499             __SkipCoreCLR=1
 500             __SkipMSCorLib=1
 501             __IncludeTests=
 502             ;;
 503
 504         skipconfigure)
 505             __SkipConfigure=1
 506             ;;
 507
 508         skipnative)
 509             # Use "skipnative" to use the same option name as build.cmd.
 510             __SkipCoreCLR=1
 511             ;;
 512
 513         skipcoreclr)
 514             # Accept "skipcoreclr" for backwards-compatibility.
 515             __SkipCoreCLR=1
 516             ;;
 517
 518         skipmscorlib)
 519             __SkipMSCorLib=1
 520             ;;
 521
 522         includetests)
 523             ;;
 524
 525         skiptests)
 526             __IncludeTests=
 527             ;;
 528
 529         disableoss)
 530             __SignTypeReal="/p:SignType=real"
 531             ;;
 532
 533         cmakeargs)
 534             if [ -n "$2" ]; then
 535                 __cmakeargs="$2"
 536                 shift
 537             else
 538                 echo "ERROR: 'cmakeargs' requires a non-empty option argument"
 539                 exit 1
 540             fi
 541             ;;
 542
 543         *)
 544             __UnprocessedBuildArgs="$__UnprocessedBuildArgs $1"
 545             ;;
 546     esac
 547
 548     shift
 549 done
 550
 551 if [[ $__ConfigureOnly == 1 && $__SkipConfigure == 1 ]]; then
 552     echo "configureonly and skipconfigure are mutually exclusive!"
 553     exit 1
 554 fi
 555
 556 # init the distro name
 557 initDistroName
 558
 559 # Set the remaining variables based upon the determined build configuration
 560 __BinDir="$__RootBinDir/Product/$__BuildOS.$__BuildArch.$__BuildType"
 561 __PackagesBinDir="$__BinDir/.nuget"
 562 __ToolsDir="$__RootBinDir/tools"
 563 __TestWorkingDir="$__RootBinDir/tests/$__BuildOS.$__BuildArch.$__BuildType"
 564 export __IntermediatesDir="$__RootBinDir/obj/$__BuildOS.$__BuildArch.$__BuildType"
 565 __TestIntermediatesDir="$__RootBinDir/tests/obj/$__BuildOS.$__BuildArch.$__BuildType"
 566 __isMSBuildOnNETCoreSupported=0
 567
 568 # Init if MSBuild for .NET Core is supported for this platform
 569 isMSBuildOnNETCoreSupported
 570
 571 # CI_SPECIFIC - On CI machines, $HOME may not be set. In such a case, create a subfolder and set the variable to set.
 572 # This is needed by CLI to function.
 573 if [ -z "$HOME" ]; then
 574     if [ ! -d "$__ProjectDir/temp_home" ]; then
 575         mkdir temp_home
 576     fi
 577     export HOME=$__ProjectDir/temp_home
 578     echo "HOME not defined; setting it to $HOME"
 579 fi
 580
 581 # Specify path to be set for CMAKE_INSTALL_PREFIX.
 582 # This is where all built CoreClr libraries will copied to.
 583 export __CMakeBinDir="$__BinDir"
 584
 585 # Configure environment if we are doing a clean build.
 586 if [ $__CleanBuild == 1 ]; then
 587     clean
 588 fi
 589
 590 # Configure environment if we are doing a verbose build
 591 if [ $__VerboseBuild == 1 ]; then
 592     export VERBOSE=1
 593 fi
 594
 595 # Configure environment if we are doing a cross compile.
 596 if [ $__CrossBuild == 1 ]; then
 597     export CROSSCOMPILE=1
 598     if ! [[ -n "$ROOTFS_DIR" ]]; then
 599         export ROOTFS_DIR="$__ProjectRoot/cross/rootfs/$__BuildArch"
 600     fi
 601 fi
 602
 603 # Make the directories necessary for build if they don't exist
 604
 605 setup_dirs
 606
 607 # Check prereqs.
 608
 609 check_prereqs
 610
 611 # Build the coreclr (native) components.
 612
 613 build_coreclr
 614
 615 # Build mscorlib.
 616
 617 build_mscorlib
 618
 619 # Generate nuget packages
 620
 621 generate_NugetPackages
 622
 623
 624 # Build complete
 625
 626 echo "Repo successfully built."
 627 echo "Product binaries are available at $__BinDir"
 628 exit 0