build.sh

   1 #!/usr/bin/env bash
   2
   3 PYTHON=${PYTHON:-python}
   4
   5 usage()
   6 {
   7     echo "Usage: $0 [BuildArch] [BuildType] [clean] [verbose] [coverage] [cross] [clangx.y] [ninja] [configureonly] [skipconfigure] [skipnative] [skipmscorlib] [skiptests] [cmakeargs]"
   8     echo "BuildArch can be: x64, x86, arm, arm64"
   9     echo "BuildType can be: Debug, Checked, Release"
  10     echo "clean - optional argument to force a clean build."
  11     echo "verbose - optional argument to enable verbose build output."
  12     echo "coverage - optional argument to enable code coverage build (currently supported only for Linux and OSX)."
  13     echo "ninja - target ninja instead of GNU make"
  14     echo "clangx.y - optional argument to build using clang version x.y."
  15     echo "cross - optional argument to signify cross compilation,"
  16     echo "      - will use ROOTFS_DIR environment variable if set."
  17     echo "configureonly - do not perform any builds; just configure the build."
  18     echo "skipconfigure - skip build configuration."
  19     echo "skipnative - do not build native components."
  20     echo "skipmscorlib - do not build mscorlib.dll even if mono is installed."
  21     echo "skiptests - skip the tests in the 'tests' subdirectory."
  22     echo "cmakeargs - user-settable additional arguments passed to CMake."
  23
  24     exit 1
  25 }
  26
  27 initDistroName()
  28 {
  29     if [ "$__BuildOS" == "Linux" ]; then
  30         # Detect Distro
  31         if [ "$(cat /etc/*-release | grep -cim1 ubuntu)" -eq 1 ]; then
  32             export __DistroName=ubuntu
  33         elif [ "$(cat /etc/*-release | grep -cim1 centos)" -eq 1 ]; then
  34             export __DistroName=rhel
  35         elif [ "$(cat /etc/*-release | grep -cim1 rhel)" -eq 1 ]; then
  36             export __DistroName=rhel
  37         elif [ "$(cat /etc/*-release | grep -cim1 debian)" -eq 1 ]; then
  38             export __DistroName=debian
  39         else
  40             export __DistroName=""
  41         fi
  42     fi
  43 }
  44
  45 setup_dirs()
  46 {
  47     echo Setting up directories for build
  48
  49     mkdir -p "$__RootBinDir"
  50     mkdir -p "$__BinDir"
  51     mkdir -p "$__LogsDir"
  52     mkdir -p "$__IntermediatesDir"
  53 }
  54
  55 # Performs "clean build" type actions (deleting and remaking directories)
  56
  57 clean()
  58 {
  59     echo Cleaning previous output for the selected configuration
  60     rm -rf "$__BinDir"
  61     rm -rf "$__IntermediatesDir"
  62
  63     rm -rf "$__TestWorkingDir"
  64     rm -rf "$__TestIntermediatesDir"
  65
  66     rm -rf "$__LogsDir/*_$__BuildOS__$__BuildArch__$__BuildType.*"
  67 }
  68
  69 # Check the system to ensure the right prereqs are in place
  70
  71 check_prereqs()
  72 {
  73     echo "Checking prerequisites..."
  74
  75     # Check presence of CMake on the path
  76     hash cmake 2>/dev/null || { echo >&2 "Please install cmake before running this script"; exit 1; }
  77
  78     # Check for clang
  79     hash clang-$__ClangMajorVersion.$__ClangMinorVersion 2>/dev/null ||  hash clang$__ClangMajorVersion$__ClangMinorVersion 2>/dev/null ||  hash clang 2>/dev/null || { echo >&2 "Please install clang before running this script"; exit 1; }
  80
  81 }
  82
  83 build_coreclr()
  84 {
  85
  86 # Event Logging Infrastructure
  87    __GeneratedIntermediate="$__IntermediatesDir/Generated"
  88    __GeneratedIntermediateEventProvider="$__GeneratedIntermediate/eventprovider_new"
  89     if [[ -d "$__GeneratedIntermediateEventProvider" ]]; then
  90         rm -rf  "$__GeneratedIntermediateEventProvider"
  91     fi
  92
  93     if [[ ! -d "$__GeneratedIntermediate/eventprovider" ]]; then
  94         mkdir -p "$__GeneratedIntermediate/eventprovider"
  95     fi
  96
  97     mkdir -p "$__GeneratedIntermediateEventProvider"
  98     if [[ $__SkipCoreCLR == 0 || $__ConfigureOnly == 1 ]]; then
  99         echo "Laying out dynamically generated files consumed by the build system "
 100         echo "Laying out dynamically generated Event Logging Test files"
 101         $PYTHON -B -Wall -Werror "$__ProjectRoot/src/scripts/genXplatEventing.py" --man "$__ProjectRoot/src/vm/ClrEtwAll.man" --exc "$__ProjectRoot/src/vm/ClrEtwAllMeta.lst" --testdir "$__GeneratedIntermediateEventProvider/tests"
 102
 103         if  [[ $? != 0 ]]; then
 104             exit
 105         fi
 106
 107         #determine the logging system
 108         case $__BuildOS in
 109             Linux)
 110                 echo "Laying out dynamically generated Event Logging Implementation of Lttng"
 111                 $PYTHON -B -Wall -Werror "$__ProjectRoot/src/scripts/genXplatLttng.py" --man "$__ProjectRoot/src/vm/ClrEtwAll.man" --intermediate "$__GeneratedIntermediateEventProvider"
 112                 if  [[ $? != 0 ]]; then
 113                     exit
 114                 fi
 115                 ;;
 116             *)
 117                 ;;
 118         esac
 119     fi
 120
 121     echo "Cleaning the temp folder of dynamically generated Event Logging files"
 122     $PYTHON -B -Wall -Werror -c "import sys;sys.path.insert(0,\"$__ProjectRoot/src/scripts\"); from Utilities import *;UpdateDirectory(\"$__GeneratedIntermediate/eventprovider\",\"$__GeneratedIntermediateEventProvider\")"
 123     if  [[ $? != 0 ]]; then
 124         exit
 125     fi
 126
 127     rm -rf "$__GeneratedIntermediateEventProvider"
 128
 129     # All set to commence the build
 130
 131     echo "Commencing build of native components for $__BuildOS.$__BuildArch.$__BuildType in $__IntermediatesDir"
 132
 133     cd "$__IntermediatesDir"
 134
 135     generator=""
 136     buildFile="Makefile"
 137     buildTool="make"
 138     if [ $__UseNinja == 1 ]; then
 139         generator="ninja"
 140         buildFile="build.ninja"
 141         buildTool="ninja"
 142     fi
 143
 144     if [ $__SkipConfigure == 0 ]; then
 145         # Regenerate the CMake solution
 146         echo "Invoking \"$__ProjectRoot/src/pal/tools/gen-buildsys-clang.sh\" \"$__ProjectRoot\" $__ClangMajorVersion $__ClangMinorVersion $__BuildArch $__BuildType $__CodeCoverage $__IncludeTests $generator $__cmakeargs"
 147         "$__ProjectRoot/src/pal/tools/gen-buildsys-clang.sh" "$__ProjectRoot" $__ClangMajorVersion $__ClangMinorVersion $__BuildArch $__BuildType $__CodeCoverage $__IncludeTests $generator "$__cmakeargs"
 148     fi
 149
 150     if [ $__SkipCoreCLR == 1 ]; then
 151         echo "Skipping CoreCLR build."
 152         return
 153     fi
 154
 155     # Check that the makefiles were created.
 156
 157     if [ ! -f "$__IntermediatesDir/$buildFile" ]; then
 158         echo "Failed to generate native component build project!"
 159         exit 1
 160     fi
 161
 162     # Get the number of processors available to the scheduler
 163     # Other techniques such as `nproc` only get the number of
 164     # processors available to a single process.
 165     if [ `uname` = "FreeBSD" ]; then
 166         NumProc=`sysctl hw.ncpu | awk '{ print $2+1 }'`
 167     elif [ `uname` = "NetBSD" ]; then
 168         NumProc=$(($(getconf NPROCESSORS_ONLN)+1))
 169     else
 170         NumProc=$(($(getconf _NPROCESSORS_ONLN)+1))
 171     fi
 172
 173     # Build CoreCLR
 174
 175     echo "Executing $buildTool install -j $NumProc $__UnprocessedBuildArgs"
 176
 177     $buildTool install -j $NumProc $__UnprocessedBuildArgs
 178     if [ $? != 0 ]; then
 179         echo "Failed to build coreclr components."
 180         exit 1
 181     fi
 182 }
 183
 184 restoreBuildTools()
 185 {
 186     echo "Restoring BuildTools..."
 187     $__ProjectRoot/init-tools.sh
 188     if [ $? -ne 0 ]; then
 189         echo "Failed to restore BuildTools."
 190         exit 1
 191     fi
 192 }
 193
 194 isMSBuildOnNETCoreSupported()
 195 {
 196     # This needs to be updated alongwith corresponding changes to netci.groovy.
 197     __isMSBuildOnNETCoreSupported=0
 198
 199     if [ "$__BuildArch" == "x64" ]; then
 200         if [ "$__BuildOS" == "Linux" ]; then
 201             if [ "$__DistroName" == "ubuntu" ]; then
 202                 __isMSBuildOnNETCoreSupported=1
 203             elif [ "$__DistroName" == "rhel" ]; then
 204                 __isMSBuildOnNETCoreSupported=1
 205             elif [ "$__DistroName" == "debian" ]; then
 206                 __isMSBuildOnNETCoreSupported=1
 207             fi
 208         elif [ "$__BuildOS" == "OSX" ]; then
 209             __isMSBuildOnNETCoreSupported=1
 210         fi
 211     fi
 212 }
 213
 214 build_mscorlib()
 215 {
 216
 217     if [ $__isMSBuildOnNETCoreSupported == 0 ]; then
 218         echo "Mscorlib.dll build unsupported."
 219         return
 220     fi
 221
 222     if [ $__SkipMSCorLib == 1 ]; then
 223        echo "Skipping building mscorlib."
 224        return
 225     fi
 226
 227     hash mono 2> /dev/null || { echo >&2 "Skipping mscorlib.dll build since Mono is not installed."; __SkipMSCorLib=1; return; }
 228
 229     # Restore buildTools
 230
 231     restoreBuildTools
 232
 233     echo "Commencing build of mscorlib components for $__BuildOS.$__BuildArch.$__BuildType"
 234
 235     # Invoke MSBuild
 236     $__ProjectRoot/Tools/corerun "$__MSBuildPath" /nologo "$__ProjectRoot/build.proj" /verbosity:minimal "/fileloggerparameters:Verbosity=normal;LogFile=$__LogsDir/MSCorLib_$__BuildOS__$__BuildArch__$__BuildType.log" /t:Build /p:__BuildOS=$__BuildOS /p:__BuildArch=$__BuildArch /p:__BuildType=$__BuildType /p:__IntermediatesDir=$__IntermediatesDir /p:UseRoslynCompiler=true /p:BuildNugetPackage=false /p:UseSharedCompilation=false
 237
 238     if [ $? -ne 0 ]; then
 239         echo "Failed to build mscorlib."
 240         exit 1
 241     fi
 242
 243     if [ $__SkipCoreCLR == 0 -a -e $__BinDir/crossgen ]; then
 244         echo "Generating native image for mscorlib."
 245         $__BinDir/crossgen $__BinDir/mscorlib.dll
 246         if [ $? -ne 0 ]; then
 247             echo "Failed to generate native image for mscorlib."
 248             exit 1
 249         fi
 250     fi
 251 }
 252
 253 generate_NugetPackages()
 254 {
 255     # We can only generate nuget package if we also support building mscorlib as part of this build.
 256     if [ $__isMSBuildOnNETCoreSupported == 0 ]; then
 257         echo "Microsoft.NETCore.Runtime.CoreCLR nuget package generation unsupported."
 258         return
 259     fi
 260
 261     # Since we can build mscorlib for this OS, did we build the native components as well?
 262     if [ $__SkipCoreCLR == 1 ]; then
 263         echo "Unable to generate Microsoft.NETCore.Runtime.CoreCLR nuget package since native components were not built."
 264         return
 265     fi
 266
 267     if [ $__SkipMSCorLib == 1 ]; then
 268        echo "Unable to generate Microsoft.NETCore.Runtime.CoreCLR nuget package since mscorlib was not built."
 269        return
 270     fi
 271
 272     echo "Generating nuget packages for "$__BuildOS
 273
 274     # Invoke MSBuild
 275     $__ProjectRoot/Tools/corerun "$__MSBuildPath" /nologo "$__ProjectRoot/src/.nuget/Microsoft.NETCore.Runtime.CoreCLR/Microsoft.NETCore.Runtime.CoreCLR.builds" /verbosity:minimal "/fileloggerparameters:Verbosity=normal;LogFile=$__LogsDir/Nuget_$__BuildOS__$__BuildArch__$__BuildType.log" /t:Build /p:__BuildOS=$__BuildOS /p:__BuildArch=$__BuildArch /p:__BuildType=$__BuildType /p:__IntermediatesDir=$__IntermediatesDir /p:UseRoslynCompiler=true /p:BuildNugetPackage=false /p:UseSharedCompilation=false
 276
 277     if [ $? -ne 0 ]; then
 278         echo "Failed to generate Nuget packages."
 279         exit 1
 280     fi
 281 }
 282
 283 echo "Commencing CoreCLR Repo build"
 284
 285 # Argument types supported by this script:
 286 #
 287 # Build architecture - valid values are: x64, ARM.
 288 # Build Type         - valid values are: Debug, Checked, Release
 289 #
 290 # Set the default arguments for build
 291
 292 # Obtain the location of the bash script to figure out where the root of the repo is.
 293 __ProjectRoot="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
 294
 295 # Use uname to determine what the CPU is.
 296 CPUName=$(uname -p)
 297 # Some Linux platforms report unknown for platform, but the arch for machine.
 298 if [ $CPUName == "unknown" ]; then
 299     CPUName=$(uname -m)
 300 fi
 301
 302 case $CPUName in
 303     i686)
 304         echo "Unsupported CPU $CPUName detected, build might not succeed!"
 305         __BuildArch=x86
 306         ;;
 307
 308     x86_64)
 309         __BuildArch=x64
 310         ;;
 311
 312     armv7l)
 313         echo "Unsupported CPU $CPUName detected, build might not succeed!"
 314         __BuildArch=arm
 315         ;;
 316
 317     aarch64)
 318         echo "Unsupported CPU $CPUName detected, build might not succeed!"
 319         __BuildArch=arm64
 320         ;;
 321
 322     *)
 323         echo "Unknown CPU $CPUName detected, configuring as if for x64"
 324         __BuildArch=x64
 325         ;;
 326 esac
 327
 328 # Use uname to determine what the OS is.
 329 OSName=$(uname -s)
 330 case $OSName in
 331     Linux)
 332         __BuildOS=Linux
 333         ;;
 334
 335     Darwin)
 336         __BuildOS=OSX
 337         ;;
 338
 339     FreeBSD)
 340         __BuildOS=FreeBSD
 341         ;;
 342
 343     OpenBSD)
 344         __BuildOS=OpenBSD
 345         ;;
 346
 347     NetBSD)
 348         __BuildOS=NetBSD
 349         ;;
 350
 351     SunOS)
 352         __BuildOS=SunOS
 353         ;;
 354
 355     *)
 356         echo "Unsupported OS $OSName detected, configuring as if for Linux"
 357         __BuildOS=Linux
 358         ;;
 359 esac
 360
 361 __BuildType=Debug
 362 __CodeCoverage=
 363 __IncludeTests=Include_Tests
 364
 365 # Set the various build properties here so that CMake and MSBuild can pick them up
 366 __ProjectDir="$__ProjectRoot"
 367 __SourceDir="$__ProjectDir/src"
 368 __PackagesDir="$__ProjectDir/packages"
 369 __RootBinDir="$__ProjectDir/bin"
 370 __LogsDir="$__RootBinDir/Logs"
 371 __UnprocessedBuildArgs=
 372 __MSBCleanBuildArgs=
 373 __UseNinja=0
 374 __ConfigureOnly=0
 375 __SkipConfigure=0
 376 __SkipCoreCLR=0
 377 __SkipMSCorLib=0
 378 __CleanBuild=0
 379 __VerboseBuild=0
 380 __CrossBuild=0
 381 __ClangMajorVersion=3
 382 __ClangMinorVersion=5
 383 __MSBuildPath=$__ProjectRoot/Tools/MSBuild.exe
 384 __NuGetPath="$__PackagesDir/NuGet.exe"
 385 __DistroName=""
 386 __cmakeargs=""
 387
 388 while :; do
 389     if [ $# -le 0 ]; then
 390         break
 391     fi
 392
 393     lowerI="$(echo $1 | awk '{print tolower($0)}')"
 394     case $lowerI in
 395         -\?|-h|--help)
 396             usage
 397             exit 1
 398             ;;
 399
 400         x86)
 401             __BuildArch=x86
 402             ;;
 403
 404         x64)
 405             __BuildArch=x64
 406             ;;
 407
 408         arm)
 409             __BuildArch=arm
 410             ;;
 411
 412         arm64)
 413             __BuildArch=arm64
 414             ;;
 415
 416         debug)
 417             __BuildType=Debug
 418             ;;
 419
 420         checked)
 421             __BuildType=Checked
 422             ;;
 423
 424         release)
 425             __BuildType=Release
 426             ;;
 427
 428         coverage)
 429             __CodeCoverage=Coverage
 430             ;;
 431
 432         clean)
 433             __CleanBuild=1
 434             ;;
 435
 436         verbose)
 437             __VerboseBuild=1
 438             ;;
 439
 440         cross)
 441             __CrossBuild=1
 442             ;;
 443
 444         clang3.5)
 445             __ClangMajorVersion=3
 446             __ClangMinorVersion=5
 447             ;;
 448
 449         clang3.6)
 450             __ClangMajorVersion=3
 451             __ClangMinorVersion=6
 452             ;;
 453
 454         clang3.7)
 455             __ClangMajorVersion=3
 456             __ClangMinorVersion=7
 457             ;;
 458
 459         ninja)
 460             __UseNinja=1
 461             ;;
 462
 463         configureonly)
 464             __ConfigureOnly=1
 465             __SkipCoreCLR=1
 466             __SkipMSCorLib=1
 467             __IncludeTests=
 468             ;;
 469
 470         skipconfigure)
 471             __SkipConfigure=1
 472             ;;
 473
 474         skipnative)
 475             # Use "skipnative" to use the same option name as build.cmd.
 476             __SkipCoreCLR=1
 477             ;;
 478
 479         skipcoreclr)
 480             # Accept "skipcoreclr" for backwards-compatibility.
 481             __SkipCoreCLR=1
 482             ;;
 483
 484         skipmscorlib)
 485             __SkipMSCorLib=1
 486             ;;
 487
 488         includetests)
 489             ;;
 490
 491         skiptests)
 492             __IncludeTests=
 493             ;;
 494
 495         cmakeargs)
 496             if [ -n "$2" ]; then
 497                 __cmakeargs="$2"
 498                 shift
 499             else
 500                 echo "ERROR: 'cmakeargs' requires a non-empty option argument"
 501                 exit 1
 502             fi
 503             ;;
 504
 505         *)
 506             __UnprocessedBuildArgs="$__UnprocessedBuildArgs $1"
 507             ;;
 508     esac
 509
 510     shift
 511 done
 512
 513 if [[ $__ConfigureOnly == 1 && $__SkipConfigure == 1 ]]; then
 514     echo "configureonly and skipconfigure are mutually exclusive!"
 515     exit 1
 516 fi
 517
 518 # init the distro name
 519 initDistroName
 520
 521 # Set the remaining variables based upon the determined build configuration
 522 __BinDir="$__RootBinDir/Product/$__BuildOS.$__BuildArch.$__BuildType"
 523 __PackagesBinDir="$__BinDir/.nuget"
 524 __ToolsDir="$__RootBinDir/tools"
 525 __TestWorkingDir="$__RootBinDir/tests/$__BuildOS.$__BuildArch.$__BuildType"
 526 export __IntermediatesDir="$__RootBinDir/obj/$__BuildOS.$__BuildArch.$__BuildType"
 527 __TestIntermediatesDir="$__RootBinDir/tests/obj/$__BuildOS.$__BuildArch.$__BuildType"
 528 __isMSBuildOnNETCoreSupported=0
 529
 530 # Init if MSBuild for .NET Core is supported for this platform
 531 isMSBuildOnNETCoreSupported
 532
 533 # CI_SPECIFIC - On CI machines, $HOME may not be set. In such a case, create a subfolder and set the variable to set.
 534 # This is needed by CLI to function.
 535 if [ -z "$HOME" ]; then
 536     if [ ! -d "$__ProjectDir/temp_home" ]; then
 537         mkdir temp_home
 538     fi
 539     export HOME=$__ProjectDir/temp_home
 540     echo "HOME not defined; setting it to $HOME"
 541 fi
 542
 543 # Specify path to be set for CMAKE_INSTALL_PREFIX.
 544 # This is where all built CoreClr libraries will copied to.
 545 export __CMakeBinDir="$__BinDir"
 546
 547 # Configure environment if we are doing a clean build.
 548 if [ $__CleanBuild == 1 ]; then
 549     clean
 550 fi
 551
 552 # Configure environment if we are doing a verbose build
 553 if [ $__VerboseBuild == 1 ]; then
 554     export VERBOSE=1
 555 fi
 556
 557 # Configure environment if we are doing a cross compile.
 558 if [ $__CrossBuild == 1 ]; then
 559     export CROSSCOMPILE=1
 560     if ! [[ -n "$ROOTFS_DIR" ]]; then
 561         export ROOTFS_DIR="$__ProjectRoot/cross/rootfs/$__BuildArch"
 562     fi
 563 fi
 564
 565 # Make the directories necessary for build if they don't exist
 566
 567 setup_dirs
 568
 569 # Check prereqs.
 570
 571 check_prereqs
 572
 573 # Build the coreclr (native) components.
 574
 575 build_coreclr
 576
 577 # Build mscorlib.
 578
 579 build_mscorlib
 580
 581 # Generate nuget packages
 582
 583 generate_NugetPackages
 584
 585
 586 # Build complete
 587
 588 echo "Repo successfully built."
 589 echo "Product binaries are available at $__BinDir"
 590 exit 0