Modify rebuild scenario.
[platform/upstream/coreclr.git] / build.sh
1 #!/usr/bin/env bash
2
3 # resolve python-version to use
4 if [ "$PYTHON" == "" ] ; then
5     if which python >/dev/null 2>&1
6     then
7        PYTHON=python
8     elif which python2 >/dev/null 2>&1
9     then
10        PYTHON=python2
11     elif which python2.7 >/dev/null 2>&1
12     then
13        PYTHON=python2.7
14     else
15        echo "Unable to locate build-dependency python2.x!" 1>&2
16        exit 1
17     fi
18 fi
19
20 # validate python-dependency
21 # useful in case of explicitly set option.
22 if ! which $PYTHON > /dev/null 2>&1
23 then
24    echo "Unable to locate build-dependency python2.x ($PYTHON)!" 1>&2
25    exit 1
26 fi
27
28 usage()
29 {
30     echo "Usage: $0 [BuildArch] [BuildType] [verbose] [coverage] [cross] [clangx.y] [ninja] [configureonly] [skipconfigure] [skipnative] [skipmscorlib] [skiptests] [cmakeargs] [bindir]"
31     echo "BuildArch can be: x64, x86, arm, arm-softfp, arm64"
32     echo "BuildType can be: debug, checked, release"
33     echo "coverage - optional argument to enable code coverage build (currently supported only for Linux and OSX)."
34     echo "ninja - target ninja instead of GNU make"
35     echo "clangx.y - optional argument to build using clang version x.y."
36     echo "cross - optional argument to signify cross compilation,"
37     echo "      - will use ROOTFS_DIR environment variable if set."
38     echo "configureonly - do not perform any builds; just configure the build."
39     echo "skipconfigure - skip build configuration."
40     echo "skipnative - do not build native components."
41     echo "skipmscorlib - do not build mscorlib.dll."
42     echo "skiptests - skip the tests in the 'tests' subdirectory."
43     echo "skipnuget - skip building nuget packages."
44     echo "verbose - optional argument to enable verbose build output."
45     echo "-skiprestore: skip restoring packages ^(default: packages are restored during build^)."
46         echo "-disableoss: Disable Open Source Signing for System.Private.CoreLib."
47         echo "-sequential: force a non-parallel build ^(default is to build in parallel"
48         echo "   using all processors^)."
49         echo "-officialbuildid=^<ID^>: specify the official build ID to be used by this build."
50         echo "-Rebuild: passes /t:rebuild to the build projects."
51     echo "skipgenerateversion - disable version generation even if MSBuild is supported."
52     echo "cmakeargs - user-settable additional arguments passed to CMake."
53     echo "bindir - output directory (defaults to $__ProjectRoot/bin)"
54
55     exit 1
56 }
57
58 initDistroRid()
59 {
60     if [ "$__BuildOS" == "Linux" ]; then
61         if [ ! -e /etc/os-release ]; then
62             echo "WARNING: Can not determine runtime id for current distro."
63             export __DistroRid=""
64         else
65             source /etc/os-release
66             export __DistroRid="$ID.$VERSION_ID-$__HostArch"
67         fi
68     fi
69 }
70
71 setup_dirs()
72 {
73     echo Setting up directories for build
74
75     mkdir -p "$__RootBinDir"
76     mkdir -p "$__BinDir"
77     mkdir -p "$__LogsDir"
78     mkdir -p "$__IntermediatesDir"
79 }
80
81 # Check the system to ensure the right prereqs are in place
82
83 check_prereqs()
84 {
85     echo "Checking prerequisites..."
86
87     # Check presence of CMake on the path
88     hash cmake 2>/dev/null || { echo >&2 "Please install cmake before running this script"; exit 1; }
89
90     # Check for clang
91     hash clang-$__ClangMajorVersion.$__ClangMinorVersion 2>/dev/null ||  hash clang$__ClangMajorVersion$__ClangMinorVersion 2>/dev/null ||  hash clang 2>/dev/null || { echo >&2 "Please install clang-$__ClangMajorVersion.$__ClangMinorVersion before running this script"; exit 1; }
92
93 }
94
95 build_coreclr()
96 {
97
98     if [ $__SkipCoreCLR == 1 ]; then
99         echo "Skipping CoreCLR build."
100         return
101     fi
102
103 # Event Logging Infrastructure
104    __GeneratedIntermediate="$__IntermediatesDir/Generated"
105    __GeneratedIntermediateEventProvider="$__GeneratedIntermediate/eventprovider_new"
106     if [[ -d "$__GeneratedIntermediateEventProvider" ]]; then
107         rm -rf  "$__GeneratedIntermediateEventProvider"
108     fi
109
110     if [[ ! -d "$__GeneratedIntermediate/eventprovider" ]]; then
111         mkdir -p "$__GeneratedIntermediate/eventprovider"
112     fi
113
114     mkdir -p "$__GeneratedIntermediateEventProvider"
115     if [[ $__SkipCoreCLR == 0 || $__ConfigureOnly == 1 ]]; then
116         echo "Laying out dynamically generated files consumed by the build system "
117         echo "Laying out dynamically generated Event Logging Test files"
118         $PYTHON -B -Wall -Werror "$__ProjectRoot/src/scripts/genXplatEventing.py" --man "$__ProjectRoot/src/vm/ClrEtwAll.man" --exc "$__ProjectRoot/src/vm/ClrEtwAllMeta.lst" --testdir "$__GeneratedIntermediateEventProvider/tests"
119
120         if  [[ $? != 0 ]]; then
121             exit
122         fi
123
124         #determine the logging system
125         case $__BuildOS in
126             Linux)
127                 echo "Laying out dynamically generated Event Logging Implementation of Lttng"
128                 $PYTHON -B -Wall -Werror "$__ProjectRoot/src/scripts/genXplatLttng.py" --man "$__ProjectRoot/src/vm/ClrEtwAll.man" --intermediate "$__GeneratedIntermediateEventProvider"
129                 if  [[ $? != 0 ]]; then
130                     exit
131                 fi
132                 ;;
133             *)
134                 ;;
135         esac
136     fi
137
138     echo "Cleaning the temp folder of dynamically generated Event Logging files"
139     $PYTHON -B -Wall -Werror -c "import sys;sys.path.insert(0,\"$__ProjectRoot/src/scripts\"); from Utilities import *;UpdateDirectory(\"$__GeneratedIntermediate/eventprovider\",\"$__GeneratedIntermediateEventProvider\")"
140     if  [[ $? != 0 ]]; then
141         exit
142     fi
143
144     rm -rf "$__GeneratedIntermediateEventProvider"
145
146     # All set to commence the build
147
148     echo "Commencing build of native components for $__BuildOS.$__BuildArch.$__BuildType in $__IntermediatesDir"
149
150     generator=""
151     buildFile="Makefile"
152     buildTool="make"
153     if [ $__UseNinja == 1 ]; then
154         generator="ninja"
155         buildFile="build.ninja"
156         buildTool="ninja"
157     fi
158
159     if [ $__SkipConfigure == 0 ]; then
160         # if msbuild is not supported, then set __SkipGenerateVersion to 1
161         if [ $__isMSBuildOnNETCoreSupported == 0 ]; then __SkipGenerateVersion=1; fi
162         # Drop version.c file
163         __versionSourceFile=$__IntermediatesDir/version.cpp
164         if [ $__SkipGenerateVersion == 0 ]; then
165             "$__ProjectRoot/run.sh" build -Project=$__ProjectDir/build.proj -generateHeaderUnix -NativeVersionSourceFile=$__versionSourceFile $__RunArgs $__UnprocessedBuildArgs
166         else
167             __versionSourceLine="static char sccsid[] __attribute__((used)) = \"@(#)No version information produced\";"
168             echo $__versionSourceLine > $__versionSourceFile
169         fi
170
171                 pushd "$__IntermediatesDir"
172         # Regenerate the CMake solution
173         echo "Invoking \"$__ProjectRoot/src/pal/tools/gen-buildsys-clang.sh\" \"$__ProjectRoot\" $__ClangMajorVersion $__ClangMinorVersion $__BuildArch $__BuildType $__CodeCoverage $__IncludeTests $generator $__cmakeargs"
174         "$__ProjectRoot/src/pal/tools/gen-buildsys-clang.sh" "$__ProjectRoot" $__ClangMajorVersion $__ClangMinorVersion $__BuildArch $__BuildType $__CodeCoverage $__IncludeTests $generator "$__cmakeargs"
175         popd
176     fi
177
178     # Check that the makefiles were created.
179     pushd "$__IntermediatesDir"
180     
181     if [ ! -f "$__IntermediatesDir/$buildFile" ]; then
182         echo "Failed to generate native component build project!"
183         exit 1
184     fi
185
186     # Get the number of processors available to the scheduler
187     # Other techniques such as `nproc` only get the number of
188     # processors available to a single process.
189     if [ `uname` = "FreeBSD" ]; then
190         NumProc=`sysctl hw.ncpu | awk '{ print $2+1 }'`
191     elif [ `uname` = "NetBSD" ]; then
192         NumProc=$(($(getconf NPROCESSORS_ONLN)+1))
193     else
194         NumProc=$(($(getconf _NPROCESSORS_ONLN)+1))
195     fi
196
197     # Build CoreCLR
198
199     echo "Executing $buildTool install -j $NumProc"
200
201     $buildTool install -j $NumProc
202     if [ $? != 0 ]; then
203         echo "Failed to build coreclr components."
204         exit 1
205     fi
206         popd
207 }
208
209 isMSBuildOnNETCoreSupported()
210 {
211     # This needs to be updated alongwith corresponding changes to netci.groovy.
212     __isMSBuildOnNETCoreSupported=0
213
214     if [ "$__HostArch" == "x64" ]; then
215         if [ "$__HostOS" == "Linux" ]; then
216             case "$__DistroRid" in
217                 "centos.7-x64")
218                     __isMSBuildOnNETCoreSupported=1
219                     ;;
220                 "debian.8-x64")
221                     __isMSBuildOnNETCoreSupported=1
222                     ;;
223                 "fedora.23-x64")
224                     __isMSBuildOnNETCoreSupported=1
225                     ;;
226                 "opensuse.13.2-x64")
227                     __isMSBuildOnNETCoreSupported=1
228                     ;;
229                 "rhel.7.2-x64")
230                     __isMSBuildOnNETCoreSupported=1
231                     ;;
232                 "ubuntu.14.04-x64")
233                     __isMSBuildOnNETCoreSupported=1
234                     ;;
235                 "ubuntu.16.04-x64")
236                     __isMSBuildOnNETCoreSupported=1
237                     ;;
238                 *)
239             esac
240         elif [ "$__HostOS" == "OSX" ]; then
241             __isMSBuildOnNETCoreSupported=1
242         fi
243     fi
244 }
245
246 build_CoreLib_ni()
247 {
248     if [ $__SkipCoreCLR == 0 -a -e $__BinDir/crossgen ]; then
249         echo "Generating native image for System.Private.CoreLib."
250         $__BinDir/crossgen $__BinDir/System.Private.CoreLib.dll
251         if [ $? -ne 0 ]; then
252             echo "Failed to generate native image for System.Private.CoreLib."
253             exit 1
254         fi
255
256         echo "Generating native image for MScorlib Facade."
257         $__BinDir/crossgen $__BinDir/mscorlib.dll
258         if [ $? -ne 0 ]; then
259             echo "Failed to generate native image for mscorlib facade."
260             exit 1
261         fi
262
263         if [ "$__BuildOS" == "Linux" ]; then
264             echo "Generating symbol file for System.Private.CoreLib."
265             $__BinDir/crossgen /CreatePerfMap $__BinDir $__BinDir/System.Private.CoreLib.ni.dll
266             if [ $? -ne 0 ]; then
267                 echo "Failed to generate symbol file for System.Private.CoreLib."
268                 exit 1
269             fi
270         fi
271     fi
272 }
273
274 build_CoreLib()
275 {
276
277     if [ $__isMSBuildOnNETCoreSupported == 0 ]; then
278         echo "System.Private.CoreLib.dll build unsupported."
279         return
280     fi
281
282     if [ $__SkipMSCorLib == 1 ]; then
283        echo "Skipping building System.Private.CoreLib."
284        return
285     fi
286
287     echo "Commencing build of managed components for $__BuildOS.$__BuildArch.$__BuildType"
288
289     # Invoke MSBuild
290     $__ProjectRoot/run.sh build -Project=$__ProjectDir/build.proj -MsBuildLog="/flp:Verbosity=normal;LogFile=$__LogsDir/System.Private.CoreLib_$__BuildOS__$__BuildArch__$__BuildType.log" -BuildTarget -__IntermediatesDir=$__IntermediatesDir -__RootBinDir=$__RootBinDir -BuildNugetPackage=false -UseSharedCompilation=false $__RunArgs $__UnprocessedBuildArgs
291
292     if [ $? -ne 0 ]; then
293         echo "Failed to build managed components."
294         exit 1
295     fi
296
297     # The cross build generates a crossgen with the target architecture.
298     if [ $__CrossBuild != 1 ]; then
299        # The architecture of host pc must be same architecture with target.
300        if [[ ( "$__HostArch" == "$__BuildArch" ) ]]; then
301            build_CoreLib_ni
302        elif [[ ( "$__HostArch" == "x64" ) && ( "$__BuildArch" == "x86" ) ]]; then
303            build_CoreLib_ni
304        elif [[ ( "$__HostArch" == "arm64" ) && ( "$__BuildArch" == "arm" ) ]]; then
305            build_CoreLib_ni
306        else 
307            exit 1
308        fi
309     fi 
310 }
311
312 generate_NugetPackages()
313 {
314     # We can only generate nuget package if we also support building mscorlib as part of this build.
315     if [ $__isMSBuildOnNETCoreSupported == 0 ]; then
316         echo "Nuget package generation unsupported."
317         return
318     fi
319
320     # Since we can build mscorlib for this OS, did we build the native components as well?
321     if [ $__SkipCoreCLR == 1 ]; then
322         echo "Unable to generate nuget packages since native components were not built."
323         return
324     fi
325
326     echo "Generating nuget packages for "$__BuildOS
327
328     # Build the packages
329     $__ProjectRoot/run.sh build -Project=$__SourceDir/.nuget/packages.builds -MsBuildLog="/flp:Verbosity=normal;LogFile=$__LogsDir/Nuget_$__BuildOS__$__BuildArch__$__BuildType.log" -BuildTarget -__IntermediatesDir=$__IntermediatesDir -__RootBinDir=$__RootBinDir -BuildNugetPackage=false -UseSharedCompilation=false $__RunArgs $__UnprocessedBuildArgs
330
331     if [ $? -ne 0 ]; then
332         echo "Failed to generate Nuget packages."
333         exit 1
334     fi
335 }
336
337 echo "Commencing CoreCLR Repo build"
338
339 # Argument types supported by this script:
340 #
341 # Build architecture - valid values are: x64, ARM.
342 # Build Type         - valid values are: Debug, Checked, Release
343 #
344 # Set the default arguments for build
345
346 # Obtain the location of the bash script to figure out where the root of the repo is.
347 __ProjectRoot="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
348
349 # Use uname to determine what the CPU is.
350 CPUName=$(uname -p)
351 # Some Linux platforms report unknown for platform, but the arch for machine.
352 if [ "$CPUName" == "unknown" ]; then
353     CPUName=$(uname -m)
354 fi
355
356 case $CPUName in
357     i686)
358         echo "Unsupported CPU $CPUName detected, build might not succeed!"
359         __BuildArch=x86
360         __HostArch=x86
361         ;;
362
363     x86_64)
364         __BuildArch=x64
365         __HostArch=x64
366         ;;
367
368     armv7l)
369         echo "Unsupported CPU $CPUName detected, build might not succeed!"
370         __BuildArch=arm
371         __HostArch=arm
372         ;;
373
374     aarch64)
375         echo "Unsupported CPU $CPUName detected, build might not succeed!"
376         __BuildArch=arm64
377         __HostArch=arm64
378         ;;
379
380     *)
381         echo "Unknown CPU $CPUName detected, configuring as if for x64"
382         __BuildArch=x64
383         __HostArch=x64
384         ;;
385 esac
386
387 # Use uname to determine what the OS is.
388 OSName=$(uname -s)
389 case $OSName in
390     Linux)
391         __BuildOS=Linux
392         __HostOS=Linux
393         ;;
394
395     Darwin)
396         __BuildOS=OSX
397         __HostOS=OSX
398         ;;
399
400     FreeBSD)
401         __BuildOS=FreeBSD
402         __HostOS=FreeBSD
403         ;;
404
405     OpenBSD)
406         __BuildOS=OpenBSD
407         __HostOS=OpenBSD
408         ;;
409
410     NetBSD)
411         __BuildOS=NetBSD
412         __HostOS=NetBSD
413         ;;
414
415     SunOS)
416         __BuildOS=SunOS
417         __HostOS=SunOS
418         ;;
419
420     *)
421         echo "Unsupported OS $OSName detected, configuring as if for Linux"
422         __BuildOS=Linux
423         __HostOS=Linux
424         ;;
425 esac
426
427 __BuildType=Debug
428 __CodeCoverage=
429 __IncludeTests=Include_Tests
430
431 # Set the various build properties here so that CMake and MSBuild can pick them up
432 __ProjectDir="$__ProjectRoot"
433 __SourceDir="$__ProjectDir/src"
434 __PackagesDir="$__ProjectDir/packages"
435 __RootBinDir="$__ProjectDir/bin"
436 __UnprocessedBuildArgs=
437 __RunArgs=
438 __MSBCleanBuildArgs=
439 __UseNinja=0
440 __VerboseBuild=0
441 __ConfigureOnly=0
442 __SkipConfigure=0
443 __SkipRestore=""
444 __SkipNuget=0
445 __SkipCoreCLR=0
446 __SkipMSCorLib=0
447 __CrossBuild=0
448 __ClangMajorVersion=0
449 __ClangMinorVersion=0
450 __NuGetPath="$__PackagesDir/NuGet.exe"
451 __DistroRid=""
452 __cmakeargs=""
453 __SkipGenerateVersion=0
454
455 while :; do
456     if [ $# -le 0 ]; then
457         break
458     fi
459
460     lowerI="$(echo $1 | awk '{print tolower($0)}')"
461     case $lowerI in
462         -\?|-h|--help)
463             usage
464             exit 1
465             ;;
466
467         x86)
468             __BuildArch=x86
469             ;;
470
471         x64)
472             __BuildArch=x64
473             ;;
474
475         arm)
476             __BuildArch=arm
477             ;;
478
479         arm-softfp)
480             __BuildArch=arm-softfp
481             ;;
482
483         arm64)
484             __BuildArch=arm64
485             ;;
486
487         debug)
488             __BuildType=Debug
489             ;;
490
491         checked)
492             __BuildType=Checked
493             ;;
494
495         release)
496             __BuildType=Release
497             ;;
498
499         coverage)
500             __CodeCoverage=Coverage
501             ;;
502
503         cross)
504             __CrossBuild=1
505             ;;
506
507                 verbose)
508         __VerboseBuild=1
509         ;;
510
511         clang3.5)
512             __ClangMajorVersion=3
513             __ClangMinorVersion=5
514             ;;
515
516         clang3.6)
517             __ClangMajorVersion=3
518             __ClangMinorVersion=6
519             ;;
520
521         clang3.7)
522             __ClangMajorVersion=3
523             __ClangMinorVersion=7
524             ;;
525
526         clang3.8)
527             __ClangMajorVersion=3
528             __ClangMinorVersion=8
529             ;;
530
531         ninja)
532             __UseNinja=1
533             ;;
534
535         configureonly)
536             __ConfigureOnly=1
537             __SkipCoreCLR=1
538             __SkipMSCorLib=1
539             __IncludeTests=
540             ;;
541
542         skipconfigure)
543             __SkipConfigure=1
544             ;;
545
546         skipnative)
547             # Use "skipnative" to use the same option name as build.cmd.
548             __SkipCoreCLR=1
549             ;;
550
551         skipcoreclr)
552             # Accept "skipcoreclr" for backwards-compatibility.
553             __SkipCoreCLR=1
554             ;;
555
556         skipmscorlib)
557             __SkipMSCorLib=1
558             ;;
559
560         skipgenerateversion)
561             __SkipGenerateVersion=1
562             ;;
563
564         includetests)
565             ;;
566
567         skiptests)
568             __IncludeTests=
569             ;;
570
571         skipnuget)
572             __SkipNuget=1
573             ;;
574
575         cmakeargs)
576             if [ -n "$2" ]; then
577                 __cmakeargs="$2"
578                 shift
579             else
580                 echo "ERROR: 'cmakeargs' requires a non-empty option argument"
581                 exit 1
582             fi
583             ;;
584
585         bindir)
586             if [ -n "$2" ]; then
587                 __RootBinDir="$2"
588                 if [ ! -d $__RootBinDir ]; then
589                     mkdir $__RootBinDir
590                 fi
591                 __RootBinParent=$(dirname $__RootBinDir)
592                 __RootBinName=${__RootBinDir##*/}
593                 __RootBinDir="$(cd $__RootBinParent &>/dev/null && printf %s/%s $PWD $__RootBinName)"
594                 shift
595             else
596                 echo "ERROR: 'bindir' requires a non-empty option argument"
597                 exit 1
598             fi
599             ;;
600
601         *)
602             __UnprocessedBuildArgs="$__UnprocessedBuildArgs $1"
603             ;;
604     esac
605
606     shift
607 done
608
609 __RunArgs="-BuildArch=$__BuildArch -BuildType=$__BuildType -BuildOS=$__BuildOS"
610
611 # Configure environment if we are doing a verbose build
612 if [ $__VerboseBuild == 1 ]; then
613     export VERBOSE=1
614         __RunArgs="$__RunArgs -verbose"
615 fi
616
617 # Set default clang version
618 if [[ $__ClangMajorVersion == 0 && $__ClangMinorVersion == 0 ]]; then
619     if [ $__CrossBuild == 1 ]; then
620         __ClangMajorVersion=3
621         __ClangMinorVersion=6
622     else
623         __ClangMajorVersion=3
624         __ClangMinorVersion=5
625     fi
626 fi
627
628 # Set dependent variables
629 __LogsDir="$__RootBinDir/Logs"
630
631 # init the distro name
632 initDistroRid
633
634 # Set the remaining variables based upon the determined build configuration
635 __BinDir="$__RootBinDir/Product/$__BuildOS.$__BuildArch.$__BuildType"
636 __PackagesBinDir="$__BinDir/.nuget"
637 __ToolsDir="$__RootBinDir/tools"
638 __TestWorkingDir="$__RootBinDir/tests/$__BuildOS.$__BuildArch.$__BuildType"
639 export __IntermediatesDir="$__RootBinDir/obj/$__BuildOS.$__BuildArch.$__BuildType"
640 __TestIntermediatesDir="$__RootBinDir/tests/obj/$__BuildOS.$__BuildArch.$__BuildType"
641 __isMSBuildOnNETCoreSupported=0
642
643 # Init if MSBuild for .NET Core is supported for this platform
644 isMSBuildOnNETCoreSupported
645
646 # CI_SPECIFIC - On CI machines, $HOME may not be set. In such a case, create a subfolder and set the variable to set.
647 # This is needed by CLI to function.
648 if [ -z "$HOME" ]; then
649     if [ ! -d "$__ProjectDir/temp_home" ]; then
650         mkdir temp_home
651     fi
652     export HOME=$__ProjectDir/temp_home
653     echo "HOME not defined; setting it to $HOME"
654 fi
655
656 # Specify path to be set for CMAKE_INSTALL_PREFIX.
657 # This is where all built CoreClr libraries will copied to.
658 export __CMakeBinDir="$__BinDir"
659
660 # Configure environment if we are doing a cross compile.
661 if [ $__CrossBuild == 1 ]; then
662     export CROSSCOMPILE=1
663     if ! [[ -n "$ROOTFS_DIR" ]]; then
664         export ROOTFS_DIR="$__ProjectRoot/cross/rootfs/$__BuildArch"
665     fi
666 fi
667
668 # Make the directories necessary for build if they don't exist
669
670 setup_dirs
671
672 # Check prereqs.
673
674 check_prereqs
675
676 # Build the coreclr (native) components.
677
678 build_coreclr
679
680 # Build System.Private.CoreLib.
681
682 build_CoreLib
683
684 # Generate nuget packages
685 if [ $__SkipNuget != 1 ]; then
686     generate_NugetPackages
687 fi
688
689
690 # Build complete
691
692 echo "Repo successfully built."
693 echo "Product binaries are available at $__BinDir"
694 exit 0