Merge pull request #6848 from BruceForstall/FixArm64SuperPmi2
[platform/upstream/coreclr.git] / build.sh
1 #!/usr/bin/env bash
2
3 # resolve python-version to use
4 if [ "$PYTHON" == "" ] ; then
5     if which python >/dev/null 2>&1
6     then
7        PYTHON=python
8     elif which python2 >/dev/null 2>&1
9     then
10        PYTHON=python2
11     elif which python2.7 >/dev/null 2>&1
12     then
13        PYTHON=python2.7
14     else
15        echo "Unable to locate build-dependency python2.x!" 1>&2
16        exit 1
17     fi
18 fi
19
20 # validate python-dependency
21 # useful in case of explicitly set option.
22 if ! which $PYTHON > /dev/null 2>&1
23 then
24    echo "Unable to locate build-dependency python2.x ($PYTHON)!" 1>&2
25    exit 1
26 fi
27
28 usage()
29 {
30     echo "Usage: $0 [BuildArch] [BuildType] [verbose] [coverage] [cross] [clangx.y] [ninja] [configureonly] [skipconfigure] [skipnative] [skipmscorlib] [skiptests] [cmakeargs] [bindir]"
31     echo "BuildArch can be: x64, x86, arm, arm-softfp, arm64"
32     echo "BuildType can be: debug, checked, release"
33     echo "coverage - optional argument to enable code coverage build (currently supported only for Linux and OSX)."
34     echo "ninja - target ninja instead of GNU make"
35     echo "clangx.y - optional argument to build using clang version x.y."
36     echo "cross - optional argument to signify cross compilation,"
37     echo "      - will use ROOTFS_DIR environment variable if set."
38     echo "configureonly - do not perform any builds; just configure the build."
39     echo "skipconfigure - skip build configuration."
40     echo "skipnative - do not build native components."
41     echo "skipmscorlib - do not build mscorlib.dll."
42     echo "skiptests - skip the tests in the 'tests' subdirectory."
43     echo "skipnuget - skip building nuget packages."
44     echo "verbose - optional argument to enable verbose build output."
45     echo "-skiprestore: skip restoring packages ^(default: packages are restored during build^)."
46         echo "-disableoss: Disable Open Source Signing for System.Private.CoreLib."
47         echo "-sequential: force a non-parallel build ^(default is to build in parallel"
48         echo "   using all processors^)."
49         echo "-officialbuildid=^<ID^>: specify the official build ID to be used by this build."
50         echo "-Rebuild: passes /t:rebuild to the build projects."
51     echo "skipgenerateversion - disable version generation even if MSBuild is supported."
52     echo "cmakeargs - user-settable additional arguments passed to CMake."
53     echo "bindir - output directory (defaults to $__ProjectRoot/bin)"
54
55     exit 1
56 }
57
58 initDistroRid()
59 {
60     if [ "$__BuildOS" == "Linux" ]; then
61         if [ ! -e /etc/os-release ]; then
62             echo "WARNING: Can not determine runtime id for current distro."
63             export __DistroRid=""
64         else
65             source /etc/os-release
66             export __DistroRid="$ID.$VERSION_ID-$__HostArch"
67         fi
68     fi
69 }
70
71 setup_dirs()
72 {
73     echo Setting up directories for build
74
75     mkdir -p "$__RootBinDir"
76     mkdir -p "$__BinDir"
77     mkdir -p "$__LogsDir"
78     mkdir -p "$__IntermediatesDir"
79 }
80
81 # Check the system to ensure the right prereqs are in place
82
83 check_prereqs()
84 {
85     echo "Checking prerequisites..."
86
87     # Check presence of CMake on the path
88     hash cmake 2>/dev/null || { echo >&2 "Please install cmake before running this script"; exit 1; }
89
90     # Check for clang
91     hash clang-$__ClangMajorVersion.$__ClangMinorVersion 2>/dev/null ||  hash clang$__ClangMajorVersion$__ClangMinorVersion 2>/dev/null ||  hash clang 2>/dev/null || { echo >&2 "Please install clang-$__ClangMajorVersion.$__ClangMinorVersion before running this script"; exit 1; }
92
93 }
94
95 build_coreclr()
96 {
97
98     if [ $__SkipCoreCLR == 1 ]; then
99         echo "Skipping CoreCLR build."
100         return
101     fi
102
103 # Event Logging Infrastructure
104    __GeneratedIntermediate="$__IntermediatesDir/Generated"
105    __GeneratedIntermediateEventProvider="$__GeneratedIntermediate/eventprovider_new"
106     if [[ -d "$__GeneratedIntermediateEventProvider" ]]; then
107         rm -rf  "$__GeneratedIntermediateEventProvider"
108     fi
109
110     if [[ ! -d "$__GeneratedIntermediate/eventprovider" ]]; then
111         mkdir -p "$__GeneratedIntermediate/eventprovider"
112     fi
113
114     mkdir -p "$__GeneratedIntermediateEventProvider"
115     if [[ $__SkipCoreCLR == 0 || $__ConfigureOnly == 1 ]]; then
116         echo "Laying out dynamically generated files consumed by the build system "
117         echo "Laying out dynamically generated Event Logging Test files"
118         $PYTHON -B -Wall -Werror "$__ProjectRoot/src/scripts/genXplatEventing.py" --man "$__ProjectRoot/src/vm/ClrEtwAll.man" --exc "$__ProjectRoot/src/vm/ClrEtwAllMeta.lst" --testdir "$__GeneratedIntermediateEventProvider/tests"
119
120         if  [[ $? != 0 ]]; then
121             exit
122         fi
123
124         #determine the logging system
125         case $__BuildOS in
126             Linux)
127                 echo "Laying out dynamically generated Event Logging Implementation of Lttng"
128                 $PYTHON -B -Wall -Werror "$__ProjectRoot/src/scripts/genXplatLttng.py" --man "$__ProjectRoot/src/vm/ClrEtwAll.man" --intermediate "$__GeneratedIntermediateEventProvider"
129                 if  [[ $? != 0 ]]; then
130                     exit
131                 fi
132                 ;;
133             *)
134                 ;;
135         esac
136     fi
137
138     echo "Cleaning the temp folder of dynamically generated Event Logging files"
139     $PYTHON -B -Wall -Werror -c "import sys;sys.path.insert(0,\"$__ProjectRoot/src/scripts\"); from Utilities import *;UpdateDirectory(\"$__GeneratedIntermediate/eventprovider\",\"$__GeneratedIntermediateEventProvider\")"
140     if  [[ $? != 0 ]]; then
141         exit
142     fi
143
144     rm -rf "$__GeneratedIntermediateEventProvider"
145
146     # All set to commence the build
147
148     echo "Commencing build of native components for $__BuildOS.$__BuildArch.$__BuildType in $__IntermediatesDir"
149
150     generator=""
151     buildFile="Makefile"
152     buildTool="make"
153     if [ $__UseNinja == 1 ]; then
154         generator="ninja"
155         buildFile="build.ninja"
156         buildTool="ninja"
157     fi
158
159     if [ $__SkipConfigure == 0 ]; then
160         # if msbuild is not supported, then set __SkipGenerateVersion to 1
161         if [ $__isMSBuildOnNETCoreSupported == 0 ]; then __SkipGenerateVersion=1; fi
162         # Drop version.c file
163         __versionSourceFile=$__IntermediatesDir/version.cpp
164         if [ $__SkipGenerateVersion == 0 ]; then
165             "$__ProjectRoot/run.sh" build -Project=$__ProjectDir/build.proj -generateHeaderUnix -NativeVersionSourceFile=$__versionSourceFile $__RunArgs $__UnprocessedBuildArgs
166         else
167             __versionSourceLine="static char sccsid[] __attribute__((used)) = \"@(#)No version information produced\";"
168             echo $__versionSourceLine > $__versionSourceFile
169         fi
170
171                 pushd "$__IntermediatesDir"
172         # Regenerate the CMake solution
173         echo "Invoking \"$__ProjectRoot/src/pal/tools/gen-buildsys-clang.sh\" \"$__ProjectRoot\" $__ClangMajorVersion $__ClangMinorVersion $__BuildArch $__BuildType $__CodeCoverage $__IncludeTests $generator $__cmakeargs"
174         "$__ProjectRoot/src/pal/tools/gen-buildsys-clang.sh" "$__ProjectRoot" $__ClangMajorVersion $__ClangMinorVersion $__BuildArch $__BuildType $__CodeCoverage $__IncludeTests $generator "$__cmakeargs"
175         popd
176     fi
177
178     # Check that the makefiles were created.
179     pushd "$__IntermediatesDir"
180     
181     if [ ! -f "$__IntermediatesDir/$buildFile" ]; then
182         echo "Failed to generate native component build project!"
183         exit 1
184     fi
185
186     # Get the number of processors available to the scheduler
187     # Other techniques such as `nproc` only get the number of
188     # processors available to a single process.
189     if [ `uname` = "FreeBSD" ]; then
190         NumProc=`sysctl hw.ncpu | awk '{ print $2+1 }'`
191     elif [ `uname` = "NetBSD" ]; then
192         NumProc=$(($(getconf NPROCESSORS_ONLN)+1))
193     else
194         NumProc=$(($(getconf _NPROCESSORS_ONLN)+1))
195     fi
196
197     # Build CoreCLR
198
199     echo "Executing $buildTool install -j $NumProc"
200
201     $buildTool install -j $NumProc
202     if [ $? != 0 ]; then
203         echo "Failed to build coreclr components."
204         exit 1
205     fi
206         popd
207 }
208
209 isMSBuildOnNETCoreSupported()
210 {
211     # This needs to be updated alongwith corresponding changes to netci.groovy.
212     __isMSBuildOnNETCoreSupported=0
213
214     if [ "$__HostArch" == "x64" ]; then
215         if [ "$__HostOS" == "Linux" ]; then
216             case "$__DistroRid" in
217                 "centos.7-x64")
218                     __isMSBuildOnNETCoreSupported=1
219                     ;;
220                 "debian.8-x64")
221                     __isMSBuildOnNETCoreSupported=1
222                     ;;
223                 "fedora.23-x64")
224                     __isMSBuildOnNETCoreSupported=1
225                     ;;
226                 "opensuse.13.2-x64")
227                     __isMSBuildOnNETCoreSupported=1
228                     ;;
229                 "opensuse.42.1-x64")
230                     __isMSBuildOnNETCoreSupported=1
231                     ;;
232                 "rhel.7.2-x64")
233                     __isMSBuildOnNETCoreSupported=1
234                     ;;
235                 "ubuntu.14.04-x64")
236                     __isMSBuildOnNETCoreSupported=1
237                     ;;
238                 "ubuntu.16.04-x64")
239                     __isMSBuildOnNETCoreSupported=1
240                     ;;
241                 *)
242             esac
243         elif [ "$__HostOS" == "OSX" ]; then
244             __isMSBuildOnNETCoreSupported=1
245         fi
246     fi
247 }
248
249 build_CoreLib_ni()
250 {
251     if [ $__SkipCoreCLR == 0 -a -e $__BinDir/crossgen ]; then
252         echo "Generating native image for System.Private.CoreLib."
253         $__BinDir/crossgen $__BinDir/System.Private.CoreLib.dll
254         if [ $? -ne 0 ]; then
255             echo "Failed to generate native image for System.Private.CoreLib."
256             exit 1
257         fi
258
259         echo "Generating native image for MScorlib Facade."
260         $__BinDir/crossgen $__BinDir/mscorlib.dll
261         if [ $? -ne 0 ]; then
262             echo "Failed to generate native image for mscorlib facade."
263             exit 1
264         fi
265
266         if [ "$__BuildOS" == "Linux" ]; then
267             echo "Generating symbol file for System.Private.CoreLib."
268             $__BinDir/crossgen /CreatePerfMap $__BinDir $__BinDir/System.Private.CoreLib.ni.dll
269             if [ $? -ne 0 ]; then
270                 echo "Failed to generate symbol file for System.Private.CoreLib."
271                 exit 1
272             fi
273         fi
274     fi
275 }
276
277 build_CoreLib()
278 {
279
280     if [ $__isMSBuildOnNETCoreSupported == 0 ]; then
281         echo "System.Private.CoreLib.dll build unsupported."
282         return
283     fi
284
285     if [ $__SkipMSCorLib == 1 ]; then
286        echo "Skipping building System.Private.CoreLib."
287        return
288     fi
289
290     echo "Commencing build of managed components for $__BuildOS.$__BuildArch.$__BuildType"
291
292     # Invoke MSBuild
293     $__ProjectRoot/run.sh build -Project=$__ProjectDir/build.proj -MsBuildLog="/flp:Verbosity=normal;LogFile=$__LogsDir/System.Private.CoreLib_$__BuildOS__$__BuildArch__$__BuildType.log" -BuildTarget -__IntermediatesDir=$__IntermediatesDir -__RootBinDir=$__RootBinDir -BuildNugetPackage=false -UseSharedCompilation=false $__RunArgs $__UnprocessedBuildArgs
294
295     if [ $? -ne 0 ]; then
296         echo "Failed to build managed components."
297         exit 1
298     fi
299
300     # The cross build generates a crossgen with the target architecture.
301     if [ $__CrossBuild != 1 ]; then
302        # The architecture of host pc must be same architecture with target.
303        if [[ ( "$__HostArch" == "$__BuildArch" ) ]]; then
304            build_CoreLib_ni
305        elif [[ ( "$__HostArch" == "x64" ) && ( "$__BuildArch" == "x86" ) ]]; then
306            build_CoreLib_ni
307        elif [[ ( "$__HostArch" == "arm64" ) && ( "$__BuildArch" == "arm" ) ]]; then
308            build_CoreLib_ni
309        else 
310            exit 1
311        fi
312     fi 
313 }
314
315 generate_NugetPackages()
316 {
317     # We can only generate nuget package if we also support building mscorlib as part of this build.
318     if [ $__isMSBuildOnNETCoreSupported == 0 ]; then
319         echo "Nuget package generation unsupported."
320         return
321     fi
322
323     # Since we can build mscorlib for this OS, did we build the native components as well?
324     if [ $__SkipCoreCLR == 1 ]; then
325         echo "Unable to generate nuget packages since native components were not built."
326         return
327     fi
328
329     echo "Generating nuget packages for "$__BuildOS
330
331     # Build the packages
332     $__ProjectRoot/run.sh build -Project=$__SourceDir/.nuget/packages.builds -MsBuildLog="/flp:Verbosity=normal;LogFile=$__LogsDir/Nuget_$__BuildOS__$__BuildArch__$__BuildType.log" -BuildTarget -__IntermediatesDir=$__IntermediatesDir -__RootBinDir=$__RootBinDir -BuildNugetPackage=false -UseSharedCompilation=false $__RunArgs $__UnprocessedBuildArgs
333
334     if [ $? -ne 0 ]; then
335         echo "Failed to generate Nuget packages."
336         exit 1
337     fi
338 }
339
340 echo "Commencing CoreCLR Repo build"
341
342 # Argument types supported by this script:
343 #
344 # Build architecture - valid values are: x64, ARM.
345 # Build Type         - valid values are: Debug, Checked, Release
346 #
347 # Set the default arguments for build
348
349 # Obtain the location of the bash script to figure out where the root of the repo is.
350 __ProjectRoot="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
351
352 # Use uname to determine what the CPU is.
353 CPUName=$(uname -p)
354 # Some Linux platforms report unknown for platform, but the arch for machine.
355 if [ "$CPUName" == "unknown" ]; then
356     CPUName=$(uname -m)
357 fi
358
359 case $CPUName in
360     i686)
361         echo "Unsupported CPU $CPUName detected, build might not succeed!"
362         __BuildArch=x86
363         __HostArch=x86
364         ;;
365
366     x86_64)
367         __BuildArch=x64
368         __HostArch=x64
369         ;;
370
371     armv7l)
372         echo "Unsupported CPU $CPUName detected, build might not succeed!"
373         __BuildArch=arm
374         __HostArch=arm
375         ;;
376
377     aarch64)
378         echo "Unsupported CPU $CPUName detected, build might not succeed!"
379         __BuildArch=arm64
380         __HostArch=arm64
381         ;;
382
383     *)
384         echo "Unknown CPU $CPUName detected, configuring as if for x64"
385         __BuildArch=x64
386         __HostArch=x64
387         ;;
388 esac
389
390 # Use uname to determine what the OS is.
391 OSName=$(uname -s)
392 case $OSName in
393     Linux)
394         __BuildOS=Linux
395         __HostOS=Linux
396         ;;
397
398     Darwin)
399         __BuildOS=OSX
400         __HostOS=OSX
401         ;;
402
403     FreeBSD)
404         __BuildOS=FreeBSD
405         __HostOS=FreeBSD
406         ;;
407
408     OpenBSD)
409         __BuildOS=OpenBSD
410         __HostOS=OpenBSD
411         ;;
412
413     NetBSD)
414         __BuildOS=NetBSD
415         __HostOS=NetBSD
416         ;;
417
418     SunOS)
419         __BuildOS=SunOS
420         __HostOS=SunOS
421         ;;
422
423     *)
424         echo "Unsupported OS $OSName detected, configuring as if for Linux"
425         __BuildOS=Linux
426         __HostOS=Linux
427         ;;
428 esac
429
430 __BuildType=Debug
431 __CodeCoverage=
432 __IncludeTests=Include_Tests
433
434 # Set the various build properties here so that CMake and MSBuild can pick them up
435 __ProjectDir="$__ProjectRoot"
436 __SourceDir="$__ProjectDir/src"
437 __PackagesDir="$__ProjectDir/packages"
438 __RootBinDir="$__ProjectDir/bin"
439 __UnprocessedBuildArgs=
440 __RunArgs=
441 __MSBCleanBuildArgs=
442 __UseNinja=0
443 __VerboseBuild=0
444 __ConfigureOnly=0
445 __SkipConfigure=0
446 __SkipRestore=""
447 __SkipNuget=0
448 __SkipCoreCLR=0
449 __SkipMSCorLib=0
450 __CrossBuild=0
451 __ClangMajorVersion=0
452 __ClangMinorVersion=0
453 __NuGetPath="$__PackagesDir/NuGet.exe"
454 __DistroRid=""
455 __cmakeargs=""
456 __SkipGenerateVersion=0
457
458 while :; do
459     if [ $# -le 0 ]; then
460         break
461     fi
462
463     lowerI="$(echo $1 | awk '{print tolower($0)}')"
464     case $lowerI in
465         -\?|-h|--help)
466             usage
467             exit 1
468             ;;
469
470         x86)
471             __BuildArch=x86
472             ;;
473
474         x64)
475             __BuildArch=x64
476             ;;
477
478         arm)
479             __BuildArch=arm
480             ;;
481
482         arm-softfp)
483             __BuildArch=arm-softfp
484             ;;
485
486         arm64)
487             __BuildArch=arm64
488             ;;
489
490         debug)
491             __BuildType=Debug
492             ;;
493
494         checked)
495             __BuildType=Checked
496             ;;
497
498         release)
499             __BuildType=Release
500             ;;
501
502         coverage)
503             __CodeCoverage=Coverage
504             ;;
505
506         cross)
507             __CrossBuild=1
508             ;;
509
510                 verbose)
511         __VerboseBuild=1
512         ;;
513
514         clang3.5)
515             __ClangMajorVersion=3
516             __ClangMinorVersion=5
517             ;;
518
519         clang3.6)
520             __ClangMajorVersion=3
521             __ClangMinorVersion=6
522             ;;
523
524         clang3.7)
525             __ClangMajorVersion=3
526             __ClangMinorVersion=7
527             ;;
528
529         clang3.8)
530             __ClangMajorVersion=3
531             __ClangMinorVersion=8
532             ;;
533
534         ninja)
535             __UseNinja=1
536             ;;
537
538         configureonly)
539             __ConfigureOnly=1
540             __SkipCoreCLR=1
541             __SkipMSCorLib=1
542             __IncludeTests=
543             ;;
544
545         skipconfigure)
546             __SkipConfigure=1
547             ;;
548
549         skipnative)
550             # Use "skipnative" to use the same option name as build.cmd.
551             __SkipCoreCLR=1
552             ;;
553
554         skipcoreclr)
555             # Accept "skipcoreclr" for backwards-compatibility.
556             __SkipCoreCLR=1
557             ;;
558
559         skipmscorlib)
560             __SkipMSCorLib=1
561             ;;
562
563         skipgenerateversion)
564             __SkipGenerateVersion=1
565             ;;
566
567         includetests)
568             ;;
569
570         skiptests)
571             __IncludeTests=
572             ;;
573
574         skipnuget)
575             __SkipNuget=1
576             ;;
577
578         cmakeargs)
579             if [ -n "$2" ]; then
580                 __cmakeargs="$2"
581                 shift
582             else
583                 echo "ERROR: 'cmakeargs' requires a non-empty option argument"
584                 exit 1
585             fi
586             ;;
587
588         bindir)
589             if [ -n "$2" ]; then
590                 __RootBinDir="$2"
591                 if [ ! -d $__RootBinDir ]; then
592                     mkdir $__RootBinDir
593                 fi
594                 __RootBinParent=$(dirname $__RootBinDir)
595                 __RootBinName=${__RootBinDir##*/}
596                 __RootBinDir="$(cd $__RootBinParent &>/dev/null && printf %s/%s $PWD $__RootBinName)"
597                 shift
598             else
599                 echo "ERROR: 'bindir' requires a non-empty option argument"
600                 exit 1
601             fi
602             ;;
603
604         *)
605             __UnprocessedBuildArgs="$__UnprocessedBuildArgs $1"
606             ;;
607     esac
608
609     shift
610 done
611
612 __RunArgs="-BuildArch=$__BuildArch -BuildType=$__BuildType -BuildOS=$__BuildOS"
613
614 # Configure environment if we are doing a verbose build
615 if [ $__VerboseBuild == 1 ]; then
616     export VERBOSE=1
617         __RunArgs="$__RunArgs -verbose"
618 fi
619
620 # Set default clang version
621 if [[ $__ClangMajorVersion == 0 && $__ClangMinorVersion == 0 ]]; then
622     if [ $__CrossBuild == 1 ]; then
623         __ClangMajorVersion=3
624         __ClangMinorVersion=6
625     else
626         __ClangMajorVersion=3
627         __ClangMinorVersion=5
628     fi
629 fi
630
631 # Set dependent variables
632 __LogsDir="$__RootBinDir/Logs"
633
634 # init the distro name
635 initDistroRid
636
637 # Set the remaining variables based upon the determined build configuration
638 __BinDir="$__RootBinDir/Product/$__BuildOS.$__BuildArch.$__BuildType"
639 __PackagesBinDir="$__BinDir/.nuget"
640 __ToolsDir="$__RootBinDir/tools"
641 __TestWorkingDir="$__RootBinDir/tests/$__BuildOS.$__BuildArch.$__BuildType"
642 export __IntermediatesDir="$__RootBinDir/obj/$__BuildOS.$__BuildArch.$__BuildType"
643 __TestIntermediatesDir="$__RootBinDir/tests/obj/$__BuildOS.$__BuildArch.$__BuildType"
644 __isMSBuildOnNETCoreSupported=0
645
646 # Init if MSBuild for .NET Core is supported for this platform
647 isMSBuildOnNETCoreSupported
648
649 # CI_SPECIFIC - On CI machines, $HOME may not be set. In such a case, create a subfolder and set the variable to set.
650 # This is needed by CLI to function.
651 if [ -z "$HOME" ]; then
652     if [ ! -d "$__ProjectDir/temp_home" ]; then
653         mkdir temp_home
654     fi
655     export HOME=$__ProjectDir/temp_home
656     echo "HOME not defined; setting it to $HOME"
657 fi
658
659 # Specify path to be set for CMAKE_INSTALL_PREFIX.
660 # This is where all built CoreClr libraries will copied to.
661 export __CMakeBinDir="$__BinDir"
662
663 # Configure environment if we are doing a cross compile.
664 if [ $__CrossBuild == 1 ]; then
665     export CROSSCOMPILE=1
666     if ! [[ -n "$ROOTFS_DIR" ]]; then
667         export ROOTFS_DIR="$__ProjectRoot/cross/rootfs/$__BuildArch"
668     fi
669 fi
670
671 # Make the directories necessary for build if they don't exist
672
673 setup_dirs
674
675 # Check prereqs.
676
677 check_prereqs
678
679 # Build the coreclr (native) components.
680
681 build_coreclr
682
683 # Build System.Private.CoreLib.
684
685 build_CoreLib
686
687 # Generate nuget packages
688 if [ $__SkipNuget != 1 ]; then
689     generate_NugetPackages
690 fi
691
692
693 # Build complete
694
695 echo "Repo successfully built."
696 echo "Product binaries are available at $__BinDir"
697 exit 0