Update docker images for arm32 (#17422)
[platform/upstream/coreclr.git] / build.sh
1 #!/usr/bin/env bash
2
3 # Work around Jenkins CI + msbuild problem: Jenkins sometimes creates very large environment
4 # variables, and msbuild can't handle environment blocks with such large variables. So clear
5 # out the variables that might be too large.
6 export ghprbCommentBody=
7
8 # resolve python-version to use
9 if [ "$PYTHON" == "" ] ; then
10     if ! PYTHON=$(command -v python2.7 || command -v python2 || command -v python)
11     then
12        echo "Unable to locate build-dependency python2.x!" 1>&2
13        exit 1
14     fi
15 fi
16
17 # validate python-dependency
18 # useful in case of explicitly set option.
19 if ! command -v $PYTHON > /dev/null
20 then
21    echo "Unable to locate build-dependency python2.x ($PYTHON)!" 1>&2
22    exit 1
23 fi
24
25 usage()
26 {
27     echo "Usage: $0 [BuildArch] [BuildType] [-verbose] [-coverage] [-cross] [-clangx.y] [-ninja] [-configureonly] [-skipconfigure] [-skipnative] [-skipmscorlib] [-skiptests] [-stripsymbols] [-ignorewarnings] [-cmakeargs] [-bindir]"
28     echo "BuildArch can be: -x64, -x86, -arm, -armel, -arm64"
29     echo "BuildType can be: -debug, -checked, -release"
30     echo "-coverage - optional argument to enable code coverage build (currently supported only for Linux and OSX)."
31     echo "-ninja - target ninja instead of GNU make"
32     echo "-clangx.y - optional argument to build using clang version x.y."
33     echo "-cross - optional argument to signify cross compilation,"
34     echo "       - will use ROOTFS_DIR environment variable if set."
35     echo "-crosscomponent - optional argument to build cross-architecture component,"
36     echo "                - will use CAC_ROOTFS_DIR environment variable if set."
37     echo "-nopgooptimize - do not use profile guided optimizations."
38     echo "-pgoinstrument - generate instrumented code for profile guided optimization enabled binaries."
39     echo "-ibcinstrument - generate IBC-tuning-enabled native images when invoking crossgen."
40     echo "-configureonly - do not perform any builds; just configure the build."
41     echo "-skipconfigure - skip build configuration."
42     echo "-skipnative - do not build native components."
43     echo "-skipmscorlib - do not build mscorlib.dll."
44     echo "-skiptests - skip the tests in the 'tests' subdirectory."
45     echo "-skipnuget - skip building nuget packages."
46     echo "-skiprestoreoptdata - skip restoring optimization data used by profile-based optimizations."
47     echo "-skipcrossgen - skip native image generation"
48     echo "-verbose - optional argument to enable verbose build output."
49     echo "-skiprestore: skip restoring packages ^(default: packages are restored during build^)."
50     echo "-disableoss: Disable Open Source Signing for System.Private.CoreLib."
51     echo "-officialbuildid=^<ID^>: specify the official build ID to be used by this build."
52     echo "-Rebuild: passes /t:rebuild to the build projects."
53     echo "-stripSymbols - Optional argument to strip native symbols during the build."
54     echo "-skipgenerateversion - disable version generation even if MSBuild is supported."
55     echo "-ignorewarnings - do not treat warnings as errors"
56     echo "-cmakeargs - user-settable additional arguments passed to CMake."
57     echo "-bindir - output directory (defaults to $__ProjectRoot/bin)"
58     echo "-msbuildonunsupportedplatform - build managed binaries even if distro is not officially supported."
59     echo "-numproc - set the number of build processes."
60     exit 1
61 }
62
63 initHostDistroRid()
64 {
65     __HostDistroRid=""
66     if [ "$__HostOS" == "Linux" ]; then
67         if [ -e /etc/os-release ]; then
68             source /etc/os-release
69             if [[ $ID == "alpine" || $ID == "rhel" ]]; then
70                 # remove the last version digit
71                 VERSION_ID=${VERSION_ID%.*}
72             fi
73             __HostDistroRid="$ID.$VERSION_ID-$__HostArch"
74         elif [ -e /etc/redhat-release ]; then
75             local redhatRelease=$(</etc/redhat-release)
76             if [[ $redhatRelease == "CentOS release 6."* || $redhatRelease == "Red Hat Enterprise Linux Server release 6."* ]]; then
77                __HostDistroRid="rhel.6-$__HostArch"
78             fi
79         fi
80     fi
81     if [ "$__HostOS" == "FreeBSD" ]; then
82         __freebsd_version=`sysctl -n kern.osrelease | cut -f1 -d'.'`
83         __HostDistroRid="freebsd.$__freebsd_version-$__HostArch"
84     fi
85
86     if [ "$__HostDistroRid" == "" ]; then
87         echo "WARNING: Can not determine runtime id for current distro."
88     fi
89 }
90
91 initTargetDistroRid()
92 {
93     if [ $__CrossBuild == 1 ]; then
94         if [ "$__BuildOS" == "Linux" ]; then
95             if [ ! -e $ROOTFS_DIR/etc/os-release ]; then
96                 if [ -e $ROOTFS_DIR/android_platform ]; then
97                     source $ROOTFS_DIR/android_platform
98                     export __DistroRid="$RID"
99                 else
100                     echo "WARNING: Can not determine runtime id for current distro."
101                     export __DistroRid=""
102                 fi
103             else
104                 source $ROOTFS_DIR/etc/os-release
105                 export __DistroRid="$ID.$VERSION_ID-$__BuildArch"
106             fi
107         fi
108     else
109         export __DistroRid="$__HostDistroRid"
110     fi
111
112     if [ "$__BuildOS" == "OSX" ]; then
113         __PortableBuild=1
114     fi
115
116     # Portable builds target the base RID
117     if [ $__PortableBuild == 1 ]; then
118         if [ "$__BuildOS" == "Linux" ]; then
119             export __DistroRid="linux-$__BuildArch"
120         elif [ "$__BuildOS" == "OSX" ]; then
121             export __DistroRid="osx-$__BuildArch"
122         elif [ "$__BuildOS" == "FreeBSD" ]; then
123             export __DistroRid="freebsd-$__BuildArch"
124         fi
125     fi
126 }
127
128 setup_dirs()
129 {
130     echo Setting up directories for build
131
132     mkdir -p "$__RootBinDir"
133     mkdir -p "$__BinDir"
134     mkdir -p "$__LogsDir"
135     mkdir -p "$__IntermediatesDir"
136
137     if [ $__CrossBuild == 1 ]; then
138         mkdir -p "$__CrossComponentBinDir"
139         mkdir -p "$__CrossCompIntermediatesDir"
140     fi
141 }
142
143 # Check the system to ensure the right prereqs are in place
144
145 check_prereqs()
146 {
147     echo "Checking prerequisites..."
148
149     # Check presence of CMake on the path
150     hash cmake 2>/dev/null || { echo >&2 "Please install cmake before running this script"; exit 1; }
151
152
153     # Minimum required version of clang is version 4.0 for arm/armel cross build
154     if [[ $__CrossBuild == 1 && ("$__BuildArch" == "arm" || "$__BuildArch" == "armel") ]]; then
155         if ! [[ "$__ClangMajorVersion" -ge "4" ]]; then
156             echo "Please install clang4.0 or latest for arm/armel cross build"; exit 1;
157         fi
158     fi
159
160     # Check for clang
161     hash clang-$__ClangMajorVersion.$__ClangMinorVersion 2>/dev/null ||  hash clang$__ClangMajorVersion$__ClangMinorVersion 2>/dev/null ||  hash clang 2>/dev/null || { echo >&2 "Please install clang-$__ClangMajorVersion.$__ClangMinorVersion before running this script"; exit 1; }
162
163 }
164
165 restore_optdata()
166 {
167     # we only need optdata on a Release build
168     if [[ "$__BuildType" != "Release" ]]; then __SkipRestoreOptData=1; fi
169
170     if [[ ( $__SkipRestoreOptData == 0 ) && ( $__isMSBuildOnNETCoreSupported == 1 ) ]]; then
171         echo "Restoring the OptimizationData package"
172         "$__ProjectRoot/run.sh" build -optdata $__RunArgs $__UnprocessedBuildArgs
173         if [ $? != 0 ]; then
174             echo "Failed to restore the optimization data package."
175             exit 1
176         fi
177     fi
178
179     if [ $__isMSBuildOnNETCoreSupported == 1 ]; then
180         # Parse the optdata package versions out of msbuild so that we can pass them on to CMake
181         local DotNetCli="$__ProjectRoot/Tools/dotnetcli/dotnet"
182         if [ ! -f $DotNetCli ]; then
183             source "$__ProjectRoot/init-tools.sh"
184             if [ $? != 0 ]; then
185                 echo "Failed to restore buildtools."
186                 exit 1
187             fi
188         fi
189         local OptDataProjectFilePath="$__ProjectRoot/src/.nuget/optdata/optdata.csproj"
190         __PgoOptDataVersion=$(DOTNET_SKIP_FIRST_TIME_EXPERIENCE=1 $DotNetCli msbuild $OptDataProjectFilePath /t:DumpPgoDataPackageVersion /nologo | sed 's/^\s*//')
191         __IbcOptDataVersion=$(DOTNET_SKIP_FIRST_TIME_EXPERIENCE=1 $DotNetCli msbuild $OptDataProjectFilePath /t:DumpIbcDataPackageVersion /nologo | sed 's/^\s*//')
192     fi
193 }
194
195 generate_event_logging_sources()
196 {
197     __OutputDir=$1
198     __ConsumingBuildSystem=$2
199
200     __OutputIncDir="$__OutputDir/src/inc"
201     __OutputEventingDir="$__OutputDir/eventing"
202     __OutputEventProviderDir="$__OutputEventingDir/eventprovider"
203
204     echo "Laying out dynamically generated files consumed by $__ConsumingBuildSystem"
205     echo "Laying out dynamically generated Event test files, etmdummy stub functions, and external linkages"
206
207     __PythonWarningFlags="-Wall"
208     if [[ $__IgnoreWarnings == 0 ]]; then
209         __PythonWarningFlags="$__PythonWarningFlags -Werror"
210     fi
211
212     $PYTHON -B $__PythonWarningFlags "$__ProjectRoot/src/scripts/genEventing.py" --inc $__OutputIncDir --dummy $__OutputIncDir/etmdummy.h --man "$__ProjectRoot/src/vm/ClrEtwAll.man" --testdir "$__OutputEventProviderDir/tests"
213     if [[ $? != 0 ]]; then
214         exit 1
215     fi
216
217     echo "Laying out dynamically generated EventPipe Implementation"
218     $PYTHON -B $__PythonWarningFlags "$__ProjectRoot/src/scripts/genEventPipe.py" --man "$__ProjectRoot/src/vm/ClrEtwAll.man" --intermediate "$__OutputEventingDir/eventpipe"
219
220     # determine the logging system
221     case $__BuildOS in
222         Linux|FreeBSD)
223             echo "Laying out dynamically generated Event Logging Implementation of Lttng"
224             $PYTHON -B $__PythonWarningFlags "$__ProjectRoot/src/scripts/genLttngProvider.py" --man "$__ProjectRoot/src/vm/ClrEtwAll.man" --intermediate "$__OutputEventProviderDir"
225             if [[ $? != 0 ]]; then
226                 exit 1
227             fi
228             ;;
229         *)
230             echo "Laying out dummy event logging provider"
231             $PYTHON -B $__PythonWarningFlags "$__ProjectRoot/src/scripts/genDummyProvider.py" --man "$__ProjectRoot/src/vm/ClrEtwAll.man" --intermediate "$__OutputEventProviderDir"
232             if [[ $? != 0 ]]; then
233                 exit 1
234             fi
235             ;;
236     esac
237 }
238
239 generate_event_logging()
240 {
241     # Event Logging Infrastructure
242     if [[ $__SkipCoreCLR == 0 || $__ConfigureOnly == 1 ]]; then
243         generate_event_logging_sources "$__IntermediatesDir" "the native build system"
244     fi
245
246     if [[ $__CrossBuild == 1 && $__DoCrossArchBuild == 1 ]]; then
247         generate_event_logging_sources "$__CrossCompIntermediatesDir" "the crossarch build system"
248     fi
249  }
250
251 build_native()
252 {
253     skipCondition=$1
254     platformArch="$2"
255     intermediatesForBuild="$3"
256     extraCmakeArguments="$4"
257     message="$5"
258
259     if [ $skipCondition == 1 ]; then
260         echo "Skipping $message build."
261         return
262     fi
263
264     # All set to commence the build
265     echo "Commencing build of $message for $__BuildOS.$__BuildArch.$__BuildType in $intermediatesForBuild"
266
267     generator=""
268     buildFile="Makefile"
269     buildTool="make"
270     if [ $__UseNinja == 1 ]; then
271         generator="ninja"
272         buildFile="build.ninja"
273         if ! buildTool=$(command -v ninja || command -v ninja-build); then
274            echo "Unable to locate ninja!" 1>&2
275            exit 1
276         fi
277     fi
278
279     if [ $__SkipConfigure == 0 ]; then
280         # if msbuild is not supported, then set __SkipGenerateVersion to 1
281         if [ $__isMSBuildOnNETCoreSupported == 0 ]; then __SkipGenerateVersion=1; fi
282         # Drop version.cpp file
283         __versionSourceFile="$intermediatesForBuild/version.cpp"
284         if [ $__SkipGenerateVersion == 0 ]; then
285             pwd
286             "$__ProjectRoot/run.sh" build -Project=$__ProjectDir/build.proj -generateHeaderUnix -NativeVersionSourceFile=$__versionSourceFile -MsBuildEventLogging="/l:BinClashLogger,Tools/Microsoft.DotNet.Build.Tasks.dll;LogFile=binclash.log"  $__RunArgs $__UnprocessedBuildArgs
287         else
288             # Generate the dummy version.cpp, but only if it didn't exist to make sure we don't trigger unnecessary rebuild
289             __versionSourceLine="static char sccsid[] __attribute__((used)) = \"@(#)No version information produced\";"
290             if [ -e $__versionSourceFile ]; then
291                 read existingVersionSourceLine < $__versionSourceFile
292             fi
293             if [ "$__versionSourceLine" != "$existingVersionSourceLine" ]; then
294                 echo $__versionSourceLine > $__versionSourceFile
295             fi
296         fi
297
298
299         pushd "$intermediatesForBuild"
300         # Regenerate the CMake solution
301         echo "Invoking \"$__ProjectRoot/src/pal/tools/gen-buildsys-clang.sh\" \"$__ProjectRoot\" $__ClangMajorVersion $__ClangMinorVersion $platformArch $__BuildType $__CodeCoverage $__IncludeTests $generator $extraCmakeArguments $__cmakeargs"
302         "$__ProjectRoot/src/pal/tools/gen-buildsys-clang.sh" "$__ProjectRoot" $__ClangMajorVersion $__ClangMinorVersion $platformArch $__BuildType $__CodeCoverage $__IncludeTests $generator "$extraCmakeArguments" "$__cmakeargs"
303         popd
304     fi
305
306     if [ ! -f "$intermediatesForBuild/$buildFile" ]; then
307         echo "Failed to generate $message build project!"
308         exit 1
309     fi
310
311     # Build
312     if [ $__ConfigureOnly == 1 ]; then
313         echo "Finish configuration & skipping $message build."
314         return
315     fi
316
317     # Check that the makefiles were created.
318     pushd "$intermediatesForBuild"
319
320     echo "Executing $buildTool install -j $__NumProc"
321
322     $buildTool install -j $__NumProc
323     if [ $? != 0 ]; then
324         echo "Failed to build $message."
325         exit 1
326     fi
327
328     popd
329 }
330
331 build_cross_arch_component()
332 {
333     __SkipCrossArchBuild=1
334     TARGET_ROOTFS=""
335     # check supported cross-architecture components host(__HostArch)/target(__BuildArch) pair
336     if [[ ("$__BuildArch" == "arm" || "$__BuildArch" == "armel") && "$__CrossArch" == "x86" ]]; then
337         export CROSSCOMPILE=0
338         __SkipCrossArchBuild=0
339
340         # building x64-host/arm-target cross-architecture component need to use cross toolchain of x86
341         if [ "$__HostArch" == "x64" ]; then
342             export CROSSCOMPILE=1
343         fi
344     elif [[ ("$__BuildArch" == "arm64") && "$__CrossArch" == "x64" ]]; then
345         export CROSSCOMPILE=0
346         __SkipCrossArchBuild=0
347     else
348         # not supported
349         return
350     fi
351
352     export __CMakeBinDir="$__CrossComponentBinDir"
353     export CROSSCOMPONENT=1
354     __IncludeTests=
355
356     if [ $CROSSCOMPILE == 1 ]; then
357         TARGET_ROOTFS="$ROOTFS_DIR"
358         if [ -n "$CAC_ROOTFS_DIR" ]; then
359             export ROOTFS_DIR="$CAC_ROOTFS_DIR"
360         else
361             export ROOTFS_DIR="$__ProjectRoot/cross/rootfs/$__CrossArch"
362         fi
363     fi
364
365     __ExtraCmakeArgs="-DCLR_CMAKE_TARGET_ARCH=$__BuildArch -DCLR_CMAKE_TARGET_OS=$__BuildOS -DCLR_CMAKE_PACKAGES_DIR=$__PackagesDir -DCLR_CMAKE_PGO_INSTRUMENT=$__PgoInstrument -DCLR_CMAKE_OPTDATA_VERSION=$__PgoOptDataVersion -DCLR_CMAKE_PGO_OPTIMIZE=$__PgoOptimize"
366     build_native $__SkipCrossArchBuild "$__CrossArch" "$__CrossCompIntermediatesDir" "$__ExtraCmakeArgs" "cross-architecture component"
367
368     # restore ROOTFS_DIR, CROSSCOMPONENT, and CROSSCOMPILE
369     if [ -n "$TARGET_ROOTFS" ]; then
370         export ROOTFS_DIR="$TARGET_ROOTFS"
371     fi
372     export CROSSCOMPONENT=
373     export CROSSCOMPILE=1
374 }
375
376 isMSBuildOnNETCoreSupported()
377 {
378     __isMSBuildOnNETCoreSupported=$__msbuildonunsupportedplatform
379
380     if [ $__isMSBuildOnNETCoreSupported == 1 ]; then
381         return
382     fi
383
384     if [ "$__HostArch" == "x64" ]; then
385         if [ "$__HostOS" == "Linux" ]; then
386             __isMSBuildOnNETCoreSupported=1
387             # note: the RIDs below can use globbing patterns
388             UNSUPPORTED_RIDS=("debian.9-x64" "ubuntu.17.04-x64")
389             for UNSUPPORTED_RID in "${UNSUPPORTED_RIDS[@]}"
390             do
391                 if [[ $__HostDistroRid == $UNSUPPORTED_RID ]]; then
392                     __isMSBuildOnNETCoreSupported=0
393                     break
394                 fi
395             done
396         elif [ "$__HostOS" == "OSX" ]; then
397             __isMSBuildOnNETCoreSupported=1
398         fi
399     fi
400 }
401
402
403 build_CoreLib_ni()
404 {
405     local __CrossGenExec=$1
406
407     echo "Generating native image for System.Private.CoreLib.dll"
408     echo "$__CrossGenExec /Platform_Assemblies_Paths $__BinDir/IL $__IbcTuning /out $__BinDir/System.Private.CoreLib.dll $__BinDir/IL/System.Private.CoreLib.dll"
409     $__CrossGenExec /Platform_Assemblies_Paths $__BinDir/IL $__IbcTuning /out $__BinDir/System.Private.CoreLib.dll $__BinDir/IL/System.Private.CoreLib.dll
410     if [ $? -ne 0 ]; then
411         echo "Failed to generate native image for System.Private.CoreLib."
412         exit 1
413     fi
414
415     if [ "$__BuildOS" == "Linux" ]; then
416         echo "Generating symbol file for System.Private.CoreLib.dll"
417         echo "$__CrossGenExec /Platform_Assemblies_Paths $__BinDir /CreatePerfMap $__BinDir $__BinDir/System.Private.CoreLib.dll"
418         $__CrossGenExec /Platform_Assemblies_Paths $__BinDir /CreatePerfMap $__BinDir $__BinDir/System.Private.CoreLib.dll
419         if [ $? -ne 0 ]; then
420             echo "Failed to generate symbol file for System.Private.CoreLib."
421             exit 1
422         fi
423     fi
424 }
425
426 build_CoreLib()
427 {
428     if [ $__isMSBuildOnNETCoreSupported == 0 ]; then
429         echo "System.Private.CoreLib.dll build unsupported."
430         return
431     fi
432
433     if [ $__SkipMSCorLib == 1 ]; then
434        echo "Skipping building System.Private.CoreLib."
435        return
436     fi
437
438     echo "Commencing build of managed components for $__BuildOS.$__BuildArch.$__BuildType"
439
440     # Invoke MSBuild
441     __ExtraBuildArgs=""
442     if [[ "$__IbcTuning" -eq "" ]]; then
443         __ExtraBuildArgs="$__ExtraBuildArgs -OptimizationDataDir=\"$__PackagesDir/optimization.$__BuildOS-$__BuildArch.IBC.CoreCLR/$__IbcOptDataVersion/data/\""
444         __ExtraBuildArgs="$__ExtraBuildArgs -EnableProfileGuidedOptimization=true"
445     fi
446     $__ProjectRoot/run.sh build -Project=$__ProjectDir/build.proj -MsBuildEventLogging="/l:BinClashLogger,Tools/Microsoft.DotNet.Build.Tasks.dll;LogFile=binclash.log" -MsBuildLog="/flp:Verbosity=normal;LogFile=$__LogsDir/System.Private.CoreLib_$__BuildOS__$__BuildArch__$__BuildType.log" -BuildTarget -__IntermediatesDir=$__IntermediatesDir -__RootBinDir=$__RootBinDir -BuildNugetPackage=false -UseSharedCompilation=false $__RunArgs $__ExtraBuildArgs $__UnprocessedBuildArgs
447
448     if [ $? -ne 0 ]; then
449         echo "Failed to build managed components."
450         exit 1
451     fi
452
453     if [ $__SkipCrossgen == 1 ]; then
454         echo "Skipping generating native image"
455         return
456     fi
457
458     # The cross build generates a crossgen with the target architecture.
459     if [ $__CrossBuild == 0 ]; then
460        if [ $__SkipCoreCLR == 1 ]; then
461            return
462        fi
463
464        # The architecture of host pc must be same architecture with target.
465        if [[ ( "$__HostArch" == "$__BuildArch" ) ]]; then
466            build_CoreLib_ni "$__BinDir/crossgen"
467        elif [[ ( "$__HostArch" == "x64" ) && ( "$__BuildArch" == "x86" ) ]]; then
468            build_CoreLib_ni "$__BinDir/crossgen"
469        elif [[ ( "$__HostArch" == "arm64" ) && ( "$__BuildArch" == "arm" ) ]]; then
470            build_CoreLib_ni "$__BinDir/crossgen"
471        else
472            exit 1
473        fi
474     elif [ $__DoCrossArchBuild == 1 ]; then
475        if [[ ( "$__CrossArch" == "x86" ) && ( "$__BuildArch" == "arm" ) ]]; then
476            build_CoreLib_ni "$__CrossComponentBinDir/crossgen"
477        elif [[ ( "$__HostArch" == "x64" ) && ( "$__BuildArch" == "arm64" ) ]]; then
478            build_CoreLib_ni "$__CrossComponentBinDir/crossgen"
479        fi
480     fi
481 }
482
483 generate_NugetPackages()
484 {
485     # We can only generate nuget package if we also support building mscorlib as part of this build.
486     if [ $__isMSBuildOnNETCoreSupported == 0 ]; then
487         echo "Nuget package generation unsupported."
488         return
489     fi
490
491     # Since we can build mscorlib for this OS, did we build the native components as well?
492     if [ $__SkipCoreCLR == 1 ]; then
493         echo "Unable to generate nuget packages since native components were not built."
494         return
495     fi
496
497     echo "Generating nuget packages for "$__BuildOS
498     echo "DistroRid is "$__DistroRid
499     echo "ROOTFS_DIR is "$ROOTFS_DIR
500     # Build the packages
501     $__ProjectRoot/run.sh build -Project=$__SourceDir/.nuget/packages.builds -MsBuildEventLogging="/l:BinClashLogger,Tools/Microsoft.DotNet.Build.Tasks.dll;LogFile=binclash.log" -MsBuildLog="/flp:Verbosity=normal;LogFile=$__LogsDir/Nuget_$__BuildOS__$__BuildArch__$__BuildType.log" -BuildTarget -__IntermediatesDir=$__IntermediatesDir -__RootBinDir=$__RootBinDir -BuildNugetPackage=false -UseSharedCompilation=false -__DoCrossArchBuild=$__DoCrossArchBuild $__RunArgs $__UnprocessedBuildArgs
502
503     if [ $? -ne 0 ]; then
504         echo "Failed to generate Nuget packages."
505         exit 1
506     fi
507 }
508
509 echo "Commencing CoreCLR Repo build"
510
511 # Argument types supported by this script:
512 #
513 # Build architecture - valid values are: x64, ARM.
514 # Build Type         - valid values are: Debug, Checked, Release
515 #
516 # Set the default arguments for build
517
518 # Obtain the location of the bash script to figure out where the root of the repo is.
519 __ProjectRoot="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
520
521 # Use uname to determine what the CPU is.
522 CPUName=$(uname -p)
523 # Some Linux platforms report unknown for platform, but the arch for machine.
524 if [ "$CPUName" == "unknown" ]; then
525     CPUName=$(uname -m)
526 fi
527
528 case $CPUName in
529     i686)
530         echo "Unsupported CPU $CPUName detected, build might not succeed!"
531         __BuildArch=x86
532         __HostArch=x86
533         ;;
534
535     x86_64)
536         __BuildArch=x64
537         __HostArch=x64
538         ;;
539
540     armv7l)
541         echo "Unsupported CPU $CPUName detected, build might not succeed!"
542         __BuildArch=arm
543         __HostArch=arm
544         ;;
545
546     aarch64)
547         __BuildArch=arm64
548         __HostArch=arm64
549         ;;
550
551     amd64)
552         __BuildArch=x64
553         __HostArch=x64
554         ;;
555     *)
556         echo "Unknown CPU $CPUName detected, configuring as if for x64"
557         __BuildArch=x64
558         __HostArch=x64
559         ;;
560 esac
561
562 # Use uname to determine what the OS is.
563 OSName=$(uname -s)
564 case $OSName in
565     Linux)
566         __BuildOS=Linux
567         __HostOS=Linux
568         ;;
569
570     Darwin)
571         __BuildOS=OSX
572         __HostOS=OSX
573         ;;
574
575     FreeBSD)
576         __BuildOS=FreeBSD
577         __HostOS=FreeBSD
578         ;;
579
580     OpenBSD)
581         __BuildOS=OpenBSD
582         __HostOS=OpenBSD
583         ;;
584
585     NetBSD)
586         __BuildOS=NetBSD
587         __HostOS=NetBSD
588         ;;
589
590     SunOS)
591         __BuildOS=SunOS
592         __HostOS=SunOS
593         ;;
594
595     *)
596         echo "Unsupported OS $OSName detected, configuring as if for Linux"
597         __BuildOS=Linux
598         __HostOS=Linux
599         ;;
600 esac
601
602 __BuildType=Debug
603 __CodeCoverage=
604 __IncludeTests=Include_Tests
605 __IgnoreWarnings=0
606
607 # Set the various build properties here so that CMake and MSBuild can pick them up
608 __ProjectDir="$__ProjectRoot"
609 __SourceDir="$__ProjectDir/src"
610 __PackagesDir="${DotNetRestorePackagesPath:-${__ProjectDir}/packages}"
611 __RootBinDir="$__ProjectDir/bin"
612 __UnprocessedBuildArgs=
613 __RunArgs=
614 __MSBCleanBuildArgs=
615 __UseNinja=0
616 __VerboseBuild=0
617 __PgoInstrument=0
618 __PgoOptimize=1
619 __IbcTuning=""
620 __ConfigureOnly=0
621 __SkipConfigure=0
622 __SkipRestore=""
623 __SkipNuget=0
624 __SkipCoreCLR=0
625 __SkipMSCorLib=0
626 __SkipRestoreOptData=0
627 __SkipCrossgen=0
628 __CrossBuild=0
629 __ClangMajorVersion=0
630 __ClangMinorVersion=0
631 __NuGetPath="$__PackagesDir/NuGet.exe"
632 __HostDistroRid=""
633 __DistroRid=""
634 __cmakeargs=""
635 __SkipGenerateVersion=0
636 __DoCrossArchBuild=0
637 __PortableBuild=1
638 __msbuildonunsupportedplatform=0
639 __PgoOptDataVersion=""
640 __IbcOptDataVersion=""
641
642 # Get the number of processors available to the scheduler
643 # Other techniques such as `nproc` only get the number of
644 # processors available to a single process.
645 if [ `uname` = "FreeBSD" ]; then
646   __NumProc=`sysctl hw.ncpu | awk '{ print $2+1 }'`
647 elif [ `uname` = "NetBSD" ]; then
648   __NumProc=$(($(getconf NPROCESSORS_ONLN)+1))
649 elif [ `uname` = "Darwin" ]; then
650   __NumProc=$(($(getconf _NPROCESSORS_ONLN)+1))
651 else
652   __NumProc=$(nproc --all)
653 fi
654
655 while :; do
656     if [ $# -le 0 ]; then
657         break
658     fi
659
660     lowerI="$(echo $1 | awk '{print tolower($0)}')"
661     case $lowerI in
662         -\?|-h|--help)
663             usage
664             exit 1
665             ;;
666
667         x86|-x86)
668             __BuildArch=x86
669             ;;
670
671         x64|-x64)
672             __BuildArch=x64
673             ;;
674
675         arm|-arm)
676             __BuildArch=arm
677             ;;
678
679         armel|-armel)
680             __BuildArch=armel
681             ;;
682
683         arm64|-arm64)
684             __BuildArch=arm64
685             ;;
686
687         debug|-debug)
688             __BuildType=Debug
689             ;;
690
691         checked|-checked)
692             __BuildType=Checked
693             ;;
694
695         release|-release)
696             __BuildType=Release
697             ;;
698
699         coverage|-coverage)
700             __CodeCoverage=Coverage
701             ;;
702
703         cross|-cross)
704             __CrossBuild=1
705             ;;
706
707         -portablebuild=false)
708             __PortableBuild=0
709             ;;
710
711         verbose|-verbose)
712             __VerboseBuild=1
713             ;;
714
715         stripsymbols|-stripsymbols)
716             __cmakeargs="$__cmakeargs -DSTRIP_SYMBOLS=true"
717             ;;
718
719         clang3.5|-clang3.5)
720             __ClangMajorVersion=3
721             __ClangMinorVersion=5
722             ;;
723
724         clang3.6|-clang3.6)
725             __ClangMajorVersion=3
726             __ClangMinorVersion=6
727             ;;
728
729         clang3.7|-clang3.7)
730             __ClangMajorVersion=3
731             __ClangMinorVersion=7
732             ;;
733
734         clang3.8|-clang3.8)
735             __ClangMajorVersion=3
736             __ClangMinorVersion=8
737             ;;
738
739         clang3.9|-clang3.9)
740             __ClangMajorVersion=3
741             __ClangMinorVersion=9
742             ;;
743
744         clang4.0|-clang4.0)
745             __ClangMajorVersion=4
746             __ClangMinorVersion=0
747             ;;
748
749         clang5.0|-clang5.0)
750             __ClangMajorVersion=5
751             __ClangMinorVersion=0
752             ;;
753
754         clang6.0|-clang6.0)
755             __ClangMajorVersion=6
756             __ClangMinorVersion=0
757             ;;
758
759         ninja|-ninja)
760             __UseNinja=1
761             ;;
762
763         pgoinstrument|-pgoinstrument)
764             __PgoInstrument=1
765             ;;
766
767         nopgooptimize|-nopgooptimize)
768             __PgoOptimize=0
769             __SkipRestoreOptData=1
770             ;;
771
772         ibcinstrument|-ibcinstrument)
773             __IbcTuning="/Tuning"
774             ;;
775
776         configureonly|-configureonly)
777             __ConfigureOnly=1
778             __SkipMSCorLib=1
779             __SkipNuget=1
780             ;;
781
782         skipconfigure|-skipconfigure)
783             __SkipConfigure=1
784             ;;
785
786         skipnative|-skipnative)
787             # Use "skipnative" to use the same option name as build.cmd.
788             __SkipCoreCLR=1
789             ;;
790
791         skipcoreclr|-skipcoreclr)
792             # Accept "skipcoreclr" for backwards-compatibility.
793             __SkipCoreCLR=1
794             ;;
795
796         crosscomponent|-crosscomponent)
797             __DoCrossArchBuild=1
798             ;;
799
800         skipmscorlib|-skipmscorlib)
801             __SkipMSCorLib=1
802             ;;
803
804         skipgenerateversion|-skipgenerateversion)
805             __SkipGenerateVersion=1
806             ;;
807
808         skiprestoreoptdata|-skiprestoreoptdata)
809             __SkipRestoreOptData=1
810             ;;
811
812         skipcrossgen|-skipcrossgen)
813             __SkipCrossgen=1
814             ;;
815
816         includetests|-includetests)
817             ;;
818
819         skiptests|-skiptests)
820             __IncludeTests=
821             ;;
822
823         skipnuget|-skipnuget)
824             __SkipNuget=1
825             ;;
826
827         ignorewarnings|-ignorewarnings)
828             __IgnoreWarnings=1
829             __cmakeargs="$__cmakeargs -DCLR_CMAKE_WARNINGS_ARE_ERRORS=OFF"
830             ;;
831
832         cmakeargs|-cmakeargs)
833             if [ -n "$2" ]; then
834                 __cmakeargs="$__cmakeargs $2"
835                 shift
836             else
837                 echo "ERROR: 'cmakeargs' requires a non-empty option argument"
838                 exit 1
839             fi
840             ;;
841
842         bindir|-bindir)
843             if [ -n "$2" ]; then
844                 __RootBinDir="$2"
845                 if [ ! -d $__RootBinDir ]; then
846                     mkdir $__RootBinDir
847                 fi
848                 __RootBinParent=$(dirname $__RootBinDir)
849                 __RootBinName=${__RootBinDir##*/}
850                 __RootBinDir="$(cd $__RootBinParent &>/dev/null && printf %s/%s $PWD $__RootBinName)"
851                 shift
852             else
853                 echo "ERROR: 'bindir' requires a non-empty option argument"
854                 exit 1
855             fi
856             ;;
857         msbuildonunsupportedplatform|-msbuildonunsupportedplatform)
858             __msbuildonunsupportedplatform=1
859             ;;
860         numproc|-numproc)
861             if [ -n "$2" ]; then
862               __NumProc="$2"
863               shift
864             else
865               echo "ERROR: 'numproc' requires a non-empty option argument"
866               exit 1
867             fi
868             ;;
869         osgroup|-osgroup)
870             if [ -n "$2" ]; then
871               __BuildOS="$2"
872               shift
873             else
874               echo "ERROR: 'osgroup' requires a non-empty option argument"
875               exit 1
876             fi
877             ;;
878
879         *)
880             __UnprocessedBuildArgs="$__UnprocessedBuildArgs $1"
881             ;;
882     esac
883
884     shift
885 done
886
887 __RunArgs="-BuildArch=$__BuildArch -BuildType=$__BuildType -BuildOS=$__BuildOS"
888
889 # Configure environment if we are doing a verbose build
890 if [ $__VerboseBuild == 1 ]; then
891     export VERBOSE=1
892         __RunArgs="$__RunArgs -verbose"
893 fi
894
895 # Set default clang version
896 if [[ $__ClangMajorVersion == 0 && $__ClangMinorVersion == 0 ]]; then
897         if [[ "$__BuildArch" == "arm" || "$__BuildArch" == "armel" ]]; then
898                 __ClangMajorVersion=5
899                 __ClangMinorVersion=0
900         else
901                 __ClangMajorVersion=3
902                 __ClangMinorVersion=9
903         fi
904 fi
905
906 if [[ "$__BuildArch" == "armel" ]]; then
907     # Armel cross build is Tizen specific and does not support Portable RID build
908     __PortableBuild=0
909 fi
910
911 if [ $__PortableBuild == 0 ]; then
912         __RunArgs="$__RunArgs -PortableBuild=false"
913 fi
914
915 # Set dependent variables
916 __LogsDir="$__RootBinDir/Logs"
917
918 # init the host distro name
919 initHostDistroRid
920
921 # Set the remaining variables based upon the determined build configuration
922 __BinDir="$__RootBinDir/Product/$__BuildOS.$__BuildArch.$__BuildType"
923 __PackagesBinDir="$__BinDir/.nuget"
924 __ToolsDir="$__RootBinDir/tools"
925 __TestWorkingDir="$__RootBinDir/tests/$__BuildOS.$__BuildArch.$__BuildType"
926 export __IntermediatesDir="$__RootBinDir/obj/$__BuildOS.$__BuildArch.$__BuildType"
927 __TestIntermediatesDir="$__RootBinDir/tests/obj/$__BuildOS.$__BuildArch.$__BuildType"
928 __isMSBuildOnNETCoreSupported=0
929 __CrossComponentBinDir="$__BinDir"
930 __CrossCompIntermediatesDir="$__IntermediatesDir/crossgen"
931
932 __CrossArch="$__HostArch"
933 if [[ "$__HostArch" == "x64" && ("$__BuildArch" == "arm" || "$__BuildArch" == "armel") ]]; then
934     __CrossArch="x86"
935 fi
936 if [ $__CrossBuild == 1 ]; then
937     __CrossComponentBinDir="$__CrossComponentBinDir/$__CrossArch"
938 fi
939 __CrossgenCoreLibLog="$__LogsDir/CrossgenCoreLib_$__BuildOS.$BuildArch.$__BuildType.log"
940 __CrossgenExe="$__CrossComponentBinDir/crossgen"
941
942 # Init if MSBuild for .NET Core is supported for this platform
943 isMSBuildOnNETCoreSupported
944
945 # CI_SPECIFIC - On CI machines, $HOME may not be set. In such a case, create a subfolder and set the variable to set.
946 # This is needed by CLI to function.
947 if [ -z "$HOME" ]; then
948     if [ ! -d "$__ProjectDir/temp_home" ]; then
949         mkdir temp_home
950     fi
951     export HOME=$__ProjectDir/temp_home
952     echo "HOME not defined; setting it to $HOME"
953 fi
954
955 # Specify path to be set for CMAKE_INSTALL_PREFIX.
956 # This is where all built CoreClr libraries will copied to.
957 export __CMakeBinDir="$__BinDir"
958
959 # Configure environment if we are doing a cross compile.
960 if [ $__CrossBuild == 1 ]; then
961     export CROSSCOMPILE=1
962     if ! [[ -n "$ROOTFS_DIR" ]]; then
963         export ROOTFS_DIR="$__ProjectRoot/cross/rootfs/$__BuildArch"
964     fi
965 fi
966
967 # init the target distro name
968 initTargetDistroRid
969
970 # Make the directories necessary for build if they don't exist
971 setup_dirs
972
973 # Check prereqs.
974 check_prereqs
975
976 # Restore the package containing profile counts for profile-guided optimizations
977 restore_optdata
978
979 # Generate event logging infrastructure sources
980 generate_event_logging
981
982 # Build the coreclr (native) components.
983 __ExtraCmakeArgs="-DCLR_CMAKE_TARGET_OS=$__BuildOS -DCLR_CMAKE_PACKAGES_DIR=$__PackagesDir -DCLR_CMAKE_PGO_INSTRUMENT=$__PgoInstrument -DCLR_CMAKE_OPTDATA_VERSION=$__PgoOptDataVersion -DCLR_CMAKE_PGO_OPTIMIZE=$__PgoOptimize"
984 build_native $__SkipCoreCLR "$__BuildArch" "$__IntermediatesDir" "$__ExtraCmakeArgs" "CoreCLR component"
985
986 # Build cross-architecture components
987 if [[ $__CrossBuild == 1 && $__DoCrossArchBuild == 1 ]]; then
988     build_cross_arch_component
989 fi
990
991 # Build System.Private.CoreLib.
992
993 build_CoreLib
994
995 # Generate nuget packages
996 if [ $__SkipNuget != 1 ]; then
997     generate_NugetPackages
998 fi
999
1000
1001 # Build complete
1002
1003 echo "Repo successfully built."
1004 echo "Product binaries are available at $__BinDir"
1005 exit 0