Merge pull request #14647 from BruceForstall/AddArm64FrameLayoutDoc
[platform/upstream/coreclr.git] / build-test.sh
1 #!/usr/bin/env bash
2
3 initHostDistroRid()
4 {
5     __HostDistroRid=""
6     if [ "$__HostOS" == "Linux" ]; then
7         if [ -e /etc/os-release ]; then
8             source /etc/os-release
9             if [[ $ID == "alpine" ]]; then
10                 # remove the last version digit
11                 VERSION_ID=${VERSION_ID%.*}
12             fi
13             __HostDistroRid="$ID.$VERSION_ID-$__HostArch"
14         elif [ -e /etc/redhat-release ]; then
15             local redhatRelease=$(</etc/redhat-release)
16             if [[ $redhatRelease == "CentOS release 6."* || $redhatRelease == "Red Hat Enterprise Linux Server release 6."* ]]; then
17                __HostDistroRid="rhel.6-$__HostArch"
18             fi
19         fi
20     fi
21     if [ "$__HostOS" == "FreeBSD" ]; then
22         __freebsd_version=`sysctl -n kern.osrelease | cut -f1 -d'.'`
23         __HostDistroRid="freebsd.$__freebsd_version-$__HostArch"
24     fi
25
26     if [ "$__HostDistroRid" == "" ]; then
27         echo "WARNING: Cannot determine runtime id for current distro."
28     fi
29 }
30
31 initTargetDistroRid()
32 {
33     if [ $__CrossBuild == 1 ]; then
34         if [ "$__BuildOS" == "Linux" ]; then
35             if [ ! -e $ROOTFS_DIR/etc/os-release ]; then
36                 echo "WARNING: Can not determine runtime id for current distro."
37                 export __DistroRid=""
38             else
39                 source $ROOTFS_DIR/etc/os-release
40                 export __DistroRid="$ID.$VERSION_ID-$__BuildArch"
41             fi
42         fi
43     else
44         export __DistroRid="$__HostDistroRid"
45     fi
46
47     if [ "$__BuildOS" == "OSX" ]; then
48         __PortableBuild=1
49     fi
50
51     # Portable builds target the base RID
52     if [ "$__PortableBuild" == 1 ]; then
53         if [ "$__BuildOS" == "Linux" ]; then
54             export __DistroRid="linux-$__BuildArch"
55         elif [ "$__BuildOS" == "OSX" ]; then
56             export __DistroRid="osx-$__BuildArch"
57         fi
58     fi
59
60    if [ "$ID.$VERSION_ID" == "ubuntu.16.04" ]; then
61      export __DistroRid="ubuntu.14.04-$__BuildArch"
62    fi
63
64    echo "__DistroRid: " $__DistroRid
65 }
66
67 isMSBuildOnNETCoreSupported()
68 {
69     __isMSBuildOnNETCoreSupported=$__msbuildonunsupportedplatform
70
71     if [ $__isMSBuildOnNETCoreSupported == 1 ]; then
72         return
73     fi
74
75     if [ "$__HostArch" == "x64" ]; then
76         if [ "$__HostOS" == "Linux" ]; then
77             __isMSBuildOnNETCoreSupported=1
78             UNSUPPORTED_RIDS=("debian.9-x64" "ubuntu.17.04-x64")
79             for UNSUPPORTED_RID in "${UNSUPPORTED_RIDS[@]}"
80             do
81                 if [ "$__HostDistroRid" == "$UNSUPPORTED_RID" ]; then
82                     __isMSBuildOnNETCoreSupported=0
83                     break
84                 fi
85             done
86         elif [ "$__HostOS" == "OSX" ]; then
87             __isMSBuildOnNETCoreSupported=1
88         fi
89     fi
90 }
91
92 build_Tests()
93 {
94     __TestDir=$__ProjectDir/tests
95     __ProjectFilesDir=$__TestDir
96     __TestBinDir=$__TestWorkingDir
97
98     if [ $__RebuildTests -ne 0 ]; then
99         if [ -d "${__TestBinDir}" ]; then
100             echo "Removing tests build dir: ${__TestBinDir}"
101             rm -rf $__TestBinDir
102         fi
103     fi
104
105     __CMakeBinDir="${__TestBinDir}"
106
107     if [ -z "$__TestIntermediateDir" ]; then
108         __TestIntermediateDir="tests/obj/${__BuildOS}.${__BuildArch}.${__BuildType}"
109     fi
110
111         echo "__BuildOS: ${__BuildOS}"
112         echo "__BuildArch: ${__BuildArch}"
113         echo "__BuildType: ${__BuildType}"
114         echo "__TestIntermediateDir: ${__TestIntermediateDir}"
115
116     if [ ! -f "$__TestBinDir" ]; then
117         echo "Creating TestBinDir: ${__TestBinDir}"
118         mkdir -p $__TestBinDir
119     fi
120     if [ ! -f "$__LogsDir" ]; then
121         echo "Creating LogsDir: ${__LogsDir}"
122         mkdir -p $__LogsDir
123     fi
124
125     __BuildProperties="-p:OSGroup=${__BuildOS} -p:BuildOS=${__BuildOS} -p:BuildArch=${__BuildArch} -p:BuildType=${__BuildType}"
126
127     # =========================================================================================
128     # ===
129     # === Restore product binaries from packages
130     # ===
131     # =========================================================================================
132
133     build_Tests_internal "Restore_Product" "${__ProjectDir}/tests/build.proj" " -BatchRestorePackages" "Restore product binaries (build tests)"
134
135     build_Tests_internal "Tests_GenerateRuntimeLayout" "${__ProjectDir}/tests/runtest.proj" "-BinPlaceRef -BinPlaceProduct -CopyCrossgenToProduct" "Restore product binaries (run tests)"
136
137     if [ -n "$__UpdateInvalidPackagesArg" ]; then
138         __up=-updateinvalidpackageversion
139     fi
140
141     # Work hardcoded path around
142     if [ ! -f "${__BuildToolsDir}/Microsoft.CSharp.Core.Targets" ]; then
143         ln -s "${__BuildToolsDir}/Microsoft.CSharp.Core.targets" "${__BuildToolsDir}/Microsoft.CSharp.Core.Targets"
144     fi
145     if [ ! -f "${__BuildToolsDir}/Microsoft.CSharp.targets" ]; then
146         ln -s "${__BuildToolsDir}/Microsoft.CSharp.Targets" "${__BuildToolsDir}/Microsoft.CSharp.targets"
147     fi
148
149     echo "Starting the Managed Tests Build..."
150
151     __ManagedTestBuiltMarker=${__TestBinDir}/managed_test_build
152
153     if [ ! -f $__ManagedTestBuiltMarker ]; then
154
155             build_Tests_internal "Tests_Managed" "$__ProjectDir/tests/build.proj" "$__up" "Managed tests build (build tests)"
156
157         if [ $? -ne 0 ]; then
158             echo "${__MsgPrefix}Error: build failed. Refer to the build log files for details (above)"
159             exit 1
160         else
161             echo "Tests have been built."
162             echo "Create marker \"${__ManagedTestBuiltMarker}\""
163             touch $__ManagedTestBuiltMarker
164         fi
165     else
166         echo "Managed Tests had been built before."
167     fi
168
169     if [ $__BuildTestWrappers -ne -0 ]; then
170         echo "${__MsgPrefix}Creating test wrappers..."
171
172         __XUnitWrapperBuiltMarker=${__TestBinDir}/xunit_wrapper_build
173
174         if [ ! -f $__XUnitWrapperBuiltMarker ]; then
175
176             build_Tests_internal "Tests_XunitWrapper" "$__ProjectDir/tests/runtest.proj" "-BuildWrappers -MsBuildEventLogging=\" \" " "Test Xunit Wrapper"
177
178             if [ $? -ne 0 ]; then
179                 echo "${__MsgPrefix}Error: build failed. Refer to the build log files for details (above)"
180                 exit 1
181             else
182                 echo "XUnit Wrappers have been built."
183                 echo "Create marker \"${__XUnitWrapperBuiltMarker}\""
184                 touch $__XUnitWrapperBuiltMarker
185             fi
186         else
187             echo "XUnit Wrappers had been built before."
188         fi
189     fi
190
191     echo "${__MsgPrefix}Creating test overlay..."
192
193     if [ -z "$XuintTestBinBase" ]; then
194       XuintTestBinBase=$__TestWorkingDir
195     fi
196
197     export CORE_ROOT=$XuintTestBinBase/Tests/Core_Root
198
199     if [ ! -f "${CORE_ROOT}" ]; then
200       mkdir -p $CORE_ROOT
201     else
202       rm -rf $CORE_ROOT/*
203     fi
204
205     cp -r $__BinDir/* $CORE_ROOT/ > /dev/null
206
207     build_Tests_internal "Tests_Overlay_Managed" "$__ProjectDir/tests/runtest.proj" "-testOverlay" "Creating test overlay"
208
209     if [ $__ZipTests -ne 0 ]; then
210         echo "${__MsgPrefix}ZIP tests packages..."
211         build_Tests_internal "Helix_Prep" "$__ProjectDir/tests/helixprep.proj" " " "Prep test binaries for Helix publishing"
212     fi
213 }
214
215 build_Tests_internal()
216 {
217         subDirectoryName=$1
218         projectName=$2
219         extraBuildParameters=$3
220         stepName="$4"
221
222         # Set up directories and file names
223         __BuildLogRootName=$subDirectoryName
224     __BuildLog="$__LogsDir/${__BuildLogRootName}.${__BuildOS}.${__BuildArch}.${__BuildType}.log"
225     __BuildWrn="$__LogsDir/${__BuildLogRootName}.${__BuildOS}.${__BuildArch}.${__BuildType}.wrn"
226     __BuildErr="$__LogsDir/${__BuildLogRootName}.${__BuildOS}.${__BuildArch}.${__BuildType}.err"
227     __msbuildLog="\"/flp:Verbosity=normal;LogFile=${__BuildLog}\""
228     __msbuildWrn="\"/flp1:WarningsOnly;LogFile=${__BuildWrn}\""
229     __msbuildErr="\"/flp2:ErrorsOnly;LogFile=${__BuildErr}\""
230
231     # Generate build command
232     buildCommand="$__ProjectRoot/run.sh build -Project=$projectName -MsBuildLog=${__msbuildLog} -MsBuildWrn=${__msbuildWrn} -MsBuildErr=${__msbuildErr} $extraBuildParameters $__RunArgs $__UnprocessedBuildArgs"
233
234     echo "Building step '$stepName' via $buildCommand"
235
236     # Invoke MSBuild
237     eval $buildCommand
238
239     # Invoke MSBuild
240     # $__ProjectRoot/run.sh build -Project=$projectName -MsBuildLog="$__msbuildLog" -MsBuildWrn="$__msbuildWrn" -MsBuildErr="$__msbuildErr" $extraBuildParameters $__RunArgs $__UnprocessedBuildArgs
241
242     # Make sure everything is OK
243     if [ $? -ne 0 ]; then
244         echo "${__MsgPrefix}Failed to build $stepName. See the build logs:"
245         echo "    $__BuildLog"
246         echo "    $__BuildWrn"
247         echo "    $__BuildErr"
248         exit 1
249     fi
250 }
251
252 usage()
253 {
254     echo "Usage: $0 [BuildArch] [BuildType] [verbose] [coverage] [cross] [clangx.y] [ninja] [runtests] [bindir]"
255     echo "BuildArch can be: x64, x86, arm, armel, arm64"
256     echo "BuildType can be: debug, checked, release"
257     echo "coverage - optional argument to enable code coverage build (currently supported only for Linux and OSX)."
258     echo "ninja - target ninja instead of GNU make"
259     echo "clangx.y - optional argument to build using clang version x.y."
260     echo "cross - optional argument to signify cross compilation,"
261     echo "      - will use ROOTFS_DIR environment variable if set."
262     echo "crosscomponent - optional argument to build cross-architecture component,"
263     echo "               - will use CAC_ROOTFS_DIR environment variable if set."
264     echo "portableLinux - build for Portable Linux Distribution"
265     echo "verbose - optional argument to enable verbose build output."
266     echo "rebuild - if tests have already been built - rebuild them"
267     echo "runtests - run tests after building them"
268     echo "ziptests - zips CoreCLR tests & Core_Root for a Helix run"
269     echo "bindir - output directory (defaults to $__ProjectRoot/bin)"
270     echo "msbuildonunsupportedplatform - build managed binaries even if distro is not officially supported."
271     exit 1
272 }
273
274
275 # Obtain the location of the bash script to figure out where the root of the repo is.
276 __ProjectRoot="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
277
278 # $__ProjectRoot/build.sh $1 $2
279
280 # Use uname to determine what the CPU is.
281 CPUName=$(uname -p)
282
283 # Some Linux platforms report unknown for platform, but the arch for machine.
284 if [ "$CPUName" == "unknown" ]; then
285     CPUName=$(uname -m)
286 fi
287
288 case $CPUName in
289     i686)
290         echo "Unsupported CPU $CPUName detected, build might not succeed!"
291         __BuildArch=x86
292         __HostArch=x86
293         ;;
294
295     x86_64)
296         __BuildArch=x64
297         __HostArch=x64
298         ;;
299
300     armv7l)
301         echo "Unsupported CPU $CPUName detected, build might not succeed!"
302         __BuildArch=arm
303         __HostArch=arm
304         ;;
305
306     aarch64)
307         __BuildArch=arm64
308         __HostArch=arm64
309         ;;
310
311     *)
312         echo "Unknown CPU $CPUName detected, configuring as if for x64"
313         __BuildArch=x64
314         __HostArch=x64
315         ;;
316 esac
317
318 # Use uname to determine what the OS is.
319 OSName=$(uname -s)
320 case $OSName in
321     Linux)
322         __BuildOS=Linux
323         __HostOS=Linux
324         ;;
325
326     Darwin)
327         __BuildOS=OSX
328         __HostOS=OSX
329         ;;
330
331     FreeBSD)
332         __BuildOS=FreeBSD
333         __HostOS=FreeBSD
334         ;;
335
336     OpenBSD)
337         __BuildOS=OpenBSD
338         __HostOS=OpenBSD
339         ;;
340
341     NetBSD)
342         __BuildOS=NetBSD
343         __HostOS=NetBSD
344         ;;
345
346     SunOS)
347         __BuildOS=SunOS
348         __HostOS=SunOS
349         ;;
350
351     *)
352         echo "Unsupported OS $OSName detected, configuring as if for Linux"
353         __BuildOS=Linux
354         __HostOS=Linux
355         ;;
356 esac
357
358 __BuildType=Debug
359 __CodeCoverage=
360 __IncludeTests=INCLUDE_TESTS
361
362 # Set the various build properties here so that CMake and MSBuild can pick them up
363 export __ProjectDir="$__ProjectRoot"
364 __SourceDir="$__ProjectDir/src"
365 __PackagesDir="$__ProjectDir/packages"
366 __RootBinDir="$__ProjectDir/bin"
367 __BuildToolsDir="$__ProjectDir/Tools"
368 __UnprocessedBuildArgs=
369 __RunArgs=
370 __MSBCleanBuildArgs=
371 __UseNinja=0
372 __VerboseBuild=0
373 __SkipRestore=""
374 __CrossBuild=0
375 __ClangMajorVersion=0
376 __ClangMinorVersion=0
377 __NuGetPath="$__PackagesDir/NuGet.exe"
378 __HostDistroRid=""
379 __DistroRid=""
380 __cmakeargs=""
381 __PortableLinux=0
382 __msbuildonunsupportedplatform=0
383 __ZipTests=0
384 __NativeTestIntermediatesDir=
385 __RunTests=0
386 __RebuildTests=0
387 __BuildTestWrappers=0
388 CORE_ROOT=
389
390
391 while :; do
392     if [ $# -le 0 ]; then
393         break
394     fi
395
396     lowerI="$(echo $1 | awk '{print tolower($0)}')"
397     case $lowerI in
398         -\?|-h|--help)
399             usage
400             exit 1
401             ;;
402
403         x86)
404             __BuildArch=x86
405             ;;
406
407         x64)
408             __BuildArch=x64
409             ;;
410
411         arm)
412             __BuildArch=arm
413             ;;
414
415         armel)
416             __BuildArch=armel
417             ;;
418
419         arm64)
420             __BuildArch=arm64
421             ;;
422
423         debug)
424             __BuildType=Debug
425             ;;
426
427         checked)
428             __BuildType=Checked
429             ;;
430
431         release)
432             __BuildType=Release
433             ;;
434
435         coverage)
436             __CodeCoverage=Coverage
437             ;;
438
439         cross)
440             __CrossBuild=1
441             ;;
442
443         portablelinux)
444             if [ "$__BuildOS" == "Linux" ]; then
445                 __PortableLinux=1
446             else
447                 echo "ERROR: portableLinux not supported for non-Linux platforms."
448                 exit 1
449             fi
450             ;;
451
452         verbose)
453         __VerboseBuild=1
454         ;;
455
456         clang3.5)
457             __ClangMajorVersion=3
458             __ClangMinorVersion=5
459             ;;
460
461         clang3.6)
462             __ClangMajorVersion=3
463             __ClangMinorVersion=6
464             ;;
465
466         clang3.7)
467             __ClangMajorVersion=3
468             __ClangMinorVersion=7
469             ;;
470
471         clang3.8)
472             __ClangMajorVersion=3
473             __ClangMinorVersion=8
474             ;;
475
476         clang3.9)
477             __ClangMajorVersion=3
478             __ClangMinorVersion=9
479             ;;
480
481         ninja)
482             __UseNinja=1
483             ;;
484
485         runtests)
486             __RunTests=1
487             ;;
488
489         rebuild)
490             __RebuildTests=1
491             ;;
492
493         ziptests)
494             __ZipTests=1
495             ;;
496
497         bindir)
498             if [ -n "$2" ]; then
499                 __RootBinDir="$2"
500                 if [ ! -d $__RootBinDir ]; then
501                     mkdir $__RootBinDir
502                 fi
503                 __RootBinParent=$(dirname $__RootBinDir)
504                 __RootBinName=${__RootBinDir##*/}
505                 __RootBinDir="$(cd $__RootBinParent &>/dev/null && printf %s/%s $PWD $__RootBinName)"
506                 shift
507             else
508                 echo "ERROR: 'bindir' requires a non-empty option argument"
509                 exit 1
510             fi
511             ;;
512
513         msbuildonunsupportedplatform)
514             __msbuildonunsupportedplatform=1
515             ;;
516         *)
517             __UnprocessedBuildArgs="$__UnprocessedBuildArgs $1"
518             ;;
519     esac
520
521     shift
522 done
523
524
525 __RunArgs="-BuildArch=$__BuildArch -BuildType=$__BuildType -BuildOS=$__BuildOS"
526
527 # Configure environment if we are doing a verbose build
528 if [ $__VerboseBuild == 1 ]; then
529     export VERBOSE=1
530         __RunArgs="$__RunArgs -verbose"
531 fi
532
533 # Set default clang version
534 if [[ $__ClangMajorVersion == 0 && $__ClangMinorVersion == 0 ]]; then
535     if [ $__CrossBuild == 1 ]; then
536         __ClangMajorVersion=3
537         __ClangMinorVersion=6
538     else
539         __ClangMajorVersion=3
540         __ClangMinorVersion=5
541     fi
542 fi
543
544
545 # Set dependent variables
546 __LogsDir="$__RootBinDir/Logs"
547
548 # init the host distro name
549 initHostDistroRid
550
551 # Set the remaining variables based upon the determined build configuration
552 __BinDir="$__RootBinDir/Product/$__BuildOS.$__BuildArch.$__BuildType"
553 __PackagesBinDir="$__BinDir/.nuget"
554 __ToolsDir="$__RootBinDir/tools"
555 __TestDir="$__ProjectDir/tests"
556 __TestWorkingDir="$__RootBinDir/tests/$__BuildOS.$__BuildArch.$__BuildType"
557 __IntermediatesDir="$__RootBinDir/obj/$__BuildOS.$__BuildArch.$__BuildType"
558 __TestIntermediatesDir="$__RootBinDir/tests/obj/$__BuildOS.$__BuildArch.$__BuildType"
559 __isMSBuildOnNETCoreSupported=0
560 __CrossComponentBinDir="$__BinDir"
561 __CrossCompIntermediatesDir="$__IntermediatesDir/crossgen"
562
563 __CrossArch="$__HostArch"
564 if [[ "$__HostArch" == "x64" && "$__BuildArch" == "arm" ]]; then
565     __CrossArch="x86"
566 fi
567 if [ $__CrossBuild == 1 ]; then
568     __CrossComponentBinDir="$__CrossComponentBinDir/$__CrossArch"
569 fi
570 __CrossgenCoreLibLog="$__LogsDir/CrossgenCoreLib_$__BuildOS.$BuildArch.$__BuildType.log"
571 __CrossgenExe="$__CrossComponentBinDir/crossgen"
572
573 isMSBuildOnNETCoreSupported
574
575 # CI_SPECIFIC - On CI machines, $HOME may not be set. In such a case, create a subfolder and set the variable to set.
576 # This is needed by CLI to function.
577 if [ -z "$HOME" ]; then
578     if [ ! -d "$__ProjectDir/temp_home" ]; then
579         mkdir temp_home
580     fi
581     export HOME=$__ProjectDir/temp_home
582     echo "HOME not defined; setting it to $HOME"
583 fi
584
585 # Specify path to be set for CMAKE_INSTALL_PREFIX.
586 # This is where all built CoreClr libraries will copied to.
587 export __CMakeBinDir="$__BinDir"
588
589 if [ ! -d "$__BinDir" ] || [ ! -d "$__BinDir/bin" ]; then
590
591     echo "Has not been found built CoreCLR instance"
592     echo "Please build it before tests using './build.sh $__BuildArch $__BuildType'"
593     exit 1
594 fi
595
596 # Configure environment if we are doing a cross compile.
597 if [ $__CrossBuild == 1 ]; then
598     export CROSSCOMPILE=1
599     if ! [[ -n "$ROOTFS_DIR" ]]; then
600         export ROOTFS_DIR="$__ProjectRoot/cross/rootfs/$__BuildArch"
601     fi
602 fi
603
604 # init the target distro name
605 initTargetDistroRid
606
607 # Override tool directory
608
609 __CoreClrVersion=1.1.0
610 __sharedFxDir=$__BuildToolsDir/dotnetcli/shared/Microsoft.NETCore.App/$__CoreClrVersion/
611
612 echo "Building Tests..."
613
614 build_Tests
615
616 if [ $? -ne 0 ]; then
617     echo "Failed to build tests"
618     exit 1
619 fi
620
621 echo "${__MsgPrefix}Test build successful."
622 echo "${__MsgPrefix}Test binaries are available at ${__TestBinDir}"
623
624 __testNativeBinDir=$__IntermediatesDir/tests
625
626 if [ $__RunTests -ne 0 ]; then
627
628     echo "Run Tests..."
629
630     echo "${__TestDir}/runtest.sh --testRootDir=$__TestBinDir --coreClrBinDir=$__BinDir --coreFxBinDir=$__sharedFxDir --testNativeBinDir=$__testNativeBinDir"
631
632     $__TestDir/runtest.sh --testRootDir=$__TestBinDir --coreClrBinDir=$__BinDir --coreFxBinDir=$CORE_ROOT --testNativeBinDir=$__testNativeBinDir
633
634     echo "Tests run successful."
635 else
636     echo "To run all tests use 'tests/runtests.sh' where:"
637     echo "    testRootDir      = $__TestBinDir"
638     echo "    coreClrBinDir    = $__BinDir"
639     echo "    coreFxBinDir     = $CORE_ROOT"
640     echo "    testNativeBinDir = $__testNativeBinDir"
641     echo " -------------------------------------------------- "
642     echo "To run single test use the following command:"
643     echo "    bash ${__TestBinDir}/__TEST_PATH__/__TEST_NAME__.sh -coreroot=${CORE_ROOT}"
644 fi
645