Merge pull request #10532 from rartemev/fix_tests_build
[platform/upstream/coreclr.git] / build-test.sh
1 #!/usr/bin/env bash
2
3 initHostDistroRid()
4 {
5     if [ "$__HostOS" == "Linux" ]; then
6         if [ ! -e /etc/os-release ]; then
7             echo "WARNING: Can not determine runtime id for current distro."
8             __HostDistroRid=""
9         else
10             source /etc/os-release
11             __HostDistroRid="$ID.$VERSION_ID-$__HostArch"
12         fi
13     fi
14 }
15
16 initTargetDistroRid()
17 {
18     if [ $__CrossBuild == 1 ]; then
19         if [ "$__BuildOS" == "Linux" ]; then
20             if [ ! -e $ROOTFS_DIR/etc/os-release ]; then
21                 echo "WARNING: Can not determine runtime id for current distro."
22                 export __DistroRid=""
23             else
24                 source $ROOTFS_DIR/etc/os-release
25                 export __DistroRid="$ID.$VERSION_ID-$__BuildArch"
26             fi
27         fi
28     else
29         export __DistroRid="$__HostDistroRid"
30     fi
31
32     # Portable builds target the base RID
33     if [ "$__PortableBuild" == 1 ]; then
34         if [ "$__BuildOS" == "Linux" ]; then
35             export __DistroRid="linux-$__BuildArch"
36         elif [ "$__BuildOS" == "OSX" ]; then
37             export __DistroRid="osx-$__BuildArch"
38         fi
39     fi
40 }
41
42 isMSBuildOnNETCoreSupported()
43 {
44     # This needs to be updated alongwith corresponding changes to netci.groovy.
45     __isMSBuildOnNETCoreSupported=0
46
47     if [ "$__HostArch" == "x64" ]; then
48         if [ "$__HostOS" == "Linux" ]; then
49             case "$__HostDistroRid" in
50                 "centos.7-x64")
51                     __isMSBuildOnNETCoreSupported=1
52                     ;;
53                 "debian.8-x64")
54                     __isMSBuildOnNETCoreSupported=1
55                     ;;
56                 "fedora.23-x64")
57                     __isMSBuildOnNETCoreSupported=1
58                     ;;
59                 "fedora.24-x64")
60                     __isMSBuildOnNETCoreSupported=1
61                     ;;
62                 "opensuse.42.1-x64")
63                     __isMSBuildOnNETCoreSupported=1
64                     ;;
65                 "rhel.7"*"-x64")
66                     __isMSBuildOnNETCoreSupported=1
67                     ;;
68                 "ubuntu.14.04-x64")
69                     __isMSBuildOnNETCoreSupported=1
70                     ;;
71                 "ubuntu.16.04-x64")
72                     __isMSBuildOnNETCoreSupported=1
73                     ;;
74                 "ubuntu.16.10-x64")
75                     __isMSBuildOnNETCoreSupported=1
76                     ;;
77                 "alpine.3.4.3-x64")
78                     __isMSBuildOnNETCoreSupported=1
79                     ;;
80                 *)
81                 __isMSBuildOnNETCoreSupported=$__msbuildonunsupportedplatform
82             esac
83         elif [ "$__HostOS" == "OSX" ]; then
84             __isMSBuildOnNETCoreSupported=1
85         fi
86     fi
87 }
88
89 build_Tests()
90 {
91     __TestDir=$__ProjectDir/tests
92     __ProjectFilesDir=$__TestDir
93     __TestBinDir=$__TestWorkingDir
94
95     if [ $__RebuildTests -ne 0 ]; then
96         if [ -d "${__TestBinDir}" ]; then
97             echo "Removing tests build dir: ${__TestBinDir}"
98             rm -rf $__TestBinDir
99         fi
100     fi
101
102     __CMakeBinDir="${__TestBinDir}"
103
104     if [ -z "$__TestIntermediateDir" ]; then
105         __TestIntermediateDir="tests/obj/${__BuildOS}.${__BuildArch}.${__BuildType}"
106     fi
107
108         echo "__BuildOS: ${__BuildOS}"
109         echo "__BuildArch: ${__BuildArch}"
110         echo "__BuildType: ${__BuildType}"
111         echo "__TestIntermediateDir: ${__TestIntermediateDir}"
112
113     if [ ! -f "$__TestBinDir" ]; then
114         echo "Creating TestBinDir: ${__TestBinDir}"
115         mkdir -p $__TestBinDir
116     fi
117     if [ ! -f "$__LogsDir" ]; then
118         echo "Creating LogsDir: ${__LogsDir}"
119         mkdir -p $__LogsDir
120     fi
121
122     __BuildProperties="-p:OSGroup=${__BuildOS} -p:BuildOS=${__BuildOS} -p:BuildArch=${__BuildArch} -p:BuildType=${__BuildType}"
123
124     # =========================================================================================
125     # ===
126     # === Restore product binaries from packages
127     # ===
128     # =========================================================================================
129
130     if [ -z "$XunitTestBinBase"]; then
131       XunitTestBinBase="$__TestWorkingDir"
132     fi
133
134     export CORE_ROOT="$XunitTestBinBase/Tests/Core_Root"
135
136     if [ ! -f "${CORE_ROOT}" ]; then
137         mkdir -p ${CORE_ROOT}
138     else
139         rm -rf ${CORE_ROOT}
140     fi
141
142     cp -r $__BinDir/* $CORE_ROOT/ > /dev/null
143
144     # make sure the correct ilasm will be used
145     export CscToolPath="${CORE_ROOT}"
146
147     build_Tests_internal "Restore_Product" "${__ProjectDir}/tests/build.proj" " -BatchRestorePackages" "Restore product binaries (build tests)"
148
149     build_Tests_internal "Tests_GenerateRuntimeLayout" "${__ProjectDir}/tests/runtest.proj" "-BinPlaceRef -BinPlaceProduct -CopyCrossgenToProduct" "Restore product binaries (run tests)"
150
151     if [ -n "$__UpdateInvalidPackagesArg" ]; then
152         __up=-updateinvalidpackageversion
153     fi
154
155     # Work hardcoded path around
156     if [ ! -f "${__BuildToolsDir}/Microsoft.CSharp.Core.Targets" ]; then
157         ln -s "${__BuildToolsDir}/Microsoft.CSharp.Core.targets" "${__BuildToolsDir}/Microsoft.CSharp.Core.Targets"
158     fi
159     if [ ! -f "${__BuildToolsDir}/Microsoft.CSharp.targets" ]; then
160         ln -s "${__BuildToolsDir}/Microsoft.CSharp.Targets" "${__BuildToolsDir}/Microsoft.CSharp.targets"
161     fi
162
163     echo "Starting the Managed Tests Build..."
164
165     __ManagedTestBuiltMarker=${__TestBinDir}/managed_test_build
166
167     if [ ! -f $__ManagedTestBuiltMarker ]; then
168
169             build_Tests_internal "Tests_Managed" "$__ProjectDir/tests/build.proj" "$__up" "Managed tests build (build tests)"
170
171         if [ $? -ne 0 ]; then
172             echo "${__MsgPrefix}Error: build failed. Refer to the build log files for details (above)"
173             exit 1
174         else
175             echo "Tests have been built."
176             echo "Create marker \"${__ManagedTestBuiltMarker}\""
177             touch $__ManagedTestBuiltMarker
178         fi
179     else
180         echo "Managed Tests had been built before."
181     fi
182
183     if [ $__BuildTestWrappers -ne -0 ]; then
184         echo "${__MsgPrefix}Creating test wrappers..."
185
186         __XUnitWrapperBuiltMarker=${__TestBinDir}/xunit_wrapper_build
187
188         if [ ! -f $__XUnitWrapperBuiltMarker ]; then
189
190             build_Tests_internal "Tests_XunitWrapper" "$__ProjectDir/tests/runtest.proj" "-BuildWrappers -MsBuildEventLogging=\" \" " "Test Xunit Wrapper"
191
192             if [ $? -ne 0 ]; then
193                 echo "${__MsgPrefix}Error: build failed. Refer to the build log files for details (above)"
194                 exit 1
195             else
196                 echo "XUnit Wrappers have been built."
197                 echo "Create marker \"${__XUnitWrapperBuiltMarker}\""
198                 touch $__XUnitWrapperBuiltMarker
199             fi
200         else
201             echo "XUnit Wrappers had been built before."
202         fi
203     fi
204
205     echo "${__MsgPrefix}Creating test overlay..."
206
207     build_Tests_internal "Tests_Overlay_Managed" "$__ProjectDir/tests/runtest.proj" "-testOverlay" "Creating test overlay"
208
209     if [ $__ZipTests -ne 0 ]; then
210         echo "${__MsgPrefix}ZIP tests packages..."
211         build_Tests_internal "Helix_Prep" "$__ProjectDir/tests/helixprep.proj" " " "Prep test binaries for Helix publishing"
212     fi
213 }
214
215 build_Tests_internal()
216 {
217         subDirectoryName=$1
218         projectName=$2
219         extraBuildParameters=$3
220         stepName="$4"
221
222         # Set up directories and file names
223         __BuildLogRootName=$subDirectoryName
224     __BuildLog="$__LogsDir/${__BuildLogRootName}.${__BuildOS}.${__BuildArch}.${__BuildType}.log"
225     __BuildWrn="$__LogsDir/${__BuildLogRootName}.${__BuildOS}.${__BuildArch}.${__BuildType}.wrn"
226     __BuildErr="$__LogsDir/${__BuildLogRootName}.${__BuildOS}.${__BuildArch}.${__BuildType}.err"
227     __msbuildLog="\"/flp:Verbosity=normal;LogFile=${__BuildLog}\""
228     __msbuildWrn="\"/flp1:WarningsOnly;LogFile=${__BuildWrn}\""
229     __msbuildErr="\"/flp2:ErrorsOnly;LogFile=${__BuildErr}\""
230
231     # Generate build command
232     buildCommand="$__ProjectRoot/run.sh build -Project=$projectName -MsBuildLog=${__msbuildLog} -MsBuildWrn=${__msbuildWrn} -MsBuildErr=${__msbuildErr} $extraBuildParameters $__RunArgs $__UnprocessedBuildArgs"
233
234     echo "Building step '$stepName' via $buildCommand"
235
236     # Invoke MSBuild
237     eval $buildCommand
238
239     # Invoke MSBuild
240     # $__ProjectRoot/run.sh build -Project=$projectName -MsBuildLog="$__msbuildLog" -MsBuildWrn="$__msbuildWrn" -MsBuildErr="$__msbuildErr" $extraBuildParameters $__RunArgs $__UnprocessedBuildArgs
241
242     # Make sure everything is OK
243     if [ $? -ne 0 ]; then
244         echo "${__MsgPrefix}Failed to build $stepName. See the build logs:"
245         echo "    $__BuildLog"
246         echo "    $__BuildWrn"
247         echo "    $__BuildErr"
248         exit 1
249     fi
250 }
251
252 usage()
253 {
254     echo "Usage: $0 [BuildArch] [BuildType] [verbose] [coverage] [cross] [clangx.y] [ninja] [runtests] [bindir]"
255     echo "BuildArch can be: x64, x86, arm, armel, arm64"
256     echo "BuildType can be: debug, checked, release"
257     echo "coverage - optional argument to enable code coverage build (currently supported only for Linux and OSX)."
258     echo "ninja - target ninja instead of GNU make"
259     echo "clangx.y - optional argument to build using clang version x.y."
260     echo "cross - optional argument to signify cross compilation,"
261     echo "      - will use ROOTFS_DIR environment variable if set."
262     echo "crosscomponent - optional argument to build cross-architecture component,"
263     echo "               - will use CAC_ROOTFS_DIR environment variable if set."
264     echo "portableLinux - build for Portable Linux Distribution"
265     echo "verbose - optional argument to enable verbose build output."
266     echo "rebuild - if tests have already been built - rebuild them"
267     echo "runtests - run tests after building them"
268     echo "ziptests - zips CoreCLR tests & Core_Root for a Helix run"
269     echo "bindir - output directory (defaults to $__ProjectRoot/bin)"
270     echo "msbuildonunsupportedplatform - build managed binaries even if distro is not officially supported."
271     exit 1
272 }
273
274
275 # Obtain the location of the bash script to figure out where the root of the repo is.
276 __ProjectRoot="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
277
278 # $__ProjectRoot/build.sh $1 $2
279
280 # Use uname to determine what the CPU is.
281 CPUName=$(uname -p)
282
283 # Some Linux platforms report unknown for platform, but the arch for machine.
284 if [ "$CPUName" == "unknown" ]; then
285     CPUName=$(uname -m)
286 fi
287
288 case $CPUName in
289     i686)
290         echo "Unsupported CPU $CPUName detected, build might not succeed!"
291         __BuildArch=x86
292         __HostArch=x86
293         ;;
294
295     x86_64)
296         __BuildArch=x64
297         __HostArch=x64
298         ;;
299
300     armv7l)
301         echo "Unsupported CPU $CPUName detected, build might not succeed!"
302         __BuildArch=arm
303         __HostArch=arm
304         ;;
305
306     aarch64)
307         __BuildArch=arm64
308         __HostArch=arm64
309         ;;
310
311     *)
312         echo "Unknown CPU $CPUName detected, configuring as if for x64"
313         __BuildArch=x64
314         __HostArch=x64
315         ;;
316 esac
317
318 # Use uname to determine what the OS is.
319 OSName=$(uname -s)
320 case $OSName in
321     Linux)
322         __BuildOS=Linux
323         __HostOS=Linux
324         ;;
325
326     Darwin)
327         __BuildOS=OSX
328         __HostOS=OSX
329         ;;
330
331     FreeBSD)
332         __BuildOS=FreeBSD
333         __HostOS=FreeBSD
334         ;;
335
336     OpenBSD)
337         __BuildOS=OpenBSD
338         __HostOS=OpenBSD
339         ;;
340
341     NetBSD)
342         __BuildOS=NetBSD
343         __HostOS=NetBSD
344         ;;
345
346     SunOS)
347         __BuildOS=SunOS
348         __HostOS=SunOS
349         ;;
350
351     *)
352         echo "Unsupported OS $OSName detected, configuring as if for Linux"
353         __BuildOS=Linux
354         __HostOS=Linux
355         ;;
356 esac
357
358 __BuildType=Debug
359 __CodeCoverage=
360 __IncludeTests=INCLUDE_TESTS
361
362 # Set the various build properties here so that CMake and MSBuild can pick them up
363 export __ProjectDir="$__ProjectRoot"
364 __SourceDir="$__ProjectDir/src"
365 __PackagesDir="$__ProjectDir/packages"
366 __RootBinDir="$__ProjectDir/bin"
367 __BuildToolsDir="$__ProjectDir/Tools"
368 __UnprocessedBuildArgs=
369 __RunArgs=
370 __MSBCleanBuildArgs=
371 __UseNinja=0
372 __VerboseBuild=0
373 __SkipRestore=""
374 __CrossBuild=0
375 __ClangMajorVersion=0
376 __ClangMinorVersion=0
377 __NuGetPath="$__PackagesDir/NuGet.exe"
378 __HostDistroRid=""
379 __DistroRid=""
380 __cmakeargs=""
381 __PortableLinux=0
382 __msbuildonunsupportedplatform=0
383 __ZipTests=0
384 __NativeTestIntermediatesDir=
385 __RunTests=0
386 __RebuildTests=0
387 __BuildTestWrappers=0
388 CORE_ROOT=
389
390
391 while :; do
392     if [ $# -le 0 ]; then
393         break
394     fi
395
396     lowerI="$(echo $1 | awk '{print tolower($0)}')"
397     case $lowerI in
398         -\?|-h|--help)
399             usage
400             exit 1
401             ;;
402
403         x86)
404             __BuildArch=x86
405             ;;
406
407         x64)
408             __BuildArch=x64
409             ;;
410
411         arm)
412             __BuildArch=arm
413             ;;
414
415         armel)
416             __BuildArch=armel
417             ;;
418
419         arm64)
420             __BuildArch=arm64
421             ;;
422
423         debug)
424             __BuildType=Debug
425             ;;
426
427         checked)
428             __BuildType=Checked
429             ;;
430
431         release)
432             __BuildType=Release
433             ;;
434
435         coverage)
436             __CodeCoverage=Coverage
437             ;;
438
439         cross)
440             __CrossBuild=1
441             ;;
442
443         portablelinux)
444             if [ "$__BuildOS" == "Linux" ]; then
445                 __PortableLinux=1
446             else
447                 echo "ERROR: portableLinux not supported for non-Linux platforms."
448                 exit 1
449             fi
450             ;;
451
452         verbose)
453         __VerboseBuild=1
454         ;;
455
456         clang3.5)
457             __ClangMajorVersion=3
458             __ClangMinorVersion=5
459             ;;
460
461         clang3.6)
462             __ClangMajorVersion=3
463             __ClangMinorVersion=6
464             ;;
465
466         clang3.7)
467             __ClangMajorVersion=3
468             __ClangMinorVersion=7
469             ;;
470
471         clang3.8)
472             __ClangMajorVersion=3
473             __ClangMinorVersion=8
474             ;;
475
476         clang3.9)
477             __ClangMajorVersion=3
478             __ClangMinorVersion=9
479             ;;
480
481         ninja)
482             __UseNinja=1
483             ;;
484
485         runtests)
486             __RunTests=1
487             ;;
488
489         rebuild)
490             __RebuildTests=1
491             ;;
492
493         ziptests)
494             __ZipTests=1
495             ;;
496
497         bindir)
498             if [ -n "$2" ]; then
499                 __RootBinDir="$2"
500                 if [ ! -d $__RootBinDir ]; then
501                     mkdir $__RootBinDir
502                 fi
503                 __RootBinParent=$(dirname $__RootBinDir)
504                 __RootBinName=${__RootBinDir##*/}
505                 __RootBinDir="$(cd $__RootBinParent &>/dev/null && printf %s/%s $PWD $__RootBinName)"
506                 shift
507             else
508                 echo "ERROR: 'bindir' requires a non-empty option argument"
509                 exit 1
510             fi
511             ;;
512
513         msbuildonunsupportedplatform)
514             __msbuildonunsupportedplatform=1
515             ;;
516         *)
517             __UnprocessedBuildArgs="$__UnprocessedBuildArgs $1"
518             ;;
519     esac
520
521     shift
522 done
523
524
525 __RunArgs="-BuildArch=$__BuildArch -BuildType=$__BuildType -BuildOS=$__BuildOS"
526
527 # Configure environment if we are doing a verbose build
528 if [ $__VerboseBuild == 1 ]; then
529     export VERBOSE=1
530         __RunArgs="$__RunArgs -verbose"
531 fi
532
533 # Set default clang version
534 if [[ $__ClangMajorVersion == 0 && $__ClangMinorVersion == 0 ]]; then
535     if [ $__CrossBuild == 1 ]; then
536         __ClangMajorVersion=3
537         __ClangMinorVersion=6
538     else
539         __ClangMajorVersion=3
540         __ClangMinorVersion=5
541     fi
542 fi
543
544
545 # Set dependent variables
546 __LogsDir="$__RootBinDir/Logs"
547
548 # init the host distro name
549 initHostDistroRid
550
551 # Set the remaining variables based upon the determined build configuration
552 __BinDir="$__RootBinDir/Product/$__BuildOS.$__BuildArch.$__BuildType"
553 __PackagesBinDir="$__BinDir/.nuget"
554 __ToolsDir="$__RootBinDir/tools"
555 __TestDir="$__ProjectDir/tests"
556 __TestWorkingDir="$__RootBinDir/tests/$__BuildOS.$__BuildArch.$__BuildType"
557 __IntermediatesDir="$__RootBinDir/obj/$__BuildOS.$__BuildArch.$__BuildType"
558 __TestIntermediatesDir="$__RootBinDir/tests/obj/$__BuildOS.$__BuildArch.$__BuildType"
559 __isMSBuildOnNETCoreSupported=0
560 __CrossComponentBinDir="$__BinDir"
561 __CrossCompIntermediatesDir="$__IntermediatesDir/crossgen"
562
563 __CrossArch="$__HostArch"
564 if [[ "$__HostArch" == "x64" && "$__BuildArch" == "arm" ]]; then
565     __CrossArch="x86"
566 fi
567 if [ $__CrossBuild == 1 ]; then
568     __CrossComponentBinDir="$__CrossComponentBinDir/$__CrossArch"
569 fi
570 __CrossgenCoreLibLog="$__LogsDir/CrossgenCoreLib_$__BuildOS.$BuildArch.$__BuildType.log"
571 __CrossgenExe="$__CrossComponentBinDir/crossgen"
572
573 isMSBuildOnNETCoreSupported
574
575 # CI_SPECIFIC - On CI machines, $HOME may not be set. In such a case, create a subfolder and set the variable to set.
576 # This is needed by CLI to function.
577 if [ -z "$HOME" ]; then
578     if [ ! -d "$__ProjectDir/temp_home" ]; then
579         mkdir temp_home
580     fi
581     export HOME=$__ProjectDir/temp_home
582     echo "HOME not defined; setting it to $HOME"
583 fi
584
585 # Specify path to be set for CMAKE_INSTALL_PREFIX.
586 # This is where all built CoreClr libraries will copied to.
587 export __CMakeBinDir="$__BinDir"
588
589 if [ ! -d "$__BinDir" ] || [ ! -d "$__BinDir/bin" ]; then
590
591     echo "Has not been found built CoreCLR instance"
592     echo "Please build it before tests using './build.sh $__BuildArch $__BuildType'"
593     exit 1
594 fi
595
596 # Configure environment if we are doing a cross compile.
597 if [ $__CrossBuild == 1 ]; then
598     export CROSSCOMPILE=1
599     if ! [[ -n "$ROOTFS_DIR" ]]; then
600         export ROOTFS_DIR="$__ProjectRoot/cross/rootfs/$__BuildArch"
601     fi
602 fi
603
604 # init the target distro name
605 initTargetDistroRid
606
607 # Override tool directory
608
609 __CoreClrVersion=1.1.0
610 __sharedFxDir=$__BuildToolsDir/dotnetcli/shared/Microsoft.NETCore.App/$__CoreClrVersion/
611
612 echo "Building Tests..."
613
614 build_Tests
615
616 if [ $? -ne 0 ]; then
617     echo "Failed to build tests"
618     exit 1
619 fi
620
621 echo "${__MsgPrefix}Test build successful."
622 echo "${__MsgPrefix}Test binaries are available at ${__TestBinDir}"
623
624 __testNativeBinDir=$__IntermediatesDir/tests
625
626 if [ $__RunTests -ne 0 ]; then
627
628     echo "Run Tests..."
629
630     echo "${__TestDir}/runtest.sh --testRootDir=$__TestBinDir --coreClrBinDir=$__BinDir --coreFxBinDir=$__sharedFxDir --testNativeBinDir=$__testNativeBinDir"
631
632     $__TestDir/runtest.sh --testRootDir=$__TestBinDir --coreClrBinDir=$__BinDir --coreFxBinDir=$CORE_ROOT --testNativeBinDir=$__testNativeBinDir
633
634     echo "Tests run successful."
635 else
636     echo "To run all tests use 'tests/runtests.sh' where:"
637     echo "    testRootDir      = $__TestBinDir"
638     echo "    coreClrBinDir    = $__BinDir"
639     echo "    coreFxBinDir     = $CORE_ROOT"
640     echo "    testNativeBinDir = $__testNativeBinDir"
641     echo " -------------------------------------------------- "
642     echo "To run single test use the following command:"
643     echo "    bash ${__TestBinDir}/__TEST_PATH__/__TEST_NAME__.sh -coreroot=${CORE_ROOT}"
644 fi
645