Set vtable offset as contained
[platform/upstream/coreclr.git] / build-test.sh
1 #!/usr/bin/env bash
2
3 __PortableBuild=1
4
5 initTargetDistroRid()
6 {
7     source init-distro-rid.sh
8
9     # Only pass ROOTFS_DIR if cross is specified.
10     if (( ${__CrossBuild} == 1 )); then
11         passedRootfsDir=${ROOTFS_DIR}
12     fi
13
14     initDistroRidGlobal ${__BuildOS} ${__BuildArch} ${__PortableBuild} ${passedRootfsDir}
15 }
16
17 isMSBuildOnNETCoreSupported()
18 {
19     __isMSBuildOnNETCoreSupported=$__msbuildonunsupportedplatform
20
21     if [ $__isMSBuildOnNETCoreSupported == 1 ]; then
22         return
23     fi
24
25     if [ "$__HostArch" == "x64" ]; then
26         if [ "$__HostOS" == "Linux" ]; then
27             __isMSBuildOnNETCoreSupported=1
28             UNSUPPORTED_RIDS=("debian.9-x64" "ubuntu.17.04-x64")
29             for UNSUPPORTED_RID in "${UNSUPPORTED_RIDS[@]}"
30             do
31                 if [ "${__DistroRid}" == "$UNSUPPORTED_RID" ]; then
32                     __isMSBuildOnNETCoreSupported=0
33                     break
34                 fi
35             done
36         elif [ "$__HostOS" == "OSX" ]; then
37             __isMSBuildOnNETCoreSupported=1
38         fi
39     fi
40 }
41
42 build_test_wrappers()
43 {
44     if [ $__BuildTestWrappers -ne -0 ]; then
45         echo "${__MsgPrefix}Creating test wrappers..."
46
47         export __Exclude="${__ProjectDir}/tests/issues.targets"
48         export __BuildLogRootName="Tests_XunitWrapper"
49
50         buildVerbosity="Summary"
51
52         if [ $__VerboseBuild == 1 ]; then
53             buildVerbosity="Diag"
54         fi
55
56         # Set up directories and file names
57         __BuildLogRootName=$subDirectoryName
58         __BuildLog="$__LogsDir/${__BuildLogRootName}.${__BuildOS}.${__BuildArch}.${__BuildType}.log"
59         __BuildWrn="$__LogsDir/${__BuildLogRootName}.${__BuildOS}.${__BuildArch}.${__BuildType}.wrn"
60         __BuildErr="$__LogsDir/${__BuildLogRootName}.${__BuildOS}.${__BuildArch}.${__BuildType}.err"
61         __MsbuildLog="/fileloggerparameters:\"Verbosity=normal;LogFile=${__BuildLog}\""
62         __MsbuildWrn="/fileloggerparameters1:\"WarningsOnly;LogFile=${__BuildWrn}\""
63         __MsbuildErr="/fileloggerparameters2:\"ErrorsOnly;LogFile=${__BuildErr}\""
64         __Logging="$__MsbuildLog $__MsbuildWrn $__MsbuildErr /consoleloggerparameters:$buildVerbosity"
65
66         nextCommand="\"${__DotNetCli}\" msbuild \"${__ProjectDir}/tests/runtest.proj\" /p:RestoreAdditionalProjectSources=https://dotnet.myget.org/F/dotnet-core/ /p:BuildWrappers=true /p:TargetsWindows=false $__Logging /p:__BuildOS=$__BuildOS /p:__BuildType=$__BuildType /p:__BuildArch=$__BuildArch"
67         eval $nextCommand
68
69         if [ $? -ne 0 ]; then
70             echo "${__MsgPrefix}Error: build failed. Refer to the build log files for details (above)"
71             exit 1
72         else
73             echo "XUnit Wrappers have been built."
74             echo { "\"build_os\"": "\"${__BuildOS}\"", "\"build_arch\"": "\"${__BuildArch}\"", "\"build_type\"": "\"${__BuildType}\"" } > "${__TestWorkingDir}/build_info.json"
75
76         fi
77     fi
78 }
79
80 generate_layout()
81 {
82     echo "${__MsgPrefix}Creating test overlay..."
83
84     __TestDir=$__ProjectDir/tests
85     __ProjectFilesDir=$__TestDir
86     __TestBinDir=$__TestWorkingDir
87
88     if [ $__RebuildTests -ne 0 ]; then
89         if [ -d "${__TestBinDir}" ]; then
90             echo "Removing tests build dir: ${__TestBinDir}"
91             rm -rf $__TestBinDir
92         fi
93     fi
94
95     __CMakeBinDir="${__TestBinDir}"
96
97     if [ -z "$__TestIntermediateDir" ]; then
98         __TestIntermediateDir="tests/obj/${__BuildOS}.${__BuildArch}.${__BuildType}"
99     fi
100
101     echo "__BuildOS: ${__BuildOS}"
102     echo "__BuildArch: ${__BuildArch}"
103     echo "__BuildType: ${__BuildType}"
104     echo "__TestIntermediateDir: ${__TestIntermediateDir}"
105
106     if [ ! -f "$__TestBinDir" ]; then
107         echo "Creating TestBinDir: ${__TestBinDir}"
108         mkdir -p $__TestBinDir
109     fi
110     if [ ! -f "$__LogsDir" ]; then
111         echo "Creating LogsDir: ${__LogsDir}"
112         mkdir -p $__LogsDir
113     fi
114     if [ ! -f "$__MsbuildDebugLogsDir" ]; then
115         echo "Creating MsbuildDebugLogsDir: ${__MsbuildDebugLogsDir}"
116         mkdir -p $__MsbuildDebugLogsDir
117     fi
118
119     # Set up the directory for MSBuild debug logs.
120     export MSBUILDDEBUGPATH="${__MsbuildDebugLogsDir}"
121
122     __BuildProperties="-p:OSGroup=${__BuildOS} -p:BuildOS=${__BuildOS} -p:BuildArch=${__BuildArch} -p:BuildType=${__BuildType}"
123
124     # =========================================================================================
125     # ===
126     # === Restore product binaries from packages
127     # ===
128     # =========================================================================================
129
130     build_MSBuild_projects "Restore_Packages" "${__ProjectDir}/tests/build.proj" "Restore product binaries (build tests)" "/t:BatchRestorePackages"
131
132     if [ -n "$__UpdateInvalidPackagesArg" ]; then
133         __up="/t:UpdateInvalidPackageVersions"
134     fi
135
136     echo "${__MsgPrefix}Creating test overlay..."
137
138     if [ -z "$xUnitTestBinBase" ]; then
139         xUnitTestBinBase=$__TestWorkingDir
140     fi
141
142     export CORE_ROOT=$xUnitTestBinBase/Tests/Core_Root
143
144     if [ -d "${CORE_ROOT}" ]; then
145         rm -rf $CORE_ROOT
146     fi
147
148     mkdir -p $CORE_ROOT
149
150     build_MSBuild_projects "Tests_Overlay_Managed" "${__ProjectDir}/tests/runtest.proj" "Creating test overlay" "/t:CreateTestOverlay"
151
152     chmod +x $__BinDir/corerun
153     chmod +x $__CrossgenExe
154
155     # Make sure to copy over the pulled down packages
156     cp -r $__BinDir/* $CORE_ROOT/ > /dev/null
157
158     if [ "$__BuildOS" != "OSX" ]; then
159         nextCommand="\"$__TestDir/setup-stress-dependencies.sh\" --arch=$__BuildArch --outputDir=$CORE_ROOT"
160         echo "Resolve runtime dependences via $nextCommand"
161         eval $nextCommand
162         if [ $? != 0 ]; then
163             echo "${__MsgPrefix}Error: setup-stress-dependencies failed."
164             exit 1
165         fi
166     fi
167
168     # Precompile framework assemblies with crossgen if required
169     if [ $__DoCrossgen -ne 0 ]; then
170         precompile_coreroot_fx
171     fi
172 }
173
174 precompile_coreroot_fx()
175 {
176     echo "${__MsgPrefix}Running crossgen on framework assemblies in CORE_ROOT: '${CORE_ROOT}'"
177
178     # Read the exclusion file for this platform
179     skipCrossGenFiles=($(read_array "$(dirname "$0")/tests/skipCrossGenFiles.${__BuildArch}.txt"))
180
181     local overlayDir=$CORE_ROOT
182
183     filesToPrecompile=$(find -L $overlayDir -iname \*.dll -not -iname \*.ni.dll -not -iname \*-ms-win-\* -not -iname xunit.\* -type f)
184     for fileToPrecompile in ${filesToPrecompile}
185     do
186         local filename=${fileToPrecompile}
187         if is_skip_crossgen_test "$(basename $filename)"; then
188                 continue
189         fi
190         echo Precompiling $filename
191         $__CrossgenExe /Platform_Assemblies_Paths $overlayDir $filename 1> $filename.stdout 2>$filename.stderr
192         local exitCode=$?
193         if [[ $exitCode != 0 ]]; then
194             if grep -q -e '0x80131018' $filename.stderr; then
195                 printf "\n\t$filename is not a managed assembly.\n\n"
196             else
197                 echo Unable to precompile $filename.
198                 cat $filename.stdout
199                 cat $filename.stderr
200                 exit $exitCode
201             fi
202         else
203             rm $filename.{stdout,stderr}
204         fi
205     done
206 }
207
208 declare -a skipCrossGenFiles
209
210 function is_skip_crossgen_test {
211     for skip in "${skipCrossGenFiles[@]}"; do
212         if [ "$1" == "$skip" ]; then
213             return 0
214         fi
215     done
216     return 1
217 }
218
219 # Get an array of items by reading the specified file line by line.
220 function read_array {
221     local theArray=()
222
223     if [ ! -f "$1" ]; then
224         return
225     fi
226
227     # bash in Mac OS X doesn't support 'readarray', so using alternate way instead.
228     # readarray -t theArray < "$1"
229     # Any line that starts with '#' is ignored.
230     while IFS='' read -r line || [ -n "$line" ]; do
231         if [[ $line != "#"* ]]; then
232             theArray[${#theArray[@]}]=$line
233         fi
234     done < "$1"
235     echo ${theArray[@]}
236 }
237
238 generate_testhost()
239 {
240     echo "${__MsgPrefix}Generating test host..."
241
242     export TEST_HOST=$xUnitTestBinBase/testhost
243
244     if [ -d "${TEST_HOST}" ]; then
245         rm -rf $TEST_HOST
246     fi
247
248     mkdir -p $TEST_HOST
249
250     build_MSBuild_projects "Tests_Generate_TestHost" "${__ProjectDir}/tests/runtest.proj" "Creating test host" "/t:CreateTestHost"
251 }
252
253
254 build_Tests()
255 {
256     echo "${__MsgPrefix}Building Tests..."
257
258     __TestDir=$__ProjectDir/tests
259     __ProjectFilesDir=$__TestDir
260     __TestBinDir=$__TestWorkingDir
261
262     if [ -f  "${__TestWorkingDir}/build_info.json" ]; then
263         rm  "${__TestWorkingDir}/build_info.json"
264     fi
265
266     if [ $__RebuildTests -ne 0 ]; then
267         if [ -d "${__TestBinDir}" ]; then
268             echo "Removing tests build dir: ${__TestBinDir}"
269             rm -rf $__TestBinDir
270         fi
271     fi
272
273     export __CMakeBinDir="${__TestBinDir}"
274     if [ ! -d "${__TestIntermediatesDir}" ]; then
275         mkdir -p ${__TestIntermediatesDir}
276     fi
277
278     __NativeTestIntermediatesDir="${__TestIntermediatesDir}/Native"
279     if [  ! -d "${__NativeTestIntermediatesDir}" ]; then
280         mkdir -p ${__NativeTestIntermediatesDir}
281     fi
282
283     __ManagedTestIntermediatesDir="${__TestIntermediatesDir}/Managed"
284     if [ ! -d "${__ManagedTestIntermediatesDir}" ]; then
285         mkdir -p ${__ManagedTestIntermediatesDir}
286     fi
287
288     echo "__BuildOS: ${__BuildOS}"
289     echo "__BuildArch: ${__BuildArch}"
290     echo "__BuildType: ${__BuildType}"
291     echo "__TestIntermediatesDir: ${__TestIntermediatesDir}"
292     echo "__NativeTestIntermediatesDir: ${__NativeTestIntermediatesDir}"
293     echo "__ManagedTestIntermediatesDir: ${__ManagedTestIntermediatesDir}"
294
295     if [ ! -f "$__TestBinDir" ]; then
296         echo "Creating TestBinDir: ${__TestBinDir}"
297         mkdir -p $__TestBinDir
298     fi
299     if [ ! -f "$__LogsDir" ]; then
300         echo "Creating LogsDir: ${__LogsDir}"
301         mkdir -p $__LogsDir
302     fi
303     if [ ! -f "$__MsbuildDebugLogsDir" ]; then
304         echo "Creating MsbuildDebugLogsDir: ${__MsbuildDebugLogsDir}"
305         mkdir -p $__MsbuildDebugLogsDir
306     fi
307
308     # Set up the directory for MSBuild debug logs.
309     export MSBUILDDEBUGPATH="${__MsbuildDebugLogsDir}"
310
311     __BuildProperties="-p:OSGroup=${__BuildOS} -p:BuildOS=${__BuildOS} -p:BuildArch=${__BuildArch} -p:BuildType=${__BuildType}"
312
313     # =========================================================================================
314     # ===
315     # === Restore product binaries from packages
316     # ===
317     # =========================================================================================
318
319     if [ ${__SkipRestorePackages} != 1 ]; then
320         build_MSBuild_projects "Restore_Product" "${__ProjectDir}/tests/build.proj" "Restore product binaries (build tests)" "/t:BatchRestorePackages"
321     fi
322
323     if [ $__SkipNative != 1 ]; then
324         build_native_projects "$__BuildArch" "${__NativeTestIntermediatesDir}"
325
326         if [ $? -ne 0 ]; then
327             echo "${__MsgPrefix}Error: build failed. Refer to the build log files for details (above)"
328             exit 1
329         fi
330     fi
331
332     if [ $__SkipManaged != 1 ]; then
333         echo "Starting the Managed Tests Build..."
334
335         build_MSBuild_projects "Tests_Managed" "$__ProjectDir/tests/build.proj" "Managed tests build (build tests)" "$__up"
336
337         if [ $? -ne 0 ]; then
338             echo "${__MsgPrefix}Error: build failed. Refer to the build log files for details (above)"
339             exit 1
340         else
341             echo "Checking the Managed Tests Build..."
342
343             build_MSBuild_projects "Check_Test_Build" "${__ProjectDir}/tests/runtest.proj" "Check Test Build" "/t:CheckTestBuild"
344
345             if [ $? -ne 0 ]; then
346                 echo "${__MsgPrefix}Error: Check Test Build failed."
347                 exit 1
348             fi
349         fi
350
351         echo "Managed tests build success!"
352     fi
353
354     build_test_wrappers
355
356     if [ -n "$__UpdateInvalidPackagesArg" ]; then
357         __up="/t:UpdateInvalidPackageVersions"
358     fi
359
360     generate_layout
361 }
362
363 build_MSBuild_projects()
364 {
365     subDirectoryName=$1
366     shift
367     projectName=$1
368     shift
369     stepName="$1"
370     shift
371     extraBuildParameters=("$@")
372
373     # Set up directories and file names
374     __BuildLogRootName=$subDirectoryName
375     __BuildLog="$__LogsDir/${__BuildLogRootName}.${__BuildOS}.${__BuildArch}.${__BuildType}.log"
376     __BuildWrn="$__LogsDir/${__BuildLogRootName}.${__BuildOS}.${__BuildArch}.${__BuildType}.wrn"
377     __BuildErr="$__LogsDir/${__BuildLogRootName}.${__BuildOS}.${__BuildArch}.${__BuildType}.err"
378
379     # Use binclashlogger by default if no other logger is specified
380     if [[ "${extraBuildParameters[*]}" == *"/l:"* ]]; then
381         __msbuildEventLogging=
382     else
383         __msbuildEventLogging="/l:BinClashLogger,Tools/Microsoft.DotNet.Build.Tasks.dll\;LogFile=binclash.log"
384     fi
385
386     if [[ "$subDirectoryName" == "Tests_Managed" ]]; then
387         # Execute msbuild managed test build in stages - workaround for excessive data retention in MSBuild ConfigCache
388         # See https://github.com/Microsoft/msbuild/issues/2993
389
390         # __SkipPackageRestore and __SkipTargetingPackBuild used  to control build by tests/src/dirs.proj
391         export __SkipPackageRestore=false
392         export __SkipTargetingPackBuild=false
393         export __NumberOfTestGroups=3
394
395         __AppendToLog=false
396
397         if [ -n "$__priority1" ]; then
398             export __NumberOfTestGroups=10
399         fi
400
401         for (( testGroupToBuild=1 ; testGroupToBuild <= __NumberOfTestGroups; testGroupToBuild = testGroupToBuild + 1 ))
402         do
403             __msbuildLog="\"/flp:Verbosity=normal;LogFile=${__BuildLog};Append=${__AppendToLog}\""
404             __msbuildWrn="\"/flp1:WarningsOnly;LogFile=${__BuildWrn};Append=${__AppendToLog}\""
405             __msbuildErr="\"/flp2:ErrorsOnly;LogFile=${__BuildErr};Append=${__AppendToLog}\""
406
407             export __TestGroupToBuild=$testGroupToBuild
408
409             # Generate build command
410             buildArgs=("/nologo" "/verbosity:minimal" "/clp:Summary")
411             buildArgs+=("/p:RestoreDefaultOptimizationDataPackage=false" "/p:PortableBuild=true")
412             buildArgs+=("/p:UsePartialNGENOptimization=false" "/maxcpucount")
413
414             buildArgs+=("$projectName" "${__msbuildLog}" "${__msbuildWrn}" "${__msbuildErr}")
415             buildArgs+=("$__msbuildEventLogging")
416             buildArgs+=("${extraBuildParameters[@]}")
417             buildArgs+=("${__CommonMSBuildArgs[@]}")
418             buildArgs+=("${__UnprocessedBuildArgs[@]}")
419
420             nextCommand="\"$__ProjectRoot/dotnet.sh\" msbuild ${buildArgs[@]}"
421             echo "Building step '$stepName' testGroupToBuild=$testGroupToBuild via $nextCommand"
422             eval $nextCommand
423
424             # Make sure everything is OK
425             if [ $? -ne 0 ]; then
426                 echo "${__MsgPrefix}Failed to build $stepName. See the build logs:"
427                 echo "    $__BuildLog"
428                 echo "    $__BuildWrn"
429                 echo "    $__BuildErr"
430                 exit 1
431             fi
432             export __SkipPackageRestore=true
433             export __SkipTargetingPackBuild=true
434             __AppendToLog=true
435         done
436     else
437         __msbuildLog="\"/flp:Verbosity=normal;LogFile=${__BuildLog}\""
438         __msbuildWrn="\"/flp1:WarningsOnly;LogFile=${__BuildWrn}\""
439         __msbuildErr="\"/flp2:ErrorsOnly;LogFile=${__BuildErr}\""
440
441         # Generate build command
442         buildArgs=("/nologo" "/verbosity:minimal" "/clp:Summary")
443         buildArgs+=("/p:RestoreDefaultOptimizationDataPackage=false" "/p:PortableBuild=true")
444         buildArgs+=("/p:UsePartialNGENOptimization=false" "/maxcpucount")
445
446         buildArgs+=("$projectName" "${__msbuildLog}" "${__msbuildWrn}" "${__msbuildErr}")
447         buildArgs+=("$__msbuildEventLogging")
448         buildArgs+=("${extraBuildParameters[@]}")
449         buildArgs+=("${__CommonMSBuildArgs[@]}")
450         buildArgs+=("${__UnprocessedBuildArgs[@]}")
451
452         nextCommand="\"$__ProjectRoot/dotnet.sh\" msbuild ${buildArgs[@]}"
453         echo "Building step '$stepName' via $nextCommand"
454         eval $nextCommand
455
456         # Make sure everything is OK
457         if [ $? -ne 0 ]; then
458             echo "${__MsgPrefix}Failed to build $stepName. See the build logs:"
459             echo "    $__BuildLog"
460             echo "    $__BuildWrn"
461             echo "    $__BuildErr"
462             exit 1
463         fi
464     fi
465 }
466
467 build_native_projects()
468 {
469     platformArch="$1"
470     intermediatesForBuild="$2"
471
472     extraCmakeArguments="-DCLR_CMAKE_TARGET_OS=${__BuildOS} -DCLR_CMAKE_HOST_ARCH=${platformArch}"
473     message="native tests assets"
474
475     # All set to commence the build
476     echo "Commencing build of $message for $__BuildOS.$__BuildArch.$__BuildType in $intermediatesForBuild"
477
478     generator=""
479     buildFile="Makefile"
480     buildTool="make"
481     if [ $__UseNinja == 1 ]; then
482         generator="ninja"
483         buildFile="build.ninja"
484         if ! buildTool=$(command -v ninja || command -v ninja-build); then
485            echo "Unable to locate ninja!" 1>&2
486            exit 1
487         fi
488     fi
489
490     if [ $__SkipConfigure == 0 ]; then
491         # if msbuild is not supported, then set __SkipGenerateVersion to 1
492         if [ $__isMSBuildOnNETCoreSupported == 0 ]; then __SkipGenerateVersion=1; fi
493         # Drop version.c file
494         __versionSourceFile="$intermediatesForBuild/version.c"
495         if [ $__SkipGenerateVersion == 0 ]; then
496             pwd
497             $__ProjectRoot/eng/common/msbuild.sh $__ProjectRoot/eng/empty.csproj \
498                                                  /p:NativeVersionFile=$__versionSourceFile \
499                                                  /p:ArcadeBuild=true /t:GenerateNativeVersionFile /restore \
500                                                  $__CommonMSBuildArgs $__UnprocessedBuildArgs
501             if [ $? -ne 0 ]; then
502                 echo "Failed to generate native version file."
503                 exit $?
504             fi
505         else
506             # Generate the dummy version.c, but only if it didn't exist to make sure we don't trigger unnecessary rebuild
507             __versionSourceLine="static char sccsid[] __attribute__((used)) = \"@(#)No version information produced\";"
508             if [ -e $__versionSourceFile ]; then
509                 read existingVersionSourceLine < $__versionSourceFile
510             fi
511             if [ "$__versionSourceLine" != "$existingVersionSourceLine" ]; then
512                 echo $__versionSourceLine > $__versionSourceFile
513             fi
514         fi
515
516         pushd "$intermediatesForBuild"
517         # Regenerate the CMake solution
518         # Force cross dir to point to project root cross dir, in case there is a cross build.
519         scriptDir="$__ProjectRoot/src/pal/tools"
520         if [[ $__GccBuild == 0 ]]; then
521             nextCommand="CONFIG_DIR=\"$__ProjectRoot/cross\" \"$scriptDir/gen-buildsys-clang.sh\" \"$__TestDir\" $__ClangMajorVersion $__ClangMinorVersion $platformArch $scriptDir $__BuildType $__CodeCoverage $generator $extraCmakeArguments $__cmakeargs"
522         else
523             nextCommand="CONFIG_DIR=\"$__ProjectRoot/cross\" \"$scriptDir/gen-buildsys-gcc.sh\" \"$__TestDir\" \"$__GccMajorVersion\" \"$__GccMinorVersion\" $platformArch $scriptDir $__BuildType $__CodeCoverage $generator $extraCmakeArguments $__cmakeargs"
524         fi
525         echo "Invoking $nextCommand"
526         eval $nextCommand
527         popd
528     fi
529
530     if [ ! -f "$intermediatesForBuild/$buildFile" ]; then
531         echo "Failed to generate $message build project!"
532         exit 1
533     fi
534
535     # Build
536     if [ $__ConfigureOnly == 1 ]; then
537         echo "Finish configuration & skipping $message build."
538         return
539     fi
540
541     pushd "$intermediatesForBuild"
542
543     echo "Executing $buildTool install -j $__NumProc"
544
545     $buildTool install -j $__NumProc
546     if [ $? != 0 ]; then
547         echo "Failed to build $message."
548         exit 1
549     fi
550
551     popd
552     echo "Native tests build success!"
553 }
554
555 usage()
556 {
557     echo "Usage: $0 [BuildArch] [BuildType] [verbose] [coverage] [cross] [clangx.y] [ninja] [runtests] [bindir]"
558     echo "BuildArch can be: x64, x86, arm, armel, arm64"
559     echo "BuildType can be: debug, checked, release"
560     echo "coverage - optional argument to enable code coverage build (currently supported only for Linux and OSX)."
561     echo "ninja - target ninja instead of GNU make"
562     echo "clangx.y - optional argument to build using clang version x.y - supported version 3.5 - 6.0"
563     echo "gccx.y - optional argument to build using gcc version x.y."
564     echo "cross - optional argument to signify cross compilation,"
565     echo "      - will use ROOTFS_DIR environment variable if set."
566     echo "portableLinux - build for Portable Linux Distribution"
567     echo "portablebuild - Use portable build."
568     echo "verbose - optional argument to enable verbose build output."
569     echo "rebuild - if tests have already been built - rebuild them"
570     echo "skipnative: skip the native tests build"
571     echo "skipmanaged: skip the managed section of the test build"
572     echo "buildtestwrappersonly - only build the test wrappers"
573     echo "generatelayoutonly - only pull down dependencies and build coreroot"
574     echo "generatetesthostonly - only pull down dependencies and build coreroot and the CoreFX testhost"
575     echo "skiprestorepackages - skip package restore"
576     echo "crossgen - Precompiles the framework managed assemblies in coreroot"
577     echo "runtests - run tests after building them"
578     echo "bindir - output directory (defaults to $__ProjectRoot/bin)"
579     echo "msbuildonunsupportedplatform - build managed binaries even if distro is not officially supported."
580     echo "priority1 - include priority=1 tests in the build"
581     exit 1
582 }
583
584
585 # Obtain the location of the bash script to figure out where the root of the repo is.
586 __ProjectRoot="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
587
588 # Use uname to determine what the CPU is.
589 CPUName=$(uname -p)
590
591 # Some Linux platforms report unknown for platform, but the arch for machine.
592 if [ "$CPUName" == "unknown" ]; then
593     CPUName=$(uname -m)
594 fi
595
596 case $CPUName in
597     i686)
598         echo "Unsupported CPU $CPUName detected, build might not succeed!"
599         __BuildArch=x86
600         __HostArch=x86
601         ;;
602
603     x86_64)
604         __BuildArch=x64
605         __HostArch=x64
606         ;;
607
608     armv7l)
609         echo "Unsupported CPU $CPUName detected, build might not succeed!"
610         __BuildArch=arm
611         __HostArch=arm
612         ;;
613
614     aarch64)
615         __BuildArch=arm64
616         __HostArch=arm64
617         ;;
618
619     *)
620         echo "Unknown CPU $CPUName detected, configuring as if for x64"
621         __BuildArch=x64
622         __HostArch=x64
623         ;;
624 esac
625
626 # Use uname to determine what the OS is.
627 OSName=$(uname -s)
628 case $OSName in
629     Linux)
630         __BuildOS=Linux
631         __HostOS=Linux
632         ;;
633
634     Darwin)
635         __BuildOS=OSX
636         __HostOS=OSX
637         ;;
638
639     FreeBSD)
640         __BuildOS=FreeBSD
641         __HostOS=FreeBSD
642         ;;
643
644     OpenBSD)
645         __BuildOS=OpenBSD
646         __HostOS=OpenBSD
647         ;;
648
649     NetBSD)
650         __BuildOS=NetBSD
651         __HostOS=NetBSD
652         ;;
653
654     SunOS)
655         __BuildOS=SunOS
656         __HostOS=SunOS
657         ;;
658
659     *)
660         echo "Unsupported OS $OSName detected, configuring as if for Linux"
661         __BuildOS=Linux
662         __HostOS=Linux
663         ;;
664 esac
665
666 __BuildType=Debug
667 __CodeCoverage=
668 __IncludeTests=INCLUDE_TESTS
669
670 # Set the various build properties here so that CMake and MSBuild can pick them up
671 export __ProjectDir="$__ProjectRoot"
672 __SourceDir="$__ProjectDir/src"
673 __PackagesDir="$__ProjectDir/.packages"
674 __RootBinDir="$__ProjectDir/bin"
675 __BuildToolsDir="$__ProjectDir/Tools"
676 __DotNetCli="$__ProjectDir/dotnet.sh"
677 __UnprocessedBuildArgs=
678 __CommonMSBuildArgs=
679 __MSBCleanBuildArgs=
680 __UseNinja=0
681 __VerboseBuild=0
682 __SkipRestore=""
683 __SkipNative=0
684 __SkipManaged=0
685 __SkipConfigure=0
686 __SkipGenerateVersion=0
687 __ConfigureOnly=0
688 __CrossBuild=0
689 __ClangMajorVersion=0
690 __ClangMinorVersion=0
691 __GccBuild=0
692 __GccMajorVersion=0
693 __GccMinorVersion=0
694 __NuGetPath="$__PackagesDir/NuGet.exe"
695 __SkipRestorePackages=0
696 __DistroRid=""
697 __cmakeargs=""
698 __PortableLinux=0
699 __msbuildonunsupportedplatform=0
700 __NativeTestIntermediatesDir=
701 __RunTests=0
702 __RebuildTests=0
703 __BuildTestWrappers=1
704 __GenerateLayoutOnly=
705 __GenerateTestHostOnly=
706 __priority1=
707 __BuildTestWrappersOnly=
708 __DoCrossgen=0
709 CORE_ROOT=
710
711 while :; do
712     if [ $# -le 0 ]; then
713         break
714     fi
715
716     lowerI="$(echo $1 | awk '{print tolower($0)}')"
717     case $lowerI in
718         -\?|-h|--help)
719             usage
720             exit 1
721             ;;
722
723         x86)
724             __BuildArch=x86
725             ;;
726
727         x64)
728             __BuildArch=x64
729             ;;
730
731         arm)
732             __BuildArch=arm
733             ;;
734
735         armel)
736             __BuildArch=armel
737             ;;
738
739         arm64)
740             __BuildArch=arm64
741             ;;
742
743         debug)
744             __BuildType=Debug
745             ;;
746
747         checked)
748             __BuildType=Checked
749             ;;
750
751         release)
752             __BuildType=Release
753             ;;
754
755         coverage)
756             __CodeCoverage=Coverage
757             ;;
758
759         cross)
760             __CrossBuild=1
761             ;;
762
763         portablebuild=false)
764             __PortableBuild=0
765             ;;
766
767         portablelinux)
768             if [ "$__BuildOS" == "Linux" ]; then
769                 __PortableLinux=1
770             else
771                 echo "ERROR: portableLinux not supported for non-Linux platforms."
772                 exit 1
773             fi
774             ;;
775
776         verbose)
777             __VerboseBuild=1
778             ;;
779
780         clang3.5|-clang3.5)
781             __ClangMajorVersion=3
782             __ClangMinorVersion=5
783             ;;
784
785         clang3.6|-clang3.6)
786             __ClangMajorVersion=3
787             __ClangMinorVersion=6
788             ;;
789
790         clang3.7|-clang3.7)
791             __ClangMajorVersion=3
792             __ClangMinorVersion=7
793             ;;
794
795         clang3.8|-clang3.8)
796             __ClangMajorVersion=3
797             __ClangMinorVersion=8
798             ;;
799
800         clang3.9|-clang3.9)
801             __ClangMajorVersion=3
802             __ClangMinorVersion=9
803             ;;
804
805         clang4.0|-clang4.0)
806             __ClangMajorVersion=4
807             __ClangMinorVersion=0
808             ;;
809
810         clang5.0|-clang5.0)
811             __ClangMajorVersion=5
812             __ClangMinorVersion=0
813             ;;
814
815         clang6.0|-clang6.0)
816             __ClangMajorVersion=6
817             __ClangMinorVersion=0
818             ;;
819
820         gcc5|-gcc5)
821             __GccMajorVersion=5
822             __GccMinorVersion=
823             __GccBuild=1
824             ;;
825
826         gcc6|-gcc6)
827             __GccMajorVersion=6
828             __GccMinorVersion=
829             __GccBuild=1
830             ;;
831
832         gcc7|-gcc7)
833             __GccMajorVersion=7
834             __GccMinorVersion=
835             __GccBuild=1
836             ;;
837
838         gcc8|-gcc8)
839             __GccMajorVersion=8
840             __GccMinorVersion=
841             __GccBuild=1
842             ;;
843
844         gcc|-gcc)
845             __GccMajorVersion=
846             __GccMinorVersion=
847             __GccBuild=1
848             ;;
849
850         ninja)
851             __UseNinja=1
852             ;;
853
854         runtests)
855             __RunTests=1
856             ;;
857
858         rebuild)
859             __RebuildTests=1
860             ;;
861
862         skipnative|-skipnative)
863             __SkipNative=1
864             ;;
865
866         skipmanaged|-skipmanaged)
867             __SkipManaged=1
868             __BuildTestWrappers=0
869             ;;
870
871         buildtestwrappersonly)
872             __BuildTestWrappersOnly=1
873             ;;
874
875         generatelayoutonly)
876             __GenerateLayoutOnly=1
877             ;;
878
879         generatetesthostonly)
880             __GenerateTestHostOnly=1
881             ;;
882
883         skiprestorepackages)
884             __SkipRestorePackages=1
885             ;;
886
887         crossgen)
888             __DoCrossgen=1
889             ;;
890
891         bindir)
892             if [ -n "$2" ]; then
893                 __RootBinDir="$2"
894                 if [ ! -d $__RootBinDir ]; then
895                     mkdir $__RootBinDir
896                 fi
897                 __RootBinParent=$(dirname $__RootBinDir)
898                 __RootBinName=${__RootBinDir##*/}
899                 __RootBinDir="$(cd $__RootBinParent &>/dev/null && printf %s/%s $PWD $__RootBinName)"
900                 shift
901             else
902                 echo "ERROR: 'bindir' requires a non-empty option argument"
903                 exit 1
904             fi
905             ;;
906
907         msbuildonunsupportedplatform)
908             __msbuildonunsupportedplatform=1
909             ;;
910
911         priority1)
912             __priority1=1
913             __UnprocessedBuildArgs+=("/p:CLRTestPriorityToBuild=1")
914             ;;
915
916         *)
917             __UnprocessedBuildArgs+=("$1")
918             ;;
919     esac
920
921     shift
922 done
923
924 # Get the number of processors available to the scheduler
925 # Other techniques such as `nproc` only get the number of
926 # processors available to a single process.
927 if [ `uname` = "FreeBSD" ]; then
928   __NumProc=`sysctl hw.ncpu | awk '{ print $2+1 }'`
929 elif [ `uname` = "NetBSD" ]; then
930   __NumProc=$(($(getconf NPROCESSORS_ONLN)+1))
931 elif [ `uname` = "Darwin" ]; then
932   __NumProc=$(($(getconf _NPROCESSORS_ONLN)+1))
933 else
934   __NumProc=$(nproc --all)
935 fi
936
937 __CommonMSBuildArgs=("/p:__BuildArch=$__BuildArch" "/p:__BuildType=$__BuildType" "/p:__BuildOS=$__BuildOS" "/nodeReuse:false")
938
939 # Configure environment if we are doing a verbose build
940 if [ $__VerboseBuild == 1 ]; then
941     export VERBOSE=1
942     __CommonMSBuildArgs+=("/v:detailed")
943 fi
944
945 # Set default clang version
946 if [[ $__ClangMajorVersion == 0 && $__ClangMinorVersion == 0 ]]; then
947     if [[ "$__BuildArch" == "arm" || "$__BuildArch" == "armel" ]]; then
948         __ClangMajorVersion=5
949         __ClangMinorVersion=0
950     else
951         __ClangMajorVersion=3
952         __ClangMinorVersion=9
953     fi
954 fi
955
956 # Set dependent variables
957 __LogsDir="$__RootBinDir/Logs"
958 __MsbuildDebugLogsDir="$__LogsDir/MsbuildDebugLogs"
959
960 # Set the remaining variables based upon the determined build configuration
961 __BinDir="$__RootBinDir/Product/$__BuildOS.$__BuildArch.$__BuildType"
962 __PackagesBinDir="$__BinDir/.nuget"
963 __TestDir="$__ProjectDir/tests"
964 __TestWorkingDir="$__RootBinDir/tests/$__BuildOS.$__BuildArch.$__BuildType"
965 __IntermediatesDir="$__RootBinDir/obj/$__BuildOS.$__BuildArch.$__BuildType"
966 __TestIntermediatesDir="$__RootBinDir/tests/obj/$__BuildOS.$__BuildArch.$__BuildType"
967 __isMSBuildOnNETCoreSupported=0
968 __CrossComponentBinDir="$__BinDir"
969 __CrossCompIntermediatesDir="$__IntermediatesDir/crossgen"
970
971 __CrossArch="$__HostArch"
972 if [ $__CrossBuild == 1 ]; then
973     __CrossComponentBinDir="$__CrossComponentBinDir/$__CrossArch"
974 fi
975 __CrossgenCoreLibLog="$__LogsDir/CrossgenCoreLib_$__BuildOS.$BuildArch.$__BuildType.log"
976 __CrossgenExe="$__CrossComponentBinDir/crossgen"
977
978 isMSBuildOnNETCoreSupported
979
980 # CI_SPECIFIC - On CI machines, $HOME may not be set. In such a case, create a subfolder and set the variable to it.
981 # This is needed by CLI to function.
982 if [ -z "$HOME" ]; then
983     if [ ! -d "$__ProjectDir/temp_home" ]; then
984         mkdir temp_home
985     fi
986     export HOME=$__ProjectDir/temp_home
987     echo "HOME not defined; setting it to $HOME"
988 fi
989
990 # Configure environment if we are doing a cross compile.
991 if [ $__CrossBuild == 1 ]; then
992     export CROSSCOMPILE=1
993     if ! [[ -n "$ROOTFS_DIR" ]]; then
994         export ROOTFS_DIR="$__ProjectRoot/cross/rootfs/$__BuildArch"
995     fi
996 fi
997
998 # init the target distro name
999 initTargetDistroRid
1000
1001 if [ $__PortableBuild == 0 ]; then
1002     __CommonMSBuildArgs="$__CommonMSBuildArgs /p:PortableBuild=false"
1003 fi
1004
1005 # Restore Build Tools
1006 source $__ProjectRoot/init-tools.sh
1007
1008 if [[ (-z "$__GenerateLayoutOnly") && (-z "$__GenerateTestHostOnly") && (-z "$__BuildTestWrappersOnly") ]]; then
1009     build_Tests
1010 elif [ ! -z "$__BuildTestWrappersOnly" ]; then
1011     build_test_wrappers
1012 else
1013     generate_layout
1014     if [ ! -z "$__GenerateTestHostOnly" ]; then
1015         generate_testhost
1016     fi
1017 fi
1018
1019 if [ $? -ne 0 ]; then
1020     echo "Failed to build tests"
1021     exit 1
1022 fi
1023
1024 echo "${__MsgPrefix}Test build successful."
1025 echo "${__MsgPrefix}Test binaries are available at ${__TestBinDir}"
1026
1027 __testNativeBinDir=$__IntermediatesDir/tests
1028
1029 if [ $__RunTests -ne 0 ]; then
1030
1031     echo "Run Tests..."
1032
1033     nextCommand="$__TestDir/runtest.sh --testRootDir=$__TestBinDir --coreClrBinDir=$__BinDir --coreFxBinDir=$CORE_ROOT --testNativeBinDir=$__testNativeBinDir"
1034     echo "$nextCommand"
1035     eval $nextCommand
1036
1037     echo "Tests run successful."
1038 else
1039     echo "To run all tests use 'tests/runtests.sh' where:"
1040     echo "    testRootDir      = $__TestBinDir"
1041     echo "    coreClrBinDir    = $__BinDir"
1042     echo "    coreFxBinDir     = $CORE_ROOT"
1043     echo "    testNativeBinDir = $__testNativeBinDir"
1044     echo " -------------------------------------------------- "
1045     echo " Example runtest.sh command"
1046     echo ""
1047     echo " ./tests/runtest.sh --coreOverlayDir=$CORE_ROOT --testNativeBinDir=$__testNativeBinDir --testRootDir=$__TestBinDir --copyNativeTestBin"
1048     echo " -------------------------------------------------- "
1049     echo "To run single test use the following command:"
1050     echo "    bash ${__TestBinDir}/__TEST_PATH__/__TEST_NAME__.sh -coreroot=${CORE_ROOT}"
1051 fi
1052