Cleanup + Small TimeZoneInfo optimization for Unix (dotnet/corert#5314)
[platform/upstream/coreclr.git] / build-test.sh
index 058b9c1..491f618 100755 (executable)
@@ -2,15 +2,51 @@
 
 initHostDistroRid()
 {
+    __HostDistroRid=""
+
+    # Some OS groups should default to use the portable packages
+    if [ "$__BuildOS" == "OSX" ]; then
+        __PortableBuild=1
+    fi
+
     if [ "$__HostOS" == "Linux" ]; then
-        if [ ! -e /etc/os-release ]; then
-            echo "WARNING: Can not determine runtime id for current distro."
-            __HostDistroRid=""
-        else
+        if [ -e /etc/redhat-release ]; then
+            local redhatRelease=$(</etc/redhat-release)
+            if [[ $redhatRelease == "CentOS release 6."* || $redhatRelease == "Red Hat Enterprise Linux Server release 6."* ]]; then
+                __HostDistroRid="rhel.6-$__HostArch"
+            else
+                __PortableBuild=1
+            fi
+        elif [ -e /etc/os-release ]; then
             source /etc/os-release
+            if [[ $ID == "alpine" ]]; then
+                # remove the last version digit
+                VERSION_ID=${VERSION_ID%.*}
+            else
+                __PortableBuild=1
+            fi
+
             __HostDistroRid="$ID.$VERSION_ID-$__HostArch"
+        fi        
+    elif [ "$__HostOS" == "FreeBSD" ]; then
+        __freebsd_version=`sysctl -n kern.osrelease | cut -f1 -d'.'`
+        __HostDistroRid="freebsd.$__freebsd_version-$__HostArch"
+    fi
+
+    # Portable builds target the base RID
+    if [ "$__PortableBuild" == 1 ]; then
+        if [ "$__BuildOS" == "OSX" ]; then
+            export __HostDistroRid="osx-$__BuildArch"
+        elif [ "$__BuildOS" == "Linux" ]; then
+            export __HostDistroRid="linux-$__BuildArch"
         fi
     fi
+
+    if [ "$__HostDistroRid" == "" ]; then
+        echo "WARNING: Cannot determine runtime id for current distro."
+    fi
+
+    echo "Setting __HostDistroRid to $__HostDistroRid"
 }
 
 initTargetDistroRid()
@@ -18,8 +54,13 @@ initTargetDistroRid()
     if [ $__CrossBuild == 1 ]; then
         if [ "$__BuildOS" == "Linux" ]; then
             if [ ! -e $ROOTFS_DIR/etc/os-release ]; then
-                echo "WARNING: Can not determine runtime id for current distro."
-                export __DistroRid=""
+                if [ -e $ROOTFS_DIR/android_platform ]; then
+                    source $ROOTFS_DIR/android_platform
+                    export __DistroRid="$RID"
+                else
+                    echo "WARNING: Cannot determine runtime id for current distro."
+                    export __DistroRid=""
+                fi
             else
                 source $ROOTFS_DIR/etc/os-release
                 export __DistroRid="$ID.$VERSION_ID-$__BuildArch"
@@ -29,64 +70,50 @@ initTargetDistroRid()
         export __DistroRid="$__HostDistroRid"
     fi
 
+    if [ "$ID.$VERSION_ID" == "ubuntu.16.04" ]; then
+     export __DistroRid="ubuntu.14.04-$__BuildArch"
+    fi
+
     # Portable builds target the base RID
     if [ "$__PortableBuild" == 1 ]; then
         if [ "$__BuildOS" == "Linux" ]; then
             export __DistroRid="linux-$__BuildArch"
+            export __RuntimeId="linux-$__BuildArch"
         elif [ "$__BuildOS" == "OSX" ]; then
             export __DistroRid="osx-$__BuildArch"
+            export __RuntimeId="osx-$__BuildArch"
         fi
     fi
+
+    echo "__DistroRid: " $__DistroRid
 }
 
 isMSBuildOnNETCoreSupported()
 {
-    # This needs to be updated alongwith corresponding changes to netci.groovy.
-    __isMSBuildOnNETCoreSupported=0
+    __isMSBuildOnNETCoreSupported=$__msbuildonunsupportedplatform
+
+    if [ $__isMSBuildOnNETCoreSupported == 1 ]; then
+        return
+    fi
 
     if [ "$__HostArch" == "x64" ]; then
         if [ "$__HostOS" == "Linux" ]; then
-            case "$__HostDistroRid" in
-                "centos.7-x64")
-                    __isMSBuildOnNETCoreSupported=1
-                    ;;
-                "debian.8-x64")
-                    __isMSBuildOnNETCoreSupported=1
-                    ;;
-                "fedora.23-x64")
-                    __isMSBuildOnNETCoreSupported=1
-                    ;;
-                "fedora.24-x64")
-                    __isMSBuildOnNETCoreSupported=1
-                    ;;
-                "opensuse.42.1-x64")
-                    __isMSBuildOnNETCoreSupported=1
-                    ;;
-                "rhel.7"*"-x64")
-                    __isMSBuildOnNETCoreSupported=1
-                    ;;
-                "ubuntu.14.04-x64")
-                    __isMSBuildOnNETCoreSupported=1
-                    ;;
-                "ubuntu.16.04-x64")
-                    __isMSBuildOnNETCoreSupported=1
-                    ;;
-                "ubuntu.16.10-x64")
-                    __isMSBuildOnNETCoreSupported=1
-                    ;;
-                "alpine.3.4.3-x64")
-                    __isMSBuildOnNETCoreSupported=1
-                    ;;
-                *)
-                __isMSBuildOnNETCoreSupported=$__msbuildonunsupportedplatform
-            esac
+            __isMSBuildOnNETCoreSupported=1
+            UNSUPPORTED_RIDS=("debian.9-x64" "ubuntu.17.04-x64")
+            for UNSUPPORTED_RID in "${UNSUPPORTED_RIDS[@]}"
+            do
+                if [ "$__HostDistroRid" == "$UNSUPPORTED_RID" ]; then
+                    __isMSBuildOnNETCoreSupported=0
+                    break
+                fi
+            done
         elif [ "$__HostOS" == "OSX" ]; then
             __isMSBuildOnNETCoreSupported=1
         fi
     fi
 }
 
-build_Tests()
+generate_layout()
 {
     __TestDir=$__ProjectDir/tests
     __ProjectFilesDir=$__TestDir
@@ -105,10 +132,10 @@ build_Tests()
         __TestIntermediateDir="tests/obj/${__BuildOS}.${__BuildArch}.${__BuildType}"
     fi
 
-       echo "__BuildOS: ${__BuildOS}"
-       echo "__BuildArch: ${__BuildArch}"
-       echo "__BuildType: ${__BuildType}"
-       echo "__TestIntermediateDir: ${__TestIntermediateDir}"
+    echo "__BuildOS: ${__BuildOS}"
+    echo "__BuildArch: ${__BuildArch}"
+    echo "__BuildType: ${__BuildType}"
+    echo "__TestIntermediateDir: ${__TestIntermediateDir}"
 
     if [ ! -f "$__TestBinDir" ]; then
         echo "Creating TestBinDir: ${__TestBinDir}"
@@ -127,30 +154,33 @@ build_Tests()
     # ===
     # =========================================================================================
 
-    if [ -z "$XunitTestBinBase"]; then
-      XunitTestBinBase="$__TestWorkingDir"
+    build_Tests_internal "Restore_Packages" "${__ProjectDir}/tests/build.proj" " -BatchRestorePackages" "Restore product binaries (build tests)"
+
+    if [ -n "$__UpdateInvalidPackagesArg" ]; then
+        __up=-updateinvalidpackageversion
     fi
 
-    export CORE_ROOT="$XunitTestBinBase/Tests/Core_Root"
+    echo "${__MsgPrefix}Creating test overlay..."
 
-    if [ ! -f "${CORE_ROOT}" ]; then
-        mkdir -p ${CORE_ROOT}
-    else
-        rm -rf ${CORE_ROOT}
+    if [ -z "$xUnitTestBinBase" ]; then
+      xUnitTestBinBase=$__TestWorkingDir
     fi
 
-    cp -r $__BinDir/* $CORE_ROOT/ > /dev/null
+    export CORE_ROOT=$xUnitTestBinBase/Tests/Core_Root
 
-    # make sure the correct ilasm will be used
-    export CscToolPath="${CORE_ROOT}"
+    if [ -d "${CORE_ROOT}" ]; then
+      rm -rf $CORE_ROOT
+    fi
 
-    build_Tests_internal "Restore_Product" "${__ProjectDir}/tests/build.proj" " -BatchRestorePackages" "Restore product binaries (build tests)"
+    mkdir -p $CORE_ROOT
 
-    build_Tests_internal "Tests_GenerateRuntimeLayout" "${__ProjectDir}/tests/runtest.proj" "-BinPlaceRef -BinPlaceProduct -CopyCrossgenToProduct" "Restore product binaries (run tests)"
+    build_Tests_internal "Tests_Overlay_Managed" "${__ProjectDir}/tests/runtest.proj" "-testOverlay" "Creating test overlay"
 
-    if [ -n "$__UpdateInvalidPackagesArg" ]; then
-        __up=-updateinvalidpackageversion
-    fi
+    chmod +x $__BinDir/corerun
+    chmod +x $__BinDir/crossgen
+
+    # Make sure to copy over the pulled down packages
+    cp -r $__BinDir/* $CORE_ROOT/ > /dev/null
 
     # Work hardcoded path around
     if [ ! -f "${__BuildToolsDir}/Microsoft.CSharp.Core.Targets" ]; then
@@ -159,6 +189,54 @@ build_Tests()
     if [ ! -f "${__BuildToolsDir}/Microsoft.CSharp.targets" ]; then
         ln -s "${__BuildToolsDir}/Microsoft.CSharp.Targets" "${__BuildToolsDir}/Microsoft.CSharp.targets"
     fi
+}
+
+build_Tests()
+{
+    __TestDir=$__ProjectDir/tests
+    __ProjectFilesDir=$__TestDir
+    __TestBinDir=$__TestWorkingDir
+
+    if [ $__RebuildTests -ne 0 ]; then
+        if [ -d "${__TestBinDir}" ]; then
+            echo "Removing tests build dir: ${__TestBinDir}"
+            rm -rf $__TestBinDir
+        fi
+    fi
+
+    __CMakeBinDir="${__TestBinDir}"
+
+    if [ -z "$__TestIntermediateDir" ]; then
+        __TestIntermediateDir="tests/obj/${__BuildOS}.${__BuildArch}.${__BuildType}"
+    fi
+
+    echo "__BuildOS: ${__BuildOS}"
+    echo "__BuildArch: ${__BuildArch}"
+    echo "__BuildType: ${__BuildType}"
+    echo "__TestIntermediateDir: ${__TestIntermediateDir}"
+
+    if [ ! -f "$__TestBinDir" ]; then
+        echo "Creating TestBinDir: ${__TestBinDir}"
+        mkdir -p $__TestBinDir
+    fi
+    if [ ! -f "$__LogsDir" ]; then
+        echo "Creating LogsDir: ${__LogsDir}"
+        mkdir -p $__LogsDir
+    fi
+
+    __BuildProperties="-p:OSGroup=${__BuildOS} -p:BuildOS=${__BuildOS} -p:BuildArch=${__BuildArch} -p:BuildType=${__BuildType}"
+
+    # =========================================================================================
+    # ===
+    # === Restore product binaries from packages
+    # ===
+    # =========================================================================================
+
+    build_Tests_internal "Restore_Product" "${__ProjectDir}/tests/build.proj" " -BatchRestorePackages" "Restore product binaries (build tests)"
+
+    if [ -n "$__BuildAgainstPackagesArg" ]; then
+        build_Tests_internal "Tests_GenerateRuntimeLayout" "${__ProjectDir}/tests/runtest.proj" "-BinPlaceRef -BinPlaceProduct -CopyCrossgenToProduct" "Restore product binaries (run tests)"
+    fi
 
     echo "Starting the Managed Tests Build..."
 
@@ -166,7 +244,7 @@ build_Tests()
 
     if [ ! -f $__ManagedTestBuiltMarker ]; then
 
-           build_Tests_internal "Tests_Managed" "$__ProjectDir/tests/build.proj" "$__up" "Managed tests build (build tests)"
+        build_Tests_internal "Tests_Managed" "$__ProjectDir/tests/build.proj" "$__up" "Managed tests build (build tests)"
 
         if [ $? -ne 0 ]; then
             echo "${__MsgPrefix}Error: build failed. Refer to the build log files for details (above)"
@@ -202,9 +280,13 @@ build_Tests()
         fi
     fi
 
+    if [ -n "$__UpdateInvalidPackagesArg" ]; then
+        __up=-updateinvalidpackageversion
+    fi
+
     echo "${__MsgPrefix}Creating test overlay..."
 
-    build_Tests_internal "Tests_Overlay_Managed" "$__ProjectDir/tests/runtest.proj" "-testOverlay" "Creating test overlay"
+    generate_layout
 
     if [ $__ZipTests -ne 0 ]; then
         echo "${__MsgPrefix}ZIP tests packages..."
@@ -214,13 +296,13 @@ build_Tests()
 
 build_Tests_internal()
 {
-       subDirectoryName=$1
-       projectName=$2
-       extraBuildParameters=$3
-       stepName="$4"
+    subDirectoryName=$1
+    projectName=$2
+    extraBuildParameters=$3
+    stepName="$4"
 
-       # Set up directories and file names
-       __BuildLogRootName=$subDirectoryName
+    # Set up directories and file names
+    __BuildLogRootName=$subDirectoryName
     __BuildLog="$__LogsDir/${__BuildLogRootName}.${__BuildOS}.${__BuildArch}.${__BuildType}.log"
     __BuildWrn="$__LogsDir/${__BuildLogRootName}.${__BuildOS}.${__BuildArch}.${__BuildType}.wrn"
     __BuildErr="$__LogsDir/${__BuildLogRootName}.${__BuildOS}.${__BuildArch}.${__BuildType}.err"
@@ -262,8 +344,11 @@ usage()
     echo "crosscomponent - optional argument to build cross-architecture component,"
     echo "               - will use CAC_ROOTFS_DIR environment variable if set."
     echo "portableLinux - build for Portable Linux Distribution"
+    echo "portablebuild - Use portable build."
     echo "verbose - optional argument to enable verbose build output."
     echo "rebuild - if tests have already been built - rebuild them"
+    echo "generatelayoutonly - only pull down dependencies and build coreroot"
+    echo "buildagainstpackages - pull down and build using packages."
     echo "runtests - run tests after building them"
     echo "ziptests - zips CoreCLR tests & Core_Root for a Helix run"
     echo "bindir - output directory (defaults to $__ProjectRoot/bin)"
@@ -376,6 +461,7 @@ __ClangMajorVersion=0
 __ClangMinorVersion=0
 __NuGetPath="$__PackagesDir/NuGet.exe"
 __HostDistroRid=""
+__BuildAgainstPackagesArg=
 __DistroRid=""
 __cmakeargs=""
 __PortableLinux=0
@@ -385,9 +471,9 @@ __NativeTestIntermediatesDir=
 __RunTests=0
 __RebuildTests=0
 __BuildTestWrappers=0
+__GenerateLayoutOnly=
 CORE_ROOT=
 
-
 while :; do
     if [ $# -le 0 ]; then
         break
@@ -440,6 +526,10 @@ while :; do
             __CrossBuild=1
             ;;
 
+        portableBuild)
+            __PortableBuild=1
+            ;;
+
         portablelinux)
             if [ "$__BuildOS" == "Linux" ]; then
                 __PortableLinux=1
@@ -494,6 +584,14 @@ while :; do
             __ZipTests=1
             ;;
 
+        generatelayoutonly)
+            __GenerateLayoutOnly=1
+            ;;
+
+        buildagainstpackages)
+            __BuildAgainstPackagesArg=1
+            ;;
+
         bindir)
             if [ -n "$2" ]; then
                 __RootBinDir="$2"
@@ -527,7 +625,7 @@ __RunArgs="-BuildArch=$__BuildArch -BuildType=$__BuildType -BuildOS=$__BuildOS"
 # Configure environment if we are doing a verbose build
 if [ $__VerboseBuild == 1 ]; then
     export VERBOSE=1
-       __RunArgs="$__RunArgs -verbose"
+    __RunArgs="$__RunArgs -verbose"
 fi
 
 # Set default clang version
@@ -588,8 +686,9 @@ export __CMakeBinDir="$__BinDir"
 
 if [ ! -d "$__BinDir" ] || [ ! -d "$__BinDir/bin" ]; then
 
-    echo "Has not been found built CoreCLR instance"
-    echo "Please build it before tests using './build.sh $__BuildArch $__BuildType'"
+    echo "Cannot find build directory for the CoreCLR Product."
+    echo "Please make sure CoreCLR is built before building tests."
+    echo "Example use: './build.sh $__BuildArch $__BuildType'"
     exit 1
 fi
 
@@ -611,7 +710,11 @@ __sharedFxDir=$__BuildToolsDir/dotnetcli/shared/Microsoft.NETCore.App/$__CoreClr
 
 echo "Building Tests..."
 
-build_Tests
+if [ -z "$__GenerateLayoutOnly" ]; then
+    build_Tests
+else
+    generate_layout
+fi
 
 if [ $? -ne 0 ]; then
     echo "Failed to build tests"
@@ -639,6 +742,10 @@ else
     echo "    coreFxBinDir     = $CORE_ROOT"
     echo "    testNativeBinDir = $__testNativeBinDir"
     echo " -------------------------------------------------- "
+    echo " Example runtest.sh command"
+    echo ""
+    echo " ./tests/runtest.sh --coreOverlayDir=$CORE_ROOT --testNativeBinDir=$__testNativeBinDir --testRootDir=$__TestBinDir --copyNativeTestBin"
+    echo " -------------------------------------------------- "
     echo "To run single test use the following command:"
     echo "    bash ${__TestBinDir}/__TEST_PATH__/__TEST_NAME__.sh -coreroot=${CORE_ROOT}"
 fi