From: Yongseop Kim Date: Wed, 20 Sep 2017 07:14:10 +0000 (+0900) Subject: Add crossgen features for CoreCLR Unittest X-Git-Url: http://review.tizen.org/git/?a=commitdiff_plain;h=refs%2Fchanges%2F84%2F151184%2F5;p=platform%2Fupstream%2Fcoreclr.git Add crossgen features for CoreCLR Unittest Change-Id: I12794fa0272a53b85c8f1d424cf9485d47abdda4 Signed-off-by: Yongseop Kim --- diff --git a/script/create_ni_files.sh b/script/create_ni_files.sh index 72c35e0..ce11eab 100755 --- a/script/create_ni_files.sh +++ b/script/create_ni_files.sh @@ -2,21 +2,29 @@ usage() { + echo "" echo "Usage: $0 --device=[name] --overlay-dir=###" - echo "--device - If there are more than one target," - echo " you should pass the device name by using 'sdb devices' command" - echo "--overlay-dir - You can specify the path of overlay" - echo "--environment - You can specify the environment variable for performing the unittest" - echo "--options - You can specify options for creating ni files" - exit 1 + echo "" + echo "--device - If there are more than one target," + echo " you should pass the device name by using 'sdb devices' command" + echo "--overlay-dir - You can specify the path of overlay" + echo "--ni-for - You can specify which to create ni for. If you leave this blank or skip, system is default value for this. You can set it to system or clrutc or full" + echo "--utc-dir - You can specify where to create ni from. If --ni-for is set to [clrutc or full], the default value is '/opt/usr/coreclr-tc/Windows_NT.${tcarch}.Release'" + echo "--environment - You can specify the environment variable for performing the unittest" + echo "--crossgen-options - You can specify options for creating ni files" + echo "" } opt="" clrdir="" overlaydir="" +nifor="system" +utcdir="" environment="" options="" +crossgen_log="" + for i in "$@" do if [ $# -le 0 ]; then @@ -37,10 +45,16 @@ do --overlay-dir=*) overlaydir=${i#*=} ;; + --ni-for=*) + nifor=${i#*=} + ;; + --utc-dir=*) + utcdir=${i#*=} + ;; --environment=*) environment=${i#*=} ;; - --options=*) + --crossgen-options=*) options=${i#*=} ;; *) @@ -76,18 +90,67 @@ if [[ ${overlaydir} == "" ]]; then overlaydir="${clrdir}/Windows_NT.${tcarch}.Release/Tests/coreoverlay" fi +if [ "$nifor" != "system" ] && [ "$nifor" != "clrutc" ] && [ "$nifor" != "full" ]; then + echo "'--ni-for' should be one of 'system' or 'clrutc' or 'full'" + exit 1 +fi + +if [ "$nifor" != "system" ] && [ -z $utcdir ]; then + utcdir="${clrdir}/Windows_NT.${tcarch}.Release" +fi + scriptdir="${PWD}/${0%/*}" sdb ${opt} push ${scriptdir}/invoked_create_ni_files.sh $clrdir + cmd="rm ${overlaydir}/*.ni.dll" echo ${cmd} sdb ${opt} shell ${cmd} -cmd="cd ${clrdir} && \ - ./invoked_create_ni_files.sh --overlay-dir=\"${overlaydir}\" --environment=\"${environment}\" \ - --options=\"${options}\" >> crossgen.log 2>&1 | tee -a crossgen.log" -echo ${cmd} -sdb ${opt} shell ${cmd} -sdb ${opt} shell "cd $clrdir && \ +function RunCrossgen() +{ + cmd="cd ${clrdir} && \ + ./invoked_create_ni_files.sh --overlay-dir=\"${overlaydir}\" \ + --ni-for=\"${nifor}\" \ + --utc-dir=\"${utcdir}\" \ + --environment=\"${environment}\" \ + --crossgen-options=\"${options}\" >> $crossgen_log 2>&1 | tee -a $crossgen_log" + echo ${cmd} + sdb ${opt} shell ${cmd} +} + +case "$nifor" in + "system") + crossgen_log="crossgen.log" + rm -f "${crossgen_log}" + RunCrossgen + ;; + "clrutc") + crossgen_log="crossgen_clrutc.log" + rm -f "${crossgen_log}" + RunCrossgen + ;; + "full") + # system + crossgen_log="crossgen.log" + rm -f "${crossgen_log}" + nifor="system" + RunCrossgen + # clrutc + crossgen_log="crossgen_clrutc.log" + rm -f "${crossgen_log}" + nifor="clrutc" + RunCrossgen + # restore + nifor="full" + ;; + *) + echo "can't enter here" + exit 1 +esac + +if [ "$nifor" == "system" ] || [ "$nifor" == "full" ]; then + sdb ${opt} shell "cd $clrdir && \ mkdir -p overlay && cp ${overlaydir}/* overlay \ && tar zcvf overlay_with_ni.tgz overlay \ && rm -rf overlay" +fi \ No newline at end of file diff --git a/script/invoked_create_ni_files.sh b/script/invoked_create_ni_files.sh index 28a47c2..f30a19c 100755 --- a/script/invoked_create_ni_files.sh +++ b/script/invoked_create_ni_files.sh @@ -2,16 +2,151 @@ usage() { - echo "Usage: $0 --overlay-dir=[name] --environment=[environment] --options=[options]" - echo "--overlay-dir - You can specify the overlay directory for creating ni files" - echo "--environment - You can specify the environment variable for creating ni files" - echo "--options - You can specify options for creating ni files" - exit 1 + echo "" + echo "Usage: $0 --overlay-dir=[path] --ni-for=[system or clrutc] --utc-dir=[path] --environment=[environment] --crossgen-options=[options]" + echo "" + echo "--overlay-dir - You can specify the overlay directory for creating ni files" + echo "--ni-for - You can specify which to create ni for" + echo "--utc-dir - You can specify where to create ni from" + echo "--environment - You can specify the environment variable for creating ni files" + echo "--crossgen-options - You can specify options for creating ni files" + echo "" } +# parameters from user overlaydir="" +nifor="system" +utcdir="" environment="" -options="-readytorun" +options="/ReadyToRun" + +# global variables +crossgen_list="" +crossgen_bin="" +opt_args="" +opt_args2="" + +function DoCrossgen() +{ + local DLL_DIR=$(dirname $1) + local DLL_FILE=$(basename $1) + local DLL_EXT="${DLL_FILE##*.}" + local DLL_NAME="${DLL_FILE%.*}" + local STATE="" + local COMMENT="" + + echo "============================================ crossgen start ============================================" + echo "DLL FILE: ${DLL_FILE}" + + $crossgen_bin $options $opt_args $opt_args2 /Out "$DLL_DIR/$DLL_NAME.ni.$DLL_EXT" $1 + + local RESULT=$? + if [[ $RESULT -ne 0 ]]; then + STATE="Fail" + COMMENT="$DLL_DIR" + elif [[ $RESULT -eq 24 ]]; then + STATE="Fail" + COMMENT="$DLL_DIR, The file is not a managed DLL. (Probably a Windows Native PE)" + else + STATE="Pass" + fi + + echo "========================================================================================================" + echo "" + echo -e "${DLL_FILE} \t ${STATE} \t ${COMMENT}" >> $crossgen_list +} + +function DoCrossgenSystem() +{ + echo "============================================ remove ni =================================================" + find ${overlaydir} \( -name "*.ni.*" \) -print | while read ni; do + rm -fv $ni + done + echo "========================================================================================================" + echo "" + + # set /Trusted_Platform_Assemblies + local TRUSTED_DLLS="$overlaydir/System.Private.CoreLib.dll" + while read f + do + TRUSTED_DLLS="$TRUSTED_DLLS:$f" + done < <(find "$overlaydir" \( -name "*.dll" -o -name "*.exe" \) -a -not -path "*.ni.*" -a -not -name "System.Private.CoreLib.dll" -print) + opt_args="/Trusted_Platform_Assemblies $TRUSTED_DLLS" + opt_args2="" + + # System.Private.CoreLib.dll + DoCrossgen "${overlaydir}/System.Private.CoreLib.dll" + mv -vf "${overlaydir}/System.Private.CoreLib.ni.dll" "${overlaydir}/System.Private.CoreLib.dll" + + # other dlls + for dll in `find "${overlaydir}" -iname "*.dll" ! -iname "*.ni.dll"`; do + if [[ $(basename $dll) == "System.Private.CoreLib.dll" ]]; then + continue + fi + DoCrossgen $dll + done +} + +# Get an array of items by reading the specified file line by line. +function read_array { + local theArray=() + + # bash in Mac OS X doesn't support 'readarray', so using alternate way instead. + # readarray -t theArray < "$1" + while IFS='' read -r line || [ -n "$line" ]; do + theArray[${#theArray[@]}]=$(dirname $line) + done < "$1" + echo ${theArray[@]} + +} + +declare -a skipped_tests +function LoadSkippedTests { + # Load the list of tests that are not supported on this platform. These tests are disabled (skipped) permanently. + skipped_tests=($(read_array "$1/testsFailingOutsideWindows.txt")) + skipped_tests+=($(read_array "$1/testsUnsupportedOutsideWindows.txt")) + skipped_tests+=($(read_array "$1/testsUnsupportedOnARM32.txt")) +} + +function IsSkippedTest { + local TEST_PATH=$(dirname $1) + for t in "${skipped_tests[@]}"; do + local FT="$utcdir/$t" + if [ "$FT" == "$TEST_PATH" ]; then + return 0 + fi + done + return 1 +} + +function DoCrossgenClrUtc() +{ + echo "============================================ remove ni =================================================" + find ${utcdir} \( -name "*.ni.*" \) -a -not -path "*/Tests/*" -print | while read ni; do + rm -fv $ni + done + echo "========================================================================================================" + echo "" + + # opt_args + opt_args="$options /Platform_Assemblies_Paths $overlaydir" + opt_args2="" + + # Load skipped tests from text files. We assume these files are in utcdir + LoadSkippedTests $utcdir + + while read dll + do + if IsSkippedTest "$dll"; then + echo "Skipping Crossgen: $dll" + continue + fi + opt_args2="/App_Paths $(dirname $dll)" + DoCrossgen $dll + done < <(find $utcdir \( -name "*.dll" -o -name "*.exe" \) -a -not -path "*.ni.*" -a -not -path "*/Tests/*" -a -not -name "*ldtokenmember.exe" -a -not -path "*readytorun/tests/*" -print) +} + +# entry point for i in "$@" do @@ -27,10 +162,16 @@ do --overlay-dir=*) overlaydir=${i#*=} ;; + --ni-for=*) + nifor=${i#*=} + ;; + --utc-dir=*) + utcdir=${i#*=} + ;; --environment=*) environment=${i#*=} ;; - --options=*) + --crossgen-options=*) options=${i#*=} ;; @@ -42,48 +183,49 @@ do shift done -rm ${overlaydir}/*.ni.dll +# handle parameters + +crossgen_bin="${overlaydir}"/crossgen +if [ ! -f "$crossgen_bin" ]; then + echo "Crossgen($crossgen_bin) doesn't exist" + exit 1 +fi + +if [ "$nifor" == "clrutc" ] && [ ! -d $utcdir ]; then + echo "UtcDir($utcdir) doesn't exist" + exit 1 +fi +# export environment echo "environment: ${environment}" for word in ${environment}; do export ${word} done - echo "overlay-dir: ${overlaydir}" -echo "" > crossgen_list.txt -echo "============================================ crossgen start ============================================" -echo "DLL NAME: System.Private.CoreLib.dll" -"${overlaydir}"/crossgen ${options} -platform_assemblies_paths "${overlaydir}" -out "${overlaydir}/System.Private.CoreLib.ni.dll" "${overlaydir}/System.Private.CoreLib.dll" -if [[ $? -ne 0 ]]; then - STATE="Fail" -else - STATE="Pass" -fi -mv ${overlaydir}/System.Private.CoreLib.ni.dll ${overlaydir}/System.Private.CoreLib.dll -echo "========================================================================================================" -echo "" -echo "System.Private.CoreLib.dll, ${STATE}" >> crossgen_list.txt +echo "ni-for: ${nifor}" +echo "options: ${options}" -for DLL_FILE in `find "${overlaydir}" -iname "*.dll" ! -iname "*.ni.dll"`; do - DLL_NAME=$(basename ${DLL_FILE}) - OUT_NAME=$(echo ${DLL_NAME} | sed "s/.dll$/.ni.dll/g") +echo "" > $crossgen_list - if [[ ${DLL_NAME} == "System.Private.CoreLib.dll" ]]; then - continue - fi - - echo "" - echo "============================================ crossgen start ============================================" - echo "DLL NAME: ${DLL_NAME}" - "${overlaydir}"/crossgen ${options} -platform_assemblies_paths "${overlaydir}" -out "${overlaydir}/${OUT_NAME}" "${DLL_FILE}" - if [[ $? -ne 0 ]]; then - STATE="Fail" - else - STATE="Pass" - fi - echo "========================================================================================================" - echo "" - echo "${DLL_NAME}, ${STATE}" >> crossgen_list.txt -done +# do crossgen for each ni-for +time_start=$(date +"%s") +case "$nifor" in + "system") + crossgen_list="crossgen_list.txt" + rm -f "${crossgen_list}" + DoCrossgenSystem + ;; + "clrutc") + crossgen_list="crossgen_list_clrutc.txt" + rm -f "${crossgen_list}" + DoCrossgenClrUtc + ;; + *) + echo "can't enter here" + exit 1 +esac +time_end=$(date +"%s") +time_diff=$(($time_end-$time_start)) +echo "$(($time_diff / 60)) minutes and $(($time_diff % 60)) seconds taken to crossgen all of tests" \ No newline at end of file