# @brief This is API set for SSAT (Shell Script Automated Tester)
#
-Black='\033[0;30m'
-DarkGray='\033[1;30m'
-Red='\033[0;31m'
-LightRed='\033[1;31m'
-Green='\033[0;32m'
-LightGreen='\033[1;32m'
-Orange='\033[0;33m'
-Yellow='\033[1;33m'
-Blue='\033[0;34m'
-LightBlue='\033[1;34m'
-Purple='\033[0;35m'
-LightPurple='\033[1;35m'
-Cyan='\033[0;36m'
-LightCyan='\033[1;36m'
-LightGray='\033[0;37m'
-White='\033[1;37m'
-NC='\033[0m'
+if [[ "$nocolor" != "1" ]]
+then
+ Black='\033[0;30m'
+ DarkGray='\033[1;30m'
+ Red='\033[0;31m'
+ LightRed='\033[1;31m'
+ Green='\033[0;32m'
+ LightGreen='\033[1;32m'
+ Orange='\033[0;33m'
+ Yellow='\033[1;33m'
+ Blue='\033[0;34m'
+ LightBlue='\033[1;34m'
+ Purple='\033[0;35m'
+ LightPurple='\033[1;35m'
+ Cyan='\033[0;36m'
+ LightCyan='\033[1;36m'
+ LightGray='\033[0;37m'
+ White='\033[1;37m'
+ NC='\033[0m'
+else
+ Black=''
+ DarkGray=''
+ Red=''
+ LightRed=''
+ Green=''
+ LightGreen=''
+ Orange=''
+ Yellow=''
+ Blue=''
+ LightBlue=''
+ Purple=''
+ LightPurple=''
+ Cyan=''
+ LightCyan=''
+ LightGray=''
+ White=''
+ NC=''
+fi
+
+
+ResultLog=""
+
+function writef {
+ ResultLog="$ResultLog$1\n"
+}
##
# @brief Report results of a test group (a "runTest.sh" in a testee directory)
-# @param $1 # total test cases
-# @param $2 # total success cases
-# @param $3 # test log
-# @param $4 # print outs from test (stdout)
#
function report {
- echo NYI
+ if (( ${_fail} == 0 ))
+ then
+ writef "${Green}==================================================${NC}"
+ writef "${LightGreen}[PASSED]${NC} Test Group ${Green}Passed${NC}"
+ else
+ if (( ${_criticalFail} > 0 ))
+ then
+ writef "${Green}==================================================${NC}"
+ writef "${Red}[FAILED]${NC} Test Group has ${Red}failed cases ($_fail)${NC}"
+ else
+ writef "${Green}==================================================${NC}"
+ writef "${LightGreen}[PASSED]${NC} Test Group has ${Red}failed cases ($_fail), but they are not critical.${NC}"
+ fi
+ fi
+
+ writef "${_cases}/${_pass}/${_fail}"
+
+ echo "${ResultLog}" > $_filename
+ printf "$_filename\n"
+
+ if (( ${_criticalFail} > 0 ))
+ then
+ exit 1
+ else
+ exit 0
+ fi
+}
+
+function testInit {
+ _pass=0
+ _fail=0
+ _criticalFail=0
+ _cases=0
+ _filename=$(mktemp)
}
##
# @param $1 1 = success / 0 = fail
# @param $2 test case ID (short string)
# @param $3 test case description
+# @param $4 set 1 if this is not critical (don't care if it's pass or fail)_
function testResult {
- echo NYI
+ _cases=$((_cases+1))
+ if [[ "${1}" == "1" ]]
+ then
+ writef "${LightGreen}[PASSED]${NC} ${Green}$2${NC}:$3${NC}"
+ _pass=$((_pass+1))
+ else
+ _fail=$((_fail+1))
+ if [[ "${4}" == "1" ]]
+ then
+ writef "${Purple}[FAILED][Ignorable] $2${NC}:${Purple}$3${NC}"
+ else
+ writef "${Red}[FAILED][Critical] $2${NC}:${Purple}$3${NC}"
+ _criticalFail=$((_criticalFail+1))
+ fi
+ fi
}
##
# @param $2 Full string of the arguments to $1 (e.g., "-q -n --dryrun")
# @param $3 test case ID
# @param $4 test case description
+# @param $5 set 1 if this is not critical (don't care if it's pass or fail)_
function callTestSuccess {
echo NYI
}
# @param $2 Full string of the arguments to $1 (e.g., "-q -n --dryrun")
# @param $3 test case ID
# @param $4 test case description
+# @param $5 set 1 if this is not critical (don't care if it's pass or fail)_
function callTestFail {
echo NYI
}
# @param $3 test case ID
# @param $4 test case description
# @param $5 Expected exit code.
+# @param $6 set 1 if this is not critical (don't care if it's pass or fail)_
function callTestExitEq {
echo NYI
}
# @param $3 test case ID
# @param $4 test case description
# @param $5 0 if the size is expected to be equal as well. 1 if golden (result 1) might be smaller (will ignore rest of result 2). 2 if the opposite of 1. If $5 > 2, it denotes the max size of compared bytes. (compare the first $5 bytes only)
+# @param $6 set 1 if this is not critical (don't care if it's pass or fail)_
function callCompareTest {
echo NYI
}
# @license Apache-2.0
# @brief This executes test groups and reports aggregated test results.
# @exit 0 if all PASSED. Positive if some FAILED.
+# @dependency sed
#
# If there is no arguments specified, this will search for all "runTest.sh" in
# the subdirectory of this file and regard them as the test groups.
#
+# If a testgroup (runTest.sh) returns 0 while there are failed testcase,
+# it implies that the failed testcases may be ignored and it's good to go.
+#
# If --help or -h is given, this will show detailed description.
TARGET=$(pwd)
BASEPATH=`dirname "$0"`
BASENAME=`basename "$0"`
TESTCASE="runTest.sh"
-source ${BASEPATH}/ssat-api.sh
#
-DOTEST=1
SILENT=1
# Handle arguments
key="$1"
case $key in
-h|--help)
- printf "usage: ${BASENAME} [--help] [<path>] [--testcase <filename>]\n\n"
+ printf "usage: ${BASENAME} [--help] [<path>] [--testcase <filename>] [--nocolor] [--showstdout]\n\n"
printf "These are common ${Red}ssat${NC} commands used:\n\n"
printf "Test all test-groups in the current ($(pwd)) directory, recursively\n"
printf " (no options specified)\n"
printf " $ ${BASENAME} --testcase cases.sh\n"
printf " Search for cases.sh instead of runTest.sh\n"
printf "\n"
+ printf "Do not emit colored text\n"
+ printf " --nocolor or -n\n"
+ printf "\n"
+ printf "Show stdout of test cases\n"
+ printf " --showstdout or -s\n"
+ printf "\n"
printf "Shows this message\n"
printf " --help or -h\n"
printf " $ ${BASENAME} --help \n"
printf "\n\n"
exit 0
;;
+ -n|--nocolor)
+ nocolor=1
+ shift
+ ;;
-t|--testcase)
TESTCASE="$2"
shift
shift
;;
+ -s|--showstdout)
+ SILENT=0
+ shift
+ ;;
*) # Unknown, which is probably target (the path to root-dir of test groups).
TARGET="$1"
esac
done
-if [[ ${#TARGET} -eq 0 ]]
+source ${BASEPATH}/ssat-api.sh
+
+if [[ "${#TARGET}" -eq "0" ]]
then
TARGET="."
fi
-find $TARGET -name $TESTCASE -print0 | while read -d $'\0' file
+TNtc=0
+TNtcpass=0
+TNtcfail=0
+TNgroup=0
+TNgrouppass=0
+TNgroupfail=0
+log=""
+groupLog=""
+
+while read -d $'\0' file
do
CASEBASEPATH=`dirname "$file"`
- pushd $CASEBASEPATH
- source $file
- popd
-done
-
+ CASENAME=`basename "$CASEBASEPATH"`
+ Ntc=0
+ Npass=0
+ Nfail=0
+ tmpfile=$(mktemp)
+ pushd $CASEBASEPATH > /dev/null
+ output=$(. $file)
+ retcode=$?
+ if [[ "${SILENT}" -eq "0" ]]
+ then
+ printf "${output}" | sed '$d'
+ fi
+ popd > /dev/null
+
+ logfile="${output##*$'\n'}"
+
+ resultlog=$(<$logfile)
+ effectiveOutput=`printf "$resultlog" | sed '$d'`
+ log="$log$effectiveOutput\n"
+
+ lastline=`printf "${resultlog}" | sed '$!d'`
+ IFS=/
+ set $lastline
+ Ntc=$1
+ Npass=$2
+ Nfail=$3
+
+ TNtc=$((TNtc+Ntc))
+ TNtcpass=$((TNtcpass+Npass))
+ TNtcfail=$((TNtcfail+Nfail))
+
+ TNgroup=$((TNgroup+1))
+ if [[ "$retcode" -eq "0" ]]
+ then
+ TNgrouppass=$((TNgrouppass+1))
+ groupLog="${groupLog}${LightGreen}[PASSED]${NC} ${Blue}${CASENAME}${NC} ($Npass passed among $Ntc cases)\n"
+ else
+ TNgroupfail=$((TNgroupfail+1))
+ groupLog="${groupLog}${Red}[FAILED]${NC} ${Blue}${CASENAME}${NC} ($Npass passed among $Ntc cases)\n"
+ fi
+
+done < <(find $TARGET -name $TESTCASE -print0)
+
+printf "\n\n==================================================\n\n"
+
+printf "$log\n"
+printf "==================================================\n\n"
+printf "$groupLog"
+printf "==================================================\n"
+
+if (( ${TNgroupfail} == 0 ))
+then
+ printf "${LightGreen}[PASSED] ${Blue}All Test Groups (${TNgroup}) Passed!${NC}\n\n"
+ exit 0
+else
+ printf "${Red}[FAILED] ${Purple}There are failed test groups! (${TNgroupfail})${NC}\n\n"
+ exit 1
+fi
# gather reports & publish them.
projects / platform / upstream / SSAT.git / commitdiff