ssat.sh

   1 #!/usr/bin/env bash
   2 ##
   3 ## @file ssat.sh
   4 ## @author MyungJoo Ham <myungjoo.ham@gmail.com>
   5 ## @date Jun 22 2018
   6 ## @brief This executes test groups and reports aggregated test results.
   7 ## @return 0 if all PASSED. Positive if some FAILED.
   8 ## @todo Separate GStreamer related functions as plugins
   9 ##
  10 ## This uses sed, date, cmp
  11 ##
  12 ## If there is no arguments specified, this will search for all "runTest.sh" in
  13 ## the subdirectory of this file and regard them as the test groups.
  14 ##
  15 ## If a testgroup (runTest.sh) returns 0 while there are failed testcase,
  16 ## it implies that the failed testcases may be ignored and it's good to go.
  17 ##
  18 ## If --help or -h is given, this will show detailed description.
  19
  20 ##
  21 ## @mainpage SSAT
  22 ## @section intro        Introduction
  23 ## - Introduction     :  Shell Script Automated Tester
  24 ## @section Program      Program Name
  25 ## - Program Name     :  ssat
  26 ## - Program Details  :  SSAT is a software testing framework for test cases written in BASH shell scripts.
  27 ##   It can search for test scripts recursively from a given path and summarize the test results.
  28 ##   If there is any "critical" fail, ssat will return non-zero values on its exit.
  29 ## @section INOUTPUT     Input/output data
  30 ## - INPUT            :  Test Cases (If not supplied, the current path is the root of test cases)
  31 ## - OUTPUT           :  Summary of test results to stdout. Exit code of 0 if success, non-zero if not success.
  32 ## @section CREATEINFO   Code information
  33 ## - Initial date     :  2018/06/22
  34 ## - Version          :  1.2.0
  35
  36 TARGET=$(pwd)
  37 BASEPATH=`dirname "$0"`
  38 BASENAME=`basename "$0"`
  39 TESTCASE="runTest.sh"
  40 SUMMARYFILENAME=""
  41
  42 #
  43 SILENT=1
  44 PROGRESSLOGLEVEL=0
  45 COUNTNEGATIVE=0
  46 COUNTNEGATIVEPOSTFIX=""
  47 VALGRIND=0
  48 date=`date +"%b %d %Y"`
  49
  50 ## @fn createTemplate()
  51 ## @brief Generate runTest template file
  52 ##
  53 ## Note that the generated template has no license.
  54 ## The SSAT user may put their own license for the generated files.
  55 ## I hereby grant the right to relicense the generated files.
  56 function createTemplate() {
  57         if [[ -f "runTest.sh" ]]
  58         then
  59                 printf "Cannot create runTest.sh here. The file already exists at $(pwd).\n\n"
  60                 exit 1
  61         fi
  62
  63         echo -e "#!/usr/bin/env bash\n\
  64 ##\n\
  65 ## @file runTest.sh\n\
  66 ## @author MyungJoo Ham <myungjoo.ham@gmail.com>\n\
  67 ## @date ${date}\n\
  68 ## @brief This is a template file for SSAT test cases. You may designate your own license.\n\
  69 #\n\
  70 if [[ \"\$SSATAPILOADED\" != \"1\" ]]\n\
  71 then\n\
  72         SILENT=0\n\
  73         INDEPENDENT=1\n\
  74         search=\"ssat-api.sh\"\n\
  75         source \$search\n\
  76         printf \"\${Blue}Independent Mode\${NC}\\n\"\n\
  77 fi\n\
  78 testInit \$1 # You may replace this with Test Group Name\n\
  79 \n\
  80 #testResult 1 T1 \"Dummy Test1\"\n\
  81 #callTestSuccess gst-launch-1.0 \"-q videotestsrc ! videoconvert ! autovideosink\" T2 \"This may run indefinitely\"\n\
  82 #callCompareTest golden.log executeResult.log T3 \"The two files must be same\" 0\n\
  83 \n\
  84 report\n" > runTest.sh
  85 chmod a+x runTest.sh
  86
  87         exit 0
  88 }
  89
  90 # Handle arguments
  91 POSITIONAL=()
  92 while [[ $# -gt 0 ]]
  93 do
  94         key="$1"
  95         case $key in
  96         -h|--help)
  97                 printf "usage: ${BASENAME} [--help] [<path>] [--testcase <filename>] [--nocolor] [--showstdout] [--createtemplate] [--countnegative <postfix>] \n\n"
  98                 printf "These are common ${Red}ssat${NC} commands used:\n\n"
  99                 printf "Test all test-groups in the current ($(pwd)) directory, recursively\n"
 100                 printf "    (no options specified)\n"
 101                 printf "    $ ${BASENAME}\n"
 102                 printf "\n"
 103                 printf "Test all test-groups in the specified directory, recursively\n"
 104                 printf "    <path>\n"
 105                 printf "    $ ${BASENAME} /home/username/test\n"
 106                 printf "    If there are multiple paths, the last one will be used\n"
 107                 printf "\n"
 108                 printf "Search for \"filename\" as the testcase scripts\n"
 109                 printf "    --testcase or -t\n"
 110                 printf "    $ ${BASENAME} --testcase cases.sh\n"
 111                 printf "    Search for cases.sh instead of runTest.sh\n"
 112                 printf "\n"
 113                 printf "Do not emit colored text\n"
 114                 printf "    --nocolor or -n\n"
 115                 printf "\n"
 116                 printf "Show stdout of test cases\n"
 117                 printf "    --showstdout or -s\n"
 118                 printf "\n"
 119                 printf "Create a template 'runTest.sh' test group at your current directory\n"
 120                 printf "    --createtemplate or -c\n"
 121                 printf "\n"
 122                 printf "Show progress during execution\n"
 123                 printf "    --progress or -p or -p=(0,1,9)\n"
 124                 printf "        '0' : Do not print logs in progress. If -p is not given, -p=0 is assumed.\n"
 125                 printf "        '1' : Print test group names only in progress.\n"
 126                 printf "        '2-9' : Print all logs in progress. If -p is given without numbers, -p=9 is used.\n"
 127                 printf "     $ ${BASENAME} -p=1\n"
 128                 printf "     $ ${BASENAME} --progress=9 (equal to --progress) \n"
 129                 printf "\n"
 130                 printf "Enable valgrind to perform memcheck\n"
 131                 printf "    --enable-valgrind or -vg\n"
 132                 printf "\n"
 133                 printf "Shows this message\n"
 134                 printf "    --help or -h\n"
 135                 printf "    $ ${BASENAME} --help \n"
 136                 printf "\n"
 137                 printf "Count negative test cases with the given postfix\n"
 138                 printf "    --countnegative or -cn\n"
 139                 printf "    $ ${BASENAME} --countnegative _n\n"
 140                 printf "    $ ${BASENAME} -cn _n\n"
 141                 printf "\n"
 142                 printf "Write result summary as a file\n"
 143                 printf "    --summary <filename>\n"
 144                 printf "\n\n"
 145                 exit 0
 146         ;;
 147         -n|--nocolor)
 148         nocolor=1
 149         shift
 150         ;;
 151         -t|--testcase)
 152         TESTCASE="$2"
 153         shift
 154         shift
 155         ;;
 156         -cn|--countnegative)
 157         COUNTNEGATIVE=1
 158         COUNTNEGATIVEPOSTFIX="$2"
 159         if [[ "${COUNTNEGATIVEPOSTFIX}" == "" ]]
 160         then
 161                 printf "${BASENAME} -cn or --countnegative requires postfix.\n\n"
 162                 exit -2
 163         fi
 164         shift
 165         shift
 166         ;;
 167         -s|--showstdout)
 168         SILENT=0
 169         shift
 170         ;;
 171         -c|--createtemplate)
 172         createTemplate
 173         shift
 174         ;;
 175         -p|-p=*|--progress|--progress=*)
 176         if [[ $key == "-p" || $key == "--progress" ]]
 177         then
 178             PROGRESSLOGLEVEL=9
 179             printf "Progress Log level is not given. Print all logs in progress.\n"
 180         else
 181             PROGRESSLOGLEVEL=${key#*=}
 182             printf "Given progress log level is ${PROGRESSLOGLEVEL}.\n"
 183         fi
 184         shift
 185         ;;
 186         -vg|--enable-valgrind)
 187         VALGRIND=1
 188         shift
 189         ;;
 190         --summary)
 191         SUMMARYFILENAME="$2"
 192         shift
 193         shift
 194         ;;
 195         *) # Unknown, which is probably target (the path to root-dir of test groups).
 196         TARGET="$1"
 197         shift
 198         esac
 199 done
 200
 201 source ${BASEPATH}/ssat-api.sh
 202
 203 if [[ "${#TARGET}" -eq "0" ]]
 204 then
 205         TARGET="."
 206 fi
 207
 208 TNtc=0
 209 TNtcpass=0
 210 TNtcfail=0
 211 TNtcignore=0
 212 TNtcneg=0
 213 TNgroup=0
 214 TNgrouppass=0
 215 TNgroupfail=0
 216 log=""
 217 groupLog=""
 218
 219 while read -d $'\0' file
 220 do
 221         CASEBASEPATH=`dirname "$file"`
 222         CASENAME=`basename "$CASEBASEPATH"`
 223         Ntc=0
 224         Npass=0
 225         Nfail=0
 226         Nneg=0
 227         tmpfile=$(mktemp)
 228
 229         if [[ ${PROGRESSLOGLEVEL} -ge 1 ]]; then
 230                 printf "[Starting] $CASENAME\n"
 231         fi
 232         pushd $CASEBASEPATH > /dev/null
 233         output=$(. $file $CASEBASEPATH)
 234         retcode=$?
 235         popd > /dev/null
 236
 237         logfile="${output##*$'\n'}"
 238         resultlog=$(<$logfile)
 239         effectiveOutput=`printf "$resultlog" | sed '$d'`
 240         log="$log$effectiveOutput\n"
 241
 242         lastline=`printf "${resultlog}" | sed '$!d'`
 243         IFS=,
 244         set $lastline
 245         Ntc=$1
 246         Npass=$2
 247         Nfail=$3
 248         Nignore=$4
 249         Nneg=$5
 250         unset IFS
 251
 252         TNtc=$((TNtc+Ntc))
 253         TNtcpass=$((TNtcpass+Npass))
 254         TNtcfail=$((TNtcfail+Nfail))
 255         TNtcignore=$((TNtcignore+Nignore))
 256         TNtcneg=$((TNtcneg+Nneg))
 257
 258         TNgroup=$((TNgroup+1))
 259         if [[ "$retcode" -eq "0" ]]
 260         then
 261                 TNgrouppass=$((TNgrouppass+1))
 262                 groupLog="${groupLog}${LightGreen}[PASSED]${NC} ${Blue}${CASENAME}${NC} ($Npass passed among $Ntc cases)\n"
 263         else
 264                 TNgroupfail=$((TNgroupfail+1))
 265                 groupLog="${groupLog}${Red}[FAILED]${NC} ${Blue}${CASENAME}${NC} ($Npass passed among $Ntc cases)\n"
 266         fi
 267
 268         printf "$log\n"
 269         log=""
 270 done < <(find $TARGET -name $TESTCASE -print0)
 271
 272 printf "\n\n==================================================\n\n"
 273
 274 printf "==================================================\n\n"
 275 printf "$groupLog"
 276 printf "==================================================\n"
 277
 278 ADDITIONALSTRING=""
 279 ADDITIONALSUMMARY=""
 280 if (( ${COUNTNEGATIVE} == 1 ))
 281 then
 282         total=$((TNtcpass+TNtcfail+TNtcignore))
 283         pos=$((total-TNtcneg))
 284         ADDITIONALSTRING="${ADDITIONALSTRING} | Positive: ${pos} / Negative: ${TNtcneg}"
 285         ADDITIONALSUMMARY="${ADDITIONALSUMMARY}, negative=${TNtcneg}"
 286 fi
 287
 288 if [ "${SUMMARYFILENAME}" != "" ]
 289 then
 290         echo "passed=${TNtcpass}, failed=${TNtcfail}, ignored=${TNtcignore}${ADDITIONALSUMMARY}" > "${SUMMARYFILENAME}"
 291 fi
 292 if (( ${TNgroupfail} == 0 ))
 293 then
 294         printf "${LightGreen}[PASSED] ${Blue}All Test Groups (${TNgroup}) Passed!${NC}\n"
 295         printf "         TC Passed: ${TNtcpass} / Failed: ${TNtcfail} / Ignored: ${TNtcignore} ${ADDITIONALSTRING}\n\n";
 296         exit 0
 297 else
 298         printf "${Red}[FAILED] ${Purple}There are failed test groups! (${TNgroupfail})${NC}\n"
 299         printf "         TC Passed: ${TNtcpass} / Failed: ${TNtcfail} / Ignored: ${TNtcignore} ${ADDITIONALSTRING}\n\n";
 300         exit 1
 301 fi
 302 # gather reports & publish them.