ssat.sh

   1 #!/usr/bin/env bash
   2 ##
   3 ## @file ssat.sh
   4 ## @author MyungJoo Ham <myungjoo.ham@gmail.com>
   5 ## @date Jun 22 2018
   6 ## @brief This executes test groups and reports aggregated test results.
   7 ## @return 0 if all PASSED. Positive if some FAILED.
   8 ## @todo Separate GStreamer related functions as plugins
   9 ##
  10 ## This uses sed, date, cmp
  11 ##
  12 ## If there is no arguments specified, this will search for all "runTest.sh" in
  13 ## the subdirectory of this file and regard them as the test groups.
  14 ##
  15 ## If a testgroup (runTest.sh) returns 0 while there are failed testcase,
  16 ## it implies that the failed testcases may be ignored and it's good to go.
  17 ##
  18 ## If --help or -h is given, this will show detailed description.
  19
  20 ##
  21 ## @mainpage SSAT
  22 ## @section intro        Introduction
  23 ## - Introduction     :  Shell Script Automated Tester
  24 ## @section Program      Program Name
  25 ## - Program Name     :  ssat
  26 ## - Program Details  :  SSAT is a software testing framework for test cases written in BASH shell scripts.
  27 ##   It can search for test scripts recursively from a given path and summarize the test results.
  28 ##   If there is any "critical" fail, ssat will return non-zero values on its exit.
  29 ## @section INOUTPUT     Input/output data
  30 ## - INPUT            :  Test Cases (If not supplied, the current path is the root of test cases)
  31 ## - OUTPUT           :  Summary of test results to stdout. Exit code of 0 if success, non-zero if not success.
  32 ## @section CREATEINFO   Code information
  33 ## - Initial date     :  2018/06/22
  34 ## - Version          :  1.4.0
  35
  36 TARGET=$(pwd)
  37 TARGET_ASSIGNED=0
  38 BASEPATH=`dirname "$0"`
  39 BASENAME=`basename "$0"`
  40 TESTCASE="runTest.sh"
  41 SUMMARYFILENAME=""
  42 VERSION="1.4.0"
  43
  44 #
  45 SILENT=1
  46 PROGRESSLOGLEVEL=0
  47 COUNTNEGATIVE=0
  48 COUNTNEGATIVEPOSTFIX=""
  49 VALGRIND=0
  50 VALGRIND_SUPPRESSION=""
  51 date=`date +"%b %d %Y"`
  52
  53 ## @fn createTemplate()
  54 ## @brief Generate runTest template file
  55 ##
  56 ## Note that the generated template has no license.
  57 ## The SSAT user may put their own license for the generated files.
  58 ## I hereby grant the right to relicense the generated files.
  59 function createTemplate() {
  60         if [[ -f "runTest.sh" ]]
  61         then
  62                 printf "Cannot create runTest.sh here. The file already exists at $(pwd).\n\n"
  63                 exit 1
  64         fi
  65
  66         echo -e "#!/usr/bin/env bash\n\
  67 ##\n\
  68 ## @file runTest.sh\n\
  69 ## @author MyungJoo Ham <myungjoo.ham@gmail.com>\n\
  70 ## @date ${date}\n\
  71 ## @brief This is a template file for SSAT test cases. You may designate your own license.\n\
  72 #\n\
  73 if [[ \"\$SSATAPILOADED\" != \"1\" ]]\n\
  74 then\n\
  75         SILENT=0\n\
  76         INDEPENDENT=1\n\
  77         search=\"ssat-api.sh\"\n\
  78         source \$search\n\
  79         printf \"\${Blue}Independent Mode\${NC}\\n\"\n\
  80 fi\n\
  81 testInit \$1 # You may replace this with Test Group Name\n\
  82 \n\
  83 #testResult 1 T1 \"Dummy Test1\"\n\
  84 #callTestSuccess gst-launch-1.0 \"-q videotestsrc ! videoconvert ! autovideosink\" T2 \"This may run indefinitely\"\n\
  85 #callCompareTest golden.log executeResult.log T3 \"The two files must be same\" 0\n\
  86 \n\
  87 report\n" > runTest.sh
  88 chmod a+x runTest.sh
  89
  90         exit 0
  91 }
  92
  93 # Handle arguments
  94 POSITIONAL=()
  95 while [[ $# -gt 0 ]]
  96 do
  97         key="$1"
  98         case $key in
  99         -h|--help)
 100                 printf "usage: ${BASENAME} [--help] [<path>] [--testcase <filename>] [--nocolor] [--showstdout] [--createtemplate] [--countnegative <postfix>] [--enable-valgrind] [--valgrind-suppression <filepath>] [--version]\n\n"
 101                 printf "These are common ${Red}ssat${NC} commands used:\n\n"
 102                 printf "Test all test-groups in the current ($(pwd)) directory, recursively\n"
 103                 printf "    (no options specified)\n"
 104                 printf "    $ ${BASENAME}\n"
 105                 printf "\n"
 106                 printf "Test all test-groups in the specified directory, recursively\n"
 107                 printf "    <path>\n"
 108                 printf "    $ ${BASENAME} /home/username/test\n"
 109                 printf "    If there are multiple paths, the last one will be used\n"
 110                 printf "\n"
 111                 printf "Search for \"filename\" as the testcase scripts\n"
 112                 printf "    --testcase or -t\n"
 113                 printf "    $ ${BASENAME} --testcase cases.sh\n"
 114                 printf "    Search for cases.sh instead of runTest.sh\n"
 115                 printf "\n"
 116                 printf "Do not emit colored text\n"
 117                 printf "    --nocolor or -n\n"
 118                 printf "\n"
 119                 printf "Show stdout of test cases\n"
 120                 printf "    --showstdout or -s\n"
 121                 printf "\n"
 122                 printf "Create a template 'runTest.sh' test group at your current directory\n"
 123                 printf "    --createtemplate or -c\n"
 124                 printf "\n"
 125                 printf "Show progress during execution\n"
 126                 printf "    --progress or -p or -p=(0,1,9)\n"
 127                 printf "        '0' : Do not print logs in progress. If -p is not given, -p=0 is assumed.\n"
 128                 printf "        '1' : Print test group names only in progress.\n"
 129                 printf "        '2-9' : Print all logs in progress. If -p is given without numbers, -p=9 is used.\n"
 130                 printf "     $ ${BASENAME} -p=1\n"
 131                 printf "     $ ${BASENAME} --progress=9 (equal to --progress) \n"
 132                 printf "\n"
 133                 printf "Enable valgrind to perform memcheck\n"
 134                 printf "    --enable-valgrind or -vg\n"
 135                 printf "\n"
 136                 printf "Suppress valgrind errors with the given suppression file\n"
 137                 printf "    --valgrind-suppression <path to the suppression file>\n"
 138                 printf "    or \n"
 139                 printf "    -vs <path to the suppression file>\n"
 140                 printf "\n"
 141                 printf "Shows this message\n"
 142                 printf "    --help or -h\n"
 143                 printf "    $ ${BASENAME} --help \n"
 144                 printf "\n"
 145                 printf "Count negative test cases with the given postfix\n"
 146                 printf "    --countnegative or -cn\n"
 147                 printf "    $ ${BASENAME} --countnegative _n\n"
 148                 printf "    $ ${BASENAME} -cn _n\n"
 149                 printf "\n"
 150                 printf "Write result summary as a file\n"
 151                 printf "    --summary <filename>\n"
 152                 printf "\n"
 153                 printf "Show the version\n"
 154                 printf "    --version or -v\n"
 155                 printf "\n\n"
 156                 exit 0
 157         ;;
 158         -n|--nocolor)
 159         nocolor=1
 160         shift
 161         ;;
 162         -t|--testcase)
 163         TESTCASE="$2"
 164         shift
 165         shift
 166         ;;
 167         -cn|--countnegative)
 168         COUNTNEGATIVE=1
 169         COUNTNEGATIVEPOSTFIX="$2"
 170         if [[ "${COUNTNEGATIVEPOSTFIX}" == "" ]]
 171         then
 172                 printf "${BASENAME} -cn or --countnegative requires postfix.\n\n"
 173                 exit -2
 174         fi
 175         shift
 176         shift
 177         ;;
 178         -s|--showstdout)
 179         SILENT=0
 180         shift
 181         ;;
 182         -c|--createtemplate)
 183         createTemplate
 184         shift
 185         ;;
 186         -p|-p=*|--progress|--progress=*)
 187         if [[ $key == "-p" || $key == "--progress" ]]
 188         then
 189             PROGRESSLOGLEVEL=9
 190             printf "Progress Log level is not given. Print all logs in progress.\n"
 191         else
 192             PROGRESSLOGLEVEL=${key#*=}
 193             printf "Given progress log level is ${PROGRESSLOGLEVEL}.\n"
 194         fi
 195         shift
 196         ;;
 197         -vg|--enable-valgrind)
 198         VALGRIND=1
 199         shift
 200         ;;
 201         -vs|--valgrind-suppression)
 202         VALGRIND_SUPPRESSION=" --suppressions=$2 "
 203         shift
 204         shift
 205         ;;
 206         --summary)
 207         SUMMARYFILENAME="$2"
 208         shift
 209         shift
 210         ;;
 211         --version|-v)
 212         printf "${VERSION}\n"
 213         exit 0
 214         shift
 215         ;;
 216         *) # Unknown, which is probably target (the path to root-dir of test groups).
 217         # If this is the second occurrence, ignore it.
 218         # Assume that the previous string is path and the later string is an invalid argument.
 219         if [ $TARGET_ASSIGNED -eq 0 ]
 220         then
 221                 TARGET="$1"
 222                 TARGET_ASSIGNED=1
 223         fi
 224         shift
 225         esac
 226 done
 227
 228 source ${BASEPATH}/ssat-api.sh
 229
 230 if [[ "${#TARGET}" -eq "0" ]]
 231 then
 232         TARGET="."
 233 fi
 234
 235 TNtc=0
 236 TNtcpass=0
 237 TNtcfail=0
 238 TNtcignore=0
 239 TNtcneg=0
 240 TNgroup=0
 241 TNgrouppass=0
 242 TNgroupfail=0
 243 log=""
 244 groupLog=""
 245
 246 while read -d $'\0' file
 247 do
 248         CASEBASEPATH=`dirname "$file"`
 249         CASENAME=`basename "$CASEBASEPATH"`
 250         Ntc=0
 251         Npass=0
 252         Nfail=0
 253         Nneg=0
 254         tmpfile=$(mktemp)
 255
 256         if [[ ${PROGRESSLOGLEVEL} -ge 1 ]]; then
 257                 printf "[Starting] $CASENAME\n"
 258         fi
 259         pushd $CASEBASEPATH > /dev/null
 260         output=$(. $file $CASEBASEPATH)
 261         retcode=$?
 262         popd > /dev/null
 263
 264         logfile="${output##*$'\n'}"
 265         resultlog=$(<$logfile)
 266         effectiveOutput=`printf "$resultlog" | sed '$d'`
 267         log="$log$effectiveOutput\n"
 268
 269         lastline=`printf "${resultlog}" | sed '$!d'`
 270         IFS=,
 271         set $lastline
 272         Ntc=$1
 273         Npass=$2
 274         Nfail=$3
 275         Nignore=$4
 276         Nneg=$5
 277         unset IFS
 278
 279         TNtc=$((TNtc+Ntc))
 280         TNtcpass=$((TNtcpass+Npass))
 281         TNtcfail=$((TNtcfail+Nfail))
 282         TNtcignore=$((TNtcignore+Nignore))
 283         TNtcneg=$((TNtcneg+Nneg))
 284
 285         TNgroup=$((TNgroup+1))
 286         if [[ "$retcode" -eq "0" ]]
 287         then
 288                 TNgrouppass=$((TNgrouppass+1))
 289                 groupLog="${groupLog}${LightGreen}[PASSED]${NC} ${Blue}${CASENAME}${NC} ($Npass passed among $Ntc cases)\n"
 290         else
 291                 TNgroupfail=$((TNgroupfail+1))
 292                 groupLog="${groupLog}${Red}[FAILED]${NC} ${Blue}${CASENAME}${NC} ($Npass passed among $Ntc cases)\n"
 293         fi
 294
 295         printf "$log\n"
 296         log=""
 297 done < <(find $TARGET -name $TESTCASE -print0)
 298
 299 printf "\n\n==================================================\n\n"
 300
 301 printf "==================================================\n\n"
 302 printf "$groupLog"
 303 printf "==================================================\n"
 304
 305 ADDITIONALSTRING=""
 306 ADDITIONALSUMMARY=""
 307 if (( ${COUNTNEGATIVE} == 1 ))
 308 then
 309         total=$((TNtcpass+TNtcfail+TNtcignore))
 310         pos=$((total-TNtcneg))
 311         ADDITIONALSTRING="${ADDITIONALSTRING} | Positive: ${pos} / Negative: ${TNtcneg}"
 312         ADDITIONALSUMMARY="${ADDITIONALSUMMARY}, negative=${TNtcneg}"
 313 fi
 314
 315 if [ "${SUMMARYFILENAME}" != "" ]
 316 then
 317         echo "passed=${TNtcpass}, failed=${TNtcfail}, ignored=${TNtcignore}${ADDITIONALSUMMARY}" > "${SUMMARYFILENAME}"
 318 fi
 319 if (( ${TNgroupfail} == 0 ))
 320 then
 321         printf "${LightGreen}[PASSED] ${Blue}All Test Groups (${TNgroup}) Passed!${NC}\n"
 322         printf "         TC Passed: ${TNtcpass} / Failed: ${TNtcfail} / Ignored: ${TNtcignore} ${ADDITIONALSTRING}\n\n";
 323         exit 0
 324 else
 325         printf "${Red}[FAILED] ${Purple}There are failed test groups! (${TNgroupfail})${NC}\n"
 326         printf "         TC Passed: ${TNtcpass} / Failed: ${TNtcfail} / Ignored: ${TNtcignore} ${ADDITIONALSTRING}\n\n";
 327         exit 1
 328 fi
 329 # gather reports & publish them.