test/tinytest.c

   1 /* tinytest.c -- Copyright 2009-2012 Nick Mathewson
   2  *
   3  * Redistribution and use in source and binary forms, with or without
   4  * modification, are permitted provided that the following conditions
   5  * are met:
   6  * 1. Redistributions of source code must retain the above copyright
   7  *    notice, this list of conditions and the following disclaimer.
   8  * 2. Redistributions in binary form must reproduce the above copyright
   9  *    notice, this list of conditions and the following disclaimer in the
  10  *    documentation and/or other materials provided with the distribution.
  11  * 3. The name of the author may not be used to endorse or promote products
  12  *    derived from this software without specific prior written permission.
  13  *
  14  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
  15  * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
  16  * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
  17  * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
  18  * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
  19  * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
  20  * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
  21  * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  22  * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
  23  * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  24  */
  25 #ifdef TINYTEST_LOCAL
  26 #include "tinytest_local.h"
  27 #endif
  28
  29 #include <stdio.h>
  30 #include <stdlib.h>
  31 #include <string.h>
  32 #include <assert.h>
  33
  34 #ifndef NO_FORKING
  35
  36 #ifdef _WIN32
  37 #include <windows.h>
  38 #else
  39 #include <sys/types.h>
  40 #include <sys/wait.h>
  41 #include <unistd.h>
  42 #endif
  43
  44 #if defined(__APPLE__) && defined(__ENVIRONMENT_MAC_OS_X_VERSION_MIN_REQUIRED__)
  45 #if (__ENVIRONMENT_MAC_OS_X_VERSION_MIN_REQUIRED__ >= 1060 && \
  46     __ENVIRONMENT_MAC_OS_X_VERSION_MIN_REQUIRED__ < 1070)
  47 /* Workaround for a stupid bug in OSX 10.6 */
  48 #define FORK_BREAKS_GCOV
  49 #include <vproc.h>
  50 #endif
  51 #endif
  52
  53 #endif /* !NO_FORKING */
  54
  55 #ifndef __GNUC__
  56 #define __attribute__(x)
  57 #endif
  58
  59 #include "tinytest.h"
  60 #include "tinytest_macros.h"
  61
  62 #define LONGEST_TEST_NAME 16384
  63
  64 #ifndef _WIN32
  65 #define DEFAULT_TESTCASE_TIMEOUT 30U
  66 #else
  67 #define DEFAULT_TESTCASE_TIMEOUT 0U
  68 #endif
  69
  70 static int in_tinytest_main = 0; /**< true if we're in tinytest_main().*/
  71 static int n_ok = 0; /**< Number of tests that have passed */
  72 static int n_bad = 0; /**< Number of tests that have failed. */
  73 static int n_skipped = 0; /**< Number of tests that have been skipped. */
  74
  75 static int opt_forked = 0; /**< True iff we're called from inside a win32 fork*/
  76 static int opt_nofork = 0; /**< Suppress calls to fork() for debugging. */
  77 static int opt_verbosity = 1; /**< -==quiet,0==terse,1==normal,2==verbose */
  78 static unsigned int opt_timeout = DEFAULT_TESTCASE_TIMEOUT; /**< Timeout for every test (using alarm()) */
  79 const char *verbosity_flag = "";
  80
  81 const struct testlist_alias_t *cfg_aliases=NULL;
  82
  83 enum outcome { SKIP=2, OK=1, FAIL=0 };
  84 static enum outcome cur_test_outcome = 0;
  85 const char *cur_test_prefix = NULL; /**< prefix of the current test group */
  86 /** Name of the current test, if we haven't logged is yet. Used for --quiet */
  87 const char *cur_test_name = NULL;
  88
  89 #ifdef _WIN32
  90 /* Copy of argv[0] for win32. */
  91 static char commandname[MAX_PATH+1];
  92 #endif
  93
  94 static void usage(struct testgroup_t *groups, int list_groups)
  95   __attribute__((noreturn));
  96 static int process_test_option(struct testgroup_t *groups, const char *test);
  97
  98 static unsigned int testcase_set_timeout_(void)
  99 {
 100         if (!opt_timeout)
 101                 return 0;
 102 #ifndef _WIN32
 103         return alarm(opt_timeout);
 104 #else
 105         /** TODO: win32 support */
 106         fprintf(stderr, "You cannot set alarm on windows\n");
 107         exit(1);
 108 #endif
 109 }
 110 static unsigned int testcase_reset_timeout_(void)
 111 {
 112 #ifndef _WIN32
 113         return alarm(0);
 114 #endif
 115 }
 116
 117 static enum outcome
 118 testcase_run_bare_(const struct testcase_t *testcase)
 119 {
 120         void *env = NULL;
 121         int outcome;
 122         if (testcase->setup) {
 123                 env = testcase->setup->setup_fn(testcase);
 124                 if (!env)
 125                         return FAIL;
 126                 else if (env == (void*)TT_SKIP)
 127                         return SKIP;
 128         }
 129
 130         cur_test_outcome = OK;
 131         {
 132                 testcase_set_timeout_();
 133                 testcase->fn(env);
 134                 testcase_reset_timeout_();
 135         }
 136         outcome = cur_test_outcome;
 137
 138         if (testcase->setup) {
 139                 if (testcase->setup->cleanup_fn(testcase, env) == 0)
 140                         outcome = FAIL;
 141         }
 142
 143         return outcome;
 144 }
 145
 146 #define MAGIC_EXITCODE 42
 147
 148 #ifndef NO_FORKING
 149
 150 static enum outcome
 151 testcase_run_forked_(const struct testgroup_t *group,
 152                      const struct testcase_t *testcase)
 153 {
 154 #ifdef _WIN32
 155         /* Fork? On Win32?  How primitive!  We'll do what the smart kids do:
 156            we'll invoke our own exe (whose name we recall from the command
 157            line) with a command line that tells it to run just the test we
 158            want, and this time without forking.
 159
 160            (No, threads aren't an option.  The whole point of forking is to
 161            share no state between tests.)
 162          */
 163         int ok;
 164         char buffer[LONGEST_TEST_NAME+256];
 165         STARTUPINFOA si;
 166         PROCESS_INFORMATION info;
 167         DWORD exitcode;
 168
 169         if (!in_tinytest_main) {
 170                 printf("\nERROR.  On Windows, testcase_run_forked_ must be"
 171                        " called from within tinytest_main.\n");
 172                 abort();
 173         }
 174         if (opt_verbosity>0)
 175                 printf("[forking] ");
 176
 177         snprintf(buffer, sizeof(buffer), "%s --RUNNING-FORKED %s %s%s",
 178                  commandname, verbosity_flag, group->prefix, testcase->name);
 179
 180         memset(&si, 0, sizeof(si));
 181         memset(&info, 0, sizeof(info));
 182         si.cb = sizeof(si);
 183
 184         ok = CreateProcessA(commandname, buffer, NULL, NULL, 0,
 185                            0, NULL, NULL, &si, &info);
 186         if (!ok) {
 187                 printf("CreateProcess failed!\n");
 188                 return 0;
 189         }
 190         WaitForSingleObject(info.hProcess, INFINITE);
 191         GetExitCodeProcess(info.hProcess, &exitcode);
 192         CloseHandle(info.hProcess);
 193         CloseHandle(info.hThread);
 194         if (exitcode == 0)
 195                 return OK;
 196         else if (exitcode == MAGIC_EXITCODE)
 197                 return SKIP;
 198         else
 199                 return FAIL;
 200 #else
 201         int outcome_pipe[2];
 202         pid_t pid;
 203         (void)group;
 204
 205         if (pipe(outcome_pipe))
 206                 perror("opening pipe");
 207
 208         if (opt_verbosity>0)
 209                 printf("[forking] ");
 210         pid = fork();
 211 #ifdef FORK_BREAKS_GCOV
 212         vproc_transaction_begin(0);
 213 #endif
 214         if (!pid) {
 215                 /* child. */
 216                 int test_r, write_r;
 217                 char b[1];
 218                 close(outcome_pipe[0]);
 219                 test_r = testcase_run_bare_(testcase);
 220                 assert(0<=(int)test_r && (int)test_r<=2);
 221                 b[0] = "NYS"[test_r];
 222                 write_r = (int)write(outcome_pipe[1], b, 1);
 223                 if (write_r != 1) {
 224                         perror("write outcome to pipe");
 225                         exit(1);
 226                 }
 227                 exit(0);
 228                 return FAIL; /* unreachable */
 229         } else {
 230                 /* parent */
 231                 int status, r;
 232                 char b[1];
 233                 /* Close this now, so that if the other side closes it,
 234                  * our read fails. */
 235                 close(outcome_pipe[1]);
 236                 r = (int)read(outcome_pipe[0], b, 1);
 237                 if (r == 0) {
 238                         printf("[Lost connection!] ");
 239                         return 0;
 240                 } else if (r != 1) {
 241                         perror("read outcome from pipe");
 242                 }
 243                 waitpid(pid, &status, 0);
 244                 close(outcome_pipe[0]);
 245                 return b[0]=='Y' ? OK : (b[0]=='S' ? SKIP : FAIL);
 246         }
 247 #endif
 248 }
 249
 250 #endif /* !NO_FORKING */
 251
 252 int
 253 testcase_run_one(const struct testgroup_t *group,
 254                  const struct testcase_t *testcase)
 255 {
 256         enum outcome outcome;
 257
 258         if (testcase->flags & (TT_SKIP|TT_OFF_BY_DEFAULT)) {
 259                 if (opt_verbosity>0)
 260                         printf("%s%s: %s\n",
 261                            group->prefix, testcase->name,
 262                            (testcase->flags & TT_SKIP) ? "SKIPPED" : "DISABLED");
 263                 ++n_skipped;
 264                 return SKIP;
 265         }
 266
 267         if (opt_verbosity>0 && !opt_forked) {
 268                 printf("%s%s: ", group->prefix, testcase->name);
 269         } else {
 270                 if (opt_verbosity==0) printf(".");
 271                 cur_test_prefix = group->prefix;
 272                 cur_test_name = testcase->name;
 273         }
 274
 275 #ifndef NO_FORKING
 276         if ((testcase->flags & TT_FORK) && !(opt_forked||opt_nofork)) {
 277                 outcome = testcase_run_forked_(group, testcase);
 278         } else {
 279 #else
 280         {
 281 #endif
 282                 outcome = testcase_run_bare_(testcase);
 283         }
 284
 285         if (outcome == OK) {
 286                 if (opt_verbosity>0 && !opt_forked)
 287                         puts(opt_verbosity==1?"OK":"");
 288         } else if (outcome == SKIP) {
 289                 if (opt_verbosity>0 && !opt_forked)
 290                         puts("SKIPPED");
 291         } else {
 292                 if (!opt_forked)
 293                         printf("\n  [%s FAILED]\n", testcase->name);
 294         }
 295
 296         if (opt_forked) {
 297                 exit(outcome==OK ? 0 : (outcome==SKIP?MAGIC_EXITCODE : 1));
 298                 return 1; /* unreachable */
 299         } else {
 300                 return (int)outcome;
 301         }
 302 }
 303
 304 int
 305 tinytest_set_flag_(struct testgroup_t *groups, const char *arg, int set, unsigned long flag)
 306 {
 307         int i, j;
 308         size_t length = LONGEST_TEST_NAME;
 309         char fullname[LONGEST_TEST_NAME];
 310         int found=0;
 311         if (strstr(arg, ".."))
 312                 length = strstr(arg,"..")-arg;
 313         for (i=0; groups[i].prefix; ++i) {
 314                 for (j=0; groups[i].cases[j].name; ++j) {
 315                         struct testcase_t *testcase = &groups[i].cases[j];
 316                         snprintf(fullname, sizeof(fullname), "%s%s",
 317                                  groups[i].prefix, testcase->name);
 318                         if (!flag) { /* Hack! */
 319                                 printf("    %s", fullname);
 320                                 if (testcase->flags & TT_OFF_BY_DEFAULT)
 321                                         puts("   (Off by default)");
 322                                 else if (testcase->flags & TT_SKIP)
 323                                         puts("  (DISABLED)");
 324                                 else
 325                                         puts("");
 326                         }
 327                         if (!strncmp(fullname, arg, length)) {
 328                                 if (set)
 329                                         testcase->flags |= flag;
 330                                 else
 331                                         testcase->flags &= ~flag;
 332                                 ++found;
 333                         }
 334                 }
 335         }
 336         return found;
 337 }
 338
 339 static void
 340 usage(struct testgroup_t *groups, int list_groups)
 341 {
 342         puts("Options are: [--verbose|--quiet|--terse] [--no-fork] [--timeout <sec>]");
 343         puts("  Specify tests by name, or using a prefix ending with '..'");
 344         puts("  To skip a test, prefix its name with a colon.");
 345         puts("  To enable a disabled test, prefix its name with a plus.");
 346         puts("  Use --list-tests for a list of tests.");
 347         if (list_groups) {
 348                 puts("Known tests are:");
 349                 tinytest_set_flag_(groups, "..", 1, 0);
 350         }
 351         exit(0);
 352 }
 353
 354 static int
 355 process_test_alias(struct testgroup_t *groups, const char *test)
 356 {
 357         int i, j, n, r;
 358         for (i=0; cfg_aliases && cfg_aliases[i].name; ++i) {
 359                 if (!strcmp(cfg_aliases[i].name, test)) {
 360                         n = 0;
 361                         for (j = 0; cfg_aliases[i].tests[j]; ++j) {
 362                                 r = process_test_option(groups, cfg_aliases[i].tests[j]);
 363                                 if (r<0)
 364                                         return -1;
 365                                 n += r;
 366                         }
 367                         return n;
 368                 }
 369         }
 370         printf("No such test alias as @%s!",test);
 371         return -1;
 372 }
 373
 374 static int
 375 process_test_option(struct testgroup_t *groups, const char *test)
 376 {
 377         int flag = TT_ENABLED_;
 378         int n = 0;
 379         if (test[0] == '@') {
 380                 return process_test_alias(groups, test + 1);
 381         } else if (test[0] == ':') {
 382                 ++test;
 383                 flag = TT_SKIP;
 384         } else if (test[0] == '+') {
 385                 ++test;
 386                 ++n;
 387                 if (!tinytest_set_flag_(groups, test, 0, TT_OFF_BY_DEFAULT)) {
 388                         printf("No such test as %s!\n", test);
 389                         return -1;
 390                 }
 391         } else {
 392                 ++n;
 393         }
 394         if (!tinytest_set_flag_(groups, test, 1, flag)) {
 395                 printf("No such test as %s!\n", test);
 396                 return -1;
 397         }
 398         return n;
 399 }
 400
 401 void
 402 tinytest_set_aliases(const struct testlist_alias_t *aliases)
 403 {
 404         cfg_aliases = aliases;
 405 }
 406
 407 int
 408 tinytest_main(int c, const char **v, struct testgroup_t *groups)
 409 {
 410         int i, j, n=0;
 411
 412 #ifdef _WIN32
 413         const char *sp = strrchr(v[0], '.');
 414         const char *extension = "";
 415         if (!sp || stricmp(sp, ".exe"))
 416                 extension = ".exe"; /* Add an exe so CreateProcess will work */
 417         snprintf(commandname, sizeof(commandname), "%s%s", v[0], extension);
 418         commandname[MAX_PATH]='\0';
 419 #endif
 420         for (i=1; i<c; ++i) {
 421                 if (v[i][0] == '-') {
 422                         if (!strcmp(v[i], "--RUNNING-FORKED")) {
 423                                 opt_forked = 1;
 424                         } else if (!strcmp(v[i], "--no-fork")) {
 425                                 opt_nofork = 1;
 426                         } else if (!strcmp(v[i], "--quiet")) {
 427                                 opt_verbosity = -1;
 428                                 verbosity_flag = "--quiet";
 429                         } else if (!strcmp(v[i], "--verbose")) {
 430                                 opt_verbosity = 2;
 431                                 verbosity_flag = "--verbose";
 432                         } else if (!strcmp(v[i], "--terse")) {
 433                                 opt_verbosity = 0;
 434                                 verbosity_flag = "--terse";
 435                         } else if (!strcmp(v[i], "--help")) {
 436                                 usage(groups, 0);
 437                         } else if (!strcmp(v[i], "--list-tests")) {
 438                                 usage(groups, 1);
 439                         } else if (!strcmp(v[i], "--timeout")) {
 440                                 ++i;
 441                                 if (i >= c) {
 442                                         fprintf(stderr, "--timeout requires argument\n");
 443                                         return -1;
 444                                 }
 445                                 opt_timeout = (unsigned)atoi(v[i]);
 446                         } else {
 447                                 fprintf(stderr, "Unknown option %s. Try --help\n", v[i]);
 448                                 return -1;
 449                         }
 450                 } else {
 451                         int r = process_test_option(groups, v[i]);
 452                         if (r<0)
 453                                 return -1;
 454                         n += r;
 455                 }
 456         }
 457         if (!n)
 458                 tinytest_set_flag_(groups, "..", 1, TT_ENABLED_);
 459
 460 #ifdef _IONBF
 461         setvbuf(stdout, NULL, _IONBF, 0);
 462 #endif
 463
 464         ++in_tinytest_main;
 465         for (i = 0; groups[i].prefix; ++i) {
 466                 struct testgroup_t *group = &groups[i];
 467                 for (j = 0; group->cases[j].name; ++j) {
 468                         struct testcase_t *testcase = &group->cases[j];
 469                         int test_attempts = 3;
 470                         int test_ret_err;
 471
 472                         if (!(testcase->flags & TT_ENABLED_))
 473                                 continue;
 474
 475                         for (;;) {
 476                                 test_ret_err = testcase_run_one(group, testcase);
 477
 478                                 if (test_ret_err == OK)
 479                                         break;
 480                                 if (!(testcase->flags & TT_RETRIABLE))
 481                                         break;
 482                                 printf("\n  [RETRYING %s (%i)]\n", testcase->name, test_attempts);
 483                                 if (!test_attempts--)
 484                                         break;
 485                         }
 486
 487                         switch (test_ret_err) {
 488                                 case OK:   ++n_ok;      break;
 489                                 case SKIP: ++n_skipped; break;
 490                                 default:   ++n_bad;     break;
 491                         }
 492                 }
 493         }
 494
 495         --in_tinytest_main;
 496
 497         if (opt_verbosity==0)
 498                 puts("");
 499
 500         if (n_bad)
 501                 printf("%d/%d TESTS FAILED. (%d skipped)\n", n_bad,
 502                        n_bad+n_ok,n_skipped);
 503         else if (opt_verbosity >= 1)
 504                 printf("%d tests ok.  (%d skipped)\n", n_ok, n_skipped);
 505
 506         return (n_bad == 0) ? 0 : 1;
 507 }
 508
 509 int
 510 tinytest_get_verbosity_(void)
 511 {
 512         return opt_verbosity;
 513 }
 514
 515 void
 516 tinytest_set_test_failed_(void)
 517 {
 518         if (opt_verbosity <= 0 && cur_test_name) {
 519                 if (opt_verbosity==0) puts("");
 520                 printf("%s%s: ", cur_test_prefix, cur_test_name);
 521                 cur_test_name = NULL;
 522         }
 523         cur_test_outcome = 0;
 524 }
 525
 526 void
 527 tinytest_set_test_skipped_(void)
 528 {
 529         if (cur_test_outcome==OK)
 530                 cur_test_outcome = SKIP;
 531 }
 532
 533 char *
 534 tinytest_format_hex_(const void *val_, unsigned long len)
 535 {
 536         const unsigned char *val = val_;
 537         char *result, *cp;
 538         size_t i;
 539
 540         if (!val)
 541                 return strdup("null");
 542         if (!(result = malloc(len*2+1)))
 543                 return strdup("<allocation failure>");
 544         cp = result;
 545         for (i=0;i<len;++i) {
 546                 *cp++ = "0123456789ABCDEF"[val[i] >> 4];
 547                 *cp++ = "0123456789ABCDEF"[val[i] & 0x0f];
 548         }
 549         *cp = 0;
 550         return result;
 551 }