test/cairo-test-runner.c

   1 /*
   2  * Copyright © 2008 Chris Wilson
   3  *
   4  * Permission to use, copy, modify, distribute, and sell this software
   5  * and its documentation for any purpose is hereby granted without
   6  * fee, provided that the above copyright notice appear in all copies
   7  * and that both that copyright notice and this permission notice
   8  * appear in supporting documentation, and that the name of
   9  * Chris Wilson not be used in advertising or publicity pertaining to
  10  * distribution of the software without specific, written prior
  11  * permission. Chris Wilson makes no representations about the
  12  * suitability of this software for any purpose.  It is provided "as
  13  * is" without express or implied warranty.
  14  *
  15  * CHRIS WILSON DISCLAIMS ALL WARRANTIES WITH REGARD TO THIS
  16  * SOFTWARE, INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY AND
  17  * FITNESS, IN NO EVENT SHALL CHRIS WILSON BE LIABLE FOR ANY SPECIAL,
  18  * INDIRECT OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER
  19  * RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION
  20  * OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR
  21  * IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
  22  *
  23  * Author: Chris Wilson <chris@chris-wilson.co.uk>
  24  */
  25
  26 #include "cairo-test-private.h"
  27 #include "cairo-boilerplate-getopt.h"
  28
  29 /* get the "real" version info instead of dummy cairo-version.h */
  30 #undef CAIRO_VERSION_H
  31 #undef CAIRO_VERSION_MAJOR
  32 #undef CAIRO_VERSION_MINOR
  33 #undef CAIRO_VERSION_MICRO
  34 #include "../cairo-version.h"
  35
  36 #include <pixman.h> /* for version information */
  37
  38 #define SHOULD_FORK HAVE_FORK && HAVE_WAITPID
  39 #if SHOULD_FORK
  40 #if HAVE_UNISTD_H
  41 #include <unistd.h>
  42 #endif
  43 #if HAVE_SIGNAL_H
  44 #include <signal.h>
  45 #endif
  46 #include <sys/types.h>
  47 #include <sys/wait.h>
  48 #endif
  49 #if HAVE_LIBGEN_H
  50 #include <libgen.h>
  51 #endif
  52
  53 #if HAVE_VALGRIND
  54 #include <valgrind.h>
  55 #else
  56 #define RUNNING_ON_VALGRIND 0
  57 #endif
  58
  59 #ifdef _MSC_VER
  60 #include <crtdbg.h>
  61 #endif
  62
  63 typedef struct _cairo_test_list {
  64     const cairo_test_t *test;
  65     struct _cairo_test_list *next;
  66 } cairo_test_list_t;
  67
  68 typedef struct _cairo_test_runner {
  69     cairo_test_context_t base;
  70
  71     unsigned int num_device_offsets;
  72
  73     cairo_bool_t passed;
  74     int num_passed;
  75     int num_skipped;
  76     int num_failed;
  77     int num_xfailed;
  78     int num_error;
  79     int num_crashed;
  80
  81     cairo_test_list_t *crashes_preamble;
  82     cairo_test_list_t *errors_preamble;
  83     cairo_test_list_t *fails_preamble;
  84
  85     cairo_test_list_t **crashes_per_target;
  86     cairo_test_list_t **errors_per_target;
  87     cairo_test_list_t **fails_per_target;
  88
  89     int *num_failed_per_target;
  90     int *num_error_per_target;
  91     int *num_crashed_per_target;
  92
  93     cairo_bool_t foreground;
  94     cairo_bool_t exit_on_failure;
  95     cairo_bool_t list_only;
  96     cairo_bool_t full_test;
  97     cairo_bool_t keyword_match;
  98     cairo_bool_t slow;
  99     cairo_bool_t force_pass;
 100 } cairo_test_runner_t;
 101
 102 typedef enum {
 103     GE,
 104     GT
 105 } cairo_test_compare_op_t;
 106
 107 static cairo_test_list_t *tests;
 108
 109 static void CAIRO_BOILERPLATE_PRINTF_FORMAT(2,3)
 110 _log (cairo_test_context_t *ctx,
 111       const char *fmt,
 112       ...)
 113 {
 114     va_list ap;
 115
 116     va_start (ap, fmt);
 117     vprintf (fmt, ap);
 118     va_end (ap);
 119
 120     va_start (ap, fmt);
 121     cairo_test_logv (ctx, fmt, ap);
 122     va_end (ap);
 123 }
 124
 125 static cairo_test_list_t *
 126 _list_prepend (cairo_test_list_t *head, const cairo_test_t *test)
 127 {
 128     cairo_test_list_t *list;
 129
 130     list = xmalloc (sizeof (cairo_test_list_t));
 131     list->test = test;
 132     list->next = head;
 133     head = list;
 134
 135     return head;
 136 }
 137
 138 static cairo_test_list_t *
 139 _list_reverse (cairo_test_list_t *head)
 140 {
 141     cairo_test_list_t *list, *next;
 142
 143     for (list = head, head = NULL; list != NULL; list = next) {
 144         next = list->next;
 145         list->next = head;
 146         head = list;
 147     }
 148
 149     return head;
 150 }
 151
 152 static void
 153 _list_free (cairo_test_list_t *list)
 154 {
 155     while (list != NULL) {
 156         cairo_test_list_t *next = list->next;
 157         free (list);
 158         list = next;
 159     }
 160 }
 161
 162 static cairo_bool_t
 163 is_running_under_debugger (void)
 164 {
 165 #if HAVE_UNISTD_H && HAVE_LIBGEN_H && __linux__
 166     char buf[1024];
 167
 168     sprintf (buf, "/proc/%d/exe", getppid ());
 169     if (readlink (buf, buf, sizeof (buf)) != -1 &&
 170         strncmp (basename (buf), "gdb", 3) == 0)
 171     {
 172         return TRUE;
 173     }
 174 #endif
 175
 176     if (RUNNING_ON_VALGRIND)
 177         return TRUE;
 178
 179     return FALSE;
 180 }
 181
 182 #if SHOULD_FORK
 183 static cairo_test_status_t
 184 _cairo_test_wait (pid_t pid)
 185 {
 186     int exitcode;
 187
 188     if (waitpid (pid, &exitcode, 0) != pid)
 189         return CAIRO_TEST_CRASHED;
 190
 191     if (WIFSIGNALED (exitcode)) {
 192         switch (WTERMSIG (exitcode)) {
 193         case SIGINT:
 194 #if HAVE_RAISE
 195             raise (SIGINT);
 196 #endif
 197             return CAIRO_TEST_UNTESTED;
 198         default:
 199             return CAIRO_TEST_CRASHED;
 200         }
 201     }
 202
 203     return WEXITSTATUS (exitcode);
 204 }
 205 #endif
 206
 207 static cairo_test_status_t
 208 _cairo_test_runner_preamble (cairo_test_runner_t *runner,
 209                              cairo_test_context_t *ctx)
 210 {
 211 #if SHOULD_FORK
 212     if (! runner->foreground) {
 213         pid_t pid;
 214
 215         switch ((pid = fork ())) {
 216         case -1: /* error */
 217             return CAIRO_TEST_UNTESTED;
 218
 219         case 0: /* child */
 220             exit (ctx->test->preamble (ctx));
 221
 222         default:
 223             return _cairo_test_wait (pid);
 224         }
 225     }
 226 #endif
 227     return ctx->test->preamble (ctx);
 228 }
 229
 230 static cairo_test_status_t
 231 _cairo_test_runner_draw (cairo_test_runner_t *runner,
 232                          cairo_test_context_t *ctx,
 233                          const cairo_boilerplate_target_t *target,
 234                          cairo_bool_t similar,
 235                          int device_offset)
 236 {
 237 #if SHOULD_FORK
 238     if (! runner->foreground) {
 239         pid_t pid;
 240
 241         switch ((pid = fork ())) {
 242         case -1: /* error */
 243             return CAIRO_TEST_UNTESTED;
 244
 245         case 0: /* child */
 246             exit (_cairo_test_context_run_for_target (ctx, target,
 247                                                       similar, device_offset));
 248
 249         default:
 250             return _cairo_test_wait (pid);
 251         }
 252     }
 253 #endif
 254     return _cairo_test_context_run_for_target (ctx, target,
 255                                                similar, device_offset);
 256 }
 257
 258 static void
 259 append_argv (int *argc, char ***argv, const char *str)
 260 {
 261     int old_argc;
 262     char **old_argv;
 263     cairo_bool_t doit;
 264     const char *s, *t;
 265     int olen;
 266     int len;
 267     int i;
 268     int args_to_add = 0;
 269
 270     if (str == NULL)
 271         return;
 272
 273     old_argc = *argc;
 274     old_argv = *argv;
 275
 276     doit = FALSE;
 277     do {
 278         if (doit)
 279             *argv = xmalloc (olen);
 280
 281         olen = sizeof (char *) * (args_to_add + *argc);
 282         for (i = 0; i < old_argc; i++) {
 283             len = strlen (old_argv[i]) + 1;
 284             if (doit) {
 285                 (*argv)[i] = (char *) *argv + olen;
 286                 memcpy ((*argv)[i], old_argv[i], len);
 287             }
 288             olen += len;
 289         }
 290
 291         s = str;
 292         while ((t = strpbrk (s, " \t,:;")) != NULL) {
 293             if (t - s) {
 294                 len = t - s;
 295                 if (doit) {
 296                     (*argv)[i] = (char *) *argv + olen;
 297                     memcpy ((*argv)[i], s, len);
 298                     (*argv)[i][len] = '\0';
 299                 } else {
 300                     olen += sizeof (char *);
 301                 }
 302                 args_to_add++;
 303                 olen += len + 1;
 304                 i++;
 305             }
 306             s = t + 1;
 307         }
 308         if (*s != '\0') {
 309             len = strlen (s) + 1;
 310             if (doit) {
 311                 (*argv)[i] = (char *) *argv + olen;
 312                 memcpy ((*argv)[i], s, len);
 313             } else {
 314                 olen += sizeof (char *);
 315             }
 316             args_to_add++;
 317             olen += len;
 318             i++;
 319         }
 320     } while (doit++ == FALSE);
 321     *argc = i;
 322 }
 323
 324 static void
 325 usage (const char *argv0)
 326 {
 327     fprintf (stderr,
 328              "Usage: %s [-afkxsl] [test-names|keywords ...]\n"
 329              "\n"
 330              "Run the cairo conformance test suite over the given tests (all by default)\n"
 331              "The command-line arguments are interpreted as follows:\n"
 332              "\n"
 333              "  -a      all; run the full set of tests. By default the test suite\n"
 334              "          skips similar surface and device offset testing.\n"
 335              "  -f      foreground; do not fork\n"
 336              "  -k      match tests by keyword\n"
 337              "  -l      list only; just list selected test case names without executing\n"
 338              "  -s      include slow, long running tests\n"
 339              "  -x      exit on first failure\n"
 340              "\n"
 341              "If test names are given they are used as matches either to a specific\n"
 342              "test case or to a keyword, so a command such as\n"
 343              "\"%s -k text\" can be used to run all text test cases, and\n"
 344              "\"%s text-transform\" to run the individual case.\n",
 345              argv0, argv0, argv0);
 346 }
 347
 348 static void
 349 _parse_cmdline (cairo_test_runner_t *runner, int *argc, char **argv[])
 350 {
 351     int c;
 352
 353     while (1) {
 354         c = _cairo_getopt (*argc, *argv, ":afklsx");
 355         if (c == -1)
 356             break;
 357
 358         switch (c) {
 359         case 'a':
 360             runner->full_test = TRUE;
 361             break;
 362         case 'f':
 363             runner->foreground = TRUE;
 364             break;
 365         case 'k':
 366             runner->keyword_match = TRUE;
 367             break;
 368         case 'l':
 369             runner->list_only = TRUE;
 370             break;
 371         case 's':
 372             runner->slow = TRUE;
 373             break;
 374         case 'x':
 375             runner->exit_on_failure = TRUE;
 376             break;
 377         default:
 378             fprintf (stderr, "Internal error: unhandled option: %c\n", c);
 379             /* fall-through */
 380         case '?':
 381             usage ((*argv)[0]);
 382             exit (1);
 383         }
 384     }
 385
 386     *argc -= optind;
 387     *argv += optind;
 388 }
 389
 390 static void
 391 _runner_init (cairo_test_runner_t *runner)
 392 {
 393     cairo_test_init (&runner->base, "cairo-test-suite", ".");
 394
 395     runner->passed = TRUE;
 396
 397     runner->fails_preamble = NULL;
 398     runner->crashes_preamble = NULL;
 399     runner->errors_preamble = NULL;
 400
 401     runner->fails_per_target = xcalloc (sizeof (cairo_test_list_t *),
 402                                         runner->base.num_targets);
 403     runner->crashes_per_target = xcalloc (sizeof (cairo_test_list_t *),
 404                                           runner->base.num_targets);
 405     runner->errors_per_target = xcalloc (sizeof (cairo_test_list_t *),
 406                                           runner->base.num_targets);
 407     runner->num_failed_per_target = xcalloc (sizeof (int),
 408                                              runner->base.num_targets);
 409     runner->num_error_per_target = xcalloc (sizeof (int),
 410                                              runner->base.num_targets);
 411     runner->num_crashed_per_target = xcalloc (sizeof (int),
 412                                               runner->base.num_targets);
 413 }
 414
 415 static void
 416 _runner_print_versions (cairo_test_runner_t *runner)
 417 {
 418     _log (&runner->base,
 419          "Compiled against cairo %s, running on %s.\n",
 420          CAIRO_VERSION_STRING, cairo_version_string ());
 421     _log (&runner->base,
 422          "Compiled against pixman %s, running on %s.\n",
 423          PIXMAN_VERSION_STRING, pixman_version_string ());
 424
 425     fflush (runner->base.log_file);
 426 }
 427
 428 static void
 429 _runner_print_summary (cairo_test_runner_t *runner)
 430 {
 431     _log (&runner->base,
 432           "%d Passed, %d Failed [%d crashed, %d expected], %d Skipped\n",
 433           runner->num_passed,
 434
 435           runner->num_failed + runner->num_crashed + runner->num_xfailed,
 436           runner->num_crashed,
 437           runner->num_xfailed,
 438
 439           runner->num_skipped);
 440 }
 441
 442 static void
 443 _runner_print_details (cairo_test_runner_t *runner)
 444 {
 445     cairo_test_list_t *list;
 446     unsigned int n;
 447
 448     if (runner->crashes_preamble) {
 449         int count = 0;
 450
 451         for (list = runner->crashes_preamble; list != NULL; list = list->next)
 452             count++;
 453
 454         _log (&runner->base, "Preamble: %d crashed! -", count);
 455
 456         for (list = runner->crashes_preamble; list != NULL; list = list->next) {
 457             char *name = cairo_test_get_name (list->test);
 458             _log (&runner->base, " %s", name);
 459             free (name);
 460         }
 461         _log (&runner->base, "\n");
 462     }
 463     if (runner->errors_preamble) {
 464         int count = 0;
 465
 466         for (list = runner->errors_preamble; list != NULL; list = list->next)
 467             count++;
 468
 469         _log (&runner->base, "Preamble: %d error -", count);
 470
 471         for (list = runner->errors_preamble; list != NULL; list = list->next) {
 472             char *name = cairo_test_get_name (list->test);
 473             _log (&runner->base, " %s", name);
 474             free (name);
 475         }
 476         _log (&runner->base, "\n");
 477     }
 478     if (runner->fails_preamble) {
 479         int count = 0;
 480
 481         for (list = runner->fails_preamble; list != NULL; list = list->next)
 482             count++;
 483
 484         _log (&runner->base, "Preamble: %d failed -", count);
 485
 486         for (list = runner->fails_preamble; list != NULL; list = list->next) {
 487             char *name = cairo_test_get_name (list->test);
 488             _log (&runner->base, " %s", name);
 489             free (name);
 490         }
 491         _log (&runner->base, "\n");
 492     }
 493
 494     for (n = 0; n < runner->base.num_targets; n++) {
 495         const cairo_boilerplate_target_t *target;
 496
 497         target = runner->base.targets_to_test[n];
 498         if (runner->num_crashed_per_target[n]) {
 499             _log (&runner->base, "%s (%s): %d crashed! -",
 500                   target->name,
 501                   cairo_boilerplate_content_name (target->content),
 502                   runner->num_crashed_per_target[n]);
 503
 504             for (list = runner->crashes_per_target[n];
 505                  list != NULL;
 506                  list = list->next)
 507             {
 508                 char *name = cairo_test_get_name (list->test);
 509                 _log (&runner->base, " %s", name);
 510                 free (name);
 511             }
 512             _log (&runner->base, "\n");
 513         }
 514         if (runner->num_error_per_target[n]) {
 515             _log (&runner->base, "%s (%s): %d error -",
 516                   target->name,
 517                   cairo_boilerplate_content_name (target->content),
 518                   runner->num_error_per_target[n]);
 519
 520             for (list = runner->errors_per_target[n];
 521                  list != NULL;
 522                  list = list->next)
 523             {
 524                 char *name = cairo_test_get_name (list->test);
 525                 _log (&runner->base, " %s", name);
 526                 free (name);
 527             }
 528             _log (&runner->base, "\n");
 529         }
 530
 531         if (runner->num_failed_per_target[n]) {
 532             _log (&runner->base, "%s (%s): %d failed -",
 533                   target->name,
 534                   cairo_boilerplate_content_name (target->content),
 535                   runner->num_failed_per_target[n]);
 536
 537             for (list = runner->fails_per_target[n];
 538                  list != NULL;
 539                  list = list->next)
 540             {
 541                 char *name = cairo_test_get_name (list->test);
 542                 _log (&runner->base, " %s", name);
 543                 free (name);
 544             }
 545             _log (&runner->base, "\n");
 546         }
 547     }
 548 }
 549
 550 static void
 551 _runner_print_results (cairo_test_runner_t *runner)
 552 {
 553     _runner_print_summary (runner);
 554     _runner_print_details (runner);
 555
 556     if (! runner->passed && ! runner->num_crashed) {
 557         _log (&runner->base,
 558 "\n"
 559 "Note: These failures may be due to external factors.\n"
 560 "Please read test/README -- \"Getting the elusive zero failures\".\n");
 561     }
 562 }
 563
 564 static cairo_test_status_t
 565 _runner_fini (cairo_test_runner_t *runner)
 566 {
 567     unsigned int n;
 568
 569     _list_free (runner->crashes_preamble);
 570     _list_free (runner->errors_preamble);
 571     _list_free (runner->fails_preamble);
 572
 573     for (n = 0; n < runner->base.num_targets; n++) {
 574         _list_free (runner->crashes_per_target[n]);
 575         _list_free (runner->errors_per_target[n]);
 576         _list_free (runner->fails_per_target[n]);
 577     }
 578     free (runner->crashes_per_target);
 579     free (runner->errors_per_target);
 580     free (runner->fails_per_target);
 581
 582     free (runner->num_crashed_per_target);
 583     free (runner->num_error_per_target);
 584     free (runner->num_failed_per_target);
 585
 586     cairo_test_fini (&runner->base);
 587
 588     if (runner->force_pass)
 589         return CAIRO_TEST_SUCCESS;
 590
 591     return runner->num_failed + runner->num_crashed ?
 592         CAIRO_TEST_FAILURE :
 593         runner->num_passed + runner->num_xfailed ?
 594         CAIRO_TEST_SUCCESS : CAIRO_TEST_UNTESTED;
 595 }
 596
 597 static cairo_bool_t
 598 _version_compare (int a, cairo_test_compare_op_t op, int b)
 599 {
 600     switch (op) {
 601     case GT: return a > b;
 602     case GE: return a >= b;
 603     default: return FALSE;
 604     }
 605 }
 606
 607
 608 static cairo_bool_t
 609 _get_required_version (const char *str,
 610                        cairo_test_compare_op_t *op,
 611                        int *major,
 612                        int *minor,
 613                        int *micro)
 614 {
 615     while (*str == ' ')
 616         str++;
 617
 618     if (strncmp (str, ">=", 2) == 0) {
 619         *op = GE;
 620         str += 2;
 621     } else if (strncmp (str, ">", 1) == 0) {
 622         *op = GT;
 623         str += 1;
 624     } else
 625         return FALSE;
 626
 627     while (*str == ' ')
 628         str++;
 629
 630     if (sscanf (str, "%d.%d.%d", major, minor, micro) != 3) {
 631         *micro = 0;
 632         if (sscanf (str, "%d.%d", major, minor) != 2)
 633             return FALSE;
 634     }
 635
 636     return TRUE;
 637 }
 638
 639 static cairo_bool_t
 640 _has_required_cairo_version (const char *str)
 641 {
 642     cairo_test_compare_op_t op;
 643     int major, minor, micro;
 644
 645     if (! _get_required_version (str + 5 /* advance over "cairo" */,
 646                                  &op, &major, &minor, &micro))
 647     {
 648         fprintf (stderr, "unrecognised cairo version requirement '%s'\n", str);
 649         return FALSE;
 650     }
 651
 652     return _version_compare (cairo_version (),
 653                              op,
 654                              CAIRO_VERSION_ENCODE (major, minor, micro));
 655 }
 656
 657 static cairo_bool_t
 658 _has_required_ghostscript_version (const char *str)
 659 {
 660 #if ! CAIRO_CAN_TEST_PS_SURFACE
 661     return TRUE;
 662 #endif
 663
 664     str += 2; /* advance over "gs" */
 665
 666     return TRUE;
 667 }
 668
 669 static cairo_bool_t
 670 _has_required_poppler_version (const char *str)
 671 {
 672 #if ! CAIRO_CAN_TEST_PDF_SURFACE
 673     return TRUE;
 674 #endif
 675
 676     str += 7; /* advance over "poppler" */
 677
 678     return TRUE;
 679 }
 680
 681 static cairo_bool_t
 682 _has_required_rsvg_version (const char *str)
 683 {
 684 #if ! CAIRO_CAN_TEST_SVG_SURFACE
 685     return TRUE;
 686 #endif
 687
 688     str += 4; /* advance over "rsvg" */
 689
 690     return TRUE;
 691 }
 692
 693 int
 694 main (int argc, char **argv)
 695 {
 696     cairo_test_runner_t runner;
 697     cairo_test_list_t *test_list;
 698     cairo_test_status_t *target_status;
 699     unsigned int n, m;
 700     char targets[4096];
 701     int len;
 702     char *cairo_tests_env;
 703
 704 #ifdef _MSC_VER
 705     /* We don't want an assert dialog, we want stderr */
 706     _CrtSetReportMode(_CRT_ERROR, _CRTDBG_MODE_FILE);
 707     _CrtSetReportFile(_CRT_ERROR, _CRTDBG_FILE_STDERR);
 708 #endif
 709
 710     _cairo_test_runner_register_tests ();
 711     tests = _list_reverse (tests);
 712
 713     memset (&runner, 0, sizeof (runner));
 714     runner.num_device_offsets = 1;
 715
 716     if (is_running_under_debugger ())
 717         runner.foreground = TRUE;
 718
 719     if (getenv ("CAIRO_TEST_MODE")) {
 720         const char *env = getenv ("CAIRO_TEST_MODE");
 721
 722         if (strstr (env, "full")) {
 723             runner.full_test = TRUE;
 724         }
 725         if (strstr (env, "foreground")) {
 726             runner.foreground = TRUE;
 727         }
 728         if (strstr (env, "exit-on-failure")) {
 729             runner.exit_on_failure = TRUE;
 730         }
 731     }
 732
 733     if (getenv ("CAIRO_TEST_FORCE_PASS")) {
 734         const char *env = getenv ("CAIRO_TEST_FORCE_PASS");
 735
 736         runner.force_pass = atoi (env);
 737     }
 738
 739     _parse_cmdline (&runner, &argc, &argv);
 740
 741     cairo_tests_env = getenv("CAIRO_TESTS");
 742     append_argv (&argc, &argv, cairo_tests_env);
 743
 744     if (runner.full_test) {
 745         runner.num_device_offsets = 2;
 746     }
 747
 748     target_status = NULL; /* silence the compiler */
 749     if (! runner.list_only) {
 750         _runner_init (&runner);
 751         _runner_print_versions (&runner);
 752         target_status = xmalloc (sizeof (cairo_test_status_t) *
 753                                  runner.base.num_targets);
 754     }
 755
 756     for (test_list = tests; test_list != NULL; test_list = test_list->next) {
 757         const cairo_test_t *test = test_list->test;
 758         cairo_test_context_t ctx;
 759         cairo_test_status_t status;
 760         cairo_bool_t failed = FALSE, xfailed = FALSE, error = FALSE, crashed = FALSE, skipped = TRUE;
 761         cairo_bool_t in_preamble = FALSE;
 762         char *name = cairo_test_get_name (test);
 763         int i;
 764
 765         /* check for restricted runs */
 766         if (argc) {
 767             cairo_bool_t found = FALSE;
 768             const char *keywords = test->keywords;
 769
 770             for (i = 0; i < argc; i++) {
 771                 const char *match = argv[i];
 772                 cairo_bool_t invert = match[0] == '!';
 773                 if (invert)
 774                     match++;
 775
 776                 if (runner.keyword_match) {
 777                     if (keywords != NULL && strstr (keywords, match) != NULL) {
 778                         found = ! invert;
 779                         break;
 780                     } else if (invert) {
 781                         found = TRUE;
 782                     }
 783                 } else {
 784                     /* exact match on test name */
 785                     if (strcmp (name, match) == 0) {
 786                         found = ! invert;
 787                         break;
 788                     } else if (invert) {
 789                         found = TRUE;
 790                     }
 791                 }
 792             }
 793
 794             if (! found) {
 795                 free (name);
 796                 continue;
 797             }
 798         }
 799
 800         /* check to see if external requirements match */
 801         if (test->requirements != NULL) {
 802             const char *requirements = test->requirements;
 803             const char *str;
 804
 805             str = strstr (requirements, "slow");
 806             if (str != NULL && ! runner.slow) {
 807                 if (runner.list_only)
 808                     goto TEST_NEXT;
 809                 else
 810                     goto TEST_SKIPPED;
 811             }
 812
 813             str = strstr (requirements, "cairo");
 814             if (str != NULL && ! _has_required_cairo_version (str)) {
 815                 if (runner.list_only)
 816                     goto TEST_NEXT;
 817                 else
 818                     goto TEST_SKIPPED;
 819             }
 820
 821             str = strstr (requirements, "gs");
 822             if (str != NULL && ! _has_required_ghostscript_version (str)) {
 823                 if (runner.list_only)
 824                     goto TEST_NEXT;
 825                 else
 826                     goto TEST_SKIPPED;
 827             }
 828
 829             str = strstr (requirements, "poppler");
 830             if (str != NULL && ! _has_required_poppler_version (str)) {
 831                 if (runner.list_only)
 832                     goto TEST_NEXT;
 833                 else
 834                     goto TEST_SKIPPED;
 835             }
 836
 837             str = strstr (requirements, "rsvg");
 838             if (str != NULL && ! _has_required_rsvg_version (str)) {
 839                 if (runner.list_only)
 840                     goto TEST_NEXT;
 841                 else
 842                     goto TEST_SKIPPED;
 843             }
 844         }
 845
 846         if (runner.list_only) {
 847             printf ("%s ", name);
 848             goto TEST_NEXT;
 849         }
 850
 851         _cairo_test_context_init_for_test (&ctx, &runner.base, test);
 852         memset (target_status, 0,
 853                 sizeof (cairo_test_status_t) * ctx.num_targets);
 854
 855         if (ctx.test->preamble != NULL) {
 856             status = _cairo_test_runner_preamble (&runner, &ctx);
 857             switch (status) {
 858             case CAIRO_TEST_SUCCESS:
 859                 in_preamble = TRUE;
 860                 skipped = FALSE;
 861                 break;
 862
 863             case CAIRO_TEST_XFAILURE:
 864                 in_preamble = TRUE;
 865                 xfailed = TRUE;
 866                 goto TEST_DONE;
 867
 868             case CAIRO_TEST_NEW:
 869             case CAIRO_TEST_FAILURE:
 870                 runner.fails_preamble = _list_prepend (runner.fails_preamble,
 871                                                        test);
 872                 in_preamble = TRUE;
 873                 failed = TRUE;
 874                 goto TEST_DONE;
 875
 876             case CAIRO_TEST_ERROR:
 877                 runner.errors_preamble = _list_prepend (runner.errors_preamble,
 878                                                          test);
 879                 in_preamble = TRUE;
 880                 failed = TRUE;
 881                 goto TEST_DONE;
 882
 883             case CAIRO_TEST_NO_MEMORY:
 884             case CAIRO_TEST_CRASHED:
 885                 runner.crashes_preamble = _list_prepend (runner.crashes_preamble,
 886                                                          test);
 887                 in_preamble = TRUE;
 888                 failed = TRUE;
 889                 goto TEST_DONE;
 890
 891             case CAIRO_TEST_UNTESTED:
 892                 goto TEST_DONE;
 893             }
 894         }
 895
 896         if (ctx.test->draw == NULL)
 897             goto TEST_DONE;
 898
 899         for (n = 0; n < ctx.num_targets; n++) {
 900             const cairo_boilerplate_target_t *target;
 901             cairo_bool_t target_failed = FALSE,
 902                          target_xfailed = FALSE,
 903                          target_error = FALSE,
 904                          target_crashed = FALSE,
 905                          target_skipped = TRUE;
 906             cairo_test_similar_t has_similar;
 907
 908             target = ctx.targets_to_test[n];
 909
 910             has_similar = runner.full_test ?
 911                           cairo_test_target_has_similar (&ctx, target) :
 912                           DIRECT;
 913             for (m = 0; m < runner.num_device_offsets; m++) {
 914                 int dev_offset = m * 25;
 915                 cairo_test_similar_t similar;
 916
 917                 for (similar = DIRECT; similar <= has_similar; similar++) {
 918                     status = _cairo_test_runner_draw (&runner, &ctx, target,
 919                                                       similar, dev_offset);
 920                     switch (status) {
 921                     case CAIRO_TEST_SUCCESS:
 922                         target_skipped = FALSE;
 923                         break;
 924                     case CAIRO_TEST_XFAILURE:
 925                         target_xfailed = TRUE;
 926                         break;
 927                     case CAIRO_TEST_NEW:
 928                     case CAIRO_TEST_FAILURE:
 929                         target_failed = TRUE;
 930                         break;
 931                     case CAIRO_TEST_ERROR:
 932                         target_error = TRUE;
 933                         break;
 934                     case CAIRO_TEST_NO_MEMORY:
 935                     case CAIRO_TEST_CRASHED:
 936                         target_crashed = TRUE;
 937                         break;
 938                     case CAIRO_TEST_UNTESTED:
 939                         break;
 940                     }
 941                 }
 942             }
 943
 944             if (target_crashed) {
 945                 target_status[n] = CAIRO_TEST_CRASHED;
 946                 runner.num_crashed_per_target[n]++;
 947                 runner.crashes_per_target[n] = _list_prepend (runner.crashes_per_target[n],
 948                                                               test);
 949                 crashed = TRUE;
 950             } else if (target_error) {
 951                 target_status[n] = CAIRO_TEST_ERROR;
 952                 runner.num_error_per_target[n]++;
 953                 runner.errors_per_target[n] = _list_prepend (runner.errors_per_target[n],
 954                                                              test);
 955
 956                 error = TRUE;
 957             } else if (target_failed) {
 958                 target_status[n] = CAIRO_TEST_FAILURE;
 959                 runner.num_failed_per_target[n]++;
 960                 runner.fails_per_target[n] = _list_prepend (runner.fails_per_target[n],
 961                                                             test);
 962
 963                 failed = TRUE;
 964             } else if (target_xfailed) {
 965                 target_status[n] = CAIRO_TEST_XFAILURE;
 966                 xfailed = TRUE;
 967             } else if (target_skipped) {
 968                 target_status[n] = CAIRO_TEST_UNTESTED;
 969             } else {
 970                 target_status[n] = CAIRO_TEST_SUCCESS;
 971                 skipped = FALSE;
 972             }
 973         }
 974
 975   TEST_DONE:
 976         cairo_test_fini (&ctx);
 977   TEST_SKIPPED:
 978         targets[0] = '\0';
 979         if (crashed) {
 980             if (! in_preamble) {
 981                 len = 0;
 982                 for (n = 0 ; n < runner.base.num_targets; n++) {
 983                     if (target_status[n] == CAIRO_TEST_CRASHED) {
 984                         if (strstr (targets,
 985                                     runner.base.targets_to_test[n]->name) == NULL)
 986                         {
 987                             len += snprintf (targets + len, sizeof (targets) - len,
 988                                              "%s, ",
 989                                              runner.base.targets_to_test[n]->name);
 990                         }
 991                     }
 992                 }
 993                 targets[len-2] = '\0';
 994                 _log (&runner.base, "\n%s: CRASH! (%s)\n", name, targets);
 995             } else {
 996                 _log (&runner.base, "\n%s: CRASH!\n", name);
 997             }
 998             runner.num_crashed++;
 999             runner.passed = FALSE;
1000         } else if (error) {
1001             if (! in_preamble) {
1002                 len = 0;
1003                 for (n = 0 ; n < runner.base.num_targets; n++) {
1004                     if (target_status[n] == CAIRO_TEST_ERROR) {
1005                         if (strstr (targets,
1006                                     runner.base.targets_to_test[n]->name) == NULL)
1007                         {
1008                             len += snprintf (targets + len,
1009                                              sizeof (targets) - len,
1010                                              "%s, ",
1011                                              runner.base.targets_to_test[n]->name);
1012                         }
1013                     }
1014                 }
1015                 targets[len-2] = '\0';
1016                 _log (&runner.base, "%s: ERROR (%s)\n", name, targets);
1017             } else {
1018                 _log (&runner.base, "%s: ERROR\n", name);
1019             }
1020             runner.num_error++;
1021             runner.passed = FALSE;
1022         } else if (failed) {
1023             if (! in_preamble) {
1024                 len = 0;
1025                 for (n = 0 ; n < runner.base.num_targets; n++) {
1026                     if (target_status[n] == CAIRO_TEST_FAILURE) {
1027                         if (strstr (targets,
1028                                     runner.base.targets_to_test[n]->name) == NULL)
1029                         {
1030                             len += snprintf (targets + len,
1031                                              sizeof (targets) - len,
1032                                              "%s, ",
1033                                              runner.base.targets_to_test[n]->name);
1034                         }
1035                     }
1036                 }
1037                 targets[len-2] = '\0';
1038                 _log (&runner.base, "%s: FAIL (%s)\n", name, targets);
1039             } else {
1040                 _log (&runner.base, "%s: FAIL\n", name);
1041             }
1042             runner.num_failed++;
1043             runner.passed = FALSE;
1044         } else if (xfailed) {
1045             _log (&runner.base, "%s: XFAIL\n", name);
1046             runner.num_xfailed++;
1047         } else if (skipped) {
1048             _log (&runner.base, "%s: UNTESTED\n", name);
1049             runner.num_skipped++;
1050         } else {
1051             _log (&runner.base, "%s: PASS\n", name);
1052             runner.num_passed++;
1053         }
1054         fflush (runner.base.log_file);
1055
1056   TEST_NEXT:
1057         free (name);
1058         if (runner.exit_on_failure && ! runner.passed)
1059             break;
1060
1061     }
1062
1063     if (cairo_tests_env)
1064         free(argv);
1065
1066     if (runner.list_only) {
1067         printf ("\n");
1068         return CAIRO_TEST_SUCCESS;
1069     }
1070
1071     for (n = 0 ; n < runner.base.num_targets; n++) {
1072         runner.crashes_per_target[n] = _list_reverse (runner.crashes_per_target[n]);
1073         runner.errors_per_target[n] = _list_reverse (runner.errors_per_target[n]);
1074         runner.fails_per_target[n] = _list_reverse (runner.fails_per_target[n]);
1075     }
1076
1077     _runner_print_results (&runner);
1078
1079     free (target_status);
1080     return _runner_fini (&runner);
1081 }
1082
1083 void
1084 cairo_test_register (cairo_test_t *test)
1085 {
1086     tests = _list_prepend (tests, test);
1087 }