test/cairo-test-runner.c

   1 /*
   2  * Copyright © 2008 Chris Wilson
   3  *
   4  * Permission to use, copy, modify, distribute, and sell this software
   5  * and its documentation for any purpose is hereby granted without
   6  * fee, provided that the above copyright notice appear in all copies
   7  * and that both that copyright notice and this permission notice
   8  * appear in supporting documentation, and that the name of
   9  * Chris Wilson not be used in advertising or publicity pertaining to
  10  * distribution of the software without specific, written prior
  11  * permission. Chris Wilson makes no representations about the
  12  * suitability of this software for any purpose.  It is provided "as
  13  * is" without express or implied warranty.
  14  *
  15  * CHRIS WILSON DISCLAIMS ALL WARRANTIES WITH REGARD TO THIS
  16  * SOFTWARE, INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY AND
  17  * FITNESS, IN NO EVENT SHALL CHRIS WILSON BE LIABLE FOR ANY SPECIAL,
  18  * INDIRECT OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER
  19  * RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION
  20  * OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR
  21  * IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
  22  *
  23  * Author: Chris Wilson <chris@chris-wilson.co.uk>
  24  */
  25
  26 #include "cairo-test-private.h"
  27 #include "cairo-boilerplate-getopt.h"
  28
  29 /* get the "real" version info instead of dummy cairo-version.h */
  30 #undef CAIRO_VERSION_H
  31 #undef CAIRO_VERSION_MAJOR
  32 #undef CAIRO_VERSION_MINOR
  33 #undef CAIRO_VERSION_MICRO
  34 #include "../cairo-version.h"
  35
  36 #include <pixman.h> /* for version information */
  37
  38 #define SHOULD_FORK HAVE_FORK && HAVE_WAITPID
  39 #if SHOULD_FORK
  40 #if HAVE_UNISTD_H
  41 #include <unistd.h>
  42 #endif
  43 #if HAVE_SIGNAL_H
  44 #include <signal.h>
  45 #endif
  46 #include <sys/types.h>
  47 #include <sys/wait.h>
  48 #endif
  49 #if HAVE_LIBGEN_H
  50 #include <libgen.h>
  51 #endif
  52
  53 #if HAVE_VALGRIND
  54 #include <valgrind.h>
  55 #else
  56 #define RUNNING_ON_VALGRIND 0
  57 #endif
  58
  59 #ifdef _MSC_VER
  60 #include <crtdbg.h>
  61 #endif
  62
  63 typedef struct _cairo_test_list {
  64     const cairo_test_t *test;
  65     struct _cairo_test_list *next;
  66 } cairo_test_list_t;
  67
  68 typedef struct _cairo_test_runner {
  69     cairo_test_context_t base;
  70
  71     unsigned int num_device_offsets;
  72     unsigned int num_device_scales;
  73
  74     cairo_bool_t passed;
  75     int num_passed;
  76     int num_skipped;
  77     int num_failed;
  78     int num_xfailed;
  79     int num_error;
  80     int num_crashed;
  81
  82     cairo_test_list_t *crashes_preamble;
  83     cairo_test_list_t *errors_preamble;
  84     cairo_test_list_t *fails_preamble;
  85
  86     cairo_test_list_t **crashes_per_target;
  87     cairo_test_list_t **errors_per_target;
  88     cairo_test_list_t **fails_per_target;
  89
  90     int *num_failed_per_target;
  91     int *num_error_per_target;
  92     int *num_crashed_per_target;
  93
  94     cairo_bool_t foreground;
  95     cairo_bool_t exit_on_failure;
  96     cairo_bool_t list_only;
  97     cairo_bool_t full_test;
  98     cairo_bool_t keyword_match;
  99     cairo_bool_t slow;
 100     cairo_bool_t force_pass;
 101 } cairo_test_runner_t;
 102
 103 typedef enum {
 104     GE,
 105     GT
 106 } cairo_test_compare_op_t;
 107
 108 static cairo_test_list_t *tests;
 109
 110 static void CAIRO_BOILERPLATE_PRINTF_FORMAT(2,3)
 111 _log (cairo_test_context_t *ctx,
 112       const char *fmt,
 113       ...)
 114 {
 115     va_list ap;
 116
 117     va_start (ap, fmt);
 118     vprintf (fmt, ap);
 119     va_end (ap);
 120
 121     va_start (ap, fmt);
 122     cairo_test_logv (ctx, fmt, ap);
 123     va_end (ap);
 124 }
 125
 126 static cairo_test_list_t *
 127 _list_prepend (cairo_test_list_t *head, const cairo_test_t *test)
 128 {
 129     cairo_test_list_t *list;
 130
 131     list = xmalloc (sizeof (cairo_test_list_t));
 132     list->test = test;
 133     list->next = head;
 134     head = list;
 135
 136     return head;
 137 }
 138
 139 static cairo_test_list_t *
 140 _list_reverse (cairo_test_list_t *head)
 141 {
 142     cairo_test_list_t *list, *next;
 143
 144     for (list = head, head = NULL; list != NULL; list = next) {
 145         next = list->next;
 146         list->next = head;
 147         head = list;
 148     }
 149
 150     return head;
 151 }
 152
 153 static void
 154 _list_free (cairo_test_list_t *list)
 155 {
 156     while (list != NULL) {
 157         cairo_test_list_t *next = list->next;
 158         free (list);
 159         list = next;
 160     }
 161 }
 162
 163 static cairo_bool_t
 164 is_running_under_debugger (void)
 165 {
 166 #if HAVE_UNISTD_H && HAVE_LIBGEN_H && __linux__
 167     char buf[1024];
 168
 169     sprintf (buf, "/proc/%d/exe", getppid ());
 170     if (readlink (buf, buf, sizeof (buf)) != -1 &&
 171         strncmp (basename (buf), "gdb", 3) == 0)
 172     {
 173         return TRUE;
 174     }
 175 #endif
 176
 177     if (RUNNING_ON_VALGRIND)
 178         return TRUE;
 179
 180     return FALSE;
 181 }
 182
 183 #if SHOULD_FORK
 184 static cairo_test_status_t
 185 _cairo_test_wait (pid_t pid)
 186 {
 187     int exitcode;
 188
 189     if (waitpid (pid, &exitcode, 0) != pid)
 190         return CAIRO_TEST_CRASHED;
 191
 192     if (WIFSIGNALED (exitcode)) {
 193         switch (WTERMSIG (exitcode)) {
 194         case SIGINT:
 195 #if HAVE_RAISE
 196             raise (SIGINT);
 197 #endif
 198             return CAIRO_TEST_UNTESTED;
 199         default:
 200             return CAIRO_TEST_CRASHED;
 201         }
 202     }
 203
 204     return WEXITSTATUS (exitcode);
 205 }
 206 #endif
 207
 208 static cairo_test_status_t
 209 _cairo_test_runner_preamble (cairo_test_runner_t *runner,
 210                              cairo_test_context_t *ctx)
 211 {
 212 #if SHOULD_FORK
 213     if (! runner->foreground) {
 214         pid_t pid;
 215
 216         switch ((pid = fork ())) {
 217         case -1: /* error */
 218             return CAIRO_TEST_UNTESTED;
 219
 220         case 0: /* child */
 221             exit (ctx->test->preamble (ctx));
 222
 223         default:
 224             return _cairo_test_wait (pid);
 225         }
 226     }
 227 #endif
 228     return ctx->test->preamble (ctx);
 229 }
 230
 231 static cairo_test_status_t
 232 _cairo_test_runner_draw (cairo_test_runner_t *runner,
 233                          cairo_test_context_t *ctx,
 234                          const cairo_boilerplate_target_t *target,
 235                          cairo_bool_t similar,
 236                          int device_offset, int device_scale)
 237 {
 238 #if SHOULD_FORK
 239     if (! runner->foreground) {
 240         pid_t pid;
 241
 242         switch ((pid = fork ())) {
 243         case -1: /* error */
 244             return CAIRO_TEST_UNTESTED;
 245
 246         case 0: /* child */
 247             exit (_cairo_test_context_run_for_target (ctx, target,
 248                                                       similar, device_offset, device_scale));
 249
 250         default:
 251             return _cairo_test_wait (pid);
 252         }
 253     }
 254 #endif
 255     return _cairo_test_context_run_for_target (ctx, target,
 256                                                similar, device_offset, device_scale);
 257 }
 258
 259 static void
 260 append_argv (int *argc, char ***argv, const char *str)
 261 {
 262     int old_argc;
 263     char **old_argv;
 264     cairo_bool_t doit;
 265     const char *s, *t;
 266     int olen;
 267     int len;
 268     int i;
 269     int args_to_add = 0;
 270
 271     if (str == NULL)
 272         return;
 273
 274     old_argc = *argc;
 275     old_argv = *argv;
 276
 277     doit = FALSE;
 278     do {
 279         if (doit)
 280             *argv = xmalloc (olen);
 281
 282         olen = sizeof (char *) * (args_to_add + *argc);
 283         for (i = 0; i < old_argc; i++) {
 284             len = strlen (old_argv[i]) + 1;
 285             if (doit) {
 286                 (*argv)[i] = (char *) *argv + olen;
 287                 memcpy ((*argv)[i], old_argv[i], len);
 288             }
 289             olen += len;
 290         }
 291
 292         s = str;
 293         while ((t = strpbrk (s, " \t,:;")) != NULL) {
 294             if (t - s) {
 295                 len = t - s;
 296                 if (doit) {
 297                     (*argv)[i] = (char *) *argv + olen;
 298                     memcpy ((*argv)[i], s, len);
 299                     (*argv)[i][len] = '\0';
 300                 } else {
 301                     olen += sizeof (char *);
 302                 }
 303                 args_to_add++;
 304                 olen += len + 1;
 305                 i++;
 306             }
 307             s = t + 1;
 308         }
 309         if (*s != '\0') {
 310             len = strlen (s) + 1;
 311             if (doit) {
 312                 (*argv)[i] = (char *) *argv + olen;
 313                 memcpy ((*argv)[i], s, len);
 314             } else {
 315                 olen += sizeof (char *);
 316             }
 317             args_to_add++;
 318             olen += len;
 319             i++;
 320         }
 321     } while (doit++ == FALSE);
 322     *argc = i;
 323 }
 324
 325 static void
 326 usage (const char *argv0)
 327 {
 328     fprintf (stderr,
 329              "Usage: %s [-afkxsl] [test-names|keywords ...]\n"
 330              "\n"
 331              "Run the cairo conformance test suite over the given tests (all by default)\n"
 332              "The command-line arguments are interpreted as follows:\n"
 333              "\n"
 334              "  -a      all; run the full set of tests. By default the test suite\n"
 335              "          skips similar surface and device offset testing.\n"
 336              "  -f      foreground; do not fork\n"
 337              "  -k      match tests by keyword\n"
 338              "  -l      list only; just list selected test case names without executing\n"
 339              "  -s      include slow, long running tests\n"
 340              "  -x      exit on first failure\n"
 341              "\n"
 342              "If test names are given they are used as matches either to a specific\n"
 343              "test case or to a keyword, so a command such as\n"
 344              "\"%s -k text\" can be used to run all text test cases, and\n"
 345              "\"%s text-transform\" to run the individual case.\n",
 346              argv0, argv0, argv0);
 347 }
 348
 349 static void
 350 _parse_cmdline (cairo_test_runner_t *runner, int *argc, char **argv[])
 351 {
 352     int c;
 353
 354     while (1) {
 355         c = _cairo_getopt (*argc, *argv, ":afklsx");
 356         if (c == -1)
 357             break;
 358
 359         switch (c) {
 360         case 'a':
 361             runner->full_test = ~0;
 362             break;
 363         case 'f':
 364             runner->foreground = TRUE;
 365             break;
 366         case 'k':
 367             runner->keyword_match = TRUE;
 368             break;
 369         case 'l':
 370             runner->list_only = TRUE;
 371             break;
 372         case 's':
 373             runner->slow = TRUE;
 374             break;
 375         case 'x':
 376             runner->exit_on_failure = TRUE;
 377             break;
 378         default:
 379             fprintf (stderr, "Internal error: unhandled option: %c\n", c);
 380             /* fall-through */
 381         case '?':
 382             usage ((*argv)[0]);
 383             exit (1);
 384         }
 385     }
 386
 387     *argc -= optind;
 388     *argv += optind;
 389 }
 390
 391 static void
 392 _runner_init (cairo_test_runner_t *runner)
 393 {
 394     cairo_test_init (&runner->base, "cairo-test-suite", ".");
 395
 396     runner->passed = TRUE;
 397
 398     runner->fails_preamble = NULL;
 399     runner->crashes_preamble = NULL;
 400     runner->errors_preamble = NULL;
 401
 402     runner->fails_per_target = xcalloc (sizeof (cairo_test_list_t *),
 403                                         runner->base.num_targets);
 404     runner->crashes_per_target = xcalloc (sizeof (cairo_test_list_t *),
 405                                           runner->base.num_targets);
 406     runner->errors_per_target = xcalloc (sizeof (cairo_test_list_t *),
 407                                           runner->base.num_targets);
 408     runner->num_failed_per_target = xcalloc (sizeof (int),
 409                                              runner->base.num_targets);
 410     runner->num_error_per_target = xcalloc (sizeof (int),
 411                                              runner->base.num_targets);
 412     runner->num_crashed_per_target = xcalloc (sizeof (int),
 413                                               runner->base.num_targets);
 414 }
 415
 416 static void
 417 _runner_print_versions (cairo_test_runner_t *runner)
 418 {
 419     _log (&runner->base,
 420          "Compiled against cairo %s, running on %s.\n",
 421          CAIRO_VERSION_STRING, cairo_version_string ());
 422     _log (&runner->base,
 423          "Compiled against pixman %s, running on %s.\n",
 424          PIXMAN_VERSION_STRING, pixman_version_string ());
 425
 426     fflush (runner->base.log_file);
 427 }
 428
 429 static void
 430 _runner_print_summary (cairo_test_runner_t *runner)
 431 {
 432     _log (&runner->base,
 433           "%d Passed, %d Failed [%d crashed, %d expected], %d Skipped\n",
 434           runner->num_passed,
 435
 436           runner->num_failed + runner->num_crashed + runner->num_xfailed,
 437           runner->num_crashed,
 438           runner->num_xfailed,
 439
 440           runner->num_skipped);
 441 }
 442
 443 static void
 444 _runner_print_details (cairo_test_runner_t *runner)
 445 {
 446     cairo_test_list_t *list;
 447     unsigned int n;
 448
 449     if (runner->crashes_preamble) {
 450         int count = 0;
 451
 452         for (list = runner->crashes_preamble; list != NULL; list = list->next)
 453             count++;
 454
 455         _log (&runner->base, "Preamble: %d crashed! -", count);
 456
 457         for (list = runner->crashes_preamble; list != NULL; list = list->next) {
 458             char *name = cairo_test_get_name (list->test);
 459             _log (&runner->base, " %s", name);
 460             free (name);
 461         }
 462         _log (&runner->base, "\n");
 463     }
 464     if (runner->errors_preamble) {
 465         int count = 0;
 466
 467         for (list = runner->errors_preamble; list != NULL; list = list->next)
 468             count++;
 469
 470         _log (&runner->base, "Preamble: %d error -", count);
 471
 472         for (list = runner->errors_preamble; list != NULL; list = list->next) {
 473             char *name = cairo_test_get_name (list->test);
 474             _log (&runner->base, " %s", name);
 475             free (name);
 476         }
 477         _log (&runner->base, "\n");
 478     }
 479     if (runner->fails_preamble) {
 480         int count = 0;
 481
 482         for (list = runner->fails_preamble; list != NULL; list = list->next)
 483             count++;
 484
 485         _log (&runner->base, "Preamble: %d failed -", count);
 486
 487         for (list = runner->fails_preamble; list != NULL; list = list->next) {
 488             char *name = cairo_test_get_name (list->test);
 489             _log (&runner->base, " %s", name);
 490             free (name);
 491         }
 492         _log (&runner->base, "\n");
 493     }
 494
 495     for (n = 0; n < runner->base.num_targets; n++) {
 496         const cairo_boilerplate_target_t *target;
 497
 498         target = runner->base.targets_to_test[n];
 499         if (runner->num_crashed_per_target[n]) {
 500             _log (&runner->base, "%s (%s): %d crashed! -",
 501                   target->name,
 502                   cairo_boilerplate_content_name (target->content),
 503                   runner->num_crashed_per_target[n]);
 504
 505             for (list = runner->crashes_per_target[n];
 506                  list != NULL;
 507                  list = list->next)
 508             {
 509                 char *name = cairo_test_get_name (list->test);
 510                 _log (&runner->base, " %s", name);
 511                 free (name);
 512             }
 513             _log (&runner->base, "\n");
 514         }
 515         if (runner->num_error_per_target[n]) {
 516             _log (&runner->base, "%s (%s): %d error -",
 517                   target->name,
 518                   cairo_boilerplate_content_name (target->content),
 519                   runner->num_error_per_target[n]);
 520
 521             for (list = runner->errors_per_target[n];
 522                  list != NULL;
 523                  list = list->next)
 524             {
 525                 char *name = cairo_test_get_name (list->test);
 526                 _log (&runner->base, " %s", name);
 527                 free (name);
 528             }
 529             _log (&runner->base, "\n");
 530         }
 531
 532         if (runner->num_failed_per_target[n]) {
 533             _log (&runner->base, "%s (%s): %d failed -",
 534                   target->name,
 535                   cairo_boilerplate_content_name (target->content),
 536                   runner->num_failed_per_target[n]);
 537
 538             for (list = runner->fails_per_target[n];
 539                  list != NULL;
 540                  list = list->next)
 541             {
 542                 char *name = cairo_test_get_name (list->test);
 543                 _log (&runner->base, " %s", name);
 544                 free (name);
 545             }
 546             _log (&runner->base, "\n");
 547         }
 548     }
 549 }
 550
 551 static void
 552 _runner_print_results (cairo_test_runner_t *runner)
 553 {
 554     _runner_print_summary (runner);
 555     _runner_print_details (runner);
 556
 557     if (! runner->passed && ! runner->num_crashed) {
 558         _log (&runner->base,
 559 "\n"
 560 "Note: These failures may be due to external factors.\n"
 561 "Please read test/README -- \"Getting the elusive zero failures\".\n");
 562     }
 563 }
 564
 565 static cairo_test_status_t
 566 _runner_fini (cairo_test_runner_t *runner)
 567 {
 568     unsigned int n;
 569
 570     _list_free (runner->crashes_preamble);
 571     _list_free (runner->errors_preamble);
 572     _list_free (runner->fails_preamble);
 573
 574     for (n = 0; n < runner->base.num_targets; n++) {
 575         _list_free (runner->crashes_per_target[n]);
 576         _list_free (runner->errors_per_target[n]);
 577         _list_free (runner->fails_per_target[n]);
 578     }
 579     free (runner->crashes_per_target);
 580     free (runner->errors_per_target);
 581     free (runner->fails_per_target);
 582
 583     free (runner->num_crashed_per_target);
 584     free (runner->num_error_per_target);
 585     free (runner->num_failed_per_target);
 586
 587     cairo_test_fini (&runner->base);
 588
 589     if (runner->force_pass)
 590         return CAIRO_TEST_SUCCESS;
 591
 592     return runner->num_failed + runner->num_crashed ?
 593         CAIRO_TEST_FAILURE :
 594         runner->num_passed + runner->num_xfailed ?
 595         CAIRO_TEST_SUCCESS : CAIRO_TEST_UNTESTED;
 596 }
 597
 598 static cairo_bool_t
 599 _version_compare (int a, cairo_test_compare_op_t op, int b)
 600 {
 601     switch (op) {
 602     case GT: return a > b;
 603     case GE: return a >= b;
 604     default: return FALSE;
 605     }
 606 }
 607
 608
 609 static cairo_bool_t
 610 _get_required_version (const char *str,
 611                        cairo_test_compare_op_t *op,
 612                        int *major,
 613                        int *minor,
 614                        int *micro)
 615 {
 616     while (*str == ' ')
 617         str++;
 618
 619     if (strncmp (str, ">=", 2) == 0) {
 620         *op = GE;
 621         str += 2;
 622     } else if (strncmp (str, ">", 1) == 0) {
 623         *op = GT;
 624         str += 1;
 625     } else
 626         return FALSE;
 627
 628     while (*str == ' ')
 629         str++;
 630
 631     if (sscanf (str, "%d.%d.%d", major, minor, micro) != 3) {
 632         *micro = 0;
 633         if (sscanf (str, "%d.%d", major, minor) != 2)
 634             return FALSE;
 635     }
 636
 637     return TRUE;
 638 }
 639
 640 static cairo_bool_t
 641 _has_required_cairo_version (const char *str)
 642 {
 643     cairo_test_compare_op_t op;
 644     int major, minor, micro;
 645
 646     if (! _get_required_version (str + 5 /* advance over "cairo" */,
 647                                  &op, &major, &minor, &micro))
 648     {
 649         fprintf (stderr, "unrecognised cairo version requirement '%s'\n", str);
 650         return FALSE;
 651     }
 652
 653     return _version_compare (cairo_version (),
 654                              op,
 655                              CAIRO_VERSION_ENCODE (major, minor, micro));
 656 }
 657
 658 static cairo_bool_t
 659 _has_required_ghostscript_version (const char *str)
 660 {
 661 #if ! CAIRO_CAN_TEST_PS_SURFACE
 662     return TRUE;
 663 #endif
 664
 665     str += 2; /* advance over "gs" */
 666
 667     return TRUE;
 668 }
 669
 670 static cairo_bool_t
 671 _has_required_poppler_version (const char *str)
 672 {
 673 #if ! CAIRO_CAN_TEST_PDF_SURFACE
 674     return TRUE;
 675 #endif
 676
 677     str += 7; /* advance over "poppler" */
 678
 679     return TRUE;
 680 }
 681
 682 static cairo_bool_t
 683 _has_required_rsvg_version (const char *str)
 684 {
 685 #if ! CAIRO_CAN_TEST_SVG_SURFACE
 686     return TRUE;
 687 #endif
 688
 689     str += 4; /* advance over "rsvg" */
 690
 691     return TRUE;
 692 }
 693
 694 #define TEST_SIMILAR    0x1
 695 #define TEST_OFFSET     0x2
 696 #define TEST_SCALE      0x4
 697 int
 698 main (int argc, char **argv)
 699 {
 700     cairo_test_runner_t runner;
 701     cairo_test_list_t *test_list;
 702     cairo_test_status_t *target_status;
 703     unsigned int n, m, k;
 704     char targets[4096];
 705     int len;
 706     char *cairo_tests_env;
 707
 708 #ifdef _MSC_VER
 709     /* We don't want an assert dialog, we want stderr */
 710     _CrtSetReportMode(_CRT_ERROR, _CRTDBG_MODE_FILE);
 711     _CrtSetReportFile(_CRT_ERROR, _CRTDBG_FILE_STDERR);
 712 #endif
 713
 714     _cairo_test_runner_register_tests ();
 715     tests = _list_reverse (tests);
 716
 717     memset (&runner, 0, sizeof (runner));
 718     runner.num_device_offsets = 1;
 719     runner.num_device_scales = 1;
 720
 721     if (is_running_under_debugger ())
 722         runner.foreground = TRUE;
 723
 724     if (getenv ("CAIRO_TEST_MODE")) {
 725         const char *env = getenv ("CAIRO_TEST_MODE");
 726
 727         if (strstr (env, "full")) {
 728             runner.full_test = ~0;
 729         }
 730         if (strstr (env, "similar")) {
 731             runner.full_test |= TEST_SIMILAR;
 732         }
 733         if (strstr (env, "offset")) {
 734             runner.full_test |= TEST_OFFSET;
 735         }
 736         if (strstr (env, "scale")) {
 737             runner.full_test |= TEST_SCALE;
 738         }
 739         if (strstr (env, "foreground")) {
 740             runner.foreground = TRUE;
 741         }
 742         if (strstr (env, "exit-on-failure")) {
 743             runner.exit_on_failure = TRUE;
 744         }
 745     }
 746
 747     if (getenv ("CAIRO_TEST_FORCE_PASS")) {
 748         const char *env = getenv ("CAIRO_TEST_FORCE_PASS");
 749
 750         runner.force_pass = atoi (env);
 751     }
 752
 753     _parse_cmdline (&runner, &argc, &argv);
 754
 755     cairo_tests_env = getenv("CAIRO_TESTS");
 756     append_argv (&argc, &argv, cairo_tests_env);
 757
 758     if (runner.full_test & TEST_OFFSET) {
 759         runner.num_device_offsets = 2;
 760     }
 761     if (runner.full_test & TEST_SCALE) {
 762         runner.num_device_scales = 2;
 763     }
 764
 765     target_status = NULL; /* silence the compiler */
 766     if (! runner.list_only) {
 767         _runner_init (&runner);
 768         _runner_print_versions (&runner);
 769         target_status = xmalloc (sizeof (cairo_test_status_t) *
 770                                  runner.base.num_targets);
 771     }
 772
 773     for (test_list = tests; test_list != NULL; test_list = test_list->next) {
 774         const cairo_test_t *test = test_list->test;
 775         cairo_test_context_t ctx;
 776         cairo_test_status_t status;
 777         cairo_bool_t failed = FALSE, xfailed = FALSE, error = FALSE, crashed = FALSE, skipped = TRUE;
 778         cairo_bool_t in_preamble = FALSE;
 779         char *name = cairo_test_get_name (test);
 780         int i;
 781
 782         /* check for restricted runs */
 783         if (argc) {
 784             cairo_bool_t found = FALSE;
 785             const char *keywords = test->keywords;
 786
 787             for (i = 0; i < argc; i++) {
 788                 const char *match = argv[i];
 789                 cairo_bool_t invert = match[0] == '!';
 790                 if (invert)
 791                     match++;
 792
 793                 if (runner.keyword_match) {
 794                     if (keywords != NULL && strstr (keywords, match) != NULL) {
 795                         found = ! invert;
 796                         break;
 797                     } else if (invert) {
 798                         found = TRUE;
 799                     }
 800                 } else {
 801                     /* exact match on test name */
 802                     if (strcmp (name, match) == 0) {
 803                         found = ! invert;
 804                         break;
 805                     } else if (invert) {
 806                         found = TRUE;
 807                     }
 808                 }
 809             }
 810
 811             if (! found) {
 812                 free (name);
 813                 continue;
 814             }
 815         }
 816
 817         /* check to see if external requirements match */
 818         if (test->requirements != NULL) {
 819             const char *requirements = test->requirements;
 820             const char *str;
 821
 822             str = strstr (requirements, "slow");
 823             if (str != NULL && ! runner.slow) {
 824                 if (runner.list_only)
 825                     goto TEST_NEXT;
 826                 else
 827                     goto TEST_SKIPPED;
 828             }
 829
 830             str = strstr (requirements, "cairo");
 831             if (str != NULL && ! _has_required_cairo_version (str)) {
 832                 if (runner.list_only)
 833                     goto TEST_NEXT;
 834                 else
 835                     goto TEST_SKIPPED;
 836             }
 837
 838             str = strstr (requirements, "gs");
 839             if (str != NULL && ! _has_required_ghostscript_version (str)) {
 840                 if (runner.list_only)
 841                     goto TEST_NEXT;
 842                 else
 843                     goto TEST_SKIPPED;
 844             }
 845
 846             str = strstr (requirements, "poppler");
 847             if (str != NULL && ! _has_required_poppler_version (str)) {
 848                 if (runner.list_only)
 849                     goto TEST_NEXT;
 850                 else
 851                     goto TEST_SKIPPED;
 852             }
 853
 854             str = strstr (requirements, "rsvg");
 855             if (str != NULL && ! _has_required_rsvg_version (str)) {
 856                 if (runner.list_only)
 857                     goto TEST_NEXT;
 858                 else
 859                     goto TEST_SKIPPED;
 860             }
 861         }
 862
 863         if (runner.list_only) {
 864             printf ("%s ", name);
 865             goto TEST_NEXT;
 866         }
 867
 868         _cairo_test_context_init_for_test (&ctx, &runner.base, test);
 869         memset (target_status, 0,
 870                 sizeof (cairo_test_status_t) * ctx.num_targets);
 871
 872         if (ctx.test->preamble != NULL) {
 873             status = _cairo_test_runner_preamble (&runner, &ctx);
 874             switch (status) {
 875             case CAIRO_TEST_SUCCESS:
 876                 in_preamble = TRUE;
 877                 skipped = FALSE;
 878                 break;
 879
 880             case CAIRO_TEST_XFAILURE:
 881                 in_preamble = TRUE;
 882                 xfailed = TRUE;
 883                 goto TEST_DONE;
 884
 885             case CAIRO_TEST_NEW:
 886             case CAIRO_TEST_FAILURE:
 887                 runner.fails_preamble = _list_prepend (runner.fails_preamble,
 888                                                        test);
 889                 in_preamble = TRUE;
 890                 failed = TRUE;
 891                 goto TEST_DONE;
 892
 893             case CAIRO_TEST_ERROR:
 894                 runner.errors_preamble = _list_prepend (runner.errors_preamble,
 895                                                          test);
 896                 in_preamble = TRUE;
 897                 failed = TRUE;
 898                 goto TEST_DONE;
 899
 900             case CAIRO_TEST_NO_MEMORY:
 901             case CAIRO_TEST_CRASHED:
 902                 runner.crashes_preamble = _list_prepend (runner.crashes_preamble,
 903                                                          test);
 904                 in_preamble = TRUE;
 905                 failed = TRUE;
 906                 goto TEST_DONE;
 907
 908             case CAIRO_TEST_UNTESTED:
 909                 goto TEST_DONE;
 910             }
 911         }
 912
 913         if (ctx.test->draw == NULL)
 914             goto TEST_DONE;
 915
 916         for (n = 0; n < ctx.num_targets; n++) {
 917             const cairo_boilerplate_target_t *target;
 918             cairo_bool_t target_failed = FALSE,
 919                          target_xfailed = FALSE,
 920                          target_error = FALSE,
 921                          target_crashed = FALSE,
 922                          target_skipped = TRUE;
 923             cairo_test_similar_t has_similar;
 924
 925             target = ctx.targets_to_test[n];
 926
 927             has_similar = runner.full_test & TEST_SIMILAR ?
 928                           cairo_test_target_has_similar (&ctx, target) :
 929                           DIRECT;
 930             for (m = 0; m < runner.num_device_offsets; m++) {
 931                 for (k = 0; k < runner.num_device_scales; k++) {
 932                     int dev_offset = m * 25;
 933                     int dev_scale = k + 1;
 934                     cairo_test_similar_t similar;
 935
 936                     for (similar = DIRECT; similar <= has_similar; similar++) {
 937                         status = _cairo_test_runner_draw (&runner, &ctx, target,
 938                                                           similar, dev_offset, dev_scale);
 939                         switch (status) {
 940                         case CAIRO_TEST_SUCCESS:
 941                             target_skipped = FALSE;
 942                             break;
 943                         case CAIRO_TEST_XFAILURE:
 944                             target_xfailed = TRUE;
 945                             break;
 946                         case CAIRO_TEST_NEW:
 947                         case CAIRO_TEST_FAILURE:
 948                             target_failed = TRUE;
 949                             break;
 950                         case CAIRO_TEST_ERROR:
 951                             target_error = TRUE;
 952                             break;
 953                         case CAIRO_TEST_NO_MEMORY:
 954                         case CAIRO_TEST_CRASHED:
 955                             target_crashed = TRUE;
 956                             break;
 957                         case CAIRO_TEST_UNTESTED:
 958                             break;
 959                         }
 960                     }
 961                 }
 962             }
 963
 964             if (target_crashed) {
 965                 target_status[n] = CAIRO_TEST_CRASHED;
 966                 runner.num_crashed_per_target[n]++;
 967                 runner.crashes_per_target[n] = _list_prepend (runner.crashes_per_target[n],
 968                                                               test);
 969                 crashed = TRUE;
 970             } else if (target_error) {
 971                 target_status[n] = CAIRO_TEST_ERROR;
 972                 runner.num_error_per_target[n]++;
 973                 runner.errors_per_target[n] = _list_prepend (runner.errors_per_target[n],
 974                                                              test);
 975
 976                 error = TRUE;
 977             } else if (target_failed) {
 978                 target_status[n] = CAIRO_TEST_FAILURE;
 979                 runner.num_failed_per_target[n]++;
 980                 runner.fails_per_target[n] = _list_prepend (runner.fails_per_target[n],
 981                                                             test);
 982
 983                 failed = TRUE;
 984             } else if (target_xfailed) {
 985                 target_status[n] = CAIRO_TEST_XFAILURE;
 986                 xfailed = TRUE;
 987             } else if (target_skipped) {
 988                 target_status[n] = CAIRO_TEST_UNTESTED;
 989             } else {
 990                 target_status[n] = CAIRO_TEST_SUCCESS;
 991                 skipped = FALSE;
 992             }
 993         }
 994
 995   TEST_DONE:
 996         cairo_test_fini (&ctx);
 997   TEST_SKIPPED:
 998         targets[0] = '\0';
 999         if (crashed) {
1000             if (! in_preamble) {
1001                 len = 0;
1002                 for (n = 0 ; n < runner.base.num_targets; n++) {
1003                     if (target_status[n] == CAIRO_TEST_CRASHED) {
1004                         if (strstr (targets,
1005                                     runner.base.targets_to_test[n]->name) == NULL)
1006                         {
1007                             len += snprintf (targets + len, sizeof (targets) - len,
1008                                              "%s, ",
1009                                              runner.base.targets_to_test[n]->name);
1010                         }
1011                     }
1012                 }
1013                 targets[len-2] = '\0';
1014                 _log (&runner.base, "\n%s: CRASH! (%s)\n", name, targets);
1015             } else {
1016                 _log (&runner.base, "\n%s: CRASH!\n", name);
1017             }
1018             runner.num_crashed++;
1019             runner.passed = FALSE;
1020         } else if (error) {
1021             if (! in_preamble) {
1022                 len = 0;
1023                 for (n = 0 ; n < runner.base.num_targets; n++) {
1024                     if (target_status[n] == CAIRO_TEST_ERROR) {
1025                         if (strstr (targets,
1026                                     runner.base.targets_to_test[n]->name) == NULL)
1027                         {
1028                             len += snprintf (targets + len,
1029                                              sizeof (targets) - len,
1030                                              "%s, ",
1031                                              runner.base.targets_to_test[n]->name);
1032                         }
1033                     }
1034                 }
1035                 targets[len-2] = '\0';
1036                 _log (&runner.base, "%s: ERROR (%s)\n", name, targets);
1037             } else {
1038                 _log (&runner.base, "%s: ERROR\n", name);
1039             }
1040             runner.num_error++;
1041             runner.passed = FALSE;
1042         } else if (failed) {
1043             if (! in_preamble) {
1044                 len = 0;
1045                 for (n = 0 ; n < runner.base.num_targets; n++) {
1046                     if (target_status[n] == CAIRO_TEST_FAILURE) {
1047                         if (strstr (targets,
1048                                     runner.base.targets_to_test[n]->name) == NULL)
1049                         {
1050                             len += snprintf (targets + len,
1051                                              sizeof (targets) - len,
1052                                              "%s, ",
1053                                              runner.base.targets_to_test[n]->name);
1054                         }
1055                     }
1056                 }
1057                 targets[len-2] = '\0';
1058                 _log (&runner.base, "%s: FAIL (%s)\n", name, targets);
1059             } else {
1060                 _log (&runner.base, "%s: FAIL\n", name);
1061             }
1062             runner.num_failed++;
1063             runner.passed = FALSE;
1064         } else if (xfailed) {
1065             _log (&runner.base, "%s: XFAIL\n", name);
1066             runner.num_xfailed++;
1067         } else if (skipped) {
1068             _log (&runner.base, "%s: UNTESTED\n", name);
1069             runner.num_skipped++;
1070         } else {
1071             _log (&runner.base, "%s: PASS\n", name);
1072             runner.num_passed++;
1073         }
1074         fflush (runner.base.log_file);
1075
1076   TEST_NEXT:
1077         free (name);
1078         if (runner.exit_on_failure && ! runner.passed)
1079             break;
1080
1081     }
1082
1083     if (cairo_tests_env)
1084         free(argv);
1085
1086     if (runner.list_only) {
1087         printf ("\n");
1088         return CAIRO_TEST_SUCCESS;
1089     }
1090
1091     for (n = 0 ; n < runner.base.num_targets; n++) {
1092         runner.crashes_per_target[n] = _list_reverse (runner.crashes_per_target[n]);
1093         runner.errors_per_target[n] = _list_reverse (runner.errors_per_target[n]);
1094         runner.fails_per_target[n] = _list_reverse (runner.fails_per_target[n]);
1095     }
1096
1097     _runner_print_results (&runner);
1098
1099     _list_free (tests);
1100     free (target_status);
1101     return _runner_fini (&runner);
1102 }
1103
1104 void
1105 cairo_test_register (cairo_test_t *test)
1106 {
1107     tests = _list_prepend (tests, test);
1108 }