test/cairo-test.c

   1 /*
   2  * Copyright © 2004 Red Hat, Inc.
   3  * Copyright © 2008 Chris Wilson
   4  *
   5  * Permission to use, copy, modify, distribute, and sell this software
   6  * and its documentation for any purpose is hereby granted without
   7  * fee, provided that the above copyright notice appear in all copies
   8  * and that both that copyright notice and this permission notice
   9  * appear in supporting documentation, and that the name of
  10  * Red Hat, Inc. not be used in advertising or publicity pertaining to
  11  * distribution of the software without specific, written prior
  12  * permission. Red Hat, Inc. makes no representations about the
  13  * suitability of this software for any purpose.  It is provided "as
  14  * is" without express or implied warranty.
  15  *
  16  * RED HAT, INC. DISCLAIMS ALL WARRANTIES WITH REGARD TO THIS
  17  * SOFTWARE, INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY AND
  18  * FITNESS, IN NO EVENT SHALL RED HAT, INC. BE LIABLE FOR ANY SPECIAL,
  19  * INDIRECT OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER
  20  * RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION
  21  * OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR
  22  * IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
  23  *
  24  * Author: Carl D. Worth <cworth@cworth.org>
  25  *         Chris Wilson <chris@chris-wilson.co.uk>
  26  */
  27
  28 #define _GNU_SOURCE 1   /* for feenableexcept() et al */
  29
  30 #if HAVE_CONFIG_H
  31 #include "config.h"
  32 #endif
  33
  34 #include <stdio.h>
  35 #include <stdlib.h>
  36 #include <stdarg.h>
  37 #include <ctype.h>
  38 #if HAVE_FEENABLEEXCEPT
  39 #include <fenv.h>
  40 #endif
  41 #include <assert.h>
  42 #if HAVE_UNISTD_H
  43 #include <unistd.h>
  44 #endif
  45 #include <errno.h>
  46 #include <string.h>
  47 #if HAVE_FCFINI
  48 #include <fontconfig/fontconfig.h>
  49 #endif
  50 #if CAIRO_HAS_REAL_PTHREAD
  51 #include <pthread.h>
  52 #endif
  53 #if HAVE_SYS_STAT_H
  54 #include <sys/stat.h>
  55 #endif
  56
  57 #if HAVE_VALGRIND
  58 #include <valgrind.h>
  59 #else
  60 #define RUNNING_ON_VALGRIND 0
  61 #endif
  62
  63 #if HAVE_MEMFAULT
  64 #include <memfault.h>
  65 #define MF(x) x
  66 #else
  67 #define MF(x)
  68 #endif
  69
  70 #include "cairo-test-private.h"
  71
  72 #include "buffer-diff.h"
  73
  74 #ifdef _MSC_VER
  75 #include <crtdbg.h>
  76 #include <direct.h>
  77 #define F_OK 0
  78 #define HAVE_MKDIR 1
  79 #define mkdir _mkdir
  80 #endif
  81
  82 #ifndef FALSE
  83 #define FALSE 0
  84 #endif
  85 #ifndef TRUE
  86 #define TRUE !FALSE
  87 #endif
  88
  89 #ifndef ARRAY_SIZE
  90 #define ARRAY_SIZE(A) (sizeof(A) / sizeof (A[0]))
  91 #endif
  92
  93 #if ! HAVE_ALARM || ! defined(SIGALRM)
  94 #define alarm(X);
  95 #endif
  96
  97 static const cairo_user_data_key_t _cairo_test_context_key;
  98
  99 static void
 100 _xunlink (const cairo_test_context_t *ctx, const char *pathname);
 101
 102 static const char *fail_face = "", *xfail_face="", *normal_face = "";
 103 static cairo_bool_t print_fail_on_stdout;
 104 static int cairo_test_timeout = 60;
 105
 106 #define NUM_DEVICE_OFFSETS 2
 107
 108 static cairo_bool_t
 109 _cairo_test_mkdir (const char *path)
 110 {
 111 #if ! HAVE_MKDIR
 112     return FALSE;
 113 #elif HAVE_MKDIR == 1
 114     if (mkdir (path) == 0)
 115         return TRUE;
 116 #elif HAVE_MKDIR == 2
 117     if (mkdir (path, 0770) == 0)
 118         return TRUE;
 119 #else
 120 #error Bad value for HAVE_MKDIR
 121 #endif
 122
 123     return errno == EEXIST;
 124 }
 125
 126 static char *
 127 _cairo_test_fixup_name (const char *original)
 128 {
 129     char *name, *s;
 130
 131     s = name = xstrdup (original);
 132     while ((s = strchr (s, '_')) != NULL)
 133         *s++ = '-';
 134
 135     return name;
 136 }
 137
 138 char *
 139 cairo_test_get_name (const cairo_test_t *test)
 140 {
 141     return _cairo_test_fixup_name (test->name);
 142 }
 143
 144 static void
 145 _cairo_test_init (cairo_test_context_t *ctx,
 146                   const cairo_test_context_t *parent,
 147                   const cairo_test_t *test,
 148                   const char *test_name,
 149                   const char *output)
 150 {
 151     char *log_name;
 152
 153     MF (MEMFAULT_DISABLE_FAULTS ());
 154
 155 #if HAVE_FEENABLEEXCEPT
 156     feenableexcept (FE_DIVBYZERO | FE_INVALID | FE_OVERFLOW);
 157 #endif
 158
 159     ctx->test = test;
 160     ctx->test_name = _cairo_test_fixup_name (test_name);
 161     ctx->output = output;
 162
 163     _cairo_test_mkdir (ctx->output);
 164
 165     ctx->malloc_failure = 0;
 166 #if HAVE_MEMFAULT
 167     if (getenv ("CAIRO_TEST_MALLOC_FAILURE"))
 168         ctx->malloc_failure = atoi (getenv ("CAIRO_TEST_MALLOC_FAILURE"));
 169     if (ctx->malloc_failure && ! RUNNING_ON_MEMFAULT ())
 170         ctx->malloc_failure = 0;
 171 #endif
 172
 173     ctx->timeout = cairo_test_timeout;
 174     if (getenv ("CAIRO_TEST_TIMEOUT"))
 175         ctx->timeout = atoi (getenv ("CAIRO_TEST_TIMEOUT"));
 176
 177     xasprintf (&log_name, "%s/%s%s", ctx->output, ctx->test_name, CAIRO_TEST_LOG_SUFFIX);
 178     _xunlink (NULL, log_name);
 179
 180     ctx->log_file = fopen (log_name, "a");
 181     if (ctx->log_file == NULL) {
 182         fprintf (stderr, "Error opening log file: %s\n", log_name);
 183         ctx->log_file = stderr;
 184     }
 185     free (log_name);
 186
 187     ctx->ref_name = NULL;
 188     ctx->ref_image = NULL;
 189     ctx->ref_image_flattened = NULL;
 190
 191     if (parent != NULL) {
 192         ctx->targets_to_test = parent->targets_to_test;
 193         ctx->num_targets = parent->num_targets;
 194         ctx->limited_targets = parent->limited_targets;
 195         ctx->own_targets = FALSE;
 196
 197         ctx->srcdir = parent->srcdir;
 198         ctx->refdir = parent->refdir;
 199     } else {
 200         int tmp_num_targets;
 201         cairo_bool_t tmp_limited_targets;
 202
 203         ctx->targets_to_test = cairo_boilerplate_get_targets (&tmp_num_targets, &tmp_limited_targets);
 204         ctx->num_targets = tmp_num_targets;
 205         ctx->limited_targets = tmp_limited_targets;
 206         ctx->own_targets = TRUE;
 207
 208         ctx->srcdir = getenv ("srcdir");
 209         if (ctx->srcdir == NULL)
 210             ctx->srcdir = ".";
 211
 212         ctx->refdir = getenv ("CAIRO_REF_DIR");
 213     }
 214
 215 #ifdef HAVE_UNISTD_H
 216     if (*fail_face == '\0' && isatty (2)) {
 217         fail_face = "\033[41;37;1m";
 218         xfail_face = "\033[43;37;1m";
 219         normal_face = "\033[m";
 220         if (isatty (1))
 221             print_fail_on_stdout = FALSE;
 222     }
 223 #endif
 224
 225     printf ("\nTESTING %s\n", ctx->test_name);
 226 }
 227
 228 void
 229 _cairo_test_context_init_for_test (cairo_test_context_t *ctx,
 230                                    const cairo_test_context_t *parent,
 231                                    const cairo_test_t *test)
 232 {
 233     _cairo_test_init (ctx, parent, test, test->name, CAIRO_TEST_OUTPUT_DIR);
 234 }
 235
 236 void
 237 cairo_test_init (cairo_test_context_t *ctx,
 238                  const char *test_name,
 239                  const char *output)
 240 {
 241     _cairo_test_init (ctx, NULL, NULL, test_name, output);
 242 }
 243
 244 void
 245 cairo_test_fini (cairo_test_context_t *ctx)
 246 {
 247     if (ctx->log_file == NULL)
 248         return;
 249
 250     if (ctx->log_file != stderr)
 251         fclose (ctx->log_file);
 252     ctx->log_file = NULL;
 253
 254     free (ctx->ref_name);
 255     cairo_surface_destroy (ctx->ref_image);
 256     cairo_surface_destroy (ctx->ref_image_flattened);
 257
 258     if (ctx->test_name != NULL)
 259         free ((char *) ctx->test_name);
 260
 261     if (ctx->own_targets)
 262         cairo_boilerplate_free_targets (ctx->targets_to_test);
 263
 264     cairo_boilerplate_fini ();
 265
 266     cairo_debug_reset_static_data ();
 267 #if HAVE_FCFINI
 268     FcFini ();
 269 #endif
 270 }
 271
 272 void
 273 cairo_test_logv (const cairo_test_context_t *ctx,
 274                 const char *fmt, va_list va)
 275 {
 276     FILE *file = ctx && ctx->log_file ? ctx->log_file : stderr;
 277     vfprintf (file, fmt, va);
 278 }
 279
 280 void
 281 cairo_test_log (const cairo_test_context_t *ctx, const char *fmt, ...)
 282 {
 283     va_list va;
 284
 285     va_start (va, fmt);
 286     cairo_test_logv (ctx, fmt, va);
 287     va_end (va);
 288 }
 289
 290 static void
 291 _xunlink (const cairo_test_context_t *ctx, const char *pathname)
 292 {
 293     if (unlink (pathname) < 0 && errno != ENOENT) {
 294         cairo_test_log (ctx, "Error: Cannot remove %s: %s\n",
 295                         pathname, strerror (errno));
 296         exit (1);
 297     }
 298 }
 299
 300 char *
 301 cairo_test_reference_filename (const cairo_test_context_t *ctx,
 302                                const char *base_name,
 303                                const char *test_name,
 304                                const char *target_name,
 305                                const char *base_target_name,
 306                                const char *format,
 307                                const char *suffix,
 308                                const char *extension)
 309 {
 310     char *ref_name = NULL;
 311
 312     /* First look for a previous build for comparison. */
 313     if (ctx->refdir != NULL) {
 314         xasprintf (&ref_name, "%s/%s%s%s",
 315                    ctx->refdir,
 316                    base_name,
 317                    suffix,
 318                    extension);
 319         if (access (ref_name, F_OK) != 0)
 320             free (ref_name);
 321         else
 322             goto done;
 323     }
 324
 325     if (target_name != NULL) {
 326         /* Next look for a target/format-specific reference image. */
 327         xasprintf (&ref_name, "%s/reference/%s.%s.%s%s%s",
 328                    ctx->srcdir,
 329                    test_name,
 330                    target_name,
 331                    format,
 332                    suffix,
 333                    extension);
 334         if (access (ref_name, F_OK) != 0)
 335             free (ref_name);
 336         else
 337             goto done;
 338
 339         /* Next, look for target-specific reference image. */
 340         xasprintf (&ref_name, "%s/reference/%s.%s%s%s",
 341                    ctx->srcdir,
 342                    test_name,
 343                    target_name,
 344                    suffix,
 345                    extension);
 346         if (access (ref_name, F_OK) != 0)
 347             free (ref_name);
 348         else
 349             goto done;
 350     }
 351
 352     if (base_target_name != NULL) {
 353         /* Next look for a base/format-specific reference image. */
 354         xasprintf (&ref_name, "%s/reference/%s.%s.%s%s%s",
 355                    ctx->srcdir,
 356                    test_name,
 357                    base_target_name,
 358                    format,
 359                    suffix,
 360                    extension);
 361         if (access (ref_name, F_OK) != 0)
 362             free (ref_name);
 363         else
 364             goto done;
 365
 366         /* Next, look for base-specific reference image. */
 367         xasprintf (&ref_name, "%s/reference/%s.%s%s%s",
 368                    ctx->srcdir,
 369                    test_name,
 370                    base_target_name,
 371                    suffix,
 372                    extension);
 373         if (access (ref_name, F_OK) != 0)
 374             free (ref_name);
 375         else
 376             goto done;
 377     }
 378
 379     /* Next, look for format-specific reference image. */
 380     xasprintf (&ref_name, "%s/reference/%s.%s%s%s",
 381                ctx->srcdir,
 382                test_name,
 383                format,
 384                suffix,
 385                extension);
 386     if (access (ref_name, F_OK) != 0)
 387         free (ref_name);
 388     else
 389         goto done;
 390
 391     /* Finally, look for the standard reference image. */
 392     xasprintf (&ref_name, "%s/reference/%s%s%s", ctx->srcdir,
 393                test_name,
 394                suffix,
 395                extension);
 396     if (access (ref_name, F_OK) != 0)
 397         free (ref_name);
 398     else
 399         goto done;
 400
 401     ref_name = NULL;
 402
 403 done:
 404     return ref_name;
 405 }
 406
 407 cairo_test_similar_t
 408 cairo_test_target_has_similar (const cairo_test_context_t *ctx,
 409                                const cairo_boilerplate_target_t *target)
 410 {
 411     cairo_surface_t *surface;
 412     cairo_test_similar_t has_similar;
 413     cairo_t * cr;
 414     cairo_surface_t *similar;
 415     cairo_status_t status;
 416     void *closure;
 417     char *path;
 418
 419     /* ignore image intermediate targets */
 420     if (target->expected_type == CAIRO_SURFACE_TYPE_IMAGE)
 421         return DIRECT;
 422
 423     if (getenv ("CAIRO_TEST_IGNORE_SIMILAR"))
 424         return DIRECT;
 425
 426     xasprintf (&path, "%s/%s",
 427                _cairo_test_mkdir (ctx->output) ? ctx->output : ".",
 428                ctx->test_name);
 429
 430     has_similar = DIRECT;
 431     do {
 432         do {
 433             surface = (target->create_surface) (path,
 434                                                 target->content,
 435                                                 ctx->test->width,
 436                                                 ctx->test->height,
 437                                                 ctx->test->width + 25 * NUM_DEVICE_OFFSETS,
 438                                                 ctx->test->height + 25 * NUM_DEVICE_OFFSETS,
 439                                                 CAIRO_BOILERPLATE_MODE_TEST,
 440                                                 &closure);
 441             if (surface == NULL)
 442                 goto out;
 443         } while (cairo_test_malloc_failure (ctx, cairo_surface_status (surface)));
 444
 445         if (cairo_surface_status (surface))
 446             goto out;
 447
 448         cr = cairo_create (surface);
 449         cairo_push_group_with_content (cr,
 450                                        cairo_boilerplate_content (target->content));
 451         similar = cairo_get_group_target (cr);
 452         status = cairo_surface_status (similar);
 453
 454         if (cairo_surface_get_type (similar) == cairo_surface_get_type (surface))
 455             has_similar = SIMILAR;
 456         else
 457             has_similar = DIRECT;
 458
 459         cairo_destroy (cr);
 460         cairo_surface_destroy (surface);
 461
 462         if (target->cleanup)
 463             target->cleanup (closure);
 464     } while (! has_similar && cairo_test_malloc_failure (ctx, status));
 465 out:
 466     free (path);
 467
 468     return has_similar;
 469 }
 470
 471 static cairo_surface_t *
 472 _cairo_test_flatten_reference_image (cairo_test_context_t *ctx,
 473                                      cairo_bool_t flatten)
 474 {
 475     cairo_surface_t *surface;
 476     cairo_t *cr;
 477
 478     if (! flatten)
 479         return ctx->ref_image;
 480
 481     if (ctx->ref_image_flattened != NULL)
 482         return ctx->ref_image_flattened;
 483
 484     surface = cairo_image_surface_create (CAIRO_FORMAT_ARGB32,
 485                                           cairo_image_surface_get_width (ctx->ref_image),
 486                                           cairo_image_surface_get_height (ctx->ref_image));
 487     cr = cairo_create (surface);
 488     cairo_surface_destroy (surface);
 489
 490     cairo_set_source_rgb (cr, 1, 1, 1);
 491     cairo_paint (cr);
 492
 493     cairo_set_source_surface (cr, ctx->ref_image, 0, 0);
 494     cairo_paint (cr);
 495
 496     surface = cairo_surface_reference (cairo_get_target (cr));
 497     cairo_destroy (cr);
 498
 499     if (cairo_surface_status (surface) == CAIRO_STATUS_SUCCESS)
 500         ctx->ref_image_flattened = surface;
 501     return surface;
 502 }
 503
 504 cairo_surface_t *
 505 cairo_test_get_reference_image (cairo_test_context_t *ctx,
 506                                 const char *filename,
 507                                 cairo_bool_t flatten)
 508 {
 509     cairo_surface_t *surface;
 510
 511     if (ctx->ref_name != NULL) {
 512         if (strcmp (ctx->ref_name, filename) == 0)
 513             return _cairo_test_flatten_reference_image (ctx, flatten);
 514
 515         cairo_surface_destroy (ctx->ref_image);
 516         ctx->ref_image = NULL;
 517
 518         cairo_surface_destroy (ctx->ref_image_flattened);
 519         ctx->ref_image_flattened = NULL;
 520
 521         free (ctx->ref_name);
 522         ctx->ref_name = NULL;
 523     }
 524
 525     surface = cairo_image_surface_create_from_png (filename);
 526     if (cairo_surface_status (surface))
 527         return surface;
 528
 529     ctx->ref_name = xstrdup (filename);
 530     ctx->ref_image = surface;
 531     return _cairo_test_flatten_reference_image (ctx, flatten);
 532 }
 533
 534 static cairo_bool_t
 535 cairo_test_file_is_older (const char *filename,
 536                           char **ref_filenames,
 537                           int num_ref_filenames)
 538 {
 539 #if HAVE_SYS_STAT_H
 540     struct stat st;
 541
 542     if (stat (filename, &st) < 0)
 543         return FALSE;
 544
 545     while (num_ref_filenames--) {
 546         struct stat ref;
 547         char *ref_filename = *ref_filenames++;
 548
 549         if (ref_filename == NULL)
 550             continue;
 551
 552         if (stat (ref_filename++, &ref) < 0)
 553             continue;
 554
 555         if (st.st_mtime <= ref.st_mtime)
 556             return TRUE;
 557     }
 558 #endif
 559
 560     return FALSE;
 561 }
 562
 563 static cairo_bool_t
 564 cairo_test_files_equal (const char *test_filename,
 565                         const char *pass_filename)
 566 {
 567     FILE *test, *pass;
 568     int t, p;
 569
 570     if (test_filename == NULL || pass_filename == NULL)
 571         return FALSE;
 572
 573     test = fopen (test_filename, "rb");
 574     if (test == NULL)
 575         return FALSE;
 576
 577     pass = fopen (pass_filename, "rb");
 578     if (pass == NULL) {
 579         fclose (test);
 580         return FALSE;
 581     }
 582
 583     /* as simple as it gets */
 584     do {
 585         t = getc (test);
 586         p = getc (pass);
 587         if (t != p)
 588             break;
 589     } while (t != EOF && p != EOF);
 590
 591     fclose (pass);
 592     fclose (test);
 593
 594     return t == p; /* both EOF */
 595 }
 596
 597 static cairo_bool_t
 598 cairo_test_copy_file (const char *src_filename,
 599                       const char *dst_filename)
 600 {
 601     FILE *src, *dst;
 602     int c;
 603
 604 #if HAVE_LINK
 605     if (link (src_filename, dst_filename) == 0)
 606         return TRUE;
 607
 608     unlink (dst_filename);
 609 #endif
 610
 611     src = fopen (src_filename, "rb");
 612     if (src == NULL)
 613         return FALSE;
 614
 615     dst = fopen (dst_filename, "wb");
 616     if (dst == NULL) {
 617         fclose (src);
 618         return FALSE;
 619     }
 620
 621     /* as simple as it gets */
 622     while ((c = getc (src)) != EOF)
 623         putc (c, dst);
 624
 625     fclose (src);
 626     fclose (dst);
 627
 628     return TRUE;
 629 }
 630
 631 static cairo_test_status_t
 632 cairo_test_for_target (cairo_test_context_t              *ctx,
 633                        const cairo_boilerplate_target_t  *target,
 634                        int                                dev_offset,
 635                        cairo_bool_t                       similar)
 636 {
 637     cairo_test_status_t status;
 638     cairo_surface_t *surface = NULL;
 639     cairo_t *cr;
 640     const char *empty_str = "";
 641     char *offset_str;
 642     char *base_name, *base_path;
 643     char *out_png_path;
 644     char *ref_path = NULL, *ref_png_path, *cmp_png_path = NULL;
 645     char *new_path = NULL, *new_png_path;
 646     char *xfail_path = NULL, *xfail_png_path;
 647     char *base_ref_png_path;
 648     char *base_new_png_path;
 649     char *base_xfail_png_path;
 650     char *diff_png_path;
 651     char *test_filename = NULL, *pass_filename = NULL, *fail_filename = NULL;
 652     cairo_test_status_t ret;
 653     cairo_content_t expected_content;
 654     cairo_font_options_t *font_options;
 655     const char *format;
 656     cairo_bool_t have_output = FALSE;
 657     cairo_bool_t have_result = FALSE;
 658     void *closure;
 659     double width, height;
 660     cairo_bool_t have_output_dir;
 661 #if HAVE_MEMFAULT
 662     int malloc_failure_iterations = ctx->malloc_failure;
 663     int last_fault_count = 0;
 664 #endif
 665
 666     /* Get the strings ready that we'll need. */
 667     format = cairo_boilerplate_content_name (target->content);
 668     if (dev_offset)
 669         xasprintf (&offset_str, ".%d", dev_offset);
 670     else
 671         offset_str = (char *) empty_str;
 672
 673     xasprintf (&base_name, "%s.%s.%s%s%s",
 674                ctx->test_name,
 675                target->name,
 676                format,
 677                similar ? ".similar" : "",
 678                offset_str);
 679
 680     if (offset_str != empty_str)
 681       free (offset_str);
 682
 683     ref_png_path = cairo_test_reference_filename (ctx,
 684                                                   base_name,
 685                                                   ctx->test_name,
 686                                                   target->name,
 687                                                   target->basename,
 688                                                   format,
 689                                                   CAIRO_TEST_REF_SUFFIX,
 690                                                   CAIRO_TEST_PNG_EXTENSION);
 691     new_png_path = cairo_test_reference_filename (ctx,
 692                                                   base_name,
 693                                                   ctx->test_name,
 694                                                   target->name,
 695                                                   target->basename,
 696                                                   format,
 697                                                   CAIRO_TEST_NEW_SUFFIX,
 698                                                   CAIRO_TEST_PNG_EXTENSION);
 699     xfail_png_path = cairo_test_reference_filename (ctx,
 700                                                     base_name,
 701                                                     ctx->test_name,
 702                                                     target->name,
 703                                                     target->basename,
 704                                                     format,
 705                                                     CAIRO_TEST_XFAIL_SUFFIX,
 706                                                     CAIRO_TEST_PNG_EXTENSION);
 707
 708     base_ref_png_path = cairo_test_reference_filename (ctx,
 709                                                   base_name,
 710                                                   ctx->test_name,
 711                                                   NULL, NULL,
 712                                                   format,
 713                                                   CAIRO_TEST_REF_SUFFIX,
 714                                                   CAIRO_TEST_PNG_EXTENSION);
 715     base_new_png_path = cairo_test_reference_filename (ctx,
 716                                                   base_name,
 717                                                   ctx->test_name,
 718                                                   NULL, NULL,
 719                                                   format,
 720                                                   CAIRO_TEST_NEW_SUFFIX,
 721                                                   CAIRO_TEST_PNG_EXTENSION);
 722     base_xfail_png_path = cairo_test_reference_filename (ctx,
 723                                                     base_name,
 724                                                     ctx->test_name,
 725                                                     NULL, NULL,
 726                                                     format,
 727                                                     CAIRO_TEST_XFAIL_SUFFIX,
 728                                                     CAIRO_TEST_PNG_EXTENSION);
 729
 730     if (target->file_extension != NULL) {
 731         ref_path = cairo_test_reference_filename (ctx,
 732                                                   base_name,
 733                                                   ctx->test_name,
 734                                                   target->name,
 735                                                   target->basename,
 736                                                   format,
 737                                                   CAIRO_TEST_REF_SUFFIX,
 738                                                   target->file_extension);
 739         new_path = cairo_test_reference_filename (ctx,
 740                                                   base_name,
 741                                                   ctx->test_name,
 742                                                   target->name,
 743                                                   target->basename,
 744                                                   format,
 745                                                   CAIRO_TEST_NEW_SUFFIX,
 746                                                   target->file_extension);
 747         xfail_path = cairo_test_reference_filename (ctx,
 748                                                     base_name,
 749                                                     ctx->test_name,
 750                                                     target->name,
 751                                                     target->basename,
 752                                                     format,
 753                                                     CAIRO_TEST_XFAIL_SUFFIX,
 754                                                     target->file_extension);
 755     }
 756
 757     have_output_dir = _cairo_test_mkdir (ctx->output);
 758     xasprintf (&base_path, "%s/%s",
 759                have_output_dir ? ctx->output : ".",
 760                base_name);
 761     xasprintf (&out_png_path, "%s" CAIRO_TEST_OUT_PNG, base_path);
 762     xasprintf (&diff_png_path, "%s" CAIRO_TEST_DIFF_PNG, base_path);
 763
 764     if (ctx->test->requirements != NULL) {
 765         const char *required;
 766
 767         required = target->is_vector ? "target=raster" : "target=vector";
 768         if (strstr (ctx->test->requirements, required) != NULL) {
 769             cairo_test_log (ctx, "Error: Skipping for %s target %s\n",
 770                             target->is_vector ? "vector" : "raster",
 771                             target->name);
 772             ret = CAIRO_TEST_UNTESTED;
 773             goto UNWIND_STRINGS;
 774         }
 775
 776         required = target->is_recording ? "target=!recording" : "target=recording";
 777         if (strstr (ctx->test->requirements, required) != NULL) {
 778             cairo_test_log (ctx, "Error: Skipping for %s target %s\n",
 779                             target->is_recording ? "recording" : "non-recording",
 780                             target->name);
 781             ret = CAIRO_TEST_UNTESTED;
 782             goto UNWIND_STRINGS;
 783         }
 784     }
 785
 786     width = ctx->test->width;
 787     height = ctx->test->height;
 788     if (width && height) {
 789         width += dev_offset;
 790         height += dev_offset;
 791     }
 792
 793 #if HAVE_MEMFAULT
 794 REPEAT:
 795     MEMFAULT_CLEAR_FAULTS ();
 796     MEMFAULT_RESET_LEAKS ();
 797     ctx->last_fault_count = 0;
 798     last_fault_count = MEMFAULT_COUNT_FAULTS ();
 799
 800     /* Pre-initialise fontconfig so that the configuration is loaded without
 801      * malloc failures (our primary goal is to test cairo fault tolerance).
 802      */
 803 #if HAVE_FCINIT
 804     FcInit ();
 805 #endif
 806
 807     MEMFAULT_ENABLE_FAULTS ();
 808 #endif
 809     have_output = FALSE;
 810     have_result = FALSE;
 811
 812     /* Run the actual drawing code. */
 813     ret = CAIRO_TEST_SUCCESS;
 814     surface = (target->create_surface) (base_path,
 815                                         target->content,
 816                                         width, height,
 817                                         ctx->test->width + 25 * NUM_DEVICE_OFFSETS,
 818                                         ctx->test->height + 25 * NUM_DEVICE_OFFSETS,
 819                                         CAIRO_BOILERPLATE_MODE_TEST,
 820                                         &closure);
 821     if (surface == NULL) {
 822         cairo_test_log (ctx, "Error: Failed to set %s target\n", target->name);
 823         ret = CAIRO_TEST_UNTESTED;
 824         goto UNWIND_STRINGS;
 825     }
 826
 827 #if HAVE_MEMFAULT
 828     if (ctx->malloc_failure &&
 829         MEMFAULT_COUNT_FAULTS () - last_fault_count > 0 &&
 830         cairo_surface_status (surface) == CAIRO_STATUS_NO_MEMORY)
 831     {
 832         goto REPEAT;
 833     }
 834 #endif
 835
 836     if (cairo_surface_status (surface)) {
 837         MF (MEMFAULT_PRINT_FAULTS ());
 838         cairo_test_log (ctx, "Error: Created an error surface: %s\n",
 839                         cairo_status_to_string (cairo_surface_status (surface)));
 840         ret = CAIRO_TEST_FAILURE;
 841         goto UNWIND_STRINGS;
 842     }
 843
 844     /* Check that we created a surface of the expected type. */
 845     if (cairo_surface_get_type (surface) != target->expected_type) {
 846         MF (MEMFAULT_PRINT_FAULTS ());
 847         cairo_test_log (ctx, "Error: Created surface is of type %d (expected %d)\n",
 848                         cairo_surface_get_type (surface), target->expected_type);
 849         ret = CAIRO_TEST_UNTESTED;
 850         goto UNWIND_SURFACE;
 851     }
 852
 853     /* Check that we created a surface of the expected content,
 854      * (ignore the artificial CAIRO_TEST_CONTENT_COLOR_ALPHA_FLATTENED value).
 855      */
 856     expected_content = cairo_boilerplate_content (target->content);
 857
 858     if (cairo_surface_get_content (surface) != expected_content) {
 859         MF (MEMFAULT_PRINT_FAULTS ());
 860         cairo_test_log (ctx, "Error: Created surface has content %d (expected %d)\n",
 861                         cairo_surface_get_content (surface), expected_content);
 862         ret = CAIRO_TEST_FAILURE;
 863         goto UNWIND_SURFACE;
 864     }
 865
 866     if (cairo_surface_set_user_data (surface,
 867                                      &cairo_boilerplate_output_basename_key,
 868                                      base_path,
 869                                      NULL))
 870     {
 871 #if HAVE_MEMFAULT
 872         cairo_surface_destroy (surface);
 873
 874         if (target->cleanup)
 875             target->cleanup (closure);
 876
 877         goto REPEAT;
 878 #else
 879         ret = CAIRO_TEST_FAILURE;
 880         goto UNWIND_SURFACE;
 881 #endif
 882     }
 883
 884     cairo_surface_set_device_offset (surface, dev_offset, dev_offset);
 885
 886     cr = cairo_create (surface);
 887     if (cairo_set_user_data (cr, &_cairo_test_context_key, (void*) ctx, NULL)) {
 888 #if HAVE_MEMFAULT
 889         cairo_destroy (cr);
 890         cairo_surface_destroy (surface);
 891
 892         if (target->cleanup)
 893             target->cleanup (closure);
 894
 895         goto REPEAT;
 896 #else
 897         ret = CAIRO_TEST_FAILURE;
 898         goto UNWIND_CAIRO;
 899 #endif
 900     }
 901
 902     if (similar)
 903         cairo_push_group_with_content (cr, expected_content);
 904
 905     /* Clear to transparent (or black) depending on whether the target
 906      * surface supports alpha. */
 907     cairo_save (cr);
 908     cairo_set_operator (cr, CAIRO_OPERATOR_CLEAR);
 909     cairo_paint (cr);
 910     cairo_restore (cr);
 911
 912     /* Set all components of font_options to avoid backend differences
 913      * and reduce number of needed reference images. */
 914     font_options = cairo_font_options_create ();
 915     cairo_font_options_set_hint_style (font_options, CAIRO_HINT_STYLE_NONE);
 916     cairo_font_options_set_hint_metrics (font_options, CAIRO_HINT_METRICS_ON);
 917     cairo_font_options_set_antialias (font_options, CAIRO_ANTIALIAS_GRAY);
 918     cairo_set_font_options (cr, font_options);
 919     cairo_font_options_destroy (font_options);
 920
 921     cairo_save (cr);
 922     alarm (ctx->timeout);
 923     status = (ctx->test->draw) (cr, ctx->test->width, ctx->test->height);
 924     alarm (0);
 925     cairo_restore (cr);
 926
 927     if (similar) {
 928         cairo_pop_group_to_source (cr);
 929         cairo_set_operator (cr, CAIRO_OPERATOR_SOURCE);
 930         cairo_paint (cr);
 931     }
 932
 933 #if HAVE_MEMFAULT
 934     MEMFAULT_DISABLE_FAULTS ();
 935
 936     /* repeat test after malloc failure injection */
 937     if (ctx->malloc_failure &&
 938         MEMFAULT_COUNT_FAULTS () - last_fault_count > 0 &&
 939         (status == CAIRO_TEST_NO_MEMORY ||
 940          cairo_status (cr) == CAIRO_STATUS_NO_MEMORY ||
 941          cairo_surface_status (surface) == CAIRO_STATUS_NO_MEMORY))
 942     {
 943         cairo_destroy (cr);
 944         cairo_surface_destroy (surface);
 945         if (target->cleanup)
 946             target->cleanup (closure);
 947         cairo_debug_reset_static_data ();
 948 #if HAVE_FCFINI
 949         FcFini ();
 950 #endif
 951         if (MEMFAULT_COUNT_LEAKS () > 0) {
 952             MEMFAULT_PRINT_FAULTS ();
 953             MEMFAULT_PRINT_LEAKS ();
 954         }
 955
 956         goto REPEAT;
 957     }
 958 #endif
 959
 960     /* Then, check all the different ways it could fail. */
 961     if (status) {
 962         cairo_test_log (ctx, "Error: Function under test failed\n");
 963         ret = status;
 964         goto UNWIND_CAIRO;
 965     }
 966
 967 #if HAVE_MEMFAULT
 968     if (MEMFAULT_COUNT_FAULTS () - last_fault_count > 0 &&
 969         MEMFAULT_HAS_FAULTS ())
 970     {
 971         VALGRIND_PRINTF ("Unreported memfaults...");
 972         MEMFAULT_PRINT_FAULTS ();
 973     }
 974 #endif
 975
 976     if (target->finish_surface != NULL) {
 977 #if HAVE_MEMFAULT
 978         /* We need to re-enable faults as most recording-surface processing
 979          * is done during cairo_surface_finish().
 980          */
 981         MEMFAULT_CLEAR_FAULTS ();
 982         last_fault_count = MEMFAULT_COUNT_FAULTS ();
 983         MEMFAULT_ENABLE_FAULTS ();
 984 #endif
 985
 986         /* also check for infinite loops whilst replaying */
 987         alarm (ctx->timeout);
 988         status = target->finish_surface (surface);
 989         alarm (0);
 990
 991 #if HAVE_MEMFAULT
 992         MEMFAULT_DISABLE_FAULTS ();
 993
 994         if (ctx->malloc_failure &&
 995             MEMFAULT_COUNT_FAULTS () - last_fault_count > 0 &&
 996             status == CAIRO_STATUS_NO_MEMORY)
 997         {
 998             cairo_destroy (cr);
 999             cairo_surface_destroy (surface);
1000             if (target->cleanup)
1001                 target->cleanup (closure);
1002             cairo_debug_reset_static_data ();
1003 #if HAVE_FCFINI
1004             FcFini ();
1005 #endif
1006             if (MEMFAULT_COUNT_LEAKS () > 0) {
1007                 MEMFAULT_PRINT_FAULTS ();
1008                 MEMFAULT_PRINT_LEAKS ();
1009             }
1010
1011             goto REPEAT;
1012         }
1013 #endif
1014         if (status) {
1015             cairo_test_log (ctx, "Error: Failed to finish surface: %s\n",
1016                             cairo_status_to_string (status));
1017             ret = CAIRO_TEST_FAILURE;
1018             goto UNWIND_CAIRO;
1019         }
1020     }
1021
1022     /* Skip image check for tests with no image (width,height == 0,0) */
1023     if (ctx->test->width != 0 && ctx->test->height != 0) {
1024         cairo_surface_t *ref_image;
1025         cairo_surface_t *test_image;
1026         cairo_surface_t *diff_image;
1027         buffer_diff_result_t result;
1028         cairo_status_t diff_status;
1029
1030         if (ref_png_path == NULL) {
1031             cairo_test_log (ctx, "Error: Cannot find reference image for %s\n",
1032                             base_name);
1033
1034             /* we may be running this test to generate reference images */
1035             _xunlink (ctx, out_png_path);
1036             /* be more generous as we may need to use external renderers */
1037             alarm (4 * ctx->timeout);
1038             test_image = target->get_image_surface (surface, 0,
1039                                                     ctx->test->width,
1040                                                     ctx->test->height);
1041             alarm (0);
1042             diff_status = cairo_surface_write_to_png (test_image, out_png_path);
1043             cairo_surface_destroy (test_image);
1044             if (diff_status) {
1045                 if (cairo_surface_status (test_image) == CAIRO_STATUS_INVALID_STATUS)
1046                     ret = CAIRO_TEST_CRASHED;
1047                 else
1048                     ret = CAIRO_TEST_FAILURE;
1049                 cairo_test_log (ctx,
1050                                 "Error: Failed to write output image: %s\n",
1051                                 cairo_status_to_string (diff_status));
1052             }
1053             have_output = TRUE;
1054
1055             ret = CAIRO_TEST_XFAILURE;
1056             goto UNWIND_CAIRO;
1057         }
1058
1059         if (target->file_extension != NULL) { /* compare vector surfaces */
1060             char *filenames[] = {
1061                 ref_png_path,
1062                 ref_path,
1063                 new_png_path,
1064                 new_path,
1065                 xfail_png_path,
1066                 xfail_path,
1067                 base_ref_png_path,
1068                 base_new_png_path,
1069                 base_xfail_png_path,
1070             };
1071
1072             xasprintf (&test_filename, "%s.out%s",
1073                        base_path, target->file_extension);
1074             xasprintf (&pass_filename, "%s.pass%s",
1075                        base_path, target->file_extension);
1076             xasprintf (&fail_filename, "%s.fail%s",
1077                        base_path, target->file_extension);
1078
1079             if (cairo_test_file_is_older (pass_filename,
1080                                           filenames,
1081                                           ARRAY_SIZE (filenames)))
1082             {
1083                 _xunlink (ctx, pass_filename);
1084             }
1085             if (cairo_test_file_is_older (fail_filename,
1086                                           filenames,
1087                                           ARRAY_SIZE (filenames)))
1088             {
1089                 _xunlink (ctx, fail_filename);
1090             }
1091
1092             if (cairo_test_files_equal (out_png_path, ref_path)) {
1093                 cairo_test_log (ctx, "Vector surface matches reference.\n");
1094                 have_output = FALSE;
1095                 ret = CAIRO_TEST_SUCCESS;
1096                 goto UNWIND_CAIRO;
1097             }
1098             if (cairo_test_files_equal (out_png_path, new_path)) {
1099                 cairo_test_log (ctx, "Vector surface matches current failure.\n");
1100                 have_output = FALSE;
1101                 ret = CAIRO_TEST_NEW;
1102                 goto UNWIND_CAIRO;
1103             }
1104             if (cairo_test_files_equal (out_png_path, xfail_path)) {
1105                 cairo_test_log (ctx, "Vector surface matches known failure.\n");
1106                 have_output = FALSE;
1107                 ret = CAIRO_TEST_XFAILURE;
1108                 goto UNWIND_CAIRO;
1109             }
1110
1111             if (cairo_test_files_equal (test_filename, pass_filename)) {
1112                 /* identical output as last known PASS */
1113                 cairo_test_log (ctx, "Vector surface matches last pass.\n");
1114                 have_output = TRUE;
1115                 ret = CAIRO_TEST_SUCCESS;
1116                 goto UNWIND_CAIRO;
1117             }
1118             if (cairo_test_files_equal (test_filename, fail_filename)) {
1119                 /* identical output as last known FAIL, fail */
1120                 cairo_test_log (ctx, "Vector surface matches last fail.\n");
1121                 have_result = TRUE; /* presume these were kept around as well */
1122                 have_output = TRUE;
1123                 ret = CAIRO_TEST_FAILURE;
1124                 goto UNWIND_CAIRO;
1125             }
1126         }
1127
1128         /* be more generous as we may need to use external renderers */
1129         alarm (4 * ctx->timeout);
1130         test_image = target->get_image_surface (surface, 0,
1131                                                 ctx->test->width,
1132                                                 ctx->test->height);
1133         alarm (0);
1134         if (cairo_surface_status (test_image)) {
1135             cairo_test_log (ctx, "Error: Failed to extract image: %s\n",
1136                             cairo_status_to_string (cairo_surface_status (test_image)));
1137             if (cairo_surface_status (test_image) == CAIRO_STATUS_INVALID_STATUS)
1138                 ret = CAIRO_TEST_CRASHED;
1139             else
1140                 ret = CAIRO_TEST_FAILURE;
1141             cairo_surface_destroy (test_image);
1142             goto UNWIND_CAIRO;
1143         }
1144
1145         _xunlink (ctx, out_png_path);
1146         diff_status = cairo_surface_write_to_png (test_image, out_png_path);
1147         if (diff_status) {
1148             cairo_test_log (ctx, "Error: Failed to write output image: %s\n",
1149                             cairo_status_to_string (diff_status));
1150             cairo_surface_destroy (test_image);
1151             ret = CAIRO_TEST_FAILURE;
1152             goto UNWIND_CAIRO;
1153         }
1154         have_output = TRUE;
1155
1156         /* binary compare png files (no decompression) */
1157         if (target->file_extension == NULL) {
1158             char *filenames[] = {
1159                 ref_png_path,
1160                 new_png_path,
1161                 xfail_png_path,
1162                 base_ref_png_path,
1163                 base_new_png_path,
1164                 base_xfail_png_path,
1165             };
1166
1167             xasprintf (&test_filename, "%s", out_png_path);
1168             xasprintf (&pass_filename, "%s.pass.png", base_path);
1169             xasprintf (&fail_filename, "%s.fail.png", base_path);
1170
1171             if (cairo_test_file_is_older (pass_filename,
1172                                           filenames,
1173                                           ARRAY_SIZE (filenames)))
1174             {
1175                 _xunlink (ctx, pass_filename);
1176             }
1177             if (cairo_test_file_is_older (fail_filename,
1178                                           filenames,
1179                                           ARRAY_SIZE (filenames)))
1180             {
1181                 _xunlink (ctx, fail_filename);
1182             }
1183
1184             if (cairo_test_files_equal (test_filename, pass_filename)) {
1185                 cairo_test_log (ctx, "PNG file exactly matches last pass.\n");
1186                 have_result = TRUE;
1187                 cairo_surface_destroy (test_image);
1188                 ret = CAIRO_TEST_SUCCESS;
1189                 goto UNWIND_CAIRO;
1190             }
1191             if (cairo_test_files_equal (out_png_path, ref_png_path)) {
1192                 cairo_test_log (ctx, "PNG file exactly matches reference image.\n");
1193                 have_result = TRUE;
1194                 cairo_surface_destroy (test_image);
1195                 ret = CAIRO_TEST_SUCCESS;
1196                 goto UNWIND_CAIRO;
1197             }
1198             if (cairo_test_files_equal (out_png_path, new_png_path)) {
1199                 cairo_test_log (ctx, "PNG file exactly matches current failure image.\n");
1200                 have_result = TRUE;
1201                 cairo_surface_destroy (test_image);
1202                 ret = CAIRO_TEST_NEW;
1203                 goto UNWIND_CAIRO;
1204             }
1205             if (cairo_test_files_equal (out_png_path, xfail_png_path)) {
1206                 cairo_test_log (ctx, "PNG file exactly matches known failure image.\n");
1207                 have_result = TRUE;
1208                 cairo_surface_destroy (test_image);
1209                 ret = CAIRO_TEST_XFAILURE;
1210                 goto UNWIND_CAIRO;
1211             }
1212             if (cairo_test_files_equal (test_filename, fail_filename)) {
1213                 cairo_test_log (ctx, "PNG file exactly matches last fail.\n");
1214                 have_result = TRUE; /* presume these were kept around as well */
1215                 cairo_surface_destroy (test_image);
1216                 ret = CAIRO_TEST_FAILURE;
1217                 goto UNWIND_CAIRO;
1218             }
1219         } else {
1220             if (cairo_test_files_equal (out_png_path, ref_png_path)) {
1221                 cairo_test_log (ctx, "PNG file exactly matches reference image.\n");
1222                 have_result = TRUE;
1223                 cairo_surface_destroy (test_image);
1224                 ret = CAIRO_TEST_SUCCESS;
1225                 goto UNWIND_CAIRO;
1226             }
1227             if (cairo_test_files_equal (out_png_path, new_png_path)) {
1228                 cairo_test_log (ctx, "PNG file exactly matches current failure image.\n");
1229                 have_result = TRUE;
1230                 cairo_surface_destroy (test_image);
1231                 ret = CAIRO_TEST_NEW;
1232                 goto UNWIND_CAIRO;
1233             }
1234             if (cairo_test_files_equal (out_png_path, xfail_png_path)) {
1235                 cairo_test_log (ctx, "PNG file exactly matches known failure image.\n");
1236                 have_result = TRUE;
1237                 cairo_surface_destroy (test_image);
1238                 ret = CAIRO_TEST_XFAILURE;
1239                 goto UNWIND_CAIRO;
1240             }
1241         }
1242
1243         if (cairo_test_files_equal (out_png_path, base_ref_png_path)) {
1244             cairo_test_log (ctx, "PNG file exactly reference image.\n");
1245             have_result = TRUE;
1246             cairo_surface_destroy (test_image);
1247             ret = CAIRO_TEST_SUCCESS;
1248             goto UNWIND_CAIRO;
1249         }
1250         if (cairo_test_files_equal (out_png_path, base_new_png_path)) {
1251             cairo_test_log (ctx, "PNG file exactly current failure image.\n");
1252             have_result = TRUE;
1253             cairo_surface_destroy (test_image);
1254             ret = CAIRO_TEST_NEW;
1255             goto UNWIND_CAIRO;
1256         }
1257         if (cairo_test_files_equal (out_png_path, base_xfail_png_path)) {
1258             cairo_test_log (ctx, "PNG file exactly known failure image.\n");
1259             have_result = TRUE;
1260             cairo_surface_destroy (test_image);
1261             ret = CAIRO_TEST_XFAILURE;
1262             goto UNWIND_CAIRO;
1263         }
1264
1265         /* first compare against the ideal reference */
1266         ref_image = cairo_test_get_reference_image (ctx, base_ref_png_path,
1267                                                     target->content == CAIRO_TEST_CONTENT_COLOR_ALPHA_FLATTENED);
1268         if (cairo_surface_status (ref_image)) {
1269             cairo_test_log (ctx, "Error: Cannot open reference image for %s: %s\n",
1270                             base_ref_png_path,
1271                             cairo_status_to_string (cairo_surface_status (ref_image)));
1272             cairo_surface_destroy (test_image);
1273             ret = CAIRO_TEST_FAILURE;
1274             goto UNWIND_CAIRO;
1275         }
1276
1277         diff_image = cairo_image_surface_create (CAIRO_FORMAT_ARGB32,
1278                                                  ctx->test->width,
1279                                                  ctx->test->height);
1280
1281         cmp_png_path = base_ref_png_path;
1282         diff_status = image_diff (ctx,
1283                                   test_image, ref_image, diff_image,
1284                                   &result);
1285         _xunlink (ctx, diff_png_path);
1286         if (diff_status ||
1287             image_diff_is_failure (&result, target->error_tolerance))
1288         {
1289             /* that failed, so check against the specific backend */
1290             ref_image = cairo_test_get_reference_image (ctx, ref_png_path,
1291                                                         target->content == CAIRO_TEST_CONTENT_COLOR_ALPHA_FLATTENED);
1292             if (cairo_surface_status (ref_image)) {
1293                 cairo_test_log (ctx, "Error: Cannot open reference image for %s: %s\n",
1294                                 ref_png_path,
1295                                 cairo_status_to_string (cairo_surface_status (ref_image)));
1296                 cairo_surface_destroy (test_image);
1297                 ret = CAIRO_TEST_FAILURE;
1298                 goto UNWIND_CAIRO;
1299             }
1300
1301             cmp_png_path = ref_png_path;
1302             diff_status = image_diff (ctx,
1303                                       test_image, ref_image,
1304                                       diff_image,
1305                                       &result);
1306             if (diff_status)
1307             {
1308                 cairo_test_log (ctx, "Error: Failed to compare images: %s\n",
1309                                 cairo_status_to_string (diff_status));
1310                 ret = CAIRO_TEST_FAILURE;
1311             }
1312             else if (image_diff_is_failure (&result, target->error_tolerance))
1313             {
1314                 ret = CAIRO_TEST_FAILURE;
1315
1316                 diff_status = cairo_surface_write_to_png (diff_image,
1317                                                           diff_png_path);
1318                 if (diff_status) {
1319                     cairo_test_log (ctx, "Error: Failed to write differences image: %s\n",
1320                                     cairo_status_to_string (diff_status));
1321                 } else {
1322                     have_result = TRUE;
1323                 }
1324
1325                 cairo_test_copy_file (test_filename, fail_filename);
1326             }
1327             else
1328             { /* success */
1329                 cairo_test_copy_file (test_filename, pass_filename);
1330             }
1331         }
1332         else
1333         { /* success */
1334             cairo_test_copy_file (test_filename, pass_filename);
1335         }
1336
1337         /* If failed, compare against the current image output,
1338          * and attempt to detect systematic failures.
1339          */
1340         if (ret == CAIRO_TEST_FAILURE) {
1341             char *image_out_path;
1342
1343             image_out_path =
1344                 cairo_test_reference_filename (ctx,
1345                                                base_name,
1346                                                ctx->test_name,
1347                                                "image",
1348                                                "image",
1349                                                format,
1350                                                CAIRO_TEST_OUT_SUFFIX,
1351                                                CAIRO_TEST_PNG_EXTENSION);
1352             if (image_out_path != NULL) {
1353                 if (cairo_test_files_equal (out_png_path,
1354                                             image_out_path))
1355                 {
1356                     ret = CAIRO_TEST_XFAILURE;
1357                 }
1358                 else
1359                 {
1360                     ref_image =
1361                         cairo_image_surface_create_from_png (image_out_path);
1362                     if (cairo_surface_status (ref_image) == CAIRO_STATUS_SUCCESS)
1363                     {
1364                         diff_status = image_diff (ctx,
1365                                                   test_image, ref_image,
1366                                                   diff_image,
1367                                                   &result);
1368                         if (diff_status == CAIRO_STATUS_SUCCESS &&
1369                             !image_diff_is_failure (&result, target->error_tolerance))
1370                         {
1371                             ret = CAIRO_TEST_XFAILURE;
1372                         }
1373
1374                         cairo_surface_destroy (ref_image);
1375                     }
1376                 }
1377
1378                 free (image_out_path);
1379             }
1380         }
1381
1382         cairo_surface_destroy (test_image);
1383         cairo_surface_destroy (diff_image);
1384     }
1385
1386     if (cairo_status (cr) != CAIRO_STATUS_SUCCESS) {
1387         cairo_test_log (ctx, "Error: Function under test left cairo status in an error state: %s\n",
1388                         cairo_status_to_string (cairo_status (cr)));
1389         ret = CAIRO_TEST_ERROR;
1390         goto UNWIND_CAIRO;
1391     }
1392
1393 UNWIND_CAIRO:
1394     free (test_filename);
1395     free (fail_filename);
1396     free (pass_filename);
1397
1398     test_filename = fail_filename = pass_filename = NULL;
1399
1400 #if HAVE_MEMFAULT
1401     if (ret == CAIRO_TEST_FAILURE)
1402         MEMFAULT_PRINT_FAULTS ();
1403 #endif
1404     cairo_destroy (cr);
1405 UNWIND_SURFACE:
1406     cairo_surface_destroy (surface);
1407
1408     if (target->cleanup)
1409         target->cleanup (closure);
1410
1411 #if HAVE_MEMFAULT
1412     cairo_debug_reset_static_data ();
1413
1414 #if HAVE_FCFINI
1415     FcFini ();
1416 #endif
1417
1418     if (MEMFAULT_COUNT_LEAKS () > 0) {
1419         if (ret != CAIRO_TEST_FAILURE)
1420             MEMFAULT_PRINT_FAULTS ();
1421         MEMFAULT_PRINT_LEAKS ();
1422     }
1423
1424     if (ret == CAIRO_TEST_SUCCESS && --malloc_failure_iterations > 0)
1425         goto REPEAT;
1426 #endif
1427
1428     if (have_output)
1429         cairo_test_log (ctx, "OUTPUT: %s\n", out_png_path);
1430
1431     if (have_result) {
1432         if (cmp_png_path == NULL) {
1433             /* XXX presume we matched the normal ref last time */
1434             cmp_png_path = ref_png_path;
1435         }
1436         cairo_test_log (ctx,
1437                         "REFERENCE: %s\nDIFFERENCE: %s\n",
1438                         cmp_png_path, diff_png_path);
1439     }
1440
1441 UNWIND_STRINGS:
1442     free (out_png_path);
1443     free (ref_png_path);
1444     free (base_ref_png_path);
1445     free (ref_path);
1446     free (new_png_path);
1447     free (base_new_png_path);
1448     free (new_path);
1449     free (xfail_png_path);
1450     free (base_xfail_png_path);
1451     free (xfail_path);
1452     free (diff_png_path);
1453     free (base_path);
1454     free (base_name);
1455
1456     return ret;
1457 }
1458
1459 #if defined(HAVE_SIGNAL_H) && defined(HAVE_SETJMP_H)
1460 #include <signal.h>
1461 #include <setjmp.h>
1462 /* Used to catch crashes in a test, so that we report it as such and
1463  * continue testing, although one crasher may already have corrupted memory in
1464  * an nonrecoverable fashion. */
1465 static jmp_buf jmpbuf;
1466
1467 static void
1468 segfault_handler (int signal)
1469 {
1470     longjmp (jmpbuf, signal);
1471 }
1472 #endif
1473
1474 cairo_test_status_t
1475 _cairo_test_context_run_for_target (cairo_test_context_t *ctx,
1476                                     const cairo_boilerplate_target_t *target,
1477                                     cairo_bool_t similar,
1478                                     int dev_offset)
1479 {
1480     cairo_test_status_t status;
1481
1482     if (target->get_image_surface == NULL)
1483         return CAIRO_TEST_UNTESTED;
1484
1485     if (similar && ! cairo_test_target_has_similar (ctx, target))
1486         return CAIRO_TEST_UNTESTED;
1487
1488     cairo_test_log (ctx,
1489                     "Testing %s with %s%s target (dev offset %d)\n",
1490                     ctx->test_name,
1491                     similar ? " (similar) " : "",
1492                     target->name,
1493                     dev_offset);
1494
1495     printf ("%s.%s.%s [%d]%s:\t", ctx->test_name, target->name,
1496             cairo_boilerplate_content_name (target->content),
1497             dev_offset,
1498             similar ? " (similar)": "");
1499     fflush (stdout);
1500
1501 #if defined(HAVE_SIGNAL_H) && defined(HAVE_SETJMP_H)
1502     if (! RUNNING_ON_VALGRIND) {
1503         void (* volatile old_segfault_handler)(int);
1504         void (* volatile old_segfpe_handler)(int);
1505         void (* volatile old_sigpipe_handler)(int);
1506         void (* volatile old_sigabrt_handler)(int);
1507         void (* volatile old_sigalrm_handler)(int);
1508
1509         /* Set up a checkpoint to get back to in case of segfaults. */
1510 #ifdef SIGSEGV
1511         old_segfault_handler = signal (SIGSEGV, segfault_handler);
1512 #endif
1513 #ifdef SIGFPE
1514         old_segfpe_handler = signal (SIGFPE, segfault_handler);
1515 #endif
1516 #ifdef SIGPIPE
1517         old_sigpipe_handler = signal (SIGPIPE, segfault_handler);
1518 #endif
1519 #ifdef SIGABRT
1520         old_sigabrt_handler = signal (SIGABRT, segfault_handler);
1521 #endif
1522 #ifdef SIGALRM
1523         old_sigalrm_handler = signal (SIGALRM, segfault_handler);
1524 #endif
1525         if (0 == setjmp (jmpbuf))
1526             status = cairo_test_for_target (ctx, target, dev_offset, similar);
1527         else
1528             status = CAIRO_TEST_CRASHED;
1529 #ifdef SIGSEGV
1530         signal (SIGSEGV, old_segfault_handler);
1531 #endif
1532 #ifdef SIGFPE
1533         signal (SIGFPE, old_segfpe_handler);
1534 #endif
1535 #ifdef SIGPIPE
1536         signal (SIGPIPE, old_sigpipe_handler);
1537 #endif
1538 #ifdef SIGABRT
1539         signal (SIGABRT, old_sigabrt_handler);
1540 #endif
1541 #ifdef SIGALRM
1542         signal (SIGALRM, old_sigalrm_handler);
1543 #endif
1544     } else {
1545         status = cairo_test_for_target (ctx, target, dev_offset, similar);
1546     }
1547 #else
1548     status = cairo_test_for_target (ctx, target, dev_offset, similar);
1549 #endif
1550
1551     cairo_test_log (ctx,
1552                     "TEST: %s TARGET: %s FORMAT: %s OFFSET: %d SIMILAR: %d RESULT: ",
1553                     ctx->test_name, target->name,
1554                     cairo_boilerplate_content_name (target->content),
1555                     dev_offset, similar);
1556     switch (status) {
1557     case CAIRO_TEST_SUCCESS:
1558         printf ("PASS\n");
1559         cairo_test_log (ctx, "PASS\n");
1560         break;
1561
1562     case CAIRO_TEST_UNTESTED:
1563         printf ("UNTESTED\n");
1564         cairo_test_log (ctx, "UNTESTED\n");
1565         break;
1566
1567     default:
1568     case CAIRO_TEST_CRASHED:
1569         if (print_fail_on_stdout) {
1570             printf ("!!!CRASHED!!!\n");
1571         } else {
1572             /* eat the test name */
1573             printf ("\r");
1574             fflush (stdout);
1575         }
1576         cairo_test_log (ctx, "CRASHED\n");
1577         fprintf (stderr, "%s.%s.%s [%d]%s:\t%s!!!CRASHED!!!%s\n",
1578                  ctx->test_name, target->name,
1579                  cairo_boilerplate_content_name (target->content), dev_offset, similar ? " (similar)" : "",
1580                  fail_face, normal_face);
1581         break;
1582
1583     case CAIRO_TEST_ERROR:
1584         if (print_fail_on_stdout) {
1585             printf ("!!!ERROR!!!\n");
1586         } else {
1587             /* eat the test name */
1588             printf ("\r");
1589             fflush (stdout);
1590         }
1591         cairo_test_log (ctx, "ERROR\n");
1592         fprintf (stderr, "%s.%s.%s [%d]%s:\t%s!!!ERROR!!!%s\n",
1593                  ctx->test_name, target->name,
1594                  cairo_boilerplate_content_name (target->content), dev_offset, similar ? " (similar)" : "",
1595                  fail_face, normal_face);
1596         break;
1597
1598     case CAIRO_TEST_XFAILURE:
1599         if (print_fail_on_stdout) {
1600             printf ("XFAIL\n");
1601         } else {
1602             /* eat the test name */
1603             printf ("\r");
1604             fflush (stdout);
1605         }
1606         fprintf (stderr, "%s.%s.%s [%d]%s:\t%sXFAIL%s\n",
1607                  ctx->test_name, target->name,
1608                  cairo_boilerplate_content_name (target->content), dev_offset, similar ? " (similar)" : "",
1609                  xfail_face, normal_face);
1610         cairo_test_log (ctx, "XFAIL\n");
1611         break;
1612
1613     case CAIRO_TEST_NEW:
1614         if (print_fail_on_stdout) {
1615             printf ("NEW\n");
1616         } else {
1617             /* eat the test name */
1618             printf ("\r");
1619             fflush (stdout);
1620         }
1621         fprintf (stderr, "%s.%s.%s [%d]%s:\t%sNEW%s\n",
1622                  ctx->test_name, target->name,
1623                  cairo_boilerplate_content_name (target->content), dev_offset, similar ? " (similar)" : "",
1624                  fail_face, normal_face);
1625         cairo_test_log (ctx, "NEW\n");
1626         break;
1627
1628     case CAIRO_TEST_NO_MEMORY:
1629     case CAIRO_TEST_FAILURE:
1630         if (print_fail_on_stdout) {
1631             printf ("FAIL\n");
1632         } else {
1633             /* eat the test name */
1634             printf ("\r");
1635             fflush (stdout);
1636         }
1637         fprintf (stderr, "%s.%s.%s [%d]%s:\t%sFAIL%s\n",
1638                  ctx->test_name, target->name,
1639                  cairo_boilerplate_content_name (target->content), dev_offset, similar ? " (similar)" : "",
1640                  fail_face, normal_face);
1641         cairo_test_log (ctx, "FAIL\n");
1642         break;
1643     }
1644     fflush (stdout);
1645
1646     return status;
1647 }
1648
1649 const cairo_test_context_t *
1650 cairo_test_get_context (cairo_t *cr)
1651 {
1652     return cairo_get_user_data (cr, &_cairo_test_context_key);
1653 }
1654
1655 cairo_surface_t *
1656 cairo_test_create_surface_from_png (const cairo_test_context_t *ctx,
1657                                     const char *filename)
1658 {
1659     cairo_surface_t *image;
1660     cairo_status_t status;
1661
1662     image = cairo_image_surface_create_from_png (filename);
1663     status = cairo_surface_status (image);
1664     if (status == CAIRO_STATUS_FILE_NOT_FOUND) {
1665         /* expect not found when running with srcdir != builddir
1666          * such as when 'make distcheck' is run
1667          */
1668         if (ctx->srcdir) {
1669             char *srcdir_filename;
1670             xasprintf (&srcdir_filename, "%s/%s", ctx->srcdir, filename);
1671             cairo_surface_destroy (image);
1672             image = cairo_image_surface_create_from_png (srcdir_filename);
1673             free (srcdir_filename);
1674         }
1675     }
1676
1677     return image;
1678 }
1679
1680 cairo_pattern_t *
1681 cairo_test_create_pattern_from_png (const cairo_test_context_t *ctx,
1682                                     const char *filename)
1683 {
1684     cairo_surface_t *image;
1685     cairo_pattern_t *pattern;
1686
1687     image = cairo_test_create_surface_from_png (ctx, filename);
1688
1689     pattern = cairo_pattern_create_for_surface (image);
1690
1691     cairo_pattern_set_extend (pattern, CAIRO_EXTEND_REPEAT);
1692
1693     cairo_surface_destroy (image);
1694
1695     return pattern;
1696 }
1697
1698 static cairo_surface_t *
1699 _draw_check (int width, int height)
1700 {
1701     cairo_surface_t *surface;
1702     cairo_t *cr;
1703
1704     surface = cairo_image_surface_create (CAIRO_FORMAT_RGB24, 12, 12);
1705     cr = cairo_create (surface);
1706     cairo_surface_destroy (surface);
1707
1708     cairo_set_source_rgb (cr, 0.75, 0.75, 0.75); /* light gray */
1709     cairo_paint (cr);
1710
1711     cairo_set_source_rgb (cr, 0.25, 0.25, 0.25); /* dark gray */
1712     cairo_rectangle (cr, width / 2,  0, width / 2, height / 2);
1713     cairo_rectangle (cr, 0, height / 2, width / 2, height / 2);
1714     cairo_fill (cr);
1715
1716     surface = cairo_surface_reference (cairo_get_target (cr));
1717     cairo_destroy (cr);
1718
1719     return surface;
1720 }
1721
1722 void
1723 cairo_test_paint_checkered (cairo_t *cr)
1724 {
1725     cairo_surface_t *check;
1726
1727     check = _draw_check (12, 12);
1728
1729     cairo_save (cr);
1730     cairo_set_source_surface (cr, check, 0, 0);
1731     cairo_surface_destroy (check);
1732
1733     cairo_pattern_set_filter (cairo_get_source (cr), CAIRO_FILTER_NEAREST);
1734     cairo_pattern_set_extend (cairo_get_source (cr), CAIRO_EXTEND_REPEAT);
1735     cairo_paint (cr);
1736
1737     cairo_restore (cr);
1738 }
1739
1740 cairo_bool_t
1741 cairo_test_is_target_enabled (const cairo_test_context_t *ctx,
1742                               const char *target)
1743 {
1744     size_t i;
1745
1746     for (i = 0; i < ctx->num_targets; i++) {
1747         const cairo_boilerplate_target_t *t = ctx->targets_to_test[i];
1748         if (strcmp (t->name, target) == 0) {
1749             /* XXX ask the target whether is it possible to run?
1750              * e.g. the xlib backend could check whether it is able to connect
1751              * to the Display.
1752              */
1753             return t->get_image_surface != NULL;
1754         }
1755     }
1756
1757     return FALSE;
1758 }
1759
1760 cairo_bool_t
1761 cairo_test_malloc_failure (const cairo_test_context_t *ctx,
1762                            cairo_status_t status)
1763 {
1764     if (! ctx->malloc_failure)
1765         return FALSE;
1766
1767     if (status != CAIRO_STATUS_NO_MEMORY)
1768         return FALSE;
1769
1770 #if HAVE_MEMFAULT
1771     {
1772         int n_faults;
1773
1774         /* prevent infinite loops... */
1775         n_faults = MEMFAULT_COUNT_FAULTS ();
1776         if (n_faults == ctx->last_fault_count)
1777             return FALSE;
1778
1779         ((cairo_test_context_t *) ctx)->last_fault_count = n_faults;
1780     }
1781 #endif
1782
1783     return TRUE;
1784 }
1785
1786 cairo_test_status_t
1787 cairo_test_status_from_status (const cairo_test_context_t *ctx,
1788                                cairo_status_t status)
1789 {
1790     if (status == CAIRO_STATUS_SUCCESS)
1791         return CAIRO_TEST_SUCCESS;
1792
1793     if (cairo_test_malloc_failure (ctx, status))
1794         return CAIRO_TEST_NO_MEMORY;
1795
1796     return CAIRO_TEST_FAILURE;
1797 }