test/cairo-test.c

   1 /*
   2  * Copyright © 2004 Red Hat, Inc.
   3  * Copyright © 2008 Chris Wilson
   4  *
   5  * Permission to use, copy, modify, distribute, and sell this software
   6  * and its documentation for any purpose is hereby granted without
   7  * fee, provided that the above copyright notice appear in all copies
   8  * and that both that copyright notice and this permission notice
   9  * appear in supporting documentation, and that the name of
  10  * Red Hat, Inc. not be used in advertising or publicity pertaining to
  11  * distribution of the software without specific, written prior
  12  * permission. Red Hat, Inc. makes no representations about the
  13  * suitability of this software for any purpose.  It is provided "as
  14  * is" without express or implied warranty.
  15  *
  16  * RED HAT, INC. DISCLAIMS ALL WARRANTIES WITH REGARD TO THIS
  17  * SOFTWARE, INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY AND
  18  * FITNESS, IN NO EVENT SHALL RED HAT, INC. BE LIABLE FOR ANY SPECIAL,
  19  * INDIRECT OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER
  20  * RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION
  21  * OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR
  22  * IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
  23  *
  24  * Author: Carl D. Worth <cworth@cworth.org>
  25  *         Chris Wilson <chris@chris-wilson.co.uk>
  26  */
  27
  28 #define _GNU_SOURCE 1   /* for feenableexcept() et al */
  29
  30 #if HAVE_CONFIG_H
  31 #include "config.h"
  32 #endif
  33
  34 #include <stdio.h>
  35 #include <stdlib.h>
  36 #include <stdarg.h>
  37 #include <ctype.h>
  38 #include <assert.h>
  39 #if HAVE_UNISTD_H
  40 #include <unistd.h>
  41 #endif
  42 #include <errno.h>
  43 #include <string.h>
  44 #if HAVE_FCFINI
  45 #include <fontconfig/fontconfig.h>
  46 #endif
  47 #if CAIRO_HAS_REAL_PTHREAD
  48 #include <pthread.h>
  49 #endif
  50 #if HAVE_SYS_STAT_H
  51 #include <sys/stat.h>
  52 #endif
  53
  54 #if HAVE_VALGRIND
  55 #include <valgrind.h>
  56 #else
  57 #define RUNNING_ON_VALGRIND 0
  58 #endif
  59
  60 #if HAVE_MEMFAULT
  61 #include <memfault.h>
  62 #define MF(x) x
  63 #else
  64 #define MF(x)
  65 #endif
  66
  67 #include "cairo-test-private.h"
  68
  69 #include "buffer-diff.h"
  70
  71 #ifdef _MSC_VER
  72 #include <crtdbg.h>
  73 #include <direct.h>
  74 #define F_OK 0
  75 #define HAVE_MKDIR 1
  76 #define mkdir _mkdir
  77 #endif
  78
  79 #ifndef FALSE
  80 #define FALSE 0
  81 #endif
  82 #ifndef TRUE
  83 #define TRUE !FALSE
  84 #endif
  85
  86 #if ! HAVE_ALARM || ! defined(SIGALRM)
  87 #define alarm(X);
  88 #endif
  89
  90 static const cairo_user_data_key_t _cairo_test_context_key;
  91
  92 static void
  93 _xunlink (const cairo_test_context_t *ctx, const char *pathname);
  94
  95 static const char *fail_face = "", *xfail_face="", *normal_face = "";
  96 static cairo_bool_t print_fail_on_stdout;
  97 static int cairo_test_timeout = 60;
  98
  99 #define NUM_DEVICE_OFFSETS 2
 100 #define NUM_DEVICE_SCALE 2
 101
 102 cairo_bool_t
 103 cairo_test_mkdir (const char *path)
 104 {
 105 #if ! HAVE_MKDIR
 106     return FALSE;
 107 #elif HAVE_MKDIR == 1
 108     if (mkdir (path) == 0)
 109         return TRUE;
 110 #elif HAVE_MKDIR == 2
 111     if (mkdir (path, 0770) == 0)
 112         return TRUE;
 113 #else
 114 #error Bad value for HAVE_MKDIR
 115 #endif
 116
 117     return errno == EEXIST;
 118 }
 119
 120 static char *
 121 _cairo_test_fixup_name (const char *original)
 122 {
 123     char *name, *s;
 124
 125     s = name = xstrdup (original);
 126     while ((s = strchr (s, '_')) != NULL)
 127         *s++ = '-';
 128
 129     return name;
 130 }
 131
 132 char *
 133 cairo_test_get_name (const cairo_test_t *test)
 134 {
 135     return _cairo_test_fixup_name (test->name);
 136 }
 137
 138 static void
 139 _cairo_test_init (cairo_test_context_t *ctx,
 140                   const cairo_test_context_t *parent,
 141                   const cairo_test_t *test,
 142                   const char *test_name,
 143                   const char *output)
 144 {
 145     char *log_name;
 146
 147     MF (MEMFAULT_DISABLE_FAULTS ());
 148
 149 #if HAVE_FEENABLEEXCEPT
 150     feenableexcept (FE_DIVBYZERO | FE_INVALID | FE_OVERFLOW);
 151 #endif
 152
 153     ctx->test = test;
 154     ctx->test_name = _cairo_test_fixup_name (test_name);
 155     ctx->output = output;
 156
 157     cairo_test_mkdir (ctx->output);
 158
 159     ctx->malloc_failure = 0;
 160 #if HAVE_MEMFAULT
 161     if (getenv ("CAIRO_TEST_MALLOC_FAILURE"))
 162         ctx->malloc_failure = atoi (getenv ("CAIRO_TEST_MALLOC_FAILURE"));
 163     if (ctx->malloc_failure && ! RUNNING_ON_MEMFAULT ())
 164         ctx->malloc_failure = 0;
 165 #endif
 166
 167     ctx->timeout = cairo_test_timeout;
 168     if (getenv ("CAIRO_TEST_TIMEOUT"))
 169         ctx->timeout = atoi (getenv ("CAIRO_TEST_TIMEOUT"));
 170
 171     xasprintf (&log_name, "%s/%s%s", ctx->output, ctx->test_name, CAIRO_TEST_LOG_SUFFIX);
 172     _xunlink (NULL, log_name);
 173
 174     ctx->log_file = fopen (log_name, "a");
 175     if (ctx->log_file == NULL) {
 176         fprintf (stderr, "Error opening log file: %s\n", log_name);
 177         ctx->log_file = stderr;
 178     }
 179     free (log_name);
 180
 181     ctx->ref_name = NULL;
 182     ctx->ref_image = NULL;
 183     ctx->ref_image_flattened = NULL;
 184
 185     if (parent != NULL) {
 186         ctx->targets_to_test = parent->targets_to_test;
 187         ctx->num_targets = parent->num_targets;
 188         ctx->limited_targets = parent->limited_targets;
 189         ctx->own_targets = FALSE;
 190
 191         ctx->srcdir = parent->srcdir;
 192         ctx->refdir = parent->refdir;
 193     } else {
 194         int tmp_num_targets;
 195         cairo_bool_t tmp_limited_targets;
 196
 197         ctx->targets_to_test = cairo_boilerplate_get_targets (&tmp_num_targets, &tmp_limited_targets);
 198         ctx->num_targets = tmp_num_targets;
 199         ctx->limited_targets = tmp_limited_targets;
 200         ctx->own_targets = TRUE;
 201
 202         ctx->srcdir = getenv ("srcdir");
 203         if (ctx->srcdir == NULL)
 204             ctx->srcdir = ".";
 205
 206         ctx->refdir = getenv ("CAIRO_REF_DIR");
 207     }
 208
 209 #ifdef HAVE_UNISTD_H
 210     if (*fail_face == '\0' && isatty (2)) {
 211         fail_face = "\033[41;37;1m";
 212         xfail_face = "\033[43;37;1m";
 213         normal_face = "\033[m";
 214         if (isatty (1))
 215             print_fail_on_stdout = FALSE;
 216     }
 217 #endif
 218
 219     printf ("\nTESTING %s\n", ctx->test_name);
 220 }
 221
 222 void
 223 _cairo_test_context_init_for_test (cairo_test_context_t *ctx,
 224                                    const cairo_test_context_t *parent,
 225                                    const cairo_test_t *test)
 226 {
 227     _cairo_test_init (ctx, parent, test, test->name, CAIRO_TEST_OUTPUT_DIR);
 228 }
 229
 230 void
 231 cairo_test_init (cairo_test_context_t *ctx,
 232                  const char *test_name,
 233                  const char *output)
 234 {
 235     _cairo_test_init (ctx, NULL, NULL, test_name, output);
 236 }
 237
 238 void
 239 cairo_test_fini (cairo_test_context_t *ctx)
 240 {
 241     if (ctx->log_file == NULL)
 242         return;
 243
 244     if (ctx->log_file != stderr)
 245         fclose (ctx->log_file);
 246     ctx->log_file = NULL;
 247
 248     free (ctx->ref_name);
 249     cairo_surface_destroy (ctx->ref_image);
 250     cairo_surface_destroy (ctx->ref_image_flattened);
 251
 252     if (ctx->test_name != NULL)
 253         free ((char *) ctx->test_name);
 254
 255     if (ctx->own_targets)
 256         cairo_boilerplate_free_targets (ctx->targets_to_test);
 257
 258     cairo_boilerplate_fini ();
 259
 260     cairo_debug_reset_static_data ();
 261 #if HAVE_FCFINI
 262     FcFini ();
 263 #endif
 264 }
 265
 266 void
 267 cairo_test_logv (const cairo_test_context_t *ctx,
 268                 const char *fmt, va_list va)
 269 {
 270     FILE *file = ctx && ctx->log_file ? ctx->log_file : stderr;
 271     vfprintf (file, fmt, va);
 272 }
 273
 274 void
 275 cairo_test_log (const cairo_test_context_t *ctx, const char *fmt, ...)
 276 {
 277     va_list va;
 278
 279     va_start (va, fmt);
 280     cairo_test_logv (ctx, fmt, va);
 281     va_end (va);
 282 }
 283
 284 static void
 285 _xunlink (const cairo_test_context_t *ctx, const char *pathname)
 286 {
 287     if (unlink (pathname) < 0 && errno != ENOENT) {
 288         cairo_test_log (ctx, "Error: Cannot remove %s: %s\n",
 289                         pathname, strerror (errno));
 290         exit (1);
 291     }
 292 }
 293
 294 char *
 295 cairo_test_reference_filename (const cairo_test_context_t *ctx,
 296                                const char *base_name,
 297                                const char *test_name,
 298                                const char *target_name,
 299                                const char *base_target_name,
 300                                const char *format,
 301                                const char *suffix,
 302                                const char *extension)
 303 {
 304     char *ref_name = NULL;
 305
 306     /* First look for a previous build for comparison. */
 307     if (ctx->refdir != NULL && strcmp(suffix, CAIRO_TEST_REF_SUFFIX) == 0) {
 308         xasprintf (&ref_name, "%s/%s" CAIRO_TEST_OUT_SUFFIX "%s",
 309                    ctx->refdir,
 310                    base_name,
 311                    extension);
 312         if (access (ref_name, F_OK) != 0)
 313             free (ref_name);
 314         else
 315             goto done;
 316     }
 317
 318     if (target_name != NULL) {
 319         /* Next look for a target/format-specific reference image. */
 320         xasprintf (&ref_name, "%s/reference/%s.%s.%s%s%s",
 321                    ctx->srcdir,
 322                    test_name,
 323                    target_name,
 324                    format,
 325                    suffix,
 326                    extension);
 327         if (access (ref_name, F_OK) != 0)
 328             free (ref_name);
 329         else
 330             goto done;
 331
 332         /* Next, look for target-specific reference image. */
 333         xasprintf (&ref_name, "%s/reference/%s.%s%s%s",
 334                    ctx->srcdir,
 335                    test_name,
 336                    target_name,
 337                    suffix,
 338                    extension);
 339         if (access (ref_name, F_OK) != 0)
 340             free (ref_name);
 341         else
 342             goto done;
 343     }
 344
 345     if (base_target_name != NULL) {
 346         /* Next look for a base/format-specific reference image. */
 347         xasprintf (&ref_name, "%s/reference/%s.%s.%s%s%s",
 348                    ctx->srcdir,
 349                    test_name,
 350                    base_target_name,
 351                    format,
 352                    suffix,
 353                    extension);
 354         if (access (ref_name, F_OK) != 0)
 355             free (ref_name);
 356         else
 357             goto done;
 358
 359         /* Next, look for base-specific reference image. */
 360         xasprintf (&ref_name, "%s/reference/%s.%s%s%s",
 361                    ctx->srcdir,
 362                    test_name,
 363                    base_target_name,
 364                    suffix,
 365                    extension);
 366         if (access (ref_name, F_OK) != 0)
 367             free (ref_name);
 368         else
 369             goto done;
 370     }
 371
 372     /* Next, look for format-specific reference image. */
 373     xasprintf (&ref_name, "%s/reference/%s.%s%s%s",
 374                ctx->srcdir,
 375                test_name,
 376                format,
 377                suffix,
 378                extension);
 379     if (access (ref_name, F_OK) != 0)
 380         free (ref_name);
 381     else
 382         goto done;
 383
 384     /* Finally, look for the standard reference image. */
 385     xasprintf (&ref_name, "%s/reference/%s%s%s", ctx->srcdir,
 386                test_name,
 387                suffix,
 388                extension);
 389     if (access (ref_name, F_OK) != 0)
 390         free (ref_name);
 391     else
 392         goto done;
 393
 394     ref_name = NULL;
 395
 396 done:
 397     return ref_name;
 398 }
 399
 400 cairo_test_similar_t
 401 cairo_test_target_has_similar (const cairo_test_context_t *ctx,
 402                                const cairo_boilerplate_target_t *target)
 403 {
 404     cairo_surface_t *surface;
 405     cairo_test_similar_t has_similar;
 406     cairo_t * cr;
 407     cairo_surface_t *similar;
 408     cairo_status_t status;
 409     void *closure;
 410     char *path;
 411
 412     /* ignore image intermediate targets */
 413     if (target->expected_type == CAIRO_SURFACE_TYPE_IMAGE)
 414         return DIRECT;
 415
 416     if (getenv ("CAIRO_TEST_IGNORE_SIMILAR"))
 417         return DIRECT;
 418
 419     xasprintf (&path, "%s/%s",
 420                cairo_test_mkdir (ctx->output) ? ctx->output : ".",
 421                ctx->test_name);
 422
 423     has_similar = DIRECT;
 424     do {
 425         do {
 426             surface = (target->create_surface) (path,
 427                                                 target->content,
 428                                                 ctx->test->width,
 429                                                 ctx->test->height,
 430                                                 ctx->test->width* NUM_DEVICE_SCALE + 25 * NUM_DEVICE_OFFSETS,
 431                                                 ctx->test->height* NUM_DEVICE_SCALE + 25 * NUM_DEVICE_OFFSETS,
 432                                                 CAIRO_BOILERPLATE_MODE_TEST,
 433                                                 &closure);
 434             if (surface == NULL)
 435                 goto out;
 436         } while (cairo_test_malloc_failure (ctx, cairo_surface_status (surface)));
 437
 438         if (cairo_surface_status (surface))
 439             goto out;
 440
 441         cr = cairo_create (surface);
 442         cairo_push_group_with_content (cr,
 443                                        cairo_boilerplate_content (target->content));
 444         similar = cairo_get_group_target (cr);
 445         status = cairo_surface_status (similar);
 446
 447         if (cairo_surface_get_type (similar) == cairo_surface_get_type (surface))
 448             has_similar = SIMILAR;
 449         else
 450             has_similar = DIRECT;
 451
 452         cairo_destroy (cr);
 453         cairo_surface_destroy (surface);
 454
 455         if (target->cleanup)
 456             target->cleanup (closure);
 457     } while (! has_similar && cairo_test_malloc_failure (ctx, status));
 458 out:
 459     free (path);
 460
 461     return has_similar;
 462 }
 463
 464 static cairo_surface_t *
 465 _cairo_test_flatten_reference_image (cairo_test_context_t *ctx,
 466                                      cairo_bool_t flatten)
 467 {
 468     cairo_surface_t *surface;
 469     cairo_t *cr;
 470
 471     if (! flatten)
 472         return ctx->ref_image;
 473
 474     if (ctx->ref_image_flattened != NULL)
 475         return ctx->ref_image_flattened;
 476
 477     surface = cairo_image_surface_create (CAIRO_FORMAT_ARGB32,
 478                                           cairo_image_surface_get_width (ctx->ref_image),
 479                                           cairo_image_surface_get_height (ctx->ref_image));
 480     cr = cairo_create (surface);
 481     cairo_surface_destroy (surface);
 482
 483     cairo_set_source_rgb (cr, 1, 1, 1);
 484     cairo_paint (cr);
 485
 486     cairo_set_source_surface (cr, ctx->ref_image, 0, 0);
 487     cairo_paint (cr);
 488
 489     surface = cairo_surface_reference (cairo_get_target (cr));
 490     cairo_destroy (cr);
 491
 492     if (cairo_surface_status (surface) == CAIRO_STATUS_SUCCESS)
 493         ctx->ref_image_flattened = surface;
 494     return surface;
 495 }
 496
 497 cairo_surface_t *
 498 cairo_test_get_reference_image (cairo_test_context_t *ctx,
 499                                 const char *filename,
 500                                 cairo_bool_t flatten)
 501 {
 502     cairo_surface_t *surface;
 503
 504     if (ctx->ref_name != NULL) {
 505         if (strcmp (ctx->ref_name, filename) == 0)
 506             return _cairo_test_flatten_reference_image (ctx, flatten);
 507
 508         cairo_surface_destroy (ctx->ref_image);
 509         ctx->ref_image = NULL;
 510
 511         cairo_surface_destroy (ctx->ref_image_flattened);
 512         ctx->ref_image_flattened = NULL;
 513
 514         free (ctx->ref_name);
 515         ctx->ref_name = NULL;
 516     }
 517
 518     surface = cairo_image_surface_create_from_png (filename);
 519     if (cairo_surface_status (surface))
 520         return surface;
 521
 522     ctx->ref_name = xstrdup (filename);
 523     ctx->ref_image = surface;
 524     return _cairo_test_flatten_reference_image (ctx, flatten);
 525 }
 526
 527 static cairo_bool_t
 528 cairo_test_file_is_older (const char *filename,
 529                           char **ref_filenames,
 530                           int num_ref_filenames)
 531 {
 532 #if HAVE_SYS_STAT_H
 533     struct stat st;
 534
 535     if (stat (filename, &st) < 0)
 536         return FALSE;
 537
 538     while (num_ref_filenames--) {
 539         struct stat ref;
 540         char *ref_filename = *ref_filenames++;
 541
 542         if (ref_filename == NULL)
 543             continue;
 544
 545         if (stat (ref_filename++, &ref) < 0)
 546             continue;
 547
 548         if (st.st_mtime <= ref.st_mtime)
 549             return TRUE;
 550     }
 551 #endif
 552
 553     return FALSE;
 554 }
 555
 556 static cairo_bool_t
 557 cairo_test_files_equal (const char *test_filename,
 558                         const char *pass_filename)
 559 {
 560     FILE *test, *pass;
 561     int t, p;
 562
 563     if (test_filename == NULL || pass_filename == NULL)
 564         return FALSE;
 565
 566     test = fopen (test_filename, "rb");
 567     if (test == NULL)
 568         return FALSE;
 569
 570     pass = fopen (pass_filename, "rb");
 571     if (pass == NULL) {
 572         fclose (test);
 573         return FALSE;
 574     }
 575
 576     /* as simple as it gets */
 577     do {
 578         t = getc (test);
 579         p = getc (pass);
 580         if (t != p)
 581             break;
 582     } while (t != EOF && p != EOF);
 583
 584     fclose (pass);
 585     fclose (test);
 586
 587     return t == p; /* both EOF */
 588 }
 589
 590 static cairo_bool_t
 591 cairo_test_copy_file (const char *src_filename,
 592                       const char *dst_filename)
 593 {
 594     FILE *src, *dst;
 595     int c;
 596
 597 #if HAVE_LINK
 598     if (link (src_filename, dst_filename) == 0)
 599         return TRUE;
 600
 601     unlink (dst_filename);
 602 #endif
 603
 604     src = fopen (src_filename, "rb");
 605     if (src == NULL)
 606         return FALSE;
 607
 608     dst = fopen (dst_filename, "wb");
 609     if (dst == NULL) {
 610         fclose (src);
 611         return FALSE;
 612     }
 613
 614     /* as simple as it gets */
 615     while ((c = getc (src)) != EOF)
 616         putc (c, dst);
 617
 618     fclose (src);
 619     fclose (dst);
 620
 621     return TRUE;
 622 }
 623
 624 static cairo_test_status_t
 625 cairo_test_for_target (cairo_test_context_t              *ctx,
 626                        const cairo_boilerplate_target_t  *target,
 627                        int                                dev_offset,
 628                        int                                dev_scale,
 629                        cairo_bool_t                       similar)
 630 {
 631     cairo_test_status_t status;
 632     cairo_surface_t *surface = NULL;
 633     cairo_t *cr;
 634     const char *empty_str = "";
 635     char *offset_str;
 636     char *scale_str;
 637     char *base_name, *base_path;
 638     char *out_png_path;
 639     char *ref_path = NULL, *ref_png_path, *cmp_png_path = NULL;
 640     char *new_path = NULL, *new_png_path;
 641     char *xfail_path = NULL, *xfail_png_path;
 642     char *base_ref_png_path;
 643     char *base_new_png_path;
 644     char *base_xfail_png_path;
 645     char *diff_png_path;
 646     char *test_filename = NULL, *pass_filename = NULL, *fail_filename = NULL;
 647     cairo_test_status_t ret;
 648     cairo_content_t expected_content;
 649     cairo_font_options_t *font_options;
 650     const char *format;
 651     cairo_bool_t have_output = FALSE;
 652     cairo_bool_t have_result = FALSE;
 653     void *closure;
 654     double width, height;
 655     cairo_bool_t have_output_dir;
 656 #if HAVE_MEMFAULT
 657     int malloc_failure_iterations = ctx->malloc_failure;
 658     int last_fault_count = 0;
 659 #endif
 660
 661     /* Get the strings ready that we'll need. */
 662     format = cairo_boilerplate_content_name (target->content);
 663     if (dev_offset)
 664         xasprintf (&offset_str, ".%d", dev_offset);
 665     else
 666         offset_str = (char *) empty_str;
 667
 668     if (dev_scale != 1)
 669         xasprintf (&scale_str, ".x%d", dev_scale);
 670     else
 671         scale_str = (char *) empty_str;
 672
 673     xasprintf (&base_name, "%s.%s.%s%s%s%s",
 674                ctx->test_name,
 675                target->name,
 676                format,
 677                similar ? ".similar" : "",
 678                offset_str,
 679                scale_str);
 680
 681     if (offset_str != empty_str)
 682       free (offset_str);
 683     if (scale_str != empty_str)
 684       free (scale_str);
 685
 686     ref_png_path = cairo_test_reference_filename (ctx,
 687                                                   base_name,
 688                                                   ctx->test_name,
 689                                                   target->name,
 690                                                   target->basename,
 691                                                   format,
 692                                                   CAIRO_TEST_REF_SUFFIX,
 693                                                   CAIRO_TEST_PNG_EXTENSION);
 694     new_png_path = cairo_test_reference_filename (ctx,
 695                                                   base_name,
 696                                                   ctx->test_name,
 697                                                   target->name,
 698                                                   target->basename,
 699                                                   format,
 700                                                   CAIRO_TEST_NEW_SUFFIX,
 701                                                   CAIRO_TEST_PNG_EXTENSION);
 702     xfail_png_path = cairo_test_reference_filename (ctx,
 703                                                     base_name,
 704                                                     ctx->test_name,
 705                                                     target->name,
 706                                                     target->basename,
 707                                                     format,
 708                                                     CAIRO_TEST_XFAIL_SUFFIX,
 709                                                     CAIRO_TEST_PNG_EXTENSION);
 710
 711     base_ref_png_path = cairo_test_reference_filename (ctx,
 712                                                   base_name,
 713                                                   ctx->test_name,
 714                                                   NULL, NULL,
 715                                                   format,
 716                                                   CAIRO_TEST_REF_SUFFIX,
 717                                                   CAIRO_TEST_PNG_EXTENSION);
 718     base_new_png_path = cairo_test_reference_filename (ctx,
 719                                                   base_name,
 720                                                   ctx->test_name,
 721                                                   NULL, NULL,
 722                                                   format,
 723                                                   CAIRO_TEST_NEW_SUFFIX,
 724                                                   CAIRO_TEST_PNG_EXTENSION);
 725     base_xfail_png_path = cairo_test_reference_filename (ctx,
 726                                                     base_name,
 727                                                     ctx->test_name,
 728                                                     NULL, NULL,
 729                                                     format,
 730                                                     CAIRO_TEST_XFAIL_SUFFIX,
 731                                                     CAIRO_TEST_PNG_EXTENSION);
 732
 733     if (target->file_extension != NULL) {
 734         ref_path = cairo_test_reference_filename (ctx,
 735                                                   base_name,
 736                                                   ctx->test_name,
 737                                                   target->name,
 738                                                   target->basename,
 739                                                   format,
 740                                                   CAIRO_TEST_REF_SUFFIX,
 741                                                   target->file_extension);
 742         new_path = cairo_test_reference_filename (ctx,
 743                                                   base_name,
 744                                                   ctx->test_name,
 745                                                   target->name,
 746                                                   target->basename,
 747                                                   format,
 748                                                   CAIRO_TEST_NEW_SUFFIX,
 749                                                   target->file_extension);
 750         xfail_path = cairo_test_reference_filename (ctx,
 751                                                     base_name,
 752                                                     ctx->test_name,
 753                                                     target->name,
 754                                                     target->basename,
 755                                                     format,
 756                                                     CAIRO_TEST_XFAIL_SUFFIX,
 757                                                     target->file_extension);
 758     }
 759
 760     have_output_dir = cairo_test_mkdir (ctx->output);
 761     xasprintf (&base_path, "%s/%s",
 762                have_output_dir ? ctx->output : ".",
 763                base_name);
 764     xasprintf (&out_png_path, "%s" CAIRO_TEST_OUT_PNG, base_path);
 765     xasprintf (&diff_png_path, "%s" CAIRO_TEST_DIFF_PNG, base_path);
 766
 767     if (ctx->test->requirements != NULL) {
 768         const char *required;
 769
 770         required = target->is_vector ? "target=raster" : "target=vector";
 771         if (strstr (ctx->test->requirements, required) != NULL) {
 772             cairo_test_log (ctx, "Error: Skipping for %s target %s\n",
 773                             target->is_vector ? "vector" : "raster",
 774                             target->name);
 775             ret = CAIRO_TEST_UNTESTED;
 776             goto UNWIND_STRINGS;
 777         }
 778
 779         required = target->is_recording ? "target=!recording" : "target=recording";
 780         if (strstr (ctx->test->requirements, required) != NULL) {
 781             cairo_test_log (ctx, "Error: Skipping for %s target %s\n",
 782                             target->is_recording ? "recording" : "non-recording",
 783                             target->name);
 784             ret = CAIRO_TEST_UNTESTED;
 785             goto UNWIND_STRINGS;
 786         }
 787     }
 788
 789     width = ctx->test->width;
 790     height = ctx->test->height;
 791     if (width && height) {
 792         width *= dev_scale;
 793         height *= dev_scale;
 794         width += dev_offset;
 795         height += dev_offset;
 796     }
 797
 798 #if HAVE_MEMFAULT
 799 REPEAT:
 800     MEMFAULT_CLEAR_FAULTS ();
 801     MEMFAULT_RESET_LEAKS ();
 802     ctx->last_fault_count = 0;
 803     last_fault_count = MEMFAULT_COUNT_FAULTS ();
 804
 805     /* Pre-initialise fontconfig so that the configuration is loaded without
 806      * malloc failures (our primary goal is to test cairo fault tolerance).
 807      */
 808 #if HAVE_FCINIT
 809     FcInit ();
 810 #endif
 811
 812     MEMFAULT_ENABLE_FAULTS ();
 813 #endif
 814     have_output = FALSE;
 815     have_result = FALSE;
 816
 817     /* Run the actual drawing code. */
 818     ret = CAIRO_TEST_SUCCESS;
 819     surface = (target->create_surface) (base_path,
 820                                         target->content,
 821                                         width, height,
 822                                         ctx->test->width * NUM_DEVICE_SCALE + 25 * NUM_DEVICE_OFFSETS,
 823                                         ctx->test->height * NUM_DEVICE_SCALE + 25 * NUM_DEVICE_OFFSETS,
 824                                         CAIRO_BOILERPLATE_MODE_TEST,
 825                                         &closure);
 826     if (surface == NULL) {
 827         cairo_test_log (ctx, "Error: Failed to set %s target\n", target->name);
 828         ret = CAIRO_TEST_UNTESTED;
 829         goto UNWIND_STRINGS;
 830     }
 831
 832 #if HAVE_MEMFAULT
 833     if (ctx->malloc_failure &&
 834         MEMFAULT_COUNT_FAULTS () - last_fault_count > 0 &&
 835         cairo_surface_status (surface) == CAIRO_STATUS_NO_MEMORY)
 836     {
 837         goto REPEAT;
 838     }
 839 #endif
 840
 841     if (cairo_surface_status (surface)) {
 842         MF (MEMFAULT_PRINT_FAULTS ());
 843         cairo_test_log (ctx, "Error: Created an error surface: %s\n",
 844                         cairo_status_to_string (cairo_surface_status (surface)));
 845         ret = CAIRO_TEST_FAILURE;
 846         goto UNWIND_STRINGS;
 847     }
 848
 849     /* Check that we created a surface of the expected type. */
 850     if (cairo_surface_get_type (surface) != target->expected_type) {
 851         MF (MEMFAULT_PRINT_FAULTS ());
 852         cairo_test_log (ctx, "Error: Created surface is of type %d (expected %d)\n",
 853                         cairo_surface_get_type (surface), target->expected_type);
 854         ret = CAIRO_TEST_UNTESTED;
 855         goto UNWIND_SURFACE;
 856     }
 857
 858     /* Check that we created a surface of the expected content,
 859      * (ignore the artificial CAIRO_TEST_CONTENT_COLOR_ALPHA_FLATTENED value).
 860      */
 861     expected_content = cairo_boilerplate_content (target->content);
 862
 863     if (cairo_surface_get_content (surface) != expected_content) {
 864         MF (MEMFAULT_PRINT_FAULTS ());
 865         cairo_test_log (ctx, "Error: Created surface has content %d (expected %d)\n",
 866                         cairo_surface_get_content (surface), expected_content);
 867         ret = CAIRO_TEST_FAILURE;
 868         goto UNWIND_SURFACE;
 869     }
 870
 871     if (cairo_surface_set_user_data (surface,
 872                                      &cairo_boilerplate_output_basename_key,
 873                                      base_path,
 874                                      NULL))
 875     {
 876 #if HAVE_MEMFAULT
 877         cairo_surface_destroy (surface);
 878
 879         if (target->cleanup)
 880             target->cleanup (closure);
 881
 882         goto REPEAT;
 883 #else
 884         ret = CAIRO_TEST_FAILURE;
 885         goto UNWIND_SURFACE;
 886 #endif
 887     }
 888
 889     cairo_surface_set_device_offset (surface, dev_offset, dev_offset);
 890     cairo_surface_set_device_scale (surface, dev_scale, dev_scale);
 891
 892     cr = cairo_create (surface);
 893     if (cairo_set_user_data (cr, &_cairo_test_context_key, (void*) ctx, NULL)) {
 894 #if HAVE_MEMFAULT
 895         cairo_destroy (cr);
 896         cairo_surface_destroy (surface);
 897
 898         if (target->cleanup)
 899             target->cleanup (closure);
 900
 901         goto REPEAT;
 902 #else
 903         ret = CAIRO_TEST_FAILURE;
 904         goto UNWIND_CAIRO;
 905 #endif
 906     }
 907
 908     if (similar)
 909         cairo_push_group_with_content (cr, expected_content);
 910
 911     /* Clear to transparent (or black) depending on whether the target
 912      * surface supports alpha. */
 913     cairo_save (cr);
 914     cairo_set_operator (cr, CAIRO_OPERATOR_CLEAR);
 915     cairo_paint (cr);
 916     cairo_restore (cr);
 917
 918     /* Set all components of font_options to avoid backend differences
 919      * and reduce number of needed reference images. */
 920     font_options = cairo_font_options_create ();
 921     cairo_font_options_set_hint_style (font_options, CAIRO_HINT_STYLE_NONE);
 922     cairo_font_options_set_hint_metrics (font_options, CAIRO_HINT_METRICS_ON);
 923     cairo_font_options_set_antialias (font_options, CAIRO_ANTIALIAS_GRAY);
 924     cairo_set_font_options (cr, font_options);
 925     cairo_font_options_destroy (font_options);
 926
 927     cairo_save (cr);
 928     alarm (ctx->timeout);
 929     status = (ctx->test->draw) (cr, ctx->test->width, ctx->test->height);
 930     alarm (0);
 931     cairo_restore (cr);
 932
 933     if (similar) {
 934         cairo_pop_group_to_source (cr);
 935         cairo_set_operator (cr, CAIRO_OPERATOR_SOURCE);
 936         cairo_paint (cr);
 937     }
 938
 939 #if HAVE_MEMFAULT
 940     MEMFAULT_DISABLE_FAULTS ();
 941
 942     /* repeat test after malloc failure injection */
 943     if (ctx->malloc_failure &&
 944         MEMFAULT_COUNT_FAULTS () - last_fault_count > 0 &&
 945         (status == CAIRO_TEST_NO_MEMORY ||
 946          cairo_status (cr) == CAIRO_STATUS_NO_MEMORY ||
 947          cairo_surface_status (surface) == CAIRO_STATUS_NO_MEMORY))
 948     {
 949         cairo_destroy (cr);
 950         cairo_surface_destroy (surface);
 951         if (target->cleanup)
 952             target->cleanup (closure);
 953         cairo_debug_reset_static_data ();
 954 #if HAVE_FCFINI
 955         FcFini ();
 956 #endif
 957         if (MEMFAULT_COUNT_LEAKS () > 0) {
 958             MEMFAULT_PRINT_FAULTS ();
 959             MEMFAULT_PRINT_LEAKS ();
 960         }
 961
 962         goto REPEAT;
 963     }
 964 #endif
 965
 966     /* Then, check all the different ways it could fail. */
 967     if (status) {
 968         cairo_test_log (ctx, "Error: Function under test failed\n");
 969         ret = status;
 970         goto UNWIND_CAIRO;
 971     }
 972
 973 #if HAVE_MEMFAULT
 974     if (MEMFAULT_COUNT_FAULTS () - last_fault_count > 0 &&
 975         MEMFAULT_HAS_FAULTS ())
 976     {
 977         VALGRIND_PRINTF ("Unreported memfaults...");
 978         MEMFAULT_PRINT_FAULTS ();
 979     }
 980 #endif
 981
 982     if (target->finish_surface != NULL) {
 983 #if HAVE_MEMFAULT
 984         /* We need to re-enable faults as most recording-surface processing
 985          * is done during cairo_surface_finish().
 986          */
 987         MEMFAULT_CLEAR_FAULTS ();
 988         last_fault_count = MEMFAULT_COUNT_FAULTS ();
 989         MEMFAULT_ENABLE_FAULTS ();
 990 #endif
 991
 992         /* also check for infinite loops whilst replaying */
 993         alarm (ctx->timeout);
 994         status = target->finish_surface (surface);
 995         alarm (0);
 996
 997 #if HAVE_MEMFAULT
 998         MEMFAULT_DISABLE_FAULTS ();
 999
1000         if (ctx->malloc_failure &&
1001             MEMFAULT_COUNT_FAULTS () - last_fault_count > 0 &&
1002             status == CAIRO_STATUS_NO_MEMORY)
1003         {
1004             cairo_destroy (cr);
1005             cairo_surface_destroy (surface);
1006             if (target->cleanup)
1007                 target->cleanup (closure);
1008             cairo_debug_reset_static_data ();
1009 #if HAVE_FCFINI
1010             FcFini ();
1011 #endif
1012             if (MEMFAULT_COUNT_LEAKS () > 0) {
1013                 MEMFAULT_PRINT_FAULTS ();
1014                 MEMFAULT_PRINT_LEAKS ();
1015             }
1016
1017             goto REPEAT;
1018         }
1019 #endif
1020         if (status) {
1021             cairo_test_log (ctx, "Error: Failed to finish surface: %s\n",
1022                             cairo_status_to_string (status));
1023             ret = CAIRO_TEST_FAILURE;
1024             goto UNWIND_CAIRO;
1025         }
1026     }
1027
1028     /* Skip image check for tests with no image (width,height == 0,0) */
1029     if (ctx->test->width != 0 && ctx->test->height != 0) {
1030         cairo_surface_t *ref_image;
1031         cairo_surface_t *test_image;
1032         cairo_surface_t *diff_image;
1033         buffer_diff_result_t result;
1034         cairo_status_t diff_status;
1035
1036         if (ref_png_path == NULL) {
1037             cairo_test_log (ctx, "Error: Cannot find reference image for %s\n",
1038                             base_name);
1039
1040             /* we may be running this test to generate reference images */
1041             _xunlink (ctx, out_png_path);
1042             /* be more generous as we may need to use external renderers */
1043             alarm (4 * ctx->timeout);
1044             test_image = target->get_image_surface (surface, 0,
1045                                                     ctx->test->width,
1046                                                     ctx->test->height);
1047             alarm (0);
1048             diff_status = cairo_surface_write_to_png (test_image, out_png_path);
1049             cairo_surface_destroy (test_image);
1050             if (diff_status) {
1051                 if (cairo_surface_status (test_image) == CAIRO_STATUS_INVALID_STATUS)
1052                     ret = CAIRO_TEST_CRASHED;
1053                 else
1054                     ret = CAIRO_TEST_FAILURE;
1055                 cairo_test_log (ctx,
1056                                 "Error: Failed to write output image: %s\n",
1057                                 cairo_status_to_string (diff_status));
1058             }
1059             have_output = TRUE;
1060
1061             ret = CAIRO_TEST_XFAILURE;
1062             goto UNWIND_CAIRO;
1063         }
1064
1065         if (target->file_extension != NULL) { /* compare vector surfaces */
1066             char *filenames[] = {
1067                 ref_png_path,
1068                 ref_path,
1069                 new_png_path,
1070                 new_path,
1071                 xfail_png_path,
1072                 xfail_path,
1073                 base_ref_png_path,
1074                 base_new_png_path,
1075                 base_xfail_png_path,
1076             };
1077
1078             xasprintf (&test_filename, "%s.out%s",
1079                        base_path, target->file_extension);
1080             xasprintf (&pass_filename, "%s.pass%s",
1081                        base_path, target->file_extension);
1082             xasprintf (&fail_filename, "%s.fail%s",
1083                        base_path, target->file_extension);
1084
1085             if (cairo_test_file_is_older (pass_filename,
1086                                           filenames,
1087                                           ARRAY_LENGTH (filenames)))
1088             {
1089                 _xunlink (ctx, pass_filename);
1090             }
1091             if (cairo_test_file_is_older (fail_filename,
1092                                           filenames,
1093                                           ARRAY_LENGTH (filenames)))
1094             {
1095                 _xunlink (ctx, fail_filename);
1096             }
1097
1098             if (cairo_test_files_equal (out_png_path, ref_path)) {
1099                 cairo_test_log (ctx, "Vector surface matches reference.\n");
1100                 have_output = FALSE;
1101                 ret = CAIRO_TEST_SUCCESS;
1102                 goto UNWIND_CAIRO;
1103             }
1104             if (cairo_test_files_equal (out_png_path, new_path)) {
1105                 cairo_test_log (ctx, "Vector surface matches current failure.\n");
1106                 have_output = FALSE;
1107                 ret = CAIRO_TEST_NEW;
1108                 goto UNWIND_CAIRO;
1109             }
1110             if (cairo_test_files_equal (out_png_path, xfail_path)) {
1111                 cairo_test_log (ctx, "Vector surface matches known failure.\n");
1112                 have_output = FALSE;
1113                 ret = CAIRO_TEST_XFAILURE;
1114                 goto UNWIND_CAIRO;
1115             }
1116
1117             if (cairo_test_files_equal (test_filename, pass_filename)) {
1118                 /* identical output as last known PASS */
1119                 cairo_test_log (ctx, "Vector surface matches last pass.\n");
1120                 have_output = TRUE;
1121                 ret = CAIRO_TEST_SUCCESS;
1122                 goto UNWIND_CAIRO;
1123             }
1124             if (cairo_test_files_equal (test_filename, fail_filename)) {
1125                 /* identical output as last known FAIL, fail */
1126                 cairo_test_log (ctx, "Vector surface matches last fail.\n");
1127                 have_result = TRUE; /* presume these were kept around as well */
1128                 have_output = TRUE;
1129                 ret = CAIRO_TEST_FAILURE;
1130                 goto UNWIND_CAIRO;
1131             }
1132         }
1133
1134         /* be more generous as we may need to use external renderers */
1135         alarm (4 * ctx->timeout);
1136         test_image = target->get_image_surface (surface, 0,
1137                                                 ctx->test->width,
1138                                                 ctx->test->height);
1139         alarm (0);
1140         if (cairo_surface_status (test_image)) {
1141             cairo_test_log (ctx, "Error: Failed to extract image: %s\n",
1142                             cairo_status_to_string (cairo_surface_status (test_image)));
1143             if (cairo_surface_status (test_image) == CAIRO_STATUS_INVALID_STATUS)
1144                 ret = CAIRO_TEST_CRASHED;
1145             else
1146                 ret = CAIRO_TEST_FAILURE;
1147             cairo_surface_destroy (test_image);
1148             goto UNWIND_CAIRO;
1149         }
1150
1151         _xunlink (ctx, out_png_path);
1152         diff_status = cairo_surface_write_to_png (test_image, out_png_path);
1153         if (diff_status) {
1154             cairo_test_log (ctx, "Error: Failed to write output image: %s\n",
1155                             cairo_status_to_string (diff_status));
1156             cairo_surface_destroy (test_image);
1157             ret = CAIRO_TEST_FAILURE;
1158             goto UNWIND_CAIRO;
1159         }
1160         have_output = TRUE;
1161
1162         /* binary compare png files (no decompression) */
1163         if (target->file_extension == NULL) {
1164             char *filenames[] = {
1165                 ref_png_path,
1166                 new_png_path,
1167                 xfail_png_path,
1168                 base_ref_png_path,
1169                 base_new_png_path,
1170                 base_xfail_png_path,
1171             };
1172
1173             xasprintf (&test_filename, "%s", out_png_path);
1174             xasprintf (&pass_filename, "%s.pass.png", base_path);
1175             xasprintf (&fail_filename, "%s.fail.png", base_path);
1176
1177             if (cairo_test_file_is_older (pass_filename,
1178                                           filenames,
1179                                           ARRAY_LENGTH (filenames)))
1180             {
1181                 _xunlink (ctx, pass_filename);
1182             }
1183             if (cairo_test_file_is_older (fail_filename,
1184                                           filenames,
1185                                           ARRAY_LENGTH (filenames)))
1186             {
1187                 _xunlink (ctx, fail_filename);
1188             }
1189
1190             if (cairo_test_files_equal (test_filename, pass_filename)) {
1191                 cairo_test_log (ctx, "PNG file exactly matches last pass.\n");
1192                 have_result = TRUE;
1193                 cairo_surface_destroy (test_image);
1194                 ret = CAIRO_TEST_SUCCESS;
1195                 goto UNWIND_CAIRO;
1196             }
1197             if (cairo_test_files_equal (out_png_path, ref_png_path)) {
1198                 cairo_test_log (ctx, "PNG file exactly matches reference image.\n");
1199                 have_result = TRUE;
1200                 cairo_surface_destroy (test_image);
1201                 ret = CAIRO_TEST_SUCCESS;
1202                 goto UNWIND_CAIRO;
1203             }
1204             if (cairo_test_files_equal (out_png_path, new_png_path)) {
1205                 cairo_test_log (ctx, "PNG file exactly matches current failure image.\n");
1206                 have_result = TRUE;
1207                 cairo_surface_destroy (test_image);
1208                 ret = CAIRO_TEST_NEW;
1209                 goto UNWIND_CAIRO;
1210             }
1211             if (cairo_test_files_equal (out_png_path, xfail_png_path)) {
1212                 cairo_test_log (ctx, "PNG file exactly matches known failure image.\n");
1213                 have_result = TRUE;
1214                 cairo_surface_destroy (test_image);
1215                 ret = CAIRO_TEST_XFAILURE;
1216                 goto UNWIND_CAIRO;
1217             }
1218             if (cairo_test_files_equal (test_filename, fail_filename)) {
1219                 cairo_test_log (ctx, "PNG file exactly matches last fail.\n");
1220                 have_result = TRUE; /* presume these were kept around as well */
1221                 cairo_surface_destroy (test_image);
1222                 ret = CAIRO_TEST_FAILURE;
1223                 goto UNWIND_CAIRO;
1224             }
1225         } else {
1226             if (cairo_test_files_equal (out_png_path, ref_png_path)) {
1227                 cairo_test_log (ctx, "PNG file exactly matches reference image.\n");
1228                 have_result = TRUE;
1229                 cairo_surface_destroy (test_image);
1230                 ret = CAIRO_TEST_SUCCESS;
1231                 goto UNWIND_CAIRO;
1232             }
1233             if (cairo_test_files_equal (out_png_path, new_png_path)) {
1234                 cairo_test_log (ctx, "PNG file exactly matches current failure image.\n");
1235                 have_result = TRUE;
1236                 cairo_surface_destroy (test_image);
1237                 ret = CAIRO_TEST_NEW;
1238                 goto UNWIND_CAIRO;
1239             }
1240             if (cairo_test_files_equal (out_png_path, xfail_png_path)) {
1241                 cairo_test_log (ctx, "PNG file exactly matches known failure image.\n");
1242                 have_result = TRUE;
1243                 cairo_surface_destroy (test_image);
1244                 ret = CAIRO_TEST_XFAILURE;
1245                 goto UNWIND_CAIRO;
1246             }
1247         }
1248
1249         if (cairo_test_files_equal (out_png_path, base_ref_png_path)) {
1250             cairo_test_log (ctx, "PNG file exactly reference image.\n");
1251             have_result = TRUE;
1252             cairo_surface_destroy (test_image);
1253             ret = CAIRO_TEST_SUCCESS;
1254             goto UNWIND_CAIRO;
1255         }
1256         if (cairo_test_files_equal (out_png_path, base_new_png_path)) {
1257             cairo_test_log (ctx, "PNG file exactly current failure image.\n");
1258             have_result = TRUE;
1259             cairo_surface_destroy (test_image);
1260             ret = CAIRO_TEST_NEW;
1261             goto UNWIND_CAIRO;
1262         }
1263         if (cairo_test_files_equal (out_png_path, base_xfail_png_path)) {
1264             cairo_test_log (ctx, "PNG file exactly known failure image.\n");
1265             have_result = TRUE;
1266             cairo_surface_destroy (test_image);
1267             ret = CAIRO_TEST_XFAILURE;
1268             goto UNWIND_CAIRO;
1269         }
1270
1271         /* first compare against the ideal reference */
1272         ref_image = cairo_test_get_reference_image (ctx, base_ref_png_path,
1273                                                     target->content == CAIRO_TEST_CONTENT_COLOR_ALPHA_FLATTENED);
1274         if (cairo_surface_status (ref_image)) {
1275             cairo_test_log (ctx, "Error: Cannot open reference image for %s: %s\n",
1276                             base_ref_png_path,
1277                             cairo_status_to_string (cairo_surface_status (ref_image)));
1278             cairo_surface_destroy (test_image);
1279             ret = CAIRO_TEST_FAILURE;
1280             goto UNWIND_CAIRO;
1281         }
1282
1283         diff_image = cairo_image_surface_create (CAIRO_FORMAT_ARGB32,
1284                                                  ctx->test->width,
1285                                                  ctx->test->height);
1286
1287         cmp_png_path = base_ref_png_path;
1288         diff_status = image_diff (ctx,
1289                                   test_image, ref_image, diff_image,
1290                                   &result);
1291         _xunlink (ctx, diff_png_path);
1292         if (diff_status ||
1293             image_diff_is_failure (&result, target->error_tolerance))
1294         {
1295             /* that failed, so check against the specific backend */
1296             ref_image = cairo_test_get_reference_image (ctx, ref_png_path,
1297                                                         target->content == CAIRO_TEST_CONTENT_COLOR_ALPHA_FLATTENED);
1298             if (cairo_surface_status (ref_image)) {
1299                 cairo_test_log (ctx, "Error: Cannot open reference image for %s: %s\n",
1300                                 ref_png_path,
1301                                 cairo_status_to_string (cairo_surface_status (ref_image)));
1302                 cairo_surface_destroy (test_image);
1303                 ret = CAIRO_TEST_FAILURE;
1304                 goto UNWIND_CAIRO;
1305             }
1306
1307             cmp_png_path = ref_png_path;
1308             diff_status = image_diff (ctx,
1309                                       test_image, ref_image,
1310                                       diff_image,
1311                                       &result);
1312             if (diff_status)
1313             {
1314                 cairo_test_log (ctx, "Error: Failed to compare images: %s\n",
1315                                 cairo_status_to_string (diff_status));
1316                 ret = CAIRO_TEST_FAILURE;
1317             }
1318             else if (image_diff_is_failure (&result, target->error_tolerance))
1319             {
1320                 ret = CAIRO_TEST_FAILURE;
1321
1322                 diff_status = cairo_surface_write_to_png (diff_image,
1323                                                           diff_png_path);
1324                 if (diff_status) {
1325                     cairo_test_log (ctx, "Error: Failed to write differences image: %s\n",
1326                                     cairo_status_to_string (diff_status));
1327                 } else {
1328                     have_result = TRUE;
1329                 }
1330
1331                 cairo_test_copy_file (test_filename, fail_filename);
1332             }
1333             else
1334             { /* success */
1335                 cairo_test_copy_file (test_filename, pass_filename);
1336             }
1337         }
1338         else
1339         { /* success */
1340             cairo_test_copy_file (test_filename, pass_filename);
1341         }
1342
1343         /* If failed, compare against the current image output,
1344          * and attempt to detect systematic failures.
1345          */
1346         if (ret == CAIRO_TEST_FAILURE) {
1347             char *image_out_path;
1348
1349             image_out_path =
1350                 cairo_test_reference_filename (ctx,
1351                                                base_name,
1352                                                ctx->test_name,
1353                                                "image",
1354                                                "image",
1355                                                format,
1356                                                CAIRO_TEST_OUT_SUFFIX,
1357                                                CAIRO_TEST_PNG_EXTENSION);
1358             if (image_out_path != NULL) {
1359                 if (cairo_test_files_equal (out_png_path,
1360                                             image_out_path))
1361                 {
1362                     ret = CAIRO_TEST_XFAILURE;
1363                 }
1364                 else
1365                 {
1366                     ref_image =
1367                         cairo_image_surface_create_from_png (image_out_path);
1368                     if (cairo_surface_status (ref_image) == CAIRO_STATUS_SUCCESS)
1369                     {
1370                         diff_status = image_diff (ctx,
1371                                                   test_image, ref_image,
1372                                                   diff_image,
1373                                                   &result);
1374                         if (diff_status == CAIRO_STATUS_SUCCESS &&
1375                             !image_diff_is_failure (&result, target->error_tolerance))
1376                         {
1377                             ret = CAIRO_TEST_XFAILURE;
1378                         }
1379
1380                         cairo_surface_destroy (ref_image);
1381                     }
1382                 }
1383
1384                 free (image_out_path);
1385             }
1386         }
1387
1388         cairo_surface_destroy (test_image);
1389         cairo_surface_destroy (diff_image);
1390     }
1391
1392     if (cairo_status (cr) != CAIRO_STATUS_SUCCESS) {
1393         cairo_test_log (ctx, "Error: Function under test left cairo status in an error state: %s\n",
1394                         cairo_status_to_string (cairo_status (cr)));
1395         ret = CAIRO_TEST_ERROR;
1396         goto UNWIND_CAIRO;
1397     }
1398
1399 UNWIND_CAIRO:
1400     free (test_filename);
1401     free (fail_filename);
1402     free (pass_filename);
1403
1404     test_filename = fail_filename = pass_filename = NULL;
1405
1406 #if HAVE_MEMFAULT
1407     if (ret == CAIRO_TEST_FAILURE)
1408         MEMFAULT_PRINT_FAULTS ();
1409 #endif
1410     cairo_destroy (cr);
1411 UNWIND_SURFACE:
1412     cairo_surface_destroy (surface);
1413
1414     if (target->cleanup)
1415         target->cleanup (closure);
1416
1417 #if HAVE_MEMFAULT
1418     cairo_debug_reset_static_data ();
1419
1420 #if HAVE_FCFINI
1421     FcFini ();
1422 #endif
1423
1424     if (MEMFAULT_COUNT_LEAKS () > 0) {
1425         if (ret != CAIRO_TEST_FAILURE)
1426             MEMFAULT_PRINT_FAULTS ();
1427         MEMFAULT_PRINT_LEAKS ();
1428     }
1429
1430     if (ret == CAIRO_TEST_SUCCESS && --malloc_failure_iterations > 0)
1431         goto REPEAT;
1432 #endif
1433
1434     if (have_output)
1435         cairo_test_log (ctx, "OUTPUT: %s\n", out_png_path);
1436
1437     if (have_result) {
1438         if (cmp_png_path == NULL) {
1439             /* XXX presume we matched the normal ref last time */
1440             cmp_png_path = ref_png_path;
1441         }
1442         cairo_test_log (ctx,
1443                         "REFERENCE: %s\nDIFFERENCE: %s\n",
1444                         cmp_png_path, diff_png_path);
1445     }
1446
1447 UNWIND_STRINGS:
1448     free (out_png_path);
1449     free (ref_png_path);
1450     free (base_ref_png_path);
1451     free (ref_path);
1452     free (new_png_path);
1453     free (base_new_png_path);
1454     free (new_path);
1455     free (xfail_png_path);
1456     free (base_xfail_png_path);
1457     free (xfail_path);
1458     free (diff_png_path);
1459     free (base_path);
1460     free (base_name);
1461
1462     return ret;
1463 }
1464
1465 #if defined(HAVE_SIGNAL_H) && defined(HAVE_SETJMP_H)
1466 #include <signal.h>
1467 #include <setjmp.h>
1468 /* Used to catch crashes in a test, so that we report it as such and
1469  * continue testing, although one crasher may already have corrupted memory in
1470  * an nonrecoverable fashion. */
1471 static jmp_buf jmpbuf;
1472
1473 static void
1474 segfault_handler (int signal)
1475 {
1476     longjmp (jmpbuf, signal);
1477 }
1478 #endif
1479
1480 cairo_test_status_t
1481 _cairo_test_context_run_for_target (cairo_test_context_t *ctx,
1482                                     const cairo_boilerplate_target_t *target,
1483                                     cairo_bool_t similar,
1484                                     int dev_offset, int dev_scale)
1485 {
1486     cairo_test_status_t status;
1487
1488     if (target->get_image_surface == NULL)
1489         return CAIRO_TEST_UNTESTED;
1490
1491     if (similar && ! cairo_test_target_has_similar (ctx, target))
1492         return CAIRO_TEST_UNTESTED;
1493
1494     cairo_test_log (ctx,
1495                     "Testing %s with %s%s target (dev offset %d scale: %d)\n",
1496                     ctx->test_name,
1497                     similar ? " (similar) " : "",
1498                     target->name,
1499                     dev_offset, dev_scale);
1500
1501     printf ("%s.%s.%s [%dx%d]%s:\t", ctx->test_name, target->name,
1502             cairo_boilerplate_content_name (target->content),
1503             dev_offset, dev_scale,
1504             similar ? " (similar)": "");
1505     fflush (stdout);
1506
1507 #if defined(HAVE_SIGNAL_H) && defined(HAVE_SETJMP_H)
1508     if (! RUNNING_ON_VALGRIND) {
1509         void (* volatile old_segfault_handler)(int);
1510         void (* volatile old_segfpe_handler)(int);
1511         void (* volatile old_sigpipe_handler)(int);
1512         void (* volatile old_sigabrt_handler)(int);
1513         void (* volatile old_sigalrm_handler)(int);
1514
1515         /* Set up a checkpoint to get back to in case of segfaults. */
1516 #ifdef SIGSEGV
1517         old_segfault_handler = signal (SIGSEGV, segfault_handler);
1518 #endif
1519 #ifdef SIGFPE
1520         old_segfpe_handler = signal (SIGFPE, segfault_handler);
1521 #endif
1522 #ifdef SIGPIPE
1523         old_sigpipe_handler = signal (SIGPIPE, segfault_handler);
1524 #endif
1525 #ifdef SIGABRT
1526         old_sigabrt_handler = signal (SIGABRT, segfault_handler);
1527 #endif
1528 #ifdef SIGALRM
1529         old_sigalrm_handler = signal (SIGALRM, segfault_handler);
1530 #endif
1531         if (0 == setjmp (jmpbuf))
1532             status = cairo_test_for_target (ctx, target, dev_offset, dev_scale, similar);
1533         else
1534             status = CAIRO_TEST_CRASHED;
1535 #ifdef SIGSEGV
1536         signal (SIGSEGV, old_segfault_handler);
1537 #endif
1538 #ifdef SIGFPE
1539         signal (SIGFPE, old_segfpe_handler);
1540 #endif
1541 #ifdef SIGPIPE
1542         signal (SIGPIPE, old_sigpipe_handler);
1543 #endif
1544 #ifdef SIGABRT
1545         signal (SIGABRT, old_sigabrt_handler);
1546 #endif
1547 #ifdef SIGALRM
1548         signal (SIGALRM, old_sigalrm_handler);
1549 #endif
1550     } else {
1551         status = cairo_test_for_target (ctx, target, dev_offset, dev_scale, similar);
1552     }
1553 #else
1554     status = cairo_test_for_target (ctx, target, dev_offset, dev_scale, similar);
1555 #endif
1556
1557     cairo_test_log (ctx,
1558                     "TEST: %s TARGET: %s FORMAT: %s OFFSET: %d SCALE: %d SIMILAR: %d RESULT: ",
1559                     ctx->test_name, target->name,
1560                     cairo_boilerplate_content_name (target->content),
1561                     dev_offset, dev_scale, similar);
1562     switch (status) {
1563     case CAIRO_TEST_SUCCESS:
1564         printf ("PASS\n");
1565         cairo_test_log (ctx, "PASS\n");
1566         break;
1567
1568     case CAIRO_TEST_UNTESTED:
1569         printf ("UNTESTED\n");
1570         cairo_test_log (ctx, "UNTESTED\n");
1571         break;
1572
1573     default:
1574     case CAIRO_TEST_CRASHED:
1575         if (print_fail_on_stdout) {
1576             printf ("!!!CRASHED!!!\n");
1577         } else {
1578             /* eat the test name */
1579             printf ("\r");
1580             fflush (stdout);
1581         }
1582         cairo_test_log (ctx, "CRASHED\n");
1583         fprintf (stderr, "%s.%s.%s [%dx%d]%s:\t%s!!!CRASHED!!!%s\n",
1584                  ctx->test_name, target->name,
1585                  cairo_boilerplate_content_name (target->content), dev_offset, dev_scale, similar ? " (similar)" : "",
1586                  fail_face, normal_face);
1587         break;
1588
1589     case CAIRO_TEST_ERROR:
1590         if (print_fail_on_stdout) {
1591             printf ("!!!ERROR!!!\n");
1592         } else {
1593             /* eat the test name */
1594             printf ("\r");
1595             fflush (stdout);
1596         }
1597         cairo_test_log (ctx, "ERROR\n");
1598         fprintf (stderr, "%s.%s.%s [%dx%d]%s:\t%s!!!ERROR!!!%s\n",
1599                  ctx->test_name, target->name,
1600                  cairo_boilerplate_content_name (target->content), dev_offset, dev_scale, similar ? " (similar)" : "",
1601                  fail_face, normal_face);
1602         break;
1603
1604     case CAIRO_TEST_XFAILURE:
1605         if (print_fail_on_stdout) {
1606             printf ("XFAIL\n");
1607         } else {
1608             /* eat the test name */
1609             printf ("\r");
1610             fflush (stdout);
1611         }
1612         fprintf (stderr, "%s.%s.%s [%dx%d]%s:\t%sXFAIL%s\n",
1613                  ctx->test_name, target->name,
1614                  cairo_boilerplate_content_name (target->content), dev_offset, dev_scale, similar ? " (similar)" : "",
1615                  xfail_face, normal_face);
1616         cairo_test_log (ctx, "XFAIL\n");
1617         break;
1618
1619     case CAIRO_TEST_NEW:
1620         if (print_fail_on_stdout) {
1621             printf ("NEW\n");
1622         } else {
1623             /* eat the test name */
1624             printf ("\r");
1625             fflush (stdout);
1626         }
1627         fprintf (stderr, "%s.%s.%s [%dx%d]%s:\t%sNEW%s\n",
1628                  ctx->test_name, target->name,
1629                  cairo_boilerplate_content_name (target->content), dev_offset, dev_scale, similar ? " (similar)" : "",
1630                  fail_face, normal_face);
1631         cairo_test_log (ctx, "NEW\n");
1632         break;
1633
1634     case CAIRO_TEST_NO_MEMORY:
1635     case CAIRO_TEST_FAILURE:
1636         if (print_fail_on_stdout) {
1637             printf ("FAIL\n");
1638         } else {
1639             /* eat the test name */
1640             printf ("\r");
1641             fflush (stdout);
1642         }
1643         fprintf (stderr, "%s.%s.%s [%dx%d]%s:\t%sFAIL%s\n",
1644                  ctx->test_name, target->name,
1645                  cairo_boilerplate_content_name (target->content), dev_offset, dev_scale, similar ? " (similar)" : "",
1646                  fail_face, normal_face);
1647         cairo_test_log (ctx, "FAIL\n");
1648         break;
1649     }
1650     fflush (stdout);
1651
1652     return status;
1653 }
1654
1655 const cairo_test_context_t *
1656 cairo_test_get_context (cairo_t *cr)
1657 {
1658     return cairo_get_user_data (cr, &_cairo_test_context_key);
1659 }
1660
1661 cairo_surface_t *
1662 cairo_test_create_surface_from_png (const cairo_test_context_t *ctx,
1663                                     const char *filename)
1664 {
1665     cairo_surface_t *image;
1666     cairo_status_t status;
1667
1668     image = cairo_image_surface_create_from_png (filename);
1669     status = cairo_surface_status (image);
1670     if (status == CAIRO_STATUS_FILE_NOT_FOUND) {
1671         /* expect not found when running with srcdir != builddir
1672          * such as when 'make distcheck' is run
1673          */
1674         if (ctx->srcdir) {
1675             char *srcdir_filename;
1676             xasprintf (&srcdir_filename, "%s/%s", ctx->srcdir, filename);
1677             cairo_surface_destroy (image);
1678             image = cairo_image_surface_create_from_png (srcdir_filename);
1679             free (srcdir_filename);
1680         }
1681     }
1682
1683     return image;
1684 }
1685
1686 cairo_pattern_t *
1687 cairo_test_create_pattern_from_png (const cairo_test_context_t *ctx,
1688                                     const char *filename)
1689 {
1690     cairo_surface_t *image;
1691     cairo_pattern_t *pattern;
1692
1693     image = cairo_test_create_surface_from_png (ctx, filename);
1694
1695     pattern = cairo_pattern_create_for_surface (image);
1696
1697     cairo_pattern_set_extend (pattern, CAIRO_EXTEND_REPEAT);
1698
1699     cairo_surface_destroy (image);
1700
1701     return pattern;
1702 }
1703
1704 static cairo_surface_t *
1705 _draw_check (int width, int height)
1706 {
1707     cairo_surface_t *surface;
1708     cairo_t *cr;
1709
1710     surface = cairo_image_surface_create (CAIRO_FORMAT_RGB24, 12, 12);
1711     cr = cairo_create (surface);
1712     cairo_surface_destroy (surface);
1713
1714     cairo_set_source_rgb (cr, 0.75, 0.75, 0.75); /* light gray */
1715     cairo_paint (cr);
1716
1717     cairo_set_source_rgb (cr, 0.25, 0.25, 0.25); /* dark gray */
1718     cairo_rectangle (cr, width / 2,  0, width / 2, height / 2);
1719     cairo_rectangle (cr, 0, height / 2, width / 2, height / 2);
1720     cairo_fill (cr);
1721
1722     surface = cairo_surface_reference (cairo_get_target (cr));
1723     cairo_destroy (cr);
1724
1725     return surface;
1726 }
1727
1728 void
1729 cairo_test_paint_checkered (cairo_t *cr)
1730 {
1731     cairo_surface_t *check;
1732
1733     check = _draw_check (12, 12);
1734
1735     cairo_save (cr);
1736     cairo_set_source_surface (cr, check, 0, 0);
1737     cairo_surface_destroy (check);
1738
1739     cairo_pattern_set_filter (cairo_get_source (cr), CAIRO_FILTER_NEAREST);
1740     cairo_pattern_set_extend (cairo_get_source (cr), CAIRO_EXTEND_REPEAT);
1741     cairo_paint (cr);
1742
1743     cairo_restore (cr);
1744 }
1745
1746 cairo_bool_t
1747 cairo_test_is_target_enabled (const cairo_test_context_t *ctx,
1748                               const char *target)
1749 {
1750     size_t i;
1751
1752     for (i = 0; i < ctx->num_targets; i++) {
1753         const cairo_boilerplate_target_t *t = ctx->targets_to_test[i];
1754         if (strcmp (t->name, target) == 0) {
1755             /* XXX ask the target whether is it possible to run?
1756              * e.g. the xlib backend could check whether it is able to connect
1757              * to the Display.
1758              */
1759             return t->get_image_surface != NULL;
1760         }
1761     }
1762
1763     return FALSE;
1764 }
1765
1766 cairo_bool_t
1767 cairo_test_malloc_failure (const cairo_test_context_t *ctx,
1768                            cairo_status_t status)
1769 {
1770     if (! ctx->malloc_failure)
1771         return FALSE;
1772
1773     if (status != CAIRO_STATUS_NO_MEMORY)
1774         return FALSE;
1775
1776 #if HAVE_MEMFAULT
1777     {
1778         int n_faults;
1779
1780         /* prevent infinite loops... */
1781         n_faults = MEMFAULT_COUNT_FAULTS ();
1782         if (n_faults == ctx->last_fault_count)
1783             return FALSE;
1784
1785         ((cairo_test_context_t *) ctx)->last_fault_count = n_faults;
1786     }
1787 #endif
1788
1789     return TRUE;
1790 }
1791
1792 cairo_test_status_t
1793 cairo_test_status_from_status (const cairo_test_context_t *ctx,
1794                                cairo_status_t status)
1795 {
1796     if (status == CAIRO_STATUS_SUCCESS)
1797         return CAIRO_TEST_SUCCESS;
1798
1799     if (cairo_test_malloc_failure (ctx, status))
1800         return CAIRO_TEST_NO_MEMORY;
1801
1802     return CAIRO_TEST_FAILURE;
1803 }