test/Makefile.am

   1 include $(top_srcdir)/build/Makefile.am.common
   2
   3 include $(top_srcdir)/test/Makefile.sources
   4
   5 SUBDIRS=pdiff .
   6
   7 # Then we have a collection of tests that are only run if certain
   8 # features are compiled into cairo
   9 if HAVE_REAL_PTHREAD
  10 test_sources += $(pthread_test_sources)
  11 endif
  12
  13 if CAIRO_HAS_FT_FONT
  14 if CAIRO_HAS_FC_FONT
  15 test_sources += $(ft_font_test_sources)
  16 endif
  17 endif
  18
  19 if CAIRO_HAS_GL_SURFACE
  20 test_sources += $(gl_surface_test_sources)
  21 endif
  22
  23 # Need to add quartz-surface-source
  24 if CAIRO_HAS_QUARTZ_SURFACE
  25 test_sources += $(quartz_surface_test_sources)
  26 endif
  27
  28 if CAIRO_HAS_PDF_SURFACE
  29 test_sources += $(pdf_surface_test_sources)
  30 endif
  31
  32 if CAIRO_HAS_PS_SURFACE
  33 test_sources += $(ps_surface_test_sources)
  34 endif
  35
  36 if CAIRO_HAS_SVG_SURFACE
  37 test_sources += $(svg_surface_test_sources)
  38 endif
  39
  40 if CAIRO_HAS_TEST_SURFACES
  41 test_sources += $(test_fallback16_surface_test_sources)
  42 endif
  43
  44 if CAIRO_HAS_XCB_SURFACE
  45 test_sources += $(xcb_surface_test_sources)
  46 endif
  47
  48 if CAIRO_HAS_XLIB_SURFACE
  49 test_sources += $(xlib_surface_test_sources)
  50 endif
  51
  52 if CAIRO_HAS_XLIB_XRENDER_SURFACE
  53 test_sources += $(xlib_xrender_surface_test_sources)
  54 endif
  55
  56 if CAIRO_HAS_MULTI_PAGE_SURFACES
  57 test_sources += $(multi_page_surface_test_sources)
  58 endif
  59
  60 # Include fallback-resolution (once!) if we have any of the vector surfaces
  61 if BUILD_ANY2PPM
  62 if CAIRO_HAS_SVG_SURFACE
  63 test = $(fallback_resolution_test_sources)
  64 endif
  65 if CAIRO_HAS_PDF_SURFACE
  66 test = $(fallback_resolution_test_sources)
  67 endif
  68 if CAIRO_HAS_PS_SURFACE
  69 test = $(fallback_resolution_test_sources)
  70 endif
  71 endif
  72 test_sources += $(test)
  73
  74 noinst_PROGRAMS = cairo-test-suite$(EXEEXT) # always build
  75
  76 TESTS += cairo-test-suite$(EXEEXT)
  77
  78 cairo-test-constructors.c: Makefile $(test_sources) make-cairo-test-constructors.sh
  79         (cd $(srcdir) && sh ./make-cairo-test-constructors.sh $(test_sources)) > $@
  80
  81 cairo_test_suite_SOURCES =              \
  82         $(cairo_test_suite_sources)     \
  83         $(cairo_test_suite_headers)     \
  84         $(test_sources)                 \
  85         cairo-test-constructors.c
  86 cairo_test_suite_CFLAGS = $(AM_CFLAGS) $(real_pthread_CFLAGS)
  87 cairo_test_suite_LDADD =                                        \
  88         $(real_pthread_LIBS)                                    \
  89         $(top_builddir)/test/pdiff/libpdiff.la                  \
  90         $(top_builddir)/boilerplate/libcairoboilerplate.la      \
  91         $(top_builddir)/src/libcairo.la                         \
  92         $(CAIRO_LDADD)
  93 cairo_test_suite_DEPENDENCIES = \
  94         $(top_builddir)/test/pdiff/libpdiff.la                  \
  95         $(top_builddir)/boilerplate/libcairoboilerplate.la      \
  96         $(top_builddir)/src/libcairo.la
  97 if BUILD_ANY2PPM
  98 cairo_test_suite_DEPENDENCIES += \
  99         any2ppm$(EXEEXT)
 100 endif
 101
 102 if HAVE_SHM
 103 EXTRA_PROGRAMS += cairo-test-trace
 104 cairo_test_trace_SOURCES =              \
 105         cairo-test-trace.c              \
 106         buffer-diff.c                   \
 107         buffer-diff.h
 108 cairo_test_trace_CFLAGS = $(AM_CFLAGS) $(real_pthread_CFLAGS)
 109 cairo_test_trace_LDADD =                \
 110         $(real_pthread_LIBS)                                    \
 111         $(top_builddir)/test/pdiff/libpdiff.la                  \
 112         $(top_builddir)/util/cairo-script/libcairo-script-interpreter.la \
 113         $(top_builddir)/boilerplate/libcairoboilerplate.la      \
 114         $(top_builddir)/src/libcairo.la                         \
 115         $(top_builddir)/util/cairo-missing/libcairo-missing.la  \
 116         $(CAIRO_LDADD) \
 117         $(SHM_LIBS)
 118 cairo_test_trace_DEPENDENCIES = \
 119         $(top_builddir)/test/pdiff/libpdiff.la                  \
 120         $(top_builddir)/util/cairo-script/libcairo-script-interpreter.la \
 121         $(top_builddir)/boilerplate/libcairoboilerplate.la      \
 122         $(top_builddir)/src/libcairo.la                         \
 123         $(top_builddir)/util/cairo-missing/libcairo-missing.la  \
 124         $(NULL)
 125 endif
 126
 127 BUILT_SOURCES += cairo-test-constructors.c
 128 EXTRA_DIST += $(BUILT_SOURCES) $(noinst_SCRIPTS) COPYING make-cairo-test-constructors.sh run-cairo-test-suite.sh generate_refs.sh tiger.inc
 129 CLEANFILES += $(BUILT_SOURCES)
 130
 131 EXTRA_DIST +=           \
 132 6x13.pcf                \
 133 index.html              \
 134 jp2.jp2                 \
 135 jpeg.jpg                \
 136 png.png                 \
 137 romedalen.jpg           \
 138 romedalen.png           \
 139 scarab.jpg              \
 140 surface-source.c        \
 141 testtable.js            \
 142 reference
 143
 144 # Any test for which the code committed to CVS is expected to fail
 145 # should be listed here.
 146 #
 147 # This way, we can avoid being bothered by reports of bugs we are
 148 # aware of, but users can still report when tests start behaving in
 149 # unexpected ways on their system.
 150 #
 151 # Of course, before any "release" of cairo we should eliminate
 152 # everything from this list by fixing the bugs. (We don't necessarily
 153 # have to be that strict for "snapshots" though.)
 154 #
 155 # Analysis of XFAIL errors:
 156 # alpha-similar         - discrepancy between backends in applying color
 157 #                         components of a pure alpha surface
 158 # big-line              - range overflow of fixed-point
 159 # big-trap              - range overflow of fixed-point
 160 # degenerate-dash       - needs path editing in PS to convert degenerate
 161 #                         end-caps into the shapes as expected by cairo
 162 #                         (Or maybe PS is the correct behaviour?)
 163 # degenerate-path       - undefined behaviour in PS, needs path editing to
 164 #                         convert degenerate segments into circles/rectangles
 165 #                         as expected by cairo
 166 # device-offset-scale   - complication of pre-multiplying device_offset
 167 #                         into the pattern_matrix and then requiring further
 168 #                         manipulation for SVG
 169 # extend-pad            - lacks implementation in pixman and consequently used
 170 #                         as an excuse for lack of support in other backends
 171 # fallback-resolution   - The essential problem here is that the recording-surface
 172 #                         has recorded a sequence of operations with one device
 173 #                         transformation, and we attempt to replay it with
 174 #                         another (basically a scale-factor for the falback
 175 #                         resolution). Carl begun to look at this with his
 176 #                         chain-of-bugs, but the can of worms is much bigger.
 177 #                         It appears to be a design flaw in the recording-surface
 178 #                         that may spread further...
 179 #                         My solution would be to lock Behad and Adrian in a
 180 #                         room, with Carl as a moderator and not let them out
 181 #                         until they have come up with an interface and
 182 #                         semantics that actually work. :-)
 183 # in-fill-empty-trapezoid The cairo_in_fill () function can sometimes
 184 #                         produce false positives when the tessellator
 185 #                         produces empty trapezoids and the query
 186 #                         point lands exactly on a trapezoid edge.
 187 # long-lines            - range overflow of fixed-point
 188 # scale-offset-image    - loss of precision converting a cairo matrix to
 189 # scale-offset-similar    pixman's fixed point format.
 190 # self-copy-overlap     - vector surfaces take snapshot of patterns in contrast
 191 #                         to the raster backends which don't. One solution
 192 #                         would be to clone overlapping areas of dst/source, so
 193 #                         patterns were effectively snapshotted across all
 194 #                         backends.
 195 # self-intersecting     - incremental trapezoidation of strokes can generate
 196 #                         overlapping traps. Needs self-intersection analysis
 197 #                         on cairo_traps_t after strokes.
 198 #                         Test case should also be expanded to hit special-case
 199 #                         tessellators as well.
 200 # surface-pattern-big...- Strange, unexplained results for SVG/PS.
 201 XFAIL_TESTS =                                   \
 202 alpha-similar$(EXEEXT)                          \
 203 big-line$(EXEEXT)                               \
 204 big-trap$(EXEEXT)                               \
 205 degenerate-dash$(EXEEXT)                        \
 206 degenerate-path$(EXEEXT)                        \
 207 device-offset-scale$(EXEEXT)                    \
 208 extend-pad$(EXEEXT)                             \
 209 fallback-resolution$(EXEEXT)                    \
 210 in-fill-empty-trapezoid$(EXEEXT)                \
 211 long-lines$(EXEEXT)                             \
 212 self-copy-overlap$(EXEEXT)                      \
 213 self-intersecting$(EXEEXT)                      \
 214 surface-pattern-big-scale-down$(EXEEXT)         \
 215 $(NULL)
 216
 217 # Any test that doesn't generate a log file goes here
 218 NOLOG_TESTS =                   \
 219 fallback-resolution             \
 220 font-options                    \
 221 multi-page                      \
 222 pdf-features                    \
 223 png                             \
 224 ps-eps                          \
 225 ps-features                     \
 226 svg-clip                        \
 227 svg-surface                     \
 228 toy-font-face                   \
 229 user-data
 230
 231 # A target to summarise the failures
 232 check-summary:
 233         @FAILED_TESTS=""; \
 234         for t in output/*.log; do \
 235             if grep -e '\<FAIL\>' $$t >/dev/null 2>&1; then \
 236                 FAILED_TESTS="$$FAILED_TESTS $$t"; \
 237             fi; \
 238         done; \
 239         if test -n "$$FAILED_TESTS"; then \
 240             echo "Failed tests:"; \
 241             surfaces=""; \
 242             for t in $$FAILED_TESTS; do \
 243                 name="$${t##output/}"; name="$${name%.log}"; \
 244                 echo -n "     $$name: "; \
 245                 grep -e '\<FAIL\>' $$t | sed -e 's/.*TARGET: \([^ ]*\).*/\1/' | sort | uniq | tr '\n' ' '; \
 246                 echo; \
 247                 for s in `grep -e '\<FAIL\>' $$t | sed -e 's/.*TARGET: \([^ ]*\).*/\1/' | sort | uniq`; do \
 248                     ss=`echo $$s | tr '-' '_'`; \
 249                     tt=`echo $$name | tr '-' '_'`; \
 250                     eval $$ss=\""$${!ss} $$tt"\"; \
 251                     echo $$surfaces | grep $$ss >/dev/null || surfaces="$$surfaces $$ss"; \
 252                 done; \
 253             done; \
 254             echo -n "Failures per surface - "; \
 255             first=""; \
 256             for s in $$surfaces; do \
 257                 ss=`echo $$s | tr '_' '-'`; \
 258                 test -n "$$first" && echo -n ", "; \
 259                 cnt=`echo $${!s} | wc -w`; \
 260                 echo -n "$$ss: $$cnt"; \
 261                 first="false"; \
 262             done; \
 263             echo "."; \
 264             for s in $$surfaces; do \
 265                 ss=`echo $$s | tr '_' '-'`; \
 266                 cnt=`echo $${!s} | wc -w`; \
 267                 echo -n "       $$ss [$$cnt]: "; \
 268                 echo $${!s} | tr '_' '-'; \
 269             done; \
 270         fi
 271
 272 AM_CPPFLAGS =                                   \
 273         -I$(srcdir)                             \
 274         -I$(srcdir)/pdiff                       \
 275         -I$(top_srcdir)/boilerplate             \
 276         -I$(top_srcdir)/util/cairo-missing      \
 277         -I$(top_srcdir)/util/cairo-script       \
 278         -I$(top_srcdir)/src                     \
 279         -I$(top_builddir)/src                   \
 280         $(CAIRO_CFLAGS)
 281 AM_LDFLAGS = $(CAIRO_LDFLAGS)
 282
 283 $(top_builddir)/boilerplate/libcairoboilerplate.la: $(top_builddir)/src/libcairo.la
 284         cd $(top_builddir)/boilerplate && $(MAKE) $(AM_MAKEFLAGS) libcairoboilerplate.la
 285
 286 $(top_builddir)/src/libcairo.la:
 287         cd $(top_builddir)/src && $(MAKE) $(AM_MAKEFLAGS) libcairo.la
 288
 289 $(top_builddir)/test/pdiff/libpdiff.la:
 290         cd $(top_builddir)/test/pdiff && $(MAKE) $(AM_MAKEFLAGS) libpdiff.la
 291
 292 $(top_builddir)/util/cairo-script/libcairo-script-interpreter.la: $(top_builddir)/src/libcairo.la
 293         cd $(top_builddir)/util/cairo-script && $(MAKE) $(AM_MAKEFLAGS) libcairo-script-interpreter.la
 294
 295 EXTRA_PROGRAMS += imagediff png-flatten
 296
 297 imagediff_SOURCES = \
 298         imagediff.c     \
 299         buffer-diff.c   \
 300         buffer-diff.h
 301 imagediff_LDADD = \
 302         $(top_builddir)/test/pdiff/libpdiff.la \
 303         $(top_builddir)/src/libcairo.la
 304
 305 png_flatten_SOURCES = png-flatten.c
 306 png_flatten_LDADD = $(top_builddir)/src/libcairo.la \
 307                     $(CAIRO_LDADD)
 308
 309 if BUILD_ANY2PPM
 310 check_PROGRAMS += any2ppm
 311 any2ppm_CFLAGS = $(AM_CFLAGS) $(POPPLER_CFLAGS) $(LIBRSVG_CFLAGS) $(LIBSPECTRE_CFLAGS)
 312 # add LDADD, so poppler/librsvg uses "our" cairo
 313 any2ppm_LDFLAGS = $(AM_LDFLAGS) $(CAIRO_TEST_UNDEFINED_LDFLAGS)
 314 any2ppm_LDADD = \
 315                 $(top_builddir)/util/cairo-script/libcairo-script-interpreter.la \
 316                 $(top_builddir)/src/libcairo.la \
 317                 $(CAIRO_LDADD) \
 318                 $(CAIROBOILERPLATE_LIBS) \
 319                 $(POPPLER_LIBS) \
 320                 $(LIBRSVG_LIBS) \
 321                 $(LIBSPECTRE_LIBS)
 322 endif
 323
 324 if CAIRO_CAN_TEST_PDF_SURFACE
 325 check_PROGRAMS += pdf2png
 326 pdf2png_CFLAGS = $(AM_CFLAGS) $(POPPLER_CFLAGS)
 327 # add LDADD, so poppler uses "our" cairo
 328 pdf2png_LDFLAGS = $(AM_LDFLAGS) $(CAIRO_TEST_UNDEFINED_LDFLAGS)
 329 pdf2png_LDADD  = $(top_builddir)/src/libcairo.la \
 330                  $(CAIRO_LDADD) \
 331                  $(POPPLER_LIBS)
 332 endif
 333
 334 if CAIRO_CAN_TEST_SVG_SURFACE
 335 check_PROGRAMS += svg2png
 336 svg2png_CFLAGS = $(AM_CFLAGS) $(LIBRSVG_CFLAGS)
 337 # add LDADD, so librsvg uses "our" cairo
 338 svg2png_LDFLAGS = $(AM_LDFLAGS) $(CAIRO_TEST_UNDEFINED_LDFLAGS)
 339 svg2png_LDADD  = $(top_builddir)/src/libcairo.la \
 340                  $(CAIRO_LDADD) \
 341                  $(LIBRSVG_LIBS)
 342 endif
 343
 344 if CAIRO_HAS_SPECTRE
 345 check_PROGRAMS += ps2png
 346 ps2png_CFLAGS = $(AM_CFLAGS) $(LIBSPECTRE_CFLAGS)
 347 # add LDADD, so ps2png uses "our" cairo
 348 ps2png_LDFLAGS = $(AM_LDFLAGS) $(CAIRO_TEST_UNDEFINED_LDFLAGS)
 349 ps2png_LDADD  = $(top_builddir)/src/libcairo.la \
 350                 $(CAIRO_LDADD) \
 351                 $(LIBSPECTRE_LIBS)
 352 endif
 353
 354 EXTRA_PROGRAMS += $(TESTS)
 355
 356 # Do a funny transition of CAIRO_TEST_TARGET through TARGETS such that
 357 # one can limit tested targets both through CAIRO_TEST_TARGET env var
 358 # and TARGETS make var on the command line.  Same for the rest.
 359 TARGETS = $(CAIRO_TEST_TARGET)
 360 TARGETS_EXCLUDE = $(CAIRO_TEST_TARGET_EXCLUDE)
 361 NUM_THREADS = $(CAIRO_TEST_NUM_THREADS)
 362 MODE = $(CAIRO_TEST_MODE)
 363
 364 # Same about ENV vs CAIRO_TEST_ENV.  ENV is used with "make run" only
 365 ENV = $(CAIRO_TEST_ENV)
 366
 367 TESTS_ENVIRONMENT = CAIRO_TEST_MODE="$(MODE)" CAIRO_TEST_TARGET="$(TARGETS)" CAIRO_TEST_TARGET_EXCLUDE="$(TARGETS_EXCLUDE)" CAIRO_TEST_NUM_THREADS="$(NUM_THREADS)" $(ENV)
 368
 369 EXTRA_VALGRIND_FLAGS = $(CAIRO_EXTRA_VALGRIND_FLAGS)
 370 VALGRIND_FLAGS = \
 371         --tool=memcheck --suppressions=$(srcdir)/.valgrind-suppressions \
 372         --track-origins=yes \
 373         --leak-check=yes --show-reachable=yes \
 374         $(EXTRA_VALGRIND_FLAGS)
 375
 376 CLEANFILES +=                                   \
 377         valgrind-log                            \
 378         ref.hash                                \
 379         ref.list                                \
 380         png-test.png                            \
 381         png.out.png                             \
 382         create-for-stream.pdf                   \
 383         create-for-stream.ps                    \
 384         create-for-stream.svg                   \
 385         svg-surface-source.out.svg              \
 386         pdf-surface-source.out.pdf              \
 387         ps-surface-source.out.ps                \
 388         pdf-features.pdf                        \
 389         pdf-mime-data.out*                      \
 390         ps-features.ps                          \
 391         svg-clip.svg                            \
 392         svg-surface.svg                         \
 393         multi-page.pdf                          \
 394         multi-page.ps                           \
 395         $(NULL)
 396
 397 # This used to be a simple 'echo ${RM} *.ps *.pdf *.svg *.etc', but
 398 # most systems cannot handle all of our clean files together.
 399 # Then it became a fancy find using many GNU extensions, but then the ugly
 400 # reality of portability was raised and it became....
 401 clean-local:
 402         rm -rf output
 403         -${FIND} . -name '*.log'      -print | ${XARGS} ${RM}
 404         -${FIND} . -name '*.[is]'     -print | ${XARGS} ${RM}
 405 clean-caches:
 406         -${FIND} output -name '*.fail.*' -print | ${XARGS} ${RM}
 407         -${FIND} output -name '*.pass.*' -print | ${XARGS} ${RM}
 408
 409 # The following definitions both should work.
 410 #FAILED_TESTS = `grep -l '\<FAIL\>' $(test_sources:.c=.log) 2>/dev/null | sed -e 's/[.]log$$//' | xargs echo`
 411 FAILED_TESTS = `grep -l '\<FAIL\>' $(test_sources:.c=.log) 2>/dev/null | tr '\n' ' ' | sed -e 's/[.]log  */ /g; s/^ //; s/ $$//'`
 412
 413 recheck = check CAIRO_TESTS="$(FAILED_TESTS)"
 414
 415 # Re-checks all failed tests, i.e. tests with a log file that has a failure
 416 recheck:
 417         @echo Re-checking failed tests
 418         @$(MAKE) $(AM_MAKEFLAGS) $(recheck)
 419
 420 # Checks tests.
 421 # Target doesn't fail if tests fail.
 422 test:
 423         @$(MAKE) $(AM_MAKEFLAGS) check
 424
 425 # Re-checks tests.
 426 # Target doesn't fail if tests fail.
 427 retest:
 428         @CAIRO_TESTS="$(FAILED_TESTS)"; \
 429         $(MAKE) $(AM_MAKEFLAGS) check
 430
 431 # Run tests under a tool specified by TOOL.  For example, make run TOOL=gdb
 432 run:
 433         $(MAKE) $(AM_MAKEFLAGS) check TESTS_ENVIRONMENT='$(TESTS_ENVIRONMENT) $(top_builddir)/libtool --mode=execute env $(TOOL)'
 434
 435 # Check tests under valgrind.  Saves log to valgrind-log
 436 check-valgrind:
 437         $(MAKE) $(AM_MAKEFLAGS) check TESTS_ENVIRONMENT='$(TESTS_ENVIRONMENT) CAIRO_TEST_MODE="$(MODE),foreground CAIRO_TEST_TIMEOUT=0" $(top_builddir)/libtool --mode=execute valgrind $(VALGRIND_FLAGS)' 2>&1 | tee valgrind-log
 438
 439 #%.log: %.c cairo-test-suite
 440 #-./cairo-test-suite $(<:.c=)
 441
 442 NOLOG_TESTS_LOG = $(NOLOG_TESTS:=.log)
 443
 444 $(NOLOG_TESTS_LOG):
 445         @echo dummy > $@
 446
 447 # Identify identical reference images
 448 check-ref-dups:
 449         @LANG=C; \
 450         ( cd "$(srcdir)" && sha1sum *.ref.png | sort ) > ref.hash; \
 451         join ref.hash ref.hash | grep -v -E '( .*.ref.png).*\1' | cut -d' ' -f 1-2 | sort -u
 452
 453 results.tar:
 454         @tar cf $@ index.html testtable.js *.log output/*.log; \
 455         for i in output/*.fail.png ; do \
 456                 testname=$${i#output/} ; \
 457                 testname=$${testname%%.*} ; \
 458                 echo tar uf $@ reference/$${testname}*.ref.png $${i%fail.png}out.png $${i%fail.png}diff.png ; \
 459                 tar uf $@ reference/$${testname}*.ref.png $${i%fail.png}out.png $${i%fail.png}diff.png ; \
 460         done
 461
 462 results.tar.gz: results.tar
 463         gzip -c $< > $@
 464
 465 release-verify-sane-tests:
 466
 467 .PHONY: check-valgrind test recheck retest check-ref-dups release-verify-sane-tests
 468
 469 EXTRA_DIST += Makefile.win32