1 include $(top_srcdir)/build/Makefile.am.common
3 include $(top_srcdir)/test/Makefile.sources
7 # Then we have a collection of tests that are only run if certain
8 # features are compiled into cairo
10 test_sources += $(pthread_test_sources)
15 test_sources += $(ft_font_test_sources)
19 if CAIRO_HAS_GL_SURFACE
20 test_sources += $(gl_surface_test_sources)
23 # Need to add quartz-surface-source
24 if CAIRO_HAS_QUARTZ_SURFACE
25 test_sources += $(quartz_surface_test_sources)
28 if CAIRO_HAS_PDF_SURFACE
29 test_sources += $(pdf_surface_test_sources)
32 if CAIRO_HAS_PS_SURFACE
33 test_sources += $(ps_surface_test_sources)
36 if CAIRO_HAS_SVG_SURFACE
37 test_sources += $(svg_surface_test_sources)
40 if CAIRO_HAS_TEST_SURFACES
41 test_sources += $(test_fallback16_surface_test_sources)
44 if CAIRO_HAS_XCB_SURFACE
45 test_sources += $(xcb_surface_test_sources)
48 if CAIRO_HAS_XLIB_SURFACE
49 test_sources += $(xlib_surface_test_sources)
52 if CAIRO_HAS_XLIB_XRENDER_SURFACE
53 test_sources += $(xlib_xrender_surface_test_sources)
56 if CAIRO_HAS_MULTI_PAGE_SURFACES
57 test_sources += $(multi_page_surface_test_sources)
60 # Include fallback-resolution (once!) if we have any of the vector surfaces
62 if CAIRO_HAS_SVG_SURFACE
63 test = $(fallback_resolution_test_sources)
65 if CAIRO_HAS_PDF_SURFACE
66 test = $(fallback_resolution_test_sources)
68 if CAIRO_HAS_PS_SURFACE
69 test = $(fallback_resolution_test_sources)
72 test_sources += $(test)
74 noinst_PROGRAMS = cairo-test-suite$(EXEEXT) # always build
76 TESTS += cairo-test-suite$(EXEEXT)
78 cairo-test-constructors.c: Makefile $(test_sources) make-cairo-test-constructors.sh
79 (cd $(srcdir) && sh ./make-cairo-test-constructors.sh $(test_sources)) > $@
81 cairo_test_suite_SOURCES = \
82 $(cairo_test_suite_sources) \
83 $(cairo_test_suite_headers) \
85 cairo-test-constructors.c
86 cairo_test_suite_CFLAGS = $(AM_CFLAGS) $(real_pthread_CFLAGS)
87 cairo_test_suite_LDADD = \
88 $(real_pthread_LIBS) \
89 $(top_builddir)/test/pdiff/libpdiff.la \
90 $(top_builddir)/boilerplate/libcairoboilerplate.la \
91 $(top_builddir)/src/libcairo.la \
93 cairo_test_suite_DEPENDENCIES = \
94 $(top_builddir)/test/pdiff/libpdiff.la \
95 $(top_builddir)/boilerplate/libcairoboilerplate.la \
96 $(top_builddir)/src/libcairo.la
98 cairo_test_suite_DEPENDENCIES += \
103 EXTRA_PROGRAMS += cairo-test-trace
104 cairo_test_trace_SOURCES = \
108 cairo_test_trace_CFLAGS = $(AM_CFLAGS) $(real_pthread_CFLAGS)
109 cairo_test_trace_LDADD = \
110 $(real_pthread_LIBS) \
111 $(top_builddir)/test/pdiff/libpdiff.la \
112 $(top_builddir)/util/cairo-script/libcairo-script-interpreter.la \
113 $(top_builddir)/boilerplate/libcairoboilerplate.la \
114 $(top_builddir)/src/libcairo.la \
115 $(top_builddir)/util/cairo-missing/libcairo-missing.la \
118 cairo_test_trace_DEPENDENCIES = \
119 $(top_builddir)/test/pdiff/libpdiff.la \
120 $(top_builddir)/util/cairo-script/libcairo-script-interpreter.la \
121 $(top_builddir)/boilerplate/libcairoboilerplate.la \
122 $(top_builddir)/src/libcairo.la \
123 $(top_builddir)/util/cairo-missing/libcairo-missing.la \
127 BUILT_SOURCES += cairo-test-constructors.c
128 EXTRA_DIST += $(BUILT_SOURCES) $(noinst_SCRIPTS) COPYING make-cairo-test-constructors.sh run-cairo-test-suite.sh generate_refs.sh tiger.inc
129 CLEANFILES += $(BUILT_SOURCES)
144 # Any test for which the code committed to CVS is expected to fail
145 # should be listed here.
147 # This way, we can avoid being bothered by reports of bugs we are
148 # aware of, but users can still report when tests start behaving in
149 # unexpected ways on their system.
151 # Of course, before any "release" of cairo we should eliminate
152 # everything from this list by fixing the bugs. (We don't necessarily
153 # have to be that strict for "snapshots" though.)
155 # Analysis of XFAIL errors:
156 # alpha-similar - discrepancy between backends in applying color
157 # components of a pure alpha surface
158 # big-line - range overflow of fixed-point
159 # big-trap - range overflow of fixed-point
160 # degenerate-dash - needs path editing in PS to convert degenerate
161 # end-caps into the shapes as expected by cairo
162 # (Or maybe PS is the correct behaviour?)
163 # degenerate-path - undefined behaviour in PS, needs path editing to
164 # convert degenerate segments into circles/rectangles
165 # as expected by cairo
166 # device-offset-scale - complication of pre-multiplying device_offset
167 # into the pattern_matrix and then requiring further
168 # manipulation for SVG
169 # extend-pad - lacks implementation in pixman and consequently used
170 # as an excuse for lack of support in other backends
171 # fallback-resolution - The essential problem here is that the recording-surface
172 # has recorded a sequence of operations with one device
173 # transformation, and we attempt to replay it with
174 # another (basically a scale-factor for the falback
175 # resolution). Carl begun to look at this with his
176 # chain-of-bugs, but the can of worms is much bigger.
177 # It appears to be a design flaw in the recording-surface
178 # that may spread further...
179 # My solution would be to lock Behad and Adrian in a
180 # room, with Carl as a moderator and not let them out
181 # until they have come up with an interface and
182 # semantics that actually work. :-)
183 # in-fill-empty-trapezoid The cairo_in_fill () function can sometimes
184 # produce false positives when the tessellator
185 # produces empty trapezoids and the query
186 # point lands exactly on a trapezoid edge.
187 # long-lines - range overflow of fixed-point
188 # scale-offset-image - loss of precision converting a cairo matrix to
189 # scale-offset-similar pixman's fixed point format.
190 # self-copy-overlap - vector surfaces take snapshot of patterns in contrast
191 # to the raster backends which don't. One solution
192 # would be to clone overlapping areas of dst/source, so
193 # patterns were effectively snapshotted across all
195 # self-intersecting - incremental trapezoidation of strokes can generate
196 # overlapping traps. Needs self-intersection analysis
197 # on cairo_traps_t after strokes.
198 # Test case should also be expanded to hit special-case
199 # tessellators as well.
200 # surface-pattern-big...- Strange, unexplained results for SVG/PS.
202 alpha-similar$(EXEEXT) \
205 degenerate-dash$(EXEEXT) \
206 degenerate-path$(EXEEXT) \
207 device-offset-scale$(EXEEXT) \
208 extend-pad$(EXEEXT) \
209 fallback-resolution$(EXEEXT) \
210 in-fill-empty-trapezoid$(EXEEXT) \
211 long-lines$(EXEEXT) \
212 self-copy-overlap$(EXEEXT) \
213 self-intersecting$(EXEEXT) \
214 surface-pattern-big-scale-down$(EXEEXT) \
217 # Any test that doesn't generate a log file goes here
219 fallback-resolution \
231 # A target to summarise the failures
234 for t in output/*.log; do \
235 if grep -e '\<FAIL\>' $$t >/dev/null 2>&1; then \
236 FAILED_TESTS="$$FAILED_TESTS $$t"; \
239 if test -n "$$FAILED_TESTS"; then \
240 echo "Failed tests:"; \
242 for t in $$FAILED_TESTS; do \
243 name="$${t##output/}"; name="$${name%.log}"; \
244 echo -n " $$name: "; \
245 grep -e '\<FAIL\>' $$t | sed -e 's/.*TARGET: \([^ ]*\).*/\1/' | sort | uniq | tr '\n' ' '; \
247 for s in `grep -e '\<FAIL\>' $$t | sed -e 's/.*TARGET: \([^ ]*\).*/\1/' | sort | uniq`; do \
248 ss=`echo $$s | tr '-' '_'`; \
249 tt=`echo $$name | tr '-' '_'`; \
250 eval $$ss=\""$${!ss} $$tt"\"; \
251 echo $$surfaces | grep $$ss >/dev/null || surfaces="$$surfaces $$ss"; \
254 echo -n "Failures per surface - "; \
256 for s in $$surfaces; do \
257 ss=`echo $$s | tr '_' '-'`; \
258 test -n "$$first" && echo -n ", "; \
259 cnt=`echo $${!s} | wc -w`; \
260 echo -n "$$ss: $$cnt"; \
264 for s in $$surfaces; do \
265 ss=`echo $$s | tr '_' '-'`; \
266 cnt=`echo $${!s} | wc -w`; \
267 echo -n " $$ss [$$cnt]: "; \
268 echo $${!s} | tr '_' '-'; \
275 -I$(top_srcdir)/boilerplate \
276 -I$(top_srcdir)/util/cairo-missing \
277 -I$(top_srcdir)/util/cairo-script \
278 -I$(top_srcdir)/src \
279 -I$(top_builddir)/src \
281 AM_LDFLAGS = $(CAIRO_LDFLAGS)
283 $(top_builddir)/boilerplate/libcairoboilerplate.la: $(top_builddir)/src/libcairo.la
284 cd $(top_builddir)/boilerplate && $(MAKE) $(AM_MAKEFLAGS) libcairoboilerplate.la
286 $(top_builddir)/src/libcairo.la:
287 cd $(top_builddir)/src && $(MAKE) $(AM_MAKEFLAGS) libcairo.la
289 $(top_builddir)/test/pdiff/libpdiff.la:
290 cd $(top_builddir)/test/pdiff && $(MAKE) $(AM_MAKEFLAGS) libpdiff.la
292 $(top_builddir)/util/cairo-script/libcairo-script-interpreter.la: $(top_builddir)/src/libcairo.la
293 cd $(top_builddir)/util/cairo-script && $(MAKE) $(AM_MAKEFLAGS) libcairo-script-interpreter.la
295 EXTRA_PROGRAMS += imagediff png-flatten
297 imagediff_SOURCES = \
302 $(top_builddir)/test/pdiff/libpdiff.la \
303 $(top_builddir)/src/libcairo.la
305 png_flatten_SOURCES = png-flatten.c
306 png_flatten_LDADD = $(top_builddir)/src/libcairo.la \
310 check_PROGRAMS += any2ppm
311 any2ppm_CFLAGS = $(AM_CFLAGS) $(POPPLER_CFLAGS) $(LIBRSVG_CFLAGS) $(LIBSPECTRE_CFLAGS)
312 # add LDADD, so poppler/librsvg uses "our" cairo
313 any2ppm_LDFLAGS = $(AM_LDFLAGS) $(CAIRO_TEST_UNDEFINED_LDFLAGS)
315 $(top_builddir)/util/cairo-script/libcairo-script-interpreter.la \
316 $(top_builddir)/src/libcairo.la \
318 $(CAIROBOILERPLATE_LIBS) \
324 if CAIRO_CAN_TEST_PDF_SURFACE
325 check_PROGRAMS += pdf2png
326 pdf2png_CFLAGS = $(AM_CFLAGS) $(POPPLER_CFLAGS)
327 # add LDADD, so poppler uses "our" cairo
328 pdf2png_LDFLAGS = $(AM_LDFLAGS) $(CAIRO_TEST_UNDEFINED_LDFLAGS)
329 pdf2png_LDADD = $(top_builddir)/src/libcairo.la \
334 if CAIRO_CAN_TEST_SVG_SURFACE
335 check_PROGRAMS += svg2png
336 svg2png_CFLAGS = $(AM_CFLAGS) $(LIBRSVG_CFLAGS)
337 # add LDADD, so librsvg uses "our" cairo
338 svg2png_LDFLAGS = $(AM_LDFLAGS) $(CAIRO_TEST_UNDEFINED_LDFLAGS)
339 svg2png_LDADD = $(top_builddir)/src/libcairo.la \
345 check_PROGRAMS += ps2png
346 ps2png_CFLAGS = $(AM_CFLAGS) $(LIBSPECTRE_CFLAGS)
347 # add LDADD, so ps2png uses "our" cairo
348 ps2png_LDFLAGS = $(AM_LDFLAGS) $(CAIRO_TEST_UNDEFINED_LDFLAGS)
349 ps2png_LDADD = $(top_builddir)/src/libcairo.la \
354 EXTRA_PROGRAMS += $(TESTS)
356 # Do a funny transition of CAIRO_TEST_TARGET through TARGETS such that
357 # one can limit tested targets both through CAIRO_TEST_TARGET env var
358 # and TARGETS make var on the command line. Same for the rest.
359 TARGETS = $(CAIRO_TEST_TARGET)
360 TARGETS_EXCLUDE = $(CAIRO_TEST_TARGET_EXCLUDE)
361 NUM_THREADS = $(CAIRO_TEST_NUM_THREADS)
362 MODE = $(CAIRO_TEST_MODE)
364 # Same about ENV vs CAIRO_TEST_ENV. ENV is used with "make run" only
365 ENV = $(CAIRO_TEST_ENV)
367 TESTS_ENVIRONMENT = CAIRO_TEST_MODE="$(MODE)" CAIRO_TEST_TARGET="$(TARGETS)" CAIRO_TEST_TARGET_EXCLUDE="$(TARGETS_EXCLUDE)" CAIRO_TEST_NUM_THREADS="$(NUM_THREADS)" $(ENV)
369 EXTRA_VALGRIND_FLAGS = $(CAIRO_EXTRA_VALGRIND_FLAGS)
371 --tool=memcheck --suppressions=$(srcdir)/.valgrind-suppressions \
372 --track-origins=yes \
373 --leak-check=yes --show-reachable=yes \
374 $(EXTRA_VALGRIND_FLAGS)
382 create-for-stream.pdf \
383 create-for-stream.ps \
384 create-for-stream.svg \
385 svg-surface-source.out.svg \
386 pdf-surface-source.out.pdf \
387 ps-surface-source.out.ps \
397 # This used to be a simple 'echo ${RM} *.ps *.pdf *.svg *.etc', but
398 # most systems cannot handle all of our clean files together.
399 # Then it became a fancy find using many GNU extensions, but then the ugly
400 # reality of portability was raised and it became....
403 -${FIND} . -name '*.log' -print | ${XARGS} ${RM}
404 -${FIND} . -name '*.[is]' -print | ${XARGS} ${RM}
406 -${FIND} output -name '*.fail.*' -print | ${XARGS} ${RM}
407 -${FIND} output -name '*.pass.*' -print | ${XARGS} ${RM}
409 # The following definitions both should work.
410 #FAILED_TESTS = `grep -l '\<FAIL\>' $(test_sources:.c=.log) 2>/dev/null | sed -e 's/[.]log$$//' | xargs echo`
411 FAILED_TESTS = `grep -l '\<FAIL\>' $(test_sources:.c=.log) 2>/dev/null | tr '\n' ' ' | sed -e 's/[.]log */ /g; s/^ //; s/ $$//'`
413 recheck = check CAIRO_TESTS="$(FAILED_TESTS)"
415 # Re-checks all failed tests, i.e. tests with a log file that has a failure
417 @echo Re-checking failed tests
418 @$(MAKE) $(AM_MAKEFLAGS) $(recheck)
421 # Target doesn't fail if tests fail.
423 @$(MAKE) $(AM_MAKEFLAGS) check
426 # Target doesn't fail if tests fail.
428 @CAIRO_TESTS="$(FAILED_TESTS)"; \
429 $(MAKE) $(AM_MAKEFLAGS) check
431 # Run tests under a tool specified by TOOL. For example, make run TOOL=gdb
433 $(MAKE) $(AM_MAKEFLAGS) check TESTS_ENVIRONMENT='$(TESTS_ENVIRONMENT) $(top_builddir)/libtool --mode=execute env $(TOOL)'
435 # Check tests under valgrind. Saves log to valgrind-log
437 $(MAKE) $(AM_MAKEFLAGS) check TESTS_ENVIRONMENT='$(TESTS_ENVIRONMENT) CAIRO_TEST_MODE="$(MODE),foreground CAIRO_TEST_TIMEOUT=0" $(top_builddir)/libtool --mode=execute valgrind $(VALGRIND_FLAGS)' 2>&1 | tee valgrind-log
439 #%.log: %.c cairo-test-suite
440 #-./cairo-test-suite $(<:.c=)
442 NOLOG_TESTS_LOG = $(NOLOG_TESTS:=.log)
447 # Identify identical reference images
450 ( cd "$(srcdir)" && sha1sum *.ref.png | sort ) > ref.hash; \
451 join ref.hash ref.hash | grep -v -E '( .*.ref.png).*\1' | cut -d' ' -f 1-2 | sort -u
454 @tar cf $@ index.html testtable.js *.log output/*.log; \
455 for i in output/*.fail.png ; do \
456 testname=$${i#output/} ; \
457 testname=$${testname%%.*} ; \
458 echo tar uf $@ reference/$${testname}*.ref.png $${i%fail.png}out.png $${i%fail.png}diff.png ; \
459 tar uf $@ reference/$${testname}*.ref.png $${i%fail.png}out.png $${i%fail.png}diff.png ; \
462 results.tar.gz: results.tar
465 release-verify-sane-tests:
467 .PHONY: check-valgrind test recheck retest check-ref-dups release-verify-sane-tests
469 EXTRA_DIST += Makefile.win32