1 /* GLib testing framework runner
2 * Copyright (C) 2007 Sven Herzberg
3 * Copyright (C) 2007 Tim Janik
5 * This library is free software; you can redistribute it and/or
6 * modify it under the terms of the GNU Lesser General Public
7 * License as published by the Free Software Foundation; either
8 * version 2 of the License, or (at your option) any later version.
10 * This library is distributed in the hope that it will be useful,
11 * but WITHOUT ANY WARRANTY; without even the implied warranty of
12 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
13 * Lesser General Public License for more details.
15 * You should have received a copy of the GNU Lesser General Public
16 * License along with this library; if not, write to the
17 * Free Software Foundation, Inc., 59 Temple Place - Suite 330,
18 * Boston, MA 02111-1307, USA.
30 /* the read buffer size in bytes */
31 #define READ_BUFFER_SIZE 4096
33 /* --- prototypes --- */
34 static int main_selftest (int argc,
36 static void parse_args (gint *argc_p,
39 /* --- variables --- */
40 static GIOChannel *ioc_report = NULL;
41 static gboolean gtester_quiet = FALSE;
42 static gboolean gtester_verbose = FALSE;
43 static gboolean gtester_list_tests = FALSE;
44 static gboolean gtester_selftest = FALSE;
45 static gboolean subtest_running = FALSE;
46 static gint subtest_exitstatus = 0;
47 static gboolean subtest_io_pending = FALSE;
48 static gboolean subtest_quiet = TRUE;
49 static gboolean subtest_verbose = FALSE;
50 static gboolean subtest_mode_fatal = TRUE;
51 static gboolean subtest_mode_perf = FALSE;
52 static gboolean subtest_mode_quick = TRUE;
53 static const gchar *subtest_seedstr = NULL;
54 static gchar *subtest_last_seed = NULL;
55 static GSList *subtest_paths = NULL;
56 static GSList *subtest_args = NULL;
57 static gboolean testcase_open = FALSE;
58 static guint testcase_count = 0;
59 static guint testcase_fail_count = 0;
60 static const gchar *output_filename = NULL;
61 static guint log_indent = 0;
62 static gint log_fd = -1;
64 /* --- functions --- */
68 static const char spaces[] = " ";
69 int l = sizeof (spaces) - 1;
71 return spaces + l - n;
74 static void G_GNUC_PRINTF (1, 2)
75 test_log_printfe (const char *format,
81 va_start (args, format);
82 result = g_markup_vprintf_escaped (format, args);
85 r = write (log_fd, result, strlen (result));
86 while (r < 0 && errno == EINTR);
93 kill (getpid(), SIGTERM);
98 testcase_close (long double duration,
102 g_return_if_fail (testcase_open > 0);
103 test_log_printfe ("%s<duration>%.6Lf</duration>\n", sindent (log_indent), duration);
104 test_log_printfe ("%s<status exit-status=\"%d\" n-forks=\"%d\" result=\"%s\"/>\n",
105 sindent (log_indent), exit_status, n_forks,
106 exit_status ? "failed" : "success");
108 test_log_printfe ("%s</testcase>\n", sindent (log_indent));
111 g_print ("%s\n", exit_status ? "FAIL" : "OK");
112 if (exit_status && subtest_last_seed)
113 g_print ("GTester: last random seed: %s\n", subtest_last_seed);
115 testcase_fail_count += 1;
116 if (subtest_mode_fatal && testcase_fail_count)
121 test_log_msg (GTestLogMsg *msg)
123 switch (msg->log_type)
127 case G_TEST_LOG_NONE:
129 case G_TEST_LOG_ERROR:
130 strv = g_strsplit (msg->strings[0], "\n", -1);
131 for (i = 0; strv[i]; i++)
132 test_log_printfe ("%s<error>%s</error>\n", sindent (log_indent), strv[i]);
135 case G_TEST_LOG_START_BINARY:
136 test_log_printfe ("%s<binary file=\"%s\"/>\n", sindent (log_indent), msg->strings[0]);
137 subtest_last_seed = g_strdup (msg->strings[1]);
138 test_log_printfe ("%s<random-seed>%s</random-seed>\n", sindent (log_indent), subtest_last_seed);
140 case G_TEST_LOG_LIST_CASE:
141 g_print ("%s\n", msg->strings[0]);
143 case G_TEST_LOG_START_CASE:
147 gchar *sc = g_strconcat (msg->strings[0], ":", NULL);
148 gchar *sleft = g_strdup_printf ("%-68s", sc);
150 g_print ("%70s ", sleft);
153 g_return_if_fail (testcase_open == 0);
155 test_log_printfe ("%s<testcase path=\"%s\">\n", sindent (log_indent), msg->strings[0]);
158 case G_TEST_LOG_SKIP_CASE:
159 if (FALSE && gtester_verbose) /* enable to debug test case skipping logic */
161 gchar *sc = g_strconcat (msg->strings[0], ":", NULL);
162 gchar *sleft = g_strdup_printf ("%-68s", sc);
164 g_print ("%70s SKIPPED\n", sleft);
167 test_log_printfe ("%s<testcase path=\"%s\" skipped=\"1\"/>\n", sindent (log_indent), msg->strings[0]);
169 case G_TEST_LOG_STOP_CASE:
170 testcase_close (msg->nums[2], (int) msg->nums[0], (int) msg->nums[1]);
172 case G_TEST_LOG_MIN_RESULT:
173 case G_TEST_LOG_MAX_RESULT:
174 test_log_printfe ("%s<performance minimize=\"%d\" maximize=\"%d\" value=\"%.16Lg\">\n",
175 sindent (log_indent), msg->log_type == G_TEST_LOG_MIN_RESULT, msg->log_type == G_TEST_LOG_MAX_RESULT, msg->nums[0]);
176 test_log_printfe ("%s%s\n", sindent (log_indent + 2), msg->strings[0]);
177 test_log_printfe ("%s</performance>\n", sindent (log_indent));
179 case G_TEST_LOG_MESSAGE:
180 test_log_printfe ("%s<message>\n%s\n%s</message>\n", sindent (log_indent), msg->strings[0], sindent (log_indent));
186 child_report_cb (GIOChannel *source,
187 GIOCondition condition,
190 GTestLogBuffer *tlb = data;
191 GIOStatus status = G_IO_STATUS_NORMAL;
192 gboolean first_read_eof = FALSE, first_read = TRUE;
196 guint8 buffer[READ_BUFFER_SIZE];
197 GError *error = NULL;
198 status = g_io_channel_read_chars (source, (gchar*) buffer, sizeof (buffer), &length, &error);
199 if (first_read && (condition & G_IO_IN))
201 /* on some unixes (MacOS) we need to detect non-blocking fd EOF
202 * by an IO_IN select/poll followed by read()==0.
204 first_read_eof = length == 0;
210 g_test_log_buffer_push (tlb, length, buffer);
213 msg = g_test_log_buffer_pop (tlb);
217 g_test_log_msg_free (msg);
222 g_clear_error (&error);
223 /* ignore the io channel status, which will report intermediate EOFs for non blocking fds */
227 /* g_print ("LASTIOSTATE: first_read_eof=%d condition=%d\n", first_read_eof, condition); */
228 if (first_read_eof || (condition & (G_IO_ERR | G_IO_HUP)))
230 /* if there's no data to read and select() reports an error or hangup,
231 * the fd must have been closed remotely
233 subtest_io_pending = FALSE;
236 return TRUE; /* keep polling */
240 child_watch_cb (GPid pid,
244 g_spawn_close_pid (pid);
245 if (WIFEXITED (status)) /* normal exit */
246 subtest_exitstatus = WEXITSTATUS (status);
247 else /* signal or core dump, etc */
248 subtest_exitstatus = 0xffffffff;
249 subtest_running = FALSE;
253 queue_gfree (GSList **slistp,
256 *slistp = g_slist_prepend (*slistp, string);
261 unset_cloexec_fdp (gpointer fdp_data)
263 int r, *fdp = fdp_data;
265 r = fcntl (*fdp, F_SETFD, 0 /* FD_CLOEXEC */);
266 while (r < 0 && errno == EINTR);
270 launch_test_binary (const char *binary,
274 GSList *slist, *free_list = NULL;
275 GError *error = NULL;
279 gint report_pipe[2] = { -1, -1 };
280 guint child_report_cb_id = 0;
281 gboolean loop_pending;
284 if (pipe (report_pipe) < 0)
286 if (subtest_mode_fatal)
287 g_error ("Failed to open pipe for test binary: %s: %s", binary, g_strerror (errno));
289 g_warning ("Failed to open pipe for test binary: %s: %s", binary, g_strerror (errno));
294 for (slist = subtest_args; slist; slist = slist->next)
301 if (!subtest_mode_fatal)
303 if (subtest_mode_quick)
307 if (subtest_mode_perf)
309 if (gtester_list_tests)
316 for (slist = subtest_paths; slist; slist = slist->next)
320 argv = g_malloc ((argc + 2) * sizeof(gchar *));
322 for (slist = subtest_args; slist; slist = slist->next)
323 argv[i++] = (gchar*) slist->data;
324 /* argv[i++] = "--debug-log"; */
326 argv[i++] = "--quiet";
328 argv[i++] = "--verbose";
329 if (!subtest_mode_fatal)
330 argv[i++] = "--keep-going";
331 if (subtest_mode_quick)
332 argv[i++] = "-m=quick";
334 argv[i++] = "-m=slow";
335 if (subtest_mode_perf)
336 argv[i++] = "-m=perf";
337 if (gtester_list_tests)
340 argv[i++] = queue_gfree (&free_list, g_strdup_printf ("--seed=%s", subtest_seedstr));
341 argv[i++] = queue_gfree (&free_list, g_strdup_printf ("--GTestLogFD=%u", report_pipe[1]));
343 argv[i++] = queue_gfree (&free_list, g_strdup_printf ("--GTestSkipCount=%u", skip_tests));
344 for (slist = subtest_paths; slist; slist = slist->next)
345 argv[i++] = queue_gfree (&free_list, g_strdup_printf ("-p=%s", (gchar*) slist->data));
348 g_spawn_async_with_pipes (NULL, /* g_get_current_dir() */
351 G_SPAWN_DO_NOT_REAP_CHILD, /* G_SPAWN_SEARCH_PATH */
352 unset_cloexec_fdp, &report_pipe[1], /* pre-exec callback */
354 NULL, /* standard_input */
355 NULL, /* standard_output */
356 NULL, /* standard_error */
358 g_slist_foreach (free_list, (void(*)(void*,void*)) g_free, NULL);
359 g_slist_free (free_list);
361 close (report_pipe[1]);
364 g_print ("(pid=%lu)\n", (unsigned long) pid);
368 close (report_pipe[0]);
369 if (subtest_mode_fatal)
370 g_error ("Failed to execute test binary: %s: %s", argv[0], error->message);
372 g_warning ("Failed to execute test binary: %s: %s", argv[0], error->message);
373 g_clear_error (&error);
379 subtest_running = TRUE;
380 subtest_io_pending = TRUE;
381 tlb = g_test_log_buffer_new();
382 if (report_pipe[0] >= 0)
384 ioc_report = g_io_channel_unix_new (report_pipe[0]);
385 g_io_channel_set_flags (ioc_report, G_IO_FLAG_NONBLOCK, NULL);
386 g_io_channel_set_encoding (ioc_report, NULL, NULL);
387 g_io_channel_set_buffered (ioc_report, FALSE);
388 child_report_cb_id = g_io_add_watch_full (ioc_report, G_PRIORITY_DEFAULT - 1, G_IO_IN | G_IO_ERR | G_IO_HUP, child_report_cb, tlb, NULL);
389 g_io_channel_unref (ioc_report);
391 g_child_watch_add_full (G_PRIORITY_DEFAULT + 1, pid, child_watch_cb, NULL, NULL);
393 loop_pending = g_main_context_pending (NULL);
394 while (subtest_running || /* FALSE once child exits */
395 subtest_io_pending || /* FALSE once ioc_report closes */
396 loop_pending) /* TRUE while idler, etc are running */
398 /* g_print ("LOOPSTATE: subtest_running=%d subtest_io_pending=%d\n", subtest_running, subtest_io_pending); */
399 /* check for unexpected hangs that are not signalled on report_pipe */
400 if (!subtest_running && /* child exited */
401 subtest_io_pending && /* no EOF detected on report_pipe */
402 !loop_pending) /* no IO events pending however */
404 g_main_context_iteration (NULL, TRUE);
405 loop_pending = g_main_context_pending (NULL);
408 g_source_remove (child_report_cb_id);
409 close (report_pipe[0]);
410 g_test_log_buffer_free (tlb);
416 launch_test (const char *binary)
418 gboolean success = TRUE;
419 GTimer *btimer = g_timer_new();
420 gboolean need_restart;
422 testcase_fail_count = 0;
424 g_print ("TEST: %s... ", binary);
427 test_log_printfe ("%s<testbinary path=\"%s\">\n", sindent (log_indent), binary);
429 g_timer_start (btimer);
430 subtest_exitstatus = 0;
431 success &= launch_test_binary (binary, testcase_count);
432 success &= subtest_exitstatus == 0;
433 need_restart = testcase_open != 0;
435 testcase_close (0, -256, 0);
436 g_timer_stop (btimer);
437 test_log_printfe ("%s<duration>%.6f</duration>\n", sindent (log_indent), g_timer_elapsed (btimer, NULL));
439 test_log_printfe ("%s</testbinary>\n", sindent (log_indent));
440 g_free (subtest_last_seed);
441 subtest_last_seed = NULL;
444 /* restart test binary, skipping processed test cases */
449 g_print ("%s: %s\n", testcase_fail_count || !success ? "FAIL" : "PASS", binary);
450 g_timer_destroy (btimer);
451 if (subtest_mode_fatal && !success)
456 usage (gboolean just_version)
460 g_print ("gtester version %d.%d.%d\n", GLIB_MAJOR_VERSION, GLIB_MINOR_VERSION, GLIB_MICRO_VERSION);
463 g_print ("Usage: gtester [OPTIONS] testprogram...\n");
464 /* 12345678901234567890123456789012345678901234567890123456789012345678901234567890 */
465 g_print ("Options:\n");
466 g_print (" -h, --help show this help message\n");
467 g_print (" -v, --version print version informations\n");
468 g_print (" --g-fatal-warnings make warnings fatal (abort)\n");
469 g_print (" -k, --keep-going continue running after tests failed\n");
470 g_print (" -l list paths of available test cases\n");
471 g_print (" -m=perf, -m=slow, -m=quick -m=thorough\n");
472 g_print (" run test cases in mode perf, slow/thorough or quick (default)\n");
473 g_print (" -p=TESTPATH only start test cases matching TESTPATH\n");
474 g_print (" --seed=SEEDSTRING start all tests with random number seed SEEDSTRING\n");
475 g_print (" -o=LOGFILE write the test log to LOGFILE\n");
476 g_print (" -q, --quiet suppress per test binary output\n");
477 g_print (" --verbose report success per testcase\n");
481 parse_args (gint *argc_p,
484 guint argc = *argc_p;
485 gchar **argv = *argv_p;
487 /* parse known args */
488 for (i = 1; i < argc; i++)
490 if (strcmp (argv[i], "--g-fatal-warnings") == 0)
492 GLogLevelFlags fatal_mask = (GLogLevelFlags) g_log_set_always_fatal ((GLogLevelFlags) G_LOG_FATAL_MASK);
493 fatal_mask = (GLogLevelFlags) (fatal_mask | G_LOG_LEVEL_WARNING | G_LOG_LEVEL_CRITICAL);
494 g_log_set_always_fatal (fatal_mask);
497 else if (strcmp (argv[i], "--gtester-selftest") == 0)
499 gtester_selftest = TRUE;
501 break; /* stop parsing regular gtester arguments */
503 else if (strcmp (argv[i], "-h") == 0 || strcmp (argv[i], "--help") == 0)
509 else if (strcmp (argv[i], "-v") == 0 || strcmp (argv[i], "--version") == 0)
515 else if (strcmp (argv[i], "--keep-going") == 0 ||
516 strcmp (argv[i], "-k") == 0)
518 subtest_mode_fatal = FALSE;
521 else if (strcmp ("-p", argv[i]) == 0 || strncmp ("-p=", argv[i], 3) == 0)
523 gchar *equal = argv[i] + 2;
525 subtest_paths = g_slist_prepend (subtest_paths, equal + 1);
526 else if (i + 1 < argc)
529 subtest_paths = g_slist_prepend (subtest_paths, argv[i]);
533 else if (strcmp ("--test-arg", argv[i]) == 0 || strncmp ("--test-arg=", argv[i], 11) == 0)
535 gchar *equal = argv[i] + 10;
537 subtest_args = g_slist_prepend (subtest_args, equal + 1);
538 else if (i + 1 < argc)
541 subtest_args = g_slist_prepend (subtest_args, argv[i]);
545 else if (strcmp ("-o", argv[i]) == 0 || strncmp ("-o=", argv[i], 3) == 0)
547 gchar *equal = argv[i] + 2;
549 output_filename = equal + 1;
550 else if (i + 1 < argc)
553 output_filename = argv[i];
557 else if (strcmp ("-m", argv[i]) == 0 || strncmp ("-m=", argv[i], 3) == 0)
559 gchar *equal = argv[i] + 2;
560 const gchar *mode = "";
563 else if (i + 1 < argc)
568 if (strcmp (mode, "perf") == 0)
569 subtest_mode_perf = TRUE;
570 else if (strcmp (mode, "slow") == 0 || strcmp (mode, "thorough") == 0)
571 subtest_mode_quick = FALSE;
572 else if (strcmp (mode, "quick") == 0)
574 subtest_mode_quick = TRUE;
575 subtest_mode_perf = FALSE;
578 g_error ("unknown test mode: -m %s", mode);
581 else if (strcmp ("-q", argv[i]) == 0 || strcmp ("--quiet", argv[i]) == 0)
583 gtester_quiet = TRUE;
584 gtester_verbose = FALSE;
587 else if (strcmp ("--verbose", argv[i]) == 0)
589 gtester_quiet = FALSE;
590 gtester_verbose = TRUE;
593 else if (strcmp ("-l", argv[i]) == 0)
595 gtester_list_tests = TRUE;
598 else if (strcmp ("--seed", argv[i]) == 0 || strncmp ("--seed=", argv[i], 7) == 0)
600 gchar *equal = argv[i] + 6;
602 subtest_seedstr = equal + 1;
603 else if (i + 1 < argc)
606 subtest_seedstr = argv[i];
613 for (i = 1; i < argc; i++)
624 do_nothing (gpointer data)
636 g_timeout_add_seconds (5, do_nothing, NULL);
638 /* some unices need SA_RESTART for SIGCHLD to return -EAGAIN for io.
639 * we must fiddle with sigaction() *before* glib is used, otherwise
640 * we could revoke signal hanmdler setups from glib initialization code.
645 struct sigaction osa;
646 sa.sa_handler = SIG_DFL;
647 sigfillset (&sa.sa_mask);
648 sa.sa_flags = SA_RESTART;
649 sigaction (SIGCHLD, &sa, &osa);
652 g_set_prgname (argv[0]);
653 parse_args (&argc, &argv);
654 if (gtester_selftest)
655 return main_selftest (argc, argv);
665 log_fd = g_open (output_filename, O_WRONLY | O_CREAT | O_TRUNC, 0666);
667 g_error ("Failed to open log file '%s': %s", output_filename, g_strerror (errno));
670 test_log_printfe ("<?xml version=\"1.0\"?>\n");
671 test_log_printfe ("%s<gtester>\n", sindent (log_indent));
673 for (ui = 1; ui < argc; ui++)
675 const char *binary = argv[ui];
676 launch_test (binary);
677 /* we only get here on success or if !subtest_mode_fatal */
680 test_log_printfe ("%s</gtester>\n", sindent (log_indent));
684 return testcase_fail_count == 0 ? 0 : 1;
688 fixture_setup (guint *fix,
689 gconstpointer test_data)
691 g_assert_cmphex (*fix, ==, 0);
695 fixture_test (guint *fix,
696 gconstpointer test_data)
698 g_assert_cmphex (*fix, ==, 0xdeadbeef);
699 g_test_message ("This is a test message API test message.");
700 g_test_bug_base ("http://www.example.com/bugtracker/");
702 g_test_bug_base ("http://www.example.com/bugtracker?bugnum=%s;cmd=showbug");
706 fixture_teardown (guint *fix,
707 gconstpointer test_data)
709 g_assert_cmphex (*fix, ==, 0xdeadbeef);
713 main_selftest (int argc,
716 /* gtester main() for --gtester-selftest invokations */
717 g_test_init (&argc, &argv, NULL);
718 g_test_add ("/gtester/fixture-test", guint, NULL, fixture_setup, fixture_test, fixture_teardown);