1 /* GLib testing framework runner
2 * Copyright (C) 2007 Sven Herzberg
3 * Copyright (C) 2007 Tim Janik
5 * SPDX-License-Identifier: LGPL-2.1-or-later
7 * This library is free software; you can redistribute it and/or
8 * modify it under the terms of the GNU Lesser General Public
9 * License as published by the Free Software Foundation; either
10 * version 2.1 of the License, or (at your option) any later version.
12 * This library is distributed in the hope that it will be useful,
13 * but WITHOUT ANY WARRANTY; without even the implied warranty of
14 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
15 * Lesser General Public License for more details.
17 * You should have received a copy of the GNU Lesser General Public
18 * License along with this library; if not, see <http://www.gnu.org/licenses/>.
23 #include <glib-unix.h>
33 /* the read buffer size in bytes */
34 #define READ_BUFFER_SIZE 4096
36 /* --- prototypes --- */
37 static int main_selftest (int argc,
39 static void parse_args (gint *argc_p,
42 /* --- variables --- */
43 static GIOChannel *ioc_report = NULL;
44 static gboolean gtester_quiet = FALSE;
45 static gboolean gtester_verbose = FALSE;
46 static gboolean gtester_list_tests = FALSE;
47 static gboolean gtester_selftest = FALSE;
48 static gboolean gtester_ignore_deprecation = FALSE;
49 static gboolean subtest_running = FALSE;
50 static gint subtest_exitstatus = 0;
51 static gboolean subtest_io_pending = FALSE;
52 static gboolean subtest_quiet = TRUE;
53 static gboolean subtest_verbose = FALSE;
54 static gboolean subtest_mode_fatal = TRUE;
55 static gboolean subtest_mode_perf = FALSE;
56 static gboolean subtest_mode_quick = TRUE;
57 static gboolean subtest_mode_undefined = TRUE;
58 static const gchar *subtest_seedstr = NULL;
59 static gchar *subtest_last_seed = NULL;
60 static GSList *subtest_paths = NULL;
61 static GSList *skipped_paths = NULL;
62 static GSList *subtest_args = NULL;
63 static gboolean testcase_open = FALSE;
64 static guint testcase_count = 0;
65 static guint testcase_fail_count = 0;
66 static const gchar *output_filename = NULL;
67 static guint log_indent = 0;
68 static gint log_fd = -1;
70 /* --- functions --- */
74 static const char spaces[] = " ";
75 gsize l = sizeof (spaces) - 1;
77 return spaces + l - n;
80 static void G_GNUC_PRINTF (1, 2)
81 test_log_printfe (const char *format,
87 va_start (args, format);
88 result = g_markup_vprintf_escaped (format, args);
91 r = write (log_fd, result, strlen (result));
92 while (r < 0 && errno == EINTR);
99 kill (getpid(), SIGTERM);
104 testcase_close (long double duration,
110 g_return_if_fail (testcase_open > 0);
111 test_log_printfe ("%s<duration>%.6Lf</duration>\n", sindent (log_indent), duration);
112 success = exit_status == G_TEST_RUN_SUCCESS || exit_status == G_TEST_RUN_SKIPPED;
113 test_log_printfe ("%s<status exit-status=\"%d\" n-forks=\"%d\" result=\"%s\"/>\n",
114 sindent (log_indent), exit_status, n_forks,
115 success ? "success" : "failed");
117 test_log_printfe ("%s</testcase>\n", sindent (log_indent));
123 case G_TEST_RUN_SUCCESS:
126 case G_TEST_RUN_SKIPPED:
134 if (!success && subtest_last_seed)
135 g_print ("GTester: last random seed: %s\n", subtest_last_seed);
137 testcase_fail_count += 1;
138 if (subtest_mode_fatal && !success)
143 test_log_msg (GTestLogMsg *msg)
145 switch (msg->log_type)
149 case G_TEST_LOG_NONE:
150 case G_TEST_LOG_START_SUITE:
151 case G_TEST_LOG_STOP_SUITE:
153 case G_TEST_LOG_ERROR:
154 strv = g_strsplit (msg->strings[0], "\n", -1);
155 for (i = 0; strv[i]; i++)
156 test_log_printfe ("%s<error>%s</error>\n", sindent (log_indent), strv[i]);
159 case G_TEST_LOG_START_BINARY:
160 test_log_printfe ("%s<binary file=\"%s\"/>\n", sindent (log_indent), msg->strings[0]);
161 subtest_last_seed = g_strdup (msg->strings[1]);
162 test_log_printfe ("%s<random-seed>%s</random-seed>\n", sindent (log_indent), subtest_last_seed);
164 case G_TEST_LOG_LIST_CASE:
165 g_print ("%s\n", msg->strings[0]);
167 case G_TEST_LOG_START_CASE:
171 gchar *sc = g_strconcat (msg->strings[0], ":", NULL);
172 gchar *sleft = g_strdup_printf ("%-68s", sc);
174 g_print ("%70s ", sleft);
177 g_return_if_fail (testcase_open == 0);
179 test_log_printfe ("%s<testcase path=\"%s\">\n", sindent (log_indent), msg->strings[0]);
182 case G_TEST_LOG_SKIP_CASE:
183 if (FALSE && gtester_verbose) /* enable to debug test case skipping logic */
185 gchar *sc = g_strconcat (msg->strings[0], ":", NULL);
186 gchar *sleft = g_strdup_printf ("%-68s", sc);
188 g_print ("%70s SKIPPED\n", sleft);
191 test_log_printfe ("%s<testcase path=\"%s\" skipped=\"1\"/>\n", sindent (log_indent), msg->strings[0]);
193 case G_TEST_LOG_STOP_CASE:
194 testcase_close (msg->nums[2], (int) msg->nums[0], (int) msg->nums[1]);
196 case G_TEST_LOG_MIN_RESULT:
197 case G_TEST_LOG_MAX_RESULT:
198 test_log_printfe ("%s<performance minimize=\"%d\" maximize=\"%d\" value=\"%.16Lg\">\n",
199 sindent (log_indent), msg->log_type == G_TEST_LOG_MIN_RESULT, msg->log_type == G_TEST_LOG_MAX_RESULT, msg->nums[0]);
200 test_log_printfe ("%s%s\n", sindent (log_indent + 2), msg->strings[0]);
201 test_log_printfe ("%s</performance>\n", sindent (log_indent));
203 case G_TEST_LOG_MESSAGE:
204 test_log_printfe ("%s<message>\n%s\n%s</message>\n", sindent (log_indent), msg->strings[0], sindent (log_indent));
210 child_report_cb (GIOChannel *source,
211 GIOCondition condition,
214 GTestLogBuffer *tlb = data;
215 GIOStatus status = G_IO_STATUS_NORMAL;
216 gboolean first_read_eof = FALSE, first_read = TRUE;
220 guint8 buffer[READ_BUFFER_SIZE];
221 GError *error = NULL;
222 status = g_io_channel_read_chars (source, (gchar*) buffer, sizeof (buffer), &length, &error);
223 if (first_read && (condition & G_IO_IN))
225 /* on some unixes (MacOS) we need to detect non-blocking fd EOF
226 * by an IO_IN select/poll followed by read()==0.
228 first_read_eof = length == 0;
234 g_test_log_buffer_push (tlb, length, buffer);
237 msg = g_test_log_buffer_pop (tlb);
241 g_test_log_msg_free (msg);
246 g_clear_error (&error);
247 /* ignore the io channel status, which will report intermediate EOFs for non blocking fds */
251 /* g_print ("LASTIOSTATE: first_read_eof=%d condition=%d\n", first_read_eof, condition); */
252 if (first_read_eof || (condition & (G_IO_ERR | G_IO_HUP)))
254 /* if there's no data to read and select() reports an error or hangup,
255 * the fd must have been closed remotely
257 subtest_io_pending = FALSE;
260 return TRUE; /* keep polling */
264 child_watch_cb (GPid pid,
268 g_spawn_close_pid (pid);
269 if (WIFEXITED (status)) /* normal exit */
270 subtest_exitstatus = WEXITSTATUS (status);
271 else /* signal or core dump, etc */
272 subtest_exitstatus = 0xffffffff;
273 subtest_running = FALSE;
277 queue_gfree (GSList **slistp,
280 *slistp = g_slist_prepend (*slistp, string);
285 unset_cloexec_fdp (gpointer fdp_data)
287 int r, *fdp = fdp_data;
289 r = fcntl (*fdp, F_SETFD, 0 /* FD_CLOEXEC */);
290 while (r < 0 && errno == EINTR);
294 launch_test_binary (const char *binary,
298 GSList *slist, *free_list = NULL;
299 GError *error = NULL;
303 gint report_pipe[2] = { -1, -1 };
304 guint child_report_cb_id = 0;
305 gboolean loop_pending;
308 if (!g_unix_open_pipe (report_pipe, O_CLOEXEC, &error))
310 if (subtest_mode_fatal)
311 g_error ("Failed to open pipe for test binary: %s: %s", binary, error->message);
313 g_warning ("Failed to open pipe for test binary: %s: %s", binary, error->message);
314 g_clear_error (&error);
319 for (slist = subtest_args; slist; slist = slist->next)
326 if (!subtest_mode_fatal)
328 /* Either -m=quick or -m=slow is always appended. */
330 if (subtest_mode_perf)
332 if (!subtest_mode_undefined)
334 if (gtester_list_tests)
341 for (slist = subtest_paths; slist; slist = slist->next)
343 for (slist = skipped_paths; slist; slist = slist->next)
347 argv = g_malloc ((argc + 2) * sizeof(gchar *));
349 for (slist = subtest_args; slist; slist = slist->next)
350 argv[i++] = (gchar*) slist->data;
351 /* argv[i++] = "--debug-log"; */
353 argv[i++] = "--quiet";
355 argv[i++] = "--verbose";
356 if (!subtest_mode_fatal)
357 argv[i++] = "--keep-going";
358 if (subtest_mode_quick)
359 argv[i++] = "-m=quick";
361 argv[i++] = "-m=slow";
362 if (subtest_mode_perf)
363 argv[i++] = "-m=perf";
364 if (!subtest_mode_undefined)
365 argv[i++] = "-m=no-undefined";
366 if (gtester_list_tests)
369 argv[i++] = queue_gfree (&free_list, g_strdup_printf ("--seed=%s", subtest_seedstr));
370 argv[i++] = queue_gfree (&free_list, g_strdup_printf ("--GTestLogFD=%u", report_pipe[1]));
372 argv[i++] = queue_gfree (&free_list, g_strdup_printf ("--GTestSkipCount=%u", skip_tests));
373 for (slist = subtest_paths; slist; slist = slist->next)
374 argv[i++] = queue_gfree (&free_list, g_strdup_printf ("-p=%s", (gchar*) slist->data));
375 for (slist = skipped_paths; slist; slist = slist->next)
376 argv[i++] = queue_gfree (&free_list, g_strdup_printf ("-s=%s", (gchar*) slist->data));
379 g_spawn_async_with_pipes (NULL, /* g_get_current_dir() */
382 G_SPAWN_DO_NOT_REAP_CHILD, /* G_SPAWN_SEARCH_PATH */
383 unset_cloexec_fdp, &report_pipe[1], /* pre-exec callback */
385 NULL, /* standard_input */
386 NULL, /* standard_output */
387 NULL, /* standard_error */
389 g_slist_foreach (free_list, (void(*)(void*,void*)) g_free, NULL);
390 g_slist_free (free_list);
392 close (report_pipe[1]);
395 g_print ("(pid=%lu)\n", (unsigned long) pid);
399 close (report_pipe[0]);
400 if (subtest_mode_fatal)
401 g_error ("Failed to execute test binary: %s: %s", argv[0], error->message);
403 g_warning ("Failed to execute test binary: %s: %s", argv[0], error->message);
404 g_clear_error (&error);
410 subtest_running = TRUE;
411 subtest_io_pending = TRUE;
412 tlb = g_test_log_buffer_new();
413 if (report_pipe[0] >= 0)
415 ioc_report = g_io_channel_unix_new (report_pipe[0]);
416 g_io_channel_set_flags (ioc_report, G_IO_FLAG_NONBLOCK, NULL);
417 g_io_channel_set_encoding (ioc_report, NULL, NULL);
418 g_io_channel_set_buffered (ioc_report, FALSE);
419 child_report_cb_id = g_io_add_watch_full (ioc_report, G_PRIORITY_DEFAULT - 1, G_IO_IN | G_IO_ERR | G_IO_HUP, child_report_cb, tlb, NULL);
420 g_io_channel_unref (ioc_report);
422 g_child_watch_add_full (G_PRIORITY_DEFAULT + 1, pid, child_watch_cb, NULL, NULL);
424 loop_pending = g_main_context_pending (NULL);
425 while (subtest_running || /* FALSE once child exits */
426 subtest_io_pending || /* FALSE once ioc_report closes */
427 loop_pending) /* TRUE while idler, etc are running */
429 /* g_print ("LOOPSTATE: subtest_running=%d subtest_io_pending=%d\n", subtest_running, subtest_io_pending); */
430 /* check for unexpected hangs that are not signalled on report_pipe */
431 if (!subtest_running && /* child exited */
432 subtest_io_pending && /* no EOF detected on report_pipe */
433 !loop_pending) /* no IO events pending however */
435 g_main_context_iteration (NULL, TRUE);
436 loop_pending = g_main_context_pending (NULL);
439 if (subtest_io_pending)
440 g_source_remove (child_report_cb_id);
442 close (report_pipe[0]);
443 g_test_log_buffer_free (tlb);
449 launch_test (const char *binary)
451 gboolean success = TRUE;
452 GTimer *btimer = g_timer_new();
453 gboolean need_restart;
457 g_print ("TEST: %s... ", binary);
460 test_log_printfe ("%s<testbinary path=\"%s\">\n", sindent (log_indent), binary);
462 g_timer_start (btimer);
463 subtest_exitstatus = 0;
464 success &= launch_test_binary (binary, testcase_count);
465 success &= subtest_exitstatus == 0;
466 need_restart = testcase_open != 0;
468 testcase_close (0, -256, 0);
469 g_timer_stop (btimer);
470 test_log_printfe ("%s<duration>%.6f</duration>\n", sindent (log_indent), g_timer_elapsed (btimer, NULL));
472 test_log_printfe ("%s</testbinary>\n", sindent (log_indent));
473 g_free (subtest_last_seed);
474 subtest_last_seed = NULL;
477 /* restart test binary, skipping processed test cases */
481 /* count the inability to run a test as a failure */
482 if (!success && testcase_count == 0)
483 testcase_fail_count++;
486 g_print ("%s: %s\n", !success ? "FAIL" : "PASS", binary);
487 g_timer_destroy (btimer);
488 if (subtest_mode_fatal && !success)
493 usage (gboolean just_version)
497 g_print ("gtester version %d.%d.%d\n", GLIB_MAJOR_VERSION, GLIB_MINOR_VERSION, GLIB_MICRO_VERSION);
500 g_print ("Usage:\n");
501 g_print ("gtester [OPTIONS] testprogram...\n\n");
502 /* 12345678901234567890123456789012345678901234567890123456789012345678901234567890 */
503 g_print ("Help Options:\n");
504 g_print (" -h, --help Show this help message\n\n");
505 g_print ("Utility Options:\n");
506 g_print (" -v, --version Print version information\n");
507 g_print (" --g-fatal-warnings Make warnings fatal (abort)\n");
508 g_print (" -k, --keep-going Continue running after tests failed\n");
509 g_print (" -l List paths of available test cases\n");
510 g_print (" -m {perf|slow|thorough|quick} Run test cases according to mode\n");
511 g_print (" -m {undefined|no-undefined} Run test cases according to mode\n");
512 g_print (" -p=TESTPATH Only start test cases matching TESTPATH\n");
513 g_print (" -s=TESTPATH Skip test cases matching TESTPATH\n");
514 g_print (" --seed=SEEDSTRING Start tests with random seed SEEDSTRING\n");
515 g_print (" -o=LOGFILE Write the test log to LOGFILE\n");
516 g_print (" -q, --quiet Suppress per test binary output\n");
517 g_print (" --verbose Report success per testcase\n");
521 parse_args (gint *argc_p,
524 guint argc = *argc_p;
525 gchar **argv = *argv_p;
527 /* parse known args */
528 for (i = 1; i < argc; i++)
530 if (strcmp (argv[i], "--g-fatal-warnings") == 0)
532 GLogLevelFlags fatal_mask = (GLogLevelFlags) g_log_set_always_fatal ((GLogLevelFlags) G_LOG_FATAL_MASK);
533 fatal_mask = (GLogLevelFlags) (fatal_mask | G_LOG_LEVEL_WARNING | G_LOG_LEVEL_CRITICAL);
534 g_log_set_always_fatal (fatal_mask);
537 else if (strcmp (argv[i], "--gtester-selftest") == 0)
539 gtester_selftest = TRUE;
541 break; /* stop parsing regular gtester arguments */
543 else if (strcmp (argv[i], "-h") == 0 || strcmp (argv[i], "--help") == 0)
549 else if (strcmp (argv[i], "-v") == 0 || strcmp (argv[i], "--version") == 0)
555 else if (strcmp (argv[i], "--keep-going") == 0 ||
556 strcmp (argv[i], "-k") == 0)
558 subtest_mode_fatal = FALSE;
561 else if (strcmp ("-p", argv[i]) == 0 || strncmp ("-p=", argv[i], 3) == 0)
563 gchar *equal = argv[i] + 2;
565 subtest_paths = g_slist_prepend (subtest_paths, equal + 1);
566 else if (i + 1 < argc)
569 subtest_paths = g_slist_prepend (subtest_paths, argv[i]);
573 else if (strcmp ("-s", argv[i]) == 0 || strncmp ("-s=", argv[i], 3) == 0)
575 gchar *equal = argv[i] + 2;
577 skipped_paths = g_slist_prepend (skipped_paths, equal + 1);
578 else if (i + 1 < argc)
581 skipped_paths = g_slist_prepend (skipped_paths, argv[i]);
585 else if (strcmp ("--test-arg", argv[i]) == 0 || strncmp ("--test-arg=", argv[i], 11) == 0)
587 gchar *equal = argv[i] + 10;
589 subtest_args = g_slist_prepend (subtest_args, equal + 1);
590 else if (i + 1 < argc)
593 subtest_args = g_slist_prepend (subtest_args, argv[i]);
597 else if (strcmp ("-o", argv[i]) == 0 || strncmp ("-o=", argv[i], 3) == 0)
599 gchar *equal = argv[i] + 2;
601 output_filename = equal + 1;
602 else if (i + 1 < argc)
605 output_filename = argv[i];
609 else if (strcmp ("-m", argv[i]) == 0 || strncmp ("-m=", argv[i], 3) == 0)
611 gchar *equal = argv[i] + 2;
612 const gchar *mode = "";
615 else if (i + 1 < argc)
620 if (strcmp (mode, "perf") == 0)
621 subtest_mode_perf = TRUE;
622 else if (strcmp (mode, "slow") == 0 || strcmp (mode, "thorough") == 0)
623 subtest_mode_quick = FALSE;
624 else if (strcmp (mode, "quick") == 0)
626 subtest_mode_quick = TRUE;
627 subtest_mode_perf = FALSE;
629 else if (strcmp (mode, "undefined") == 0)
630 subtest_mode_undefined = TRUE;
631 else if (strcmp (mode, "no-undefined") == 0)
632 subtest_mode_undefined = FALSE;
634 g_error ("unknown test mode: -m %s", mode);
637 else if (strcmp ("-q", argv[i]) == 0 || strcmp ("--quiet", argv[i]) == 0)
639 gtester_quiet = TRUE;
640 gtester_verbose = FALSE;
643 else if (strcmp ("--verbose", argv[i]) == 0)
645 gtester_quiet = FALSE;
646 gtester_verbose = TRUE;
649 else if (strcmp ("-l", argv[i]) == 0)
651 gtester_list_tests = TRUE;
654 else if (strcmp ("--seed", argv[i]) == 0 || strncmp ("--seed=", argv[i], 7) == 0)
656 gchar *equal = argv[i] + 6;
658 subtest_seedstr = equal + 1;
659 else if (i + 1 < argc)
662 subtest_seedstr = argv[i];
666 else if (strcmp ("--i-know-this-is-deprecated", argv[i]) == 0)
668 gtester_ignore_deprecation = TRUE;
674 for (i = 0; i < argc; i++)
690 g_set_prgname (argv[0]);
691 parse_args (&argc, &argv);
692 if (gtester_selftest)
693 return main_selftest (argc, argv);
701 if (!gtester_ignore_deprecation)
702 g_warning ("Deprecated: Since GLib 2.62, gtester and gtester-report are "
703 "deprecated. Port to TAP.");
708 log_fd = g_open (output_filename, O_WRONLY | O_CREAT | O_TRUNC, 0666);
711 g_error ("Failed to open log file '%s': %s", output_filename, g_strerror (errsv));
714 test_log_printfe ("<?xml version=\"1.0\"?>\n");
715 test_log_printfe ("<!-- Deprecated: Since GLib 2.62, gtester and "
716 "gtester-report are deprecated. Port to TAP. -->\n");
717 test_log_printfe ("%s<gtester>\n", sindent (log_indent));
719 for (ui = 1; ui < argc; ui++)
721 const char *binary = argv[ui];
722 launch_test (binary);
723 /* we only get here on success or if !subtest_mode_fatal */
726 test_log_printfe ("%s</gtester>\n", sindent (log_indent));
730 return testcase_fail_count == 0 ? EXIT_SUCCESS : EXIT_FAILURE;
734 fixture_setup (guint *fix,
735 gconstpointer test_data)
737 g_assert_cmphex (*fix, ==, 0);
741 fixture_test (guint *fix,
742 gconstpointer test_data)
744 g_assert_cmphex (*fix, ==, 0xdeadbeef);
745 g_test_message ("This is a test message API test message.");
746 g_test_bug_base ("http://www.example.com/bugtracker/");
748 g_test_bug_base ("http://www.example.com/bugtracker?bugnum=%s;cmd=showbug");
750 g_test_bug ("https://example.com/no-base-used");
753 fixture_teardown (guint *fix,
754 gconstpointer test_data)
756 g_assert_cmphex (*fix, ==, 0xdeadbeef);
760 main_selftest (int argc,
763 /* gtester main() for --gtester-selftest invocations */
764 g_test_init (&argc, &argv, NULL);
765 g_test_add ("/gtester/fixture-test", guint, NULL, fixture_setup, fixture_test, fixture_teardown);