packaging/test-runner.c

   1 /*
   2  * This file contains standalone test-runner.
   3  * This file is NOT part of GLib project.
   4  *
   5  * Copyright (c) 2016 Samsung Electronics Co., Ltd All Rights Reserved
   6  * Author: Kazimierz Krosman <k.krosman@samsung.com>
   7  *
   8  * MIT License
   9  *
  10  * Permission is hereby granted, free of charge, to any person
  11  * obtaining a copy of this software and associated documentation
  12  * files (the "Software"), to deal in the Software without
  13  * restriction, including without limitation the rights to use, copy,
  14  * modify, merge, publish, distribute, sublicense, and/or sell copies
  15  * of the Software, and to permit persons to whom the Software is
  16  * furnished to do so, subject to the following conditions:
  17  *
  18  * The above copyright notice and this permission notice shall be
  19  * included in all copies or substantial portions of the Software.
  20  *
  21  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  22  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  23  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
  24  * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  25  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
  26  * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
  27  * THE SOFTWARE.
  28  */
  29 #include <unistd.h>
  30 #include <stdio.h>
  31 #include <stdlib.h>
  32 #include <string.h>
  33 #include <getopt.h>
  34 #include <fcntl.h>
  35 #include <limits.h>
  36 #include <signal.h>
  37 #include <sys/types.h>
  38 #include <sys/stat.h>
  39 #include <sys/select.h>
  40 #include <sys/time.h>
  41 #include <ctype.h>
  42 #include <time.h>
  43 #include <sys/select.h>
  44 #include <errno.h>
  45 #include <sys/types.h>
  46 #include <sys/wait.h>
  47 #include <assert.h>
  48 #include <stdbool.h>
  49
  50 #define MAX_TC_NUM 1024
  51 #define MAX_BUFFER (64*1024)
  52 #define MAX_COMMENT 1024
  53
  54 enum {
  55         INIT_TEST,
  56         NEW_STDOUT,
  57         NEW_STDERR,
  58         RESULT_CODE,
  59         RESULT_SIGNAL,
  60         RESULT_ERROR,
  61         RESULT_TIMEOUT
  62 };
  63
  64 struct test_result {
  65         bool is_positive;
  66         char comment[MAX_COMMENT];
  67         char result[MAX_COMMENT];
  68         char name[MAX_COMMENT];
  69 };
  70
  71 struct test_case {
  72         const char* name;
  73         const char* description;
  74 };
  75
  76 struct binary {
  77         const char* path;
  78         const char* name;
  79         struct test_case* test_cases;
  80         int timeout;
  81
  82         char** (*prepare_args) (const struct binary* b, const char* test_name);
  83         void (*parse) (const struct binary* b, const char* test_name, char* buffer, int state_change, int state_option);
  84         int (*init)(void);
  85         int (*clean)(void);
  86 };
  87
  88 char* get_test_id(char* dest, const struct binary* b, const char* test_name);
  89 void add_test_result(const char* test_id, const char* result, const char* comment, int res);
  90
  91 enum {
  92         PIPE_READ,
  93         PIPE_WRITE,
  94 };
  95
  96 void parse_test_state(const struct binary* b, const char* test_name, char* buffer, int state_change, int state_option);
  97 void parse_one_test_one_binary(const struct binary* b, const char* test_name, char* buffer, int state_change, int state_option);
  98 char** prepare_args_for_binary(const struct binary* b, const char* test_name);
  99
 100 static struct test_case test_case_desc_01[] = {
 101         {"", "Checks if the library supports format of addresses described in the D-Bus specification (key/value pairs for each transport are valid)"},
 102         {NULL, NULL}
 103 };
 104
 105 static struct test_case test_case_desc_02[] = {
 106         {"", "Test case for GNOME #662395"},
 107         {NULL, NULL}
 108 };
 109
 110 static struct test_case test_case_desc_03[] = {
 111         {"", "Test dbus authentication methods"},
 112         {NULL, NULL}
 113 };
 114
 115 static struct test_case test_case_desc_04[] = {
 116         {"", "Test 'lock' and 'copy' operations on GDBusMessage object"},
 117         {NULL, NULL}
 118 };
 119
 120 static struct test_case test_case_desc_05[] = {
 121         {"", "Test 'non-socket' connections (connection via pipes and GIOStreams objects)"},
 122         {NULL, NULL}
 123 };
 124
 125 static struct test_case test_case_desc_06[] = {
 126         {"", "Test overflowing socket buffer in producer/consumer scenario (UNIX sockets as a transport mechanism)"},
 127         {NULL, NULL}
 128 };
 129
 130 static struct test_case test_case_desc_07[] = {
 131         {"", "Test that peer-to-peer connections work"},
 132         {NULL, NULL}
 133 };
 134
 135 static struct test_case test_case_desc_08[] = {
 136         {"", "Test that peer-to-peer connections work (using GDBusObjectManagerServer and GDBusObjectManager objects)"},
 137         {NULL, NULL}
 138 };
 139
 140 /* This table is used to start binaries */
 141 struct binary tests[] = {
 142         /*path, name, TC_table, timeout in us, prepare_args_handler, parse_function_handler, init_handler, clean_handler*/
 143         {TESTS_DIR "/gdbus-addresses",               "gdbus-addresses",               test_case_desc_01, 5000*1000, prepare_args_for_binary, parse_one_test_one_binary, NULL, NULL},
 144         {TESTS_DIR "/gdbus-connection-flush",        "gdbus-connection-flush",        test_case_desc_02, 5000*1000, prepare_args_for_binary, parse_one_test_one_binary, NULL, NULL},
 145         {TESTS_DIR "/gdbus-server-auth",             "gdbus-server-auth",             test_case_desc_03, 5000*1000, prepare_args_for_binary, parse_one_test_one_binary, NULL, NULL},
 146         {TESTS_DIR "/gdbus-message",                 "gdbus-message",                 test_case_desc_04, 5000*1000, prepare_args_for_binary, parse_one_test_one_binary, NULL, NULL},
 147         {TESTS_DIR "/gdbus-non-socket",              "gdbus-non-socket",              test_case_desc_05, 5000*1000, prepare_args_for_binary, parse_one_test_one_binary, NULL, NULL},
 148         {TESTS_DIR "/gdbus-overflow",                "gdbus-overflow",                test_case_desc_06, 5000*1000, prepare_args_for_binary, parse_one_test_one_binary, NULL, NULL},
 149         {TESTS_DIR "/gdbus-peer",                    "gdbus-peer",                    test_case_desc_07, 5000*1000, prepare_args_for_binary, parse_one_test_one_binary, NULL, NULL},
 150         {TESTS_DIR "/gdbus-peer-object-manager",     "gdbus-peer-object-manager",     test_case_desc_08, 5000*1000, prepare_args_for_binary, parse_one_test_one_binary, NULL, NULL},
 151 };
 152
 153 static char* args[3];
 154 char** prepare_args_for_binary(const struct binary* b, const char* test_name)
 155 {
 156         args[0] = (char*)b->name;
 157         args[1] = NULL;
 158         return args;
 159 }
 160
 161 void parse_one_test_one_binary(const struct binary* b, const char* test_name, char* buffer, int state_change, int state_option)
 162 {
 163         char test_id[MAX_COMMENT];
 164
 165         switch(state_change) {
 166         case INIT_TEST:
 167                 break;
 168         case NEW_STDOUT:
 169                 buffer[state_option] = 0;
 170                 get_test_id(test_id, b, test_name);
 171                 fprintf(stderr, "[stdout][%s]%s\n",test_id, buffer);
 172                 break;
 173         case NEW_STDERR:
 174                 buffer[state_option] = 0;
 175                 get_test_id(test_id, b, test_name);
 176                 fprintf(stderr, "[stderr][%s]%s\n",test_id, buffer);
 177                 break;
 178         case RESULT_CODE:
 179                 get_test_id(test_id, b, test_name);
 180                 if (state_option == 0)
 181                         add_test_result(test_id, "PASS", "", 1);
 182                 else if (state_option == 77)
 183                         add_test_result(test_id, "SKIP", "", 0);
 184                 else
 185                         add_test_result(test_id, "FAIL", "", 0);
 186                 break;
 187         case RESULT_SIGNAL:
 188                 get_test_id(test_id, b, test_name);
 189                 add_test_result(test_id, "FAIL", "Finished by SIGNAL", 0);
 190                 break;
 191         case RESULT_TIMEOUT:
 192                 get_test_id(test_id, b, test_name);
 193                 add_test_result(test_id, "FAIL", "Test TIMEOUT", 0);
 194                 break;
 195         }
 196 }
 197
 198 static struct option long_options[] = {
 199         {"list",        no_argument,       0, 'l'},
 200         {"run",         required_argument, 0, 'r'},
 201         {"description", required_argument, 0, 'd'},
 202         {0,             0,                 0,  0 }
 203 };
 204
 205 static int stdin_pipe[2];
 206 static int stdout_pipe[2];
 207 static int stderr_pipe[2];
 208 static int gravedigger_pipe[2];
 209 static char buffer[MAX_BUFFER];
 210 static const char* requested_tc[MAX_TC_NUM];
 211
 212 char* get_test_id(char* dest, const struct binary* b, const char* test_name)
 213 {
 214         int len = strlen(b->name);
 215         memcpy(dest, b->name, len);
 216         memcpy(dest + len, test_name, strlen(test_name)+1);
 217         return dest;
 218 }
 219
 220 static void print_description(const char* name, const char* description)
 221 {
 222         printf("%s;%s\n",name, description);
 223 }
 224
 225 static void print_list(const char* test_name)
 226 {
 227         unsigned int i;
 228         char full_name[MAX_COMMENT];
 229         for (i = 0;i < sizeof(tests)/sizeof(struct binary); i++) {
 230                 int j = 0;
 231                 int l = strlen(tests[i].name);
 232                 memcpy(full_name, tests[i].name, l+1);
 233                 if (test_name && strncmp(test_name, full_name, l) != 0)
 234                         continue;
 235
 236                 while (tests[i].test_cases[j].name) {
 237                         memcpy(full_name + l, tests[i].test_cases[j].name, strlen(tests[i].test_cases[j].name) + 1);
 238                         if (!test_name || strncmp(full_name, test_name, sizeof(full_name)) == 0)
 239                                 print_description(full_name,tests[i].test_cases[j].description);
 240                         j++;
 241                 }
 242         }
 243 }
 244
 245
 246 static void stop_binary(const struct binary* b, pid_t pid, const char* test_name, int w_res)
 247 {
 248         int status = 0;
 249         int res = 0;
 250         if (w_res == 0)
 251                 res = waitpid(pid, &status, WNOHANG);
 252         else
 253                 res = waitpid(pid, &status, 0);
 254
 255         if (res == 0) {
 256                 //timeouted
 257                 kill(pid, SIGKILL);
 258                 res = waitpid(pid, &status, WNOHANG);
 259                 b->parse(b, test_name, buffer, RESULT_TIMEOUT, res);
 260         } else if (res < 0) {
 261                 //errno check
 262                 kill(pid, SIGKILL);
 263                 res = waitpid(pid, &status, WNOHANG);
 264                 b->parse(b, test_name, buffer, RESULT_ERROR, res);
 265         } else if (res > 0) {
 266                 if (WIFEXITED(status)) {
 267                         b->parse(b, test_name, buffer, RESULT_CODE, WEXITSTATUS(status));
 268                 } else if (WIFSIGNALED(status)) {
 269                         b->parse(b, test_name, buffer, RESULT_SIGNAL, WTERMSIG(status));
 270                 } else if (WIFSTOPPED(status)) {
 271                         b->parse(b, test_name, buffer, RESULT_SIGNAL, WSTOPSIG(status));
 272         } else if (WIFCONTINUED(status)) {
 273                         kill(pid, SIGKILL);
 274                         b->parse(b, test_name, buffer, RESULT_SIGNAL, -1);
 275                 }
 276         }
 277 }
 278
 279 static void parse_output_with_timeout(const struct binary* b, pid_t pid, const char* test_name)
 280 {
 281         struct timeval tv;
 282         fd_set rfds;
 283         int nfds;
 284         int res;
 285         int w_res = 0;
 286         tv.tv_sec = b->timeout/(1000*1000);
 287         tv.tv_usec = (b->timeout-tv.tv_sec*1000*1000);
 288         while (1) {
 289                 FD_ZERO(&rfds);
 290                 if (stdout_pipe[PIPE_READ] > -1) {
 291                         assert(stdout_pipe[PIPE_READ] > -1);
 292                         assert(stdout_pipe[PIPE_READ] < 1024);
 293                         FD_SET(stdout_pipe[PIPE_READ], &rfds);
 294                 }
 295                 if (stderr_pipe[PIPE_READ] > -1) {
 296                         assert(stderr_pipe[PIPE_READ] > -1);
 297                         assert(stderr_pipe[PIPE_READ] < 1024);
 298                         FD_SET(stderr_pipe[PIPE_READ], &rfds);
 299                 }
 300                 FD_SET(gravedigger_pipe[PIPE_READ], &rfds);
 301
 302                 nfds = select(FD_SETSIZE, &rfds, NULL, NULL, &tv);
 303                 if (nfds == -1) {
 304                         if (errno != EINTR) {
 305                                 w_res = 0;
 306                                 break;
 307                         }
 308                 } else if (nfds > 0) {
 309                         if (stdout_pipe[PIPE_READ] > -1 && FD_ISSET(stdout_pipe[PIPE_READ], &rfds)) {
 310                                 res = read(stdout_pipe[PIPE_READ], buffer, MAX_BUFFER-1);
 311                                 if (res == 0 || (res < 0 && errno != EINTR)) {
 312                                         close (stdout_pipe[PIPE_READ]);
 313                                         stdout_pipe[PIPE_READ] = -1;
 314                                         continue;
 315                                 } else if (res >=0) {
 316                                         b->parse(b, test_name, buffer, NEW_STDOUT, res);
 317                                 }
 318                         }
 319
 320                         if (stderr_pipe[PIPE_READ] > -1 && FD_ISSET(stderr_pipe[PIPE_READ], &rfds)) {
 321                                 res = read(stderr_pipe[PIPE_READ], buffer, MAX_BUFFER-1);
 322                                 if (res == 0 || (res < 0 && errno != EINTR)) {
 323                                         close (stderr_pipe[PIPE_READ]);
 324                                         stderr_pipe[PIPE_READ] = -1;
 325                                         continue;
 326                                 }
 327                                 b->parse(b, test_name, buffer, NEW_STDERR, res);
 328                         }
 329
 330                         if (FD_ISSET(gravedigger_pipe[PIPE_READ], &rfds)) {
 331                                 w_res = 1;
 332                                 break; //it has ended
 333                         }
 334                 } else {
 335                         //timeout
 336                         w_res = 0;
 337                         break;
 338                 }
 339         }
 340         stop_binary(b, pid, test_name, w_res);
 341 }
 342
 343 static int create_child(const char* path, char* const arguments[])
 344 {
 345         int child;
 346         int nResult;
 347         if (pipe(gravedigger_pipe) < 0) {
 348                 perror("allocating pipe for gravedigger failed");
 349                 goto error1;
 350         }
 351
 352         if (pipe(stdin_pipe) < 0) {
 353                 perror("allocating pipe for child input redirect failed");
 354                 goto error1;
 355         }
 356
 357         if (pipe(stdout_pipe) < 0) {
 358                 perror("allocating pipe for child output redirect failed");
 359                 goto error2;
 360         }
 361
 362         if (pipe(stderr_pipe) < 0) {
 363                 perror("allocating pipe for child output redirect failed");
 364                 goto error3;
 365         }
 366
 367         child = fork();
 368         if (!child) {
 369                 char ld_path[512];
 370                 sprintf(ld_path, TESTS_DIR ":");
 371                 // redirect stdin
 372                 if (dup2(stdin_pipe[PIPE_READ], STDIN_FILENO) == -1) {
 373                         perror("redirecting stdin failed");
 374                         return -1;
 375                 }
 376
 377                 if (dup2(stdout_pipe[PIPE_WRITE], STDOUT_FILENO) == -1) {
 378                         perror("redirecting stdout failed");
 379                         return -1;
 380                 }
 381
 382                 if (dup2(stderr_pipe[PIPE_WRITE], STDERR_FILENO) == -1) {
 383                         perror("redirecting stderr failed");
 384                         return -1;
 385                 }
 386
 387                 close(stdin_pipe[PIPE_READ]);
 388                 close(stdin_pipe[PIPE_WRITE]);
 389                 close(stdout_pipe[PIPE_READ]);
 390                 close(stdout_pipe[PIPE_WRITE]);
 391                 close(stderr_pipe[PIPE_READ]);
 392                 close(stderr_pipe[PIPE_WRITE]);
 393                 close(gravedigger_pipe[PIPE_READ]);
 394
 395                 char* ld_path_b = getenv("LD_LIBRARY_PATH");
 396                 if (ld_path_b != NULL)
 397                         memcpy(ld_path + strlen(ld_path), ld_path_b, strlen(ld_path_b)+1);
 398                 setenv("LD_LIBRARY_PATH", ld_path, 1);
 399                 setenv("G_TEST_SRCDIR", TESTS_DIR, 1);
 400
 401                 // run child process image
 402                 nResult = execv(path, arguments);
 403
 404                 // if we get here at all, an error occurred, but we are in the child
 405                 // process, so just exit
 406                 perror("exec of the child process  failed");
 407                 exit(nResult);
 408         } else if (child > 0) {
 409                 // close unused file descriptors, these are for child only
 410                 close(stdin_pipe[PIPE_READ]);
 411                 close(stdout_pipe[PIPE_WRITE]);
 412                 close(stderr_pipe[PIPE_WRITE]);
 413                 close(gravedigger_pipe[PIPE_WRITE]);
 414         } else {
 415                 // failed to create child
 416                 goto error4;
 417         }
 418
 419         return child;
 420
 421 error4:
 422         close(stderr_pipe[PIPE_READ]);
 423         close(stderr_pipe[PIPE_WRITE]);
 424 error3:
 425         close(stdout_pipe[PIPE_READ]);
 426         close(stdout_pipe[PIPE_WRITE]);
 427 error2:
 428         close(stdin_pipe[PIPE_READ]);
 429         close(stdin_pipe[PIPE_WRITE]);
 430 error1:
 431         return -1;
 432 }
 433
 434 static void run_test(const struct binary* b, const char* test_name)
 435 {
 436         int res = -1;
 437         char** arg;
 438         char test_id[MAX_COMMENT];
 439
 440         assert(b);
 441         assert(b->name);
 442         assert(b->path);
 443         assert(test_name);
 444
 445         arg = b->prepare_args(b, test_name);
 446
 447         if (b->init)
 448                 if (!b->init()) {
 449                         add_test_result(get_test_id(test_id, b, test_name), "ERROR", "Cannot init test", 0);
 450                         return;
 451                 }
 452
 453         res = create_child(b->path, arg);
 454         if (res > 0)
 455                 parse_output_with_timeout(b, res, test_name);
 456         else
 457                 add_test_result(get_test_id(test_id, b, test_name), "ERROR", "Cannot start test", 0);
 458
 459         if (b->clean)
 460                 b->clean();
 461 }
 462
 463 static void parse_run_test(const char* tc) {
 464         unsigned int i = 0;
 465         for (i = 0;i < sizeof(tests)/sizeof(struct binary); i++) {
 466                 int len = strlen(tests[i].name);
 467                 if (strncmp(tc, tests[i].name, len) == 0) {
 468                         if (tc[len] == '*' || tc[len] == '\0')
 469                                 run_test(&tests[i], "");
 470             else
 471                                 run_test(&tests[i], tc + len);
 472                 }
 473         }
 474 }
 475
 476 static int parse_option(int argc, char* argv[])
 477 {
 478         int ch = 0;
 479         int c = 0;
 480         while ((ch = getopt_long(argc, argv, "lr:d:", long_options, NULL)) != -1) {
 481                 switch (ch) {
 482                 case 'l':
 483                         print_list(NULL);
 484                         return 1;
 485                 case 'r':
 486                         if (c >= MAX_TC_NUM - 1) //NULL at the end
 487                                 return 0;
 488
 489                         if (optarg)
 490                                 requested_tc[c++] = optarg;
 491
 492                         break;
 493                 case 'd':
 494                         print_list(optarg);
 495             return 1;
 496                 }
 497         }
 498         return 0;
 499 }
 500
 501 void add_test_result(const char* test_id, const char* result, const char* comment, int res)
 502 {
 503         printf("%s;%s;%s\n", test_id, result, comment);
 504         fflush(stdout);
 505 }
 506
 507 static void prepare_results(void)
 508 {
 509 }
 510
 511 int main(int argc, char* argv[])
 512 {
 513         unsigned int i;
 514         signal(SIGPIPE, SIG_IGN);
 515         if (parse_option(argc, argv))
 516                 return 0;
 517
 518         prepare_results();
 519
 520         if (!requested_tc[0]) {
 521                 for (i = 0;i < sizeof(tests)/sizeof(struct binary); i++)
 522                     run_test(&tests[i], "");
 523         } else {
 524                 i = 0;
 525                 while(requested_tc[i]) {
 526                     parse_run_test(requested_tc[i]);
 527                         i++;
 528                 }
 529         }
 530
 531         return 0;
 532 }