src/coredump.c

   1 #define _GNU_SOURCE
   2 /*
   3  * Copyright 2007, Intel Corporation
   4  *
   5  * This file is part of corewatcher.org
   6  *
   7  * This program file is free software; you can redistribute it and/or modify it
   8  * under the terms of the GNU General Public License as published by the
   9  * Free Software Foundation; version 2 of the License.
  10  *
  11  * This program is distributed in the hope that it will be useful, but WITHOUT
  12  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  13  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
  14  * for more details.
  15  *
  16  * You should have received a copy of the GNU General Public License
  17  * along with this program in a file named COPYING; if not, write to the
  18  * Free Software Foundation, Inc.,
  19  * 51 Franklin Street, Fifth Floor,
  20  * Boston, MA 02110-1301 USA
  21  *
  22  * Authors:
  23  *      Arjan van de Ven <arjan@linux.intel.com>
  24  *      William Douglas <william.douglas@intel.com>
  25  */
  26
  27 #include <unistd.h>
  28 #include <stdlib.h>
  29 #include <stdio.h>
  30 #include <string.h>
  31 #include <assert.h>
  32 #include <fcntl.h>
  33 #include <pthread.h>
  34 #include <asm/unistd.h>
  35 #include <sys/types.h>
  36 #include <sys/stat.h>
  37 #include <sys/time.h>
  38 #include <dirent.h>
  39 #include <signal.h>
  40 #include <glib.h>
  41 #include <errno.h>
  42
  43 #include "corewatcher.h"
  44
  45 int uid = 0;
  46 int sig = 0;
  47
  48 const char *core_folder = "/var/lib/corewatcher/";
  49 const char *processed_folder = "/var/lib/corewatcher/processed/";
  50
  51 /* Always pick up the processing_mtx and then the
  52    processing_queue_mtx, reverse for setting down */
  53 /* Always pick up the gdb_mtx and then the
  54    processing_queue_mtx, reverse for setting down */
  55 /* Always pick up the processing_mtx and then the
  56    gdb_mtx, reverse for setting down */
  57 /* so order for pick up should be:
  58    processing_mtx -> gdb_mtx -> processing_queue_mtx
  59    and the reverse for setting down */
  60 static pthread_mutex_t processing_queue_mtx = PTHREAD_MUTEX_INITIALIZER;
  61 static char *processing_queue[MAX_PROCESSING_OOPS];
  62 static pthread_mutex_t gdb_mtx = PTHREAD_MUTEX_INITIALIZER;
  63 static int tail = 0;
  64 static int head = 0;
  65
  66 static char *get_release(void)
  67 {
  68         FILE *file = NULL;
  69         char *line = NULL;
  70         size_t dummy = 0;
  71
  72         file = fopen("/etc/os-release", "r");
  73         if (!file) {
  74                 line = strdup("Unknown");
  75                 return line;
  76         }
  77
  78         while (!feof(file)) {
  79                 if (getline(&line, &dummy, file) == -1)
  80                         break;
  81                 if (strstr(line, "VERSION_ID=")) {
  82                         char *c = NULL;
  83
  84                         c = strchr(line, '\n');
  85                         if (c) {
  86                                 *c = 0;
  87                                 c = strdup(&line[11]);
  88                                 fclose(file);
  89                                 free(line);
  90                                 return c;
  91                         }
  92                 }
  93         }
  94
  95         fclose(file);
  96         free(line);
  97
  98         line = strdup("Unknown");
  99
 100         return line;
 101 }
 102
 103 static char *set_wrapped_app(char *line)
 104 {
 105         char *dline = NULL, *app = NULL, *appfile = NULL, *abs_path = NULL;
 106         char delim[] = " '";
 107         char app_folder[] = "/usr/share/";
 108         int r = 0;
 109
 110         if (!line)
 111                 return NULL;
 112
 113         dline = strdup(line);
 114
 115         app = strtok(dline, delim);
 116         while(app) {
 117                 if (strcmp(app, "--app") == 0) {
 118                         app = strtok(NULL, delim);
 119                         break;
 120                 }
 121                 app = strtok(NULL, delim);
 122         }
 123         if (!app)
 124                 goto cleanup_set_wrapped_app;
 125         r = asprintf(&abs_path, "%s%s", app_folder, app);
 126         if (r == -1) {
 127                 abs_path = NULL;
 128                 goto cleanup_set_wrapped_app;
 129         } else if (((unsigned int)r) != strlen(app_folder) + strlen(app)) {
 130                 goto cleanup_set_wrapped_app;
 131         }
 132
 133         appfile = find_executable(abs_path);
 134
 135 cleanup_set_wrapped_app:
 136         free(abs_path);
 137         free(dline);
 138         return appfile;
 139 }
 140
 141 /*
 142  * Scan core dump in case a wrapper was used
 143  * to run the process and get the actual binary name
 144  */
 145 static char *wrapper_scan(char *command)
 146 {
 147         char *line = NULL, *appfile = NULL;
 148         FILE *file = NULL;
 149
 150         file = popen(command, "r");
 151         if (!file)
 152                 return NULL;
 153
 154         while (!feof(file)) {
 155                 size_t size = 0;
 156                 int ret = 0;
 157                 free(line);
 158                 ret = getline(&line, &size, file);
 159                 if (!size)
 160                         break;
 161                 if (ret < 0)
 162                         break;
 163
 164                 if (strstr(line, "Core was generated by") &&
 165                     strstr(line, "--app")) {
 166                         /* attempt to update appfile */
 167                         appfile = set_wrapped_app(line);
 168                         break;
 169                 }
 170         }
 171         if (line)
 172                 free(line);
 173         pclose(file);
 174
 175         return appfile;
 176 }
 177
 178 /*
 179  * Strip the directories from the path
 180  * given by fullname
 181  */
 182 char *strip_directories(char *fullpath)
 183 {
 184         char *dfile = NULL, *c1 = NULL, *c2 = NULL, *r = NULL;
 185         char delim[] = "/";
 186
 187         if (!fullpath)
 188                 return NULL;
 189
 190         dfile = strdup(fullpath);
 191         if (!dfile)
 192                 return NULL;
 193
 194         c1 = strtok(dfile, delim);
 195         while(c1) {
 196                 c2 = c1;
 197                 c1 = strtok(NULL, delim);
 198         }
 199
 200         if (c2)
 201                 r = strdup(c2);
 202         free(dfile);
 203
 204         return r;
 205 }
 206
 207 /*
 208  * Move corefile from core_folder to processed_folder subdir.
 209  * Add extension if given and attempt to create directories if needed.
 210  */
 211 int move_core(char *fullpath, char *extension)
 212 {
 213         char *corefilename = NULL, newpath[8192];
 214
 215         if (!fullpath)
 216                 return -1;
 217
 218         if (!(corefilename = strip_directories(fullpath))) {
 219                 unlink(fullpath);
 220                 return -ENOMEM;
 221         }
 222
 223         if (extension)
 224                 snprintf(newpath, 8192, "%s%s.%s", processed_folder, corefilename, extension);
 225         else
 226                 snprintf(newpath, 8192, "%s%s", processed_folder, corefilename);
 227
 228         free(corefilename);
 229         rename(fullpath, newpath);
 230
 231         return 0;
 232 }
 233
 234 /*
 235  * Finds the full path for the application that crashed,
 236  * and depending on what opted_in was configured as will:
 237  * opted_in 2 (always submit) -> move file to processed_folder
 238  * to be processed further
 239  * opted_in 1 (ask user) -> ask user if we should submit
 240  * the crash and add to asked_oops hash so we don't get
 241  * called again for this corefile
 242  * opted_in 0 (don't submit) -> do nothing
 243  *
 244  * Picks up and sets down the asked_mtx.
 245  */
 246 static char *get_appfile(char *fullpath)
 247 {
 248         char *appname = NULL, *appfile = NULL;
 249
 250         if (!fullpath)
 251                 return NULL;
 252
 253         appname = find_coredump(fullpath);
 254         if (!appname)
 255                 return NULL;
 256
 257         /* don't try and do anything for rpm, gdb or corewatcher crashes */
 258         if (!(strcmp(appname, "rpm") && strcmp(appname, "gdb") && strcmp(appname, "corewatcher")))
 259                 return NULL;
 260
 261         appfile = find_executable(appname);
 262         /* appname no longer used, so free it as it was strdup'd */
 263         free(appname);
 264         if (!appfile)
 265                 return NULL;
 266
 267         move_core(fullpath, "to-process");
 268
 269         return appfile;
 270 }
 271
 272 /*
 273  * Use GDB to extract backtrace information from corefile
 274  */
 275 static struct oops *extract_core(char *fullpath, char *appfile)
 276 {
 277         struct oops *oops = NULL;
 278         int ret = 0;
 279         char *command = NULL, *h1 = NULL, *c1 = NULL, *c2 = NULL, *line = NULL, *text = NULL, *at = NULL;
 280         char *m1 = NULL, *m2 = NULL;
 281         FILE *file = NULL;
 282         char *badchar = NULL;
 283         char *release = get_release();
 284         int parsing_maps = 0;
 285
 286         if (asprintf(&command, "LANG=C gdb --batch -f %s %s -x /etc/corewatcher/gdb.command 2> /dev/null", appfile, fullpath) == -1)
 287                 return NULL;
 288
 289         if ((at = wrapper_scan(command))) {
 290                 free(appfile);
 291                 appfile = at;
 292         }
 293
 294         ret = asprintf(&h1,
 295                        "cmdline: %s\n"
 296                        "release: %s\n",
 297                        appfile,
 298                        release);
 299         free(release);
 300         if (ret == -1)
 301                 h1 = strdup("Unknown");
 302
 303         file = popen(command, "r");
 304
 305         while (file && !feof(file)) {
 306                 size_t size = 0;
 307
 308                 free(line);
 309                 line = NULL;
 310                 ret = getline(&line, &size, file);
 311                 if (!size)
 312                         break;
 313                 if (ret == -1)
 314                         break;
 315
 316                 if (strncmp(line, "From", 4) == 0) {
 317                         parsing_maps = 1;
 318                         /*continue;*/
 319                 }
 320
 321                 if (!parsing_maps) { /* parsing backtrace */
 322                         c2 = c1;
 323                         if (line[0] != '#')
 324                                 continue;
 325 fixup:                  /* gdb outputs some 0x1a's which break XML */
 326                         badchar = memchr(line, 0x1a, size);
 327                         if (badchar) {
 328                                 *badchar = ' ';
 329                                 goto fixup;
 330                         }
 331
 332                         if (c1) {
 333                                 c1 = NULL;
 334                                 if (asprintf(&c1, "%s        %s", c2, line) == -1)
 335                                         continue;
 336                                 free(c2);
 337                         } else {
 338                                 /* keep going even if asprintf has errors */
 339                                 ret = asprintf(&c1, "        %s", line);
 340                         }
 341                 } else { /* parsing maps */
 342                         m2 = m1;
 343                         if (m1) {
 344                                 m1 = NULL;
 345                                 if (asprintf(&m1, "%s        %s", m2, line) == -1)
 346                                         continue;
 347                                 free(m2);
 348                         } else {
 349                                 /* keep going even if asprintf has errors */
 350                                 ret = asprintf(&m1, "        %s", line);
 351                         }
 352                 }
 353         }
 354         if (line)
 355                 free(line);
 356         if (file)
 357                 pclose(file);
 358         free(command);
 359
 360         ret = asprintf(&text,
 361                        "%s"
 362                        "backtrace: |\n"
 363                        "%s"
 364                        "maps: |\n"
 365                        "%s",
 366                        h1, c1, m1);
 367         if (ret == -1)
 368                 text = NULL;
 369
 370         free(h1);
 371         free(c1);
 372
 373         oops = malloc(sizeof(struct oops));
 374         if (!oops) {
 375                 free(text);
 376                 return NULL;
 377         }
 378         memset(oops, 0, sizeof(struct oops));
 379         oops->application = strdup(appfile);
 380         oops->text = text;
 381         oops->filename = strdup(fullpath);
 382         return oops;
 383 }
 384
 385 /*
 386  * filename is of the form core_XXXX[.blah]
 387  * we need to get the pid out as we want
 388  * output of the form XXXX[.ext]
 389  */
 390 char *get_core_filename(char *filename, char *ext)
 391 {
 392         char *pid = NULL, *c = NULL, *s = NULL, *detail_filename = NULL;
 393
 394         if (!filename)
 395                 return NULL;
 396
 397         if (!(s = strstr(filename, "_")))
 398                 return NULL;
 399
 400         if (!(++s))
 401                 return NULL;
 402         /* causes valgrind whining because we copy from middle of a string */
 403         if (!(pid = strdup(s)))
 404                 return NULL;
 405
 406         c = strstr(pid, ".");
 407
 408         if (c)
 409                 *c = '\0';
 410
 411         if (ext) {
 412                 /* causes valgrind whining because we copy from middle of a string */
 413                 if ((asprintf(&detail_filename, "%s%s.%s", processed_folder, pid, ext)) == -1) {
 414                         free(pid);
 415                         return NULL;
 416                 }
 417         } else {
 418                 /* causes valgrind whining because we copy from middle of a string */
 419                 if ((asprintf(&detail_filename, "%s%s", processed_folder, pid)) == -1) {
 420                         free(pid);
 421                         return NULL;
 422                 }
 423         }
 424
 425         free(pid);
 426         return detail_filename;
 427 }
 428
 429 /*
 430  * Write the backtrace from the core file into a text
 431  * file named after the pid
 432  */
 433 static void write_core_detail_file(char *filename, char *text)
 434 {
 435         int fd = 0;
 436         char *detail_filename = NULL;
 437
 438         if (!filename || !text)
 439                 return;
 440
 441         if (!(detail_filename = get_core_filename(filename, "txt")))
 442                 return;
 443
 444         if ((fd = open(detail_filename, O_WRONLY | O_CREAT | O_TRUNC, 0)) != -1) {
 445                 if(write(fd, text, strlen(text)) >= 0)
 446                         fchmod(fd, 0644);
 447                 else
 448                         unlink(detail_filename);
 449                 close(fd);
 450         }
 451         free(detail_filename);
 452 }
 453
 454 /*
 455  * Removes corefile (core.XXXX) from the processing_queue.
 456  *
 457  * Expects the processing_queue_mtx to be held.
 458  */
 459 static void remove_from_processing_queue(void)
 460 {
 461         free(processing_queue[head]);
 462         processing_queue[head++] = NULL;
 463
 464         if (head == MAX_PROCESSING_OOPS)
 465                 head = 0;
 466 }
 467
 468 /*
 469  * Removes file from processing_oops hash based on pid.
 470  * Extracts pid from the fullpath such that
 471  * /home/user/core.pid will be tranformed into just the pid.
 472  *
 473  * Expects the lock on the given hash to be held.
 474  */
 475 void remove_pid_from_hash(char *fullpath, GHashTable *ht)
 476 {
 477         char *c1 = NULL, *c2 = NULL;
 478
 479         if (!(c1 = strip_directories(fullpath)))
 480                 return;
 481
 482         if (!(c2 = get_core_filename(c1, NULL))) {
 483                 free(c1);
 484                 return;
 485         }
 486
 487         free(c1);
 488
 489         g_hash_table_remove(ht, c2);
 490
 491         free(c2);
 492 }
 493
 494 /*
 495  * Common function for processing core
 496  * files to generate oops structures
 497  */
 498 static struct oops *process_common(char *fullpath)
 499 {
 500         struct oops *oops = NULL;
 501         char *appname = NULL, *appfile = NULL;
 502
 503         if(!(appname = find_coredump(fullpath))) {
 504                 return NULL;
 505         }
 506
 507         if (!(appfile = find_executable(appname))) {
 508                 free(appname);
 509                 return NULL;
 510         }
 511         free(appname);
 512
 513         if (!(oops = extract_core(fullpath, appfile))) {
 514                 free(appfile);
 515                 return NULL;
 516         }
 517         free(appfile);
 518
 519         return oops;
 520 }
 521
 522
 523 /*
 524  * Processes .to-process core files.
 525  * Also creates the pid.txt file and adds
 526  * the oops struct to the submit queue
 527  *
 528  * Picks up and sets down the gdb_mtx and
 529  * picks up and sets down the processing_queue_mtx.
 530  * (held at the same time in that order)
 531  * Also will pick up and sets down the queued_mtx.
 532  */
 533 static void *process_new(void __unused *vp)
 534 {
 535         struct oops *oops = NULL;
 536         char *procfn = NULL, *corefn = NULL, *fullpath = NULL;
 537
 538         pthread_mutex_lock(&core_status.processing_mtx);
 539         pthread_mutex_lock(&gdb_mtx);
 540         pthread_mutex_lock(&processing_queue_mtx);
 541
 542         if (!(fullpath = processing_queue[head])) {
 543                 /* something went quite wrong */
 544                 pthread_mutex_unlock(&processing_queue_mtx);
 545                 pthread_mutex_unlock(&gdb_mtx);
 546                 pthread_mutex_unlock(&core_status.processing_mtx);
 547                 return NULL;
 548         }
 549
 550         if (!(corefn = strip_directories(fullpath)))
 551                 goto clean_process_new;
 552
 553         if (!(procfn = replace_name(fullpath, ".to-process", ".processed")))
 554                 goto clean_process_new;
 555
 556         if (!(oops = process_common(fullpath)))
 557                 goto clean_process_new;
 558
 559         if (!(oops->detail_filename = get_core_filename(corefn, "txt")))
 560                 goto clean_process_new;
 561
 562         if (rename(fullpath, procfn))
 563                 goto clean_process_new;
 564
 565         free(oops->filename);
 566         oops->filename = procfn;
 567
 568         remove_from_processing_queue();
 569
 570         pthread_mutex_unlock(&processing_queue_mtx);
 571         pthread_mutex_unlock(&gdb_mtx);
 572         pthread_mutex_unlock(&core_status.processing_mtx);
 573
 574         write_core_detail_file(corefn, oops->text);
 575
 576         pthread_mutex_lock(&core_status.queued_mtx);
 577         queue_backtrace(oops);
 578         pthread_mutex_unlock(&core_status.queued_mtx);
 579
 580         /* don't need to free procfn because was set to oops->filename and that gets free'd */
 581         free(corefn);
 582         FREE_OOPS(oops);
 583         return NULL;
 584
 585 clean_process_new:
 586         remove_pid_from_hash(fullpath, core_status.processing_oops);
 587         remove_from_processing_queue();
 588         free(procfn);
 589         procfn = NULL; /* don't know if oops->filename == procfn so be safe */
 590         free(corefn);
 591         FREE_OOPS(oops);
 592         pthread_mutex_unlock(&processing_queue_mtx);
 593         pthread_mutex_unlock(&gdb_mtx);
 594         pthread_mutex_unlock(&core_status.processing_mtx);
 595         return NULL;
 596 }
 597
 598 /*
 599  * Reprocesses .processed core files.
 600  *
 601  * Picks up and sets down the gdb_mtx.
 602  * Picks up and sets down the processing_queue_mtx.
 603  * (held at the same time in that order)
 604  * Also will pick up and sets down the queued_mtx.
 605  */
 606 static void *process_old(void __unused *vp)
 607 {
 608         struct oops *oops = NULL;
 609         char *corefn = NULL, *fullpath = NULL;
 610
 611         pthread_mutex_lock(&gdb_mtx);
 612         pthread_mutex_lock(&processing_queue_mtx);
 613
 614         if (!(fullpath = processing_queue[head])) {
 615                 /* something went quite wrong */
 616                 pthread_mutex_unlock(&processing_queue_mtx);
 617                 pthread_mutex_unlock(&gdb_mtx);
 618                 return NULL;
 619         }
 620
 621         if (!(corefn = strip_directories(fullpath)))
 622                 goto clean_process_old;
 623
 624         if (!(oops = process_common(fullpath)))
 625                 goto clean_process_old;
 626
 627         if (!(oops->detail_filename = get_core_filename(corefn, "txt")))
 628                 goto clean_process_old;
 629
 630         remove_from_processing_queue();
 631
 632         pthread_mutex_unlock(&processing_queue_mtx);
 633         pthread_mutex_unlock(&gdb_mtx);
 634
 635         pthread_mutex_lock(&core_status.queued_mtx);
 636         queue_backtrace(oops);
 637         pthread_mutex_unlock(&core_status.queued_mtx);
 638
 639         free(corefn);
 640         FREE_OOPS(oops);
 641         return NULL;
 642
 643 clean_process_old:
 644         remove_pid_from_hash(fullpath, core_status.processing_oops);
 645         remove_from_processing_queue();
 646         free(corefn);
 647         FREE_OOPS(oops);
 648         pthread_mutex_unlock(&processing_queue_mtx);
 649         pthread_mutex_unlock(&gdb_mtx);
 650         return NULL;
 651 }
 652
 653 /*
 654  * Adds corefile (based on pid) to the processing_oops
 655  * hash table if it is not already there, then
 656  * tries to add to the processing_queue.
 657  *
 658  * Picks up and sets down the processing_mtx.
 659  * Picks up and sets down the processing_queue_mtx.
 660  */
 661 static int add_to_processing(char *fullpath)
 662 {
 663         char *c1 = NULL, *c2 = NULL, *fp = NULL;
 664
 665         if (!fullpath)
 666                 return -1;
 667
 668         if (!(fp = strdup(fullpath)))
 669                 goto clean_add_to_processing;
 670
 671         if (!(c1 = get_core_filename(fp, NULL)))
 672                 goto clean_add_to_processing;
 673
 674         if (!(c2 = strip_directories(c1)))
 675                 goto clean_add_to_processing;
 676
 677         free(c1);
 678         c1 = NULL;
 679
 680         pthread_mutex_lock(&core_status.processing_mtx);
 681         if (g_hash_table_lookup(core_status.processing_oops, c2)) {
 682                 pthread_mutex_unlock(&core_status.processing_mtx);
 683                 goto clean_add_to_processing;
 684         }
 685
 686         pthread_mutex_lock(&processing_queue_mtx);
 687         if (processing_queue[tail]) {
 688                 pthread_mutex_unlock(&processing_queue_mtx);
 689                 pthread_mutex_unlock(&core_status.processing_mtx);
 690                 goto clean_add_to_processing;
 691         }
 692
 693         g_hash_table_insert(core_status.processing_oops, c2, c2);
 694         processing_queue[tail++] = fp;
 695         if (tail == MAX_PROCESSING_OOPS)
 696                 tail = 0;
 697
 698         pthread_mutex_unlock(&processing_queue_mtx);
 699         pthread_mutex_unlock(&core_status.processing_mtx);
 700         return 0;
 701 clean_add_to_processing:
 702         free(fp);
 703         free(c1);
 704         free(c2);
 705         return -1;
 706 }
 707
 708 /*
 709  * Entry for processing new core files.
 710  */
 711 static void process_corefile(char *fullpath)
 712 {
 713         pthread_t thrd;
 714         int r = 1;
 715
 716         r = add_to_processing(fullpath);
 717
 718         if (r)
 719                 return;
 720
 721         if (pthread_create(&thrd, NULL, process_new, NULL))
 722                 fprintf(stderr, "Couldn't start up gdb extract core thread\n");
 723 }
 724
 725 /*
 726  * Entry for processing already seen core files.
 727  */
 728 static void reprocess_corefile(char *fullpath)
 729 {
 730         pthread_t thrd;
 731         int r = 0;
 732
 733         r = add_to_processing(fullpath);
 734
 735         if (r)
 736                 return;
 737
 738         if (pthread_create(&thrd, NULL, process_old, NULL))
 739                 fprintf(stderr, "Couldn't start up gdb extract core thread\n");
 740 }
 741
 742 int scan_corefolders(void __unused *unused)
 743 {
 744         DIR *dir = NULL;
 745         struct dirent *entry = NULL;
 746         char *fullpath = NULL, *appfile = NULL;
 747         int r = 0;
 748
 749         pthread_mutex_init(&core_status.processing_mtx, NULL);
 750         pthread_mutex_init(&core_status.queued_mtx, NULL);
 751         pthread_mutex_init(&core_status.asked_mtx, NULL);
 752
 753         /* scan for new crash data */
 754         dir = opendir(core_folder);
 755         if (!dir)
 756                 return 1;
 757         fprintf(stderr, "+ scanning %s...\n", core_folder);
 758         while(1) {
 759                 free(fullpath);
 760                 fullpath = NULL;
 761
 762                 entry = readdir(dir);
 763                 if (!entry || !entry->d_name)
 764                         break;
 765                 if (entry->d_name[0] == '.')
 766                         continue;
 767                 if (strncmp(entry->d_name, "core_", 5))
 768                         continue;
 769
 770                 /* matched core_#### where #### is the pid of the process */
 771                 r = asprintf(&fullpath, "%s%s", core_folder, entry->d_name);
 772                 if (r == -1) {
 773                         fullpath = NULL;
 774                         continue;
 775                 } else if (((unsigned int)r) != strlen(core_folder) + strlen(entry->d_name)) {
 776                         continue;
 777                 }
 778                 /* already found, waiting for response from user */
 779                 pthread_mutex_lock(&core_status.asked_mtx);
 780                 if (g_hash_table_lookup(core_status.asked_oops, fullpath)) {
 781                         pthread_mutex_unlock(&core_status.asked_mtx);
 782                         continue;
 783                 }
 784                 pthread_mutex_unlock(&core_status.asked_mtx);
 785                 fprintf(stderr, "+ Looking at %s\n", fullpath);
 786                 appfile = get_appfile(fullpath);
 787
 788                 if (!appfile) {
 789                         unlink(fullpath);
 790                 } else {
 791                         free(appfile);
 792                         appfile = NULL;
 793                 }
 794         }
 795         closedir(dir);
 796
 797         /* scan for partially processed crash data */
 798         dir = opendir(processed_folder);
 799         if (!dir)
 800                 return 1;
 801         fprintf(stderr, "+ scanning %s...\n", processed_folder);
 802         while(1) {
 803                 free(fullpath);
 804                 fullpath = NULL;
 805
 806                 entry = readdir(dir);
 807                 if (!entry || !entry->d_name)
 808                         break;
 809                 if (entry->d_name[0] == '.')
 810                         continue;
 811                 if (!strstr(entry->d_name, "process"))
 812                         continue;
 813
 814                 r = asprintf(&fullpath, "%s%s", processed_folder, entry->d_name);
 815                 if (r == -1) {
 816                         fullpath = NULL;
 817                         continue;
 818                 } else if (((unsigned int)r) != strlen(processed_folder) + strlen(entry->d_name)) {
 819                         continue;
 820                 }
 821
 822                 fprintf(stderr, "+ Looking at %s\n", fullpath);
 823                 if (strstr(fullpath, "to-process"))
 824                         process_corefile(fullpath);
 825                 else
 826                         reprocess_corefile(fullpath);
 827         }
 828         closedir(dir);
 829
 830         submit_queue();
 831
 832         return 1;
 833 }