src/coredump.c

   1 #define _GNU_SOURCE
   2 /*
   3  * Copyright 2007, Intel Corporation
   4  *
   5  * This file is part of corewatcher.org
   6  *
   7  * This program file is free software; you can redistribute it and/or modify it
   8  * under the terms of the GNU General Public License as published by the
   9  * Free Software Foundation; version 2 of the License.
  10  *
  11  * This program is distributed in the hope that it will be useful, but WITHOUT
  12  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  13  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
  14  * for more details.
  15  *
  16  * You should have received a copy of the GNU General Public License
  17  * along with this program in a file named COPYING; if not, write to the
  18  * Free Software Foundation, Inc.,
  19  * 51 Franklin Street, Fifth Floor,
  20  * Boston, MA 02110-1301 USA
  21  *
  22  * Authors:
  23  *      Arjan van de Ven <arjan@linux.intel.com>
  24  *      William Douglas <william.douglas@intel.com>
  25  *      Tim Pepper <timothy.c.pepper@linux.intel.com>
  26  */
  27
  28 #include <unistd.h>
  29 #include <stdlib.h>
  30 #include <stdio.h>
  31 #include <string.h>
  32 #include <assert.h>
  33 #include <fcntl.h>
  34 #include <asm/unistd.h>
  35 #include <sys/types.h>
  36 #include <sys/stat.h>
  37 #include <sys/time.h>
  38 #include <dirent.h>
  39 #include <glib.h>
  40 #include <errno.h>
  41
  42 #include "corewatcher.h"
  43
  44 /*
  45  * processing "queue" loop's condition variable and associated
  46  * lock.  Note the queue is an implicit data structure consisting
  47  * of the non-submitted core files in the filesystem, but the bool pq is
  48  * used to mark whether the "queue" holds something to prevent the possible
  49  * race where the condition is set before the thread is awaiting it and
  50  * thus is not woken.
  51  */
  52 static GMutex pq_mtx;
  53 static gboolean pq = FALSE;
  54 static GCond pq_work;
  55
  56 static char *get_release(void)
  57 {
  58         FILE *file = NULL;
  59         char *line = NULL;
  60         size_t dummy = 0;
  61
  62         file = fopen("/etc/os-release", "r");
  63         if (!file)
  64                 return NULL;
  65
  66         while (!feof(file)) {
  67                 if (getline(&line, &dummy, file) == -1)
  68                         break;
  69                 if (strstr(line, "VERSION_ID=")) {
  70                         char *c = NULL;
  71
  72                         c = strchr(line, '\n');
  73                         if (c) {
  74                                 *c = 0;
  75                                 c = strdup(&line[11]);
  76                                 fclose(file);
  77                                 free(line);
  78                                 return c;
  79                         }
  80                 }
  81         }
  82
  83         fclose(file);
  84         free(line);
  85
  86         return NULL;
  87 }
  88
  89 /*
  90  * Strip the directories from the path
  91  * given by fullname
  92  */
  93 char *strip_directories(char *fullpath)
  94 {
  95         char *dfile = NULL, *c1 = NULL, *c2 = NULL, *r = NULL;
  96         char delim[] = "/";
  97         char *saveptr;
  98
  99         if (!fullpath)
 100                 return NULL;
 101
 102         dfile = strdup(fullpath);
 103         if (!dfile)
 104                 return NULL;
 105
 106         c1 = strtok_r(dfile, delim, &saveptr);
 107         while(c1) {
 108                 c2 = c1;
 109                 c1 = strtok_r(NULL, delim, &saveptr);
 110         }
 111
 112         if (c2)
 113                 r = strdup(c2);
 114
 115         free(dfile);
 116
 117         return r;
 118 }
 119
 120 /*
 121  * Move corefile from core_folder to processed_folder subdir.
 122  * If this type of core has recently been seen, unlink this more recent
 123  * example in order to rate limit submissions of extremely crashy
 124  * applications.
 125  * Add extension and attempt to create directories if needed.
 126  */
 127 static int move_core(char *fullpath, char *extension)
 128 {
 129         char *corefilename = NULL, *newpath = NULL, *coreprefix = NULL;
 130         char *s = NULL;
 131         size_t prefix_len;
 132         DIR *dir = NULL;
 133         struct dirent *entry = NULL;
 134
 135         if (!fullpath)
 136                 return -1;
 137
 138         corefilename = strip_directories(fullpath);
 139         if (!corefilename)
 140                 return -ENOMEM;
 141
 142         /* if the corefile's name minus any suffixes (such as .$PID) and
 143          * minus two additional characters (ie: last two digits of
 144          * timestamp assuming core_%e_%t) matches another core file in the
 145          * processed_folder, simply unlink it instead of processing it for
 146          * submission.  TODO: consider a (configurable) time delta greater
 147          * than which the cores must be separated, stat'ing the files, etc.
 148          */
 149         coreprefix = strdup(corefilename);
 150         if (!coreprefix) {
 151                 free(corefilename);
 152                 return -ENOMEM;
 153         }
 154         s = strstr(coreprefix, ".");
 155         if (!s) {
 156                 free(coreprefix);
 157                 free(corefilename);
 158                 return -1;
 159         }
 160         *s = '\0';
 161         prefix_len = strlen(coreprefix);
 162         if (prefix_len > 2) {
 163                 s = strndup(coreprefix, prefix_len - 2);
 164                 free(coreprefix);
 165                 coreprefix = s;
 166         } else {
 167                 goto error;
 168         }
 169         dir = opendir(processed_folder);
 170         if (!dir)
 171                 goto error;
 172         while(1) {
 173                 entry = readdir(dir);
 174                 if (!entry || !entry->d_name)
 175                         break;
 176                 if (entry->d_name[0] == '.')
 177                         continue;
 178                 if (!strstr(entry->d_name, coreprefix))
 179                         continue;
 180                 fprintf(stderr, "+ ...ignoring/unlinking %s\n", fullpath);
 181                 unlink(fullpath);
 182                 closedir(dir);
 183                 goto error;
 184         }
 185         closedir(dir);
 186
 187         if (asprintf(&newpath, "%s%s.%s", processed_folder, corefilename, extension) == -1)
 188                 goto error;
 189
 190         free(coreprefix);
 191         free(corefilename);
 192         rename(fullpath, newpath);
 193         free(newpath);
 194         return 0;
 195
 196 error:
 197         free(coreprefix);
 198         free(corefilename);
 199         return -1;
 200 }
 201
 202
 203 /*
 204  * Use GDB to extract backtrace information from corefile
 205  */
 206 static struct oops *extract_core(char *fullpath, char *appfile, char *reportname)
 207 {
 208         struct oops *oops = NULL;
 209         int ret = 0;
 210         char *command = NULL, *h1 = NULL, *c1 = NULL, *c2 = NULL, *line = NULL;
 211         char *text = NULL, *coretime = NULL;
 212         char *m1 = NULL, *m2 = NULL;
 213         int bt_lines = 0, maps_lines = 0;
 214         FILE *file = NULL;
 215         char *badchar = NULL;
 216         char *release = get_release();
 217         int parsing_maps = 0;
 218         struct stat stat_buf;
 219         size_t size = 0;
 220         ssize_t bytesread = 0;
 221
 222         fprintf(stderr, "+ extract_core() called for %s\n", fullpath);
 223
 224         if (asprintf(&command, "LANG=C gdb --batch -f %s %s -x /etc/corewatcher/gdb.command 2> /dev/null", appfile, fullpath) == -1)
 225                 return NULL;
 226
 227         file = popen(command, "r");
 228         free(command);
 229         if (!file)
 230                 fprintf(stderr, "+ gdb failed for %s\n", fullpath);
 231
 232         if (stat(fullpath, &stat_buf) != -1) {
 233                 coretime = malloc(26);
 234                 if (coretime)
 235                         ctime_r(&stat_buf.st_mtime, coretime);
 236         }
 237
 238         ret = asprintf(&h1,
 239                        "cmdline: %s\n"
 240                        "release: %s\n"
 241                        "time: %s",
 242                        appfile,
 243                        release ? release : "Unknown",
 244                        coretime ? coretime : "Unknown");
 245         if (release)
 246                 free(release);
 247         if (coretime)
 248                 free(coretime);
 249         if (ret == -1)
 250                 return NULL;
 251
 252         while (file && !feof(file)) {
 253                 bytesread = getline(&line, &size, file);
 254                 if (!size)
 255                         break;
 256                 if (bytesread == -1)
 257                         break;
 258
 259                 /* try to figure out if we're onto the maps output yet */
 260                 if (strncmp(line, "From", 4) == 0) {
 261                         parsing_maps = 1;
 262                 }
 263                 /* maps might not be present */
 264                 if (strncmp(line, "No shared libraries", 19) == 0) {
 265                         break;
 266                 }
 267
 268                 if (!parsing_maps) { /* parsing backtrace */
 269                         c2 = c1;
 270
 271                         /* gdb's backtrace lines start with a line number */
 272                         if (line[0] != '#')
 273                                 continue;
 274
 275                         /* gdb prints some initial info which may include the
 276                          * "#0" line of the backtrace, then prints the
 277                          * backtrace in its entirety, leading to a
 278                          * duplicate "#0" in our summary if we do do: */
 279                         if ((bt_lines == 1) && (strncmp(line, "#0 ", 3) == 0))
 280                                 continue;
 281                         bt_lines++;
 282
 283                         /* gdb outputs some 0x1a's which break XML */
 284                         do {
 285                                 badchar = memchr(line, 0x1a, bytesread);
 286                                 if (badchar)
 287                                         *badchar = ' ';
 288                         } while (badchar);
 289
 290                         if (c1) {
 291                                 c1 = NULL;
 292                                 if (asprintf(&c1, "%s        %s", c2, line) == -1)
 293                                         continue;
 294                                 free(c2);
 295                         } else {
 296                                 /* keep going even if asprintf has errors */
 297                                 ret = asprintf(&c1, "        %s", line);
 298                         }
 299                 } else { /* parsing maps */
 300                         m2 = m1;
 301                         maps_lines++;
 302                         if (m1) {
 303                                 m1 = NULL;
 304                                 if (asprintf(&m1, "%s        %s", m2, line) == -1)
 305                                         continue;
 306                                 free(m2);
 307                         } else {
 308                                 /* keep going even if asprintf has errors */
 309                                 ret = asprintf(&m1, "        %s", line);
 310                         }
 311                 }
 312         }
 313         if (line)
 314                 free(line);
 315         if (file)
 316                 pclose(file);
 317
 318         ret = asprintf(&text,
 319                        "%s"
 320                        "backtrace: |\n"
 321                        "%s"
 322                        "maps: |\n"
 323                        "%s",
 324                        h1,
 325                        c1 ? c1 : "        Unknown",
 326                        m1 ? m1 : "        Unknown");
 327         free(h1);
 328         if (c1)
 329                 free(c1);
 330         if (m1)
 331                 free(m1);
 332
 333         if (ret == -1)
 334                 return NULL;
 335
 336         oops = malloc(sizeof(struct oops));
 337         if (!oops) {
 338                 free(text);
 339                 return NULL;
 340         }
 341         memset(oops, 0, sizeof(struct oops));
 342         oops->next = NULL;
 343         oops->application = strdup(appfile);
 344         oops->text = text;
 345         oops->filename = strdup(fullpath);
 346         oops->detail_filename = strdup(reportname);
 347         return oops;
 348 }
 349
 350 /*
 351  * input filename has the form: core_$APP_$TIMESTAMP[.$PID]
 352  * output filename has form of: $APP_$TIMESTAMP.txt
 353  */
 354 static char *make_report_filename(char *filename)
 355 {
 356         char *name = NULL, *dotpid = NULL, *stamp = NULL, *detail_filename = NULL;
 357
 358         if (!filename)
 359                 return NULL;
 360
 361         if (!(stamp = strstr(filename, "_")))
 362                 return NULL;
 363
 364         if (!(++stamp))
 365                 return NULL;
 366
 367         if (!(name = strdup(stamp)))
 368                 return NULL;
 369
 370         /* strip trailing .PID if present */
 371         dotpid = strstr(name, ".");
 372         if (dotpid)
 373                 *dotpid = '\0';
 374
 375         if ((asprintf(&detail_filename, "%s%s.txt", processed_folder, name)) == -1) {
 376                 free(name);
 377                 return NULL;
 378         }
 379         free(name);
 380
 381         return detail_filename;
 382 }
 383
 384 /*
 385  * Write the backtrace from the core file into a text
 386  * file named as $APP_$TIMESTAMP.txt
 387  */
 388 static void write_core_detail_file(struct oops *oops)
 389 {
 390         int fd = 0;
 391
 392         if (!oops->detail_filename)
 393                 return;
 394
 395         fd = open(oops->detail_filename, O_WRONLY | O_CREAT | O_TRUNC, 0);
 396         if (fd == -1) {
 397                 fprintf(stderr, "+ Error creating/opening %s for write\n", oops->detail_filename);
 398                 return;
 399         }
 400
 401         if(write(fd, oops->text, strlen(oops->text)) >= 0) {
 402                 fprintf(stderr, "+ Wrote %s\n", oops->detail_filename);
 403                 fchmod(fd, 0644);
 404         } else {
 405                 fprintf(stderr, "+ Error writing %s\n", oops->detail_filename);
 406                 unlink(oops->detail_filename);
 407         }
 408         close(fd);
 409 }
 410
 411 /*
 412  * Common function for processing core
 413  * files to generate oops structures and write *.txt
 414  * if not already present
 415  */
 416 static struct oops *process_common(char *fullpath)
 417 {
 418         struct oops *oops = NULL;
 419         char *appname = NULL, *appfile = NULL, *corefn = NULL, *reportname = NULL;
 420         struct stat stat_buf;
 421
 422         corefn = strip_directories(fullpath);
 423         if (!corefn) {
 424                 fprintf(stderr, "+ No corefile? (%s)\n", fullpath);
 425                 return NULL;
 426         }
 427
 428         appname = find_causingapp(fullpath);
 429         if (!appname) {
 430                 free(corefn);
 431                 free(reportname);
 432                 return NULL;
 433         }
 434         /*
 435          * don't process rpm, gdb or corewatcher crashes,
 436          * also skip apps which don't appear to be part of the OS
 437          */
 438         appfile = find_apppath(appname);
 439         if (!appfile ||
 440             !strncmp(appname, "rpm", 3) ||
 441             !strncmp(appname, "gdb", 3) ||
 442             !strncmp(appname, "corewatcher", 11)) {
 443                 free(corefn);
 444                 free(appname);
 445                 fprintf(stderr, "+ ...ignoring %s's %s\n", appname, fullpath);
 446                 move_core(fullpath, "skipped");
 447                 return NULL;
 448         }
 449         free(appname);
 450
 451         reportname = make_report_filename(corefn);
 452         if (!reportname) {
 453                 fprintf(stderr, "+ Couldn't make report name for %s\n", corefn);
 454                 free(corefn);
 455                 free(appfile);
 456                 return NULL;
 457         }
 458         free(corefn);
 459         if (stat(reportname, &stat_buf) == 0) {
 460                 int fd, ret;
 461                 /*
 462                  * TODO:
 463                  *   If the file already has trailing ".processed" and the txt file
 464                  *   is a low quality report, then create a new report.
 465                  */
 466                 fprintf(stderr, "+ Report already exists in %s\n", reportname);
 467
 468                 oops = malloc(sizeof(struct oops));
 469                 if (!oops) {
 470                         fprintf(stderr, "+ Malloc failed for struct oops\n");
 471                         free(reportname);
 472                         free(appfile);
 473                         return NULL;
 474                 }
 475                 memset(oops, 0, sizeof(struct oops));
 476
 477                 oops->next = NULL;
 478                 oops->application = strdup(appfile);
 479                 oops->filename = strdup(fullpath);
 480                 oops->detail_filename = strdup(reportname);
 481                 free(reportname);
 482                 free(appfile);
 483
 484                 oops->text = malloc(stat_buf.st_size + 1);
 485                 if (!oops->text) {
 486                         fprintf(stderr, "+ Malloc failed for oops text\n");
 487                         goto err;
 488                 }
 489                 fd = open(oops->detail_filename, O_RDONLY);
 490                 if (fd == -1) {
 491                         fprintf(stderr, "+ Open failed for oops text\n");
 492                         goto err;
 493                 }
 494                 ret = read(fd, oops->text, stat_buf.st_size);
 495                 close(fd);
 496                 if (ret != stat_buf.st_size) {
 497                         fprintf(stderr, "+ Read failed for oops text\n");
 498                         goto err;
 499                 }
 500                 oops->text[stat_buf.st_size] = '\0';
 501                 return oops;
 502         }
 503
 504         oops = extract_core(fullpath, appfile, reportname);
 505         write_core_detail_file(oops);
 506         free(reportname);
 507         free(appfile);
 508         return oops;
 509 err:
 510         FREE_OOPS(oops);
 511         return NULL;
 512 }
 513
 514
 515 /*
 516  * Creates $APP_$TIMESTAMP.txt report summaries if they don't exist and
 517  * adds the oops struct to the submit queue
 518  */
 519 static void *create_report(char *fullpath)
 520 {
 521         struct oops *oops = NULL;
 522         char *procfn = NULL;
 523         int new = 0, ret;
 524
 525         fprintf(stderr, "+ Entered create_report() for %s\n", fullpath);
 526
 527         /*
 528          * If the file has trailing ".to-process", create a new report.
 529          */
 530         if (strstr(fullpath, ".to-process"))
 531                 new = 1;
 532
 533         oops = process_common(fullpath);
 534         if (!oops) {
 535                 fprintf(stderr, "+ Did not generate struct oops for %s\n", fullpath);
 536                 return NULL;
 537         }
 538
 539         if (new) {
 540                 procfn = replace_name(fullpath, ".to-process", ".processed");
 541                 if (!procfn) {
 542                         fprintf(stderr, "+ Problems with filename manipulation for %s\n", fullpath);
 543                         goto clean_process_new;
 544                 }
 545                 ret = rename(fullpath, procfn);
 546                 if (ret) {
 547                         fprintf(stderr, "+ Unable to move %s to %s\n", fullpath, procfn);
 548                         free(procfn);
 549                         goto clean_process_new;
 550                 }
 551                 free(oops->filename);
 552                 oops->filename = strdup(procfn);
 553                 free(procfn);
 554         }
 555
 556         return oops;
 557
 558 clean_process_new:
 559         FREE_OOPS(oops);
 560         return NULL;
 561 }
 562
 563 /*
 564  * scan once for core files in core_folder, moving any to the
 565  * processed_folder with ".to-process" appended to their name
 566  */
 567 int scan_core_folder(void __unused *unused)
 568 {
 569         DIR *dir = NULL;
 570         struct dirent *entry = NULL;
 571         char *fullpath = NULL;
 572         int ret, work = 0;
 573
 574         dir = opendir(core_folder);
 575         if (!dir) {
 576                 fprintf(stderr, "+ Unable to open %s\n", core_folder);
 577                 return -1;
 578         }
 579         fprintf(stderr, "+ Begin scanning %s...\n", core_folder);
 580         while(1) {
 581                 free(fullpath);
 582                 fullpath = NULL;
 583
 584                 entry = readdir(dir);
 585                 if (!entry || !entry->d_name)
 586                         break;
 587                 if (entry->d_name[0] == '.')
 588                         continue;
 589                 if (strncmp(entry->d_name, "core_", 5))
 590                         continue;
 591
 592                 /* matched core_#### */
 593                 if (asprintf(&fullpath, "%s%s", core_folder, entry->d_name) == -1) {
 594                         fullpath = NULL;
 595                         continue;
 596                 }
 597
 598                 /* If one were to prompt the user before submitting, that
 599                  * might happen here.  */
 600
 601                 fprintf(stderr, "+ Looking at %s\n", fullpath);
 602
 603                 ret = move_core(fullpath, "to-process");
 604                 if (ret == 0)
 605                         work++;
 606         }
 607         closedir(dir);
 608
 609         if (work) {
 610                 fprintf(stderr, "+ Found %d files, setting pq_work condition\n", work);
 611                 g_mutex_lock(&pq_mtx);
 612                 g_cond_signal(&pq_work);
 613                 pq = TRUE;
 614                 g_mutex_unlock(&pq_mtx);
 615         }
 616
 617         fprintf(stderr, "+ End scanning %s...\n", core_folder);
 618         return TRUE;
 619 }
 620
 621 /*
 622  * scan for core_*.to-process and core_*.processed,
 623  * insure a summary *.txt report exists, then queue it
 624  */
 625 void *scan_processed_folder(void __unused *unused)
 626 {
 627         DIR *dir = NULL;
 628         struct dirent *entry = NULL;
 629         char *fullpath = NULL;
 630         struct oops *oops = NULL;
 631
 632         while(1) {
 633                 g_mutex_lock(&pq_mtx);
 634                 while (pq != TRUE) {
 635                         fprintf(stderr, "+ Awaiting work in %s...\n", processed_folder);
 636                         g_cond_wait(&pq_work, &pq_mtx);
 637                 }
 638                 pq = FALSE;
 639                 g_mutex_unlock(&pq_mtx);
 640
 641                 fprintf(stderr, "+ Begin scanning %s...\n", processed_folder);
 642
 643                 dir = opendir(processed_folder);
 644                 if (!dir) {
 645                         fprintf(stderr, "+ Unable to open %s\n", processed_folder);
 646                         continue;
 647                 }
 648                 while(1) {
 649                         entry = readdir(dir);
 650                         if (!entry || !entry->d_name)
 651                                 break;
 652                         if (entry->d_name[0] == '.')
 653                                 continue;
 654
 655                         /* files with trailing ".to-process" or "processed" represent new work */
 656                         if (!strstr(entry->d_name, "process"))
 657                                 continue;
 658
 659                         if (asprintf(&fullpath, "%s%s", processed_folder, entry->d_name) == -1) {
 660                                 fullpath = NULL;
 661                                 continue;
 662                         }
 663
 664                         fprintf(stderr, "+ Looking at %s\n", fullpath);
 665
 666                         oops = create_report(fullpath);
 667
 668                         if (oops) {
 669                                 fprintf(stderr, "+ Queued backtrace from %s\n", oops->detail_filename);
 670                                 queue_backtrace(oops);
 671                         }
 672
 673                         free(fullpath);
 674                         fullpath = NULL;
 675                 }
 676                 closedir(dir);
 677                 fprintf(stderr, "+ End scanning %s...\n", processed_folder);
 678         }
 679
 680         return NULL;
 681 }
 682
 683 /* do everything, called from timer event */
 684 int scan_folders(void __unused *unused)
 685 {
 686         scan_core_folder(NULL);
 687
 688         g_mutex_lock(&pq_mtx);
 689         g_cond_signal(&pq_work);
 690         pq = TRUE;
 691         g_mutex_unlock(&pq_mtx);
 692
 693         return TRUE;
 694 }