3 * Copyright 2007, Intel Corporation
5 * This file is part of corewatcher.org
7 * This program file is free software; you can redistribute it and/or modify it
8 * under the terms of the GNU General Public License as published by the
9 * Free Software Foundation; version 2 of the License.
11 * This program is distributed in the hope that it will be useful, but WITHOUT
12 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
13 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
16 * You should have received a copy of the GNU General Public License
17 * along with this program in a file named COPYING; if not, write to the
18 * Free Software Foundation, Inc.,
19 * 51 Franklin Street, Fifth Floor,
20 * Boston, MA 02110-1301 USA
23 * Arjan van de Ven <arjan@linux.intel.com>
24 * William Douglas <william.douglas@intel.com>
25 * Tim Pepper <timothy.c.pepper@linux.intel.com>
34 #include <asm/unistd.h>
35 #include <sys/types.h>
43 #include "corewatcher.h"
48 const char *core_folder = "/var/lib/corewatcher/";
49 const char *processed_folder = "/var/lib/corewatcher/processed/";
51 /* Always pick up the processing_mtx and then the
52 processing_queue_mtx, reverse for setting down */
53 /* Always pick up the gdb_mtx and then the
54 processing_queue_mtx, reverse for setting down */
55 /* Always pick up the processing_mtx and then the
56 gdb_mtx, reverse for setting down */
57 /* so order for pick up should be:
58 processing_mtx -> gdb_mtx -> processing_queue_mtx
59 and the reverse for setting down */
60 GMutex processing_queue_mtx;
61 static char *processing_queue[MAX_PROCESSING_OOPS];
62 static int pq_tail = 0;
63 static int pq_head = 0;
66 static char *get_release(void)
72 file = fopen("/etc/os-release", "r");
77 if (getline(&line, &dummy, file) == -1)
79 if (strstr(line, "VERSION_ID=")) {
82 c = strchr(line, '\n');
85 c = strdup(&line[11]);
100 * Strip the directories from the path
103 char *strip_directories(char *fullpath)
105 char *dfile = NULL, *c1 = NULL, *c2 = NULL, *r = NULL;
111 dfile = strdup(fullpath);
115 c1 = strtok(dfile, delim);
118 c1 = strtok(NULL, delim);
129 * Move corefile from core_folder to processed_folder subdir.
130 * If this type of core has recently been seen, unlink this more recent
131 * example in order to rate limit submissions of extremely crashy
133 * Add extension if given and attempt to create directories if needed.
135 int move_core(char *fullpath, char *extension)
137 char *corefilename = NULL, newpath[8192], *coreprefix = NULL;
141 struct dirent *entry = NULL;
146 if (!(corefilename = strip_directories(fullpath)))
149 /* if the corefile's name minus any suffixes (such as .$PID) and
150 * minus two additional characters (ie: last two digits of
151 * timestamp assuming core_%e_%t) matches another core file in the
152 * processed_folder, simply unlink it instead of processing it for
153 * submission. TODO: consider a (configurable) time delta greater
154 * than which the cores must be separated, stat'ing the files, etc.
156 if (!(coreprefix = strdup(corefilename)))
158 if (!(s = strstr(coreprefix, ".")))
161 prefix_len = strlen(coreprefix);
162 if (prefix_len > 2) {
163 s = strndup(coreprefix, prefix_len - 2);
169 dir = opendir(processed_folder);
173 entry = readdir(dir);
174 if (!entry || !entry->d_name)
176 if (entry->d_name[0] == '.')
178 if (!strstr(entry->d_name, coreprefix))
180 fprintf(stderr, "+ ...ignoring/unlinking %s\n", fullpath);
186 snprintf(newpath, 8192, "%s%s.%s", processed_folder, corefilename, extension);
188 snprintf(newpath, 8192, "%s%s", processed_folder, corefilename);
192 rename(fullpath, newpath);
202 * Finds the full path for the application that crashed,
203 * and moves file to processed_folder for processing
205 static char *get_appfile(char *fullpath)
207 char *appname = NULL, *appfile = NULL;
212 appname = find_coredump(fullpath);
216 /* don't try and do anything for rpm, gdb or corewatcher crashes */
217 if (!(strcmp(appname, "rpm") && strcmp(appname, "gdb") && strcmp(appname, "corewatcher")))
220 appfile = find_executable(appname);
221 /* appname no longer used, so free it as it was strdup'd */
226 move_core(fullpath, "to-process");
232 * Use GDB to extract backtrace information from corefile
234 static struct oops *extract_core(char *fullpath, char *appfile)
236 struct oops *oops = NULL;
238 char *command = NULL, *h1 = NULL, *c1 = NULL, *c2 = NULL, *line = NULL;
239 char *text = NULL, *coretime = NULL;
240 char *m1 = NULL, *m2 = NULL;
241 int bt_lines = 0, maps_lines = 0;
243 char *badchar = NULL;
244 char *release = get_release();
245 int parsing_maps = 0;
246 struct stat stat_buf;
248 ssize_t bytesread = 0;
250 fprintf(stderr, "+ extract_core() called for %s\n", fullpath);
252 if (asprintf(&command, "LANG=C gdb --batch -f %s %s -x /etc/corewatcher/gdb.command 2> /dev/null", appfile, fullpath) == -1)
255 file = popen(command, "r");
258 fprintf(stderr, "+ gdb failed for %s\n", fullpath);
260 if (stat(fullpath, &stat_buf) != -1) {
261 coretime = malloc(26);
263 ctime_r(&stat_buf.st_mtime, coretime);
271 release ? release : "Unknown",
272 coretime ? coretime : "Unknown");
280 while (file && !feof(file)) {
281 bytesread = getline(&line, &size, file);
287 /* try to figure out if we're onto the maps output yet */
288 if (strncmp(line, "From", 4) == 0) {
291 /* maps might not be present */
292 if (strncmp(line, "No shared libraries", 19) == 0) {
296 if (!parsing_maps) { /* parsing backtrace */
299 /* gdb's backtrace lines start with a line number */
303 /* gdb prints some initial info which may include the
304 * "#0" line of the backtrace, then prints the
305 * backtrace in its entirety, leading to a
306 * duplicate "#0" in our summary if we do do: */
307 if ((bt_lines == 1) && (strncmp(line, "#0 ", 3) == 0))
311 /* gdb outputs some 0x1a's which break XML */
313 badchar = memchr(line, 0x1a, bytesread);
320 if (asprintf(&c1, "%s %s", c2, line) == -1)
324 /* keep going even if asprintf has errors */
325 ret = asprintf(&c1, " %s", line);
327 } else { /* parsing maps */
332 if (asprintf(&m1, "%s %s", m2, line) == -1)
336 /* keep going even if asprintf has errors */
337 ret = asprintf(&m1, " %s", line);
346 ret = asprintf(&text,
353 c1 ? c1 : " Unknown",
354 m1 ? m1 : " Unknown");
364 oops = malloc(sizeof(struct oops));
369 memset(oops, 0, sizeof(struct oops));
370 oops->application = appfile;
372 oops->filename = strdup(fullpath);
377 * input filename has the form: core_$APP_$TIMESTAMP[.$PID]
378 * output filename has form of: $APP_$TIMESTAMP[.ext]
380 char *get_core_filename(char *filename, char *ext)
382 char *name = NULL, *dotpid = NULL, *stamp = NULL, *detail_filename = NULL;
387 if (!(stamp = strstr(filename, "_")))
393 if (!(name = strdup(stamp)))
396 /* strip trailing .PID if present */
397 dotpid = strstr(name, ".");
402 if ((asprintf(&detail_filename, "%s%s.%s", processed_folder, name, ext)) == -1) {
407 if ((asprintf(&detail_filename, "%s%s", processed_folder, name)) == -1) {
414 return detail_filename;
418 * Write the backtrace from the core file into a text
419 * file named as $APP_$TIMESTAMP.txt
421 static void write_core_detail_file(char *filename, char *text)
424 char *detail_filename = NULL;
426 if (!filename || !text)
429 if (!(detail_filename = get_core_filename(filename, "txt")))
432 if ((fd = open(detail_filename, O_WRONLY | O_CREAT | O_TRUNC, 0)) != -1) {
433 if(write(fd, text, strlen(text)) >= 0) {
436 fprintf(stderr, "+ ...ignoring/unlinking %s\n", detail_filename);
437 unlink(detail_filename);
441 free(detail_filename);
445 * Removes corefile (core.XXXX) from the processing_queue.
447 * Expects the processing_queue_mtx to be held.
449 static void remove_from_processing_queue(void)
451 fprintf(stderr, "+ removing processing_queue head\n");
452 free(processing_queue[pq_head]);
453 processing_queue[pq_head++] = NULL;
455 if (pq_head == MAX_PROCESSING_OOPS) {
456 fprintf(stderr, "+ wrapping processing_queue head to 0 (bugs here?)\n");
462 * Removes file from processing_oops hash based on core name,
463 * extracting that core name from a fullpath such as
464 * "/${processed_folder}/core_$APP_$TIMESTAMP.$PID"
465 * in order to get just "$APP_$TIMESTAMP"
467 * Expects the lock on the given hash to be held.
469 void remove_name_from_hash(char *fullpath, GHashTable *ht)
471 char *name = NULL, *corename = NULL, *shortname = NULL;
473 if (!(name = strip_directories(fullpath)))
476 if (!(corename = get_core_filename(name, NULL))) {
482 if (!(shortname = strip_directories(corename))) {
488 if (g_hash_table_remove(ht, shortname))
489 fprintf(stderr, "+ core %s removed from processing_oops hash table\n", shortname);
491 fprintf(stderr, "+ core %s not in processing_oops hash table\n", shortname);
497 * Common function for processing core
498 * files to generate oops structures
500 static struct oops *process_common(char *fullpath)
502 struct oops *oops = NULL;
503 char *appname = NULL, *appfile = NULL;
505 if(!(appname = find_coredump(fullpath))) {
509 if (!(appfile = find_executable(appname))) {
515 if (!(oops = extract_core(fullpath, appfile))) {
525 * Processes .to-process core files.
526 * Also creates the $APP_$TIMESTAMP.txt file and adds
527 * the oops struct to the submit queue
529 * Picks up and sets down the gdb_mtx and
530 * picks up and sets down the processing_queue_mtx.
531 * (held at the same time in that order)
533 static void *process_new(void __unused *vp)
535 struct oops *oops = NULL;
536 char *procfn = NULL, *corefn = NULL, *fullpath = NULL;
538 g_mutex_lock(&core_status.processing_mtx);
539 g_mutex_lock(&gdb_mtx);
540 g_mutex_lock(&processing_queue_mtx);
542 fprintf(stderr, "+ Entered process_new()\n");
544 if (!(fullpath = processing_queue[pq_head])) {
545 fprintf(stderr, "+ processing_queue corruption?\n");
546 g_mutex_unlock(&processing_queue_mtx);
547 g_mutex_unlock(&gdb_mtx);
548 g_mutex_unlock(&core_status.processing_mtx);
552 if (!(corefn = strip_directories(fullpath))) {
553 fprintf(stderr, "+ No corefile? (%s)\n", fullpath);
554 goto clean_process_new;
557 if (!(procfn = replace_name(fullpath, ".to-process", ".processed"))) {
558 fprintf(stderr, "+ Problems with filename manipulation for %s\n", corefn);
559 goto clean_process_new;
562 if (!(oops = process_common(fullpath))) {
563 fprintf(stderr, "+ Problems processing %s\n", procfn);
564 goto clean_process_new;
567 if (!(oops->detail_filename = get_core_filename(corefn, "txt"))) {
568 fprintf(stderr, "+ Problems with filename manipulation for %s\n", procfn);
569 goto clean_process_new;
572 if (rename(fullpath, procfn)) {
573 fprintf(stderr, "+ Unable to move %s to %s\n", fullpath, procfn);
574 goto clean_process_new;
577 free(oops->filename);
578 oops->filename = procfn;
580 remove_from_processing_queue();
582 g_mutex_unlock(&processing_queue_mtx);
583 g_mutex_unlock(&gdb_mtx);
584 g_mutex_unlock(&core_status.processing_mtx);
586 write_core_detail_file(corefn, oops->text);
588 queue_backtrace(oops);
590 fprintf(stderr, "+ Leaving process_new() with %s queued\n", oops->detail_filename);
592 /* mustn't free procfn because it was hung on oops->filename */
597 remove_name_from_hash(fullpath, core_status.processing_oops);
598 remove_from_processing_queue();
602 g_mutex_unlock(&processing_queue_mtx);
603 g_mutex_unlock(&gdb_mtx);
604 g_mutex_unlock(&core_status.processing_mtx);
609 * Reprocesses .processed core files.
611 * Picks up and sets down the gdb_mtx.
612 * Picks up and sets down the processing_queue_mtx.
613 * (held at the same time in that order)
615 static void *process_old(void __unused *vp)
617 struct oops *oops = NULL;
618 char *corefn = NULL, *fullpath = NULL;
620 g_mutex_lock(&gdb_mtx);
621 g_mutex_lock(&processing_queue_mtx);
623 fprintf(stderr, "+ Entered process_old()\n");
625 if (!(fullpath = processing_queue[pq_head])) {
626 fprintf(stderr, "+ processing_queue corruption?\n");
627 g_mutex_unlock(&processing_queue_mtx);
628 g_mutex_unlock(&gdb_mtx);
631 fprintf(stderr, "+ Reprocessing %s\n", fullpath);
633 if (!(corefn = strip_directories(fullpath))) {
634 fprintf(stderr, "+ No corefile? (%s)\n", fullpath);
635 goto clean_process_old;
638 if (!(oops = process_common(fullpath))) {
639 fprintf(stderr, "+ Problems processing %s\n", corefn);
640 goto clean_process_old;
643 if (!(oops->detail_filename = get_core_filename(corefn, "txt"))) {
644 fprintf(stderr, "+ Problems with filename manipulation for %s\n", corefn);
645 goto clean_process_old;
648 remove_from_processing_queue();
650 g_mutex_unlock(&processing_queue_mtx);
651 g_mutex_unlock(&gdb_mtx);
653 queue_backtrace(oops);
655 fprintf(stderr, "+ Leaving process_old() with %s queued\n", oops->detail_filename);
661 remove_name_from_hash(fullpath, core_status.processing_oops);
662 remove_from_processing_queue();
665 g_mutex_unlock(&processing_queue_mtx);
666 g_mutex_unlock(&gdb_mtx);
671 * Adds corefile (based on name) to the processing_oops
672 * hash table if it is not already there, then
673 * tries to add to the processing_queue.
675 * Picks up and sets down the processing_mtx.
676 * Picks up and sets down the processing_queue_mtx.
678 static int add_to_processing(char *fullpath)
680 char *c1 = NULL, *c2 = NULL, *fp = NULL;
685 if (!(fp = strdup(fullpath)))
686 goto clean_add_to_processing;
688 if (!(c1 = get_core_filename(fp, NULL)))
689 goto clean_add_to_processing;
691 if (!(c2 = strip_directories(c1)))
692 goto clean_add_to_processing;
697 g_mutex_lock(&core_status.processing_mtx);
698 if (g_hash_table_lookup(core_status.processing_oops, c2)) {
699 g_mutex_unlock(&core_status.processing_mtx);
700 fprintf(stderr, "+ ...name %s already in processing_oops hash table\n", c2);
701 goto clean_add_to_processing;
704 g_mutex_lock(&processing_queue_mtx);
705 if (processing_queue[pq_tail]) {
706 g_mutex_unlock(&processing_queue_mtx);
707 g_mutex_unlock(&core_status.processing_mtx);
708 fprintf(stderr, "+ ...processing_queue full\n");
709 goto clean_add_to_processing;
712 g_hash_table_insert(core_status.processing_oops, c2, c2);
713 processing_queue[pq_tail++] = fp;
714 if (pq_tail == MAX_PROCESSING_OOPS)
717 g_mutex_unlock(&processing_queue_mtx);
718 g_mutex_unlock(&core_status.processing_mtx);
720 clean_add_to_processing:
728 * Entry for processing new core files.
730 static void process_corefile(char *fullpath)
732 GThread *thrd = NULL;
735 r = add_to_processing(fullpath);
740 thrd = g_thread_new("process_new", process_new, NULL);
742 fprintf(stderr, "Couldn't start thread for process_new()\n");
746 * Entry for processing already seen core files.
748 static void reprocess_corefile(char *fullpath)
750 GThread *thrd = NULL;
753 r = add_to_processing(fullpath);
758 thrd = g_thread_new("process_old", process_old, NULL);
760 fprintf(stderr, "Couldn't start thread for process_old()\n");
763 static void scan_core_folder(void __unused *unused)
765 /* scan for new crash data */
767 struct dirent *entry = NULL;
768 char *fullpath = NULL, *appfile = NULL;
771 dir = opendir(core_folder);
774 fprintf(stderr, "+ Begin scanning %s...\n", core_folder);
779 entry = readdir(dir);
780 if (!entry || !entry->d_name)
782 if (entry->d_name[0] == '.')
784 if (strncmp(entry->d_name, "core_", 5))
787 /* matched core_#### */
788 r = asprintf(&fullpath, "%s%s", core_folder, entry->d_name);
792 } else if (((unsigned int)r) != strlen(core_folder) + strlen(entry->d_name)) {
796 /* If one were to prompt the user before submitting, that
797 * might happen here. */
799 fprintf(stderr, "+ Looking at %s\n", fullpath);
800 appfile = get_appfile(fullpath);
803 fprintf(stderr, "+ ...ignoring/unlinking %s\n", fullpath);
811 fprintf(stderr, "+ End scanning %s...\n", core_folder);
814 static void scan_processed_folder(void __unused *unused)
816 /* scan for partially processed crash data */
818 struct dirent *entry = NULL;
819 char *fullpath = NULL;
822 dir = opendir(processed_folder);
825 fprintf(stderr, "+ Begin scanning %s...\n", processed_folder);
830 entry = readdir(dir);
831 if (!entry || !entry->d_name)
833 if (entry->d_name[0] == '.')
835 if (!strstr(entry->d_name, "process"))
838 r = asprintf(&fullpath, "%s%s", processed_folder, entry->d_name);
842 } else if (((unsigned int)r) != strlen(processed_folder) + strlen(entry->d_name)) {
846 fprintf(stderr, "+ Looking at %s\n", fullpath);
847 if (strstr(fullpath, "to-process"))
848 process_corefile(fullpath);
850 reprocess_corefile(fullpath);
853 fprintf(stderr, "+ End scanning %s...\n", processed_folder);
856 int scan_corefolders(void __unused *unused)
858 scan_core_folder(NULL);
859 scan_processed_folder(NULL);