3 * Copyright 2007, Intel Corporation
5 * This file is part of corewatcher.org
7 * This program file is free software; you can redistribute it and/or modify it
8 * under the terms of the GNU General Public License as published by the
9 * Free Software Foundation; version 2 of the License.
11 * This program is distributed in the hope that it will be useful, but WITHOUT
12 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
13 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
16 * You should have received a copy of the GNU General Public License
17 * along with this program in a file named COPYING; if not, write to the
18 * Free Software Foundation, Inc.,
19 * 51 Franklin Street, Fifth Floor,
20 * Boston, MA 02110-1301 USA
23 * Arjan van de Ven <arjan@linux.intel.com>
24 * William Douglas <william.douglas@intel.com>
33 #include <asm/unistd.h>
34 #include <sys/types.h>
42 #include "corewatcher.h"
47 const char *core_folder = "/var/lib/corewatcher/";
48 const char *processed_folder = "/var/lib/corewatcher/processed/";
51 * the application must initialize the GMutex's
52 * core_status.processing_mtx, core_status.queued_mtx,
53 * processing_queue_mtx and gdb_mtx
54 * before calling into this file's scan_corefolders()
55 * (also since that calls submit_queue() there are dependencies
56 * there which need taken care of too)
58 /* Always pick up the processing_mtx and then the
59 processing_queue_mtx, reverse for setting down */
60 /* Always pick up the gdb_mtx and then the
61 processing_queue_mtx, reverse for setting down */
62 /* Always pick up the processing_mtx and then the
63 gdb_mtx, reverse for setting down */
64 /* so order for pick up should be:
65 processing_mtx -> gdb_mtx -> processing_queue_mtx
66 and the reverse for setting down */
67 GMutex processing_queue_mtx;
68 static char *processing_queue[MAX_PROCESSING_OOPS];
69 static int pq_tail = 0;
70 static int pq_head = 0;
73 static char *get_release(void)
79 file = fopen("/etc/os-release", "r");
81 line = strdup("Unknown");
86 if (getline(&line, &dummy, file) == -1)
88 if (strstr(line, "VERSION_ID=")) {
91 c = strchr(line, '\n');
94 c = strdup(&line[11]);
105 line = strdup("Unknown");
110 static char *set_wrapped_app(char *line)
112 char *dline = NULL, *app = NULL, *appfile = NULL, *abs_path = NULL;
114 char app_folder[] = "/usr/share/";
120 dline = strdup(line);
122 app = strtok(dline, delim);
124 if (strcmp(app, "--app") == 0) {
125 app = strtok(NULL, delim);
128 app = strtok(NULL, delim);
131 goto cleanup_set_wrapped_app;
132 r = asprintf(&abs_path, "%s%s", app_folder, app);
135 goto cleanup_set_wrapped_app;
136 } else if (((unsigned int)r) != strlen(app_folder) + strlen(app)) {
137 goto cleanup_set_wrapped_app;
140 appfile = find_executable(abs_path);
142 cleanup_set_wrapped_app:
149 * Scan core dump in case a wrapper was used
150 * to run the process and get the actual binary name
152 static char *wrapper_scan(char *command)
154 char *line = NULL, *appfile = NULL;
157 file = popen(command, "r");
161 while (!feof(file)) {
165 ret = getline(&line, &size, file);
171 if (strstr(line, "Core was generated by") &&
172 strstr(line, "--app")) {
173 /* attempt to update appfile */
174 appfile = set_wrapped_app(line);
186 * Strip the directories from the path
189 char *strip_directories(char *fullpath)
191 char *dfile = NULL, *c1 = NULL, *c2 = NULL, *r = NULL;
197 dfile = strdup(fullpath);
201 c1 = strtok(dfile, delim);
204 c1 = strtok(NULL, delim);
215 * Move corefile from core_folder to processed_folder subdir.
216 * If this type of core has recently been seen, unlink this more recent
217 * example in order to rate limit submissions of extremely crashy
219 * Add extension if given and attempt to create directories if needed.
221 int move_core(char *fullpath, char *extension)
223 char *corefilename = NULL, newpath[8192], *coreprefix = NULL;
227 struct dirent *entry = NULL;
232 if (!(corefilename = strip_directories(fullpath)))
235 /* if the corefile's name minus any suffixes (such as .$PID) and
236 * minus two additional characters (ie: last two digits of
237 * timestamp assuming core_%e_%t) matches another core file in the
238 * processed_folder, simply unlink it instead of processing it for
239 * submission. TODO: consider a (configurable) time delta greater
240 * than which the cores must be separated, stat'ing the files, etc.
242 if (!(coreprefix = strdup(corefilename)))
244 if (!(s = strstr(coreprefix, ".")))
247 prefix_len = strlen(coreprefix);
248 if (prefix_len > 2) {
249 s = strndup(coreprefix, prefix_len - 2);
255 dir = opendir(processed_folder);
259 entry = readdir(dir);
260 if (!entry || !entry->d_name)
262 if (entry->d_name[0] == '.')
264 if (!strstr(entry->d_name, coreprefix))
266 fprintf(stderr, "+ ...ignoring/unlinking %s\n", fullpath);
272 snprintf(newpath, 8192, "%s%s.%s", processed_folder, corefilename, extension);
274 snprintf(newpath, 8192, "%s%s", processed_folder, corefilename);
278 rename(fullpath, newpath);
288 * Finds the full path for the application that crashed,
289 * and moves file to processed_folder for processing
291 static char *get_appfile(char *fullpath)
293 char *appname = NULL, *appfile = NULL;
298 appname = find_coredump(fullpath);
302 /* don't try and do anything for rpm, gdb or corewatcher crashes */
303 if (!(strcmp(appname, "rpm") && strcmp(appname, "gdb") && strcmp(appname, "corewatcher")))
306 appfile = find_executable(appname);
307 /* appname no longer used, so free it as it was strdup'd */
312 move_core(fullpath, "to-process");
318 * Use GDB to extract backtrace information from corefile
320 static struct oops *extract_core(char *fullpath, char *appfile)
322 struct oops *oops = NULL;
324 char *command = NULL, *h1 = NULL, *c1 = NULL, *c2 = NULL, *line = NULL;
325 char *text = NULL, *at = NULL, *coretime = NULL;
326 char *m1 = NULL, *m2 = NULL;
328 char *badchar = NULL;
329 char *release = get_release();
330 int parsing_maps = 0;
331 struct stat stat_buf;
333 if (asprintf(&command, "LANG=C gdb --batch -f %s %s -x /etc/corewatcher/gdb.command 2> /dev/null", appfile, fullpath) == -1)
336 if (stat(fullpath, &stat_buf) != -1) {
337 coretime = ctime(&stat_buf.st_mtime);
339 if (coretime == NULL) {
340 if (asprintf(&coretime, "Unknown\n") == -1)
344 if ((at = wrapper_scan(command))) {
358 h1 = strdup("Unknown");
360 file = popen(command, "r");
362 while (file && !feof(file)) {
367 ret = getline(&line, &size, file);
373 if (strncmp(line, "From", 4) == 0) {
378 if (!parsing_maps) { /* parsing backtrace */
382 fixup: /* gdb outputs some 0x1a's which break XML */
383 badchar = memchr(line, 0x1a, size);
391 if (asprintf(&c1, "%s %s", c2, line) == -1)
395 /* keep going even if asprintf has errors */
396 ret = asprintf(&c1, " %s", line);
398 } else { /* parsing maps */
402 if (asprintf(&m1, "%s %s", m2, line) == -1)
406 /* keep going even if asprintf has errors */
407 ret = asprintf(&m1, " %s", line);
417 ret = asprintf(&text,
430 oops = malloc(sizeof(struct oops));
435 memset(oops, 0, sizeof(struct oops));
436 oops->application = strdup(appfile);
438 oops->filename = strdup(fullpath);
443 * filename is of the form core_XXXX[.blah]
444 * we need to get the pid out as we want
445 * output of the form XXXX[.ext]
447 char *get_core_filename(char *filename, char *ext)
449 char *pid = NULL, *c = NULL, *s = NULL, *detail_filename = NULL;
454 if (!(s = strstr(filename, "_")))
459 /* causes valgrind whining because we copy from middle of a string */
460 if (!(pid = strdup(s)))
463 c = strstr(pid, ".");
469 /* causes valgrind whining because we copy from middle of a string */
470 if ((asprintf(&detail_filename, "%s%s.%s", processed_folder, pid, ext)) == -1) {
475 /* causes valgrind whining because we copy from middle of a string */
476 if ((asprintf(&detail_filename, "%s%s", processed_folder, pid)) == -1) {
483 return detail_filename;
487 * Write the backtrace from the core file into a text
488 * file named after the pid
490 static void write_core_detail_file(char *filename, char *text)
493 char *detail_filename = NULL;
495 if (!filename || !text)
498 if (!(detail_filename = get_core_filename(filename, "txt")))
501 if ((fd = open(detail_filename, O_WRONLY | O_CREAT | O_TRUNC, 0)) != -1) {
502 if(write(fd, text, strlen(text)) >= 0) {
505 fprintf(stderr, "+ ...ignoring/unlinking %s\n", detail_filename);
506 unlink(detail_filename);
510 free(detail_filename);
514 * Removes corefile (core.XXXX) from the processing_queue.
516 * Expects the processing_queue_mtx to be held.
518 static void remove_from_processing_queue(void)
520 free(processing_queue[pq_head]);
521 processing_queue[pq_head++] = NULL;
523 if (pq_head == MAX_PROCESSING_OOPS)
528 * Removes file from processing_oops hash based on pid.
529 * Extracts pid from the fullpath such that
530 * /home/user/core.pid will be tranformed into just the pid.
532 * Expects the lock on the given hash to be held.
534 void remove_pid_from_hash(char *fullpath, GHashTable *ht)
536 char *c1 = NULL, *c2 = NULL;
538 if (!(c1 = strip_directories(fullpath)))
541 if (!(c2 = get_core_filename(c1, NULL))) {
548 g_hash_table_remove(ht, c2);
554 * Common function for processing core
555 * files to generate oops structures
557 static struct oops *process_common(char *fullpath)
559 struct oops *oops = NULL;
560 char *appname = NULL, *appfile = NULL;
562 if(!(appname = find_coredump(fullpath))) {
566 if (!(appfile = find_executable(appname))) {
572 if (!(oops = extract_core(fullpath, appfile))) {
583 * Processes .to-process core files.
584 * Also creates the pid.txt file and adds
585 * the oops struct to the submit queue
587 * Picks up and sets down the gdb_mtx and
588 * picks up and sets down the processing_queue_mtx.
589 * (held at the same time in that order)
590 * Also will pick up and sets down the queued_mtx.
592 static void *process_new(void __unused *vp)
594 struct oops *oops = NULL;
595 char *procfn = NULL, *corefn = NULL, *fullpath = NULL;
597 g_mutex_lock(&core_status.processing_mtx);
598 g_mutex_lock(&gdb_mtx);
599 g_mutex_lock(&processing_queue_mtx);
601 if (!(fullpath = processing_queue[pq_head])) {
602 /* something went quite wrong */
603 g_mutex_unlock(&processing_queue_mtx);
604 g_mutex_unlock(&gdb_mtx);
605 g_mutex_unlock(&core_status.processing_mtx);
609 if (!(corefn = strip_directories(fullpath)))
610 goto clean_process_new;
612 if (!(procfn = replace_name(fullpath, ".to-process", ".processed")))
613 goto clean_process_new;
615 if (!(oops = process_common(fullpath)))
616 goto clean_process_new;
618 if (!(oops->detail_filename = get_core_filename(corefn, "txt")))
619 goto clean_process_new;
621 if (rename(fullpath, procfn))
622 goto clean_process_new;
624 free(oops->filename);
625 oops->filename = procfn;
627 remove_from_processing_queue();
629 g_mutex_unlock(&processing_queue_mtx);
630 g_mutex_unlock(&gdb_mtx);
631 g_mutex_unlock(&core_status.processing_mtx);
633 write_core_detail_file(corefn, oops->text);
635 g_mutex_lock(&core_status.queued_mtx);
636 queue_backtrace(oops);
637 g_mutex_unlock(&core_status.queued_mtx);
639 /* don't need to free procfn because was set to oops->filename and that gets free'd */
645 remove_pid_from_hash(fullpath, core_status.processing_oops);
646 remove_from_processing_queue();
648 procfn = NULL; /* don't know if oops->filename == procfn so be safe */
651 g_mutex_unlock(&processing_queue_mtx);
652 g_mutex_unlock(&gdb_mtx);
653 g_mutex_unlock(&core_status.processing_mtx);
658 * Reprocesses .processed core files.
660 * Picks up and sets down the gdb_mtx.
661 * Picks up and sets down the processing_queue_mtx.
662 * (held at the same time in that order)
663 * Also will pick up and sets down the queued_mtx.
665 static void *process_old(void __unused *vp)
667 struct oops *oops = NULL;
668 char *corefn = NULL, *fullpath = NULL;
670 g_mutex_lock(&gdb_mtx);
671 g_mutex_lock(&processing_queue_mtx);
673 if (!(fullpath = processing_queue[pq_head])) {
674 /* something went quite wrong */
675 g_mutex_unlock(&processing_queue_mtx);
676 g_mutex_unlock(&gdb_mtx);
680 if (!(corefn = strip_directories(fullpath)))
681 goto clean_process_old;
683 if (!(oops = process_common(fullpath)))
684 goto clean_process_old;
686 if (!(oops->detail_filename = get_core_filename(corefn, "txt")))
687 goto clean_process_old;
689 remove_from_processing_queue();
691 g_mutex_unlock(&processing_queue_mtx);
692 g_mutex_unlock(&gdb_mtx);
694 g_mutex_lock(&core_status.queued_mtx);
695 queue_backtrace(oops);
696 g_mutex_unlock(&core_status.queued_mtx);
703 remove_pid_from_hash(fullpath, core_status.processing_oops);
704 remove_from_processing_queue();
707 g_mutex_unlock(&processing_queue_mtx);
708 g_mutex_unlock(&gdb_mtx);
713 * Adds corefile (based on pid) to the processing_oops
714 * hash table if it is not already there, then
715 * tries to add to the processing_queue.
717 * Picks up and sets down the processing_mtx.
718 * Picks up and sets down the processing_queue_mtx.
720 static int add_to_processing(char *fullpath)
722 char *c1 = NULL, *c2 = NULL, *fp = NULL;
727 if (!(fp = strdup(fullpath)))
728 goto clean_add_to_processing;
730 if (!(c1 = get_core_filename(fp, NULL)))
731 goto clean_add_to_processing;
733 if (!(c2 = strip_directories(c1)))
734 goto clean_add_to_processing;
739 g_mutex_lock(&core_status.processing_mtx);
740 if (g_hash_table_lookup(core_status.processing_oops, c2)) {
741 g_mutex_unlock(&core_status.processing_mtx);
742 goto clean_add_to_processing;
745 g_mutex_lock(&processing_queue_mtx);
746 if (processing_queue[pq_tail]) {
747 g_mutex_unlock(&processing_queue_mtx);
748 g_mutex_unlock(&core_status.processing_mtx);
749 goto clean_add_to_processing;
752 g_hash_table_insert(core_status.processing_oops, c2, c2);
753 processing_queue[pq_tail++] = fp;
754 if (pq_tail == MAX_PROCESSING_OOPS)
757 g_mutex_unlock(&processing_queue_mtx);
758 g_mutex_unlock(&core_status.processing_mtx);
760 clean_add_to_processing:
768 * Entry for processing new core files.
770 static void process_corefile(char *fullpath)
772 GThread *thrd = NULL;
775 r = add_to_processing(fullpath);
780 thrd = g_thread_new("process_new", process_new, NULL);
782 fprintf(stderr, "Couldn't start thread for process_new()\n");
786 * Entry for processing already seen core files.
788 static void reprocess_corefile(char *fullpath)
790 GThread *thrd = NULL;
793 r = add_to_processing(fullpath);
798 thrd = g_thread_new("process_old", process_old, NULL);
800 fprintf(stderr, "Couldn't start thread for process_old()\n");
803 static void scan_core_folder(void __unused *unused)
805 /* scan for new crash data */
807 struct dirent *entry = NULL;
808 char *fullpath = NULL, *appfile = NULL;
811 dir = opendir(core_folder);
814 fprintf(stderr, "+ scanning %s...\n", core_folder);
819 entry = readdir(dir);
820 if (!entry || !entry->d_name)
822 if (entry->d_name[0] == '.')
824 if (strncmp(entry->d_name, "core_", 5))
827 /* matched core_#### where #### is the pid of the process */
828 r = asprintf(&fullpath, "%s%s", core_folder, entry->d_name);
832 } else if (((unsigned int)r) != strlen(core_folder) + strlen(entry->d_name)) {
836 /* If one were to prompt the user before submitting, that
837 * might happen here. */
839 fprintf(stderr, "+ Looking at %s\n", fullpath);
840 appfile = get_appfile(fullpath);
843 fprintf(stderr, "+ ...ignoring/unlinking %s\n", fullpath);
853 static void scan_processed_folder(void __unused *unused)
855 /* scan for partially processed crash data */
857 struct dirent *entry = NULL;
858 char *fullpath = NULL;
861 dir = opendir(processed_folder);
864 fprintf(stderr, "+ scanning %s...\n", processed_folder);
869 entry = readdir(dir);
870 if (!entry || !entry->d_name)
872 if (entry->d_name[0] == '.')
874 if (!strstr(entry->d_name, "process"))
877 r = asprintf(&fullpath, "%s%s", processed_folder, entry->d_name);
881 } else if (((unsigned int)r) != strlen(processed_folder) + strlen(entry->d_name)) {
885 fprintf(stderr, "+ Looking at %s\n", fullpath);
886 if (strstr(fullpath, "to-process"))
887 process_corefile(fullpath);
889 reprocess_corefile(fullpath);
894 int scan_corefolders(void __unused *unused)
896 scan_core_folder(NULL);
897 scan_processed_folder(NULL);