3 * Copyright 2007, Intel Corporation
5 * This file is part of corewatcher.org
7 * This program file is free software; you can redistribute it and/or modify it
8 * under the terms of the GNU General Public License as published by the
9 * Free Software Foundation; version 2 of the License.
11 * This program is distributed in the hope that it will be useful, but WITHOUT
12 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
13 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
16 * You should have received a copy of the GNU General Public License
17 * along with this program in a file named COPYING; if not, write to the
18 * Free Software Foundation, Inc.,
19 * 51 Franklin Street, Fifth Floor,
20 * Boston, MA 02110-1301 USA
23 * Arjan van de Ven <arjan@linux.intel.com>
24 * William Douglas <william.douglas@intel.com>
34 #include <asm/unistd.h>
35 #include <sys/types.h>
43 #include "corewatcher.h"
48 const char *core_folder = "/var/lib/corewatcher/";
49 const char *processed_folder = "/var/lib/corewatcher/processed/";
51 /* Always pick up the processing_mtx and then the
52 processing_queue_mtx, reverse for setting down */
53 /* Always pick up the gdb_mtx and then the
54 processing_queue_mtx, reverse for setting down */
55 /* Always pick up the processing_mtx and then the
56 gdb_mtx, reverse for setting down */
57 /* so order for pick up should be:
58 processing_mtx -> gdb_mtx -> processing_queue_mtx
59 and the reverse for setting down */
60 static pthread_mutex_t processing_queue_mtx = PTHREAD_MUTEX_INITIALIZER;
61 static char *processing_queue[MAX_PROCESSING_OOPS];
62 static pthread_mutex_t gdb_mtx = PTHREAD_MUTEX_INITIALIZER;
66 static char *get_release(void)
72 file = fopen("/etc/os-release", "r");
74 line = strdup("Unknown");
79 if (getline(&line, &dummy, file) == -1)
81 if (strstr(line, "VERSION_ID=")) {
84 c = strchr(line, '\n');
87 c = strdup(&line[11]);
98 line = strdup("Unknown");
103 static char *set_wrapped_app(char *line)
105 char *dline = NULL, *app = NULL, *appfile = NULL, *abs_path = NULL;
107 char app_folder[] = "/usr/share/";
113 dline = strdup(line);
115 app = strtok(dline, delim);
117 if (strcmp(app, "--app") == 0) {
118 app = strtok(NULL, delim);
121 app = strtok(NULL, delim);
124 goto cleanup_set_wrapped_app;
125 r = asprintf(&abs_path, "%s%s", app_folder, app);
128 goto cleanup_set_wrapped_app;
129 } else if (((unsigned int)r) != strlen(app_folder) + strlen(app)) {
130 goto cleanup_set_wrapped_app;
133 appfile = find_executable(abs_path);
135 cleanup_set_wrapped_app:
142 * Scan core dump in case a wrapper was used
143 * to run the process and get the actual binary name
145 static char *wrapper_scan(char *command)
147 char *line = NULL, *appfile = NULL;
150 file = popen(command, "r");
154 while (!feof(file)) {
158 ret = getline(&line, &size, file);
164 if (strstr(line, "Core was generated by") &&
165 strstr(line, "--app")) {
166 /* attempt to update appfile */
167 appfile = set_wrapped_app(line);
179 * Strip the directories from the path
182 char *strip_directories(char *fullpath)
184 char *dfile = NULL, *c1 = NULL, *c2 = NULL, *r = NULL;
190 dfile = strdup(fullpath);
194 c1 = strtok(dfile, delim);
197 c1 = strtok(NULL, delim);
208 * Move corefile from core_folder to processed_folder subdir.
209 * If this type of core has recently been seen, unlink this more recent
210 * example in order to rate limit submissions of extremely crashy
212 * Add extension if given and attempt to create directories if needed.
214 int move_core(char *fullpath, char *extension)
216 char *corefilename = NULL, newpath[8192], *coreprefix = NULL;
220 struct dirent *entry = NULL;
225 if (!(corefilename = strip_directories(fullpath)))
228 /* if the corefile's name minus any suffixes (such as .$PID) and
229 * minus two additional characters (ie: last two digits of
230 * timestamp assuming core_%e_%t) matches another core file in the
231 * processed_folder, simply unlink it instead of processing it for
232 * submission. TODO: consider a (configurable) time delta greater
233 * than which the cores must be separated, stat'ing the files, etc.
235 if (!(coreprefix = strdup(corefilename)))
237 if (!(s = strstr(coreprefix, ".")))
240 prefix_len = strlen(coreprefix);
241 if (prefix_len > 2) {
242 s = strndup(coreprefix, prefix_len - 2);
248 dir = opendir(processed_folder);
252 entry = readdir(dir);
253 if (!entry || !entry->d_name)
255 if (entry->d_name[0] == '.')
257 if (!strstr(entry->d_name, coreprefix))
259 fprintf(stderr, "+ ...ignoring/unlinking %s\n", fullpath);
265 snprintf(newpath, 8192, "%s%s.%s", processed_folder, corefilename, extension);
267 snprintf(newpath, 8192, "%s%s", processed_folder, corefilename);
271 rename(fullpath, newpath);
281 * Finds the full path for the application that crashed,
282 * and depending on what opted_in was configured as will:
283 * opted_in 2 (always submit) -> move file to processed_folder
284 * to be processed further
285 * opted_in 1 (ask user) -> ask user if we should submit
286 * the crash and add to asked_oops hash so we don't get
287 * called again for this corefile
288 * opted_in 0 (don't submit) -> do nothing
290 * Picks up and sets down the asked_mtx.
292 static char *get_appfile(char *fullpath)
294 char *appname = NULL, *appfile = NULL;
299 appname = find_coredump(fullpath);
303 /* don't try and do anything for rpm, gdb or corewatcher crashes */
304 if (!(strcmp(appname, "rpm") && strcmp(appname, "gdb") && strcmp(appname, "corewatcher")))
307 appfile = find_executable(appname);
308 /* appname no longer used, so free it as it was strdup'd */
313 move_core(fullpath, "to-process");
319 * Use GDB to extract backtrace information from corefile
321 static struct oops *extract_core(char *fullpath, char *appfile)
323 struct oops *oops = NULL;
325 char *command = NULL, *h1 = NULL, *c1 = NULL, *c2 = NULL, *line = NULL, *text = NULL, *at = NULL;
326 char *m1 = NULL, *m2 = NULL;
328 char *badchar = NULL;
329 char *release = get_release();
330 int parsing_maps = 0;
332 if (asprintf(&command, "LANG=C gdb --batch -f %s %s -x /etc/corewatcher/gdb.command 2> /dev/null", appfile, fullpath) == -1)
335 if ((at = wrapper_scan(command))) {
347 h1 = strdup("Unknown");
349 file = popen(command, "r");
351 while (file && !feof(file)) {
356 ret = getline(&line, &size, file);
362 if (strncmp(line, "From", 4) == 0) {
367 if (!parsing_maps) { /* parsing backtrace */
371 fixup: /* gdb outputs some 0x1a's which break XML */
372 badchar = memchr(line, 0x1a, size);
380 if (asprintf(&c1, "%s %s", c2, line) == -1)
384 /* keep going even if asprintf has errors */
385 ret = asprintf(&c1, " %s", line);
387 } else { /* parsing maps */
391 if (asprintf(&m1, "%s %s", m2, line) == -1)
395 /* keep going even if asprintf has errors */
396 ret = asprintf(&m1, " %s", line);
406 ret = asprintf(&text,
419 oops = malloc(sizeof(struct oops));
424 memset(oops, 0, sizeof(struct oops));
425 oops->application = strdup(appfile);
427 oops->filename = strdup(fullpath);
432 * filename is of the form core_XXXX[.blah]
433 * we need to get the pid out as we want
434 * output of the form XXXX[.ext]
436 char *get_core_filename(char *filename, char *ext)
438 char *pid = NULL, *c = NULL, *s = NULL, *detail_filename = NULL;
443 if (!(s = strstr(filename, "_")))
448 /* causes valgrind whining because we copy from middle of a string */
449 if (!(pid = strdup(s)))
452 c = strstr(pid, ".");
458 /* causes valgrind whining because we copy from middle of a string */
459 if ((asprintf(&detail_filename, "%s%s.%s", processed_folder, pid, ext)) == -1) {
464 /* causes valgrind whining because we copy from middle of a string */
465 if ((asprintf(&detail_filename, "%s%s", processed_folder, pid)) == -1) {
472 return detail_filename;
476 * Write the backtrace from the core file into a text
477 * file named after the pid
479 static void write_core_detail_file(char *filename, char *text)
482 char *detail_filename = NULL;
484 if (!filename || !text)
487 if (!(detail_filename = get_core_filename(filename, "txt")))
490 if ((fd = open(detail_filename, O_WRONLY | O_CREAT | O_TRUNC, 0)) != -1) {
491 if(write(fd, text, strlen(text)) >= 0) {
494 fprintf(stderr, "+ ...ignoring/unlinking %s\n", detail_filename);
495 unlink(detail_filename);
499 free(detail_filename);
503 * Removes corefile (core.XXXX) from the processing_queue.
505 * Expects the processing_queue_mtx to be held.
507 static void remove_from_processing_queue(void)
509 free(processing_queue[head]);
510 processing_queue[head++] = NULL;
512 if (head == MAX_PROCESSING_OOPS)
517 * Removes file from processing_oops hash based on pid.
518 * Extracts pid from the fullpath such that
519 * /home/user/core.pid will be tranformed into just the pid.
521 * Expects the lock on the given hash to be held.
523 void remove_pid_from_hash(char *fullpath, GHashTable *ht)
525 char *c1 = NULL, *c2 = NULL;
527 if (!(c1 = strip_directories(fullpath)))
530 if (!(c2 = get_core_filename(c1, NULL))) {
537 g_hash_table_remove(ht, c2);
543 * Common function for processing core
544 * files to generate oops structures
546 static struct oops *process_common(char *fullpath)
548 struct oops *oops = NULL;
549 char *appname = NULL, *appfile = NULL;
551 if(!(appname = find_coredump(fullpath))) {
555 if (!(appfile = find_executable(appname))) {
561 if (!(oops = extract_core(fullpath, appfile))) {
572 * Processes .to-process core files.
573 * Also creates the pid.txt file and adds
574 * the oops struct to the submit queue
576 * Picks up and sets down the gdb_mtx and
577 * picks up and sets down the processing_queue_mtx.
578 * (held at the same time in that order)
579 * Also will pick up and sets down the queued_mtx.
581 static void *process_new(void __unused *vp)
583 struct oops *oops = NULL;
584 char *procfn = NULL, *corefn = NULL, *fullpath = NULL;
586 pthread_mutex_lock(&core_status.processing_mtx);
587 pthread_mutex_lock(&gdb_mtx);
588 pthread_mutex_lock(&processing_queue_mtx);
590 if (!(fullpath = processing_queue[head])) {
591 /* something went quite wrong */
592 pthread_mutex_unlock(&processing_queue_mtx);
593 pthread_mutex_unlock(&gdb_mtx);
594 pthread_mutex_unlock(&core_status.processing_mtx);
598 if (!(corefn = strip_directories(fullpath)))
599 goto clean_process_new;
601 if (!(procfn = replace_name(fullpath, ".to-process", ".processed")))
602 goto clean_process_new;
604 if (!(oops = process_common(fullpath)))
605 goto clean_process_new;
607 if (!(oops->detail_filename = get_core_filename(corefn, "txt")))
608 goto clean_process_new;
610 if (rename(fullpath, procfn))
611 goto clean_process_new;
613 free(oops->filename);
614 oops->filename = procfn;
616 remove_from_processing_queue();
618 pthread_mutex_unlock(&processing_queue_mtx);
619 pthread_mutex_unlock(&gdb_mtx);
620 pthread_mutex_unlock(&core_status.processing_mtx);
622 write_core_detail_file(corefn, oops->text);
624 pthread_mutex_lock(&core_status.queued_mtx);
625 queue_backtrace(oops);
626 pthread_mutex_unlock(&core_status.queued_mtx);
628 /* don't need to free procfn because was set to oops->filename and that gets free'd */
634 remove_pid_from_hash(fullpath, core_status.processing_oops);
635 remove_from_processing_queue();
637 procfn = NULL; /* don't know if oops->filename == procfn so be safe */
640 pthread_mutex_unlock(&processing_queue_mtx);
641 pthread_mutex_unlock(&gdb_mtx);
642 pthread_mutex_unlock(&core_status.processing_mtx);
647 * Reprocesses .processed core files.
649 * Picks up and sets down the gdb_mtx.
650 * Picks up and sets down the processing_queue_mtx.
651 * (held at the same time in that order)
652 * Also will pick up and sets down the queued_mtx.
654 static void *process_old(void __unused *vp)
656 struct oops *oops = NULL;
657 char *corefn = NULL, *fullpath = NULL;
659 pthread_mutex_lock(&gdb_mtx);
660 pthread_mutex_lock(&processing_queue_mtx);
662 if (!(fullpath = processing_queue[head])) {
663 /* something went quite wrong */
664 pthread_mutex_unlock(&processing_queue_mtx);
665 pthread_mutex_unlock(&gdb_mtx);
669 if (!(corefn = strip_directories(fullpath)))
670 goto clean_process_old;
672 if (!(oops = process_common(fullpath)))
673 goto clean_process_old;
675 if (!(oops->detail_filename = get_core_filename(corefn, "txt")))
676 goto clean_process_old;
678 remove_from_processing_queue();
680 pthread_mutex_unlock(&processing_queue_mtx);
681 pthread_mutex_unlock(&gdb_mtx);
683 pthread_mutex_lock(&core_status.queued_mtx);
684 queue_backtrace(oops);
685 pthread_mutex_unlock(&core_status.queued_mtx);
692 remove_pid_from_hash(fullpath, core_status.processing_oops);
693 remove_from_processing_queue();
696 pthread_mutex_unlock(&processing_queue_mtx);
697 pthread_mutex_unlock(&gdb_mtx);
702 * Adds corefile (based on pid) to the processing_oops
703 * hash table if it is not already there, then
704 * tries to add to the processing_queue.
706 * Picks up and sets down the processing_mtx.
707 * Picks up and sets down the processing_queue_mtx.
709 static int add_to_processing(char *fullpath)
711 char *c1 = NULL, *c2 = NULL, *fp = NULL;
716 if (!(fp = strdup(fullpath)))
717 goto clean_add_to_processing;
719 if (!(c1 = get_core_filename(fp, NULL)))
720 goto clean_add_to_processing;
722 if (!(c2 = strip_directories(c1)))
723 goto clean_add_to_processing;
728 pthread_mutex_lock(&core_status.processing_mtx);
729 if (g_hash_table_lookup(core_status.processing_oops, c2)) {
730 pthread_mutex_unlock(&core_status.processing_mtx);
731 /* This should only happen when the same core happened
732 * multiple times in the same second. Go ahead and
733 * ignore/remove the newer one here */
734 fprintf(stderr, "+ ...ignoring/unlinking %s\n", fullpath);
736 goto clean_add_to_processing;
739 pthread_mutex_lock(&processing_queue_mtx);
740 if (processing_queue[tail]) {
741 pthread_mutex_unlock(&processing_queue_mtx);
742 pthread_mutex_unlock(&core_status.processing_mtx);
743 goto clean_add_to_processing;
746 g_hash_table_insert(core_status.processing_oops, c2, c2);
747 processing_queue[tail++] = fp;
748 if (tail == MAX_PROCESSING_OOPS)
751 pthread_mutex_unlock(&processing_queue_mtx);
752 pthread_mutex_unlock(&core_status.processing_mtx);
754 clean_add_to_processing:
762 * Entry for processing new core files.
764 static void process_corefile(char *fullpath)
769 r = add_to_processing(fullpath);
774 if (pthread_create(&thrd, NULL, process_new, NULL))
775 fprintf(stderr, "Couldn't start up gdb extract core thread\n");
779 * Entry for processing already seen core files.
781 static void reprocess_corefile(char *fullpath)
786 r = add_to_processing(fullpath);
791 if (pthread_create(&thrd, NULL, process_old, NULL))
792 fprintf(stderr, "Couldn't start up gdb extract core thread\n");
795 int scan_corefolders(void __unused *unused)
798 struct dirent *entry = NULL;
799 char *fullpath = NULL, *appfile = NULL;
802 pthread_mutex_init(&core_status.processing_mtx, NULL);
803 pthread_mutex_init(&core_status.queued_mtx, NULL);
804 pthread_mutex_init(&core_status.asked_mtx, NULL);
806 /* scan for new crash data */
807 dir = opendir(core_folder);
810 fprintf(stderr, "+ scanning %s...\n", core_folder);
815 entry = readdir(dir);
816 if (!entry || !entry->d_name)
818 if (entry->d_name[0] == '.')
820 if (strncmp(entry->d_name, "core_", 5))
823 /* matched core_#### where #### is the pid of the process */
824 r = asprintf(&fullpath, "%s%s", core_folder, entry->d_name);
828 } else if (((unsigned int)r) != strlen(core_folder) + strlen(entry->d_name)) {
831 /* already found, waiting for response from user */
832 pthread_mutex_lock(&core_status.asked_mtx);
833 if (g_hash_table_lookup(core_status.asked_oops, fullpath)) {
834 pthread_mutex_unlock(&core_status.asked_mtx);
837 pthread_mutex_unlock(&core_status.asked_mtx);
838 fprintf(stderr, "+ Looking at %s\n", fullpath);
839 appfile = get_appfile(fullpath);
842 fprintf(stderr, "+ ...ignoring/unlinking %s\n", fullpath);
851 /* scan for partially processed crash data */
852 dir = opendir(processed_folder);
855 fprintf(stderr, "+ scanning %s...\n", processed_folder);
860 entry = readdir(dir);
861 if (!entry || !entry->d_name)
863 if (entry->d_name[0] == '.')
865 if (!strstr(entry->d_name, "process"))
868 r = asprintf(&fullpath, "%s%s", processed_folder, entry->d_name);
872 } else if (((unsigned int)r) != strlen(processed_folder) + strlen(entry->d_name)) {
876 fprintf(stderr, "+ Looking at %s\n", fullpath);
877 if (strstr(fullpath, "to-process"))
878 process_corefile(fullpath);
880 reprocess_corefile(fullpath);