* Authors:
* Arjan van de Ven <arjan@linux.intel.com>
* William Douglas <william.douglas@intel.com>
+ * Tim Pepper <timothy.c.pepper@linux.intel.com>
*/
#include <unistd.h>
#include <string.h>
#include <assert.h>
#include <fcntl.h>
-#include <pthread.h>
#include <asm/unistd.h>
#include <sys/types.h>
#include <sys/stat.h>
#include <sys/time.h>
+#include <sys/statvfs.h>
+#include <syslog.h>
#include <dirent.h>
-#include <signal.h>
#include <glib.h>
#include <errno.h>
#include "corewatcher.h"
-int uid = 0;
-int sig = 0;
-
-/* Always pick up the processing_mtx and then the
- processing_queue_mtx, reverse for setting down */
-/* Always pick up the gdb_mtx and then the
- processing_queue_mtx, reverse for setting down */
-/* Always pick up the processing_mtx and then the
- gdb_mtx, reverse for setting down */
-/* so order for pick up should be:
- processing_mtx -> gdb_mtx -> processing_queue_mtx
- and the reverse for setting down */
-static pthread_mutex_t processing_queue_mtx = PTHREAD_MUTEX_INITIALIZER;
-static char *processing_queue[MAX_PROCESSING_OOPS];
-static pthread_mutex_t gdb_mtx = PTHREAD_MUTEX_INITIALIZER;
-static int tail = 0;
-static int head = 0;
-
-static long int get_time(char *filename)
-{
- struct stat st;
- if (stat(filename, &st)) {
- return 0;
- }
- return st.st_mtim.tv_sec;
-}
-
-static char *get_build(void)
-{
- FILE *file = NULL;
- char *line = NULL, *c = NULL, *build = NULL;
- size_t dummy = 0;
-
- file = fopen(build_release, "r");
- if (!file) {
- line = strdup("Unknown");
- return line;
- }
-
- while (!feof(file)) {
- if (getline(&line, &dummy, file) == -1)
- break;
- if ((c = strstr(line, "BUILD"))) {
- c += 7;
- if (c >= line + strlen(line))
- break;
-
- /* glibc does things that scare valgrind */
- /* ignore valgrind error for the line below */
- build = strdup(c);
- if (!build)
- break;
-
- c = strchr(build, '\n');
- if (c) *c = 0;
-
- free(line);
- fclose(file);
- return build;
- }
- }
-
- fclose(file);
- free(line);
+/*
+ * processing "queue" loop's condition variable and associated
+ * lock. Note the queue is an implicit data structure consisting
+ * of the non-submitted core files in the filesystem, but the bool pq is
+ * used to mark whether the "queue" holds something to prevent the possible
+ * race where the condition is set before the thread is awaiting it and
+ * thus is not woken.
+ */
+GMutex *pq_mtx;
+static gboolean pq = FALSE;
+GCond *pq_work;
- line = strdup("Unknown");
-
- return line;
-}
+static int diskfree = 100;
static char *get_release(void)
{
char *line = NULL;
size_t dummy = 0;
- file = fopen("/etc/issue", "r");
- if (!file) {
- line = strdup("Unknown");
- return line;
- }
+ file = fopen("/etc/os-release", "r");
+ if (!file)
+ return NULL;
while (!feof(file)) {
if (getline(&line, &dummy, file) == -1)
break;
- if (strstr(line, "release")) {
+ if (strstr(line, "VERSION_ID=")) {
char *c = NULL;
c = strchr(line, '\n');
- if (c) *c = 0;
-
- fclose(file);
- return line;
+ if (c) {
+ *c = 0;
+ c = strdup(&line[11]);
+ fclose(file);
+ free(line);
+ return c;
+ }
}
}
fclose(file);
free(line);
- line = strdup("Unknown");
-
- return line;
-}
-
-static char *set_wrapped_app(char *line)
-{
- char *dline = NULL, *app = NULL, *appfile = NULL, *abs_path = NULL;
- char delim[] = " '";
- char app_folder[] = "/usr/share/";
- int r = 0;
-
- if (!line)
- return NULL;
-
- dline = strdup(line);
-
- app = strtok(dline, delim);
- while(app) {
- if (strcmp(app, "--app") == 0) {
- app = strtok(NULL, delim);
- break;
- }
- app = strtok(NULL, delim);
- }
- if (!app)
- goto cleanup_set_wrapped_app;
- r = asprintf(&abs_path, "%s%s", app_folder, app);
- if (r == -1) {
- abs_path = NULL;
- goto cleanup_set_wrapped_app;
- } else if (((unsigned int)r) != strlen(app_folder) + strlen(app)) {
- goto cleanup_set_wrapped_app;
- }
-
- appfile = find_executable(abs_path);
-
-cleanup_set_wrapped_app:
- free(abs_path);
- free(dline);
- return appfile;
-}
-
-static GList *get_core_file_list(char *appfile, char *dump_text)
-{
- char *txt = NULL, *part = NULL, *c = NULL;
- char delim[] = " ',`;\n\"";
- GList *files = NULL;
-
- if (!(txt = strdup(dump_text)))
- return NULL;
-
- part = strtok(txt, delim);
- while(part) {
- if (strstr(part, "/")) {
- if (!(c = strdup(part)))
- continue;
- files = g_list_prepend(files, c);
- }
- part = strtok(NULL, delim);
- }
- if ((c = strdup(appfile))) {
- files = g_list_prepend(files, c);
- }
-
- free(txt);
- return files;
-}
-
-static char *run_cmd(char *cmd)
-{
- char *line = NULL, *str = NULL;
- char *c = NULL;
- FILE *file = NULL;
- size_t size = 0;
-
- if (!cmd)
- return NULL;
- file = popen(cmd, "r");
- if (!file)
- return NULL;
-
- if (getline(&line, &size, file) != -1) {
- c = strchr(line, '\n');
- if (c) *c = 0;
- str = strdup(line);
- }
- free(line);
- pclose(file);
-
- return str;
-}
-
-static char *lookup_part(char *check, char *line)
-{
- char *c = NULL, *c1 = NULL, *c2 = NULL;
-
- if (!check || !line)
- return NULL;
-
- if (strncmp(check, line, strlen(check)))
- return NULL;
- if (!(c1 = strstr(line, ":")))
- return NULL;
- c1 += 2;
- if (c1 >= line + strlen(line))
- return NULL;
- if (!(c2 = strstr(c1, " ")))
- return NULL;
- *c2 = 0;
- if (!(c = strdup(c1)))
- return NULL;
- return c;
-}
-
-static int append(char **s, char *e, char *a)
-{
- char *t = NULL;
- int r = 0;
-
- if (!s || !(*s) || !e || !a)
- return -1;
- t = *s;
- *s = NULL;
- r = asprintf(s, "%s%s%s", t, a, e);
- if (r == -1) {
- *s = t;
- return -1;
- } else if (((unsigned int)r) != strlen(t) + strlen(a) + strlen(e)) {
- free(*s);
- *s = t;
- return -1;
- }
- free(t);
- return 0;
-}
-
-static void build_times(char *cmd, GHashTable *ht_p2p, GHashTable *ht_p2d)
-{
- FILE *file = NULL;
- int ret = 0, i = 0;
- char *line = NULL, *dline = NULL, *pack = NULL, *date = NULL;
- char *nm = NULL, *vr = NULL, *rl = NULL, *c = NULL, *p = NULL;
- size_t size = 0;
- char name[] = "Name";
- char version[] = "Version";
- char release[] = "Release";
- char delim[] = " ";
-
- file = popen(cmd, "r");
- if (!file)
- return;
- while (!feof(file)) {
- pack = nm = vr = rl = NULL;
- if (getline(&line, &size, file) == -1)
- goto cleanup;
- if (!(nm = lookup_part(name, line)))
- goto cleanup;
- if (getline(&line, &size, file) == -1)
- goto cleanup;
- if (!(vr = lookup_part(version, line)))
- goto cleanup;
- if (getline(&line, &size, file) == -1)
- goto cleanup;
- if (!(rl = lookup_part(release, line)))
- goto cleanup;
- ret = asprintf(&pack, "%s-%s-%s", nm, vr, rl);
- if (ret == -1)
- goto cleanup;
- else if (((unsigned int)ret) != strlen(nm) + strlen(vr) + strlen(rl) + 2)
- goto cleanup;
- /* using p instead of pack to keep freeing the hashtables uniform */
- if (!(p = g_hash_table_lookup(ht_p2p, pack)))
- goto cleanup;
-
- while (!feof(file)) {
- c = NULL;
- if (getline(&dline, &size, file) == -1)
- goto cleanup;
- if (strncmp("*", dline, 1))
- continue;
- /* twice to skip the leading '*' */
- c = strtok(dline, delim);
- if (!c) continue;
- c = strtok(NULL, delim);
- if (!c) continue;
-
- if (!(date = strdup(c)))
- goto cleanup;
-
- for (i = 0; i < 3; i++) {
- c = strtok(NULL, delim);
- if (!c) goto cleanup;
- if ((ret = append(&date, c, " ")) < 0)
- goto cleanup;
- }
- g_hash_table_insert(ht_p2d, p, date);
- date = NULL;
- break;
- }
- cleanup:
- free(nm);
- free(vr);
- free(rl);
- free(pack);
- free(date);
- date = NULL;
- }
- pclose(file);
- free(dline);
- free(line);
-
- return;
-}
-
-static char *get_package_info(char *appfile, char *dump_text)
-{
- GList *l = NULL, *files = NULL, *hfiles = NULL, *tmpl = NULL;
- GHashTable *ht_f2f = NULL, *ht_f2p = NULL, *ht_p2p = NULL, *ht_p2d = NULL;
- char *c1 = NULL, *cmd = NULL, *out = NULL;
- char find_pkg[] = "rpm -qf --queryformat \"%{NAME}-%{VERSION}-%{RELEASE}\" ";
- char find_date[] = "rpm -qi --changelog";
- char dev_null[] = "2>/dev/null";
- int r = 0;
-
- if (!(ht_f2f = g_hash_table_new(g_str_hash, g_str_equal)))
- goto clean_up;
- if (!(ht_f2p = g_hash_table_new_full(g_str_hash, g_str_equal, NULL, free)))
- goto clean_up;
- if (!(ht_p2p = g_hash_table_new(g_str_hash, g_str_equal)))
- goto clean_up;
- if (!(ht_p2d = g_hash_table_new_full(g_str_hash, g_str_equal, NULL, free)))
- goto clean_up;
- if (!(files = get_core_file_list(appfile, dump_text)))
- goto clean_up;
-
- /* get files in hash to remove duplicates */
- for (l = files; l; l = l->next) {
- if (!g_hash_table_lookup(ht_f2f, l->data))
- g_hash_table_insert(ht_f2f, l->data, l->data);
- }
-
- hfiles = g_hash_table_get_keys(ht_f2f);
-
- /* run through files one at a time in case some files don't have packages and it */
- /* isn't guaranteed we will see the error correspond with the file we are testing */
- for (l = hfiles; l; l = l->next) {
- r = asprintf(&cmd, "%s%s %s", find_pkg, (char *)l->data, dev_null);
- if (r == -1)
- goto clean_up;
- else if (((unsigned int)r) != sizeof(find_pkg) + sizeof((char *)l->data) + sizeof(dev_null) + 1) {
- free(cmd);
- goto clean_up;
- }
- c1 = run_cmd(cmd);
- free(cmd);
- cmd = NULL;
-
- if (c1 && strlen(c1) > 0) {
- g_hash_table_insert(ht_f2p, l->data, c1);
- } else {
- g_hash_table_insert(ht_f2p, l->data, NULL);
- free(c1);
- }
- }
-
- tmpl = g_hash_table_get_values(ht_f2p);
- for (l = tmpl; l; l = l->next) {
- if (l->data && !g_hash_table_lookup(ht_p2p, l->data))
- g_hash_table_insert(ht_p2p, l->data, l->data);
- }
-
- g_list_free(tmpl);
- tmpl = NULL;
- tmpl = g_hash_table_get_keys(ht_p2p);
- cmd = strdup(find_date);
- if (!cmd)
- goto clean_up;
- for (l = tmpl; l; l = l->next) {
- append(&cmd, l->data, " ");
- }
- g_list_free(tmpl);
- tmpl = NULL;
- build_times(cmd, ht_p2p, ht_p2d);
- free(cmd);
-
- if (!(out = strdup("")))
- goto clean_up;
- for (l = hfiles; l; l = l->next) {
- if (append(&out, l->data, "") < 0)
- continue;
-
- if (!(c1 = g_hash_table_lookup(ht_f2p, l->data))) {
- if (append(&out, "\n", "") < 0)
- goto clean_out;
- continue;
- } else
- if (append(&out, c1, ":") < 0)
- goto clean_out;
-
- if (!(c1 = g_hash_table_lookup(ht_p2d, c1))) {
- if (append(&out, "\n", "") < 0)
- goto clean_out;
- continue;
- } else
- if (append(&out, c1, ":") < 0)
- goto clean_out;
-
- if (append(&out, "\n", "") < 0)
- goto clean_out;
- }
- goto clean_up;
-
-clean_out:
- free(out);
- out = NULL;
-
-clean_up:
- if (ht_p2d)
- g_hash_table_destroy(ht_p2d);
- if (ht_p2p)
- g_hash_table_destroy(ht_p2p);
- if (ht_f2p)
- g_hash_table_destroy(ht_f2p);
- if (ht_f2f)
- g_hash_table_destroy(ht_f2f);
- if (files)
- g_list_free_full(files, free);
- if (hfiles)
- g_list_free(hfiles);
- if (tmpl)
- g_list_free(tmpl);
-
- return out;
-}
-
-static char *signame(int sig)
-{
- switch(sig) {
- case SIGINT: return "SIGINT";
- case SIGILL: return "SIGILL";
- case SIGABRT: return "SIGABRT";
- case SIGFPE: return "SIGFPE";
- case SIGSEGV: return "SIGSEGV";
- case SIGPIPE: return "SIGPIPE";
- case SIGBUS: return "SIGBUS";
- default: return strsignal(sig);
- }
return NULL;
}
-static char *get_kernel(void)
-{
- char *line = NULL;
- FILE *file = NULL;
- int ret = 0;
- size_t size = 0;
- char command[] = "uname -r";
-
- file = popen(command, "r");
-
- if (!file) {
- line = strdup("Unknown");
- return line;
- }
-
- ret = getline(&line, &size, file);
- if (!size || ret <= 0) {
- pclose(file);
- line = strdup("Unknown");
- return line;
- }
-
- size = strlen(line);
- line[size - 1] = '\0';
-
- pclose(file);
- return line;
-}
-
-static char *build_core_header(char *appfile, char *corefile, char * processed_fullpath)
-{
- int ret = 0;
- char *result = NULL;
- char *build = get_build();
- char *release = get_release();
- char *kernel = get_kernel();
- long int time = get_time(corefile);
-
- ret = asprintf(&result,
- "analyzer: corewatcher-gdb\n"
- "coredump: %s\n"
- "executable: %s\n"
- "kernel: %s\n"
- "reason: Process %s was killed by signal %d (%s)\n"
- "release: %s\n"
- "build: %s\n"
- "time: %lu\n"
- "uid: %d\n",
- processed_fullpath,
- appfile,
- kernel,
- appfile, sig, signame(sig),
- release,
- build,
- time,
- uid);
-
- free(kernel);
- free(release);
- free(build);
-
- if (ret == -1)
- result = strdup("Unknown");
-
- return result;
-}
-
-/*
- * Scan core dump in case a wrapper was used
- * to run the process and get the actual binary name
- */
-static char *wrapper_scan(char *command)
-{
- char *line = NULL, *appfile = NULL;
- FILE *file = NULL;
-
- file = popen(command, "r");
- if (!file)
- return NULL;
-
- while (!feof(file)) {
- size_t size = 0;
- int ret = 0;
- free(line);
- ret = getline(&line, &size, file);
- if (!size)
- break;
- if (ret < 0)
- break;
-
- if (strstr(line, "Core was generated by") &&
- strstr(line, "--app")) {
- /* attempt to update appfile */
- appfile = set_wrapped_app(line);
- break;
- }
- }
- if (line)
- free(line);
- pclose(file);
-
- return appfile;
-}
-
/*
* Strip the directories from the path
* given by fullname
{
char *dfile = NULL, *c1 = NULL, *c2 = NULL, *r = NULL;
char delim[] = "/";
+ char *saveptr;
if (!fullpath)
return NULL;
if (!dfile)
return NULL;
- c1 = strtok(dfile, delim);
+ c1 = strtok_r(dfile, delim, &saveptr);
while(c1) {
c2 = c1;
- c1 = strtok(NULL, delim);
+ c1 = strtok_r(NULL, delim, &saveptr);
}
if (c2)
r = strdup(c2);
+
free(dfile);
return r;
}
/*
- * Move corefile from /tmp to core_folder.
- * Add extension if given and attempt to create core_folder.
+ * Move corefile from core_folder to processed_folder subdir.
+ * If this type of core has recently been seen, unlink this more recent
+ * example in order to rate limit submissions of extremely crashy
+ * applications.
+ * Add extension and attempt to create directories if needed.
*/
-int move_core(char *fullpath, char *extension)
+static int move_core(char *fullpath, char *extension)
{
- char *corefn = NULL, newpath[8192];
+ char *corefilename = NULL, *newpath = NULL, *coreprefix = NULL;
+ char *s = NULL;
+ size_t prefix_len;
+ DIR *dir = NULL;
+ struct dirent *entry = NULL;
- if (!core_folder || !fullpath)
+ if (!fullpath)
return -1;
- if (!(corefn = strip_directories(fullpath))) {
- unlink(fullpath);
+ corefilename = strip_directories(fullpath);
+ if (!corefilename)
return -ENOMEM;
- }
- if (!mkdir(core_folder, S_IRWXU | S_IRWXG | S_IRWXO)
- && errno != EEXIST) {
- free(corefn);
- return -errno;
+ /* if the corefile's name minus any suffixes (such as .$PID) and
+ * minus two additional characters (ie: last two digits of
+ * timestamp assuming core_%e_%t) matches another core file in the
+ * processed_folder, simply unlink it instead of processing it for
+ * submission. TODO: consider a (configurable) time delta greater
+ * than which the cores must be separated, stat'ing the files, etc.
+ */
+ coreprefix = strdup(corefilename);
+ if (!coreprefix) {
+ free(corefilename);
+ return -ENOMEM;
+ }
+ s = strstr(coreprefix, ".");
+ if (!s) {
+ free(coreprefix);
+ free(corefilename);
+ return -1;
+ }
+ *s = '\0';
+ prefix_len = strlen(coreprefix);
+ if (prefix_len > 2) {
+ s = strndup(coreprefix, prefix_len - 2);
+ free(coreprefix);
+ coreprefix = s;
+ } else {
+ goto error;
+ }
+ dir = opendir(processed_folder);
+ if (!dir)
+ goto error;
+ while(1) {
+ entry = readdir(dir);
+ if (!entry || !entry->d_name)
+ break;
+ if (entry->d_name[0] == '.')
+ continue;
+ if (!strstr(entry->d_name, coreprefix))
+ continue;
+ fprintf(stderr, "+ ...ignoring/unlinking %s\n", fullpath);
+ unlink(fullpath);
+ closedir(dir);
+ goto error;
}
+ closedir(dir);
- if (extension)
- snprintf(newpath, 8192, "%s%s.%s", core_folder, corefn, extension);
- else
- snprintf(newpath, 8192, "%s%s", core_folder, corefn);
+ if (asprintf(&newpath, "%s%s.%s", processed_folder, corefilename, extension) == -1)
+ goto error;
- free(corefn);
+ free(coreprefix);
+ free(corefilename);
rename(fullpath, newpath);
-
+ free(newpath);
return 0;
-}
-
-/*
- * Finds the full path for the application that crashed,
- * and depending on what opted_in was configured as will:
- * opted_in 2 (always submit) -> move file to core_folder
- * to be processed further
- * opted_in 1 (ask user) -> ask user if we should submit
- * the crash and add to asked_oops hash so we don't get
- * called again for this corefile
- * opted_in 0 (don't submit) -> do nothing
- *
- * Picks up and sets down the asked_mtx.
- */
-static char *get_appfile(char *fullpath)
-{
- char *appname = NULL, *appfile = NULL;
-
- if (!fullpath)
- return NULL;
-
- appname = find_coredump(fullpath);
- if (!appname)
- return NULL;
-
- /* don't try and do anything for rpm, gdb or corewatcher crashes */
- if (!(strcmp(appname, "rpm") && strcmp(appname, "gdb") && strcmp(appname, "corewatcher")))
- return NULL;
-
- appfile = find_executable(appname);
- /* appname no longer used, so free it as it was strdup'd */
- free(appname);
- if (!appfile)
- return NULL;
- if (opted_in == 2) {
- move_core(fullpath, "to-process");
- } else {
- free(appfile);
- return NULL;
- }
-
- return appfile;
+error:
+ free(coreprefix);
+ free(corefilename);
+ return -1;
}
+
/*
* Use GDB to extract backtrace information from corefile
*/
-static struct oops *extract_core(char *fullpath, char *appfile, char *processed_fullpath)
+static struct oops *extract_core(char *fullpath, char *appfile, char *reportname)
{
struct oops *oops = NULL;
int ret = 0;
- char *command = NULL, *h1 = NULL, *h2 = NULL, *c1 = NULL, *c2 = NULL, *line = NULL, *text = NULL, *at = NULL;
+ char *command = NULL, *h1 = NULL, *c1 = NULL, *c2 = NULL, *line = NULL;
+ char *text = NULL, *coretime = NULL;
+ char *m1 = NULL, *m2 = NULL;
+ int bt_lines = 0, maps_lines = 0;
FILE *file = NULL;
- char *private = private_report ? "private: yes\n" : "";
+ char *badchar = NULL;
+ char *release = get_release();
+ int parsing_maps = 0;
+ struct stat stat_buf;
+ size_t size = 0;
+ ssize_t bytesread = 0;
+
+ fprintf(stderr, "+ extract_core() called for %s\n", fullpath);
- if (asprintf(&command, "LANG=C gdb --batch -f %s %s -x /var/lib/corewatcher/gdb.command 2> /dev/null", appfile, fullpath) == -1)
+ if (asprintf(&command, "LANG=C gdb --batch -f '%s' '%s' -x /etc/corewatcher/gdb.command 2> /dev/null", appfile, fullpath) == -1)
return NULL;
- if ((at = wrapper_scan(command))) {
- free(appfile);
- appfile = at;
- }
+ file = popen(command, "r");
+ free(command);
+ if (!file)
+ fprintf(stderr, "+ gdb failed for %s\n", fullpath);
- h1 = build_core_header(appfile, fullpath, processed_fullpath);
+ if (stat(fullpath, &stat_buf) != -1) {
+ coretime = malloc(26);
+ if (coretime)
+ ctime_r(&stat_buf.st_mtime, coretime);
+ }
- file = popen(command, "r");
+ ret = asprintf(&h1,
+ "cmdline: %s\n"
+ "release: %s\n"
+ "time: %s",
+ appfile,
+ release ? release : "Unknown",
+ coretime ? coretime : "Unknown");
+ if (release)
+ free(release);
+ if (coretime)
+ free(coretime);
+ if (ret == -1)
+ return NULL;
while (file && !feof(file)) {
- size_t size = 0;
-
- c2 = c1;
- free(line);
- ret = getline(&line, &size, file);
+ bytesread = getline(&line, &size, file);
if (!size)
break;
- if (ret == -1)
+ if (bytesread == -1)
break;
- if (strstr(line, "no debugging symbols found")) {
- continue;
+ /* try to figure out if we're onto the maps output yet */
+ if (strncmp(line, "From", 4) == 0) {
+ parsing_maps = 1;
}
- if (strstr(line, "reason: ")) {
- continue;
+ /* maps might not be present */
+ if (strncmp(line, "No shared libraries", 19) == 0) {
+ break;
}
- if (c1) {
- c1 = NULL;
- if (asprintf(&c1, "%s%s", c2, line) == -1)
+ if (!parsing_maps) { /* parsing backtrace */
+ c2 = c1;
+
+ /* gdb's backtrace lines start with a line number */
+ if (line[0] != '#')
continue;
- free(c2);
- } else {
- /* keep going even if asprintf has errors */
- ret = asprintf(&c1, "%s", line);
+
+ /* gdb prints some initial info which may include the
+ * "#0" line of the backtrace, then prints the
+ * backtrace in its entirety, leading to a
+ * duplicate "#0" in our summary if we do do: */
+ if ((bt_lines == 1) && (strncmp(line, "#0 ", 3) == 0))
+ continue;
+ bt_lines++;
+
+ /* gdb outputs some 0x1a's which break XML */
+ do {
+ badchar = memchr(line, 0x1a, bytesread);
+ if (badchar)
+ *badchar = ' ';
+ } while (badchar);
+
+ if (c1) {
+ c1 = NULL;
+ if (asprintf(&c1, "%s %s", c2, line) == -1)
+ continue;
+ free(c2);
+ } else {
+ /* keep going even if asprintf has errors */
+ ret = asprintf(&c1, " %s", line);
+ }
+ } else { /* parsing maps */
+ m2 = m1;
+ maps_lines++;
+ if (m1) {
+ m1 = NULL;
+ if (asprintf(&m1, "%s %s", m2, line) == -1)
+ continue;
+ free(m2);
+ } else {
+ /* keep going even if asprintf has errors */
+ ret = asprintf(&m1, " %s", line);
+ }
}
}
if (line)
free(line);
- pclose(file);
- free(command);
-
- if (!(h2 = get_package_info(appfile, c1)))
- h2 = strdup("Unknown");
+ if (file)
+ pclose(file);
ret = asprintf(&text,
"%s"
- "package-info\n-----\n"
+ "backtrace: |\n"
"%s"
- "\n-----\n"
- "%s"
- "\nbacktrace\n-----\n"
+ "maps: |\n"
"%s",
- h1, h2, private, c1);
- if (ret == -1)
- text = NULL;
+ h1,
+ c1 ? c1 : " Unknown\n",
+ m1 ? m1 : " Unknown\n");
free(h1);
- free(h2);
- free(c1);
+ if (c1)
+ free(c1);
+ if (m1)
+ free(m1);
+
+ if (ret == -1)
+ return NULL;
oops = malloc(sizeof(struct oops));
if (!oops) {
return NULL;
}
memset(oops, 0, sizeof(struct oops));
+ oops->next = NULL;
oops->application = strdup(appfile);
oops->text = text;
oops->filename = strdup(fullpath);
+ oops->detail_filename = strdup(reportname);
return oops;
}
/*
- * filename is of the form core.XXXX[.blah]
- * we need to get the pid out as we want
- * output of the form XXXX[.ext]
+ * input filename has the form: core_$APP_$TIMESTAMP[.$PID]
+ * output filename has form of: $APP_$TIMESTAMP.txt
*/
-char *get_core_filename(char *filename, char *ext)
+static char *make_report_filename(char *filename)
{
- char *pid = NULL, *c = NULL, *s = NULL, *detail_filename = NULL;
+ char *name = NULL, *dotpid = NULL, *stamp = NULL, *detail_filename = NULL;
if (!filename)
return NULL;
- if (!(s = strstr(filename, ".")))
+ if (!(stamp = strstr(filename, "_")))
return NULL;
- if (!(++s))
- return NULL;
- /* causes valgrind whining because we copy from middle of a string */
- if (!(pid = strdup(s)))
+ if (!(++stamp))
return NULL;
- c = strstr(pid, ".");
+ if (!(name = strdup(stamp)))
+ return NULL;
- if (c)
- *c = '\0';
+ /* strip trailing .PID if present */
+ dotpid = strstr(name, ".");
+ if (dotpid)
+ *dotpid = '\0';
- if (ext) {
- /* causes valgrind whining because we copy from middle of a string */
- if ((asprintf(&detail_filename, "%s%s.%s", core_folder, pid, ext)) == -1) {
- free(pid);
- return NULL;
- }
- } else {
- /* causes valgrind whining because we copy from middle of a string */
- if ((asprintf(&detail_filename, "%s%s", core_folder, pid)) == -1) {
- free(pid);
- return NULL;
- }
+ if ((asprintf(&detail_filename, "%s%s.txt", processed_folder, name)) == -1) {
+ free(name);
+ return NULL;
}
+ free(name);
- free(pid);
return detail_filename;
}
/*
* Write the backtrace from the core file into a text
- * file named after the pid
+ * file named as $APP_$TIMESTAMP.txt
*/
-static void write_core_detail_file(char *filename, char *text)
+static void write_core_detail_file(struct oops *oops)
{
int fd = 0;
- char *detail_filename = NULL;
- if (!filename || !text)
+ if (!oops->detail_filename)
return;
- if (!(detail_filename = get_core_filename(filename, "txt")))
+ fd = open(oops->detail_filename, O_WRONLY | O_CREAT | O_TRUNC, 0);
+ if (fd == -1) {
+ fprintf(stderr, "+ Error creating/opening %s for write\n", oops->detail_filename);
return;
-
- if ((fd = open(detail_filename, O_WRONLY | O_CREAT | O_TRUNC, 0)) != -1) {
- if(write(fd, text, strlen(text)) >= 0)
- fchmod(fd, 0644);
- else
- unlink(detail_filename);
- close(fd);
}
- free(detail_filename);
-}
-
-/*
- * Removes corefile (core.XXXX) from the processing_queue.
- *
- * Expects the processing_queue_mtx to be held.
- */
-static void remove_from_processing_queue(void)
-{
- free(processing_queue[head]);
- processing_queue[head++] = NULL;
-
- if (head == 100)
- head = 0;
-}
-/*
- * Removes file from processing_oops hash based on pid.
- * Extracts pid from the fullpath such that
- * /home/user/core.pid will be tranformed into just the pid.
- *
- * Expects the lock on the given hash to be held.
- */
-void remove_pid_from_hash(char *fullpath, GHashTable *ht)
-{
- char *c1 = NULL, *c2 = NULL;
-
- if (!(c1 = strip_directories(fullpath)))
- return;
-
- if (!(c2 = get_core_filename(c1, NULL))) {
- free(c1);
- return;
+ if(write(fd, oops->text, strlen(oops->text)) >= 0) {
+ fprintf(stderr, "+ Wrote %s\n", oops->detail_filename);
+ fchmod(fd, 0644);
+ } else {
+ fprintf(stderr, "+ Error writing %s\n", oops->detail_filename);
+ unlink(oops->detail_filename);
}
-
- free(c1);
-
- g_hash_table_remove(ht, c2);
-
- free(c2);
+ close(fd);
}
/*
* Common function for processing core
- * files to generate oops structures
+ * files to generate oops structures and write *.txt
+ * if not already present
*/
-static struct oops *process_common(char *fullpath, char *processed_fullpath)
+static struct oops *process_common(char *fullpath)
{
struct oops *oops = NULL;
- char *appname = NULL, *appfile = NULL;
+ char *appname = NULL, *appfile = NULL, *corefn = NULL, *reportname = NULL;
+ struct stat stat_buf;
- if(!(appname = find_coredump(fullpath))) {
+ corefn = strip_directories(fullpath);
+ if (!corefn) {
+ fprintf(stderr, "+ No corefile? (%s)\n", fullpath);
return NULL;
}
- if (!(appfile = find_executable(appname))) {
+ appname = find_causingapp(fullpath);
+ if (!appname) {
+ free(corefn);
+ free(reportname);
+ return NULL;
+ }
+ /*
+ * don't process rpm, gdb or corewatcher crashes,
+ * also skip apps which don't appear to be part of the OS
+ */
+ appfile = find_apppath(appname);
+ if (!appfile ||
+ !strncmp(appname, "rpm", 3) ||
+ !strncmp(appname, "gdb", 3) ||
+ !strncmp(appname, "corewatcher", 11)) {
+ free(corefn);
free(appname);
+ fprintf(stderr, "+ ...ignoring %s's %s\n", appname, fullpath);
+ move_core(fullpath, "skipped");
return NULL;
}
free(appname);
- if (!(oops = extract_core(fullpath, appfile, processed_fullpath))) {
+ reportname = make_report_filename(corefn);
+ if (!reportname) {
+ fprintf(stderr, "+ Couldn't make report name for %s\n", corefn);
+ free(corefn);
free(appfile);
return NULL;
}
- free(appfile);
+ free(corefn);
+ if (stat(reportname, &stat_buf) == 0) {
+ int fd, ret;
+ /*
+ * TODO:
+ * If the file already has trailing ".processed" and the txt file
+ * is a low quality report, then create a new report.
+ */
+ fprintf(stderr, "+ Report already exists in %s\n", reportname);
+
+ oops = malloc(sizeof(struct oops));
+ if (!oops) {
+ fprintf(stderr, "+ Malloc failed for struct oops\n");
+ free(reportname);
+ free(appfile);
+ return NULL;
+ }
+ memset(oops, 0, sizeof(struct oops));
+
+ oops->next = NULL;
+ oops->application = strdup(appfile);
+ oops->filename = strdup(fullpath);
+ oops->detail_filename = strdup(reportname);
+ free(reportname);
+ free(appfile);
+ oops->text = malloc(stat_buf.st_size + 1);
+ if (!oops->text) {
+ fprintf(stderr, "+ Malloc failed for oops text\n");
+ goto err;
+ }
+ fd = open(oops->detail_filename, O_RDONLY);
+ if (fd == -1) {
+ fprintf(stderr, "+ Open failed for oops text\n");
+ goto err;
+ }
+ ret = read(fd, oops->text, stat_buf.st_size);
+ close(fd);
+ if (ret != stat_buf.st_size) {
+ fprintf(stderr, "+ Read failed for oops text\n");
+ goto err;
+ }
+ oops->text[stat_buf.st_size] = '\0';
+ return oops;
+ }
+
+ oops = extract_core(fullpath, appfile, reportname);
+ write_core_detail_file(oops);
+ free(reportname);
+ free(appfile);
return oops;
+err:
+ FREE_OOPS(oops);
+ return NULL;
}
/*
- * Processes .to-process core files.
- * Also creates the pid.txt file and adds
- * the oops struct to the submit queue
- *
- * Picks up and sets down the gdb_mtx and
- * picks up and sets down the processing_queue_mtx.
- * (held at the same time in that order)
- * Also will pick up and sets down the queued_mtx.
+ * Creates $APP_$TIMESTAMP.txt report summaries if they don't exist and
+ * adds the oops struct to the submit queue
*/
-static void *process_new(void __unused *vp)
+static void *create_report(char *fullpath)
{
struct oops *oops = NULL;
- char *procfn = NULL, *corefn = NULL, *fullpath = NULL;
+ char *procfn = NULL;
+ int new = 0, ret;
+
+ fprintf(stderr, "+ Entered create_report() for %s\n", fullpath);
- pthread_mutex_lock(&core_status.processing_mtx);
- pthread_mutex_lock(&gdb_mtx);
- pthread_mutex_lock(&processing_queue_mtx);
+ /*
+ * If the file has trailing ".to-process", create a new report.
+ */
+ if (strstr(fullpath, ".to-process"))
+ new = 1;
- if (!(fullpath = processing_queue[head])) {
- /* something went quite wrong */
- pthread_mutex_unlock(&processing_queue_mtx);
- pthread_mutex_unlock(&gdb_mtx);
- pthread_mutex_unlock(&core_status.processing_mtx);
+ oops = process_common(fullpath);
+ if (!oops) {
+ fprintf(stderr, "+ Did not generate struct oops for %s\n", fullpath);
return NULL;
}
- if (!(corefn = strip_directories(fullpath)))
- goto clean_process_new;
-
- if (!(procfn = replace_name(fullpath, ".to-process", ".processed")))
- goto clean_process_new;
-
- if (!(oops = process_common(fullpath, procfn)))
- goto clean_process_new;
-
- if (!(oops->detail_filename = get_core_filename(corefn, "txt")))
- goto clean_process_new;
-
- if (rename(fullpath, procfn))
- goto clean_process_new;
-
- free(oops->filename);
- oops->filename = procfn;
-
- remove_from_processing_queue();
-
- pthread_mutex_unlock(&processing_queue_mtx);
- pthread_mutex_unlock(&gdb_mtx);
- pthread_mutex_unlock(&core_status.processing_mtx);
-
- write_core_detail_file(corefn, oops->text);
-
- pthread_mutex_lock(&core_status.queued_mtx);
- queue_backtrace(oops);
- pthread_mutex_unlock(&core_status.queued_mtx);
+ if (new) {
+ procfn = replace_name(fullpath, ".to-process", ".processed");
+ if (!procfn) {
+ fprintf(stderr, "+ Problems with filename manipulation for %s\n", fullpath);
+ goto clean_process_new;
+ }
+ ret = rename(fullpath, procfn);
+ if (ret) {
+ fprintf(stderr, "+ Unable to move %s to %s\n", fullpath, procfn);
+ free(procfn);
+ goto clean_process_new;
+ }
+ free(oops->filename);
+ oops->filename = strdup(procfn);
+ free(procfn);
+ }
- /* don't need to free procfn because was set to oops->filename and that gets free'd */
- free(corefn);
- FREE_OOPS(oops);
- return NULL;
+ return oops;
clean_process_new:
- remove_pid_from_hash(fullpath, core_status.processing_oops);
- remove_from_processing_queue();
- free(procfn);
- procfn = NULL; /* don't know if oops->filename == procfn so be safe */
- free(corefn);
FREE_OOPS(oops);
- pthread_mutex_unlock(&processing_queue_mtx);
- pthread_mutex_unlock(&gdb_mtx);
- pthread_mutex_unlock(&core_status.processing_mtx);
return NULL;
}
/*
- * Reprocesses .processed core files.
- *
- * Picks up and sets down the gdb_mtx.
- * Picks up and sets down the processing_queue_mtx.
- * (held at the same time in that order)
- * Also will pick up and sets down the queued_mtx.
+ * scan once for core files in core_folder, moving any to the
+ * processed_folder with ".to-process" appended to their name
*/
-static void *process_old(void __unused *vp)
+int scan_core_folder(void __unused *unused)
{
- struct oops *oops = NULL;
- char *corefn = NULL, *fullpath = NULL;
-
- pthread_mutex_lock(&gdb_mtx);
- pthread_mutex_lock(&processing_queue_mtx);
+ DIR *dir = NULL;
+ struct dirent *entry = NULL;
+ char *fullpath = NULL;
+ int ret, work = 0;
- if (!(fullpath = processing_queue[head])) {
- /* something went quite wrong */
- pthread_mutex_unlock(&processing_queue_mtx);
- pthread_mutex_unlock(&gdb_mtx);
- return NULL;
+ dir = opendir(core_folder);
+ if (!dir) {
+ fprintf(stderr, "+ Unable to open %s\n", core_folder);
+ return -1;
}
+ fprintf(stderr, "+ Begin scanning %s...\n", core_folder);
+ while(1) {
+ entry = readdir(dir);
+ if (!entry || !entry->d_name)
+ break;
+ if (entry->d_name[0] == '.')
+ continue;
+ if (strncmp(entry->d_name, "core_", 5))
+ continue;
- if (!(corefn = strip_directories(fullpath)))
- goto clean_process_old;
-
- if (!(oops = process_common(fullpath, fullpath)))
- goto clean_process_old;
+ /* matched core_#### */
+ if (asprintf(&fullpath, "%s%s", core_folder, entry->d_name) == -1) {
+ fullpath = NULL;
+ continue;
+ }
- if (!(oops->detail_filename = get_core_filename(corefn, "txt")))
- goto clean_process_old;
+ /* If one were to prompt the user before submitting, that
+ * might happen here. */
- remove_from_processing_queue();
+ fprintf(stderr, "+ Looking at %s\n", fullpath);
- pthread_mutex_unlock(&processing_queue_mtx);
- pthread_mutex_unlock(&gdb_mtx);
+ ret = move_core(fullpath, "to-process");
+ if (ret == 0)
+ work++;
- pthread_mutex_lock(&core_status.queued_mtx);
- queue_backtrace(oops);
- pthread_mutex_unlock(&core_status.queued_mtx);
+ free(fullpath);
+ fullpath = NULL;
+ }
+ closedir(dir);
- free(corefn);
- FREE_OOPS(oops);
- return NULL;
+ if (work) {
+ fprintf(stderr, "+ Found %d files, setting pq_work condition\n", work);
+ g_mutex_lock(pq_mtx);
+ g_cond_signal(pq_work);
+ pq = TRUE;
+ g_mutex_unlock(pq_mtx);
+ }
-clean_process_old:
- remove_pid_from_hash(fullpath, core_status.processing_oops);
- remove_from_processing_queue();
- free(corefn);
- FREE_OOPS(oops);
- pthread_mutex_unlock(&processing_queue_mtx);
- pthread_mutex_unlock(&gdb_mtx);
- return NULL;
+ fprintf(stderr, "+ End scanning %s...\n", core_folder);
+ return TRUE;
}
/*
- * Adds corefile (based on pid) to the processing_oops
- * hash table if it is not already there, then
- * tries to add to the processing_queue.
- *
- * Picks up and sets down the processing_mtx.
- * Picks up and sets down the processing_queue_mtx.
+ * scan for core_*.to-process and core_*.processed,
+ * insure a summary *.txt report exists, then queue it
*/
-static int add_to_processing(char *fullpath)
+void *scan_processed_folder(void __unused *unused)
{
- char *c1 = NULL, *c2 = NULL, *fp = NULL;
+ DIR *dir = NULL;
+ struct dirent *entry = NULL;
+ char *fullpath = NULL;
+ struct oops *oops = NULL;
- if (!fullpath)
- return -1;
+ while(1) {
+ g_mutex_lock(pq_mtx);
+ while (pq != TRUE) {
+ fprintf(stderr, "+ Awaiting work in %s...\n", processed_folder);
+ g_cond_wait(pq_work, pq_mtx);
+ }
+ pq = FALSE;
+ g_mutex_unlock(pq_mtx);
- if (!(fp = strdup(fullpath)))
- goto clean_add_to_processing;
+ fprintf(stderr, "+ Begin scanning %s...\n", processed_folder);
+
+ dir = opendir(processed_folder);
+ if (!dir) {
+ fprintf(stderr, "+ Unable to open %s\n", processed_folder);
+ continue;
+ }
+ while(1) {
+ entry = readdir(dir);
+ if (!entry || !entry->d_name)
+ break;
+ if (entry->d_name[0] == '.')
+ continue;
- if (!(c1 = get_core_filename(fp, NULL)))
- goto clean_add_to_processing;
+ /* files with trailing ".to-process" or "processed" represent new work */
+ if (!strstr(entry->d_name, "process"))
+ continue;
- if (!(c2 = strip_directories(c1)))
- goto clean_add_to_processing;
+ if (asprintf(&fullpath, "%s%s", processed_folder, entry->d_name) == -1) {
+ fullpath = NULL;
+ continue;
+ }
- free(c1);
- c1 = NULL;
+ fprintf(stderr, "+ Looking at %s\n", fullpath);
- pthread_mutex_lock(&core_status.processing_mtx);
- if (g_hash_table_lookup(core_status.processing_oops, c2)) {
- pthread_mutex_unlock(&core_status.processing_mtx);
- goto clean_add_to_processing;
- }
+ oops = create_report(fullpath);
- pthread_mutex_lock(&processing_queue_mtx);
- if (processing_queue[tail]) {
- pthread_mutex_unlock(&processing_queue_mtx);
- pthread_mutex_unlock(&core_status.processing_mtx);
- goto clean_add_to_processing;
- }
+ if (oops) {
+ fprintf(stderr, "+ Queued backtrace from %s\n", oops->detail_filename);
+ queue_backtrace(oops);
+ }
- g_hash_table_insert(core_status.processing_oops, c2, c2);
- processing_queue[tail++] = fp;
- if (tail == 100)
- tail = 0;
+ free(fullpath);
+ fullpath = NULL;
+ }
+ closedir(dir);
+ fprintf(stderr, "+ End scanning %s...\n", processed_folder);
+ }
- pthread_mutex_unlock(&processing_queue_mtx);
- pthread_mutex_unlock(&core_status.processing_mtx);
- return 0;
-clean_add_to_processing:
- free(fp);
- free(c1);
- free(c2);
- return -1;
+ return NULL;
}
-/*
- * Entry for processing new core files.
- */
-static void process_corefile(char *fullpath)
+static void disable_corefiles(int diskfree)
{
- pthread_t thrd;
- int r = 1;
-
- r = add_to_processing(fullpath);
-
- if (r)
- return;
-
- if (pthread_create(&thrd, NULL, process_new, NULL))
- fprintf(stderr, "Couldn't start up gdb extract core thread\n");
+ int ret;
+ ret = system("echo \"\" > /proc/sys/kernel/core_pattern");
+ if (ret != -1) {
+ fprintf(stderr, "+ disabled core pattern, disk low %d%%\n", diskfree);
+ syslog(LOG_WARNING,
+ "Disabled kernel core_pattern, %s only has %d%% available",
+ core_folder, diskfree);
+ }
}
-/*
- * Entry for processing already seen core files.
- */
-static void reprocess_corefile(char *fullpath)
+void enable_corefiles(int diskfree)
{
- pthread_t thrd;
- int r = 0;
-
- r = add_to_processing(fullpath);
+ int ret;
+ char * proc_core_string = NULL;
+ ret = asprintf(&proc_core_string,
+ "echo \"%score_%%e_%%t\" > /proc/sys/kernel/core_pattern",
+ core_folder);
+ if (ret == -1)
+ goto err;
- if (r)
- return;
+ ret = system(proc_core_string);
+ free(proc_core_string);
+ if (ret == -1)
+ goto err;
- if (pthread_create(&thrd, NULL, process_old, NULL))
- fprintf(stderr, "Couldn't start up gdb extract core thread\n");
+ if (diskfree == -1) {
+ fprintf(stderr, "+ enabled core pattern\n");
+ syslog(LOG_INFO, "Enabled corewatcher kernel core_pattern\n");
+ } else {
+ fprintf(stderr, "+ reenabled core pattern, disk %d%%", diskfree);
+ syslog(LOG_WARNING,
+ "Reenabled corewatcher kernel core_pattern, %s now has %d%% available",
+ core_folder, diskfree);
+ }
+ return;
+err:
+ fprintf(stderr, "+ unable to enable core pattern\n");
+ syslog(LOG_WARNING, "Unable to enable kernel core_pattern\n");
+ return;
}
-int scan_corefolders(void __unused *unused)
+/* do everything, called from timer event */
+int scan_folders(void __unused *unused)
{
- DIR *dir = NULL;
- struct dirent *entry = NULL;
- char *fullpath = NULL, *appfile = NULL;
- char tmp_folder[] = "/tmp/";
- int r = 0;
-
- dir = opendir(tmp_folder);
- if (!dir)
- return 1;
+ struct statvfs stat;
+ int newdiskfree;
- fprintf(stderr, "+ scanning %s...\n", tmp_folder);
- while(1) {
- free(fullpath);
- fullpath = NULL;
-
- entry = readdir(dir);
- if (!entry || !entry->d_name)
- break;
- if (entry->d_name[0] == '.')
- continue;
- if (strncmp(entry->d_name, "core.", 5))
- continue;
+ if (statvfs(core_folder, &stat) == 0) {
+ newdiskfree = (int)(100 * stat.f_bavail / stat.f_blocks);
- /* matched core.#### where #### is the processes pid */
- r = asprintf(&fullpath, "%s%s", tmp_folder, entry->d_name);
- if (r == -1) {
- fullpath = NULL;
- continue;
- } else if (((unsigned int)r) != strlen(tmp_folder) + strlen(entry->d_name)) {
- continue;
- }
- /* already found, waiting for response from user */
- pthread_mutex_lock(&core_status.asked_mtx);
- if (g_hash_table_lookup(core_status.asked_oops, fullpath)) {
- pthread_mutex_unlock(&core_status.asked_mtx);
- continue;
- }
- pthread_mutex_unlock(&core_status.asked_mtx);
- fprintf(stderr, "+ Looking at %s\n", fullpath);
- appfile = get_appfile(fullpath);
+ openlog("corewatcher", 0, LOG_KERN);
+ if ((newdiskfree < 10) && (diskfree >= 10))
+ disable_corefiles(newdiskfree);
+ if ((newdiskfree > 12) && (diskfree <= 12))
+ enable_corefiles(newdiskfree);
+ closelog();
- if (!appfile) {
- unlink(fullpath);
- } else {
- free(appfile);
- appfile = NULL;
- }
+ diskfree = newdiskfree;
}
- closedir(dir);
-
- if (!core_folder)
- return 1;
- dir = opendir(core_folder);
- if (!dir)
- return 1;
-
- fprintf(stderr, "+ scanning %s...\n", core_folder);
- while(1) {
- free(fullpath);
- fullpath = NULL;
-
- entry = readdir(dir);
- if (!entry || !entry->d_name)
- break;
- if (entry->d_name[0] == '.')
- continue;
- if (!strstr(entry->d_name, "process"))
- continue;
-
- r = asprintf(&fullpath, "%s%s", core_folder, entry->d_name);
- if (r == -1) {
- fullpath = NULL;
- continue;
- } else if (((unsigned int)r) != strlen(core_folder) + strlen(entry->d_name)) {
- continue;
- }
- fprintf(stderr, "+ Looking at %s\n", fullpath);
- if (strstr(fullpath, "to-process"))
- process_corefile(fullpath);
- else
- reprocess_corefile(fullpath);
- }
- closedir(dir);
+ scan_core_folder(NULL);
- submit_queue();
+ g_mutex_lock(pq_mtx);
+ g_cond_signal(pq_work);
+ pq = TRUE;
+ g_mutex_unlock(pq_mtx);
- return 1;
+ return TRUE;
}