set pthread stack size to at least PTHREAD_STACK_MIN

Setting the stacksize too small just causes
pthread_attr_setstacksize() to fail, leaving you with the default stack
size. On some architectures, the default stacksize is large, like 10Mb.
Since you start one waiter thread per multipath device, every 100
devices eats up 1Gb of memory.

The other problem is that when I actually read the pthread_attr_init man
page (it can fail. who knew?), I saw that it can fail with ENOMEM. Also,
that it had a function to free it, and that the result of reinitializing
an attr that hadn't been freed was undefined.  Clearly, this function
wasn't intended to be called over and over without ever freeing the
attr, which is how we've been using it in multipathd. So, in the spirit
of writing code to the interface, instead of to how it appears to be
currently implemented, how about this.

diff --git a/libmultipath/log_pthread.c b/libmultipath/log_pthread.c
index 6997cff..96823a8 100644 (file)
--- a/libmultipath/log_pthread.c
+++ b/libmultipath/log_pthread.c
@@ -58,10 +58,8 @@ static void * log_thread (void * et)
        }
 }
 
-void log_thread_start (void)
+void log_thread_start (pthread_attr_t *attr)
 {
-       pthread_attr_t attr;
-
        logdbg(stderr,"enter log_thread_start\n");
 
        logq_lock = (pthread_mutex_t *) malloc(sizeof(pthread_mutex_t));
@@ -72,14 +70,11 @@ void log_thread_start (void)
        pthread_mutex_init(logev_lock, NULL);
        pthread_cond_init(logev_cond, NULL);
 
-       pthread_attr_init(&attr);
-       pthread_attr_setstacksize(&attr, 64 * 1024);
-
        if (log_init("multipathd", 0)) {
                fprintf(stderr,"can't initialize log buffer\n");
                exit(1);
        }
-       pthread_create(&log_thr, &attr, log_thread, NULL);
+       pthread_create(&log_thr, attr, log_thread, NULL);
 
        return;
 }

diff --git a/libmultipath/log_pthread.h b/libmultipath/log_pthread.h
index 2b18f59..77780d8 100644 (file)
--- a/libmultipath/log_pthread.h
+++ b/libmultipath/log_pthread.h
@@ -1,6 +1,8 @@
 #ifndef _LOG_PTHREAD_H
 #define _LOG_PTHREAD_H
 
+#include <pthread.h>
+
 pthread_t log_thr;
 
 pthread_mutex_t *logq_lock;
@@ -8,7 +10,7 @@ pthread_mutex_t *logev_lock;
 pthread_cond_t *logev_cond;
 
 void log_safe(int prio, const char * fmt, va_list ap);
-void log_thread_start(void);
+void log_thread_start(pthread_attr_t *attr);
 void log_thread_stop(void);
 
 #endif /* _LOG_PTHREAD_H */

diff --git a/libmultipath/waiter.c b/libmultipath/waiter.c
index 99bfdbf..4fb2cff 100644 (file)
--- a/libmultipath/waiter.c
+++ b/libmultipath/waiter.c
@@ -20,6 +20,8 @@
 #include "lock.h"
 #include "waiter.h"
 
+pthread_attr_t waiter_attr;
+
 struct event_thread *alloc_waiter (void)
 {
 
@@ -205,18 +207,11 @@ void *waitevent (void *et)
 
 int start_waiter_thread (struct multipath *mpp, struct vectors *vecs)
 {
-       pthread_attr_t attr;
        struct event_thread *wp;
 
        if (!mpp)
                return 0;
 
-       if (pthread_attr_init(&attr))
-               goto out;
-
-       pthread_attr_setstacksize(&attr, 32 * 1024);
-       pthread_attr_setdetachstate(&attr, PTHREAD_CREATE_DETACHED);
-
        wp = alloc_waiter();
 
        if (!wp)
@@ -227,7 +222,7 @@ int start_waiter_thread (struct multipath *mpp, struct vectors *vecs)
        wp->vecs = vecs;
        wp->mpp = mpp;
 
-       if (pthread_create(&wp->thread, &attr, waitevent, wp)) {
+       if (pthread_create(&wp->thread, &waiter_attr, waitevent, wp)) {
                condlog(0, "%s: cannot create event checker", wp->mapname);
                goto out1;
        }

diff --git a/libmultipath/waiter.h b/libmultipath/waiter.h
index 468ce5f..ab362d1 100644 (file)
--- a/libmultipath/waiter.h
+++ b/libmultipath/waiter.h
@@ -1,6 +1,8 @@
 #ifndef _WAITER_H
 #define _WAITER_H
 
+extern pthread_attr_t waiter_attr;
+
 struct event_thread {
        struct dm_task *dmt;
        pthread_t thread;

diff --git a/multipathd/main.c b/multipathd/main.c
index a5b61e6..f5e3e38 100644 (file)
--- a/multipathd/main.c
+++ b/multipathd/main.c
@@ -14,6 +14,7 @@
 #include <errno.h>
 #include <sys/time.h>
 #include <sys/resource.h>
+#include <limits.h>
 
 /*
  * libcheckers
@@ -1275,17 +1276,47 @@ set_oom_adj (int val)
        fclose(fp);
 }
 
+void
+setup_thread_attr(pthread_attr_t *attr, size_t stacksize, int detached)
+{
+       if (pthread_attr_init(attr)) {
+               fprintf(stderr, "can't initialize thread attr: %s\n",
+                       strerror(errno));
+               exit(1);
+       }
+       if (stacksize < PTHREAD_STACK_MIN)
+               stacksize = PTHREAD_STACK_MIN;
+
+       if (pthread_attr_setstacksize(attr, stacksize)) {
+               fprintf(stderr, "can't set thread stack size to %lu: %s\n",
+                       (unsigned long)stacksize, strerror(errno));
+               exit(1);
+       }
+       if (detached && pthread_attr_setdetachstate(attr,
+                                                   PTHREAD_CREATE_DETACHED)) {
+               fprintf(stderr, "can't set thread to detached: %s\n",
+                       strerror(errno));
+               exit(1);
+       }
+}
+
 static int
 child (void * param)
 {
        pthread_t check_thr, uevent_thr, uxlsnr_thr;
-       pthread_attr_t attr;
+       pthread_attr_t log_attr, misc_attr;
        struct vectors * vecs;
 
        mlockall(MCL_CURRENT | MCL_FUTURE);
 
-       if (logsink)
-               log_thread_start();
+       setup_thread_attr(&misc_attr, 64 * 1024, 1);
+       setup_thread_attr(&waiter_attr, 32 * 1024, 1);
+
+       if (logsink) {
+               setup_thread_attr(&log_attr, 64 * 1024, 0);
+               log_thread_start(&log_attr);
+               pthread_attr_destroy(&log_attr);
+       }
 
        condlog(2, "--------start up--------");
        condlog(2, "read " DEFAULT_CONFIGFILE);
@@ -1356,13 +1387,10 @@ child (void * param)
        /*
         * start threads
         */
-       pthread_attr_init(&attr);
-       pthread_attr_setstacksize(&attr, 64 * 1024);
-       pthread_attr_setdetachstate(&attr, PTHREAD_CREATE_DETACHED);
-
-       pthread_create(&check_thr, &attr, checkerloop, vecs);
-       pthread_create(&uevent_thr, &attr, ueventloop, vecs);
-       pthread_create(&uxlsnr_thr, &attr, uxlsnrloop, vecs);
+       pthread_create(&check_thr, &misc_attr, checkerloop, vecs);
+       pthread_create(&uevent_thr, &misc_attr, ueventloop, vecs);
+       pthread_create(&uxlsnr_thr, &misc_attr, uxlsnrloop, vecs);
+       pthread_attr_destroy(&misc_attr);
 
        pthread_cond_wait(&exit_cond, &exit_mutex);