2 * Original author : tridge@samba.org, January 2002
4 * Copyright (c) 2005 Christophe Varoqui
5 * Copyright (c) 2005 Benjamin Marzinski, Redhat
9 * A simple domain socket listener
18 #include <sys/ioctl.h>
19 #include <sys/types.h>
20 #include <sys/socket.h>
26 #include <sys/inotify.h>
27 #include <sys/eventfd.h>
32 #include "structs_vec.h"
36 #include "mpath_cmd.h"
37 #include "time-util.h"
45 /* state of client connection */
55 struct list_head node;
56 struct timespec expires;
61 char cmd[_MAX_CMD_LEN + 1];
63 struct handler *handler;
69 /* Indices for array of poll fds */
77 #define POLLFD_CHUNK (4096 / sizeof(struct pollfd))
78 /* Minimum number of pollfds to reserve for clients */
79 #define MIN_POLLS (POLLFD_CHUNK - POLLFDS_BASE)
81 * Max number of client connections allowed
82 * During coldplug, there may be a large number of "multipath -u"
83 * processes connecting.
85 #define MAX_CLIENTS (16384 - POLLFDS_BASE)
87 /* Compile-time error if POLLFD_CHUNK is too small */
88 static __attribute__((unused)) char ___a[-(MIN_POLLS <= 0)];
90 static LIST_HEAD(clients);
91 static struct pollfd *polls;
92 static int notify_fd = -1;
93 static int idle_fd = -1;
94 static char *watch_config_dir;
96 static bool _socket_client_is_root(int fd)
101 len = sizeof(struct ucred);
103 (getsockopt(fd, SOL_SOCKET, SO_PEERCRED, &uc, &len) == 0) &&
107 /* Treat error as not root client */
112 * handle a new client joining
114 static void new_client(int ux_sock)
117 struct sockaddr addr;
118 socklen_t len = sizeof(addr);
121 fd = accept(ux_sock, &addr, &len);
126 c = (struct client *)calloc(1, sizeof(*c));
131 INIT_LIST_HEAD(&c->node);
134 c->is_root = _socket_client_is_root(c->fd);
136 /* put it in our linked list */
137 list_add_tail(&c->node, &clients);
141 * kill off a dead client
143 static void dead_client(struct client *c)
146 list_del_init(&c->node);
148 reset_strbuf(&c->reply);
150 free_keys(c->cmdvec);
155 static void free_polls (void)
162 void uxsock_cleanup(void *arg)
164 struct client *client_loop;
165 struct client *client_tmp;
166 long ux_sock = (long)arg;
170 free(watch_config_dir);
172 list_for_each_entry_safe(client_loop, client_tmp, &clients, node) {
173 dead_client(client_loop);
180 void wakeup_cleanup(void *arg)
182 struct mutex_lock *lck = arg;
186 set_wakeup_fn(lck, NULL);
191 struct watch_descriptors {
196 /* failing to set the watch descriptor is o.k. we just miss a warning
198 static void reset_watch(int notify_fd, struct watch_descriptors *wds,
199 unsigned int *sequence_nr)
208 conf = get_multipath_config();
209 /* instead of repeatedly try to reset the inotify watch if
210 * the config directory or multipath.conf isn't there, just
211 * do it once per reconfigure */
212 if (*sequence_nr != conf->sequence_nr) {
213 *sequence_nr = conf->sequence_nr;
214 if (wds->conf_wd == -1)
216 if (!watch_config_dir || !conf->config_dir ||
217 strcmp(watch_config_dir, conf->config_dir)) {
219 if (watch_config_dir)
220 free(watch_config_dir);
221 if (conf->config_dir)
222 watch_config_dir = strdup(conf->config_dir);
224 watch_config_dir = NULL;
225 } else if (wds->dir_wd == -1)
228 put_multipath_config(conf);
231 if (wds->dir_wd != -1) {
232 inotify_rm_watch(notify_fd, wds->dir_wd);
235 if (watch_config_dir) {
236 wds->dir_wd = inotify_add_watch(notify_fd,
239 IN_DELETE | IN_ONLYDIR);
240 if (wds->dir_wd == -1)
241 condlog(3, "didn't set up notifications on %s: %m", watch_config_dir);
245 wds->conf_wd = inotify_add_watch(notify_fd, DEFAULT_CONFIGFILE,
247 if (wds->conf_wd == -1)
248 condlog(3, "didn't set up notifications on /etc/multipath.conf: %m");
253 static void handle_inotify(int fd, struct watch_descriptors *wds)
256 __attribute__ ((aligned(__alignof__(struct inotify_event))));
257 const struct inotify_event *event;
263 len = read(fd, buff, sizeof(buff));
265 if (len < 0 && errno != EAGAIN) {
266 condlog(3, "error reading from inotify_fd");
267 if (wds->conf_wd != -1)
268 inotify_rm_watch(fd, wds->conf_wd);
269 if (wds->dir_wd != -1)
270 inotify_rm_watch(fd, wds->dir_wd);
271 wds->conf_wd = wds->dir_wd = -1;
277 for (ptr = buff; ptr < buff + len;
278 ptr += sizeof(struct inotify_event) + event->len) {
279 event = (const struct inotify_event *) ptr;
281 if (event->mask & IN_IGNORED) {
282 /* multipathd.conf may have been overwritten.
283 * Try once to reset the notification */
284 if (wds->conf_wd == event->wd)
285 wds->conf_wd = inotify_add_watch(notify_fd, DEFAULT_CONFIGFILE, IN_CLOSE_WRITE);
286 else if (wds->dir_wd == event->wd)
292 condlog(1, "Multipath configuration updated.\nReload multipathd for changes to take effect");
295 static const struct timespec ts_zero = { .tv_sec = 0, };
296 static const struct timespec ts_max = { .tv_sec = LONG_MAX, .tv_nsec = 999999999 };
298 static struct timespec *get_soonest_timeout(struct timespec *ts)
300 struct timespec ts_min = ts_max, now;
304 list_for_each_entry(c, &clients, node) {
305 if (timespeccmp(&c->expires, &ts_zero) != 0 &&
306 timespeccmp(&c->expires, &ts_min) < 0) {
315 get_monotonic_time(&now);
316 timespecsub(&ts_min, &now, ts);
317 if (timespeccmp(ts, &ts_zero) < 0)
320 condlog(4, "%s: next client expires in %ld.%03lds", __func__,
321 (long)ts->tv_sec, ts->tv_nsec / 1000000);
325 static bool need_vecs_lock(void)
329 list_for_each_entry(c, &clients, node) {
330 if (c->state == CLT_LOCKED_WORK)
336 static int parse_cmd(struct client *c)
340 r = get_cmdvec(c->cmd, &c->cmdvec);
345 c->handler = find_handler_for_cmdvec(c->cmdvec);
347 if (!c->handler || !c->handler->fn)
353 static int execute_handler(struct client *c, struct vectors *vecs)
356 if (!c->handler || !c->handler->fn)
359 return c->handler->fn(c->cmdvec, &c->reply, vecs);
362 static void wakeup_listener(void)
367 write(idle_fd, &one, sizeof(one)) != sizeof(one))
368 condlog(1, "%s: failed", __func__);
371 static void drain_idle_fd(int fd)
376 rc = read(fd, &val, sizeof(val));
377 condlog(4, "%s: %d, %"PRIu64, __func__, rc, val);
380 void default_reply(struct client *c, int r)
386 /* return codes from get_cmdvec() */
387 genhelp_handler(c->cmd, -r, &c->reply);
390 append_strbuf_str(&c->reply,
391 "permission deny: need to be root\n");
394 append_strbuf_str(&c->reply, "timeout\n");
397 append_strbuf_str(&c->reply, "ok\n");
400 /* cli_handler functions return 1 on unspecified error */
401 append_strbuf_str(&c->reply, "fail\n");
406 static void set_client_state(struct client *c, int state)
411 reset_strbuf(&c->reply);
412 memset(c->cmd, '\0', sizeof(c->cmd));
413 c->expires = ts_zero;
417 /* reuse these fields for next data transfer */
418 c->len = c->cmd_len = 0;
431 static int client_state_machine(struct client *c, struct vectors *vecs,
436 condlog(4, "%s: cli[%d] poll=%x state=%d cmd=\"%s\" repl \"%s\"", __func__,
437 c->fd, revents, c->state, c->cmd, get_strbuf_str(&c->reply));
441 if (!(revents & POLLIN))
443 if (c->cmd_len == 0) {
446 * We got POLLIN; assume that at least the length can
447 * be read immediately.
449 get_monotonic_time(&c->expires);
450 c->expires.tv_sec += uxsock_timeout / 1000;
451 c->expires.tv_nsec += (uxsock_timeout % 1000) * 1000000;
452 normalize_timespec(&c->expires);
453 n = recv(c->fd, &len, sizeof(len), 0);
454 if (n < (ssize_t)sizeof(len)) {
455 condlog(1, "%s: cli[%d]: failed to receive reply len: %zd",
457 c->error = -ECONNRESET;
458 } else if (len <= 0 || len > _MAX_CMD_LEN) {
459 condlog(1, "%s: cli[%d]: invalid command length (%zu bytes)",
460 __func__, c->fd, len);
461 c->error = -ECONNRESET;
464 condlog(4, "%s: cli[%d]: connected", __func__, c->fd);
468 } else if (c->len < c->cmd_len) {
469 n = recv(c->fd, c->cmd + c->len, c->cmd_len - c->len, 0);
470 if (n <= 0 && errno != EINTR && errno != EAGAIN) {
471 condlog(1, "%s: cli[%d]: error in recv: %m",
473 c->error = -ECONNRESET;
477 if (c->len < c->cmd_len)
478 /* continue polling */
481 condlog(4, "cli[%d]: Got request [%s]", c->fd, c->cmd);
482 set_client_state(c, CLT_PARSE);
486 c->error = parse_cmd(c);
488 /* Permission check */
489 struct key *kw = VECTOR_SLOT(c->cmdvec, 0);
491 if (!c->is_root && kw->code != LIST) {
493 condlog(0, "%s: cli[%d]: unauthorized cmd \"%s\"",
494 __func__, c->fd, c->cmd);
498 set_client_state(c, CLT_SEND);
499 else if (c->handler->locked)
500 set_client_state(c, CLT_LOCKED_WORK);
502 set_client_state(c, CLT_WORK);
505 case CLT_LOCKED_WORK:
506 if (trylock(&vecs->lock) == 0) {
507 /* don't use cleanup_lock(), lest we wakeup ourselves */
508 pthread_cleanup_push_cast(__unlock, &vecs->lock);
509 c->error = execute_handler(c, vecs);
510 pthread_cleanup_pop(1);
511 condlog(4, "%s: cli[%d] grabbed lock", __func__, c->fd);
512 free_keys(c->cmdvec);
514 set_client_state(c, CLT_SEND);
515 /* Wait for POLLOUT */
518 condlog(4, "%s: cli[%d] waiting for lock", __func__, c->fd);
523 c->error = execute_handler(c, vecs);
524 free_keys(c->cmdvec);
526 set_client_state(c, CLT_SEND);
527 /* Wait for POLLOUT */
531 if (get_strbuf_len(&c->reply) == 0)
532 default_reply(c, c->error);
534 if (c->cmd_len == 0) {
535 size_t len = get_strbuf_len(&c->reply) + 1;
537 if (send(c->fd, &len, sizeof(len), MSG_NOSIGNAL)
539 c->error = -ECONNRESET;
544 if (c->len < c->cmd_len) {
545 const char *buf = get_strbuf_str(&c->reply);
547 n = send(c->fd, buf + c->len, c->cmd_len, MSG_NOSIGNAL);
549 if (!(errno == EAGAIN || errno == EINTR))
550 c->error = -ECONNRESET;
555 if (c->len >= c->cmd_len) {
556 condlog(4, "cli[%d]: Reply [%zu bytes]", c->fd, c->cmd_len);
557 set_client_state(c, CLT_RECV);
566 static void check_timeout(struct client *c)
570 if (timespeccmp(&c->expires, &ts_zero) == 0)
573 get_monotonic_time(&now);
574 if (timespeccmp(&c->expires, &now) > 0)
577 condlog(2, "%s: cli[%d]: timed out at %ld.%03ld", __func__,
578 c->fd, (long)c->expires.tv_sec, c->expires.tv_nsec / 1000000);
580 c->error = -ETIMEDOUT;
581 set_client_state(c, CLT_SEND);
584 static void handle_client(struct client *c, struct vectors *vecs, short revents)
586 if (revents & (POLLHUP|POLLERR)) {
587 c->error = -ECONNRESET;
592 while (client_state_machine(c, vecs, revents) == STM_CONT);
598 void *uxsock_listen(long ux_sock, void *trigger_data)
601 int max_pfds = MIN_POLLS + POLLFDS_BASE;
602 /* conf->sequence_nr will be 1 when uxsock_listen is first called */
603 unsigned int sequence_nr = 0;
604 struct watch_descriptors wds = { .conf_wd = -1, .dir_wd = -1 };
605 struct vectors *vecs = trigger_data;
607 condlog(3, "uxsock: startup listener");
608 polls = calloc(1, max_pfds * sizeof(*polls));
610 condlog(0, "uxsock: failed to allocate poll fds");
614 notify_fd = inotify_init1(IN_NONBLOCK);
615 if (notify_fd == -1) /* it's fine if notifications fail */
616 condlog(3, "failed to start up configuration notifications");
618 pthread_cleanup_push(wakeup_cleanup, &vecs->lock);
619 idle_fd = eventfd(0, EFD_NONBLOCK|EFD_CLOEXEC);
621 condlog(1, "failed to create idle fd");
624 set_wakeup_fn(&vecs->lock, wakeup_listener);
627 sigdelset(&mask, SIGINT);
628 sigdelset(&mask, SIGTERM);
629 sigdelset(&mask, SIGHUP);
630 sigdelset(&mask, SIGUSR1);
632 struct client *c, *tmp;
633 int i, n_pfds, poll_count, num_clients;
634 struct timespec __timeout, *timeout;
636 /* setup for a poll */
638 list_for_each_entry(c, &clients, node) {
641 if (num_clients + POLLFDS_BASE > max_pfds) {
643 int n_new = max_pfds + POLLFD_CHUNK;
645 new = realloc(polls, n_new * sizeof(*polls));
650 condlog(1, "%s: realloc failure, %d clients not served",
652 num_clients + POLLFDS_BASE - max_pfds);
653 num_clients = max_pfds - POLLFDS_BASE;
656 if (num_clients < MAX_CLIENTS) {
657 polls[POLLFD_UX].fd = ux_sock;
658 polls[POLLFD_UX].events = POLLIN;
661 * New clients can't connect, num_clients won't grow
662 * to MAX_CLIENTS or higher
664 condlog(1, "%s: max client connections reached, pausing polling",
666 polls[POLLFD_UX].fd = -1;
669 reset_watch(notify_fd, &wds, &sequence_nr);
670 polls[POLLFD_NOTIFY].fd = notify_fd;
671 if (notify_fd == -1 || (wds.conf_wd == -1 && wds.dir_wd == -1))
672 polls[POLLFD_NOTIFY].events = 0;
674 polls[POLLFD_NOTIFY].events = POLLIN;
676 polls[POLLFD_IDLE].fd = idle_fd;
677 if (need_vecs_lock())
678 polls[POLLFD_IDLE].events = POLLIN;
680 polls[POLLFD_IDLE].events = 0;
682 /* setup the clients */
684 list_for_each_entry(c, &clients, node) {
687 polls[i].events = POLLIN;
690 polls[i].events = POLLOUT;
693 /* don't poll for this client */
702 timeout = get_soonest_timeout(&__timeout);
704 /* most of our life is spent in this call */
705 poll_count = ppoll(polls, n_pfds, timeout, &mask);
707 handle_signals(false);
708 if (poll_count == -1) {
709 if (errno == EINTR) {
710 handle_signals(true);
714 /* something went badly wrong! */
715 condlog(0, "uxsock: poll failed with %d", errno);
720 if (polls[POLLFD_IDLE].fd != -1 &&
721 polls[POLLFD_IDLE].revents & POLLIN)
722 drain_idle_fd(idle_fd);
724 /* see if a client needs handling */
725 list_for_each_entry_safe(c, tmp, &clients, node) {
728 for (i = POLLFDS_BASE; i < n_pfds; i++) {
729 if (polls[i].fd == c->fd) {
730 revents = polls[i].revents;
735 handle_client(c, trigger_data, revents);
737 if (c->error == -ECONNRESET) {
738 condlog(4, "cli[%d]: disconnected", c->fd);
744 /* see if we got a non-fatal signal */
745 handle_signals(true);
747 /* see if we got a new client */
748 if (polls[POLLFD_UX].revents & POLLIN) {
752 /* handle inotify events on config files */
753 if (polls[POLLFD_NOTIFY].revents & POLLIN)
754 handle_inotify(notify_fd, &wds);
757 pthread_cleanup_pop(1);