2 * Copyright (c) 2004, 2005 Christophe Varoqui
3 * Copyright (c) 2005 Kiyoshi Ueda, NEC
4 * Copyright (c) 2005 Benjamin Marzinski, Redhat
5 * Copyright (c) 2005 Edward Goggin, EMC
9 #include <libdevmapper.h>
12 #include <sys/types.h>
16 #include <sys/resource.h>
18 #include <linux/oom.h>
19 #include <mpath_persist.h>
38 #include <blacklist.h>
39 #include <structs_vec.h>
41 #include <devmapper.h>
44 #include <discovery.h>
48 #include <switchgroup.h>
50 #include <configure.h>
52 #include <pgpolicies.h>
60 #include "cli_handlers.h"
64 #define FILE_NAME_SIZE 256
67 #define LOG_MSG(a, b) \
70 condlog(a, "%s: %s - path offline", pp->mpp->alias, pp->dev); \
72 condlog(a, "%s: %s - %s", pp->mpp->alias, pp->dev, b); \
75 struct mpath_event_param
78 struct multipath *mpp;
81 unsigned int mpath_mx_alloc_len;
83 pthread_cond_t exit_cond = PTHREAD_COND_INITIALIZER;
84 pthread_mutex_t exit_mutex = PTHREAD_MUTEX_INITIALIZER;
87 enum daemon_status running_state;
91 * global copy of vecs for use in sig handlers
93 struct vectors * gvecs;
96 need_switch_pathgroup (struct multipath * mpp, int refresh)
98 struct pathgroup * pgp;
102 if (!mpp || mpp->pgfailback == -FAILBACK_MANUAL)
106 * Refresh path priority values
109 vector_foreach_slot (mpp->pg, pgp, i)
110 vector_foreach_slot (pgp->paths, pp, j)
111 pathinfo(pp, conf->hwtable, DI_PRIO);
113 mpp->bestpg = select_path_group(mpp);
115 if (mpp->bestpg != mpp->nextpg)
122 switch_pathgroup (struct multipath * mpp)
124 mpp->stat_switchgroup++;
125 dm_switchgroup(mpp->alias, mpp->bestpg);
126 condlog(2, "%s: switch to path group #%i",
127 mpp->alias, mpp->bestpg);
131 coalesce_maps(struct vectors *vecs, vector nmpv)
133 struct multipath * ompp;
134 vector ompv = vecs->mpvec;
138 vector_foreach_slot (ompv, ompp, i) {
139 if (!find_mp_by_wwid(nmpv, ompp->wwid)) {
141 * remove all current maps not allowed by the
142 * current configuration
144 if (dm_flush_map(ompp->alias)) {
145 condlog(0, "%s: unable to flush devmap",
148 * may be just because the device is open
150 if (!vector_alloc_slot(nmpv))
153 vector_set_slot(nmpv, ompp);
154 setup_multipath(vecs, ompp);
156 if ((j = find_slot(ompv, (void *)ompp)) != -1)
157 vector_del_slot(ompv, j);
163 condlog(2, "%s devmap removed", ompp->alias);
165 } else if (conf->reassign_maps) {
166 condlog(3, "%s: Reassign existing device-mapper"
167 " devices", ompp->alias);
168 dm_reassign(ompp->alias);
175 sync_map_state(struct multipath *mpp)
177 struct pathgroup *pgp;
184 vector_foreach_slot (mpp->pg, pgp, i){
185 vector_foreach_slot (pgp->paths, pp, j){
186 if (pp->state == PATH_UNCHECKED ||
187 pp->state == PATH_WILD)
189 if ((pp->dmstate == PSTATE_FAILED ||
190 pp->dmstate == PSTATE_UNDEF) &&
191 (pp->state == PATH_UP || pp->state == PATH_GHOST))
192 dm_reinstate_path(mpp->alias, pp->dev_t);
193 else if ((pp->dmstate == PSTATE_ACTIVE ||
194 pp->dmstate == PSTATE_UNDEF) &&
195 (pp->state == PATH_DOWN ||
196 pp->state == PATH_SHAKY))
197 dm_fail_path(mpp->alias, pp->dev_t);
203 sync_maps_state(vector mpvec)
206 struct multipath *mpp;
208 vector_foreach_slot (mpvec, mpp, i)
213 flush_map(struct multipath * mpp, struct vectors * vecs)
216 * clear references to this map before flushing so we can ignore
217 * the spurious uevent we may generate with the dm_flush_map call below
219 if (dm_flush_map(mpp->alias)) {
221 * May not really be an error -- if the map was already flushed
222 * from the device mapper by dmsetup(8) for instance.
224 condlog(0, "%s: can't flush", mpp->alias);
229 condlog(2, "%s: devmap removed", mpp->alias);
232 orphan_paths(vecs->pathvec, mpp);
233 remove_map_and_stop_waiter(mpp, vecs, 1);
239 uev_add_map (struct uevent * uev, struct vectors * vecs)
242 int major = -1, minor = -1, rc;
244 condlog(3, "%s: add map (uevent)", uev->kernel);
245 alias = uevent_get_dm_name(uev);
247 condlog(3, "%s: No DM_NAME in uevent", uev->kernel);
248 major = uevent_get_major(uev);
249 minor = uevent_get_minor(uev);
250 alias = dm_mapname(major, minor);
252 condlog(2, "%s: mapname not found for %d:%d",
253 uev->kernel, major, minor);
257 rc = ev_add_map(uev->kernel, alias, vecs);
263 ev_add_map (char * dev, char * alias, struct vectors * vecs)
266 struct multipath * mpp;
270 map_present = dm_map_present(alias);
272 if (map_present && dm_type(alias, TGT_MPATH) <= 0) {
273 condlog(4, "%s: not a multipath map", alias);
277 mpp = find_mp_by_alias(vecs->mpvec, alias);
281 * Not really an error -- we generate our own uevent
282 * if we create a multipath mapped device as a result
285 if (conf->reassign_maps) {
286 condlog(3, "%s: Reassign existing device-mapper devices",
292 condlog(2, "%s: adding map", alias);
295 * now we can register the map
297 if (map_present && (mpp = add_map_without_path(vecs, alias))) {
299 condlog(2, "%s: devmap %s registered", alias, dev);
302 refwwid = get_refwwid(dev, DEV_DEVMAP, vecs->pathvec);
305 r = coalesce_paths(vecs, NULL, refwwid, 0);
310 condlog(2, "%s: devmap %s added", alias, dev);
312 condlog(0, "%s: uev_add_map %s failed", alias, dev);
319 uev_remove_map (struct uevent * uev, struct vectors * vecs)
323 struct multipath *mpp;
325 condlog(2, "%s: remove map (uevent)", uev->kernel);
326 alias = uevent_get_dm_name(uev);
328 condlog(3, "%s: No DM_NAME in uevent, ignoring", uev->kernel);
331 minor = uevent_get_minor(uev);
332 mpp = find_mp_by_minor(vecs->mpvec, minor);
335 condlog(2, "%s: devmap not registered, can't remove",
339 if (strcmp(mpp->alias, alias)) {
340 condlog(2, "%s: minor number mismatch (map %d, event %d)",
341 mpp->alias, mpp->dmi->minor, minor);
345 orphan_paths(vecs->pathvec, mpp);
346 remove_map_and_stop_waiter(mpp, vecs, 1);
353 ev_remove_map (char * devname, char * alias, int minor, struct vectors * vecs)
355 struct multipath * mpp;
357 mpp = find_mp_by_minor(vecs->mpvec, minor);
360 condlog(2, "%s: devmap not registered, can't remove",
364 if (strcmp(mpp->alias, alias)) {
365 condlog(2, "%s: minor number mismatch (map %d, event %d)",
366 mpp->alias, mpp->dmi->minor, minor);
369 return flush_map(mpp, vecs);
373 uev_add_path (struct uevent *uev, struct vectors * vecs)
375 struct sysfs_device * dev;
377 dev = sysfs_device_get(uev->devpath);
379 condlog(2, "%s: not found in sysfs", uev->devpath);
382 condlog(2, "%s: add path (uevent)", dev->kernel);
383 return (ev_add_path(dev->kernel, vecs) != 1)? 0 : 1;
393 ev_add_path (char * devname, struct vectors * vecs)
395 struct multipath * mpp;
397 char empty_buff[WWID_SIZE] = {0};
398 char params[PARAMS_SIZE] = {0};
400 int start_waiter = 0;
402 if (strstr(devname, "..") != NULL) {
404 * Don't allow relative device names in the pathvec
406 condlog(0, "%s: path name is invalid", devname);
410 pp = find_path_by_dev(vecs->pathvec, devname);
413 condlog(0, "%s: spurious uevent, path already in pathvec",
420 * get path vital state
422 if (!(pp = store_pathinfo(vecs->pathvec, conf->hwtable,
424 condlog(0, "%s: failed to store path info", devname);
427 pp->checkint = conf->checkint;
431 * need path UID to go any further
433 if (memcmp(empty_buff, pp->wwid, WWID_SIZE) == 0) {
434 condlog(0, "%s: failed to get path uid", devname);
435 goto fail; /* leave path added to pathvec */
437 if (filter_path(conf, pp) > 0){
438 int i = find_slot(vecs->pathvec, (void *)pp);
440 vector_del_slot(vecs->pathvec, i);
444 mpp = pp->mpp = find_mp_by_wwid(vecs->mpvec, pp->wwid);
447 if ((!pp->size) || (mpp->size != pp->size)) {
449 condlog(0, "%s: failed to add new path %s, "
453 condlog(0, "%s: failed to add new path %s, "
454 "device size mismatch",
456 int i = find_slot(vecs->pathvec, (void *)pp);
458 vector_del_slot(vecs->pathvec, i);
463 condlog(4,"%s: adopting all paths for path %s",
464 mpp->alias, pp->dev);
465 if (adopt_paths(vecs->pathvec, mpp, 1))
466 goto fail; /* leave path added to pathvec */
468 verify_paths(mpp, vecs, NULL);
469 mpp->flush_on_last_del = FLUSH_UNDEF;
470 mpp->action = ACT_RELOAD;
474 condlog(0, "%s: failed to create new map,"
475 " %s device size is 0 ", devname, pp->dev);
476 int i = find_slot(vecs->pathvec, (void *)pp);
478 vector_del_slot(vecs->pathvec, i);
483 condlog(4,"%s: creating new map", pp->dev);
484 if ((mpp = add_map_with_path(vecs, pp, 1))) {
485 mpp->action = ACT_CREATE;
487 * We don't depend on ACT_CREATE, as domap will
488 * set it to ACT_NOTHING when complete.
493 goto fail; /* leave path added to pathvec */
496 /* persistent reseravtion check*/
497 mpath_pr_event_handle(pp);
500 * push the map to the device-mapper
502 if (setup_map(mpp, params, PARAMS_SIZE)) {
503 condlog(0, "%s: failed to setup map for addition of new "
504 "path %s", mpp->alias, devname);
508 * reload the map for the multipath mapped device
510 if (domap(mpp, params) <= 0) {
511 condlog(0, "%s: failed in domap for addition of new "
512 "path %s", mpp->alias, devname);
514 * deal with asynchronous uevents :((
516 if (mpp->action == ACT_RELOAD && retries-- > 0) {
517 condlog(0, "%s: uev_add_path sleep", mpp->alias);
519 update_mpp_paths(mpp, vecs->pathvec);
522 else if (mpp->action == ACT_RELOAD)
523 condlog(0, "%s: giving up reload", mpp->alias);
530 * update our state from kernel regardless of create or reload
532 if (setup_multipath(vecs, mpp))
537 if ((mpp->action == ACT_CREATE ||
538 (mpp->action == ACT_NOTHING && start_waiter && !mpp->waiter)) &&
539 start_waiter_thread(mpp, vecs))
543 condlog(2, "%s path added to devmap %s", devname, mpp->alias);
550 remove_map(mpp, vecs, 1);
557 uev_remove_path (struct uevent *uev, struct vectors * vecs)
559 struct sysfs_device * dev;
562 dev = sysfs_device_get(uev->devpath);
564 condlog(2, "%s: not found in sysfs", uev->devpath);
567 condlog(2, "%s: remove path (uevent)", uev->kernel);
568 retval = ev_remove_path(uev->kernel, vecs);
571 sysfs_device_put(dev);
577 ev_remove_path (char * devname, struct vectors * vecs)
579 struct multipath * mpp;
582 char params[PARAMS_SIZE] = {0};
584 pp = find_path_by_dev(vecs->pathvec, devname);
587 /* Not an error; path might have been purged earlier */
588 condlog(0, "%s: path already removed", devname);
593 * avoid referring to the map of an orphaned path
595 if ((mpp = pp->mpp)) {
597 * transform the mp->pg vector of vectors of paths
598 * into a mp->params string to feed the device-mapper
600 if (update_mpp_paths(mpp, vecs->pathvec)) {
601 condlog(0, "%s: failed to update paths",
605 if ((i = find_slot(mpp->paths, (void *)pp)) != -1)
606 vector_del_slot(mpp->paths, i);
609 * remove the map IFF removing the last path
611 if (VECTOR_SIZE(mpp->paths) == 0) {
612 char alias[WWID_SIZE];
615 * flush_map will fail if the device is open
617 strncpy(alias, mpp->alias, WWID_SIZE);
618 if (mpp->flush_on_last_del == FLUSH_ENABLED) {
619 condlog(2, "%s Last path deleted, disabling queueing", mpp->alias);
621 mpp->no_path_retry = NO_PATH_RETRY_FAIL;
622 mpp->flush_on_last_del = FLUSH_IN_PROGRESS;
623 dm_queue_if_no_path(mpp->alias, 0);
625 if (!flush_map(mpp, vecs)) {
626 condlog(2, "%s: removed map after"
627 " removing all paths",
633 * Not an error, continue
637 if (setup_map(mpp, params, PARAMS_SIZE)) {
638 condlog(0, "%s: failed to setup map for"
639 " removal of path %s", mpp->alias,
646 mpp->action = ACT_RELOAD;
647 if (domap(mpp, params) <= 0) {
648 condlog(0, "%s: failed in domap for "
649 "removal of path %s",
650 mpp->alias, devname);
654 * update our state from kernel
656 if (setup_multipath(vecs, mpp)) {
661 condlog(2, "%s: path removed from map %s",
662 devname, mpp->alias);
667 if ((i = find_slot(vecs->pathvec, (void *)pp)) != -1)
668 vector_del_slot(vecs->pathvec, i);
675 remove_map_and_stop_waiter(mpp, vecs, 1);
680 uev_update_path (struct uevent *uev, struct vectors * vecs)
682 struct sysfs_device * dev;
685 dev = sysfs_device_get(uev->devpath);
687 condlog(2, "%s: not found in sysfs", uev->devpath);
690 ro = uevent_get_disk_ro(uev);
695 condlog(2, "%s: update path write_protect to '%d' (uevent)",
697 pp = find_path_by_dev(vecs->pathvec, uev->kernel);
699 condlog(0, "%s: spurious uevent, path not found",
704 retval = reload_map(vecs, pp->mpp);
706 condlog(2, "%s: map %s reloaded (retval %d)",
707 uev->kernel, pp->mpp->alias, retval);
711 sysfs_device_put(dev);
717 map_discovery (struct vectors * vecs)
719 struct multipath * mpp;
722 if (dm_get_maps(vecs->mpvec))
725 vector_foreach_slot (vecs->mpvec, mpp, i)
726 if (setup_multipath(vecs, mpp))
733 uxsock_trigger (char * str, char ** reply, int * len, void * trigger_data)
735 struct vectors * vecs;
740 vecs = (struct vectors *)trigger_data;
742 pthread_cleanup_push(cleanup_lock, &vecs->lock);
745 r = parse_cmd(str, reply, len, vecs);
748 *reply = STRDUP("fail\n");
749 *len = strlen(*reply) + 1;
752 else if (!r && *len == 0) {
753 *reply = STRDUP("ok\n");
754 *len = strlen(*reply) + 1;
757 /* else if (r < 0) leave *reply alone */
759 lock_cleanup_pop(vecs->lock);
764 uev_discard(char * devpath)
770 * keep only block devices, discard partitions
772 tmp = strstr(devpath, "/block/");
774 condlog(4, "no /block/ in '%s'", devpath);
777 if (sscanf(tmp, "/block/%10s", a) != 1 ||
778 sscanf(tmp, "/block/%10[^/]/%10s", a, b) == 2) {
779 condlog(4, "discard event on %s", devpath);
786 uev_trigger (struct uevent * uev, void * trigger_data)
789 struct vectors * vecs;
791 vecs = (struct vectors *)trigger_data;
793 if (uev_discard(uev->devpath))
800 * Add events are ignored here as the tables
801 * are not fully initialised then.
803 if (!strncmp(uev->kernel, "dm-", 3)) {
804 if (!strncmp(uev->action, "change", 6)) {
805 r = uev_add_map(uev, vecs);
808 if (!strncmp(uev->action, "remove", 6)) {
809 r = uev_remove_map(uev, vecs);
816 * path add/remove event
818 if (filter_devnode(conf->blist_devnode, conf->elist_devnode,
822 if (!strncmp(uev->action, "add", 3)) {
823 r = uev_add_path(uev, vecs);
826 if (!strncmp(uev->action, "remove", 6)) {
827 r = uev_remove_path(uev, vecs);
830 if (!strncmp(uev->action, "change", 6)) {
831 r = uev_update_path(uev, vecs);
841 ueventloop (void * ap)
843 block_signal(SIGUSR1, NULL);
844 block_signal(SIGHUP, NULL);
847 condlog(0, "error starting uevent listener");
855 block_signal(SIGUSR1, NULL);
856 block_signal(SIGHUP, NULL);
858 if (uevent_dispatch(&uev_trigger, ap))
859 condlog(0, "error starting uevent dispatcher");
864 uxlsnrloop (void * ap)
866 block_signal(SIGUSR1, NULL);
867 block_signal(SIGHUP, NULL);
872 set_handler_callback(LIST+PATHS, cli_list_paths);
873 set_handler_callback(LIST+PATHS+FMT, cli_list_paths_fmt);
874 set_handler_callback(LIST+MAPS, cli_list_maps);
875 set_handler_callback(LIST+STATUS, cli_list_status);
876 set_handler_callback(LIST+DAEMON, cli_list_daemon);
877 set_handler_callback(LIST+MAPS+STATUS, cli_list_maps_status);
878 set_handler_callback(LIST+MAPS+STATS, cli_list_maps_stats);
879 set_handler_callback(LIST+MAPS+FMT, cli_list_maps_fmt);
880 set_handler_callback(LIST+MAPS+TOPOLOGY, cli_list_maps_topology);
881 set_handler_callback(LIST+TOPOLOGY, cli_list_maps_topology);
882 set_handler_callback(LIST+MAP+TOPOLOGY, cli_list_map_topology);
883 set_handler_callback(LIST+CONFIG, cli_list_config);
884 set_handler_callback(LIST+BLACKLIST, cli_list_blacklist);
885 set_handler_callback(LIST+DEVICES, cli_list_devices);
886 set_handler_callback(LIST+WILDCARDS, cli_list_wildcards);
887 set_handler_callback(ADD+PATH, cli_add_path);
888 set_handler_callback(DEL+PATH, cli_del_path);
889 set_handler_callback(ADD+MAP, cli_add_map);
890 set_handler_callback(DEL+MAP, cli_del_map);
891 set_handler_callback(SWITCH+MAP+GROUP, cli_switch_group);
892 set_handler_callback(RECONFIGURE, cli_reconfigure);
893 set_handler_callback(SUSPEND+MAP, cli_suspend);
894 set_handler_callback(RESUME+MAP, cli_resume);
895 set_handler_callback(RESIZE+MAP, cli_resize);
896 set_handler_callback(RELOAD+MAP, cli_reload);
897 set_handler_callback(RESET+MAP, cli_reassign);
898 set_handler_callback(REINSTATE+PATH, cli_reinstate);
899 set_handler_callback(FAIL+PATH, cli_fail);
900 set_handler_callback(DISABLEQ+MAP, cli_disable_queueing);
901 set_handler_callback(RESTOREQ+MAP, cli_restore_queueing);
902 set_handler_callback(DISABLEQ+MAPS, cli_disable_all_queueing);
903 set_handler_callback(RESTOREQ+MAPS, cli_restore_all_queueing);
904 set_handler_callback(QUIT, cli_quit);
905 set_handler_callback(SHUTDOWN, cli_shutdown);
906 set_handler_callback(GETPRSTATUS+MAP, cli_getprstatus);
907 set_handler_callback(SETPRSTATUS+MAP, cli_setprstatus);
908 set_handler_callback(UNSETPRSTATUS+MAP, cli_unsetprstatus);
911 uxsock_listen(&uxsock_trigger, ap);
917 exit_daemon (int status)
920 fprintf(stderr, "bad exit status. see daemon.log\n");
922 condlog(3, "unlink pidfile");
923 unlink(DEFAULT_PIDFILE);
925 pthread_mutex_lock(&exit_mutex);
926 pthread_cond_signal(&exit_cond);
927 pthread_mutex_unlock(&exit_mutex);
935 switch (running_state) {
940 case DAEMON_CONFIGURE:
944 case DAEMON_SHUTDOWN:
951 fail_path (struct path * pp, int del_active)
956 condlog(2, "checker failed path %s in map %s",
957 pp->dev_t, pp->mpp->alias);
959 dm_fail_path(pp->mpp->alias, pp->dev_t);
961 update_queue_mode_del_path(pp->mpp);
965 * caller must have locked the path list before calling that function
968 reinstate_path (struct path * pp, int add_active)
973 if (dm_reinstate_path(pp->mpp->alias, pp->dev_t))
974 condlog(0, "%s: reinstate failed", pp->dev_t);
976 condlog(2, "%s: reinstated", pp->dev_t);
978 update_queue_mode_add_path(pp->mpp);
983 enable_group(struct path * pp)
985 struct pathgroup * pgp;
988 * if path is added through uev_add_path, pgindex can be unset.
989 * next update_strings() will set it, upon map reload event.
991 * we can safely return here, because upon map reload, all
992 * PG will be enabled.
994 if (!pp->mpp->pg || !pp->pgindex)
997 pgp = VECTOR_SLOT(pp->mpp->pg, pp->pgindex - 1);
999 if (pgp->status == PGSTATE_DISABLED) {
1000 condlog(2, "%s: enable group #%i", pp->mpp->alias, pp->pgindex);
1001 dm_enablegroup(pp->mpp->alias, pp->pgindex);
1006 mpvec_garbage_collector (struct vectors * vecs)
1008 struct multipath * mpp;
1014 vector_foreach_slot (vecs->mpvec, mpp, i) {
1015 if (mpp && mpp->alias && !dm_map_present(mpp->alias)) {
1016 condlog(2, "%s: remove dead map", mpp->alias);
1017 remove_map_and_stop_waiter(mpp, vecs, 1);
1024 defered_failback_tick (vector mpvec)
1026 struct multipath * mpp;
1029 vector_foreach_slot (mpvec, mpp, i) {
1031 * defered failback getting sooner
1033 if (mpp->pgfailback > 0 && mpp->failback_tick > 0) {
1034 mpp->failback_tick--;
1036 if (!mpp->failback_tick && need_switch_pathgroup(mpp, 1))
1037 switch_pathgroup(mpp);
1043 retry_count_tick(vector mpvec)
1045 struct multipath *mpp;
1048 vector_foreach_slot (mpvec, mpp, i) {
1049 if (mpp->retry_tick) {
1050 mpp->stat_total_queueing_time++;
1051 condlog(4, "%s: Retrying.. No active path", mpp->alias);
1052 if(--mpp->retry_tick == 0) {
1053 dm_queue_if_no_path(mpp->alias, 0);
1054 condlog(2, "%s: Disable queueing", mpp->alias);
1060 int update_prio(struct path *pp, int refresh_all)
1064 struct pathgroup * pgp;
1065 int i, j, changed = 0;
1068 vector_foreach_slot (pp->mpp->pg, pgp, i) {
1069 vector_foreach_slot (pgp->paths, pp1, j) {
1070 oldpriority = pp1->priority;
1071 pathinfo(pp1, conf->hwtable, DI_PRIO);
1072 if (pp1->priority != oldpriority)
1078 oldpriority = pp->priority;
1079 pathinfo(pp, conf->hwtable, DI_PRIO);
1081 if (pp->priority == oldpriority)
1086 int update_path_groups(struct multipath *mpp, struct vectors *vecs, int refresh)
1090 char params[PARAMS_SIZE];
1092 update_mpp_paths(mpp, vecs->pathvec);
1094 vector_foreach_slot (mpp->paths, pp, i)
1095 pathinfo(pp, conf->hwtable, DI_PRIO);
1098 if (setup_map(mpp, params, PARAMS_SIZE))
1101 mpp->action = ACT_RELOAD;
1102 if (domap(mpp, params) <= 0) {
1103 condlog(0, "%s: failed to update map : %s", mpp->alias,
1108 if (setup_multipath(vecs, mpp) != 0)
1110 sync_map_state(mpp);
1116 check_path (struct vectors * vecs, struct path * pp)
1119 int new_path_up = 0;
1124 if (pp->tick && --pp->tick)
1125 return; /* don't check this path yet */
1128 * provision a next check soonest,
1129 * in case we exit abnormaly from here
1131 pp->tick = conf->checkint;
1133 newstate = path_offline(pp);
1134 if (newstate == PATH_UP)
1135 newstate = get_state(pp, 1);
1137 if (newstate == PATH_WILD || newstate == PATH_UNCHECKED) {
1138 condlog(2, "%s: unusable path", pp->dev);
1139 pathinfo(pp, conf->hwtable, 0);
1143 * Async IO in flight. Keep the previous path state
1144 * and reschedule as soon as possible
1146 if (newstate == PATH_PENDING) {
1151 * Synchronize with kernel state
1153 if (update_multipath_strings(pp->mpp, vecs->pathvec)) {
1154 condlog(1, "%s: Could not synchronize with kernel state\n",
1156 pp->dmstate = PSTATE_UNDEF;
1158 if (newstate != pp->state) {
1159 int oldstate = pp->state;
1160 pp->state = newstate;
1161 LOG_MSG(1, checker_message(&pp->checker));
1164 * upon state change, reset the checkint
1165 * to the shortest delay
1167 pp->checkint = conf->checkint;
1169 if (newstate == PATH_DOWN || newstate == PATH_SHAKY) {
1171 * proactively fail path in the DM
1173 if (oldstate == PATH_UP ||
1174 oldstate == PATH_GHOST)
1180 * cancel scheduled failback
1182 pp->mpp->failback_tick = 0;
1184 pp->mpp->stat_path_failures++;
1188 if(newstate == PATH_UP || newstate == PATH_GHOST){
1189 if ( pp->mpp && pp->mpp->prflag ){
1191 * Check Persistent Reservation.
1193 condlog(2, "%s: checking persistent reservation "
1194 "registration", pp->dev);
1195 mpath_pr_event_handle(pp);
1200 * reinstate this path
1202 if (oldstate != PATH_UP &&
1203 oldstate != PATH_GHOST)
1204 reinstate_path(pp, 1);
1206 reinstate_path(pp, 0);
1211 * if at least one path is up in a group, and
1212 * the group is disabled, re-enable it
1214 if (newstate == PATH_UP)
1217 else if (newstate == PATH_UP || newstate == PATH_GHOST) {
1218 if (pp->dmstate == PSTATE_FAILED ||
1219 pp->dmstate == PSTATE_UNDEF) {
1220 /* Clear IO errors */
1221 reinstate_path(pp, 0);
1223 LOG_MSG(4, checker_message(&pp->checker));
1225 * double the next check delay.
1226 * max at conf->max_checkint
1228 if (pp->checkint < (conf->max_checkint / 2))
1229 pp->checkint = 2 * pp->checkint;
1231 pp->checkint = conf->max_checkint;
1233 pp->tick = pp->checkint;
1234 condlog(4, "%s: delay next check %is",
1235 pp->dev_t, pp->tick);
1238 else if (newstate == PATH_DOWN) {
1239 if (conf->log_checker_err == LOG_CHKR_ERR_ONCE)
1240 LOG_MSG(3, checker_message(&pp->checker));
1242 LOG_MSG(2, checker_message(&pp->checker));
1245 pp->state = newstate;
1248 * path prio refreshing
1250 condlog(4, "path prio refresh");
1252 if (update_prio(pp, new_path_up) &&
1253 (pp->mpp->pgpolicyfn == (pgpolicyfn *)group_by_prio) &&
1254 pp->mpp->pgfailback == -FAILBACK_IMMEDIATE)
1255 update_path_groups(pp->mpp, vecs, !new_path_up);
1256 else if (need_switch_pathgroup(pp->mpp, 0)) {
1257 if (pp->mpp->pgfailback > 0 &&
1258 (new_path_up || pp->mpp->failback_tick <= 0))
1259 pp->mpp->failback_tick =
1260 pp->mpp->pgfailback + 1;
1261 else if (pp->mpp->pgfailback == -FAILBACK_IMMEDIATE)
1262 switch_pathgroup(pp->mpp);
1267 checkerloop (void *ap)
1269 struct vectors *vecs;
1275 mlockall(MCL_CURRENT | MCL_FUTURE);
1276 vecs = (struct vectors *)ap;
1277 condlog(2, "path checkers start up");
1280 * init the path check interval
1282 vector_foreach_slot (vecs->pathvec, pp, i) {
1283 pp->checkint = conf->checkint;
1287 block_signal(SIGHUP, &old);
1288 pthread_cleanup_push(cleanup_lock, &vecs->lock);
1292 if (vecs->pathvec) {
1293 vector_foreach_slot (vecs->pathvec, pp, i) {
1294 check_path(vecs, pp);
1298 defered_failback_tick(vecs->mpvec);
1299 retry_count_tick(vecs->mpvec);
1304 condlog(4, "map garbage collection");
1305 mpvec_garbage_collector(vecs);
1309 lock_cleanup_pop(vecs->lock);
1310 pthread_sigmask(SIG_SETMASK, &old, NULL);
1317 configure (struct vectors * vecs, int start_waiters)
1319 struct multipath * mpp;
1324 if (!vecs->pathvec && !(vecs->pathvec = vector_alloc()))
1327 if (!vecs->mpvec && !(vecs->mpvec = vector_alloc()))
1330 if (!(mpvec = vector_alloc()))
1334 * probe for current path (from sysfs) and map (from dm) sets
1336 path_discovery(vecs->pathvec, conf, DI_ALL);
1338 vector_foreach_slot (vecs->pathvec, pp, i){
1339 if (filter_path(conf, pp) > 0){
1340 vector_del_slot(vecs->pathvec, i);
1345 pp->checkint = conf->checkint;
1347 if (map_discovery(vecs))
1351 * create new set of maps & push changed ones into dm
1353 if (coalesce_paths(vecs, mpvec, NULL, 1))
1357 * may need to remove some maps which are no longer relevant
1358 * e.g., due to blacklist changes in conf file
1360 if (coalesce_maps(vecs, mpvec))
1365 sync_maps_state(mpvec);
1366 vector_foreach_slot(mpvec, mpp, i){
1371 * purge dm of old maps
1376 * save new set of maps formed by considering current path state
1378 vector_free(vecs->mpvec);
1379 vecs->mpvec = mpvec;
1382 * start dm event waiter threads for these new maps
1384 vector_foreach_slot(vecs->mpvec, mpp, i) {
1385 if (setup_multipath(vecs, mpp))
1388 if (start_waiter_thread(mpp, vecs))
1395 reconfigure (struct vectors * vecs)
1397 struct config * old = conf;
1401 * free old map and path vectors ... they use old conf state
1403 if (VECTOR_SIZE(vecs->mpvec))
1404 remove_maps_and_stop_waiters(vecs);
1406 if (VECTOR_SIZE(vecs->pathvec))
1407 free_pathvec(vecs->pathvec, FREE_PATHS);
1409 vecs->pathvec = NULL;
1412 if (!load_config(DEFAULT_CONFIGFILE)) {
1413 conf->verbosity = old->verbosity;
1423 static struct vectors *
1426 struct vectors * vecs;
1428 vecs = (struct vectors *)MALLOC(sizeof(struct vectors));
1434 (pthread_mutex_t *)MALLOC(sizeof(pthread_mutex_t));
1436 if (!vecs->lock.mutex)
1439 pthread_mutex_init(vecs->lock.mutex, NULL);
1440 vecs->lock.depth = 0;
1446 condlog(0, "failed to init paths");
1451 signal_set(int signo, void (*func) (int))
1454 struct sigaction sig;
1455 struct sigaction osig;
1457 sig.sa_handler = func;
1458 sigemptyset(&sig.sa_mask);
1461 r = sigaction(signo, &sig, &osig);
1466 return (osig.sa_handler);
1472 condlog(2, "reconfigure (SIGHUP)");
1474 if (running_state != DAEMON_RUNNING)
1479 unlock(gvecs->lock);
1482 dbg_free_final(NULL);
1495 condlog(3, "SIGUSR1 received");
1501 signal_set(SIGHUP, sighup);
1502 signal_set(SIGUSR1, sigusr1);
1503 signal_set(SIGINT, sigend);
1504 signal_set(SIGTERM, sigend);
1505 signal(SIGPIPE, SIG_IGN);
1512 static struct sched_param sched_param = {
1513 .sched_priority = 99
1516 res = sched_setscheduler (0, SCHED_RR, &sched_param);
1519 condlog(LOG_WARNING, "Could not set SCHED_RR at priority 99");
1527 char *file = "/proc/self/oom_score_adj";
1528 int score = OOM_SCORE_ADJ_MIN;
1533 if (stat(file, &st) == 0){
1534 fp = fopen(file, "w");
1536 condlog(0, "couldn't fopen %s : %s", file,
1540 fprintf(fp, "%i", score);
1544 if (errno != ENOENT) {
1545 condlog(0, "couldn't stat %s : %s", file,
1549 file = "/proc/self/oom_adj";
1550 score = OOM_ADJUST_MIN;
1552 condlog(0, "couldn't adjust oom score");
1556 child (void * param)
1558 pthread_t check_thr, uevent_thr, uxlsnr_thr, uevq_thr;
1559 pthread_attr_t log_attr, misc_attr;
1560 struct vectors * vecs;
1561 struct multipath * mpp;
1565 mlockall(MCL_CURRENT | MCL_FUTURE);
1567 setup_thread_attr(&misc_attr, 64 * 1024, 1);
1568 setup_thread_attr(&waiter_attr, 32 * 1024, 1);
1571 setup_thread_attr(&log_attr, 64 * 1024, 0);
1572 log_thread_start(&log_attr);
1573 pthread_attr_destroy(&log_attr);
1576 running_state = DAEMON_START;
1578 condlog(2, "--------start up--------");
1579 condlog(2, "read " DEFAULT_CONFIGFILE);
1581 if (load_config(DEFAULT_CONFIGFILE))
1584 if (init_checkers()) {
1585 condlog(0, "failed to initialize checkers");
1589 condlog(0, "failed to initialize prioritizers");
1593 setlogmask(LOG_UPTO(conf->verbosity + 3));
1595 if (conf->max_fds) {
1596 struct rlimit fd_limit;
1598 if (getrlimit(RLIMIT_NOFILE, &fd_limit) < 0) {
1599 condlog(0, "can't get open fds limit: %s\n",
1601 fd_limit.rlim_cur = 0;
1602 fd_limit.rlim_max = 0;
1604 if (fd_limit.rlim_cur < conf->max_fds) {
1605 fd_limit.rlim_cur = conf->max_fds;
1606 if (fd_limit.rlim_max < conf->max_fds)
1607 fd_limit.rlim_max = conf->max_fds;
1608 if (setrlimit(RLIMIT_NOFILE, &fd_limit) < 0) {
1609 condlog(0, "can't set open fds limit to "
1611 fd_limit.rlim_cur, fd_limit.rlim_max,
1614 condlog(3, "set open fds limit to %lu/%lu\n",
1615 fd_limit.rlim_cur, fd_limit.rlim_max);
1624 vecs = gvecs = init_vecs();
1629 if (sysfs_init(conf->sysfs_dir, FILE_NAME_SIZE)) {
1630 condlog(0, "can not find sysfs mount point");
1634 udev_set_sync_support(0);
1636 * Start uevent listener early to catch events
1638 if ((rc = pthread_create(&uevent_thr, &misc_attr, ueventloop, vecs))) {
1639 condlog(0, "failed to create uevent thread: %d", rc);
1642 if ((rc = pthread_create(&uxlsnr_thr, &misc_attr, uxlsnrloop, vecs))) {
1643 condlog(0, "failed to create cli listener: %d", rc);
1647 * fetch and configure both paths and multipaths
1650 running_state = DAEMON_CONFIGURE;
1652 if (configure(vecs, 1)) {
1654 condlog(0, "failure during configuration");
1662 if ((rc = pthread_create(&check_thr, &misc_attr, checkerloop, vecs))) {
1663 condlog(0,"failed to create checker loop thread: %d", rc);
1666 if ((rc = pthread_create(&uevq_thr, &misc_attr, uevqloop, vecs))) {
1667 condlog(0, "failed to create uevent dispatcher: %d", rc);
1670 pthread_attr_destroy(&misc_attr);
1672 pthread_mutex_lock(&exit_mutex);
1673 /* Startup complete, create logfile */
1674 if (pidfile_create(DEFAULT_PIDFILE, daemon_pid))
1675 /* Ignore errors, we can live without */
1676 condlog(1, "failed to create pidfile");
1678 running_state = DAEMON_RUNNING;
1679 pthread_cond_wait(&exit_cond, &exit_mutex);
1684 running_state = DAEMON_SHUTDOWN;
1685 block_signal(SIGHUP, NULL);
1687 if (conf->queue_without_daemon == QUE_NO_DAEMON_OFF)
1688 vector_foreach_slot(vecs->mpvec, mpp, i)
1689 dm_queue_if_no_path(mpp->alias, 0);
1690 remove_maps_and_stop_waiters(vecs);
1693 pthread_cancel(check_thr);
1694 pthread_cancel(uevent_thr);
1695 pthread_cancel(uxlsnr_thr);
1696 pthread_cancel(uevq_thr);
1701 free_pathvec(vecs->pathvec, FREE_PATHS);
1702 vecs->pathvec = NULL;
1704 /* Now all the waitevent threads will start rushing in. */
1705 while (vecs->lock.depth > 0) {
1706 sleep (1); /* This is weak. */
1707 condlog(3,"Have %d wait event checkers threads to de-alloc, waiting..\n", vecs->lock.depth);
1709 pthread_mutex_destroy(vecs->lock.mutex);
1710 FREE(vecs->lock.mutex);
1711 vecs->lock.depth = 0;
1712 vecs->lock.mutex = NULL;
1722 /* We're done here */
1723 condlog(3, "unlink pidfile");
1724 unlink(DEFAULT_PIDFILE);
1726 condlog(2, "--------shut down-------");
1732 * Freeing config must be done after condlog() and dm_lib_exit(),
1733 * because logging functions like dlog() and dm_write_log()
1734 * reference the config.
1740 dbg_free_final(NULL);
1752 if( (pid = fork()) < 0){
1753 fprintf(stderr, "Failed first fork : %s\n", strerror(errno));
1761 if ( (pid = fork()) < 0)
1762 fprintf(stderr, "Failed second fork : %s\n", strerror(errno));
1767 fprintf(stderr, "cannot chdir to '/', continuing\n");
1769 dev_null_fd = open("/dev/null", O_RDWR);
1770 if (dev_null_fd < 0){
1771 fprintf(stderr, "cannot open /dev/null for input & output : %s\n",
1776 close(STDIN_FILENO);
1778 close(STDOUT_FILENO);
1780 close(STDERR_FILENO);
1783 daemon_pid = getpid();
1788 main (int argc, char *argv[])
1790 extern char *optarg;
1796 running_state = DAEMON_INIT;
1799 if (getuid() != 0) {
1800 fprintf(stderr, "need to be root\n");
1804 /* make sure we don't lock any path */
1806 umask(umask(077) | 022);
1808 conf = alloc_config();
1813 while ((arg = getopt(argc, argv, ":dv:k::")) != EOF ) {
1817 //debug=1; /* ### comment me out ### */
1820 if (sizeof(optarg) > sizeof(char *) ||
1821 !isdigit(optarg[0]))
1824 conf->verbosity = atoi(optarg);
1833 if (optind < argc) {
1838 while (optind < argc) {
1839 if (strchr(argv[optind], ' '))
1840 c += snprintf(c, s + CMDSIZE - c, "\"%s\" ", argv[optind]);
1842 c += snprintf(c, s + CMDSIZE - c, "%s ", argv[optind]);
1845 c += snprintf(c, s + CMDSIZE - c, "\n");
1863 return (child(NULL));
1866 void * mpath_pr_event_handler_fn (void * pathp )
1868 struct multipath * mpp;
1869 int i,j, ret, isFound;
1870 struct path * pp = (struct path *)pathp;
1871 unsigned char *keyp;
1873 struct prout_param_descriptor *param;
1874 struct prin_resp *resp;
1878 resp = mpath_alloc_prin_response(MPATH_PRIN_RKEY_SA);
1880 condlog(0,"%s Alloc failed for prin response \n", pp->dev);
1884 ret = prin_do_scsi_ioctl(pp->dev, MPATH_PRIN_RKEY_SA, resp, 0);
1885 if (ret != MPATH_PR_SUCCESS )
1887 condlog(0,"%s : pr in read keys service action failed. Error=%d\n", pp->dev, ret);
1891 condlog(3, " event pr=%d addlen=%d\n",resp->prin_descriptor.prin_readkeys.prgeneration,
1892 resp->prin_descriptor.prin_readkeys.additional_length );
1894 if (resp->prin_descriptor.prin_readkeys.additional_length == 0 )
1896 condlog(1, "%s: No key found. Device may not be registered.", pp->dev);
1897 ret = MPATH_PR_SUCCESS;
1901 keyp = (unsigned char *)mpp->reservation_key;
1902 for (j = 0; j < 8; ++j) {
1908 condlog(2, "Multipath reservation_key: 0x%" PRIx64 " ", prkey);
1911 for (i = 0; i < resp->prin_descriptor.prin_readkeys.additional_length/8; i++ )
1913 condlog(2, "PR IN READKEYS[%d] reservation key:\n",i);
1914 dumpHex((char *)&resp->prin_descriptor.prin_readkeys.key_list[i*8], 8 , -1);
1915 if (!memcmp(mpp->reservation_key, &resp->prin_descriptor.prin_readkeys.key_list[i*8], 8))
1917 condlog(2, "%s: pr key found in prin readkeys response", mpp->alias);
1924 condlog(0, "%s: Either device not registered or ", pp->dev);
1925 condlog(0, "host is not authorised for registration. Skip path\n");
1926 ret = MPATH_PR_OTHER;
1930 param= malloc(sizeof(struct prout_param_descriptor));
1931 memset(param, 0 , sizeof(struct prout_param_descriptor));
1933 for (j = 7; j >= 0; --j) {
1934 param->sa_key[j] = (prkey & 0xff);
1937 param->num_transportid = 0;
1939 condlog(3, "device %s:%s \n", pp->dev, pp->mpp->wwid);
1941 ret = prout_do_scsi_ioctl(pp->dev, MPATH_PROUT_REG_IGN_SA, 0, 0, param, 0);
1942 if (ret != MPATH_PR_SUCCESS )
1944 condlog(0,"%s: Reservation registration failed. Error: %d\n", pp->dev, ret);
1954 int mpath_pr_event_handle(struct path *pp)
1958 pthread_attr_t attr;
1959 struct multipath * mpp;
1963 if (!mpp->reservation_key)
1966 pthread_attr_init(&attr);
1967 pthread_attr_setdetachstate(&attr, PTHREAD_CREATE_JOINABLE);
1969 rc = pthread_create(&thread, NULL , mpath_pr_event_handler_fn, pp);
1971 condlog(0, "%s: ERROR; return code from pthread_create() is %d\n", pp->dev, rc);
1974 pthread_attr_destroy(&attr);
1975 rc = pthread_join(thread, NULL);