From 47d818a43a1dca755129dff8b29539461a203ea8 Mon Sep 17 00:00:00 2001 From: Benjamin Marzinski Date: Fri, 27 Jul 2012 15:56:19 -0500 Subject: [PATCH] multipath: add followover failback mode This patch adds a new failback mode, followover, to deal with multiple computers accessing the same active/passive storage devices. In these cases, if only one node loses access to the primary paths, it will force a trespass to the secondary paths. If the nodes are configured with immediate failback, the other nodes with trespass back to the primary paths, and the machines will ping-pong the storage. If the nodes are configured with manual failback, this won't happen. However when the primary path is restored on the node that lost access to it, the nodes won't automatically failback to it. In followover mode, they will. Followover mode works by only failing back when a path comes back online from a pathgroup that previously had no working paths. For this to work, the paths need an additional attribute, chkrstate. This is just like the path state, except it is not updated when the paths state is changed by the kernel, only when the path checker function sees that the path is down. This is necessary because when a trespass occurs, all the outstanding IO to the previously active paths will fail, and the kernel will mark the path as down. But for failback to happen in followover mode, the paths must actually be down, not just in a ghost state. Signed-off-by: Benjamin Marzinski --- libmultipath/dict.c | 12 ++++++++++++ libmultipath/discovery.c | 6 +++--- libmultipath/print.c | 2 ++ libmultipath/structs.h | 4 +++- multipath/main.c | 2 +- multipath/multipath.conf.5 | 5 +++++ multipathd/main.c | 35 ++++++++++++++++++++++++++++++++++- 7 files changed, 60 insertions(+), 6 deletions(-) diff --git a/libmultipath/dict.c b/libmultipath/dict.c index 9cd4c37..f17ffaa 100644 --- a/libmultipath/dict.c +++ b/libmultipath/dict.c @@ -398,6 +398,8 @@ default_failback_handler(vector strvec) conf->pgfailback = -FAILBACK_MANUAL; else if (strlen(buff) == 9 && !strcmp(buff, "immediate")) conf->pgfailback = -FAILBACK_IMMEDIATE; + else if (strlen(buff) == 10 && !strcmp(buff, "followover")) + conf->pgfailback = -FAILBACK_FOLLOWOVER; else conf->pgfailback = atoi(buff); @@ -1053,6 +1055,8 @@ hw_failback_handler(vector strvec) hwe->pgfailback = -FAILBACK_MANUAL; else if (strlen(buff) == 9 && !strcmp(buff, "immediate")) hwe->pgfailback = -FAILBACK_IMMEDIATE; + else if (strlen(buff) == 10 && !strcmp(buff, "followover")) + hwe->pgfailback = -FAILBACK_FOLLOWOVER; else hwe->pgfailback = atoi(buff); @@ -1351,6 +1355,8 @@ mp_failback_handler(vector strvec) mpe->pgfailback = -FAILBACK_MANUAL; else if (strlen(buff) == 9 && !strcmp(buff, "immediate")) mpe->pgfailback = -FAILBACK_IMMEDIATE; + else if (strlen(buff) == 10 && !strcmp(buff, "followover")) + mpe->pgfailback = -FAILBACK_FOLLOWOVER; else mpe->pgfailback = atoi(buff); @@ -1769,6 +1775,8 @@ snprint_mp_failback (char * buff, int len, void * data) return snprintf(buff, len, "manual"); case -FAILBACK_IMMEDIATE: return snprintf(buff, len, "immediate"); + case -FAILBACK_FOLLOWOVER: + return snprintf(buff, len, "followover"); default: return snprintf(buff, len, "%i", mpe->pgfailback); } @@ -2130,6 +2138,8 @@ snprint_hw_failback (char * buff, int len, void * data) return snprintf(buff, len, "manual"); case -FAILBACK_IMMEDIATE: return snprintf(buff, len, "immediate"); + case -FAILBACK_FOLLOWOVER: + return snprintf(buff, len, "followover"); default: return snprintf(buff, len, "%i", hwe->pgfailback); } @@ -2394,6 +2404,8 @@ snprint_def_failback (char * buff, int len, void * data) return snprintf(buff, len, "manual"); case -FAILBACK_IMMEDIATE: return snprintf(buff, len, "immediate"); + case -FAILBACK_FOLLOWOVER: + return snprintf(buff, len, "followover"); default: return snprintf(buff, len, "%i", conf->pgfailback); } diff --git a/libmultipath/discovery.c b/libmultipath/discovery.c index 33391c6..abc686f 100644 --- a/libmultipath/discovery.c +++ b/libmultipath/discovery.c @@ -876,13 +876,13 @@ pathinfo (struct path *pp, vector hwtable, int mask) if (mask & DI_CHECKER) { if (path_state == PATH_UP) { - pp->state = get_state(pp, 0); + pp->chkrstate = pp->state = get_state(pp, 0); if (pp->state == PATH_UNCHECKED || pp->state == PATH_WILD) goto blank; } else { condlog(3, "%s: path inaccessible", pp->dev); - pp->state = path_state; + pp->chkrstate = pp->state = path_state; } } @@ -910,7 +910,7 @@ blank: * Recoverable error, for example faulty or offline path */ memset(pp->wwid, 0, WWID_SIZE); - pp->state = PATH_DOWN; + pp->chkrstate = pp->state = PATH_DOWN; return 0; } diff --git a/libmultipath/print.c b/libmultipath/print.c index ba4210e..a29b885 100644 --- a/libmultipath/print.c +++ b/libmultipath/print.c @@ -143,6 +143,8 @@ snprint_failback (char * buff, size_t len, struct multipath * mpp) { if (mpp->pgfailback == -FAILBACK_IMMEDIATE) return snprintf(buff, len, "immediate"); + if (mpp->pgfailback == -FAILBACK_FOLLOWOVER) + return snprintf(buff, len, "followover"); if (!mpp->failback_tick) return snprintf(buff, len, "-"); diff --git a/libmultipath/structs.h b/libmultipath/structs.h index 705cb29..991ea6e 100644 --- a/libmultipath/structs.h +++ b/libmultipath/structs.h @@ -39,7 +39,8 @@ enum rr_weight_mode { enum failback_mode { FAILBACK_UNDEF, FAILBACK_MANUAL, - FAILBACK_IMMEDIATE + FAILBACK_IMMEDIATE, + FAILBACK_FOLLOWOVER }; enum sysfs_buses { @@ -151,6 +152,7 @@ struct path { int offline; int state; int dmstate; + int chkrstate; int failcount; int priority; int pgindex; diff --git a/multipath/main.c b/multipath/main.c index 628da6b..883ef59 100644 --- a/multipath/main.c +++ b/multipath/main.c @@ -142,7 +142,7 @@ update_paths (struct multipath * mpp) /* * path is not in sysfs anymore */ - pp->state = PATH_DOWN; + pp->chkrstate = pp->state = PATH_DOWN; continue; } pp->mpp = mpp; diff --git a/multipath/multipath.conf.5 b/multipath/multipath.conf.5 index 8dd4a4b..24662af 100644 --- a/multipath/multipath.conf.5 +++ b/multipath/multipath.conf.5 @@ -254,6 +254,11 @@ active paths. .B manual Do not perform automatic failback. .TP +.B followover +Only perform automatic failback when the first path of a pathgroup +becomes active. This keeps a node from automatically failing back when +another node requested the failover. +.TP .B values > 0 deferred failback (time to defer in seconds) .TP diff --git a/multipathd/main.c b/multipathd/main.c index 6e33b11..64c1a0c 100644 --- a/multipathd/main.c +++ b/multipathd/main.c @@ -995,6 +995,32 @@ mpvec_garbage_collector (struct vectors * vecs) } } +/* This is called after a path has started working again. It the multipath + * device for this path uses the followover failback type, and this is the + * best pathgroup, and this is the first path in the pathgroup to come back + * up, then switch to this pathgroup */ +static int +followover_should_failback(struct path * pp) +{ + struct pathgroup * pgp; + struct path *pp1; + int i; + + if (pp->mpp->pgfailback != -FAILBACK_FOLLOWOVER || + !pp->mpp->pg || !pp->pgindex || + pp->pgindex != pp->mpp->bestpg) + return 0; + + pgp = VECTOR_SLOT(pp->mpp->pg, pp->pgindex - 1); + vector_foreach_slot(pgp->paths, pp1, i) { + if (pp1 == pp) + continue; + if (pp1->chkrstate != PATH_DOWN && pp1->chkrstate != PATH_SHAKY) + return 0; + } + return 1; +} + static void defered_failback_tick (vector mpvec) { @@ -1092,6 +1118,8 @@ check_path (struct vectors * vecs, struct path * pp) { int newstate; int new_path_up = 0; + int chkr_new_path_up = 0; + int oldchkrstate = pp->chkrstate; if (!pp->mpp) return; @@ -1130,6 +1158,7 @@ check_path (struct vectors * vecs, struct path * pp) pp->dev); pp->dmstate = PSTATE_UNDEF; } + pp->chkrstate = newstate; if (newstate != pp->state) { int oldstate = pp->state; pp->state = newstate; @@ -1182,6 +1211,9 @@ check_path (struct vectors * vecs, struct path * pp) new_path_up = 1; + if (oldchkrstate != PATH_UP && oldchkrstate != PATH_GHOST) + chkr_new_path_up = 1; + /* * if at least one path is up in a group, and * the group is disabled, re-enable it @@ -1233,7 +1265,8 @@ check_path (struct vectors * vecs, struct path * pp) (new_path_up || pp->mpp->failback_tick <= 0)) pp->mpp->failback_tick = pp->mpp->pgfailback + 1; - else if (pp->mpp->pgfailback == -FAILBACK_IMMEDIATE) + else if (pp->mpp->pgfailback == -FAILBACK_IMMEDIATE || + (chkr_new_path_up && followover_should_failback(pp))) switch_pathgroup(pp->mpp); } } -- 2.34.1