multipath: add followover failback mode

This patch adds a new failback mode, followover, to deal with multiple
computers accessing the same active/passive storage devices.  In these
cases, if only one node loses access to the primary paths, it will
force a trespass to the secondary paths.  If the nodes are configured
with immediate failback, the other nodes with trespass back to the
primary paths, and the machines will ping-pong the storage. If the
nodes are configured with manual failback, this won't happen. However
when the primary path is restored on the node that lost access to it,
the nodes won't automatically failback to it.  In followover mode, they
will.

Followover mode works by only failing back when a path comes back online
from a pathgroup that previously had no working paths.  For this to
work, the paths need an additional attribute, chkrstate. This is just like
the path state, except it is not updated when the paths state is changed
by the kernel, only when the path checker function sees that the path is
down.  This is necessary because when a trespass occurs, all the outstanding
IO to the previously active paths will fail, and the kernel will mark the
path as down.  But for failback to happen in followover mode, the paths must
actually be down, not just in a ghost state.

Signed-off-by: Benjamin Marzinski <bmarzins@redhat.com>

diff --git a/libmultipath/dict.c b/libmultipath/dict.c
index 9cd4c37..f17ffaa 100644 (file)
--- a/libmultipath/dict.c
+++ b/libmultipath/dict.c
@@ -398,6 +398,8 @@ default_failback_handler(vector strvec)
                conf->pgfailback = -FAILBACK_MANUAL;
        else if (strlen(buff) == 9 && !strcmp(buff, "immediate"))
                conf->pgfailback = -FAILBACK_IMMEDIATE;
+       else if (strlen(buff) == 10 && !strcmp(buff, "followover"))
+               conf->pgfailback = -FAILBACK_FOLLOWOVER;
        else
                conf->pgfailback = atoi(buff);
 
@@ -1053,6 +1055,8 @@ hw_failback_handler(vector strvec)
                hwe->pgfailback = -FAILBACK_MANUAL;
        else if (strlen(buff) == 9 && !strcmp(buff, "immediate"))
                hwe->pgfailback = -FAILBACK_IMMEDIATE;
+       else if (strlen(buff) == 10 && !strcmp(buff, "followover"))
+               hwe->pgfailback = -FAILBACK_FOLLOWOVER;
        else
                hwe->pgfailback = atoi(buff);
 
@@ -1351,6 +1355,8 @@ mp_failback_handler(vector strvec)
                mpe->pgfailback = -FAILBACK_MANUAL;
        else if (strlen(buff) == 9 && !strcmp(buff, "immediate"))
                mpe->pgfailback = -FAILBACK_IMMEDIATE;
+       else if (strlen(buff) == 10 && !strcmp(buff, "followover"))
+               mpe->pgfailback = -FAILBACK_FOLLOWOVER;
        else
                mpe->pgfailback = atoi(buff);
 
@@ -1769,6 +1775,8 @@ snprint_mp_failback (char * buff, int len, void * data)
                return snprintf(buff, len, "manual");
        case -FAILBACK_IMMEDIATE:
                return snprintf(buff, len, "immediate");
+       case -FAILBACK_FOLLOWOVER:
+               return snprintf(buff, len, "followover");
        default:
                return snprintf(buff, len, "%i", mpe->pgfailback);
        }
@@ -2130,6 +2138,8 @@ snprint_hw_failback (char * buff, int len, void * data)
                return snprintf(buff, len, "manual");
        case -FAILBACK_IMMEDIATE:
                return snprintf(buff, len, "immediate");
+       case -FAILBACK_FOLLOWOVER:
+               return snprintf(buff, len, "followover");
        default:
                return snprintf(buff, len, "%i", hwe->pgfailback);
        }
@@ -2394,6 +2404,8 @@ snprint_def_failback (char * buff, int len, void * data)
                return snprintf(buff, len, "manual");
        case -FAILBACK_IMMEDIATE:
                return snprintf(buff, len, "immediate");
+       case -FAILBACK_FOLLOWOVER:
+               return snprintf(buff, len, "followover");
        default:
                return snprintf(buff, len, "%i", conf->pgfailback);
        }

diff --git a/libmultipath/discovery.c b/libmultipath/discovery.c
index 33391c6..abc686f 100644 (file)
--- a/libmultipath/discovery.c
+++ b/libmultipath/discovery.c
@@ -876,13 +876,13 @@ pathinfo (struct path *pp, vector hwtable, int mask)
 
        if (mask & DI_CHECKER) {
                if (path_state == PATH_UP) {
-                       pp->state = get_state(pp, 0);
+                       pp->chkrstate = pp->state = get_state(pp, 0);
                        if (pp->state == PATH_UNCHECKED ||
                            pp->state == PATH_WILD)
                                goto blank;
                } else {
                        condlog(3, "%s: path inaccessible", pp->dev);
-                       pp->state = path_state;
+                       pp->chkrstate = pp->state = path_state;
                }
        }
 
@@ -910,7 +910,7 @@ blank:
         * Recoverable error, for example faulty or offline path
         */
        memset(pp->wwid, 0, WWID_SIZE);
-       pp->state = PATH_DOWN;
+       pp->chkrstate = pp->state = PATH_DOWN;
 
        return 0;
 }

diff --git a/libmultipath/print.c b/libmultipath/print.c
index ba4210e..a29b885 100644 (file)
--- a/libmultipath/print.c
+++ b/libmultipath/print.c
@@ -143,6 +143,8 @@ snprint_failback (char * buff, size_t len, struct multipath * mpp)
 {
        if (mpp->pgfailback == -FAILBACK_IMMEDIATE)
                return snprintf(buff, len, "immediate");
+       if (mpp->pgfailback == -FAILBACK_FOLLOWOVER)
+               return snprintf(buff, len, "followover");
 
        if (!mpp->failback_tick)
                return snprintf(buff, len, "-");

diff --git a/libmultipath/structs.h b/libmultipath/structs.h
index 705cb29..991ea6e 100644 (file)
--- a/libmultipath/structs.h
+++ b/libmultipath/structs.h
@@ -39,7 +39,8 @@ enum rr_weight_mode {
 enum failback_mode {
        FAILBACK_UNDEF,
        FAILBACK_MANUAL,
-       FAILBACK_IMMEDIATE
+       FAILBACK_IMMEDIATE,
+       FAILBACK_FOLLOWOVER
 };
 
 enum sysfs_buses {
@@ -151,6 +152,7 @@ struct path {
        int offline;
        int state;
        int dmstate;
+       int chkrstate;
        int failcount;
        int priority;
        int pgindex;

diff --git a/multipath/main.c b/multipath/main.c
index 628da6b..883ef59 100644 (file)
--- a/multipath/main.c
+++ b/multipath/main.c
@@ -142,7 +142,7 @@ update_paths (struct multipath * mpp)
                                        /*
                                         * path is not in sysfs anymore
                                         */
-                                       pp->state = PATH_DOWN;
+                                       pp->chkrstate = pp->state = PATH_DOWN;
                                        continue;
                                }
                                pp->mpp = mpp;

diff --git a/multipath/multipath.conf.5 b/multipath/multipath.conf.5
index 8dd4a4b..24662af 100644 (file)
--- a/multipath/multipath.conf.5
+++ b/multipath/multipath.conf.5
@@ -254,6 +254,11 @@ active paths.
 .B manual
 Do not perform automatic failback.
 .TP
+.B followover
+Only perform automatic failback when the first path of a pathgroup
+becomes active. This keeps a node from automatically failing back when
+another node requested the failover.
+.TP
 .B values > 0
 deferred failback (time to defer in seconds)
 .TP

diff --git a/multipathd/main.c b/multipathd/main.c
index 6e33b11..64c1a0c 100644 (file)
--- a/multipathd/main.c
+++ b/multipathd/main.c
@@ -995,6 +995,32 @@ mpvec_garbage_collector (struct vectors * vecs)
        }
 }
 
+/* This is called after a path has started working again. It the multipath
+ * device for this path uses the followover failback type, and this is the
+ * best pathgroup, and this is the first path in the pathgroup to come back
+ * up, then switch to this pathgroup */
+static int
+followover_should_failback(struct path * pp)
+{
+       struct pathgroup * pgp;
+       struct path *pp1;
+       int i;
+
+       if (pp->mpp->pgfailback != -FAILBACK_FOLLOWOVER ||
+           !pp->mpp->pg || !pp->pgindex ||
+           pp->pgindex != pp->mpp->bestpg)
+               return 0;
+
+       pgp = VECTOR_SLOT(pp->mpp->pg, pp->pgindex - 1);
+       vector_foreach_slot(pgp->paths, pp1, i) {
+               if (pp1 == pp)
+                       continue;
+               if (pp1->chkrstate != PATH_DOWN && pp1->chkrstate != PATH_SHAKY)
+                       return 0;
+       }
+       return 1;
+}
+
 static void
 defered_failback_tick (vector mpvec)
 {
@@ -1092,6 +1118,8 @@ check_path (struct vectors * vecs, struct path * pp)
 {
        int newstate;
        int new_path_up = 0;
+       int chkr_new_path_up = 0;
+       int oldchkrstate = pp->chkrstate;
 
        if (!pp->mpp)
                return;
@@ -1130,6 +1158,7 @@ check_path (struct vectors * vecs, struct path * pp)
                        pp->dev);
                pp->dmstate = PSTATE_UNDEF;
        }
+       pp->chkrstate = newstate;
        if (newstate != pp->state) {
                int oldstate = pp->state;
                pp->state = newstate;
@@ -1182,6 +1211,9 @@ check_path (struct vectors * vecs, struct path * pp)
 
                new_path_up = 1;
 
+               if (oldchkrstate != PATH_UP && oldchkrstate != PATH_GHOST)
+                       chkr_new_path_up = 1;
+
                /*
                 * if at least one path is up in a group, and
                 * the group is disabled, re-enable it
@@ -1233,7 +1265,8 @@ check_path (struct vectors * vecs, struct path * pp)
                    (new_path_up || pp->mpp->failback_tick <= 0))
                        pp->mpp->failback_tick =
                                pp->mpp->pgfailback + 1;
-               else if (pp->mpp->pgfailback == -FAILBACK_IMMEDIATE)
+               else if (pp->mpp->pgfailback == -FAILBACK_IMMEDIATE ||
+                        (chkr_new_path_up && followover_should_failback(pp)))
                        switch_pathgroup(pp->mpp);
        }
 }