1 // SPDX-License-Identifier: GPL-2.0
3 * Multipath support for RPC
5 * Copyright (c) 2015, 2016, Primary Data, Inc. All rights reserved.
7 * Trond Myklebust <trond.myklebust@primarydata.com>
10 #include <linux/atomic.h>
11 #include <linux/types.h>
12 #include <linux/kref.h>
13 #include <linux/list.h>
14 #include <linux/rcupdate.h>
15 #include <linux/rculist.h>
16 #include <linux/slab.h>
17 #include <linux/spinlock.h>
18 #include <linux/sunrpc/xprt.h>
19 #include <linux/sunrpc/addr.h>
20 #include <linux/sunrpc/xprtmultipath.h>
24 typedef struct rpc_xprt *(*xprt_switch_find_xprt_t)(struct rpc_xprt_switch *xps,
25 const struct rpc_xprt *cur);
27 static const struct rpc_xprt_iter_ops rpc_xprt_iter_singular;
28 static const struct rpc_xprt_iter_ops rpc_xprt_iter_roundrobin;
29 static const struct rpc_xprt_iter_ops rpc_xprt_iter_listall;
31 static void xprt_switch_add_xprt_locked(struct rpc_xprt_switch *xps,
32 struct rpc_xprt *xprt)
34 if (unlikely(xprt_get(xprt) == NULL))
36 list_add_tail_rcu(&xprt->xprt_switch, &xps->xps_xprt_list);
38 if (xps->xps_nxprts == 0)
39 xps->xps_net = xprt->xprt_net;
45 * rpc_xprt_switch_add_xprt - Add a new rpc_xprt to an rpc_xprt_switch
46 * @xps: pointer to struct rpc_xprt_switch
47 * @xprt: pointer to struct rpc_xprt
49 * Adds xprt to the end of the list of struct rpc_xprt in xps.
51 void rpc_xprt_switch_add_xprt(struct rpc_xprt_switch *xps,
52 struct rpc_xprt *xprt)
56 spin_lock(&xps->xps_lock);
57 if (xps->xps_net == xprt->xprt_net || xps->xps_net == NULL)
58 xprt_switch_add_xprt_locked(xps, xprt);
59 spin_unlock(&xps->xps_lock);
60 rpc_sysfs_xprt_setup(xps, xprt, GFP_KERNEL);
63 static void xprt_switch_remove_xprt_locked(struct rpc_xprt_switch *xps,
64 struct rpc_xprt *xprt)
66 if (unlikely(xprt == NULL))
68 if (!test_bit(XPRT_OFFLINE, &xprt->state))
71 if (xps->xps_nxprts == 0)
74 list_del_rcu(&xprt->xprt_switch);
78 * rpc_xprt_switch_remove_xprt - Removes an rpc_xprt from a rpc_xprt_switch
79 * @xps: pointer to struct rpc_xprt_switch
80 * @xprt: pointer to struct rpc_xprt
82 * Removes xprt from the list of struct rpc_xprt in xps.
84 void rpc_xprt_switch_remove_xprt(struct rpc_xprt_switch *xps,
85 struct rpc_xprt *xprt)
87 spin_lock(&xps->xps_lock);
88 xprt_switch_remove_xprt_locked(xps, xprt);
89 spin_unlock(&xps->xps_lock);
93 static DEFINE_IDA(rpc_xprtswitch_ids);
95 void xprt_multipath_cleanup_ids(void)
97 ida_destroy(&rpc_xprtswitch_ids);
100 static int xprt_switch_alloc_id(struct rpc_xprt_switch *xps, gfp_t gfp_flags)
104 id = ida_simple_get(&rpc_xprtswitch_ids, 0, 0, gfp_flags);
112 static void xprt_switch_free_id(struct rpc_xprt_switch *xps)
114 ida_simple_remove(&rpc_xprtswitch_ids, xps->xps_id);
118 * xprt_switch_alloc - Allocate a new struct rpc_xprt_switch
119 * @xprt: pointer to struct rpc_xprt
120 * @gfp_flags: allocation flags
122 * On success, returns an initialised struct rpc_xprt_switch, containing
123 * the entry xprt. Returns NULL on failure.
125 struct rpc_xprt_switch *xprt_switch_alloc(struct rpc_xprt *xprt,
128 struct rpc_xprt_switch *xps;
130 xps = kmalloc(sizeof(*xps), gfp_flags);
132 spin_lock_init(&xps->xps_lock);
133 kref_init(&xps->xps_kref);
134 xprt_switch_alloc_id(xps, gfp_flags);
135 xps->xps_nxprts = xps->xps_nactive = 0;
136 atomic_long_set(&xps->xps_queuelen, 0);
138 INIT_LIST_HEAD(&xps->xps_xprt_list);
139 xps->xps_iter_ops = &rpc_xprt_iter_singular;
140 rpc_sysfs_xprt_switch_setup(xps, xprt, gfp_flags);
141 xprt_switch_add_xprt_locked(xps, xprt);
142 rpc_sysfs_xprt_setup(xps, xprt, gfp_flags);
148 static void xprt_switch_free_entries(struct rpc_xprt_switch *xps)
150 spin_lock(&xps->xps_lock);
151 while (!list_empty(&xps->xps_xprt_list)) {
152 struct rpc_xprt *xprt;
154 xprt = list_first_entry(&xps->xps_xprt_list,
155 struct rpc_xprt, xprt_switch);
156 xprt_switch_remove_xprt_locked(xps, xprt);
157 spin_unlock(&xps->xps_lock);
159 spin_lock(&xps->xps_lock);
161 spin_unlock(&xps->xps_lock);
164 static void xprt_switch_free(struct kref *kref)
166 struct rpc_xprt_switch *xps = container_of(kref,
167 struct rpc_xprt_switch, xps_kref);
169 xprt_switch_free_entries(xps);
170 rpc_sysfs_xprt_switch_destroy(xps);
171 xprt_switch_free_id(xps);
172 kfree_rcu(xps, xps_rcu);
176 * xprt_switch_get - Return a reference to a rpc_xprt_switch
177 * @xps: pointer to struct rpc_xprt_switch
179 * Returns a reference to xps unless the refcount is already zero.
181 struct rpc_xprt_switch *xprt_switch_get(struct rpc_xprt_switch *xps)
183 if (xps != NULL && kref_get_unless_zero(&xps->xps_kref))
189 * xprt_switch_put - Release a reference to a rpc_xprt_switch
190 * @xps: pointer to struct rpc_xprt_switch
192 * Release the reference to xps, and free it once the refcount is zero.
194 void xprt_switch_put(struct rpc_xprt_switch *xps)
197 kref_put(&xps->xps_kref, xprt_switch_free);
201 * rpc_xprt_switch_set_roundrobin - Set a round-robin policy on rpc_xprt_switch
202 * @xps: pointer to struct rpc_xprt_switch
204 * Sets a round-robin default policy for iterators acting on xps.
206 void rpc_xprt_switch_set_roundrobin(struct rpc_xprt_switch *xps)
208 if (READ_ONCE(xps->xps_iter_ops) != &rpc_xprt_iter_roundrobin)
209 WRITE_ONCE(xps->xps_iter_ops, &rpc_xprt_iter_roundrobin);
213 const struct rpc_xprt_iter_ops *xprt_iter_ops(const struct rpc_xprt_iter *xpi)
215 if (xpi->xpi_ops != NULL)
217 return rcu_dereference(xpi->xpi_xpswitch)->xps_iter_ops;
221 void xprt_iter_no_rewind(struct rpc_xprt_iter *xpi)
226 void xprt_iter_default_rewind(struct rpc_xprt_iter *xpi)
228 WRITE_ONCE(xpi->xpi_cursor, NULL);
232 bool xprt_is_active(const struct rpc_xprt *xprt)
234 return (kref_read(&xprt->kref) != 0 &&
235 !test_bit(XPRT_OFFLINE, &xprt->state));
239 struct rpc_xprt *xprt_switch_find_first_entry(struct list_head *head)
241 struct rpc_xprt *pos;
243 list_for_each_entry_rcu(pos, head, xprt_switch) {
244 if (xprt_is_active(pos))
251 struct rpc_xprt *xprt_iter_first_entry(struct rpc_xprt_iter *xpi)
253 struct rpc_xprt_switch *xps = rcu_dereference(xpi->xpi_xpswitch);
257 return xprt_switch_find_first_entry(&xps->xps_xprt_list);
261 struct rpc_xprt *xprt_switch_find_current_entry(struct list_head *head,
262 const struct rpc_xprt *cur)
264 struct rpc_xprt *pos;
267 list_for_each_entry_rcu(pos, head, xprt_switch) {
270 if (found && xprt_is_active(pos))
277 struct rpc_xprt *xprt_iter_current_entry(struct rpc_xprt_iter *xpi)
279 struct rpc_xprt_switch *xps = rcu_dereference(xpi->xpi_xpswitch);
280 struct list_head *head;
284 head = &xps->xps_xprt_list;
285 if (xpi->xpi_cursor == NULL || xps->xps_nxprts < 2)
286 return xprt_switch_find_first_entry(head);
287 return xprt_switch_find_current_entry(head, xpi->xpi_cursor);
290 bool rpc_xprt_switch_has_addr(struct rpc_xprt_switch *xps,
291 const struct sockaddr *sap)
293 struct list_head *head;
294 struct rpc_xprt *pos;
296 if (xps == NULL || sap == NULL)
299 head = &xps->xps_xprt_list;
300 list_for_each_entry_rcu(pos, head, xprt_switch) {
301 if (rpc_cmp_addr_port(sap, (struct sockaddr *)&pos->addr)) {
302 pr_info("RPC: addr %s already in xprt switch\n",
303 pos->address_strings[RPC_DISPLAY_ADDR]);
311 struct rpc_xprt *xprt_switch_find_next_entry(struct list_head *head,
312 const struct rpc_xprt *cur)
314 struct rpc_xprt *pos, *prev = NULL;
317 list_for_each_entry_rcu(pos, head, xprt_switch) {
320 if (found && xprt_is_active(pos))
328 struct rpc_xprt *xprt_switch_set_next_cursor(struct rpc_xprt_switch *xps,
329 struct rpc_xprt **cursor,
330 xprt_switch_find_xprt_t find_next)
332 struct rpc_xprt *pos, *old;
334 old = smp_load_acquire(cursor);
335 pos = find_next(xps, old);
336 smp_store_release(cursor, pos);
341 struct rpc_xprt *xprt_iter_next_entry_multiple(struct rpc_xprt_iter *xpi,
342 xprt_switch_find_xprt_t find_next)
344 struct rpc_xprt_switch *xps = rcu_dereference(xpi->xpi_xpswitch);
348 return xprt_switch_set_next_cursor(xps, &xpi->xpi_cursor, find_next);
352 struct rpc_xprt *__xprt_switch_find_next_entry_roundrobin(struct list_head *head,
353 const struct rpc_xprt *cur)
355 struct rpc_xprt *ret;
357 ret = xprt_switch_find_next_entry(head, cur);
360 return xprt_switch_find_first_entry(head);
364 struct rpc_xprt *xprt_switch_find_next_entry_roundrobin(struct rpc_xprt_switch *xps,
365 const struct rpc_xprt *cur)
367 struct list_head *head = &xps->xps_xprt_list;
368 struct rpc_xprt *xprt;
369 unsigned int nactive;
372 unsigned long xprt_queuelen, xps_queuelen;
374 xprt = __xprt_switch_find_next_entry_roundrobin(head, cur);
377 xprt_queuelen = atomic_long_read(&xprt->queuelen);
378 xps_queuelen = atomic_long_read(&xps->xps_queuelen);
379 nactive = READ_ONCE(xps->xps_nactive);
380 /* Exit loop if xprt_queuelen <= average queue length */
381 if (xprt_queuelen * nactive <= xps_queuelen)
389 struct rpc_xprt *xprt_iter_next_entry_roundrobin(struct rpc_xprt_iter *xpi)
391 return xprt_iter_next_entry_multiple(xpi,
392 xprt_switch_find_next_entry_roundrobin);
396 struct rpc_xprt *xprt_switch_find_next_entry_all(struct rpc_xprt_switch *xps,
397 const struct rpc_xprt *cur)
399 return xprt_switch_find_next_entry(&xps->xps_xprt_list, cur);
403 struct rpc_xprt *xprt_iter_next_entry_all(struct rpc_xprt_iter *xpi)
405 return xprt_iter_next_entry_multiple(xpi,
406 xprt_switch_find_next_entry_all);
410 * xprt_iter_rewind - Resets the xprt iterator
411 * @xpi: pointer to rpc_xprt_iter
413 * Resets xpi to ensure that it points to the first entry in the list
417 void xprt_iter_rewind(struct rpc_xprt_iter *xpi)
420 xprt_iter_ops(xpi)->xpi_rewind(xpi);
424 static void __xprt_iter_init(struct rpc_xprt_iter *xpi,
425 struct rpc_xprt_switch *xps,
426 const struct rpc_xprt_iter_ops *ops)
428 rcu_assign_pointer(xpi->xpi_xpswitch, xprt_switch_get(xps));
429 xpi->xpi_cursor = NULL;
434 * xprt_iter_init - Initialise an xprt iterator
435 * @xpi: pointer to rpc_xprt_iter
436 * @xps: pointer to rpc_xprt_switch
438 * Initialises the iterator to use the default iterator ops
439 * as set in xps. This function is mainly intended for internal
440 * use in the rpc_client.
442 void xprt_iter_init(struct rpc_xprt_iter *xpi,
443 struct rpc_xprt_switch *xps)
445 __xprt_iter_init(xpi, xps, NULL);
449 * xprt_iter_init_listall - Initialise an xprt iterator
450 * @xpi: pointer to rpc_xprt_iter
451 * @xps: pointer to rpc_xprt_switch
453 * Initialises the iterator to iterate once through the entire list
456 void xprt_iter_init_listall(struct rpc_xprt_iter *xpi,
457 struct rpc_xprt_switch *xps)
459 __xprt_iter_init(xpi, xps, &rpc_xprt_iter_listall);
463 * xprt_iter_xchg_switch - Atomically swap out the rpc_xprt_switch
464 * @xpi: pointer to rpc_xprt_iter
465 * @newswitch: pointer to a new rpc_xprt_switch or NULL
467 * Swaps out the existing xpi->xpi_xpswitch with a new value.
469 struct rpc_xprt_switch *xprt_iter_xchg_switch(struct rpc_xprt_iter *xpi,
470 struct rpc_xprt_switch *newswitch)
472 struct rpc_xprt_switch __rcu *oldswitch;
474 /* Atomically swap out the old xpswitch */
475 oldswitch = xchg(&xpi->xpi_xpswitch, RCU_INITIALIZER(newswitch));
476 if (newswitch != NULL)
477 xprt_iter_rewind(xpi);
478 return rcu_dereference_protected(oldswitch, true);
482 * xprt_iter_destroy - Destroys the xprt iterator
483 * @xpi: pointer to rpc_xprt_iter
485 void xprt_iter_destroy(struct rpc_xprt_iter *xpi)
487 xprt_switch_put(xprt_iter_xchg_switch(xpi, NULL));
491 * xprt_iter_xprt - Returns the rpc_xprt pointed to by the cursor
492 * @xpi: pointer to rpc_xprt_iter
494 * Returns a pointer to the struct rpc_xprt that is currently
495 * pointed to by the cursor.
496 * Caller must be holding rcu_read_lock().
498 struct rpc_xprt *xprt_iter_xprt(struct rpc_xprt_iter *xpi)
500 WARN_ON_ONCE(!rcu_read_lock_held());
501 return xprt_iter_ops(xpi)->xpi_xprt(xpi);
505 struct rpc_xprt *xprt_iter_get_helper(struct rpc_xprt_iter *xpi,
506 struct rpc_xprt *(*fn)(struct rpc_xprt_iter *))
508 struct rpc_xprt *ret;
515 } while (ret == NULL);
520 * xprt_iter_get_xprt - Returns the rpc_xprt pointed to by the cursor
521 * @xpi: pointer to rpc_xprt_iter
523 * Returns a reference to the struct rpc_xprt that is currently
524 * pointed to by the cursor.
526 struct rpc_xprt *xprt_iter_get_xprt(struct rpc_xprt_iter *xpi)
528 struct rpc_xprt *xprt;
531 xprt = xprt_iter_get_helper(xpi, xprt_iter_ops(xpi)->xpi_xprt);
537 * xprt_iter_get_next - Returns the next rpc_xprt following the cursor
538 * @xpi: pointer to rpc_xprt_iter
540 * Returns a reference to the struct rpc_xprt that immediately follows the
541 * entry pointed to by the cursor.
543 struct rpc_xprt *xprt_iter_get_next(struct rpc_xprt_iter *xpi)
545 struct rpc_xprt *xprt;
548 xprt = xprt_iter_get_helper(xpi, xprt_iter_ops(xpi)->xpi_next);
553 /* Policy for always returning the first entry in the rpc_xprt_switch */
555 const struct rpc_xprt_iter_ops rpc_xprt_iter_singular = {
556 .xpi_rewind = xprt_iter_no_rewind,
557 .xpi_xprt = xprt_iter_first_entry,
558 .xpi_next = xprt_iter_first_entry,
561 /* Policy for round-robin iteration of entries in the rpc_xprt_switch */
563 const struct rpc_xprt_iter_ops rpc_xprt_iter_roundrobin = {
564 .xpi_rewind = xprt_iter_default_rewind,
565 .xpi_xprt = xprt_iter_current_entry,
566 .xpi_next = xprt_iter_next_entry_roundrobin,
569 /* Policy for once-through iteration of entries in the rpc_xprt_switch */
571 const struct rpc_xprt_iter_ops rpc_xprt_iter_listall = {
572 .xpi_rewind = xprt_iter_default_rewind,
573 .xpi_xprt = xprt_iter_current_entry,
574 .xpi_next = xprt_iter_next_entry_all,