libceph: tcp_nodelay support

TCP_NODELAY socket option set on connection sockets,
disables Nagle’s algorithm and improves latency characteristics.
tcp_nodelay(default)/notcp_nodelay option flags provided to
enable/disable setting the socket option.

Signed-off-by: Chaitanya Huilgol <chaitanya.huilgol@sandisk.com>
[idryomov@redhat.com: NO_TCP_NODELAY -> TCP_NODELAY, minor adjustments]
Signed-off-by: Ilya Dryomov <idryomov@redhat.com>

diff --git a/include/linux/ceph/libceph.h b/include/linux/ceph/libceph.h
index 8b11a79..16fff96 100644 (file)
--- a/include/linux/ceph/libceph.h
+++ b/include/linux/ceph/libceph.h
@@ -30,8 +30,9 @@
 #define CEPH_OPT_MYIP             (1<<2) /* specified my ip */
 #define CEPH_OPT_NOCRC            (1<<3) /* no data crc on writes */
 #define CEPH_OPT_NOMSGAUTH       (1<<4) /* not require cephx message signature */
+#define CEPH_OPT_TCP_NODELAY     (1<<5) /* TCP_NODELAY on TCP sockets */
 
-#define CEPH_OPT_DEFAULT   (0)
+#define CEPH_OPT_DEFAULT   (CEPH_OPT_TCP_NODELAY)
 
 #define ceph_set_opt(client, opt) \
        (client)->options->flags |= CEPH_OPT_##opt;

diff --git a/include/linux/ceph/messenger.h b/include/linux/ceph/messenger.h
index d9d396c..e154994 100644 (file)
--- a/include/linux/ceph/messenger.h
+++ b/include/linux/ceph/messenger.h
@@ -57,6 +57,7 @@ struct ceph_messenger {
 
        atomic_t stopping;
        bool nocrc;
+       bool tcp_nodelay;
 
        /*
         * the global_seq counts connections i (attempt to) initiate
@@ -264,7 +265,8 @@ extern void ceph_messenger_init(struct ceph_messenger *msgr,
                        struct ceph_entity_addr *myaddr,
                        u64 supported_features,
                        u64 required_features,
-                       bool nocrc);
+                       bool nocrc,
+                       bool tcp_nodelay);
 
 extern void ceph_con_init(struct ceph_connection *con, void *private,
                        const struct ceph_connection_operations *ops,

diff --git a/net/ceph/ceph_common.c b/net/ceph/ceph_common.c
index 5d5ab67..ec56550 100644 (file)
--- a/net/ceph/ceph_common.c
+++ b/net/ceph/ceph_common.c
@@ -239,6 +239,8 @@ enum {
        Opt_nocrc,
        Opt_cephx_require_signatures,
        Opt_nocephx_require_signatures,
+       Opt_tcp_nodelay,
+       Opt_notcp_nodelay,
 };
 
 static match_table_t opt_tokens = {
@@ -259,6 +261,8 @@ static match_table_t opt_tokens = {
        {Opt_nocrc, "nocrc"},
        {Opt_cephx_require_signatures, "cephx_require_signatures"},
        {Opt_nocephx_require_signatures, "nocephx_require_signatures"},
+       {Opt_tcp_nodelay, "tcp_nodelay"},
+       {Opt_notcp_nodelay, "notcp_nodelay"},
        {-1, NULL}
 };
 
@@ -457,6 +461,7 @@ ceph_parse_options(char *options, const char *dev_name,
                case Opt_nocrc:
                        opt->flags |= CEPH_OPT_NOCRC;
                        break;
+
                case Opt_cephx_require_signatures:
                        opt->flags &= ~CEPH_OPT_NOMSGAUTH;
                        break;
@@ -464,6 +469,13 @@ ceph_parse_options(char *options, const char *dev_name,
                        opt->flags |= CEPH_OPT_NOMSGAUTH;
                        break;
 
+               case Opt_tcp_nodelay:
+                       opt->flags |= CEPH_OPT_TCP_NODELAY;
+                       break;
+               case Opt_notcp_nodelay:
+                       opt->flags &= ~CEPH_OPT_TCP_NODELAY;
+                       break;
+
                default:
                        BUG_ON(token);
                }
@@ -518,10 +530,12 @@ struct ceph_client *ceph_create_client(struct ceph_options *opt, void *private,
        /* msgr */
        if (ceph_test_opt(client, MYIP))
                myaddr = &client->options->my_addr;
+
        ceph_messenger_init(&client->msgr, myaddr,
                client->supported_features,
                client->required_features,
-               ceph_test_opt(client, NOCRC));
+               ceph_test_opt(client, NOCRC),
+               ceph_test_opt(client, TCP_NODELAY));
 
        /* subsystems */
        err = ceph_monc_init(&client->monc, client);

diff --git a/net/ceph/messenger.c b/net/ceph/messenger.c
index 33a2f20..6b3f54e 100644 (file)
--- a/net/ceph/messenger.c
+++ b/net/ceph/messenger.c
@@ -510,6 +510,16 @@ static int ceph_tcp_connect(struct ceph_connection *con)
                return ret;
        }
 
+       if (con->msgr->tcp_nodelay) {
+               int optval = 1;
+
+               ret = kernel_setsockopt(sock, SOL_TCP, TCP_NODELAY,
+                                       (char *)&optval, sizeof(optval));
+               if (ret)
+                       pr_err("kernel_setsockopt(TCP_NODELAY) failed: %d",
+                              ret);
+       }
+
        sk_set_memalloc(sock->sk);
 
        con->sock = sock;
@@ -2922,7 +2932,8 @@ void ceph_messenger_init(struct ceph_messenger *msgr,
                        struct ceph_entity_addr *myaddr,
                        u64 supported_features,
                        u64 required_features,
-                       bool nocrc)
+                       bool nocrc,
+                       bool tcp_nodelay)
 {
        msgr->supported_features = supported_features;
        msgr->required_features = required_features;
@@ -2937,6 +2948,7 @@ void ceph_messenger_init(struct ceph_messenger *msgr,
        get_random_bytes(&msgr->inst.addr.nonce, sizeof(msgr->inst.addr.nonce));
        encode_my_addr(msgr);
        msgr->nocrc = nocrc;
+       msgr->tcp_nodelay = tcp_nodelay;
 
        atomic_set(&msgr->stopping, 0);