SUNRPC: Add tracepoint that fires when an RPC is retransmitted

A separate tracepoint can be left enabled all the time to capture
rare but important retransmission events. So for example:

kworker/u26:3-568   [009]   156.967933: xprt_retransmit:      task:44093@5 xid=0xa25dbc79 nfsv3 WRITE ntrans=2

Or, for example, enable all nfs and nfs4 tracepoints, and set up a
trigger to disable tracing when xprt_retransmit fires to capture
everything that leads up to it.

Signed-off-by: Chuck Lever <chuck.lever@oracle.com>
Signed-off-by: Trond Myklebust <trond.myklebust@hammerspace.com>

diff --git a/include/trace/events/sunrpc.h b/include/trace/events/sunrpc.h
index 036eb1f..430b42f 100644 (file)
--- a/include/trace/events/sunrpc.h
+++ b/include/trace/events/sunrpc.h
@@ -1079,6 +1079,46 @@ TRACE_EVENT(xprt_transmit,
                __entry->seqno, __entry->status)
 );
 
+TRACE_EVENT(xprt_retransmit,
+       TP_PROTO(
+               const struct rpc_rqst *rqst
+       ),
+
+       TP_ARGS(rqst),
+
+       TP_STRUCT__entry(
+               __field(unsigned int, task_id)
+               __field(unsigned int, client_id)
+               __field(u32, xid)
+               __field(int, ntrans)
+               __field(int, version)
+               __string(progname,
+                        rqst->rq_task->tk_client->cl_program->name)
+               __string(procedure,
+                        rqst->rq_task->tk_msg.rpc_proc->p_name)
+       ),
+
+       TP_fast_assign(
+               struct rpc_task *task = rqst->rq_task;
+
+               __entry->task_id = task->tk_pid;
+               __entry->client_id = task->tk_client ?
+                       task->tk_client->cl_clid : -1;
+               __entry->xid = be32_to_cpu(rqst->rq_xid);
+               __entry->ntrans = rqst->rq_ntrans;
+               __assign_str(progname,
+                            task->tk_client->cl_program->name)
+               __entry->version = task->tk_client->cl_vers;
+               __assign_str(procedure, task->tk_msg.rpc_proc->p_name)
+       ),
+
+       TP_printk(
+               "task:%u@%u xid=0x%08x %sv%d %s ntrans=%d",
+               __entry->task_id, __entry->client_id, __entry->xid,
+               __get_str(progname), __entry->version, __get_str(procedure),
+               __entry->ntrans)
+);
+
 TRACE_EVENT(xprt_ping,
        TP_PROTO(const struct rpc_xprt *xprt, int status),
 

diff --git a/net/sunrpc/xprt.c b/net/sunrpc/xprt.c
index 80c94ea..67039ee 100644 (file)
--- a/net/sunrpc/xprt.c
+++ b/net/sunrpc/xprt.c
@@ -1542,8 +1542,10 @@ xprt_request_transmit(struct rpc_rqst *req, struct rpc_task *snd_task)
                return status;
        }
 
-       if (is_retrans)
+       if (is_retrans) {
                task->tk_client->cl_stats->rpcretrans++;
+               trace_xprt_retransmit(req);
+       }
 
        xprt_inject_disconnect(xprt);