RDMA/rtrs-clt: Add event tracing support

Add event tracing mechanism for following routines:
- rtrs_clt_reconnect_work()
- rtrs_clt_close_conns()
- rtrs_rdma_error_recovery()

How to use:
1. Load the rtrs_client module
2. cd /sys/kernel/debug/tracing
3. If all the events need to be enabled:
        echo 1 > events/rtrs_clt/enable
4. OR only speific routine/event needs to be enabled e.g.
        echo 1 > events/rtrs_clt/rtrs_clt_close_conns/enable
5. cat trace
6. Run some workload which can trigger rtrs_clt_close_conns()

Link: https://lore.kernel.org/r/20220818105240.110234-2-haris.iqbal@ionos.com
Signed-off-by: Santosh Pradhan <santosh.pradhan@ionos.com>
Signed-off-by: Jack Wang <jinpu.wang@ionos.com>
Signed-off-by: Md Haris Iqbal <haris.iqbal@ionos.com>
Signed-off-by: Leon Romanovsky <leon@kernel.org>

diff --git a/drivers/infiniband/ulp/rtrs/Makefile b/drivers/infiniband/ulp/rtrs/Makefile
index 3898509..1fdf918 100644 (file)
--- a/drivers/infiniband/ulp/rtrs/Makefile
+++ b/drivers/infiniband/ulp/rtrs/Makefile
@@ -1,8 +1,11 @@
 # SPDX-License-Identifier: GPL-2.0-or-later
 
+CFLAGS_rtrs-clt-trace.o = -I$(src)
+
 rtrs-client-y := rtrs-clt.o \
                  rtrs-clt-stats.o \
-                 rtrs-clt-sysfs.o
+                 rtrs-clt-sysfs.o \
+                 rtrs-clt-trace.o
 
 rtrs-server-y := rtrs-srv.o \
                  rtrs-srv-stats.o \

diff --git a/drivers/infiniband/ulp/rtrs/rtrs-clt-trace.c b/drivers/infiniband/ulp/rtrs/rtrs-clt-trace.c
new file mode 100644 (file)
index 0000000..f14fa1f
--- /dev/null
+++ b/drivers/infiniband/ulp/rtrs/rtrs-clt-trace.c
@@ -0,0 +1,15 @@
+// SPDX-License-Identifier: GPL-2.0+
+/*
+ * RDMA Network Block Driver
+ *
+ * Copyright (c) 2022 1&1 IONOS SE. All rights reserved.
+ */
+#include "rtrs.h"
+#include "rtrs-clt.h"
+
+/*
+ * We include this last to have the helpers above available for the trace
+ * event implementations.
+ */
+#define CREATE_TRACE_POINTS
+#include "rtrs-clt-trace.h"

diff --git a/drivers/infiniband/ulp/rtrs/rtrs-clt-trace.h b/drivers/infiniband/ulp/rtrs/rtrs-clt-trace.h
new file mode 100644 (file)
index 0000000..7738e26
--- /dev/null
+++ b/drivers/infiniband/ulp/rtrs/rtrs-clt-trace.h
@@ -0,0 +1,86 @@
+/* SPDX-License-Identifier: GPL-2.0+ */
+/*
+ * RDMA Network Block Driver
+ *
+ * Copyright (c) 2022 1&1 IONOS SE. All rights reserved.
+ */
+#undef TRACE_SYSTEM
+#define TRACE_SYSTEM rtrs_clt
+
+#if !defined(_TRACE_RTRS_CLT_H) || defined(TRACE_HEADER_MULTI_READ)
+#define _TRACE_RTRS_CLT_H
+
+#include <linux/tracepoint.h>
+
+struct rtrs_clt_path;
+struct rtrs_clt_sess;
+
+TRACE_DEFINE_ENUM(RTRS_CLT_CONNECTING);
+TRACE_DEFINE_ENUM(RTRS_CLT_CONNECTING_ERR);
+TRACE_DEFINE_ENUM(RTRS_CLT_RECONNECTING);
+TRACE_DEFINE_ENUM(RTRS_CLT_CONNECTED);
+TRACE_DEFINE_ENUM(RTRS_CLT_CLOSING);
+TRACE_DEFINE_ENUM(RTRS_CLT_CLOSED);
+TRACE_DEFINE_ENUM(RTRS_CLT_DEAD);
+
+#define show_rtrs_clt_state(x) \
+       __print_symbolic(x, \
+               { RTRS_CLT_CONNECTING,          "CONNECTING" }, \
+               { RTRS_CLT_CONNECTING_ERR,      "CONNECTING_ERR" }, \
+               { RTRS_CLT_RECONNECTING,        "RECONNECTING" }, \
+               { RTRS_CLT_CONNECTED,           "CONNECTED" }, \
+               { RTRS_CLT_CLOSING,             "CLOSING" }, \
+               { RTRS_CLT_CLOSED,              "CLOSED" }, \
+               { RTRS_CLT_DEAD,                "DEAD" })
+
+DECLARE_EVENT_CLASS(rtrs_clt_conn_class,
+       TP_PROTO(struct rtrs_clt_path *clt_path),
+
+       TP_ARGS(clt_path),
+
+       TP_STRUCT__entry(
+               __field(int, state)
+               __field(int, reconnect_attempts)
+               __field(int, max_reconnect_attempts)
+               __field(int, fail_cnt)
+               __field(int, success_cnt)
+               __array(char, sessname, NAME_MAX)
+       ),
+
+       TP_fast_assign(
+               struct rtrs_clt_sess *clt = clt_path->clt;
+
+               __entry->state = clt_path->state;
+               __entry->reconnect_attempts = clt_path->reconnect_attempts;
+               __entry->max_reconnect_attempts = clt->max_reconnect_attempts;
+               __entry->fail_cnt = clt_path->stats->reconnects.fail_cnt;
+               __entry->success_cnt = clt_path->stats->reconnects.successful_cnt;
+               memcpy(__entry->sessname, kobject_name(&clt_path->kobj), NAME_MAX);
+       ),
+
+       TP_printk("RTRS-CLT: sess='%s' state=%s attempts='%d' max-attempts='%d' fail='%d' success='%d'",
+                  __entry->sessname,
+                  show_rtrs_clt_state(__entry->state),
+                  __entry->reconnect_attempts,
+                  __entry->max_reconnect_attempts,
+                  __entry->fail_cnt,
+                  __entry->success_cnt
+       )
+);
+
+#define DEFINE_CLT_CONN_EVENT(name) \
+DEFINE_EVENT(rtrs_clt_conn_class, rtrs_##name, \
+       TP_PROTO(struct rtrs_clt_path *clt_path), \
+       TP_ARGS(clt_path))
+
+DEFINE_CLT_CONN_EVENT(clt_reconnect_work);
+DEFINE_CLT_CONN_EVENT(clt_close_conns);
+DEFINE_CLT_CONN_EVENT(rdma_error_recovery);
+
+#endif /* _TRACE_RTRS_CLT_H */
+
+#undef TRACE_INCLUDE_PATH
+#define TRACE_INCLUDE_PATH .
+#define TRACE_INCLUDE_FILE rtrs-clt-trace
+#include <trace/define_trace.h>
+

diff --git a/drivers/infiniband/ulp/rtrs/rtrs-clt.c b/drivers/infiniband/ulp/rtrs/rtrs-clt.c
index baecde4..5219bb1 100644 (file)
--- a/drivers/infiniband/ulp/rtrs/rtrs-clt.c
+++ b/drivers/infiniband/ulp/rtrs/rtrs-clt.c
@@ -16,6 +16,7 @@
 
 #include "rtrs-clt.h"
 #include "rtrs-log.h"
+#include "rtrs-clt-trace.h"
 
 #define RTRS_CONNECT_TIMEOUT_MS 30000
 /*
@@ -302,6 +303,8 @@ static void rtrs_rdma_error_recovery(struct rtrs_clt_con *con)
 {
        struct rtrs_clt_path *clt_path = to_clt_path(con->c.path);
 
+       trace_rtrs_rdma_error_recovery(clt_path);
+
        if (rtrs_clt_change_state_from_to(clt_path,
                                           RTRS_CLT_CONNECTED,
                                           RTRS_CLT_RECONNECTING)) {
@@ -1942,6 +1945,8 @@ static int rtrs_rdma_conn_rejected(struct rtrs_clt_con *con,
 
 void rtrs_clt_close_conns(struct rtrs_clt_path *clt_path, bool wait)
 {
+       trace_rtrs_clt_close_conns(clt_path);
+
        if (rtrs_clt_change_state_get_old(clt_path, RTRS_CLT_CLOSING, NULL))
                queue_work(rtrs_wq, &clt_path->close_work);
        if (wait)
@@ -2648,6 +2653,8 @@ static void rtrs_clt_reconnect_work(struct work_struct *work)
                                reconnect_dwork);
        clt = clt_path->clt;
 
+       trace_rtrs_clt_reconnect_work(clt_path);
+
        if (READ_ONCE(clt_path->state) != RTRS_CLT_RECONNECTING)
                return;