NFS4: Trace lock reclaims

One of the most frustrating messages our sustaining team sees is
the "Lock reclaim failed!" message. Add some observability in the
client's lock reclaim logic so we can capture better data the
first time a problem occurs.

Signed-off-by: Chuck Lever <chuck.lever@oracle.com>
Signed-off-by: Trond Myklebust <trond.myklebust@hammerspace.com>

diff --git a/fs/nfs/nfs4_fs.h b/fs/nfs/nfs4_fs.h
index a452011..a7a73b1 100644 (file)
--- a/fs/nfs/nfs4_fs.h
+++ b/fs/nfs/nfs4_fs.h
@@ -166,11 +166,9 @@ enum {
        NFS_STATE_RECOVERY_FAILED,      /* OPEN stateid state recovery failed */
        NFS_STATE_MAY_NOTIFY_LOCK,      /* server may CB_NOTIFY_LOCK */
        NFS_STATE_CHANGE_WAIT,          /* A state changing operation is outstanding */
-#ifdef CONFIG_NFS_V4_2
        NFS_CLNT_DST_SSC_COPY_STATE,    /* dst server open state on client*/
        NFS_CLNT_SRC_SSC_COPY_STATE,    /* src server open state on client*/
        NFS_SRV_SSC_COPY_STATE,         /* ssc state on the dst server */
-#endif /* CONFIG_NFS_V4_2 */
 };
 
 struct nfs4_state {

diff --git a/fs/nfs/nfs4state.c b/fs/nfs/nfs4state.c
index 4c4d05d..3455232 100644 (file)
--- a/fs/nfs/nfs4state.c
+++ b/fs/nfs/nfs4state.c
@@ -1611,6 +1611,7 @@ static int __nfs4_reclaim_open_state(struct nfs4_state_owner *sp, struct nfs4_st
        if (!test_bit(NFS_DELEGATED_STATE, &state->flags)) {
                spin_lock(&state->state_lock);
                list_for_each_entry(lock, &state->lock_states, ls_locks) {
+                       trace_nfs4_state_lock_reclaim(state, lock);
                        if (!test_bit(NFS_LOCK_INITIALIZED, &lock->ls_flags))
                                pr_warn_ratelimited("NFS: %s: Lock reclaim failed!\n", __func__);
                }

diff --git a/fs/nfs/nfs4trace.h b/fs/nfs/nfs4trace.h
index 0c298dc..e60b6fb 100644 (file)
--- a/fs/nfs/nfs4trace.h
+++ b/fs/nfs/nfs4trace.h
@@ -1022,6 +1022,88 @@ TRACE_EVENT(nfs4_set_lock,
                )
 );
 
+TRACE_DEFINE_ENUM(LK_STATE_IN_USE);
+TRACE_DEFINE_ENUM(NFS_DELEGATED_STATE);
+TRACE_DEFINE_ENUM(NFS_OPEN_STATE);
+TRACE_DEFINE_ENUM(NFS_O_RDONLY_STATE);
+TRACE_DEFINE_ENUM(NFS_O_WRONLY_STATE);
+TRACE_DEFINE_ENUM(NFS_O_RDWR_STATE);
+TRACE_DEFINE_ENUM(NFS_STATE_RECLAIM_REBOOT);
+TRACE_DEFINE_ENUM(NFS_STATE_RECLAIM_NOGRACE);
+TRACE_DEFINE_ENUM(NFS_STATE_POSIX_LOCKS);
+TRACE_DEFINE_ENUM(NFS_STATE_RECOVERY_FAILED);
+TRACE_DEFINE_ENUM(NFS_STATE_MAY_NOTIFY_LOCK);
+TRACE_DEFINE_ENUM(NFS_STATE_CHANGE_WAIT);
+TRACE_DEFINE_ENUM(NFS_CLNT_DST_SSC_COPY_STATE);
+TRACE_DEFINE_ENUM(NFS_CLNT_SRC_SSC_COPY_STATE);
+TRACE_DEFINE_ENUM(NFS_SRV_SSC_COPY_STATE);
+
+#define show_nfs4_state_flags(flags) \
+       __print_flags(flags, "|", \
+               { LK_STATE_IN_USE,              "IN_USE" }, \
+               { NFS_DELEGATED_STATE,          "DELEGATED" }, \
+               { NFS_OPEN_STATE,               "OPEN" }, \
+               { NFS_O_RDONLY_STATE,           "O_RDONLY" }, \
+               { NFS_O_WRONLY_STATE,           "O_WRONLY" }, \
+               { NFS_O_RDWR_STATE,             "O_RDWR" }, \
+               { NFS_STATE_RECLAIM_REBOOT,     "RECLAIM_REBOOT" }, \
+               { NFS_STATE_RECLAIM_NOGRACE,    "RECLAIM_NOGRACE" }, \
+               { NFS_STATE_POSIX_LOCKS,        "POSIX_LOCKS" }, \
+               { NFS_STATE_RECOVERY_FAILED,    "RECOVERY_FAILED" }, \
+               { NFS_STATE_MAY_NOTIFY_LOCK,    "MAY_NOTIFY_LOCK" }, \
+               { NFS_STATE_CHANGE_WAIT,        "CHANGE_WAIT" }, \
+               { NFS_CLNT_DST_SSC_COPY_STATE,  "CLNT_DST_SSC_COPY" }, \
+               { NFS_CLNT_SRC_SSC_COPY_STATE,  "CLNT_SRC_SSC_COPY" }, \
+               { NFS_SRV_SSC_COPY_STATE,       "SRV_SSC_COPY" })
+
+#define show_nfs4_lock_flags(flags) \
+       __print_flags(flags, "|", \
+               { BIT(NFS_LOCK_INITIALIZED),    "INITIALIZED" }, \
+               { BIT(NFS_LOCK_LOST),           "LOST" })
+
+TRACE_EVENT(nfs4_state_lock_reclaim,
+               TP_PROTO(
+                       const struct nfs4_state *state,
+                       const struct nfs4_lock_state *lock
+               ),
+
+               TP_ARGS(state, lock),
+
+               TP_STRUCT__entry(
+                       __field(dev_t, dev)
+                       __field(u32, fhandle)
+                       __field(u64, fileid)
+                       __field(unsigned long, state_flags)
+                       __field(unsigned long, lock_flags)
+                       __field(int, stateid_seq)
+                       __field(u32, stateid_hash)
+               ),
+
+               TP_fast_assign(
+                       const struct inode *inode = state->inode;
+
+                       __entry->dev = inode->i_sb->s_dev;
+                       __entry->fileid = NFS_FILEID(inode);
+                       __entry->fhandle = nfs_fhandle_hash(NFS_FH(inode));
+                       __entry->state_flags = state->flags;
+                       __entry->lock_flags = lock->ls_flags;
+                       __entry->stateid_seq =
+                               be32_to_cpu(state->stateid.seqid);
+                       __entry->stateid_hash =
+                               nfs_stateid_hash(&state->stateid);
+               ),
+
+               TP_printk(
+                       "fileid=%02x:%02x:%llu fhandle=0x%08x "
+                       "stateid=%d:0x%08x state_flags=%s lock_flags=%s",
+                       MAJOR(__entry->dev), MINOR(__entry->dev),
+                       (unsigned long long)__entry->fileid, __entry->fhandle,
+                       __entry->stateid_seq, __entry->stateid_hash,
+                       show_nfs4_state_flags(__entry->state_flags),
+                       show_nfs4_lock_flags(__entry->lock_flags)
+               )
+)
+
 DECLARE_EVENT_CLASS(nfs4_set_delegation_event,
                TP_PROTO(
                        const struct inode *inode,