config XFS_DRAIN_INTENTS
bool
+ select JUMP_LABEL if HAVE_ARCH_JUMP_LABEL
config XFS_ONLINE_SCRUB
bool "XFS online metadata check support"
#include "scrub/scrub.h"
#include "scrub/common.h"
+int
+xchk_setup_agheader(
+ struct xfs_scrub *sc)
+{
+ if (xchk_need_intent_drain(sc))
+ xchk_fsgates_enable(sc, XCHK_FSGATES_DRAIN);
+ return xchk_setup_fs(sc);
+}
+
/* Superblock */
/* Cross-reference with the other btrees. */
xchk_setup_ag_allocbt(
struct xfs_scrub *sc)
{
+ if (xchk_need_intent_drain(sc))
+ xchk_fsgates_enable(sc, XCHK_FSGATES_DRAIN);
+
return xchk_setup_ag_btree(sc, false);
}
{
int error;
+ if (xchk_need_intent_drain(sc))
+ xchk_fsgates_enable(sc, XCHK_FSGATES_DRAIN);
+
error = xchk_get_inode(sc);
if (error)
goto out;
sa->agi_bp = NULL;
}
+ if (!(sc->flags & XCHK_FSGATES_DRAIN))
+ return -EDEADLOCK;
error = xfs_perag_intent_drain(sa->pag);
if (error == -ERESTARTSYS)
error = -EINTR;
}
sc->flags &= ~XCHK_REAPING_DISABLED;
}
+
+/*
+ * Enable filesystem hooks (i.e. runtime code patching) before starting a scrub
+ * operation. Callers must not hold any locks that intersect with the CPU
+ * hotplug lock (e.g. writeback locks) because code patching must halt the CPUs
+ * to change kernel code.
+ */
+void
+xchk_fsgates_enable(
+ struct xfs_scrub *sc,
+ unsigned int scrub_fsgates)
+{
+ ASSERT(!(scrub_fsgates & ~XCHK_FSGATES_ALL));
+ ASSERT(!(sc->flags & scrub_fsgates));
+
+ trace_xchk_fsgates_enable(sc, scrub_fsgates);
+
+ if (scrub_fsgates & XCHK_FSGATES_DRAIN)
+ xfs_drain_wait_enable();
+
+ sc->flags |= scrub_fsgates;
+}
struct xfs_btree_cur **curpp);
/* Setup functions */
+int xchk_setup_agheader(struct xfs_scrub *sc);
int xchk_setup_fs(struct xfs_scrub *sc);
int xchk_setup_ag_allocbt(struct xfs_scrub *sc);
int xchk_setup_ag_iallocbt(struct xfs_scrub *sc);
void xchk_stop_reaping(struct xfs_scrub *sc);
void xchk_start_reaping(struct xfs_scrub *sc);
+/*
+ * Setting up a hook to wait for intents to drain is costly -- we have to take
+ * the CPU hotplug lock and force an i-cache flush on all CPUs once to set it
+ * up, and again to tear it down. These costs add up quickly, so we only want
+ * to enable the drain waiter if the drain actually detected a conflict with
+ * running intent chains.
+ */
+static inline bool xchk_need_intent_drain(struct xfs_scrub *sc)
+{
+ return sc->flags & XCHK_TRY_HARDER;
+}
+
+void xchk_fsgates_enable(struct xfs_scrub *sc, unsigned int scrub_fshooks);
+
#endif /* __XFS_SCRUB_COMMON_H__ */
struct xchk_fscounters *fsc;
int error;
+ /*
+ * If the AGF doesn't track btreeblks, we have to lock the AGF to count
+ * btree block usage by walking the actual btrees.
+ */
+ if (!xfs_has_lazysbcount(sc->mp))
+ xchk_fsgates_enable(sc, XCHK_FSGATES_DRAIN);
+
sc->buf = kzalloc(sizeof(struct xchk_fscounters), XCHK_GFP_FLAGS);
if (!sc->buf)
return -ENOMEM;
xchk_setup_ag_iallocbt(
struct xfs_scrub *sc)
{
+ if (xchk_need_intent_drain(sc))
+ xchk_fsgates_enable(sc, XCHK_FSGATES_DRAIN);
return xchk_setup_ag_btree(sc, sc->flags & XCHK_TRY_HARDER);
}
{
int error;
+ if (xchk_need_intent_drain(sc))
+ xchk_fsgates_enable(sc, XCHK_FSGATES_DRAIN);
+
/*
* Try to get the inode. If the verifiers fail, we try again
* in raw mode.
if (!xfs_this_quota_on(sc->mp, dqtype))
return -ENOENT;
+ if (xchk_need_intent_drain(sc))
+ xchk_fsgates_enable(sc, XCHK_FSGATES_DRAIN);
+
error = xchk_setup_fs(sc);
if (error)
return error;
xchk_setup_ag_refcountbt(
struct xfs_scrub *sc)
{
+ if (xchk_need_intent_drain(sc))
+ xchk_fsgates_enable(sc, XCHK_FSGATES_DRAIN);
return xchk_setup_ag_btree(sc, false);
}
xchk_setup_ag_rmapbt(
struct xfs_scrub *sc)
{
+ if (xchk_need_intent_drain(sc))
+ xchk_fsgates_enable(sc, XCHK_FSGATES_DRAIN);
+
return xchk_setup_ag_btree(sc, false);
}
/* Scrub setup and teardown */
+static inline void
+xchk_fsgates_disable(
+ struct xfs_scrub *sc)
+{
+ if (!(sc->flags & XCHK_FSGATES_ALL))
+ return;
+
+ trace_xchk_fsgates_disable(sc, sc->flags & XCHK_FSGATES_ALL);
+
+ if (sc->flags & XCHK_FSGATES_DRAIN)
+ xfs_drain_wait_disable();
+
+ sc->flags &= ~XCHK_FSGATES_ALL;
+}
+
/* Free all the resources and finish the transactions. */
STATIC int
xchk_teardown(
kvfree(sc->buf);
sc->buf = NULL;
}
+
+ xchk_fsgates_disable(sc);
return error;
}
},
[XFS_SCRUB_TYPE_SB] = { /* superblock */
.type = ST_PERAG,
- .setup = xchk_setup_fs,
+ .setup = xchk_setup_agheader,
.scrub = xchk_superblock,
.repair = xrep_superblock,
},
[XFS_SCRUB_TYPE_AGF] = { /* agf */
.type = ST_PERAG,
- .setup = xchk_setup_fs,
+ .setup = xchk_setup_agheader,
.scrub = xchk_agf,
.repair = xrep_agf,
},
[XFS_SCRUB_TYPE_AGFL]= { /* agfl */
.type = ST_PERAG,
- .setup = xchk_setup_fs,
+ .setup = xchk_setup_agheader,
.scrub = xchk_agfl,
.repair = xrep_agfl,
},
[XFS_SCRUB_TYPE_AGI] = { /* agi */
.type = ST_PERAG,
- .setup = xchk_setup_fs,
+ .setup = xchk_setup_agheader,
.scrub = xchk_agi,
.repair = xrep_agi,
},
/* XCHK state flags grow up from zero, XREP state flags grown down from 2^31 */
#define XCHK_TRY_HARDER (1 << 0) /* can't get resources, try again */
-#define XCHK_REAPING_DISABLED (1 << 2) /* background block reaping paused */
+#define XCHK_REAPING_DISABLED (1 << 1) /* background block reaping paused */
+#define XCHK_FSGATES_DRAIN (1 << 2) /* defer ops draining enabled */
#define XREP_ALREADY_FIXED (1 << 31) /* checking our repair work */
+/*
+ * The XCHK_FSGATES* flags reflect functionality in the main filesystem that
+ * are only enabled for this particular online fsck. When not in use, the
+ * features are gated off via dynamic code patching, which is why the state
+ * must be enabled during scrub setup and can only be torn down afterwards.
+ */
+#define XCHK_FSGATES_ALL (XCHK_FSGATES_DRAIN)
+
/* Metadata scrubbers */
int xchk_tester(struct xfs_scrub *sc);
int xchk_superblock(struct xfs_scrub *sc);
{ XFS_SCRUB_OFLAG_WARNING, "warning" }, \
{ XFS_SCRUB_OFLAG_NO_REPAIR_NEEDED, "norepair" }
+#define XFS_SCRUB_STATE_STRINGS \
+ { XCHK_TRY_HARDER, "try_harder" }, \
+ { XCHK_REAPING_DISABLED, "reaping_disabled" }, \
+ { XCHK_FSGATES_DRAIN, "fsgates_drain" }, \
+ { XREP_ALREADY_FIXED, "already_fixed" }
+
DECLARE_EVENT_CLASS(xchk_class,
TP_PROTO(struct xfs_inode *ip, struct xfs_scrub_metadata *sm,
int error),
DEFINE_SCRUB_EVENT(xrep_attempt);
DEFINE_SCRUB_EVENT(xrep_done);
+DECLARE_EVENT_CLASS(xchk_fsgate_class,
+ TP_PROTO(struct xfs_scrub *sc, unsigned int fsgate_flags),
+ TP_ARGS(sc, fsgate_flags),
+ TP_STRUCT__entry(
+ __field(dev_t, dev)
+ __field(unsigned int, type)
+ __field(unsigned int, fsgate_flags)
+ ),
+ TP_fast_assign(
+ __entry->dev = sc->mp->m_super->s_dev;
+ __entry->type = sc->sm->sm_type;
+ __entry->fsgate_flags = fsgate_flags;
+ ),
+ TP_printk("dev %d:%d type %s fsgates '%s'",
+ MAJOR(__entry->dev), MINOR(__entry->dev),
+ __print_symbolic(__entry->type, XFS_SCRUB_TYPE_STRINGS),
+ __print_flags(__entry->fsgate_flags, "|", XFS_SCRUB_STATE_STRINGS))
+)
+
+#define DEFINE_SCRUB_FSHOOK_EVENT(name) \
+DEFINE_EVENT(xchk_fsgate_class, name, \
+ TP_PROTO(struct xfs_scrub *sc, unsigned int fsgates_flags), \
+ TP_ARGS(sc, fsgates_flags))
+
+DEFINE_SCRUB_FSHOOK_EVENT(xchk_fsgates_enable);
+DEFINE_SCRUB_FSHOOK_EVENT(xchk_fsgates_disable);
+
TRACE_EVENT(xchk_op_error,
TP_PROTO(struct xfs_scrub *sc, xfs_agnumber_t agno,
xfs_agblock_t bno, int error, void *ret_ip),
#include "xfs_ag.h"
#include "xfs_trace.h"
+/*
+ * Use a static key here to reduce the overhead of xfs_drain_rele. If the
+ * compiler supports jump labels, the static branch will be replaced by a nop
+ * sled when there are no xfs_drain_wait callers. Online fsck is currently
+ * the only caller, so this is a reasonable tradeoff.
+ *
+ * Note: Patching the kernel code requires taking the cpu hotplug lock. Other
+ * parts of the kernel allocate memory with that lock held, which means that
+ * XFS callers cannot hold any locks that might be used by memory reclaim or
+ * writeback when calling the static_branch_{inc,dec} functions.
+ */
+static DEFINE_STATIC_KEY_FALSE(xfs_drain_waiter_gate);
+
+void
+xfs_drain_wait_disable(void)
+{
+ static_branch_dec(&xfs_drain_waiter_gate);
+}
+
+void
+xfs_drain_wait_enable(void)
+{
+ static_branch_inc(&xfs_drain_waiter_gate);
+}
+
void
xfs_defer_drain_init(
struct xfs_defer_drain *dr)
static inline void xfs_defer_drain_rele(struct xfs_defer_drain *dr)
{
if (atomic_dec_and_test(&dr->dr_count) &&
+ static_branch_unlikely(&xfs_drain_waiter_gate) &&
has_waiters(&dr->dr_waiters))
wake_up(&dr->dr_waiters);
}
void xfs_defer_drain_init(struct xfs_defer_drain *dr);
void xfs_defer_drain_free(struct xfs_defer_drain *dr);
+void xfs_drain_wait_disable(void);
+void xfs_drain_wait_enable(void);
+
/*
* Deferred Work Intent Drains
* ===========================