patch-5.15.79-rt54.patch

[platform/kernel/linux-rpi.git] / include / linux / preempt.h

diff --git a/include/linux/preempt.h b/include/linux/preempt.h
index 4d244e2..3da73c9 100644 (file)
--- a/include/linux/preempt.h
+++ b/include/linux/preempt.h
@@ -122,9 +122,10 @@
  * The preempt_count offset after spin_lock()
  */
 #if !defined(CONFIG_PREEMPT_RT)
-#define PREEMPT_LOCK_OFFSET    PREEMPT_DISABLE_OFFSET
+#define PREEMPT_LOCK_OFFSET            PREEMPT_DISABLE_OFFSET
 #else
-#define PREEMPT_LOCK_OFFSET    0
+/* Locks on RT do not disable preemption */
+#define PREEMPT_LOCK_OFFSET            0
 #endif
 
 /*
@@ -174,6 +175,20 @@ extern void preempt_count_sub(int val);
 #define preempt_count_inc() preempt_count_add(1)
 #define preempt_count_dec() preempt_count_sub(1)
 
+#ifdef CONFIG_PREEMPT_LAZY
+#define add_preempt_lazy_count(val)    do { preempt_lazy_count() += (val); } while (0)
+#define sub_preempt_lazy_count(val)    do { preempt_lazy_count() -= (val); } while (0)
+#define inc_preempt_lazy_count()       add_preempt_lazy_count(1)
+#define dec_preempt_lazy_count()       sub_preempt_lazy_count(1)
+#define preempt_lazy_count()           (current_thread_info()->preempt_lazy_count)
+#else
+#define add_preempt_lazy_count(val)    do { } while (0)
+#define sub_preempt_lazy_count(val)    do { } while (0)
+#define inc_preempt_lazy_count()       do { } while (0)
+#define dec_preempt_lazy_count()       do { } while (0)
+#define preempt_lazy_count()           (0)
+#endif
+
 #ifdef CONFIG_PREEMPT_COUNT
 
 #define preempt_disable() \
@@ -182,13 +197,25 @@ do { \
        barrier(); \
 } while (0)
 
+#define preempt_lazy_disable() \
+do { \
+       inc_preempt_lazy_count(); \
+       barrier(); \
+} while (0)
+
 #define sched_preempt_enable_no_resched() \
 do { \
        barrier(); \
        preempt_count_dec(); \
 } while (0)
 
-#define preempt_enable_no_resched() sched_preempt_enable_no_resched()
+#ifndef CONFIG_PREEMPT_RT
+# define preempt_enable_no_resched() sched_preempt_enable_no_resched()
+# define preempt_check_resched_rt() barrier();
+#else
+# define preempt_enable_no_resched() preempt_enable()
+# define preempt_check_resched_rt() preempt_check_resched()
+#endif
 
 #define preemptible()  (preempt_count() == 0 && !irqs_disabled())
 
@@ -213,6 +240,18 @@ do { \
                __preempt_schedule(); \
 } while (0)
 
+/*
+ * open code preempt_check_resched() because it is not exported to modules and
+ * used by local_unlock() or bpf_enable_instrumentation().
+ */
+#define preempt_lazy_enable() \
+do { \
+       dec_preempt_lazy_count(); \
+       barrier(); \
+       if (should_resched(0)) \
+               __preempt_schedule(); \
+} while (0)
+
 #else /* !CONFIG_PREEMPTION */
 #define preempt_enable() \
 do { \
@@ -220,6 +259,12 @@ do { \
        preempt_count_dec(); \
 } while (0)
 
+#define preempt_lazy_enable() \
+do { \
+       dec_preempt_lazy_count(); \
+       barrier(); \
+} while (0)
+
 #define preempt_enable_notrace() \
 do { \
        barrier(); \
@@ -258,8 +303,12 @@ do { \
 #define preempt_disable_notrace()              barrier()
 #define preempt_enable_no_resched_notrace()    barrier()
 #define preempt_enable_notrace()               barrier()
+#define preempt_check_resched_rt()             barrier()
 #define preemptible()                          0
 
+#define preempt_lazy_disable()                 barrier()
+#define preempt_lazy_enable()                  barrier()
+
 #endif /* CONFIG_PREEMPT_COUNT */
 
 #ifdef MODULE
@@ -278,7 +327,7 @@ do { \
 } while (0)
 #define preempt_fold_need_resched() \
 do { \
-       if (tif_need_resched()) \
+       if (tif_need_resched_now()) \
                set_preempt_need_resched(); \
 } while (0)
 
@@ -394,8 +443,15 @@ extern void migrate_enable(void);
 
 #else
 
-static inline void migrate_disable(void) { }
-static inline void migrate_enable(void) { }
+static inline void migrate_disable(void)
+{
+       preempt_lazy_disable();
+}
+
+static inline void migrate_enable(void)
+{
+       preempt_lazy_enable();
+}
 
 #endif /* CONFIG_SMP */