Merge tag 'percpu-for-6.6' of git://git.kernel.org/pub/scm/linux/kernel/git/dennis...

Pull percpu updates from Dennis Zhou:
 "One bigger change to percpu_counter's api allowing for init and
  destroy of multiple counters via percpu_counter_init_many() and
  percpu_counter_destroy_many(). This is used to help begin remediating
  a performance regression with percpu rss stats.

  Additionally, it seems larger core count machines are feeling the
  burden of the single threaded allocation of percpu. Mateusz is
  thinking about it and I will spend some time on it too.

  percpu:

   - A couple cleanups by Baoquan He and Bibo Mao. The only behavior
     change is to start printing messages if we're under the warn limit
     for failed atomic allocations.

  percpu_counter:

   - Shakeel introduced percpu counters into mm_struct which caused
     percpu allocations be on the hot path [1]. Originally I spent some
     time trying to improve the percpu allocator, but instead preferred
     what Mateusz Guzik proposed grouping at the allocation site,
     percpu_counter_init_many(). This allows a single percpu allocation
     to be shared by the counters. I like this approach because it
     creates a shared lifetime by the allocations. Additionally, I
     believe many inits have higher level synchronization requirements,
     like percpu_counter does against HOTPLUG_CPU. Therefore we can
     group these optimizations together"

Link: https://lore.kernel.org/linux-mm/20221024052841.3291983-1-shakeelb@google.com/
* tag 'percpu-for-6.6' of git://git.kernel.org/pub/scm/linux/kernel/git/dennis/percpu:
  kernel/fork: group allocation/free of per-cpu counters for mm struct
  pcpcntr: add group allocation/free
  mm/percpu.c: print error message too if atomic alloc failed
  mm/percpu.c: optimize the code in pcpu_setup_first_chunk() a little bit
  mm/percpu.c: remove redundant check
  mm/percpu: Remove some local variables in pcpu_populate_pte

diff --combined kernel/fork.c
index a9c18d4,afd198b..3b6d20d
--- 1/kernel/fork.c
--- 2/kernel/fork.c
+++ b/kernel/fork.c
@@@ -909,8 -909,6 +909,6 @@@ static void cleanup_lazy_tlbs(struct mm
   */
  void __mmdrop(struct mm_struct *mm)
  {
-       int i;
- 
        BUG_ON(mm == &init_mm);
        WARN_ON_ONCE(mm == current->mm);
  
@@@ -925,9 -923,8 +923,8 @@@
        put_user_ns(mm->user_ns);
        mm_pasid_drop(mm);
        mm_destroy_cid(mm);
+       percpu_counter_destroy_many(mm->rss_stat, NR_MM_COUNTERS);
  
-       for (i = 0; i < NR_MM_COUNTERS; i++)
-               percpu_counter_destroy(&mm->rss_stat[i]);
        free_mm(mm);
  }
  EXPORT_SYMBOL_GPL(__mmdrop);
@@@ -985,14 -982,6 +982,14 @@@ void __put_task_struct(struct task_stru
  }
  EXPORT_SYMBOL_GPL(__put_task_struct);
  
 +void __put_task_struct_rcu_cb(struct rcu_head *rhp)
 +{
 +      struct task_struct *task = container_of(rhp, struct task_struct, rcu);
 +
 +      __put_task_struct(task);
 +}
 +EXPORT_SYMBOL_GPL(__put_task_struct_rcu_cb);
 +
  void __init __weak arch_task_cache_init(void) { }
  
  /*
@@@ -1260,8 -1249,6 +1257,6 @@@ static void mm_init_uprobes_state(struc
  static struct mm_struct *mm_init(struct mm_struct *mm, struct task_struct *p,
        struct user_namespace *user_ns)
  {
-       int i;
- 
        mt_init_flags(&mm->mm_mt, MM_MT_FLAGS);
        mt_set_external_lock(&mm->mm_mt, &mm->mmap_lock);
        atomic_set(&mm->mm_users, 1);
@@@ -1309,17 -1296,15 +1304,15 @@@
        if (mm_alloc_cid(mm))
                goto fail_cid;
  
-       for (i = 0; i < NR_MM_COUNTERS; i++)
-               if (percpu_counter_init(&mm->rss_stat[i], 0, GFP_KERNEL_ACCOUNT))
-                       goto fail_pcpu;
+       if (percpu_counter_init_many(mm->rss_stat, 0, GFP_KERNEL_ACCOUNT,
+                                    NR_MM_COUNTERS))
+               goto fail_pcpu;
  
        mm->user_ns = get_user_ns(user_ns);
        lru_gen_init_mm(mm);
        return mm;
  
  fail_pcpu:
-       while (i > 0)
-               percpu_counter_destroy(&mm->rss_stat[--i]);
        mm_destroy_cid(mm);
  fail_cid:
        destroy_context(mm);
@@@ -1404,8 -1389,8 +1397,8 @@@ EXPORT_SYMBOL_GPL(mmput_async)
   * This changes mm's executable file (shown as symlink /proc/[pid]/exe).
   *
   * Main users are mmput() and sys_execve(). Callers prevent concurrent
 - * invocations: in mmput() nobody alive left, in execve task is single
 - * threaded.
 + * invocations: in mmput() nobody alive left, in execve it happens before
 + * the new mm is made visible to anyone.
   *
   * Can only fail if new_exe_file != NULL.
   */
@@@ -1440,7 -1425,9 +1433,7 @@@ int set_mm_exe_file(struct mm_struct *m
  /**
   * replace_mm_exe_file - replace a reference to the mm's executable file
   *
 - * This changes mm's executable file (shown as symlink /proc/[pid]/exe),
 - * dealing with concurrent invocation and without grabbing the mmap lock in
 - * write mode.
 + * This changes mm's executable file (shown as symlink /proc/[pid]/exe).
   *
   * Main user is sys_prctl(PR_SET_MM_MAP/EXE_FILE).
   */
@@@ -1470,20 -1457,22 +1463,20 @@@ int replace_mm_exe_file(struct mm_struc
                        return ret;
        }
  
 -      /* set the new file, lockless */
        ret = deny_write_access(new_exe_file);
        if (ret)
                return -EACCES;
        get_file(new_exe_file);
  
 -      old_exe_file = xchg(&mm->exe_file, new_exe_file);
 +      /* set the new file */
 +      mmap_write_lock(mm);
 +      old_exe_file = rcu_dereference_raw(mm->exe_file);
 +      rcu_assign_pointer(mm->exe_file, new_exe_file);
 +      mmap_write_unlock(mm);
 +
        if (old_exe_file) {
 -              /*
 -               * Don't race with dup_mmap() getting the file and disallowing
 -               * write access while someone might open the file writable.
 -               */
 -              mmap_read_lock(mm);
                allow_write_access(old_exe_file);
                fput(old_exe_file);
 -              mmap_read_unlock(mm);
        }
        return 0;
  }