Merge branch 'dmi-for-linus' of git://git.kernel.org/pub/scm/linux/kernel/git/jdelvar...
[platform/kernel/linux-starfive.git] / kernel / exit.c
1 // SPDX-License-Identifier: GPL-2.0-only
2 /*
3  *  linux/kernel/exit.c
4  *
5  *  Copyright (C) 1991, 1992  Linus Torvalds
6  */
7
8 #include <linux/mm.h>
9 #include <linux/slab.h>
10 #include <linux/sched/autogroup.h>
11 #include <linux/sched/mm.h>
12 #include <linux/sched/stat.h>
13 #include <linux/sched/task.h>
14 #include <linux/sched/task_stack.h>
15 #include <linux/sched/cputime.h>
16 #include <linux/interrupt.h>
17 #include <linux/module.h>
18 #include <linux/capability.h>
19 #include <linux/completion.h>
20 #include <linux/personality.h>
21 #include <linux/tty.h>
22 #include <linux/iocontext.h>
23 #include <linux/key.h>
24 #include <linux/cpu.h>
25 #include <linux/acct.h>
26 #include <linux/tsacct_kern.h>
27 #include <linux/file.h>
28 #include <linux/fdtable.h>
29 #include <linux/freezer.h>
30 #include <linux/binfmts.h>
31 #include <linux/nsproxy.h>
32 #include <linux/pid_namespace.h>
33 #include <linux/ptrace.h>
34 #include <linux/profile.h>
35 #include <linux/mount.h>
36 #include <linux/proc_fs.h>
37 #include <linux/kthread.h>
38 #include <linux/mempolicy.h>
39 #include <linux/taskstats_kern.h>
40 #include <linux/delayacct.h>
41 #include <linux/cgroup.h>
42 #include <linux/syscalls.h>
43 #include <linux/signal.h>
44 #include <linux/posix-timers.h>
45 #include <linux/cn_proc.h>
46 #include <linux/mutex.h>
47 #include <linux/futex.h>
48 #include <linux/pipe_fs_i.h>
49 #include <linux/audit.h> /* for audit_free() */
50 #include <linux/resource.h>
51 #include <linux/task_io_accounting_ops.h>
52 #include <linux/blkdev.h>
53 #include <linux/task_work.h>
54 #include <linux/fs_struct.h>
55 #include <linux/init_task.h>
56 #include <linux/perf_event.h>
57 #include <trace/events/sched.h>
58 #include <linux/hw_breakpoint.h>
59 #include <linux/oom.h>
60 #include <linux/writeback.h>
61 #include <linux/shm.h>
62 #include <linux/kcov.h>
63 #include <linux/kmsan.h>
64 #include <linux/random.h>
65 #include <linux/rcuwait.h>
66 #include <linux/compat.h>
67 #include <linux/io_uring.h>
68 #include <linux/kprobes.h>
69 #include <linux/rethook.h>
70
71 #include <linux/uaccess.h>
72 #include <asm/unistd.h>
73 #include <asm/mmu_context.h>
74
75 static void __unhash_process(struct task_struct *p, bool group_dead)
76 {
77         nr_threads--;
78         detach_pid(p, PIDTYPE_PID);
79         if (group_dead) {
80                 detach_pid(p, PIDTYPE_TGID);
81                 detach_pid(p, PIDTYPE_PGID);
82                 detach_pid(p, PIDTYPE_SID);
83
84                 list_del_rcu(&p->tasks);
85                 list_del_init(&p->sibling);
86                 __this_cpu_dec(process_counts);
87         }
88         list_del_rcu(&p->thread_group);
89         list_del_rcu(&p->thread_node);
90 }
91
92 /*
93  * This function expects the tasklist_lock write-locked.
94  */
95 static void __exit_signal(struct task_struct *tsk)
96 {
97         struct signal_struct *sig = tsk->signal;
98         bool group_dead = thread_group_leader(tsk);
99         struct sighand_struct *sighand;
100         struct tty_struct *tty;
101         u64 utime, stime;
102
103         sighand = rcu_dereference_check(tsk->sighand,
104                                         lockdep_tasklist_lock_is_held());
105         spin_lock(&sighand->siglock);
106
107 #ifdef CONFIG_POSIX_TIMERS
108         posix_cpu_timers_exit(tsk);
109         if (group_dead)
110                 posix_cpu_timers_exit_group(tsk);
111 #endif
112
113         if (group_dead) {
114                 tty = sig->tty;
115                 sig->tty = NULL;
116         } else {
117                 /*
118                  * If there is any task waiting for the group exit
119                  * then notify it:
120                  */
121                 if (sig->notify_count > 0 && !--sig->notify_count)
122                         wake_up_process(sig->group_exec_task);
123
124                 if (tsk == sig->curr_target)
125                         sig->curr_target = next_thread(tsk);
126         }
127
128         add_device_randomness((const void*) &tsk->se.sum_exec_runtime,
129                               sizeof(unsigned long long));
130
131         /*
132          * Accumulate here the counters for all threads as they die. We could
133          * skip the group leader because it is the last user of signal_struct,
134          * but we want to avoid the race with thread_group_cputime() which can
135          * see the empty ->thread_head list.
136          */
137         task_cputime(tsk, &utime, &stime);
138         write_seqlock(&sig->stats_lock);
139         sig->utime += utime;
140         sig->stime += stime;
141         sig->gtime += task_gtime(tsk);
142         sig->min_flt += tsk->min_flt;
143         sig->maj_flt += tsk->maj_flt;
144         sig->nvcsw += tsk->nvcsw;
145         sig->nivcsw += tsk->nivcsw;
146         sig->inblock += task_io_get_inblock(tsk);
147         sig->oublock += task_io_get_oublock(tsk);
148         task_io_accounting_add(&sig->ioac, &tsk->ioac);
149         sig->sum_sched_runtime += tsk->se.sum_exec_runtime;
150         sig->nr_threads--;
151         __unhash_process(tsk, group_dead);
152         write_sequnlock(&sig->stats_lock);
153
154         /*
155          * Do this under ->siglock, we can race with another thread
156          * doing sigqueue_free() if we have SIGQUEUE_PREALLOC signals.
157          */
158         flush_sigqueue(&tsk->pending);
159         tsk->sighand = NULL;
160         spin_unlock(&sighand->siglock);
161
162         __cleanup_sighand(sighand);
163         clear_tsk_thread_flag(tsk, TIF_SIGPENDING);
164         if (group_dead) {
165                 flush_sigqueue(&sig->shared_pending);
166                 tty_kref_put(tty);
167         }
168 }
169
170 static void delayed_put_task_struct(struct rcu_head *rhp)
171 {
172         struct task_struct *tsk = container_of(rhp, struct task_struct, rcu);
173
174         kprobe_flush_task(tsk);
175         rethook_flush_task(tsk);
176         perf_event_delayed_put(tsk);
177         trace_sched_process_free(tsk);
178         put_task_struct(tsk);
179 }
180
181 void put_task_struct_rcu_user(struct task_struct *task)
182 {
183         if (refcount_dec_and_test(&task->rcu_users))
184                 call_rcu(&task->rcu, delayed_put_task_struct);
185 }
186
187 void release_task(struct task_struct *p)
188 {
189         struct task_struct *leader;
190         struct pid *thread_pid;
191         int zap_leader;
192 repeat:
193         /* don't need to get the RCU readlock here - the process is dead and
194          * can't be modifying its own credentials. But shut RCU-lockdep up */
195         rcu_read_lock();
196         dec_rlimit_ucounts(task_ucounts(p), UCOUNT_RLIMIT_NPROC, 1);
197         rcu_read_unlock();
198
199         cgroup_release(p);
200
201         write_lock_irq(&tasklist_lock);
202         ptrace_release_task(p);
203         thread_pid = get_pid(p->thread_pid);
204         __exit_signal(p);
205
206         /*
207          * If we are the last non-leader member of the thread
208          * group, and the leader is zombie, then notify the
209          * group leader's parent process. (if it wants notification.)
210          */
211         zap_leader = 0;
212         leader = p->group_leader;
213         if (leader != p && thread_group_empty(leader)
214                         && leader->exit_state == EXIT_ZOMBIE) {
215                 /*
216                  * If we were the last child thread and the leader has
217                  * exited already, and the leader's parent ignores SIGCHLD,
218                  * then we are the one who should release the leader.
219                  */
220                 zap_leader = do_notify_parent(leader, leader->exit_signal);
221                 if (zap_leader)
222                         leader->exit_state = EXIT_DEAD;
223         }
224
225         write_unlock_irq(&tasklist_lock);
226         seccomp_filter_release(p);
227         proc_flush_pid(thread_pid);
228         put_pid(thread_pid);
229         release_thread(p);
230         put_task_struct_rcu_user(p);
231
232         p = leader;
233         if (unlikely(zap_leader))
234                 goto repeat;
235 }
236
237 int rcuwait_wake_up(struct rcuwait *w)
238 {
239         int ret = 0;
240         struct task_struct *task;
241
242         rcu_read_lock();
243
244         /*
245          * Order condition vs @task, such that everything prior to the load
246          * of @task is visible. This is the condition as to why the user called
247          * rcuwait_wake() in the first place. Pairs with set_current_state()
248          * barrier (A) in rcuwait_wait_event().
249          *
250          *    WAIT                WAKE
251          *    [S] tsk = current   [S] cond = true
252          *        MB (A)              MB (B)
253          *    [L] cond            [L] tsk
254          */
255         smp_mb(); /* (B) */
256
257         task = rcu_dereference(w->task);
258         if (task)
259                 ret = wake_up_process(task);
260         rcu_read_unlock();
261
262         return ret;
263 }
264 EXPORT_SYMBOL_GPL(rcuwait_wake_up);
265
266 /*
267  * Determine if a process group is "orphaned", according to the POSIX
268  * definition in 2.2.2.52.  Orphaned process groups are not to be affected
269  * by terminal-generated stop signals.  Newly orphaned process groups are
270  * to receive a SIGHUP and a SIGCONT.
271  *
272  * "I ask you, have you ever known what it is to be an orphan?"
273  */
274 static int will_become_orphaned_pgrp(struct pid *pgrp,
275                                         struct task_struct *ignored_task)
276 {
277         struct task_struct *p;
278
279         do_each_pid_task(pgrp, PIDTYPE_PGID, p) {
280                 if ((p == ignored_task) ||
281                     (p->exit_state && thread_group_empty(p)) ||
282                     is_global_init(p->real_parent))
283                         continue;
284
285                 if (task_pgrp(p->real_parent) != pgrp &&
286                     task_session(p->real_parent) == task_session(p))
287                         return 0;
288         } while_each_pid_task(pgrp, PIDTYPE_PGID, p);
289
290         return 1;
291 }
292
293 int is_current_pgrp_orphaned(void)
294 {
295         int retval;
296
297         read_lock(&tasklist_lock);
298         retval = will_become_orphaned_pgrp(task_pgrp(current), NULL);
299         read_unlock(&tasklist_lock);
300
301         return retval;
302 }
303
304 static bool has_stopped_jobs(struct pid *pgrp)
305 {
306         struct task_struct *p;
307
308         do_each_pid_task(pgrp, PIDTYPE_PGID, p) {
309                 if (p->signal->flags & SIGNAL_STOP_STOPPED)
310                         return true;
311         } while_each_pid_task(pgrp, PIDTYPE_PGID, p);
312
313         return false;
314 }
315
316 /*
317  * Check to see if any process groups have become orphaned as
318  * a result of our exiting, and if they have any stopped jobs,
319  * send them a SIGHUP and then a SIGCONT. (POSIX 3.2.2.2)
320  */
321 static void
322 kill_orphaned_pgrp(struct task_struct *tsk, struct task_struct *parent)
323 {
324         struct pid *pgrp = task_pgrp(tsk);
325         struct task_struct *ignored_task = tsk;
326
327         if (!parent)
328                 /* exit: our father is in a different pgrp than
329                  * we are and we were the only connection outside.
330                  */
331                 parent = tsk->real_parent;
332         else
333                 /* reparent: our child is in a different pgrp than
334                  * we are, and it was the only connection outside.
335                  */
336                 ignored_task = NULL;
337
338         if (task_pgrp(parent) != pgrp &&
339             task_session(parent) == task_session(tsk) &&
340             will_become_orphaned_pgrp(pgrp, ignored_task) &&
341             has_stopped_jobs(pgrp)) {
342                 __kill_pgrp_info(SIGHUP, SEND_SIG_PRIV, pgrp);
343                 __kill_pgrp_info(SIGCONT, SEND_SIG_PRIV, pgrp);
344         }
345 }
346
347 static void coredump_task_exit(struct task_struct *tsk)
348 {
349         struct core_state *core_state;
350
351         /*
352          * Serialize with any possible pending coredump.
353          * We must hold siglock around checking core_state
354          * and setting PF_POSTCOREDUMP.  The core-inducing thread
355          * will increment ->nr_threads for each thread in the
356          * group without PF_POSTCOREDUMP set.
357          */
358         spin_lock_irq(&tsk->sighand->siglock);
359         tsk->flags |= PF_POSTCOREDUMP;
360         core_state = tsk->signal->core_state;
361         spin_unlock_irq(&tsk->sighand->siglock);
362         if (core_state) {
363                 struct core_thread self;
364
365                 self.task = current;
366                 if (self.task->flags & PF_SIGNALED)
367                         self.next = xchg(&core_state->dumper.next, &self);
368                 else
369                         self.task = NULL;
370                 /*
371                  * Implies mb(), the result of xchg() must be visible
372                  * to core_state->dumper.
373                  */
374                 if (atomic_dec_and_test(&core_state->nr_threads))
375                         complete(&core_state->startup);
376
377                 for (;;) {
378                         set_current_state(TASK_UNINTERRUPTIBLE|TASK_FREEZABLE);
379                         if (!self.task) /* see coredump_finish() */
380                                 break;
381                         schedule();
382                 }
383                 __set_current_state(TASK_RUNNING);
384         }
385 }
386
387 #ifdef CONFIG_MEMCG
388 /*
389  * A task is exiting.   If it owned this mm, find a new owner for the mm.
390  */
391 void mm_update_next_owner(struct mm_struct *mm)
392 {
393         struct task_struct *c, *g, *p = current;
394
395 retry:
396         /*
397          * If the exiting or execing task is not the owner, it's
398          * someone else's problem.
399          */
400         if (mm->owner != p)
401                 return;
402         /*
403          * The current owner is exiting/execing and there are no other
404          * candidates.  Do not leave the mm pointing to a possibly
405          * freed task structure.
406          */
407         if (atomic_read(&mm->mm_users) <= 1) {
408                 WRITE_ONCE(mm->owner, NULL);
409                 return;
410         }
411
412         read_lock(&tasklist_lock);
413         /*
414          * Search in the children
415          */
416         list_for_each_entry(c, &p->children, sibling) {
417                 if (c->mm == mm)
418                         goto assign_new_owner;
419         }
420
421         /*
422          * Search in the siblings
423          */
424         list_for_each_entry(c, &p->real_parent->children, sibling) {
425                 if (c->mm == mm)
426                         goto assign_new_owner;
427         }
428
429         /*
430          * Search through everything else, we should not get here often.
431          */
432         for_each_process(g) {
433                 if (g->flags & PF_KTHREAD)
434                         continue;
435                 for_each_thread(g, c) {
436                         if (c->mm == mm)
437                                 goto assign_new_owner;
438                         if (c->mm)
439                                 break;
440                 }
441         }
442         read_unlock(&tasklist_lock);
443         /*
444          * We found no owner yet mm_users > 1: this implies that we are
445          * most likely racing with swapoff (try_to_unuse()) or /proc or
446          * ptrace or page migration (get_task_mm()).  Mark owner as NULL.
447          */
448         WRITE_ONCE(mm->owner, NULL);
449         return;
450
451 assign_new_owner:
452         BUG_ON(c == p);
453         get_task_struct(c);
454         /*
455          * The task_lock protects c->mm from changing.
456          * We always want mm->owner->mm == mm
457          */
458         task_lock(c);
459         /*
460          * Delay read_unlock() till we have the task_lock()
461          * to ensure that c does not slip away underneath us
462          */
463         read_unlock(&tasklist_lock);
464         if (c->mm != mm) {
465                 task_unlock(c);
466                 put_task_struct(c);
467                 goto retry;
468         }
469         WRITE_ONCE(mm->owner, c);
470         lru_gen_migrate_mm(mm);
471         task_unlock(c);
472         put_task_struct(c);
473 }
474 #endif /* CONFIG_MEMCG */
475
476 /*
477  * Turn us into a lazy TLB process if we
478  * aren't already..
479  */
480 static void exit_mm(void)
481 {
482         struct mm_struct *mm = current->mm;
483
484         exit_mm_release(current, mm);
485         if (!mm)
486                 return;
487         sync_mm_rss(mm);
488         mmap_read_lock(mm);
489         mmgrab(mm);
490         BUG_ON(mm != current->active_mm);
491         /* more a memory barrier than a real lock */
492         task_lock(current);
493         /*
494          * When a thread stops operating on an address space, the loop
495          * in membarrier_private_expedited() may not observe that
496          * tsk->mm, and the loop in membarrier_global_expedited() may
497          * not observe a MEMBARRIER_STATE_GLOBAL_EXPEDITED
498          * rq->membarrier_state, so those would not issue an IPI.
499          * Membarrier requires a memory barrier after accessing
500          * user-space memory, before clearing tsk->mm or the
501          * rq->membarrier_state.
502          */
503         smp_mb__after_spinlock();
504         local_irq_disable();
505         current->mm = NULL;
506         membarrier_update_current_mm(NULL);
507         enter_lazy_tlb(mm, current);
508         local_irq_enable();
509         task_unlock(current);
510         mmap_read_unlock(mm);
511         mm_update_next_owner(mm);
512         mmput(mm);
513         if (test_thread_flag(TIF_MEMDIE))
514                 exit_oom_victim();
515 }
516
517 static struct task_struct *find_alive_thread(struct task_struct *p)
518 {
519         struct task_struct *t;
520
521         for_each_thread(p, t) {
522                 if (!(t->flags & PF_EXITING))
523                         return t;
524         }
525         return NULL;
526 }
527
528 static struct task_struct *find_child_reaper(struct task_struct *father,
529                                                 struct list_head *dead)
530         __releases(&tasklist_lock)
531         __acquires(&tasklist_lock)
532 {
533         struct pid_namespace *pid_ns = task_active_pid_ns(father);
534         struct task_struct *reaper = pid_ns->child_reaper;
535         struct task_struct *p, *n;
536
537         if (likely(reaper != father))
538                 return reaper;
539
540         reaper = find_alive_thread(father);
541         if (reaper) {
542                 pid_ns->child_reaper = reaper;
543                 return reaper;
544         }
545
546         write_unlock_irq(&tasklist_lock);
547
548         list_for_each_entry_safe(p, n, dead, ptrace_entry) {
549                 list_del_init(&p->ptrace_entry);
550                 release_task(p);
551         }
552
553         zap_pid_ns_processes(pid_ns);
554         write_lock_irq(&tasklist_lock);
555
556         return father;
557 }
558
559 /*
560  * When we die, we re-parent all our children, and try to:
561  * 1. give them to another thread in our thread group, if such a member exists
562  * 2. give it to the first ancestor process which prctl'd itself as a
563  *    child_subreaper for its children (like a service manager)
564  * 3. give it to the init process (PID 1) in our pid namespace
565  */
566 static struct task_struct *find_new_reaper(struct task_struct *father,
567                                            struct task_struct *child_reaper)
568 {
569         struct task_struct *thread, *reaper;
570
571         thread = find_alive_thread(father);
572         if (thread)
573                 return thread;
574
575         if (father->signal->has_child_subreaper) {
576                 unsigned int ns_level = task_pid(father)->level;
577                 /*
578                  * Find the first ->is_child_subreaper ancestor in our pid_ns.
579                  * We can't check reaper != child_reaper to ensure we do not
580                  * cross the namespaces, the exiting parent could be injected
581                  * by setns() + fork().
582                  * We check pid->level, this is slightly more efficient than
583                  * task_active_pid_ns(reaper) != task_active_pid_ns(father).
584                  */
585                 for (reaper = father->real_parent;
586                      task_pid(reaper)->level == ns_level;
587                      reaper = reaper->real_parent) {
588                         if (reaper == &init_task)
589                                 break;
590                         if (!reaper->signal->is_child_subreaper)
591                                 continue;
592                         thread = find_alive_thread(reaper);
593                         if (thread)
594                                 return thread;
595                 }
596         }
597
598         return child_reaper;
599 }
600
601 /*
602 * Any that need to be release_task'd are put on the @dead list.
603  */
604 static void reparent_leader(struct task_struct *father, struct task_struct *p,
605                                 struct list_head *dead)
606 {
607         if (unlikely(p->exit_state == EXIT_DEAD))
608                 return;
609
610         /* We don't want people slaying init. */
611         p->exit_signal = SIGCHLD;
612
613         /* If it has exited notify the new parent about this child's death. */
614         if (!p->ptrace &&
615             p->exit_state == EXIT_ZOMBIE && thread_group_empty(p)) {
616                 if (do_notify_parent(p, p->exit_signal)) {
617                         p->exit_state = EXIT_DEAD;
618                         list_add(&p->ptrace_entry, dead);
619                 }
620         }
621
622         kill_orphaned_pgrp(p, father);
623 }
624
625 /*
626  * This does two things:
627  *
628  * A.  Make init inherit all the child processes
629  * B.  Check to see if any process groups have become orphaned
630  *      as a result of our exiting, and if they have any stopped
631  *      jobs, send them a SIGHUP and then a SIGCONT.  (POSIX 3.2.2.2)
632  */
633 static void forget_original_parent(struct task_struct *father,
634                                         struct list_head *dead)
635 {
636         struct task_struct *p, *t, *reaper;
637
638         if (unlikely(!list_empty(&father->ptraced)))
639                 exit_ptrace(father, dead);
640
641         /* Can drop and reacquire tasklist_lock */
642         reaper = find_child_reaper(father, dead);
643         if (list_empty(&father->children))
644                 return;
645
646         reaper = find_new_reaper(father, reaper);
647         list_for_each_entry(p, &father->children, sibling) {
648                 for_each_thread(p, t) {
649                         RCU_INIT_POINTER(t->real_parent, reaper);
650                         BUG_ON((!t->ptrace) != (rcu_access_pointer(t->parent) == father));
651                         if (likely(!t->ptrace))
652                                 t->parent = t->real_parent;
653                         if (t->pdeath_signal)
654                                 group_send_sig_info(t->pdeath_signal,
655                                                     SEND_SIG_NOINFO, t,
656                                                     PIDTYPE_TGID);
657                 }
658                 /*
659                  * If this is a threaded reparent there is no need to
660                  * notify anyone anything has happened.
661                  */
662                 if (!same_thread_group(reaper, father))
663                         reparent_leader(father, p, dead);
664         }
665         list_splice_tail_init(&father->children, &reaper->children);
666 }
667
668 /*
669  * Send signals to all our closest relatives so that they know
670  * to properly mourn us..
671  */
672 static void exit_notify(struct task_struct *tsk, int group_dead)
673 {
674         bool autoreap;
675         struct task_struct *p, *n;
676         LIST_HEAD(dead);
677
678         write_lock_irq(&tasklist_lock);
679         forget_original_parent(tsk, &dead);
680
681         if (group_dead)
682                 kill_orphaned_pgrp(tsk->group_leader, NULL);
683
684         tsk->exit_state = EXIT_ZOMBIE;
685         if (unlikely(tsk->ptrace)) {
686                 int sig = thread_group_leader(tsk) &&
687                                 thread_group_empty(tsk) &&
688                                 !ptrace_reparented(tsk) ?
689                         tsk->exit_signal : SIGCHLD;
690                 autoreap = do_notify_parent(tsk, sig);
691         } else if (thread_group_leader(tsk)) {
692                 autoreap = thread_group_empty(tsk) &&
693                         do_notify_parent(tsk, tsk->exit_signal);
694         } else {
695                 autoreap = true;
696         }
697
698         if (autoreap) {
699                 tsk->exit_state = EXIT_DEAD;
700                 list_add(&tsk->ptrace_entry, &dead);
701         }
702
703         /* mt-exec, de_thread() is waiting for group leader */
704         if (unlikely(tsk->signal->notify_count < 0))
705                 wake_up_process(tsk->signal->group_exec_task);
706         write_unlock_irq(&tasklist_lock);
707
708         list_for_each_entry_safe(p, n, &dead, ptrace_entry) {
709                 list_del_init(&p->ptrace_entry);
710                 release_task(p);
711         }
712 }
713
714 #ifdef CONFIG_DEBUG_STACK_USAGE
715 static void check_stack_usage(void)
716 {
717         static DEFINE_SPINLOCK(low_water_lock);
718         static int lowest_to_date = THREAD_SIZE;
719         unsigned long free;
720
721         free = stack_not_used(current);
722
723         if (free >= lowest_to_date)
724                 return;
725
726         spin_lock(&low_water_lock);
727         if (free < lowest_to_date) {
728                 pr_info("%s (%d) used greatest stack depth: %lu bytes left\n",
729                         current->comm, task_pid_nr(current), free);
730                 lowest_to_date = free;
731         }
732         spin_unlock(&low_water_lock);
733 }
734 #else
735 static inline void check_stack_usage(void) {}
736 #endif
737
738 static void synchronize_group_exit(struct task_struct *tsk, long code)
739 {
740         struct sighand_struct *sighand = tsk->sighand;
741         struct signal_struct *signal = tsk->signal;
742
743         spin_lock_irq(&sighand->siglock);
744         signal->quick_threads--;
745         if ((signal->quick_threads == 0) &&
746             !(signal->flags & SIGNAL_GROUP_EXIT)) {
747                 signal->flags = SIGNAL_GROUP_EXIT;
748                 signal->group_exit_code = code;
749                 signal->group_stop_count = 0;
750         }
751         spin_unlock_irq(&sighand->siglock);
752 }
753
754 void __noreturn do_exit(long code)
755 {
756         struct task_struct *tsk = current;
757         int group_dead;
758
759         synchronize_group_exit(tsk, code);
760
761         WARN_ON(tsk->plug);
762
763         kcov_task_exit(tsk);
764         kmsan_task_exit(tsk);
765
766         coredump_task_exit(tsk);
767         ptrace_event(PTRACE_EVENT_EXIT, code);
768
769         validate_creds_for_do_exit(tsk);
770
771         io_uring_files_cancel();
772         exit_signals(tsk);  /* sets PF_EXITING */
773
774         /* sync mm's RSS info before statistics gathering */
775         if (tsk->mm)
776                 sync_mm_rss(tsk->mm);
777         acct_update_integrals(tsk);
778         group_dead = atomic_dec_and_test(&tsk->signal->live);
779         if (group_dead) {
780                 /*
781                  * If the last thread of global init has exited, panic
782                  * immediately to get a useable coredump.
783                  */
784                 if (unlikely(is_global_init(tsk)))
785                         panic("Attempted to kill init! exitcode=0x%08x\n",
786                                 tsk->signal->group_exit_code ?: (int)code);
787
788 #ifdef CONFIG_POSIX_TIMERS
789                 hrtimer_cancel(&tsk->signal->real_timer);
790                 exit_itimers(tsk);
791 #endif
792                 if (tsk->mm)
793                         setmax_mm_hiwater_rss(&tsk->signal->maxrss, tsk->mm);
794         }
795         acct_collect(code, group_dead);
796         if (group_dead)
797                 tty_audit_exit();
798         audit_free(tsk);
799
800         tsk->exit_code = code;
801         taskstats_exit(tsk, group_dead);
802
803         exit_mm();
804
805         if (group_dead)
806                 acct_process();
807         trace_sched_process_exit(tsk);
808
809         exit_sem(tsk);
810         exit_shm(tsk);
811         exit_files(tsk);
812         exit_fs(tsk);
813         if (group_dead)
814                 disassociate_ctty(1);
815         exit_task_namespaces(tsk);
816         exit_task_work(tsk);
817         exit_thread(tsk);
818
819         /*
820          * Flush inherited counters to the parent - before the parent
821          * gets woken up by child-exit notifications.
822          *
823          * because of cgroup mode, must be called before cgroup_exit()
824          */
825         perf_event_exit_task(tsk);
826
827         sched_autogroup_exit_task(tsk);
828         cgroup_exit(tsk);
829
830         /*
831          * FIXME: do that only when needed, using sched_exit tracepoint
832          */
833         flush_ptrace_hw_breakpoint(tsk);
834
835         exit_tasks_rcu_start();
836         exit_notify(tsk, group_dead);
837         proc_exit_connector(tsk);
838         mpol_put_task_policy(tsk);
839 #ifdef CONFIG_FUTEX
840         if (unlikely(current->pi_state_cache))
841                 kfree(current->pi_state_cache);
842 #endif
843         /*
844          * Make sure we are holding no locks:
845          */
846         debug_check_no_locks_held();
847
848         if (tsk->io_context)
849                 exit_io_context(tsk);
850
851         if (tsk->splice_pipe)
852                 free_pipe_info(tsk->splice_pipe);
853
854         if (tsk->task_frag.page)
855                 put_page(tsk->task_frag.page);
856
857         validate_creds_for_do_exit(tsk);
858         exit_task_stack_account(tsk);
859
860         check_stack_usage();
861         preempt_disable();
862         if (tsk->nr_dirtied)
863                 __this_cpu_add(dirty_throttle_leaks, tsk->nr_dirtied);
864         exit_rcu();
865         exit_tasks_rcu_finish();
866
867         lockdep_free_task(tsk);
868         do_task_dead();
869 }
870
871 void __noreturn make_task_dead(int signr)
872 {
873         /*
874          * Take the task off the cpu after something catastrophic has
875          * happened.
876          *
877          * We can get here from a kernel oops, sometimes with preemption off.
878          * Start by checking for critical errors.
879          * Then fix up important state like USER_DS and preemption.
880          * Then do everything else.
881          */
882         struct task_struct *tsk = current;
883
884         if (unlikely(in_interrupt()))
885                 panic("Aiee, killing interrupt handler!");
886         if (unlikely(!tsk->pid))
887                 panic("Attempted to kill the idle task!");
888
889         if (unlikely(in_atomic())) {
890                 pr_info("note: %s[%d] exited with preempt_count %d\n",
891                         current->comm, task_pid_nr(current),
892                         preempt_count());
893                 preempt_count_set(PREEMPT_ENABLED);
894         }
895
896         /*
897          * We're taking recursive faults here in make_task_dead. Safest is to just
898          * leave this task alone and wait for reboot.
899          */
900         if (unlikely(tsk->flags & PF_EXITING)) {
901                 pr_alert("Fixing recursive fault but reboot is needed!\n");
902                 futex_exit_recursive(tsk);
903                 tsk->exit_state = EXIT_DEAD;
904                 refcount_inc(&tsk->rcu_users);
905                 do_task_dead();
906         }
907
908         do_exit(signr);
909 }
910
911 SYSCALL_DEFINE1(exit, int, error_code)
912 {
913         do_exit((error_code&0xff)<<8);
914 }
915
916 /*
917  * Take down every thread in the group.  This is called by fatal signals
918  * as well as by sys_exit_group (below).
919  */
920 void __noreturn
921 do_group_exit(int exit_code)
922 {
923         struct signal_struct *sig = current->signal;
924
925         if (sig->flags & SIGNAL_GROUP_EXIT)
926                 exit_code = sig->group_exit_code;
927         else if (sig->group_exec_task)
928                 exit_code = 0;
929         else {
930                 struct sighand_struct *const sighand = current->sighand;
931
932                 spin_lock_irq(&sighand->siglock);
933                 if (sig->flags & SIGNAL_GROUP_EXIT)
934                         /* Another thread got here before we took the lock.  */
935                         exit_code = sig->group_exit_code;
936                 else if (sig->group_exec_task)
937                         exit_code = 0;
938                 else {
939                         sig->group_exit_code = exit_code;
940                         sig->flags = SIGNAL_GROUP_EXIT;
941                         zap_other_threads(current);
942                 }
943                 spin_unlock_irq(&sighand->siglock);
944         }
945
946         do_exit(exit_code);
947         /* NOTREACHED */
948 }
949
950 /*
951  * this kills every thread in the thread group. Note that any externally
952  * wait4()-ing process will get the correct exit code - even if this
953  * thread is not the thread group leader.
954  */
955 SYSCALL_DEFINE1(exit_group, int, error_code)
956 {
957         do_group_exit((error_code & 0xff) << 8);
958         /* NOTREACHED */
959         return 0;
960 }
961
962 struct waitid_info {
963         pid_t pid;
964         uid_t uid;
965         int status;
966         int cause;
967 };
968
969 struct wait_opts {
970         enum pid_type           wo_type;
971         int                     wo_flags;
972         struct pid              *wo_pid;
973
974         struct waitid_info      *wo_info;
975         int                     wo_stat;
976         struct rusage           *wo_rusage;
977
978         wait_queue_entry_t              child_wait;
979         int                     notask_error;
980 };
981
982 static int eligible_pid(struct wait_opts *wo, struct task_struct *p)
983 {
984         return  wo->wo_type == PIDTYPE_MAX ||
985                 task_pid_type(p, wo->wo_type) == wo->wo_pid;
986 }
987
988 static int
989 eligible_child(struct wait_opts *wo, bool ptrace, struct task_struct *p)
990 {
991         if (!eligible_pid(wo, p))
992                 return 0;
993
994         /*
995          * Wait for all children (clone and not) if __WALL is set or
996          * if it is traced by us.
997          */
998         if (ptrace || (wo->wo_flags & __WALL))
999                 return 1;
1000
1001         /*
1002          * Otherwise, wait for clone children *only* if __WCLONE is set;
1003          * otherwise, wait for non-clone children *only*.
1004          *
1005          * Note: a "clone" child here is one that reports to its parent
1006          * using a signal other than SIGCHLD, or a non-leader thread which
1007          * we can only see if it is traced by us.
1008          */
1009         if ((p->exit_signal != SIGCHLD) ^ !!(wo->wo_flags & __WCLONE))
1010                 return 0;
1011
1012         return 1;
1013 }
1014
1015 /*
1016  * Handle sys_wait4 work for one task in state EXIT_ZOMBIE.  We hold
1017  * read_lock(&tasklist_lock) on entry.  If we return zero, we still hold
1018  * the lock and this task is uninteresting.  If we return nonzero, we have
1019  * released the lock and the system call should return.
1020  */
1021 static int wait_task_zombie(struct wait_opts *wo, struct task_struct *p)
1022 {
1023         int state, status;
1024         pid_t pid = task_pid_vnr(p);
1025         uid_t uid = from_kuid_munged(current_user_ns(), task_uid(p));
1026         struct waitid_info *infop;
1027
1028         if (!likely(wo->wo_flags & WEXITED))
1029                 return 0;
1030
1031         if (unlikely(wo->wo_flags & WNOWAIT)) {
1032                 status = (p->signal->flags & SIGNAL_GROUP_EXIT)
1033                         ? p->signal->group_exit_code : p->exit_code;
1034                 get_task_struct(p);
1035                 read_unlock(&tasklist_lock);
1036                 sched_annotate_sleep();
1037                 if (wo->wo_rusage)
1038                         getrusage(p, RUSAGE_BOTH, wo->wo_rusage);
1039                 put_task_struct(p);
1040                 goto out_info;
1041         }
1042         /*
1043          * Move the task's state to DEAD/TRACE, only one thread can do this.
1044          */
1045         state = (ptrace_reparented(p) && thread_group_leader(p)) ?
1046                 EXIT_TRACE : EXIT_DEAD;
1047         if (cmpxchg(&p->exit_state, EXIT_ZOMBIE, state) != EXIT_ZOMBIE)
1048                 return 0;
1049         /*
1050          * We own this thread, nobody else can reap it.
1051          */
1052         read_unlock(&tasklist_lock);
1053         sched_annotate_sleep();
1054
1055         /*
1056          * Check thread_group_leader() to exclude the traced sub-threads.
1057          */
1058         if (state == EXIT_DEAD && thread_group_leader(p)) {
1059                 struct signal_struct *sig = p->signal;
1060                 struct signal_struct *psig = current->signal;
1061                 unsigned long maxrss;
1062                 u64 tgutime, tgstime;
1063
1064                 /*
1065                  * The resource counters for the group leader are in its
1066                  * own task_struct.  Those for dead threads in the group
1067                  * are in its signal_struct, as are those for the child
1068                  * processes it has previously reaped.  All these
1069                  * accumulate in the parent's signal_struct c* fields.
1070                  *
1071                  * We don't bother to take a lock here to protect these
1072                  * p->signal fields because the whole thread group is dead
1073                  * and nobody can change them.
1074                  *
1075                  * psig->stats_lock also protects us from our sub-threads
1076                  * which can reap other children at the same time. Until
1077                  * we change k_getrusage()-like users to rely on this lock
1078                  * we have to take ->siglock as well.
1079                  *
1080                  * We use thread_group_cputime_adjusted() to get times for
1081                  * the thread group, which consolidates times for all threads
1082                  * in the group including the group leader.
1083                  */
1084                 thread_group_cputime_adjusted(p, &tgutime, &tgstime);
1085                 spin_lock_irq(&current->sighand->siglock);
1086                 write_seqlock(&psig->stats_lock);
1087                 psig->cutime += tgutime + sig->cutime;
1088                 psig->cstime += tgstime + sig->cstime;
1089                 psig->cgtime += task_gtime(p) + sig->gtime + sig->cgtime;
1090                 psig->cmin_flt +=
1091                         p->min_flt + sig->min_flt + sig->cmin_flt;
1092                 psig->cmaj_flt +=
1093                         p->maj_flt + sig->maj_flt + sig->cmaj_flt;
1094                 psig->cnvcsw +=
1095                         p->nvcsw + sig->nvcsw + sig->cnvcsw;
1096                 psig->cnivcsw +=
1097                         p->nivcsw + sig->nivcsw + sig->cnivcsw;
1098                 psig->cinblock +=
1099                         task_io_get_inblock(p) +
1100                         sig->inblock + sig->cinblock;
1101                 psig->coublock +=
1102                         task_io_get_oublock(p) +
1103                         sig->oublock + sig->coublock;
1104                 maxrss = max(sig->maxrss, sig->cmaxrss);
1105                 if (psig->cmaxrss < maxrss)
1106                         psig->cmaxrss = maxrss;
1107                 task_io_accounting_add(&psig->ioac, &p->ioac);
1108                 task_io_accounting_add(&psig->ioac, &sig->ioac);
1109                 write_sequnlock(&psig->stats_lock);
1110                 spin_unlock_irq(&current->sighand->siglock);
1111         }
1112
1113         if (wo->wo_rusage)
1114                 getrusage(p, RUSAGE_BOTH, wo->wo_rusage);
1115         status = (p->signal->flags & SIGNAL_GROUP_EXIT)
1116                 ? p->signal->group_exit_code : p->exit_code;
1117         wo->wo_stat = status;
1118
1119         if (state == EXIT_TRACE) {
1120                 write_lock_irq(&tasklist_lock);
1121                 /* We dropped tasklist, ptracer could die and untrace */
1122                 ptrace_unlink(p);
1123
1124                 /* If parent wants a zombie, don't release it now */
1125                 state = EXIT_ZOMBIE;
1126                 if (do_notify_parent(p, p->exit_signal))
1127                         state = EXIT_DEAD;
1128                 p->exit_state = state;
1129                 write_unlock_irq(&tasklist_lock);
1130         }
1131         if (state == EXIT_DEAD)
1132                 release_task(p);
1133
1134 out_info:
1135         infop = wo->wo_info;
1136         if (infop) {
1137                 if ((status & 0x7f) == 0) {
1138                         infop->cause = CLD_EXITED;
1139                         infop->status = status >> 8;
1140                 } else {
1141                         infop->cause = (status & 0x80) ? CLD_DUMPED : CLD_KILLED;
1142                         infop->status = status & 0x7f;
1143                 }
1144                 infop->pid = pid;
1145                 infop->uid = uid;
1146         }
1147
1148         return pid;
1149 }
1150
1151 static int *task_stopped_code(struct task_struct *p, bool ptrace)
1152 {
1153         if (ptrace) {
1154                 if (task_is_traced(p) && !(p->jobctl & JOBCTL_LISTENING))
1155                         return &p->exit_code;
1156         } else {
1157                 if (p->signal->flags & SIGNAL_STOP_STOPPED)
1158                         return &p->signal->group_exit_code;
1159         }
1160         return NULL;
1161 }
1162
1163 /**
1164  * wait_task_stopped - Wait for %TASK_STOPPED or %TASK_TRACED
1165  * @wo: wait options
1166  * @ptrace: is the wait for ptrace
1167  * @p: task to wait for
1168  *
1169  * Handle sys_wait4() work for %p in state %TASK_STOPPED or %TASK_TRACED.
1170  *
1171  * CONTEXT:
1172  * read_lock(&tasklist_lock), which is released if return value is
1173  * non-zero.  Also, grabs and releases @p->sighand->siglock.
1174  *
1175  * RETURNS:
1176  * 0 if wait condition didn't exist and search for other wait conditions
1177  * should continue.  Non-zero return, -errno on failure and @p's pid on
1178  * success, implies that tasklist_lock is released and wait condition
1179  * search should terminate.
1180  */
1181 static int wait_task_stopped(struct wait_opts *wo,
1182                                 int ptrace, struct task_struct *p)
1183 {
1184         struct waitid_info *infop;
1185         int exit_code, *p_code, why;
1186         uid_t uid = 0; /* unneeded, required by compiler */
1187         pid_t pid;
1188
1189         /*
1190          * Traditionally we see ptrace'd stopped tasks regardless of options.
1191          */
1192         if (!ptrace && !(wo->wo_flags & WUNTRACED))
1193                 return 0;
1194
1195         if (!task_stopped_code(p, ptrace))
1196                 return 0;
1197
1198         exit_code = 0;
1199         spin_lock_irq(&p->sighand->siglock);
1200
1201         p_code = task_stopped_code(p, ptrace);
1202         if (unlikely(!p_code))
1203                 goto unlock_sig;
1204
1205         exit_code = *p_code;
1206         if (!exit_code)
1207                 goto unlock_sig;
1208
1209         if (!unlikely(wo->wo_flags & WNOWAIT))
1210                 *p_code = 0;
1211
1212         uid = from_kuid_munged(current_user_ns(), task_uid(p));
1213 unlock_sig:
1214         spin_unlock_irq(&p->sighand->siglock);
1215         if (!exit_code)
1216                 return 0;
1217
1218         /*
1219          * Now we are pretty sure this task is interesting.
1220          * Make sure it doesn't get reaped out from under us while we
1221          * give up the lock and then examine it below.  We don't want to
1222          * keep holding onto the tasklist_lock while we call getrusage and
1223          * possibly take page faults for user memory.
1224          */
1225         get_task_struct(p);
1226         pid = task_pid_vnr(p);
1227         why = ptrace ? CLD_TRAPPED : CLD_STOPPED;
1228         read_unlock(&tasklist_lock);
1229         sched_annotate_sleep();
1230         if (wo->wo_rusage)
1231                 getrusage(p, RUSAGE_BOTH, wo->wo_rusage);
1232         put_task_struct(p);
1233
1234         if (likely(!(wo->wo_flags & WNOWAIT)))
1235                 wo->wo_stat = (exit_code << 8) | 0x7f;
1236
1237         infop = wo->wo_info;
1238         if (infop) {
1239                 infop->cause = why;
1240                 infop->status = exit_code;
1241                 infop->pid = pid;
1242                 infop->uid = uid;
1243         }
1244         return pid;
1245 }
1246
1247 /*
1248  * Handle do_wait work for one task in a live, non-stopped state.
1249  * read_lock(&tasklist_lock) on entry.  If we return zero, we still hold
1250  * the lock and this task is uninteresting.  If we return nonzero, we have
1251  * released the lock and the system call should return.
1252  */
1253 static int wait_task_continued(struct wait_opts *wo, struct task_struct *p)
1254 {
1255         struct waitid_info *infop;
1256         pid_t pid;
1257         uid_t uid;
1258
1259         if (!unlikely(wo->wo_flags & WCONTINUED))
1260                 return 0;
1261
1262         if (!(p->signal->flags & SIGNAL_STOP_CONTINUED))
1263                 return 0;
1264
1265         spin_lock_irq(&p->sighand->siglock);
1266         /* Re-check with the lock held.  */
1267         if (!(p->signal->flags & SIGNAL_STOP_CONTINUED)) {
1268                 spin_unlock_irq(&p->sighand->siglock);
1269                 return 0;
1270         }
1271         if (!unlikely(wo->wo_flags & WNOWAIT))
1272                 p->signal->flags &= ~SIGNAL_STOP_CONTINUED;
1273         uid = from_kuid_munged(current_user_ns(), task_uid(p));
1274         spin_unlock_irq(&p->sighand->siglock);
1275
1276         pid = task_pid_vnr(p);
1277         get_task_struct(p);
1278         read_unlock(&tasklist_lock);
1279         sched_annotate_sleep();
1280         if (wo->wo_rusage)
1281                 getrusage(p, RUSAGE_BOTH, wo->wo_rusage);
1282         put_task_struct(p);
1283
1284         infop = wo->wo_info;
1285         if (!infop) {
1286                 wo->wo_stat = 0xffff;
1287         } else {
1288                 infop->cause = CLD_CONTINUED;
1289                 infop->pid = pid;
1290                 infop->uid = uid;
1291                 infop->status = SIGCONT;
1292         }
1293         return pid;
1294 }
1295
1296 /*
1297  * Consider @p for a wait by @parent.
1298  *
1299  * -ECHILD should be in ->notask_error before the first call.
1300  * Returns nonzero for a final return, when we have unlocked tasklist_lock.
1301  * Returns zero if the search for a child should continue;
1302  * then ->notask_error is 0 if @p is an eligible child,
1303  * or still -ECHILD.
1304  */
1305 static int wait_consider_task(struct wait_opts *wo, int ptrace,
1306                                 struct task_struct *p)
1307 {
1308         /*
1309          * We can race with wait_task_zombie() from another thread.
1310          * Ensure that EXIT_ZOMBIE -> EXIT_DEAD/EXIT_TRACE transition
1311          * can't confuse the checks below.
1312          */
1313         int exit_state = READ_ONCE(p->exit_state);
1314         int ret;
1315
1316         if (unlikely(exit_state == EXIT_DEAD))
1317                 return 0;
1318
1319         ret = eligible_child(wo, ptrace, p);
1320         if (!ret)
1321                 return ret;
1322
1323         if (unlikely(exit_state == EXIT_TRACE)) {
1324                 /*
1325                  * ptrace == 0 means we are the natural parent. In this case
1326                  * we should clear notask_error, debugger will notify us.
1327                  */
1328                 if (likely(!ptrace))
1329                         wo->notask_error = 0;
1330                 return 0;
1331         }
1332
1333         if (likely(!ptrace) && unlikely(p->ptrace)) {
1334                 /*
1335                  * If it is traced by its real parent's group, just pretend
1336                  * the caller is ptrace_do_wait() and reap this child if it
1337                  * is zombie.
1338                  *
1339                  * This also hides group stop state from real parent; otherwise
1340                  * a single stop can be reported twice as group and ptrace stop.
1341                  * If a ptracer wants to distinguish these two events for its
1342                  * own children it should create a separate process which takes
1343                  * the role of real parent.
1344                  */
1345                 if (!ptrace_reparented(p))
1346                         ptrace = 1;
1347         }
1348
1349         /* slay zombie? */
1350         if (exit_state == EXIT_ZOMBIE) {
1351                 /* we don't reap group leaders with subthreads */
1352                 if (!delay_group_leader(p)) {
1353                         /*
1354                          * A zombie ptracee is only visible to its ptracer.
1355                          * Notification and reaping will be cascaded to the
1356                          * real parent when the ptracer detaches.
1357                          */
1358                         if (unlikely(ptrace) || likely(!p->ptrace))
1359                                 return wait_task_zombie(wo, p);
1360                 }
1361
1362                 /*
1363                  * Allow access to stopped/continued state via zombie by
1364                  * falling through.  Clearing of notask_error is complex.
1365                  *
1366                  * When !@ptrace:
1367                  *
1368                  * If WEXITED is set, notask_error should naturally be
1369                  * cleared.  If not, subset of WSTOPPED|WCONTINUED is set,
1370                  * so, if there are live subthreads, there are events to
1371                  * wait for.  If all subthreads are dead, it's still safe
1372                  * to clear - this function will be called again in finite
1373                  * amount time once all the subthreads are released and
1374                  * will then return without clearing.
1375                  *
1376                  * When @ptrace:
1377                  *
1378                  * Stopped state is per-task and thus can't change once the
1379                  * target task dies.  Only continued and exited can happen.
1380                  * Clear notask_error if WCONTINUED | WEXITED.
1381                  */
1382                 if (likely(!ptrace) || (wo->wo_flags & (WCONTINUED | WEXITED)))
1383                         wo->notask_error = 0;
1384         } else {
1385                 /*
1386                  * @p is alive and it's gonna stop, continue or exit, so
1387                  * there always is something to wait for.
1388                  */
1389                 wo->notask_error = 0;
1390         }
1391
1392         /*
1393          * Wait for stopped.  Depending on @ptrace, different stopped state
1394          * is used and the two don't interact with each other.
1395          */
1396         ret = wait_task_stopped(wo, ptrace, p);
1397         if (ret)
1398                 return ret;
1399
1400         /*
1401          * Wait for continued.  There's only one continued state and the
1402          * ptracer can consume it which can confuse the real parent.  Don't
1403          * use WCONTINUED from ptracer.  You don't need or want it.
1404          */
1405         return wait_task_continued(wo, p);
1406 }
1407
1408 /*
1409  * Do the work of do_wait() for one thread in the group, @tsk.
1410  *
1411  * -ECHILD should be in ->notask_error before the first call.
1412  * Returns nonzero for a final return, when we have unlocked tasklist_lock.
1413  * Returns zero if the search for a child should continue; then
1414  * ->notask_error is 0 if there were any eligible children,
1415  * or still -ECHILD.
1416  */
1417 static int do_wait_thread(struct wait_opts *wo, struct task_struct *tsk)
1418 {
1419         struct task_struct *p;
1420
1421         list_for_each_entry(p, &tsk->children, sibling) {
1422                 int ret = wait_consider_task(wo, 0, p);
1423
1424                 if (ret)
1425                         return ret;
1426         }
1427
1428         return 0;
1429 }
1430
1431 static int ptrace_do_wait(struct wait_opts *wo, struct task_struct *tsk)
1432 {
1433         struct task_struct *p;
1434
1435         list_for_each_entry(p, &tsk->ptraced, ptrace_entry) {
1436                 int ret = wait_consider_task(wo, 1, p);
1437
1438                 if (ret)
1439                         return ret;
1440         }
1441
1442         return 0;
1443 }
1444
1445 static int child_wait_callback(wait_queue_entry_t *wait, unsigned mode,
1446                                 int sync, void *key)
1447 {
1448         struct wait_opts *wo = container_of(wait, struct wait_opts,
1449                                                 child_wait);
1450         struct task_struct *p = key;
1451
1452         if (!eligible_pid(wo, p))
1453                 return 0;
1454
1455         if ((wo->wo_flags & __WNOTHREAD) && wait->private != p->parent)
1456                 return 0;
1457
1458         return default_wake_function(wait, mode, sync, key);
1459 }
1460
1461 void __wake_up_parent(struct task_struct *p, struct task_struct *parent)
1462 {
1463         __wake_up_sync_key(&parent->signal->wait_chldexit,
1464                            TASK_INTERRUPTIBLE, p);
1465 }
1466
1467 static bool is_effectively_child(struct wait_opts *wo, bool ptrace,
1468                                  struct task_struct *target)
1469 {
1470         struct task_struct *parent =
1471                 !ptrace ? target->real_parent : target->parent;
1472
1473         return current == parent || (!(wo->wo_flags & __WNOTHREAD) &&
1474                                      same_thread_group(current, parent));
1475 }
1476
1477 /*
1478  * Optimization for waiting on PIDTYPE_PID. No need to iterate through child
1479  * and tracee lists to find the target task.
1480  */
1481 static int do_wait_pid(struct wait_opts *wo)
1482 {
1483         bool ptrace;
1484         struct task_struct *target;
1485         int retval;
1486
1487         ptrace = false;
1488         target = pid_task(wo->wo_pid, PIDTYPE_TGID);
1489         if (target && is_effectively_child(wo, ptrace, target)) {
1490                 retval = wait_consider_task(wo, ptrace, target);
1491                 if (retval)
1492                         return retval;
1493         }
1494
1495         ptrace = true;
1496         target = pid_task(wo->wo_pid, PIDTYPE_PID);
1497         if (target && target->ptrace &&
1498             is_effectively_child(wo, ptrace, target)) {
1499                 retval = wait_consider_task(wo, ptrace, target);
1500                 if (retval)
1501                         return retval;
1502         }
1503
1504         return 0;
1505 }
1506
1507 static long do_wait(struct wait_opts *wo)
1508 {
1509         int retval;
1510
1511         trace_sched_process_wait(wo->wo_pid);
1512
1513         init_waitqueue_func_entry(&wo->child_wait, child_wait_callback);
1514         wo->child_wait.private = current;
1515         add_wait_queue(&current->signal->wait_chldexit, &wo->child_wait);
1516 repeat:
1517         /*
1518          * If there is nothing that can match our criteria, just get out.
1519          * We will clear ->notask_error to zero if we see any child that
1520          * might later match our criteria, even if we are not able to reap
1521          * it yet.
1522          */
1523         wo->notask_error = -ECHILD;
1524         if ((wo->wo_type < PIDTYPE_MAX) &&
1525            (!wo->wo_pid || !pid_has_task(wo->wo_pid, wo->wo_type)))
1526                 goto notask;
1527
1528         set_current_state(TASK_INTERRUPTIBLE);
1529         read_lock(&tasklist_lock);
1530
1531         if (wo->wo_type == PIDTYPE_PID) {
1532                 retval = do_wait_pid(wo);
1533                 if (retval)
1534                         goto end;
1535         } else {
1536                 struct task_struct *tsk = current;
1537
1538                 do {
1539                         retval = do_wait_thread(wo, tsk);
1540                         if (retval)
1541                                 goto end;
1542
1543                         retval = ptrace_do_wait(wo, tsk);
1544                         if (retval)
1545                                 goto end;
1546
1547                         if (wo->wo_flags & __WNOTHREAD)
1548                                 break;
1549                 } while_each_thread(current, tsk);
1550         }
1551         read_unlock(&tasklist_lock);
1552
1553 notask:
1554         retval = wo->notask_error;
1555         if (!retval && !(wo->wo_flags & WNOHANG)) {
1556                 retval = -ERESTARTSYS;
1557                 if (!signal_pending(current)) {
1558                         schedule();
1559                         goto repeat;
1560                 }
1561         }
1562 end:
1563         __set_current_state(TASK_RUNNING);
1564         remove_wait_queue(&current->signal->wait_chldexit, &wo->child_wait);
1565         return retval;
1566 }
1567
1568 static long kernel_waitid(int which, pid_t upid, struct waitid_info *infop,
1569                           int options, struct rusage *ru)
1570 {
1571         struct wait_opts wo;
1572         struct pid *pid = NULL;
1573         enum pid_type type;
1574         long ret;
1575         unsigned int f_flags = 0;
1576
1577         if (options & ~(WNOHANG|WNOWAIT|WEXITED|WSTOPPED|WCONTINUED|
1578                         __WNOTHREAD|__WCLONE|__WALL))
1579                 return -EINVAL;
1580         if (!(options & (WEXITED|WSTOPPED|WCONTINUED)))
1581                 return -EINVAL;
1582
1583         switch (which) {
1584         case P_ALL:
1585                 type = PIDTYPE_MAX;
1586                 break;
1587         case P_PID:
1588                 type = PIDTYPE_PID;
1589                 if (upid <= 0)
1590                         return -EINVAL;
1591
1592                 pid = find_get_pid(upid);
1593                 break;
1594         case P_PGID:
1595                 type = PIDTYPE_PGID;
1596                 if (upid < 0)
1597                         return -EINVAL;
1598
1599                 if (upid)
1600                         pid = find_get_pid(upid);
1601                 else
1602                         pid = get_task_pid(current, PIDTYPE_PGID);
1603                 break;
1604         case P_PIDFD:
1605                 type = PIDTYPE_PID;
1606                 if (upid < 0)
1607                         return -EINVAL;
1608
1609                 pid = pidfd_get_pid(upid, &f_flags);
1610                 if (IS_ERR(pid))
1611                         return PTR_ERR(pid);
1612
1613                 break;
1614         default:
1615                 return -EINVAL;
1616         }
1617
1618         wo.wo_type      = type;
1619         wo.wo_pid       = pid;
1620         wo.wo_flags     = options;
1621         wo.wo_info      = infop;
1622         wo.wo_rusage    = ru;
1623         if (f_flags & O_NONBLOCK)
1624                 wo.wo_flags |= WNOHANG;
1625
1626         ret = do_wait(&wo);
1627         if (!ret && !(options & WNOHANG) && (f_flags & O_NONBLOCK))
1628                 ret = -EAGAIN;
1629
1630         put_pid(pid);
1631         return ret;
1632 }
1633
1634 SYSCALL_DEFINE5(waitid, int, which, pid_t, upid, struct siginfo __user *,
1635                 infop, int, options, struct rusage __user *, ru)
1636 {
1637         struct rusage r;
1638         struct waitid_info info = {.status = 0};
1639         long err = kernel_waitid(which, upid, &info, options, ru ? &r : NULL);
1640         int signo = 0;
1641
1642         if (err > 0) {
1643                 signo = SIGCHLD;
1644                 err = 0;
1645                 if (ru && copy_to_user(ru, &r, sizeof(struct rusage)))
1646                         return -EFAULT;
1647         }
1648         if (!infop)
1649                 return err;
1650
1651         if (!user_write_access_begin(infop, sizeof(*infop)))
1652                 return -EFAULT;
1653
1654         unsafe_put_user(signo, &infop->si_signo, Efault);
1655         unsafe_put_user(0, &infop->si_errno, Efault);
1656         unsafe_put_user(info.cause, &infop->si_code, Efault);
1657         unsafe_put_user(info.pid, &infop->si_pid, Efault);
1658         unsafe_put_user(info.uid, &infop->si_uid, Efault);
1659         unsafe_put_user(info.status, &infop->si_status, Efault);
1660         user_write_access_end();
1661         return err;
1662 Efault:
1663         user_write_access_end();
1664         return -EFAULT;
1665 }
1666
1667 long kernel_wait4(pid_t upid, int __user *stat_addr, int options,
1668                   struct rusage *ru)
1669 {
1670         struct wait_opts wo;
1671         struct pid *pid = NULL;
1672         enum pid_type type;
1673         long ret;
1674
1675         if (options & ~(WNOHANG|WUNTRACED|WCONTINUED|
1676                         __WNOTHREAD|__WCLONE|__WALL))
1677                 return -EINVAL;
1678
1679         /* -INT_MIN is not defined */
1680         if (upid == INT_MIN)
1681                 return -ESRCH;
1682
1683         if (upid == -1)
1684                 type = PIDTYPE_MAX;
1685         else if (upid < 0) {
1686                 type = PIDTYPE_PGID;
1687                 pid = find_get_pid(-upid);
1688         } else if (upid == 0) {
1689                 type = PIDTYPE_PGID;
1690                 pid = get_task_pid(current, PIDTYPE_PGID);
1691         } else /* upid > 0 */ {
1692                 type = PIDTYPE_PID;
1693                 pid = find_get_pid(upid);
1694         }
1695
1696         wo.wo_type      = type;
1697         wo.wo_pid       = pid;
1698         wo.wo_flags     = options | WEXITED;
1699         wo.wo_info      = NULL;
1700         wo.wo_stat      = 0;
1701         wo.wo_rusage    = ru;
1702         ret = do_wait(&wo);
1703         put_pid(pid);
1704         if (ret > 0 && stat_addr && put_user(wo.wo_stat, stat_addr))
1705                 ret = -EFAULT;
1706
1707         return ret;
1708 }
1709
1710 int kernel_wait(pid_t pid, int *stat)
1711 {
1712         struct wait_opts wo = {
1713                 .wo_type        = PIDTYPE_PID,
1714                 .wo_pid         = find_get_pid(pid),
1715                 .wo_flags       = WEXITED,
1716         };
1717         int ret;
1718
1719         ret = do_wait(&wo);
1720         if (ret > 0 && wo.wo_stat)
1721                 *stat = wo.wo_stat;
1722         put_pid(wo.wo_pid);
1723         return ret;
1724 }
1725
1726 SYSCALL_DEFINE4(wait4, pid_t, upid, int __user *, stat_addr,
1727                 int, options, struct rusage __user *, ru)
1728 {
1729         struct rusage r;
1730         long err = kernel_wait4(upid, stat_addr, options, ru ? &r : NULL);
1731
1732         if (err > 0) {
1733                 if (ru && copy_to_user(ru, &r, sizeof(struct rusage)))
1734                         return -EFAULT;
1735         }
1736         return err;
1737 }
1738
1739 #ifdef __ARCH_WANT_SYS_WAITPID
1740
1741 /*
1742  * sys_waitpid() remains for compatibility. waitpid() should be
1743  * implemented by calling sys_wait4() from libc.a.
1744  */
1745 SYSCALL_DEFINE3(waitpid, pid_t, pid, int __user *, stat_addr, int, options)
1746 {
1747         return kernel_wait4(pid, stat_addr, options, NULL);
1748 }
1749
1750 #endif
1751
1752 #ifdef CONFIG_COMPAT
1753 COMPAT_SYSCALL_DEFINE4(wait4,
1754         compat_pid_t, pid,
1755         compat_uint_t __user *, stat_addr,
1756         int, options,
1757         struct compat_rusage __user *, ru)
1758 {
1759         struct rusage r;
1760         long err = kernel_wait4(pid, stat_addr, options, ru ? &r : NULL);
1761         if (err > 0) {
1762                 if (ru && put_compat_rusage(&r, ru))
1763                         return -EFAULT;
1764         }
1765         return err;
1766 }
1767
1768 COMPAT_SYSCALL_DEFINE5(waitid,
1769                 int, which, compat_pid_t, pid,
1770                 struct compat_siginfo __user *, infop, int, options,
1771                 struct compat_rusage __user *, uru)
1772 {
1773         struct rusage ru;
1774         struct waitid_info info = {.status = 0};
1775         long err = kernel_waitid(which, pid, &info, options, uru ? &ru : NULL);
1776         int signo = 0;
1777         if (err > 0) {
1778                 signo = SIGCHLD;
1779                 err = 0;
1780                 if (uru) {
1781                         /* kernel_waitid() overwrites everything in ru */
1782                         if (COMPAT_USE_64BIT_TIME)
1783                                 err = copy_to_user(uru, &ru, sizeof(ru));
1784                         else
1785                                 err = put_compat_rusage(&ru, uru);
1786                         if (err)
1787                                 return -EFAULT;
1788                 }
1789         }
1790
1791         if (!infop)
1792                 return err;
1793
1794         if (!user_write_access_begin(infop, sizeof(*infop)))
1795                 return -EFAULT;
1796
1797         unsafe_put_user(signo, &infop->si_signo, Efault);
1798         unsafe_put_user(0, &infop->si_errno, Efault);
1799         unsafe_put_user(info.cause, &infop->si_code, Efault);
1800         unsafe_put_user(info.pid, &infop->si_pid, Efault);
1801         unsafe_put_user(info.uid, &infop->si_uid, Efault);
1802         unsafe_put_user(info.status, &infop->si_status, Efault);
1803         user_write_access_end();
1804         return err;
1805 Efault:
1806         user_write_access_end();
1807         return -EFAULT;
1808 }
1809 #endif
1810
1811 /**
1812  * thread_group_exited - check that a thread group has exited
1813  * @pid: tgid of thread group to be checked.
1814  *
1815  * Test if the thread group represented by tgid has exited (all
1816  * threads are zombies, dead or completely gone).
1817  *
1818  * Return: true if the thread group has exited. false otherwise.
1819  */
1820 bool thread_group_exited(struct pid *pid)
1821 {
1822         struct task_struct *task;
1823         bool exited;
1824
1825         rcu_read_lock();
1826         task = pid_task(pid, PIDTYPE_PID);
1827         exited = !task ||
1828                 (READ_ONCE(task->exit_state) && thread_group_empty(task));
1829         rcu_read_unlock();
1830
1831         return exited;
1832 }
1833 EXPORT_SYMBOL(thread_group_exited);
1834
1835 __weak void abort(void)
1836 {
1837         BUG();
1838
1839         /* if that doesn't kill us, halt */
1840         panic("Oops failed to kill thread");
1841 }
1842 EXPORT_SYMBOL(abort);