Merge tag 'irqchip-fixes-5.11-1' of git://git.kernel.org/pub/scm/linux/kernel/git...
[platform/kernel/linux-starfive.git] / kernel / scs.c
1 // SPDX-License-Identifier: GPL-2.0
2 /*
3  * Shadow Call Stack support.
4  *
5  * Copyright (C) 2019 Google LLC
6  */
7
8 #include <linux/cpuhotplug.h>
9 #include <linux/kasan.h>
10 #include <linux/mm.h>
11 #include <linux/scs.h>
12 #include <linux/vmalloc.h>
13 #include <linux/vmstat.h>
14
15 static void __scs_account(void *s, int account)
16 {
17         struct page *scs_page = vmalloc_to_page(s);
18
19         mod_node_page_state(page_pgdat(scs_page), NR_KERNEL_SCS_KB,
20                             account * (SCS_SIZE / SZ_1K));
21 }
22
23 /* Matches NR_CACHED_STACKS for VMAP_STACK */
24 #define NR_CACHED_SCS 2
25 static DEFINE_PER_CPU(void *, scs_cache[NR_CACHED_SCS]);
26
27 static void *__scs_alloc(int node)
28 {
29         int i;
30         void *s;
31
32         for (i = 0; i < NR_CACHED_SCS; i++) {
33                 s = this_cpu_xchg(scs_cache[i], NULL);
34                 if (s) {
35                         kasan_unpoison_vmalloc(s, SCS_SIZE);
36                         memset(s, 0, SCS_SIZE);
37                         return s;
38                 }
39         }
40
41         return __vmalloc_node_range(SCS_SIZE, 1, VMALLOC_START, VMALLOC_END,
42                                     GFP_SCS, PAGE_KERNEL, 0, node,
43                                     __builtin_return_address(0));
44 }
45
46 void *scs_alloc(int node)
47 {
48         void *s;
49
50         s = __scs_alloc(node);
51         if (!s)
52                 return NULL;
53
54         *__scs_magic(s) = SCS_END_MAGIC;
55
56         /*
57          * Poison the allocation to catch unintentional accesses to
58          * the shadow stack when KASAN is enabled.
59          */
60         kasan_poison_vmalloc(s, SCS_SIZE);
61         __scs_account(s, 1);
62         return s;
63 }
64
65 void scs_free(void *s)
66 {
67         int i;
68
69         __scs_account(s, -1);
70
71         /*
72          * We cannot sleep as this can be called in interrupt context,
73          * so use this_cpu_cmpxchg to update the cache, and vfree_atomic
74          * to free the stack.
75          */
76
77         for (i = 0; i < NR_CACHED_SCS; i++)
78                 if (this_cpu_cmpxchg(scs_cache[i], 0, s) == NULL)
79                         return;
80
81         vfree_atomic(s);
82 }
83
84 static int scs_cleanup(unsigned int cpu)
85 {
86         int i;
87         void **cache = per_cpu_ptr(scs_cache, cpu);
88
89         for (i = 0; i < NR_CACHED_SCS; i++) {
90                 vfree(cache[i]);
91                 cache[i] = NULL;
92         }
93
94         return 0;
95 }
96
97 void __init scs_init(void)
98 {
99         cpuhp_setup_state(CPUHP_BP_PREPARE_DYN, "scs:scs_cache", NULL,
100                           scs_cleanup);
101 }
102
103 int scs_prepare(struct task_struct *tsk, int node)
104 {
105         void *s = scs_alloc(node);
106
107         if (!s)
108                 return -ENOMEM;
109
110         task_scs(tsk) = task_scs_sp(tsk) = s;
111         return 0;
112 }
113
114 static void scs_check_usage(struct task_struct *tsk)
115 {
116         static unsigned long highest;
117
118         unsigned long *p, prev, curr = highest, used = 0;
119
120         if (!IS_ENABLED(CONFIG_DEBUG_STACK_USAGE))
121                 return;
122
123         for (p = task_scs(tsk); p < __scs_magic(tsk); ++p) {
124                 if (!READ_ONCE_NOCHECK(*p))
125                         break;
126                 used += sizeof(*p);
127         }
128
129         while (used > curr) {
130                 prev = cmpxchg_relaxed(&highest, curr, used);
131
132                 if (prev == curr) {
133                         pr_info("%s (%d): highest shadow stack usage: %lu bytes\n",
134                                 tsk->comm, task_pid_nr(tsk), used);
135                         break;
136                 }
137
138                 curr = prev;
139         }
140 }
141
142 void scs_release(struct task_struct *tsk)
143 {
144         void *s = task_scs(tsk);
145
146         if (!s)
147                 return;
148
149         WARN(task_scs_end_corrupted(tsk),
150              "corrupted shadow stack detected when freeing task\n");
151         scs_check_usage(tsk);
152         scs_free(s);
153 }