1 // SPDX-License-Identifier: GPL-2.0
2 #include <linux/hugetlb.h>
5 #ifdef CONFIG_RISCV_ISA_SVNAPOT
6 pte_t *huge_pte_alloc(struct mm_struct *mm,
7 struct vm_area_struct *vma,
18 pgd = pgd_offset(mm, addr);
19 p4d = p4d_alloc(mm, pgd, addr);
23 pud = pud_alloc(mm, p4d, addr);
33 if (want_pmd_share(vma, addr) && pud_none(*pud))
34 pte = huge_pmd_share(mm, vma, addr, pud);
36 pte = (pte_t *)pmd_alloc(mm, pud, addr);
40 pmd = pmd_alloc(mm, pud, addr);
44 for_each_napot_order(order) {
45 if (napot_cont_size(order) == sz) {
46 pte = pte_alloc_map(mm, pmd, addr & napot_cont_mask(order));
52 WARN_ON_ONCE(pte && pte_present(*pte) && !pte_huge(*pte));
56 pte_t *huge_pte_offset(struct mm_struct *mm,
67 pgd = pgd_offset(mm, addr);
68 if (!pgd_present(*pgd))
71 p4d = p4d_offset(pgd, addr);
72 if (!p4d_present(*p4d))
75 pud = pud_offset(p4d, addr);
77 /* must be pud huge, non-present or none */
80 if (!pud_present(*pud))
83 pmd = pmd_offset(pud, addr);
85 /* must be pmd huge, non-present or none */
88 if (!pmd_present(*pmd))
91 for_each_napot_order(order) {
92 if (napot_cont_size(order) == sz) {
93 pte = pte_offset_kernel(pmd, addr & napot_cont_mask(order));
100 static pte_t get_clear_contig(struct mm_struct *mm,
103 unsigned long pte_num)
105 pte_t orig_pte = ptep_get(ptep);
108 for (i = 0; i < pte_num; i++, addr += PAGE_SIZE, ptep++) {
109 pte_t pte = ptep_get_and_clear(mm, addr, ptep);
112 orig_pte = pte_mkdirty(orig_pte);
115 orig_pte = pte_mkyoung(orig_pte);
121 static pte_t get_clear_contig_flush(struct mm_struct *mm,
124 unsigned long pte_num)
126 pte_t orig_pte = get_clear_contig(mm, addr, ptep, pte_num);
127 struct vm_area_struct vma = TLB_FLUSH_VMA(mm, 0);
128 bool valid = !pte_none(orig_pte);
131 flush_tlb_range(&vma, addr, addr + (PAGE_SIZE * pte_num));
136 pte_t arch_make_huge_pte(pte_t entry, unsigned int shift, vm_flags_t flags)
140 for_each_napot_order(order) {
141 if (shift == napot_cont_shift(order)) {
142 entry = pte_mknapot(entry, order);
146 if (order == NAPOT_ORDER_MAX)
147 entry = pte_mkhuge(entry);
152 void set_huge_pte_at(struct mm_struct *mm,
159 if (!pte_napot(pte)) {
160 set_pte_at(mm, addr, ptep, pte);
164 pte_num = napot_pte_num(napot_cont_order(pte));
165 for (i = 0; i < pte_num; i++, ptep++, addr += PAGE_SIZE)
166 set_pte_at(mm, addr, ptep, pte);
169 int huge_ptep_set_access_flags(struct vm_area_struct *vma,
175 struct mm_struct *mm = vma->vm_mm;
181 return ptep_set_access_flags(vma, addr, ptep, pte, dirty);
183 order = napot_cont_order(pte);
184 pte_num = napot_pte_num(order);
185 ptep = huge_pte_offset(mm, addr, napot_cont_size(order));
186 orig_pte = get_clear_contig_flush(mm, addr, ptep, pte_num);
188 if (pte_dirty(orig_pte))
189 pte = pte_mkdirty(pte);
191 if (pte_young(orig_pte))
192 pte = pte_mkyoung(pte);
194 for (i = 0; i < pte_num; i++, addr += PAGE_SIZE, ptep++)
195 set_pte_at(mm, addr, ptep, pte);
200 pte_t huge_ptep_get_and_clear(struct mm_struct *mm,
204 pte_t orig_pte = ptep_get(ptep);
207 if (!pte_napot(orig_pte))
208 return ptep_get_and_clear(mm, addr, ptep);
210 pte_num = napot_pte_num(napot_cont_order(orig_pte));
212 return get_clear_contig(mm, addr, ptep, pte_num);
215 void huge_ptep_set_wrprotect(struct mm_struct *mm,
219 pte_t pte = ptep_get(ptep);
224 if (!pte_napot(pte)) {
225 ptep_set_wrprotect(mm, addr, ptep);
229 order = napot_cont_order(pte);
230 pte_num = napot_pte_num(order);
231 ptep = huge_pte_offset(mm, addr, napot_cont_size(order));
232 orig_pte = get_clear_contig_flush(mm, addr, ptep, pte_num);
234 orig_pte = pte_wrprotect(orig_pte);
236 for (i = 0; i < pte_num; i++, addr += PAGE_SIZE, ptep++)
237 set_pte_at(mm, addr, ptep, orig_pte);
240 pte_t huge_ptep_clear_flush(struct vm_area_struct *vma,
244 pte_t pte = ptep_get(ptep);
248 return ptep_clear_flush(vma, addr, ptep);
250 pte_num = napot_pte_num(napot_cont_order(pte));
252 return get_clear_contig_flush(vma->vm_mm, addr, ptep, pte_num);
255 void huge_pte_clear(struct mm_struct *mm,
260 pte_t pte = READ_ONCE(*ptep);
263 if (!pte_napot(pte)) {
264 pte_clear(mm, addr, ptep);
268 pte_num = napot_pte_num(napot_cont_order(pte));
269 for (i = 0; i < pte_num; i++, addr += PAGE_SIZE, ptep++)
270 pte_clear(mm, addr, ptep);
273 static __init bool is_napot_size(unsigned long size)
280 for_each_napot_order(order) {
281 if (size == napot_cont_size(order))
287 static __init int napot_hugetlbpages_init(void)
292 for_each_napot_order(order)
293 hugetlb_add_hstate(order);
297 arch_initcall(napot_hugetlbpages_init);
301 static __init bool is_napot_size(unsigned long size)
306 #endif /*CONFIG_RISCV_ISA_SVNAPOT*/
308 int pud_huge(pud_t pud)
310 return pud_leaf(pud);
313 int pmd_huge(pmd_t pmd)
315 return pmd_leaf(pmd);
318 bool __init arch_hugetlb_valid_size(unsigned long size)
320 if (size == HPAGE_SIZE)
322 else if (IS_ENABLED(CONFIG_64BIT) && size == PUD_SIZE)
324 else if (is_napot_size(size))
330 #ifdef CONFIG_CONTIG_ALLOC
331 static __init int gigantic_pages_init(void)
333 /* With CONTIG_ALLOC, we can allocate gigantic pages at runtime */
334 if (IS_ENABLED(CONFIG_64BIT))
335 hugetlb_add_hstate(PUD_SHIFT - PAGE_SHIFT);
338 arch_initcall(gigantic_pages_init);