1 // SPDX-License-Identifier: GPL-2.0
2 #include <linux/hugetlb.h>
5 #ifdef CONFIG_RISCV_ISA_SVNAPOT
6 pte_t *huge_pte_alloc(struct mm_struct *mm,
7 struct vm_area_struct *vma,
18 pgd = pgd_offset(mm, addr);
19 p4d = p4d_alloc(mm, pgd, addr);
23 pud = pud_alloc(mm, p4d, addr);
33 if (want_pmd_share(vma, addr) && pud_none(*pud))
34 pte = huge_pmd_share(mm, vma, addr, pud);
36 pte = (pte_t *)pmd_alloc(mm, pud, addr);
40 pmd = pmd_alloc(mm, pud, addr);
44 for_each_napot_order(order) {
45 if (napot_cont_size(order) == sz) {
46 pte = pte_alloc_map(mm, pmd, addr & napot_cont_mask(order));
52 WARN_ON_ONCE(pte && pte_present(*pte) && !pte_huge(*pte));
56 pte_t *huge_pte_offset(struct mm_struct *mm,
67 pgd = pgd_offset(mm, addr);
68 if (!pgd_present(*pgd))
71 p4d = p4d_offset(pgd, addr);
72 if (!p4d_present(*p4d))
75 pud = pud_offset(p4d, addr);
77 /* must be pud huge, non-present or none */
80 if (!pud_present(*pud))
83 pmd = pmd_offset(pud, addr);
85 /* must be pmd huge, non-present or none */
88 if (!pmd_present(*pmd))
91 for_each_napot_order(order) {
92 if (napot_cont_size(order) == sz) {
93 pte = pte_offset_kernel(pmd, addr & napot_cont_mask(order));
100 static pte_t get_clear_contig(struct mm_struct *mm,
103 unsigned long pte_num)
105 pte_t orig_pte = ptep_get(ptep);
108 for (i = 0; i < pte_num; i++, addr += PAGE_SIZE, ptep++) {
109 pte_t pte = ptep_get_and_clear(mm, addr, ptep);
112 orig_pte = pte_mkdirty(orig_pte);
115 orig_pte = pte_mkyoung(orig_pte);
121 static pte_t get_clear_contig_flush(struct mm_struct *mm,
124 unsigned long pte_num)
126 pte_t orig_pte = get_clear_contig(mm, addr, ptep, pte_num);
127 struct vm_area_struct vma = TLB_FLUSH_VMA(mm, 0);
128 bool valid = !pte_none(orig_pte);
131 flush_tlb_range(&vma, addr, addr + (PAGE_SIZE * pte_num));
136 pte_t arch_make_huge_pte(pte_t entry, unsigned int shift, vm_flags_t flags)
140 for_each_napot_order(order) {
141 if (shift == napot_cont_shift(order)) {
142 entry = pte_mknapot(entry, order);
146 if (order == NAPOT_ORDER_MAX)
147 entry = pte_mkhuge(entry);
152 void set_huge_pte_at(struct mm_struct *mm,
159 if (!pte_napot(pte)) {
160 set_pte_at(mm, addr, ptep, pte);
164 pte_num = napot_pte_num(napot_cont_order(pte));
165 for (i = 0; i < pte_num; i++, ptep++, addr += PAGE_SIZE)
166 set_pte_at(mm, addr, ptep, pte);
169 int huge_ptep_set_access_flags(struct vm_area_struct *vma,
175 struct mm_struct *mm = vma->vm_mm;
181 return ptep_set_access_flags(vma, addr, ptep, pte, dirty);
183 order = napot_cont_order(pte);
184 pte_num = napot_pte_num(order);
185 ptep = huge_pte_offset(mm, addr, napot_cont_size(order));
186 orig_pte = get_clear_contig_flush(mm, addr, ptep, pte_num);
188 if (pte_dirty(orig_pte))
189 pte = pte_mkdirty(pte);
191 if (pte_young(orig_pte))
192 pte = pte_mkyoung(pte);
194 for (i = 0; i < pte_num; i++, addr += PAGE_SIZE, ptep++)
195 set_pte_at(mm, addr, ptep, pte);
200 pte_t huge_ptep_get_and_clear(struct mm_struct *mm,
204 pte_t orig_pte = ptep_get(ptep);
207 if (!pte_napot(orig_pte))
208 return ptep_get_and_clear(mm, addr, ptep);
210 pte_num = napot_pte_num(napot_cont_order(orig_pte));
212 return get_clear_contig(mm, addr, ptep, pte_num);
215 void huge_ptep_set_wrprotect(struct mm_struct *mm,
219 pte_t pte = ptep_get(ptep);
223 if (!pte_napot(pte)) {
224 ptep_set_wrprotect(mm, addr, ptep);
228 order = napot_cont_order(pte);
229 pte_num = napot_pte_num(order);
230 ptep = huge_pte_offset(mm, addr, napot_cont_size(order));
232 for (i = 0; i < pte_num; i++, addr += PAGE_SIZE, ptep++)
233 ptep_set_wrprotect(mm, addr, ptep);
236 pte_t huge_ptep_clear_flush(struct vm_area_struct *vma,
240 pte_t pte = ptep_get(ptep);
244 return ptep_clear_flush(vma, addr, ptep);
246 pte_num = napot_pte_num(napot_cont_order(pte));
248 return get_clear_contig_flush(vma->vm_mm, addr, ptep, pte_num);
251 void huge_pte_clear(struct mm_struct *mm,
256 pte_t pte = READ_ONCE(*ptep);
259 if (!pte_napot(pte)) {
260 pte_clear(mm, addr, ptep);
264 pte_num = napot_pte_num(napot_cont_order(pte));
265 for (i = 0; i < pte_num; i++, addr += PAGE_SIZE, ptep++)
266 pte_clear(mm, addr, ptep);
269 static __init bool is_napot_size(unsigned long size)
276 for_each_napot_order(order) {
277 if (size == napot_cont_size(order))
283 static __init int napot_hugetlbpages_init(void)
288 for_each_napot_order(order)
289 hugetlb_add_hstate(order);
293 arch_initcall(napot_hugetlbpages_init);
297 static __init bool is_napot_size(unsigned long size)
302 #endif /*CONFIG_RISCV_ISA_SVNAPOT*/
304 int pud_huge(pud_t pud)
306 return pud_leaf(pud);
309 int pmd_huge(pmd_t pmd)
311 return pmd_leaf(pmd);
314 bool __init arch_hugetlb_valid_size(unsigned long size)
316 if (size == HPAGE_SIZE)
318 else if (IS_ENABLED(CONFIG_64BIT) && size == PUD_SIZE)
320 else if (is_napot_size(size))
326 #ifdef CONFIG_CONTIG_ALLOC
327 static __init int gigantic_pages_init(void)
329 /* With CONTIG_ALLOC, we can allocate gigantic pages at runtime */
330 if (IS_ENABLED(CONFIG_64BIT))
331 hugetlb_add_hstate(PUD_SHIFT - PAGE_SHIFT);
334 arch_initcall(gigantic_pages_init);