x86: convert various functions to use ptdescs

In order to split struct ptdesc from struct page, convert various
functions to use ptdescs.

Some of the functions use the *get*page*() helper functions.  Convert
these to use pagetable_alloc() and ptdesc_address() instead to help
standardize page tables further.

Link: https://lkml.kernel.org/r/20230807230513.102486-14-vishal.moola@gmail.com
Signed-off-by: Vishal Moola (Oracle) <vishal.moola@gmail.com>
Cc: Arnd Bergmann <arnd@arndb.de>
Cc: Catalin Marinas <catalin.marinas@arm.com>
Cc: Christophe Leroy <christophe.leroy@csgroup.eu>
Cc: Claudio Imbrenda <imbrenda@linux.ibm.com>
Cc: Dave Hansen <dave.hansen@linux.intel.com>
Cc: David Hildenbrand <david@redhat.com>
Cc: "David S. Miller" <davem@davemloft.net>
Cc: Dinh Nguyen <dinguyen@kernel.org>
Cc: Geert Uytterhoeven <geert@linux-m68k.org>
Cc: Geert Uytterhoeven <geert+renesas@glider.be>
Cc: Guo Ren <guoren@kernel.org>
Cc: Huacai Chen <chenhuacai@kernel.org>
Cc: Hugh Dickins <hughd@google.com>
Cc: John Paul Adrian Glaubitz <glaubitz@physik.fu-berlin.de>
Cc: Jonas Bonn <jonas@southpole.se>
Cc: Matthew Wilcox <willy@infradead.org>
Cc: Mike Rapoport (IBM) <rppt@kernel.org>
Cc: Palmer Dabbelt <palmer@rivosinc.com>
Cc: Paul Walmsley <paul.walmsley@sifive.com>
Cc: Richard Weinberger <richard@nod.at>
Cc: Thomas Bogendoerfer <tsbogend@alpha.franken.de>
Cc: Yoshinori Sato <ysato@users.sourceforge.jp>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>

diff --git a/arch/x86/mm/pgtable.c b/arch/x86/mm/pgtable.c
index 15a8009a4480a98abcd25aca53ab8a45da42bef9..d3a93e8766eeaebccdb67a20692f9ad813c596b2 100644 (file)
--- a/arch/x86/mm/pgtable.c
+++ b/arch/x86/mm/pgtable.c
@@ -52,7 +52,7 @@ early_param("userpte", setup_userpte);
 
 void ___pte_free_tlb(struct mmu_gather *tlb, struct page *pte)
 {
-       pgtable_pte_page_dtor(pte);
+       pagetable_pte_dtor(page_ptdesc(pte));
        paravirt_release_pte(page_to_pfn(pte));
        paravirt_tlb_remove_table(tlb, pte);
 }
@@ -60,7 +60,7 @@ void ___pte_free_tlb(struct mmu_gather *tlb, struct page *pte)
 #if CONFIG_PGTABLE_LEVELS > 2
 void ___pmd_free_tlb(struct mmu_gather *tlb, pmd_t *pmd)
 {
-       struct page *page = virt_to_page(pmd);
+       struct ptdesc *ptdesc = virt_to_ptdesc(pmd);
        paravirt_release_pmd(__pa(pmd) >> PAGE_SHIFT);
        /*
         * NOTE! For PAE, any changes to the top page-directory-pointer-table
@@ -69,8 +69,8 @@ void ___pmd_free_tlb(struct mmu_gather *tlb, pmd_t *pmd)
 #ifdef CONFIG_X86_PAE
        tlb->need_flush_all = 1;
 #endif
-       pgtable_pmd_page_dtor(page);
-       paravirt_tlb_remove_table(tlb, page);
+       pagetable_pmd_dtor(ptdesc);
+       paravirt_tlb_remove_table(tlb, ptdesc_page(ptdesc));
 }
 
 #if CONFIG_PGTABLE_LEVELS > 3
@@ -92,16 +92,16 @@ void ___p4d_free_tlb(struct mmu_gather *tlb, p4d_t *p4d)
 
 static inline void pgd_list_add(pgd_t *pgd)
 {
-       struct page *page = virt_to_page(pgd);
+       struct ptdesc *ptdesc = virt_to_ptdesc(pgd);
 
-       list_add(&page->lru, &pgd_list);
+       list_add(&ptdesc->pt_list, &pgd_list);
 }
 
 static inline void pgd_list_del(pgd_t *pgd)
 {
-       struct page *page = virt_to_page(pgd);
+       struct ptdesc *ptdesc = virt_to_ptdesc(pgd);
 
-       list_del(&page->lru);
+       list_del(&ptdesc->pt_list);
 }
 
 #define UNSHARED_PTRS_PER_PGD                          \
@@ -112,12 +112,12 @@ static inline void pgd_list_del(pgd_t *pgd)
 
 static void pgd_set_mm(pgd_t *pgd, struct mm_struct *mm)
 {
-       virt_to_page(pgd)->pt_mm = mm;
+       virt_to_ptdesc(pgd)->pt_mm = mm;
 }
 
 struct mm_struct *pgd_page_get_mm(struct page *page)
 {
-       return page->pt_mm;
+       return page_ptdesc(page)->pt_mm;
 }
 
 static void pgd_ctor(struct mm_struct *mm, pgd_t *pgd)
@@ -213,11 +213,14 @@ void pud_populate(struct mm_struct *mm, pud_t *pudp, pmd_t *pmd)
 static void free_pmds(struct mm_struct *mm, pmd_t *pmds[], int count)
 {
        int i;
+       struct ptdesc *ptdesc;
 
        for (i = 0; i < count; i++)
                if (pmds[i]) {
-                       pgtable_pmd_page_dtor(virt_to_page(pmds[i]));
-                       free_page((unsigned long)pmds[i]);
+                       ptdesc = virt_to_ptdesc(pmds[i]);
+
+                       pagetable_pmd_dtor(ptdesc);
+                       pagetable_free(ptdesc);
                        mm_dec_nr_pmds(mm);
                }
 }
@@ -230,18 +233,24 @@ static int preallocate_pmds(struct mm_struct *mm, pmd_t *pmds[], int count)
 
        if (mm == &init_mm)
                gfp &= ~__GFP_ACCOUNT;
+       gfp &= ~__GFP_HIGHMEM;
 
        for (i = 0; i < count; i++) {
-               pmd_t *pmd = (pmd_t *)__get_free_page(gfp);
-               if (!pmd)
+               pmd_t *pmd = NULL;
+               struct ptdesc *ptdesc = pagetable_alloc(gfp, 0);
+
+               if (!ptdesc)
                        failed = true;
-               if (pmd && !pgtable_pmd_page_ctor(virt_to_page(pmd))) {
-                       free_page((unsigned long)pmd);
-                       pmd = NULL;
+               if (ptdesc && !pagetable_pmd_ctor(ptdesc)) {
+                       pagetable_free(ptdesc);
+                       ptdesc = NULL;
                        failed = true;
                }
-               if (pmd)
+               if (ptdesc) {
                        mm_inc_nr_pmds(mm);
+                       pmd = ptdesc_address(ptdesc);
+               }
+
                pmds[i] = pmd;
        }
 
@@ -830,7 +839,7 @@ int pud_free_pmd_page(pud_t *pud, unsigned long addr)
 
        free_page((unsigned long)pmd_sv);
 
-       pgtable_pmd_page_dtor(virt_to_page(pmd));
+       pagetable_pmd_dtor(virt_to_ptdesc(pmd));
        free_page((unsigned long)pmd);
 
        return 1;