x86/efistub: Perform 4/5 level paging switch from the stub

[platform/kernel/linux-rpi.git] / drivers / firmware / efi / libstub / x86-stub.c

diff --git a/drivers/firmware/efi/libstub/x86-stub.c b/drivers/firmware/efi/libstub/x86-stub.c
index 220be75..af5f506 100644 (file)
--- a/drivers/firmware/efi/libstub/x86-stub.c
+++ b/drivers/firmware/efi/libstub/x86-stub.c
@@ -17,6 +17,7 @@
 #include <asm/boot.h>
 
 #include "efistub.h"
+#include "x86-stub.h"
 
 /* Maximum physical address for 64-bit kernel with 4-level paging */
 #define MAXMEM_X86_64_4LEVEL (1ull << 46)
@@ -223,8 +224,8 @@ static void retrieve_apple_device_properties(struct boot_params *boot_params)
        }
 }
 
-static void
-adjust_memory_range_protection(unsigned long start, unsigned long size)
+void efi_adjust_memory_range_protection(unsigned long start,
+                                       unsigned long size)
 {
        efi_status_t status;
        efi_gcd_memory_space_desc_t desc;
@@ -278,35 +279,14 @@ adjust_memory_range_protection(unsigned long start, unsigned long size)
        }
 }
 
-/*
- * Trampoline takes 2 pages and can be loaded in first megabyte of memory
- * with its end placed between 128k and 640k where BIOS might start.
- * (see arch/x86/boot/compressed/pgtable_64.c)
- *
- * We cannot find exact trampoline placement since memory map
- * can be modified by UEFI, and it can alter the computed address.
- */
-
-#define TRAMPOLINE_PLACEMENT_BASE ((128 - 8)*1024)
-#define TRAMPOLINE_PLACEMENT_SIZE (640*1024 - (128 - 8)*1024)
-
-void startup_32(struct boot_params *boot_params);
+extern const u8 startup_32[], startup_64[];
 
 static void
 setup_memory_protection(unsigned long image_base, unsigned long image_size)
 {
-       /*
-        * Allow execution of possible trampoline used
-        * for switching between 4- and 5-level page tables
-        * and relocated kernel image.
-        */
-
-       adjust_memory_range_protection(TRAMPOLINE_PLACEMENT_BASE,
-                                      TRAMPOLINE_PLACEMENT_SIZE);
-
 #ifdef CONFIG_64BIT
        if (image_base != (unsigned long)startup_32)
-               adjust_memory_range_protection(image_base, image_size);
+               efi_adjust_memory_range_protection(image_base, image_size);
 #else
        /*
         * Clear protection flags on a whole range of possible
@@ -316,8 +296,8 @@ setup_memory_protection(unsigned long image_base, unsigned long image_size)
         * need to remove possible protection on relocated image
         * itself disregarding further relocations.
         */
-       adjust_memory_range_protection(LOAD_PHYSICAL_ADDR,
-                                      KERNEL_IMAGE_SIZE - LOAD_PHYSICAL_ADDR);
+       efi_adjust_memory_range_protection(LOAD_PHYSICAL_ADDR,
+                                          KERNEL_IMAGE_SIZE - LOAD_PHYSICAL_ADDR);
 #endif
 }
 
@@ -803,14 +783,23 @@ static efi_status_t exit_boot(struct boot_params *boot_params, void *handle)
        return EFI_SUCCESS;
 }
 
+static void __noreturn enter_kernel(unsigned long kernel_addr,
+                                   struct boot_params *boot_params)
+{
+       /* enter decompressed kernel with boot_params pointer in RSI/ESI */
+       asm("jmp *%0"::"r"(kernel_addr), "S"(boot_params));
+
+       unreachable();
+}
+
 /*
- * On success, we return the address of startup_32, which has potentially been
- * relocated by efi_relocate_kernel.
- * On failure, we exit to the firmware via efi_exit instead of returning.
+ * On success, this routine will jump to the relocated image directly and never
+ * return.  On failure, it will exit to the firmware via efi_exit() instead of
+ * returning.
  */
-asmlinkage unsigned long efi_main(efi_handle_t handle,
-                                 efi_system_table_t *sys_table_arg,
-                                 struct boot_params *boot_params)
+void __noreturn efi_stub_entry(efi_handle_t handle,
+                              efi_system_table_t *sys_table_arg,
+                              struct boot_params *boot_params)
 {
        unsigned long bzimage_addr = (unsigned long)startup_32;
        unsigned long buffer_start, buffer_end;
@@ -830,6 +819,12 @@ asmlinkage unsigned long efi_main(efi_handle_t handle,
                efi_dxe_table = NULL;
        }
 
+       status = efi_setup_5level_paging();
+       if (status != EFI_SUCCESS) {
+               efi_err("efi_setup_5level_paging() failed!\n");
+               goto fail;
+       }
+
        /*
         * If the kernel isn't already loaded at a suitable address,
         * relocate it.
@@ -950,9 +945,35 @@ asmlinkage unsigned long efi_main(efi_handle_t handle,
                goto fail;
        }
 
-       return bzimage_addr;
+       efi_5level_switch();
+
+       if (IS_ENABLED(CONFIG_X86_64))
+               bzimage_addr += startup_64 - startup_32;
+
+       enter_kernel(bzimage_addr, boot_params);
 fail:
-       efi_err("efi_main() failed!\n");
+       efi_err("efi_stub_entry() failed!\n");
 
        efi_exit(handle, status);
 }
+
+#ifdef CONFIG_EFI_HANDOVER_PROTOCOL
+void efi_handover_entry(efi_handle_t handle, efi_system_table_t *sys_table_arg,
+                       struct boot_params *boot_params)
+{
+       extern char _bss[], _ebss[];
+
+       memset(_bss, 0, _ebss - _bss);
+       efi_stub_entry(handle, sys_table_arg, boot_params);
+}
+
+#ifndef CONFIG_EFI_MIXED
+extern __alias(efi_handover_entry)
+void efi32_stub_entry(efi_handle_t handle, efi_system_table_t *sys_table_arg,
+                     struct boot_params *boot_params);
+
+extern __alias(efi_handover_entry)
+void efi64_stub_entry(efi_handle_t handle, efi_system_table_t *sys_table_arg,
+                     struct boot_params *boot_params);
+#endif
+#endif