#define ALT_ORIG_PTR(a) __ALT_PTR(a, orig_offset)
#define ALT_REPL_PTR(a) __ALT_PTR(a, alt_offset)
+#define ALT_CAP(a) ((a)->cpufeature & ~ARM64_CB_BIT)
+#define ALT_HAS_CB(a) ((a)->cpufeature & ARM64_CB_BIT)
+
/* Volatile, as we may be patching the guts of READ_ONCE() */
static volatile int all_alternatives_applied;
} while (cur += d_size, cur < end);
}
-static void __nocfi __apply_alternatives(struct alt_region *region, bool is_module,
- unsigned long *feature_mask)
+static void __nocfi __apply_alternatives(const struct alt_region *region,
+ bool is_module,
+ unsigned long *feature_mask)
{
struct alt_instr *alt;
__le32 *origptr, *updptr;
for (alt = region->begin; alt < region->end; alt++) {
int nr_inst;
+ int cap = ALT_CAP(alt);
- if (!test_bit(alt->cpufeature, feature_mask))
+ if (!test_bit(cap, feature_mask))
continue;
- /* Use ARM64_CB_PATCH as an unconditional patch */
- if (alt->cpufeature < ARM64_CB_PATCH &&
- !cpus_have_cap(alt->cpufeature))
+ if (!cpus_have_cap(cap))
continue;
- if (alt->cpufeature == ARM64_CB_PATCH)
+ if (ALT_HAS_CB(alt))
BUG_ON(alt->alt_len != 0);
else
BUG_ON(alt->alt_len != alt->orig_len);
- pr_info_once("patching kernel code\n");
-
origptr = ALT_ORIG_PTR(alt);
updptr = is_module ? origptr : lm_alias(origptr);
nr_inst = alt->orig_len / AARCH64_INSN_SIZE;
- if (alt->cpufeature < ARM64_CB_PATCH)
- alt_cb = patch_alternative;
- else
+ if (ALT_HAS_CB(alt))
alt_cb = ALT_REPL_PTR(alt);
+ else
+ alt_cb = patch_alternative;
alt_cb(alt, origptr, updptr, nr_inst);
const struct elf64_hdr *hdr;
const struct elf64_shdr *shdr;
const struct elf64_shdr *alt;
- DECLARE_BITMAP(all_capabilities, ARM64_NPATCHABLE);
+ DECLARE_BITMAP(all_capabilities, ARM64_NCAPS);
- bitmap_fill(all_capabilities, ARM64_NPATCHABLE);
+ bitmap_fill(all_capabilities, ARM64_NCAPS);
hdr = (struct elf64_hdr *)vdso_start;
shdr = (void *)hdr + hdr->e_shoff;
__apply_alternatives(®ion, false, &all_capabilities[0]);
}
+static const struct alt_region kernel_alternatives = {
+ .begin = (struct alt_instr *)__alt_instructions,
+ .end = (struct alt_instr *)__alt_instructions_end,
+};
+
/*
* We might be patching the stop_machine state machine, so implement a
* really simple polling protocol here.
*/
static int __apply_alternatives_multi_stop(void *unused)
{
- struct alt_region region = {
- .begin = (struct alt_instr *)__alt_instructions,
- .end = (struct alt_instr *)__alt_instructions_end,
- };
-
/* We always have a CPU 0 at this point (__init) */
if (smp_processor_id()) {
while (!all_alternatives_applied)
cpu_relax();
isb();
} else {
- DECLARE_BITMAP(remaining_capabilities, ARM64_NPATCHABLE);
+ DECLARE_BITMAP(remaining_capabilities, ARM64_NCAPS);
bitmap_complement(remaining_capabilities, boot_capabilities,
- ARM64_NPATCHABLE);
+ ARM64_NCAPS);
BUG_ON(all_alternatives_applied);
- __apply_alternatives(®ion, false, remaining_capabilities);
+ __apply_alternatives(&kernel_alternatives, false,
+ remaining_capabilities);
/* Barriers provided by the cache flushing */
all_alternatives_applied = 1;
}
void __init apply_alternatives_all(void)
{
+ pr_info("applying system-wide alternatives\n");
+
apply_alternatives_vdso();
/* better not try code patching on a live SMP system */
stop_machine(__apply_alternatives_multi_stop, NULL, cpu_online_mask);
*/
void __init apply_boot_alternatives(void)
{
- struct alt_region region = {
- .begin = (struct alt_instr *)__alt_instructions,
- .end = (struct alt_instr *)__alt_instructions_end,
- };
-
/* If called on non-boot cpu things could go wrong */
WARN_ON(smp_processor_id() != 0);
- __apply_alternatives(®ion, false, &boot_capabilities[0]);
+ pr_info("applying boot alternatives\n");
+
+ __apply_alternatives(&kernel_alternatives, false,
+ &boot_capabilities[0]);
}
#ifdef CONFIG_MODULES
.begin = start,
.end = start + length,
};
- DECLARE_BITMAP(all_capabilities, ARM64_NPATCHABLE);
+ DECLARE_BITMAP(all_capabilities, ARM64_NCAPS);
- bitmap_fill(all_capabilities, ARM64_NPATCHABLE);
+ bitmap_fill(all_capabilities, ARM64_NCAPS);
__apply_alternatives(®ion, true, &all_capabilities[0]);
}
#endif
+
+noinstr void alt_cb_patch_nops(struct alt_instr *alt, __le32 *origptr,
+ __le32 *updptr, int nr_inst)
+{
+ for (int i = 0; i < nr_inst; i++)
+ updptr[i] = cpu_to_le32(aarch64_insn_gen_nop());
+}
+EXPORT_SYMBOL(alt_cb_patch_nops);