resume it if we have a value of 3 or more percent; consider information about
the amount of free space valid for 30 seconds
-audit_argv_kb
--------------
-
-The file contains a single value denoting the limit on the argv array size
-for execve (in KiB). This limit is only applied when system call auditing for
-execve is enabled, otherwise the value is ignored.
-
ctrl-alt-del
------------
j kernel_entry
#endif
- __INIT_REFOK
+ __REF
NESTED(kernel_entry, 16, sp) # kernel entry point
#
obj-y += cache.o dma-default.o extable.o fault.o \
- init.o pgtable.o tlbex.o tlbex-fault.o
+ init.o pgtable.o tlbex.o tlbex-fault.o \
+ uasm.o
obj-$(CONFIG_32BIT) += ioremap.o pgtable-32.o
obj-$(CONFIG_64BIT) += pgtable-64.o
*
* Synthesize TLB refill handlers at runtime.
*
- * Copyright (C) 2004,2005,2006 by Thiemo Seufer
+ * Copyright (C) 2004, 2005, 2006, 2008 Thiemo Seufer
* Copyright (C) 2005, 2007 Maciej W. Rozycki
* Copyright (C) 2006 Ralf Baechle (ralf@linux-mips.org)
*
#include <linux/string.h>
#include <linux/init.h>
-#include <asm/bugs.h>
#include <asm/mmu_context.h>
-#include <asm/inst.h>
-#include <asm/elf.h>
#include <asm/war.h>
+#include "uasm.h"
+
static inline int r45k_bvahwbug(void)
{
/* XXX: We should probe for the presence of this bug, but we don't. */
(PRID_COMP_MIPS | PRID_IMP_4KC);
}
-/*
- * A little micro-assembler, intended for TLB refill handler
- * synthesizing. It is intentionally kept simple, does only support
- * a subset of instructions, and does not try to hide pipeline effects
- * like branch delay slots.
- */
-
-enum fields
-{
- RS = 0x001,
- RT = 0x002,
- RD = 0x004,
- RE = 0x008,
- SIMM = 0x010,
- UIMM = 0x020,
- BIMM = 0x040,
- JIMM = 0x080,
- FUNC = 0x100,
- SET = 0x200
-};
-
-#define OP_MASK 0x3f
-#define OP_SH 26
-#define RS_MASK 0x1f
-#define RS_SH 21
-#define RT_MASK 0x1f
-#define RT_SH 16
-#define RD_MASK 0x1f
-#define RD_SH 11
-#define RE_MASK 0x1f
-#define RE_SH 6
-#define IMM_MASK 0xffff
-#define IMM_SH 0
-#define JIMM_MASK 0x3ffffff
-#define JIMM_SH 0
-#define FUNC_MASK 0x3f
-#define FUNC_SH 0
-#define SET_MASK 0x7
-#define SET_SH 0
-
-enum opcode {
- insn_invalid,
- insn_addu, insn_addiu, insn_and, insn_andi, insn_beq,
- insn_beql, insn_bgez, insn_bgezl, insn_bltz, insn_bltzl,
- insn_bne, insn_daddu, insn_daddiu, insn_dmfc0, insn_dmtc0,
- insn_dsll, insn_dsll32, insn_dsra, insn_dsrl, insn_dsrl32,
- insn_dsubu, insn_eret, insn_j, insn_jal, insn_jr, insn_ld,
- insn_ll, insn_lld, insn_lui, insn_lw, insn_mfc0, insn_mtc0,
- insn_ori, insn_rfe, insn_sc, insn_scd, insn_sd, insn_sll,
- insn_sra, insn_srl, insn_subu, insn_sw, insn_tlbp, insn_tlbwi,
- insn_tlbwr, insn_xor, insn_xori
-};
-
-struct insn {
- enum opcode opcode;
- u32 match;
- enum fields fields;
-};
-
-/* This macro sets the non-variable bits of an instruction. */
-#define M(a, b, c, d, e, f) \
- ((a) << OP_SH \
- | (b) << RS_SH \
- | (c) << RT_SH \
- | (d) << RD_SH \
- | (e) << RE_SH \
- | (f) << FUNC_SH)
-
-static struct insn insn_table[] __initdata = {
- { insn_addiu, M(addiu_op, 0, 0, 0, 0, 0), RS | RT | SIMM },
- { insn_addu, M(spec_op, 0, 0, 0, 0, addu_op), RS | RT | RD },
- { insn_and, M(spec_op, 0, 0, 0, 0, and_op), RS | RT | RD },
- { insn_andi, M(andi_op, 0, 0, 0, 0, 0), RS | RT | UIMM },
- { insn_beq, M(beq_op, 0, 0, 0, 0, 0), RS | RT | BIMM },
- { insn_beql, M(beql_op, 0, 0, 0, 0, 0), RS | RT | BIMM },
- { insn_bgez, M(bcond_op, 0, bgez_op, 0, 0, 0), RS | BIMM },
- { insn_bgezl, M(bcond_op, 0, bgezl_op, 0, 0, 0), RS | BIMM },
- { insn_bltz, M(bcond_op, 0, bltz_op, 0, 0, 0), RS | BIMM },
- { insn_bltzl, M(bcond_op, 0, bltzl_op, 0, 0, 0), RS | BIMM },
- { insn_bne, M(bne_op, 0, 0, 0, 0, 0), RS | RT | BIMM },
- { insn_daddiu, M(daddiu_op, 0, 0, 0, 0, 0), RS | RT | SIMM },
- { insn_daddu, M(spec_op, 0, 0, 0, 0, daddu_op), RS | RT | RD },
- { insn_dmfc0, M(cop0_op, dmfc_op, 0, 0, 0, 0), RT | RD | SET},
- { insn_dmtc0, M(cop0_op, dmtc_op, 0, 0, 0, 0), RT | RD | SET},
- { insn_dsll, M(spec_op, 0, 0, 0, 0, dsll_op), RT | RD | RE },
- { insn_dsll32, M(spec_op, 0, 0, 0, 0, dsll32_op), RT | RD | RE },
- { insn_dsra, M(spec_op, 0, 0, 0, 0, dsra_op), RT | RD | RE },
- { insn_dsrl, M(spec_op, 0, 0, 0, 0, dsrl_op), RT | RD | RE },
- { insn_dsrl32, M(spec_op, 0, 0, 0, 0, dsrl32_op), RT | RD | RE },
- { insn_dsubu, M(spec_op, 0, 0, 0, 0, dsubu_op), RS | RT | RD },
- { insn_eret, M(cop0_op, cop_op, 0, 0, 0, eret_op), 0 },
- { insn_j, M(j_op, 0, 0, 0, 0, 0), JIMM },
- { insn_jal, M(jal_op, 0, 0, 0, 0, 0), JIMM },
- { insn_jr, M(spec_op, 0, 0, 0, 0, jr_op), RS },
- { insn_ld, M(ld_op, 0, 0, 0, 0, 0), RS | RT | SIMM },
- { insn_ll, M(ll_op, 0, 0, 0, 0, 0), RS | RT | SIMM },
- { insn_lld, M(lld_op, 0, 0, 0, 0, 0), RS | RT | SIMM },
- { insn_lui, M(lui_op, 0, 0, 0, 0, 0), RT | SIMM },
- { insn_lw, M(lw_op, 0, 0, 0, 0, 0), RS | RT | SIMM },
- { insn_mfc0, M(cop0_op, mfc_op, 0, 0, 0, 0), RT | RD | SET},
- { insn_mtc0, M(cop0_op, mtc_op, 0, 0, 0, 0), RT | RD | SET},
- { insn_ori, M(ori_op, 0, 0, 0, 0, 0), RS | RT | UIMM },
- { insn_rfe, M(cop0_op, cop_op, 0, 0, 0, rfe_op), 0 },
- { insn_sc, M(sc_op, 0, 0, 0, 0, 0), RS | RT | SIMM },
- { insn_scd, M(scd_op, 0, 0, 0, 0, 0), RS | RT | SIMM },
- { insn_sd, M(sd_op, 0, 0, 0, 0, 0), RS | RT | SIMM },
- { insn_sll, M(spec_op, 0, 0, 0, 0, sll_op), RT | RD | RE },
- { insn_sra, M(spec_op, 0, 0, 0, 0, sra_op), RT | RD | RE },
- { insn_srl, M(spec_op, 0, 0, 0, 0, srl_op), RT | RD | RE },
- { insn_subu, M(spec_op, 0, 0, 0, 0, subu_op), RS | RT | RD },
- { insn_sw, M(sw_op, 0, 0, 0, 0, 0), RS | RT | SIMM },
- { insn_tlbp, M(cop0_op, cop_op, 0, 0, 0, tlbp_op), 0 },
- { insn_tlbwi, M(cop0_op, cop_op, 0, 0, 0, tlbwi_op), 0 },
- { insn_tlbwr, M(cop0_op, cop_op, 0, 0, 0, tlbwr_op), 0 },
- { insn_xor, M(spec_op, 0, 0, 0, 0, xor_op), RS | RT | RD },
- { insn_xori, M(xori_op, 0, 0, 0, 0, 0), RS | RT | UIMM },
- { insn_invalid, 0, 0 }
-};
-
-#undef M
-
-static u32 __init build_rs(u32 arg)
-{
- if (arg & ~RS_MASK)
- printk(KERN_WARNING "TLB synthesizer field overflow\n");
-
- return (arg & RS_MASK) << RS_SH;
-}
-
-static u32 __init build_rt(u32 arg)
-{
- if (arg & ~RT_MASK)
- printk(KERN_WARNING "TLB synthesizer field overflow\n");
-
- return (arg & RT_MASK) << RT_SH;
-}
-
-static u32 __init build_rd(u32 arg)
-{
- if (arg & ~RD_MASK)
- printk(KERN_WARNING "TLB synthesizer field overflow\n");
-
- return (arg & RD_MASK) << RD_SH;
-}
-
-static u32 __init build_re(u32 arg)
-{
- if (arg & ~RE_MASK)
- printk(KERN_WARNING "TLB synthesizer field overflow\n");
-
- return (arg & RE_MASK) << RE_SH;
-}
-
-static u32 __init build_simm(s32 arg)
-{
- if (arg > 0x7fff || arg < -0x8000)
- printk(KERN_WARNING "TLB synthesizer field overflow\n");
-
- return arg & 0xffff;
-}
-
-static u32 __init build_uimm(u32 arg)
-{
- if (arg & ~IMM_MASK)
- printk(KERN_WARNING "TLB synthesizer field overflow\n");
-
- return arg & IMM_MASK;
-}
-
-static u32 __init build_bimm(s32 arg)
-{
- if (arg > 0x1ffff || arg < -0x20000)
- printk(KERN_WARNING "TLB synthesizer field overflow\n");
-
- if (arg & 0x3)
- printk(KERN_WARNING "Invalid TLB synthesizer branch target\n");
-
- return ((arg < 0) ? (1 << 15) : 0) | ((arg >> 2) & 0x7fff);
-}
-
-static u32 __init build_jimm(u32 arg)
-{
- if (arg & ~((JIMM_MASK) << 2))
- printk(KERN_WARNING "TLB synthesizer field overflow\n");
-
- return (arg >> 2) & JIMM_MASK;
-}
-
-static u32 __init build_func(u32 arg)
-{
- if (arg & ~FUNC_MASK)
- printk(KERN_WARNING "TLB synthesizer field overflow\n");
-
- return arg & FUNC_MASK;
-}
-
-static u32 __init build_set(u32 arg)
-{
- if (arg & ~SET_MASK)
- printk(KERN_WARNING "TLB synthesizer field overflow\n");
-
- return arg & SET_MASK;
-}
-
-/*
- * The order of opcode arguments is implicitly left to right,
- * starting with RS and ending with FUNC or IMM.
- */
-static void __init build_insn(u32 **buf, enum opcode opc, ...)
-{
- struct insn *ip = NULL;
- unsigned int i;
- va_list ap;
- u32 op;
-
- for (i = 0; insn_table[i].opcode != insn_invalid; i++)
- if (insn_table[i].opcode == opc) {
- ip = &insn_table[i];
- break;
- }
-
- if (!ip || (opc == insn_daddiu && r4k_daddiu_bug()))
- panic("Unsupported TLB synthesizer instruction %d", opc);
-
- op = ip->match;
- va_start(ap, opc);
- if (ip->fields & RS) op |= build_rs(va_arg(ap, u32));
- if (ip->fields & RT) op |= build_rt(va_arg(ap, u32));
- if (ip->fields & RD) op |= build_rd(va_arg(ap, u32));
- if (ip->fields & RE) op |= build_re(va_arg(ap, u32));
- if (ip->fields & SIMM) op |= build_simm(va_arg(ap, s32));
- if (ip->fields & UIMM) op |= build_uimm(va_arg(ap, u32));
- if (ip->fields & BIMM) op |= build_bimm(va_arg(ap, s32));
- if (ip->fields & JIMM) op |= build_jimm(va_arg(ap, u32));
- if (ip->fields & FUNC) op |= build_func(va_arg(ap, u32));
- if (ip->fields & SET) op |= build_set(va_arg(ap, u32));
- va_end(ap);
-
- **buf = op;
- (*buf)++;
-}
-
-#define I_u1u2u3(op) \
- static void __init __maybe_unused i##op(u32 **buf, unsigned int a, \
- unsigned int b, unsigned int c) \
- { \
- build_insn(buf, insn##op, a, b, c); \
- }
-
-#define I_u2u1u3(op) \
- static void __init __maybe_unused i##op(u32 **buf, unsigned int a, \
- unsigned int b, unsigned int c) \
- { \
- build_insn(buf, insn##op, b, a, c); \
- }
-
-#define I_u3u1u2(op) \
- static void __init __maybe_unused i##op(u32 **buf, unsigned int a, \
- unsigned int b, unsigned int c) \
- { \
- build_insn(buf, insn##op, b, c, a); \
- }
-
-#define I_u1u2s3(op) \
- static void __init __maybe_unused i##op(u32 **buf, unsigned int a, \
- unsigned int b, signed int c) \
- { \
- build_insn(buf, insn##op, a, b, c); \
- }
-
-#define I_u2s3u1(op) \
- static void __init __maybe_unused i##op(u32 **buf, unsigned int a, \
- signed int b, unsigned int c) \
- { \
- build_insn(buf, insn##op, c, a, b); \
- }
-
-#define I_u2u1s3(op) \
- static void __init __maybe_unused i##op(u32 **buf, unsigned int a, \
- unsigned int b, signed int c) \
- { \
- build_insn(buf, insn##op, b, a, c); \
- }
-
-#define I_u1u2(op) \
- static void __init __maybe_unused i##op(u32 **buf, unsigned int a, \
- unsigned int b) \
- { \
- build_insn(buf, insn##op, a, b); \
- }
-
-#define I_u1s2(op) \
- static void __init __maybe_unused i##op(u32 **buf, unsigned int a, \
- signed int b) \
- { \
- build_insn(buf, insn##op, a, b); \
- }
-
-#define I_u1(op) \
- static void __init __maybe_unused i##op(u32 **buf, unsigned int a) \
- { \
- build_insn(buf, insn##op, a); \
- }
-
-#define I_0(op) \
- static void __init __maybe_unused i##op(u32 **buf) \
- { \
- build_insn(buf, insn##op); \
- }
-
-I_u2u1s3(_addiu);
-I_u3u1u2(_addu);
-I_u2u1u3(_andi);
-I_u3u1u2(_and);
-I_u1u2s3(_beq);
-I_u1u2s3(_beql);
-I_u1s2(_bgez);
-I_u1s2(_bgezl);
-I_u1s2(_bltz);
-I_u1s2(_bltzl);
-I_u1u2s3(_bne);
-I_u1u2u3(_dmfc0);
-I_u1u2u3(_dmtc0);
-I_u2u1s3(_daddiu);
-I_u3u1u2(_daddu);
-I_u2u1u3(_dsll);
-I_u2u1u3(_dsll32);
-I_u2u1u3(_dsra);
-I_u2u1u3(_dsrl);
-I_u2u1u3(_dsrl32);
-I_u3u1u2(_dsubu);
-I_0(_eret);
-I_u1(_j);
-I_u1(_jal);
-I_u1(_jr);
-I_u2s3u1(_ld);
-I_u2s3u1(_ll);
-I_u2s3u1(_lld);
-I_u1s2(_lui);
-I_u2s3u1(_lw);
-I_u1u2u3(_mfc0);
-I_u1u2u3(_mtc0);
-I_u2u1u3(_ori);
-I_0(_rfe);
-I_u2s3u1(_sc);
-I_u2s3u1(_scd);
-I_u2s3u1(_sd);
-I_u2u1u3(_sll);
-I_u2u1u3(_sra);
-I_u2u1u3(_srl);
-I_u3u1u2(_subu);
-I_u2s3u1(_sw);
-I_0(_tlbp);
-I_0(_tlbwi);
-I_0(_tlbwr);
-I_u3u1u2(_xor)
-I_u2u1u3(_xori);
-
-/*
- * handling labels
- */
-
+/* Handle labels (which must be positive integers). */
enum label_id {
- label_invalid,
- label_second_part,
+ label_second_part = 1,
label_leave,
#ifdef MODULE_START
label_module_alloc,
label_r3000_write_probe_fail,
};
-struct label {
- u32 *addr;
- enum label_id lab;
-};
-
-static void __init build_label(struct label **lab, u32 *addr,
- enum label_id l)
-{
- (*lab)->addr = addr;
- (*lab)->lab = l;
- (*lab)++;
-}
-
-#define L_LA(lb) \
- static inline void __init l##lb(struct label **lab, u32 *addr) \
- { \
- build_label(lab, addr, label##lb); \
- }
-
-L_LA(_second_part)
-L_LA(_leave)
+UASM_L_LA(_second_part)
+UASM_L_LA(_leave)
#ifdef MODULE_START
-L_LA(_module_alloc)
-#endif
-L_LA(_vmalloc)
-L_LA(_vmalloc_done)
-L_LA(_tlbw_hazard)
-L_LA(_split)
-L_LA(_nopage_tlbl)
-L_LA(_nopage_tlbs)
-L_LA(_nopage_tlbm)
-L_LA(_smp_pgtable_change)
-L_LA(_r3000_write_probe_fail)
-
-/* convenience macros for instructions */
-#ifdef CONFIG_64BIT
-# define i_LW(buf, rs, rt, off) i_ld(buf, rs, rt, off)
-# define i_SW(buf, rs, rt, off) i_sd(buf, rs, rt, off)
-# define i_SLL(buf, rs, rt, sh) i_dsll(buf, rs, rt, sh)
-# define i_SRA(buf, rs, rt, sh) i_dsra(buf, rs, rt, sh)
-# define i_SRL(buf, rs, rt, sh) i_dsrl(buf, rs, rt, sh)
-# define i_MFC0(buf, rt, rd...) i_dmfc0(buf, rt, rd)
-# define i_MTC0(buf, rt, rd...) i_dmtc0(buf, rt, rd)
-# define i_ADDIU(buf, rs, rt, val) i_daddiu(buf, rs, rt, val)
-# define i_ADDU(buf, rs, rt, rd) i_daddu(buf, rs, rt, rd)
-# define i_SUBU(buf, rs, rt, rd) i_dsubu(buf, rs, rt, rd)
-# define i_LL(buf, rs, rt, off) i_lld(buf, rs, rt, off)
-# define i_SC(buf, rs, rt, off) i_scd(buf, rs, rt, off)
-#else
-# define i_LW(buf, rs, rt, off) i_lw(buf, rs, rt, off)
-# define i_SW(buf, rs, rt, off) i_sw(buf, rs, rt, off)
-# define i_SLL(buf, rs, rt, sh) i_sll(buf, rs, rt, sh)
-# define i_SRA(buf, rs, rt, sh) i_sra(buf, rs, rt, sh)
-# define i_SRL(buf, rs, rt, sh) i_srl(buf, rs, rt, sh)
-# define i_MFC0(buf, rt, rd...) i_mfc0(buf, rt, rd)
-# define i_MTC0(buf, rt, rd...) i_mtc0(buf, rt, rd)
-# define i_ADDIU(buf, rs, rt, val) i_addiu(buf, rs, rt, val)
-# define i_ADDU(buf, rs, rt, rd) i_addu(buf, rs, rt, rd)
-# define i_SUBU(buf, rs, rt, rd) i_subu(buf, rs, rt, rd)
-# define i_LL(buf, rs, rt, off) i_ll(buf, rs, rt, off)
-# define i_SC(buf, rs, rt, off) i_sc(buf, rs, rt, off)
-#endif
-
-#define i_b(buf, off) i_beq(buf, 0, 0, off)
-#define i_beqz(buf, rs, off) i_beq(buf, rs, 0, off)
-#define i_beqzl(buf, rs, off) i_beql(buf, rs, 0, off)
-#define i_bnez(buf, rs, off) i_bne(buf, rs, 0, off)
-#define i_bnezl(buf, rs, off) i_bnel(buf, rs, 0, off)
-#define i_move(buf, a, b) i_ADDU(buf, a, 0, b)
-#define i_nop(buf) i_sll(buf, 0, 0, 0)
-#define i_ssnop(buf) i_sll(buf, 0, 0, 1)
-#define i_ehb(buf) i_sll(buf, 0, 0, 3)
-
-static int __init __maybe_unused in_compat_space_p(long addr)
-{
- /* Is this address in 32bit compat space? */
-#ifdef CONFIG_64BIT
- return (((addr) & 0xffffffff00000000L) == 0xffffffff00000000L);
-#else
- return 1;
+UASM_L_LA(_module_alloc)
#endif
-}
-
-static int __init __maybe_unused rel_highest(long val)
-{
-#ifdef CONFIG_64BIT
- return ((((val + 0x800080008000L) >> 48) & 0xffff) ^ 0x8000) - 0x8000;
-#else
- return 0;
-#endif
-}
-
-static int __init __maybe_unused rel_higher(long val)
-{
-#ifdef CONFIG_64BIT
- return ((((val + 0x80008000L) >> 32) & 0xffff) ^ 0x8000) - 0x8000;
-#else
- return 0;
-#endif
-}
-
-static int __init rel_hi(long val)
-{
- return ((((val + 0x8000L) >> 16) & 0xffff) ^ 0x8000) - 0x8000;
-}
-
-static int __init rel_lo(long val)
-{
- return ((val & 0xffff) ^ 0x8000) - 0x8000;
-}
-
-static void __init i_LA_mostly(u32 **buf, unsigned int rs, long addr)
-{
- if (!in_compat_space_p(addr)) {
- i_lui(buf, rs, rel_highest(addr));
- if (rel_higher(addr))
- i_daddiu(buf, rs, rs, rel_higher(addr));
- if (rel_hi(addr)) {
- i_dsll(buf, rs, rs, 16);
- i_daddiu(buf, rs, rs, rel_hi(addr));
- i_dsll(buf, rs, rs, 16);
- } else
- i_dsll32(buf, rs, rs, 0);
- } else
- i_lui(buf, rs, rel_hi(addr));
-}
-
-static void __init __maybe_unused i_LA(u32 **buf, unsigned int rs, long addr)
-{
- i_LA_mostly(buf, rs, addr);
- if (rel_lo(addr)) {
- if (!in_compat_space_p(addr))
- i_daddiu(buf, rs, rs, rel_lo(addr));
- else
- i_addiu(buf, rs, rs, rel_lo(addr));
- }
-}
-
-/*
- * handle relocations
- */
-
-struct reloc {
- u32 *addr;
- unsigned int type;
- enum label_id lab;
-};
-
-static void __init r_mips_pc16(struct reloc **rel, u32 *addr,
- enum label_id l)
-{
- (*rel)->addr = addr;
- (*rel)->type = R_MIPS_PC16;
- (*rel)->lab = l;
- (*rel)++;
-}
-
-static inline void __resolve_relocs(struct reloc *rel, struct label *lab)
-{
- long laddr = (long)lab->addr;
- long raddr = (long)rel->addr;
-
- switch (rel->type) {
- case R_MIPS_PC16:
- *rel->addr |= build_bimm(laddr - (raddr + 4));
- break;
-
- default:
- panic("Unsupported TLB synthesizer relocation %d",
- rel->type);
- }
-}
-
-static void __init resolve_relocs(struct reloc *rel, struct label *lab)
-{
- struct label *l;
-
- for (; rel->lab != label_invalid; rel++)
- for (l = lab; l->lab != label_invalid; l++)
- if (rel->lab == l->lab)
- __resolve_relocs(rel, l);
-}
-
-static void __init move_relocs(struct reloc *rel, u32 *first, u32 *end,
- long off)
-{
- for (; rel->lab != label_invalid; rel++)
- if (rel->addr >= first && rel->addr < end)
- rel->addr += off;
-}
-
-static void __init move_labels(struct label *lab, u32 *first, u32 *end,
- long off)
-{
- for (; lab->lab != label_invalid; lab++)
- if (lab->addr >= first && lab->addr < end)
- lab->addr += off;
-}
-
-static void __init copy_handler(struct reloc *rel, struct label *lab,
- u32 *first, u32 *end, u32 *target)
-{
- long off = (long)(target - first);
-
- memcpy(target, first, (end - first) * sizeof(u32));
-
- move_relocs(rel, first, end, off);
- move_labels(lab, first, end, off);
-}
-
-static int __init __maybe_unused insn_has_bdelay(struct reloc *rel,
- u32 *addr)
-{
- for (; rel->lab != label_invalid; rel++) {
- if (rel->addr == addr
- && (rel->type == R_MIPS_PC16
- || rel->type == R_MIPS_26))
- return 1;
- }
-
- return 0;
-}
-
-/* convenience functions for labeled branches */
-static void __init __maybe_unused
- il_bltz(u32 **p, struct reloc **r, unsigned int reg, enum label_id l)
-{
- r_mips_pc16(r, *p, l);
- i_bltz(p, reg, 0);
-}
-
-static void __init __maybe_unused il_b(u32 **p, struct reloc **r,
- enum label_id l)
-{
- r_mips_pc16(r, *p, l);
- i_b(p, 0);
-}
-
-static void __init il_beqz(u32 **p, struct reloc **r, unsigned int reg,
- enum label_id l)
-{
- r_mips_pc16(r, *p, l);
- i_beqz(p, reg, 0);
-}
-
-static void __init __maybe_unused
-il_beqzl(u32 **p, struct reloc **r, unsigned int reg, enum label_id l)
-{
- r_mips_pc16(r, *p, l);
- i_beqzl(p, reg, 0);
-}
-
-static void __init il_bnez(u32 **p, struct reloc **r, unsigned int reg,
- enum label_id l)
-{
- r_mips_pc16(r, *p, l);
- i_bnez(p, reg, 0);
-}
-
-static void __init il_bgezl(u32 **p, struct reloc **r, unsigned int reg,
- enum label_id l)
-{
- r_mips_pc16(r, *p, l);
- i_bgezl(p, reg, 0);
-}
-
-static void __init __maybe_unused
-il_bgez(u32 **p, struct reloc **r, unsigned int reg, enum label_id l)
-{
- r_mips_pc16(r, *p, l);
- i_bgez(p, reg, 0);
-}
+UASM_L_LA(_vmalloc)
+UASM_L_LA(_vmalloc_done)
+UASM_L_LA(_tlbw_hazard)
+UASM_L_LA(_split)
+UASM_L_LA(_nopage_tlbl)
+UASM_L_LA(_nopage_tlbs)
+UASM_L_LA(_nopage_tlbm)
+UASM_L_LA(_smp_pgtable_change)
+UASM_L_LA(_r3000_write_probe_fail)
/*
* For debug purposes.
#define C0_XCONTEXT 20, 0
#ifdef CONFIG_64BIT
-# define GET_CONTEXT(buf, reg) i_MFC0(buf, reg, C0_XCONTEXT)
+# define GET_CONTEXT(buf, reg) UASM_i_MFC0(buf, reg, C0_XCONTEXT)
#else
-# define GET_CONTEXT(buf, reg) i_MFC0(buf, reg, C0_CONTEXT)
+# define GET_CONTEXT(buf, reg) UASM_i_MFC0(buf, reg, C0_CONTEXT)
#endif
/* The worst case length of the handler is around 18 instructions for
static u32 tlb_handler[128] __initdata;
/* simply assume worst case size for labels and relocs */
-static struct label labels[128] __initdata;
-static struct reloc relocs[128] __initdata;
+static struct uasm_label labels[128] __initdata;
+static struct uasm_reloc relocs[128] __initdata;
/*
* The R3000 TLB handler is simple.
memset(tlb_handler, 0, sizeof(tlb_handler));
p = tlb_handler;
- i_mfc0(&p, K0, C0_BADVADDR);
- i_lui(&p, K1, rel_hi(pgdc)); /* cp0 delay */
- i_lw(&p, K1, rel_lo(pgdc), K1);
- i_srl(&p, K0, K0, 22); /* load delay */
- i_sll(&p, K0, K0, 2);
- i_addu(&p, K1, K1, K0);
- i_mfc0(&p, K0, C0_CONTEXT);
- i_lw(&p, K1, 0, K1); /* cp0 delay */
- i_andi(&p, K0, K0, 0xffc); /* load delay */
- i_addu(&p, K1, K1, K0);
- i_lw(&p, K0, 0, K1);
- i_nop(&p); /* load delay */
- i_mtc0(&p, K0, C0_ENTRYLO0);
- i_mfc0(&p, K1, C0_EPC); /* cp0 delay */
- i_tlbwr(&p); /* cp0 delay */
- i_jr(&p, K1);
- i_rfe(&p); /* branch delay */
+ uasm_i_mfc0(&p, K0, C0_BADVADDR);
+ uasm_i_lui(&p, K1, uasm_rel_hi(pgdc)); /* cp0 delay */
+ uasm_i_lw(&p, K1, uasm_rel_lo(pgdc), K1);
+ uasm_i_srl(&p, K0, K0, 22); /* load delay */
+ uasm_i_sll(&p, K0, K0, 2);
+ uasm_i_addu(&p, K1, K1, K0);
+ uasm_i_mfc0(&p, K0, C0_CONTEXT);
+ uasm_i_lw(&p, K1, 0, K1); /* cp0 delay */
+ uasm_i_andi(&p, K0, K0, 0xffc); /* load delay */
+ uasm_i_addu(&p, K1, K1, K0);
+ uasm_i_lw(&p, K0, 0, K1);
+ uasm_i_nop(&p); /* load delay */
+ uasm_i_mtc0(&p, K0, C0_ENTRYLO0);
+ uasm_i_mfc0(&p, K1, C0_EPC); /* cp0 delay */
+ uasm_i_tlbwr(&p); /* cp0 delay */
+ uasm_i_jr(&p, K1);
+ uasm_i_rfe(&p); /* branch delay */
if (p > tlb_handler + 32)
panic("TLB refill handler space exceeded");
- pr_info("Synthesized TLB refill handler (%u instructions).\n",
- (unsigned int)(p - tlb_handler));
+ pr_debug("Wrote TLB refill handler (%u instructions).\n",
+ (unsigned int)(p - tlb_handler));
memcpy((void *)ebase, tlb_handler, 0x80);
case CPU_R5000:
case CPU_R5000A:
case CPU_NEVADA:
- i_nop(p);
- i_tlbp(p);
+ uasm_i_nop(p);
+ uasm_i_tlbp(p);
break;
default:
- i_tlbp(p);
+ uasm_i_tlbp(p);
break;
}
}
*/
enum tlb_write_entry { tlb_random, tlb_indexed };
-static void __init build_tlb_write_entry(u32 **p, struct label **l,
- struct reloc **r,
+static void __init build_tlb_write_entry(u32 **p, struct uasm_label **l,
+ struct uasm_reloc **r,
enum tlb_write_entry wmode)
{
void(*tlbw)(u32 **) = NULL;
switch (wmode) {
- case tlb_random: tlbw = i_tlbwr; break;
- case tlb_indexed: tlbw = i_tlbwi; break;
+ case tlb_random: tlbw = uasm_i_tlbwr; break;
+ case tlb_indexed: tlbw = uasm_i_tlbwi; break;
}
if (cpu_has_mips_r2) {
- i_ehb(p);
+ uasm_i_ehb(p);
tlbw(p);
return;
}
* This branch uses up a mtc0 hazard nop slot and saves
* two nops after the tlbw instruction.
*/
- il_bgezl(p, r, 0, label_tlbw_hazard);
+ uasm_il_bgezl(p, r, 0, label_tlbw_hazard);
tlbw(p);
- l_tlbw_hazard(l, *p);
- i_nop(p);
+ uasm_l_tlbw_hazard(l, *p);
+ uasm_i_nop(p);
break;
case CPU_R4600:
case CPU_R4700:
case CPU_R5000:
case CPU_R5000A:
- i_nop(p);
+ uasm_i_nop(p);
tlbw(p);
- i_nop(p);
+ uasm_i_nop(p);
break;
case CPU_R4300:
case CPU_AU1210:
case CPU_AU1250:
case CPU_PR4450:
- i_nop(p);
+ uasm_i_nop(p);
tlbw(p);
break;
case CPU_BCM4710:
case CPU_LOONGSON2:
if (m4kc_tlbp_war())
- i_nop(p);
+ uasm_i_nop(p);
tlbw(p);
break;
case CPU_NEVADA:
- i_nop(p); /* QED specifies 2 nops hazard */
+ uasm_i_nop(p); /* QED specifies 2 nops hazard */
/*
* This branch uses up a mtc0 hazard nop slot and saves
* a nop after the tlbw instruction.
*/
- il_bgezl(p, r, 0, label_tlbw_hazard);
+ uasm_il_bgezl(p, r, 0, label_tlbw_hazard);
tlbw(p);
- l_tlbw_hazard(l, *p);
+ uasm_l_tlbw_hazard(l, *p);
break;
case CPU_RM7000:
- i_nop(p);
- i_nop(p);
- i_nop(p);
- i_nop(p);
+ uasm_i_nop(p);
+ uasm_i_nop(p);
+ uasm_i_nop(p);
+ uasm_i_nop(p);
tlbw(p);
break;
* cpu cycles and use for data translations should not occur
* for 3 cpu cycles.
*/
- i_ssnop(p);
- i_ssnop(p);
- i_ssnop(p);
- i_ssnop(p);
+ uasm_i_ssnop(p);
+ uasm_i_ssnop(p);
+ uasm_i_ssnop(p);
+ uasm_i_ssnop(p);
tlbw(p);
- i_ssnop(p);
- i_ssnop(p);
- i_ssnop(p);
- i_ssnop(p);
+ uasm_i_ssnop(p);
+ uasm_i_ssnop(p);
+ uasm_i_ssnop(p);
+ uasm_i_ssnop(p);
break;
case CPU_VR4111:
case CPU_VR4122:
case CPU_VR4181:
case CPU_VR4181A:
- i_nop(p);
- i_nop(p);
+ uasm_i_nop(p);
+ uasm_i_nop(p);
tlbw(p);
- i_nop(p);
- i_nop(p);
+ uasm_i_nop(p);
+ uasm_i_nop(p);
break;
case CPU_VR4131:
case CPU_VR4133:
case CPU_R5432:
- i_nop(p);
- i_nop(p);
+ uasm_i_nop(p);
+ uasm_i_nop(p);
tlbw(p);
break;
* TMP will be clobbered, PTR will hold the pmd entry.
*/
static void __init
-build_get_pmde64(u32 **p, struct label **l, struct reloc **r,
+build_get_pmde64(u32 **p, struct uasm_label **l, struct uasm_reloc **r,
unsigned int tmp, unsigned int ptr)
{
long pgdc = (long)pgd_current;
/*
* The vmalloc handling is not in the hotpath.
*/
- i_dmfc0(p, tmp, C0_BADVADDR);
+ uasm_i_dmfc0(p, tmp, C0_BADVADDR);
#ifdef MODULE_START
- il_bltz(p, r, tmp, label_module_alloc);
+ uasm_il_bltz(p, r, tmp, label_module_alloc);
#else
- il_bltz(p, r, tmp, label_vmalloc);
+ uasm_il_bltz(p, r, tmp, label_vmalloc);
#endif
- /* No i_nop needed here, since the next insn doesn't touch TMP. */
+ /* No uasm_i_nop needed here, since the next insn doesn't touch TMP. */
#ifdef CONFIG_SMP
# ifdef CONFIG_MIPS_MT_SMTC
/*
* SMTC uses TCBind value as "CPU" index
*/
- i_mfc0(p, ptr, C0_TCBIND);
- i_dsrl(p, ptr, ptr, 19);
+ uasm_i_mfc0(p, ptr, C0_TCBIND);
+ uasm_i_dsrl(p, ptr, ptr, 19);
# else
/*
* 64 bit SMP running in XKPHYS has smp_processor_id() << 3
* stored in CONTEXT.
*/
- i_dmfc0(p, ptr, C0_CONTEXT);
- i_dsrl(p, ptr, ptr, 23);
+ uasm_i_dmfc0(p, ptr, C0_CONTEXT);
+ uasm_i_dsrl(p, ptr, ptr, 23);
#endif
- i_LA_mostly(p, tmp, pgdc);
- i_daddu(p, ptr, ptr, tmp);
- i_dmfc0(p, tmp, C0_BADVADDR);
- i_ld(p, ptr, rel_lo(pgdc), ptr);
+ UASM_i_LA_mostly(p, tmp, pgdc);
+ uasm_i_daddu(p, ptr, ptr, tmp);
+ uasm_i_dmfc0(p, tmp, C0_BADVADDR);
+ uasm_i_ld(p, ptr, uasm_rel_lo(pgdc), ptr);
#else
- i_LA_mostly(p, ptr, pgdc);
- i_ld(p, ptr, rel_lo(pgdc), ptr);
+ UASM_i_LA_mostly(p, ptr, pgdc);
+ uasm_i_ld(p, ptr, uasm_rel_lo(pgdc), ptr);
#endif
- l_vmalloc_done(l, *p);
+ uasm_l_vmalloc_done(l, *p);
if (PGDIR_SHIFT - 3 < 32) /* get pgd offset in bytes */
- i_dsrl(p, tmp, tmp, PGDIR_SHIFT-3);
+ uasm_i_dsrl(p, tmp, tmp, PGDIR_SHIFT-3);
else
- i_dsrl32(p, tmp, tmp, PGDIR_SHIFT - 3 - 32);
-
- i_andi(p, tmp, tmp, (PTRS_PER_PGD - 1)<<3);
- i_daddu(p, ptr, ptr, tmp); /* add in pgd offset */
- i_dmfc0(p, tmp, C0_BADVADDR); /* get faulting address */
- i_ld(p, ptr, 0, ptr); /* get pmd pointer */
- i_dsrl(p, tmp, tmp, PMD_SHIFT-3); /* get pmd offset in bytes */
- i_andi(p, tmp, tmp, (PTRS_PER_PMD - 1)<<3);
- i_daddu(p, ptr, ptr, tmp); /* add in pmd offset */
+ uasm_i_dsrl32(p, tmp, tmp, PGDIR_SHIFT - 3 - 32);
+
+ uasm_i_andi(p, tmp, tmp, (PTRS_PER_PGD - 1)<<3);
+ uasm_i_daddu(p, ptr, ptr, tmp); /* add in pgd offset */
+ uasm_i_dmfc0(p, tmp, C0_BADVADDR); /* get faulting address */
+ uasm_i_ld(p, ptr, 0, ptr); /* get pmd pointer */
+ uasm_i_dsrl(p, tmp, tmp, PMD_SHIFT-3); /* get pmd offset in bytes */
+ uasm_i_andi(p, tmp, tmp, (PTRS_PER_PMD - 1)<<3);
+ uasm_i_daddu(p, ptr, ptr, tmp); /* add in pmd offset */
}
/*
* PTR will hold the pgd for vmalloc.
*/
static void __init
-build_get_pgd_vmalloc64(u32 **p, struct label **l, struct reloc **r,
+build_get_pgd_vmalloc64(u32 **p, struct uasm_label **l, struct uasm_reloc **r,
unsigned int bvaddr, unsigned int ptr)
{
long swpd = (long)swapper_pg_dir;
#ifdef MODULE_START
long modd = (long)module_pg_dir;
- l_module_alloc(l, *p);
+ uasm_l_module_alloc(l, *p);
/*
* Assumption:
* VMALLOC_START >= 0xc000000000000000UL
* MODULE_START >= 0xe000000000000000UL
*/
- i_SLL(p, ptr, bvaddr, 2);
- il_bgez(p, r, ptr, label_vmalloc);
+ UASM_i_SLL(p, ptr, bvaddr, 2);
+ uasm_il_bgez(p, r, ptr, label_vmalloc);
- if (in_compat_space_p(MODULE_START) && !rel_lo(MODULE_START)) {
- i_lui(p, ptr, rel_hi(MODULE_START)); /* delay slot */
+ if (uasm_in_compat_space_p(MODULE_START) &&
+ !uasm_rel_lo(MODULE_START)) {
+ uasm_i_lui(p, ptr, uasm_rel_hi(MODULE_START)); /* delay slot */
} else {
/* unlikely configuration */
- i_nop(p); /* delay slot */
- i_LA(p, ptr, MODULE_START);
+ uasm_i_nop(p); /* delay slot */
+ UASM_i_LA(p, ptr, MODULE_START);
}
- i_dsubu(p, bvaddr, bvaddr, ptr);
+ uasm_i_dsubu(p, bvaddr, bvaddr, ptr);
- if (in_compat_space_p(modd) && !rel_lo(modd)) {
- il_b(p, r, label_vmalloc_done);
- i_lui(p, ptr, rel_hi(modd));
+ if (uasm_in_compat_space_p(modd) && !uasm_rel_lo(modd)) {
+ uasm_il_b(p, r, label_vmalloc_done);
+ uasm_i_lui(p, ptr, uasm_rel_hi(modd));
} else {
- i_LA_mostly(p, ptr, modd);
- il_b(p, r, label_vmalloc_done);
- if (in_compat_space_p(modd))
- i_addiu(p, ptr, ptr, rel_lo(modd));
+ UASM_i_LA_mostly(p, ptr, modd);
+ uasm_il_b(p, r, label_vmalloc_done);
+ if (uasm_in_compat_space_p(modd))
+ uasm_i_addiu(p, ptr, ptr, uasm_rel_lo(modd));
else
- i_daddiu(p, ptr, ptr, rel_lo(modd));
+ uasm_i_daddiu(p, ptr, ptr, uasm_rel_lo(modd));
}
- l_vmalloc(l, *p);
- if (in_compat_space_p(MODULE_START) && !rel_lo(MODULE_START) &&
+ uasm_l_vmalloc(l, *p);
+ if (uasm_in_compat_space_p(MODULE_START) &&
+ !uasm_rel_lo(MODULE_START) &&
MODULE_START << 32 == VMALLOC_START)
- i_dsll32(p, ptr, ptr, 0); /* typical case */
+ uasm_i_dsll32(p, ptr, ptr, 0); /* typical case */
else
- i_LA(p, ptr, VMALLOC_START);
+ UASM_i_LA(p, ptr, VMALLOC_START);
#else
- l_vmalloc(l, *p);
- i_LA(p, ptr, VMALLOC_START);
+ uasm_l_vmalloc(l, *p);
+ UASM_i_LA(p, ptr, VMALLOC_START);
#endif
- i_dsubu(p, bvaddr, bvaddr, ptr);
+ uasm_i_dsubu(p, bvaddr, bvaddr, ptr);
- if (in_compat_space_p(swpd) && !rel_lo(swpd)) {
- il_b(p, r, label_vmalloc_done);
- i_lui(p, ptr, rel_hi(swpd));
+ if (uasm_in_compat_space_p(swpd) && !uasm_rel_lo(swpd)) {
+ uasm_il_b(p, r, label_vmalloc_done);
+ uasm_i_lui(p, ptr, uasm_rel_hi(swpd));
} else {
- i_LA_mostly(p, ptr, swpd);
- il_b(p, r, label_vmalloc_done);
- if (in_compat_space_p(swpd))
- i_addiu(p, ptr, ptr, rel_lo(swpd));
+ UASM_i_LA_mostly(p, ptr, swpd);
+ uasm_il_b(p, r, label_vmalloc_done);
+ if (uasm_in_compat_space_p(swpd))
+ uasm_i_addiu(p, ptr, ptr, uasm_rel_lo(swpd));
else
- i_daddiu(p, ptr, ptr, rel_lo(swpd));
+ uasm_i_daddiu(p, ptr, ptr, uasm_rel_lo(swpd));
}
}
/*
* SMTC uses TCBind value as "CPU" index
*/
- i_mfc0(p, ptr, C0_TCBIND);
- i_LA_mostly(p, tmp, pgdc);
- i_srl(p, ptr, ptr, 19);
+ uasm_i_mfc0(p, ptr, C0_TCBIND);
+ UASM_i_LA_mostly(p, tmp, pgdc);
+ uasm_i_srl(p, ptr, ptr, 19);
#else
/*
* smp_processor_id() << 3 is stored in CONTEXT.
*/
- i_mfc0(p, ptr, C0_CONTEXT);
- i_LA_mostly(p, tmp, pgdc);
- i_srl(p, ptr, ptr, 23);
+ uasm_i_mfc0(p, ptr, C0_CONTEXT);
+ UASM_i_LA_mostly(p, tmp, pgdc);
+ uasm_i_srl(p, ptr, ptr, 23);
#endif
- i_addu(p, ptr, tmp, ptr);
+ uasm_i_addu(p, ptr, tmp, ptr);
#else
- i_LA_mostly(p, ptr, pgdc);
+ UASM_i_LA_mostly(p, ptr, pgdc);
#endif
- i_mfc0(p, tmp, C0_BADVADDR); /* get faulting address */
- i_lw(p, ptr, rel_lo(pgdc), ptr);
- i_srl(p, tmp, tmp, PGDIR_SHIFT); /* get pgd only bits */
- i_sll(p, tmp, tmp, PGD_T_LOG2);
- i_addu(p, ptr, ptr, tmp); /* add in pgd offset */
+ uasm_i_mfc0(p, tmp, C0_BADVADDR); /* get faulting address */
+ uasm_i_lw(p, ptr, uasm_rel_lo(pgdc), ptr);
+ uasm_i_srl(p, tmp, tmp, PGDIR_SHIFT); /* get pgd only bits */
+ uasm_i_sll(p, tmp, tmp, PGD_T_LOG2);
+ uasm_i_addu(p, ptr, ptr, tmp); /* add in pgd offset */
}
#endif /* !CONFIG_64BIT */
}
if (shift)
- i_SRL(p, ctx, ctx, shift);
- i_andi(p, ctx, ctx, mask);
+ UASM_i_SRL(p, ctx, ctx, shift);
+ uasm_i_andi(p, ctx, ctx, mask);
}
static void __init build_get_ptep(u32 **p, unsigned int tmp, unsigned int ptr)
*/
switch (current_cpu_type()) {
case CPU_NEVADA:
- i_LW(p, ptr, 0, ptr);
+ UASM_i_LW(p, ptr, 0, ptr);
GET_CONTEXT(p, tmp); /* get context reg */
break;
default:
GET_CONTEXT(p, tmp); /* get context reg */
- i_LW(p, ptr, 0, ptr);
+ UASM_i_LW(p, ptr, 0, ptr);
break;
}
build_adjust_context(p, tmp);
- i_ADDU(p, ptr, ptr, tmp); /* add in offset */
+ UASM_i_ADDU(p, ptr, ptr, tmp); /* add in offset */
}
static void __init build_update_entries(u32 **p, unsigned int tmp,
*/
#ifdef CONFIG_64BIT_PHYS_ADDR
if (cpu_has_64bits) {
- i_ld(p, tmp, 0, ptep); /* get even pte */
- i_ld(p, ptep, sizeof(pte_t), ptep); /* get odd pte */
- i_dsrl(p, tmp, tmp, 6); /* convert to entrylo0 */
- i_mtc0(p, tmp, C0_ENTRYLO0); /* load it */
- i_dsrl(p, ptep, ptep, 6); /* convert to entrylo1 */
- i_mtc0(p, ptep, C0_ENTRYLO1); /* load it */
+ uasm_i_ld(p, tmp, 0, ptep); /* get even pte */
+ uasm_i_ld(p, ptep, sizeof(pte_t), ptep); /* get odd pte */
+ uasm_i_dsrl(p, tmp, tmp, 6); /* convert to entrylo0 */
+ uasm_i_mtc0(p, tmp, C0_ENTRYLO0); /* load it */
+ uasm_i_dsrl(p, ptep, ptep, 6); /* convert to entrylo1 */
+ uasm_i_mtc0(p, ptep, C0_ENTRYLO1); /* load it */
} else {
int pte_off_even = sizeof(pte_t) / 2;
int pte_off_odd = pte_off_even + sizeof(pte_t);
/* The pte entries are pre-shifted */
- i_lw(p, tmp, pte_off_even, ptep); /* get even pte */
- i_mtc0(p, tmp, C0_ENTRYLO0); /* load it */
- i_lw(p, ptep, pte_off_odd, ptep); /* get odd pte */
- i_mtc0(p, ptep, C0_ENTRYLO1); /* load it */
+ uasm_i_lw(p, tmp, pte_off_even, ptep); /* get even pte */
+ uasm_i_mtc0(p, tmp, C0_ENTRYLO0); /* load it */
+ uasm_i_lw(p, ptep, pte_off_odd, ptep); /* get odd pte */
+ uasm_i_mtc0(p, ptep, C0_ENTRYLO1); /* load it */
}
#else
- i_LW(p, tmp, 0, ptep); /* get even pte */
- i_LW(p, ptep, sizeof(pte_t), ptep); /* get odd pte */
+ UASM_i_LW(p, tmp, 0, ptep); /* get even pte */
+ UASM_i_LW(p, ptep, sizeof(pte_t), ptep); /* get odd pte */
if (r45k_bvahwbug())
build_tlb_probe_entry(p);
- i_SRL(p, tmp, tmp, 6); /* convert to entrylo0 */
+ UASM_i_SRL(p, tmp, tmp, 6); /* convert to entrylo0 */
if (r4k_250MHZhwbug())
- i_mtc0(p, 0, C0_ENTRYLO0);
- i_mtc0(p, tmp, C0_ENTRYLO0); /* load it */
- i_SRL(p, ptep, ptep, 6); /* convert to entrylo1 */
+ uasm_i_mtc0(p, 0, C0_ENTRYLO0);
+ uasm_i_mtc0(p, tmp, C0_ENTRYLO0); /* load it */
+ UASM_i_SRL(p, ptep, ptep, 6); /* convert to entrylo1 */
if (r45k_bvahwbug())
- i_mfc0(p, tmp, C0_INDEX);
+ uasm_i_mfc0(p, tmp, C0_INDEX);
if (r4k_250MHZhwbug())
- i_mtc0(p, 0, C0_ENTRYLO1);
- i_mtc0(p, ptep, C0_ENTRYLO1); /* load it */
+ uasm_i_mtc0(p, 0, C0_ENTRYLO1);
+ uasm_i_mtc0(p, ptep, C0_ENTRYLO1); /* load it */
#endif
}
static void __init build_r4000_tlb_refill_handler(void)
{
u32 *p = tlb_handler;
- struct label *l = labels;
- struct reloc *r = relocs;
+ struct uasm_label *l = labels;
+ struct uasm_reloc *r = relocs;
u32 *f;
unsigned int final_len;
* create the plain linear handler
*/
if (bcm1250_m3_war()) {
- i_MFC0(&p, K0, C0_BADVADDR);
- i_MFC0(&p, K1, C0_ENTRYHI);
- i_xor(&p, K0, K0, K1);
- i_SRL(&p, K0, K0, PAGE_SHIFT + 1);
- il_bnez(&p, &r, K0, label_leave);
- /* No need for i_nop */
+ UASM_i_MFC0(&p, K0, C0_BADVADDR);
+ UASM_i_MFC0(&p, K1, C0_ENTRYHI);
+ uasm_i_xor(&p, K0, K0, K1);
+ UASM_i_SRL(&p, K0, K0, PAGE_SHIFT + 1);
+ uasm_il_bnez(&p, &r, K0, label_leave);
+ /* No need for uasm_i_nop */
}
#ifdef CONFIG_64BIT
build_get_ptep(&p, K0, K1);
build_update_entries(&p, K0, K1);
build_tlb_write_entry(&p, &l, &r, tlb_random);
- l_leave(&l, p);
- i_eret(&p); /* return from trap */
+ uasm_l_leave(&l, p);
+ uasm_i_eret(&p); /* return from trap */
#ifdef CONFIG_64BIT
build_get_pgd_vmalloc64(&p, &l, &r, K0, K1);
#else
if (((p - tlb_handler) > 63)
|| (((p - tlb_handler) > 61)
- && insn_has_bdelay(relocs, tlb_handler + 29)))
+ && uasm_insn_has_bdelay(relocs, tlb_handler + 29)))
panic("TLB refill handler space exceeded");
#endif
#if defined(CONFIG_32BIT) || defined(CONFIG_CPU_LOONGSON2)
f = final_handler;
/* Simplest case, just copy the handler. */
- copy_handler(relocs, labels, tlb_handler, p, f);
+ uasm_copy_handler(relocs, labels, tlb_handler, p, f);
final_len = p - tlb_handler;
#else /* CONFIG_64BIT */
f = final_handler + 32;
if ((p - tlb_handler) <= 32) {
/* Just copy the handler. */
- copy_handler(relocs, labels, tlb_handler, p, f);
+ uasm_copy_handler(relocs, labels, tlb_handler, p, f);
final_len = p - tlb_handler;
} else {
u32 *split = tlb_handler + 30;
/*
* Find the split point.
*/
- if (insn_has_bdelay(relocs, split - 1))
+ if (uasm_insn_has_bdelay(relocs, split - 1))
split--;
/* Copy first part of the handler. */
- copy_handler(relocs, labels, tlb_handler, split, f);
+ uasm_copy_handler(relocs, labels, tlb_handler, split, f);
f += split - tlb_handler;
/* Insert branch. */
- l_split(&l, final_handler);
- il_b(&f, &r, label_split);
- if (insn_has_bdelay(relocs, split))
- i_nop(&f);
+ uasm_l_split(&l, final_handler);
+ uasm_il_b(&f, &r, label_split);
+ if (uasm_insn_has_bdelay(relocs, split))
+ uasm_i_nop(&f);
else {
- copy_handler(relocs, labels, split, split + 1, f);
- move_labels(labels, f, f + 1, -1);
+ uasm_copy_handler(relocs, labels, split, split + 1, f);
+ uasm_move_labels(labels, f, f + 1, -1);
f++;
split++;
}
/* Copy the rest of the handler. */
- copy_handler(relocs, labels, split, p, final_handler);
+ uasm_copy_handler(relocs, labels, split, p, final_handler);
final_len = (f - (final_handler + 32)) + (p - split);
}
#endif /* CONFIG_64BIT */
- resolve_relocs(relocs, labels);
- pr_info("Synthesized TLB refill handler (%u instructions).\n",
- final_len);
+ uasm_resolve_relocs(relocs, labels);
+ pr_debug("Wrote TLB refill handler (%u instructions).\n",
+ final_len);
memcpy((void *)ebase, final_handler, 0x100);
u32 handle_tlbm[FASTPATH_SIZE] __cacheline_aligned;
static void __init
-iPTE_LW(u32 **p, struct label **l, unsigned int pte, unsigned int ptr)
+iPTE_LW(u32 **p, struct uasm_label **l, unsigned int pte, unsigned int ptr)
{
#ifdef CONFIG_SMP
# ifdef CONFIG_64BIT_PHYS_ADDR
if (cpu_has_64bits)
- i_lld(p, pte, 0, ptr);
+ uasm_i_lld(p, pte, 0, ptr);
else
# endif
- i_LL(p, pte, 0, ptr);
+ UASM_i_LL(p, pte, 0, ptr);
#else
# ifdef CONFIG_64BIT_PHYS_ADDR
if (cpu_has_64bits)
- i_ld(p, pte, 0, ptr);
+ uasm_i_ld(p, pte, 0, ptr);
else
# endif
- i_LW(p, pte, 0, ptr);
+ UASM_i_LW(p, pte, 0, ptr);
#endif
}
static void __init
-iPTE_SW(u32 **p, struct reloc **r, unsigned int pte, unsigned int ptr,
+iPTE_SW(u32 **p, struct uasm_reloc **r, unsigned int pte, unsigned int ptr,
unsigned int mode)
{
#ifdef CONFIG_64BIT_PHYS_ADDR
unsigned int hwmode = mode & (_PAGE_VALID | _PAGE_DIRTY);
#endif
- i_ori(p, pte, pte, mode);
+ uasm_i_ori(p, pte, pte, mode);
#ifdef CONFIG_SMP
# ifdef CONFIG_64BIT_PHYS_ADDR
if (cpu_has_64bits)
- i_scd(p, pte, 0, ptr);
+ uasm_i_scd(p, pte, 0, ptr);
else
# endif
- i_SC(p, pte, 0, ptr);
+ UASM_i_SC(p, pte, 0, ptr);
if (r10000_llsc_war())
- il_beqzl(p, r, pte, label_smp_pgtable_change);
+ uasm_il_beqzl(p, r, pte, label_smp_pgtable_change);
else
- il_beqz(p, r, pte, label_smp_pgtable_change);
+ uasm_il_beqz(p, r, pte, label_smp_pgtable_change);
# ifdef CONFIG_64BIT_PHYS_ADDR
if (!cpu_has_64bits) {
- /* no i_nop needed */
- i_ll(p, pte, sizeof(pte_t) / 2, ptr);
- i_ori(p, pte, pte, hwmode);
- i_sc(p, pte, sizeof(pte_t) / 2, ptr);
- il_beqz(p, r, pte, label_smp_pgtable_change);
- /* no i_nop needed */
- i_lw(p, pte, 0, ptr);
+ /* no uasm_i_nop needed */
+ uasm_i_ll(p, pte, sizeof(pte_t) / 2, ptr);
+ uasm_i_ori(p, pte, pte, hwmode);
+ uasm_i_sc(p, pte, sizeof(pte_t) / 2, ptr);
+ uasm_il_beqz(p, r, pte, label_smp_pgtable_change);
+ /* no uasm_i_nop needed */
+ uasm_i_lw(p, pte, 0, ptr);
} else
- i_nop(p);
+ uasm_i_nop(p);
# else
- i_nop(p);
+ uasm_i_nop(p);
# endif
#else
# ifdef CONFIG_64BIT_PHYS_ADDR
if (cpu_has_64bits)
- i_sd(p, pte, 0, ptr);
+ uasm_i_sd(p, pte, 0, ptr);
else
# endif
- i_SW(p, pte, 0, ptr);
+ UASM_i_SW(p, pte, 0, ptr);
# ifdef CONFIG_64BIT_PHYS_ADDR
if (!cpu_has_64bits) {
- i_lw(p, pte, sizeof(pte_t) / 2, ptr);
- i_ori(p, pte, pte, hwmode);
- i_sw(p, pte, sizeof(pte_t) / 2, ptr);
- i_lw(p, pte, 0, ptr);
+ uasm_i_lw(p, pte, sizeof(pte_t) / 2, ptr);
+ uasm_i_ori(p, pte, pte, hwmode);
+ uasm_i_sw(p, pte, sizeof(pte_t) / 2, ptr);
+ uasm_i_lw(p, pte, 0, ptr);
}
# endif
#endif
* with it's original value.
*/
static void __init
-build_pte_present(u32 **p, struct label **l, struct reloc **r,
+build_pte_present(u32 **p, struct uasm_label **l, struct uasm_reloc **r,
unsigned int pte, unsigned int ptr, enum label_id lid)
{
- i_andi(p, pte, pte, _PAGE_PRESENT | _PAGE_READ);
- i_xori(p, pte, pte, _PAGE_PRESENT | _PAGE_READ);
- il_bnez(p, r, pte, lid);
+ uasm_i_andi(p, pte, pte, _PAGE_PRESENT | _PAGE_READ);
+ uasm_i_xori(p, pte, pte, _PAGE_PRESENT | _PAGE_READ);
+ uasm_il_bnez(p, r, pte, lid);
iPTE_LW(p, l, pte, ptr);
}
/* Make PTE valid, store result in PTR. */
static void __init
-build_make_valid(u32 **p, struct reloc **r, unsigned int pte,
+build_make_valid(u32 **p, struct uasm_reloc **r, unsigned int pte,
unsigned int ptr)
{
unsigned int mode = _PAGE_VALID | _PAGE_ACCESSED;
* restore PTE with value from PTR when done.
*/
static void __init
-build_pte_writable(u32 **p, struct label **l, struct reloc **r,
+build_pte_writable(u32 **p, struct uasm_label **l, struct uasm_reloc **r,
unsigned int pte, unsigned int ptr, enum label_id lid)
{
- i_andi(p, pte, pte, _PAGE_PRESENT | _PAGE_WRITE);
- i_xori(p, pte, pte, _PAGE_PRESENT | _PAGE_WRITE);
- il_bnez(p, r, pte, lid);
+ uasm_i_andi(p, pte, pte, _PAGE_PRESENT | _PAGE_WRITE);
+ uasm_i_xori(p, pte, pte, _PAGE_PRESENT | _PAGE_WRITE);
+ uasm_il_bnez(p, r, pte, lid);
iPTE_LW(p, l, pte, ptr);
}
* at PTR.
*/
static void __init
-build_make_write(u32 **p, struct reloc **r, unsigned int pte,
+build_make_write(u32 **p, struct uasm_reloc **r, unsigned int pte,
unsigned int ptr)
{
unsigned int mode = (_PAGE_ACCESSED | _PAGE_MODIFIED | _PAGE_VALID
* restore PTE with value from PTR when done.
*/
static void __init
-build_pte_modifiable(u32 **p, struct label **l, struct reloc **r,
+build_pte_modifiable(u32 **p, struct uasm_label **l, struct uasm_reloc **r,
unsigned int pte, unsigned int ptr, enum label_id lid)
{
- i_andi(p, pte, pte, _PAGE_WRITE);
- il_beqz(p, r, pte, lid);
+ uasm_i_andi(p, pte, pte, _PAGE_WRITE);
+ uasm_il_beqz(p, r, pte, lid);
iPTE_LW(p, l, pte, ptr);
}
static void __init
build_r3000_pte_reload_tlbwi(u32 **p, unsigned int pte, unsigned int tmp)
{
- i_mtc0(p, pte, C0_ENTRYLO0); /* cp0 delay */
- i_mfc0(p, tmp, C0_EPC); /* cp0 delay */
- i_tlbwi(p);
- i_jr(p, tmp);
- i_rfe(p); /* branch delay */
+ uasm_i_mtc0(p, pte, C0_ENTRYLO0); /* cp0 delay */
+ uasm_i_mfc0(p, tmp, C0_EPC); /* cp0 delay */
+ uasm_i_tlbwi(p);
+ uasm_i_jr(p, tmp);
+ uasm_i_rfe(p); /* branch delay */
}
/*
* kseg2 access, i.e. without refill. Then it returns.
*/
static void __init
-build_r3000_tlb_reload_write(u32 **p, struct label **l, struct reloc **r,
- unsigned int pte, unsigned int tmp)
-{
- i_mfc0(p, tmp, C0_INDEX);
- i_mtc0(p, pte, C0_ENTRYLO0); /* cp0 delay */
- il_bltz(p, r, tmp, label_r3000_write_probe_fail); /* cp0 delay */
- i_mfc0(p, tmp, C0_EPC); /* branch delay */
- i_tlbwi(p); /* cp0 delay */
- i_jr(p, tmp);
- i_rfe(p); /* branch delay */
- l_r3000_write_probe_fail(l, *p);
- i_tlbwr(p); /* cp0 delay */
- i_jr(p, tmp);
- i_rfe(p); /* branch delay */
+build_r3000_tlb_reload_write(u32 **p, struct uasm_label **l,
+ struct uasm_reloc **r, unsigned int pte,
+ unsigned int tmp)
+{
+ uasm_i_mfc0(p, tmp, C0_INDEX);
+ uasm_i_mtc0(p, pte, C0_ENTRYLO0); /* cp0 delay */
+ uasm_il_bltz(p, r, tmp, label_r3000_write_probe_fail); /* cp0 delay */
+ uasm_i_mfc0(p, tmp, C0_EPC); /* branch delay */
+ uasm_i_tlbwi(p); /* cp0 delay */
+ uasm_i_jr(p, tmp);
+ uasm_i_rfe(p); /* branch delay */
+ uasm_l_r3000_write_probe_fail(l, *p);
+ uasm_i_tlbwr(p); /* cp0 delay */
+ uasm_i_jr(p, tmp);
+ uasm_i_rfe(p); /* branch delay */
}
static void __init
{
long pgdc = (long)pgd_current;
- i_mfc0(p, pte, C0_BADVADDR);
- i_lui(p, ptr, rel_hi(pgdc)); /* cp0 delay */
- i_lw(p, ptr, rel_lo(pgdc), ptr);
- i_srl(p, pte, pte, 22); /* load delay */
- i_sll(p, pte, pte, 2);
- i_addu(p, ptr, ptr, pte);
- i_mfc0(p, pte, C0_CONTEXT);
- i_lw(p, ptr, 0, ptr); /* cp0 delay */
- i_andi(p, pte, pte, 0xffc); /* load delay */
- i_addu(p, ptr, ptr, pte);
- i_lw(p, pte, 0, ptr);
- i_tlbp(p); /* load delay */
+ uasm_i_mfc0(p, pte, C0_BADVADDR);
+ uasm_i_lui(p, ptr, uasm_rel_hi(pgdc)); /* cp0 delay */
+ uasm_i_lw(p, ptr, uasm_rel_lo(pgdc), ptr);
+ uasm_i_srl(p, pte, pte, 22); /* load delay */
+ uasm_i_sll(p, pte, pte, 2);
+ uasm_i_addu(p, ptr, ptr, pte);
+ uasm_i_mfc0(p, pte, C0_CONTEXT);
+ uasm_i_lw(p, ptr, 0, ptr); /* cp0 delay */
+ uasm_i_andi(p, pte, pte, 0xffc); /* load delay */
+ uasm_i_addu(p, ptr, ptr, pte);
+ uasm_i_lw(p, pte, 0, ptr);
+ uasm_i_tlbp(p); /* load delay */
}
static void __init build_r3000_tlb_load_handler(void)
{
u32 *p = handle_tlbl;
- struct label *l = labels;
- struct reloc *r = relocs;
+ struct uasm_label *l = labels;
+ struct uasm_reloc *r = relocs;
memset(handle_tlbl, 0, sizeof(handle_tlbl));
memset(labels, 0, sizeof(labels));
build_r3000_tlbchange_handler_head(&p, K0, K1);
build_pte_present(&p, &l, &r, K0, K1, label_nopage_tlbl);
- i_nop(&p); /* load delay */
+ uasm_i_nop(&p); /* load delay */
build_make_valid(&p, &r, K0, K1);
build_r3000_tlb_reload_write(&p, &l, &r, K0, K1);
- l_nopage_tlbl(&l, p);
- i_j(&p, (unsigned long)tlb_do_page_fault_0 & 0x0fffffff);
- i_nop(&p);
+ uasm_l_nopage_tlbl(&l, p);
+ uasm_i_j(&p, (unsigned long)tlb_do_page_fault_0 & 0x0fffffff);
+ uasm_i_nop(&p);
if ((p - handle_tlbl) > FASTPATH_SIZE)
panic("TLB load handler fastpath space exceeded");
- resolve_relocs(relocs, labels);
- pr_info("Synthesized TLB load handler fastpath (%u instructions).\n",
- (unsigned int)(p - handle_tlbl));
+ uasm_resolve_relocs(relocs, labels);
+ pr_debug("Wrote TLB load handler fastpath (%u instructions).\n",
+ (unsigned int)(p - handle_tlbl));
dump_handler(handle_tlbl, ARRAY_SIZE(handle_tlbl));
}
static void __init build_r3000_tlb_store_handler(void)
{
u32 *p = handle_tlbs;
- struct label *l = labels;
- struct reloc *r = relocs;
+ struct uasm_label *l = labels;
+ struct uasm_reloc *r = relocs;
memset(handle_tlbs, 0, sizeof(handle_tlbs));
memset(labels, 0, sizeof(labels));
build_r3000_tlbchange_handler_head(&p, K0, K1);
build_pte_writable(&p, &l, &r, K0, K1, label_nopage_tlbs);
- i_nop(&p); /* load delay */
+ uasm_i_nop(&p); /* load delay */
build_make_write(&p, &r, K0, K1);
build_r3000_tlb_reload_write(&p, &l, &r, K0, K1);
- l_nopage_tlbs(&l, p);
- i_j(&p, (unsigned long)tlb_do_page_fault_1 & 0x0fffffff);
- i_nop(&p);
+ uasm_l_nopage_tlbs(&l, p);
+ uasm_i_j(&p, (unsigned long)tlb_do_page_fault_1 & 0x0fffffff);
+ uasm_i_nop(&p);
if ((p - handle_tlbs) > FASTPATH_SIZE)
panic("TLB store handler fastpath space exceeded");
- resolve_relocs(relocs, labels);
- pr_info("Synthesized TLB store handler fastpath (%u instructions).\n",
- (unsigned int)(p - handle_tlbs));
+ uasm_resolve_relocs(relocs, labels);
+ pr_debug("Wrote TLB store handler fastpath (%u instructions).\n",
+ (unsigned int)(p - handle_tlbs));
dump_handler(handle_tlbs, ARRAY_SIZE(handle_tlbs));
}
static void __init build_r3000_tlb_modify_handler(void)
{
u32 *p = handle_tlbm;
- struct label *l = labels;
- struct reloc *r = relocs;
+ struct uasm_label *l = labels;
+ struct uasm_reloc *r = relocs;
memset(handle_tlbm, 0, sizeof(handle_tlbm));
memset(labels, 0, sizeof(labels));
build_r3000_tlbchange_handler_head(&p, K0, K1);
build_pte_modifiable(&p, &l, &r, K0, K1, label_nopage_tlbm);
- i_nop(&p); /* load delay */
+ uasm_i_nop(&p); /* load delay */
build_make_write(&p, &r, K0, K1);
build_r3000_pte_reload_tlbwi(&p, K0, K1);
- l_nopage_tlbm(&l, p);
- i_j(&p, (unsigned long)tlb_do_page_fault_1 & 0x0fffffff);
- i_nop(&p);
+ uasm_l_nopage_tlbm(&l, p);
+ uasm_i_j(&p, (unsigned long)tlb_do_page_fault_1 & 0x0fffffff);
+ uasm_i_nop(&p);
if ((p - handle_tlbm) > FASTPATH_SIZE)
panic("TLB modify handler fastpath space exceeded");
- resolve_relocs(relocs, labels);
- pr_info("Synthesized TLB modify handler fastpath (%u instructions).\n",
- (unsigned int)(p - handle_tlbm));
+ uasm_resolve_relocs(relocs, labels);
+ pr_debug("Wrote TLB modify handler fastpath (%u instructions).\n",
+ (unsigned int)(p - handle_tlbm));
dump_handler(handle_tlbm, ARRAY_SIZE(handle_tlbm));
}
* R4000 style TLB load/store/modify handlers.
*/
static void __init
-build_r4000_tlbchange_handler_head(u32 **p, struct label **l,
- struct reloc **r, unsigned int pte,
+build_r4000_tlbchange_handler_head(u32 **p, struct uasm_label **l,
+ struct uasm_reloc **r, unsigned int pte,
unsigned int ptr)
{
#ifdef CONFIG_64BIT
build_get_pgde32(p, pte, ptr); /* get pgd in ptr */
#endif
- i_MFC0(p, pte, C0_BADVADDR);
- i_LW(p, ptr, 0, ptr);
- i_SRL(p, pte, pte, PAGE_SHIFT + PTE_ORDER - PTE_T_LOG2);
- i_andi(p, pte, pte, (PTRS_PER_PTE - 1) << PTE_T_LOG2);
- i_ADDU(p, ptr, ptr, pte);
+ UASM_i_MFC0(p, pte, C0_BADVADDR);
+ UASM_i_LW(p, ptr, 0, ptr);
+ UASM_i_SRL(p, pte, pte, PAGE_SHIFT + PTE_ORDER - PTE_T_LOG2);
+ uasm_i_andi(p, pte, pte, (PTRS_PER_PTE - 1) << PTE_T_LOG2);
+ UASM_i_ADDU(p, ptr, ptr, pte);
#ifdef CONFIG_SMP
- l_smp_pgtable_change(l, *p);
-# endif
+ uasm_l_smp_pgtable_change(l, *p);
+#endif
iPTE_LW(p, l, pte, ptr); /* get even pte */
if (!m4kc_tlbp_war())
build_tlb_probe_entry(p);
}
static void __init
-build_r4000_tlbchange_handler_tail(u32 **p, struct label **l,
- struct reloc **r, unsigned int tmp,
+build_r4000_tlbchange_handler_tail(u32 **p, struct uasm_label **l,
+ struct uasm_reloc **r, unsigned int tmp,
unsigned int ptr)
{
- i_ori(p, ptr, ptr, sizeof(pte_t));
- i_xori(p, ptr, ptr, sizeof(pte_t));
+ uasm_i_ori(p, ptr, ptr, sizeof(pte_t));
+ uasm_i_xori(p, ptr, ptr, sizeof(pte_t));
build_update_entries(p, tmp, ptr);
build_tlb_write_entry(p, l, r, tlb_indexed);
- l_leave(l, *p);
- i_eret(p); /* return from trap */
+ uasm_l_leave(l, *p);
+ uasm_i_eret(p); /* return from trap */
#ifdef CONFIG_64BIT
build_get_pgd_vmalloc64(p, l, r, tmp, ptr);
static void __init build_r4000_tlb_load_handler(void)
{
u32 *p = handle_tlbl;
- struct label *l = labels;
- struct reloc *r = relocs;
+ struct uasm_label *l = labels;
+ struct uasm_reloc *r = relocs;
memset(handle_tlbl, 0, sizeof(handle_tlbl));
memset(labels, 0, sizeof(labels));
memset(relocs, 0, sizeof(relocs));
if (bcm1250_m3_war()) {
- i_MFC0(&p, K0, C0_BADVADDR);
- i_MFC0(&p, K1, C0_ENTRYHI);
- i_xor(&p, K0, K0, K1);
- i_SRL(&p, K0, K0, PAGE_SHIFT + 1);
- il_bnez(&p, &r, K0, label_leave);
- /* No need for i_nop */
+ UASM_i_MFC0(&p, K0, C0_BADVADDR);
+ UASM_i_MFC0(&p, K1, C0_ENTRYHI);
+ uasm_i_xor(&p, K0, K0, K1);
+ UASM_i_SRL(&p, K0, K0, PAGE_SHIFT + 1);
+ uasm_il_bnez(&p, &r, K0, label_leave);
+ /* No need for uasm_i_nop */
}
build_r4000_tlbchange_handler_head(&p, &l, &r, K0, K1);
build_make_valid(&p, &r, K0, K1);
build_r4000_tlbchange_handler_tail(&p, &l, &r, K0, K1);
- l_nopage_tlbl(&l, p);
- i_j(&p, (unsigned long)tlb_do_page_fault_0 & 0x0fffffff);
- i_nop(&p);
+ uasm_l_nopage_tlbl(&l, p);
+ uasm_i_j(&p, (unsigned long)tlb_do_page_fault_0 & 0x0fffffff);
+ uasm_i_nop(&p);
if ((p - handle_tlbl) > FASTPATH_SIZE)
panic("TLB load handler fastpath space exceeded");
- resolve_relocs(relocs, labels);
- pr_info("Synthesized TLB load handler fastpath (%u instructions).\n",
- (unsigned int)(p - handle_tlbl));
+ uasm_resolve_relocs(relocs, labels);
+ pr_debug("Wrote TLB load handler fastpath (%u instructions).\n",
+ (unsigned int)(p - handle_tlbl));
dump_handler(handle_tlbl, ARRAY_SIZE(handle_tlbl));
}
static void __init build_r4000_tlb_store_handler(void)
{
u32 *p = handle_tlbs;
- struct label *l = labels;
- struct reloc *r = relocs;
+ struct uasm_label *l = labels;
+ struct uasm_reloc *r = relocs;
memset(handle_tlbs, 0, sizeof(handle_tlbs));
memset(labels, 0, sizeof(labels));
build_make_write(&p, &r, K0, K1);
build_r4000_tlbchange_handler_tail(&p, &l, &r, K0, K1);
- l_nopage_tlbs(&l, p);
- i_j(&p, (unsigned long)tlb_do_page_fault_1 & 0x0fffffff);
- i_nop(&p);
+ uasm_l_nopage_tlbs(&l, p);
+ uasm_i_j(&p, (unsigned long)tlb_do_page_fault_1 & 0x0fffffff);
+ uasm_i_nop(&p);
if ((p - handle_tlbs) > FASTPATH_SIZE)
panic("TLB store handler fastpath space exceeded");
- resolve_relocs(relocs, labels);
- pr_info("Synthesized TLB store handler fastpath (%u instructions).\n",
- (unsigned int)(p - handle_tlbs));
+ uasm_resolve_relocs(relocs, labels);
+ pr_debug("Wrote TLB store handler fastpath (%u instructions).\n",
+ (unsigned int)(p - handle_tlbs));
dump_handler(handle_tlbs, ARRAY_SIZE(handle_tlbs));
}
static void __init build_r4000_tlb_modify_handler(void)
{
u32 *p = handle_tlbm;
- struct label *l = labels;
- struct reloc *r = relocs;
+ struct uasm_label *l = labels;
+ struct uasm_reloc *r = relocs;
memset(handle_tlbm, 0, sizeof(handle_tlbm));
memset(labels, 0, sizeof(labels));
build_make_write(&p, &r, K0, K1);
build_r4000_tlbchange_handler_tail(&p, &l, &r, K0, K1);
- l_nopage_tlbm(&l, p);
- i_j(&p, (unsigned long)tlb_do_page_fault_1 & 0x0fffffff);
- i_nop(&p);
+ uasm_l_nopage_tlbm(&l, p);
+ uasm_i_j(&p, (unsigned long)tlb_do_page_fault_1 & 0x0fffffff);
+ uasm_i_nop(&p);
if ((p - handle_tlbm) > FASTPATH_SIZE)
panic("TLB modify handler fastpath space exceeded");
- resolve_relocs(relocs, labels);
- pr_info("Synthesized TLB modify handler fastpath (%u instructions).\n",
- (unsigned int)(p - handle_tlbm));
+ uasm_resolve_relocs(relocs, labels);
+ pr_debug("Wrote TLB modify handler fastpath (%u instructions).\n",
+ (unsigned int)(p - handle_tlbm));
dump_handler(handle_tlbm, ARRAY_SIZE(handle_tlbm));
}
--- /dev/null
+/*
+ * This file is subject to the terms and conditions of the GNU General Public
+ * License. See the file "COPYING" in the main directory of this archive
+ * for more details.
+ *
+ * A small micro-assembler. It is intentionally kept simple, does only
+ * support a subset of instructions, and does not try to hide pipeline
+ * effects like branch delay slots.
+ *
+ * Copyright (C) 2004, 2005, 2006, 2008 Thiemo Seufer
+ * Copyright (C) 2005, 2007 Maciej W. Rozycki
+ * Copyright (C) 2006 Ralf Baechle (ralf@linux-mips.org)
+ */
+
+#include <linux/kernel.h>
+#include <linux/types.h>
+#include <linux/init.h>
+
+#include <asm/inst.h>
+#include <asm/elf.h>
+#include <asm/bugs.h>
+
+#include "uasm.h"
+
+enum fields {
+ RS = 0x001,
+ RT = 0x002,
+ RD = 0x004,
+ RE = 0x008,
+ SIMM = 0x010,
+ UIMM = 0x020,
+ BIMM = 0x040,
+ JIMM = 0x080,
+ FUNC = 0x100,
+ SET = 0x200
+};
+
+#define OP_MASK 0x3f
+#define OP_SH 26
+#define RS_MASK 0x1f
+#define RS_SH 21
+#define RT_MASK 0x1f
+#define RT_SH 16
+#define RD_MASK 0x1f
+#define RD_SH 11
+#define RE_MASK 0x1f
+#define RE_SH 6
+#define IMM_MASK 0xffff
+#define IMM_SH 0
+#define JIMM_MASK 0x3ffffff
+#define JIMM_SH 0
+#define FUNC_MASK 0x3f
+#define FUNC_SH 0
+#define SET_MASK 0x7
+#define SET_SH 0
+
+enum opcode {
+ insn_invalid,
+ insn_addu, insn_addiu, insn_and, insn_andi, insn_beq,
+ insn_beql, insn_bgez, insn_bgezl, insn_bltz, insn_bltzl,
+ insn_bne, insn_daddu, insn_daddiu, insn_dmfc0, insn_dmtc0,
+ insn_dsll, insn_dsll32, insn_dsra, insn_dsrl, insn_dsrl32,
+ insn_dsubu, insn_eret, insn_j, insn_jal, insn_jr, insn_ld,
+ insn_ll, insn_lld, insn_lui, insn_lw, insn_mfc0, insn_mtc0,
+ insn_ori, insn_rfe, insn_sc, insn_scd, insn_sd, insn_sll,
+ insn_sra, insn_srl, insn_subu, insn_sw, insn_tlbp, insn_tlbwi,
+ insn_tlbwr, insn_xor, insn_xori
+};
+
+struct insn {
+ enum opcode opcode;
+ u32 match;
+ enum fields fields;
+};
+
+/* This macro sets the non-variable bits of an instruction. */
+#define M(a, b, c, d, e, f) \
+ ((a) << OP_SH \
+ | (b) << RS_SH \
+ | (c) << RT_SH \
+ | (d) << RD_SH \
+ | (e) << RE_SH \
+ | (f) << FUNC_SH)
+
+static struct insn insn_table[] __initdata = {
+ { insn_addiu, M(addiu_op, 0, 0, 0, 0, 0), RS | RT | SIMM },
+ { insn_addu, M(spec_op, 0, 0, 0, 0, addu_op), RS | RT | RD },
+ { insn_and, M(spec_op, 0, 0, 0, 0, and_op), RS | RT | RD },
+ { insn_andi, M(andi_op, 0, 0, 0, 0, 0), RS | RT | UIMM },
+ { insn_beq, M(beq_op, 0, 0, 0, 0, 0), RS | RT | BIMM },
+ { insn_beql, M(beql_op, 0, 0, 0, 0, 0), RS | RT | BIMM },
+ { insn_bgez, M(bcond_op, 0, bgez_op, 0, 0, 0), RS | BIMM },
+ { insn_bgezl, M(bcond_op, 0, bgezl_op, 0, 0, 0), RS | BIMM },
+ { insn_bltz, M(bcond_op, 0, bltz_op, 0, 0, 0), RS | BIMM },
+ { insn_bltzl, M(bcond_op, 0, bltzl_op, 0, 0, 0), RS | BIMM },
+ { insn_bne, M(bne_op, 0, 0, 0, 0, 0), RS | RT | BIMM },
+ { insn_daddiu, M(daddiu_op, 0, 0, 0, 0, 0), RS | RT | SIMM },
+ { insn_daddu, M(spec_op, 0, 0, 0, 0, daddu_op), RS | RT | RD },
+ { insn_dmfc0, M(cop0_op, dmfc_op, 0, 0, 0, 0), RT | RD | SET},
+ { insn_dmtc0, M(cop0_op, dmtc_op, 0, 0, 0, 0), RT | RD | SET},
+ { insn_dsll, M(spec_op, 0, 0, 0, 0, dsll_op), RT | RD | RE },
+ { insn_dsll32, M(spec_op, 0, 0, 0, 0, dsll32_op), RT | RD | RE },
+ { insn_dsra, M(spec_op, 0, 0, 0, 0, dsra_op), RT | RD | RE },
+ { insn_dsrl, M(spec_op, 0, 0, 0, 0, dsrl_op), RT | RD | RE },
+ { insn_dsrl32, M(spec_op, 0, 0, 0, 0, dsrl32_op), RT | RD | RE },
+ { insn_dsubu, M(spec_op, 0, 0, 0, 0, dsubu_op), RS | RT | RD },
+ { insn_eret, M(cop0_op, cop_op, 0, 0, 0, eret_op), 0 },
+ { insn_j, M(j_op, 0, 0, 0, 0, 0), JIMM },
+ { insn_jal, M(jal_op, 0, 0, 0, 0, 0), JIMM },
+ { insn_jr, M(spec_op, 0, 0, 0, 0, jr_op), RS },
+ { insn_ld, M(ld_op, 0, 0, 0, 0, 0), RS | RT | SIMM },
+ { insn_ll, M(ll_op, 0, 0, 0, 0, 0), RS | RT | SIMM },
+ { insn_lld, M(lld_op, 0, 0, 0, 0, 0), RS | RT | SIMM },
+ { insn_lui, M(lui_op, 0, 0, 0, 0, 0), RT | SIMM },
+ { insn_lw, M(lw_op, 0, 0, 0, 0, 0), RS | RT | SIMM },
+ { insn_mfc0, M(cop0_op, mfc_op, 0, 0, 0, 0), RT | RD | SET},
+ { insn_mtc0, M(cop0_op, mtc_op, 0, 0, 0, 0), RT | RD | SET},
+ { insn_ori, M(ori_op, 0, 0, 0, 0, 0), RS | RT | UIMM },
+ { insn_rfe, M(cop0_op, cop_op, 0, 0, 0, rfe_op), 0 },
+ { insn_sc, M(sc_op, 0, 0, 0, 0, 0), RS | RT | SIMM },
+ { insn_scd, M(scd_op, 0, 0, 0, 0, 0), RS | RT | SIMM },
+ { insn_sd, M(sd_op, 0, 0, 0, 0, 0), RS | RT | SIMM },
+ { insn_sll, M(spec_op, 0, 0, 0, 0, sll_op), RT | RD | RE },
+ { insn_sra, M(spec_op, 0, 0, 0, 0, sra_op), RT | RD | RE },
+ { insn_srl, M(spec_op, 0, 0, 0, 0, srl_op), RT | RD | RE },
+ { insn_subu, M(spec_op, 0, 0, 0, 0, subu_op), RS | RT | RD },
+ { insn_sw, M(sw_op, 0, 0, 0, 0, 0), RS | RT | SIMM },
+ { insn_tlbp, M(cop0_op, cop_op, 0, 0, 0, tlbp_op), 0 },
+ { insn_tlbwi, M(cop0_op, cop_op, 0, 0, 0, tlbwi_op), 0 },
+ { insn_tlbwr, M(cop0_op, cop_op, 0, 0, 0, tlbwr_op), 0 },
+ { insn_xor, M(spec_op, 0, 0, 0, 0, xor_op), RS | RT | RD },
+ { insn_xori, M(xori_op, 0, 0, 0, 0, 0), RS | RT | UIMM },
+ { insn_invalid, 0, 0 }
+};
+
+#undef M
+
+static inline __init u32 build_rs(u32 arg)
+{
+ if (arg & ~RS_MASK)
+ printk(KERN_WARNING "Micro-assembler field overflow\n");
+
+ return (arg & RS_MASK) << RS_SH;
+}
+
+static inline __init u32 build_rt(u32 arg)
+{
+ if (arg & ~RT_MASK)
+ printk(KERN_WARNING "Micro-assembler field overflow\n");
+
+ return (arg & RT_MASK) << RT_SH;
+}
+
+static inline __init u32 build_rd(u32 arg)
+{
+ if (arg & ~RD_MASK)
+ printk(KERN_WARNING "Micro-assembler field overflow\n");
+
+ return (arg & RD_MASK) << RD_SH;
+}
+
+static inline __init u32 build_re(u32 arg)
+{
+ if (arg & ~RE_MASK)
+ printk(KERN_WARNING "Micro-assembler field overflow\n");
+
+ return (arg & RE_MASK) << RE_SH;
+}
+
+static inline __init u32 build_simm(s32 arg)
+{
+ if (arg > 0x7fff || arg < -0x8000)
+ printk(KERN_WARNING "Micro-assembler field overflow\n");
+
+ return arg & 0xffff;
+}
+
+static inline __init u32 build_uimm(u32 arg)
+{
+ if (arg & ~IMM_MASK)
+ printk(KERN_WARNING "Micro-assembler field overflow\n");
+
+ return arg & IMM_MASK;
+}
+
+static inline __init u32 build_bimm(s32 arg)
+{
+ if (arg > 0x1ffff || arg < -0x20000)
+ printk(KERN_WARNING "Micro-assembler field overflow\n");
+
+ if (arg & 0x3)
+ printk(KERN_WARNING "Invalid micro-assembler branch target\n");
+
+ return ((arg < 0) ? (1 << 15) : 0) | ((arg >> 2) & 0x7fff);
+}
+
+static inline __init u32 build_jimm(u32 arg)
+{
+ if (arg & ~((JIMM_MASK) << 2))
+ printk(KERN_WARNING "Micro-assembler field overflow\n");
+
+ return (arg >> 2) & JIMM_MASK;
+}
+
+static inline __init u32 build_func(u32 arg)
+{
+ if (arg & ~FUNC_MASK)
+ printk(KERN_WARNING "Micro-assembler field overflow\n");
+
+ return arg & FUNC_MASK;
+}
+
+static inline __init u32 build_set(u32 arg)
+{
+ if (arg & ~SET_MASK)
+ printk(KERN_WARNING "Micro-assembler field overflow\n");
+
+ return arg & SET_MASK;
+}
+
+/*
+ * The order of opcode arguments is implicitly left to right,
+ * starting with RS and ending with FUNC or IMM.
+ */
+static void __init build_insn(u32 **buf, enum opcode opc, ...)
+{
+ struct insn *ip = NULL;
+ unsigned int i;
+ va_list ap;
+ u32 op;
+
+ for (i = 0; insn_table[i].opcode != insn_invalid; i++)
+ if (insn_table[i].opcode == opc) {
+ ip = &insn_table[i];
+ break;
+ }
+
+ if (!ip || (opc == insn_daddiu && r4k_daddiu_bug()))
+ panic("Unsupported Micro-assembler instruction %d", opc);
+
+ op = ip->match;
+ va_start(ap, opc);
+ if (ip->fields & RS)
+ op |= build_rs(va_arg(ap, u32));
+ if (ip->fields & RT)
+ op |= build_rt(va_arg(ap, u32));
+ if (ip->fields & RD)
+ op |= build_rd(va_arg(ap, u32));
+ if (ip->fields & RE)
+ op |= build_re(va_arg(ap, u32));
+ if (ip->fields & SIMM)
+ op |= build_simm(va_arg(ap, s32));
+ if (ip->fields & UIMM)
+ op |= build_uimm(va_arg(ap, u32));
+ if (ip->fields & BIMM)
+ op |= build_bimm(va_arg(ap, s32));
+ if (ip->fields & JIMM)
+ op |= build_jimm(va_arg(ap, u32));
+ if (ip->fields & FUNC)
+ op |= build_func(va_arg(ap, u32));
+ if (ip->fields & SET)
+ op |= build_set(va_arg(ap, u32));
+ va_end(ap);
+
+ **buf = op;
+ (*buf)++;
+}
+
+#define I_u1u2u3(op) \
+Ip_u1u2u3(op) \
+{ \
+ build_insn(buf, insn##op, a, b, c); \
+}
+
+#define I_u2u1u3(op) \
+Ip_u2u1u3(op) \
+{ \
+ build_insn(buf, insn##op, b, a, c); \
+}
+
+#define I_u3u1u2(op) \
+Ip_u3u1u2(op) \
+{ \
+ build_insn(buf, insn##op, b, c, a); \
+}
+
+#define I_u1u2s3(op) \
+Ip_u1u2s3(op) \
+{ \
+ build_insn(buf, insn##op, a, b, c); \
+}
+
+#define I_u2s3u1(op) \
+Ip_u2s3u1(op) \
+{ \
+ build_insn(buf, insn##op, c, a, b); \
+}
+
+#define I_u2u1s3(op) \
+Ip_u2u1s3(op) \
+{ \
+ build_insn(buf, insn##op, b, a, c); \
+}
+
+#define I_u1u2(op) \
+Ip_u1u2(op) \
+{ \
+ build_insn(buf, insn##op, a, b); \
+}
+
+#define I_u1s2(op) \
+Ip_u1s2(op) \
+{ \
+ build_insn(buf, insn##op, a, b); \
+}
+
+#define I_u1(op) \
+Ip_u1(op) \
+{ \
+ build_insn(buf, insn##op, a); \
+}
+
+#define I_0(op) \
+Ip_0(op) \
+{ \
+ build_insn(buf, insn##op); \
+}
+
+I_u2u1s3(_addiu)
+I_u3u1u2(_addu)
+I_u2u1u3(_andi)
+I_u3u1u2(_and)
+I_u1u2s3(_beq)
+I_u1u2s3(_beql)
+I_u1s2(_bgez)
+I_u1s2(_bgezl)
+I_u1s2(_bltz)
+I_u1s2(_bltzl)
+I_u1u2s3(_bne)
+I_u1u2u3(_dmfc0)
+I_u1u2u3(_dmtc0)
+I_u2u1s3(_daddiu)
+I_u3u1u2(_daddu)
+I_u2u1u3(_dsll)
+I_u2u1u3(_dsll32)
+I_u2u1u3(_dsra)
+I_u2u1u3(_dsrl)
+I_u2u1u3(_dsrl32)
+I_u3u1u2(_dsubu)
+I_0(_eret)
+I_u1(_j)
+I_u1(_jal)
+I_u1(_jr)
+I_u2s3u1(_ld)
+I_u2s3u1(_ll)
+I_u2s3u1(_lld)
+I_u1s2(_lui)
+I_u2s3u1(_lw)
+I_u1u2u3(_mfc0)
+I_u1u2u3(_mtc0)
+I_u2u1u3(_ori)
+I_0(_rfe)
+I_u2s3u1(_sc)
+I_u2s3u1(_scd)
+I_u2s3u1(_sd)
+I_u2u1u3(_sll)
+I_u2u1u3(_sra)
+I_u2u1u3(_srl)
+I_u3u1u2(_subu)
+I_u2s3u1(_sw)
+I_0(_tlbp)
+I_0(_tlbwi)
+I_0(_tlbwr)
+I_u3u1u2(_xor)
+I_u2u1u3(_xori)
+
+/* Handle labels. */
+void __init uasm_build_label(struct uasm_label **lab, u32 *addr, int lid)
+{
+ (*lab)->addr = addr;
+ (*lab)->lab = lid;
+ (*lab)++;
+}
+
+int __init uasm_in_compat_space_p(long addr)
+{
+ /* Is this address in 32bit compat space? */
+#ifdef CONFIG_64BIT
+ return (((addr) & 0xffffffff00000000L) == 0xffffffff00000000L);
+#else
+ return 1;
+#endif
+}
+
+int __init uasm_rel_highest(long val)
+{
+#ifdef CONFIG_64BIT
+ return ((((val + 0x800080008000L) >> 48) & 0xffff) ^ 0x8000) - 0x8000;
+#else
+ return 0;
+#endif
+}
+
+int __init uasm_rel_higher(long val)
+{
+#ifdef CONFIG_64BIT
+ return ((((val + 0x80008000L) >> 32) & 0xffff) ^ 0x8000) - 0x8000;
+#else
+ return 0;
+#endif
+}
+
+int __init uasm_rel_hi(long val)
+{
+ return ((((val + 0x8000L) >> 16) & 0xffff) ^ 0x8000) - 0x8000;
+}
+
+int __init uasm_rel_lo(long val)
+{
+ return ((val & 0xffff) ^ 0x8000) - 0x8000;
+}
+
+void __init UASM_i_LA_mostly(u32 **buf, unsigned int rs, long addr)
+{
+ if (!uasm_in_compat_space_p(addr)) {
+ uasm_i_lui(buf, rs, uasm_rel_highest(addr));
+ if (uasm_rel_higher(addr))
+ uasm_i_daddiu(buf, rs, rs, uasm_rel_higher(addr));
+ if (uasm_rel_hi(addr)) {
+ uasm_i_dsll(buf, rs, rs, 16);
+ uasm_i_daddiu(buf, rs, rs, uasm_rel_hi(addr));
+ uasm_i_dsll(buf, rs, rs, 16);
+ } else
+ uasm_i_dsll32(buf, rs, rs, 0);
+ } else
+ uasm_i_lui(buf, rs, uasm_rel_hi(addr));
+}
+
+void __init UASM_i_LA(u32 **buf, unsigned int rs, long addr)
+{
+ UASM_i_LA_mostly(buf, rs, addr);
+ if (uasm_rel_lo(addr)) {
+ if (!uasm_in_compat_space_p(addr))
+ uasm_i_daddiu(buf, rs, rs, uasm_rel_lo(addr));
+ else
+ uasm_i_addiu(buf, rs, rs, uasm_rel_lo(addr));
+ }
+}
+
+/* Handle relocations. */
+void __init
+uasm_r_mips_pc16(struct uasm_reloc **rel, u32 *addr, int lid)
+{
+ (*rel)->addr = addr;
+ (*rel)->type = R_MIPS_PC16;
+ (*rel)->lab = lid;
+ (*rel)++;
+}
+
+static inline void __init
+__resolve_relocs(struct uasm_reloc *rel, struct uasm_label *lab)
+{
+ long laddr = (long)lab->addr;
+ long raddr = (long)rel->addr;
+
+ switch (rel->type) {
+ case R_MIPS_PC16:
+ *rel->addr |= build_bimm(laddr - (raddr + 4));
+ break;
+
+ default:
+ panic("Unsupported Micro-assembler relocation %d",
+ rel->type);
+ }
+}
+
+void __init
+uasm_resolve_relocs(struct uasm_reloc *rel, struct uasm_label *lab)
+{
+ struct uasm_label *l;
+
+ for (; rel->lab != UASM_LABEL_INVALID; rel++)
+ for (l = lab; l->lab != UASM_LABEL_INVALID; l++)
+ if (rel->lab == l->lab)
+ __resolve_relocs(rel, l);
+}
+
+void __init
+uasm_move_relocs(struct uasm_reloc *rel, u32 *first, u32 *end, long off)
+{
+ for (; rel->lab != UASM_LABEL_INVALID; rel++)
+ if (rel->addr >= first && rel->addr < end)
+ rel->addr += off;
+}
+
+void __init
+uasm_move_labels(struct uasm_label *lab, u32 *first, u32 *end, long off)
+{
+ for (; lab->lab != UASM_LABEL_INVALID; lab++)
+ if (lab->addr >= first && lab->addr < end)
+ lab->addr += off;
+}
+
+void __init
+uasm_copy_handler(struct uasm_reloc *rel, struct uasm_label *lab, u32 *first,
+ u32 *end, u32 *target)
+{
+ long off = (long)(target - first);
+
+ memcpy(target, first, (end - first) * sizeof(u32));
+
+ uasm_move_relocs(rel, first, end, off);
+ uasm_move_labels(lab, first, end, off);
+}
+
+int __init uasm_insn_has_bdelay(struct uasm_reloc *rel, u32 *addr)
+{
+ for (; rel->lab != UASM_LABEL_INVALID; rel++) {
+ if (rel->addr == addr
+ && (rel->type == R_MIPS_PC16
+ || rel->type == R_MIPS_26))
+ return 1;
+ }
+
+ return 0;
+}
+
+/* Convenience functions for labeled branches. */
+void __init
+uasm_il_bltz(u32 **p, struct uasm_reloc **r, unsigned int reg, int lid)
+{
+ uasm_r_mips_pc16(r, *p, lid);
+ uasm_i_bltz(p, reg, 0);
+}
+
+void __init
+uasm_il_b(u32 **p, struct uasm_reloc **r, int lid)
+{
+ uasm_r_mips_pc16(r, *p, lid);
+ uasm_i_b(p, 0);
+}
+
+void __init
+uasm_il_beqz(u32 **p, struct uasm_reloc **r, unsigned int reg, int lid)
+{
+ uasm_r_mips_pc16(r, *p, lid);
+ uasm_i_beqz(p, reg, 0);
+}
+
+void __init
+uasm_il_beqzl(u32 **p, struct uasm_reloc **r, unsigned int reg, int lid)
+{
+ uasm_r_mips_pc16(r, *p, lid);
+ uasm_i_beqzl(p, reg, 0);
+}
+
+void __init
+uasm_il_bnez(u32 **p, struct uasm_reloc **r, unsigned int reg, int lid)
+{
+ uasm_r_mips_pc16(r, *p, lid);
+ uasm_i_bnez(p, reg, 0);
+}
+
+void __init
+uasm_il_bgezl(u32 **p, struct uasm_reloc **r, unsigned int reg, int lid)
+{
+ uasm_r_mips_pc16(r, *p, lid);
+ uasm_i_bgezl(p, reg, 0);
+}
+
+void __init
+uasm_il_bgez(u32 **p, struct uasm_reloc **r, unsigned int reg, int lid)
+{
+ uasm_r_mips_pc16(r, *p, lid);
+ uasm_i_bgez(p, reg, 0);
+}
--- /dev/null
+/*
+ * This file is subject to the terms and conditions of the GNU General Public
+ * License. See the file "COPYING" in the main directory of this archive
+ * for more details.
+ *
+ * Copyright (C) 2004, 2005, 2006, 2008 Thiemo Seufer
+ * Copyright (C) 2005 Maciej W. Rozycki
+ * Copyright (C) 2006 Ralf Baechle (ralf@linux-mips.org)
+ */
+
+#include <linux/types.h>
+
+#define Ip_u1u2u3(op) \
+void __init \
+uasm_i##op(u32 **buf, unsigned int a, unsigned int b, unsigned int c)
+
+#define Ip_u2u1u3(op) \
+void __init \
+uasm_i##op(u32 **buf, unsigned int a, unsigned int b, unsigned int c)
+
+#define Ip_u3u1u2(op) \
+void __init \
+uasm_i##op(u32 **buf, unsigned int a, unsigned int b, unsigned int c)
+
+#define Ip_u1u2s3(op) \
+void __init \
+uasm_i##op(u32 **buf, unsigned int a, unsigned int b, signed int c)
+
+#define Ip_u2s3u1(op) \
+void __init \
+uasm_i##op(u32 **buf, unsigned int a, signed int b, unsigned int c)
+
+#define Ip_u2u1s3(op) \
+void __init \
+uasm_i##op(u32 **buf, unsigned int a, unsigned int b, signed int c)
+
+#define Ip_u1u2(op) \
+void __init uasm_i##op(u32 **buf, unsigned int a, unsigned int b)
+
+#define Ip_u1s2(op) \
+void __init uasm_i##op(u32 **buf, unsigned int a, signed int b)
+
+#define Ip_u1(op) void __init uasm_i##op(u32 **buf, unsigned int a)
+
+#define Ip_0(op) void __init uasm_i##op(u32 **buf)
+
+Ip_u2u1s3(_addiu);
+Ip_u3u1u2(_addu);
+Ip_u2u1u3(_andi);
+Ip_u3u1u2(_and);
+Ip_u1u2s3(_beq);
+Ip_u1u2s3(_beql);
+Ip_u1s2(_bgez);
+Ip_u1s2(_bgezl);
+Ip_u1s2(_bltz);
+Ip_u1s2(_bltzl);
+Ip_u1u2s3(_bne);
+Ip_u1u2u3(_dmfc0);
+Ip_u1u2u3(_dmtc0);
+Ip_u2u1s3(_daddiu);
+Ip_u3u1u2(_daddu);
+Ip_u2u1u3(_dsll);
+Ip_u2u1u3(_dsll32);
+Ip_u2u1u3(_dsra);
+Ip_u2u1u3(_dsrl);
+Ip_u2u1u3(_dsrl32);
+Ip_u3u1u2(_dsubu);
+Ip_0(_eret);
+Ip_u1(_j);
+Ip_u1(_jal);
+Ip_u1(_jr);
+Ip_u2s3u1(_ld);
+Ip_u2s3u1(_ll);
+Ip_u2s3u1(_lld);
+Ip_u1s2(_lui);
+Ip_u2s3u1(_lw);
+Ip_u1u2u3(_mfc0);
+Ip_u1u2u3(_mtc0);
+Ip_u2u1u3(_ori);
+Ip_0(_rfe);
+Ip_u2s3u1(_sc);
+Ip_u2s3u1(_scd);
+Ip_u2s3u1(_sd);
+Ip_u2u1u3(_sll);
+Ip_u2u1u3(_sra);
+Ip_u2u1u3(_srl);
+Ip_u3u1u2(_subu);
+Ip_u2s3u1(_sw);
+Ip_0(_tlbp);
+Ip_0(_tlbwi);
+Ip_0(_tlbwr);
+Ip_u3u1u2(_xor);
+Ip_u2u1u3(_xori);
+
+/* Handle labels. */
+struct uasm_label {
+ u32 *addr;
+ int lab;
+};
+
+void __init uasm_build_label(struct uasm_label **lab, u32 *addr, int lid);
+#ifdef CONFIG_64BIT
+int __init uasm_in_compat_space_p(long addr);
+int __init uasm_rel_highest(long val);
+int __init uasm_rel_higher(long val);
+#endif
+int __init uasm_rel_hi(long val);
+int __init uasm_rel_lo(long val);
+void __init UASM_i_LA_mostly(u32 **buf, unsigned int rs, long addr);
+void __init UASM_i_LA(u32 **buf, unsigned int rs, long addr);
+
+#define UASM_L_LA(lb) \
+static inline void __init uasm_l##lb(struct uasm_label **lab, u32 *addr) \
+{ \
+ uasm_build_label(lab, addr, label##lb); \
+}
+
+/* convenience macros for instructions */
+#ifdef CONFIG_64BIT
+# define UASM_i_LW(buf, rs, rt, off) uasm_i_ld(buf, rs, rt, off)
+# define UASM_i_SW(buf, rs, rt, off) uasm_i_sd(buf, rs, rt, off)
+# define UASM_i_SLL(buf, rs, rt, sh) uasm_i_dsll(buf, rs, rt, sh)
+# define UASM_i_SRA(buf, rs, rt, sh) uasm_i_dsra(buf, rs, rt, sh)
+# define UASM_i_SRL(buf, rs, rt, sh) uasm_i_dsrl(buf, rs, rt, sh)
+# define UASM_i_MFC0(buf, rt, rd...) uasm_i_dmfc0(buf, rt, rd)
+# define UASM_i_MTC0(buf, rt, rd...) uasm_i_dmtc0(buf, rt, rd)
+# define UASM_i_ADDIU(buf, rs, rt, val) uasm_i_daddiu(buf, rs, rt, val)
+# define UASM_i_ADDU(buf, rs, rt, rd) uasm_i_daddu(buf, rs, rt, rd)
+# define UASM_i_SUBU(buf, rs, rt, rd) uasm_i_dsubu(buf, rs, rt, rd)
+# define UASM_i_LL(buf, rs, rt, off) uasm_i_lld(buf, rs, rt, off)
+# define UASM_i_SC(buf, rs, rt, off) uasm_i_scd(buf, rs, rt, off)
+#else
+# define UASM_i_LW(buf, rs, rt, off) uasm_i_lw(buf, rs, rt, off)
+# define UASM_i_SW(buf, rs, rt, off) uasm_i_sw(buf, rs, rt, off)
+# define UASM_i_SLL(buf, rs, rt, sh) uasm_i_sll(buf, rs, rt, sh)
+# define UASM_i_SRA(buf, rs, rt, sh) uasm_i_sra(buf, rs, rt, sh)
+# define UASM_i_SRL(buf, rs, rt, sh) uasm_i_srl(buf, rs, rt, sh)
+# define UASM_i_MFC0(buf, rt, rd...) uasm_i_mfc0(buf, rt, rd)
+# define UASM_i_MTC0(buf, rt, rd...) uasm_i_mtc0(buf, rt, rd)
+# define UASM_i_ADDIU(buf, rs, rt, val) uasm_i_addiu(buf, rs, rt, val)
+# define UASM_i_ADDU(buf, rs, rt, rd) uasm_i_addu(buf, rs, rt, rd)
+# define UASM_i_SUBU(buf, rs, rt, rd) uasm_i_subu(buf, rs, rt, rd)
+# define UASM_i_LL(buf, rs, rt, off) uasm_i_ll(buf, rs, rt, off)
+# define UASM_i_SC(buf, rs, rt, off) uasm_i_sc(buf, rs, rt, off)
+#endif
+
+#define uasm_i_b(buf, off) uasm_i_beq(buf, 0, 0, off)
+#define uasm_i_beqz(buf, rs, off) uasm_i_beq(buf, rs, 0, off)
+#define uasm_i_beqzl(buf, rs, off) uasm_i_beql(buf, rs, 0, off)
+#define uasm_i_bnez(buf, rs, off) uasm_i_bne(buf, rs, 0, off)
+#define uasm_i_bnezl(buf, rs, off) uasm_i_bnel(buf, rs, 0, off)
+#define uasm_i_move(buf, a, b) UASM_i_ADDU(buf, a, 0, b)
+#define uasm_i_nop(buf) uasm_i_sll(buf, 0, 0, 0)
+#define uasm_i_ssnop(buf) uasm_i_sll(buf, 0, 0, 1)
+#define uasm_i_ehb(buf) uasm_i_sll(buf, 0, 0, 3)
+
+/* Handle relocations. */
+struct uasm_reloc {
+ u32 *addr;
+ unsigned int type;
+ int lab;
+};
+
+/* This is zero so we can use zeroed label arrays. */
+#define UASM_LABEL_INVALID 0
+
+void __init uasm_r_mips_pc16(struct uasm_reloc **rel, u32 *addr, int lid);
+void __init
+uasm_resolve_relocs(struct uasm_reloc *rel, struct uasm_label *lab);
+void __init
+uasm_move_relocs(struct uasm_reloc *rel, u32 *first, u32 *end, long off);
+void __init
+uasm_move_labels(struct uasm_label *lab, u32 *first, u32 *end, long off);
+void __init
+uasm_copy_handler(struct uasm_reloc *rel, struct uasm_label *lab, u32 *first,
+ u32 *end, u32 *target);
+int __init uasm_insn_has_bdelay(struct uasm_reloc *rel, u32 *addr);
+
+/* Convenience functions for labeled branches. */
+void __init
+uasm_il_bltz(u32 **p, struct uasm_reloc **r, unsigned int reg, int lid);
+void __init uasm_il_b(u32 **p, struct uasm_reloc **r, int lid);
+void __init
+uasm_il_beqz(u32 **p, struct uasm_reloc **r, unsigned int reg, int lid);
+void __init
+uasm_il_beqzl(u32 **p, struct uasm_reloc **r, unsigned int reg, int lid);
+void __init
+uasm_il_bnez(u32 **p, struct uasm_reloc **r, unsigned int reg, int lid);
+void __init
+uasm_il_bgezl(u32 **p, struct uasm_reloc **r, unsigned int reg, int lid);
+void __init
+uasm_il_bgez(u32 **p, struct uasm_reloc **r, unsigned int reg, int lid);
config STACKTRACE_SUPPORT
def_bool y
+config HAVE_LATENCYTOP_SUPPORT
+ def_bool y
+
config SEMAPHORE_SLEEPERS
def_bool y
ENTRY(startup_64)
SECTIONS
{
- /* Be careful parts of head_64.S assume startup_64 is at
+ /* Be careful parts of head_64.S assume startup_32 is at
* address 0.
*/
. = 0;
* ACPI based hotplug support for CPU
*/
#ifdef CONFIG_ACPI_HOTPLUG_CPU
-int acpi_map_lsapic(acpi_handle handle, int *pcpu)
+
+static int __cpuinit _acpi_map_lsapic(acpi_handle handle, int *pcpu)
{
struct acpi_buffer buffer = { ACPI_ALLOCATE_BUFFER, NULL };
union acpi_object *obj;
return 0;
}
+/* wrapper to silence section mismatch warning */
+int __ref acpi_map_lsapic(acpi_handle handle, int *pcpu)
+{
+ return _acpi_map_lsapic(handle, pcpu);
+}
EXPORT_SYMBOL(acpi_map_lsapic);
int acpi_unmap_lsapic(int cpu)
void __init cpu_detect(struct cpuinfo_x86 *c)
{
/* Get vendor name */
- cpuid(0x00000000, &c->cpuid_level,
- (int *)&c->x86_vendor_id[0],
- (int *)&c->x86_vendor_id[8],
- (int *)&c->x86_vendor_id[4]);
+ cpuid(0x00000000, (unsigned int *)&c->cpuid_level,
+ (unsigned int *)&c->x86_vendor_id[0],
+ (unsigned int *)&c->x86_vendor_id[8],
+ (unsigned int *)&c->x86_vendor_id[4]);
c->x86 = 4;
if (c->cpuid_level >= 0x00000001) {
static void __cpuinit early_get_cap(struct cpuinfo_x86 *c)
{
u32 tfms, xlvl;
- int ebx;
+ unsigned int ebx;
memset(&c->x86_capability, 0, sizeof c->x86_capability);
if (have_cpuid_p()) {
static void __cpuinit generic_identify(struct cpuinfo_x86 * c)
{
u32 tfms, xlvl;
- int ebx;
+ unsigned int ebx;
if (have_cpuid_p()) {
/* Get vendor name */
- cpuid(0x00000000, &c->cpuid_level,
- (int *)&c->x86_vendor_id[0],
- (int *)&c->x86_vendor_id[8],
- (int *)&c->x86_vendor_id[4]);
+ cpuid(0x00000000, (unsigned int *)&c->cpuid_level,
+ (unsigned int *)&c->x86_vendor_id[0],
+ (unsigned int *)&c->x86_vendor_id[8],
+ (unsigned int *)&c->x86_vendor_id[4]);
get_cpu_vendor(c, 0);
/* Initialize the standard set of capabilities */
return err ? NOTIFY_BAD : NOTIFY_OK;
}
-static struct notifier_block __cpuinitdata cpuid_class_cpu_notifier =
+static struct notifier_block __refdata cpuid_class_cpu_notifier =
{
.notifier_call = cpuid_class_cpu_callback,
};
struct early_res {
unsigned long start, end;
+ char name[16];
};
static struct early_res early_res[MAX_EARLY_RES] __initdata = {
- { 0, PAGE_SIZE }, /* BIOS data page */
+ { 0, PAGE_SIZE, "BIOS data page" }, /* BIOS data page */
#ifdef CONFIG_SMP
- { SMP_TRAMPOLINE_BASE, SMP_TRAMPOLINE_BASE + 2*PAGE_SIZE },
+ { SMP_TRAMPOLINE_BASE, SMP_TRAMPOLINE_BASE + 2*PAGE_SIZE, "SMP_TRAMPOLINE" },
#endif
{}
};
-void __init reserve_early(unsigned long start, unsigned long end)
+void __init reserve_early(unsigned long start, unsigned long end, char *name)
{
int i;
struct early_res *r;
for (i = 0; i < MAX_EARLY_RES && early_res[i].end; i++) {
r = &early_res[i];
if (end > r->start && start < r->end)
- panic("Overlapping early reservations %lx-%lx to %lx-%lx\n",
- start, end, r->start, r->end);
+ panic("Overlapping early reservations %lx-%lx %s to %lx-%lx %s\n",
+ start, end - 1, name?name:"", r->start, r->end - 1, r->name);
}
if (i >= MAX_EARLY_RES)
panic("Too many early reservations");
r = &early_res[i];
r->start = start;
r->end = end;
+ if (name)
+ strncpy(r->name, name, sizeof(r->name) - 1);
}
void __init early_res_to_bootmem(void)
int i;
for (i = 0; i < MAX_EARLY_RES && early_res[i].end; i++) {
struct early_res *r = &early_res[i];
+ printk(KERN_INFO "early res: %d [%lx-%lx] %s\n", i,
+ r->start, r->end - 1, r->name);
reserve_bootmem_generic(r->start, r->end - r->start);
}
}
}
/*
- * Find a free area in a specific range.
+ * Find a free area with specified alignment in a specific range.
*/
unsigned long __init find_e820_area(unsigned long start, unsigned long end,
- unsigned size)
+ unsigned size, unsigned long align)
{
int i;
+ unsigned long mask = ~(align - 1);
for (i = 0; i < e820.nr_map; i++) {
struct e820entry *ei = &e820.map[i];
continue;
while (bad_addr(&addr, size) && addr+size <= ei->addr+ei->size)
;
- last = PAGE_ALIGN(addr) + size;
+ addr = (addr + align - 1) & mask;
+ last = addr + size;
if (last > ei->addr + ei->size)
continue;
if (last > end)
};
/* Direct interface for emergencies */
-struct console *early_console = &early_vga_console;
+static struct console *early_console = &early_vga_console;
static int early_console_initialized = 0;
void early_printk(const char *fmt, ...)
int executable)
{
pte_t *kpte;
- int level;
+ unsigned int level;
while (start < end) {
kpte = lookup_address((unsigned long)__va(start), &level);
if (ebda_size > 64*1024)
ebda_size = 64*1024;
- reserve_early(ebda_addr, ebda_addr + ebda_size);
+ reserve_early(ebda_addr, ebda_addr + ebda_size, "EBDA");
}
void __init x86_64_start_kernel(char * real_mode_data)
pda_init(0);
copy_bootdata(__va(real_mode_data));
- reserve_early(__pa_symbol(&_text), __pa_symbol(&_end));
+ reserve_early(__pa_symbol(&_text), __pa_symbol(&_end), "TEXT DATA BSS");
/* Reserve INITRD */
if (boot_params.hdr.type_of_loader && boot_params.hdr.ramdisk_image) {
unsigned long ramdisk_image = boot_params.hdr.ramdisk_image;
unsigned long ramdisk_size = boot_params.hdr.ramdisk_size;
unsigned long ramdisk_end = ramdisk_image + ramdisk_size;
- reserve_early(ramdisk_image, ramdisk_end);
+ reserve_early(ramdisk_image, ramdisk_end, "RAMDISK");
}
reserve_ebda();
return NOTIFY_OK;
}
-static struct notifier_block __cpuinitdata mc_cpu_notifier = {
+static struct notifier_block __refdata mc_cpu_notifier = {
.notifier_call = mc_cpu_callback,
};
return err ? NOTIFY_BAD : NOTIFY_OK;
}
-static struct notifier_block __cpuinitdata msr_class_cpu_notifier = {
+static struct notifier_block __refdata msr_class_cpu_notifier = {
.notifier_call = msr_class_cpu_callback,
};
readq(target); /* flush */
}
-static void calioc2_handle_quirks(struct iommu_table *tbl, struct pci_dev *dev)
+static void __init calioc2_handle_quirks(struct iommu_table *tbl, struct pci_dev *dev)
{
unsigned char busnum = dev->bus->number;
void __iomem *bbar = tbl->bbar;
writel(cpu_to_be32(val), target);
}
-static void calgary_handle_quirks(struct iommu_table *tbl, struct pci_dev *dev)
+static void __init calgary_handle_quirks(struct iommu_table *tbl, struct pci_dev *dev)
{
unsigned char busnum = dev->bus->number;
unsigned long bootmap_size, bootmap;
bootmap_size = bootmem_bootmap_pages(end_pfn)<<PAGE_SHIFT;
- bootmap = find_e820_area(0, end_pfn<<PAGE_SHIFT, bootmap_size);
+ bootmap = find_e820_area(0, end_pfn<<PAGE_SHIFT, bootmap_size,
+ PAGE_SIZE);
if (bootmap == -1L)
panic("Cannot find bootmem map of size %ld\n", bootmap_size);
bootmap_size = init_bootmem(bootmap >> PAGE_SHIFT, end_pfn);
cpu_clear(cpu, cpu_sibling_setup_map);
}
-void remove_cpu_from_maps(void)
+static void __ref remove_cpu_from_maps(void)
{
int cpu = smp_processor_id();
}
EXPORT_SYMBOL(arch_unregister_cpu);
#else
-int arch_register_cpu(int num)
+static int __init arch_register_cpu(int num)
{
return register_cpu(&per_cpu(cpu_devices, num).cpu, num);
}
-EXPORT_SYMBOL(arch_register_cpu);
#endif /*CONFIG_HOTPLUG_CPU*/
static int __init topology_init(void)
#ifdef CONFIG_X86_PAE
if (error_code & PF_INSTR) {
- int level;
+ unsigned int level;
pte_t *pte = lookup_address(address, &level);
if (pte && pte_present(*pte) && !pte_exec(*pte))
* need roughly 0.5KB per GB.
*/
start = 0x8000;
- table_start = find_e820_area(start, end, tables);
+ table_start = find_e820_area(start, end, tables, PAGE_SIZE);
if (table_start == -1UL)
panic("Cannot find space for the kernel page tables");
- /*
- * When you have a lot of RAM like 256GB, early_table will not fit
- * into 0x8000 range, find_e820_area() will find area after kernel
- * bss but the table_start is not page aligned, so need to round it
- * up to avoid overlap with bss:
- */
- table_start = round_up(table_start, PAGE_SIZE);
table_start >>= PAGE_SHIFT;
table_end = table_start;
mmu_cr4_features = read_cr4();
__flush_tlb_all();
- reserve_early(table_start << PAGE_SHIFT, table_end << PAGE_SHIFT);
+ if (!after_bootmem)
+ reserve_early(table_start << PAGE_SHIFT,
+ table_end << PAGE_SHIFT, "PGTABLE");
}
#ifndef CONFIG_NUMA
{
unsigned long vaddr = (unsigned long)__va(paddr);
unsigned long nrpages = size >> PAGE_SHIFT;
- int err, level;
+ unsigned int level;
+ int err;
/* No change for pages after the last mapping */
if ((paddr + size - 1) >= (max_pfn_mapped << PAGE_SHIFT))
static int __init allocate_cachealigned_memnodemap(void)
{
- unsigned long pad, pad_addr;
+ unsigned long addr;
memnodemap = memnode.embedded_map;
if (memnodemapsize <= ARRAY_SIZE(memnode.embedded_map))
return 0;
- pad = L1_CACHE_BYTES - 1;
- pad_addr = 0x8000;
- nodemap_size = pad + sizeof(s16) * memnodemapsize;
- nodemap_addr = find_e820_area(pad_addr, end_pfn<<PAGE_SHIFT,
- nodemap_size);
+ addr = 0x8000;
+ nodemap_size = round_up(sizeof(s16) * memnodemapsize, L1_CACHE_BYTES);
+ nodemap_addr = find_e820_area(addr, end_pfn<<PAGE_SHIFT,
+ nodemap_size, L1_CACHE_BYTES);
if (nodemap_addr == -1UL) {
printk(KERN_ERR
"NUMA: Unable to allocate Memory to Node hash map\n");
nodemap_addr = nodemap_size = 0;
return -1;
}
- pad_addr = (nodemap_addr + pad) & ~pad;
- memnodemap = phys_to_virt(pad_addr);
- reserve_early(nodemap_addr, nodemap_addr + nodemap_size);
+ memnodemap = phys_to_virt(nodemap_addr);
+ reserve_early(nodemap_addr, nodemap_addr + nodemap_size, "MEMNODEMAP");
printk(KERN_DEBUG "NUMA: Allocated memnodemap from %lx - %lx\n",
nodemap_addr, nodemap_addr + nodemap_size);
}
static void * __init early_node_mem(int nodeid, unsigned long start,
- unsigned long end, unsigned long size)
+ unsigned long end, unsigned long size,
+ unsigned long align)
{
- unsigned long mem = find_e820_area(start, end, size);
+ unsigned long mem = find_e820_area(start, end, size, align);
void *ptr;
if (mem != -1L)
return __va(mem);
- ptr = __alloc_bootmem_nopanic(size,
- SMP_CACHE_BYTES, __pa(MAX_DMA_ADDRESS));
+
+ ptr = __alloc_bootmem_nopanic(size, align, __pa(MAX_DMA_ADDRESS));
if (ptr == NULL) {
printk(KERN_ERR "Cannot find %lu bytes in node %d\n",
size, nodeid);
start_pfn = start >> PAGE_SHIFT;
end_pfn = end >> PAGE_SHIFT;
- node_data[nodeid] = early_node_mem(nodeid, start, end, pgdat_size);
+ node_data[nodeid] = early_node_mem(nodeid, start, end, pgdat_size,
+ SMP_CACHE_BYTES);
if (node_data[nodeid] == NULL)
return;
nodedata_phys = __pa(node_data[nodeid]);
/* Find a place for the bootmem map */
bootmap_pages = bootmem_bootmap_pages(end_pfn - start_pfn);
bootmap_start = round_up(nodedata_phys + pgdat_size, PAGE_SIZE);
+ /*
+ * SMP_CAHCE_BYTES could be enough, but init_bootmem_node like
+ * to use that to align to PAGE_SIZE
+ */
bootmap = early_node_mem(nodeid, bootmap_start, end,
- bootmap_pages<<PAGE_SHIFT);
+ bootmap_pages<<PAGE_SHIFT, PAGE_SIZE);
if (bootmap == NULL) {
if (nodedata_phys < start || nodedata_phys >= end)
free_bootmem((unsigned long)node_data[nodeid],
s->max_exec = 0;
for (i = 0; i < max_pfn_mapped; ) {
unsigned long addr = (unsigned long)__va(i << PAGE_SHIFT);
- int level;
+ unsigned int level;
pte_t *pte;
pte = lookup_address(addr, &level);
unsigned long *bm;
pte_t *pte, pte0;
int failed = 0;
- int level;
+ unsigned int level;
int i, k;
int err;
config SATA_MV
tristate "Marvell SATA support (HIGHLY EXPERIMENTAL)"
- depends on PCI && EXPERIMENTAL
+ depends on EXPERIMENTAL
help
This option enables support for the Marvell Serial ATA family.
Currently supports 88SX[56]0[48][01] chips.
{ PCI_VDEVICE(INTEL, 0x294e), board_ahci }, /* ICH9M */
{ PCI_VDEVICE(INTEL, 0x502a), board_ahci }, /* Tolapai */
{ PCI_VDEVICE(INTEL, 0x502b), board_ahci }, /* Tolapai */
+ { PCI_VDEVICE(INTEL, 0x3a05), board_ahci }, /* ICH10 */
+ { PCI_VDEVICE(INTEL, 0x3a25), board_ahci }, /* ICH10 */
/* JMicron 360/1/3/5/6, match class to avoid IDE function */
{ PCI_VENDOR_ID_JMICRON, PCI_ANY_ID, PCI_ANY_ID, PCI_ANY_ID,
{ 0x8086, 0x292e, PCI_ANY_ID, PCI_ANY_ID, 0, 0, ich8_sata_ahci },
/* SATA Controller IDE (Tolapai) */
{ 0x8086, 0x5028, PCI_ANY_ID, PCI_ANY_ID, 0, 0, tolapai_sata_ahci },
+ /* SATA Controller IDE (ICH10) */
+ { 0x8086, 0x3a00, PCI_ANY_ID, PCI_ANY_ID, 0, 0, ich8_sata_ahci },
+ /* SATA Controller IDE (ICH10) */
+ { 0x8086, 0x3a06, PCI_ANY_ID, PCI_ANY_ID, 0, 0, ich8_2port_sata },
+ /* SATA Controller IDE (ICH10) */
+ { 0x8086, 0x3a20, PCI_ANY_ID, PCI_ANY_ID, 0, 0, ich8_sata_ahci },
+ /* SATA Controller IDE (ICH10) */
+ { 0x8086, 0x3a26, PCI_ANY_ID, PCI_ANY_ID, 0, 0, ich8_2port_sata },
{ } /* terminate list */
};
iowrite32(val, hpriv->sidpr + PIIX_SIDPR_DATA);
}
-u32 piix_merge_scr(u32 val0, u32 val1, const int * const *merge_tbl)
+static u32 piix_merge_scr(u32 val0, u32 val1, const int * const *merge_tbl)
{
u32 val = 0;
int i, mi;
*/
n6 = num_clocks_min(t6min, fsclk);
if (mode >= 0 && mode <= 4 && n6 >= 1) {
- pr_debug("set piomode: mode=%d, fsclk=%ud\n", mode, fsclk);
+ dev_dbg(adev->link->ap->dev, "set piomode: mode=%d, fsclk=%ud\n", mode, fsclk);
/* calculate the timing values for register transfers. */
while (mode > 0 && pio_fsclk[mode] > fsclk)
mode--;
mode = adev->dma_mode - XFER_UDMA_0;
if (mode >= 0 && mode <= 5) {
- pr_debug("set udmamode: mode=%d\n", mode);
+ dev_dbg(adev->link->ap->dev, "set udmamode: mode=%d\n", mode);
/* the most restrictive timing value is t6 and tc,
* the DIOW - data hold. If one SCLK pulse is longer
* than this minimum value then register
mode = adev->dma_mode - XFER_MW_DMA_0;
if (mode >= 0 && mode <= 2) {
- pr_debug("set mdmamode: mode=%d\n", mode);
+ dev_dbg(adev->link->ap->dev, "set mdmamode: mode=%d\n", mode);
/* the most restrictive timing value is tf, the DMACK to
* read data released. If one SCLK pulse is longer than
* this maximum value then the MDMA mode
write_atapi_register(base, ATA_REG_LBAL, tf->hob_lbal);
write_atapi_register(base, ATA_REG_LBAM, tf->hob_lbam);
write_atapi_register(base, ATA_REG_LBAH, tf->hob_lbah);
- pr_debug("hob: feat 0x%X nsect 0x%X, lba 0x%X "
+ dev_dbg(ap->dev, "hob: feat 0x%X nsect 0x%X, lba 0x%X "
"0x%X 0x%X\n",
tf->hob_feature,
tf->hob_nsect,
write_atapi_register(base, ATA_REG_LBAL, tf->lbal);
write_atapi_register(base, ATA_REG_LBAM, tf->lbam);
write_atapi_register(base, ATA_REG_LBAH, tf->lbah);
- pr_debug("feat 0x%X nsect 0x%X lba 0x%X 0x%X 0x%X\n",
+ dev_dbg(ap->dev, "feat 0x%X nsect 0x%X lba 0x%X 0x%X 0x%X\n",
tf->feature,
tf->nsect,
tf->lbal,
if (tf->flags & ATA_TFLAG_DEVICE) {
write_atapi_register(base, ATA_REG_DEVICE, tf->device);
- pr_debug("device 0x%X\n", tf->device);
+ dev_dbg(ap->dev, "device 0x%X\n", tf->device);
}
ata_wait_idle(ap);
const struct ata_taskfile *tf)
{
void __iomem *base = (void __iomem *)ap->ioaddr.ctl_addr;
- pr_debug("ata%u: cmd 0x%X\n", ap->print_id, tf->command);
+ dev_dbg(ap->dev, "ata%u: cmd 0x%X\n", ap->print_id, tf->command);
write_atapi_register(base, ATA_REG_CMD, tf->command);
ata_pause(ap);
struct scatterlist *sg;
unsigned int si;
- pr_debug("in atapi dma setup\n");
+ dev_dbg(qc->ap->dev, "in atapi dma setup\n");
/* Program the ATA_CTRL register with dir */
if (qc->tf.flags & ATA_TFLAG_WRITE) {
/* fill the ATAPI DMA controller */
struct scatterlist *sg;
unsigned int si;
- pr_debug("in atapi dma start\n");
+ dev_dbg(qc->ap->dev, "in atapi dma start\n");
if (!(ap->udma_mask || ap->mwdma_mask))
return;
sg_dma_address(sg) + sg_dma_len(sg));
}
enable_dma(CH_ATAPI_TX);
- pr_debug("enable udma write\n");
+ dev_dbg(qc->ap->dev, "enable udma write\n");
/* Send ATA DMA write command */
bfin_exec_command(ap, &qc->tf);
| XFER_DIR));
} else {
enable_dma(CH_ATAPI_RX);
- pr_debug("enable udma read\n");
+ dev_dbg(qc->ap->dev, "enable udma read\n");
/* Send ATA DMA read command */
bfin_exec_command(ap, &qc->tf);
struct scatterlist *sg;
unsigned int si;
- pr_debug("in atapi dma stop\n");
+ dev_dbg(qc->ap->dev, "in atapi dma stop\n");
if (!(ap->udma_mask || ap->mwdma_mask))
return;
void __iomem *base = (void __iomem *)ap->ioaddr.ctl_addr;
unsigned short int_status = ATAPI_GET_INT_STATUS(base);
- if (ATAPI_GET_STATUS(base) & (MULTI_XFER_ON|ULTRA_XFER_ON)) {
+ if (ATAPI_GET_STATUS(base) & (MULTI_XFER_ON|ULTRA_XFER_ON))
host_stat |= ATA_DMA_ACTIVE;
- }
- if (int_status & (MULTI_DONE_INT|UDMAIN_DONE_INT|UDMAOUT_DONE_INT)) {
+ if (int_status & (MULTI_DONE_INT|UDMAIN_DONE_INT|UDMAOUT_DONE_INT|
+ ATAPI_DEV_INT))
host_stat |= ATA_DMA_INTR;
- }
- if (int_status & (MULTI_TERM_INT|UDMAIN_TERM_INT|UDMAOUT_TERM_INT)) {
- host_stat |= ATA_DMA_ERR;
- }
+ if (int_status & (MULTI_TERM_INT|UDMAIN_TERM_INT|UDMAOUT_TERM_INT))
+ host_stat |= ATA_DMA_ERR|ATA_DMA_INTR;
+
+ dev_dbg(ap->dev, "ATAPI: host_stat=0x%x\n", host_stat);
return host_stat;
}
{
void __iomem *base = (void __iomem *)ap->ioaddr.ctl_addr;
- pr_debug("in atapi irq clear\n");
-
+ dev_dbg(ap->dev, "in atapi irq clear\n");
ATAPI_SET_INT_STATUS(base, ATAPI_GET_INT_STATUS(base)|ATAPI_DEV_INT
| MULTI_DONE_INT | UDMAIN_DONE_INT | UDMAOUT_DONE_INT
| MULTI_TERM_INT | UDMAIN_TERM_INT | UDMAOUT_TERM_INT);
void __iomem *base = (void __iomem *)ap->ioaddr.ctl_addr;
u8 tmp;
- pr_debug("in atapi irq on\n");
+ dev_dbg(ap->dev, "in atapi irq on\n");
ap->ctl &= ~ATA_NIEN;
ap->last_ctl = ap->ctl;
{
void __iomem *base = (void __iomem *)ap->ioaddr.ctl_addr;
- pr_debug("in atapi dma freeze\n");
+ dev_dbg(ap->dev, "in atapi dma freeze\n");
ap->ctl |= ATA_NIEN;
ap->last_ctl = ap->ctl;
static void bfin_port_stop(struct ata_port *ap)
{
- pr_debug("in atapi port stop\n");
+ dev_dbg(ap->dev, "in atapi port stop\n");
if (ap->udma_mask != 0 || ap->mwdma_mask != 0) {
free_dma(CH_ATAPI_RX);
free_dma(CH_ATAPI_TX);
static int bfin_port_start(struct ata_port *ap)
{
- pr_debug("in atapi port start\n");
+ dev_dbg(ap->dev, "in atapi port start\n");
if (!(ap->udma_mask || ap->mwdma_mask))
return 0;
.slave_configure = ata_scsi_slave_config,
.slave_destroy = ata_scsi_slave_destroy,
.bios_param = ata_std_bios_param,
-#ifdef CONFIG_PM
- .resume = ata_scsi_device_resume,
- .suspend = ata_scsi_device_suspend,
-#endif
};
static const struct ata_port_operations bfin_pata_ops = {
#include <linux/libata.h>
#define DRV_NAME "pata_sl82c105"
-#define DRV_VERSION "0.3.2"
+#define DRV_VERSION "0.3.3"
enum {
/*
sl82c105_set_piomode(ap, qc->dev);
}
+/**
+ * sl82c105_qc_defer - implement serialization
+ * @qc: command
+ *
+ * We must issue one command per host not per channel because
+ * of the reset bug.
+ *
+ * Q: is the scsi host lock sufficient ?
+ */
+
+static int sl82c105_qc_defer(struct ata_queued_cmd *qc)
+{
+ struct ata_host *host = qc->ap->host;
+ struct ata_port *alt = host->ports[1 ^ qc->ap->port_no];
+ int rc;
+
+ /* First apply the usual rules */
+ rc = ata_std_qc_defer(qc);
+ if (rc != 0)
+ return rc;
+
+ /* Now apply serialization rules. Only allow a command if the
+ other channel state machine is idle */
+ if (alt && alt->qc_active)
+ return ATA_DEFER_PORT;
+ return 0;
+}
+
static struct scsi_host_template sl82c105_sht = {
.module = THIS_MODULE,
.name = DRV_NAME,
.bmdma_stop = sl82c105_bmdma_stop,
.bmdma_status = ata_bmdma_status,
+ .qc_defer = sl82c105_qc_defer,
.qc_prep = ata_qc_prep,
.qc_issue = ata_qc_issue_prot,
};
/* for now use only the first port */
const struct ata_port_info *ppi[] = { &info_early,
- &ata_dummy_port_info };
+ NULL };
u32 val;
int rev;
I distinctly remember a couple workarounds (one related to PCI-X)
are still needed.
- 4) Add NCQ support (easy to intermediate, once new-EH support appears)
+ 2) Improve/fix IRQ and error handling sequences.
+
+ 3) ATAPI support (Marvell claims the 60xx/70xx chips can do it).
+
+ 4) Think about TCQ support here, and for libata in general
+ with controllers that suppport it via host-queuing hardware
+ (a software-only implementation could be a nightmare).
5) Investigate problems with PCI Message Signalled Interrupts (MSI).
Target mode, for those without docs, is the ability to directly
connect two SATA controllers.
- 13) Verify that 7042 is fully supported. I only have a 6042.
-
*/
#include <linux/libata.h>
#define DRV_NAME "sata_mv"
-#define DRV_VERSION "1.01"
+#define DRV_VERSION "1.20"
enum {
/* BAR's are enumerated in terms of pci_resource_start() terms */
/* CRQB needs alignment on a 1KB boundary. Size == 1KB
* CRPB needs alignment on a 256B boundary. Size == 256B
- * SG count of 176 leads to MV_PORT_PRIV_DMA_SZ == 4KB
* ePRD (SG) entries need alignment on a 16B boundary. Size == 16B
*/
MV_CRQB_Q_SZ = (32 * MV_MAX_Q_DEPTH),
MV_CRPB_Q_SZ = (8 * MV_MAX_Q_DEPTH),
- MV_MAX_SG_CT = 176,
+ MV_MAX_SG_CT = 256,
MV_SG_TBL_SZ = (16 * MV_MAX_SG_CT),
- MV_PORT_PRIV_DMA_SZ = (MV_CRQB_Q_SZ + MV_CRPB_Q_SZ + MV_SG_TBL_SZ),
MV_PORTS_PER_HC = 4,
/* == (port / MV_PORTS_PER_HC) to determine HC from 0-7 port */
/* Host Flags */
MV_FLAG_DUAL_HC = (1 << 30), /* two SATA Host Controllers */
MV_FLAG_IRQ_COALESCE = (1 << 29), /* IRQ coalescing capability */
+ /* SoC integrated controllers, no PCI interface */
+ MV_FLAG_SOC = (1 << 28),
+
MV_COMMON_FLAGS = ATA_FLAG_SATA | ATA_FLAG_NO_LEGACY |
ATA_FLAG_MMIO | ATA_FLAG_NO_ATAPI |
ATA_FLAG_PIO_POLLING,
PCIE_IRQ_CAUSE_OFS = 0x1900,
PCIE_IRQ_MASK_OFS = 0x1910,
- PCIE_UNMASK_ALL_IRQS = 0x70a, /* assorted bits */
+ PCIE_UNMASK_ALL_IRQS = 0x40a, /* assorted bits */
HC_MAIN_IRQ_CAUSE_OFS = 0x1d60,
HC_MAIN_IRQ_MASK_OFS = 0x1d64,
/* SATA registers */
SATA_STATUS_OFS = 0x300, /* ctrl, err regs follow status */
SATA_ACTIVE_OFS = 0x350,
+ SATA_FIS_IRQ_CAUSE_OFS = 0x364,
PHY_MODE3 = 0x310,
PHY_MODE4 = 0x314,
PHY_MODE2 = 0x330,
/* Port registers */
EDMA_CFG_OFS = 0,
- EDMA_CFG_Q_DEPTH = 0, /* queueing disabled */
- EDMA_CFG_NCQ = (1 << 5),
- EDMA_CFG_NCQ_GO_ON_ERR = (1 << 14), /* continue on error */
- EDMA_CFG_RD_BRST_EXT = (1 << 11), /* read burst 512B */
- EDMA_CFG_WR_BUFF_LEN = (1 << 13), /* write buffer 512B */
+ EDMA_CFG_Q_DEPTH = 0x1f, /* max device queue depth */
+ EDMA_CFG_NCQ = (1 << 5), /* for R/W FPDMA queued */
+ EDMA_CFG_NCQ_GO_ON_ERR = (1 << 14), /* continue on error */
+ EDMA_CFG_RD_BRST_EXT = (1 << 11), /* read burst 512B */
+ EDMA_CFG_WR_BUFF_LEN = (1 << 13), /* write buffer 512B */
EDMA_ERR_IRQ_CAUSE_OFS = 0x8,
EDMA_ERR_IRQ_MASK_OFS = 0xc,
EDMA_ERR_CRPB_PAR = (1 << 10), /* CRPB parity error */
EDMA_ERR_INTRL_PAR = (1 << 11), /* internal parity error */
EDMA_ERR_IORDY = (1 << 12), /* IORdy timeout */
+
EDMA_ERR_LNK_CTRL_RX = (0xf << 13), /* link ctrl rx error */
- EDMA_ERR_LNK_CTRL_RX_2 = (1 << 15),
+ EDMA_ERR_LNK_CTRL_RX_0 = (1 << 13), /* transient: CRC err */
+ EDMA_ERR_LNK_CTRL_RX_1 = (1 << 14), /* transient: FIFO err */
+ EDMA_ERR_LNK_CTRL_RX_2 = (1 << 15), /* fatal: caught SYNC */
+ EDMA_ERR_LNK_CTRL_RX_3 = (1 << 16), /* transient: FIS rx err */
+
EDMA_ERR_LNK_DATA_RX = (0xf << 17), /* link data rx error */
+
EDMA_ERR_LNK_CTRL_TX = (0x1f << 21), /* link ctrl tx error */
+ EDMA_ERR_LNK_CTRL_TX_0 = (1 << 21), /* transient: CRC err */
+ EDMA_ERR_LNK_CTRL_TX_1 = (1 << 22), /* transient: FIFO err */
+ EDMA_ERR_LNK_CTRL_TX_2 = (1 << 23), /* transient: caught SYNC */
+ EDMA_ERR_LNK_CTRL_TX_3 = (1 << 24), /* transient: caught DMAT */
+ EDMA_ERR_LNK_CTRL_TX_4 = (1 << 25), /* transient: FIS collision */
+
EDMA_ERR_LNK_DATA_TX = (0x1f << 26), /* link data tx error */
+
EDMA_ERR_TRANS_PROTO = (1 << 31), /* transport protocol error */
EDMA_ERR_OVERRUN_5 = (1 << 5),
EDMA_ERR_UNDERRUN_5 = (1 << 6),
+
+ EDMA_ERR_IRQ_TRANSIENT = EDMA_ERR_LNK_CTRL_RX_0 |
+ EDMA_ERR_LNK_CTRL_RX_1 |
+ EDMA_ERR_LNK_CTRL_RX_3 |
+ EDMA_ERR_LNK_CTRL_TX,
+
EDMA_EH_FREEZE = EDMA_ERR_D_PAR |
EDMA_ERR_PRD_PAR |
EDMA_ERR_DEV_DCON |
/* Port private flags (pp_flags) */
MV_PP_FLAG_EDMA_EN = (1 << 0), /* is EDMA engine enabled? */
+ MV_PP_FLAG_NCQ_EN = (1 << 1), /* is EDMA set up for NCQ? */
MV_PP_FLAG_HAD_A_RESET = (1 << 2), /* 1st hard reset complete? */
};
#define IS_GEN_I(hpriv) ((hpriv)->hp_flags & MV_HP_GEN_I)
#define IS_GEN_II(hpriv) ((hpriv)->hp_flags & MV_HP_GEN_II)
#define IS_GEN_IIE(hpriv) ((hpriv)->hp_flags & MV_HP_GEN_IIE)
+#define HAS_PCI(host) (!((host)->ports[0]->flags & MV_FLAG_SOC))
enum {
/* DMA boundary 0xffff is required by the s/g splitting
dma_addr_t crqb_dma;
struct mv_crpb *crpb;
dma_addr_t crpb_dma;
- struct mv_sg *sg_tbl;
- dma_addr_t sg_tbl_dma;
+ struct mv_sg *sg_tbl[MV_MAX_Q_DEPTH];
+ dma_addr_t sg_tbl_dma[MV_MAX_Q_DEPTH];
unsigned int req_idx;
unsigned int resp_idx;
u32 irq_cause_ofs;
u32 irq_mask_ofs;
u32 unmask_all_irqs;
+ /*
+ * These consistent DMA memory pools give us guaranteed
+ * alignment for hardware-accessed data structures,
+ * and less memory waste in accomplishing the alignment.
+ */
+ struct dma_pool *crqb_pool;
+ struct dma_pool *crpb_pool;
+ struct dma_pool *sg_tbl_pool;
};
struct mv_hw_ops {
int (*reset_hc)(struct mv_host_priv *hpriv, void __iomem *mmio,
unsigned int n_hc);
void (*reset_flash)(struct mv_host_priv *hpriv, void __iomem *mmio);
- void (*reset_bus)(struct pci_dev *pdev, void __iomem *mmio);
+ void (*reset_bus)(struct ata_host *host, void __iomem *mmio);
};
static void mv_irq_clear(struct ata_port *ap);
static void mv_qc_prep_iie(struct ata_queued_cmd *qc);
static unsigned int mv_qc_issue(struct ata_queued_cmd *qc);
static void mv_error_handler(struct ata_port *ap);
-static void mv_post_int_cmd(struct ata_queued_cmd *qc);
static void mv_eh_freeze(struct ata_port *ap);
static void mv_eh_thaw(struct ata_port *ap);
-static int mv_init_one(struct pci_dev *pdev, const struct pci_device_id *ent);
+static void mv6_dev_config(struct ata_device *dev);
static void mv5_phy_errata(struct mv_host_priv *hpriv, void __iomem *mmio,
unsigned int port);
static int mv5_reset_hc(struct mv_host_priv *hpriv, void __iomem *mmio,
unsigned int n_hc);
static void mv5_reset_flash(struct mv_host_priv *hpriv, void __iomem *mmio);
-static void mv5_reset_bus(struct pci_dev *pdev, void __iomem *mmio);
+static void mv5_reset_bus(struct ata_host *host, void __iomem *mmio);
static void mv6_phy_errata(struct mv_host_priv *hpriv, void __iomem *mmio,
unsigned int port);
static int mv6_reset_hc(struct mv_host_priv *hpriv, void __iomem *mmio,
unsigned int n_hc);
static void mv6_reset_flash(struct mv_host_priv *hpriv, void __iomem *mmio);
-static void mv_reset_pci_bus(struct pci_dev *pdev, void __iomem *mmio);
+static void mv_reset_pci_bus(struct ata_host *host, void __iomem *mmio);
static void mv_channel_reset(struct mv_host_priv *hpriv, void __iomem *mmio,
unsigned int port_no);
+static void mv_edma_cfg(struct mv_port_priv *pp, struct mv_host_priv *hpriv,
+ void __iomem *port_mmio, int want_ncq);
+static int __mv_stop_dma(struct ata_port *ap);
+/* .sg_tablesize is (MV_MAX_SG_CT / 2) in the structures below
+ * because we have to allow room for worst case splitting of
+ * PRDs for 64K boundaries in mv_fill_sg().
+ */
static struct scsi_host_template mv5_sht = {
.module = THIS_MODULE,
.name = DRV_NAME,
.name = DRV_NAME,
.ioctl = ata_scsi_ioctl,
.queuecommand = ata_scsi_queuecmd,
- .can_queue = ATA_DEF_QUEUE,
+ .change_queue_depth = ata_scsi_change_queue_depth,
+ .can_queue = MV_MAX_Q_DEPTH - 1,
.this_id = ATA_SHT_THIS_ID,
.sg_tablesize = MV_MAX_SG_CT / 2,
.cmd_per_lun = ATA_SHT_CMD_PER_LUN,
.irq_on = ata_irq_on,
.error_handler = mv_error_handler,
- .post_internal_cmd = mv_post_int_cmd,
.freeze = mv_eh_freeze,
.thaw = mv_eh_thaw,
};
static const struct ata_port_operations mv6_ops = {
+ .dev_config = mv6_dev_config,
.tf_load = ata_tf_load,
.tf_read = ata_tf_read,
.check_status = ata_check_status,
.irq_on = ata_irq_on,
.error_handler = mv_error_handler,
- .post_internal_cmd = mv_post_int_cmd,
.freeze = mv_eh_freeze,
.thaw = mv_eh_thaw,
+ .qc_defer = ata_std_qc_defer,
.scr_read = mv_scr_read,
.scr_write = mv_scr_write,
.irq_on = ata_irq_on,
.error_handler = mv_error_handler,
- .post_internal_cmd = mv_post_int_cmd,
.freeze = mv_eh_freeze,
.thaw = mv_eh_thaw,
+ .qc_defer = ata_std_qc_defer,
.scr_read = mv_scr_read,
.scr_write = mv_scr_write,
.port_ops = &mv5_ops,
},
{ /* chip_604x */
- .flags = MV_COMMON_FLAGS | MV_6XXX_FLAGS,
+ .flags = MV_COMMON_FLAGS | MV_6XXX_FLAGS |
+ ATA_FLAG_NCQ,
.pio_mask = 0x1f, /* pio0-4 */
.udma_mask = ATA_UDMA6,
.port_ops = &mv6_ops,
},
{ /* chip_608x */
.flags = MV_COMMON_FLAGS | MV_6XXX_FLAGS |
- MV_FLAG_DUAL_HC,
+ ATA_FLAG_NCQ | MV_FLAG_DUAL_HC,
.pio_mask = 0x1f, /* pio0-4 */
.udma_mask = ATA_UDMA6,
.port_ops = &mv6_ops,
},
{ /* chip_6042 */
- .flags = MV_COMMON_FLAGS | MV_6XXX_FLAGS,
+ .flags = MV_COMMON_FLAGS | MV_6XXX_FLAGS |
+ ATA_FLAG_NCQ,
.pio_mask = 0x1f, /* pio0-4 */
.udma_mask = ATA_UDMA6,
.port_ops = &mv_iie_ops,
},
{ /* chip_7042 */
- .flags = MV_COMMON_FLAGS | MV_6XXX_FLAGS,
+ .flags = MV_COMMON_FLAGS | MV_6XXX_FLAGS |
+ ATA_FLAG_NCQ,
.pio_mask = 0x1f, /* pio0-4 */
.udma_mask = ATA_UDMA6,
.port_ops = &mv_iie_ops,
{ } /* terminate list */
};
-static struct pci_driver mv_pci_driver = {
- .name = DRV_NAME,
- .id_table = mv_pci_tbl,
- .probe = mv_init_one,
- .remove = ata_pci_remove_one,
-};
-
static const struct mv_hw_ops mv5xxx_ops = {
.phy_errata = mv5_phy_errata,
.enable_leds = mv5_enable_leds,
};
/*
- * module options
- */
-static int msi; /* Use PCI msi; either zero (off, default) or non-zero */
-
-
-/* move to PCI layer or libata core? */
-static int pci_go_64(struct pci_dev *pdev)
-{
- int rc;
-
- if (!pci_set_dma_mask(pdev, DMA_64BIT_MASK)) {
- rc = pci_set_consistent_dma_mask(pdev, DMA_64BIT_MASK);
- if (rc) {
- rc = pci_set_consistent_dma_mask(pdev, DMA_32BIT_MASK);
- if (rc) {
- dev_printk(KERN_ERR, &pdev->dev,
- "64-bit DMA enable failed\n");
- return rc;
- }
- }
- } else {
- rc = pci_set_dma_mask(pdev, DMA_32BIT_MASK);
- if (rc) {
- dev_printk(KERN_ERR, &pdev->dev,
- "32-bit DMA enable failed\n");
- return rc;
- }
- rc = pci_set_consistent_dma_mask(pdev, DMA_32BIT_MASK);
- if (rc) {
- dev_printk(KERN_ERR, &pdev->dev,
- "32-bit consistent DMA enable failed\n");
- return rc;
- }
- }
-
- return rc;
-}
-
-/*
* Functions
*/
* LOCKING:
* Inherited from caller.
*/
-static void mv_start_dma(void __iomem *base, struct mv_host_priv *hpriv,
- struct mv_port_priv *pp)
+static void mv_start_dma(struct ata_port *ap, void __iomem *port_mmio,
+ struct mv_port_priv *pp, u8 protocol)
{
+ int want_ncq = (protocol == ATA_PROT_NCQ);
+
+ if (pp->pp_flags & MV_PP_FLAG_EDMA_EN) {
+ int using_ncq = ((pp->pp_flags & MV_PP_FLAG_NCQ_EN) != 0);
+ if (want_ncq != using_ncq)
+ __mv_stop_dma(ap);
+ }
if (!(pp->pp_flags & MV_PP_FLAG_EDMA_EN)) {
+ struct mv_host_priv *hpriv = ap->host->private_data;
+ int hard_port = mv_hardport_from_port(ap->port_no);
+ void __iomem *hc_mmio = mv_hc_base_from_port(
+ ap->host->iomap[MV_PRIMARY_BAR], hard_port);
+ u32 hc_irq_cause, ipending;
+
/* clear EDMA event indicators, if any */
- writelfl(0, base + EDMA_ERR_IRQ_CAUSE_OFS);
+ writelfl(0, port_mmio + EDMA_ERR_IRQ_CAUSE_OFS);
+
+ /* clear EDMA interrupt indicator, if any */
+ hc_irq_cause = readl(hc_mmio + HC_IRQ_CAUSE_OFS);
+ ipending = (DEV_IRQ << hard_port) |
+ (CRPB_DMA_DONE << hard_port);
+ if (hc_irq_cause & ipending) {
+ writelfl(hc_irq_cause & ~ipending,
+ hc_mmio + HC_IRQ_CAUSE_OFS);
+ }
+
+ mv_edma_cfg(pp, hpriv, port_mmio, want_ncq);
+
+ /* clear FIS IRQ Cause */
+ writelfl(0, port_mmio + SATA_FIS_IRQ_CAUSE_OFS);
- mv_set_edma_ptrs(base, hpriv, pp);
+ mv_set_edma_ptrs(port_mmio, hpriv, pp);
- writelfl(EDMA_EN, base + EDMA_CMD_OFS);
+ writelfl(EDMA_EN, port_mmio + EDMA_CMD_OFS);
pp->pp_flags |= MV_PP_FLAG_EDMA_EN;
}
- WARN_ON(!(EDMA_EN & readl(base + EDMA_CMD_OFS)));
+ WARN_ON(!(EDMA_EN & readl(port_mmio + EDMA_CMD_OFS)));
}
/**
return -EINVAL;
}
-static void mv_edma_cfg(struct ata_port *ap, struct mv_host_priv *hpriv,
- void __iomem *port_mmio)
+static void mv6_dev_config(struct ata_device *adev)
{
- u32 cfg = readl(port_mmio + EDMA_CFG_OFS);
+ /*
+ * We don't have hob_nsect when doing NCQ commands on Gen-II.
+ * See mv_qc_prep() for more info.
+ */
+ if (adev->flags & ATA_DFLAG_NCQ)
+ if (adev->max_sectors > ATA_MAX_SECTORS)
+ adev->max_sectors = ATA_MAX_SECTORS;
+}
+
+static void mv_edma_cfg(struct mv_port_priv *pp, struct mv_host_priv *hpriv,
+ void __iomem *port_mmio, int want_ncq)
+{
+ u32 cfg;
/* set up non-NCQ EDMA configuration */
- cfg &= ~(1 << 9); /* disable eQue */
+ cfg = EDMA_CFG_Q_DEPTH; /* always 0x1f for *all* chips */
- if (IS_GEN_I(hpriv)) {
- cfg &= ~0x1f; /* clear queue depth */
+ if (IS_GEN_I(hpriv))
cfg |= (1 << 8); /* enab config burst size mask */
- }
- else if (IS_GEN_II(hpriv)) {
- cfg &= ~0x1f; /* clear queue depth */
+ else if (IS_GEN_II(hpriv))
cfg |= EDMA_CFG_RD_BRST_EXT | EDMA_CFG_WR_BUFF_LEN;
- cfg &= ~(EDMA_CFG_NCQ | EDMA_CFG_NCQ_GO_ON_ERR); /* clear NCQ */
- }
else if (IS_GEN_IIE(hpriv)) {
cfg |= (1 << 23); /* do not mask PM field in rx'd FIS */
cfg |= (1 << 22); /* enab 4-entry host queue cache */
- cfg &= ~(1 << 19); /* dis 128-entry queue (for now?) */
cfg |= (1 << 18); /* enab early completion */
cfg |= (1 << 17); /* enab cut-through (dis stor&forwrd) */
- cfg &= ~(1 << 16); /* dis FIS-based switching (for now) */
- cfg &= ~(EDMA_CFG_NCQ); /* clear NCQ */
}
+ if (want_ncq) {
+ cfg |= EDMA_CFG_NCQ;
+ pp->pp_flags |= MV_PP_FLAG_NCQ_EN;
+ } else
+ pp->pp_flags &= ~MV_PP_FLAG_NCQ_EN;
+
writelfl(cfg, port_mmio + EDMA_CFG_OFS);
}
+static void mv_port_free_dma_mem(struct ata_port *ap)
+{
+ struct mv_host_priv *hpriv = ap->host->private_data;
+ struct mv_port_priv *pp = ap->private_data;
+ int tag;
+
+ if (pp->crqb) {
+ dma_pool_free(hpriv->crqb_pool, pp->crqb, pp->crqb_dma);
+ pp->crqb = NULL;
+ }
+ if (pp->crpb) {
+ dma_pool_free(hpriv->crpb_pool, pp->crpb, pp->crpb_dma);
+ pp->crpb = NULL;
+ }
+ /*
+ * For GEN_I, there's no NCQ, so we have only a single sg_tbl.
+ * For later hardware, we have one unique sg_tbl per NCQ tag.
+ */
+ for (tag = 0; tag < MV_MAX_Q_DEPTH; ++tag) {
+ if (pp->sg_tbl[tag]) {
+ if (tag == 0 || !IS_GEN_I(hpriv))
+ dma_pool_free(hpriv->sg_tbl_pool,
+ pp->sg_tbl[tag],
+ pp->sg_tbl_dma[tag]);
+ pp->sg_tbl[tag] = NULL;
+ }
+ }
+}
+
/**
* mv_port_start - Port specific init/start routine.
* @ap: ATA channel to manipulate
struct mv_host_priv *hpriv = ap->host->private_data;
struct mv_port_priv *pp;
void __iomem *port_mmio = mv_ap_base(ap);
- void *mem;
- dma_addr_t mem_dma;
unsigned long flags;
- int rc;
+ int tag, rc;
pp = devm_kzalloc(dev, sizeof(*pp), GFP_KERNEL);
if (!pp)
return -ENOMEM;
-
- mem = dmam_alloc_coherent(dev, MV_PORT_PRIV_DMA_SZ, &mem_dma,
- GFP_KERNEL);
- if (!mem)
- return -ENOMEM;
- memset(mem, 0, MV_PORT_PRIV_DMA_SZ);
+ ap->private_data = pp;
rc = ata_pad_alloc(ap, dev);
if (rc)
return rc;
- /* First item in chunk of DMA memory:
- * 32-slot command request table (CRQB), 32 bytes each in size
- */
- pp->crqb = mem;
- pp->crqb_dma = mem_dma;
- mem += MV_CRQB_Q_SZ;
- mem_dma += MV_CRQB_Q_SZ;
+ pp->crqb = dma_pool_alloc(hpriv->crqb_pool, GFP_KERNEL, &pp->crqb_dma);
+ if (!pp->crqb)
+ return -ENOMEM;
+ memset(pp->crqb, 0, MV_CRQB_Q_SZ);
- /* Second item:
- * 32-slot command response table (CRPB), 8 bytes each in size
- */
- pp->crpb = mem;
- pp->crpb_dma = mem_dma;
- mem += MV_CRPB_Q_SZ;
- mem_dma += MV_CRPB_Q_SZ;
+ pp->crpb = dma_pool_alloc(hpriv->crpb_pool, GFP_KERNEL, &pp->crpb_dma);
+ if (!pp->crpb)
+ goto out_port_free_dma_mem;
+ memset(pp->crpb, 0, MV_CRPB_Q_SZ);
- /* Third item:
- * Table of scatter-gather descriptors (ePRD), 16 bytes each
+ /*
+ * For GEN_I, there's no NCQ, so we only allocate a single sg_tbl.
+ * For later hardware, we need one unique sg_tbl per NCQ tag.
*/
- pp->sg_tbl = mem;
- pp->sg_tbl_dma = mem_dma;
+ for (tag = 0; tag < MV_MAX_Q_DEPTH; ++tag) {
+ if (tag == 0 || !IS_GEN_I(hpriv)) {
+ pp->sg_tbl[tag] = dma_pool_alloc(hpriv->sg_tbl_pool,
+ GFP_KERNEL, &pp->sg_tbl_dma[tag]);
+ if (!pp->sg_tbl[tag])
+ goto out_port_free_dma_mem;
+ } else {
+ pp->sg_tbl[tag] = pp->sg_tbl[0];
+ pp->sg_tbl_dma[tag] = pp->sg_tbl_dma[0];
+ }
+ }
spin_lock_irqsave(&ap->host->lock, flags);
- mv_edma_cfg(ap, hpriv, port_mmio);
-
+ mv_edma_cfg(pp, hpriv, port_mmio, 0);
mv_set_edma_ptrs(port_mmio, hpriv, pp);
spin_unlock_irqrestore(&ap->host->lock, flags);
* we'll be unable to send non-data, PIO, etc due to restricted access
* to shadow regs.
*/
- ap->private_data = pp;
return 0;
+
+out_port_free_dma_mem:
+ mv_port_free_dma_mem(ap);
+ return -ENOMEM;
}
/**
static void mv_port_stop(struct ata_port *ap)
{
mv_stop_dma(ap);
+ mv_port_free_dma_mem(ap);
}
/**
struct mv_sg *mv_sg, *last_sg = NULL;
unsigned int si;
- mv_sg = pp->sg_tbl;
+ mv_sg = pp->sg_tbl[qc->tag];
for_each_sg(qc->sg, sg, qc->n_elem, si) {
dma_addr_t addr = sg_dma_address(sg);
u32 sg_len = sg_dma_len(sg);
u16 flags = 0;
unsigned in_index;
- if (qc->tf.protocol != ATA_PROT_DMA)
+ if ((qc->tf.protocol != ATA_PROT_DMA) &&
+ (qc->tf.protocol != ATA_PROT_NCQ))
return;
/* Fill in command request block
flags |= CRQB_FLAG_READ;
WARN_ON(MV_MAX_Q_DEPTH <= qc->tag);
flags |= qc->tag << CRQB_TAG_SHIFT;
- flags |= qc->tag << CRQB_IOID_SHIFT; /* 50xx appears to ignore this*/
/* get current queue index from software */
in_index = pp->req_idx & MV_MAX_Q_DEPTH_MASK;
pp->crqb[in_index].sg_addr =
- cpu_to_le32(pp->sg_tbl_dma & 0xffffffff);
+ cpu_to_le32(pp->sg_tbl_dma[qc->tag] & 0xffffffff);
pp->crqb[in_index].sg_addr_hi =
- cpu_to_le32((pp->sg_tbl_dma >> 16) >> 16);
+ cpu_to_le32((pp->sg_tbl_dma[qc->tag] >> 16) >> 16);
pp->crqb[in_index].ctrl_flags = cpu_to_le16(flags);
cw = &pp->crqb[in_index].ata_cmd[0];
case ATA_CMD_WRITE_FUA_EXT:
mv_crqb_pack_cmd(cw++, tf->hob_nsect, ATA_REG_NSECT, 0);
break;
-#ifdef LIBATA_NCQ /* FIXME: remove this line when NCQ added */
case ATA_CMD_FPDMA_READ:
case ATA_CMD_FPDMA_WRITE:
mv_crqb_pack_cmd(cw++, tf->hob_feature, ATA_REG_FEATURE, 0);
mv_crqb_pack_cmd(cw++, tf->feature, ATA_REG_FEATURE, 0);
break;
-#endif /* FIXME: remove this line when NCQ added */
default:
/* The only other commands EDMA supports in non-queued and
* non-NCQ mode are: [RW] STREAM DMA and W DMA FUA EXT, none
unsigned in_index;
u32 flags = 0;
- if (qc->tf.protocol != ATA_PROT_DMA)
+ if ((qc->tf.protocol != ATA_PROT_DMA) &&
+ (qc->tf.protocol != ATA_PROT_NCQ))
return;
/* Fill in Gen IIE command request block
WARN_ON(MV_MAX_Q_DEPTH <= qc->tag);
flags |= qc->tag << CRQB_TAG_SHIFT;
- flags |= qc->tag << CRQB_IOID_SHIFT; /* "I/O Id" is -really-
- what we use as our tag */
+ flags |= qc->tag << CRQB_HOSTQ_SHIFT;
/* get current queue index from software */
in_index = pp->req_idx & MV_MAX_Q_DEPTH_MASK;
crqb = (struct mv_crqb_iie *) &pp->crqb[in_index];
- crqb->addr = cpu_to_le32(pp->sg_tbl_dma & 0xffffffff);
- crqb->addr_hi = cpu_to_le32((pp->sg_tbl_dma >> 16) >> 16);
+ crqb->addr = cpu_to_le32(pp->sg_tbl_dma[qc->tag] & 0xffffffff);
+ crqb->addr_hi = cpu_to_le32((pp->sg_tbl_dma[qc->tag] >> 16) >> 16);
crqb->flags = cpu_to_le32(flags);
tf = &qc->tf;
struct ata_port *ap = qc->ap;
void __iomem *port_mmio = mv_ap_base(ap);
struct mv_port_priv *pp = ap->private_data;
- struct mv_host_priv *hpriv = ap->host->private_data;
u32 in_index;
- if (qc->tf.protocol != ATA_PROT_DMA) {
+ if ((qc->tf.protocol != ATA_PROT_DMA) &&
+ (qc->tf.protocol != ATA_PROT_NCQ)) {
/* We're about to send a non-EDMA capable command to the
* port. Turn off EDMA so there won't be problems accessing
* shadow block, etc registers.
return ata_qc_issue_prot(qc);
}
- mv_start_dma(port_mmio, hpriv, pp);
-
- in_index = pp->req_idx & MV_MAX_Q_DEPTH_MASK;
-
- /* until we do queuing, the queue should be empty at this point */
- WARN_ON(in_index != ((readl(port_mmio + EDMA_REQ_Q_OUT_PTR_OFS)
- >> EDMA_REQ_Q_PTR_SHIFT) & MV_MAX_Q_DEPTH_MASK));
+ mv_start_dma(ap, port_mmio, pp, qc->tf.protocol);
pp->req_idx++;
ata_ehi_hotplugged(ehi);
ata_ehi_push_desc(ehi, edma_err_cause & EDMA_ERR_DEV_DCON ?
"dev disconnect" : "dev connect");
+ action |= ATA_EH_HARDRESET;
}
if (IS_GEN_I(hpriv)) {
}
/* Clear EDMA now that SERR cleanup done */
- writelfl(0, port_mmio + EDMA_ERR_IRQ_CAUSE_OFS);
+ writelfl(~edma_err_cause, port_mmio + EDMA_ERR_IRQ_CAUSE_OFS);
if (!err_mask) {
err_mask = AC_ERR_OTHER;
* support for queueing. this works transparently for
* queued and non-queued modes.
*/
- else if (IS_GEN_II(hpriv))
- tag = (le16_to_cpu(pp->crpb[out_index].id)
- >> CRPB_IOID_SHIFT_6) & 0x3f;
-
- else /* IS_GEN_IIE */
- tag = (le16_to_cpu(pp->crpb[out_index].id)
- >> CRPB_IOID_SHIFT_7) & 0x3f;
+ else
+ tag = le16_to_cpu(pp->crpb[out_index].id) & 0x1f;
qc = ata_qc_from_tag(ap, tag);
- /* lower 8 bits of status are EDMA_ERR_IRQ_CAUSE_OFS
- * bits (WARNING: might not necessarily be associated
- * with this command), which -should- be clear
- * if all is well
+ /* For non-NCQ mode, the lower 8 bits of status
+ * are from EDMA_ERR_IRQ_CAUSE_OFS,
+ * which should be zero if all went well.
*/
status = le16_to_cpu(pp->crpb[out_index].flags);
- if (unlikely(status & 0xff)) {
+ if ((status & 0xff) && !(pp->pp_flags & MV_PP_FLAG_NCQ_EN)) {
mv_err_intr(ap, qc);
return;
}
struct ata_host *host = dev_instance;
unsigned int hc, handled = 0, n_hcs;
void __iomem *mmio = host->iomap[MV_PRIMARY_BAR];
- u32 irq_stat;
+ u32 irq_stat, irq_mask;
+ spin_lock(&host->lock);
irq_stat = readl(mmio + HC_MAIN_IRQ_CAUSE_OFS);
+ irq_mask = readl(mmio + HC_MAIN_IRQ_MASK_OFS);
/* check the cases where we either have nothing pending or have read
* a bogus register value which can indicate HW removal or PCI fault
*/
- if (!irq_stat || (0xffffffffU == irq_stat))
- return IRQ_NONE;
+ if (!(irq_stat & irq_mask) || (0xffffffffU == irq_stat))
+ goto out_unlock;
n_hcs = mv_get_hc_count(host->ports[0]->flags);
- spin_lock(&host->lock);
- if (unlikely(irq_stat & PCI_ERR)) {
+ if (unlikely((irq_stat & PCI_ERR) && HAS_PCI(host))) {
mv_pci_error(host, mmio);
handled = 1;
goto out_unlock; /* skip all other HC irq handling */
return -EINVAL;
}
-static void mv5_reset_bus(struct pci_dev *pdev, void __iomem *mmio)
+static void mv5_reset_bus(struct ata_host *host, void __iomem *mmio)
{
+ struct pci_dev *pdev = to_pci_dev(host->dev);
int early_5080;
early_5080 = (pdev->device == 0x5080) && (pdev->revision == 0);
writel(tmp, mmio + MV_PCI_EXP_ROM_BAR_CTL);
}
- mv_reset_pci_bus(pdev, mmio);
+ mv_reset_pci_bus(host, mmio);
}
static void mv5_reset_flash(struct mv_host_priv *hpriv, void __iomem *mmio)
#undef ZERO
#define ZERO(reg) writel(0, mmio + (reg))
-static void mv_reset_pci_bus(struct pci_dev *pdev, void __iomem *mmio)
+static void mv_reset_pci_bus(struct ata_host *host, void __iomem *mmio)
{
- struct ata_host *host = dev_get_drvdata(&pdev->dev);
struct mv_host_priv *hpriv = host->private_data;
u32 tmp;
mv_hardreset, mv_postreset);
}
-static void mv_post_int_cmd(struct ata_queued_cmd *qc)
-{
- mv_stop_dma(qc->ap);
-}
-
static void mv_eh_freeze(struct ata_port *ap)
{
void __iomem *mmio = ap->host->iomap[MV_PRIMARY_BAR];
writelfl(readl(port_mmio + serr_ofs), port_mmio + serr_ofs);
writelfl(0, port_mmio + EDMA_ERR_IRQ_CAUSE_OFS);
- /* unmask all EDMA error interrupts */
- writelfl(~0, port_mmio + EDMA_ERR_IRQ_MASK_OFS);
+ /* unmask all non-transient EDMA error interrupts */
+ writelfl(~EDMA_ERR_IRQ_TRANSIENT, port_mmio + EDMA_ERR_IRQ_MASK_OFS);
VPRINTK("EDMA cfg=0x%08x EDMA IRQ err cause/mask=0x%08x/0x%08x\n",
readl(port_mmio + EDMA_CFG_OFS),
static int mv_init_host(struct ata_host *host, unsigned int board_idx)
{
int rc = 0, n_hc, port, hc;
- struct pci_dev *pdev = to_pci_dev(host->dev);
void __iomem *mmio = host->iomap[MV_PRIMARY_BAR];
struct mv_host_priv *hpriv = host->private_data;
goto done;
hpriv->ops->reset_flash(hpriv, mmio);
- hpriv->ops->reset_bus(pdev, mmio);
+ hpriv->ops->reset_bus(host, mmio);
hpriv->ops->enable_leds(hpriv, mmio);
for (port = 0; port < host->n_ports; port++) {
mv_port_init(&ap->ioaddr, port_mmio);
+#ifdef CONFIG_PCI
ata_port_pbar_desc(ap, MV_PRIMARY_BAR, -1, "mmio");
ata_port_pbar_desc(ap, MV_PRIMARY_BAR, offset, "port");
+#endif
}
for (hc = 0; hc < n_hc; hc++) {
return rc;
}
+#ifdef CONFIG_PCI
+static int mv_init_one(struct pci_dev *pdev, const struct pci_device_id *ent);
+
+static struct pci_driver mv_pci_driver = {
+ .name = DRV_NAME,
+ .id_table = mv_pci_tbl,
+ .probe = mv_init_one,
+ .remove = ata_pci_remove_one,
+};
+
+/*
+ * module options
+ */
+static int msi; /* Use PCI msi; either zero (off, default) or non-zero */
+
+
+/* move to PCI layer or libata core? */
+static int pci_go_64(struct pci_dev *pdev)
+{
+ int rc;
+
+ if (!pci_set_dma_mask(pdev, DMA_64BIT_MASK)) {
+ rc = pci_set_consistent_dma_mask(pdev, DMA_64BIT_MASK);
+ if (rc) {
+ rc = pci_set_consistent_dma_mask(pdev, DMA_32BIT_MASK);
+ if (rc) {
+ dev_printk(KERN_ERR, &pdev->dev,
+ "64-bit DMA enable failed\n");
+ return rc;
+ }
+ }
+ } else {
+ rc = pci_set_dma_mask(pdev, DMA_32BIT_MASK);
+ if (rc) {
+ dev_printk(KERN_ERR, &pdev->dev,
+ "32-bit DMA enable failed\n");
+ return rc;
+ }
+ rc = pci_set_consistent_dma_mask(pdev, DMA_32BIT_MASK);
+ if (rc) {
+ dev_printk(KERN_ERR, &pdev->dev,
+ "32-bit consistent DMA enable failed\n");
+ return rc;
+ }
+ }
+
+ return rc;
+}
+
/**
* mv_print_info - Dump key info to kernel log for perusal.
* @host: ATA host to print info about
scc_s, (MV_HP_FLAG_MSI & hpriv->hp_flags) ? "MSI" : "INTx");
}
+static int mv_create_dma_pools(struct mv_host_priv *hpriv, struct device *dev)
+{
+ hpriv->crqb_pool = dmam_pool_create("crqb_q", dev, MV_CRQB_Q_SZ,
+ MV_CRQB_Q_SZ, 0);
+ if (!hpriv->crqb_pool)
+ return -ENOMEM;
+
+ hpriv->crpb_pool = dmam_pool_create("crpb_q", dev, MV_CRPB_Q_SZ,
+ MV_CRPB_Q_SZ, 0);
+ if (!hpriv->crpb_pool)
+ return -ENOMEM;
+
+ hpriv->sg_tbl_pool = dmam_pool_create("sg_tbl", dev, MV_SG_TBL_SZ,
+ MV_SG_TBL_SZ, 0);
+ if (!hpriv->sg_tbl_pool)
+ return -ENOMEM;
+
+ return 0;
+}
+
/**
* mv_init_one - handle a positive probe of a Marvell host
* @pdev: PCI device found
if (rc)
return rc;
+ rc = mv_create_dma_pools(hpriv, &pdev->dev);
+ if (rc)
+ return rc;
+
/* initialize adapter */
rc = mv_init_host(host, board_idx);
if (rc)
return ata_host_activate(host, pdev->irq, mv_interrupt, IRQF_SHARED,
IS_GEN_I(hpriv) ? &mv5_sht : &mv6_sht);
}
+#endif
static int __init mv_init(void)
{
- return pci_register_driver(&mv_pci_driver);
+ int rc = -ENODEV;
+#ifdef CONFIG_PCI
+ rc = pci_register_driver(&mv_pci_driver);
+#endif
+ return rc;
}
static void __exit mv_exit(void)
{
+#ifdef CONFIG_PCI
pci_unregister_driver(&mv_pci_driver);
+#endif
}
MODULE_AUTHOR("Brett Russ");
MODULE_DEVICE_TABLE(pci, mv_pci_tbl);
MODULE_VERSION(DRV_VERSION);
+#ifdef CONFIG_PCI
module_param(msi, int, 0444);
MODULE_PARM_DESC(msi, "Enable use of PCI MSI (0=off, 1=on)");
+#endif
module_init(mv_init);
module_exit(mv_exit);
}
if (status & (NV_ADMA_STAT_DONE |
- NV_ADMA_STAT_CPBERR)) {
- u32 check_commands;
+ NV_ADMA_STAT_CPBERR |
+ NV_ADMA_STAT_CMD_COMPLETE)) {
+ u32 check_commands = notifier_clears[i];
int pos, error = 0;
- if (ata_tag_valid(ap->link.active_tag))
- check_commands = 1 << ap->link.active_tag;
- else
- check_commands = ap->link.sactive;
+ if (status & NV_ADMA_STAT_CPBERR) {
+ /* Check all active commands */
+ if (ata_tag_valid(ap->link.active_tag))
+ check_commands = 1 <<
+ ap->link.active_tag;
+ else
+ check_commands = ap->
+ link.sactive;
+ }
/** Check CPBs for completed commands */
while ((pos = ffs(check_commands)) && !error) {
* @tsk with @loginuid. @buf->mutex must be locked.
*/
static void tty_audit_buf_push(struct task_struct *tsk, uid_t loginuid,
+ unsigned int sessionid,
struct tty_audit_buf *buf)
{
struct audit_buffer *ab;
if (ab) {
char name[sizeof(tsk->comm)];
- audit_log_format(ab, "tty pid=%u uid=%u auid=%u major=%d "
- "minor=%d comm=", tsk->pid, tsk->uid,
- loginuid, buf->major, buf->minor);
+ audit_log_format(ab, "tty pid=%u uid=%u auid=%u ses=%u "
+ "major=%d minor=%d comm=", tsk->pid, tsk->uid,
+ loginuid, sessionid, buf->major, buf->minor);
get_task_comm(name, tsk);
audit_log_untrustedstring(ab, name);
audit_log_format(ab, " data=");
*/
static void tty_audit_buf_push_current(struct tty_audit_buf *buf)
{
- tty_audit_buf_push(current, audit_get_loginuid(current->audit_context),
- buf);
+ uid_t auid = audit_get_loginuid(current);
+ unsigned int sessionid = audit_get_sessionid(current);
+ tty_audit_buf_push(current, auid, sessionid, buf);
}
/**
void tty_audit_push_task(struct task_struct *tsk, uid_t loginuid)
{
struct tty_audit_buf *buf;
+ /* FIXME I think this is correct. Check against netlink once that is
+ * I really need to read this code more closely. But that's for
+ * another patch.
+ */
+ unsigned int sessionid = audit_get_sessionid(tsk);
spin_lock_irq(&tsk->sighand->siglock);
buf = tsk->signal->tty_audit_buf;
return;
mutex_lock(&buf->mutex);
- tty_audit_buf_push(tsk, loginuid, buf);
+ tty_audit_buf_push(tsk, loginuid, sessionid, buf);
mutex_unlock(&buf->mutex);
tty_audit_buf_put(buf);
dr = get_pci_dr(pdev);
if (unlikely(!dr))
return -ENOMEM;
- WARN_ON(!!dr->enabled);
+ if (dr->enabled)
+ return 0;
rc = pci_enable_device(pdev);
if (!rc) {
if (!task)
return -ESRCH;
length = scnprintf(tmpbuf, TMPBUFLEN, "%u",
- audit_get_loginuid(task->audit_context));
+ audit_get_loginuid(task));
put_task_struct(task);
return simple_read_from_buffer(buf, count, ppos, tmpbuf, length);
}
struct x86_cpu {
struct cpu cpu;
};
-extern int arch_register_cpu(int num);
+
#ifdef CONFIG_HOTPLUG_CPU
+extern int arch_register_cpu(int num);
extern void arch_unregister_cpu(int);
#endif
#ifndef __ASSEMBLY__
extern unsigned long find_e820_area(unsigned long start, unsigned long end,
- unsigned size);
+ unsigned size, unsigned long align);
extern void add_memory_region(unsigned long start, unsigned long size,
int type);
extern void setup_memory_region(void);
extern struct e820map e820;
extern void update_e820(void);
-extern void reserve_early(unsigned long start, unsigned long end);
+extern void reserve_early(unsigned long start, unsigned long end, char *name);
extern void early_res_to_bootmem(void);
#endif/*!__ASSEMBLY__*/
"1: movl %2, %0\n \
movl %0, %3\n" \
insn "\n" \
-"2: " LOCK_PREFIX "cmpxchgl %3, %2\n \
+"2: lock; cmpxchgl %3, %2\n \
jnz 1b\n \
3: .section .fixup,\"ax\"\n \
4: mov %5, %1\n \
__futex_atomic_op1("xchgl %0, %2", ret, oldval, uaddr, oparg);
break;
case FUTEX_OP_ADD:
- __futex_atomic_op1(LOCK_PREFIX "xaddl %0, %2", ret, oldval,
+ __futex_atomic_op1("lock; xaddl %0, %2", ret, oldval,
uaddr, oparg);
break;
case FUTEX_OP_OR:
return -EFAULT;
__asm__ __volatile__(
- "1: " LOCK_PREFIX "cmpxchgl %3, %1 \n"
+ "1: lock; cmpxchgl %3, %1 \n"
"2: .section .fixup, \"ax\" \n"
"3: mov %2, %0 \n"
" jmp 2b \n"
#define AUDIT_FD_PAIR 1317 /* audit record for pipe/socketpair */
#define AUDIT_OBJ_PID 1318 /* ptrace target */
#define AUDIT_TTY 1319 /* Input on an administrative TTY */
+#define AUDIT_EOE 1320 /* End of multi-record event */
#define AUDIT_AVC 1400 /* SE Linux avc denial or grant */
#define AUDIT_SELINUX_ERR 1401 /* Internal SE Linux Errors */
extern void auditsc_get_stamp(struct audit_context *ctx,
struct timespec *t, unsigned int *serial);
extern int audit_set_loginuid(struct task_struct *task, uid_t loginuid);
-extern uid_t audit_get_loginuid(struct audit_context *ctx);
+#define audit_get_loginuid(t) ((t)->loginuid)
+#define audit_get_sessionid(t) ((t)->sessionid)
extern void audit_log_task_context(struct audit_buffer *ab);
extern int __audit_ipc_obj(struct kern_ipc_perm *ipcp);
extern int __audit_ipc_set_perm(unsigned long qbytes, uid_t uid, gid_t gid, mode_t mode);
#define audit_inode_child(d,i,p) do { ; } while (0)
#define audit_core_dumps(i) do { ; } while (0)
#define auditsc_get_stamp(c,t,s) do { BUG(); } while (0)
-#define audit_get_loginuid(c) ({ -1; })
+#define audit_get_loginuid(t) (-1)
+#define audit_get_sessionid(t) (-1)
#define audit_log_task_context(b) do { ; } while (0)
#define audit_ipc_obj(i) ({ 0; })
#define audit_ipc_set_perm(q,u,g,m) ({ 0; })
extern void audit_log_hex(struct audit_buffer *ab,
const unsigned char *buf,
size_t len);
-extern const char * audit_log_untrustedstring(struct audit_buffer *ab,
+extern int audit_string_contains_control(const char *string,
+ size_t len);
+extern void audit_log_untrustedstring(struct audit_buffer *ab,
const char *string);
-extern const char * audit_log_n_untrustedstring(struct audit_buffer *ab,
+extern void audit_log_n_untrustedstring(struct audit_buffer *ab,
size_t n,
const char *string);
extern void audit_log_d_path(struct audit_buffer *ab,
#define FUTEX_LOCK_PI 6
#define FUTEX_UNLOCK_PI 7
#define FUTEX_TRYLOCK_PI 8
+#define FUTEX_WAIT_BITSET 9
+#define FUTEX_WAKE_BITSET 10
#define FUTEX_PRIVATE_FLAG 128
#define FUTEX_CMD_MASK ~FUTEX_PRIVATE_FLAG
#define FUTEX_LOCK_PI_PRIVATE (FUTEX_LOCK_PI | FUTEX_PRIVATE_FLAG)
#define FUTEX_UNLOCK_PI_PRIVATE (FUTEX_UNLOCK_PI | FUTEX_PRIVATE_FLAG)
#define FUTEX_TRYLOCK_PI_PRIVATE (FUTEX_TRYLOCK_PI | FUTEX_PRIVATE_FLAG)
+#define FUTEX_WAIT_BITSET_PRIVATE (FUTEX_WAIT_BITS | FUTEX_PRIVATE_FLAG)
+#define FUTEX_WAKE_BITSET_PRIVATE (FUTEX_WAKE_BITS | FUTEX_PRIVATE_FLAG)
/*
* Support for robust futexes: the kernel cleans up held futexes at
*/
#define ROBUST_LIST_LIMIT 2048
+/*
+ * bitset with all bits set for the FUTEX_xxx_BITSET OPs to request a
+ * match of any bit.
+ */
+#define FUTEX_BITSET_MATCH_ANY 0xffffffff
+
#ifdef __KERNEL__
long do_futex(u32 __user *uaddr, int op, u32 val, union ktime *timeout,
u32 __user *uaddr2, u32 val2, u32 val3);
.pid = &init_struct_pid, \
}
+#ifdef CONFIG_AUDITSYSCALL
+#define INIT_IDS \
+ .loginuid = -1, \
+ .sessionid = -1,
+#else
+#define INIT_IDS
+#endif
/*
* INIT_TASK is used to set up the first task table, touch at
* your own risk!. Base=0, limit=0x1fffff (=2MB)
[PIDTYPE_SID] = INIT_PID_LINK(PIDTYPE_SID), \
}, \
.dirties = INIT_PROP_LOCAL_SINGLE(dirties), \
+ INIT_IDS \
INIT_TRACE_IRQFLAGS \
INIT_LOCKDEP \
}
void *security;
#endif
struct audit_context *audit_context;
+#ifdef CONFIG_AUDITSYSCALL
+ uid_t loginuid;
+ unsigned int sessionid;
+#endif
seccomp_t seccomp;
/* Thread group tracking */
u32 *uaddr;
u32 val;
u32 flags;
+ u32 bitset;
u64 time;
} futex;
};
* @idle_calls: Total number of idle calls
* @idle_sleeps: Number of idle calls, where the sched tick was stopped
* @idle_entrytime: Time when the idle call was entered
+ * @idle_waketime: Time when the idle was interrupted
+ * @idle_exittime: Time when the idle state was left
* @idle_sleeptime: Sum of the time slept in idle with sched tick stopped
* @sleep_length: Duration of the current idle sleep
*/
unsigned long idle_sleeps;
int idle_active;
ktime_t idle_entrytime;
+ ktime_t idle_waketime;
+ ktime_t idle_exittime;
ktime_t idle_sleeptime;
ktime_t idle_lastupdate;
ktime_t sleep_length;
extern struct timespec timespec_trunc(struct timespec t, unsigned gran);
extern int timekeeping_is_continuous(void);
extern void update_wall_time(void);
+extern void update_xtime_cache(u64 nsec);
/**
* timespec_to_ns - Convert timespec to nanoseconds
* (Initialization happens after skb_init is called.) */
static int audit_initialized;
-/* 0 - no auditing
- * 1 - auditing enabled
- * 2 - auditing enabled and configuration is locked/unchangeable. */
+#define AUDIT_OFF 0
+#define AUDIT_ON 1
+#define AUDIT_LOCKED 2
int audit_enabled;
+int audit_ever_enabled;
/* Default state when kernel boots without any parameters. */
static int audit_default;
static void audit_set_pid(struct audit_buffer *ab, pid_t pid)
{
- struct nlmsghdr *nlh = nlmsg_hdr(ab->skb);
- nlh->nlmsg_pid = pid;
+ if (ab) {
+ struct nlmsghdr *nlh = nlmsg_hdr(ab->skb);
+ nlh->nlmsg_pid = pid;
+ }
}
void audit_panic(const char *message)
case AUDIT_FAIL_SILENT:
break;
case AUDIT_FAIL_PRINTK:
- printk(KERN_ERR "audit: %s\n", message);
+ if (printk_ratelimit())
+ printk(KERN_ERR "audit: %s\n", message);
break;
case AUDIT_FAIL_PANIC:
panic("audit: %s\n", message);
}
if (print) {
- printk(KERN_WARNING
- "audit: audit_lost=%d audit_rate_limit=%d audit_backlog_limit=%d\n",
- atomic_read(&audit_lost),
- audit_rate_limit,
- audit_backlog_limit);
+ if (printk_ratelimit())
+ printk(KERN_WARNING
+ "audit: audit_lost=%d audit_rate_limit=%d "
+ "audit_backlog_limit=%d\n",
+ atomic_read(&audit_lost),
+ audit_rate_limit,
+ audit_backlog_limit);
audit_panic(message);
}
}
-static int audit_set_rate_limit(int limit, uid_t loginuid, u32 sid)
+static int audit_log_config_change(char *function_name, int new, int old,
+ uid_t loginuid, u32 sid, int allow_changes)
{
- int res, rc = 0, old = audit_rate_limit;
-
- /* check if we are locked */
- if (audit_enabled == 2)
- res = 0;
- else
- res = 1;
+ struct audit_buffer *ab;
+ int rc = 0;
+ ab = audit_log_start(NULL, GFP_KERNEL, AUDIT_CONFIG_CHANGE);
+ audit_log_format(ab, "%s=%d old=%d by auid=%u", function_name, new,
+ old, loginuid);
if (sid) {
char *ctx = NULL;
u32 len;
- if ((rc = selinux_sid_to_string(sid, &ctx, &len)) == 0) {
- audit_log(NULL, GFP_KERNEL, AUDIT_CONFIG_CHANGE,
- "audit_rate_limit=%d old=%d by auid=%u"
- " subj=%s res=%d",
- limit, old, loginuid, ctx, res);
+
+ rc = selinux_sid_to_string(sid, &ctx, &len);
+ if (rc) {
+ audit_log_format(ab, " sid=%u", sid);
+ allow_changes = 0; /* Something weird, deny request */
+ } else {
+ audit_log_format(ab, " subj=%s", ctx);
kfree(ctx);
- } else
- res = 0; /* Something weird, deny request */
+ }
}
- audit_log(NULL, GFP_KERNEL, AUDIT_CONFIG_CHANGE,
- "audit_rate_limit=%d old=%d by auid=%u res=%d",
- limit, old, loginuid, res);
-
- /* If we are allowed, make the change */
- if (res == 1)
- audit_rate_limit = limit;
- /* Not allowed, update reason */
- else if (rc == 0)
- rc = -EPERM;
+ audit_log_format(ab, " res=%d", allow_changes);
+ audit_log_end(ab);
return rc;
}
-static int audit_set_backlog_limit(int limit, uid_t loginuid, u32 sid)
+static int audit_do_config_change(char *function_name, int *to_change,
+ int new, uid_t loginuid, u32 sid)
{
- int res, rc = 0, old = audit_backlog_limit;
+ int allow_changes, rc = 0, old = *to_change;
/* check if we are locked */
- if (audit_enabled == 2)
- res = 0;
+ if (audit_enabled == AUDIT_LOCKED)
+ allow_changes = 0;
else
- res = 1;
+ allow_changes = 1;
- if (sid) {
- char *ctx = NULL;
- u32 len;
- if ((rc = selinux_sid_to_string(sid, &ctx, &len)) == 0) {
- audit_log(NULL, GFP_KERNEL, AUDIT_CONFIG_CHANGE,
- "audit_backlog_limit=%d old=%d by auid=%u"
- " subj=%s res=%d",
- limit, old, loginuid, ctx, res);
- kfree(ctx);
- } else
- res = 0; /* Something weird, deny request */
+ if (audit_enabled != AUDIT_OFF) {
+ rc = audit_log_config_change(function_name, new, old,
+ loginuid, sid, allow_changes);
+ if (rc)
+ allow_changes = 0;
}
- audit_log(NULL, GFP_KERNEL, AUDIT_CONFIG_CHANGE,
- "audit_backlog_limit=%d old=%d by auid=%u res=%d",
- limit, old, loginuid, res);
/* If we are allowed, make the change */
- if (res == 1)
- audit_backlog_limit = limit;
+ if (allow_changes == 1)
+ *to_change = new;
/* Not allowed, update reason */
else if (rc == 0)
rc = -EPERM;
return rc;
}
-static int audit_set_enabled(int state, uid_t loginuid, u32 sid)
+static int audit_set_rate_limit(int limit, uid_t loginuid, u32 sid)
+{
+ return audit_do_config_change("audit_rate_limit", &audit_rate_limit,
+ limit, loginuid, sid);
+}
+
+static int audit_set_backlog_limit(int limit, uid_t loginuid, u32 sid)
{
- int res, rc = 0, old = audit_enabled;
+ return audit_do_config_change("audit_backlog_limit", &audit_backlog_limit,
+ limit, loginuid, sid);
+}
- if (state < 0 || state > 2)
+static int audit_set_enabled(int state, uid_t loginuid, u32 sid)
+{
+ int rc;
+ if (state < AUDIT_OFF || state > AUDIT_LOCKED)
return -EINVAL;
- /* check if we are locked */
- if (audit_enabled == 2)
- res = 0;
- else
- res = 1;
+ rc = audit_do_config_change("audit_enabled", &audit_enabled, state,
+ loginuid, sid);
- if (sid) {
- char *ctx = NULL;
- u32 len;
- if ((rc = selinux_sid_to_string(sid, &ctx, &len)) == 0) {
- audit_log(NULL, GFP_KERNEL, AUDIT_CONFIG_CHANGE,
- "audit_enabled=%d old=%d by auid=%u"
- " subj=%s res=%d",
- state, old, loginuid, ctx, res);
- kfree(ctx);
- } else
- res = 0; /* Something weird, deny request */
- }
- audit_log(NULL, GFP_KERNEL, AUDIT_CONFIG_CHANGE,
- "audit_enabled=%d old=%d by auid=%u res=%d",
- state, old, loginuid, res);
+ if (!rc)
+ audit_ever_enabled |= !!state;
- /* If we are allowed, make the change */
- if (res == 1)
- audit_enabled = state;
- /* Not allowed, update reason */
- else if (rc == 0)
- rc = -EPERM;
return rc;
}
static int audit_set_failure(int state, uid_t loginuid, u32 sid)
{
- int res, rc = 0, old = audit_failure;
-
if (state != AUDIT_FAIL_SILENT
&& state != AUDIT_FAIL_PRINTK
&& state != AUDIT_FAIL_PANIC)
return -EINVAL;
- /* check if we are locked */
- if (audit_enabled == 2)
- res = 0;
- else
- res = 1;
-
- if (sid) {
- char *ctx = NULL;
- u32 len;
- if ((rc = selinux_sid_to_string(sid, &ctx, &len)) == 0) {
- audit_log(NULL, GFP_KERNEL, AUDIT_CONFIG_CHANGE,
- "audit_failure=%d old=%d by auid=%u"
- " subj=%s res=%d",
- state, old, loginuid, ctx, res);
- kfree(ctx);
- } else
- res = 0; /* Something weird, deny request */
- }
- audit_log(NULL, GFP_KERNEL, AUDIT_CONFIG_CHANGE,
- "audit_failure=%d old=%d by auid=%u res=%d",
- state, old, loginuid, res);
-
- /* If we are allowed, make the change */
- if (res == 1)
- audit_failure = state;
- /* Not allowed, update reason */
- else if (rc == 0)
- rc = -EPERM;
- return rc;
+ return audit_do_config_change("audit_failure", &audit_failure, state,
+ loginuid, sid);
}
static int kauditd_thread(void *dummy)
audit_pid = 0;
}
} else {
- printk(KERN_NOTICE "%s\n", skb->data + NLMSG_SPACE(0));
+ if (printk_ratelimit())
+ printk(KERN_NOTICE "%s\n", skb->data +
+ NLMSG_SPACE(0));
+ else
+ audit_log_lost("printk limit exceeded\n");
kfree_skb(skb);
}
} else {
return err;
}
+static int audit_log_common_recv_msg(struct audit_buffer **ab, u16 msg_type,
+ u32 pid, u32 uid, uid_t auid, u32 sid)
+{
+ int rc = 0;
+ char *ctx = NULL;
+ u32 len;
+
+ if (!audit_enabled) {
+ *ab = NULL;
+ return rc;
+ }
+
+ *ab = audit_log_start(NULL, GFP_KERNEL, msg_type);
+ audit_log_format(*ab, "user pid=%d uid=%u auid=%u",
+ pid, uid, auid);
+ if (sid) {
+ rc = selinux_sid_to_string(sid, &ctx, &len);
+ if (rc)
+ audit_log_format(*ab, " ssid=%u", sid);
+ else
+ audit_log_format(*ab, " subj=%s", ctx);
+ kfree(ctx);
+ }
+
+ return rc;
+}
+
static int audit_receive_msg(struct sk_buff *skb, struct nlmsghdr *nlh)
{
u32 uid, pid, seq, sid;
u16 msg_type = nlh->nlmsg_type;
uid_t loginuid; /* loginuid of sender */
struct audit_sig_info *sig_data;
- char *ctx;
+ char *ctx = NULL;
u32 len;
err = audit_netlink_ok(skb, msg_type);
if (err < 0) return err;
}
if (status_get->mask & AUDIT_STATUS_PID) {
- int old = audit_pid;
- if (sid) {
- if ((err = selinux_sid_to_string(
- sid, &ctx, &len)))
- return err;
- else
- audit_log(NULL, GFP_KERNEL,
- AUDIT_CONFIG_CHANGE,
- "audit_pid=%d old=%d by auid=%u subj=%s",
- status_get->pid, old,
- loginuid, ctx);
- kfree(ctx);
- } else
- audit_log(NULL, GFP_KERNEL, AUDIT_CONFIG_CHANGE,
- "audit_pid=%d old=%d by auid=%u",
- status_get->pid, old, loginuid);
- audit_pid = status_get->pid;
+ int new_pid = status_get->pid;
+
+ if (audit_enabled != AUDIT_OFF)
+ audit_log_config_change("audit_pid", new_pid,
+ audit_pid, loginuid,
+ sid, 1);
+
+ audit_pid = new_pid;
}
if (status_get->mask & AUDIT_STATUS_RATE_LIMIT)
err = audit_set_rate_limit(status_get->rate_limit,
if (err)
break;
}
- ab = audit_log_start(NULL, GFP_KERNEL, msg_type);
- if (ab) {
- audit_log_format(ab,
- "user pid=%d uid=%u auid=%u",
- pid, uid, loginuid);
- if (sid) {
- if (selinux_sid_to_string(
- sid, &ctx, &len)) {
- audit_log_format(ab,
- " ssid=%u", sid);
- /* Maybe call audit_panic? */
- } else
- audit_log_format(ab,
- " subj=%s", ctx);
- kfree(ctx);
- }
- if (msg_type != AUDIT_USER_TTY)
- audit_log_format(ab, " msg='%.1024s'",
- (char *)data);
- else {
- int size;
-
- audit_log_format(ab, " msg=");
- size = nlmsg_len(nlh);
- audit_log_n_untrustedstring(ab, size,
- data);
- }
- audit_set_pid(ab, pid);
- audit_log_end(ab);
+ audit_log_common_recv_msg(&ab, msg_type, pid, uid,
+ loginuid, sid);
+
+ if (msg_type != AUDIT_USER_TTY)
+ audit_log_format(ab, " msg='%.1024s'",
+ (char *)data);
+ else {
+ int size;
+
+ audit_log_format(ab, " msg=");
+ size = nlmsg_len(nlh);
+ audit_log_n_untrustedstring(ab, size,
+ data);
}
+ audit_set_pid(ab, pid);
+ audit_log_end(ab);
}
break;
case AUDIT_ADD:
case AUDIT_DEL:
if (nlmsg_len(nlh) < sizeof(struct audit_rule))
return -EINVAL;
- if (audit_enabled == 2) {
- ab = audit_log_start(NULL, GFP_KERNEL,
- AUDIT_CONFIG_CHANGE);
- if (ab) {
- audit_log_format(ab,
- "pid=%d uid=%u auid=%u",
- pid, uid, loginuid);
- if (sid) {
- if (selinux_sid_to_string(
- sid, &ctx, &len)) {
- audit_log_format(ab,
- " ssid=%u", sid);
- /* Maybe call audit_panic? */
- } else
- audit_log_format(ab,
- " subj=%s", ctx);
- kfree(ctx);
- }
- audit_log_format(ab, " audit_enabled=%d res=0",
- audit_enabled);
- audit_log_end(ab);
- }
+ if (audit_enabled == AUDIT_LOCKED) {
+ audit_log_common_recv_msg(&ab, AUDIT_CONFIG_CHANGE, pid,
+ uid, loginuid, sid);
+
+ audit_log_format(ab, " audit_enabled=%d res=0",
+ audit_enabled);
+ audit_log_end(ab);
return -EPERM;
}
/* fallthrough */
case AUDIT_DEL_RULE:
if (nlmsg_len(nlh) < sizeof(struct audit_rule_data))
return -EINVAL;
- if (audit_enabled == 2) {
- ab = audit_log_start(NULL, GFP_KERNEL,
- AUDIT_CONFIG_CHANGE);
- if (ab) {
- audit_log_format(ab,
- "pid=%d uid=%u auid=%u",
- pid, uid, loginuid);
- if (sid) {
- if (selinux_sid_to_string(
- sid, &ctx, &len)) {
- audit_log_format(ab,
- " ssid=%u", sid);
- /* Maybe call audit_panic? */
- } else
- audit_log_format(ab,
- " subj=%s", ctx);
- kfree(ctx);
- }
- audit_log_format(ab, " audit_enabled=%d res=0",
- audit_enabled);
- audit_log_end(ab);
- }
+ if (audit_enabled == AUDIT_LOCKED) {
+ audit_log_common_recv_msg(&ab, AUDIT_CONFIG_CHANGE, pid,
+ uid, loginuid, sid);
+
+ audit_log_format(ab, " audit_enabled=%d res=0",
+ audit_enabled);
+ audit_log_end(ab);
return -EPERM;
}
/* fallthrough */
break;
case AUDIT_TRIM:
audit_trim_trees();
- ab = audit_log_start(NULL, GFP_KERNEL, AUDIT_CONFIG_CHANGE);
- if (!ab)
- break;
- audit_log_format(ab, "auid=%u", loginuid);
- if (sid) {
- u32 len;
- ctx = NULL;
- if (selinux_sid_to_string(sid, &ctx, &len))
- audit_log_format(ab, " ssid=%u", sid);
- else
- audit_log_format(ab, " subj=%s", ctx);
- kfree(ctx);
- }
+
+ audit_log_common_recv_msg(&ab, AUDIT_CONFIG_CHANGE, pid,
+ uid, loginuid, sid);
+
audit_log_format(ab, " op=trim res=1");
audit_log_end(ab);
break;
/* OK, here comes... */
err = audit_tag_tree(old, new);
- ab = audit_log_start(NULL, GFP_KERNEL, AUDIT_CONFIG_CHANGE);
- if (!ab) {
- kfree(old);
- kfree(new);
- break;
- }
- audit_log_format(ab, "auid=%u", loginuid);
- if (sid) {
- u32 len;
- ctx = NULL;
- if (selinux_sid_to_string(sid, &ctx, &len))
- audit_log_format(ab, " ssid=%u", sid);
- else
- audit_log_format(ab, " subj=%s", ctx);
- kfree(ctx);
- }
+ audit_log_common_recv_msg(&ab, AUDIT_CONFIG_CHANGE, pid,
+ uid, loginuid, sid);
+
audit_log_format(ab, " op=make_equiv old=");
audit_log_untrustedstring(ab, old);
audit_log_format(ab, " new=");
skb_queue_head_init(&audit_skb_queue);
audit_initialized = 1;
audit_enabled = audit_default;
+ audit_ever_enabled |= !!audit_default;
/* Register the callback with selinux. This callback will be invoked
* when a new policy is loaded. */
printk(KERN_INFO "audit: %s%s\n",
audit_default ? "enabled" : "disabled",
audit_initialized ? "" : " (after initialization)");
- if (audit_initialized)
+ if (audit_initialized) {
audit_enabled = audit_default;
+ audit_ever_enabled |= !!audit_default;
+ }
return 1;
}
{
struct audit_buffer *ab = NULL;
struct timespec t;
- unsigned int serial;
+ unsigned int uninitialized_var(serial);
int reserve;
unsigned long timeout_start = jiffies;
remove_wait_queue(&audit_backlog_wait, &wait);
continue;
}
- if (audit_rate_check())
+ if (audit_rate_check() && printk_ratelimit())
printk(KERN_WARNING
"audit: audit_backlog=%d > "
"audit_backlog_limit=%d\n",
goto out;
len = vsnprintf(skb_tail_pointer(skb), avail, fmt, args2);
}
+ va_end(args2);
if (len > 0)
skb_put(skb, len);
out:
}
/**
+ * audit_string_contains_control - does a string need to be logged in hex
+ * @string - string to be checked
+ * @len - max length of the string to check
+ */
+int audit_string_contains_control(const char *string, size_t len)
+{
+ const unsigned char *p;
+ for (p = string; p < (const unsigned char *)string + len && *p; p++) {
+ if (*p == '"' || *p < 0x21 || *p > 0x7f)
+ return 1;
+ }
+ return 0;
+}
+
+/**
* audit_log_n_untrustedstring - log a string that may contain random characters
* @ab: audit_buffer
* @len: lenth of string (not including trailing null)
* The caller specifies the number of characters in the string to log, which may
* or may not be the entire string.
*/
-const char *audit_log_n_untrustedstring(struct audit_buffer *ab, size_t len,
- const char *string)
+void audit_log_n_untrustedstring(struct audit_buffer *ab, size_t len,
+ const char *string)
{
- const unsigned char *p;
-
- for (p = string; p < (const unsigned char *)string + len && *p; p++) {
- if (*p == '"' || *p < 0x21 || *p > 0x7f) {
- audit_log_hex(ab, string, len);
- return string + len + 1;
- }
- }
- audit_log_n_string(ab, len, string);
- return p + 1;
+ if (audit_string_contains_control(string, len))
+ audit_log_hex(ab, string, len);
+ else
+ audit_log_n_string(ab, len, string);
}
/**
* Same as audit_log_n_untrustedstring(), except that strlen is used to
* determine string length.
*/
-const char *audit_log_untrustedstring(struct audit_buffer *ab, const char *string)
+void audit_log_untrustedstring(struct audit_buffer *ab, const char *string)
{
- return audit_log_n_untrustedstring(ab, strlen(string), string);
+ audit_log_n_untrustedstring(ab, strlen(string), string);
}
/* This is a helper-function to print the escaped d_path */
skb_queue_tail(&audit_skb_queue, ab->skb);
ab->skb = NULL;
wake_up_interruptible(&kauditd_wait);
+ } else if (printk_ratelimit()) {
+ struct nlmsghdr *nlh = nlmsg_hdr(ab->skb);
+ printk(KERN_NOTICE "type=%d %s\n", nlh->nlmsg_type, ab->skb->data + NLMSG_SPACE(0));
} else {
- printk(KERN_NOTICE "%s\n", ab->skb->data + NLMSG_SPACE(0));
+ audit_log_lost("printk limit exceeded\n");
}
}
audit_buffer_free(ab);
/* Inotify events we care about. */
#define AUDIT_IN_WATCH IN_MOVE|IN_CREATE|IN_DELETE|IN_DELETE_SELF|IN_MOVE_SELF
+extern int audit_enabled;
+
void audit_free_parent(struct inotify_watch *i_watch)
{
struct audit_parent *parent;
struct audit_watch *owatch, *nwatch, *nextw;
struct audit_krule *r, *nextr;
struct audit_entry *oentry, *nentry;
- struct audit_buffer *ab;
mutex_lock(&audit_filter_mutex);
list_for_each_entry_safe(owatch, nextw, &parent->watches, wlist) {
call_rcu(&oentry->rcu, audit_free_rule_rcu);
}
- ab = audit_log_start(NULL, GFP_KERNEL, AUDIT_CONFIG_CHANGE);
- audit_log_format(ab, "op=updated rules specifying path=");
- audit_log_untrustedstring(ab, owatch->path);
- audit_log_format(ab, " with dev=%u ino=%lu\n", dev, ino);
- audit_log_format(ab, " list=%d res=1", r->listnr);
- audit_log_end(ab);
-
+ if (audit_enabled) {
+ struct audit_buffer *ab;
+ ab = audit_log_start(NULL, GFP_KERNEL,
+ AUDIT_CONFIG_CHANGE);
+ audit_log_format(ab,
+ "op=updated rules specifying path=");
+ audit_log_untrustedstring(ab, owatch->path);
+ audit_log_format(ab, " with dev=%u ino=%lu\n",
+ dev, ino);
+ audit_log_format(ab, " list=%d res=1", r->listnr);
+ audit_log_end(ab);
+ }
audit_remove_watch(owatch);
goto add_watch_to_parent; /* event applies to a single watch */
}
struct audit_watch *w, *nextw;
struct audit_krule *r, *nextr;
struct audit_entry *e;
- struct audit_buffer *ab;
mutex_lock(&audit_filter_mutex);
parent->flags |= AUDIT_PARENT_INVALID;
list_for_each_entry_safe(w, nextw, &parent->watches, wlist) {
list_for_each_entry_safe(r, nextr, &w->rules, rlist) {
e = container_of(r, struct audit_entry, rule);
-
- ab = audit_log_start(NULL, GFP_KERNEL, AUDIT_CONFIG_CHANGE);
- audit_log_format(ab, "op=remove rule path=");
- audit_log_untrustedstring(ab, w->path);
- if (r->filterkey) {
- audit_log_format(ab, " key=");
- audit_log_untrustedstring(ab, r->filterkey);
- } else
- audit_log_format(ab, " key=(null)");
- audit_log_format(ab, " list=%d res=1", r->listnr);
- audit_log_end(ab);
-
+ if (audit_enabled) {
+ struct audit_buffer *ab;
+ ab = audit_log_start(NULL, GFP_KERNEL,
+ AUDIT_CONFIG_CHANGE);
+ audit_log_format(ab, "op=remove rule path=");
+ audit_log_untrustedstring(ab, w->path);
+ if (r->filterkey) {
+ audit_log_format(ab, " key=");
+ audit_log_untrustedstring(ab,
+ r->filterkey);
+ } else
+ audit_log_format(ab, " key=(null)");
+ audit_log_format(ab, " list=%d res=1",
+ r->listnr);
+ audit_log_end(ab);
+ }
list_del(&r->rlist);
list_del_rcu(&e->list);
call_rcu(&e->rcu, audit_free_rule_rcu);
{
struct audit_buffer *ab;
+ if (!audit_enabled)
+ return;
+
ab = audit_log_start(NULL, GFP_KERNEL, AUDIT_CONFIG_CHANGE);
if (!ab)
return;
#include "audit.h"
extern struct list_head audit_filter_list[];
+extern int audit_ever_enabled;
/* AUDIT_NAMES is the number of slots we reserve in the audit_context
* for saving names from getname(). */
/* Indicates that audit should log the full pathname. */
#define AUDIT_NAME_FULL -1
+/* no execve audit message should be longer than this (userspace limits) */
+#define MAX_EXECVE_AUDIT_LEN 7500
+
/* number of audit rules */
int audit_n_rules;
struct audit_aux_data_pids {
struct audit_aux_data d;
pid_t target_pid[AUDIT_AUX_PIDS];
+ uid_t target_auid[AUDIT_AUX_PIDS];
+ uid_t target_uid[AUDIT_AUX_PIDS];
+ unsigned int target_sessionid[AUDIT_AUX_PIDS];
u32 target_sid[AUDIT_AUX_PIDS];
+ char target_comm[AUDIT_AUX_PIDS][TASK_COMM_LEN];
int pid_count;
};
enum audit_state state;
unsigned int serial; /* serial number for record */
struct timespec ctime; /* time of syscall entry */
- uid_t loginuid; /* login uid (identity) */
int major; /* syscall number */
unsigned long argv[4]; /* syscall arguments */
int return_valid; /* return code is valid */
int arch;
pid_t target_pid;
+ uid_t target_auid;
+ uid_t target_uid;
+ unsigned int target_sessionid;
u32 target_sid;
+ char target_comm[TASK_COMM_LEN];
struct audit_tree_refs *trees, *first_trees;
int tree_count;
case AUDIT_LOGINUID:
result = 0;
if (ctx)
- result = audit_comparator(ctx->loginuid, f->op, f->val);
+ result = audit_comparator(tsk->loginuid, f->op, f->val);
break;
case AUDIT_SUBJ_USER:
case AUDIT_SUBJ_ROLE:
if (likely(!context))
return NULL;
context->return_valid = return_valid;
- context->return_code = return_code;
+
+ /*
+ * we need to fix up the return code in the audit logs if the actual
+ * return codes are later going to be fixed up by the arch specific
+ * signal handlers
+ *
+ * This is actually a test for:
+ * (rc == ERESTARTSYS ) || (rc == ERESTARTNOINTR) ||
+ * (rc == ERESTARTNOHAND) || (rc == ERESTART_RESTARTBLOCK)
+ *
+ * but is faster than a bunch of ||
+ */
+ if (unlikely(return_code <= -ERESTARTSYS) &&
+ (return_code >= -ERESTART_RESTARTBLOCK) &&
+ (return_code != -ENOIOCTLCMD))
+ context->return_code = -EINTR;
+ else
+ context->return_code = return_code;
if (context->in_syscall && !context->dummy && !context->auditable) {
enum audit_state state;
static inline void audit_zero_context(struct audit_context *context,
enum audit_state state)
{
- uid_t loginuid = context->loginuid;
-
memset(context, 0, sizeof(*context));
context->state = state;
- context->loginuid = loginuid;
}
static inline struct audit_context *audit_alloc_context(enum audit_state state)
struct audit_context *context;
enum audit_state state;
- if (likely(!audit_enabled))
+ if (likely(!audit_ever_enabled))
return 0; /* Return if not auditing. */
state = audit_filter_task(tsk);
return -ENOMEM;
}
- /* Preserve login uid */
- context->loginuid = -1;
- if (current->audit_context)
- context->loginuid = current->audit_context->loginuid;
-
tsk->audit_context = context;
set_tsk_thread_flag(tsk, TIF_SYSCALL_AUDIT);
return 0;
}
static int audit_log_pid_context(struct audit_context *context, pid_t pid,
- u32 sid)
+ uid_t auid, uid_t uid, unsigned int sessionid,
+ u32 sid, char *comm)
{
struct audit_buffer *ab;
char *s = NULL;
ab = audit_log_start(context, GFP_KERNEL, AUDIT_OBJ_PID);
if (!ab)
- return 1;
+ return rc;
+ audit_log_format(ab, "opid=%d oauid=%d ouid=%d oses=%d", pid, auid,
+ uid, sessionid);
if (selinux_sid_to_string(sid, &s, &len)) {
- audit_log_format(ab, "opid=%d obj=(none)", pid);
+ audit_log_format(ab, " obj=(none)");
rc = 1;
} else
- audit_log_format(ab, "opid=%d obj=%s", pid, s);
+ audit_log_format(ab, " obj=%s", s);
+ audit_log_format(ab, " ocomm=");
+ audit_log_untrustedstring(ab, comm);
audit_log_end(ab);
kfree(s);
return rc;
}
-static void audit_log_execve_info(struct audit_buffer *ab,
- struct audit_aux_data_execve *axi)
+/*
+ * to_send and len_sent accounting are very loose estimates. We aren't
+ * really worried about a hard cap to MAX_EXECVE_AUDIT_LEN so much as being
+ * within about 500 bytes (next page boundry)
+ *
+ * why snprintf? an int is up to 12 digits long. if we just assumed when
+ * logging that a[%d]= was going to be 16 characters long we would be wasting
+ * space in every audit message. In one 7500 byte message we can log up to
+ * about 1000 min size arguments. That comes down to about 50% waste of space
+ * if we didn't do the snprintf to find out how long arg_num_len was.
+ */
+static int audit_log_single_execve_arg(struct audit_context *context,
+ struct audit_buffer **ab,
+ int arg_num,
+ size_t *len_sent,
+ const char __user *p,
+ char *buf)
{
- int i;
- long len, ret;
- const char __user *p;
- char *buf;
+ char arg_num_len_buf[12];
+ const char __user *tmp_p = p;
+ /* how many digits are in arg_num? 3 is the length of a=\n */
+ size_t arg_num_len = snprintf(arg_num_len_buf, 12, "%d", arg_num) + 3;
+ size_t len, len_left, to_send;
+ size_t max_execve_audit_len = MAX_EXECVE_AUDIT_LEN;
+ unsigned int i, has_cntl = 0, too_long = 0;
+ int ret;
+
+ /* strnlen_user includes the null we don't want to send */
+ len_left = len = strnlen_user(p, MAX_ARG_STRLEN) - 1;
- if (axi->mm != current->mm)
- return; /* execve failed, no additional info */
-
- p = (const char __user *)axi->mm->arg_start;
+ /*
+ * We just created this mm, if we can't find the strings
+ * we just copied into it something is _very_ wrong. Similar
+ * for strings that are too long, we should not have created
+ * any.
+ */
+ if (unlikely((len = -1) || len > MAX_ARG_STRLEN - 1)) {
+ WARN_ON(1);
+ send_sig(SIGKILL, current, 0);
+ }
- for (i = 0; i < axi->argc; i++, p += len) {
- len = strnlen_user(p, MAX_ARG_STRLEN);
+ /* walk the whole argument looking for non-ascii chars */
+ do {
+ if (len_left > MAX_EXECVE_AUDIT_LEN)
+ to_send = MAX_EXECVE_AUDIT_LEN;
+ else
+ to_send = len_left;
+ ret = copy_from_user(buf, tmp_p, to_send);
/*
- * We just created this mm, if we can't find the strings
- * we just copied into it something is _very_ wrong. Similar
- * for strings that are too long, we should not have created
- * any.
+ * There is no reason for this copy to be short. We just
+ * copied them here, and the mm hasn't been exposed to user-
+ * space yet.
*/
- if (!len || len > MAX_ARG_STRLEN) {
+ if (ret) {
WARN_ON(1);
send_sig(SIGKILL, current, 0);
}
-
- buf = kmalloc(len, GFP_KERNEL);
- if (!buf) {
- audit_panic("out of memory for argv string\n");
+ buf[to_send] = '\0';
+ has_cntl = audit_string_contains_control(buf, to_send);
+ if (has_cntl) {
+ /*
+ * hex messages get logged as 2 bytes, so we can only
+ * send half as much in each message
+ */
+ max_execve_audit_len = MAX_EXECVE_AUDIT_LEN / 2;
break;
}
+ len_left -= to_send;
+ tmp_p += to_send;
+ } while (len_left > 0);
+
+ len_left = len;
+
+ if (len > max_execve_audit_len)
+ too_long = 1;
+
+ /* rewalk the argument actually logging the message */
+ for (i = 0; len_left > 0; i++) {
+ int room_left;
+
+ if (len_left > max_execve_audit_len)
+ to_send = max_execve_audit_len;
+ else
+ to_send = len_left;
+
+ /* do we have space left to send this argument in this ab? */
+ room_left = MAX_EXECVE_AUDIT_LEN - arg_num_len - *len_sent;
+ if (has_cntl)
+ room_left -= (to_send * 2);
+ else
+ room_left -= to_send;
+ if (room_left < 0) {
+ *len_sent = 0;
+ audit_log_end(*ab);
+ *ab = audit_log_start(context, GFP_KERNEL, AUDIT_EXECVE);
+ if (!*ab)
+ return 0;
+ }
- ret = copy_from_user(buf, p, len);
/*
- * There is no reason for this copy to be short. We just
- * copied them here, and the mm hasn't been exposed to user-
- * space yet.
+ * first record needs to say how long the original string was
+ * so we can be sure nothing was lost.
+ */
+ if ((i == 0) && (too_long))
+ audit_log_format(*ab, "a%d_len=%ld ", arg_num,
+ has_cntl ? 2*len : len);
+
+ /*
+ * normally arguments are small enough to fit and we already
+ * filled buf above when we checked for control characters
+ * so don't bother with another copy_from_user
*/
+ if (len >= max_execve_audit_len)
+ ret = copy_from_user(buf, p, to_send);
+ else
+ ret = 0;
if (ret) {
WARN_ON(1);
send_sig(SIGKILL, current, 0);
}
+ buf[to_send] = '\0';
+
+ /* actually log it */
+ audit_log_format(*ab, "a%d", arg_num);
+ if (too_long)
+ audit_log_format(*ab, "[%d]", i);
+ audit_log_format(*ab, "=");
+ if (has_cntl)
+ audit_log_hex(*ab, buf, to_send);
+ else
+ audit_log_format(*ab, "\"%s\"", buf);
+ audit_log_format(*ab, "\n");
+
+ p += to_send;
+ len_left -= to_send;
+ *len_sent += arg_num_len;
+ if (has_cntl)
+ *len_sent += to_send * 2;
+ else
+ *len_sent += to_send;
+ }
+ /* include the null we didn't log */
+ return len + 1;
+}
- audit_log_format(ab, "a%d=", i);
- audit_log_untrustedstring(ab, buf);
- audit_log_format(ab, "\n");
+static void audit_log_execve_info(struct audit_context *context,
+ struct audit_buffer **ab,
+ struct audit_aux_data_execve *axi)
+{
+ int i;
+ size_t len, len_sent = 0;
+ const char __user *p;
+ char *buf;
+
+ if (axi->mm != current->mm)
+ return; /* execve failed, no additional info */
+
+ p = (const char __user *)axi->mm->arg_start;
+
+ audit_log_format(*ab, "argc=%d ", axi->argc);
+
+ /*
+ * we need some kernel buffer to hold the userspace args. Just
+ * allocate one big one rather than allocating one of the right size
+ * for every single argument inside audit_log_single_execve_arg()
+ * should be <8k allocation so should be pretty safe.
+ */
+ buf = kmalloc(MAX_EXECVE_AUDIT_LEN + 1, GFP_KERNEL);
+ if (!buf) {
+ audit_panic("out of memory for argv string\n");
+ return;
+ }
- kfree(buf);
+ for (i = 0; i < axi->argc; i++) {
+ len = audit_log_single_execve_arg(context, ab, i,
+ &len_sent, p, buf);
+ if (len <= 0)
+ break;
+ p += len;
}
+ kfree(buf);
}
static void audit_log_exit(struct audit_context *context, struct task_struct *tsk)
" a0=%lx a1=%lx a2=%lx a3=%lx items=%d"
" ppid=%d pid=%d auid=%u uid=%u gid=%u"
" euid=%u suid=%u fsuid=%u"
- " egid=%u sgid=%u fsgid=%u tty=%s",
+ " egid=%u sgid=%u fsgid=%u tty=%s ses=%u",
context->argv[0],
context->argv[1],
context->argv[2],
context->name_count,
context->ppid,
context->pid,
- context->loginuid,
+ tsk->loginuid,
context->uid,
context->gid,
context->euid, context->suid, context->fsuid,
- context->egid, context->sgid, context->fsgid, tty);
+ context->egid, context->sgid, context->fsgid, tty,
+ tsk->sessionid);
mutex_unlock(&tty_mutex);
case AUDIT_EXECVE: {
struct audit_aux_data_execve *axi = (void *)aux;
- audit_log_execve_info(ab, axi);
+ audit_log_execve_info(context, &ab, axi);
break; }
case AUDIT_SOCKETCALL: {
for (i = 0; i < axs->pid_count; i++)
if (audit_log_pid_context(context, axs->target_pid[i],
- axs->target_sid[i]))
+ axs->target_auid[i],
+ axs->target_uid[i],
+ axs->target_sessionid[i],
+ axs->target_sid[i],
+ axs->target_comm[i]))
call_panic = 1;
}
if (context->target_pid &&
audit_log_pid_context(context, context->target_pid,
- context->target_sid))
+ context->target_auid, context->target_uid,
+ context->target_sessionid,
+ context->target_sid, context->target_comm))
call_panic = 1;
if (context->pwd && context->pwdmnt) {
audit_log_end(ab);
}
+
+ /* Send end of event record to help user space know we are finished */
+ ab = audit_log_start(context, GFP_KERNEL, AUDIT_EOE);
+ if (ab)
+ audit_log_end(ab);
if (call_panic)
audit_panic("error converting sid to string");
}
ctx->auditable = 1;
}
+/* global counter which is incremented every time something logs in */
+static atomic_t session_id = ATOMIC_INIT(0);
+
/**
* audit_set_loginuid - set a task's audit_context loginuid
* @task: task whose audit context is being modified
*/
int audit_set_loginuid(struct task_struct *task, uid_t loginuid)
{
+ unsigned int sessionid = atomic_inc_return(&session_id);
struct audit_context *context = task->audit_context;
- if (context) {
- /* Only log if audit is enabled */
- if (context->in_syscall) {
- struct audit_buffer *ab;
-
- ab = audit_log_start(NULL, GFP_KERNEL, AUDIT_LOGIN);
- if (ab) {
- audit_log_format(ab, "login pid=%d uid=%u "
- "old auid=%u new auid=%u",
- task->pid, task->uid,
- context->loginuid, loginuid);
- audit_log_end(ab);
- }
+ if (context && context->in_syscall) {
+ struct audit_buffer *ab;
+
+ ab = audit_log_start(NULL, GFP_KERNEL, AUDIT_LOGIN);
+ if (ab) {
+ audit_log_format(ab, "login pid=%d uid=%u "
+ "old auid=%u new auid=%u"
+ " old ses=%u new ses=%u",
+ task->pid, task->uid,
+ task->loginuid, loginuid,
+ task->sessionid, sessionid);
+ audit_log_end(ab);
}
- context->loginuid = loginuid;
}
+ task->sessionid = sessionid;
+ task->loginuid = loginuid;
return 0;
}
/**
- * audit_get_loginuid - get the loginuid for an audit_context
- * @ctx: the audit_context
- *
- * Returns the context's loginuid or -1 if @ctx is NULL.
- */
-uid_t audit_get_loginuid(struct audit_context *ctx)
-{
- return ctx ? ctx->loginuid : -1;
-}
-
-EXPORT_SYMBOL(audit_get_loginuid);
-
-/**
* __audit_mq_open - record audit data for a POSIX MQ open
* @oflag: open flag
* @mode: mode bits
return 0;
}
-int audit_argv_kb = 32;
-
int audit_bprm(struct linux_binprm *bprm)
{
struct audit_aux_data_execve *ax;
if (likely(!audit_enabled || !context || context->dummy))
return 0;
- /*
- * Even though the stack code doesn't limit the arg+env size any more,
- * the audit code requires that _all_ arguments be logged in a single
- * netlink skb. Hence cap it :-(
- */
- if (bprm->argv_len > (audit_argv_kb << 10))
- return -E2BIG;
-
ax = kmalloc(sizeof(*ax), GFP_KERNEL);
if (!ax)
return -ENOMEM;
struct audit_context *context = current->audit_context;
context->target_pid = t->pid;
+ context->target_auid = audit_get_loginuid(t);
+ context->target_uid = t->uid;
+ context->target_sessionid = audit_get_sessionid(t);
selinux_get_task_sid(t, &context->target_sid);
+ memcpy(context->target_comm, t->comm, TASK_COMM_LEN);
}
/**
if (audit_pid && t->tgid == audit_pid) {
if (sig == SIGTERM || sig == SIGHUP || sig == SIGUSR1) {
audit_sig_pid = tsk->pid;
- if (ctx)
- audit_sig_uid = ctx->loginuid;
+ if (tsk->loginuid != -1)
+ audit_sig_uid = tsk->loginuid;
else
audit_sig_uid = tsk->uid;
selinux_get_task_sid(tsk, &audit_sig_sid);
* in audit_context */
if (!ctx->target_pid) {
ctx->target_pid = t->tgid;
+ ctx->target_auid = audit_get_loginuid(t);
+ ctx->target_uid = t->uid;
+ ctx->target_sessionid = audit_get_sessionid(t);
selinux_get_task_sid(t, &ctx->target_sid);
+ memcpy(ctx->target_comm, t->comm, TASK_COMM_LEN);
return 0;
}
BUG_ON(axp->pid_count >= AUDIT_AUX_PIDS);
axp->target_pid[axp->pid_count] = t->tgid;
+ axp->target_auid[axp->pid_count] = audit_get_loginuid(t);
+ axp->target_uid[axp->pid_count] = t->uid;
+ axp->target_sessionid[axp->pid_count] = audit_get_sessionid(t);
selinux_get_task_sid(t, &axp->target_sid[axp->pid_count]);
+ memcpy(axp->target_comm[axp->pid_count], t->comm, TASK_COMM_LEN);
axp->pid_count++;
return 0;
{
struct audit_buffer *ab;
u32 sid;
+ uid_t auid = audit_get_loginuid(current);
+ unsigned int sessionid = audit_get_sessionid(current);
if (!audit_enabled)
return;
return;
ab = audit_log_start(NULL, GFP_KERNEL, AUDIT_ANOM_ABEND);
- audit_log_format(ab, "auid=%u uid=%u gid=%u",
- audit_get_loginuid(current->audit_context),
- current->uid, current->gid);
+ audit_log_format(ab, "auid=%u uid=%u gid=%u ses=%u",
+ auid, current->uid, current->gid, sessionid);
selinux_get_task_sid(current, &sid);
if (sid) {
char *ctx = NULL;
/* Optional priority inheritance state: */
struct futex_pi_state *pi_state;
struct task_struct *task;
+
+ /* Bitset for the optional bitmasked wakeup */
+ u32 bitset;
};
/*
* to this virtual address:
*/
static int futex_wake(u32 __user *uaddr, struct rw_semaphore *fshared,
- int nr_wake)
+ int nr_wake, u32 bitset)
{
struct futex_hash_bucket *hb;
struct futex_q *this, *next;
union futex_key key;
int ret;
+ if (!bitset)
+ return -EINVAL;
+
futex_lock_mm(fshared);
ret = get_futex_key(uaddr, fshared, &key);
ret = -EINVAL;
break;
}
+
+ /* Check if one of the bits is set in both bitsets */
+ if (!(this->bitset & bitset))
+ continue;
+
wake_futex(this);
if (++ret >= nr_wake)
break;
static long futex_wait_restart(struct restart_block *restart);
static int futex_wait(u32 __user *uaddr, struct rw_semaphore *fshared,
- u32 val, ktime_t *abs_time)
+ u32 val, ktime_t *abs_time, u32 bitset)
{
struct task_struct *curr = current;
DECLARE_WAITQUEUE(wait, curr);
struct hrtimer_sleeper t;
int rem = 0;
+ if (!bitset)
+ return -EINVAL;
+
q.pi_state = NULL;
+ q.bitset = bitset;
retry:
futex_lock_mm(fshared);
t.timer.expires = *abs_time;
hrtimer_start(&t.timer, t.timer.expires, HRTIMER_MODE_ABS);
+ if (!hrtimer_active(&t.timer))
+ t.task = NULL;
/*
* the timer could have already expired, in which
restart->futex.uaddr = (u32 *)uaddr;
restart->futex.val = val;
restart->futex.time = abs_time->tv64;
+ restart->futex.bitset = bitset;
restart->futex.flags = 0;
if (fshared)
restart->fn = do_no_restart_syscall;
if (restart->futex.flags & FLAGS_SHARED)
fshared = ¤t->mm->mmap_sem;
- return (long)futex_wait(uaddr, fshared, restart->futex.val, &t);
+ return (long)futex_wait(uaddr, fshared, restart->futex.val, &t,
+ restart->futex.bitset);
}
owner = rt_mutex_owner(&q.pi_state->pi_mutex);
res = fixup_pi_state_owner(uaddr, &q, owner);
- WARN_ON(rt_mutex_owner(&q.pi_state->pi_mutex) !=
- owner);
-
/* propagate -EFAULT, if the fixup failed */
if (res)
ret = res;
* PI futexes happens in exit_pi_state():
*/
if (!pi && (uval & FUTEX_WAITERS))
- futex_wake(uaddr, &curr->mm->mmap_sem, 1);
+ futex_wake(uaddr, &curr->mm->mmap_sem, 1,
+ FUTEX_BITSET_MATCH_ANY);
}
return 0;
}
switch (cmd) {
case FUTEX_WAIT:
- ret = futex_wait(uaddr, fshared, val, timeout);
+ val3 = FUTEX_BITSET_MATCH_ANY;
+ case FUTEX_WAIT_BITSET:
+ ret = futex_wait(uaddr, fshared, val, timeout, val3);
break;
case FUTEX_WAKE:
- ret = futex_wake(uaddr, fshared, val);
+ val3 = FUTEX_BITSET_MATCH_ANY;
+ case FUTEX_WAKE_BITSET:
+ ret = futex_wake(uaddr, fshared, val, val3);
break;
case FUTEX_FD:
/* non-zero val means F_SETOWN(getpid()) & F_SETSIG(val) */
u32 val2 = 0;
int cmd = op & FUTEX_CMD_MASK;
- if (utime && (cmd == FUTEX_WAIT || cmd == FUTEX_LOCK_PI)) {
+ if (utime && (cmd == FUTEX_WAIT || cmd == FUTEX_LOCK_PI ||
+ cmd == FUTEX_WAIT_BITSET)) {
if (copy_from_user(&ts, utime, sizeof(ts)) != 0)
return -EFAULT;
if (!timespec_valid(&ts))
int val2 = 0;
int cmd = op & FUTEX_CMD_MASK;
- if (utime && (cmd == FUTEX_WAIT || cmd == FUTEX_LOCK_PI)) {
+ if (utime && (cmd == FUTEX_WAIT || cmd == FUTEX_LOCK_PI ||
+ cmd == FUTEX_WAIT_BITSET)) {
if (get_compat_timespec(&ts, utime))
return -EFAULT;
if (!timespec_valid(&ts))
} while (t->task && !signal_pending(current));
+ __set_current_state(TASK_RUNNING);
+
return t->task == NULL;
}
extern int compat_log;
extern int maps_protect;
extern int sysctl_stat_interval;
-extern int audit_argv_kb;
extern int latencytop_enabled;
/* Constants used for minimum and maximum */
.mode = 0644,
.proc_handler = &proc_dointvec,
},
-#ifdef CONFIG_AUDITSYSCALL
- {
- .ctl_name = CTL_UNNUMBERED,
- .procname = "audit_argv_kb",
- .data = &audit_argv_kb,
- .maxlen = sizeof(int),
- .mode = 0644,
- .proc_handler = &proc_dointvec,
- },
-#endif
{
.ctl_name = KERN_CORE_PATTERN,
.procname = "core_pattern",
write_seqlock_irq(&xtime_lock);
wall_to_monotonic.tv_sec -= sys_tz.tz_minuteswest * 60;
xtime.tv_sec += sys_tz.tz_minuteswest * 60;
+ update_xtime_cache(0);
write_sequnlock_irq(&xtime_lock);
clock_was_set();
}
cpu_clear(cpu, nohz_cpu_mask);
now = ktime_get();
+ ts->idle_waketime = now;
local_irq_save(flags);
tick_do_update_jiffies64(now);
* Cancel the scheduled timer and restore the tick
*/
ts->tick_stopped = 0;
+ ts->idle_exittime = now;
hrtimer_cancel(&ts->sched_timer);
ts->sched_timer.expires = ts->idle_tick;
static unsigned long total_sleep_time; /* seconds */
static struct timespec xtime_cache __attribute__ ((aligned (16)));
-static inline void update_xtime_cache(u64 nsec)
+void update_xtime_cache(u64 nsec)
{
xtime_cache = xtime;
timespec_add_ns(&xtime_cache, nsec);
set_normalized_timespec(&xtime, sec, nsec);
set_normalized_timespec(&wall_to_monotonic, wtm_sec, wtm_nsec);
+ update_xtime_cache(0);
clock->error = 0;
ntp_clear();
xtime.tv_nsec = 0;
set_normalized_timespec(&wall_to_monotonic,
-xtime.tv_sec, -xtime.tv_nsec);
+ update_xtime_cache(0);
total_sleep_time = 0;
-
write_sequnlock_irqrestore(&xtime_lock, flags);
}
}
/* Make sure that we have the correct xtime reference */
timespec_add_ns(&xtime, timekeeping_suspend_nsecs);
+ update_xtime_cache(0);
/* re-base the last cycle value */
clock->cycle_last = clocksource_read(clock);
clock->error = 0;
P(idle_calls);
P(idle_sleeps);
P_ns(idle_entrytime);
+ P_ns(idle_waketime);
+ P_ns(idle_exittime);
P_ns(idle_sleeptime);
P(last_jiffies);
P(next_jiffies);
select STACKTRACE
select SCHEDSTATS
select SCHED_DEBUG
- depends on X86 || X86_64
+ depends on HAVE_LATENCYTOP_SUPPORT
help
Enable this option if you want to use the LatencyTOP tool
to find out which userspace is blocking on what kernel operations.
printk(KERN_INFO "device %s %s promiscuous mode\n",
dev->name, (dev->flags & IFF_PROMISC) ? "entered" :
"left");
- audit_log(current->audit_context, GFP_ATOMIC,
- AUDIT_ANOM_PROMISCUOUS,
- "dev=%s prom=%d old_prom=%d auid=%u",
- dev->name, (dev->flags & IFF_PROMISC),
- (old_flags & IFF_PROMISC),
- audit_get_loginuid(current->audit_context));
+ if (audit_enabled)
+ audit_log(current->audit_context, GFP_ATOMIC,
+ AUDIT_ANOM_PROMISCUOUS,
+ "dev=%s prom=%d old_prom=%d auid=%u uid=%u gid=%u ses=%u",
+ dev->name, (dev->flags & IFF_PROMISC),
+ (old_flags & IFF_PROMISC),
+ audit_get_loginuid(current),
+ current->uid, current->gid,
+ audit_get_sessionid(current));
if (dev->change_rx_flags)
dev->change_rx_flags(dev, IFF_PROMISC);
err = xfrm_state_update(x);
xfrm_audit_state_add(x, err ? 0 : 1,
- audit_get_loginuid(current->audit_context), 0);
+ audit_get_loginuid(current), 0);
if (err < 0) {
x->km.state = XFRM_STATE_DEAD;
km_state_notify(x, &c);
out:
xfrm_audit_state_delete(x, err ? 0 : 1,
- audit_get_loginuid(current->audit_context), 0);
+ audit_get_loginuid(current), 0);
xfrm_state_put(x);
return err;
if (proto == 0)
return -EINVAL;
- audit_info.loginuid = audit_get_loginuid(current->audit_context);
+ audit_info.loginuid = audit_get_loginuid(current);
audit_info.secid = 0;
err = xfrm_state_flush(proto, &audit_info);
if (err)
hdr->sadb_msg_type != SADB_X_SPDUPDATE);
xfrm_audit_policy_add(xp, err ? 0 : 1,
- audit_get_loginuid(current->audit_context), 0);
+ audit_get_loginuid(current), 0);
if (err)
goto out;
return -ENOENT;
xfrm_audit_policy_delete(xp, err ? 0 : 1,
- audit_get_loginuid(current->audit_context), 0);
+ audit_get_loginuid(current), 0);
if (err)
goto out;
if (delete) {
xfrm_audit_policy_delete(xp, err ? 0 : 1,
- audit_get_loginuid(current->audit_context), 0);
+ audit_get_loginuid(current), 0);
if (err)
goto out;
struct xfrm_audit audit_info;
int err;
- audit_info.loginuid = audit_get_loginuid(current->audit_context);
+ audit_info.loginuid = audit_get_loginuid(current);
audit_info.secid = 0;
err = xfrm_policy_flush(XFRM_POLICY_TYPE_MAIN, &audit_info);
if (err)
NETLINK_CB(skb).pid = nlk->pid;
NETLINK_CB(skb).dst_group = dst_group;
- NETLINK_CB(skb).loginuid = audit_get_loginuid(current->audit_context);
+ NETLINK_CB(skb).loginuid = audit_get_loginuid(current);
selinux_get_task_sid(current, &(NETLINK_CB(skb).sid));
memcpy(NETLINK_CREDS(skb), &siocb->scm->creds, sizeof(struct ucred));
km_state_expired(x, 1, 0);
xfrm_audit_state_delete(x, err ? 0 : 1,
- audit_get_loginuid(current->audit_context), 0);
+ audit_get_loginuid(current), 0);
out:
spin_unlock(&x->lock);
if (length)
goto out;
audit_log(current->audit_context, GFP_KERNEL, AUDIT_MAC_STATUS,
- "enforcing=%d old_enforcing=%d auid=%u", new_value,
- selinux_enforcing,
- audit_get_loginuid(current->audit_context));
+ "enforcing=%d old_enforcing=%d auid=%u ses=%u",
+ new_value, selinux_enforcing,
+ audit_get_loginuid(current),
+ audit_get_sessionid(current));
selinux_enforcing = new_value;
if (selinux_enforcing)
avc_ss_reset(0);
if (length < 0)
goto out;
audit_log(current->audit_context, GFP_KERNEL, AUDIT_MAC_STATUS,
- "selinux=0 auid=%u",
- audit_get_loginuid(current->audit_context));
+ "selinux=0 auid=%u ses=%u",
+ audit_get_loginuid(current),
+ audit_get_sessionid(current));
}
length = count;
(security_get_allow_unknown() ? "allow" : "deny")));
audit_log(current->audit_context, GFP_KERNEL, AUDIT_MAC_POLICY_LOAD,
- "policy loaded auid=%u",
- audit_get_loginuid(current->audit_context));
+ "policy loaded auid=%u ses=%u",
+ audit_get_loginuid(current),
+ audit_get_sessionid(current));
out:
mutex_unlock(&sel_mutex);
vfree(data);
if (!!values[i] != policydb.bool_val_to_struct[i]->state) {
audit_log(current->audit_context, GFP_ATOMIC,
AUDIT_MAC_CONFIG_CHANGE,
- "bool=%s val=%d old_val=%d auid=%u",
+ "bool=%s val=%d old_val=%d auid=%u ses=%u",
policydb.p_bool_val_to_name[i],
!!values[i],
policydb.bool_val_to_struct[i]->state,
- audit_get_loginuid(current->audit_context));
+ audit_get_loginuid(current),
+ audit_get_sessionid(current));
}
if (values[i]) {
policydb.bool_val_to_struct[i]->state = 1;
projects / platform / adaptation / renesas_rcar / renesas_kernel.git / commitdiff