mips: sync asm/mipsregs.h with Linux 5.7
authorDaniel Schwierzeck <daniel.schwierzeck@gmail.com>
Sat, 11 Jul 2020 23:46:18 +0000 (01:46 +0200)
committerDaniel Schwierzeck <daniel.schwierzeck@gmail.com>
Sat, 18 Jul 2020 12:23:25 +0000 (14:23 +0200)
Sync asm/mipsregs.h with Linux 5.7. Also replace the custom
symbols EBASE_CPUNUM and EBASE_WG with the according symbols
from Linux.

Signed-off-by: Daniel Schwierzeck <daniel.schwierzeck@gmail.com>
Reviewed-by: Stefan Roese <sr@denx.de>
Tested-by: Stefan Roese <sr@denx.de>
arch/mips/cpu/start.S
arch/mips/include/asm/compiler.h [new file with mode: 0644]
arch/mips/include/asm/isa-rev.h [new file with mode: 0644]
arch/mips/include/asm/mipsregs.h
arch/mips/lib/traps.c

index 0c30303..e227717 100644 (file)
@@ -138,7 +138,7 @@ reset:
        and     t0, t0, (1 << 31)
 #else
 1:     mfc0    t0, CP0_EBASE
-       and     t0, t0, EBASE_CPUNUM
+       and     t0, t0, MIPS_EBASE_CPUNUM
 #endif
 
        /* Hang if this isn't the first CPU in the system */
diff --git a/arch/mips/include/asm/compiler.h b/arch/mips/include/asm/compiler.h
new file mode 100644 (file)
index 0000000..c498b42
--- /dev/null
@@ -0,0 +1,69 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+/*
+ * Copyright (C) 2004, 2007  Maciej W. Rozycki
+ *
+ * This file is subject to the terms and conditions of the GNU General Public
+ * License.  See the file "COPYING" in the main directory of this archive
+ * for more details.
+ */
+#ifndef _ASM_COMPILER_H
+#define _ASM_COMPILER_H
+
+/*
+ * With GCC 4.5 onwards we can use __builtin_unreachable to indicate to the
+ * compiler that a particular code path will never be hit. This allows it to be
+ * optimised out of the generated binary.
+ *
+ * Unfortunately at least GCC 4.6.3 through 7.3.0 inclusive suffer from a bug
+ * that can lead to instructions from beyond an unreachable statement being
+ * incorrectly reordered into earlier delay slots if the unreachable statement
+ * is the only content of a case in a switch statement. This can lead to
+ * seemingly random behaviour, such as invalid memory accesses from incorrectly
+ * reordered loads or stores. See this potential GCC fix for details:
+ *
+ *   https://gcc.gnu.org/ml/gcc-patches/2015-09/msg00360.html
+ *
+ * It is unclear whether GCC 8 onwards suffer from the same issue - nothing
+ * relevant is mentioned in GCC 8 release notes and nothing obviously relevant
+ * stands out in GCC commit logs, but these newer GCC versions generate very
+ * different code for the testcase which doesn't exhibit the bug.
+ *
+ * GCC also handles stack allocation suboptimally when calling noreturn
+ * functions or calling __builtin_unreachable():
+ *
+ *   https://gcc.gnu.org/bugzilla/show_bug.cgi?id=82365
+ *
+ * We work around both of these issues by placing a volatile asm statement,
+ * which GCC is prevented from reordering past, prior to __builtin_unreachable
+ * calls.
+ *
+ * The .insn statement is required to ensure that any branches to the
+ * statement, which sadly must be kept due to the asm statement, are known to
+ * be branches to code and satisfy linker requirements for microMIPS kernels.
+ */
+#undef barrier_before_unreachable
+#define barrier_before_unreachable() asm volatile(".insn")
+
+#if !defined(CONFIG_CC_IS_GCC) || \
+    (__GNUC__ > 4) || (__GNUC__ == 4 && __GNUC_MINOR__ >= 9)
+# define GCC_OFF_SMALL_ASM() "ZC"
+#elif defined(CONFIG_CPU_MICROMIPS)
+# error "microMIPS compilation unsupported with GCC older than 4.9"
+#else
+# define GCC_OFF_SMALL_ASM() "R"
+#endif
+
+#ifdef CONFIG_CPU_MIPSR6
+#define MIPS_ISA_LEVEL "mips64r6"
+#define MIPS_ISA_ARCH_LEVEL MIPS_ISA_LEVEL
+#define MIPS_ISA_LEVEL_RAW mips64r6
+#define MIPS_ISA_ARCH_LEVEL_RAW MIPS_ISA_LEVEL_RAW
+#else
+/* MIPS64 is a superset of MIPS32 */
+#define MIPS_ISA_LEVEL "mips64r2"
+#define MIPS_ISA_ARCH_LEVEL "arch=r4000"
+#define MIPS_ISA_LEVEL_RAW mips64r2
+#define MIPS_ISA_ARCH_LEVEL_RAW MIPS_ISA_LEVEL_RAW
+#endif /* CONFIG_CPU_MIPSR6 */
+
+#endif /* _ASM_COMPILER_H */
diff --git a/arch/mips/include/asm/isa-rev.h b/arch/mips/include/asm/isa-rev.h
new file mode 100644 (file)
index 0000000..683ea34
--- /dev/null
@@ -0,0 +1,24 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+/*
+ * Copyright (C) 2018 MIPS Tech, LLC
+ * Author: Matt Redfearn <matt.redfearn@mips.com>
+ */
+
+#ifndef __MIPS_ASM_ISA_REV_H__
+#define __MIPS_ASM_ISA_REV_H__
+
+/*
+ * The ISA revision level. This is 0 for MIPS I to V and N for
+ * MIPS{32,64}rN.
+ */
+
+/* If the compiler has defined __mips_isa_rev, believe it. */
+#ifdef __mips_isa_rev
+#define MIPS_ISA_REV __mips_isa_rev
+#else
+/* The compiler hasn't defined the isa rev so assume it's MIPS I - V (0) */
+#define MIPS_ISA_REV 0
+#endif
+
+
+#endif /* __MIPS_ASM_ISA_REV_H__ */
index 1738130..e65485b 100644 (file)
@@ -10,6 +10,8 @@
 #ifndef _ASM_MIPSREGS_H
 #define _ASM_MIPSREGS_H
 
+#include <asm/compiler.h>
+#include <asm/isa-rev.h>
 /*
  * The following macros are especially useful for __asm__
  * inline assembler.
  */
 #ifdef __ASSEMBLY__
 #define _ULCAST_
+#define _U64CAST_
 #else
-#include <linux/bitops.h>
 #define _ULCAST_ (unsigned long)
+#define _U64CAST_ (u64)
 #endif
 
 /*
 #define CP0_GLOBALNUMBER $3, 1
 #define CP0_CONTEXT $4
 #define CP0_PAGEMASK $5
+#define CP0_PAGEGRAIN $5, 1
+#define CP0_SEGCTL0 $5, 2
+#define CP0_SEGCTL1 $5, 3
+#define CP0_SEGCTL2 $5, 4
 #define CP0_WIRED $6
 #define CP0_INFO $7
-#define CP0_HWRENA $7, 0
+#define CP0_HWRENA $7
 #define CP0_BADVADDR $8
 #define CP0_BADINSTR $8, 1
 #define CP0_COUNT $9
 #define CP0_ENTRYHI $10
+#define CP0_GUESTCTL1 $10, 4
+#define CP0_GUESTCTL2 $10, 5
+#define CP0_GUESTCTL3 $10, 6
 #define CP0_COMPARE $11
+#define CP0_GUESTCTL0EXT $11, 4
 #define CP0_STATUS $12
+#define CP0_GUESTCTL0 $12, 6
+#define CP0_GTOFFSET $12, 7
 #define CP0_CAUSE $13
 #define CP0_EPC $14
 #define CP0_PRID $15
@@ -59,6 +72,7 @@
 #define CP0_CONFIG $16
 #define CP0_CONFIG3 $16, 3
 #define CP0_CONFIG5 $16, 5
+#define CP0_CONFIG6 $16, 6
 #define CP0_LLADDR $17
 #define CP0_WATCHLO $18
 #define CP0_WATCHHI $19
 #define MIPS_ENTRYLO_RI                (_ULCAST_(1) << (BITS_PER_LONG - 1))
 
 /*
+ * MIPSr6+ GlobalNumber register definitions
+ */
+#define MIPS_GLOBALNUMBER_VP_SHF       0
+#define MIPS_GLOBALNUMBER_VP           (_ULCAST_(0xff) << MIPS_GLOBALNUMBER_VP_SHF)
+#define MIPS_GLOBALNUMBER_CORE_SHF     8
+#define MIPS_GLOBALNUMBER_CORE         (_ULCAST_(0xff) << MIPS_GLOBALNUMBER_CORE_SHF)
+#define MIPS_GLOBALNUMBER_CLUSTER_SHF  16
+#define MIPS_GLOBALNUMBER_CLUSTER      (_ULCAST_(0xf) << MIPS_GLOBALNUMBER_CLUSTER_SHF)
+
+/*
  * Values for PageMask register
  */
 #ifdef CONFIG_CPU_VR41XX
 
 /* MIPS32/64 EntryHI bit definitions */
 #define MIPS_ENTRYHI_EHINV     (_ULCAST_(1) << 10)
+#define MIPS_ENTRYHI_ASIDX     (_ULCAST_(0x3) << 8)
+#define MIPS_ENTRYHI_ASID      (_ULCAST_(0xff) << 0)
 
 /*
  * R4x00 interrupt enable / cause bits
 #define         CAUSEF_IP7             (_ULCAST_(1)   << 15)
 #define CAUSEB_FDCI            21
 #define CAUSEF_FDCI            (_ULCAST_(1)   << 21)
+#define CAUSEB_WP              22
+#define CAUSEF_WP              (_ULCAST_(1)   << 22)
 #define CAUSEB_IV              23
 #define CAUSEF_IV              (_ULCAST_(1)   << 23)
 #define CAUSEB_PCI             26
 #define CAUSEF_PCI             (_ULCAST_(1)   << 26)
+#define CAUSEB_DC              27
+#define CAUSEF_DC              (_ULCAST_(1)   << 27)
 #define CAUSEB_CE              28
 #define CAUSEF_CE              (_ULCAST_(3)   << 28)
 #define CAUSEB_TI              30
 #define CAUSEF_BD              (_ULCAST_(1)   << 31)
 
 /*
- * Bits in the coprocessor 0 EBase register.
+ * Cause.ExcCode trap codes.
  */
-#define EBASE_CPUNUM           0x3ff
-#define EBASE_WG               (_ULCAST_(1) << 11)
+#define EXCCODE_INT            0       /* Interrupt pending */
+#define EXCCODE_MOD            1       /* TLB modified fault */
+#define EXCCODE_TLBL           2       /* TLB miss on load or ifetch */
+#define EXCCODE_TLBS           3       /* TLB miss on a store */
+#define EXCCODE_ADEL           4       /* Address error on a load or ifetch */
+#define EXCCODE_ADES           5       /* Address error on a store */
+#define EXCCODE_IBE            6       /* Bus error on an ifetch */
+#define EXCCODE_DBE            7       /* Bus error on a load or store */
+#define EXCCODE_SYS            8       /* System call */
+#define EXCCODE_BP             9       /* Breakpoint */
+#define EXCCODE_RI             10      /* Reserved instruction exception */
+#define EXCCODE_CPU            11      /* Coprocessor unusable */
+#define EXCCODE_OV             12      /* Arithmetic overflow */
+#define EXCCODE_TR             13      /* Trap instruction */
+#define EXCCODE_MSAFPE         14      /* MSA floating point exception */
+#define EXCCODE_FPE            15      /* Floating point exception */
+#define EXCCODE_TLBRI          19      /* TLB Read-Inhibit exception */
+#define EXCCODE_TLBXI          20      /* TLB Execution-Inhibit exception */
+#define EXCCODE_MSADIS         21      /* MSA disabled exception */
+#define EXCCODE_MDMX           22      /* MDMX unusable exception */
+#define EXCCODE_WATCH          23      /* Watch address reference */
+#define EXCCODE_MCHECK         24      /* Machine check */
+#define EXCCODE_THREAD         25      /* Thread exceptions (MT) */
+#define EXCCODE_DSPDIS         26      /* DSP disabled exception */
+#define EXCCODE_GE             27      /* Virtualized guest exception (VZ) */
+
+/* Implementation specific trap codes used by MIPS cores */
+#define MIPS_EXCCODE_TLBPAR    16      /* TLB parity error exception */
 
 /*
  * Bits in the coprocessor 0 config register.
 #define CONF_SM                        (_ULCAST_(1) << 16)
 #define CONF_SC                        (_ULCAST_(1) << 17)
 #define CONF_EW                        (_ULCAST_(3) << 18)
-#define CONF_EP                        (_ULCAST_(15) << 24)
+#define CONF_EP                        (_ULCAST_(15)<< 24)
 #define CONF_EC                        (_ULCAST_(7) << 28)
 #define CONF_CM                        (_ULCAST_(1) << 31)
 
 #define R10K_CONF_CT           (_ULCAST_(1) <<  5)
 #define R10K_CONF_PE           (_ULCAST_(1) <<  6)
 #define R10K_CONF_PM           (_ULCAST_(3) <<  7)
-#define R10K_CONF_EC           (_ULCAST_(15) << 9)
+#define R10K_CONF_EC           (_ULCAST_(15)<<  9)
 #define R10K_CONF_SB           (_ULCAST_(1) << 13)
 #define R10K_CONF_SK           (_ULCAST_(1) << 14)
 #define R10K_CONF_SS           (_ULCAST_(7) << 16)
 #define TX49_CONF_CWFON                (_ULCAST_(1) << 27)
 
 /* Bits specific to the MIPS32/64 PRA. */
+#define MIPS_CONF_VI           (_ULCAST_(1) <<  3)
 #define MIPS_CONF_MT           (_ULCAST_(7) <<  7)
 #define MIPS_CONF_MT_TLB       (_ULCAST_(1) <<  7)
 #define MIPS_CONF_MT_FTLB      (_ULCAST_(4) <<  7)
 #define MIPS_CONF4_MMUEXTDEF_MMUSIZEEXT (_ULCAST_(1) << 14)
 #define MIPS_CONF4_MMUEXTDEF_FTLBSIZEEXT       (_ULCAST_(2) << 14)
 #define MIPS_CONF4_MMUEXTDEF_VTLBSIZEEXT       (_ULCAST_(3) << 14)
-#define MIPS_CONF4_KSCREXIST   (_ULCAST_(255) << 16)
+#define MIPS_CONF4_KSCREXIST_SHIFT     (16)
+#define MIPS_CONF4_KSCREXIST   (_ULCAST_(255) << MIPS_CONF4_KSCREXIST_SHIFT)
 #define MIPS_CONF4_VTLBSIZEEXT_SHIFT   (24)
 #define MIPS_CONF4_VTLBSIZEEXT (_ULCAST_(15) << MIPS_CONF4_VTLBSIZEEXT_SHIFT)
 #define MIPS_CONF4_AE          (_ULCAST_(1) << 28)
 #define MIPS_CONF5_LLB         (_ULCAST_(1) << 4)
 #define MIPS_CONF5_MVH         (_ULCAST_(1) << 5)
 #define MIPS_CONF5_VP          (_ULCAST_(1) << 7)
+#define MIPS_CONF5_SBRI                (_ULCAST_(1) << 6)
 #define MIPS_CONF5_FRE         (_ULCAST_(1) << 8)
 #define MIPS_CONF5_UFE         (_ULCAST_(1) << 9)
 #define MIPS_CONF5_L2C         (_ULCAST_(1) << 10)
+#define MIPS_CONF5_CA2         (_ULCAST_(1) << 14)
+#define MIPS_CONF5_MI          (_ULCAST_(1) << 17)
+#define MIPS_CONF5_CRCP                (_ULCAST_(1) << 18)
 #define MIPS_CONF5_MSAEN       (_ULCAST_(1) << 27)
 #define MIPS_CONF5_EVA         (_ULCAST_(1) << 28)
 #define MIPS_CONF5_CV          (_ULCAST_(1) << 29)
 #define MIPS_CONF6_SYND                (_ULCAST_(1) << 13)
 /* proAptiv FTLB on/off bit */
 #define MIPS_CONF6_FTLBEN      (_ULCAST_(1) << 15)
+/* Loongson-3 FTLB on/off bit */
+#define MIPS_CONF6_FTLBDIS     (_ULCAST_(1) << 22)
 /* FTLB probability bits */
 #define MIPS_CONF6_FTLBP_SHIFT (16)
 
 
 #define MIPS_CONF7_IAR         (_ULCAST_(1) << 10)
 #define MIPS_CONF7_AR          (_ULCAST_(1) << 16)
-/* FTLB probability bits for R6 */
-#define MIPS_CONF7_FTLBP_SHIFT (18)
+
+/* Ingenic HPTLB off bits */
+#define XBURST_PAGECTRL_HPTLB_DIS 0xa9000000
+
+/* Ingenic Config7 bits */
+#define MIPS_CONF7_BTB_LOOP_EN (_ULCAST_(1) << 4)
+
+/* Config7 Bits specific to MIPS Technologies. */
+
+/* Performance counters implemented Per TC */
+#define MTI_CONF7_PTC          (_ULCAST_(1) << 19)
+
+/* WatchLo* register definitions */
+#define MIPS_WATCHLO_IRW       (_ULCAST_(0x7) << 0)
+
+/* WatchHi* register definitions */
+#define MIPS_WATCHHI_M         (_ULCAST_(1) << 31)
+#define MIPS_WATCHHI_G         (_ULCAST_(1) << 30)
+#define MIPS_WATCHHI_WM                (_ULCAST_(0x3) << 28)
+#define MIPS_WATCHHI_WM_R_RVA  (_ULCAST_(0) << 28)
+#define MIPS_WATCHHI_WM_R_GPA  (_ULCAST_(1) << 28)
+#define MIPS_WATCHHI_WM_G_GVA  (_ULCAST_(2) << 28)
+#define MIPS_WATCHHI_EAS       (_ULCAST_(0x3) << 24)
+#define MIPS_WATCHHI_ASID      (_ULCAST_(0xff) << 16)
+#define MIPS_WATCHHI_MASK      (_ULCAST_(0x1ff) << 3)
+#define MIPS_WATCHHI_I         (_ULCAST_(1) << 2)
+#define MIPS_WATCHHI_R         (_ULCAST_(1) << 1)
+#define MIPS_WATCHHI_W         (_ULCAST_(1) << 0)
+#define MIPS_WATCHHI_IRW       (_ULCAST_(0x7) << 0)
+
+/* PerfCnt control register definitions */
+#define MIPS_PERFCTRL_EXL      (_ULCAST_(1) << 0)
+#define MIPS_PERFCTRL_K                (_ULCAST_(1) << 1)
+#define MIPS_PERFCTRL_S                (_ULCAST_(1) << 2)
+#define MIPS_PERFCTRL_U                (_ULCAST_(1) << 3)
+#define MIPS_PERFCTRL_IE       (_ULCAST_(1) << 4)
+#define MIPS_PERFCTRL_EVENT_S  5
+#define MIPS_PERFCTRL_EVENT    (_ULCAST_(0x3ff) << MIPS_PERFCTRL_EVENT_S)
+#define MIPS_PERFCTRL_PCTD     (_ULCAST_(1) << 15)
+#define MIPS_PERFCTRL_EC       (_ULCAST_(0x3) << 23)
+#define MIPS_PERFCTRL_EC_R     (_ULCAST_(0) << 23)
+#define MIPS_PERFCTRL_EC_RI    (_ULCAST_(1) << 23)
+#define MIPS_PERFCTRL_EC_G     (_ULCAST_(2) << 23)
+#define MIPS_PERFCTRL_EC_GRI   (_ULCAST_(3) << 23)
+#define MIPS_PERFCTRL_W                (_ULCAST_(1) << 30)
+#define MIPS_PERFCTRL_M                (_ULCAST_(1) << 31)
+
+/* PerfCnt control register MT extensions used by MIPS cores */
+#define MIPS_PERFCTRL_VPEID_S  16
+#define MIPS_PERFCTRL_VPEID    (_ULCAST_(0xf) << MIPS_PERFCTRL_VPEID_S)
+#define MIPS_PERFCTRL_TCID_S   22
+#define MIPS_PERFCTRL_TCID     (_ULCAST_(0xff) << MIPS_PERFCTRL_TCID_S)
+#define MIPS_PERFCTRL_MT_EN    (_ULCAST_(0x3) << 20)
+#define MIPS_PERFCTRL_MT_EN_ALL        (_ULCAST_(0) << 20)
+#define MIPS_PERFCTRL_MT_EN_VPE        (_ULCAST_(1) << 20)
+#define MIPS_PERFCTRL_MT_EN_TC (_ULCAST_(2) << 20)
+
+/* PerfCnt control register MT extensions used by BMIPS5000 */
+#define BRCM_PERFCTRL_TC       (_ULCAST_(1) << 30)
+
+/* PerfCnt control register MT extensions used by Netlogic XLR */
+#define XLR_PERFCTRL_ALLTHREADS        (_ULCAST_(1) << 13)
 
 /* MAAR bit definitions */
+#define MIPS_MAAR_VH           (_U64CAST_(1) << 63)
 #define MIPS_MAAR_ADDR         ((BIT_ULL(BITS_PER_LONG - 12) - 1) << 12)
 #define MIPS_MAAR_ADDR_SHIFT   12
 #define MIPS_MAAR_S            (_ULCAST_(1) << 1)
-#define MIPS_MAAR_V            (_ULCAST_(1) << 0)
+#define MIPS_MAAR_VL           (_ULCAST_(1) << 0)
+
+/* MAARI bit definitions */
+#define MIPS_MAARI_INDEX       (_ULCAST_(0x3f) << 0)
+
+/* EBase bit definitions */
+#define MIPS_EBASE_CPUNUM_SHIFT        0
+#define MIPS_EBASE_CPUNUM      (_ULCAST_(0x3ff) << 0)
+#define MIPS_EBASE_WG_SHIFT    11
+#define MIPS_EBASE_WG          (_ULCAST_(1) << 11)
+#define MIPS_EBASE_BASE_SHIFT  12
+#define MIPS_EBASE_BASE                (~_ULCAST_((1 << MIPS_EBASE_BASE_SHIFT) - 1))
 
 /* CMGCRBase bit definitions */
 #define MIPS_CMGCRB_BASE       11
 #define MIPS_CMGCRF_BASE       (~_ULCAST_((1 << MIPS_CMGCRB_BASE) - 1))
 
+/* LLAddr bit definitions */
+#define MIPS_LLADDR_LLB_SHIFT  0
+#define MIPS_LLADDR_LLB                (_ULCAST_(1) << MIPS_LLADDR_LLB_SHIFT)
+
 /*
  * Bits in the MIPS32 Memory Segmentation registers.
  */
 #define MIPS_PWFIELD_PTEI_SHIFT        0
 #define MIPS_PWFIELD_PTEI_MASK 0x0000003f
 
+#define MIPS_PWSIZE_PS_SHIFT   30
+#define MIPS_PWSIZE_PS_MASK    0x40000000
 #define MIPS_PWSIZE_GDW_SHIFT  24
 #define MIPS_PWSIZE_GDW_MASK   0x3f000000
 #define MIPS_PWSIZE_UDW_SHIFT  18
 
 #define MIPS_PWCTL_PWEN_SHIFT  31
 #define MIPS_PWCTL_PWEN_MASK   0x80000000
+#define MIPS_PWCTL_XK_SHIFT    28
+#define MIPS_PWCTL_XK_MASK     0x10000000
+#define MIPS_PWCTL_XS_SHIFT    27
+#define MIPS_PWCTL_XS_MASK     0x08000000
+#define MIPS_PWCTL_XU_SHIFT    26
+#define MIPS_PWCTL_XU_MASK     0x04000000
 #define MIPS_PWCTL_DPH_SHIFT   7
 #define MIPS_PWCTL_DPH_MASK    0x00000080
 #define MIPS_PWCTL_HUGEPG_SHIFT        6
 #define MIPS_PWCTL_PSN_SHIFT   0
 #define MIPS_PWCTL_PSN_MASK    0x0000003f
 
+/* GuestCtl0 fields */
+#define MIPS_GCTL0_GM_SHIFT    31
+#define MIPS_GCTL0_GM          (_ULCAST_(1) << MIPS_GCTL0_GM_SHIFT)
+#define MIPS_GCTL0_RI_SHIFT    30
+#define MIPS_GCTL0_RI          (_ULCAST_(1) << MIPS_GCTL0_RI_SHIFT)
+#define MIPS_GCTL0_MC_SHIFT    29
+#define MIPS_GCTL0_MC          (_ULCAST_(1) << MIPS_GCTL0_MC_SHIFT)
+#define MIPS_GCTL0_CP0_SHIFT   28
+#define MIPS_GCTL0_CP0         (_ULCAST_(1) << MIPS_GCTL0_CP0_SHIFT)
+#define MIPS_GCTL0_AT_SHIFT    26
+#define MIPS_GCTL0_AT          (_ULCAST_(0x3) << MIPS_GCTL0_AT_SHIFT)
+#define MIPS_GCTL0_GT_SHIFT    25
+#define MIPS_GCTL0_GT          (_ULCAST_(1) << MIPS_GCTL0_GT_SHIFT)
+#define MIPS_GCTL0_CG_SHIFT    24
+#define MIPS_GCTL0_CG          (_ULCAST_(1) << MIPS_GCTL0_CG_SHIFT)
+#define MIPS_GCTL0_CF_SHIFT    23
+#define MIPS_GCTL0_CF          (_ULCAST_(1) << MIPS_GCTL0_CF_SHIFT)
+#define MIPS_GCTL0_G1_SHIFT    22
+#define MIPS_GCTL0_G1          (_ULCAST_(1) << MIPS_GCTL0_G1_SHIFT)
+#define MIPS_GCTL0_G0E_SHIFT   19
+#define MIPS_GCTL0_G0E         (_ULCAST_(1) << MIPS_GCTL0_G0E_SHIFT)
+#define MIPS_GCTL0_PT_SHIFT    18
+#define MIPS_GCTL0_PT          (_ULCAST_(1) << MIPS_GCTL0_PT_SHIFT)
+#define MIPS_GCTL0_RAD_SHIFT   9
+#define MIPS_GCTL0_RAD         (_ULCAST_(1) << MIPS_GCTL0_RAD_SHIFT)
+#define MIPS_GCTL0_DRG_SHIFT   8
+#define MIPS_GCTL0_DRG         (_ULCAST_(1) << MIPS_GCTL0_DRG_SHIFT)
+#define MIPS_GCTL0_G2_SHIFT    7
+#define MIPS_GCTL0_G2          (_ULCAST_(1) << MIPS_GCTL0_G2_SHIFT)
+#define MIPS_GCTL0_GEXC_SHIFT  2
+#define MIPS_GCTL0_GEXC                (_ULCAST_(0x1f) << MIPS_GCTL0_GEXC_SHIFT)
+#define MIPS_GCTL0_SFC2_SHIFT  1
+#define MIPS_GCTL0_SFC2                (_ULCAST_(1) << MIPS_GCTL0_SFC2_SHIFT)
+#define MIPS_GCTL0_SFC1_SHIFT  0
+#define MIPS_GCTL0_SFC1                (_ULCAST_(1) << MIPS_GCTL0_SFC1_SHIFT)
+
+/* GuestCtl0.AT Guest address translation control */
+#define MIPS_GCTL0_AT_ROOT     1  /* Guest MMU under Root control */
+#define MIPS_GCTL0_AT_GUEST    3  /* Guest MMU under Guest control */
+
+/* GuestCtl0.GExcCode Hypervisor exception cause codes */
+#define MIPS_GCTL0_GEXC_GPSI   0  /* Guest Privileged Sensitive Instruction */
+#define MIPS_GCTL0_GEXC_GSFC   1  /* Guest Software Field Change */
+#define MIPS_GCTL0_GEXC_HC     2  /* Hypercall */
+#define MIPS_GCTL0_GEXC_GRR    3  /* Guest Reserved Instruction Redirect */
+#define MIPS_GCTL0_GEXC_GVA    8  /* Guest Virtual Address available */
+#define MIPS_GCTL0_GEXC_GHFC   9  /* Guest Hardware Field Change */
+#define MIPS_GCTL0_GEXC_GPA    10 /* Guest Physical Address available */
+
+/* GuestCtl0Ext fields */
+#define MIPS_GCTL0EXT_RPW_SHIFT        8
+#define MIPS_GCTL0EXT_RPW      (_ULCAST_(0x3) << MIPS_GCTL0EXT_RPW_SHIFT)
+#define MIPS_GCTL0EXT_NCC_SHIFT        6
+#define MIPS_GCTL0EXT_NCC      (_ULCAST_(0x3) << MIPS_GCTL0EXT_NCC_SHIFT)
+#define MIPS_GCTL0EXT_CGI_SHIFT        4
+#define MIPS_GCTL0EXT_CGI      (_ULCAST_(1) << MIPS_GCTL0EXT_CGI_SHIFT)
+#define MIPS_GCTL0EXT_FCD_SHIFT        3
+#define MIPS_GCTL0EXT_FCD      (_ULCAST_(1) << MIPS_GCTL0EXT_FCD_SHIFT)
+#define MIPS_GCTL0EXT_OG_SHIFT 2
+#define MIPS_GCTL0EXT_OG       (_ULCAST_(1) << MIPS_GCTL0EXT_OG_SHIFT)
+#define MIPS_GCTL0EXT_BG_SHIFT 1
+#define MIPS_GCTL0EXT_BG       (_ULCAST_(1) << MIPS_GCTL0EXT_BG_SHIFT)
+#define MIPS_GCTL0EXT_MG_SHIFT 0
+#define MIPS_GCTL0EXT_MG       (_ULCAST_(1) << MIPS_GCTL0EXT_MG_SHIFT)
+
+/* GuestCtl0Ext.RPW Root page walk configuration */
+#define MIPS_GCTL0EXT_RPW_BOTH 0  /* Root PW for GPA->RPA and RVA->RPA */
+#define MIPS_GCTL0EXT_RPW_GPA  2  /* Root PW for GPA->RPA */
+#define MIPS_GCTL0EXT_RPW_RVA  3  /* Root PW for RVA->RPA */
+
+/* GuestCtl0Ext.NCC Nested cache coherency attributes */
+#define MIPS_GCTL0EXT_NCC_IND  0  /* Guest CCA independent of Root CCA */
+#define MIPS_GCTL0EXT_NCC_MOD  1  /* Guest CCA modified by Root CCA */
+
+/* GuestCtl1 fields */
+#define MIPS_GCTL1_ID_SHIFT    0
+#define MIPS_GCTL1_ID_WIDTH    8
+#define MIPS_GCTL1_ID          (_ULCAST_(0xff) << MIPS_GCTL1_ID_SHIFT)
+#define MIPS_GCTL1_RID_SHIFT   16
+#define MIPS_GCTL1_RID_WIDTH   8
+#define MIPS_GCTL1_RID         (_ULCAST_(0xff) << MIPS_GCTL1_RID_SHIFT)
+#define MIPS_GCTL1_EID_SHIFT   24
+#define MIPS_GCTL1_EID_WIDTH   8
+#define MIPS_GCTL1_EID         (_ULCAST_(0xff) << MIPS_GCTL1_EID_SHIFT)
+
+/* GuestID reserved for root context */
+#define MIPS_GCTL1_ROOT_GUESTID        0
+
 /* CDMMBase register bit definitions */
 #define MIPS_CDMMBASE_SIZE_SHIFT 0
 #define MIPS_CDMMBASE_SIZE     (_ULCAST_(511) << MIPS_CDMMBASE_SIZE_SHIFT)
 #define MIPS_CDMMBASE_ADDR_SHIFT 11
 #define MIPS_CDMMBASE_ADDR_START 15
 
+/* RDHWR register numbers */
+#define MIPS_HWR_CPUNUM                0       /* CPU number */
+#define MIPS_HWR_SYNCISTEP     1       /* SYNCI step size */
+#define MIPS_HWR_CC            2       /* Cycle counter */
+#define MIPS_HWR_CCRES         3       /* Cycle counter resolution */
+#define MIPS_HWR_ULR           29      /* UserLocal */
+#define MIPS_HWR_IMPL1         30      /* Implementation dependent */
+#define MIPS_HWR_IMPL2         31      /* Implementation dependent */
+
+/* Bits in HWREna register */
+#define MIPS_HWRENA_CPUNUM     (_ULCAST_(1) << MIPS_HWR_CPUNUM)
+#define MIPS_HWRENA_SYNCISTEP  (_ULCAST_(1) << MIPS_HWR_SYNCISTEP)
+#define MIPS_HWRENA_CC         (_ULCAST_(1) << MIPS_HWR_CC)
+#define MIPS_HWRENA_CCRES      (_ULCAST_(1) << MIPS_HWR_CCRES)
+#define MIPS_HWRENA_ULR                (_ULCAST_(1) << MIPS_HWR_ULR)
+#define MIPS_HWRENA_IMPL1      (_ULCAST_(1) << MIPS_HWR_IMPL1)
+#define MIPS_HWRENA_IMPL2      (_ULCAST_(1) << MIPS_HWR_IMPL2)
+
 /*
  * Bitfields in the TX39 family CP0 Configuration Register 3
  */
 /* Disable Branch Return Cache */
 #define R10K_DIAG_D_BRC                (_ULCAST_(1) << 22)
 
+/* Flush ITLB */
+#define LOONGSON_DIAG_ITLB     (_ULCAST_(1) << 2)
+/* Flush DTLB */
+#define LOONGSON_DIAG_DTLB     (_ULCAST_(1) << 3)
+/* Flush VTLB */
+#define LOONGSON_DIAG_VTLB     (_ULCAST_(1) << 12)
+/* Flush FTLB */
+#define LOONGSON_DIAG_FTLB     (_ULCAST_(1) << 13)
+
+/* CvmCtl register field definitions */
+#define CVMCTL_IPPCI_SHIFT     7
+#define CVMCTL_IPPCI           (_U64CAST_(0x7) << CVMCTL_IPPCI_SHIFT)
+#define CVMCTL_IPTI_SHIFT      4
+#define CVMCTL_IPTI            (_U64CAST_(0x7) << CVMCTL_IPTI_SHIFT)
+
+/* CvmMemCtl2 register field definitions */
+#define CVMMEMCTL2_INHIBITTS   (_U64CAST_(1) << 17)
+
+/* CvmVMConfig register field definitions */
+#define CVMVMCONF_DGHT         (_U64CAST_(1) << 60)
+#define CVMVMCONF_MMUSIZEM1_S  12
+#define CVMVMCONF_MMUSIZEM1    (_U64CAST_(0xff) << CVMVMCONF_MMUSIZEM1_S)
+#define CVMVMCONF_RMMUSIZEM1_S 0
+#define CVMVMCONF_RMMUSIZEM1   (_U64CAST_(0xff) << CVMVMCONF_RMMUSIZEM1_S)
+
 /*
  * Coprocessor 1 (FPU) register names
  */
 /*
  * Bits 22:20 of the FPU Status Register will be read as 0,
  * and should be written as zero.
+ * MAC2008 was removed in Release 5 so we still treat it as
+ * reserved.
  */
 #define FPU_CSR_RSVD   (_ULCAST_(7) << 20)
 
+#define FPU_CSR_MAC2008        (_ULCAST_(1) << 20)
 #define FPU_CSR_ABS2008        (_ULCAST_(1) << 19)
 #define FPU_CSR_NAN2008        (_ULCAST_(1) << 18)
 
  * Macros for handling the ISA mode bit for MIPS16 and microMIPS.
  */
 #if defined(CONFIG_SYS_SUPPORTS_MIPS16) || \
-       defined(CONFIG_SYS_SUPPORTS_MICROMIPS)
+    defined(CONFIG_SYS_SUPPORTS_MICROMIPS)
 #define get_isa16_mode(x)              ((x) & 0x1)
 #define msk_isa16_mode(x)              ((x) & ~0x1)
-#define set_isa16_mode(x)              do { (x) |= 0x1; } while (0)
+#define set_isa16_mode(x)              do { (x) |= 0x1; } while(0)
 #else
 #define get_isa16_mode(x)              0
 #define msk_isa16_mode(x)              (x)
-#define set_isa16_mode(x)              do { } while (0)
+#define set_isa16_mode(x)              do { } while(0)
 #endif
 
 /*
@@ -849,6 +1131,123 @@ static inline int mm_insn_16bit(u16 insn)
 }
 
 /*
+ * Helper macros for generating raw instruction encodings in inline asm.
+ */
+#ifdef CONFIG_CPU_MICROMIPS
+#define _ASM_INSN16_IF_MM(_enc)                        \
+       ".insn\n\t"                             \
+       ".hword (" #_enc ")\n\t"
+#define _ASM_INSN32_IF_MM(_enc)                        \
+       ".insn\n\t"                             \
+       ".hword ((" #_enc ") >> 16)\n\t"        \
+       ".hword ((" #_enc ") & 0xffff)\n\t"
+#else
+#define _ASM_INSN_IF_MIPS(_enc)                        \
+       ".insn\n\t"                             \
+       ".word (" #_enc ")\n\t"
+#endif
+
+#ifndef _ASM_INSN16_IF_MM
+#define _ASM_INSN16_IF_MM(_enc)
+#endif
+#ifndef _ASM_INSN32_IF_MM
+#define _ASM_INSN32_IF_MM(_enc)
+#endif
+#ifndef _ASM_INSN_IF_MIPS
+#define _ASM_INSN_IF_MIPS(_enc)
+#endif
+
+/*
+ * parse_r var, r - Helper assembler macro for parsing register names.
+ *
+ * This converts the register name in $n form provided in \r to the
+ * corresponding register number, which is assigned to the variable \var. It is
+ * needed to allow explicit encoding of instructions in inline assembly where
+ * registers are chosen by the compiler in $n form, allowing us to avoid using
+ * fixed register numbers.
+ *
+ * It also allows newer instructions (not implemented by the assembler) to be
+ * transparently implemented using assembler macros, instead of needing separate
+ * cases depending on toolchain support.
+ *
+ * Simple usage example:
+ * __asm__ __volatile__("parse_r __rt, %0\n\t"
+ *                     ".insn\n\t"
+ *                     "# di    %0\n\t"
+ *                     ".word   (0x41606000 | (__rt << 16))"
+ *                     : "=r" (status);
+ */
+
+/* Match an individual register number and assign to \var */
+#define _IFC_REG(n)                            \
+       ".ifc   \\r, $" #n "\n\t"               \
+       "\\var  = " #n "\n\t"                   \
+       ".endif\n\t"
+
+__asm__(".macro        parse_r var r\n\t"
+       "\\var  = -1\n\t"
+       _IFC_REG(0)  _IFC_REG(1)  _IFC_REG(2)  _IFC_REG(3)
+       _IFC_REG(4)  _IFC_REG(5)  _IFC_REG(6)  _IFC_REG(7)
+       _IFC_REG(8)  _IFC_REG(9)  _IFC_REG(10) _IFC_REG(11)
+       _IFC_REG(12) _IFC_REG(13) _IFC_REG(14) _IFC_REG(15)
+       _IFC_REG(16) _IFC_REG(17) _IFC_REG(18) _IFC_REG(19)
+       _IFC_REG(20) _IFC_REG(21) _IFC_REG(22) _IFC_REG(23)
+       _IFC_REG(24) _IFC_REG(25) _IFC_REG(26) _IFC_REG(27)
+       _IFC_REG(28) _IFC_REG(29) _IFC_REG(30) _IFC_REG(31)
+       ".iflt  \\var\n\t"
+       ".error \"Unable to parse register name \\r\"\n\t"
+       ".endif\n\t"
+       ".endm");
+
+#undef _IFC_REG
+
+/*
+ * C macros for generating assembler macros for common instruction formats.
+ *
+ * The names of the operands can be chosen by the caller, and the encoding of
+ * register operand \<Rn> is assigned to __<Rn> where it can be accessed from
+ * the ENC encodings.
+ */
+
+/* Instructions with no operands */
+#define _ASM_MACRO_0(OP, ENC)                                          \
+       __asm__(".macro " #OP "\n\t"                                    \
+               ENC                                                     \
+               ".endm")
+
+/* Instructions with 1 register operand & 1 immediate operand */
+#define _ASM_MACRO_1R1I(OP, R1, I2, ENC)                               \
+       __asm__(".macro " #OP " " #R1 ", " #I2 "\n\t"                   \
+               "parse_r __" #R1 ", \\" #R1 "\n\t"                      \
+               ENC                                                     \
+               ".endm")
+
+/* Instructions with 2 register operands */
+#define _ASM_MACRO_2R(OP, R1, R2, ENC)                                 \
+       __asm__(".macro " #OP " " #R1 ", " #R2 "\n\t"                   \
+               "parse_r __" #R1 ", \\" #R1 "\n\t"                      \
+               "parse_r __" #R2 ", \\" #R2 "\n\t"                      \
+               ENC                                                     \
+               ".endm")
+
+/* Instructions with 3 register operands */
+#define _ASM_MACRO_3R(OP, R1, R2, R3, ENC)                             \
+       __asm__(".macro " #OP " " #R1 ", " #R2 ", " #R3 "\n\t"          \
+               "parse_r __" #R1 ", \\" #R1 "\n\t"                      \
+               "parse_r __" #R2 ", \\" #R2 "\n\t"                      \
+               "parse_r __" #R3 ", \\" #R3 "\n\t"                      \
+               ENC                                                     \
+               ".endm")
+
+/* Instructions with 2 register operands and 1 optional select operand */
+#define _ASM_MACRO_2R_1S(OP, R1, R2, SEL3, ENC)                                \
+       __asm__(".macro " #OP " " #R1 ", " #R2 ", " #SEL3 " = 0\n\t"    \
+               "parse_r __" #R1 ", \\" #R1 "\n\t"                      \
+               "parse_r __" #R2 ", \\" #R2 "\n\t"                      \
+               ENC                                                     \
+               ".endm")
+
+/*
  * TLB Invalidate Flush
  */
 static inline void tlbinvf(void)
@@ -856,7 +1255,9 @@ static inline void tlbinvf(void)
        __asm__ __volatile__(
                ".set push\n\t"
                ".set noreorder\n\t"
-               ".word 0x42000004\n\t" /* tlbinvf */
+               "# tlbinvf\n\t"
+               _ASM_INSN_IF_MIPS(0x42000004)
+               _ASM_INSN32_IF_MM(0x0000537c)
                ".set pop");
 }
 
@@ -911,40 +1312,55 @@ do {                                                             \
  * Macros to access the system control coprocessor
  */
 
-#define __read_32bit_c0_register(source, sel)                          \
+#define ___read_32bit_c0_register(source, sel, vol)                    \
 ({ unsigned int __res;                                                 \
        if (sel == 0)                                                   \
-               __asm__ __volatile__(                                   \
+               __asm__ vol(                                            \
                        "mfc0\t%0, " #source "\n\t"                     \
                        : "=r" (__res));                                \
        else                                                            \
-               __asm__ __volatile__(                                   \
+               __asm__ vol(                                            \
+                       ".set\tpush\n\t"                                \
                        ".set\tmips32\n\t"                              \
                        "mfc0\t%0, " #source ", " #sel "\n\t"           \
-                       ".set\tmips0\n\t"                               \
+                       ".set\tpop\n\t"                                 \
                        : "=r" (__res));                                \
        __res;                                                          \
 })
 
-#define __read_64bit_c0_register(source, sel)                          \
+#define ___read_64bit_c0_register(source, sel, vol)                    \
 ({ unsigned long long __res;                                           \
        if (sizeof(unsigned long) == 4)                                 \
-               __res = __read_64bit_c0_split(source, sel);             \
+               __res = __read_64bit_c0_split(source, sel, vol);        \
        else if (sel == 0)                                              \
-               __asm__ __volatile__(                                   \
+               __asm__ vol(                                            \
+                       ".set\tpush\n\t"                                \
                        ".set\tmips3\n\t"                               \
                        "dmfc0\t%0, " #source "\n\t"                    \
-                       ".set\tmips0"                                   \
+                       ".set\tpop"                                     \
                        : "=r" (__res));                                \
        else                                                            \
-               __asm__ __volatile__(                                   \
+               __asm__ vol(                                            \
+                       ".set\tpush\n\t"                                \
                        ".set\tmips64\n\t"                              \
                        "dmfc0\t%0, " #source ", " #sel "\n\t"          \
-                       ".set\tmips0"                                   \
+                       ".set\tpop"                                     \
                        : "=r" (__res));                                \
        __res;                                                          \
 })
 
+#define __read_32bit_c0_register(source, sel)                          \
+       ___read_32bit_c0_register(source, sel, __volatile__)
+
+#define __read_const_32bit_c0_register(source, sel)                    \
+       ___read_32bit_c0_register(source, sel,)
+
+#define __read_64bit_c0_register(source, sel)                          \
+       ___read_64bit_c0_register(source, sel, __volatile__)
+
+#define __read_const_64bit_c0_register(source, sel)                    \
+       ___read_64bit_c0_register(source, sel,)
+
 #define __write_32bit_c0_register(register, sel, value)                        \
 do {                                                                   \
        if (sel == 0)                                                   \
@@ -953,9 +1369,10 @@ do {                                                                      \
                        : : "Jr" ((unsigned int)(value)));              \
        else                                                            \
                __asm__ __volatile__(                                   \
+                       ".set\tpush\n\t"                                \
                        ".set\tmips32\n\t"                              \
                        "mtc0\t%z0, " #register ", " #sel "\n\t"        \
-                       ".set\tmips0"                                   \
+                       ".set\tpop"                                     \
                        : : "Jr" ((unsigned int)(value)));              \
 } while (0)
 
@@ -965,15 +1382,17 @@ do {                                                                     \
                __write_64bit_c0_split(register, sel, value);           \
        else if (sel == 0)                                              \
                __asm__ __volatile__(                                   \
+                       ".set\tpush\n\t"                                \
                        ".set\tmips3\n\t"                               \
                        "dmtc0\t%z0, " #register "\n\t"                 \
-                       ".set\tmips0"                                   \
+                       ".set\tpop"                                     \
                        : : "Jr" (value));                              \
        else                                                            \
                __asm__ __volatile__(                                   \
+                       ".set\tpush\n\t"                                \
                        ".set\tmips64\n\t"                              \
                        "dmtc0\t%z0, " #register ", " #sel "\n\t"       \
-                       ".set\tmips0"                                   \
+                       ".set\tpop"                                     \
                        : : "Jr" (value));                              \
 } while (0)
 
@@ -982,6 +1401,11 @@ do {                                                                      \
        (unsigned long) __read_32bit_c0_register(reg, sel) :            \
        (unsigned long) __read_64bit_c0_register(reg, sel))
 
+#define __read_const_ulong_c0_register(reg, sel)                       \
+       ((sizeof(unsigned long) == 4) ?                                 \
+       (unsigned long) __read_const_32bit_c0_register(reg, sel) :      \
+       (unsigned long) __read_const_64bit_c0_register(reg, sel))
+
 #define __write_ulong_c0_register(reg, sel, val)                       \
 do {                                                                   \
        if (sizeof(unsigned long) == 4)                                 \
@@ -1012,27 +1436,27 @@ do {                                                                    \
  * These versions are only needed for systems with more than 38 bits of
  * physical address space running the 32-bit kernel.  That's none atm :-)
  */
-#define __read_64bit_c0_split(source, sel)                             \
+#define __read_64bit_c0_split(source, sel, vol)                                \
 ({                                                                     \
        unsigned long long __val;                                       \
                                                                        \
        if (sel == 0)                                                   \
-               __asm__ __volatile__(                                   \
+               __asm__ vol(                                            \
+                       ".set\tpush\n\t"                                \
                        ".set\tmips64\n\t"                              \
-                       "dmfc0\t%M0, " #source "\n\t"                   \
-                       "dsll\t%L0, %M0, 32\n\t"                        \
-                       "dsra\t%M0, %M0, 32\n\t"                        \
-                       "dsra\t%L0, %L0, 32\n\t"                        \
-                       ".set\tmips0"                                   \
+                       "dmfc0\t%L0, " #source "\n\t"                   \
+                       "dsra\t%M0, %L0, 32\n\t"                        \
+                       "sll\t%L0, %L0, 0\n\t"                          \
+                       ".set\tpop"                                     \
                        : "=r" (__val));                                \
        else                                                            \
-               __asm__ __volatile__(                                   \
+               __asm__ vol(                                            \
+                       ".set\tpush\n\t"                                \
                        ".set\tmips64\n\t"                              \
-                       "dmfc0\t%M0, " #source ", " #sel "\n\t"         \
-                       "dsll\t%L0, %M0, 32\n\t"                        \
-                       "dsra\t%M0, %M0, 32\n\t"                        \
-                       "dsra\t%L0, %L0, 32\n\t"                        \
-                       ".set\tmips0"                                   \
+                       "dmfc0\t%L0, " #source ", " #sel "\n\t"         \
+                       "dsra\t%M0, %L0, 32\n\t"                        \
+                       "sll\t%L0, %L0, 0\n\t"                          \
+                       ".set\tpop"                                     \
                        : "=r" (__val));                                \
                                                                        \
        __val;                                                          \
@@ -1040,60 +1464,77 @@ do {                                                                    \
 
 #define __write_64bit_c0_split(source, sel, val)                       \
 do {                                                                   \
-       if (sel == 0)                                                   \
+       unsigned long long __tmp = (val);                               \
+       if (MIPS_ISA_REV >= 2)                                          \
                __asm__ __volatile__(                                   \
+                       ".set\tpush\n\t"                                \
+                       ".set\t" MIPS_ISA_LEVEL "\n\t"                  \
+                       "dins\t%L0, %M0, 32, 32\n\t"                    \
+                       "dmtc0\t%L0, " #source ", " #sel "\n\t"         \
+                       ".set\tpop"                                     \
+                       : "+r" (__tmp));                                \
+       else if (sel == 0)                                              \
+               __asm__ __volatile__(                                   \
+                       ".set\tpush\n\t"                                \
                        ".set\tmips64\n\t"                              \
                        "dsll\t%L0, %L0, 32\n\t"                        \
                        "dsrl\t%L0, %L0, 32\n\t"                        \
                        "dsll\t%M0, %M0, 32\n\t"                        \
                        "or\t%L0, %L0, %M0\n\t"                         \
                        "dmtc0\t%L0, " #source "\n\t"                   \
-                       ".set\tmips0"                                   \
-                       : : "r" (val));                                 \
+                       ".set\tpop"                                     \
+                       : "+r" (__tmp));                                \
        else                                                            \
                __asm__ __volatile__(                                   \
+                       ".set\tpush\n\t"                                \
                        ".set\tmips64\n\t"                              \
                        "dsll\t%L0, %L0, 32\n\t"                        \
                        "dsrl\t%L0, %L0, 32\n\t"                        \
                        "dsll\t%M0, %M0, 32\n\t"                        \
                        "or\t%L0, %L0, %M0\n\t"                         \
                        "dmtc0\t%L0, " #source ", " #sel "\n\t"         \
-                       ".set\tmips0"                                   \
-                       : : "r" (val));                                 \
+                       ".set\tpop"                                     \
+                       : "+r" (__tmp));                                \
 } while (0)
 
-#define __readx_32bit_c0_register(source)                              \
+#ifndef TOOLCHAIN_SUPPORTS_XPA
+_ASM_MACRO_2R_1S(mfhc0, rt, rs, sel,
+       _ASM_INSN_IF_MIPS(0x40400000 | __rt << 16 | __rs << 11 | \\sel)
+       _ASM_INSN32_IF_MM(0x000000f4 | __rt << 21 | __rs << 16 | \\sel << 11));
+_ASM_MACRO_2R_1S(mthc0, rt, rd, sel,
+       _ASM_INSN_IF_MIPS(0x40c00000 | __rt << 16 | __rd << 11 | \\sel)
+       _ASM_INSN32_IF_MM(0x000002f4 | __rt << 21 | __rd << 16 | \\sel << 11));
+#define _ASM_SET_XPA ""
+#else  /* !TOOLCHAIN_SUPPORTS_XPA */
+#define _ASM_SET_XPA ".set\txpa\n\t"
+#endif
+
+#define __readx_32bit_c0_register(source, sel)                         \
 ({                                                                     \
        unsigned int __res;                                             \
                                                                        \
        __asm__ __volatile__(                                           \
        "       .set    push                                    \n"     \
-       "       .set    noat                                    \n"     \
        "       .set    mips32r2                                \n"     \
-       "       .insn                                           \n"     \
-       "       # mfhc0 $1, %1                                  \n"     \
-       "       .word   (0x40410000 | ((%1 & 0x1f) << 11))      \n"     \
-       "       move    %0, $1                                  \n"     \
+       _ASM_SET_XPA                                                    \
+       "       mfhc0   %0, " #source ", %1                     \n"     \
        "       .set    pop                                     \n"     \
        : "=r" (__res)                                                  \
-       : "i" (source));                                                \
+       : "i" (sel));                                                   \
        __res;                                                          \
 })
 
-#define __writex_32bit_c0_register(register, value)                    \
-({                                                                     \
+#define __writex_32bit_c0_register(register, sel, value)               \
+do {                                                                   \
        __asm__ __volatile__(                                           \
        "       .set    push                                    \n"     \
-       "       .set    noat                                    \n"     \
        "       .set    mips32r2                                \n"     \
-       "       move    $1, %0                                  \n"     \
-       "       # mthc0 $1, %1                                  \n"     \
-       "       .insn                                           \n"     \
-       "       .word   (0x40c10000 | ((%1 & 0x1f) << 11))      \n"     \
+       _ASM_SET_XPA                                                    \
+       "       mthc0   %z0, " #register ", %1                  \n"     \
        "       .set    pop                                     \n"     \
        :                                                               \
-       : "r" (value), "i" (register));                                 \
-})
+       : "Jr" (value), "i" (sel));                                     \
+} while (0)
 
 #define read_c0_index()                __read_32bit_c0_register($0, 0)
 #define write_c0_index(val)    __write_32bit_c0_register($0, 0, val)
@@ -1104,24 +1545,35 @@ do {                                                                    \
 #define read_c0_entrylo0()     __read_ulong_c0_register($2, 0)
 #define write_c0_entrylo0(val) __write_ulong_c0_register($2, 0, val)
 
-#define readx_c0_entrylo0()    __readx_32bit_c0_register(2)
-#define writex_c0_entrylo0(val)        __writex_32bit_c0_register(2, val)
+#define readx_c0_entrylo0()    __readx_32bit_c0_register($2, 0)
+#define writex_c0_entrylo0(val)        __writex_32bit_c0_register($2, 0, val)
 
 #define read_c0_entrylo1()     __read_ulong_c0_register($3, 0)
 #define write_c0_entrylo1(val) __write_ulong_c0_register($3, 0, val)
 
-#define readx_c0_entrylo1()    __readx_32bit_c0_register(3)
-#define writex_c0_entrylo1(val)        __writex_32bit_c0_register(3, val)
+#define readx_c0_entrylo1()    __readx_32bit_c0_register($3, 0)
+#define writex_c0_entrylo1(val)        __writex_32bit_c0_register($3, 0, val)
 
 #define read_c0_conf()         __read_32bit_c0_register($3, 0)
 #define write_c0_conf(val)     __write_32bit_c0_register($3, 0, val)
 
+#define read_c0_globalnumber() __read_32bit_c0_register($3, 1)
+
 #define read_c0_context()      __read_ulong_c0_register($4, 0)
 #define write_c0_context(val)  __write_ulong_c0_register($4, 0, val)
 
+#define read_c0_contextconfig()                __read_32bit_c0_register($4, 1)
+#define write_c0_contextconfig(val)    __write_32bit_c0_register($4, 1, val)
+
 #define read_c0_userlocal()    __read_ulong_c0_register($4, 2)
 #define write_c0_userlocal(val) __write_ulong_c0_register($4, 2, val)
 
+#define read_c0_xcontextconfig()       __read_ulong_c0_register($4, 3)
+#define write_c0_xcontextconfig(val)   __write_ulong_c0_register($4, 3, val)
+
+#define read_c0_memorymapid()          __read_32bit_c0_register($4, 5)
+#define write_c0_memorymapid(val)      __write_32bit_c0_register($4, 5, val)
+
 #define read_c0_pagemask()     __read_32bit_c0_register($5, 0)
 #define write_c0_pagemask(val) __write_32bit_c0_register($5, 0, val)
 
@@ -1139,6 +1591,9 @@ do {                                                                      \
 #define read_c0_badvaddr()     __read_ulong_c0_register($8, 0)
 #define write_c0_badvaddr(val) __write_ulong_c0_register($8, 0, val)
 
+#define read_c0_badinstr()     __read_32bit_c0_register($8, 1)
+#define read_c0_badinstrp()    __read_32bit_c0_register($8, 2)
+
 #define read_c0_count()                __read_32bit_c0_register($9, 0)
 #define write_c0_count(val)    __write_32bit_c0_register($9, 0, val)
 
@@ -1151,9 +1606,21 @@ do {                                                                     \
 #define read_c0_entryhi()      __read_ulong_c0_register($10, 0)
 #define write_c0_entryhi(val)  __write_ulong_c0_register($10, 0, val)
 
+#define read_c0_guestctl1()    __read_32bit_c0_register($10, 4)
+#define write_c0_guestctl1(val)        __write_32bit_c0_register($10, 4, val)
+
+#define read_c0_guestctl2()    __read_32bit_c0_register($10, 5)
+#define write_c0_guestctl2(val)        __write_32bit_c0_register($10, 5, val)
+
+#define read_c0_guestctl3()    __read_32bit_c0_register($10, 6)
+#define write_c0_guestctl3(val)        __write_32bit_c0_register($10, 6, val)
+
 #define read_c0_compare()      __read_32bit_c0_register($11, 0)
 #define write_c0_compare(val)  __write_32bit_c0_register($11, 0, val)
 
+#define read_c0_guestctl0ext() __read_32bit_c0_register($11, 4)
+#define write_c0_guestctl0ext(val) __write_32bit_c0_register($11, 4, val)
+
 #define read_c0_compare2()     __read_32bit_c0_register($11, 6) /* pnx8550 */
 #define write_c0_compare2(val) __write_32bit_c0_register($11, 6, val)
 
@@ -1164,13 +1631,19 @@ do {                                                                    \
 
 #define write_c0_status(val)   __write_32bit_c0_register($12, 0, val)
 
+#define read_c0_guestctl0()    __read_32bit_c0_register($12, 6)
+#define write_c0_guestctl0(val)        __write_32bit_c0_register($12, 6, val)
+
+#define read_c0_gtoffset()     __read_32bit_c0_register($12, 7)
+#define write_c0_gtoffset(val) __write_32bit_c0_register($12, 7, val)
+
 #define read_c0_cause()                __read_32bit_c0_register($13, 0)
 #define write_c0_cause(val)    __write_32bit_c0_register($13, 0, val)
 
 #define read_c0_epc()          __read_ulong_c0_register($14, 0)
 #define write_c0_epc(val)      __write_ulong_c0_register($14, 0, val)
 
-#define read_c0_prid()         __read_32bit_c0_register($15, 0)
+#define read_c0_prid()         __read_const_32bit_c0_register($15, 0)
 
 #define read_c0_cmgcrbase()    __read_ulong_c0_register($15, 3)
 
@@ -1349,6 +1822,9 @@ do {                                                                      \
 #define read_c0_ebase()                __read_32bit_c0_register($15, 1)
 #define write_c0_ebase(val)    __write_32bit_c0_register($15, 1, val)
 
+#define read_c0_ebase_64()     __read_64bit_c0_register($15, 1)
+#define write_c0_ebase_64(val) __write_64bit_c0_register($15, 1, val)
+
 #define read_c0_cdmmbase()     __read_ulong_c0_register($15, 2)
 #define write_c0_cdmmbase(val) __write_ulong_c0_register($15, 2, val)
 
@@ -1375,6 +1851,12 @@ do {                                                                     \
 #define read_c0_pwctl()                __read_32bit_c0_register($6, 6)
 #define write_c0_pwctl(val)    __write_32bit_c0_register($6, 6, val)
 
+#define read_c0_pgd()          __read_64bit_c0_register($9, 7)
+#define write_c0_pgd(val)      __write_64bit_c0_register($9, 7, val)
+
+#define read_c0_kpgd()         __read_64bit_c0_register($31, 7)
+#define write_c0_kpgd(val)     __write_64bit_c0_register($31, 7, val)
+
 /* Cavium OCTEON (cnMIPS) */
 #define read_c0_cvmcount()     __read_ulong_c0_register($9, 6)
 #define write_c0_cvmcount(val) __write_ulong_c0_register($9, 6, val)
@@ -1384,6 +1866,13 @@ do {                                                                     \
 
 #define read_c0_cvmmemctl()    __read_64bit_c0_register($11, 7)
 #define write_c0_cvmmemctl(val) __write_64bit_c0_register($11, 7, val)
+
+#define read_c0_cvmmemctl2()   __read_64bit_c0_register($16, 6)
+#define write_c0_cvmmemctl2(val) __write_64bit_c0_register($16, 6, val)
+
+#define read_c0_cvmvmconfig()  __read_64bit_c0_register($16, 7)
+#define write_c0_cvmvmconfig(val) __write_64bit_c0_register($16, 7, val)
+
 /*
  * The cacheerr registers are not standardized.         On OCTEON, they are
  * 64 bits wide.
@@ -1439,6 +1928,306 @@ do {                                                                    \
 #define read_c0_brcm_sleepcount()      __read_32bit_c0_register($22, 7)
 #define write_c0_brcm_sleepcount(val)  __write_32bit_c0_register($22, 7, val)
 
+/* Ingenic page ctrl register */
+#define write_c0_page_ctrl(val)        __write_32bit_c0_register($5, 4, val)
+
+/*
+ * Macros to access the guest system control coprocessor
+ */
+
+#ifndef TOOLCHAIN_SUPPORTS_VIRT
+_ASM_MACRO_2R_1S(mfgc0, rt, rs, sel,
+       _ASM_INSN_IF_MIPS(0x40600000 | __rt << 16 | __rs << 11 | \\sel)
+       _ASM_INSN32_IF_MM(0x000004fc | __rt << 21 | __rs << 16 | \\sel << 11));
+_ASM_MACRO_2R_1S(dmfgc0, rt, rs, sel,
+       _ASM_INSN_IF_MIPS(0x40600100 | __rt << 16 | __rs << 11 | \\sel)
+       _ASM_INSN32_IF_MM(0x580004fc | __rt << 21 | __rs << 16 | \\sel << 11));
+_ASM_MACRO_2R_1S(mtgc0, rt, rd, sel,
+       _ASM_INSN_IF_MIPS(0x40600200 | __rt << 16 | __rd << 11 | \\sel)
+       _ASM_INSN32_IF_MM(0x000006fc | __rt << 21 | __rd << 16 | \\sel << 11));
+_ASM_MACRO_2R_1S(dmtgc0, rt, rd, sel,
+       _ASM_INSN_IF_MIPS(0x40600300 | __rt << 16 | __rd << 11 | \\sel)
+       _ASM_INSN32_IF_MM(0x580006fc | __rt << 21 | __rd << 16 | \\sel << 11));
+_ASM_MACRO_0(tlbgp,    _ASM_INSN_IF_MIPS(0x42000010)
+                      _ASM_INSN32_IF_MM(0x0000017c));
+_ASM_MACRO_0(tlbgr,    _ASM_INSN_IF_MIPS(0x42000009)
+                      _ASM_INSN32_IF_MM(0x0000117c));
+_ASM_MACRO_0(tlbgwi,   _ASM_INSN_IF_MIPS(0x4200000a)
+                      _ASM_INSN32_IF_MM(0x0000217c));
+_ASM_MACRO_0(tlbgwr,   _ASM_INSN_IF_MIPS(0x4200000e)
+                      _ASM_INSN32_IF_MM(0x0000317c));
+_ASM_MACRO_0(tlbginvf, _ASM_INSN_IF_MIPS(0x4200000c)
+                      _ASM_INSN32_IF_MM(0x0000517c));
+#define _ASM_SET_VIRT ""
+#else  /* !TOOLCHAIN_SUPPORTS_VIRT */
+#define _ASM_SET_VIRT ".set\tvirt\n\t"
+#endif
+
+#define __read_32bit_gc0_register(source, sel)                         \
+({ int __res;                                                          \
+       __asm__ __volatile__(                                           \
+               ".set\tpush\n\t"                                        \
+               ".set\tmips32r2\n\t"                                    \
+               _ASM_SET_VIRT                                           \
+               "mfgc0\t%0, " #source ", %1\n\t"                        \
+               ".set\tpop"                                             \
+               : "=r" (__res)                                          \
+               : "i" (sel));                                           \
+       __res;                                                          \
+})
+
+#define __read_64bit_gc0_register(source, sel)                         \
+({ unsigned long long __res;                                           \
+       __asm__ __volatile__(                                           \
+               ".set\tpush\n\t"                                        \
+               ".set\tmips64r2\n\t"                                    \
+               _ASM_SET_VIRT                                           \
+               "dmfgc0\t%0, " #source ", %1\n\t"                       \
+               ".set\tpop"                                             \
+               : "=r" (__res)                                          \
+               : "i" (sel));                                           \
+       __res;                                                          \
+})
+
+#define __write_32bit_gc0_register(register, sel, value)               \
+do {                                                                   \
+       __asm__ __volatile__(                                           \
+               ".set\tpush\n\t"                                        \
+               ".set\tmips32r2\n\t"                                    \
+               _ASM_SET_VIRT                                           \
+               "mtgc0\t%z0, " #register ", %1\n\t"                     \
+               ".set\tpop"                                             \
+               : : "Jr" ((unsigned int)(value)),                       \
+                   "i" (sel));                                         \
+} while (0)
+
+#define __write_64bit_gc0_register(register, sel, value)               \
+do {                                                                   \
+       __asm__ __volatile__(                                           \
+               ".set\tpush\n\t"                                        \
+               ".set\tmips64r2\n\t"                                    \
+               _ASM_SET_VIRT                                           \
+               "dmtgc0\t%z0, " #register ", %1\n\t"                    \
+               ".set\tpop"                                             \
+               : : "Jr" (value),                                       \
+                   "i" (sel));                                         \
+} while (0)
+
+#define __read_ulong_gc0_register(reg, sel)                            \
+       ((sizeof(unsigned long) == 4) ?                                 \
+       (unsigned long) __read_32bit_gc0_register(reg, sel) :           \
+       (unsigned long) __read_64bit_gc0_register(reg, sel))
+
+#define __write_ulong_gc0_register(reg, sel, val)                      \
+do {                                                                   \
+       if (sizeof(unsigned long) == 4)                                 \
+               __write_32bit_gc0_register(reg, sel, val);              \
+       else                                                            \
+               __write_64bit_gc0_register(reg, sel, val);              \
+} while (0)
+
+#define read_gc0_index()               __read_32bit_gc0_register($0, 0)
+#define write_gc0_index(val)           __write_32bit_gc0_register($0, 0, val)
+
+#define read_gc0_entrylo0()            __read_ulong_gc0_register($2, 0)
+#define write_gc0_entrylo0(val)                __write_ulong_gc0_register($2, 0, val)
+
+#define read_gc0_entrylo1()            __read_ulong_gc0_register($3, 0)
+#define write_gc0_entrylo1(val)                __write_ulong_gc0_register($3, 0, val)
+
+#define read_gc0_context()             __read_ulong_gc0_register($4, 0)
+#define write_gc0_context(val)         __write_ulong_gc0_register($4, 0, val)
+
+#define read_gc0_contextconfig()       __read_32bit_gc0_register($4, 1)
+#define write_gc0_contextconfig(val)   __write_32bit_gc0_register($4, 1, val)
+
+#define read_gc0_userlocal()           __read_ulong_gc0_register($4, 2)
+#define write_gc0_userlocal(val)       __write_ulong_gc0_register($4, 2, val)
+
+#define read_gc0_xcontextconfig()      __read_ulong_gc0_register($4, 3)
+#define write_gc0_xcontextconfig(val)  __write_ulong_gc0_register($4, 3, val)
+
+#define read_gc0_pagemask()            __read_32bit_gc0_register($5, 0)
+#define write_gc0_pagemask(val)                __write_32bit_gc0_register($5, 0, val)
+
+#define read_gc0_pagegrain()           __read_32bit_gc0_register($5, 1)
+#define write_gc0_pagegrain(val)       __write_32bit_gc0_register($5, 1, val)
+
+#define read_gc0_segctl0()             __read_ulong_gc0_register($5, 2)
+#define write_gc0_segctl0(val)         __write_ulong_gc0_register($5, 2, val)
+
+#define read_gc0_segctl1()             __read_ulong_gc0_register($5, 3)
+#define write_gc0_segctl1(val)         __write_ulong_gc0_register($5, 3, val)
+
+#define read_gc0_segctl2()             __read_ulong_gc0_register($5, 4)
+#define write_gc0_segctl2(val)         __write_ulong_gc0_register($5, 4, val)
+
+#define read_gc0_pwbase()              __read_ulong_gc0_register($5, 5)
+#define write_gc0_pwbase(val)          __write_ulong_gc0_register($5, 5, val)
+
+#define read_gc0_pwfield()             __read_ulong_gc0_register($5, 6)
+#define write_gc0_pwfield(val)         __write_ulong_gc0_register($5, 6, val)
+
+#define read_gc0_pwsize()              __read_ulong_gc0_register($5, 7)
+#define write_gc0_pwsize(val)          __write_ulong_gc0_register($5, 7, val)
+
+#define read_gc0_wired()               __read_32bit_gc0_register($6, 0)
+#define write_gc0_wired(val)           __write_32bit_gc0_register($6, 0, val)
+
+#define read_gc0_pwctl()               __read_32bit_gc0_register($6, 6)
+#define write_gc0_pwctl(val)           __write_32bit_gc0_register($6, 6, val)
+
+#define read_gc0_hwrena()              __read_32bit_gc0_register($7, 0)
+#define write_gc0_hwrena(val)          __write_32bit_gc0_register($7, 0, val)
+
+#define read_gc0_badvaddr()            __read_ulong_gc0_register($8, 0)
+#define write_gc0_badvaddr(val)                __write_ulong_gc0_register($8, 0, val)
+
+#define read_gc0_badinstr()            __read_32bit_gc0_register($8, 1)
+#define write_gc0_badinstr(val)                __write_32bit_gc0_register($8, 1, val)
+
+#define read_gc0_badinstrp()           __read_32bit_gc0_register($8, 2)
+#define write_gc0_badinstrp(val)       __write_32bit_gc0_register($8, 2, val)
+
+#define read_gc0_count()               __read_32bit_gc0_register($9, 0)
+
+#define read_gc0_entryhi()             __read_ulong_gc0_register($10, 0)
+#define write_gc0_entryhi(val)         __write_ulong_gc0_register($10, 0, val)
+
+#define read_gc0_compare()             __read_32bit_gc0_register($11, 0)
+#define write_gc0_compare(val)         __write_32bit_gc0_register($11, 0, val)
+
+#define read_gc0_status()              __read_32bit_gc0_register($12, 0)
+#define write_gc0_status(val)          __write_32bit_gc0_register($12, 0, val)
+
+#define read_gc0_intctl()              __read_32bit_gc0_register($12, 1)
+#define write_gc0_intctl(val)          __write_32bit_gc0_register($12, 1, val)
+
+#define read_gc0_cause()               __read_32bit_gc0_register($13, 0)
+#define write_gc0_cause(val)           __write_32bit_gc0_register($13, 0, val)
+
+#define read_gc0_epc()                 __read_ulong_gc0_register($14, 0)
+#define write_gc0_epc(val)             __write_ulong_gc0_register($14, 0, val)
+
+#define read_gc0_prid()                        __read_32bit_gc0_register($15, 0)
+
+#define read_gc0_ebase()               __read_32bit_gc0_register($15, 1)
+#define write_gc0_ebase(val)           __write_32bit_gc0_register($15, 1, val)
+
+#define read_gc0_ebase_64()            __read_64bit_gc0_register($15, 1)
+#define write_gc0_ebase_64(val)                __write_64bit_gc0_register($15, 1, val)
+
+#define read_gc0_config()              __read_32bit_gc0_register($16, 0)
+#define read_gc0_config1()             __read_32bit_gc0_register($16, 1)
+#define read_gc0_config2()             __read_32bit_gc0_register($16, 2)
+#define read_gc0_config3()             __read_32bit_gc0_register($16, 3)
+#define read_gc0_config4()             __read_32bit_gc0_register($16, 4)
+#define read_gc0_config5()             __read_32bit_gc0_register($16, 5)
+#define read_gc0_config6()             __read_32bit_gc0_register($16, 6)
+#define read_gc0_config7()             __read_32bit_gc0_register($16, 7)
+#define write_gc0_config(val)          __write_32bit_gc0_register($16, 0, val)
+#define write_gc0_config1(val)         __write_32bit_gc0_register($16, 1, val)
+#define write_gc0_config2(val)         __write_32bit_gc0_register($16, 2, val)
+#define write_gc0_config3(val)         __write_32bit_gc0_register($16, 3, val)
+#define write_gc0_config4(val)         __write_32bit_gc0_register($16, 4, val)
+#define write_gc0_config5(val)         __write_32bit_gc0_register($16, 5, val)
+#define write_gc0_config6(val)         __write_32bit_gc0_register($16, 6, val)
+#define write_gc0_config7(val)         __write_32bit_gc0_register($16, 7, val)
+
+#define read_gc0_lladdr()              __read_ulong_gc0_register($17, 0)
+#define write_gc0_lladdr(val)          __write_ulong_gc0_register($17, 0, val)
+
+#define read_gc0_watchlo0()            __read_ulong_gc0_register($18, 0)
+#define read_gc0_watchlo1()            __read_ulong_gc0_register($18, 1)
+#define read_gc0_watchlo2()            __read_ulong_gc0_register($18, 2)
+#define read_gc0_watchlo3()            __read_ulong_gc0_register($18, 3)
+#define read_gc0_watchlo4()            __read_ulong_gc0_register($18, 4)
+#define read_gc0_watchlo5()            __read_ulong_gc0_register($18, 5)
+#define read_gc0_watchlo6()            __read_ulong_gc0_register($18, 6)
+#define read_gc0_watchlo7()            __read_ulong_gc0_register($18, 7)
+#define write_gc0_watchlo0(val)                __write_ulong_gc0_register($18, 0, val)
+#define write_gc0_watchlo1(val)                __write_ulong_gc0_register($18, 1, val)
+#define write_gc0_watchlo2(val)                __write_ulong_gc0_register($18, 2, val)
+#define write_gc0_watchlo3(val)                __write_ulong_gc0_register($18, 3, val)
+#define write_gc0_watchlo4(val)                __write_ulong_gc0_register($18, 4, val)
+#define write_gc0_watchlo5(val)                __write_ulong_gc0_register($18, 5, val)
+#define write_gc0_watchlo6(val)                __write_ulong_gc0_register($18, 6, val)
+#define write_gc0_watchlo7(val)                __write_ulong_gc0_register($18, 7, val)
+
+#define read_gc0_watchhi0()            __read_32bit_gc0_register($19, 0)
+#define read_gc0_watchhi1()            __read_32bit_gc0_register($19, 1)
+#define read_gc0_watchhi2()            __read_32bit_gc0_register($19, 2)
+#define read_gc0_watchhi3()            __read_32bit_gc0_register($19, 3)
+#define read_gc0_watchhi4()            __read_32bit_gc0_register($19, 4)
+#define read_gc0_watchhi5()            __read_32bit_gc0_register($19, 5)
+#define read_gc0_watchhi6()            __read_32bit_gc0_register($19, 6)
+#define read_gc0_watchhi7()            __read_32bit_gc0_register($19, 7)
+#define write_gc0_watchhi0(val)                __write_32bit_gc0_register($19, 0, val)
+#define write_gc0_watchhi1(val)                __write_32bit_gc0_register($19, 1, val)
+#define write_gc0_watchhi2(val)                __write_32bit_gc0_register($19, 2, val)
+#define write_gc0_watchhi3(val)                __write_32bit_gc0_register($19, 3, val)
+#define write_gc0_watchhi4(val)                __write_32bit_gc0_register($19, 4, val)
+#define write_gc0_watchhi5(val)                __write_32bit_gc0_register($19, 5, val)
+#define write_gc0_watchhi6(val)                __write_32bit_gc0_register($19, 6, val)
+#define write_gc0_watchhi7(val)                __write_32bit_gc0_register($19, 7, val)
+
+#define read_gc0_xcontext()            __read_ulong_gc0_register($20, 0)
+#define write_gc0_xcontext(val)                __write_ulong_gc0_register($20, 0, val)
+
+#define read_gc0_perfctrl0()           __read_32bit_gc0_register($25, 0)
+#define write_gc0_perfctrl0(val)       __write_32bit_gc0_register($25, 0, val)
+#define read_gc0_perfcntr0()           __read_32bit_gc0_register($25, 1)
+#define write_gc0_perfcntr0(val)       __write_32bit_gc0_register($25, 1, val)
+#define read_gc0_perfcntr0_64()                __read_64bit_gc0_register($25, 1)
+#define write_gc0_perfcntr0_64(val)    __write_64bit_gc0_register($25, 1, val)
+#define read_gc0_perfctrl1()           __read_32bit_gc0_register($25, 2)
+#define write_gc0_perfctrl1(val)       __write_32bit_gc0_register($25, 2, val)
+#define read_gc0_perfcntr1()           __read_32bit_gc0_register($25, 3)
+#define write_gc0_perfcntr1(val)       __write_32bit_gc0_register($25, 3, val)
+#define read_gc0_perfcntr1_64()                __read_64bit_gc0_register($25, 3)
+#define write_gc0_perfcntr1_64(val)    __write_64bit_gc0_register($25, 3, val)
+#define read_gc0_perfctrl2()           __read_32bit_gc0_register($25, 4)
+#define write_gc0_perfctrl2(val)       __write_32bit_gc0_register($25, 4, val)
+#define read_gc0_perfcntr2()           __read_32bit_gc0_register($25, 5)
+#define write_gc0_perfcntr2(val)       __write_32bit_gc0_register($25, 5, val)
+#define read_gc0_perfcntr2_64()                __read_64bit_gc0_register($25, 5)
+#define write_gc0_perfcntr2_64(val)    __write_64bit_gc0_register($25, 5, val)
+#define read_gc0_perfctrl3()           __read_32bit_gc0_register($25, 6)
+#define write_gc0_perfctrl3(val)       __write_32bit_gc0_register($25, 6, val)
+#define read_gc0_perfcntr3()           __read_32bit_gc0_register($25, 7)
+#define write_gc0_perfcntr3(val)       __write_32bit_gc0_register($25, 7, val)
+#define read_gc0_perfcntr3_64()                __read_64bit_gc0_register($25, 7)
+#define write_gc0_perfcntr3_64(val)    __write_64bit_gc0_register($25, 7, val)
+
+#define read_gc0_errorepc()            __read_ulong_gc0_register($30, 0)
+#define write_gc0_errorepc(val)                __write_ulong_gc0_register($30, 0, val)
+
+#define read_gc0_kscratch1()           __read_ulong_gc0_register($31, 2)
+#define read_gc0_kscratch2()           __read_ulong_gc0_register($31, 3)
+#define read_gc0_kscratch3()           __read_ulong_gc0_register($31, 4)
+#define read_gc0_kscratch4()           __read_ulong_gc0_register($31, 5)
+#define read_gc0_kscratch5()           __read_ulong_gc0_register($31, 6)
+#define read_gc0_kscratch6()           __read_ulong_gc0_register($31, 7)
+#define write_gc0_kscratch1(val)       __write_ulong_gc0_register($31, 2, val)
+#define write_gc0_kscratch2(val)       __write_ulong_gc0_register($31, 3, val)
+#define write_gc0_kscratch3(val)       __write_ulong_gc0_register($31, 4, val)
+#define write_gc0_kscratch4(val)       __write_ulong_gc0_register($31, 5, val)
+#define write_gc0_kscratch5(val)       __write_ulong_gc0_register($31, 6, val)
+#define write_gc0_kscratch6(val)       __write_ulong_gc0_register($31, 7, val)
+
+/* Cavium OCTEON (cnMIPS) */
+#define read_gc0_cvmcount()            __read_ulong_gc0_register($9, 6)
+#define write_gc0_cvmcount(val)                __write_ulong_gc0_register($9, 6, val)
+
+#define read_gc0_cvmctl()              __read_64bit_gc0_register($9, 7)
+#define write_gc0_cvmctl(val)          __write_64bit_gc0_register($9, 7, val)
+
+#define read_gc0_cvmmemctl()           __read_64bit_gc0_register($11, 7)
+#define write_gc0_cvmmemctl(val)       __write_64bit_gc0_register($11, 7, val)
+
+#define read_gc0_cvmmemctl2()          __read_64bit_gc0_register($16, 6)
+#define write_gc0_cvmmemctl2(val)      __write_64bit_gc0_register($16, 6, val)
+
 /*
  * Macros to access the floating point coprocessor control registers
  */
@@ -1460,7 +2249,7 @@ do {                                                                      \
 })
 
 #define _write_32bit_cp1_register(dest, val, gas_hardfloat)            \
-({                                                                     \
+do {                                                                   \
        __asm__ __volatile__(                                           \
        "       .set    push                                    \n"     \
        "       .set    reorder                                 \n"     \
@@ -1468,7 +2257,7 @@ do {                                                                      \
        "       ctc1    %0,"STR(dest)"                          \n"     \
        "       .set    pop                                     \n"     \
        : : "r" (val));                                                 \
-})
+} while (0)
 
 #ifdef GAS_HAS_SET_HARDFLOAT
 #define read_32bit_cp1_register(source)                                        \
@@ -1482,13 +2271,14 @@ do {                                                                    \
        _write_32bit_cp1_register(dest, val, )
 #endif
 
-#ifdef HAVE_AS_DSP
+#ifdef TOOLCHAIN_SUPPORTS_DSP
 #define rddsp(mask)                                                    \
 ({                                                                     \
        unsigned int __dspctl;                                          \
                                                                        \
        __asm__ __volatile__(                                           \
        "       .set push                                       \n"     \
+       "       .set " MIPS_ISA_LEVEL "                         \n"     \
        "       .set dsp                                        \n"     \
        "       rddsp   %0, %x1                                 \n"     \
        "       .set pop                                        \n"     \
@@ -1498,25 +2288,27 @@ do {                                                                    \
 })
 
 #define wrdsp(val, mask)                                               \
-({                                                                     \
+do {                                                                   \
        __asm__ __volatile__(                                           \
        "       .set push                                       \n"     \
+       "       .set " MIPS_ISA_LEVEL "                         \n"     \
        "       .set dsp                                        \n"     \
        "       wrdsp   %0, %x1                                 \n"     \
        "       .set pop                                        \n"     \
        :                                                               \
        : "r" (val), "i" (mask));                                       \
-})
+} while (0)
 
 #define mflo0()                                                                \
 ({                                                                     \
        long mflo0;                                                     \
        __asm__(                                                        \
        "       .set push                                       \n"     \
+       "       .set " MIPS_ISA_LEVEL "                         \n"     \
        "       .set dsp                                        \n"     \
        "       mflo %0, $ac0                                   \n"     \
-       "       .set pop                                        \n"     \
-       : "=r" (mflo0));                                                \
+       "       .set pop                                        \n"     \
+       : "=r" (mflo0));                                                \
        mflo0;                                                          \
 })
 
@@ -1525,10 +2317,11 @@ do {                                                                    \
        long mflo1;                                                     \
        __asm__(                                                        \
        "       .set push                                       \n"     \
+       "       .set " MIPS_ISA_LEVEL "                         \n"     \
        "       .set dsp                                        \n"     \
        "       mflo %0, $ac1                                   \n"     \
-       "       .set pop                                        \n"     \
-       : "=r" (mflo1));                                                \
+       "       .set pop                                        \n"     \
+       : "=r" (mflo1));                                                \
        mflo1;                                                          \
 })
 
@@ -1537,10 +2330,11 @@ do {                                                                    \
        long mflo2;                                                     \
        __asm__(                                                        \
        "       .set push                                       \n"     \
+       "       .set " MIPS_ISA_LEVEL "                         \n"     \
        "       .set dsp                                        \n"     \
        "       mflo %0, $ac2                                   \n"     \
-       "       .set pop                                        \n"     \
-       : "=r" (mflo2));                                                \
+       "       .set pop                                        \n"     \
+       : "=r" (mflo2));                                                \
        mflo2;                                                          \
 })
 
@@ -1549,10 +2343,11 @@ do {                                                                    \
        long mflo3;                                                     \
        __asm__(                                                        \
        "       .set push                                       \n"     \
+       "       .set " MIPS_ISA_LEVEL "                         \n"     \
        "       .set dsp                                        \n"     \
        "       mflo %0, $ac3                                   \n"     \
-       "       .set pop                                        \n"     \
-       : "=r" (mflo3));                                                \
+       "       .set pop                                        \n"     \
+       : "=r" (mflo3));                                                \
        mflo3;                                                          \
 })
 
@@ -1561,10 +2356,11 @@ do {                                                                    \
        long mfhi0;                                                     \
        __asm__(                                                        \
        "       .set push                                       \n"     \
+       "       .set " MIPS_ISA_LEVEL "                         \n"     \
        "       .set dsp                                        \n"     \
        "       mfhi %0, $ac0                                   \n"     \
-       "       .set pop                                        \n"     \
-       : "=r" (mfhi0));                                                \
+       "       .set pop                                        \n"     \
+       : "=r" (mfhi0));                                                \
        mfhi0;                                                          \
 })
 
@@ -1573,10 +2369,11 @@ do {                                                                    \
        long mfhi1;                                                     \
        __asm__(                                                        \
        "       .set push                                       \n"     \
+       "       .set " MIPS_ISA_LEVEL "                         \n"     \
        "       .set dsp                                        \n"     \
        "       mfhi %0, $ac1                                   \n"     \
-       "       .set pop                                        \n"     \
-       : "=r" (mfhi1));                                                \
+       "       .set pop                                        \n"     \
+       : "=r" (mfhi1));                                                \
        mfhi1;                                                          \
 })
 
@@ -1585,10 +2382,11 @@ do {                                                                    \
        long mfhi2;                                                     \
        __asm__(                                                        \
        "       .set push                                       \n"     \
+       "       .set " MIPS_ISA_LEVEL "                         \n"     \
        "       .set dsp                                        \n"     \
        "       mfhi %0, $ac2                                   \n"     \
-       "       .set pop                                        \n"     \
-       : "=r" (mfhi2));                                                \
+       "       .set pop                                        \n"     \
+       : "=r" (mfhi2));                                                \
        mfhi2;                                                          \
 })
 
@@ -1597,10 +2395,11 @@ do {                                                                    \
        long mfhi3;                                                     \
        __asm__(                                                        \
        "       .set push                                       \n"     \
+       "       .set " MIPS_ISA_LEVEL "                         \n"     \
        "       .set dsp                                        \n"     \
        "       mfhi %0, $ac3                                   \n"     \
-       "       .set pop                                        \n"     \
-       : "=r" (mfhi3));                                                \
+       "       .set pop                                        \n"     \
+       : "=r" (mfhi3));                                                \
        mfhi3;                                                          \
 })
 
@@ -1609,6 +2408,7 @@ do {                                                                      \
 ({                                                                     \
        __asm__(                                                        \
        "       .set push                                       \n"     \
+       "       .set " MIPS_ISA_LEVEL "                         \n"     \
        "       .set dsp                                        \n"     \
        "       mtlo %0, $ac0                                   \n"     \
        "       .set pop                                        \n"     \
@@ -1620,6 +2420,7 @@ do {                                                                      \
 ({                                                                     \
        __asm__(                                                        \
        "       .set push                                       \n"     \
+       "       .set " MIPS_ISA_LEVEL "                         \n"     \
        "       .set dsp                                        \n"     \
        "       mtlo %0, $ac1                                   \n"     \
        "       .set pop                                        \n"     \
@@ -1631,6 +2432,7 @@ do {                                                                      \
 ({                                                                     \
        __asm__(                                                        \
        "       .set push                                       \n"     \
+       "       .set " MIPS_ISA_LEVEL "                         \n"     \
        "       .set dsp                                        \n"     \
        "       mtlo %0, $ac2                                   \n"     \
        "       .set pop                                        \n"     \
@@ -1642,6 +2444,7 @@ do {                                                                      \
 ({                                                                     \
        __asm__(                                                        \
        "       .set push                                       \n"     \
+       "       .set " MIPS_ISA_LEVEL "                         \n"     \
        "       .set dsp                                        \n"     \
        "       mtlo %0, $ac3                                   \n"     \
        "       .set pop                                        \n"     \
@@ -1653,6 +2456,7 @@ do {                                                                      \
 ({                                                                     \
        __asm__(                                                        \
        "       .set push                                       \n"     \
+       "       .set " MIPS_ISA_LEVEL "                         \n"     \
        "       .set dsp                                        \n"     \
        "       mthi %0, $ac0                                   \n"     \
        "       .set pop                                        \n"     \
@@ -1664,6 +2468,7 @@ do {                                                                      \
 ({                                                                     \
        __asm__(                                                        \
        "       .set push                                       \n"     \
+       "       .set " MIPS_ISA_LEVEL "                         \n"     \
        "       .set dsp                                        \n"     \
        "       mthi %0, $ac1                                   \n"     \
        "       .set pop                                        \n"     \
@@ -1675,6 +2480,7 @@ do {                                                                      \
 ({                                                                     \
        __asm__(                                                        \
        "       .set push                                       \n"     \
+       "       .set " MIPS_ISA_LEVEL "                         \n"     \
        "       .set dsp                                        \n"     \
        "       mthi %0, $ac2                                   \n"     \
        "       .set pop                                        \n"     \
@@ -1686,6 +2492,7 @@ do {                                                                      \
 ({                                                                     \
        __asm__(                                                        \
        "       .set push                                       \n"     \
+       "       .set " MIPS_ISA_LEVEL "                         \n"     \
        "       .set dsp                                        \n"     \
        "       mthi %0, $ac3                                   \n"     \
        "       .set pop                                        \n"     \
@@ -1695,7 +2502,6 @@ do {                                                                      \
 
 #else
 
-#ifdef CONFIG_CPU_MICROMIPS
 #define rddsp(mask)                                                    \
 ({                                                                     \
        unsigned int __res;                                             \
@@ -1704,8 +2510,8 @@ do {                                                                      \
        "       .set    push                                    \n"     \
        "       .set    noat                                    \n"     \
        "       # rddsp $1, %x1                                 \n"     \
-       "       .hword  ((0x0020067c | (%x1 << 14)) >> 16)      \n"     \
-       "       .hword  ((0x0020067c | (%x1 << 14)) & 0xffff)   \n"     \
+       _ASM_INSN_IF_MIPS(0x7c000cb8 | (%x1 << 16))                     \
+       _ASM_INSN32_IF_MM(0x0020067c | (%x1 << 14))                     \
        "       move    %0, $1                                  \n"     \
        "       .set    pop                                     \n"     \
        : "=r" (__res)                                                  \
@@ -1714,28 +2520,28 @@ do {                                                                    \
 })
 
 #define wrdsp(val, mask)                                               \
-({                                                                     \
+do {                                                                   \
        __asm__ __volatile__(                                           \
        "       .set    push                                    \n"     \
        "       .set    noat                                    \n"     \
        "       move    $1, %0                                  \n"     \
        "       # wrdsp $1, %x1                                 \n"     \
-       "       .hword  ((0x0020167c | (%x1 << 14)) >> 16)      \n"     \
-       "       .hword  ((0x0020167c | (%x1 << 14)) & 0xffff)   \n"     \
+       _ASM_INSN_IF_MIPS(0x7c2004f8 | (%x1 << 11))                     \
+       _ASM_INSN32_IF_MM(0x0020167c | (%x1 << 14))                     \
        "       .set    pop                                     \n"     \
        :                                                               \
        : "r" (val), "i" (mask));                                       \
-})
+} while (0)
 
-#define _umips_dsp_mfxxx(ins)                                          \
+#define _dsp_mfxxx(ins)                                                        \
 ({                                                                     \
        unsigned long __treg;                                           \
                                                                        \
        __asm__ __volatile__(                                           \
        "       .set    push                                    \n"     \
        "       .set    noat                                    \n"     \
-       "       .hword  0x0001                                  \n"     \
-       "       .hword  %x1                                     \n"     \
+       _ASM_INSN_IF_MIPS(0x00000810 | %X1)                             \
+       _ASM_INSN32_IF_MM(0x0001007c | %x1)                             \
        "       move    %0, $1                                  \n"     \
        "       .set    pop                                     \n"     \
        : "=r" (__treg)                                                 \
@@ -1743,101 +2549,28 @@ do {                                                                   \
        __treg;                                                         \
 })
 
-#define _umips_dsp_mtxxx(val, ins)                                     \
-({                                                                     \
+#define _dsp_mtxxx(val, ins)                                           \
+do {                                                                   \
        __asm__ __volatile__(                                           \
        "       .set    push                                    \n"     \
        "       .set    noat                                    \n"     \
        "       move    $1, %0                                  \n"     \
-       "       .hword  0x0001                                  \n"     \
-       "       .hword  %x1                                     \n"     \
+       _ASM_INSN_IF_MIPS(0x00200011 | %X1)                             \
+       _ASM_INSN32_IF_MM(0x0001207c | %x1)                             \
        "       .set    pop                                     \n"     \
        :                                                               \
        : "r" (val), "i" (ins));                                        \
-})
-
-#define _umips_dsp_mflo(reg) _umips_dsp_mfxxx((reg << 14) | 0x107c)
-#define _umips_dsp_mfhi(reg) _umips_dsp_mfxxx((reg << 14) | 0x007c)
-
-#define _umips_dsp_mtlo(val, reg) _umips_dsp_mtxxx(val, ((reg << 14) | 0x307c))
-#define _umips_dsp_mthi(val, reg) _umips_dsp_mtxxx(val, ((reg << 14) | 0x207c))
-
-#define mflo0() _umips_dsp_mflo(0)
-#define mflo1() _umips_dsp_mflo(1)
-#define mflo2() _umips_dsp_mflo(2)
-#define mflo3() _umips_dsp_mflo(3)
+} while (0)
 
-#define mfhi0() _umips_dsp_mfhi(0)
-#define mfhi1() _umips_dsp_mfhi(1)
-#define mfhi2() _umips_dsp_mfhi(2)
-#define mfhi3() _umips_dsp_mfhi(3)
+#ifdef CONFIG_CPU_MICROMIPS
 
-#define mtlo0(x) _umips_dsp_mtlo(x, 0)
-#define mtlo1(x) _umips_dsp_mtlo(x, 1)
-#define mtlo2(x) _umips_dsp_mtlo(x, 2)
-#define mtlo3(x) _umips_dsp_mtlo(x, 3)
+#define _dsp_mflo(reg) _dsp_mfxxx((reg << 14) | 0x1000)
+#define _dsp_mfhi(reg) _dsp_mfxxx((reg << 14) | 0x0000)
 
-#define mthi0(x) _umips_dsp_mthi(x, 0)
-#define mthi1(x) _umips_dsp_mthi(x, 1)
-#define mthi2(x) _umips_dsp_mthi(x, 2)
-#define mthi3(x) _umips_dsp_mthi(x, 3)
+#define _dsp_mtlo(val, reg) _dsp_mtxxx(val, ((reg << 14) | 0x1000))
+#define _dsp_mthi(val, reg) _dsp_mtxxx(val, ((reg << 14) | 0x0000))
 
 #else  /* !CONFIG_CPU_MICROMIPS */
-#define rddsp(mask)                                                    \
-({                                                                     \
-       unsigned int __res;                                             \
-                                                                       \
-       __asm__ __volatile__(                                           \
-       "       .set    push                            \n"             \
-       "       .set    noat                            \n"             \
-       "       # rddsp $1, %x1                         \n"             \
-       "       .word   0x7c000cb8 | (%x1 << 16)        \n"             \
-       "       move    %0, $1                          \n"             \
-       "       .set    pop                             \n"             \
-       : "=r" (__res)                                                  \
-       : "i" (mask));                                                  \
-       __res;                                                          \
-})
-
-#define wrdsp(val, mask)                                               \
-({                                                                     \
-       __asm__ __volatile__(                                           \
-       "       .set    push                                    \n"     \
-       "       .set    noat                                    \n"     \
-       "       move    $1, %0                                  \n"     \
-       "       # wrdsp $1, %x1                                 \n"     \
-       "       .word   0x7c2004f8 | (%x1 << 11)                \n"     \
-       "       .set    pop                                     \n"     \
-       :                                                               \
-       : "r" (val), "i" (mask));                                       \
-})
-
-#define _dsp_mfxxx(ins)                                                        \
-({                                                                     \
-       unsigned long __treg;                                           \
-                                                                       \
-       __asm__ __volatile__(                                           \
-       "       .set    push                                    \n"     \
-       "       .set    noat                                    \n"     \
-       "       .word   (0x00000810 | %1)                       \n"     \
-       "       move    %0, $1                                  \n"     \
-       "       .set    pop                                     \n"     \
-       : "=r" (__treg)                                                 \
-       : "i" (ins));                                                   \
-       __treg;                                                         \
-})
-
-#define _dsp_mtxxx(val, ins)                                           \
-({                                                                     \
-       __asm__ __volatile__(                                           \
-       "       .set    push                                    \n"     \
-       "       .set    noat                                    \n"     \
-       "       move    $1, %0                                  \n"     \
-       "       .word   (0x00200011 | %1)                       \n"     \
-       "       .set    pop                                     \n"     \
-       :                                                               \
-       : "r" (val), "i" (ins));                                        \
-})
 
 #define _dsp_mflo(reg) _dsp_mfxxx((reg << 21) | 0x0002)
 #define _dsp_mfhi(reg) _dsp_mfxxx((reg << 21) | 0x0000)
@@ -1845,6 +2578,8 @@ do {                                                                      \
 #define _dsp_mtlo(val, reg) _dsp_mtxxx(val, ((reg << 11) | 0x0002))
 #define _dsp_mthi(val, reg) _dsp_mtxxx(val, ((reg << 11) | 0x0000))
 
+#endif /* CONFIG_CPU_MICROMIPS */
+
 #define mflo0() _dsp_mflo(0)
 #define mflo1() _dsp_mflo(1)
 #define mflo2() _dsp_mflo(2)
@@ -1865,7 +2600,6 @@ do {                                                                      \
 #define mthi2(x) _dsp_mthi(x, 2)
 #define mthi3(x) _dsp_mthi(x, 3)
 
-#endif /* CONFIG_CPU_MICROMIPS */
 #endif
 
 /*
@@ -1935,54 +2669,123 @@ static inline void tlb_write_random(void)
 }
 
 /*
- * Manipulate bits in a c0 register.
+ * Guest TLB operations.
+ *
+ * It is responsibility of the caller to take care of any TLB hazards.
  */
-#define __BUILD_SET_C0(name)                                   \
+static inline void guest_tlb_probe(void)
+{
+       __asm__ __volatile__(
+               ".set push\n\t"
+               ".set noreorder\n\t"
+               _ASM_SET_VIRT
+               "tlbgp\n\t"
+               ".set pop");
+}
+
+static inline void guest_tlb_read(void)
+{
+       __asm__ __volatile__(
+               ".set push\n\t"
+               ".set noreorder\n\t"
+               _ASM_SET_VIRT
+               "tlbgr\n\t"
+               ".set pop");
+}
+
+static inline void guest_tlb_write_indexed(void)
+{
+       __asm__ __volatile__(
+               ".set push\n\t"
+               ".set noreorder\n\t"
+               _ASM_SET_VIRT
+               "tlbgwi\n\t"
+               ".set pop");
+}
+
+static inline void guest_tlb_write_random(void)
+{
+       __asm__ __volatile__(
+               ".set push\n\t"
+               ".set noreorder\n\t"
+               _ASM_SET_VIRT
+               "tlbgwr\n\t"
+               ".set pop");
+}
+
+/*
+ * Guest TLB Invalidate Flush
+ */
+static inline void guest_tlbinvf(void)
+{
+       __asm__ __volatile__(
+               ".set push\n\t"
+               ".set noreorder\n\t"
+               _ASM_SET_VIRT
+               "tlbginvf\n\t"
+               ".set pop");
+}
+
+/*
+ * Manipulate bits in a register.
+ */
+#define __BUILD_SET_COMMON(name)                               \
 static inline unsigned int                                     \
-set_c0_##name(unsigned int set)                                        \
+set_##name(unsigned int set)                                   \
 {                                                              \
        unsigned int res, new;                                  \
                                                                \
-       res = read_c0_##name();                                 \
+       res = read_##name();                                    \
        new = res | set;                                        \
-       write_c0_##name(new);                                   \
+       write_##name(new);                                      \
                                                                \
        return res;                                             \
 }                                                              \
                                                                \
 static inline unsigned int                                     \
-clear_c0_##name(unsigned int clear)                            \
+clear_##name(unsigned int clear)                               \
 {                                                              \
        unsigned int res, new;                                  \
                                                                \
-       res = read_c0_##name();                                 \
+       res = read_##name();                                    \
        new = res & ~clear;                                     \
-       write_c0_##name(new);                                   \
+       write_##name(new);                                      \
                                                                \
        return res;                                             \
 }                                                              \
                                                                \
 static inline unsigned int                                     \
-change_c0_##name(unsigned int change, unsigned int val)                \
+change_##name(unsigned int change, unsigned int val)           \
 {                                                              \
        unsigned int res, new;                                  \
                                                                \
-       res = read_c0_##name();                                 \
+       res = read_##name();                                    \
        new = res & ~change;                                    \
        new |= (val & change);                                  \
-       write_c0_##name(new);                                   \
+       write_##name(new);                                      \
                                                                \
        return res;                                             \
 }
 
+/*
+ * Manipulate bits in a c0 register.
+ */
+#define __BUILD_SET_C0(name)   __BUILD_SET_COMMON(c0_##name)
+
 __BUILD_SET_C0(status)
 __BUILD_SET_C0(cause)
 __BUILD_SET_C0(config)
 __BUILD_SET_C0(config5)
+__BUILD_SET_C0(config7)
 __BUILD_SET_C0(intcontrol)
 __BUILD_SET_C0(intctl)
 __BUILD_SET_C0(srsmap)
 __BUILD_SET_C0(pagegrain)
+__BUILD_SET_C0(guestctl0)
+__BUILD_SET_C0(guestctl0ext)
+__BUILD_SET_C0(guestctl1)
+__BUILD_SET_C0(guestctl2)
+__BUILD_SET_C0(guestctl3)
 __BUILD_SET_C0(brcm_config_0)
 __BUILD_SET_C0(brcm_bus_pll)
 __BUILD_SET_C0(brcm_reset)
@@ -1992,12 +2795,23 @@ __BUILD_SET_C0(brcm_config)
 __BUILD_SET_C0(brcm_mode)
 
 /*
+ * Manipulate bits in a guest c0 register.
+ */
+#define __BUILD_SET_GC0(name)  __BUILD_SET_COMMON(gc0_##name)
+
+__BUILD_SET_GC0(wired)
+__BUILD_SET_GC0(status)
+__BUILD_SET_GC0(cause)
+__BUILD_SET_GC0(ebase)
+__BUILD_SET_GC0(config1)
+
+/*
  * Return low 10 bits of ebase.
  * Note that under KVM (MIPSVZ) this returns vcpu id.
  */
 static inline unsigned int get_ebase_cpunum(void)
 {
-       return read_c0_ebase() & 0x3ff;
+       return read_c0_ebase() & MIPS_EBASE_CPUNUM;
 }
 
 static inline void write_one_tlb(int index, u32 pagemask, u32 hi, u32 low0,
index 6ff9d20..df8b63f 100644 (file)
@@ -110,7 +110,7 @@ void trap_init(ulong reloc_addr)
 
        /* Set WG bit on Octeon to enable writing to bits 63:30 */
        if (IS_ENABLED(CONFIG_ARCH_OCTEON))
-               ebase |= EBASE_WG;
+               ebase |= MIPS_EBASE_WG;
 
        write_c0_ebase(ebase);
        clear_c0_status(ST0_BEV);