F: include/linux/lightnvm.h
F: include/uapi/linux/lightnvm.h
-LINUX FOR IBM pSERIES (RS/6000)
-M: Paul Mackerras <paulus@au.ibm.com>
-W: http://www.ibm.com/linux/ltc/projects/ppc
-S: Supported
-F: arch/powerpc/boot/rs6000.h
-
LINUX FOR POWERPC (32-BIT AND 64-BIT)
M: Benjamin Herrenschmidt <benh@kernel.crashing.org>
M: Paul Mackerras <paulus@samba.org>
M: Michael Ellerman <mpe@ellerman.id.au>
-W: http://www.penguinppc.org/
+W: https://github.com/linuxppc/linux/wiki
L: linuxppc-dev@lists.ozlabs.org
Q: http://patchwork.ozlabs.org/project/linuxppc-dev/list/
T: git git://git.kernel.org/pub/scm/linux/kernel/git/powerpc/linux.git
F: drivers/pci/hotplug/pnv_php.c
F: drivers/pci/hotplug/rpa*
F: drivers/scsi/ibmvscsi/
+F: tools/testing/selftests/powerpc
N: opal
N: /pmac
N: powermac
F: arch/powerpc/platforms/85xx/
LINUX FOR POWERPC PA SEMI PWRFICIENT
-M: Olof Johansson <olof@lixom.net>
L: linuxppc-dev@lists.ozlabs.org
-S: Maintained
+S: Orphan
F: arch/powerpc/platforms/pasemi/
F: drivers/*/*pasemi*
F: drivers/*/*/*pasemi*
F: drivers/net/wireless/intersil/p54/
PA SEMI ETHERNET DRIVER
-M: Olof Johansson <olof@lixom.net>
L: netdev@vger.kernel.org
-S: Maintained
+S: Orphan
F: drivers/net/ethernet/pasemi/*
PA SEMI SMBUS DRIVER
-M: Olof Johansson <olof@lixom.net>
L: linux-i2c@vger.kernel.org
-S: Maintained
+S: Orphan
F: drivers/i2c/busses/i2c-pasemi.c
PADATA PARALLEL EXECUTION MECHANISM
select VIRT_TO_BUS if !PPC64
select HAVE_IDE
select HAVE_IOREMAP_PROT
- select HAVE_EFFICIENT_UNALIGNED_ACCESS if !CPU_LITTLE_ENDIAN
+ select HAVE_EFFICIENT_UNALIGNED_ACCESS if !(CPU_LITTLE_ENDIAN && POWER7_CPU)
select HAVE_KPROBES
select HAVE_ARCH_KGDB
select HAVE_KRETPROBES
select GENERIC_CPU_AUTOPROBE
select HAVE_VIRT_CPU_ACCOUNTING
select HAVE_ARCH_HARDENED_USERCOPY
+ select HAVE_KERNEL_GZIP
config GENERIC_CSUM
def_bool CPU_LITTLE_ENDIAN
all: $(obj)/zImage
+compress-$(CONFIG_KERNEL_GZIP) := CONFIG_KERNEL_GZIP
+compress-$(CONFIG_KERNEL_XZ) := CONFIG_KERNEL_XZ
+
BOOTCFLAGS := -Wall -Wundef -Wstrict-prototypes -Wno-trigraphs \
-fno-strict-aliasing -Os -msoft-float -pipe \
-fomit-frame-pointer -fno-builtin -fPIC -nostdinc \
- -isystem $(shell $(CROSS32CC) -print-file-name=include)
+ -isystem $(shell $(CROSS32CC) -print-file-name=include) \
+ -D$(compress-y)
+
ifdef CONFIG_PPC64_BOOT_WRAPPER
BOOTCFLAGS += -m64
endif
$(obj)/treeboot-akebono.o: BOOTCFLAGS += -mcpu=405
$(obj)/virtex405-head.o: BOOTAFLAGS += -mcpu=405
+# The pre-boot decompressors pull in a lot of kernel headers and other source
+# files. This creates a bit of a dependency headache since we need to copy
+# these files into the build dir, fix up any includes and ensure that dependent
+# files are copied in the right order.
+
+# these need to be seperate variables because they are copied out of different
+# directories in the kernel tree. Sure you COULd merge them, but it's a
+# cure-is-worse-than-disease situation.
+zlib-decomp-$(CONFIG_KERNEL_GZIP) := decompress_inflate.c
+zlib-$(CONFIG_KERNEL_GZIP) := inffast.c inflate.c inftrees.c
+zlibheader-$(CONFIG_KERNEL_GZIP) := inffast.h inffixed.h inflate.h inftrees.h infutil.h
+zliblinuxheader-$(CONFIG_KERNEL_GZIP) := zlib.h zconf.h zutil.h
-zlib := inffast.c inflate.c inftrees.c
-zlibheader := inffast.h inffixed.h inflate.h inftrees.h infutil.h
-zliblinuxheader := zlib.h zconf.h zutil.h
+$(addprefix $(obj)/, decompress.o): \
+ $(addprefix $(obj)/,$(zlib-decomp-y))
-$(addprefix $(obj)/,$(zlib) cuboot-c2k.o gunzip_util.o main.o): \
- $(addprefix $(obj)/,$(zliblinuxheader)) $(addprefix $(obj)/,$(zlibheader))
+$(addprefix $(obj)/, $(zlib-decomp-y)): \
+ $(addprefix $(obj)/,$(zliblinuxheader-y)) \
+ $(addprefix $(obj)/,$(zlibheader-y)) \
+ $(addprefix $(obj)/,$(zlib-y))
+
+$(addprefix $(obj)/,$(zlib-y)): \
+ $(addprefix $(obj)/,$(zliblinuxheader-y)) \
+ $(addprefix $(obj)/,$(zlibheader-y))
libfdt := fdt.c fdt_ro.c fdt_wip.c fdt_sw.c fdt_rw.c fdt_strerror.c
libfdtheader := fdt.h libfdt.h libfdt_internal.h
$(addprefix $(obj)/,$(libfdt) libfdt-wrapper.o simpleboot.o epapr.o opal.o): \
$(addprefix $(obj)/,$(libfdtheader))
-src-wlib-y := string.S crt0.S crtsavres.S stdio.c main.c \
+src-wlib-y := string.S crt0.S crtsavres.S stdio.c decompress.c main.c \
$(libfdt) libfdt-wrapper.c \
ns16550.c serial.c simple_alloc.c div64.S util.S \
- gunzip_util.c elf_util.c $(zlib) devtree.c stdlib.c \
+ elf_util.c $(zlib-y) devtree.c stdlib.c \
oflib.c ofconsole.c cuboot.c mpsc.c cpm-serial.c \
uartlite.c mpc52xx-psc.c opal.c opal-calls.S
src-wlib-$(CONFIG_40x) += 4xx.c planetcore.c
obj-plat := $(addsuffix .o, $(basename $(addprefix $(obj)/, $(src-plat))))
obj-plat: $(libfdt)
-quiet_cmd_copy_zlib = COPY $@
- cmd_copy_zlib = sed "s@__used@@;s@<linux/\([^>]*\).*@\"\1\"@" $< > $@
+quiet_cmd_copy_kern_src = COPY $@
+ cmd_copy_kern_src = sed -f $(srctree)/arch/powerpc/boot/fixup-headers.sed $< > $@
-quiet_cmd_copy_zlibheader = COPY $@
- cmd_copy_zlibheader = sed "s@<linux/\([^>]*\).*@\"\1\"@" $< > $@
-# stddef.h for NULL
-quiet_cmd_copy_zliblinuxheader = COPY $@
- cmd_copy_zliblinuxheader = sed "s@<linux/string.h>@\"string.h\"@;s@<linux/kernel.h>@<stddef.h>@;s@<linux/\([^>]*\).*@\"\1\"@" $< > $@
+$(addprefix $(obj)/,$(zlib-y)): $(obj)/%: $(srctree)/lib/zlib_inflate/%
+ $(call cmd,copy_kern_src)
-$(addprefix $(obj)/,$(zlib)): $(obj)/%: $(srctree)/lib/zlib_inflate/%
- $(call cmd,copy_zlib)
+$(addprefix $(obj)/,$(zlibheader-y)): $(obj)/%: $(srctree)/lib/zlib_inflate/%
+ $(call cmd,copy_kern_src)
-$(addprefix $(obj)/,$(zlibheader)): $(obj)/%: $(srctree)/lib/zlib_inflate/%
- $(call cmd,copy_zlibheader)
+$(addprefix $(obj)/,$(zliblinuxheader-y)): $(obj)/%: $(srctree)/include/linux/%
+ $(call cmd,copy_kern_src)
-$(addprefix $(obj)/,$(zliblinuxheader)): $(obj)/%: $(srctree)/include/linux/%
- $(call cmd,copy_zliblinuxheader)
+$(addprefix $(obj)/,$(zlib-decomp-y)): $(obj)/%: $(srctree)/lib/%
+ $(call cmd,copy_kern_src)
quiet_cmd_copy_libfdt = COPY $@
cmd_copy_libfdt = cp $< $@
$(obj)/zImage.coff.lds $(obj)/zImage.ps3.lds : $(obj)/%: $(srctree)/$(src)/%.S
$(Q)cp $< $@
-clean-files := $(zlib) $(zlibheader) $(zliblinuxheader) \
- $(libfdt) $(libfdtheader) \
+clean-files := $(zlib-) $(zlibheader-) $(zliblinuxheader-) \
+ $(zlib-decomp-) $(libfdt) $(libfdtheader) \
empty.c zImage.coff.lds zImage.ps3.lds zImage.lds
quiet_cmd_bootcc = BOOTCC $@
endif
endif
+compressor-$(CONFIG_KERNEL_GZIP) := gz
+compressor-$(CONFIG_KERNEL_XZ) := xz
+
# args (to if_changed): 1 = (this rule), 2 = platform, 3 = dts 4=dtb 5=initrd
quiet_cmd_wrap = WRAP $@
- cmd_wrap =$(CONFIG_SHELL) $(wrapper) -c -o $@ -p $2 $(CROSSWRAP) \
- $(if $3, -s $3)$(if $4, -d $4)$(if $5, -i $5) vmlinux
+ cmd_wrap =$(CONFIG_SHELL) $(wrapper) -Z $(compressor-y) -c -o $@ -p $2 \
+ $(CROSSWRAP) $(if $3, -s $3)$(if $4, -d $4)$(if $5, -i $5) \
+ vmlinux
image-$(CONFIG_PPC_PSERIES) += zImage.pseries
image-$(CONFIG_PPC_POWERNV) += zImage.pseries
zImage.maple simpleImage.* otheros.bld *.dtb
# clean up files cached by wrapper
-clean-kernel := vmlinux.strip vmlinux.bin
-clean-kernel += $(addsuffix .gz,$(clean-kernel))
+clean-kernel-base := vmlinux.strip vmlinux.bin
+clean-kernel := $(addsuffix .gz,$(clean-kernel-base))
+clean-kernel += $(addsuffix .xz,$(clean-kernel-base))
# If not absolute clean-files are relative to $(obj).
clean-files += $(addprefix $(objtree)/, $(clean-kernel))
#include "io.h"
#include "ops.h"
#include "elf.h"
-#include "gunzip_util.h"
#include "mv64x60.h"
#include "cuboot.h"
#include "ppcboot.h"
--- /dev/null
+/*
+ * Wrapper around the kernel's pre-boot decompression library.
+ *
+ * Copyright (C) IBM Corporation 2016.
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License
+ * as published by the Free Software Foundation; either version
+ * 2 of the License, or (at your option) any later version.
+ */
+
+#include "elf.h"
+#include "page.h"
+#include "string.h"
+#include "stdio.h"
+#include "ops.h"
+#include "reg.h"
+#include "types.h"
+
+/*
+ * The decompressor_*.c files play #ifdef games so they can be used in both
+ * pre-boot and regular kernel code. We need these definitions to make the
+ * includes work.
+ */
+
+#define STATIC static
+#define INIT
+#define __always_inline inline
+
+/*
+ * The build process will copy the required zlib source files and headers
+ * out of lib/ and "fix" the includes so they do not pull in other kernel
+ * headers.
+ */
+
+#ifdef CONFIG_KERNEL_GZIP
+# include "decompress_inflate.c"
+#endif
+
+#ifdef CONFIG_KERNEL_XZ
+# include "xz_config.h"
+# include "../../../lib/decompress_unxz.c"
+#endif
+
+/* globals for tracking the state of the decompression */
+static unsigned long decompressed_bytes;
+static unsigned long limit;
+static unsigned long skip;
+static char *output_buffer;
+
+/*
+ * flush() is called by __decompress() when the decompressor's scratch buffer is
+ * full.
+ */
+static long flush(void *v, unsigned long buffer_size)
+{
+ unsigned long end = decompressed_bytes + buffer_size;
+ unsigned long size = buffer_size;
+ unsigned long offset = 0;
+ char *in = v;
+ char *out;
+
+ /*
+ * if we hit our decompression limit, we need to fake an error to abort
+ * the in-progress decompression.
+ */
+ if (decompressed_bytes >= limit)
+ return -1;
+
+ /* skip this entire block */
+ if (end <= skip) {
+ decompressed_bytes += buffer_size;
+ return buffer_size;
+ }
+
+ /* skip some data at the start, but keep the rest of the block */
+ if (decompressed_bytes < skip && end > skip) {
+ offset = skip - decompressed_bytes;
+
+ in += offset;
+ size -= offset;
+ decompressed_bytes += offset;
+ }
+
+ out = &output_buffer[decompressed_bytes - skip];
+ size = min(decompressed_bytes + size, limit) - decompressed_bytes;
+
+ memcpy(out, in, size);
+ decompressed_bytes += size;
+
+ return buffer_size;
+}
+
+static void print_err(char *s)
+{
+ /* suppress the "error" when we terminate the decompressor */
+ if (decompressed_bytes >= limit)
+ return;
+
+ printf("Decompression error: '%s'\n\r", s);
+}
+
+/**
+ * partial_decompress - decompresses part or all of a compressed buffer
+ * @inbuf: input buffer
+ * @input_size: length of the input buffer
+ * @outbuf: input buffer
+ * @output_size: length of the input buffer
+ * @skip number of output bytes to ignore
+ *
+ * This function takes compressed data from inbuf, decompresses and write it to
+ * outbuf. Once output_size bytes are written to the output buffer, or the
+ * stream is exhausted the function will return the number of bytes that were
+ * decompressed. Otherwise it will return whatever error code the decompressor
+ * reported (NB: This is specific to each decompressor type).
+ *
+ * The skip functionality is mainly there so the program and discover
+ * the size of the compressed image so that it can ask firmware (if present)
+ * for an appropriately sized buffer.
+ */
+long partial_decompress(void *inbuf, unsigned long input_size,
+ void *outbuf, unsigned long output_size, unsigned long _skip)
+{
+ int ret;
+
+ /*
+ * The skipped bytes needs to be included in the size of data we want
+ * to decompress.
+ */
+ output_size += _skip;
+
+ decompressed_bytes = 0;
+ output_buffer = outbuf;
+ limit = output_size;
+ skip = _skip;
+
+ ret = __decompress(inbuf, input_size, NULL, flush, outbuf,
+ output_size, NULL, print_err);
+
+ /*
+ * If decompression was aborted due to an actual error rather than
+ * a fake error that we used to abort, then we should report it.
+ */
+ if (decompressed_bytes < limit)
+ return ret;
+
+ return decompressed_bytes - skip;
+}
--- /dev/null
+# Copyright 2016 IBM Corporation.
+#
+# This program is free software; you can redistribute it and/or modify it
+# under the terms of the GNU General Public License version 2 or later as
+# published by the Free Software Foundation.
+
+s@#include <linux/decompress/mm\.h>@@;
+s@\"zlib_inflate/\([^\"]*\).*@"\1"@;
+s@<linux/kernel.h>@<stddef.h>@;
+
+s@__used@@;
+s@<linux/\([^>]*\).*@"\1"@;
+++ /dev/null
-/*
- * Copyright 2007 David Gibson, IBM Corporation.
- * Based on earlier work, Copyright (C) Paul Mackerras 1997.
- *
- * This program is free software; you can redistribute it and/or
- * modify it under the terms of the GNU General Public License
- * as published by the Free Software Foundation; either version
- * 2 of the License, or (at your option) any later version.
- */
-
-#include <stddef.h>
-#include "string.h"
-#include "stdio.h"
-#include "ops.h"
-#include "gunzip_util.h"
-
-#define HEAD_CRC 2
-#define EXTRA_FIELD 4
-#define ORIG_NAME 8
-#define COMMENT 0x10
-#define RESERVED 0xe0
-
-/**
- * gunzip_start - prepare to decompress gzip data
- * @state: decompressor state structure to be initialized
- * @src: buffer containing gzip compressed or uncompressed data
- * @srclen: size in bytes of the buffer at src
- *
- * If the buffer at @src contains a gzip header, this function
- * initializes zlib to decompress the data, storing the decompression
- * state in @state. The other functions in this file can then be used
- * to decompress data from the gzipped stream.
- *
- * If the buffer at @src does not contain a gzip header, it is assumed
- * to contain uncompressed data. The buffer information is recorded
- * in @state and the other functions in this file will simply copy
- * data from the uncompressed data stream at @src.
- *
- * Any errors, such as bad compressed data, cause an error to be
- * printed an the platform's exit() function to be called.
- */
-void gunzip_start(struct gunzip_state *state, void *src, int srclen)
-{
- char *hdr = src;
- int hdrlen = 0;
-
- memset(state, 0, sizeof(*state));
-
- /* Check for gzip magic number */
- if ((hdr[0] == 0x1f) && (hdr[1] == 0x8b)) {
- /* gzip data, initialize zlib parameters */
- int r, flags;
-
- state->s.workspace = state->scratch;
- if (zlib_inflate_workspacesize() > sizeof(state->scratch))
- fatal("insufficient scratch space for gunzip\n\r");
-
- /* skip header */
- hdrlen = 10;
- flags = hdr[3];
- if (hdr[2] != Z_DEFLATED || (flags & RESERVED) != 0)
- fatal("bad gzipped data\n\r");
- if ((flags & EXTRA_FIELD) != 0)
- hdrlen = 12 + hdr[10] + (hdr[11] << 8);
- if ((flags & ORIG_NAME) != 0)
- while (hdr[hdrlen++] != 0)
- ;
- if ((flags & COMMENT) != 0)
- while (hdr[hdrlen++] != 0)
- ;
- if ((flags & HEAD_CRC) != 0)
- hdrlen += 2;
- if (hdrlen >= srclen)
- fatal("gunzip_start: ran out of data in header\n\r");
-
- r = zlib_inflateInit2(&state->s, -MAX_WBITS);
- if (r != Z_OK)
- fatal("inflateInit2 returned %d\n\r", r);
- }
-
- state->s.total_in = hdrlen;
- state->s.next_in = src + hdrlen;
- state->s.avail_in = srclen - hdrlen;
-}
-
-/**
- * gunzip_partial - extract bytes from a gzip data stream
- * @state: gzip state structure previously initialized by gunzip_start()
- * @dst: buffer to store extracted data
- * @dstlen: maximum number of bytes to extract
- *
- * This function extracts at most @dstlen bytes from the data stream
- * previously associated with @state by gunzip_start(), decompressing
- * if necessary. Exactly @dstlen bytes are extracted unless the data
- * stream doesn't contain enough bytes, in which case the entire
- * remainder of the stream is decompressed.
- *
- * Returns the actual number of bytes extracted. If any errors occur,
- * such as a corrupted compressed stream, an error is printed an the
- * platform's exit() function is called.
- */
-int gunzip_partial(struct gunzip_state *state, void *dst, int dstlen)
-{
- int len;
-
- if (state->s.workspace) {
- /* gunzipping */
- int r;
-
- state->s.next_out = dst;
- state->s.avail_out = dstlen;
- r = zlib_inflate(&state->s, Z_FULL_FLUSH);
- if (r != Z_OK && r != Z_STREAM_END)
- fatal("inflate returned %d msg: %s\n\r", r, state->s.msg);
- len = state->s.next_out - (Byte *)dst;
- } else {
- /* uncompressed image */
- len = min(state->s.avail_in, (uLong)dstlen);
- memcpy(dst, state->s.next_in, len);
- state->s.next_in += len;
- state->s.avail_in -= len;
- }
- return len;
-}
-
-/**
- * gunzip_exactly - extract a fixed number of bytes from a gzip data stream
- * @state: gzip state structure previously initialized by gunzip_start()
- * @dst: buffer to store extracted data
- * @dstlen: number of bytes to extract
- *
- * This function extracts exactly @dstlen bytes from the data stream
- * previously associated with @state by gunzip_start(), decompressing
- * if necessary.
- *
- * If there are less @dstlen bytes available in the data stream, or if
- * any other errors occur, such as a corrupted compressed stream, an
- * error is printed an the platform's exit() function is called.
- */
-void gunzip_exactly(struct gunzip_state *state, void *dst, int dstlen)
-{
- int len;
-
- len = gunzip_partial(state, dst, dstlen);
- if (len < dstlen)
- fatal("\n\rgunzip_exactly: ran out of data!"
- " Wanted %d, got %d.\n\r", dstlen, len);
-}
-
-/**
- * gunzip_discard - discard bytes from a gzip data stream
- * @state: gzip state structure previously initialized by gunzip_start()
- * @len: number of bytes to discard
- *
- * This function extracts, then discards exactly @len bytes from the
- * data stream previously associated with @state by gunzip_start().
- * Subsequent gunzip_partial(), gunzip_exactly() or gunzip_finish()
- * calls will extract the data following the discarded bytes in the
- * data stream.
- *
- * If there are less @len bytes available in the data stream, or if
- * any other errors occur, such as a corrupted compressed stream, an
- * error is printed an the platform's exit() function is called.
- */
-void gunzip_discard(struct gunzip_state *state, int len)
-{
- static char discard_buf[128];
-
- while (len > sizeof(discard_buf)) {
- gunzip_exactly(state, discard_buf, sizeof(discard_buf));
- len -= sizeof(discard_buf);
- }
-
- if (len > 0)
- gunzip_exactly(state, discard_buf, len);
-}
-
-/**
- * gunzip_finish - extract all remaining bytes from a gzip data stream
- * @state: gzip state structure previously initialized by gunzip_start()
- * @dst: buffer to store extracted data
- * @dstlen: maximum number of bytes to extract
- *
- * This function extracts all remaining data, or at most @dstlen
- * bytes, from the stream previously associated with @state by
- * gunzip_start(). zlib is then shut down, so it is an error to use
- * any of the functions in this file on @state until it is
- * re-initialized with another call to gunzip_start().
- *
- * If any errors occur, such as a corrupted compressed stream, an
- * error is printed an the platform's exit() function is called.
- */
-int gunzip_finish(struct gunzip_state *state, void *dst, int dstlen)
-{
- int len;
-
- len = gunzip_partial(state, dst, dstlen);
-
- if (state->s.workspace) {
- zlib_inflateEnd(&state->s);
- }
-
- return len;
-}
+++ /dev/null
-/*
- * Decompression convenience functions
- *
- * Copyright 2007 David Gibson, IBM Corporation.
- *
- * This file is licensed under the terms of the GNU General Public
- * License version 2. This program is licensed "as is" without any
- * warranty of any kind, whether express or implied.
- */
-#ifndef _PPC_BOOT_GUNZIP_UTIL_H_
-#define _PPC_BOOT_GUNZIP_UTIL_H_
-
-#include "zlib.h"
-
-/*
- * These functions are designed to make life easy for decompressing
- * kernel images, initrd images or any other gzip compressed image,
- * particularly if its useful to decompress part of the image (e.g. to
- * examine headers) before decompressing the remainder.
- *
- * To use:
- * - declare a gunzip_state structure
- * - use gunzip_start() to initialize the state, associating it
- * with a stream of compressed data
- * - use gunzip_partial(), gunzip_exactly() and gunzip_discard()
- * in any combination to extract pieces of data from the stream
- * - Finally use gunzip_finish() to extract the tail of the
- * compressed stream and wind up zlib
- */
-
-/* scratch space for gunzip; 46912 is from zlib_inflate_workspacesize() */
-#define GUNZIP_SCRATCH_SIZE 46912
-
-struct gunzip_state {
- z_stream s;
- char scratch[46912];
-};
-
-void gunzip_start(struct gunzip_state *state, void *src, int srclen);
-int gunzip_partial(struct gunzip_state *state, void *dst, int dstlen);
-void gunzip_exactly(struct gunzip_state *state, void *dst, int len);
-void gunzip_discard(struct gunzip_state *state, int len);
-int gunzip_finish(struct gunzip_state *state, void *dst, int len);
-
-#endif /* _PPC_BOOT_GUNZIP_UTIL_H_ */
#include "string.h"
#include "stdio.h"
#include "ops.h"
-#include "gunzip_util.h"
#include "reg.h"
-static struct gunzip_state gzstate;
-
struct addr_range {
void *addr;
unsigned long size;
static struct addr_range prep_kernel(void)
{
char elfheader[256];
- void *vmlinuz_addr = _vmlinux_start;
+ unsigned char *vmlinuz_addr = (unsigned char *)_vmlinux_start;
unsigned long vmlinuz_size = _vmlinux_end - _vmlinux_start;
void *addr = 0;
struct elf_info ei;
- int len;
+ long len;
- /* gunzip the ELF header of the kernel */
- gunzip_start(&gzstate, vmlinuz_addr, vmlinuz_size);
- gunzip_exactly(&gzstate, elfheader, sizeof(elfheader));
+ partial_decompress(vmlinuz_addr, vmlinuz_size,
+ elfheader, sizeof(elfheader), 0);
if (!parse_elf64(elfheader, &ei) && !parse_elf32(elfheader, &ei))
fatal("Error: not a valid PPC32 or PPC64 ELF file!\n\r");
* the kernel bss must be claimed (it will be zero'd by the
* kernel itself)
*/
- printf("Allocating 0x%lx bytes for kernel ...\n\r", ei.memsize);
+ printf("Allocating 0x%lx bytes for kernel...\n\r", ei.memsize);
if (platform_ops.vmlinux_alloc) {
addr = platform_ops.vmlinux_alloc(ei.memsize);
"device tree\n\r");
}
- /* Finally, gunzip the kernel */
- printf("gunzipping (0x%p <- 0x%p:0x%p)...", addr,
+ /* Finally, decompress the kernel */
+ printf("Decompressing (0x%p <- 0x%p:0x%p)...\n\r", addr,
vmlinuz_addr, vmlinuz_addr+vmlinuz_size);
- /* discard up to the actual load data */
- gunzip_discard(&gzstate, ei.elfoffset - sizeof(elfheader));
- len = gunzip_finish(&gzstate, addr, ei.loadsize);
+
+ len = partial_decompress(vmlinuz_addr, vmlinuz_size,
+ addr, ei.loadsize, ei.elfoffset);
+
+ if (len < 0)
+ fatal("Decompression failed with error code %ld\n\r", len);
+
if (len != ei.loadsize)
- fatal("ran out of data! only got 0x%x of 0x%lx bytes.\n\r",
- len, ei.loadsize);
- printf("done 0x%x bytes\n\r", len);
+ fatal("Decompression error: got 0x%lx bytes, expected 0x%lx.\n\r",
+ len, ei.loadsize);
+
+ printf("Done! Decompressed 0x%lx bytes\n\r", len);
flush_cache(addr, ei.loadsize);
return 31 - bit;
}
+long partial_decompress(void *inbuf, unsigned long input_size, void *outbuf,
+ unsigned long output_size, unsigned long skip);
+
#endif /* _PPC_BOOT_OPS_H_ */
--- /dev/null
+/*
+ * Copyright (C) IBM Corporation 2016.
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License
+ * as published by the Free Software Foundation; either version
+ * 2 of the License, or (at your option) any later version.
+ *
+ * This file is only necessary because some of the pre-boot decompressors
+ * expect stdbool.h to be available.
+ *
+ */
+
+#include "types.h"
--- /dev/null
+/*
+ * Copyright (C) IBM Corporation 2016.
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License
+ * as published by the Free Software Foundation; either version
+ * 2 of the License, or (at your option) any later version.
+ *
+ * This file is only necessary because some of the pre-boot decompressors
+ * expect stdint.h to be available.
+ */
+
+#include "types.h"
#ifndef _TYPES_H_
#define _TYPES_H_
+#include <stdbool.h>
+
#define ARRAY_SIZE(x) (sizeof(x) / sizeof((x)[0]))
typedef unsigned char u8;
(void) (&_x == &_y); \
_x > _y ? _x : _y; })
+#define min_t(type, a, b) min(((type) a), ((type) b))
+#define max_t(type, a, b) max(((type) a), ((type) b))
+
+typedef int bool;
+
+#ifndef true
+#define true 1
+#endif
+
+#ifndef false
+#define false 0
+#endif
#endif /* _TYPES_H_ */
# -D dir specify directory containing data files used by script
# (default ./arch/powerpc/boot)
# -W dir specify working directory for temporary files (default .)
+# -z use gzip (legacy)
+# -Z zsuffix compression to use (gz, xz or none)
# Stop execution if any command fails
set -e
dts=
cacheit=
binary=
-gzip=.gz
+compression=.gz
pie=
format=
usage() {
echo 'Usage: wrapper [-o output] [-p platform] [-i initrd]' >&2
echo ' [-d devtree] [-s tree.dts] [-c] [-C cross-prefix]' >&2
- echo ' [-D datadir] [-W workingdir] [--no-gzip] [vmlinux]' >&2
+ echo ' [-D datadir] [-W workingdir] [-Z (gz|xz|none)]' >&2
+ echo ' [--no-compression] [vmlinux]' >&2
exit 1
}
[ "$#" -gt 0 ] || usage
tmpdir="$1"
;;
+ -z)
+ compression=.gz
+ ;;
+ -Z)
+ shift
+ [ "$#" -gt 0 ] || usage
+ [ "$1" != "gz" -o "$1" != "xz" -o "$1" != "none" ] || usage
+
+ compression=".$1"
+
+ if [ $compression = ".none" ]; then
+ compression=
+ fi
+ ;;
--no-gzip)
- gzip=
+ # a "feature" of the the wrapper script is that it can be used outside
+ # the kernel tree. So keeping this around for backwards compatibility.
+ compression=
;;
-?)
usage
shift
done
+
if [ -n "$dts" ]; then
if [ ! -r "$dts" -a -r "$object/dts/$dts" ]; then
dts="$object/dts/$dts"
;;
cuboot*)
binary=y
- gzip=
+ compression=
case "$platform" in
*-mpc866ads|*-mpc885ads|*-adder875*|*-ep88xc)
platformo=$object/cuboot-8xx.o
ps3)
platformo="$object/ps3-head.o $object/ps3-hvcall.o $object/ps3.o"
lds=$object/zImage.ps3.lds
- gzip=
+ compression=
ext=bin
objflags="-O binary --set-section-flags=.bss=contents,alloc,load,data"
ksection=.kernel:vmlinux.bin
esac
vmz="$tmpdir/`basename \"$kernel\"`.$ext"
-if [ -z "$cacheit" -o ! -f "$vmz$gzip" -o "$vmz$gzip" -ot "$kernel" ]; then
- ${CROSS}objcopy $objflags "$kernel" "$vmz.$$"
- strip_size=$(stat -c %s $vmz.$$)
+# Calculate the vmlinux.strip size
+${CROSS}objcopy $objflags "$kernel" "$vmz.$$"
+strip_size=$(stat -c %s $vmz.$$)
- if [ -n "$gzip" ]; then
+if [ -z "$cacheit" -o ! -f "$vmz$compression" -o "$vmz$compression" -ot "$kernel" ]; then
+ # recompress the image if we need to
+ case $compression in
+ .xz)
+ xz --check=crc32 -f -6 "$vmz.$$"
+ ;;
+ .gz)
gzip -n -f -9 "$vmz.$$"
- fi
+ ;;
+ *)
+ # drop the compression suffix so the stripped vmlinux is used
+ compression=
+ ;;
+ esac
if [ -n "$cacheit" ]; then
- mv -f "$vmz.$$$gzip" "$vmz$gzip"
+ mv -f "$vmz.$$$compression" "$vmz$compression"
else
vmz="$vmz.$$"
fi
else
- # Calculate the vmlinux.strip size
- ${CROSS}objcopy $objflags "$kernel" "$vmz.$$"
- strip_size=$(stat -c %s $vmz.$$)
rm -f $vmz.$$
fi
+vmz="$vmz$compression"
+
if [ "$make_space" = "y" ]; then
# Round the size to next higher MB limit
round_size=$(((strip_size + 0xfffff) & 0xfff00000))
fi
fi
-vmz="$vmz$gzip"
-
# Extract kernel version information, some platforms want to include
# it in the image header
version=`${CROSS}strings "$kernel" | grep '^Linux version [-0-9.]' | \
if [ -n "$link_address" ] ; then
text_start="-Ttext $link_address"
fi
+#link everything
${CROSS}ld -m $format -T $lds $text_start $pie -o "$ofile" \
$platformo $tmp $object/wrapper.a
rm $tmp
--- /dev/null
+#ifndef __XZ_CONFIG_H__
+#define __XZ_CONFIG_H__
+
+/*
+ * most of this is copied from lib/xz/xz_private.h, we can't use their defines
+ * since the boot wrapper is not built in the same environment as the rest of
+ * the kernel.
+ */
+
+#include "types.h"
+#include "swab.h"
+
+static inline uint32_t swab32p(void *p)
+{
+ uint32_t *q = p;
+
+ return swab32(*q);
+}
+
+#ifdef __LITTLE_ENDIAN__
+#define get_le32(p) (*((uint32_t *) (p)))
+#else
+#define get_le32(p) swab32p(p)
+#endif
+
+#define memeq(a, b, size) (memcmp(a, b, size) == 0)
+#define memzero(buf, size) memset(buf, 0, size)
+
+/* prevent the inclusion of the xz-preboot MM headers */
+#define DECOMPR_MM_H
+#define memmove memmove
+#define XZ_EXTERN static
+
+/* xz.h needs to be included directly since we need enum xz_mode */
+#include "../../../include/linux/xz.h"
+
+#undef XZ_EXTERN
+
+#endif
CONFIG_TASK_IO_ACCOUNTING=y
CONFIG_IKCONFIG=y
CONFIG_IKCONFIG_PROC=y
+CONFIG_LOG_BUF_SHIFT=18
+CONFIG_LOG_CPU_MAX_BUF_SHIFT=13
CONFIG_NUMA_BALANCING=y
CONFIG_CGROUPS=y
CONFIG_MEMCG=y
CONFIG_BLK_DEV_GENERIC=y
CONFIG_BLK_DEV_AMD74XX=y
CONFIG_BLK_DEV_SD=y
-CONFIG_CHR_DEV_ST=y
+CONFIG_CHR_DEV_ST=m
CONFIG_BLK_DEV_SR=y
CONFIG_BLK_DEV_SR_VENDOR=y
CONFIG_CHR_DEV_SG=y
CONFIG_SCSI_BNX2_ISCSI=m
CONFIG_BE2ISCSI=m
CONFIG_SCSI_MPT2SAS=m
-CONFIG_SCSI_SYM53C8XX_2=y
+CONFIG_SCSI_SYM53C8XX_2=m
CONFIG_SCSI_SYM53C8XX_DMA_ADDRESSING_MODE=0
CONFIG_SCSI_IPR=y
CONFIG_SCSI_QLA_FC=m
CONFIG_VETH=m
CONFIG_VIRTIO_NET=m
CONFIG_VHOST_NET=m
-CONFIG_VORTEX=y
+CONFIG_VORTEX=m
CONFIG_ACENIC=m
CONFIG_ACENIC_OMIT_TIGON_I=y
-CONFIG_PCNET32=y
+CONFIG_PCNET32=m
CONFIG_TIGON3=y
CONFIG_BNX2X=m
CONFIG_CHELSIO_T1=m
CONFIG_E1000E=y
CONFIG_IXGB=m
CONFIG_IXGBE=m
+CONFIG_I40E=m
CONFIG_MLX4_EN=m
CONFIG_MYRI10GE=m
CONFIG_QLGE=m
CONFIG_EXT4_FS=y
CONFIG_EXT4_FS_POSIX_ACL=y
CONFIG_EXT4_FS_SECURITY=y
-CONFIG_REISERFS_FS=y
+CONFIG_REISERFS_FS=m
CONFIG_REISERFS_FS_XATTR=y
CONFIG_REISERFS_FS_POSIX_ACL=y
CONFIG_REISERFS_FS_SECURITY=y
CONFIG_AUTOFS4_FS=m
CONFIG_FUSE_FS=m
CONFIG_OVERLAY_FS=m
-CONFIG_ISO9660_FS=y
+CONFIG_ISO9660_FS=m
CONFIG_UDF_FS=m
CONFIG_MSDOS_FS=y
-CONFIG_VFAT_FS=y
+CONFIG_VFAT_FS=m
CONFIG_PROC_KCORE=y
CONFIG_TMPFS=y
CONFIG_TMPFS_POSIX_ACL=y
CONFIG_CRYPTO_TWOFISH=m
CONFIG_CRYPTO_LZO=m
CONFIG_CRYPTO_DEV_NX=y
+CONFIG_CRYPTO_DEV_VMX=y
+CONFIG_CRYPTO_DEV_VMX_ENCRYPT=m
CONFIG_VIRTUALIZATION=y
CONFIG_KVM_BOOK3S_64=m
CONFIG_KVM_BOOK3S_64_HV=m
CONFIG_TASK_DELAY_ACCT=y
CONFIG_IKCONFIG=y
CONFIG_IKCONFIG_PROC=y
+CONFIG_LOG_BUF_SHIFT=18
+CONFIG_LOG_CPU_MAX_BUF_SHIFT=13
CONFIG_CGROUPS=y
CONFIG_CPUSETS=y
CONFIG_BLK_DEV_INITRD=y
CONFIG_BLK_DEV_IDE_PMAC=y
CONFIG_BLK_DEV_IDE_PMAC_ATA100FIRST=y
CONFIG_BLK_DEV_SD=y
-CONFIG_CHR_DEV_ST=y
+CONFIG_CHR_DEV_ST=m
CONFIG_BLK_DEV_SR=y
CONFIG_BLK_DEV_SR_VENDOR=y
CONFIG_CHR_DEV_SG=y
CONFIG_SCSI_MPT2SAS=m
CONFIG_SCSI_IBMVSCSI=y
CONFIG_SCSI_IBMVFC=m
-CONFIG_SCSI_SYM53C8XX_2=y
+CONFIG_SCSI_SYM53C8XX_2=m
CONFIG_SCSI_SYM53C8XX_DMA_ADDRESSING_MODE=0
CONFIG_SCSI_IPR=y
CONFIG_SCSI_QLA_FC=m
CONFIG_TUN=m
CONFIG_VIRTIO_NET=m
CONFIG_VHOST_NET=m
-CONFIG_VORTEX=y
+CONFIG_VORTEX=m
CONFIG_ACENIC=m
CONFIG_ACENIC_OMIT_TIGON_I=y
-CONFIG_PCNET32=y
+CONFIG_PCNET32=m
CONFIG_TIGON3=y
CONFIG_BNX2X=m
CONFIG_CHELSIO_T1=m
CONFIG_E1000E=y
CONFIG_IXGB=m
CONFIG_IXGBE=m
+CONFIG_I40E=m
CONFIG_MLX4_EN=m
CONFIG_MYRI10GE=m
CONFIG_PASEMI_MAC=y
CONFIG_EXT4_FS=y
CONFIG_EXT4_FS_POSIX_ACL=y
CONFIG_EXT4_FS_SECURITY=y
-CONFIG_REISERFS_FS=y
+CONFIG_REISERFS_FS=m
CONFIG_REISERFS_FS_XATTR=y
CONFIG_REISERFS_FS_POSIX_ACL=y
CONFIG_REISERFS_FS_SECURITY=y
CONFIG_AUTOFS4_FS=m
CONFIG_FUSE_FS=m
CONFIG_OVERLAY_FS=m
-CONFIG_ISO9660_FS=y
+CONFIG_ISO9660_FS=m
CONFIG_UDF_FS=m
CONFIG_MSDOS_FS=y
-CONFIG_VFAT_FS=y
+CONFIG_VFAT_FS=m
CONFIG_PROC_KCORE=y
CONFIG_TMPFS=y
CONFIG_TMPFS_POSIX_ACL=y
# CONFIG_CRYPTO_ANSI_CPRNG is not set
CONFIG_CRYPTO_DEV_NX=y
CONFIG_CRYPTO_DEV_NX_ENCRYPT=m
+CONFIG_CRYPTO_DEV_VMX=y
+CONFIG_CRYPTO_DEV_VMX_ENCRYPT=m
CONFIG_VIRTUALIZATION=y
CONFIG_KVM_BOOK3S_64=m
CONFIG_KVM_BOOK3S_64_HV=m
CONFIG_TASK_IO_ACCOUNTING=y
CONFIG_IKCONFIG=y
CONFIG_IKCONFIG_PROC=y
+CONFIG_LOG_BUF_SHIFT=18
+CONFIG_LOG_CPU_MAX_BUF_SHIFT=13
CONFIG_NUMA_BALANCING=y
CONFIG_NUMA_BALANCING_DEFAULT_ENABLED=y
CONFIG_CGROUPS=y
CONFIG_BLK_DEV_GENERIC=y
CONFIG_BLK_DEV_AMD74XX=y
CONFIG_BLK_DEV_SD=y
-CONFIG_CHR_DEV_ST=y
+CONFIG_CHR_DEV_ST=m
CONFIG_BLK_DEV_SR=y
CONFIG_BLK_DEV_SR_VENDOR=y
CONFIG_CHR_DEV_SG=y
CONFIG_SCSI_MPT2SAS=m
CONFIG_SCSI_IBMVSCSI=y
CONFIG_SCSI_IBMVFC=m
-CONFIG_SCSI_SYM53C8XX_2=y
+CONFIG_SCSI_SYM53C8XX_2=m
CONFIG_SCSI_SYM53C8XX_DMA_ADDRESSING_MODE=0
CONFIG_SCSI_IPR=y
CONFIG_SCSI_QLA_FC=m
CONFIG_VETH=m
CONFIG_VIRTIO_NET=m
CONFIG_VHOST_NET=m
-CONFIG_VORTEX=y
+CONFIG_VORTEX=m
CONFIG_ACENIC=m
CONFIG_ACENIC_OMIT_TIGON_I=y
-CONFIG_PCNET32=y
+CONFIG_PCNET32=m
CONFIG_TIGON3=y
CONFIG_BNX2X=m
CONFIG_CHELSIO_T1=m
CONFIG_E1000E=y
CONFIG_IXGB=m
CONFIG_IXGBE=m
+CONFIG_I40E=m
CONFIG_MLX4_EN=m
CONFIG_MYRI10GE=m
CONFIG_QLGE=m
CONFIG_EXT4_FS=y
CONFIG_EXT4_FS_POSIX_ACL=y
CONFIG_EXT4_FS_SECURITY=y
-CONFIG_REISERFS_FS=y
+CONFIG_REISERFS_FS=m
CONFIG_REISERFS_FS_XATTR=y
CONFIG_REISERFS_FS_POSIX_ACL=y
CONFIG_REISERFS_FS_SECURITY=y
CONFIG_AUTOFS4_FS=m
CONFIG_FUSE_FS=m
CONFIG_OVERLAY_FS=m
-CONFIG_ISO9660_FS=y
+CONFIG_ISO9660_FS=m
CONFIG_UDF_FS=m
CONFIG_MSDOS_FS=y
-CONFIG_VFAT_FS=y
+CONFIG_VFAT_FS=m
CONFIG_PROC_KCORE=y
CONFIG_TMPFS=y
CONFIG_TMPFS_POSIX_ACL=y
# CONFIG_CRYPTO_ANSI_CPRNG is not set
CONFIG_CRYPTO_DEV_NX=y
CONFIG_CRYPTO_DEV_NX_ENCRYPT=m
+CONFIG_CRYPTO_DEV_VMX=y
+CONFIG_CRYPTO_DEV_VMX_ENCRYPT=m
CONFIG_VIRTUALIZATION=y
CONFIG_KVM_BOOK3S_64=m
CONFIG_KVM_BOOK3S_64_HV=m
PPC_ATOMIC_ENTRY_BARRIER
"1: lwarx %0,0,%1 # __atomic_add_unless\n\
cmpw 0,%0,%3 \n\
- beq- 2f \n\
+ beq 2f \n\
add %0,%2,%0 \n"
PPC405_ERR77(0,%2)
" stwcx. %0,0,%1 \n\
PPC_ATOMIC_ENTRY_BARRIER
"1: ldarx %0,0,%1 # __atomic_add_unless\n\
cmpd 0,%0,%3 \n\
- beq- 2f \n\
+ beq 2f \n\
add %0,%2,%0 \n"
" stdcx. %0,0,%1 \n\
bne- 1b \n"
* exception handlers (including pSeries LPAR) and iSeries LPAR
* implementations as possible.
*/
+#include <asm/head-64.h>
#define EX_R9 0
#define EX_R10 8
*/
#define LOAD_HANDLER(reg, label) \
ld reg,PACAKBASE(r13); /* get high part of &label */ \
- ori reg,reg,((label)-_stext)@l; /* virt addr of handler ... */
+ ori reg,reg,(FIXED_SYMBOL_ABS_ADDR(label))@l;
/* Exception register prefixes */
#define EXC_HV H
EXCEPTION_PROLOG_1(area, extra, vec); \
EXCEPTION_PROLOG_PSERIES_1(label, h);
-#define __KVMTEST(n) \
- lbz r10,HSTATE_IN_GUEST(r13); \
+#define __KVMTEST(h, n) \
+ lbz r10,HSTATE_IN_GUEST(r13); \
cmpwi r10,0; \
- bne do_kvm_##n
+ bne do_kvm_##h##n
#ifdef CONFIG_KVM_BOOK3S_HV_POSSIBLE
/*
#define kvmppc_interrupt kvmppc_interrupt_pr
#endif
-#define __KVM_HANDLER(area, h, n) \
-do_kvm_##n: \
+#define __KVM_HANDLER_PROLOG(area, n) \
BEGIN_FTR_SECTION_NESTED(947) \
ld r10,area+EX_CFAR(r13); \
std r10,HSTATE_CFAR(r13); \
stw r9,HSTATE_SCRATCH1(r13); \
ld r9,area+EX_R9(r13); \
std r12,HSTATE_SCRATCH0(r13); \
+
+#define __KVM_HANDLER(area, h, n) \
+ __KVM_HANDLER_PROLOG(area, n) \
li r12,n; \
b kvmppc_interrupt
#define __KVM_HANDLER_SKIP(area, h, n) \
-do_kvm_##n: \
cmpwi r10,KVM_GUEST_MODE_SKIP; \
ld r10,area+EX_R10(r13); \
beq 89f; \
- stw r9,HSTATE_SCRATCH1(r13); \
+ stw r9,HSTATE_SCRATCH1(r13); \
BEGIN_FTR_SECTION_NESTED(948) \
ld r9,area+EX_PPR(r13); \
std r9,HSTATE_PPR(r13); \
END_FTR_SECTION_NESTED(CPU_FTR_HAS_PPR,CPU_FTR_HAS_PPR,948); \
ld r9,area+EX_R9(r13); \
- std r12,HSTATE_SCRATCH0(r13); \
+ std r12,HSTATE_SCRATCH0(r13); \
li r12,n; \
b kvmppc_interrupt; \
89: mtocrf 0x80,r9; \
b kvmppc_skip_##h##interrupt
#ifdef CONFIG_KVM_BOOK3S_64_HANDLER
-#define KVMTEST(n) __KVMTEST(n)
+#define KVMTEST(h, n) __KVMTEST(h, n)
#define KVM_HANDLER(area, h, n) __KVM_HANDLER(area, h, n)
#define KVM_HANDLER_SKIP(area, h, n) __KVM_HANDLER_SKIP(area, h, n)
#else
-#define KVMTEST(n)
+#define KVMTEST(h, n)
#define KVM_HANDLER(area, h, n)
#define KVM_HANDLER_SKIP(area, h, n)
#endif
/*
* Exception vectors.
*/
-#define STD_EXCEPTION_PSERIES(vec, label) \
- . = vec; \
- .globl label##_pSeries; \
-label##_pSeries: \
+#define STD_EXCEPTION_PSERIES(vec, label) \
SET_SCRATCH0(r13); /* save r13 */ \
- EXCEPTION_PROLOG_PSERIES(PACA_EXGEN, label##_common, \
- EXC_STD, KVMTEST, vec)
+ EXCEPTION_PROLOG_PSERIES(PACA_EXGEN, label, \
+ EXC_STD, KVMTEST_PR, vec); \
/* Version of above for when we have to branch out-of-line */
+#define __OOL_EXCEPTION(vec, label, hdlr) \
+ SET_SCRATCH0(r13) \
+ EXCEPTION_PROLOG_0(PACA_EXGEN) \
+ b hdlr;
+
#define STD_EXCEPTION_PSERIES_OOL(vec, label) \
- .globl label##_pSeries; \
-label##_pSeries: \
- EXCEPTION_PROLOG_1(PACA_EXGEN, KVMTEST, vec); \
- EXCEPTION_PROLOG_PSERIES_1(label##_common, EXC_STD)
-
-#define STD_EXCEPTION_HV(loc, vec, label) \
- . = loc; \
- .globl label##_hv; \
-label##_hv: \
+ EXCEPTION_PROLOG_1(PACA_EXGEN, KVMTEST_PR, vec); \
+ EXCEPTION_PROLOG_PSERIES_1(label, EXC_STD)
+
+#define STD_EXCEPTION_HV(loc, vec, label) \
SET_SCRATCH0(r13); /* save r13 */ \
- EXCEPTION_PROLOG_PSERIES(PACA_EXGEN, label##_common, \
- EXC_HV, KVMTEST, vec)
+ EXCEPTION_PROLOG_PSERIES(PACA_EXGEN, label, \
+ EXC_HV, KVMTEST_HV, vec);
-/* Version of above for when we have to branch out-of-line */
-#define STD_EXCEPTION_HV_OOL(vec, label) \
- .globl label##_hv; \
-label##_hv: \
- EXCEPTION_PROLOG_1(PACA_EXGEN, KVMTEST, vec); \
- EXCEPTION_PROLOG_PSERIES_1(label##_common, EXC_HV)
+#define STD_EXCEPTION_HV_OOL(vec, label) \
+ EXCEPTION_PROLOG_1(PACA_EXGEN, KVMTEST_HV, vec); \
+ EXCEPTION_PROLOG_PSERIES_1(label, EXC_HV)
#define STD_RELON_EXCEPTION_PSERIES(loc, vec, label) \
- . = loc; \
- .globl label##_relon_pSeries; \
-label##_relon_pSeries: \
/* No guest interrupts come through here */ \
SET_SCRATCH0(r13); /* save r13 */ \
- EXCEPTION_RELON_PROLOG_PSERIES(PACA_EXGEN, label##_common, \
- EXC_STD, NOTEST, vec)
+ EXCEPTION_RELON_PROLOG_PSERIES(PACA_EXGEN, label, EXC_STD, NOTEST, vec);
#define STD_RELON_EXCEPTION_PSERIES_OOL(vec, label) \
- .globl label##_relon_pSeries; \
-label##_relon_pSeries: \
EXCEPTION_PROLOG_1(PACA_EXGEN, NOTEST, vec); \
- EXCEPTION_RELON_PROLOG_PSERIES_1(label##_common, EXC_STD)
+ EXCEPTION_RELON_PROLOG_PSERIES_1(label, EXC_STD)
#define STD_RELON_EXCEPTION_HV(loc, vec, label) \
- . = loc; \
- .globl label##_relon_hv; \
-label##_relon_hv: \
/* No guest interrupts come through here */ \
SET_SCRATCH0(r13); /* save r13 */ \
- EXCEPTION_RELON_PROLOG_PSERIES(PACA_EXGEN, label##_common, \
- EXC_HV, NOTEST, vec)
+ EXCEPTION_RELON_PROLOG_PSERIES(PACA_EXGEN, label, EXC_HV, NOTEST, vec);
#define STD_RELON_EXCEPTION_HV_OOL(vec, label) \
- .globl label##_relon_hv; \
-label##_relon_hv: \
EXCEPTION_PROLOG_1(PACA_EXGEN, NOTEST, vec); \
- EXCEPTION_RELON_PROLOG_PSERIES_1(label##_common, EXC_HV)
+ EXCEPTION_RELON_PROLOG_PSERIES_1(label, EXC_HV)
/* This associate vector numbers with bits in paca->irq_happened */
#define SOFTEN_VALUE_0x500 PACA_IRQ_EE
-#define SOFTEN_VALUE_0x502 PACA_IRQ_EE
#define SOFTEN_VALUE_0x900 PACA_IRQ_DEC
-#define SOFTEN_VALUE_0x982 PACA_IRQ_DEC
+#define SOFTEN_VALUE_0x980 PACA_IRQ_DEC
#define SOFTEN_VALUE_0xa00 PACA_IRQ_DBELL
#define SOFTEN_VALUE_0xe80 PACA_IRQ_DBELL
-#define SOFTEN_VALUE_0xe82 PACA_IRQ_DBELL
#define SOFTEN_VALUE_0xe60 PACA_IRQ_HMI
-#define SOFTEN_VALUE_0xe62 PACA_IRQ_HMI
#define SOFTEN_VALUE_0xea0 PACA_IRQ_EE
-#define SOFTEN_VALUE_0xea2 PACA_IRQ_EE
#define __SOFTEN_TEST(h, vec) \
lbz r10,PACASOFTIRQEN(r13); \
cmpwi r10,0; \
li r10,SOFTEN_VALUE_##vec; \
beq masked_##h##interrupt
+
#define _SOFTEN_TEST(h, vec) __SOFTEN_TEST(h, vec)
#define SOFTEN_TEST_PR(vec) \
- KVMTEST(vec); \
+ KVMTEST(EXC_STD, vec); \
_SOFTEN_TEST(EXC_STD, vec)
#define SOFTEN_TEST_HV(vec) \
- KVMTEST(vec); \
+ KVMTEST(EXC_HV, vec); \
_SOFTEN_TEST(EXC_HV, vec)
+#define KVMTEST_PR(vec) \
+ KVMTEST(EXC_STD, vec)
+
+#define KVMTEST_HV(vec) \
+ KVMTEST(EXC_HV, vec)
+
#define SOFTEN_NOTEST_PR(vec) _SOFTEN_TEST(EXC_STD, vec)
#define SOFTEN_NOTEST_HV(vec) _SOFTEN_TEST(EXC_HV, vec)
SET_SCRATCH0(r13); /* save r13 */ \
EXCEPTION_PROLOG_0(PACA_EXGEN); \
__EXCEPTION_PROLOG_1(PACA_EXGEN, extra, vec); \
- EXCEPTION_PROLOG_PSERIES_1(label##_common, h);
+ EXCEPTION_PROLOG_PSERIES_1(label, h);
#define _MASKABLE_EXCEPTION_PSERIES(vec, label, h, extra) \
__MASKABLE_EXCEPTION_PSERIES(vec, label, h, extra)
#define MASKABLE_EXCEPTION_PSERIES(loc, vec, label) \
- . = loc; \
- .globl label##_pSeries; \
-label##_pSeries: \
_MASKABLE_EXCEPTION_PSERIES(vec, label, \
EXC_STD, SOFTEN_TEST_PR)
+#define MASKABLE_EXCEPTION_PSERIES_OOL(vec, label) \
+ EXCEPTION_PROLOG_1(PACA_EXGEN, SOFTEN_TEST_PR, vec); \
+ EXCEPTION_PROLOG_PSERIES_1(label, EXC_STD)
+
#define MASKABLE_EXCEPTION_HV(loc, vec, label) \
- . = loc; \
- .globl label##_hv; \
-label##_hv: \
_MASKABLE_EXCEPTION_PSERIES(vec, label, \
EXC_HV, SOFTEN_TEST_HV)
#define MASKABLE_EXCEPTION_HV_OOL(vec, label) \
- .globl label##_hv; \
-label##_hv: \
EXCEPTION_PROLOG_1(PACA_EXGEN, SOFTEN_TEST_HV, vec); \
- EXCEPTION_PROLOG_PSERIES_1(label##_common, EXC_HV);
+ EXCEPTION_PROLOG_PSERIES_1(label, EXC_HV)
#define __MASKABLE_RELON_EXCEPTION_PSERIES(vec, label, h, extra) \
SET_SCRATCH0(r13); /* save r13 */ \
EXCEPTION_PROLOG_0(PACA_EXGEN); \
- __EXCEPTION_PROLOG_1(PACA_EXGEN, extra, vec); \
- EXCEPTION_RELON_PROLOG_PSERIES_1(label##_common, h);
-#define _MASKABLE_RELON_EXCEPTION_PSERIES(vec, label, h, extra) \
+ __EXCEPTION_PROLOG_1(PACA_EXGEN, extra, vec); \
+ EXCEPTION_RELON_PROLOG_PSERIES_1(label, h)
+
+#define _MASKABLE_RELON_EXCEPTION_PSERIES(vec, label, h, extra) \
__MASKABLE_RELON_EXCEPTION_PSERIES(vec, label, h, extra)
#define MASKABLE_RELON_EXCEPTION_PSERIES(loc, vec, label) \
- . = loc; \
- .globl label##_relon_pSeries; \
-label##_relon_pSeries: \
_MASKABLE_RELON_EXCEPTION_PSERIES(vec, label, \
EXC_STD, SOFTEN_NOTEST_PR)
#define MASKABLE_RELON_EXCEPTION_HV(loc, vec, label) \
- . = loc; \
- .globl label##_relon_hv; \
-label##_relon_hv: \
_MASKABLE_RELON_EXCEPTION_PSERIES(vec, label, \
EXC_HV, SOFTEN_NOTEST_HV)
#define MASKABLE_RELON_EXCEPTION_HV_OOL(vec, label) \
- .globl label##_relon_hv; \
-label##_relon_hv: \
EXCEPTION_PROLOG_1(PACA_EXGEN, SOFTEN_NOTEST_HV, vec); \
- EXCEPTION_PROLOG_PSERIES_1(label##_common, EXC_HV);
+ EXCEPTION_PROLOG_PSERIES_1(label, EXC_HV)
/*
* Our exception common code can be passed various "additions"
END_FTR_SECTION_IFSET(CPU_FTR_CTRL)
#define EXCEPTION_COMMON(trap, label, hdlr, ret, additions) \
- .align 7; \
- .globl label##_common; \
-label##_common: \
EXCEPTION_PROLOG_COMMON(trap, PACA_EXGEN); \
/* Volatile regs are potentially clobbered here */ \
additions; \
--- /dev/null
+#ifndef _ASM_POWERPC_HEAD_64_H
+#define _ASM_POWERPC_HEAD_64_H
+
+#include <asm/cache.h>
+
+/*
+ * We can't do CPP stringification and concatination directly into the section
+ * name for some reason, so these macros can do it for us.
+ */
+.macro define_ftsec name
+ .section ".head.text.\name\()","ax",@progbits
+.endm
+.macro define_data_ftsec name
+ .section ".head.data.\name\()","a",@progbits
+.endm
+.macro use_ftsec name
+ .section ".head.text.\name\()"
+.endm
+
+/*
+ * Fixed (location) sections are used by opening fixed sections and emitting
+ * fixed section entries into them before closing them. Multiple fixed sections
+ * can be open at any time.
+ *
+ * Each fixed section created in a .S file must have corresponding linkage
+ * directives including location, added to arch/powerpc/kernel/vmlinux.lds.S
+ *
+ * For each fixed section, code is generated into it in the order which it
+ * appears in the source. Fixed section entries can be placed at a fixed
+ * location within the section using _LOCATION postifx variants. These must
+ * be ordered according to their relative placements within the section.
+ *
+ * OPEN_FIXED_SECTION(section_name, start_address, end_address)
+ * FIXED_SECTION_ENTRY_BEGIN(section_name, label1)
+ *
+ * USE_FIXED_SECTION(section_name)
+ * label3:
+ * li r10,128
+ * mv r11,r10
+
+ * FIXED_SECTION_ENTRY_BEGIN_LOCATION(section_name, label2, start_address)
+ * FIXED_SECTION_ENTRY_END_LOCATION(section_name, label2, end_address)
+ * CLOSE_FIXED_SECTION(section_name)
+ *
+ * ZERO_FIXED_SECTION can be used to emit zeroed data.
+ *
+ * Troubleshooting:
+ * - If the build dies with "Error: attempt to move .org backwards" at
+ * CLOSE_FIXED_SECTION() or elsewhere, there may be something
+ * unexpected being added there. Remove the '. = x_len' line, rebuild, and
+ * check what is pushing the section down.
+ * - If the build dies in linking, check arch/powerpc/kernel/vmlinux.lds.S
+ * for instructions.
+ * - If the kernel crashes or hangs in very early boot, it could be linker
+ * stubs at the start of the main text.
+ */
+
+#define OPEN_FIXED_SECTION(sname, start, end) \
+ sname##_start = (start); \
+ sname##_end = (end); \
+ sname##_len = (end) - (start); \
+ define_ftsec sname; \
+ . = 0x0; \
+start_##sname:
+
+#define OPEN_TEXT_SECTION(start) \
+ text_start = (start); \
+ .section ".text","ax",@progbits; \
+ . = 0x0; \
+start_text:
+
+#define ZERO_FIXED_SECTION(sname, start, end) \
+ sname##_start = (start); \
+ sname##_end = (end); \
+ sname##_len = (end) - (start); \
+ define_data_ftsec sname; \
+ . = 0x0; \
+ . = sname##_len;
+
+#define USE_FIXED_SECTION(sname) \
+ fs_label = start_##sname; \
+ fs_start = sname##_start; \
+ use_ftsec sname;
+
+#define USE_TEXT_SECTION() \
+ fs_label = start_text; \
+ fs_start = text_start; \
+ .text
+
+#define CLOSE_FIXED_SECTION(sname) \
+ USE_FIXED_SECTION(sname); \
+ . = sname##_len; \
+end_##sname:
+
+
+#define __FIXED_SECTION_ENTRY_BEGIN(sname, name, __align) \
+ USE_FIXED_SECTION(sname); \
+ .align __align; \
+ .global name; \
+name:
+
+#define FIXED_SECTION_ENTRY_BEGIN(sname, name) \
+ __FIXED_SECTION_ENTRY_BEGIN(sname, name, 0)
+
+#define FIXED_SECTION_ENTRY_BEGIN_LOCATION(sname, name, start) \
+ USE_FIXED_SECTION(sname); \
+ name##_start = (start); \
+ .if (start) < sname##_start; \
+ .error "Fixed section underflow"; \
+ .abort; \
+ .endif; \
+ . = (start) - sname##_start; \
+ .global name; \
+name:
+
+#define FIXED_SECTION_ENTRY_END_LOCATION(sname, name, end) \
+ .if (end) > sname##_end; \
+ .error "Fixed section overflow"; \
+ .abort; \
+ .endif; \
+ .if (. - name > end - name##_start); \
+ .error "Fixed entry overflow"; \
+ .abort; \
+ .endif; \
+ . = ((end) - sname##_start); \
+
+
+/*
+ * These macros are used to change symbols in other fixed sections to be
+ * absolute or related to our current fixed section.
+ *
+ * - DEFINE_FIXED_SYMBOL / FIXED_SYMBOL_ABS_ADDR is used to find the
+ * absolute address of a symbol within a fixed section, from any section.
+ *
+ * - ABS_ADDR is used to find the absolute address of any symbol, from within
+ * a fixed section.
+ */
+#define DEFINE_FIXED_SYMBOL(label) \
+ label##_absolute = (label - fs_label + fs_start)
+
+#define FIXED_SYMBOL_ABS_ADDR(label) \
+ (label##_absolute)
+
+#define ABS_ADDR(label) (label - fs_label + fs_start)
+
+/*
+ * Following are the BOOK3S exception handler helper macros.
+ * Handlers come in a number of types, and each type has a number of varieties.
+ *
+ * EXC_REAL_* - real, unrelocated exception vectors
+ * EXC_VIRT_* - virt (AIL), unrelocated exception vectors
+ * TRAMP_REAL_* - real, unrelocated helpers (virt can call these)
+ * TRAMP_VIRT_* - virt, unreloc helpers (in practice, real can use)
+ * TRAMP_KVM - KVM handlers that get put into real, unrelocated
+ * EXC_COMMON_* - virt, relocated common handlers
+ *
+ * The EXC handlers are given a name, and branch to name_common, or the
+ * appropriate KVM or masking function. Vector handler verieties are as
+ * follows:
+ *
+ * EXC_{REAL|VIRT}_BEGIN/END - used to open-code the exception
+ *
+ * EXC_{REAL|VIRT} - standard exception
+ *
+ * EXC_{REAL|VIRT}_suffix
+ * where _suffix is:
+ * - _MASKABLE - maskable exception
+ * - _OOL - out of line with trampoline to common handler
+ * - _HV - HV exception
+ *
+ * There can be combinations, e.g., EXC_VIRT_OOL_MASKABLE_HV
+ *
+ * The one unusual case is __EXC_REAL_OOL_HV_DIRECT, which is
+ * an OOL vector that branches to a specified handler rather than the usual
+ * trampoline that goes to common. It, and other underscore macros, should
+ * be used with care.
+ *
+ * KVM handlers come in the following verieties:
+ * TRAMP_KVM
+ * TRAMP_KVM_SKIP
+ * TRAMP_KVM_HV
+ * TRAMP_KVM_HV_SKIP
+ *
+ * COMMON handlers come in the following verieties:
+ * EXC_COMMON_BEGIN/END - used to open-code the handler
+ * EXC_COMMON
+ * EXC_COMMON_ASYNC
+ * EXC_COMMON_HV
+ *
+ * TRAMP_REAL and TRAMP_VIRT can be used with BEGIN/END. KVM
+ * and OOL handlers are implemented as types of TRAMP and TRAMP_VIRT handlers.
+ */
+
+#define EXC_REAL_BEGIN(name, start, end) \
+ FIXED_SECTION_ENTRY_BEGIN_LOCATION(real_vectors, exc_real_##start##_##name, start)
+
+#define EXC_REAL_END(name, start, end) \
+ FIXED_SECTION_ENTRY_END_LOCATION(real_vectors, exc_real_##start##_##name, end)
+
+#define EXC_VIRT_BEGIN(name, start, end) \
+ FIXED_SECTION_ENTRY_BEGIN_LOCATION(virt_vectors, exc_virt_##start##_##name, start)
+
+#define EXC_VIRT_END(name, start, end) \
+ FIXED_SECTION_ENTRY_END_LOCATION(virt_vectors, exc_virt_##start##_##name, end)
+
+#define EXC_COMMON_BEGIN(name) \
+ USE_TEXT_SECTION(); \
+ .align 7; \
+ .global name; \
+ DEFINE_FIXED_SYMBOL(name); \
+name:
+
+#define TRAMP_REAL_BEGIN(name) \
+ FIXED_SECTION_ENTRY_BEGIN(real_trampolines, name)
+
+#define TRAMP_VIRT_BEGIN(name) \
+ FIXED_SECTION_ENTRY_BEGIN(virt_trampolines, name)
+
+#ifdef CONFIG_KVM_BOOK3S_64_HANDLER
+#define TRAMP_KVM_BEGIN(name) \
+ TRAMP_REAL_BEGIN(name)
+#else
+#define TRAMP_KVM_BEGIN(name)
+#endif
+
+#define EXC_REAL_NONE(start, end) \
+ FIXED_SECTION_ENTRY_BEGIN_LOCATION(real_vectors, exc_real_##start##_##unused, start); \
+ FIXED_SECTION_ENTRY_END_LOCATION(real_vectors, exc_real_##start##_##unused, end)
+
+#define EXC_VIRT_NONE(start, end) \
+ FIXED_SECTION_ENTRY_BEGIN_LOCATION(virt_vectors, exc_virt_##start##_##unused, start); \
+ FIXED_SECTION_ENTRY_END_LOCATION(virt_vectors, exc_virt_##start##_##unused, end);
+
+
+#define EXC_REAL(name, start, end) \
+ EXC_REAL_BEGIN(name, start, end); \
+ STD_EXCEPTION_PSERIES(start, name##_common); \
+ EXC_REAL_END(name, start, end);
+
+#define EXC_VIRT(name, start, end, realvec) \
+ EXC_VIRT_BEGIN(name, start, end); \
+ STD_RELON_EXCEPTION_PSERIES(start, realvec, name##_common); \
+ EXC_VIRT_END(name, start, end);
+
+#define EXC_REAL_MASKABLE(name, start, end) \
+ EXC_REAL_BEGIN(name, start, end); \
+ MASKABLE_EXCEPTION_PSERIES(start, start, name##_common); \
+ EXC_REAL_END(name, start, end);
+
+#define EXC_VIRT_MASKABLE(name, start, end, realvec) \
+ EXC_VIRT_BEGIN(name, start, end); \
+ MASKABLE_RELON_EXCEPTION_PSERIES(start, realvec, name##_common); \
+ EXC_VIRT_END(name, start, end);
+
+#define EXC_REAL_HV(name, start, end) \
+ EXC_REAL_BEGIN(name, start, end); \
+ STD_EXCEPTION_HV(start, start, name##_common); \
+ EXC_REAL_END(name, start, end);
+
+#define EXC_VIRT_HV(name, start, end, realvec) \
+ EXC_VIRT_BEGIN(name, start, end); \
+ STD_RELON_EXCEPTION_HV(start, realvec, name##_common); \
+ EXC_VIRT_END(name, start, end);
+
+#define __EXC_REAL_OOL(name, start, end) \
+ EXC_REAL_BEGIN(name, start, end); \
+ __OOL_EXCEPTION(start, label, tramp_real_##name); \
+ EXC_REAL_END(name, start, end);
+
+#define __TRAMP_REAL_REAL_OOL(name, vec) \
+ TRAMP_REAL_BEGIN(tramp_real_##name); \
+ STD_EXCEPTION_PSERIES_OOL(vec, name##_common); \
+
+#define EXC_REAL_OOL(name, start, end) \
+ __EXC_REAL_OOL(name, start, end); \
+ __TRAMP_REAL_REAL_OOL(name, start);
+
+#define __EXC_REAL_OOL_MASKABLE(name, start, end) \
+ __EXC_REAL_OOL(name, start, end);
+
+#define __TRAMP_REAL_REAL_OOL_MASKABLE(name, vec) \
+ TRAMP_REAL_BEGIN(tramp_real_##name); \
+ MASKABLE_EXCEPTION_PSERIES_OOL(vec, name##_common); \
+
+#define EXC_REAL_OOL_MASKABLE(name, start, end) \
+ __EXC_REAL_OOL_MASKABLE(name, start, end); \
+ __TRAMP_REAL_REAL_OOL_MASKABLE(name, start);
+
+#define __EXC_REAL_OOL_HV_DIRECT(name, start, end, handler) \
+ EXC_REAL_BEGIN(name, start, end); \
+ __OOL_EXCEPTION(start, label, handler); \
+ EXC_REAL_END(name, start, end);
+
+#define __EXC_REAL_OOL_HV(name, start, end) \
+ __EXC_REAL_OOL(name, start, end);
+
+#define __TRAMP_REAL_REAL_OOL_HV(name, vec) \
+ TRAMP_REAL_BEGIN(tramp_real_##name); \
+ STD_EXCEPTION_HV_OOL(vec, name##_common); \
+
+#define EXC_REAL_OOL_HV(name, start, end) \
+ __EXC_REAL_OOL_HV(name, start, end); \
+ __TRAMP_REAL_REAL_OOL_HV(name, start);
+
+#define __EXC_REAL_OOL_MASKABLE_HV(name, start, end) \
+ __EXC_REAL_OOL(name, start, end);
+
+#define __TRAMP_REAL_REAL_OOL_MASKABLE_HV(name, vec) \
+ TRAMP_REAL_BEGIN(tramp_real_##name); \
+ MASKABLE_EXCEPTION_HV_OOL(vec, name##_common); \
+
+#define EXC_REAL_OOL_MASKABLE_HV(name, start, end) \
+ __EXC_REAL_OOL_MASKABLE_HV(name, start, end); \
+ __TRAMP_REAL_REAL_OOL_MASKABLE_HV(name, start);
+
+#define __EXC_VIRT_OOL(name, start, end) \
+ EXC_VIRT_BEGIN(name, start, end); \
+ __OOL_EXCEPTION(start, label, tramp_virt_##name); \
+ EXC_VIRT_END(name, start, end);
+
+#define __TRAMP_REAL_VIRT_OOL(name, realvec) \
+ TRAMP_VIRT_BEGIN(tramp_virt_##name); \
+ STD_RELON_EXCEPTION_PSERIES_OOL(realvec, name##_common); \
+
+#define EXC_VIRT_OOL(name, start, end, realvec) \
+ __EXC_VIRT_OOL(name, start, end); \
+ __TRAMP_REAL_VIRT_OOL(name, realvec);
+
+#define __EXC_VIRT_OOL_MASKABLE(name, start, end) \
+ __EXC_VIRT_OOL(name, start, end);
+
+#define __TRAMP_REAL_VIRT_OOL_MASKABLE(name, realvec) \
+ TRAMP_VIRT_BEGIN(tramp_virt_##name); \
+ MASKABLE_RELON_EXCEPTION_PSERIES_OOL(realvec, name##_common); \
+
+#define EXC_VIRT_OOL_MASKABLE(name, start, end, realvec) \
+ __EXC_VIRT_OOL_MASKABLE(name, start, end); \
+ __TRAMP_REAL_VIRT_OOL_MASKABLE(name, realvec);
+
+#define __EXC_VIRT_OOL_HV(name, start, end) \
+ __EXC_VIRT_OOL(name, start, end);
+
+#define __TRAMP_REAL_VIRT_OOL_HV(name, realvec) \
+ TRAMP_VIRT_BEGIN(tramp_virt_##name); \
+ STD_RELON_EXCEPTION_HV_OOL(realvec, name##_common); \
+
+#define EXC_VIRT_OOL_HV(name, start, end, realvec) \
+ __EXC_VIRT_OOL_HV(name, start, end); \
+ __TRAMP_REAL_VIRT_OOL_HV(name, realvec);
+
+#define __EXC_VIRT_OOL_MASKABLE_HV(name, start, end) \
+ __EXC_VIRT_OOL(name, start, end);
+
+#define __TRAMP_REAL_VIRT_OOL_MASKABLE_HV(name, realvec) \
+ TRAMP_VIRT_BEGIN(tramp_virt_##name); \
+ MASKABLE_RELON_EXCEPTION_HV_OOL(realvec, name##_common); \
+
+#define EXC_VIRT_OOL_MASKABLE_HV(name, start, end, realvec) \
+ __EXC_VIRT_OOL_MASKABLE_HV(name, start, end); \
+ __TRAMP_REAL_VIRT_OOL_MASKABLE_HV(name, realvec);
+
+#define TRAMP_KVM(area, n) \
+ TRAMP_KVM_BEGIN(do_kvm_##n); \
+ KVM_HANDLER(area, EXC_STD, n); \
+
+#define TRAMP_KVM_SKIP(area, n) \
+ TRAMP_KVM_BEGIN(do_kvm_##n); \
+ KVM_HANDLER_SKIP(area, EXC_STD, n); \
+
+/*
+ * HV variant exceptions get the 0x2 bit added to their trap number.
+ */
+#define TRAMP_KVM_HV(area, n) \
+ TRAMP_KVM_BEGIN(do_kvm_H##n); \
+ KVM_HANDLER(area, EXC_HV, n + 0x2); \
+
+#define TRAMP_KVM_HV_SKIP(area, n) \
+ TRAMP_KVM_BEGIN(do_kvm_H##n); \
+ KVM_HANDLER_SKIP(area, EXC_HV, n + 0x2); \
+
+#define EXC_COMMON(name, realvec, hdlr) \
+ EXC_COMMON_BEGIN(name); \
+ STD_EXCEPTION_COMMON(realvec, name, hdlr); \
+
+#define EXC_COMMON_ASYNC(name, realvec, hdlr) \
+ EXC_COMMON_BEGIN(name); \
+ STD_EXCEPTION_COMMON_ASYNC(realvec, name, hdlr); \
+
+#define EXC_COMMON_HV(name, realvec, hdlr) \
+ EXC_COMMON_BEGIN(name); \
+ STD_EXCEPTION_COMMON(realvec + 0x2, name, hdlr); \
+
+#endif /* _ASM_POWERPC_HEAD_64_H */
#ifdef CONFIG_SPAPR_TCE_IOMMU
struct mm_iommu_table_group_mem_t;
+extern int isolate_lru_page(struct page *page); /* from internal.h */
extern bool mm_iommu_preregistered(void);
extern long mm_iommu_get(unsigned long ua, unsigned long entries,
struct mm_iommu_table_group_mem_t **pmem);
struct task_struct *tsk)
{
/* Mark this context has been used on the new CPU */
- cpumask_set_cpu(smp_processor_id(), mm_cpumask(next));
+ if (!cpumask_test_cpu(smp_processor_id(), mm_cpumask(next)))
+ cpumask_set_cpu(smp_processor_id(), mm_cpumask(next));
/* 32-bit keeps track of the current PGDIR in the thread struct */
#ifdef CONFIG_PPC32
#include <misc/cxl-base.h>
#include <asm/opal-api.h>
-#define PCI_SLOT_ID_PREFIX 0x8000000000000000
+#define PCI_SLOT_ID_PREFIX (1UL << 63)
#define PCI_SLOT_ID(phb_id, bdfn) \
(PCI_SLOT_ID_PREFIX | ((uint64_t)(bdfn) << 16) | (phb_id))
#define PNV_PHP_STATE_POPULATED 2
#define PNV_PHP_STATE_OFFLINE 3
int state;
+ int irq;
+ struct workqueue_struct *wq;
struct device_node *dn;
struct pci_dev *pdev;
struct pci_bus *bus;
#define PPC_INST_STWU 0x94000000
#define PPC_INST_MFLR 0x7c0802a6
#define PPC_INST_MTLR 0x7c0803a6
+#define PPC_INST_MTCTR 0x7c0903a6
#define PPC_INST_CMPWI 0x2c000000
#define PPC_INST_CMPDI 0x2c200000
#define PPC_INST_CMPW 0x7c000000
#define PPC_INST_SUB 0x7c000050
#define PPC_INST_BLR 0x4e800020
#define PPC_INST_BLRL 0x4e800021
+#define PPC_INST_BCTR 0x4e800420
#define PPC_INST_MULLD 0x7c0001d2
#define PPC_INST_MULLW 0x7c0001d6
#define PPC_INST_MULHWU 0x7c000016
} mm_segment_t;
#define TS_FPR(i) fp_state.fpr[i][TS_FPROFFSET]
-#define TS_TRANS_FPR(i) transact_fp.fpr[i][TS_FPROFFSET]
+#define TS_CKFPR(i) ckfp_state.fpr[i][TS_FPROFFSET]
/* FP and VSX 0-31 register set */
struct thread_fp_state {
int used_spe; /* set if process has used spe */
#endif /* CONFIG_SPE */
#ifdef CONFIG_PPC_TRANSACTIONAL_MEM
+ u8 load_tm;
u64 tm_tfhar; /* Transaction fail handler addr */
u64 tm_texasr; /* Transaction exception & summary */
u64 tm_tfiar; /* Transaction fail instr address reg */
unsigned long tm_dscr;
/*
- * Transactional FP and VSX 0-31 register set.
- * NOTE: the sense of these is the opposite of the integer ckpt_regs!
+ * Checkpointed FP and VSX 0-31 register set.
*
* When a transaction is active/signalled/scheduled etc., *regs is the
* most recent set of/speculated GPRs with ckpt_regs being the older
* checkpointed regs to which we roll back if transaction aborts.
*
- * However, fpr[] is the checkpointed 'base state' of FP regs, and
- * transact_fpr[] is the new set of transactional values.
- * VRs work the same way.
+ * These are analogous to how ckpt_regs and pt_regs work
*/
- struct thread_fp_state transact_fp;
- struct thread_vr_state transact_vr;
- unsigned long transact_vrsave;
+ struct thread_fp_state ckfp_state; /* Checkpointed FP state */
+ struct thread_vr_state ckvr_state; /* Checkpointed VR state */
+ unsigned long ckvrsave; /* Checkpointed VRSAVE */
#endif /* CONFIG_PPC_TRANSACTIONAL_MEM */
#ifdef CONFIG_KVM_BOOK3S_32_HANDLER
void* kvm_shadow_vcpu; /* KVM internal data */
#define wrtspr(rn) asm volatile("mtspr " __stringify(rn) ",0" : \
: : "memory")
-extern void msr_check_and_set(unsigned long bits);
+extern unsigned long msr_check_and_set(unsigned long bits);
extern bool strict_msr_control;
extern void __msr_check_and_clear(unsigned long bits);
static inline void msr_check_and_clear(unsigned long bits)
#include <uapi/asm/signal.h>
#include <uapi/asm/ptrace.h>
-extern unsigned long get_tm_stackpointer(struct pt_regs *regs);
-
#endif /* _ASM_POWERPC_SIGNAL_H */
#ifndef __ASSEMBLY__
-#ifdef CONFIG_PPC_TRANSACTIONAL_MEM
-extern void do_load_up_transact_fpu(struct thread_struct *thread);
-extern void do_load_up_transact_altivec(struct thread_struct *thread);
-#endif
-
extern void tm_enable(void);
extern void tm_reclaim(struct thread_struct *thread,
unsigned long orig_msr, uint8_t cause);
DEFINE(THREAD_TM_PPR, offsetof(struct thread_struct, tm_ppr));
DEFINE(THREAD_TM_DSCR, offsetof(struct thread_struct, tm_dscr));
DEFINE(PT_CKPT_REGS, offsetof(struct thread_struct, ckpt_regs));
- DEFINE(THREAD_TRANSACT_VRSTATE, offsetof(struct thread_struct,
- transact_vr));
- DEFINE(THREAD_TRANSACT_VRSAVE, offsetof(struct thread_struct,
- transact_vrsave));
- DEFINE(THREAD_TRANSACT_FPSTATE, offsetof(struct thread_struct,
- transact_fp));
+ DEFINE(THREAD_CKVRSTATE, offsetof(struct thread_struct,
+ ckvr_state));
+ DEFINE(THREAD_CKVRSAVE, offsetof(struct thread_struct,
+ ckvrsave));
+ DEFINE(THREAD_CKFPSTATE, offsetof(struct thread_struct,
+ ckfp_state));
/* Local pt_regs on stack for Transactional Memory funcs. */
DEFINE(TM_FRAME_SIZE, STACK_FRAME_OVERHEAD +
sizeof(struct pt_regs) + 16);
/* Lock to avoid races due to multiple reports of an error */
DEFINE_RAW_SPINLOCK(confirm_error_lock);
+EXPORT_SYMBOL_GPL(confirm_error_lock);
/* Lock to protect passed flags */
static DEFINE_MUTEX(eeh_dev_mutex);
if (eeh_enabled())
pr_info("EEH: PCI Enhanced I/O Error Handling Enabled\n");
else
- pr_warn("EEH: No capable adapters found\n");
+ pr_info("EEH: No capable adapters found\n");
return ret;
}
break;
case EEH_OPT_THAW_MMIO:
case EEH_OPT_THAW_DMA:
+ case EEH_OPT_FREEZE_PE:
if (!eeh_ops || !eeh_ops->set_option) {
ret = -ENOENT;
break;
{
eeh_pe_traverse(pe, __eeh_pe_state_mark, &state);
}
+EXPORT_SYMBOL_GPL(eeh_pe_state_mark);
static void *__eeh_pe_dev_mode_mark(void *data, void *flag)
{
#include <asm/exception-64s.h>
#include <asm/ptrace.h>
#include <asm/cpuidle.h>
+#include <asm/head-64.h>
/*
+ * There are a few constraints to be concerned with.
+ * - Real mode exceptions code/data must be located at their physical location.
+ * - Virtual mode exceptions must be mapped at their 0xc000... location.
+ * - Fixed location code must not call directly beyond the __end_interrupts
+ * area when built with CONFIG_RELOCATABLE. LOAD_HANDLER / bctr sequence
+ * must be used.
+ * - LOAD_HANDLER targets must be within first 64K of physical 0 /
+ * virtual 0xc00...
+ * - Conditional branch targets must be within +/-32K of caller.
+ *
+ * "Virtual exceptions" run with relocation on (MSR_IR=1, MSR_DR=1), and
+ * therefore don't have to run in physically located code or rfid to
+ * virtual mode kernel code. However on relocatable kernels they do have
+ * to branch to KERNELBASE offset because the rest of the kernel (outside
+ * the exception vectors) may be located elsewhere.
+ *
+ * Virtual exceptions correspond with physical, except their entry points
+ * are offset by 0xc000000000000000 and also tend to get an added 0x4000
+ * offset applied. Virtual exceptions are enabled with the Alternate
+ * Interrupt Location (AIL) bit set in the LPCR. However this does not
+ * guarantee they will be delivered virtually. Some conditions (see the ISA)
+ * cause exceptions to be delivered in real mode.
+ *
+ * It's impossible to receive interrupts below 0x300 via AIL.
+ *
+ * KVM: None of the virtual exceptions are from the guest. Anything that
+ * escalated to HV=1 from HV=0 is delivered via real mode handlers.
+ *
+ *
* We layout physical memory as follows:
* 0x0000 - 0x00ff : Secondary processor spin code
- * 0x0100 - 0x17ff : pSeries Interrupt prologs
- * 0x1800 - 0x4000 : interrupt support common interrupt prologs
- * 0x4000 - 0x5fff : pSeries interrupts with IR=1,DR=1
- * 0x6000 - 0x6fff : more interrupt support including for IR=1,DR=1
+ * 0x0100 - 0x18ff : Real mode pSeries interrupt vectors
+ * 0x1900 - 0x3fff : Real mode trampolines
+ * 0x4000 - 0x58ff : Relon (IR=1,DR=1) mode pSeries interrupt vectors
+ * 0x5900 - 0x6fff : Relon mode trampolines
* 0x7000 - 0x7fff : FWNMI data area
- * 0x8000 - 0x8fff : Initial (CPU0) segment table
- * 0x9000 - : Early init and support code
+ * 0x8000 - .... : Common interrupt handlers, remaining early
+ * setup code, rest of kernel.
+ *
+ * We could reclaim 0x4000-0x42ff for real mode trampolines if the space
+ * is necessary. Until then it's more consistent to explicitly put VIRT_NONE
+ * vectors there.
*/
- /* Syscall routine is used twice, in reloc-off and reloc-on paths */
-#define SYSCALL_PSERIES_1 \
-BEGIN_FTR_SECTION \
- cmpdi r0,0x1ebe ; \
- beq- 1f ; \
-END_FTR_SECTION_IFSET(CPU_FTR_REAL_LE) \
- mr r9,r13 ; \
- GET_PACA(r13) ; \
- mfspr r11,SPRN_SRR0 ; \
-0:
-
-#define SYSCALL_PSERIES_2_RFID \
- mfspr r12,SPRN_SRR1 ; \
- LOAD_HANDLER(r10, system_call_common) ; \
- mtspr SPRN_SRR0,r10 ; \
- ld r10,PACAKMSR(r13) ; \
- mtspr SPRN_SRR1,r10 ; \
- rfid ; \
- b . ; /* prevent speculative execution */
-
-#define SYSCALL_PSERIES_3 \
- /* Fast LE/BE switch system call */ \
-1: mfspr r12,SPRN_SRR1 ; \
- xori r12,r12,MSR_LE ; \
- mtspr SPRN_SRR1,r12 ; \
- rfid ; /* return to userspace */ \
- b . ; /* prevent speculative execution */
-
-#if defined(CONFIG_RELOCATABLE)
- /*
- * We can't branch directly so we do it via the CTR which
- * is volatile across system calls.
- */
-#define SYSCALL_PSERIES_2_DIRECT \
- LOAD_HANDLER(r12, system_call_common) ; \
- mtctr r12 ; \
- mfspr r12,SPRN_SRR1 ; \
- li r10,MSR_RI ; \
- mtmsrd r10,1 ; \
- bctr ;
+OPEN_FIXED_SECTION(real_vectors, 0x0100, 0x1900)
+OPEN_FIXED_SECTION(real_trampolines, 0x1900, 0x4000)
+OPEN_FIXED_SECTION(virt_vectors, 0x4000, 0x5900)
+OPEN_FIXED_SECTION(virt_trampolines, 0x5900, 0x7000)
+#if defined(CONFIG_PPC_PSERIES) || defined(CONFIG_PPC_POWERNV)
+/*
+ * Data area reserved for FWNMI option.
+ * This address (0x7000) is fixed by the RPA.
+ * pseries and powernv need to keep the whole page from
+ * 0x7000 to 0x8000 free for use by the firmware
+ */
+ZERO_FIXED_SECTION(fwnmi_page, 0x7000, 0x8000)
+OPEN_TEXT_SECTION(0x8000)
#else
- /* We can branch directly */
-#define SYSCALL_PSERIES_2_DIRECT \
- mfspr r12,SPRN_SRR1 ; \
- li r10,MSR_RI ; \
- mtmsrd r10,1 ; /* Set RI (EE=0) */ \
- b system_call_common ;
+OPEN_TEXT_SECTION(0x7000)
#endif
+USE_FIXED_SECTION(real_vectors)
+
/*
* This is the start of the interrupt handlers for pSeries
* This code runs with relocation off.
* Therefore any relative branches in this section must only
* branch to labels in this section.
*/
- . = 0x100
.globl __start_interrupts
__start_interrupts:
- .globl system_reset_pSeries;
-system_reset_pSeries:
+/* No virt vectors corresponding with 0x0..0x100 */
+EXC_VIRT_NONE(0x4000, 0x4100)
+
+EXC_REAL_BEGIN(system_reset, 0x100, 0x200)
SET_SCRATCH0(r13)
#ifdef CONFIG_PPC_P7_NAP
BEGIN_FTR_SECTION
#endif /* CONFIG_PPC_P7_NAP */
EXCEPTION_PROLOG_PSERIES(PACA_EXGEN, system_reset_common, EXC_STD,
NOTEST, 0x100)
+EXC_REAL_END(system_reset, 0x100, 0x200)
+EXC_VIRT_NONE(0x4100, 0x4200)
+EXC_COMMON(system_reset_common, 0x100, system_reset_exception)
+
+#ifdef CONFIG_PPC_PSERIES
+/*
+ * Vectors for the FWNMI option. Share common code.
+ */
+TRAMP_REAL_BEGIN(system_reset_fwnmi)
+ SET_SCRATCH0(r13) /* save r13 */
+ EXCEPTION_PROLOG_PSERIES(PACA_EXGEN, system_reset_common, EXC_STD,
+ NOTEST, 0x100)
+#endif /* CONFIG_PPC_PSERIES */
- . = 0x200
-machine_check_pSeries_1:
+
+EXC_REAL_BEGIN(machine_check, 0x200, 0x300)
/* This is moved out of line as it can be patched by FW, but
* some code path might still want to branch into the original
* vector
FTR_SECTION_ELSE
b machine_check_pSeries_0
ALT_FTR_SECTION_END_IFSET(CPU_FTR_HVMODE)
-
- . = 0x300
- .globl data_access_pSeries
-data_access_pSeries:
- SET_SCRATCH0(r13)
- EXCEPTION_PROLOG_PSERIES(PACA_EXGEN, data_access_common, EXC_STD,
- KVMTEST, 0x300)
-
- . = 0x380
- .globl data_access_slb_pSeries
-data_access_slb_pSeries:
- SET_SCRATCH0(r13)
- EXCEPTION_PROLOG_0(PACA_EXSLB)
- EXCEPTION_PROLOG_1(PACA_EXSLB, KVMTEST, 0x380)
- std r3,PACA_EXSLB+EX_R3(r13)
- mfspr r3,SPRN_DAR
- mfspr r12,SPRN_SRR1
- crset 4*cr6+eq
-#ifndef CONFIG_RELOCATABLE
- b slb_miss_realmode
-#else
- /*
- * We can't just use a direct branch to slb_miss_realmode
- * because the distance from here to there depends on where
- * the kernel ends up being put.
- */
- mfctr r11
- LOAD_HANDLER(r10, slb_miss_realmode)
- mtctr r10
- bctr
-#endif
-
- STD_EXCEPTION_PSERIES(0x400, instruction_access)
-
- . = 0x480
- .globl instruction_access_slb_pSeries
-instruction_access_slb_pSeries:
- SET_SCRATCH0(r13)
- EXCEPTION_PROLOG_0(PACA_EXSLB)
- EXCEPTION_PROLOG_1(PACA_EXSLB, KVMTEST, 0x480)
- std r3,PACA_EXSLB+EX_R3(r13)
- mfspr r3,SPRN_SRR0 /* SRR0 is faulting address */
- mfspr r12,SPRN_SRR1
- crclr 4*cr6+eq
-#ifndef CONFIG_RELOCATABLE
- b slb_miss_realmode
-#else
- mfctr r11
- LOAD_HANDLER(r10, slb_miss_realmode)
- mtctr r10
- bctr
-#endif
-
- /* We open code these as we can't have a ". = x" (even with
- * x = "." within a feature section
- */
- . = 0x500;
- .globl hardware_interrupt_pSeries;
- .globl hardware_interrupt_hv;
-hardware_interrupt_pSeries:
-hardware_interrupt_hv:
- BEGIN_FTR_SECTION
- _MASKABLE_EXCEPTION_PSERIES(0x502, hardware_interrupt,
- EXC_HV, SOFTEN_TEST_HV)
- KVM_HANDLER(PACA_EXGEN, EXC_HV, 0x502)
- FTR_SECTION_ELSE
- _MASKABLE_EXCEPTION_PSERIES(0x500, hardware_interrupt,
- EXC_STD, SOFTEN_TEST_PR)
- KVM_HANDLER(PACA_EXGEN, EXC_STD, 0x500)
- ALT_FTR_SECTION_END_IFSET(CPU_FTR_HVMODE | CPU_FTR_ARCH_206)
-
- STD_EXCEPTION_PSERIES(0x600, alignment)
- KVM_HANDLER(PACA_EXGEN, EXC_STD, 0x600)
-
- STD_EXCEPTION_PSERIES(0x700, program_check)
- KVM_HANDLER(PACA_EXGEN, EXC_STD, 0x700)
-
- STD_EXCEPTION_PSERIES(0x800, fp_unavailable)
- KVM_HANDLER(PACA_EXGEN, EXC_STD, 0x800)
-
- . = 0x900
- .globl decrementer_pSeries
-decrementer_pSeries:
- _MASKABLE_EXCEPTION_PSERIES(0x900, decrementer, EXC_STD, SOFTEN_TEST_PR)
-
- STD_EXCEPTION_HV(0x980, 0x982, hdecrementer)
-
- MASKABLE_EXCEPTION_PSERIES(0xa00, 0xa00, doorbell_super)
- KVM_HANDLER(PACA_EXGEN, EXC_STD, 0xa00)
-
- STD_EXCEPTION_PSERIES(0xb00, trap_0b)
- KVM_HANDLER(PACA_EXGEN, EXC_STD, 0xb00)
-
- . = 0xc00
- .globl system_call_pSeries
-system_call_pSeries:
- /*
- * If CONFIG_KVM_BOOK3S_64_HANDLER is set, save the PPR (on systems
- * that support it) before changing to HMT_MEDIUM. That allows the KVM
- * code to save that value into the guest state (it is the guest's PPR
- * value). Otherwise just change to HMT_MEDIUM as userspace has
- * already saved the PPR.
- */
-#ifdef CONFIG_KVM_BOOK3S_64_HANDLER
- SET_SCRATCH0(r13)
- GET_PACA(r13)
- std r9,PACA_EXGEN+EX_R9(r13)
- OPT_GET_SPR(r9, SPRN_PPR, CPU_FTR_HAS_PPR);
- HMT_MEDIUM;
- std r10,PACA_EXGEN+EX_R10(r13)
- OPT_SAVE_REG_TO_PACA(PACA_EXGEN+EX_PPR, r9, CPU_FTR_HAS_PPR);
- mfcr r9
- KVMTEST(0xc00)
- GET_SCRATCH0(r13)
-#else
- HMT_MEDIUM;
-#endif
- SYSCALL_PSERIES_1
- SYSCALL_PSERIES_2_RFID
- SYSCALL_PSERIES_3
- KVM_HANDLER(PACA_EXGEN, EXC_STD, 0xc00)
-
- STD_EXCEPTION_PSERIES(0xd00, single_step)
- KVM_HANDLER(PACA_EXGEN, EXC_STD, 0xd00)
-
- /* At 0xe??? we have a bunch of hypervisor exceptions, we branch
- * out of line to handle them
- */
- . = 0xe00
-hv_data_storage_trampoline:
- SET_SCRATCH0(r13)
- EXCEPTION_PROLOG_0(PACA_EXGEN)
- b h_data_storage_hv
-
- . = 0xe20
-hv_instr_storage_trampoline:
- SET_SCRATCH0(r13)
- EXCEPTION_PROLOG_0(PACA_EXGEN)
- b h_instr_storage_hv
-
- . = 0xe40
-emulation_assist_trampoline:
- SET_SCRATCH0(r13)
- EXCEPTION_PROLOG_0(PACA_EXGEN)
- b emulation_assist_hv
-
- . = 0xe60
-hv_exception_trampoline:
- SET_SCRATCH0(r13)
- EXCEPTION_PROLOG_0(PACA_EXGEN)
- b hmi_exception_early
-
- . = 0xe80
-hv_doorbell_trampoline:
- SET_SCRATCH0(r13)
- EXCEPTION_PROLOG_0(PACA_EXGEN)
- b h_doorbell_hv
-
- . = 0xea0
-hv_virt_irq_trampoline:
- SET_SCRATCH0(r13)
- EXCEPTION_PROLOG_0(PACA_EXGEN)
- b h_virt_irq_hv
-
- /* We need to deal with the Altivec unavailable exception
- * here which is at 0xf20, thus in the middle of the
- * prolog code of the PerformanceMonitor one. A little
- * trickery is thus necessary
- */
- . = 0xf00
-performance_monitor_pseries_trampoline:
- SET_SCRATCH0(r13)
- EXCEPTION_PROLOG_0(PACA_EXGEN)
- b performance_monitor_pSeries
-
- . = 0xf20
-altivec_unavailable_pseries_trampoline:
- SET_SCRATCH0(r13)
- EXCEPTION_PROLOG_0(PACA_EXGEN)
- b altivec_unavailable_pSeries
-
- . = 0xf40
-vsx_unavailable_pseries_trampoline:
- SET_SCRATCH0(r13)
- EXCEPTION_PROLOG_0(PACA_EXGEN)
- b vsx_unavailable_pSeries
-
- . = 0xf60
-facility_unavailable_trampoline:
- SET_SCRATCH0(r13)
- EXCEPTION_PROLOG_0(PACA_EXGEN)
- b facility_unavailable_pSeries
-
- . = 0xf80
-hv_facility_unavailable_trampoline:
- SET_SCRATCH0(r13)
- EXCEPTION_PROLOG_0(PACA_EXGEN)
- b hv_facility_unavailable_hv
-
-#ifdef CONFIG_CBE_RAS
- STD_EXCEPTION_HV(0x1200, 0x1202, cbe_system_error)
- KVM_HANDLER_SKIP(PACA_EXGEN, EXC_HV, 0x1202)
-#endif /* CONFIG_CBE_RAS */
-
- STD_EXCEPTION_PSERIES(0x1300, instruction_breakpoint)
- KVM_HANDLER_SKIP(PACA_EXGEN, EXC_STD, 0x1300)
-
- . = 0x1500
- .global denorm_exception_hv
-denorm_exception_hv:
- mtspr SPRN_SPRG_HSCRATCH0,r13
- EXCEPTION_PROLOG_0(PACA_EXGEN)
- EXCEPTION_PROLOG_1(PACA_EXGEN, NOTEST, 0x1500)
-
-#ifdef CONFIG_PPC_DENORMALISATION
- mfspr r10,SPRN_HSRR1
- mfspr r11,SPRN_HSRR0 /* save HSRR0 */
- andis. r10,r10,(HSRR1_DENORM)@h /* denorm? */
- addi r11,r11,-4 /* HSRR0 is next instruction */
- bne+ denorm_assist
-#endif
-
- KVMTEST(0x1500)
- EXCEPTION_PROLOG_PSERIES_1(denorm_common, EXC_HV)
- KVM_HANDLER_SKIP(PACA_EXGEN, EXC_STD, 0x1500)
-
-#ifdef CONFIG_CBE_RAS
- STD_EXCEPTION_HV(0x1600, 0x1602, cbe_maintenance)
- KVM_HANDLER_SKIP(PACA_EXGEN, EXC_HV, 0x1602)
-#endif /* CONFIG_CBE_RAS */
-
- STD_EXCEPTION_PSERIES(0x1700, altivec_assist)
- KVM_HANDLER(PACA_EXGEN, EXC_STD, 0x1700)
-
-#ifdef CONFIG_CBE_RAS
- STD_EXCEPTION_HV(0x1800, 0x1802, cbe_thermal)
- KVM_HANDLER_SKIP(PACA_EXGEN, EXC_HV, 0x1802)
-#else
- . = 0x1800
-#endif /* CONFIG_CBE_RAS */
-
-
-/*** Out of line interrupts support ***/
-
- .align 7
- /* moved from 0x200 */
-machine_check_powernv_early:
+EXC_REAL_END(machine_check, 0x200, 0x300)
+EXC_VIRT_NONE(0x4200, 0x4300)
+TRAMP_REAL_BEGIN(machine_check_powernv_early)
BEGIN_FTR_SECTION
EXCEPTION_PROLOG_1(PACA_EXMC, NOTEST, 0x200)
/*
b . /* prevent speculative execution */
END_FTR_SECTION_IFSET(CPU_FTR_HVMODE)
-machine_check_pSeries:
+TRAMP_REAL_BEGIN(machine_check_pSeries)
.globl machine_check_fwnmi
machine_check_fwnmi:
SET_SCRATCH0(r13) /* save r13 */
EXCEPTION_PROLOG_0(PACA_EXMC)
machine_check_pSeries_0:
- EXCEPTION_PROLOG_1(PACA_EXMC, KVMTEST, 0x200)
+ EXCEPTION_PROLOG_1(PACA_EXMC, KVMTEST_PR, 0x200)
/*
* The following is essentially EXCEPTION_PROLOG_PSERIES_1 with the
* difference that MSR_RI is not enabled, because PACA_EXMC is being
rfid
b . /* prevent speculative execution */
- KVM_HANDLER_SKIP(PACA_EXMC, EXC_STD, 0x200)
- KVM_HANDLER_SKIP(PACA_EXGEN, EXC_STD, 0x300)
- KVM_HANDLER_SKIP(PACA_EXSLB, EXC_STD, 0x380)
- KVM_HANDLER(PACA_EXGEN, EXC_STD, 0x400)
- KVM_HANDLER(PACA_EXSLB, EXC_STD, 0x480)
- KVM_HANDLER(PACA_EXGEN, EXC_STD, 0x900)
- KVM_HANDLER(PACA_EXGEN, EXC_HV, 0x982)
-
-#ifdef CONFIG_PPC_DENORMALISATION
-denorm_assist:
-BEGIN_FTR_SECTION
-/*
- * To denormalise we need to move a copy of the register to itself.
- * For POWER6 do that here for all FP regs.
- */
- mfmsr r10
- ori r10,r10,(MSR_FP|MSR_FE0|MSR_FE1)
- xori r10,r10,(MSR_FE0|MSR_FE1)
- mtmsrd r10
- sync
+TRAMP_KVM_SKIP(PACA_EXMC, 0x200)
-#define FMR2(n) fmr (n), (n) ; fmr n+1, n+1
-#define FMR4(n) FMR2(n) ; FMR2(n+2)
-#define FMR8(n) FMR4(n) ; FMR4(n+4)
-#define FMR16(n) FMR8(n) ; FMR8(n+8)
-#define FMR32(n) FMR16(n) ; FMR16(n+16)
- FMR32(0)
+EXC_COMMON_BEGIN(machine_check_common)
+ /*
+ * Machine check is different because we use a different
+ * save area: PACA_EXMC instead of PACA_EXGEN.
+ */
+ mfspr r10,SPRN_DAR
+ std r10,PACA_EXMC+EX_DAR(r13)
+ mfspr r10,SPRN_DSISR
+ stw r10,PACA_EXMC+EX_DSISR(r13)
+ EXCEPTION_PROLOG_COMMON(0x200, PACA_EXMC)
+ FINISH_NAP
+ RECONCILE_IRQ_STATE(r10, r11)
+ ld r3,PACA_EXMC+EX_DAR(r13)
+ lwz r4,PACA_EXMC+EX_DSISR(r13)
+ /* Enable MSR_RI when finished with PACA_EXMC */
+ li r10,MSR_RI
+ mtmsrd r10,1
+ std r3,_DAR(r1)
+ std r4,_DSISR(r1)
+ bl save_nvgprs
+ addi r3,r1,STACK_FRAME_OVERHEAD
+ bl machine_check_exception
+ b ret_from_except
-FTR_SECTION_ELSE
-/*
- * To denormalise we need to move a copy of the register to itself.
- * For POWER7 do that here for the first 32 VSX registers only.
- */
- mfmsr r10
- oris r10,r10,MSR_VSX@h
- mtmsrd r10
- sync
-
-#define XVCPSGNDP2(n) XVCPSGNDP(n,n,n) ; XVCPSGNDP(n+1,n+1,n+1)
-#define XVCPSGNDP4(n) XVCPSGNDP2(n) ; XVCPSGNDP2(n+2)
-#define XVCPSGNDP8(n) XVCPSGNDP4(n) ; XVCPSGNDP4(n+4)
-#define XVCPSGNDP16(n) XVCPSGNDP8(n) ; XVCPSGNDP8(n+8)
-#define XVCPSGNDP32(n) XVCPSGNDP16(n) ; XVCPSGNDP16(n+16)
- XVCPSGNDP32(0)
+#define MACHINE_CHECK_HANDLER_WINDUP \
+ /* Clear MSR_RI before setting SRR0 and SRR1. */\
+ li r0,MSR_RI; \
+ mfmsr r9; /* get MSR value */ \
+ andc r9,r9,r0; \
+ mtmsrd r9,1; /* Clear MSR_RI */ \
+ /* Move original SRR0 and SRR1 into the respective regs */ \
+ ld r9,_MSR(r1); \
+ mtspr SPRN_SRR1,r9; \
+ ld r3,_NIP(r1); \
+ mtspr SPRN_SRR0,r3; \
+ ld r9,_CTR(r1); \
+ mtctr r9; \
+ ld r9,_XER(r1); \
+ mtxer r9; \
+ ld r9,_LINK(r1); \
+ mtlr r9; \
+ REST_GPR(0, r1); \
+ REST_8GPRS(2, r1); \
+ REST_GPR(10, r1); \
+ ld r11,_CCR(r1); \
+ mtcr r11; \
+ /* Decrement paca->in_mce. */ \
+ lhz r12,PACA_IN_MCE(r13); \
+ subi r12,r12,1; \
+ sth r12,PACA_IN_MCE(r13); \
+ REST_GPR(11, r1); \
+ REST_2GPRS(12, r1); \
+ /* restore original r1. */ \
+ ld r1,GPR1(r1)
-ALT_FTR_SECTION_END_IFCLR(CPU_FTR_ARCH_206)
+ /*
+ * Handle machine check early in real mode. We come here with
+ * ME=1, MMU (IR=0 and DR=0) off and using MC emergency stack.
+ */
+EXC_COMMON_BEGIN(machine_check_handle_early)
+ std r0,GPR0(r1) /* Save r0 */
+ EXCEPTION_PROLOG_COMMON_3(0x200)
+ bl save_nvgprs
+ addi r3,r1,STACK_FRAME_OVERHEAD
+ bl machine_check_early
+ std r3,RESULT(r1) /* Save result */
+ ld r12,_MSR(r1)
+#ifdef CONFIG_PPC_P7_NAP
+ /*
+ * Check if thread was in power saving mode. We come here when any
+ * of the following is true:
+ * a. thread wasn't in power saving mode
+ * b. thread was in power saving mode with no state loss,
+ * supervisor state loss or hypervisor state loss.
+ *
+ * Go back to nap/sleep/winkle mode again if (b) is true.
+ */
+ rlwinm. r11,r12,47-31,30,31 /* Was it in power saving mode? */
+ beq 4f /* No, it wasn;t */
+ /* Thread was in power saving mode. Go back to nap again. */
+ cmpwi r11,2
+ blt 3f
+ /* Supervisor/Hypervisor state loss */
+ li r0,1
+ stb r0,PACA_NAPSTATELOST(r13)
+3: bl machine_check_queue_event
+ MACHINE_CHECK_HANDLER_WINDUP
+ GET_PACA(r13)
+ ld r1,PACAR1(r13)
+ /*
+ * Check what idle state this CPU was in and go back to same mode
+ * again.
+ */
+ lbz r3,PACA_THREAD_IDLE_STATE(r13)
+ cmpwi r3,PNV_THREAD_NAP
+ bgt 10f
+ IDLE_STATE_ENTER_SEQ(PPC_NAP)
+ /* No return */
+10:
+ cmpwi r3,PNV_THREAD_SLEEP
+ bgt 2f
+ IDLE_STATE_ENTER_SEQ(PPC_SLEEP)
+ /* No return */
-BEGIN_FTR_SECTION
- b denorm_done
-END_FTR_SECTION_IFCLR(CPU_FTR_ARCH_207S)
-/*
- * To denormalise we need to move a copy of the register to itself.
- * For POWER8 we need to do that for all 64 VSX registers
- */
- XVCPSGNDP32(32)
-denorm_done:
- mtspr SPRN_HSRR0,r11
- mtcrf 0x80,r9
- ld r9,PACA_EXGEN+EX_R9(r13)
- RESTORE_PPR_PACA(PACA_EXGEN, r10)
-BEGIN_FTR_SECTION
- ld r10,PACA_EXGEN+EX_CFAR(r13)
- mtspr SPRN_CFAR,r10
-END_FTR_SECTION_IFSET(CPU_FTR_CFAR)
- ld r10,PACA_EXGEN+EX_R10(r13)
- ld r11,PACA_EXGEN+EX_R11(r13)
- ld r12,PACA_EXGEN+EX_R12(r13)
- ld r13,PACA_EXGEN+EX_R13(r13)
- HRFID
- b .
+2:
+ /*
+ * Go back to winkle. Please note that this thread was woken up in
+ * machine check from winkle and have not restored the per-subcore
+ * state. Hence before going back to winkle, set last bit of HSPGR0
+ * to 1. This will make sure that if this thread gets woken up
+ * again at reset vector 0x100 then it will get chance to restore
+ * the subcore state.
+ */
+ ori r13,r13,1
+ SET_PACA(r13)
+ IDLE_STATE_ENTER_SEQ(PPC_WINKLE)
+ /* No return */
+4:
#endif
-
- .align 7
- /* moved from 0xe00 */
- STD_EXCEPTION_HV_OOL(0xe02, h_data_storage)
- KVM_HANDLER_SKIP(PACA_EXGEN, EXC_HV, 0xe02)
- STD_EXCEPTION_HV_OOL(0xe22, h_instr_storage)
- KVM_HANDLER(PACA_EXGEN, EXC_HV, 0xe22)
- STD_EXCEPTION_HV_OOL(0xe42, emulation_assist)
- KVM_HANDLER(PACA_EXGEN, EXC_HV, 0xe42)
- MASKABLE_EXCEPTION_HV_OOL(0xe62, hmi_exception)
- KVM_HANDLER(PACA_EXGEN, EXC_HV, 0xe62)
-
- MASKABLE_EXCEPTION_HV_OOL(0xe82, h_doorbell)
- KVM_HANDLER(PACA_EXGEN, EXC_HV, 0xe82)
-
- MASKABLE_EXCEPTION_HV_OOL(0xea2, h_virt_irq)
- KVM_HANDLER(PACA_EXGEN, EXC_HV, 0xea2)
-
- /* moved from 0xf00 */
- STD_EXCEPTION_PSERIES_OOL(0xf00, performance_monitor)
- KVM_HANDLER(PACA_EXGEN, EXC_STD, 0xf00)
- STD_EXCEPTION_PSERIES_OOL(0xf20, altivec_unavailable)
- KVM_HANDLER(PACA_EXGEN, EXC_STD, 0xf20)
- STD_EXCEPTION_PSERIES_OOL(0xf40, vsx_unavailable)
- KVM_HANDLER(PACA_EXGEN, EXC_STD, 0xf40)
- STD_EXCEPTION_PSERIES_OOL(0xf60, facility_unavailable)
- KVM_HANDLER(PACA_EXGEN, EXC_STD, 0xf60)
- STD_EXCEPTION_HV_OOL(0xf82, hv_facility_unavailable)
- KVM_HANDLER(PACA_EXGEN, EXC_HV, 0xf82)
-
-/*
- * An interrupt came in while soft-disabled. We set paca->irq_happened, then:
- * - If it was a decrementer interrupt, we bump the dec to max and and return.
- * - If it was a doorbell we return immediately since doorbells are edge
- * triggered and won't automatically refire.
- * - If it was a HMI we return immediately since we handled it in realmode
- * and it won't refire.
- * - else we hard disable and return.
- * This is called with r10 containing the value to OR to the paca field.
- */
-#define MASKED_INTERRUPT(_H) \
-masked_##_H##interrupt: \
- std r11,PACA_EXGEN+EX_R11(r13); \
- lbz r11,PACAIRQHAPPENED(r13); \
- or r11,r11,r10; \
- stb r11,PACAIRQHAPPENED(r13); \
- cmpwi r10,PACA_IRQ_DEC; \
- bne 1f; \
- lis r10,0x7fff; \
- ori r10,r10,0xffff; \
- mtspr SPRN_DEC,r10; \
- b 2f; \
-1: cmpwi r10,PACA_IRQ_DBELL; \
- beq 2f; \
- cmpwi r10,PACA_IRQ_HMI; \
- beq 2f; \
- mfspr r10,SPRN_##_H##SRR1; \
- rldicl r10,r10,48,1; /* clear MSR_EE */ \
- rotldi r10,r10,16; \
- mtspr SPRN_##_H##SRR1,r10; \
-2: mtcrf 0x80,r9; \
- ld r9,PACA_EXGEN+EX_R9(r13); \
- ld r10,PACA_EXGEN+EX_R10(r13); \
- ld r11,PACA_EXGEN+EX_R11(r13); \
- GET_SCRATCH0(r13); \
- ##_H##rfid; \
- b .
-
- MASKED_INTERRUPT()
- MASKED_INTERRUPT(H)
-
-/*
- * Called from arch_local_irq_enable when an interrupt needs
- * to be resent. r3 contains 0x500, 0x900, 0xa00 or 0xe80 to indicate
- * which kind of interrupt. MSR:EE is already off. We generate a
- * stackframe like if a real interrupt had happened.
- *
- * Note: While MSR:EE is off, we need to make sure that _MSR
- * in the generated frame has EE set to 1 or the exception
- * handler will not properly re-enable them.
- */
-_GLOBAL(__replay_interrupt)
- /* We are going to jump to the exception common code which
- * will retrieve various register values from the PACA which
- * we don't give a damn about, so we don't bother storing them.
+ /*
+ * Check if we are coming from hypervisor userspace. If yes then we
+ * continue in host kernel in V mode to deliver the MC event.
*/
- mfmsr r12
- mflr r11
- mfcr r9
- ori r12,r12,MSR_EE
- cmpwi r3,0x900
- beq decrementer_common
- cmpwi r3,0x500
- beq hardware_interrupt_common
-BEGIN_FTR_SECTION
- cmpwi r3,0xe80
- beq h_doorbell_common
- cmpwi r3,0xea0
- beq h_virt_irq_common
- cmpwi r3,0xe60
- beq hmi_exception_common
-FTR_SECTION_ELSE
- cmpwi r3,0xa00
- beq doorbell_super_common
-ALT_FTR_SECTION_END_IFSET(CPU_FTR_HVMODE)
- blr
-
-#ifdef CONFIG_PPC_PSERIES
-/*
- * Vectors for the FWNMI option. Share common code.
- */
- .globl system_reset_fwnmi
- .align 7
-system_reset_fwnmi:
- SET_SCRATCH0(r13) /* save r13 */
- EXCEPTION_PROLOG_PSERIES(PACA_EXGEN, system_reset_common, EXC_STD,
- NOTEST, 0x100)
-
-#endif /* CONFIG_PPC_PSERIES */
+ rldicl. r11,r12,4,63 /* See if MC hit while in HV mode. */
+ beq 5f
+ andi. r11,r12,MSR_PR /* See if coming from user. */
+ bne 9f /* continue in V mode if we are. */
+5:
#ifdef CONFIG_KVM_BOOK3S_64_HANDLER
-kvmppc_skip_interrupt:
/*
- * Here all GPRs are unchanged from when the interrupt happened
- * except for r13, which is saved in SPRG_SCRATCH0.
+ * We are coming from kernel context. Check if we are coming from
+ * guest. if yes, then we can continue. We will fall through
+ * do_kvm_200->kvmppc_interrupt to deliver the MC event to guest.
*/
- mfspr r13, SPRN_SRR0
- addi r13, r13, 4
- mtspr SPRN_SRR0, r13
- GET_SCRATCH0(r13)
+ lbz r11,HSTATE_IN_GUEST(r13)
+ cmpwi r11,0 /* Check if coming from guest */
+ bne 9f /* continue if we are. */
+#endif
+ /*
+ * At this point we are not sure about what context we come from.
+ * Queue up the MCE event and return from the interrupt.
+ * But before that, check if this is an un-recoverable exception.
+ * If yes, then stay on emergency stack and panic.
+ */
+ andi. r11,r12,MSR_RI
+ bne 2f
+1: mfspr r11,SPRN_SRR0
+ LOAD_HANDLER(r10,unrecover_mce)
+ mtspr SPRN_SRR0,r10
+ ld r10,PACAKMSR(r13)
+ /*
+ * We are going down. But there are chances that we might get hit by
+ * another MCE during panic path and we may run into unstable state
+ * with no way out. Hence, turn ME bit off while going down, so that
+ * when another MCE is hit during panic path, system will checkstop
+ * and hypervisor will get restarted cleanly by SP.
+ */
+ li r3,MSR_ME
+ andc r10,r10,r3 /* Turn off MSR_ME */
+ mtspr SPRN_SRR1,r10
rfid
b .
-
-kvmppc_skip_Hinterrupt:
+2:
/*
- * Here all GPRs are unchanged from when the interrupt happened
- * except for r13, which is saved in SPRG_SCRATCH0.
+ * Check if we have successfully handled/recovered from error, if not
+ * then stay on emergency stack and panic.
*/
- mfspr r13, SPRN_HSRR0
- addi r13, r13, 4
- mtspr SPRN_HSRR0, r13
- GET_SCRATCH0(r13)
- hrfid
- b .
-#endif
+ ld r3,RESULT(r1) /* Load result */
+ cmpdi r3,0 /* see if we handled MCE successfully */
-/*
- * Ensure that any handlers that get invoked from the exception prologs
- * above are below the first 64KB (0x10000) of the kernel image because
- * the prologs assemble the addresses of these handlers using the
- * LOAD_HANDLER macro, which uses an ori instruction.
- */
+ beq 1b /* if !handled then panic */
+ /*
+ * Return from MC interrupt.
+ * Queue up the MCE event so that we can log it later, while
+ * returning from kernel or opal call.
+ */
+ bl machine_check_queue_event
+ MACHINE_CHECK_HANDLER_WINDUP
+ rfid
+9:
+ /* Deliver the machine check to host kernel in V mode. */
+ MACHINE_CHECK_HANDLER_WINDUP
+ b machine_check_pSeries
-/*** Common interrupt handlers ***/
+EXC_COMMON_BEGIN(unrecover_mce)
+ /* Invoke machine_check_exception to print MCE event and panic. */
+ addi r3,r1,STACK_FRAME_OVERHEAD
+ bl machine_check_exception
+ /*
+ * We will not reach here. Even if we did, there is no way out. Call
+ * unrecoverable_exception and die.
+ */
+1: addi r3,r1,STACK_FRAME_OVERHEAD
+ bl unrecoverable_exception
+ b 1b
- STD_EXCEPTION_COMMON(0x100, system_reset, system_reset_exception)
- STD_EXCEPTION_COMMON_ASYNC(0x500, hardware_interrupt, do_IRQ)
- STD_EXCEPTION_COMMON_ASYNC(0x900, decrementer, timer_interrupt)
- STD_EXCEPTION_COMMON(0x980, hdecrementer, hdec_interrupt)
-#ifdef CONFIG_PPC_DOORBELL
- STD_EXCEPTION_COMMON_ASYNC(0xa00, doorbell_super, doorbell_exception)
-#else
- STD_EXCEPTION_COMMON_ASYNC(0xa00, doorbell_super, unknown_exception)
-#endif
- STD_EXCEPTION_COMMON(0xb00, trap_0b, unknown_exception)
- STD_EXCEPTION_COMMON(0xd00, single_step, single_step_exception)
- STD_EXCEPTION_COMMON(0xe00, trap_0e, unknown_exception)
- STD_EXCEPTION_COMMON(0xe40, emulation_assist, emulation_assist_interrupt)
- STD_EXCEPTION_COMMON_ASYNC(0xe60, hmi_exception, handle_hmi_exception)
-#ifdef CONFIG_PPC_DOORBELL
- STD_EXCEPTION_COMMON_ASYNC(0xe80, h_doorbell, doorbell_exception)
-#else
- STD_EXCEPTION_COMMON_ASYNC(0xe80, h_doorbell, unknown_exception)
-#endif
- STD_EXCEPTION_COMMON_ASYNC(0xea0, h_virt_irq, do_IRQ)
- STD_EXCEPTION_COMMON_ASYNC(0xf00, performance_monitor, performance_monitor_exception)
- STD_EXCEPTION_COMMON(0x1300, instruction_breakpoint, instruction_breakpoint_exception)
- STD_EXCEPTION_COMMON(0x1502, denorm, unknown_exception)
-#ifdef CONFIG_ALTIVEC
- STD_EXCEPTION_COMMON(0x1700, altivec_assist, altivec_assist_exception)
-#else
- STD_EXCEPTION_COMMON(0x1700, altivec_assist, unknown_exception)
-#endif
+EXC_REAL(data_access, 0x300, 0x380)
+EXC_VIRT(data_access, 0x4300, 0x4380, 0x300)
+TRAMP_KVM_SKIP(PACA_EXGEN, 0x300)
+EXC_COMMON_BEGIN(data_access_common)
/*
- * Relocation-on interrupts: A subset of the interrupts can be delivered
- * with IR=1/DR=1, if AIL==2 and MSR.HV won't be changed by delivering
- * it. Addresses are the same as the original interrupt addresses, but
- * offset by 0xc000000000004000.
- * It's impossible to receive interrupts below 0x300 via this mechanism.
- * KVM: None of these traps are from the guest ; anything that escalated
- * to HV=1 from HV=0 is delivered via real mode handlers.
+ * Here r13 points to the paca, r9 contains the saved CR,
+ * SRR0 and SRR1 are saved in r11 and r12,
+ * r9 - r13 are saved in paca->exgen.
*/
+ mfspr r10,SPRN_DAR
+ std r10,PACA_EXGEN+EX_DAR(r13)
+ mfspr r10,SPRN_DSISR
+ stw r10,PACA_EXGEN+EX_DSISR(r13)
+ EXCEPTION_PROLOG_COMMON(0x300, PACA_EXGEN)
+ RECONCILE_IRQ_STATE(r10, r11)
+ ld r12,_MSR(r1)
+ ld r3,PACA_EXGEN+EX_DAR(r13)
+ lwz r4,PACA_EXGEN+EX_DSISR(r13)
+ li r5,0x300
+ std r3,_DAR(r1)
+ std r4,_DSISR(r1)
+BEGIN_MMU_FTR_SECTION
+ b do_hash_page /* Try to handle as hpte fault */
+MMU_FTR_SECTION_ELSE
+ b handle_page_fault
+ALT_MMU_FTR_SECTION_END_IFCLR(MMU_FTR_TYPE_RADIX)
+
+EXC_REAL_BEGIN(data_access_slb, 0x380, 0x400)
+ SET_SCRATCH0(r13)
+ EXCEPTION_PROLOG_0(PACA_EXSLB)
+ EXCEPTION_PROLOG_1(PACA_EXSLB, KVMTEST_PR, 0x380)
+ std r3,PACA_EXSLB+EX_R3(r13)
+ mfspr r3,SPRN_DAR
+ mfspr r12,SPRN_SRR1
+ crset 4*cr6+eq
+#ifndef CONFIG_RELOCATABLE
+ b slb_miss_realmode
+#else
/*
- * This uses the standard macro, since the original 0x300 vector
- * only has extra guff for STAB-based processors -- which never
- * come here.
+ * We can't just use a direct branch to slb_miss_realmode
+ * because the distance from here to there depends on where
+ * the kernel ends up being put.
*/
- STD_RELON_EXCEPTION_PSERIES(0x4300, 0x300, data_access)
- . = 0x4380
- .globl data_access_slb_relon_pSeries
-data_access_slb_relon_pSeries:
+ mfctr r11
+ LOAD_HANDLER(r10, slb_miss_realmode)
+ mtctr r10
+ bctr
+#endif
+EXC_REAL_END(data_access_slb, 0x380, 0x400)
+
+EXC_VIRT_BEGIN(data_access_slb, 0x4380, 0x4400)
SET_SCRATCH0(r13)
EXCEPTION_PROLOG_0(PACA_EXSLB)
EXCEPTION_PROLOG_1(PACA_EXSLB, NOTEST, 0x380)
mtctr r10
bctr
#endif
+EXC_VIRT_END(data_access_slb, 0x4380, 0x4400)
+TRAMP_KVM_SKIP(PACA_EXSLB, 0x380)
+
+
+EXC_REAL(instruction_access, 0x400, 0x480)
+EXC_VIRT(instruction_access, 0x4400, 0x4480, 0x400)
+TRAMP_KVM(PACA_EXGEN, 0x400)
+
+EXC_COMMON_BEGIN(instruction_access_common)
+ EXCEPTION_PROLOG_COMMON(0x400, PACA_EXGEN)
+ RECONCILE_IRQ_STATE(r10, r11)
+ ld r12,_MSR(r1)
+ ld r3,_NIP(r1)
+ andis. r4,r12,0x5820
+ li r5,0x400
+ std r3,_DAR(r1)
+ std r4,_DSISR(r1)
+BEGIN_MMU_FTR_SECTION
+ b do_hash_page /* Try to handle as hpte fault */
+MMU_FTR_SECTION_ELSE
+ b handle_page_fault
+ALT_MMU_FTR_SECTION_END_IFCLR(MMU_FTR_TYPE_RADIX)
+
- STD_RELON_EXCEPTION_PSERIES(0x4400, 0x400, instruction_access)
- . = 0x4480
- .globl instruction_access_slb_relon_pSeries
-instruction_access_slb_relon_pSeries:
+EXC_REAL_BEGIN(instruction_access_slb, 0x480, 0x500)
SET_SCRATCH0(r13)
EXCEPTION_PROLOG_0(PACA_EXSLB)
- EXCEPTION_PROLOG_1(PACA_EXSLB, NOTEST, 0x480)
+ EXCEPTION_PROLOG_1(PACA_EXSLB, KVMTEST_PR, 0x480)
std r3,PACA_EXSLB+EX_R3(r13)
mfspr r3,SPRN_SRR0 /* SRR0 is faulting address */
mfspr r12,SPRN_SRR1
mtctr r10
bctr
#endif
+EXC_REAL_END(instruction_access_slb, 0x480, 0x500)
- . = 0x4500
- .globl hardware_interrupt_relon_pSeries;
- .globl hardware_interrupt_relon_hv;
-hardware_interrupt_relon_pSeries:
-hardware_interrupt_relon_hv:
- BEGIN_FTR_SECTION
- _MASKABLE_RELON_EXCEPTION_PSERIES(0x502, hardware_interrupt, EXC_HV, SOFTEN_TEST_HV)
- FTR_SECTION_ELSE
- _MASKABLE_RELON_EXCEPTION_PSERIES(0x500, hardware_interrupt, EXC_STD, SOFTEN_TEST_PR)
- ALT_FTR_SECTION_END_IFSET(CPU_FTR_HVMODE)
- STD_RELON_EXCEPTION_PSERIES(0x4600, 0x600, alignment)
- STD_RELON_EXCEPTION_PSERIES(0x4700, 0x700, program_check)
- STD_RELON_EXCEPTION_PSERIES(0x4800, 0x800, fp_unavailable)
- MASKABLE_RELON_EXCEPTION_PSERIES(0x4900, 0x900, decrementer)
- STD_RELON_EXCEPTION_HV(0x4980, 0x982, hdecrementer)
- MASKABLE_RELON_EXCEPTION_PSERIES(0x4a00, 0xa00, doorbell_super)
- STD_RELON_EXCEPTION_PSERIES(0x4b00, 0xb00, trap_0b)
-
- . = 0x4c00
- .globl system_call_relon_pSeries
-system_call_relon_pSeries:
- HMT_MEDIUM
- SYSCALL_PSERIES_1
- SYSCALL_PSERIES_2_DIRECT
- SYSCALL_PSERIES_3
-
- STD_RELON_EXCEPTION_PSERIES(0x4d00, 0xd00, single_step)
-
- . = 0x4e00
- b . /* Can't happen, see v2.07 Book III-S section 6.5 */
-
- . = 0x4e20
- b . /* Can't happen, see v2.07 Book III-S section 6.5 */
-
- . = 0x4e40
-emulation_assist_relon_trampoline:
+EXC_VIRT_BEGIN(instruction_access_slb, 0x4480, 0x4500)
SET_SCRATCH0(r13)
- EXCEPTION_PROLOG_0(PACA_EXGEN)
- b emulation_assist_relon_hv
+ EXCEPTION_PROLOG_0(PACA_EXSLB)
+ EXCEPTION_PROLOG_1(PACA_EXSLB, NOTEST, 0x480)
+ std r3,PACA_EXSLB+EX_R3(r13)
+ mfspr r3,SPRN_SRR0 /* SRR0 is faulting address */
+ mfspr r12,SPRN_SRR1
+ crclr 4*cr6+eq
+#ifndef CONFIG_RELOCATABLE
+ b slb_miss_realmode
+#else
+ mfctr r11
+ LOAD_HANDLER(r10, slb_miss_realmode)
+ mtctr r10
+ bctr
+#endif
+EXC_VIRT_END(instruction_access_slb, 0x4480, 0x4500)
+TRAMP_KVM(PACA_EXSLB, 0x480)
- . = 0x4e60
- b . /* Can't happen, see v2.07 Book III-S section 6.5 */
- . = 0x4e80
-h_doorbell_relon_trampoline:
- SET_SCRATCH0(r13)
- EXCEPTION_PROLOG_0(PACA_EXGEN)
- b h_doorbell_relon_hv
+/* This handler is used by both 0x380 and 0x480 slb miss interrupts */
+EXC_COMMON_BEGIN(slb_miss_realmode)
+ /*
+ * r13 points to the PACA, r9 contains the saved CR,
+ * r12 contain the saved SRR1, SRR0 is still ready for return
+ * r3 has the faulting address
+ * r9 - r13 are saved in paca->exslb.
+ * r3 is saved in paca->slb_r3
+ * cr6.eq is set for a D-SLB miss, clear for a I-SLB miss
+ * We assume we aren't going to take any exceptions during this
+ * procedure.
+ */
+ mflr r10
+#ifdef CONFIG_RELOCATABLE
+ mtctr r11
+#endif
- . = 0x4ea0
-h_virt_irq_relon_trampoline:
- SET_SCRATCH0(r13)
- EXCEPTION_PROLOG_0(PACA_EXGEN)
- b h_virt_irq_relon_hv
+ stw r9,PACA_EXSLB+EX_CCR(r13) /* save CR in exc. frame */
+ std r10,PACA_EXSLB+EX_LR(r13) /* save LR */
+ std r3,PACA_EXSLB+EX_DAR(r13)
- . = 0x4f00
-performance_monitor_relon_pseries_trampoline:
- SET_SCRATCH0(r13)
- EXCEPTION_PROLOG_0(PACA_EXGEN)
- b performance_monitor_relon_pSeries
+ crset 4*cr0+eq
+#ifdef CONFIG_PPC_STD_MMU_64
+BEGIN_MMU_FTR_SECTION
+ bl slb_allocate_realmode
+END_MMU_FTR_SECTION_IFCLR(MMU_FTR_TYPE_RADIX)
+#endif
- . = 0x4f20
-altivec_unavailable_relon_pseries_trampoline:
- SET_SCRATCH0(r13)
- EXCEPTION_PROLOG_0(PACA_EXGEN)
- b altivec_unavailable_relon_pSeries
+ ld r10,PACA_EXSLB+EX_LR(r13)
+ ld r3,PACA_EXSLB+EX_R3(r13)
+ lwz r9,PACA_EXSLB+EX_CCR(r13) /* get saved CR */
+ mtlr r10
- . = 0x4f40
-vsx_unavailable_relon_pseries_trampoline:
- SET_SCRATCH0(r13)
- EXCEPTION_PROLOG_0(PACA_EXGEN)
- b vsx_unavailable_relon_pSeries
+ beq 8f /* if bad address, make full stack frame */
- . = 0x4f60
-facility_unavailable_relon_trampoline:
- SET_SCRATCH0(r13)
- EXCEPTION_PROLOG_0(PACA_EXGEN)
- b facility_unavailable_relon_pSeries
+ andi. r10,r12,MSR_RI /* check for unrecoverable exception */
+ beq- 2f
- . = 0x4f80
-hv_facility_unavailable_relon_trampoline:
- SET_SCRATCH0(r13)
- EXCEPTION_PROLOG_0(PACA_EXGEN)
- b hv_facility_unavailable_relon_hv
+ /* All done -- return from exception. */
- STD_RELON_EXCEPTION_PSERIES(0x5300, 0x1300, instruction_breakpoint)
-#ifdef CONFIG_PPC_DENORMALISATION
- . = 0x5500
- b denorm_exception_hv
-#endif
- STD_RELON_EXCEPTION_PSERIES(0x5700, 0x1700, altivec_assist)
+.machine push
+.machine "power4"
+ mtcrf 0x80,r9
+ mtcrf 0x02,r9 /* I/D indication is in cr6 */
+ mtcrf 0x01,r9 /* slb_allocate uses cr0 and cr7 */
+.machine pop
-ppc64_runlatch_on_trampoline:
- b __ppc64_runlatch_on
+ RESTORE_PPR_PACA(PACA_EXSLB, r9)
+ ld r9,PACA_EXSLB+EX_R9(r13)
+ ld r10,PACA_EXSLB+EX_R10(r13)
+ ld r11,PACA_EXSLB+EX_R11(r13)
+ ld r12,PACA_EXSLB+EX_R12(r13)
+ ld r13,PACA_EXSLB+EX_R13(r13)
+ rfid
+ b . /* prevent speculative execution */
-/*
- * Here r13 points to the paca, r9 contains the saved CR,
- * SRR0 and SRR1 are saved in r11 and r12,
- * r9 - r13 are saved in paca->exgen.
- */
- .align 7
- .globl data_access_common
-data_access_common:
- mfspr r10,SPRN_DAR
- std r10,PACA_EXGEN+EX_DAR(r13)
- mfspr r10,SPRN_DSISR
- stw r10,PACA_EXGEN+EX_DSISR(r13)
- EXCEPTION_PROLOG_COMMON(0x300, PACA_EXGEN)
- RECONCILE_IRQ_STATE(r10, r11)
- ld r12,_MSR(r1)
- ld r3,PACA_EXGEN+EX_DAR(r13)
- lwz r4,PACA_EXGEN+EX_DSISR(r13)
- li r5,0x300
- std r3,_DAR(r1)
- std r4,_DSISR(r1)
-BEGIN_MMU_FTR_SECTION
- b do_hash_page /* Try to handle as hpte fault */
-MMU_FTR_SECTION_ELSE
- b handle_page_fault
-ALT_MMU_FTR_SECTION_END_IFCLR(MMU_FTR_TYPE_RADIX)
+2: mfspr r11,SPRN_SRR0
+ LOAD_HANDLER(r10,unrecov_slb)
+ mtspr SPRN_SRR0,r10
+ ld r10,PACAKMSR(r13)
+ mtspr SPRN_SRR1,r10
+ rfid
+ b .
- .align 7
- .globl h_data_storage_common
-h_data_storage_common:
- mfspr r10,SPRN_HDAR
- std r10,PACA_EXGEN+EX_DAR(r13)
- mfspr r10,SPRN_HDSISR
- stw r10,PACA_EXGEN+EX_DSISR(r13)
- EXCEPTION_PROLOG_COMMON(0xe00, PACA_EXGEN)
- bl save_nvgprs
+8: mfspr r11,SPRN_SRR0
+ LOAD_HANDLER(r10,bad_addr_slb)
+ mtspr SPRN_SRR0,r10
+ ld r10,PACAKMSR(r13)
+ mtspr SPRN_SRR1,r10
+ rfid
+ b .
+
+EXC_COMMON_BEGIN(unrecov_slb)
+ EXCEPTION_PROLOG_COMMON(0x4100, PACA_EXSLB)
RECONCILE_IRQ_STATE(r10, r11)
- addi r3,r1,STACK_FRAME_OVERHEAD
- bl unknown_exception
- b ret_from_except
+ bl save_nvgprs
+1: addi r3,r1,STACK_FRAME_OVERHEAD
+ bl unrecoverable_exception
+ b 1b
- .align 7
- .globl instruction_access_common
-instruction_access_common:
- EXCEPTION_PROLOG_COMMON(0x400, PACA_EXGEN)
+EXC_COMMON_BEGIN(bad_addr_slb)
+ EXCEPTION_PROLOG_COMMON(0x380, PACA_EXSLB)
RECONCILE_IRQ_STATE(r10, r11)
- ld r12,_MSR(r1)
- ld r3,_NIP(r1)
- andis. r4,r12,0x5820
- li r5,0x400
- std r3,_DAR(r1)
- std r4,_DSISR(r1)
-BEGIN_MMU_FTR_SECTION
- b do_hash_page /* Try to handle as hpte fault */
-MMU_FTR_SECTION_ELSE
- b handle_page_fault
-ALT_MMU_FTR_SECTION_END_IFCLR(MMU_FTR_TYPE_RADIX)
+ ld r3, PACA_EXSLB+EX_DAR(r13)
+ std r3, _DAR(r1)
+ beq cr6, 2f
+ li r10, 0x480 /* fix trap number for I-SLB miss */
+ std r10, _TRAP(r1)
+2: bl save_nvgprs
+ addi r3, r1, STACK_FRAME_OVERHEAD
+ bl slb_miss_bad_addr
+ b ret_from_except
- STD_EXCEPTION_COMMON(0xe20, h_instr_storage, unknown_exception)
+EXC_REAL_BEGIN(hardware_interrupt, 0x500, 0x600)
+ .globl hardware_interrupt_hv;
+hardware_interrupt_hv:
+ BEGIN_FTR_SECTION
+ _MASKABLE_EXCEPTION_PSERIES(0x500, hardware_interrupt_common,
+ EXC_HV, SOFTEN_TEST_HV)
+do_kvm_H0x500:
+ KVM_HANDLER(PACA_EXGEN, EXC_HV, 0x502)
+ FTR_SECTION_ELSE
+ _MASKABLE_EXCEPTION_PSERIES(0x500, hardware_interrupt_common,
+ EXC_STD, SOFTEN_TEST_PR)
+do_kvm_0x500:
+ KVM_HANDLER(PACA_EXGEN, EXC_STD, 0x500)
+ ALT_FTR_SECTION_END_IFSET(CPU_FTR_HVMODE | CPU_FTR_ARCH_206)
+EXC_REAL_END(hardware_interrupt, 0x500, 0x600)
- /*
- * Machine check is different because we use a different
- * save area: PACA_EXMC instead of PACA_EXGEN.
- */
- .align 7
- .globl machine_check_common
-machine_check_common:
+EXC_VIRT_BEGIN(hardware_interrupt, 0x4500, 0x4600)
+ .globl hardware_interrupt_relon_hv;
+hardware_interrupt_relon_hv:
+ BEGIN_FTR_SECTION
+ _MASKABLE_RELON_EXCEPTION_PSERIES(0x500, hardware_interrupt_common, EXC_HV, SOFTEN_TEST_HV)
+ FTR_SECTION_ELSE
+ _MASKABLE_RELON_EXCEPTION_PSERIES(0x500, hardware_interrupt_common, EXC_STD, SOFTEN_TEST_PR)
+ ALT_FTR_SECTION_END_IFSET(CPU_FTR_HVMODE)
+EXC_VIRT_END(hardware_interrupt, 0x4500, 0x4600)
- mfspr r10,SPRN_DAR
- std r10,PACA_EXMC+EX_DAR(r13)
- mfspr r10,SPRN_DSISR
- stw r10,PACA_EXMC+EX_DSISR(r13)
- EXCEPTION_PROLOG_COMMON(0x200, PACA_EXMC)
- FINISH_NAP
- RECONCILE_IRQ_STATE(r10, r11)
- ld r3,PACA_EXMC+EX_DAR(r13)
- lwz r4,PACA_EXMC+EX_DSISR(r13)
- /* Enable MSR_RI when finished with PACA_EXMC */
- li r10,MSR_RI
- mtmsrd r10,1
- std r3,_DAR(r1)
- std r4,_DSISR(r1)
- bl save_nvgprs
- addi r3,r1,STACK_FRAME_OVERHEAD
- bl machine_check_exception
- b ret_from_except
+EXC_COMMON_ASYNC(hardware_interrupt_common, 0x500, do_IRQ)
- .align 7
- .globl alignment_common
-alignment_common:
+
+EXC_REAL(alignment, 0x600, 0x700)
+EXC_VIRT(alignment, 0x4600, 0x4700, 0x600)
+TRAMP_KVM(PACA_EXGEN, 0x600)
+EXC_COMMON_BEGIN(alignment_common)
mfspr r10,SPRN_DAR
std r10,PACA_EXGEN+EX_DAR(r13)
mfspr r10,SPRN_DSISR
bl alignment_exception
b ret_from_except
- .align 7
- .globl program_check_common
-program_check_common:
+
+EXC_REAL(program_check, 0x700, 0x800)
+EXC_VIRT(program_check, 0x4700, 0x4800, 0x700)
+TRAMP_KVM(PACA_EXGEN, 0x700)
+EXC_COMMON_BEGIN(program_check_common)
EXCEPTION_PROLOG_COMMON(0x700, PACA_EXGEN)
bl save_nvgprs
RECONCILE_IRQ_STATE(r10, r11)
bl program_check_exception
b ret_from_except
- .align 7
- .globl fp_unavailable_common
-fp_unavailable_common:
+
+EXC_REAL(fp_unavailable, 0x800, 0x900)
+EXC_VIRT(fp_unavailable, 0x4800, 0x4900, 0x800)
+TRAMP_KVM(PACA_EXGEN, 0x800)
+EXC_COMMON_BEGIN(fp_unavailable_common)
EXCEPTION_PROLOG_COMMON(0x800, PACA_EXGEN)
bne 1f /* if from user, just load it up */
bl save_nvgprs
bl fp_unavailable_tm
b ret_from_except
#endif
- .align 7
- .globl altivec_unavailable_common
-altivec_unavailable_common:
+
+
+EXC_REAL_MASKABLE(decrementer, 0x900, 0x980)
+EXC_VIRT_MASKABLE(decrementer, 0x4900, 0x4980, 0x900)
+TRAMP_KVM(PACA_EXGEN, 0x900)
+EXC_COMMON_ASYNC(decrementer_common, 0x900, timer_interrupt)
+
+
+EXC_REAL_HV(hdecrementer, 0x980, 0xa00)
+EXC_VIRT_HV(hdecrementer, 0x4980, 0x4a00, 0x980)
+TRAMP_KVM_HV(PACA_EXGEN, 0x980)
+EXC_COMMON(hdecrementer_common, 0x980, hdec_interrupt)
+
+
+EXC_REAL_MASKABLE(doorbell_super, 0xa00, 0xb00)
+EXC_VIRT_MASKABLE(doorbell_super, 0x4a00, 0x4b00, 0xa00)
+TRAMP_KVM(PACA_EXGEN, 0xa00)
+#ifdef CONFIG_PPC_DOORBELL
+EXC_COMMON_ASYNC(doorbell_super_common, 0xa00, doorbell_exception)
+#else
+EXC_COMMON_ASYNC(doorbell_super_common, 0xa00, unknown_exception)
+#endif
+
+
+EXC_REAL(trap_0b, 0xb00, 0xc00)
+EXC_VIRT(trap_0b, 0x4b00, 0x4c00, 0xb00)
+TRAMP_KVM(PACA_EXGEN, 0xb00)
+EXC_COMMON(trap_0b_common, 0xb00, unknown_exception)
+
+
+#define LOAD_SYSCALL_HANDLER(reg) \
+ ld reg,PACAKBASE(r13); \
+ ori reg,reg,(ABS_ADDR(system_call_common))@l;
+
+/* Syscall routine is used twice, in reloc-off and reloc-on paths */
+#define SYSCALL_PSERIES_1 \
+BEGIN_FTR_SECTION \
+ cmpdi r0,0x1ebe ; \
+ beq- 1f ; \
+END_FTR_SECTION_IFSET(CPU_FTR_REAL_LE) \
+ mr r9,r13 ; \
+ GET_PACA(r13) ; \
+ mfspr r11,SPRN_SRR0 ; \
+0:
+
+#define SYSCALL_PSERIES_2_RFID \
+ mfspr r12,SPRN_SRR1 ; \
+ LOAD_SYSCALL_HANDLER(r10) ; \
+ mtspr SPRN_SRR0,r10 ; \
+ ld r10,PACAKMSR(r13) ; \
+ mtspr SPRN_SRR1,r10 ; \
+ rfid ; \
+ b . ; /* prevent speculative execution */
+
+#define SYSCALL_PSERIES_3 \
+ /* Fast LE/BE switch system call */ \
+1: mfspr r12,SPRN_SRR1 ; \
+ xori r12,r12,MSR_LE ; \
+ mtspr SPRN_SRR1,r12 ; \
+ rfid ; /* return to userspace */ \
+ b . ; /* prevent speculative execution */
+
+#if defined(CONFIG_RELOCATABLE)
+ /*
+ * We can't branch directly so we do it via the CTR which
+ * is volatile across system calls.
+ */
+#define SYSCALL_PSERIES_2_DIRECT \
+ LOAD_SYSCALL_HANDLER(r12) ; \
+ mtctr r12 ; \
+ mfspr r12,SPRN_SRR1 ; \
+ li r10,MSR_RI ; \
+ mtmsrd r10,1 ; \
+ bctr ;
+#else
+ /* We can branch directly */
+#define SYSCALL_PSERIES_2_DIRECT \
+ mfspr r12,SPRN_SRR1 ; \
+ li r10,MSR_RI ; \
+ mtmsrd r10,1 ; /* Set RI (EE=0) */ \
+ b system_call_common ;
+#endif
+
+EXC_REAL_BEGIN(system_call, 0xc00, 0xd00)
+ /*
+ * If CONFIG_KVM_BOOK3S_64_HANDLER is set, save the PPR (on systems
+ * that support it) before changing to HMT_MEDIUM. That allows the KVM
+ * code to save that value into the guest state (it is the guest's PPR
+ * value). Otherwise just change to HMT_MEDIUM as userspace has
+ * already saved the PPR.
+ */
+#ifdef CONFIG_KVM_BOOK3S_64_HANDLER
+ SET_SCRATCH0(r13)
+ GET_PACA(r13)
+ std r9,PACA_EXGEN+EX_R9(r13)
+ OPT_GET_SPR(r9, SPRN_PPR, CPU_FTR_HAS_PPR);
+ HMT_MEDIUM;
+ std r10,PACA_EXGEN+EX_R10(r13)
+ OPT_SAVE_REG_TO_PACA(PACA_EXGEN+EX_PPR, r9, CPU_FTR_HAS_PPR);
+ mfcr r9
+ KVMTEST_PR(0xc00)
+ GET_SCRATCH0(r13)
+#else
+ HMT_MEDIUM;
+#endif
+ SYSCALL_PSERIES_1
+ SYSCALL_PSERIES_2_RFID
+ SYSCALL_PSERIES_3
+EXC_REAL_END(system_call, 0xc00, 0xd00)
+
+EXC_VIRT_BEGIN(system_call, 0x4c00, 0x4d00)
+ HMT_MEDIUM
+ SYSCALL_PSERIES_1
+ SYSCALL_PSERIES_2_DIRECT
+ SYSCALL_PSERIES_3
+EXC_VIRT_END(system_call, 0x4c00, 0x4d00)
+
+TRAMP_KVM(PACA_EXGEN, 0xc00)
+
+
+EXC_REAL(single_step, 0xd00, 0xe00)
+EXC_VIRT(single_step, 0x4d00, 0x4e00, 0xd00)
+TRAMP_KVM(PACA_EXGEN, 0xd00)
+EXC_COMMON(single_step_common, 0xd00, single_step_exception)
+
+EXC_REAL_OOL_HV(h_data_storage, 0xe00, 0xe20)
+EXC_VIRT_NONE(0x4e00, 0x4e20)
+TRAMP_KVM_HV_SKIP(PACA_EXGEN, 0xe00)
+EXC_COMMON_BEGIN(h_data_storage_common)
+ mfspr r10,SPRN_HDAR
+ std r10,PACA_EXGEN+EX_DAR(r13)
+ mfspr r10,SPRN_HDSISR
+ stw r10,PACA_EXGEN+EX_DSISR(r13)
+ EXCEPTION_PROLOG_COMMON(0xe00, PACA_EXGEN)
+ bl save_nvgprs
+ RECONCILE_IRQ_STATE(r10, r11)
+ addi r3,r1,STACK_FRAME_OVERHEAD
+ bl unknown_exception
+ b ret_from_except
+
+
+EXC_REAL_OOL_HV(h_instr_storage, 0xe20, 0xe40)
+EXC_VIRT_NONE(0x4e20, 0x4e40)
+TRAMP_KVM_HV(PACA_EXGEN, 0xe20)
+EXC_COMMON(h_instr_storage_common, 0xe20, unknown_exception)
+
+
+EXC_REAL_OOL_HV(emulation_assist, 0xe40, 0xe60)
+EXC_VIRT_OOL_HV(emulation_assist, 0x4e40, 0x4e60, 0xe40)
+TRAMP_KVM_HV(PACA_EXGEN, 0xe40)
+EXC_COMMON(emulation_assist_common, 0xe40, emulation_assist_interrupt)
+
+
+/*
+ * hmi_exception trampoline is a special case. It jumps to hmi_exception_early
+ * first, and then eventaully from there to the trampoline to get into virtual
+ * mode.
+ */
+__EXC_REAL_OOL_HV_DIRECT(hmi_exception, 0xe60, 0xe80, hmi_exception_early)
+__TRAMP_REAL_REAL_OOL_MASKABLE_HV(hmi_exception, 0xe60)
+EXC_VIRT_NONE(0x4e60, 0x4e80)
+TRAMP_KVM_HV(PACA_EXGEN, 0xe60)
+TRAMP_REAL_BEGIN(hmi_exception_early)
+ EXCEPTION_PROLOG_1(PACA_EXGEN, KVMTEST_HV, 0xe60)
+ mr r10,r1 /* Save r1 */
+ ld r1,PACAEMERGSP(r13) /* Use emergency stack */
+ subi r1,r1,INT_FRAME_SIZE /* alloc stack frame */
+ std r9,_CCR(r1) /* save CR in stackframe */
+ mfspr r11,SPRN_HSRR0 /* Save HSRR0 */
+ std r11,_NIP(r1) /* save HSRR0 in stackframe */
+ mfspr r12,SPRN_HSRR1 /* Save SRR1 */
+ std r12,_MSR(r1) /* save SRR1 in stackframe */
+ std r10,0(r1) /* make stack chain pointer */
+ std r0,GPR0(r1) /* save r0 in stackframe */
+ std r10,GPR1(r1) /* save r1 in stackframe */
+ EXCEPTION_PROLOG_COMMON_2(PACA_EXGEN)
+ EXCEPTION_PROLOG_COMMON_3(0xe60)
+ addi r3,r1,STACK_FRAME_OVERHEAD
+ bl hmi_exception_realmode
+ /* Windup the stack. */
+ /* Move original HSRR0 and HSRR1 into the respective regs */
+ ld r9,_MSR(r1)
+ mtspr SPRN_HSRR1,r9
+ ld r3,_NIP(r1)
+ mtspr SPRN_HSRR0,r3
+ ld r9,_CTR(r1)
+ mtctr r9
+ ld r9,_XER(r1)
+ mtxer r9
+ ld r9,_LINK(r1)
+ mtlr r9
+ REST_GPR(0, r1)
+ REST_8GPRS(2, r1)
+ REST_GPR(10, r1)
+ ld r11,_CCR(r1)
+ mtcr r11
+ REST_GPR(11, r1)
+ REST_2GPRS(12, r1)
+ /* restore original r1. */
+ ld r1,GPR1(r1)
+
+ /*
+ * Go to virtual mode and pull the HMI event information from
+ * firmware.
+ */
+ .globl hmi_exception_after_realmode
+hmi_exception_after_realmode:
+ SET_SCRATCH0(r13)
+ EXCEPTION_PROLOG_0(PACA_EXGEN)
+ b tramp_real_hmi_exception
+
+EXC_COMMON_ASYNC(hmi_exception_common, 0xe60, handle_hmi_exception)
+
+
+EXC_REAL_OOL_MASKABLE_HV(h_doorbell, 0xe80, 0xea0)
+EXC_VIRT_OOL_MASKABLE_HV(h_doorbell, 0x4e80, 0x4ea0, 0xe80)
+TRAMP_KVM_HV(PACA_EXGEN, 0xe80)
+#ifdef CONFIG_PPC_DOORBELL
+EXC_COMMON_ASYNC(h_doorbell_common, 0xe80, doorbell_exception)
+#else
+EXC_COMMON_ASYNC(h_doorbell_common, 0xe80, unknown_exception)
+#endif
+
+
+EXC_REAL_OOL_MASKABLE_HV(h_virt_irq, 0xea0, 0xec0)
+EXC_VIRT_OOL_MASKABLE_HV(h_virt_irq, 0x4ea0, 0x4ec0, 0xea0)
+TRAMP_KVM_HV(PACA_EXGEN, 0xea0)
+EXC_COMMON_ASYNC(h_virt_irq_common, 0xea0, do_IRQ)
+
+
+EXC_REAL_NONE(0xec0, 0xf00)
+EXC_VIRT_NONE(0x4ec0, 0x4f00)
+
+
+EXC_REAL_OOL(performance_monitor, 0xf00, 0xf20)
+EXC_VIRT_OOL(performance_monitor, 0x4f00, 0x4f20, 0xf00)
+TRAMP_KVM(PACA_EXGEN, 0xf00)
+EXC_COMMON_ASYNC(performance_monitor_common, 0xf00, performance_monitor_exception)
+
+
+EXC_REAL_OOL(altivec_unavailable, 0xf20, 0xf40)
+EXC_VIRT_OOL(altivec_unavailable, 0x4f20, 0x4f40, 0xf20)
+TRAMP_KVM(PACA_EXGEN, 0xf20)
+EXC_COMMON_BEGIN(altivec_unavailable_common)
EXCEPTION_PROLOG_COMMON(0xf20, PACA_EXGEN)
#ifdef CONFIG_ALTIVEC
BEGIN_FTR_SECTION
bl altivec_unavailable_exception
b ret_from_except
- .align 7
- .globl vsx_unavailable_common
-vsx_unavailable_common:
+
+EXC_REAL_OOL(vsx_unavailable, 0xf40, 0xf60)
+EXC_VIRT_OOL(vsx_unavailable, 0x4f40, 0x4f60, 0xf40)
+TRAMP_KVM(PACA_EXGEN, 0xf40)
+EXC_COMMON_BEGIN(vsx_unavailable_common)
EXCEPTION_PROLOG_COMMON(0xf40, PACA_EXGEN)
#ifdef CONFIG_VSX
BEGIN_FTR_SECTION
bl vsx_unavailable_exception
b ret_from_except
- /* Equivalents to the above handlers for relocation-on interrupt vectors */
- STD_RELON_EXCEPTION_HV_OOL(0xe40, emulation_assist)
- MASKABLE_RELON_EXCEPTION_HV_OOL(0xe80, h_doorbell)
- MASKABLE_RELON_EXCEPTION_HV_OOL(0xea0, h_virt_irq)
- STD_RELON_EXCEPTION_PSERIES_OOL(0xf00, performance_monitor)
- STD_RELON_EXCEPTION_PSERIES_OOL(0xf20, altivec_unavailable)
- STD_RELON_EXCEPTION_PSERIES_OOL(0xf40, vsx_unavailable)
- STD_RELON_EXCEPTION_PSERIES_OOL(0xf60, facility_unavailable)
- STD_RELON_EXCEPTION_HV_OOL(0xf80, hv_facility_unavailable)
+EXC_REAL_OOL(facility_unavailable, 0xf60, 0xf80)
+EXC_VIRT_OOL(facility_unavailable, 0x4f60, 0x4f80, 0xf60)
+TRAMP_KVM(PACA_EXGEN, 0xf60)
+EXC_COMMON(facility_unavailable_common, 0xf60, facility_unavailable_exception)
- /*
- * The __end_interrupts marker must be past the out-of-line (OOL)
- * handlers, so that they are copied to real address 0x100 when running
- * a relocatable kernel. This ensures they can be reached from the short
- * trampoline handlers (like 0x4f00, 0x4f20, etc.) which branch
- * directly, without using LOAD_HANDLER().
- */
- .align 7
- .globl __end_interrupts
-__end_interrupts:
-#if defined(CONFIG_PPC_PSERIES) || defined(CONFIG_PPC_POWERNV)
-/*
- * Data area reserved for FWNMI option.
- * This address (0x7000) is fixed by the RPA.
- */
- .= 0x7000
- .globl fwnmi_data_area
-fwnmi_data_area:
+EXC_REAL_OOL_HV(h_facility_unavailable, 0xf80, 0xfa0)
+EXC_VIRT_OOL_HV(h_facility_unavailable, 0x4f80, 0x4fa0, 0xf80)
+TRAMP_KVM_HV(PACA_EXGEN, 0xf80)
+EXC_COMMON(h_facility_unavailable_common, 0xf80, facility_unavailable_exception)
- /* pseries and powernv need to keep the whole page from
- * 0x7000 to 0x8000 free for use by the firmware
- */
- . = 0x8000
-#endif /* defined(CONFIG_PPC_PSERIES) || defined(CONFIG_PPC_POWERNV) */
- STD_EXCEPTION_COMMON(0xf60, facility_unavailable, facility_unavailable_exception)
- STD_EXCEPTION_COMMON(0xf80, hv_facility_unavailable, facility_unavailable_exception)
+EXC_REAL_NONE(0xfa0, 0x1200)
+EXC_VIRT_NONE(0x4fa0, 0x5200)
#ifdef CONFIG_CBE_RAS
- STD_EXCEPTION_COMMON(0x1200, cbe_system_error, cbe_system_error_exception)
- STD_EXCEPTION_COMMON(0x1600, cbe_maintenance, cbe_maintenance_exception)
- STD_EXCEPTION_COMMON(0x1800, cbe_thermal, cbe_thermal_exception)
-#endif /* CONFIG_CBE_RAS */
-
- .globl hmi_exception_early
-hmi_exception_early:
- EXCEPTION_PROLOG_1(PACA_EXGEN, KVMTEST, 0xe62)
- mr r10,r1 /* Save r1 */
- ld r1,PACAEMERGSP(r13) /* Use emergency stack */
- subi r1,r1,INT_FRAME_SIZE /* alloc stack frame */
- std r9,_CCR(r1) /* save CR in stackframe */
- mfspr r11,SPRN_HSRR0 /* Save HSRR0 */
- std r11,_NIP(r1) /* save HSRR0 in stackframe */
- mfspr r12,SPRN_HSRR1 /* Save SRR1 */
- std r12,_MSR(r1) /* save SRR1 in stackframe */
- std r10,0(r1) /* make stack chain pointer */
- std r0,GPR0(r1) /* save r0 in stackframe */
- std r10,GPR1(r1) /* save r1 in stackframe */
- EXCEPTION_PROLOG_COMMON_2(PACA_EXGEN)
- EXCEPTION_PROLOG_COMMON_3(0xe60)
- addi r3,r1,STACK_FRAME_OVERHEAD
- bl hmi_exception_realmode
- /* Windup the stack. */
- /* Move original HSRR0 and HSRR1 into the respective regs */
- ld r9,_MSR(r1)
- mtspr SPRN_HSRR1,r9
- ld r3,_NIP(r1)
- mtspr SPRN_HSRR0,r3
- ld r9,_CTR(r1)
- mtctr r9
- ld r9,_XER(r1)
- mtxer r9
- ld r9,_LINK(r1)
- mtlr r9
- REST_GPR(0, r1)
- REST_8GPRS(2, r1)
- REST_GPR(10, r1)
- ld r11,_CCR(r1)
- mtcr r11
- REST_GPR(11, r1)
- REST_2GPRS(12, r1)
- /* restore original r1. */
- ld r1,GPR1(r1)
+EXC_REAL_HV(cbe_system_error, 0x1200, 0x1300)
+EXC_VIRT_NONE(0x5200, 0x5300)
+TRAMP_KVM_HV_SKIP(PACA_EXGEN, 0x1200)
+EXC_COMMON(cbe_system_error_common, 0x1200, cbe_system_error_exception)
+#else /* CONFIG_CBE_RAS */
+EXC_REAL_NONE(0x1200, 0x1300)
+EXC_VIRT_NONE(0x5200, 0x5300)
+#endif
- /*
- * Go to virtual mode and pull the HMI event information from
- * firmware.
- */
- .globl hmi_exception_after_realmode
-hmi_exception_after_realmode:
- SET_SCRATCH0(r13)
- EXCEPTION_PROLOG_0(PACA_EXGEN)
- b hmi_exception_hv
+EXC_REAL(instruction_breakpoint, 0x1300, 0x1400)
+EXC_VIRT(instruction_breakpoint, 0x5300, 0x5400, 0x1300)
+TRAMP_KVM_SKIP(PACA_EXGEN, 0x1300)
+EXC_COMMON(instruction_breakpoint_common, 0x1300, instruction_breakpoint_exception)
-#define MACHINE_CHECK_HANDLER_WINDUP \
- /* Clear MSR_RI before setting SRR0 and SRR1. */\
- li r0,MSR_RI; \
- mfmsr r9; /* get MSR value */ \
- andc r9,r9,r0; \
- mtmsrd r9,1; /* Clear MSR_RI */ \
- /* Move original SRR0 and SRR1 into the respective regs */ \
- ld r9,_MSR(r1); \
- mtspr SPRN_SRR1,r9; \
- ld r3,_NIP(r1); \
- mtspr SPRN_SRR0,r3; \
- ld r9,_CTR(r1); \
- mtctr r9; \
- ld r9,_XER(r1); \
- mtxer r9; \
- ld r9,_LINK(r1); \
- mtlr r9; \
- REST_GPR(0, r1); \
- REST_8GPRS(2, r1); \
- REST_GPR(10, r1); \
- ld r11,_CCR(r1); \
- mtcr r11; \
- /* Decrement paca->in_mce. */ \
- lhz r12,PACA_IN_MCE(r13); \
- subi r12,r12,1; \
- sth r12,PACA_IN_MCE(r13); \
- REST_GPR(11, r1); \
- REST_2GPRS(12, r1); \
- /* restore original r1. */ \
- ld r1,GPR1(r1)
+EXC_REAL_NONE(0x1400, 0x1500)
+EXC_VIRT_NONE(0x5400, 0x5500)
- /*
- * Handle machine check early in real mode. We come here with
- * ME=1, MMU (IR=0 and DR=0) off and using MC emergency stack.
- */
- .align 7
- .globl machine_check_handle_early
-machine_check_handle_early:
- std r0,GPR0(r1) /* Save r0 */
- EXCEPTION_PROLOG_COMMON_3(0x200)
- bl save_nvgprs
- addi r3,r1,STACK_FRAME_OVERHEAD
- bl machine_check_early
- std r3,RESULT(r1) /* Save result */
- ld r12,_MSR(r1)
-#ifdef CONFIG_PPC_P7_NAP
- /*
- * Check if thread was in power saving mode. We come here when any
- * of the following is true:
- * a. thread wasn't in power saving mode
- * b. thread was in power saving mode with no state loss,
- * supervisor state loss or hypervisor state loss.
- *
- * Go back to nap/sleep/winkle mode again if (b) is true.
- */
- rlwinm. r11,r12,47-31,30,31 /* Was it in power saving mode? */
- beq 4f /* No, it wasn;t */
- /* Thread was in power saving mode. Go back to nap again. */
- cmpwi r11,2
- blt 3f
- /* Supervisor/Hypervisor state loss */
- li r0,1
- stb r0,PACA_NAPSTATELOST(r13)
-3: bl machine_check_queue_event
- MACHINE_CHECK_HANDLER_WINDUP
- GET_PACA(r13)
- ld r1,PACAR1(r13)
- /*
- * Check what idle state this CPU was in and go back to same mode
- * again.
- */
- lbz r3,PACA_THREAD_IDLE_STATE(r13)
- cmpwi r3,PNV_THREAD_NAP
- bgt 10f
- IDLE_STATE_ENTER_SEQ(PPC_NAP)
- /* No return */
-10:
- cmpwi r3,PNV_THREAD_SLEEP
- bgt 2f
- IDLE_STATE_ENTER_SEQ(PPC_SLEEP)
- /* No return */
+EXC_REAL_BEGIN(denorm_exception_hv, 0x1500, 0x1600)
+ mtspr SPRN_SPRG_HSCRATCH0,r13
+ EXCEPTION_PROLOG_0(PACA_EXGEN)
+ EXCEPTION_PROLOG_1(PACA_EXGEN, NOTEST, 0x1500)
-2:
- /*
- * Go back to winkle. Please note that this thread was woken up in
- * machine check from winkle and have not restored the per-subcore
- * state. Hence before going back to winkle, set last bit of HSPGR0
- * to 1. This will make sure that if this thread gets woken up
- * again at reset vector 0x100 then it will get chance to restore
- * the subcore state.
- */
- ori r13,r13,1
- SET_PACA(r13)
- IDLE_STATE_ENTER_SEQ(PPC_WINKLE)
- /* No return */
-4:
+#ifdef CONFIG_PPC_DENORMALISATION
+ mfspr r10,SPRN_HSRR1
+ mfspr r11,SPRN_HSRR0 /* save HSRR0 */
+ andis. r10,r10,(HSRR1_DENORM)@h /* denorm? */
+ addi r11,r11,-4 /* HSRR0 is next instruction */
+ bne+ denorm_assist
#endif
- /*
- * Check if we are coming from hypervisor userspace. If yes then we
- * continue in host kernel in V mode to deliver the MC event.
- */
- rldicl. r11,r12,4,63 /* See if MC hit while in HV mode. */
- beq 5f
- andi. r11,r12,MSR_PR /* See if coming from user. */
- bne 9f /* continue in V mode if we are. */
-5:
-#ifdef CONFIG_KVM_BOOK3S_64_HANDLER
- /*
- * We are coming from kernel context. Check if we are coming from
- * guest. if yes, then we can continue. We will fall through
- * do_kvm_200->kvmppc_interrupt to deliver the MC event to guest.
- */
- lbz r11,HSTATE_IN_GUEST(r13)
- cmpwi r11,0 /* Check if coming from guest */
- bne 9f /* continue if we are. */
+ KVMTEST_PR(0x1500)
+ EXCEPTION_PROLOG_PSERIES_1(denorm_common, EXC_HV)
+EXC_REAL_END(denorm_exception_hv, 0x1500, 0x1600)
+
+#ifdef CONFIG_PPC_DENORMALISATION
+EXC_VIRT_BEGIN(denorm_exception, 0x5500, 0x5600)
+ b exc_real_0x1500_denorm_exception_hv
+EXC_VIRT_END(denorm_exception, 0x5500, 0x5600)
+#else
+EXC_VIRT_NONE(0x5500, 0x5600)
#endif
- /*
- * At this point we are not sure about what context we come from.
- * Queue up the MCE event and return from the interrupt.
- * But before that, check if this is an un-recoverable exception.
- * If yes, then stay on emergency stack and panic.
- */
- andi. r11,r12,MSR_RI
- bne 2f
-1: mfspr r11,SPRN_SRR0
- LOAD_HANDLER(r10,unrecover_mce)
- mtspr SPRN_SRR0,r10
- ld r10,PACAKMSR(r13)
- /*
- * We are going down. But there are chances that we might get hit by
- * another MCE during panic path and we may run into unstable state
- * with no way out. Hence, turn ME bit off while going down, so that
- * when another MCE is hit during panic path, system will checkstop
- * and hypervisor will get restarted cleanly by SP.
- */
- li r3,MSR_ME
- andc r10,r10,r3 /* Turn off MSR_ME */
- mtspr SPRN_SRR1,r10
- rfid
- b .
-2:
- /*
- * Check if we have successfully handled/recovered from error, if not
- * then stay on emergency stack and panic.
- */
- ld r3,RESULT(r1) /* Load result */
- cmpdi r3,0 /* see if we handled MCE successfully */
- beq 1b /* if !handled then panic */
- /*
- * Return from MC interrupt.
- * Queue up the MCE event so that we can log it later, while
- * returning from kernel or opal call.
- */
- bl machine_check_queue_event
- MACHINE_CHECK_HANDLER_WINDUP
- rfid
-9:
- /* Deliver the machine check to host kernel in V mode. */
- MACHINE_CHECK_HANDLER_WINDUP
- b machine_check_pSeries
+TRAMP_KVM_SKIP(PACA_EXGEN, 0x1500)
+
+#ifdef CONFIG_PPC_DENORMALISATION
+TRAMP_REAL_BEGIN(denorm_assist)
+BEGIN_FTR_SECTION
+/*
+ * To denormalise we need to move a copy of the register to itself.
+ * For POWER6 do that here for all FP regs.
+ */
+ mfmsr r10
+ ori r10,r10,(MSR_FP|MSR_FE0|MSR_FE1)
+ xori r10,r10,(MSR_FE0|MSR_FE1)
+ mtmsrd r10
+ sync
+
+#define FMR2(n) fmr (n), (n) ; fmr n+1, n+1
+#define FMR4(n) FMR2(n) ; FMR2(n+2)
+#define FMR8(n) FMR4(n) ; FMR4(n+4)
+#define FMR16(n) FMR8(n) ; FMR8(n+8)
+#define FMR32(n) FMR16(n) ; FMR16(n+16)
+ FMR32(0)
+
+FTR_SECTION_ELSE
+/*
+ * To denormalise we need to move a copy of the register to itself.
+ * For POWER7 do that here for the first 32 VSX registers only.
+ */
+ mfmsr r10
+ oris r10,r10,MSR_VSX@h
+ mtmsrd r10
+ sync
+
+#define XVCPSGNDP2(n) XVCPSGNDP(n,n,n) ; XVCPSGNDP(n+1,n+1,n+1)
+#define XVCPSGNDP4(n) XVCPSGNDP2(n) ; XVCPSGNDP2(n+2)
+#define XVCPSGNDP8(n) XVCPSGNDP4(n) ; XVCPSGNDP4(n+4)
+#define XVCPSGNDP16(n) XVCPSGNDP8(n) ; XVCPSGNDP8(n+8)
+#define XVCPSGNDP32(n) XVCPSGNDP16(n) ; XVCPSGNDP16(n+16)
+ XVCPSGNDP32(0)
+
+ALT_FTR_SECTION_END_IFCLR(CPU_FTR_ARCH_206)
-unrecover_mce:
- /* Invoke machine_check_exception to print MCE event and panic. */
- addi r3,r1,STACK_FRAME_OVERHEAD
- bl machine_check_exception
- /*
- * We will not reach here. Even if we did, there is no way out. Call
- * unrecoverable_exception and die.
- */
-1: addi r3,r1,STACK_FRAME_OVERHEAD
- bl unrecoverable_exception
- b 1b
+BEGIN_FTR_SECTION
+ b denorm_done
+END_FTR_SECTION_IFCLR(CPU_FTR_ARCH_207S)
/*
- * r13 points to the PACA, r9 contains the saved CR,
- * r12 contain the saved SRR1, SRR0 is still ready for return
- * r3 has the faulting address
- * r9 - r13 are saved in paca->exslb.
- * r3 is saved in paca->slb_r3
- * cr6.eq is set for a D-SLB miss, clear for a I-SLB miss
- * We assume we aren't going to take any exceptions during this procedure.
+ * To denormalise we need to move a copy of the register to itself.
+ * For POWER8 we need to do that for all 64 VSX registers
*/
-slb_miss_realmode:
- mflr r10
-#ifdef CONFIG_RELOCATABLE
- mtctr r11
+ XVCPSGNDP32(32)
+denorm_done:
+ mtspr SPRN_HSRR0,r11
+ mtcrf 0x80,r9
+ ld r9,PACA_EXGEN+EX_R9(r13)
+ RESTORE_PPR_PACA(PACA_EXGEN, r10)
+BEGIN_FTR_SECTION
+ ld r10,PACA_EXGEN+EX_CFAR(r13)
+ mtspr SPRN_CFAR,r10
+END_FTR_SECTION_IFSET(CPU_FTR_CFAR)
+ ld r10,PACA_EXGEN+EX_R10(r13)
+ ld r11,PACA_EXGEN+EX_R11(r13)
+ ld r12,PACA_EXGEN+EX_R12(r13)
+ ld r13,PACA_EXGEN+EX_R13(r13)
+ HRFID
+ b .
#endif
- stw r9,PACA_EXSLB+EX_CCR(r13) /* save CR in exc. frame */
- std r10,PACA_EXSLB+EX_LR(r13) /* save LR */
- std r3,PACA_EXSLB+EX_DAR(r13)
+EXC_COMMON_HV(denorm_common, 0x1500, unknown_exception)
- crset 4*cr0+eq
-#ifdef CONFIG_PPC_STD_MMU_64
-BEGIN_MMU_FTR_SECTION
- bl slb_allocate_realmode
-END_MMU_FTR_SECTION_IFCLR(MMU_FTR_TYPE_RADIX)
-#endif
- ld r10,PACA_EXSLB+EX_LR(r13)
- ld r3,PACA_EXSLB+EX_R3(r13)
- lwz r9,PACA_EXSLB+EX_CCR(r13) /* get saved CR */
- mtlr r10
+#ifdef CONFIG_CBE_RAS
+EXC_REAL_HV(cbe_maintenance, 0x1600, 0x1700)
+EXC_VIRT_NONE(0x5600, 0x5700)
+TRAMP_KVM_HV_SKIP(PACA_EXGEN, 0x1600)
+EXC_COMMON(cbe_maintenance_common, 0x1600, cbe_maintenance_exception)
+#else /* CONFIG_CBE_RAS */
+EXC_REAL_NONE(0x1600, 0x1700)
+EXC_VIRT_NONE(0x5600, 0x5700)
+#endif
- beq 8f /* if bad address, make full stack frame */
- andi. r10,r12,MSR_RI /* check for unrecoverable exception */
- beq- 2f
+EXC_REAL(altivec_assist, 0x1700, 0x1800)
+EXC_VIRT(altivec_assist, 0x5700, 0x5800, 0x1700)
+TRAMP_KVM(PACA_EXGEN, 0x1700)
+#ifdef CONFIG_ALTIVEC
+EXC_COMMON(altivec_assist_common, 0x1700, altivec_assist_exception)
+#else
+EXC_COMMON(altivec_assist_common, 0x1700, unknown_exception)
+#endif
- /* All done -- return from exception. */
-.machine push
-.machine "power4"
- mtcrf 0x80,r9
- mtcrf 0x02,r9 /* I/D indication is in cr6 */
- mtcrf 0x01,r9 /* slb_allocate uses cr0 and cr7 */
-.machine pop
+#ifdef CONFIG_CBE_RAS
+EXC_REAL_HV(cbe_thermal, 0x1800, 0x1900)
+EXC_VIRT_NONE(0x5800, 0x5900)
+TRAMP_KVM_HV_SKIP(PACA_EXGEN, 0x1800)
+EXC_COMMON(cbe_thermal_common, 0x1800, cbe_thermal_exception)
+#else /* CONFIG_CBE_RAS */
+EXC_REAL_NONE(0x1800, 0x1900)
+EXC_VIRT_NONE(0x5800, 0x5900)
+#endif
- RESTORE_PPR_PACA(PACA_EXSLB, r9)
- ld r9,PACA_EXSLB+EX_R9(r13)
- ld r10,PACA_EXSLB+EX_R10(r13)
- ld r11,PACA_EXSLB+EX_R11(r13)
- ld r12,PACA_EXSLB+EX_R12(r13)
- ld r13,PACA_EXSLB+EX_R13(r13)
- rfid
- b . /* prevent speculative execution */
-2: mfspr r11,SPRN_SRR0
- LOAD_HANDLER(r10,unrecov_slb)
- mtspr SPRN_SRR0,r10
- ld r10,PACAKMSR(r13)
- mtspr SPRN_SRR1,r10
- rfid
+/*
+ * An interrupt came in while soft-disabled. We set paca->irq_happened, then:
+ * - If it was a decrementer interrupt, we bump the dec to max and and return.
+ * - If it was a doorbell we return immediately since doorbells are edge
+ * triggered and won't automatically refire.
+ * - If it was a HMI we return immediately since we handled it in realmode
+ * and it won't refire.
+ * - else we hard disable and return.
+ * This is called with r10 containing the value to OR to the paca field.
+ */
+#define MASKED_INTERRUPT(_H) \
+masked_##_H##interrupt: \
+ std r11,PACA_EXGEN+EX_R11(r13); \
+ lbz r11,PACAIRQHAPPENED(r13); \
+ or r11,r11,r10; \
+ stb r11,PACAIRQHAPPENED(r13); \
+ cmpwi r10,PACA_IRQ_DEC; \
+ bne 1f; \
+ lis r10,0x7fff; \
+ ori r10,r10,0xffff; \
+ mtspr SPRN_DEC,r10; \
+ b 2f; \
+1: cmpwi r10,PACA_IRQ_DBELL; \
+ beq 2f; \
+ cmpwi r10,PACA_IRQ_HMI; \
+ beq 2f; \
+ mfspr r10,SPRN_##_H##SRR1; \
+ rldicl r10,r10,48,1; /* clear MSR_EE */ \
+ rotldi r10,r10,16; \
+ mtspr SPRN_##_H##SRR1,r10; \
+2: mtcrf 0x80,r9; \
+ ld r9,PACA_EXGEN+EX_R9(r13); \
+ ld r10,PACA_EXGEN+EX_R10(r13); \
+ ld r11,PACA_EXGEN+EX_R11(r13); \
+ GET_SCRATCH0(r13); \
+ ##_H##rfid; \
b .
-unrecov_slb:
- EXCEPTION_PROLOG_COMMON(0x4100, PACA_EXSLB)
- RECONCILE_IRQ_STATE(r10, r11)
- bl save_nvgprs
-1: addi r3,r1,STACK_FRAME_OVERHEAD
- bl unrecoverable_exception
- b 1b
+/*
+ * Real mode exceptions actually use this too, but alternate
+ * instruction code patches (which end up in the common .text area)
+ * cannot reach these if they are put there.
+ */
+USE_FIXED_SECTION(virt_trampolines)
+ MASKED_INTERRUPT()
+ MASKED_INTERRUPT(H)
-8: mfspr r11,SPRN_SRR0
- LOAD_HANDLER(r10,bad_addr_slb)
- mtspr SPRN_SRR0,r10
- ld r10,PACAKMSR(r13)
- mtspr SPRN_SRR1,r10
+#ifdef CONFIG_KVM_BOOK3S_64_HANDLER
+TRAMP_REAL_BEGIN(kvmppc_skip_interrupt)
+ /*
+ * Here all GPRs are unchanged from when the interrupt happened
+ * except for r13, which is saved in SPRG_SCRATCH0.
+ */
+ mfspr r13, SPRN_SRR0
+ addi r13, r13, 4
+ mtspr SPRN_SRR0, r13
+ GET_SCRATCH0(r13)
rfid
b .
-bad_addr_slb:
- EXCEPTION_PROLOG_COMMON(0x380, PACA_EXSLB)
- RECONCILE_IRQ_STATE(r10, r11)
- ld r3, PACA_EXSLB+EX_DAR(r13)
- std r3, _DAR(r1)
- beq cr6, 2f
- li r10, 0x480 /* fix trap number for I-SLB miss */
- std r10, _TRAP(r1)
-2: bl save_nvgprs
- addi r3, r1, STACK_FRAME_OVERHEAD
- bl slb_miss_bad_addr
- b ret_from_except
+TRAMP_REAL_BEGIN(kvmppc_skip_Hinterrupt)
+ /*
+ * Here all GPRs are unchanged from when the interrupt happened
+ * except for r13, which is saved in SPRG_SCRATCH0.
+ */
+ mfspr r13, SPRN_HSRR0
+ addi r13, r13, 4
+ mtspr SPRN_HSRR0, r13
+ GET_SCRATCH0(r13)
+ hrfid
+ b .
+#endif
+
+/*
+ * Ensure that any handlers that get invoked from the exception prologs
+ * above are below the first 64KB (0x10000) of the kernel image because
+ * the prologs assemble the addresses of these handlers using the
+ * LOAD_HANDLER macro, which uses an ori instruction.
+ */
+
+/*** Common interrupt handlers ***/
+
+
+ /*
+ * Relocation-on interrupts: A subset of the interrupts can be delivered
+ * with IR=1/DR=1, if AIL==2 and MSR.HV won't be changed by delivering
+ * it. Addresses are the same as the original interrupt addresses, but
+ * offset by 0xc000000000004000.
+ * It's impossible to receive interrupts below 0x300 via this mechanism.
+ * KVM: None of these traps are from the guest ; anything that escalated
+ * to HV=1 from HV=0 is delivered via real mode handlers.
+ */
+
+ /*
+ * This uses the standard macro, since the original 0x300 vector
+ * only has extra guff for STAB-based processors -- which never
+ * come here.
+ */
+
+EXC_COMMON_BEGIN(ppc64_runlatch_on_trampoline)
+ b __ppc64_runlatch_on
+
+USE_FIXED_SECTION(virt_trampolines)
+ /*
+ * The __end_interrupts marker must be past the out-of-line (OOL)
+ * handlers, so that they are copied to real address 0x100 when running
+ * a relocatable kernel. This ensures they can be reached from the short
+ * trampoline handlers (like 0x4f00, 0x4f20, etc.) which branch
+ * directly, without using LOAD_HANDLER().
+ */
+ .align 7
+ .globl __end_interrupts
+__end_interrupts:
+DEFINE_FIXED_SYMBOL(__end_interrupts)
#ifdef CONFIG_PPC_970_NAP
-power4_fixup_nap:
+TRAMP_REAL_BEGIN(power4_fixup_nap)
andc r9,r9,r10
std r9,TI_LOCAL_FLAGS(r11)
ld r10,_LINK(r1) /* make idle task do the */
blr
#endif
+CLOSE_FIXED_SECTION(real_vectors);
+CLOSE_FIXED_SECTION(real_trampolines);
+CLOSE_FIXED_SECTION(virt_vectors);
+CLOSE_FIXED_SECTION(virt_trampolines);
+
+USE_TEXT_SECTION()
+
/*
* Hash table stuff
*/
1: addi r3,r1,STACK_FRAME_OVERHEAD
bl kernel_bad_stack
b 1b
+
+/*
+ * Called from arch_local_irq_enable when an interrupt needs
+ * to be resent. r3 contains 0x500, 0x900, 0xa00 or 0xe80 to indicate
+ * which kind of interrupt. MSR:EE is already off. We generate a
+ * stackframe like if a real interrupt had happened.
+ *
+ * Note: While MSR:EE is off, we need to make sure that _MSR
+ * in the generated frame has EE set to 1 or the exception
+ * handler will not properly re-enable them.
+ */
+_GLOBAL(__replay_interrupt)
+ /* We are going to jump to the exception common code which
+ * will retrieve various register values from the PACA which
+ * we don't give a damn about, so we don't bother storing them.
+ */
+ mfmsr r12
+ mflr r11
+ mfcr r9
+ ori r12,r12,MSR_EE
+ cmpwi r3,0x900
+ beq decrementer_common
+ cmpwi r3,0x500
+ beq hardware_interrupt_common
+BEGIN_FTR_SECTION
+ cmpwi r3,0xe80
+ beq h_doorbell_common
+ cmpwi r3,0xea0
+ beq h_virt_irq_common
+ cmpwi r3,0xe60
+ beq hmi_exception_common
+FTR_SECTION_ELSE
+ cmpwi r3,0xa00
+ beq doorbell_super_common
+ALT_FTR_SECTION_END_IFSET(CPU_FTR_HVMODE)
+ blr
* Take away the '/proc/vmcore'. We are releasing the dump
* memory, hence it will not be valid anymore.
*/
+#ifdef CONFIG_PROC_VMCORE
vmcore_cleanup();
+#endif
fadump_invalidate_release_mem();
} else
#define REST_32FPVSRS(n,c,base) __REST_32FPVSRS(n,__REG_##c,__REG_##base)
#define SAVE_32FPVSRS(n,c,base) __SAVE_32FPVSRS(n,__REG_##c,__REG_##base)
-#ifdef CONFIG_PPC_TRANSACTIONAL_MEM
-/* void do_load_up_transact_fpu(struct thread_struct *thread)
- *
- * This is similar to load_up_fpu but for the transactional version of the FP
- * register set. It doesn't mess with the task MSR or valid flags.
- * Furthermore, we don't do lazy FP with TM currently.
- */
-_GLOBAL(do_load_up_transact_fpu)
- mfmsr r6
- ori r5,r6,MSR_FP
-#ifdef CONFIG_VSX
-BEGIN_FTR_SECTION
- oris r5,r5,MSR_VSX@h
-END_FTR_SECTION_IFSET(CPU_FTR_VSX)
-#endif
- SYNC
- MTMSRD(r5)
-
- addi r7,r3,THREAD_TRANSACT_FPSTATE
- lfd fr0,FPSTATE_FPSCR(r7)
- MTFSF_L(fr0)
- REST_32FPVSRS(0, R4, R7)
-
- blr
-#endif /* CONFIG_PPC_TRANSACTIONAL_MEM */
-
/*
* Load state from memory into FP registers including FPSCR.
* Assumes the caller has enabled FP in the MSR.
#include <asm/page.h>
#include <asm/mmu.h>
#include <asm/ppc_asm.h>
+#include <asm/head-64.h>
#include <asm/asm-offsets.h>
#include <asm/bug.h>
#include <asm/cputable.h>
* 2. The kernel is entered at __start
*/
- .text
- .globl _stext
-_stext:
+OPEN_FIXED_SECTION(first_256B, 0x0, 0x100)
+USE_FIXED_SECTION(first_256B)
+ /*
+ * Offsets are relative from the start of fixed section, and
+ * first_256B starts at 0. Offsets are a bit easier to use here
+ * than the fixed section entry macros.
+ */
+ . = 0x0
_GLOBAL(__start)
/* NOP this out unconditionally */
BEGIN_FTR_SECTION
. = 0x5c
.globl __run_at_load
__run_at_load:
+DEFINE_FIXED_SYMBOL(__run_at_load)
.long 0x72756e30 /* "run0" -- relocate to 0 by default */
#endif
/* Tell the master cpu we're here */
/* Relocation is off & we are located at an address less */
/* than 0x100, so only need to grab low order offset. */
- std r24,__secondary_hold_acknowledge-_stext(0)
+ std r24,(ABS_ADDR(__secondary_hold_acknowledge))(0)
sync
li r26,0
tovirt(r26,r26)
#endif
/* All secondary cpus wait here until told to start. */
-100: ld r12,__secondary_hold_spinloop-_stext(r26)
+100: ld r12,(ABS_ADDR(__secondary_hold_spinloop))(r26)
cmpdi 0,r12,0
beq 100b
#else
BUG_OPCODE
#endif
+CLOSE_FIXED_SECTION(first_256B)
/* This value is used to mark exception frames on the stack. */
.section ".toc","aw"
exception_marker:
.tc ID_72656773_68657265[TC],0x7265677368657265
- .text
+ .previous
/*
* On server, we include the exception vectors code here as it
*/
#ifdef CONFIG_PPC_BOOK3S
#include "exceptions-64s.S"
+#else
+OPEN_TEXT_SECTION(0x100)
#endif
+USE_TEXT_SECTION()
+
#ifdef CONFIG_PPC_BOOK3E
/*
* The booting_thread_hwid holds the thread id we want to boot in cpu
#if defined(CONFIG_PPC_BOOK3E)
tovirt(r26,r26) /* on booke, we already run at PAGE_OFFSET */
#endif
- lwz r7,__run_at_load-_stext(r26)
+ lwz r7,(FIXED_SYMBOL_ABS_ADDR(__run_at_load))(r26)
#if defined(CONFIG_PPC_BOOK3E)
tophys(r26,r26)
#endif
#if defined(CONFIG_PPC_BOOK3E)
tovirt(r26,r26) /* on booke, we already run at PAGE_OFFSET */
#endif
- lwz r7,__run_at_load-_stext(r26)
+ lwz r7,(FIXED_SYMBOL_ABS_ADDR(__run_at_load))(r26)
cmplwi cr0,r7,1
bne 3f
sub r5,r5,r11
#else
/* just copy interrupts */
- LOAD_REG_IMMEDIATE(r5, __end_interrupts - _stext)
+ LOAD_REG_IMMEDIATE(r5, FIXED_SYMBOL_ABS_ADDR(__end_interrupts))
#endif
b 5f
3:
#endif
- lis r5,(copy_to_here - _stext)@ha
- addi r5,r5,(copy_to_here - _stext)@l /* # bytes of memory to copy */
+ /* # bytes of memory to copy */
+ lis r5,(ABS_ADDR(copy_to_here))@ha
+ addi r5,r5,(ABS_ADDR(copy_to_here))@l
bl copy_and_flush /* copy the first n bytes */
/* this includes the code being */
/* executed here. */
- addis r8,r3,(4f - _stext)@ha /* Jump to the copy of this code */
- addi r12,r8,(4f - _stext)@l /* that we just made */
+ /* Jump to the copy of this code that we just made */
+ addis r8,r3,(ABS_ADDR(4f))@ha
+ addi r12,r8,(ABS_ADDR(4f))@l
mtctr r12
bctr
.balign 8
-p_end: .llong _end - _stext
+p_end: .llong _end - copy_to_here
-4: /* Now copy the rest of the kernel up to _end */
- addis r5,r26,(p_end - _stext)@ha
- ld r5,(p_end - _stext)@l(r5) /* get _end */
+4:
+ /*
+ * Now copy the rest of the kernel up to _end, add
+ * _end - copy_to_here to the copy limit and run again.
+ */
+ addis r8,r26,(ABS_ADDR(p_end))@ha
+ ld r8,(ABS_ADDR(p_end))@l(r8)
+ add r5,r5,r8
5: bl copy_and_flush /* copy the rest */
9: b start_here_multiplatform
set_thread_flag(TIF_RESTORE_TM);
}
}
+
+static inline bool msr_tm_active(unsigned long msr)
+{
+ return MSR_TM_ACTIVE(msr);
+}
#else
+static inline bool msr_tm_active(unsigned long msr) { return false; }
static inline void check_if_tm_restore_required(struct task_struct *tsk) { }
#endif /* CONFIG_PPC_TRANSACTIONAL_MEM */
}
early_param("ppc_strict_facility_enable", enable_strict_msr_control);
-void msr_check_and_set(unsigned long bits)
+unsigned long msr_check_and_set(unsigned long bits)
{
unsigned long oldmsr = mfmsr();
unsigned long newmsr;
if (oldmsr != newmsr)
mtmsr_isync(newmsr);
+
+ return newmsr;
}
void __msr_check_and_clear(unsigned long bits)
void enable_kernel_fp(void)
{
+ unsigned long cpumsr;
+
WARN_ON(preemptible());
- msr_check_and_set(MSR_FP);
+ cpumsr = msr_check_and_set(MSR_FP);
if (current->thread.regs && (current->thread.regs->msr & MSR_FP)) {
check_if_tm_restore_required(current);
+ /*
+ * If a thread has already been reclaimed then the
+ * checkpointed registers are on the CPU but have definitely
+ * been saved by the reclaim code. Don't need to and *cannot*
+ * giveup as this would save to the 'live' structure not the
+ * checkpointed structure.
+ */
+ if(!msr_tm_active(cpumsr) && msr_tm_active(current->thread.regs->msr))
+ return;
__giveup_fpu(current);
}
}
EXPORT_SYMBOL(enable_kernel_fp);
static int restore_fp(struct task_struct *tsk) {
- if (tsk->thread.load_fp) {
+ if (tsk->thread.load_fp || msr_tm_active(tsk->thread.regs->msr)) {
load_fp_state(¤t->thread.fp_state);
current->thread.load_fp++;
return 1;
void enable_kernel_altivec(void)
{
+ unsigned long cpumsr;
+
WARN_ON(preemptible());
- msr_check_and_set(MSR_VEC);
+ cpumsr = msr_check_and_set(MSR_VEC);
if (current->thread.regs && (current->thread.regs->msr & MSR_VEC)) {
check_if_tm_restore_required(current);
+ /*
+ * If a thread has already been reclaimed then the
+ * checkpointed registers are on the CPU but have definitely
+ * been saved by the reclaim code. Don't need to and *cannot*
+ * giveup as this would save to the 'live' structure not the
+ * checkpointed structure.
+ */
+ if(!msr_tm_active(cpumsr) && msr_tm_active(current->thread.regs->msr))
+ return;
__giveup_altivec(current);
}
}
static int restore_altivec(struct task_struct *tsk)
{
- if (cpu_has_feature(CPU_FTR_ALTIVEC) && tsk->thread.load_vec) {
+ if (cpu_has_feature(CPU_FTR_ALTIVEC) &&
+ (tsk->thread.load_vec || msr_tm_active(tsk->thread.regs->msr))) {
load_vr_state(&tsk->thread.vr_state);
tsk->thread.used_vr = 1;
tsk->thread.load_vec++;
void enable_kernel_vsx(void)
{
+ unsigned long cpumsr;
+
WARN_ON(preemptible());
- msr_check_and_set(MSR_FP|MSR_VEC|MSR_VSX);
+ cpumsr = msr_check_and_set(MSR_FP|MSR_VEC|MSR_VSX);
if (current->thread.regs && (current->thread.regs->msr & MSR_VSX)) {
check_if_tm_restore_required(current);
+ /*
+ * If a thread has already been reclaimed then the
+ * checkpointed registers are on the CPU but have definitely
+ * been saved by the reclaim code. Don't need to and *cannot*
+ * giveup as this would save to the 'live' structure not the
+ * checkpointed structure.
+ */
+ if(!msr_tm_active(cpumsr) && msr_tm_active(current->thread.regs->msr))
+ return;
if (current->thread.regs->msr & MSR_FP)
__giveup_fpu(current);
if (current->thread.regs->msr & MSR_VEC)
return;
msr_check_and_set(msr_all_available);
+ check_if_tm_restore_required(tsk);
#ifdef CONFIG_PPC_FPU
if (usermsr & MSR_FP)
{
unsigned long msr;
- if (!current->thread.load_fp && !loadvec(current->thread))
+ if (!msr_tm_active(regs->msr) &&
+ !current->thread.load_fp && !loadvec(current->thread))
return;
msr = regs->msr;
}
#ifdef CONFIG_PPC_TRANSACTIONAL_MEM
+
+static inline bool tm_enabled(struct task_struct *tsk)
+{
+ return tsk && tsk->thread.regs && (tsk->thread.regs->msr & MSR_TM);
+}
+
static void tm_reclaim_thread(struct thread_struct *thr,
struct thread_info *ti, uint8_t cause)
{
- unsigned long msr_diff = 0;
-
- /*
- * If FP/VSX registers have been already saved to the
- * thread_struct, move them to the transact_fp array.
- * We clear the TIF_RESTORE_TM bit since after the reclaim
- * the thread will no longer be transactional.
- */
- if (test_ti_thread_flag(ti, TIF_RESTORE_TM)) {
- msr_diff = thr->ckpt_regs.msr & ~thr->regs->msr;
- if (msr_diff & MSR_FP)
- memcpy(&thr->transact_fp, &thr->fp_state,
- sizeof(struct thread_fp_state));
- if (msr_diff & MSR_VEC)
- memcpy(&thr->transact_vr, &thr->vr_state,
- sizeof(struct thread_vr_state));
- clear_ti_thread_flag(ti, TIF_RESTORE_TM);
- msr_diff &= MSR_FP | MSR_VEC | MSR_VSX | MSR_FE0 | MSR_FE1;
- }
-
/*
* Use the current MSR TM suspended bit to track if we have
* checkpointed state outstanding.
if (!MSR_TM_SUSPENDED(mfmsr()))
return;
- tm_reclaim(thr, thr->regs->msr, cause);
+ giveup_all(container_of(thr, struct task_struct, thread));
- /* Having done the reclaim, we now have the checkpointed
- * FP/VSX values in the registers. These might be valid
- * even if we have previously called enable_kernel_fp() or
- * flush_fp_to_thread(), so update thr->regs->msr to
- * indicate their current validity.
- */
- thr->regs->msr |= msr_diff;
+ tm_reclaim(thr, thr->ckpt_regs.msr, cause);
}
void tm_reclaim_current(uint8_t cause)
*
* In switching we need to maintain a 2nd register state as
* oldtask->thread.ckpt_regs. We tm_reclaim(oldproc); this saves the
- * checkpointed (tbegin) state in ckpt_regs and saves the transactional
- * (current) FPRs into oldtask->thread.transact_fpr[].
+ * checkpointed (tbegin) state in ckpt_regs, ckfp_state and
+ * ckvr_state
*
* We also context switch (save) TFHAR/TEXASR/TFIAR in here.
*/
if (!MSR_TM_ACTIVE(thr->regs->msr))
goto out_and_saveregs;
- /* Stash the original thread MSR, as giveup_fpu et al will
- * modify it. We hold onto it to see whether the task used
- * FP & vector regs. If the TIF_RESTORE_TM flag is set,
- * ckpt_regs.msr is already set.
- */
- if (!test_ti_thread_flag(task_thread_info(tsk), TIF_RESTORE_TM))
- thr->ckpt_regs.msr = thr->regs->msr;
-
TM_DEBUG("--- tm_reclaim on pid %d (NIP=%lx, "
"ccr=%lx, msr=%lx, trap=%lx)\n",
tsk->pid, thr->regs->nip,
{
unsigned long flags;
+ if (!(thread->regs->msr & MSR_TM))
+ return;
+
/* We really can't be interrupted here as the TEXASR registers can't
* change and later in the trecheckpoint code, we have a userspace R1.
* So let's hard disable over this region.
* If the task was using FP, we non-lazily reload both the original and
* the speculative FP register states. This is because the kernel
* doesn't see if/when a TM rollback occurs, so if we take an FP
- * unavoidable later, we are unable to determine which set of FP regs
+ * unavailable later, we are unable to determine which set of FP regs
* need to be restored.
*/
- if (!new->thread.regs)
+ if (!tm_enabled(new))
return;
if (!MSR_TM_ACTIVE(new->thread.regs->msr)){
"(new->msr 0x%lx, new->origmsr 0x%lx)\n",
new->pid, new->thread.regs->msr, msr);
- /* This loads the checkpointed FP/VEC state, if used */
tm_recheckpoint(&new->thread, msr);
- /* This loads the speculative FP/VEC state, if used */
- if (msr & MSR_FP) {
- do_load_up_transact_fpu(&new->thread);
- new->thread.regs->msr |=
- (MSR_FP | new->thread.fpexc_mode);
- }
-#ifdef CONFIG_ALTIVEC
- if (msr & MSR_VEC) {
- do_load_up_transact_altivec(&new->thread);
- new->thread.regs->msr |= MSR_VEC;
- }
-#endif
- /* We may as well turn on VSX too since all the state is restored now */
- if (msr & MSR_VSX)
- new->thread.regs->msr |= MSR_VSX;
+ /*
+ * The checkpointed state has been restored but the live state has
+ * not, ensure all the math functionality is turned off to trigger
+ * restore_math() to reload.
+ */
+ new->thread.regs->msr &= ~(MSR_FP | MSR_VEC | MSR_VSX);
TM_DEBUG("*** tm_recheckpoint of pid %d complete "
"(kernel msr 0x%lx)\n",
new->pid, mfmsr());
}
-static inline void __switch_to_tm(struct task_struct *prev)
+static inline void __switch_to_tm(struct task_struct *prev,
+ struct task_struct *new)
{
if (cpu_has_feature(CPU_FTR_TM)) {
- tm_enable();
- tm_reclaim_task(prev);
+ if (tm_enabled(prev) || tm_enabled(new))
+ tm_enable();
+
+ if (tm_enabled(prev)) {
+ prev->thread.load_tm++;
+ tm_reclaim_task(prev);
+ if (!MSR_TM_ACTIVE(prev->thread.regs->msr) && prev->thread.load_tm == 0)
+ prev->thread.regs->msr &= ~MSR_TM;
+ }
+
+ tm_recheckpoint_new_task(new);
}
}
{
unsigned long msr_diff;
+ /*
+ * This is the only moment we should clear TIF_RESTORE_TM as
+ * it is here that ckpt_regs.msr and pt_regs.msr become the same
+ * again, anything else could lead to an incorrect ckpt_msr being
+ * saved and therefore incorrect signal contexts.
+ */
clear_thread_flag(TIF_RESTORE_TM);
if (!MSR_TM_ACTIVE(regs->msr))
return;
msr_diff = current->thread.ckpt_regs.msr & ~regs->msr;
msr_diff &= MSR_FP | MSR_VEC | MSR_VSX;
+ /* Ensure that restore_math() will restore */
+ if (msr_diff & MSR_FP)
+ current->thread.load_fp = 1;
+#ifdef CONFIG_ALIVEC
+ if (cpu_has_feature(CPU_FTR_ALTIVEC) && msr_diff & MSR_VEC)
+ current->thread.load_vec = 1;
+#endif
restore_math(regs);
regs->msr |= msr_diff;
#else
#define tm_recheckpoint_new_task(new)
-#define __switch_to_tm(prev)
+#define __switch_to_tm(prev, new)
#endif /* CONFIG_PPC_TRANSACTIONAL_MEM */
static inline void save_sprs(struct thread_struct *t)
*/
save_sprs(&prev->thread);
- __switch_to_tm(prev);
-
/* Save FPU, Altivec, VSX and SPE state */
giveup_all(prev);
+ __switch_to_tm(prev, new);
+
/*
* We can't take a PMU exception inside _switch() since there is a
* window where the kernel stack SLB and the kernel stack are out
*/
hard_irq_disable();
- tm_recheckpoint_new_task(new);
-
/*
* Call restore_sprs() before calling _switch(). If we move it after
* _switch() then we miss out on calling it for new tasks. The reason
* transitions the CPU out of TM mode. Hence we need to call
* tm_recheckpoint_new_task() (on the same task) to restore the
* checkpointed state back and the TM mode.
+ *
+ * Can't pass dst because it isn't ready. Doesn't matter, passing
+ * dst is only important for __switch_to()
*/
- __switch_to_tm(src);
- tm_recheckpoint_new_task(src);
+ __switch_to_tm(src, src);
*dst = *src;
current->thread.used_spe = 0;
#endif /* CONFIG_SPE */
#ifdef CONFIG_PPC_TRANSACTIONAL_MEM
- if (cpu_has_feature(CPU_FTR_TM))
- regs->msr |= MSR_TM;
current->thread.tm_tfhar = 0;
current->thread.tm_texasr = 0;
current->thread.tm_tfiar = 0;
}
/*
- * When the transaction is active, 'transact_fp' holds the current running
- * value of all FPR registers and 'fp_state' holds the last checkpointed
- * value of all FPR registers for the current transaction. When transaction
- * is not active 'fp_state' holds the current running state of all the FPR
- * registers. So this function which returns the current running values of
- * all the FPR registers, needs to know whether any transaction is active
- * or not.
+ * Regardless of transactions, 'fp_state' holds the current running
+ * value of all FPR registers and 'ckfp_state' holds the last checkpointed
+ * value of all FPR registers for the current transaction.
*
* Userspace interface buffer layout:
*
* u64 fpr[32];
* u64 fpscr;
* };
- *
- * There are two config options CONFIG_VSX and CONFIG_PPC_TRANSACTIONAL_MEM
- * which determines the final code in this function. All the combinations of
- * these two config options are possible except the one below as transactional
- * memory config pulls in CONFIG_VSX automatically.
- *
- * !defined(CONFIG_VSX) && defined(CONFIG_PPC_TRANSACTIONAL_MEM)
*/
static int fpr_get(struct task_struct *target, const struct user_regset *regset,
unsigned int pos, unsigned int count,
#ifdef CONFIG_VSX
u64 buf[33];
int i;
-#endif
- flush_fp_to_thread(target);
-#if defined(CONFIG_VSX) && defined(CONFIG_PPC_TRANSACTIONAL_MEM)
- /* copy to local buffer then write that out */
- if (MSR_TM_ACTIVE(target->thread.regs->msr)) {
- flush_altivec_to_thread(target);
- flush_tmregs_to_thread(target);
- for (i = 0; i < 32 ; i++)
- buf[i] = target->thread.TS_TRANS_FPR(i);
- buf[32] = target->thread.transact_fp.fpscr;
- } else {
- for (i = 0; i < 32 ; i++)
- buf[i] = target->thread.TS_FPR(i);
- buf[32] = target->thread.fp_state.fpscr;
- }
- return user_regset_copyout(&pos, &count, &kbuf, &ubuf, buf, 0, -1);
-#endif
+ flush_fp_to_thread(target);
-#if defined(CONFIG_VSX) && !defined(CONFIG_PPC_TRANSACTIONAL_MEM)
/* copy to local buffer then write that out */
for (i = 0; i < 32 ; i++)
buf[i] = target->thread.TS_FPR(i);
buf[32] = target->thread.fp_state.fpscr;
return user_regset_copyout(&pos, &count, &kbuf, &ubuf, buf, 0, -1);
-#endif
-
-#if !defined(CONFIG_VSX) && !defined(CONFIG_PPC_TRANSACTIONAL_MEM)
+#else
BUILD_BUG_ON(offsetof(struct thread_fp_state, fpscr) !=
offsetof(struct thread_fp_state, fpr[32]));
+ flush_fp_to_thread(target);
+
return user_regset_copyout(&pos, &count, &kbuf, &ubuf,
&target->thread.fp_state, 0, -1);
#endif
}
/*
- * When the transaction is active, 'transact_fp' holds the current running
- * value of all FPR registers and 'fp_state' holds the last checkpointed
- * value of all FPR registers for the current transaction. When transaction
- * is not active 'fp_state' holds the current running state of all the FPR
- * registers. So this function which setss the current running values of
- * all the FPR registers, needs to know whether any transaction is active
- * or not.
+ * Regardless of transactions, 'fp_state' holds the current running
+ * value of all FPR registers and 'ckfp_state' holds the last checkpointed
+ * value of all FPR registers for the current transaction.
*
* Userspace interface buffer layout:
*
* u64 fpscr;
* };
*
- * There are two config options CONFIG_VSX and CONFIG_PPC_TRANSACTIONAL_MEM
- * which determines the final code in this function. All the combinations of
- * these two config options are possible except the one below as transactional
- * memory config pulls in CONFIG_VSX automatically.
- *
- * !defined(CONFIG_VSX) && defined(CONFIG_PPC_TRANSACTIONAL_MEM)
*/
static int fpr_set(struct task_struct *target, const struct user_regset *regset,
unsigned int pos, unsigned int count,
#ifdef CONFIG_VSX
u64 buf[33];
int i;
-#endif
+
flush_fp_to_thread(target);
-#if defined(CONFIG_VSX) && defined(CONFIG_PPC_TRANSACTIONAL_MEM)
/* copy to local buffer then write that out */
i = user_regset_copyin(&pos, &count, &kbuf, &ubuf, buf, 0, -1);
if (i)
return i;
- if (MSR_TM_ACTIVE(target->thread.regs->msr)) {
- flush_altivec_to_thread(target);
- flush_tmregs_to_thread(target);
- for (i = 0; i < 32 ; i++)
- target->thread.TS_TRANS_FPR(i) = buf[i];
- target->thread.transact_fp.fpscr = buf[32];
- } else {
- for (i = 0; i < 32 ; i++)
- target->thread.TS_FPR(i) = buf[i];
- target->thread.fp_state.fpscr = buf[32];
- }
- return 0;
-#endif
-
-#if defined(CONFIG_VSX) && !defined(CONFIG_PPC_TRANSACTIONAL_MEM)
- /* copy to local buffer then write that out */
- i = user_regset_copyin(&pos, &count, &kbuf, &ubuf, buf, 0, -1);
- if (i)
- return i;
for (i = 0; i < 32 ; i++)
target->thread.TS_FPR(i) = buf[i];
target->thread.fp_state.fpscr = buf[32];
return 0;
-#endif
-
-#if !defined(CONFIG_VSX) && !defined(CONFIG_PPC_TRANSACTIONAL_MEM)
+#else
BUILD_BUG_ON(offsetof(struct thread_fp_state, fpscr) !=
offsetof(struct thread_fp_state, fpr[32]));
+ flush_fp_to_thread(target);
+
return user_regset_copyin(&pos, &count, &kbuf, &ubuf,
&target->thread.fp_state, 0, -1);
#endif
}
/*
- * When the transaction is active, 'transact_vr' holds the current running
- * value of all the VMX registers and 'vr_state' holds the last checkpointed
- * value of all the VMX registers for the current transaction to fall back
- * on in case it aborts. When transaction is not active 'vr_state' holds
- * the current running state of all the VMX registers. So this function which
- * gets the current running values of all the VMX registers, needs to know
- * whether any transaction is active or not.
+ * Regardless of transactions, 'vr_state' holds the current running
+ * value of all the VMX registers and 'ckvr_state' holds the last
+ * checkpointed value of all the VMX registers for the current
+ * transaction to fall back on in case it aborts.
*
* Userspace interface buffer layout:
*
unsigned int pos, unsigned int count,
void *kbuf, void __user *ubuf)
{
- struct thread_vr_state *addr;
int ret;
flush_altivec_to_thread(target);
BUILD_BUG_ON(offsetof(struct thread_vr_state, vscr) !=
offsetof(struct thread_vr_state, vr[32]));
-#ifdef CONFIG_PPC_TRANSACTIONAL_MEM
- if (MSR_TM_ACTIVE(target->thread.regs->msr)) {
- flush_fp_to_thread(target);
- flush_tmregs_to_thread(target);
- addr = &target->thread.transact_vr;
- } else {
- addr = &target->thread.vr_state;
- }
-#else
- addr = &target->thread.vr_state;
-#endif
ret = user_regset_copyout(&pos, &count, &kbuf, &ubuf,
- addr, 0,
+ &target->thread.vr_state, 0,
33 * sizeof(vector128));
if (!ret) {
/*
} vrsave;
memset(&vrsave, 0, sizeof(vrsave));
-#ifdef CONFIG_PPC_TRANSACTIONAL_MEM
- if (MSR_TM_ACTIVE(target->thread.regs->msr))
- vrsave.word = target->thread.transact_vrsave;
- else
- vrsave.word = target->thread.vrsave;
-#else
vrsave.word = target->thread.vrsave;
-#endif
ret = user_regset_copyout(&pos, &count, &kbuf, &ubuf, &vrsave,
33 * sizeof(vector128), -1);
}
/*
- * When the transaction is active, 'transact_vr' holds the current running
- * value of all the VMX registers and 'vr_state' holds the last checkpointed
- * value of all the VMX registers for the current transaction to fall back
- * on in case it aborts. When transaction is not active 'vr_state' holds
- * the current running state of all the VMX registers. So this function which
- * sets the current running values of all the VMX registers, needs to know
- * whether any transaction is active or not.
+ * Regardless of transactions, 'vr_state' holds the current running
+ * value of all the VMX registers and 'ckvr_state' holds the last
+ * checkpointed value of all the VMX registers for the current
+ * transaction to fall back on in case it aborts.
*
* Userspace interface buffer layout:
*
unsigned int pos, unsigned int count,
const void *kbuf, const void __user *ubuf)
{
- struct thread_vr_state *addr;
int ret;
flush_altivec_to_thread(target);
BUILD_BUG_ON(offsetof(struct thread_vr_state, vscr) !=
offsetof(struct thread_vr_state, vr[32]));
-#ifdef CONFIG_PPC_TRANSACTIONAL_MEM
- if (MSR_TM_ACTIVE(target->thread.regs->msr)) {
- flush_fp_to_thread(target);
- flush_tmregs_to_thread(target);
- addr = &target->thread.transact_vr;
- } else {
- addr = &target->thread.vr_state;
- }
-#else
- addr = &target->thread.vr_state;
-#endif
ret = user_regset_copyin(&pos, &count, &kbuf, &ubuf,
- addr, 0,
+ &target->thread.vr_state, 0,
33 * sizeof(vector128));
if (!ret && count > 0) {
/*
} vrsave;
memset(&vrsave, 0, sizeof(vrsave));
-#ifdef CONFIG_PPC_TRANSACTIONAL_MEM
- if (MSR_TM_ACTIVE(target->thread.regs->msr))
- vrsave.word = target->thread.transact_vrsave;
- else
- vrsave.word = target->thread.vrsave;
-#else
vrsave.word = target->thread.vrsave;
-#endif
+
ret = user_regset_copyin(&pos, &count, &kbuf, &ubuf, &vrsave,
33 * sizeof(vector128), -1);
- if (!ret) {
-
-#ifdef CONFIG_PPC_TRANSACTIONAL_MEM
- if (MSR_TM_ACTIVE(target->thread.regs->msr))
- target->thread.transact_vrsave = vrsave.word;
- else
- target->thread.vrsave = vrsave.word;
-#else
+ if (!ret)
target->thread.vrsave = vrsave.word;
-#endif
- }
}
return ret;
}
/*
- * When the transaction is active, 'transact_fp' holds the current running
- * value of all FPR registers and 'fp_state' holds the last checkpointed
- * value of all FPR registers for the current transaction. When transaction
- * is not active 'fp_state' holds the current running state of all the FPR
- * registers. So this function which returns the current running values of
- * all the FPR registers, needs to know whether any transaction is active
- * or not.
+ * Regardless of transactions, 'fp_state' holds the current running
+ * value of all FPR registers and 'ckfp_state' holds the last
+ * checkpointed value of all FPR registers for the current
+ * transaction.
*
* Userspace interface buffer layout:
*
u64 buf[32];
int ret, i;
-#ifdef CONFIG_PPC_TRANSACTIONAL_MEM
+ flush_tmregs_to_thread(target);
flush_fp_to_thread(target);
flush_altivec_to_thread(target);
- flush_tmregs_to_thread(target);
-#endif
flush_vsx_to_thread(target);
-#ifdef CONFIG_PPC_TRANSACTIONAL_MEM
- if (MSR_TM_ACTIVE(target->thread.regs->msr)) {
- for (i = 0; i < 32 ; i++)
- buf[i] = target->thread.
- transact_fp.fpr[i][TS_VSRLOWOFFSET];
- } else {
- for (i = 0; i < 32 ; i++)
- buf[i] = target->thread.
- fp_state.fpr[i][TS_VSRLOWOFFSET];
- }
-#else
for (i = 0; i < 32 ; i++)
buf[i] = target->thread.fp_state.fpr[i][TS_VSRLOWOFFSET];
-#endif
+
ret = user_regset_copyout(&pos, &count, &kbuf, &ubuf,
buf, 0, 32 * sizeof(double));
}
/*
- * When the transaction is active, 'transact_fp' holds the current running
- * value of all FPR registers and 'fp_state' holds the last checkpointed
- * value of all FPR registers for the current transaction. When transaction
- * is not active 'fp_state' holds the current running state of all the FPR
- * registers. So this function which sets the current running values of all
- * the FPR registers, needs to know whether any transaction is active or not.
+ * Regardless of transactions, 'fp_state' holds the current running
+ * value of all FPR registers and 'ckfp_state' holds the last
+ * checkpointed value of all FPR registers for the current
+ * transaction.
*
* Userspace interface buffer layout:
*
u64 buf[32];
int ret,i;
-#ifdef CONFIG_PPC_TRANSACTIONAL_MEM
+ flush_tmregs_to_thread(target);
flush_fp_to_thread(target);
flush_altivec_to_thread(target);
- flush_tmregs_to_thread(target);
-#endif
flush_vsx_to_thread(target);
ret = user_regset_copyin(&pos, &count, &kbuf, &ubuf,
buf, 0, 32 * sizeof(double));
-
-#ifdef CONFIG_PPC_TRANSACTIONAL_MEM
- if (MSR_TM_ACTIVE(target->thread.regs->msr)) {
- for (i = 0; i < 32 ; i++)
- target->thread.transact_fp.
- fpr[i][TS_VSRLOWOFFSET] = buf[i];
- } else {
+ if (!ret)
for (i = 0; i < 32 ; i++)
- target->thread.fp_state.
- fpr[i][TS_VSRLOWOFFSET] = buf[i];
- }
-#else
- for (i = 0; i < 32 ; i++)
- target->thread.fp_state.fpr[i][TS_VSRLOWOFFSET] = buf[i];
-#endif
-
+ target->thread.fp_state.fpr[i][TS_VSRLOWOFFSET] = buf[i];
return ret;
}
if (!MSR_TM_ACTIVE(target->thread.regs->msr))
return -ENODATA;
+ flush_tmregs_to_thread(target);
flush_fp_to_thread(target);
flush_altivec_to_thread(target);
- flush_tmregs_to_thread(target);
ret = user_regset_copyout(&pos, &count, &kbuf, &ubuf,
&target->thread.ckpt_regs,
if (!MSR_TM_ACTIVE(target->thread.regs->msr))
return -ENODATA;
+ flush_tmregs_to_thread(target);
flush_fp_to_thread(target);
flush_altivec_to_thread(target);
- flush_tmregs_to_thread(target);
ret = user_regset_copyin(&pos, &count, &kbuf, &ubuf,
&target->thread.ckpt_regs,
*
* This function gets in transaction checkpointed FPR registers.
*
- * When the transaction is active 'fp_state' holds the checkpointed
+ * When the transaction is active 'ckfp_state' holds the checkpointed
* values for the current transaction to fall back on if it aborts
* in between. This function gets those checkpointed FPR registers.
* The userspace interface buffer layout is as follows.
if (!MSR_TM_ACTIVE(target->thread.regs->msr))
return -ENODATA;
+ flush_tmregs_to_thread(target);
flush_fp_to_thread(target);
flush_altivec_to_thread(target);
- flush_tmregs_to_thread(target);
/* copy to local buffer then write that out */
for (i = 0; i < 32 ; i++)
- buf[i] = target->thread.TS_FPR(i);
- buf[32] = target->thread.fp_state.fpscr;
+ buf[i] = target->thread.TS_CKFPR(i);
+ buf[32] = target->thread.ckfp_state.fpscr;
return user_regset_copyout(&pos, &count, &kbuf, &ubuf, buf, 0, -1);
}
*
* This function sets in transaction checkpointed FPR registers.
*
- * When the transaction is active 'fp_state' holds the checkpointed
+ * When the transaction is active 'ckfp_state' holds the checkpointed
* FPR register values for the current transaction to fall back on
* if it aborts in between. This function sets these checkpointed
* FPR registers. The userspace interface buffer layout is as follows.
if (!MSR_TM_ACTIVE(target->thread.regs->msr))
return -ENODATA;
+ flush_tmregs_to_thread(target);
flush_fp_to_thread(target);
flush_altivec_to_thread(target);
- flush_tmregs_to_thread(target);
/* copy to local buffer then write that out */
i = user_regset_copyin(&pos, &count, &kbuf, &ubuf, buf, 0, -1);
if (i)
return i;
for (i = 0; i < 32 ; i++)
- target->thread.TS_FPR(i) = buf[i];
- target->thread.fp_state.fpscr = buf[32];
+ target->thread.TS_CKFPR(i) = buf[i];
+ target->thread.ckfp_state.fpscr = buf[32];
return 0;
}
*
* This function gets in transaction checkpointed VMX registers.
*
- * When the transaction is active 'vr_state' and 'vr_save' hold
+ * When the transaction is active 'ckvr_state' and 'ckvrsave' hold
* the checkpointed values for the current transaction to fall
* back on if it aborts in between. The userspace interface buffer
* layout is as follows.
return -ENODATA;
/* Flush the state */
+ flush_tmregs_to_thread(target);
flush_fp_to_thread(target);
flush_altivec_to_thread(target);
- flush_tmregs_to_thread(target);
ret = user_regset_copyout(&pos, &count, &kbuf, &ubuf,
- &target->thread.vr_state, 0,
+ &target->thread.ckvr_state, 0,
33 * sizeof(vector128));
if (!ret) {
/*
u32 word;
} vrsave;
memset(&vrsave, 0, sizeof(vrsave));
- vrsave.word = target->thread.vrsave;
+ vrsave.word = target->thread.ckvrsave;
ret = user_regset_copyout(&pos, &count, &kbuf, &ubuf, &vrsave,
33 * sizeof(vector128), -1);
}
*
* This function sets in transaction checkpointed VMX registers.
*
- * When the transaction is active 'vr_state' and 'vr_save' hold
+ * When the transaction is active 'ckvr_state' and 'ckvrsave' hold
* the checkpointed values for the current transaction to fall
* back on if it aborts in between. The userspace interface buffer
* layout is as follows.
if (!MSR_TM_ACTIVE(target->thread.regs->msr))
return -ENODATA;
+ flush_tmregs_to_thread(target);
flush_fp_to_thread(target);
flush_altivec_to_thread(target);
- flush_tmregs_to_thread(target);
ret = user_regset_copyin(&pos, &count, &kbuf, &ubuf,
- &target->thread.vr_state, 0,
+ &target->thread.ckvr_state, 0,
33 * sizeof(vector128));
if (!ret && count > 0) {
/*
u32 word;
} vrsave;
memset(&vrsave, 0, sizeof(vrsave));
- vrsave.word = target->thread.vrsave;
+ vrsave.word = target->thread.ckvrsave;
ret = user_regset_copyin(&pos, &count, &kbuf, &ubuf, &vrsave,
33 * sizeof(vector128), -1);
if (!ret)
- target->thread.vrsave = vrsave.word;
+ target->thread.ckvrsave = vrsave.word;
}
return ret;
*
* This function gets in transaction checkpointed VSX registers.
*
- * When the transaction is active 'fp_state' holds the checkpointed
+ * When the transaction is active 'ckfp_state' holds the checkpointed
* values for the current transaction to fall back on if it aborts
* in between. This function gets those checkpointed VSX registers.
* The userspace interface buffer layout is as follows.
return -ENODATA;
/* Flush the state */
+ flush_tmregs_to_thread(target);
flush_fp_to_thread(target);
flush_altivec_to_thread(target);
- flush_tmregs_to_thread(target);
flush_vsx_to_thread(target);
for (i = 0; i < 32 ; i++)
- buf[i] = target->thread.fp_state.fpr[i][TS_VSRLOWOFFSET];
+ buf[i] = target->thread.ckfp_state.fpr[i][TS_VSRLOWOFFSET];
ret = user_regset_copyout(&pos, &count, &kbuf, &ubuf,
buf, 0, 32 * sizeof(double));
*
* This function sets in transaction checkpointed VSX registers.
*
- * When the transaction is active 'fp_state' holds the checkpointed
+ * When the transaction is active 'ckfp_state' holds the checkpointed
* VSX register values for the current transaction to fall back on
* if it aborts in between. This function sets these checkpointed
* FPR registers. The userspace interface buffer layout is as follows.
return -ENODATA;
/* Flush the state */
+ flush_tmregs_to_thread(target);
flush_fp_to_thread(target);
flush_altivec_to_thread(target);
- flush_tmregs_to_thread(target);
flush_vsx_to_thread(target);
ret = user_regset_copyin(&pos, &count, &kbuf, &ubuf,
buf, 0, 32 * sizeof(double));
- for (i = 0; i < 32 ; i++)
- target->thread.fp_state.fpr[i][TS_VSRLOWOFFSET] = buf[i];
+ if (!ret)
+ for (i = 0; i < 32 ; i++)
+ target->thread.ckfp_state.fpr[i][TS_VSRLOWOFFSET] = buf[i];
return ret;
}
return -ENODEV;
/* Flush the states */
+ flush_tmregs_to_thread(target);
flush_fp_to_thread(target);
flush_altivec_to_thread(target);
- flush_tmregs_to_thread(target);
/* TFHAR register */
ret = user_regset_copyout(&pos, &count, &kbuf, &ubuf,
return -ENODEV;
/* Flush the states */
+ flush_tmregs_to_thread(target);
flush_fp_to_thread(target);
flush_altivec_to_thread(target);
- flush_tmregs_to_thread(target);
/* TFHAR register */
ret = user_regset_copyin(&pos, &count, &kbuf, &ubuf,
}
}
-static void do_signal(struct pt_regs *regs)
+static void do_signal(struct task_struct *tsk)
{
sigset_t *oldset = sigmask_to_save();
struct ksignal ksig;
int ret;
int is32 = is_32bit_task();
+ BUG_ON(tsk != current);
+
get_signal(&ksig);
/* Is there any syscall restart business here ? */
- check_syscall_restart(regs, &ksig.ka, ksig.sig > 0);
+ check_syscall_restart(tsk->thread.regs, &ksig.ka, ksig.sig > 0);
if (ksig.sig <= 0) {
/* No signal to deliver -- put the saved sigmask back */
restore_saved_sigmask();
- regs->trap = 0;
+ tsk->thread.regs->trap = 0;
return; /* no signals delivered */
}
* user space. The DABR will have been cleared if it
* triggered inside the kernel.
*/
- if (current->thread.hw_brk.address &&
- current->thread.hw_brk.type)
- __set_breakpoint(¤t->thread.hw_brk);
+ if (tsk->thread.hw_brk.address && tsk->thread.hw_brk.type)
+ __set_breakpoint(&tsk->thread.hw_brk);
#endif
/* Re-enable the breakpoints for the signal stack */
- thread_change_pc(current, regs);
+ thread_change_pc(tsk, tsk->thread.regs);
if (is32) {
if (ksig.ka.sa.sa_flags & SA_SIGINFO)
- ret = handle_rt_signal32(&ksig, oldset, regs);
+ ret = handle_rt_signal32(&ksig, oldset, tsk);
else
- ret = handle_signal32(&ksig, oldset, regs);
+ ret = handle_signal32(&ksig, oldset, tsk);
} else {
- ret = handle_rt_signal64(&ksig, oldset, regs);
+ ret = handle_rt_signal64(&ksig, oldset, tsk);
}
- regs->trap = 0;
+ tsk->thread.regs->trap = 0;
signal_setup_done(ret, &ksig, test_thread_flag(TIF_SINGLESTEP));
}
if (thread_info_flags & _TIF_UPROBE)
uprobe_notify_resume(regs);
- if (thread_info_flags & _TIF_SIGPENDING)
- do_signal(regs);
+ if (thread_info_flags & _TIF_SIGPENDING) {
+ BUG_ON(regs != current->thread.regs);
+ do_signal(current);
+ }
if (thread_info_flags & _TIF_NOTIFY_RESUME) {
clear_thread_flag(TIF_NOTIFY_RESUME);
user_enter();
}
-unsigned long get_tm_stackpointer(struct pt_regs *regs)
+unsigned long get_tm_stackpointer(struct task_struct *tsk)
{
/* When in an active transaction that takes a signal, we need to be
* careful with the stack. It's possible that the stack has moved back
*/
#ifdef CONFIG_PPC_TRANSACTIONAL_MEM
- if (MSR_TM_ACTIVE(regs->msr)) {
+ BUG_ON(tsk != current);
+
+ if (MSR_TM_ACTIVE(tsk->thread.regs->msr)) {
tm_reclaim_current(TM_CAUSE_SIGNAL);
- if (MSR_TM_TRANSACTIONAL(regs->msr))
- return current->thread.ckpt_regs.gpr[1];
+ if (MSR_TM_TRANSACTIONAL(tsk->thread.regs->msr))
+ return tsk->thread.ckpt_regs.gpr[1];
}
#endif
- return regs->gpr[1];
+ return tsk->thread.regs->gpr[1];
}
size_t frame_size, int is_32);
extern int handle_signal32(struct ksignal *ksig, sigset_t *oldset,
- struct pt_regs *regs);
+ struct task_struct *tsk);
extern int handle_rt_signal32(struct ksignal *ksig, sigset_t *oldset,
- struct pt_regs *regs);
+ struct task_struct *tsk);
extern unsigned long copy_fpr_to_user(void __user *to,
struct task_struct *task);
-extern unsigned long copy_transact_fpr_to_user(void __user *to,
+extern unsigned long copy_ckfpr_to_user(void __user *to,
struct task_struct *task);
extern unsigned long copy_fpr_from_user(struct task_struct *task,
void __user *from);
-extern unsigned long copy_transact_fpr_from_user(struct task_struct *task,
+extern unsigned long copy_ckfpr_from_user(struct task_struct *task,
void __user *from);
+extern unsigned long get_tm_stackpointer(struct task_struct *tsk);
+
#ifdef CONFIG_VSX
extern unsigned long copy_vsx_to_user(void __user *to,
struct task_struct *task);
-extern unsigned long copy_transact_vsx_to_user(void __user *to,
+extern unsigned long copy_ckvsx_to_user(void __user *to,
struct task_struct *task);
extern unsigned long copy_vsx_from_user(struct task_struct *task,
void __user *from);
-extern unsigned long copy_transact_vsx_from_user(struct task_struct *task,
+extern unsigned long copy_ckvsx_from_user(struct task_struct *task,
void __user *from);
#endif
#ifdef CONFIG_PPC64
extern int handle_rt_signal64(struct ksignal *ksig, sigset_t *set,
- struct pt_regs *regs);
+ struct task_struct *tsk);
#else /* CONFIG_PPC64 */
static inline int handle_rt_signal64(struct ksignal *ksig, sigset_t *set,
- struct pt_regs *regs)
+ struct task_struct *tsk)
{
return -EFAULT;
}
}
#ifdef CONFIG_PPC_TRANSACTIONAL_MEM
-unsigned long copy_transact_fpr_to_user(void __user *to,
+unsigned long copy_ckfpr_to_user(void __user *to,
struct task_struct *task)
{
u64 buf[ELF_NFPREG];
/* save FPR copy to local buffer then write to the thread_struct */
for (i = 0; i < (ELF_NFPREG - 1) ; i++)
- buf[i] = task->thread.TS_TRANS_FPR(i);
- buf[i] = task->thread.transact_fp.fpscr;
+ buf[i] = task->thread.TS_CKFPR(i);
+ buf[i] = task->thread.ckfp_state.fpscr;
return __copy_to_user(to, buf, ELF_NFPREG * sizeof(double));
}
-unsigned long copy_transact_fpr_from_user(struct task_struct *task,
+unsigned long copy_ckfpr_from_user(struct task_struct *task,
void __user *from)
{
u64 buf[ELF_NFPREG];
if (__copy_from_user(buf, from, ELF_NFPREG * sizeof(double)))
return 1;
for (i = 0; i < (ELF_NFPREG - 1) ; i++)
- task->thread.TS_TRANS_FPR(i) = buf[i];
- task->thread.transact_fp.fpscr = buf[i];
+ task->thread.TS_CKFPR(i) = buf[i];
+ task->thread.ckfp_state.fpscr = buf[i];
return 0;
}
-unsigned long copy_transact_vsx_to_user(void __user *to,
+unsigned long copy_ckvsx_to_user(void __user *to,
struct task_struct *task)
{
u64 buf[ELF_NVSRHALFREG];
/* save FPR copy to local buffer then write to the thread_struct */
for (i = 0; i < ELF_NVSRHALFREG; i++)
- buf[i] = task->thread.transact_fp.fpr[i][TS_VSRLOWOFFSET];
+ buf[i] = task->thread.ckfp_state.fpr[i][TS_VSRLOWOFFSET];
return __copy_to_user(to, buf, ELF_NVSRHALFREG * sizeof(double));
}
-unsigned long copy_transact_vsx_from_user(struct task_struct *task,
+unsigned long copy_ckvsx_from_user(struct task_struct *task,
void __user *from)
{
u64 buf[ELF_NVSRHALFREG];
if (__copy_from_user(buf, from, ELF_NVSRHALFREG * sizeof(double)))
return 1;
for (i = 0; i < ELF_NVSRHALFREG ; i++)
- task->thread.transact_fp.fpr[i][TS_VSRLOWOFFSET] = buf[i];
+ task->thread.ckfp_state.fpr[i][TS_VSRLOWOFFSET] = buf[i];
return 0;
}
#endif /* CONFIG_PPC_TRANSACTIONAL_MEM */
}
#ifdef CONFIG_PPC_TRANSACTIONAL_MEM
-inline unsigned long copy_transact_fpr_to_user(void __user *to,
+inline unsigned long copy_ckfpr_to_user(void __user *to,
struct task_struct *task)
{
- return __copy_to_user(to, task->thread.transact_fp.fpr,
+ return __copy_to_user(to, task->thread.ckfp_state.fpr,
ELF_NFPREG * sizeof(double));
}
-inline unsigned long copy_transact_fpr_from_user(struct task_struct *task,
+inline unsigned long copy_ckfpr_from_user(struct task_struct *task,
void __user *from)
{
- return __copy_from_user(task->thread.transact_fp.fpr, from,
+ return __copy_from_user(task->thread.ckfp_state.fpr, from,
ELF_NFPREG * sizeof(double));
}
#endif /* CONFIG_PPC_TRANSACTIONAL_MEM */
*/
regs->msr &= ~MSR_TS_MASK;
- /* Make sure floating point registers are stored in regs */
- flush_fp_to_thread(current);
-
/* Save both sets of general registers */
if (save_general_regs(¤t->thread.ckpt_regs, frame)
|| save_general_regs(regs, tm_frame))
#ifdef CONFIG_ALTIVEC
/* save altivec registers */
if (current->thread.used_vr) {
- flush_altivec_to_thread(current);
- if (__copy_to_user(&frame->mc_vregs, ¤t->thread.vr_state,
+ if (__copy_to_user(&frame->mc_vregs, ¤t->thread.ckvr_state,
ELF_NVRREG * sizeof(vector128)))
return 1;
if (msr & MSR_VEC) {
if (__copy_to_user(&tm_frame->mc_vregs,
- ¤t->thread.transact_vr,
+ ¤t->thread.vr_state,
ELF_NVRREG * sizeof(vector128)))
return 1;
} else {
if (__copy_to_user(&tm_frame->mc_vregs,
- ¤t->thread.vr_state,
+ ¤t->thread.ckvr_state,
ELF_NVRREG * sizeof(vector128)))
return 1;
}
* most significant bits of that same vector. --BenH
*/
if (cpu_has_feature(CPU_FTR_ALTIVEC))
- current->thread.vrsave = mfspr(SPRN_VRSAVE);
- if (__put_user(current->thread.vrsave,
+ current->thread.ckvrsave = mfspr(SPRN_VRSAVE);
+ if (__put_user(current->thread.ckvrsave,
(u32 __user *)&frame->mc_vregs[32]))
return 1;
if (msr & MSR_VEC) {
- if (__put_user(current->thread.transact_vrsave,
+ if (__put_user(current->thread.vrsave,
(u32 __user *)&tm_frame->mc_vregs[32]))
return 1;
} else {
- if (__put_user(current->thread.vrsave,
+ if (__put_user(current->thread.ckvrsave,
(u32 __user *)&tm_frame->mc_vregs[32]))
return 1;
}
#endif /* CONFIG_ALTIVEC */
- if (copy_fpr_to_user(&frame->mc_fregs, current))
+ if (copy_ckfpr_to_user(&frame->mc_fregs, current))
return 1;
if (msr & MSR_FP) {
- if (copy_transact_fpr_to_user(&tm_frame->mc_fregs, current))
+ if (copy_fpr_to_user(&tm_frame->mc_fregs, current))
return 1;
} else {
- if (copy_fpr_to_user(&tm_frame->mc_fregs, current))
+ if (copy_ckfpr_to_user(&tm_frame->mc_fregs, current))
return 1;
}
* contains valid data
*/
if (current->thread.used_vsr) {
- flush_vsx_to_thread(current);
- if (copy_vsx_to_user(&frame->mc_vsregs, current))
+ if (copy_ckvsx_to_user(&frame->mc_vsregs, current))
return 1;
if (msr & MSR_VSX) {
- if (copy_transact_vsx_to_user(&tm_frame->mc_vsregs,
+ if (copy_vsx_to_user(&tm_frame->mc_vsregs,
current))
return 1;
} else {
- if (copy_vsx_to_user(&tm_frame->mc_vsregs, current))
+ if (copy_ckvsx_to_user(&tm_frame->mc_vsregs, current))
return 1;
}
regs->msr &= ~MSR_VEC;
if (msr & MSR_VEC) {
/* restore altivec registers from the stack */
- if (__copy_from_user(¤t->thread.vr_state, &sr->mc_vregs,
+ if (__copy_from_user(¤t->thread.ckvr_state, &sr->mc_vregs,
sizeof(sr->mc_vregs)) ||
- __copy_from_user(¤t->thread.transact_vr,
+ __copy_from_user(¤t->thread.vr_state,
&tm_sr->mc_vregs,
sizeof(sr->mc_vregs)))
return 1;
} else if (current->thread.used_vr) {
memset(¤t->thread.vr_state, 0,
ELF_NVRREG * sizeof(vector128));
- memset(¤t->thread.transact_vr, 0,
+ memset(¤t->thread.ckvr_state, 0,
ELF_NVRREG * sizeof(vector128));
}
/* Always get VRSAVE back */
- if (__get_user(current->thread.vrsave,
+ if (__get_user(current->thread.ckvrsave,
(u32 __user *)&sr->mc_vregs[32]) ||
- __get_user(current->thread.transact_vrsave,
+ __get_user(current->thread.vrsave,
(u32 __user *)&tm_sr->mc_vregs[32]))
return 1;
if (cpu_has_feature(CPU_FTR_ALTIVEC))
- mtspr(SPRN_VRSAVE, current->thread.vrsave);
+ mtspr(SPRN_VRSAVE, current->thread.ckvrsave);
#endif /* CONFIG_ALTIVEC */
regs->msr &= ~(MSR_FP | MSR_FE0 | MSR_FE1);
if (copy_fpr_from_user(current, &sr->mc_fregs) ||
- copy_transact_fpr_from_user(current, &tm_sr->mc_fregs))
+ copy_ckfpr_from_user(current, &tm_sr->mc_fregs))
return 1;
#ifdef CONFIG_VSX
* Restore altivec registers from the stack to a local
* buffer, then write this out to the thread_struct
*/
- if (copy_vsx_from_user(current, &sr->mc_vsregs) ||
- copy_transact_vsx_from_user(current, &tm_sr->mc_vsregs))
+ if (copy_vsx_from_user(current, &tm_sr->mc_vsregs) ||
+ copy_ckvsx_from_user(current, &sr->mc_vsregs))
return 1;
current->thread.used_vsr = true;
} else if (current->thread.used_vsr)
for (i = 0; i < 32 ; i++) {
current->thread.fp_state.fpr[i][TS_VSRLOWOFFSET] = 0;
- current->thread.transact_fp.fpr[i][TS_VSRLOWOFFSET] = 0;
+ current->thread.ckfp_state.fpr[i][TS_VSRLOWOFFSET] = 0;
}
#endif /* CONFIG_VSX */
tm_recheckpoint(¤t->thread, msr);
/* This loads the speculative FP/VEC state, if used */
+ msr_check_and_set(msr & (MSR_FP | MSR_VEC));
if (msr & MSR_FP) {
- do_load_up_transact_fpu(¤t->thread);
+ load_fp_state(¤t->thread.fp_state);
regs->msr |= (MSR_FP | current->thread.fpexc_mode);
}
#ifdef CONFIG_ALTIVEC
if (msr & MSR_VEC) {
- do_load_up_transact_altivec(¤t->thread);
+ load_vr_state(¤t->thread.vr_state);
regs->msr |= MSR_VEC;
}
#endif
* (one which gets siginfo).
*/
int handle_rt_signal32(struct ksignal *ksig, sigset_t *oldset,
- struct pt_regs *regs)
+ struct task_struct *tsk)
{
struct rt_sigframe __user *rt_sf;
struct mcontext __user *frame;
unsigned long newsp = 0;
int sigret;
unsigned long tramp;
+ struct pt_regs *regs = tsk->thread.regs;
+
+ BUG_ON(tsk != current);
/* Set up Signal Frame */
/* Put a Real Time Context onto stack */
- rt_sf = get_sigframe(ksig, get_tm_stackpointer(regs), sizeof(*rt_sf), 1);
+ rt_sf = get_sigframe(ksig, get_tm_stackpointer(tsk), sizeof(*rt_sf), 1);
addr = rt_sf;
if (unlikely(rt_sf == NULL))
goto badframe;
/* Save user registers on the stack */
frame = &rt_sf->uc.uc_mcontext;
addr = frame;
- if (vdso32_rt_sigtramp && current->mm->context.vdso_base) {
+ if (vdso32_rt_sigtramp && tsk->mm->context.vdso_base) {
sigret = 0;
- tramp = current->mm->context.vdso_base + vdso32_rt_sigtramp;
+ tramp = tsk->mm->context.vdso_base + vdso32_rt_sigtramp;
} else {
sigret = __NR_rt_sigreturn;
tramp = (unsigned long) frame->tramp;
}
regs->link = tramp;
- current->thread.fp_state.fpscr = 0; /* turn off all fp exceptions */
+ tsk->thread.fp_state.fpscr = 0; /* turn off all fp exceptions */
/* create a stack frame for the caller of the handler */
newsp = ((unsigned long)rt_sf) - (__SIGNAL_FRAMESIZE + 16);
printk_ratelimited(KERN_INFO
"%s[%d]: bad frame in handle_rt_signal32: "
"%p nip %08lx lr %08lx\n",
- current->comm, current->pid,
+ tsk->comm, tsk->pid,
addr, regs->nip, regs->link);
return 1;
/*
* OK, we're invoking a handler
*/
-int handle_signal32(struct ksignal *ksig, sigset_t *oldset, struct pt_regs *regs)
+int handle_signal32(struct ksignal *ksig, sigset_t *oldset,
+ struct task_struct *tsk)
{
struct sigcontext __user *sc;
struct sigframe __user *frame;
unsigned long newsp = 0;
int sigret;
unsigned long tramp;
+ struct pt_regs *regs = tsk->thread.regs;
+
+ BUG_ON(tsk != current);
/* Set up Signal Frame */
- frame = get_sigframe(ksig, get_tm_stackpointer(regs), sizeof(*frame), 1);
+ frame = get_sigframe(ksig, get_tm_stackpointer(tsk), sizeof(*frame), 1);
if (unlikely(frame == NULL))
goto badframe;
sc = (struct sigcontext __user *) &frame->sctx;
|| __put_user(ksig->sig, &sc->signal))
goto badframe;
- if (vdso32_sigtramp && current->mm->context.vdso_base) {
+ if (vdso32_sigtramp && tsk->mm->context.vdso_base) {
sigret = 0;
- tramp = current->mm->context.vdso_base + vdso32_sigtramp;
+ tramp = tsk->mm->context.vdso_base + vdso32_sigtramp;
} else {
sigret = __NR_sigreturn;
tramp = (unsigned long) frame->mctx.tramp;
regs->link = tramp;
- current->thread.fp_state.fpscr = 0; /* turn off all fp exceptions */
+ tsk->thread.fp_state.fpscr = 0; /* turn off all fp exceptions */
/* create a stack frame for the caller of the handler */
newsp = ((unsigned long)frame) - __SIGNAL_FRAMESIZE;
printk_ratelimited(KERN_INFO
"%s[%d]: bad frame in handle_signal32: "
"%p nip %08lx lr %08lx\n",
- current->comm, current->pid,
+ tsk->comm, tsk->pid,
frame, regs->nip, regs->link);
return 1;
* Set up the sigcontext for the signal frame.
*/
-static long setup_sigcontext(struct sigcontext __user *sc, struct pt_regs *regs,
- int signr, sigset_t *set, unsigned long handler,
- int ctx_has_vsx_region)
+static long setup_sigcontext(struct sigcontext __user *sc,
+ struct task_struct *tsk, int signr, sigset_t *set,
+ unsigned long handler, int ctx_has_vsx_region)
{
/* When CONFIG_ALTIVEC is set, we _always_ setup v_regs even if the
* process never used altivec yet (MSR_VEC is zero in pt_regs of
elf_vrreg_t __user *v_regs = sigcontext_vmx_regs(sc);
unsigned long vrsave;
#endif
+ struct pt_regs *regs = tsk->thread.regs;
unsigned long msr = regs->msr;
long err = 0;
+ BUG_ON(tsk != current);
+
#ifdef CONFIG_ALTIVEC
err |= __put_user(v_regs, &sc->v_regs);
/* save altivec registers */
- if (current->thread.used_vr) {
- flush_altivec_to_thread(current);
+ if (tsk->thread.used_vr) {
+ flush_altivec_to_thread(tsk);
/* Copy 33 vec registers (vr0..31 and vscr) to the stack */
- err |= __copy_to_user(v_regs, ¤t->thread.vr_state,
+ err |= __copy_to_user(v_regs, &tsk->thread.vr_state,
33 * sizeof(vector128));
/* set MSR_VEC in the MSR value in the frame to indicate that sc->v_reg)
* contains valid data.
vrsave = 0;
if (cpu_has_feature(CPU_FTR_ALTIVEC)) {
vrsave = mfspr(SPRN_VRSAVE);
- current->thread.vrsave = vrsave;
+ tsk->thread.vrsave = vrsave;
}
err |= __put_user(vrsave, (u32 __user *)&v_regs[33]);
#else /* CONFIG_ALTIVEC */
err |= __put_user(0, &sc->v_regs);
#endif /* CONFIG_ALTIVEC */
- flush_fp_to_thread(current);
+ flush_fp_to_thread(tsk);
/* copy fpr regs and fpscr */
- err |= copy_fpr_to_user(&sc->fp_regs, current);
+ err |= copy_fpr_to_user(&sc->fp_regs, tsk);
/*
* Clear the MSR VSX bit to indicate there is no valid state attached
* then out to userspace. Update v_regs to point after the
* VMX data.
*/
- if (current->thread.used_vsr && ctx_has_vsx_region) {
- flush_vsx_to_thread(current);
+ if (tsk->thread.used_vsr && ctx_has_vsx_region) {
+ flush_vsx_to_thread(tsk);
v_regs += ELF_NVRREG;
- err |= copy_vsx_to_user(v_regs, current);
+ err |= copy_vsx_to_user(v_regs, tsk);
/* set MSR_VSX in the MSR value in the frame to
* indicate that sc->vs_reg) contains valid data.
*/
*/
static long setup_tm_sigcontexts(struct sigcontext __user *sc,
struct sigcontext __user *tm_sc,
- struct pt_regs *regs,
+ struct task_struct *tsk,
int signr, sigset_t *set, unsigned long handler)
{
/* When CONFIG_ALTIVEC is set, we _always_ setup v_regs even if the
elf_vrreg_t __user *v_regs = sigcontext_vmx_regs(sc);
elf_vrreg_t __user *tm_v_regs = sigcontext_vmx_regs(tm_sc);
#endif
- unsigned long msr = regs->msr;
+ struct pt_regs *regs = tsk->thread.regs;
+ unsigned long msr = tsk->thread.ckpt_regs.msr;
long err = 0;
+ BUG_ON(tsk != current);
+
BUG_ON(!MSR_TM_ACTIVE(regs->msr));
/* Remove TM bits from thread's MSR. The MSR in the sigcontext
*/
regs->msr &= ~MSR_TS_MASK;
- flush_fp_to_thread(current);
-
#ifdef CONFIG_ALTIVEC
err |= __put_user(v_regs, &sc->v_regs);
err |= __put_user(tm_v_regs, &tm_sc->v_regs);
/* save altivec registers */
- if (current->thread.used_vr) {
- flush_altivec_to_thread(current);
+ if (tsk->thread.used_vr) {
/* Copy 33 vec registers (vr0..31 and vscr) to the stack */
- err |= __copy_to_user(v_regs, ¤t->thread.vr_state,
+ err |= __copy_to_user(v_regs, &tsk->thread.ckvr_state,
33 * sizeof(vector128));
/* If VEC was enabled there are transactional VRs valid too,
* else they're a copy of the checkpointed VRs.
*/
if (msr & MSR_VEC)
err |= __copy_to_user(tm_v_regs,
- ¤t->thread.transact_vr,
+ &tsk->thread.vr_state,
33 * sizeof(vector128));
else
err |= __copy_to_user(tm_v_regs,
- ¤t->thread.vr_state,
+ &tsk->thread.ckvr_state,
33 * sizeof(vector128));
/* set MSR_VEC in the MSR value in the frame to indicate
* use altivec.
*/
if (cpu_has_feature(CPU_FTR_ALTIVEC))
- current->thread.vrsave = mfspr(SPRN_VRSAVE);
- err |= __put_user(current->thread.vrsave, (u32 __user *)&v_regs[33]);
+ tsk->thread.ckvrsave = mfspr(SPRN_VRSAVE);
+ err |= __put_user(tsk->thread.ckvrsave, (u32 __user *)&v_regs[33]);
if (msr & MSR_VEC)
- err |= __put_user(current->thread.transact_vrsave,
+ err |= __put_user(tsk->thread.vrsave,
(u32 __user *)&tm_v_regs[33]);
else
- err |= __put_user(current->thread.vrsave,
+ err |= __put_user(tsk->thread.ckvrsave,
(u32 __user *)&tm_v_regs[33]);
#else /* CONFIG_ALTIVEC */
#endif /* CONFIG_ALTIVEC */
/* copy fpr regs and fpscr */
- err |= copy_fpr_to_user(&sc->fp_regs, current);
+ err |= copy_ckfpr_to_user(&sc->fp_regs, tsk);
if (msr & MSR_FP)
- err |= copy_transact_fpr_to_user(&tm_sc->fp_regs, current);
+ err |= copy_fpr_to_user(&tm_sc->fp_regs, tsk);
else
- err |= copy_fpr_to_user(&tm_sc->fp_regs, current);
+ err |= copy_ckfpr_to_user(&tm_sc->fp_regs, tsk);
#ifdef CONFIG_VSX
/*
* then out to userspace. Update v_regs to point after the
* VMX data.
*/
- if (current->thread.used_vsr) {
- flush_vsx_to_thread(current);
+ if (tsk->thread.used_vsr) {
v_regs += ELF_NVRREG;
tm_v_regs += ELF_NVRREG;
- err |= copy_vsx_to_user(v_regs, current);
+ err |= copy_ckvsx_to_user(v_regs, tsk);
if (msr & MSR_VSX)
- err |= copy_transact_vsx_to_user(tm_v_regs, current);
+ err |= copy_vsx_to_user(tm_v_regs, tsk);
else
- err |= copy_vsx_to_user(tm_v_regs, current);
+ err |= copy_ckvsx_to_user(tm_v_regs, tsk);
/* set MSR_VSX in the MSR value in the frame to
* indicate that sc->vs_reg) contains valid data.
WARN_ON(!FULL_REGS(regs));
err |= __copy_to_user(&tm_sc->gp_regs, regs, GP_REGS_SIZE);
err |= __copy_to_user(&sc->gp_regs,
- ¤t->thread.ckpt_regs, GP_REGS_SIZE);
+ &tsk->thread.ckpt_regs, GP_REGS_SIZE);
err |= __put_user(msr, &tm_sc->gp_regs[PT_MSR]);
err |= __put_user(msr, &sc->gp_regs[PT_MSR]);
err |= __put_user(signr, &sc->signal);
* Restore the sigcontext from the signal frame.
*/
-static long restore_sigcontext(struct pt_regs *regs, sigset_t *set, int sig,
+static long restore_sigcontext(struct task_struct *tsk, sigset_t *set, int sig,
struct sigcontext __user *sc)
{
#ifdef CONFIG_ALTIVEC
unsigned long err = 0;
unsigned long save_r13 = 0;
unsigned long msr;
+ struct pt_regs *regs = tsk->thread.regs;
#ifdef CONFIG_VSX
int i;
#endif
+ BUG_ON(tsk != current);
+
/* If this is not a signal return, we preserve the TLS in r13 */
if (!sig)
save_r13 = regs->gpr[13];
/*
* Force reload of FP/VEC.
- * This has to be done before copying stuff into current->thread.fpr/vr
+ * This has to be done before copying stuff into tsk->thread.fpr/vr
* for the reasons explained in the previous comment.
*/
regs->msr &= ~(MSR_FP | MSR_FE0 | MSR_FE1 | MSR_VEC | MSR_VSX);
return -EFAULT;
/* Copy 33 vec registers (vr0..31 and vscr) from the stack */
if (v_regs != NULL && (msr & MSR_VEC) != 0) {
- err |= __copy_from_user(¤t->thread.vr_state, v_regs,
+ err |= __copy_from_user(&tsk->thread.vr_state, v_regs,
33 * sizeof(vector128));
- current->thread.used_vr = true;
+ tsk->thread.used_vr = true;
+ } else if (tsk->thread.used_vr) {
+ memset(&tsk->thread.vr_state, 0, 33 * sizeof(vector128));
}
- else if (current->thread.used_vr)
- memset(¤t->thread.vr_state, 0, 33 * sizeof(vector128));
/* Always get VRSAVE back */
if (v_regs != NULL)
- err |= __get_user(current->thread.vrsave, (u32 __user *)&v_regs[33]);
+ err |= __get_user(tsk->thread.vrsave, (u32 __user *)&v_regs[33]);
else
- current->thread.vrsave = 0;
+ tsk->thread.vrsave = 0;
if (cpu_has_feature(CPU_FTR_ALTIVEC))
- mtspr(SPRN_VRSAVE, current->thread.vrsave);
+ mtspr(SPRN_VRSAVE, tsk->thread.vrsave);
#endif /* CONFIG_ALTIVEC */
/* restore floating point */
- err |= copy_fpr_from_user(current, &sc->fp_regs);
+ err |= copy_fpr_from_user(tsk, &sc->fp_regs);
#ifdef CONFIG_VSX
/*
* Get additional VSX data. Update v_regs to point after the
*/
v_regs += ELF_NVRREG;
if ((msr & MSR_VSX) != 0) {
- err |= copy_vsx_from_user(current, v_regs);
- current->thread.used_vsr = true;
- } else
+ err |= copy_vsx_from_user(tsk, v_regs);
+ tsk->thread.used_vsr = true;
+ } else {
for (i = 0; i < 32 ; i++)
- current->thread.fp_state.fpr[i][TS_VSRLOWOFFSET] = 0;
+ tsk->thread.fp_state.fpr[i][TS_VSRLOWOFFSET] = 0;
+ }
#endif
return err;
}
* Restore the two sigcontexts from the frame of a transactional processes.
*/
-static long restore_tm_sigcontexts(struct pt_regs *regs,
+static long restore_tm_sigcontexts(struct task_struct *tsk,
struct sigcontext __user *sc,
struct sigcontext __user *tm_sc)
{
#endif
unsigned long err = 0;
unsigned long msr;
+ struct pt_regs *regs = tsk->thread.regs;
#ifdef CONFIG_VSX
int i;
#endif
+
+ BUG_ON(tsk != current);
+
/* copy the GPRs */
err |= __copy_from_user(regs->gpr, tm_sc->gp_regs, sizeof(regs->gpr));
- err |= __copy_from_user(¤t->thread.ckpt_regs, sc->gp_regs,
+ err |= __copy_from_user(&tsk->thread.ckpt_regs, sc->gp_regs,
sizeof(regs->gpr));
/*
* we don't need to re-copy them here.
*/
err |= __get_user(regs->nip, &tm_sc->gp_regs[PT_NIP]);
- err |= __get_user(current->thread.tm_tfhar, &sc->gp_regs[PT_NIP]);
+ err |= __get_user(tsk->thread.tm_tfhar, &sc->gp_regs[PT_NIP]);
/* get MSR separately, transfer the LE bit if doing signal return */
err |= __get_user(msr, &sc->gp_regs[PT_MSR]);
err |= __get_user(regs->link, &tm_sc->gp_regs[PT_LNK]);
err |= __get_user(regs->xer, &tm_sc->gp_regs[PT_XER]);
err |= __get_user(regs->ccr, &tm_sc->gp_regs[PT_CCR]);
- err |= __get_user(current->thread.ckpt_regs.ctr,
+ err |= __get_user(tsk->thread.ckpt_regs.ctr,
&sc->gp_regs[PT_CTR]);
- err |= __get_user(current->thread.ckpt_regs.link,
+ err |= __get_user(tsk->thread.ckpt_regs.link,
&sc->gp_regs[PT_LNK]);
- err |= __get_user(current->thread.ckpt_regs.xer,
+ err |= __get_user(tsk->thread.ckpt_regs.xer,
&sc->gp_regs[PT_XER]);
- err |= __get_user(current->thread.ckpt_regs.ccr,
+ err |= __get_user(tsk->thread.ckpt_regs.ccr,
&sc->gp_regs[PT_CCR]);
/* These regs are not checkpointed; they can go in 'regs'. */
/*
* Force reload of FP/VEC.
- * This has to be done before copying stuff into current->thread.fpr/vr
+ * This has to be done before copying stuff into tsk->thread.fpr/vr
* for the reasons explained in the previous comment.
*/
regs->msr &= ~(MSR_FP | MSR_FE0 | MSR_FE1 | MSR_VEC | MSR_VSX);
return -EFAULT;
/* Copy 33 vec registers (vr0..31 and vscr) from the stack */
if (v_regs != NULL && tm_v_regs != NULL && (msr & MSR_VEC) != 0) {
- err |= __copy_from_user(¤t->thread.vr_state, v_regs,
+ err |= __copy_from_user(&tsk->thread.ckvr_state, v_regs,
33 * sizeof(vector128));
- err |= __copy_from_user(¤t->thread.transact_vr, tm_v_regs,
+ err |= __copy_from_user(&tsk->thread.vr_state, tm_v_regs,
33 * sizeof(vector128));
current->thread.used_vr = true;
}
- else if (current->thread.used_vr) {
- memset(¤t->thread.vr_state, 0, 33 * sizeof(vector128));
- memset(¤t->thread.transact_vr, 0, 33 * sizeof(vector128));
+ else if (tsk->thread.used_vr) {
+ memset(&tsk->thread.vr_state, 0, 33 * sizeof(vector128));
+ memset(&tsk->thread.ckvr_state, 0, 33 * sizeof(vector128));
}
/* Always get VRSAVE back */
if (v_regs != NULL && tm_v_regs != NULL) {
- err |= __get_user(current->thread.vrsave,
+ err |= __get_user(tsk->thread.ckvrsave,
(u32 __user *)&v_regs[33]);
- err |= __get_user(current->thread.transact_vrsave,
+ err |= __get_user(tsk->thread.vrsave,
(u32 __user *)&tm_v_regs[33]);
}
else {
- current->thread.vrsave = 0;
- current->thread.transact_vrsave = 0;
+ tsk->thread.vrsave = 0;
+ tsk->thread.ckvrsave = 0;
}
if (cpu_has_feature(CPU_FTR_ALTIVEC))
- mtspr(SPRN_VRSAVE, current->thread.vrsave);
+ mtspr(SPRN_VRSAVE, tsk->thread.vrsave);
#endif /* CONFIG_ALTIVEC */
/* restore floating point */
- err |= copy_fpr_from_user(current, &sc->fp_regs);
- err |= copy_transact_fpr_from_user(current, &tm_sc->fp_regs);
+ err |= copy_fpr_from_user(tsk, &tm_sc->fp_regs);
+ err |= copy_ckfpr_from_user(tsk, &sc->fp_regs);
#ifdef CONFIG_VSX
/*
* Get additional VSX data. Update v_regs to point after the
if (v_regs && ((msr & MSR_VSX) != 0)) {
v_regs += ELF_NVRREG;
tm_v_regs += ELF_NVRREG;
- err |= copy_vsx_from_user(current, v_regs);
- err |= copy_transact_vsx_from_user(current, tm_v_regs);
- current->thread.used_vsr = true;
+ err |= copy_vsx_from_user(tsk, tm_v_regs);
+ err |= copy_ckvsx_from_user(tsk, v_regs);
+ tsk->thread.used_vsr = true;
} else {
for (i = 0; i < 32 ; i++) {
- current->thread.fp_state.fpr[i][TS_VSRLOWOFFSET] = 0;
- current->thread.transact_fp.fpr[i][TS_VSRLOWOFFSET] = 0;
+ tsk->thread.fp_state.fpr[i][TS_VSRLOWOFFSET] = 0;
+ tsk->thread.ckfp_state.fpr[i][TS_VSRLOWOFFSET] = 0;
}
}
#endif
tm_enable();
/* Make sure the transaction is marked as failed */
- current->thread.tm_texasr |= TEXASR_FS;
+ tsk->thread.tm_texasr |= TEXASR_FS;
/* This loads the checkpointed FP/VEC state, if used */
- tm_recheckpoint(¤t->thread, msr);
+ tm_recheckpoint(&tsk->thread, msr);
- /* This loads the speculative FP/VEC state, if used */
+ msr_check_and_set(msr & (MSR_FP | MSR_VEC));
if (msr & MSR_FP) {
- do_load_up_transact_fpu(¤t->thread);
- regs->msr |= (MSR_FP | current->thread.fpexc_mode);
+ load_fp_state(&tsk->thread.fp_state);
+ regs->msr |= (MSR_FP | tsk->thread.fpexc_mode);
}
-#ifdef CONFIG_ALTIVEC
if (msr & MSR_VEC) {
- do_load_up_transact_altivec(¤t->thread);
+ load_vr_state(&tsk->thread.vr_state);
regs->msr |= MSR_VEC;
}
-#endif
return err;
}
unsigned long new_msr = 0;
int ctx_has_vsx_region = 0;
+ BUG_ON(regs != current->thread.regs);
+
if (new_ctx &&
get_user(new_msr, &new_ctx->uc_mcontext.gp_regs[PT_MSR]))
return -EFAULT;
if (old_ctx != NULL) {
if (!access_ok(VERIFY_WRITE, old_ctx, ctx_size)
- || setup_sigcontext(&old_ctx->uc_mcontext, regs, 0, NULL, 0,
+ || setup_sigcontext(&old_ctx->uc_mcontext, current, 0, NULL, 0,
ctx_has_vsx_region)
|| __copy_to_user(&old_ctx->uc_sigmask,
¤t->blocked, sizeof(sigset_t)))
if (__copy_from_user(&set, &new_ctx->uc_sigmask, sizeof(set)))
do_exit(SIGSEGV);
set_current_blocked(&set);
- if (restore_sigcontext(regs, NULL, 0, &new_ctx->uc_mcontext))
+ if (restore_sigcontext(current, NULL, 0, &new_ctx->uc_mcontext))
do_exit(SIGSEGV);
/* This returns like rt_sigreturn */
unsigned long msr;
#endif
+ BUG_ON(current->thread.regs != regs);
+
/* Always make any pending restarted system calls return -EINTR */
current->restart_block.fn = do_no_restart_syscall;
struct ucontext __user *uc_transact;
if (__get_user(uc_transact, &uc->uc_link))
goto badframe;
- if (restore_tm_sigcontexts(regs, &uc->uc_mcontext,
+ if (restore_tm_sigcontexts(current, &uc->uc_mcontext,
&uc_transact->uc_mcontext))
goto badframe;
}
else
/* Fall through, for non-TM restore */
#endif
- if (restore_sigcontext(regs, NULL, 1, &uc->uc_mcontext))
+ if (restore_sigcontext(current, NULL, 1, &uc->uc_mcontext))
goto badframe;
if (restore_altstack(&uc->uc_stack))
return 0;
}
-int handle_rt_signal64(struct ksignal *ksig, sigset_t *set, struct pt_regs *regs)
+int handle_rt_signal64(struct ksignal *ksig, sigset_t *set,
+ struct task_struct *tsk)
{
struct rt_sigframe __user *frame;
unsigned long newsp = 0;
long err = 0;
+ struct pt_regs *regs = tsk->thread.regs;
+
+ BUG_ON(tsk != current);
- frame = get_sigframe(ksig, get_tm_stackpointer(regs), sizeof(*frame), 0);
+ frame = get_sigframe(ksig, get_tm_stackpointer(tsk), sizeof(*frame), 0);
if (unlikely(frame == NULL))
goto badframe;
err |= __put_user(&frame->uc_transact, &frame->uc.uc_link);
err |= setup_tm_sigcontexts(&frame->uc.uc_mcontext,
&frame->uc_transact.uc_mcontext,
- regs, ksig->sig,
- NULL,
+ tsk, ksig->sig, NULL,
(unsigned long)ksig->ka.sa.sa_handler);
} else
#endif
{
err |= __put_user(0, &frame->uc.uc_link);
- err |= setup_sigcontext(&frame->uc.uc_mcontext, regs, ksig->sig,
+ err |= setup_sigcontext(&frame->uc.uc_mcontext, tsk, ksig->sig,
NULL, (unsigned long)ksig->ka.sa.sa_handler,
1);
}
goto badframe;
/* Make sure signal handler doesn't get spurious FP exceptions */
- current->thread.fp_state.fpscr = 0;
+ tsk->thread.fp_state.fpscr = 0;
/* Set up to return from userspace. */
- if (vdso64_rt_sigtramp && current->mm->context.vdso_base) {
- regs->link = current->mm->context.vdso_base + vdso64_rt_sigtramp;
+ if (vdso64_rt_sigtramp && tsk->mm->context.vdso_base) {
+ regs->link = tsk->mm->context.vdso_base + vdso64_rt_sigtramp;
} else {
err |= setup_trampoline(__NR_rt_sigreturn, &frame->tramp[0]);
if (err)
badframe:
if (show_unhandled_signals)
printk_ratelimited(regs->msr & MSR_64BIT ? fmt64 : fmt32,
- current->comm, current->pid, "setup_rt_frame",
+ tsk->comm, tsk->pid, "setup_rt_frame",
(long)frame, regs->nip, regs->link);
return 1;
/* We've a struct pt_regs at [r1+STACK_FRAME_OVERHEAD]. */
std r3, STK_PARAM(R3)(r1)
+ std r4, STK_PARAM(R4)(r1)
SAVE_NVGPRS(r1)
/* We need to setup MSR for VSX register save instructions. */
mtmsrd r15
std r14, TM_FRAME_L0(r1)
- /* Stash the stack pointer away for use after reclaim */
- std r1, PACAR1(r13)
-
- /* ******************** FPR/VR/VSRs ************
- * Before reclaiming, capture the current/transactional FPR/VR
- * versions /if used/.
- *
- * (If VSX used, FP and VMX are implied. Or, we don't need to look
- * at MSR.VSX as copying FP regs if .FP, vector regs if .VMX covers it.)
- *
- * We're passed the thread's MSR as parameter 2.
- *
- * We enabled VEC/FP/VSX in the msr above, so we can execute these
- * instructions!
- */
- andis. r0, r4, MSR_VEC@h
- beq dont_backup_vec
-
- addi r7, r3, THREAD_TRANSACT_VRSTATE
- SAVE_32VRS(0, r6, r7) /* r6 scratch, r7 transact vr state */
- mfvscr v0
- li r6, VRSTATE_VSCR
- stvx v0, r7, r6
-dont_backup_vec:
- mfspr r0, SPRN_VRSAVE
- std r0, THREAD_TRANSACT_VRSAVE(r3)
-
- andi. r0, r4, MSR_FP
- beq dont_backup_fp
-
- addi r7, r3, THREAD_TRANSACT_FPSTATE
- SAVE_32FPRS_VSRS(0, R6, R7) /* r6 scratch, r7 transact fp state */
-
- mffs fr0
- stfd fr0,FPSTATE_FPSCR(r7)
-
-dont_backup_fp:
/* Do sanity check on MSR to make sure we are suspended */
li r7, (MSR_TS_S)@higher
srdi r6, r14, 32
1: tdeqi r6, 0
EMIT_BUG_ENTRY 1b,__FILE__,__LINE__,0
+ /* Stash the stack pointer away for use after reclaim */
+ std r1, PACAR1(r13)
+
/* Clear MSR RI since we are about to change r1, EE is already off. */
li r4, 0
mtmsrd r4, 1
* MSR.
*/
+
+ /* ******************** FPR/VR/VSRs ************
+ * After reclaiming, capture the checkpointed FPRs/VRs /if used/.
+ *
+ * (If VSX used, FP and VMX are implied. Or, we don't need to look
+ * at MSR.VSX as copying FP regs if .FP, vector regs if .VMX covers it.)
+ *
+ * We're passed the thread's MSR as the second parameter
+ *
+ * We enabled VEC/FP/VSX in the msr above, so we can execute these
+ * instructions!
+ */
+ ld r4, STK_PARAM(R4)(r1) /* Second parameter, MSR * */
+ mr r3, r12
+ andis. r0, r4, MSR_VEC@h
+ beq dont_backup_vec
+
+ addi r7, r3, THREAD_CKVRSTATE
+ SAVE_32VRS(0, r6, r7) /* r6 scratch, r7 transact vr state */
+ mfvscr v0
+ li r6, VRSTATE_VSCR
+ stvx v0, r7, r6
+dont_backup_vec:
+ mfspr r0, SPRN_VRSAVE
+ std r0, THREAD_CKVRSAVE(r3)
+
+ andi. r0, r4, MSR_FP
+ beq dont_backup_fp
+
+ addi r7, r3, THREAD_CKFPSTATE
+ SAVE_32FPRS_VSRS(0, R6, R7) /* r6 scratch, r7 transact fp state */
+
+ mffs fr0
+ stfd fr0,FPSTATE_FPSCR(r7)
+
+dont_backup_fp:
+
/* TM regs, incl TEXASR -- these live in thread_struct. Note they've
* been updated by the treclaim, to explain to userland the failure
* cause (aborted).
/* Restore original MSR/IRQ state & clear TM mode */
ld r14, TM_FRAME_L0(r1) /* Orig MSR */
+
li r15, 0
rldimi r14, r15, MSR_TS_LG, (63-MSR_TS_LG)-1
mtmsrd r14
mtmsr r5
#ifdef CONFIG_ALTIVEC
- /* FP and VEC registers: These are recheckpointed from thread.fpr[]
- * and thread.vr[] respectively. The thread.transact_fpr[] version
- * is more modern, and will be loaded subsequently by any FPUnavailable
- * trap.
+ /*
+ * FP and VEC registers: These are recheckpointed from
+ * thread.ckfp_state and thread.ckvr_state respectively. The
+ * thread.fp_state[] version holds the 'live' (transactional)
+ * and will be loaded subsequently by any FPUnavailable trap.
*/
andis. r0, r4, MSR_VEC@h
beq dont_restore_vec
- addi r8, r3, THREAD_VRSTATE
+ addi r8, r3, THREAD_CKVRSTATE
li r5, VRSTATE_VSCR
lvx v0, r8, r5
mtvscr v0
REST_32VRS(0, r5, r8) /* r5 scratch, r8 ptr */
dont_restore_vec:
- ld r5, THREAD_VRSAVE(r3)
+ ld r5, THREAD_CKVRSAVE(r3)
mtspr SPRN_VRSAVE, r5
#endif
andi. r0, r4, MSR_FP
beq dont_restore_fp
- addi r8, r3, THREAD_FPSTATE
+ addi r8, r3, THREAD_CKFPSTATE
lfd fr0, FPSTATE_FPSCR(r8)
MTFSF_L(fr0)
REST_32FPRS_VSRS(0, R4, R8)
}
#ifdef CONFIG_PPC64
+static void tm_unavailable(struct pt_regs *regs)
+{
+#ifdef CONFIG_PPC_TRANSACTIONAL_MEM
+ if (user_mode(regs)) {
+ current->thread.load_tm++;
+ regs->msr |= MSR_TM;
+ tm_enable();
+ tm_restore_sprs(¤t->thread);
+ return;
+ }
+#endif
+ pr_emerg("Unrecoverable TM Unavailable Exception "
+ "%lx at %lx\n", regs->trap, regs->nip);
+ die("Unrecoverable TM Unavailable Exception", regs, SIGABRT);
+}
+
void facility_unavailable_exception(struct pt_regs *regs)
{
static char *facility_strings[] = {
return;
}
+ if (status == FSCR_TM_LG) {
+ /*
+ * If we're here then the hardware is TM aware because it
+ * generated an exception with FSRM_TM set.
+ *
+ * If cpu_has_feature(CPU_FTR_TM) is false, then either firmware
+ * told us not to do TM, or the kernel is not built with TM
+ * support.
+ *
+ * If both of those things are true, then userspace can spam the
+ * console by triggering the printk() below just by continually
+ * doing tbegin (or any TM instruction). So in that case just
+ * send the process a SIGILL immediately.
+ */
+ if (!cpu_has_feature(CPU_FTR_TM))
+ goto out;
+
+ tm_unavailable(regs);
+ return;
+ }
+
if ((status < ARRAY_SIZE(facility_strings)) &&
facility_strings[status])
facility = facility_strings[status];
"%sFacility '%s' unavailable, exception at 0x%lx, MSR=%lx\n",
hv ? "Hypervisor " : "", facility, regs->nip, regs->msr);
+out:
if (user_mode(regs)) {
_exception(SIGILL, regs, ILL_ILLOPC, regs->nip);
return;
/* If VMX is in use, get the transactional values back */
if (regs->msr & MSR_VEC) {
- do_load_up_transact_altivec(¤t->thread);
+ msr_check_and_set(MSR_VEC);
+ load_vr_state(¤t->thread.vr_state);
/* At this point all the VSX state is loaded, so enable it */
regs->msr |= MSR_VSX;
}
current->thread.used_vr = 1;
if (regs->msr & MSR_FP) {
- do_load_up_transact_fpu(¤t->thread);
+ msr_check_and_set(MSR_FP);
+ load_fp_state(¤t->thread.fp_state);
regs->msr |= MSR_VSX;
}
}
*/
tm_recheckpoint(¤t->thread, regs->msr & ~orig_msr);
+ msr_check_and_set(orig_msr & (MSR_FP | MSR_VEC));
+
if (orig_msr & MSR_FP)
- do_load_up_transact_fpu(¤t->thread);
+ load_fp_state(¤t->thread.fp_state);
if (orig_msr & MSR_VEC)
- do_load_up_transact_altivec(¤t->thread);
+ load_vr_state(¤t->thread.vr_state);
}
#endif /* CONFIG_PPC_TRANSACTIONAL_MEM */
bl V_LOCAL_FUNC(__get_datapage)
mtlr r12
addi r3,r3,CFG_SYSCALL_MAP64
- cmpli cr0,r4,0
+ cmpldi cr0,r4,0
crclr cr0*4+so
beqlr
li r0,NR_syscalls
bne cr0,99f
li r3,0
- cmpli cr0,r4,0
+ cmpldi cr0,r4,0
crclr cr0*4+so
beqlr
lis r5,CLOCK_REALTIME_RES@h
#include <asm/page.h>
#include <asm/ptrace.h>
-#ifdef CONFIG_PPC_TRANSACTIONAL_MEM
-/* void do_load_up_transact_altivec(struct thread_struct *thread)
- *
- * This is similar to load_up_altivec but for the transactional version of the
- * vector regs. It doesn't mess with the task MSR or valid flags.
- * Furthermore, VEC laziness is not supported with TM currently.
- */
-_GLOBAL(do_load_up_transact_altivec)
- mfmsr r6
- oris r5,r6,MSR_VEC@h
- MTMSRD(r5)
- isync
-
- li r4,1
- stw r4,THREAD_USED_VR(r3)
-
- li r10,THREAD_TRANSACT_VRSTATE+VRSTATE_VSCR
- lvx v0,r10,r3
- mtvscr v0
- addi r10,r3,THREAD_TRANSACT_VRSTATE
- REST_32VRS(0,r4,r10)
-
- blr
-#endif
-
/*
* Load state from memory into VMX registers including VSCR.
* Assumes the caller has enabled VMX in the MSR.
* Text, read only data and other permanent read-only sections
*/
- /* Text and gots */
+ _text = .;
+ _stext = .;
+
+ /*
+ * Head text.
+ * This needs to be in its own output section to avoid ld placing
+ * branch trampoline stubs randomly throughout the fixed sections,
+ * which it will do (even if the branch comes from another section)
+ * in order to optimize stub generation.
+ */
+ .head.text : AT(ADDR(.head.text) - LOAD_OFFSET) {
+#ifdef CONFIG_PPC64
+ KEEP(*(.head.text.first_256B));
+#ifdef CONFIG_PPC_BOOK3E
+# define END_FIXED 0x100
+#else
+ KEEP(*(.head.text.real_vectors));
+ *(.head.text.real_trampolines);
+ KEEP(*(.head.text.virt_vectors));
+ *(.head.text.virt_trampolines);
+# if defined(CONFIG_PPC_PSERIES) || defined(CONFIG_PPC_POWERNV)
+ KEEP(*(.head.data.fwnmi_page));
+# define END_FIXED 0x8000
+# else
+# define END_FIXED 0x7000
+# endif
+#endif
+ ASSERT((. == END_FIXED), "vmlinux.lds.S: fixed section overflow error");
+#else /* !CONFIG_PPC64 */
+ HEAD_TEXT
+#endif
+ } :kernel
+
+ /*
+ * If the build dies here, it's likely code in head_64.S is referencing
+ * labels it can't reach, and the linker inserting stubs without the
+ * assembler's knowledge. To debug, remove the above assert and
+ * rebuild. Look for branch stubs in the fixed section region.
+ *
+ * Linker stub generation could be allowed in "trampoline"
+ * sections if absolutely necessary, but this would require
+ * some rework of the fixed sections. Before resorting to this,
+ * consider references that have sufficient addressing range,
+ * (e.g., hand coded trampolines) so the linker does not have
+ * to add stubs.
+ *
+ * Linker stubs at the top of the main text section are currently not
+ * detected, and will result in a crash at boot due to offsets being
+ * wrong.
+ */
.text : AT(ADDR(.text) - LOAD_OFFSET) {
ALIGN_FUNCTION();
- HEAD_TEXT
- _text = .;
/* careful! __ftr_alt_* sections need to be close to .text */
*(.text .fixup __ftr_alt_* .ref.text)
SCHED_TEXT
clrldi r5,r5,58
mtctr r0
beq 5f
+ .balign 16
4: std r4,0(r6)
std r4,8(r6)
std r4,16(r6)
andi. r0,r6,3
mtctr r7
bne 5f
+ .balign 16
1: lwz r7,-4(r4)
lwzu r8,-8(r4)
stw r7,-4(r6)
#include <linux/rculist.h>
#include <linux/vmalloc.h>
#include <linux/mutex.h>
+#include <linux/migrate.h>
+#include <linux/hugetlb.h>
+#include <linux/swap.h>
#include <asm/mmu_context.h>
static DEFINE_MUTEX(mem_list_mutex);
}
EXPORT_SYMBOL_GPL(mm_iommu_preregistered);
+/*
+ * Taken from alloc_migrate_target with changes to remove CMA allocations
+ */
+struct page *new_iommu_non_cma_page(struct page *page, unsigned long private,
+ int **resultp)
+{
+ gfp_t gfp_mask = GFP_USER;
+ struct page *new_page;
+
+ if (PageHuge(page) || PageTransHuge(page) || PageCompound(page))
+ return NULL;
+
+ if (PageHighMem(page))
+ gfp_mask |= __GFP_HIGHMEM;
+
+ /*
+ * We don't want the allocation to force an OOM if possibe
+ */
+ new_page = alloc_page(gfp_mask | __GFP_NORETRY | __GFP_NOWARN);
+ return new_page;
+}
+
+static int mm_iommu_move_page_from_cma(struct page *page)
+{
+ int ret = 0;
+ LIST_HEAD(cma_migrate_pages);
+
+ /* Ignore huge pages for now */
+ if (PageHuge(page) || PageTransHuge(page) || PageCompound(page))
+ return -EBUSY;
+
+ lru_add_drain();
+ ret = isolate_lru_page(page);
+ if (ret)
+ return ret;
+
+ list_add(&page->lru, &cma_migrate_pages);
+ put_page(page); /* Drop the gup reference */
+
+ ret = migrate_pages(&cma_migrate_pages, new_iommu_non_cma_page,
+ NULL, 0, MIGRATE_SYNC, MR_CMA);
+ if (ret) {
+ if (!list_empty(&cma_migrate_pages))
+ putback_movable_pages(&cma_migrate_pages);
+ }
+
+ return 0;
+}
+
long mm_iommu_get(unsigned long ua, unsigned long entries,
struct mm_iommu_table_group_mem_t **pmem)
{
for (i = 0; i < entries; ++i) {
if (1 != get_user_pages_fast(ua + (i << PAGE_SHIFT),
1/* pages */, 1/* iswrite */, &page)) {
+ ret = -EFAULT;
for (j = 0; j < i; ++j)
- put_page(pfn_to_page(
- mem->hpas[j] >> PAGE_SHIFT));
+ put_page(pfn_to_page(mem->hpas[j] >>
+ PAGE_SHIFT));
vfree(mem->hpas);
kfree(mem);
- ret = -EFAULT;
goto unlock_exit;
}
-
+ /*
+ * If we get a page from the CMA zone, since we are going to
+ * be pinning these entries, we might as well move them out
+ * of the CMA zone if possible. NOTE: faulting in + migration
+ * can be expensive. Batching can be considered later
+ */
+ if (get_pageblock_migratetype(page) == MIGRATE_CMA) {
+ if (mm_iommu_move_page_from_cma(page))
+ goto populate;
+ if (1 != get_user_pages_fast(ua + (i << PAGE_SHIFT),
+ 1/* pages */, 1/* iswrite */,
+ &page)) {
+ ret = -EFAULT;
+ for (j = 0; j < i; ++j)
+ put_page(pfn_to_page(mem->hpas[j] >>
+ PAGE_SHIFT));
+ vfree(mem->hpas);
+ kfree(mem);
+ goto unlock_exit;
+ }
+ }
+populate:
mem->hpas[i] = page_to_pfn(page) << PAGE_SHIFT;
}
#define PPC_BLR() EMIT(PPC_INST_BLR)
#define PPC_BLRL() EMIT(PPC_INST_BLRL)
#define PPC_MTLR(r) EMIT(PPC_INST_MTLR | ___PPC_RT(r))
+#define PPC_BCTR() EMIT(PPC_INST_BCTR)
+#define PPC_MTCTR(r) EMIT(PPC_INST_MTCTR | ___PPC_RT(r))
#define PPC_ADDI(d, a, i) EMIT(PPC_INST_ADDI | ___PPC_RT(d) | \
___PPC_RA(a) | IMM_L(i))
#define PPC_MR(d, a) PPC_OR(d, a, a)
/*
* Stack layout:
+ * Ensure the top half (upto local_tmp_var) stays consistent
+ * with our redzone usage.
*
* [ prev sp ] <-------------
* [ nv gpr save area ] 8*8 |
+ * [ tail_call_cnt ] 8 |
+ * [ local_tmp_var ] 8 |
* fp (r31) --> [ ebpf stack space ] 512 |
- * [ local/tmp var space ] 16 |
* [ frame header ] 32/112 |
* sp (r1) ---> [ stack pointer ] --------------
*/
-/* for bpf JIT code internal usage */
-#define BPF_PPC_STACK_LOCALS 16
/* for gpr non volatile registers BPG_REG_6 to 10, plus skb cache registers */
#define BPF_PPC_STACK_SAVE (8*8)
+/* for bpf JIT code internal usage */
+#define BPF_PPC_STACK_LOCALS 16
/* Ensure this is quadword aligned */
-#define BPF_PPC_STACKFRAME (STACK_FRAME_MIN_SIZE + BPF_PPC_STACK_LOCALS + \
- MAX_BPF_STACK + BPF_PPC_STACK_SAVE)
+#define BPF_PPC_STACKFRAME (STACK_FRAME_MIN_SIZE + MAX_BPF_STACK + \
+ BPF_PPC_STACK_LOCALS + BPF_PPC_STACK_SAVE)
#ifndef __ASSEMBLY__
/* BPF register usage */
-#define SKB_HLEN_REG (MAX_BPF_REG + 0)
-#define SKB_DATA_REG (MAX_BPF_REG + 1)
-#define TMP_REG_1 (MAX_BPF_REG + 2)
-#define TMP_REG_2 (MAX_BPF_REG + 3)
+#define SKB_HLEN_REG (MAX_BPF_JIT_REG + 0)
+#define SKB_DATA_REG (MAX_BPF_JIT_REG + 1)
+#define TMP_REG_1 (MAX_BPF_JIT_REG + 2)
+#define TMP_REG_2 (MAX_BPF_JIT_REG + 3)
/* BPF to ppc register mappings */
static const int b2p[] = {
/* frame pointer aka BPF_REG_10 */
[BPF_REG_FP] = 31,
/* eBPF jit internal registers */
+ [BPF_REG_AX] = 2,
[SKB_HLEN_REG] = 25,
[SKB_DATA_REG] = 26,
[TMP_REG_1] = 9,
[TMP_REG_2] = 10
};
+/* PPC NVR range -- update this if we ever use NVRs below r24 */
+#define BPF_PPC_NVR_MIN 24
+
/* Assembly helpers */
#define DECLARE_LOAD_FUNC(func) u64 func(u64 r3, u64 r4); \
u64 func##_negative_offset(u64 r3, u64 r4); \
#define SEEN_FUNC 0x1000 /* might call external helpers */
#define SEEN_STACK 0x2000 /* uses BPF stack */
#define SEEN_SKB 0x4000 /* uses sk_buff */
+#define SEEN_TAILCALL 0x8000 /* uses tail calls */
struct codegen_context {
/*
#include <linux/filter.h>
#include <linux/if_vlan.h>
#include <asm/kprobes.h>
+#include <linux/bpf.h>
#include "bpf_jit64.h"
return ctx->seen & SEEN_FUNC || bpf_is_seen_register(ctx, BPF_REG_FP);
}
+/*
+ * When not setting up our own stackframe, the redzone usage is:
+ *
+ * [ prev sp ] <-------------
+ * [ ... ] |
+ * sp (r1) ---> [ stack pointer ] --------------
+ * [ nv gpr save area ] 8*8
+ * [ tail_call_cnt ] 8
+ * [ local_tmp_var ] 8
+ * [ unused red zone ] 208 bytes protected
+ */
+static int bpf_jit_stack_local(struct codegen_context *ctx)
+{
+ if (bpf_has_stack_frame(ctx))
+ return STACK_FRAME_MIN_SIZE + MAX_BPF_STACK;
+ else
+ return -(BPF_PPC_STACK_SAVE + 16);
+}
+
+static int bpf_jit_stack_tailcallcnt(struct codegen_context *ctx)
+{
+ return bpf_jit_stack_local(ctx) + 8;
+}
+
+static int bpf_jit_stack_offsetof(struct codegen_context *ctx, int reg)
+{
+ if (reg >= BPF_PPC_NVR_MIN && reg < 32)
+ return (bpf_has_stack_frame(ctx) ? BPF_PPC_STACKFRAME : 0)
+ - (8 * (32 - reg));
+
+ pr_err("BPF JIT is asking about unknown registers");
+ BUG();
+}
+
static void bpf_jit_emit_skb_loads(u32 *image, struct codegen_context *ctx)
{
/*
PPC_BPF_LL(b2p[SKB_DATA_REG], 3, offsetof(struct sk_buff, data));
}
-static void bpf_jit_emit_func_call(u32 *image, struct codegen_context *ctx, u64 func)
+static void bpf_jit_build_prologue(u32 *image, struct codegen_context *ctx)
{
-#ifdef PPC64_ELF_ABI_v1
- /* func points to the function descriptor */
- PPC_LI64(b2p[TMP_REG_2], func);
- /* Load actual entry point from function descriptor */
- PPC_BPF_LL(b2p[TMP_REG_1], b2p[TMP_REG_2], 0);
- /* ... and move it to LR */
- PPC_MTLR(b2p[TMP_REG_1]);
+ int i;
+
/*
- * Load TOC from function descriptor at offset 8.
- * We can clobber r2 since we get called through a
- * function pointer (so caller will save/restore r2)
- * and since we don't use a TOC ourself.
+ * Initialize tail_call_cnt if we do tail calls.
+ * Otherwise, put in NOPs so that it can be skipped when we are
+ * invoked through a tail call.
*/
- PPC_BPF_LL(2, b2p[TMP_REG_2], 8);
-#else
- /* We can clobber r12 */
- PPC_FUNC_ADDR(12, func);
- PPC_MTLR(12);
-#endif
- PPC_BLRL();
-}
+ if (ctx->seen & SEEN_TAILCALL) {
+ PPC_LI(b2p[TMP_REG_1], 0);
+ /* this goes in the redzone */
+ PPC_BPF_STL(b2p[TMP_REG_1], 1, -(BPF_PPC_STACK_SAVE + 8));
+ } else {
+ PPC_NOP();
+ PPC_NOP();
+ }
-static void bpf_jit_build_prologue(u32 *image, struct codegen_context *ctx)
-{
- int i;
- bool new_stack_frame = bpf_has_stack_frame(ctx);
+#define BPF_TAILCALL_PROLOGUE_SIZE 8
- if (new_stack_frame) {
+ if (bpf_has_stack_frame(ctx)) {
/*
* We need a stack frame, but we don't necessarily need to
* save/restore LR unless we call other functions
*/
for (i = BPF_REG_6; i <= BPF_REG_10; i++)
if (bpf_is_seen_register(ctx, i))
- PPC_BPF_STL(b2p[i], 1,
- (new_stack_frame ? BPF_PPC_STACKFRAME : 0) -
- (8 * (32 - b2p[i])));
+ PPC_BPF_STL(b2p[i], 1, bpf_jit_stack_offsetof(ctx, b2p[i]));
/*
* Save additional non-volatile regs if we cache skb
*/
if (ctx->seen & SEEN_SKB) {
PPC_BPF_STL(b2p[SKB_HLEN_REG], 1,
- BPF_PPC_STACKFRAME - (8 * (32 - b2p[SKB_HLEN_REG])));
+ bpf_jit_stack_offsetof(ctx, b2p[SKB_HLEN_REG]));
PPC_BPF_STL(b2p[SKB_DATA_REG], 1,
- BPF_PPC_STACKFRAME - (8 * (32 - b2p[SKB_DATA_REG])));
+ bpf_jit_stack_offsetof(ctx, b2p[SKB_DATA_REG]));
bpf_jit_emit_skb_loads(image, ctx);
}
/* Setup frame pointer to point to the bpf stack area */
if (bpf_is_seen_register(ctx, BPF_REG_FP))
PPC_ADDI(b2p[BPF_REG_FP], 1,
- BPF_PPC_STACKFRAME - BPF_PPC_STACK_SAVE);
+ STACK_FRAME_MIN_SIZE + MAX_BPF_STACK);
}
-static void bpf_jit_build_epilogue(u32 *image, struct codegen_context *ctx)
+static void bpf_jit_emit_common_epilogue(u32 *image, struct codegen_context *ctx)
{
int i;
- bool new_stack_frame = bpf_has_stack_frame(ctx);
-
- /* Move result to r3 */
- PPC_MR(3, b2p[BPF_REG_0]);
/* Restore NVRs */
for (i = BPF_REG_6; i <= BPF_REG_10; i++)
if (bpf_is_seen_register(ctx, i))
- PPC_BPF_LL(b2p[i], 1,
- (new_stack_frame ? BPF_PPC_STACKFRAME : 0) -
- (8 * (32 - b2p[i])));
+ PPC_BPF_LL(b2p[i], 1, bpf_jit_stack_offsetof(ctx, b2p[i]));
/* Restore non-volatile registers used for skb cache */
if (ctx->seen & SEEN_SKB) {
PPC_BPF_LL(b2p[SKB_HLEN_REG], 1,
- BPF_PPC_STACKFRAME - (8 * (32 - b2p[SKB_HLEN_REG])));
+ bpf_jit_stack_offsetof(ctx, b2p[SKB_HLEN_REG]));
PPC_BPF_LL(b2p[SKB_DATA_REG], 1,
- BPF_PPC_STACKFRAME - (8 * (32 - b2p[SKB_DATA_REG])));
+ bpf_jit_stack_offsetof(ctx, b2p[SKB_DATA_REG]));
}
/* Tear down our stack frame */
- if (new_stack_frame) {
+ if (bpf_has_stack_frame(ctx)) {
PPC_ADDI(1, 1, BPF_PPC_STACKFRAME);
if (ctx->seen & SEEN_FUNC) {
PPC_BPF_LL(0, 1, PPC_LR_STKOFF);
PPC_MTLR(0);
}
}
+}
+
+static void bpf_jit_build_epilogue(u32 *image, struct codegen_context *ctx)
+{
+ bpf_jit_emit_common_epilogue(image, ctx);
+
+ /* Move result to r3 */
+ PPC_MR(3, b2p[BPF_REG_0]);
PPC_BLR();
}
+static void bpf_jit_emit_func_call(u32 *image, struct codegen_context *ctx, u64 func)
+{
+#ifdef PPC64_ELF_ABI_v1
+ /* func points to the function descriptor */
+ PPC_LI64(b2p[TMP_REG_2], func);
+ /* Load actual entry point from function descriptor */
+ PPC_BPF_LL(b2p[TMP_REG_1], b2p[TMP_REG_2], 0);
+ /* ... and move it to LR */
+ PPC_MTLR(b2p[TMP_REG_1]);
+ /*
+ * Load TOC from function descriptor at offset 8.
+ * We can clobber r2 since we get called through a
+ * function pointer (so caller will save/restore r2)
+ * and since we don't use a TOC ourself.
+ */
+ PPC_BPF_LL(2, b2p[TMP_REG_2], 8);
+#else
+ /* We can clobber r12 */
+ PPC_FUNC_ADDR(12, func);
+ PPC_MTLR(12);
+#endif
+ PPC_BLRL();
+}
+
+static void bpf_jit_emit_tail_call(u32 *image, struct codegen_context *ctx, u32 out)
+{
+ /*
+ * By now, the eBPF program has already setup parameters in r3, r4 and r5
+ * r3/BPF_REG_1 - pointer to ctx -- passed as is to the next bpf program
+ * r4/BPF_REG_2 - pointer to bpf_array
+ * r5/BPF_REG_3 - index in bpf_array
+ */
+ int b2p_bpf_array = b2p[BPF_REG_2];
+ int b2p_index = b2p[BPF_REG_3];
+
+ /*
+ * if (index >= array->map.max_entries)
+ * goto out;
+ */
+ PPC_LWZ(b2p[TMP_REG_1], b2p_bpf_array, offsetof(struct bpf_array, map.max_entries));
+ PPC_CMPLW(b2p_index, b2p[TMP_REG_1]);
+ PPC_BCC(COND_GE, out);
+
+ /*
+ * if (tail_call_cnt > MAX_TAIL_CALL_CNT)
+ * goto out;
+ */
+ PPC_LD(b2p[TMP_REG_1], 1, bpf_jit_stack_tailcallcnt(ctx));
+ PPC_CMPLWI(b2p[TMP_REG_1], MAX_TAIL_CALL_CNT);
+ PPC_BCC(COND_GT, out);
+
+ /*
+ * tail_call_cnt++;
+ */
+ PPC_ADDI(b2p[TMP_REG_1], b2p[TMP_REG_1], 1);
+ PPC_BPF_STL(b2p[TMP_REG_1], 1, bpf_jit_stack_tailcallcnt(ctx));
+
+ /* prog = array->ptrs[index]; */
+ PPC_MULI(b2p[TMP_REG_1], b2p_index, 8);
+ PPC_ADD(b2p[TMP_REG_1], b2p[TMP_REG_1], b2p_bpf_array);
+ PPC_LD(b2p[TMP_REG_1], b2p[TMP_REG_1], offsetof(struct bpf_array, ptrs));
+
+ /*
+ * if (prog == NULL)
+ * goto out;
+ */
+ PPC_CMPLDI(b2p[TMP_REG_1], 0);
+ PPC_BCC(COND_EQ, out);
+
+ /* goto *(prog->bpf_func + prologue_size); */
+ PPC_LD(b2p[TMP_REG_1], b2p[TMP_REG_1], offsetof(struct bpf_prog, bpf_func));
+#ifdef PPC64_ELF_ABI_v1
+ /* skip past the function descriptor */
+ PPC_ADDI(b2p[TMP_REG_1], b2p[TMP_REG_1],
+ FUNCTION_DESCR_SIZE + BPF_TAILCALL_PROLOGUE_SIZE);
+#else
+ PPC_ADDI(b2p[TMP_REG_1], b2p[TMP_REG_1], BPF_TAILCALL_PROLOGUE_SIZE);
+#endif
+ PPC_MTCTR(b2p[TMP_REG_1]);
+
+ /* tear down stack, restore NVRs, ... */
+ bpf_jit_emit_common_epilogue(image, ctx);
+
+ PPC_BCTR();
+ /* out: */
+}
+
/* Assemble the body code between the prologue & epilogue */
static int bpf_jit_build_body(struct bpf_prog *fp, u32 *image,
struct codegen_context *ctx,
u64 imm64;
u8 *func;
u32 true_cond;
- int stack_local_off;
/*
* addrs[] maps a BPF bytecode address into a real offset from
* optimization but everything else should work without
* any issues.
*/
- if (dst_reg >= 24 && dst_reg <= 31)
+ if (dst_reg >= BPF_PPC_NVR_MIN && dst_reg < 32)
bpf_set_seen_register(ctx, insn[i].dst_reg);
- if (src_reg >= 24 && src_reg <= 31)
+ if (src_reg >= BPF_PPC_NVR_MIN && src_reg < 32)
bpf_set_seen_register(ctx, insn[i].src_reg);
switch (code) {
* Way easier and faster(?) to store the value
* into stack and then use ldbrx
*
- * First, determine where in stack we can store
- * this:
- * - if we have allotted a stack frame, then we
- * will utilize the area set aside by
- * BPF_PPC_STACK_LOCALS
- * - else, we use the area beneath the NV GPR
- * save area
- *
* ctx->seen will be reliable in pass2, but
* the instructions generated will remain the
* same across all passes
*/
- if (bpf_has_stack_frame(ctx))
- stack_local_off = STACK_FRAME_MIN_SIZE;
- else
- stack_local_off = -(BPF_PPC_STACK_SAVE + 8);
-
- PPC_STD(dst_reg, 1, stack_local_off);
- PPC_ADDI(b2p[TMP_REG_1], 1, stack_local_off);
+ PPC_STD(dst_reg, 1, bpf_jit_stack_local(ctx));
+ PPC_ADDI(b2p[TMP_REG_1], 1, bpf_jit_stack_local(ctx));
PPC_LDBRX(dst_reg, 0, b2p[TMP_REG_1]);
break;
}
/* Save skb pointer if we need to re-cache skb data */
if (bpf_helper_changes_skb_data(func))
- PPC_BPF_STL(3, 1, STACK_FRAME_MIN_SIZE);
+ PPC_BPF_STL(3, 1, bpf_jit_stack_local(ctx));
bpf_jit_emit_func_call(image, ctx, (u64)func);
/* refresh skb cache */
if (bpf_helper_changes_skb_data(func)) {
/* reload skb pointer to r3 */
- PPC_BPF_LL(3, 1, STACK_FRAME_MIN_SIZE);
+ PPC_BPF_LL(3, 1, bpf_jit_stack_local(ctx));
bpf_jit_emit_skb_loads(image, ctx);
}
break;
break;
/*
- * TODO: Tail call
+ * Tail call
*/
case BPF_JMP | BPF_CALL | BPF_X:
+ ctx->seen |= SEEN_TAILCALL;
+ bpf_jit_emit_tail_call(image, ctx, addrs[i + 1]);
+ break;
default:
/*
int pass;
int flen;
struct bpf_binary_header *bpf_hdr;
+ struct bpf_prog *org_fp = fp;
+ struct bpf_prog *tmp_fp;
+ bool bpf_blinded = false;
if (!bpf_jit_enable)
- return fp;
+ return org_fp;
+
+ tmp_fp = bpf_jit_blind_constants(org_fp);
+ if (IS_ERR(tmp_fp))
+ return org_fp;
+
+ if (tmp_fp != org_fp) {
+ bpf_blinded = true;
+ fp = tmp_fp;
+ }
flen = fp->len;
addrs = kzalloc((flen+1) * sizeof(*addrs), GFP_KERNEL);
- if (addrs == NULL)
- return fp;
+ if (addrs == NULL) {
+ fp = org_fp;
+ goto out;
+ }
+
+ memset(&cgctx, 0, sizeof(struct codegen_context));
- cgctx.idx = 0;
- cgctx.seen = 0;
/* Scouting faux-generate pass 0 */
- if (bpf_jit_build_body(fp, 0, &cgctx, addrs))
+ if (bpf_jit_build_body(fp, 0, &cgctx, addrs)) {
/* We hit something illegal or unsupported. */
+ fp = org_fp;
goto out;
+ }
/*
* Pretend to build prologue, given the features we've seen. This will
bpf_hdr = bpf_jit_binary_alloc(alloclen, &image, 4,
bpf_jit_fill_ill_insns);
- if (!bpf_hdr)
+ if (!bpf_hdr) {
+ fp = org_fp;
goto out;
+ }
code_base = (u32 *)(image + FUNCTION_DESCR_SIZE);
out:
kfree(addrs);
+
+ if (bpf_blinded)
+ bpf_jit_prog_release_other(fp, fp == org_fp ? tmp_fp : org_fp);
+
return fp;
}
select HAVE_ARCH_TRANSPARENT_HUGEPAGE
select ARCH_SUPPORTS_NUMA_BALANCING
select IRQ_WORK
+ select HAVE_KERNEL_XZ
config PPC_BOOK3E_64
bool "Embedded processors"
choice
prompt "CPU selection"
depends on PPC64
+ default POWER8_CPU if CPU_LITTLE_ENDIAN
default GENERIC_CPU
help
This will create a kernel which is optimised for a particular CPU.
* reset followed by hot reset on root bus. So we also
* need the PCI bus settlement delay.
*/
- rc = pnv_eeh_poll(phb->opal_id);
+ if (rc > 0)
+ rc = pnv_eeh_poll(phb->opal_id);
if (option == EEH_RESET_DEACTIVATE) {
if (system_state < SYSTEM_RUNNING)
udelay(1000 * EEH_PE_RST_SETTLE_TIME);
goto out;
/* Poll state of the PHB until the request is done */
- rc = pnv_eeh_poll(phb->opal_id);
+ if (rc > 0)
+ rc = pnv_eeh_poll(phb->opal_id);
if (option == EEH_RESET_DEACTIVATE)
msleep(EEH_PE_RST_SETTLE_TIME);
out:
return;
}
- switch (data->type) {
+ switch (be16_to_cpu(data->type)) {
case OPAL_P7IOC_DIAG_TYPE_RGC:
pr_info("P7IOC diag-data for RGC\n\n");
pnv_eeh_dump_hub_diag_common(data);
/* Try best to clear it */
opal_pci_eeh_freeze_clear(phb->opal_id,
- frozen_pe_no,
+ be64_to_cpu(frozen_pe_no),
OPAL_EEH_ACTION_CLEAR_FREEZE_ALL);
ret = EEH_NEXT_ERR_NONE;
} else if ((*pe)->state & EEH_PE_ISOLATED ||
static struct pnv_ioda_pe *pnv_ioda_init_pe(struct pnv_phb *phb, int pe_no)
{
+ s64 rc;
+
phb->ioda.pe_array[pe_no].phb = phb;
phb->ioda.pe_array[pe_no].pe_number = pe_no;
+ /*
+ * Clear the PE frozen state as it might be put into frozen state
+ * in the last PCI remove path. It's not harmful to do so when the
+ * PE is already in unfrozen state.
+ */
+ rc = opal_pci_eeh_freeze_clear(phb->opal_id, pe_no,
+ OPAL_EEH_ACTION_CLEAR_FREEZE_ALL);
+ if (rc != OPAL_SUCCESS)
+ pr_warn("%s: Error %lld unfreezing PHB#%d-PE#%d\n",
+ __func__, rc, phb->hose->global_number, pe_no);
+
return &phb->ioda.pe_array[pe_no];
}
struct device_node *dn = hose->dn;
struct resource *res;
u32 m64_range[2], i;
- const u32 *r;
+ const __be32 *r;
u64 pci_addr;
if (phb->type != PNV_PHB_IODA1 && phb->type != PNV_PHB_IODA2) {
be64_to_cpu(data->dma1ErrorLog1));
for (i = 0; i < OPAL_P7IOC_NUM_PEST_REGS; i++) {
- if ((data->pestA[i] >> 63) == 0 &&
- (data->pestB[i] >> 63) == 0)
+ if ((be64_to_cpu(data->pestA[i]) >> 63) == 0 &&
+ (be64_to_cpu(data->pestB[i]) >> 63) == 0)
continue;
pr_info("PE[%3d] A/B: %016llx %016llx\n",
#define CXL_PSL_SPAP_V 0x0000000000000001ULL
/****** CXL_PSL_Control ****************************************************/
-#define CXL_PSL_Control_tb 0x0000000000000001ULL
+#define CXL_PSL_Control_tb (0x1ull << (63-63))
+#define CXL_PSL_Control_Fr (0x1ull << (63-31))
+#define CXL_PSL_Control_Fs_MASK (0x3ull << (63-29))
+#define CXL_PSL_Control_Fs_Complete (0x3ull << (63-29))
/****** CXL_PSL_DLCNTL *****************************************************/
#define CXL_PSL_DLCNTL_D (0x1ull << (63-28))
int cxl_check_error(struct cxl_afu *afu);
int cxl_afu_slbia(struct cxl_afu *afu);
int cxl_tlb_slb_invalidate(struct cxl *adapter);
+int cxl_data_cache_flush(struct cxl *adapter);
int cxl_afu_disable(struct cxl_afu *afu);
int cxl_psl_purge(struct cxl_afu *afu);
return 0;
}
+int cxl_data_cache_flush(struct cxl *adapter)
+{
+ u64 reg;
+ unsigned long timeout = jiffies + (HZ * CXL_TIMEOUT);
+
+ pr_devel("Flushing data cache\n");
+
+ reg = cxl_p1_read(adapter, CXL_PSL_Control);
+ reg |= CXL_PSL_Control_Fr;
+ cxl_p1_write(adapter, CXL_PSL_Control, reg);
+
+ reg = cxl_p1_read(adapter, CXL_PSL_Control);
+ while ((reg & CXL_PSL_Control_Fs_MASK) != CXL_PSL_Control_Fs_Complete) {
+ if (time_after_eq(jiffies, timeout)) {
+ dev_warn(&adapter->dev, "WARNING: cache flush timed out!\n");
+ return -EBUSY;
+ }
+
+ if (!cxl_ops->link_ok(adapter, NULL)) {
+ dev_warn(&adapter->dev, "WARNING: link down when flushing cache\n");
+ return -EIO;
+ }
+ cpu_relax();
+ reg = cxl_p1_read(adapter, CXL_PSL_Control);
+ }
+
+ reg &= ~CXL_PSL_Control_Fr;
+ cxl_p1_write(adapter, CXL_PSL_Control, reg);
+ return 0;
+}
+
static int cxl_write_sstp(struct cxl_afu *afu, u64 sstp0, u64 sstp1)
{
int rc;
struct device_node *afu_np = NULL;
struct cxl *adapter = NULL;
int ret;
- int slice, slice_ok;
+ int slice = 0, slice_ok = 0;
pr_devel("in %s\n", __func__);
}
/* init afu */
- slice_ok = 0;
- for (afu_np = NULL, slice = 0; (afu_np = of_get_next_child(np, afu_np)); slice++) {
+ for_each_child_of_node(np, afu_np) {
if ((ret = cxl_guest_init_afu(adapter, slice, afu_np)))
dev_err(&pdev->dev, "AFU %i failed to initialise: %i\n",
slice, ret);
else
slice_ok++;
+ slice++;
}
if (slice_ok == 0) {
adapter->slices = 0;
}
- if (afu_np)
- of_node_put(afu_np);
return 0;
}
dev_info(&dev->dev, "CXL reset\n");
+ /* the adapter is about to be reset, so ignore errors */
+ cxl_data_cache_flush(adapter);
+
/* pcie_warm_reset requests a fundamental pci reset which includes a
* PERST assert/deassert. PERST triggers a loading of the image
* if "user" or "factory" is selected in sysfs */
#define DRIVER_AUTHOR "Gavin Shan, IBM Corporation"
#define DRIVER_DESC "PowerPC PowerNV PCI Hotplug Driver"
+struct pnv_php_event {
+ bool added;
+ struct pnv_php_slot *php_slot;
+ struct work_struct work;
+};
+
static LIST_HEAD(pnv_php_slot_list);
static DEFINE_SPINLOCK(pnv_php_lock);
static void pnv_php_unregister_one(struct device_node *dn);
static void pnv_php_unregister(struct device_node *dn);
+static void pnv_php_disable_irq(struct pnv_php_slot *php_slot)
+{
+ struct pci_dev *pdev = php_slot->pdev;
+ u16 ctrl;
+
+ if (php_slot->irq > 0) {
+ pcie_capability_read_word(pdev, PCI_EXP_SLTCTL, &ctrl);
+ ctrl &= ~(PCI_EXP_SLTCTL_HPIE |
+ PCI_EXP_SLTCTL_PDCE |
+ PCI_EXP_SLTCTL_DLLSCE);
+ pcie_capability_write_word(pdev, PCI_EXP_SLTCTL, ctrl);
+
+ free_irq(php_slot->irq, php_slot);
+ php_slot->irq = 0;
+ }
+
+ if (php_slot->wq) {
+ destroy_workqueue(php_slot->wq);
+ php_slot->wq = NULL;
+ }
+
+ if (pdev->msix_enabled)
+ pci_disable_msix(pdev);
+ else if (pdev->msi_enabled)
+ pci_disable_msi(pdev);
+}
+
static void pnv_php_free_slot(struct kref *kref)
{
struct pnv_php_slot *php_slot = container_of(kref,
struct pnv_php_slot, kref);
WARN_ON(!list_empty(&php_slot->children));
+ pnv_php_disable_irq(php_slot);
kfree(php_slot->name);
kfree(php_slot);
}
of_node_put(dn);
refcount = atomic_read(&dn->kobj.kref.refcount);
- if (unlikely(refcount != 1))
+ if (refcount != 1)
pr_warn("Invalid refcount %d on <%s>\n",
refcount, of_node_full_name(dn));
for_each_child_of_node(dn, child) {
ret = of_changeset_attach_node(ocs, child);
- if (unlikely(ret))
+ if (ret)
break;
ret = pnv_php_populate_changeset(ocs, child);
- if (unlikely(ret))
+ if (ret)
break;
}
struct pci_dn *pdn;
pdn = pci_add_device_node_info(hose, dn);
- if (unlikely(!pdn))
+ if (!pdn)
return ERR_PTR(-ENOMEM);
return NULL;
* fits the real size.
*/
fdt1 = kzalloc(0x10000, GFP_KERNEL);
- if (unlikely(!fdt1)) {
+ if (!fdt1) {
ret = -ENOMEM;
dev_warn(&php_slot->pdev->dev, "Cannot alloc FDT blob\n");
goto out;
}
ret = pnv_pci_get_device_tree(php_slot->dn->phandle, fdt1, 0x10000);
- if (unlikely(ret)) {
+ if (ret) {
dev_warn(&php_slot->pdev->dev, "Error %d getting FDT blob\n",
ret);
goto free_fdt1;
}
fdt = kzalloc(fdt_totalsize(fdt1), GFP_KERNEL);
- if (unlikely(!fdt)) {
+ if (!fdt) {
ret = -ENOMEM;
dev_warn(&php_slot->pdev->dev, "Cannot %d bytes memory\n",
fdt_totalsize(fdt1));
/* Unflatten device tree blob */
memcpy(fdt, fdt1, fdt_totalsize(fdt1));
dt = of_fdt_unflatten_tree(fdt, php_slot->dn, NULL);
- if (unlikely(!dt)) {
+ if (!dt) {
ret = -EINVAL;
dev_warn(&php_slot->pdev->dev, "Cannot unflatten FDT\n");
goto free_fdt;
of_changeset_init(&php_slot->ocs);
pnv_php_reverse_nodes(php_slot->dn);
ret = pnv_php_populate_changeset(&php_slot->ocs, php_slot->dn);
- if (unlikely(ret)) {
+ if (ret) {
pnv_php_reverse_nodes(php_slot->dn);
dev_warn(&php_slot->pdev->dev, "Error %d populating changeset\n",
ret);
php_slot->dn->child = NULL;
ret = of_changeset_apply(&php_slot->ocs);
- if (unlikely(ret)) {
+ if (ret) {
dev_warn(&php_slot->pdev->dev, "Error %d applying changeset\n",
ret);
goto destroy_changeset;
int ret;
ret = pnv_pci_set_power_state(php_slot->id, state, &msg);
- if (likely(ret > 0)) {
+ if (ret > 0) {
if (be64_to_cpu(msg.params[1]) != php_slot->dn->phandle ||
be64_to_cpu(msg.params[2]) != state ||
be64_to_cpu(msg.params[3]) != OPAL_SUCCESS) {
be64_to_cpu(msg.params[3]));
return -ENOMSG;
}
- } else if (unlikely(ret < 0)) {
+ } else if (ret < 0) {
dev_warn(&php_slot->pdev->dev, "Error %d powering %s\n",
ret, (state == OPAL_PCI_SLOT_POWER_ON) ? "on" : "off");
return ret;
* be on.
*/
ret = pnv_pci_get_power_state(php_slot->id, &power_state);
- if (unlikely(ret)) {
+ if (ret) {
dev_warn(&php_slot->pdev->dev, "Error %d getting power status\n",
ret);
} else {
* get that, it will fail back to be empty.
*/
ret = pnv_pci_get_presence_state(php_slot->id, &presence);
- if (likely(ret >= 0)) {
+ if (ret >= 0) {
*state = presence;
slot->info->adapter_status = presence;
ret = 0;
/* Retrieve slot presence status */
ret = pnv_php_get_adapter_state(slot, &presence);
- if (unlikely(ret))
+ if (ret)
return ret;
/* Proceed if there have nothing behind the slot */
php_slot->power_state_check = true;
ret = pnv_php_get_power_state(slot, &power_status);
- if (unlikely(ret))
+ if (ret)
return ret;
if (power_status != OPAL_PCI_SLOT_POWER_ON)
/* Check the power status. Scan the slot if it is already on */
ret = pnv_php_get_power_state(slot, &power_status);
- if (unlikely(ret))
+ if (ret)
return ret;
if (power_status == OPAL_PCI_SLOT_POWER_ON)
/* Power is off, turn it on and then scan the slot */
ret = pnv_php_set_slot_power_state(slot, OPAL_PCI_SLOT_POWER_ON);
- if (unlikely(ret))
+ if (ret)
return ret;
scan:
struct pci_bus *bus;
const char *label;
uint64_t id;
+ int ret;
- label = of_get_property(dn, "ibm,slot-label", NULL);
- if (unlikely(!label))
+ ret = of_property_read_string(dn, "ibm,slot-label", &label);
+ if (ret)
return NULL;
if (pnv_pci_get_slot_id(dn, &id))
return NULL;
bus = pci_find_bus_by_node(dn);
- if (unlikely(!bus))
+ if (!bus)
return NULL;
php_slot = kzalloc(sizeof(*php_slot), GFP_KERNEL);
- if (unlikely(!php_slot))
+ if (!php_slot)
return NULL;
php_slot->name = kstrdup(label, GFP_KERNEL);
- if (unlikely(!php_slot->name)) {
+ if (!php_slot->name) {
kfree(php_slot);
return NULL;
}
- if (likely(dn->child && PCI_DN(dn->child)))
+ if (dn->child && PCI_DN(dn->child))
php_slot->slot_no = PCI_SLOT(PCI_DN(dn->child)->devfn);
else
php_slot->slot_no = -1; /* Placeholder slot */
/* Check if the slot is registered or not */
parent = pnv_php_find_slot(php_slot->dn);
- if (unlikely(parent)) {
+ if (parent) {
pnv_php_put_slot(parent);
return -EEXIST;
}
/* Register PCI slot */
ret = pci_hp_register(&php_slot->slot, php_slot->bus,
php_slot->slot_no, php_slot->name);
- if (unlikely(ret)) {
+ if (ret) {
dev_warn(&php_slot->pdev->dev, "Error %d registering slot\n",
ret);
return ret;
return 0;
}
+static int pnv_php_enable_msix(struct pnv_php_slot *php_slot)
+{
+ struct pci_dev *pdev = php_slot->pdev;
+ struct msix_entry entry;
+ int nr_entries, ret;
+ u16 pcie_flag;
+
+ /* Get total number of MSIx entries */
+ nr_entries = pci_msix_vec_count(pdev);
+ if (nr_entries < 0)
+ return nr_entries;
+
+ /* Check hotplug MSIx entry is in range */
+ pcie_capability_read_word(pdev, PCI_EXP_FLAGS, &pcie_flag);
+ entry.entry = (pcie_flag & PCI_EXP_FLAGS_IRQ) >> 9;
+ if (entry.entry >= nr_entries)
+ return -ERANGE;
+
+ /* Enable MSIx */
+ ret = pci_enable_msix_exact(pdev, &entry, 1);
+ if (ret) {
+ dev_warn(&pdev->dev, "Error %d enabling MSIx\n", ret);
+ return ret;
+ }
+
+ return entry.vector;
+}
+
+static void pnv_php_event_handler(struct work_struct *work)
+{
+ struct pnv_php_event *event =
+ container_of(work, struct pnv_php_event, work);
+ struct pnv_php_slot *php_slot = event->php_slot;
+
+ if (event->added)
+ pnv_php_enable_slot(&php_slot->slot);
+ else
+ pnv_php_disable_slot(&php_slot->slot);
+
+ kfree(event);
+}
+
+static irqreturn_t pnv_php_interrupt(int irq, void *data)
+{
+ struct pnv_php_slot *php_slot = data;
+ struct pci_dev *pchild, *pdev = php_slot->pdev;
+ struct eeh_dev *edev;
+ struct eeh_pe *pe;
+ struct pnv_php_event *event;
+ u16 sts, lsts;
+ u8 presence;
+ bool added;
+ unsigned long flags;
+ int ret;
+
+ pcie_capability_read_word(pdev, PCI_EXP_SLTSTA, &sts);
+ sts &= (PCI_EXP_SLTSTA_PDC | PCI_EXP_SLTSTA_DLLSC);
+ pcie_capability_write_word(pdev, PCI_EXP_SLTSTA, sts);
+ if (sts & PCI_EXP_SLTSTA_DLLSC) {
+ pcie_capability_read_word(pdev, PCI_EXP_LNKSTA, &lsts);
+ added = !!(lsts & PCI_EXP_LNKSTA_DLLLA);
+ } else if (sts & PCI_EXP_SLTSTA_PDC) {
+ ret = pnv_pci_get_presence_state(php_slot->id, &presence);
+ if (!ret)
+ return IRQ_HANDLED;
+ added = !!(presence == OPAL_PCI_SLOT_PRESENT);
+ } else {
+ return IRQ_NONE;
+ }
+
+ /* Freeze the removed PE to avoid unexpected error reporting */
+ if (!added) {
+ pchild = list_first_entry_or_null(&php_slot->bus->devices,
+ struct pci_dev, bus_list);
+ edev = pchild ? pci_dev_to_eeh_dev(pchild) : NULL;
+ pe = edev ? edev->pe : NULL;
+ if (pe) {
+ eeh_serialize_lock(&flags);
+ eeh_pe_state_mark(pe, EEH_PE_ISOLATED);
+ eeh_serialize_unlock(flags);
+ eeh_pe_set_option(pe, EEH_OPT_FREEZE_PE);
+ }
+ }
+
+ /*
+ * The PE is left in frozen state if the event is missed. It's
+ * fine as the PCI devices (PE) aren't functional any more.
+ */
+ event = kzalloc(sizeof(*event), GFP_ATOMIC);
+ if (!event) {
+ dev_warn(&pdev->dev, "PCI slot [%s] missed hotplug event 0x%04x\n",
+ php_slot->name, sts);
+ return IRQ_HANDLED;
+ }
+
+ dev_info(&pdev->dev, "PCI slot [%s] %s (IRQ: %d)\n",
+ php_slot->name, added ? "added" : "removed", irq);
+ INIT_WORK(&event->work, pnv_php_event_handler);
+ event->added = added;
+ event->php_slot = php_slot;
+ queue_work(php_slot->wq, &event->work);
+
+ return IRQ_HANDLED;
+}
+
+static void pnv_php_init_irq(struct pnv_php_slot *php_slot, int irq)
+{
+ struct pci_dev *pdev = php_slot->pdev;
+ u16 sts, ctrl;
+ int ret;
+
+ /* Allocate workqueue */
+ php_slot->wq = alloc_workqueue("pciehp-%s", 0, 0, php_slot->name);
+ if (!php_slot->wq) {
+ dev_warn(&pdev->dev, "Cannot alloc workqueue\n");
+ pnv_php_disable_irq(php_slot);
+ return;
+ }
+
+ /* Clear pending interrupts */
+ pcie_capability_read_word(pdev, PCI_EXP_SLTSTA, &sts);
+ sts |= (PCI_EXP_SLTSTA_PDC | PCI_EXP_SLTSTA_DLLSC);
+ pcie_capability_write_word(pdev, PCI_EXP_SLTSTA, sts);
+
+ /* Request the interrupt */
+ ret = request_irq(irq, pnv_php_interrupt, IRQF_SHARED,
+ php_slot->name, php_slot);
+ if (ret) {
+ pnv_php_disable_irq(php_slot);
+ dev_warn(&pdev->dev, "Error %d enabling IRQ %d\n", ret, irq);
+ return;
+ }
+
+ /* Enable the interrupts */
+ pcie_capability_read_word(pdev, PCI_EXP_SLTCTL, &ctrl);
+ ctrl |= (PCI_EXP_SLTCTL_HPIE |
+ PCI_EXP_SLTCTL_PDCE |
+ PCI_EXP_SLTCTL_DLLSCE);
+ pcie_capability_write_word(pdev, PCI_EXP_SLTCTL, ctrl);
+
+ /* The interrupt is initialized successfully when @irq is valid */
+ php_slot->irq = irq;
+}
+
+static void pnv_php_enable_irq(struct pnv_php_slot *php_slot)
+{
+ struct pci_dev *pdev = php_slot->pdev;
+ int irq, ret;
+
+ ret = pci_enable_device(pdev);
+ if (ret) {
+ dev_warn(&pdev->dev, "Error %d enabling device\n", ret);
+ return;
+ }
+
+ pci_set_master(pdev);
+
+ /* Enable MSIx interrupt */
+ irq = pnv_php_enable_msix(php_slot);
+ if (irq > 0) {
+ pnv_php_init_irq(php_slot, irq);
+ return;
+ }
+
+ /*
+ * Use MSI if MSIx doesn't work. Fail back to legacy INTx
+ * if MSI doesn't work either
+ */
+ ret = pci_enable_msi(pdev);
+ if (!ret || pdev->irq) {
+ irq = pdev->irq;
+ pnv_php_init_irq(php_slot, irq);
+ }
+}
+
static int pnv_php_register_one(struct device_node *dn)
{
struct pnv_php_slot *php_slot;
- const __be32 *prop32;
+ u32 prop32;
int ret;
/* Check if it's hotpluggable slot */
- prop32 = of_get_property(dn, "ibm,slot-pluggable", NULL);
- if (!prop32 || !of_read_number(prop32, 1))
+ ret = of_property_read_u32(dn, "ibm,slot-pluggable", &prop32);
+ if (ret || !prop32)
return -ENXIO;
- prop32 = of_get_property(dn, "ibm,reset-by-firmware", NULL);
- if (!prop32 || !of_read_number(prop32, 1))
+ ret = of_property_read_u32(dn, "ibm,reset-by-firmware", &prop32);
+ if (ret || !prop32)
return -ENXIO;
php_slot = pnv_php_alloc_slot(dn);
- if (unlikely(!php_slot))
+ if (!php_slot)
return -ENODEV;
ret = pnv_php_register_slot(php_slot);
- if (unlikely(ret))
+ if (ret)
goto free_slot;
ret = pnv_php_enable(php_slot, false);
- if (unlikely(ret))
+ if (ret)
goto unregister_slot;
+ /* Enable interrupt if the slot supports surprise hotplug */
+ ret = of_property_read_u32(dn, "ibm,slot-surprise-pluggable", &prop32);
+ if (!ret && prop32)
+ pnv_php_enable_irq(php_slot);
+
return 0;
unregister_slot:
dscr \
mm \
pmu \
+ signal \
primitives \
stringloops \
switch_endian \
--- /dev/null
+#define EXPORT_SYMBOL(x)
--- /dev/null
+/*
+ * Copyright 2016, Cyril Bur, IBM Corp.
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License
+ * as published by the Free Software Foundation; either version
+ * 2 of the License, or (at your option) any later version.
+ */
+
+#ifndef _SELFTESTS_POWERPC_FPU_ASM_H
+#define _SELFTESTS_POWERPC_FPU_ASM_H
+#include "basic_asm.h"
+
+#define PUSH_FPU(stack_size) \
+ stfd f31,(stack_size + STACK_FRAME_MIN_SIZE)(%r1); \
+ stfd f30,(stack_size + STACK_FRAME_MIN_SIZE - 8)(%r1); \
+ stfd f29,(stack_size + STACK_FRAME_MIN_SIZE - 16)(%r1); \
+ stfd f28,(stack_size + STACK_FRAME_MIN_SIZE - 24)(%r1); \
+ stfd f27,(stack_size + STACK_FRAME_MIN_SIZE - 32)(%r1); \
+ stfd f26,(stack_size + STACK_FRAME_MIN_SIZE - 40)(%r1); \
+ stfd f25,(stack_size + STACK_FRAME_MIN_SIZE - 48)(%r1); \
+ stfd f24,(stack_size + STACK_FRAME_MIN_SIZE - 56)(%r1); \
+ stfd f23,(stack_size + STACK_FRAME_MIN_SIZE - 64)(%r1); \
+ stfd f22,(stack_size + STACK_FRAME_MIN_SIZE - 72)(%r1); \
+ stfd f21,(stack_size + STACK_FRAME_MIN_SIZE - 80)(%r1); \
+ stfd f20,(stack_size + STACK_FRAME_MIN_SIZE - 88)(%r1); \
+ stfd f19,(stack_size + STACK_FRAME_MIN_SIZE - 96)(%r1); \
+ stfd f18,(stack_size + STACK_FRAME_MIN_SIZE - 104)(%r1); \
+ stfd f17,(stack_size + STACK_FRAME_MIN_SIZE - 112)(%r1); \
+ stfd f16,(stack_size + STACK_FRAME_MIN_SIZE - 120)(%r1); \
+ stfd f15,(stack_size + STACK_FRAME_MIN_SIZE - 128)(%r1); \
+ stfd f14,(stack_size + STACK_FRAME_MIN_SIZE - 136)(%r1);
+
+#define POP_FPU(stack_size) \
+ lfd f31,(stack_size + STACK_FRAME_MIN_SIZE)(%r1); \
+ lfd f30,(stack_size + STACK_FRAME_MIN_SIZE - 8)(%r1); \
+ lfd f29,(stack_size + STACK_FRAME_MIN_SIZE - 16)(%r1); \
+ lfd f28,(stack_size + STACK_FRAME_MIN_SIZE - 24)(%r1); \
+ lfd f27,(stack_size + STACK_FRAME_MIN_SIZE - 32)(%r1); \
+ lfd f26,(stack_size + STACK_FRAME_MIN_SIZE - 40)(%r1); \
+ lfd f25,(stack_size + STACK_FRAME_MIN_SIZE - 48)(%r1); \
+ lfd f24,(stack_size + STACK_FRAME_MIN_SIZE - 56)(%r1); \
+ lfd f23,(stack_size + STACK_FRAME_MIN_SIZE - 64)(%r1); \
+ lfd f22,(stack_size + STACK_FRAME_MIN_SIZE - 72)(%r1); \
+ lfd f21,(stack_size + STACK_FRAME_MIN_SIZE - 80)(%r1); \
+ lfd f20,(stack_size + STACK_FRAME_MIN_SIZE - 88)(%r1); \
+ lfd f19,(stack_size + STACK_FRAME_MIN_SIZE - 96)(%r1); \
+ lfd f18,(stack_size + STACK_FRAME_MIN_SIZE - 104)(%r1); \
+ lfd f17,(stack_size + STACK_FRAME_MIN_SIZE - 112)(%r1); \
+ lfd f16,(stack_size + STACK_FRAME_MIN_SIZE - 120)(%r1); \
+ lfd f15,(stack_size + STACK_FRAME_MIN_SIZE - 128)(%r1); \
+ lfd f14,(stack_size + STACK_FRAME_MIN_SIZE - 136)(%r1);
+
+/*
+ * Careful calling this, it will 'clobber' fpu (by design)
+ * Don't call this from C
+ */
+FUNC_START(load_fpu)
+ lfd f14,0(r3)
+ lfd f15,8(r3)
+ lfd f16,16(r3)
+ lfd f17,24(r3)
+ lfd f18,32(r3)
+ lfd f19,40(r3)
+ lfd f20,48(r3)
+ lfd f21,56(r3)
+ lfd f22,64(r3)
+ lfd f23,72(r3)
+ lfd f24,80(r3)
+ lfd f25,88(r3)
+ lfd f26,96(r3)
+ lfd f27,104(r3)
+ lfd f28,112(r3)
+ lfd f29,120(r3)
+ lfd f30,128(r3)
+ lfd f31,136(r3)
+ blr
+FUNC_END(load_fpu)
+
+#endif /* _SELFTESTS_POWERPC_FPU_ASM_H */
--- /dev/null
+/*
+ * Copyright 2016, Cyril Bur, IBM Corp.
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License
+ * as published by the Free Software Foundation; either version
+ * 2 of the License, or (at your option) any later version.
+ */
+
+#ifndef _SELFTESTS_POWERPC_GPR_ASM_H
+#define _SELFTESTS_POWERPC_GPR_ASM_H
+
+#include "basic_asm.h"
+
+#define __PUSH_NVREGS(top_pos); \
+ std r31,(top_pos)(%r1); \
+ std r30,(top_pos - 8)(%r1); \
+ std r29,(top_pos - 16)(%r1); \
+ std r28,(top_pos - 24)(%r1); \
+ std r27,(top_pos - 32)(%r1); \
+ std r26,(top_pos - 40)(%r1); \
+ std r25,(top_pos - 48)(%r1); \
+ std r24,(top_pos - 56)(%r1); \
+ std r23,(top_pos - 64)(%r1); \
+ std r22,(top_pos - 72)(%r1); \
+ std r21,(top_pos - 80)(%r1); \
+ std r20,(top_pos - 88)(%r1); \
+ std r19,(top_pos - 96)(%r1); \
+ std r18,(top_pos - 104)(%r1); \
+ std r17,(top_pos - 112)(%r1); \
+ std r16,(top_pos - 120)(%r1); \
+ std r15,(top_pos - 128)(%r1); \
+ std r14,(top_pos - 136)(%r1)
+
+#define __POP_NVREGS(top_pos); \
+ ld r31,(top_pos)(%r1); \
+ ld r30,(top_pos - 8)(%r1); \
+ ld r29,(top_pos - 16)(%r1); \
+ ld r28,(top_pos - 24)(%r1); \
+ ld r27,(top_pos - 32)(%r1); \
+ ld r26,(top_pos - 40)(%r1); \
+ ld r25,(top_pos - 48)(%r1); \
+ ld r24,(top_pos - 56)(%r1); \
+ ld r23,(top_pos - 64)(%r1); \
+ ld r22,(top_pos - 72)(%r1); \
+ ld r21,(top_pos - 80)(%r1); \
+ ld r20,(top_pos - 88)(%r1); \
+ ld r19,(top_pos - 96)(%r1); \
+ ld r18,(top_pos - 104)(%r1); \
+ ld r17,(top_pos - 112)(%r1); \
+ ld r16,(top_pos - 120)(%r1); \
+ ld r15,(top_pos - 128)(%r1); \
+ ld r14,(top_pos - 136)(%r1)
+
+#define PUSH_NVREGS(stack_size) \
+ __PUSH_NVREGS(stack_size + STACK_FRAME_MIN_SIZE)
+
+/* 18 NV FPU REGS */
+#define PUSH_NVREGS_BELOW_FPU(stack_size) \
+ __PUSH_NVREGS(stack_size + STACK_FRAME_MIN_SIZE - (18 * 8))
+
+#define POP_NVREGS(stack_size) \
+ __POP_NVREGS(stack_size + STACK_FRAME_MIN_SIZE)
+
+/* 18 NV FPU REGS */
+#define POP_NVREGS_BELOW_FPU(stack_size) \
+ __POP_NVREGS(stack_size + STACK_FRAME_MIN_SIZE - (18 * 8))
+
+/*
+ * Careful calling this, it will 'clobber' NVGPRs (by design)
+ * Don't call this from C
+ */
+FUNC_START(load_gpr)
+ ld r14,0(r3)
+ ld r15,8(r3)
+ ld r16,16(r3)
+ ld r17,24(r3)
+ ld r18,32(r3)
+ ld r19,40(r3)
+ ld r20,48(r3)
+ ld r21,56(r3)
+ ld r22,64(r3)
+ ld r23,72(r3)
+ ld r24,80(r3)
+ ld r25,88(r3)
+ ld r26,96(r3)
+ ld r27,104(r3)
+ ld r28,112(r3)
+ ld r29,120(r3)
+ ld r30,128(r3)
+ ld r31,136(r3)
+ blr
+FUNC_END(load_gpr)
+
+
+#endif /* _SELFTESTS_POWERPC_GPR_ASM_H */
#include "subunit.h"
#include "utils.h"
-#define TIMEOUT 120
#define KILL_TIMEOUT 5
+static uint64_t timeout = 120;
int run_test(int (test_function)(void), char *name)
{
setpgid(pid, pid);
/* Wake us up in timeout seconds */
- alarm(TIMEOUT);
+ alarm(timeout);
terminated = false;
wait:
.sa_handler = alarm_handler,
};
+void test_harness_set_timeout(uint64_t time)
+{
+ timeout = time;
+}
+
int test_harness(int (test_function)(void), char *name)
{
int rc;
vmx_preempt
fpu_signal
vmx_signal
+vsx_preempt
-TEST_PROGS := fpu_syscall fpu_preempt fpu_signal vmx_syscall vmx_preempt vmx_signal
+TEST_PROGS := fpu_syscall fpu_preempt fpu_signal vmx_syscall vmx_preempt vmx_signal vsx_preempt
all: $(TEST_PROGS)
vmx_preempt: vmx_asm.S
vmx_signal: vmx_asm.S
+vsx_preempt: CFLAGS += -mvsx
+vsx_preempt: vsx_asm.S
+
include ../../lib.mk
clean:
*/
#include "../basic_asm.h"
-
-#define PUSH_FPU(pos) \
- stfd f14,pos(sp); \
- stfd f15,pos+8(sp); \
- stfd f16,pos+16(sp); \
- stfd f17,pos+24(sp); \
- stfd f18,pos+32(sp); \
- stfd f19,pos+40(sp); \
- stfd f20,pos+48(sp); \
- stfd f21,pos+56(sp); \
- stfd f22,pos+64(sp); \
- stfd f23,pos+72(sp); \
- stfd f24,pos+80(sp); \
- stfd f25,pos+88(sp); \
- stfd f26,pos+96(sp); \
- stfd f27,pos+104(sp); \
- stfd f28,pos+112(sp); \
- stfd f29,pos+120(sp); \
- stfd f30,pos+128(sp); \
- stfd f31,pos+136(sp);
-
-#define POP_FPU(pos) \
- lfd f14,pos(sp); \
- lfd f15,pos+8(sp); \
- lfd f16,pos+16(sp); \
- lfd f17,pos+24(sp); \
- lfd f18,pos+32(sp); \
- lfd f19,pos+40(sp); \
- lfd f20,pos+48(sp); \
- lfd f21,pos+56(sp); \
- lfd f22,pos+64(sp); \
- lfd f23,pos+72(sp); \
- lfd f24,pos+80(sp); \
- lfd f25,pos+88(sp); \
- lfd f26,pos+96(sp); \
- lfd f27,pos+104(sp); \
- lfd f28,pos+112(sp); \
- lfd f29,pos+120(sp); \
- lfd f30,pos+128(sp); \
- lfd f31,pos+136(sp);
-
-# Careful calling this, it will 'clobber' fpu (by design)
-# Don't call this from C
-FUNC_START(load_fpu)
- lfd f14,0(r3)
- lfd f15,8(r3)
- lfd f16,16(r3)
- lfd f17,24(r3)
- lfd f18,32(r3)
- lfd f19,40(r3)
- lfd f20,48(r3)
- lfd f21,56(r3)
- lfd f22,64(r3)
- lfd f23,72(r3)
- lfd f24,80(r3)
- lfd f25,88(r3)
- lfd f26,96(r3)
- lfd f27,104(r3)
- lfd f28,112(r3)
- lfd f29,120(r3)
- lfd f30,128(r3)
- lfd f31,136(r3)
- blr
-FUNC_END(load_fpu)
+#include "../fpu_asm.h"
FUNC_START(check_fpu)
mr r4,r3
# r4 holds pointer to the pid
# f14-f31 are non volatiles
PUSH_BASIC_STACK(256)
+ PUSH_FPU(256)
std r3,STACK_FRAME_PARAM(0)(sp) # Address of darray
std r4,STACK_FRAME_PARAM(1)(sp) # Address of pid
- PUSH_FPU(STACK_FRAME_LOCAL(2,0))
bl load_fpu
nop
bl check_fpu
nop
- POP_FPU(STACK_FRAME_LOCAL(2,0))
+ POP_FPU(256)
POP_BASIC_STACK(256)
blr
FUNC_END(test_fpu)
# registers while running is not zero.
FUNC_START(preempt_fpu)
PUSH_BASIC_STACK(256)
+ PUSH_FPU(256)
std r3,STACK_FRAME_PARAM(0)(sp) # double *darray
std r4,STACK_FRAME_PARAM(1)(sp) # int *threads_starting
std r5,STACK_FRAME_PARAM(2)(sp) # int *running
- PUSH_FPU(STACK_FRAME_LOCAL(3,0))
bl load_fpu
nop
cmpwi r5,0
bne 2b
-3: POP_FPU(STACK_FRAME_LOCAL(3,0))
+3: POP_FPU(256)
POP_BASIC_STACK(256)
blr
FUNC_END(preempt_fpu)
*/
#include "../basic_asm.h"
-
-# POS MUST BE 16 ALIGNED!
-#define PUSH_VMX(pos,reg) \
- li reg,pos; \
- stvx v20,reg,sp; \
- addi reg,reg,16; \
- stvx v21,reg,sp; \
- addi reg,reg,16; \
- stvx v22,reg,sp; \
- addi reg,reg,16; \
- stvx v23,reg,sp; \
- addi reg,reg,16; \
- stvx v24,reg,sp; \
- addi reg,reg,16; \
- stvx v25,reg,sp; \
- addi reg,reg,16; \
- stvx v26,reg,sp; \
- addi reg,reg,16; \
- stvx v27,reg,sp; \
- addi reg,reg,16; \
- stvx v28,reg,sp; \
- addi reg,reg,16; \
- stvx v29,reg,sp; \
- addi reg,reg,16; \
- stvx v30,reg,sp; \
- addi reg,reg,16; \
- stvx v31,reg,sp;
-
-# POS MUST BE 16 ALIGNED!
-#define POP_VMX(pos,reg) \
- li reg,pos; \
- lvx v20,reg,sp; \
- addi reg,reg,16; \
- lvx v21,reg,sp; \
- addi reg,reg,16; \
- lvx v22,reg,sp; \
- addi reg,reg,16; \
- lvx v23,reg,sp; \
- addi reg,reg,16; \
- lvx v24,reg,sp; \
- addi reg,reg,16; \
- lvx v25,reg,sp; \
- addi reg,reg,16; \
- lvx v26,reg,sp; \
- addi reg,reg,16; \
- lvx v27,reg,sp; \
- addi reg,reg,16; \
- lvx v28,reg,sp; \
- addi reg,reg,16; \
- lvx v29,reg,sp; \
- addi reg,reg,16; \
- lvx v30,reg,sp; \
- addi reg,reg,16; \
- lvx v31,reg,sp;
-
-# Carefull this will 'clobber' vmx (by design)
-# Don't call this from C
-FUNC_START(load_vmx)
- li r5,0
- lvx v20,r5,r3
- addi r5,r5,16
- lvx v21,r5,r3
- addi r5,r5,16
- lvx v22,r5,r3
- addi r5,r5,16
- lvx v23,r5,r3
- addi r5,r5,16
- lvx v24,r5,r3
- addi r5,r5,16
- lvx v25,r5,r3
- addi r5,r5,16
- lvx v26,r5,r3
- addi r5,r5,16
- lvx v27,r5,r3
- addi r5,r5,16
- lvx v28,r5,r3
- addi r5,r5,16
- lvx v29,r5,r3
- addi r5,r5,16
- lvx v30,r5,r3
- addi r5,r5,16
- lvx v31,r5,r3
- blr
-FUNC_END(load_vmx)
+#include "../vmx_asm.h"
# Should be safe from C, only touches r4, r5 and v0,v1,v2
FUNC_START(check_vmx)
--- /dev/null
+/*
+ * Copyright 2015, Cyril Bur, IBM Corp.
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License
+ * as published by the Free Software Foundation; either version
+ * 2 of the License, or (at your option) any later version.
+ */
+
+#include "../basic_asm.h"
+#include "../vsx_asm.h"
+
+#long check_vsx(vector int *r3);
+#This function wraps storeing VSX regs to the end of an array and a
+#call to a comparison function in C which boils down to a memcmp()
+FUNC_START(check_vsx)
+ PUSH_BASIC_STACK(32)
+ std r3,STACK_FRAME_PARAM(0)(sp)
+ addi r3, r3, 16 * 12 #Second half of array
+ bl store_vsx
+ ld r3,STACK_FRAME_PARAM(0)(sp)
+ bl vsx_memcmp
+ POP_BASIC_STACK(32)
+ blr
+FUNC_END(check_vsx)
+
+# int preempt_vmx(vector int *varray, int *threads_starting,
+# int *running);
+# On starting will (atomically) decrement threads_starting as a signal
+# that the VMX have been loaded with varray. Will proceed to check the
+# validity of the VMX registers while running is not zero.
+FUNC_START(preempt_vsx)
+ PUSH_BASIC_STACK(512)
+ std r3,STACK_FRAME_PARAM(0)(sp) # vector int *varray
+ std r4,STACK_FRAME_PARAM(1)(sp) # int *threads_starting
+ std r5,STACK_FRAME_PARAM(2)(sp) # int *running
+
+ bl load_vsx
+ nop
+
+ sync
+ # Atomic DEC
+ ld r3,STACK_FRAME_PARAM(1)(sp)
+1: lwarx r4,0,r3
+ addi r4,r4,-1
+ stwcx. r4,0,r3
+ bne- 1b
+
+2: ld r3,STACK_FRAME_PARAM(0)(sp)
+ bl check_vsx
+ nop
+ cmpdi r3,0
+ bne 3f
+ ld r4,STACK_FRAME_PARAM(2)(sp)
+ ld r5,0(r4)
+ cmpwi r5,0
+ bne 2b
+
+3: POP_BASIC_STACK(512)
+ blr
+FUNC_END(preempt_vsx)
--- /dev/null
+/*
+ * Copyright 2015, Cyril Bur, IBM Corp.
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License
+ * as published by the Free Software Foundation; either version
+ * 2 of the License, or (at your option) any later version.
+ *
+ * This test attempts to see if the VSX registers change across preemption.
+ * There is no way to be sure preemption happened so this test just
+ * uses many threads and a long wait. As such, a successful test
+ * doesn't mean much but a failure is bad.
+ */
+
+#include <stdio.h>
+#include <string.h>
+#include <unistd.h>
+#include <sys/syscall.h>
+#include <sys/time.h>
+#include <sys/types.h>
+#include <sys/wait.h>
+#include <stdlib.h>
+#include <pthread.h>
+
+#include "utils.h"
+
+/* Time to wait for workers to get preempted (seconds) */
+#define PREEMPT_TIME 20
+/*
+ * Factor by which to multiply number of online CPUs for total number of
+ * worker threads
+ */
+#define THREAD_FACTOR 8
+
+/*
+ * Ensure there is twice the number of non-volatile VMX regs!
+ * check_vmx() is going to use the other half as space to put the live
+ * registers before calling vsx_memcmp()
+ */
+__thread vector int varray[24] = {
+ {1, 2, 3, 4 }, {5, 6, 7, 8 }, {9, 10,11,12},
+ {13,14,15,16}, {17,18,19,20}, {21,22,23,24},
+ {25,26,27,28}, {29,30,31,32}, {33,34,35,36},
+ {37,38,39,40}, {41,42,43,44}, {45,46,47,48}
+};
+
+int threads_starting;
+int running;
+
+extern long preempt_vsx(vector int *varray, int *threads_starting, int *running);
+
+long vsx_memcmp(vector int *a) {
+ vector int zero = {0, 0, 0, 0};
+ int i;
+
+ FAIL_IF(a != varray);
+
+ for(i = 0; i < 12; i++) {
+ if (memcmp(&a[i + 12], &zero, sizeof(vector int)) == 0) {
+ fprintf(stderr, "Detected zero from the VSX reg %d\n", i + 12);
+ return 2;
+ }
+ }
+
+ if (memcmp(a, &a[12], 12 * sizeof(vector int))) {
+ long *p = (long *)a;
+ fprintf(stderr, "VSX mismatch\n");
+ for (i = 0; i < 24; i=i+2)
+ fprintf(stderr, "%d: 0x%08lx%08lx | 0x%08lx%08lx\n",
+ i/2 + i%2 + 20, p[i], p[i + 1], p[i + 24], p[i + 25]);
+ return 1;
+ }
+ return 0;
+}
+
+void *preempt_vsx_c(void *p)
+{
+ int i, j;
+ long rc;
+ srand(pthread_self());
+ for (i = 0; i < 12; i++)
+ for (j = 0; j < 4; j++) {
+ varray[i][j] = rand();
+ /* Don't want zero because it hides kernel problems */
+ if (varray[i][j] == 0)
+ j--;
+ }
+ rc = preempt_vsx(varray, &threads_starting, &running);
+ if (rc == 2)
+ fprintf(stderr, "Caught zeros in VSX compares\n");
+ return (void *)rc;
+}
+
+int test_preempt_vsx(void)
+{
+ int i, rc, threads;
+ pthread_t *tids;
+
+ threads = sysconf(_SC_NPROCESSORS_ONLN) * THREAD_FACTOR;
+ tids = malloc(threads * sizeof(pthread_t));
+ FAIL_IF(!tids);
+
+ running = true;
+ threads_starting = threads;
+ for (i = 0; i < threads; i++) {
+ rc = pthread_create(&tids[i], NULL, preempt_vsx_c, NULL);
+ FAIL_IF(rc);
+ }
+
+ setbuf(stdout, NULL);
+ /* Not really nessesary but nice to wait for every thread to start */
+ printf("\tWaiting for %d workers to start...", threads_starting);
+ while(threads_starting)
+ asm volatile("": : :"memory");
+ printf("done\n");
+
+ printf("\tWaiting for %d seconds to let some workers get preempted...", PREEMPT_TIME);
+ sleep(PREEMPT_TIME);
+ printf("done\n");
+
+ printf("\tStopping workers...");
+ /*
+ * Working are checking this value every loop. In preempt_vsx 'cmpwi r5,0; bne 2b'.
+ * r5 will have loaded the value of running.
+ */
+ running = 0;
+ for (i = 0; i < threads; i++) {
+ void *rc_p;
+ pthread_join(tids[i], &rc_p);
+
+ /*
+ * Harness will say the fail was here, look at why preempt_vsx
+ * returned
+ */
+ if ((long) rc_p)
+ printf("oops\n");
+ FAIL_IF((long) rc_p);
+ }
+ printf("done\n");
+
+ return 0;
+}
+
+int main(int argc, char *argv[])
+{
+ return test_harness(test_preempt_vsx, "vsx_preempt");
+}
--- /dev/null
+signal
+signal_tm
--- /dev/null
+TEST_PROGS := signal signal_tm
+
+all: $(TEST_PROGS)
+
+$(TEST_PROGS): ../harness.c ../utils.c signal.S
+
+CFLAGS += -maltivec
+signal_tm: CFLAGS += -mhtm
+
+include ../../lib.mk
+
+clean:
+ rm -f $(TEST_PROGS) *.o
--- /dev/null
+/*
+ * Copyright 2015, Cyril Bur, IBM Corp.
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License
+ * as published by the Free Software Foundation; either version
+ * 2 of the License, or (at your option) any later version.
+ */
+
+#include "../basic_asm.h"
+
+/* long signal_self(pid_t pid, int sig); */
+FUNC_START(signal_self)
+ li r0,37 /* sys_kill */
+ /* r3 already has our pid in it */
+ /* r4 already has signal type in it */
+ sc
+ bc 4,3,1f
+ subfze r3,r3
+1: blr
+FUNC_END(signal_self)
+
+/* long tm_signal_self(pid_t pid, int sig, int *ret); */
+FUNC_START(tm_signal_self)
+ PUSH_BASIC_STACK(8)
+ std r5,STACK_FRAME_PARAM(0)(sp) /* ret */
+ tbegin.
+ beq 1f
+ tsuspend.
+ li r0,37 /* sys_kill */
+ /* r3 already has our pid in it */
+ /* r4 already has signal type in it */
+ sc
+ ld r5,STACK_FRAME_PARAM(0)(sp) /* ret */
+ bc 4,3,2f
+ subfze r3,r3
+2: std r3,0(r5)
+ tabort. 0
+ tresume. /* Be nice to some cleanup, jumps back to tbegin then to 1: */
+ /*
+ * Transaction should be proper doomed and we should never get
+ * here
+ */
+ li r3,1
+ POP_BASIC_STACK(8)
+ blr
+1: li r3,0
+ POP_BASIC_STACK(8)
+ blr
+FUNC_END(tm_signal_self)
--- /dev/null
+/*
+ * Copyright 2016, Cyril Bur, IBM Corp.
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License
+ * as published by the Free Software Foundation; either version
+ * 2 of the License, or (at your option) any later version.
+ *
+ * Sending one self a signal should always get delivered.
+ */
+
+#include <signal.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <sys/types.h>
+#include <sys/wait.h>
+#include <unistd.h>
+
+#include <altivec.h>
+
+#include "utils.h"
+
+#define MAX_ATTEMPT 500000
+#define TIMEOUT 5
+
+extern long signal_self(pid_t pid, int sig);
+
+static sig_atomic_t signaled;
+static sig_atomic_t fail;
+
+static void signal_handler(int sig)
+{
+ if (sig == SIGUSR1)
+ signaled = 1;
+ else
+ fail = 1;
+}
+
+static int test_signal()
+{
+ int i;
+ struct sigaction act;
+ pid_t ppid = getpid();
+ pid_t pid;
+
+ act.sa_handler = signal_handler;
+ act.sa_flags = 0;
+ sigemptyset(&act.sa_mask);
+ if (sigaction(SIGUSR1, &act, NULL) < 0) {
+ perror("sigaction SIGUSR1");
+ exit(1);
+ }
+ if (sigaction(SIGALRM, &act, NULL) < 0) {
+ perror("sigaction SIGALRM");
+ exit(1);
+ }
+
+ /* Don't do this for MAX_ATTEMPT, its simply too long */
+ for(i = 0; i < 1000; i++) {
+ pid = fork();
+ if (pid == -1) {
+ perror("fork");
+ exit(1);
+ }
+ if (pid == 0) {
+ signal_self(ppid, SIGUSR1);
+ exit(1);
+ } else {
+ alarm(0); /* Disable any pending */
+ alarm(2);
+ while (!signaled && !fail)
+ asm volatile("": : :"memory");
+ if (!signaled) {
+ fprintf(stderr, "Didn't get signal from child\n");
+ FAIL_IF(1); /* For the line number */
+ }
+ /* Otherwise we'll loop too fast and fork() will eventually fail */
+ waitpid(pid, NULL, 0);
+ }
+ }
+
+ for (i = 0; i < MAX_ATTEMPT; i++) {
+ long rc;
+
+ alarm(0); /* Disable any pending */
+ signaled = 0;
+ alarm(TIMEOUT);
+ rc = signal_self(ppid, SIGUSR1);
+ if (rc) {
+ fprintf(stderr, "(%d) Fail reason: %d rc=0x%lx",
+ i, fail, rc);
+ FAIL_IF(1); /* For the line number */
+ }
+ while (!signaled && !fail)
+ asm volatile("": : :"memory");
+ if (!signaled) {
+ fprintf(stderr, "(%d) Fail reason: %d rc=0x%lx",
+ i, fail, rc);
+ FAIL_IF(1); /* For the line number */
+ }
+ }
+
+ return 0;
+}
+
+int main(void)
+{
+ test_harness_set_timeout(300);
+ return test_harness(test_signal, "signal");
+}
--- /dev/null
+/*
+ * Copyright 2016, Cyril Bur, IBM Corp.
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License
+ * as published by the Free Software Foundation; either version
+ * 2 of the License, or (at your option) any later version.
+ *
+ * Sending one self a signal should always get delivered.
+ */
+
+#include <errno.h>
+#include <stdlib.h>
+#include <stdio.h>
+#include <string.h>
+#include <signal.h>
+#include <unistd.h>
+
+#include <altivec.h>
+
+#include "utils.h"
+#include "../tm/tm.h"
+
+#define MAX_ATTEMPT 500000
+#define TIMEOUT 10
+
+extern long tm_signal_self(pid_t pid, int sig, long *ret);
+
+static sig_atomic_t signaled;
+static sig_atomic_t fail;
+
+static void signal_handler(int sig)
+{
+ if (tcheck_active()) {
+ fail = 2;
+ return;
+ }
+
+ if (sig == SIGUSR1)
+ signaled = 1;
+ else
+ fail = 1;
+}
+
+static int test_signal_tm()
+{
+ int i;
+ struct sigaction act;
+
+ act.sa_handler = signal_handler;
+ act.sa_flags = 0;
+ sigemptyset(&act.sa_mask);
+ if (sigaction(SIGUSR1, &act, NULL) < 0) {
+ perror("sigaction SIGUSR1");
+ exit(1);
+ }
+ if (sigaction(SIGALRM, &act, NULL) < 0) {
+ perror("sigaction SIGALRM");
+ exit(1);
+ }
+
+ SKIP_IF(!have_htm());
+
+ for (i = 0; i < MAX_ATTEMPT; i++) {
+ /*
+ * If anything bad happens in ASM and we fail to set ret
+ * because *handwave* TM this will cause failure
+ */
+ long ret = 0xdead;
+ long rc = 0xbeef;
+
+ alarm(0); /* Disable any pending */
+ signaled = 0;
+ alarm(TIMEOUT);
+ FAIL_IF(tcheck_transactional());
+ rc = tm_signal_self(getpid(), SIGUSR1, &ret);
+ if (ret == 0xdead)
+ /*
+ * This basically means the transaction aborted before we
+ * even got to the suspend... this is crazy but it
+ * happens.
+ * Yes this also means we might never make forward
+ * progress... the alarm() will trip eventually...
+ */
+ continue;
+
+ if (rc || ret) {
+ /* Ret is actually an errno */
+ printf("TEXASR 0x%016lx, TFIAR 0x%016lx\n",
+ __builtin_get_texasr(), __builtin_get_tfiar());
+ fprintf(stderr, "(%d) Fail reason: %d rc=0x%lx ret=0x%lx\n",
+ i, fail, rc, ret);
+ FAIL_IF(ret);
+ }
+ while(!signaled && !fail)
+ asm volatile("": : :"memory");
+ if (!signaled) {
+ fprintf(stderr, "(%d) Fail reason: %d rc=0x%lx ret=0x%lx\n",
+ i, fail, rc, ret);
+ FAIL_IF(fail); /* For the line number */
+ }
+ }
+
+ return 0;
+}
+
+int main(void)
+{
+ return test_harness(test_signal_tm, "signal_tm");
+}
--- /dev/null
+#define EXPORT_SYMBOL(x)
tm-tar
tm-tmspr
tm-exec
+tm-signal-context-chk-fpu
+tm-signal-context-chk-gpr
+tm-signal-context-chk-vmx
+tm-signal-context-chk-vsx
+SIGNAL_CONTEXT_CHK_TESTS := tm-signal-context-chk-gpr tm-signal-context-chk-fpu \
+ tm-signal-context-chk-vmx tm-signal-context-chk-vsx
+
TEST_PROGS := tm-resched-dscr tm-syscall tm-signal-msr-resv tm-signal-stack \
- tm-vmxcopy tm-fork tm-tar tm-tmspr tm-exec tm-execed
+ tm-vmxcopy tm-fork tm-tar tm-tmspr $(SIGNAL_CONTEXT_CHK_TESTS)
all: $(TEST_PROGS)
tm-syscall: CFLAGS += -I../../../../../usr/include
tm-tmspr: CFLAGS += -pthread
+$(SIGNAL_CONTEXT_CHK_TESTS): tm-signal.S
+$(SIGNAL_CONTEXT_CHK_TESTS): CFLAGS += -mhtm -m64 -mvsx
+
include ../../lib.mk
clean:
--- /dev/null
+/*
+ * Copyright 2016, Cyril Bur, IBM Corp.
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License
+ * as published by the Free Software Foundation; either version
+ * 2 of the License, or (at your option) any later version.
+ *
+ *
+ * Test the kernel's signal frame code.
+ *
+ * The kernel sets up two sets of ucontexts if the signal was to be
+ * delivered while the thread was in a transaction.
+ * Expected behaviour is that the checkpointed state is in the user
+ * context passed to the signal handler. The speculated state can be
+ * accessed with the uc_link pointer.
+ *
+ * The rationale for this is that if TM unaware code (which linked
+ * against TM libs) installs a signal handler it will not know of the
+ * speculative nature of the 'live' registers and may infer the wrong
+ * thing.
+ */
+
+#include <stdlib.h>
+#include <stdio.h>
+#include <signal.h>
+#include <unistd.h>
+
+#include <altivec.h>
+
+#include "utils.h"
+#include "tm.h"
+
+#define MAX_ATTEMPT 500000
+
+#define NV_FPU_REGS 18
+
+long tm_signal_self_context_load(pid_t pid, long *gprs, double *fps, vector int *vms, vector int *vss);
+
+/* Be sure there are 2x as many as there are NV FPU regs (2x18) */
+static double fps[] = {
+ 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18,
+ -1,-2,-3,-4,-5,-6,-7,-8,-9,-10,-11,-12,-13,-14,-15,-16,-17,-18
+};
+
+static sig_atomic_t fail;
+
+static void signal_usr1(int signum, siginfo_t *info, void *uc)
+{
+ int i;
+ ucontext_t *ucp = uc;
+ ucontext_t *tm_ucp = ucp->uc_link;
+
+ for (i = 0; i < NV_FPU_REGS && !fail; i++) {
+ fail = (ucp->uc_mcontext.fp_regs[i + 14] != fps[i]);
+ fail |= (tm_ucp->uc_mcontext.fp_regs[i + 14] != fps[i + NV_FPU_REGS]);
+ if (fail)
+ printf("Failed on %d FP %g or %g\n", i, ucp->uc_mcontext.fp_regs[i + 14], tm_ucp->uc_mcontext.fp_regs[i + 14]);
+ }
+}
+
+static int tm_signal_context_chk_fpu()
+{
+ struct sigaction act;
+ int i;
+ long rc;
+ pid_t pid = getpid();
+
+ SKIP_IF(!have_htm());
+
+ act.sa_sigaction = signal_usr1;
+ sigemptyset(&act.sa_mask);
+ act.sa_flags = SA_SIGINFO;
+ if (sigaction(SIGUSR1, &act, NULL) < 0) {
+ perror("sigaction sigusr1");
+ exit(1);
+ }
+
+ i = 0;
+ while (i < MAX_ATTEMPT && !fail) {
+ rc = tm_signal_self_context_load(pid, NULL, fps, NULL, NULL);
+ FAIL_IF(rc != pid);
+ i++;
+ }
+
+ return fail;
+}
+
+int main(void)
+{
+ return test_harness(tm_signal_context_chk_fpu, "tm_signal_context_chk_fpu");
+}
--- /dev/null
+/*
+ * Copyright 2016, Cyril Bur, IBM Corp.
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License
+ * as published by the Free Software Foundation; either version
+ * 2 of the License, or (at your option) any later version.
+ *
+ *
+ * Test the kernel's signal frame code.
+ *
+ * The kernel sets up two sets of ucontexts if the signal was to be
+ * delivered while the thread was in a transaction.
+ * Expected behaviour is that the checkpointed state is in the user
+ * context passed to the signal handler. The speculated state can be
+ * accessed with the uc_link pointer.
+ *
+ * The rationale for this is that if TM unaware code (which linked
+ * against TM libs) installs a signal handler it will not know of the
+ * speculative nature of the 'live' registers and may infer the wrong
+ * thing.
+ */
+
+#include <stdlib.h>
+#include <stdio.h>
+#include <signal.h>
+#include <unistd.h>
+
+#include <altivec.h>
+
+#include "utils.h"
+#include "tm.h"
+
+#define MAX_ATTEMPT 500000
+
+#define NV_GPR_REGS 18
+
+long tm_signal_self_context_load(pid_t pid, long *gprs, double *fps, vector int *vms, vector int *vss);
+
+static sig_atomic_t fail;
+
+static long gps[] = { 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18,
+ -1,-2,-3,-4,-5,-6,-7,-8,-9,-10,-11,-12,-13,-14,-15,-16,-17,-18};
+
+static void signal_usr1(int signum, siginfo_t *info, void *uc)
+{
+ int i;
+ ucontext_t *ucp = uc;
+ ucontext_t *tm_ucp = ucp->uc_link;
+
+ for (i = 0; i < NV_GPR_REGS && !fail; i++) {
+ fail = (ucp->uc_mcontext.gp_regs[i + 14] != gps[i]);
+ fail |= (tm_ucp->uc_mcontext.gp_regs[i + 14] != gps[i + NV_GPR_REGS]);
+ if (fail)
+ printf("Failed on %d GPR %lu or %lu\n", i,
+ ucp->uc_mcontext.gp_regs[i + 14], tm_ucp->uc_mcontext.gp_regs[i + 14]);
+ }
+}
+
+static int tm_signal_context_chk_gpr()
+{
+ struct sigaction act;
+ int i;
+ long rc;
+ pid_t pid = getpid();
+
+ SKIP_IF(!have_htm());
+
+ act.sa_sigaction = signal_usr1;
+ sigemptyset(&act.sa_mask);
+ act.sa_flags = SA_SIGINFO;
+ if (sigaction(SIGUSR1, &act, NULL) < 0) {
+ perror("sigaction sigusr1");
+ exit(1);
+ }
+
+ i = 0;
+ while (i < MAX_ATTEMPT && !fail) {
+ rc = tm_signal_self_context_load(pid, gps, NULL, NULL, NULL);
+ FAIL_IF(rc != pid);
+ i++;
+ }
+
+ return fail;
+}
+
+int main(void)
+{
+ return test_harness(tm_signal_context_chk_gpr, "tm_signal_context_chk_gpr");
+}
--- /dev/null
+/*
+ * Copyright 2016, Cyril Bur, IBM Corp.
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License
+ * as published by the Free Software Foundation; either version
+ * 2 of the License, or (at your option) any later version.
+ *
+ *
+ * Test the kernel's signal frame code.
+ *
+ * The kernel sets up two sets of ucontexts if the signal was to be
+ * delivered while the thread was in a transaction.
+ * Expected behaviour is that the checkpointed state is in the user
+ * context passed to the signal handler. The speculated state can be
+ * accessed with the uc_link pointer.
+ *
+ * The rationale for this is that if TM unaware code (which linked
+ * against TM libs) installs a signal handler it will not know of the
+ * speculative nature of the 'live' registers and may infer the wrong
+ * thing.
+ */
+
+#include <stdlib.h>
+#include <stdio.h>
+#include <string.h>
+#include <signal.h>
+#include <unistd.h>
+
+#include <altivec.h>
+
+#include "utils.h"
+#include "tm.h"
+
+#define MAX_ATTEMPT 500000
+
+#define NV_VMX_REGS 12
+
+long tm_signal_self_context_load(pid_t pid, long *gprs, double *fps, vector int *vms, vector int *vss);
+
+static sig_atomic_t fail;
+
+vector int vms[] = {
+ {1, 2, 3, 4 },{5, 6, 7, 8 },{9, 10,11,12},
+ {13,14,15,16},{17,18,19,20},{21,22,23,24},
+ {25,26,27,28},{29,30,31,32},{33,34,35,36},
+ {37,38,39,40},{41,42,43,44},{45,46,47,48},
+ {-1, -2, -3, -4}, {-5, -6, -7, -8}, {-9, -10,-11,-12},
+ {-13,-14,-15,-16},{-17,-18,-19,-20},{-21,-22,-23,-24},
+ {-25,-26,-27,-28},{-29,-30,-31,-32},{-33,-34,-35,-36},
+ {-37,-38,-39,-40},{-41,-42,-43,-44},{-45,-46,-47,-48}
+};
+
+static void signal_usr1(int signum, siginfo_t *info, void *uc)
+{
+ int i;
+ ucontext_t *ucp = uc;
+ ucontext_t *tm_ucp = ucp->uc_link;
+
+ for (i = 0; i < NV_VMX_REGS && !fail; i++) {
+ fail = memcmp(ucp->uc_mcontext.v_regs->vrregs[i + 20],
+ &vms[i], sizeof(vector int));
+ fail |= memcmp(tm_ucp->uc_mcontext.v_regs->vrregs[i + 20],
+ &vms[i + NV_VMX_REGS], sizeof (vector int));
+
+ if (fail) {
+ int j;
+
+ fprintf(stderr, "Failed on %d vmx 0x", i);
+ for (j = 0; j < 4; j++)
+ fprintf(stderr, "%04x", ucp->uc_mcontext.v_regs->vrregs[i + 20][j]);
+ fprintf(stderr, " vs 0x");
+ for (j = 0 ; j < 4; j++)
+ fprintf(stderr, "%04x", tm_ucp->uc_mcontext.v_regs->vrregs[i + 20][j]);
+ fprintf(stderr, "\n");
+ }
+ }
+}
+
+static int tm_signal_context_chk()
+{
+ struct sigaction act;
+ int i;
+ long rc;
+ pid_t pid = getpid();
+
+ SKIP_IF(!have_htm());
+
+ act.sa_sigaction = signal_usr1;
+ sigemptyset(&act.sa_mask);
+ act.sa_flags = SA_SIGINFO;
+ if (sigaction(SIGUSR1, &act, NULL) < 0) {
+ perror("sigaction sigusr1");
+ exit(1);
+ }
+
+ i = 0;
+ while (i < MAX_ATTEMPT && !fail) {
+ rc = tm_signal_self_context_load(pid, NULL, NULL, vms, NULL);
+ FAIL_IF(rc != pid);
+ i++;
+ }
+
+ return fail;
+}
+
+int main(void)
+{
+ return test_harness(tm_signal_context_chk, "tm_signal_context_chk_vmx");
+}
--- /dev/null
+/*
+ * Copyright 2016, Cyril Bur, IBM Corp.
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License
+ * as published by the Free Software Foundation; either version
+ * 2 of the License, or (at your option) any later version.
+ *
+ *
+ * Test the kernel's signal frame code.
+ *
+ * The kernel sets up two sets of ucontexts if the signal was to be
+ * delivered while the thread was in a transaction.
+ * Expected behaviour is that the checkpointed state is in the user
+ * context passed to the signal handler. The speculated state can be
+ * accessed with the uc_link pointer.
+ *
+ * The rationale for this is that if TM unaware code (which linked
+ * against TM libs) installs a signal handler it will not know of the
+ * speculative nature of the 'live' registers and may infer the wrong
+ * thing.
+ */
+
+#include <stdlib.h>
+#include <stdio.h>
+#include <string.h>
+#include <signal.h>
+#include <unistd.h>
+
+#include <altivec.h>
+
+#include "utils.h"
+#include "tm.h"
+
+#define MAX_ATTEMPT 500000
+
+#define NV_VSX_REGS 12
+
+long tm_signal_self_context_load(pid_t pid, long *gprs, double *fps, vector int *vms, vector int *vss);
+
+static sig_atomic_t fail;
+
+vector int vss[] = {
+ {1, 2, 3, 4 },{5, 6, 7, 8 },{9, 10,11,12},
+ {13,14,15,16},{17,18,19,20},{21,22,23,24},
+ {25,26,27,28},{29,30,31,32},{33,34,35,36},
+ {37,38,39,40},{41,42,43,44},{45,46,47,48},
+ {-1, -2, -3, -4 },{-5, -6, -7, -8 },{-9, -10,-11,-12},
+ {-13,-14,-15,-16},{-17,-18,-19,-20},{-21,-22,-23,-24},
+ {-25,-26,-27,-28},{-29,-30,-31,-32},{-33,-34,-35,-36},
+ {-37,-38,-39,-40},{-41,-42,-43,-44},{-45,-46,-47,-48}
+};
+
+static void signal_usr1(int signum, siginfo_t *info, void *uc)
+{
+ int i;
+ uint8_t vsc[sizeof(vector int)];
+ uint8_t vst[sizeof(vector int)];
+ ucontext_t *ucp = uc;
+ ucontext_t *tm_ucp = ucp->uc_link;
+
+ /*
+ * The other half of the VSX regs will be after v_regs.
+ *
+ * In short, vmx_reserve array holds everything. v_regs is a 16
+ * byte aligned pointer at the start of vmx_reserve (vmx_reserve
+ * may or may not be 16 aligned) where the v_regs structure exists.
+ * (half of) The VSX regsters are directly after v_regs so the
+ * easiest way to find them below.
+ */
+ long *vsx_ptr = (long *)(ucp->uc_mcontext.v_regs + 1);
+ long *tm_vsx_ptr = (long *)(tm_ucp->uc_mcontext.v_regs + 1);
+ for (i = 0; i < NV_VSX_REGS && !fail; i++) {
+ memcpy(vsc, &ucp->uc_mcontext.fp_regs[i + 20], 8);
+ memcpy(vsc + 8, &vsx_ptr[20 + i], 8);
+ fail = memcmp(vsc, &vss[i], sizeof(vector int));
+ memcpy(vst, &tm_ucp->uc_mcontext.fp_regs[i + 20], 8);
+ memcpy(vst + 8, &tm_vsx_ptr[20 + i], 8);
+ fail |= memcmp(vst, &vss[i + NV_VSX_REGS], sizeof(vector int));
+
+ if (fail) {
+ int j;
+
+ fprintf(stderr, "Failed on %d vsx 0x", i);
+ for (j = 0; j < 16; j++)
+ fprintf(stderr, "%02x", vsc[j]);
+ fprintf(stderr, " vs 0x");
+ for (j = 0; j < 16; j++)
+ fprintf(stderr, "%02x", vst[j]);
+ fprintf(stderr, "\n");
+ }
+ }
+}
+
+static int tm_signal_context_chk()
+{
+ struct sigaction act;
+ int i;
+ long rc;
+ pid_t pid = getpid();
+
+ SKIP_IF(!have_htm());
+
+ act.sa_sigaction = signal_usr1;
+ sigemptyset(&act.sa_mask);
+ act.sa_flags = SA_SIGINFO;
+ if (sigaction(SIGUSR1, &act, NULL) < 0) {
+ perror("sigaction sigusr1");
+ exit(1);
+ }
+
+ i = 0;
+ while (i < MAX_ATTEMPT && !fail) {
+ rc = tm_signal_self_context_load(pid, NULL, NULL, NULL, vss);
+ FAIL_IF(rc != pid);
+ i++;
+ }
+
+ return fail;
+}
+
+int main(void)
+{
+ return test_harness(tm_signal_context_chk, "tm_signal_context_chk_vsx");
+}
--- /dev/null
+/*
+ * Copyright 2015, Cyril Bur, IBM Corp.
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License
+ * as published by the Free Software Foundation; either version
+ * 2 of the License, or (at your option) any later version.
+ */
+
+#include "../basic_asm.h"
+#include "../gpr_asm.h"
+#include "../fpu_asm.h"
+#include "../vmx_asm.h"
+#include "../vsx_asm.h"
+
+/*
+ * Large caveat here being that the caller cannot expect the
+ * signal to always be sent! The hardware can (AND WILL!) abort
+ * the transaction between the tbegin and the tsuspend (however
+ * unlikely it seems or infrequently it actually happens).
+ * You have been warned.
+ */
+/* long tm_signal_self(pid_t pid, long *gprs, double *fps, vector *vms, vector *vss); */
+FUNC_START(tm_signal_self_context_load)
+ PUSH_BASIC_STACK(512)
+ /*
+ * Don't strictly need to save and restore as it depends on if
+ * we're going to use them, however this reduces messy logic
+ */
+ PUSH_VMX(STACK_FRAME_LOCAL(5,0),r8)
+ PUSH_FPU(512)
+ PUSH_NVREGS_BELOW_FPU(512)
+ std r3, STACK_FRAME_PARAM(0)(sp) /* pid */
+ std r4, STACK_FRAME_PARAM(1)(sp) /* gps */
+ std r5, STACK_FRAME_PARAM(2)(sp) /* fps */
+ std r6, STACK_FRAME_PARAM(3)(sp) /* vms */
+ std r7, STACK_FRAME_PARAM(4)(sp) /* vss */
+
+ ld r3, STACK_FRAME_PARAM(1)(sp)
+ cmpdi r3, 0
+ beq skip_gpr_lc
+ bl load_gpr
+skip_gpr_lc:
+ ld r3, STACK_FRAME_PARAM(2)(sp)
+ cmpdi r3, 0
+ beq skip_fpu_lc
+ bl load_fpu
+skip_fpu_lc:
+ ld r3, STACK_FRAME_PARAM(3)(sp)
+ cmpdi r3, 0
+ beq skip_vmx_lc
+ bl load_vmx
+skip_vmx_lc:
+ ld r3, STACK_FRAME_PARAM(4)(sp)
+ cmpdi r3, 0
+ beq skip_vsx_lc
+ bl load_vsx
+skip_vsx_lc:
+ /*
+ * Set r3 (return value) before tbegin. Use the pid as a known
+ * 'all good' return value, zero is used to indicate a non-doomed
+ * transaction.
+ */
+ ld r3, STACK_FRAME_PARAM(0)(sp)
+ tbegin.
+ beq 1f
+ tsuspend. /* Can't enter a syscall transactionally */
+ ld r3, STACK_FRAME_PARAM(1)(sp)
+ cmpdi r3, 0
+ beq skip_gpr_lt
+ /* Get the second half of the array */
+ addi r3, r3, 8 * 18
+ bl load_gpr
+skip_gpr_lt:
+ ld r3, STACK_FRAME_PARAM(2)(sp)
+ cmpdi r3, 0
+ beq skip_fpu_lt
+ /* Get the second half of the array */
+ addi r3, r3, 8 * 18
+ bl load_fpu
+skip_fpu_lt:
+ ld r3, STACK_FRAME_PARAM(3)(sp)
+ cmpdi r3, 0
+ beq skip_vmx_lt
+ /* Get the second half of the array */
+ addi r3, r3, 16 * 12
+ bl load_vmx
+skip_vmx_lt:
+ ld r3, STACK_FRAME_PARAM(4)(sp)
+ cmpdi r3, 0
+ beq skip_vsx_lt
+ /* Get the second half of the array */
+ addi r3, r3, 16 * 12
+ bl load_vsx
+skip_vsx_lt:
+ li r0, 37 /* sys_kill */
+ ld r3, STACK_FRAME_PARAM(0)(sp) /* pid */
+ li r4, 10 /* SIGUSR1 */
+ sc /* Taking the signal will doom the transaction */
+ tabort. 0
+ tresume. /* Be super sure we abort */
+ /*
+ * This will cause us to resume doomed transaction and cause
+ * hardware to cleanup, we'll end up at 1: anything between
+ * tresume. and 1: shouldn't ever run.
+ */
+ li r3, 0
+ 1:
+ POP_VMX(STACK_FRAME_LOCAL(5,0),r4)
+ POP_FPU(512)
+ POP_NVREGS_BELOW_FPU(512)
+ POP_BASIC_STACK(512)
+ blr
+FUNC_END(tm_signal_self_context_load)
return (__builtin_get_texasru() & 0x400000);
}
+static inline int tcheck(void)
+{
+ long cr;
+ asm volatile ("tcheck 0" : "=r"(cr) : : "cr0");
+ return (cr >> 28) & 4;
+}
+
+static inline bool tcheck_doomed(void)
+{
+ return tcheck() & 8;
+}
+
+static inline bool tcheck_active(void)
+{
+ return tcheck() & 4;
+}
+
+static inline bool tcheck_suspended(void)
+{
+ return tcheck() & 2;
+}
+
+static inline bool tcheck_transactional(void)
+{
+ return tcheck() & 6;
+}
+
#endif /* _SELFTESTS_POWERPC_TM_TM_H */
typedef uint16_t u16;
typedef uint8_t u8;
-
+void test_harness_set_timeout(uint64_t time);
int test_harness(int (test_function)(void), char *name);
extern void *get_auxv_entry(int type);
int pick_online_cpu(void);
return ((unsigned long)get_auxv_entry(AT_HWCAP) & ftr) == ftr;
}
+#ifdef AT_HWCAP2
static inline bool have_hwcap2(unsigned long ftr2)
{
return ((unsigned long)get_auxv_entry(AT_HWCAP2) & ftr2) == ftr2;
}
+#else
+static inline bool have_hwcap2(unsigned long ftr2)
+{
+ return false;
+}
+#endif
/* Yes, this is evil */
#define FAIL_IF(x) \
--- /dev/null
+/*
+ * Copyright 2015, Cyril Bur, IBM Corp.
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License
+ * as published by the Free Software Foundation; either version
+ * 2 of the License, or (at your option) any later version.
+ */
+
+#include "basic_asm.h"
+
+/* POS MUST BE 16 ALIGNED! */
+#define PUSH_VMX(pos,reg) \
+ li reg,pos; \
+ stvx v20,reg,%r1; \
+ addi reg,reg,16; \
+ stvx v21,reg,%r1; \
+ addi reg,reg,16; \
+ stvx v22,reg,%r1; \
+ addi reg,reg,16; \
+ stvx v23,reg,%r1; \
+ addi reg,reg,16; \
+ stvx v24,reg,%r1; \
+ addi reg,reg,16; \
+ stvx v25,reg,%r1; \
+ addi reg,reg,16; \
+ stvx v26,reg,%r1; \
+ addi reg,reg,16; \
+ stvx v27,reg,%r1; \
+ addi reg,reg,16; \
+ stvx v28,reg,%r1; \
+ addi reg,reg,16; \
+ stvx v29,reg,%r1; \
+ addi reg,reg,16; \
+ stvx v30,reg,%r1; \
+ addi reg,reg,16; \
+ stvx v31,reg,%r1;
+
+/* POS MUST BE 16 ALIGNED! */
+#define POP_VMX(pos,reg) \
+ li reg,pos; \
+ lvx v20,reg,%r1; \
+ addi reg,reg,16; \
+ lvx v21,reg,%r1; \
+ addi reg,reg,16; \
+ lvx v22,reg,%r1; \
+ addi reg,reg,16; \
+ lvx v23,reg,%r1; \
+ addi reg,reg,16; \
+ lvx v24,reg,%r1; \
+ addi reg,reg,16; \
+ lvx v25,reg,%r1; \
+ addi reg,reg,16; \
+ lvx v26,reg,%r1; \
+ addi reg,reg,16; \
+ lvx v27,reg,%r1; \
+ addi reg,reg,16; \
+ lvx v28,reg,%r1; \
+ addi reg,reg,16; \
+ lvx v29,reg,%r1; \
+ addi reg,reg,16; \
+ lvx v30,reg,%r1; \
+ addi reg,reg,16; \
+ lvx v31,reg,%r1;
+
+/*
+ * Careful this will 'clobber' vmx (by design)
+ * Don't call this from C
+ */
+FUNC_START(load_vmx)
+ li r5,0
+ lvx v20,r5,r3
+ addi r5,r5,16
+ lvx v21,r5,r3
+ addi r5,r5,16
+ lvx v22,r5,r3
+ addi r5,r5,16
+ lvx v23,r5,r3
+ addi r5,r5,16
+ lvx v24,r5,r3
+ addi r5,r5,16
+ lvx v25,r5,r3
+ addi r5,r5,16
+ lvx v26,r5,r3
+ addi r5,r5,16
+ lvx v27,r5,r3
+ addi r5,r5,16
+ lvx v28,r5,r3
+ addi r5,r5,16
+ lvx v29,r5,r3
+ addi r5,r5,16
+ lvx v30,r5,r3
+ addi r5,r5,16
+ lvx v31,r5,r3
+ blr
+FUNC_END(load_vmx)
--- /dev/null
+/*
+ * Copyright 2015, Cyril Bur, IBM Corp.
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License
+ * as published by the Free Software Foundation; either version
+ * 2 of the License, or (at your option) any later version.
+ */
+
+#include "basic_asm.h"
+
+/*
+ * Careful this will 'clobber' vsx (by design), VSX are always
+ * volatile though so unlike vmx this isn't so much of an issue
+ * Still should avoid calling from C
+ */
+FUNC_START(load_vsx)
+ li r5,0
+ lxvx vs20,r5,r3
+ addi r5,r5,16
+ lxvx vs21,r5,r3
+ addi r5,r5,16
+ lxvx vs22,r5,r3
+ addi r5,r5,16
+ lxvx vs23,r5,r3
+ addi r5,r5,16
+ lxvx vs24,r5,r3
+ addi r5,r5,16
+ lxvx vs25,r5,r3
+ addi r5,r5,16
+ lxvx vs26,r5,r3
+ addi r5,r5,16
+ lxvx vs27,r5,r3
+ addi r5,r5,16
+ lxvx vs28,r5,r3
+ addi r5,r5,16
+ lxvx vs29,r5,r3
+ addi r5,r5,16
+ lxvx vs30,r5,r3
+ addi r5,r5,16
+ lxvx vs31,r5,r3
+ blr
+FUNC_END(load_vsx)
+
+FUNC_START(store_vsx)
+ li r5,0
+ stxvx vs20,r5,r3
+ addi r5,r5,16
+ stxvx vs21,r5,r3
+ addi r5,r5,16
+ stxvx vs22,r5,r3
+ addi r5,r5,16
+ stxvx vs23,r5,r3
+ addi r5,r5,16
+ stxvx vs24,r5,r3
+ addi r5,r5,16
+ stxvx vs25,r5,r3
+ addi r5,r5,16
+ stxvx vs26,r5,r3
+ addi r5,r5,16
+ stxvx vs27,r5,r3
+ addi r5,r5,16
+ stxvx vs28,r5,r3
+ addi r5,r5,16
+ stxvx vs29,r5,r3
+ addi r5,r5,16
+ stxvx vs30,r5,r3
+ addi r5,r5,16
+ stxvx vs31,r5,r3
+ blr
+FUNC_END(store_vsx)
projects / platform / kernel / linux-rpi.git / commitdiff