Bit Operations
--------------
-.. kernel-doc:: include/asm-generic/bitops-instrumented.h
+.. kernel-doc:: include/asm-generic/bitops/instrumented-atomic.h
+ :internal:
+
+.. kernel-doc:: include/asm-generic/bitops/instrumented-non-atomic.h
+ :internal:
+
+.. kernel-doc:: include/asm-generic/bitops/instrumented-lock.h
:internal:
Bitmap Operations
- iommus: phandle to the adreno iommu
- operating-points-v2: phandle to the OPP operating points
+Optional properties:
+- sram: phandle to the On Chip Memory (OCMEM) that's present on some Snapdragon
+ SoCs. See Documentation/devicetree/bindings/sram/qcom,ocmem.yaml.
+
Example:
/ {
operating-points-v2 = <&gmu_opp_table>;
};
};
+
+a3xx example with OCMEM support:
+
+/ {
+ ...
+
+ gpu: adreno@fdb00000 {
+ compatible = "qcom,adreno-330.2",
+ "qcom,adreno";
+ reg = <0xfdb00000 0x10000>;
+ reg-names = "kgsl_3d0_reg_memory";
+ interrupts = <GIC_SPI 33 IRQ_TYPE_LEVEL_HIGH>;
+ interrupt-names = "kgsl_3d0_irq";
+ clock-names = "core",
+ "iface",
+ "mem_iface";
+ clocks = <&mmcc OXILI_GFX3D_CLK>,
+ <&mmcc OXILICX_AHB_CLK>,
+ <&mmcc OXILICX_AXI_CLK>;
+ sram = <&gmu_sram>;
+ power-domains = <&mmcc OXILICX_GDSC>;
+ operating-points-v2 = <&gpu_opp_table>;
+ iommus = <&gpu_iommu 0>;
+ };
+
+ ocmem@fdd00000 {
+ compatible = "qcom,msm8974-ocmem";
+
+ reg = <0xfdd00000 0x2000>,
+ <0xfec00000 0x180000>;
+ reg-names = "ctrl",
+ "mem";
+
+ clocks = <&rpmcc RPM_SMD_OCMEMGX_CLK>,
+ <&mmcc OCMEMCX_OCMEMNOC_CLK>;
+ clock-names = "core",
+ "iface";
+
+ #address-cells = <1>;
+ #size-cells = <1>;
+
+ gmu_sram: gmu-sram@0 {
+ reg = <0x0 0x100000>;
+ ranges = <0 0 0xfec00000 0x100000>;
+ };
+ };
+};
Optional properties:
- clock-names: the following clocks are optional:
* "lut"
+ * "tbu"
+ * "tbu_rt"
Example:
--- /dev/null
+# SPDX-License-Identifier: (GPL-2.0 OR BSD-2-Clause)
+%YAML 1.2
+---
+$id: http://devicetree.org/schemas/sram/qcom,ocmem.yaml#
+$schema: http://devicetree.org/meta-schemas/core.yaml#
+
+title: On Chip Memory (OCMEM) that is present on some Qualcomm Snapdragon SoCs.
+
+maintainers:
+ - Brian Masney <masneyb@onstation.org>
+
+description: |
+ The On Chip Memory (OCMEM) is typically used by the GPU, camera/video, and
+ audio components on some Snapdragon SoCs.
+
+properties:
+ compatible:
+ const: qcom,msm8974-ocmem
+
+ reg:
+ items:
+ - description: Control registers
+ - description: OCMEM address range
+
+ reg-names:
+ items:
+ - const: ctrl
+ - const: mem
+
+ clocks:
+ items:
+ - description: Core clock
+ - description: Interface clock
+
+ clock-names:
+ items:
+ - const: core
+ - const: iface
+
+ '#address-cells':
+ const: 1
+
+ '#size-cells':
+ const: 1
+
+required:
+ - compatible
+ - reg
+ - reg-names
+ - clocks
+ - clock-names
+ - '#address-cells'
+ - '#size-cells'
+
+patternProperties:
+ "^.+-sram$":
+ type: object
+ description: A region of reserved memory.
+
+ properties:
+ reg:
+ maxItems: 1
+
+ ranges:
+ maxItems: 1
+
+ required:
+ - reg
+ - ranges
+
+examples:
+ - |
+ #include <dt-bindings/clock/qcom,rpmcc.h>
+ #include <dt-bindings/clock/qcom,mmcc-msm8974.h>
+
+ ocmem: ocmem@fdd00000 {
+ compatible = "qcom,msm8974-ocmem";
+
+ reg = <0xfdd00000 0x2000>,
+ <0xfec00000 0x180000>;
+ reg-names = "ctrl",
+ "mem";
+
+ clocks = <&rpmcc RPM_SMD_OCMEMGX_CLK>,
+ <&mmcc OCMEMCX_OCMEMNOC_CLK>;
+ clock-names = "core",
+ "iface";
+
+ #address-cells = <1>;
+ #size-cells = <1>;
+
+ gmu-sram@0 {
+ reg = <0x0 0x100000>;
+ ranges = <0 0 0xfec00000 0x100000>;
+ };
+ };
F: drivers/i2c/busses/i2c-amd-mp2*
AMD POWERPLAY
-M: Rex Zhu <rex.zhu@amd.com>
M: Evan Quan <evan.quan@amd.com>
L: amd-gfx@lists.freedesktop.org
S: Supported
RADOS BLOCK DEVICE (RBD)
M: Ilya Dryomov <idryomov@gmail.com>
M: Sage Weil <sage@redhat.com>
-M: Alex Elder <elder@kernel.org>
+R: Dongsheng Yang <dongsheng.yang@easystack.cn>
L: ceph-devel@vger.kernel.org
W: http://ceph.com/
T: git git://git.kernel.org/pub/scm/linux/kernel/git/sage/ceph-client.git
void kvm_update_va_mask(struct alt_instr *alt,
__le32 *origptr, __le32 *updptr, int nr_inst);
+void kvm_compute_layout(void);
static inline unsigned long __kern_hyp_va(unsigned long v)
{
extern char __idmap_text_start[], __idmap_text_end[];
extern char __initdata_begin[], __initdata_end[];
extern char __inittext_begin[], __inittext_end[];
+extern char __exittext_begin[], __exittext_end[];
extern char __irqentry_text_start[], __irqentry_text_end[];
extern char __mmuoff_data_start[], __mmuoff_data_end[];
extern char __entry_tramp_text_start[], __entry_tramp_text_end[];
{
unsigned long ret, limit = current_thread_info()->addr_limit;
+ /*
+ * Asynchronous I/O running in a kernel thread does not have the
+ * TIF_TAGGED_ADDR flag of the process owning the mm, so always untag
+ * the user address before checking.
+ */
if (IS_ENABLED(CONFIG_ARM64_TAGGED_ADDR_ABI) &&
- test_thread_flag(TIF_TAGGED_ADDR))
+ (current->flags & PF_KTHREAD || test_thread_flag(TIF_TAGGED_ADDR)))
addr = untagged_addr(addr);
__chk_user_ptr(addr);
bl prepare_ftrace_return
b ftrace_common_return
ENDPROC(ftrace_graph_caller)
-#else
#endif
#else /* CONFIG_DYNAMIC_FTRACE_WITH_REGS */
mcount_exit
ENDPROC(ftrace_caller)
+#endif /* CONFIG_DYNAMIC_FTRACE */
#ifdef CONFIG_FUNCTION_GRAPH_TRACER
/*
mcount_exit
ENDPROC(ftrace_graph_caller)
#endif /* CONFIG_FUNCTION_GRAPH_TRACER */
-#endif /* CONFIG_DYNAMIC_FTRACE */
#endif /* CONFIG_DYNAMIC_FTRACE_WITH_REGS */
ENTRY(ftrace_stub)
#ifdef CONFIG_VMAP_STACK
/*
* Test whether the SP has overflowed, without corrupting a GPR.
- * Task and IRQ stacks are aligned to (1 << THREAD_SHIFT).
+ * Task and IRQ stacks are aligned so that SP & (1 << THREAD_SHIFT)
+ * should always be zero.
*/
add sp, sp, x0 // sp' = sp + x0
sub x0, sp, x0 // x0' = sp' - x0 = (sp + x0) - x0 = sp
#include <asm/fixmap.h>
#include <asm/insn.h>
#include <asm/kprobes.h>
+#include <asm/sections.h>
#define AARCH64_INSN_SF_BIT BIT(31)
#define AARCH64_INSN_N_BIT BIT(22)
static DEFINE_RAW_SPINLOCK(patch_lock);
+static bool is_exit_text(unsigned long addr)
+{
+ /* discarded with init text/data */
+ return system_state < SYSTEM_RUNNING &&
+ addr >= (unsigned long)__exittext_begin &&
+ addr < (unsigned long)__exittext_end;
+}
+
+static bool is_image_text(unsigned long addr)
+{
+ return core_kernel_text(addr) || is_exit_text(addr);
+}
+
static void __kprobes *patch_map(void *addr, int fixmap)
{
unsigned long uintaddr = (uintptr_t) addr;
- bool module = !core_kernel_text(uintaddr);
+ bool image = is_image_text(uintaddr);
struct page *page;
- if (module && IS_ENABLED(CONFIG_STRICT_MODULE_RWX))
- page = vmalloc_to_page(addr);
- else if (!module)
+ if (image)
page = phys_to_page(__pa_symbol(addr));
+ else if (IS_ENABLED(CONFIG_STRICT_MODULE_RWX))
+ page = vmalloc_to_page(addr);
else
return addr;
#include <linux/of.h>
#include <linux/irq_work.h>
#include <linux/kexec.h>
+#include <linux/kvm_host.h>
#include <asm/alternative.h>
#include <asm/atomic.h>
#include <asm/cputype.h>
#include <asm/cpu_ops.h>
#include <asm/daifflags.h>
+#include <asm/kvm_mmu.h>
#include <asm/mmu_context.h>
#include <asm/numa.h>
#include <asm/pgtable.h>
"CPU: CPUs started in inconsistent modes");
else
pr_info("CPU: All CPU(s) started at EL1\n");
+ if (IS_ENABLED(CONFIG_KVM_ARM_HOST))
+ kvm_compute_layout();
}
void __init smp_cpus_done(unsigned int max_cpus)
__inittext_begin = .;
INIT_TEXT_SECTION(8)
+
+ __exittext_begin = .;
.exit.text : {
ARM_EXIT_KEEP(EXIT_TEXT)
}
+ __exittext_end = .;
. = ALIGN(4);
.altinstructions : {
static u64 tag_val;
static u64 va_mask;
-static void compute_layout(void)
+__init void kvm_compute_layout(void)
{
phys_addr_t idmap_addr = __pa_symbol(__hyp_idmap_text_start);
u64 hyp_va_msb;
BUG_ON(nr_inst != 5);
- if (!has_vhe() && !va_mask)
- compute_layout();
-
for (i = 0; i < nr_inst; i++) {
u32 rd, rn, insn, oinsn;
return;
}
- if (!va_mask)
- compute_layout();
-
/*
* Compute HYP VA by using the same computation as kern_hyp_va()
*/
.mask = PTE_UXN,
.val = PTE_UXN,
.set = "UXN",
+ .clear = " ",
}, {
.mask = PTE_ATTRINDX_MASK,
.val = PTE_ATTRINDX(MT_DEVICE_nGnRnE),
{
struct memblock_region *reg;
unsigned long zone_size[MAX_NR_ZONES], zhole_size[MAX_NR_ZONES];
- unsigned long max_dma32 = min;
- unsigned long __maybe_unused max_dma = min;
+ unsigned long __maybe_unused max_dma, max_dma32;
memset(zone_size, 0, sizeof(zone_size));
+ max_dma = max_dma32 = min;
#ifdef CONFIG_ZONE_DMA
- max_dma = PFN_DOWN(arm64_dma_phys_limit);
+ max_dma = max_dma32 = PFN_DOWN(arm64_dma_phys_limit);
zone_size[ZONE_DMA] = max_dma - min;
- max_dma32 = max_dma;
#endif
#ifdef CONFIG_ZONE_DMA32
max_dma32 = PFN_DOWN(arm64_dma32_phys_limit);
unsigned long start = memblock_region_memory_base_pfn(reg);
unsigned long end = memblock_region_memory_end_pfn(reg);
- if (start >= max)
- continue;
#ifdef CONFIG_ZONE_DMA
- if (start < max_dma) {
- unsigned long dma_end = min_not_zero(end, max_dma);
+ if (start >= min && start < max_dma) {
+ unsigned long dma_end = min(end, max_dma);
zhole_size[ZONE_DMA] -= dma_end - start;
+ start = dma_end;
}
#endif
#ifdef CONFIG_ZONE_DMA32
- if (start < max_dma32) {
+ if (start >= max_dma && start < max_dma32) {
unsigned long dma32_end = min(end, max_dma32);
- unsigned long dma32_start = max(start, max_dma);
- zhole_size[ZONE_DMA32] -= dma32_end - dma32_start;
+ zhole_size[ZONE_DMA32] -= dma32_end - start;
+ start = dma32_end;
}
#endif
- if (end > max_dma32) {
+ if (start >= max_dma32 && start < max) {
unsigned long normal_end = min(end, max);
- unsigned long normal_start = max(start, max_dma32);
- zhole_size[ZONE_NORMAL] -= normal_end - normal_start;
+ zhole_size[ZONE_NORMAL] -= normal_end - start;
}
}
* in coherent mode, which lets us map the AGP memory as normal (write-back) memory
* (unlike x86, where it gets mapped "write-coalescing").
*/
-#define map_page_into_agp(page) /* nothing */
-#define unmap_page_from_agp(page) /* nothing */
+#define map_page_into_agp(page) do { } while (0)
+#define unmap_page_from_agp(page) do { } while (0)
#define flush_agp_cache() mb()
/* GATT allocation. Returns/accepts GATT kernel virtual address. */
btst #5,%sp@(PT_OFF_SR) /* check if returning to kernel */
jeq Luser_return /* if so, skip resched, signals */
-#ifdef CONFIG_PREEMPT
+#ifdef CONFIG_PREEMPTION
movel %sp,%d1 /* get thread_info pointer */
andl #-THREAD_SIZE,%d1 /* at base of kernel stack */
movel %d1,%a0
unsigned long val;
int rc;
- rc = arch_get_random_long(&val);
+ rc = arch_get_random_seed_long(&val);
if (rc)
*v = val;
/* Macro for generating the ***_bits() functions */
#define DEFINE_BITOP(fn, op, prefix) \
-static __inline__ void fn(unsigned long mask, \
+static inline void fn(unsigned long mask, \
volatile unsigned long *_p) \
{ \
unsigned long old; \
DEFINE_BITOP(clear_bits_unlock, andc, PPC_RELEASE_BARRIER)
DEFINE_BITOP(change_bits, xor, "")
-static __inline__ void set_bit(int nr, volatile unsigned long *addr)
+static inline void arch_set_bit(int nr, volatile unsigned long *addr)
{
set_bits(BIT_MASK(nr), addr + BIT_WORD(nr));
}
-static __inline__ void clear_bit(int nr, volatile unsigned long *addr)
+static inline void arch_clear_bit(int nr, volatile unsigned long *addr)
{
clear_bits(BIT_MASK(nr), addr + BIT_WORD(nr));
}
-static __inline__ void clear_bit_unlock(int nr, volatile unsigned long *addr)
+static inline void arch_clear_bit_unlock(int nr, volatile unsigned long *addr)
{
clear_bits_unlock(BIT_MASK(nr), addr + BIT_WORD(nr));
}
-static __inline__ void change_bit(int nr, volatile unsigned long *addr)
+static inline void arch_change_bit(int nr, volatile unsigned long *addr)
{
change_bits(BIT_MASK(nr), addr + BIT_WORD(nr));
}
/* Like DEFINE_BITOP(), with changes to the arguments to 'op' and the output
* operands. */
#define DEFINE_TESTOP(fn, op, prefix, postfix, eh) \
-static __inline__ unsigned long fn( \
+static inline unsigned long fn( \
unsigned long mask, \
volatile unsigned long *_p) \
{ \
DEFINE_TESTOP(test_and_change_bits, xor, PPC_ATOMIC_ENTRY_BARRIER,
PPC_ATOMIC_EXIT_BARRIER, 0)
-static __inline__ int test_and_set_bit(unsigned long nr,
- volatile unsigned long *addr)
+static inline int arch_test_and_set_bit(unsigned long nr,
+ volatile unsigned long *addr)
{
return test_and_set_bits(BIT_MASK(nr), addr + BIT_WORD(nr)) != 0;
}
-static __inline__ int test_and_set_bit_lock(unsigned long nr,
- volatile unsigned long *addr)
+static inline int arch_test_and_set_bit_lock(unsigned long nr,
+ volatile unsigned long *addr)
{
return test_and_set_bits_lock(BIT_MASK(nr),
addr + BIT_WORD(nr)) != 0;
}
-static __inline__ int test_and_clear_bit(unsigned long nr,
- volatile unsigned long *addr)
+static inline int arch_test_and_clear_bit(unsigned long nr,
+ volatile unsigned long *addr)
{
return test_and_clear_bits(BIT_MASK(nr), addr + BIT_WORD(nr)) != 0;
}
-static __inline__ int test_and_change_bit(unsigned long nr,
- volatile unsigned long *addr)
+static inline int arch_test_and_change_bit(unsigned long nr,
+ volatile unsigned long *addr)
{
return test_and_change_bits(BIT_MASK(nr), addr + BIT_WORD(nr)) != 0;
}
#ifdef CONFIG_PPC64
-static __inline__ unsigned long clear_bit_unlock_return_word(int nr,
- volatile unsigned long *addr)
+static inline unsigned long
+clear_bit_unlock_return_word(int nr, volatile unsigned long *addr)
{
unsigned long old, t;
unsigned long *p = (unsigned long *)addr + BIT_WORD(nr);
return old;
}
-/* This is a special function for mm/filemap.c */
-#define clear_bit_unlock_is_negative_byte(nr, addr) \
- (clear_bit_unlock_return_word(nr, addr) & BIT_MASK(PG_waiters))
+/*
+ * This is a special function for mm/filemap.c
+ * Bit 7 corresponds to PG_waiters.
+ */
+#define arch_clear_bit_unlock_is_negative_byte(nr, addr) \
+ (clear_bit_unlock_return_word(nr, addr) & BIT_MASK(7))
#endif /* CONFIG_PPC64 */
#include <asm-generic/bitops/non-atomic.h>
-static __inline__ void __clear_bit_unlock(int nr, volatile unsigned long *addr)
+static inline void arch___clear_bit_unlock(int nr, volatile unsigned long *addr)
{
__asm__ __volatile__(PPC_RELEASE_BARRIER "" ::: "memory");
__clear_bit(nr, addr);
* fls: find last (most-significant) bit set.
* Note fls(0) = 0, fls(1) = 1, fls(0x80000000) = 32.
*/
-static __inline__ int fls(unsigned int x)
+static inline int fls(unsigned int x)
{
return 32 - __builtin_clz(x);
}
#include <asm-generic/bitops/builtin-__fls.h>
-static __inline__ int fls64(__u64 x)
+static inline int fls64(__u64 x)
{
return 64 - __builtin_clzll(x);
}
#include <asm-generic/bitops/find.h>
+/* wrappers that deal with KASAN instrumentation */
+#include <asm-generic/bitops/instrumented-atomic.h>
+#include <asm-generic/bitops/instrumented-lock.h>
+
/* Little-endian versions */
#include <asm-generic/bitops/le.h>
__s64 wtom_clock_sec; /* Wall to monotonic clock sec */
__s64 stamp_xtime_sec; /* xtime secs as at tb_orig_stamp */
__s64 stamp_xtime_nsec; /* xtime nsecs as at tb_orig_stamp */
+ __u32 hrtimer_res; /* hrtimer resolution */
__u32 syscall_map_64[SYSCALL_MAP_SIZE]; /* map of syscalls */
__u32 syscall_map_32[SYSCALL_MAP_SIZE]; /* map of syscalls */
};
__s32 stamp_xtime_sec; /* xtime seconds as at tb_orig_stamp */
__s32 stamp_xtime_nsec; /* xtime nsecs as at tb_orig_stamp */
__u32 stamp_sec_fraction; /* fractional seconds of stamp_xtime */
+ __u32 hrtimer_res; /* hrtimer resolution */
__u32 syscall_map_32[SYSCALL_MAP_SIZE]; /* map of syscalls */
__u32 dcache_block_size; /* L1 d-cache block size */
__u32 icache_block_size; /* L1 i-cache block size */
OFFSET(STAMP_XTIME_SEC, vdso_data, stamp_xtime_sec);
OFFSET(STAMP_XTIME_NSEC, vdso_data, stamp_xtime_nsec);
OFFSET(STAMP_SEC_FRAC, vdso_data, stamp_sec_fraction);
+ OFFSET(CLOCK_HRTIMER_RES, vdso_data, hrtimer_res);
OFFSET(CFG_ICACHE_BLOCKSZ, vdso_data, icache_block_size);
OFFSET(CFG_DCACHE_BLOCKSZ, vdso_data, dcache_block_size);
OFFSET(CFG_ICACHE_LOGBLOCKSZ, vdso_data, icache_log_block_size);
DEFINE(CLOCK_REALTIME_COARSE, CLOCK_REALTIME_COARSE);
DEFINE(CLOCK_MONOTONIC_COARSE, CLOCK_MONOTONIC_COARSE);
DEFINE(NSEC_PER_SEC, NSEC_PER_SEC);
- DEFINE(CLOCK_REALTIME_RES, MONOTONIC_RES_NSEC);
#ifdef CONFIG_BUG
DEFINE(BUG_ENTRY_SIZE, sizeof(struct bug_entry));
bl early_init
+#ifdef CONFIG_KASAN
+ bl kasan_early_init
+#endif
#ifdef CONFIG_RELOCATABLE
mr r3,r30
mr r4,r31
/*
* Decide what sort of machine this is and initialize the MMU.
*/
-#ifdef CONFIG_KASAN
- bl kasan_early_init
-#endif
mr r3,r30
mr r4,r31
bl machine_init
vdso_data->stamp_xtime_sec = xt.tv_sec;
vdso_data->stamp_xtime_nsec = xt.tv_nsec;
vdso_data->stamp_sec_fraction = frac_sec;
+ vdso_data->hrtimer_res = hrtimer_resolution;
smp_wmb();
++(vdso_data->tb_update_count);
}
cror cr0*4+eq,cr0*4+eq,cr1*4+eq
bne cr0,99f
+ mflr r12
+ .cfi_register lr,r12
+ bl __get_datapage@local /* get data page */
+ lwz r5, CLOCK_HRTIMER_RES(r3)
+ mtlr r12
li r3,0
cmpli cr0,r4,0
crclr cr0*4+so
beqlr
- lis r5,CLOCK_REALTIME_RES@h
- ori r5,r5,CLOCK_REALTIME_RES@l
stw r3,TSPC32_TV_SEC(r4)
stw r5,TSPC32_TV_NSEC(r4)
blr
cror cr0*4+eq,cr0*4+eq,cr1*4+eq
bne cr0,99f
+ mflr r12
+ .cfi_register lr,r12
+ bl V_LOCAL_FUNC(__get_datapage)
+ lwz r5, CLOCK_HRTIMER_RES(r3)
+ mtlr r12
li r3,0
cmpldi cr0,r4,0
crclr cr0*4+so
beqlr
- lis r5,CLOCK_REALTIME_RES@h
- ori r5,r5,CLOCK_REALTIME_RES@l
std r3,TSPC64_TV_SEC(r4)
std r5,TSPC64_TV_NSEC(r4)
blr
unsigned long start = (unsigned long) addr;
flush_dcache_range(start, start + size);
}
-EXPORT_SYMBOL(arch_wb_cache_pmem);
+EXPORT_SYMBOL_GPL(arch_wb_cache_pmem);
void arch_invalidate_pmem(void *addr, size_t size)
{
unsigned long start = (unsigned long) addr;
flush_dcache_range(start, start + size);
}
-EXPORT_SYMBOL(arch_invalidate_pmem);
+EXPORT_SYMBOL_GPL(arch_invalidate_pmem);
/*
* CONFIG_ARCH_HAS_UACCESS_FLUSHCACHE symbols
unsigned long i;
for (i = start; i < stop; i += chunk) {
- flush_dcache_range(i, min(stop, start + chunk));
+ flush_dcache_range(i, min(stop, i + chunk));
cond_resched();
}
}
imc_debugfs_parent = debugfs_create_dir("imc", powerpc_debugfs_root);
- /*
- * Return here, either because 'imc' directory already exists,
- * Or failed to create a new one.
- */
if (!imc_debugfs_parent)
return;
}
pmu_ptr->imc_counter_mmaped = true;
- export_imc_mode_and_cmd(node, pmu_ptr);
kfree(base_addr_arr);
kfree(chipid_arr);
return 0;
* and domain as the inputs.
* Allocates memory for the struct imc_pmu, sets up its domain, size and offsets
*/
-static int imc_pmu_create(struct device_node *parent, int pmu_index, int domain)
+static struct imc_pmu *imc_pmu_create(struct device_node *parent, int pmu_index, int domain)
{
int ret = 0;
struct imc_pmu *pmu_ptr;
/* Return for unknown domain */
if (domain < 0)
- return -EINVAL;
+ return NULL;
/* memory for pmu */
pmu_ptr = kzalloc(sizeof(*pmu_ptr), GFP_KERNEL);
if (!pmu_ptr)
- return -ENOMEM;
+ return NULL;
/* Set the domain */
pmu_ptr->domain = domain;
ret = of_property_read_u32(parent, "size", &pmu_ptr->counter_mem_size);
- if (ret) {
- ret = -EINVAL;
+ if (ret)
goto free_pmu;
- }
if (!of_property_read_u32(parent, "offset", &offset)) {
- if (imc_get_mem_addr_nest(parent, pmu_ptr, offset)) {
- ret = -EINVAL;
+ if (imc_get_mem_addr_nest(parent, pmu_ptr, offset))
goto free_pmu;
- }
}
/* Function to register IMC pmu */
if (pmu_ptr->domain == IMC_DOMAIN_NEST)
kfree(pmu_ptr->mem_info);
kfree(pmu_ptr);
- return ret;
+ return NULL;
}
- return 0;
+ return pmu_ptr;
free_pmu:
kfree(pmu_ptr);
- return ret;
+ return NULL;
}
static void disable_nest_pmu_counters(void)
static int opal_imc_counters_probe(struct platform_device *pdev)
{
struct device_node *imc_dev = pdev->dev.of_node;
+ struct imc_pmu *pmu;
int pmu_count = 0, domain;
bool core_imc_reg = false, thread_imc_reg = false;
u32 type;
}
for_each_compatible_node(imc_dev, NULL, IMC_DTB_UNIT_COMPAT) {
+ pmu = NULL;
if (of_property_read_u32(imc_dev, "type", &type)) {
pr_warn("IMC Device without type property\n");
continue;
domain = IMC_DOMAIN_THREAD;
break;
case IMC_TYPE_TRACE:
- domain = IMC_DOMAIN_TRACE;
+ /*
+ * FIXME. Using trace_imc events to monitor application
+ * or KVM thread performance can cause a checkstop
+ * (system crash).
+ * Disable it for now.
+ */
+ pr_info_once("IMC: disabling trace_imc PMU\n");
+ domain = -1;
break;
default:
pr_warn("IMC Unknown Device type \n");
break;
}
- if (!imc_pmu_create(imc_dev, pmu_count, domain)) {
- if (domain == IMC_DOMAIN_NEST)
+ pmu = imc_pmu_create(imc_dev, pmu_count, domain);
+ if (pmu != NULL) {
+ if (domain == IMC_DOMAIN_NEST) {
+ if (!imc_debugfs_parent)
+ export_imc_mode_and_cmd(imc_dev, pmu);
pmu_count++;
+ }
if (domain == IMC_DOMAIN_CORE)
core_imc_reg = true;
if (domain == IMC_DOMAIN_THREAD)
}
}
- /* If none of the nest units are registered, remove debugfs interface */
- if (pmu_count == 0)
- debugfs_remove_recursive(imc_debugfs_parent);
-
/* If core imc is not registered, unregister thread-imc */
if (!core_imc_reg && thread_imc_reg)
unregister_thread_imc();
data->esb_shift = esb_shift;
data->trig_page = trig_page;
+ data->hw_irq = hw_irq;
+
/*
* No chip-id for the sPAPR backend. This has an impact how we
* pick a target. See xive_pick_irq_target().
*/
data->src_chip = XIVE_INVALID_CHIP_ID;
+ /*
+ * When the H_INT_ESB flag is set, the H_INT_ESB hcall should
+ * be used for interrupt management. Skip the remapping of the
+ * ESB pages which are not available.
+ */
+ if (data->flags & XIVE_IRQ_FLAG_H_INT_ESB)
+ return 0;
+
data->eoi_mmio = ioremap(data->eoi_page, 1u << data->esb_shift);
if (!data->eoi_mmio) {
pr_err("Failed to map EOI page for irq 0x%x\n", hw_irq);
return -ENOMEM;
}
- data->hw_irq = hw_irq;
-
/* Full function page supports trigger */
if (flags & XIVE_SRC_TRIGGER) {
data->trig_mmio = data->eoi_mmio;
arch___clear_bit(nr, ptr);
}
-#include <asm-generic/bitops-instrumented.h>
+#include <asm-generic/bitops/instrumented-atomic.h>
+#include <asm-generic/bitops/instrumented-non-atomic.h>
+#include <asm-generic/bitops/instrumented-lock.h>
/*
* Functions which use MSB0 bit numbering.
#include <asm-generic/bitops/const_hweight.h>
-#include <asm-generic/bitops-instrumented.h>
+#include <asm-generic/bitops/instrumented-atomic.h>
+#include <asm-generic/bitops/instrumented-non-atomic.h>
+#include <asm-generic/bitops/instrumented-lock.h>
#include <asm-generic/bitops/le.h>
{
struct bfq_group *bfqg = blkg_to_bfqg(rq->bio->bi_blkg);
+ if (!bfqg)
+ return;
+
blkg_rwstat_add(&bfqg->stats.bytes, rq->cmd_flags, blk_rq_bytes(rq));
blkg_rwstat_add(&bfqg->stats.ios, rq->cmd_flags, 1);
}
* Description: Used to free the integrity portion of a bio. Usually
* called from bio_free().
*/
-static void bio_integrity_free(struct bio *bio)
+void bio_integrity_free(struct bio *bio)
{
struct bio_integrity_payload *bip = bio_integrity(bio);
struct bio_set *bs = bio->bi_pool;
void bio_uninit(struct bio *bio)
{
bio_disassociate_blkg(bio);
+
+ if (bio_integrity(bio))
+ bio_integrity_free(bio);
}
EXPORT_SYMBOL(bio_uninit);
}
EXPORT_SYMBOL_GPL(__blk_req_zone_write_unlock);
-static inline unsigned int __blkdev_nr_zones(struct request_queue *q,
- sector_t nr_sectors)
-{
- sector_t zone_sectors = blk_queue_zone_sectors(q);
-
- return (nr_sectors + zone_sectors - 1) >> ilog2(zone_sectors);
-}
-
/**
* blkdev_nr_zones - Get number of zones
- * @bdev: Target block device
+ * @disk: Target gendisk
*
- * Description:
- * Return the total number of zones of a zoned block device.
- * For a regular block device, the number of zones is always 0.
+ * Return the total number of zones of a zoned block device. For a block
+ * device without zone capabilities, the number of zones is always 0.
*/
-unsigned int blkdev_nr_zones(struct block_device *bdev)
+unsigned int blkdev_nr_zones(struct gendisk *disk)
{
- struct request_queue *q = bdev_get_queue(bdev);
+ sector_t zone_sectors = blk_queue_zone_sectors(disk->queue);
- if (!blk_queue_is_zoned(q))
+ if (!blk_queue_is_zoned(disk->queue))
return 0;
-
- return __blkdev_nr_zones(q, get_capacity(bdev->bd_disk));
+ return (get_capacity(disk) + zone_sectors - 1) >> ilog2(zone_sectors);
}
EXPORT_SYMBOL_GPL(blkdev_nr_zones);
void blk_queue_free_zone_bitmaps(struct request_queue *q)
{
- kfree(q->seq_zones_bitmap);
- q->seq_zones_bitmap = NULL;
+ kfree(q->conv_zones_bitmap);
+ q->conv_zones_bitmap = NULL;
kfree(q->seq_zones_wlock);
q->seq_zones_wlock = NULL;
}
struct blk_revalidate_zone_args {
struct gendisk *disk;
- unsigned long *seq_zones_bitmap;
+ unsigned long *conv_zones_bitmap;
unsigned long *seq_zones_wlock;
+ unsigned int nr_zones;
+ sector_t zone_sectors;
sector_t sector;
};
struct blk_revalidate_zone_args *args = data;
struct gendisk *disk = args->disk;
struct request_queue *q = disk->queue;
- sector_t zone_sectors = blk_queue_zone_sectors(q);
sector_t capacity = get_capacity(disk);
/*
* All zones must have the same size, with the exception on an eventual
* smaller last zone.
*/
- if (zone->start + zone_sectors < capacity &&
- zone->len != zone_sectors) {
- pr_warn("%s: Invalid zoned device with non constant zone size\n",
- disk->disk_name);
- return false;
- }
+ if (zone->start == 0) {
+ if (zone->len == 0 || !is_power_of_2(zone->len)) {
+ pr_warn("%s: Invalid zoned device with non power of two zone size (%llu)\n",
+ disk->disk_name, zone->len);
+ return -ENODEV;
+ }
- if (zone->start + zone->len >= capacity &&
- zone->len > zone_sectors) {
- pr_warn("%s: Invalid zoned device with larger last zone size\n",
- disk->disk_name);
- return -ENODEV;
+ args->zone_sectors = zone->len;
+ args->nr_zones = (capacity + zone->len - 1) >> ilog2(zone->len);
+ } else if (zone->start + args->zone_sectors < capacity) {
+ if (zone->len != args->zone_sectors) {
+ pr_warn("%s: Invalid zoned device with non constant zone size\n",
+ disk->disk_name);
+ return -ENODEV;
+ }
+ } else {
+ if (zone->len > args->zone_sectors) {
+ pr_warn("%s: Invalid zoned device with larger last zone size\n",
+ disk->disk_name);
+ return -ENODEV;
+ }
}
/* Check for holes in the zone report */
/* Check zone type */
switch (zone->type) {
case BLK_ZONE_TYPE_CONVENTIONAL:
+ if (!args->conv_zones_bitmap) {
+ args->conv_zones_bitmap =
+ blk_alloc_zone_bitmap(q->node, args->nr_zones);
+ if (!args->conv_zones_bitmap)
+ return -ENOMEM;
+ }
+ set_bit(idx, args->conv_zones_bitmap);
+ break;
case BLK_ZONE_TYPE_SEQWRITE_REQ:
case BLK_ZONE_TYPE_SEQWRITE_PREF:
+ if (!args->seq_zones_wlock) {
+ args->seq_zones_wlock =
+ blk_alloc_zone_bitmap(q->node, args->nr_zones);
+ if (!args->seq_zones_wlock)
+ return -ENOMEM;
+ }
break;
default:
pr_warn("%s: Invalid zone type 0x%x at sectors %llu\n",
return -ENODEV;
}
- if (zone->type != BLK_ZONE_TYPE_CONVENTIONAL)
- set_bit(idx, args->seq_zones_bitmap);
-
args->sector += zone->len;
return 0;
}
-static int blk_update_zone_info(struct gendisk *disk, unsigned int nr_zones,
- struct blk_revalidate_zone_args *args)
-{
- /*
- * Ensure that all memory allocations in this context are done as
- * if GFP_NOIO was specified.
- */
- unsigned int noio_flag = memalloc_noio_save();
- struct request_queue *q = disk->queue;
- int ret;
-
- args->seq_zones_wlock = blk_alloc_zone_bitmap(q->node, nr_zones);
- if (!args->seq_zones_wlock)
- return -ENOMEM;
- args->seq_zones_bitmap = blk_alloc_zone_bitmap(q->node, nr_zones);
- if (!args->seq_zones_bitmap)
- return -ENOMEM;
-
- ret = disk->fops->report_zones(disk, 0, nr_zones,
- blk_revalidate_zone_cb, args);
- memalloc_noio_restore(noio_flag);
- return ret;
-}
-
/**
* blk_revalidate_disk_zones - (re)allocate and initialize zone bitmaps
* @disk: Target disk
*
* Helper function for low-level device drivers to (re) allocate and initialize
* a disk request queue zone bitmaps. This functions should normally be called
- * within the disk ->revalidate method. For BIO based queues, no zone bitmap
- * is allocated.
+ * within the disk ->revalidate method for blk-mq based drivers. For BIO based
+ * drivers only q->nr_zones needs to be updated so that the sysfs exposed value
+ * is correct.
*/
int blk_revalidate_disk_zones(struct gendisk *disk)
{
struct request_queue *q = disk->queue;
- unsigned int nr_zones = __blkdev_nr_zones(q, get_capacity(disk));
- struct blk_revalidate_zone_args args = { .disk = disk };
- int ret = 0;
+ struct blk_revalidate_zone_args args = {
+ .disk = disk,
+ };
+ unsigned int noio_flag;
+ int ret;
if (WARN_ON_ONCE(!blk_queue_is_zoned(q)))
return -EIO;
+ if (WARN_ON_ONCE(!queue_is_mq(q)))
+ return -EIO;
/*
- * BIO based queues do not use a scheduler so only q->nr_zones
- * needs to be updated so that the sysfs exposed value is correct.
+ * Ensure that all memory allocations in this context are done as if
+ * GFP_NOIO was specified.
*/
- if (!queue_is_mq(q)) {
- q->nr_zones = nr_zones;
- return 0;
- }
-
- if (nr_zones)
- ret = blk_update_zone_info(disk, nr_zones, &args);
+ noio_flag = memalloc_noio_save();
+ ret = disk->fops->report_zones(disk, 0, UINT_MAX,
+ blk_revalidate_zone_cb, &args);
+ memalloc_noio_restore(noio_flag);
/*
- * Install the new bitmaps, making sure the queue is stopped and
- * all I/Os are completed (i.e. a scheduler is not referencing the
- * bitmaps).
+ * Install the new bitmaps and update nr_zones only once the queue is
+ * stopped and all I/Os are completed (i.e. a scheduler is not
+ * referencing the bitmaps).
*/
blk_mq_freeze_queue(q);
if (ret >= 0) {
- q->nr_zones = nr_zones;
+ blk_queue_chunk_sectors(q, args.zone_sectors);
+ q->nr_zones = args.nr_zones;
swap(q->seq_zones_wlock, args.seq_zones_wlock);
- swap(q->seq_zones_bitmap, args.seq_zones_bitmap);
+ swap(q->conv_zones_bitmap, args.conv_zones_bitmap);
ret = 0;
} else {
pr_warn("%s: failed to revalidate zones\n", disk->disk_name);
blk_mq_unfreeze_queue(q);
kfree(args.seq_zones_wlock);
- kfree(args.seq_zones_bitmap);
+ kfree(args.conv_zones_bitmap);
return ret;
}
EXPORT_SYMBOL_GPL(blk_revalidate_disk_zones);
-
#ifdef CONFIG_BLK_DEV_INTEGRITY
void blk_flush_integrity(void);
bool __bio_integrity_endio(struct bio *);
+void bio_integrity_free(struct bio *bio);
static inline bool bio_integrity_endio(struct bio *bio)
{
if (bio_integrity(bio))
{
return true;
}
+static inline void bio_integrity_free(struct bio *bio)
+{
+}
#endif /* CONFIG_BLK_DEV_INTEGRITY */
unsigned long blk_rq_timeout(unsigned long timeout);
case BLKGETZONESZ:
return put_uint(arg, bdev_zone_sectors(bdev));
case BLKGETNRZONES:
- return put_uint(arg, blkdev_nr_zones(bdev));
+ return put_uint(arg, blkdev_nr_zones(bdev->bd_disk));
case HDIO_GETGEO:
return blkdev_getgeo(bdev, argp);
case BLKRAGET:
unsigned int len = bvec.bv_len;
int err;
+ /* Don't support un-aligned buffer */
+ WARN_ON_ONCE((bvec.bv_offset & (SECTOR_SIZE - 1)) ||
+ (len & (SECTOR_SIZE - 1)));
+
err = brd_do_bvec(brd, bvec.bv_page, len, bvec.bv_offset,
bio_op(bio), sector);
if (err)
goto out_free_dev;
blk_queue_make_request(brd->brd_queue, brd_make_request);
- blk_queue_max_hw_sectors(brd->brd_queue, 1024);
/* This is so fdisk will align partitions on 4k, because of
* direct_access API needing 4k alignment, returning a PFN
static int null_gendisk_register(struct nullb *nullb)
{
+ sector_t size = ((sector_t)nullb->dev->size * SZ_1M) >> SECTOR_SHIFT;
struct gendisk *disk;
- sector_t size;
disk = nullb->disk = alloc_disk_node(1, nullb->dev->home_node);
if (!disk)
return -ENOMEM;
- size = (sector_t)nullb->dev->size * 1024 * 1024ULL;
- set_capacity(disk, size >> 9);
+ set_capacity(disk, size);
disk->flags |= GENHD_FL_EXT_DEVT | GENHD_FL_SUPPRESS_PARTITION_INFO;
disk->major = null_major;
disk->queue = nullb->q;
strncpy(disk->disk_name, nullb->disk_name, DISK_NAME_LEN);
+#ifdef CONFIG_BLK_DEV_ZONED
if (nullb->dev->zoned) {
- int ret = blk_revalidate_disk_zones(disk);
-
- if (ret != 0)
- return ret;
+ if (queue_is_mq(nullb->q)) {
+ int ret = blk_revalidate_disk_zones(disk);
+ if (ret)
+ return ret;
+ } else {
+ blk_queue_chunk_sectors(nullb->q,
+ nullb->dev->zone_size_sects);
+ nullb->q->nr_zones = blkdev_nr_zones(disk);
+ }
}
+#endif
add_disk(disk);
return 0;
return blk_mq_alloc_tag_set(set);
}
-static void null_validate_conf(struct nullb_device *dev)
+static int null_validate_conf(struct nullb_device *dev)
{
dev->blocksize = round_down(dev->blocksize, 512);
dev->blocksize = clamp_t(unsigned int, dev->blocksize, 512, 4096);
/* can not stop a queue */
if (dev->queue_mode == NULL_Q_BIO)
dev->mbps = 0;
+
+ if (dev->zoned &&
+ (!dev->zone_size || !is_power_of_2(dev->zone_size))) {
+ pr_err("zone_size must be power-of-two\n");
+ return -EINVAL;
+ }
+
+ return 0;
}
#ifdef CONFIG_BLK_DEV_NULL_BLK_FAULT_INJECTION
struct nullb *nullb;
int rv;
- null_validate_conf(dev);
+ rv = null_validate_conf(dev);
+ if (rv)
+ return rv;
nullb = kzalloc_node(sizeof(*nullb), GFP_KERNEL, dev->home_node);
if (!nullb) {
if (rv)
goto out_cleanup_blk_queue;
- blk_queue_chunk_sectors(nullb->q, dev->zone_size_sects);
nullb->q->limits.zoned = BLK_ZONED_HM;
blk_queue_flag_set(QUEUE_FLAG_ZONE_RESETALL, nullb->q);
blk_queue_required_elevator_features(nullb->q,
g_bs = PAGE_SIZE;
}
- if (!is_power_of_2(g_zone_size)) {
- pr_err("zone_size must be power-of-two\n");
- return -EINVAL;
- }
-
if (g_home_node != NUMA_NO_NODE && g_home_node >= nr_online_nodes) {
pr_err("invalid home_node value\n");
g_home_node = NUMA_NO_NODE;
#include <linux/ceph/cls_lock_client.h>
#include <linux/ceph/striper.h>
#include <linux/ceph/decode.h>
-#include <linux/parser.h>
+#include <linux/fs_parser.h>
#include <linux/bsearch.h>
#include <linux/kernel.h>
struct rbd_mapping {
u64 size;
- u64 features;
};
/*
* by rbd_dev->lock
*/
enum rbd_dev_flags {
- RBD_DEV_FLAG_EXISTS, /* mapped snapshot has not been deleted */
+ RBD_DEV_FLAG_EXISTS, /* rbd_dev_device_setup() ran */
RBD_DEV_FLAG_REMOVING, /* this mapping is being removed */
+ RBD_DEV_FLAG_READONLY, /* -o ro or snapshot */
};
static DEFINE_MUTEX(client_mutex); /* Serialize client creation */
return minor >> RBD_SINGLE_MAJOR_PART_SHIFT;
}
+static bool rbd_is_ro(struct rbd_device *rbd_dev)
+{
+ return test_bit(RBD_DEV_FLAG_READONLY, &rbd_dev->flags);
+}
+
+static bool rbd_is_snap(struct rbd_device *rbd_dev)
+{
+ return rbd_dev->spec->snap_id != CEPH_NOSNAP;
+}
+
static bool __rbd_is_lock_owner(struct rbd_device *rbd_dev)
{
lockdep_assert_held(&rbd_dev->lock_rwsem);
u64 snap_id);
static int _rbd_dev_v2_snap_size(struct rbd_device *rbd_dev, u64 snap_id,
u8 *order, u64 *snap_size);
-static int _rbd_dev_v2_snap_features(struct rbd_device *rbd_dev, u64 snap_id,
- u64 *snap_features);
static int rbd_dev_v2_get_flags(struct rbd_device *rbd_dev);
static void rbd_obj_handle_request(struct rbd_obj_request *obj_req, int result);
if (get_user(ro, (int __user *)arg))
return -EFAULT;
- /* Snapshots can't be marked read-write */
- if (rbd_dev->spec->snap_id != CEPH_NOSNAP && !ro)
- return -EROFS;
+ /*
+ * Both images mapped read-only and snapshots can't be marked
+ * read-write.
+ */
+ if (!ro) {
+ if (rbd_is_ro(rbd_dev))
+ return -EROFS;
+
+ rbd_assert(!rbd_is_snap(rbd_dev));
+ }
/* Let blkdev_roset() handle it */
return -ENOTTY;
Opt_queue_depth,
Opt_alloc_size,
Opt_lock_timeout,
- Opt_last_int,
/* int args above */
Opt_pool_ns,
- Opt_last_string,
/* string args above */
Opt_read_only,
Opt_read_write,
Opt_lock_on_read,
Opt_exclusive,
Opt_notrim,
- Opt_err
};
-static match_table_t rbd_opts_tokens = {
- {Opt_queue_depth, "queue_depth=%d"},
- {Opt_alloc_size, "alloc_size=%d"},
- {Opt_lock_timeout, "lock_timeout=%d"},
- /* int args above */
- {Opt_pool_ns, "_pool_ns=%s"},
- /* string args above */
- {Opt_read_only, "read_only"},
- {Opt_read_only, "ro"}, /* Alternate spelling */
- {Opt_read_write, "read_write"},
- {Opt_read_write, "rw"}, /* Alternate spelling */
- {Opt_lock_on_read, "lock_on_read"},
- {Opt_exclusive, "exclusive"},
- {Opt_notrim, "notrim"},
- {Opt_err, NULL}
+static const struct fs_parameter_spec rbd_param_specs[] = {
+ fsparam_u32 ("alloc_size", Opt_alloc_size),
+ fsparam_flag ("exclusive", Opt_exclusive),
+ fsparam_flag ("lock_on_read", Opt_lock_on_read),
+ fsparam_u32 ("lock_timeout", Opt_lock_timeout),
+ fsparam_flag ("notrim", Opt_notrim),
+ fsparam_string ("_pool_ns", Opt_pool_ns),
+ fsparam_u32 ("queue_depth", Opt_queue_depth),
+ fsparam_flag ("read_only", Opt_read_only),
+ fsparam_flag ("read_write", Opt_read_write),
+ fsparam_flag ("ro", Opt_read_only),
+ fsparam_flag ("rw", Opt_read_write),
+ {}
+};
+
+static const struct fs_parameter_description rbd_parameters = {
+ .name = "rbd",
+ .specs = rbd_param_specs,
};
struct rbd_options {
#define RBD_EXCLUSIVE_DEFAULT false
#define RBD_TRIM_DEFAULT true
-struct parse_rbd_opts_ctx {
+struct rbd_parse_opts_ctx {
struct rbd_spec *spec;
+ struct ceph_options *copts;
struct rbd_options *opts;
};
-static int parse_rbd_opts_token(char *c, void *private)
-{
- struct parse_rbd_opts_ctx *pctx = private;
- substring_t argstr[MAX_OPT_ARGS];
- int token, intval, ret;
-
- token = match_token(c, rbd_opts_tokens, argstr);
- if (token < Opt_last_int) {
- ret = match_int(&argstr[0], &intval);
- if (ret < 0) {
- pr_err("bad option arg (not int) at '%s'\n", c);
- return ret;
- }
- dout("got int token %d val %d\n", token, intval);
- } else if (token > Opt_last_int && token < Opt_last_string) {
- dout("got string token %d val %s\n", token, argstr[0].from);
- } else {
- dout("got token %d\n", token);
- }
-
- switch (token) {
- case Opt_queue_depth:
- if (intval < 1) {
- pr_err("queue_depth out of range\n");
- return -EINVAL;
- }
- pctx->opts->queue_depth = intval;
- break;
- case Opt_alloc_size:
- if (intval < SECTOR_SIZE) {
- pr_err("alloc_size out of range\n");
- return -EINVAL;
- }
- if (!is_power_of_2(intval)) {
- pr_err("alloc_size must be a power of 2\n");
- return -EINVAL;
- }
- pctx->opts->alloc_size = intval;
- break;
- case Opt_lock_timeout:
- /* 0 is "wait forever" (i.e. infinite timeout) */
- if (intval < 0 || intval > INT_MAX / 1000) {
- pr_err("lock_timeout out of range\n");
- return -EINVAL;
- }
- pctx->opts->lock_timeout = msecs_to_jiffies(intval * 1000);
- break;
- case Opt_pool_ns:
- kfree(pctx->spec->pool_ns);
- pctx->spec->pool_ns = match_strdup(argstr);
- if (!pctx->spec->pool_ns)
- return -ENOMEM;
- break;
- case Opt_read_only:
- pctx->opts->read_only = true;
- break;
- case Opt_read_write:
- pctx->opts->read_only = false;
- break;
- case Opt_lock_on_read:
- pctx->opts->lock_on_read = true;
- break;
- case Opt_exclusive:
- pctx->opts->exclusive = true;
- break;
- case Opt_notrim:
- pctx->opts->trim = false;
- break;
- default:
- /* libceph prints "bad option" msg */
- return -EINVAL;
- }
-
- return 0;
-}
-
static char* obj_op_name(enum obj_operation_type op_type)
{
switch (op_type) {
return 0;
}
-static int rbd_snap_features(struct rbd_device *rbd_dev, u64 snap_id,
- u64 *snap_features)
-{
- rbd_assert(rbd_image_format_valid(rbd_dev->image_format));
- if (snap_id == CEPH_NOSNAP) {
- *snap_features = rbd_dev->header.features;
- } else if (rbd_dev->image_format == 1) {
- *snap_features = 0; /* No features for format 1 */
- } else {
- u64 features = 0;
- int ret;
-
- ret = _rbd_dev_v2_snap_features(rbd_dev, snap_id, &features);
- if (ret)
- return ret;
-
- *snap_features = features;
- }
- return 0;
-}
-
static int rbd_dev_mapping_set(struct rbd_device *rbd_dev)
{
u64 snap_id = rbd_dev->spec->snap_id;
u64 size = 0;
- u64 features = 0;
int ret;
ret = rbd_snap_size(rbd_dev, snap_id, &size);
if (ret)
return ret;
- ret = rbd_snap_features(rbd_dev, snap_id, &features);
- if (ret)
- return ret;
rbd_dev->mapping.size = size;
- rbd_dev->mapping.features = features;
-
return 0;
}
static void rbd_dev_mapping_clear(struct rbd_device *rbd_dev)
{
rbd_dev->mapping.size = 0;
- rbd_dev->mapping.features = 0;
}
static void zero_bvec(struct bio_vec *bv)
static bool use_object_map(struct rbd_device *rbd_dev)
{
+ /*
+ * An image mapped read-only can't use the object map -- it isn't
+ * loaded because the header lock isn't acquired. Someone else can
+ * write to the image and update the object map behind our back.
+ *
+ * A snapshot can't be written to, so using the object map is always
+ * safe.
+ */
+ if (!rbd_is_snap(rbd_dev) && rbd_is_ro(rbd_dev))
+ return false;
+
return ((rbd_dev->header.features & RBD_FEATURE_OBJECT_MAP) &&
!(rbd_dev->object_map_flags & RBD_FLAG_OBJECT_MAP_INVALID));
}
if (!(rbd_dev->header.features & RBD_FEATURE_EXCLUSIVE_LOCK))
return false;
- if (rbd_dev->spec->snap_id != CEPH_NOSNAP)
+ if (rbd_is_ro(rbd_dev))
return false;
rbd_assert(!test_bit(IMG_REQ_CHILD, &img_req->flags));
* lock owner acked, but resend if we don't see them
* release the lock
*/
- dout("%s rbd_dev %p requeueing lock_dwork\n", __func__,
+ dout("%s rbd_dev %p requeuing lock_dwork\n", __func__,
rbd_dev);
mod_delayed_work(rbd_dev->task_wq, &rbd_dev->lock_dwork,
msecs_to_jiffies(2 * RBD_NOTIFY_TIMEOUT * MSEC_PER_SEC));
goto err_rq;
}
- if (op_type != OBJ_OP_READ && rbd_dev->spec->snap_id != CEPH_NOSNAP) {
- rbd_warn(rbd_dev, "%s on read-only snapshot",
- obj_op_name(op_type));
- result = -EIO;
- goto err;
- }
-
- /*
- * Quit early if the mapped snapshot no longer exists. It's
- * still possible the snapshot will have disappeared by the
- * time our request arrives at the osd, but there's no sense in
- * sending it if we already know.
- */
- if (!test_bit(RBD_DEV_FLAG_EXISTS, &rbd_dev->flags)) {
- dout("request for non-existent snapshot");
- rbd_assert(rbd_dev->spec->snap_id != CEPH_NOSNAP);
- result = -ENXIO;
- goto err_rq;
+ if (op_type != OBJ_OP_READ) {
+ if (rbd_is_ro(rbd_dev)) {
+ rbd_warn(rbd_dev, "%s on read-only mapping",
+ obj_op_name(op_type));
+ result = -EIO;
+ goto err;
+ }
+ rbd_assert(!rbd_is_snap(rbd_dev));
}
if (offset && length > U64_MAX - offset + 1) {
return ret;
}
-/*
- * Clear the rbd device's EXISTS flag if the snapshot it's mapped to
- * has disappeared from the (just updated) snapshot context.
- */
-static void rbd_exists_validate(struct rbd_device *rbd_dev)
-{
- u64 snap_id;
-
- if (!test_bit(RBD_DEV_FLAG_EXISTS, &rbd_dev->flags))
- return;
-
- snap_id = rbd_dev->spec->snap_id;
- if (snap_id == CEPH_NOSNAP)
- return;
-
- if (rbd_dev_snap_index(rbd_dev, snap_id) == BAD_SNAP_INDEX)
- clear_bit(RBD_DEV_FLAG_EXISTS, &rbd_dev->flags);
-}
-
static void rbd_dev_update_size(struct rbd_device *rbd_dev)
{
sector_t size;
goto out;
}
- if (rbd_dev->spec->snap_id == CEPH_NOSNAP) {
- rbd_dev->mapping.size = rbd_dev->header.image_size;
- } else {
- /* validate mapped snapshot's EXISTS flag */
- rbd_exists_validate(rbd_dev);
- }
+ rbd_assert(!rbd_is_snap(rbd_dev));
+ rbd_dev->mapping.size = rbd_dev->header.image_size;
out:
up_write(&rbd_dev->header_rwsem);
(unsigned long long)rbd_dev->mapping.size);
}
-/*
- * Note this shows the features for whatever's mapped, which is not
- * necessarily the base image.
- */
static ssize_t rbd_features_show(struct device *dev,
struct device_attribute *attr, char *buf)
{
struct rbd_device *rbd_dev = dev_to_rbd_dev(dev);
- return sprintf(buf, "0x%016llx\n",
- (unsigned long long)rbd_dev->mapping.features);
+ return sprintf(buf, "0x%016llx\n", rbd_dev->header.features);
}
static ssize_t rbd_major_show(struct device *dev,
}
static int _rbd_dev_v2_snap_features(struct rbd_device *rbd_dev, u64 snap_id,
- u64 *snap_features)
+ bool read_only, u64 *snap_features)
{
- __le64 snapid = cpu_to_le64(snap_id);
+ struct {
+ __le64 snap_id;
+ u8 read_only;
+ } features_in;
struct {
__le64 features;
__le64 incompat;
u64 unsup;
int ret;
+ features_in.snap_id = cpu_to_le64(snap_id);
+ features_in.read_only = read_only;
+
ret = rbd_obj_method_sync(rbd_dev, &rbd_dev->header_oid,
&rbd_dev->header_oloc, "get_features",
- &snapid, sizeof(snapid),
+ &features_in, sizeof(features_in),
&features_buf, sizeof(features_buf));
dout("%s: rbd_obj_method_sync returned %d\n", __func__, ret);
if (ret < 0)
static int rbd_dev_v2_features(struct rbd_device *rbd_dev)
{
return _rbd_dev_v2_snap_features(rbd_dev, CEPH_NOSNAP,
- &rbd_dev->header.features);
+ rbd_is_ro(rbd_dev),
+ &rbd_dev->header.features);
}
/*
return dup;
}
+static int rbd_parse_param(struct fs_parameter *param,
+ struct rbd_parse_opts_ctx *pctx)
+{
+ struct rbd_options *opt = pctx->opts;
+ struct fs_parse_result result;
+ int token, ret;
+
+ ret = ceph_parse_param(param, pctx->copts, NULL);
+ if (ret != -ENOPARAM)
+ return ret;
+
+ token = fs_parse(NULL, &rbd_parameters, param, &result);
+ dout("%s fs_parse '%s' token %d\n", __func__, param->key, token);
+ if (token < 0) {
+ if (token == -ENOPARAM) {
+ return invalf(NULL, "rbd: Unknown parameter '%s'",
+ param->key);
+ }
+ return token;
+ }
+
+ switch (token) {
+ case Opt_queue_depth:
+ if (result.uint_32 < 1)
+ goto out_of_range;
+ opt->queue_depth = result.uint_32;
+ break;
+ case Opt_alloc_size:
+ if (result.uint_32 < SECTOR_SIZE)
+ goto out_of_range;
+ if (!is_power_of_2(result.uint_32)) {
+ return invalf(NULL, "rbd: alloc_size must be a power of 2");
+ }
+ opt->alloc_size = result.uint_32;
+ break;
+ case Opt_lock_timeout:
+ /* 0 is "wait forever" (i.e. infinite timeout) */
+ if (result.uint_32 > INT_MAX / 1000)
+ goto out_of_range;
+ opt->lock_timeout = msecs_to_jiffies(result.uint_32 * 1000);
+ break;
+ case Opt_pool_ns:
+ kfree(pctx->spec->pool_ns);
+ pctx->spec->pool_ns = param->string;
+ param->string = NULL;
+ break;
+ case Opt_read_only:
+ opt->read_only = true;
+ break;
+ case Opt_read_write:
+ opt->read_only = false;
+ break;
+ case Opt_lock_on_read:
+ opt->lock_on_read = true;
+ break;
+ case Opt_exclusive:
+ opt->exclusive = true;
+ break;
+ case Opt_notrim:
+ opt->trim = false;
+ break;
+ default:
+ BUG();
+ }
+
+ return 0;
+
+out_of_range:
+ return invalf(NULL, "rbd: %s out of range", param->key);
+}
+
+/*
+ * This duplicates most of generic_parse_monolithic(), untying it from
+ * fs_context and skipping standard superblock and security options.
+ */
+static int rbd_parse_options(char *options, struct rbd_parse_opts_ctx *pctx)
+{
+ char *key;
+ int ret = 0;
+
+ dout("%s '%s'\n", __func__, options);
+ while ((key = strsep(&options, ",")) != NULL) {
+ if (*key) {
+ struct fs_parameter param = {
+ .key = key,
+ .type = fs_value_is_string,
+ };
+ char *value = strchr(key, '=');
+ size_t v_len = 0;
+
+ if (value) {
+ if (value == key)
+ continue;
+ *value++ = 0;
+ v_len = strlen(value);
+ }
+
+
+ if (v_len > 0) {
+ param.string = kmemdup_nul(value, v_len,
+ GFP_KERNEL);
+ if (!param.string)
+ return -ENOMEM;
+ }
+ param.size = v_len;
+
+ ret = rbd_parse_param(¶m, pctx);
+ kfree(param.string);
+ if (ret)
+ break;
+ }
+ }
+
+ return ret;
+}
+
/*
* Parse the options provided for an "rbd add" (i.e., rbd image
* mapping) request. These arrive via a write to /sys/bus/rbd/add,
const char *mon_addrs;
char *snap_name;
size_t mon_addrs_size;
- struct parse_rbd_opts_ctx pctx = { 0 };
- struct ceph_options *copts;
+ struct rbd_parse_opts_ctx pctx = { 0 };
int ret;
/* The first four tokens are required */
return -EINVAL;
}
mon_addrs = buf;
- mon_addrs_size = len + 1;
+ mon_addrs_size = len;
buf += len;
ret = -EINVAL;
*(snap_name + len) = '\0';
pctx.spec->snap_name = snap_name;
+ pctx.copts = ceph_alloc_options();
+ if (!pctx.copts)
+ goto out_mem;
+
/* Initialize all rbd options to the defaults */
pctx.opts = kzalloc(sizeof(*pctx.opts), GFP_KERNEL);
pctx.opts->exclusive = RBD_EXCLUSIVE_DEFAULT;
pctx.opts->trim = RBD_TRIM_DEFAULT;
- copts = ceph_parse_options(options, mon_addrs,
- mon_addrs + mon_addrs_size - 1,
- parse_rbd_opts_token, &pctx);
- if (IS_ERR(copts)) {
- ret = PTR_ERR(copts);
+ ret = ceph_parse_mon_ips(mon_addrs, mon_addrs_size, pctx.copts, NULL);
+ if (ret)
+ goto out_err;
+
+ ret = rbd_parse_options(options, &pctx);
+ if (ret)
goto out_err;
- }
- kfree(options);
- *ceph_opts = copts;
+ *ceph_opts = pctx.copts;
*opts = pctx.opts;
*rbd_spec = pctx.spec;
-
+ kfree(options);
return 0;
+
out_mem:
ret = -ENOMEM;
out_err:
kfree(pctx.opts);
+ ceph_destroy_options(pctx.copts);
rbd_spec_put(pctx.spec);
kfree(options);
-
return ret;
}
return -EINVAL;
}
- if (rbd_dev->spec->snap_id != CEPH_NOSNAP)
+ if (rbd_is_ro(rbd_dev))
return 0;
rbd_assert(!rbd_is_lock_owner(rbd_dev));
__rbd_get_client(rbd_dev->rbd_client);
rbd_spec_get(rbd_dev->parent_spec);
+ __set_bit(RBD_DEV_FLAG_READONLY, &parent->flags);
+
ret = rbd_dev_image_probe(parent, depth);
if (ret < 0)
goto out_err;
goto err_out_blkdev;
set_capacity(rbd_dev->disk, rbd_dev->mapping.size / SECTOR_SIZE);
- set_disk_ro(rbd_dev->disk, rbd_dev->opts->read_only);
+ set_disk_ro(rbd_dev->disk, rbd_is_ro(rbd_dev));
ret = dev_set_name(&rbd_dev->dev, "%d", rbd_dev->dev_id);
if (ret)
return ret;
}
+static void rbd_print_dne(struct rbd_device *rbd_dev, bool is_snap)
+{
+ if (!is_snap) {
+ pr_info("image %s/%s%s%s does not exist\n",
+ rbd_dev->spec->pool_name,
+ rbd_dev->spec->pool_ns ?: "",
+ rbd_dev->spec->pool_ns ? "/" : "",
+ rbd_dev->spec->image_name);
+ } else {
+ pr_info("snap %s/%s%s%s@%s does not exist\n",
+ rbd_dev->spec->pool_name,
+ rbd_dev->spec->pool_ns ?: "",
+ rbd_dev->spec->pool_ns ? "/" : "",
+ rbd_dev->spec->image_name,
+ rbd_dev->spec->snap_name);
+ }
+}
+
static void rbd_dev_image_release(struct rbd_device *rbd_dev)
{
rbd_dev_unprobe(rbd_dev);
*/
static int rbd_dev_image_probe(struct rbd_device *rbd_dev, int depth)
{
+ bool need_watch = !rbd_is_ro(rbd_dev);
int ret;
/*
if (ret)
goto err_out_format;
- if (!depth) {
+ if (need_watch) {
ret = rbd_register_watch(rbd_dev);
if (ret) {
if (ret == -ENOENT)
- pr_info("image %s/%s%s%s does not exist\n",
- rbd_dev->spec->pool_name,
- rbd_dev->spec->pool_ns ?: "",
- rbd_dev->spec->pool_ns ? "/" : "",
- rbd_dev->spec->image_name);
+ rbd_print_dne(rbd_dev, false);
goto err_out_format;
}
}
ret = rbd_dev_header_info(rbd_dev);
- if (ret)
+ if (ret) {
+ if (ret == -ENOENT && !need_watch)
+ rbd_print_dne(rbd_dev, false);
goto err_out_watch;
+ }
/*
* If this image is the one being mapped, we have pool name and
ret = rbd_spec_fill_names(rbd_dev);
if (ret) {
if (ret == -ENOENT)
- pr_info("snap %s/%s%s%s@%s does not exist\n",
- rbd_dev->spec->pool_name,
- rbd_dev->spec->pool_ns ?: "",
- rbd_dev->spec->pool_ns ? "/" : "",
- rbd_dev->spec->image_name,
- rbd_dev->spec->snap_name);
+ rbd_print_dne(rbd_dev, true);
goto err_out_probe;
}
if (ret)
goto err_out_probe;
- if (rbd_dev->spec->snap_id != CEPH_NOSNAP &&
+ if (rbd_is_snap(rbd_dev) &&
(rbd_dev->header.features & RBD_FEATURE_OBJECT_MAP)) {
ret = rbd_object_map_load(rbd_dev);
if (ret)
err_out_probe:
rbd_dev_unprobe(rbd_dev);
err_out_watch:
- if (!depth)
+ if (need_watch)
rbd_unregister_watch(rbd_dev);
err_out_format:
rbd_dev->image_format = 0;
spec = NULL; /* rbd_dev now owns this */
rbd_opts = NULL; /* rbd_dev now owns this */
+ /* if we are mapping a snapshot it will be a read-only mapping */
+ if (rbd_dev->opts->read_only ||
+ strcmp(rbd_dev->spec->snap_name, RBD_SNAP_HEAD_NAME))
+ __set_bit(RBD_DEV_FLAG_READONLY, &rbd_dev->flags);
+
rbd_dev->config_info = kstrdup(buf, GFP_KERNEL);
if (!rbd_dev->config_info) {
rc = -ENOMEM;
goto err_out_rbd_dev;
}
- /* If we are mapping a snapshot it must be marked read-only */
- if (rbd_dev->spec->snap_id != CEPH_NOSNAP)
- rbd_dev->opts->read_only = true;
-
if (rbd_dev->opts->alloc_size > rbd_dev->layout.object_size) {
rbd_warn(rbd_dev, "alloc_size adjusted to %u",
rbd_dev->layout.object_size);
out_of_memory:
pr_alert("%s: out of memory\n", __func__);
put_free_pages(ring, pages_to_gnt, segs_to_map);
+ for (i = last_map; i < num; i++)
+ pages[i]->handle = BLKBACK_INVALID_HANDLE;
return -ENOMEM;
}
int size, pgprot_t page_prot)
{
struct agp_segment_priv *seg;
- int num_segments, i;
+ int i;
off_t pg_start;
size_t pg_count;
pg_start = offset / 4096;
pg_count = size / 4096;
seg = *(client->segments);
- num_segments = client->num_segments;
for (i = 0; i < client->num_segments; i++) {
if ((seg[i].pg_start == pg_start) &&
/**
* agp_allocate_memory - allocate a group of pages of a certain type.
*
+ * @bridge: an agp_bridge_data struct allocated for the AGP host bridge.
* @page_count: size_t argument of the number of pages
* @type: u32 argument of the type of memory to be allocated.
*
/**
* agp_copy_info - copy bridge state information
*
+ * @bridge: an agp_bridge_data struct allocated for the AGP host bridge.
* @info: agp_kern_info pointer. The caller should insure that this pointer is valid.
*
* This function copies information about the agp bridge device and the state of
{
char *table;
char *table_end;
- int size;
int page_order;
int num_entries;
int i;
table = NULL;
i = bridge->aperture_size_idx;
temp = bridge->current_size;
- size = page_order = num_entries = 0;
+ page_order = num_entries = 0;
if (bridge->driver->size_type != FIXED_APER_SIZE) {
do {
switch (bridge->driver->size_type) {
case U8_APER_SIZE:
- size = A_SIZE_8(temp)->size;
page_order =
A_SIZE_8(temp)->page_order;
num_entries =
A_SIZE_8(temp)->num_entries;
break;
case U16_APER_SIZE:
- size = A_SIZE_16(temp)->size;
page_order = A_SIZE_16(temp)->page_order;
num_entries = A_SIZE_16(temp)->num_entries;
break;
case U32_APER_SIZE:
- size = A_SIZE_32(temp)->size;
page_order = A_SIZE_32(temp)->page_order;
num_entries = A_SIZE_32(temp)->num_entries;
break;
case FIXED_APER_SIZE:
case LVL2_APER_SIZE:
default:
- size = page_order = num_entries = 0;
+ page_order = num_entries = 0;
break;
}
}
} while (!table && (i < bridge->driver->num_aperture_sizes));
} else {
- size = ((struct aper_size_info_fixed *) temp)->size;
page_order = ((struct aper_size_info_fixed *) temp)->page_order;
num_entries = ((struct aper_size_info_fixed *) temp)->num_entries;
table = alloc_gatt_pages(page_order);
/**
* agp_enable - initialise the agp point-to-point connection.
*
+ * @bridge: an agp_bridge_data struct allocated for the AGP host bridge.
* @mode: agp mode register value to configure with.
*/
void agp_enable(struct agp_bridge_data *bridge, u32 mode)
req, req_cnt * sizeof(*req), resp, sizeof(*resp));
}
+int __qcom_scm_ocmem_lock(struct device *dev, u32 id, u32 offset, u32 size,
+ u32 mode)
+{
+ struct ocmem_tz_lock {
+ __le32 id;
+ __le32 offset;
+ __le32 size;
+ __le32 mode;
+ } request;
+
+ request.id = cpu_to_le32(id);
+ request.offset = cpu_to_le32(offset);
+ request.size = cpu_to_le32(size);
+ request.mode = cpu_to_le32(mode);
+
+ return qcom_scm_call(dev, QCOM_SCM_OCMEM_SVC, QCOM_SCM_OCMEM_LOCK_CMD,
+ &request, sizeof(request), NULL, 0);
+}
+
+int __qcom_scm_ocmem_unlock(struct device *dev, u32 id, u32 offset, u32 size)
+{
+ struct ocmem_tz_unlock {
+ __le32 id;
+ __le32 offset;
+ __le32 size;
+ } request;
+
+ request.id = cpu_to_le32(id);
+ request.offset = cpu_to_le32(offset);
+ request.size = cpu_to_le32(size);
+
+ return qcom_scm_call(dev, QCOM_SCM_OCMEM_SVC, QCOM_SCM_OCMEM_UNLOCK_CMD,
+ &request, sizeof(request), NULL, 0);
+}
+
void __qcom_scm_init(void)
{
}
int __qcom_scm_restore_sec_cfg(struct device *dev, u32 device_id,
u32 spare)
{
- return -ENODEV;
+ struct msm_scm_sec_cfg {
+ __le32 id;
+ __le32 ctx_bank_num;
+ } cfg;
+ int ret, scm_ret = 0;
+
+ cfg.id = cpu_to_le32(device_id);
+ cfg.ctx_bank_num = cpu_to_le32(spare);
+
+ ret = qcom_scm_call(dev, QCOM_SCM_SVC_MP, QCOM_SCM_RESTORE_SEC_CFG,
+ &cfg, sizeof(cfg), &scm_ret, sizeof(scm_ret));
+
+ if (ret || scm_ret)
+ return ret ? ret : -EINVAL;
+
+ return 0;
}
int __qcom_scm_iommu_secure_ptbl_size(struct device *dev, u32 spare,
return ret;
}
+int __qcom_scm_ocmem_lock(struct device *dev, uint32_t id, uint32_t offset,
+ uint32_t size, uint32_t mode)
+{
+ return -ENOTSUPP;
+}
+
+int __qcom_scm_ocmem_unlock(struct device *dev, uint32_t id, uint32_t offset,
+ uint32_t size)
+{
+ return -ENOTSUPP;
+}
+
void __qcom_scm_init(void)
{
u64 cmd;
EXPORT_SYMBOL(qcom_scm_pas_supported);
/**
+ * qcom_scm_ocmem_lock_available() - is OCMEM lock/unlock interface available
+ */
+bool qcom_scm_ocmem_lock_available(void)
+{
+ return __qcom_scm_is_call_available(__scm->dev, QCOM_SCM_OCMEM_SVC,
+ QCOM_SCM_OCMEM_LOCK_CMD);
+}
+EXPORT_SYMBOL(qcom_scm_ocmem_lock_available);
+
+/**
+ * qcom_scm_ocmem_lock() - call OCMEM lock interface to assign an OCMEM
+ * region to the specified initiator
+ *
+ * @id: tz initiator id
+ * @offset: OCMEM offset
+ * @size: OCMEM size
+ * @mode: access mode (WIDE/NARROW)
+ */
+int qcom_scm_ocmem_lock(enum qcom_scm_ocmem_client id, u32 offset, u32 size,
+ u32 mode)
+{
+ return __qcom_scm_ocmem_lock(__scm->dev, id, offset, size, mode);
+}
+EXPORT_SYMBOL(qcom_scm_ocmem_lock);
+
+/**
+ * qcom_scm_ocmem_unlock() - call OCMEM unlock interface to release an OCMEM
+ * region from the specified initiator
+ *
+ * @id: tz initiator id
+ * @offset: OCMEM offset
+ * @size: OCMEM size
+ */
+int qcom_scm_ocmem_unlock(enum qcom_scm_ocmem_client id, u32 offset, u32 size)
+{
+ return __qcom_scm_ocmem_unlock(__scm->dev, id, offset, size);
+}
+EXPORT_SYMBOL(qcom_scm_ocmem_unlock);
+
+/**
* qcom_scm_pas_init_image() - Initialize peripheral authentication service
* state machine for a given peripheral, using the
* metadata
.deassert = qcom_scm_pas_reset_deassert,
};
+/**
+ * qcom_scm_restore_sec_cfg_available() - Check if secure environment
+ * supports restore security config interface.
+ *
+ * Return true if restore-cfg interface is supported, false if not.
+ */
+bool qcom_scm_restore_sec_cfg_available(void)
+{
+ return __qcom_scm_is_call_available(__scm->dev, QCOM_SCM_SVC_MP,
+ QCOM_SCM_RESTORE_SEC_CFG);
+}
+EXPORT_SYMBOL(qcom_scm_restore_sec_cfg_available);
+
int qcom_scm_restore_sec_cfg(u32 device_id, u32 spare)
{
return __qcom_scm_restore_sec_cfg(__scm->dev, device_id, spare);
extern void __qcom_scm_init(void);
+#define QCOM_SCM_OCMEM_SVC 0xf
+#define QCOM_SCM_OCMEM_LOCK_CMD 0x1
+#define QCOM_SCM_OCMEM_UNLOCK_CMD 0x2
+
+extern int __qcom_scm_ocmem_lock(struct device *dev, u32 id, u32 offset,
+ u32 size, u32 mode);
+extern int __qcom_scm_ocmem_unlock(struct device *dev, u32 id, u32 offset,
+ u32 size);
+
#define QCOM_SCM_SVC_PIL 0x2
#define QCOM_SCM_PAS_INIT_IMAGE_CMD 0x1
#define QCOM_SCM_PAS_MEM_SETUP_CMD 0x2
config DRM_DEBUG_DP_MST_TOPOLOGY_REFS
bool "Enable refcount backtrace history in the DP MST helpers"
+ depends on STACKTRACE_SUPPORT
select STACKDEPOT
depends on DRM_KMS_HELPER
depends on DEBUG_KERNEL
(kfd_mem_limit.max_ttm_mem_limit >> 20));
}
+/* Estimate page table size needed to represent a given memory size
+ *
+ * With 4KB pages, we need one 8 byte PTE for each 4KB of memory
+ * (factor 512, >> 9). With 2MB pages, we need one 8 byte PTE for 2MB
+ * of memory (factor 256K, >> 18). ROCm user mode tries to optimize
+ * for 2MB pages for TLB efficiency. However, small allocations and
+ * fragmented system memory still need some 4KB pages. We choose a
+ * compromise that should work in most cases without reserving too
+ * much memory for page tables unnecessarily (factor 16K, >> 14).
+ */
+#define ESTIMATE_PT_SIZE(mem_size) ((mem_size) >> 14)
+
static int amdgpu_amdkfd_reserve_mem_limit(struct amdgpu_device *adev,
uint64_t size, u32 domain, bool sg)
{
+ uint64_t reserved_for_pt =
+ ESTIMATE_PT_SIZE(amdgpu_amdkfd_total_mem_size);
size_t acc_size, system_mem_needed, ttm_mem_needed, vram_needed;
- uint64_t reserved_for_pt = amdgpu_amdkfd_total_mem_size >> 9;
int ret = 0;
acc_size = ttm_bo_dma_acc_size(&adev->mman.bdev, size,
return ret;
/* Start rlc autoload after psp recieved all the gfx firmware */
- if (psp->autoload_supported && ucode->ucode_id ==
- AMDGPU_UCODE_ID_RLC_RESTORE_LIST_SRM_MEM) {
+ if (psp->autoload_supported && ucode->ucode_id == (amdgpu_sriov_vf(adev) ?
+ AMDGPU_UCODE_ID_CP_MEC2 : AMDGPU_UCODE_ID_RLC_RESTORE_LIST_SRM_MEM)) {
ret = psp_rlc_autoload(psp);
if (ret) {
DRM_ERROR("Failed to start rlc autoload\n");
#include <linux/bits.h>
#include "smu_v11_0_i2c.h"
-#define EEPROM_I2C_TARGET_ADDR 0xA0
+#define EEPROM_I2C_TARGET_ADDR_ARCTURUS 0xA8
+#define EEPROM_I2C_TARGET_ADDR_VEGA20 0xA0
/*
* The 2 macros bellow represent the actual size in bytes that
{
int ret = 0;
struct i2c_msg msg = {
- .addr = EEPROM_I2C_TARGET_ADDR,
+ .addr = 0,
.flags = 0,
.len = EEPROM_ADDRESS_SIZE + EEPROM_TABLE_HEADER_SIZE,
.buf = buff,
*(uint16_t *)buff = EEPROM_HDR_START;
__encode_table_header_to_buff(&control->tbl_hdr, buff + EEPROM_ADDRESS_SIZE);
+ msg.addr = control->i2c_address;
+
ret = i2c_transfer(&control->eeprom_accessor, &msg, 1);
if (ret < 1)
DRM_ERROR("Failed to write EEPROM table header, ret:%d", ret);
unsigned char buff[EEPROM_ADDRESS_SIZE + EEPROM_TABLE_HEADER_SIZE] = { 0 };
struct amdgpu_ras_eeprom_table_header *hdr = &control->tbl_hdr;
struct i2c_msg msg = {
- .addr = EEPROM_I2C_TARGET_ADDR,
+ .addr = 0,
.flags = I2C_M_RD,
.len = EEPROM_ADDRESS_SIZE + EEPROM_TABLE_HEADER_SIZE,
.buf = buff,
switch (adev->asic_type) {
case CHIP_VEGA20:
+ control->i2c_address = EEPROM_I2C_TARGET_ADDR_VEGA20;
ret = smu_v11_0_i2c_eeprom_control_init(&control->eeprom_accessor);
break;
case CHIP_ARCTURUS:
+ control->i2c_address = EEPROM_I2C_TARGET_ADDR_ARCTURUS;
ret = smu_i2c_eeprom_init(&adev->smu, &control->eeprom_accessor);
break;
return ret;
}
+ msg.addr = control->i2c_address;
+
/* Read/Create table header from EEPROM address 0 */
ret = i2c_transfer(&control->eeprom_accessor, &msg, 1);
if (ret < 1) {
* Update bits 16,17 of EEPROM address in I2C address by setting them
* to bits 1,2 of Device address byte
*/
- msg->addr = EEPROM_I2C_TARGET_ADDR |
- ((control->next_addr & EEPROM_ADDR_MSB_MASK) >> 15);
+ msg->addr = control->i2c_address |
+ ((control->next_addr & EEPROM_ADDR_MSB_MASK) >> 15);
msg->flags = write ? 0 : I2C_M_RD;
msg->len = EEPROM_ADDRESS_SIZE + EEPROM_TABLE_RECORD_SIZE;
msg->buf = buff;
struct mutex tbl_mutex;
bool bus_locked;
uint32_t tbl_byte_sum;
+ uint16_t i2c_address; // 8-bit represented address
};
/*
*/
int amdgpu_gfx_rlc_init_csb(struct amdgpu_device *adev)
{
- volatile u32 *dst_ptr;
u32 dws;
int r;
/* allocate clear state block */
adev->gfx.rlc.clear_state_size = dws = adev->gfx.rlc.funcs->get_csb_size(adev);
- r = amdgpu_bo_create_reserved(adev, dws * 4, PAGE_SIZE,
+ r = amdgpu_bo_create_kernel(adev, dws * 4, PAGE_SIZE,
AMDGPU_GEM_DOMAIN_VRAM,
&adev->gfx.rlc.clear_state_obj,
&adev->gfx.rlc.clear_state_gpu_addr,
return r;
}
- /* set up the cs buffer */
- dst_ptr = adev->gfx.rlc.cs_ptr;
- adev->gfx.rlc.funcs->get_csb_buffer(adev, dst_ptr);
- amdgpu_bo_kunmap(adev->gfx.rlc.clear_state_obj);
- amdgpu_bo_unpin(adev->gfx.rlc.clear_state_obj);
- amdgpu_bo_unreserve(adev->gfx.rlc.clear_state_obj);
-
return 0;
}
{
int r;
- if (cik_asic_reset_method(adev) == AMD_RESET_METHOD_BACO)
+ if (cik_asic_reset_method(adev) == AMD_RESET_METHOD_BACO) {
+ if (!adev->in_suspend)
+ amdgpu_inc_vram_lost(adev);
r = smu7_asic_baco_reset(adev);
- else
+ } else {
r = cik_asic_pci_config_reset(adev);
+ }
return r;
}
adev->gfx.ce_fw_version = le32_to_cpu(cp_hdr->header.ucode_version);
adev->gfx.ce_feature_version = le32_to_cpu(cp_hdr->ucode_feature_version);
- snprintf(fw_name, sizeof(fw_name), "amdgpu/%s_rlc.bin", chip_name);
- err = request_firmware(&adev->gfx.rlc_fw, fw_name, adev->dev);
- if (err)
- goto out;
- err = amdgpu_ucode_validate(adev->gfx.rlc_fw);
- rlc_hdr = (const struct rlc_firmware_header_v2_0 *)adev->gfx.rlc_fw->data;
- version_major = le16_to_cpu(rlc_hdr->header.header_version_major);
- version_minor = le16_to_cpu(rlc_hdr->header.header_version_minor);
- if (version_major == 2 && version_minor == 1)
- adev->gfx.rlc.is_rlc_v2_1 = true;
-
- adev->gfx.rlc_fw_version = le32_to_cpu(rlc_hdr->header.ucode_version);
- adev->gfx.rlc_feature_version = le32_to_cpu(rlc_hdr->ucode_feature_version);
- adev->gfx.rlc.save_and_restore_offset =
+ if (!amdgpu_sriov_vf(adev)) {
+ snprintf(fw_name, sizeof(fw_name), "amdgpu/%s_rlc.bin", chip_name);
+ err = request_firmware(&adev->gfx.rlc_fw, fw_name, adev->dev);
+ if (err)
+ goto out;
+ err = amdgpu_ucode_validate(adev->gfx.rlc_fw);
+ rlc_hdr = (const struct rlc_firmware_header_v2_0 *)adev->gfx.rlc_fw->data;
+ version_major = le16_to_cpu(rlc_hdr->header.header_version_major);
+ version_minor = le16_to_cpu(rlc_hdr->header.header_version_minor);
+ if (version_major == 2 && version_minor == 1)
+ adev->gfx.rlc.is_rlc_v2_1 = true;
+
+ adev->gfx.rlc_fw_version = le32_to_cpu(rlc_hdr->header.ucode_version);
+ adev->gfx.rlc_feature_version = le32_to_cpu(rlc_hdr->ucode_feature_version);
+ adev->gfx.rlc.save_and_restore_offset =
le32_to_cpu(rlc_hdr->save_and_restore_offset);
- adev->gfx.rlc.clear_state_descriptor_offset =
+ adev->gfx.rlc.clear_state_descriptor_offset =
le32_to_cpu(rlc_hdr->clear_state_descriptor_offset);
- adev->gfx.rlc.avail_scratch_ram_locations =
+ adev->gfx.rlc.avail_scratch_ram_locations =
le32_to_cpu(rlc_hdr->avail_scratch_ram_locations);
- adev->gfx.rlc.reg_restore_list_size =
+ adev->gfx.rlc.reg_restore_list_size =
le32_to_cpu(rlc_hdr->reg_restore_list_size);
- adev->gfx.rlc.reg_list_format_start =
+ adev->gfx.rlc.reg_list_format_start =
le32_to_cpu(rlc_hdr->reg_list_format_start);
- adev->gfx.rlc.reg_list_format_separate_start =
+ adev->gfx.rlc.reg_list_format_separate_start =
le32_to_cpu(rlc_hdr->reg_list_format_separate_start);
- adev->gfx.rlc.starting_offsets_start =
+ adev->gfx.rlc.starting_offsets_start =
le32_to_cpu(rlc_hdr->starting_offsets_start);
- adev->gfx.rlc.reg_list_format_size_bytes =
+ adev->gfx.rlc.reg_list_format_size_bytes =
le32_to_cpu(rlc_hdr->reg_list_format_size_bytes);
- adev->gfx.rlc.reg_list_size_bytes =
+ adev->gfx.rlc.reg_list_size_bytes =
le32_to_cpu(rlc_hdr->reg_list_size_bytes);
- adev->gfx.rlc.register_list_format =
+ adev->gfx.rlc.register_list_format =
kmalloc(adev->gfx.rlc.reg_list_format_size_bytes +
- adev->gfx.rlc.reg_list_size_bytes, GFP_KERNEL);
- if (!adev->gfx.rlc.register_list_format) {
- err = -ENOMEM;
- goto out;
- }
+ adev->gfx.rlc.reg_list_size_bytes, GFP_KERNEL);
+ if (!adev->gfx.rlc.register_list_format) {
+ err = -ENOMEM;
+ goto out;
+ }
- tmp = (unsigned int *)((uintptr_t)rlc_hdr +
- le32_to_cpu(rlc_hdr->reg_list_format_array_offset_bytes));
- for (i = 0 ; i < (rlc_hdr->reg_list_format_size_bytes >> 2); i++)
- adev->gfx.rlc.register_list_format[i] = le32_to_cpu(tmp[i]);
+ tmp = (unsigned int *)((uintptr_t)rlc_hdr +
+ le32_to_cpu(rlc_hdr->reg_list_format_array_offset_bytes));
+ for (i = 0 ; i < (rlc_hdr->reg_list_format_size_bytes >> 2); i++)
+ adev->gfx.rlc.register_list_format[i] = le32_to_cpu(tmp[i]);
- adev->gfx.rlc.register_restore = adev->gfx.rlc.register_list_format + i;
+ adev->gfx.rlc.register_restore = adev->gfx.rlc.register_list_format + i;
- tmp = (unsigned int *)((uintptr_t)rlc_hdr +
- le32_to_cpu(rlc_hdr->reg_list_array_offset_bytes));
- for (i = 0 ; i < (rlc_hdr->reg_list_size_bytes >> 2); i++)
- adev->gfx.rlc.register_restore[i] = le32_to_cpu(tmp[i]);
+ tmp = (unsigned int *)((uintptr_t)rlc_hdr +
+ le32_to_cpu(rlc_hdr->reg_list_array_offset_bytes));
+ for (i = 0 ; i < (rlc_hdr->reg_list_size_bytes >> 2); i++)
+ adev->gfx.rlc.register_restore[i] = le32_to_cpu(tmp[i]);
- if (adev->gfx.rlc.is_rlc_v2_1)
- gfx_v10_0_init_rlc_ext_microcode(adev);
+ if (adev->gfx.rlc.is_rlc_v2_1)
+ gfx_v10_0_init_rlc_ext_microcode(adev);
+ }
snprintf(fw_name, sizeof(fw_name), "amdgpu/%s_mec%s.bin", chip_name, wks);
err = request_firmware(&adev->gfx.mec_fw, fw_name, adev->dev);
return 0;
}
-static int gfx_v10_0_csb_vram_pin(struct amdgpu_device *adev)
-{
- int r;
-
- r = amdgpu_bo_reserve(adev->gfx.rlc.clear_state_obj, false);
- if (unlikely(r != 0))
- return r;
-
- r = amdgpu_bo_pin(adev->gfx.rlc.clear_state_obj,
- AMDGPU_GEM_DOMAIN_VRAM);
- if (!r)
- adev->gfx.rlc.clear_state_gpu_addr =
- amdgpu_bo_gpu_offset(adev->gfx.rlc.clear_state_obj);
-
- amdgpu_bo_unreserve(adev->gfx.rlc.clear_state_obj);
-
- return r;
-}
-
-static void gfx_v10_0_csb_vram_unpin(struct amdgpu_device *adev)
-{
- int r;
-
- if (!adev->gfx.rlc.clear_state_obj)
- return;
-
- r = amdgpu_bo_reserve(adev->gfx.rlc.clear_state_obj, true);
- if (likely(r == 0)) {
- amdgpu_bo_unpin(adev->gfx.rlc.clear_state_obj);
- amdgpu_bo_unreserve(adev->gfx.rlc.clear_state_obj);
- }
-}
-
static void gfx_v10_0_mec_fini(struct amdgpu_device *adev)
{
amdgpu_bo_free_kernel(&adev->gfx.mec.hpd_eop_obj, NULL, NULL);
static int gfx_v10_0_init_csb(struct amdgpu_device *adev)
{
- int r;
-
- if (adev->in_gpu_reset) {
- r = amdgpu_bo_reserve(adev->gfx.rlc.clear_state_obj, false);
- if (r)
- return r;
-
- r = amdgpu_bo_kmap(adev->gfx.rlc.clear_state_obj,
- (void **)&adev->gfx.rlc.cs_ptr);
- if (!r) {
- adev->gfx.rlc.funcs->get_csb_buffer(adev,
- adev->gfx.rlc.cs_ptr);
- amdgpu_bo_kunmap(adev->gfx.rlc.clear_state_obj);
- }
-
- amdgpu_bo_unreserve(adev->gfx.rlc.clear_state_obj);
- if (r)
- return r;
- }
+ adev->gfx.rlc.funcs->get_csb_buffer(adev, adev->gfx.rlc.cs_ptr);
/* csib */
WREG32_SOC15(GC, 0, mmRLC_CSIB_ADDR_HI,
return 0;
}
-static int gfx_v10_0_init_pg(struct amdgpu_device *adev)
-{
- int i;
- int r;
-
- r = gfx_v10_0_init_csb(adev);
- if (r)
- return r;
-
- for (i = 0; i < adev->num_vmhubs; i++)
- amdgpu_gmc_flush_gpu_tlb(adev, 0, i, 0);
-
- /* TODO: init power gating */
- return 0;
-}
-
void gfx_v10_0_rlc_stop(struct amdgpu_device *adev)
{
u32 tmp = RREG32_SOC15(GC, 0, mmRLC_CNTL);
{
int r;
- if (amdgpu_sriov_vf(adev))
- return 0;
-
if (adev->firmware.load_type == AMDGPU_FW_LOAD_PSP) {
- r = gfx_v10_0_wait_for_rlc_autoload_complete(adev);
- if (r)
- return r;
- r = gfx_v10_0_init_pg(adev);
+ r = gfx_v10_0_wait_for_rlc_autoload_complete(adev);
if (r)
return r;
- /* enable RLC SRM */
- gfx_v10_0_rlc_enable_srm(adev);
+ gfx_v10_0_init_csb(adev);
+ if (!amdgpu_sriov_vf(adev)) /* enable RLC SRM */
+ gfx_v10_0_rlc_enable_srm(adev);
} else {
adev->gfx.rlc.funcs->stop(adev);
return r;
}
- r = gfx_v10_0_init_pg(adev);
- if (r)
- return r;
+ gfx_v10_0_init_csb(adev);
adev->gfx.rlc.funcs->start(adev);
/* Init gfx ring 0 for pipe 0 */
mutex_lock(&adev->srbm_mutex);
gfx_v10_0_cp_gfx_switch_pipe(adev, PIPE_ID0);
- mutex_unlock(&adev->srbm_mutex);
+
/* Set ring buffer size */
ring = &adev->gfx.gfx_ring[0];
rb_bufsz = order_base_2(ring->ring_size / 8);
WREG32_SOC15(GC, 0, mmCP_RB_ACTIVE, 1);
gfx_v10_0_cp_gfx_set_doorbell(adev, ring);
+ mutex_unlock(&adev->srbm_mutex);
/* Init gfx ring 1 for pipe 1 */
mutex_lock(&adev->srbm_mutex);
gfx_v10_0_cp_gfx_switch_pipe(adev, PIPE_ID1);
- mutex_unlock(&adev->srbm_mutex);
ring = &adev->gfx.gfx_ring[1];
rb_bufsz = order_base_2(ring->ring_size / 8);
tmp = REG_SET_FIELD(0, CP_RB1_CNTL, RB_BUFSZ, rb_bufsz);
WREG32_SOC15(GC, 0, mmCP_RB1_ACTIVE, 1);
gfx_v10_0_cp_gfx_set_doorbell(adev, ring);
+ mutex_unlock(&adev->srbm_mutex);
/* Switch to pipe 0 */
mutex_lock(&adev->srbm_mutex);
int r;
struct amdgpu_device *adev = (struct amdgpu_device *)handle;
- r = gfx_v10_0_csb_vram_pin(adev);
- if (r)
- return r;
-
if (!amdgpu_emu_mode)
gfx_v10_0_init_golden_registers(adev);
if (amdgpu_gfx_disable_kcq(adev))
DRM_ERROR("KCQ disable failed\n");
if (amdgpu_sriov_vf(adev)) {
- pr_debug("For SRIOV client, shouldn't do anything.\n");
+ gfx_v10_0_cp_gfx_enable(adev, false);
return 0;
}
gfx_v10_0_cp_enable(adev, false);
gfx_v10_0_enable_gui_idle_interrupt(adev, false);
- gfx_v10_0_csb_vram_unpin(adev);
return 0;
}
gfx_v7_0_constants_init(adev);
+ /* init CSB */
+ adev->gfx.rlc.funcs->get_csb_buffer(adev, adev->gfx.rlc.cs_ptr);
/* init rlc */
r = adev->gfx.rlc.funcs->resume(adev);
if (r)
return 0;
}
-static int gfx_v8_0_csb_vram_pin(struct amdgpu_device *adev)
-{
- int r;
-
- r = amdgpu_bo_reserve(adev->gfx.rlc.clear_state_obj, false);
- if (unlikely(r != 0))
- return r;
-
- r = amdgpu_bo_pin(adev->gfx.rlc.clear_state_obj,
- AMDGPU_GEM_DOMAIN_VRAM);
- if (!r)
- adev->gfx.rlc.clear_state_gpu_addr =
- amdgpu_bo_gpu_offset(adev->gfx.rlc.clear_state_obj);
-
- amdgpu_bo_unreserve(adev->gfx.rlc.clear_state_obj);
-
- return r;
-}
-
-static void gfx_v8_0_csb_vram_unpin(struct amdgpu_device *adev)
-{
- int r;
-
- if (!adev->gfx.rlc.clear_state_obj)
- return;
-
- r = amdgpu_bo_reserve(adev->gfx.rlc.clear_state_obj, true);
- if (likely(r == 0)) {
- amdgpu_bo_unpin(adev->gfx.rlc.clear_state_obj);
- amdgpu_bo_unreserve(adev->gfx.rlc.clear_state_obj);
- }
-}
-
static void gfx_v8_0_mec_fini(struct amdgpu_device *adev)
{
amdgpu_bo_free_kernel(&adev->gfx.mec.hpd_eop_obj, NULL, NULL);
static void gfx_v8_0_init_csb(struct amdgpu_device *adev)
{
+ adev->gfx.rlc.funcs->get_csb_buffer(adev, adev->gfx.rlc.cs_ptr);
/* csib */
WREG32(mmRLC_CSIB_ADDR_HI,
adev->gfx.rlc.clear_state_gpu_addr >> 32);
gfx_v8_0_init_golden_registers(adev);
gfx_v8_0_constants_init(adev);
- r = gfx_v8_0_csb_vram_pin(adev);
- if (r)
- return r;
-
r = adev->gfx.rlc.funcs->resume(adev);
if (r)
return r;
pr_err("rlc is busy, skip halt rlc\n");
amdgpu_gfx_rlc_exit_safe_mode(adev);
- gfx_v8_0_csb_vram_unpin(adev);
-
return 0;
}
return 0;
}
-static int gfx_v9_0_csb_vram_pin(struct amdgpu_device *adev)
-{
- int r;
-
- r = amdgpu_bo_reserve(adev->gfx.rlc.clear_state_obj, false);
- if (unlikely(r != 0))
- return r;
-
- r = amdgpu_bo_pin(adev->gfx.rlc.clear_state_obj,
- AMDGPU_GEM_DOMAIN_VRAM);
- if (!r)
- adev->gfx.rlc.clear_state_gpu_addr =
- amdgpu_bo_gpu_offset(adev->gfx.rlc.clear_state_obj);
-
- amdgpu_bo_unreserve(adev->gfx.rlc.clear_state_obj);
-
- return r;
-}
-
-static void gfx_v9_0_csb_vram_unpin(struct amdgpu_device *adev)
-{
- int r;
-
- if (!adev->gfx.rlc.clear_state_obj)
- return;
-
- r = amdgpu_bo_reserve(adev->gfx.rlc.clear_state_obj, true);
- if (likely(r == 0)) {
- amdgpu_bo_unpin(adev->gfx.rlc.clear_state_obj);
- amdgpu_bo_unreserve(adev->gfx.rlc.clear_state_obj);
- }
-}
-
static void gfx_v9_0_mec_fini(struct amdgpu_device *adev)
{
amdgpu_bo_free_kernel(&adev->gfx.mec.hpd_eop_obj, NULL, NULL);
static void gfx_v9_0_init_csb(struct amdgpu_device *adev)
{
+ adev->gfx.rlc.funcs->get_csb_buffer(adev, adev->gfx.rlc.cs_ptr);
/* csib */
WREG32_RLC(SOC15_REG_OFFSET(GC, 0, mmRLC_CSIB_ADDR_HI),
adev->gfx.rlc.clear_state_gpu_addr >> 32);
gfx_v9_0_constants_init(adev);
- r = gfx_v9_0_csb_vram_pin(adev);
- if (r)
- return r;
-
r = adev->gfx.rlc.funcs->resume(adev);
if (r)
return r;
gfx_v9_0_cp_enable(adev, false);
adev->gfx.rlc.funcs->stop(adev);
- gfx_v9_0_csb_vram_unpin(adev);
-
return 0;
}
u32 xgmi_lfb_cntl = RREG32_SOC15(GC, 0, mmMC_VM_XGMI_LFB_CNTL);
u32 max_region =
REG_GET_FIELD(xgmi_lfb_cntl, MC_VM_XGMI_LFB_CNTL, PF_MAX_REGION);
+ u32 max_num_physical_nodes = 0;
+ u32 max_physical_node_id = 0;
+
+ switch (adev->asic_type) {
+ case CHIP_VEGA20:
+ max_num_physical_nodes = 4;
+ max_physical_node_id = 3;
+ break;
+ case CHIP_ARCTURUS:
+ max_num_physical_nodes = 8;
+ max_physical_node_id = 7;
+ break;
+ default:
+ return -EINVAL;
+ }
/* PF_MAX_REGION=0 means xgmi is disabled */
if (max_region) {
adev->gmc.xgmi.num_physical_nodes = max_region + 1;
- if (adev->gmc.xgmi.num_physical_nodes > 4)
+ if (adev->gmc.xgmi.num_physical_nodes > max_num_physical_nodes)
return -EINVAL;
adev->gmc.xgmi.physical_node_id =
REG_GET_FIELD(xgmi_lfb_cntl, MC_VM_XGMI_LFB_CNTL, PF_LFB_REGION);
- if (adev->gmc.xgmi.physical_node_id > 3)
+ if (adev->gmc.xgmi.physical_node_id > max_physical_node_id)
return -EINVAL;
adev->gmc.xgmi.node_segment_size = REG_GET_FIELD(
RREG32_SOC15(GC, 0, mmMC_VM_XGMI_LFB_SIZE),
if (!adev->mman.buffer_funcs_enabled ||
!adev->ib_pool_ready ||
- adev->in_gpu_reset) {
+ adev->in_gpu_reset ||
+ ring->sched.ready == false) {
gmc_v10_0_flush_vm_hub(adev, vmid, AMDGPU_GFXHUB_0, 0);
mutex_unlock(&adev->mman.gtt_window_lock);
return;
{
int r;
- if (vi_asic_reset_method(adev) == AMD_RESET_METHOD_BACO)
+ if (vi_asic_reset_method(adev) == AMD_RESET_METHOD_BACO) {
+ if (!adev->in_suspend)
+ amdgpu_inc_vram_lost(adev);
r = smu7_asic_baco_reset(adev);
- else
+ } else {
r = vi_asic_pci_config_reset(adev);
+ }
return r;
}
config HSA_AMD
bool "HSA kernel driver for AMD GPU devices"
- depends on DRM_AMDGPU && (X86_64 || ARM64)
+ depends on DRM_AMDGPU && (X86_64 || ARM64 || PPC64)
imply AMD_IOMMU_V2 if X86_64
select MMU_NOTIFIER
help
if (adev->powerplay.pp_funcs && adev->powerplay.pp_funcs->get_clock_by_type) {
if (adev->powerplay.pp_funcs->get_clock_by_type(pp_handle,
dc_to_pp_clock_type(clk_type), &pp_clks)) {
- /* Error in pplib. Provide default values. */
+ /* Error in pplib. Provide default values. */
+ get_default_clock_levels(clk_type, dc_clks);
return true;
}
} else if (adev->smu.ppt_funcs && adev->smu.ppt_funcs->get_clock_by_type) {
if (pipe->plane_state != NULL)
flip_immediate = pipe->plane_state->flip_immediate;
+ if (flip_immediate && lock) {
+ const int TIMEOUT_FOR_FLIP_PENDING = 100000;
+ int i;
+
+ for (i = 0; i < TIMEOUT_FOR_FLIP_PENDING; ++i) {
+ if (!pipe->plane_res.hubp->funcs->hubp_is_flip_pending(pipe->plane_res.hubp))
+ break;
+ udelay(1);
+ }
+
+ if (pipe->bottom_pipe != NULL) {
+ for (i = 0; i < TIMEOUT_FOR_FLIP_PENDING; ++i) {
+ if (!pipe->bottom_pipe->plane_res.hubp->funcs->hubp_is_flip_pending(pipe->bottom_pipe->plane_res.hubp))
+ break;
+ udelay(1);
+ }
+ }
+ }
+
/* In flip immediate and pipe splitting case, we need to use GSL
* for synchronization. Only do setup on locking and on flip type change.
*/
.xfc_fill_constant_bytes = 0,
};
+struct _vcs_dpi_ip_params_st dcn2_0_nv14_ip = {
+ .odm_capable = 1,
+ .gpuvm_enable = 0,
+ .hostvm_enable = 0,
+ .gpuvm_max_page_table_levels = 4,
+ .hostvm_max_page_table_levels = 4,
+ .hostvm_cached_page_table_levels = 0,
+ .num_dsc = 5,
+ .rob_buffer_size_kbytes = 168,
+ .det_buffer_size_kbytes = 164,
+ .dpte_buffer_size_in_pte_reqs_luma = 84,
+ .dpte_buffer_size_in_pte_reqs_chroma = 42,//todo
+ .dpp_output_buffer_pixels = 2560,
+ .opp_output_buffer_lines = 1,
+ .pixel_chunk_size_kbytes = 8,
+ .pte_enable = 1,
+ .max_page_table_levels = 4,
+ .pte_chunk_size_kbytes = 2,
+ .meta_chunk_size_kbytes = 2,
+ .writeback_chunk_size_kbytes = 2,
+ .line_buffer_size_bits = 789504,
+ .is_line_buffer_bpp_fixed = 0,
+ .line_buffer_fixed_bpp = 0,
+ .dcc_supported = true,
+ .max_line_buffer_lines = 12,
+ .writeback_luma_buffer_size_kbytes = 12,
+ .writeback_chroma_buffer_size_kbytes = 8,
+ .writeback_chroma_line_buffer_width_pixels = 4,
+ .writeback_max_hscl_ratio = 1,
+ .writeback_max_vscl_ratio = 1,
+ .writeback_min_hscl_ratio = 1,
+ .writeback_min_vscl_ratio = 1,
+ .writeback_max_hscl_taps = 12,
+ .writeback_max_vscl_taps = 12,
+ .writeback_line_buffer_luma_buffer_size = 0,
+ .writeback_line_buffer_chroma_buffer_size = 14643,
+ .cursor_buffer_size = 8,
+ .cursor_chunk_size = 2,
+ .max_num_otg = 5,
+ .max_num_dpp = 5,
+ .max_num_wb = 1,
+ .max_dchub_pscl_bw_pix_per_clk = 4,
+ .max_pscl_lb_bw_pix_per_clk = 2,
+ .max_lb_vscl_bw_pix_per_clk = 4,
+ .max_vscl_hscl_bw_pix_per_clk = 4,
+ .max_hscl_ratio = 8,
+ .max_vscl_ratio = 8,
+ .hscl_mults = 4,
+ .vscl_mults = 4,
+ .max_hscl_taps = 8,
+ .max_vscl_taps = 8,
+ .dispclk_ramp_margin_percent = 1,
+ .underscan_factor = 1.10,
+ .min_vblank_lines = 32, //
+ .dppclk_delay_subtotal = 77, //
+ .dppclk_delay_scl_lb_only = 16,
+ .dppclk_delay_scl = 50,
+ .dppclk_delay_cnvc_formatter = 8,
+ .dppclk_delay_cnvc_cursor = 6,
+ .dispclk_delay_subtotal = 87, //
+ .dcfclk_cstate_latency = 10, // SRExitTime
+ .max_inter_dcn_tile_repeaters = 8,
+ .xfc_supported = true,
+ .xfc_fill_bw_overhead_percent = 10.0,
+ .xfc_fill_constant_bytes = 0,
+ .ptoi_supported = 0
+};
+
struct _vcs_dpi_soc_bounding_box_st dcn2_0_soc = {
/* Defaults that get patched on driver load from firmware. */
.clock_limits = {
.num_pll = 5,
.num_dwb = 1,
.num_ddc = 5,
+ .num_vmid = 16,
+ .num_dsc = 5,
};
static const struct dc_debug_options debug_defaults_drv = {
static struct _vcs_dpi_ip_params_st *get_asic_rev_ip_params(
uint32_t hw_internal_rev)
{
+ /* NV14 */
+ if (ASICREV_IS_NAVI14_M(hw_internal_rev))
+ return &dcn2_0_nv14_ip;
+
/* NV12 and NV10 */
return &dcn2_0_ip;
}
return ret;
}
+
+int smu_send_smc_msg(struct smu_context *smu,
+ enum smu_message_type msg)
+{
+ int ret;
+
+ ret = smu_send_smc_msg_with_param(smu, msg, 0);
+ return ret;
+}
.set_tool_table_location = smu_v11_0_set_tool_table_location,
.notify_memory_pool_location = smu_v11_0_notify_memory_pool_location,
.system_features_control = smu_v11_0_system_features_control,
- .send_smc_msg = smu_v11_0_send_msg,
.send_smc_msg_with_param = smu_v11_0_send_msg_with_param,
.read_smc_arg = smu_v11_0_read_arg,
.init_display_count = smu_v11_0_init_display_count,
int (*notify_memory_pool_location)(struct smu_context *smu);
int (*set_last_dcef_min_deep_sleep_clk)(struct smu_context *smu);
int (*system_features_control)(struct smu_context *smu, bool en);
- int (*send_smc_msg)(struct smu_context *smu, uint16_t msg);
- int (*send_smc_msg_with_param)(struct smu_context *smu, uint16_t msg, uint32_t param);
+ int (*send_smc_msg_with_param)(struct smu_context *smu,
+ enum smu_message_type msg, uint32_t param);
int (*read_smc_arg)(struct smu_context *smu, uint32_t *arg);
int (*init_display_count)(struct smu_context *smu, uint32_t count);
int (*set_allowed_mask)(struct smu_context *smu);
int smu_v11_0_system_features_control(struct smu_context *smu,
bool en);
-int smu_v11_0_send_msg(struct smu_context *smu, uint16_t msg);
-
int
-smu_v11_0_send_msg_with_param(struct smu_context *smu, uint16_t msg,
+smu_v11_0_send_msg_with_param(struct smu_context *smu,
+ enum smu_message_type msg,
uint32_t param);
int smu_v11_0_read_arg(struct smu_context *smu, uint32_t *arg);
int smu_v12_0_wait_for_response(struct smu_context *smu);
-int smu_v12_0_send_msg(struct smu_context *smu, uint16_t msg);
-
int
-smu_v12_0_send_msg_with_param(struct smu_context *smu, uint16_t msg,
+smu_v12_0_send_msg_with_param(struct smu_context *smu,
+ enum smu_message_type msg,
uint32_t param);
int smu_v12_0_check_fw_status(struct smu_context *smu);
.set_tool_table_location = smu_v11_0_set_tool_table_location,
.notify_memory_pool_location = smu_v11_0_notify_memory_pool_location,
.system_features_control = smu_v11_0_system_features_control,
- .send_smc_msg = smu_v11_0_send_msg,
.send_smc_msg_with_param = smu_v11_0_send_msg_with_param,
.read_smc_arg = smu_v11_0_read_arg,
.init_display_count = smu_v11_0_init_display_count,
.check_fw_version = smu_v12_0_check_fw_version,
.powergate_sdma = smu_v12_0_powergate_sdma,
.powergate_vcn = smu_v12_0_powergate_vcn,
- .send_smc_msg = smu_v12_0_send_msg,
.send_smc_msg_with_param = smu_v12_0_send_msg_with_param,
.read_smc_arg = smu_v12_0_read_arg,
.set_gfx_cgpg = smu_v12_0_set_gfx_cgpg,
#define smu_set_default_od_settings(smu, initialize) \
((smu)->ppt_funcs->set_default_od_settings ? (smu)->ppt_funcs->set_default_od_settings((smu), (initialize)) : 0)
-#define smu_send_smc_msg(smu, msg) \
- ((smu)->ppt_funcs->send_smc_msg? (smu)->ppt_funcs->send_smc_msg((smu), (msg)) : 0)
+int smu_send_smc_msg(struct smu_context *smu, enum smu_message_type msg);
+
#define smu_send_smc_msg_with_param(smu, msg, param) \
((smu)->ppt_funcs->send_smc_msg_with_param? (smu)->ppt_funcs->send_smc_msg_with_param((smu), (msg), (param)) : 0)
#define smu_read_smc_arg(smu, arg) \
return RREG32_SOC15(MP1, 0, mmMP1_SMN_C2PMSG_90) == 0x1 ? 0 : -EIO;
}
-int smu_v11_0_send_msg(struct smu_context *smu, uint16_t msg)
-{
- struct amdgpu_device *adev = smu->adev;
- int ret = 0, index = 0;
-
- index = smu_msg_get_index(smu, msg);
- if (index < 0)
- return index;
-
- smu_v11_0_wait_for_response(smu);
-
- WREG32_SOC15(MP1, 0, mmMP1_SMN_C2PMSG_90, 0);
-
- smu_v11_0_send_msg_without_waiting(smu, (uint16_t)index);
-
- ret = smu_v11_0_wait_for_response(smu);
-
- if (ret)
- pr_err("failed send message: %10s (%d) response %#x\n",
- smu_get_message_name(smu, msg), index, ret);
-
- return ret;
-
-}
-
int
-smu_v11_0_send_msg_with_param(struct smu_context *smu, uint16_t msg,
+smu_v11_0_send_msg_with_param(struct smu_context *smu,
+ enum smu_message_type msg,
uint32_t param)
{
-
struct amdgpu_device *adev = smu->adev;
int ret = 0, index = 0;
return RREG32_SOC15(MP1, 0, mmMP1_SMN_C2PMSG_90) == 0x1 ? 0 : -EIO;
}
-int smu_v12_0_send_msg(struct smu_context *smu, uint16_t msg)
-{
- struct amdgpu_device *adev = smu->adev;
- int ret = 0, index = 0;
-
- index = smu_msg_get_index(smu, msg);
- if (index < 0)
- return index;
-
- smu_v12_0_wait_for_response(smu);
-
- WREG32_SOC15(MP1, 0, mmMP1_SMN_C2PMSG_90, 0);
-
- smu_v12_0_send_msg_without_waiting(smu, (uint16_t)index);
-
- ret = smu_v12_0_wait_for_response(smu);
-
- if (ret)
- pr_err("Failed to send message 0x%x, response 0x%x\n", index,
- ret);
-
- return ret;
-
-}
-
int
-smu_v12_0_send_msg_with_param(struct smu_context *smu, uint16_t msg,
+smu_v12_0_send_msg_with_param(struct smu_context *smu,
+ enum smu_message_type msg,
uint32_t param)
{
struct amdgpu_device *adev = smu->adev;
.set_tool_table_location = smu_v11_0_set_tool_table_location,
.notify_memory_pool_location = smu_v11_0_notify_memory_pool_location,
.system_features_control = smu_v11_0_system_features_control,
- .send_smc_msg = smu_v11_0_send_msg,
.send_smc_msg_with_param = smu_v11_0_send_msg_with_param,
.read_smc_arg = smu_v11_0_read_arg,
.init_display_count = smu_v11_0_init_display_count,
drm_dp_mst_topology_put_port(port);
}
- for (i = 0; i < mgr->max_payloads; i++) {
- if (mgr->payloads[i].payload_state != DP_PAYLOAD_DELETE_LOCAL)
+ for (i = 0; i < mgr->max_payloads; /* do nothing */) {
+ if (mgr->payloads[i].payload_state != DP_PAYLOAD_DELETE_LOCAL) {
+ i++;
continue;
+ }
DRM_DEBUG_KMS("removing payload %d\n", i);
for (j = i; j < mgr->max_payloads - 1; j++) {
config DRM_I915_PREEMPT_TIMEOUT
int "Preempt timeout (ms, jiffy granularity)"
- default 100 # milliseconds
+ default 640 # milliseconds
help
How long to wait (in milliseconds) for a preemption event to occur
when submitting a new context via execlists. If the current context
static u8 ehl_calc_voltage_level(int cdclk)
{
- if (cdclk > 312000)
+ if (cdclk > 326400)
+ return 3;
+ else if (cdclk > 312000)
return 2;
else if (cdclk > 180000)
return 1;
u32 dkl_de_emphasis_control;
};
-static const struct tgl_dkl_phy_ddi_buf_trans tgl_dkl_phy_ddi_translations[] = {
+static const struct tgl_dkl_phy_ddi_buf_trans tgl_dkl_phy_dp_ddi_trans[] = {
/* VS pre-emp Non-trans mV Pre-emph dB */
{ 0x7, 0x0, 0x00 }, /* 0 0 400mV 0 dB */
{ 0x5, 0x0, 0x03 }, /* 0 1 400mV 3.5 dB */
{ 0x0, 0x0, 0x00 }, /* 3 0 1200mV 0 dB HDMI default */
};
+static const struct tgl_dkl_phy_ddi_buf_trans tgl_dkl_phy_hdmi_ddi_trans[] = {
+ /* HDMI Preset VS Pre-emph */
+ { 0x7, 0x0, 0x0 }, /* 1 400mV 0dB */
+ { 0x6, 0x0, 0x0 }, /* 2 500mV 0dB */
+ { 0x4, 0x0, 0x0 }, /* 3 650mV 0dB */
+ { 0x2, 0x0, 0x0 }, /* 4 800mV 0dB */
+ { 0x0, 0x0, 0x0 }, /* 5 1000mV 0dB */
+ { 0x0, 0x0, 0x5 }, /* 6 Full -1.5 dB */
+ { 0x0, 0x0, 0x6 }, /* 7 Full -1.8 dB */
+ { 0x0, 0x0, 0x7 }, /* 8 Full -2 dB */
+ { 0x0, 0x0, 0x8 }, /* 9 Full -2.5 dB */
+ { 0x0, 0x0, 0xA }, /* 10 Full -3 dB */
+};
+
static const struct ddi_buf_trans *
bdw_get_buf_trans_edp(struct drm_i915_private *dev_priv, int *n_entries)
{
icl_get_combo_buf_trans(dev_priv, INTEL_OUTPUT_HDMI,
0, &n_entries);
else
- n_entries = ARRAY_SIZE(tgl_dkl_phy_ddi_translations);
+ n_entries = ARRAY_SIZE(tgl_dkl_phy_hdmi_ddi_trans);
default_entry = n_entries - 1;
} else if (INTEL_GEN(dev_priv) == 11) {
if (intel_phy_is_combo(dev_priv, phy))
icl_get_combo_buf_trans(dev_priv, encoder->type,
intel_dp->link_rate, &n_entries);
else
- n_entries = ARRAY_SIZE(tgl_dkl_phy_ddi_translations);
+ n_entries = ARRAY_SIZE(tgl_dkl_phy_dp_ddi_trans);
} else if (INTEL_GEN(dev_priv) == 11) {
if (intel_phy_is_combo(dev_priv, phy))
icl_get_combo_buf_trans(dev_priv, encoder->type,
const struct tgl_dkl_phy_ddi_buf_trans *ddi_translations;
u32 n_entries, val, ln, dpcnt_mask, dpcnt_val;
- n_entries = ARRAY_SIZE(tgl_dkl_phy_ddi_translations);
- ddi_translations = tgl_dkl_phy_ddi_translations;
+ if (encoder->type == INTEL_OUTPUT_HDMI) {
+ n_entries = ARRAY_SIZE(tgl_dkl_phy_hdmi_ddi_trans);
+ ddi_translations = tgl_dkl_phy_hdmi_ddi_trans;
+ } else {
+ n_entries = ARRAY_SIZE(tgl_dkl_phy_dp_ddi_trans);
+ ddi_translations = tgl_dkl_phy_dp_ddi_trans;
+ }
if (level >= n_entries)
level = n_entries - 1;
return I915_READ(GEN8_DE_PORT_ISR) & bit;
}
-static bool icl_combo_port_connected(struct drm_i915_private *dev_priv,
- struct intel_digital_port *intel_dig_port)
+static bool intel_combo_phy_connected(struct drm_i915_private *dev_priv,
+ enum phy phy)
{
- enum port port = intel_dig_port->base.port;
-
- if (HAS_PCH_MCC(dev_priv) && port == PORT_C)
+ if (HAS_PCH_MCC(dev_priv) && phy == PHY_C)
return I915_READ(SDEISR) & SDE_TC_HOTPLUG_ICP(PORT_TC1);
- return I915_READ(SDEISR) & SDE_DDI_HOTPLUG_ICP(port);
+ return I915_READ(SDEISR) & SDE_DDI_HOTPLUG_ICP(phy);
}
static bool icl_digital_port_connected(struct intel_encoder *encoder)
enum phy phy = intel_port_to_phy(dev_priv, encoder->port);
if (intel_phy_is_combo(dev_priv, phy))
- return icl_combo_port_connected(dev_priv, dig_port);
+ return intel_combo_phy_connected(dev_priv, phy);
else if (intel_phy_is_tc(dev_priv, phy))
return intel_tc_port_connected(dig_port);
else
if (!ce->timeline)
return NULL;
- rcu_read_lock();
+ mutex_lock(&ce->timeline->mutex);
list_for_each_entry_reverse(rq, &ce->timeline->requests, link) {
if (i915_request_completed(rq))
break;
if (engine)
break;
}
- rcu_read_unlock();
+ mutex_unlock(&ce->timeline->mutex);
return engine;
}
GEM_BUG_ON(rq->hw_context == ce);
if (rcu_access_pointer(rq->timeline) != tl) { /* timeline sharing! */
- err = mutex_lock_interruptible_nested(&tl->mutex,
- SINGLE_DEPTH_NESTING);
- if (err)
- return err;
+ /*
+ * Ideally, we just want to insert our foreign fence as
+ * a barrier into the remove context, such that this operation
+ * occurs after all current operations in that context, and
+ * all future operations must occur after this.
+ *
+ * Currently, the timeline->last_request tracking is guarded
+ * by its mutex and so we must obtain that to atomically
+ * insert our barrier. However, since we already hold our
+ * timeline->mutex, we must be careful against potential
+ * inversion if we are the kernel_context as the remote context
+ * will itself poke at the kernel_context when it needs to
+ * unpin. Ergo, if already locked, we drop both locks and
+ * try again (through the magic of userspace repeating EAGAIN).
+ */
+ if (!mutex_trylock(&tl->mutex))
+ return -EAGAIN;
/* Queue this switch after current activity by this context. */
err = i915_active_fence_set(&tl->last_request, rq);
static inline struct i915_request *
execlists_active(const struct intel_engine_execlists *execlists)
{
- GEM_BUG_ON(execlists->active - execlists->inflight >
- execlists_num_ports(execlists));
- return READ_ONCE(*execlists->active);
+ return *READ_ONCE(execlists->active);
}
static inline void
#include "i915_drv.h"
-#include "gt/intel_gt.h"
-
+#include "intel_context.h"
#include "intel_engine.h"
#include "intel_engine_pm.h"
#include "intel_engine_pool.h"
#include "intel_engine_user.h"
-#include "intel_context.h"
+#include "intel_gt.h"
+#include "intel_gt_requests.h"
#include "intel_lrc.h"
#include "intel_reset.h"
#include "intel_ring.h"
intel_engine_init_execlists(engine);
intel_engine_init_cmd_parser(engine);
intel_engine_init__pm(engine);
+ intel_engine_init_retire(engine);
intel_engine_pool_init(&engine->pool);
cleanup_status_page(engine);
+ intel_engine_fini_retire(engine);
intel_engine_pool_fini(&engine->pool);
intel_engine_fini_breadcrumbs(engine);
intel_engine_cleanup_cmd_parser(engine);
#endif /* !IS_ENABLED(CONFIG_LOCKDEP) */
+static void
+__queue_and_release_pm(struct i915_request *rq,
+ struct intel_timeline *tl,
+ struct intel_engine_cs *engine)
+{
+ struct intel_gt_timelines *timelines = &engine->gt->timelines;
+
+ GEM_TRACE("%s\n", engine->name);
+
+ /*
+ * We have to serialise all potential retirement paths with our
+ * submission, as we don't want to underflow either the
+ * engine->wakeref.counter or our timeline->active_count.
+ *
+ * Equally, we cannot allow a new submission to start until
+ * after we finish queueing, nor could we allow that submitter
+ * to retire us before we are ready!
+ */
+ spin_lock(&timelines->lock);
+
+ /* Let intel_gt_retire_requests() retire us (acquired under lock) */
+ if (!atomic_fetch_inc(&tl->active_count))
+ list_add_tail(&tl->link, &timelines->active_list);
+
+ /* Hand the request over to HW and so engine_retire() */
+ __i915_request_queue(rq, NULL);
+
+ /* Let new submissions commence (and maybe retire this timeline) */
+ __intel_wakeref_defer_park(&engine->wakeref);
+
+ spin_unlock(&timelines->lock);
+}
+
static bool switch_to_kernel_context(struct intel_engine_cs *engine)
{
+ struct intel_context *ce = engine->kernel_context;
struct i915_request *rq;
unsigned long flags;
bool result = true;
* This should hold true as we can only park the engine after
* retiring the last request, thus all rings should be empty and
* all timelines idle.
+ *
+ * For unlocking, there are 2 other parties and the GPU who have a
+ * stake here.
+ *
+ * A new gpu user will be waiting on the engine-pm to start their
+ * engine_unpark. New waiters are predicated on engine->wakeref.count
+ * and so intel_wakeref_defer_park() acts like a mutex_unlock of the
+ * engine->wakeref.
+ *
+ * The other party is intel_gt_retire_requests(), which is walking the
+ * list of active timelines looking for completions. Meanwhile as soon
+ * as we call __i915_request_queue(), the GPU may complete our request.
+ * Ergo, if we put ourselves on the timelines.active_list
+ * (se intel_timeline_enter()) before we increment the
+ * engine->wakeref.count, we may see the request completion and retire
+ * it causing an undeflow of the engine->wakeref.
*/
- flags = __timeline_mark_lock(engine->kernel_context);
+ flags = __timeline_mark_lock(ce);
+ GEM_BUG_ON(atomic_read(&ce->timeline->active_count) < 0);
- rq = __i915_request_create(engine->kernel_context, GFP_NOWAIT);
+ rq = __i915_request_create(ce, GFP_NOWAIT);
if (IS_ERR(rq))
/* Context switch failed, hope for the best! Maybe reset? */
goto out_unlock;
- intel_timeline_enter(i915_request_timeline(rq));
-
/* Check again on the next retirement. */
engine->wakeref_serial = engine->serial + 1;
i915_request_add_active_barriers(rq);
rq->sched.attr.priority = I915_PRIORITY_BARRIER;
__i915_request_commit(rq);
- /* Release our exclusive hold on the engine */
- __intel_wakeref_defer_park(&engine->wakeref);
- __i915_request_queue(rq, NULL);
+ /* Expose ourselves to the world */
+ __queue_and_release_pm(rq, ce->timeline, engine);
result = false;
out_unlock:
- __timeline_mark_unlock(engine->kernel_context, flags);
+ __timeline_mark_unlock(ce, flags);
return result;
}
engine->execlists.no_priolist = false;
- intel_gt_pm_put(engine->gt);
+ /* While gt calls i915_vma_parked(), we have to break the lock cycle */
+ intel_gt_pm_put_async(engine->gt);
return 0;
}
intel_wakeref_put(&engine->wakeref);
}
+static inline void intel_engine_pm_put_async(struct intel_engine_cs *engine)
+{
+ intel_wakeref_put_async(&engine->wakeref);
+}
+
+static inline void intel_engine_pm_flush(struct intel_engine_cs *engine)
+{
+ intel_wakeref_unlock_wait(&engine->wakeref);
+}
+
void intel_engine_init__pm(struct intel_engine_cs *engine);
#endif /* INTEL_ENGINE_PM_H */
struct intel_engine_execlists execlists;
+ /*
+ * Keep track of completed timelines on this engine for early
+ * retirement with the goal of quickly enabling powersaving as
+ * soon as the engine is idle.
+ */
+ struct intel_timeline *retire;
+ struct work_struct retire_work;
+
/* status_notifier: list of callbacks for context-switch changes */
struct atomic_notifier_head context_status_notifier;
static const struct intel_wakeref_ops wf_ops = {
.get = __gt_unpark,
.put = __gt_park,
- .flags = INTEL_WAKEREF_PUT_ASYNC,
};
void intel_gt_pm_init_early(struct intel_gt *gt)
static suspend_state_t pm_suspend_target(void)
{
-#if IS_ENABLED(CONFIG_PM_SLEEP)
+#if IS_ENABLED(CONFIG_SUSPEND) && IS_ENABLED(CONFIG_PM_SLEEP)
return pm_suspend_target_state;
#else
return PM_SUSPEND_TO_IDLE;
intel_wakeref_put(>->wakeref);
}
+static inline void intel_gt_pm_put_async(struct intel_gt *gt)
+{
+ intel_wakeref_put_async(>->wakeref);
+}
+
static inline int intel_gt_pm_wait_for_idle(struct intel_gt *gt)
{
return intel_wakeref_wait_for_idle(>->wakeref);
* Copyright © 2019 Intel Corporation
*/
+#include <linux/workqueue.h>
+
#include "i915_drv.h" /* for_each_engine() */
#include "i915_request.h"
#include "intel_gt.h"
intel_engine_flush_submission(engine);
}
+static void engine_retire(struct work_struct *work)
+{
+ struct intel_engine_cs *engine =
+ container_of(work, typeof(*engine), retire_work);
+ struct intel_timeline *tl = xchg(&engine->retire, NULL);
+
+ do {
+ struct intel_timeline *next = xchg(&tl->retire, NULL);
+
+ /*
+ * Our goal here is to retire _idle_ timelines as soon as
+ * possible (as they are idle, we do not expect userspace
+ * to be cleaning up anytime soon).
+ *
+ * If the timeline is currently locked, either it is being
+ * retired elsewhere or about to be!
+ */
+ if (mutex_trylock(&tl->mutex)) {
+ retire_requests(tl);
+ mutex_unlock(&tl->mutex);
+ }
+ intel_timeline_put(tl);
+
+ GEM_BUG_ON(!next);
+ tl = ptr_mask_bits(next, 1);
+ } while (tl);
+}
+
+static bool add_retire(struct intel_engine_cs *engine,
+ struct intel_timeline *tl)
+{
+ struct intel_timeline *first;
+
+ /*
+ * We open-code a llist here to include the additional tag [BIT(0)]
+ * so that we know when the timeline is already on a
+ * retirement queue: either this engine or another.
+ *
+ * However, we rely on that a timeline can only be active on a single
+ * engine at any one time and that add_retire() is called before the
+ * engine releases the timeline and transferred to another to retire.
+ */
+
+ if (READ_ONCE(tl->retire)) /* already queued */
+ return false;
+
+ intel_timeline_get(tl);
+ first = READ_ONCE(engine->retire);
+ do
+ tl->retire = ptr_pack_bits(first, 1, 1);
+ while (!try_cmpxchg(&engine->retire, &first, tl));
+
+ return !first;
+}
+
+void intel_engine_add_retire(struct intel_engine_cs *engine,
+ struct intel_timeline *tl)
+{
+ if (add_retire(engine, tl))
+ schedule_work(&engine->retire_work);
+}
+
+void intel_engine_init_retire(struct intel_engine_cs *engine)
+{
+ INIT_WORK(&engine->retire_work, engine_retire);
+}
+
+void intel_engine_fini_retire(struct intel_engine_cs *engine)
+{
+ flush_work(&engine->retire_work);
+ GEM_BUG_ON(engine->retire);
+}
+
long intel_gt_retire_requests_timeout(struct intel_gt *gt, long timeout)
{
struct intel_gt_timelines *timelines = >->timelines;
}
intel_timeline_get(tl);
- GEM_BUG_ON(!tl->active_count);
- tl->active_count++; /* pin the list element */
+ GEM_BUG_ON(!atomic_read(&tl->active_count));
+ atomic_inc(&tl->active_count); /* pin the list element */
spin_unlock_irqrestore(&timelines->lock, flags);
if (timeout > 0) {
/* Resume iteration after dropping lock */
list_safe_reset_next(tl, tn, link);
- if (!--tl->active_count)
+ if (atomic_dec_and_test(&tl->active_count))
list_del(&tl->link);
else
active_count += !!rcu_access_pointer(tl->last_request.fence);
/* Defer the final release to after the spinlock */
if (refcount_dec_and_test(&tl->kref.refcount)) {
- GEM_BUG_ON(tl->active_count);
+ GEM_BUG_ON(atomic_read(&tl->active_count));
list_add(&tl->link, &free);
}
}
#ifndef INTEL_GT_REQUESTS_H
#define INTEL_GT_REQUESTS_H
+struct intel_engine_cs;
struct intel_gt;
+struct intel_timeline;
long intel_gt_retire_requests_timeout(struct intel_gt *gt, long timeout);
static inline void intel_gt_retire_requests(struct intel_gt *gt)
intel_gt_retire_requests_timeout(gt, 0);
}
+void intel_engine_init_retire(struct intel_engine_cs *engine);
+void intel_engine_add_retire(struct intel_engine_cs *engine,
+ struct intel_timeline *tl);
+void intel_engine_fini_retire(struct intel_engine_cs *engine);
+
int intel_gt_wait_for_idle(struct intel_gt *gt, long timeout);
void intel_gt_init_requests(struct intel_gt *gt);
#include "intel_engine_pm.h"
#include "intel_gt.h"
#include "intel_gt_pm.h"
+#include "intel_gt_requests.h"
#include "intel_lrc_reg.h"
#include "intel_mocs.h"
#include "intel_reset.h"
* refrain from doing non-trivial work here.
*/
+ /*
+ * If we have just completed this context, the engine may now be
+ * idle and we want to re-enter powersaving.
+ */
+ if (list_is_last(&rq->link, &ce->timeline->requests) &&
+ i915_request_completed(rq))
+ intel_engine_add_retire(engine, ce->timeline);
+
intel_engine_context_out(engine);
execlists_context_status_change(rq, INTEL_CONTEXT_SCHEDULE_OUT);
- intel_gt_pm_put(engine->gt);
+ intel_gt_pm_put_async(engine->gt);
/*
* If this is part of a virtual engine, its next request may
static void
cancel_port_requests(struct intel_engine_execlists * const execlists)
{
- struct i915_request * const *port, *rq;
+ struct i915_request * const *port;
- for (port = execlists->pending; (rq = *port); port++)
- execlists_schedule_out(rq);
+ for (port = execlists->pending; *port; port++)
+ execlists_schedule_out(*port);
memset(execlists->pending, 0, sizeof(execlists->pending));
- for (port = execlists->active; (rq = *port); port++)
- execlists_schedule_out(rq);
- execlists->active =
- memset(execlists->inflight, 0, sizeof(execlists->inflight));
+ /* Mark the end of active before we overwrite *active */
+ for (port = xchg(&execlists->active, execlists->pending); *port; port++)
+ execlists_schedule_out(*port);
+ WRITE_ONCE(execlists->active,
+ memset(execlists->inflight, 0, sizeof(execlists->inflight)));
}
static inline void
else
promote = gen8_csb_parse(execlists, buf + 2 * head);
if (promote) {
+ struct i915_request * const *old = execlists->active;
+
+ /* Point active to the new ELSP; prevent overwriting */
+ WRITE_ONCE(execlists->active, execlists->pending);
+ set_timeslice(engine);
+
if (!inject_preempt_hang(execlists))
ring_set_paused(engine, 0);
/* cancel old inflight, prepare for switch */
- trace_ports(execlists, "preempted", execlists->active);
- while (*execlists->active)
- execlists_schedule_out(*execlists->active++);
+ trace_ports(execlists, "preempted", old);
+ while (*old)
+ execlists_schedule_out(*old++);
/* switch pending to inflight */
GEM_BUG_ON(!assert_pending_valid(execlists, "promote"));
- execlists->active =
- memcpy(execlists->inflight,
- execlists->pending,
- execlists_num_ports(execlists) *
- sizeof(*execlists->pending));
-
- set_timeslice(engine);
+ WRITE_ONCE(execlists->active,
+ memcpy(execlists->inflight,
+ execlists->pending,
+ execlists_num_ports(execlists) *
+ sizeof(*execlists->pending)));
WRITE_ONCE(execlists->pending[0], NULL);
} else {
out:
intel_engine_cancel_stop_cs(engine);
reset_finish_engine(engine);
- intel_engine_pm_put(engine);
+ intel_engine_pm_put_async(engine);
return ret;
}
i915_vma_make_unshrinkable(vma);
- GEM_BUG_ON(ring->vaddr);
- ring->vaddr = addr;
+ /* Discard any unused bytes beyond that submitted to hw. */
+ intel_ring_reset(ring, ring->emit);
+ ring->vaddr = addr;
return 0;
err_ring:
if (!atomic_dec_and_test(&ring->pin_count))
return;
- /* Discard any unused bytes beyond that submitted to hw. */
- intel_ring_reset(ring, ring->emit);
-
i915_vma_unset_ggtt_write(vma);
if (i915_vma_is_map_and_fenceable(vma))
i915_vma_unpin_iomap(vma);
else
i915_gem_object_unpin_map(vma->obj);
- GEM_BUG_ON(!ring->vaddr);
- ring->vaddr = NULL;
-
- i915_vma_unpin(vma);
i915_vma_make_purgeable(vma);
+ i915_vma_unpin(vma);
}
static struct i915_vma *create_ring_vma(struct i915_ggtt *ggtt, int size)
{
GEM_BUG_ON(atomic_read(&timeline->pin_count));
GEM_BUG_ON(!list_empty(&timeline->requests));
+ GEM_BUG_ON(timeline->retire);
if (timeline->hwsp_cacheline)
cacheline_free(timeline->hwsp_cacheline);
struct intel_gt_timelines *timelines = &tl->gt->timelines;
unsigned long flags;
+ /*
+ * Pretend we are serialised by the timeline->mutex.
+ *
+ * While generally true, there are a few exceptions to the rule
+ * for the engine->kernel_context being used to manage power
+ * transitions. As the engine_park may be called from under any
+ * timeline, it uses the power mutex as a global serialisation
+ * lock to prevent any other request entering its timeline.
+ *
+ * The rule is generally tl->mutex, otherwise engine->wakeref.mutex.
+ *
+ * However, intel_gt_retire_request() does not know which engine
+ * it is retiring along and so cannot partake in the engine-pm
+ * barrier, and there we use the tl->active_count as a means to
+ * pin the timeline in the active_list while the locks are dropped.
+ * Ergo, as that is outside of the engine-pm barrier, we need to
+ * use atomic to manipulate tl->active_count.
+ */
lockdep_assert_held(&tl->mutex);
-
GEM_BUG_ON(!atomic_read(&tl->pin_count));
- if (tl->active_count++)
+
+ if (atomic_add_unless(&tl->active_count, 1, 0))
return;
- GEM_BUG_ON(!tl->active_count); /* overflow? */
spin_lock_irqsave(&timelines->lock, flags);
- list_add(&tl->link, &timelines->active_list);
+ if (!atomic_fetch_inc(&tl->active_count))
+ list_add_tail(&tl->link, &timelines->active_list);
spin_unlock_irqrestore(&timelines->lock, flags);
}
struct intel_gt_timelines *timelines = &tl->gt->timelines;
unsigned long flags;
+ /* See intel_timeline_enter() */
lockdep_assert_held(&tl->mutex);
- GEM_BUG_ON(!tl->active_count);
- if (--tl->active_count)
+ GEM_BUG_ON(!atomic_read(&tl->active_count));
+ if (atomic_add_unless(&tl->active_count, -1, 1))
return;
spin_lock_irqsave(&timelines->lock, flags);
- list_del(&tl->link);
+ if (atomic_dec_and_test(&tl->active_count))
+ list_del(&tl->link);
spin_unlock_irqrestore(&timelines->lock, flags);
/*
* from the intel_context caller plus internal atomicity.
*/
atomic_t pin_count;
- unsigned int active_count;
+ atomic_t active_count;
const u32 *hwsp_seqno;
struct i915_vma *hwsp_ggtt;
*/
struct i915_active_fence last_request;
+ /** A chain of completed timelines ready for early retirement. */
+ struct intel_timeline *retire;
+
/**
* We track the most recent seqno that we wait on in every context so
* that we only have to emit a new await and dependency on a more
pr_err("intel_engine_pm_get_if_awake(%s) failed under %s\n",
engine->name, p->name);
else
- intel_engine_pm_put(engine);
- intel_engine_pm_put(engine);
+ intel_engine_pm_put_async(engine);
+ intel_engine_pm_put_async(engine);
p->critical_section_end();
- /* engine wakeref is sync (instant) */
+ intel_engine_pm_flush(engine);
+
if (intel_engine_pm_is_awake(engine)) {
pr_err("%s is still awake after flushing pm\n",
engine->name);
if (!(cmd_val(s, 0) & (1 << 22)))
return ret;
- /* check if QWORD */
- if (DWORD_FIELD(0, 20, 19) == 1)
- valid_len += 8;
+ /* check inline data */
+ if (cmd_val(s, 0) & BIT(18))
+ valid_len = CMD_LEN(9);
ret = gvt_check_valid_cmd_length(cmd_length(s),
valid_len);
if (ret)
static i915_reg_t force_nonpriv_white_list[] = {
GEN9_CS_DEBUG_MODE1, //_MMIO(0x20ec)
GEN9_CTX_PREEMPT_REG,//_MMIO(0x2248)
+ PS_INVOCATION_COUNT,//_MMIO(0x2348)
GEN8_CS_CHICKEN1,//_MMIO(0x2580)
_MMIO(0x2690),
_MMIO(0x2694),
static int force_nonpriv_write(struct intel_vgpu *vgpu,
unsigned int offset, void *p_data, unsigned int bytes)
{
- u32 reg_nonpriv = *(u32 *)p_data;
+ u32 reg_nonpriv = (*(u32 *)p_data) & REG_GENMASK(25, 2);
int ring_id = intel_gvt_render_mmio_to_ring_id(vgpu->gvt, offset);
u32 ring_base;
struct drm_i915_private *dev_priv = vgpu->gvt->dev_priv;
bytes);
} else
gvt_err("vgpu(%d) Invalid FORCE_NONPRIV write %x at offset %x\n",
- vgpu->id, reg_nonpriv, offset);
+ vgpu->id, *(u32 *)p_data, offset);
return 0;
}
* populated by i915_request_add_active_barriers() to point to the
* request that will eventually release them.
*/
- spin_lock_irqsave_nested(&ref->tree_lock, flags, SINGLE_DEPTH_NESTING);
llist_for_each_safe(pos, next, take_preallocated_barriers(ref)) {
struct active_node *node = barrier_from_ll(pos);
struct intel_engine_cs *engine = barrier_to_engine(node);
struct rb_node **p, *parent;
+ spin_lock_irqsave_nested(&ref->tree_lock, flags,
+ SINGLE_DEPTH_NESTING);
parent = NULL;
p = &ref->tree.rb_node;
while (*p) {
}
rb_link_node(&node->node, parent, p);
rb_insert_color(&node->node, &ref->tree);
+ spin_unlock_irqrestore(&ref->tree_lock, flags);
GEM_BUG_ON(!intel_engine_pm_is_awake(engine));
llist_add(barrier_to_ll(node), &engine->barrier_tasks);
intel_engine_pm_put(engine);
}
- spin_unlock_irqrestore(&ref->tree_lock, flags);
}
void i915_request_add_active_barriers(struct i915_request *rq)
val = 0;
if (intel_gt_pm_get_if_awake(gt)) {
val = __get_rc6(gt);
- intel_gt_pm_put(gt);
+ intel_gt_pm_put_async(gt);
}
spin_lock_irqsave(&pmu->lock, flags);
skip:
spin_unlock_irqrestore(&engine->uncore->lock, flags);
- intel_engine_pm_put(engine);
+ intel_engine_pm_put_async(engine);
}
}
if (intel_gt_pm_get_if_awake(gt)) {
val = intel_uncore_read_notrace(uncore, GEN6_RPSTAT1);
val = intel_get_cagf(rps, val);
- intel_gt_pm_put(gt);
+ intel_gt_pm_put_async(gt);
}
add_sample_mult(&pmu->sample[__I915_SAMPLE_FREQ_ACT],
struct drm_i915_engine_info __user *info_ptr;
struct drm_i915_query_engine_info query;
struct drm_i915_engine_info info = { };
+ unsigned int num_uabi_engines = 0;
struct intel_engine_cs *engine;
int len, ret;
if (query_item->flags)
return -EINVAL;
+ for_each_uabi_engine(engine, i915)
+ num_uabi_engines++;
+
len = sizeof(struct drm_i915_query_engine_info) +
- RUNTIME_INFO(i915)->num_engines *
- sizeof(struct drm_i915_engine_info);
+ num_uabi_engines * sizeof(struct drm_i915_engine_info);
ret = copy_query_item(&query, sizeof(query), len, query_item);
if (ret != 0)
static void ____intel_wakeref_put_last(struct intel_wakeref *wf)
{
- if (!atomic_dec_and_test(&wf->count))
+ INTEL_WAKEREF_BUG_ON(atomic_read(&wf->count) <= 0);
+ if (unlikely(!atomic_dec_and_test(&wf->count)))
goto unlock;
/* ops->put() must reschedule its own release on error/deferral */
mutex_unlock(&wf->mutex);
}
-void __intel_wakeref_put_last(struct intel_wakeref *wf)
+void __intel_wakeref_put_last(struct intel_wakeref *wf, unsigned long flags)
{
INTEL_WAKEREF_BUG_ON(work_pending(&wf->work));
/* Assume we are not in process context and so cannot sleep. */
- if (wf->ops->flags & INTEL_WAKEREF_PUT_ASYNC ||
- !mutex_trylock(&wf->mutex)) {
+ if (flags & INTEL_WAKEREF_PUT_ASYNC || !mutex_trylock(&wf->mutex)) {
schedule_work(&wf->work);
return;
}
int intel_wakeref_wait_for_idle(struct intel_wakeref *wf)
{
- return wait_var_event_killable(&wf->wakeref,
- !intel_wakeref_is_active(wf));
+ int err;
+
+ might_sleep();
+
+ err = wait_var_event_killable(&wf->wakeref,
+ !intel_wakeref_is_active(wf));
+ if (err)
+ return err;
+
+ intel_wakeref_unlock_wait(wf);
+ return 0;
}
static void wakeref_auto_timeout(struct timer_list *t)
#include <linux/atomic.h>
#include <linux/bits.h>
+#include <linux/lockdep.h>
#include <linux/mutex.h>
#include <linux/refcount.h>
#include <linux/stackdepot.h>
struct intel_wakeref_ops {
int (*get)(struct intel_wakeref *wf);
int (*put)(struct intel_wakeref *wf);
-
- unsigned long flags;
-#define INTEL_WAKEREF_PUT_ASYNC BIT(0)
};
struct intel_wakeref {
} while (0)
int __intel_wakeref_get_first(struct intel_wakeref *wf);
-void __intel_wakeref_put_last(struct intel_wakeref *wf);
+void __intel_wakeref_put_last(struct intel_wakeref *wf, unsigned long flags);
/**
* intel_wakeref_get: Acquire the wakeref
}
/**
- * intel_wakeref_put: Release the wakeref
- * @i915: the drm_i915_private device
+ * intel_wakeref_put_flags: Release the wakeref
* @wf: the wakeref
- * @fn: callback for releasing the wakeref, called only on final release.
+ * @flags: control flags
*
* Release our hold on the wakeref. When there are no more users,
* the runtime pm wakeref will be released after the @fn callback is called
* code otherwise.
*/
static inline void
-intel_wakeref_put(struct intel_wakeref *wf)
+__intel_wakeref_put(struct intel_wakeref *wf, unsigned long flags)
+#define INTEL_WAKEREF_PUT_ASYNC BIT(0)
{
INTEL_WAKEREF_BUG_ON(atomic_read(&wf->count) <= 0);
if (unlikely(!atomic_add_unless(&wf->count, -1, 1)))
- __intel_wakeref_put_last(wf);
+ __intel_wakeref_put_last(wf, flags);
+}
+
+static inline void
+intel_wakeref_put(struct intel_wakeref *wf)
+{
+ might_sleep();
+ __intel_wakeref_put(wf, 0);
+}
+
+static inline void
+intel_wakeref_put_async(struct intel_wakeref *wf)
+{
+ __intel_wakeref_put(wf, INTEL_WAKEREF_PUT_ASYNC);
}
/**
}
/**
+ * intel_wakeref_unlock_wait: Wait until the active callback is complete
+ * @wf: the wakeref
+ *
+ * Waits for the active callback (under the @wf->mutex or another CPU) is
+ * complete.
+ */
+static inline void
+intel_wakeref_unlock_wait(struct intel_wakeref *wf)
+{
+ mutex_lock(&wf->mutex);
+ mutex_unlock(&wf->mutex);
+ flush_work(&wf->work);
+}
+
+/**
* intel_wakeref_is_active: Query whether the wakeref is currently held
* @wf: the wakeref
*
static inline void
__intel_wakeref_defer_park(struct intel_wakeref *wf)
{
+ lockdep_assert_held(&wf->mutex);
INTEL_WAKEREF_BUG_ON(atomic_read(&wf->count));
atomic_set_release(&wf->count, 1);
}
static struct drm_driver driver;
static const struct pci_device_id pciidlist[] = {
+ { PCI_VENDOR_ID_MATROX, 0x522, PCI_VENDOR_ID_SUN, 0x4852, 0, 0,
+ G200_SE_A | MGAG200_FLAG_HW_BUG_NO_STARTADD},
{ PCI_VENDOR_ID_MATROX, 0x522, PCI_ANY_ID, PCI_ANY_ID, 0, 0, G200_SE_A },
{ PCI_VENDOR_ID_MATROX, 0x524, PCI_ANY_ID, PCI_ANY_ID, 0, 0, G200_SE_B },
{ PCI_VENDOR_ID_MATROX, 0x530, PCI_ANY_ID, PCI_ANY_ID, 0, 0, G200_EV },
DEFINE_DRM_GEM_FOPS(mgag200_driver_fops);
+static bool mgag200_pin_bo_at_0(const struct mga_device *mdev)
+{
+ return mdev->flags & MGAG200_FLAG_HW_BUG_NO_STARTADD;
+}
+
+int mgag200_driver_dumb_create(struct drm_file *file,
+ struct drm_device *dev,
+ struct drm_mode_create_dumb *args)
+{
+ struct mga_device *mdev = dev->dev_private;
+ unsigned long pg_align;
+
+ if (WARN_ONCE(!dev->vram_mm, "VRAM MM not initialized"))
+ return -EINVAL;
+
+ pg_align = 0ul;
+
+ /*
+ * Aligning scanout buffers to the size of the video ram forces
+ * placement at offset 0. Works around a bug where HW does not
+ * respect 'startadd' field.
+ */
+ if (mgag200_pin_bo_at_0(mdev))
+ pg_align = PFN_UP(mdev->mc.vram_size);
+
+ return drm_gem_vram_fill_create_dumb(file, dev, &dev->vram_mm->bdev,
+ pg_align, false, args);
+}
+
static struct drm_driver driver = {
.driver_features = DRIVER_GEM | DRIVER_MODESET,
.load = mgag200_driver_load,
.major = DRIVER_MAJOR,
.minor = DRIVER_MINOR,
.patchlevel = DRIVER_PATCHLEVEL,
- DRM_GEM_VRAM_DRIVER
+ .debugfs_init = drm_vram_mm_debugfs_init,
+ .dumb_create = mgag200_driver_dumb_create,
+ .dumb_map_offset = drm_gem_vram_driver_dumb_mmap_offset,
+ .gem_prime_mmap = drm_gem_prime_mmap,
};
static struct pci_driver mgag200_pci_driver = {
G200_EW3,
};
+/* HW does not handle 'startadd' field correct. */
+#define MGAG200_FLAG_HW_BUG_NO_STARTADD (1ul << 8)
+
+#define MGAG200_TYPE_MASK (0x000000ff)
+#define MGAG200_FLAG_MASK (0x00ffff00)
+
#define IS_G200_SE(mdev) (mdev->type == G200_SE_A || mdev->type == G200_SE_B)
struct mga_device {
u32 unique_rev_id;
};
+static inline enum mga_type
+mgag200_type_from_driver_data(kernel_ulong_t driver_data)
+{
+ return (enum mga_type)(driver_data & MGAG200_TYPE_MASK);
+}
+
+static inline unsigned long
+mgag200_flags_from_driver_data(kernel_ulong_t driver_data)
+{
+ return driver_data & MGAG200_FLAG_MASK;
+}
+
/* mgag200_mode.c */
int mgag200_modeset_init(struct mga_device *mdev);
void mgag200_modeset_fini(struct mga_device *mdev);
struct mga_device *mdev = dev->dev_private;
int ret, option;
- mdev->type = flags;
+ mdev->flags = mgag200_flags_from_driver_data(flags);
+ mdev->type = mgag200_type_from_driver_data(flags);
/* Hardcode the number of CRTCs to 1 */
mdev->num_crtc = 1;
depends on OF && COMMON_CLK
depends on MMU
depends on INTERCONNECT || !INTERCONNECT
+ depends on QCOM_OCMEM || QCOM_OCMEM=n
select QCOM_MDT_LOADER if ARCH_QCOM
select REGULATOR
select DRM_KMS_HELPER
* Copyright (c) 2014 The Linux Foundation. All rights reserved.
*/
-#ifdef CONFIG_MSM_OCMEM
-# include <mach/ocmem.h>
-#endif
-
#include "a3xx_gpu.h"
#define A3XX_INT0_MASK \
gpu_write(gpu, REG_A3XX_RBBM_GPR0_CTL, 0x00000000);
/* Set the OCMEM base address for A330, etc */
- if (a3xx_gpu->ocmem_hdl) {
+ if (a3xx_gpu->ocmem.hdl) {
gpu_write(gpu, REG_A3XX_RB_GMEM_BASE_ADDR,
- (unsigned int)(a3xx_gpu->ocmem_base >> 14));
+ (unsigned int)(a3xx_gpu->ocmem.base >> 14));
}
/* Turn on performance counters: */
adreno_gpu_cleanup(adreno_gpu);
-#ifdef CONFIG_MSM_OCMEM
- if (a3xx_gpu->ocmem_base)
- ocmem_free(OCMEM_GRAPHICS, a3xx_gpu->ocmem_hdl);
-#endif
+ adreno_gpu_ocmem_cleanup(&a3xx_gpu->ocmem);
kfree(a3xx_gpu);
}
/* if needed, allocate gmem: */
if (adreno_is_a330(adreno_gpu)) {
-#ifdef CONFIG_MSM_OCMEM
- /* TODO this is different/missing upstream: */
- struct ocmem_buf *ocmem_hdl =
- ocmem_allocate(OCMEM_GRAPHICS, adreno_gpu->gmem);
-
- a3xx_gpu->ocmem_hdl = ocmem_hdl;
- a3xx_gpu->ocmem_base = ocmem_hdl->addr;
- adreno_gpu->gmem = ocmem_hdl->len;
- DBG("using %dK of OCMEM at 0x%08x", adreno_gpu->gmem / 1024,
- a3xx_gpu->ocmem_base);
-#endif
+ ret = adreno_gpu_ocmem_init(&adreno_gpu->base.pdev->dev,
+ adreno_gpu, &a3xx_gpu->ocmem);
+ if (ret)
+ goto fail;
}
if (!gpu->aspace) {
struct adreno_gpu base;
/* if OCMEM is used for GMEM: */
- uint32_t ocmem_base;
- void *ocmem_hdl;
+ struct adreno_ocmem ocmem;
};
#define to_a3xx_gpu(x) container_of(x, struct a3xx_gpu, base)
/* Copyright (c) 2014 The Linux Foundation. All rights reserved.
*/
#include "a4xx_gpu.h"
-#ifdef CONFIG_MSM_OCMEM
-# include <soc/qcom/ocmem.h>
-#endif
#define A4XX_INT0_MASK \
(A4XX_INT0_RBBM_AHB_ERROR | \
(1 << 30) | 0xFFFF);
gpu_write(gpu, REG_A4XX_RB_GMEM_BASE_ADDR,
- (unsigned int)(a4xx_gpu->ocmem_base >> 14));
+ (unsigned int)(a4xx_gpu->ocmem.base >> 14));
/* Turn on performance counters: */
gpu_write(gpu, REG_A4XX_RBBM_PERFCTR_CTL, 0x01);
adreno_gpu_cleanup(adreno_gpu);
-#ifdef CONFIG_MSM_OCMEM
- if (a4xx_gpu->ocmem_base)
- ocmem_free(OCMEM_GRAPHICS, a4xx_gpu->ocmem_hdl);
-#endif
+ adreno_gpu_ocmem_cleanup(&a4xx_gpu->ocmem);
kfree(a4xx_gpu);
}
/* if needed, allocate gmem: */
if (adreno_is_a4xx(adreno_gpu)) {
-#ifdef CONFIG_MSM_OCMEM
- /* TODO this is different/missing upstream: */
- struct ocmem_buf *ocmem_hdl =
- ocmem_allocate(OCMEM_GRAPHICS, adreno_gpu->gmem);
-
- a4xx_gpu->ocmem_hdl = ocmem_hdl;
- a4xx_gpu->ocmem_base = ocmem_hdl->addr;
- adreno_gpu->gmem = ocmem_hdl->len;
- DBG("using %dK of OCMEM at 0x%08x", adreno_gpu->gmem / 1024,
- a4xx_gpu->ocmem_base);
-#endif
+ ret = adreno_gpu_ocmem_init(dev->dev, adreno_gpu,
+ &a4xx_gpu->ocmem);
+ if (ret)
+ goto fail;
}
if (!gpu->aspace) {
struct adreno_gpu base;
/* if OCMEM is used for GMEM: */
- uint32_t ocmem_base;
- void *ocmem_hdl;
+ struct adreno_ocmem ocmem;
};
#define to_a4xx_gpu(x) container_of(x, struct a4xx_gpu, base)
* 2D mode 3 draw
*/
OUT_RING(ring, 0x0000000B);
+ } else if (adreno_is_a510(adreno_gpu)) {
+ /* Workaround for token and syncs */
+ OUT_RING(ring, 0x00000001);
} else {
/* No workarounds enabled */
OUT_RING(ring, 0x00000000);
0x00100000 + adreno_gpu->gmem - 1);
gpu_write(gpu, REG_A5XX_UCHE_GMEM_RANGE_MAX_HI, 0x00000000);
- gpu_write(gpu, REG_A5XX_CP_MEQ_THRESHOLDS, 0x40);
- if (adreno_is_a530(adreno_gpu))
- gpu_write(gpu, REG_A5XX_CP_MERCIU_SIZE, 0x40);
- if (adreno_is_a540(adreno_gpu))
- gpu_write(gpu, REG_A5XX_CP_MERCIU_SIZE, 0x400);
- gpu_write(gpu, REG_A5XX_CP_ROQ_THRESHOLDS_2, 0x80000060);
- gpu_write(gpu, REG_A5XX_CP_ROQ_THRESHOLDS_1, 0x40201B16);
-
- gpu_write(gpu, REG_A5XX_PC_DBG_ECO_CNTL, (0x400 << 11 | 0x300 << 22));
+ if (adreno_is_a510(adreno_gpu)) {
+ gpu_write(gpu, REG_A5XX_CP_MEQ_THRESHOLDS, 0x20);
+ gpu_write(gpu, REG_A5XX_CP_MERCIU_SIZE, 0x20);
+ gpu_write(gpu, REG_A5XX_CP_ROQ_THRESHOLDS_2, 0x40000030);
+ gpu_write(gpu, REG_A5XX_CP_ROQ_THRESHOLDS_1, 0x20100D0A);
+ gpu_write(gpu, REG_A5XX_PC_DBG_ECO_CNTL,
+ (0x200 << 11 | 0x200 << 22));
+ } else {
+ gpu_write(gpu, REG_A5XX_CP_MEQ_THRESHOLDS, 0x40);
+ if (adreno_is_a530(adreno_gpu))
+ gpu_write(gpu, REG_A5XX_CP_MERCIU_SIZE, 0x40);
+ if (adreno_is_a540(adreno_gpu))
+ gpu_write(gpu, REG_A5XX_CP_MERCIU_SIZE, 0x400);
+ gpu_write(gpu, REG_A5XX_CP_ROQ_THRESHOLDS_2, 0x80000060);
+ gpu_write(gpu, REG_A5XX_CP_ROQ_THRESHOLDS_1, 0x40201B16);
+ gpu_write(gpu, REG_A5XX_PC_DBG_ECO_CNTL,
+ (0x400 << 11 | 0x300 << 22));
+ }
if (adreno_gpu->info->quirks & ADRENO_QUIRK_TWO_PASS_USE_WFI)
gpu_rmw(gpu, REG_A5XX_PC_DBG_ECO_CNTL, 0, (1 << 8));
/* Enable ME/PFP split notification */
gpu_write(gpu, REG_A5XX_RBBM_AHB_CNTL1, 0xA6FFFFFF);
+ /*
+ * In A5x, CCU can send context_done event of a particular context to
+ * UCHE which ultimately reaches CP even when there is valid
+ * transaction of that context inside CCU. This can let CP to program
+ * config registers, which will make the "valid transaction" inside
+ * CCU to be interpreted differently. This can cause gpu fault. This
+ * bug is fixed in latest A510 revision. To enable this bug fix -
+ * bit[11] of RB_DBG_ECO_CNTL need to be set to 0, default is 1
+ * (disable). For older A510 version this bit is unused.
+ */
+ if (adreno_is_a510(adreno_gpu))
+ gpu_rmw(gpu, REG_A5XX_RB_DBG_ECO_CNTL, (1 << 11), 0);
+
/* Enable HWCG */
a5xx_set_hwcg(gpu, true);
/* UCHE */
gpu_write(gpu, REG_A5XX_CP_PROTECT(16), ADRENO_PROTECT_RW(0xE80, 16));
- if (adreno_is_a530(adreno_gpu))
+ if (adreno_is_a530(adreno_gpu) || adreno_is_a510(adreno_gpu))
gpu_write(gpu, REG_A5XX_CP_PROTECT(17),
ADRENO_PROTECT_RW(0x10000, 0x8000));
a5xx_preempt_hw_init(gpu);
- a5xx_gpmu_ucode_init(gpu);
+ if (!adreno_is_a510(adreno_gpu))
+ a5xx_gpmu_ucode_init(gpu);
ret = a5xx_ucode_init(gpu);
if (ret)
}
/*
- * Try to load a zap shader into the secure world. If successful
+ * If the chip that we are using does support loading one, then
+ * try to load a zap shader into the secure world. If successful
* we can use the CP to switch out of secure mode. If not then we
* have no resource but to try to switch ourselves out manually. If we
* guessed wrong then access to the RBBM_SECVID_TRUST_CNTL register will
static int a5xx_pm_resume(struct msm_gpu *gpu)
{
+ struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu);
int ret;
/* Turn on the core power */
if (ret)
return ret;
+ if (adreno_is_a510(adreno_gpu)) {
+ /* Halt the sp_input_clk at HM level */
+ gpu_write(gpu, REG_A5XX_RBBM_CLOCK_CNTL, 0x00000055);
+ a5xx_set_hwcg(gpu, true);
+ /* Turn on sp_input_clk at HM level */
+ gpu_rmw(gpu, REG_A5XX_RBBM_CLOCK_CNTL, 0xff, 0);
+ return 0;
+ }
+
/* Turn the RBCCU domain first to limit the chances of voltage droop */
gpu_write(gpu, REG_A5XX_GPMU_RBCCU_POWER_CNTL, 0x778000);
static int a5xx_pm_suspend(struct msm_gpu *gpu)
{
+ struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu);
+ u32 mask = 0xf;
+
+ /* A510 has 3 XIN ports in VBIF */
+ if (adreno_is_a510(adreno_gpu))
+ mask = 0x7;
+
/* Clear the VBIF pipe before shutting down */
- gpu_write(gpu, REG_A5XX_VBIF_XIN_HALT_CTRL0, 0xF);
- spin_until((gpu_read(gpu, REG_A5XX_VBIF_XIN_HALT_CTRL1) & 0xF) == 0xF);
+ gpu_write(gpu, REG_A5XX_VBIF_XIN_HALT_CTRL0, mask);
+ spin_until((gpu_read(gpu, REG_A5XX_VBIF_XIN_HALT_CTRL1) &
+ mask) == mask);
gpu_write(gpu, REG_A5XX_VBIF_XIN_HALT_CTRL0, 0);
kfree(a5xx_state);
}
-int a5xx_gpu_state_put(struct msm_gpu_state *state)
+static int a5xx_gpu_state_put(struct msm_gpu_state *state)
{
if (IS_ERR_OR_NULL(state))
return 1;
#if defined(CONFIG_DEBUG_FS) || defined(CONFIG_DEV_COREDUMP)
-void a5xx_show(struct msm_gpu *gpu, struct msm_gpu_state *state,
- struct drm_printer *p)
+static void a5xx_show(struct msm_gpu *gpu, struct msm_gpu_state *state,
+ struct drm_printer *p)
{
int i, j;
u32 pos = 0;
struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu);
int ret;
+ /* Not all A5xx chips have a GPMU */
+ if (adreno_is_a510(adreno_gpu))
+ return 0;
+
/* Set up the limits management */
if (adreno_is_a530(adreno_gpu))
a530_lm_setup(gpu);
unsigned int *data, *ptr, *cmds;
unsigned int cmds_size;
+ if (adreno_is_a510(adreno_gpu))
+ return;
+
if (a5xx_gpu->gpmu_bo)
return;
.inactive_period = DRM_MSM_INACTIVE_PERIOD,
.init = a4xx_gpu_init,
}, {
+ .rev = ADRENO_REV(5, 1, 0, ANY_ID),
+ .revn = 510,
+ .name = "A510",
+ .fw = {
+ [ADRENO_FW_PM4] = "a530_pm4.fw",
+ [ADRENO_FW_PFP] = "a530_pfp.fw",
+ },
+ .gmem = SZ_256K,
+ /*
+ * Increase inactive period to 250 to avoid bouncing
+ * the GDSC which appears to make it grumpy
+ */
+ .inactive_period = 250,
+ .init = a5xx_gpu_init,
+ }, {
.rev = ADRENO_REV(5, 3, 0, 2),
.revn = 530,
.name = "A530",
#include <linux/pm_opp.h>
#include <linux/slab.h>
#include <linux/soc/qcom/mdt_loader.h>
+#include <soc/qcom/ocmem.h>
#include "adreno_gpu.h"
#include "msm_gem.h"
#include "msm_mmu.h"
return 0;
}
+int adreno_gpu_ocmem_init(struct device *dev, struct adreno_gpu *adreno_gpu,
+ struct adreno_ocmem *adreno_ocmem)
+{
+ struct ocmem_buf *ocmem_hdl;
+ struct ocmem *ocmem;
+
+ ocmem = of_get_ocmem(dev);
+ if (IS_ERR(ocmem)) {
+ if (PTR_ERR(ocmem) == -ENODEV) {
+ /*
+ * Return success since either the ocmem property was
+ * not specified in device tree, or ocmem support is
+ * not compiled into the kernel.
+ */
+ return 0;
+ }
+
+ return PTR_ERR(ocmem);
+ }
+
+ ocmem_hdl = ocmem_allocate(ocmem, OCMEM_GRAPHICS, adreno_gpu->gmem);
+ if (IS_ERR(ocmem_hdl))
+ return PTR_ERR(ocmem_hdl);
+
+ adreno_ocmem->ocmem = ocmem;
+ adreno_ocmem->base = ocmem_hdl->addr;
+ adreno_ocmem->hdl = ocmem_hdl;
+ adreno_gpu->gmem = ocmem_hdl->len;
+
+ return 0;
+}
+
+void adreno_gpu_ocmem_cleanup(struct adreno_ocmem *adreno_ocmem)
+{
+ if (adreno_ocmem && adreno_ocmem->base)
+ ocmem_free(adreno_ocmem->ocmem, OCMEM_GRAPHICS,
+ adreno_ocmem->hdl);
+}
+
int adreno_gpu_init(struct drm_device *drm, struct platform_device *pdev,
struct adreno_gpu *adreno_gpu,
const struct adreno_gpu_funcs *funcs, int nr_rings)
};
#define to_adreno_gpu(x) container_of(x, struct adreno_gpu, base)
+struct adreno_ocmem {
+ struct ocmem *ocmem;
+ unsigned long base;
+ void *hdl;
+};
+
/* platform config data (ie. from DT, or pdata) */
struct adreno_platform_config {
struct adreno_rev rev;
return gpu->revn == 430;
}
+static inline int adreno_is_a510(struct adreno_gpu *gpu)
+{
+ return gpu->revn == 510;
+}
+
static inline int adreno_is_a530(struct adreno_gpu *gpu)
{
return gpu->revn == 530;
void adreno_wait_ring(struct msm_ringbuffer *ring, uint32_t ndwords);
struct msm_ringbuffer *adreno_active_ring(struct msm_gpu *gpu);
+int adreno_gpu_ocmem_init(struct device *dev, struct adreno_gpu *adreno_gpu,
+ struct adreno_ocmem *ocmem);
+void adreno_gpu_ocmem_cleanup(struct adreno_ocmem *ocmem);
+
int adreno_gpu_init(struct drm_device *drm, struct platform_device *pdev,
struct adreno_gpu *gpu, const struct adreno_gpu_funcs *funcs,
int nr_rings);
int dpu_core_irq_idx_lookup(struct dpu_kms *dpu_kms,
enum dpu_intr_type intr_type, u32 instance_idx)
{
- if (!dpu_kms || !dpu_kms->hw_intr ||
- !dpu_kms->hw_intr->ops.irq_idx_lookup)
+ if (!dpu_kms->hw_intr || !dpu_kms->hw_intr->ops.irq_idx_lookup)
return -EINVAL;
return dpu_kms->hw_intr->ops.irq_idx_lookup(intr_type,
unsigned long irq_flags;
int ret = 0, enable_count;
- if (!dpu_kms || !dpu_kms->hw_intr ||
+ if (!dpu_kms->hw_intr ||
!dpu_kms->irq_obj.enable_counts ||
!dpu_kms->irq_obj.irq_counts) {
DPU_ERROR("invalid params\n");
{
int i, ret = 0, counts;
- if (!dpu_kms || !irq_idxs || !irq_count) {
+ if (!irq_idxs || !irq_count) {
DPU_ERROR("invalid params\n");
return -EINVAL;
}
{
int ret = 0, enable_count;
- if (!dpu_kms || !dpu_kms->hw_intr || !dpu_kms->irq_obj.enable_counts) {
+ if (!dpu_kms->hw_intr || !dpu_kms->irq_obj.enable_counts) {
DPU_ERROR("invalid params\n");
return -EINVAL;
}
{
int i, ret = 0, counts;
- if (!dpu_kms || !irq_idxs || !irq_count) {
+ if (!irq_idxs || !irq_count) {
DPU_ERROR("invalid params\n");
return -EINVAL;
}
u32 dpu_core_irq_read(struct dpu_kms *dpu_kms, int irq_idx, bool clear)
{
- if (!dpu_kms || !dpu_kms->hw_intr ||
+ if (!dpu_kms->hw_intr ||
!dpu_kms->hw_intr->ops.get_interrupt_status)
return 0;
{
unsigned long irq_flags;
- if (!dpu_kms || !dpu_kms->irq_obj.irq_cb_tbl) {
+ if (!dpu_kms->irq_obj.irq_cb_tbl) {
DPU_ERROR("invalid params\n");
return -EINVAL;
}
{
unsigned long irq_flags;
- if (!dpu_kms || !dpu_kms->irq_obj.irq_cb_tbl) {
+ if (!dpu_kms->irq_obj.irq_cb_tbl) {
DPU_ERROR("invalid params\n");
return -EINVAL;
}
static void dpu_clear_all_irqs(struct dpu_kms *dpu_kms)
{
- if (!dpu_kms || !dpu_kms->hw_intr ||
- !dpu_kms->hw_intr->ops.clear_all_irqs)
+ if (!dpu_kms->hw_intr || !dpu_kms->hw_intr->ops.clear_all_irqs)
return;
dpu_kms->hw_intr->ops.clear_all_irqs(dpu_kms->hw_intr);
static void dpu_disable_all_irqs(struct dpu_kms *dpu_kms)
{
- if (!dpu_kms || !dpu_kms->hw_intr ||
- !dpu_kms->hw_intr->ops.disable_all_irqs)
+ if (!dpu_kms->hw_intr || !dpu_kms->hw_intr->ops.disable_all_irqs)
return;
dpu_kms->hw_intr->ops.disable_all_irqs(dpu_kms->hw_intr);
void dpu_core_irq_preinstall(struct dpu_kms *dpu_kms)
{
- struct msm_drm_private *priv;
int i;
- if (!dpu_kms->dev) {
- DPU_ERROR("invalid drm device\n");
- return;
- } else if (!dpu_kms->dev->dev_private) {
- DPU_ERROR("invalid device private\n");
- return;
- }
- priv = dpu_kms->dev->dev_private;
-
pm_runtime_get_sync(&dpu_kms->pdev->dev);
dpu_clear_all_irqs(dpu_kms);
dpu_disable_all_irqs(dpu_kms);
void dpu_core_irq_uninstall(struct dpu_kms *dpu_kms)
{
- struct msm_drm_private *priv;
int i;
- if (!dpu_kms->dev) {
- DPU_ERROR("invalid drm device\n");
- return;
- } else if (!dpu_kms->dev->dev_private) {
- DPU_ERROR("invalid device private\n");
- return;
- }
- priv = dpu_kms->dev->dev_private;
-
pm_runtime_get_sync(&dpu_kms->pdev->dev);
for (i = 0; i < dpu_kms->irq_obj.total_irqs; i++)
if (atomic_read(&dpu_kms->irq_obj.enable_counts[i]) ||
static struct dpu_kms *_dpu_crtc_get_kms(struct drm_crtc *crtc)
{
struct msm_drm_private *priv;
-
- if (!crtc->dev || !crtc->dev->dev_private) {
- DPU_ERROR("invalid device\n");
- return NULL;
- }
-
priv = crtc->dev->dev_private;
- if (!priv || !priv->kms) {
- DPU_ERROR("invalid kms\n");
- return NULL;
- }
-
return to_dpu_kms(priv->kms);
}
}
kms = _dpu_crtc_get_kms(crtc);
- if (!kms || !kms->catalog) {
+ if (!kms->catalog) {
DPU_ERROR("invalid parameters\n");
return 0;
}
void dpu_core_perf_crtc_release_bw(struct drm_crtc *crtc)
{
struct dpu_crtc *dpu_crtc;
- struct dpu_crtc_state *dpu_cstate;
struct dpu_kms *kms;
if (!crtc) {
}
kms = _dpu_crtc_get_kms(crtc);
- if (!kms || !kms->catalog) {
+ if (!kms->catalog) {
DPU_ERROR("invalid kms\n");
return;
}
dpu_crtc = to_dpu_crtc(crtc);
- dpu_cstate = to_dpu_crtc_state(crtc->state);
if (atomic_dec_return(&kms->bandwidth_ref) > 0)
return;
u64 clk_rate = 0;
struct dpu_crtc *dpu_crtc;
struct dpu_crtc_state *dpu_cstate;
- struct msm_drm_private *priv;
struct dpu_kms *kms;
int ret;
}
kms = _dpu_crtc_get_kms(crtc);
- if (!kms || !kms->catalog) {
+ if (!kms->catalog) {
DPU_ERROR("invalid kms\n");
return -EINVAL;
}
- priv = kms->dev->dev_private;
dpu_crtc = to_dpu_crtc(crtc);
dpu_cstate = to_dpu_crtc_state(crtc->state);
{
struct drm_encoder *encoder;
- if (!crtc || !crtc->dev) {
+ if (!crtc) {
DPU_ERROR("invalid crtc\n");
return INTF_MODE_NONE;
}
+ /*
+ * TODO: This function is called from dpu debugfs and as part of atomic
+ * check. When called from debugfs, the crtc->mutex must be held to
+ * read crtc->state. However reading crtc->state from atomic check isn't
+ * allowed (unless you have a good reason, a big comment, and a deep
+ * understanding of how the atomic/modeset locks work (<- and this is
+ * probably not possible)). So we'll keep the WARN_ON here for now, but
+ * really we need to figure out a better way to track our operating mode
+ */
WARN_ON(!drm_modeset_is_locked(&crtc->mutex));
/* TODO: Returns the first INTF_MODE, could there be multiple values? */
unsigned long flags;
bool release_bandwidth = false;
- if (!crtc || !crtc->dev || !crtc->dev->dev_private || !crtc->state) {
+ if (!crtc || !crtc->state) {
DPU_ERROR("invalid crtc\n");
return;
}
struct msm_drm_private *priv;
bool request_bandwidth;
- if (!crtc || !crtc->dev || !crtc->dev->dev_private) {
+ if (!crtc) {
DPU_ERROR("invalid crtc\n");
return;
}
{
struct drm_crtc *crtc = NULL;
struct dpu_crtc *dpu_crtc = NULL;
- struct msm_drm_private *priv = NULL;
- struct dpu_kms *kms = NULL;
int i;
- priv = dev->dev_private;
- kms = to_dpu_kms(priv->kms);
-
dpu_crtc = kzalloc(sizeof(*dpu_crtc), GFP_KERNEL);
if (!dpu_crtc)
return ERR_PTR(-ENOMEM);
priv = drm_enc->dev->dev_private;
dpu_kms = to_dpu_kms(priv->kms);
- if (!dpu_kms) {
- DPU_ERROR("invalid dpu_kms\n");
- return;
- }
-
hw_mdptop = dpu_kms->hw_mdp;
if (!hw_mdptop) {
DPU_ERROR("invalid mdptop\n");
struct msm_drm_private *priv;
bool is_vid_mode = false;
- if (!drm_enc || !drm_enc->dev || !drm_enc->dev->dev_private ||
- !drm_enc->crtc) {
+ if (!drm_enc || !drm_enc->dev || !drm_enc->crtc) {
DPU_ERROR("invalid parameters\n");
return -EINVAL;
}
struct msm_drm_private *priv;
struct dpu_kms *dpu_kms;
- if (!drm_enc || !drm_enc->dev || !drm_enc->dev->dev_private) {
+ if (!drm_enc || !drm_enc->dev) {
DPU_ERROR("invalid parameters\n");
return;
}
priv = drm_enc->dev->dev_private;
dpu_kms = to_dpu_kms(priv->kms);
- if (!dpu_kms) {
- DPU_ERROR("invalid dpu_kms\n");
- return;
- }
dpu_enc = to_dpu_encoder_virt(drm_enc);
if (!dpu_enc || !dpu_enc->cur_master) {
struct dpu_encoder_virt *dpu_enc = NULL;
struct msm_drm_private *priv;
struct dpu_kms *dpu_kms;
- struct drm_display_mode *mode;
int i = 0;
if (!drm_enc) {
} else if (!drm_enc->dev) {
DPU_ERROR("invalid dev\n");
return;
- } else if (!drm_enc->dev->dev_private) {
- DPU_ERROR("invalid dev_private\n");
- return;
}
dpu_enc = to_dpu_encoder_virt(drm_enc);
mutex_lock(&dpu_enc->enc_lock);
dpu_enc->enabled = false;
- mode = &drm_enc->crtc->state->adjusted_mode;
-
priv = drm_enc->dev->dev_private;
dpu_kms = to_dpu_kms(priv->kms);
struct msm_drm_private *priv;
struct msm_drm_thread *event_thread;
- if (!drm_enc->dev || !drm_enc->dev->dev_private ||
- !drm_enc->crtc) {
+ if (!drm_enc->dev || !drm_enc->crtc) {
DPU_ERROR("invalid parameters\n");
return;
}
static int _dpu_encoder_init_debugfs(struct drm_encoder *drm_enc)
{
struct dpu_encoder_virt *dpu_enc = to_dpu_encoder_virt(drm_enc);
- struct msm_drm_private *priv;
- struct dpu_kms *dpu_kms;
int i;
static const struct file_operations debugfs_status_fops = {
char name[DPU_NAME_SIZE];
- if (!drm_enc->dev || !drm_enc->dev->dev_private) {
+ if (!drm_enc->dev) {
DPU_ERROR("invalid encoder or kms\n");
return -EINVAL;
}
- priv = drm_enc->dev->dev_private;
- dpu_kms = to_dpu_kms(priv->kms);
-
snprintf(name, DPU_NAME_SIZE, "encoder%u", drm_enc->base.id);
/* create overall sub-directory for the encoder */
enum dpu_intf_type intf_type;
struct dpu_enc_phys_init_params phys_params;
- if (!dpu_enc || !dpu_kms) {
- DPU_ERROR("invalid arg(s), enc %d kms %d\n",
- dpu_enc != 0, dpu_kms != 0);
+ if (!dpu_enc) {
+ DPU_ERROR("invalid arg(s), enc %d\n", dpu_enc != 0);
return -EINVAL;
}
struct dpu_encoder_virt *dpu_enc = from_timer(dpu_enc, t,
frame_done_timer);
struct drm_encoder *drm_enc = &dpu_enc->base;
- struct msm_drm_private *priv;
u32 event;
- if (!drm_enc->dev || !drm_enc->dev->dev_private) {
+ if (!drm_enc->dev) {
DPU_ERROR("invalid parameters\n");
return;
}
- priv = drm_enc->dev->dev_private;
if (!dpu_enc->frame_busy_mask[0] || !dpu_enc->crtc_frame_event_cb) {
DRM_DEBUG_KMS("id:%u invalid timeout frame_busy_mask=%lu\n",
static void dpu_encoder_phys_cmd_ctl_start_irq(void *arg, int irq_idx)
{
struct dpu_encoder_phys *phys_enc = arg;
- struct dpu_encoder_phys_cmd *cmd_enc;
if (!phys_enc || !phys_enc->hw_ctl)
return;
DPU_ATRACE_BEGIN("ctl_start_irq");
- cmd_enc = to_dpu_encoder_phys_cmd(phys_enc);
atomic_add_unless(&phys_enc->pending_ctlstart_cnt, -1, 0);
static void dpu_encoder_phys_cmd_irq_control(struct dpu_encoder_phys *phys_enc,
bool enable)
{
- struct dpu_encoder_phys_cmd *cmd_enc;
-
if (!phys_enc)
return;
- cmd_enc = to_dpu_encoder_phys_cmd(phys_enc);
-
trace_dpu_enc_phys_cmd_irq_ctrl(DRMID(phys_enc->parent),
phys_enc->hw_pp->idx - PINGPONG_0,
enable, atomic_read(&phys_enc->vblank_refcount));
struct drm_display_mode *mode;
bool tc_enable = true;
u32 vsync_hz;
- struct msm_drm_private *priv;
struct dpu_kms *dpu_kms;
if (!phys_enc || !phys_enc->hw_pp) {
}
dpu_kms = phys_enc->dpu_kms;
- if (!dpu_kms || !dpu_kms->dev || !dpu_kms->dev->dev_private) {
- DPU_ERROR("invalid device\n");
- return;
- }
- priv = dpu_kms->dev->dev_private;
/*
* TE default: dsi byte clock calculated base on 70 fps;
struct dpu_encoder_phys *phys_enc)
{
int rc;
- struct dpu_encoder_phys_cmd *cmd_enc;
if (!phys_enc)
return -EINVAL;
- cmd_enc = to_dpu_encoder_phys_cmd(phys_enc);
-
rc = _dpu_encoder_phys_cmd_wait_for_idle(phys_enc);
if (rc) {
DRM_ERROR("failed wait_for_idle: id:%u ret:%d intf:%d\n",
struct drm_display_mode *mode,
struct drm_display_mode *adj_mode)
{
- if (!phys_enc || !phys_enc->dpu_kms) {
+ if (!phys_enc) {
DPU_ERROR("invalid encoder/kms\n");
return;
}
static void dpu_encoder_phys_vid_disable(struct dpu_encoder_phys *phys_enc)
{
- struct msm_drm_private *priv;
unsigned long lock_flags;
int ret;
- if (!phys_enc || !phys_enc->parent || !phys_enc->parent->dev ||
- !phys_enc->parent->dev->dev_private) {
+ if (!phys_enc || !phys_enc->parent || !phys_enc->parent->dev) {
DPU_ERROR("invalid encoder/device\n");
return;
}
- priv = phys_enc->parent->dev->dev_private;
if (!phys_enc->hw_intf || !phys_enc->hw_ctl) {
DPU_ERROR("invalid hw_intf %d hw_ctl %d\n",
#define CREATE_TRACE_POINTS
#include "dpu_trace.h"
-static const char * const iommu_ports[] = {
- "mdp_0",
-};
-
/*
* To enable overall DRM driver logging
* # echo 0x2 > /sys/module/drm/parameters/debug
bool danger_status)
{
struct dpu_kms *kms = (struct dpu_kms *)s->private;
- struct msm_drm_private *priv;
struct dpu_danger_safe_status status;
int i;
- if (!kms->dev || !kms->dev->dev_private || !kms->hw_mdp) {
+ if (!kms->hw_mdp) {
DPU_ERROR("invalid arg(s)\n");
return 0;
}
- priv = kms->dev->dev_private;
memset(&status, 0, sizeof(struct dpu_danger_safe_status));
pm_runtime_get_sync(&kms->pdev->dev);
return 0;
dev = dpu_kms->dev;
- if (!dev)
- return 0;
-
priv = dev->dev_private;
- if (!priv)
- return 0;
-
base = dpu_kms->mmio + regset->offset;
/* insert padding spaces, if needed */
struct drm_atomic_state *state)
{
struct dpu_kms *dpu_kms;
- struct msm_drm_private *priv;
struct drm_device *dev;
struct drm_crtc *crtc;
struct drm_crtc_state *crtc_state;
dpu_kms = to_dpu_kms(kms);
dev = dpu_kms->dev;
- if (!dev || !dev->dev_private)
- return;
- priv = dev->dev_private;
-
/* Call prepare_commit for all affected encoders */
for_each_new_crtc_in_state(state, crtc, crtc_state, i) {
drm_for_each_encoder_mask(encoder, crtc->dev,
if (funcs && funcs->commit)
funcs->commit(encoder);
- WARN_ON(!drm_modeset_is_locked(&dev->mode_config.connection_mutex));
drm_for_each_crtc(crtc, dev) {
if (!(crtc->state->encoder_mask & drm_encoder_mask(encoder)))
continue;
struct msm_drm_private *priv;
int i;
- if (!dpu_kms) {
- DPU_ERROR("invalid dpu_kms\n");
- return;
- } else if (!dpu_kms->dev) {
- DPU_ERROR("invalid dev\n");
- return;
- } else if (!dpu_kms->dev->dev_private) {
- DPU_ERROR("invalid dev_private\n");
- return;
- }
priv = dpu_kms->dev->dev_private;
for (i = 0; i < priv->num_crtcs; i++)
int primary_planes_idx = 0, cursor_planes_idx = 0, i, ret;
int max_crtc_count;
-
dev = dpu_kms->dev;
priv = dev->dev_private;
catalog = dpu_kms->catalog;
int i;
dev = dpu_kms->dev;
- if (!dev)
- return;
if (dpu_kms->hw_intr)
dpu_hw_intr_destroy(dpu_kms->hw_intr);
mmu = dpu_kms->base.aspace->mmu;
- mmu->funcs->detach(mmu, (const char **)iommu_ports,
- ARRAY_SIZE(iommu_ports));
+ mmu->funcs->detach(mmu);
msm_gem_address_space_put(dpu_kms->base.aspace);
dpu_kms->base.aspace = NULL;
return PTR_ERR(aspace);
}
- ret = aspace->mmu->funcs->attach(aspace->mmu, iommu_ports,
- ARRAY_SIZE(iommu_ports));
+ ret = aspace->mmu->funcs->attach(aspace->mmu);
if (ret) {
DPU_ERROR("failed to attach iommu %d\n", ret);
msm_gem_address_space_put(aspace);
dpu_kms = to_dpu_kms(kms);
dev = dpu_kms->dev;
- if (!dev) {
- DPU_ERROR("invalid device\n");
- return rc;
- }
-
priv = dev->dev_private;
- if (!priv) {
- DPU_ERROR("invalid private data\n");
- return rc;
- }
atomic_set(&dpu_kms->bandwidth_ref, 0);
struct dpu_kms *dpu_kms;
int irq;
- if (!dev || !dev->dev_private) {
+ if (!dev) {
DPU_ERROR("drm device node invalid\n");
return ERR_PTR(-EINVAL);
}
struct dss_module_power *mp = &dpu_kms->mp;
ddev = dpu_kms->dev;
- if (!ddev) {
- DPU_ERROR("invalid drm_device\n");
- return rc;
- }
-
rc = msm_dss_enable_clk(mp->clk_config, mp->num_clk, false);
if (rc)
DPU_ERROR("clock disable failed rc:%d\n", rc);
struct dss_module_power *mp = &dpu_kms->mp;
ddev = dpu_kms->dev;
- if (!ddev) {
- DPU_ERROR("invalid drm_device\n");
- return rc;
- }
-
rc = msm_dss_enable_clk(mp->clk_config, mp->num_clk, true);
if (rc) {
DPU_ERROR("clock enable failed rc:%d\n", rc);
#define to_dpu_kms(x) container_of(x, struct dpu_kms, base)
-/* get struct msm_kms * from drm_device * */
-#define ddev_to_msm_kms(D) ((D) && (D)->dev_private ? \
- ((struct msm_drm_private *)((D)->dev_private))->kms : NULL)
-
/**
* Debugfs functions - extra helper functions for debugfs support
*
u32 ot_lim;
int ret, i;
- if (!dpu_kms) {
- DPU_ERROR("invalid arguments\n");
- return;
- }
mdp = dpu_kms->hw_mdp;
for (i = 0; i < ARRAY_SIZE(dpu_kms->hw_vbif); i++) {
const struct dpu_vbif_qos_tbl *qos_tbl;
int i;
- if (!dpu_kms || !params || !dpu_kms->hw_mdp) {
+ if (!params || !dpu_kms->hw_mdp) {
DPU_ERROR("invalid arguments\n");
return;
}
}
}
-static const char * const iommu_ports[] = {
- "mdp_port0_cb0", "mdp_port1_cb0",
-};
-
static void mdp4_destroy(struct msm_kms *kms)
{
struct mdp4_kms *mdp4_kms = to_mdp4_kms(to_mdp_kms(kms));
drm_gem_object_put_unlocked(mdp4_kms->blank_cursor_bo);
if (aspace) {
- aspace->mmu->funcs->detach(aspace->mmu,
- iommu_ports, ARRAY_SIZE(iommu_ports));
+ aspace->mmu->funcs->detach(aspace->mmu);
msm_gem_address_space_put(aspace);
}
kms->aspace = aspace;
- ret = aspace->mmu->funcs->attach(aspace->mmu, iommu_ports,
- ARRAY_SIZE(iommu_ports));
+ ret = aspace->mmu->funcs->attach(aspace->mmu);
if (ret)
goto fail;
} else {
/* mdp5_cfg must be exposed (used in mdp5.xml.h) */
const struct mdp5_cfg_hw *mdp5_cfg = NULL;
-const struct mdp5_cfg_hw msm8x74v1_config = {
+static const struct mdp5_cfg_hw msm8x74v1_config = {
.name = "msm8x74v1",
.mdp = {
.count = 1,
.max_clk = 200000000,
};
-const struct mdp5_cfg_hw msm8x74v2_config = {
+static const struct mdp5_cfg_hw msm8x74v2_config = {
.name = "msm8x74",
.mdp = {
.count = 1,
.max_clk = 200000000,
};
-const struct mdp5_cfg_hw apq8084_config = {
+static const struct mdp5_cfg_hw apq8084_config = {
.name = "apq8084",
.mdp = {
.count = 1,
.max_clk = 320000000,
};
-const struct mdp5_cfg_hw msm8x16_config = {
+static const struct mdp5_cfg_hw msm8x16_config = {
.name = "msm8x16",
.mdp = {
.count = 1,
.max_clk = 320000000,
};
-const struct mdp5_cfg_hw msm8x94_config = {
+static const struct mdp5_cfg_hw msm8x94_config = {
.name = "msm8x94",
.mdp = {
.count = 1,
.max_clk = 400000000,
};
-const struct mdp5_cfg_hw msm8x96_config = {
+static const struct mdp5_cfg_hw msm8x96_config = {
.name = "msm8x96",
.mdp = {
.count = 1,
.max_clk = 412500000,
};
-const struct mdp5_cfg_hw msm8917_config = {
+const struct mdp5_cfg_hw msm8x76_config = {
+ .name = "msm8x76",
+ .mdp = {
+ .count = 1,
+ .caps = MDP_CAP_SMP |
+ MDP_CAP_DSC |
+ MDP_CAP_SRC_SPLIT |
+ 0,
+ },
+ .ctl = {
+ .count = 3,
+ .base = { 0x01000, 0x01200, 0x01400 },
+ .flush_hw_mask = 0xffffffff,
+ },
+ .smp = {
+ .mmb_count = 10,
+ .mmb_size = 10240,
+ .clients = {
+ [SSPP_VIG0] = 1, [SSPP_VIG1] = 9,
+ [SSPP_DMA0] = 4,
+ [SSPP_RGB0] = 7, [SSPP_RGB1] = 8,
+ },
+ },
+ .pipe_vig = {
+ .count = 2,
+ .base = { 0x04000, 0x06000 },
+ .caps = MDP_PIPE_CAP_HFLIP |
+ MDP_PIPE_CAP_VFLIP |
+ MDP_PIPE_CAP_SCALE |
+ MDP_PIPE_CAP_CSC |
+ MDP_PIPE_CAP_DECIMATION |
+ MDP_PIPE_CAP_SW_PIX_EXT |
+ 0,
+ },
+ .pipe_rgb = {
+ .count = 2,
+ .base = { 0x14000, 0x16000 },
+ .caps = MDP_PIPE_CAP_HFLIP |
+ MDP_PIPE_CAP_VFLIP |
+ MDP_PIPE_CAP_DECIMATION |
+ MDP_PIPE_CAP_SW_PIX_EXT |
+ 0,
+ },
+ .pipe_dma = {
+ .count = 1,
+ .base = { 0x24000 },
+ .caps = MDP_PIPE_CAP_HFLIP |
+ MDP_PIPE_CAP_VFLIP |
+ MDP_PIPE_CAP_SW_PIX_EXT |
+ 0,
+ },
+ .pipe_cursor = {
+ .count = 1,
+ .base = { 0x440DC },
+ .caps = MDP_PIPE_CAP_HFLIP |
+ MDP_PIPE_CAP_VFLIP |
+ MDP_PIPE_CAP_SW_PIX_EXT |
+ MDP_PIPE_CAP_CURSOR |
+ 0,
+ },
+ .lm = {
+ .count = 2,
+ .base = { 0x44000, 0x45000 },
+ .instances = {
+ { .id = 0, .pp = 0, .dspp = 0,
+ .caps = MDP_LM_CAP_DISPLAY, },
+ { .id = 1, .pp = -1, .dspp = -1,
+ .caps = MDP_LM_CAP_WB },
+ },
+ .nb_stages = 8,
+ .max_width = 2560,
+ .max_height = 0xFFFF,
+ },
+ .dspp = {
+ .count = 1,
+ .base = { 0x54000 },
+
+ },
+ .pp = {
+ .count = 3,
+ .base = { 0x70000, 0x70800, 0x72000 },
+ },
+ .dsc = {
+ .count = 2,
+ .base = { 0x80000, 0x80400 },
+ },
+ .intf = {
+ .base = { 0x6a000, 0x6a800, 0x6b000 },
+ .connect = {
+ [0] = INTF_DISABLED,
+ [1] = INTF_DSI,
+ [2] = INTF_DSI,
+ },
+ },
+ .max_clk = 360000000,
+};
+
+static const struct mdp5_cfg_hw msm8917_config = {
.name = "msm8917",
.mdp = {
.count = 1,
.max_clk = 320000000,
};
-const struct mdp5_cfg_hw msm8998_config = {
+static const struct mdp5_cfg_hw msm8998_config = {
.name = "msm8998",
.mdp = {
.count = 1,
{ .revision = 6, .config = { .hw = &msm8x16_config } },
{ .revision = 9, .config = { .hw = &msm8x94_config } },
{ .revision = 7, .config = { .hw = &msm8x96_config } },
+ { .revision = 11, .config = { .hw = &msm8x76_config } },
{ .revision = 15, .config = { .hw = &msm8917_config } },
};
struct mdp5_pipeline *pipeline = &mdp5_cstate->pipeline;
struct mdp5_kms *mdp5_kms = get_kms(crtc);
struct drm_plane *plane;
- const struct mdp5_cfg_hw *hw_cfg;
struct mdp5_plane_state *pstate, *pstates[STAGE_MAX + 1] = {NULL};
const struct mdp_format *format;
struct mdp5_hw_mixer *mixer = pipeline->mixer;
u32 val;
#define blender(stage) ((stage) - STAGE0)
- hw_cfg = mdp5_cfg_get_hw_config(mdp5_kms->cfg);
-
spin_lock_irqsave(&mdp5_crtc->lm_lock, flags);
/* ctl could be released already when we are shutting down: */
#include "msm_mmu.h"
#include "mdp5_kms.h"
-static const char *iommu_ports[] = {
- "mdp_0",
-};
-
static int mdp5_hw_init(struct msm_kms *kms)
{
struct mdp5_kms *mdp5_kms = to_mdp5_kms(to_mdp_kms(kms));
mdp5_pipe_destroy(mdp5_kms->hwpipes[i]);
if (aspace) {
- aspace->mmu->funcs->detach(aspace->mmu,
- iommu_ports, ARRAY_SIZE(iommu_ports));
+ aspace->mmu->funcs->detach(aspace->mmu);
msm_gem_address_space_put(aspace);
}
}
mdp5_kms->enable_count--;
WARN_ON(mdp5_kms->enable_count < 0);
+ if (mdp5_kms->tbu_rt_clk)
+ clk_disable_unprepare(mdp5_kms->tbu_rt_clk);
+ if (mdp5_kms->tbu_clk)
+ clk_disable_unprepare(mdp5_kms->tbu_clk);
clk_disable_unprepare(mdp5_kms->ahb_clk);
clk_disable_unprepare(mdp5_kms->axi_clk);
clk_disable_unprepare(mdp5_kms->core_clk);
clk_prepare_enable(mdp5_kms->core_clk);
if (mdp5_kms->lut_clk)
clk_prepare_enable(mdp5_kms->lut_clk);
+ if (mdp5_kms->tbu_clk)
+ clk_prepare_enable(mdp5_kms->tbu_clk);
+ if (mdp5_kms->tbu_rt_clk)
+ clk_prepare_enable(mdp5_kms->tbu_rt_clk);
return 0;
}
{
struct drm_device *dev = mdp5_kms->dev;
struct msm_drm_private *priv = dev->dev_private;
- const struct mdp5_cfg_hw *hw_cfg;
unsigned int num_crtcs;
int i, ret, pi = 0, ci = 0;
struct drm_plane *primary[MAX_BASES] = { NULL };
struct drm_plane *cursor[MAX_BASES] = { NULL };
- hw_cfg = mdp5_cfg_get_hw_config(mdp5_kms->cfg);
-
/*
* Construct encoders and modeset initialize connector devices
* for each external display interface.
kms->aspace = aspace;
- ret = aspace->mmu->funcs->attach(aspace->mmu, iommu_ports,
- ARRAY_SIZE(iommu_ports));
+ ret = aspace->mmu->funcs->attach(aspace->mmu);
if (ret) {
DRM_DEV_ERROR(&pdev->dev, "failed to attach iommu: %d\n",
ret);
/* optional clocks: */
get_clk(pdev, &mdp5_kms->lut_clk, "lut", false);
+ get_clk(pdev, &mdp5_kms->tbu_clk, "tbu", false);
+ get_clk(pdev, &mdp5_kms->tbu_rt_clk, "tbu_rt", false);
/* we need to set a default rate before enabling. Set a safe
* rate first, then figure out hw revision, and then set a
struct clk *ahb_clk;
struct clk *core_clk;
struct clk *lut_clk;
+ struct clk *tbu_clk;
+ struct clk *tbu_rt_clk;
struct clk *vsync_clk;
/*
struct mdp5_kms *mdp5_kms = get_kms(smp);
int rev = mdp5_cfg_get_hw_rev(mdp5_kms->cfg);
int i, hsub, nplanes, nlines;
- u32 fmt = format->base.pixel_format;
uint32_t blkcfg = 0;
nplanes = info->num_planes;
* them together, writes to SMP using a single client.
*/
if ((rev > 0) && (format->chroma_sample > CHROMA_FULL)) {
- fmt = DRM_FORMAT_NV24;
nplanes = 2;
/* if decimation is enabled, HW decimates less on the
.num_dsi = 1,
};
+static const char * const dsi_8976_bus_clk_names[] = {
+ "mdp_core", "iface", "bus",
+};
+
+static const struct msm_dsi_config msm8976_dsi_cfg = {
+ .io_offset = DSI_6G_REG_SHIFT,
+ .reg_cfg = {
+ .num = 3,
+ .regs = {
+ {"gdsc", -1, -1},
+ {"vdda", 100000, 100}, /* 1.2 V */
+ {"vddio", 100000, 100}, /* 1.8 V */
+ },
+ },
+ .bus_clk_names = dsi_8976_bus_clk_names,
+ .num_bus_clks = ARRAY_SIZE(dsi_8976_bus_clk_names),
+ .io_start = { 0x1a94000, 0x1a96000 },
+ .num_dsi = 2,
+};
+
static const struct msm_dsi_config msm8994_dsi_cfg = {
.io_offset = DSI_6G_REG_SHIFT,
.reg_cfg = {
.num_dsi = 2,
};
-const static struct msm_dsi_host_cfg_ops msm_dsi_v2_host_ops = {
+static const struct msm_dsi_host_cfg_ops msm_dsi_v2_host_ops = {
.link_clk_enable = dsi_link_clk_enable_v2,
.link_clk_disable = dsi_link_clk_disable_v2,
.clk_init_ver = dsi_clk_init_v2,
.calc_clk_rate = dsi_calc_clk_rate_v2,
};
-const static struct msm_dsi_host_cfg_ops msm_dsi_6g_host_ops = {
+static const struct msm_dsi_host_cfg_ops msm_dsi_6g_host_ops = {
.link_clk_enable = dsi_link_clk_enable_6g,
.link_clk_disable = dsi_link_clk_disable_6g,
.clk_init_ver = NULL,
.calc_clk_rate = dsi_calc_clk_rate_6g,
};
-const static struct msm_dsi_host_cfg_ops msm_dsi_6g_v2_host_ops = {
+static const struct msm_dsi_host_cfg_ops msm_dsi_6g_v2_host_ops = {
.link_clk_enable = dsi_link_clk_enable_6g,
.link_clk_disable = dsi_link_clk_disable_6g,
.clk_init_ver = dsi_clk_init_6g_v2,
&msm8916_dsi_cfg, &msm_dsi_6g_host_ops},
{MSM_DSI_VER_MAJOR_6G, MSM_DSI_6G_VER_MINOR_V1_4_1,
&msm8996_dsi_cfg, &msm_dsi_6g_host_ops},
+ {MSM_DSI_VER_MAJOR_6G, MSM_DSI_6G_VER_MINOR_V1_4_2,
+ &msm8976_dsi_cfg, &msm_dsi_6g_host_ops},
{MSM_DSI_VER_MAJOR_6G, MSM_DSI_6G_VER_MINOR_V2_2_0,
&msm8998_dsi_cfg, &msm_dsi_6g_v2_host_ops},
{MSM_DSI_VER_MAJOR_6G, MSM_DSI_6G_VER_MINOR_V2_2_1,
#define MSM_DSI_6G_VER_MINOR_V1_3 0x10030000
#define MSM_DSI_6G_VER_MINOR_V1_3_1 0x10030001
#define MSM_DSI_6G_VER_MINOR_V1_4_1 0x10040001
+#define MSM_DSI_6G_VER_MINOR_V1_4_2 0x10040002
#define MSM_DSI_6G_VER_MINOR_V2_2_0 0x20000000
#define MSM_DSI_6G_VER_MINOR_V2_2_1 0x20020001
static int dsi_cmd_dma_rx(struct msm_dsi_host *msm_host,
u8 *buf, int rx_byte, int pkt_size)
{
- u32 *lp, *temp, data;
+ u32 *temp, data;
int i, j = 0, cnt;
u32 read_cnt;
u8 reg[16];
int repeated_bytes = 0;
int buf_offset = buf - msm_host->rx_buf;
- lp = (u32 *)buf;
temp = (u32 *)reg;
cnt = (rx_byte + 3) >> 2;
if (cnt > 4)
{
const unsigned long bit_rate = clk_req->bitclk_rate;
const unsigned long esc_rate = clk_req->escclk_rate;
- s32 ui, ui_x8, lpx;
+ s32 ui, ui_x8;
s32 tmax, tmin;
s32 pcnt0 = 50;
s32 pcnt1 = 50;
ui = mult_frac(NSEC_PER_MSEC, coeff, bit_rate / 1000);
ui_x8 = ui << 3;
- lpx = mult_frac(NSEC_PER_MSEC, coeff, esc_rate / 1000);
temp = S_DIV_ROUND_UP(38 * coeff - val_ckln * ui, ui_x8);
tmin = max_t(s32, temp, 0);
{
const unsigned long bit_rate = clk_req->bitclk_rate;
const unsigned long esc_rate = clk_req->escclk_rate;
- s32 ui, ui_x8, lpx;
+ s32 ui, ui_x8;
s32 tmax, tmin;
s32 pcnt0 = 50;
s32 pcnt1 = 50;
ui = mult_frac(NSEC_PER_MSEC, coeff, bit_rate / 1000);
ui_x8 = ui << 3;
- lpx = mult_frac(NSEC_PER_MSEC, coeff, esc_rate / 1000);
temp = S_DIV_ROUND_UP(38 * coeff, ui_x8);
tmin = max_t(s32, temp, 0);
#ifdef CONFIG_DRM_MSM_DSI_28NM_PHY
{ .compatible = "qcom,dsi-phy-28nm-hpm",
.data = &dsi_phy_28nm_hpm_cfgs },
+ { .compatible = "qcom,dsi-phy-28nm-hpm-fam-b",
+ .data = &dsi_phy_28nm_hpm_famb_cfgs },
{ .compatible = "qcom,dsi-phy-28nm-lp",
.data = &dsi_phy_28nm_lp_cfgs },
#endif
};
extern const struct msm_dsi_phy_cfg dsi_phy_28nm_hpm_cfgs;
+extern const struct msm_dsi_phy_cfg dsi_phy_28nm_hpm_famb_cfgs;
extern const struct msm_dsi_phy_cfg dsi_phy_28nm_lp_cfgs;
extern const struct msm_dsi_phy_cfg dsi_phy_20nm_cfgs;
extern const struct msm_dsi_phy_cfg dsi_phy_28nm_8960_cfgs;
DSI_28nm_PHY_TIMING_CTRL_11_TRIG3_CMD(0));
}
-static void dsi_28nm_phy_regulator_ctrl(struct msm_dsi_phy *phy, bool enable)
+static void dsi_28nm_phy_regulator_enable_dcdc(struct msm_dsi_phy *phy)
{
void __iomem *base = phy->reg_base;
- if (!enable) {
- dsi_phy_write(base + REG_DSI_28nm_PHY_REGULATOR_CAL_PWR_CFG, 0);
- return;
- }
-
dsi_phy_write(base + REG_DSI_28nm_PHY_REGULATOR_CTRL_0, 0x0);
dsi_phy_write(base + REG_DSI_28nm_PHY_REGULATOR_CAL_PWR_CFG, 1);
dsi_phy_write(base + REG_DSI_28nm_PHY_REGULATOR_CTRL_5, 0);
dsi_phy_write(base + REG_DSI_28nm_PHY_REGULATOR_CTRL_1, 0x9);
dsi_phy_write(base + REG_DSI_28nm_PHY_REGULATOR_CTRL_0, 0x7);
dsi_phy_write(base + REG_DSI_28nm_PHY_REGULATOR_CTRL_4, 0x20);
+ dsi_phy_write(phy->base + REG_DSI_28nm_PHY_LDO_CNTRL, 0x00);
+}
+
+static void dsi_28nm_phy_regulator_enable_ldo(struct msm_dsi_phy *phy)
+{
+ void __iomem *base = phy->reg_base;
+
+ dsi_phy_write(base + REG_DSI_28nm_PHY_REGULATOR_CTRL_0, 0x0);
+ dsi_phy_write(base + REG_DSI_28nm_PHY_REGULATOR_CAL_PWR_CFG, 0);
+ dsi_phy_write(base + REG_DSI_28nm_PHY_REGULATOR_CTRL_5, 0x7);
+ dsi_phy_write(base + REG_DSI_28nm_PHY_REGULATOR_CTRL_3, 0);
+ dsi_phy_write(base + REG_DSI_28nm_PHY_REGULATOR_CTRL_2, 0x1);
+ dsi_phy_write(base + REG_DSI_28nm_PHY_REGULATOR_CTRL_1, 0x1);
+ dsi_phy_write(base + REG_DSI_28nm_PHY_REGULATOR_CTRL_4, 0x20);
+
+ if (phy->cfg->type == MSM_DSI_PHY_28NM_LP)
+ dsi_phy_write(phy->base + REG_DSI_28nm_PHY_LDO_CNTRL, 0x05);
+ else
+ dsi_phy_write(phy->base + REG_DSI_28nm_PHY_LDO_CNTRL, 0x0d);
+}
+
+static void dsi_28nm_phy_regulator_ctrl(struct msm_dsi_phy *phy, bool enable)
+{
+ if (!enable) {
+ dsi_phy_write(phy->reg_base +
+ REG_DSI_28nm_PHY_REGULATOR_CAL_PWR_CFG, 0);
+ return;
+ }
+
+ if (phy->regulator_ldo_mode)
+ dsi_28nm_phy_regulator_enable_ldo(phy);
+ else
+ dsi_28nm_phy_regulator_enable_dcdc(phy);
}
static int dsi_28nm_phy_enable(struct msm_dsi_phy *phy, int src_pll_id,
dsi_28nm_phy_regulator_ctrl(phy, true);
- dsi_phy_write(base + REG_DSI_28nm_PHY_LDO_CNTRL, 0x00);
-
dsi_28nm_dphy_set_timing(phy, timing);
dsi_phy_write(base + REG_DSI_28nm_PHY_CTRL_1, 0x00);
.num_dsi_phy = 2,
};
+const struct msm_dsi_phy_cfg dsi_phy_28nm_hpm_famb_cfgs = {
+ .type = MSM_DSI_PHY_28NM_HPM,
+ .src_pll_truthtable = { {true, true}, {false, true} },
+ .reg_cfg = {
+ .num = 1,
+ .regs = {
+ {"vddio", 100000, 100},
+ },
+ },
+ .ops = {
+ .enable = dsi_28nm_phy_enable,
+ .disable = dsi_28nm_phy_disable,
+ .init = msm_dsi_phy_init_common,
+ },
+ .io_start = { 0x1a94400, 0x1a96400 },
+ .num_dsi_phy = 2,
+};
+
const struct msm_dsi_phy_cfg dsi_phy_28nm_lp_cfgs = {
.type = MSM_DSI_PHY_28NM_LP,
.src_pll_truthtable = { {true, true}, {true, true} },
reg = devm_regulator_get(dev, cfg->reg_names[i]);
if (IS_ERR(reg)) {
ret = PTR_ERR(reg);
- DRM_DEV_ERROR(dev, "failed to get phy regulator: %s (%d)\n",
- cfg->reg_names[i], ret);
+ if (ret != -EPROBE_DEFER) {
+ DRM_DEV_ERROR(dev,
+ "failed to get phy regulator: %s (%d)\n",
+ cfg->reg_names[i], ret);
+ }
+
return ret;
}
#include <linux/pm_opp.h>
#include <linux/devfreq.h>
#include <linux/devcoredump.h>
+#include <linux/sched/task.h>
/*
* Power Management:
return ERR_CAST(aspace);
}
- ret = aspace->mmu->funcs->attach(aspace->mmu, NULL, 0);
+ ret = aspace->mmu->funcs->attach(aspace->mmu);
if (ret) {
msm_gem_address_space_put(aspace);
return ERR_PTR(ret);
msm_gem_kernel_put(gpu->memptrs_bo, gpu->aspace, false);
if (!IS_ERR_OR_NULL(gpu->aspace)) {
- gpu->aspace->mmu->funcs->detach(gpu->aspace->mmu,
- NULL, 0);
+ gpu->aspace->mmu->funcs->detach(gpu->aspace->mmu);
msm_gem_address_space_put(gpu->aspace);
}
}
#define GPUMMU_PAGE_SIZE SZ_4K
#define TABLE_SIZE (sizeof(uint32_t) * GPUMMU_VA_RANGE / GPUMMU_PAGE_SIZE)
-static int msm_gpummu_attach(struct msm_mmu *mmu, const char * const *names,
- int cnt)
+static int msm_gpummu_attach(struct msm_mmu *mmu)
{
return 0;
}
-static void msm_gpummu_detach(struct msm_mmu *mmu, const char * const *names,
- int cnt)
+static void msm_gpummu_detach(struct msm_mmu *mmu)
{
}
return 0;
}
-static int msm_iommu_attach(struct msm_mmu *mmu, const char * const *names,
- int cnt)
+static int msm_iommu_attach(struct msm_mmu *mmu)
{
struct msm_iommu *iommu = to_msm_iommu(mmu);
return iommu_attach_device(iommu->domain, mmu->dev);
}
-static void msm_iommu_detach(struct msm_mmu *mmu, const char * const *names,
- int cnt)
+static void msm_iommu_detach(struct msm_mmu *mmu)
{
struct msm_iommu *iommu = to_msm_iommu(mmu);
#include <linux/iommu.h>
struct msm_mmu_funcs {
- int (*attach)(struct msm_mmu *mmu, const char * const *names, int cnt);
- void (*detach)(struct msm_mmu *mmu, const char * const *names, int cnt);
+ int (*attach)(struct msm_mmu *mmu);
+ void (*detach)(struct msm_mmu *mmu);
int (*map)(struct msm_mmu *mmu, uint64_t iova, struct sg_table *sgt,
unsigned len, int prot);
int (*unmap)(struct msm_mmu *mmu, uint64_t iova, unsigned len);
static void snapshot_buf(struct msm_rd_state *rd,
struct msm_gem_submit *submit, int idx,
- uint64_t iova, uint32_t size)
+ uint64_t iova, uint32_t size, bool full)
{
struct msm_gem_object *obj = submit->bos[idx].obj;
unsigned offset = 0;
rd_write_section(rd, RD_GPUADDR,
(uint32_t[3]){ iova, size, iova >> 32 }, 12);
+ if (!full)
+ return;
+
/* But only dump the contents of buffers marked READ */
if (!(submit->bos[idx].flags & MSM_SUBMIT_BO_READ))
return;
rd_write_section(rd, RD_CMD, msg, ALIGN(n, 4));
for (i = 0; i < submit->nr_bos; i++)
- if (should_dump(submit, i))
- snapshot_buf(rd, submit, i, 0, 0);
+ snapshot_buf(rd, submit, i, 0, 0, should_dump(submit, i));
for (i = 0; i < submit->nr_cmds; i++) {
- uint64_t iova = submit->cmd[i].iova;
uint32_t szd = submit->cmd[i].size; /* in dwords */
/* snapshot cmdstream bo's (if we haven't already): */
if (!should_dump(submit, i)) {
snapshot_buf(rd, submit, submit->cmd[i].idx,
- submit->cmd[i].iova, szd * 4);
+ submit->cmd[i].iova, szd * 4, true);
}
+ }
+
+ for (i = 0; i < submit->nr_cmds; i++) {
+ uint64_t iova = submit->cmd[i].iova;
+ uint32_t szd = submit->cmd[i].size; /* in dwords */
switch (submit->cmd[i].type) {
case MSM_SUBMIT_CMD_IB_TARGET_BUF:
*/
static void omap_gem_unpin_locked(struct drm_gem_object *obj)
{
+ struct omap_drm_private *priv = obj->dev->dev_private;
struct omap_gem_object *omap_obj = to_omap_bo(obj);
int ret;
+ if (omap_gem_is_contiguous(omap_obj) || !priv->has_dmm)
+ return;
+
if (refcount_dec_and_test(&omap_obj->dma_addr_cnt)) {
ret = tiler_unpin(omap_obj->block);
if (ret) {
track->textures[i].use_pitch = 1;
} else {
track->textures[i].use_pitch = 0;
- track->textures[i].width = 1 << ((idx_value >> RADEON_TXFORMAT_WIDTH_SHIFT) & RADEON_TXFORMAT_WIDTH_MASK);
- track->textures[i].height = 1 << ((idx_value >> RADEON_TXFORMAT_HEIGHT_SHIFT) & RADEON_TXFORMAT_HEIGHT_MASK);
+ track->textures[i].width = 1 << ((idx_value & RADEON_TXFORMAT_WIDTH_MASK) >> RADEON_TXFORMAT_WIDTH_SHIFT);
+ track->textures[i].height = 1 << ((idx_value & RADEON_TXFORMAT_HEIGHT_MASK) >> RADEON_TXFORMAT_HEIGHT_SHIFT);
}
if (idx_value & RADEON_TXFORMAT_CUBIC_MAP_ENABLE)
track->textures[i].tex_coord_type = 2;
track->textures[i].use_pitch = 1;
} else {
track->textures[i].use_pitch = 0;
- track->textures[i].width = 1 << ((idx_value >> RADEON_TXFORMAT_WIDTH_SHIFT) & RADEON_TXFORMAT_WIDTH_MASK);
- track->textures[i].height = 1 << ((idx_value >> RADEON_TXFORMAT_HEIGHT_SHIFT) & RADEON_TXFORMAT_HEIGHT_MASK);
+ track->textures[i].width = 1 << ((idx_value & RADEON_TXFORMAT_WIDTH_MASK) >> RADEON_TXFORMAT_WIDTH_SHIFT);
+ track->textures[i].height = 1 << ((idx_value & RADEON_TXFORMAT_HEIGHT_MASK) >> RADEON_TXFORMAT_HEIGHT_SHIFT);
}
if (idx_value & R200_TXFORMAT_LOOKUP_DISABLE)
track->textures[i].lookup_disable = true;
static void tegra_cursor_atomic_update(struct drm_plane *plane,
struct drm_plane_state *old_state)
{
- struct tegra_bo *bo = tegra_fb_get_plane(plane->state->fb, 0);
+ struct tegra_plane_state *state = to_tegra_plane_state(plane->state);
struct tegra_dc *dc = to_tegra_dc(plane->state->crtc);
- struct drm_plane_state *state = plane->state;
u32 value = CURSOR_CLIP_DISPLAY;
/* rien ne va plus */
if (!plane->state->crtc || !plane->state->fb)
return;
- switch (state->crtc_w) {
+ switch (plane->state->crtc_w) {
case 32:
value |= CURSOR_SIZE_32x32;
break;
break;
default:
- WARN(1, "cursor size %ux%u not supported\n", state->crtc_w,
- state->crtc_h);
+ WARN(1, "cursor size %ux%u not supported\n",
+ plane->state->crtc_w, plane->state->crtc_h);
return;
}
- value |= (bo->iova >> 10) & 0x3fffff;
+ value |= (state->iova[0] >> 10) & 0x3fffff;
tegra_dc_writel(dc, value, DC_DISP_CURSOR_START_ADDR);
#ifdef CONFIG_ARCH_DMA_ADDR_T_64BIT
- value = (bo->iova >> 32) & 0x3;
+ value = (state->iova[0] >> 32) & 0x3;
tegra_dc_writel(dc, value, DC_DISP_CURSOR_START_ADDR_HI);
#endif
tegra_dc_writel(dc, value, DC_DISP_BLEND_CURSOR_CONTROL);
/* position the cursor */
- value = (state->crtc_y & 0x3fff) << 16 | (state->crtc_x & 0x3fff);
+ value = (plane->state->crtc_y & 0x3fff) << 16 |
+ (plane->state->crtc_x & 0x3fff);
tegra_dc_writel(dc, value, DC_DISP_CURSOR_POSITION);
}
dev_warn(dc->dev, "failed to allocate syncpoint\n");
err = host1x_client_iommu_attach(client);
- if (err < 0) {
+ if (err < 0 && err != -ENODEV) {
dev_err(client->dev, "failed to attach to domain: %d\n", err);
return err;
}
if (tegra->domain) {
group = iommu_group_get(client->dev);
- if (!group) {
- dev_err(client->dev, "failed to get IOMMU group\n");
+ if (!group)
return -ENODEV;
- }
if (domain != tegra->domain) {
err = iommu_attach_group(tegra->domain, group);
drm_atomic_helper_shutdown(drm);
drm_mode_config_cleanup(drm);
+ if (tegra->hub)
+ tegra_display_hub_cleanup(tegra->hub);
+
err = host1x_device_exit(dev);
if (err < 0)
dev_err(&dev->dev, "host1x device cleanup failed: %d\n", err);
drm_gem_object_put_unlocked(&obj->gem);
}
+/* XXX move this into lib/scatterlist.c? */
+static int sg_alloc_table_from_sg(struct sg_table *sgt, struct scatterlist *sg,
+ unsigned int nents, gfp_t gfp_mask)
+{
+ struct scatterlist *dst;
+ unsigned int i;
+ int err;
+
+ err = sg_alloc_table(sgt, nents, gfp_mask);
+ if (err < 0)
+ return err;
+
+ dst = sgt->sgl;
+
+ for (i = 0; i < nents; i++) {
+ sg_set_page(dst, sg_page(sg), sg->length, 0);
+ dst = sg_next(dst);
+ sg = sg_next(sg);
+ }
+
+ return 0;
+}
+
static struct sg_table *tegra_bo_pin(struct device *dev, struct host1x_bo *bo,
dma_addr_t *phys)
{
return ERR_PTR(-ENOMEM);
if (obj->pages) {
+ /*
+ * If the buffer object was allocated from the explicit IOMMU
+ * API code paths, construct an SG table from the pages.
+ */
err = sg_alloc_table_from_pages(sgt, obj->pages, obj->num_pages,
0, obj->gem.size, GFP_KERNEL);
if (err < 0)
goto free;
+ } else if (obj->sgt) {
+ /*
+ * If the buffer object already has an SG table but no pages
+ * were allocated for it, it means the buffer was imported and
+ * the SG table needs to be copied to avoid overwriting any
+ * other potential users of the original SG table.
+ */
+ err = sg_alloc_table_from_sg(sgt, obj->sgt->sgl, obj->sgt->nents,
+ GFP_KERNEL);
+ if (err < 0)
+ goto free;
} else {
+ /*
+ * If the buffer object had no pages allocated and if it was
+ * not imported, it had to be allocated with the DMA API, so
+ * the DMA API helper can be used.
+ */
err = dma_get_sgtable(dev, sgt, obj->vaddr, obj->iova,
obj->gem.size);
if (err < 0)
err = tegra_bo_iommu_map(tegra, bo);
if (err < 0)
goto detach;
- } else {
- if (bo->sgt->nents > 1) {
- err = -EINVAL;
- goto detach;
- }
-
- bo->iova = sg_dma_address(bo->sgt->sgl);
}
bo->gem.import_attach = attach;
tegra_display_hub_get_state(struct tegra_display_hub *hub,
struct drm_atomic_state *state)
{
- struct drm_device *drm = dev_get_drvdata(hub->client.parent);
struct drm_private_state *priv;
- WARN_ON(!drm_modeset_is_locked(&drm->mode_config.connection_mutex));
-
priv = drm_atomic_get_private_obj_state(state, &hub->base);
if (IS_ERR(priv))
return ERR_CAST(priv);
goto unpin;
}
+ /*
+ * The display controller needs contiguous memory, so
+ * fail if the buffer is discontiguous and we fail to
+ * map its SG table to a single contiguous chunk of
+ * I/O virtual memory.
+ */
+ if (err > 1) {
+ err = -EINVAL;
+ goto unpin;
+ }
+
state->iova[i] = sg_dma_address(sgt->sgl);
state->sgt[i] = sgt;
} else {
return 0;
}
-#ifdef CONFIG_PM
-static int tegra_sor_suspend(struct device *dev)
+static int tegra_sor_runtime_suspend(struct device *dev)
{
struct tegra_sor *sor = dev_get_drvdata(dev);
int err;
return 0;
}
-static int tegra_sor_resume(struct device *dev)
+static int tegra_sor_runtime_resume(struct device *dev)
{
struct tegra_sor *sor = dev_get_drvdata(dev);
int err;
return 0;
}
-#endif
+
+static int tegra_sor_suspend(struct device *dev)
+{
+ struct tegra_sor *sor = dev_get_drvdata(dev);
+ int err;
+
+ if (sor->hdmi_supply) {
+ err = regulator_disable(sor->hdmi_supply);
+ if (err < 0)
+ return err;
+ }
+
+ return 0;
+}
+
+static int tegra_sor_resume(struct device *dev)
+{
+ struct tegra_sor *sor = dev_get_drvdata(dev);
+ int err;
+
+ if (sor->hdmi_supply) {
+ err = regulator_enable(sor->hdmi_supply);
+ if (err < 0)
+ return err;
+ }
+
+ return 0;
+}
static const struct dev_pm_ops tegra_sor_pm_ops = {
- SET_RUNTIME_PM_OPS(tegra_sor_suspend, tegra_sor_resume, NULL)
+ SET_RUNTIME_PM_OPS(tegra_sor_runtime_suspend, tegra_sor_runtime_resume,
+ NULL)
+ SET_SYSTEM_SLEEP_PM_OPS(tegra_sor_suspend, tegra_sor_resume)
};
struct platform_driver tegra_sor_driver = {
int err;
err = host1x_client_iommu_attach(client);
- if (err < 0) {
+ if (err < 0 && err != -ENODEV) {
dev_err(vic->dev, "failed to attach to domain: %d\n", err);
return err;
}
.supports_sid = true,
};
-static const struct of_device_id vic_match[] = {
+static const struct of_device_id tegra_vic_of_match[] = {
{ .compatible = "nvidia,tegra124-vic", .data = &vic_t124_config },
{ .compatible = "nvidia,tegra210-vic", .data = &vic_t210_config },
{ .compatible = "nvidia,tegra186-vic", .data = &vic_t186_config },
{ .compatible = "nvidia,tegra194-vic", .data = &vic_t194_config },
{ },
};
+MODULE_DEVICE_TABLE(of, tegra_vic_of_match);
static int vic_probe(struct platform_device *pdev)
{
struct platform_driver tegra_vic_driver = {
.driver = {
.name = "tegra-vic",
- .of_match_table = vic_match,
+ .of_match_table = tegra_vic_of_match,
.pm = &vic_pm_ops
},
.probe = vic_probe,
/*
* For a zoned target, the number of zones should be updated for the
* correct value to be exposed in sysfs queue/nr_zones. For a BIO based
- * target, this is all that is needed. For a request based target, the
- * queue zone bitmaps must also be updated.
- * Use blk_revalidate_disk_zones() to handle this.
+ * target, this is all that is needed.
*/
- if (blk_queue_is_zoned(q))
- blk_revalidate_disk_zones(t->md->disk);
+#ifdef CONFIG_BLK_DEV_ZONED
+ if (blk_queue_is_zoned(q)) {
+ WARN_ON_ONCE(queue_is_mq(q));
+ q->nr_zones = blkdev_nr_zones(t->md->disk);
+ }
+#endif
/* Allow reads to exceed readahead limits */
q->backing_dev_info->io_pages = limits->max_sectors >> (PAGE_SHIFT - 9);
dev->zone_nr_blocks = dmz_sect2blk(dev->zone_nr_sectors);
dev->zone_nr_blocks_shift = ilog2(dev->zone_nr_blocks);
- dev->nr_zones = blkdev_nr_zones(dev->bdev);
+ dev->nr_zones = blkdev_nr_zones(dev->bdev->bd_disk);
dmz->dev = dev;
goto err;
/* The drive satisfies the kernel restrictions: set it up */
- blk_queue_chunk_sectors(sdkp->disk->queue,
- logical_to_sectors(sdkp->device, zone_blocks));
blk_queue_flag_set(QUEUE_FLAG_ZONE_RESETALL, sdkp->disk->queue);
blk_queue_required_elevator_features(sdkp->disk->queue,
ELEVATOR_F_ZBD_SEQ_WRITE);
tristate
select QCOM_SCM
+config QCOM_OCMEM
+ tristate "Qualcomm On Chip Memory (OCMEM) driver"
+ depends on ARCH_QCOM
+ select QCOM_SCM
+ help
+ The On Chip Memory (OCMEM) allocator allows various clients to
+ allocate memory from OCMEM based on performance, latency and power
+ requirements. This is typically used by the GPU, camera/video, and
+ audio components on some Snapdragon SoCs.
+
config QCOM_PM
bool "Qualcomm Power Management"
depends on ARCH_QCOM && !ARM64
obj-$(CONFIG_QCOM_GLINK_SSR) += glink_ssr.o
obj-$(CONFIG_QCOM_GSBI) += qcom_gsbi.o
obj-$(CONFIG_QCOM_MDT_LOADER) += mdt_loader.o
+obj-$(CONFIG_QCOM_OCMEM) += ocmem.o
obj-$(CONFIG_QCOM_PM) += spm.o
obj-$(CONFIG_QCOM_QMI_HELPERS) += qmi_helpers.o
qmi_helpers-y += qmi_encdec.o qmi_interface.o
--- /dev/null
+// SPDX-License-Identifier: GPL-2.0-only
+/*
+ * The On Chip Memory (OCMEM) allocator allows various clients to allocate
+ * memory from OCMEM based on performance, latency and power requirements.
+ * This is typically used by the GPU, camera/video, and audio components on
+ * some Snapdragon SoCs.
+ *
+ * Copyright (C) 2019 Brian Masney <masneyb@onstation.org>
+ * Copyright (C) 2015 Red Hat. Author: Rob Clark <robdclark@gmail.com>
+ */
+
+#include <linux/bitfield.h>
+#include <linux/clk.h>
+#include <linux/io.h>
+#include <linux/kernel.h>
+#include <linux/module.h>
+#include <linux/of_device.h>
+#include <linux/platform_device.h>
+#include <linux/qcom_scm.h>
+#include <linux/sizes.h>
+#include <linux/slab.h>
+#include <linux/types.h>
+#include <soc/qcom/ocmem.h>
+
+enum region_mode {
+ WIDE_MODE = 0x0,
+ THIN_MODE,
+ MODE_DEFAULT = WIDE_MODE,
+};
+
+enum ocmem_macro_state {
+ PASSTHROUGH = 0,
+ PERI_ON = 1,
+ CORE_ON = 2,
+ CLK_OFF = 4,
+};
+
+struct ocmem_region {
+ bool interleaved;
+ enum region_mode mode;
+ unsigned int num_macros;
+ enum ocmem_macro_state macro_state[4];
+ unsigned long macro_size;
+ unsigned long region_size;
+};
+
+struct ocmem_config {
+ uint8_t num_regions;
+ unsigned long macro_size;
+};
+
+struct ocmem {
+ struct device *dev;
+ const struct ocmem_config *config;
+ struct resource *memory;
+ void __iomem *mmio;
+ unsigned int num_ports;
+ unsigned int num_macros;
+ bool interleaved;
+ struct ocmem_region *regions;
+ unsigned long active_allocations;
+};
+
+#define OCMEM_MIN_ALIGN SZ_64K
+#define OCMEM_MIN_ALLOC SZ_64K
+
+#define OCMEM_REG_HW_VERSION 0x00000000
+#define OCMEM_REG_HW_PROFILE 0x00000004
+
+#define OCMEM_REG_REGION_MODE_CTL 0x00001000
+#define OCMEM_REGION_MODE_CTL_REG0_THIN 0x00000001
+#define OCMEM_REGION_MODE_CTL_REG1_THIN 0x00000002
+#define OCMEM_REGION_MODE_CTL_REG2_THIN 0x00000004
+#define OCMEM_REGION_MODE_CTL_REG3_THIN 0x00000008
+
+#define OCMEM_REG_GFX_MPU_START 0x00001004
+#define OCMEM_REG_GFX_MPU_END 0x00001008
+
+#define OCMEM_HW_PROFILE_NUM_PORTS(val) FIELD_PREP(0x0000000f, (val))
+#define OCMEM_HW_PROFILE_NUM_MACROS(val) FIELD_PREP(0x00003f00, (val))
+
+#define OCMEM_HW_PROFILE_LAST_REGN_HALFSIZE 0x00010000
+#define OCMEM_HW_PROFILE_INTERLEAVING 0x00020000
+#define OCMEM_REG_GEN_STATUS 0x0000000c
+
+#define OCMEM_REG_PSGSC_STATUS 0x00000038
+#define OCMEM_REG_PSGSC_CTL(i0) (0x0000003c + 0x1*(i0))
+
+#define OCMEM_PSGSC_CTL_MACRO0_MODE(val) FIELD_PREP(0x00000007, (val))
+#define OCMEM_PSGSC_CTL_MACRO1_MODE(val) FIELD_PREP(0x00000070, (val))
+#define OCMEM_PSGSC_CTL_MACRO2_MODE(val) FIELD_PREP(0x00000700, (val))
+#define OCMEM_PSGSC_CTL_MACRO3_MODE(val) FIELD_PREP(0x00007000, (val))
+
+#define OCMEM_CLK_CORE_IDX 0
+static struct clk_bulk_data ocmem_clks[] = {
+ {
+ .id = "core",
+ },
+ {
+ .id = "iface",
+ },
+};
+
+static inline void ocmem_write(struct ocmem *ocmem, u32 reg, u32 data)
+{
+ writel(data, ocmem->mmio + reg);
+}
+
+static inline u32 ocmem_read(struct ocmem *ocmem, u32 reg)
+{
+ return readl(ocmem->mmio + reg);
+}
+
+static void update_ocmem(struct ocmem *ocmem)
+{
+ uint32_t region_mode_ctrl = 0x0;
+ int i;
+
+ if (!qcom_scm_ocmem_lock_available()) {
+ for (i = 0; i < ocmem->config->num_regions; i++) {
+ struct ocmem_region *region = &ocmem->regions[i];
+
+ if (region->mode == THIN_MODE)
+ region_mode_ctrl |= BIT(i);
+ }
+
+ dev_dbg(ocmem->dev, "ocmem_region_mode_control %x\n",
+ region_mode_ctrl);
+ ocmem_write(ocmem, OCMEM_REG_REGION_MODE_CTL, region_mode_ctrl);
+ }
+
+ for (i = 0; i < ocmem->config->num_regions; i++) {
+ struct ocmem_region *region = &ocmem->regions[i];
+ u32 data;
+
+ data = OCMEM_PSGSC_CTL_MACRO0_MODE(region->macro_state[0]) |
+ OCMEM_PSGSC_CTL_MACRO1_MODE(region->macro_state[1]) |
+ OCMEM_PSGSC_CTL_MACRO2_MODE(region->macro_state[2]) |
+ OCMEM_PSGSC_CTL_MACRO3_MODE(region->macro_state[3]);
+
+ ocmem_write(ocmem, OCMEM_REG_PSGSC_CTL(i), data);
+ }
+}
+
+static unsigned long phys_to_offset(struct ocmem *ocmem,
+ unsigned long addr)
+{
+ if (addr < ocmem->memory->start || addr >= ocmem->memory->end)
+ return 0;
+
+ return addr - ocmem->memory->start;
+}
+
+static unsigned long device_address(struct ocmem *ocmem,
+ enum ocmem_client client,
+ unsigned long addr)
+{
+ WARN_ON(client != OCMEM_GRAPHICS);
+
+ /* TODO: gpu uses phys_to_offset, but others do not.. */
+ return phys_to_offset(ocmem, addr);
+}
+
+static void update_range(struct ocmem *ocmem, struct ocmem_buf *buf,
+ enum ocmem_macro_state mstate, enum region_mode rmode)
+{
+ unsigned long offset = 0;
+ int i, j;
+
+ for (i = 0; i < ocmem->config->num_regions; i++) {
+ struct ocmem_region *region = &ocmem->regions[i];
+
+ if (buf->offset <= offset && offset < buf->offset + buf->len)
+ region->mode = rmode;
+
+ for (j = 0; j < region->num_macros; j++) {
+ if (buf->offset <= offset &&
+ offset < buf->offset + buf->len)
+ region->macro_state[j] = mstate;
+
+ offset += region->macro_size;
+ }
+ }
+
+ update_ocmem(ocmem);
+}
+
+struct ocmem *of_get_ocmem(struct device *dev)
+{
+ struct platform_device *pdev;
+ struct device_node *devnode;
+
+ devnode = of_parse_phandle(dev->of_node, "sram", 0);
+ if (!devnode || !devnode->parent) {
+ dev_err(dev, "Cannot look up sram phandle\n");
+ return ERR_PTR(-ENODEV);
+ }
+
+ pdev = of_find_device_by_node(devnode->parent);
+ if (!pdev) {
+ dev_err(dev, "Cannot find device node %s\n", devnode->name);
+ return ERR_PTR(-EPROBE_DEFER);
+ }
+
+ return platform_get_drvdata(pdev);
+}
+EXPORT_SYMBOL(of_get_ocmem);
+
+struct ocmem_buf *ocmem_allocate(struct ocmem *ocmem, enum ocmem_client client,
+ unsigned long size)
+{
+ struct ocmem_buf *buf;
+ int ret;
+
+ /* TODO: add support for other clients... */
+ if (WARN_ON(client != OCMEM_GRAPHICS))
+ return ERR_PTR(-ENODEV);
+
+ if (size < OCMEM_MIN_ALLOC || !IS_ALIGNED(size, OCMEM_MIN_ALIGN))
+ return ERR_PTR(-EINVAL);
+
+ if (test_and_set_bit_lock(BIT(client), &ocmem->active_allocations))
+ return ERR_PTR(-EBUSY);
+
+ buf = kzalloc(sizeof(*buf), GFP_KERNEL);
+ if (!buf) {
+ ret = -ENOMEM;
+ goto err_unlock;
+ }
+
+ buf->offset = 0;
+ buf->addr = device_address(ocmem, client, buf->offset);
+ buf->len = size;
+
+ update_range(ocmem, buf, CORE_ON, WIDE_MODE);
+
+ if (qcom_scm_ocmem_lock_available()) {
+ ret = qcom_scm_ocmem_lock(QCOM_SCM_OCMEM_GRAPHICS_ID,
+ buf->offset, buf->len, WIDE_MODE);
+ if (ret) {
+ dev_err(ocmem->dev, "could not lock: %d\n", ret);
+ ret = -EINVAL;
+ goto err_kfree;
+ }
+ } else {
+ ocmem_write(ocmem, OCMEM_REG_GFX_MPU_START, buf->offset);
+ ocmem_write(ocmem, OCMEM_REG_GFX_MPU_END,
+ buf->offset + buf->len);
+ }
+
+ dev_dbg(ocmem->dev, "using %ldK of OCMEM at 0x%08lx for client %d\n",
+ size / 1024, buf->addr, client);
+
+ return buf;
+
+err_kfree:
+ kfree(buf);
+err_unlock:
+ clear_bit_unlock(BIT(client), &ocmem->active_allocations);
+
+ return ERR_PTR(ret);
+}
+EXPORT_SYMBOL(ocmem_allocate);
+
+void ocmem_free(struct ocmem *ocmem, enum ocmem_client client,
+ struct ocmem_buf *buf)
+{
+ /* TODO: add support for other clients... */
+ if (WARN_ON(client != OCMEM_GRAPHICS))
+ return;
+
+ update_range(ocmem, buf, CLK_OFF, MODE_DEFAULT);
+
+ if (qcom_scm_ocmem_lock_available()) {
+ int ret;
+
+ ret = qcom_scm_ocmem_unlock(QCOM_SCM_OCMEM_GRAPHICS_ID,
+ buf->offset, buf->len);
+ if (ret)
+ dev_err(ocmem->dev, "could not unlock: %d\n", ret);
+ } else {
+ ocmem_write(ocmem, OCMEM_REG_GFX_MPU_START, 0x0);
+ ocmem_write(ocmem, OCMEM_REG_GFX_MPU_END, 0x0);
+ }
+
+ kfree(buf);
+
+ clear_bit_unlock(BIT(client), &ocmem->active_allocations);
+}
+EXPORT_SYMBOL(ocmem_free);
+
+static int ocmem_dev_probe(struct platform_device *pdev)
+{
+ struct device *dev = &pdev->dev;
+ unsigned long reg, region_size;
+ int i, j, ret, num_banks;
+ struct resource *res;
+ struct ocmem *ocmem;
+
+ if (!qcom_scm_is_available())
+ return -EPROBE_DEFER;
+
+ ocmem = devm_kzalloc(dev, sizeof(*ocmem), GFP_KERNEL);
+ if (!ocmem)
+ return -ENOMEM;
+
+ ocmem->dev = dev;
+ ocmem->config = device_get_match_data(dev);
+
+ ret = devm_clk_bulk_get(dev, ARRAY_SIZE(ocmem_clks), ocmem_clks);
+ if (ret) {
+ if (ret != -EPROBE_DEFER)
+ dev_err(dev, "Unable to get clocks\n");
+
+ return ret;
+ }
+
+ res = platform_get_resource_byname(pdev, IORESOURCE_MEM, "ctrl");
+ ocmem->mmio = devm_ioremap_resource(&pdev->dev, res);
+ if (IS_ERR(ocmem->mmio)) {
+ dev_err(&pdev->dev, "Failed to ioremap ocmem_ctrl resource\n");
+ return PTR_ERR(ocmem->mmio);
+ }
+
+ ocmem->memory = platform_get_resource_byname(pdev, IORESOURCE_MEM,
+ "mem");
+ if (!ocmem->memory) {
+ dev_err(dev, "Could not get mem region\n");
+ return -ENXIO;
+ }
+
+ /* The core clock is synchronous with graphics */
+ WARN_ON(clk_set_rate(ocmem_clks[OCMEM_CLK_CORE_IDX].clk, 1000) < 0);
+
+ ret = clk_bulk_prepare_enable(ARRAY_SIZE(ocmem_clks), ocmem_clks);
+ if (ret) {
+ dev_info(ocmem->dev, "Failed to enable clocks\n");
+ return ret;
+ }
+
+ if (qcom_scm_restore_sec_cfg_available()) {
+ dev_dbg(dev, "configuring scm\n");
+ ret = qcom_scm_restore_sec_cfg(QCOM_SCM_OCMEM_DEV_ID, 0);
+ if (ret) {
+ dev_err(dev, "Could not enable secure configuration\n");
+ goto err_clk_disable;
+ }
+ }
+
+ reg = ocmem_read(ocmem, OCMEM_REG_HW_PROFILE);
+ ocmem->num_ports = OCMEM_HW_PROFILE_NUM_PORTS(reg);
+ ocmem->num_macros = OCMEM_HW_PROFILE_NUM_MACROS(reg);
+ ocmem->interleaved = !!(reg & OCMEM_HW_PROFILE_INTERLEAVING);
+
+ num_banks = ocmem->num_ports / 2;
+ region_size = ocmem->config->macro_size * num_banks;
+
+ dev_info(dev, "%u ports, %u regions, %u macros, %sinterleaved\n",
+ ocmem->num_ports, ocmem->config->num_regions,
+ ocmem->num_macros, ocmem->interleaved ? "" : "not ");
+
+ ocmem->regions = devm_kcalloc(dev, ocmem->config->num_regions,
+ sizeof(struct ocmem_region), GFP_KERNEL);
+ if (!ocmem->regions) {
+ ret = -ENOMEM;
+ goto err_clk_disable;
+ }
+
+ for (i = 0; i < ocmem->config->num_regions; i++) {
+ struct ocmem_region *region = &ocmem->regions[i];
+
+ if (WARN_ON(num_banks > ARRAY_SIZE(region->macro_state))) {
+ ret = -EINVAL;
+ goto err_clk_disable;
+ }
+
+ region->mode = MODE_DEFAULT;
+ region->num_macros = num_banks;
+
+ if (i == (ocmem->config->num_regions - 1) &&
+ reg & OCMEM_HW_PROFILE_LAST_REGN_HALFSIZE) {
+ region->macro_size = ocmem->config->macro_size / 2;
+ region->region_size = region_size / 2;
+ } else {
+ region->macro_size = ocmem->config->macro_size;
+ region->region_size = region_size;
+ }
+
+ for (j = 0; j < ARRAY_SIZE(region->macro_state); j++)
+ region->macro_state[j] = CLK_OFF;
+ }
+
+ platform_set_drvdata(pdev, ocmem);
+
+ return 0;
+
+err_clk_disable:
+ clk_bulk_disable_unprepare(ARRAY_SIZE(ocmem_clks), ocmem_clks);
+ return ret;
+}
+
+static int ocmem_dev_remove(struct platform_device *pdev)
+{
+ clk_bulk_disable_unprepare(ARRAY_SIZE(ocmem_clks), ocmem_clks);
+
+ return 0;
+}
+
+static const struct ocmem_config ocmem_8974_config = {
+ .num_regions = 3,
+ .macro_size = SZ_128K,
+};
+
+static const struct of_device_id ocmem_of_match[] = {
+ { .compatible = "qcom,msm8974-ocmem", .data = &ocmem_8974_config },
+ { }
+};
+
+MODULE_DEVICE_TABLE(of, ocmem_of_match);
+
+static struct platform_driver ocmem_driver = {
+ .probe = ocmem_dev_probe,
+ .remove = ocmem_dev_remove,
+ .driver = {
+ .name = "ocmem",
+ .of_match_table = ocmem_of_match,
+ },
+};
+
+module_platform_driver(ocmem_driver);
+
+MODULE_DESCRIPTION("On Chip Memory (OCMEM) allocator for some Snapdragon SoCs");
+MODULE_LICENSE("GPL v2");
struct autofs_sb_info *sbi;
unsigned long last_used;
- atomic_t count;
+ int count;
kuid_t uid;
kgid_t gid;
}
} else {
struct autofs_info *ino = autofs_dentry_ino(p);
- unsigned int ino_count = atomic_read(&ino->count);
+ unsigned int ino_count = READ_ONCE(ino->count);
/* allow for dget above and top is already dgot */
if (p == top)
/* Not a forced expire? */
if (!(how & AUTOFS_EXP_FORCED)) {
/* ref-walk currently on this dentry? */
- ino_count = atomic_read(&ino->count) + 1;
+ ino_count = READ_ONCE(ino->count) + 1;
if (d_count(dentry) > ino_count)
return NULL;
}
/* Not a forced expire? */
if (!(how & AUTOFS_EXP_FORCED)) {
/* ref-walk currently on this dentry? */
- ino_count = atomic_read(&ino->count) + 1;
+ ino_count = READ_ONCE(ino->count) + 1;
if (d_count(dentry) > ino_count)
return NULL;
}
d_add(dentry, inode);
dget(dentry);
- atomic_inc(&ino->count);
+ ino->count++;
p_ino = autofs_dentry_ino(dentry->d_parent);
- if (p_ino && !IS_ROOT(dentry))
- atomic_inc(&p_ino->count);
+ p_ino->count++;
dir->i_mtime = current_time(dir);
if (sbi->flags & AUTOFS_SBI_CATATONIC)
return -EACCES;
- if (atomic_dec_and_test(&ino->count)) {
- p_ino = autofs_dentry_ino(dentry->d_parent);
- if (p_ino && !IS_ROOT(dentry))
- atomic_dec(&p_ino->count);
- }
+ ino->count--;
+ p_ino = autofs_dentry_ino(dentry->d_parent);
+ p_ino->count--;
dput(ino->dentry);
d_inode(dentry)->i_size = 0;
static void autofs_clear_leaf_automount_flags(struct dentry *dentry)
{
- struct list_head *d_child;
struct dentry *parent;
/* flags for dentrys in the root are handled elsewhere */
/* only consider parents below dentrys in the root */
if (IS_ROOT(parent->d_parent))
return;
- d_child = &dentry->d_child;
- /* Set parent managed if it's becoming empty */
- if (d_child->next == &parent->d_subdirs &&
- d_child->prev == &parent->d_subdirs)
+ if (autofs_dentry_ino(parent)->count == 2)
managed_dentry_set_managed(parent);
}
if (sbi->flags & AUTOFS_SBI_CATATONIC)
return -EACCES;
- spin_lock(&sbi->lookup_lock);
- if (!simple_empty(dentry)) {
- spin_unlock(&sbi->lookup_lock);
+ if (ino->count != 1)
return -ENOTEMPTY;
- }
+
+ spin_lock(&sbi->lookup_lock);
__autofs_add_expiring(dentry);
d_drop(dentry);
spin_unlock(&sbi->lookup_lock);
if (sbi->version < 5)
autofs_clear_leaf_automount_flags(dentry);
- if (atomic_dec_and_test(&ino->count)) {
- p_ino = autofs_dentry_ino(dentry->d_parent);
- if (p_ino && dentry->d_parent != dentry)
- atomic_dec(&p_ino->count);
- }
+ ino->count--;
+ p_ino = autofs_dentry_ino(dentry->d_parent);
+ p_ino->count--;
dput(ino->dentry);
d_inode(dentry)->i_size = 0;
clear_nlink(d_inode(dentry));
autofs_set_leaf_automount_flags(dentry);
dget(dentry);
- atomic_inc(&ino->count);
+ ino->count++;
p_ino = autofs_dentry_ino(dentry->d_parent);
- if (p_ino && !IS_ROOT(dentry))
- atomic_inc(&p_ino->count);
+ p_ino->count++;
inc_nlink(dir);
dir->i_mtime = current_time(dir);
ret = blk_add_partitions(disk, bdev);
if (ret == -EAGAIN)
goto rescan;
- } else {
+ } else if (invalidate) {
/*
* Tell userspace that the media / partition table may have
* changed.
#include <linux/ceph/ceph_debug.h>
+#include <linux/fs_context.h>
#include "super.h"
#include "cache.h"
fscache_unregister_netfs(&ceph_cache_netfs);
}
-int ceph_fscache_register_fs(struct ceph_fs_client* fsc)
+int ceph_fscache_register_fs(struct ceph_fs_client* fsc, struct fs_context *fc)
{
const struct ceph_fsid *fsid = &fsc->client->fsid;
const char *fscache_uniq = fsc->mount_options->fscache_uniq;
if (uniq_len && memcmp(ent->uniquifier, fscache_uniq, uniq_len))
continue;
- pr_err("fscache cookie already registered for fsid %pU\n", fsid);
- pr_err(" use fsc=%%s mount option to specify a uniquifier\n");
+ errorf(fc, "ceph: fscache cookie already registered for fsid %pU, use fsc=<uniquifier> option",
+ fsid);
err = -EBUSY;
goto out_unlock;
}
list_add_tail(&ent->list, &ceph_fscache_list);
} else {
kfree(ent);
- pr_err("unable to register fscache cookie for fsid %pU\n",
+ errorf(fc, "ceph: unable to register fscache cookie for fsid %pU",
fsid);
/* all other fs ignore this error */
}
int ceph_fscache_register(void);
void ceph_fscache_unregister(void);
-int ceph_fscache_register_fs(struct ceph_fs_client* fsc);
+int ceph_fscache_register_fs(struct ceph_fs_client* fsc, struct fs_context *fc);
void ceph_fscache_unregister_fs(struct ceph_fs_client* fsc);
void ceph_fscache_register_inode_cookie(struct inode *inode);
{
}
-static inline int ceph_fscache_register_fs(struct ceph_fs_client* fsc)
+static inline int ceph_fscache_register_fs(struct ceph_fs_client* fsc,
+ struct fs_context *fc)
{
return 0;
}
}
base = ceph_ino(d_inode(temp));
rcu_read_unlock();
- if (pos < 0 || read_seqretry(&rename_lock, seq)) {
- pr_err("build_path did not end path lookup where "
- "expected, pos is %d\n", pos);
- /* presumably this is only possible if racing with a
- rename of one of the parent directories (we can not
- lock the dentries above us to prevent this, but
- retrying should be harmless) */
+
+ if (read_seqretry(&rename_lock, seq))
+ goto retry;
+
+ if (pos < 0) {
+ /*
+ * A rename didn't occur, but somehow we didn't end up where
+ * we thought we would. Throw a warning and try again.
+ */
+ pr_warn("build_path did not end path lookup where "
+ "expected, pos is %d\n", pos);
goto retry;
}
head->op = cpu_to_le32(req->r_op);
head->caller_uid = cpu_to_le32(from_kuid(&init_user_ns, req->r_uid));
head->caller_gid = cpu_to_le32(from_kgid(&init_user_ns, req->r_gid));
+ head->ino = 0;
head->args = req->r_args;
ceph_encode_filepath(&p, end, ino1, path1);
int ceph_mdsmap_get_random_mds(struct ceph_mdsmap *m)
{
int n = 0;
- int i;
+ int i, j;
/* special case for one mds */
if (1 == m->m_num_mds && m->m_info[0].state > 0)
/* pick */
n = prandom_u32() % n;
- for (i = 0; n > 0; i++, n--)
- while (m->m_info[i].state <= 0)
- i++;
+ for (j = 0, i = 0; i < m->m_num_mds; i++) {
+ if (m->m_info[i].state > 0)
+ j++;
+ if (j > n)
+ break;
+ }
return i;
}
#include <linux/in6.h>
#include <linux/module.h>
#include <linux/mount.h>
-#include <linux/parser.h>
+#include <linux/fs_context.h>
+#include <linux/fs_parser.h>
#include <linux/sched.h>
#include <linux/seq_file.h>
#include <linux/slab.h>
Opt_readdir_max_entries,
Opt_readdir_max_bytes,
Opt_congestion_kb,
- Opt_last_int,
/* int args above */
Opt_snapdirname,
Opt_mds_namespace,
- Opt_fscache_uniq,
Opt_recover_session,
- Opt_last_string,
+ Opt_source,
/* string args above */
Opt_dirstat,
- Opt_nodirstat,
Opt_rbytes,
- Opt_norbytes,
Opt_asyncreaddir,
- Opt_noasyncreaddir,
Opt_dcache,
- Opt_nodcache,
Opt_ino32,
- Opt_noino32,
Opt_fscache,
- Opt_nofscache,
Opt_poolperm,
- Opt_nopoolperm,
Opt_require_active_mds,
- Opt_norequire_active_mds,
-#ifdef CONFIG_CEPH_FS_POSIX_ACL
Opt_acl,
-#endif
- Opt_noacl,
Opt_quotadf,
- Opt_noquotadf,
Opt_copyfrom,
- Opt_nocopyfrom,
};
-static match_table_t fsopt_tokens = {
- {Opt_wsize, "wsize=%d"},
- {Opt_rsize, "rsize=%d"},
- {Opt_rasize, "rasize=%d"},
- {Opt_caps_wanted_delay_min, "caps_wanted_delay_min=%d"},
- {Opt_caps_wanted_delay_max, "caps_wanted_delay_max=%d"},
- {Opt_caps_max, "caps_max=%d"},
- {Opt_readdir_max_entries, "readdir_max_entries=%d"},
- {Opt_readdir_max_bytes, "readdir_max_bytes=%d"},
- {Opt_congestion_kb, "write_congestion_kb=%d"},
- /* int args above */
- {Opt_snapdirname, "snapdirname=%s"},
- {Opt_mds_namespace, "mds_namespace=%s"},
- {Opt_recover_session, "recover_session=%s"},
- {Opt_fscache_uniq, "fsc=%s"},
- /* string args above */
- {Opt_dirstat, "dirstat"},
- {Opt_nodirstat, "nodirstat"},
- {Opt_rbytes, "rbytes"},
- {Opt_norbytes, "norbytes"},
- {Opt_asyncreaddir, "asyncreaddir"},
- {Opt_noasyncreaddir, "noasyncreaddir"},
- {Opt_dcache, "dcache"},
- {Opt_nodcache, "nodcache"},
- {Opt_ino32, "ino32"},
- {Opt_noino32, "noino32"},
- {Opt_fscache, "fsc"},
- {Opt_nofscache, "nofsc"},
- {Opt_poolperm, "poolperm"},
- {Opt_nopoolperm, "nopoolperm"},
- {Opt_require_active_mds, "require_active_mds"},
- {Opt_norequire_active_mds, "norequire_active_mds"},
-#ifdef CONFIG_CEPH_FS_POSIX_ACL
- {Opt_acl, "acl"},
-#endif
- {Opt_noacl, "noacl"},
- {Opt_quotadf, "quotadf"},
- {Opt_noquotadf, "noquotadf"},
- {Opt_copyfrom, "copyfrom"},
- {Opt_nocopyfrom, "nocopyfrom"},
- {-1, NULL}
+enum ceph_recover_session_mode {
+ ceph_recover_session_no,
+ ceph_recover_session_clean
+};
+
+static const struct fs_parameter_enum ceph_mount_param_enums[] = {
+ { Opt_recover_session, "no", ceph_recover_session_no },
+ { Opt_recover_session, "clean", ceph_recover_session_clean },
+ {}
+};
+
+static const struct fs_parameter_spec ceph_mount_param_specs[] = {
+ fsparam_flag_no ("acl", Opt_acl),
+ fsparam_flag_no ("asyncreaddir", Opt_asyncreaddir),
+ fsparam_u32 ("caps_max", Opt_caps_max),
+ fsparam_u32 ("caps_wanted_delay_max", Opt_caps_wanted_delay_max),
+ fsparam_u32 ("caps_wanted_delay_min", Opt_caps_wanted_delay_min),
+ fsparam_s32 ("write_congestion_kb", Opt_congestion_kb),
+ fsparam_flag_no ("copyfrom", Opt_copyfrom),
+ fsparam_flag_no ("dcache", Opt_dcache),
+ fsparam_flag_no ("dirstat", Opt_dirstat),
+ __fsparam (fs_param_is_string, "fsc", Opt_fscache,
+ fs_param_neg_with_no | fs_param_v_optional),
+ fsparam_flag_no ("ino32", Opt_ino32),
+ fsparam_string ("mds_namespace", Opt_mds_namespace),
+ fsparam_flag_no ("poolperm", Opt_poolperm),
+ fsparam_flag_no ("quotadf", Opt_quotadf),
+ fsparam_u32 ("rasize", Opt_rasize),
+ fsparam_flag_no ("rbytes", Opt_rbytes),
+ fsparam_s32 ("readdir_max_bytes", Opt_readdir_max_bytes),
+ fsparam_s32 ("readdir_max_entries", Opt_readdir_max_entries),
+ fsparam_enum ("recover_session", Opt_recover_session),
+ fsparam_flag_no ("require_active_mds", Opt_require_active_mds),
+ fsparam_u32 ("rsize", Opt_rsize),
+ fsparam_string ("snapdirname", Opt_snapdirname),
+ fsparam_string ("source", Opt_source),
+ fsparam_u32 ("wsize", Opt_wsize),
+ {}
+};
+
+static const struct fs_parameter_description ceph_mount_parameters = {
+ .name = "ceph",
+ .specs = ceph_mount_param_specs,
+ .enums = ceph_mount_param_enums,
};
-static int parse_fsopt_token(char *c, void *private)
+struct ceph_parse_opts_ctx {
+ struct ceph_options *copts;
+ struct ceph_mount_options *opts;
+};
+
+/*
+ * Parse the source parameter. Distinguish the server list from the path.
+ * Internally we do not include the leading '/' in the path.
+ *
+ * The source will look like:
+ * <server_spec>[,<server_spec>...]:[<path>]
+ * where
+ * <server_spec> is <ip>[:<port>]
+ * <path> is optional, but if present must begin with '/'
+ */
+static int ceph_parse_source(struct fs_parameter *param, struct fs_context *fc)
{
- struct ceph_mount_options *fsopt = private;
- substring_t argstr[MAX_OPT_ARGS];
- int token, intval, ret;
+ struct ceph_parse_opts_ctx *pctx = fc->fs_private;
+ struct ceph_mount_options *fsopt = pctx->opts;
+ char *dev_name = param->string, *dev_name_end;
+ int ret;
- token = match_token((char *)c, fsopt_tokens, argstr);
- if (token < 0)
- return -EINVAL;
+ dout("%s '%s'\n", __func__, dev_name);
+ if (!dev_name || !*dev_name)
+ return invalf(fc, "ceph: Empty source");
- if (token < Opt_last_int) {
- ret = match_int(&argstr[0], &intval);
- if (ret < 0) {
- pr_err("bad option arg (not int) at '%s'\n", c);
- return ret;
+ dev_name_end = strchr(dev_name, '/');
+ if (dev_name_end) {
+ if (strlen(dev_name_end) > 1) {
+ kfree(fsopt->server_path);
+ fsopt->server_path = kstrdup(dev_name_end, GFP_KERNEL);
+ if (!fsopt->server_path)
+ return -ENOMEM;
}
- dout("got int token %d val %d\n", token, intval);
- } else if (token > Opt_last_int && token < Opt_last_string) {
- dout("got string token %d val %s\n", token,
- argstr[0].from);
} else {
- dout("got token %d\n", token);
+ dev_name_end = dev_name + strlen(dev_name);
}
+ dev_name_end--; /* back up to ':' separator */
+ if (dev_name_end < dev_name || *dev_name_end != ':')
+ return invalf(fc, "ceph: No path or : separator in source");
+
+ dout("device name '%.*s'\n", (int)(dev_name_end - dev_name), dev_name);
+ if (fsopt->server_path)
+ dout("server path '%s'\n", fsopt->server_path);
+
+ ret = ceph_parse_mon_ips(param->string, dev_name_end - dev_name,
+ pctx->copts, fc);
+ if (ret)
+ return ret;
+
+ fc->source = param->string;
+ param->string = NULL;
+ return 0;
+}
+
+static int ceph_parse_mount_param(struct fs_context *fc,
+ struct fs_parameter *param)
+{
+ struct ceph_parse_opts_ctx *pctx = fc->fs_private;
+ struct ceph_mount_options *fsopt = pctx->opts;
+ struct fs_parse_result result;
+ unsigned int mode;
+ int token, ret;
+
+ ret = ceph_parse_param(param, pctx->copts, fc);
+ if (ret != -ENOPARAM)
+ return ret;
+
+ token = fs_parse(fc, &ceph_mount_parameters, param, &result);
+ dout("%s fs_parse '%s' token %d\n", __func__, param->key, token);
+ if (token < 0)
+ return token;
+
switch (token) {
case Opt_snapdirname:
kfree(fsopt->snapdir_name);
- fsopt->snapdir_name = kstrndup(argstr[0].from,
- argstr[0].to-argstr[0].from,
- GFP_KERNEL);
- if (!fsopt->snapdir_name)
- return -ENOMEM;
+ fsopt->snapdir_name = param->string;
+ param->string = NULL;
break;
case Opt_mds_namespace:
kfree(fsopt->mds_namespace);
- fsopt->mds_namespace = kstrndup(argstr[0].from,
- argstr[0].to-argstr[0].from,
- GFP_KERNEL);
- if (!fsopt->mds_namespace)
- return -ENOMEM;
+ fsopt->mds_namespace = param->string;
+ param->string = NULL;
break;
case Opt_recover_session:
- if (!strncmp(argstr[0].from, "no",
- argstr[0].to - argstr[0].from)) {
+ mode = result.uint_32;
+ if (mode == ceph_recover_session_no)
fsopt->flags &= ~CEPH_MOUNT_OPT_CLEANRECOVER;
- } else if (!strncmp(argstr[0].from, "clean",
- argstr[0].to - argstr[0].from)) {
+ else if (mode == ceph_recover_session_clean)
fsopt->flags |= CEPH_MOUNT_OPT_CLEANRECOVER;
- } else {
- return -EINVAL;
- }
- break;
- case Opt_fscache_uniq:
-#ifdef CONFIG_CEPH_FSCACHE
- kfree(fsopt->fscache_uniq);
- fsopt->fscache_uniq = kstrndup(argstr[0].from,
- argstr[0].to-argstr[0].from,
- GFP_KERNEL);
- if (!fsopt->fscache_uniq)
- return -ENOMEM;
- fsopt->flags |= CEPH_MOUNT_OPT_FSCACHE;
+ else
+ BUG();
break;
-#else
- pr_err("fscache support is disabled\n");
- return -EINVAL;
-#endif
+ case Opt_source:
+ if (fc->source)
+ return invalf(fc, "ceph: Multiple sources specified");
+ return ceph_parse_source(param, fc);
case Opt_wsize:
- if (intval < (int)PAGE_SIZE || intval > CEPH_MAX_WRITE_SIZE)
- return -EINVAL;
- fsopt->wsize = ALIGN(intval, PAGE_SIZE);
+ if (result.uint_32 < PAGE_SIZE ||
+ result.uint_32 > CEPH_MAX_WRITE_SIZE)
+ goto out_of_range;
+ fsopt->wsize = ALIGN(result.uint_32, PAGE_SIZE);
break;
case Opt_rsize:
- if (intval < (int)PAGE_SIZE || intval > CEPH_MAX_READ_SIZE)
- return -EINVAL;
- fsopt->rsize = ALIGN(intval, PAGE_SIZE);
+ if (result.uint_32 < PAGE_SIZE ||
+ result.uint_32 > CEPH_MAX_READ_SIZE)
+ goto out_of_range;
+ fsopt->rsize = ALIGN(result.uint_32, PAGE_SIZE);
break;
case Opt_rasize:
- if (intval < 0)
- return -EINVAL;
- fsopt->rasize = ALIGN(intval, PAGE_SIZE);
+ fsopt->rasize = ALIGN(result.uint_32, PAGE_SIZE);
break;
case Opt_caps_wanted_delay_min:
- if (intval < 1)
- return -EINVAL;
- fsopt->caps_wanted_delay_min = intval;
+ if (result.uint_32 < 1)
+ goto out_of_range;
+ fsopt->caps_wanted_delay_min = result.uint_32;
break;
case Opt_caps_wanted_delay_max:
- if (intval < 1)
- return -EINVAL;
- fsopt->caps_wanted_delay_max = intval;
+ if (result.uint_32 < 1)
+ goto out_of_range;
+ fsopt->caps_wanted_delay_max = result.uint_32;
break;
case Opt_caps_max:
- if (intval < 0)
- return -EINVAL;
- fsopt->caps_max = intval;
+ fsopt->caps_max = result.uint_32;
break;
case Opt_readdir_max_entries:
- if (intval < 1)
- return -EINVAL;
- fsopt->max_readdir = intval;
+ if (result.uint_32 < 1)
+ goto out_of_range;
+ fsopt->max_readdir = result.uint_32;
break;
case Opt_readdir_max_bytes:
- if (intval < (int)PAGE_SIZE && intval != 0)
- return -EINVAL;
- fsopt->max_readdir_bytes = intval;
+ if (result.uint_32 < PAGE_SIZE && result.uint_32 != 0)
+ goto out_of_range;
+ fsopt->max_readdir_bytes = result.uint_32;
break;
case Opt_congestion_kb:
- if (intval < 1024) /* at least 1M */
- return -EINVAL;
- fsopt->congestion_kb = intval;
+ if (result.uint_32 < 1024) /* at least 1M */
+ goto out_of_range;
+ fsopt->congestion_kb = result.uint_32;
break;
case Opt_dirstat:
- fsopt->flags |= CEPH_MOUNT_OPT_DIRSTAT;
- break;
- case Opt_nodirstat:
- fsopt->flags &= ~CEPH_MOUNT_OPT_DIRSTAT;
+ if (!result.negated)
+ fsopt->flags |= CEPH_MOUNT_OPT_DIRSTAT;
+ else
+ fsopt->flags &= ~CEPH_MOUNT_OPT_DIRSTAT;
break;
case Opt_rbytes:
- fsopt->flags |= CEPH_MOUNT_OPT_RBYTES;
- break;
- case Opt_norbytes:
- fsopt->flags &= ~CEPH_MOUNT_OPT_RBYTES;
+ if (!result.negated)
+ fsopt->flags |= CEPH_MOUNT_OPT_RBYTES;
+ else
+ fsopt->flags &= ~CEPH_MOUNT_OPT_RBYTES;
break;
case Opt_asyncreaddir:
- fsopt->flags &= ~CEPH_MOUNT_OPT_NOASYNCREADDIR;
- break;
- case Opt_noasyncreaddir:
- fsopt->flags |= CEPH_MOUNT_OPT_NOASYNCREADDIR;
+ if (!result.negated)
+ fsopt->flags &= ~CEPH_MOUNT_OPT_NOASYNCREADDIR;
+ else
+ fsopt->flags |= CEPH_MOUNT_OPT_NOASYNCREADDIR;
break;
case Opt_dcache:
- fsopt->flags |= CEPH_MOUNT_OPT_DCACHE;
- break;
- case Opt_nodcache:
- fsopt->flags &= ~CEPH_MOUNT_OPT_DCACHE;
+ if (!result.negated)
+ fsopt->flags |= CEPH_MOUNT_OPT_DCACHE;
+ else
+ fsopt->flags &= ~CEPH_MOUNT_OPT_DCACHE;
break;
case Opt_ino32:
- fsopt->flags |= CEPH_MOUNT_OPT_INO32;
- break;
- case Opt_noino32:
- fsopt->flags &= ~CEPH_MOUNT_OPT_INO32;
+ if (!result.negated)
+ fsopt->flags |= CEPH_MOUNT_OPT_INO32;
+ else
+ fsopt->flags &= ~CEPH_MOUNT_OPT_INO32;
break;
+
case Opt_fscache:
#ifdef CONFIG_CEPH_FSCACHE
- fsopt->flags |= CEPH_MOUNT_OPT_FSCACHE;
kfree(fsopt->fscache_uniq);
fsopt->fscache_uniq = NULL;
+ if (result.negated) {
+ fsopt->flags &= ~CEPH_MOUNT_OPT_FSCACHE;
+ } else {
+ fsopt->flags |= CEPH_MOUNT_OPT_FSCACHE;
+ fsopt->fscache_uniq = param->string;
+ param->string = NULL;
+ }
break;
#else
- pr_err("fscache support is disabled\n");
- return -EINVAL;
+ return invalf(fc, "ceph: fscache support is disabled");
#endif
- case Opt_nofscache:
- fsopt->flags &= ~CEPH_MOUNT_OPT_FSCACHE;
- kfree(fsopt->fscache_uniq);
- fsopt->fscache_uniq = NULL;
- break;
case Opt_poolperm:
- fsopt->flags &= ~CEPH_MOUNT_OPT_NOPOOLPERM;
- break;
- case Opt_nopoolperm:
- fsopt->flags |= CEPH_MOUNT_OPT_NOPOOLPERM;
+ if (!result.negated)
+ fsopt->flags &= ~CEPH_MOUNT_OPT_NOPOOLPERM;
+ else
+ fsopt->flags |= CEPH_MOUNT_OPT_NOPOOLPERM;
break;
case Opt_require_active_mds:
- fsopt->flags &= ~CEPH_MOUNT_OPT_MOUNTWAIT;
- break;
- case Opt_norequire_active_mds:
- fsopt->flags |= CEPH_MOUNT_OPT_MOUNTWAIT;
+ if (!result.negated)
+ fsopt->flags &= ~CEPH_MOUNT_OPT_MOUNTWAIT;
+ else
+ fsopt->flags |= CEPH_MOUNT_OPT_MOUNTWAIT;
break;
case Opt_quotadf:
- fsopt->flags &= ~CEPH_MOUNT_OPT_NOQUOTADF;
- break;
- case Opt_noquotadf:
- fsopt->flags |= CEPH_MOUNT_OPT_NOQUOTADF;
+ if (!result.negated)
+ fsopt->flags &= ~CEPH_MOUNT_OPT_NOQUOTADF;
+ else
+ fsopt->flags |= CEPH_MOUNT_OPT_NOQUOTADF;
break;
case Opt_copyfrom:
- fsopt->flags &= ~CEPH_MOUNT_OPT_NOCOPYFROM;
- break;
- case Opt_nocopyfrom:
- fsopt->flags |= CEPH_MOUNT_OPT_NOCOPYFROM;
+ if (!result.negated)
+ fsopt->flags &= ~CEPH_MOUNT_OPT_NOCOPYFROM;
+ else
+ fsopt->flags |= CEPH_MOUNT_OPT_NOCOPYFROM;
break;
-#ifdef CONFIG_CEPH_FS_POSIX_ACL
case Opt_acl:
- fsopt->sb_flags |= SB_POSIXACL;
- break;
+ if (!result.negated) {
+#ifdef CONFIG_CEPH_FS_POSIX_ACL
+ fc->sb_flags |= SB_POSIXACL;
+#else
+ return invalf(fc, "ceph: POSIX ACL support is disabled");
#endif
- case Opt_noacl:
- fsopt->sb_flags &= ~SB_POSIXACL;
+ } else {
+ fc->sb_flags &= ~SB_POSIXACL;
+ }
break;
default:
- BUG_ON(token);
+ BUG();
}
return 0;
+
+out_of_range:
+ return invalf(fc, "ceph: %s out of range", param->key);
}
static void destroy_mount_options(struct ceph_mount_options *args)
{
dout("destroy_mount_options %p\n", args);
+ if (!args)
+ return;
+
kfree(args->snapdir_name);
kfree(args->mds_namespace);
kfree(args->server_path);
return ceph_compare_options(new_opt, fsc->client);
}
-static int parse_mount_options(struct ceph_mount_options **pfsopt,
- struct ceph_options **popt,
- int flags, char *options,
- const char *dev_name)
-{
- struct ceph_mount_options *fsopt;
- const char *dev_name_end;
- int err;
-
- if (!dev_name || !*dev_name)
- return -EINVAL;
-
- fsopt = kzalloc(sizeof(*fsopt), GFP_KERNEL);
- if (!fsopt)
- return -ENOMEM;
-
- dout("parse_mount_options %p, dev_name '%s'\n", fsopt, dev_name);
-
- fsopt->sb_flags = flags;
- fsopt->flags = CEPH_MOUNT_OPT_DEFAULT;
-
- fsopt->wsize = CEPH_MAX_WRITE_SIZE;
- fsopt->rsize = CEPH_MAX_READ_SIZE;
- fsopt->rasize = CEPH_RASIZE_DEFAULT;
- fsopt->snapdir_name = kstrdup(CEPH_SNAPDIRNAME_DEFAULT, GFP_KERNEL);
- if (!fsopt->snapdir_name) {
- err = -ENOMEM;
- goto out;
- }
-
- fsopt->caps_wanted_delay_min = CEPH_CAPS_WANTED_DELAY_MIN_DEFAULT;
- fsopt->caps_wanted_delay_max = CEPH_CAPS_WANTED_DELAY_MAX_DEFAULT;
- fsopt->max_readdir = CEPH_MAX_READDIR_DEFAULT;
- fsopt->max_readdir_bytes = CEPH_MAX_READDIR_BYTES_DEFAULT;
- fsopt->congestion_kb = default_congestion_kb();
-
- /*
- * Distinguish the server list from the path in "dev_name".
- * Internally we do not include the leading '/' in the path.
- *
- * "dev_name" will look like:
- * <server_spec>[,<server_spec>...]:[<path>]
- * where
- * <server_spec> is <ip>[:<port>]
- * <path> is optional, but if present must begin with '/'
- */
- dev_name_end = strchr(dev_name, '/');
- if (dev_name_end) {
- if (strlen(dev_name_end) > 1) {
- fsopt->server_path = kstrdup(dev_name_end, GFP_KERNEL);
- if (!fsopt->server_path) {
- err = -ENOMEM;
- goto out;
- }
- }
- } else {
- dev_name_end = dev_name + strlen(dev_name);
- }
- err = -EINVAL;
- dev_name_end--; /* back up to ':' separator */
- if (dev_name_end < dev_name || *dev_name_end != ':') {
- pr_err("device name is missing path (no : separator in %s)\n",
- dev_name);
- goto out;
- }
- dout("device name '%.*s'\n", (int)(dev_name_end - dev_name), dev_name);
- if (fsopt->server_path)
- dout("server path '%s'\n", fsopt->server_path);
-
- *popt = ceph_parse_options(options, dev_name, dev_name_end,
- parse_fsopt_token, (void *)fsopt);
- if (IS_ERR(*popt)) {
- err = PTR_ERR(*popt);
- goto out;
- }
-
- /* success */
- *pfsopt = fsopt;
- return 0;
-
-out:
- destroy_mount_options(fsopt);
- return err;
-}
-
/**
* ceph_show_options - Show mount options in /proc/mounts
* @m: seq_file to write to
seq_puts(m, ",noquotadf");
#ifdef CONFIG_CEPH_FS_POSIX_ACL
- if (fsopt->sb_flags & SB_POSIXACL)
+ if (root->d_sb->s_flags & SB_POSIXACL)
seq_puts(m, ",acl");
else
seq_puts(m, ",noacl");
fsc->filp_gen++; // invalidate open files
}
-static int ceph_remount(struct super_block *sb, int *flags, char *data)
-{
- sync_filesystem(sb);
- return 0;
-}
-
static const struct super_operations ceph_super_ops = {
.alloc_inode = ceph_alloc_inode,
.free_inode = ceph_free_inode,
.evict_inode = ceph_evict_inode,
.sync_fs = ceph_sync_fs,
.put_super = ceph_put_super,
- .remount_fs = ceph_remount,
.show_options = ceph_show_options,
.statfs = ceph_statfs,
.umount_begin = ceph_umount_begin,
/*
* mount: join the ceph cluster, and open root directory.
*/
-static struct dentry *ceph_real_mount(struct ceph_fs_client *fsc)
+static struct dentry *ceph_real_mount(struct ceph_fs_client *fsc,
+ struct fs_context *fc)
{
int err;
unsigned long started = jiffies; /* note the start time */
/* setup fscache */
if (fsc->mount_options->flags & CEPH_MOUNT_OPT_FSCACHE) {
- err = ceph_fscache_register_fs(fsc);
+ err = ceph_fscache_register_fs(fsc, fc);
if (err < 0)
goto out;
}
return ERR_PTR(err);
}
-static int ceph_set_super(struct super_block *s, void *data)
+static int ceph_set_super(struct super_block *s, struct fs_context *fc)
{
- struct ceph_fs_client *fsc = data;
+ struct ceph_fs_client *fsc = s->s_fs_info;
int ret;
- dout("set_super %p data %p\n", s, data);
+ dout("set_super %p\n", s);
- s->s_flags = fsc->mount_options->sb_flags;
s->s_maxbytes = MAX_LFS_FILESIZE;
s->s_xattr = ceph_xattr_handlers;
- s->s_fs_info = fsc;
fsc->sb = s;
fsc->max_file_size = 1ULL << 40; /* temp value until we get mdsmap */
s->s_time_min = 0;
s->s_time_max = U32_MAX;
- ret = set_anon_super(s, NULL); /* what is that second arg for? */
+ ret = set_anon_super_fc(s, fc);
if (ret != 0)
- goto fail;
-
- return ret;
-
-fail:
- s->s_fs_info = NULL;
- fsc->sb = NULL;
+ fsc->sb = NULL;
return ret;
}
/*
* share superblock if same fs AND options
*/
-static int ceph_compare_super(struct super_block *sb, void *data)
+static int ceph_compare_super(struct super_block *sb, struct fs_context *fc)
{
- struct ceph_fs_client *new = data;
+ struct ceph_fs_client *new = fc->s_fs_info;
struct ceph_mount_options *fsopt = new->mount_options;
struct ceph_options *opt = new->client->options;
struct ceph_fs_client *other = ceph_sb_to_client(sb);
dout("fsid doesn't match\n");
return 0;
}
- if (fsopt->sb_flags != other->mount_options->sb_flags) {
+ if (fc->sb_flags != (sb->s_flags & ~SB_BORN)) {
dout("flags differ\n");
return 0;
}
return 0;
}
-static struct dentry *ceph_mount(struct file_system_type *fs_type,
- int flags, const char *dev_name, void *data)
+static int ceph_get_tree(struct fs_context *fc)
{
+ struct ceph_parse_opts_ctx *pctx = fc->fs_private;
struct super_block *sb;
struct ceph_fs_client *fsc;
struct dentry *res;
+ int (*compare_super)(struct super_block *, struct fs_context *) =
+ ceph_compare_super;
int err;
- int (*compare_super)(struct super_block *, void *) = ceph_compare_super;
- struct ceph_mount_options *fsopt = NULL;
- struct ceph_options *opt = NULL;
- dout("ceph_mount\n");
+ dout("ceph_get_tree\n");
+
+ if (!fc->source)
+ return invalf(fc, "ceph: No source");
#ifdef CONFIG_CEPH_FS_POSIX_ACL
- flags |= SB_POSIXACL;
+ fc->sb_flags |= SB_POSIXACL;
#endif
- err = parse_mount_options(&fsopt, &opt, flags, data, dev_name);
- if (err < 0) {
- res = ERR_PTR(err);
- goto out_final;
- }
/* create client (which we may/may not use) */
- fsc = create_fs_client(fsopt, opt);
+ fsc = create_fs_client(pctx->opts, pctx->copts);
+ pctx->opts = NULL;
+ pctx->copts = NULL;
if (IS_ERR(fsc)) {
- res = ERR_CAST(fsc);
+ err = PTR_ERR(fsc);
goto out_final;
}
err = ceph_mdsc_init(fsc);
- if (err < 0) {
- res = ERR_PTR(err);
+ if (err < 0)
goto out;
- }
if (ceph_test_opt(fsc->client, NOSHARE))
compare_super = NULL;
- sb = sget(fs_type, compare_super, ceph_set_super, flags, fsc);
+
+ fc->s_fs_info = fsc;
+ sb = sget_fc(fc, compare_super, ceph_set_super);
+ fc->s_fs_info = NULL;
if (IS_ERR(sb)) {
- res = ERR_CAST(sb);
+ err = PTR_ERR(sb);
goto out;
}
} else {
dout("get_sb using new client %p\n", fsc);
err = ceph_setup_bdi(sb, fsc);
- if (err < 0) {
- res = ERR_PTR(err);
+ if (err < 0)
goto out_splat;
- }
}
- res = ceph_real_mount(fsc);
- if (IS_ERR(res))
+ res = ceph_real_mount(fsc, fc);
+ if (IS_ERR(res)) {
+ err = PTR_ERR(res);
goto out_splat;
+ }
dout("root %p inode %p ino %llx.%llx\n", res,
d_inode(res), ceph_vinop(d_inode(res)));
- return res;
+ fc->root = fsc->sb->s_root;
+ return 0;
out_splat:
ceph_mdsc_close_sessions(fsc->mdsc);
out:
destroy_fs_client(fsc);
out_final:
- dout("ceph_mount fail %ld\n", PTR_ERR(res));
- return res;
+ dout("ceph_get_tree fail %d\n", err);
+ return err;
+}
+
+static void ceph_free_fc(struct fs_context *fc)
+{
+ struct ceph_parse_opts_ctx *pctx = fc->fs_private;
+
+ if (pctx) {
+ destroy_mount_options(pctx->opts);
+ ceph_destroy_options(pctx->copts);
+ kfree(pctx);
+ }
+}
+
+static int ceph_reconfigure_fc(struct fs_context *fc)
+{
+ sync_filesystem(fc->root->d_sb);
+ return 0;
+}
+
+static const struct fs_context_operations ceph_context_ops = {
+ .free = ceph_free_fc,
+ .parse_param = ceph_parse_mount_param,
+ .get_tree = ceph_get_tree,
+ .reconfigure = ceph_reconfigure_fc,
+};
+
+/*
+ * Set up the filesystem mount context.
+ */
+static int ceph_init_fs_context(struct fs_context *fc)
+{
+ struct ceph_parse_opts_ctx *pctx;
+ struct ceph_mount_options *fsopt;
+
+ pctx = kzalloc(sizeof(*pctx), GFP_KERNEL);
+ if (!pctx)
+ return -ENOMEM;
+
+ pctx->copts = ceph_alloc_options();
+ if (!pctx->copts)
+ goto nomem;
+
+ pctx->opts = kzalloc(sizeof(*pctx->opts), GFP_KERNEL);
+ if (!pctx->opts)
+ goto nomem;
+
+ fsopt = pctx->opts;
+ fsopt->flags = CEPH_MOUNT_OPT_DEFAULT;
+
+ fsopt->wsize = CEPH_MAX_WRITE_SIZE;
+ fsopt->rsize = CEPH_MAX_READ_SIZE;
+ fsopt->rasize = CEPH_RASIZE_DEFAULT;
+ fsopt->snapdir_name = kstrdup(CEPH_SNAPDIRNAME_DEFAULT, GFP_KERNEL);
+ if (!fsopt->snapdir_name)
+ goto nomem;
+
+ fsopt->caps_wanted_delay_min = CEPH_CAPS_WANTED_DELAY_MIN_DEFAULT;
+ fsopt->caps_wanted_delay_max = CEPH_CAPS_WANTED_DELAY_MAX_DEFAULT;
+ fsopt->max_readdir = CEPH_MAX_READDIR_DEFAULT;
+ fsopt->max_readdir_bytes = CEPH_MAX_READDIR_BYTES_DEFAULT;
+ fsopt->congestion_kb = default_congestion_kb();
+
+ fc->fs_private = pctx;
+ fc->ops = &ceph_context_ops;
+ return 0;
+
+nomem:
+ destroy_mount_options(pctx->opts);
+ ceph_destroy_options(pctx->copts);
+ kfree(pctx);
+ return -ENOMEM;
}
static void ceph_kill_sb(struct super_block *s)
static struct file_system_type ceph_fs_type = {
.owner = THIS_MODULE,
.name = "ceph",
- .mount = ceph_mount,
+ .init_fs_context = ceph_init_fs_context,
.kill_sb = ceph_kill_sb,
.fs_flags = FS_RENAME_DOES_D_MOVE,
};
struct ceph_mount_options {
int flags;
- int sb_flags;
int wsize; /* max write size */
int rsize; /* max read size */
struct inode vfs_inode; /* at end */
};
-static inline struct ceph_inode_info *ceph_inode(struct inode *inode)
+static inline struct ceph_inode_info *
+ceph_inode(const struct inode *inode)
{
return container_of(inode, struct ceph_inode_info, vfs_inode);
}
-static inline struct ceph_fs_client *ceph_inode_to_client(struct inode *inode)
+static inline struct ceph_fs_client *
+ceph_inode_to_client(const struct inode *inode)
{
return (struct ceph_fs_client *)inode->i_sb->s_fs_info;
}
-static inline struct ceph_fs_client *ceph_sb_to_client(struct super_block *sb)
+static inline struct ceph_fs_client *
+ceph_sb_to_client(const struct super_block *sb)
{
return (struct ceph_fs_client *)sb->s_fs_info;
}
-static inline struct ceph_vino ceph_vino(struct inode *inode)
+static inline struct ceph_vino
+ceph_vino(const struct inode *inode)
{
return ceph_inode(inode)->i_vino;
}
struct inode *dir = d_inode(dentry);
struct dentry *child;
- if (!dir) {
- dput(dentry);
- dentry = ERR_PTR(-ENOENT);
- break;
- }
if (!S_ISDIR(dir->i_mode)) {
dput(dentry);
dentry = ERR_PTR(-ENOTDIR);
while (*s && *s != sep)
s++;
- child = lookup_one_len_unlocked(p, dentry, s - p);
+ child = lookup_positive_unlocked(p, dentry, s - p);
dput(dentry);
dentry = child;
} while (!IS_ERR(dentry));
flags = READ_ONCE(dentry->d_flags);
flags &= ~(DCACHE_ENTRY_TYPE | DCACHE_FALLTHRU);
flags |= type_flags;
- WRITE_ONCE(dentry->d_flags, flags);
+ smp_store_release(&dentry->d_flags, flags);
}
static inline void __d_clear_type_and_inode(struct dentry *dentry)
{
int gotref;
struct dentry *ret;
+ unsigned seq;
/*
* Do optimistic parent lookup without any
* locking.
*/
rcu_read_lock();
+ seq = raw_seqcount_begin(&dentry->d_seq);
ret = READ_ONCE(dentry->d_parent);
gotref = lockref_get_not_zero(&ret->d_lockref);
rcu_read_unlock();
if (likely(gotref)) {
- if (likely(ret == READ_ONCE(dentry->d_parent)))
+ if (!read_seqcount_retry(&dentry->d_seq, seq))
return ret;
dput(ret);
}
if (!parent)
parent = debugfs_mount->mnt_root;
- dentry = lookup_one_len_unlocked(name, parent, strlen(name));
+ dentry = lookup_positive_unlocked(name, parent, strlen(name));
if (IS_ERR(dentry))
return NULL;
- if (!d_really_is_positive(dentry)) {
- dput(dentry);
- return NULL;
- }
return dentry;
}
EXPORT_SYMBOL_GPL(debugfs_lookup);
select VIRTIO
help
The Virtio Filesystem allows guests to mount file systems from the
- host.
+ host.
If you want to share files between guests or with the host, answer Y
- or M.
+ or M.
nbuf = 0;
rem = 0;
- for (idx = tail; idx < head && rem < len; idx++)
+ for (idx = tail; idx != head && rem < len; idx++)
rem += pipe->bufs[idx & mask].len;
ret = -EINVAL;
kfree(forget);
if (ret == -ENOMEM)
goto out;
- if (ret || (outarg.attr.mode ^ inode->i_mode) & S_IFMT)
+ if (ret || fuse_invalid_attr(&outarg.attr) ||
+ (outarg.attr.mode ^ inode->i_mode) & S_IFMT)
goto invalid;
forget_all_cached_acls(inode);
S_ISBLK(m) || S_ISFIFO(m) || S_ISSOCK(m);
}
+bool fuse_invalid_attr(struct fuse_attr *attr)
+{
+ return !fuse_valid_type(attr->mode) ||
+ attr->size > LLONG_MAX;
+}
+
int fuse_lookup_name(struct super_block *sb, u64 nodeid, const struct qstr *name,
struct fuse_entry_out *outarg, struct inode **inode)
{
err = -EIO;
if (!outarg->nodeid)
goto out_put_forget;
- if (!fuse_valid_type(outarg->attr.mode))
+ if (fuse_invalid_attr(&outarg->attr))
goto out_put_forget;
*inode = fuse_iget(sb, outarg->nodeid, outarg->generation,
goto out_free_ff;
err = -EIO;
- if (!S_ISREG(outentry.attr.mode) || invalid_nodeid(outentry.nodeid))
+ if (!S_ISREG(outentry.attr.mode) || invalid_nodeid(outentry.nodeid) ||
+ fuse_invalid_attr(&outentry.attr))
goto out_free_ff;
ff->fh = outopen.fh;
goto out_put_forget_req;
err = -EIO;
- if (invalid_nodeid(outarg.nodeid))
+ if (invalid_nodeid(outarg.nodeid) || fuse_invalid_attr(&outarg.attr))
goto out_put_forget_req;
if ((outarg.attr.mode ^ mode) & S_IFMT)
spin_lock(&fi->lock);
fi->attr_version = atomic64_inc_return(&fc->attr_version);
- inc_nlink(inode);
+ if (likely(inode->i_nlink < UINT_MAX))
+ inc_nlink(inode);
spin_unlock(&fi->lock);
fuse_invalidate_attr(inode);
fuse_update_ctime(inode);
args.out_args[0].value = &outarg;
err = fuse_simple_request(fc, &args);
if (!err) {
- if ((inode->i_mode ^ outarg.attr.mode) & S_IFMT) {
+ if (fuse_invalid_attr(&outarg.attr) ||
+ (inode->i_mode ^ outarg.attr.mode) & S_IFMT) {
make_bad_inode(inode);
err = -EIO;
} else {
goto error;
}
- if ((inode->i_mode ^ outarg.attr.mode) & S_IFMT) {
+ if (fuse_invalid_attr(&outarg.attr) ||
+ (inode->i_mode ^ outarg.attr.mode) & S_IFMT) {
make_bad_inode(inode);
err = -EIO;
goto error;
ia->ap.args.end = fuse_aio_complete_req;
err = fuse_simple_background(fc, &ia->ap.args, GFP_KERNEL);
+ if (err)
+ fuse_aio_complete_req(fc, &ia->ap.args, err);
- return err ?: num_bytes;
+ return num_bytes;
}
static ssize_t fuse_send_read(struct fuse_io_args *ia, loff_t pos, size_t count,
ia->write.in.flags = fuse_write_flags(iocb);
err = fuse_simple_request(fc, &ap->args);
+ if (!err && ia->write.out.size > count)
+ err = -EIO;
offset = ap->descs[0].offset;
count = ia->write.out.size;
*/
int fuse_valid_type(int m);
+bool fuse_invalid_attr(struct fuse_attr *attr);
+
/**
* Is current process allowed to perform filesystem operation?
*/
if (invalid_nodeid(o->nodeid))
return -EIO;
- if (!fuse_valid_type(o->attr.mode))
+ if (fuse_invalid_attr(&o->attr))
return -EIO;
fc = get_fuse_conn(dir);
struct fuse_dev *fud;
bool connected;
long in_flight;
+ struct completion in_flight_zero; /* No inflight requests */
char name[24];
} ____cacheline_aligned_in_smp;
unsigned int num_request_queues; /* number of request queues */
};
-struct virtio_fs_forget {
+struct virtio_fs_forget_req {
struct fuse_in_header ih;
struct fuse_forget_in arg;
+};
+
+struct virtio_fs_forget {
/* This request can be temporarily queued on virt queue */
struct list_head list;
+ struct virtio_fs_forget_req req;
};
static int virtio_fs_enqueue_req(struct virtio_fs_vq *fsvq,
{
WARN_ON(fsvq->in_flight <= 0);
fsvq->in_flight--;
+ if (!fsvq->in_flight)
+ complete(&fsvq->in_flight_zero);
}
static void release_virtio_fs_obj(struct kref *ref)
WARN_ON(fsvq->in_flight < 0);
/* Wait for in flight requests to finish.*/
- while (1) {
- spin_lock(&fsvq->lock);
- if (!fsvq->in_flight) {
- spin_unlock(&fsvq->lock);
- break;
- }
+ spin_lock(&fsvq->lock);
+ if (fsvq->in_flight) {
+ /* We are holding virtio_fs_mutex. There should not be any
+ * waiters waiting for completion.
+ */
+ reinit_completion(&fsvq->in_flight_zero);
+ spin_unlock(&fsvq->lock);
+ wait_for_completion(&fsvq->in_flight_zero);
+ } else {
spin_unlock(&fsvq->lock);
- /* TODO use completion instead of timeout */
- usleep_range(1000, 2000);
}
flush_work(&fsvq->done_work);
flush_delayed_work(&fsvq->dispatch_work);
}
-static void virtio_fs_drain_all_queues(struct virtio_fs *fs)
+static void virtio_fs_drain_all_queues_locked(struct virtio_fs *fs)
{
struct virtio_fs_vq *fsvq;
int i;
}
}
+static void virtio_fs_drain_all_queues(struct virtio_fs *fs)
+{
+ /* Provides mutual exclusion between ->remove and ->kill_sb
+ * paths. We don't want both of these draining queue at the
+ * same time. Current completion logic reinits completion
+ * and that means there should not be any other thread
+ * doing reinit or waiting for completion already.
+ */
+ mutex_lock(&virtio_fs_mutex);
+ virtio_fs_drain_all_queues_locked(fs);
+ mutex_unlock(&virtio_fs_mutex);
+}
+
static void virtio_fs_start_all_queues(struct virtio_fs *fs)
{
struct virtio_fs_vq *fsvq;
}
}
+/*
+ * Returns 1 if queue is full and sender should wait a bit before sending
+ * next request, 0 otherwise.
+ */
+static int send_forget_request(struct virtio_fs_vq *fsvq,
+ struct virtio_fs_forget *forget,
+ bool in_flight)
+{
+ struct scatterlist sg;
+ struct virtqueue *vq;
+ int ret = 0;
+ bool notify;
+ struct virtio_fs_forget_req *req = &forget->req;
+
+ spin_lock(&fsvq->lock);
+ if (!fsvq->connected) {
+ if (in_flight)
+ dec_in_flight_req(fsvq);
+ kfree(forget);
+ goto out;
+ }
+
+ sg_init_one(&sg, req, sizeof(*req));
+ vq = fsvq->vq;
+ dev_dbg(&vq->vdev->dev, "%s\n", __func__);
+
+ ret = virtqueue_add_outbuf(vq, &sg, 1, forget, GFP_ATOMIC);
+ if (ret < 0) {
+ if (ret == -ENOMEM || ret == -ENOSPC) {
+ pr_debug("virtio-fs: Could not queue FORGET: err=%d. Will try later\n",
+ ret);
+ list_add_tail(&forget->list, &fsvq->queued_reqs);
+ schedule_delayed_work(&fsvq->dispatch_work,
+ msecs_to_jiffies(1));
+ if (!in_flight)
+ inc_in_flight_req(fsvq);
+ /* Queue is full */
+ ret = 1;
+ } else {
+ pr_debug("virtio-fs: Could not queue FORGET: err=%d. Dropping it.\n",
+ ret);
+ kfree(forget);
+ if (in_flight)
+ dec_in_flight_req(fsvq);
+ }
+ goto out;
+ }
+
+ if (!in_flight)
+ inc_in_flight_req(fsvq);
+ notify = virtqueue_kick_prepare(vq);
+ spin_unlock(&fsvq->lock);
+
+ if (notify)
+ virtqueue_notify(vq);
+ return ret;
+out:
+ spin_unlock(&fsvq->lock);
+ return ret;
+}
+
static void virtio_fs_hiprio_dispatch_work(struct work_struct *work)
{
struct virtio_fs_forget *forget;
struct virtio_fs_vq *fsvq = container_of(work, struct virtio_fs_vq,
dispatch_work.work);
- struct virtqueue *vq = fsvq->vq;
- struct scatterlist sg;
- struct scatterlist *sgs[] = {&sg};
- bool notify;
- int ret;
-
pr_debug("virtio-fs: worker %s called.\n", __func__);
while (1) {
spin_lock(&fsvq->lock);
}
list_del(&forget->list);
- if (!fsvq->connected) {
- dec_in_flight_req(fsvq);
- spin_unlock(&fsvq->lock);
- kfree(forget);
- continue;
- }
-
- sg_init_one(&sg, forget, sizeof(*forget));
-
- /* Enqueue the request */
- dev_dbg(&vq->vdev->dev, "%s\n", __func__);
- ret = virtqueue_add_sgs(vq, sgs, 1, 0, forget, GFP_ATOMIC);
- if (ret < 0) {
- if (ret == -ENOMEM || ret == -ENOSPC) {
- pr_debug("virtio-fs: Could not queue FORGET: err=%d. Will try later\n",
- ret);
- list_add_tail(&forget->list,
- &fsvq->queued_reqs);
- schedule_delayed_work(&fsvq->dispatch_work,
- msecs_to_jiffies(1));
- } else {
- pr_debug("virtio-fs: Could not queue FORGET: err=%d. Dropping it.\n",
- ret);
- dec_in_flight_req(fsvq);
- kfree(forget);
- }
- spin_unlock(&fsvq->lock);
- return;
- }
-
- notify = virtqueue_kick_prepare(vq);
spin_unlock(&fsvq->lock);
-
- if (notify)
- virtqueue_notify(vq);
- pr_debug("virtio-fs: worker %s dispatched one forget request.\n",
- __func__);
+ if (send_forget_request(fsvq, forget, true))
+ return;
}
}
INIT_LIST_HEAD(&fs->vqs[VQ_HIPRIO].end_reqs);
INIT_DELAYED_WORK(&fs->vqs[VQ_HIPRIO].dispatch_work,
virtio_fs_hiprio_dispatch_work);
+ init_completion(&fs->vqs[VQ_HIPRIO].in_flight_zero);
spin_lock_init(&fs->vqs[VQ_HIPRIO].lock);
/* Initialize the requests virtqueues */
virtio_fs_request_dispatch_work);
INIT_LIST_HEAD(&fs->vqs[i].queued_reqs);
INIT_LIST_HEAD(&fs->vqs[i].end_reqs);
+ init_completion(&fs->vqs[i].in_flight_zero);
snprintf(fs->vqs[i].name, sizeof(fs->vqs[i].name),
"requests.%u", i - VQ_REQUEST);
callbacks[i] = virtio_fs_vq_done;
/* This device is going away. No one should get new reference */
list_del_init(&fs->list);
virtio_fs_stop_all_queues(fs);
- virtio_fs_drain_all_queues(fs);
+ virtio_fs_drain_all_queues_locked(fs);
vdev->config->reset(vdev);
virtio_fs_cleanup_vqs(vdev, fs);
}
#endif /* CONFIG_PM_SLEEP */
-const static struct virtio_device_id id_table[] = {
+static const struct virtio_device_id id_table[] = {
{ VIRTIO_ID_FS, VIRTIO_DEV_ANY_ID },
{},
};
-const static unsigned int feature_table[] = {};
+static const unsigned int feature_table[] = {};
static struct virtio_driver virtio_fs_driver = {
.driver.name = KBUILD_MODNAME,
{
struct fuse_forget_link *link;
struct virtio_fs_forget *forget;
- struct scatterlist sg;
- struct scatterlist *sgs[] = {&sg};
+ struct virtio_fs_forget_req *req;
struct virtio_fs *fs;
- struct virtqueue *vq;
struct virtio_fs_vq *fsvq;
- bool notify;
u64 unique;
- int ret;
link = fuse_dequeue_forget(fiq, 1, NULL);
unique = fuse_get_unique(fiq);
/* Allocate a buffer for the request */
forget = kmalloc(sizeof(*forget), GFP_NOFS | __GFP_NOFAIL);
+ req = &forget->req;
- forget->ih = (struct fuse_in_header){
+ req->ih = (struct fuse_in_header){
.opcode = FUSE_FORGET,
.nodeid = link->forget_one.nodeid,
.unique = unique,
- .len = sizeof(*forget),
+ .len = sizeof(*req),
};
- forget->arg = (struct fuse_forget_in){
+ req->arg = (struct fuse_forget_in){
.nlookup = link->forget_one.nlookup,
};
- sg_init_one(&sg, forget, sizeof(*forget));
-
- /* Enqueue the request */
- spin_lock(&fsvq->lock);
-
- if (!fsvq->connected) {
- kfree(forget);
- spin_unlock(&fsvq->lock);
- goto out;
- }
-
- vq = fsvq->vq;
- dev_dbg(&vq->vdev->dev, "%s\n", __func__);
-
- ret = virtqueue_add_sgs(vq, sgs, 1, 0, forget, GFP_ATOMIC);
- if (ret < 0) {
- if (ret == -ENOMEM || ret == -ENOSPC) {
- pr_debug("virtio-fs: Could not queue FORGET: err=%d. Will try later.\n",
- ret);
- list_add_tail(&forget->list, &fsvq->queued_reqs);
- schedule_delayed_work(&fsvq->dispatch_work,
- msecs_to_jiffies(1));
- inc_in_flight_req(fsvq);
- } else {
- pr_debug("virtio-fs: Could not queue FORGET: err=%d. Dropping it.\n",
- ret);
- kfree(forget);
- }
- spin_unlock(&fsvq->lock);
- goto out;
- }
-
- inc_in_flight_req(fsvq);
- notify = virtqueue_kick_prepare(vq);
-
- spin_unlock(&fsvq->lock);
-
- if (notify)
- virtqueue_notify(vq);
-out:
+ send_forget_request(fsvq, forget, false);
kfree(link);
}
}
}
-const static struct fuse_iqueue_ops virtio_fs_fiq_ops = {
+static const struct fuse_iqueue_ops virtio_fs_fiq_ops = {
.wake_forget_and_unlock = virtio_fs_wake_forget_and_unlock,
.wake_interrupt_and_unlock = virtio_fs_wake_interrupt_and_unlock,
.wake_pending_and_unlock = virtio_fs_wake_pending_and_unlock,
* the page size, the remaining memory is zeroed when mapped, and
* writes to that region are not written out to the file."
*/
- offset = i_size & (PAGE_SIZE-1);
+ offset = i_size & (PAGE_SIZE - 1);
if (page->index == end_index && offset)
zero_user_segment(page, offset, PAGE_SIZE);
error = mpage_readpage(page, gfs2_block_map);
}
- if (unlikely(test_bit(SDF_WITHDRAWN, &sdp->sd_flags)))
+ if (unlikely(gfs2_withdrawn(sdp)))
return -EIO;
return error;
gfs2_glock_dq(&gh);
out_uninit:
gfs2_holder_uninit(&gh);
- if (unlikely(test_bit(SDF_WITHDRAWN, &sdp->sd_flags)))
+ if (unlikely(gfs2_withdrawn(sdp)))
ret = -EIO;
return ret;
}
struct inode *inode = file_inode(file);
struct gfs2_inode *ip = GFS2_I(inode);
struct gfs2_sbd *sdp = GFS2_SB(inode);
+ unsigned int blocksize = i_blocksize(inode);
+ loff_t start, end;
int error;
+ start = round_down(offset, blocksize);
+ end = round_up(offset + length, blocksize) - 1;
+ error = filemap_write_and_wait_range(inode->i_mapping, start, end);
+ if (error)
+ return error;
+
if (gfs2_is_jdata(ip))
error = gfs2_trans_begin(sdp, RES_DINODE + 2 * RES_JDATA,
GFS2_JTRUNC_REVOKES);
if (error)
goto out;
} else {
- unsigned int start_off, end_len, blocksize;
+ unsigned int start_off, end_len;
- blocksize = i_blocksize(inode);
start_off = offset & (blocksize - 1);
end_len = (offset + length) & (blocksize - 1);
if (start_off) {
/**
* gfs2_allocate_page_backing - Allocate blocks for a write fault
* @page: The (locked) page to allocate backing for
+ * @length: Size of the allocation
*
* We try to allocate all the blocks required for the page in one go. This
* might fail for various reasons, so we keep trying until all the blocks to
* back this page are allocated. If some of the blocks are already allocated,
* that is ok too.
*/
-static int gfs2_allocate_page_backing(struct page *page)
+static int gfs2_allocate_page_backing(struct page *page, unsigned int length)
{
u64 pos = page_offset(page);
- u64 size = PAGE_SIZE;
do {
struct iomap iomap = { };
- if (gfs2_iomap_get_alloc(page->mapping->host, pos, 1, &iomap))
+ if (gfs2_iomap_get_alloc(page->mapping->host, pos, length, &iomap))
return -EIO;
- iomap.length = min(iomap.length, size);
- size -= iomap.length;
+ if (length < iomap.length)
+ iomap.length = length;
+ length -= iomap.length;
pos += iomap.length;
- } while (size > 0);
+ } while (length > 0);
return 0;
}
struct gfs2_inode *ip = GFS2_I(inode);
struct gfs2_sbd *sdp = GFS2_SB(inode);
struct gfs2_alloc_parms ap = { .aflags = 0, };
- unsigned long last_index;
- u64 pos = page_offset(page);
+ u64 offset = page_offset(page);
unsigned int data_blocks, ind_blocks, rblocks;
struct gfs2_holder gh;
+ unsigned int length;
loff_t size;
int ret;
if (ret)
goto out;
- gfs2_size_hint(vmf->vma->vm_file, pos, PAGE_SIZE);
-
gfs2_holder_init(ip->i_gl, LM_ST_EXCLUSIVE, 0, &gh);
ret = gfs2_glock_nq(&gh);
if (ret)
goto out_uninit;
+ /* Check page index against inode size */
+ size = i_size_read(inode);
+ if (offset >= size) {
+ ret = -EINVAL;
+ goto out_unlock;
+ }
+
/* Update file times before taking page lock */
file_update_time(vmf->vma->vm_file);
+ /* page is wholly or partially inside EOF */
+ if (offset > size - PAGE_SIZE)
+ length = offset_in_page(size);
+ else
+ length = PAGE_SIZE;
+
+ gfs2_size_hint(vmf->vma->vm_file, offset, length);
+
set_bit(GLF_DIRTY, &ip->i_gl->gl_flags);
set_bit(GIF_SW_PAGED, &ip->i_flags);
- if (!gfs2_write_alloc_required(ip, pos, PAGE_SIZE)) {
+ /*
+ * iomap_writepage / iomap_writepages currently don't support inline
+ * files, so always unstuff here.
+ */
+
+ if (!gfs2_is_stuffed(ip) &&
+ !gfs2_write_alloc_required(ip, offset, length)) {
lock_page(page);
if (!PageUptodate(page) || page->mapping != inode->i_mapping) {
ret = -EAGAIN;
if (ret)
goto out_unlock;
- gfs2_write_calc_reserv(ip, PAGE_SIZE, &data_blocks, &ind_blocks);
+ gfs2_write_calc_reserv(ip, length, &data_blocks, &ind_blocks);
ap.target = data_blocks + ind_blocks;
ret = gfs2_quota_lock_check(ip, &ap);
if (ret)
goto out_trans_fail;
lock_page(page);
- ret = -EINVAL;
- size = i_size_read(inode);
- last_index = (size - 1) >> PAGE_SHIFT;
- /* Check page index against inode size */
- if (size == 0 || (page->index > last_index))
- goto out_trans_end;
-
ret = -EAGAIN;
/* If truncated, we must retry the operation, we may have raced
* with the glock demotion code.
if (gfs2_is_stuffed(ip))
ret = gfs2_unstuff_dinode(ip, page);
if (ret == 0)
- ret = gfs2_allocate_page_backing(page);
+ ret = gfs2_allocate_page_backing(page, length);
out_trans_end:
if (ret)
brelse(dibh);
return error;
}
+
/**
* calc_max_reserv() - Reverse of write_calc_reserv. Given a number of
* blocks, determine how many bytes can be written.
cmd = F_SETLK;
fl->fl_type = F_UNLCK;
}
- if (unlikely(test_bit(SDF_WITHDRAWN, &sdp->sd_flags))) {
+ if (unlikely(gfs2_withdrawn(sdp))) {
if (fl->fl_type == F_UNLCK)
locks_lock_file_wait(file, fl);
return -EIO;
unsigned int lck_flags = (unsigned int)(gh ? gh->gh_flags : 0);
int ret;
- if (unlikely(test_bit(SDF_WITHDRAWN, &sdp->sd_flags)) &&
+ if (unlikely(gfs2_withdrawn(sdp)) &&
target != LM_ST_UNLOCKED)
return;
lck_flags &= (LM_FLAG_TRY | LM_FLAG_TRY_1CB | LM_FLAG_NOEXP |
GLOCK_BUG_ON(gl, gl->gl_state == gl->gl_target);
if ((target == LM_ST_UNLOCKED || target == LM_ST_DEFERRED) &&
glops->go_inval) {
- set_bit(GLF_INVALIDATE_IN_PROGRESS, &gl->gl_flags);
+ /*
+ * If another process is already doing the invalidate, let that
+ * finish first. The glock state machine will get back to this
+ * holder again later.
+ */
+ if (test_and_set_bit(GLF_INVALIDATE_IN_PROGRESS,
+ &gl->gl_flags))
+ return;
do_error(gl, 0); /* Fail queued try locks */
}
gl->gl_req = target;
}
else if (ret) {
fs_err(sdp, "lm_lock ret %d\n", ret);
- GLOCK_BUG_ON(gl, !test_bit(SDF_WITHDRAWN,
- &sdp->sd_flags));
+ GLOCK_BUG_ON(gl, !gfs2_withdrawn(sdp));
}
} else { /* lock_nolock */
finish_xmote(gl, target);
struct gfs2_sbd *sdp = gl->gl_name.ln_sbd;
int error = 0;
- if (unlikely(test_bit(SDF_WITHDRAWN, &sdp->sd_flags)))
+ if (unlikely(gfs2_withdrawn(sdp)))
return -EIO;
if (test_bit(GLF_LRU, &gl->gl_flags))
ip->i_inode.i_rdev = MKDEV(be32_to_cpu(str->di_major),
be32_to_cpu(str->di_minor));
break;
- };
+ }
i_uid_write(&ip->i_inode, be32_to_cpu(str->di_uid));
i_gid_write(&ip->i_inode, be32_to_cpu(str->di_gid));
gfs2_consist(sdp);
/* Initialize some head of the log stuff */
- if (!test_bit(SDF_WITHDRAWN, &sdp->sd_flags)) {
+ if (!gfs2_withdrawn(sdp)) {
sdp->sd_log_sequence = head.lh_sequence + 1;
gfs2_log_pointers_init(sdp, head.lh_blkno);
}
inode->i_rdev = dev;
inode->i_size = size;
inode->i_atime = inode->i_mtime = inode->i_ctime = current_time(inode);
- gfs2_set_inode_blocks(inode, 1);
munge_mode_uid_gid(dip, inode);
check_and_update_goal(dip);
ip->i_goal = dip->i_goal;
error = gfs2_trans_begin(sdp, blocks, 0);
if (error)
- goto fail_gunlock2;
+ goto fail_free_inode;
if (blocks > 1) {
ip->i_eattr = ip->i_no_addr + 1;
error = gfs2_glock_get(sdp, ip->i_no_addr, &gfs2_iopen_glops, CREATE, &io_gl);
if (error)
- goto fail_gunlock2;
+ goto fail_free_inode;
BUG_ON(test_and_set_bit(GLF_INODE_CREATING, &io_gl->gl_flags));
goto fail_gunlock2;
glock_set_object(ip->i_iopen_gh.gh_gl, ip);
- gfs2_glock_put(io_gl);
gfs2_set_iop(inode);
insert_inode_hash(inode);
mark_inode_dirty(inode);
d_instantiate(dentry, inode);
+ /* After instantiate, errors should result in evict which will destroy
+ * both inode and iopen glocks properly. */
if (file) {
file->f_mode |= FMODE_CREATED;
error = finish_open(file, dentry, gfs2_open_common);
gfs2_glock_dq_uninit(ghs);
gfs2_glock_dq_uninit(ghs + 1);
clear_bit(GLF_INODE_CREATING, &io_gl->gl_flags);
+ gfs2_glock_put(io_gl);
return error;
fail_gunlock3:
glock_clear_object(io_gl, ip);
gfs2_glock_dq_uninit(&ip->i_iopen_gh);
- gfs2_glock_put(io_gl);
fail_gunlock2:
- if (io_gl)
- clear_bit(GLF_INODE_CREATING, &io_gl->gl_flags);
+ clear_bit(GLF_INODE_CREATING, &io_gl->gl_flags);
+ gfs2_glock_put(io_gl);
fail_free_inode:
if (ip->i_gl) {
glock_clear_object(ip->i_gl, ip);
error = -EEXIST;
default:
goto out_gunlock;
- };
+ }
if (odip != ndip) {
if (!ndip->i_inode.i_nlink) {
#include "dir.h"
#include "trace_gfs2.h"
+static void gfs2_log_shutdown(struct gfs2_sbd *sdp);
+
/**
* gfs2_struct2blk - compute stuff
* @sdp: the filesystem
list_for_each_entry_reverse(tr, head, tr_list) {
if (wbc->nr_to_write <= 0)
break;
- if (gfs2_ail1_start_one(sdp, wbc, tr, &withdraw))
+ if (gfs2_ail1_start_one(sdp, wbc, tr, &withdraw) &&
+ !gfs2_withdrawn(sdp))
goto restart;
}
spin_unlock(&sdp->sd_ail_lock);
list_add(&bd->bd_list, &sdp->sd_log_revokes);
}
+void gfs2_glock_remove_revoke(struct gfs2_glock *gl)
+{
+ if (atomic_dec_return(&gl->gl_revokes) == 0) {
+ clear_bit(GLF_LFLUSH, &gl->gl_flags);
+ gfs2_glock_queue_put(gl);
+ }
+}
+
void gfs2_write_revokes(struct gfs2_sbd *sdp)
{
struct gfs2_trans *tr;
{
struct gfs2_log_header *lh;
u32 hash, crc;
- struct page *page = mempool_alloc(gfs2_page_pool, GFP_NOIO);
+ struct page *page;
struct gfs2_statfs_change_host *l_sc = &sdp->sd_statfs_local;
struct timespec64 tv;
struct super_block *sb = sdp->sd_vfs;
u64 dblock;
+ if (gfs2_withdrawn(sdp))
+ goto out;
+
+ page = mempool_alloc(gfs2_page_pool, GFP_NOIO);
lh = page_address(page);
clear_page(lh);
lh->lh_nsec = cpu_to_be32(tv.tv_nsec);
lh->lh_sec = cpu_to_be64(tv.tv_sec);
if (!list_empty(&jd->extent_list))
- dblock = gfs2_log_bmap(sdp);
+ dblock = gfs2_log_bmap(jd, lblock);
else {
int ret = gfs2_lblk_to_dblk(jd->jd_inode, lblock, &dblock);
if (gfs2_assert_withdraw(sdp, ret == 0))
gfs2_log_write(sdp, page, sb->s_blocksize, 0, dblock);
gfs2_log_submit_bio(&sdp->sd_log_bio, REQ_OP_WRITE | op_flags);
+out:
log_flush_wait(sdp);
}
sdp->sd_log_idle = (tail == sdp->sd_log_flush_head);
gfs2_write_log_header(sdp, sdp->sd_jdesc, sdp->sd_log_sequence++, tail,
sdp->sd_log_flush_head, flags, op_flags);
+ gfs2_log_incr_head(sdp);
if (sdp->sd_log_tail != tail)
log_pull_tail(sdp, tail);
*
*/
-void gfs2_log_shutdown(struct gfs2_sbd *sdp)
+static void gfs2_log_shutdown(struct gfs2_sbd *sdp)
{
gfs2_assert_withdraw(sdp, !sdp->sd_log_blks_reserved);
gfs2_assert_withdraw(sdp, !sdp->sd_log_num_revoke);
extern void gfs2_log_commit(struct gfs2_sbd *sdp, struct gfs2_trans *trans);
extern void gfs2_ail1_flush(struct gfs2_sbd *sdp, struct writeback_control *wbc);
-extern void gfs2_log_shutdown(struct gfs2_sbd *sdp);
extern int gfs2_logd(void *data);
extern void gfs2_add_revoke(struct gfs2_sbd *sdp, struct gfs2_bufdata *bd);
+extern void gfs2_glock_remove_revoke(struct gfs2_glock *gl);
extern void gfs2_write_revokes(struct gfs2_sbd *sdp);
#endif /* __LOG_DOT_H__ */
atomic_dec(&sdp->sd_log_pinned);
}
-static void gfs2_log_incr_head(struct gfs2_sbd *sdp)
+void gfs2_log_incr_head(struct gfs2_sbd *sdp)
{
BUG_ON((sdp->sd_log_flush_head == sdp->sd_log_tail) &&
(sdp->sd_log_flush_head != sdp->sd_log_head));
sdp->sd_log_flush_head = 0;
}
-u64 gfs2_log_bmap(struct gfs2_sbd *sdp)
+u64 gfs2_log_bmap(struct gfs2_jdesc *jd, unsigned int lblock)
{
- unsigned int lbn = sdp->sd_log_flush_head;
struct gfs2_journal_extent *je;
- u64 block;
- list_for_each_entry(je, &sdp->sd_jdesc->extent_list, list) {
- if ((lbn >= je->lblock) && (lbn < (je->lblock + je->blocks))) {
- block = je->dblock + lbn - je->lblock;
- gfs2_log_incr_head(sdp);
- return block;
- }
+ list_for_each_entry(je, &jd->extent_list, list) {
+ if (lblock >= je->lblock && lblock < je->lblock + je->blocks)
+ return je->dblock + lblock - je->lblock;
}
return -1;
static void gfs2_log_write_bh(struct gfs2_sbd *sdp, struct buffer_head *bh)
{
- gfs2_log_write(sdp, bh->b_page, bh->b_size, bh_offset(bh),
- gfs2_log_bmap(sdp));
+ u64 dblock;
+
+ dblock = gfs2_log_bmap(sdp->sd_jdesc, sdp->sd_log_flush_head);
+ gfs2_log_incr_head(sdp);
+ gfs2_log_write(sdp, bh->b_page, bh->b_size, bh_offset(bh), dblock);
}
/**
void gfs2_log_write_page(struct gfs2_sbd *sdp, struct page *page)
{
struct super_block *sb = sdp->sd_vfs;
- gfs2_log_write(sdp, page, sb->s_blocksize, 0,
- gfs2_log_bmap(sdp));
+ u64 dblock;
+
+ dblock = gfs2_log_bmap(sdp->sd_jdesc, sdp->sd_log_flush_head);
+ gfs2_log_incr_head(sdp);
+ gfs2_log_write(sdp, page, sb->s_blocksize, 0, dblock);
}
/**
bd = list_entry(head->next, struct gfs2_bufdata, bd_list);
list_del_init(&bd->bd_list);
gl = bd->bd_gl;
- if (atomic_dec_return(&gl->gl_revokes) == 0) {
- clear_bit(GLF_LFLUSH, &gl->gl_flags);
- gfs2_glock_queue_put(gl);
- }
+ gfs2_glock_remove_revoke(gl);
kmem_cache_free(gfs2_bufdata_cachep, bd);
}
}
~(2 * sizeof(__be64) - 1))
extern const struct gfs2_log_operations *gfs2_log_ops[];
-extern u64 gfs2_log_bmap(struct gfs2_sbd *sdp);
+extern void gfs2_log_incr_head(struct gfs2_sbd *sdp);
+extern u64 gfs2_log_bmap(struct gfs2_jdesc *jd, unsigned int lbn);
extern void gfs2_log_write(struct gfs2_sbd *sdp, struct page *page,
unsigned size, unsigned offset, u64 blkno);
extern void gfs2_log_write_page(struct gfs2_sbd *sdp, struct page *page);
struct buffer_head *bh, *bhs[2];
int num = 0;
- if (unlikely(test_bit(SDF_WITHDRAWN, &sdp->sd_flags))) {
+ if (unlikely(gfs2_withdrawn(sdp))) {
*bhp = NULL;
return -EIO;
}
int gfs2_meta_wait(struct gfs2_sbd *sdp, struct buffer_head *bh)
{
- if (unlikely(test_bit(SDF_WITHDRAWN, &sdp->sd_flags)))
+ if (unlikely(gfs2_withdrawn(sdp)))
return -EIO;
wait_on_buffer(bh);
gfs2_io_error_bh_wd(sdp, bh);
return -EIO;
}
- if (unlikely(test_bit(SDF_WITHDRAWN, &sdp->sd_flags)))
+ if (unlikely(gfs2_withdrawn(sdp)))
return -EIO;
return 0;
void gfs2_lm_unmount(struct gfs2_sbd *sdp)
{
const struct lm_lockops *lm = sdp->sd_lockstruct.ls_ops;
- if (likely(!test_bit(SDF_WITHDRAWN, &sdp->sd_flags)) &&
- lm->lm_unmount)
+ if (likely(!gfs2_withdrawn(sdp)) && lm->lm_unmount)
lm->lm_unmount(sdp);
}
{}
};
-const struct fs_parameter_description gfs2_fs_parameters = {
+static const struct fs_parameter_description gfs2_fs_parameters = {
.name = "gfs2",
.specs = gfs2_param_specs,
.enums = gfs2_param_enums,
{
struct gfs2_sbd *sdp = sb->s_fs_info;
struct gfs2_quota_data **qda;
- unsigned int max_qd = PAGE_SIZE/sizeof(struct gfs2_holder);
+ unsigned int max_qd = PAGE_SIZE / sizeof(struct gfs2_holder);
unsigned int num_qd;
unsigned int x;
int error = 0;
{
if (error == 0 || error == -EROFS)
return;
- if (!test_bit(SDF_WITHDRAWN, &sdp->sd_flags)) {
+ if (!gfs2_withdrawn(sdp)) {
fs_err(sdp, "gfs2_quotad: %s error %d\n", msg, error);
sdp->sd_log_error = error;
wake_up(&sdp->sd_logd_waitq);
u32 lblock = head->lh_blkno;
gfs2_replay_incr_blk(jd, &lblock);
- if (jd->jd_jid == sdp->sd_lockstruct.ls_jid)
- sdp->sd_log_flush_head = lblock;
gfs2_write_log_header(sdp, jd, head->lh_sequence + 1, 0, lblock,
GFS2_LOG_HEAD_UNMOUNT | GFS2_LOG_HEAD_RECOVERY,
REQ_PREFLUSH | REQ_FUA | REQ_META | REQ_SYNC);
+ if (jd->jd_jid == sdp->sd_lockstruct.ls_jid) {
+ sdp->sd_log_flush_head = lblock;
+ gfs2_log_incr_head(sdp);
+ }
}
default:
goto fail;
- };
+ }
error = gfs2_glock_nq_init(ip->i_gl, LM_ST_SHARED,
LM_FLAG_NOEXP | GL_NOCACHE, &ji_gh);
* Returns: errno
*/
-static int gfs2_lock_fs_check_clean(struct gfs2_sbd *sdp,
- struct gfs2_holder *freeze_gh)
+static int gfs2_lock_fs_check_clean(struct gfs2_sbd *sdp)
{
struct gfs2_inode *ip;
struct gfs2_jdesc *jd;
}
error = gfs2_glock_nq_init(sdp->sd_freeze_gl, LM_ST_EXCLUSIVE,
- GL_NOCACHE, freeze_gh);
+ GL_NOCACHE, &sdp->sd_freeze_gh);
+ if (error)
+ goto out;
list_for_each_entry(jd, &sdp->sd_jindex_list, jd_list) {
error = gfs2_jdesc_check(jd);
}
if (error)
- gfs2_glock_dq_uninit(freeze_gh);
+ gfs2_glock_dq_uninit(&sdp->sd_freeze_gh);
out:
while (!list_empty(&list)) {
if (!(flags & I_DIRTY_INODE))
return;
- if (unlikely(test_bit(SDF_WITHDRAWN, &sdp->sd_flags)))
+ if (unlikely(gfs2_withdrawn(sdp)))
return;
if (!gfs2_glock_is_locked_by_me(ip->i_gl)) {
ret = gfs2_glock_nq_init(ip->i_gl, LM_ST_EXCLUSIVE, 0, &gh);
error = gfs2_glock_nq_init(sdp->sd_freeze_gl, LM_ST_SHARED, GL_NOCACHE,
&freeze_gh);
- if (error && !test_bit(SDF_WITHDRAWN, &sdp->sd_flags))
+ if (error && !gfs2_withdrawn(sdp))
return error;
flush_workqueue(gfs2_delete_workqueue);
if (atomic_read(&sdp->sd_freeze_state) != SFS_UNFROZEN)
goto out;
- if (test_bit(SDF_WITHDRAWN, &sdp->sd_flags)) {
- error = -EINVAL;
- goto out;
- }
-
for (;;) {
- error = gfs2_lock_fs_check_clean(sdp, &sdp->sd_freeze_gh);
+ if (gfs2_withdrawn(sdp)) {
+ error = -EINVAL;
+ goto out;
+ }
+
+ error = gfs2_lock_fs_check_clean(sdp);
if (!error)
break;
if (error == -EBUSY)
fs_err(sdp, "waiting for recovery before freeze\n");
- else
+ else if (error == -EIO) {
+ fs_err(sdp, "Fatal IO error: cannot freeze gfs2 due "
+ "to recovery error.\n");
+ goto out;
+ } else {
fs_err(sdp, "error freezing FS: %d\n", error);
-
+ }
fs_err(sdp, "retrying...\n");
msleep(1000);
}
static ssize_t withdraw_show(struct gfs2_sbd *sdp, char *buf)
{
- unsigned int b = test_bit(SDF_WITHDRAWN, &sdp->sd_flags);
+ unsigned int b = gfs2_withdrawn(sdp);
return snprintf(buf, PAGE_SIZE, "%u\n", b);
}
list_del_init(&bd->bd_list);
gfs2_assert_withdraw(sdp, sdp->sd_log_num_revoke);
sdp->sd_log_num_revoke--;
+ if (bd->bd_gl)
+ gfs2_glock_remove_revoke(bd->bd_gl);
kmem_cache_free(gfs2_bufdata_cachep, bd);
tr->tr_num_revoke--;
if (--n == 0)
const char *function, char *file, unsigned int line,
bool withdraw)
{
- if (!test_bit(SDF_WITHDRAWN, &sdp->sd_flags))
+ if (!gfs2_withdrawn(sdp))
fs_err(sdp,
"fatal: I/O error\n"
" block = %llu\n"
return x;
}
+/**
+ * gfs2_withdrawn - test whether the file system is withdrawing or withdrawn
+ * @sdp: the superblock
+ */
+static inline bool gfs2_withdrawn(struct gfs2_sbd *sdp)
+{
+ return test_bit(SDF_WITHDRAWN, &sdp->sd_flags);
+}
+
#define gfs2_tune_get(sdp, field) \
gfs2_tune_get_i(&(sdp)->sd_tune, &(sdp)->sd_tune.field)
struct task_struct *manager;
struct user_struct *user;
- struct cred *creds;
+ const struct cred *creds;
struct mm_struct *mm;
refcount_t refs;
struct completion done;
list->last = prev;
if (prev)
prev->next = node->next;
+ node->next = NULL;
}
#define wq_list_for_each(pos, prv, head) \
struct io_wq_data {
struct mm_struct *mm;
struct user_struct *user;
- struct cred *creds;
+ const struct cred *creds;
get_work_fn *get_work;
put_work_fn *put_work;
static inline void io_wq_worker_running(struct task_struct *tsk)
{
}
-#endif
+#endif /* CONFIG_IO_WQ */
-static inline bool io_wq_current_is_worker(void)
-{
- return in_task() && (current->flags & PF_IO_WORKER);
-}
-#endif
+#endif /* INTERNAL_IO_WQ_H */
/*
* Number of completion events lost because the queue was full;
* this should be avoided by the application by making sure
- * there are not more requests pending thatn there is space in
+ * there are not more requests pending than there is space in
* the completion queue.
*
* Written by the kernel, shouldn't be modified by the
struct user_struct *user;
- struct cred *creds;
+ const struct cred *creds;
/* 0 is for ctx quiesce/reinit/free, 1 is for sqo_thread started */
struct completion *completions;
* manipulate the list, hence no extra locking is needed there.
*/
struct list_head poll_list;
- struct rb_root cancel_tree;
+ struct hlist_head *cancel_hash;
+ unsigned cancel_hash_bits;
spinlock_t inflight_lock;
struct list_head inflight_list;
u32 seq_offset;
};
-struct io_timeout {
- struct file *file;
- struct io_timeout_data *data;
+struct io_async_connect {
+ struct sockaddr_storage address;
+};
+
+struct io_async_msghdr {
+ struct iovec fast_iov[UIO_FASTIOV];
+ struct iovec *iov;
+ struct sockaddr __user *uaddr;
+ struct msghdr msg;
+};
+
+struct io_async_rw {
+ struct iovec fast_iov[UIO_FASTIOV];
+ struct iovec *iov;
+ ssize_t nr_segs;
+ ssize_t size;
+};
+
+struct io_async_ctx {
+ struct io_uring_sqe sqe;
+ union {
+ struct io_async_rw rw;
+ struct io_async_msghdr msg;
+ struct io_async_connect connect;
+ struct io_timeout_data timeout;
+ };
};
/*
struct file *file;
struct kiocb rw;
struct io_poll_iocb poll;
- struct io_timeout timeout;
};
const struct io_uring_sqe *sqe;
+ struct io_async_ctx *io;
struct file *ring_file;
int ring_fd;
bool has_user;
struct io_ring_ctx *ctx;
union {
struct list_head list;
- struct rb_node rb_node;
+ struct hlist_node hash_node;
};
struct list_head link_list;
unsigned int flags;
#define REQ_F_TIMEOUT_NOSEQ 8192 /* no timeout sequence */
#define REQ_F_INFLIGHT 16384 /* on inflight list */
#define REQ_F_COMP_LOCKED 32768 /* completion under lock */
-#define REQ_F_FREE_SQE 65536 /* free sqe if not async queued */
u64 user_data;
u32 result;
u32 sequence;
static struct io_ring_ctx *io_ring_ctx_alloc(struct io_uring_params *p)
{
struct io_ring_ctx *ctx;
+ int hash_bits;
ctx = kzalloc(sizeof(*ctx), GFP_KERNEL);
if (!ctx)
if (!ctx->completions)
goto err;
+ /*
+ * Use 5 bits less than the max cq entries, that should give us around
+ * 32 entries per hash list if totally full and uniformly spread.
+ */
+ hash_bits = ilog2(p->cq_entries);
+ hash_bits -= 5;
+ if (hash_bits <= 0)
+ hash_bits = 1;
+ ctx->cancel_hash_bits = hash_bits;
+ ctx->cancel_hash = kmalloc((1U << hash_bits) * sizeof(struct hlist_head),
+ GFP_KERNEL);
+ if (!ctx->cancel_hash)
+ goto err;
+ __hash_init(ctx->cancel_hash, 1U << hash_bits);
+
if (percpu_ref_init(&ctx->refs, io_ring_ctx_ref_free,
PERCPU_REF_ALLOW_REINIT, GFP_KERNEL))
goto err;
init_waitqueue_head(&ctx->wait);
spin_lock_init(&ctx->completion_lock);
INIT_LIST_HEAD(&ctx->poll_list);
- ctx->cancel_tree = RB_ROOT;
INIT_LIST_HEAD(&ctx->defer_list);
INIT_LIST_HEAD(&ctx->timeout_list);
init_waitqueue_head(&ctx->inflight_wait);
if (ctx->fallback_req)
kmem_cache_free(req_cachep, ctx->fallback_req);
kfree(ctx->completions);
+ kfree(ctx->cancel_hash);
kfree(ctx);
return NULL;
}
{
int ret;
- ret = hrtimer_try_to_cancel(&req->timeout.data->timer);
+ ret = hrtimer_try_to_cancel(&req->io->timeout.timer);
if (ret != -1) {
atomic_inc(&req->ctx->cq_timeouts);
list_del_init(&req->list);
}
got_it:
+ req->io = NULL;
req->ring_file = NULL;
req->file = NULL;
req->ctx = ctx;
{
struct io_ring_ctx *ctx = req->ctx;
- if (req->flags & REQ_F_FREE_SQE)
- kfree(req->sqe);
+ if (req->io)
+ kfree(req->io);
if (req->file && !(req->flags & REQ_F_FIXED_FILE))
fput(req->file);
if (req->flags & REQ_F_INFLIGHT) {
wake_up(&ctx->inflight_wait);
spin_unlock_irqrestore(&ctx->inflight_lock, flags);
}
- if (req->flags & REQ_F_TIMEOUT)
- kfree(req->timeout.data);
percpu_ref_put(&ctx->refs);
if (likely(!io_is_fallback_req(req)))
kmem_cache_free(req_cachep, req);
struct io_ring_ctx *ctx = req->ctx;
int ret;
- ret = hrtimer_try_to_cancel(&req->timeout.data->timer);
+ ret = hrtimer_try_to_cancel(&req->io->timeout.timer);
if (ret != -1) {
io_cqring_fill_event(req, -ECANCELED);
io_commit_cqring(ctx);
static void io_req_link_next(struct io_kiocb *req, struct io_kiocb **nxtptr)
{
struct io_ring_ctx *ctx = req->ctx;
- struct io_kiocb *nxt;
bool wake_ev = false;
/* Already got next link */
* potentially happen if the chain is messed up, check to be on the
* safe side.
*/
- nxt = list_first_entry_or_null(&req->link_list, struct io_kiocb, list);
- while (nxt) {
- list_del_init(&nxt->list);
+ while (!list_empty(&req->link_list)) {
+ struct io_kiocb *nxt = list_first_entry(&req->link_list,
+ struct io_kiocb, link_list);
- if ((req->flags & REQ_F_LINK_TIMEOUT) &&
- (nxt->flags & REQ_F_TIMEOUT)) {
+ if (unlikely((req->flags & REQ_F_LINK_TIMEOUT) &&
+ (nxt->flags & REQ_F_TIMEOUT))) {
+ list_del_init(&nxt->link_list);
wake_ev |= io_link_cancel_timeout(nxt);
- nxt = list_first_entry_or_null(&req->link_list,
- struct io_kiocb, list);
req->flags &= ~REQ_F_LINK_TIMEOUT;
continue;
}
- if (!list_empty(&req->link_list)) {
- INIT_LIST_HEAD(&nxt->link_list);
- list_splice(&req->link_list, &nxt->link_list);
- nxt->flags |= REQ_F_LINK;
- }
+ list_del_init(&req->link_list);
+ if (!list_empty(&nxt->link_list))
+ nxt->flags |= REQ_F_LINK;
*nxtptr = nxt;
break;
}
static void io_fail_links(struct io_kiocb *req)
{
struct io_ring_ctx *ctx = req->ctx;
- struct io_kiocb *link;
unsigned long flags;
spin_lock_irqsave(&ctx->completion_lock, flags);
while (!list_empty(&req->link_list)) {
- link = list_first_entry(&req->link_list, struct io_kiocb, list);
- list_del_init(&link->list);
+ struct io_kiocb *link = list_first_entry(&req->link_list,
+ struct io_kiocb, link_list);
+ list_del_init(&link->link_list);
trace_io_uring_fail_link(req, link);
if ((req->flags & REQ_F_LINK_TIMEOUT) &&
* completions for those, only batch free for fixed
* file and non-linked commands.
*/
- if (((req->flags &
- (REQ_F_FIXED_FILE|REQ_F_LINK|REQ_F_FREE_SQE)) ==
- REQ_F_FIXED_FILE) && !io_is_fallback_req(req)) {
+ if (((req->flags & (REQ_F_FIXED_FILE|REQ_F_LINK)) ==
+ REQ_F_FIXED_FILE) && !io_is_fallback_req(req) &&
+ !req->io) {
reqs[to_free++] = req;
if (to_free == ARRAY_SIZE(reqs))
io_free_req_many(ctx, reqs, &to_free);
if (S_ISREG(file_inode(req->file)->i_mode))
req->flags |= REQ_F_ISREG;
- /*
- * If the file doesn't support async, mark it as REQ_F_MUST_PUNT so
- * we know to async punt it even if it was opened O_NONBLOCK
- */
- if (force_nonblock && !io_file_supports_async(req->file)) {
- req->flags |= REQ_F_MUST_PUNT;
- return -EAGAIN;
- }
-
kiocb->ki_pos = READ_ONCE(sqe->off);
kiocb->ki_flags = iocb_flags(kiocb->ki_filp);
kiocb->ki_hint = ki_hint_validate(file_write_hint(kiocb->ki_filp));
return io_import_fixed(req->ctx, rw, sqe, iter);
}
+ if (req->io) {
+ struct io_async_rw *iorw = &req->io->rw;
+
+ *iovec = iorw->iov;
+ iov_iter_init(iter, rw, *iovec, iorw->nr_segs, iorw->size);
+ if (iorw->iov == iorw->fast_iov)
+ *iovec = NULL;
+ return iorw->size;
+ }
+
if (!req->has_user)
return -EFAULT;
return ret;
}
+static void io_req_map_io(struct io_kiocb *req, ssize_t io_size,
+ struct iovec *iovec, struct iovec *fast_iov,
+ struct iov_iter *iter)
+{
+ req->io->rw.nr_segs = iter->nr_segs;
+ req->io->rw.size = io_size;
+ req->io->rw.iov = iovec;
+ if (!req->io->rw.iov) {
+ req->io->rw.iov = req->io->rw.fast_iov;
+ memcpy(req->io->rw.iov, fast_iov,
+ sizeof(struct iovec) * iter->nr_segs);
+ }
+}
+
+static int io_setup_async_io(struct io_kiocb *req, ssize_t io_size,
+ struct iovec *iovec, struct iovec *fast_iov,
+ struct iov_iter *iter)
+{
+ req->io = kmalloc(sizeof(*req->io), GFP_KERNEL);
+ if (req->io) {
+ io_req_map_io(req, io_size, iovec, fast_iov, iter);
+ memcpy(&req->io->sqe, req->sqe, sizeof(req->io->sqe));
+ req->sqe = &req->io->sqe;
+ return 0;
+ }
+
+ return -ENOMEM;
+}
+
+static int io_read_prep(struct io_kiocb *req, struct iovec **iovec,
+ struct iov_iter *iter, bool force_nonblock)
+{
+ ssize_t ret;
+
+ ret = io_prep_rw(req, force_nonblock);
+ if (ret)
+ return ret;
+
+ if (unlikely(!(req->file->f_mode & FMODE_READ)))
+ return -EBADF;
+
+ return io_import_iovec(READ, req, iovec, iter);
+}
+
static int io_read(struct io_kiocb *req, struct io_kiocb **nxt,
bool force_nonblock)
{
struct iov_iter iter;
struct file *file;
size_t iov_count;
- ssize_t read_size, ret;
-
- ret = io_prep_rw(req, force_nonblock);
- if (ret)
- return ret;
- file = kiocb->ki_filp;
+ ssize_t io_size, ret;
- if (unlikely(!(file->f_mode & FMODE_READ)))
- return -EBADF;
-
- ret = io_import_iovec(READ, req, &iovec, &iter);
- if (ret < 0)
- return ret;
+ if (!req->io) {
+ ret = io_read_prep(req, &iovec, &iter, force_nonblock);
+ if (ret < 0)
+ return ret;
+ } else {
+ ret = io_import_iovec(READ, req, &iovec, &iter);
+ if (ret < 0)
+ return ret;
+ }
- read_size = ret;
+ file = req->file;
+ io_size = ret;
if (req->flags & REQ_F_LINK)
- req->result = read_size;
+ req->result = io_size;
+
+ /*
+ * If the file doesn't support async, mark it as REQ_F_MUST_PUNT so
+ * we know to async punt it even if it was opened O_NONBLOCK
+ */
+ if (force_nonblock && !io_file_supports_async(file)) {
+ req->flags |= REQ_F_MUST_PUNT;
+ goto copy_iov;
+ }
iov_count = iov_iter_count(&iter);
ret = rw_verify_area(READ, file, &kiocb->ki_pos, iov_count);
*/
if (force_nonblock && !(req->flags & REQ_F_NOWAIT) &&
(req->flags & REQ_F_ISREG) &&
- ret2 > 0 && ret2 < read_size)
+ ret2 > 0 && ret2 < io_size)
ret2 = -EAGAIN;
/* Catch -EAGAIN return for forced non-blocking submission */
- if (!force_nonblock || ret2 != -EAGAIN)
+ if (!force_nonblock || ret2 != -EAGAIN) {
kiocb_done(kiocb, ret2, nxt, req->in_async);
- else
- ret = -EAGAIN;
+ } else {
+copy_iov:
+ ret = io_setup_async_io(req, io_size, iovec,
+ inline_vecs, &iter);
+ if (ret)
+ goto out_free;
+ return -EAGAIN;
+ }
}
+out_free:
kfree(iovec);
return ret;
}
+static int io_write_prep(struct io_kiocb *req, struct iovec **iovec,
+ struct iov_iter *iter, bool force_nonblock)
+{
+ ssize_t ret;
+
+ ret = io_prep_rw(req, force_nonblock);
+ if (ret)
+ return ret;
+
+ if (unlikely(!(req->file->f_mode & FMODE_WRITE)))
+ return -EBADF;
+
+ return io_import_iovec(WRITE, req, iovec, iter);
+}
+
static int io_write(struct io_kiocb *req, struct io_kiocb **nxt,
bool force_nonblock)
{
struct iov_iter iter;
struct file *file;
size_t iov_count;
- ssize_t ret;
+ ssize_t ret, io_size;
- ret = io_prep_rw(req, force_nonblock);
- if (ret)
- return ret;
+ if (!req->io) {
+ ret = io_write_prep(req, &iovec, &iter, force_nonblock);
+ if (ret < 0)
+ return ret;
+ } else {
+ ret = io_import_iovec(WRITE, req, &iovec, &iter);
+ if (ret < 0)
+ return ret;
+ }
file = kiocb->ki_filp;
- if (unlikely(!(file->f_mode & FMODE_WRITE)))
- return -EBADF;
-
- ret = io_import_iovec(WRITE, req, &iovec, &iter);
- if (ret < 0)
- return ret;
-
+ io_size = ret;
if (req->flags & REQ_F_LINK)
- req->result = ret;
+ req->result = io_size;
- iov_count = iov_iter_count(&iter);
+ /*
+ * If the file doesn't support async, mark it as REQ_F_MUST_PUNT so
+ * we know to async punt it even if it was opened O_NONBLOCK
+ */
+ if (force_nonblock && !io_file_supports_async(req->file)) {
+ req->flags |= REQ_F_MUST_PUNT;
+ goto copy_iov;
+ }
- ret = -EAGAIN;
if (force_nonblock && !(kiocb->ki_flags & IOCB_DIRECT))
- goto out_free;
+ goto copy_iov;
+ iov_count = iov_iter_count(&iter);
ret = rw_verify_area(WRITE, file, &kiocb->ki_pos, iov_count);
if (!ret) {
ssize_t ret2;
ret2 = call_write_iter(file, kiocb, &iter);
else
ret2 = loop_rw_iter(WRITE, file, kiocb, &iter);
- if (!force_nonblock || ret2 != -EAGAIN)
+ if (!force_nonblock || ret2 != -EAGAIN) {
kiocb_done(kiocb, ret2, nxt, req->in_async);
- else
- ret = -EAGAIN;
+ } else {
+copy_iov:
+ ret = io_setup_async_io(req, io_size, iovec,
+ inline_vecs, &iter);
+ if (ret)
+ goto out_free;
+ return -EAGAIN;
+ }
}
out_free:
kfree(iovec);
return 0;
}
+static int io_sendmsg_prep(struct io_kiocb *req, struct io_async_ctx *io)
+{
#if defined(CONFIG_NET)
-static int io_send_recvmsg(struct io_kiocb *req, const struct io_uring_sqe *sqe,
- struct io_kiocb **nxt, bool force_nonblock,
- long (*fn)(struct socket *, struct user_msghdr __user *,
- unsigned int))
+ const struct io_uring_sqe *sqe = req->sqe;
+ struct user_msghdr __user *msg;
+ unsigned flags;
+
+ flags = READ_ONCE(sqe->msg_flags);
+ msg = (struct user_msghdr __user *)(unsigned long) READ_ONCE(sqe->addr);
+ return sendmsg_copy_msghdr(&io->msg.msg, msg, flags, &io->msg.iov);
+#else
+ return 0;
+#endif
+}
+
+static int io_sendmsg(struct io_kiocb *req, const struct io_uring_sqe *sqe,
+ struct io_kiocb **nxt, bool force_nonblock)
{
+#if defined(CONFIG_NET)
struct socket *sock;
int ret;
sock = sock_from_file(req->file, &ret);
if (sock) {
- struct user_msghdr __user *msg;
+ struct io_async_ctx io, *copy;
+ struct sockaddr_storage addr;
+ struct msghdr *kmsg;
unsigned flags;
flags = READ_ONCE(sqe->msg_flags);
else if (force_nonblock)
flags |= MSG_DONTWAIT;
- msg = (struct user_msghdr __user *) (unsigned long)
- READ_ONCE(sqe->addr);
+ if (req->io) {
+ kmsg = &req->io->msg.msg;
+ kmsg->msg_name = &addr;
+ } else {
+ kmsg = &io.msg.msg;
+ kmsg->msg_name = &addr;
+ io.msg.iov = io.msg.fast_iov;
+ ret = io_sendmsg_prep(req, &io);
+ if (ret)
+ goto out;
+ }
- ret = fn(sock, msg, flags);
- if (force_nonblock && ret == -EAGAIN)
+ ret = __sys_sendmsg_sock(sock, kmsg, flags);
+ if (force_nonblock && ret == -EAGAIN) {
+ copy = kmalloc(sizeof(*copy), GFP_KERNEL);
+ if (!copy) {
+ ret = -ENOMEM;
+ goto out;
+ }
+ memcpy(©->msg, &io.msg, sizeof(copy->msg));
+ req->io = copy;
+ memcpy(&req->io->sqe, req->sqe, sizeof(*req->sqe));
+ req->sqe = &req->io->sqe;
return ret;
+ }
+ if (ret == -ERESTARTSYS)
+ ret = -EINTR;
}
+out:
io_cqring_add_event(req, ret);
if (ret < 0 && (req->flags & REQ_F_LINK))
req->flags |= REQ_F_FAIL_LINK;
io_put_req_find_next(req, nxt);
return 0;
-}
+#else
+ return -EOPNOTSUPP;
#endif
+}
-static int io_sendmsg(struct io_kiocb *req, const struct io_uring_sqe *sqe,
- struct io_kiocb **nxt, bool force_nonblock)
+static int io_recvmsg_prep(struct io_kiocb *req, struct io_async_ctx *io)
{
#if defined(CONFIG_NET)
- return io_send_recvmsg(req, sqe, nxt, force_nonblock,
- __sys_sendmsg_sock);
+ const struct io_uring_sqe *sqe = req->sqe;
+ struct user_msghdr __user *msg;
+ unsigned flags;
+
+ flags = READ_ONCE(sqe->msg_flags);
+ msg = (struct user_msghdr __user *)(unsigned long) READ_ONCE(sqe->addr);
+ return recvmsg_copy_msghdr(&io->msg.msg, msg, flags, &io->msg.uaddr,
+ &io->msg.iov);
#else
- return -EOPNOTSUPP;
+ return 0;
#endif
}
struct io_kiocb **nxt, bool force_nonblock)
{
#if defined(CONFIG_NET)
- return io_send_recvmsg(req, sqe, nxt, force_nonblock,
- __sys_recvmsg_sock);
+ struct socket *sock;
+ int ret;
+
+ if (unlikely(req->ctx->flags & IORING_SETUP_IOPOLL))
+ return -EINVAL;
+
+ sock = sock_from_file(req->file, &ret);
+ if (sock) {
+ struct user_msghdr __user *msg;
+ struct io_async_ctx io, *copy;
+ struct sockaddr_storage addr;
+ struct msghdr *kmsg;
+ unsigned flags;
+
+ flags = READ_ONCE(sqe->msg_flags);
+ if (flags & MSG_DONTWAIT)
+ req->flags |= REQ_F_NOWAIT;
+ else if (force_nonblock)
+ flags |= MSG_DONTWAIT;
+
+ msg = (struct user_msghdr __user *) (unsigned long)
+ READ_ONCE(sqe->addr);
+ if (req->io) {
+ kmsg = &req->io->msg.msg;
+ kmsg->msg_name = &addr;
+ } else {
+ kmsg = &io.msg.msg;
+ kmsg->msg_name = &addr;
+ io.msg.iov = io.msg.fast_iov;
+ ret = io_recvmsg_prep(req, &io);
+ if (ret)
+ goto out;
+ }
+
+ ret = __sys_recvmsg_sock(sock, kmsg, msg, io.msg.uaddr, flags);
+ if (force_nonblock && ret == -EAGAIN) {
+ copy = kmalloc(sizeof(*copy), GFP_KERNEL);
+ if (!copy) {
+ ret = -ENOMEM;
+ goto out;
+ }
+ memcpy(copy, &io, sizeof(*copy));
+ req->io = copy;
+ memcpy(&req->io->sqe, req->sqe, sizeof(*req->sqe));
+ req->sqe = &req->io->sqe;
+ return ret;
+ }
+ if (ret == -ERESTARTSYS)
+ ret = -EINTR;
+ }
+
+out:
+ io_cqring_add_event(req, ret);
+ if (ret < 0 && (req->flags & REQ_F_LINK))
+ req->flags |= REQ_F_FAIL_LINK;
+ io_put_req_find_next(req, nxt);
+ return 0;
#else
return -EOPNOTSUPP;
#endif
#endif
}
+static int io_connect_prep(struct io_kiocb *req, struct io_async_ctx *io)
+{
+#if defined(CONFIG_NET)
+ const struct io_uring_sqe *sqe = req->sqe;
+ struct sockaddr __user *addr;
+ int addr_len;
+
+ addr = (struct sockaddr __user *) (unsigned long) READ_ONCE(sqe->addr);
+ addr_len = READ_ONCE(sqe->addr2);
+ return move_addr_to_kernel(addr, addr_len, &io->connect.address);
+#else
+ return 0;
+#endif
+}
+
static int io_connect(struct io_kiocb *req, const struct io_uring_sqe *sqe,
struct io_kiocb **nxt, bool force_nonblock)
{
#if defined(CONFIG_NET)
- struct sockaddr __user *addr;
+ struct io_async_ctx __io, *io;
unsigned file_flags;
int addr_len, ret;
if (sqe->ioprio || sqe->len || sqe->buf_index || sqe->rw_flags)
return -EINVAL;
- addr = (struct sockaddr __user *) (unsigned long) READ_ONCE(sqe->addr);
addr_len = READ_ONCE(sqe->addr2);
file_flags = force_nonblock ? O_NONBLOCK : 0;
- ret = __sys_connect_file(req->file, addr, addr_len, file_flags);
- if (ret == -EAGAIN && force_nonblock)
+ if (req->io) {
+ io = req->io;
+ } else {
+ ret = io_connect_prep(req, &__io);
+ if (ret)
+ goto out;
+ io = &__io;
+ }
+
+ ret = __sys_connect_file(req->file, &io->connect.address, addr_len,
+ file_flags);
+ if ((ret == -EAGAIN || ret == -EINPROGRESS) && force_nonblock) {
+ io = kmalloc(sizeof(*io), GFP_KERNEL);
+ if (!io) {
+ ret = -ENOMEM;
+ goto out;
+ }
+ memcpy(&io->connect, &__io.connect, sizeof(io->connect));
+ req->io = io;
+ memcpy(&io->sqe, req->sqe, sizeof(*req->sqe));
+ req->sqe = &io->sqe;
return -EAGAIN;
+ }
if (ret == -ERESTARTSYS)
ret = -EINTR;
+out:
if (ret < 0 && (req->flags & REQ_F_LINK))
req->flags |= REQ_F_FAIL_LINK;
io_cqring_add_event(req, ret);
#endif
}
-static inline void io_poll_remove_req(struct io_kiocb *req)
-{
- if (!RB_EMPTY_NODE(&req->rb_node)) {
- rb_erase(&req->rb_node, &req->ctx->cancel_tree);
- RB_CLEAR_NODE(&req->rb_node);
- }
-}
-
static void io_poll_remove_one(struct io_kiocb *req)
{
struct io_poll_iocb *poll = &req->poll;
io_queue_async_work(req);
}
spin_unlock(&poll->head->lock);
- io_poll_remove_req(req);
+ hash_del(&req->hash_node);
}
static void io_poll_remove_all(struct io_ring_ctx *ctx)
{
- struct rb_node *node;
+ struct hlist_node *tmp;
struct io_kiocb *req;
+ int i;
spin_lock_irq(&ctx->completion_lock);
- while ((node = rb_first(&ctx->cancel_tree)) != NULL) {
- req = rb_entry(node, struct io_kiocb, rb_node);
- io_poll_remove_one(req);
+ for (i = 0; i < (1U << ctx->cancel_hash_bits); i++) {
+ struct hlist_head *list;
+
+ list = &ctx->cancel_hash[i];
+ hlist_for_each_entry_safe(req, tmp, list, hash_node)
+ io_poll_remove_one(req);
}
spin_unlock_irq(&ctx->completion_lock);
}
static int io_poll_cancel(struct io_ring_ctx *ctx, __u64 sqe_addr)
{
- struct rb_node *p, *parent = NULL;
+ struct hlist_head *list;
struct io_kiocb *req;
- p = ctx->cancel_tree.rb_node;
- while (p) {
- parent = p;
- req = rb_entry(parent, struct io_kiocb, rb_node);
- if (sqe_addr < req->user_data) {
- p = p->rb_left;
- } else if (sqe_addr > req->user_data) {
- p = p->rb_right;
- } else {
+ list = &ctx->cancel_hash[hash_long(sqe_addr, ctx->cancel_hash_bits)];
+ hlist_for_each_entry(req, list, hash_node) {
+ if (sqe_addr == req->user_data) {
io_poll_remove_one(req);
return 0;
}
spin_unlock_irq(&ctx->completion_lock);
return;
}
- io_poll_remove_req(req);
+ hash_del(&req->hash_node);
io_poll_complete(req, mask, ret);
spin_unlock_irq(&ctx->completion_lock);
* for finalizing the request, mark us as having grabbed that already.
*/
if (mask && spin_trylock_irqsave(&ctx->completion_lock, flags)) {
- io_poll_remove_req(req);
+ hash_del(&req->hash_node);
io_poll_complete(req, mask, 0);
req->flags |= REQ_F_COMP_LOCKED;
io_put_req(req);
static void io_poll_req_insert(struct io_kiocb *req)
{
struct io_ring_ctx *ctx = req->ctx;
- struct rb_node **p = &ctx->cancel_tree.rb_node;
- struct rb_node *parent = NULL;
- struct io_kiocb *tmp;
-
- while (*p) {
- parent = *p;
- tmp = rb_entry(parent, struct io_kiocb, rb_node);
- if (req->user_data < tmp->user_data)
- p = &(*p)->rb_left;
- else
- p = &(*p)->rb_right;
- }
- rb_link_node(&req->rb_node, parent, p);
- rb_insert_color(&req->rb_node, &ctx->cancel_tree);
+ struct hlist_head *list;
+
+ list = &ctx->cancel_hash[hash_long(req->user_data, ctx->cancel_hash_bits)];
+ hlist_add_head(&req->hash_node, list);
}
static int io_poll_add(struct io_kiocb *req, const struct io_uring_sqe *sqe,
if (!poll->wait)
return -ENOMEM;
- req->sqe = NULL;
+ req->io = NULL;
INIT_IO_WORK(&req->work, io_poll_complete_work);
events = READ_ONCE(sqe->poll_events);
poll->events = demangle_poll(events) | EPOLLERR | EPOLLHUP;
- RB_CLEAR_NODE(&req->rb_node);
+ INIT_HLIST_NODE(&req->hash_node);
poll->head = NULL;
poll->done = false;
if (ret == -ENOENT)
return ret;
- ret = hrtimer_try_to_cancel(&req->timeout.data->timer);
+ ret = hrtimer_try_to_cancel(&req->io->timeout.timer);
if (ret == -1)
return -EALREADY;
return 0;
}
-static int io_timeout_setup(struct io_kiocb *req)
+static int io_timeout_prep(struct io_kiocb *req, struct io_async_ctx *io,
+ bool is_timeout_link)
{
const struct io_uring_sqe *sqe = req->sqe;
struct io_timeout_data *data;
return -EINVAL;
if (sqe->ioprio || sqe->buf_index || sqe->len != 1)
return -EINVAL;
+ if (sqe->off && is_timeout_link)
+ return -EINVAL;
flags = READ_ONCE(sqe->timeout_flags);
if (flags & ~IORING_TIMEOUT_ABS)
return -EINVAL;
- data = kzalloc(sizeof(struct io_timeout_data), GFP_KERNEL);
- if (!data)
- return -ENOMEM;
+ data = &io->timeout;
data->req = req;
- req->timeout.data = data;
req->flags |= REQ_F_TIMEOUT;
if (get_timespec64(&data->ts, u64_to_user_ptr(sqe->addr)))
data->mode = HRTIMER_MODE_REL;
hrtimer_init(&data->timer, CLOCK_MONOTONIC, data->mode);
+ req->io = io;
return 0;
}
unsigned count;
struct io_ring_ctx *ctx = req->ctx;
struct io_timeout_data *data;
+ struct io_async_ctx *io;
struct list_head *entry;
unsigned span = 0;
- int ret;
- ret = io_timeout_setup(req);
- /* common setup allows flags (like links) set, we don't */
- if (!ret && sqe->flags)
- ret = -EINVAL;
- if (ret)
- return ret;
+ io = req->io;
+ if (!io) {
+ int ret;
+
+ io = kmalloc(sizeof(*io), GFP_KERNEL);
+ if (!io)
+ return -ENOMEM;
+ ret = io_timeout_prep(req, io, false);
+ if (ret) {
+ kfree(io);
+ return ret;
+ }
+ }
+ data = &req->io->timeout;
/*
* sqe->off holds how many events that need to occur for this
}
req->sequence = ctx->cached_sq_head + count - 1;
- req->timeout.data->seq_offset = count;
+ data->seq_offset = count;
/*
* Insertion sort, ensuring the first entry in the list is always
struct io_kiocb *nxt = list_entry(entry, struct io_kiocb, list);
unsigned nxt_sq_head;
long long tmp, tmp_nxt;
- u32 nxt_offset = nxt->timeout.data->seq_offset;
+ u32 nxt_offset = nxt->io->timeout.seq_offset;
if (nxt->flags & REQ_F_TIMEOUT_NOSEQ)
continue;
req->sequence -= span;
add:
list_add(&req->list, entry);
- data = req->timeout.data;
data->timer.function = io_timeout_fn;
hrtimer_start(&data->timer, timespec64_to_ktime(data->ts), data->mode);
spin_unlock_irq(&ctx->completion_lock);
return 0;
}
+static int io_req_defer_prep(struct io_kiocb *req, struct io_async_ctx *io)
+{
+ struct iovec inline_vecs[UIO_FASTIOV], *iovec = inline_vecs;
+ struct iov_iter iter;
+ ssize_t ret;
+
+ memcpy(&io->sqe, req->sqe, sizeof(io->sqe));
+ req->sqe = &io->sqe;
+
+ switch (io->sqe.opcode) {
+ case IORING_OP_READV:
+ case IORING_OP_READ_FIXED:
+ ret = io_read_prep(req, &iovec, &iter, true);
+ break;
+ case IORING_OP_WRITEV:
+ case IORING_OP_WRITE_FIXED:
+ ret = io_write_prep(req, &iovec, &iter, true);
+ break;
+ case IORING_OP_SENDMSG:
+ ret = io_sendmsg_prep(req, io);
+ break;
+ case IORING_OP_RECVMSG:
+ ret = io_recvmsg_prep(req, io);
+ break;
+ case IORING_OP_CONNECT:
+ ret = io_connect_prep(req, io);
+ break;
+ case IORING_OP_TIMEOUT:
+ return io_timeout_prep(req, io, false);
+ case IORING_OP_LINK_TIMEOUT:
+ return io_timeout_prep(req, io, true);
+ default:
+ req->io = io;
+ return 0;
+ }
+
+ if (ret < 0)
+ return ret;
+
+ req->io = io;
+ io_req_map_io(req, ret, iovec, inline_vecs, &iter);
+ return 0;
+}
+
static int io_req_defer(struct io_kiocb *req)
{
- struct io_uring_sqe *sqe_copy;
struct io_ring_ctx *ctx = req->ctx;
+ struct io_async_ctx *io;
+ int ret;
/* Still need defer if there is pending req in defer list. */
if (!req_need_defer(req) && list_empty(&ctx->defer_list))
return 0;
- sqe_copy = kmalloc(sizeof(*sqe_copy), GFP_KERNEL);
- if (!sqe_copy)
+ io = kmalloc(sizeof(*io), GFP_KERNEL);
+ if (!io)
return -EAGAIN;
+ ret = io_req_defer_prep(req, io);
+ if (ret < 0) {
+ kfree(io);
+ return ret;
+ }
+
spin_lock_irq(&ctx->completion_lock);
if (!req_need_defer(req) && list_empty(&ctx->defer_list)) {
spin_unlock_irq(&ctx->completion_lock);
- kfree(sqe_copy);
return 0;
}
- memcpy(sqe_copy, req->sqe, sizeof(*sqe_copy));
- req->flags |= REQ_F_FREE_SQE;
- req->sqe = sqe_copy;
-
trace_io_uring_defer(ctx, req, req->user_data);
list_add_tail(&req->list, &ctx->defer_list);
spin_unlock_irq(&ctx->completion_lock);
* We don't expect the list to be empty, that will only happen if we
* race with the completion of the linked work.
*/
- if (!list_empty(&req->list)) {
- prev = list_entry(req->list.prev, struct io_kiocb, link_list);
+ if (!list_empty(&req->link_list)) {
+ prev = list_entry(req->link_list.prev, struct io_kiocb,
+ link_list);
if (refcount_inc_not_zero(&prev->refs)) {
- list_del_init(&req->list);
+ list_del_init(&req->link_list);
prev->flags &= ~REQ_F_LINK_TIMEOUT;
} else
prev = NULL;
* we got a chance to setup the timer
*/
spin_lock_irq(&ctx->completion_lock);
- if (!list_empty(&req->list)) {
- struct io_timeout_data *data = req->timeout.data;
+ if (!list_empty(&req->link_list)) {
+ struct io_timeout_data *data = &req->io->timeout;
data->timer.function = io_link_timeout_fn;
hrtimer_start(&data->timer, timespec64_to_ktime(data->ts),
if (!(req->flags & REQ_F_LINK))
return NULL;
- nxt = list_first_entry_or_null(&req->link_list, struct io_kiocb, list);
+ nxt = list_first_entry_or_null(&req->link_list, struct io_kiocb,
+ link_list);
if (!nxt || nxt->sqe->opcode != IORING_OP_LINK_TIMEOUT)
return NULL;
*/
if (ret == -EAGAIN && (!(req->flags & REQ_F_NOWAIT) ||
(req->flags & REQ_F_MUST_PUNT))) {
- struct io_uring_sqe *sqe_copy;
-
- sqe_copy = kmemdup(req->sqe, sizeof(*sqe_copy), GFP_KERNEL);
- if (!sqe_copy)
- goto err;
-
- req->sqe = sqe_copy;
- req->flags |= REQ_F_FREE_SQE;
-
if (req->work.flags & IO_WQ_WORK_NEEDS_FILES) {
ret = io_grab_files(req);
if (ret)
#define SQE_VALID_FLAGS (IOSQE_FIXED_FILE|IOSQE_IO_DRAIN|IOSQE_IO_LINK)
-static void io_submit_sqe(struct io_kiocb *req, struct io_submit_state *state,
+static bool io_submit_sqe(struct io_kiocb *req, struct io_submit_state *state,
struct io_kiocb **link)
{
struct io_ring_ctx *ctx = req->ctx;
err_req:
io_cqring_add_event(req, ret);
io_double_put_req(req);
- return;
+ return false;
}
/*
*/
if (*link) {
struct io_kiocb *prev = *link;
- struct io_uring_sqe *sqe_copy;
+ struct io_async_ctx *io;
if (req->sqe->flags & IOSQE_IO_DRAIN)
(*link)->flags |= REQ_F_DRAIN_LINK | REQ_F_IO_DRAIN;
- if (READ_ONCE(req->sqe->opcode) == IORING_OP_LINK_TIMEOUT) {
- ret = io_timeout_setup(req);
- /* common setup allows offset being set, we don't */
- if (!ret && req->sqe->off)
- ret = -EINVAL;
- if (ret) {
- prev->flags |= REQ_F_FAIL_LINK;
- goto err_req;
- }
- }
-
- sqe_copy = kmemdup(req->sqe, sizeof(*sqe_copy), GFP_KERNEL);
- if (!sqe_copy) {
+ io = kmalloc(sizeof(*io), GFP_KERNEL);
+ if (!io) {
ret = -EAGAIN;
goto err_req;
}
- req->sqe = sqe_copy;
- req->flags |= REQ_F_FREE_SQE;
+ ret = io_req_defer_prep(req, io);
+ if (ret) {
+ kfree(io);
+ prev->flags |= REQ_F_FAIL_LINK;
+ goto err_req;
+ }
trace_io_uring_link(ctx, req, prev);
- list_add_tail(&req->list, &prev->link_list);
+ list_add_tail(&req->link_list, &prev->link_list);
} else if (req->sqe->flags & IOSQE_IO_LINK) {
req->flags |= REQ_F_LINK;
} else {
io_queue_sqe(req);
}
+
+ return true;
}
/*
* Start submission side cache.
*/
static void io_submit_state_start(struct io_submit_state *state,
- struct io_ring_ctx *ctx, unsigned max_ios)
+ unsigned int max_ios)
{
blk_start_plug(&state->plug);
state->free_reqs = 0;
return -EBUSY;
if (nr > IO_PLUG_THRESHOLD) {
- io_submit_state_start(&state, ctx, nr);
+ io_submit_state_start(&state, nr);
statep = &state;
}
}
}
+ submitted++;
sqe_flags = req->sqe->flags;
req->ring_file = ring_file;
req->needs_fixed_file = async;
trace_io_uring_submit_sqe(ctx, req->sqe->user_data,
true, async);
- io_submit_sqe(req, statep, &link);
- submitted++;
-
+ if (!io_submit_sqe(req, statep, &link))
+ break;
/*
* If previous wasn't linked and we have a linked command,
* that's the end of the chain. Submit the previous link.
free_uid(ctx->user);
put_cred(ctx->creds);
kfree(ctx->completions);
+ kfree(ctx->cancel_hash);
kmem_cache_free(req_cachep, ctx->fallback_req);
kfree(ctx);
}
ctx->compat = in_compat_syscall();
ctx->account_mem = account_mem;
ctx->user = user;
- ctx->creds = prepare_creds();
+ ctx->creds = get_current_cred();
ret = io_allocate_scq_urings(ctx, p);
if (ret)
if (ret < 0)
goto err;
- p->features = IORING_FEAT_SINGLE_MMAP | IORING_FEAT_NODROP;
+ p->features = IORING_FEAT_SINGLE_MMAP | IORING_FEAT_NODROP |
+ IORING_FEAT_SUBMIT_STABLE;
trace_io_uring_create(ret, ctx, p->sq_entries, p->cq_entries, p->flags);
return ret;
err:
dput(dentry);
return ERR_PTR(-EINVAL);
}
- dtmp = lookup_one_len_unlocked(kntmp->name, dentry,
+ dtmp = lookup_positive_unlocked(kntmp->name, dentry,
strlen(kntmp->name));
dput(dentry);
if (IS_ERR(dtmp))
* - Flagged as automount point
*
* This may only be called in refwalk mode.
+ * On success path->dentry is known positive.
*
* Serialization is taken care of in namespace.c
*/
static int follow_managed(struct path *path, struct nameidata *nd)
{
struct vfsmount *mnt = path->mnt; /* held by caller, must be left alone */
- unsigned managed;
+ unsigned flags;
bool need_mntput = false;
int ret = 0;
/* Given that we're not holding a lock here, we retain the value in a
* local variable for each dentry as we look at it so that we don't see
* the components of that value change under us */
- while (managed = READ_ONCE(path->dentry->d_flags),
- managed &= DCACHE_MANAGED_DENTRY,
- unlikely(managed != 0)) {
+ while (flags = smp_load_acquire(&path->dentry->d_flags),
+ unlikely(flags & DCACHE_MANAGED_DENTRY)) {
/* Allow the filesystem to manage the transit without i_mutex
* being held. */
- if (managed & DCACHE_MANAGE_TRANSIT) {
+ if (flags & DCACHE_MANAGE_TRANSIT) {
BUG_ON(!path->dentry->d_op);
BUG_ON(!path->dentry->d_op->d_manage);
ret = path->dentry->d_op->d_manage(path, false);
}
/* Transit to a mounted filesystem. */
- if (managed & DCACHE_MOUNTED) {
+ if (flags & DCACHE_MOUNTED) {
struct vfsmount *mounted = lookup_mnt(path);
if (mounted) {
dput(path->dentry);
}
/* Handle an automount point */
- if (managed & DCACHE_NEED_AUTOMOUNT) {
+ if (flags & DCACHE_NEED_AUTOMOUNT) {
ret = follow_automount(path, nd, &need_mntput);
if (ret < 0)
break;
if (need_mntput && path->mnt == mnt)
mntput(path->mnt);
- if (ret == -EISDIR || !ret)
- ret = 1;
if (need_mntput)
nd->flags |= LOOKUP_JUMPED;
+ if (ret == -EISDIR || !ret)
+ ret = 1;
+ if (ret > 0 && unlikely(d_flags_negative(flags)))
+ ret = -ENOENT;
if (unlikely(ret < 0))
path_put_conditional(path, nd);
return ret;
dput(dentry);
return status;
}
- if (unlikely(d_is_negative(dentry))) {
- dput(dentry);
- return -ENOENT;
- }
path->mnt = mnt;
path->dentry = dentry;
if (unlikely(err < 0))
return err;
- if (unlikely(d_is_negative(path.dentry))) {
- path_to_nameidata(&path, nd);
- return -ENOENT;
- }
-
seq = 0; /* we are already out of RCU mode */
inode = d_backing_inode(path.dentry);
}
}
EXPORT_SYMBOL(lookup_one_len_unlocked);
+/*
+ * Like lookup_one_len_unlocked(), except that it yields ERR_PTR(-ENOENT)
+ * on negatives. Returns known positive or ERR_PTR(); that's what
+ * most of the users want. Note that pinned negative with unlocked parent
+ * _can_ become positive at any time, so callers of lookup_one_len_unlocked()
+ * need to be very careful; pinned positives have ->d_inode stable, so
+ * this one avoids such problems.
+ */
+struct dentry *lookup_positive_unlocked(const char *name,
+ struct dentry *base, int len)
+{
+ struct dentry *ret = lookup_one_len_unlocked(name, base, len);
+ if (!IS_ERR(ret) && d_flags_negative(smp_load_acquire(&ret->d_flags))) {
+ dput(ret);
+ ret = ERR_PTR(-ENOENT);
+ }
+ return ret;
+}
+EXPORT_SYMBOL(lookup_positive_unlocked);
+
#ifdef CONFIG_UNIX98_PTYS
int path_pts(struct path *path)
{
return PTR_ERR(path.dentry);
}
}
- if (d_is_negative(path.dentry)) {
+ if (d_flags_negative(smp_load_acquire(&path.dentry->d_flags))) {
dput(path.dentry);
return -ENOENT;
}
if (unlikely(error < 0))
return error;
- if (unlikely(d_is_negative(path.dentry))) {
- path_to_nameidata(&path, nd);
- return -ENOENT;
- }
-
/*
* create/update audit record if it already exists.
*/
} else
dchild = dget(dparent);
} else
- dchild = lookup_one_len_unlocked(name, dparent, namlen);
+ dchild = lookup_positive_unlocked(name, dparent, namlen);
if (IS_ERR(dchild))
return rv;
if (d_mountpoint(dchild))
goto out;
- if (d_really_is_negative(dchild))
- goto out;
if (dchild->d_inode->i_ino != ino)
goto out;
rv = fh_compose(fhp, exp, dchild, &cd->fh);
__be32 nfserr;
int ignore_crossmnt = 0;
- dentry = lookup_one_len_unlocked(name, cd->rd_fhp->fh_dentry, namlen);
+ dentry = lookup_positive_unlocked(name, cd->rd_fhp->fh_dentry, namlen);
if (IS_ERR(dentry))
return nfserrno(PTR_ERR(dentry));
- if (d_really_is_negative(dentry)) {
- /*
- * we're not holding the i_mutex here, so there's
- * a window where this directory entry could have gone
- * away.
- */
- dput(dentry);
- return nfserr_noent;
- }
exp_get(exp);
/*
int err;
bool last_element = !post[0];
- this = lookup_one_len_unlocked(name, base, namelen);
+ this = lookup_positive_unlocked(name, base, namelen);
if (IS_ERR(this)) {
err = PTR_ERR(this);
this = NULL;
goto out;
goto out_err;
}
- if (!this->d_inode)
- goto put_and_out;
if (ovl_dentry_weird(this)) {
/* Don't support traversing automounts and other weirdness */
if (err)
return ERR_PTR(err);
- index = lookup_one_len_unlocked(name.name, ofs->indexdir, name.len);
+ index = lookup_positive_unlocked(name.name, ofs->indexdir, name.len);
kfree(name.name);
if (IS_ERR(index)) {
if (PTR_ERR(index) == -ENOENT)
return index;
}
- if (d_is_negative(index))
- err = 0;
- else if (ovl_is_whiteout(index))
+ if (ovl_is_whiteout(index))
err = -ESTALE;
else if (ovl_dentry_weird(index))
err = -EIO;
if (err)
return ERR_PTR(err);
- index = lookup_one_len_unlocked(name.name, ofs->indexdir, name.len);
+ index = lookup_positive_unlocked(name.name, ofs->indexdir, name.len);
if (IS_ERR(index)) {
err = PTR_ERR(index);
if (err == -ENOENT) {
}
inode = d_inode(index);
- if (d_is_negative(index)) {
- goto out_dput;
- } else if (ovl_is_whiteout(index) && !verify) {
+ if (ovl_is_whiteout(index) && !verify) {
/*
* When index lookup is called with !verify for decoding an
* overlay file handle, a whiteout index implies that decode
struct dentry *this;
struct dentry *lowerdir = poe->lowerstack[i].dentry;
- this = lookup_one_len_unlocked(name->name, lowerdir,
+ this = lookup_positive_unlocked(name->name, lowerdir,
name->len);
if (IS_ERR(this)) {
switch (PTR_ERR(this)) {
break;
}
} else {
- if (this->d_inode) {
- positive = !ovl_is_whiteout(this);
- done = true;
- }
+ positive = !ovl_is_whiteout(this);
+ done = true;
dput(this);
}
}
{
struct file *filp = iocb->ki_filp;
struct pipe_inode_info *pipe = filp->private_data;
- unsigned int head, max_usage, mask;
+ unsigned int head;
ssize_t ret = 0;
int do_wakeup = 0;
size_t total_len = iov_iter_count(from);
}
head = pipe->head;
- max_usage = pipe->max_usage;
- mask = pipe->ring_size - 1;
/* We try to merge small writes */
chars = total_len & (PAGE_SIZE-1); /* size of the last buffer */
if (!pipe_empty(head, pipe->tail) && chars != 0) {
+ unsigned int mask = pipe->ring_size - 1;
struct pipe_buffer *buf = &pipe->bufs[(head - 1) & mask];
int offset = buf->offset + buf->len;
}
head = pipe->head;
- if (!pipe_full(head, pipe->tail, max_usage)) {
+ if (!pipe_full(head, pipe->tail, pipe->max_usage)) {
+ unsigned int mask = pipe->ring_size - 1;
struct pipe_buffer *buf = &pipe->bufs[head & mask];
struct page *page = pipe->tmp_page;
int copied;
spin_lock_irq(&pipe->wait.lock);
head = pipe->head;
- if (pipe_full(head, pipe->tail, max_usage)) {
+ if (pipe_full(head, pipe->tail, pipe->max_usage)) {
spin_unlock_irq(&pipe->wait.lock);
continue;
}
break;
}
- if (!pipe_full(head, pipe->tail, max_usage))
+ if (!pipe_full(head, pipe->tail, pipe->max_usage))
continue;
/* Wait for buffer space to become available. */
poll_wait(filp, &pipe->wait, wait);
- BUG_ON(pipe_occupancy(head, tail) > pipe->ring_size);
-
/* Reading only -- no need for acquiring the semaphore. */
mask = 0;
if (filp->f_mode & FMODE_READ) {
pipe->max_usage = nr_slots;
pipe->tail = tail;
pipe->head = head;
+ wake_up_interruptible_all(&pipe->wait);
return pipe->max_usage * PAGE_SIZE;
out_revert_acct:
struct dentry *dentry;
int error;
- dentry = lookup_one_len_unlocked(qf_name, sb->s_root, strlen(qf_name));
+ dentry = lookup_positive_unlocked(qf_name, sb->s_root, strlen(qf_name));
if (IS_ERR(dentry))
return PTR_ERR(dentry);
- if (d_really_is_negative(dentry)) {
- error = -ENOENT;
- goto out;
- }
-
error = security_quota_on(dentry);
if (!error)
error = dquot_load_quota_inode(d_inode(dentry), type, format_id,
DQUOT_USAGE_ENABLED | DQUOT_LIMITS_ENABLED);
-out:
dput(dentry);
return error;
}
unsigned int mask = pipe->ring_size - 1;
int ret;
- while (!pipe_empty(tail, head)) {
+ while (!pipe_empty(head, tail)) {
struct pipe_buffer *buf = &pipe->bufs[tail & mask];
sd->len = buf->len;
splice_from_pipe_begin(&sd);
while (sd.total_len) {
struct iov_iter from;
- unsigned int head = pipe->head;
- unsigned int tail = pipe->tail;
- unsigned int mask = pipe->ring_size - 1;
+ unsigned int head, tail, mask;
size_t left;
int n;
}
}
+ head = pipe->head;
+ tail = pipe->tail;
+ mask = pipe->ring_size - 1;
+
/* build the vector */
left = sd.total_len;
for (n = 0; !pipe_empty(head, tail) && left && n < nbufs; tail++, n++) {
+++ /dev/null
-/* SPDX-License-Identifier: GPL-2.0 */
-
-/*
- * This file provides wrappers with sanitizer instrumentation for bit
- * operations.
- *
- * To use this functionality, an arch's bitops.h file needs to define each of
- * the below bit operations with an arch_ prefix (e.g. arch_set_bit(),
- * arch___set_bit(), etc.).
- */
-#ifndef _ASM_GENERIC_BITOPS_INSTRUMENTED_H
-#define _ASM_GENERIC_BITOPS_INSTRUMENTED_H
-
-#include <linux/kasan-checks.h>
-
-/**
- * set_bit - Atomically set a bit in memory
- * @nr: the bit to set
- * @addr: the address to start counting from
- *
- * This is a relaxed atomic operation (no implied memory barriers).
- *
- * Note that @nr may be almost arbitrarily large; this function is not
- * restricted to acting on a single-word quantity.
- */
-static inline void set_bit(long nr, volatile unsigned long *addr)
-{
- kasan_check_write(addr + BIT_WORD(nr), sizeof(long));
- arch_set_bit(nr, addr);
-}
-
-/**
- * __set_bit - Set a bit in memory
- * @nr: the bit to set
- * @addr: the address to start counting from
- *
- * Unlike set_bit(), this function is non-atomic. If it is called on the same
- * region of memory concurrently, the effect may be that only one operation
- * succeeds.
- */
-static inline void __set_bit(long nr, volatile unsigned long *addr)
-{
- kasan_check_write(addr + BIT_WORD(nr), sizeof(long));
- arch___set_bit(nr, addr);
-}
-
-/**
- * clear_bit - Clears a bit in memory
- * @nr: Bit to clear
- * @addr: Address to start counting from
- *
- * This is a relaxed atomic operation (no implied memory barriers).
- */
-static inline void clear_bit(long nr, volatile unsigned long *addr)
-{
- kasan_check_write(addr + BIT_WORD(nr), sizeof(long));
- arch_clear_bit(nr, addr);
-}
-
-/**
- * __clear_bit - Clears a bit in memory
- * @nr: the bit to clear
- * @addr: the address to start counting from
- *
- * Unlike clear_bit(), this function is non-atomic. If it is called on the same
- * region of memory concurrently, the effect may be that only one operation
- * succeeds.
- */
-static inline void __clear_bit(long nr, volatile unsigned long *addr)
-{
- kasan_check_write(addr + BIT_WORD(nr), sizeof(long));
- arch___clear_bit(nr, addr);
-}
-
-/**
- * clear_bit_unlock - Clear a bit in memory, for unlock
- * @nr: the bit to set
- * @addr: the address to start counting from
- *
- * This operation is atomic and provides release barrier semantics.
- */
-static inline void clear_bit_unlock(long nr, volatile unsigned long *addr)
-{
- kasan_check_write(addr + BIT_WORD(nr), sizeof(long));
- arch_clear_bit_unlock(nr, addr);
-}
-
-/**
- * __clear_bit_unlock - Clears a bit in memory
- * @nr: Bit to clear
- * @addr: Address to start counting from
- *
- * This is a non-atomic operation but implies a release barrier before the
- * memory operation. It can be used for an unlock if no other CPUs can
- * concurrently modify other bits in the word.
- */
-static inline void __clear_bit_unlock(long nr, volatile unsigned long *addr)
-{
- kasan_check_write(addr + BIT_WORD(nr), sizeof(long));
- arch___clear_bit_unlock(nr, addr);
-}
-
-/**
- * change_bit - Toggle a bit in memory
- * @nr: Bit to change
- * @addr: Address to start counting from
- *
- * This is a relaxed atomic operation (no implied memory barriers).
- *
- * Note that @nr may be almost arbitrarily large; this function is not
- * restricted to acting on a single-word quantity.
- */
-static inline void change_bit(long nr, volatile unsigned long *addr)
-{
- kasan_check_write(addr + BIT_WORD(nr), sizeof(long));
- arch_change_bit(nr, addr);
-}
-
-/**
- * __change_bit - Toggle a bit in memory
- * @nr: the bit to change
- * @addr: the address to start counting from
- *
- * Unlike change_bit(), this function is non-atomic. If it is called on the same
- * region of memory concurrently, the effect may be that only one operation
- * succeeds.
- */
-static inline void __change_bit(long nr, volatile unsigned long *addr)
-{
- kasan_check_write(addr + BIT_WORD(nr), sizeof(long));
- arch___change_bit(nr, addr);
-}
-
-/**
- * test_and_set_bit - Set a bit and return its old value
- * @nr: Bit to set
- * @addr: Address to count from
- *
- * This is an atomic fully-ordered operation (implied full memory barrier).
- */
-static inline bool test_and_set_bit(long nr, volatile unsigned long *addr)
-{
- kasan_check_write(addr + BIT_WORD(nr), sizeof(long));
- return arch_test_and_set_bit(nr, addr);
-}
-
-/**
- * __test_and_set_bit - Set a bit and return its old value
- * @nr: Bit to set
- * @addr: Address to count from
- *
- * This operation is non-atomic. If two instances of this operation race, one
- * can appear to succeed but actually fail.
- */
-static inline bool __test_and_set_bit(long nr, volatile unsigned long *addr)
-{
- kasan_check_write(addr + BIT_WORD(nr), sizeof(long));
- return arch___test_and_set_bit(nr, addr);
-}
-
-/**
- * test_and_set_bit_lock - Set a bit and return its old value, for lock
- * @nr: Bit to set
- * @addr: Address to count from
- *
- * This operation is atomic and provides acquire barrier semantics if
- * the returned value is 0.
- * It can be used to implement bit locks.
- */
-static inline bool test_and_set_bit_lock(long nr, volatile unsigned long *addr)
-{
- kasan_check_write(addr + BIT_WORD(nr), sizeof(long));
- return arch_test_and_set_bit_lock(nr, addr);
-}
-
-/**
- * test_and_clear_bit - Clear a bit and return its old value
- * @nr: Bit to clear
- * @addr: Address to count from
- *
- * This is an atomic fully-ordered operation (implied full memory barrier).
- */
-static inline bool test_and_clear_bit(long nr, volatile unsigned long *addr)
-{
- kasan_check_write(addr + BIT_WORD(nr), sizeof(long));
- return arch_test_and_clear_bit(nr, addr);
-}
-
-/**
- * __test_and_clear_bit - Clear a bit and return its old value
- * @nr: Bit to clear
- * @addr: Address to count from
- *
- * This operation is non-atomic. If two instances of this operation race, one
- * can appear to succeed but actually fail.
- */
-static inline bool __test_and_clear_bit(long nr, volatile unsigned long *addr)
-{
- kasan_check_write(addr + BIT_WORD(nr), sizeof(long));
- return arch___test_and_clear_bit(nr, addr);
-}
-
-/**
- * test_and_change_bit - Change a bit and return its old value
- * @nr: Bit to change
- * @addr: Address to count from
- *
- * This is an atomic fully-ordered operation (implied full memory barrier).
- */
-static inline bool test_and_change_bit(long nr, volatile unsigned long *addr)
-{
- kasan_check_write(addr + BIT_WORD(nr), sizeof(long));
- return arch_test_and_change_bit(nr, addr);
-}
-
-/**
- * __test_and_change_bit - Change a bit and return its old value
- * @nr: Bit to change
- * @addr: Address to count from
- *
- * This operation is non-atomic. If two instances of this operation race, one
- * can appear to succeed but actually fail.
- */
-static inline bool __test_and_change_bit(long nr, volatile unsigned long *addr)
-{
- kasan_check_write(addr + BIT_WORD(nr), sizeof(long));
- return arch___test_and_change_bit(nr, addr);
-}
-
-/**
- * test_bit - Determine whether a bit is set
- * @nr: bit number to test
- * @addr: Address to start counting from
- */
-static inline bool test_bit(long nr, const volatile unsigned long *addr)
-{
- kasan_check_read(addr + BIT_WORD(nr), sizeof(long));
- return arch_test_bit(nr, addr);
-}
-
-#if defined(arch_clear_bit_unlock_is_negative_byte)
-/**
- * clear_bit_unlock_is_negative_byte - Clear a bit in memory and test if bottom
- * byte is negative, for unlock.
- * @nr: the bit to clear
- * @addr: the address to start counting from
- *
- * This operation is atomic and provides release barrier semantics.
- *
- * This is a bit of a one-trick-pony for the filemap code, which clears
- * PG_locked and tests PG_waiters,
- */
-static inline bool
-clear_bit_unlock_is_negative_byte(long nr, volatile unsigned long *addr)
-{
- kasan_check_write(addr + BIT_WORD(nr), sizeof(long));
- return arch_clear_bit_unlock_is_negative_byte(nr, addr);
-}
-/* Let everybody know we have it. */
-#define clear_bit_unlock_is_negative_byte clear_bit_unlock_is_negative_byte
-#endif
-
-#endif /* _ASM_GENERIC_BITOPS_INSTRUMENTED_H */
--- /dev/null
+/* SPDX-License-Identifier: GPL-2.0 */
+
+/*
+ * This file provides wrappers with sanitizer instrumentation for atomic bit
+ * operations.
+ *
+ * To use this functionality, an arch's bitops.h file needs to define each of
+ * the below bit operations with an arch_ prefix (e.g. arch_set_bit(),
+ * arch___set_bit(), etc.).
+ */
+#ifndef _ASM_GENERIC_BITOPS_INSTRUMENTED_ATOMIC_H
+#define _ASM_GENERIC_BITOPS_INSTRUMENTED_ATOMIC_H
+
+#include <linux/kasan-checks.h>
+
+/**
+ * set_bit - Atomically set a bit in memory
+ * @nr: the bit to set
+ * @addr: the address to start counting from
+ *
+ * This is a relaxed atomic operation (no implied memory barriers).
+ *
+ * Note that @nr may be almost arbitrarily large; this function is not
+ * restricted to acting on a single-word quantity.
+ */
+static inline void set_bit(long nr, volatile unsigned long *addr)
+{
+ kasan_check_write(addr + BIT_WORD(nr), sizeof(long));
+ arch_set_bit(nr, addr);
+}
+
+/**
+ * clear_bit - Clears a bit in memory
+ * @nr: Bit to clear
+ * @addr: Address to start counting from
+ *
+ * This is a relaxed atomic operation (no implied memory barriers).
+ */
+static inline void clear_bit(long nr, volatile unsigned long *addr)
+{
+ kasan_check_write(addr + BIT_WORD(nr), sizeof(long));
+ arch_clear_bit(nr, addr);
+}
+
+/**
+ * change_bit - Toggle a bit in memory
+ * @nr: Bit to change
+ * @addr: Address to start counting from
+ *
+ * This is a relaxed atomic operation (no implied memory barriers).
+ *
+ * Note that @nr may be almost arbitrarily large; this function is not
+ * restricted to acting on a single-word quantity.
+ */
+static inline void change_bit(long nr, volatile unsigned long *addr)
+{
+ kasan_check_write(addr + BIT_WORD(nr), sizeof(long));
+ arch_change_bit(nr, addr);
+}
+
+/**
+ * test_and_set_bit - Set a bit and return its old value
+ * @nr: Bit to set
+ * @addr: Address to count from
+ *
+ * This is an atomic fully-ordered operation (implied full memory barrier).
+ */
+static inline bool test_and_set_bit(long nr, volatile unsigned long *addr)
+{
+ kasan_check_write(addr + BIT_WORD(nr), sizeof(long));
+ return arch_test_and_set_bit(nr, addr);
+}
+
+/**
+ * test_and_clear_bit - Clear a bit and return its old value
+ * @nr: Bit to clear
+ * @addr: Address to count from
+ *
+ * This is an atomic fully-ordered operation (implied full memory barrier).
+ */
+static inline bool test_and_clear_bit(long nr, volatile unsigned long *addr)
+{
+ kasan_check_write(addr + BIT_WORD(nr), sizeof(long));
+ return arch_test_and_clear_bit(nr, addr);
+}
+
+/**
+ * test_and_change_bit - Change a bit and return its old value
+ * @nr: Bit to change
+ * @addr: Address to count from
+ *
+ * This is an atomic fully-ordered operation (implied full memory barrier).
+ */
+static inline bool test_and_change_bit(long nr, volatile unsigned long *addr)
+{
+ kasan_check_write(addr + BIT_WORD(nr), sizeof(long));
+ return arch_test_and_change_bit(nr, addr);
+}
+
+#endif /* _ASM_GENERIC_BITOPS_INSTRUMENTED_NON_ATOMIC_H */
--- /dev/null
+/* SPDX-License-Identifier: GPL-2.0 */
+
+/*
+ * This file provides wrappers with sanitizer instrumentation for bit
+ * locking operations.
+ *
+ * To use this functionality, an arch's bitops.h file needs to define each of
+ * the below bit operations with an arch_ prefix (e.g. arch_set_bit(),
+ * arch___set_bit(), etc.).
+ */
+#ifndef _ASM_GENERIC_BITOPS_INSTRUMENTED_LOCK_H
+#define _ASM_GENERIC_BITOPS_INSTRUMENTED_LOCK_H
+
+#include <linux/kasan-checks.h>
+
+/**
+ * clear_bit_unlock - Clear a bit in memory, for unlock
+ * @nr: the bit to set
+ * @addr: the address to start counting from
+ *
+ * This operation is atomic and provides release barrier semantics.
+ */
+static inline void clear_bit_unlock(long nr, volatile unsigned long *addr)
+{
+ kasan_check_write(addr + BIT_WORD(nr), sizeof(long));
+ arch_clear_bit_unlock(nr, addr);
+}
+
+/**
+ * __clear_bit_unlock - Clears a bit in memory
+ * @nr: Bit to clear
+ * @addr: Address to start counting from
+ *
+ * This is a non-atomic operation but implies a release barrier before the
+ * memory operation. It can be used for an unlock if no other CPUs can
+ * concurrently modify other bits in the word.
+ */
+static inline void __clear_bit_unlock(long nr, volatile unsigned long *addr)
+{
+ kasan_check_write(addr + BIT_WORD(nr), sizeof(long));
+ arch___clear_bit_unlock(nr, addr);
+}
+
+/**
+ * test_and_set_bit_lock - Set a bit and return its old value, for lock
+ * @nr: Bit to set
+ * @addr: Address to count from
+ *
+ * This operation is atomic and provides acquire barrier semantics if
+ * the returned value is 0.
+ * It can be used to implement bit locks.
+ */
+static inline bool test_and_set_bit_lock(long nr, volatile unsigned long *addr)
+{
+ kasan_check_write(addr + BIT_WORD(nr), sizeof(long));
+ return arch_test_and_set_bit_lock(nr, addr);
+}
+
+#if defined(arch_clear_bit_unlock_is_negative_byte)
+/**
+ * clear_bit_unlock_is_negative_byte - Clear a bit in memory and test if bottom
+ * byte is negative, for unlock.
+ * @nr: the bit to clear
+ * @addr: the address to start counting from
+ *
+ * This operation is atomic and provides release barrier semantics.
+ *
+ * This is a bit of a one-trick-pony for the filemap code, which clears
+ * PG_locked and tests PG_waiters,
+ */
+static inline bool
+clear_bit_unlock_is_negative_byte(long nr, volatile unsigned long *addr)
+{
+ kasan_check_write(addr + BIT_WORD(nr), sizeof(long));
+ return arch_clear_bit_unlock_is_negative_byte(nr, addr);
+}
+/* Let everybody know we have it. */
+#define clear_bit_unlock_is_negative_byte clear_bit_unlock_is_negative_byte
+#endif
+
+#endif /* _ASM_GENERIC_BITOPS_INSTRUMENTED_LOCK_H */
--- /dev/null
+/* SPDX-License-Identifier: GPL-2.0 */
+
+/*
+ * This file provides wrappers with sanitizer instrumentation for non-atomic
+ * bit operations.
+ *
+ * To use this functionality, an arch's bitops.h file needs to define each of
+ * the below bit operations with an arch_ prefix (e.g. arch_set_bit(),
+ * arch___set_bit(), etc.).
+ */
+#ifndef _ASM_GENERIC_BITOPS_INSTRUMENTED_NON_ATOMIC_H
+#define _ASM_GENERIC_BITOPS_INSTRUMENTED_NON_ATOMIC_H
+
+#include <linux/kasan-checks.h>
+
+/**
+ * __set_bit - Set a bit in memory
+ * @nr: the bit to set
+ * @addr: the address to start counting from
+ *
+ * Unlike set_bit(), this function is non-atomic. If it is called on the same
+ * region of memory concurrently, the effect may be that only one operation
+ * succeeds.
+ */
+static inline void __set_bit(long nr, volatile unsigned long *addr)
+{
+ kasan_check_write(addr + BIT_WORD(nr), sizeof(long));
+ arch___set_bit(nr, addr);
+}
+
+/**
+ * __clear_bit - Clears a bit in memory
+ * @nr: the bit to clear
+ * @addr: the address to start counting from
+ *
+ * Unlike clear_bit(), this function is non-atomic. If it is called on the same
+ * region of memory concurrently, the effect may be that only one operation
+ * succeeds.
+ */
+static inline void __clear_bit(long nr, volatile unsigned long *addr)
+{
+ kasan_check_write(addr + BIT_WORD(nr), sizeof(long));
+ arch___clear_bit(nr, addr);
+}
+
+/**
+ * __change_bit - Toggle a bit in memory
+ * @nr: the bit to change
+ * @addr: the address to start counting from
+ *
+ * Unlike change_bit(), this function is non-atomic. If it is called on the same
+ * region of memory concurrently, the effect may be that only one operation
+ * succeeds.
+ */
+static inline void __change_bit(long nr, volatile unsigned long *addr)
+{
+ kasan_check_write(addr + BIT_WORD(nr), sizeof(long));
+ arch___change_bit(nr, addr);
+}
+
+/**
+ * __test_and_set_bit - Set a bit and return its old value
+ * @nr: Bit to set
+ * @addr: Address to count from
+ *
+ * This operation is non-atomic. If two instances of this operation race, one
+ * can appear to succeed but actually fail.
+ */
+static inline bool __test_and_set_bit(long nr, volatile unsigned long *addr)
+{
+ kasan_check_write(addr + BIT_WORD(nr), sizeof(long));
+ return arch___test_and_set_bit(nr, addr);
+}
+
+/**
+ * __test_and_clear_bit - Clear a bit and return its old value
+ * @nr: Bit to clear
+ * @addr: Address to count from
+ *
+ * This operation is non-atomic. If two instances of this operation race, one
+ * can appear to succeed but actually fail.
+ */
+static inline bool __test_and_clear_bit(long nr, volatile unsigned long *addr)
+{
+ kasan_check_write(addr + BIT_WORD(nr), sizeof(long));
+ return arch___test_and_clear_bit(nr, addr);
+}
+
+/**
+ * __test_and_change_bit - Change a bit and return its old value
+ * @nr: Bit to change
+ * @addr: Address to count from
+ *
+ * This operation is non-atomic. If two instances of this operation race, one
+ * can appear to succeed but actually fail.
+ */
+static inline bool __test_and_change_bit(long nr, volatile unsigned long *addr)
+{
+ kasan_check_write(addr + BIT_WORD(nr), sizeof(long));
+ return arch___test_and_change_bit(nr, addr);
+}
+
+/**
+ * test_bit - Determine whether a bit is set
+ * @nr: bit number to test
+ * @addr: Address to start counting from
+ */
+static inline bool test_bit(long nr, const volatile unsigned long *addr)
+{
+ kasan_check_read(addr + BIT_WORD(nr), sizeof(long));
+ return arch_test_bit(nr, addr);
+}
+
+#endif /* _ASM_GENERIC_BITOPS_INSTRUMENTED_NON_ATOMIC_H */
#include <linux/agp_backend.h>
#include <uapi/linux/agpgart.h>
-#define AGPGART_MINOR 175
-
struct agp_info {
struct agp_version version; /* version of the driver */
u32 bridge_id; /* bridge vendor/device */
#define BLK_ALL_ZONES ((unsigned int)-1)
int blkdev_report_zones(struct block_device *bdev, sector_t sector,
unsigned int nr_zones, report_zones_cb cb, void *data);
-
-extern unsigned int blkdev_nr_zones(struct block_device *bdev);
+unsigned int blkdev_nr_zones(struct gendisk *disk);
extern int blkdev_zone_mgmt(struct block_device *bdev, enum req_opf op,
sector_t sectors, sector_t nr_sectors,
gfp_t gfp_mask);
#else /* CONFIG_BLK_DEV_ZONED */
-static inline unsigned int blkdev_nr_zones(struct block_device *bdev)
-{
- return 0;
-}
-
-static inline int blk_revalidate_disk_zones(struct gendisk *disk)
+static inline unsigned int blkdev_nr_zones(struct gendisk *disk)
{
return 0;
}
/*
* Zoned block device information for request dispatch control.
* nr_zones is the total number of zones of the device. This is always
- * 0 for regular block devices. seq_zones_bitmap is a bitmap of nr_zones
- * bits which indicates if a zone is conventional (bit clear) or
- * sequential (bit set). seq_zones_wlock is a bitmap of nr_zones
+ * 0 for regular block devices. conv_zones_bitmap is a bitmap of nr_zones
+ * bits which indicates if a zone is conventional (bit set) or
+ * sequential (bit clear). seq_zones_wlock is a bitmap of nr_zones
* bits which indicates if a zone is write locked, that is, if a write
* request targeting the zone was dispatched. All three fields are
* initialized by the low level device driver (e.g. scsi/sd.c).
* blk_mq_unfreeze_queue().
*/
unsigned int nr_zones;
- unsigned long *seq_zones_bitmap;
+ unsigned long *conv_zones_bitmap;
unsigned long *seq_zones_wlock;
#endif /* CONFIG_BLK_DEV_ZONED */
static inline bool blk_queue_zone_is_seq(struct request_queue *q,
sector_t sector)
{
- if (!blk_queue_is_zoned(q) || !q->seq_zones_bitmap)
+ if (!blk_queue_is_zoned(q))
return false;
- return test_bit(blk_queue_zone_no(q, sector), q->seq_zones_bitmap);
+ if (!q->conv_zones_bitmap)
+ return true;
+ return !test_bit(blk_queue_zone_no(q, sector), q->conv_zones_bitmap);
}
#else /* CONFIG_BLK_DEV_ZONED */
static inline unsigned int blk_queue_nr_zones(struct request_queue *q)
static inline bool bvec_iter_advance(const struct bio_vec *bv,
struct bvec_iter *iter, unsigned bytes)
{
+ unsigned int idx = iter->bi_idx;
+
if (WARN_ONCE(bytes > iter->bi_size,
"Attempted to advance past end of bvec iter\n")) {
iter->bi_size = 0;
return false;
}
- while (bytes) {
- const struct bio_vec *cur = bv + iter->bi_idx;
- unsigned len = min3(bytes, iter->bi_size,
- cur->bv_len - iter->bi_bvec_done);
-
- bytes -= len;
- iter->bi_size -= len;
- iter->bi_bvec_done += len;
+ iter->bi_size -= bytes;
+ bytes += iter->bi_bvec_done;
- if (iter->bi_bvec_done == cur->bv_len) {
- iter->bi_bvec_done = 0;
- iter->bi_idx++;
- }
+ while (bytes && bytes >= bv[idx].bv_len) {
+ bytes -= bv[idx].bv_len;
+ idx++;
}
+
+ iter->bi_idx = idx;
+ iter->bi_bvec_done = bytes;
return true;
}
extern int ceph_check_fsid(struct ceph_client *client, struct ceph_fsid *fsid);
extern void *ceph_kvmalloc(size_t size, gfp_t flags);
-extern struct ceph_options *ceph_parse_options(char *options,
- const char *dev_name, const char *dev_name_end,
- int (*parse_extra_token)(char *c, void *private),
- void *private);
+struct fs_parameter;
+struct ceph_options *ceph_alloc_options(void);
+int ceph_parse_mon_ips(const char *buf, size_t len, struct ceph_options *opt,
+ struct fs_context *fc);
+int ceph_parse_param(struct fs_parameter *param, struct ceph_options *opt,
+ struct fs_context *fc);
int ceph_print_client_options(struct seq_file *m, struct ceph_client *client,
bool show_all);
extern void ceph_destroy_options(struct ceph_options *opt);
return d_is_miss(dentry);
}
+static inline bool d_flags_negative(unsigned flags)
+{
+ return (flags & DCACHE_ENTRY_TYPE) == DCACHE_MISS_TYPE;
+}
+
static inline bool d_is_positive(const struct dentry *dentry)
{
return !d_is_negative(dentry);
* absolute relocations that require runtime processing on relocatable
* kernels.
*/
-#define __KSYMTAB_ENTRY_NS(sym, sec) \
+#define __KSYMTAB_ENTRY(sym, sec) \
__ADDRESSABLE(sym) \
asm(" .section \"___ksymtab" sec "+" #sym "\", \"a\" \n" \
" .balign 4 \n" \
" .long __kstrtabns_" #sym "- . \n" \
" .previous \n")
-#define __KSYMTAB_ENTRY(sym, sec) \
- __ADDRESSABLE(sym) \
- asm(" .section \"___ksymtab" sec "+" #sym "\", \"a\" \n" \
- " .balign 4 \n" \
- "__ksymtab_" #sym ": \n" \
- " .long " #sym "- . \n" \
- " .long __kstrtab_" #sym "- . \n" \
- " .long 0 \n" \
- " .previous \n")
-
struct kernel_symbol {
int value_offset;
int name_offset;
int namespace_offset;
};
#else
-#define __KSYMTAB_ENTRY_NS(sym, sec) \
- static const struct kernel_symbol __ksymtab_##sym \
- __attribute__((section("___ksymtab" sec "+" #sym), used)) \
- __aligned(sizeof(void *)) \
- = { (unsigned long)&sym, __kstrtab_##sym, __kstrtabns_##sym }
-
#define __KSYMTAB_ENTRY(sym, sec) \
static const struct kernel_symbol __ksymtab_##sym \
__attribute__((section("___ksymtab" sec "+" #sym), used)) \
__aligned(sizeof(void *)) \
- = { (unsigned long)&sym, __kstrtab_##sym, NULL }
+ = { (unsigned long)&sym, __kstrtab_##sym, __kstrtabns_##sym }
struct kernel_symbol {
unsigned long value;
#ifdef __GENKSYMS__
-#define ___EXPORT_SYMBOL(sym,sec) __GENKSYMS_EXPORT_SYMBOL(sym)
-#define ___EXPORT_SYMBOL_NS(sym,sec,ns) __GENKSYMS_EXPORT_SYMBOL(sym)
+#define ___EXPORT_SYMBOL(sym, sec, ns) __GENKSYMS_EXPORT_SYMBOL(sym)
#else
-#define ___export_symbol_common(sym, sec) \
+/* For every exported symbol, place a struct in the __ksymtab section */
+#define ___EXPORT_SYMBOL(sym, sec, ns) \
extern typeof(sym) sym; \
__CRC_SYMBOL(sym, sec); \
static const char __kstrtab_##sym[] \
__attribute__((section("__ksymtab_strings"), used, aligned(1))) \
- = #sym \
-
-/* For every exported symbol, place a struct in the __ksymtab section */
-#define ___EXPORT_SYMBOL_NS(sym, sec, ns) \
- ___export_symbol_common(sym, sec); \
+ = #sym; \
static const char __kstrtabns_##sym[] \
__attribute__((section("__ksymtab_strings"), used, aligned(1))) \
- = #ns; \
- __KSYMTAB_ENTRY_NS(sym, sec)
-
-#define ___EXPORT_SYMBOL(sym, sec) \
- ___export_symbol_common(sym, sec); \
+ = ns; \
__KSYMTAB_ENTRY(sym, sec)
#endif
* be reused in other execution contexts such as the UEFI stub or the
* decompressor.
*/
-#define __EXPORT_SYMBOL_NS(sym, sec, ns)
-#define __EXPORT_SYMBOL(sym, sec)
+#define __EXPORT_SYMBOL(sym, sec, ns)
#elif defined(CONFIG_TRIM_UNUSED_KSYMS)
#define __ksym_marker(sym) \
static int __ksym_marker_##sym[0] __section(".discard.ksym") __used
-#define __EXPORT_SYMBOL(sym, sec) \
- __ksym_marker(sym); \
- __cond_export_sym(sym, sec, __is_defined(__KSYM_##sym))
-#define __cond_export_sym(sym, sec, conf) \
- ___cond_export_sym(sym, sec, conf)
-#define ___cond_export_sym(sym, sec, enabled) \
- __cond_export_sym_##enabled(sym, sec)
-#define __cond_export_sym_1(sym, sec) ___EXPORT_SYMBOL(sym, sec)
-#define __cond_export_sym_0(sym, sec) /* nothing */
-
-#define __EXPORT_SYMBOL_NS(sym, sec, ns) \
+#define __EXPORT_SYMBOL(sym, sec, ns) \
__ksym_marker(sym); \
- __cond_export_ns_sym(sym, sec, ns, __is_defined(__KSYM_##sym))
-#define __cond_export_ns_sym(sym, sec, ns, conf) \
- ___cond_export_ns_sym(sym, sec, ns, conf)
-#define ___cond_export_ns_sym(sym, sec, ns, enabled) \
- __cond_export_ns_sym_##enabled(sym, sec, ns)
-#define __cond_export_ns_sym_1(sym, sec, ns) ___EXPORT_SYMBOL_NS(sym, sec, ns)
-#define __cond_export_ns_sym_0(sym, sec, ns) /* nothing */
+ __cond_export_sym(sym, sec, ns, __is_defined(__KSYM_##sym))
+#define __cond_export_sym(sym, sec, ns, conf) \
+ ___cond_export_sym(sym, sec, ns, conf)
+#define ___cond_export_sym(sym, sec, ns, enabled) \
+ __cond_export_sym_##enabled(sym, sec, ns)
+#define __cond_export_sym_1(sym, sec, ns) ___EXPORT_SYMBOL(sym, sec, ns)
+#define __cond_export_sym_0(sym, sec, ns) /* nothing */
#else
-#define __EXPORT_SYMBOL_NS(sym,sec,ns) ___EXPORT_SYMBOL_NS(sym,sec,ns)
-#define __EXPORT_SYMBOL(sym,sec) ___EXPORT_SYMBOL(sym,sec)
+#define __EXPORT_SYMBOL(sym, sec, ns) ___EXPORT_SYMBOL(sym, sec, ns)
#endif /* CONFIG_MODULES */
#ifdef DEFAULT_SYMBOL_NAMESPACE
-#undef __EXPORT_SYMBOL
-#define __EXPORT_SYMBOL(sym, sec) \
- __EXPORT_SYMBOL_NS(sym, sec, DEFAULT_SYMBOL_NAMESPACE)
+#include <linux/stringify.h>
+#define _EXPORT_SYMBOL(sym, sec) __EXPORT_SYMBOL(sym, sec, __stringify(DEFAULT_SYMBOL_NAMESPACE))
+#else
+#define _EXPORT_SYMBOL(sym, sec) __EXPORT_SYMBOL(sym, sec, "")
#endif
-#define EXPORT_SYMBOL(sym) __EXPORT_SYMBOL(sym, "")
-#define EXPORT_SYMBOL_GPL(sym) __EXPORT_SYMBOL(sym, "_gpl")
-#define EXPORT_SYMBOL_GPL_FUTURE(sym) __EXPORT_SYMBOL(sym, "_gpl_future")
-#define EXPORT_SYMBOL_NS(sym, ns) __EXPORT_SYMBOL_NS(sym, "", ns)
-#define EXPORT_SYMBOL_NS_GPL(sym, ns) __EXPORT_SYMBOL_NS(sym, "_gpl", ns)
+#define EXPORT_SYMBOL(sym) _EXPORT_SYMBOL(sym, "")
+#define EXPORT_SYMBOL_GPL(sym) _EXPORT_SYMBOL(sym, "_gpl")
+#define EXPORT_SYMBOL_GPL_FUTURE(sym) _EXPORT_SYMBOL(sym, "_gpl_future")
+#define EXPORT_SYMBOL_NS(sym, ns) __EXPORT_SYMBOL(sym, "", #ns)
+#define EXPORT_SYMBOL_NS_GPL(sym, ns) __EXPORT_SYMBOL(sym, "_gpl", #ns)
#ifdef CONFIG_UNUSED_SYMBOLS
-#define EXPORT_UNUSED_SYMBOL(sym) __EXPORT_SYMBOL(sym, "_unused")
-#define EXPORT_UNUSED_SYMBOL_GPL(sym) __EXPORT_SYMBOL(sym, "_unused_gpl")
+#define EXPORT_UNUSED_SYMBOL(sym) _EXPORT_SYMBOL(sym, "_unused")
+#define EXPORT_UNUSED_SYMBOL_GPL(sym) _EXPORT_SYMBOL(sym, "_unused_gpl")
#else
#define EXPORT_UNUSED_SYMBOL(sym)
#define EXPORT_UNUSED_SYMBOL_GPL(sym)
#define SGI_MMTIMER 153
#define STORE_QUEUE_MINOR 155 /* unused */
#define I2O_MINOR 166
+#define AGPGART_MINOR 175
#define HWRNG_MINOR 183
#define MICROCODE_MINOR 184
#define IRNET_MINOR 187
/**
* module_param - typesafe helper for a module/cmdline parameter
- * @value: the variable to alter, and exposed parameter name.
+ * @name: the variable to alter, and exposed parameter name.
* @type: the type of the parameter
* @perm: visibility in sysfs.
*
- * @value becomes the module parameter, or (prefixed by KBUILD_MODNAME and a
+ * @name becomes the module parameter, or (prefixed by KBUILD_MODNAME and a
* ".") the kernel commandline parameter. Note that - is changed to _, so
* the user can use "foo-bar=1" even for variable "foo_bar".
*
extern struct dentry *try_lookup_one_len(const char *, struct dentry *, int);
extern struct dentry *lookup_one_len(const char *, struct dentry *, int);
extern struct dentry *lookup_one_len_unlocked(const char *, struct dentry *, int);
+extern struct dentry *lookup_positive_unlocked(const char *, struct dentry *, int);
extern int follow_down_one(struct path *);
extern int follow_down(struct path *);
int perm;
};
+enum qcom_scm_ocmem_client {
+ QCOM_SCM_OCMEM_UNUSED_ID = 0x0,
+ QCOM_SCM_OCMEM_GRAPHICS_ID,
+ QCOM_SCM_OCMEM_VIDEO_ID,
+ QCOM_SCM_OCMEM_LP_AUDIO_ID,
+ QCOM_SCM_OCMEM_SENSORS_ID,
+ QCOM_SCM_OCMEM_OTHER_OS_ID,
+ QCOM_SCM_OCMEM_DEBUG_ID,
+};
+
+enum qcom_scm_sec_dev_id {
+ QCOM_SCM_MDSS_DEV_ID = 1,
+ QCOM_SCM_OCMEM_DEV_ID = 5,
+ QCOM_SCM_PCIE0_DEV_ID = 11,
+ QCOM_SCM_PCIE1_DEV_ID = 12,
+ QCOM_SCM_GFX_DEV_ID = 18,
+ QCOM_SCM_UFS_DEV_ID = 19,
+ QCOM_SCM_ICE_DEV_ID = 20,
+};
+
#define QCOM_SCM_VMID_HLOS 0x3
#define QCOM_SCM_VMID_MSS_MSA 0xF
#define QCOM_SCM_VMID_WLAN 0x18
extern bool qcom_scm_hdcp_available(void);
extern int qcom_scm_hdcp_req(struct qcom_scm_hdcp_req *req, u32 req_cnt,
u32 *resp);
+extern bool qcom_scm_ocmem_lock_available(void);
+extern int qcom_scm_ocmem_lock(enum qcom_scm_ocmem_client id, u32 offset,
+ u32 size, u32 mode);
+extern int qcom_scm_ocmem_unlock(enum qcom_scm_ocmem_client id, u32 offset,
+ u32 size);
extern bool qcom_scm_pas_supported(u32 peripheral);
extern int qcom_scm_pas_init_image(u32 peripheral, const void *metadata,
size_t size);
extern void qcom_scm_cpu_power_down(u32 flags);
extern u32 qcom_scm_get_version(void);
extern int qcom_scm_set_remote_state(u32 state, u32 id);
+extern bool qcom_scm_restore_sec_cfg_available(void);
extern int qcom_scm_restore_sec_cfg(u32 device_id, u32 spare);
extern int qcom_scm_iommu_secure_ptbl_size(u32 spare, size_t *size);
extern int qcom_scm_iommu_secure_ptbl_init(u64 addr, u32 size, u32 spare);
extern int __sys_sendmmsg(int fd, struct mmsghdr __user *mmsg,
unsigned int vlen, unsigned int flags,
bool forbid_cmsg_compat);
-extern long __sys_sendmsg_sock(struct socket *sock,
- struct user_msghdr __user *msg,
+extern long __sys_sendmsg_sock(struct socket *sock, struct msghdr *msg,
unsigned int flags);
-extern long __sys_recvmsg_sock(struct socket *sock,
- struct user_msghdr __user *msg,
+extern long __sys_recvmsg_sock(struct socket *sock, struct msghdr *msg,
+ struct user_msghdr __user *umsg,
+ struct sockaddr __user *uaddr,
unsigned int flags);
+extern int sendmsg_copy_msghdr(struct msghdr *msg,
+ struct user_msghdr __user *umsg, unsigned flags,
+ struct iovec **iov);
+extern int recvmsg_copy_msghdr(struct msghdr *msg,
+ struct user_msghdr __user *umsg, unsigned flags,
+ struct sockaddr __user **uaddr,
+ struct iovec **iov);
/* helpers which do the actual work for syscalls */
extern int __sys_recvfrom(int fd, void __user *ubuf, size_t size,
int __user *upeer_addrlen, int flags);
extern int __sys_socket(int family, int type, int protocol);
extern int __sys_bind(int fd, struct sockaddr __user *umyaddr, int addrlen);
-extern int __sys_connect_file(struct file *file,
- struct sockaddr __user *uservaddr, int addrlen,
- int file_flags);
+extern int __sys_connect_file(struct file *file, struct sockaddr_storage *addr,
+ int addrlen, int file_flags);
extern int __sys_connect(int fd, struct sockaddr __user *uservaddr,
int addrlen);
extern int __sys_listen(int fd, int backlog);
--- /dev/null
+/* SPDX-License-Identifier: GPL-2.0-only */
+/*
+ * The On Chip Memory (OCMEM) allocator allows various clients to allocate
+ * memory from OCMEM based on performance, latency and power requirements.
+ * This is typically used by the GPU, camera/video, and audio components on
+ * some Snapdragon SoCs.
+ *
+ * Copyright (C) 2019 Brian Masney <masneyb@onstation.org>
+ * Copyright (C) 2015 Red Hat. Author: Rob Clark <robdclark@gmail.com>
+ */
+
+#include <linux/device.h>
+#include <linux/err.h>
+
+#ifndef __OCMEM_H__
+#define __OCMEM_H__
+
+enum ocmem_client {
+ /* GMEM clients */
+ OCMEM_GRAPHICS = 0x0,
+ /*
+ * TODO add more once ocmem_allocate() is clever enough to
+ * deal with multiple clients.
+ */
+ OCMEM_CLIENT_MAX,
+};
+
+struct ocmem;
+
+struct ocmem_buf {
+ unsigned long offset;
+ unsigned long addr;
+ unsigned long len;
+};
+
+#if IS_ENABLED(CONFIG_QCOM_OCMEM)
+
+struct ocmem *of_get_ocmem(struct device *dev);
+struct ocmem_buf *ocmem_allocate(struct ocmem *ocmem, enum ocmem_client client,
+ unsigned long size);
+void ocmem_free(struct ocmem *ocmem, enum ocmem_client client,
+ struct ocmem_buf *buf);
+
+#else /* IS_ENABLED(CONFIG_QCOM_OCMEM) */
+
+static inline struct ocmem *of_get_ocmem(struct device *dev)
+{
+ return ERR_PTR(-ENODEV);
+}
+
+static inline struct ocmem_buf *ocmem_allocate(struct ocmem *ocmem,
+ enum ocmem_client client,
+ unsigned long size)
+{
+ return ERR_PTR(-ENODEV);
+}
+
+static inline void ocmem_free(struct ocmem *ocmem, enum ocmem_client client,
+ struct ocmem_buf *buf)
+{
+}
+
+#endif /* IS_ENABLED(CONFIG_QCOM_OCMEM) */
+
+#endif /* __OCMEM_H__ */
bool prepared:1;
bool no_period_wakeup:1;
bool locked:1;
+ bool stripe:1; /* apply stripe control */
/* timestamp */
unsigned long start_wallclk; /* start + minimum wallclk */
*/
#define IORING_FEAT_SINGLE_MMAP (1U << 0)
#define IORING_FEAT_NODROP (1U << 1)
+#define IORING_FEAT_SUBMIT_STABLE (1U << 2)
/*
* io_uring_register(2) opcodes and arguments
strlcpy(last_unloaded_module, mod->name, sizeof(last_unloaded_module));
free_module(mod);
+ /* someone could wait for the module in add_unformed_module() */
+ wake_up_all(&module_wq);
return 0;
out:
mutex_unlock(&module_mutex);
char *imported_namespace;
namespace = kernel_symbol_namespace(sym);
- if (namespace) {
+ if (namespace && namespace[0]) {
imported_namespace = get_modinfo(info, "import_ns");
while (imported_namespace) {
if (strcmp(namespace, imported_namespace) == 0)
#include <linux/module.h>
#include <linux/mount.h>
#include <linux/nsproxy.h>
-#include <linux/parser.h>
+#include <linux/fs_parser.h>
#include <linux/sched.h>
#include <linux/sched/mm.h>
#include <linux/seq_file.h>
Opt_mount_timeout,
Opt_osd_idle_ttl,
Opt_osd_request_timeout,
- Opt_last_int,
/* int args above */
Opt_fsid,
Opt_name,
Opt_secret,
Opt_key,
Opt_ip,
- Opt_last_string,
/* string args above */
Opt_share,
- Opt_noshare,
Opt_crc,
- Opt_nocrc,
Opt_cephx_require_signatures,
- Opt_nocephx_require_signatures,
Opt_cephx_sign_messages,
- Opt_nocephx_sign_messages,
Opt_tcp_nodelay,
- Opt_notcp_nodelay,
Opt_abort_on_full,
};
-static match_table_t opt_tokens = {
- {Opt_osdtimeout, "osdtimeout=%d"},
- {Opt_osdkeepalivetimeout, "osdkeepalive=%d"},
- {Opt_mount_timeout, "mount_timeout=%d"},
- {Opt_osd_idle_ttl, "osd_idle_ttl=%d"},
- {Opt_osd_request_timeout, "osd_request_timeout=%d"},
- /* int args above */
- {Opt_fsid, "fsid=%s"},
- {Opt_name, "name=%s"},
- {Opt_secret, "secret=%s"},
- {Opt_key, "key=%s"},
- {Opt_ip, "ip=%s"},
- /* string args above */
- {Opt_share, "share"},
- {Opt_noshare, "noshare"},
- {Opt_crc, "crc"},
- {Opt_nocrc, "nocrc"},
- {Opt_cephx_require_signatures, "cephx_require_signatures"},
- {Opt_nocephx_require_signatures, "nocephx_require_signatures"},
- {Opt_cephx_sign_messages, "cephx_sign_messages"},
- {Opt_nocephx_sign_messages, "nocephx_sign_messages"},
- {Opt_tcp_nodelay, "tcp_nodelay"},
- {Opt_notcp_nodelay, "notcp_nodelay"},
- {Opt_abort_on_full, "abort_on_full"},
- {-1, NULL}
+static const struct fs_parameter_spec ceph_param_specs[] = {
+ fsparam_flag ("abort_on_full", Opt_abort_on_full),
+ fsparam_flag_no ("cephx_require_signatures", Opt_cephx_require_signatures),
+ fsparam_flag_no ("cephx_sign_messages", Opt_cephx_sign_messages),
+ fsparam_flag_no ("crc", Opt_crc),
+ fsparam_string ("fsid", Opt_fsid),
+ fsparam_string ("ip", Opt_ip),
+ fsparam_string ("key", Opt_key),
+ fsparam_u32 ("mount_timeout", Opt_mount_timeout),
+ fsparam_string ("name", Opt_name),
+ fsparam_u32 ("osd_idle_ttl", Opt_osd_idle_ttl),
+ fsparam_u32 ("osd_request_timeout", Opt_osd_request_timeout),
+ fsparam_u32 ("osdkeepalive", Opt_osdkeepalivetimeout),
+ __fsparam (fs_param_is_s32, "osdtimeout", Opt_osdtimeout,
+ fs_param_deprecated),
+ fsparam_string ("secret", Opt_secret),
+ fsparam_flag_no ("share", Opt_share),
+ fsparam_flag_no ("tcp_nodelay", Opt_tcp_nodelay),
+ {}
+};
+
+static const struct fs_parameter_description ceph_parameters = {
+ .name = "libceph",
+ .specs = ceph_param_specs,
};
+struct ceph_options *ceph_alloc_options(void)
+{
+ struct ceph_options *opt;
+
+ opt = kzalloc(sizeof(*opt), GFP_KERNEL);
+ if (!opt)
+ return NULL;
+
+ opt->mon_addr = kcalloc(CEPH_MAX_MON, sizeof(*opt->mon_addr),
+ GFP_KERNEL);
+ if (!opt->mon_addr) {
+ kfree(opt);
+ return NULL;
+ }
+
+ opt->flags = CEPH_OPT_DEFAULT;
+ opt->osd_keepalive_timeout = CEPH_OSD_KEEPALIVE_DEFAULT;
+ opt->mount_timeout = CEPH_MOUNT_TIMEOUT_DEFAULT;
+ opt->osd_idle_ttl = CEPH_OSD_IDLE_TTL_DEFAULT;
+ opt->osd_request_timeout = CEPH_OSD_REQUEST_TIMEOUT_DEFAULT;
+ return opt;
+}
+EXPORT_SYMBOL(ceph_alloc_options);
+
void ceph_destroy_options(struct ceph_options *opt)
{
dout("destroy_options %p\n", opt);
+ if (!opt)
+ return;
+
kfree(opt->name);
if (opt->key) {
ceph_crypto_key_destroy(opt->key);
EXPORT_SYMBOL(ceph_destroy_options);
/* get secret from key store */
-static int get_secret(struct ceph_crypto_key *dst, const char *name) {
+static int get_secret(struct ceph_crypto_key *dst, const char *name,
+ struct fs_context *fc)
+{
struct key *ukey;
int key_err;
int err = 0;
key_err = PTR_ERR(ukey);
switch (key_err) {
case -ENOKEY:
- pr_warn("ceph: Mount failed due to key not found: %s\n",
- name);
+ errorf(fc, "libceph: Failed due to key not found: %s",
+ name);
break;
case -EKEYEXPIRED:
- pr_warn("ceph: Mount failed due to expired key: %s\n",
- name);
+ errorf(fc, "libceph: Failed due to expired key: %s",
+ name);
break;
case -EKEYREVOKED:
- pr_warn("ceph: Mount failed due to revoked key: %s\n",
- name);
+ errorf(fc, "libceph: Failed due to revoked key: %s",
+ name);
break;
default:
- pr_warn("ceph: Mount failed due to unknown key error %d: %s\n",
- key_err, name);
+ errorf(fc, "libceph: Failed due to key error %d: %s",
+ key_err, name);
}
err = -EPERM;
goto out;
return err;
}
-struct ceph_options *
-ceph_parse_options(char *options, const char *dev_name,
- const char *dev_name_end,
- int (*parse_extra_token)(char *c, void *private),
- void *private)
+int ceph_parse_mon_ips(const char *buf, size_t len, struct ceph_options *opt,
+ struct fs_context *fc)
{
- struct ceph_options *opt;
- const char *c;
- int err = -ENOMEM;
- substring_t argstr[MAX_OPT_ARGS];
-
- opt = kzalloc(sizeof(*opt), GFP_KERNEL);
- if (!opt)
- return ERR_PTR(-ENOMEM);
- opt->mon_addr = kcalloc(CEPH_MAX_MON, sizeof(*opt->mon_addr),
- GFP_KERNEL);
- if (!opt->mon_addr)
- goto out;
-
- dout("parse_options %p options '%s' dev_name '%s'\n", opt, options,
- dev_name);
-
- /* start with defaults */
- opt->flags = CEPH_OPT_DEFAULT;
- opt->osd_keepalive_timeout = CEPH_OSD_KEEPALIVE_DEFAULT;
- opt->mount_timeout = CEPH_MOUNT_TIMEOUT_DEFAULT;
- opt->osd_idle_ttl = CEPH_OSD_IDLE_TTL_DEFAULT;
- opt->osd_request_timeout = CEPH_OSD_REQUEST_TIMEOUT_DEFAULT;
+ int ret;
- /* get mon ip(s) */
/* ip1[:port1][,ip2[:port2]...] */
- err = ceph_parse_ips(dev_name, dev_name_end, opt->mon_addr,
- CEPH_MAX_MON, &opt->num_mon);
- if (err < 0)
- goto out;
+ ret = ceph_parse_ips(buf, buf + len, opt->mon_addr, CEPH_MAX_MON,
+ &opt->num_mon);
+ if (ret) {
+ errorf(fc, "libceph: Failed to parse monitor IPs: %d", ret);
+ return ret;
+ }
- /* parse mount options */
- while ((c = strsep(&options, ",")) != NULL) {
- int token, intval;
- if (!*c)
- continue;
- err = -EINVAL;
- token = match_token((char *)c, opt_tokens, argstr);
- if (token < 0 && parse_extra_token) {
- /* extra? */
- err = parse_extra_token((char *)c, private);
- if (err < 0) {
- pr_err("bad option at '%s'\n", c);
- goto out;
- }
- continue;
- }
- if (token < Opt_last_int) {
- err = match_int(&argstr[0], &intval);
- if (err < 0) {
- pr_err("bad option arg (not int) at '%s'\n", c);
- goto out;
- }
- dout("got int token %d val %d\n", token, intval);
- } else if (token > Opt_last_int && token < Opt_last_string) {
- dout("got string token %d val %s\n", token,
- argstr[0].from);
- } else {
- dout("got token %d\n", token);
+ return 0;
+}
+EXPORT_SYMBOL(ceph_parse_mon_ips);
+
+int ceph_parse_param(struct fs_parameter *param, struct ceph_options *opt,
+ struct fs_context *fc)
+{
+ struct fs_parse_result result;
+ int token, err;
+
+ token = fs_parse(fc, &ceph_parameters, param, &result);
+ dout("%s fs_parse '%s' token %d\n", __func__, param->key, token);
+ if (token < 0)
+ return token;
+
+ switch (token) {
+ case Opt_ip:
+ err = ceph_parse_ips(param->string,
+ param->string + param->size,
+ &opt->my_addr,
+ 1, NULL);
+ if (err) {
+ errorf(fc, "libceph: Failed to parse ip: %d", err);
+ return err;
}
- switch (token) {
- case Opt_ip:
- err = ceph_parse_ips(argstr[0].from,
- argstr[0].to,
- &opt->my_addr,
- 1, NULL);
- if (err < 0)
- goto out;
- opt->flags |= CEPH_OPT_MYIP;
- break;
+ opt->flags |= CEPH_OPT_MYIP;
+ break;
- case Opt_fsid:
- err = parse_fsid(argstr[0].from, &opt->fsid);
- if (err == 0)
- opt->flags |= CEPH_OPT_FSID;
- break;
- case Opt_name:
- kfree(opt->name);
- opt->name = kstrndup(argstr[0].from,
- argstr[0].to-argstr[0].from,
- GFP_KERNEL);
- if (!opt->name) {
- err = -ENOMEM;
- goto out;
- }
- break;
- case Opt_secret:
- ceph_crypto_key_destroy(opt->key);
- kfree(opt->key);
-
- opt->key = kzalloc(sizeof(*opt->key), GFP_KERNEL);
- if (!opt->key) {
- err = -ENOMEM;
- goto out;
- }
- err = ceph_crypto_key_unarmor(opt->key, argstr[0].from);
- if (err < 0)
- goto out;
- break;
- case Opt_key:
- ceph_crypto_key_destroy(opt->key);
- kfree(opt->key);
-
- opt->key = kzalloc(sizeof(*opt->key), GFP_KERNEL);
- if (!opt->key) {
- err = -ENOMEM;
- goto out;
- }
- err = get_secret(opt->key, argstr[0].from);
- if (err < 0)
- goto out;
- break;
+ case Opt_fsid:
+ err = parse_fsid(param->string, &opt->fsid);
+ if (err) {
+ errorf(fc, "libceph: Failed to parse fsid: %d", err);
+ return err;
+ }
+ opt->flags |= CEPH_OPT_FSID;
+ break;
+ case Opt_name:
+ kfree(opt->name);
+ opt->name = param->string;
+ param->string = NULL;
+ break;
+ case Opt_secret:
+ ceph_crypto_key_destroy(opt->key);
+ kfree(opt->key);
- /* misc */
- case Opt_osdtimeout:
- pr_warn("ignoring deprecated osdtimeout option\n");
- break;
- case Opt_osdkeepalivetimeout:
- /* 0 isn't well defined right now, reject it */
- if (intval < 1 || intval > INT_MAX / 1000) {
- pr_err("osdkeepalive out of range\n");
- err = -EINVAL;
- goto out;
- }
- opt->osd_keepalive_timeout =
- msecs_to_jiffies(intval * 1000);
- break;
- case Opt_osd_idle_ttl:
- /* 0 isn't well defined right now, reject it */
- if (intval < 1 || intval > INT_MAX / 1000) {
- pr_err("osd_idle_ttl out of range\n");
- err = -EINVAL;
- goto out;
- }
- opt->osd_idle_ttl = msecs_to_jiffies(intval * 1000);
- break;
- case Opt_mount_timeout:
- /* 0 is "wait forever" (i.e. infinite timeout) */
- if (intval < 0 || intval > INT_MAX / 1000) {
- pr_err("mount_timeout out of range\n");
- err = -EINVAL;
- goto out;
- }
- opt->mount_timeout = msecs_to_jiffies(intval * 1000);
- break;
- case Opt_osd_request_timeout:
- /* 0 is "wait forever" (i.e. infinite timeout) */
- if (intval < 0 || intval > INT_MAX / 1000) {
- pr_err("osd_request_timeout out of range\n");
- err = -EINVAL;
- goto out;
- }
- opt->osd_request_timeout = msecs_to_jiffies(intval * 1000);
- break;
+ opt->key = kzalloc(sizeof(*opt->key), GFP_KERNEL);
+ if (!opt->key)
+ return -ENOMEM;
+ err = ceph_crypto_key_unarmor(opt->key, param->string);
+ if (err) {
+ errorf(fc, "libceph: Failed to parse secret: %d", err);
+ return err;
+ }
+ break;
+ case Opt_key:
+ ceph_crypto_key_destroy(opt->key);
+ kfree(opt->key);
- case Opt_share:
+ opt->key = kzalloc(sizeof(*opt->key), GFP_KERNEL);
+ if (!opt->key)
+ return -ENOMEM;
+ return get_secret(opt->key, param->string, fc);
+
+ case Opt_osdtimeout:
+ warnf(fc, "libceph: Ignoring osdtimeout");
+ break;
+ case Opt_osdkeepalivetimeout:
+ /* 0 isn't well defined right now, reject it */
+ if (result.uint_32 < 1 || result.uint_32 > INT_MAX / 1000)
+ goto out_of_range;
+ opt->osd_keepalive_timeout =
+ msecs_to_jiffies(result.uint_32 * 1000);
+ break;
+ case Opt_osd_idle_ttl:
+ /* 0 isn't well defined right now, reject it */
+ if (result.uint_32 < 1 || result.uint_32 > INT_MAX / 1000)
+ goto out_of_range;
+ opt->osd_idle_ttl = msecs_to_jiffies(result.uint_32 * 1000);
+ break;
+ case Opt_mount_timeout:
+ /* 0 is "wait forever" (i.e. infinite timeout) */
+ if (result.uint_32 > INT_MAX / 1000)
+ goto out_of_range;
+ opt->mount_timeout = msecs_to_jiffies(result.uint_32 * 1000);
+ break;
+ case Opt_osd_request_timeout:
+ /* 0 is "wait forever" (i.e. infinite timeout) */
+ if (result.uint_32 > INT_MAX / 1000)
+ goto out_of_range;
+ opt->osd_request_timeout =
+ msecs_to_jiffies(result.uint_32 * 1000);
+ break;
+
+ case Opt_share:
+ if (!result.negated)
opt->flags &= ~CEPH_OPT_NOSHARE;
- break;
- case Opt_noshare:
+ else
opt->flags |= CEPH_OPT_NOSHARE;
- break;
-
- case Opt_crc:
+ break;
+ case Opt_crc:
+ if (!result.negated)
opt->flags &= ~CEPH_OPT_NOCRC;
- break;
- case Opt_nocrc:
+ else
opt->flags |= CEPH_OPT_NOCRC;
- break;
-
- case Opt_cephx_require_signatures:
+ break;
+ case Opt_cephx_require_signatures:
+ if (!result.negated)
opt->flags &= ~CEPH_OPT_NOMSGAUTH;
- break;
- case Opt_nocephx_require_signatures:
+ else
opt->flags |= CEPH_OPT_NOMSGAUTH;
- break;
- case Opt_cephx_sign_messages:
+ break;
+ case Opt_cephx_sign_messages:
+ if (!result.negated)
opt->flags &= ~CEPH_OPT_NOMSGSIGN;
- break;
- case Opt_nocephx_sign_messages:
+ else
opt->flags |= CEPH_OPT_NOMSGSIGN;
- break;
-
- case Opt_tcp_nodelay:
+ break;
+ case Opt_tcp_nodelay:
+ if (!result.negated)
opt->flags |= CEPH_OPT_TCP_NODELAY;
- break;
- case Opt_notcp_nodelay:
+ else
opt->flags &= ~CEPH_OPT_TCP_NODELAY;
- break;
+ break;
- case Opt_abort_on_full:
- opt->flags |= CEPH_OPT_ABORT_ON_FULL;
- break;
+ case Opt_abort_on_full:
+ opt->flags |= CEPH_OPT_ABORT_ON_FULL;
+ break;
- default:
- BUG_ON(token);
- }
+ default:
+ BUG();
}
- /* success */
- return opt;
+ return 0;
-out:
- ceph_destroy_options(opt);
- return ERR_PTR(err);
+out_of_range:
+ return invalf(fc, "libceph: %s out of range", param->key);
}
-EXPORT_SYMBOL(ceph_parse_options);
+EXPORT_SYMBOL(ceph_parse_param);
int ceph_print_client_options(struct seq_file *m, struct ceph_client *client,
bool show_all)
return 0;
bad:
- pr_err("parse_ips bad ip '%.*s'\n", (int)(end - c), c);
return ret;
}
-EXPORT_SYMBOL(ceph_parse_ips);
static int process_banner(struct ceph_connection *con)
{
struct ceph_mon_client *monc = con->private;
int type = le16_to_cpu(msg->hdr.type);
- if (!monc)
- return;
-
switch (type) {
case CEPH_MSG_AUTH_REPLY:
handle_auth_reply(monc, msg);
* include the -EINPROGRESS status for such sockets.
*/
-int __sys_connect_file(struct file *file, struct sockaddr __user *uservaddr,
+int __sys_connect_file(struct file *file, struct sockaddr_storage *address,
int addrlen, int file_flags)
{
struct socket *sock;
- struct sockaddr_storage address;
int err;
sock = sock_from_file(file, &err);
if (!sock)
goto out;
- err = move_addr_to_kernel(uservaddr, addrlen, &address);
- if (err < 0)
- goto out;
err =
- security_socket_connect(sock, (struct sockaddr *)&address, addrlen);
+ security_socket_connect(sock, (struct sockaddr *)address, addrlen);
if (err)
goto out;
- err = sock->ops->connect(sock, (struct sockaddr *)&address, addrlen,
+ err = sock->ops->connect(sock, (struct sockaddr *)address, addrlen,
sock->file->f_flags | file_flags);
out:
return err;
f = fdget(fd);
if (f.file) {
- ret = __sys_connect_file(f.file, uservaddr, addrlen, 0);
+ struct sockaddr_storage address;
+
+ ret = move_addr_to_kernel(uservaddr, addrlen, &address);
+ if (!ret)
+ ret = __sys_connect_file(f.file, &address, addrlen, 0);
if (f.flags)
fput(f.file);
}
return err;
}
-static int sendmsg_copy_msghdr(struct msghdr *msg,
- struct user_msghdr __user *umsg, unsigned flags,
- struct iovec **iov)
+int sendmsg_copy_msghdr(struct msghdr *msg,
+ struct user_msghdr __user *umsg, unsigned flags,
+ struct iovec **iov)
{
int err;
/*
* BSD sendmsg interface
*/
-long __sys_sendmsg_sock(struct socket *sock, struct user_msghdr __user *umsg,
+long __sys_sendmsg_sock(struct socket *sock, struct msghdr *msg,
unsigned int flags)
{
- struct iovec iovstack[UIO_FASTIOV], *iov = iovstack;
- struct sockaddr_storage address;
- struct msghdr msg = { .msg_name = &address };
- ssize_t err;
-
- err = sendmsg_copy_msghdr(&msg, umsg, flags, &iov);
- if (err)
- return err;
/* disallow ancillary data requests from this path */
- if (msg.msg_control || msg.msg_controllen) {
- err = -EINVAL;
- goto out;
- }
+ if (msg->msg_control || msg->msg_controllen)
+ return -EINVAL;
- err = ____sys_sendmsg(sock, &msg, flags, NULL, 0);
-out:
- kfree(iov);
- return err;
+ return ____sys_sendmsg(sock, msg, flags, NULL, 0);
}
long __sys_sendmsg(int fd, struct user_msghdr __user *msg, unsigned int flags,
return __sys_sendmmsg(fd, mmsg, vlen, flags, true);
}
-static int recvmsg_copy_msghdr(struct msghdr *msg,
- struct user_msghdr __user *umsg, unsigned flags,
- struct sockaddr __user **uaddr,
- struct iovec **iov)
+int recvmsg_copy_msghdr(struct msghdr *msg,
+ struct user_msghdr __user *umsg, unsigned flags,
+ struct sockaddr __user **uaddr,
+ struct iovec **iov)
{
ssize_t err;
* BSD recvmsg interface
*/
-long __sys_recvmsg_sock(struct socket *sock, struct user_msghdr __user *umsg,
- unsigned int flags)
+long __sys_recvmsg_sock(struct socket *sock, struct msghdr *msg,
+ struct user_msghdr __user *umsg,
+ struct sockaddr __user *uaddr, unsigned int flags)
{
- struct iovec iovstack[UIO_FASTIOV], *iov = iovstack;
- struct sockaddr_storage address;
- struct msghdr msg = { .msg_name = &address };
- struct sockaddr __user *uaddr;
- ssize_t err;
-
- err = recvmsg_copy_msghdr(&msg, umsg, flags, &uaddr, &iov);
- if (err)
- return err;
/* disallow ancillary data requests from this path */
- if (msg.msg_control || msg.msg_controllen) {
- err = -EINVAL;
- goto out;
- }
+ if (msg->msg_control || msg->msg_controllen)
+ return -EINVAL;
- err = ____sys_recvmsg(sock, &msg, umsg, uaddr, flags, 0);
-out:
- kfree(iov);
- return err;
+ return ____sys_recvmsg(sock, msg, umsg, uaddr, flags, 0);
}
long __sys_recvmsg(int fd, struct user_msghdr __user *msg, unsigned int flags,
}
}
#endif
+ if (frames > dst_channels[0].frames)
+ frames = dst_channels[0].frames;
convert(plugin, src_channels, dst_channels, frames);
return frames;
}
}
}
#endif
+ if (frames > dst_channels[0].frames)
+ frames = dst_channels[0].frames;
data = (struct mulaw_priv *)plugin->extra_data;
data->func(plugin, src_channels, dst_channels, frames);
return frames;
return -ENXIO;
if (frames == 0)
return 0;
+ if (frames > dst_channels[0].frames)
+ frames = dst_channels[0].frames;
nsrcs = plugin->src_format.channels;
ndsts = plugin->dst_format.channels;
dpcm_play = cable->streams[SNDRV_PCM_STREAM_PLAYBACK];
dpcm_capt = cable->streams[SNDRV_PCM_STREAM_CAPTURE];
- substream_play = (running & (1 << SNDRV_PCM_STREAM_PLAYBACK)) ?
- dpcm_play->substream : NULL;
- substream_capt = (running & (1 << SNDRV_PCM_STREAM_CAPTURE)) ?
- dpcm_capt->substream : NULL;
if (event == SNDRV_TIMER_EVENT_MSTOP) {
if (!dpcm_play ||
}
}
+ substream_play = (running & (1 << SNDRV_PCM_STREAM_PLAYBACK)) ?
+ dpcm_play->substream : NULL;
+ substream_capt = (running & (1 << SNDRV_PCM_STREAM_CAPTURE)) ?
+ dpcm_capt->substream : NULL;
valid_runtime = (running & (1 << SNDRV_PCM_STREAM_PLAYBACK)) ?
dpcm_play->substream->runtime :
dpcm_capt->substream->runtime;
1 << azx_dev->index,
1 << azx_dev->index);
/* set stripe control */
- if (azx_dev->substream)
- stripe_ctl = snd_hdac_get_stream_stripe_ctl(bus, azx_dev->substream);
- else
- stripe_ctl = 0;
- snd_hdac_stream_updateb(azx_dev, SD_CTL_3B, SD_CTL_STRIPE_MASK,
- stripe_ctl);
+ if (azx_dev->stripe) {
+ if (azx_dev->substream)
+ stripe_ctl = snd_hdac_get_stream_stripe_ctl(bus, azx_dev->substream);
+ else
+ stripe_ctl = 0;
+ snd_hdac_stream_updateb(azx_dev, SD_CTL_3B, SD_CTL_STRIPE_MASK,
+ stripe_ctl);
+ }
/* set DMA start and interrupt mask */
snd_hdac_stream_updateb(azx_dev, SD_CTL,
0, SD_CTL_DMA_START | SD_INT_MASK);
snd_hdac_stream_updateb(azx_dev, SD_CTL,
SD_CTL_DMA_START | SD_INT_MASK, 0);
snd_hdac_stream_writeb(azx_dev, SD_STS, SD_INT_MASK); /* to be sure */
- snd_hdac_stream_updateb(azx_dev, SD_CTL_3B, SD_CTL_STRIPE_MASK, 0);
+ if (azx_dev->stripe) {
+ snd_hdac_stream_updateb(azx_dev, SD_CTL_3B, SD_CTL_STRIPE_MASK, 0);
+ azx_dev->stripe = 0;
+ }
azx_dev->running = false;
}
EXPORT_SYMBOL_GPL(snd_hdac_stream_clear);
#include <linux/clocksource.h>
#include <linux/time.h>
#include <linux/completion.h>
+#include <linux/acpi.h>
#ifdef CONFIG_X86
/* for snoop control */
}
#ifdef SUPPORT_VGA_SWITCHEROO
+#ifdef CONFIG_ACPI
+/* ATPX is in the integrated GPU's namespace */
+static bool atpx_present(void)
+{
+ struct pci_dev *pdev = NULL;
+ acpi_handle dhandle, atpx_handle;
+ acpi_status status;
+
+ while ((pdev = pci_get_class(PCI_BASE_CLASS_DISPLAY << 16, pdev)) != NULL) {
+ dhandle = ACPI_HANDLE(&pdev->dev);
+ if (dhandle) {
+ status = acpi_get_handle(dhandle, "ATPX", &atpx_handle);
+ if (!ACPI_FAILURE(status)) {
+ pci_dev_put(pdev);
+ return true;
+ }
+ }
+ pci_dev_put(pdev);
+ }
+ return false;
+}
+#else
+static bool atpx_present(void)
+{
+ return false;
+}
+#endif
+
/*
* Check of disabled HDMI controller by vga_switcheroo
*/
switch (pci->vendor) {
case PCI_VENDOR_ID_ATI:
case PCI_VENDOR_ID_AMD:
+ if (pci->devfn == 1) {
+ p = pci_get_domain_bus_and_slot(pci_domain_nr(pci->bus),
+ pci->bus->number, 0);
+ if (p) {
+ /* ATPX is in the integrated GPU's ACPI namespace
+ * rather than the dGPU's namespace. However,
+ * the dGPU is the one who is involved in
+ * vgaswitcheroo.
+ */
+ if (((p->class >> 16) == PCI_BASE_CLASS_DISPLAY) &&
+ atpx_present())
+ return p;
+ pci_dev_put(p);
+ }
+ }
+ break;
case PCI_VENDOR_ID_NVIDIA:
if (pci->devfn == 1) {
p = pci_get_domain_bus_and_slot(pci_domain_nr(pci->bus),
{ PCI_DEVICE(0x1002, 0xaac8),
.driver_data = AZX_DRIVER_ATIHDMI_NS | AZX_DCAPS_PRESET_ATI_HDMI_NS },
{ PCI_DEVICE(0x1002, 0xaad8),
- .driver_data = AZX_DRIVER_ATIHDMI_NS | AZX_DCAPS_PRESET_ATI_HDMI_NS },
- { PCI_DEVICE(0x1002, 0xaae8),
- .driver_data = AZX_DRIVER_ATIHDMI_NS | AZX_DCAPS_PRESET_ATI_HDMI_NS },
+ .driver_data = AZX_DRIVER_ATIHDMI_NS | AZX_DCAPS_PRESET_ATI_HDMI_NS |
+ AZX_DCAPS_PM_RUNTIME },
{ PCI_DEVICE(0x1002, 0xaae0),
- .driver_data = AZX_DRIVER_ATIHDMI_NS | AZX_DCAPS_PRESET_ATI_HDMI_NS },
+ .driver_data = AZX_DRIVER_ATIHDMI_NS | AZX_DCAPS_PRESET_ATI_HDMI_NS |
+ AZX_DCAPS_PM_RUNTIME },
+ { PCI_DEVICE(0x1002, 0xaae8),
+ .driver_data = AZX_DRIVER_ATIHDMI_NS | AZX_DCAPS_PRESET_ATI_HDMI_NS |
+ AZX_DCAPS_PM_RUNTIME },
{ PCI_DEVICE(0x1002, 0xaaf0),
- .driver_data = AZX_DRIVER_ATIHDMI_NS | AZX_DCAPS_PRESET_ATI_HDMI_NS },
+ .driver_data = AZX_DRIVER_ATIHDMI_NS | AZX_DCAPS_PRESET_ATI_HDMI_NS |
+ AZX_DCAPS_PM_RUNTIME },
+ { PCI_DEVICE(0x1002, 0xaaf8),
+ .driver_data = AZX_DRIVER_ATIHDMI_NS | AZX_DCAPS_PRESET_ATI_HDMI_NS |
+ AZX_DCAPS_PM_RUNTIME },
+ { PCI_DEVICE(0x1002, 0xab00),
+ .driver_data = AZX_DRIVER_ATIHDMI_NS | AZX_DCAPS_PRESET_ATI_HDMI_NS |
+ AZX_DCAPS_PM_RUNTIME },
+ { PCI_DEVICE(0x1002, 0xab08),
+ .driver_data = AZX_DRIVER_ATIHDMI_NS | AZX_DCAPS_PRESET_ATI_HDMI_NS |
+ AZX_DCAPS_PM_RUNTIME },
+ { PCI_DEVICE(0x1002, 0xab10),
+ .driver_data = AZX_DRIVER_ATIHDMI_NS | AZX_DCAPS_PRESET_ATI_HDMI_NS |
+ AZX_DCAPS_PM_RUNTIME },
+ { PCI_DEVICE(0x1002, 0xab18),
+ .driver_data = AZX_DRIVER_ATIHDMI_NS | AZX_DCAPS_PRESET_ATI_HDMI_NS |
+ AZX_DCAPS_PM_RUNTIME },
+ { PCI_DEVICE(0x1002, 0xab20),
+ .driver_data = AZX_DRIVER_ATIHDMI_NS | AZX_DCAPS_PRESET_ATI_HDMI_NS |
+ AZX_DCAPS_PM_RUNTIME },
+ { PCI_DEVICE(0x1002, 0xab38),
+ .driver_data = AZX_DRIVER_ATIHDMI_NS | AZX_DCAPS_PRESET_ATI_HDMI_NS |
+ AZX_DCAPS_PM_RUNTIME },
/* VIA VT8251/VT8237A */
{ PCI_DEVICE(0x1106, 0x3288), .driver_data = AZX_DRIVER_VIA },
/* VIA GFX VT7122/VX900 */
#include <sound/hda_codec.h>
#include "hda_local.h"
#include "hda_jack.h"
+#include "hda_controller.h"
static bool static_hdmi_pcm;
module_param(static_hdmi_pcm, bool, 0644);
per_pin->cvt_nid = per_cvt->cvt_nid;
hinfo->nid = per_cvt->cvt_nid;
+ /* flip stripe flag for the assigned stream if supported */
+ if (get_wcaps(codec, per_cvt->cvt_nid) & AC_WCAP_STRIPE)
+ azx_stream(get_azx_dev(substream))->stripe = 1;
+
snd_hda_set_dev_select(codec, per_pin->pin_nid, per_pin->dev_id);
snd_hda_codec_write_cache(codec, per_pin->pin_nid, 0,
AC_VERB_SET_CONNECT_SEL,
struct hdmi_spec_per_pin *per_pin = get_pin(spec, pin_idx);
hda_nid_t pin_nid = per_pin->pin_nid;
int dev_id = per_pin->dev_id;
+ int conns;
if (!(get_wcaps(codec, pin_nid) & AC_WCAP_CONN_LIST)) {
codec_warn(codec,
snd_hda_set_dev_select(codec, pin_nid, dev_id);
+ if (spec->intel_hsw_fixup) {
+ conns = spec->num_cvts;
+ memcpy(per_pin->mux_nids, spec->cvt_nids,
+ sizeof(hda_nid_t) * conns);
+ } else {
+ conns = snd_hda_get_raw_connections(codec, pin_nid,
+ per_pin->mux_nids,
+ HDA_MAX_CONNECTIONS);
+ }
+
/* all the device entries on the same pin have the same conn list */
- per_pin->num_mux_nids =
- snd_hda_get_raw_connections(codec, pin_nid, per_pin->mux_nids,
- HDA_MAX_CONNECTIONS);
+ per_pin->num_mux_nids = conns;
return 0;
}
int i;
/*
- * generic_hdmi_build_pcms() allocates (num_nids + dev_num - 1)
- * number of pcms.
+ * generic_hdmi_build_pcms() may allocate extra PCMs on some
+ * platforms (with maximum of 'num_nids + dev_num - 1')
*
* The per_pin of pin_nid_idx=n and dev_id=m prefers to get pcm-n
* if m==0. This guarantees that dynamic pcm assignments are compatible
- * with the legacy static per_pin-pmc assignment that existed in the
+ * with the legacy static per_pin-pcm assignment that existed in the
* days before DP-MST.
*
+ * Intel DP-MST prefers this legacy behavior for compatibility, too.
+ *
* per_pin of m!=0 prefers to get pcm=(num_nids + (m - 1)).
*/
- if (per_pin->dev_id == 0 &&
- !test_bit(per_pin->pin_nid_idx, &spec->pcm_bitmap))
- return per_pin->pin_nid_idx;
-
- if (per_pin->dev_id != 0 &&
- !(test_bit(spec->num_nids + (per_pin->dev_id - 1),
- &spec->pcm_bitmap))) {
- return spec->num_nids + (per_pin->dev_id - 1);
+
+ if (per_pin->dev_id == 0 || spec->intel_hsw_fixup) {
+ if (!test_bit(per_pin->pin_nid_idx, &spec->pcm_bitmap))
+ return per_pin->pin_nid_idx;
+ } else {
+ i = spec->num_nids + (per_pin->dev_id - 1);
+ if (i < spec->pcm_used && !(test_bit(i, &spec->pcm_bitmap)))
+ return i;
}
/* have a second try; check the area over num_nids */
mutex_unlock(&spec->pcm_lock);
}
-static void intel_haswell_fixup_connect_list(struct hda_codec *codec,
- hda_nid_t nid);
-
static int hdmi_add_pin(struct hda_codec *codec, hda_nid_t pin_nid)
{
struct hdmi_spec *spec = codec->spec;
per_pin->dev_id = i;
per_pin->non_pcm = false;
snd_hda_set_dev_select(codec, pin_nid, i);
- if (spec->intel_hsw_fixup)
- intel_haswell_fixup_connect_list(codec, pin_nid);
err = hdmi_read_pin_conn(codec, pin_idx);
if (err < 0)
return err;
* Intel codec parsers and helpers
*/
-static void intel_haswell_fixup_connect_list(struct hda_codec *codec,
- hda_nid_t nid)
-{
- struct hdmi_spec *spec = codec->spec;
- hda_nid_t conns[4];
- int nconns;
-
- nconns = snd_hda_get_raw_connections(codec, nid, conns,
- ARRAY_SIZE(conns));
- if (nconns == spec->num_cvts &&
- !memcmp(conns, spec->cvt_nids, spec->num_cvts * sizeof(hda_nid_t)))
- return;
-
- /* override pins connection list */
- codec_dbg(codec, "hdmi: haswell: override pin connection 0x%x\n", nid);
- snd_hda_override_conn_list(codec, nid, spec->num_cvts, spec->cvt_nids);
-}
-
#define INTEL_GET_VENDOR_VERB 0xf81
#define INTEL_SET_VENDOR_VERB 0x781
#define INTEL_EN_DP12 0x02 /* enable DP 1.2 features */
ATI_VERB_SET_MULTICHANNEL_MODE,
ATI_MULTICHANNEL_MODE_SINGLE);
}
+ codec->auto_runtime_pm = 1;
return 0;
}
case 0x10ec0215:
case 0x10ec0233:
case 0x10ec0235:
- case 0x10ec0236:
case 0x10ec0255:
- case 0x10ec0256:
case 0x10ec0257:
case 0x10ec0282:
case 0x10ec0283:
case 0x10ec0300:
alc_update_coef_idx(codec, 0x10, 1<<9, 0);
break;
+ case 0x10ec0236:
+ case 0x10ec0256:
+ alc_write_coef_idx(codec, 0x36, 0x5757);
+ alc_update_coef_idx(codec, 0x10, 1<<9, 0);
+ break;
case 0x10ec0275:
alc_update_coef_idx(codec, 0xe, 0, 1<<0);
break;
}
}
+/* force NID 0x17 (Bass Speaker) to DAC1 to share it with the main speaker */
+static void alc285_fixup_speaker2_to_dac1(struct hda_codec *codec,
+ const struct hda_fixup *fix, int action)
+{
+ if (action == HDA_FIXUP_ACT_PRE_PROBE) {
+ hda_nid_t conn[1] = { 0x02 };
+ snd_hda_override_conn_list(codec, 0x17, 1, conn);
+ }
+}
+
/* Hook to update amp GPIO4 for automute */
static void alc280_hp_gpio4_automute_hook(struct hda_codec *codec,
struct hda_jack_callback *jack)
ALC225_FIXUP_DISABLE_MIC_VREF,
ALC225_FIXUP_DELL1_MIC_NO_PRESENCE,
ALC295_FIXUP_DISABLE_DAC3,
+ ALC285_FIXUP_SPEAKER2_TO_DAC1,
ALC280_FIXUP_HP_HEADSET_MIC,
ALC221_FIXUP_HP_FRONT_MIC,
ALC292_FIXUP_TPT460,
.type = HDA_FIXUP_FUNC,
.v.func = alc295_fixup_disable_dac3,
},
+ [ALC285_FIXUP_SPEAKER2_TO_DAC1] = {
+ .type = HDA_FIXUP_FUNC,
+ .v.func = alc285_fixup_speaker2_to_dac1,
+ },
[ALC256_FIXUP_DELL_INSPIRON_7559_SUBWOOFER] = {
.type = HDA_FIXUP_PINS,
.v.pins = (const struct hda_pintbl[]) {
SND_PCI_QUIRK(0x17aa, 0x224c, "Thinkpad", ALC298_FIXUP_TPT470_DOCK),
SND_PCI_QUIRK(0x17aa, 0x224d, "Thinkpad", ALC298_FIXUP_TPT470_DOCK),
SND_PCI_QUIRK(0x17aa, 0x225d, "Thinkpad T480", ALC269_FIXUP_LIMIT_INT_MIC_BOOST),
+ SND_PCI_QUIRK(0x17aa, 0x2293, "Thinkpad X1 Carbon 7th", ALC285_FIXUP_SPEAKER2_TO_DAC1),
SND_PCI_QUIRK(0x17aa, 0x30bb, "ThinkCentre AIO", ALC233_FIXUP_LENOVO_LINE2_MIC_HOTKEY),
SND_PCI_QUIRK(0x17aa, 0x30e2, "ThinkCentre AIO", ALC233_FIXUP_LENOVO_LINE2_MIC_HOTKEY),
SND_PCI_QUIRK(0x17aa, 0x310c, "ThinkCentre Station", ALC294_FIXUP_LENOVO_MIC_LOCATION),
{.id = ALC255_FIXUP_DELL_SPK_NOISE, .name = "dell-spk-noise"},
{.id = ALC225_FIXUP_DELL1_MIC_NO_PRESENCE, .name = "alc225-dell1"},
{.id = ALC295_FIXUP_DISABLE_DAC3, .name = "alc295-disable-dac3"},
+ {.id = ALC285_FIXUP_SPEAKER2_TO_DAC1, .name = "alc285-speaker2-to-dac1"},
{.id = ALC280_FIXUP_HP_HEADSET_MIC, .name = "alc280-hp-headset"},
{.id = ALC221_FIXUP_HP_FRONT_MIC, .name = "alc221-hp-mic"},
{.id = ALC298_FIXUP_SPK_VOLUME, .name = "alc298-spk-volume"},
case HDA_FIXUP_ACT_PRE_PROBE:
snd_hda_jack_detect_enable_callback(codec, 0x1b,
alc662_aspire_ethos_mute_speakers);
+ /* subwoofer needs an extra GPIO setting to become audible */
+ alc_setup_gpio(codec, 0x02);
break;
case HDA_FIXUP_ACT_INIT:
/* Make sure to start in a correct state, i.e. if
ALC662_FIXUP_USI_HEADSET_MODE,
ALC662_FIXUP_LENOVO_MULTI_CODECS,
ALC669_FIXUP_ACER_ASPIRE_ETHOS,
- ALC669_FIXUP_ACER_ASPIRE_ETHOS_SUBWOOFER,
ALC669_FIXUP_ACER_ASPIRE_ETHOS_HEADSET,
};
.type = HDA_FIXUP_FUNC,
.v.func = alc662_fixup_aspire_ethos_hp,
},
- [ALC669_FIXUP_ACER_ASPIRE_ETHOS_SUBWOOFER] = {
- .type = HDA_FIXUP_VERBS,
- /* subwoofer needs an extra GPIO setting to become audible */
- .v.verbs = (const struct hda_verb[]) {
- {0x01, AC_VERB_SET_GPIO_MASK, 0x02},
- {0x01, AC_VERB_SET_GPIO_DIRECTION, 0x02},
- {0x01, AC_VERB_SET_GPIO_DATA, 0x00},
- { }
- },
- .chained = true,
- .chain_id = ALC669_FIXUP_ACER_ASPIRE_ETHOS_HEADSET
- },
[ALC669_FIXUP_ACER_ASPIRE_ETHOS] = {
.type = HDA_FIXUP_PINS,
.v.pins = (const struct hda_pintbl[]) {
{ }
},
.chained = true,
- .chain_id = ALC669_FIXUP_ACER_ASPIRE_ETHOS_SUBWOOFER
+ .chain_id = ALC669_FIXUP_ACER_ASPIRE_ETHOS_HEADSET
},
};
projects / platform / kernel / linux-rpi.git / commitdiff