}
/* Context descriptor manipulation functions */
- ---static u64 arm_smmu_cpu_tcr_to_cd(u64 tcr)
+ +++static void arm_smmu_sync_cd(struct arm_smmu_domain *smmu_domain,
+ +++ int ssid, bool leaf)
{
- --- u64 val = 0;
+ +++ size_t i;
+ +++ unsigned long flags;
+ +++ struct arm_smmu_master *master;
+ +++ struct arm_smmu_device *smmu = smmu_domain->smmu;
+ +++ struct arm_smmu_cmdq_ent cmd = {
+ +++ .opcode = CMDQ_OP_CFGI_CD,
+ +++ .cfgi = {
+ +++ .ssid = ssid,
+ +++ .leaf = leaf,
+ +++ },
+ +++ };
+ +++
+ +++ spin_lock_irqsave(&smmu_domain->devices_lock, flags);
+ +++ list_for_each_entry(master, &smmu_domain->devices, domain_head) {
+ +++ for (i = 0; i < master->num_sids; i++) {
+ +++ cmd.cfgi.sid = master->sids[i];
+ +++ arm_smmu_cmdq_issue_cmd(smmu, &cmd);
+ +++ }
+ +++ }
+ +++ spin_unlock_irqrestore(&smmu_domain->devices_lock, flags);
+ +++
+ +++ arm_smmu_cmdq_issue_sync(smmu);
+ +++}
- --- /* Repack the TCR. Just care about TTBR0 for now */
- --- val |= ARM_SMMU_TCR2CD(tcr, T0SZ);
- --- val |= ARM_SMMU_TCR2CD(tcr, TG0);
- --- val |= ARM_SMMU_TCR2CD(tcr, IRGN0);
- --- val |= ARM_SMMU_TCR2CD(tcr, ORGN0);
- --- val |= ARM_SMMU_TCR2CD(tcr, SH0);
- --- val |= ARM_SMMU_TCR2CD(tcr, EPD0);
- --- val |= ARM_SMMU_TCR2CD(tcr, EPD1);
- --- val |= ARM_SMMU_TCR2CD(tcr, IPS);
+ +++static int arm_smmu_alloc_cd_leaf_table(struct arm_smmu_device *smmu,
+ +++ struct arm_smmu_l1_ctx_desc *l1_desc)
+ +++{
+ +++ size_t size = CTXDESC_L2_ENTRIES * (CTXDESC_CD_DWORDS << 3);
- --- return val;
+ +++ l1_desc->l2ptr = dmam_alloc_coherent(smmu->dev, size,
+ +++ &l1_desc->l2ptr_dma, GFP_KERNEL);
+ +++ if (!l1_desc->l2ptr) {
+ +++ dev_warn(smmu->dev,
+ +++ "failed to allocate context descriptor table\n");
+ +++ return -ENOMEM;
+ +++ }
+ +++ return 0;
}
- ---static void arm_smmu_write_ctx_desc(struct arm_smmu_device *smmu,
- --- struct arm_smmu_s1_cfg *cfg)
+ +++static void arm_smmu_write_cd_l1_desc(__le64 *dst,
+ +++ struct arm_smmu_l1_ctx_desc *l1_desc)
{
- --- u64 val;
+ +++ u64 val = (l1_desc->l2ptr_dma & CTXDESC_L1_DESC_L2PTR_MASK) |
+ +++ CTXDESC_L1_DESC_V;
+ ++
+ +++ WRITE_ONCE(*dst, cpu_to_le64(val));
+ +++}
+ +++
+ +++static __le64 *arm_smmu_get_cd_ptr(struct arm_smmu_domain *smmu_domain,
+ +++ u32 ssid)
+ +++{
+ +++ __le64 *l1ptr;
+ +++ unsigned int idx;
+ +++ struct arm_smmu_l1_ctx_desc *l1_desc;
+ +++ struct arm_smmu_device *smmu = smmu_domain->smmu;
+ +++ struct arm_smmu_ctx_desc_cfg *cdcfg = &smmu_domain->s1_cfg.cdcfg;
+
+ +++ if (smmu_domain->s1_cfg.s1fmt == STRTAB_STE_0_S1FMT_LINEAR)
+ +++ return cdcfg->cdtab + ssid * CTXDESC_CD_DWORDS;
+ +++
+ +++ idx = ssid >> CTXDESC_SPLIT;
+ +++ l1_desc = &cdcfg->l1_desc[idx];
+ +++ if (!l1_desc->l2ptr) {
+ +++ if (arm_smmu_alloc_cd_leaf_table(smmu, l1_desc))
+ +++ return NULL;
+ +++
+ +++ l1ptr = cdcfg->cdtab + idx * CTXDESC_L1_DESC_DWORDS;
+ +++ arm_smmu_write_cd_l1_desc(l1ptr, l1_desc);
+ +++ /* An invalid L1CD can be cached */
+ +++ arm_smmu_sync_cd(smmu_domain, ssid, false);
+ +++ }
+ +++ idx = ssid & (CTXDESC_L2_ENTRIES - 1);
+ +++ return l1_desc->l2ptr + idx * CTXDESC_CD_DWORDS;
+ +++}
+ +++
+ +++static int arm_smmu_write_ctx_desc(struct arm_smmu_domain *smmu_domain,
+ +++ int ssid, struct arm_smmu_ctx_desc *cd)
+ +++{
/*
- --- * We don't need to issue any invalidation here, as we'll invalidate
- --- * the STE when installing the new entry anyway.
+ +++ * This function handles the following cases:
+ +++ *
+ +++ * (1) Install primary CD, for normal DMA traffic (SSID = 0).
+ +++ * (2) Install a secondary CD, for SID+SSID traffic.
+ +++ * (3) Update ASID of a CD. Atomically write the first 64 bits of the
+ +++ * CD, then invalidate the old entry and mappings.
+ +++ * (4) Remove a secondary CD.
*/
- --- val = arm_smmu_cpu_tcr_to_cd(cfg->cd.tcr) |
+ +++ u64 val;
+ +++ bool cd_live;
+ +++ __le64 *cdptr;
+ +++ struct arm_smmu_device *smmu = smmu_domain->smmu;
+ +++
+ +++ if (WARN_ON(ssid >= (1 << smmu_domain->s1_cfg.s1cdmax)))
+ +++ return -E2BIG;
+ +++
+ +++ cdptr = arm_smmu_get_cd_ptr(smmu_domain, ssid);
+ +++ if (!cdptr)
+ +++ return -ENOMEM;
+ +++
+ +++ val = le64_to_cpu(cdptr[0]);
+ +++ cd_live = !!(val & CTXDESC_CD_0_V);
+ +++
+ +++ if (!cd) { /* (4) */
+ +++ val = 0;
+ +++ } else if (cd_live) { /* (3) */
+ +++ val &= ~CTXDESC_CD_0_ASID;
+ +++ val |= FIELD_PREP(CTXDESC_CD_0_ASID, cd->asid);
+ +++ /*
+ +++ * Until CD+TLB invalidation, both ASIDs may be used for tagging
+ +++ * this substream's traffic
+ +++ */
+ +++ } else { /* (1) and (2) */
+ +++ cdptr[1] = cpu_to_le64(cd->ttbr & CTXDESC_CD_1_TTB0_MASK);
+ +++ cdptr[2] = 0;
+ +++ cdptr[3] = cpu_to_le64(cd->mair);
+ +++
+ +++ /*
+ +++ * STE is live, and the SMMU might read dwords of this CD in any
+ +++ * order. Ensure that it observes valid values before reading
+ +++ * V=1.
+ +++ */
+ +++ arm_smmu_sync_cd(smmu_domain, ssid, true);
+ +++
+ +++ val = cd->tcr |
#ifdef __BIG_ENDIAN
- --- CTXDESC_CD_0_ENDI |
+ +++ CTXDESC_CD_0_ENDI |
#endif
- --- CTXDESC_CD_0_R | CTXDESC_CD_0_A | CTXDESC_CD_0_ASET |
- --- CTXDESC_CD_0_AA64 | FIELD_PREP(CTXDESC_CD_0_ASID, cfg->cd.asid) |
- --- CTXDESC_CD_0_V;
+ +++ CTXDESC_CD_0_R | CTXDESC_CD_0_A | CTXDESC_CD_0_ASET |
+ +++ CTXDESC_CD_0_AA64 |
+ +++ FIELD_PREP(CTXDESC_CD_0_ASID, cd->asid) |
+ +++ CTXDESC_CD_0_V;
+
- -- /* STALL_MODEL==0b10 && CD.S==0 is ILLEGAL */
- -- if (smmu->features & ARM_SMMU_FEAT_STALL_FORCE)
- -- val |= CTXDESC_CD_0_S;
+ +++ /* STALL_MODEL==0b10 && CD.S==0 is ILLEGAL */
+ +++ if (smmu->features & ARM_SMMU_FEAT_STALL_FORCE)
+ +++ val |= CTXDESC_CD_0_S;
+ +++ }
+
- -- cfg->cdptr[0] = cpu_to_le64(val);
+ +++ /*
+ +++ * The SMMU accesses 64-bit values atomically. See IHI0070Ca 3.21.3
+ +++ * "Configuration structures and configuration invalidation completion"
+ +++ *
+ +++ * The size of single-copy atomic reads made by the SMMU is
+ +++ * IMPLEMENTATION DEFINED but must be at least 64 bits. Any single
+ +++ * field within an aligned 64-bit span of a structure can be altered
+ +++ * without first making the structure invalid.
+ +++ */
+ +++ WRITE_ONCE(cdptr[0], cpu_to_le64(val));
+ +++ arm_smmu_sync_cd(smmu_domain, ssid, true);
+ +++ return 0;
+ +++}
+ +++
+ +++static int arm_smmu_alloc_cd_tables(struct arm_smmu_domain *smmu_domain)
+ +++{
+ +++ int ret;
+ +++ size_t l1size;
+ +++ size_t max_contexts;
+ +++ struct arm_smmu_device *smmu = smmu_domain->smmu;
+ +++ struct arm_smmu_s1_cfg *cfg = &smmu_domain->s1_cfg;
+ +++ struct arm_smmu_ctx_desc_cfg *cdcfg = &cfg->cdcfg;
+ +++
+ +++ max_contexts = 1 << cfg->s1cdmax;
+ +++
+ +++ if (!(smmu->features & ARM_SMMU_FEAT_2_LVL_CDTAB) ||
+ +++ max_contexts <= CTXDESC_L2_ENTRIES) {
+ +++ cfg->s1fmt = STRTAB_STE_0_S1FMT_LINEAR;
+ +++ cdcfg->num_l1_ents = max_contexts;
+
- -- val = cfg->cd.ttbr & CTXDESC_CD_1_TTB0_MASK;
- -- cfg->cdptr[1] = cpu_to_le64(val);
+ +++ l1size = max_contexts * (CTXDESC_CD_DWORDS << 3);
+ +++ } else {
+ +++ cfg->s1fmt = STRTAB_STE_0_S1FMT_64K_L2;
+ +++ cdcfg->num_l1_ents = DIV_ROUND_UP(max_contexts,
+ +++ CTXDESC_L2_ENTRIES);
+ +++
+ +++ cdcfg->l1_desc = devm_kcalloc(smmu->dev, cdcfg->num_l1_ents,
+ +++ sizeof(*cdcfg->l1_desc),
+ +++ GFP_KERNEL);
+ +++ if (!cdcfg->l1_desc)
+ +++ return -ENOMEM;
+ +++
+ +++ l1size = cdcfg->num_l1_ents * (CTXDESC_L1_DESC_DWORDS << 3);
+ +++ }
+ ++
- /* STALL_MODEL==0b10 && CD.S==0 is ILLEGAL */
- if (smmu->features & ARM_SMMU_FEAT_STALL_FORCE)
- val |= CTXDESC_CD_0_S;
+ +++ cdcfg->cdtab = dmam_alloc_coherent(smmu->dev, l1size, &cdcfg->cdtab_dma,
+ +++ GFP_KERNEL);
+ +++ if (!cdcfg->cdtab) {
+ +++ dev_warn(smmu->dev, "failed to allocate context descriptor\n");
+ +++ ret = -ENOMEM;
+ +++ goto err_free_l1;
+ +++ }
+ +++
+ +++ return 0;
+
- -- cfg->cdptr[3] = cpu_to_le64(cfg->cd.mair);
+ +++err_free_l1:
+ +++ if (cdcfg->l1_desc) {
+ +++ devm_kfree(smmu->dev, cdcfg->l1_desc);
+ +++ cdcfg->l1_desc = NULL;
+ +++ }
+ +++ return ret;
+ +++}
+ +++
+ +++static void arm_smmu_free_cd_tables(struct arm_smmu_domain *smmu_domain)
+ +++{
+ +++ int i;
+ +++ size_t size, l1size;
+ +++ struct arm_smmu_device *smmu = smmu_domain->smmu;
+ +++ struct arm_smmu_ctx_desc_cfg *cdcfg = &smmu_domain->s1_cfg.cdcfg;
+ +++
+ +++ if (cdcfg->l1_desc) {
+ +++ size = CTXDESC_L2_ENTRIES * (CTXDESC_CD_DWORDS << 3);
+ +++
+ +++ for (i = 0; i < cdcfg->num_l1_ents; i++) {
+ +++ if (!cdcfg->l1_desc[i].l2ptr)
+ +++ continue;
+ ++
- cfg->cdptr[0] = cpu_to_le64(val);
+ +++ dmam_free_coherent(smmu->dev, size,
+ +++ cdcfg->l1_desc[i].l2ptr,
+ +++ cdcfg->l1_desc[i].l2ptr_dma);
+ +++ }
+ +++ devm_kfree(smmu->dev, cdcfg->l1_desc);
+ +++ cdcfg->l1_desc = NULL;
+ ++
- val = cfg->cd.ttbr & CTXDESC_CD_1_TTB0_MASK;
- cfg->cdptr[1] = cpu_to_le64(val);
+ +++ l1size = cdcfg->num_l1_ents * (CTXDESC_L1_DESC_DWORDS << 3);
+ +++ } else {
+ +++ l1size = cdcfg->num_l1_ents * (CTXDESC_CD_DWORDS << 3);
+ +++ }
+ ++
- cfg->cdptr[3] = cpu_to_le64(cfg->cd.mair);
+ +++ dmam_free_coherent(smmu->dev, l1size, cdcfg->cdtab, cdcfg->cdtab_dma);
+ +++ cdcfg->cdtab_dma = 0;
+ +++ cdcfg->cdtab = NULL;
}
/* Stream table manipulation functions */
}
#endif
#ifdef CONFIG_FSL_MC_BUS
- --- if (!iommu_present(&fsl_mc_bus_type))
- --- bus_set_iommu(&fsl_mc_bus_type, &arm_smmu_ops);
+ +++ if (!iommu_present(&fsl_mc_bus_type)) {
+ +++ err = bus_set_iommu(&fsl_mc_bus_type, ops);
+ +++ if (err)
+ +++ goto err_reset_pci_ops;
+ +++ }
+#endif
+ +++ return 0;
+ +++
+ +++err_reset_pci_ops: __maybe_unused;
+ +++#ifdef CONFIG_PCI
+ +++ bus_set_iommu(&pci_bus_type, NULL);
+ ++ #endif
+ +++err_reset_amba_ops: __maybe_unused;
+ +++#ifdef CONFIG_ARM_AMBA
+ +++ bus_set_iommu(&amba_bustype, NULL);
+ +++#endif
+ +++err_reset_platform_ops: __maybe_unused;
+ +++ bus_set_iommu(&platform_bus_type, NULL);
+ +++ return err;
}
static int arm_smmu_device_probe(struct platform_device *pdev)
* ready to handle default domain setup as soon as any SMMU exists.
*/
if (!using_legacy_binding)
- --- arm_smmu_bus_init();
-
- return 0;
-}
+ +++ return arm_smmu_bus_init(&arm_smmu_ops);
-/*
- * With the legacy DT binding in play, though, we have no guarantees about
- * probe order, but then we're also not doing default domains, so we can
- * delay setting bus ops until we're sure every possible SMMU is ready,
- * and that way ensure that no add_device() calls get missed.
- */
-static int arm_smmu_legacy_bus_init(void)
-{
- if (using_legacy_binding)
- arm_smmu_bus_init();
return 0;
}
-device_initcall_sync(arm_smmu_legacy_bus_init);
- -- /*
- -- * With the legacy DT binding in play, though, we have no guarantees about
- -- * probe order, but then we're also not doing default domains, so we can
- -- * delay setting bus ops until we're sure every possible SMMU is ready,
- -- * and that way ensure that no add_device() calls get missed.
- -- */
- -- static int arm_smmu_legacy_bus_init(void)
- -- {
- -- if (using_legacy_binding)
- -- arm_smmu_bus_init();
- -- return 0;
- -- }
- -- device_initcall_sync(arm_smmu_legacy_bus_init);
- --
- ---static void arm_smmu_device_shutdown(struct platform_device *pdev)
+ +++static int arm_smmu_device_remove(struct platform_device *pdev)
{
struct arm_smmu_device *smmu = platform_get_drvdata(pdev);
projects / platform / kernel / linux-rpi.git / commitdiff