From 556f99ac886635e8da15528995f06d1d7028cfca Mon Sep 17 00:00:00 2001 From: Lu Baolu Date: Mon, 8 Nov 2021 14:13:49 +0800 Subject: iommu: Extend mutex lock scope in iommu_probe_device() Extend the scope of holding group->mutex so that it can cover the default domain check/attachment and direct mappings of reserved regions. Cc: Ashish Mhetre Fixes: 211ff31b3d33b ("iommu: Fix race condition during default domain allocation") Signed-off-by: Lu Baolu Link: https://lore.kernel.org/r/20211108061349.1985579-1-baolu.lu@linux.intel.com Signed-off-by: Joerg Roedel --- drivers/iommu/iommu.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) (limited to 'drivers/iommu') diff --git a/drivers/iommu/iommu.c b/drivers/iommu/iommu.c index dd7863e453a5..8b86406b7162 100644 --- a/drivers/iommu/iommu.c +++ b/drivers/iommu/iommu.c @@ -288,11 +288,11 @@ int iommu_probe_device(struct device *dev) */ mutex_lock(&group->mutex); iommu_alloc_default_domain(group, dev); - mutex_unlock(&group->mutex); if (group->default_domain) { ret = __iommu_attach_device(group->default_domain, dev); if (ret) { + mutex_unlock(&group->mutex); iommu_group_put(group); goto err_release; } @@ -300,6 +300,7 @@ int iommu_probe_device(struct device *dev) iommu_create_device_direct_mappings(group, dev); + mutex_unlock(&group->mutex); iommu_group_put(group); if (ops->probe_finalize) -- cgit v1.2.3-73-gaa49b From 9abe2ac834851a7d0b0756e295cf7a292c45ca53 Mon Sep 17 00:00:00 2001 From: Hector Martin Date: Sat, 20 Nov 2021 12:13:43 +0900 Subject: iommu/io-pgtable-arm: Fix table descriptor paddr formatting Table descriptors were being installed without properly formatting the address using paddr_to_iopte, which does not match up with the iopte_deref in __arm_lpae_map. This is incorrect for the LPAE pte format, as it does not handle the high bits properly. This was found on Apple T6000 DARTs, which require a new pte format (different shift); adding support for that to paddr_to_iopte/iopte_to_paddr caused it to break badly, as even <48-bit addresses would end up incorrect in that case. Fixes: 6c89928ff7a0 ("iommu/io-pgtable-arm: Support 52-bit physical address") Acked-by: Robin Murphy Signed-off-by: Hector Martin Link: https://lore.kernel.org/r/20211120031343.88034-1-marcan@marcan.st Signed-off-by: Joerg Roedel --- drivers/iommu/io-pgtable-arm.c | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) (limited to 'drivers/iommu') diff --git a/drivers/iommu/io-pgtable-arm.c b/drivers/iommu/io-pgtable-arm.c index dd9e47189d0d..94ff319ae8ac 100644 --- a/drivers/iommu/io-pgtable-arm.c +++ b/drivers/iommu/io-pgtable-arm.c @@ -315,11 +315,12 @@ static int arm_lpae_init_pte(struct arm_lpae_io_pgtable *data, static arm_lpae_iopte arm_lpae_install_table(arm_lpae_iopte *table, arm_lpae_iopte *ptep, arm_lpae_iopte curr, - struct io_pgtable_cfg *cfg) + struct arm_lpae_io_pgtable *data) { arm_lpae_iopte old, new; + struct io_pgtable_cfg *cfg = &data->iop.cfg; - new = __pa(table) | ARM_LPAE_PTE_TYPE_TABLE; + new = paddr_to_iopte(__pa(table), data) | ARM_LPAE_PTE_TYPE_TABLE; if (cfg->quirks & IO_PGTABLE_QUIRK_ARM_NS) new |= ARM_LPAE_PTE_NSTABLE; @@ -380,7 +381,7 @@ static int __arm_lpae_map(struct arm_lpae_io_pgtable *data, unsigned long iova, if (!cptep) return -ENOMEM; - pte = arm_lpae_install_table(cptep, ptep, 0, cfg); + pte = arm_lpae_install_table(cptep, ptep, 0, data); if (pte) __arm_lpae_free_pages(cptep, tblsz, cfg); } else if (!cfg->coherent_walk && !(pte & ARM_LPAE_PTE_SW_SYNC)) { @@ -592,7 +593,7 @@ static size_t arm_lpae_split_blk_unmap(struct arm_lpae_io_pgtable *data, __arm_lpae_init_pte(data, blk_paddr, pte, lvl, 1, &tablep[i]); } - pte = arm_lpae_install_table(tablep, ptep, blk_pte, cfg); + pte = arm_lpae_install_table(tablep, ptep, blk_pte, data); if (pte != blk_pte) { __arm_lpae_free_pages(tablep, tablesz, cfg); /* -- cgit v1.2.3-73-gaa49b From f0f07a8462dc2a889745b9e436b4d86d66ccc787 Mon Sep 17 00:00:00 2001 From: Jean-Philippe Brucker Date: Wed, 1 Dec 2021 17:33:22 +0000 Subject: iommu/virtio: Support bypass domains The VIRTIO_IOMMU_F_BYPASS_CONFIG feature adds a new flag to the ATTACH request, that creates a bypass domain. Use it to enable identity domains. When VIRTIO_IOMMU_F_BYPASS_CONFIG is not supported by the device, we currently fail attaching to an identity domain. Future patches will instead create identity mappings in this case. Reviewed-by: Kevin Tian Signed-off-by: Jean-Philippe Brucker Reviewed-by: Eric Auger Link: https://lore.kernel.org/r/20211201173323.1045819-3-jean-philippe@linaro.org Signed-off-by: Joerg Roedel --- drivers/iommu/virtio-iommu.c | 20 +++++++++++++++++++- 1 file changed, 19 insertions(+), 1 deletion(-) (limited to 'drivers/iommu') diff --git a/drivers/iommu/virtio-iommu.c b/drivers/iommu/virtio-iommu.c index 80930ce04a16..14dfee76fd19 100644 --- a/drivers/iommu/virtio-iommu.c +++ b/drivers/iommu/virtio-iommu.c @@ -71,6 +71,7 @@ struct viommu_domain { struct rb_root_cached mappings; unsigned long nr_endpoints; + bool bypass; }; struct viommu_endpoint { @@ -587,7 +588,9 @@ static struct iommu_domain *viommu_domain_alloc(unsigned type) { struct viommu_domain *vdomain; - if (type != IOMMU_DOMAIN_UNMANAGED && type != IOMMU_DOMAIN_DMA) + if (type != IOMMU_DOMAIN_UNMANAGED && + type != IOMMU_DOMAIN_DMA && + type != IOMMU_DOMAIN_IDENTITY) return NULL; vdomain = kzalloc(sizeof(*vdomain), GFP_KERNEL); @@ -630,6 +633,17 @@ static int viommu_domain_finalise(struct viommu_endpoint *vdev, vdomain->map_flags = viommu->map_flags; vdomain->viommu = viommu; + if (domain->type == IOMMU_DOMAIN_IDENTITY) { + if (!virtio_has_feature(viommu->vdev, + VIRTIO_IOMMU_F_BYPASS_CONFIG)) { + ida_free(&viommu->domain_ids, vdomain->id); + vdomain->viommu = NULL; + return -EOPNOTSUPP; + } + + vdomain->bypass = true; + } + return 0; } @@ -691,6 +705,9 @@ static int viommu_attach_dev(struct iommu_domain *domain, struct device *dev) .domain = cpu_to_le32(vdomain->id), }; + if (vdomain->bypass) + req.flags |= cpu_to_le32(VIRTIO_IOMMU_ATTACH_F_BYPASS); + for (i = 0; i < fwspec->num_ids; i++) { req.endpoint = cpu_to_le32(fwspec->ids[i]); @@ -1132,6 +1149,7 @@ static unsigned int features[] = { VIRTIO_IOMMU_F_DOMAIN_RANGE, VIRTIO_IOMMU_F_PROBE, VIRTIO_IOMMU_F_MMIO, + VIRTIO_IOMMU_F_BYPASS_CONFIG, }; static struct virtio_device_id id_table[] = { -- cgit v1.2.3-73-gaa49b From 5610979415649f3743a7c3de03ab46c9a3bfff16 Mon Sep 17 00:00:00 2001 From: Jean-Philippe Brucker Date: Wed, 1 Dec 2021 17:33:23 +0000 Subject: iommu/virtio: Sort reserved regions To ease identity mapping support, keep the list of reserved regions sorted. Reviewed-by: Eric Auger Reviewed-by: Kevin Tian Signed-off-by: Jean-Philippe Brucker Link: https://lore.kernel.org/r/20211201173323.1045819-4-jean-philippe@linaro.org Signed-off-by: Joerg Roedel --- drivers/iommu/virtio-iommu.c | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) (limited to 'drivers/iommu') diff --git a/drivers/iommu/virtio-iommu.c b/drivers/iommu/virtio-iommu.c index 14dfee76fd19..1b3c1f2741c6 100644 --- a/drivers/iommu/virtio-iommu.c +++ b/drivers/iommu/virtio-iommu.c @@ -423,7 +423,7 @@ static int viommu_add_resv_mem(struct viommu_endpoint *vdev, size_t size; u64 start64, end64; phys_addr_t start, end; - struct iommu_resv_region *region = NULL; + struct iommu_resv_region *region = NULL, *next; unsigned long prot = IOMMU_WRITE | IOMMU_NOEXEC | IOMMU_MMIO; start = start64 = le64_to_cpu(mem->start); @@ -454,7 +454,12 @@ static int viommu_add_resv_mem(struct viommu_endpoint *vdev, if (!region) return -ENOMEM; - list_add(®ion->list, &vdev->resv_regions); + /* Keep the list sorted */ + list_for_each_entry(next, &vdev->resv_regions, list) { + if (next->start > region->start) + break; + } + list_add_tail(®ion->list, &next->list); return 0; } -- cgit v1.2.3-73-gaa49b From c0c763598960153e10622ff0a802012a073174a0 Mon Sep 17 00:00:00 2001 From: Jean-Philippe Brucker Date: Wed, 1 Dec 2021 17:33:24 +0000 Subject: iommu/virtio: Pass end address to viommu_add_mapping() To support identity mappings, the virtio-iommu driver must be able to represent full 64-bit ranges internally. Pass (start, end) instead of (start, size) to viommu_add/del_mapping(). Clean comments. The one about the returned size was never true: when sweeping the whole address space the returned size will most certainly be smaller than 2^64. Reviewed-by: Eric Auger Reviewed-by: Kevin Tian Signed-off-by: Jean-Philippe Brucker Link: https://lore.kernel.org/r/20211201173323.1045819-5-jean-philippe@linaro.org Signed-off-by: Joerg Roedel --- drivers/iommu/virtio-iommu.c | 31 +++++++++++++++---------------- 1 file changed, 15 insertions(+), 16 deletions(-) (limited to 'drivers/iommu') diff --git a/drivers/iommu/virtio-iommu.c b/drivers/iommu/virtio-iommu.c index 1b3c1f2741c6..2fa370c2659c 100644 --- a/drivers/iommu/virtio-iommu.c +++ b/drivers/iommu/virtio-iommu.c @@ -311,8 +311,8 @@ out_unlock: * * On success, return the new mapping. Otherwise return NULL. */ -static int viommu_add_mapping(struct viommu_domain *vdomain, unsigned long iova, - phys_addr_t paddr, size_t size, u32 flags) +static int viommu_add_mapping(struct viommu_domain *vdomain, u64 iova, u64 end, + phys_addr_t paddr, u32 flags) { unsigned long irqflags; struct viommu_mapping *mapping; @@ -323,7 +323,7 @@ static int viommu_add_mapping(struct viommu_domain *vdomain, unsigned long iova, mapping->paddr = paddr; mapping->iova.start = iova; - mapping->iova.last = iova + size - 1; + mapping->iova.last = end; mapping->flags = flags; spin_lock_irqsave(&vdomain->mappings_lock, irqflags); @@ -338,26 +338,24 @@ static int viommu_add_mapping(struct viommu_domain *vdomain, unsigned long iova, * * @vdomain: the domain * @iova: start of the range - * @size: size of the range. A size of 0 corresponds to the entire address - * space. + * @end: end of the range * - * On success, returns the number of unmapped bytes (>= size) + * On success, returns the number of unmapped bytes */ static size_t viommu_del_mappings(struct viommu_domain *vdomain, - unsigned long iova, size_t size) + u64 iova, u64 end) { size_t unmapped = 0; unsigned long flags; - unsigned long last = iova + size - 1; struct viommu_mapping *mapping = NULL; struct interval_tree_node *node, *next; spin_lock_irqsave(&vdomain->mappings_lock, flags); - next = interval_tree_iter_first(&vdomain->mappings, iova, last); + next = interval_tree_iter_first(&vdomain->mappings, iova, end); while (next) { node = next; mapping = container_of(node, struct viommu_mapping, iova); - next = interval_tree_iter_next(node, iova, last); + next = interval_tree_iter_next(node, iova, end); /* Trying to split a mapping? */ if (mapping->iova.start < iova) @@ -656,8 +654,8 @@ static void viommu_domain_free(struct iommu_domain *domain) { struct viommu_domain *vdomain = to_viommu_domain(domain); - /* Free all remaining mappings (size 2^64) */ - viommu_del_mappings(vdomain, 0, 0); + /* Free all remaining mappings */ + viommu_del_mappings(vdomain, 0, ULLONG_MAX); if (vdomain->viommu) ida_free(&vdomain->viommu->domain_ids, vdomain->id); @@ -742,6 +740,7 @@ static int viommu_map(struct iommu_domain *domain, unsigned long iova, { int ret; u32 flags; + u64 end = iova + size - 1; struct virtio_iommu_req_map map; struct viommu_domain *vdomain = to_viommu_domain(domain); @@ -752,7 +751,7 @@ static int viommu_map(struct iommu_domain *domain, unsigned long iova, if (flags & ~vdomain->map_flags) return -EINVAL; - ret = viommu_add_mapping(vdomain, iova, paddr, size, flags); + ret = viommu_add_mapping(vdomain, iova, end, paddr, flags); if (ret) return ret; @@ -761,7 +760,7 @@ static int viommu_map(struct iommu_domain *domain, unsigned long iova, .domain = cpu_to_le32(vdomain->id), .virt_start = cpu_to_le64(iova), .phys_start = cpu_to_le64(paddr), - .virt_end = cpu_to_le64(iova + size - 1), + .virt_end = cpu_to_le64(end), .flags = cpu_to_le32(flags), }; @@ -770,7 +769,7 @@ static int viommu_map(struct iommu_domain *domain, unsigned long iova, ret = viommu_send_req_sync(vdomain->viommu, &map, sizeof(map)); if (ret) - viommu_del_mappings(vdomain, iova, size); + viommu_del_mappings(vdomain, iova, end); return ret; } @@ -783,7 +782,7 @@ static size_t viommu_unmap(struct iommu_domain *domain, unsigned long iova, struct virtio_iommu_req_unmap unmap; struct viommu_domain *vdomain = to_viommu_domain(domain); - unmapped = viommu_del_mappings(vdomain, iova, size); + unmapped = viommu_del_mappings(vdomain, iova, iova + size - 1); if (unmapped < size) return 0; -- cgit v1.2.3-73-gaa49b From b03cbca48d6408cea866057654675dd63a1bdcd3 Mon Sep 17 00:00:00 2001 From: Jean-Philippe Brucker Date: Wed, 1 Dec 2021 17:33:25 +0000 Subject: iommu/virtio: Support identity-mapped domains Support identity domains for devices that do not offer the VIRTIO_IOMMU_F_BYPASS_CONFIG feature, by creating 1:1 mappings between the virtual and physical address space. Identity domains created this way still perform noticeably better than DMA domains, because they don't have the overhead of setting up and tearing down mappings at runtime. The performance difference between this and bypass is minimal in comparison. It does not matter that the physical addresses in the identity mappings do not all correspond to memory. By enabling passthrough we are trusting the device driver and the device itself to only perform DMA to suitable locations. In some cases it may even be desirable to perform DMA to MMIO regions. Reviewed-by: Eric Auger Reviewed-by: Kevin Tian Signed-off-by: Jean-Philippe Brucker Link: https://lore.kernel.org/r/20211201173323.1045819-6-jean-philippe@linaro.org Signed-off-by: Joerg Roedel --- drivers/iommu/virtio-iommu.c | 61 +++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 57 insertions(+), 4 deletions(-) (limited to 'drivers/iommu') diff --git a/drivers/iommu/virtio-iommu.c b/drivers/iommu/virtio-iommu.c index 2fa370c2659c..6a8a52b4297b 100644 --- a/drivers/iommu/virtio-iommu.c +++ b/drivers/iommu/virtio-iommu.c @@ -375,6 +375,55 @@ static size_t viommu_del_mappings(struct viommu_domain *vdomain, return unmapped; } +/* + * Fill the domain with identity mappings, skipping the device's reserved + * regions. + */ +static int viommu_domain_map_identity(struct viommu_endpoint *vdev, + struct viommu_domain *vdomain) +{ + int ret; + struct iommu_resv_region *resv; + u64 iova = vdomain->domain.geometry.aperture_start; + u64 limit = vdomain->domain.geometry.aperture_end; + u32 flags = VIRTIO_IOMMU_MAP_F_READ | VIRTIO_IOMMU_MAP_F_WRITE; + unsigned long granule = 1UL << __ffs(vdomain->domain.pgsize_bitmap); + + iova = ALIGN(iova, granule); + limit = ALIGN_DOWN(limit + 1, granule) - 1; + + list_for_each_entry(resv, &vdev->resv_regions, list) { + u64 resv_start = ALIGN_DOWN(resv->start, granule); + u64 resv_end = ALIGN(resv->start + resv->length, granule) - 1; + + if (resv_end < iova || resv_start > limit) + /* No overlap */ + continue; + + if (resv_start > iova) { + ret = viommu_add_mapping(vdomain, iova, resv_start - 1, + (phys_addr_t)iova, flags); + if (ret) + goto err_unmap; + } + + if (resv_end >= limit) + return 0; + + iova = resv_end + 1; + } + + ret = viommu_add_mapping(vdomain, iova, limit, (phys_addr_t)iova, + flags); + if (ret) + goto err_unmap; + return 0; + +err_unmap: + viommu_del_mappings(vdomain, 0, iova); + return ret; +} + /* * viommu_replay_mappings - re-send MAP requests * @@ -637,14 +686,18 @@ static int viommu_domain_finalise(struct viommu_endpoint *vdev, vdomain->viommu = viommu; if (domain->type == IOMMU_DOMAIN_IDENTITY) { - if (!virtio_has_feature(viommu->vdev, - VIRTIO_IOMMU_F_BYPASS_CONFIG)) { + if (virtio_has_feature(viommu->vdev, + VIRTIO_IOMMU_F_BYPASS_CONFIG)) { + vdomain->bypass = true; + return 0; + } + + ret = viommu_domain_map_identity(vdev, vdomain); + if (ret) { ida_free(&viommu->domain_ids, vdomain->id); vdomain->viommu = NULL; return -EOPNOTSUPP; } - - vdomain->bypass = true; } return 0; -- cgit v1.2.3-73-gaa49b From c31112fbd4077a51a14ff338038c82e9571dc821 Mon Sep 17 00:00:00 2001 From: Rob Clark Date: Mon, 8 Nov 2021 09:17:23 -0800 Subject: iommu/arm-smmu-qcom: Fix TTBR0 read It is a 64b register, lets not lose the upper bits. Fixes: ab5df7b953d8 ("iommu/arm-smmu-qcom: Add an adreno-smmu-priv callback to get pagefault info") Signed-off-by: Rob Clark Reviewed-by: Bjorn Andersson Link: https://lore.kernel.org/r/20211108171724.470973-1-robdclark@gmail.com Signed-off-by: Will Deacon --- drivers/iommu/arm/arm-smmu/arm-smmu-qcom.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'drivers/iommu') diff --git a/drivers/iommu/arm/arm-smmu/arm-smmu-qcom.c b/drivers/iommu/arm/arm-smmu/arm-smmu-qcom.c index ca736b065dd0..40c91dd368a4 100644 --- a/drivers/iommu/arm/arm-smmu/arm-smmu-qcom.c +++ b/drivers/iommu/arm/arm-smmu/arm-smmu-qcom.c @@ -51,7 +51,7 @@ static void qcom_adreno_smmu_get_fault_info(const void *cookie, info->fsynr1 = arm_smmu_cb_read(smmu, cfg->cbndx, ARM_SMMU_CB_FSYNR1); info->far = arm_smmu_cb_readq(smmu, cfg->cbndx, ARM_SMMU_CB_FAR); info->cbfrsynra = arm_smmu_gr1_read(smmu, ARM_SMMU_GR1_CBFRSYNRA(cfg->cbndx)); - info->ttbr0 = arm_smmu_cb_read(smmu, cfg->cbndx, ARM_SMMU_CB_TTBR0); + info->ttbr0 = arm_smmu_cb_readq(smmu, cfg->cbndx, ARM_SMMU_CB_TTBR0); info->contextidr = arm_smmu_cb_read(smmu, cfg->cbndx, ARM_SMMU_CB_CONTEXTIDR); } -- cgit v1.2.3-73-gaa49b From cd76990c94bb2239e1bcf38087ce4be0512c1232 Mon Sep 17 00:00:00 2001 From: Vinod Koul Date: Wed, 1 Dec 2021 13:09:43 +0530 Subject: iommu: arm-smmu-impl: Add SM8450 qcom iommu implementation Add SM8450 qcom iommu implementation to the table of qcom_smmu_impl_of_match table which brings in iommu support for SM8450 SoC Signed-off-by: Vinod Koul Tested-by: Dmitry Baryshkov Acked-by: Konrad Dybcio Link: https://lore.kernel.org/r/20211201073943.3969549-3-vkoul@kernel.org Signed-off-by: Will Deacon --- drivers/iommu/arm/arm-smmu/arm-smmu-qcom.c | 1 + 1 file changed, 1 insertion(+) (limited to 'drivers/iommu') diff --git a/drivers/iommu/arm/arm-smmu/arm-smmu-qcom.c b/drivers/iommu/arm/arm-smmu/arm-smmu-qcom.c index 40c91dd368a4..ba6298c7140e 100644 --- a/drivers/iommu/arm/arm-smmu/arm-smmu-qcom.c +++ b/drivers/iommu/arm/arm-smmu/arm-smmu-qcom.c @@ -415,6 +415,7 @@ static const struct of_device_id __maybe_unused qcom_smmu_impl_of_match[] = { { .compatible = "qcom,sm8150-smmu-500" }, { .compatible = "qcom,sm8250-smmu-500" }, { .compatible = "qcom,sm8350-smmu-500" }, + { .compatible = "qcom,sm8450-smmu-500" }, { } }; -- cgit v1.2.3-73-gaa49b From 17d9a4b43b284c333bf2c0c347da28bf22fe7a97 Mon Sep 17 00:00:00 2001 From: Rikard Falkeborn Date: Sat, 4 Dec 2021 23:33:01 +0100 Subject: iommu/arm-smmu-v3: Constify arm_smmu_mmu_notifier_ops The only usage of arm_smmu_mmu_notifier_ops is to assign its address to the ops field in the mmu_notifier struct, which is a pointer to const struct mmu_notifier_ops. Make it const to allow the compiler to put it in read-only memory. Signed-off-by: Rikard Falkeborn Reviewed-by: Jean-Philippe Brucker Link: https://lore.kernel.org/r/20211204223301.100649-1-rikard.falkeborn@gmail.com Signed-off-by: Will Deacon --- drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3-sva.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'drivers/iommu') diff --git a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3-sva.c b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3-sva.c index ee66d1f4cb81..a737ba5f727e 100644 --- a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3-sva.c +++ b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3-sva.c @@ -220,7 +220,7 @@ static void arm_smmu_mmu_notifier_free(struct mmu_notifier *mn) kfree(mn_to_smmu(mn)); } -static struct mmu_notifier_ops arm_smmu_mmu_notifier_ops = { +static const struct mmu_notifier_ops arm_smmu_mmu_notifier_ops = { .invalidate_range = arm_smmu_mm_invalidate_range, .release = arm_smmu_mm_release, .free_notifier = arm_smmu_mmu_notifier_free, -- cgit v1.2.3-73-gaa49b From a556cfe4cabc6d79cbb7733f118bbb420b376fe6 Mon Sep 17 00:00:00 2001 From: Yunfei Wang Date: Tue, 7 Dec 2021 19:33:15 +0800 Subject: iommu/io-pgtable-arm-v7s: Add error handle for page table allocation failure In __arm_v7s_alloc_table function: iommu call kmem_cache_alloc to allocate page table, this function allocate memory may fail, when kmem_cache_alloc fails to allocate table, call virt_to_phys will be abnomal and return unexpected phys and goto out_free, then call kmem_cache_free to release table will trigger KE, __get_free_pages and free_pages have similar problem, so add error handle for page table allocation failure. Fixes: 29859aeb8a6e ("iommu/io-pgtable-arm-v7s: Abort allocation when table address overflows the PTE") Signed-off-by: Yunfei Wang Cc: # 5.10.* Acked-by: Robin Murphy Link: https://lore.kernel.org/r/20211207113315.29109-1-yf.wang@mediatek.com Signed-off-by: Will Deacon --- drivers/iommu/io-pgtable-arm-v7s.c | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) (limited to 'drivers/iommu') diff --git a/drivers/iommu/io-pgtable-arm-v7s.c b/drivers/iommu/io-pgtable-arm-v7s.c index bfb6acb651e5..be066c1503d3 100644 --- a/drivers/iommu/io-pgtable-arm-v7s.c +++ b/drivers/iommu/io-pgtable-arm-v7s.c @@ -246,13 +246,17 @@ static void *__arm_v7s_alloc_table(int lvl, gfp_t gfp, __GFP_ZERO | ARM_V7S_TABLE_GFP_DMA, get_order(size)); else if (lvl == 2) table = kmem_cache_zalloc(data->l2_tables, gfp); + + if (!table) + return NULL; + phys = virt_to_phys(table); if (phys != (arm_v7s_iopte)phys) { /* Doesn't fit in PTE */ dev_err(dev, "Page table does not fit in PTE: %pa", &phys); goto out_free; } - if (table && !cfg->coherent_walk) { + if (!cfg->coherent_walk) { dma = dma_map_single(dev, table, size, DMA_TO_DEVICE); if (dma_mapping_error(dev, dma)) goto out_free; -- cgit v1.2.3-73-gaa49b From 477436699e7801276fa7306e20318156cb535249 Mon Sep 17 00:00:00 2001 From: Zhou Wang Date: Tue, 7 Dec 2021 14:32:48 +0800 Subject: Revert "iommu/arm-smmu-v3: Decrease the queue size of evtq and priq" The commit f115f3c0d5d8 ("iommu/arm-smmu-v3: Decrease the queue size of evtq and priq") decreases evtq and priq, which may lead evtq/priq to be full with fault events, e.g HiSilicon ZIP/SEC/HPRE have maximum 1024 queues in one device, every queue could be binded with one process and trigger a fault event. So let's revert f115f3c0d5d8. In fact, if an implementation of SMMU really does not need so long evtq and priq, value of IDR1_EVTQS and IDR1_PRIQS can be set to proper ones. Signed-off-by: Zhou Wang Acked-by: Zhen Lei Link: https://lore.kernel.org/r/1638858768-9971-1-git-send-email-wangzhou1@hisilicon.com Signed-off-by: Will Deacon --- drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.h | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) (limited to 'drivers/iommu') diff --git a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.h b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.h index 4cb136f07914..cd48590ada30 100644 --- a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.h +++ b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.h @@ -184,7 +184,6 @@ #else #define Q_MAX_SZ_SHIFT (PAGE_SHIFT + MAX_ORDER - 1) #endif -#define Q_MIN_SZ_SHIFT (PAGE_SHIFT) /* * Stream table. @@ -374,7 +373,7 @@ /* Event queue */ #define EVTQ_ENT_SZ_SHIFT 5 #define EVTQ_ENT_DWORDS ((1 << EVTQ_ENT_SZ_SHIFT) >> 3) -#define EVTQ_MAX_SZ_SHIFT (Q_MIN_SZ_SHIFT - EVTQ_ENT_SZ_SHIFT) +#define EVTQ_MAX_SZ_SHIFT (Q_MAX_SZ_SHIFT - EVTQ_ENT_SZ_SHIFT) #define EVTQ_0_ID GENMASK_ULL(7, 0) @@ -400,7 +399,7 @@ /* PRI queue */ #define PRIQ_ENT_SZ_SHIFT 4 #define PRIQ_ENT_DWORDS ((1 << PRIQ_ENT_SZ_SHIFT) >> 3) -#define PRIQ_MAX_SZ_SHIFT (Q_MIN_SZ_SHIFT - PRIQ_ENT_SZ_SHIFT) +#define PRIQ_MAX_SZ_SHIFT (Q_MAX_SZ_SHIFT - PRIQ_ENT_SZ_SHIFT) #define PRIQ_0_SID GENMASK_ULL(31, 0) #define PRIQ_0_SSID GENMASK_ULL(51, 32) -- cgit v1.2.3-73-gaa49b From 4599d78a820eb0a8ce2b3a1bd619620837cf906e Mon Sep 17 00:00:00 2001 From: Kees Cook Date: Wed, 15 Dec 2021 15:24:32 -0800 Subject: iommu/vt-d: Use correctly sized arguments for bit field The find.h APIs are designed to be used only on unsigned long arguments. This can technically result in a over-read, but it is harmless in this case. Regardless, fix it to avoid the warning seen under -Warray-bounds, which we'd like to enable globally: In file included from ./include/linux/bitmap.h:9, from drivers/iommu/intel/iommu.c:17: drivers/iommu/intel/iommu.c: In function 'domain_context_mapping_one': ./include/linux/find.h:119:37: warning: array subscript 'long unsigned int[0]' is partly outside array bounds of 'int[1]' [-Warray-bounds] 119 | unsigned long val = *addr & GENMASK(size - 1, 0); | ^~~~~ drivers/iommu/intel/iommu.c:2115:18: note: while referencing 'max_pde' 2115 | int pds, max_pde; | ^~~~~~~ Signed-off-by: Kees Cook Acked-by: Yury Norov Link: https://lore.kernel.org/r/20211215232432.2069605-1-keescook@chromium.org Signed-off-by: Joerg Roedel --- drivers/iommu/intel/iommu.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'drivers/iommu') diff --git a/drivers/iommu/intel/iommu.c b/drivers/iommu/intel/iommu.c index b6a8f3282411..99f9e8229384 100644 --- a/drivers/iommu/intel/iommu.c +++ b/drivers/iommu/intel/iommu.c @@ -2112,10 +2112,10 @@ static void domain_exit(struct dmar_domain *domain) */ static inline unsigned long context_get_sm_pds(struct pasid_table *table) { - int pds, max_pde; + unsigned long pds, max_pde; max_pde = table->max_pasid >> PASID_PDE_SHIFT; - pds = find_first_bit((unsigned long *)&max_pde, MAX_NR_PASID_BITS); + pds = find_first_bit(&max_pde, MAX_NR_PASID_BITS); if (pds < 7) return 0; -- cgit v1.2.3-73-gaa49b From 4cb3600e5eaf629452c362ec3535b4f30716d4e7 Mon Sep 17 00:00:00 2001 From: Xiang wangx Date: Thu, 16 Dec 2021 16:33:02 +0800 Subject: iommu/virtio: Fix typo in a comment The double `as' in a comment is repeated, thus it should be removed. Signed-off-by: Xiang wangx Link: https://lore.kernel.org/r/20211216083302.18049-1-wangxiang@cdjrlc.com Signed-off-by: Joerg Roedel --- drivers/iommu/virtio-iommu.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'drivers/iommu') diff --git a/drivers/iommu/virtio-iommu.c b/drivers/iommu/virtio-iommu.c index 6a8a52b4297b..1f793219cac6 100644 --- a/drivers/iommu/virtio-iommu.c +++ b/drivers/iommu/virtio-iommu.c @@ -743,7 +743,7 @@ static int viommu_attach_dev(struct iommu_domain *domain, struct device *dev) /* * In the virtio-iommu device, when attaching the endpoint to a new - * domain, it is detached from the old one and, if as as a result the + * domain, it is detached from the old one and, if as a result the * old domain isn't attached to any endpoint, all mappings are removed * from the old domain and it is freed. * -- cgit v1.2.3-73-gaa49b From 972bf252f86062e50f9c9ea81f84f5df0e9f1302 Mon Sep 17 00:00:00 2001 From: John Garry via iommu Date: Tue, 7 Dec 2021 19:17:26 +0800 Subject: iommu/iova: Move fast alloc size roundup into alloc_iova_fast() It really is a property of the IOVA rcache code that we need to alloc a power-of-2 size, so relocate the functionality to resize into alloc_iova_fast(), rather than the callsites. Signed-off-by: John Garry Acked-by: Will Deacon Reviewed-by: Xie Yongji Acked-by: Jason Wang Acked-by: Michael S. Tsirkin Acked-by: Robin Murphy Link: https://lore.kernel.org/r/1638875846-23993-1-git-send-email-john.garry@huawei.com Signed-off-by: Joerg Roedel --- drivers/iommu/dma-iommu.c | 8 -------- drivers/iommu/iova.c | 9 +++++++++ drivers/vdpa/vdpa_user/iova_domain.c | 8 -------- 3 files changed, 9 insertions(+), 16 deletions(-) (limited to 'drivers/iommu') diff --git a/drivers/iommu/dma-iommu.c b/drivers/iommu/dma-iommu.c index b42e38a0dbe2..84dee53fe892 100644 --- a/drivers/iommu/dma-iommu.c +++ b/drivers/iommu/dma-iommu.c @@ -442,14 +442,6 @@ static dma_addr_t iommu_dma_alloc_iova(struct iommu_domain *domain, shift = iova_shift(iovad); iova_len = size >> shift; - /* - * Freeing non-power-of-two-sized allocations back into the IOVA caches - * will come back to bite us badly, so we have to waste a bit of space - * rounding up anything cacheable to make sure that can't happen. The - * order of the unadjusted size will still match upon freeing. - */ - if (iova_len < (1 << (IOVA_RANGE_CACHE_MAX_SIZE - 1))) - iova_len = roundup_pow_of_two(iova_len); dma_limit = min_not_zero(dma_limit, dev->bus_dma_limit); diff --git a/drivers/iommu/iova.c b/drivers/iommu/iova.c index 9e8bc802ac05..ff567cbc42f7 100644 --- a/drivers/iommu/iova.c +++ b/drivers/iommu/iova.c @@ -497,6 +497,15 @@ alloc_iova_fast(struct iova_domain *iovad, unsigned long size, unsigned long iova_pfn; struct iova *new_iova; + /* + * Freeing non-power-of-two-sized allocations back into the IOVA caches + * will come back to bite us badly, so we have to waste a bit of space + * rounding up anything cacheable to make sure that can't happen. The + * order of the unadjusted size will still match upon freeing. + */ + if (size < (1 << (IOVA_RANGE_CACHE_MAX_SIZE - 1))) + size = roundup_pow_of_two(size); + iova_pfn = iova_rcache_get(iovad, size, limit_pfn + 1); if (iova_pfn) return iova_pfn; diff --git a/drivers/vdpa/vdpa_user/iova_domain.c b/drivers/vdpa/vdpa_user/iova_domain.c index 1daae2608860..2b1143f11d8f 100644 --- a/drivers/vdpa/vdpa_user/iova_domain.c +++ b/drivers/vdpa/vdpa_user/iova_domain.c @@ -292,14 +292,6 @@ vduse_domain_alloc_iova(struct iova_domain *iovad, unsigned long iova_len = iova_align(iovad, size) >> shift; unsigned long iova_pfn; - /* - * Freeing non-power-of-two-sized allocations back into the IOVA caches - * will come back to bite us badly, so we have to waste a bit of space - * rounding up anything cacheable to make sure that can't happen. The - * order of the unadjusted size will still match upon freeing. - */ - if (iova_len < (1 << (IOVA_RANGE_CACHE_MAX_SIZE - 1))) - iova_len = roundup_pow_of_two(iova_len); iova_pfn = alloc_iova_fast(iovad, iova_len, limit >> shift, true); return iova_pfn << shift; -- cgit v1.2.3-73-gaa49b From a8d4a37d1bb93608501d0d0545f902061152669a Mon Sep 17 00:00:00 2001 From: Maxim Levitsky Date: Tue, 23 Nov 2021 18:10:34 +0200 Subject: iommu/amd: Restore GA log/tail pointer on host resume This will give IOMMU GA log a chance to work after resume from s3/s4. Fixes: 8bda0cfbdc1a6 ("iommu/amd: Detect and initialize guest vAPIC log") Signed-off-by: Maxim Levitsky Link: https://lore.kernel.org/r/20211123161038.48009-2-mlevitsk@redhat.com Signed-off-by: Joerg Roedel --- drivers/iommu/amd/init.c | 31 +++++++++++++++---------------- 1 file changed, 15 insertions(+), 16 deletions(-) (limited to 'drivers/iommu') diff --git a/drivers/iommu/amd/init.c b/drivers/iommu/amd/init.c index 1eacd43cb436..8dae85fcfc2e 100644 --- a/drivers/iommu/amd/init.c +++ b/drivers/iommu/amd/init.c @@ -806,16 +806,27 @@ static int iommu_ga_log_enable(struct amd_iommu *iommu) { #ifdef CONFIG_IRQ_REMAP u32 status, i; + u64 entry; if (!iommu->ga_log) return -EINVAL; - status = readl(iommu->mmio_base + MMIO_STATUS_OFFSET); - /* Check if already running */ - if (status & (MMIO_STATUS_GALOG_RUN_MASK)) + status = readl(iommu->mmio_base + MMIO_STATUS_OFFSET); + if (WARN_ON(status & (MMIO_STATUS_GALOG_RUN_MASK))) return 0; + entry = iommu_virt_to_phys(iommu->ga_log) | GA_LOG_SIZE_512; + memcpy_toio(iommu->mmio_base + MMIO_GA_LOG_BASE_OFFSET, + &entry, sizeof(entry)); + entry = (iommu_virt_to_phys(iommu->ga_log_tail) & + (BIT_ULL(52)-1)) & ~7ULL; + memcpy_toio(iommu->mmio_base + MMIO_GA_LOG_TAIL_OFFSET, + &entry, sizeof(entry)); + writel(0x00, iommu->mmio_base + MMIO_GA_HEAD_OFFSET); + writel(0x00, iommu->mmio_base + MMIO_GA_TAIL_OFFSET); + + iommu_feature_enable(iommu, CONTROL_GAINT_EN); iommu_feature_enable(iommu, CONTROL_GALOG_EN); @@ -825,7 +836,7 @@ static int iommu_ga_log_enable(struct amd_iommu *iommu) break; } - if (i >= LOOP_TIMEOUT) + if (WARN_ON(i >= LOOP_TIMEOUT)) return -EINVAL; #endif /* CONFIG_IRQ_REMAP */ return 0; @@ -834,8 +845,6 @@ static int iommu_ga_log_enable(struct amd_iommu *iommu) static int iommu_init_ga_log(struct amd_iommu *iommu) { #ifdef CONFIG_IRQ_REMAP - u64 entry; - if (!AMD_IOMMU_GUEST_IR_VAPIC(amd_iommu_guest_ir)) return 0; @@ -849,16 +858,6 @@ static int iommu_init_ga_log(struct amd_iommu *iommu) if (!iommu->ga_log_tail) goto err_out; - entry = iommu_virt_to_phys(iommu->ga_log) | GA_LOG_SIZE_512; - memcpy_toio(iommu->mmio_base + MMIO_GA_LOG_BASE_OFFSET, - &entry, sizeof(entry)); - entry = (iommu_virt_to_phys(iommu->ga_log_tail) & - (BIT_ULL(52)-1)) & ~7ULL; - memcpy_toio(iommu->mmio_base + MMIO_GA_LOG_TAIL_OFFSET, - &entry, sizeof(entry)); - writel(0x00, iommu->mmio_base + MMIO_GA_HEAD_OFFSET); - writel(0x00, iommu->mmio_base + MMIO_GA_TAIL_OFFSET); - return 0; err_out: free_ga_log(iommu); -- cgit v1.2.3-73-gaa49b From 01b297a48a26bcb96769505ac948db4603b72bd1 Mon Sep 17 00:00:00 2001 From: Maxim Levitsky Date: Tue, 23 Nov 2021 18:10:35 +0200 Subject: iommu/amd: X2apic mode: re-enable after resume Otherwise it is guaranteed to not work after the resume... Fixes: 66929812955bb ("iommu/amd: Add support for X2APIC IOMMU interrupts") Signed-off-by: Maxim Levitsky Link: https://lore.kernel.org/r/20211123161038.48009-3-mlevitsk@redhat.com Signed-off-by: Joerg Roedel --- drivers/iommu/amd/init.c | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) (limited to 'drivers/iommu') diff --git a/drivers/iommu/amd/init.c b/drivers/iommu/amd/init.c index 8dae85fcfc2e..b905604f434e 100644 --- a/drivers/iommu/amd/init.c +++ b/drivers/iommu/amd/init.c @@ -2172,7 +2172,6 @@ static int iommu_setup_intcapxt(struct amd_iommu *iommu) return ret; } - iommu_feature_enable(iommu, CONTROL_INTCAPXT_EN); return 0; } @@ -2195,6 +2194,10 @@ static int iommu_init_irq(struct amd_iommu *iommu) iommu->int_enabled = true; enable_faults: + + if (amd_iommu_xt_mode == IRQ_REMAP_X2APIC_MODE) + iommu_feature_enable(iommu, CONTROL_INTCAPXT_EN); + iommu_feature_enable(iommu, CONTROL_EVT_INT_EN); if (iommu->ppr_log != NULL) -- cgit v1.2.3-73-gaa49b From 4691f79d62a637958f7b5f55c232a65399500b7a Mon Sep 17 00:00:00 2001 From: Maxim Levitsky Date: Tue, 23 Nov 2021 18:10:36 +0200 Subject: iommu/amd: X2apic mode: setup the INTX registers on mask/unmask This is more logically correct and will also allow us to to use mask/unmask logic to restore INTX setttings after the resume from s3/s4. Fixes: 66929812955bb ("iommu/amd: Add support for X2APIC IOMMU interrupts") Signed-off-by: Maxim Levitsky Link: https://lore.kernel.org/r/20211123161038.48009-4-mlevitsk@redhat.com Signed-off-by: Joerg Roedel --- drivers/iommu/amd/init.c | 65 ++++++++++++++++++++++++------------------------ 1 file changed, 33 insertions(+), 32 deletions(-) (limited to 'drivers/iommu') diff --git a/drivers/iommu/amd/init.c b/drivers/iommu/amd/init.c index b905604f434e..9e895bb8086a 100644 --- a/drivers/iommu/amd/init.c +++ b/drivers/iommu/amd/init.c @@ -2015,48 +2015,18 @@ union intcapxt { }; } __attribute__ ((packed)); -/* - * There isn't really any need to mask/unmask at the irqchip level because - * the 64-bit INTCAPXT registers can be updated atomically without tearing - * when the affinity is being updated. - */ -static void intcapxt_unmask_irq(struct irq_data *data) -{ -} - -static void intcapxt_mask_irq(struct irq_data *data) -{ -} static struct irq_chip intcapxt_controller; static int intcapxt_irqdomain_activate(struct irq_domain *domain, struct irq_data *irqd, bool reserve) { - struct amd_iommu *iommu = irqd->chip_data; - struct irq_cfg *cfg = irqd_cfg(irqd); - union intcapxt xt; - - xt.capxt = 0ULL; - xt.dest_mode_logical = apic->dest_mode_logical; - xt.vector = cfg->vector; - xt.destid_0_23 = cfg->dest_apicid & GENMASK(23, 0); - xt.destid_24_31 = cfg->dest_apicid >> 24; - - /** - * Current IOMMU implemtation uses the same IRQ for all - * 3 IOMMU interrupts. - */ - writeq(xt.capxt, iommu->mmio_base + MMIO_INTCAPXT_EVT_OFFSET); - writeq(xt.capxt, iommu->mmio_base + MMIO_INTCAPXT_PPR_OFFSET); - writeq(xt.capxt, iommu->mmio_base + MMIO_INTCAPXT_GALOG_OFFSET); return 0; } static void intcapxt_irqdomain_deactivate(struct irq_domain *domain, struct irq_data *irqd) { - intcapxt_mask_irq(irqd); } @@ -2090,6 +2060,38 @@ static void intcapxt_irqdomain_free(struct irq_domain *domain, unsigned int virq irq_domain_free_irqs_top(domain, virq, nr_irqs); } + +static void intcapxt_unmask_irq(struct irq_data *irqd) +{ + struct amd_iommu *iommu = irqd->chip_data; + struct irq_cfg *cfg = irqd_cfg(irqd); + union intcapxt xt; + + xt.capxt = 0ULL; + xt.dest_mode_logical = apic->dest_mode_logical; + xt.vector = cfg->vector; + xt.destid_0_23 = cfg->dest_apicid & GENMASK(23, 0); + xt.destid_24_31 = cfg->dest_apicid >> 24; + + /** + * Current IOMMU implementation uses the same IRQ for all + * 3 IOMMU interrupts. + */ + writeq(xt.capxt, iommu->mmio_base + MMIO_INTCAPXT_EVT_OFFSET); + writeq(xt.capxt, iommu->mmio_base + MMIO_INTCAPXT_PPR_OFFSET); + writeq(xt.capxt, iommu->mmio_base + MMIO_INTCAPXT_GALOG_OFFSET); +} + +static void intcapxt_mask_irq(struct irq_data *irqd) +{ + struct amd_iommu *iommu = irqd->chip_data; + + writeq(0, iommu->mmio_base + MMIO_INTCAPXT_EVT_OFFSET); + writeq(0, iommu->mmio_base + MMIO_INTCAPXT_PPR_OFFSET); + writeq(0, iommu->mmio_base + MMIO_INTCAPXT_GALOG_OFFSET); +} + + static int intcapxt_set_affinity(struct irq_data *irqd, const struct cpumask *mask, bool force) { @@ -2099,8 +2101,7 @@ static int intcapxt_set_affinity(struct irq_data *irqd, ret = parent->chip->irq_set_affinity(parent, mask, force); if (ret < 0 || ret == IRQ_SET_MASK_OK_DONE) return ret; - - return intcapxt_irqdomain_activate(irqd->domain, irqd, false); + return 0; } static struct irq_chip intcapxt_controller = { -- cgit v1.2.3-73-gaa49b From 1980105e3cfc2215c75b4f6b172661d675c467d1 Mon Sep 17 00:00:00 2001 From: Maxim Levitsky Date: Tue, 23 Nov 2021 18:10:37 +0200 Subject: iommu/amd: X2apic mode: mask/unmask interrupts on suspend/resume Use IRQCHIP_MASK_ON_SUSPEND to make the core irq code to mask the iommu interrupt on suspend and unmask it on the resume. Since now the unmask function updates the INTX settings, that will restore them on resume from s3/s4. Since IRQCHIP_MASK_ON_SUSPEND is only effective for interrupts which are not wakeup sources, remove IRQCHIP_SKIP_SET_WAKE flag and instead implement a dummy .irq_set_wake which doesn't allow the interrupt to become a wakeup source. Fixes: 66929812955bb ("iommu/amd: Add support for X2APIC IOMMU interrupts") Signed-off-by: Maxim Levitsky Link: https://lore.kernel.org/r/20211123161038.48009-5-mlevitsk@redhat.com Signed-off-by: Joerg Roedel --- drivers/iommu/amd/init.c | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) (limited to 'drivers/iommu') diff --git a/drivers/iommu/amd/init.c b/drivers/iommu/amd/init.c index 9e895bb8086a..b94822fc2c9f 100644 --- a/drivers/iommu/amd/init.c +++ b/drivers/iommu/amd/init.c @@ -2104,6 +2104,11 @@ static int intcapxt_set_affinity(struct irq_data *irqd, return 0; } +static int intcapxt_set_wake(struct irq_data *irqd, unsigned int on) +{ + return on ? -EOPNOTSUPP : 0; +} + static struct irq_chip intcapxt_controller = { .name = "IOMMU-MSI", .irq_unmask = intcapxt_unmask_irq, @@ -2111,7 +2116,8 @@ static struct irq_chip intcapxt_controller = { .irq_ack = irq_chip_ack_parent, .irq_retrigger = irq_chip_retrigger_hierarchy, .irq_set_affinity = intcapxt_set_affinity, - .flags = IRQCHIP_SKIP_SET_WAKE, + .irq_set_wake = intcapxt_set_wake, + .flags = IRQCHIP_MASK_ON_SUSPEND, }; static const struct irq_domain_ops intcapxt_domain_ops = { -- cgit v1.2.3-73-gaa49b From 575f5cfb13c84f324f9898383fa4a5694e53c9ef Mon Sep 17 00:00:00 2001 From: Maxim Levitsky Date: Tue, 23 Nov 2021 18:10:38 +0200 Subject: iommu/amd: Remove useless irq affinity notifier iommu->intcapxt_notify field is no longer used after a switch to a separate domain was done Fixes: d1adcfbb520c ("iommu/amd: Fix IOMMU interrupt generation in X2APIC mode") Signed-off-by: Maxim Levitsky Link: https://lore.kernel.org/r/20211123161038.48009-6-mlevitsk@redhat.com Signed-off-by: Joerg Roedel --- drivers/iommu/amd/amd_iommu_types.h | 2 -- 1 file changed, 2 deletions(-) (limited to 'drivers/iommu') diff --git a/drivers/iommu/amd/amd_iommu_types.h b/drivers/iommu/amd/amd_iommu_types.h index 867535eb0ce9..ffc89c4fb120 100644 --- a/drivers/iommu/amd/amd_iommu_types.h +++ b/drivers/iommu/amd/amd_iommu_types.h @@ -645,8 +645,6 @@ struct amd_iommu { /* DebugFS Info */ struct dentry *debugfs; #endif - /* IRQ notifier for IntCapXT interrupt */ - struct irq_affinity_notify intcapxt_notify; }; static inline struct amd_iommu *dev_to_amd_iommu(struct device *dev) -- cgit v1.2.3-73-gaa49b From bb7125739611acbbce8182ed98ad700f03ebac18 Mon Sep 17 00:00:00 2001 From: Christophe JAILLET Date: Fri, 17 Dec 2021 16:38:14 +0800 Subject: iommu/vt-d: Use bitmap_zalloc() when applicable 'iommu->domain_ids' is a bitmap. So use 'bitmap_zalloc()' to simplify code and improve the semantic. Also change the corresponding 'kfree()' into 'bitmap_free()' to keep consistency. Signed-off-by: Christophe JAILLET Link: https://lore.kernel.org/r/cb7a3e0a8d522447a06298a4f244c3df072f948b.1635018498.git.christophe.jaillet@wanadoo.fr Signed-off-by: Lu Baolu Link: https://lore.kernel.org/r/20211217083817.1745419-2-baolu.lu@linux.intel.com Signed-off-by: Joerg Roedel --- drivers/iommu/intel/iommu.c | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) (limited to 'drivers/iommu') diff --git a/drivers/iommu/intel/iommu.c b/drivers/iommu/intel/iommu.c index 99f9e8229384..16e5fe35b3dc 100644 --- a/drivers/iommu/intel/iommu.c +++ b/drivers/iommu/intel/iommu.c @@ -1878,17 +1878,16 @@ static void iommu_disable_translation(struct intel_iommu *iommu) static int iommu_init_domains(struct intel_iommu *iommu) { - u32 ndomains, nlongs; + u32 ndomains; size_t size; ndomains = cap_ndoms(iommu->cap); pr_debug("%s: Number of Domains supported <%d>\n", iommu->name, ndomains); - nlongs = BITS_TO_LONGS(ndomains); spin_lock_init(&iommu->lock); - iommu->domain_ids = kcalloc(nlongs, sizeof(unsigned long), GFP_KERNEL); + iommu->domain_ids = bitmap_zalloc(ndomains, GFP_KERNEL); if (!iommu->domain_ids) return -ENOMEM; @@ -1903,7 +1902,7 @@ static int iommu_init_domains(struct intel_iommu *iommu) if (!iommu->domains || !iommu->domains[0]) { pr_err("%s: Allocating domain array failed\n", iommu->name); - kfree(iommu->domain_ids); + bitmap_free(iommu->domain_ids); kfree(iommu->domains); iommu->domain_ids = NULL; iommu->domains = NULL; @@ -1964,7 +1963,7 @@ static void free_dmar_iommu(struct intel_iommu *iommu) for (i = 0; i < elems; i++) kfree(iommu->domains[i]); kfree(iommu->domains); - kfree(iommu->domain_ids); + bitmap_free(iommu->domain_ids); iommu->domains = NULL; iommu->domain_ids = NULL; } -- cgit v1.2.3-73-gaa49b From f5209f91272295c6c3ee0cd5f9589baf93cd00e8 Mon Sep 17 00:00:00 2001 From: Kefeng Wang Date: Fri, 17 Dec 2021 16:38:15 +0800 Subject: iommu/vt-d: Drop duplicate check in dma_pte_free_pagetable() The BUG_ON check exists in dma_pte_clear_range(), kill the duplicate check. Signed-off-by: Kefeng Wang Link: https://lore.kernel.org/r/20211025032307.182974-1-wangkefeng.wang@huawei.com Signed-off-by: Lu Baolu Link: https://lore.kernel.org/r/20211217083817.1745419-3-baolu.lu@linux.intel.com Signed-off-by: Joerg Roedel --- drivers/iommu/intel/iommu.c | 4 ---- 1 file changed, 4 deletions(-) (limited to 'drivers/iommu') diff --git a/drivers/iommu/intel/iommu.c b/drivers/iommu/intel/iommu.c index 16e5fe35b3dc..261d931ee76e 100644 --- a/drivers/iommu/intel/iommu.c +++ b/drivers/iommu/intel/iommu.c @@ -1280,10 +1280,6 @@ static void dma_pte_free_pagetable(struct dmar_domain *domain, unsigned long last_pfn, int retain_level) { - BUG_ON(!domain_pfn_supported(domain, start_pfn)); - BUG_ON(!domain_pfn_supported(domain, last_pfn)); - BUG_ON(start_pfn > last_pfn); - dma_pte_clear_range(domain, start_pfn, last_pfn); /* We don't need lock here; nobody else touches the iova range */ -- cgit v1.2.3-73-gaa49b From c95a9c278df8733a4370182a20f1355d6a81c235 Mon Sep 17 00:00:00 2001 From: Maíra Canal Date: Fri, 17 Dec 2021 16:38:16 +0800 Subject: iommu/vt-d: Remove unused dma_to_mm_pfn function MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Remove dma_to_buf_pfn function, which is not used in the codebase. This was pointed by clang with the following warning: 'dma_to_mm_pfn' [-Wunused-function] static inline unsigned long dma_to_mm_pfn(unsigned long dma_pfn) ^ https://lore.kernel.org/r/YYhY7GqlrcTZlzuA@fedora drivers/iommu/intel/iommu.c:136:29: warning: unused function Signed-off-by: Maíra Canal Reviewed-by: Nick Desaulniers Signed-off-by: Lu Baolu Link: https://lore.kernel.org/r/20211217083817.1745419-4-baolu.lu@linux.intel.com Signed-off-by: Joerg Roedel --- drivers/iommu/intel/iommu.c | 5 ----- 1 file changed, 5 deletions(-) (limited to 'drivers/iommu') diff --git a/drivers/iommu/intel/iommu.c b/drivers/iommu/intel/iommu.c index 261d931ee76e..56671c7f4c08 100644 --- a/drivers/iommu/intel/iommu.c +++ b/drivers/iommu/intel/iommu.c @@ -133,11 +133,6 @@ static inline unsigned long lvl_to_nr_pages(unsigned int lvl) /* VT-d pages must always be _smaller_ than MM pages. Otherwise things are never going to work. */ -static inline unsigned long dma_to_mm_pfn(unsigned long dma_pfn) -{ - return dma_pfn >> (PAGE_SHIFT - VTD_PAGE_SHIFT); -} - static inline unsigned long mm_to_dma_pfn(unsigned long mm_pfn) { return mm_pfn << (PAGE_SHIFT - VTD_PAGE_SHIFT); -- cgit v1.2.3-73-gaa49b From 664c0b58e0252f20c66084b467525fd9fb2892eb Mon Sep 17 00:00:00 2001 From: Paul Menzel Date: Fri, 17 Dec 2021 14:49:16 +0100 Subject: iommu/amd: Fix typo in *glues … together* in comment MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Paul Menzel Link: https://lore.kernel.org/r/20211217134916.43698-1-pmenzel@molgen.mpg.de Signed-off-by: Joerg Roedel --- drivers/iommu/amd/init.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'drivers/iommu') diff --git a/drivers/iommu/amd/init.c b/drivers/iommu/amd/init.c index b94822fc2c9f..dc338acf3338 100644 --- a/drivers/iommu/amd/init.c +++ b/drivers/iommu/amd/init.c @@ -1522,7 +1522,7 @@ static void amd_iommu_ats_write_check_workaround(struct amd_iommu *iommu) } /* - * This function clues the initialization function for one IOMMU + * This function glues the initialization function for one IOMMU * together and also allocates the command buffer and programs the * hardware. It does NOT enable the IOMMU. This is done afterwards. */ -- cgit v1.2.3-73-gaa49b From d7061627d701c90e1cac1e1e60c45292f64f3470 Mon Sep 17 00:00:00 2001 From: Xiongfeng Wang Date: Fri, 17 Dec 2021 15:30:55 +0000 Subject: iommu/iova: Fix race between FQ timeout and teardown It turns out to be possible for hotplugging out a device to reach the stage of tearing down the device's group and default domain before the domain's flush queue has drained naturally. At this point, it is then possible for the timeout to expire just before the del_timer() call in free_iova_flush_queue(), such that we then proceed to free the FQ resources while fq_flush_timeout() is still accessing them on another CPU. Crashes due to this have been observed in the wild while removing NVMe devices. Close the race window by using del_timer_sync() to safely wait for any active timeout handler to finish before we start to free things. We already avoid any locking in free_iova_flush_queue() since the FQ is supposed to be inactive anyway, so the potential deadlock scenario does not apply. Fixes: 9a005a800ae8 ("iommu/iova: Add flush timer") Reviewed-by: John Garry Signed-off-by: Xiongfeng Wang [ rm: rewrite commit message ] Signed-off-by: Robin Murphy Link: https://lore.kernel.org/r/0a365e5b07f14b7344677ad6a9a734966a8422ce.1639753638.git.robin.murphy@arm.com Signed-off-by: Joerg Roedel --- drivers/iommu/iova.c | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) (limited to 'drivers/iommu') diff --git a/drivers/iommu/iova.c b/drivers/iommu/iova.c index ff567cbc42f7..670211e41771 100644 --- a/drivers/iommu/iova.c +++ b/drivers/iommu/iova.c @@ -83,8 +83,7 @@ static void free_iova_flush_queue(struct iova_domain *iovad) if (!has_iova_flush_queue(iovad)) return; - if (timer_pending(&iovad->fq_timer)) - del_timer(&iovad->fq_timer); + del_timer_sync(&iovad->fq_timer); fq_destroy_all_entries(iovad); -- cgit v1.2.3-73-gaa49b From d5c383f2c98ac58c210b266cdaf7b86bc32d1ad1 Mon Sep 17 00:00:00 2001 From: Robin Murphy Date: Fri, 17 Dec 2021 15:30:56 +0000 Subject: iommu/iova: Squash entry_dtor abstraction All flush queues are driven by iommu-dma now, so there is no need to abstract entry_dtor or its data any more. Squash the now-canonical implementation directly into the IOVA code to get it out of the way. Reviewed-by: John Garry Reviewed-by: Christoph Hellwig Signed-off-by: Robin Murphy Link: https://lore.kernel.org/r/2260f8de00ab5e0f9d2a1cf8978e6ae7cd4f182c.1639753638.git.robin.murphy@arm.com Signed-off-by: Joerg Roedel --- drivers/iommu/dma-iommu.c | 17 ++--------------- drivers/iommu/iova.c | 28 +++++++++++++++------------- include/linux/iova.h | 26 +++----------------------- 3 files changed, 20 insertions(+), 51 deletions(-) (limited to 'drivers/iommu') diff --git a/drivers/iommu/dma-iommu.c b/drivers/iommu/dma-iommu.c index 84dee53fe892..6691f3cd768f 100644 --- a/drivers/iommu/dma-iommu.c +++ b/drivers/iommu/dma-iommu.c @@ -64,18 +64,6 @@ static int __init iommu_dma_forcedac_setup(char *str) } early_param("iommu.forcedac", iommu_dma_forcedac_setup); -static void iommu_dma_entry_dtor(unsigned long data) -{ - struct page *freelist = (struct page *)data; - - while (freelist) { - unsigned long p = (unsigned long)page_address(freelist); - - freelist = freelist->freelist; - free_page(p); - } -} - static inline size_t cookie_msi_granule(struct iommu_dma_cookie *cookie) { if (cookie->type == IOMMU_DMA_IOVA_COOKIE) @@ -324,8 +312,7 @@ int iommu_dma_init_fq(struct iommu_domain *domain) if (cookie->fq_domain) return 0; - ret = init_iova_flush_queue(&cookie->iovad, iommu_dma_flush_iotlb_all, - iommu_dma_entry_dtor); + ret = init_iova_flush_queue(&cookie->iovad, iommu_dma_flush_iotlb_all); if (ret) { pr_warn("iova flush queue initialization failed\n"); return ret; @@ -471,7 +458,7 @@ static void iommu_dma_free_iova(struct iommu_dma_cookie *cookie, else if (gather && gather->queued) queue_iova(iovad, iova_pfn(iovad, iova), size >> iova_shift(iovad), - (unsigned long)gather->freelist); + gather->freelist); else free_iova_fast(iovad, iova_pfn(iovad, iova), size >> iova_shift(iovad)); diff --git a/drivers/iommu/iova.c b/drivers/iommu/iova.c index 670211e41771..541857ca4fd5 100644 --- a/drivers/iommu/iova.c +++ b/drivers/iommu/iova.c @@ -91,11 +91,9 @@ static void free_iova_flush_queue(struct iova_domain *iovad) iovad->fq = NULL; iovad->flush_cb = NULL; - iovad->entry_dtor = NULL; } -int init_iova_flush_queue(struct iova_domain *iovad, - iova_flush_cb flush_cb, iova_entry_dtor entry_dtor) +int init_iova_flush_queue(struct iova_domain *iovad, iova_flush_cb flush_cb) { struct iova_fq __percpu *queue; int cpu; @@ -108,7 +106,6 @@ int init_iova_flush_queue(struct iova_domain *iovad, return -ENOMEM; iovad->flush_cb = flush_cb; - iovad->entry_dtor = entry_dtor; for_each_possible_cpu(cpu) { struct iova_fq *fq; @@ -547,6 +544,16 @@ free_iova_fast(struct iova_domain *iovad, unsigned long pfn, unsigned long size) } EXPORT_SYMBOL_GPL(free_iova_fast); +static void fq_entry_dtor(struct page *freelist) +{ + while (freelist) { + unsigned long p = (unsigned long)page_address(freelist); + + freelist = freelist->freelist; + free_page(p); + } +} + #define fq_ring_for_each(i, fq) \ for ((i) = (fq)->head; (i) != (fq)->tail; (i) = ((i) + 1) % IOVA_FQ_SIZE) @@ -579,9 +586,7 @@ static void fq_ring_free(struct iova_domain *iovad, struct iova_fq *fq) if (fq->entries[idx].counter >= counter) break; - if (iovad->entry_dtor) - iovad->entry_dtor(fq->entries[idx].data); - + fq_entry_dtor(fq->entries[idx].freelist); free_iova_fast(iovad, fq->entries[idx].iova_pfn, fq->entries[idx].pages); @@ -606,15 +611,12 @@ static void fq_destroy_all_entries(struct iova_domain *iovad) * bother to free iovas, just call the entry_dtor on all remaining * entries. */ - if (!iovad->entry_dtor) - return; - for_each_possible_cpu(cpu) { struct iova_fq *fq = per_cpu_ptr(iovad->fq, cpu); int idx; fq_ring_for_each(idx, fq) - iovad->entry_dtor(fq->entries[idx].data); + fq_entry_dtor(fq->entries[idx].freelist); } } @@ -639,7 +641,7 @@ static void fq_flush_timeout(struct timer_list *t) void queue_iova(struct iova_domain *iovad, unsigned long pfn, unsigned long pages, - unsigned long data) + struct page *freelist) { struct iova_fq *fq; unsigned long flags; @@ -673,7 +675,7 @@ void queue_iova(struct iova_domain *iovad, fq->entries[idx].iova_pfn = pfn; fq->entries[idx].pages = pages; - fq->entries[idx].data = data; + fq->entries[idx].freelist = freelist; fq->entries[idx].counter = atomic64_read(&iovad->fq_flush_start_cnt); spin_unlock_irqrestore(&fq->lock, flags); diff --git a/include/linux/iova.h b/include/linux/iova.h index 71d8a2de6635..e746d8e41449 100644 --- a/include/linux/iova.h +++ b/include/linux/iova.h @@ -40,9 +40,6 @@ struct iova_domain; /* Call-Back from IOVA code into IOMMU drivers */ typedef void (* iova_flush_cb)(struct iova_domain *domain); -/* Destructor for per-entry data */ -typedef void (* iova_entry_dtor)(unsigned long data); - /* Number of entries per Flush Queue */ #define IOVA_FQ_SIZE 256 @@ -53,7 +50,7 @@ typedef void (* iova_entry_dtor)(unsigned long data); struct iova_fq_entry { unsigned long iova_pfn; unsigned long pages; - unsigned long data; + struct page *freelist; u64 counter; /* Flush counter when this entrie was added */ }; @@ -88,9 +85,6 @@ struct iova_domain { iova_flush_cb flush_cb; /* Call-Back function to flush IOMMU TLBs */ - iova_entry_dtor entry_dtor; /* IOMMU driver specific destructor for - iova entry */ - struct timer_list fq_timer; /* Timer to regularily empty the flush-queues */ atomic_t fq_timer_on; /* 1 when timer is active, 0 @@ -146,15 +140,14 @@ void free_iova_fast(struct iova_domain *iovad, unsigned long pfn, unsigned long size); void queue_iova(struct iova_domain *iovad, unsigned long pfn, unsigned long pages, - unsigned long data); + struct page *freelist); unsigned long alloc_iova_fast(struct iova_domain *iovad, unsigned long size, unsigned long limit_pfn, bool flush_rcache); struct iova *reserve_iova(struct iova_domain *iovad, unsigned long pfn_lo, unsigned long pfn_hi); void init_iova_domain(struct iova_domain *iovad, unsigned long granule, unsigned long start_pfn); -int init_iova_flush_queue(struct iova_domain *iovad, - iova_flush_cb flush_cb, iova_entry_dtor entry_dtor); +int init_iova_flush_queue(struct iova_domain *iovad, iova_flush_cb flush_cb); struct iova *find_iova(struct iova_domain *iovad, unsigned long pfn); void put_iova_domain(struct iova_domain *iovad); #else @@ -189,12 +182,6 @@ static inline void free_iova_fast(struct iova_domain *iovad, { } -static inline void queue_iova(struct iova_domain *iovad, - unsigned long pfn, unsigned long pages, - unsigned long data) -{ -} - static inline unsigned long alloc_iova_fast(struct iova_domain *iovad, unsigned long size, unsigned long limit_pfn, @@ -216,13 +203,6 @@ static inline void init_iova_domain(struct iova_domain *iovad, { } -static inline int init_iova_flush_queue(struct iova_domain *iovad, - iova_flush_cb flush_cb, - iova_entry_dtor entry_dtor) -{ - return -ENODEV; -} - static inline struct iova *find_iova(struct iova_domain *iovad, unsigned long pfn) { -- cgit v1.2.3-73-gaa49b From 649ad9835a3783bcb6c69368fa939e0010abb2c6 Mon Sep 17 00:00:00 2001 From: Robin Murphy Date: Fri, 17 Dec 2021 15:30:57 +0000 Subject: iommu/iova: Squash flush_cb abstraction Once again, with iommu-dma now being the only flush queue user, we no longer need the extra level of indirection through flush_cb. Squash that and let the flush queue code call the domain method directly. This does mean temporarily having to carry an additional copy of the IOMMU domain pointer around instead, but only until a later patch untangles it again. Reviewed-by: John Garry Reviewed-by: Christoph Hellwig Signed-off-by: Robin Murphy Link: https://lore.kernel.org/r/e3f9b4acdd6640012ef4fbc819ac868d727b64a9.1639753638.git.robin.murphy@arm.com Signed-off-by: Joerg Roedel --- drivers/iommu/dma-iommu.c | 13 +------------ drivers/iommu/iova.c | 11 +++++------ include/linux/iova.h | 11 +++-------- 3 files changed, 9 insertions(+), 26 deletions(-) (limited to 'drivers/iommu') diff --git a/drivers/iommu/dma-iommu.c b/drivers/iommu/dma-iommu.c index 6691f3cd768f..c63d93581a4e 100644 --- a/drivers/iommu/dma-iommu.c +++ b/drivers/iommu/dma-iommu.c @@ -282,17 +282,6 @@ static int iova_reserve_iommu_regions(struct device *dev, return ret; } -static void iommu_dma_flush_iotlb_all(struct iova_domain *iovad) -{ - struct iommu_dma_cookie *cookie; - struct iommu_domain *domain; - - cookie = container_of(iovad, struct iommu_dma_cookie, iovad); - domain = cookie->fq_domain; - - domain->ops->flush_iotlb_all(domain); -} - static bool dev_is_untrusted(struct device *dev) { return dev_is_pci(dev) && to_pci_dev(dev)->untrusted; @@ -312,7 +301,7 @@ int iommu_dma_init_fq(struct iommu_domain *domain) if (cookie->fq_domain) return 0; - ret = init_iova_flush_queue(&cookie->iovad, iommu_dma_flush_iotlb_all); + ret = init_iova_flush_queue(&cookie->iovad, domain); if (ret) { pr_warn("iova flush queue initialization failed\n"); return ret; diff --git a/drivers/iommu/iova.c b/drivers/iommu/iova.c index 541857ca4fd5..bbf642940988 100644 --- a/drivers/iommu/iova.c +++ b/drivers/iommu/iova.c @@ -63,7 +63,7 @@ init_iova_domain(struct iova_domain *iovad, unsigned long granule, iovad->start_pfn = start_pfn; iovad->dma_32bit_pfn = 1UL << (32 - iova_shift(iovad)); iovad->max32_alloc_size = iovad->dma_32bit_pfn; - iovad->flush_cb = NULL; + iovad->fq_domain = NULL; iovad->fq = NULL; iovad->anchor.pfn_lo = iovad->anchor.pfn_hi = IOVA_ANCHOR; rb_link_node(&iovad->anchor.node, NULL, &iovad->rbroot.rb_node); @@ -90,10 +90,10 @@ static void free_iova_flush_queue(struct iova_domain *iovad) free_percpu(iovad->fq); iovad->fq = NULL; - iovad->flush_cb = NULL; + iovad->fq_domain = NULL; } -int init_iova_flush_queue(struct iova_domain *iovad, iova_flush_cb flush_cb) +int init_iova_flush_queue(struct iova_domain *iovad, struct iommu_domain *fq_domain) { struct iova_fq __percpu *queue; int cpu; @@ -105,8 +105,6 @@ int init_iova_flush_queue(struct iova_domain *iovad, iova_flush_cb flush_cb) if (!queue) return -ENOMEM; - iovad->flush_cb = flush_cb; - for_each_possible_cpu(cpu) { struct iova_fq *fq; @@ -117,6 +115,7 @@ int init_iova_flush_queue(struct iova_domain *iovad, iova_flush_cb flush_cb) spin_lock_init(&fq->lock); } + iovad->fq_domain = fq_domain; iovad->fq = queue; timer_setup(&iovad->fq_timer, fq_flush_timeout, 0); @@ -598,7 +597,7 @@ static void fq_ring_free(struct iova_domain *iovad, struct iova_fq *fq) static void iova_domain_flush(struct iova_domain *iovad) { atomic64_inc(&iovad->fq_flush_start_cnt); - iovad->flush_cb(iovad); + iovad->fq_domain->ops->flush_iotlb_all(iovad->fq_domain); atomic64_inc(&iovad->fq_flush_finish_cnt); } diff --git a/include/linux/iova.h b/include/linux/iova.h index e746d8e41449..99be4fcea4f3 100644 --- a/include/linux/iova.h +++ b/include/linux/iova.h @@ -14,6 +14,7 @@ #include #include #include +#include /* iova structure */ struct iova { @@ -35,11 +36,6 @@ struct iova_rcache { struct iova_cpu_rcache __percpu *cpu_rcaches; }; -struct iova_domain; - -/* Call-Back from IOVA code into IOMMU drivers */ -typedef void (* iova_flush_cb)(struct iova_domain *domain); - /* Number of entries per Flush Queue */ #define IOVA_FQ_SIZE 256 @@ -82,8 +78,7 @@ struct iova_domain { struct iova anchor; /* rbtree lookup anchor */ struct iova_rcache rcaches[IOVA_RANGE_CACHE_MAX_SIZE]; /* IOVA range caches */ - iova_flush_cb flush_cb; /* Call-Back function to flush IOMMU - TLBs */ + struct iommu_domain *fq_domain; struct timer_list fq_timer; /* Timer to regularily empty the flush-queues */ @@ -147,7 +142,7 @@ struct iova *reserve_iova(struct iova_domain *iovad, unsigned long pfn_lo, unsigned long pfn_hi); void init_iova_domain(struct iova_domain *iovad, unsigned long granule, unsigned long start_pfn); -int init_iova_flush_queue(struct iova_domain *iovad, iova_flush_cb flush_cb); +int init_iova_flush_queue(struct iova_domain *iovad, struct iommu_domain *fq_domain); struct iova *find_iova(struct iova_domain *iovad, unsigned long pfn); void put_iova_domain(struct iova_domain *iovad); #else -- cgit v1.2.3-73-gaa49b From 6b3106e9ba2de7320a71291cedcefdcf1195ad58 Mon Sep 17 00:00:00 2001 From: Robin Murphy Date: Fri, 17 Dec 2021 15:30:58 +0000 Subject: iommu/amd: Simplify pagetable freeing For reasons unclear, pagetable freeing is an effectively recursive method implemented via an elaborate system of templated functions that turns out to account for 25% of the object file size. Implementing it using regular straightforward recursion makes the code simpler, and seems like a good thing to do before we work on it further. As part of that, also fix the types to avoid all the needless casting back and forth which just gets in the way. Signed-off-by: Robin Murphy Link: https://lore.kernel.org/r/d3d00c9f3fa0df4756b867072c201e6e82f9ce39.1639753638.git.robin.murphy@arm.com Signed-off-by: Joerg Roedel --- drivers/iommu/amd/io_pgtable.c | 82 ++++++++++++++++++------------------------ 1 file changed, 34 insertions(+), 48 deletions(-) (limited to 'drivers/iommu') diff --git a/drivers/iommu/amd/io_pgtable.c b/drivers/iommu/amd/io_pgtable.c index 182c93a43efd..4165e1372b6e 100644 --- a/drivers/iommu/amd/io_pgtable.c +++ b/drivers/iommu/amd/io_pgtable.c @@ -84,49 +84,45 @@ static void free_page_list(struct page *freelist) } } -static struct page *free_pt_page(unsigned long pt, struct page *freelist) +static struct page *free_pt_page(u64 *pt, struct page *freelist) { - struct page *p = virt_to_page((void *)pt); + struct page *p = virt_to_page(pt); p->freelist = freelist; return p; } -#define DEFINE_FREE_PT_FN(LVL, FN) \ -static struct page *free_pt_##LVL (unsigned long __pt, struct page *freelist) \ -{ \ - unsigned long p; \ - u64 *pt; \ - int i; \ - \ - pt = (u64 *)__pt; \ - \ - for (i = 0; i < 512; ++i) { \ - /* PTE present? */ \ - if (!IOMMU_PTE_PRESENT(pt[i])) \ - continue; \ - \ - /* Large PTE? */ \ - if (PM_PTE_LEVEL(pt[i]) == 0 || \ - PM_PTE_LEVEL(pt[i]) == 7) \ - continue; \ - \ - p = (unsigned long)IOMMU_PTE_PAGE(pt[i]); \ - freelist = FN(p, freelist); \ - } \ - \ - return free_pt_page((unsigned long)pt, freelist); \ -} +static struct page *free_pt_lvl(u64 *pt, struct page *freelist, int lvl) +{ + u64 *p; + int i; + + for (i = 0; i < 512; ++i) { + /* PTE present? */ + if (!IOMMU_PTE_PRESENT(pt[i])) + continue; -DEFINE_FREE_PT_FN(l2, free_pt_page) -DEFINE_FREE_PT_FN(l3, free_pt_l2) -DEFINE_FREE_PT_FN(l4, free_pt_l3) -DEFINE_FREE_PT_FN(l5, free_pt_l4) -DEFINE_FREE_PT_FN(l6, free_pt_l5) + /* Large PTE? */ + if (PM_PTE_LEVEL(pt[i]) == 0 || + PM_PTE_LEVEL(pt[i]) == 7) + continue; -static struct page *free_sub_pt(unsigned long root, int mode, - struct page *freelist) + /* + * Free the next level. No need to look at l1 tables here since + * they can only contain leaf PTEs; just free them directly. + */ + p = IOMMU_PTE_PAGE(pt[i]); + if (lvl > 2) + freelist = free_pt_lvl(p, freelist, lvl - 1); + else + freelist = free_pt_page(p, freelist); + } + + return free_pt_page(pt, freelist); +} + +static struct page *free_sub_pt(u64 *root, int mode, struct page *freelist) { switch (mode) { case PAGE_MODE_NONE: @@ -136,19 +132,11 @@ static struct page *free_sub_pt(unsigned long root, int mode, freelist = free_pt_page(root, freelist); break; case PAGE_MODE_2_LEVEL: - freelist = free_pt_l2(root, freelist); - break; case PAGE_MODE_3_LEVEL: - freelist = free_pt_l3(root, freelist); - break; case PAGE_MODE_4_LEVEL: - freelist = free_pt_l4(root, freelist); - break; case PAGE_MODE_5_LEVEL: - freelist = free_pt_l5(root, freelist); - break; case PAGE_MODE_6_LEVEL: - freelist = free_pt_l6(root, freelist); + free_pt_lvl(root, freelist, mode); break; default: BUG(); @@ -364,7 +352,7 @@ static u64 *fetch_pte(struct amd_io_pgtable *pgtable, static struct page *free_clear_pte(u64 *pte, u64 pteval, struct page *freelist) { - unsigned long pt; + u64 *pt; int mode; while (cmpxchg64(pte, pteval, 0) != pteval) { @@ -375,7 +363,7 @@ static struct page *free_clear_pte(u64 *pte, u64 pteval, struct page *freelist) if (!IOMMU_PTE_PRESENT(pteval)) return freelist; - pt = (unsigned long)IOMMU_PTE_PAGE(pteval); + pt = IOMMU_PTE_PAGE(pteval); mode = IOMMU_PTE_MODE(pteval); return free_sub_pt(pt, mode, freelist); @@ -512,7 +500,6 @@ static void v1_free_pgtable(struct io_pgtable *iop) struct amd_io_pgtable *pgtable = container_of(iop, struct amd_io_pgtable, iop); struct protection_domain *dom; struct page *freelist = NULL; - unsigned long root; if (pgtable->mode == PAGE_MODE_NONE) return; @@ -529,8 +516,7 @@ static void v1_free_pgtable(struct io_pgtable *iop) BUG_ON(pgtable->mode < PAGE_MODE_NONE || pgtable->mode > PAGE_MODE_6_LEVEL); - root = (unsigned long)pgtable->root; - freelist = free_sub_pt(root, pgtable->mode, freelist); + freelist = free_sub_pt(pgtable->root, pgtable->mode, freelist); free_page_list(freelist); } -- cgit v1.2.3-73-gaa49b From ce00eece6909c266da123fd147172d745a4f14a0 Mon Sep 17 00:00:00 2001 From: "Matthew Wilcox (Oracle)" Date: Fri, 17 Dec 2021 15:30:59 +0000 Subject: iommu/amd: Use put_pages_list page->freelist is for the use of slab. We already have the ability to free a list of pages in the core mm, but it requires the use of a list_head and for the pages to be chained together through page->lru. Switch the AMD IOMMU code over to using free_pages_list(). Signed-off-by: Matthew Wilcox (Oracle) [rm: split from original patch, cosmetic tweaks] Signed-off-by: Robin Murphy Link: https://lore.kernel.org/r/73af128f651aaa1f38f69e586c66765a88ad2de0.1639753638.git.robin.murphy@arm.com Signed-off-by: Joerg Roedel --- drivers/iommu/amd/io_pgtable.c | 50 +++++++++++++++--------------------------- 1 file changed, 18 insertions(+), 32 deletions(-) (limited to 'drivers/iommu') diff --git a/drivers/iommu/amd/io_pgtable.c b/drivers/iommu/amd/io_pgtable.c index 4165e1372b6e..b1bf4125b0f7 100644 --- a/drivers/iommu/amd/io_pgtable.c +++ b/drivers/iommu/amd/io_pgtable.c @@ -74,26 +74,14 @@ static u64 *first_pte_l7(u64 *pte, unsigned long *page_size, * ****************************************************************************/ -static void free_page_list(struct page *freelist) -{ - while (freelist != NULL) { - unsigned long p = (unsigned long)page_address(freelist); - - freelist = freelist->freelist; - free_page(p); - } -} - -static struct page *free_pt_page(u64 *pt, struct page *freelist) +static void free_pt_page(u64 *pt, struct list_head *freelist) { struct page *p = virt_to_page(pt); - p->freelist = freelist; - - return p; + list_add_tail(&p->lru, freelist); } -static struct page *free_pt_lvl(u64 *pt, struct page *freelist, int lvl) +static void free_pt_lvl(u64 *pt, struct list_head *freelist, int lvl) { u64 *p; int i; @@ -114,22 +102,22 @@ static struct page *free_pt_lvl(u64 *pt, struct page *freelist, int lvl) */ p = IOMMU_PTE_PAGE(pt[i]); if (lvl > 2) - freelist = free_pt_lvl(p, freelist, lvl - 1); + free_pt_lvl(p, freelist, lvl - 1); else - freelist = free_pt_page(p, freelist); + free_pt_page(p, freelist); } - return free_pt_page(pt, freelist); + free_pt_page(pt, freelist); } -static struct page *free_sub_pt(u64 *root, int mode, struct page *freelist) +static void free_sub_pt(u64 *root, int mode, struct list_head *freelist) { switch (mode) { case PAGE_MODE_NONE: case PAGE_MODE_7_LEVEL: break; case PAGE_MODE_1_LEVEL: - freelist = free_pt_page(root, freelist); + free_pt_page(root, freelist); break; case PAGE_MODE_2_LEVEL: case PAGE_MODE_3_LEVEL: @@ -141,8 +129,6 @@ static struct page *free_sub_pt(u64 *root, int mode, struct page *freelist) default: BUG(); } - - return freelist; } void amd_iommu_domain_set_pgtable(struct protection_domain *domain, @@ -350,7 +336,7 @@ static u64 *fetch_pte(struct amd_io_pgtable *pgtable, return pte; } -static struct page *free_clear_pte(u64 *pte, u64 pteval, struct page *freelist) +static void free_clear_pte(u64 *pte, u64 pteval, struct list_head *freelist) { u64 *pt; int mode; @@ -361,12 +347,12 @@ static struct page *free_clear_pte(u64 *pte, u64 pteval, struct page *freelist) } if (!IOMMU_PTE_PRESENT(pteval)) - return freelist; + return; pt = IOMMU_PTE_PAGE(pteval); mode = IOMMU_PTE_MODE(pteval); - return free_sub_pt(pt, mode, freelist); + free_sub_pt(pt, mode, freelist); } /* @@ -380,7 +366,7 @@ static int iommu_v1_map_page(struct io_pgtable_ops *ops, unsigned long iova, phys_addr_t paddr, size_t size, int prot, gfp_t gfp) { struct protection_domain *dom = io_pgtable_ops_to_domain(ops); - struct page *freelist = NULL; + LIST_HEAD(freelist); bool updated = false; u64 __pte, *pte; int ret, i, count; @@ -400,9 +386,9 @@ static int iommu_v1_map_page(struct io_pgtable_ops *ops, unsigned long iova, goto out; for (i = 0; i < count; ++i) - freelist = free_clear_pte(&pte[i], pte[i], freelist); + free_clear_pte(&pte[i], pte[i], &freelist); - if (freelist != NULL) + if (!list_empty(&freelist)) updated = true; if (count > 1) { @@ -437,7 +423,7 @@ out: } /* Everything flushed out, free pages now */ - free_page_list(freelist); + put_pages_list(&freelist); return ret; } @@ -499,7 +485,7 @@ static void v1_free_pgtable(struct io_pgtable *iop) { struct amd_io_pgtable *pgtable = container_of(iop, struct amd_io_pgtable, iop); struct protection_domain *dom; - struct page *freelist = NULL; + LIST_HEAD(freelist); if (pgtable->mode == PAGE_MODE_NONE) return; @@ -516,9 +502,9 @@ static void v1_free_pgtable(struct io_pgtable *iop) BUG_ON(pgtable->mode < PAGE_MODE_NONE || pgtable->mode > PAGE_MODE_6_LEVEL); - freelist = free_sub_pt(pgtable->root, pgtable->mode, freelist); + free_sub_pt(pgtable->root, pgtable->mode, &freelist); - free_page_list(freelist); + put_pages_list(&freelist); } static struct io_pgtable *v1_alloc_pgtable(struct io_pgtable_cfg *cfg, void *cookie) -- cgit v1.2.3-73-gaa49b From 87f60cc65d24939353b40aa1d9297fea080cdf8d Mon Sep 17 00:00:00 2001 From: "Matthew Wilcox (Oracle)" Date: Fri, 17 Dec 2021 15:31:00 +0000 Subject: iommu/vt-d: Use put_pages_list page->freelist is for the use of slab. We already have the ability to free a list of pages in the core mm, but it requires the use of a list_head and for the pages to be chained together through page->lru. Switch the Intel IOMMU and IOVA code over to using free_pages_list(). Signed-off-by: Matthew Wilcox (Oracle) [rm: split from original patch, cosmetic tweaks, fix fq entries] Signed-off-by: Robin Murphy Reviewed-by: Lu Baolu Link: https://lore.kernel.org/r/2115b560d9a0ce7cd4b948bd51a2b7bde8fdfd59.1639753638.git.robin.murphy@arm.com Signed-off-by: Joerg Roedel --- drivers/iommu/dma-iommu.c | 2 +- drivers/iommu/intel/iommu.c | 89 ++++++++++++++++----------------------------- drivers/iommu/iova.c | 26 +++++-------- include/linux/iommu.h | 3 +- include/linux/iova.h | 4 +- 5 files changed, 45 insertions(+), 79 deletions(-) (limited to 'drivers/iommu') diff --git a/drivers/iommu/dma-iommu.c b/drivers/iommu/dma-iommu.c index c63d93581a4e..ccacd0f71443 100644 --- a/drivers/iommu/dma-iommu.c +++ b/drivers/iommu/dma-iommu.c @@ -447,7 +447,7 @@ static void iommu_dma_free_iova(struct iommu_dma_cookie *cookie, else if (gather && gather->queued) queue_iova(iovad, iova_pfn(iovad, iova), size >> iova_shift(iovad), - gather->freelist); + &gather->freelist); else free_iova_fast(iovad, iova_pfn(iovad, iova), size >> iova_shift(iovad)); diff --git a/drivers/iommu/intel/iommu.c b/drivers/iommu/intel/iommu.c index b6a8f3282411..17b3d97111f3 100644 --- a/drivers/iommu/intel/iommu.c +++ b/drivers/iommu/intel/iommu.c @@ -1303,35 +1303,30 @@ static void dma_pte_free_pagetable(struct dmar_domain *domain, know the hardware page-walk will no longer touch them. The 'pte' argument is the *parent* PTE, pointing to the page that is to be freed. */ -static struct page *dma_pte_list_pagetables(struct dmar_domain *domain, - int level, struct dma_pte *pte, - struct page *freelist) +static void dma_pte_list_pagetables(struct dmar_domain *domain, + int level, struct dma_pte *pte, + struct list_head *freelist) { struct page *pg; pg = pfn_to_page(dma_pte_addr(pte) >> PAGE_SHIFT); - pg->freelist = freelist; - freelist = pg; + list_add_tail(&pg->lru, freelist); if (level == 1) - return freelist; + return; pte = page_address(pg); do { if (dma_pte_present(pte) && !dma_pte_superpage(pte)) - freelist = dma_pte_list_pagetables(domain, level - 1, - pte, freelist); + dma_pte_list_pagetables(domain, level - 1, pte, freelist); pte++; } while (!first_pte_in_page(pte)); - - return freelist; } -static struct page *dma_pte_clear_level(struct dmar_domain *domain, int level, - struct dma_pte *pte, unsigned long pfn, - unsigned long start_pfn, - unsigned long last_pfn, - struct page *freelist) +static void dma_pte_clear_level(struct dmar_domain *domain, int level, + struct dma_pte *pte, unsigned long pfn, + unsigned long start_pfn, unsigned long last_pfn, + struct list_head *freelist) { struct dma_pte *first_pte = NULL, *last_pte = NULL; @@ -1350,7 +1345,7 @@ static struct page *dma_pte_clear_level(struct dmar_domain *domain, int level, /* These suborbinate page tables are going away entirely. Don't bother to clear them; we're just going to *free* them. */ if (level > 1 && !dma_pte_superpage(pte)) - freelist = dma_pte_list_pagetables(domain, level - 1, pte, freelist); + dma_pte_list_pagetables(domain, level - 1, pte, freelist); dma_clear_pte(pte); if (!first_pte) @@ -1358,10 +1353,10 @@ static struct page *dma_pte_clear_level(struct dmar_domain *domain, int level, last_pte = pte; } else if (level > 1) { /* Recurse down into a level that isn't *entirely* obsolete */ - freelist = dma_pte_clear_level(domain, level - 1, - phys_to_virt(dma_pte_addr(pte)), - level_pfn, start_pfn, last_pfn, - freelist); + dma_pte_clear_level(domain, level - 1, + phys_to_virt(dma_pte_addr(pte)), + level_pfn, start_pfn, last_pfn, + freelist); } next: pfn = level_pfn + level_size(level); @@ -1370,47 +1365,28 @@ next: if (first_pte) domain_flush_cache(domain, first_pte, (void *)++last_pte - (void *)first_pte); - - return freelist; } /* We can't just free the pages because the IOMMU may still be walking the page tables, and may have cached the intermediate levels. The pages can only be freed after the IOTLB flush has been done. */ -static struct page *domain_unmap(struct dmar_domain *domain, - unsigned long start_pfn, - unsigned long last_pfn, - struct page *freelist) +static void domain_unmap(struct dmar_domain *domain, unsigned long start_pfn, + unsigned long last_pfn, struct list_head *freelist) { BUG_ON(!domain_pfn_supported(domain, start_pfn)); BUG_ON(!domain_pfn_supported(domain, last_pfn)); BUG_ON(start_pfn > last_pfn); /* we don't need lock here; nobody else touches the iova range */ - freelist = dma_pte_clear_level(domain, agaw_to_level(domain->agaw), - domain->pgd, 0, start_pfn, last_pfn, - freelist); + dma_pte_clear_level(domain, agaw_to_level(domain->agaw), + domain->pgd, 0, start_pfn, last_pfn, freelist); /* free pgd */ if (start_pfn == 0 && last_pfn == DOMAIN_MAX_PFN(domain->gaw)) { struct page *pgd_page = virt_to_page(domain->pgd); - pgd_page->freelist = freelist; - freelist = pgd_page; - + list_add_tail(&pgd_page->lru, freelist); domain->pgd = NULL; } - - return freelist; -} - -static void dma_free_pagelist(struct page *freelist) -{ - struct page *pg; - - while ((pg = freelist)) { - freelist = pg->freelist; - free_pgtable_page(page_address(pg)); - } } /* iommu handling */ @@ -2095,11 +2071,10 @@ static void domain_exit(struct dmar_domain *domain) domain_remove_dev_info(domain); if (domain->pgd) { - struct page *freelist; + LIST_HEAD(freelist); - freelist = domain_unmap(domain, 0, - DOMAIN_MAX_PFN(domain->gaw), NULL); - dma_free_pagelist(freelist); + domain_unmap(domain, 0, DOMAIN_MAX_PFN(domain->gaw), &freelist); + put_pages_list(&freelist); } free_domain_mem(domain); @@ -4192,19 +4167,17 @@ static int intel_iommu_memory_notifier(struct notifier_block *nb, { struct dmar_drhd_unit *drhd; struct intel_iommu *iommu; - struct page *freelist; + LIST_HEAD(freelist); - freelist = domain_unmap(si_domain, - start_vpfn, last_vpfn, - NULL); + domain_unmap(si_domain, start_vpfn, last_vpfn, &freelist); rcu_read_lock(); for_each_active_iommu(iommu, drhd) iommu_flush_iotlb_psi(iommu, si_domain, start_vpfn, mhp->nr_pages, - !freelist, 0); + list_empty(&freelist), 0); rcu_read_unlock(); - dma_free_pagelist(freelist); + put_pages_list(&freelist); } break; } @@ -5211,8 +5184,7 @@ static size_t intel_iommu_unmap(struct iommu_domain *domain, start_pfn = iova >> VTD_PAGE_SHIFT; last_pfn = (iova + size - 1) >> VTD_PAGE_SHIFT; - gather->freelist = domain_unmap(dmar_domain, start_pfn, - last_pfn, gather->freelist); + domain_unmap(dmar_domain, start_pfn, last_pfn, &gather->freelist); if (dmar_domain->max_addr == iova + size) dmar_domain->max_addr = iova; @@ -5248,9 +5220,10 @@ static void intel_iommu_tlb_sync(struct iommu_domain *domain, for_each_domain_iommu(iommu_id, dmar_domain) iommu_flush_iotlb_psi(g_iommus[iommu_id], dmar_domain, - start_pfn, nrpages, !gather->freelist, 0); + start_pfn, nrpages, + list_empty(&gather->freelist), 0); - dma_free_pagelist(gather->freelist); + put_pages_list(&gather->freelist); } static phys_addr_t intel_iommu_iova_to_phys(struct iommu_domain *domain, diff --git a/drivers/iommu/iova.c b/drivers/iommu/iova.c index bbf642940988..962614c7afbb 100644 --- a/drivers/iommu/iova.c +++ b/drivers/iommu/iova.c @@ -96,7 +96,7 @@ static void free_iova_flush_queue(struct iova_domain *iovad) int init_iova_flush_queue(struct iova_domain *iovad, struct iommu_domain *fq_domain) { struct iova_fq __percpu *queue; - int cpu; + int i, cpu; atomic64_set(&iovad->fq_flush_start_cnt, 0); atomic64_set(&iovad->fq_flush_finish_cnt, 0); @@ -113,6 +113,9 @@ int init_iova_flush_queue(struct iova_domain *iovad, struct iommu_domain *fq_dom fq->tail = 0; spin_lock_init(&fq->lock); + + for (i = 0; i < IOVA_FQ_SIZE; i++) + INIT_LIST_HEAD(&fq->entries[i].freelist); } iovad->fq_domain = fq_domain; @@ -543,16 +546,6 @@ free_iova_fast(struct iova_domain *iovad, unsigned long pfn, unsigned long size) } EXPORT_SYMBOL_GPL(free_iova_fast); -static void fq_entry_dtor(struct page *freelist) -{ - while (freelist) { - unsigned long p = (unsigned long)page_address(freelist); - - freelist = freelist->freelist; - free_page(p); - } -} - #define fq_ring_for_each(i, fq) \ for ((i) = (fq)->head; (i) != (fq)->tail; (i) = ((i) + 1) % IOVA_FQ_SIZE) @@ -585,7 +578,7 @@ static void fq_ring_free(struct iova_domain *iovad, struct iova_fq *fq) if (fq->entries[idx].counter >= counter) break; - fq_entry_dtor(fq->entries[idx].freelist); + put_pages_list(&fq->entries[idx].freelist); free_iova_fast(iovad, fq->entries[idx].iova_pfn, fq->entries[idx].pages); @@ -607,15 +600,14 @@ static void fq_destroy_all_entries(struct iova_domain *iovad) /* * This code runs when the iova_domain is being detroyed, so don't - * bother to free iovas, just call the entry_dtor on all remaining - * entries. + * bother to free iovas, just free any remaining pagetable pages. */ for_each_possible_cpu(cpu) { struct iova_fq *fq = per_cpu_ptr(iovad->fq, cpu); int idx; fq_ring_for_each(idx, fq) - fq_entry_dtor(fq->entries[idx].freelist); + put_pages_list(&fq->entries[idx].freelist); } } @@ -640,7 +632,7 @@ static void fq_flush_timeout(struct timer_list *t) void queue_iova(struct iova_domain *iovad, unsigned long pfn, unsigned long pages, - struct page *freelist) + struct list_head *freelist) { struct iova_fq *fq; unsigned long flags; @@ -674,8 +666,8 @@ void queue_iova(struct iova_domain *iovad, fq->entries[idx].iova_pfn = pfn; fq->entries[idx].pages = pages; - fq->entries[idx].freelist = freelist; fq->entries[idx].counter = atomic64_read(&iovad->fq_flush_start_cnt); + list_splice(freelist, &fq->entries[idx].freelist); spin_unlock_irqrestore(&fq->lock, flags); diff --git a/include/linux/iommu.h b/include/linux/iommu.h index d2f3435e7d17..de0c57a567c8 100644 --- a/include/linux/iommu.h +++ b/include/linux/iommu.h @@ -186,7 +186,7 @@ struct iommu_iotlb_gather { unsigned long start; unsigned long end; size_t pgsize; - struct page *freelist; + struct list_head freelist; bool queued; }; @@ -399,6 +399,7 @@ static inline void iommu_iotlb_gather_init(struct iommu_iotlb_gather *gather) { *gather = (struct iommu_iotlb_gather) { .start = ULONG_MAX, + .freelist = LIST_HEAD_INIT(gather->freelist), }; } diff --git a/include/linux/iova.h b/include/linux/iova.h index 99be4fcea4f3..072a09c06e8a 100644 --- a/include/linux/iova.h +++ b/include/linux/iova.h @@ -46,7 +46,7 @@ struct iova_rcache { struct iova_fq_entry { unsigned long iova_pfn; unsigned long pages; - struct page *freelist; + struct list_head freelist; u64 counter; /* Flush counter when this entrie was added */ }; @@ -135,7 +135,7 @@ void free_iova_fast(struct iova_domain *iovad, unsigned long pfn, unsigned long size); void queue_iova(struct iova_domain *iovad, unsigned long pfn, unsigned long pages, - struct page *freelist); + struct list_head *freelist); unsigned long alloc_iova_fast(struct iova_domain *iovad, unsigned long size, unsigned long limit_pfn, bool flush_rcache); struct iova *reserve_iova(struct iova_domain *iovad, unsigned long pfn_lo, -- cgit v1.2.3-73-gaa49b From ea4d71bb5e3fc5c2b7b856bc7506439298f47a23 Mon Sep 17 00:00:00 2001 From: Robin Murphy Date: Fri, 17 Dec 2021 15:31:01 +0000 Subject: iommu/iova: Consolidate flush queue code Squash and simplify some of the freeing code, and move the init and free routines down into the rest of the flush queue code to obviate the forward declarations. Reviewed-by: John Garry Signed-off-by: Robin Murphy Link: https://lore.kernel.org/r/b0dd4565e6646b6489599d7a1eaa362c75f53c95.1639753638.git.robin.murphy@arm.com Signed-off-by: Joerg Roedel --- drivers/iommu/iova.c | 131 +++++++++++++++++++++++---------------------------- 1 file changed, 58 insertions(+), 73 deletions(-) (limited to 'drivers/iommu') diff --git a/drivers/iommu/iova.c b/drivers/iommu/iova.c index 962614c7afbb..e368fd3da0d2 100644 --- a/drivers/iommu/iova.c +++ b/drivers/iommu/iova.c @@ -24,8 +24,6 @@ static unsigned long iova_rcache_get(struct iova_domain *iovad, static void init_iova_rcaches(struct iova_domain *iovad); static void free_cpu_cached_iovas(unsigned int cpu, struct iova_domain *iovad); static void free_iova_rcaches(struct iova_domain *iovad); -static void fq_destroy_all_entries(struct iova_domain *iovad); -static void fq_flush_timeout(struct timer_list *t); static int iova_cpuhp_dead(unsigned int cpu, struct hlist_node *node) { @@ -73,60 +71,6 @@ init_iova_domain(struct iova_domain *iovad, unsigned long granule, } EXPORT_SYMBOL_GPL(init_iova_domain); -static bool has_iova_flush_queue(struct iova_domain *iovad) -{ - return !!iovad->fq; -} - -static void free_iova_flush_queue(struct iova_domain *iovad) -{ - if (!has_iova_flush_queue(iovad)) - return; - - del_timer_sync(&iovad->fq_timer); - - fq_destroy_all_entries(iovad); - - free_percpu(iovad->fq); - - iovad->fq = NULL; - iovad->fq_domain = NULL; -} - -int init_iova_flush_queue(struct iova_domain *iovad, struct iommu_domain *fq_domain) -{ - struct iova_fq __percpu *queue; - int i, cpu; - - atomic64_set(&iovad->fq_flush_start_cnt, 0); - atomic64_set(&iovad->fq_flush_finish_cnt, 0); - - queue = alloc_percpu(struct iova_fq); - if (!queue) - return -ENOMEM; - - for_each_possible_cpu(cpu) { - struct iova_fq *fq; - - fq = per_cpu_ptr(queue, cpu); - fq->head = 0; - fq->tail = 0; - - spin_lock_init(&fq->lock); - - for (i = 0; i < IOVA_FQ_SIZE; i++) - INIT_LIST_HEAD(&fq->entries[i].freelist); - } - - iovad->fq_domain = fq_domain; - iovad->fq = queue; - - timer_setup(&iovad->fq_timer, fq_flush_timeout, 0); - atomic_set(&iovad->fq_timer_on, 0); - - return 0; -} - static struct rb_node * __get_cached_rbnode(struct iova_domain *iovad, unsigned long limit_pfn) { @@ -594,23 +538,6 @@ static void iova_domain_flush(struct iova_domain *iovad) atomic64_inc(&iovad->fq_flush_finish_cnt); } -static void fq_destroy_all_entries(struct iova_domain *iovad) -{ - int cpu; - - /* - * This code runs when the iova_domain is being detroyed, so don't - * bother to free iovas, just free any remaining pagetable pages. - */ - for_each_possible_cpu(cpu) { - struct iova_fq *fq = per_cpu_ptr(iovad->fq, cpu); - int idx; - - fq_ring_for_each(idx, fq) - put_pages_list(&fq->entries[idx].freelist); - } -} - static void fq_flush_timeout(struct timer_list *t) { struct iova_domain *iovad = from_timer(iovad, t, fq_timer); @@ -678,6 +605,64 @@ void queue_iova(struct iova_domain *iovad, jiffies + msecs_to_jiffies(IOVA_FQ_TIMEOUT)); } +static void free_iova_flush_queue(struct iova_domain *iovad) +{ + int cpu, idx; + + if (!iovad->fq) + return; + + del_timer_sync(&iovad->fq_timer); + /* + * This code runs when the iova_domain is being detroyed, so don't + * bother to free iovas, just free any remaining pagetable pages. + */ + for_each_possible_cpu(cpu) { + struct iova_fq *fq = per_cpu_ptr(iovad->fq, cpu); + + fq_ring_for_each(idx, fq) + put_pages_list(&fq->entries[idx].freelist); + } + + free_percpu(iovad->fq); + + iovad->fq = NULL; + iovad->fq_domain = NULL; +} + +int init_iova_flush_queue(struct iova_domain *iovad, struct iommu_domain *fq_domain) +{ + struct iova_fq __percpu *queue; + int i, cpu; + + atomic64_set(&iovad->fq_flush_start_cnt, 0); + atomic64_set(&iovad->fq_flush_finish_cnt, 0); + + queue = alloc_percpu(struct iova_fq); + if (!queue) + return -ENOMEM; + + for_each_possible_cpu(cpu) { + struct iova_fq *fq = per_cpu_ptr(queue, cpu); + + fq->head = 0; + fq->tail = 0; + + spin_lock_init(&fq->lock); + + for (i = 0; i < IOVA_FQ_SIZE; i++) + INIT_LIST_HEAD(&fq->entries[i].freelist); + } + + iovad->fq_domain = fq_domain; + iovad->fq = queue; + + timer_setup(&iovad->fq_timer, fq_flush_timeout, 0); + atomic_set(&iovad->fq_timer_on, 0); + + return 0; +} + /** * put_iova_domain - destroys the iova domain * @iovad: - iova domain in question. -- cgit v1.2.3-73-gaa49b From f7f07484542fae833025da86eb4bd068f5253fe0 Mon Sep 17 00:00:00 2001 From: Robin Murphy Date: Fri, 17 Dec 2021 15:31:02 +0000 Subject: iommu/iova: Move flush queue code to iommu-dma Flush queues are specific to DMA ops, which are now handled exclusively by iommu-dma. As such, now that the historical artefacts from being shared directly with drivers have been cleaned up, move the flush queue code into iommu-dma itself to get it out of the way of other IOVA users. This is pure code movement with no functional change; refactoring to clean up the headers and definitions will follow. Reviewed-by: John Garry Signed-off-by: Robin Murphy Link: https://lore.kernel.org/r/1d9a1ee1392e96eaae5e6467181b3e83edfdfbad.1639753638.git.robin.murphy@arm.com Signed-off-by: Joerg Roedel --- drivers/iommu/dma-iommu.c | 177 +++++++++++++++++++++++++++++++++++++++++++++- drivers/iommu/iova.c | 175 --------------------------------------------- 2 files changed, 176 insertions(+), 176 deletions(-) (limited to 'drivers/iommu') diff --git a/drivers/iommu/dma-iommu.c b/drivers/iommu/dma-iommu.c index ccacd0f71443..96057bec4164 100644 --- a/drivers/iommu/dma-iommu.c +++ b/drivers/iommu/dma-iommu.c @@ -64,6 +64,179 @@ static int __init iommu_dma_forcedac_setup(char *str) } early_param("iommu.forcedac", iommu_dma_forcedac_setup); +#define fq_ring_for_each(i, fq) \ + for ((i) = (fq)->head; (i) != (fq)->tail; (i) = ((i) + 1) % IOVA_FQ_SIZE) + +static inline bool fq_full(struct iova_fq *fq) +{ + assert_spin_locked(&fq->lock); + return (((fq->tail + 1) % IOVA_FQ_SIZE) == fq->head); +} + +static inline unsigned fq_ring_add(struct iova_fq *fq) +{ + unsigned idx = fq->tail; + + assert_spin_locked(&fq->lock); + + fq->tail = (idx + 1) % IOVA_FQ_SIZE; + + return idx; +} + +static void fq_ring_free(struct iova_domain *iovad, struct iova_fq *fq) +{ + u64 counter = atomic64_read(&iovad->fq_flush_finish_cnt); + unsigned idx; + + assert_spin_locked(&fq->lock); + + fq_ring_for_each(idx, fq) { + + if (fq->entries[idx].counter >= counter) + break; + + put_pages_list(&fq->entries[idx].freelist); + free_iova_fast(iovad, + fq->entries[idx].iova_pfn, + fq->entries[idx].pages); + + fq->head = (fq->head + 1) % IOVA_FQ_SIZE; + } +} + +static void iova_domain_flush(struct iova_domain *iovad) +{ + atomic64_inc(&iovad->fq_flush_start_cnt); + iovad->fq_domain->ops->flush_iotlb_all(iovad->fq_domain); + atomic64_inc(&iovad->fq_flush_finish_cnt); +} + +static void fq_flush_timeout(struct timer_list *t) +{ + struct iova_domain *iovad = from_timer(iovad, t, fq_timer); + int cpu; + + atomic_set(&iovad->fq_timer_on, 0); + iova_domain_flush(iovad); + + for_each_possible_cpu(cpu) { + unsigned long flags; + struct iova_fq *fq; + + fq = per_cpu_ptr(iovad->fq, cpu); + spin_lock_irqsave(&fq->lock, flags); + fq_ring_free(iovad, fq); + spin_unlock_irqrestore(&fq->lock, flags); + } +} + +void queue_iova(struct iova_domain *iovad, + unsigned long pfn, unsigned long pages, + struct list_head *freelist) +{ + struct iova_fq *fq; + unsigned long flags; + unsigned idx; + + /* + * Order against the IOMMU driver's pagetable update from unmapping + * @pte, to guarantee that iova_domain_flush() observes that if called + * from a different CPU before we release the lock below. Full barrier + * so it also pairs with iommu_dma_init_fq() to avoid seeing partially + * written fq state here. + */ + smp_mb(); + + fq = raw_cpu_ptr(iovad->fq); + spin_lock_irqsave(&fq->lock, flags); + + /* + * First remove all entries from the flush queue that have already been + * flushed out on another CPU. This makes the fq_full() check below less + * likely to be true. + */ + fq_ring_free(iovad, fq); + + if (fq_full(fq)) { + iova_domain_flush(iovad); + fq_ring_free(iovad, fq); + } + + idx = fq_ring_add(fq); + + fq->entries[idx].iova_pfn = pfn; + fq->entries[idx].pages = pages; + fq->entries[idx].counter = atomic64_read(&iovad->fq_flush_start_cnt); + list_splice(freelist, &fq->entries[idx].freelist); + + spin_unlock_irqrestore(&fq->lock, flags); + + /* Avoid false sharing as much as possible. */ + if (!atomic_read(&iovad->fq_timer_on) && + !atomic_xchg(&iovad->fq_timer_on, 1)) + mod_timer(&iovad->fq_timer, + jiffies + msecs_to_jiffies(IOVA_FQ_TIMEOUT)); +} + +static void free_iova_flush_queue(struct iova_domain *iovad) +{ + int cpu, idx; + + if (!iovad->fq) + return; + + del_timer_sync(&iovad->fq_timer); + /* + * This code runs when the iova_domain is being detroyed, so don't + * bother to free iovas, just free any remaining pagetable pages. + */ + for_each_possible_cpu(cpu) { + struct iova_fq *fq = per_cpu_ptr(iovad->fq, cpu); + + fq_ring_for_each(idx, fq) + put_pages_list(&fq->entries[idx].freelist); + } + + free_percpu(iovad->fq); + + iovad->fq = NULL; + iovad->fq_domain = NULL; +} + +int init_iova_flush_queue(struct iova_domain *iovad, struct iommu_domain *fq_domain) +{ + struct iova_fq __percpu *queue; + int i, cpu; + + atomic64_set(&iovad->fq_flush_start_cnt, 0); + atomic64_set(&iovad->fq_flush_finish_cnt, 0); + + queue = alloc_percpu(struct iova_fq); + if (!queue) + return -ENOMEM; + + for_each_possible_cpu(cpu) { + struct iova_fq *fq = per_cpu_ptr(queue, cpu); + + fq->head = 0; + fq->tail = 0; + + spin_lock_init(&fq->lock); + + for (i = 0; i < IOVA_FQ_SIZE; i++) + INIT_LIST_HEAD(&fq->entries[i].freelist); + } + + iovad->fq_domain = fq_domain; + iovad->fq = queue; + + timer_setup(&iovad->fq_timer, fq_flush_timeout, 0); + atomic_set(&iovad->fq_timer_on, 0); + + return 0; +} + static inline size_t cookie_msi_granule(struct iommu_dma_cookie *cookie) { if (cookie->type == IOMMU_DMA_IOVA_COOKIE) @@ -144,8 +317,10 @@ void iommu_put_dma_cookie(struct iommu_domain *domain) if (!cookie) return; - if (cookie->type == IOMMU_DMA_IOVA_COOKIE && cookie->iovad.granule) + if (cookie->type == IOMMU_DMA_IOVA_COOKIE && cookie->iovad.granule) { + free_iova_flush_queue(&cookie->iovad); put_iova_domain(&cookie->iovad); + } list_for_each_entry_safe(msi, tmp, &cookie->msi_page_list, list) { list_del(&msi->list); diff --git a/drivers/iommu/iova.c b/drivers/iommu/iova.c index e368fd3da0d2..081e5c0cf940 100644 --- a/drivers/iommu/iova.c +++ b/drivers/iommu/iova.c @@ -490,179 +490,6 @@ free_iova_fast(struct iova_domain *iovad, unsigned long pfn, unsigned long size) } EXPORT_SYMBOL_GPL(free_iova_fast); -#define fq_ring_for_each(i, fq) \ - for ((i) = (fq)->head; (i) != (fq)->tail; (i) = ((i) + 1) % IOVA_FQ_SIZE) - -static inline bool fq_full(struct iova_fq *fq) -{ - assert_spin_locked(&fq->lock); - return (((fq->tail + 1) % IOVA_FQ_SIZE) == fq->head); -} - -static inline unsigned fq_ring_add(struct iova_fq *fq) -{ - unsigned idx = fq->tail; - - assert_spin_locked(&fq->lock); - - fq->tail = (idx + 1) % IOVA_FQ_SIZE; - - return idx; -} - -static void fq_ring_free(struct iova_domain *iovad, struct iova_fq *fq) -{ - u64 counter = atomic64_read(&iovad->fq_flush_finish_cnt); - unsigned idx; - - assert_spin_locked(&fq->lock); - - fq_ring_for_each(idx, fq) { - - if (fq->entries[idx].counter >= counter) - break; - - put_pages_list(&fq->entries[idx].freelist); - free_iova_fast(iovad, - fq->entries[idx].iova_pfn, - fq->entries[idx].pages); - - fq->head = (fq->head + 1) % IOVA_FQ_SIZE; - } -} - -static void iova_domain_flush(struct iova_domain *iovad) -{ - atomic64_inc(&iovad->fq_flush_start_cnt); - iovad->fq_domain->ops->flush_iotlb_all(iovad->fq_domain); - atomic64_inc(&iovad->fq_flush_finish_cnt); -} - -static void fq_flush_timeout(struct timer_list *t) -{ - struct iova_domain *iovad = from_timer(iovad, t, fq_timer); - int cpu; - - atomic_set(&iovad->fq_timer_on, 0); - iova_domain_flush(iovad); - - for_each_possible_cpu(cpu) { - unsigned long flags; - struct iova_fq *fq; - - fq = per_cpu_ptr(iovad->fq, cpu); - spin_lock_irqsave(&fq->lock, flags); - fq_ring_free(iovad, fq); - spin_unlock_irqrestore(&fq->lock, flags); - } -} - -void queue_iova(struct iova_domain *iovad, - unsigned long pfn, unsigned long pages, - struct list_head *freelist) -{ - struct iova_fq *fq; - unsigned long flags; - unsigned idx; - - /* - * Order against the IOMMU driver's pagetable update from unmapping - * @pte, to guarantee that iova_domain_flush() observes that if called - * from a different CPU before we release the lock below. Full barrier - * so it also pairs with iommu_dma_init_fq() to avoid seeing partially - * written fq state here. - */ - smp_mb(); - - fq = raw_cpu_ptr(iovad->fq); - spin_lock_irqsave(&fq->lock, flags); - - /* - * First remove all entries from the flush queue that have already been - * flushed out on another CPU. This makes the fq_full() check below less - * likely to be true. - */ - fq_ring_free(iovad, fq); - - if (fq_full(fq)) { - iova_domain_flush(iovad); - fq_ring_free(iovad, fq); - } - - idx = fq_ring_add(fq); - - fq->entries[idx].iova_pfn = pfn; - fq->entries[idx].pages = pages; - fq->entries[idx].counter = atomic64_read(&iovad->fq_flush_start_cnt); - list_splice(freelist, &fq->entries[idx].freelist); - - spin_unlock_irqrestore(&fq->lock, flags); - - /* Avoid false sharing as much as possible. */ - if (!atomic_read(&iovad->fq_timer_on) && - !atomic_xchg(&iovad->fq_timer_on, 1)) - mod_timer(&iovad->fq_timer, - jiffies + msecs_to_jiffies(IOVA_FQ_TIMEOUT)); -} - -static void free_iova_flush_queue(struct iova_domain *iovad) -{ - int cpu, idx; - - if (!iovad->fq) - return; - - del_timer_sync(&iovad->fq_timer); - /* - * This code runs when the iova_domain is being detroyed, so don't - * bother to free iovas, just free any remaining pagetable pages. - */ - for_each_possible_cpu(cpu) { - struct iova_fq *fq = per_cpu_ptr(iovad->fq, cpu); - - fq_ring_for_each(idx, fq) - put_pages_list(&fq->entries[idx].freelist); - } - - free_percpu(iovad->fq); - - iovad->fq = NULL; - iovad->fq_domain = NULL; -} - -int init_iova_flush_queue(struct iova_domain *iovad, struct iommu_domain *fq_domain) -{ - struct iova_fq __percpu *queue; - int i, cpu; - - atomic64_set(&iovad->fq_flush_start_cnt, 0); - atomic64_set(&iovad->fq_flush_finish_cnt, 0); - - queue = alloc_percpu(struct iova_fq); - if (!queue) - return -ENOMEM; - - for_each_possible_cpu(cpu) { - struct iova_fq *fq = per_cpu_ptr(queue, cpu); - - fq->head = 0; - fq->tail = 0; - - spin_lock_init(&fq->lock); - - for (i = 0; i < IOVA_FQ_SIZE; i++) - INIT_LIST_HEAD(&fq->entries[i].freelist); - } - - iovad->fq_domain = fq_domain; - iovad->fq = queue; - - timer_setup(&iovad->fq_timer, fq_flush_timeout, 0); - atomic_set(&iovad->fq_timer_on, 0); - - return 0; -} - /** * put_iova_domain - destroys the iova domain * @iovad: - iova domain in question. @@ -674,8 +501,6 @@ void put_iova_domain(struct iova_domain *iovad) cpuhp_state_remove_instance_nocalls(CPUHP_IOMMU_IOVA_DEAD, &iovad->cpuhp_dead); - - free_iova_flush_queue(iovad); free_iova_rcaches(iovad); rbtree_postorder_for_each_entry_safe(iova, tmp, &iovad->rbroot, node) free_iova_mem(iova); -- cgit v1.2.3-73-gaa49b From a17e3026bc4da9135ca9a42ec0b1fa67f95172e3 Mon Sep 17 00:00:00 2001 From: Robin Murphy Date: Fri, 17 Dec 2021 15:31:03 +0000 Subject: iommu: Move flush queue data into iommu_dma_cookie Complete the move into iommu-dma by refactoring the flush queues themselves to belong to the DMA cookie rather than the IOVA domain. The refactoring may as well extend to some minor cosmetic aspects too, to help us stay one step ahead of the style police. Signed-off-by: Robin Murphy Link: https://lore.kernel.org/r/24304722005bc6f144e2a1fdd865d1465722fc2e.1639753638.git.robin.murphy@arm.com Signed-off-by: Joerg Roedel --- drivers/iommu/dma-iommu.c | 171 +++++++++++++++++++++++++--------------------- drivers/iommu/iova.c | 2 - include/linux/iova.h | 44 +----------- 3 files changed, 95 insertions(+), 122 deletions(-) (limited to 'drivers/iommu') diff --git a/drivers/iommu/dma-iommu.c b/drivers/iommu/dma-iommu.c index 96057bec4164..d85d54f2b549 100644 --- a/drivers/iommu/dma-iommu.c +++ b/drivers/iommu/dma-iommu.c @@ -9,9 +9,12 @@ */ #include +#include +#include #include -#include +#include #include +#include #include #include #include @@ -20,11 +23,10 @@ #include #include #include -#include #include +#include +#include #include -#include -#include struct iommu_dma_msi_page { struct list_head list; @@ -41,7 +43,19 @@ struct iommu_dma_cookie { enum iommu_dma_cookie_type type; union { /* Full allocator for IOMMU_DMA_IOVA_COOKIE */ - struct iova_domain iovad; + struct { + struct iova_domain iovad; + + struct iova_fq __percpu *fq; /* Flush queue */ + /* Number of TLB flushes that have been started */ + atomic64_t fq_flush_start_cnt; + /* Number of TLB flushes that have been finished */ + atomic64_t fq_flush_finish_cnt; + /* Timer to regularily empty the flush queues */ + struct timer_list fq_timer; + /* 1 when timer is active, 0 when not */ + atomic_t fq_timer_on; + }; /* Trivial linear page allocator for IOMMU_DMA_MSI_COOKIE */ dma_addr_t msi_iova; }; @@ -64,6 +78,27 @@ static int __init iommu_dma_forcedac_setup(char *str) } early_param("iommu.forcedac", iommu_dma_forcedac_setup); +/* Number of entries per flush queue */ +#define IOVA_FQ_SIZE 256 + +/* Timeout (in ms) after which entries are flushed from the queue */ +#define IOVA_FQ_TIMEOUT 10 + +/* Flush queue entry for deferred flushing */ +struct iova_fq_entry { + unsigned long iova_pfn; + unsigned long pages; + struct list_head freelist; + u64 counter; /* Flush counter when this entry was added */ +}; + +/* Per-CPU flush queue structure */ +struct iova_fq { + struct iova_fq_entry entries[IOVA_FQ_SIZE]; + unsigned int head, tail; + spinlock_t lock; +}; + #define fq_ring_for_each(i, fq) \ for ((i) = (fq)->head; (i) != (fq)->tail; (i) = ((i) + 1) % IOVA_FQ_SIZE) @@ -73,9 +108,9 @@ static inline bool fq_full(struct iova_fq *fq) return (((fq->tail + 1) % IOVA_FQ_SIZE) == fq->head); } -static inline unsigned fq_ring_add(struct iova_fq *fq) +static inline unsigned int fq_ring_add(struct iova_fq *fq) { - unsigned idx = fq->tail; + unsigned int idx = fq->tail; assert_spin_locked(&fq->lock); @@ -84,10 +119,10 @@ static inline unsigned fq_ring_add(struct iova_fq *fq) return idx; } -static void fq_ring_free(struct iova_domain *iovad, struct iova_fq *fq) +static void fq_ring_free(struct iommu_dma_cookie *cookie, struct iova_fq *fq) { - u64 counter = atomic64_read(&iovad->fq_flush_finish_cnt); - unsigned idx; + u64 counter = atomic64_read(&cookie->fq_flush_finish_cnt); + unsigned int idx; assert_spin_locked(&fq->lock); @@ -97,7 +132,7 @@ static void fq_ring_free(struct iova_domain *iovad, struct iova_fq *fq) break; put_pages_list(&fq->entries[idx].freelist); - free_iova_fast(iovad, + free_iova_fast(&cookie->iovad, fq->entries[idx].iova_pfn, fq->entries[idx].pages); @@ -105,50 +140,50 @@ static void fq_ring_free(struct iova_domain *iovad, struct iova_fq *fq) } } -static void iova_domain_flush(struct iova_domain *iovad) +static void fq_flush_iotlb(struct iommu_dma_cookie *cookie) { - atomic64_inc(&iovad->fq_flush_start_cnt); - iovad->fq_domain->ops->flush_iotlb_all(iovad->fq_domain); - atomic64_inc(&iovad->fq_flush_finish_cnt); + atomic64_inc(&cookie->fq_flush_start_cnt); + cookie->fq_domain->ops->flush_iotlb_all(cookie->fq_domain); + atomic64_inc(&cookie->fq_flush_finish_cnt); } static void fq_flush_timeout(struct timer_list *t) { - struct iova_domain *iovad = from_timer(iovad, t, fq_timer); + struct iommu_dma_cookie *cookie = from_timer(cookie, t, fq_timer); int cpu; - atomic_set(&iovad->fq_timer_on, 0); - iova_domain_flush(iovad); + atomic_set(&cookie->fq_timer_on, 0); + fq_flush_iotlb(cookie); for_each_possible_cpu(cpu) { unsigned long flags; struct iova_fq *fq; - fq = per_cpu_ptr(iovad->fq, cpu); + fq = per_cpu_ptr(cookie->fq, cpu); spin_lock_irqsave(&fq->lock, flags); - fq_ring_free(iovad, fq); + fq_ring_free(cookie, fq); spin_unlock_irqrestore(&fq->lock, flags); } } -void queue_iova(struct iova_domain *iovad, +static void queue_iova(struct iommu_dma_cookie *cookie, unsigned long pfn, unsigned long pages, struct list_head *freelist) { struct iova_fq *fq; unsigned long flags; - unsigned idx; + unsigned int idx; /* * Order against the IOMMU driver's pagetable update from unmapping - * @pte, to guarantee that iova_domain_flush() observes that if called + * @pte, to guarantee that fq_flush_iotlb() observes that if called * from a different CPU before we release the lock below. Full barrier * so it also pairs with iommu_dma_init_fq() to avoid seeing partially * written fq state here. */ smp_mb(); - fq = raw_cpu_ptr(iovad->fq); + fq = raw_cpu_ptr(cookie->fq); spin_lock_irqsave(&fq->lock, flags); /* @@ -156,65 +191,66 @@ void queue_iova(struct iova_domain *iovad, * flushed out on another CPU. This makes the fq_full() check below less * likely to be true. */ - fq_ring_free(iovad, fq); + fq_ring_free(cookie, fq); if (fq_full(fq)) { - iova_domain_flush(iovad); - fq_ring_free(iovad, fq); + fq_flush_iotlb(cookie); + fq_ring_free(cookie, fq); } idx = fq_ring_add(fq); fq->entries[idx].iova_pfn = pfn; fq->entries[idx].pages = pages; - fq->entries[idx].counter = atomic64_read(&iovad->fq_flush_start_cnt); + fq->entries[idx].counter = atomic64_read(&cookie->fq_flush_start_cnt); list_splice(freelist, &fq->entries[idx].freelist); spin_unlock_irqrestore(&fq->lock, flags); /* Avoid false sharing as much as possible. */ - if (!atomic_read(&iovad->fq_timer_on) && - !atomic_xchg(&iovad->fq_timer_on, 1)) - mod_timer(&iovad->fq_timer, + if (!atomic_read(&cookie->fq_timer_on) && + !atomic_xchg(&cookie->fq_timer_on, 1)) + mod_timer(&cookie->fq_timer, jiffies + msecs_to_jiffies(IOVA_FQ_TIMEOUT)); } -static void free_iova_flush_queue(struct iova_domain *iovad) +static void iommu_dma_free_fq(struct iommu_dma_cookie *cookie) { int cpu, idx; - if (!iovad->fq) + if (!cookie->fq) return; - del_timer_sync(&iovad->fq_timer); - /* - * This code runs when the iova_domain is being detroyed, so don't - * bother to free iovas, just free any remaining pagetable pages. - */ + del_timer_sync(&cookie->fq_timer); + /* The IOVAs will be torn down separately, so just free our queued pages */ for_each_possible_cpu(cpu) { - struct iova_fq *fq = per_cpu_ptr(iovad->fq, cpu); + struct iova_fq *fq = per_cpu_ptr(cookie->fq, cpu); fq_ring_for_each(idx, fq) put_pages_list(&fq->entries[idx].freelist); } - free_percpu(iovad->fq); - - iovad->fq = NULL; - iovad->fq_domain = NULL; + free_percpu(cookie->fq); } -int init_iova_flush_queue(struct iova_domain *iovad, struct iommu_domain *fq_domain) +/* sysfs updates are serialised by the mutex of the group owning @domain */ +int iommu_dma_init_fq(struct iommu_domain *domain) { + struct iommu_dma_cookie *cookie = domain->iova_cookie; struct iova_fq __percpu *queue; int i, cpu; - atomic64_set(&iovad->fq_flush_start_cnt, 0); - atomic64_set(&iovad->fq_flush_finish_cnt, 0); + if (cookie->fq_domain) + return 0; + + atomic64_set(&cookie->fq_flush_start_cnt, 0); + atomic64_set(&cookie->fq_flush_finish_cnt, 0); queue = alloc_percpu(struct iova_fq); - if (!queue) + if (!queue) { + pr_warn("iova flush queue initialization failed\n"); return -ENOMEM; + } for_each_possible_cpu(cpu) { struct iova_fq *fq = per_cpu_ptr(queue, cpu); @@ -228,12 +264,16 @@ int init_iova_flush_queue(struct iova_domain *iovad, struct iommu_domain *fq_dom INIT_LIST_HEAD(&fq->entries[i].freelist); } - iovad->fq_domain = fq_domain; - iovad->fq = queue; - - timer_setup(&iovad->fq_timer, fq_flush_timeout, 0); - atomic_set(&iovad->fq_timer_on, 0); + cookie->fq = queue; + timer_setup(&cookie->fq_timer, fq_flush_timeout, 0); + atomic_set(&cookie->fq_timer_on, 0); + /* + * Prevent incomplete fq state being observable. Pairs with path from + * __iommu_dma_unmap() through iommu_dma_free_iova() to queue_iova() + */ + smp_wmb(); + WRITE_ONCE(cookie->fq_domain, domain); return 0; } @@ -318,7 +358,7 @@ void iommu_put_dma_cookie(struct iommu_domain *domain) return; if (cookie->type == IOMMU_DMA_IOVA_COOKIE && cookie->iovad.granule) { - free_iova_flush_queue(&cookie->iovad); + iommu_dma_free_fq(cookie); put_iova_domain(&cookie->iovad); } @@ -467,29 +507,6 @@ static bool dev_use_swiotlb(struct device *dev) return IS_ENABLED(CONFIG_SWIOTLB) && dev_is_untrusted(dev); } -/* sysfs updates are serialised by the mutex of the group owning @domain */ -int iommu_dma_init_fq(struct iommu_domain *domain) -{ - struct iommu_dma_cookie *cookie = domain->iova_cookie; - int ret; - - if (cookie->fq_domain) - return 0; - - ret = init_iova_flush_queue(&cookie->iovad, domain); - if (ret) { - pr_warn("iova flush queue initialization failed\n"); - return ret; - } - /* - * Prevent incomplete iovad->fq being observable. Pairs with path from - * __iommu_dma_unmap() through iommu_dma_free_iova() to queue_iova() - */ - smp_wmb(); - WRITE_ONCE(cookie->fq_domain, domain); - return 0; -} - /** * iommu_dma_init_domain - Initialise a DMA mapping domain * @domain: IOMMU domain previously prepared by iommu_get_dma_cookie() @@ -620,7 +637,7 @@ static void iommu_dma_free_iova(struct iommu_dma_cookie *cookie, if (cookie->type == IOMMU_DMA_MSI_COOKIE) cookie->msi_iova -= size; else if (gather && gather->queued) - queue_iova(iovad, iova_pfn(iovad, iova), + queue_iova(cookie, iova_pfn(iovad, iova), size >> iova_shift(iovad), &gather->freelist); else diff --git a/drivers/iommu/iova.c b/drivers/iommu/iova.c index 081e5c0cf940..b28c9435b898 100644 --- a/drivers/iommu/iova.c +++ b/drivers/iommu/iova.c @@ -61,8 +61,6 @@ init_iova_domain(struct iova_domain *iovad, unsigned long granule, iovad->start_pfn = start_pfn; iovad->dma_32bit_pfn = 1UL << (32 - iova_shift(iovad)); iovad->max32_alloc_size = iovad->dma_32bit_pfn; - iovad->fq_domain = NULL; - iovad->fq = NULL; iovad->anchor.pfn_lo = iovad->anchor.pfn_hi = IOVA_ANCHOR; rb_link_node(&iovad->anchor.node, NULL, &iovad->rbroot.rb_node); rb_insert_color(&iovad->anchor.node, &iovad->rbroot); diff --git a/include/linux/iova.h b/include/linux/iova.h index 072a09c06e8a..0abd48c5e622 100644 --- a/include/linux/iova.h +++ b/include/linux/iova.h @@ -12,9 +12,6 @@ #include #include #include -#include -#include -#include /* iova structure */ struct iova { @@ -36,27 +33,6 @@ struct iova_rcache { struct iova_cpu_rcache __percpu *cpu_rcaches; }; -/* Number of entries per Flush Queue */ -#define IOVA_FQ_SIZE 256 - -/* Timeout (in ms) after which entries are flushed from the Flush-Queue */ -#define IOVA_FQ_TIMEOUT 10 - -/* Flush Queue entry for defered flushing */ -struct iova_fq_entry { - unsigned long iova_pfn; - unsigned long pages; - struct list_head freelist; - u64 counter; /* Flush counter when this entrie was added */ -}; - -/* Per-CPU Flush Queue structure */ -struct iova_fq { - struct iova_fq_entry entries[IOVA_FQ_SIZE]; - unsigned head, tail; - spinlock_t lock; -}; - /* holds all the iova translations for a domain */ struct iova_domain { spinlock_t iova_rbtree_lock; /* Lock to protect update of rbtree */ @@ -67,23 +43,9 @@ struct iova_domain { unsigned long start_pfn; /* Lower limit for this domain */ unsigned long dma_32bit_pfn; unsigned long max32_alloc_size; /* Size of last failed allocation */ - struct iova_fq __percpu *fq; /* Flush Queue */ - - atomic64_t fq_flush_start_cnt; /* Number of TLB flushes that - have been started */ - - atomic64_t fq_flush_finish_cnt; /* Number of TLB flushes that - have been finished */ - struct iova anchor; /* rbtree lookup anchor */ - struct iova_rcache rcaches[IOVA_RANGE_CACHE_MAX_SIZE]; /* IOVA range caches */ - - struct iommu_domain *fq_domain; - struct timer_list fq_timer; /* Timer to regularily empty the - flush-queues */ - atomic_t fq_timer_on; /* 1 when timer is active, 0 - when not */ + struct iova_rcache rcaches[IOVA_RANGE_CACHE_MAX_SIZE]; /* IOVA range caches */ struct hlist_node cpuhp_dead; }; @@ -133,16 +95,12 @@ struct iova *alloc_iova(struct iova_domain *iovad, unsigned long size, bool size_aligned); void free_iova_fast(struct iova_domain *iovad, unsigned long pfn, unsigned long size); -void queue_iova(struct iova_domain *iovad, - unsigned long pfn, unsigned long pages, - struct list_head *freelist); unsigned long alloc_iova_fast(struct iova_domain *iovad, unsigned long size, unsigned long limit_pfn, bool flush_rcache); struct iova *reserve_iova(struct iova_domain *iovad, unsigned long pfn_lo, unsigned long pfn_hi); void init_iova_domain(struct iova_domain *iovad, unsigned long granule, unsigned long start_pfn); -int init_iova_flush_queue(struct iova_domain *iovad, struct iommu_domain *fq_domain); struct iova *find_iova(struct iova_domain *iovad, unsigned long pfn); void put_iova_domain(struct iova_domain *iovad); #else -- cgit v1.2.3-73-gaa49b