diff options
41 files changed, 224 insertions, 593 deletions
diff --git a/MAINTAINERS b/MAINTAINERS index d30f26e07cd3..09abd5444fe7 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -10626,8 +10626,8 @@ L: [email protected] S: Maintained T: git git://git.kernel.org/pub/scm/linux/kernel/git/joro/iommu.git F: drivers/iommu/dma-iommu.c +F: drivers/iommu/dma-iommu.h F: drivers/iommu/iova.c -F: include/linux/dma-iommu.h F: include/linux/iova.h IOMMU SUBSYSTEM diff --git a/arch/arm64/Kconfig b/arch/arm64/Kconfig index 9fb9fff08c94..ff4755fd2cb3 100644 --- a/arch/arm64/Kconfig +++ b/arch/arm64/Kconfig @@ -209,7 +209,6 @@ config ARM64 select HAVE_KPROBES select HAVE_KRETPROBES select HAVE_GENERIC_VDSO - select IOMMU_DMA if IOMMU_SUPPORT select IRQ_DOMAIN select IRQ_FORCED_THREADING select KASAN_VMALLOC if KASAN diff --git a/arch/arm64/mm/dma-mapping.c b/arch/arm64/mm/dma-mapping.c index 599cf81f5685..7d7e9a046305 100644 --- a/arch/arm64/mm/dma-mapping.c +++ b/arch/arm64/mm/dma-mapping.c @@ -7,7 +7,7 @@ #include <linux/gfp.h> #include <linux/cache.h> #include <linux/dma-map-ops.h> -#include <linux/dma-iommu.h> +#include <linux/iommu.h> #include <xen/xen.h> #include <asm/cacheflush.h> diff --git a/drivers/acpi/viot.c b/drivers/acpi/viot.c index 6132092dab2a..ed752cbbe636 100644 --- a/drivers/acpi/viot.c +++ b/drivers/acpi/viot.c @@ -19,7 +19,6 @@ #define pr_fmt(fmt) "ACPI: VIOT: " fmt #include <linux/acpi_viot.h> -#include <linux/dma-iommu.h> #include <linux/fwnode.h> #include <linux/iommu.h> #include <linux/list.h> diff --git a/drivers/gpu/drm/exynos/exynos_drm_dma.c b/drivers/gpu/drm/exynos/exynos_drm_dma.c index bf33c3084cb4..a971590b8132 100644 --- a/drivers/gpu/drm/exynos/exynos_drm_dma.c +++ b/drivers/gpu/drm/exynos/exynos_drm_dma.c @@ -4,7 +4,6 @@ // Author: Inki Dae <[email protected]> // Author: Andrzej Hajda <[email protected]> -#include <linux/dma-iommu.h> #include <linux/dma-map-ops.h> #include <linux/iommu.h> #include <linux/platform_device.h> diff --git a/drivers/iommu/Kconfig b/drivers/iommu/Kconfig index 5c5cb5bee8b6..1d99c2d984fb 100644 --- a/drivers/iommu/Kconfig +++ b/drivers/iommu/Kconfig @@ -137,7 +137,7 @@ config OF_IOMMU # IOMMU-agnostic DMA-mapping layer config IOMMU_DMA - bool + def_bool ARM64 || IA64 || X86 select DMA_OPS select IOMMU_API select IOMMU_IOVA @@ -476,7 +476,6 @@ config VIRTIO_IOMMU depends on VIRTIO depends on (ARM64 || X86) select IOMMU_API - select IOMMU_DMA select INTERVAL_TREE select ACPI_VIOT if ACPI help diff --git a/drivers/iommu/amd/Kconfig b/drivers/iommu/amd/Kconfig index a3cbafb603f5..9b5fc3356bf2 100644 --- a/drivers/iommu/amd/Kconfig +++ b/drivers/iommu/amd/Kconfig @@ -9,7 +9,6 @@ config AMD_IOMMU select PCI_PASID select IOMMU_API select IOMMU_IOVA - select IOMMU_DMA select IOMMU_IO_PGTABLE depends on X86_64 && PCI && ACPI && HAVE_CMPXCHG_DOUBLE help diff --git a/drivers/iommu/amd/amd_iommu.h b/drivers/iommu/amd/amd_iommu.h index 84e5bb1bf01b..c160a332ce33 100644 --- a/drivers/iommu/amd/amd_iommu.h +++ b/drivers/iommu/amd/amd_iommu.h @@ -18,7 +18,6 @@ extern void amd_iommu_restart_event_logging(struct amd_iommu *iommu); extern int amd_iommu_init_devices(void); extern void amd_iommu_uninit_devices(void); extern void amd_iommu_init_notifier(void); -extern int amd_iommu_init_api(void); extern void amd_iommu_set_rlookup_table(struct amd_iommu *iommu, u16 devid); #ifdef CONFIG_AMD_IOMMU_DEBUGFS diff --git a/drivers/iommu/amd/init.c b/drivers/iommu/amd/init.c index fdc642362c14..79e0286da6ce 100644 --- a/drivers/iommu/amd/init.c +++ b/drivers/iommu/amd/init.c @@ -2168,20 +2168,13 @@ static int __init amd_iommu_init_pci(void) /* * Order is important here to make sure any unity map requirements are * fulfilled. The unity mappings are created and written to the device - * table during the amd_iommu_init_api() call. + * table during the iommu_init_pci() call. * * After that we call init_device_table_dma() to make sure any * uninitialized DTE will block DMA, and in the end we flush the caches * of all IOMMUs to make sure the changes to the device table are * active. */ - ret = amd_iommu_init_api(); - if (ret) { - pr_err("IOMMU: Failed to initialize IOMMU-API interface (error=%d)!\n", - ret); - goto out; - } - for_each_pci_segment(pci_seg) init_device_table_dma(pci_seg); diff --git a/drivers/iommu/amd/iommu.c b/drivers/iommu/amd/iommu.c index 828672a46a3d..67de9c42205a 100644 --- a/drivers/iommu/amd/iommu.c +++ b/drivers/iommu/amd/iommu.c @@ -11,8 +11,6 @@ #include <linux/ratelimit.h> #include <linux/pci.h> #include <linux/acpi.h> -#include <linux/amba/bus.h> -#include <linux/platform_device.h> #include <linux/pci-ats.h> #include <linux/bitmap.h> #include <linux/slab.h> @@ -20,7 +18,6 @@ #include <linux/scatterlist.h> #include <linux/dma-map-ops.h> #include <linux/dma-direct.h> -#include <linux/dma-iommu.h> #include <linux/iommu-helper.h> #include <linux/delay.h> #include <linux/amd-iommu.h> @@ -42,6 +39,7 @@ #include <asm/dma.h> #include "amd_iommu.h" +#include "../dma-iommu.h" #include "../irq_remapping.h" #define CMD_SET_TYPE(cmd, t) ((cmd)->data[1] |= ((t) << 28)) @@ -1852,6 +1850,10 @@ static struct iommu_device *amd_iommu_probe_device(struct device *dev) if (!iommu) return ERR_PTR(-ENODEV); + /* Not registered yet? */ + if (!iommu->iommu.ops) + return ERR_PTR(-ENODEV); + if (dev_iommu_priv_get(dev)) return &iommu->iommu; @@ -1938,25 +1940,6 @@ void amd_iommu_domain_update(struct protection_domain *domain) amd_iommu_domain_flush_complete(domain); } -int __init amd_iommu_init_api(void) -{ - int err; - - err = bus_set_iommu(&pci_bus_type, &amd_iommu_ops); - if (err) - return err; -#ifdef CONFIG_ARM_AMBA - err = bus_set_iommu(&amba_bustype, &amd_iommu_ops); - if (err) - return err; -#endif - err = bus_set_iommu(&platform_bus_type, &amd_iommu_ops); - if (err) - return err; - - return 0; -} - /***************************************************************************** * * The following functions belong to the exported interface of AMD IOMMU @@ -2252,7 +2235,7 @@ static phys_addr_t amd_iommu_iova_to_phys(struct iommu_domain *dom, return ops->iova_to_phys(ops, iova); } -static bool amd_iommu_capable(enum iommu_cap cap) +static bool amd_iommu_capable(struct device *dev, enum iommu_cap cap) { switch (cap) { case IOMMU_CAP_CACHE_COHERENCY: diff --git a/drivers/iommu/apple-dart.c b/drivers/iommu/apple-dart.c index 1b1725759262..79643b84cb14 100644 --- a/drivers/iommu/apple-dart.c +++ b/drivers/iommu/apple-dart.c @@ -15,7 +15,6 @@ #include <linux/bitfield.h> #include <linux/clk.h> #include <linux/dev_printk.h> -#include <linux/dma-iommu.h> #include <linux/dma-mapping.h> #include <linux/err.h> #include <linux/interrupt.h> @@ -33,6 +32,8 @@ #include <linux/swab.h> #include <linux/types.h> +#include "dma-iommu.h" + #define DART_MAX_STREAMS 16 #define DART_MAX_TTBR 4 #define MAX_DARTS_PER_DEVICE 2 @@ -820,27 +821,6 @@ static irqreturn_t apple_dart_irq(int irq, void *dev) return IRQ_HANDLED; } -static int apple_dart_set_bus_ops(const struct iommu_ops *ops) -{ - int ret; - - if (!iommu_present(&platform_bus_type)) { - ret = bus_set_iommu(&platform_bus_type, ops); - if (ret) - return ret; - } -#ifdef CONFIG_PCI - if (!iommu_present(&pci_bus_type)) { - ret = bus_set_iommu(&pci_bus_type, ops); - if (ret) { - bus_set_iommu(&platform_bus_type, NULL); - return ret; - } - } -#endif - return 0; -} - static int apple_dart_probe(struct platform_device *pdev) { int ret; @@ -895,14 +875,10 @@ static int apple_dart_probe(struct platform_device *pdev) platform_set_drvdata(pdev, dart); - ret = apple_dart_set_bus_ops(&apple_dart_iommu_ops); - if (ret) - goto err_free_irq; - ret = iommu_device_sysfs_add(&dart->iommu, dev, NULL, "apple-dart.%s", dev_name(&pdev->dev)); if (ret) - goto err_remove_bus_ops; + goto err_free_irq; ret = iommu_device_register(&dart->iommu, &apple_dart_iommu_ops, dev); if (ret) @@ -916,8 +892,6 @@ static int apple_dart_probe(struct platform_device *pdev) err_sysfs_remove: iommu_device_sysfs_remove(&dart->iommu); -err_remove_bus_ops: - apple_dart_set_bus_ops(NULL); err_free_irq: free_irq(dart->irq, dart); err_clk_disable: @@ -932,7 +906,6 @@ static int apple_dart_remove(struct platform_device *pdev) apple_dart_hw_reset(dart); free_irq(dart->irq, dart); - apple_dart_set_bus_ops(NULL); iommu_device_unregister(&dart->iommu); iommu_device_sysfs_remove(&dart->iommu); diff --git a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c index d32b02336411..b788a38d8fdf 100644 --- a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c +++ b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c @@ -14,7 +14,6 @@ #include <linux/bitops.h> #include <linux/crash_dump.h> #include <linux/delay.h> -#include <linux/dma-iommu.h> #include <linux/err.h> #include <linux/interrupt.h> #include <linux/io-pgtable.h> @@ -28,9 +27,8 @@ #include <linux/pci-ats.h> #include <linux/platform_device.h> -#include <linux/amba/bus.h> - #include "arm-smmu-v3.h" +#include "../../dma-iommu.h" #include "../../iommu-sva-lib.h" static bool disable_bypass = true; @@ -1992,11 +1990,14 @@ static const struct iommu_flush_ops arm_smmu_flush_ops = { }; /* IOMMU API */ -static bool arm_smmu_capable(enum iommu_cap cap) +static bool arm_smmu_capable(struct device *dev, enum iommu_cap cap) { + struct arm_smmu_master *master = dev_iommu_priv_get(dev); + switch (cap) { case IOMMU_CAP_CACHE_COHERENCY: - return true; + /* Assume that a coherent TCU implies coherent TBUs */ + return master->smmu->features & ARM_SMMU_FEAT_COHERENCY; case IOMMU_CAP_NOEXEC: return true; default: @@ -3673,43 +3674,6 @@ static unsigned long arm_smmu_resource_size(struct arm_smmu_device *smmu) return SZ_128K; } -static int arm_smmu_set_bus_ops(struct iommu_ops *ops) -{ - int err; - -#ifdef CONFIG_PCI - if (pci_bus_type.iommu_ops != ops) { - err = bus_set_iommu(&pci_bus_type, ops); - if (err) - return err; - } -#endif -#ifdef CONFIG_ARM_AMBA - if (amba_bustype.iommu_ops != ops) { - err = bus_set_iommu(&amba_bustype, ops); - if (err) - goto err_reset_pci_ops; - } -#endif - if (platform_bus_type.iommu_ops != ops) { - err = bus_set_iommu(&platform_bus_type, ops); - if (err) - goto err_reset_amba_ops; - } - - return 0; - -err_reset_amba_ops: -#ifdef CONFIG_ARM_AMBA - bus_set_iommu(&amba_bustype, NULL); -#endif -err_reset_pci_ops: __maybe_unused; -#ifdef CONFIG_PCI - bus_set_iommu(&pci_bus_type, NULL); -#endif - return err; -} - static void __iomem *arm_smmu_ioremap(struct device *dev, resource_size_t start, resource_size_t size) { @@ -3848,27 +3812,17 @@ static int arm_smmu_device_probe(struct platform_device *pdev) ret = iommu_device_register(&smmu->iommu, &arm_smmu_ops, dev); if (ret) { dev_err(dev, "Failed to register iommu\n"); - goto err_sysfs_remove; + iommu_device_sysfs_remove(&smmu->iommu); + return ret; } - ret = arm_smmu_set_bus_ops(&arm_smmu_ops); - if (ret) - goto err_unregister_device; - return 0; - -err_unregister_device: - iommu_device_unregister(&smmu->iommu); -err_sysfs_remove: - iommu_device_sysfs_remove(&smmu->iommu); - return ret; } static int arm_smmu_device_remove(struct platform_device *pdev) { struct arm_smmu_device *smmu = platform_get_drvdata(pdev); - arm_smmu_set_bus_ops(NULL); iommu_device_unregister(&smmu->iommu); iommu_device_sysfs_remove(&smmu->iommu); arm_smmu_device_disable(smmu); diff --git a/drivers/iommu/arm/arm-smmu/arm-smmu.c b/drivers/iommu/arm/arm-smmu/arm-smmu.c index dfa82df00342..6c1114a4d6cc 100644 --- a/drivers/iommu/arm/arm-smmu/arm-smmu.c +++ b/drivers/iommu/arm/arm-smmu/arm-smmu.c @@ -21,7 +21,6 @@ #include <linux/acpi_iort.h> #include <linux/bitfield.h> #include <linux/delay.h> -#include <linux/dma-iommu.h> #include <linux/dma-mapping.h> #include <linux/err.h> #include <linux/interrupt.h> @@ -37,10 +36,10 @@ #include <linux/ratelimit.h> #include <linux/slab.h> -#include <linux/amba/bus.h> #include <linux/fsl/mc.h> #include "arm-smmu.h" +#include "../../dma-iommu.h" /* * Apparently, some Qualcomm arm64 platforms which appear to expose their SMMU @@ -93,8 +92,6 @@ static struct platform_driver arm_smmu_driver; static struct iommu_ops arm_smmu_ops; #ifdef CONFIG_ARM_SMMU_LEGACY_DT_BINDINGS -static int arm_smmu_bus_init(struct iommu_ops *ops); - static struct device_node *dev_get_dev_node(struct device *dev) { if (dev_is_pci(dev)) { @@ -180,20 +177,6 @@ static int arm_smmu_register_legacy_master(struct device *dev, kfree(sids); return err; } - -/* - * With the legacy DT binding in play, we have no guarantees about - * probe order, but then we're also not doing default domains, so we can - * delay setting bus ops until we're sure every possible SMMU is ready, - * and that way ensure that no probe_device() calls get missed. - */ -static int arm_smmu_legacy_bus_init(void) -{ - if (using_legacy_binding) - return arm_smmu_bus_init(&arm_smmu_ops); - return 0; -} -device_initcall_sync(arm_smmu_legacy_bus_init); #else static int arm_smmu_register_legacy_master(struct device *dev, struct arm_smmu_device **smmu) @@ -1330,15 +1313,14 @@ static phys_addr_t arm_smmu_iova_to_phys(struct iommu_domain *domain, return ops->iova_to_phys(ops, iova); } -static bool arm_smmu_capable(enum iommu_cap cap) +static bool arm_smmu_capable(struct device *dev, enum iommu_cap cap) { + struct arm_smmu_master_cfg *cfg = dev_iommu_priv_get(dev); + switch (cap) { case IOMMU_CAP_CACHE_COHERENCY: - /* - * Return true here as the SMMU can always send out coherent - * requests. - */ - return true; + /* Assume that a coherent TCU implies coherent TBUs */ + return cfg->smmu->features & ARM_SMMU_FEAT_COHERENT_WALK; case IOMMU_CAP_NOEXEC: return true; default: @@ -2016,52 +1998,6 @@ static int arm_smmu_device_dt_probe(struct arm_smmu_device *smmu, return 0; } -static int arm_smmu_bus_init(struct iommu_ops *ops) -{ - int err; - - /* Oh, for a proper bus abstraction */ - if (!iommu_present(&platform_bus_type)) { - err = bus_set_iommu(&platform_bus_type, ops); - if (err) - return err; - } -#ifdef CONFIG_ARM_AMBA - if (!iommu_present(&amba_bustype)) { - err = bus_set_iommu(&amba_bustype, ops); - if (err) - goto err_reset_platform_ops; - } -#endif -#ifdef CONFIG_PCI - if (!iommu_present(&pci_bus_type)) { - err = bus_set_iommu(&pci_bus_type, ops); - if (err) - goto err_reset_amba_ops; - } -#endif -#ifdef CONFIG_FSL_MC_BUS - if (!iommu_present(&fsl_mc_bus_type)) { - err = bus_set_iommu(&fsl_mc_bus_type, ops); - if (err) - goto err_reset_pci_ops; - } -#endif - return 0; - -err_reset_pci_ops: __maybe_unused; -#ifdef CONFIG_PCI - bus_set_iommu(&pci_bus_type, NULL); -#endif -err_reset_amba_ops: __maybe_unused; -#ifdef CONFIG_ARM_AMBA - bus_set_iommu(&amba_bustype, NULL); -#endif -err_reset_platform_ops: __maybe_unused; - bus_set_iommu(&platform_bus_type, NULL); - return err; -} - static void arm_smmu_rmr_install_bypass_smr(struct arm_smmu_device *smmu) { struct list_head rmr_list; @@ -2226,7 +2162,8 @@ static int arm_smmu_device_probe(struct platform_device *pdev) err = iommu_device_register(&smmu->iommu, &arm_smmu_ops, dev); if (err) { dev_err(dev, "Failed to register iommu\n"); - goto err_sysfs_remove; + iommu_device_sysfs_remove(&smmu->iommu); + return err; } platform_set_drvdata(pdev, smmu); @@ -2248,24 +2185,7 @@ static int arm_smmu_device_probe(struct platform_device *pdev) pm_runtime_enable(dev); } - /* - * For ACPI and generic DT bindings, an SMMU will be probed before - * any device which might need it, so we want the bus ops in place - * ready to handle default domain setup as soon as any SMMU exists. - */ - if (!using_legacy_binding) { - err = arm_smmu_bus_init(&arm_smmu_ops); - if (err) - goto err_unregister_device; - } - return 0; - -err_unregister_device: - iommu_device_unregister(&smmu->iommu); -err_sysfs_remove: - iommu_device_sysfs_remove(&smmu->iommu); - return err; } static int arm_smmu_device_remove(struct platform_device *pdev) @@ -2278,7 +2198,6 @@ static int arm_smmu_device_remove(struct platform_device *pdev) if (!bitmap_empty(smmu->context_map, ARM_SMMU_MAX_CBS)) dev_notice(&pdev->dev, "disabling translation\n"); - arm_smmu_bus_init(NULL); iommu_device_unregister(&smmu->iommu); iommu_device_sysfs_remove(&smmu->iommu); diff --git a/drivers/iommu/arm/arm-smmu/qcom_iommu.c b/drivers/iommu/arm/arm-smmu/qcom_iommu.c index 17235116d3bb..3869c3ecda8c 100644 --- a/drivers/iommu/arm/arm-smmu/qcom_iommu.c +++ b/drivers/iommu/arm/arm-smmu/qcom_iommu.c @@ -493,7 +493,7 @@ static phys_addr_t qcom_iommu_iova_to_phys(struct iommu_domain *domain, return ret; } -static bool qcom_iommu_capable(enum iommu_cap cap) +static bool qcom_iommu_capable(struct device *dev, enum iommu_cap cap) { switch (cap) { case IOMMU_CAP_CACHE_COHERENCY: @@ -837,8 +837,6 @@ static int qcom_iommu_device_probe(struct platform_device *pdev) goto err_pm_disable; } - bus_set_iommu(&platform_bus_type, &qcom_iommu_ops); - if (qcom_iommu->local_base) { pm_runtime_get_sync(dev); writel_relaxed(0xffffffff, qcom_iommu->local_base + SMMU_INTR_SEL_NS); @@ -856,8 +854,6 @@ static int qcom_iommu_device_remove(struct platform_device *pdev) { struct qcom_iommu_dev *qcom_iommu = platform_get_drvdata(pdev); - bus_set_iommu(&platform_bus_type, NULL); - pm_runtime_force_suspend(&pdev->dev); platform_set_drvdata(pdev, NULL); iommu_device_sysfs_remove(&qcom_iommu->iommu); diff --git a/drivers/iommu/dma-iommu.c b/drivers/iommu/dma-iommu.c index 17dd683b2fce..9297b741f5e8 100644 --- a/drivers/iommu/dma-iommu.c +++ b/drivers/iommu/dma-iommu.c @@ -13,7 +13,6 @@ #include <linux/crash_dump.h> #include <linux/device.h> #include <linux/dma-direct.h> -#include <linux/dma-iommu.h> #include <linux/dma-map-ops.h> #include <linux/gfp.h> #include <linux/huge_mm.h> @@ -30,6 +29,8 @@ #include <linux/swiotlb.h> #include <linux/vmalloc.h> +#include "dma-iommu.h" + struct iommu_dma_msi_page { struct list_head list; dma_addr_t iova; @@ -1633,6 +1634,13 @@ out_free_page: return NULL; } +/** + * iommu_dma_prepare_msi() - Map the MSI page in the IOMMU domain + * @desc: MSI descriptor, will store the MSI page + * @msi_addr: MSI target address to be mapped + * + * Return: 0 on success or negative error code if the mapping failed. + */ int iommu_dma_prepare_msi(struct msi_desc *desc, phys_addr_t msi_addr) { struct device *dev = msi_desc_to_dev(desc); @@ -1661,8 +1669,12 @@ int iommu_dma_prepare_msi(struct msi_desc *desc, phys_addr_t msi_addr) return 0; } -void iommu_dma_compose_msi_msg(struct msi_desc *desc, - struct msi_msg *msg) +/** + * iommu_dma_compose_msi_msg() - Apply translation to an MSI message + * @desc: MSI descriptor prepared by iommu_dma_prepare_msi() + * @msg: MSI message containing target physical address + */ +void iommu_dma_compose_msi_msg(struct msi_desc *desc, struct msi_msg *msg) { struct device *dev = msi_desc_to_dev(desc); const struct iommu_domain *domain = iommu_get_domain_for_dev(dev); diff --git a/drivers/iommu/dma-iommu.h b/drivers/iommu/dma-iommu.h new file mode 100644 index 000000000000..942790009292 --- /dev/null +++ b/drivers/iommu/dma-iommu.h @@ -0,0 +1,42 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * Copyright (C) 2014-2015 ARM Ltd. + */ +#ifndef __DMA_IOMMU_H +#define __DMA_IOMMU_H + +#include <linux/iommu.h> + +#ifdef CONFIG_IOMMU_DMA + +int iommu_get_dma_cookie(struct iommu_domain *domain); +void iommu_put_dma_cookie(struct iommu_domain *domain); + +int iommu_dma_init_fq(struct iommu_domain *domain); + +void iommu_dma_get_resv_regions(struct device *dev, struct list_head *list); + +extern bool iommu_dma_forcedac; + +#else /* CONFIG_IOMMU_DMA */ + +static inline int iommu_dma_init_fq(struct iommu_domain *domain) +{ + return -EINVAL; +} + +static inline int iommu_get_dma_cookie(struct iommu_domain *domain) +{ + return -ENODEV; +} + +static inline void iommu_put_dma_cookie(struct iommu_domain *domain) +{ +} + +static inline void iommu_dma_get_resv_regions(struct device *dev, struct list_head *list) +{ +} + +#endif /* CONFIG_IOMMU_DMA */ +#endif /* __DMA_IOMMU_H */ diff --git a/drivers/iommu/exynos-iommu.c b/drivers/iommu/exynos-iommu.c index 8e18984a0c4f..45fd4850bacb 100644 --- a/drivers/iommu/exynos-iommu.c +++ b/drivers/iommu/exynos-iommu.c @@ -1446,16 +1446,7 @@ static int __init exynos_iommu_init(void) goto err_zero_lv2; } - ret = bus_set_iommu(&platform_bus_type, &exynos_iommu_ops); - if (ret) { - pr_err("%s: Failed to register exynos-iommu driver.\n", - __func__); - goto err_set_iommu; - } - return 0; -err_set_iommu: - kmem_cache_free(lv2table_kmem_cache, zero_lv2_table); err_zero_lv2: platform_driver_unregister(&exynos_sysmmu_driver); err_reg_driver: diff --git a/drivers/iommu/fsl_pamu_domain.c b/drivers/iommu/fsl_pamu_domain.c index 011f9ab7f743..fa20f4b03e12 100644 --- a/drivers/iommu/fsl_pamu_domain.c +++ b/drivers/iommu/fsl_pamu_domain.c @@ -178,7 +178,7 @@ static phys_addr_t fsl_pamu_iova_to_phys(struct iommu_domain *domain, return iova; } -static bool fsl_pamu_capable(enum iommu_cap cap) +static bool fsl_pamu_capable(struct device *dev, enum iommu_cap cap) { return cap == IOMMU_CAP_CACHE_COHERENCY; } @@ -476,11 +476,7 @@ int __init pamu_domain_init(void) if (ret) { iommu_device_sysfs_remove(&pamu_iommu); pr_err("Can't register iommu device\n"); - return ret; } - bus_set_iommu(&platform_bus_type, &fsl_pamu_ops); - bus_set_iommu(&pci_bus_type, &fsl_pamu_ops); - return ret; } diff --git a/drivers/iommu/intel/Kconfig b/drivers/iommu/intel/Kconfig index 39a06d245f12..c48005147ac5 100644 --- a/drivers/iommu/intel/Kconfig +++ b/drivers/iommu/intel/Kconfig @@ -19,7 +19,6 @@ config INTEL_IOMMU select DMAR_TABLE select SWIOTLB select IOASID - select IOMMU_DMA select PCI_ATS help DMA remapping (DMAR) devices support enables independent address diff --git a/drivers/iommu/intel/iommu.c b/drivers/iommu/intel/iommu.c index 64d30895a4c8..2c6eca037f0a 100644 --- a/drivers/iommu/intel/iommu.c +++ b/drivers/iommu/intel/iommu.c @@ -15,7 +15,6 @@ #include <linux/crash_dump.h> #include <linux/dma-direct.h> -#include <linux/dma-iommu.h> #include <linux/dmi.h> #include <linux/intel-svm.h> #include <linux/memory.h> @@ -26,6 +25,7 @@ #include <linux/tboot.h> #include "iommu.h" +#include "../dma-iommu.h" #include "../irq_remapping.h" #include "../iommu-sva-lib.h" #include "pasid.h" @@ -3896,7 +3896,6 @@ static int __init probe_acpi_namespace_devices(void) continue; } - pn->dev->bus->iommu_ops = &intel_iommu_ops; ret = iommu_probe_device(pn->dev); if (ret) break; @@ -4029,7 +4028,6 @@ int __init intel_iommu_init(void) } up_read(&dmar_global_lock); - bus_set_iommu(&pci_bus_type, &intel_iommu_ops); if (si_domain && !hw_pass_through) register_memory_notifier(&intel_iommu_memory_nb); @@ -4437,7 +4435,7 @@ static bool intel_iommu_enforce_cache_coherency(struct iommu_domain *domain) return true; } -static bool intel_iommu_capable(enum iommu_cap cap) +static bool intel_iommu_capable(struct device *dev, enum iommu_cap cap) { if (cap == IOMMU_CAP_CACHE_COHERENCY) return true; @@ -4457,7 +4455,7 @@ static struct iommu_device *intel_iommu_probe_device(struct device *dev) u8 bus, devfn; iommu = device_to_iommu(dev, &bus, &devfn); - if (!iommu) + if (!iommu || !iommu->iommu.ops) return ERR_PTR(-ENODEV); info = kzalloc(sizeof(*info), GFP_KERNEL); diff --git a/drivers/iommu/io-pgtable-arm.c b/drivers/iommu/io-pgtable-arm.c index 94ff319ae8ac..873a92bbd80c 100644 --- a/drivers/iommu/io-pgtable-arm.c +++ b/drivers/iommu/io-pgtable-arm.c @@ -200,8 +200,7 @@ static void *__arm_lpae_alloc_pages(size_t size, gfp_t gfp, void *pages; VM_BUG_ON((gfp & __GFP_HIGHMEM)); - p = alloc_pages_node(dev ? dev_to_node(dev) : NUMA_NO_NODE, - gfp | __GFP_ZERO, order); + p = alloc_pages_node(dev_to_node(dev), gfp | __GFP_ZERO, order); if (!p) return NULL; @@ -1343,12 +1342,17 @@ static int __init arm_lpae_do_selftests(void) }; int i, j, pass = 0, fail = 0; + struct device dev; struct io_pgtable_cfg cfg = { .tlb = &dummy_tlb_ops, .oas = 48, .coherent_walk = true, + .iommu_dev = &dev, }; + /* __arm_lpae_alloc_pages() merely needs dev_to_node() to work */ + set_dev_node(&dev, NUMA_NO_NODE); + for (i = 0; i < ARRAY_SIZE(pgsize); ++i) { for (j = 0; j < ARRAY_SIZE(ias); ++j) { cfg.pgsize_bitmap = pgsize[i]; diff --git a/drivers/iommu/iommu.c b/drivers/iommu/iommu.c index 780fb7071577..c864cbb16523 100644 --- a/drivers/iommu/iommu.c +++ b/drivers/iommu/iommu.c @@ -6,8 +6,8 @@ #define pr_fmt(fmt) "iommu: " fmt +#include <linux/amba/bus.h> #include <linux/device.h> -#include <linux/dma-iommu.h> #include <linux/kernel.h> #include <linux/bits.h> #include <linux/bug.h> @@ -16,17 +16,21 @@ #include <linux/export.h> #include <linux/slab.h> #include <linux/errno.h> +#include <linux/host1x_context_bus.h> #include <linux/iommu.h> #include <linux/idr.h> #include <linux/err.h> #include <linux/pci.h> #include <linux/bitops.h> +#include <linux/platform_device.h> #include <linux/property.h> #include <linux/fsl/mc.h> #include <linux/module.h> #include <linux/cc_platform.h> #include <trace/events/iommu.h> +#include "dma-iommu.h" + static struct kset *iommu_group_kset; static DEFINE_IDA(iommu_group_ida); @@ -75,6 +79,8 @@ static const char * const iommu_group_resv_type_string[] = { #define IOMMU_CMD_LINE_DMA_API BIT(0) #define IOMMU_CMD_LINE_STRICT BIT(1) +static int iommu_bus_notifier(struct notifier_block *nb, + unsigned long action, void *data); static int iommu_alloc_default_domain(struct iommu_group *group, struct device *dev); static struct iommu_domain *__iommu_domain_alloc(struct bus_type *bus, @@ -103,6 +109,22 @@ struct iommu_group_attribute iommu_group_attr_##_name = \ static LIST_HEAD(iommu_device_list); static DEFINE_SPINLOCK(iommu_device_lock); +static struct bus_type * const iommu_buses[] = { + &platform_bus_type, +#ifdef CONFIG_PCI + &pci_bus_type, +#endif +#ifdef CONFIG_ARM_AMBA + &amba_bustype, +#endif +#ifdef CONFIG_FSL_MC_BUS + &fsl_mc_bus_type, +#endif +#ifdef CONFIG_TEGRA_HOST1X_CONTEXT_BUS + &host1x_context_device_bus_type, +#endif +}; + /* * Use a function instead of an array here because the domain-type is a * bit-field, so an array would waste memory. @@ -126,6 +148,8 @@ static const char *iommu_domain_type_str(unsigned int t) static int __init iommu_subsys_init(void) { + struct notifier_block *nb; + if (!(iommu_cmd_line & IOMMU_CMD_LINE_DMA_API)) { if (IS_ENABLED(CONFIG_IOMMU_DEFAULT_PASSTHROUGH)) iommu_set_default_passthrough(false); @@ -152,10 +176,27 @@ static int __init iommu_subsys_init(void) (iommu_cmd_line & IOMMU_CMD_LINE_STRICT) ? "(set via kernel command line)" : ""); + nb = kcalloc(ARRAY_SIZE(iommu_buses), sizeof(*nb), GFP_KERNEL); + if (!nb) + return -ENOMEM; + + for (int i = 0; i < ARRAY_SIZE(iommu_buses); i++) { + nb[i].notifier_call = iommu_bus_notifier; + bus_register_notifier(iommu_buses[i], &nb[i]); + } + return 0; } subsys_initcall(iommu_subsys_init); +static int remove_iommu_group(struct device *dev, void *data) +{ + if (dev->iommu && dev->iommu->iommu_dev == data) + iommu_release_device(dev); + + return 0; +} + /** * iommu_device_register() - Register an IOMMU hardware instance * @iommu: IOMMU handle for the instance @@ -167,23 +208,42 @@ subsys_initcall(iommu_subsys_init); int iommu_device_register(struct iommu_device *iommu, const struct iommu_ops *ops, struct device *hwdev) { + int err = 0; + /* We need to be able to take module references appropriately */ if (WARN_ON(is_module_address((unsigned long)ops) && !ops->owner)) return -EINVAL; + /* + * Temporarily enforce global restriction to a single driver. This was + * already the de-facto behaviour, since any possible combination of + * existing drivers would compete for at least the PCI or platform bus. + */ + if (iommu_buses[0]->iommu_ops && iommu_buses[0]->iommu_ops != ops) + return -EBUSY; iommu->ops = ops; if (hwdev) - iommu->fwnode = hwdev->fwnode; + iommu->fwnode = dev_fwnode(hwdev); spin_lock(&iommu_device_lock); list_add_tail(&iommu->list, &iommu_device_list); spin_unlock(&iommu_device_lock); - return 0; + + for (int i = 0; i < ARRAY_SIZE(iommu_buses) && !err; i++) { + iommu_buses[i]->iommu_ops = ops; + err = bus_iommu_probe(iommu_buses[i]); + } + if (err) + iommu_device_unregister(iommu); + return err; } EXPORT_SYMBOL_GPL(iommu_device_register); void iommu_device_unregister(struct iommu_device *iommu) { + for (int i = 0; i < ARRAY_SIZE(iommu_buses); i++) + bus_for_each_dev(iommu_buses[i], NULL, iommu, remove_iommu_group); + spin_lock(&iommu_device_lock); list_del(&iommu->list); spin_unlock(&iommu_device_lock); @@ -654,7 +714,6 @@ struct iommu_group *iommu_group_alloc(void) ret = kobject_init_and_add(&group->kobj, &iommu_group_ktype, NULL, "%d", group->id); if (ret) { - ida_free(&iommu_group_ida, group->id); kobject_put(&group->kobj); return ERR_PTR(ret); } @@ -1612,13 +1671,6 @@ static int probe_iommu_group(struct device *dev, void *data) return ret; } -static int remove_iommu_group(struct device *dev, void *data) -{ - iommu_release_device(dev); - - return 0; -} - static int iommu_bus_notifier(struct notifier_block *nb, unsigned long action, void *data) { @@ -1775,75 +1827,6 @@ int bus_iommu_probe(struct bus_type *bus) return ret; } -static int iommu_bus_init(struct bus_type *bus, const struct iommu_ops *ops) -{ - struct notifier_block *nb; - int err; - - nb = kzalloc(sizeof(struct notifier_block), GFP_KERNEL); - if (!nb) - return -ENOMEM; - - nb->notifier_call = iommu_bus_notifier; - - err = bus_register_notifier(bus, nb); - if (err) - goto out_free; - - err = bus_iommu_probe(bus); - if (err) - goto out_err; - - - return 0; - -out_err: - /* Clean up */ - bus_for_each_dev(bus, NULL, NULL, remove_iommu_group); - bus_unregister_notifier(bus, nb); - -out_free: - kfree(nb); - - return err; -} - -/** - * bus_set_iommu - set iommu-callbacks for the bus - * @bus: bus. - * @ops: the callbacks provided by the iommu-driver - * - * This function is called by an iommu driver to set the iommu methods - * used for a particular bus. Drivers for devices on that bus can use - * the iommu-api after these ops are registered. - * This special function is needed because IOMMUs are usually devices on - * the bus itself, so the iommu drivers are not initialized when the bus - * is set up. With this function the iommu-driver can set the iommu-ops - * afterwards. - */ -int bus_set_iommu(struct bus_type *bus, const struct iommu_ops *ops) -{ - int err; - - if (ops == NULL) { - bus->iommu_ops = NULL; - return 0; - } - - if (bus->iommu_ops != NULL) - return -EBUSY; - - bus->iommu_ops = ops; - - /* Do IOMMU specific setup for this bus-type */ - err = iommu_bus_init(bus, ops); - if (err) - bus->iommu_ops = NULL; - - return err; -} -EXPORT_SYMBOL_GPL(bus_set_iommu); - bool iommu_present(struct bus_type *bus) { return bus->iommu_ops != NULL; @@ -1869,19 +1852,10 @@ bool device_iommu_capable(struct device *dev, enum iommu_cap cap) if (!ops->capable) return false; - return ops->capable(cap); + return ops->capable(dev, cap); } EXPORT_SYMBOL_GPL(device_iommu_capable); -bool iommu_capable(struct bus_type *bus, enum iommu_cap cap) -{ - if (!bus->iommu_ops || !bus->iommu_ops->capable) - return false; - - return bus->iommu_ops->capable(cap); -} -EXPORT_SYMBOL_GPL(iommu_capable); - /** * iommu_set_fault_handler() - set a fault handler for an iommu domain * @domain: iommu domain diff --git a/drivers/iommu/iova.c b/drivers/iommu/iova.c index 47d1983dfa2a..a44ad92fc5eb 100644 --- a/drivers/iommu/iova.c +++ b/drivers/iommu/iova.c @@ -661,9 +661,6 @@ iova_magazine_free_pfns(struct iova_magazine *mag, struct iova_domain *iovad) unsigned long flags; int i; - if (!mag) - return; - spin_lock_irqsave(&iovad->iova_rbtree_lock, flags); for (i = 0 ; i < mag->size; ++i) { @@ -683,12 +680,12 @@ iova_magazine_free_pfns(struct iova_magazine *mag, struct iova_domain *iovad) static bool iova_magazine_full(struct iova_magazine *mag) { - return (mag && mag->size == IOVA_MAG_SIZE); + return mag->size == IOVA_MAG_SIZE; } static bool iova_magazine_empty(struct iova_magazine *mag) { - return (!mag || mag->size == 0); + return mag->size == 0; } static unsigned long iova_magazine_pop(struct iova_magazine *mag, @@ -697,8 +694,6 @@ static unsigned long iova_magazine_pop(struct iova_magazine *mag, int i; unsigned long pfn; - BUG_ON(iova_magazine_empty(mag)); - /* Only fall back to the rbtree if we have no suitable pfns at all */ for (i = mag->size - 1; mag->pfns[i] > limit_pfn; i--) if (i == 0) @@ -713,8 +708,6 @@ static unsigned long iova_magazine_pop(struct iova_magazine *mag, static void iova_magazine_push(struct iova_magazine *mag, unsigned long pfn) { - BUG_ON(iova_magazine_full(mag)); - mag->pfns[mag->size++] = pfn; } @@ -882,7 +875,7 @@ static unsigned long iova_rcache_get(struct iova_domain *iovad, { unsigned int log_size = order_base_2(size); - if (log_size >= IOVA_RANGE_CACHE_MAX_SIZE || !iovad->rcaches) + if (log_size >= IOVA_RANGE_CACHE_MAX_SIZE) return 0; return __iova_rcache_get(&iovad->rcaches[log_size], limit_pfn - size); diff --git a/drivers/iommu/ipmmu-vmsa.c b/drivers/iommu/ipmmu-vmsa.c index 1d42084d0276..3b30c0752274 100644 --- a/drivers/iommu/ipmmu-vmsa.c +++ b/drivers/iommu/ipmmu-vmsa.c @@ -1090,11 +1090,6 @@ static int ipmmu_probe(struct platform_device *pdev) ret = iommu_device_register(&mmu->iommu, &ipmmu_ops, &pdev->dev); if (ret) return ret; - -#if defined(CONFIG_IOMMU_DMA) - if (!iommu_present(&platform_bus_type)) - bus_set_iommu(&platform_bus_type, &ipmmu_ops); -#endif } /* @@ -1168,32 +1163,4 @@ static struct platform_driver ipmmu_driver = { .probe = ipmmu_probe, .remove = ipmmu_remove, }; - -static int __init ipmmu_init(void) -{ - struct device_node *np; - static bool setup_done; - int ret; - - if (setup_done) - return 0; - - np = of_find_matching_node(NULL, ipmmu_of_ids); - if (!np) - return 0; - - of_node_put(np); - - ret = platform_driver_register(&ipmmu_driver); - if (ret < 0) - return ret; - -#if defined(CONFIG_ARM) && !defined(CONFIG_IOMMU_DMA) - if (!iommu_present(&platform_bus_type)) - bus_set_iommu(&platform_bus_type, &ipmmu_ops); -#endif - - setup_done = true; - return 0; -} -subsys_initcall(ipmmu_init); +builtin_platform_driver(ipmmu_driver); diff --git a/drivers/iommu/msm_iommu.c b/drivers/iommu/msm_iommu.c index 6a24aa804ea3..16179a9a7283 100644 --- a/drivers/iommu/msm_iommu.c +++ b/drivers/iommu/msm_iommu.c @@ -792,8 +792,6 @@ static int msm_iommu_probe(struct platform_device *pdev) goto fail; } - bus_set_iommu(&platform_bus_type, &msm_iommu_ops); - pr_info("device mapped at %p, irq %d with %d ctx banks\n", iommu->base, iommu->irq, iommu->ncb); diff --git a/drivers/iommu/mtk_iommu.c b/drivers/iommu/mtk_iommu.c index 7e363b1f24df..552e4eb8c610 100644 --- a/drivers/iommu/mtk_iommu.c +++ b/drivers/iommu/mtk_iommu.c @@ -1243,30 +1243,13 @@ static int mtk_iommu_probe(struct platform_device *pdev) data->hw_list = &data->hw_list_head; } - if (!iommu_present(&platform_bus_type)) { - ret = bus_set_iommu(&platform_bus_type, &mtk_iommu_ops); - if (ret) - goto out_list_del; - } - if (MTK_IOMMU_IS_TYPE(data->plat_data, MTK_IOMMU_TYPE_MM)) { ret = component_master_add_with_match(dev, &mtk_iommu_com_ops, match); if (ret) - goto out_bus_set_null; - } else if (MTK_IOMMU_IS_TYPE(data->plat_data, MTK_IOMMU_TYPE_INFRA) && - MTK_IOMMU_HAS_FLAG(data->plat_data, IFA_IOMMU_PCIE_SUPPORT)) { -#ifdef CONFIG_PCI - if (!iommu_present(&pci_bus_type)) { - ret = bus_set_iommu(&pci_bus_type, &mtk_iommu_ops); - if (ret) /* PCIe fail don't affect platform_bus. */ - goto out_list_del; - } -#endif + goto out_list_del; } return ret; -out_bus_set_null: - bus_set_iommu(&platform_bus_type, NULL); out_list_del: list_del(&data->list); iommu_device_unregister(&data->iommu); @@ -1294,11 +1277,6 @@ static int mtk_iommu_remove(struct platform_device *pdev) if (MTK_IOMMU_IS_TYPE(data->plat_data, MTK_IOMMU_TYPE_MM)) { device_link_remove(data->smicomm_dev, &pdev->dev); component_master_del(&pdev->dev, &mtk_iommu_com_ops); - } else if (MTK_IOMMU_IS_TYPE(data->plat_data, MTK_IOMMU_TYPE_INFRA) && - MTK_IOMMU_HAS_FLAG(data->plat_data, IFA_IOMMU_PCIE_SUPPORT)) { -#ifdef CONFIG_PCI - bus_set_iommu(&pci_bus_type, NULL); -#endif } pm_runtime_disable(&pdev->dev); for (i = 0; i < data->plat_data->banks_num; i++) { diff --git a/drivers/iommu/mtk_iommu_v1.c b/drivers/iommu/mtk_iommu_v1.c index 128c7a3f1778..6e0e65831eb7 100644 --- a/drivers/iommu/mtk_iommu_v1.c +++ b/drivers/iommu/mtk_iommu_v1.c @@ -691,19 +691,11 @@ static int mtk_iommu_v1_probe(struct platform_device *pdev) if (ret) goto out_sysfs_remove; - if (!iommu_present(&platform_bus_type)) { - ret = bus_set_iommu(&platform_bus_type, &mtk_iommu_v1_ops); - if (ret) - goto out_dev_unreg; - } - ret = component_master_add_with_match(dev, &mtk_iommu_v1_com_ops, match); if (ret) - goto out_bus_set_null; + goto out_dev_unreg; return ret; -out_bus_set_null: - bus_set_iommu(&platform_bus_type, NULL); out_dev_unreg: iommu_device_unregister(&data->iommu); out_sysfs_remove: @@ -718,9 +710,6 @@ static int mtk_iommu_v1_remove(struct platform_device *pdev) iommu_device_sysfs_remove(&data->iommu); iommu_device_unregister(&data->iommu); - if (iommu_present(&platform_bus_type)) - bus_set_iommu(&platform_bus_type, NULL); - clk_disable_unprepare(data->bclk); devm_free_irq(&pdev->dev, data->irq, data); component_master_del(&pdev->dev, &mtk_iommu_v1_com_ops); diff --git a/drivers/iommu/omap-iommu.c b/drivers/iommu/omap-iommu.c index d9cf2820c02e..07ee2600113c 100644 --- a/drivers/iommu/omap-iommu.c +++ b/drivers/iommu/omap-iommu.c @@ -1776,14 +1776,8 @@ static int __init omap_iommu_init(void) goto fail_driver; } - ret = bus_set_iommu(&platform_bus_type, &omap_iommu_ops); - if (ret) - goto fail_bus; - return 0; -fail_bus: - platform_driver_unregister(&omap_iommu_driver); fail_driver: kmem_cache_destroy(iopte_cachep); return ret; diff --git a/drivers/iommu/rockchip-iommu.c b/drivers/iommu/rockchip-iommu.c index ab57c4b8fade..a3fc59b814ab 100644 --- a/drivers/iommu/rockchip-iommu.c +++ b/drivers/iommu/rockchip-iommu.c @@ -1300,8 +1300,6 @@ static int rk_iommu_probe(struct platform_device *pdev) if (!dma_dev) dma_dev = &pdev->dev; - bus_set_iommu(&platform_bus_type, &rk_iommu_ops); - pm_runtime_enable(dev); for (i = 0; i < iommu->num_irq; i++) { diff --git a/drivers/iommu/s390-iommu.c b/drivers/iommu/s390-iommu.c index c898bcbbce11..3c071782f6f1 100644 --- a/drivers/iommu/s390-iommu.c +++ b/drivers/iommu/s390-iommu.c @@ -39,7 +39,7 @@ static struct s390_domain *to_s390_domain(struct iommu_domain *dom) return container_of(dom, struct s390_domain, domain); } -static bool s390_iommu_capable(enum iommu_cap cap) +static bool s390_iommu_capable(struct device *dev, enum iommu_cap cap) { switch (cap) { case IOMMU_CAP_CACHE_COHERENCY: @@ -185,7 +185,12 @@ static void s390_iommu_detach_device(struct iommu_domain *domain, static struct iommu_device *s390_iommu_probe_device(struct device *dev) { - struct zpci_dev *zdev = to_zpci_dev(dev); + struct zpci_dev *zdev; + + if (!dev_is_pci(dev)) + return ERR_PTR(-ENODEV); + + zdev = to_zpci_dev(dev); return &zdev->iommu_dev; } @@ -385,9 +390,3 @@ static const struct iommu_ops s390_iommu_ops = { .free = s390_domain_free, } }; - -static int __init s390_iommu_init(void) -{ - return bus_set_iommu(&pci_bus_type, &s390_iommu_ops); -} -subsys_initcall(s390_iommu_init); diff --git a/drivers/iommu/sprd-iommu.c b/drivers/iommu/sprd-iommu.c index 511959c8a14d..fadd2c907222 100644 --- a/drivers/iommu/sprd-iommu.c +++ b/drivers/iommu/sprd-iommu.c @@ -496,9 +496,6 @@ static int sprd_iommu_probe(struct platform_device *pdev) if (ret) goto remove_sysfs; - if (!iommu_present(&platform_bus_type)) - bus_set_iommu(&platform_bus_type, &sprd_iommu_ops); - ret = sprd_iommu_clk_enable(sdev); if (ret) goto unregister_iommu; @@ -534,8 +531,6 @@ static int sprd_iommu_remove(struct platform_device *pdev) iommu_group_put(sdev->group); sdev->group = NULL; - bus_set_iommu(&platform_bus_type, NULL); - platform_set_drvdata(pdev, NULL); iommu_device_sysfs_remove(&sdev->iommu); iommu_device_unregister(&sdev->iommu); diff --git a/drivers/iommu/sun50i-iommu.c b/drivers/iommu/sun50i-iommu.c index a84c63518773..cd9b74ee24de 100644 --- a/drivers/iommu/sun50i-iommu.c +++ b/drivers/iommu/sun50i-iommu.c @@ -965,8 +965,6 @@ static int sun50i_iommu_probe(struct platform_device *pdev) if (ret < 0) goto err_unregister; - bus_set_iommu(&platform_bus_type, &sun50i_iommu_ops); - return 0; err_unregister: diff --git a/drivers/iommu/tegra-smmu.c b/drivers/iommu/tegra-smmu.c index 2a8de975fe63..5b1af40221ec 100644 --- a/drivers/iommu/tegra-smmu.c +++ b/drivers/iommu/tegra-smmu.c @@ -1083,8 +1083,8 @@ struct tegra_smmu *tegra_smmu_probe(struct device *dev, /* * This is a bit of a hack. Ideally we'd want to simply return this - * value. However the IOMMU registration process will attempt to add - * all devices to the IOMMU when bus_set_iommu() is called. In order + * value. However iommu_device_register() will attempt to add + * all devices to the IOMMU before we get that far. In order * not to rely on global variables to track the IOMMU instance, we * set it here so that it can be looked up from the .probe_device() * callback via the IOMMU device's .drvdata field. @@ -1138,32 +1138,15 @@ struct tegra_smmu *tegra_smmu_probe(struct device *dev, return ERR_PTR(err); err = iommu_device_register(&smmu->iommu, &tegra_smmu_ops, dev); - if (err) - goto remove_sysfs; - - err = bus_set_iommu(&platform_bus_type, &tegra_smmu_ops); - if (err < 0) - goto unregister; - -#ifdef CONFIG_PCI - err = bus_set_iommu(&pci_bus_type, &tegra_smmu_ops); - if (err < 0) - goto unset_platform_bus; -#endif + if (err) { + iommu_device_sysfs_remove(&smmu->iommu); + return ERR_PTR(err); + } if (IS_ENABLED(CONFIG_DEBUG_FS)) tegra_smmu_debugfs_init(smmu); return smmu; - -unset_platform_bus: __maybe_unused; - bus_set_iommu(&platform_bus_type, NULL); -unregister: - iommu_device_unregister(&smmu->iommu); -remove_sysfs: - iommu_device_sysfs_remove(&smmu->iommu); - - return ERR_PTR(err); } void tegra_smmu_remove(struct tegra_smmu *smmu) diff --git a/drivers/iommu/virtio-iommu.c b/drivers/iommu/virtio-iommu.c index 80151176ba12..da463db9f12a 100644 --- a/drivers/iommu/virtio-iommu.c +++ b/drivers/iommu/virtio-iommu.c @@ -7,9 +7,7 @@ #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt -#include <linux/amba/bus.h> #include <linux/delay.h> -#include <linux/dma-iommu.h> #include <linux/dma-map-ops.h> #include <linux/freezer.h> #include <linux/interval_tree.h> @@ -17,7 +15,6 @@ #include <linux/module.h> #include <linux/of_platform.h> #include <linux/pci.h> -#include <linux/platform_device.h> #include <linux/virtio.h> #include <linux/virtio_config.h> #include <linux/virtio_ids.h> @@ -25,6 +22,8 @@ #include <uapi/linux/virtio_iommu.h> +#include "dma-iommu.h" + #define MSI_IOVA_BASE 0x8000000 #define MSI_IOVA_LENGTH 0x100000 @@ -1156,26 +1155,6 @@ static int viommu_probe(struct virtio_device *vdev) iommu_device_register(&viommu->iommu, &viommu_ops, parent_dev); -#ifdef CONFIG_PCI - if (pci_bus_type.iommu_ops != &viommu_ops) { - ret = bus_set_iommu(&pci_bus_type, &viommu_ops); - if (ret) - goto err_unregister; - } -#endif -#ifdef CONFIG_ARM_AMBA - if (amba_bustype.iommu_ops != &viommu_ops) { - ret = bus_set_iommu(&amba_bustype, &viommu_ops); - if (ret) - goto err_unregister; - } -#endif - if (platform_bus_type.iommu_ops != &viommu_ops) { - ret = bus_set_iommu(&platform_bus_type, &viommu_ops); - if (ret) - goto err_unregister; - } - vdev->priv = viommu; dev_info(dev, "input address: %u bits\n", @@ -1184,9 +1163,6 @@ static int viommu_probe(struct virtio_device *vdev) return 0; -err_unregister: - iommu_device_sysfs_remove(&viommu->iommu); - iommu_device_unregister(&viommu->iommu); err_free_vqs: vdev->config->del_vqs(vdev); diff --git a/drivers/irqchip/irq-gic-v2m.c b/drivers/irqchip/irq-gic-v2m.c index b249d4df899e..6e1ac330d7a6 100644 --- a/drivers/irqchip/irq-gic-v2m.c +++ b/drivers/irqchip/irq-gic-v2m.c @@ -13,7 +13,7 @@ #define pr_fmt(fmt) "GICv2m: " fmt #include <linux/acpi.h> -#include <linux/dma-iommu.h> +#include <linux/iommu.h> #include <linux/irq.h> #include <linux/irqdomain.h> #include <linux/kernel.h> diff --git a/drivers/irqchip/irq-gic-v3-its.c b/drivers/irqchip/irq-gic-v3-its.c index 5ff09de6c48f..e7d8d4208ee6 100644 --- a/drivers/irqchip/irq-gic-v3-its.c +++ b/drivers/irqchip/irq-gic-v3-its.c @@ -11,9 +11,9 @@ #include <linux/cpu.h> #include <linux/crash_dump.h> #include <linux/delay.h> -#include <linux/dma-iommu.h> #include <linux/efi.h> #include <linux/interrupt.h> +#include <linux/iommu.h> #include <linux/iopoll.h> #include <linux/irqdomain.h> #include <linux/list.h> diff --git a/drivers/irqchip/irq-gic-v3-mbi.c b/drivers/irqchip/irq-gic-v3-mbi.c index a2163d32f17d..e1efdec9e9ac 100644 --- a/drivers/irqchip/irq-gic-v3-mbi.c +++ b/drivers/irqchip/irq-gic-v3-mbi.c @@ -6,7 +6,7 @@ #define pr_fmt(fmt) "GICv3: " fmt -#include <linux/dma-iommu.h> +#include <linux/iommu.h> #include <linux/irq.h> #include <linux/irqdomain.h> #include <linux/kernel.h> diff --git a/drivers/irqchip/irq-ls-scfg-msi.c b/drivers/irqchip/irq-ls-scfg-msi.c index b4927e425f7b..527c90e0920e 100644 --- a/drivers/irqchip/irq-ls-scfg-msi.c +++ b/drivers/irqchip/irq-ls-scfg-msi.c @@ -11,6 +11,7 @@ #include <linux/module.h> #include <linux/msi.h> #include <linux/interrupt.h> +#include <linux/iommu.h> #include <linux/irq.h> #include <linux/irqchip/chained_irq.h> #include <linux/irqdomain.h> @@ -18,7 +19,6 @@ #include <linux/of_pci.h> #include <linux/of_platform.h> #include <linux/spinlock.h> -#include <linux/dma-iommu.h> #define MSI_IRQS_PER_MSIR 32 #define MSI_MSIR_OFFSET 4 diff --git a/drivers/vfio/vfio_iommu_type1.c b/drivers/vfio/vfio_iommu_type1.c index db516c90a977..2f41e32c640c 100644 --- a/drivers/vfio/vfio_iommu_type1.c +++ b/drivers/vfio/vfio_iommu_type1.c @@ -37,7 +37,6 @@ #include <linux/vfio.h> #include <linux/workqueue.h> #include <linux/notifier.h> -#include <linux/dma-iommu.h> #include <linux/irqdomain.h> #include "vfio.h" diff --git a/include/linux/dma-iommu.h b/include/linux/dma-iommu.h deleted file mode 100644 index 24607dc3c2ac..000000000000 --- a/include/linux/dma-iommu.h +++ /dev/null @@ -1,93 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0-only */ -/* - * Copyright (C) 2014-2015 ARM Ltd. - */ -#ifndef __DMA_IOMMU_H -#define __DMA_IOMMU_H - -#include <linux/errno.h> -#include <linux/types.h> - -#ifdef CONFIG_IOMMU_DMA -#include <linux/dma-mapping.h> -#include <linux/iommu.h> -#include <linux/msi.h> - -/* Domain management interface for IOMMU drivers */ -int iommu_get_dma_cookie(struct iommu_domain *domain); -int iommu_get_msi_cookie(struct iommu_domain *domain, dma_addr_t base); -void iommu_put_dma_cookie(struct iommu_domain *domain); - -/* Setup call for arch DMA mapping code */ -void iommu_setup_dma_ops(struct device *dev, u64 dma_base, u64 dma_limit); -int iommu_dma_init_fq(struct iommu_domain *domain); - -/* The DMA API isn't _quite_ the whole story, though... */ -/* - * iommu_dma_prepare_msi() - Map the MSI page in the IOMMU device - * - * The MSI page will be stored in @desc. - * - * Return: 0 on success otherwise an error describing the failure. - */ -int iommu_dma_prepare_msi(struct msi_desc *desc, phys_addr_t msi_addr); - -/* Update the MSI message if required. */ -void iommu_dma_compose_msi_msg(struct msi_desc *desc, - struct msi_msg *msg); - -void iommu_dma_get_resv_regions(struct device *dev, struct list_head *list); - -void iommu_dma_free_cpu_cached_iovas(unsigned int cpu, - struct iommu_domain *domain); - -extern bool iommu_dma_forcedac; - -#else /* CONFIG_IOMMU_DMA */ - -struct iommu_domain; -struct msi_desc; -struct msi_msg; -struct device; - -static inline void iommu_setup_dma_ops(struct device *dev, u64 dma_base, - u64 dma_limit) -{ -} - -static inline int iommu_dma_init_fq(struct iommu_domain *domain) -{ - return -EINVAL; -} - -static inline int iommu_get_dma_cookie(struct iommu_domain *domain) -{ - return -ENODEV; -} - -static inline int iommu_get_msi_cookie(struct iommu_domain *domain, dma_addr_t base) -{ - return -ENODEV; -} - -static inline void iommu_put_dma_cookie(struct iommu_domain *domain) -{ -} - -static inline int iommu_dma_prepare_msi(struct msi_desc *desc, - phys_addr_t msi_addr) -{ - return 0; -} - -static inline void iommu_dma_compose_msi_msg(struct msi_desc *desc, - struct msi_msg *msg) -{ -} - -static inline void iommu_dma_get_resv_regions(struct device *dev, struct list_head *list) -{ -} - -#endif /* CONFIG_IOMMU_DMA */ -#endif /* __DMA_IOMMU_H */ diff --git a/include/linux/iommu.h b/include/linux/iommu.h index ea30f00dc145..a325532aeab5 100644 --- a/include/linux/iommu.h +++ b/include/linux/iommu.h @@ -212,7 +212,7 @@ struct iommu_iotlb_gather { * @of_xlate: add OF master IDs to iommu grouping * @is_attach_deferred: Check if domain attach should be deferred from iommu * driver init to device driver init (default no) - * @dev_has/enable/disable_feat: per device entries to check/enable/disable + * @dev_enable/disable_feat: per device entries to enable/disable * iommu specific features. * @sva_bind: Bind process address space to device * @sva_unbind: Unbind process address space from device @@ -227,7 +227,7 @@ struct iommu_iotlb_gather { * @owner: Driver module providing these ops */ struct iommu_ops { - bool (*capable)(enum iommu_cap); + bool (*capable)(struct device *dev, enum iommu_cap); /* Domain allocation and freeing by the iommu driver */ struct iommu_domain *(*domain_alloc)(unsigned iommu_domain_type); @@ -416,11 +416,9 @@ static inline const struct iommu_ops *dev_iommu_ops(struct device *dev) return dev->iommu->iommu_dev->ops; } -extern int bus_set_iommu(struct bus_type *bus, const struct iommu_ops *ops); extern int bus_iommu_probe(struct bus_type *bus); extern bool iommu_present(struct bus_type *bus); extern bool device_iommu_capable(struct device *dev, enum iommu_cap cap); -extern bool iommu_capable(struct bus_type *bus, enum iommu_cap cap); extern struct iommu_domain *iommu_domain_alloc(struct bus_type *bus); extern struct iommu_group *iommu_group_get_by_id(int id); extern void iommu_domain_free(struct iommu_domain *domain); @@ -697,11 +695,6 @@ static inline bool device_iommu_capable(struct device *dev, enum iommu_cap cap) return false; } -static inline bool iommu_capable(struct bus_type *bus, enum iommu_cap cap) -{ - return false; -} - static inline struct iommu_domain *iommu_domain_alloc(struct bus_type *bus) { return NULL; @@ -1070,4 +1063,40 @@ void iommu_debugfs_setup(void); static inline void iommu_debugfs_setup(void) {} #endif +#ifdef CONFIG_IOMMU_DMA +#include <linux/msi.h> + +/* Setup call for arch DMA mapping code */ +void iommu_setup_dma_ops(struct device *dev, u64 dma_base, u64 dma_limit); + +int iommu_get_msi_cookie(struct iommu_domain *domain, dma_addr_t base); + +int iommu_dma_prepare_msi(struct msi_desc *desc, phys_addr_t msi_addr); +void iommu_dma_compose_msi_msg(struct msi_desc *desc, struct msi_msg *msg); + +#else /* CONFIG_IOMMU_DMA */ + +struct msi_desc; +struct msi_msg; + +static inline void iommu_setup_dma_ops(struct device *dev, u64 dma_base, u64 dma_limit) +{ +} + +static inline int iommu_get_msi_cookie(struct iommu_domain *domain, dma_addr_t base) +{ + return -ENODEV; +} + +static inline int iommu_dma_prepare_msi(struct msi_desc *desc, phys_addr_t msi_addr) +{ + return 0; +} + +static inline void iommu_dma_compose_msi_msg(struct msi_desc *desc, struct msi_msg *msg) +{ +} + +#endif /* CONFIG_IOMMU_DMA */ + #endif /* __LINUX_IOMMU_H */ |