diff options
Diffstat (limited to 'include/linux/dma-mapping.h')
| -rw-r--r-- | include/linux/dma-mapping.h | 418 |
1 files changed, 229 insertions, 189 deletions
diff --git a/include/linux/dma-mapping.h b/include/linux/dma-mapping.h index ba521d5506c9..75e60be91e5f 100644 --- a/include/linux/dma-mapping.h +++ b/include/linux/dma-mapping.h @@ -130,6 +130,7 @@ struct dma_map_ops { enum dma_data_direction direction); int (*dma_supported)(struct device *dev, u64 mask); u64 (*get_required_mask)(struct device *dev); + size_t (*max_mapping_size)(struct device *dev); }; #define DMA_MAPPING_ERROR (~(dma_addr_t)0) @@ -153,7 +154,7 @@ static inline int is_device_dma_capable(struct device *dev) return dev->dma_mask != NULL && *dev->dma_mask != DMA_MASK_NONE; } -#ifdef CONFIG_HAVE_GENERIC_DMA_COHERENT +#ifdef CONFIG_DMA_DECLARE_COHERENT /* * These three functions are only for dma allocator. * Don't use them in device drivers. @@ -192,34 +193,7 @@ static inline int dma_mmap_from_global_coherent(struct vm_area_struct *vma, { return 0; } -#endif /* CONFIG_HAVE_GENERIC_DMA_COHERENT */ - -#ifdef CONFIG_HAS_DMA -#include <asm/dma-mapping.h> -static inline const struct dma_map_ops *get_dma_ops(struct device *dev) -{ - if (dev && dev->dma_ops) - return dev->dma_ops; - return get_arch_dma_ops(dev ? dev->bus : NULL); -} - -static inline void set_dma_ops(struct device *dev, - const struct dma_map_ops *dma_ops) -{ - dev->dma_ops = dma_ops; -} -#else -/* - * Define the dma api to allow compilation of dma dependent code. - * Code that depends on the dma-mapping API needs to set 'depends on HAS_DMA' - * in its Kconfig, unless it already depends on <something> || COMPILE_TEST, - * where <something> guarantuees the availability of the dma-mapping API. - */ -static inline const struct dma_map_ops *get_dma_ops(struct device *dev) -{ - return NULL; -} -#endif +#endif /* CONFIG_DMA_DECLARE_COHERENT */ static inline bool dma_is_direct(const struct dma_map_ops *ops) { @@ -235,6 +209,8 @@ dma_addr_t dma_direct_map_page(struct device *dev, struct page *page, unsigned long attrs); int dma_direct_map_sg(struct device *dev, struct scatterlist *sgl, int nents, enum dma_data_direction dir, unsigned long attrs); +dma_addr_t dma_direct_map_resource(struct device *dev, phys_addr_t paddr, + size_t size, enum dma_data_direction dir, unsigned long attrs); #if defined(CONFIG_ARCH_HAS_SYNC_DMA_FOR_DEVICE) || \ defined(CONFIG_SWIOTLB) @@ -284,32 +260,43 @@ static inline void dma_direct_sync_sg_for_cpu(struct device *dev, } #endif -static inline dma_addr_t dma_map_single_attrs(struct device *dev, void *ptr, - size_t size, - enum dma_data_direction dir, - unsigned long attrs) +size_t dma_direct_max_mapping_size(struct device *dev); + +#ifdef CONFIG_HAS_DMA +#include <asm/dma-mapping.h> + +static inline const struct dma_map_ops *get_dma_ops(struct device *dev) +{ + if (dev && dev->dma_ops) + return dev->dma_ops; + return get_arch_dma_ops(dev ? dev->bus : NULL); +} + +static inline void set_dma_ops(struct device *dev, + const struct dma_map_ops *dma_ops) +{ + dev->dma_ops = dma_ops; +} + +static inline dma_addr_t dma_map_page_attrs(struct device *dev, + struct page *page, size_t offset, size_t size, + enum dma_data_direction dir, unsigned long attrs) { const struct dma_map_ops *ops = get_dma_ops(dev); dma_addr_t addr; BUG_ON(!valid_dma_direction(dir)); - debug_dma_map_single(dev, ptr, size); if (dma_is_direct(ops)) - addr = dma_direct_map_page(dev, virt_to_page(ptr), - offset_in_page(ptr), size, dir, attrs); + addr = dma_direct_map_page(dev, page, offset, size, dir, attrs); else - addr = ops->map_page(dev, virt_to_page(ptr), - offset_in_page(ptr), size, dir, attrs); - debug_dma_map_page(dev, virt_to_page(ptr), - offset_in_page(ptr), size, - dir, addr, true); + addr = ops->map_page(dev, page, offset, size, dir, attrs); + debug_dma_map_page(dev, page, offset, size, dir, addr); + return addr; } -static inline void dma_unmap_single_attrs(struct device *dev, dma_addr_t addr, - size_t size, - enum dma_data_direction dir, - unsigned long attrs) +static inline void dma_unmap_page_attrs(struct device *dev, dma_addr_t addr, + size_t size, enum dma_data_direction dir, unsigned long attrs) { const struct dma_map_ops *ops = get_dma_ops(dev); @@ -318,13 +305,7 @@ static inline void dma_unmap_single_attrs(struct device *dev, dma_addr_t addr, dma_direct_unmap_page(dev, addr, size, dir, attrs); else if (ops->unmap_page) ops->unmap_page(dev, addr, size, dir, attrs); - debug_dma_unmap_page(dev, addr, size, dir, true); -} - -static inline void dma_unmap_page_attrs(struct device *dev, dma_addr_t addr, - size_t size, enum dma_data_direction dir, unsigned long attrs) -{ - return dma_unmap_single_attrs(dev, addr, size, dir, attrs); + debug_dma_unmap_page(dev, addr, size, dir); } /* @@ -363,25 +344,6 @@ static inline void dma_unmap_sg_attrs(struct device *dev, struct scatterlist *sg ops->unmap_sg(dev, sg, nents, dir, attrs); } -static inline dma_addr_t dma_map_page_attrs(struct device *dev, - struct page *page, - size_t offset, size_t size, - enum dma_data_direction dir, - unsigned long attrs) -{ - const struct dma_map_ops *ops = get_dma_ops(dev); - dma_addr_t addr; - - BUG_ON(!valid_dma_direction(dir)); - if (dma_is_direct(ops)) - addr = dma_direct_map_page(dev, page, offset, size, dir, attrs); - else - addr = ops->map_page(dev, page, offset, size, dir, attrs); - debug_dma_map_page(dev, page, offset, size, dir, addr, false); - - return addr; -} - static inline dma_addr_t dma_map_resource(struct device *dev, phys_addr_t phys_addr, size_t size, @@ -389,19 +351,20 @@ static inline dma_addr_t dma_map_resource(struct device *dev, unsigned long attrs) { const struct dma_map_ops *ops = get_dma_ops(dev); - dma_addr_t addr; + dma_addr_t addr = DMA_MAPPING_ERROR; BUG_ON(!valid_dma_direction(dir)); /* Don't allow RAM to be mapped */ - BUG_ON(pfn_valid(PHYS_PFN(phys_addr))); + if (WARN_ON_ONCE(pfn_valid(PHYS_PFN(phys_addr)))) + return DMA_MAPPING_ERROR; - addr = phys_addr; - if (ops && ops->map_resource) + if (dma_is_direct(ops)) + addr = dma_direct_map_resource(dev, phys_addr, size, dir, attrs); + else if (ops->map_resource) addr = ops->map_resource(dev, phys_addr, size, dir, attrs); debug_dma_map_resource(dev, phys_addr, size, dir, addr); - return addr; } @@ -412,7 +375,7 @@ static inline void dma_unmap_resource(struct device *dev, dma_addr_t addr, const struct dma_map_ops *ops = get_dma_ops(dev); BUG_ON(!valid_dma_direction(dir)); - if (ops && ops->unmap_resource) + if (!dma_is_direct(ops) && ops->unmap_resource) ops->unmap_resource(dev, addr, size, dir, attrs); debug_dma_unmap_resource(dev, addr, size, dir); } @@ -431,13 +394,6 @@ static inline void dma_sync_single_for_cpu(struct device *dev, dma_addr_t addr, debug_dma_sync_single_for_cpu(dev, addr, size, dir); } -static inline void dma_sync_single_range_for_cpu(struct device *dev, - dma_addr_t addr, unsigned long offset, size_t size, - enum dma_data_direction dir) -{ - return dma_sync_single_for_cpu(dev, addr + offset, size, dir); -} - static inline void dma_sync_single_for_device(struct device *dev, dma_addr_t addr, size_t size, enum dma_data_direction dir) @@ -452,13 +408,6 @@ static inline void dma_sync_single_for_device(struct device *dev, debug_dma_sync_single_for_device(dev, addr, size, dir); } -static inline void dma_sync_single_range_for_device(struct device *dev, - dma_addr_t addr, unsigned long offset, size_t size, - enum dma_data_direction dir) -{ - return dma_sync_single_for_device(dev, addr + offset, size, dir); -} - static inline void dma_sync_sg_for_cpu(struct device *dev, struct scatterlist *sg, int nelems, enum dma_data_direction dir) @@ -488,15 +437,179 @@ dma_sync_sg_for_device(struct device *dev, struct scatterlist *sg, } +static inline int dma_mapping_error(struct device *dev, dma_addr_t dma_addr) +{ + debug_dma_mapping_error(dev, dma_addr); + + if (dma_addr == DMA_MAPPING_ERROR) + return -ENOMEM; + return 0; +} + +void *dma_alloc_attrs(struct device *dev, size_t size, dma_addr_t *dma_handle, + gfp_t flag, unsigned long attrs); +void dma_free_attrs(struct device *dev, size_t size, void *cpu_addr, + dma_addr_t dma_handle, unsigned long attrs); +void *dmam_alloc_attrs(struct device *dev, size_t size, dma_addr_t *dma_handle, + gfp_t gfp, unsigned long attrs); +void dmam_free_coherent(struct device *dev, size_t size, void *vaddr, + dma_addr_t dma_handle); +void dma_cache_sync(struct device *dev, void *vaddr, size_t size, + enum dma_data_direction dir); +int dma_get_sgtable_attrs(struct device *dev, struct sg_table *sgt, + void *cpu_addr, dma_addr_t dma_addr, size_t size, + unsigned long attrs); +int dma_mmap_attrs(struct device *dev, struct vm_area_struct *vma, + void *cpu_addr, dma_addr_t dma_addr, size_t size, + unsigned long attrs); +int dma_supported(struct device *dev, u64 mask); +int dma_set_mask(struct device *dev, u64 mask); +int dma_set_coherent_mask(struct device *dev, u64 mask); +u64 dma_get_required_mask(struct device *dev); +size_t dma_max_mapping_size(struct device *dev); +#else /* CONFIG_HAS_DMA */ +static inline dma_addr_t dma_map_page_attrs(struct device *dev, + struct page *page, size_t offset, size_t size, + enum dma_data_direction dir, unsigned long attrs) +{ + return DMA_MAPPING_ERROR; +} +static inline void dma_unmap_page_attrs(struct device *dev, dma_addr_t addr, + size_t size, enum dma_data_direction dir, unsigned long attrs) +{ +} +static inline int dma_map_sg_attrs(struct device *dev, struct scatterlist *sg, + int nents, enum dma_data_direction dir, unsigned long attrs) +{ + return 0; +} +static inline void dma_unmap_sg_attrs(struct device *dev, + struct scatterlist *sg, int nents, enum dma_data_direction dir, + unsigned long attrs) +{ +} +static inline dma_addr_t dma_map_resource(struct device *dev, + phys_addr_t phys_addr, size_t size, enum dma_data_direction dir, + unsigned long attrs) +{ + return DMA_MAPPING_ERROR; +} +static inline void dma_unmap_resource(struct device *dev, dma_addr_t addr, + size_t size, enum dma_data_direction dir, unsigned long attrs) +{ +} +static inline void dma_sync_single_for_cpu(struct device *dev, dma_addr_t addr, + size_t size, enum dma_data_direction dir) +{ +} +static inline void dma_sync_single_for_device(struct device *dev, + dma_addr_t addr, size_t size, enum dma_data_direction dir) +{ +} +static inline void dma_sync_sg_for_cpu(struct device *dev, + struct scatterlist *sg, int nelems, enum dma_data_direction dir) +{ +} +static inline void dma_sync_sg_for_device(struct device *dev, + struct scatterlist *sg, int nelems, enum dma_data_direction dir) +{ +} +static inline int dma_mapping_error(struct device *dev, dma_addr_t dma_addr) +{ + return -ENOMEM; +} +static inline void *dma_alloc_attrs(struct device *dev, size_t size, + dma_addr_t *dma_handle, gfp_t flag, unsigned long attrs) +{ + return NULL; +} +static void dma_free_attrs(struct device *dev, size_t size, void *cpu_addr, + dma_addr_t dma_handle, unsigned long attrs) +{ +} +static inline void *dmam_alloc_attrs(struct device *dev, size_t size, + dma_addr_t *dma_handle, gfp_t gfp, unsigned long attrs) +{ + return NULL; +} +static inline void dmam_free_coherent(struct device *dev, size_t size, + void *vaddr, dma_addr_t dma_handle) +{ +} +static inline void dma_cache_sync(struct device *dev, void *vaddr, size_t size, + enum dma_data_direction dir) +{ +} +static inline int dma_get_sgtable_attrs(struct device *dev, + struct sg_table *sgt, void *cpu_addr, dma_addr_t dma_addr, + size_t size, unsigned long attrs) +{ + return -ENXIO; +} +static inline int dma_mmap_attrs(struct device *dev, struct vm_area_struct *vma, + void *cpu_addr, dma_addr_t dma_addr, size_t size, + unsigned long attrs) +{ + return -ENXIO; +} +static inline int dma_supported(struct device *dev, u64 mask) +{ + return 0; +} +static inline int dma_set_mask(struct device *dev, u64 mask) +{ + return -EIO; +} +static inline int dma_set_coherent_mask(struct device *dev, u64 mask) +{ + return -EIO; +} +static inline u64 dma_get_required_mask(struct device *dev) +{ + return 0; +} +static inline size_t dma_max_mapping_size(struct device *dev) +{ + return 0; +} +#endif /* CONFIG_HAS_DMA */ + +static inline dma_addr_t dma_map_single_attrs(struct device *dev, void *ptr, + size_t size, enum dma_data_direction dir, unsigned long attrs) +{ + debug_dma_map_single(dev, ptr, size); + return dma_map_page_attrs(dev, virt_to_page(ptr), offset_in_page(ptr), + size, dir, attrs); +} + +static inline void dma_unmap_single_attrs(struct device *dev, dma_addr_t addr, + size_t size, enum dma_data_direction dir, unsigned long attrs) +{ + return dma_unmap_page_attrs(dev, addr, size, dir, attrs); +} + +static inline void dma_sync_single_range_for_cpu(struct device *dev, + dma_addr_t addr, unsigned long offset, size_t size, + enum dma_data_direction dir) +{ + return dma_sync_single_for_cpu(dev, addr + offset, size, dir); +} + +static inline void dma_sync_single_range_for_device(struct device *dev, + dma_addr_t addr, unsigned long offset, size_t size, + enum dma_data_direction dir) +{ + return dma_sync_single_for_device(dev, addr + offset, size, dir); +} + #define dma_map_single(d, a, s, r) dma_map_single_attrs(d, a, s, r, 0) #define dma_unmap_single(d, a, s, r) dma_unmap_single_attrs(d, a, s, r, 0) #define dma_map_sg(d, s, n, r) dma_map_sg_attrs(d, s, n, r, 0) #define dma_unmap_sg(d, s, n, r) dma_unmap_sg_attrs(d, s, n, r, 0) #define dma_map_page(d, p, o, s, r) dma_map_page_attrs(d, p, o, s, r, 0) #define dma_unmap_page(d, a, s, r) dma_unmap_page_attrs(d, a, s, r, 0) - -void dma_cache_sync(struct device *dev, void *vaddr, size_t size, - enum dma_data_direction dir); +#define dma_get_sgtable(d, t, v, h, s) dma_get_sgtable_attrs(d, t, v, h, s, 0) +#define dma_mmap_coherent(d, v, c, h, s) dma_mmap_attrs(d, v, c, h, s, 0) extern int dma_common_mmap(struct device *dev, struct vm_area_struct *vma, void *cpu_addr, dma_addr_t dma_addr, size_t size, @@ -516,25 +629,10 @@ bool dma_in_atomic_pool(void *start, size_t size); void *dma_alloc_from_pool(size_t size, struct page **ret_page, gfp_t flags); bool dma_free_from_pool(void *start, size_t size); -int dma_mmap_attrs(struct device *dev, struct vm_area_struct *vma, - void *cpu_addr, dma_addr_t dma_addr, size_t size, - unsigned long attrs); -#define dma_mmap_coherent(d, v, c, h, s) dma_mmap_attrs(d, v, c, h, s, 0) - int dma_common_get_sgtable(struct device *dev, struct sg_table *sgt, void *cpu_addr, dma_addr_t dma_addr, size_t size, unsigned long attrs); -int dma_get_sgtable_attrs(struct device *dev, struct sg_table *sgt, - void *cpu_addr, dma_addr_t dma_addr, size_t size, - unsigned long attrs); -#define dma_get_sgtable(d, t, v, h, s) dma_get_sgtable_attrs(d, t, v, h, s, 0) - -void *dma_alloc_attrs(struct device *dev, size_t size, dma_addr_t *dma_handle, - gfp_t flag, unsigned long attrs); -void dma_free_attrs(struct device *dev, size_t size, void *cpu_addr, - dma_addr_t dma_handle, unsigned long attrs); - static inline void *dma_alloc_coherent(struct device *dev, size_t size, dma_addr_t *dma_handle, gfp_t gfp) { @@ -549,18 +647,6 @@ static inline void dma_free_coherent(struct device *dev, size_t size, return dma_free_attrs(dev, size, cpu_addr, dma_handle, 0); } -static inline int dma_mapping_error(struct device *dev, dma_addr_t dma_addr) -{ - debug_dma_mapping_error(dev, dma_addr); - - if (dma_addr == DMA_MAPPING_ERROR) - return -ENOMEM; - return 0; -} - -int dma_supported(struct device *dev, u64 mask); -int dma_set_mask(struct device *dev, u64 mask); -int dma_set_coherent_mask(struct device *dev, u64 mask); static inline u64 dma_get_mask(struct device *dev) { @@ -593,17 +679,23 @@ static inline int dma_coerce_mask_and_coherent(struct device *dev, u64 mask) return dma_set_mask_and_coherent(dev, mask); } -extern u64 dma_get_required_mask(struct device *dev); - -#ifndef arch_setup_dma_ops +#ifdef CONFIG_ARCH_HAS_SETUP_DMA_OPS +void arch_setup_dma_ops(struct device *dev, u64 dma_base, u64 size, + const struct iommu_ops *iommu, bool coherent); +#else static inline void arch_setup_dma_ops(struct device *dev, u64 dma_base, - u64 size, const struct iommu_ops *iommu, - bool coherent) { } -#endif + u64 size, const struct iommu_ops *iommu, bool coherent) +{ +} +#endif /* CONFIG_ARCH_HAS_SETUP_DMA_OPS */ -#ifndef arch_teardown_dma_ops -static inline void arch_teardown_dma_ops(struct device *dev) { } -#endif +#ifdef CONFIG_ARCH_HAS_TEARDOWN_DMA_OPS +void arch_teardown_dma_ops(struct device *dev); +#else +static inline void arch_teardown_dma_ops(struct device *dev) +{ +} +#endif /* CONFIG_ARCH_HAS_TEARDOWN_DMA_OPS */ static inline unsigned int dma_get_max_seg_size(struct device *dev) { @@ -644,15 +736,6 @@ static inline unsigned long dma_max_pfn(struct device *dev) } #endif -/* - * Please always use dma_alloc_coherent instead as it already zeroes the memory! - */ -static inline void *dma_zalloc_coherent(struct device *dev, size_t size, - dma_addr_t *dma_handle, gfp_t flag) -{ - return dma_alloc_coherent(dev, size, dma_handle, flag); -} - static inline int dma_get_cache_alignment(void) { #ifdef ARCH_DMA_MINALIGN @@ -661,19 +744,14 @@ static inline int dma_get_cache_alignment(void) return 1; } -/* flags for the coherent memory api */ -#define DMA_MEMORY_EXCLUSIVE 0x01 - -#ifdef CONFIG_HAVE_GENERIC_DMA_COHERENT +#ifdef CONFIG_DMA_DECLARE_COHERENT int dma_declare_coherent_memory(struct device *dev, phys_addr_t phys_addr, - dma_addr_t device_addr, size_t size, int flags); + dma_addr_t device_addr, size_t size); void dma_release_declared_memory(struct device *dev); -void *dma_mark_declared_memory_occupied(struct device *dev, - dma_addr_t device_addr, size_t size); #else static inline int dma_declare_coherent_memory(struct device *dev, phys_addr_t phys_addr, - dma_addr_t device_addr, size_t size, int flags) + dma_addr_t device_addr, size_t size) { return -ENOSYS; } @@ -682,52 +760,14 @@ static inline void dma_release_declared_memory(struct device *dev) { } +#endif /* CONFIG_DMA_DECLARE_COHERENT */ -static inline void * -dma_mark_declared_memory_occupied(struct device *dev, - dma_addr_t device_addr, size_t size) -{ - return ERR_PTR(-EBUSY); -} -#endif /* CONFIG_HAVE_GENERIC_DMA_COHERENT */ - -/* - * Managed DMA API - */ -#ifdef CONFIG_HAS_DMA -extern void *dmam_alloc_coherent(struct device *dev, size_t size, - dma_addr_t *dma_handle, gfp_t gfp); -extern void dmam_free_coherent(struct device *dev, size_t size, void *vaddr, - dma_addr_t dma_handle); -#else /* !CONFIG_HAS_DMA */ static inline void *dmam_alloc_coherent(struct device *dev, size_t size, - dma_addr_t *dma_handle, gfp_t gfp) -{ return NULL; } -static inline void dmam_free_coherent(struct device *dev, size_t size, - void *vaddr, dma_addr_t dma_handle) { } -#endif /* !CONFIG_HAS_DMA */ - -extern void *dmam_alloc_attrs(struct device *dev, size_t size, - dma_addr_t *dma_handle, gfp_t gfp, - unsigned long attrs); -#ifdef CONFIG_HAVE_GENERIC_DMA_COHERENT -extern int dmam_declare_coherent_memory(struct device *dev, - phys_addr_t phys_addr, - dma_addr_t device_addr, size_t size, - int flags); -extern void dmam_release_declared_memory(struct device *dev); -#else /* CONFIG_HAVE_GENERIC_DMA_COHERENT */ -static inline int dmam_declare_coherent_memory(struct device *dev, - phys_addr_t phys_addr, dma_addr_t device_addr, - size_t size, gfp_t gfp) -{ - return 0; -} - -static inline void dmam_release_declared_memory(struct device *dev) + dma_addr_t *dma_handle, gfp_t gfp) { + return dmam_alloc_attrs(dev, size, dma_handle, gfp, + (gfp & __GFP_NOWARN) ? DMA_ATTR_NO_WARN : 0); } -#endif /* CONFIG_HAVE_GENERIC_DMA_COHERENT */ static inline void *dma_alloc_wc(struct device *dev, size_t size, dma_addr_t *dma_addr, gfp_t gfp) |