From 4561b366c70e312698361154bdfe1e04f596056b Mon Sep 17 00:00:00 2001 From: Christian König Date: Thu, 1 Oct 2020 16:17:55 +0200 Subject: drm/ttm: drop glob parameter from ttm_bo_swapout MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit We can always access the global state. Signed-off-by: Christian König Acked-by: Daniel Vetter Link: https://patchwork.freedesktop.org/patch/393499/ --- drivers/gpu/drm/ttm/ttm_memory.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'drivers/gpu/drm/ttm/ttm_memory.c') diff --git a/drivers/gpu/drm/ttm/ttm_memory.c b/drivers/gpu/drm/ttm/ttm_memory.c index 89d50f38c0f2..69cf622e79e5 100644 --- a/drivers/gpu/drm/ttm/ttm_memory.c +++ b/drivers/gpu/drm/ttm/ttm_memory.c @@ -275,7 +275,7 @@ static void ttm_shrink(struct ttm_mem_global *glob, bool from_wq, while (ttm_zones_above_swap_target(glob, from_wq, extra)) { spin_unlock(&glob->lock); - ret = ttm_bo_swapout(&ttm_bo_glob, ctx); + ret = ttm_bo_swapout(ctx); spin_lock(&glob->lock); if (unlikely(ret != 0)) break; -- cgit From d099fc8f540add80f725014fdd4f7f49f3c58911 Mon Sep 17 00:00:00 2001 From: Christian König Date: Thu, 22 Oct 2020 18:26:58 +0200 Subject: drm/ttm: new TT backend allocation pool v3 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This replaces the spaghetti code in the two existing page pools. First of all depending on the allocation size it is between 3 (1GiB) and 5 (1MiB) times faster than the old implementation. It makes better use of buddy pages to allow for larger physical contiguous allocations which should result in better TLB utilization at least for amdgpu. Instead of a completely braindead approach of filling the pool with one CPU while another one is trying to shrink it we only give back freed pages. This also results in much less locking contention and a trylock free MM shrinker callback, so we can guarantee that pages are given back to the system when needed. Downside of this is that it takes longer for many small allocations until the pool is filled up. We could address this, but I couldn't find an use case where this actually matters. We also don't bother freeing large chunks of pages any more since the CPU overhead in that path isn't really that important. The sysfs files are replaced with a single module parameter, allowing users to override how many pages should be globally pooled in TTM. This unfortunately breaks the UAPI slightly, but as far as we know nobody ever depended on this. Zeroing memory coming from the pool was handled inconsistently. The alloc_pages() based pool was zeroing it, the dma_alloc_attr() based one wasn't. For now the new implementation isn't zeroing pages from the pool either and only sets the __GFP_ZERO flag when necessary. The implementation has only 768 lines of code compared to the over 2600 of the old one, and also allows for saving quite a bunch of code in the drivers since we don't need specialized handling there any more based on kernel config. Additional to all of that there was a neat bug with IOMMU, coherent DMA mappings and huge pages which is now fixed in the new code as well. v2: make ttm_pool_apply_caching static as reported by the kernel bot, add some more checks v3: fix some more checkpatch.pl warnings Signed-off-by: Christian König Reviewed-by: Dave Airlie Reviewed-by: Madhav Chauhan Tested-by: Huang Rui Link: https://patchwork.freedesktop.org/patch/397080/?series=83051&rev=1 --- drivers/gpu/drm/ttm/Makefile | 2 +- drivers/gpu/drm/ttm/ttm_memory.c | 3 + drivers/gpu/drm/ttm/ttm_pool.c | 667 +++++++++++++++++++++++++++++++++++++++ include/drm/ttm/ttm_caching.h | 2 + include/drm/ttm/ttm_pool.h | 91 ++++++ 5 files changed, 764 insertions(+), 1 deletion(-) create mode 100644 drivers/gpu/drm/ttm/ttm_pool.c create mode 100644 include/drm/ttm/ttm_pool.h (limited to 'drivers/gpu/drm/ttm/ttm_memory.c') diff --git a/drivers/gpu/drm/ttm/Makefile b/drivers/gpu/drm/ttm/Makefile index 90c0da88cc98..0096bacbcf32 100644 --- a/drivers/gpu/drm/ttm/Makefile +++ b/drivers/gpu/drm/ttm/Makefile @@ -5,7 +5,7 @@ ttm-y := ttm_memory.o ttm_tt.o ttm_bo.o \ ttm_bo_util.o ttm_bo_vm.o ttm_module.o \ ttm_execbuf_util.o ttm_page_alloc.o ttm_range_manager.o \ - ttm_resource.o + ttm_resource.o ttm_pool.o ttm-$(CONFIG_AGP) += ttm_agp_backend.o ttm-$(CONFIG_DRM_TTM_DMA_PAGE_POOL) += ttm_page_alloc_dma.o diff --git a/drivers/gpu/drm/ttm/ttm_memory.c b/drivers/gpu/drm/ttm/ttm_memory.c index 69cf622e79e5..3012d0388c51 100644 --- a/drivers/gpu/drm/ttm/ttm_memory.c +++ b/drivers/gpu/drm/ttm/ttm_memory.c @@ -38,6 +38,7 @@ #include #include #include +#include #define TTM_MEMORY_ALLOC_RETRIES 4 @@ -453,6 +454,7 @@ int ttm_mem_global_init(struct ttm_mem_global *glob) } ttm_page_alloc_init(glob, glob->zone_kernel->max_mem/(2*PAGE_SIZE)); ttm_dma_page_alloc_init(glob, glob->zone_kernel->max_mem/(2*PAGE_SIZE)); + ttm_pool_mgr_init(glob->zone_kernel->max_mem/(2*PAGE_SIZE)); return 0; out_no_zone: ttm_mem_global_release(glob); @@ -467,6 +469,7 @@ void ttm_mem_global_release(struct ttm_mem_global *glob) /* let the page allocator first stop the shrink work. */ ttm_page_alloc_fini(); ttm_dma_page_alloc_fini(); + ttm_pool_mgr_fini(); flush_workqueue(glob->swap_queue); destroy_workqueue(glob->swap_queue); diff --git a/drivers/gpu/drm/ttm/ttm_pool.c b/drivers/gpu/drm/ttm/ttm_pool.c new file mode 100644 index 000000000000..1e50deefb5d5 --- /dev/null +++ b/drivers/gpu/drm/ttm/ttm_pool.c @@ -0,0 +1,667 @@ +// SPDX-License-Identifier: GPL-2.0 OR MIT +/* + * Copyright 2020 Advanced Micro Devices, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + * Authors: Christian König + */ + +/* Pooling of allocated pages is necessary because changing the caching + * attributes on x86 of the linear mapping requires a costly cross CPU TLB + * invalidate for those addresses. + * + * Additional to that allocations from the DMA coherent API are pooled as well + * cause they are rather slow compared to alloc_pages+map. + */ + +#include +#include + +#ifdef CONFIG_X86 +#include +#endif + +#include +#include +#include + +/** + * struct ttm_pool_dma - Helper object for coherent DMA mappings + * + * @addr: original DMA address returned for the mapping + * @vaddr: original vaddr return for the mapping and order in the lower bits + */ +struct ttm_pool_dma { + dma_addr_t addr; + unsigned long vaddr; +}; + +static unsigned long page_pool_size; + +MODULE_PARM_DESC(page_pool_size, "Number of pages in the WC/UC/DMA pool"); +module_param(page_pool_size, ulong, 0644); + +static atomic_long_t allocated_pages; + +static struct ttm_pool_type global_write_combined[MAX_ORDER]; +static struct ttm_pool_type global_uncached[MAX_ORDER]; + +static spinlock_t shrinker_lock; +static struct list_head shrinker_list; +static struct shrinker mm_shrinker; + +/* Allocate pages of size 1 << order with the given gfp_flags */ +static struct page *ttm_pool_alloc_page(struct ttm_pool *pool, gfp_t gfp_flags, + unsigned int order) +{ + unsigned long attr = DMA_ATTR_FORCE_CONTIGUOUS; + struct ttm_pool_dma *dma; + struct page *p; + void *vaddr; + + if (order) { + gfp_flags |= GFP_TRANSHUGE_LIGHT | __GFP_NORETRY | + __GFP_KSWAPD_RECLAIM; + gfp_flags &= ~__GFP_MOVABLE; + gfp_flags &= ~__GFP_COMP; + } + + if (!pool->use_dma_alloc) { + p = alloc_pages(gfp_flags, order); + if (p) + p->private = order; + return p; + } + + dma = kmalloc(sizeof(*dma), GFP_KERNEL); + if (!dma) + return NULL; + + if (order) + attr |= DMA_ATTR_NO_WARN; + + vaddr = dma_alloc_attrs(pool->dev, (1ULL << order) * PAGE_SIZE, + &dma->addr, gfp_flags, attr); + if (!vaddr) + goto error_free; + + /* TODO: This is an illegal abuse of the DMA API, but we need to rework + * TTM page fault handling and extend the DMA API to clean this up. + */ + if (is_vmalloc_addr(vaddr)) + p = vmalloc_to_page(vaddr); + else + p = virt_to_page(vaddr); + + dma->vaddr = (unsigned long)vaddr | order; + p->private = (unsigned long)dma; + return p; + +error_free: + kfree(dma); + return NULL; +} + +/* Reset the caching and pages of size 1 << order */ +static void ttm_pool_free_page(struct ttm_pool *pool, enum ttm_caching caching, + unsigned int order, struct page *p) +{ + unsigned long attr = DMA_ATTR_FORCE_CONTIGUOUS; + struct ttm_pool_dma *dma; + void *vaddr; + +#ifdef CONFIG_X86 + /* We don't care that set_pages_wb is inefficient here. This is only + * used when we have to shrink and CPU overhead is irrelevant then. + */ + if (caching != ttm_cached && !PageHighMem(p)) + set_pages_wb(p, 1 << order); +#endif + + if (!pool->use_dma_alloc) { + __free_pages(p, order); + return; + } + + if (order) + attr |= DMA_ATTR_NO_WARN; + + dma = (void *)p->private; + vaddr = (void *)(dma->vaddr & PAGE_MASK); + dma_free_attrs(pool->dev, (1UL << order) * PAGE_SIZE, vaddr, dma->addr, + attr); + kfree(dma); +} + +/* Apply a new caching to an array of pages */ +static int ttm_pool_apply_caching(struct page **first, struct page **last, + enum ttm_caching caching) +{ +#ifdef CONFIG_X86 + unsigned int num_pages = last - first; + + if (!num_pages) + return 0; + + switch (caching) { + case ttm_cached: + break; + case ttm_write_combined: + return set_pages_array_wc(first, num_pages); + case ttm_uncached: + return set_pages_array_uc(first, num_pages); + } +#endif + return 0; +} + +/* Map pages of 1 << order size and fill the DMA address array */ +static int ttm_pool_map(struct ttm_pool *pool, unsigned int order, + struct page *p, dma_addr_t **dma_addr) +{ + dma_addr_t addr; + unsigned int i; + + if (pool->use_dma_alloc) { + struct ttm_pool_dma *dma = (void *)p->private; + + addr = dma->addr; + } else { + size_t size = (1ULL << order) * PAGE_SIZE; + + addr = dma_map_page(pool->dev, p, 0, size, DMA_BIDIRECTIONAL); + if (dma_mapping_error(pool->dev, **dma_addr)) + return -EFAULT; + } + + for (i = 1 << order; i ; --i) { + *(*dma_addr)++ = addr; + addr += PAGE_SIZE; + } + + return 0; +} + +/* Unmap pages of 1 << order size */ +static void ttm_pool_unmap(struct ttm_pool *pool, dma_addr_t dma_addr, + unsigned int num_pages) +{ + /* Unmapped while freeing the page */ + if (pool->use_dma_alloc) + return; + + dma_unmap_page(pool->dev, dma_addr, (long)num_pages << PAGE_SHIFT, + DMA_BIDIRECTIONAL); +} + +/* Give pages into a specific pool_type */ +static void ttm_pool_type_give(struct ttm_pool_type *pt, struct page *p) +{ + spin_lock(&pt->lock); + list_add(&p->lru, &pt->pages); + spin_unlock(&pt->lock); + atomic_long_add(1 << pt->order, &allocated_pages); +} + +/* Take pages from a specific pool_type, return NULL when nothing available */ +static struct page *ttm_pool_type_take(struct ttm_pool_type *pt) +{ + struct page *p; + + spin_lock(&pt->lock); + p = list_first_entry_or_null(&pt->pages, typeof(*p), lru); + if (p) { + atomic_long_sub(1 << pt->order, &allocated_pages); + list_del(&p->lru); + } + spin_unlock(&pt->lock); + + return p; +} + +/* Count the number of pages available in a pool_type */ +static unsigned int ttm_pool_type_count(struct ttm_pool_type *pt) +{ + unsigned int count = 0; + struct page *p; + + spin_lock(&pt->lock); + /* Only used for debugfs, the overhead doesn't matter */ + list_for_each_entry(p, &pt->pages, lru) + ++count; + spin_unlock(&pt->lock); + + return count; +} + +/* Initialize and add a pool type to the global shrinker list */ +static void ttm_pool_type_init(struct ttm_pool_type *pt, struct ttm_pool *pool, + enum ttm_caching caching, unsigned int order) +{ + pt->pool = pool; + pt->caching = caching; + pt->order = order; + spin_lock_init(&pt->lock); + INIT_LIST_HEAD(&pt->pages); + + spin_lock(&shrinker_lock); + list_add_tail(&pt->shrinker_list, &shrinker_list); + spin_unlock(&shrinker_lock); +} + +/* Remove a pool_type from the global shrinker list and free all pages */ +static void ttm_pool_type_fini(struct ttm_pool_type *pt) +{ + struct page *p, *tmp; + + spin_lock(&shrinker_lock); + list_del(&pt->shrinker_list); + spin_unlock(&shrinker_lock); + + list_for_each_entry_safe(p, tmp, &pt->pages, lru) + ttm_pool_free_page(pt->pool, pt->caching, pt->order, p); +} + +/* Return the pool_type to use for the given caching and order */ +static struct ttm_pool_type *ttm_pool_select_type(struct ttm_pool *pool, + enum ttm_caching caching, + unsigned int order) +{ + if (pool->use_dma_alloc) + return &pool->caching[caching].orders[order]; + +#ifdef CONFIG_X86 + switch (caching) { + case ttm_write_combined: + return &global_write_combined[order]; + case ttm_uncached: + return &global_uncached[order]; + default: + break; + } +#endif + + return NULL; +} + +/* Free pages using the global shrinker list */ +static unsigned int ttm_pool_shrink(void) +{ + struct ttm_pool_type *pt; + unsigned int num_freed; + struct page *p; + + spin_lock(&shrinker_lock); + pt = list_first_entry(&shrinker_list, typeof(*pt), shrinker_list); + + p = ttm_pool_type_take(pt); + if (p) { + ttm_pool_free_page(pt->pool, pt->caching, pt->order, p); + num_freed = 1 << pt->order; + } else { + num_freed = 0; + } + + list_move_tail(&pt->shrinker_list, &shrinker_list); + spin_unlock(&shrinker_lock); + + return num_freed; +} + +/* Return the allocation order based for a page */ +static unsigned int ttm_pool_page_order(struct ttm_pool *pool, struct page *p) +{ + if (pool->use_dma_alloc) { + struct ttm_pool_dma *dma = (void *)p->private; + + return dma->vaddr & ~PAGE_MASK; + } + + return p->private; +} + +/** + * ttm_pool_alloc - Fill a ttm_tt object + * + * @pool: ttm_pool to use + * @tt: ttm_tt object to fill + * @ctx: operation context + * + * Fill the ttm_tt object with pages and also make sure to DMA map them when + * necessary. + * + * Returns: 0 on successe, negative error code otherwise. + */ +int ttm_pool_alloc(struct ttm_pool *pool, struct ttm_tt *tt, + struct ttm_operation_ctx *ctx) +{ + unsigned long num_pages = tt->num_pages; + dma_addr_t *dma_addr = tt->dma_address; + struct page **caching = tt->pages; + struct page **pages = tt->pages; + gfp_t gfp_flags = GFP_USER; + unsigned int i, order; + struct page *p; + int r; + + WARN_ON(!num_pages || ttm_tt_is_populated(tt)); + WARN_ON(dma_addr && !pool->dev); + + if (tt->page_flags & TTM_PAGE_FLAG_ZERO_ALLOC) + gfp_flags |= __GFP_ZERO; + + if (tt->page_flags & TTM_PAGE_FLAG_NO_RETRY) + gfp_flags |= __GFP_RETRY_MAYFAIL; + + if (pool->use_dma32) + gfp_flags |= GFP_DMA32; + else + gfp_flags |= GFP_HIGHUSER; + + for (order = min(MAX_ORDER - 1UL, __fls(num_pages)); num_pages; + order = min_t(unsigned int, order, __fls(num_pages))) { + bool apply_caching = false; + struct ttm_pool_type *pt; + + pt = ttm_pool_select_type(pool, tt->caching, order); + p = pt ? ttm_pool_type_take(pt) : NULL; + if (p) { + apply_caching = true; + } else { + p = ttm_pool_alloc_page(pool, gfp_flags, order); + if (p && PageHighMem(p)) + apply_caching = true; + } + + if (!p) { + if (order) { + --order; + continue; + } + r = -ENOMEM; + goto error_free_all; + } + + if (apply_caching) { + r = ttm_pool_apply_caching(caching, pages, + tt->caching); + if (r) + goto error_free_page; + caching = pages + (1 << order); + } + + r = ttm_mem_global_alloc_page(&ttm_mem_glob, p, + (1 << order) * PAGE_SIZE, + ctx); + if (r) + goto error_free_page; + + if (dma_addr) { + r = ttm_pool_map(pool, order, p, &dma_addr); + if (r) + goto error_global_free; + } + + num_pages -= 1 << order; + for (i = 1 << order; i; --i) + *(pages++) = p++; + } + + r = ttm_pool_apply_caching(caching, pages, tt->caching); + if (r) + goto error_free_all; + + return 0; + +error_global_free: + ttm_mem_global_free_page(&ttm_mem_glob, p, (1 << order) * PAGE_SIZE); + +error_free_page: + ttm_pool_free_page(pool, tt->caching, order, p); + +error_free_all: + num_pages = tt->num_pages - num_pages; + for (i = 0; i < num_pages; ) { + order = ttm_pool_page_order(pool, tt->pages[i]); + ttm_pool_free_page(pool, tt->caching, order, tt->pages[i]); + i += 1 << order; + } + + return r; +} +EXPORT_SYMBOL(ttm_pool_alloc); + +/** + * ttm_pool_free - Free the backing pages from a ttm_tt object + * + * @pool: Pool to give pages back to. + * @tt: ttm_tt object to unpopulate + * + * Give the packing pages back to a pool or free them + */ +void ttm_pool_free(struct ttm_pool *pool, struct ttm_tt *tt) +{ + unsigned int i; + + for (i = 0; i < tt->num_pages; ) { + struct page *p = tt->pages[i]; + unsigned int order, num_pages; + struct ttm_pool_type *pt; + + order = ttm_pool_page_order(pool, p); + num_pages = 1ULL << order; + ttm_mem_global_free_page(&ttm_mem_glob, p, + num_pages * PAGE_SIZE); + if (tt->dma_address) + ttm_pool_unmap(pool, tt->dma_address[i], num_pages); + + pt = ttm_pool_select_type(pool, tt->caching, order); + if (pt) + ttm_pool_type_give(pt, tt->pages[i]); + else + ttm_pool_free_page(pool, tt->caching, order, + tt->pages[i]); + + i += num_pages; + } + + while (atomic_long_read(&allocated_pages) > page_pool_size) + ttm_pool_shrink(); +} +EXPORT_SYMBOL(ttm_pool_free); + +/** + * ttm_pool_init - Initialize a pool + * + * @pool: the pool to initialize + * @dev: device for DMA allocations and mappings + * @use_dma_alloc: true if coherent DMA alloc should be used + * @use_dma32: true if GFP_DMA32 should be used + * + * Initialize the pool and its pool types. + */ +void ttm_pool_init(struct ttm_pool *pool, struct device *dev, + bool use_dma_alloc, bool use_dma32) +{ + unsigned int i, j; + + WARN_ON(!dev && use_dma_alloc); + + pool->dev = dev; + pool->use_dma_alloc = use_dma_alloc; + pool->use_dma32 = use_dma32; + + for (i = 0; i < TTM_NUM_CACHING_TYPES; ++i) + for (j = 0; j < MAX_ORDER; ++j) + ttm_pool_type_init(&pool->caching[i].orders[j], + pool, i, j); +} +EXPORT_SYMBOL(ttm_pool_init); + +/** + * ttm_pool_fini - Cleanup a pool + * + * @pool: the pool to clean up + * + * Free all pages in the pool and unregister the types from the global + * shrinker. + */ +void ttm_pool_fini(struct ttm_pool *pool) +{ + unsigned int i, j; + + for (i = 0; i < TTM_NUM_CACHING_TYPES; ++i) + for (j = 0; j < MAX_ORDER; ++j) + ttm_pool_type_fini(&pool->caching[i].orders[j]); +} +EXPORT_SYMBOL(ttm_pool_fini); + +#ifdef CONFIG_DEBUG_FS + +/* Dump information about the different pool types */ +static void ttm_pool_debugfs_orders(struct ttm_pool_type *pt, + struct seq_file *m) +{ + unsigned int i; + + for (i = 0; i < MAX_ORDER; ++i) + seq_printf(m, " %8u", ttm_pool_type_count(&pt[i])); + seq_puts(m, "\n"); +} + +/** + * ttm_pool_debugfs - Debugfs dump function for a pool + * + * @pool: the pool to dump the information for + * @m: seq_file to dump to + * + * Make a debugfs dump with the per pool and global information. + */ +int ttm_pool_debugfs(struct ttm_pool *pool, struct seq_file *m) +{ + unsigned int i; + + spin_lock(&shrinker_lock); + + seq_puts(m, "\t "); + for (i = 0; i < MAX_ORDER; ++i) + seq_printf(m, " ---%2u---", i); + seq_puts(m, "\n"); + + seq_puts(m, "wc\t:"); + ttm_pool_debugfs_orders(global_write_combined, m); + seq_puts(m, "uc\t:"); + ttm_pool_debugfs_orders(global_uncached, m); + + for (i = 0; i < TTM_NUM_CACHING_TYPES; ++i) { + seq_puts(m, "DMA "); + switch (i) { + case ttm_cached: + seq_puts(m, "\t:"); + break; + case ttm_write_combined: + seq_puts(m, "wc\t:"); + break; + case ttm_uncached: + seq_puts(m, "uc\t:"); + break; + } + ttm_pool_debugfs_orders(pool->caching[i].orders, m); + } + + seq_printf(m, "\ntotal\t: %8lu of %8lu\n", + atomic_long_read(&allocated_pages), page_pool_size); + + spin_unlock(&shrinker_lock); + + return 0; +} +EXPORT_SYMBOL(ttm_pool_debugfs); + +#endif + +/* As long as pages are available make sure to release at least one */ +static unsigned long ttm_pool_shrinker_scan(struct shrinker *shrink, + struct shrink_control *sc) +{ + unsigned long num_freed = 0; + + do + num_freed += ttm_pool_shrink(); + while (!num_freed && atomic_long_read(&allocated_pages)); + + return num_freed; +} + +/* Return the number of pages available or SHRINK_EMPTY if we have none */ +static unsigned long ttm_pool_shrinker_count(struct shrinker *shrink, + struct shrink_control *sc) +{ + unsigned long num_pages = atomic_long_read(&allocated_pages); + + return num_pages ? num_pages : SHRINK_EMPTY; +} + +/** + * ttm_pool_mgr_init - Initialize globals + * + * @num_pages: default number of pages + * + * Initialize the global locks and lists for the MM shrinker. + */ +int ttm_pool_mgr_init(unsigned long num_pages) +{ + unsigned int i; + + if (!page_pool_size) + page_pool_size = num_pages; + + spin_lock_init(&shrinker_lock); + INIT_LIST_HEAD(&shrinker_list); + + for (i = 0; i < MAX_ORDER; ++i) { + ttm_pool_type_init(&global_write_combined[i], NULL, + ttm_write_combined, i); + ttm_pool_type_init(&global_uncached[i], NULL, ttm_uncached, i); + } + + mm_shrinker.count_objects = ttm_pool_shrinker_count; + mm_shrinker.scan_objects = ttm_pool_shrinker_scan; + mm_shrinker.seeks = 1; + return register_shrinker(&mm_shrinker); +} + +/** + * ttm_pool_mgr_fini - Finalize globals + * + * Cleanup the global pools and unregister the MM shrinker. + */ +void ttm_pool_mgr_fini(void) +{ + unsigned int i; + + for (i = 0; i < MAX_ORDER; ++i) { + ttm_pool_type_fini(&global_write_combined[i]); + ttm_pool_type_fini(&global_uncached[i]); + } + + unregister_shrinker(&mm_shrinker); + WARN_ON(!list_empty(&shrinker_list)); +} diff --git a/include/drm/ttm/ttm_caching.h b/include/drm/ttm/ttm_caching.h index 161624dcf6be..a0b4a49fa432 100644 --- a/include/drm/ttm/ttm_caching.h +++ b/include/drm/ttm/ttm_caching.h @@ -25,6 +25,8 @@ #ifndef _TTM_CACHING_H_ #define _TTM_CACHING_H_ +#define TTM_NUM_CACHING_TYPES 3 + enum ttm_caching { ttm_uncached, ttm_write_combined, diff --git a/include/drm/ttm/ttm_pool.h b/include/drm/ttm/ttm_pool.h new file mode 100644 index 000000000000..4321728bdd11 --- /dev/null +++ b/include/drm/ttm/ttm_pool.h @@ -0,0 +1,91 @@ +/* SPDX-License-Identifier: GPL-2.0 OR MIT */ +/* + * Copyright 2020 Advanced Micro Devices, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + * Authors: Christian König + */ + +#ifndef _TTM_PAGE_POOL_H_ +#define _TTM_PAGE_POOL_H_ + +#include +#include +#include +#include + +struct device; +struct ttm_tt; +struct ttm_pool; +struct ttm_operation_ctx; + +/** + * ttm_pool_type - Pool for a certain memory type + * + * @pool: the pool we belong to, might be NULL for the global ones + * @order: the allocation order our pages have + * @caching: the caching type our pages have + * @shrinker_list: our place on the global shrinker list + * @lock: protection of the page list + * @pages: the list of pages in the pool + */ +struct ttm_pool_type { + struct ttm_pool *pool; + unsigned int order; + enum ttm_caching caching; + + struct list_head shrinker_list; + + spinlock_t lock; + struct list_head pages; +}; + +/** + * ttm_pool - Pool for all caching and orders + * + * @use_dma_alloc: if coherent DMA allocations should be used + * @use_dma32: if GFP_DMA32 should be used + * @caching: pools for each caching/order + */ +struct ttm_pool { + struct device *dev; + + bool use_dma_alloc; + bool use_dma32; + + struct { + struct ttm_pool_type orders[MAX_ORDER]; + } caching[TTM_NUM_CACHING_TYPES]; +}; + +int ttm_pool_alloc(struct ttm_pool *pool, struct ttm_tt *tt, + struct ttm_operation_ctx *ctx); +void ttm_pool_free(struct ttm_pool *pool, struct ttm_tt *tt); + +void ttm_pool_init(struct ttm_pool *pool, struct device *dev, + bool use_dma_alloc, bool use_dma32); +void ttm_pool_fini(struct ttm_pool *pool); + +int ttm_pool_debugfs(struct ttm_pool *pool, struct seq_file *m); + +int ttm_pool_mgr_init(unsigned long num_pages); +void ttm_pool_mgr_fini(void); + +#endif -- cgit From ee5d2a8e549e90325fcc31825269f89647cd6fac Mon Sep 17 00:00:00 2001 From: Christian König Date: Sat, 24 Oct 2020 13:10:28 +0200 Subject: drm/ttm: wire up the new pool as default one v2 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Provide the necessary parameters by all drivers and use the new pool alloc when no driver specific function is provided. v2: fix the GEM VRAM helpers Signed-off-by: Christian König Reviewed-by: Dave Airlie Reviewed-by: Madhav Chauhan Tested-by: Huang Rui Link: https://patchwork.freedesktop.org/patch/397081/?series=83051&rev=1 --- drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c | 4 ++-- drivers/gpu/drm/drm_gem_vram_helper.c | 4 ++-- drivers/gpu/drm/nouveau/nouveau_ttm.c | 14 +++++++++----- drivers/gpu/drm/qxl/qxl_ttm.c | 5 ++--- drivers/gpu/drm/radeon/radeon_ttm.c | 4 ++-- drivers/gpu/drm/ttm/ttm_bo.c | 8 ++++++-- drivers/gpu/drm/ttm/ttm_memory.c | 2 +- drivers/gpu/drm/ttm/ttm_tt.c | 5 ++--- drivers/gpu/drm/vmwgfx/vmwgfx_drv.c | 5 +++-- include/drm/ttm/ttm_bo_driver.h | 11 +++++++---- 10 files changed, 36 insertions(+), 26 deletions(-) (limited to 'drivers/gpu/drm/ttm/ttm_memory.c') diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c index beacd00221d8..34944927838e 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c @@ -1914,10 +1914,10 @@ int amdgpu_ttm_init(struct amdgpu_device *adev) mutex_init(&adev->mman.gtt_window_lock); /* No others user of address space so set it to 0 */ - r = ttm_bo_device_init(&adev->mman.bdev, - &amdgpu_bo_driver, + r = ttm_bo_device_init(&adev->mman.bdev, &amdgpu_bo_driver, adev->dev, adev_to_drm(adev)->anon_inode->i_mapping, adev_to_drm(adev)->vma_offset_manager, + adev->need_swiotlb, dma_addressing_limited(adev->dev)); if (r) { DRM_ERROR("failed initializing buffer object driver(%d).\n", r); diff --git a/drivers/gpu/drm/drm_gem_vram_helper.c b/drivers/gpu/drm/drm_gem_vram_helper.c index 9da823eb0edd..683762b2f9a8 100644 --- a/drivers/gpu/drm/drm_gem_vram_helper.c +++ b/drivers/gpu/drm/drm_gem_vram_helper.c @@ -1045,10 +1045,10 @@ static int drm_vram_mm_init(struct drm_vram_mm *vmm, struct drm_device *dev, vmm->vram_base = vram_base; vmm->vram_size = vram_size; - ret = ttm_bo_device_init(&vmm->bdev, &bo_driver, + ret = ttm_bo_device_init(&vmm->bdev, &bo_driver, dev->dev, dev->anon_inode->i_mapping, dev->vma_offset_manager, - true); + false, true); if (ret) return ret; diff --git a/drivers/gpu/drm/nouveau/nouveau_ttm.c b/drivers/gpu/drm/nouveau/nouveau_ttm.c index 0592ed6eaad1..d696d882c9eb 100644 --- a/drivers/gpu/drm/nouveau/nouveau_ttm.c +++ b/drivers/gpu/drm/nouveau/nouveau_ttm.c @@ -281,6 +281,7 @@ nouveau_ttm_init(struct nouveau_drm *drm) struct nvkm_pci *pci = device->pci; struct nvif_mmu *mmu = &drm->client.mmu; struct drm_device *dev = drm->dev; + bool need_swiotlb = false; int typei, ret; ret = nouveau_ttm_init_host(drm, 0); @@ -315,11 +316,14 @@ nouveau_ttm_init(struct nouveau_drm *drm) drm->agp.cma = pci->agp.cma; } - ret = ttm_bo_device_init(&drm->ttm.bdev, - &nouveau_bo_driver, - dev->anon_inode->i_mapping, - dev->vma_offset_manager, - drm->client.mmu.dmabits <= 32 ? true : false); +#if IS_ENABLED(CONFIG_SWIOTLB) && IS_ENABLED(CONFIG_X86) + need_swiotlb = !!swiotlb_nr_tbl(); +#endif + + ret = ttm_bo_device_init(&drm->ttm.bdev, &nouveau_bo_driver, + drm->dev->dev, dev->anon_inode->i_mapping, + dev->vma_offset_manager, need_swiotlb, + drm->client.mmu.dmabits <= 32); if (ret) { NV_ERROR(drm, "error initialising bo driver, %d\n", ret); return ret; diff --git a/drivers/gpu/drm/qxl/qxl_ttm.c b/drivers/gpu/drm/qxl/qxl_ttm.c index 9609eeb52821..d8ecfb8b3193 100644 --- a/drivers/gpu/drm/qxl/qxl_ttm.c +++ b/drivers/gpu/drm/qxl/qxl_ttm.c @@ -194,11 +194,10 @@ int qxl_ttm_init(struct qxl_device *qdev) int num_io_pages; /* != rom->num_io_pages, we include surface0 */ /* No others user of address space so set it to 0 */ - r = ttm_bo_device_init(&qdev->mman.bdev, - &qxl_bo_driver, + r = ttm_bo_device_init(&qdev->mman.bdev, &qxl_bo_driver, NULL, qdev->ddev.anon_inode->i_mapping, qdev->ddev.vma_offset_manager, - false); + false, false); if (r) { DRM_ERROR("failed initializing buffer object driver(%d).\n", r); return r; diff --git a/drivers/gpu/drm/radeon/radeon_ttm.c b/drivers/gpu/drm/radeon/radeon_ttm.c index 0a6d7ea847db..1add3918519c 100644 --- a/drivers/gpu/drm/radeon/radeon_ttm.c +++ b/drivers/gpu/drm/radeon/radeon_ttm.c @@ -846,10 +846,10 @@ int radeon_ttm_init(struct radeon_device *rdev) int r; /* No others user of address space so set it to 0 */ - r = ttm_bo_device_init(&rdev->mman.bdev, - &radeon_bo_driver, + r = ttm_bo_device_init(&rdev->mman.bdev, &radeon_bo_driver, rdev->dev, rdev->ddev->anon_inode->i_mapping, rdev->ddev->vma_offset_manager, + rdev->need_swiotlb, dma_addressing_limited(&rdev->pdev->dev)); if (r) { DRM_ERROR("failed initializing buffer object driver(%d).\n", r); diff --git a/drivers/gpu/drm/ttm/ttm_bo.c b/drivers/gpu/drm/ttm/ttm_bo.c index 40c72a0f9325..e9f91cfce5e9 100644 --- a/drivers/gpu/drm/ttm/ttm_bo.c +++ b/drivers/gpu/drm/ttm/ttm_bo.c @@ -1283,6 +1283,8 @@ int ttm_bo_device_release(struct ttm_bo_device *bdev) pr_debug("Swap list %d was clean\n", i); spin_unlock(&glob->lru_lock); + ttm_pool_fini(&bdev->pool); + if (!ret) ttm_bo_global_release(); @@ -1307,9 +1309,10 @@ static void ttm_bo_init_sysman(struct ttm_bo_device *bdev) int ttm_bo_device_init(struct ttm_bo_device *bdev, struct ttm_bo_driver *driver, + struct device *dev, struct address_space *mapping, struct drm_vma_offset_manager *vma_manager, - bool need_dma32) + bool use_dma_alloc, bool use_dma32) { struct ttm_bo_global *glob = &ttm_bo_glob; int ret; @@ -1324,12 +1327,13 @@ int ttm_bo_device_init(struct ttm_bo_device *bdev, bdev->driver = driver; ttm_bo_init_sysman(bdev); + ttm_pool_init(&bdev->pool, dev, use_dma_alloc, use_dma32); bdev->vma_manager = vma_manager; INIT_DELAYED_WORK(&bdev->wq, ttm_bo_delayed_workqueue); INIT_LIST_HEAD(&bdev->ddestroy); bdev->dev_mapping = mapping; - bdev->need_dma32 = need_dma32; + bdev->need_dma32 = use_dma32; mutex_lock(&ttm_global_mutex); list_add_tail(&bdev->device_list, &glob->device_list); mutex_unlock(&ttm_global_mutex); diff --git a/drivers/gpu/drm/ttm/ttm_memory.c b/drivers/gpu/drm/ttm/ttm_memory.c index 3012d0388c51..b15a91c90271 100644 --- a/drivers/gpu/drm/ttm/ttm_memory.c +++ b/drivers/gpu/drm/ttm/ttm_memory.c @@ -454,7 +454,7 @@ int ttm_mem_global_init(struct ttm_mem_global *glob) } ttm_page_alloc_init(glob, glob->zone_kernel->max_mem/(2*PAGE_SIZE)); ttm_dma_page_alloc_init(glob, glob->zone_kernel->max_mem/(2*PAGE_SIZE)); - ttm_pool_mgr_init(glob->zone_kernel->max_mem/(2*PAGE_SIZE)); + ttm_pool_mgr_init(glob->zone_kernel->max_mem / (2 * PAGE_SIZE)); return 0; out_no_zone: ttm_mem_global_release(glob); diff --git a/drivers/gpu/drm/ttm/ttm_tt.c b/drivers/gpu/drm/ttm/ttm_tt.c index 65c4254eea5c..90054280cd8f 100644 --- a/drivers/gpu/drm/ttm/ttm_tt.c +++ b/drivers/gpu/drm/ttm/ttm_tt.c @@ -37,7 +37,6 @@ #include #include #include -#include /** * Allocates a ttm structure for the given BO. @@ -321,7 +320,7 @@ int ttm_tt_populate(struct ttm_bo_device *bdev, if (bdev->driver->ttm_tt_populate) ret = bdev->driver->ttm_tt_populate(bdev, ttm, ctx); else - ret = ttm_pool_populate(ttm, ctx); + ret = ttm_pool_alloc(&bdev->pool, ttm, ctx); if (ret) return ret; @@ -363,6 +362,6 @@ void ttm_tt_unpopulate(struct ttm_bo_device *bdev, if (bdev->driver->ttm_tt_unpopulate) bdev->driver->ttm_tt_unpopulate(bdev, ttm); else - ttm_pool_unpopulate(ttm); + ttm_pool_free(&bdev->pool, ttm); ttm->page_flags &= ~TTM_PAGE_FLAG_PRIV_POPULATED; } diff --git a/drivers/gpu/drm/vmwgfx/vmwgfx_drv.c b/drivers/gpu/drm/vmwgfx/vmwgfx_drv.c index 4860370740e0..7bd1be26afe4 100644 --- a/drivers/gpu/drm/vmwgfx/vmwgfx_drv.c +++ b/drivers/gpu/drm/vmwgfx/vmwgfx_drv.c @@ -878,10 +878,11 @@ static int vmw_driver_load(struct drm_device *dev, unsigned long chipset) drm_vma_offset_manager_init(&dev_priv->vma_manager, DRM_FILE_PAGE_OFFSET_START, DRM_FILE_PAGE_OFFSET_SIZE); - ret = ttm_bo_device_init(&dev_priv->bdev, - &vmw_bo_driver, + ret = ttm_bo_device_init(&dev_priv->bdev, &vmw_bo_driver, + dev_priv->dev->dev, dev->anon_inode->i_mapping, &dev_priv->vma_manager, + dev_priv->map_mode == vmw_dma_alloc_coherent, false); if (unlikely(ret != 0)) { DRM_ERROR("Failed initializing TTM buffer object driver.\n"); diff --git a/include/drm/ttm/ttm_bo_driver.h b/include/drm/ttm/ttm_bo_driver.h index 29f6a1d1c853..45ae87640909 100644 --- a/include/drm/ttm/ttm_bo_driver.h +++ b/include/drm/ttm/ttm_bo_driver.h @@ -42,6 +42,7 @@ #include "ttm_module.h" #include "ttm_placement.h" #include "ttm_tt.h" +#include "ttm_pool.h" /** * struct ttm_bo_driver @@ -295,6 +296,7 @@ struct ttm_bo_device { * Protected by internal locks. */ struct drm_vma_offset_manager *vma_manager; + struct ttm_pool pool; /* * Protected by the global:lru lock. @@ -395,11 +397,11 @@ int ttm_bo_device_release(struct ttm_bo_device *bdev); * @bdev: A pointer to a struct ttm_bo_device to initialize. * @glob: A pointer to an initialized struct ttm_bo_global. * @driver: A pointer to a struct ttm_bo_driver set up by the caller. + * @dev: The core kernel device pointer for DMA mappings and allocations. * @mapping: The address space to use for this bo. * @vma_manager: A pointer to a vma manager. - * @file_page_offset: Offset into the device address space that is available - * for buffer data. This ensures compatibility with other users of the - * address space. + * @use_dma_alloc: If coherent DMA allocation API should be used. + * @use_dma32: If we should use GFP_DMA32 for device memory allocations. * * Initializes a struct ttm_bo_device: * Returns: @@ -407,9 +409,10 @@ int ttm_bo_device_release(struct ttm_bo_device *bdev); */ int ttm_bo_device_init(struct ttm_bo_device *bdev, struct ttm_bo_driver *driver, + struct device *dev, struct address_space *mapping, struct drm_vma_offset_manager *vma_manager, - bool need_dma32); + bool use_dma_alloc, bool use_dma32); /** * ttm_bo_unmap_virtual -- cgit From 256dd44bd897055571c131703afdd02b2e4f1f29 Mon Sep 17 00:00:00 2001 From: Christian König Date: Sat, 24 Oct 2020 13:17:49 +0200 Subject: drm/ttm: nuke old page allocator MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Not used any more. Signed-off-by: Christian König Reviewed-by: Dave Airlie Reviewed-by: Madhav Chauhan Tested-by: Huang Rui Link: https://patchwork.freedesktop.org/patch/397087/?series=83051&rev=1 --- drivers/gpu/drm/Kconfig | 7 - drivers/gpu/drm/ttm/Makefile | 3 +- drivers/gpu/drm/ttm/ttm_agp_backend.c | 1 - drivers/gpu/drm/ttm/ttm_bo.c | 1 - drivers/gpu/drm/ttm/ttm_memory.c | 7 +- drivers/gpu/drm/ttm/ttm_page_alloc.c | 1176 ---------------------------- drivers/gpu/drm/ttm/ttm_page_alloc_dma.c | 1226 ------------------------------ drivers/gpu/drm/ttm/ttm_set_memory.h | 84 -- drivers/gpu/drm/ttm/ttm_tt.c | 4 - include/drm/ttm/ttm_bo_driver.h | 2 - include/drm/ttm/ttm_page_alloc.h | 122 --- include/drm/ttm/ttm_tt.h | 2 - 12 files changed, 2 insertions(+), 2633 deletions(-) delete mode 100644 drivers/gpu/drm/ttm/ttm_page_alloc.c delete mode 100644 drivers/gpu/drm/ttm/ttm_page_alloc_dma.c delete mode 100644 drivers/gpu/drm/ttm/ttm_set_memory.h delete mode 100644 include/drm/ttm/ttm_page_alloc.h (limited to 'drivers/gpu/drm/ttm/ttm_memory.c') diff --git a/drivers/gpu/drm/Kconfig b/drivers/gpu/drm/Kconfig index 32257189e09b..64376dd298ed 100644 --- a/drivers/gpu/drm/Kconfig +++ b/drivers/gpu/drm/Kconfig @@ -182,13 +182,6 @@ config DRM_TTM GPU memory types. Will be enabled automatically if a device driver uses it. -config DRM_TTM_DMA_PAGE_POOL - bool - depends on DRM_TTM && (SWIOTLB || INTEL_IOMMU) - default y - help - Choose this if you need the TTM dma page pool - config DRM_VRAM_HELPER tristate depends on DRM diff --git a/drivers/gpu/drm/ttm/Makefile b/drivers/gpu/drm/ttm/Makefile index 0096bacbcf32..b6f5f87b270f 100644 --- a/drivers/gpu/drm/ttm/Makefile +++ b/drivers/gpu/drm/ttm/Makefile @@ -4,9 +4,8 @@ ttm-y := ttm_memory.o ttm_tt.o ttm_bo.o \ ttm_bo_util.o ttm_bo_vm.o ttm_module.o \ - ttm_execbuf_util.o ttm_page_alloc.o ttm_range_manager.o \ + ttm_execbuf_util.o ttm_range_manager.o \ ttm_resource.o ttm_pool.o ttm-$(CONFIG_AGP) += ttm_agp_backend.o -ttm-$(CONFIG_DRM_TTM_DMA_PAGE_POOL) += ttm_page_alloc_dma.o obj-$(CONFIG_DRM_TTM) += ttm.o diff --git a/drivers/gpu/drm/ttm/ttm_agp_backend.c b/drivers/gpu/drm/ttm/ttm_agp_backend.c index 4f76c9287159..03c86628e4ac 100644 --- a/drivers/gpu/drm/ttm/ttm_agp_backend.c +++ b/drivers/gpu/drm/ttm/ttm_agp_backend.c @@ -34,7 +34,6 @@ #include #include -#include #include #include #include diff --git a/drivers/gpu/drm/ttm/ttm_bo.c b/drivers/gpu/drm/ttm/ttm_bo.c index e9f91cfce5e9..530c9a4af09f 100644 --- a/drivers/gpu/drm/ttm/ttm_bo.c +++ b/drivers/gpu/drm/ttm/ttm_bo.c @@ -1333,7 +1333,6 @@ int ttm_bo_device_init(struct ttm_bo_device *bdev, INIT_DELAYED_WORK(&bdev->wq, ttm_bo_delayed_workqueue); INIT_LIST_HEAD(&bdev->ddestroy); bdev->dev_mapping = mapping; - bdev->need_dma32 = use_dma32; mutex_lock(&ttm_global_mutex); list_add_tail(&bdev->device_list, &glob->device_list); mutex_unlock(&ttm_global_mutex); diff --git a/drivers/gpu/drm/ttm/ttm_memory.c b/drivers/gpu/drm/ttm/ttm_memory.c index b15a91c90271..f9a90bfaa3c1 100644 --- a/drivers/gpu/drm/ttm/ttm_memory.c +++ b/drivers/gpu/drm/ttm/ttm_memory.c @@ -30,7 +30,6 @@ #include #include -#include #include #include #include @@ -452,9 +451,7 @@ int ttm_mem_global_init(struct ttm_mem_global *glob) pr_info("Zone %7s: Available graphics memory: %llu KiB\n", zone->name, (unsigned long long)zone->max_mem >> 10); } - ttm_page_alloc_init(glob, glob->zone_kernel->max_mem/(2*PAGE_SIZE)); - ttm_dma_page_alloc_init(glob, glob->zone_kernel->max_mem/(2*PAGE_SIZE)); - ttm_pool_mgr_init(glob->zone_kernel->max_mem / (2 * PAGE_SIZE)); + ttm_pool_mgr_init(glob->zone_kernel->max_mem/(2*PAGE_SIZE)); return 0; out_no_zone: ttm_mem_global_release(glob); @@ -467,8 +464,6 @@ void ttm_mem_global_release(struct ttm_mem_global *glob) unsigned int i; /* let the page allocator first stop the shrink work. */ - ttm_page_alloc_fini(); - ttm_dma_page_alloc_fini(); ttm_pool_mgr_fini(); flush_workqueue(glob->swap_queue); diff --git a/drivers/gpu/drm/ttm/ttm_page_alloc.c b/drivers/gpu/drm/ttm/ttm_page_alloc.c deleted file mode 100644 index 29e6c29ad60e..000000000000 --- a/drivers/gpu/drm/ttm/ttm_page_alloc.c +++ /dev/null @@ -1,1176 +0,0 @@ -/* - * Copyright (c) Red Hat Inc. - - * Permission is hereby granted, free of charge, to any person obtaining a - * copy of this software and associated documentation files (the "Software"), - * to deal in the Software without restriction, including without limitation - * the rights to use, copy, modify, merge, publish, distribute, sub license, - * and/or sell copies of the Software, and to permit persons to whom the - * Software is furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice (including the - * next paragraph) shall be included in all copies or substantial portions - * of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL - * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING - * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER - * DEALINGS IN THE SOFTWARE. - * - * Authors: Dave Airlie - * Jerome Glisse - * Pauli Nieminen - */ - -/* simple list based uncached page pool - * - Pool collects resently freed pages for reuse - * - Use page->lru to keep a free list - * - doesn't track currently in use pages - */ - -#define pr_fmt(fmt) "[TTM] " fmt - -#include -#include -#include -#include -#include -#include -#include /* for seq_printf */ -#include -#include - -#include - -#include -#include - -#include "ttm_set_memory.h" - -#define NUM_PAGES_TO_ALLOC (PAGE_SIZE/sizeof(struct page *)) -#define SMALL_ALLOCATION 16 -#define FREE_ALL_PAGES (~0U) -/* times are in msecs */ -#define PAGE_FREE_INTERVAL 1000 - -/** - * struct ttm_page_pool - Pool to reuse recently allocated uc/wc pages. - * - * @lock: Protects the shared pool from concurrnet access. Must be used with - * irqsave/irqrestore variants because pool allocator maybe called from - * delayed work. - * @fill_lock: Prevent concurrent calls to fill. - * @list: Pool of free uc/wc pages for fast reuse. - * @gfp_flags: Flags to pass for alloc_page. - * @npages: Number of pages in pool. - */ -struct ttm_page_pool { - spinlock_t lock; - bool fill_lock; - struct list_head list; - gfp_t gfp_flags; - unsigned npages; - char *name; - unsigned long nfrees; - unsigned long nrefills; - unsigned int order; -}; - -/** - * Limits for the pool. They are handled without locks because only place where - * they may change is in sysfs store. They won't have immediate effect anyway - * so forcing serialization to access them is pointless. - */ - -struct ttm_pool_opts { - unsigned alloc_size; - unsigned max_size; - unsigned small; -}; - -#define NUM_POOLS 6 - -/** - * struct ttm_pool_manager - Holds memory pools for fst allocation - * - * Manager is read only object for pool code so it doesn't need locking. - * - * @free_interval: minimum number of jiffies between freeing pages from pool. - * @page_alloc_inited: reference counting for pool allocation. - * @work: Work that is used to shrink the pool. Work is only run when there is - * some pages to free. - * @small_allocation: Limit in number of pages what is small allocation. - * - * @pools: All pool objects in use. - **/ -struct ttm_pool_manager { - struct kobject kobj; - struct shrinker mm_shrink; - struct ttm_pool_opts options; - - union { - struct ttm_page_pool pools[NUM_POOLS]; - struct { - struct ttm_page_pool wc_pool; - struct ttm_page_pool uc_pool; - struct ttm_page_pool wc_pool_dma32; - struct ttm_page_pool uc_pool_dma32; - struct ttm_page_pool wc_pool_huge; - struct ttm_page_pool uc_pool_huge; - } ; - }; -}; - -static struct attribute ttm_page_pool_max = { - .name = "pool_max_size", - .mode = S_IRUGO | S_IWUSR -}; -static struct attribute ttm_page_pool_small = { - .name = "pool_small_allocation", - .mode = S_IRUGO | S_IWUSR -}; -static struct attribute ttm_page_pool_alloc_size = { - .name = "pool_allocation_size", - .mode = S_IRUGO | S_IWUSR -}; - -static struct attribute *ttm_pool_attrs[] = { - &ttm_page_pool_max, - &ttm_page_pool_small, - &ttm_page_pool_alloc_size, - NULL -}; - -static void ttm_pool_kobj_release(struct kobject *kobj) -{ - struct ttm_pool_manager *m = - container_of(kobj, struct ttm_pool_manager, kobj); - kfree(m); -} - -static ssize_t ttm_pool_store(struct kobject *kobj, - struct attribute *attr, const char *buffer, size_t size) -{ - struct ttm_pool_manager *m = - container_of(kobj, struct ttm_pool_manager, kobj); - int chars; - unsigned val; - chars = sscanf(buffer, "%u", &val); - if (chars == 0) - return size; - - /* Convert kb to number of pages */ - val = val / (PAGE_SIZE >> 10); - - if (attr == &ttm_page_pool_max) - m->options.max_size = val; - else if (attr == &ttm_page_pool_small) - m->options.small = val; - else if (attr == &ttm_page_pool_alloc_size) { - if (val > NUM_PAGES_TO_ALLOC*8) { - pr_err("Setting allocation size to %lu is not allowed. Recommended size is %lu\n", - NUM_PAGES_TO_ALLOC*(PAGE_SIZE >> 7), - NUM_PAGES_TO_ALLOC*(PAGE_SIZE >> 10)); - return size; - } else if (val > NUM_PAGES_TO_ALLOC) { - pr_warn("Setting allocation size to larger than %lu is not recommended\n", - NUM_PAGES_TO_ALLOC*(PAGE_SIZE >> 10)); - } - m->options.alloc_size = val; - } - - return size; -} - -static ssize_t ttm_pool_show(struct kobject *kobj, - struct attribute *attr, char *buffer) -{ - struct ttm_pool_manager *m = - container_of(kobj, struct ttm_pool_manager, kobj); - unsigned val = 0; - - if (attr == &ttm_page_pool_max) - val = m->options.max_size; - else if (attr == &ttm_page_pool_small) - val = m->options.small; - else if (attr == &ttm_page_pool_alloc_size) - val = m->options.alloc_size; - - val = val * (PAGE_SIZE >> 10); - - return snprintf(buffer, PAGE_SIZE, "%u\n", val); -} - -static const struct sysfs_ops ttm_pool_sysfs_ops = { - .show = &ttm_pool_show, - .store = &ttm_pool_store, -}; - -static struct kobj_type ttm_pool_kobj_type = { - .release = &ttm_pool_kobj_release, - .sysfs_ops = &ttm_pool_sysfs_ops, - .default_attrs = ttm_pool_attrs, -}; - -static struct ttm_pool_manager *_manager; - -/** - * Select the right pool or requested caching state and ttm flags. */ -static struct ttm_page_pool *ttm_get_pool(int flags, bool huge, - enum ttm_caching cstate) -{ - int pool_index; - - if (cstate == ttm_cached) - return NULL; - - if (cstate == ttm_write_combined) - pool_index = 0x0; - else - pool_index = 0x1; - - if (flags & TTM_PAGE_FLAG_DMA32) { - if (huge) - return NULL; - pool_index |= 0x2; - - } else if (huge) { - pool_index |= 0x4; - } - - return &_manager->pools[pool_index]; -} - -/* set memory back to wb and free the pages. */ -static void ttm_pages_put(struct page *pages[], unsigned npages, - unsigned int order) -{ - unsigned int i, pages_nr = (1 << order); - - if (order == 0) { - if (ttm_set_pages_array_wb(pages, npages)) - pr_err("Failed to set %d pages to wb!\n", npages); - } - - for (i = 0; i < npages; ++i) { - if (order > 0) { - if (ttm_set_pages_wb(pages[i], pages_nr)) - pr_err("Failed to set %d pages to wb!\n", pages_nr); - } - __free_pages(pages[i], order); - } -} - -static void ttm_pool_update_free_locked(struct ttm_page_pool *pool, - unsigned freed_pages) -{ - pool->npages -= freed_pages; - pool->nfrees += freed_pages; -} - -/** - * Free pages from pool. - * - * To prevent hogging the ttm_swap process we only free NUM_PAGES_TO_ALLOC - * number of pages in one go. - * - * @pool: to free the pages from - * @free_all: If set to true will free all pages in pool - * @use_static: Safe to use static buffer - **/ -static int ttm_page_pool_free(struct ttm_page_pool *pool, unsigned nr_free, - bool use_static) -{ - static struct page *static_buf[NUM_PAGES_TO_ALLOC]; - unsigned long irq_flags; - struct page *p; - struct page **pages_to_free; - unsigned freed_pages = 0, - npages_to_free = nr_free; - - if (NUM_PAGES_TO_ALLOC < nr_free) - npages_to_free = NUM_PAGES_TO_ALLOC; - - if (use_static) - pages_to_free = static_buf; - else - pages_to_free = kmalloc_array(npages_to_free, - sizeof(struct page *), - GFP_KERNEL); - if (!pages_to_free) { - pr_debug("Failed to allocate memory for pool free operation\n"); - return 0; - } - -restart: - spin_lock_irqsave(&pool->lock, irq_flags); - - list_for_each_entry_reverse(p, &pool->list, lru) { - if (freed_pages >= npages_to_free) - break; - - pages_to_free[freed_pages++] = p; - /* We can only remove NUM_PAGES_TO_ALLOC at a time. */ - if (freed_pages >= NUM_PAGES_TO_ALLOC) { - /* remove range of pages from the pool */ - __list_del(p->lru.prev, &pool->list); - - ttm_pool_update_free_locked(pool, freed_pages); - /** - * Because changing page caching is costly - * we unlock the pool to prevent stalling. - */ - spin_unlock_irqrestore(&pool->lock, irq_flags); - - ttm_pages_put(pages_to_free, freed_pages, pool->order); - if (likely(nr_free != FREE_ALL_PAGES)) - nr_free -= freed_pages; - - if (NUM_PAGES_TO_ALLOC >= nr_free) - npages_to_free = nr_free; - else - npages_to_free = NUM_PAGES_TO_ALLOC; - - freed_pages = 0; - - /* free all so restart the processing */ - if (nr_free) - goto restart; - - /* Not allowed to fall through or break because - * following context is inside spinlock while we are - * outside here. - */ - goto out; - - } - } - - /* remove range of pages from the pool */ - if (freed_pages) { - __list_del(&p->lru, &pool->list); - - ttm_pool_update_free_locked(pool, freed_pages); - nr_free -= freed_pages; - } - - spin_unlock_irqrestore(&pool->lock, irq_flags); - - if (freed_pages) - ttm_pages_put(pages_to_free, freed_pages, pool->order); -out: - if (pages_to_free != static_buf) - kfree(pages_to_free); - return nr_free; -} - -/** - * Callback for mm to request pool to reduce number of page held. - * - * XXX: (dchinner) Deadlock warning! - * - * This code is crying out for a shrinker per pool.... - */ -static unsigned long -ttm_pool_shrink_scan(struct shrinker *shrink, struct shrink_control *sc) -{ - static DEFINE_MUTEX(lock); - static unsigned start_pool; - unsigned i; - unsigned pool_offset; - struct ttm_page_pool *pool; - int shrink_pages = sc->nr_to_scan; - unsigned long freed = 0; - unsigned int nr_free_pool; - - if (!mutex_trylock(&lock)) - return SHRINK_STOP; - pool_offset = ++start_pool % NUM_POOLS; - /* select start pool in round robin fashion */ - for (i = 0; i < NUM_POOLS; ++i) { - unsigned nr_free = shrink_pages; - unsigned page_nr; - - if (shrink_pages == 0) - break; - - pool = &_manager->pools[(i + pool_offset)%NUM_POOLS]; - page_nr = (1 << pool->order); - /* OK to use static buffer since global mutex is held. */ - nr_free_pool = roundup(nr_free, page_nr) >> pool->order; - shrink_pages = ttm_page_pool_free(pool, nr_free_pool, true); - freed += (nr_free_pool - shrink_pages) << pool->order; - if (freed >= sc->nr_to_scan) - break; - shrink_pages <<= pool->order; - } - mutex_unlock(&lock); - return freed; -} - - -static unsigned long -ttm_pool_shrink_count(struct shrinker *shrink, struct shrink_control *sc) -{ - unsigned i; - unsigned long count = 0; - struct ttm_page_pool *pool; - - for (i = 0; i < NUM_POOLS; ++i) { - pool = &_manager->pools[i]; - count += (pool->npages << pool->order); - } - - return count; -} - -static int ttm_pool_mm_shrink_init(struct ttm_pool_manager *manager) -{ - manager->mm_shrink.count_objects = ttm_pool_shrink_count; - manager->mm_shrink.scan_objects = ttm_pool_shrink_scan; - manager->mm_shrink.seeks = 1; - return register_shrinker(&manager->mm_shrink); -} - -static void ttm_pool_mm_shrink_fini(struct ttm_pool_manager *manager) -{ - unregister_shrinker(&manager->mm_shrink); -} - -static int ttm_set_pages_caching(struct page **pages, - enum ttm_caching cstate, unsigned cpages) -{ - int r = 0; - /* Set page caching */ - switch (cstate) { - case ttm_uncached: - r = ttm_set_pages_array_uc(pages, cpages); - if (r) - pr_err("Failed to set %d pages to uc!\n", cpages); - break; - case ttm_write_combined: - r = ttm_set_pages_array_wc(pages, cpages); - if (r) - pr_err("Failed to set %d pages to wc!\n", cpages); - break; - default: - break; - } - return r; -} - -/** - * Free pages the pages that failed to change the caching state. If there is - * any pages that have changed their caching state already put them to the - * pool. - */ -static void ttm_handle_caching_failure(struct page **failed_pages, - unsigned cpages) -{ - unsigned i; - - /* Failed pages have to be freed */ - for (i = 0; i < cpages; ++i) { - list_del(&failed_pages[i]->lru); - __free_page(failed_pages[i]); - } -} - -/** - * Allocate new pages with correct caching. - * - * This function is reentrant if caller updates count depending on number of - * pages returned in pages array. - */ -static int ttm_alloc_new_pages(struct list_head *pages, gfp_t gfp_flags, - int ttm_flags, enum ttm_caching cstate, - unsigned count, unsigned order) -{ - struct page **caching_array; - struct page *p; - int r = 0; - unsigned i, j, cpages; - unsigned npages = 1 << order; - unsigned max_cpages = min(count << order, (unsigned)NUM_PAGES_TO_ALLOC); - - /* allocate array for page caching change */ - caching_array = kmalloc_array(max_cpages, sizeof(struct page *), - GFP_KERNEL); - - if (!caching_array) { - pr_debug("Unable to allocate table for new pages\n"); - return -ENOMEM; - } - - for (i = 0, cpages = 0; i < count; ++i) { - p = alloc_pages(gfp_flags, order); - - if (!p) { - pr_debug("Unable to get page %u\n", i); - - /* store already allocated pages in the pool after - * setting the caching state */ - if (cpages) { - r = ttm_set_pages_caching(caching_array, - cstate, cpages); - if (r) - ttm_handle_caching_failure(caching_array, - cpages); - } - r = -ENOMEM; - goto out; - } - - list_add(&p->lru, pages); - -#ifdef CONFIG_HIGHMEM - /* gfp flags of highmem page should never be dma32 so we - * we should be fine in such case - */ - if (PageHighMem(p)) - continue; - -#endif - for (j = 0; j < npages; ++j) { - caching_array[cpages++] = p++; - if (cpages == max_cpages) { - - r = ttm_set_pages_caching(caching_array, - cstate, cpages); - if (r) { - ttm_handle_caching_failure(caching_array, - cpages); - goto out; - } - cpages = 0; - } - } - } - - if (cpages) { - r = ttm_set_pages_caching(caching_array, cstate, cpages); - if (r) - ttm_handle_caching_failure(caching_array, cpages); - } -out: - kfree(caching_array); - - return r; -} - -/** - * Fill the given pool if there aren't enough pages and the requested number of - * pages is small. - */ -static void ttm_page_pool_fill_locked(struct ttm_page_pool *pool, int ttm_flags, - enum ttm_caching cstate, - unsigned count, unsigned long *irq_flags) -{ - struct page *p; - int r; - unsigned cpages = 0; - /** - * Only allow one pool fill operation at a time. - * If pool doesn't have enough pages for the allocation new pages are - * allocated from outside of pool. - */ - if (pool->fill_lock) - return; - - pool->fill_lock = true; - - /* If allocation request is small and there are not enough - * pages in a pool we fill the pool up first. */ - if (count < _manager->options.small - && count > pool->npages) { - struct list_head new_pages; - unsigned alloc_size = _manager->options.alloc_size; - - /** - * Can't change page caching if in irqsave context. We have to - * drop the pool->lock. - */ - spin_unlock_irqrestore(&pool->lock, *irq_flags); - - INIT_LIST_HEAD(&new_pages); - r = ttm_alloc_new_pages(&new_pages, pool->gfp_flags, ttm_flags, - cstate, alloc_size, 0); - spin_lock_irqsave(&pool->lock, *irq_flags); - - if (!r) { - list_splice(&new_pages, &pool->list); - ++pool->nrefills; - pool->npages += alloc_size; - } else { - pr_debug("Failed to fill pool (%p)\n", pool); - /* If we have any pages left put them to the pool. */ - list_for_each_entry(p, &new_pages, lru) { - ++cpages; - } - list_splice(&new_pages, &pool->list); - pool->npages += cpages; - } - - } - pool->fill_lock = false; -} - -/** - * Allocate pages from the pool and put them on the return list. - * - * @return zero for success or negative error code. - */ -static int ttm_page_pool_get_pages(struct ttm_page_pool *pool, - struct list_head *pages, - int ttm_flags, - enum ttm_caching cstate, - unsigned count, unsigned order) -{ - unsigned long irq_flags; - struct list_head *p; - unsigned i; - int r = 0; - - spin_lock_irqsave(&pool->lock, irq_flags); - if (!order) - ttm_page_pool_fill_locked(pool, ttm_flags, cstate, count, - &irq_flags); - - if (count >= pool->npages) { - /* take all pages from the pool */ - list_splice_init(&pool->list, pages); - count -= pool->npages; - pool->npages = 0; - goto out; - } - /* find the last pages to include for requested number of pages. Split - * pool to begin and halve it to reduce search space. */ - if (count <= pool->npages/2) { - i = 0; - list_for_each(p, &pool->list) { - if (++i == count) - break; - } - } else { - i = pool->npages + 1; - list_for_each_prev(p, &pool->list) { - if (--i == count) - break; - } - } - /* Cut 'count' number of pages from the pool */ - list_cut_position(pages, &pool->list, p); - pool->npages -= count; - count = 0; -out: - spin_unlock_irqrestore(&pool->lock, irq_flags); - - /* clear the pages coming from the pool if requested */ - if (ttm_flags & TTM_PAGE_FLAG_ZERO_ALLOC) { - struct page *page; - - list_for_each_entry(page, pages, lru) { - if (PageHighMem(page)) - clear_highpage(page); - else - clear_page(page_address(page)); - } - } - - /* If pool didn't have enough pages allocate new one. */ - if (count) { - gfp_t gfp_flags = pool->gfp_flags; - - /* set zero flag for page allocation if required */ - if (ttm_flags & TTM_PAGE_FLAG_ZERO_ALLOC) - gfp_flags |= __GFP_ZERO; - - if (ttm_flags & TTM_PAGE_FLAG_NO_RETRY) - gfp_flags |= __GFP_RETRY_MAYFAIL; - - /* ttm_alloc_new_pages doesn't reference pool so we can run - * multiple requests in parallel. - **/ - r = ttm_alloc_new_pages(pages, gfp_flags, ttm_flags, cstate, - count, order); - } - - return r; -} - -/* Put all pages in pages list to correct pool to wait for reuse */ -static void ttm_put_pages(struct page **pages, unsigned npages, int flags, - enum ttm_caching cstate) -{ - struct ttm_page_pool *pool = ttm_get_pool(flags, false, cstate); -#ifdef CONFIG_TRANSPARENT_HUGEPAGE - struct ttm_page_pool *huge = ttm_get_pool(flags, true, cstate); -#endif - unsigned long irq_flags; - unsigned i; - - if (pool == NULL) { - /* No pool for this memory type so free the pages */ - i = 0; - while (i < npages) { -#ifdef CONFIG_TRANSPARENT_HUGEPAGE - struct page *p = pages[i]; -#endif - unsigned order = 0, j; - - if (!pages[i]) { - ++i; - continue; - } - -#ifdef CONFIG_TRANSPARENT_HUGEPAGE - if (!(flags & TTM_PAGE_FLAG_DMA32) && - (npages - i) >= HPAGE_PMD_NR) { - for (j = 1; j < HPAGE_PMD_NR; ++j) - if (++p != pages[i + j]) - break; - - if (j == HPAGE_PMD_NR) - order = HPAGE_PMD_ORDER; - } -#endif - - if (page_count(pages[i]) != 1) - pr_err("Erroneous page count. Leaking pages.\n"); - __free_pages(pages[i], order); - - j = 1 << order; - while (j) { - pages[i++] = NULL; - --j; - } - } - return; - } - - i = 0; -#ifdef CONFIG_TRANSPARENT_HUGEPAGE - if (huge) { - unsigned max_size, n2free; - - spin_lock_irqsave(&huge->lock, irq_flags); - while ((npages - i) >= HPAGE_PMD_NR) { - struct page *p = pages[i]; - unsigned j; - - if (!p) - break; - - for (j = 1; j < HPAGE_PMD_NR; ++j) - if (++p != pages[i + j]) - break; - - if (j != HPAGE_PMD_NR) - break; - - list_add_tail(&pages[i]->lru, &huge->list); - - for (j = 0; j < HPAGE_PMD_NR; ++j) - pages[i++] = NULL; - huge->npages++; - } - - /* Check that we don't go over the pool limit */ - max_size = _manager->options.max_size; - max_size /= HPAGE_PMD_NR; - if (huge->npages > max_size) - n2free = huge->npages - max_size; - else - n2free = 0; - spin_unlock_irqrestore(&huge->lock, irq_flags); - if (n2free) - ttm_page_pool_free(huge, n2free, false); - } -#endif - - spin_lock_irqsave(&pool->lock, irq_flags); - while (i < npages) { - if (pages[i]) { - if (page_count(pages[i]) != 1) - pr_err("Erroneous page count. Leaking pages.\n"); - list_add_tail(&pages[i]->lru, &pool->list); - pages[i] = NULL; - pool->npages++; - } - ++i; - } - /* Check that we don't go over the pool limit */ - npages = 0; - if (pool->npages > _manager->options.max_size) { - npages = pool->npages - _manager->options.max_size; - /* free at least NUM_PAGES_TO_ALLOC number of pages - * to reduce calls to set_memory_wb */ - if (npages < NUM_PAGES_TO_ALLOC) - npages = NUM_PAGES_TO_ALLOC; - } - spin_unlock_irqrestore(&pool->lock, irq_flags); - if (npages) - ttm_page_pool_free(pool, npages, false); -} - -/* - * On success pages list will hold count number of correctly - * cached pages. - */ -static int ttm_get_pages(struct page **pages, unsigned npages, int flags, - enum ttm_caching cstate) -{ - struct ttm_page_pool *pool = ttm_get_pool(flags, false, cstate); -#ifdef CONFIG_TRANSPARENT_HUGEPAGE - struct ttm_page_pool *huge = ttm_get_pool(flags, true, cstate); -#endif - struct list_head plist; - struct page *p = NULL; - unsigned count, first; - int r; - - /* No pool for cached pages */ - if (pool == NULL) { - gfp_t gfp_flags = GFP_USER; - unsigned i; -#ifdef CONFIG_TRANSPARENT_HUGEPAGE - unsigned j; -#endif - - /* set zero flag for page allocation if required */ - if (flags & TTM_PAGE_FLAG_ZERO_ALLOC) - gfp_flags |= __GFP_ZERO; - - if (flags & TTM_PAGE_FLAG_NO_RETRY) - gfp_flags |= __GFP_RETRY_MAYFAIL; - - if (flags & TTM_PAGE_FLAG_DMA32) - gfp_flags |= GFP_DMA32; - else - gfp_flags |= GFP_HIGHUSER; - - i = 0; -#ifdef CONFIG_TRANSPARENT_HUGEPAGE - if (!(gfp_flags & GFP_DMA32)) { - while (npages >= HPAGE_PMD_NR) { - gfp_t huge_flags = gfp_flags; - - huge_flags |= GFP_TRANSHUGE_LIGHT | __GFP_NORETRY | - __GFP_KSWAPD_RECLAIM; - huge_flags &= ~__GFP_MOVABLE; - huge_flags &= ~__GFP_COMP; - p = alloc_pages(huge_flags, HPAGE_PMD_ORDER); - if (!p) - break; - - for (j = 0; j < HPAGE_PMD_NR; ++j) - pages[i++] = p++; - - npages -= HPAGE_PMD_NR; - } - } -#endif - - first = i; - while (npages) { - p = alloc_page(gfp_flags); - if (!p) { - pr_debug("Unable to allocate page\n"); - return -ENOMEM; - } - - /* Swap the pages if we detect consecutive order */ - if (i > first && pages[i - 1] == p - 1) - swap(p, pages[i - 1]); - - pages[i++] = p; - --npages; - } - return 0; - } - - count = 0; - -#ifdef CONFIG_TRANSPARENT_HUGEPAGE - if (huge && npages >= HPAGE_PMD_NR) { - INIT_LIST_HEAD(&plist); - ttm_page_pool_get_pages(huge, &plist, flags, cstate, - npages / HPAGE_PMD_NR, - HPAGE_PMD_ORDER); - - list_for_each_entry(p, &plist, lru) { - unsigned j; - - for (j = 0; j < HPAGE_PMD_NR; ++j) - pages[count++] = &p[j]; - } - } -#endif - - INIT_LIST_HEAD(&plist); - r = ttm_page_pool_get_pages(pool, &plist, flags, cstate, - npages - count, 0); - - first = count; - list_for_each_entry(p, &plist, lru) { - struct page *tmp = p; - - /* Swap the pages if we detect consecutive order */ - if (count > first && pages[count - 1] == tmp - 1) - swap(tmp, pages[count - 1]); - pages[count++] = tmp; - } - - if (r) { - /* If there is any pages in the list put them back to - * the pool. - */ - pr_debug("Failed to allocate extra pages for large request\n"); - ttm_put_pages(pages, count, flags, cstate); - return r; - } - - return 0; -} - -static void ttm_page_pool_init_locked(struct ttm_page_pool *pool, gfp_t flags, - char *name, unsigned int order) -{ - spin_lock_init(&pool->lock); - pool->fill_lock = false; - INIT_LIST_HEAD(&pool->list); - pool->npages = pool->nfrees = 0; - pool->gfp_flags = flags; - pool->name = name; - pool->order = order; -} - -int ttm_page_alloc_init(struct ttm_mem_global *glob, unsigned max_pages) -{ - int ret; -#ifdef CONFIG_TRANSPARENT_HUGEPAGE - unsigned order = HPAGE_PMD_ORDER; -#else - unsigned order = 0; -#endif - - WARN_ON(_manager); - - pr_info("Initializing pool allocator\n"); - - _manager = kzalloc(sizeof(*_manager), GFP_KERNEL); - if (!_manager) - return -ENOMEM; - - ttm_page_pool_init_locked(&_manager->wc_pool, GFP_HIGHUSER, "wc", 0); - - ttm_page_pool_init_locked(&_manager->uc_pool, GFP_HIGHUSER, "uc", 0); - - ttm_page_pool_init_locked(&_manager->wc_pool_dma32, - GFP_USER | GFP_DMA32, "wc dma", 0); - - ttm_page_pool_init_locked(&_manager->uc_pool_dma32, - GFP_USER | GFP_DMA32, "uc dma", 0); - - ttm_page_pool_init_locked(&_manager->wc_pool_huge, - (GFP_TRANSHUGE_LIGHT | __GFP_NORETRY | - __GFP_KSWAPD_RECLAIM) & - ~(__GFP_MOVABLE | __GFP_COMP), - "wc huge", order); - - ttm_page_pool_init_locked(&_manager->uc_pool_huge, - (GFP_TRANSHUGE_LIGHT | __GFP_NORETRY | - __GFP_KSWAPD_RECLAIM) & - ~(__GFP_MOVABLE | __GFP_COMP) - , "uc huge", order); - - _manager->options.max_size = max_pages; - _manager->options.small = SMALL_ALLOCATION; - _manager->options.alloc_size = NUM_PAGES_TO_ALLOC; - - ret = kobject_init_and_add(&_manager->kobj, &ttm_pool_kobj_type, - &glob->kobj, "pool"); - if (unlikely(ret != 0)) - goto error; - - ret = ttm_pool_mm_shrink_init(_manager); - if (unlikely(ret != 0)) - goto error; - return 0; - -error: - kobject_put(&_manager->kobj); - _manager = NULL; - return ret; -} - -void ttm_page_alloc_fini(void) -{ - int i; - - pr_info("Finalizing pool allocator\n"); - ttm_pool_mm_shrink_fini(_manager); - - /* OK to use static buffer since global mutex is no longer used. */ - for (i = 0; i < NUM_POOLS; ++i) - ttm_page_pool_free(&_manager->pools[i], FREE_ALL_PAGES, true); - - kobject_put(&_manager->kobj); - _manager = NULL; -} - -static void -ttm_pool_unpopulate_helper(struct ttm_tt *ttm, unsigned mem_count_update) -{ - struct ttm_mem_global *mem_glob = &ttm_mem_glob; - unsigned i; - - if (mem_count_update == 0) - goto put_pages; - - for (i = 0; i < mem_count_update; ++i) { - if (!ttm->pages[i]) - continue; - - ttm_mem_global_free_page(mem_glob, ttm->pages[i], PAGE_SIZE); - } - -put_pages: - ttm_put_pages(ttm->pages, ttm->num_pages, ttm->page_flags, - ttm->caching); -} - -int ttm_pool_populate(struct ttm_tt *ttm, struct ttm_operation_ctx *ctx) -{ - struct ttm_mem_global *mem_glob = &ttm_mem_glob; - unsigned i; - int ret; - - if (ttm_tt_is_populated(ttm)) - return 0; - - if (ttm_check_under_lowerlimit(mem_glob, ttm->num_pages, ctx)) - return -ENOMEM; - - ret = ttm_get_pages(ttm->pages, ttm->num_pages, ttm->page_flags, - ttm->caching); - if (unlikely(ret != 0)) { - ttm_pool_unpopulate_helper(ttm, 0); - return ret; - } - - for (i = 0; i < ttm->num_pages; ++i) { - ret = ttm_mem_global_alloc_page(mem_glob, ttm->pages[i], - PAGE_SIZE, ctx); - if (unlikely(ret != 0)) { - ttm_pool_unpopulate_helper(ttm, i); - return -ENOMEM; - } - } - - return 0; -} -EXPORT_SYMBOL(ttm_pool_populate); - -void ttm_pool_unpopulate(struct ttm_tt *ttm) -{ - ttm_pool_unpopulate_helper(ttm, ttm->num_pages); -} -EXPORT_SYMBOL(ttm_pool_unpopulate); - -int ttm_populate_and_map_pages(struct device *dev, struct ttm_tt *tt, - struct ttm_operation_ctx *ctx) -{ - unsigned i, j; - int r; - - r = ttm_pool_populate(tt, ctx); - if (r) - return r; - - for (i = 0; i < tt->num_pages; ++i) { - struct page *p = tt->pages[i]; - size_t num_pages = 1; - - for (j = i + 1; j < tt->num_pages; ++j) { - if (++p != tt->pages[j]) - break; - - ++num_pages; - } - - tt->dma_address[i] = dma_map_page(dev, tt->pages[i], - 0, num_pages * PAGE_SIZE, - DMA_BIDIRECTIONAL); - if (dma_mapping_error(dev, tt->dma_address[i])) { - while (i--) { - dma_unmap_page(dev, tt->dma_address[i], - PAGE_SIZE, DMA_BIDIRECTIONAL); - tt->dma_address[i] = 0; - } - ttm_pool_unpopulate(tt); - return -EFAULT; - } - - for (j = 1; j < num_pages; ++j) { - tt->dma_address[i + 1] = tt->dma_address[i] + PAGE_SIZE; - ++i; - } - } - return 0; -} -EXPORT_SYMBOL(ttm_populate_and_map_pages); - -void ttm_unmap_and_unpopulate_pages(struct device *dev, struct ttm_tt *tt) -{ - unsigned i, j; - - for (i = 0; i < tt->num_pages;) { - struct page *p = tt->pages[i]; - size_t num_pages = 1; - - if (!tt->dma_address[i] || !tt->pages[i]) { - ++i; - continue; - } - - for (j = i + 1; j < tt->num_pages; ++j) { - if (++p != tt->pages[j]) - break; - - ++num_pages; - } - - dma_unmap_page(dev, tt->dma_address[i], num_pages * PAGE_SIZE, - DMA_BIDIRECTIONAL); - - i += num_pages; - } - ttm_pool_unpopulate(tt); -} -EXPORT_SYMBOL(ttm_unmap_and_unpopulate_pages); - -int ttm_page_alloc_debugfs(struct seq_file *m, void *data) -{ - struct ttm_page_pool *p; - unsigned i; - char *h[] = {"pool", "refills", "pages freed", "size"}; - if (!_manager) { - seq_printf(m, "No pool allocator running.\n"); - return 0; - } - seq_printf(m, "%7s %12s %13s %8s\n", - h[0], h[1], h[2], h[3]); - for (i = 0; i < NUM_POOLS; ++i) { - p = &_manager->pools[i]; - - seq_printf(m, "%7s %12ld %13ld %8d\n", - p->name, p->nrefills, - p->nfrees, p->npages); - } - return 0; -} -EXPORT_SYMBOL(ttm_page_alloc_debugfs); diff --git a/drivers/gpu/drm/ttm/ttm_page_alloc_dma.c b/drivers/gpu/drm/ttm/ttm_page_alloc_dma.c deleted file mode 100644 index c0353c25efd6..000000000000 --- a/drivers/gpu/drm/ttm/ttm_page_alloc_dma.c +++ /dev/null @@ -1,1226 +0,0 @@ -/* - * Copyright 2011 (c) Oracle Corp. - - * Permission is hereby granted, free of charge, to any person obtaining a - * copy of this software and associated documentation files (the "Software"), - * to deal in the Software without restriction, including without limitation - * the rights to use, copy, modify, merge, publish, distribute, sub license, - * and/or sell copies of the Software, and to permit persons to whom the - * Software is furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice (including the - * next paragraph) shall be included in all copies or substantial portions - * of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL - * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING - * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER - * DEALINGS IN THE SOFTWARE. - * - * Author: Konrad Rzeszutek Wilk - */ - -/* - * A simple DMA pool losely based on dmapool.c. It has certain advantages - * over the DMA pools: - * - Pool collects resently freed pages for reuse (and hooks up to - * the shrinker). - * - Tracks currently in use pages - * - Tracks whether the page is UC, WB or cached (and reverts to WB - * when freed). - */ - -#define pr_fmt(fmt) "[TTM] " fmt - -#include -#include -#include /* for seq_printf */ -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include "ttm_set_memory.h" - -#define NUM_PAGES_TO_ALLOC (PAGE_SIZE/sizeof(struct page *)) -#define SMALL_ALLOCATION 4 -#define FREE_ALL_PAGES (~0U) -#define VADDR_FLAG_HUGE_POOL 1UL -#define VADDR_FLAG_UPDATED_COUNT 2UL - -enum pool_type { - IS_UNDEFINED = 0, - IS_WC = 1 << 1, - IS_UC = 1 << 2, - IS_CACHED = 1 << 3, - IS_DMA32 = 1 << 4, - IS_HUGE = 1 << 5 -}; - -/* - * The pool structure. There are up to nine pools: - * - generic (not restricted to DMA32): - * - write combined, uncached, cached. - * - dma32 (up to 2^32 - so up 4GB): - * - write combined, uncached, cached. - * - huge (not restricted to DMA32): - * - write combined, uncached, cached. - * for each 'struct device'. The 'cached' is for pages that are actively used. - * The other ones can be shrunk by the shrinker API if neccessary. - * @pools: The 'struct device->dma_pools' link. - * @type: Type of the pool - * @lock: Protects the free_list from concurrnet access. Must be - * used with irqsave/irqrestore variants because pool allocator maybe called - * from delayed work. - * @free_list: Pool of pages that are free to be used. No order requirements. - * @dev: The device that is associated with these pools. - * @size: Size used during DMA allocation. - * @npages_free: Count of available pages for re-use. - * @npages_in_use: Count of pages that are in use. - * @nfrees: Stats when pool is shrinking. - * @nrefills: Stats when the pool is grown. - * @gfp_flags: Flags to pass for alloc_page. - * @name: Name of the pool. - * @dev_name: Name derieved from dev - similar to how dev_info works. - * Used during shutdown as the dev_info during release is unavailable. - */ -struct dma_pool { - struct list_head pools; /* The 'struct device->dma_pools link */ - enum pool_type type; - spinlock_t lock; - struct list_head free_list; - struct device *dev; - unsigned size; - unsigned npages_free; - unsigned npages_in_use; - unsigned long nfrees; /* Stats when shrunk. */ - unsigned long nrefills; /* Stats when grown. */ - gfp_t gfp_flags; - char name[13]; /* "cached dma32" */ - char dev_name[64]; /* Constructed from dev */ -}; - -/* - * The accounting page keeping track of the allocated page along with - * the DMA address. - * @page_list: The link to the 'page_list' in 'struct dma_pool'. - * @vaddr: The virtual address of the page and a flag if the page belongs to a - * huge pool - * @dma: The bus address of the page. If the page is not allocated - * via the DMA API, it will be -1. - */ -struct dma_page { - struct list_head page_list; - unsigned long vaddr; - struct page *p; - dma_addr_t dma; -}; - -/* - * Limits for the pool. They are handled without locks because only place where - * they may change is in sysfs store. They won't have immediate effect anyway - * so forcing serialization to access them is pointless. - */ - -struct ttm_pool_opts { - unsigned alloc_size; - unsigned max_size; - unsigned small; -}; - -/* - * Contains the list of all of the 'struct device' and their corresponding - * DMA pools. Guarded by _mutex->lock. - * @pools: The link to 'struct ttm_pool_manager->pools' - * @dev: The 'struct device' associated with the 'pool' - * @pool: The 'struct dma_pool' associated with the 'dev' - */ -struct device_pools { - struct list_head pools; - struct device *dev; - struct dma_pool *pool; -}; - -/* - * struct ttm_pool_manager - Holds memory pools for fast allocation - * - * @lock: Lock used when adding/removing from pools - * @pools: List of 'struct device' and 'struct dma_pool' tuples. - * @options: Limits for the pool. - * @npools: Total amount of pools in existence. - * @shrinker: The structure used by [un|]register_shrinker - */ -struct ttm_pool_manager { - struct mutex lock; - struct list_head pools; - struct ttm_pool_opts options; - unsigned npools; - struct shrinker mm_shrink; - struct kobject kobj; -}; - -static struct ttm_pool_manager *_manager; - -static struct attribute ttm_page_pool_max = { - .name = "pool_max_size", - .mode = S_IRUGO | S_IWUSR -}; -static struct attribute ttm_page_pool_small = { - .name = "pool_small_allocation", - .mode = S_IRUGO | S_IWUSR -}; -static struct attribute ttm_page_pool_alloc_size = { - .name = "pool_allocation_size", - .mode = S_IRUGO | S_IWUSR -}; - -static struct attribute *ttm_pool_attrs[] = { - &ttm_page_pool_max, - &ttm_page_pool_small, - &ttm_page_pool_alloc_size, - NULL -}; - -static void ttm_pool_kobj_release(struct kobject *kobj) -{ - struct ttm_pool_manager *m = - container_of(kobj, struct ttm_pool_manager, kobj); - kfree(m); -} - -static ssize_t ttm_pool_store(struct kobject *kobj, struct attribute *attr, - const char *buffer, size_t size) -{ - struct ttm_pool_manager *m = - container_of(kobj, struct ttm_pool_manager, kobj); - int chars; - unsigned val; - - chars = sscanf(buffer, "%u", &val); - if (chars == 0) - return size; - - /* Convert kb to number of pages */ - val = val / (PAGE_SIZE >> 10); - - if (attr == &ttm_page_pool_max) { - m->options.max_size = val; - } else if (attr == &ttm_page_pool_small) { - m->options.small = val; - } else if (attr == &ttm_page_pool_alloc_size) { - if (val > NUM_PAGES_TO_ALLOC*8) { - pr_err("Setting allocation size to %lu is not allowed. Recommended size is %lu\n", - NUM_PAGES_TO_ALLOC*(PAGE_SIZE >> 7), - NUM_PAGES_TO_ALLOC*(PAGE_SIZE >> 10)); - return size; - } else if (val > NUM_PAGES_TO_ALLOC) { - pr_warn("Setting allocation size to larger than %lu is not recommended\n", - NUM_PAGES_TO_ALLOC*(PAGE_SIZE >> 10)); - } - m->options.alloc_size = val; - } - - return size; -} - -static ssize_t ttm_pool_show(struct kobject *kobj, struct attribute *attr, - char *buffer) -{ - struct ttm_pool_manager *m = - container_of(kobj, struct ttm_pool_manager, kobj); - unsigned val = 0; - - if (attr == &ttm_page_pool_max) - val = m->options.max_size; - else if (attr == &ttm_page_pool_small) - val = m->options.small; - else if (attr == &ttm_page_pool_alloc_size) - val = m->options.alloc_size; - - val = val * (PAGE_SIZE >> 10); - - return snprintf(buffer, PAGE_SIZE, "%u\n", val); -} - -static const struct sysfs_ops ttm_pool_sysfs_ops = { - .show = &ttm_pool_show, - .store = &ttm_pool_store, -}; - -static struct kobj_type ttm_pool_kobj_type = { - .release = &ttm_pool_kobj_release, - .sysfs_ops = &ttm_pool_sysfs_ops, - .default_attrs = ttm_pool_attrs, -}; - -static int ttm_set_pages_caching(struct dma_pool *pool, - struct page **pages, unsigned cpages) -{ - int r = 0; - /* Set page caching */ - if (pool->type & IS_UC) { - r = ttm_set_pages_array_uc(pages, cpages); - if (r) - pr_err("%s: Failed to set %d pages to uc!\n", - pool->dev_name, cpages); - } - if (pool->type & IS_WC) { - r = ttm_set_pages_array_wc(pages, cpages); - if (r) - pr_err("%s: Failed to set %d pages to wc!\n", - pool->dev_name, cpages); - } - return r; -} - -static void __ttm_dma_free_page(struct dma_pool *pool, struct dma_page *d_page) -{ - unsigned long attrs = 0; - dma_addr_t dma = d_page->dma; - d_page->vaddr &= ~VADDR_FLAG_HUGE_POOL; - if (pool->type & IS_HUGE) - attrs = DMA_ATTR_NO_WARN; - - dma_free_attrs(pool->dev, pool->size, (void *)d_page->vaddr, dma, attrs); - - kfree(d_page); - d_page = NULL; -} -static struct dma_page *__ttm_dma_alloc_page(struct dma_pool *pool) -{ - struct dma_page *d_page; - unsigned long attrs = 0; - void *vaddr; - - d_page = kmalloc(sizeof(struct dma_page), GFP_KERNEL); - if (!d_page) - return NULL; - - if (pool->type & IS_HUGE) - attrs = DMA_ATTR_NO_WARN; - - vaddr = dma_alloc_attrs(pool->dev, pool->size, &d_page->dma, - pool->gfp_flags, attrs); - if (vaddr) { - if (is_vmalloc_addr(vaddr)) - d_page->p = vmalloc_to_page(vaddr); - else - d_page->p = virt_to_page(vaddr); - d_page->vaddr = (unsigned long)vaddr; - if (pool->type & IS_HUGE) - d_page->vaddr |= VADDR_FLAG_HUGE_POOL; - } else { - kfree(d_page); - d_page = NULL; - } - return d_page; -} -static enum pool_type ttm_to_type(int flags, enum ttm_caching cstate) -{ - enum pool_type type = IS_UNDEFINED; - - if (flags & TTM_PAGE_FLAG_DMA32) - type |= IS_DMA32; - if (cstate == ttm_cached) - type |= IS_CACHED; - else if (cstate == ttm_uncached) - type |= IS_UC; - else - type |= IS_WC; - - return type; -} - -static void ttm_pool_update_free_locked(struct dma_pool *pool, - unsigned freed_pages) -{ - pool->npages_free -= freed_pages; - pool->nfrees += freed_pages; - -} - -/* set memory back to wb and free the pages. */ -static void ttm_dma_page_put(struct dma_pool *pool, struct dma_page *d_page) -{ - struct page *page = d_page->p; - unsigned num_pages; - - /* Don't set WB on WB page pool. */ - if (!(pool->type & IS_CACHED)) { - num_pages = pool->size / PAGE_SIZE; - if (ttm_set_pages_wb(page, num_pages)) - pr_err("%s: Failed to set %d pages to wb!\n", - pool->dev_name, num_pages); - } - - list_del(&d_page->page_list); - __ttm_dma_free_page(pool, d_page); -} - -static void ttm_dma_pages_put(struct dma_pool *pool, struct list_head *d_pages, - struct page *pages[], unsigned npages) -{ - struct dma_page *d_page, *tmp; - - if (pool->type & IS_HUGE) { - list_for_each_entry_safe(d_page, tmp, d_pages, page_list) - ttm_dma_page_put(pool, d_page); - - return; - } - - /* Don't set WB on WB page pool. */ - if (npages && !(pool->type & IS_CACHED) && - ttm_set_pages_array_wb(pages, npages)) - pr_err("%s: Failed to set %d pages to wb!\n", - pool->dev_name, npages); - - list_for_each_entry_safe(d_page, tmp, d_pages, page_list) { - list_del(&d_page->page_list); - __ttm_dma_free_page(pool, d_page); - } -} - -/* - * Free pages from pool. - * - * To prevent hogging the ttm_swap process we only free NUM_PAGES_TO_ALLOC - * number of pages in one go. - * - * @pool: to free the pages from - * @nr_free: If set to true will free all pages in pool - * @use_static: Safe to use static buffer - **/ -static unsigned ttm_dma_page_pool_free(struct dma_pool *pool, unsigned nr_free, - bool use_static) -{ - static struct page *static_buf[NUM_PAGES_TO_ALLOC]; - unsigned long irq_flags; - struct dma_page *dma_p, *tmp; - struct page **pages_to_free; - struct list_head d_pages; - unsigned freed_pages = 0, - npages_to_free = nr_free; - - if (NUM_PAGES_TO_ALLOC < nr_free) - npages_to_free = NUM_PAGES_TO_ALLOC; - - if (use_static) - pages_to_free = static_buf; - else - pages_to_free = kmalloc_array(npages_to_free, - sizeof(struct page *), - GFP_KERNEL); - - if (!pages_to_free) { - pr_debug("%s: Failed to allocate memory for pool free operation\n", - pool->dev_name); - return 0; - } - INIT_LIST_HEAD(&d_pages); -restart: - spin_lock_irqsave(&pool->lock, irq_flags); - - /* We picking the oldest ones off the list */ - list_for_each_entry_safe_reverse(dma_p, tmp, &pool->free_list, - page_list) { - if (freed_pages >= npages_to_free) - break; - - /* Move the dma_page from one list to another. */ - list_move(&dma_p->page_list, &d_pages); - - pages_to_free[freed_pages++] = dma_p->p; - /* We can only remove NUM_PAGES_TO_ALLOC at a time. */ - if (freed_pages >= NUM_PAGES_TO_ALLOC) { - - ttm_pool_update_free_locked(pool, freed_pages); - /** - * Because changing page caching is costly - * we unlock the pool to prevent stalling. - */ - spin_unlock_irqrestore(&pool->lock, irq_flags); - - ttm_dma_pages_put(pool, &d_pages, pages_to_free, - freed_pages); - - INIT_LIST_HEAD(&d_pages); - - if (likely(nr_free != FREE_ALL_PAGES)) - nr_free -= freed_pages; - - if (NUM_PAGES_TO_ALLOC >= nr_free) - npages_to_free = nr_free; - else - npages_to_free = NUM_PAGES_TO_ALLOC; - - freed_pages = 0; - - /* free all so restart the processing */ - if (nr_free) - goto restart; - - /* Not allowed to fall through or break because - * following context is inside spinlock while we are - * outside here. - */ - goto out; - - } - } - - /* remove range of pages from the pool */ - if (freed_pages) { - ttm_pool_update_free_locked(pool, freed_pages); - nr_free -= freed_pages; - } - - spin_unlock_irqrestore(&pool->lock, irq_flags); - - if (freed_pages) - ttm_dma_pages_put(pool, &d_pages, pages_to_free, freed_pages); -out: - if (pages_to_free != static_buf) - kfree(pages_to_free); - return nr_free; -} - -static void ttm_dma_free_pool(struct device *dev, enum pool_type type) -{ - struct device_pools *p; - struct dma_pool *pool; - - if (!dev) - return; - - mutex_lock(&_manager->lock); - list_for_each_entry_reverse(p, &_manager->pools, pools) { - if (p->dev != dev) - continue; - pool = p->pool; - if (pool->type != type) - continue; - - list_del(&p->pools); - kfree(p); - _manager->npools--; - break; - } - list_for_each_entry_reverse(pool, &dev->dma_pools, pools) { - if (pool->type != type) - continue; - /* Takes a spinlock.. */ - /* OK to use static buffer since global mutex is held. */ - ttm_dma_page_pool_free(pool, FREE_ALL_PAGES, true); - WARN_ON(((pool->npages_in_use + pool->npages_free) != 0)); - /* This code path is called after _all_ references to the - * struct device has been dropped - so nobody should be - * touching it. In case somebody is trying to _add_ we are - * guarded by the mutex. */ - list_del(&pool->pools); - kfree(pool); - break; - } - mutex_unlock(&_manager->lock); -} - -/* - * On free-ing of the 'struct device' this deconstructor is run. - * Albeit the pool might have already been freed earlier. - */ -static void ttm_dma_pool_release(struct device *dev, void *res) -{ - struct dma_pool *pool = *(struct dma_pool **)res; - - if (pool) - ttm_dma_free_pool(dev, pool->type); -} - -static int ttm_dma_pool_match(struct device *dev, void *res, void *match_data) -{ - return *(struct dma_pool **)res == match_data; -} - -static struct dma_pool *ttm_dma_pool_init(struct device *dev, gfp_t flags, - enum pool_type type) -{ - const char *n[] = {"wc", "uc", "cached", " dma32", "huge"}; - enum pool_type t[] = {IS_WC, IS_UC, IS_CACHED, IS_DMA32, IS_HUGE}; - struct device_pools *sec_pool = NULL; - struct dma_pool *pool = NULL, **ptr; - unsigned i; - int ret = -ENODEV; - char *p; - - if (!dev) - return NULL; - - ptr = devres_alloc(ttm_dma_pool_release, sizeof(*ptr), GFP_KERNEL); - if (!ptr) - return NULL; - - ret = -ENOMEM; - - pool = kmalloc_node(sizeof(struct dma_pool), GFP_KERNEL, - dev_to_node(dev)); - if (!pool) - goto err_mem; - - sec_pool = kmalloc_node(sizeof(struct device_pools), GFP_KERNEL, - dev_to_node(dev)); - if (!sec_pool) - goto err_mem; - - INIT_LIST_HEAD(&sec_pool->pools); - sec_pool->dev = dev; - sec_pool->pool = pool; - - INIT_LIST_HEAD(&pool->free_list); - INIT_LIST_HEAD(&pool->pools); - spin_lock_init(&pool->lock); - pool->dev = dev; - pool->npages_free = pool->npages_in_use = 0; - pool->nfrees = 0; - pool->gfp_flags = flags; - if (type & IS_HUGE) -#ifdef CONFIG_TRANSPARENT_HUGEPAGE - pool->size = HPAGE_PMD_SIZE; -#else - BUG(); -#endif - else - pool->size = PAGE_SIZE; - pool->type = type; - pool->nrefills = 0; - p = pool->name; - for (i = 0; i < ARRAY_SIZE(t); i++) { - if (type & t[i]) { - p += scnprintf(p, sizeof(pool->name) - (p - pool->name), - "%s", n[i]); - } - } - *p = 0; - /* We copy the name for pr_ calls b/c when dma_pool_destroy is called - * - the kobj->name has already been deallocated.*/ - snprintf(pool->dev_name, sizeof(pool->dev_name), "%s %s", - dev_driver_string(dev), dev_name(dev)); - mutex_lock(&_manager->lock); - /* You can get the dma_pool from either the global: */ - list_add(&sec_pool->pools, &_manager->pools); - _manager->npools++; - /* or from 'struct device': */ - list_add(&pool->pools, &dev->dma_pools); - mutex_unlock(&_manager->lock); - - *ptr = pool; - devres_add(dev, ptr); - - return pool; -err_mem: - devres_free(ptr); - kfree(sec_pool); - kfree(pool); - return ERR_PTR(ret); -} - -static struct dma_pool *ttm_dma_find_pool(struct device *dev, - enum pool_type type) -{ - struct dma_pool *pool, *tmp; - - if (type == IS_UNDEFINED) - return NULL; - - /* NB: We iterate on the 'struct dev' which has no spinlock, but - * it does have a kref which we have taken. The kref is taken during - * graphic driver loading - in the drm_pci_init it calls either - * pci_dev_get or pci_register_driver which both end up taking a kref - * on 'struct device'. - * - * On teardown, the graphic drivers end up quiescing the TTM (put_pages) - * and calls the dev_res deconstructors: ttm_dma_pool_release. The nice - * thing is at that point of time there are no pages associated with the - * driver so this function will not be called. - */ - list_for_each_entry_safe(pool, tmp, &dev->dma_pools, pools) - if (pool->type == type) - return pool; - return NULL; -} - -/* - * Free pages the pages that failed to change the caching state. If there - * are pages that have changed their caching state already put them to the - * pool. - */ -static void ttm_dma_handle_caching_failure(struct dma_pool *pool, - struct list_head *d_pages, - struct page **failed_pages, - unsigned cpages) -{ - struct dma_page *d_page, *tmp; - struct page *p; - unsigned i = 0; - - p = failed_pages[0]; - if (!p) - return; - /* Find the failed page. */ - list_for_each_entry_safe(d_page, tmp, d_pages, page_list) { - if (d_page->p != p) - continue; - /* .. and then progress over the full list. */ - list_del(&d_page->page_list); - __ttm_dma_free_page(pool, d_page); - if (++i < cpages) - p = failed_pages[i]; - else - break; - } - -} - -/* - * Allocate 'count' pages, and put 'need' number of them on the - * 'pages' and as well on the 'dma_address' starting at 'dma_offset' offset. - * The full list of pages should also be on 'd_pages'. - * We return zero for success, and negative numbers as errors. - */ -static int ttm_dma_pool_alloc_new_pages(struct dma_pool *pool, - struct list_head *d_pages, - unsigned count) -{ - struct page **caching_array; - struct dma_page *dma_p; - struct page *p; - int r = 0; - unsigned i, j, npages, cpages; - unsigned max_cpages = min(count, - (unsigned)(PAGE_SIZE/sizeof(struct page *))); - - /* allocate array for page caching change */ - caching_array = kmalloc_array(max_cpages, sizeof(struct page *), - GFP_KERNEL); - - if (!caching_array) { - pr_debug("%s: Unable to allocate table for new pages\n", - pool->dev_name); - return -ENOMEM; - } - - if (count > 1) - pr_debug("%s: (%s:%d) Getting %d pages\n", - pool->dev_name, pool->name, current->pid, count); - - for (i = 0, cpages = 0; i < count; ++i) { - dma_p = __ttm_dma_alloc_page(pool); - if (!dma_p) { - pr_debug("%s: Unable to get page %u\n", - pool->dev_name, i); - - /* store already allocated pages in the pool after - * setting the caching state */ - if (cpages) { - r = ttm_set_pages_caching(pool, caching_array, - cpages); - if (r) - ttm_dma_handle_caching_failure( - pool, d_pages, caching_array, - cpages); - } - r = -ENOMEM; - goto out; - } - p = dma_p->p; - list_add(&dma_p->page_list, d_pages); - -#ifdef CONFIG_HIGHMEM - /* gfp flags of highmem page should never be dma32 so we - * we should be fine in such case - */ - if (PageHighMem(p)) - continue; -#endif - - npages = pool->size / PAGE_SIZE; - for (j = 0; j < npages; ++j) { - caching_array[cpages++] = p + j; - if (cpages == max_cpages) { - /* Note: Cannot hold the spinlock */ - r = ttm_set_pages_caching(pool, caching_array, - cpages); - if (r) { - ttm_dma_handle_caching_failure( - pool, d_pages, caching_array, - cpages); - goto out; - } - cpages = 0; - } - } - } - - if (cpages) { - r = ttm_set_pages_caching(pool, caching_array, cpages); - if (r) - ttm_dma_handle_caching_failure(pool, d_pages, - caching_array, cpages); - } -out: - kfree(caching_array); - return r; -} - -/* - * @return count of pages still required to fulfill the request. - */ -static int ttm_dma_page_pool_fill_locked(struct dma_pool *pool, - unsigned long *irq_flags) -{ - unsigned count = _manager->options.small; - int r = pool->npages_free; - - if (count > pool->npages_free) { - struct list_head d_pages; - - INIT_LIST_HEAD(&d_pages); - - spin_unlock_irqrestore(&pool->lock, *irq_flags); - - /* Returns how many more are neccessary to fulfill the - * request. */ - r = ttm_dma_pool_alloc_new_pages(pool, &d_pages, count); - - spin_lock_irqsave(&pool->lock, *irq_flags); - if (!r) { - /* Add the fresh to the end.. */ - list_splice(&d_pages, &pool->free_list); - ++pool->nrefills; - pool->npages_free += count; - r = count; - } else { - struct dma_page *d_page; - unsigned cpages = 0; - - pr_debug("%s: Failed to fill %s pool (r:%d)!\n", - pool->dev_name, pool->name, r); - - list_for_each_entry(d_page, &d_pages, page_list) { - cpages++; - } - list_splice_tail(&d_pages, &pool->free_list); - pool->npages_free += cpages; - r = cpages; - } - } - return r; -} - -/* - * The populate list is actually a stack (not that is matters as TTM - * allocates one page at a time. - * return dma_page pointer if success, otherwise NULL. - */ -static struct dma_page *ttm_dma_pool_get_pages(struct dma_pool *pool, - struct ttm_tt *ttm, - unsigned index) -{ - struct dma_page *d_page = NULL; - unsigned long irq_flags; - int count; - - spin_lock_irqsave(&pool->lock, irq_flags); - count = ttm_dma_page_pool_fill_locked(pool, &irq_flags); - if (count) { - d_page = list_first_entry(&pool->free_list, struct dma_page, page_list); - ttm->pages[index] = d_page->p; - ttm->dma_address[index] = d_page->dma; - list_move_tail(&d_page->page_list, &ttm->pages_list); - pool->npages_in_use += 1; - pool->npages_free -= 1; - } - spin_unlock_irqrestore(&pool->lock, irq_flags); - return d_page; -} - -static gfp_t ttm_dma_pool_gfp_flags(struct ttm_tt *ttm, bool huge) -{ - gfp_t gfp_flags; - - if (ttm->page_flags & TTM_PAGE_FLAG_DMA32) - gfp_flags = GFP_USER | GFP_DMA32; - else - gfp_flags = GFP_HIGHUSER; - if (ttm->page_flags & TTM_PAGE_FLAG_ZERO_ALLOC) - gfp_flags |= __GFP_ZERO; - - if (huge) { - gfp_flags |= GFP_TRANSHUGE_LIGHT | __GFP_NORETRY | - __GFP_KSWAPD_RECLAIM; - gfp_flags &= ~__GFP_MOVABLE; - gfp_flags &= ~__GFP_COMP; - } - - if (ttm->page_flags & TTM_PAGE_FLAG_NO_RETRY) - gfp_flags |= __GFP_RETRY_MAYFAIL; - - return gfp_flags; -} - -/* - * On success pages list will hold count number of correctly - * cached pages. On failure will hold the negative return value (-ENOMEM, etc). - */ -int ttm_dma_populate(struct ttm_tt *ttm, struct device *dev, - struct ttm_operation_ctx *ctx) -{ - struct ttm_mem_global *mem_glob = &ttm_mem_glob; - unsigned long num_pages = ttm->num_pages; - struct dma_pool *pool; - struct dma_page *d_page; - enum pool_type type; - unsigned i; - int ret; - - if (ttm_tt_is_populated(ttm)) - return 0; - - if (ttm_check_under_lowerlimit(mem_glob, num_pages, ctx)) - return -ENOMEM; - - INIT_LIST_HEAD(&ttm->pages_list); - i = 0; - - type = ttm_to_type(ttm->page_flags, ttm->caching); - -#ifdef CONFIG_TRANSPARENT_HUGEPAGE - if (ttm->page_flags & TTM_PAGE_FLAG_DMA32) - goto skip_huge; - - pool = ttm_dma_find_pool(dev, type | IS_HUGE); - if (!pool) { - gfp_t gfp_flags = ttm_dma_pool_gfp_flags(ttm, true); - - pool = ttm_dma_pool_init(dev, gfp_flags, type | IS_HUGE); - if (IS_ERR_OR_NULL(pool)) - goto skip_huge; - } - - while (num_pages >= HPAGE_PMD_NR) { - unsigned j; - - d_page = ttm_dma_pool_get_pages(pool, ttm, i); - if (!d_page) - break; - - ret = ttm_mem_global_alloc_page(mem_glob, ttm->pages[i], - pool->size, ctx); - if (unlikely(ret != 0)) { - ttm_dma_unpopulate(ttm, dev); - return -ENOMEM; - } - - d_page->vaddr |= VADDR_FLAG_UPDATED_COUNT; - for (j = i + 1; j < (i + HPAGE_PMD_NR); ++j) { - ttm->pages[j] = ttm->pages[j - 1] + 1; - ttm->dma_address[j] = ttm->dma_address[j - 1] + - PAGE_SIZE; - } - - i += HPAGE_PMD_NR; - num_pages -= HPAGE_PMD_NR; - } - -skip_huge: -#endif - - pool = ttm_dma_find_pool(dev, type); - if (!pool) { - gfp_t gfp_flags = ttm_dma_pool_gfp_flags(ttm, false); - - pool = ttm_dma_pool_init(dev, gfp_flags, type); - if (IS_ERR_OR_NULL(pool)) - return -ENOMEM; - } - - while (num_pages) { - d_page = ttm_dma_pool_get_pages(pool, ttm, i); - if (!d_page) { - ttm_dma_unpopulate(ttm, dev); - return -ENOMEM; - } - - ret = ttm_mem_global_alloc_page(mem_glob, ttm->pages[i], - pool->size, ctx); - if (unlikely(ret != 0)) { - ttm_dma_unpopulate(ttm, dev); - return -ENOMEM; - } - - d_page->vaddr |= VADDR_FLAG_UPDATED_COUNT; - ++i; - --num_pages; - } - - return 0; -} -EXPORT_SYMBOL_GPL(ttm_dma_populate); - -/* Put all pages in pages list to correct pool to wait for reuse */ -void ttm_dma_unpopulate(struct ttm_tt *ttm, struct device *dev) -{ - struct ttm_mem_global *mem_glob = &ttm_mem_glob; - struct dma_pool *pool; - struct dma_page *d_page, *next; - enum pool_type type; - bool is_cached = false; - unsigned count, i, npages = 0; - unsigned long irq_flags; - - type = ttm_to_type(ttm->page_flags, ttm->caching); - -#ifdef CONFIG_TRANSPARENT_HUGEPAGE - pool = ttm_dma_find_pool(dev, type | IS_HUGE); - if (pool) { - count = 0; - list_for_each_entry_safe(d_page, next, &ttm->pages_list, - page_list) { - if (!(d_page->vaddr & VADDR_FLAG_HUGE_POOL)) - continue; - - count++; - if (d_page->vaddr & VADDR_FLAG_UPDATED_COUNT) { - ttm_mem_global_free_page(mem_glob, d_page->p, - pool->size); - d_page->vaddr &= ~VADDR_FLAG_UPDATED_COUNT; - } - ttm_dma_page_put(pool, d_page); - } - - spin_lock_irqsave(&pool->lock, irq_flags); - pool->npages_in_use -= count; - pool->nfrees += count; - spin_unlock_irqrestore(&pool->lock, irq_flags); - } -#endif - - pool = ttm_dma_find_pool(dev, type); - if (!pool) - return; - - is_cached = (ttm_dma_find_pool(pool->dev, - ttm_to_type(ttm->page_flags, ttm_cached)) == pool); - - /* make sure pages array match list and count number of pages */ - count = 0; - list_for_each_entry_safe(d_page, next, &ttm->pages_list, - page_list) { - ttm->pages[count] = d_page->p; - count++; - - if (d_page->vaddr & VADDR_FLAG_UPDATED_COUNT) { - ttm_mem_global_free_page(mem_glob, d_page->p, - pool->size); - d_page->vaddr &= ~VADDR_FLAG_UPDATED_COUNT; - } - - if (is_cached) - ttm_dma_page_put(pool, d_page); - } - - spin_lock_irqsave(&pool->lock, irq_flags); - pool->npages_in_use -= count; - if (is_cached) { - pool->nfrees += count; - } else { - pool->npages_free += count; - list_splice(&ttm->pages_list, &pool->free_list); - /* - * Wait to have at at least NUM_PAGES_TO_ALLOC number of pages - * to free in order to minimize calls to set_memory_wb(). - */ - if (pool->npages_free >= (_manager->options.max_size + - NUM_PAGES_TO_ALLOC)) - npages = pool->npages_free - _manager->options.max_size; - } - spin_unlock_irqrestore(&pool->lock, irq_flags); - - INIT_LIST_HEAD(&ttm->pages_list); - for (i = 0; i < ttm->num_pages; i++) { - ttm->pages[i] = NULL; - ttm->dma_address[i] = 0; - } - - /* shrink pool if necessary (only on !is_cached pools)*/ - if (npages) - ttm_dma_page_pool_free(pool, npages, false); -} -EXPORT_SYMBOL_GPL(ttm_dma_unpopulate); - -/** - * Callback for mm to request pool to reduce number of page held. - * - * XXX: (dchinner) Deadlock warning! - * - * I'm getting sadder as I hear more pathetical whimpers about needing per-pool - * shrinkers - */ -static unsigned long -ttm_dma_pool_shrink_scan(struct shrinker *shrink, struct shrink_control *sc) -{ - static unsigned start_pool; - unsigned idx = 0; - unsigned pool_offset; - unsigned shrink_pages = sc->nr_to_scan; - struct device_pools *p; - unsigned long freed = 0; - - if (list_empty(&_manager->pools)) - return SHRINK_STOP; - - if (!mutex_trylock(&_manager->lock)) - return SHRINK_STOP; - if (!_manager->npools) - goto out; - pool_offset = ++start_pool % _manager->npools; - list_for_each_entry(p, &_manager->pools, pools) { - unsigned nr_free; - - if (!p->dev) - continue; - if (shrink_pages == 0) - break; - /* Do it in round-robin fashion. */ - if (++idx < pool_offset) - continue; - nr_free = shrink_pages; - /* OK to use static buffer since global mutex is held. */ - shrink_pages = ttm_dma_page_pool_free(p->pool, nr_free, true); - freed += nr_free - shrink_pages; - - pr_debug("%s: (%s:%d) Asked to shrink %d, have %d more to go\n", - p->pool->dev_name, p->pool->name, current->pid, - nr_free, shrink_pages); - } -out: - mutex_unlock(&_manager->lock); - return freed; -} - -static unsigned long -ttm_dma_pool_shrink_count(struct shrinker *shrink, struct shrink_control *sc) -{ - struct device_pools *p; - unsigned long count = 0; - - if (!mutex_trylock(&_manager->lock)) - return 0; - list_for_each_entry(p, &_manager->pools, pools) - count += p->pool->npages_free; - mutex_unlock(&_manager->lock); - return count; -} - -static int ttm_dma_pool_mm_shrink_init(struct ttm_pool_manager *manager) -{ - manager->mm_shrink.count_objects = ttm_dma_pool_shrink_count; - manager->mm_shrink.scan_objects = &ttm_dma_pool_shrink_scan; - manager->mm_shrink.seeks = 1; - return register_shrinker(&manager->mm_shrink); -} - -static void ttm_dma_pool_mm_shrink_fini(struct ttm_pool_manager *manager) -{ - unregister_shrinker(&manager->mm_shrink); -} - -int ttm_dma_page_alloc_init(struct ttm_mem_global *glob, unsigned max_pages) -{ - int ret; - - WARN_ON(_manager); - - pr_info("Initializing DMA pool allocator\n"); - - _manager = kzalloc(sizeof(*_manager), GFP_KERNEL); - if (!_manager) - return -ENOMEM; - - mutex_init(&_manager->lock); - INIT_LIST_HEAD(&_manager->pools); - - _manager->options.max_size = max_pages; - _manager->options.small = SMALL_ALLOCATION; - _manager->options.alloc_size = NUM_PAGES_TO_ALLOC; - - /* This takes care of auto-freeing the _manager */ - ret = kobject_init_and_add(&_manager->kobj, &ttm_pool_kobj_type, - &glob->kobj, "dma_pool"); - if (unlikely(ret != 0)) - goto error; - - ret = ttm_dma_pool_mm_shrink_init(_manager); - if (unlikely(ret != 0)) - goto error; - return 0; - -error: - kobject_put(&_manager->kobj); - _manager = NULL; - return ret; -} - -void ttm_dma_page_alloc_fini(void) -{ - struct device_pools *p, *t; - - pr_info("Finalizing DMA pool allocator\n"); - ttm_dma_pool_mm_shrink_fini(_manager); - - list_for_each_entry_safe_reverse(p, t, &_manager->pools, pools) { - dev_dbg(p->dev, "(%s:%d) Freeing.\n", p->pool->name, - current->pid); - WARN_ON(devres_destroy(p->dev, ttm_dma_pool_release, - ttm_dma_pool_match, p->pool)); - ttm_dma_free_pool(p->dev, p->pool->type); - } - kobject_put(&_manager->kobj); - _manager = NULL; -} - -int ttm_dma_page_alloc_debugfs(struct seq_file *m, void *data) -{ - struct device_pools *p; - struct dma_pool *pool = NULL; - - if (!_manager) { - seq_printf(m, "No pool allocator running.\n"); - return 0; - } - seq_printf(m, " pool refills pages freed inuse available name\n"); - mutex_lock(&_manager->lock); - list_for_each_entry(p, &_manager->pools, pools) { - struct device *dev = p->dev; - if (!dev) - continue; - pool = p->pool; - seq_printf(m, "%13s %12ld %13ld %8d %8d %8s\n", - pool->name, pool->nrefills, - pool->nfrees, pool->npages_in_use, - pool->npages_free, - pool->dev_name); - } - mutex_unlock(&_manager->lock); - return 0; -} -EXPORT_SYMBOL_GPL(ttm_dma_page_alloc_debugfs); diff --git a/drivers/gpu/drm/ttm/ttm_set_memory.h b/drivers/gpu/drm/ttm/ttm_set_memory.h deleted file mode 100644 index 2343c18a6133..000000000000 --- a/drivers/gpu/drm/ttm/ttm_set_memory.h +++ /dev/null @@ -1,84 +0,0 @@ -/************************************************************************** - * - * Copyright (c) 2018 Advanced Micro Devices, Inc. - * All Rights Reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a - * copy of this software and associated documentation files (the - * "Software"), to deal in the Software without restriction, including - * without limitation the rights to use, copy, modify, merge, publish, - * distribute, sub license, and/or sell copies of the Software, and to - * permit persons to whom the Software is furnished to do so, subject to - * the following conditions: - * - * The above copyright notice and this permission notice (including the - * next paragraph) shall be included in all copies or substantial portions - * of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL - * THE COPYRIGHT HOLDERS, AUTHORS AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM, - * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR - * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE - * USE OR OTHER DEALINGS IN THE SOFTWARE. - * - **************************************************************************/ -/* - * Authors: Huang Rui - */ - -#ifndef TTM_SET_MEMORY -#define TTM_SET_MEMORY - -#include - -#ifdef CONFIG_X86 - -#include - -static inline int ttm_set_pages_array_wb(struct page **pages, int addrinarray) -{ - return set_pages_array_wb(pages, addrinarray); -} - -static inline int ttm_set_pages_array_wc(struct page **pages, int addrinarray) -{ - return set_pages_array_wc(pages, addrinarray); -} - -static inline int ttm_set_pages_array_uc(struct page **pages, int addrinarray) -{ - return set_pages_array_uc(pages, addrinarray); -} - -static inline int ttm_set_pages_wb(struct page *page, int numpages) -{ - return set_pages_wb(page, numpages); -} - -#else /* for CONFIG_X86 */ - -static inline int ttm_set_pages_array_wb(struct page **pages, int addrinarray) -{ - return 0; -} - -static inline int ttm_set_pages_array_wc(struct page **pages, int addrinarray) -{ - return 0; -} - -static inline int ttm_set_pages_array_uc(struct page **pages, int addrinarray) -{ - return 0; -} - -static inline int ttm_set_pages_wb(struct page *page, int numpages) -{ - return 0; -} - -#endif /* for CONFIG_X86 */ - -#endif diff --git a/drivers/gpu/drm/ttm/ttm_tt.c b/drivers/gpu/drm/ttm/ttm_tt.c index 90054280cd8f..8861a74ac335 100644 --- a/drivers/gpu/drm/ttm/ttm_tt.c +++ b/drivers/gpu/drm/ttm/ttm_tt.c @@ -51,9 +51,6 @@ int ttm_tt_create(struct ttm_buffer_object *bo, bool zero_alloc) if (bo->ttm) return 0; - if (bdev->need_dma32) - page_flags |= TTM_PAGE_FLAG_DMA32; - if (bdev->no_retry) page_flags |= TTM_PAGE_FLAG_NO_RETRY; @@ -141,7 +138,6 @@ static void ttm_tt_init_fields(struct ttm_tt *ttm, ttm->dma_address = NULL; ttm->swap_storage = NULL; ttm->sg = bo->sg; - INIT_LIST_HEAD(&ttm->pages_list); ttm->caching = caching; } diff --git a/include/drm/ttm/ttm_bo_driver.h b/include/drm/ttm/ttm_bo_driver.h index 45ae87640909..e9f683fa72dc 100644 --- a/include/drm/ttm/ttm_bo_driver.h +++ b/include/drm/ttm/ttm_bo_driver.h @@ -315,8 +315,6 @@ struct ttm_bo_device { struct delayed_work wq; - bool need_dma32; - bool no_retry; }; diff --git a/include/drm/ttm/ttm_page_alloc.h b/include/drm/ttm/ttm_page_alloc.h deleted file mode 100644 index 8fa1e7df6213..000000000000 --- a/include/drm/ttm/ttm_page_alloc.h +++ /dev/null @@ -1,122 +0,0 @@ -/* - * Copyright (c) Red Hat Inc. - - * Permission is hereby granted, free of charge, to any person obtaining a - * copy of this software and associated documentation files (the "Software"), - * to deal in the Software without restriction, including without limitation - * the rights to use, copy, modify, merge, publish, distribute, sub license, - * and/or sell copies of the Software, and to permit persons to whom the - * Software is furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice (including the - * next paragraph) shall be included in all copies or substantial portions - * of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL - * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING - * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER - * DEALINGS IN THE SOFTWARE. - * - * Authors: Dave Airlie - * Jerome Glisse - */ -#ifndef TTM_PAGE_ALLOC -#define TTM_PAGE_ALLOC - -#include -#include - -struct device; - -/** - * Initialize pool allocator. - */ -int ttm_page_alloc_init(struct ttm_mem_global *glob, unsigned max_pages); -/** - * Free pool allocator. - */ -void ttm_page_alloc_fini(void); - -/** - * ttm_pool_populate: - * - * @ttm: The struct ttm_tt to contain the backing pages. - * - * Add backing pages to all of @ttm - */ -int ttm_pool_populate(struct ttm_tt *ttm, struct ttm_operation_ctx *ctx); - -/** - * ttm_pool_unpopulate: - * - * @ttm: The struct ttm_tt which to free backing pages. - * - * Free all pages of @ttm - */ -void ttm_pool_unpopulate(struct ttm_tt *ttm); - -/** - * Populates and DMA maps pages to fullfil a ttm_dma_populate() request - */ -int ttm_populate_and_map_pages(struct device *dev, struct ttm_tt *tt, - struct ttm_operation_ctx *ctx); - -/** - * Unpopulates and DMA unmaps pages as part of a - * ttm_dma_unpopulate() request */ -void ttm_unmap_and_unpopulate_pages(struct device *dev, struct ttm_tt *tt); - -/** - * Output the state of pools to debugfs file - */ -int ttm_page_alloc_debugfs(struct seq_file *m, void *data); - -#if defined(CONFIG_DRM_TTM_DMA_PAGE_POOL) -/** - * Initialize pool allocator. - */ -int ttm_dma_page_alloc_init(struct ttm_mem_global *glob, unsigned max_pages); - -/** - * Free pool allocator. - */ -void ttm_dma_page_alloc_fini(void); - -/** - * Output the state of pools to debugfs file - */ -int ttm_dma_page_alloc_debugfs(struct seq_file *m, void *data); - -int ttm_dma_populate(struct ttm_tt *ttm_dma, struct device *dev, - struct ttm_operation_ctx *ctx); -void ttm_dma_unpopulate(struct ttm_tt *ttm_dma, struct device *dev); - -#else -static inline int ttm_dma_page_alloc_init(struct ttm_mem_global *glob, - unsigned max_pages) -{ - return -ENODEV; -} - -static inline void ttm_dma_page_alloc_fini(void) { return; } - -static inline int ttm_dma_page_alloc_debugfs(struct seq_file *m, void *data) -{ - return 0; -} -static inline int ttm_dma_populate(struct ttm_tt *ttm_dma, - struct device *dev, - struct ttm_operation_ctx *ctx) -{ - return -ENOMEM; -} -static inline void ttm_dma_unpopulate(struct ttm_tt *ttm_dma, - struct device *dev) -{ -} -#endif - -#endif diff --git a/include/drm/ttm/ttm_tt.h b/include/drm/ttm/ttm_tt.h index df9a80650feb..da27e9d8fa64 100644 --- a/include/drm/ttm/ttm_tt.h +++ b/include/drm/ttm/ttm_tt.h @@ -37,7 +37,6 @@ struct ttm_operation_ctx; #define TTM_PAGE_FLAG_SWAPPED (1 << 4) #define TTM_PAGE_FLAG_ZERO_ALLOC (1 << 6) -#define TTM_PAGE_FLAG_DMA32 (1 << 7) #define TTM_PAGE_FLAG_SG (1 << 8) #define TTM_PAGE_FLAG_NO_RETRY (1 << 9) @@ -66,7 +65,6 @@ struct ttm_tt { struct sg_table *sg; dma_addr_t *dma_address; struct file *swap_storage; - struct list_head pages_list; enum ttm_caching caching; }; -- cgit From c44dfe4de053914cae61ed0a36421b3017f428bd Mon Sep 17 00:00:00 2001 From: Christian König Date: Mon, 2 Nov 2020 13:16:13 +0100 Subject: drm/ttm: replace context flags with bools v2 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The ttm_operation_ctx structure has a mixture of flags and bools. Drop the flags and replace them with bools as well. v2: fix typos, improve comments Signed-off-by: Christian König Reviewed-by: Daniel Vetter Link: https://patchwork.freedesktop.org/patch/398686/ --- drivers/gpu/drm/amd/amdgpu/amdgpu_cs.c | 3 +-- drivers/gpu/drm/amd/amdgpu/amdgpu_object.c | 5 ++--- drivers/gpu/drm/ttm/ttm_bo.c | 2 +- drivers/gpu/drm/ttm/ttm_bo_vm.c | 3 +-- drivers/gpu/drm/ttm/ttm_memory.c | 3 ++- drivers/gpu/drm/ttm/ttm_resource.c | 2 +- include/drm/ttm/ttm_bo_api.h | 13 ++++++------- 7 files changed, 14 insertions(+), 17 deletions(-) (limited to 'drivers/gpu/drm/ttm/ttm_memory.c') diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_cs.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_cs.c index d50b63a93d37..8466558d0d93 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_cs.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_cs.c @@ -404,8 +404,7 @@ static int amdgpu_cs_bo_validate(struct amdgpu_cs_parser *p, struct ttm_operation_ctx ctx = { .interruptible = true, .no_wait_gpu = false, - .resv = bo->tbo.base.resv, - .flags = 0 + .resv = bo->tbo.base.resv }; uint32_t domain; int r; diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_object.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_object.c index 4e9dfbea31c6..e1f64ef8c765 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_object.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_object.c @@ -518,9 +518,8 @@ static int amdgpu_bo_do_create(struct amdgpu_device *adev, .no_wait_gpu = bp->no_wait_gpu, /* We opt to avoid OOM on system pages allocations */ .gfp_retry_mayfail = true, - .resv = bp->resv, - .flags = bp->type != ttm_bo_type_kernel ? - TTM_OPT_FLAG_ALLOW_RES_EVICT : 0 + .allow_res_evict = bp->type != ttm_bo_type_kernel, + .resv = bp->resv }; struct amdgpu_bo *bo; unsigned long page_align, size = bp->size; diff --git a/drivers/gpu/drm/ttm/ttm_bo.c b/drivers/gpu/drm/ttm/ttm_bo.c index c63b7ea1cd5d..e2a124b3affb 100644 --- a/drivers/gpu/drm/ttm/ttm_bo.c +++ b/drivers/gpu/drm/ttm/ttm_bo.c @@ -637,7 +637,7 @@ static bool ttm_bo_evict_swapout_allowable(struct ttm_buffer_object *bo, if (bo->base.resv == ctx->resv) { dma_resv_assert_held(bo->base.resv); - if (ctx->flags & TTM_OPT_FLAG_ALLOW_RES_EVICT) + if (ctx->allow_res_evict) ret = true; *locked = false; if (busy) diff --git a/drivers/gpu/drm/ttm/ttm_bo_vm.c b/drivers/gpu/drm/ttm/ttm_bo_vm.c index eeaca5d1efe3..2944fa0af493 100644 --- a/drivers/gpu/drm/ttm/ttm_bo_vm.c +++ b/drivers/gpu/drm/ttm/ttm_bo_vm.c @@ -315,8 +315,7 @@ vm_fault_t ttm_bo_vm_fault_reserved(struct vm_fault *vmf, struct ttm_operation_ctx ctx = { .interruptible = false, .no_wait_gpu = false, - .flags = TTM_OPT_FLAG_FORCE_ALLOC - + .force_alloc = true }; ttm = bo->ttm; diff --git a/drivers/gpu/drm/ttm/ttm_memory.c b/drivers/gpu/drm/ttm/ttm_memory.c index f9a90bfaa3c1..5ed1fc8f2ace 100644 --- a/drivers/gpu/drm/ttm/ttm_memory.c +++ b/drivers/gpu/drm/ttm/ttm_memory.c @@ -542,7 +542,8 @@ ttm_check_under_lowerlimit(struct ttm_mem_global *glob, { int64_t available; - if (ctx->flags & TTM_OPT_FLAG_FORCE_ALLOC) + /* We allow over commit during suspend */ + if (ctx->force_alloc) return false; available = get_nr_swap_pages() + si_mem_available(); diff --git a/drivers/gpu/drm/ttm/ttm_resource.c b/drivers/gpu/drm/ttm/ttm_resource.c index 4ebc043e2867..b60699bf4816 100644 --- a/drivers/gpu/drm/ttm/ttm_resource.c +++ b/drivers/gpu/drm/ttm/ttm_resource.c @@ -89,7 +89,7 @@ int ttm_resource_manager_evict_all(struct ttm_bo_device *bdev, struct ttm_operation_ctx ctx = { .interruptible = false, .no_wait_gpu = false, - .flags = TTM_OPT_FLAG_FORCE_ALLOC + .force_alloc = true }; struct ttm_bo_global *glob = &ttm_bo_glob; struct dma_fence *fence; diff --git a/include/drm/ttm/ttm_bo_api.h b/include/drm/ttm/ttm_bo_api.h index 4637357ba856..5ddad88ae6ed 100644 --- a/include/drm/ttm/ttm_bo_api.h +++ b/include/drm/ttm/ttm_bo_api.h @@ -196,8 +196,11 @@ struct ttm_bo_kmap_obj { * @interruptible: Sleep interruptible if sleeping. * @no_wait_gpu: Return immediately if the GPU is busy. * @gfp_retry_mayfail: Set the __GFP_RETRY_MAYFAIL when allocation pages. + * @allow_res_evict: Allow eviction of reserved BOs. Can be used when multiple + * BOs share the same reservation object. + * @force_alloc: Don't check the memory account during suspend or CPU page + * faults. Should only be used by TTM internally. * @resv: Reservation object to allow reserved evictions with. - * @flags: Including the following flags * * Context for TTM operations like changing buffer placement or general memory * allocation. @@ -206,16 +209,12 @@ struct ttm_operation_ctx { bool interruptible; bool no_wait_gpu; bool gfp_retry_mayfail; + bool allow_res_evict; + bool force_alloc; struct dma_resv *resv; uint64_t bytes_moved; - uint32_t flags; }; -/* Allow eviction of reserved BOs */ -#define TTM_OPT_FLAG_ALLOW_RES_EVICT 0x1 -/* when serving page fault or suspend, allow alloc anyway */ -#define TTM_OPT_FLAG_FORCE_ALLOC 0x2 - /** * ttm_bo_get - reference a struct ttm_buffer_object * -- cgit