diff options
Diffstat (limited to 'drivers/gpu/drm/ttm')
| -rw-r--r-- | drivers/gpu/drm/ttm/ttm_bo.c | 230 | ||||
| -rw-r--r-- | drivers/gpu/drm/ttm/ttm_bo_util.c | 137 | ||||
| -rw-r--r-- | drivers/gpu/drm/ttm/ttm_bo_vm.c | 23 | ||||
| -rw-r--r-- | drivers/gpu/drm/ttm/ttm_device.c | 26 | ||||
| -rw-r--r-- | drivers/gpu/drm/ttm/ttm_execbuf_util.c | 6 | ||||
| -rw-r--r-- | drivers/gpu/drm/ttm/ttm_pool.c | 85 | ||||
| -rw-r--r-- | drivers/gpu/drm/ttm/ttm_range_manager.c | 5 | ||||
| -rw-r--r-- | drivers/gpu/drm/ttm/ttm_resource.c | 17 | ||||
| -rw-r--r-- | drivers/gpu/drm/ttm/ttm_tt.c | 3 |
9 files changed, 329 insertions, 203 deletions
diff --git a/drivers/gpu/drm/ttm/ttm_bo.c b/drivers/gpu/drm/ttm/ttm_bo.c index 7c8e8be774f1..326a3d13a829 100644 --- a/drivers/gpu/drm/ttm/ttm_bo.c +++ b/drivers/gpu/drm/ttm/ttm_bo.c @@ -31,8 +31,10 @@ #define pr_fmt(fmt) "[TTM] " fmt -#include <drm/ttm/ttm_bo_driver.h> +#include <drm/ttm/ttm_bo.h> #include <drm/ttm/ttm_placement.h> +#include <drm/ttm/ttm_tt.h> + #include <linux/jiffies.h> #include <linux/slab.h> #include <linux/sched.h> @@ -51,9 +53,6 @@ static void ttm_bo_mem_space_debug(struct ttm_buffer_object *bo, struct ttm_resource_manager *man; int i, mem_type; - drm_printf(&p, "No space for %p (%lu pages, %zuK, %zuM)\n", - bo, bo->resource->num_pages, bo->base.size >> 10, - bo->base.size >> 20); for (i = 0; i < placement->num_placement; i++) { mem_type = placement->placement[i].mem_type; drm_printf(&p, " placement[%d]=0x%08X (%d)\n", @@ -283,14 +282,13 @@ static int ttm_bo_cleanup_refs(struct ttm_buffer_object *bo, ret = 0; } - if (ret || unlikely(list_empty(&bo->ddestroy))) { + if (ret) { if (unlock_resv) dma_resv_unlock(bo->base.resv); spin_unlock(&bo->bdev->lru_lock); return ret; } - list_del_init(&bo->ddestroy); spin_unlock(&bo->bdev->lru_lock); ttm_bo_cleanup_memtype_use(bo); @@ -303,47 +301,21 @@ static int ttm_bo_cleanup_refs(struct ttm_buffer_object *bo, } /* - * Traverse the delayed list, and call ttm_bo_cleanup_refs on all - * encountered buffers. + * Block for the dma_resv object to become idle, lock the buffer and clean up + * the resource and tt object. */ -bool ttm_bo_delayed_delete(struct ttm_device *bdev, bool remove_all) +static void ttm_bo_delayed_delete(struct work_struct *work) { - struct list_head removed; - bool empty; - - INIT_LIST_HEAD(&removed); - - spin_lock(&bdev->lru_lock); - while (!list_empty(&bdev->ddestroy)) { - struct ttm_buffer_object *bo; - - bo = list_first_entry(&bdev->ddestroy, struct ttm_buffer_object, - ddestroy); - list_move_tail(&bo->ddestroy, &removed); - if (!ttm_bo_get_unless_zero(bo)) - continue; - - if (remove_all || bo->base.resv != &bo->base._resv) { - spin_unlock(&bdev->lru_lock); - dma_resv_lock(bo->base.resv, NULL); - - spin_lock(&bdev->lru_lock); - ttm_bo_cleanup_refs(bo, false, !remove_all, true); + struct ttm_buffer_object *bo; - } else if (dma_resv_trylock(bo->base.resv)) { - ttm_bo_cleanup_refs(bo, false, !remove_all, true); - } else { - spin_unlock(&bdev->lru_lock); - } - - ttm_bo_put(bo); - spin_lock(&bdev->lru_lock); - } - list_splice_tail(&removed, &bdev->ddestroy); - empty = list_empty(&bdev->ddestroy); - spin_unlock(&bdev->lru_lock); + bo = container_of(work, typeof(*bo), delayed_delete); - return empty; + dma_resv_wait_timeout(bo->base.resv, DMA_RESV_USAGE_BOOKKEEP, false, + MAX_SCHEDULE_TIMEOUT); + dma_resv_lock(bo->base.resv, NULL); + ttm_bo_cleanup_memtype_use(bo); + dma_resv_unlock(bo->base.resv); + ttm_bo_put(bo); } static void ttm_bo_release(struct kref *kref) @@ -372,69 +344,58 @@ static void ttm_bo_release(struct kref *kref) drm_vma_offset_remove(bdev->vma_manager, &bo->base.vma_node); ttm_mem_io_free(bdev, bo->resource); - } - if (!dma_resv_test_signaled(bo->base.resv, DMA_RESV_USAGE_BOOKKEEP) || - !dma_resv_trylock(bo->base.resv)) { - /* The BO is not idle, resurrect it for delayed destroy */ - ttm_bo_flush_all_fences(bo); - bo->deleted = true; + if (!dma_resv_test_signaled(bo->base.resv, + DMA_RESV_USAGE_BOOKKEEP) || + !dma_resv_trylock(bo->base.resv)) { + /* The BO is not idle, resurrect it for delayed destroy */ + ttm_bo_flush_all_fences(bo); + bo->deleted = true; - spin_lock(&bo->bdev->lru_lock); - - /* - * Make pinned bos immediately available to - * shrinkers, now that they are queued for - * destruction. - * - * FIXME: QXL is triggering this. Can be removed when the - * driver is fixed. - */ - if (bo->pin_count) { - bo->pin_count = 0; - ttm_resource_move_to_lru_tail(bo->resource); - } + spin_lock(&bo->bdev->lru_lock); - kref_init(&bo->kref); - list_add_tail(&bo->ddestroy, &bdev->ddestroy); - spin_unlock(&bo->bdev->lru_lock); + /* + * Make pinned bos immediately available to + * shrinkers, now that they are queued for + * destruction. + * + * FIXME: QXL is triggering this. Can be removed when the + * driver is fixed. + */ + if (bo->pin_count) { + bo->pin_count = 0; + ttm_resource_move_to_lru_tail(bo->resource); + } - schedule_delayed_work(&bdev->wq, - ((HZ / 100) < 1) ? 1 : HZ / 100); - return; - } + kref_init(&bo->kref); + spin_unlock(&bo->bdev->lru_lock); - spin_lock(&bo->bdev->lru_lock); - list_del(&bo->ddestroy); - spin_unlock(&bo->bdev->lru_lock); + INIT_WORK(&bo->delayed_delete, ttm_bo_delayed_delete); + queue_work(bdev->wq, &bo->delayed_delete); + return; + } - ttm_bo_cleanup_memtype_use(bo); - dma_resv_unlock(bo->base.resv); + ttm_bo_cleanup_memtype_use(bo); + dma_resv_unlock(bo->base.resv); + } atomic_dec(&ttm_glob.bo_count); bo->destroy(bo); } +/** + * ttm_bo_put + * + * @bo: The buffer object. + * + * Unreference a buffer object. + */ void ttm_bo_put(struct ttm_buffer_object *bo) { kref_put(&bo->kref, ttm_bo_release); } EXPORT_SYMBOL(ttm_bo_put); -int ttm_bo_lock_delayed_workqueue(struct ttm_device *bdev) -{ - return cancel_delayed_work_sync(&bdev->wq); -} -EXPORT_SYMBOL(ttm_bo_lock_delayed_workqueue); - -void ttm_bo_unlock_delayed_workqueue(struct ttm_device *bdev, int resched) -{ - if (resched) - schedule_delayed_work(&bdev->wq, - ((HZ / 100) < 1) ? 1 : HZ / 100); -} -EXPORT_SYMBOL(ttm_bo_unlock_delayed_workqueue); - static int ttm_bo_bounce_temp_buffer(struct ttm_buffer_object *bo, struct ttm_resource **mem, struct ttm_operation_ctx *ctx, @@ -478,7 +439,7 @@ static int ttm_bo_evict(struct ttm_buffer_object *bo, bdev->funcs->evict_flags(bo, &placement); if (!placement.num_placement && !placement.num_busy_placement) { - ret = ttm_bo_wait(bo, true, false); + ret = ttm_bo_wait_ctx(bo, ctx); if (ret) return ret; @@ -515,6 +476,14 @@ out: return ret; } +/** + * ttm_bo_eviction_valuable + * + * @bo: The buffer object to evict + * @place: the placement we need to make room for + * + * Check if it is valuable to evict the BO to make room for the given placement. + */ bool ttm_bo_eviction_valuable(struct ttm_buffer_object *bo, const struct ttm_place *place) { @@ -774,13 +743,23 @@ static int ttm_bo_mem_force_space(struct ttm_buffer_object *bo, return ttm_bo_add_move_fence(bo, man, *mem, ctx->no_wait_gpu); } -/* - * Creates space for memory region @mem according to its type. +/** + * ttm_bo_mem_space * - * This function first searches for free space in compatible memory types in - * the priority order defined by the driver. If free space isn't found, then - * ttm_bo_mem_force_space is attempted in priority order to evict and find - * space. + * @bo: Pointer to a struct ttm_buffer_object. the data of which + * we want to allocate space for. + * @proposed_placement: Proposed new placement for the buffer object. + * @mem: A struct ttm_resource. + * @ctx: if and how to sleep, lock buffers and alloc memory + * + * Allocate memory space for the buffer object pointed to by @bo, using + * the placement flags in @placement, potentially evicting other idle buffer objects. + * This function may sleep while waiting for space to become available. + * Returns: + * -EBUSY: No space available (only if no_wait == 1). + * -ENOMEM: Could not allocate memory for the buffer object, either due to + * fragmentation or concurrent allocators. + * -ERESTARTSYS: An interruptible sleep was interrupted by a signal. */ int ttm_bo_mem_space(struct ttm_buffer_object *bo, struct ttm_placement *placement, @@ -886,6 +865,21 @@ out: return ret; } +/** + * ttm_bo_validate + * + * @bo: The buffer object. + * @placement: Proposed placement for the buffer object. + * @ctx: validation parameters. + * + * Changes placement and caching policy of the buffer object + * according proposed placement. + * Returns + * -EINVAL on invalid proposed placement. + * -ENOMEM on out-of-memory condition. + * -EBUSY if no_wait is true and buffer busy. + * -ERESTARTSYS if interrupted by a signal. + */ int ttm_bo_validate(struct ttm_buffer_object *bo, struct ttm_placement *placement, struct ttm_operation_ctx *ctx) @@ -963,7 +957,6 @@ int ttm_bo_init_reserved(struct ttm_device *bdev, struct ttm_buffer_object *bo, int ret; kref_init(&bo->kref); - INIT_LIST_HEAD(&bo->ddestroy); bo->bdev = bdev; bo->type = type; bo->page_alignment = alignment; @@ -1079,6 +1072,11 @@ EXPORT_SYMBOL(ttm_bo_init_validate); * buffer object vm functions. */ +/** + * ttm_bo_unmap_virtual + * + * @bo: tear down the virtual mappings for this BO + */ void ttm_bo_unmap_virtual(struct ttm_buffer_object *bo) { struct ttm_device *bdev = bo->bdev; @@ -1088,36 +1086,44 @@ void ttm_bo_unmap_virtual(struct ttm_buffer_object *bo) } EXPORT_SYMBOL(ttm_bo_unmap_virtual); -int ttm_bo_wait(struct ttm_buffer_object *bo, - bool interruptible, bool no_wait) +/** + * ttm_bo_wait_ctx - wait for buffer idle. + * + * @bo: The buffer object. + * @ctx: defines how to wait + * + * Waits for the buffer to be idle. Used timeout depends on the context. + * Returns -EBUSY if wait timed outt, -ERESTARTSYS if interrupted by a signal or + * zero on success. + */ +int ttm_bo_wait_ctx(struct ttm_buffer_object *bo, struct ttm_operation_ctx *ctx) { - long timeout = 15 * HZ; + long ret; - if (no_wait) { - if (dma_resv_test_signaled(bo->base.resv, DMA_RESV_USAGE_BOOKKEEP)) + if (ctx->no_wait_gpu) { + if (dma_resv_test_signaled(bo->base.resv, + DMA_RESV_USAGE_BOOKKEEP)) return 0; else return -EBUSY; } - timeout = dma_resv_wait_timeout(bo->base.resv, DMA_RESV_USAGE_BOOKKEEP, - interruptible, timeout); - if (timeout < 0) - return timeout; - - if (timeout == 0) + ret = dma_resv_wait_timeout(bo->base.resv, DMA_RESV_USAGE_BOOKKEEP, + ctx->interruptible, 15 * HZ); + if (unlikely(ret < 0)) + return ret; + if (unlikely(ret == 0)) return -EBUSY; - return 0; } -EXPORT_SYMBOL(ttm_bo_wait); +EXPORT_SYMBOL(ttm_bo_wait_ctx); int ttm_bo_swapout(struct ttm_buffer_object *bo, struct ttm_operation_ctx *ctx, gfp_t gfp_flags) { struct ttm_place place; bool locked; - int ret; + long ret; /* * While the bo may already reside in SYSTEM placement, set @@ -1172,7 +1178,7 @@ int ttm_bo_swapout(struct ttm_buffer_object *bo, struct ttm_operation_ctx *ctx, /* * Make sure BO is idle. */ - ret = ttm_bo_wait(bo, false, false); + ret = ttm_bo_wait_ctx(bo, ctx); if (unlikely(ret != 0)) goto out; diff --git a/drivers/gpu/drm/ttm/ttm_bo_util.c b/drivers/gpu/drm/ttm/ttm_bo_util.c index fa04e62202c1..7635d7d6b13b 100644 --- a/drivers/gpu/drm/ttm/ttm_bo_util.c +++ b/drivers/gpu/drm/ttm/ttm_bo_util.c @@ -29,18 +29,13 @@ * Authors: Thomas Hellstrom <thellstrom-at-vmware-dot-com> */ -#include <drm/ttm/ttm_bo_driver.h> +#include <linux/vmalloc.h> + +#include <drm/ttm/ttm_bo.h> #include <drm/ttm/ttm_placement.h> +#include <drm/ttm/ttm_tt.h> + #include <drm/drm_cache.h> -#include <drm/drm_vma_manager.h> -#include <linux/iosys-map.h> -#include <linux/io.h> -#include <linux/highmem.h> -#include <linux/wait.h> -#include <linux/slab.h> -#include <linux/vmalloc.h> -#include <linux/module.h> -#include <linux/dma-resv.h> struct ttm_transfer_obj { struct ttm_buffer_object base; @@ -128,6 +123,22 @@ void ttm_move_memcpy(bool clear, } EXPORT_SYMBOL(ttm_move_memcpy); +/** + * ttm_bo_move_memcpy + * + * @bo: A pointer to a struct ttm_buffer_object. + * @ctx: operation context + * @dst_mem: struct ttm_resource indicating where to move. + * + * Fallback move function for a mappable buffer object in mappable memory. + * The function will, if successful, + * free any old aperture space, and set (@new_mem)->mm_node to NULL, + * and update the (@bo)->mem placement flags. If unsuccessful, the old + * data remains untouched, and it's up to the caller to free the + * memory space indicated by @new_mem. + * Returns: + * !0: Failure. + */ int ttm_bo_move_memcpy(struct ttm_buffer_object *bo, struct ttm_operation_ctx *ctx, struct ttm_resource *dst_mem) @@ -173,7 +184,7 @@ int ttm_bo_move_memcpy(struct ttm_buffer_object *bo, clear = src_iter->ops->maps_tt && (!ttm || !ttm_tt_is_populated(ttm)); if (!(clear && ttm && !(ttm->page_flags & TTM_TT_FLAG_ZERO_ALLOC))) - ttm_move_memcpy(clear, dst_mem->num_pages, dst_iter, src_iter); + ttm_move_memcpy(clear, PFN_UP(dst_mem->size), dst_iter, src_iter); if (!src_iter->ops->maps_tt) ttm_kmap_iter_linear_io_fini(&_src_iter.io, bdev, src_mem); @@ -230,7 +241,6 @@ static int ttm_buffer_object_transfer(struct ttm_buffer_object *bo, */ atomic_inc(&ttm_glob.bo_count); - INIT_LIST_HEAD(&fbo->base.ddestroy); drm_vma_node_reset(&fbo->base.base.vma_node); kref_init(&fbo->base.kref); @@ -267,6 +277,16 @@ static int ttm_buffer_object_transfer(struct ttm_buffer_object *bo, return 0; } +/** + * ttm_io_prot + * + * @bo: ttm buffer object + * @res: ttm resource object + * @tmp: Page protection flag for a normal, cached mapping. + * + * Utility function that returns the pgprot_t that should be used for + * setting up a PTE with the caching model indicated by @c_state. + */ pgprot_t ttm_io_prot(struct ttm_buffer_object *bo, struct ttm_resource *res, pgprot_t tmp) { @@ -348,6 +368,22 @@ static int ttm_bo_kmap_ttm(struct ttm_buffer_object *bo, return (!map->virtual) ? -ENOMEM : 0; } +/** + * ttm_bo_kmap + * + * @bo: The buffer object. + * @start_page: The first page to map. + * @num_pages: Number of pages to map. + * @map: pointer to a struct ttm_bo_kmap_obj representing the map. + * + * Sets up a kernel virtual mapping, using ioremap, vmap or kmap to the + * data in the buffer object. The ttm_kmap_obj_virtual function can then be + * used to obtain a virtual address to the data. + * + * Returns + * -ENOMEM: Out of memory. + * -EINVAL: Invalid range. + */ int ttm_bo_kmap(struct ttm_buffer_object *bo, unsigned long start_page, unsigned long num_pages, struct ttm_bo_kmap_obj *map) @@ -357,9 +393,9 @@ int ttm_bo_kmap(struct ttm_buffer_object *bo, map->virtual = NULL; map->bo = bo; - if (num_pages > bo->resource->num_pages) + if (num_pages > PFN_UP(bo->resource->size)) return -EINVAL; - if ((start_page + num_pages) > bo->resource->num_pages) + if ((start_page + num_pages) > PFN_UP(bo->resource->size)) return -EINVAL; ret = ttm_mem_io_reserve(bo->bdev, bo->resource); @@ -375,6 +411,13 @@ int ttm_bo_kmap(struct ttm_buffer_object *bo, } EXPORT_SYMBOL(ttm_bo_kmap); +/** + * ttm_bo_kunmap + * + * @map: Object describing the map to unmap. + * + * Unmaps a kernel map set up by ttm_bo_kmap. + */ void ttm_bo_kunmap(struct ttm_bo_kmap_obj *map) { if (!map->virtual) @@ -400,6 +443,20 @@ void ttm_bo_kunmap(struct ttm_bo_kmap_obj *map) } EXPORT_SYMBOL(ttm_bo_kunmap); +/** + * ttm_bo_vmap + * + * @bo: The buffer object. + * @map: pointer to a struct iosys_map representing the map. + * + * Sets up a kernel virtual mapping, using ioremap or vmap to the + * data in the buffer object. The parameter @map returns the virtual + * address as struct iosys_map. Unmap the buffer with ttm_bo_vunmap(). + * + * Returns + * -ENOMEM: Out of memory. + * -EINVAL: Invalid range. + */ int ttm_bo_vmap(struct ttm_buffer_object *bo, struct iosys_map *map) { struct ttm_resource *mem = bo->resource; @@ -461,6 +518,14 @@ int ttm_bo_vmap(struct ttm_buffer_object *bo, struct iosys_map *map) } EXPORT_SYMBOL(ttm_bo_vmap); +/** + * ttm_bo_vunmap + * + * @bo: The buffer object. + * @map: Object describing the map to unmap. + * + * Unmaps a kernel map set up by ttm_bo_vmap(). + */ void ttm_bo_vunmap(struct ttm_buffer_object *bo, struct iosys_map *map) { struct ttm_resource *mem = bo->resource; @@ -483,9 +548,13 @@ EXPORT_SYMBOL(ttm_bo_vunmap); static int ttm_bo_wait_free_node(struct ttm_buffer_object *bo, bool dst_use_tt) { - int ret; - ret = ttm_bo_wait(bo, false, false); - if (ret) + long ret; + + ret = dma_resv_wait_timeout(bo->base.resv, DMA_RESV_USAGE_BOOKKEEP, + false, 15 * HZ); + if (ret == 0) + return -EBUSY; + if (ret < 0) return ret; if (!dst_use_tt) @@ -554,6 +623,22 @@ static void ttm_bo_move_pipeline_evict(struct ttm_buffer_object *bo, ttm_resource_free(bo, &bo->resource); } +/** + * ttm_bo_move_accel_cleanup - cleanup helper for hw copies + * + * @bo: A pointer to a struct ttm_buffer_object. + * @fence: A fence object that signals when moving is complete. + * @evict: This is an evict move. Don't return until the buffer is idle. + * @pipeline: evictions are to be pipelined. + * @new_mem: struct ttm_resource indicating where to move. + * + * Accelerated move function to be called when an accelerated move + * has been scheduled. The function will create a new temporary buffer object + * representing the old placement, and put the sync object on both buffer + * objects. After that the newly created buffer object is unref'd to be + * destroyed when the move is complete. This will help pipeline + * buffer moves. + */ int ttm_bo_move_accel_cleanup(struct ttm_buffer_object *bo, struct dma_fence *fence, bool evict, @@ -582,6 +667,15 @@ int ttm_bo_move_accel_cleanup(struct ttm_buffer_object *bo, } EXPORT_SYMBOL(ttm_bo_move_accel_cleanup); +/** + * ttm_bo_move_sync_cleanup - cleanup by waiting for the move to finish + * + * @bo: A pointer to a struct ttm_buffer_object. + * @new_mem: struct ttm_resource indicating where to move. + * + * Special case of ttm_bo_move_accel_cleanup where the bo is guaranteed + * by the caller to be idle. Typically used after memcpy buffer moves. + */ void ttm_bo_move_sync_cleanup(struct ttm_buffer_object *bo, struct ttm_resource *new_mem) { @@ -621,8 +715,7 @@ int ttm_bo_pipeline_gutting(struct ttm_buffer_object *bo) return ret; /* If already idle, no need for ghost object dance. */ - ret = ttm_bo_wait(bo, false, true); - if (ret != -EBUSY) { + if (dma_resv_test_signaled(bo->base.resv, DMA_RESV_USAGE_BOOKKEEP)) { if (!bo->ttm) { /* See comment below about clearing. */ ret = ttm_tt_create(bo, true); @@ -659,8 +752,10 @@ int ttm_bo_pipeline_gutting(struct ttm_buffer_object *bo) ret = dma_resv_copy_fences(&ghost->base._resv, bo->base.resv); /* Last resort, wait for the BO to be idle when we are OOM */ - if (ret) - ttm_bo_wait(bo, false, false); + if (ret) { + dma_resv_wait_timeout(bo->base.resv, DMA_RESV_USAGE_BOOKKEEP, + false, MAX_SCHEDULE_TIMEOUT); + } dma_resv_unlock(&ghost->base._resv); ttm_bo_put(ghost); diff --git a/drivers/gpu/drm/ttm/ttm_bo_vm.c b/drivers/gpu/drm/ttm/ttm_bo_vm.c index 38119311284d..3ecda6db24b8 100644 --- a/drivers/gpu/drm/ttm/ttm_bo_vm.c +++ b/drivers/gpu/drm/ttm/ttm_bo_vm.c @@ -31,17 +31,12 @@ #define pr_fmt(fmt) "[TTM] " fmt -#include <drm/ttm/ttm_bo_driver.h> +#include <drm/ttm/ttm_bo.h> #include <drm/ttm/ttm_placement.h> -#include <drm/drm_vma_manager.h> +#include <drm/ttm/ttm_tt.h> + #include <drm/drm_drv.h> #include <drm/drm_managed.h> -#include <linux/mm.h> -#include <linux/pfn_t.h> -#include <linux/rbtree.h> -#include <linux/module.h> -#include <linux/uaccess.h> -#include <linux/mem_encrypt.h> static vm_fault_t ttm_bo_vm_fault_idle(struct ttm_buffer_object *bo, struct vm_fault *vmf) @@ -217,7 +212,7 @@ vm_fault_t ttm_bo_vm_fault_reserved(struct vm_fault *vmf, page_last = vma_pages(vma) + vma->vm_pgoff - drm_vma_node_start(&bo->base.vma_node); - if (unlikely(page_offset >= bo->resource->num_pages)) + if (unlikely(page_offset >= PFN_UP(bo->base.size))) return VM_FAULT_SIGBUS; prot = ttm_io_prot(bo, bo->resource, prot); @@ -412,7 +407,7 @@ int ttm_bo_vm_access(struct vm_area_struct *vma, unsigned long addr, << PAGE_SHIFT); int ret; - if (len < 1 || (offset + len) >> PAGE_SHIFT > bo->resource->num_pages) + if (len < 1 || (offset + len) > bo->base.size) return -EIO; ret = ttm_bo_reserve(bo, true, false, NULL); @@ -446,6 +441,14 @@ static const struct vm_operations_struct ttm_bo_vm_ops = { .access = ttm_bo_vm_access, }; +/** + * ttm_bo_mmap_obj - mmap memory backed by a ttm buffer object. + * + * @vma: vma as input from the fbdev mmap method. + * @bo: The bo backing the address space. + * + * Maps a buffer object. + */ int ttm_bo_mmap_obj(struct vm_area_struct *vma, struct ttm_buffer_object *bo) { /* Enforce no COW since would have really strange behavior with it. */ diff --git a/drivers/gpu/drm/ttm/ttm_device.c b/drivers/gpu/drm/ttm/ttm_device.c index e7147e304637..c7a1862f322a 100644 --- a/drivers/gpu/drm/ttm/ttm_device.c +++ b/drivers/gpu/drm/ttm/ttm_device.c @@ -29,10 +29,10 @@ #include <linux/mm.h> +#include <drm/ttm/ttm_bo.h> #include <drm/ttm/ttm_device.h> #include <drm/ttm/ttm_tt.h> #include <drm/ttm/ttm_placement.h> -#include <drm/ttm/ttm_bo_api.h> #include "ttm_module.h" @@ -175,16 +175,6 @@ int ttm_device_swapout(struct ttm_device *bdev, struct ttm_operation_ctx *ctx, } EXPORT_SYMBOL(ttm_device_swapout); -static void ttm_device_delayed_workqueue(struct work_struct *work) -{ - struct ttm_device *bdev = - container_of(work, struct ttm_device, wq.work); - - if (!ttm_bo_delayed_delete(bdev, false)) - schedule_delayed_work(&bdev->wq, - ((HZ / 100) < 1) ? 1 : HZ / 100); -} - /** * ttm_device_init * @@ -215,15 +205,19 @@ int ttm_device_init(struct ttm_device *bdev, struct ttm_device_funcs *funcs, if (ret) return ret; + bdev->wq = alloc_workqueue("ttm", WQ_MEM_RECLAIM | WQ_HIGHPRI, 16); + if (!bdev->wq) { + ttm_global_release(); + return -ENOMEM; + } + bdev->funcs = funcs; ttm_sys_man_init(bdev); ttm_pool_init(&bdev->pool, dev, use_dma_alloc, use_dma32); bdev->vma_manager = vma_manager; - INIT_DELAYED_WORK(&bdev->wq, ttm_device_delayed_workqueue); spin_lock_init(&bdev->lru_lock); - INIT_LIST_HEAD(&bdev->ddestroy); INIT_LIST_HEAD(&bdev->pinned); bdev->dev_mapping = mapping; mutex_lock(&ttm_global_mutex); @@ -247,10 +241,8 @@ void ttm_device_fini(struct ttm_device *bdev) list_del(&bdev->device_list); mutex_unlock(&ttm_global_mutex); - cancel_delayed_work_sync(&bdev->wq); - - if (ttm_bo_delayed_delete(bdev, true)) - pr_debug("Delayed destroy list was clean\n"); + drain_workqueue(bdev->wq); + destroy_workqueue(bdev->wq); spin_lock(&bdev->lru_lock); for (i = 0; i < TTM_MAX_BO_PRIORITY; ++i) diff --git a/drivers/gpu/drm/ttm/ttm_execbuf_util.c b/drivers/gpu/drm/ttm/ttm_execbuf_util.c index dbee34a058df..f1c60fa80c2d 100644 --- a/drivers/gpu/drm/ttm/ttm_execbuf_util.c +++ b/drivers/gpu/drm/ttm/ttm_execbuf_util.c @@ -27,11 +27,7 @@ **************************************************************************/ #include <drm/ttm/ttm_execbuf_util.h> -#include <drm/ttm/ttm_bo_driver.h> -#include <drm/ttm/ttm_placement.h> -#include <linux/wait.h> -#include <linux/sched.h> -#include <linux/module.h> +#include <drm/ttm/ttm_bo.h> static void ttm_eu_backoff_reservation_reverse(struct list_head *list, struct ttm_validate_buffer *entry) diff --git a/drivers/gpu/drm/ttm/ttm_pool.c b/drivers/gpu/drm/ttm/ttm_pool.c index 21b61631f73a..aa116a7bbae3 100644 --- a/drivers/gpu/drm/ttm/ttm_pool.c +++ b/drivers/gpu/drm/ttm/ttm_pool.c @@ -33,6 +33,7 @@ #include <linux/module.h> #include <linux/dma-mapping.h> +#include <linux/debugfs.h> #include <linux/highmem.h> #include <linux/sched/mm.h> @@ -41,8 +42,8 @@ #endif #include <drm/ttm/ttm_pool.h> -#include <drm/ttm/ttm_bo_driver.h> #include <drm/ttm/ttm_tt.h> +#include <drm/ttm/ttm_bo.h> #include "ttm_module.h" @@ -344,6 +345,28 @@ static unsigned int ttm_pool_page_order(struct ttm_pool *pool, struct page *p) return p->private; } +/* Called when we got a page, either from a pool or newly allocated */ +static int ttm_pool_page_allocated(struct ttm_pool *pool, unsigned int order, + struct page *p, dma_addr_t **dma_addr, + unsigned long *num_pages, + struct page ***pages) +{ + unsigned int i; + int r; + + if (*dma_addr) { + r = ttm_pool_map(pool, order, p, dma_addr); + if (r) + return r; + } + + *num_pages -= 1 << order; + for (i = 1 << order; i; --i, ++(*pages), ++p) + **pages = p; + + return 0; +} + /** * ttm_pool_alloc - Fill a ttm_tt object * @@ -385,45 +408,57 @@ int ttm_pool_alloc(struct ttm_pool *pool, struct ttm_tt *tt, for (order = min_t(unsigned int, MAX_ORDER - 1, __fls(num_pages)); num_pages; order = min_t(unsigned int, order, __fls(num_pages))) { - bool apply_caching = false; struct ttm_pool_type *pt; pt = ttm_pool_select_type(pool, tt->caching, order); p = pt ? ttm_pool_type_take(pt) : NULL; if (p) { - apply_caching = true; - } else { - p = ttm_pool_alloc_page(pool, gfp_flags, order); - if (p && PageHighMem(p)) - apply_caching = true; - } - - if (!p) { - if (order) { - --order; - continue; - } - r = -ENOMEM; - goto error_free_all; - } - - if (apply_caching) { r = ttm_pool_apply_caching(caching, pages, tt->caching); if (r) goto error_free_page; - caching = pages + (1 << order); + + do { + r = ttm_pool_page_allocated(pool, order, p, + &dma_addr, + &num_pages, + &pages); + if (r) + goto error_free_page; + + if (num_pages < (1 << order)) + break; + + p = ttm_pool_type_take(pt); + } while (p); + caching = pages; } - if (dma_addr) { - r = ttm_pool_map(pool, order, p, &dma_addr); + while (num_pages >= (1 << order) && + (p = ttm_pool_alloc_page(pool, gfp_flags, order))) { + + if (PageHighMem(p)) { + r = ttm_pool_apply_caching(caching, pages, + tt->caching); + if (r) + goto error_free_page; + } + r = ttm_pool_page_allocated(pool, order, p, &dma_addr, + &num_pages, &pages); if (r) goto error_free_page; + if (PageHighMem(p)) + caching = pages; } - num_pages -= 1 << order; - for (i = 1 << order; i; --i) - *(pages++) = p++; + if (!p) { + if (order) { + --order; + continue; + } + r = -ENOMEM; + goto error_free_all; + } } r = ttm_pool_apply_caching(caching, pages, tt->caching); diff --git a/drivers/gpu/drm/ttm/ttm_range_manager.c b/drivers/gpu/drm/ttm/ttm_range_manager.c index 4cfef2b3514d..ae11d07eb63a 100644 --- a/drivers/gpu/drm/ttm/ttm_range_manager.c +++ b/drivers/gpu/drm/ttm/ttm_range_manager.c @@ -32,7 +32,7 @@ #include <drm/ttm/ttm_device.h> #include <drm/ttm/ttm_placement.h> #include <drm/ttm/ttm_range_manager.h> -#include <drm/ttm/ttm_bo_api.h> +#include <drm/ttm/ttm_bo.h> #include <drm/drm_mm.h> #include <linux/slab.h> #include <linux/spinlock.h> @@ -83,7 +83,7 @@ static int ttm_range_man_alloc(struct ttm_resource_manager *man, spin_lock(&rman->lock); ret = drm_mm_insert_node_in_range(mm, &node->mm_nodes[0], - node->base.num_pages, + PFN_UP(node->base.size), bo->page_alignment, 0, place->fpfn, lpfn, mode); spin_unlock(&rman->lock); @@ -229,7 +229,6 @@ int ttm_range_man_fini_nocheck(struct ttm_device *bdev, return ret; spin_lock(&rman->lock); - drm_mm_clean(mm); drm_mm_takedown(mm); spin_unlock(&rman->lock); diff --git a/drivers/gpu/drm/ttm/ttm_resource.c b/drivers/gpu/drm/ttm/ttm_resource.c index a729c32a1e48..b8a826a24fb2 100644 --- a/drivers/gpu/drm/ttm/ttm_resource.c +++ b/drivers/gpu/drm/ttm/ttm_resource.c @@ -26,8 +26,9 @@ #include <linux/io-mapping.h> #include <linux/scatterlist.h> +#include <drm/ttm/ttm_bo.h> +#include <drm/ttm/ttm_placement.h> #include <drm/ttm/ttm_resource.h> -#include <drm/ttm/ttm_bo_driver.h> /** * ttm_lru_bulk_move_init - initialize a bulk move structure @@ -177,7 +178,7 @@ void ttm_resource_init(struct ttm_buffer_object *bo, struct ttm_resource_manager *man; res->start = 0; - res->num_pages = PFN_UP(bo->base.size); + res->size = bo->base.size; res->mem_type = place->mem_type; res->placement = place->flags; res->bus.addr = NULL; @@ -192,7 +193,7 @@ void ttm_resource_init(struct ttm_buffer_object *bo, list_add_tail(&res->lru, &bo->bdev->pinned); else list_add_tail(&res->lru, &man->lru[bo->priority]); - man->usage += res->num_pages << PAGE_SHIFT; + man->usage += res->size; spin_unlock(&bo->bdev->lru_lock); } EXPORT_SYMBOL(ttm_resource_init); @@ -214,7 +215,7 @@ void ttm_resource_fini(struct ttm_resource_manager *man, spin_lock(&bdev->lru_lock); list_del_init(&res->lru); - man->usage -= res->num_pages << PAGE_SHIFT; + man->usage -= res->size; spin_unlock(&bdev->lru_lock); } EXPORT_SYMBOL(ttm_resource_fini); @@ -665,17 +666,15 @@ ttm_kmap_iter_linear_io_init(struct ttm_kmap_iter_linear_io *iter_io, iosys_map_set_vaddr(&iter_io->dmap, mem->bus.addr); iter_io->needs_unmap = false; } else { - size_t bus_size = (size_t)mem->num_pages << PAGE_SHIFT; - iter_io->needs_unmap = true; memset(&iter_io->dmap, 0, sizeof(iter_io->dmap)); if (mem->bus.caching == ttm_write_combined) iosys_map_set_vaddr_iomem(&iter_io->dmap, ioremap_wc(mem->bus.offset, - bus_size)); + mem->size)); else if (mem->bus.caching == ttm_cached) iosys_map_set_vaddr(&iter_io->dmap, - memremap(mem->bus.offset, bus_size, + memremap(mem->bus.offset, mem->size, MEMREMAP_WB | MEMREMAP_WT | MEMREMAP_WC)); @@ -684,7 +683,7 @@ ttm_kmap_iter_linear_io_init(struct ttm_kmap_iter_linear_io *iter_io, if (iosys_map_is_null(&iter_io->dmap)) iosys_map_set_vaddr_iomem(&iter_io->dmap, ioremap(mem->bus.offset, - bus_size)); + mem->size)); if (iosys_map_is_null(&iter_io->dmap)) { ret = -ENOMEM; diff --git a/drivers/gpu/drm/ttm/ttm_tt.c b/drivers/gpu/drm/ttm/ttm_tt.c index d505603930a7..ab725d9d14a6 100644 --- a/drivers/gpu/drm/ttm/ttm_tt.c +++ b/drivers/gpu/drm/ttm/ttm_tt.c @@ -36,7 +36,8 @@ #include <linux/file.h> #include <linux/module.h> #include <drm/drm_cache.h> -#include <drm/ttm/ttm_bo_driver.h> +#include <drm/ttm/ttm_bo.h> +#include <drm/ttm/ttm_tt.h> #include "ttm_module.h" |