diff options
Diffstat (limited to 'drivers/gpu/drm/ttm')
| -rw-r--r-- | drivers/gpu/drm/ttm/ttm_bo.c | 284 | ||||
| -rw-r--r-- | drivers/gpu/drm/ttm/ttm_bo_util.c | 10 | ||||
| -rw-r--r-- | drivers/gpu/drm/ttm/ttm_execbuf_util.c | 8 | ||||
| -rw-r--r-- | drivers/gpu/drm/ttm/ttm_page_alloc.c | 137 | ||||
| -rw-r--r-- | drivers/gpu/drm/ttm/ttm_page_alloc_dma.c | 5 |
5 files changed, 239 insertions, 205 deletions
diff --git a/drivers/gpu/drm/ttm/ttm_bo.c b/drivers/gpu/drm/ttm/ttm_bo.c index c088703777e2..60bb5c12b568 100644 --- a/drivers/gpu/drm/ttm/ttm_bo.c +++ b/drivers/gpu/drm/ttm/ttm_bo.c @@ -42,10 +42,6 @@ #include <linux/atomic.h> #include <linux/reservation.h> -#define TTM_ASSERT_LOCKED(param) -#define TTM_DEBUG(fmt, arg...) -#define TTM_BO_HASH_ORDER 13 - static int ttm_bo_swapout(struct ttm_mem_shrink *shrink); static void ttm_bo_global_kobj_release(struct kobject *kobj); @@ -165,7 +161,7 @@ void ttm_bo_add_to_lru(struct ttm_buffer_object *bo) struct ttm_bo_device *bdev = bo->bdev; struct ttm_mem_type_manager *man; - lockdep_assert_held(&bo->resv->lock.base); + reservation_object_assert_held(bo->resv); if (!(bo->mem.placement & TTM_PL_FLAG_NO_EVICT)) { @@ -216,7 +212,7 @@ EXPORT_SYMBOL(ttm_bo_del_sub_from_lru); void ttm_bo_move_to_lru_tail(struct ttm_buffer_object *bo) { - lockdep_assert_held(&bo->resv->lock.base); + reservation_object_assert_held(bo->resv); ttm_bo_del_from_lru(bo); ttm_bo_add_to_lru(bo); @@ -233,7 +229,7 @@ static int ttm_bo_add_ttm(struct ttm_buffer_object *bo, bool zero_alloc) int ret = 0; uint32_t page_flags = 0; - TTM_ASSERT_LOCKED(&bo->mutex); + reservation_object_assert_held(bo->resv); bo->ttm = NULL; if (bdev->need_dma32) @@ -269,9 +265,8 @@ static int ttm_bo_add_ttm(struct ttm_buffer_object *bo, bool zero_alloc) } static int ttm_bo_handle_move_mem(struct ttm_buffer_object *bo, - struct ttm_mem_reg *mem, - bool evict, bool interruptible, - bool no_wait_gpu) + struct ttm_mem_reg *mem, bool evict, + struct ttm_operation_ctx *ctx) { struct ttm_bo_device *bdev = bo->bdev; bool old_is_pci = ttm_mem_reg_is_pci(bdev, &bo->mem); @@ -325,12 +320,11 @@ static int ttm_bo_handle_move_mem(struct ttm_buffer_object *bo, if (!(old_man->flags & TTM_MEMTYPE_FLAG_FIXED) && !(new_man->flags & TTM_MEMTYPE_FLAG_FIXED)) - ret = ttm_bo_move_ttm(bo, interruptible, no_wait_gpu, mem); + ret = ttm_bo_move_ttm(bo, ctx, mem); else if (bdev->driver->move) - ret = bdev->driver->move(bo, evict, interruptible, - no_wait_gpu, mem); + ret = bdev->driver->move(bo, evict, ctx, mem); else - ret = ttm_bo_move_memcpy(bo, interruptible, no_wait_gpu, mem); + ret = ttm_bo_move_memcpy(bo, ctx, mem); if (ret) { if (bdev->driver->move_notify) { @@ -355,13 +349,13 @@ moved: bo->evicted = false; } - if (bo->mem.mm_node) { + if (bo->mem.mm_node) bo->offset = (bo->mem.start << PAGE_SHIFT) + bdev->man[bo->mem.mem_type].gpu_offset; - bo->cur_placement = bo->mem.placement; - } else + else bo->offset = 0; + ctx->bytes_moved += bo->num_pages << PAGE_SHIFT; return 0; out_err: @@ -390,8 +384,6 @@ static void ttm_bo_cleanup_memtype_use(struct ttm_buffer_object *bo) ttm_tt_destroy(bo->ttm); bo->ttm = NULL; ttm_bo_mem_put(bo, &bo->mem); - - ww_mutex_unlock (&bo->resv->lock); } static int ttm_bo_individualize_resv(struct ttm_buffer_object *bo) @@ -448,7 +440,7 @@ static void ttm_bo_cleanup_refs_or_queue(struct ttm_buffer_object *bo) } spin_lock(&glob->lru_lock); - ret = __ttm_bo_reserve(bo, false, true, NULL); + ret = reservation_object_trylock(bo->resv) ? 0 : -EBUSY; if (!ret) { if (reservation_object_test_signaled_rcu(&bo->ttm_resv, true)) { ttm_bo_del_from_lru(bo); @@ -457,6 +449,7 @@ static void ttm_bo_cleanup_refs_or_queue(struct ttm_buffer_object *bo) reservation_object_unlock(&bo->ttm_resv); ttm_bo_cleanup_memtype_use(bo); + reservation_object_unlock(bo->resv); return; } @@ -472,7 +465,7 @@ static void ttm_bo_cleanup_refs_or_queue(struct ttm_buffer_object *bo) ttm_bo_add_to_lru(bo); } - __ttm_bo_unreserve(bo); + reservation_object_unlock(bo->resv); } if (bo->resv != &bo->ttm_resv) reservation_object_unlock(&bo->ttm_resv); @@ -487,20 +480,21 @@ error: } /** - * function ttm_bo_cleanup_refs_and_unlock + * function ttm_bo_cleanup_refs * If bo idle, remove from delayed- and lru lists, and unref. * If not idle, do nothing. * * Must be called with lru_lock and reservation held, this function - * will drop both before returning. + * will drop the lru lock and optionally the reservation lock before returning. * * @interruptible Any sleeps should occur interruptibly. * @no_wait_gpu Never wait for gpu. Return -EBUSY instead. + * @unlock_resv Unlock the reservation lock as well. */ -static int ttm_bo_cleanup_refs_and_unlock(struct ttm_buffer_object *bo, - bool interruptible, - bool no_wait_gpu) +static int ttm_bo_cleanup_refs(struct ttm_buffer_object *bo, + bool interruptible, bool no_wait_gpu, + bool unlock_resv) { struct ttm_bo_global *glob = bo->glob; struct reservation_object *resv; @@ -518,7 +512,9 @@ static int ttm_bo_cleanup_refs_and_unlock(struct ttm_buffer_object *bo, if (ret && !no_wait_gpu) { long lret; - ww_mutex_unlock(&bo->resv->lock); + + if (unlock_resv) + reservation_object_unlock(bo->resv); spin_unlock(&glob->lru_lock); lret = reservation_object_wait_timeout_rcu(resv, true, @@ -531,24 +527,24 @@ static int ttm_bo_cleanup_refs_and_unlock(struct ttm_buffer_object *bo, return -EBUSY; spin_lock(&glob->lru_lock); - ret = __ttm_bo_reserve(bo, false, true, NULL); - - /* - * We raced, and lost, someone else holds the reservation now, - * and is probably busy in ttm_bo_cleanup_memtype_use. - * - * Even if it's not the case, because we finished waiting any - * delayed destruction would succeed, so just return success - * here. - */ - if (ret) { + if (unlock_resv && !reservation_object_trylock(bo->resv)) { + /* + * We raced, and lost, someone else holds the reservation now, + * and is probably busy in ttm_bo_cleanup_memtype_use. + * + * Even if it's not the case, because we finished waiting any + * delayed destruction would succeed, so just return success + * here. + */ spin_unlock(&glob->lru_lock); return 0; } + ret = 0; } if (ret || unlikely(list_empty(&bo->ddestroy))) { - __ttm_bo_unreserve(bo); + if (unlock_resv) + reservation_object_unlock(bo->resv); spin_unlock(&glob->lru_lock); return ret; } @@ -560,6 +556,9 @@ static int ttm_bo_cleanup_refs_and_unlock(struct ttm_buffer_object *bo, spin_unlock(&glob->lru_lock); ttm_bo_cleanup_memtype_use(bo); + if (unlock_resv) + reservation_object_unlock(bo->resv); + return 0; } @@ -567,60 +566,42 @@ static int ttm_bo_cleanup_refs_and_unlock(struct ttm_buffer_object *bo, * Traverse the delayed list, and call ttm_bo_cleanup_refs on all * encountered buffers. */ - -static int ttm_bo_delayed_delete(struct ttm_bo_device *bdev, bool remove_all) +static bool ttm_bo_delayed_delete(struct ttm_bo_device *bdev, bool remove_all) { struct ttm_bo_global *glob = bdev->glob; - struct ttm_buffer_object *entry = NULL; - int ret = 0; - - spin_lock(&glob->lru_lock); - if (list_empty(&bdev->ddestroy)) - goto out_unlock; - - entry = list_first_entry(&bdev->ddestroy, - struct ttm_buffer_object, ddestroy); - kref_get(&entry->list_kref); + struct list_head removed; + bool empty; - for (;;) { - struct ttm_buffer_object *nentry = NULL; + INIT_LIST_HEAD(&removed); - if (entry->ddestroy.next != &bdev->ddestroy) { - nentry = list_first_entry(&entry->ddestroy, - struct ttm_buffer_object, ddestroy); - kref_get(&nentry->list_kref); - } + spin_lock(&glob->lru_lock); + while (!list_empty(&bdev->ddestroy)) { + struct ttm_buffer_object *bo; - ret = __ttm_bo_reserve(entry, false, true, NULL); - if (remove_all && ret) { - spin_unlock(&glob->lru_lock); - ret = __ttm_bo_reserve(entry, false, false, NULL); - spin_lock(&glob->lru_lock); - } + bo = list_first_entry(&bdev->ddestroy, struct ttm_buffer_object, + ddestroy); + kref_get(&bo->list_kref); + list_move_tail(&bo->ddestroy, &removed); - if (!ret) - ret = ttm_bo_cleanup_refs_and_unlock(entry, false, - !remove_all); - else + if (remove_all || bo->resv != &bo->ttm_resv) { spin_unlock(&glob->lru_lock); + reservation_object_lock(bo->resv, NULL); - kref_put(&entry->list_kref, ttm_bo_release_list); - entry = nentry; + spin_lock(&glob->lru_lock); + ttm_bo_cleanup_refs(bo, false, !remove_all, true); - if (ret || !entry) - goto out; + } else if (reservation_object_trylock(bo->resv)) { + ttm_bo_cleanup_refs(bo, false, !remove_all, true); + } + kref_put(&bo->list_kref, ttm_bo_release_list); spin_lock(&glob->lru_lock); - if (list_empty(&entry->ddestroy)) - break; } - -out_unlock: + list_splice_tail(&removed, &bdev->ddestroy); + empty = list_empty(&bdev->ddestroy); spin_unlock(&glob->lru_lock); -out: - if (entry) - kref_put(&entry->list_kref, ttm_bo_release_list); - return ret; + + return empty; } static void ttm_bo_delayed_workqueue(struct work_struct *work) @@ -628,7 +609,7 @@ static void ttm_bo_delayed_workqueue(struct work_struct *work) struct ttm_bo_device *bdev = container_of(work, struct ttm_bo_device, wq.work); - if (ttm_bo_delayed_delete(bdev, false)) { + if (!ttm_bo_delayed_delete(bdev, false)) { schedule_delayed_work(&bdev->wq, ((HZ / 100) < 1) ? 1 : HZ / 100); } @@ -672,15 +653,15 @@ void ttm_bo_unlock_delayed_workqueue(struct ttm_bo_device *bdev, int resched) } EXPORT_SYMBOL(ttm_bo_unlock_delayed_workqueue); -static int ttm_bo_evict(struct ttm_buffer_object *bo, bool interruptible, - bool no_wait_gpu) +static int ttm_bo_evict(struct ttm_buffer_object *bo, + struct ttm_operation_ctx *ctx) { struct ttm_bo_device *bdev = bo->bdev; struct ttm_mem_reg evict_mem; struct ttm_placement placement; int ret = 0; - lockdep_assert_held(&bo->resv->lock.base); + reservation_object_assert_held(bo->resv); evict_mem = bo->mem; evict_mem.mm_node = NULL; @@ -690,8 +671,7 @@ static int ttm_bo_evict(struct ttm_buffer_object *bo, bool interruptible, placement.num_placement = 0; placement.num_busy_placement = 0; bdev->driver->evict_flags(bo, &placement); - ret = ttm_bo_mem_space(bo, &placement, &evict_mem, interruptible, - no_wait_gpu); + ret = ttm_bo_mem_space(bo, &placement, &evict_mem, ctx); if (ret) { if (ret != -ERESTARTSYS) { pr_err("Failed to find memory space for buffer 0x%p eviction\n", @@ -701,8 +681,7 @@ static int ttm_bo_evict(struct ttm_buffer_object *bo, bool interruptible, goto out; } - ret = ttm_bo_handle_move_mem(bo, &evict_mem, true, interruptible, - no_wait_gpu); + ret = ttm_bo_handle_move_mem(bo, &evict_mem, true, ctx); if (unlikely(ret)) { if (ret != -ERESTARTSYS) pr_err("Buffer eviction failed\n"); @@ -729,48 +708,57 @@ bool ttm_bo_eviction_valuable(struct ttm_buffer_object *bo, EXPORT_SYMBOL(ttm_bo_eviction_valuable); static int ttm_mem_evict_first(struct ttm_bo_device *bdev, - uint32_t mem_type, - const struct ttm_place *place, - bool interruptible, - bool no_wait_gpu) + uint32_t mem_type, + const struct ttm_place *place, + struct ttm_operation_ctx *ctx) { struct ttm_bo_global *glob = bdev->glob; struct ttm_mem_type_manager *man = &bdev->man[mem_type]; - struct ttm_buffer_object *bo; - int ret = -EBUSY; + struct ttm_buffer_object *bo = NULL; + bool locked = false; unsigned i; + int ret; spin_lock(&glob->lru_lock); for (i = 0; i < TTM_MAX_BO_PRIORITY; ++i) { list_for_each_entry(bo, &man->lru[i], lru) { - ret = __ttm_bo_reserve(bo, false, true, NULL); - if (ret) - continue; + if (bo->resv == ctx->resv) { + if (!ctx->allow_reserved_eviction && + list_empty(&bo->ddestroy)) + continue; + } else { + locked = reservation_object_trylock(bo->resv); + if (!locked) + continue; + } if (place && !bdev->driver->eviction_valuable(bo, place)) { - __ttm_bo_unreserve(bo); - ret = -EBUSY; + if (locked) + reservation_object_unlock(bo->resv); + locked = false; continue; } - break; } - if (!ret) + /* If the inner loop terminated early, we have our candidate */ + if (&bo->lru != &man->lru[i]) break; + + bo = NULL; } - if (ret) { + if (!bo) { spin_unlock(&glob->lru_lock); - return ret; + return -EBUSY; } kref_get(&bo->list_kref); if (!list_empty(&bo->ddestroy)) { - ret = ttm_bo_cleanup_refs_and_unlock(bo, interruptible, - no_wait_gpu); + ret = ttm_bo_cleanup_refs(bo, ctx->interruptible, + ctx->no_wait_gpu, locked); kref_put(&bo->list_kref, ttm_bo_release_list); return ret; } @@ -778,10 +766,14 @@ static int ttm_mem_evict_first(struct ttm_bo_device *bdev, ttm_bo_del_from_lru(bo); spin_unlock(&glob->lru_lock); - BUG_ON(ret != 0); - - ret = ttm_bo_evict(bo, interruptible, no_wait_gpu); - ttm_bo_unreserve(bo); + ret = ttm_bo_evict(bo, ctx); + if (locked) { + ttm_bo_unreserve(bo); + } else { + spin_lock(&glob->lru_lock); + ttm_bo_add_to_lru(bo); + spin_unlock(&glob->lru_lock); + } kref_put(&bo->list_kref, ttm_bo_release_list); return ret; @@ -832,8 +824,7 @@ static int ttm_bo_mem_force_space(struct ttm_buffer_object *bo, uint32_t mem_type, const struct ttm_place *place, struct ttm_mem_reg *mem, - bool interruptible, - bool no_wait_gpu) + struct ttm_operation_ctx *ctx) { struct ttm_bo_device *bdev = bo->bdev; struct ttm_mem_type_manager *man = &bdev->man[mem_type]; @@ -845,8 +836,7 @@ static int ttm_bo_mem_force_space(struct ttm_buffer_object *bo, return ret; if (mem->mm_node) break; - ret = ttm_mem_evict_first(bdev, mem_type, place, - interruptible, no_wait_gpu); + ret = ttm_mem_evict_first(bdev, mem_type, place, ctx); if (unlikely(ret != 0)) return ret; } while (1); @@ -909,8 +899,7 @@ static bool ttm_bo_mt_compatible(struct ttm_mem_type_manager *man, int ttm_bo_mem_space(struct ttm_buffer_object *bo, struct ttm_placement *placement, struct ttm_mem_reg *mem, - bool interruptible, - bool no_wait_gpu) + struct ttm_operation_ctx *ctx) { struct ttm_bo_device *bdev = bo->bdev; struct ttm_mem_type_manager *man; @@ -1004,8 +993,7 @@ int ttm_bo_mem_space(struct ttm_buffer_object *bo, return 0; } - ret = ttm_bo_mem_force_space(bo, mem_type, place, mem, - interruptible, no_wait_gpu); + ret = ttm_bo_mem_force_space(bo, mem_type, place, mem, ctx); if (ret == 0 && mem->mm_node) { mem->placement = cur_flags; return 0; @@ -1024,14 +1012,13 @@ int ttm_bo_mem_space(struct ttm_buffer_object *bo, EXPORT_SYMBOL(ttm_bo_mem_space); static int ttm_bo_move_buffer(struct ttm_buffer_object *bo, - struct ttm_placement *placement, - bool interruptible, - bool no_wait_gpu) + struct ttm_placement *placement, + struct ttm_operation_ctx *ctx) { int ret = 0; struct ttm_mem_reg mem; - lockdep_assert_held(&bo->resv->lock.base); + reservation_object_assert_held(bo->resv); mem.num_pages = bo->num_pages; mem.size = mem.num_pages << PAGE_SHIFT; @@ -1041,12 +1028,10 @@ static int ttm_bo_move_buffer(struct ttm_buffer_object *bo, /* * Determine where to move the buffer. */ - ret = ttm_bo_mem_space(bo, placement, &mem, - interruptible, no_wait_gpu); + ret = ttm_bo_mem_space(bo, placement, &mem, ctx); if (ret) goto out_unlock; - ret = ttm_bo_handle_move_mem(bo, &mem, false, - interruptible, no_wait_gpu); + ret = ttm_bo_handle_move_mem(bo, &mem, false, ctx); out_unlock: if (ret && mem.mm_node) ttm_bo_mem_put(bo, &mem); @@ -1097,20 +1082,18 @@ bool ttm_bo_mem_compat(struct ttm_placement *placement, EXPORT_SYMBOL(ttm_bo_mem_compat); int ttm_bo_validate(struct ttm_buffer_object *bo, - struct ttm_placement *placement, - bool interruptible, - bool no_wait_gpu) + struct ttm_placement *placement, + struct ttm_operation_ctx *ctx) { int ret; uint32_t new_flags; - lockdep_assert_held(&bo->resv->lock.base); + reservation_object_assert_held(bo->resv); /* * Check whether we need to move buffer. */ if (!ttm_bo_mem_compat(placement, &bo->mem, &new_flags)) { - ret = ttm_bo_move_buffer(bo, placement, interruptible, - no_wait_gpu); + ret = ttm_bo_move_buffer(bo, placement, ctx); if (ret) return ret; } else { @@ -1139,7 +1122,7 @@ int ttm_bo_init_reserved(struct ttm_bo_device *bdev, enum ttm_bo_type type, struct ttm_placement *placement, uint32_t page_alignment, - bool interruptible, + struct ttm_operation_ctx *ctx, struct file *persistent_swap_storage, size_t acc_size, struct sg_table *sg, @@ -1199,7 +1182,7 @@ int ttm_bo_init_reserved(struct ttm_bo_device *bdev, bo->sg = sg; if (resv) { bo->resv = resv; - lockdep_assert_held(&bo->resv->lock.base); + reservation_object_assert_held(bo->resv); } else { bo->resv = &bo->ttm_resv; } @@ -1221,12 +1204,12 @@ int ttm_bo_init_reserved(struct ttm_bo_device *bdev, * since otherwise lockdep will be angered in radeon. */ if (!resv) { - locked = ww_mutex_trylock(&bo->resv->lock); + locked = reservation_object_trylock(bo->resv); WARN_ON(!locked); } if (likely(!ret)) - ret = ttm_bo_validate(bo, placement, interruptible, false); + ret = ttm_bo_validate(bo, placement, ctx); if (unlikely(ret)) { if (!resv) @@ -1259,10 +1242,11 @@ int ttm_bo_init(struct ttm_bo_device *bdev, struct reservation_object *resv, void (*destroy) (struct ttm_buffer_object *)) { + struct ttm_operation_ctx ctx = { interruptible, false }; int ret; ret = ttm_bo_init_reserved(bdev, bo, size, type, placement, - page_alignment, interruptible, + page_alignment, &ctx, persistent_swap_storage, acc_size, sg, resv, destroy); if (ret) @@ -1334,6 +1318,7 @@ EXPORT_SYMBOL(ttm_bo_create); static int ttm_bo_force_list_clean(struct ttm_bo_device *bdev, unsigned mem_type) { + struct ttm_operation_ctx ctx = { false, false }; struct ttm_mem_type_manager *man = &bdev->man[mem_type]; struct ttm_bo_global *glob = bdev->glob; struct dma_fence *fence; @@ -1348,7 +1333,7 @@ static int ttm_bo_force_list_clean(struct ttm_bo_device *bdev, for (i = 0; i < TTM_MAX_BO_PRIORITY; ++i) { while (!list_empty(&man->lru[i])) { spin_unlock(&glob->lru_lock); - ret = ttm_mem_evict_first(bdev, mem_type, NULL, false, false); + ret = ttm_mem_evict_first(bdev, mem_type, NULL, &ctx); if (ret) return ret; spin_lock(&glob->lru_lock); @@ -1554,16 +1539,13 @@ int ttm_bo_device_release(struct ttm_bo_device *bdev) cancel_delayed_work_sync(&bdev->wq); - while (ttm_bo_delayed_delete(bdev, true)) - ; + if (ttm_bo_delayed_delete(bdev, true)) + pr_debug("Delayed destroy list was clean\n"); spin_lock(&glob->lru_lock); - if (list_empty(&bdev->ddestroy)) - TTM_DEBUG("Delayed destroy list was clean\n"); - for (i = 0; i < TTM_MAX_BO_PRIORITY; ++i) if (list_empty(&bdev->man[0].lru[0])) - TTM_DEBUG("Swap list %d was clean\n", i); + pr_debug("Swap list %d was clean\n", i); spin_unlock(&glob->lru_lock); drm_vma_offset_manager_destroy(&bdev->vma_manager); @@ -1718,7 +1700,7 @@ static int ttm_bo_swapout(struct ttm_mem_shrink *shrink) spin_lock(&glob->lru_lock); for (i = 0; i < TTM_MAX_BO_PRIORITY; ++i) { list_for_each_entry(bo, &glob->swap_lru[i], swap) { - ret = __ttm_bo_reserve(bo, false, true, NULL); + ret = reservation_object_trylock(bo->resv) ? 0 : -EBUSY; if (!ret) break; } @@ -1734,7 +1716,7 @@ static int ttm_bo_swapout(struct ttm_mem_shrink *shrink) kref_get(&bo->list_kref); if (!list_empty(&bo->ddestroy)) { - ret = ttm_bo_cleanup_refs_and_unlock(bo, false, false); + ret = ttm_bo_cleanup_refs(bo, false, false, true); kref_put(&bo->list_kref, ttm_bo_release_list); return ret; } @@ -1748,6 +1730,7 @@ static int ttm_bo_swapout(struct ttm_mem_shrink *shrink) if (bo->mem.mem_type != TTM_PL_SYSTEM || bo->ttm->caching_state != tt_cached) { + struct ttm_operation_ctx ctx = { false, false }; struct ttm_mem_reg evict_mem; evict_mem = bo->mem; @@ -1755,8 +1738,7 @@ static int ttm_bo_swapout(struct ttm_mem_shrink *shrink) evict_mem.placement = TTM_PL_FLAG_SYSTEM | TTM_PL_FLAG_CACHED; evict_mem.mem_type = TTM_PL_SYSTEM; - ret = ttm_bo_handle_move_mem(bo, &evict_mem, true, - false, false); + ret = ttm_bo_handle_move_mem(bo, &evict_mem, true, &ctx); if (unlikely(ret != 0)) goto out; } @@ -1788,7 +1770,7 @@ out: * already swapped buffer. */ - __ttm_bo_unreserve(bo); + reservation_object_unlock(bo->resv); kref_put(&bo->list_kref, ttm_bo_release_list); return ret; } @@ -1822,10 +1804,12 @@ int ttm_bo_wait_unreserved(struct ttm_buffer_object *bo) return -ERESTARTSYS; if (!ww_mutex_is_locked(&bo->resv->lock)) goto out_unlock; - ret = __ttm_bo_reserve(bo, true, false, NULL); + ret = reservation_object_lock_interruptible(bo->resv, NULL); + if (ret == -EINTR) + ret = -ERESTARTSYS; if (unlikely(ret != 0)) goto out_unlock; - __ttm_bo_unreserve(bo); + reservation_object_unlock(bo->resv); out_unlock: mutex_unlock(&bo->wu_mutex); diff --git a/drivers/gpu/drm/ttm/ttm_bo_util.c b/drivers/gpu/drm/ttm/ttm_bo_util.c index e7a519f1849b..6e353df4e4bc 100644 --- a/drivers/gpu/drm/ttm/ttm_bo_util.c +++ b/drivers/gpu/drm/ttm/ttm_bo_util.c @@ -45,7 +45,7 @@ void ttm_bo_free_old_node(struct ttm_buffer_object *bo) } int ttm_bo_move_ttm(struct ttm_buffer_object *bo, - bool interruptible, bool no_wait_gpu, + struct ttm_operation_ctx *ctx, struct ttm_mem_reg *new_mem) { struct ttm_tt *ttm = bo->ttm; @@ -53,7 +53,7 @@ int ttm_bo_move_ttm(struct ttm_buffer_object *bo, int ret; if (old_mem->mem_type != TTM_PL_SYSTEM) { - ret = ttm_bo_wait(bo, interruptible, no_wait_gpu); + ret = ttm_bo_wait(bo, ctx->interruptible, ctx->no_wait_gpu); if (unlikely(ret != 0)) { if (ret != -ERESTARTSYS) @@ -329,7 +329,7 @@ static int ttm_copy_ttm_io_page(struct ttm_tt *ttm, void *dst, } int ttm_bo_move_memcpy(struct ttm_buffer_object *bo, - bool interruptible, bool no_wait_gpu, + struct ttm_operation_ctx *ctx, struct ttm_mem_reg *new_mem) { struct ttm_bo_device *bdev = bo->bdev; @@ -345,7 +345,7 @@ int ttm_bo_move_memcpy(struct ttm_buffer_object *bo, unsigned long add = 0; int dir; - ret = ttm_bo_wait(bo, interruptible, no_wait_gpu); + ret = ttm_bo_wait(bo, ctx->interruptible, ctx->no_wait_gpu); if (ret) return ret; @@ -485,7 +485,7 @@ static int ttm_buffer_object_transfer(struct ttm_buffer_object *bo, fbo->acc_size = 0; fbo->resv = &fbo->ttm_resv; reservation_object_init(fbo->resv); - ret = ww_mutex_trylock(&fbo->resv->lock); + ret = reservation_object_trylock(fbo->resv); WARN_ON(!ret); *new_obj = fbo; diff --git a/drivers/gpu/drm/ttm/ttm_execbuf_util.c b/drivers/gpu/drm/ttm/ttm_execbuf_util.c index 5e1bcabffef5..373ced0b2fc2 100644 --- a/drivers/gpu/drm/ttm/ttm_execbuf_util.c +++ b/drivers/gpu/drm/ttm/ttm_execbuf_util.c @@ -38,7 +38,7 @@ static void ttm_eu_backoff_reservation_reverse(struct list_head *list, list_for_each_entry_continue_reverse(entry, list, head) { struct ttm_buffer_object *bo = entry->bo; - __ttm_bo_unreserve(bo); + reservation_object_unlock(bo->resv); } } @@ -69,7 +69,7 @@ void ttm_eu_backoff_reservation(struct ww_acquire_ctx *ticket, struct ttm_buffer_object *bo = entry->bo; ttm_bo_add_to_lru(bo); - __ttm_bo_unreserve(bo); + reservation_object_unlock(bo->resv); } spin_unlock(&glob->lru_lock); @@ -112,7 +112,7 @@ int ttm_eu_reserve_buffers(struct ww_acquire_ctx *ticket, ret = __ttm_bo_reserve(bo, intr, (ticket == NULL), ticket); if (!ret && unlikely(atomic_read(&bo->cpu_writers) > 0)) { - __ttm_bo_unreserve(bo); + reservation_object_unlock(bo->resv); ret = -EBUSY; @@ -203,7 +203,7 @@ void ttm_eu_fence_buffer_objects(struct ww_acquire_ctx *ticket, else reservation_object_add_excl_fence(bo->resv, fence); ttm_bo_add_to_lru(bo); - __ttm_bo_unreserve(bo); + reservation_object_unlock(bo->resv); } spin_unlock(&glob->lru_lock); if (ticket) diff --git a/drivers/gpu/drm/ttm/ttm_page_alloc.c b/drivers/gpu/drm/ttm/ttm_page_alloc.c index 316f831ad5f0..b5ba6441489f 100644 --- a/drivers/gpu/drm/ttm/ttm_page_alloc.c +++ b/drivers/gpu/drm/ttm/ttm_page_alloc.c @@ -81,6 +81,7 @@ struct ttm_page_pool { char *name; unsigned long nfrees; unsigned long nrefills; + unsigned int order; }; /** @@ -222,6 +223,17 @@ static struct kobj_type ttm_pool_kobj_type = { static struct ttm_pool_manager *_manager; #ifndef CONFIG_X86 +static int set_pages_wb(struct page *page, int numpages) +{ +#if IS_ENABLED(CONFIG_AGP) + int i; + + for (i = 0; i < numpages; i++) + unmap_page_from_agp(page++); +#endif + return 0; +} + static int set_pages_array_wb(struct page **pages, int addrinarray) { #if IS_ENABLED(CONFIG_AGP) @@ -284,13 +296,23 @@ static struct ttm_page_pool *ttm_get_pool(int flags, bool huge, } /* set memory back to wb and free the pages. */ -static void ttm_pages_put(struct page *pages[], unsigned npages) +static void ttm_pages_put(struct page *pages[], unsigned npages, + unsigned int order) { - unsigned i; - if (set_pages_array_wb(pages, npages)) - pr_err("Failed to set %d pages to wb!\n", npages); - for (i = 0; i < npages; ++i) - __free_page(pages[i]); + unsigned int i, pages_nr = (1 << order); + + if (order == 0) { + if (set_pages_array_wb(pages, npages)) + pr_err("Failed to set %d pages to wb!\n", npages); + } + + for (i = 0; i < npages; ++i) { + if (order > 0) { + if (set_pages_wb(pages[i], pages_nr)) + pr_err("Failed to set %d pages to wb!\n", pages_nr); + } + __free_pages(pages[i], order); + } } static void ttm_pool_update_free_locked(struct ttm_page_pool *pool, @@ -353,7 +375,7 @@ restart: */ spin_unlock_irqrestore(&pool->lock, irq_flags); - ttm_pages_put(pages_to_free, freed_pages); + ttm_pages_put(pages_to_free, freed_pages, pool->order); if (likely(nr_free != FREE_ALL_PAGES)) nr_free -= freed_pages; @@ -388,7 +410,7 @@ restart: spin_unlock_irqrestore(&pool->lock, irq_flags); if (freed_pages) - ttm_pages_put(pages_to_free, freed_pages); + ttm_pages_put(pages_to_free, freed_pages, pool->order); out: if (pages_to_free != static_buf) kfree(pages_to_free); @@ -412,6 +434,7 @@ ttm_pool_shrink_scan(struct shrinker *shrink, struct shrink_control *sc) struct ttm_page_pool *pool; int shrink_pages = sc->nr_to_scan; unsigned long freed = 0; + unsigned int nr_free_pool; if (!mutex_trylock(&lock)) return SHRINK_STOP; @@ -419,12 +442,20 @@ ttm_pool_shrink_scan(struct shrinker *shrink, struct shrink_control *sc) /* select start pool in round robin fashion */ for (i = 0; i < NUM_POOLS; ++i) { unsigned nr_free = shrink_pages; + unsigned page_nr; + if (shrink_pages == 0) break; + pool = &_manager->pools[(i + pool_offset)%NUM_POOLS]; + page_nr = (1 << pool->order); /* OK to use static buffer since global mutex is held. */ - shrink_pages = ttm_page_pool_free(pool, nr_free, true); - freed += nr_free - shrink_pages; + nr_free_pool = roundup(nr_free, page_nr) >> pool->order; + shrink_pages = ttm_page_pool_free(pool, nr_free_pool, true); + freed += (nr_free_pool - shrink_pages) << pool->order; + if (freed >= sc->nr_to_scan) + break; + shrink_pages <<= pool->order; } mutex_unlock(&lock); return freed; @@ -436,9 +467,12 @@ ttm_pool_shrink_count(struct shrinker *shrink, struct shrink_control *sc) { unsigned i; unsigned long count = 0; + struct ttm_page_pool *pool; - for (i = 0; i < NUM_POOLS; ++i) - count += _manager->pools[i].npages; + for (i = 0; i < NUM_POOLS; ++i) { + pool = &_manager->pools[i]; + count += (pool->npages << pool->order); + } return count; } @@ -510,8 +544,7 @@ static int ttm_alloc_new_pages(struct list_head *pages, gfp_t gfp_flags, int r = 0; unsigned i, j, cpages; unsigned npages = 1 << order; - unsigned max_cpages = min(count, - (unsigned)(PAGE_SIZE/sizeof(struct page *))); + unsigned max_cpages = min(count << order, (unsigned)NUM_PAGES_TO_ALLOC); /* allocate array for page caching change */ caching_array = kmalloc(max_cpages*sizeof(struct page *), GFP_KERNEL); @@ -744,12 +777,14 @@ static void ttm_put_pages(struct page **pages, unsigned npages, int flags, } #ifdef CONFIG_TRANSPARENT_HUGEPAGE - for (j = 0; j < HPAGE_PMD_NR; ++j) - if (p++ != pages[i + j]) - break; + if (!(flags & TTM_PAGE_FLAG_DMA32)) { + for (j = 0; j < HPAGE_PMD_NR; ++j) + if (p++ != pages[i + j]) + break; - if (j == HPAGE_PMD_NR) - order = HPAGE_PMD_ORDER; + if (j == HPAGE_PMD_NR) + order = HPAGE_PMD_ORDER; + } #endif if (page_count(pages[i]) != 1) @@ -843,7 +878,7 @@ static int ttm_get_pages(struct page **pages, unsigned npages, int flags, #endif struct list_head plist; struct page *p = NULL; - unsigned count; + unsigned count, first; int r; /* No pool for cached pages */ @@ -865,23 +900,26 @@ static int ttm_get_pages(struct page **pages, unsigned npages, int flags, i = 0; #ifdef CONFIG_TRANSPARENT_HUGEPAGE - while (npages >= HPAGE_PMD_NR) { - gfp_t huge_flags = gfp_flags; + if (!(gfp_flags & GFP_DMA32)) { + while (npages >= HPAGE_PMD_NR) { + gfp_t huge_flags = gfp_flags; - huge_flags |= GFP_TRANSHUGE; - huge_flags &= ~__GFP_MOVABLE; - huge_flags &= ~__GFP_COMP; - p = alloc_pages(huge_flags, HPAGE_PMD_ORDER); - if (!p) - break; + huge_flags |= GFP_TRANSHUGE; + huge_flags &= ~__GFP_MOVABLE; + huge_flags &= ~__GFP_COMP; + p = alloc_pages(huge_flags, HPAGE_PMD_ORDER); + if (!p) + break; - for (j = 0; j < HPAGE_PMD_NR; ++j) - pages[i++] = p++; + for (j = 0; j < HPAGE_PMD_NR; ++j) + pages[i++] = p++; - npages -= HPAGE_PMD_NR; + npages -= HPAGE_PMD_NR; + } } #endif + first = i; while (npages) { p = alloc_page(gfp_flags); if (!p) { @@ -889,6 +927,10 @@ static int ttm_get_pages(struct page **pages, unsigned npages, int flags, return -ENOMEM; } + /* Swap the pages if we detect consecutive order */ + if (i > first && pages[i - 1] == p - 1) + swap(p, pages[i - 1]); + pages[i++] = p; --npages; } @@ -917,8 +959,15 @@ static int ttm_get_pages(struct page **pages, unsigned npages, int flags, r = ttm_page_pool_get_pages(pool, &plist, flags, cstate, npages - count, 0); - list_for_each_entry(p, &plist, lru) - pages[count++] = p; + first = count; + list_for_each_entry(p, &plist, lru) { + struct page *tmp = p; + + /* Swap the pages if we detect consecutive order */ + if (count > first && pages[count - 1] == tmp - 1) + swap(tmp, pages[count - 1]); + pages[count++] = tmp; + } if (r) { /* If there is any pages in the list put them back to @@ -933,7 +982,7 @@ static int ttm_get_pages(struct page **pages, unsigned npages, int flags, } static void ttm_page_pool_init_locked(struct ttm_page_pool *pool, gfp_t flags, - char *name) + char *name, unsigned int order) { spin_lock_init(&pool->lock); pool->fill_lock = false; @@ -941,11 +990,17 @@ static void ttm_page_pool_init_locked(struct ttm_page_pool *pool, gfp_t flags, pool->npages = pool->nfrees = 0; pool->gfp_flags = flags; pool->name = name; + pool->order = order; } int ttm_page_alloc_init(struct ttm_mem_global *glob, unsigned max_pages) { int ret; +#ifdef CONFIG_TRANSPARENT_HUGEPAGE + unsigned order = HPAGE_PMD_ORDER; +#else + unsigned order = 0; +#endif WARN_ON(_manager); @@ -953,23 +1008,23 @@ int ttm_page_alloc_init(struct ttm_mem_global *glob, unsigned max_pages) _manager = kzalloc(sizeof(*_manager), GFP_KERNEL); - ttm_page_pool_init_locked(&_manager->wc_pool, GFP_HIGHUSER, "wc"); + ttm_page_pool_init_locked(&_manager->wc_pool, GFP_HIGHUSER, "wc", 0); - ttm_page_pool_init_locked(&_manager->uc_pool, GFP_HIGHUSER, "uc"); + ttm_page_pool_init_locked(&_manager->uc_pool, GFP_HIGHUSER, "uc", 0); ttm_page_pool_init_locked(&_manager->wc_pool_dma32, - GFP_USER | GFP_DMA32, "wc dma"); + GFP_USER | GFP_DMA32, "wc dma", 0); ttm_page_pool_init_locked(&_manager->uc_pool_dma32, - GFP_USER | GFP_DMA32, "uc dma"); + GFP_USER | GFP_DMA32, "uc dma", 0); ttm_page_pool_init_locked(&_manager->wc_pool_huge, GFP_TRANSHUGE & ~(__GFP_MOVABLE | __GFP_COMP), - "wc huge"); + "wc huge", order); ttm_page_pool_init_locked(&_manager->uc_pool_huge, GFP_TRANSHUGE & ~(__GFP_MOVABLE | __GFP_COMP) - , "uc huge"); + , "uc huge", order); _manager->options.max_size = max_pages; _manager->options.small = SMALL_ALLOCATION; @@ -1058,7 +1113,6 @@ void ttm_pool_unpopulate(struct ttm_tt *ttm) } EXPORT_SYMBOL(ttm_pool_unpopulate); -#if defined(CONFIG_SWIOTLB) || defined(CONFIG_INTEL_IOMMU) int ttm_populate_and_map_pages(struct device *dev, struct ttm_dma_tt *tt) { unsigned i, j; @@ -1129,7 +1183,6 @@ void ttm_unmap_and_unpopulate_pages(struct device *dev, struct ttm_dma_tt *tt) ttm_pool_unpopulate(&tt->ttm); } EXPORT_SYMBOL(ttm_unmap_and_unpopulate_pages); -#endif int ttm_page_alloc_debugfs(struct seq_file *m, void *data) { diff --git a/drivers/gpu/drm/ttm/ttm_page_alloc_dma.c b/drivers/gpu/drm/ttm/ttm_page_alloc_dma.c index 6b2627fe9bc1..bda00b2ab51c 100644 --- a/drivers/gpu/drm/ttm/ttm_page_alloc_dma.c +++ b/drivers/gpu/drm/ttm/ttm_page_alloc_dma.c @@ -1244,15 +1244,12 @@ int ttm_dma_page_alloc_debugfs(struct seq_file *m, void *data) { struct device_pools *p; struct dma_pool *pool = NULL; - char *h[] = {"pool", "refills", "pages freed", "inuse", "available", - "name", "virt", "busaddr"}; if (!_manager) { seq_printf(m, "No pool allocator running.\n"); return 0; } - seq_printf(m, "%13s %12s %13s %8s %8s %8s\n", - h[0], h[1], h[2], h[3], h[4], h[5]); + seq_printf(m, " pool refills pages freed inuse available name\n"); mutex_lock(&_manager->lock); list_for_each_entry(p, &_manager->pools, pools) { struct device *dev = p->dev; |