diff options
Diffstat (limited to 'drivers/gpu/drm/ttm')
| -rw-r--r-- | drivers/gpu/drm/ttm/Makefile | 7 | ||||
| -rw-r--r-- | drivers/gpu/drm/ttm/ttm_agp_backend.c | 2 | ||||
| -rw-r--r-- | drivers/gpu/drm/ttm/ttm_bo.c | 427 | ||||
| -rw-r--r-- | drivers/gpu/drm/ttm/ttm_bo_util.c | 40 | ||||
| -rw-r--r-- | drivers/gpu/drm/ttm/ttm_bo_vm.c | 24 | ||||
| -rw-r--r-- | drivers/gpu/drm/ttm/ttm_device.c | 268 | ||||
| -rw-r--r-- | drivers/gpu/drm/ttm/ttm_execbuf_util.c | 8 | ||||
| -rw-r--r-- | drivers/gpu/drm/ttm/ttm_memory.c | 683 | ||||
| -rw-r--r-- | drivers/gpu/drm/ttm/ttm_module.c | 54 | ||||
| -rw-r--r-- | drivers/gpu/drm/ttm/ttm_module.h | 8 | ||||
| -rw-r--r-- | drivers/gpu/drm/ttm/ttm_pool.c | 157 | ||||
| -rw-r--r-- | drivers/gpu/drm/ttm/ttm_range_manager.c | 4 | ||||
| -rw-r--r-- | drivers/gpu/drm/ttm/ttm_resource.c | 11 | ||||
| -rw-r--r-- | drivers/gpu/drm/ttm/ttm_tt.c | 114 |
14 files changed, 592 insertions, 1215 deletions
diff --git a/drivers/gpu/drm/ttm/Makefile b/drivers/gpu/drm/ttm/Makefile index b6f5f87b270f..40e5e9da7953 100644 --- a/drivers/gpu/drm/ttm/Makefile +++ b/drivers/gpu/drm/ttm/Makefile @@ -2,10 +2,9 @@ # # Makefile for the drm device driver. This driver provides support for the -ttm-y := ttm_memory.o ttm_tt.o ttm_bo.o \ - ttm_bo_util.o ttm_bo_vm.o ttm_module.o \ - ttm_execbuf_util.o ttm_range_manager.o \ - ttm_resource.o ttm_pool.o +ttm-y := ttm_tt.o ttm_bo.o ttm_bo_util.o ttm_bo_vm.o ttm_module.o \ + ttm_execbuf_util.o ttm_range_manager.o ttm_resource.o ttm_pool.o \ + ttm_device.o ttm-$(CONFIG_AGP) += ttm_agp_backend.o obj-$(CONFIG_DRM_TTM) += ttm.o diff --git a/drivers/gpu/drm/ttm/ttm_agp_backend.c b/drivers/gpu/drm/ttm/ttm_agp_backend.c index 8f9fa4188897..0226ae69d3ab 100644 --- a/drivers/gpu/drm/ttm/ttm_agp_backend.c +++ b/drivers/gpu/drm/ttm/ttm_agp_backend.c @@ -49,7 +49,7 @@ struct ttm_agp_backend { int ttm_agp_bind(struct ttm_tt *ttm, struct ttm_resource *bo_mem) { struct ttm_agp_backend *agp_be = container_of(ttm, struct ttm_agp_backend, ttm); - struct page *dummy_read_page = ttm_bo_glob.dummy_read_page; + struct page *dummy_read_page = ttm_glob.dummy_read_page; struct drm_mm_node *node = bo_mem->mm_node; struct agp_memory *mem; int ret, cached = ttm->caching == ttm_cached; diff --git a/drivers/gpu/drm/ttm/ttm_bo.c b/drivers/gpu/drm/ttm/ttm_bo.c index 101a68dc615b..cfd0b9292397 100644 --- a/drivers/gpu/drm/ttm/ttm_bo.c +++ b/drivers/gpu/drm/ttm/ttm_bo.c @@ -44,21 +44,6 @@ #include "ttm_module.h" -static void ttm_bo_global_kobj_release(struct kobject *kobj); - -/* - * ttm_global_mutex - protecting the global BO state - */ -DEFINE_MUTEX(ttm_global_mutex); -unsigned ttm_bo_glob_use_count; -struct ttm_bo_global ttm_bo_glob; -EXPORT_SYMBOL(ttm_bo_glob); - -static struct attribute ttm_bo_count = { - .name = "bo_count", - .mode = S_IRUGO -}; - /* default destructor */ static void ttm_bo_default_destroy(struct ttm_buffer_object *bo) { @@ -84,41 +69,14 @@ static void ttm_bo_mem_space_debug(struct ttm_buffer_object *bo, } } -static ssize_t ttm_bo_global_show(struct kobject *kobj, - struct attribute *attr, - char *buffer) -{ - struct ttm_bo_global *glob = - container_of(kobj, struct ttm_bo_global, kobj); - - return snprintf(buffer, PAGE_SIZE, "%d\n", - atomic_read(&glob->bo_count)); -} - -static struct attribute *ttm_bo_global_attrs[] = { - &ttm_bo_count, - NULL -}; - -static const struct sysfs_ops ttm_bo_global_ops = { - .show = &ttm_bo_global_show -}; - -static struct kobj_type ttm_bo_glob_kobj_type = { - .release = &ttm_bo_global_kobj_release, - .sysfs_ops = &ttm_bo_global_ops, - .default_attrs = ttm_bo_global_attrs -}; - static void ttm_bo_del_from_lru(struct ttm_buffer_object *bo) { - struct ttm_bo_device *bdev = bo->bdev; + struct ttm_device *bdev = bo->bdev; - list_del_init(&bo->swap); list_del_init(&bo->lru); - if (bdev->driver->del_from_lru_notify) - bdev->driver->del_from_lru_notify(bo); + if (bdev->funcs->del_from_lru_notify) + bdev->funcs->del_from_lru_notify(bo); } static void ttm_bo_bulk_move_set_pos(struct ttm_lru_bulk_move_pos *pos, @@ -133,7 +91,7 @@ void ttm_bo_move_to_lru_tail(struct ttm_buffer_object *bo, struct ttm_resource *mem, struct ttm_lru_bulk_move *bulk) { - struct ttm_bo_device *bdev = bo->bdev; + struct ttm_device *bdev = bo->bdev; struct ttm_resource_manager *man; if (!bo->deleted) @@ -146,17 +104,9 @@ void ttm_bo_move_to_lru_tail(struct ttm_buffer_object *bo, man = ttm_manager_type(bdev, mem->mem_type); list_move_tail(&bo->lru, &man->lru[bo->priority]); - if (man->use_tt && bo->ttm && - !(bo->ttm->page_flags & (TTM_PAGE_FLAG_SG | - TTM_PAGE_FLAG_SWAPPED))) { - struct list_head *swap; - - swap = &ttm_bo_glob.swap_lru[bo->priority]; - list_move_tail(&bo->swap, swap); - } - if (bdev->driver->del_from_lru_notify) - bdev->driver->del_from_lru_notify(bo); + if (bdev->funcs->del_from_lru_notify) + bdev->funcs->del_from_lru_notify(bo); if (bulk && !bo->pin_count) { switch (bo->mem.mem_type) { @@ -168,9 +118,6 @@ void ttm_bo_move_to_lru_tail(struct ttm_buffer_object *bo, ttm_bo_bulk_move_set_pos(&bulk->vram[bo->priority], bo); break; } - if (bo->ttm && !(bo->ttm->page_flags & - (TTM_PAGE_FLAG_SG | TTM_PAGE_FLAG_SWAPPED))) - ttm_bo_bulk_move_set_pos(&bulk->swap[bo->priority], bo); } } EXPORT_SYMBOL(ttm_bo_move_to_lru_tail); @@ -208,20 +155,6 @@ void ttm_bo_bulk_move_lru_tail(struct ttm_lru_bulk_move *bulk) list_bulk_move_tail(&man->lru[i], &pos->first->lru, &pos->last->lru); } - - for (i = 0; i < TTM_MAX_BO_PRIORITY; ++i) { - struct ttm_lru_bulk_move_pos *pos = &bulk->swap[i]; - struct list_head *lru; - - if (!pos->first) - continue; - - dma_resv_assert_held(pos->first->base.resv); - dma_resv_assert_held(pos->last->base.resv); - - lru = &ttm_bo_glob.swap_lru[i]; - list_bulk_move_tail(lru, &pos->first->swap, &pos->last->swap); - } } EXPORT_SYMBOL(ttm_bo_bulk_move_lru_tail); @@ -230,7 +163,7 @@ static int ttm_bo_handle_move_mem(struct ttm_buffer_object *bo, struct ttm_operation_ctx *ctx, struct ttm_place *hop) { - struct ttm_bo_device *bdev = bo->bdev; + struct ttm_device *bdev = bo->bdev; struct ttm_resource_manager *old_man = ttm_manager_type(bdev, bo->mem.mem_type); struct ttm_resource_manager *new_man = ttm_manager_type(bdev, mem->mem_type); int ret; @@ -256,7 +189,7 @@ static int ttm_bo_handle_move_mem(struct ttm_buffer_object *bo, } } - ret = bdev->driver->move(bo, evict, ctx, mem, hop); + ret = bdev->funcs->move(bo, evict, ctx, mem, hop); if (ret) { if (ret == -EMULTIHOP) return ret; @@ -284,8 +217,8 @@ out_err: static void ttm_bo_cleanup_memtype_use(struct ttm_buffer_object *bo) { - if (bo->bdev->driver->delete_mem_notify) - bo->bdev->driver->delete_mem_notify(bo); + if (bo->bdev->funcs->delete_mem_notify) + bo->bdev->funcs->delete_mem_notify(bo); ttm_bo_tt_destroy(bo); ttm_resource_free(bo, &bo->mem); @@ -310,9 +243,9 @@ static int ttm_bo_individualize_resv(struct ttm_buffer_object *bo) * reference it any more. The only tricky case is the trylock on * the resv object while holding the lru_lock. */ - spin_lock(&ttm_bo_glob.lru_lock); + spin_lock(&bo->bdev->lru_lock); bo->base.resv = &bo->base._resv; - spin_unlock(&ttm_bo_glob.lru_lock); + spin_unlock(&bo->bdev->lru_lock); } return r; @@ -371,7 +304,7 @@ static int ttm_bo_cleanup_refs(struct ttm_buffer_object *bo, if (unlock_resv) dma_resv_unlock(bo->base.resv); - spin_unlock(&ttm_bo_glob.lru_lock); + spin_unlock(&bo->bdev->lru_lock); lret = dma_resv_wait_timeout_rcu(resv, true, interruptible, 30 * HZ); @@ -381,7 +314,7 @@ static int ttm_bo_cleanup_refs(struct ttm_buffer_object *bo, else if (lret == 0) return -EBUSY; - spin_lock(&ttm_bo_glob.lru_lock); + spin_lock(&bo->bdev->lru_lock); if (unlock_resv && !dma_resv_trylock(bo->base.resv)) { /* * We raced, and lost, someone else holds the reservation now, @@ -391,7 +324,7 @@ static int ttm_bo_cleanup_refs(struct ttm_buffer_object *bo, * delayed destruction would succeed, so just return success * here. */ - spin_unlock(&ttm_bo_glob.lru_lock); + spin_unlock(&bo->bdev->lru_lock); return 0; } ret = 0; @@ -400,13 +333,13 @@ static int ttm_bo_cleanup_refs(struct ttm_buffer_object *bo, if (ret || unlikely(list_empty(&bo->ddestroy))) { if (unlock_resv) dma_resv_unlock(bo->base.resv); - spin_unlock(&ttm_bo_glob.lru_lock); + spin_unlock(&bo->bdev->lru_lock); return ret; } ttm_bo_del_from_lru(bo); list_del_init(&bo->ddestroy); - spin_unlock(&ttm_bo_glob.lru_lock); + spin_unlock(&bo->bdev->lru_lock); ttm_bo_cleanup_memtype_use(bo); if (unlock_resv) @@ -421,15 +354,14 @@ static int ttm_bo_cleanup_refs(struct ttm_buffer_object *bo, * Traverse the delayed list, and call ttm_bo_cleanup_refs on all * encountered buffers. */ -static bool ttm_bo_delayed_delete(struct ttm_bo_device *bdev, bool remove_all) +bool ttm_bo_delayed_delete(struct ttm_device *bdev, bool remove_all) { - struct ttm_bo_global *glob = &ttm_bo_glob; struct list_head removed; bool empty; INIT_LIST_HEAD(&removed); - spin_lock(&glob->lru_lock); + spin_lock(&bdev->lru_lock); while (!list_empty(&bdev->ddestroy)) { struct ttm_buffer_object *bo; @@ -440,44 +372,33 @@ static bool ttm_bo_delayed_delete(struct ttm_bo_device *bdev, bool remove_all) continue; if (remove_all || bo->base.resv != &bo->base._resv) { - spin_unlock(&glob->lru_lock); + spin_unlock(&bdev->lru_lock); dma_resv_lock(bo->base.resv, NULL); - spin_lock(&glob->lru_lock); + spin_lock(&bdev->lru_lock); ttm_bo_cleanup_refs(bo, false, !remove_all, true); } else if (dma_resv_trylock(bo->base.resv)) { ttm_bo_cleanup_refs(bo, false, !remove_all, true); } else { - spin_unlock(&glob->lru_lock); + spin_unlock(&bdev->lru_lock); } ttm_bo_put(bo); - spin_lock(&glob->lru_lock); + spin_lock(&bdev->lru_lock); } list_splice_tail(&removed, &bdev->ddestroy); empty = list_empty(&bdev->ddestroy); - spin_unlock(&glob->lru_lock); + spin_unlock(&bdev->lru_lock); return empty; } -static void ttm_bo_delayed_workqueue(struct work_struct *work) -{ - struct ttm_bo_device *bdev = - container_of(work, struct ttm_bo_device, wq.work); - - if (!ttm_bo_delayed_delete(bdev, false)) - schedule_delayed_work(&bdev->wq, - ((HZ / 100) < 1) ? 1 : HZ / 100); -} - static void ttm_bo_release(struct kref *kref) { struct ttm_buffer_object *bo = container_of(kref, struct ttm_buffer_object, kref); - struct ttm_bo_device *bdev = bo->bdev; - size_t acc_size = bo->acc_size; + struct ttm_device *bdev = bo->bdev; int ret; if (!bo->deleted) { @@ -490,8 +411,8 @@ static void ttm_bo_release(struct kref *kref) 30 * HZ); } - if (bo->bdev->driver->release_notify) - bo->bdev->driver->release_notify(bo); + if (bo->bdev->funcs->release_notify) + bo->bdev->funcs->release_notify(bo); drm_vma_offset_remove(bdev->vma_manager, &bo->base.vma_node); ttm_mem_io_free(bdev, &bo->mem); @@ -503,7 +424,7 @@ static void ttm_bo_release(struct kref *kref) ttm_bo_flush_all_fences(bo); bo->deleted = true; - spin_lock(&ttm_bo_glob.lru_lock); + spin_lock(&bo->bdev->lru_lock); /* * Make pinned bos immediately available to @@ -520,27 +441,26 @@ static void ttm_bo_release(struct kref *kref) kref_init(&bo->kref); list_add_tail(&bo->ddestroy, &bdev->ddestroy); - spin_unlock(&ttm_bo_glob.lru_lock); + spin_unlock(&bo->bdev->lru_lock); schedule_delayed_work(&bdev->wq, ((HZ / 100) < 1) ? 1 : HZ / 100); return; } - spin_lock(&ttm_bo_glob.lru_lock); + spin_lock(&bo->bdev->lru_lock); ttm_bo_del_from_lru(bo); list_del(&bo->ddestroy); - spin_unlock(&ttm_bo_glob.lru_lock); + spin_unlock(&bo->bdev->lru_lock); ttm_bo_cleanup_memtype_use(bo); dma_resv_unlock(bo->base.resv); - atomic_dec(&ttm_bo_glob.bo_count); + atomic_dec(&ttm_glob.bo_count); dma_fence_put(bo->moving); if (!ttm_bo_uses_embedded_gem_object(bo)) dma_resv_fini(&bo->base._resv); bo->destroy(bo); - ttm_mem_global_free(&ttm_mem_glob, acc_size); } void ttm_bo_put(struct ttm_buffer_object *bo) @@ -549,13 +469,13 @@ void ttm_bo_put(struct ttm_buffer_object *bo) } EXPORT_SYMBOL(ttm_bo_put); -int ttm_bo_lock_delayed_workqueue(struct ttm_bo_device *bdev) +int ttm_bo_lock_delayed_workqueue(struct ttm_device *bdev) { return cancel_delayed_work_sync(&bdev->wq); } EXPORT_SYMBOL(ttm_bo_lock_delayed_workqueue); -void ttm_bo_unlock_delayed_workqueue(struct ttm_bo_device *bdev, int resched) +void ttm_bo_unlock_delayed_workqueue(struct ttm_device *bdev, int resched) { if (resched) schedule_delayed_work(&bdev->wq, @@ -566,7 +486,7 @@ EXPORT_SYMBOL(ttm_bo_unlock_delayed_workqueue); static int ttm_bo_evict(struct ttm_buffer_object *bo, struct ttm_operation_ctx *ctx) { - struct ttm_bo_device *bdev = bo->bdev; + struct ttm_device *bdev = bo->bdev; struct ttm_resource evict_mem; struct ttm_placement placement; struct ttm_place hop; @@ -578,7 +498,7 @@ static int ttm_bo_evict(struct ttm_buffer_object *bo, placement.num_placement = 0; placement.num_busy_placement = 0; - bdev->driver->evict_flags(bo, &placement); + bdev->funcs->evict_flags(bo, &placement); if (!placement.num_placement && !placement.num_busy_placement) { ttm_bo_wait(bo, false, false); @@ -694,7 +614,7 @@ static int ttm_mem_evict_wait_busy(struct ttm_buffer_object *busy_bo, return r == -EDEADLK ? -EBUSY : r; } -int ttm_mem_evict_first(struct ttm_bo_device *bdev, +int ttm_mem_evict_first(struct ttm_device *bdev, struct ttm_resource_manager *man, const struct ttm_place *place, struct ttm_operation_ctx *ctx, @@ -705,7 +625,7 @@ int ttm_mem_evict_first(struct ttm_bo_device *bdev, unsigned i; int ret; - spin_lock(&ttm_bo_glob.lru_lock); + spin_lock(&bdev->lru_lock); for (i = 0; i < TTM_MAX_BO_PRIORITY; ++i) { list_for_each_entry(bo, &man->lru[i], lru) { bool busy; @@ -718,7 +638,7 @@ int ttm_mem_evict_first(struct ttm_bo_device *bdev, continue; } - if (place && !bdev->driver->eviction_valuable(bo, + if (place && !bdev->funcs->eviction_valuable(bo, place)) { if (locked) dma_resv_unlock(bo->base.resv); @@ -742,7 +662,7 @@ int ttm_mem_evict_first(struct ttm_bo_device *bdev, if (!bo) { if (busy_bo && !ttm_bo_get_unless_zero(busy_bo)) busy_bo = NULL; - spin_unlock(&ttm_bo_glob.lru_lock); + spin_unlock(&bdev->lru_lock); ret = ttm_mem_evict_wait_busy(busy_bo, ctx, ticket); if (busy_bo) ttm_bo_put(busy_bo); @@ -756,7 +676,7 @@ int ttm_mem_evict_first(struct ttm_bo_device *bdev, return ret; } - spin_unlock(&ttm_bo_glob.lru_lock); + spin_unlock(&bdev->lru_lock); ret = ttm_bo_evict(bo, ctx); if (locked) @@ -785,8 +705,9 @@ static int ttm_bo_add_move_fence(struct ttm_buffer_object *bo, return 0; if (no_wait_gpu) { + ret = dma_fence_is_signaled(fence) ? 0 : -EBUSY; dma_fence_put(fence); - return -EBUSY; + return ret; } dma_resv_add_shared_fence(bo->base.resv, fence); @@ -811,7 +732,7 @@ static int ttm_bo_mem_force_space(struct ttm_buffer_object *bo, struct ttm_resource *mem, struct ttm_operation_ctx *ctx) { - struct ttm_bo_device *bdev = bo->bdev; + struct ttm_device *bdev = bo->bdev; struct ttm_resource_manager *man = ttm_manager_type(bdev, mem->mem_type); struct ww_acquire_ctx *ticket; int ret; @@ -846,7 +767,7 @@ static int ttm_bo_mem_placement(struct ttm_buffer_object *bo, const struct ttm_place *place, struct ttm_resource *mem) { - struct ttm_bo_device *bdev = bo->bdev; + struct ttm_device *bdev = bo->bdev; struct ttm_resource_manager *man; man = ttm_manager_type(bdev, place->mem_type); @@ -856,10 +777,9 @@ static int ttm_bo_mem_placement(struct ttm_buffer_object *bo, mem->mem_type = place->mem_type; mem->placement = place->flags; - spin_lock(&ttm_bo_glob.lru_lock); + spin_lock(&bo->bdev->lru_lock); ttm_bo_move_to_lru_tail(bo, mem, NULL); - spin_unlock(&ttm_bo_glob.lru_lock); - + spin_unlock(&bo->bdev->lru_lock); return 0; } @@ -876,7 +796,7 @@ int ttm_bo_mem_space(struct ttm_buffer_object *bo, struct ttm_resource *mem, struct ttm_operation_ctx *ctx) { - struct ttm_bo_device *bdev = bo->bdev; + struct ttm_device *bdev = bo->bdev; bool type_found = false; int i, ret; @@ -1097,38 +1017,25 @@ int ttm_bo_validate(struct ttm_buffer_object *bo, } EXPORT_SYMBOL(ttm_bo_validate); -int ttm_bo_init_reserved(struct ttm_bo_device *bdev, +int ttm_bo_init_reserved(struct ttm_device *bdev, struct ttm_buffer_object *bo, size_t size, enum ttm_bo_type type, struct ttm_placement *placement, uint32_t page_alignment, struct ttm_operation_ctx *ctx, - size_t acc_size, struct sg_table *sg, struct dma_resv *resv, void (*destroy) (struct ttm_buffer_object *)) { - struct ttm_mem_global *mem_glob = &ttm_mem_glob; bool locked; int ret = 0; - ret = ttm_mem_global_alloc(mem_glob, acc_size, ctx); - if (ret) { - pr_err("Out of kernel memory\n"); - if (destroy) - (*destroy)(bo); - else - kfree(bo); - return -ENOMEM; - } - bo->destroy = destroy ? destroy : ttm_bo_default_destroy; kref_init(&bo->kref); INIT_LIST_HEAD(&bo->lru); INIT_LIST_HEAD(&bo->ddestroy); - INIT_LIST_HEAD(&bo->swap); bo->bdev = bdev; bo->type = type; bo->mem.mem_type = TTM_PL_SYSTEM; @@ -1139,7 +1046,6 @@ int ttm_bo_init_reserved(struct ttm_bo_device *bdev, bo->mem.bus.addr = NULL; bo->moving = NULL; bo->mem.placement = 0; - bo->acc_size = acc_size; bo->pin_count = 0; bo->sg = sg; if (resv) { @@ -1157,7 +1063,7 @@ int ttm_bo_init_reserved(struct ttm_bo_device *bdev, dma_resv_init(&bo->base._resv); drm_vma_node_reset(&bo->base.vma_node); } - atomic_inc(&ttm_bo_glob.bo_count); + atomic_inc(&ttm_glob.bo_count); /* * For ttm_bo_type_device buffers, allocate @@ -1193,14 +1099,13 @@ int ttm_bo_init_reserved(struct ttm_bo_device *bdev, } EXPORT_SYMBOL(ttm_bo_init_reserved); -int ttm_bo_init(struct ttm_bo_device *bdev, +int ttm_bo_init(struct ttm_device *bdev, struct ttm_buffer_object *bo, size_t size, enum ttm_bo_type type, struct ttm_placement *placement, uint32_t page_alignment, bool interruptible, - size_t acc_size, struct sg_table *sg, struct dma_resv *resv, void (*destroy) (struct ttm_buffer_object *)) @@ -1209,8 +1114,7 @@ int ttm_bo_init(struct ttm_bo_device *bdev, int ret; ret = ttm_bo_init_reserved(bdev, bo, size, type, placement, - page_alignment, &ctx, acc_size, - sg, resv, destroy); + page_alignment, &ctx, sg, resv, destroy); if (ret) return ret; @@ -1221,171 +1125,13 @@ int ttm_bo_init(struct ttm_bo_device *bdev, } EXPORT_SYMBOL(ttm_bo_init); -size_t ttm_bo_dma_acc_size(struct ttm_bo_device *bdev, - unsigned long bo_size, - unsigned struct_size) -{ - unsigned npages = (PAGE_ALIGN(bo_size)) >> PAGE_SHIFT; - size_t size = 0; - - size += ttm_round_pot(struct_size); - size += ttm_round_pot(npages * (2*sizeof(void *) + sizeof(dma_addr_t))); - size += ttm_round_pot(sizeof(struct ttm_tt)); - return size; -} -EXPORT_SYMBOL(ttm_bo_dma_acc_size); - -static void ttm_bo_global_kobj_release(struct kobject *kobj) -{ - struct ttm_bo_global *glob = - container_of(kobj, struct ttm_bo_global, kobj); - - __free_page(glob->dummy_read_page); -} - -static void ttm_bo_global_release(void) -{ - struct ttm_bo_global *glob = &ttm_bo_glob; - - mutex_lock(&ttm_global_mutex); - if (--ttm_bo_glob_use_count > 0) - goto out; - - kobject_del(&glob->kobj); - kobject_put(&glob->kobj); - ttm_mem_global_release(&ttm_mem_glob); - memset(glob, 0, sizeof(*glob)); -out: - mutex_unlock(&ttm_global_mutex); -} - -static int ttm_bo_global_init(void) -{ - struct ttm_bo_global *glob = &ttm_bo_glob; - int ret = 0; - unsigned i; - - mutex_lock(&ttm_global_mutex); - if (++ttm_bo_glob_use_count > 1) - goto out; - - ret = ttm_mem_global_init(&ttm_mem_glob); - if (ret) - goto out; - - spin_lock_init(&glob->lru_lock); - glob->dummy_read_page = alloc_page(__GFP_ZERO | GFP_DMA32); - - if (unlikely(glob->dummy_read_page == NULL)) { - ret = -ENOMEM; - goto out; - } - - for (i = 0; i < TTM_MAX_BO_PRIORITY; ++i) - INIT_LIST_HEAD(&glob->swap_lru[i]); - INIT_LIST_HEAD(&glob->device_list); - atomic_set(&glob->bo_count, 0); - - ret = kobject_init_and_add( - &glob->kobj, &ttm_bo_glob_kobj_type, ttm_get_kobj(), "buffer_objects"); - if (unlikely(ret != 0)) - kobject_put(&glob->kobj); -out: - mutex_unlock(&ttm_global_mutex); - return ret; -} - -int ttm_bo_device_release(struct ttm_bo_device *bdev) -{ - struct ttm_bo_global *glob = &ttm_bo_glob; - int ret = 0; - unsigned i; - struct ttm_resource_manager *man; - - man = ttm_manager_type(bdev, TTM_PL_SYSTEM); - ttm_resource_manager_set_used(man, false); - ttm_set_driver_manager(bdev, TTM_PL_SYSTEM, NULL); - - mutex_lock(&ttm_global_mutex); - list_del(&bdev->device_list); - mutex_unlock(&ttm_global_mutex); - - cancel_delayed_work_sync(&bdev->wq); - - if (ttm_bo_delayed_delete(bdev, true)) - pr_debug("Delayed destroy list was clean\n"); - - spin_lock(&glob->lru_lock); - for (i = 0; i < TTM_MAX_BO_PRIORITY; ++i) - if (list_empty(&man->lru[0])) - pr_debug("Swap list %d was clean\n", i); - spin_unlock(&glob->lru_lock); - - ttm_pool_fini(&bdev->pool); - - if (!ret) - ttm_bo_global_release(); - - return ret; -} -EXPORT_SYMBOL(ttm_bo_device_release); - -static void ttm_bo_init_sysman(struct ttm_bo_device *bdev) -{ - struct ttm_resource_manager *man = &bdev->sysman; - - /* - * Initialize the system memory buffer type. - * Other types need to be driver / IOCTL initialized. - */ - man->use_tt = true; - - ttm_resource_manager_init(man, 0); - ttm_set_driver_manager(bdev, TTM_PL_SYSTEM, man); - ttm_resource_manager_set_used(man, true); -} - -int ttm_bo_device_init(struct ttm_bo_device *bdev, - struct ttm_bo_driver *driver, - struct device *dev, - struct address_space *mapping, - struct drm_vma_offset_manager *vma_manager, - bool use_dma_alloc, bool use_dma32) -{ - struct ttm_bo_global *glob = &ttm_bo_glob; - int ret; - - if (WARN_ON(vma_manager == NULL)) - return -EINVAL; - - ret = ttm_bo_global_init(); - if (ret) - return ret; - - bdev->driver = driver; - - ttm_bo_init_sysman(bdev); - ttm_pool_init(&bdev->pool, dev, use_dma_alloc, use_dma32); - - bdev->vma_manager = vma_manager; - INIT_DELAYED_WORK(&bdev->wq, ttm_bo_delayed_workqueue); - INIT_LIST_HEAD(&bdev->ddestroy); - bdev->dev_mapping = mapping; - mutex_lock(&ttm_global_mutex); - list_add_tail(&bdev->device_list, &glob->device_list); - mutex_unlock(&ttm_global_mutex); - - return 0; -} -EXPORT_SYMBOL(ttm_bo_device_init); - /* * buffer object vm functions. */ void ttm_bo_unmap_virtual(struct ttm_buffer_object *bo) { - struct ttm_bo_device *bdev = bo->bdev; + struct ttm_device *bdev = bo->bdev; drm_vma_node_unmap(&bo->base.vma_node, bdev->dev_mapping); ttm_mem_io_free(bdev, &bo->mem); @@ -1417,56 +1163,34 @@ int ttm_bo_wait(struct ttm_buffer_object *bo, } EXPORT_SYMBOL(ttm_bo_wait); -/* - * A buffer object shrink method that tries to swap out the first - * buffer object on the bo_global::swap_lru list. - */ -int ttm_bo_swapout(struct ttm_operation_ctx *ctx) +int ttm_bo_swapout(struct ttm_buffer_object *bo, struct ttm_operation_ctx *ctx, + gfp_t gfp_flags) { - struct ttm_bo_global *glob = &ttm_bo_glob; - struct ttm_buffer_object *bo; - int ret = -EBUSY; bool locked; - unsigned i; - - spin_lock(&glob->lru_lock); - for (i = 0; i < TTM_MAX_BO_PRIORITY; ++i) { - list_for_each_entry(bo, &glob->swap_lru[i], swap) { - if (!ttm_bo_evict_swapout_allowable(bo, ctx, &locked, - NULL)) - continue; - - if (!ttm_bo_get_unless_zero(bo)) { - if (locked) - dma_resv_unlock(bo->base.resv); - continue; - } + int ret; - ret = 0; - break; - } - if (!ret) - break; - } + if (!ttm_bo_evict_swapout_allowable(bo, ctx, &locked, NULL)) + return -EBUSY; - if (ret) { - spin_unlock(&glob->lru_lock); - return ret; + if (!ttm_bo_get_unless_zero(bo)) { + if (locked) + dma_resv_unlock(bo->base.resv); + return -EBUSY; } if (bo->deleted) { - ret = ttm_bo_cleanup_refs(bo, false, false, locked); + ttm_bo_cleanup_refs(bo, false, false, locked); ttm_bo_put(bo); - return ret; + return 0; } ttm_bo_del_from_lru(bo); - spin_unlock(&glob->lru_lock); + /* TODO: Cleanup the locking */ + spin_unlock(&bo->bdev->lru_lock); - /** + /* * Move to system cached */ - if (bo->mem.mem_type != TTM_PL_SYSTEM) { struct ttm_operation_ctx ctx = { false, false }; struct ttm_resource evict_mem; @@ -1486,29 +1210,26 @@ int ttm_bo_swapout(struct ttm_operation_ctx *ctx) } } - /** + /* * Make sure BO is idle. */ - ret = ttm_bo_wait(bo, false, false); if (unlikely(ret != 0)) goto out; ttm_bo_unmap_virtual(bo); - /** + /* * Swap out. Buffer will be swapped in again as soon as * anyone tries to access a ttm page. */ + if (bo->bdev->funcs->swap_notify) + bo->bdev->funcs->swap_notify(bo); - if (bo->bdev->driver->swap_notify) - bo->bdev->driver->swap_notify(bo); - - ret = ttm_tt_swapout(bo->bdev, bo->ttm); + ret = ttm_tt_swapout(bo->bdev, bo->ttm, gfp_flags); out: - /** - * + /* * Unreserve without putting on LRU to avoid swapping out an * already swapped buffer. */ @@ -1517,7 +1238,6 @@ out: ttm_bo_put(bo); return ret; } -EXPORT_SYMBOL(ttm_bo_swapout); void ttm_bo_tt_destroy(struct ttm_buffer_object *bo) { @@ -1527,4 +1247,3 @@ void ttm_bo_tt_destroy(struct ttm_buffer_object *bo) ttm_tt_destroy(bo->bdev, bo->ttm); bo->ttm = NULL; } - diff --git a/drivers/gpu/drm/ttm/ttm_bo_util.c b/drivers/gpu/drm/ttm/ttm_bo_util.c index 398d5013fc39..efb7e9c34ab4 100644 --- a/drivers/gpu/drm/ttm/ttm_bo_util.c +++ b/drivers/gpu/drm/ttm/ttm_bo_util.c @@ -46,33 +46,33 @@ struct ttm_transfer_obj { struct ttm_buffer_object *bo; }; -int ttm_mem_io_reserve(struct ttm_bo_device *bdev, +int ttm_mem_io_reserve(struct ttm_device *bdev, struct ttm_resource *mem) { if (mem->bus.offset || mem->bus.addr) return 0; mem->bus.is_iomem = false; - if (!bdev->driver->io_mem_reserve) + if (!bdev->funcs->io_mem_reserve) return 0; - return bdev->driver->io_mem_reserve(bdev, mem); + return bdev->funcs->io_mem_reserve(bdev, mem); } -void ttm_mem_io_free(struct ttm_bo_device *bdev, +void ttm_mem_io_free(struct ttm_device *bdev, struct ttm_resource *mem) { if (!mem->bus.offset && !mem->bus.addr) return; - if (bdev->driver->io_mem_free) - bdev->driver->io_mem_free(bdev, mem); + if (bdev->funcs->io_mem_free) + bdev->funcs->io_mem_free(bdev, mem); mem->bus.offset = 0; mem->bus.addr = NULL; } -static int ttm_resource_ioremap(struct ttm_bo_device *bdev, +static int ttm_resource_ioremap(struct ttm_device *bdev, struct ttm_resource *mem, void **virtual) { @@ -91,6 +91,10 @@ static int ttm_resource_ioremap(struct ttm_bo_device *bdev, if (mem->bus.caching == ttm_write_combined) addr = ioremap_wc(mem->bus.offset, bus_size); +#ifdef CONFIG_X86 + else if (mem->bus.caching == ttm_cached) + addr = ioremap_cache(mem->bus.offset, bus_size); +#endif else addr = ioremap(mem->bus.offset, bus_size); if (!addr) { @@ -102,7 +106,7 @@ static int ttm_resource_ioremap(struct ttm_bo_device *bdev, return 0; } -static void ttm_resource_iounmap(struct ttm_bo_device *bdev, +static void ttm_resource_iounmap(struct ttm_device *bdev, struct ttm_resource *mem, void *virtual) { @@ -172,7 +176,7 @@ int ttm_bo_move_memcpy(struct ttm_buffer_object *bo, struct ttm_operation_ctx *ctx, struct ttm_resource *new_mem) { - struct ttm_bo_device *bdev = bo->bdev; + struct ttm_device *bdev = bo->bdev; struct ttm_resource_manager *man = ttm_manager_type(bdev, new_mem->mem_type); struct ttm_tt *ttm = bo->ttm; struct ttm_resource *old_mem = &bo->mem; @@ -300,16 +304,14 @@ static int ttm_buffer_object_transfer(struct ttm_buffer_object *bo, * TODO: Explicit member copy would probably be better here. */ - atomic_inc(&ttm_bo_glob.bo_count); + atomic_inc(&ttm_glob.bo_count); INIT_LIST_HEAD(&fbo->base.ddestroy); INIT_LIST_HEAD(&fbo->base.lru); - INIT_LIST_HEAD(&fbo->base.swap); fbo->base.moving = NULL; drm_vma_node_reset(&fbo->base.base.vma_node); kref_init(&fbo->base.kref); fbo->base.destroy = &ttm_transfered_destroy; - fbo->base.acc_size = 0; fbo->base.pin_count = 0; if (bo->type != ttm_bo_type_sg) fbo->base.base.resv = &fbo->base.base._resv; @@ -373,6 +375,11 @@ static int ttm_bo_ioremap(struct ttm_buffer_object *bo, if (mem->bus.caching == ttm_write_combined) map->virtual = ioremap_wc(bo->mem.bus.offset + offset, size); +#ifdef CONFIG_X86 + else if (mem->bus.caching == ttm_cached) + map->virtual = ioremap_cache(bo->mem.bus.offset + offset, + size); +#endif else map->virtual = ioremap(bo->mem.bus.offset + offset, size); @@ -491,6 +498,11 @@ int ttm_bo_vmap(struct ttm_buffer_object *bo, struct dma_buf_map *map) else if (mem->bus.caching == ttm_write_combined) vaddr_iomem = ioremap_wc(mem->bus.offset, bo->base.size); +#ifdef CONFIG_X86 + else if (mem->bus.caching == ttm_cached) + vaddr_iomem = ioremap_cache(mem->bus.offset, + bo->base.size); +#endif else vaddr_iomem = ioremap(mem->bus.offset, bo->base.size); @@ -602,7 +614,7 @@ static int ttm_bo_move_to_ghost(struct ttm_buffer_object *bo, static void ttm_bo_move_pipeline_evict(struct ttm_buffer_object *bo, struct dma_fence *fence) { - struct ttm_bo_device *bdev = bo->bdev; + struct ttm_device *bdev = bo->bdev; struct ttm_resource_manager *from = ttm_manager_type(bdev, bo->mem.mem_type); /** @@ -628,7 +640,7 @@ int ttm_bo_move_accel_cleanup(struct ttm_buffer_object *bo, bool pipeline, struct ttm_resource *new_mem) { - struct ttm_bo_device *bdev = bo->bdev; + struct ttm_device *bdev = bo->bdev; struct ttm_resource_manager *from = ttm_manager_type(bdev, bo->mem.mem_type); struct ttm_resource_manager *man = ttm_manager_type(bdev, new_mem->mem_type); int ret = 0; diff --git a/drivers/gpu/drm/ttm/ttm_bo_vm.c b/drivers/gpu/drm/ttm/ttm_bo_vm.c index 6dc96cf66744..b31b18058965 100644 --- a/drivers/gpu/drm/ttm/ttm_bo_vm.c +++ b/drivers/gpu/drm/ttm/ttm_bo_vm.c @@ -95,10 +95,10 @@ out_unlock: static unsigned long ttm_bo_io_mem_pfn(struct ttm_buffer_object *bo, unsigned long page_offset) { - struct ttm_bo_device *bdev = bo->bdev; + struct ttm_device *bdev = bo->bdev; - if (bdev->driver->io_mem_pfn) - return bdev->driver->io_mem_pfn(bo, page_offset); + if (bdev->funcs->io_mem_pfn) + return bdev->funcs->io_mem_pfn(bo, page_offset); return (bo->mem.bus.offset >> PAGE_SHIFT) + page_offset; } @@ -216,7 +216,7 @@ static vm_fault_t ttm_bo_vm_insert_huge(struct vm_fault *vmf, if (page_to_pfn(ttm->pages[page_offset + i]) != pfn + i) goto out_fallback; } - } else if (bo->bdev->driver->io_mem_pfn) { + } else if (bo->bdev->funcs->io_mem_pfn) { for (i = 1; i < fault_page_size; ++i) { if (ttm_bo_io_mem_pfn(bo, page_offset + i) != pfn + i) goto out_fallback; @@ -278,7 +278,7 @@ vm_fault_t ttm_bo_vm_fault_reserved(struct vm_fault *vmf, { struct vm_area_struct *vma = vmf->vma; struct ttm_buffer_object *bo = vma->vm_private_data; - struct ttm_bo_device *bdev = bo->bdev; + struct ttm_device *bdev = bo->bdev; unsigned long page_offset; unsigned long page_last; unsigned long pfn; @@ -488,8 +488,8 @@ int ttm_bo_vm_access(struct vm_area_struct *vma, unsigned long addr, ret = ttm_bo_vm_access_kmap(bo, offset, buf, len, write); break; default: - if (bo->bdev->driver->access_memory) - ret = bo->bdev->driver->access_memory( + if (bo->bdev->funcs->access_memory) + ret = bo->bdev->funcs->access_memory( bo, offset, buf, len, write); else ret = -EIO; @@ -508,7 +508,7 @@ static const struct vm_operations_struct ttm_bo_vm_ops = { .access = ttm_bo_vm_access, }; -static struct ttm_buffer_object *ttm_bo_vm_lookup(struct ttm_bo_device *bdev, +static struct ttm_buffer_object *ttm_bo_vm_lookup(struct ttm_device *bdev, unsigned long offset, unsigned long pages) { @@ -555,9 +555,8 @@ static void ttm_bo_mmap_vma_setup(struct ttm_buffer_object *bo, struct vm_area_s } int ttm_bo_mmap(struct file *filp, struct vm_area_struct *vma, - struct ttm_bo_device *bdev) + struct ttm_device *bdev) { - struct ttm_bo_driver *driver; struct ttm_buffer_object *bo; int ret; @@ -568,12 +567,11 @@ int ttm_bo_mmap(struct file *filp, struct vm_area_struct *vma, if (unlikely(!bo)) return -EINVAL; - driver = bo->bdev->driver; - if (unlikely(!driver->verify_access)) { + if (unlikely(!bo->bdev->funcs->verify_access)) { ret = -EPERM; goto out_unref; } - ret = driver->verify_access(bo, filp); + ret = bo->bdev->funcs->verify_access(bo, filp); if (unlikely(ret != 0)) goto out_unref; diff --git a/drivers/gpu/drm/ttm/ttm_device.c b/drivers/gpu/drm/ttm/ttm_device.c new file mode 100644 index 000000000000..9b787b3caeb5 --- /dev/null +++ b/drivers/gpu/drm/ttm/ttm_device.c @@ -0,0 +1,268 @@ +/* SPDX-License-Identifier: GPL-2.0 OR MIT */ + +/* + * Copyright (c) 2006-2009 VMware, Inc., Palo Alto, CA., USA + * Copyright 2020 Advanced Micro Devices, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + * Authors: Christian König + */ + +#define pr_fmt(fmt) "[TTM DEVICE] " fmt + +#include <linux/mm.h> + +#include <drm/ttm/ttm_device.h> +#include <drm/ttm/ttm_tt.h> +#include <drm/ttm/ttm_placement.h> +#include <drm/ttm/ttm_bo_api.h> + +#include "ttm_module.h" + +/** + * ttm_global_mutex - protecting the global state + */ +DEFINE_MUTEX(ttm_global_mutex); +unsigned ttm_glob_use_count; +struct ttm_global ttm_glob; +EXPORT_SYMBOL(ttm_glob); + +static void ttm_global_release(void) +{ + struct ttm_global *glob = &ttm_glob; + + mutex_lock(&ttm_global_mutex); + if (--ttm_glob_use_count > 0) + goto out; + + ttm_pool_mgr_fini(); + + __free_page(glob->dummy_read_page); + memset(glob, 0, sizeof(*glob)); +out: + mutex_unlock(&ttm_global_mutex); +} + +static int ttm_global_init(void) +{ + struct ttm_global *glob = &ttm_glob; + unsigned long num_pages, num_dma32; + struct sysinfo si; + int ret = 0; + + mutex_lock(&ttm_global_mutex); + if (++ttm_glob_use_count > 1) + goto out; + + si_meminfo(&si); + + /* Limit the number of pages in the pool to about 50% of the total + * system memory. + */ + num_pages = ((u64)si.totalram * si.mem_unit) >> PAGE_SHIFT; + num_pages /= 2; + + /* But for DMA32 we limit ourself to only use 2GiB maximum. */ + num_dma32 = (u64)(si.totalram - si.totalhigh) * si.mem_unit + >> PAGE_SHIFT; + num_dma32 = min(num_dma32, 2UL << (30 - PAGE_SHIFT)); + + ttm_pool_mgr_init(num_pages); + ttm_tt_mgr_init(num_pages, num_dma32); + + glob->dummy_read_page = alloc_page(__GFP_ZERO | GFP_DMA32); + + if (unlikely(glob->dummy_read_page == NULL)) { + ret = -ENOMEM; + goto out; + } + + INIT_LIST_HEAD(&glob->device_list); + atomic_set(&glob->bo_count, 0); + + debugfs_create_atomic_t("buffer_objects", 0444, ttm_debugfs_root, + &glob->bo_count); +out: + mutex_unlock(&ttm_global_mutex); + return ret; +} + +/** + * A buffer object shrink method that tries to swap out the first + * buffer object on the global::swap_lru list. + */ +int ttm_global_swapout(struct ttm_operation_ctx *ctx, gfp_t gfp_flags) +{ + struct ttm_global *glob = &ttm_glob; + struct ttm_device *bdev; + int ret = -EBUSY; + + mutex_lock(&ttm_global_mutex); + list_for_each_entry(bdev, &glob->device_list, device_list) { + ret = ttm_device_swapout(bdev, ctx, gfp_flags); + if (ret > 0) { + list_move_tail(&bdev->device_list, &glob->device_list); + break; + } + } + mutex_unlock(&ttm_global_mutex); + return ret; +} +EXPORT_SYMBOL(ttm_global_swapout); + +int ttm_device_swapout(struct ttm_device *bdev, struct ttm_operation_ctx *ctx, + gfp_t gfp_flags) +{ + struct ttm_resource_manager *man; + struct ttm_buffer_object *bo; + unsigned i, j; + int ret; + + spin_lock(&bdev->lru_lock); + for (i = TTM_PL_SYSTEM; i < TTM_NUM_MEM_TYPES; ++i) { + man = ttm_manager_type(bdev, i); + if (!man || !man->use_tt) + continue; + + for (j = 0; j < TTM_MAX_BO_PRIORITY; ++j) { + list_for_each_entry(bo, &man->lru[j], lru) { + uint32_t num_pages; + + if (!bo->ttm || + bo->ttm->page_flags & TTM_PAGE_FLAG_SG || + bo->ttm->page_flags & TTM_PAGE_FLAG_SWAPPED) + continue; + + num_pages = bo->ttm->num_pages; + ret = ttm_bo_swapout(bo, ctx, gfp_flags); + /* ttm_bo_swapout has dropped the lru_lock */ + if (!ret) + return num_pages; + if (ret != -EBUSY) + return ret; + } + } + } + spin_unlock(&bdev->lru_lock); + return 0; +} +EXPORT_SYMBOL(ttm_device_swapout); + +static void ttm_init_sysman(struct ttm_device *bdev) +{ + struct ttm_resource_manager *man = &bdev->sysman; + + /* + * Initialize the system memory buffer type. + * Other types need to be driver / IOCTL initialized. + */ + man->use_tt = true; + + ttm_resource_manager_init(man, 0); + ttm_set_driver_manager(bdev, TTM_PL_SYSTEM, man); + ttm_resource_manager_set_used(man, true); +} + +static void ttm_device_delayed_workqueue(struct work_struct *work) +{ + struct ttm_device *bdev = + container_of(work, struct ttm_device, wq.work); + + if (!ttm_bo_delayed_delete(bdev, false)) + schedule_delayed_work(&bdev->wq, + ((HZ / 100) < 1) ? 1 : HZ / 100); +} + +/** + * ttm_device_init + * + * @bdev: A pointer to a struct ttm_device to initialize. + * @funcs: Function table for the device. + * @dev: The core kernel device pointer for DMA mappings and allocations. + * @mapping: The address space to use for this bo. + * @vma_manager: A pointer to a vma manager. + * @use_dma_alloc: If coherent DMA allocation API should be used. + * @use_dma32: If we should use GFP_DMA32 for device memory allocations. + * + * Initializes a struct ttm_device: + * Returns: + * !0: Failure. + */ +int ttm_device_init(struct ttm_device *bdev, struct ttm_device_funcs *funcs, + struct device *dev, struct address_space *mapping, + struct drm_vma_offset_manager *vma_manager, + bool use_dma_alloc, bool use_dma32) +{ + struct ttm_global *glob = &ttm_glob; + int ret; + + if (WARN_ON(vma_manager == NULL)) + return -EINVAL; + + ret = ttm_global_init(); + if (ret) + return ret; + + bdev->funcs = funcs; + + ttm_init_sysman(bdev); + ttm_pool_init(&bdev->pool, dev, use_dma_alloc, use_dma32); + + bdev->vma_manager = vma_manager; + INIT_DELAYED_WORK(&bdev->wq, ttm_device_delayed_workqueue); + spin_lock_init(&bdev->lru_lock); + INIT_LIST_HEAD(&bdev->ddestroy); + bdev->dev_mapping = mapping; + mutex_lock(&ttm_global_mutex); + list_add_tail(&bdev->device_list, &glob->device_list); + mutex_unlock(&ttm_global_mutex); + + return 0; +} +EXPORT_SYMBOL(ttm_device_init); + +void ttm_device_fini(struct ttm_device *bdev) +{ + struct ttm_resource_manager *man; + unsigned i; + + man = ttm_manager_type(bdev, TTM_PL_SYSTEM); + ttm_resource_manager_set_used(man, false); + ttm_set_driver_manager(bdev, TTM_PL_SYSTEM, NULL); + + mutex_lock(&ttm_global_mutex); + list_del(&bdev->device_list); + mutex_unlock(&ttm_global_mutex); + + cancel_delayed_work_sync(&bdev->wq); + + if (ttm_bo_delayed_delete(bdev, true)) + pr_debug("Delayed destroy list was clean\n"); + + spin_lock(&bdev->lru_lock); + for (i = 0; i < TTM_MAX_BO_PRIORITY; ++i) + if (list_empty(&man->lru[0])) + pr_debug("Swap list %d was clean\n", i); + spin_unlock(&bdev->lru_lock); + + ttm_pool_fini(&bdev->pool); + ttm_global_release(); +} +EXPORT_SYMBOL(ttm_device_fini); diff --git a/drivers/gpu/drm/ttm/ttm_execbuf_util.c b/drivers/gpu/drm/ttm/ttm_execbuf_util.c index 9fa36ed59429..071c48d672c6 100644 --- a/drivers/gpu/drm/ttm/ttm_execbuf_util.c +++ b/drivers/gpu/drm/ttm/ttm_execbuf_util.c @@ -51,14 +51,12 @@ void ttm_eu_backoff_reservation(struct ww_acquire_ctx *ticket, if (list_empty(list)) return; - spin_lock(&ttm_bo_glob.lru_lock); list_for_each_entry(entry, list, head) { struct ttm_buffer_object *bo = entry->bo; - ttm_bo_move_to_lru_tail(bo, &bo->mem, NULL); + ttm_bo_move_to_lru_tail_unlocked(bo); dma_resv_unlock(bo->base.resv); } - spin_unlock(&ttm_bo_glob.lru_lock); if (ticket) ww_acquire_fini(ticket); @@ -154,7 +152,6 @@ void ttm_eu_fence_buffer_objects(struct ww_acquire_ctx *ticket, if (list_empty(list)) return; - spin_lock(&ttm_bo_glob.lru_lock); list_for_each_entry(entry, list, head) { struct ttm_buffer_object *bo = entry->bo; @@ -162,10 +159,9 @@ void ttm_eu_fence_buffer_objects(struct ww_acquire_ctx *ticket, dma_resv_add_shared_fence(bo->base.resv, fence); else dma_resv_add_excl_fence(bo->base.resv, fence); - ttm_bo_move_to_lru_tail(bo, &bo->mem, NULL); + ttm_bo_move_to_lru_tail_unlocked(bo); dma_resv_unlock(bo->base.resv); } - spin_unlock(&ttm_bo_glob.lru_lock); if (ticket) ww_acquire_fini(ticket); } diff --git a/drivers/gpu/drm/ttm/ttm_memory.c b/drivers/gpu/drm/ttm/ttm_memory.c deleted file mode 100644 index a3bfbd9cea68..000000000000 --- a/drivers/gpu/drm/ttm/ttm_memory.c +++ /dev/null @@ -1,683 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0 OR MIT */ -/************************************************************************** - * - * Copyright (c) 2006-2009 VMware, Inc., Palo Alto, CA., USA - * All Rights Reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a - * copy of this software and associated documentation files (the - * "Software"), to deal in the Software without restriction, including - * without limitation the rights to use, copy, modify, merge, publish, - * distribute, sub license, and/or sell copies of the Software, and to - * permit persons to whom the Software is furnished to do so, subject to - * the following conditions: - * - * The above copyright notice and this permission notice (including the - * next paragraph) shall be included in all copies or substantial portions - * of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL - * THE COPYRIGHT HOLDERS, AUTHORS AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM, - * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR - * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE - * USE OR OTHER DEALINGS IN THE SOFTWARE. - * - **************************************************************************/ - -#define pr_fmt(fmt) "[TTM] " fmt - -#include <drm/ttm/ttm_memory.h> -#include <linux/spinlock.h> -#include <linux/sched.h> -#include <linux/wait.h> -#include <linux/mm.h> -#include <linux/module.h> -#include <linux/slab.h> -#include <linux/swap.h> -#include <drm/ttm/ttm_pool.h> - -#include "ttm_module.h" - -#define TTM_MEMORY_ALLOC_RETRIES 4 - -struct ttm_mem_global ttm_mem_glob; -EXPORT_SYMBOL(ttm_mem_glob); - -struct ttm_mem_zone { - struct kobject kobj; - struct ttm_mem_global *glob; - const char *name; - uint64_t zone_mem; - uint64_t emer_mem; - uint64_t max_mem; - uint64_t swap_limit; - uint64_t used_mem; -}; - -static struct attribute ttm_mem_sys = { - .name = "zone_memory", - .mode = S_IRUGO -}; -static struct attribute ttm_mem_emer = { - .name = "emergency_memory", - .mode = S_IRUGO | S_IWUSR -}; -static struct attribute ttm_mem_max = { - .name = "available_memory", - .mode = S_IRUGO | S_IWUSR -}; -static struct attribute ttm_mem_swap = { - .name = "swap_limit", - .mode = S_IRUGO | S_IWUSR -}; -static struct attribute ttm_mem_used = { - .name = "used_memory", - .mode = S_IRUGO -}; - -static void ttm_mem_zone_kobj_release(struct kobject *kobj) -{ - struct ttm_mem_zone *zone = - container_of(kobj, struct ttm_mem_zone, kobj); - - pr_info("Zone %7s: Used memory at exit: %llu KiB\n", - zone->name, (unsigned long long)zone->used_mem >> 10); - kfree(zone); -} - -static ssize_t ttm_mem_zone_show(struct kobject *kobj, - struct attribute *attr, - char *buffer) -{ - struct ttm_mem_zone *zone = - container_of(kobj, struct ttm_mem_zone, kobj); - uint64_t val = 0; - - spin_lock(&zone->glob->lock); - if (attr == &ttm_mem_sys) - val = zone->zone_mem; - else if (attr == &ttm_mem_emer) - val = zone->emer_mem; - else if (attr == &ttm_mem_max) - val = zone->max_mem; - else if (attr == &ttm_mem_swap) - val = zone->swap_limit; - else if (attr == &ttm_mem_used) - val = zone->used_mem; - spin_unlock(&zone->glob->lock); - - return snprintf(buffer, PAGE_SIZE, "%llu\n", - (unsigned long long) val >> 10); -} - -static void ttm_check_swapping(struct ttm_mem_global *glob); - -static ssize_t ttm_mem_zone_store(struct kobject *kobj, - struct attribute *attr, - const char *buffer, - size_t size) -{ - struct ttm_mem_zone *zone = - container_of(kobj, struct ttm_mem_zone, kobj); - int chars; - unsigned long val; - uint64_t val64; - - chars = sscanf(buffer, "%lu", &val); - if (chars == 0) - return size; - - val64 = val; - val64 <<= 10; - - spin_lock(&zone->glob->lock); - if (val64 > zone->zone_mem) - val64 = zone->zone_mem; - if (attr == &ttm_mem_emer) { - zone->emer_mem = val64; - if (zone->max_mem > val64) - zone->max_mem = val64; - } else if (attr == &ttm_mem_max) { - zone->max_mem = val64; - if (zone->emer_mem < val64) - zone->emer_mem = val64; - } else if (attr == &ttm_mem_swap) - zone->swap_limit = val64; - spin_unlock(&zone->glob->lock); - - ttm_check_swapping(zone->glob); - - return size; -} - -static struct attribute *ttm_mem_zone_attrs[] = { - &ttm_mem_sys, - &ttm_mem_emer, - &ttm_mem_max, - &ttm_mem_swap, - &ttm_mem_used, - NULL -}; - -static const struct sysfs_ops ttm_mem_zone_ops = { - .show = &ttm_mem_zone_show, - .store = &ttm_mem_zone_store -}; - -static struct kobj_type ttm_mem_zone_kobj_type = { - .release = &ttm_mem_zone_kobj_release, - .sysfs_ops = &ttm_mem_zone_ops, - .default_attrs = ttm_mem_zone_attrs, -}; - -static struct attribute ttm_mem_global_lower_mem_limit = { - .name = "lower_mem_limit", - .mode = S_IRUGO | S_IWUSR -}; - -static ssize_t ttm_mem_global_show(struct kobject *kobj, - struct attribute *attr, - char *buffer) -{ - struct ttm_mem_global *glob = - container_of(kobj, struct ttm_mem_global, kobj); - uint64_t val = 0; - - spin_lock(&glob->lock); - val = glob->lower_mem_limit; - spin_unlock(&glob->lock); - /* convert from number of pages to KB */ - val <<= (PAGE_SHIFT - 10); - return snprintf(buffer, PAGE_SIZE, "%llu\n", - (unsigned long long) val); -} - -static ssize_t ttm_mem_global_store(struct kobject *kobj, - struct attribute *attr, - const char *buffer, - size_t size) -{ - int chars; - uint64_t val64; - unsigned long val; - struct ttm_mem_global *glob = - container_of(kobj, struct ttm_mem_global, kobj); - - chars = sscanf(buffer, "%lu", &val); - if (chars == 0) - return size; - - val64 = val; - /* convert from KB to number of pages */ - val64 >>= (PAGE_SHIFT - 10); - - spin_lock(&glob->lock); - glob->lower_mem_limit = val64; - spin_unlock(&glob->lock); - - return size; -} - -static struct attribute *ttm_mem_global_attrs[] = { - &ttm_mem_global_lower_mem_limit, - NULL -}; - -static const struct sysfs_ops ttm_mem_global_ops = { - .show = &ttm_mem_global_show, - .store = &ttm_mem_global_store, -}; - -static struct kobj_type ttm_mem_glob_kobj_type = { - .sysfs_ops = &ttm_mem_global_ops, - .default_attrs = ttm_mem_global_attrs, -}; - -static bool ttm_zones_above_swap_target(struct ttm_mem_global *glob, - bool from_wq, uint64_t extra) -{ - unsigned int i; - struct ttm_mem_zone *zone; - uint64_t target; - - for (i = 0; i < glob->num_zones; ++i) { - zone = glob->zones[i]; - - if (from_wq) - target = zone->swap_limit; - else if (capable(CAP_SYS_ADMIN)) - target = zone->emer_mem; - else - target = zone->max_mem; - - target = (extra > target) ? 0ULL : target; - - if (zone->used_mem > target) - return true; - } - return false; -} - -/* - * At this point we only support a single shrink callback. - * Extend this if needed, perhaps using a linked list of callbacks. - * Note that this function is reentrant: - * many threads may try to swap out at any given time. - */ - -static void ttm_shrink(struct ttm_mem_global *glob, bool from_wq, - uint64_t extra, struct ttm_operation_ctx *ctx) -{ - int ret; - - spin_lock(&glob->lock); - - while (ttm_zones_above_swap_target(glob, from_wq, extra)) { - spin_unlock(&glob->lock); - ret = ttm_bo_swapout(ctx); - spin_lock(&glob->lock); - if (unlikely(ret != 0)) - break; - } - - spin_unlock(&glob->lock); -} - -static void ttm_shrink_work(struct work_struct *work) -{ - struct ttm_operation_ctx ctx = { - .interruptible = false, - .no_wait_gpu = false - }; - struct ttm_mem_global *glob = - container_of(work, struct ttm_mem_global, work); - - ttm_shrink(glob, true, 0ULL, &ctx); -} - -static int ttm_mem_init_kernel_zone(struct ttm_mem_global *glob, - const struct sysinfo *si) -{ - struct ttm_mem_zone *zone = kzalloc(sizeof(*zone), GFP_KERNEL); - uint64_t mem; - int ret; - - if (unlikely(!zone)) - return -ENOMEM; - - mem = si->totalram - si->totalhigh; - mem *= si->mem_unit; - - zone->name = "kernel"; - zone->zone_mem = mem; - zone->max_mem = mem >> 1; - zone->emer_mem = (mem >> 1) + (mem >> 2); - zone->swap_limit = zone->max_mem - (mem >> 3); - zone->used_mem = 0; - zone->glob = glob; - glob->zone_kernel = zone; - ret = kobject_init_and_add( - &zone->kobj, &ttm_mem_zone_kobj_type, &glob->kobj, zone->name); - if (unlikely(ret != 0)) { - kobject_put(&zone->kobj); - return ret; - } - glob->zones[glob->num_zones++] = zone; - return 0; -} - -#ifdef CONFIG_HIGHMEM -static int ttm_mem_init_highmem_zone(struct ttm_mem_global *glob, - const struct sysinfo *si) -{ - struct ttm_mem_zone *zone; - uint64_t mem; - int ret; - - if (si->totalhigh == 0) - return 0; - - zone = kzalloc(sizeof(*zone), GFP_KERNEL); - if (unlikely(!zone)) - return -ENOMEM; - - mem = si->totalram; - mem *= si->mem_unit; - - zone->name = "highmem"; - zone->zone_mem = mem; - zone->max_mem = mem >> 1; - zone->emer_mem = (mem >> 1) + (mem >> 2); - zone->swap_limit = zone->max_mem - (mem >> 3); - zone->used_mem = 0; - zone->glob = glob; - glob->zone_highmem = zone; - ret = kobject_init_and_add( - &zone->kobj, &ttm_mem_zone_kobj_type, &glob->kobj, "%s", - zone->name); - if (unlikely(ret != 0)) { - kobject_put(&zone->kobj); - return ret; - } - glob->zones[glob->num_zones++] = zone; - return 0; -} -#else -static int ttm_mem_init_dma32_zone(struct ttm_mem_global *glob, - const struct sysinfo *si) -{ - struct ttm_mem_zone *zone = kzalloc(sizeof(*zone), GFP_KERNEL); - uint64_t mem; - int ret; - - if (unlikely(!zone)) - return -ENOMEM; - - mem = si->totalram; - mem *= si->mem_unit; - - /** - * No special dma32 zone needed. - */ - - if (mem <= ((uint64_t) 1ULL << 32)) { - kfree(zone); - return 0; - } - - /* - * Limit max dma32 memory to 4GB for now - * until we can figure out how big this - * zone really is. - */ - - mem = ((uint64_t) 1ULL << 32); - zone->name = "dma32"; - zone->zone_mem = mem; - zone->max_mem = mem >> 1; - zone->emer_mem = (mem >> 1) + (mem >> 2); - zone->swap_limit = zone->max_mem - (mem >> 3); - zone->used_mem = 0; - zone->glob = glob; - glob->zone_dma32 = zone; - ret = kobject_init_and_add( - &zone->kobj, &ttm_mem_zone_kobj_type, &glob->kobj, zone->name); - if (unlikely(ret != 0)) { - kobject_put(&zone->kobj); - return ret; - } - glob->zones[glob->num_zones++] = zone; - return 0; -} -#endif - -int ttm_mem_global_init(struct ttm_mem_global *glob) -{ - struct sysinfo si; - int ret; - int i; - struct ttm_mem_zone *zone; - - spin_lock_init(&glob->lock); - glob->swap_queue = create_singlethread_workqueue("ttm_swap"); - INIT_WORK(&glob->work, ttm_shrink_work); - ret = kobject_init_and_add( - &glob->kobj, &ttm_mem_glob_kobj_type, ttm_get_kobj(), "memory_accounting"); - if (unlikely(ret != 0)) { - kobject_put(&glob->kobj); - return ret; - } - - si_meminfo(&si); - - /* set it as 0 by default to keep original behavior of OOM */ - glob->lower_mem_limit = 0; - - ret = ttm_mem_init_kernel_zone(glob, &si); - if (unlikely(ret != 0)) - goto out_no_zone; -#ifdef CONFIG_HIGHMEM - ret = ttm_mem_init_highmem_zone(glob, &si); - if (unlikely(ret != 0)) - goto out_no_zone; -#else - ret = ttm_mem_init_dma32_zone(glob, &si); - if (unlikely(ret != 0)) - goto out_no_zone; -#endif - for (i = 0; i < glob->num_zones; ++i) { - zone = glob->zones[i]; - pr_info("Zone %7s: Available graphics memory: %llu KiB\n", - zone->name, (unsigned long long)zone->max_mem >> 10); - } - ttm_pool_mgr_init(glob->zone_kernel->max_mem/(2*PAGE_SIZE)); - return 0; -out_no_zone: - ttm_mem_global_release(glob); - return ret; -} - -void ttm_mem_global_release(struct ttm_mem_global *glob) -{ - struct ttm_mem_zone *zone; - unsigned int i; - - /* let the page allocator first stop the shrink work. */ - ttm_pool_mgr_fini(); - - flush_workqueue(glob->swap_queue); - destroy_workqueue(glob->swap_queue); - glob->swap_queue = NULL; - for (i = 0; i < glob->num_zones; ++i) { - zone = glob->zones[i]; - kobject_del(&zone->kobj); - kobject_put(&zone->kobj); - } - kobject_del(&glob->kobj); - kobject_put(&glob->kobj); - memset(glob, 0, sizeof(*glob)); -} - -static void ttm_check_swapping(struct ttm_mem_global *glob) -{ - bool needs_swapping = false; - unsigned int i; - struct ttm_mem_zone *zone; - - spin_lock(&glob->lock); - for (i = 0; i < glob->num_zones; ++i) { - zone = glob->zones[i]; - if (zone->used_mem > zone->swap_limit) { - needs_swapping = true; - break; - } - } - - spin_unlock(&glob->lock); - - if (unlikely(needs_swapping)) - (void)queue_work(glob->swap_queue, &glob->work); - -} - -static void ttm_mem_global_free_zone(struct ttm_mem_global *glob, - struct ttm_mem_zone *single_zone, - uint64_t amount) -{ - unsigned int i; - struct ttm_mem_zone *zone; - - spin_lock(&glob->lock); - for (i = 0; i < glob->num_zones; ++i) { - zone = glob->zones[i]; - if (single_zone && zone != single_zone) - continue; - zone->used_mem -= amount; - } - spin_unlock(&glob->lock); -} - -void ttm_mem_global_free(struct ttm_mem_global *glob, - uint64_t amount) -{ - return ttm_mem_global_free_zone(glob, glob->zone_kernel, amount); -} -EXPORT_SYMBOL(ttm_mem_global_free); - -/* - * check if the available mem is under lower memory limit - * - * a. if no swap disk at all or free swap space is under swap_mem_limit - * but available system mem is bigger than sys_mem_limit, allow TTM - * allocation; - * - * b. if the available system mem is less than sys_mem_limit but free - * swap disk is bigger than swap_mem_limit, allow TTM allocation. - */ -bool -ttm_check_under_lowerlimit(struct ttm_mem_global *glob, - uint64_t num_pages, - struct ttm_operation_ctx *ctx) -{ - int64_t available; - - /* We allow over commit during suspend */ - if (ctx->force_alloc) - return false; - - available = get_nr_swap_pages() + si_mem_available(); - available -= num_pages; - if (available < glob->lower_mem_limit) - return true; - - return false; -} - -static int ttm_mem_global_reserve(struct ttm_mem_global *glob, - struct ttm_mem_zone *single_zone, - uint64_t amount, bool reserve) -{ - uint64_t limit; - int ret = -ENOMEM; - unsigned int i; - struct ttm_mem_zone *zone; - - spin_lock(&glob->lock); - for (i = 0; i < glob->num_zones; ++i) { - zone = glob->zones[i]; - if (single_zone && zone != single_zone) - continue; - - limit = (capable(CAP_SYS_ADMIN)) ? - zone->emer_mem : zone->max_mem; - - if (zone->used_mem > limit) - goto out_unlock; - } - - if (reserve) { - for (i = 0; i < glob->num_zones; ++i) { - zone = glob->zones[i]; - if (single_zone && zone != single_zone) - continue; - zone->used_mem += amount; - } - } - - ret = 0; -out_unlock: - spin_unlock(&glob->lock); - ttm_check_swapping(glob); - - return ret; -} - - -static int ttm_mem_global_alloc_zone(struct ttm_mem_global *glob, - struct ttm_mem_zone *single_zone, - uint64_t memory, - struct ttm_operation_ctx *ctx) -{ - int count = TTM_MEMORY_ALLOC_RETRIES; - - while (unlikely(ttm_mem_global_reserve(glob, - single_zone, - memory, true) - != 0)) { - if (ctx->no_wait_gpu) - return -ENOMEM; - if (unlikely(count-- == 0)) - return -ENOMEM; - ttm_shrink(glob, false, memory + (memory >> 2) + 16, ctx); - } - - return 0; -} - -int ttm_mem_global_alloc(struct ttm_mem_global *glob, uint64_t memory, - struct ttm_operation_ctx *ctx) -{ - /** - * Normal allocations of kernel memory are registered in - * the kernel zone. - */ - - return ttm_mem_global_alloc_zone(glob, glob->zone_kernel, memory, ctx); -} -EXPORT_SYMBOL(ttm_mem_global_alloc); - -int ttm_mem_global_alloc_page(struct ttm_mem_global *glob, - struct page *page, uint64_t size, - struct ttm_operation_ctx *ctx) -{ - struct ttm_mem_zone *zone = NULL; - - /** - * Page allocations may be registed in a single zone - * only if highmem or !dma32. - */ - -#ifdef CONFIG_HIGHMEM - if (PageHighMem(page) && glob->zone_highmem != NULL) - zone = glob->zone_highmem; -#else - if (glob->zone_dma32 && page_to_pfn(page) > 0x00100000UL) - zone = glob->zone_kernel; -#endif - return ttm_mem_global_alloc_zone(glob, zone, size, ctx); -} - -void ttm_mem_global_free_page(struct ttm_mem_global *glob, struct page *page, - uint64_t size) -{ - struct ttm_mem_zone *zone = NULL; - -#ifdef CONFIG_HIGHMEM - if (PageHighMem(page) && glob->zone_highmem != NULL) - zone = glob->zone_highmem; -#else - if (glob->zone_dma32 && page_to_pfn(page) > 0x00100000UL) - zone = glob->zone_kernel; -#endif - ttm_mem_global_free_zone(glob, zone, size); -} - -size_t ttm_round_pot(size_t size) -{ - if ((size & (size - 1)) == 0) - return size; - else if (size > PAGE_SIZE) - return PAGE_ALIGN(size); - else { - size_t tmp_size = 4; - - while (tmp_size < size) - tmp_size <<= 1; - - return tmp_size; - } - return 0; -} -EXPORT_SYMBOL(ttm_round_pot); diff --git a/drivers/gpu/drm/ttm/ttm_module.c b/drivers/gpu/drm/ttm/ttm_module.c index c0906437cb1c..56b0efdba1a9 100644 --- a/drivers/gpu/drm/ttm/ttm_module.c +++ b/drivers/gpu/drm/ttm/ttm_module.c @@ -32,68 +32,22 @@ #include <linux/module.h> #include <linux/device.h> #include <linux/sched.h> +#include <linux/debugfs.h> #include <drm/drm_sysfs.h> #include "ttm_module.h" -static DECLARE_WAIT_QUEUE_HEAD(exit_q); -static atomic_t device_released; - -static struct device_type ttm_drm_class_type = { - .name = "ttm", - /** - * Add pm ops here. - */ -}; - -static void ttm_drm_class_device_release(struct device *dev) -{ - atomic_set(&device_released, 1); - wake_up_all(&exit_q); -} - -static struct device ttm_drm_class_device = { - .type = &ttm_drm_class_type, - .release = &ttm_drm_class_device_release -}; - -struct kobject *ttm_get_kobj(void) -{ - struct kobject *kobj = &ttm_drm_class_device.kobj; - BUG_ON(kobj == NULL); - return kobj; -} +struct dentry *ttm_debugfs_root; static int __init ttm_init(void) { - int ret; - - ret = dev_set_name(&ttm_drm_class_device, "ttm"); - if (unlikely(ret != 0)) - return ret; - - atomic_set(&device_released, 0); - ret = drm_class_device_register(&ttm_drm_class_device); - if (unlikely(ret != 0)) - goto out_no_dev_reg; - + ttm_debugfs_root = debugfs_create_dir("ttm", NULL); return 0; -out_no_dev_reg: - atomic_set(&device_released, 1); - wake_up_all(&exit_q); - return ret; } static void __exit ttm_exit(void) { - drm_class_device_unregister(&ttm_drm_class_device); - - /** - * Refuse to unload until the TTM device is released. - * Not sure this is 100% needed. - */ - - wait_event(exit_q, atomic_read(&device_released) == 1); + debugfs_remove(ttm_debugfs_root); } module_init(ttm_init); diff --git a/drivers/gpu/drm/ttm/ttm_module.h b/drivers/gpu/drm/ttm/ttm_module.h index 45fa318c1585..d7cac5d4b835 100644 --- a/drivers/gpu/drm/ttm/ttm_module.h +++ b/drivers/gpu/drm/ttm/ttm_module.h @@ -31,10 +31,10 @@ #ifndef _TTM_MODULE_H_ #define _TTM_MODULE_H_ -#include <linux/kernel.h> -struct kobject; - #define TTM_PFX "[TTM] " -extern struct kobject *ttm_get_kobj(void); + +struct dentry; + +extern struct dentry *ttm_debugfs_root; #endif /* _TTM_MODULE_H_ */ diff --git a/drivers/gpu/drm/ttm/ttm_pool.c b/drivers/gpu/drm/ttm/ttm_pool.c index 4eb6efb8b8c0..cb38b1a17b09 100644 --- a/drivers/gpu/drm/ttm/ttm_pool.c +++ b/drivers/gpu/drm/ttm/ttm_pool.c @@ -34,6 +34,7 @@ #include <linux/module.h> #include <linux/dma-mapping.h> #include <linux/highmem.h> +#include <linux/sched/mm.h> #ifdef CONFIG_X86 #include <asm/set_memory.h> @@ -43,6 +44,8 @@ #include <drm/ttm/ttm_bo_driver.h> #include <drm/ttm/ttm_tt.h> +#include "ttm_module.h" + /** * struct ttm_pool_dma - Helper object for coherent DMA mappings * @@ -412,16 +415,10 @@ int ttm_pool_alloc(struct ttm_pool *pool, struct ttm_tt *tt, caching = pages + (1 << order); } - r = ttm_mem_global_alloc_page(&ttm_mem_glob, p, - (1 << order) * PAGE_SIZE, - ctx); - if (r) - goto error_free_page; - if (dma_addr) { r = ttm_pool_map(pool, order, p, &dma_addr); if (r) - goto error_global_free; + goto error_free_page; } num_pages -= 1 << order; @@ -435,9 +432,6 @@ int ttm_pool_alloc(struct ttm_pool *pool, struct ttm_tt *tt, return 0; -error_global_free: - ttm_mem_global_free_page(&ttm_mem_glob, p, (1 << order) * PAGE_SIZE); - error_free_page: ttm_pool_free_page(pool, tt->caching, order, p); @@ -472,8 +466,6 @@ void ttm_pool_free(struct ttm_pool *pool, struct ttm_tt *tt) order = ttm_pool_page_order(pool, p); num_pages = 1ULL << order; - ttm_mem_global_free_page(&ttm_mem_glob, p, - num_pages * PAGE_SIZE); if (tt->dma_address) ttm_pool_unmap(pool, tt->dma_address[i], num_pages); @@ -513,10 +505,12 @@ void ttm_pool_init(struct ttm_pool *pool, struct device *dev, pool->use_dma_alloc = use_dma_alloc; pool->use_dma32 = use_dma32; - for (i = 0; i < TTM_NUM_CACHING_TYPES; ++i) - for (j = 0; j < MAX_ORDER; ++j) - ttm_pool_type_init(&pool->caching[i].orders[j], - pool, i, j); + if (use_dma_alloc) { + for (i = 0; i < TTM_NUM_CACHING_TYPES; ++i) + for (j = 0; j < MAX_ORDER; ++j) + ttm_pool_type_init(&pool->caching[i].orders[j], + pool, i, j); + } } /** @@ -531,9 +525,33 @@ void ttm_pool_fini(struct ttm_pool *pool) { unsigned int i, j; - for (i = 0; i < TTM_NUM_CACHING_TYPES; ++i) - for (j = 0; j < MAX_ORDER; ++j) - ttm_pool_type_fini(&pool->caching[i].orders[j]); + if (pool->use_dma_alloc) { + for (i = 0; i < TTM_NUM_CACHING_TYPES; ++i) + for (j = 0; j < MAX_ORDER; ++j) + ttm_pool_type_fini(&pool->caching[i].orders[j]); + } +} + +/* As long as pages are available make sure to release at least one */ +static unsigned long ttm_pool_shrinker_scan(struct shrinker *shrink, + struct shrink_control *sc) +{ + unsigned long num_freed = 0; + + do + num_freed += ttm_pool_shrink(); + while (!num_freed && atomic_long_read(&allocated_pages)); + + return num_freed; +} + +/* Return the number of pages available or SHRINK_EMPTY if we have none */ +static unsigned long ttm_pool_shrinker_count(struct shrinker *shrink, + struct shrink_control *sc) +{ + unsigned long num_pages = atomic_long_read(&allocated_pages); + + return num_pages ? num_pages : SHRINK_EMPTY; } #ifdef CONFIG_DEBUG_FS @@ -552,6 +570,17 @@ static unsigned int ttm_pool_type_count(struct ttm_pool_type *pt) return count; } +/* Print a nice header for the order */ +static void ttm_pool_debugfs_header(struct seq_file *m) +{ + unsigned int i; + + seq_puts(m, "\t "); + for (i = 0; i < MAX_ORDER; ++i) + seq_printf(m, " ---%2u---", i); + seq_puts(m, "\n"); +} + /* Dump information about the different pool types */ static void ttm_pool_debugfs_orders(struct ttm_pool_type *pt, struct seq_file *m) @@ -563,6 +592,35 @@ static void ttm_pool_debugfs_orders(struct ttm_pool_type *pt, seq_puts(m, "\n"); } +/* Dump the total amount of allocated pages */ +static void ttm_pool_debugfs_footer(struct seq_file *m) +{ + seq_printf(m, "\ntotal\t: %8lu of %8lu\n", + atomic_long_read(&allocated_pages), page_pool_size); +} + +/* Dump the information for the global pools */ +static int ttm_pool_debugfs_globals_show(struct seq_file *m, void *data) +{ + ttm_pool_debugfs_header(m); + + mutex_lock(&shrinker_lock); + seq_puts(m, "wc\t:"); + ttm_pool_debugfs_orders(global_write_combined, m); + seq_puts(m, "uc\t:"); + ttm_pool_debugfs_orders(global_uncached, m); + seq_puts(m, "wc 32\t:"); + ttm_pool_debugfs_orders(global_dma32_write_combined, m); + seq_puts(m, "uc 32\t:"); + ttm_pool_debugfs_orders(global_dma32_uncached, m); + mutex_unlock(&shrinker_lock); + + ttm_pool_debugfs_footer(m); + + return 0; +} +DEFINE_SHOW_ATTRIBUTE(ttm_pool_debugfs_globals); + /** * ttm_pool_debugfs - Debugfs dump function for a pool * @@ -575,23 +633,14 @@ int ttm_pool_debugfs(struct ttm_pool *pool, struct seq_file *m) { unsigned int i; - mutex_lock(&shrinker_lock); - - seq_puts(m, "\t "); - for (i = 0; i < MAX_ORDER; ++i) - seq_printf(m, " ---%2u---", i); - seq_puts(m, "\n"); - - seq_puts(m, "wc\t:"); - ttm_pool_debugfs_orders(global_write_combined, m); - seq_puts(m, "uc\t:"); - ttm_pool_debugfs_orders(global_uncached, m); + if (!pool->use_dma_alloc) { + seq_puts(m, "unused\n"); + return 0; + } - seq_puts(m, "wc 32\t:"); - ttm_pool_debugfs_orders(global_dma32_write_combined, m); - seq_puts(m, "uc 32\t:"); - ttm_pool_debugfs_orders(global_dma32_uncached, m); + ttm_pool_debugfs_header(m); + mutex_lock(&shrinker_lock); for (i = 0; i < TTM_NUM_CACHING_TYPES; ++i) { seq_puts(m, "DMA "); switch (i) { @@ -607,39 +656,28 @@ int ttm_pool_debugfs(struct ttm_pool *pool, struct seq_file *m) } ttm_pool_debugfs_orders(pool->caching[i].orders, m); } - - seq_printf(m, "\ntotal\t: %8lu of %8lu\n", - atomic_long_read(&allocated_pages), page_pool_size); - mutex_unlock(&shrinker_lock); + ttm_pool_debugfs_footer(m); return 0; } EXPORT_SYMBOL(ttm_pool_debugfs); -#endif - -/* As long as pages are available make sure to release at least one */ -static unsigned long ttm_pool_shrinker_scan(struct shrinker *shrink, - struct shrink_control *sc) +/* Test the shrinker functions and dump the result */ +static int ttm_pool_debugfs_shrink_show(struct seq_file *m, void *data) { - unsigned long num_freed = 0; + struct shrink_control sc = { .gfp_mask = GFP_NOFS }; - do - num_freed += ttm_pool_shrink(); - while (!num_freed && atomic_long_read(&allocated_pages)); + fs_reclaim_acquire(GFP_KERNEL); + seq_printf(m, "%lu/%lu\n", ttm_pool_shrinker_count(&mm_shrinker, &sc), + ttm_pool_shrinker_scan(&mm_shrinker, &sc)); + fs_reclaim_release(GFP_KERNEL); - return num_freed; + return 0; } +DEFINE_SHOW_ATTRIBUTE(ttm_pool_debugfs_shrink); -/* Return the number of pages available or SHRINK_EMPTY if we have none */ -static unsigned long ttm_pool_shrinker_count(struct shrinker *shrink, - struct shrink_control *sc) -{ - unsigned long num_pages = atomic_long_read(&allocated_pages); - - return num_pages ? num_pages : SHRINK_EMPTY; -} +#endif /** * ttm_pool_mgr_init - Initialize globals @@ -669,6 +707,13 @@ int ttm_pool_mgr_init(unsigned long num_pages) ttm_uncached, i); } +#ifdef CONFIG_DEBUG_FS + debugfs_create_file("page_pool", 0444, ttm_debugfs_root, NULL, + &ttm_pool_debugfs_globals_fops); + debugfs_create_file("page_pool_shrink", 0400, ttm_debugfs_root, NULL, + &ttm_pool_debugfs_shrink_fops); +#endif + mm_shrinker.count_objects = ttm_pool_shrinker_count; mm_shrinker.scan_objects = ttm_pool_shrinker_scan; mm_shrinker.seeks = 1; diff --git a/drivers/gpu/drm/ttm/ttm_range_manager.c b/drivers/gpu/drm/ttm/ttm_range_manager.c index a39305f742da..707e5c152896 100644 --- a/drivers/gpu/drm/ttm/ttm_range_manager.c +++ b/drivers/gpu/drm/ttm/ttm_range_manager.c @@ -111,7 +111,7 @@ static void ttm_range_man_free(struct ttm_resource_manager *man, static const struct ttm_resource_manager_func ttm_range_manager_func; -int ttm_range_man_init(struct ttm_bo_device *bdev, +int ttm_range_man_init(struct ttm_device *bdev, unsigned type, bool use_tt, unsigned long p_size) { @@ -138,7 +138,7 @@ int ttm_range_man_init(struct ttm_bo_device *bdev, } EXPORT_SYMBOL(ttm_range_man_init); -int ttm_range_man_fini(struct ttm_bo_device *bdev, +int ttm_range_man_fini(struct ttm_device *bdev, unsigned type) { struct ttm_resource_manager *man = ttm_manager_type(bdev, type); diff --git a/drivers/gpu/drm/ttm/ttm_resource.c b/drivers/gpu/drm/ttm/ttm_resource.c index b60699bf4816..04f2eef653ab 100644 --- a/drivers/gpu/drm/ttm/ttm_resource.c +++ b/drivers/gpu/drm/ttm/ttm_resource.c @@ -83,7 +83,7 @@ EXPORT_SYMBOL(ttm_resource_manager_init); * Evict all the objects out of a memory manager until it is empty. * Part of memory manager cleanup sequence. */ -int ttm_resource_manager_evict_all(struct ttm_bo_device *bdev, +int ttm_resource_manager_evict_all(struct ttm_device *bdev, struct ttm_resource_manager *man) { struct ttm_operation_ctx ctx = { @@ -91,7 +91,6 @@ int ttm_resource_manager_evict_all(struct ttm_bo_device *bdev, .no_wait_gpu = false, .force_alloc = true }; - struct ttm_bo_global *glob = &ttm_bo_glob; struct dma_fence *fence; int ret; unsigned i; @@ -100,18 +99,18 @@ int ttm_resource_manager_evict_all(struct ttm_bo_device *bdev, * Can't use standard list traversal since we're unlocking. */ - spin_lock(&glob->lru_lock); + spin_lock(&bdev->lru_lock); for (i = 0; i < TTM_MAX_BO_PRIORITY; ++i) { while (!list_empty(&man->lru[i])) { - spin_unlock(&glob->lru_lock); + spin_unlock(&bdev->lru_lock); ret = ttm_mem_evict_first(bdev, man, NULL, &ctx, NULL); if (ret) return ret; - spin_lock(&glob->lru_lock); + spin_lock(&bdev->lru_lock); } } - spin_unlock(&glob->lru_lock); + spin_unlock(&bdev->lru_lock); spin_lock(&man->move_lock); fence = dma_fence_get(man->move); diff --git a/drivers/gpu/drm/ttm/ttm_tt.c b/drivers/gpu/drm/ttm/ttm_tt.c index 7f75a13163f0..eecc930e97ab 100644 --- a/drivers/gpu/drm/ttm/ttm_tt.c +++ b/drivers/gpu/drm/ttm/ttm_tt.c @@ -38,12 +38,27 @@ #include <drm/drm_cache.h> #include <drm/ttm/ttm_bo_driver.h> +#include "ttm_module.h" + +static unsigned long ttm_pages_limit; + +MODULE_PARM_DESC(pages_limit, "Limit for the allocated pages"); +module_param_named(pages_limit, ttm_pages_limit, ulong, 0644); + +static unsigned long ttm_dma32_pages_limit; + +MODULE_PARM_DESC(dma32_pages_limit, "Limit for the allocated DMA32 pages"); +module_param_named(dma32_pages_limit, ttm_dma32_pages_limit, ulong, 0644); + +static atomic_long_t ttm_pages_allocated; +static atomic_long_t ttm_dma32_pages_allocated; + /* * Allocates a ttm structure for the given BO. */ int ttm_tt_create(struct ttm_buffer_object *bo, bool zero_alloc) { - struct ttm_bo_device *bdev = bo->bdev; + struct ttm_device *bdev = bo->bdev; uint32_t page_flags = 0; dma_resv_assert_held(bo->base.resv); @@ -66,7 +81,7 @@ int ttm_tt_create(struct ttm_buffer_object *bo, bool zero_alloc) return -EINVAL; } - bo->ttm = bdev->driver->ttm_tt_create(bo, page_flags); + bo->ttm = bdev->funcs->ttm_tt_create(bo, page_flags); if (unlikely(bo->ttm == NULL)) return -ENOMEM; @@ -108,7 +123,7 @@ static int ttm_sg_tt_alloc_page_directory(struct ttm_tt *ttm) return 0; } -void ttm_tt_destroy_common(struct ttm_bo_device *bdev, struct ttm_tt *ttm) +void ttm_tt_destroy_common(struct ttm_device *bdev, struct ttm_tt *ttm) { ttm_tt_unpopulate(bdev, ttm); @@ -119,9 +134,9 @@ void ttm_tt_destroy_common(struct ttm_bo_device *bdev, struct ttm_tt *ttm) } EXPORT_SYMBOL(ttm_tt_destroy_common); -void ttm_tt_destroy(struct ttm_bo_device *bdev, struct ttm_tt *ttm) +void ttm_tt_destroy(struct ttm_device *bdev, struct ttm_tt *ttm) { - bdev->driver->ttm_tt_destroy(bdev, ttm); + bdev->funcs->ttm_tt_destroy(bdev, ttm); } static void ttm_tt_init_fields(struct ttm_tt *ttm, @@ -223,32 +238,41 @@ out_err: return ret; } -int ttm_tt_swapout(struct ttm_bo_device *bdev, struct ttm_tt *ttm) +/** + * ttm_tt_swapout - swap out tt object + * + * @bdev: TTM device structure. + * @ttm: The struct ttm_tt. + * @gfp_flags: Flags to use for memory allocation. + * + * Swapout a TT object to a shmem_file, return number of pages swapped out or + * negative error code. + */ +int ttm_tt_swapout(struct ttm_device *bdev, struct ttm_tt *ttm, + gfp_t gfp_flags) { + loff_t size = (loff_t)ttm->num_pages << PAGE_SHIFT; struct address_space *swap_space; struct file *swap_storage; struct page *from_page; struct page *to_page; - gfp_t gfp_mask; int i, ret; - swap_storage = shmem_file_setup("ttm swap", - ttm->num_pages << PAGE_SHIFT, - 0); + swap_storage = shmem_file_setup("ttm swap", size, 0); if (IS_ERR(swap_storage)) { pr_err("Failed allocating swap storage\n"); return PTR_ERR(swap_storage); } swap_space = swap_storage->f_mapping; - gfp_mask = mapping_gfp_mask(swap_space); + gfp_flags &= mapping_gfp_mask(swap_space); for (i = 0; i < ttm->num_pages; ++i) { from_page = ttm->pages[i]; if (unlikely(from_page == NULL)) continue; - to_page = shmem_read_mapping_page_gfp(swap_space, i, gfp_mask); + to_page = shmem_read_mapping_page_gfp(swap_space, i, gfp_flags); if (IS_ERR(to_page)) { ret = PTR_ERR(to_page); goto out_err; @@ -263,7 +287,7 @@ int ttm_tt_swapout(struct ttm_bo_device *bdev, struct ttm_tt *ttm) ttm->swap_storage = swap_storage; ttm->page_flags |= TTM_PAGE_FLAG_SWAPPED; - return 0; + return ttm->num_pages; out_err: fput(swap_storage); @@ -271,7 +295,7 @@ out_err: return ret; } -static void ttm_tt_add_mapping(struct ttm_bo_device *bdev, struct ttm_tt *ttm) +static void ttm_tt_add_mapping(struct ttm_device *bdev, struct ttm_tt *ttm) { pgoff_t i; @@ -282,7 +306,7 @@ static void ttm_tt_add_mapping(struct ttm_bo_device *bdev, struct ttm_tt *ttm) ttm->pages[i]->mapping = bdev->dev_mapping; } -int ttm_tt_populate(struct ttm_bo_device *bdev, +int ttm_tt_populate(struct ttm_device *bdev, struct ttm_tt *ttm, struct ttm_operation_ctx *ctx) { int ret; @@ -293,12 +317,28 @@ int ttm_tt_populate(struct ttm_bo_device *bdev, if (ttm_tt_is_populated(ttm)) return 0; - if (bdev->driver->ttm_tt_populate) - ret = bdev->driver->ttm_tt_populate(bdev, ttm, ctx); + if (!(ttm->page_flags & TTM_PAGE_FLAG_SG)) { + atomic_long_add(ttm->num_pages, &ttm_pages_allocated); + if (bdev->pool.use_dma32) + atomic_long_add(ttm->num_pages, + &ttm_dma32_pages_allocated); + } + + while (atomic_long_read(&ttm_pages_allocated) > ttm_pages_limit || + atomic_long_read(&ttm_dma32_pages_allocated) > + ttm_dma32_pages_limit) { + + ret = ttm_global_swapout(ctx, GFP_KERNEL); + if (ret < 0) + goto error; + } + + if (bdev->funcs->ttm_tt_populate) + ret = bdev->funcs->ttm_tt_populate(bdev, ttm, ctx); else ret = ttm_pool_alloc(&bdev->pool, ttm, ctx); if (ret) - return ret; + goto error; ttm_tt_add_mapping(bdev, ttm); ttm->page_flags |= TTM_PAGE_FLAG_PRIV_POPULATED; @@ -311,6 +351,15 @@ int ttm_tt_populate(struct ttm_bo_device *bdev, } return 0; + +error: + if (!(ttm->page_flags & TTM_PAGE_FLAG_SG)) { + atomic_long_sub(ttm->num_pages, &ttm_pages_allocated); + if (bdev->pool.use_dma32) + atomic_long_sub(ttm->num_pages, + &ttm_dma32_pages_allocated); + } + return ret; } EXPORT_SYMBOL(ttm_tt_populate); @@ -328,16 +377,37 @@ static void ttm_tt_clear_mapping(struct ttm_tt *ttm) } } -void ttm_tt_unpopulate(struct ttm_bo_device *bdev, - struct ttm_tt *ttm) +void ttm_tt_unpopulate(struct ttm_device *bdev, struct ttm_tt *ttm) { if (!ttm_tt_is_populated(ttm)) return; ttm_tt_clear_mapping(ttm); - if (bdev->driver->ttm_tt_unpopulate) - bdev->driver->ttm_tt_unpopulate(bdev, ttm); + if (bdev->funcs->ttm_tt_unpopulate) + bdev->funcs->ttm_tt_unpopulate(bdev, ttm); else ttm_pool_free(&bdev->pool, ttm); + + if (!(ttm->page_flags & TTM_PAGE_FLAG_SG)) { + atomic_long_sub(ttm->num_pages, &ttm_pages_allocated); + if (bdev->pool.use_dma32) + atomic_long_sub(ttm->num_pages, + &ttm_dma32_pages_allocated); + } + ttm->page_flags &= ~TTM_PAGE_FLAG_PRIV_POPULATED; } + +/** + * ttm_tt_mgr_init - register with the MM shrinker + * + * Register with the MM shrinker for swapping out BOs. + */ +void ttm_tt_mgr_init(unsigned long num_pages, unsigned long num_dma32_pages) +{ + if (!ttm_pages_limit) + ttm_pages_limit = num_pages; + + if (!ttm_dma32_pages_limit) + ttm_dma32_pages_limit = num_dma32_pages; +} |