From c1b69ed0c62f9d86599600f4c1a3bd82db1b7362 Mon Sep 17 00:00:00 2001 From: Chunming Zhou Date: Tue, 21 Jul 2015 13:45:14 +0800 Subject: drm/amdgpu: add backend implementation of gpu scheduler (v2) v2: fix rebase breakage Signed-off-by: Chunming Zhou Acked-by: Christian K?nig Reviewed-by: Jammy Zhou --- drivers/gpu/drm/amd/amdgpu/amdgpu_sched.c | 107 ++++++++++++++++++++++++++++++ 1 file changed, 107 insertions(+) create mode 100644 drivers/gpu/drm/amd/amdgpu/amdgpu_sched.c (limited to 'drivers/gpu/drm/amd/amdgpu/amdgpu_sched.c') diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_sched.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_sched.c new file mode 100644 index 000000000000..1f7bf31da7fc --- /dev/null +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_sched.c @@ -0,0 +1,107 @@ +/* + * Copyright 2015 Advanced Micro Devices, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + * + */ +#include +#include +#include +#include +#include "amdgpu.h" + +static int amdgpu_sched_prepare_job(struct amd_gpu_scheduler *sched, + struct amd_context_entity *c_entity, + void *job) +{ + int r = 0; + struct amdgpu_cs_parser *sched_job = (struct amdgpu_cs_parser *)job; + if (sched_job->prepare_job) + r = sched_job->prepare_job(sched_job); + if (r) { + DRM_ERROR("Prepare job error\n"); + schedule_work(&sched_job->job_work); + } + return r; +} + +static void amdgpu_sched_run_job(struct amd_gpu_scheduler *sched, + struct amd_context_entity *c_entity, + void *job) +{ + int r = 0; + struct amdgpu_cs_parser *sched_job = (struct amdgpu_cs_parser *)job; + + mutex_lock(&sched_job->job_lock); + r = amdgpu_ib_schedule(sched_job->adev, + sched_job->num_ibs, + sched_job->ibs, + sched_job->filp); + if (r) + goto err; + + if (sched_job->run_job) { + r = sched_job->run_job(sched_job); + if (r) + goto err; + } + mutex_unlock(&sched_job->job_lock); + return; +err: + DRM_ERROR("Run job error\n"); + mutex_unlock(&sched_job->job_lock); + schedule_work(&sched_job->job_work); +} + +static void amdgpu_sched_process_job(struct amd_gpu_scheduler *sched, void *job) +{ + struct amdgpu_cs_parser *sched_job = NULL; + struct amdgpu_fence *fence = NULL; + struct amdgpu_ring *ring = NULL; + struct amdgpu_device *adev = NULL; + struct amd_context_entity *c_entity = NULL; + + if (!job) + return; + sched_job = (struct amdgpu_cs_parser *)job; + fence = sched_job->ibs[sched_job->num_ibs - 1].fence; + if (!fence) + return; + ring = fence->ring; + adev = ring->adev; + + if (sched_job->ctx) { + c_entity = &sched_job->ctx->rings[ring->idx].c_entity; + atomic64_set(&c_entity->last_signaled_v_seq, + sched_job->uf.sequence); + } + + /* wake up users waiting for time stamp */ + wake_up_all(&c_entity->wait_queue); + + schedule_work(&sched_job->job_work); +} + +struct amd_sched_backend_ops amdgpu_sched_ops = { + .prepare_job = amdgpu_sched_prepare_job, + .run_job = amdgpu_sched_run_job, + .process_job = amdgpu_sched_process_job +}; + -- cgit From 4b559c90bc1870313f02cceef680884519af6b2b Mon Sep 17 00:00:00 2001 From: Chunming Zhou Date: Tue, 21 Jul 2015 15:53:04 +0800 Subject: drm/amdgpu: make sure the fence is emitted before ring to get it. Signed-off-by: Chunming Zhou Acked-by: Christian K?nig Reviewed-by: Jammy Zhou --- drivers/gpu/drm/amd/amdgpu/amdgpu.h | 2 ++ drivers/gpu/drm/amd/amdgpu/amdgpu_cs.c | 26 +++++++++----------------- drivers/gpu/drm/amd/amdgpu/amdgpu_ctx.c | 10 ++++++++++ drivers/gpu/drm/amd/amdgpu/amdgpu_sched.c | 5 ++++- 4 files changed, 25 insertions(+), 18 deletions(-) (limited to 'drivers/gpu/drm/amd/amdgpu/amdgpu_sched.c') diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu.h b/drivers/gpu/drm/amd/amdgpu/amdgpu.h index becb26317467..127867c2fc37 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu.h +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu.h @@ -81,6 +81,7 @@ extern int amdgpu_vm_size; extern int amdgpu_vm_block_size; extern int amdgpu_enable_scheduler; +#define AMDGPU_WAIT_IDLE_TIMEOUT_IN_MS 3000 #define AMDGPU_MAX_USEC_TIMEOUT 100000 /* 100 ms */ #define AMDGPU_FENCE_JIFFIES_TIMEOUT (HZ / 2) /* AMDGPU_IB_POOL_SIZE must be a power of 2 */ @@ -1239,6 +1240,7 @@ struct amdgpu_cs_parser { /* user fence */ struct amdgpu_user_fence uf; + struct amdgpu_ring *ring; struct mutex job_lock; struct work_struct job_work; int (*prepare_job)(struct amdgpu_cs_parser *sched_job); diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_cs.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_cs.c index f9d4fe985668..5f2403898b06 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_cs.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_cs.c @@ -915,7 +915,7 @@ int amdgpu_cs_ioctl(struct drm_device *dev, void *data, struct drm_file *filp) goto out; } else parser->prepare_job = amdgpu_cs_parser_prepare_job; - + parser->ring = ring; parser->run_job = amdgpu_cs_parser_run_job; parser->free_job = amdgpu_cs_parser_free_job; amd_sched_push_job(ring->scheduler, @@ -965,24 +965,16 @@ int amdgpu_cs_wait_ioctl(struct drm_device *dev, void *data, ctx = amdgpu_ctx_get(filp->driver_priv, wait->in.ctx_id); if (ctx == NULL) return -EINVAL; - if (amdgpu_enable_scheduler) { - r = amd_sched_wait_ts(&ctx->rings[ring->idx].c_entity, - wait->in.handle, true, timeout); - if (r) - return r; - r = 1; - } else { - fence = amdgpu_ctx_get_fence(ctx, ring, wait->in.handle); - if (IS_ERR(fence)) - r = PTR_ERR(fence); - else if (fence) { - r = fence_wait_timeout(fence, true, timeout); - fence_put(fence); + fence = amdgpu_ctx_get_fence(ctx, ring, wait->in.handle); + if (IS_ERR(fence)) + r = PTR_ERR(fence); + else if (fence) { + r = fence_wait_timeout(fence, true, timeout); + fence_put(fence); + } else + r = 1; - } else - r = 1; - } amdgpu_ctx_put(ctx); if (r < 0) return r; diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_ctx.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_ctx.c index b9be250cb206..41bc7fc0ebf6 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_ctx.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_ctx.c @@ -261,6 +261,16 @@ struct fence *amdgpu_ctx_get_fence(struct amdgpu_ctx *ctx, struct amdgpu_ctx_ring *cring = & ctx->rings[ring->idx]; struct fence *fence; uint64_t queued_seq; + int r; + + if (amdgpu_enable_scheduler) { + r = amd_sched_wait_emit(&cring->c_entity, + seq, + true, + AMDGPU_WAIT_IDLE_TIMEOUT_IN_MS); + if (r) + return NULL; + } spin_lock(&ctx->ring_lock); if (amdgpu_enable_scheduler) diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_sched.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_sched.c index 1f7bf31da7fc..46ec915c9344 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_sched.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_sched.c @@ -56,12 +56,15 @@ static void amdgpu_sched_run_job(struct amd_gpu_scheduler *sched, sched_job->filp); if (r) goto err; - if (sched_job->run_job) { r = sched_job->run_job(sched_job); if (r) goto err; } + atomic64_set(&c_entity->last_emitted_v_seq, + sched_job->uf.sequence); + wake_up_all(&c_entity->wait_emit); + mutex_unlock(&sched_job->job_lock); return; err: -- cgit From d1ff9086c1b8e67390161599006a34056b437a72 Mon Sep 17 00:00:00 2001 From: Chunming Zhou Date: Thu, 30 Jul 2015 17:59:43 +0800 Subject: drm/amdgpu: fix seq in ctx_add_fence if enabling scheduler, then the queued seq is assigned when pushing job before emitting job. Signed-off-by: Chunming Zhou Reviewed-by: Christian K?nig --- drivers/gpu/drm/amd/amdgpu/amdgpu.h | 3 +-- drivers/gpu/drm/amd/amdgpu/amdgpu_cs.c | 5 ++--- drivers/gpu/drm/amd/amdgpu/amdgpu_ctx.c | 4 ++-- drivers/gpu/drm/amd/amdgpu/amdgpu_ib.c | 6 +++++- drivers/gpu/drm/amd/amdgpu/amdgpu_sched.c | 4 ++-- drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c | 6 +++--- 6 files changed, 15 insertions(+), 13 deletions(-) (limited to 'drivers/gpu/drm/amd/amdgpu/amdgpu_sched.c') diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu.h b/drivers/gpu/drm/amd/amdgpu/amdgpu.h index 6e1fea473a66..2619c78ec303 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu.h +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu.h @@ -419,7 +419,6 @@ struct amdgpu_user_fence { struct amdgpu_bo *bo; /* write-back address offset to bo start */ uint32_t offset; - uint64_t sequence; }; int amdgpu_fence_driver_init(struct amdgpu_device *adev); @@ -1031,7 +1030,7 @@ struct amdgpu_ctx *amdgpu_ctx_get(struct amdgpu_fpriv *fpriv, uint32_t id); int amdgpu_ctx_put(struct amdgpu_ctx *ctx); uint64_t amdgpu_ctx_add_fence(struct amdgpu_ctx *ctx, struct amdgpu_ring *ring, - struct fence *fence); + struct fence *fence, uint64_t queued_seq); struct fence *amdgpu_ctx_get_fence(struct amdgpu_ctx *ctx, struct amdgpu_ring *ring, uint64_t seq); diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_cs.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_cs.c index c41360e443be..40e85bfcdf91 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_cs.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_cs.c @@ -739,7 +739,6 @@ static int amdgpu_cs_ib_fill(struct amdgpu_device *adev, ib->oa_size = amdgpu_bo_size(oa); } } - /* wrap the last IB with user fence */ if (parser->uf.bo) { struct amdgpu_ib *ib = &parser->ibs[parser->num_ibs - 1]; @@ -908,7 +907,7 @@ int amdgpu_cs_ioctl(struct drm_device *dev, void *data, struct drm_file *filp) if (amdgpu_enable_scheduler && parser->num_ibs) { struct amdgpu_ring * ring = amdgpu_cs_parser_get_ring(adev, parser); - parser->uf.sequence = atomic64_inc_return( + parser->ibs[parser->num_ibs - 1].sequence = atomic64_inc_return( &parser->ctx->rings[ring->idx].c_entity.last_queued_v_seq); if (ring->is_pte_ring || (parser->bo_list && parser->bo_list->has_userptr)) { r = amdgpu_cs_parser_prepare_job(parser); @@ -922,7 +921,7 @@ int amdgpu_cs_ioctl(struct drm_device *dev, void *data, struct drm_file *filp) amd_sched_push_job(ring->scheduler, &parser->ctx->rings[ring->idx].c_entity, parser); - cs->out.handle = parser->uf.sequence; + cs->out.handle = parser->ibs[parser->num_ibs - 1].sequence; up_read(&adev->exclusive_lock); return 0; } diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_ctx.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_ctx.c index 95807b678b6a..e0eaa55bf636 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_ctx.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_ctx.c @@ -258,7 +258,7 @@ int amdgpu_ctx_put(struct amdgpu_ctx *ctx) } uint64_t amdgpu_ctx_add_fence(struct amdgpu_ctx *ctx, struct amdgpu_ring *ring, - struct fence *fence) + struct fence *fence, uint64_t queued_seq) { struct amdgpu_ctx_ring *cring = & ctx->rings[ring->idx]; uint64_t seq = 0; @@ -266,7 +266,7 @@ uint64_t amdgpu_ctx_add_fence(struct amdgpu_ctx *ctx, struct amdgpu_ring *ring, struct fence *other = NULL; if (amdgpu_enable_scheduler) - seq = atomic64_read(&cring->c_entity.last_queued_v_seq); + seq = queued_seq; else seq = cring->sequence; idx = seq % AMDGPU_CTX_MAX_CS_PENDING; diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_ib.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_ib.c index 42d6298eb9d7..eed409c59492 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_ib.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_ib.c @@ -143,6 +143,7 @@ int amdgpu_ib_schedule(struct amdgpu_device *adev, unsigned num_ibs, struct amdgpu_ring *ring; struct amdgpu_ctx *ctx, *old_ctx; struct amdgpu_vm *vm; + uint64_t sequence; unsigned i; int r = 0; @@ -215,9 +216,12 @@ int amdgpu_ib_schedule(struct amdgpu_device *adev, unsigned num_ibs, return r; } + sequence = amdgpu_enable_scheduler ? ib->sequence : 0; + if (ib->ctx) ib->sequence = amdgpu_ctx_add_fence(ib->ctx, ring, - &ib->fence->base); + &ib->fence->base, + sequence); /* wrap the last IB with fence */ if (ib->user) { diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_sched.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_sched.c index 46ec915c9344..b913c22dd6b2 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_sched.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_sched.c @@ -62,7 +62,7 @@ static void amdgpu_sched_run_job(struct amd_gpu_scheduler *sched, goto err; } atomic64_set(&c_entity->last_emitted_v_seq, - sched_job->uf.sequence); + sched_job->ibs[sched_job->num_ibs - 1].sequence); wake_up_all(&c_entity->wait_emit); mutex_unlock(&sched_job->job_lock); @@ -93,7 +93,7 @@ static void amdgpu_sched_process_job(struct amd_gpu_scheduler *sched, void *job) if (sched_job->ctx) { c_entity = &sched_job->ctx->rings[ring->idx].c_entity; atomic64_set(&c_entity->last_signaled_v_seq, - sched_job->uf.sequence); + sched_job->ibs[sched_job->num_ibs - 1].sequence); } /* wake up users waiting for time stamp */ diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c index 26c55a7a1a88..5624d4484fb6 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c @@ -380,7 +380,7 @@ static int amdgpu_vm_clear_bo(struct amdgpu_device *adev, sched_job->run_job = amdgpu_vm_run_job; sched_job->free_job = amdgpu_vm_free_job; v_seq = atomic64_inc_return(&adev->kernel_ctx->rings[ring->idx].c_entity.last_queued_v_seq); - sched_job->uf.sequence = v_seq; + ib->sequence = v_seq; amd_sched_push_job(ring->scheduler, &adev->kernel_ctx->rings[ring->idx].c_entity, sched_job); @@ -531,7 +531,7 @@ int amdgpu_vm_update_page_directory(struct amdgpu_device *adev, sched_job->run_job = amdgpu_vm_run_job; sched_job->free_job = amdgpu_vm_free_job; v_seq = atomic64_inc_return(&adev->kernel_ctx->rings[ring->idx].c_entity.last_queued_v_seq); - sched_job->uf.sequence = v_seq; + ib->sequence = v_seq; amd_sched_push_job(ring->scheduler, &adev->kernel_ctx->rings[ring->idx].c_entity, sched_job); @@ -884,7 +884,7 @@ static int amdgpu_vm_bo_update_mapping(struct amdgpu_device *adev, sched_job->run_job = amdgpu_vm_bo_update_mapping_run_job; sched_job->free_job = amdgpu_vm_free_job; v_seq = atomic64_inc_return(&adev->kernel_ctx->rings[ring->idx].c_entity.last_queued_v_seq); - sched_job->uf.sequence = v_seq; + ib->sequence = v_seq; amd_sched_push_job(ring->scheduler, &adev->kernel_ctx->rings[ring->idx].c_entity, sched_job); -- cgit From 3c704e934d07bcb5fdf9725db190e2ae60fba1bd Mon Sep 17 00:00:00 2001 From: Chunming Zhou Date: Wed, 29 Jul 2015 10:33:14 +0800 Subject: drm/amdgpu: add helper function for kernel submission Signed-off-by: Chunming Zhou Reviewed-by: Christian K?nig --- drivers/gpu/drm/amd/amdgpu/amdgpu.h | 7 +++++++ drivers/gpu/drm/amd/amdgpu/amdgpu_sched.c | 35 +++++++++++++++++++++++++++++++ 2 files changed, 42 insertions(+) (limited to 'drivers/gpu/drm/amd/amdgpu/amdgpu_sched.c') diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu.h b/drivers/gpu/drm/amd/amdgpu/amdgpu.h index 2619c78ec303..1e87acf35892 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu.h +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu.h @@ -864,6 +864,13 @@ enum amdgpu_ring_type { extern struct amd_sched_backend_ops amdgpu_sched_ops; +int amdgpu_sched_ib_submit_kernel_helper(struct amdgpu_device *adev, + struct amdgpu_ring *ring, + struct amdgpu_ib *ibs, + unsigned num_ibs, + int (*free_job)(struct amdgpu_cs_parser *), + void *owner); + struct amdgpu_ring { struct amdgpu_device *adev; const struct amdgpu_ring_funcs *funcs; diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_sched.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_sched.c index b913c22dd6b2..d682fabca958 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_sched.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_sched.c @@ -108,3 +108,38 @@ struct amd_sched_backend_ops amdgpu_sched_ops = { .process_job = amdgpu_sched_process_job }; +int amdgpu_sched_ib_submit_kernel_helper(struct amdgpu_device *adev, + struct amdgpu_ring *ring, + struct amdgpu_ib *ibs, + unsigned num_ibs, + int (*free_job)(struct amdgpu_cs_parser *), + void *owner) +{ + int r = 0; + if (amdgpu_enable_scheduler) { + uint64_t v_seq; + struct amdgpu_cs_parser *sched_job = + amdgpu_cs_parser_create(adev, + owner, + adev->kernel_ctx, + ibs, 1); + if(!sched_job) { + return -ENOMEM; + } + sched_job->free_job = free_job; + v_seq = atomic64_inc_return(&adev->kernel_ctx->rings[ring->idx].c_entity.last_queued_v_seq); + ibs[num_ibs - 1].sequence = v_seq; + amd_sched_push_job(ring->scheduler, + &adev->kernel_ctx->rings[ring->idx].c_entity, + sched_job); + r = amd_sched_wait_emit( + &adev->kernel_ctx->rings[ring->idx].c_entity, + v_seq, + false, + -1); + if (r) + WARN(true, "emit timeout\n"); + } else + r = amdgpu_ib_schedule(adev, 1, ibs, owner); + return r; +} -- cgit From 1d7dd229f5dded247bc8800f8f4551d3d6314afa Mon Sep 17 00:00:00 2001 From: Christian König Date: Fri, 31 Jul 2015 14:31:49 +0200 Subject: drm/amdgpu: clean up amd sched wait_ts and wait_signal MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Remove code not used at the moment. Signed-off-by: Christian König Reviewed-by: Chunming Zhou --- drivers/gpu/drm/amd/amdgpu/amdgpu_sched.c | 6 ---- drivers/gpu/drm/amd/scheduler/gpu_scheduler.c | 47 ++++----------------------- drivers/gpu/drm/amd/scheduler/gpu_scheduler.h | 5 --- 3 files changed, 7 insertions(+), 51 deletions(-) (limited to 'drivers/gpu/drm/amd/amdgpu/amdgpu_sched.c') diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_sched.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_sched.c index d682fabca958..161c83ad9349 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_sched.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_sched.c @@ -90,12 +90,6 @@ static void amdgpu_sched_process_job(struct amd_gpu_scheduler *sched, void *job) ring = fence->ring; adev = ring->adev; - if (sched_job->ctx) { - c_entity = &sched_job->ctx->rings[ring->idx].c_entity; - atomic64_set(&c_entity->last_signaled_v_seq, - sched_job->ibs[sched_job->num_ibs - 1].sequence); - } - /* wake up users waiting for time stamp */ wake_up_all(&c_entity->wait_queue); diff --git a/drivers/gpu/drm/amd/scheduler/gpu_scheduler.c b/drivers/gpu/drm/amd/scheduler/gpu_scheduler.c index 042da7ddf8db..89799eb86083 100644 --- a/drivers/gpu/drm/amd/scheduler/gpu_scheduler.c +++ b/drivers/gpu/drm/amd/scheduler/gpu_scheduler.c @@ -208,7 +208,6 @@ int amd_context_entity_init(struct amd_gpu_scheduler *sched, entity->context_id = context_id; atomic64_set(&entity->last_emitted_v_seq, seq_ring); atomic64_set(&entity->last_queued_v_seq, seq_ring); - atomic64_set(&entity->last_signaled_v_seq, seq_ring); /* Add the entity to the run queue */ mutex_lock(&rq->lock); @@ -317,20 +316,7 @@ int amd_sched_push_job(struct amd_gpu_scheduler *sched, } /** - * Check the virtual sequence number for specified context - * - * @seq The virtual sequence number to check - * @c_entity The pointer to a valid amd_context_entity - * - * return 0 if signaled, -1 else. -*/ -int amd_sched_check_ts(struct amd_context_entity *c_entity, uint64_t seq) -{ - return (seq <= atomic64_read(&c_entity->last_signaled_v_seq)) ? 0 : -1; -} - -/** - * Wait for a virtual sequence number to be signaled or timeout + * Wait for a virtual sequence number to be emitted. * * @c_entity The pointer to a valid context entity * @seq The virtual sequence number to wait @@ -340,16 +326,13 @@ int amd_sched_check_ts(struct amd_context_entity *c_entity, uint64_t seq) * * return =0 signaled , <0 failed */ -static int amd_sched_wait(struct amd_context_entity *c_entity, - uint64_t seq, - bool intr, - long timeout, - bool emit) +int amd_sched_wait_emit(struct amd_context_entity *c_entity, + uint64_t seq, + bool intr, + long timeout) { - atomic64_t *v_seq = emit ? &c_entity->last_emitted_v_seq : - &c_entity->last_signaled_v_seq; - wait_queue_head_t *wait_queue = emit ? &c_entity->wait_emit : - &c_entity->wait_queue; + atomic64_t *v_seq = &c_entity->last_emitted_v_seq; + wait_queue_head_t *wait_queue = &c_entity->wait_emit; if (intr && (timeout < 0)) { wait_event_interruptible( @@ -379,22 +362,6 @@ static int amd_sched_wait(struct amd_context_entity *c_entity, return 0; } -int amd_sched_wait_signal(struct amd_context_entity *c_entity, - uint64_t seq, - bool intr, - long timeout) -{ - return amd_sched_wait(c_entity, seq, intr, timeout, false); -} - -int amd_sched_wait_emit(struct amd_context_entity *c_entity, - uint64_t seq, - bool intr, - long timeout) -{ - return amd_sched_wait(c_entity, seq, intr, timeout, true); -} - static int amd_sched_main(void *param) { int r; diff --git a/drivers/gpu/drm/amd/scheduler/gpu_scheduler.h b/drivers/gpu/drm/amd/scheduler/gpu_scheduler.h index 7f6bc26d2d29..9ab3adc1fa32 100644 --- a/drivers/gpu/drm/amd/scheduler/gpu_scheduler.h +++ b/drivers/gpu/drm/amd/scheduler/gpu_scheduler.h @@ -74,7 +74,6 @@ struct amd_context_entity { /* the virtual_seq is unique per context per ring */ atomic64_t last_queued_v_seq; atomic64_t last_emitted_v_seq; - atomic64_t last_signaled_v_seq; pid_t tgid; uint32_t context_id; /* the job_queue maintains the jobs submitted by clients */ @@ -134,10 +133,6 @@ int amd_sched_push_job(struct amd_gpu_scheduler *sched, struct amd_context_entity *c_entity, void *job); -int amd_sched_check_ts(struct amd_context_entity *c_entity, uint64_t seq); - -int amd_sched_wait_signal(struct amd_context_entity *c_entity, - uint64_t seq, bool intr, long timeout); int amd_sched_wait_emit(struct amd_context_entity *c_entity, uint64_t seq, bool intr, -- cgit From 1763552ee8a7f39a1788d24e27b50d4dee383520 Mon Sep 17 00:00:00 2001 From: Chunming Zhou Date: Mon, 3 Aug 2015 11:43:19 +0800 Subject: drm/amdgpu: add kernel fence in ib_submit_kernel_helper every sbumission should be able to get a fence. Signed-off-by: Chunming Zhou Reviewed-by: Christian K?nig Reviewed-by: Jammy Zhou --- drivers/gpu/drm/amd/amdgpu/amdgpu.h | 3 ++- drivers/gpu/drm/amd/amdgpu/amdgpu_sched.c | 8 ++++++-- drivers/gpu/drm/amd/amdgpu/amdgpu_uvd.c | 8 +++++--- drivers/gpu/drm/amd/amdgpu/amdgpu_vce.c | 12 ++++++++---- drivers/gpu/drm/amd/amdgpu/cik_sdma.c | 6 ++++-- drivers/gpu/drm/amd/amdgpu/gfx_v7_0.c | 6 ++++-- drivers/gpu/drm/amd/amdgpu/gfx_v8_0.c | 6 ++++-- drivers/gpu/drm/amd/amdgpu/sdma_v2_4.c | 6 ++++-- drivers/gpu/drm/amd/amdgpu/sdma_v3_0.c | 6 ++++-- 9 files changed, 41 insertions(+), 20 deletions(-) (limited to 'drivers/gpu/drm/amd/amdgpu/amdgpu_sched.c') diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu.h b/drivers/gpu/drm/amd/amdgpu/amdgpu.h index 371ff0845989..e1f093c1f011 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu.h +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu.h @@ -872,7 +872,8 @@ int amdgpu_sched_ib_submit_kernel_helper(struct amdgpu_device *adev, struct amdgpu_ib *ibs, unsigned num_ibs, int (*free_job)(struct amdgpu_cs_parser *), - void *owner); + void *owner, + struct fence **fence); struct amdgpu_ring { struct amdgpu_device *adev; diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_sched.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_sched.c index 161c83ad9349..23a17ec239c0 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_sched.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_sched.c @@ -107,7 +107,8 @@ int amdgpu_sched_ib_submit_kernel_helper(struct amdgpu_device *adev, struct amdgpu_ib *ibs, unsigned num_ibs, int (*free_job)(struct amdgpu_cs_parser *), - void *owner) + void *owner, + struct fence **f) { int r = 0; if (amdgpu_enable_scheduler) { @@ -135,5 +136,8 @@ int amdgpu_sched_ib_submit_kernel_helper(struct amdgpu_device *adev, WARN(true, "emit timeout\n"); } else r = amdgpu_ib_schedule(adev, 1, ibs, owner); - return r; + if (r) + return r; + *f = &ibs[num_ibs - 1].fence->base; + return 0; } diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_uvd.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_uvd.c index 9b2730599134..f114c6b49b9d 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_uvd.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_uvd.c @@ -825,6 +825,7 @@ static int amdgpu_uvd_send_msg(struct amdgpu_ring *ring, struct ww_acquire_ctx ticket; struct list_head head; struct amdgpu_ib *ib = NULL; + struct fence *f = NULL; struct amdgpu_device *adev = ring->adev; uint64_t addr; int i, r; @@ -869,14 +870,15 @@ static int amdgpu_uvd_send_msg(struct amdgpu_ring *ring, r = amdgpu_sched_ib_submit_kernel_helper(adev, ring, ib, 1, &amdgpu_uvd_free_job, - AMDGPU_FENCE_OWNER_UNDEFINED); + AMDGPU_FENCE_OWNER_UNDEFINED, + &f); if (r) goto err2; - ttm_eu_fence_buffer_objects(&ticket, &head, &ib->fence->base); + ttm_eu_fence_buffer_objects(&ticket, &head, f); if (fence) - *fence = fence_get(&ib->fence->base); + *fence = fence_get(f); amdgpu_bo_unref(&bo); if (amdgpu_enable_scheduler) diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_vce.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_vce.c index 94c40ca5ba74..38660eac67d6 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_vce.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_vce.c @@ -362,6 +362,7 @@ int amdgpu_vce_get_create_msg(struct amdgpu_ring *ring, uint32_t handle, { const unsigned ib_size_dw = 1024; struct amdgpu_ib *ib = NULL; + struct fence *f = NULL; struct amdgpu_device *adev = ring->adev; uint64_t dummy; int i, r; @@ -408,11 +409,12 @@ int amdgpu_vce_get_create_msg(struct amdgpu_ring *ring, uint32_t handle, r = amdgpu_sched_ib_submit_kernel_helper(adev, ring, ib, 1, &amdgpu_vce_free_job, - AMDGPU_FENCE_OWNER_UNDEFINED); + AMDGPU_FENCE_OWNER_UNDEFINED, + &f); if (r) goto err; if (fence) - *fence = fence_get(&ib->fence->base); + *fence = fence_get(f); if (amdgpu_enable_scheduler) return 0; err: @@ -436,6 +438,7 @@ int amdgpu_vce_get_destroy_msg(struct amdgpu_ring *ring, uint32_t handle, { const unsigned ib_size_dw = 1024; struct amdgpu_ib *ib = NULL; + struct fence *f = NULL; struct amdgpu_device *adev = ring->adev; uint64_t dummy; int i, r; @@ -472,11 +475,12 @@ int amdgpu_vce_get_destroy_msg(struct amdgpu_ring *ring, uint32_t handle, ib->ptr[i] = 0x0; r = amdgpu_sched_ib_submit_kernel_helper(adev, ring, ib, 1, &amdgpu_vce_free_job, - AMDGPU_FENCE_OWNER_UNDEFINED); + AMDGPU_FENCE_OWNER_UNDEFINED, + &f); if (r) goto err; if (fence) - *fence = fence_get(&ib->fence->base); + *fence = fence_get(f); if (amdgpu_enable_scheduler) return 0; err: diff --git a/drivers/gpu/drm/amd/amdgpu/cik_sdma.c b/drivers/gpu/drm/amd/amdgpu/cik_sdma.c index 115b7706e025..c3ed5b22d732 100644 --- a/drivers/gpu/drm/amd/amdgpu/cik_sdma.c +++ b/drivers/gpu/drm/amd/amdgpu/cik_sdma.c @@ -614,6 +614,7 @@ static int cik_sdma_ring_test_ib(struct amdgpu_ring *ring) { struct amdgpu_device *adev = ring->adev; struct amdgpu_ib ib; + struct fence *f = NULL; unsigned i; unsigned index; int r; @@ -642,11 +643,12 @@ static int cik_sdma_ring_test_ib(struct amdgpu_ring *ring) ib.ptr[4] = 0xDEADBEEF; ib.length_dw = 5; r = amdgpu_sched_ib_submit_kernel_helper(adev, ring, &ib, 1, NULL, - AMDGPU_FENCE_OWNER_UNDEFINED); + AMDGPU_FENCE_OWNER_UNDEFINED, + &f); if (r) goto err1; - r = fence_wait(&ib.fence->base, false); + r = fence_wait(f, false); if (r) { DRM_ERROR("amdgpu: fence wait failed (%d).\n", r); goto err1; diff --git a/drivers/gpu/drm/amd/amdgpu/gfx_v7_0.c b/drivers/gpu/drm/amd/amdgpu/gfx_v7_0.c index 521d811996f9..ee1c47f9a2b6 100644 --- a/drivers/gpu/drm/amd/amdgpu/gfx_v7_0.c +++ b/drivers/gpu/drm/amd/amdgpu/gfx_v7_0.c @@ -2648,6 +2648,7 @@ static int gfx_v7_0_ring_test_ib(struct amdgpu_ring *ring) { struct amdgpu_device *adev = ring->adev; struct amdgpu_ib ib; + struct fence *f = NULL; uint32_t scratch; uint32_t tmp = 0; unsigned i; @@ -2670,11 +2671,12 @@ static int gfx_v7_0_ring_test_ib(struct amdgpu_ring *ring) ib.length_dw = 3; r = amdgpu_sched_ib_submit_kernel_helper(adev, ring, &ib, 1, NULL, - AMDGPU_FENCE_OWNER_UNDEFINED); + AMDGPU_FENCE_OWNER_UNDEFINED, + &f); if (r) goto err2; - r = fence_wait(&ib.fence->base, false); + r = fence_wait(f, false); if (r) { DRM_ERROR("amdgpu: fence wait failed (%d).\n", r); goto err2; diff --git a/drivers/gpu/drm/amd/amdgpu/gfx_v8_0.c b/drivers/gpu/drm/amd/amdgpu/gfx_v8_0.c index 59eae93804c3..a865d96b67af 100644 --- a/drivers/gpu/drm/amd/amdgpu/gfx_v8_0.c +++ b/drivers/gpu/drm/amd/amdgpu/gfx_v8_0.c @@ -610,6 +610,7 @@ static int gfx_v8_0_ring_test_ib(struct amdgpu_ring *ring) { struct amdgpu_device *adev = ring->adev; struct amdgpu_ib ib; + struct fence *f = NULL; uint32_t scratch; uint32_t tmp = 0; unsigned i; @@ -632,11 +633,12 @@ static int gfx_v8_0_ring_test_ib(struct amdgpu_ring *ring) ib.length_dw = 3; r = amdgpu_sched_ib_submit_kernel_helper(adev, ring, &ib, 1, NULL, - AMDGPU_FENCE_OWNER_UNDEFINED); + AMDGPU_FENCE_OWNER_UNDEFINED, + &f); if (r) goto err2; - r = fence_wait(&ib.fence->base, false); + r = fence_wait(f, false); if (r) { DRM_ERROR("amdgpu: fence wait failed (%d).\n", r); goto err2; diff --git a/drivers/gpu/drm/amd/amdgpu/sdma_v2_4.c b/drivers/gpu/drm/amd/amdgpu/sdma_v2_4.c index 2b7ce9393929..6de7dc88d53c 100644 --- a/drivers/gpu/drm/amd/amdgpu/sdma_v2_4.c +++ b/drivers/gpu/drm/amd/amdgpu/sdma_v2_4.c @@ -673,6 +673,7 @@ static int sdma_v2_4_ring_test_ib(struct amdgpu_ring *ring) { struct amdgpu_device *adev = ring->adev; struct amdgpu_ib ib; + struct fence *f = NULL; unsigned i; unsigned index; int r; @@ -706,11 +707,12 @@ static int sdma_v2_4_ring_test_ib(struct amdgpu_ring *ring) ib.length_dw = 8; r = amdgpu_sched_ib_submit_kernel_helper(adev, ring, &ib, 1, NULL, - AMDGPU_FENCE_OWNER_UNDEFINED); + AMDGPU_FENCE_OWNER_UNDEFINED, + &f); if (r) goto err1; - r = fence_wait(&ib.fence->base, false); + r = fence_wait(f, false); if (r) { DRM_ERROR("amdgpu: fence wait failed (%d).\n", r); goto err1; diff --git a/drivers/gpu/drm/amd/amdgpu/sdma_v3_0.c b/drivers/gpu/drm/amd/amdgpu/sdma_v3_0.c index cba45e62ae4b..963a991fea00 100644 --- a/drivers/gpu/drm/amd/amdgpu/sdma_v3_0.c +++ b/drivers/gpu/drm/amd/amdgpu/sdma_v3_0.c @@ -794,6 +794,7 @@ static int sdma_v3_0_ring_test_ib(struct amdgpu_ring *ring) { struct amdgpu_device *adev = ring->adev; struct amdgpu_ib ib; + struct fence *f = NULL; unsigned i; unsigned index; int r; @@ -827,11 +828,12 @@ static int sdma_v3_0_ring_test_ib(struct amdgpu_ring *ring) ib.length_dw = 8; r = amdgpu_sched_ib_submit_kernel_helper(adev, ring, &ib, 1, NULL, - AMDGPU_FENCE_OWNER_UNDEFINED); + AMDGPU_FENCE_OWNER_UNDEFINED, + &f); if (r) goto err1; - r = fence_wait(&ib.fence->base, false); + r = fence_wait(f, false); if (r) { DRM_ERROR("amdgpu: fence wait failed (%d).\n", r); goto err1; -- cgit From 03d3a3e634894259a27979f4a372273162611e78 Mon Sep 17 00:00:00 2001 From: Chunming Zhou Date: Mon, 3 Aug 2015 20:02:43 +0800 Subject: drm/amdgpu: fix null pointer by previous cleanup Signed-off-by: Chunming Zhou Reviewed-by: Christian K?nig --- drivers/gpu/drm/amd/amdgpu/amdgpu_sched.c | 4 ---- 1 file changed, 4 deletions(-) (limited to 'drivers/gpu/drm/amd/amdgpu/amdgpu_sched.c') diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_sched.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_sched.c index 23a17ec239c0..0f55c05c80b1 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_sched.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_sched.c @@ -79,7 +79,6 @@ static void amdgpu_sched_process_job(struct amd_gpu_scheduler *sched, void *job) struct amdgpu_fence *fence = NULL; struct amdgpu_ring *ring = NULL; struct amdgpu_device *adev = NULL; - struct amd_context_entity *c_entity = NULL; if (!job) return; @@ -90,9 +89,6 @@ static void amdgpu_sched_process_job(struct amd_gpu_scheduler *sched, void *job) ring = fence->ring; adev = ring->adev; - /* wake up users waiting for time stamp */ - wake_up_all(&c_entity->wait_queue); - schedule_work(&sched_job->job_work); } -- cgit From dd01d071957ded58d9bae3d3bf6061ada1d84692 Mon Sep 17 00:00:00 2001 From: Jammy Zhou Date: Thu, 30 Jul 2015 17:19:52 +0800 Subject: drm/amdgpu: some code refinement v2 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Fix the code alignment, etc. v2: rebase the code Signed-off-by: Jammy Zhou Reviewed-by: Christian König Reviewed-by: Alex Deucher --- drivers/gpu/drm/amd/amdgpu/amdgpu_cs.c | 26 +++++++++++++------------- drivers/gpu/drm/amd/amdgpu/amdgpu_sched.c | 9 +++++---- 2 files changed, 18 insertions(+), 17 deletions(-) (limited to 'drivers/gpu/drm/amd/amdgpu/amdgpu_sched.c') diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_cs.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_cs.c index aee59110735f..fda7792d9e08 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_cs.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_cs.c @@ -817,14 +817,14 @@ static int amdgpu_cs_dependencies(struct amdgpu_device *adev, static int amdgpu_cs_parser_prepare_job(struct amdgpu_cs_parser *sched_job) { - int r, i; + int r, i; struct amdgpu_cs_parser *parser = sched_job; struct amdgpu_device *adev = sched_job->adev; - bool reserved_buffers = false; + bool reserved_buffers = false; - r = amdgpu_cs_parser_relocs(parser); - if (r) { - if (r != -ERESTARTSYS) { + r = amdgpu_cs_parser_relocs(parser); + if (r) { + if (r != -ERESTARTSYS) { if (r == -ENOMEM) DRM_ERROR("Not enough memory for command submission!\n"); else @@ -841,16 +841,16 @@ static int amdgpu_cs_parser_prepare_job(struct amdgpu_cs_parser *sched_job) if (r) DRM_ERROR("Failed in the dependencies handling %d!\n", r); } - if (r) { - amdgpu_cs_parser_fini(parser, r, reserved_buffers); - return r; - } + if (r) { + amdgpu_cs_parser_fini(parser, r, reserved_buffers); + return r; + } - for (i = 0; i < parser->num_ibs; i++) - trace_amdgpu_cs(parser, i); + for (i = 0; i < parser->num_ibs; i++) + trace_amdgpu_cs(parser, i); - r = amdgpu_cs_ib_vm_chunk(adev, parser); - return r; + r = amdgpu_cs_ib_vm_chunk(adev, parser); + return r; } static struct amdgpu_ring *amdgpu_cs_parser_get_ring( diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_sched.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_sched.c index 0f55c05c80b1..788dd348a650 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_sched.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_sched.c @@ -33,11 +33,12 @@ static int amdgpu_sched_prepare_job(struct amd_gpu_scheduler *sched, { int r = 0; struct amdgpu_cs_parser *sched_job = (struct amdgpu_cs_parser *)job; - if (sched_job->prepare_job) + if (sched_job->prepare_job) { r = sched_job->prepare_job(sched_job); - if (r) { - DRM_ERROR("Prepare job error\n"); - schedule_work(&sched_job->job_work); + if (r) { + DRM_ERROR("Prepare job error\n"); + schedule_work(&sched_job->job_work); + } } return r; } -- cgit From ea199cc9f825f3ef5aab3db5f00dcc639f8a8b02 Mon Sep 17 00:00:00 2001 From: Jammy Zhou Date: Fri, 31 Jul 2015 16:47:28 +0800 Subject: drm/amdgpu: return new seq_no for amd_sched_push_job MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit It is clean to update last_queued_v_seq in the scheduler module Signed-off-by: Jammy Zhou Reviewed-by: Christian König Reviewed-by: Alex Deucher --- drivers/gpu/drm/amd/amdgpu/amdgpu_cs.c | 5 ++--- drivers/gpu/drm/amd/amdgpu/amdgpu_sched.c | 9 ++------- drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c | 27 ++++++--------------------- drivers/gpu/drm/amd/scheduler/gpu_scheduler.c | 10 ++++------ drivers/gpu/drm/amd/scheduler/gpu_scheduler.h | 2 +- 5 files changed, 15 insertions(+), 38 deletions(-) (limited to 'drivers/gpu/drm/amd/amdgpu/amdgpu_sched.c') diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_cs.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_cs.c index fda7792d9e08..468f884271b3 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_cs.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_cs.c @@ -907,8 +907,6 @@ int amdgpu_cs_ioctl(struct drm_device *dev, void *data, struct drm_file *filp) if (amdgpu_enable_scheduler && parser->num_ibs) { struct amdgpu_ring * ring = amdgpu_cs_parser_get_ring(adev, parser); - parser->ibs[parser->num_ibs - 1].sequence = atomic64_inc_return( - &parser->ctx->rings[ring->idx].c_entity.last_queued_v_seq); if (ring->is_pte_ring || (parser->bo_list && parser->bo_list->has_userptr)) { r = amdgpu_cs_parser_prepare_job(parser); if (r) @@ -918,7 +916,8 @@ int amdgpu_cs_ioctl(struct drm_device *dev, void *data, struct drm_file *filp) parser->ring = ring; parser->run_job = amdgpu_cs_parser_run_job; parser->free_job = amdgpu_cs_parser_free_job; - amd_sched_push_job(ring->scheduler, + parser->ibs[parser->num_ibs - 1].sequence = + amd_sched_push_job(ring->scheduler, &parser->ctx->rings[ring->idx].c_entity, parser); cs->out.handle = parser->ibs[parser->num_ibs - 1].sequence; diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_sched.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_sched.c index 788dd348a650..8c01c51aac41 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_sched.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_sched.c @@ -109,7 +109,6 @@ int amdgpu_sched_ib_submit_kernel_helper(struct amdgpu_device *adev, { int r = 0; if (amdgpu_enable_scheduler) { - uint64_t v_seq; struct amdgpu_cs_parser *sched_job = amdgpu_cs_parser_create(adev, owner, @@ -119,16 +118,12 @@ int amdgpu_sched_ib_submit_kernel_helper(struct amdgpu_device *adev, return -ENOMEM; } sched_job->free_job = free_job; - v_seq = atomic64_inc_return(&adev->kernel_ctx->rings[ring->idx].c_entity.last_queued_v_seq); - ibs[num_ibs - 1].sequence = v_seq; - amd_sched_push_job(ring->scheduler, + ibs[num_ibs - 1].sequence = amd_sched_push_job(ring->scheduler, &adev->kernel_ctx->rings[ring->idx].c_entity, sched_job); r = amd_sched_wait_emit( &adev->kernel_ctx->rings[ring->idx].c_entity, - v_seq, - false, - -1); + ibs[num_ibs - 1].sequence, false, -1); if (r) WARN(true, "emit timeout\n"); } else diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c index d90254f5ca6a..ab9c65a245ba 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c @@ -371,7 +371,6 @@ static int amdgpu_vm_clear_bo(struct amdgpu_device *adev, if (amdgpu_enable_scheduler) { int r; - uint64_t v_seq; sched_job = amdgpu_cs_parser_create(adev, AMDGPU_FENCE_OWNER_VM, adev->kernel_ctx, ib, 1); if(!sched_job) @@ -379,15 +378,11 @@ static int amdgpu_vm_clear_bo(struct amdgpu_device *adev, sched_job->job_param.vm.bo = bo; sched_job->run_job = amdgpu_vm_run_job; sched_job->free_job = amdgpu_vm_free_job; - v_seq = atomic64_inc_return(&adev->kernel_ctx->rings[ring->idx].c_entity.last_queued_v_seq); - ib->sequence = v_seq; - amd_sched_push_job(ring->scheduler, + ib->sequence = amd_sched_push_job(ring->scheduler, &adev->kernel_ctx->rings[ring->idx].c_entity, sched_job); r = amd_sched_wait_emit(&adev->kernel_ctx->rings[ring->idx].c_entity, - v_seq, - false, - -1); + ib->sequence, false, -1); if (r) DRM_ERROR("emit timeout\n"); @@ -521,7 +516,6 @@ int amdgpu_vm_update_page_directory(struct amdgpu_device *adev, if (amdgpu_enable_scheduler) { int r; - uint64_t v_seq; sched_job = amdgpu_cs_parser_create(adev, AMDGPU_FENCE_OWNER_VM, adev->kernel_ctx, ib, 1); @@ -530,15 +524,11 @@ int amdgpu_vm_update_page_directory(struct amdgpu_device *adev, sched_job->job_param.vm.bo = pd; sched_job->run_job = amdgpu_vm_run_job; sched_job->free_job = amdgpu_vm_free_job; - v_seq = atomic64_inc_return(&adev->kernel_ctx->rings[ring->idx].c_entity.last_queued_v_seq); - ib->sequence = v_seq; - amd_sched_push_job(ring->scheduler, + ib->sequence = amd_sched_push_job(ring->scheduler, &adev->kernel_ctx->rings[ring->idx].c_entity, sched_job); r = amd_sched_wait_emit(&adev->kernel_ctx->rings[ring->idx].c_entity, - v_seq, - false, - -1); + ib->sequence, false, -1); if (r) DRM_ERROR("emit timeout\n"); } else { @@ -872,7 +862,6 @@ static int amdgpu_vm_bo_update_mapping(struct amdgpu_device *adev, if (amdgpu_enable_scheduler) { int r; - uint64_t v_seq; sched_job = amdgpu_cs_parser_create(adev, AMDGPU_FENCE_OWNER_VM, adev->kernel_ctx, ib, 1); if(!sched_job) @@ -883,15 +872,11 @@ static int amdgpu_vm_bo_update_mapping(struct amdgpu_device *adev, sched_job->job_param.vm_mapping.fence = fence; sched_job->run_job = amdgpu_vm_bo_update_mapping_run_job; sched_job->free_job = amdgpu_vm_free_job; - v_seq = atomic64_inc_return(&adev->kernel_ctx->rings[ring->idx].c_entity.last_queued_v_seq); - ib->sequence = v_seq; - amd_sched_push_job(ring->scheduler, + ib->sequence = amd_sched_push_job(ring->scheduler, &adev->kernel_ctx->rings[ring->idx].c_entity, sched_job); r = amd_sched_wait_emit(&adev->kernel_ctx->rings[ring->idx].c_entity, - v_seq, - false, - -1); + ib->sequence, false, -1); if (r) DRM_ERROR("emit timeout\n"); } else { diff --git a/drivers/gpu/drm/amd/scheduler/gpu_scheduler.c b/drivers/gpu/drm/amd/scheduler/gpu_scheduler.c index 89799eb86083..2c4c261ff928 100644 --- a/drivers/gpu/drm/amd/scheduler/gpu_scheduler.c +++ b/drivers/gpu/drm/amd/scheduler/gpu_scheduler.c @@ -293,12 +293,9 @@ int amd_context_entity_fini(struct amd_gpu_scheduler *sched, * @sched The pointer to the scheduler * @c_entity The pointer to amd_context_entity * @job The pointer to job required to submit - * return 0 if succeed. -1 if failed. - * -2 indicate queue is full for this client, client should wait untill - * scheduler consum some queued command. - * -1 other fail. + * return the virtual sequence number */ -int amd_sched_push_job(struct amd_gpu_scheduler *sched, +uint64_t amd_sched_push_job(struct amd_gpu_scheduler *sched, struct amd_context_entity *c_entity, void *job) { @@ -312,7 +309,8 @@ int amd_sched_push_job(struct amd_gpu_scheduler *sched, } wake_up_interruptible(&sched->wait_queue); - return 0; + + return atomic64_inc_return(&c_entity->last_queued_v_seq); } /** diff --git a/drivers/gpu/drm/amd/scheduler/gpu_scheduler.h b/drivers/gpu/drm/amd/scheduler/gpu_scheduler.h index 9ab3adc1fa32..37dd6370bd98 100644 --- a/drivers/gpu/drm/amd/scheduler/gpu_scheduler.h +++ b/drivers/gpu/drm/amd/scheduler/gpu_scheduler.h @@ -129,7 +129,7 @@ struct amd_gpu_scheduler *amd_sched_create(void *device, int amd_sched_destroy(struct amd_gpu_scheduler *sched); -int amd_sched_push_job(struct amd_gpu_scheduler *sched, +uint64_t amd_sched_push_job(struct amd_gpu_scheduler *sched, struct amd_context_entity *c_entity, void *job); -- cgit From f95b7e3e8664fbea4e60f15e7e8a975e4b2b7c3f Mon Sep 17 00:00:00 2001 From: Jammy Zhou Date: Fri, 31 Jul 2015 17:18:15 +0800 Subject: drm/amdgpu: add amd_sched_commit MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This function is to update last_emitted_v_seq and wake up the waiters. It should be called by driver in the run_job backend function Signed-off-by: Jammy Zhou Reviewed-by: Christian König Reviewed-by: Alex Deucher --- drivers/gpu/drm/amd/amdgpu/amdgpu_sched.c | 5 ++--- drivers/gpu/drm/amd/scheduler/gpu_scheduler.c | 12 ++++++++++++ drivers/gpu/drm/amd/scheduler/gpu_scheduler.h | 2 ++ 3 files changed, 16 insertions(+), 3 deletions(-) (limited to 'drivers/gpu/drm/amd/amdgpu/amdgpu_sched.c') diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_sched.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_sched.c index 8c01c51aac41..83138a6c54b5 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_sched.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_sched.c @@ -62,9 +62,8 @@ static void amdgpu_sched_run_job(struct amd_gpu_scheduler *sched, if (r) goto err; } - atomic64_set(&c_entity->last_emitted_v_seq, - sched_job->ibs[sched_job->num_ibs - 1].sequence); - wake_up_all(&c_entity->wait_emit); + + amd_sched_emit(c_entity, sched_job->ibs[sched_job->num_ibs - 1].sequence); mutex_unlock(&sched_job->job_lock); return; diff --git a/drivers/gpu/drm/amd/scheduler/gpu_scheduler.c b/drivers/gpu/drm/amd/scheduler/gpu_scheduler.c index 2c4c261ff928..66938f1597bb 100644 --- a/drivers/gpu/drm/amd/scheduler/gpu_scheduler.c +++ b/drivers/gpu/drm/amd/scheduler/gpu_scheduler.c @@ -495,3 +495,15 @@ int amd_sched_destroy(struct amd_gpu_scheduler *sched) return 0; } +/** + * Update emitted sequence and wake up the waiters, called by run_job + * in driver side + * + * @entity The context entity + * @seq The sequence number for the latest emitted job +*/ +void amd_sched_emit(struct amd_context_entity *c_entity, uint64_t seq) +{ + atomic64_set(&c_entity->last_emitted_v_seq, seq); + wake_up_all(&c_entity->wait_emit); +} diff --git a/drivers/gpu/drm/amd/scheduler/gpu_scheduler.h b/drivers/gpu/drm/amd/scheduler/gpu_scheduler.h index 37dd6370bd98..af2afdf2309c 100644 --- a/drivers/gpu/drm/amd/scheduler/gpu_scheduler.h +++ b/drivers/gpu/drm/amd/scheduler/gpu_scheduler.h @@ -151,4 +151,6 @@ int amd_context_entity_init(struct amd_gpu_scheduler *sched, uint32_t context_id, uint32_t jobs); +void amd_sched_emit(struct amd_context_entity *c_entity, uint64_t seq); + #endif -- cgit From 7484667c6a8a9122d139a287454bc9c8799c3def Mon Sep 17 00:00:00 2001 From: Chunming Zhou Date: Tue, 4 Aug 2015 11:30:09 +0800 Subject: drm/amdgpu: move sched job process from isr to fence callback This way can avoid interrupt lost, and can process sched job exactly. Signed-off-by: Chunming Zhou Reviewed-by: Jammy Zhou --- drivers/gpu/drm/amd/amdgpu/amdgpu.h | 2 +- drivers/gpu/drm/amd/amdgpu/amdgpu_fence.c | 19 +------------------ drivers/gpu/drm/amd/amdgpu/amdgpu_sched.c | 13 +++++++++++++ 3 files changed, 15 insertions(+), 19 deletions(-) (limited to 'drivers/gpu/drm/amd/amdgpu/amdgpu_sched.c') diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu.h b/drivers/gpu/drm/amd/amdgpu/amdgpu.h index e1f093c1f011..4d6a3e825096 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu.h +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu.h @@ -404,7 +404,7 @@ struct amdgpu_fence_driver { struct amdgpu_fence { struct fence base; - + struct fence_cb cb; /* RB, DMA, etc. */ struct amdgpu_ring *ring; uint64_t seq; diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_fence.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_fence.c index 60e6d668f6b4..eb419791d1b2 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_fence.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_fence.c @@ -350,25 +350,8 @@ void amdgpu_fence_process(struct amdgpu_ring *ring) } } while (atomic64_xchg(&ring->fence_drv.last_seq, seq) > seq); - if (wake) { - if (amdgpu_enable_scheduler) { - uint64_t handled_seq = - amd_sched_get_handled_seq(ring->scheduler); - uint64_t latest_seq = - atomic64_read(&ring->fence_drv.last_seq); - if (handled_seq == latest_seq) { - DRM_ERROR("ring %d, EOP without seq update (lastest_seq=%llu)\n", - ring->idx, latest_seq); - goto exit; - } - do { - amd_sched_isr(ring->scheduler); - } while (amd_sched_get_handled_seq(ring->scheduler) < latest_seq); - } - + if (wake) wake_up_all(&ring->fence_drv.fence_queue); - } -exit: spin_unlock_irqrestore(&ring->fence_lock, irqflags); } diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_sched.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_sched.c index 83138a6c54b5..9f2f19cc4625 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_sched.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_sched.c @@ -43,12 +43,20 @@ static int amdgpu_sched_prepare_job(struct amd_gpu_scheduler *sched, return r; } +static void amdgpu_fence_sched_cb(struct fence *f, struct fence_cb *cb) +{ + struct amdgpu_fence *fence = + container_of(cb, struct amdgpu_fence, cb); + amd_sched_isr(fence->ring->scheduler); +} + static void amdgpu_sched_run_job(struct amd_gpu_scheduler *sched, struct amd_context_entity *c_entity, void *job) { int r = 0; struct amdgpu_cs_parser *sched_job = (struct amdgpu_cs_parser *)job; + struct amdgpu_fence *fence; mutex_lock(&sched_job->job_lock); r = amdgpu_ib_schedule(sched_job->adev, @@ -57,6 +65,11 @@ static void amdgpu_sched_run_job(struct amd_gpu_scheduler *sched, sched_job->filp); if (r) goto err; + fence = sched_job->ibs[sched_job->num_ibs - 1].fence; + if (fence_add_callback(&fence->base, + &fence->cb, amdgpu_fence_sched_cb)) + goto err; + if (sched_job->run_job) { r = sched_job->run_job(sched_job); if (r) -- cgit From 47f38501f11fa45d8a7797f1965448c1e20049d4 Mon Sep 17 00:00:00 2001 From: Christian König Date: Tue, 4 Aug 2015 17:51:05 +0200 Subject: drm/amdgpu: cleanup amdgpu_ctx inti/fini v2 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Cleanup the kernel context handling. v2: rebased Signed-off-by: Christian König Reviewed-by: Chunming Zhou (v1) --- drivers/gpu/drm/amd/amdgpu/amdgpu.h | 9 +- drivers/gpu/drm/amd/amdgpu/amdgpu_ctx.c | 145 ++++++++++++++--------------- drivers/gpu/drm/amd/amdgpu/amdgpu_device.c | 13 +-- drivers/gpu/drm/amd/amdgpu/amdgpu_sched.c | 8 +- drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c | 18 ++-- 5 files changed, 89 insertions(+), 104 deletions(-) (limited to 'drivers/gpu/drm/amd/amdgpu/amdgpu_sched.c') diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu.h b/drivers/gpu/drm/amd/amdgpu/amdgpu.h index 0cd776a55f05..53d70f766afe 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu.h +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu.h @@ -1033,10 +1033,9 @@ struct amdgpu_ctx_mgr { struct idr ctx_handles; }; -int amdgpu_ctx_alloc(struct amdgpu_device *adev, struct amdgpu_fpriv *fpriv, - uint32_t *id); -int amdgpu_ctx_free(struct amdgpu_device *adev, struct amdgpu_fpriv *fpriv, - uint32_t id); +int amdgpu_ctx_init(struct amdgpu_device *adev, bool kernel, + struct amdgpu_ctx *ctx); +void amdgpu_ctx_fini(struct amdgpu_ctx *ctx); struct amdgpu_ctx *amdgpu_ctx_get(struct amdgpu_fpriv *fpriv, uint32_t id); int amdgpu_ctx_put(struct amdgpu_ctx *ctx); @@ -2095,7 +2094,7 @@ struct amdgpu_device { struct kfd_dev *kfd; /* kernel conext for IB submission */ - struct amdgpu_ctx *kernel_ctx; + struct amdgpu_ctx kernel_ctx; }; bool amdgpu_device_is_px(struct drm_device *dev); diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_ctx.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_ctx.c index c2290ae20312..08a9292729dc 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_ctx.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_ctx.c @@ -25,82 +25,27 @@ #include #include "amdgpu.h" -static void amdgpu_ctx_do_release(struct kref *ref) +int amdgpu_ctx_init(struct amdgpu_device *adev, bool kernel, + struct amdgpu_ctx *ctx) { - struct amdgpu_ctx *ctx; - struct amdgpu_device *adev; unsigned i, j; + int r; - ctx = container_of(ref, struct amdgpu_ctx, refcount); - adev = ctx->adev; - - - for (i = 0; i < AMDGPU_MAX_RINGS; ++i) - for (j = 0; j < AMDGPU_CTX_MAX_CS_PENDING; ++j) - fence_put(ctx->rings[i].fences[j]); - - if (amdgpu_enable_scheduler) { - for (i = 0; i < adev->num_rings; i++) - amd_context_entity_fini(adev->rings[i]->scheduler, - &ctx->rings[i].c_entity); - } - - kfree(ctx); -} - -static void amdgpu_ctx_init(struct amdgpu_device *adev, - struct amdgpu_fpriv *fpriv, - struct amdgpu_ctx *ctx) -{ - int i; memset(ctx, 0, sizeof(*ctx)); ctx->adev = adev; kref_init(&ctx->refcount); spin_lock_init(&ctx->ring_lock); for (i = 0; i < AMDGPU_MAX_RINGS; ++i) ctx->rings[i].sequence = 1; -} - -int amdgpu_ctx_alloc(struct amdgpu_device *adev, struct amdgpu_fpriv *fpriv, - uint32_t *id) -{ - struct amdgpu_ctx *ctx; - int i, j, r; - - ctx = kmalloc(sizeof(*ctx), GFP_KERNEL); - if (!ctx) - return -ENOMEM; - if (fpriv) { - struct amdgpu_ctx_mgr *mgr = &fpriv->ctx_mgr; - mutex_lock(&mgr->lock); - r = idr_alloc(&mgr->ctx_handles, ctx, 1, 0, GFP_KERNEL); - if (r < 0) { - mutex_unlock(&mgr->lock); - kfree(ctx); - return r; - } - *id = (uint32_t)r; - amdgpu_ctx_init(adev, fpriv, ctx); - mutex_unlock(&mgr->lock); - } else { - if (adev->kernel_ctx) { - DRM_ERROR("kernel cnotext has been created.\n"); - kfree(ctx); - return 0; - } - amdgpu_ctx_init(adev, fpriv, ctx); - - adev->kernel_ctx = ctx; - } if (amdgpu_enable_scheduler) { /* create context entity for each ring */ for (i = 0; i < adev->num_rings; i++) { struct amd_run_queue *rq; - if (fpriv) - rq = &adev->rings[i]->scheduler->sched_rq; - else + if (kernel) rq = &adev->rings[i]->scheduler->kernel_rq; + else + rq = &adev->rings[i]->scheduler->sched_rq; r = amd_context_entity_init(adev->rings[i]->scheduler, &ctx->rings[i].c_entity, NULL, rq, amdgpu_sched_jobs); @@ -113,33 +58,79 @@ int amdgpu_ctx_alloc(struct amdgpu_device *adev, struct amdgpu_fpriv *fpriv, amd_context_entity_fini(adev->rings[j]->scheduler, &ctx->rings[j].c_entity); kfree(ctx); - return -EINVAL; + return r; } } - return 0; } -int amdgpu_ctx_free(struct amdgpu_device *adev, struct amdgpu_fpriv *fpriv, uint32_t id) +void amdgpu_ctx_fini(struct amdgpu_ctx *ctx) { + struct amdgpu_device *adev = ctx->adev; + unsigned i, j; + + for (i = 0; i < AMDGPU_MAX_RINGS; ++i) + for (j = 0; j < AMDGPU_CTX_MAX_CS_PENDING; ++j) + fence_put(ctx->rings[i].fences[j]); + + if (amdgpu_enable_scheduler) { + for (i = 0; i < adev->num_rings; i++) + amd_context_entity_fini(adev->rings[i]->scheduler, + &ctx->rings[i].c_entity); + } +} + +static int amdgpu_ctx_alloc(struct amdgpu_device *adev, + struct amdgpu_fpriv *fpriv, + uint32_t *id) +{ + struct amdgpu_ctx_mgr *mgr = &fpriv->ctx_mgr; struct amdgpu_ctx *ctx; + int r; - if (fpriv) { - struct amdgpu_ctx_mgr *mgr = &fpriv->ctx_mgr; - mutex_lock(&mgr->lock); - ctx = idr_find(&mgr->ctx_handles, id); - if (ctx) { - idr_remove(&mgr->ctx_handles, id); - kref_put(&ctx->refcount, amdgpu_ctx_do_release); - mutex_unlock(&mgr->lock); - return 0; - } + ctx = kmalloc(sizeof(*ctx), GFP_KERNEL); + if (!ctx) + return -ENOMEM; + + mutex_lock(&mgr->lock); + r = idr_alloc(&mgr->ctx_handles, ctx, 1, 0, GFP_KERNEL); + if (r < 0) { mutex_unlock(&mgr->lock); - } else { - ctx = adev->kernel_ctx; + kfree(ctx); + return r; + } + *id = (uint32_t)r; + r = amdgpu_ctx_init(adev, false, ctx); + mutex_unlock(&mgr->lock); + + return r; +} + +static void amdgpu_ctx_do_release(struct kref *ref) +{ + struct amdgpu_ctx *ctx; + + ctx = container_of(ref, struct amdgpu_ctx, refcount); + + amdgpu_ctx_fini(ctx); + + kfree(ctx); +} + +static int amdgpu_ctx_free(struct amdgpu_fpriv *fpriv, uint32_t id) +{ + struct amdgpu_ctx_mgr *mgr = &fpriv->ctx_mgr; + struct amdgpu_ctx *ctx; + + mutex_lock(&mgr->lock); + ctx = idr_find(&mgr->ctx_handles, id); + if (ctx) { + idr_remove(&mgr->ctx_handles, id); kref_put(&ctx->refcount, amdgpu_ctx_do_release); + mutex_unlock(&mgr->lock); return 0; } + mutex_unlock(&mgr->lock); return -EINVAL; } @@ -198,7 +189,7 @@ int amdgpu_ctx_ioctl(struct drm_device *dev, void *data, args->out.alloc.ctx_id = id; break; case AMDGPU_CTX_OP_FREE_CTX: - r = amdgpu_ctx_free(adev, fpriv, id); + r = amdgpu_ctx_free(fpriv, id); break; case AMDGPU_CTX_OP_QUERY_STATE: r = amdgpu_ctx_query(adev, fpriv, id, &args->out); diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c index 801ebfc44034..42d1a22c1199 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c @@ -1525,13 +1525,10 @@ int amdgpu_device_init(struct amdgpu_device *adev, return r; } - if (!adev->kernel_ctx) { - uint32_t id = 0; - r = amdgpu_ctx_alloc(adev, NULL, &id); - if (r) { - dev_err(adev->dev, "failed to create kernel context (%d).\n", r); - return r; - } + r = amdgpu_ctx_init(adev, true, &adev->kernel_ctx); + if (r) { + dev_err(adev->dev, "failed to create kernel context (%d).\n", r); + return r; } r = amdgpu_ib_ring_tests(adev); if (r) @@ -1594,7 +1591,7 @@ void amdgpu_device_fini(struct amdgpu_device *adev) adev->shutdown = true; /* evict vram memory */ amdgpu_bo_evict_vram(adev); - amdgpu_ctx_free(adev, NULL, 0); + amdgpu_ctx_fini(&adev->kernel_ctx); amdgpu_ib_pool_fini(adev); amdgpu_fence_driver_fini(adev); amdgpu_fbdev_fini(adev); diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_sched.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_sched.c index 9f2f19cc4625..995901b9e428 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_sched.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_sched.c @@ -122,19 +122,17 @@ int amdgpu_sched_ib_submit_kernel_helper(struct amdgpu_device *adev, int r = 0; if (amdgpu_enable_scheduler) { struct amdgpu_cs_parser *sched_job = - amdgpu_cs_parser_create(adev, - owner, - adev->kernel_ctx, + amdgpu_cs_parser_create(adev, owner, &adev->kernel_ctx, ibs, 1); if(!sched_job) { return -ENOMEM; } sched_job->free_job = free_job; ibs[num_ibs - 1].sequence = amd_sched_push_job(ring->scheduler, - &adev->kernel_ctx->rings[ring->idx].c_entity, + &adev->kernel_ctx.rings[ring->idx].c_entity, sched_job); r = amd_sched_wait_emit( - &adev->kernel_ctx->rings[ring->idx].c_entity, + &adev->kernel_ctx.rings[ring->idx].c_entity, ibs[num_ibs - 1].sequence, false, -1); if (r) WARN(true, "emit timeout\n"); diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c index ab9c65a245ba..78713ae3b158 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c @@ -372,16 +372,16 @@ static int amdgpu_vm_clear_bo(struct amdgpu_device *adev, if (amdgpu_enable_scheduler) { int r; sched_job = amdgpu_cs_parser_create(adev, AMDGPU_FENCE_OWNER_VM, - adev->kernel_ctx, ib, 1); + &adev->kernel_ctx, ib, 1); if(!sched_job) goto error_free; sched_job->job_param.vm.bo = bo; sched_job->run_job = amdgpu_vm_run_job; sched_job->free_job = amdgpu_vm_free_job; ib->sequence = amd_sched_push_job(ring->scheduler, - &adev->kernel_ctx->rings[ring->idx].c_entity, + &adev->kernel_ctx.rings[ring->idx].c_entity, sched_job); - r = amd_sched_wait_emit(&adev->kernel_ctx->rings[ring->idx].c_entity, + r = amd_sched_wait_emit(&adev->kernel_ctx.rings[ring->idx].c_entity, ib->sequence, false, -1); if (r) DRM_ERROR("emit timeout\n"); @@ -517,7 +517,7 @@ int amdgpu_vm_update_page_directory(struct amdgpu_device *adev, if (amdgpu_enable_scheduler) { int r; sched_job = amdgpu_cs_parser_create(adev, AMDGPU_FENCE_OWNER_VM, - adev->kernel_ctx, + &adev->kernel_ctx, ib, 1); if(!sched_job) goto error_free; @@ -525,9 +525,9 @@ int amdgpu_vm_update_page_directory(struct amdgpu_device *adev, sched_job->run_job = amdgpu_vm_run_job; sched_job->free_job = amdgpu_vm_free_job; ib->sequence = amd_sched_push_job(ring->scheduler, - &adev->kernel_ctx->rings[ring->idx].c_entity, + &adev->kernel_ctx.rings[ring->idx].c_entity, sched_job); - r = amd_sched_wait_emit(&adev->kernel_ctx->rings[ring->idx].c_entity, + r = amd_sched_wait_emit(&adev->kernel_ctx.rings[ring->idx].c_entity, ib->sequence, false, -1); if (r) DRM_ERROR("emit timeout\n"); @@ -863,7 +863,7 @@ static int amdgpu_vm_bo_update_mapping(struct amdgpu_device *adev, if (amdgpu_enable_scheduler) { int r; sched_job = amdgpu_cs_parser_create(adev, AMDGPU_FENCE_OWNER_VM, - adev->kernel_ctx, ib, 1); + &adev->kernel_ctx, ib, 1); if(!sched_job) goto error_free; sched_job->job_param.vm_mapping.vm = vm; @@ -873,9 +873,9 @@ static int amdgpu_vm_bo_update_mapping(struct amdgpu_device *adev, sched_job->run_job = amdgpu_vm_bo_update_mapping_run_job; sched_job->free_job = amdgpu_vm_free_job; ib->sequence = amd_sched_push_job(ring->scheduler, - &adev->kernel_ctx->rings[ring->idx].c_entity, + &adev->kernel_ctx.rings[ring->idx].c_entity, sched_job); - r = amd_sched_wait_emit(&adev->kernel_ctx->rings[ring->idx].c_entity, + r = amd_sched_wait_emit(&adev->kernel_ctx.rings[ring->idx].c_entity, ib->sequence, false, -1); if (r) DRM_ERROR("emit timeout\n"); -- cgit From 80de5913cf31c86d64547af0715de4822c9b1abe Mon Sep 17 00:00:00 2001 From: Chunming Zhou Date: Wed, 5 Aug 2015 19:07:08 +0800 Subject: Revert "drm/amdgpu: return new seq_no for amd_sched_push_job" This reverts commit d1d33da8eb86b8ca41dd9ed95738030df5267b95. Reviewed-by: Christian K?nig Conflicts: drivers/gpu/drm/amd/amdgpu/amdgpu_sched.c drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c --- drivers/gpu/drm/amd/amdgpu/amdgpu_cs.c | 5 +++-- drivers/gpu/drm/amd/amdgpu/amdgpu_sched.c | 9 +++++++-- drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c | 27 +++++++++++++++++++++------ drivers/gpu/drm/amd/scheduler/gpu_scheduler.c | 10 ++++++---- drivers/gpu/drm/amd/scheduler/gpu_scheduler.h | 2 +- 5 files changed, 38 insertions(+), 15 deletions(-) (limited to 'drivers/gpu/drm/amd/amdgpu/amdgpu_sched.c') diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_cs.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_cs.c index aa1bc24b7edb..f72a8583b1a9 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_cs.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_cs.c @@ -901,6 +901,8 @@ int amdgpu_cs_ioctl(struct drm_device *dev, void *data, struct drm_file *filp) if (amdgpu_enable_scheduler && parser->num_ibs) { struct amdgpu_ring * ring = amdgpu_cs_parser_get_ring(adev, parser); + parser->ibs[parser->num_ibs - 1].sequence = atomic64_inc_return( + &parser->ctx->rings[ring->idx].c_entity.last_queued_v_seq); if (ring->is_pte_ring || (parser->bo_list && parser->bo_list->has_userptr)) { r = amdgpu_cs_parser_prepare_job(parser); if (r) @@ -910,8 +912,7 @@ int amdgpu_cs_ioctl(struct drm_device *dev, void *data, struct drm_file *filp) parser->ring = ring; parser->run_job = amdgpu_cs_parser_run_job; parser->free_job = amdgpu_cs_parser_free_job; - parser->ibs[parser->num_ibs - 1].sequence = - amd_sched_push_job(ring->scheduler, + amd_sched_push_job(ring->scheduler, &parser->ctx->rings[ring->idx].c_entity, parser); cs->out.handle = parser->ibs[parser->num_ibs - 1].sequence; diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_sched.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_sched.c index 995901b9e428..0fcf020917d0 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_sched.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_sched.c @@ -121,6 +121,7 @@ int amdgpu_sched_ib_submit_kernel_helper(struct amdgpu_device *adev, { int r = 0; if (amdgpu_enable_scheduler) { + uint64_t v_seq; struct amdgpu_cs_parser *sched_job = amdgpu_cs_parser_create(adev, owner, &adev->kernel_ctx, ibs, 1); @@ -128,12 +129,16 @@ int amdgpu_sched_ib_submit_kernel_helper(struct amdgpu_device *adev, return -ENOMEM; } sched_job->free_job = free_job; - ibs[num_ibs - 1].sequence = amd_sched_push_job(ring->scheduler, + v_seq = atomic64_inc_return(&adev->kernel_ctx.rings[ring->idx].c_entity.last_queued_v_seq); + ibs[num_ibs - 1].sequence = v_seq; + amd_sched_push_job(ring->scheduler, &adev->kernel_ctx.rings[ring->idx].c_entity, sched_job); r = amd_sched_wait_emit( &adev->kernel_ctx.rings[ring->idx].c_entity, - ibs[num_ibs - 1].sequence, false, -1); + v_seq, + false, + -1); if (r) WARN(true, "emit timeout\n"); } else diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c index 78713ae3b158..9d5043c42fc5 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c @@ -371,6 +371,7 @@ static int amdgpu_vm_clear_bo(struct amdgpu_device *adev, if (amdgpu_enable_scheduler) { int r; + uint64_t v_seq; sched_job = amdgpu_cs_parser_create(adev, AMDGPU_FENCE_OWNER_VM, &adev->kernel_ctx, ib, 1); if(!sched_job) @@ -378,11 +379,15 @@ static int amdgpu_vm_clear_bo(struct amdgpu_device *adev, sched_job->job_param.vm.bo = bo; sched_job->run_job = amdgpu_vm_run_job; sched_job->free_job = amdgpu_vm_free_job; - ib->sequence = amd_sched_push_job(ring->scheduler, + v_seq = atomic64_inc_return(&adev->kernel_ctx.rings[ring->idx].c_entity.last_queued_v_seq); + ib->sequence = v_seq; + amd_sched_push_job(ring->scheduler, &adev->kernel_ctx.rings[ring->idx].c_entity, sched_job); r = amd_sched_wait_emit(&adev->kernel_ctx.rings[ring->idx].c_entity, - ib->sequence, false, -1); + v_seq, + false, + -1); if (r) DRM_ERROR("emit timeout\n"); @@ -516,6 +521,7 @@ int amdgpu_vm_update_page_directory(struct amdgpu_device *adev, if (amdgpu_enable_scheduler) { int r; + uint64_t v_seq; sched_job = amdgpu_cs_parser_create(adev, AMDGPU_FENCE_OWNER_VM, &adev->kernel_ctx, ib, 1); @@ -524,11 +530,15 @@ int amdgpu_vm_update_page_directory(struct amdgpu_device *adev, sched_job->job_param.vm.bo = pd; sched_job->run_job = amdgpu_vm_run_job; sched_job->free_job = amdgpu_vm_free_job; - ib->sequence = amd_sched_push_job(ring->scheduler, + v_seq = atomic64_inc_return(&adev->kernel_ctx.rings[ring->idx].c_entity.last_queued_v_seq); + ib->sequence = v_seq; + amd_sched_push_job(ring->scheduler, &adev->kernel_ctx.rings[ring->idx].c_entity, sched_job); r = amd_sched_wait_emit(&adev->kernel_ctx.rings[ring->idx].c_entity, - ib->sequence, false, -1); + v_seq, + false, + -1); if (r) DRM_ERROR("emit timeout\n"); } else { @@ -862,6 +872,7 @@ static int amdgpu_vm_bo_update_mapping(struct amdgpu_device *adev, if (amdgpu_enable_scheduler) { int r; + uint64_t v_seq; sched_job = amdgpu_cs_parser_create(adev, AMDGPU_FENCE_OWNER_VM, &adev->kernel_ctx, ib, 1); if(!sched_job) @@ -872,11 +883,15 @@ static int amdgpu_vm_bo_update_mapping(struct amdgpu_device *adev, sched_job->job_param.vm_mapping.fence = fence; sched_job->run_job = amdgpu_vm_bo_update_mapping_run_job; sched_job->free_job = amdgpu_vm_free_job; - ib->sequence = amd_sched_push_job(ring->scheduler, + v_seq = atomic64_inc_return(&adev->kernel_ctx.rings[ring->idx].c_entity.last_queued_v_seq); + ib->sequence = v_seq; + amd_sched_push_job(ring->scheduler, &adev->kernel_ctx.rings[ring->idx].c_entity, sched_job); r = amd_sched_wait_emit(&adev->kernel_ctx.rings[ring->idx].c_entity, - ib->sequence, false, -1); + v_seq, + false, + -1); if (r) DRM_ERROR("emit timeout\n"); } else { diff --git a/drivers/gpu/drm/amd/scheduler/gpu_scheduler.c b/drivers/gpu/drm/amd/scheduler/gpu_scheduler.c index b9aa572980d2..1204b7386b39 100644 --- a/drivers/gpu/drm/amd/scheduler/gpu_scheduler.c +++ b/drivers/gpu/drm/amd/scheduler/gpu_scheduler.c @@ -289,9 +289,12 @@ int amd_context_entity_fini(struct amd_gpu_scheduler *sched, * @sched The pointer to the scheduler * @c_entity The pointer to amd_context_entity * @job The pointer to job required to submit - * return the virtual sequence number + * return 0 if succeed. -1 if failed. + * -2 indicate queue is full for this client, client should wait untill + * scheduler consum some queued command. + * -1 other fail. */ -uint64_t amd_sched_push_job(struct amd_gpu_scheduler *sched, +int amd_sched_push_job(struct amd_gpu_scheduler *sched, struct amd_context_entity *c_entity, void *job) { @@ -305,8 +308,7 @@ uint64_t amd_sched_push_job(struct amd_gpu_scheduler *sched, } wake_up_interruptible(&sched->wait_queue); - - return atomic64_inc_return(&c_entity->last_queued_v_seq); + return 0; } /** diff --git a/drivers/gpu/drm/amd/scheduler/gpu_scheduler.h b/drivers/gpu/drm/amd/scheduler/gpu_scheduler.h index c46d0854ab75..1a01ac45cd4c 100644 --- a/drivers/gpu/drm/amd/scheduler/gpu_scheduler.h +++ b/drivers/gpu/drm/amd/scheduler/gpu_scheduler.h @@ -124,7 +124,7 @@ struct amd_gpu_scheduler *amd_sched_create(void *device, int amd_sched_destroy(struct amd_gpu_scheduler *sched); -uint64_t amd_sched_push_job(struct amd_gpu_scheduler *sched, +int amd_sched_push_job(struct amd_gpu_scheduler *sched, struct amd_context_entity *c_entity, void *job); -- cgit From 4cef92670bc908aaa48771fc9c72f4bcfb7d6a35 Mon Sep 17 00:00:00 2001 From: Chunming Zhou Date: Wed, 5 Aug 2015 19:52:14 +0800 Subject: drm/amdgpu: process sched job exactly triggered by fence signal Signed-off-by: Chunming Zhou Reviewed-by: Christian K?nig --- drivers/gpu/drm/amd/amdgpu/amdgpu.h | 2 +- drivers/gpu/drm/amd/amdgpu/amdgpu_sched.c | 19 ++++--- drivers/gpu/drm/amd/scheduler/gpu_scheduler.c | 77 ++++++++++++++------------- drivers/gpu/drm/amd/scheduler/gpu_scheduler.h | 18 +++++-- 4 files changed, 68 insertions(+), 48 deletions(-) (limited to 'drivers/gpu/drm/amd/amdgpu/amdgpu_sched.c') diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu.h b/drivers/gpu/drm/amd/amdgpu/amdgpu.h index 53d70f766afe..423cf91ef652 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu.h +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu.h @@ -404,7 +404,7 @@ struct amdgpu_fence_driver { struct amdgpu_fence { struct fence base; - struct fence_cb cb; + /* RB, DMA, etc. */ struct amdgpu_ring *ring; uint64_t seq; diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_sched.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_sched.c index 0fcf020917d0..71a4a7e4b1ae 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_sched.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_sched.c @@ -45,19 +45,24 @@ static int amdgpu_sched_prepare_job(struct amd_gpu_scheduler *sched, static void amdgpu_fence_sched_cb(struct fence *f, struct fence_cb *cb) { - struct amdgpu_fence *fence = - container_of(cb, struct amdgpu_fence, cb); - amd_sched_isr(fence->ring->scheduler); + struct amd_sched_job *sched_job = + container_of(cb, struct amd_sched_job, cb); + amd_sched_process_job(sched_job); } static void amdgpu_sched_run_job(struct amd_gpu_scheduler *sched, struct amd_context_entity *c_entity, - void *job) + struct amd_sched_job *job) { int r = 0; - struct amdgpu_cs_parser *sched_job = (struct amdgpu_cs_parser *)job; + struct amdgpu_cs_parser *sched_job; struct amdgpu_fence *fence; + if (!job || !job->job) { + DRM_ERROR("job is null\n"); + return; + } + sched_job = (struct amdgpu_cs_parser *)job->job; mutex_lock(&sched_job->job_lock); r = amdgpu_ib_schedule(sched_job->adev, sched_job->num_ibs, @@ -67,8 +72,10 @@ static void amdgpu_sched_run_job(struct amd_gpu_scheduler *sched, goto err; fence = sched_job->ibs[sched_job->num_ibs - 1].fence; if (fence_add_callback(&fence->base, - &fence->cb, amdgpu_fence_sched_cb)) + &job->cb, amdgpu_fence_sched_cb)) { + DRM_ERROR("fence add callback failed\n"); goto err; + } if (sched_job->run_job) { r = sched_job->run_job(sched_job); diff --git a/drivers/gpu/drm/amd/scheduler/gpu_scheduler.c b/drivers/gpu/drm/amd/scheduler/gpu_scheduler.c index 1204b7386b39..4c2c5adbc537 100644 --- a/drivers/gpu/drm/amd/scheduler/gpu_scheduler.c +++ b/drivers/gpu/drm/amd/scheduler/gpu_scheduler.c @@ -107,7 +107,14 @@ static int gpu_entity_check_status(struct amd_sched_entity *entity) */ static bool is_scheduler_ready(struct amd_gpu_scheduler *sched) { - return !kfifo_is_full(&sched->active_hw_rq); + unsigned long flags; + bool full; + spin_lock_irqsave(&sched->queue_lock, flags); + full = atomic64_read(&sched->hw_rq_count) < + sched->hw_submission_limit ? true : false; + spin_unlock_irqrestore(&sched->queue_lock, flags); + + return full; } /** @@ -369,6 +376,7 @@ static int amd_sched_main(void *param) sched_setscheduler(current, SCHED_FIFO, &sparam); while (!kthread_should_stop()) { + struct amd_sched_job *sched_job = NULL; wait_event_interruptible(sched->wait_queue, is_scheduler_ready(sched) && (c_entity = select_context(sched))); @@ -376,43 +384,48 @@ static int amd_sched_main(void *param) if (r != sizeof(void *)) continue; r = sched->ops->prepare_job(sched, c_entity, job); - if (!r) - WARN_ON(kfifo_in_spinlocked( - &sched->active_hw_rq, - &job, - sizeof(void *), - &sched->queue_lock) != sizeof(void *)); + if (!r) { + unsigned long flags; + sched_job = kzalloc(sizeof(struct amd_sched_job), + GFP_KERNEL); + if (!sched_job) { + WARN(true, "No memory to allocate\n"); + continue; + } + sched_job->job = job; + sched_job->sched = sched; + spin_lock_irqsave(&sched->queue_lock, flags); + list_add_tail(&sched_job->list, &sched->active_hw_rq); + atomic64_inc(&sched->hw_rq_count); + spin_unlock_irqrestore(&sched->queue_lock, flags); + } mutex_lock(&sched->sched_lock); - sched->ops->run_job(sched, c_entity, job); + sched->ops->run_job(sched, c_entity, sched_job); mutex_unlock(&sched->sched_lock); } return 0; } -uint64_t amd_sched_get_handled_seq(struct amd_gpu_scheduler *sched) -{ - return atomic64_read(&sched->last_handled_seq); -} - /** * ISR to handle EOP inetrrupts * * @sched: gpu scheduler * */ -void amd_sched_isr(struct amd_gpu_scheduler *sched) +void amd_sched_process_job(struct amd_sched_job *sched_job) { - int r; - void *job; - r = kfifo_out_spinlocked(&sched->active_hw_rq, - &job, sizeof(void *), - &sched->queue_lock); - - if (r != sizeof(void *)) - job = NULL; - - sched->ops->process_job(sched, job); - atomic64_inc(&sched->last_handled_seq); + unsigned long flags; + struct amd_gpu_scheduler *sched; + if (!sched_job) + return; + sched = sched_job->sched; + spin_lock_irqsave(&sched->queue_lock, flags); + list_del(&sched_job->list); + atomic64_dec(&sched->hw_rq_count); + spin_unlock_irqrestore(&sched->queue_lock, flags); + + sched->ops->process_job(sched, sched_job->job); + kfree(sched_job); wake_up_interruptible(&sched->wait_queue); } @@ -446,8 +459,7 @@ struct amd_gpu_scheduler *amd_sched_create(void *device, sched->granularity = granularity; sched->ring_id = ring; sched->preemption = preemption; - atomic64_set(&sched->last_handled_seq, 0); - + sched->hw_submission_limit = hw_submission; snprintf(name, sizeof(name), "gpu_sched[%d]", ring); mutex_init(&sched->sched_lock); spin_lock_init(&sched->queue_lock); @@ -458,13 +470,8 @@ struct amd_gpu_scheduler *amd_sched_create(void *device, sched->kernel_rq.check_entity_status = gpu_entity_check_status; init_waitqueue_head(&sched->wait_queue); - if(kfifo_alloc(&sched->active_hw_rq, - hw_submission * sizeof(void *), - GFP_KERNEL)) { - kfree(sched); - return NULL; - } - + INIT_LIST_HEAD(&sched->active_hw_rq); + atomic64_set(&sched->hw_rq_count, 0); /* Each scheduler will run on a seperate kernel thread */ sched->thread = kthread_create(amd_sched_main, sched, name); if (sched->thread) { @@ -473,7 +480,6 @@ struct amd_gpu_scheduler *amd_sched_create(void *device, } DRM_ERROR("Failed to create scheduler for id %d.\n", ring); - kfifo_free(&sched->active_hw_rq); kfree(sched); return NULL; } @@ -488,7 +494,6 @@ struct amd_gpu_scheduler *amd_sched_create(void *device, int amd_sched_destroy(struct amd_gpu_scheduler *sched) { kthread_stop(sched->thread); - kfifo_free(&sched->active_hw_rq); kfree(sched); return 0; } diff --git a/drivers/gpu/drm/amd/scheduler/gpu_scheduler.h b/drivers/gpu/drm/amd/scheduler/gpu_scheduler.h index 1a01ac45cd4c..8a756a565583 100644 --- a/drivers/gpu/drm/amd/scheduler/gpu_scheduler.h +++ b/drivers/gpu/drm/amd/scheduler/gpu_scheduler.h @@ -25,6 +25,7 @@ #define _GPU_SCHEDULER_H_ #include +#include #define AMD_GPU_WAIT_IDLE_TIMEOUT_IN_MS 3000 @@ -80,6 +81,13 @@ struct amd_context_entity { bool is_pending; }; +struct amd_sched_job { + struct list_head list; + struct fence_cb cb; + struct amd_gpu_scheduler *sched; + void *job; +}; + /** * Define the backend operations called by the scheduler, * these functions should be implemented in driver side @@ -90,7 +98,7 @@ struct amd_sched_backend_ops { void *job); void (*run_job)(struct amd_gpu_scheduler *sched, struct amd_context_entity *c_entity, - void *job); + struct amd_sched_job *job); void (*process_job)(struct amd_gpu_scheduler *sched, void *job); }; @@ -102,19 +110,19 @@ struct amd_gpu_scheduler { struct task_struct *thread; struct amd_run_queue sched_rq; struct amd_run_queue kernel_rq; - struct kfifo active_hw_rq; + struct list_head active_hw_rq; + atomic64_t hw_rq_count; struct amd_sched_backend_ops *ops; uint32_t ring_id; uint32_t granularity; /* in ms unit */ uint32_t preemption; - atomic64_t last_handled_seq; wait_queue_head_t wait_queue; struct amd_context_entity *current_entity; struct mutex sched_lock; spinlock_t queue_lock; + uint32_t hw_submission_limit; }; - struct amd_gpu_scheduler *amd_sched_create(void *device, struct amd_sched_backend_ops *ops, uint32_t ring, @@ -133,7 +141,7 @@ int amd_sched_wait_emit(struct amd_context_entity *c_entity, bool intr, long timeout); -void amd_sched_isr(struct amd_gpu_scheduler *sched); +void amd_sched_process_job(struct amd_sched_job *sched_job); uint64_t amd_sched_get_handled_seq(struct amd_gpu_scheduler *sched); int amd_context_entity_fini(struct amd_gpu_scheduler *sched, -- cgit From 91404fb20825418fd9ab8e6533bc336e1ffc748e Mon Sep 17 00:00:00 2001 From: Christian König Date: Wed, 5 Aug 2015 18:33:21 +0200 Subject: drm/amdgpu: merge amd_sched_entity and amd_context_entity v2 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Avoiding a couple of casts. v2: rename c_entity to entity as well Signed-off-by: Christian König Reviewed-by: Chunming Zhou --- drivers/gpu/drm/amd/amdgpu/amdgpu.h | 6 +-- drivers/gpu/drm/amd/amdgpu/amdgpu_cs.c | 4 +- drivers/gpu/drm/amd/amdgpu/amdgpu_ctx.c | 18 +++---- drivers/gpu/drm/amd/amdgpu/amdgpu_sched.c | 12 ++--- drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c | 18 +++---- drivers/gpu/drm/amd/scheduler/gpu_scheduler.c | 67 +++++++++++---------------- drivers/gpu/drm/amd/scheduler/gpu_scheduler.h | 57 ++++++++++------------- 7 files changed, 81 insertions(+), 101 deletions(-) (limited to 'drivers/gpu/drm/amd/amdgpu/amdgpu_sched.c') diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu.h b/drivers/gpu/drm/amd/amdgpu/amdgpu.h index 423cf91ef652..1e6800050ad8 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu.h +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu.h @@ -1013,9 +1013,9 @@ struct amdgpu_vm_manager { #define AMDGPU_CTX_MAX_CS_PENDING 16 struct amdgpu_ctx_ring { - uint64_t sequence; - struct fence *fences[AMDGPU_CTX_MAX_CS_PENDING]; - struct amd_context_entity c_entity; + uint64_t sequence; + struct fence *fences[AMDGPU_CTX_MAX_CS_PENDING]; + struct amd_sched_entity entity; }; struct amdgpu_ctx { diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_cs.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_cs.c index d26688ddaa20..b1dc7e1ed271 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_cs.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_cs.c @@ -900,7 +900,7 @@ int amdgpu_cs_ioctl(struct drm_device *dev, void *data, struct drm_file *filp) struct amdgpu_ring * ring = amdgpu_cs_parser_get_ring(adev, parser); parser->ibs[parser->num_ibs - 1].sequence = atomic64_inc_return( - &parser->ctx->rings[ring->idx].c_entity.last_queued_v_seq); + &parser->ctx->rings[ring->idx].entity.last_queued_v_seq); if (ring->is_pte_ring || (parser->bo_list && parser->bo_list->has_userptr)) { r = amdgpu_cs_parser_prepare_job(parser); if (r) @@ -911,7 +911,7 @@ int amdgpu_cs_ioctl(struct drm_device *dev, void *data, struct drm_file *filp) parser->run_job = amdgpu_cs_parser_run_job; parser->free_job = amdgpu_cs_parser_free_job; amd_sched_push_job(ring->scheduler, - &parser->ctx->rings[ring->idx].c_entity, + &parser->ctx->rings[ring->idx].entity, parser); cs->out.handle = parser->ibs[parser->num_ibs - 1].sequence; up_read(&adev->exclusive_lock); diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_ctx.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_ctx.c index e04364cdcc9e..232e800eea56 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_ctx.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_ctx.c @@ -46,17 +46,17 @@ int amdgpu_ctx_init(struct amdgpu_device *adev, bool kernel, rq = &adev->rings[i]->scheduler->kernel_rq; else rq = &adev->rings[i]->scheduler->sched_rq; - r = amd_context_entity_init(adev->rings[i]->scheduler, - &ctx->rings[i].c_entity, - rq, amdgpu_sched_jobs); + r = amd_sched_entity_init(adev->rings[i]->scheduler, + &ctx->rings[i].entity, + rq, amdgpu_sched_jobs); if (r) break; } if (i < adev->num_rings) { for (j = 0; j < i; j++) - amd_context_entity_fini(adev->rings[j]->scheduler, - &ctx->rings[j].c_entity); + amd_sched_entity_fini(adev->rings[j]->scheduler, + &ctx->rings[j].entity); kfree(ctx); return r; } @@ -75,8 +75,8 @@ void amdgpu_ctx_fini(struct amdgpu_ctx *ctx) if (amdgpu_enable_scheduler) { for (i = 0; i < adev->num_rings; i++) - amd_context_entity_fini(adev->rings[i]->scheduler, - &ctx->rings[i].c_entity); + amd_sched_entity_fini(adev->rings[i]->scheduler, + &ctx->rings[i].entity); } } @@ -271,7 +271,7 @@ struct fence *amdgpu_ctx_get_fence(struct amdgpu_ctx *ctx, int r; if (amdgpu_enable_scheduler) { - r = amd_sched_wait_emit(&cring->c_entity, + r = amd_sched_wait_emit(&cring->entity, seq, false, -1); @@ -281,7 +281,7 @@ struct fence *amdgpu_ctx_get_fence(struct amdgpu_ctx *ctx, spin_lock(&ctx->ring_lock); if (amdgpu_enable_scheduler) - queued_seq = amd_sched_next_queued_seq(&cring->c_entity); + queued_seq = amd_sched_next_queued_seq(&cring->entity); else queued_seq = cring->sequence; diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_sched.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_sched.c index 71a4a7e4b1ae..787b93db6796 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_sched.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_sched.c @@ -28,7 +28,7 @@ #include "amdgpu.h" static int amdgpu_sched_prepare_job(struct amd_gpu_scheduler *sched, - struct amd_context_entity *c_entity, + struct amd_sched_entity *entity, void *job) { int r = 0; @@ -51,7 +51,7 @@ static void amdgpu_fence_sched_cb(struct fence *f, struct fence_cb *cb) } static void amdgpu_sched_run_job(struct amd_gpu_scheduler *sched, - struct amd_context_entity *c_entity, + struct amd_sched_entity *entity, struct amd_sched_job *job) { int r = 0; @@ -83,7 +83,7 @@ static void amdgpu_sched_run_job(struct amd_gpu_scheduler *sched, goto err; } - amd_sched_emit(c_entity, sched_job->ibs[sched_job->num_ibs - 1].sequence); + amd_sched_emit(entity, sched_job->ibs[sched_job->num_ibs - 1].sequence); mutex_unlock(&sched_job->job_lock); return; @@ -136,13 +136,13 @@ int amdgpu_sched_ib_submit_kernel_helper(struct amdgpu_device *adev, return -ENOMEM; } sched_job->free_job = free_job; - v_seq = atomic64_inc_return(&adev->kernel_ctx.rings[ring->idx].c_entity.last_queued_v_seq); + v_seq = atomic64_inc_return(&adev->kernel_ctx.rings[ring->idx].entity.last_queued_v_seq); ibs[num_ibs - 1].sequence = v_seq; amd_sched_push_job(ring->scheduler, - &adev->kernel_ctx.rings[ring->idx].c_entity, + &adev->kernel_ctx.rings[ring->idx].entity, sched_job); r = amd_sched_wait_emit( - &adev->kernel_ctx.rings[ring->idx].c_entity, + &adev->kernel_ctx.rings[ring->idx].entity, v_seq, false, -1); diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c index 9d5043c42fc5..230bf1f34ead 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c @@ -379,12 +379,12 @@ static int amdgpu_vm_clear_bo(struct amdgpu_device *adev, sched_job->job_param.vm.bo = bo; sched_job->run_job = amdgpu_vm_run_job; sched_job->free_job = amdgpu_vm_free_job; - v_seq = atomic64_inc_return(&adev->kernel_ctx.rings[ring->idx].c_entity.last_queued_v_seq); + v_seq = atomic64_inc_return(&adev->kernel_ctx.rings[ring->idx].entity.last_queued_v_seq); ib->sequence = v_seq; amd_sched_push_job(ring->scheduler, - &adev->kernel_ctx.rings[ring->idx].c_entity, + &adev->kernel_ctx.rings[ring->idx].entity, sched_job); - r = amd_sched_wait_emit(&adev->kernel_ctx.rings[ring->idx].c_entity, + r = amd_sched_wait_emit(&adev->kernel_ctx.rings[ring->idx].entity, v_seq, false, -1); @@ -530,12 +530,12 @@ int amdgpu_vm_update_page_directory(struct amdgpu_device *adev, sched_job->job_param.vm.bo = pd; sched_job->run_job = amdgpu_vm_run_job; sched_job->free_job = amdgpu_vm_free_job; - v_seq = atomic64_inc_return(&adev->kernel_ctx.rings[ring->idx].c_entity.last_queued_v_seq); + v_seq = atomic64_inc_return(&adev->kernel_ctx.rings[ring->idx].entity.last_queued_v_seq); ib->sequence = v_seq; amd_sched_push_job(ring->scheduler, - &adev->kernel_ctx.rings[ring->idx].c_entity, + &adev->kernel_ctx.rings[ring->idx].entity, sched_job); - r = amd_sched_wait_emit(&adev->kernel_ctx.rings[ring->idx].c_entity, + r = amd_sched_wait_emit(&adev->kernel_ctx.rings[ring->idx].entity, v_seq, false, -1); @@ -883,12 +883,12 @@ static int amdgpu_vm_bo_update_mapping(struct amdgpu_device *adev, sched_job->job_param.vm_mapping.fence = fence; sched_job->run_job = amdgpu_vm_bo_update_mapping_run_job; sched_job->free_job = amdgpu_vm_free_job; - v_seq = atomic64_inc_return(&adev->kernel_ctx.rings[ring->idx].c_entity.last_queued_v_seq); + v_seq = atomic64_inc_return(&adev->kernel_ctx.rings[ring->idx].entity.last_queued_v_seq); ib->sequence = v_seq; amd_sched_push_job(ring->scheduler, - &adev->kernel_ctx.rings[ring->idx].c_entity, + &adev->kernel_ctx.rings[ring->idx].entity, sched_job); - r = amd_sched_wait_emit(&adev->kernel_ctx.rings[ring->idx].c_entity, + r = amd_sched_wait_emit(&adev->kernel_ctx.rings[ring->idx].entity, v_seq, false, -1); diff --git a/drivers/gpu/drm/amd/scheduler/gpu_scheduler.c b/drivers/gpu/drm/amd/scheduler/gpu_scheduler.c index 1f78ad60224a..eb3b0993a8cd 100644 --- a/drivers/gpu/drm/amd/scheduler/gpu_scheduler.c +++ b/drivers/gpu/drm/amd/scheduler/gpu_scheduler.c @@ -76,7 +76,7 @@ static struct amd_sched_entity *rq_select_entity(struct amd_run_queue *rq) return i ? p : NULL; } -static bool context_entity_is_waiting(struct amd_context_entity *entity) +static bool context_entity_is_waiting(struct amd_sched_entity *entity) { /* TODO: sync obj for multi-ring synchronization */ return false; @@ -84,14 +84,11 @@ static bool context_entity_is_waiting(struct amd_context_entity *entity) static int gpu_entity_check_status(struct amd_sched_entity *entity) { - struct amd_context_entity *tmp; - if (entity == &entity->belongto_rq->head) return -1; - tmp = container_of(entity, typeof(*tmp), generic_entity); - if (kfifo_is_empty(&tmp->job_queue) || - context_entity_is_waiting(tmp)) + if (kfifo_is_empty(&entity->job_queue) || + context_entity_is_waiting(entity)) return -1; return 0; @@ -123,31 +120,26 @@ static bool is_scheduler_ready(struct amd_gpu_scheduler *sched) * Select next entity from the kernel run queue, if not available, * return null. */ -static struct amd_context_entity * +static struct amd_sched_entity * kernel_rq_select_context(struct amd_gpu_scheduler *sched) { struct amd_sched_entity *sched_entity; - struct amd_context_entity *tmp = NULL; struct amd_run_queue *rq = &sched->kernel_rq; mutex_lock(&rq->lock); sched_entity = rq_select_entity(rq); - if (sched_entity) - tmp = container_of(sched_entity, - typeof(*tmp), - generic_entity); mutex_unlock(&rq->lock); - return tmp; + return sched_entity; } /** * Select next entity containing real IB submissions */ -static struct amd_context_entity * +static struct amd_sched_entity * select_context(struct amd_gpu_scheduler *sched) { - struct amd_context_entity *wake_entity = NULL; - struct amd_context_entity *tmp; + struct amd_sched_entity *wake_entity = NULL; + struct amd_sched_entity *tmp; struct amd_run_queue *rq; if (!is_scheduler_ready(sched)) @@ -158,12 +150,9 @@ select_context(struct amd_gpu_scheduler *sched) if (tmp != NULL) goto exit; - WARN_ON(offsetof(struct amd_context_entity, generic_entity) != 0); - rq = &sched->sched_rq; mutex_lock(&rq->lock); - tmp = container_of(rq_select_entity(rq), - typeof(*tmp), generic_entity); + tmp = rq_select_entity(rq); mutex_unlock(&rq->lock); exit: if (sched->current_entity && (sched->current_entity != tmp)) @@ -178,15 +167,15 @@ exit: * Init a context entity used by scheduler when submit to HW ring. * * @sched The pointer to the scheduler - * @entity The pointer to a valid amd_context_entity + * @entity The pointer to a valid amd_sched_entity * @rq The run queue this entity belongs * @kernel If this is an entity for the kernel * @jobs The max number of jobs in the job queue * * return 0 if succeed. negative error code on failure */ -int amd_context_entity_init(struct amd_gpu_scheduler *sched, - struct amd_context_entity *entity, +int amd_sched_entity_init(struct amd_gpu_scheduler *sched, + struct amd_sched_entity *entity, struct amd_run_queue *rq, uint32_t jobs) { @@ -195,10 +184,10 @@ int amd_context_entity_init(struct amd_gpu_scheduler *sched, if (!(sched && entity && rq)) return -EINVAL; - memset(entity, 0, sizeof(struct amd_context_entity)); + memset(entity, 0, sizeof(struct amd_sched_entity)); seq_ring = ((uint64_t)sched->ring_id) << 60; spin_lock_init(&entity->lock); - entity->generic_entity.belongto_rq = rq; + entity->belongto_rq = rq; entity->scheduler = sched; init_waitqueue_head(&entity->wait_queue); init_waitqueue_head(&entity->wait_emit); @@ -213,7 +202,7 @@ int amd_context_entity_init(struct amd_gpu_scheduler *sched, /* Add the entity to the run queue */ mutex_lock(&rq->lock); - rq_add_entity(rq, &entity->generic_entity); + rq_add_entity(rq, entity); mutex_unlock(&rq->lock); return 0; } @@ -227,14 +216,14 @@ int amd_context_entity_init(struct amd_gpu_scheduler *sched, * return true if entity is initialized, false otherwise */ static bool is_context_entity_initialized(struct amd_gpu_scheduler *sched, - struct amd_context_entity *entity) + struct amd_sched_entity *entity) { return entity->scheduler == sched && - entity->generic_entity.belongto_rq != NULL; + entity->belongto_rq != NULL; } static bool is_context_entity_idle(struct amd_gpu_scheduler *sched, - struct amd_context_entity *entity) + struct amd_sched_entity *entity) { /** * Idle means no pending IBs, and the entity is not @@ -256,11 +245,11 @@ static bool is_context_entity_idle(struct amd_gpu_scheduler *sched, * * return 0 if succeed. negative error code on failure */ -int amd_context_entity_fini(struct amd_gpu_scheduler *sched, - struct amd_context_entity *entity) +int amd_sched_entity_fini(struct amd_gpu_scheduler *sched, + struct amd_sched_entity *entity) { int r = 0; - struct amd_run_queue *rq = entity->generic_entity.belongto_rq; + struct amd_run_queue *rq = entity->belongto_rq; if (!is_context_entity_initialized(sched, entity)) return 0; @@ -283,7 +272,7 @@ int amd_context_entity_fini(struct amd_gpu_scheduler *sched, } mutex_lock(&rq->lock); - rq_remove_entity(rq, &entity->generic_entity); + rq_remove_entity(rq, entity); mutex_unlock(&rq->lock); kfifo_free(&entity->job_queue); return r; @@ -293,7 +282,7 @@ int amd_context_entity_fini(struct amd_gpu_scheduler *sched, * Submit a normal job to the job queue * * @sched The pointer to the scheduler - * @c_entity The pointer to amd_context_entity + * @c_entity The pointer to amd_sched_entity * @job The pointer to job required to submit * return 0 if succeed. -1 if failed. * -2 indicate queue is full for this client, client should wait untill @@ -301,7 +290,7 @@ int amd_context_entity_fini(struct amd_gpu_scheduler *sched, * -1 other fail. */ int amd_sched_push_job(struct amd_gpu_scheduler *sched, - struct amd_context_entity *c_entity, + struct amd_sched_entity *c_entity, void *job) { while (kfifo_in_spinlocked(&c_entity->job_queue, &job, sizeof(void *), @@ -328,7 +317,7 @@ int amd_sched_push_job(struct amd_gpu_scheduler *sched, * * return =0 signaled , <0 failed */ -int amd_sched_wait_emit(struct amd_context_entity *c_entity, +int amd_sched_wait_emit(struct amd_sched_entity *c_entity, uint64_t seq, bool intr, long timeout) @@ -369,7 +358,7 @@ static int amd_sched_main(void *param) int r; void *job; struct sched_param sparam = {.sched_priority = 1}; - struct amd_context_entity *c_entity = NULL; + struct amd_sched_entity *c_entity = NULL; struct amd_gpu_scheduler *sched = (struct amd_gpu_scheduler *)param; sched_setscheduler(current, SCHED_FIFO, &sparam); @@ -505,7 +494,7 @@ int amd_sched_destroy(struct amd_gpu_scheduler *sched) * @entity The context entity * @seq The sequence number for the latest emitted job */ -void amd_sched_emit(struct amd_context_entity *c_entity, uint64_t seq) +void amd_sched_emit(struct amd_sched_entity *c_entity, uint64_t seq) { atomic64_set(&c_entity->last_emitted_v_seq, seq); wake_up_all(&c_entity->wait_emit); @@ -518,7 +507,7 @@ void amd_sched_emit(struct amd_context_entity *c_entity, uint64_t seq) * * return the next queued sequence number */ -uint64_t amd_sched_next_queued_seq(struct amd_context_entity *c_entity) +uint64_t amd_sched_next_queued_seq(struct amd_sched_entity *c_entity) { return atomic64_read(&c_entity->last_queued_v_seq) + 1; } diff --git a/drivers/gpu/drm/amd/scheduler/gpu_scheduler.h b/drivers/gpu/drm/amd/scheduler/gpu_scheduler.h index 64ef0e2b1543..a3e29df957fc 100644 --- a/drivers/gpu/drm/amd/scheduler/gpu_scheduler.h +++ b/drivers/gpu/drm/amd/scheduler/gpu_scheduler.h @@ -41,6 +41,17 @@ struct amd_run_queue; struct amd_sched_entity { struct list_head list; struct amd_run_queue *belongto_rq; + spinlock_t lock; + /* the virtual_seq is unique per context per ring */ + atomic64_t last_queued_v_seq; + atomic64_t last_emitted_v_seq; + /* the job_queue maintains the jobs submitted by clients */ + struct kfifo job_queue; + spinlock_t queue_lock; + struct amd_gpu_scheduler *scheduler; + wait_queue_head_t wait_queue; + wait_queue_head_t wait_emit; + bool is_pending; }; /** @@ -61,25 +72,6 @@ struct amd_run_queue { int (*check_entity_status)(struct amd_sched_entity *entity); }; -/** - * Context based scheduler entity, there can be multiple entities for - * each context, and one entity per ring -*/ -struct amd_context_entity { - struct amd_sched_entity generic_entity; - spinlock_t lock; - /* the virtual_seq is unique per context per ring */ - atomic64_t last_queued_v_seq; - atomic64_t last_emitted_v_seq; - /* the job_queue maintains the jobs submitted by clients */ - struct kfifo job_queue; - spinlock_t queue_lock; - struct amd_gpu_scheduler *scheduler; - wait_queue_head_t wait_queue; - wait_queue_head_t wait_emit; - bool is_pending; -}; - struct amd_sched_job { struct list_head list; struct fence_cb cb; @@ -93,10 +85,10 @@ struct amd_sched_job { */ struct amd_sched_backend_ops { int (*prepare_job)(struct amd_gpu_scheduler *sched, - struct amd_context_entity *c_entity, + struct amd_sched_entity *c_entity, void *job); void (*run_job)(struct amd_gpu_scheduler *sched, - struct amd_context_entity *c_entity, + struct amd_sched_entity *c_entity, struct amd_sched_job *job); void (*process_job)(struct amd_gpu_scheduler *sched, void *job); }; @@ -116,7 +108,7 @@ struct amd_gpu_scheduler { uint32_t granularity; /* in ms unit */ uint32_t preemption; wait_queue_head_t wait_queue; - struct amd_context_entity *current_entity; + struct amd_sched_entity *current_entity; struct mutex sched_lock; spinlock_t queue_lock; uint32_t hw_submission_limit; @@ -132,10 +124,10 @@ struct amd_gpu_scheduler *amd_sched_create(void *device, int amd_sched_destroy(struct amd_gpu_scheduler *sched); int amd_sched_push_job(struct amd_gpu_scheduler *sched, - struct amd_context_entity *c_entity, + struct amd_sched_entity *c_entity, void *job); -int amd_sched_wait_emit(struct amd_context_entity *c_entity, +int amd_sched_wait_emit(struct amd_sched_entity *c_entity, uint64_t seq, bool intr, long timeout); @@ -143,16 +135,15 @@ int amd_sched_wait_emit(struct amd_context_entity *c_entity, void amd_sched_process_job(struct amd_sched_job *sched_job); uint64_t amd_sched_get_handled_seq(struct amd_gpu_scheduler *sched); -int amd_context_entity_fini(struct amd_gpu_scheduler *sched, - struct amd_context_entity *entity); - -int amd_context_entity_init(struct amd_gpu_scheduler *sched, - struct amd_context_entity *entity, - struct amd_run_queue *rq, - uint32_t jobs); +int amd_sched_entity_init(struct amd_gpu_scheduler *sched, + struct amd_sched_entity *entity, + struct amd_run_queue *rq, + uint32_t jobs); +int amd_sched_entity_fini(struct amd_gpu_scheduler *sched, + struct amd_sched_entity *entity); -void amd_sched_emit(struct amd_context_entity *c_entity, uint64_t seq); +void amd_sched_emit(struct amd_sched_entity *c_entity, uint64_t seq); -uint64_t amd_sched_next_queued_seq(struct amd_context_entity *c_entity); +uint64_t amd_sched_next_queued_seq(struct amd_sched_entity *c_entity); #endif -- cgit From 6f0e54a964932d3d5252ac1ff7ab153c984a5d51 Mon Sep 17 00:00:00 2001 From: Christian König Date: Wed, 5 Aug 2015 21:22:10 +0200 Subject: drm/amdgpu: cleanup and fix scheduler fence handling v2 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit v2: rebased Signed-off-by: Christian König Reviewed-by: Alex Deucher (v1) Reviewed-by: Chunming Zhou --- drivers/gpu/drm/amd/amdgpu/amdgpu_sched.c | 26 ++++-------- drivers/gpu/drm/amd/scheduler/gpu_scheduler.c | 61 +++++++++++++++------------ drivers/gpu/drm/amd/scheduler/gpu_scheduler.h | 7 ++- 3 files changed, 44 insertions(+), 50 deletions(-) (limited to 'drivers/gpu/drm/amd/amdgpu/amdgpu_sched.c') diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_sched.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_sched.c index 787b93db6796..039bd1f748f0 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_sched.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_sched.c @@ -43,16 +43,9 @@ static int amdgpu_sched_prepare_job(struct amd_gpu_scheduler *sched, return r; } -static void amdgpu_fence_sched_cb(struct fence *f, struct fence_cb *cb) -{ - struct amd_sched_job *sched_job = - container_of(cb, struct amd_sched_job, cb); - amd_sched_process_job(sched_job); -} - -static void amdgpu_sched_run_job(struct amd_gpu_scheduler *sched, - struct amd_sched_entity *entity, - struct amd_sched_job *job) +static struct fence *amdgpu_sched_run_job(struct amd_gpu_scheduler *sched, + struct amd_sched_entity *entity, + struct amd_sched_job *job) { int r = 0; struct amdgpu_cs_parser *sched_job; @@ -60,7 +53,7 @@ static void amdgpu_sched_run_job(struct amd_gpu_scheduler *sched, if (!job || !job->job) { DRM_ERROR("job is null\n"); - return; + return NULL; } sched_job = (struct amdgpu_cs_parser *)job->job; mutex_lock(&sched_job->job_lock); @@ -70,12 +63,7 @@ static void amdgpu_sched_run_job(struct amd_gpu_scheduler *sched, sched_job->filp); if (r) goto err; - fence = sched_job->ibs[sched_job->num_ibs - 1].fence; - if (fence_add_callback(&fence->base, - &job->cb, amdgpu_fence_sched_cb)) { - DRM_ERROR("fence add callback failed\n"); - goto err; - } + fence = amdgpu_fence_ref(sched_job->ibs[sched_job->num_ibs - 1].fence); if (sched_job->run_job) { r = sched_job->run_job(sched_job); @@ -86,11 +74,13 @@ static void amdgpu_sched_run_job(struct amd_gpu_scheduler *sched, amd_sched_emit(entity, sched_job->ibs[sched_job->num_ibs - 1].sequence); mutex_unlock(&sched_job->job_lock); - return; + return &fence->base; + err: DRM_ERROR("Run job error\n"); mutex_unlock(&sched_job->job_lock); schedule_work(&sched_job->job_work); + return NULL; } static void amdgpu_sched_process_job(struct amd_gpu_scheduler *sched, void *job) diff --git a/drivers/gpu/drm/amd/scheduler/gpu_scheduler.c b/drivers/gpu/drm/amd/scheduler/gpu_scheduler.c index eb3b0993a8cd..438dc23f4bb3 100644 --- a/drivers/gpu/drm/amd/scheduler/gpu_scheduler.c +++ b/drivers/gpu/drm/amd/scheduler/gpu_scheduler.c @@ -175,9 +175,9 @@ exit: * return 0 if succeed. negative error code on failure */ int amd_sched_entity_init(struct amd_gpu_scheduler *sched, - struct amd_sched_entity *entity, - struct amd_run_queue *rq, - uint32_t jobs) + struct amd_sched_entity *entity, + struct amd_run_queue *rq, + uint32_t jobs) { uint64_t seq_ring = 0; @@ -353,6 +353,24 @@ int amd_sched_wait_emit(struct amd_sched_entity *c_entity, return 0; } +static void amd_sched_process_job(struct fence *f, struct fence_cb *cb) +{ + struct amd_sched_job *sched_job = + container_of(cb, struct amd_sched_job, cb); + struct amd_gpu_scheduler *sched; + unsigned long flags; + + sched = sched_job->sched; + spin_lock_irqsave(&sched->queue_lock, flags); + list_del(&sched_job->list); + atomic64_dec(&sched->hw_rq_count); + spin_unlock_irqrestore(&sched->queue_lock, flags); + + sched->ops->process_job(sched, sched_job->job); + kfree(sched_job); + wake_up_interruptible(&sched->wait_queue); +} + static int amd_sched_main(void *param) { int r; @@ -365,6 +383,8 @@ static int amd_sched_main(void *param) while (!kthread_should_stop()) { struct amd_sched_job *sched_job = NULL; + struct fence *fence; + wait_event_interruptible(sched->wait_queue, is_scheduler_ready(sched) && (c_entity = select_context(sched))); @@ -388,36 +408,21 @@ static int amd_sched_main(void *param) spin_unlock_irqrestore(&sched->queue_lock, flags); } mutex_lock(&sched->sched_lock); - sched->ops->run_job(sched, c_entity, sched_job); + fence = sched->ops->run_job(sched, c_entity, sched_job); + if (fence) { + r = fence_add_callback(fence, &sched_job->cb, + amd_sched_process_job); + if (r == -ENOENT) + amd_sched_process_job(fence, &sched_job->cb); + else if (r) + DRM_ERROR("fence add callback failed (%d)\n", r); + fence_put(fence); + } mutex_unlock(&sched->sched_lock); } return 0; } -/** - * ISR to handle EOP inetrrupts - * - * @sched: gpu scheduler - * -*/ -void amd_sched_process_job(struct amd_sched_job *sched_job) -{ - unsigned long flags; - struct amd_gpu_scheduler *sched; - - if (!sched_job) - return; - sched = sched_job->sched; - spin_lock_irqsave(&sched->queue_lock, flags); - list_del(&sched_job->list); - atomic64_dec(&sched->hw_rq_count); - spin_unlock_irqrestore(&sched->queue_lock, flags); - - sched->ops->process_job(sched, sched_job->job); - kfree(sched_job); - wake_up_interruptible(&sched->wait_queue); -} - /** * Create a gpu scheduler * diff --git a/drivers/gpu/drm/amd/scheduler/gpu_scheduler.h b/drivers/gpu/drm/amd/scheduler/gpu_scheduler.h index a3e29df957fc..e7cc40a6993b 100644 --- a/drivers/gpu/drm/amd/scheduler/gpu_scheduler.h +++ b/drivers/gpu/drm/amd/scheduler/gpu_scheduler.h @@ -87,9 +87,9 @@ struct amd_sched_backend_ops { int (*prepare_job)(struct amd_gpu_scheduler *sched, struct amd_sched_entity *c_entity, void *job); - void (*run_job)(struct amd_gpu_scheduler *sched, - struct amd_sched_entity *c_entity, - struct amd_sched_job *job); + struct fence *(*run_job)(struct amd_gpu_scheduler *sched, + struct amd_sched_entity *c_entity, + struct amd_sched_job *job); void (*process_job)(struct amd_gpu_scheduler *sched, void *job); }; @@ -132,7 +132,6 @@ int amd_sched_wait_emit(struct amd_sched_entity *c_entity, bool intr, long timeout); -void amd_sched_process_job(struct amd_sched_job *sched_job); uint64_t amd_sched_get_handled_seq(struct amd_gpu_scheduler *sched); int amd_sched_entity_init(struct amd_gpu_scheduler *sched, -- cgit From 953e8fd4e734857f6dabbaf325035bf10c4a9c7a Mon Sep 17 00:00:00 2001 From: Chunming Zhou Date: Thu, 6 Aug 2015 15:19:12 +0800 Subject: drm/amdgpu: use amd_sched_job in its backend ops Signed-off-by: Chunming Zhou Reviewed-by: Christian K?nig --- drivers/gpu/drm/amd/amdgpu/amdgpu_sched.c | 34 +++++++++++++-------------- drivers/gpu/drm/amd/scheduler/gpu_scheduler.c | 30 +++++++++++------------ drivers/gpu/drm/amd/scheduler/gpu_scheduler.h | 11 +++++---- 3 files changed, 37 insertions(+), 38 deletions(-) (limited to 'drivers/gpu/drm/amd/amdgpu/amdgpu_sched.c') diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_sched.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_sched.c index 039bd1f748f0..d13d01511694 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_sched.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_sched.c @@ -29,10 +29,16 @@ static int amdgpu_sched_prepare_job(struct amd_gpu_scheduler *sched, struct amd_sched_entity *entity, - void *job) + struct amd_sched_job *job) { int r = 0; - struct amdgpu_cs_parser *sched_job = (struct amdgpu_cs_parser *)job; + struct amdgpu_cs_parser *sched_job; + if (!job || !job->data) { + DRM_ERROR("job is null\n"); + return -EINVAL; + } + + sched_job = (struct amdgpu_cs_parser *)job->data; if (sched_job->prepare_job) { r = sched_job->prepare_job(sched_job); if (r) { @@ -51,11 +57,11 @@ static struct fence *amdgpu_sched_run_job(struct amd_gpu_scheduler *sched, struct amdgpu_cs_parser *sched_job; struct amdgpu_fence *fence; - if (!job || !job->job) { + if (!job || !job->data) { DRM_ERROR("job is null\n"); return NULL; } - sched_job = (struct amdgpu_cs_parser *)job->job; + sched_job = (struct amdgpu_cs_parser *)job->data; mutex_lock(&sched_job->job_lock); r = amdgpu_ib_schedule(sched_job->adev, sched_job->num_ibs, @@ -83,22 +89,16 @@ err: return NULL; } -static void amdgpu_sched_process_job(struct amd_gpu_scheduler *sched, void *job) +static void amdgpu_sched_process_job(struct amd_gpu_scheduler *sched, + struct amd_sched_job *job) { - struct amdgpu_cs_parser *sched_job = NULL; - struct amdgpu_fence *fence = NULL; - struct amdgpu_ring *ring = NULL; - struct amdgpu_device *adev = NULL; + struct amdgpu_cs_parser *sched_job; - if (!job) - return; - sched_job = (struct amdgpu_cs_parser *)job; - fence = sched_job->ibs[sched_job->num_ibs - 1].fence; - if (!fence) + if (!job || !job->data) { + DRM_ERROR("job is null\n"); return; - ring = fence->ring; - adev = ring->adev; - + } + sched_job = (struct amdgpu_cs_parser *)job->data; schedule_work(&sched_job->job_work); } diff --git a/drivers/gpu/drm/amd/scheduler/gpu_scheduler.c b/drivers/gpu/drm/amd/scheduler/gpu_scheduler.c index 438dc23f4bb3..33b4f55e48b1 100644 --- a/drivers/gpu/drm/amd/scheduler/gpu_scheduler.c +++ b/drivers/gpu/drm/amd/scheduler/gpu_scheduler.c @@ -291,8 +291,15 @@ int amd_sched_entity_fini(struct amd_gpu_scheduler *sched, */ int amd_sched_push_job(struct amd_gpu_scheduler *sched, struct amd_sched_entity *c_entity, - void *job) + void *data) { + struct amd_sched_job *job = kzalloc(sizeof(struct amd_sched_job), + GFP_KERNEL); + if (!job) + return -ENOMEM; + job->sched = sched; + job->s_entity = c_entity; + job->data = data; while (kfifo_in_spinlocked(&c_entity->job_queue, &job, sizeof(void *), &c_entity->queue_lock) != sizeof(void *)) { /** @@ -366,7 +373,7 @@ static void amd_sched_process_job(struct fence *f, struct fence_cb *cb) atomic64_dec(&sched->hw_rq_count); spin_unlock_irqrestore(&sched->queue_lock, flags); - sched->ops->process_job(sched, sched_job->job); + sched->ops->process_job(sched, sched_job); kfree(sched_job); wake_up_interruptible(&sched->wait_queue); } @@ -374,7 +381,7 @@ static void amd_sched_process_job(struct fence *f, struct fence_cb *cb) static int amd_sched_main(void *param) { int r; - void *job; + struct amd_sched_job *job; struct sched_param sparam = {.sched_priority = 1}; struct amd_sched_entity *c_entity = NULL; struct amd_gpu_scheduler *sched = (struct amd_gpu_scheduler *)param; @@ -382,7 +389,6 @@ static int amd_sched_main(void *param) sched_setscheduler(current, SCHED_FIFO, &sparam); while (!kthread_should_stop()) { - struct amd_sched_job *sched_job = NULL; struct fence *fence; wait_event_interruptible(sched->wait_queue, @@ -394,26 +400,18 @@ static int amd_sched_main(void *param) r = sched->ops->prepare_job(sched, c_entity, job); if (!r) { unsigned long flags; - sched_job = kzalloc(sizeof(struct amd_sched_job), - GFP_KERNEL); - if (!sched_job) { - WARN(true, "No memory to allocate\n"); - continue; - } - sched_job->job = job; - sched_job->sched = sched; spin_lock_irqsave(&sched->queue_lock, flags); - list_add_tail(&sched_job->list, &sched->active_hw_rq); + list_add_tail(&job->list, &sched->active_hw_rq); atomic64_inc(&sched->hw_rq_count); spin_unlock_irqrestore(&sched->queue_lock, flags); } mutex_lock(&sched->sched_lock); - fence = sched->ops->run_job(sched, c_entity, sched_job); + fence = sched->ops->run_job(sched, c_entity, job); if (fence) { - r = fence_add_callback(fence, &sched_job->cb, + r = fence_add_callback(fence, &job->cb, amd_sched_process_job); if (r == -ENOENT) - amd_sched_process_job(fence, &sched_job->cb); + amd_sched_process_job(fence, &job->cb); else if (r) DRM_ERROR("fence add callback failed (%d)\n", r); fence_put(fence); diff --git a/drivers/gpu/drm/amd/scheduler/gpu_scheduler.h b/drivers/gpu/drm/amd/scheduler/gpu_scheduler.h index e7cc40a6993b..f54615d6a500 100644 --- a/drivers/gpu/drm/amd/scheduler/gpu_scheduler.h +++ b/drivers/gpu/drm/amd/scheduler/gpu_scheduler.h @@ -76,7 +76,8 @@ struct amd_sched_job { struct list_head list; struct fence_cb cb; struct amd_gpu_scheduler *sched; - void *job; + struct amd_sched_entity *s_entity; + void *data; }; /** @@ -86,11 +87,12 @@ struct amd_sched_job { struct amd_sched_backend_ops { int (*prepare_job)(struct amd_gpu_scheduler *sched, struct amd_sched_entity *c_entity, - void *job); + struct amd_sched_job *job); struct fence *(*run_job)(struct amd_gpu_scheduler *sched, struct amd_sched_entity *c_entity, struct amd_sched_job *job); - void (*process_job)(struct amd_gpu_scheduler *sched, void *job); + void (*process_job)(struct amd_gpu_scheduler *sched, + struct amd_sched_job *job); }; /** @@ -120,12 +122,11 @@ struct amd_gpu_scheduler *amd_sched_create(void *device, uint32_t granularity, uint32_t preemption, uint32_t hw_submission); - int amd_sched_destroy(struct amd_gpu_scheduler *sched); int amd_sched_push_job(struct amd_gpu_scheduler *sched, struct amd_sched_entity *c_entity, - void *job); + void *data); int amd_sched_wait_emit(struct amd_sched_entity *c_entity, uint64_t seq, -- cgit From 4af9f07ccdac96e16f7a0ddaf983891a29ebd11a Mon Sep 17 00:00:00 2001 From: Chunming Zhou Date: Mon, 3 Aug 2015 12:57:31 +0800 Subject: drm/amdgpu: use kernel submit helper in vm Signed-off-by: Chunming Zhou Reviewed-by: Christian K?nig --- drivers/gpu/drm/amd/amdgpu/amdgpu.h | 14 --- drivers/gpu/drm/amd/amdgpu/amdgpu_sched.c | 4 +- drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c | 159 ++++++------------------------ 3 files changed, 33 insertions(+), 144 deletions(-) (limited to 'drivers/gpu/drm/amd/amdgpu/amdgpu_sched.c') diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu.h b/drivers/gpu/drm/amd/amdgpu/amdgpu.h index 1e6800050ad8..987e3075a03f 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu.h +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu.h @@ -1235,19 +1235,6 @@ struct amdgpu_cs_chunk { void __user *user_ptr; }; -union amdgpu_sched_job_param { - struct { - struct amdgpu_vm *vm; - uint64_t start; - uint64_t last; - struct fence **fence; - - } vm_mapping; - struct { - struct amdgpu_bo *bo; - } vm; -}; - struct amdgpu_cs_parser { struct amdgpu_device *adev; struct drm_file *filp; @@ -1272,7 +1259,6 @@ struct amdgpu_cs_parser { struct mutex job_lock; struct work_struct job_work; int (*prepare_job)(struct amdgpu_cs_parser *sched_job); - union amdgpu_sched_job_param job_param; int (*run_job)(struct amdgpu_cs_parser *sched_job); int (*free_job)(struct amdgpu_cs_parser *sched_job); }; diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_sched.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_sched.c index d13d01511694..d82f2481bd0e 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_sched.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_sched.c @@ -121,7 +121,7 @@ int amdgpu_sched_ib_submit_kernel_helper(struct amdgpu_device *adev, uint64_t v_seq; struct amdgpu_cs_parser *sched_job = amdgpu_cs_parser_create(adev, owner, &adev->kernel_ctx, - ibs, 1); + ibs, num_ibs); if(!sched_job) { return -ENOMEM; } @@ -139,7 +139,7 @@ int amdgpu_sched_ib_submit_kernel_helper(struct amdgpu_device *adev, if (r) WARN(true, "emit timeout\n"); } else - r = amdgpu_ib_schedule(adev, 1, ibs, owner); + r = amdgpu_ib_schedule(adev, num_ibs, ibs, owner); if (r) return r; *f = &ibs[num_ibs - 1].fence->base; diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c index 230bf1f34ead..b3f5d0484980 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c @@ -316,14 +316,6 @@ static int amdgpu_vm_free_job( return 0; } -static int amdgpu_vm_run_job( - struct amdgpu_cs_parser *sched_job) -{ - amdgpu_bo_fence(sched_job->job_param.vm.bo, - &sched_job->ibs[sched_job->num_ibs -1].fence->base, true); - return 0; -} - /** * amdgpu_vm_clear_bo - initially clear the page dir/table * @@ -334,7 +326,7 @@ static int amdgpu_vm_clear_bo(struct amdgpu_device *adev, struct amdgpu_bo *bo) { struct amdgpu_ring *ring = adev->vm_manager.vm_pte_funcs_ring; - struct amdgpu_cs_parser *sched_job = NULL; + struct fence *fence = NULL; struct amdgpu_ib *ib; unsigned entries; uint64_t addr; @@ -368,38 +360,16 @@ static int amdgpu_vm_clear_bo(struct amdgpu_device *adev, amdgpu_vm_update_pages(adev, ib, addr, 0, entries, 0, 0, 0); amdgpu_vm_pad_ib(adev, ib); WARN_ON(ib->length_dw > 64); - + r = amdgpu_sched_ib_submit_kernel_helper(adev, ring, ib, 1, + &amdgpu_vm_free_job, + AMDGPU_FENCE_OWNER_VM, + &fence); + if (!r) + amdgpu_bo_fence(bo, fence, true); if (amdgpu_enable_scheduler) { - int r; - uint64_t v_seq; - sched_job = amdgpu_cs_parser_create(adev, AMDGPU_FENCE_OWNER_VM, - &adev->kernel_ctx, ib, 1); - if(!sched_job) - goto error_free; - sched_job->job_param.vm.bo = bo; - sched_job->run_job = amdgpu_vm_run_job; - sched_job->free_job = amdgpu_vm_free_job; - v_seq = atomic64_inc_return(&adev->kernel_ctx.rings[ring->idx].entity.last_queued_v_seq); - ib->sequence = v_seq; - amd_sched_push_job(ring->scheduler, - &adev->kernel_ctx.rings[ring->idx].entity, - sched_job); - r = amd_sched_wait_emit(&adev->kernel_ctx.rings[ring->idx].entity, - v_seq, - false, - -1); - if (r) - DRM_ERROR("emit timeout\n"); - amdgpu_bo_unreserve(bo); return 0; - } else { - r = amdgpu_ib_schedule(adev, 1, ib, AMDGPU_FENCE_OWNER_VM); - if (r) - goto error_free; - amdgpu_bo_fence(bo, &ib->fence->base, true); } - error_free: amdgpu_ib_free(adev, ib); kfree(ib); @@ -456,7 +426,7 @@ int amdgpu_vm_update_page_directory(struct amdgpu_device *adev, uint64_t last_pde = ~0, last_pt = ~0; unsigned count = 0, pt_idx, ndw; struct amdgpu_ib *ib; - struct amdgpu_cs_parser *sched_job = NULL; + struct fence *fence = NULL; int r; @@ -518,37 +488,13 @@ int amdgpu_vm_update_page_directory(struct amdgpu_device *adev, amdgpu_vm_pad_ib(adev, ib); amdgpu_sync_resv(adev, &ib->sync, pd->tbo.resv, AMDGPU_FENCE_OWNER_VM); WARN_ON(ib->length_dw > ndw); - - if (amdgpu_enable_scheduler) { - int r; - uint64_t v_seq; - sched_job = amdgpu_cs_parser_create(adev, AMDGPU_FENCE_OWNER_VM, - &adev->kernel_ctx, - ib, 1); - if(!sched_job) - goto error_free; - sched_job->job_param.vm.bo = pd; - sched_job->run_job = amdgpu_vm_run_job; - sched_job->free_job = amdgpu_vm_free_job; - v_seq = atomic64_inc_return(&adev->kernel_ctx.rings[ring->idx].entity.last_queued_v_seq); - ib->sequence = v_seq; - amd_sched_push_job(ring->scheduler, - &adev->kernel_ctx.rings[ring->idx].entity, - sched_job); - r = amd_sched_wait_emit(&adev->kernel_ctx.rings[ring->idx].entity, - v_seq, - false, - -1); - if (r) - DRM_ERROR("emit timeout\n"); - } else { - r = amdgpu_ib_schedule(adev, 1, ib, AMDGPU_FENCE_OWNER_VM); - if (r) { - amdgpu_ib_free(adev, ib); - return r; - } - amdgpu_bo_fence(pd, &ib->fence->base, true); - } + r = amdgpu_sched_ib_submit_kernel_helper(adev, ring, ib, 1, + &amdgpu_vm_free_job, + AMDGPU_FENCE_OWNER_VM, + &fence); + if (r) + goto error_free; + amdgpu_bo_fence(pd, fence, true); } if (!amdgpu_enable_scheduler || ib->length_dw == 0) { @@ -559,11 +505,9 @@ int amdgpu_vm_update_page_directory(struct amdgpu_device *adev, return 0; error_free: - if (sched_job) - kfree(sched_job); amdgpu_ib_free(adev, ib); kfree(ib); - return -ENOMEM; + return r; } /** @@ -748,20 +692,6 @@ static void amdgpu_vm_fence_pts(struct amdgpu_vm *vm, amdgpu_bo_fence(vm->page_tables[i].bo, fence, true); } -static int amdgpu_vm_bo_update_mapping_run_job( - struct amdgpu_cs_parser *sched_job) -{ - struct fence **fence = sched_job->job_param.vm_mapping.fence; - amdgpu_vm_fence_pts(sched_job->job_param.vm_mapping.vm, - sched_job->job_param.vm_mapping.start, - sched_job->job_param.vm_mapping.last + 1, - &sched_job->ibs[sched_job->num_ibs -1].fence->base); - if (fence) { - fence_put(*fence); - *fence = fence_get(&sched_job->ibs[sched_job->num_ibs -1].fence->base); - } - return 0; -} /** * amdgpu_vm_bo_update_mapping - update a mapping in the vm page table * @@ -787,7 +717,7 @@ static int amdgpu_vm_bo_update_mapping(struct amdgpu_device *adev, unsigned nptes, ncmds, ndw; uint32_t flags = gtt_flags; struct amdgpu_ib *ib; - struct amdgpu_cs_parser *sched_job = NULL; + struct fence *f = NULL; int r; /* normally,bo_va->flags only contians READABLE and WIRTEABLE bit go here @@ -869,56 +799,29 @@ static int amdgpu_vm_bo_update_mapping(struct amdgpu_device *adev, amdgpu_vm_pad_ib(adev, ib); WARN_ON(ib->length_dw > ndw); + r = amdgpu_sched_ib_submit_kernel_helper(adev, ring, ib, 1, + &amdgpu_vm_free_job, + AMDGPU_FENCE_OWNER_VM, + &f); + if (r) + goto error_free; - if (amdgpu_enable_scheduler) { - int r; - uint64_t v_seq; - sched_job = amdgpu_cs_parser_create(adev, AMDGPU_FENCE_OWNER_VM, - &adev->kernel_ctx, ib, 1); - if(!sched_job) - goto error_free; - sched_job->job_param.vm_mapping.vm = vm; - sched_job->job_param.vm_mapping.start = mapping->it.start; - sched_job->job_param.vm_mapping.last = mapping->it.last; - sched_job->job_param.vm_mapping.fence = fence; - sched_job->run_job = amdgpu_vm_bo_update_mapping_run_job; - sched_job->free_job = amdgpu_vm_free_job; - v_seq = atomic64_inc_return(&adev->kernel_ctx.rings[ring->idx].entity.last_queued_v_seq); - ib->sequence = v_seq; - amd_sched_push_job(ring->scheduler, - &adev->kernel_ctx.rings[ring->idx].entity, - sched_job); - r = amd_sched_wait_emit(&adev->kernel_ctx.rings[ring->idx].entity, - v_seq, - false, - -1); - if (r) - DRM_ERROR("emit timeout\n"); - } else { - r = amdgpu_ib_schedule(adev, 1, ib, AMDGPU_FENCE_OWNER_VM); - if (r) { - amdgpu_ib_free(adev, ib); - return r; - } - - amdgpu_vm_fence_pts(vm, mapping->it.start, - mapping->it.last + 1, &ib->fence->base); - if (fence) { - fence_put(*fence); - *fence = fence_get(&ib->fence->base); - } - + amdgpu_vm_fence_pts(vm, mapping->it.start, + mapping->it.last + 1, f); + if (fence) { + fence_put(*fence); + *fence = fence_get(f); + } + if (!amdgpu_enable_scheduler) { amdgpu_ib_free(adev, ib); kfree(ib); } return 0; error_free: - if (sched_job) - kfree(sched_job); amdgpu_ib_free(adev, ib); kfree(ib); - return -ENOMEM; + return r; } /** -- cgit From f556cb0caeec1ba9b8e5e2aa85b47e76277f5d4b Mon Sep 17 00:00:00 2001 From: Chunming Zhou Date: Sun, 2 Aug 2015 11:18:04 +0800 Subject: drm/amd: add scheduler fence implementation (v2) scheduler fence is based on kernel fence framework. v2: squash in Christian's build fix Signed-off-by: Chunming Zhou Reviewed-by: Christian K?nig --- drivers/gpu/drm/amd/amdgpu/Makefile | 1 + drivers/gpu/drm/amd/amdgpu/amdgpu.h | 1 + drivers/gpu/drm/amd/amdgpu/amdgpu_cs.c | 21 +++-- drivers/gpu/drm/amd/amdgpu/amdgpu_ctx.c | 10 --- drivers/gpu/drm/amd/amdgpu/amdgpu_ib.c | 2 +- drivers/gpu/drm/amd/amdgpu/amdgpu_sched.c | 34 ++++---- drivers/gpu/drm/amd/scheduler/gpu_scheduler.c | 26 +++++- drivers/gpu/drm/amd/scheduler/gpu_scheduler.h | 33 +++++++- drivers/gpu/drm/amd/scheduler/sched_fence.c | 112 ++++++++++++++++++++++++++ 9 files changed, 202 insertions(+), 38 deletions(-) create mode 100644 drivers/gpu/drm/amd/scheduler/sched_fence.c (limited to 'drivers/gpu/drm/amd/amdgpu/amdgpu_sched.c') diff --git a/drivers/gpu/drm/amd/amdgpu/Makefile b/drivers/gpu/drm/amd/amdgpu/Makefile index f1cb7d2fa411..04c270757030 100644 --- a/drivers/gpu/drm/amd/amdgpu/Makefile +++ b/drivers/gpu/drm/amd/amdgpu/Makefile @@ -86,6 +86,7 @@ amdgpu-y += amdgpu_cgs.o # GPU scheduler amdgpu-y += \ ../scheduler/gpu_scheduler.o \ + ../scheduler/sched_fence.o \ amdgpu_sched.o amdgpu-$(CONFIG_COMPAT) += amdgpu_ioc32.o diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu.h b/drivers/gpu/drm/amd/amdgpu/amdgpu.h index 987e3075a03f..2ba448ee948b 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu.h +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu.h @@ -1261,6 +1261,7 @@ struct amdgpu_cs_parser { int (*prepare_job)(struct amdgpu_cs_parser *sched_job); int (*run_job)(struct amdgpu_cs_parser *sched_job); int (*free_job)(struct amdgpu_cs_parser *sched_job); + struct amd_sched_fence *s_fence; }; static inline u32 amdgpu_get_ib_value(struct amdgpu_cs_parser *p, uint32_t ib_idx, int idx) diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_cs.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_cs.c index b1dc7e1ed271..f428288d8363 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_cs.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_cs.c @@ -899,8 +899,6 @@ int amdgpu_cs_ioctl(struct drm_device *dev, void *data, struct drm_file *filp) if (amdgpu_enable_scheduler && parser->num_ibs) { struct amdgpu_ring * ring = amdgpu_cs_parser_get_ring(adev, parser); - parser->ibs[parser->num_ibs - 1].sequence = atomic64_inc_return( - &parser->ctx->rings[ring->idx].entity.last_queued_v_seq); if (ring->is_pte_ring || (parser->bo_list && parser->bo_list->has_userptr)) { r = amdgpu_cs_parser_prepare_job(parser); if (r) @@ -910,10 +908,21 @@ int amdgpu_cs_ioctl(struct drm_device *dev, void *data, struct drm_file *filp) parser->ring = ring; parser->run_job = amdgpu_cs_parser_run_job; parser->free_job = amdgpu_cs_parser_free_job; - amd_sched_push_job(ring->scheduler, - &parser->ctx->rings[ring->idx].entity, - parser); - cs->out.handle = parser->ibs[parser->num_ibs - 1].sequence; + mutex_lock(&parser->job_lock); + r = amd_sched_push_job(ring->scheduler, + &parser->ctx->rings[ring->idx].entity, + parser, + &parser->s_fence); + if (r) { + mutex_unlock(&parser->job_lock); + goto out; + } + parser->ibs[parser->num_ibs - 1].sequence = + amdgpu_ctx_add_fence(parser->ctx, ring, + &parser->s_fence->base, + parser->s_fence->v_seq); + cs->out.handle = parser->s_fence->v_seq; + mutex_unlock(&parser->job_lock); up_read(&adev->exclusive_lock); return 0; } diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_ctx.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_ctx.c index 232e800eea56..1833f05c7e0b 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_ctx.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_ctx.c @@ -268,16 +268,6 @@ struct fence *amdgpu_ctx_get_fence(struct amdgpu_ctx *ctx, struct amdgpu_ctx_ring *cring = & ctx->rings[ring->idx]; struct fence *fence; uint64_t queued_seq; - int r; - - if (amdgpu_enable_scheduler) { - r = amd_sched_wait_emit(&cring->entity, - seq, - false, - -1); - if (r) - return NULL; - } spin_lock(&ctx->ring_lock); if (amdgpu_enable_scheduler) diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_ib.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_ib.c index eed409c59492..5104e64e9ad8 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_ib.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_ib.c @@ -218,7 +218,7 @@ int amdgpu_ib_schedule(struct amdgpu_device *adev, unsigned num_ibs, sequence = amdgpu_enable_scheduler ? ib->sequence : 0; - if (ib->ctx) + if (!amdgpu_enable_scheduler && ib->ctx) ib->sequence = amdgpu_ctx_add_fence(ib->ctx, ring, &ib->fence->base, sequence); diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_sched.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_sched.c index d82f2481bd0e..6a7e83edcaa7 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_sched.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_sched.c @@ -118,7 +118,6 @@ int amdgpu_sched_ib_submit_kernel_helper(struct amdgpu_device *adev, { int r = 0; if (amdgpu_enable_scheduler) { - uint64_t v_seq; struct amdgpu_cs_parser *sched_job = amdgpu_cs_parser_create(adev, owner, &adev->kernel_ctx, ibs, num_ibs); @@ -126,22 +125,23 @@ int amdgpu_sched_ib_submit_kernel_helper(struct amdgpu_device *adev, return -ENOMEM; } sched_job->free_job = free_job; - v_seq = atomic64_inc_return(&adev->kernel_ctx.rings[ring->idx].entity.last_queued_v_seq); - ibs[num_ibs - 1].sequence = v_seq; - amd_sched_push_job(ring->scheduler, - &adev->kernel_ctx.rings[ring->idx].entity, - sched_job); - r = amd_sched_wait_emit( - &adev->kernel_ctx.rings[ring->idx].entity, - v_seq, - false, - -1); - if (r) - WARN(true, "emit timeout\n"); - } else + mutex_lock(&sched_job->job_lock); + r = amd_sched_push_job(ring->scheduler, + &adev->kernel_ctx.rings[ring->idx].entity, + sched_job, &sched_job->s_fence); + if (r) { + mutex_unlock(&sched_job->job_lock); + kfree(sched_job); + return r; + } + ibs[num_ibs - 1].sequence = sched_job->s_fence->v_seq; + *f = &sched_job->s_fence->base; + mutex_unlock(&sched_job->job_lock); + } else { r = amdgpu_ib_schedule(adev, num_ibs, ibs, owner); - if (r) - return r; - *f = &ibs[num_ibs - 1].fence->base; + if (r) + return r; + *f = &ibs[num_ibs - 1].fence->base; + } return 0; } diff --git a/drivers/gpu/drm/amd/scheduler/gpu_scheduler.c b/drivers/gpu/drm/amd/scheduler/gpu_scheduler.c index 33b4f55e48b1..402086d96889 100644 --- a/drivers/gpu/drm/amd/scheduler/gpu_scheduler.c +++ b/drivers/gpu/drm/amd/scheduler/gpu_scheduler.c @@ -180,6 +180,7 @@ int amd_sched_entity_init(struct amd_gpu_scheduler *sched, uint32_t jobs) { uint64_t seq_ring = 0; + char name[20]; if (!(sched && entity && rq)) return -EINVAL; @@ -191,6 +192,10 @@ int amd_sched_entity_init(struct amd_gpu_scheduler *sched, entity->scheduler = sched; init_waitqueue_head(&entity->wait_queue); init_waitqueue_head(&entity->wait_emit); + entity->fence_context = fence_context_alloc(1); + snprintf(name, sizeof(name), "c_entity[%llu]", entity->fence_context); + memcpy(entity->name, name, 20); + INIT_LIST_HEAD(&entity->fence_list); if(kfifo_alloc(&entity->job_queue, jobs * sizeof(void *), GFP_KERNEL)) @@ -199,6 +204,7 @@ int amd_sched_entity_init(struct amd_gpu_scheduler *sched, spin_lock_init(&entity->queue_lock); atomic64_set(&entity->last_emitted_v_seq, seq_ring); atomic64_set(&entity->last_queued_v_seq, seq_ring); + atomic64_set(&entity->last_signaled_v_seq, seq_ring); /* Add the entity to the run queue */ mutex_lock(&rq->lock); @@ -291,15 +297,25 @@ int amd_sched_entity_fini(struct amd_gpu_scheduler *sched, */ int amd_sched_push_job(struct amd_gpu_scheduler *sched, struct amd_sched_entity *c_entity, - void *data) + void *data, + struct amd_sched_fence **fence) { - struct amd_sched_job *job = kzalloc(sizeof(struct amd_sched_job), - GFP_KERNEL); + struct amd_sched_job *job; + + if (!fence) + return -EINVAL; + job = kzalloc(sizeof(struct amd_sched_job), GFP_KERNEL); if (!job) return -ENOMEM; job->sched = sched; job->s_entity = c_entity; job->data = data; + *fence = amd_sched_fence_create(c_entity); + if ((*fence) == NULL) { + kfree(job); + return -EINVAL; + } + job->s_fence = *fence; while (kfifo_in_spinlocked(&c_entity->job_queue, &job, sizeof(void *), &c_entity->queue_lock) != sizeof(void *)) { /** @@ -368,12 +384,16 @@ static void amd_sched_process_job(struct fence *f, struct fence_cb *cb) unsigned long flags; sched = sched_job->sched; + atomic64_set(&sched_job->s_entity->last_signaled_v_seq, + sched_job->s_fence->v_seq); + amd_sched_fence_signal(sched_job->s_fence); spin_lock_irqsave(&sched->queue_lock, flags); list_del(&sched_job->list); atomic64_dec(&sched->hw_rq_count); spin_unlock_irqrestore(&sched->queue_lock, flags); sched->ops->process_job(sched, sched_job); + fence_put(&sched_job->s_fence->base); kfree(sched_job); wake_up_interruptible(&sched->wait_queue); } diff --git a/drivers/gpu/drm/amd/scheduler/gpu_scheduler.h b/drivers/gpu/drm/amd/scheduler/gpu_scheduler.h index f54615d6a500..300132f14d74 100644 --- a/drivers/gpu/drm/amd/scheduler/gpu_scheduler.h +++ b/drivers/gpu/drm/amd/scheduler/gpu_scheduler.h @@ -45,6 +45,7 @@ struct amd_sched_entity { /* the virtual_seq is unique per context per ring */ atomic64_t last_queued_v_seq; atomic64_t last_emitted_v_seq; + atomic64_t last_signaled_v_seq; /* the job_queue maintains the jobs submitted by clients */ struct kfifo job_queue; spinlock_t queue_lock; @@ -52,6 +53,9 @@ struct amd_sched_entity { wait_queue_head_t wait_queue; wait_queue_head_t wait_emit; bool is_pending; + uint64_t fence_context; + struct list_head fence_list; + char name[20]; }; /** @@ -72,14 +76,35 @@ struct amd_run_queue { int (*check_entity_status)(struct amd_sched_entity *entity); }; +struct amd_sched_fence { + struct fence base; + struct fence_cb cb; + struct list_head list; + struct amd_sched_entity *entity; + uint64_t v_seq; + spinlock_t lock; +}; + struct amd_sched_job { struct list_head list; struct fence_cb cb; struct amd_gpu_scheduler *sched; struct amd_sched_entity *s_entity; void *data; + struct amd_sched_fence *s_fence; }; +extern const struct fence_ops amd_sched_fence_ops; +static inline struct amd_sched_fence *to_amd_sched_fence(struct fence *f) +{ + struct amd_sched_fence *__f = container_of(f, struct amd_sched_fence, base); + + if (__f->base.ops == &amd_sched_fence_ops) + return __f; + + return NULL; +} + /** * Define the backend operations called by the scheduler, * these functions should be implemented in driver side @@ -126,7 +151,8 @@ int amd_sched_destroy(struct amd_gpu_scheduler *sched); int amd_sched_push_job(struct amd_gpu_scheduler *sched, struct amd_sched_entity *c_entity, - void *data); + void *data, + struct amd_sched_fence **fence); int amd_sched_wait_emit(struct amd_sched_entity *c_entity, uint64_t seq, @@ -146,4 +172,9 @@ void amd_sched_emit(struct amd_sched_entity *c_entity, uint64_t seq); uint64_t amd_sched_next_queued_seq(struct amd_sched_entity *c_entity); +struct amd_sched_fence *amd_sched_fence_create( + struct amd_sched_entity *s_entity); +void amd_sched_fence_signal(struct amd_sched_fence *fence); + + #endif diff --git a/drivers/gpu/drm/amd/scheduler/sched_fence.c b/drivers/gpu/drm/amd/scheduler/sched_fence.c new file mode 100644 index 000000000000..d580a357c547 --- /dev/null +++ b/drivers/gpu/drm/amd/scheduler/sched_fence.c @@ -0,0 +1,112 @@ +/* + * Copyright 2015 Advanced Micro Devices, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + * + */ +#include +#include +#include +#include +#include "gpu_scheduler.h" + +static void amd_sched_fence_wait_cb(struct fence *f, struct fence_cb *cb) +{ + struct amd_sched_fence *fence = + container_of(cb, struct amd_sched_fence, cb); + list_del_init(&fence->list); + fence_put(&fence->base); +} + +struct amd_sched_fence *amd_sched_fence_create( + struct amd_sched_entity *s_entity) +{ + struct amd_sched_fence *fence = NULL; + fence = kzalloc(sizeof(struct amd_sched_fence), GFP_KERNEL); + if (fence == NULL) + return NULL; + fence->v_seq = atomic64_inc_return(&s_entity->last_queued_v_seq); + fence->entity = s_entity; + spin_lock_init(&fence->lock); + fence_init(&fence->base, &amd_sched_fence_ops, + &fence->lock, + s_entity->fence_context, + fence->v_seq); + fence_get(&fence->base); + list_add_tail(&fence->list, &s_entity->fence_list); + if (fence_add_callback(&fence->base,&fence->cb, + amd_sched_fence_wait_cb)) { + fence_put(&fence->base); + kfree(fence); + return NULL; + } + return fence; +} + +bool amd_sched_check_ts(struct amd_sched_entity *s_entity, uint64_t v_seq) +{ + return atomic64_read(&s_entity->last_signaled_v_seq) >= v_seq ? true : false; +} + +void amd_sched_fence_signal(struct amd_sched_fence *fence) +{ + if (amd_sched_check_ts(fence->entity, fence->v_seq)) { + int ret = fence_signal_locked(&fence->base); + if (!ret) + FENCE_TRACE(&fence->base, "signaled from irq context\n"); + else + FENCE_TRACE(&fence->base, "was already signaled\n"); + } else + WARN(true, "fence process dismattch with job!\n"); +} + +static const char *amd_sched_fence_get_driver_name(struct fence *fence) +{ + return "amd_sched"; +} + +static const char *amd_sched_fence_get_timeline_name(struct fence *f) +{ + struct amd_sched_fence *fence = to_amd_sched_fence(f); + return (const char *)fence->entity->name; +} + +static bool amd_sched_fence_enable_signaling(struct fence *f) +{ + struct amd_sched_fence *fence = to_amd_sched_fence(f); + + return !amd_sched_check_ts(fence->entity, fence->v_seq); +} + +static bool amd_sched_fence_is_signaled(struct fence *f) +{ + struct amd_sched_fence *fence = to_amd_sched_fence(f); + + return amd_sched_check_ts(fence->entity, fence->v_seq); +} + +const struct fence_ops amd_sched_fence_ops = { + .get_driver_name = amd_sched_fence_get_driver_name, + .get_timeline_name = amd_sched_fence_get_timeline_name, + .enable_signaling = amd_sched_fence_enable_signaling, + .signaled = amd_sched_fence_is_signaled, + .wait = fence_default_wait, + .release = NULL, +}; -- cgit From 05caae8515e12073f4a3beb048e0d289cbe687b7 Mon Sep 17 00:00:00 2001 From: Christian König Date: Mon, 10 Aug 2015 14:04:12 +0200 Subject: drm/amdgpu: remove amd_sched_wait_emit v2 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Not used any more. v2: remove amd_sched_emit as well. Signed-off-by: Christian König Reviewed-by: Chunming Zhou --- drivers/gpu/drm/amd/amdgpu/amdgpu_sched.c | 2 - drivers/gpu/drm/amd/scheduler/gpu_scheduler.c | 61 --------------------------- drivers/gpu/drm/amd/scheduler/gpu_scheduler.h | 10 ----- 3 files changed, 73 deletions(-) (limited to 'drivers/gpu/drm/amd/amdgpu/amdgpu_sched.c') diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_sched.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_sched.c index 6a7e83edcaa7..d2e5f3b90a3c 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_sched.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_sched.c @@ -77,8 +77,6 @@ static struct fence *amdgpu_sched_run_job(struct amd_gpu_scheduler *sched, goto err; } - amd_sched_emit(entity, sched_job->ibs[sched_job->num_ibs - 1].sequence); - mutex_unlock(&sched_job->job_lock); return &fence->base; diff --git a/drivers/gpu/drm/amd/scheduler/gpu_scheduler.c b/drivers/gpu/drm/amd/scheduler/gpu_scheduler.c index 402086d96889..90abefed86cc 100644 --- a/drivers/gpu/drm/amd/scheduler/gpu_scheduler.c +++ b/drivers/gpu/drm/amd/scheduler/gpu_scheduler.c @@ -202,7 +202,6 @@ int amd_sched_entity_init(struct amd_gpu_scheduler *sched, return -EINVAL; spin_lock_init(&entity->queue_lock); - atomic64_set(&entity->last_emitted_v_seq, seq_ring); atomic64_set(&entity->last_queued_v_seq, seq_ring); atomic64_set(&entity->last_signaled_v_seq, seq_ring); @@ -329,53 +328,6 @@ int amd_sched_push_job(struct amd_gpu_scheduler *sched, return 0; } -/** - * Wait for a virtual sequence number to be emitted. - * - * @c_entity The pointer to a valid context entity - * @seq The virtual sequence number to wait - * @intr Interruptible or not - * @timeout Timeout in ms, wait infinitely if <0 - * @emit wait for emit or signal - * - * return =0 signaled , <0 failed -*/ -int amd_sched_wait_emit(struct amd_sched_entity *c_entity, - uint64_t seq, - bool intr, - long timeout) -{ - atomic64_t *v_seq = &c_entity->last_emitted_v_seq; - wait_queue_head_t *wait_queue = &c_entity->wait_emit; - - if (intr && (timeout < 0)) { - wait_event_interruptible( - *wait_queue, - seq <= atomic64_read(v_seq)); - return 0; - } else if (intr && (timeout >= 0)) { - wait_event_interruptible_timeout( - *wait_queue, - seq <= atomic64_read(v_seq), - msecs_to_jiffies(timeout)); - return (seq <= atomic64_read(v_seq)) ? - 0 : -1; - } else if (!intr && (timeout < 0)) { - wait_event( - *wait_queue, - seq <= atomic64_read(v_seq)); - return 0; - } else if (!intr && (timeout >= 0)) { - wait_event_timeout( - *wait_queue, - seq <= atomic64_read(v_seq), - msecs_to_jiffies(timeout)); - return (seq <= atomic64_read(v_seq)) ? - 0 : -1; - } - return 0; -} - static void amd_sched_process_job(struct fence *f, struct fence_cb *cb) { struct amd_sched_job *sched_job = @@ -510,19 +462,6 @@ int amd_sched_destroy(struct amd_gpu_scheduler *sched) return 0; } -/** - * Update emitted sequence and wake up the waiters, called by run_job - * in driver side - * - * @entity The context entity - * @seq The sequence number for the latest emitted job -*/ -void amd_sched_emit(struct amd_sched_entity *c_entity, uint64_t seq) -{ - atomic64_set(&c_entity->last_emitted_v_seq, seq); - wake_up_all(&c_entity->wait_emit); -} - /** * Get next queued sequence number * diff --git a/drivers/gpu/drm/amd/scheduler/gpu_scheduler.h b/drivers/gpu/drm/amd/scheduler/gpu_scheduler.h index 300132f14d74..aa942033d4b3 100644 --- a/drivers/gpu/drm/amd/scheduler/gpu_scheduler.h +++ b/drivers/gpu/drm/amd/scheduler/gpu_scheduler.h @@ -44,7 +44,6 @@ struct amd_sched_entity { spinlock_t lock; /* the virtual_seq is unique per context per ring */ atomic64_t last_queued_v_seq; - atomic64_t last_emitted_v_seq; atomic64_t last_signaled_v_seq; /* the job_queue maintains the jobs submitted by clients */ struct kfifo job_queue; @@ -154,13 +153,6 @@ int amd_sched_push_job(struct amd_gpu_scheduler *sched, void *data, struct amd_sched_fence **fence); -int amd_sched_wait_emit(struct amd_sched_entity *c_entity, - uint64_t seq, - bool intr, - long timeout); - -uint64_t amd_sched_get_handled_seq(struct amd_gpu_scheduler *sched); - int amd_sched_entity_init(struct amd_gpu_scheduler *sched, struct amd_sched_entity *entity, struct amd_run_queue *rq, @@ -168,8 +160,6 @@ int amd_sched_entity_init(struct amd_gpu_scheduler *sched, int amd_sched_entity_fini(struct amd_gpu_scheduler *sched, struct amd_sched_entity *entity); -void amd_sched_emit(struct amd_sched_entity *c_entity, uint64_t seq); - uint64_t amd_sched_next_queued_seq(struct amd_sched_entity *c_entity); struct amd_sched_fence *amd_sched_fence_create( -- cgit From 281b42230175608dec0cd8dab9908250e7aa36a9 Mon Sep 17 00:00:00 2001 From: Chunming Zhou Date: Wed, 12 Aug 2015 12:58:31 +0800 Subject: drm/amdgpu: add reference for **fence fix fence is released when pass to **fence sometimes. add reference for it. Signed-off-by: Chunming Zhou Reviewed-by: Christian K?nig --- drivers/gpu/drm/amd/amdgpu/amdgpu_cs.c | 1 + drivers/gpu/drm/amd/amdgpu/amdgpu_sched.c | 4 ++-- drivers/gpu/drm/amd/amdgpu/amdgpu_uvd.c | 2 +- drivers/gpu/drm/amd/amdgpu/amdgpu_vce.c | 2 ++ drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c | 3 +++ drivers/gpu/drm/amd/amdgpu/cik_sdma.c | 1 + drivers/gpu/drm/amd/amdgpu/gfx_v7_0.c | 1 + drivers/gpu/drm/amd/amdgpu/gfx_v8_0.c | 1 + drivers/gpu/drm/amd/amdgpu/sdma_v2_4.c | 1 + drivers/gpu/drm/amd/amdgpu/sdma_v3_0.c | 1 + drivers/gpu/drm/amd/scheduler/gpu_scheduler.c | 1 + 11 files changed, 15 insertions(+), 3 deletions(-) (limited to 'drivers/gpu/drm/amd/amdgpu/amdgpu_sched.c') diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_cs.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_cs.c index f428288d8363..8796938216d6 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_cs.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_cs.c @@ -136,6 +136,7 @@ static void amdgpu_job_work_func(struct work_struct *work) sched_job->free_job(sched_job); mutex_unlock(&sched_job->job_lock); /* after processing job, free memory */ + fence_put(&sched_job->s_fence->base); kfree(sched_job); } struct amdgpu_cs_parser *amdgpu_cs_parser_create(struct amdgpu_device *adev, diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_sched.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_sched.c index d2e5f3b90a3c..a86e38158afa 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_sched.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_sched.c @@ -133,13 +133,13 @@ int amdgpu_sched_ib_submit_kernel_helper(struct amdgpu_device *adev, return r; } ibs[num_ibs - 1].sequence = sched_job->s_fence->v_seq; - *f = &sched_job->s_fence->base; + *f = fence_get(&sched_job->s_fence->base); mutex_unlock(&sched_job->job_lock); } else { r = amdgpu_ib_schedule(adev, num_ibs, ibs, owner); if (r) return r; - *f = &ibs[num_ibs - 1].fence->base; + *f = fence_get(&ibs[num_ibs - 1].fence->base); } return 0; } diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_uvd.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_uvd.c index e7336a95fe59..68369cf1e318 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_uvd.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_uvd.c @@ -877,7 +877,7 @@ static int amdgpu_uvd_send_msg(struct amdgpu_ring *ring, if (fence) *fence = fence_get(f); amdgpu_bo_unref(&bo); - + fence_put(f); if (amdgpu_enable_scheduler) return 0; diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_vce.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_vce.c index 38660eac67d6..33ee6ae28f37 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_vce.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_vce.c @@ -415,6 +415,7 @@ int amdgpu_vce_get_create_msg(struct amdgpu_ring *ring, uint32_t handle, goto err; if (fence) *fence = fence_get(f); + fence_put(f); if (amdgpu_enable_scheduler) return 0; err: @@ -481,6 +482,7 @@ int amdgpu_vce_get_destroy_msg(struct amdgpu_ring *ring, uint32_t handle, goto err; if (fence) *fence = fence_get(f); + fence_put(f); if (amdgpu_enable_scheduler) return 0; err: diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c index b3f5d0484980..de882b0db350 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c @@ -366,6 +366,7 @@ static int amdgpu_vm_clear_bo(struct amdgpu_device *adev, &fence); if (!r) amdgpu_bo_fence(bo, fence, true); + fence_put(fence); if (amdgpu_enable_scheduler) { amdgpu_bo_unreserve(bo); return 0; @@ -495,6 +496,7 @@ int amdgpu_vm_update_page_directory(struct amdgpu_device *adev, if (r) goto error_free; amdgpu_bo_fence(pd, fence, true); + fence_put(fence); } if (!amdgpu_enable_scheduler || ib->length_dw == 0) { @@ -812,6 +814,7 @@ static int amdgpu_vm_bo_update_mapping(struct amdgpu_device *adev, fence_put(*fence); *fence = fence_get(f); } + fence_put(f); if (!amdgpu_enable_scheduler) { amdgpu_ib_free(adev, ib); kfree(ib); diff --git a/drivers/gpu/drm/amd/amdgpu/cik_sdma.c b/drivers/gpu/drm/amd/amdgpu/cik_sdma.c index c3ed5b22d732..2b4242b39b0a 100644 --- a/drivers/gpu/drm/amd/amdgpu/cik_sdma.c +++ b/drivers/gpu/drm/amd/amdgpu/cik_sdma.c @@ -669,6 +669,7 @@ static int cik_sdma_ring_test_ib(struct amdgpu_ring *ring) } err1: + fence_put(f); amdgpu_ib_free(adev, &ib); err0: amdgpu_wb_free(adev, index); diff --git a/drivers/gpu/drm/amd/amdgpu/gfx_v7_0.c b/drivers/gpu/drm/amd/amdgpu/gfx_v7_0.c index ee1c47f9a2b6..9b0cab413677 100644 --- a/drivers/gpu/drm/amd/amdgpu/gfx_v7_0.c +++ b/drivers/gpu/drm/amd/amdgpu/gfx_v7_0.c @@ -2698,6 +2698,7 @@ static int gfx_v7_0_ring_test_ib(struct amdgpu_ring *ring) } err2: + fence_put(f); amdgpu_ib_free(adev, &ib); err1: amdgpu_gfx_scratch_free(adev, scratch); diff --git a/drivers/gpu/drm/amd/amdgpu/gfx_v8_0.c b/drivers/gpu/drm/amd/amdgpu/gfx_v8_0.c index a865d96b67af..4b68e6306f40 100644 --- a/drivers/gpu/drm/amd/amdgpu/gfx_v8_0.c +++ b/drivers/gpu/drm/amd/amdgpu/gfx_v8_0.c @@ -659,6 +659,7 @@ static int gfx_v8_0_ring_test_ib(struct amdgpu_ring *ring) r = -EINVAL; } err2: + fence_put(f); amdgpu_ib_free(adev, &ib); err1: amdgpu_gfx_scratch_free(adev, scratch); diff --git a/drivers/gpu/drm/amd/amdgpu/sdma_v2_4.c b/drivers/gpu/drm/amd/amdgpu/sdma_v2_4.c index 6de7dc88d53c..9de8104eddeb 100644 --- a/drivers/gpu/drm/amd/amdgpu/sdma_v2_4.c +++ b/drivers/gpu/drm/amd/amdgpu/sdma_v2_4.c @@ -733,6 +733,7 @@ static int sdma_v2_4_ring_test_ib(struct amdgpu_ring *ring) } err1: + fence_put(f); amdgpu_ib_free(adev, &ib); err0: amdgpu_wb_free(adev, index); diff --git a/drivers/gpu/drm/amd/amdgpu/sdma_v3_0.c b/drivers/gpu/drm/amd/amdgpu/sdma_v3_0.c index 963a991fea00..029f3455f9f9 100644 --- a/drivers/gpu/drm/amd/amdgpu/sdma_v3_0.c +++ b/drivers/gpu/drm/amd/amdgpu/sdma_v3_0.c @@ -853,6 +853,7 @@ static int sdma_v3_0_ring_test_ib(struct amdgpu_ring *ring) r = -EINVAL; } err1: + fence_put(f); amdgpu_ib_free(adev, &ib); err0: amdgpu_wb_free(adev, index); diff --git a/drivers/gpu/drm/amd/scheduler/gpu_scheduler.c b/drivers/gpu/drm/amd/scheduler/gpu_scheduler.c index 39577f6c0241..5017c71ba700 100644 --- a/drivers/gpu/drm/amd/scheduler/gpu_scheduler.c +++ b/drivers/gpu/drm/amd/scheduler/gpu_scheduler.c @@ -313,6 +313,7 @@ int amd_sched_push_job(struct amd_gpu_scheduler *sched, kfree(job); return -EINVAL; } + fence_get(&(*fence)->base); job->s_fence = *fence; while (kfifo_in_spinlocked(&c_entity->job_queue, &job, sizeof(void *), &c_entity->queue_lock) != sizeof(void *)) { -- cgit