diff options
Diffstat (limited to 'drivers/gpu/drm/amd/scheduler')
| -rw-r--r-- | drivers/gpu/drm/amd/scheduler/gpu_scheduler.c | 121 | ||||
| -rw-r--r-- | drivers/gpu/drm/amd/scheduler/gpu_scheduler.h | 42 | ||||
| -rw-r--r-- | drivers/gpu/drm/amd/scheduler/sched_fence.c | 10 |
3 files changed, 127 insertions, 46 deletions
diff --git a/drivers/gpu/drm/amd/scheduler/gpu_scheduler.c b/drivers/gpu/drm/amd/scheduler/gpu_scheduler.c index a5ff9458d359..c16248cee779 100644 --- a/drivers/gpu/drm/amd/scheduler/gpu_scheduler.c +++ b/drivers/gpu/drm/amd/scheduler/gpu_scheduler.c @@ -319,6 +319,48 @@ static bool amd_sched_entity_in(struct amd_sched_job *sched_job) return added; } +static void amd_sched_free_job(struct fence *f, struct fence_cb *cb) { + struct amd_sched_job *job = container_of(cb, struct amd_sched_job, cb_free_job); + schedule_work(&job->work_free_job); +} + +/* job_finish is called after hw fence signaled, and + * the job had already been deleted from ring_mirror_list + */ +void amd_sched_job_finish(struct amd_sched_job *s_job) +{ + struct amd_sched_job *next; + struct amd_gpu_scheduler *sched = s_job->sched; + + if (sched->timeout != MAX_SCHEDULE_TIMEOUT) { + if (cancel_delayed_work(&s_job->work_tdr)) + amd_sched_job_put(s_job); + + /* queue TDR for next job */ + next = list_first_entry_or_null(&sched->ring_mirror_list, + struct amd_sched_job, node); + + if (next) { + INIT_DELAYED_WORK(&next->work_tdr, s_job->timeout_callback); + amd_sched_job_get(next); + schedule_delayed_work(&next->work_tdr, sched->timeout); + } + } +} + +void amd_sched_job_begin(struct amd_sched_job *s_job) +{ + struct amd_gpu_scheduler *sched = s_job->sched; + + if (sched->timeout != MAX_SCHEDULE_TIMEOUT && + list_first_entry_or_null(&sched->ring_mirror_list, struct amd_sched_job, node) == s_job) + { + INIT_DELAYED_WORK(&s_job->work_tdr, s_job->timeout_callback); + amd_sched_job_get(s_job); + schedule_delayed_work(&s_job->work_tdr, sched->timeout); + } +} + /** * Submit a job to the job queue * @@ -330,11 +372,39 @@ void amd_sched_entity_push_job(struct amd_sched_job *sched_job) { struct amd_sched_entity *entity = sched_job->s_entity; + sched_job->use_sched = 1; + fence_add_callback(&sched_job->s_fence->base, + &sched_job->cb_free_job, amd_sched_free_job); trace_amd_sched_job(sched_job); wait_event(entity->sched->job_scheduled, amd_sched_entity_in(sched_job)); } +/* init a sched_job with basic field */ +int amd_sched_job_init(struct amd_sched_job *job, + struct amd_gpu_scheduler *sched, + struct amd_sched_entity *entity, + void (*timeout_cb)(struct work_struct *work), + void (*free_cb)(struct kref *refcount), + void *owner, struct fence **fence) +{ + INIT_LIST_HEAD(&job->node); + kref_init(&job->refcount); + job->sched = sched; + job->s_entity = entity; + job->s_fence = amd_sched_fence_create(entity, owner); + if (!job->s_fence) + return -ENOMEM; + + job->s_fence->s_job = job; + job->timeout_callback = timeout_cb; + job->free_callback = free_cb; + + if (fence) + *fence = &job->s_fence->base; + return 0; +} + /** * Return ture if we can push more jobs to the hw. */ @@ -383,47 +453,26 @@ static void amd_sched_process_job(struct fence *f, struct fence_cb *cb) unsigned long flags; atomic_dec(&sched->hw_rq_count); + + /* remove job from ring_mirror_list */ + spin_lock_irqsave(&sched->job_list_lock, flags); + list_del_init(&s_fence->s_job->node); + sched->ops->finish_job(s_fence->s_job); + spin_unlock_irqrestore(&sched->job_list_lock, flags); + amd_sched_fence_signal(s_fence); - if (sched->timeout != MAX_SCHEDULE_TIMEOUT) { - cancel_delayed_work(&s_fence->dwork); - spin_lock_irqsave(&sched->fence_list_lock, flags); - list_del_init(&s_fence->list); - spin_unlock_irqrestore(&sched->fence_list_lock, flags); - } + trace_amd_sched_process_job(s_fence); fence_put(&s_fence->base); wake_up_interruptible(&sched->wake_up_worker); } -static void amd_sched_fence_work_func(struct work_struct *work) -{ - struct amd_sched_fence *s_fence = - container_of(work, struct amd_sched_fence, dwork.work); - struct amd_gpu_scheduler *sched = s_fence->sched; - struct amd_sched_fence *entity, *tmp; - unsigned long flags; - - DRM_ERROR("[%s] scheduler is timeout!\n", sched->name); - - /* Clean all pending fences */ - spin_lock_irqsave(&sched->fence_list_lock, flags); - list_for_each_entry_safe(entity, tmp, &sched->fence_list, list) { - DRM_ERROR(" fence no %d\n", entity->base.seqno); - cancel_delayed_work(&entity->dwork); - list_del_init(&entity->list); - fence_put(&entity->base); - } - spin_unlock_irqrestore(&sched->fence_list_lock, flags); -} - static int amd_sched_main(void *param) { struct sched_param sparam = {.sched_priority = 1}; struct amd_gpu_scheduler *sched = (struct amd_gpu_scheduler *)param; int r, count; - spin_lock_init(&sched->fence_list_lock); - INIT_LIST_HEAD(&sched->fence_list); sched_setscheduler(current, SCHED_FIFO, &sparam); while (!kthread_should_stop()) { @@ -431,7 +480,6 @@ static int amd_sched_main(void *param) struct amd_sched_fence *s_fence; struct amd_sched_job *sched_job; struct fence *fence; - unsigned long flags; wait_event_interruptible(sched->wake_up_worker, (entity = amd_sched_select_entity(sched)) || @@ -446,15 +494,8 @@ static int amd_sched_main(void *param) s_fence = sched_job->s_fence; - if (sched->timeout != MAX_SCHEDULE_TIMEOUT) { - INIT_DELAYED_WORK(&s_fence->dwork, amd_sched_fence_work_func); - schedule_delayed_work(&s_fence->dwork, sched->timeout); - spin_lock_irqsave(&sched->fence_list_lock, flags); - list_add_tail(&s_fence->list, &sched->fence_list); - spin_unlock_irqrestore(&sched->fence_list_lock, flags); - } - atomic_inc(&sched->hw_rq_count); + amd_sched_job_pre_schedule(sched, sched_job); fence = sched->ops->run_job(sched_job); amd_sched_fence_scheduled(s_fence); if (fence) { @@ -489,7 +530,7 @@ static int amd_sched_main(void *param) * Return 0 on success, otherwise error code. */ int amd_sched_init(struct amd_gpu_scheduler *sched, - struct amd_sched_backend_ops *ops, + const struct amd_sched_backend_ops *ops, unsigned hw_submission, long timeout, const char *name) { int i; @@ -502,6 +543,8 @@ int amd_sched_init(struct amd_gpu_scheduler *sched, init_waitqueue_head(&sched->wake_up_worker); init_waitqueue_head(&sched->job_scheduled); + INIT_LIST_HEAD(&sched->ring_mirror_list); + spin_lock_init(&sched->job_list_lock); atomic_set(&sched->hw_rq_count, 0); if (atomic_inc_return(&sched_fence_slab_ref) == 1) { sched_fence_slab = kmem_cache_create( diff --git a/drivers/gpu/drm/amd/scheduler/gpu_scheduler.h b/drivers/gpu/drm/amd/scheduler/gpu_scheduler.h index 9403145d7bee..070095a9433c 100644 --- a/drivers/gpu/drm/amd/scheduler/gpu_scheduler.h +++ b/drivers/gpu/drm/amd/scheduler/gpu_scheduler.h @@ -37,7 +37,7 @@ extern atomic_t sched_fence_slab_ref; /** * A scheduler entity is a wrapper around a job queue or a group - * of other entities. Entities take turns emitting jobs from their + * of other entities. Entities take turns emitting jobs from their * job queues to corresponding hardware ring based on scheduling * policy. */ @@ -74,14 +74,21 @@ struct amd_sched_fence { struct amd_gpu_scheduler *sched; spinlock_t lock; void *owner; - struct delayed_work dwork; - struct list_head list; + struct amd_sched_job *s_job; }; struct amd_sched_job { + struct kref refcount; struct amd_gpu_scheduler *sched; struct amd_sched_entity *s_entity; struct amd_sched_fence *s_fence; + bool use_sched; /* true if the job goes to scheduler */ + struct fence_cb cb_free_job; + struct work_struct work_free_job; + struct list_head node; + struct delayed_work work_tdr; + void (*timeout_callback) (struct work_struct *work); + void (*free_callback)(struct kref *refcount); }; extern const struct fence_ops amd_sched_fence_ops; @@ -102,6 +109,8 @@ static inline struct amd_sched_fence *to_amd_sched_fence(struct fence *f) struct amd_sched_backend_ops { struct fence *(*dependency)(struct amd_sched_job *sched_job); struct fence *(*run_job)(struct amd_sched_job *sched_job); + void (*begin_job)(struct amd_sched_job *sched_job); + void (*finish_job)(struct amd_sched_job *sched_job); }; enum amd_sched_priority { @@ -114,7 +123,7 @@ enum amd_sched_priority { * One scheduler is implemented for each hardware ring */ struct amd_gpu_scheduler { - struct amd_sched_backend_ops *ops; + const struct amd_sched_backend_ops *ops; uint32_t hw_submission_limit; long timeout; const char *name; @@ -122,13 +131,13 @@ struct amd_gpu_scheduler { wait_queue_head_t wake_up_worker; wait_queue_head_t job_scheduled; atomic_t hw_rq_count; - struct list_head fence_list; - spinlock_t fence_list_lock; struct task_struct *thread; + struct list_head ring_mirror_list; + spinlock_t job_list_lock; }; int amd_sched_init(struct amd_gpu_scheduler *sched, - struct amd_sched_backend_ops *ops, + const struct amd_sched_backend_ops *ops, uint32_t hw_submission, long timeout, const char *name); void amd_sched_fini(struct amd_gpu_scheduler *sched); @@ -144,5 +153,24 @@ struct amd_sched_fence *amd_sched_fence_create( struct amd_sched_entity *s_entity, void *owner); void amd_sched_fence_scheduled(struct amd_sched_fence *fence); void amd_sched_fence_signal(struct amd_sched_fence *fence); +int amd_sched_job_init(struct amd_sched_job *job, + struct amd_gpu_scheduler *sched, + struct amd_sched_entity *entity, + void (*timeout_cb)(struct work_struct *work), + void (*free_cb)(struct kref* refcount), + void *owner, struct fence **fence); +void amd_sched_job_pre_schedule(struct amd_gpu_scheduler *sched , + struct amd_sched_job *s_job); +void amd_sched_job_finish(struct amd_sched_job *s_job); +void amd_sched_job_begin(struct amd_sched_job *s_job); +static inline void amd_sched_job_get(struct amd_sched_job *job) { + if (job) + kref_get(&job->refcount); +} + +static inline void amd_sched_job_put(struct amd_sched_job *job) { + if (job) + kref_put(&job->refcount, job->free_callback); +} #endif diff --git a/drivers/gpu/drm/amd/scheduler/sched_fence.c b/drivers/gpu/drm/amd/scheduler/sched_fence.c index dc115aea352b..2a732c490375 100644 --- a/drivers/gpu/drm/amd/scheduler/sched_fence.c +++ b/drivers/gpu/drm/amd/scheduler/sched_fence.c @@ -57,6 +57,16 @@ void amd_sched_fence_signal(struct amd_sched_fence *fence) FENCE_TRACE(&fence->base, "was already signaled\n"); } +void amd_sched_job_pre_schedule(struct amd_gpu_scheduler *sched , + struct amd_sched_job *s_job) +{ + unsigned long flags; + spin_lock_irqsave(&sched->job_list_lock, flags); + list_add_tail(&s_job->node, &sched->ring_mirror_list); + sched->ops->begin_job(s_job); + spin_unlock_irqrestore(&sched->job_list_lock, flags); +} + void amd_sched_fence_scheduled(struct amd_sched_fence *s_fence) { struct fence_cb *cur, *tmp; |