drm/scheduler: move entity handling into separate file
authorChristian König <christian.koenig@amd.com>
Mon, 6 Aug 2018 12:25:32 +0000 (14:25 +0200)
committerAlex Deucher <alexander.deucher@amd.com>
Mon, 27 Aug 2018 16:10:43 +0000 (11:10 -0500)
This is complex enough on it's own. Move it into a separate C file.

Signed-off-by: Christian König <christian.koenig@amd.com>
Reviewed-by: Huang Rui <ray.huang@amd.com>
Signed-off-by: Alex Deucher <alexander.deucher@amd.com>
drivers/gpu/drm/scheduler/Makefile
drivers/gpu/drm/scheduler/gpu_scheduler.c
drivers/gpu/drm/scheduler/sched_entity.c [new file with mode: 0644]
include/drm/gpu_scheduler.h

index 7665883f81d4b7d6756510cf22688665817963e1..f23785d4b3c8ecb4979d8c649cb2cdf0c6aa5f5b 100644 (file)
@@ -20,6 +20,6 @@
 # OTHER DEALINGS IN THE SOFTWARE.
 #
 #
-gpu-sched-y := gpu_scheduler.o sched_fence.o
+gpu-sched-y := gpu_scheduler.o sched_fence.o sched_entity.o
 
 obj-$(CONFIG_DRM_SCHED) += gpu-sched.o
index 85c1f95752cc47e6d46e3d8deb3c3c256c7f1357..9ca741f3a0bc861f1aef17cedd5fb7954339b724 100644 (file)
@@ -58,8 +58,6 @@
 #define to_drm_sched_job(sched_job)            \
                container_of((sched_job), struct drm_sched_job, queue_node)
 
-static bool drm_sched_entity_is_ready(struct drm_sched_entity *entity);
-static void drm_sched_wakeup(struct drm_gpu_scheduler *sched);
 static void drm_sched_process_job(struct dma_fence *f, struct dma_fence_cb *cb);
 
 /**
@@ -86,8 +84,8 @@ static void drm_sched_rq_init(struct drm_gpu_scheduler *sched,
  *
  * Adds a scheduler entity to the run queue.
  */
-static void drm_sched_rq_add_entity(struct drm_sched_rq *rq,
-                                   struct drm_sched_entity *entity)
+void drm_sched_rq_add_entity(struct drm_sched_rq *rq,
+                            struct drm_sched_entity *entity)
 {
        if (!list_empty(&entity->list))
                return;
@@ -104,8 +102,8 @@ static void drm_sched_rq_add_entity(struct drm_sched_rq *rq,
  *
  * Removes a scheduler entity from the run queue.
  */
-static void drm_sched_rq_remove_entity(struct drm_sched_rq *rq,
-                                      struct drm_sched_entity *entity)
+void drm_sched_rq_remove_entity(struct drm_sched_rq *rq,
+                               struct drm_sched_entity *entity)
 {
        if (list_empty(&entity->list))
                return;
@@ -158,301 +156,6 @@ drm_sched_rq_select_entity(struct drm_sched_rq *rq)
        return NULL;
 }
 
-/**
- * drm_sched_entity_init - Init a context entity used by scheduler when
- * submit to HW ring.
- *
- * @entity: scheduler entity to init
- * @rq_list: the list of run queue on which jobs from this
- *           entity can be submitted
- * @num_rq_list: number of run queue in rq_list
- * @guilty: atomic_t set to 1 when a job on this queue
- *          is found to be guilty causing a timeout
- *
- * Note: the rq_list should have atleast one element to schedule
- *       the entity
- *
- * Returns 0 on success or a negative error code on failure.
-*/
-int drm_sched_entity_init(struct drm_sched_entity *entity,
-                         struct drm_sched_rq **rq_list,
-                         unsigned int num_rq_list,
-                         atomic_t *guilty)
-{
-       int i;
-
-       if (!(entity && rq_list && num_rq_list > 0 && rq_list[0]))
-               return -EINVAL;
-
-       memset(entity, 0, sizeof(struct drm_sched_entity));
-       INIT_LIST_HEAD(&entity->list);
-       entity->rq = rq_list[0];
-       entity->guilty = guilty;
-       entity->num_rq_list = num_rq_list;
-       entity->rq_list = kcalloc(num_rq_list, sizeof(struct drm_sched_rq *),
-                               GFP_KERNEL);
-       if (!entity->rq_list)
-               return -ENOMEM;
-
-       for (i = 0; i < num_rq_list; ++i)
-               entity->rq_list[i] = rq_list[i];
-       entity->last_scheduled = NULL;
-
-       spin_lock_init(&entity->rq_lock);
-       spsc_queue_init(&entity->job_queue);
-
-       atomic_set(&entity->fence_seq, 0);
-       entity->fence_context = dma_fence_context_alloc(2);
-
-       return 0;
-}
-EXPORT_SYMBOL(drm_sched_entity_init);
-
-/**
- * drm_sched_entity_is_idle - Check if entity is idle
- *
- * @entity: scheduler entity
- *
- * Returns true if the entity does not have any unscheduled jobs.
- */
-static bool drm_sched_entity_is_idle(struct drm_sched_entity *entity)
-{
-       rmb();
-
-       if (list_empty(&entity->list) ||
-           spsc_queue_peek(&entity->job_queue) == NULL)
-               return true;
-
-       return false;
-}
-
-/**
- * drm_sched_entity_is_ready - Check if entity is ready
- *
- * @entity: scheduler entity
- *
- * Return true if entity could provide a job.
- */
-static bool drm_sched_entity_is_ready(struct drm_sched_entity *entity)
-{
-       if (spsc_queue_peek(&entity->job_queue) == NULL)
-               return false;
-
-       if (READ_ONCE(entity->dependency))
-               return false;
-
-       return true;
-}
-
-/**
- * drm_sched_entity_get_free_sched - Get the rq from rq_list with least load
- *
- * @entity: scheduler entity
- *
- * Return the pointer to the rq with least load.
- */
-static struct drm_sched_rq *
-drm_sched_entity_get_free_sched(struct drm_sched_entity *entity)
-{
-       struct drm_sched_rq *rq = NULL;
-       unsigned int min_jobs = UINT_MAX, num_jobs;
-       int i;
-
-       for (i = 0; i < entity->num_rq_list; ++i) {
-               num_jobs = atomic_read(&entity->rq_list[i]->sched->num_jobs);
-               if (num_jobs < min_jobs) {
-                       min_jobs = num_jobs;
-                       rq = entity->rq_list[i];
-               }
-       }
-
-       return rq;
-}
-
-static void drm_sched_entity_kill_jobs_cb(struct dma_fence *f,
-                                   struct dma_fence_cb *cb)
-{
-       struct drm_sched_job *job = container_of(cb, struct drm_sched_job,
-                                                finish_cb);
-       drm_sched_fence_finished(job->s_fence);
-       WARN_ON(job->s_fence->parent);
-       dma_fence_put(&job->s_fence->finished);
-       job->sched->ops->free_job(job);
-}
-
-
-/**
- * drm_sched_entity_flush - Flush a context entity
- *
- * @entity: scheduler entity
- * @timeout: time to wait in for Q to become empty in jiffies.
- *
- * Splitting drm_sched_entity_fini() into two functions, The first one does the waiting,
- * removes the entity from the runqueue and returns an error when the process was killed.
- *
- * Returns the remaining time in jiffies left from the input timeout
- */
-long drm_sched_entity_flush(struct drm_sched_entity *entity, long timeout)
-{
-       struct drm_gpu_scheduler *sched;
-       struct task_struct *last_user;
-       long ret = timeout;
-
-       sched = entity->rq->sched;
-       /**
-        * The client will not queue more IBs during this fini, consume existing
-        * queued IBs or discard them on SIGKILL
-       */
-       if (current->flags & PF_EXITING) {
-               if (timeout)
-                       ret = wait_event_timeout(
-                                       sched->job_scheduled,
-                                       drm_sched_entity_is_idle(entity),
-                                       timeout);
-       } else
-               wait_event_killable(sched->job_scheduled, drm_sched_entity_is_idle(entity));
-
-
-       /* For killed process disable any more IBs enqueue right now */
-       last_user = cmpxchg(&entity->last_user, current->group_leader, NULL);
-       if ((!last_user || last_user == current->group_leader) &&
-           (current->flags & PF_EXITING) && (current->exit_code == SIGKILL))
-               drm_sched_rq_remove_entity(entity->rq, entity);
-
-       return ret;
-}
-EXPORT_SYMBOL(drm_sched_entity_flush);
-
-/**
- * drm_sched_entity_cleanup - Destroy a context entity
- *
- * @entity: scheduler entity
- *
- * This should be called after @drm_sched_entity_do_release. It goes over the
- * entity and signals all jobs with an error code if the process was killed.
- *
- */
-void drm_sched_entity_fini(struct drm_sched_entity *entity)
-{
-       struct drm_gpu_scheduler *sched;
-
-       sched = entity->rq->sched;
-       drm_sched_rq_remove_entity(entity->rq, entity);
-
-       /* Consumption of existing IBs wasn't completed. Forcefully
-        * remove them here.
-        */
-       if (spsc_queue_peek(&entity->job_queue)) {
-               struct drm_sched_job *job;
-               int r;
-
-               /* Park the kernel for a moment to make sure it isn't processing
-                * our enity.
-                */
-               kthread_park(sched->thread);
-               kthread_unpark(sched->thread);
-               if (entity->dependency) {
-                       dma_fence_remove_callback(entity->dependency,
-                                                 &entity->cb);
-                       dma_fence_put(entity->dependency);
-                       entity->dependency = NULL;
-               }
-
-               while ((job = to_drm_sched_job(spsc_queue_pop(&entity->job_queue)))) {
-                       struct drm_sched_fence *s_fence = job->s_fence;
-                       drm_sched_fence_scheduled(s_fence);
-                       dma_fence_set_error(&s_fence->finished, -ESRCH);
-
-                       /*
-                        * When pipe is hanged by older entity, new entity might
-                        * not even have chance to submit it's first job to HW
-                        * and so entity->last_scheduled will remain NULL
-                        */
-                       if (!entity->last_scheduled) {
-                               drm_sched_entity_kill_jobs_cb(NULL, &job->finish_cb);
-                       } else {
-                               r = dma_fence_add_callback(entity->last_scheduled, &job->finish_cb,
-                                                               drm_sched_entity_kill_jobs_cb);
-                               if (r == -ENOENT)
-                                       drm_sched_entity_kill_jobs_cb(NULL, &job->finish_cb);
-                               else if (r)
-                                       DRM_ERROR("fence add callback failed (%d)\n", r);
-                       }
-               }
-       }
-
-       dma_fence_put(entity->last_scheduled);
-       entity->last_scheduled = NULL;
-       kfree(entity->rq_list);
-}
-EXPORT_SYMBOL(drm_sched_entity_fini);
-
-/**
- * drm_sched_entity_fini - Destroy a context entity
- *
- * @entity: scheduler entity
- *
- * Calls drm_sched_entity_do_release() and drm_sched_entity_cleanup()
- */
-void drm_sched_entity_destroy(struct drm_sched_entity *entity)
-{
-       drm_sched_entity_flush(entity, MAX_WAIT_SCHED_ENTITY_Q_EMPTY);
-       drm_sched_entity_fini(entity);
-}
-EXPORT_SYMBOL(drm_sched_entity_destroy);
-
-static void drm_sched_entity_wakeup(struct dma_fence *f, struct dma_fence_cb *cb)
-{
-       struct drm_sched_entity *entity =
-               container_of(cb, struct drm_sched_entity, cb);
-       entity->dependency = NULL;
-       dma_fence_put(f);
-       drm_sched_wakeup(entity->rq->sched);
-}
-
-static void drm_sched_entity_clear_dep(struct dma_fence *f, struct dma_fence_cb *cb)
-{
-       struct drm_sched_entity *entity =
-               container_of(cb, struct drm_sched_entity, cb);
-       entity->dependency = NULL;
-       dma_fence_put(f);
-}
-
-/**
- * drm_sched_entity_set_rq_priority - helper for drm_sched_entity_set_priority
- */
-static void drm_sched_entity_set_rq_priority(struct drm_sched_rq **rq,
-                                            enum drm_sched_priority priority)
-{
-       *rq = &(*rq)->sched->sched_rq[priority];
-}
-
-/**
- * drm_sched_entity_set_priority - Sets priority of the entity
- *
- * @entity: scheduler entity
- * @priority: scheduler priority
- *
- * Update the priority of runqueus used for the entity.
- */
-void drm_sched_entity_set_priority(struct drm_sched_entity *entity,
-                                  enum drm_sched_priority priority)
-{
-       unsigned int i;
-
-       spin_lock(&entity->rq_lock);
-
-       for (i = 0; i < entity->num_rq_list; ++i)
-               drm_sched_entity_set_rq_priority(&entity->rq_list[i], priority);
-
-       drm_sched_rq_remove_entity(entity->rq, entity);
-       drm_sched_entity_set_rq_priority(&entity->rq, priority);
-       drm_sched_rq_add_entity(entity->rq, entity);
-
-       spin_unlock(&entity->rq_lock);
-}
-EXPORT_SYMBOL(drm_sched_entity_set_priority);
-
 /**
  * drm_sched_dependency_optimized
  *
@@ -479,140 +182,6 @@ bool drm_sched_dependency_optimized(struct dma_fence* fence,
 }
 EXPORT_SYMBOL(drm_sched_dependency_optimized);
 
-static bool drm_sched_entity_add_dependency_cb(struct drm_sched_entity *entity)
-{
-       struct drm_gpu_scheduler *sched = entity->rq->sched;
-       struct dma_fence * fence = entity->dependency;
-       struct drm_sched_fence *s_fence;
-
-       if (fence->context == entity->fence_context ||
-            fence->context == entity->fence_context + 1) {
-                /*
-                 * Fence is a scheduled/finished fence from a job
-                 * which belongs to the same entity, we can ignore
-                 * fences from ourself
-                 */
-               dma_fence_put(entity->dependency);
-               return false;
-       }
-
-       s_fence = to_drm_sched_fence(fence);
-       if (s_fence && s_fence->sched == sched) {
-
-               /*
-                * Fence is from the same scheduler, only need to wait for
-                * it to be scheduled
-                */
-               fence = dma_fence_get(&s_fence->scheduled);
-               dma_fence_put(entity->dependency);
-               entity->dependency = fence;
-               if (!dma_fence_add_callback(fence, &entity->cb,
-                                           drm_sched_entity_clear_dep))
-                       return true;
-
-               /* Ignore it when it is already scheduled */
-               dma_fence_put(fence);
-               return false;
-       }
-
-       if (!dma_fence_add_callback(entity->dependency, &entity->cb,
-                                   drm_sched_entity_wakeup))
-               return true;
-
-       dma_fence_put(entity->dependency);
-       return false;
-}
-
-static struct drm_sched_job *
-drm_sched_entity_pop_job(struct drm_sched_entity *entity)
-{
-       struct drm_gpu_scheduler *sched = entity->rq->sched;
-       struct drm_sched_job *sched_job = to_drm_sched_job(
-                                               spsc_queue_peek(&entity->job_queue));
-
-       if (!sched_job)
-               return NULL;
-
-       while ((entity->dependency = sched->ops->dependency(sched_job, entity))) {
-               if (drm_sched_entity_add_dependency_cb(entity)) {
-
-                       trace_drm_sched_job_wait_dep(sched_job, entity->dependency);
-                       return NULL;
-               }
-       }
-
-       /* skip jobs from entity that marked guilty */
-       if (entity->guilty && atomic_read(entity->guilty))
-               dma_fence_set_error(&sched_job->s_fence->finished, -ECANCELED);
-
-       dma_fence_put(entity->last_scheduled);
-       entity->last_scheduled = dma_fence_get(&sched_job->s_fence->finished);
-
-       spsc_queue_pop(&entity->job_queue);
-       return sched_job;
-}
-
-/**
- * drm_sched_entity_select_rq - select a new rq for the entity
- *
- * @entity: scheduler entity
- *
- * Check all prerequisites and select a new rq for the entity for load
- * balancing.
- */
-static void drm_sched_entity_select_rq(struct drm_sched_entity *entity)
-{
-       struct dma_fence *fence;
-       struct drm_sched_rq *rq;
-
-       if (!spsc_queue_count(&entity->job_queue) == 0 ||
-           entity->num_rq_list <= 1)
-               return;
-
-       fence = READ_ONCE(entity->last_scheduled);
-       if (fence && !dma_fence_is_signaled(fence))
-               return;
-
-       rq = drm_sched_entity_get_free_sched(entity);
-       spin_lock(&entity->rq_lock);
-       drm_sched_rq_remove_entity(entity->rq, entity);
-       entity->rq = rq;
-       spin_unlock(&entity->rq_lock);
-}
-
-/**
- * drm_sched_entity_push_job - Submit a job to the entity's job queue
- *
- * @sched_job: job to submit
- * @entity: scheduler entity
- *
- * Note: To guarantee that the order of insertion to queue matches
- * the job's fence sequence number this function should be
- * called with drm_sched_job_init under common lock.
- *
- * Returns 0 for success, negative error code otherwise.
- */
-void drm_sched_entity_push_job(struct drm_sched_job *sched_job,
-                              struct drm_sched_entity *entity)
-{
-       bool first;
-
-       trace_drm_sched_job(sched_job, entity);
-       atomic_inc(&entity->rq->sched->num_jobs);
-       WRITE_ONCE(entity->last_user, current->group_leader);
-       first = spsc_queue_push(&entity->job_queue, &sched_job->queue_node);
-
-       /* first job wakes up scheduler */
-       if (first) {
-               /* Add the entity to the run queue */
-               spin_lock(&entity->rq_lock);
-               drm_sched_rq_add_entity(entity->rq, entity);
-               spin_unlock(&entity->rq_lock);
-               drm_sched_wakeup(entity->rq->sched);
-       }
-}
-EXPORT_SYMBOL(drm_sched_entity_push_job);
-
 /* job_finish is called after hw fence signaled
  */
 static void drm_sched_job_finish(struct work_struct *work)
@@ -840,7 +409,7 @@ static bool drm_sched_ready(struct drm_gpu_scheduler *sched)
  * @sched: scheduler instance
  *
  */
-static void drm_sched_wakeup(struct drm_gpu_scheduler *sched)
+void drm_sched_wakeup(struct drm_gpu_scheduler *sched)
 {
        if (drm_sched_ready(sched))
                wake_up_interruptible(&sched->wake_up_worker);
diff --git a/drivers/gpu/drm/scheduler/sched_entity.c b/drivers/gpu/drm/scheduler/sched_entity.c
new file mode 100644 (file)
index 0000000..1053f27
--- /dev/null
@@ -0,0 +1,459 @@
+/*
+ * Copyright 2015 Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ */
+
+#include <linux/kthread.h>
+#include <drm/gpu_scheduler.h>
+
+#include "gpu_scheduler_trace.h"
+
+#define to_drm_sched_job(sched_job)            \
+               container_of((sched_job), struct drm_sched_job, queue_node)
+
+/**
+ * drm_sched_entity_init - Init a context entity used by scheduler when
+ * submit to HW ring.
+ *
+ * @entity: scheduler entity to init
+ * @rq_list: the list of run queue on which jobs from this
+ *           entity can be submitted
+ * @num_rq_list: number of run queue in rq_list
+ * @guilty: atomic_t set to 1 when a job on this queue
+ *          is found to be guilty causing a timeout
+ *
+ * Note: the rq_list should have atleast one element to schedule
+ *       the entity
+ *
+ * Returns 0 on success or a negative error code on failure.
+*/
+int drm_sched_entity_init(struct drm_sched_entity *entity,
+                         struct drm_sched_rq **rq_list,
+                         unsigned int num_rq_list,
+                         atomic_t *guilty)
+{
+       int i;
+
+       if (!(entity && rq_list && num_rq_list > 0 && rq_list[0]))
+               return -EINVAL;
+
+       memset(entity, 0, sizeof(struct drm_sched_entity));
+       INIT_LIST_HEAD(&entity->list);
+       entity->rq = rq_list[0];
+       entity->guilty = guilty;
+       entity->num_rq_list = num_rq_list;
+       entity->rq_list = kcalloc(num_rq_list, sizeof(struct drm_sched_rq *),
+                               GFP_KERNEL);
+       if (!entity->rq_list)
+               return -ENOMEM;
+
+       for (i = 0; i < num_rq_list; ++i)
+               entity->rq_list[i] = rq_list[i];
+       entity->last_scheduled = NULL;
+
+       spin_lock_init(&entity->rq_lock);
+       spsc_queue_init(&entity->job_queue);
+
+       atomic_set(&entity->fence_seq, 0);
+       entity->fence_context = dma_fence_context_alloc(2);
+
+       return 0;
+}
+EXPORT_SYMBOL(drm_sched_entity_init);
+
+/**
+ * drm_sched_entity_is_idle - Check if entity is idle
+ *
+ * @entity: scheduler entity
+ *
+ * Returns true if the entity does not have any unscheduled jobs.
+ */
+static bool drm_sched_entity_is_idle(struct drm_sched_entity *entity)
+{
+       rmb();
+
+       if (list_empty(&entity->list) ||
+           spsc_queue_peek(&entity->job_queue) == NULL)
+               return true;
+
+       return false;
+}
+
+/**
+ * drm_sched_entity_is_ready - Check if entity is ready
+ *
+ * @entity: scheduler entity
+ *
+ * Return true if entity could provide a job.
+ */
+bool drm_sched_entity_is_ready(struct drm_sched_entity *entity)
+{
+       if (spsc_queue_peek(&entity->job_queue) == NULL)
+               return false;
+
+       if (READ_ONCE(entity->dependency))
+               return false;
+
+       return true;
+}
+
+/**
+ * drm_sched_entity_get_free_sched - Get the rq from rq_list with least load
+ *
+ * @entity: scheduler entity
+ *
+ * Return the pointer to the rq with least load.
+ */
+static struct drm_sched_rq *
+drm_sched_entity_get_free_sched(struct drm_sched_entity *entity)
+{
+       struct drm_sched_rq *rq = NULL;
+       unsigned int min_jobs = UINT_MAX, num_jobs;
+       int i;
+
+       for (i = 0; i < entity->num_rq_list; ++i) {
+               num_jobs = atomic_read(&entity->rq_list[i]->sched->num_jobs);
+               if (num_jobs < min_jobs) {
+                       min_jobs = num_jobs;
+                       rq = entity->rq_list[i];
+               }
+       }
+
+       return rq;
+}
+
+static void drm_sched_entity_kill_jobs_cb(struct dma_fence *f,
+                                   struct dma_fence_cb *cb)
+{
+       struct drm_sched_job *job = container_of(cb, struct drm_sched_job,
+                                                finish_cb);
+       drm_sched_fence_finished(job->s_fence);
+       WARN_ON(job->s_fence->parent);
+       dma_fence_put(&job->s_fence->finished);
+       job->sched->ops->free_job(job);
+}
+
+
+/**
+ * drm_sched_entity_flush - Flush a context entity
+ *
+ * @entity: scheduler entity
+ * @timeout: time to wait in for Q to become empty in jiffies.
+ *
+ * Splitting drm_sched_entity_fini() into two functions, The first one does the waiting,
+ * removes the entity from the runqueue and returns an error when the process was killed.
+ *
+ * Returns the remaining time in jiffies left from the input timeout
+ */
+long drm_sched_entity_flush(struct drm_sched_entity *entity, long timeout)
+{
+       struct drm_gpu_scheduler *sched;
+       struct task_struct *last_user;
+       long ret = timeout;
+
+       sched = entity->rq->sched;
+       /**
+        * The client will not queue more IBs during this fini, consume existing
+        * queued IBs or discard them on SIGKILL
+       */
+       if (current->flags & PF_EXITING) {
+               if (timeout)
+                       ret = wait_event_timeout(
+                                       sched->job_scheduled,
+                                       drm_sched_entity_is_idle(entity),
+                                       timeout);
+       } else {
+               wait_event_killable(sched->job_scheduled,
+                                   drm_sched_entity_is_idle(entity));
+       }
+
+       /* For killed process disable any more IBs enqueue right now */
+       last_user = cmpxchg(&entity->last_user, current->group_leader, NULL);
+       if ((!last_user || last_user == current->group_leader) &&
+           (current->flags & PF_EXITING) && (current->exit_code == SIGKILL))
+               drm_sched_rq_remove_entity(entity->rq, entity);
+
+       return ret;
+}
+EXPORT_SYMBOL(drm_sched_entity_flush);
+
+/**
+ * drm_sched_entity_cleanup - Destroy a context entity
+ *
+ * @entity: scheduler entity
+ *
+ * This should be called after @drm_sched_entity_do_release. It goes over the
+ * entity and signals all jobs with an error code if the process was killed.
+ *
+ */
+void drm_sched_entity_fini(struct drm_sched_entity *entity)
+{
+       struct drm_gpu_scheduler *sched;
+
+       sched = entity->rq->sched;
+       drm_sched_rq_remove_entity(entity->rq, entity);
+
+       /* Consumption of existing IBs wasn't completed. Forcefully
+        * remove them here.
+        */
+       if (spsc_queue_peek(&entity->job_queue)) {
+               struct drm_sched_job *job;
+               int r;
+
+               /* Park the kernel for a moment to make sure it isn't processing
+                * our enity.
+                */
+               kthread_park(sched->thread);
+               kthread_unpark(sched->thread);
+               if (entity->dependency) {
+                       dma_fence_remove_callback(entity->dependency,
+                                                 &entity->cb);
+                       dma_fence_put(entity->dependency);
+                       entity->dependency = NULL;
+               }
+
+               while ((job = to_drm_sched_job(spsc_queue_pop(&entity->job_queue)))) {
+                       struct drm_sched_fence *s_fence = job->s_fence;
+                       drm_sched_fence_scheduled(s_fence);
+                       dma_fence_set_error(&s_fence->finished, -ESRCH);
+
+                       /*
+                        * When pipe is hanged by older entity, new entity might
+                        * not even have chance to submit it's first job to HW
+                        * and so entity->last_scheduled will remain NULL
+                        */
+                       if (!entity->last_scheduled) {
+                               drm_sched_entity_kill_jobs_cb(NULL, &job->finish_cb);
+                       } else {
+                               r = dma_fence_add_callback(entity->last_scheduled, &job->finish_cb,
+                                                               drm_sched_entity_kill_jobs_cb);
+                               if (r == -ENOENT)
+                                       drm_sched_entity_kill_jobs_cb(NULL, &job->finish_cb);
+                               else if (r)
+                                       DRM_ERROR("fence add callback failed (%d)\n", r);
+                       }
+               }
+       }
+
+       dma_fence_put(entity->last_scheduled);
+       entity->last_scheduled = NULL;
+       kfree(entity->rq_list);
+}
+EXPORT_SYMBOL(drm_sched_entity_fini);
+
+/**
+ * drm_sched_entity_fini - Destroy a context entity
+ *
+ * @entity: scheduler entity
+ *
+ * Calls drm_sched_entity_do_release() and drm_sched_entity_cleanup()
+ */
+void drm_sched_entity_destroy(struct drm_sched_entity *entity)
+{
+       drm_sched_entity_flush(entity, MAX_WAIT_SCHED_ENTITY_Q_EMPTY);
+       drm_sched_entity_fini(entity);
+}
+EXPORT_SYMBOL(drm_sched_entity_destroy);
+
+static void drm_sched_entity_wakeup(struct dma_fence *f, struct dma_fence_cb *cb)
+{
+       struct drm_sched_entity *entity =
+               container_of(cb, struct drm_sched_entity, cb);
+       entity->dependency = NULL;
+       dma_fence_put(f);
+       drm_sched_wakeup(entity->rq->sched);
+}
+
+static void drm_sched_entity_clear_dep(struct dma_fence *f, struct dma_fence_cb *cb)
+{
+       struct drm_sched_entity *entity =
+               container_of(cb, struct drm_sched_entity, cb);
+       entity->dependency = NULL;
+       dma_fence_put(f);
+}
+
+/**
+ * drm_sched_entity_set_rq_priority - helper for drm_sched_entity_set_priority
+ */
+static void drm_sched_entity_set_rq_priority(struct drm_sched_rq **rq,
+                                            enum drm_sched_priority priority)
+{
+       *rq = &(*rq)->sched->sched_rq[priority];
+}
+
+/**
+ * drm_sched_entity_set_priority - Sets priority of the entity
+ *
+ * @entity: scheduler entity
+ * @priority: scheduler priority
+ *
+ * Update the priority of runqueus used for the entity.
+ */
+void drm_sched_entity_set_priority(struct drm_sched_entity *entity,
+                                  enum drm_sched_priority priority)
+{
+       unsigned int i;
+
+       spin_lock(&entity->rq_lock);
+
+       for (i = 0; i < entity->num_rq_list; ++i)
+               drm_sched_entity_set_rq_priority(&entity->rq_list[i], priority);
+
+       drm_sched_rq_remove_entity(entity->rq, entity);
+       drm_sched_entity_set_rq_priority(&entity->rq, priority);
+       drm_sched_rq_add_entity(entity->rq, entity);
+
+       spin_unlock(&entity->rq_lock);
+}
+EXPORT_SYMBOL(drm_sched_entity_set_priority);
+
+static bool drm_sched_entity_add_dependency_cb(struct drm_sched_entity *entity)
+{
+       struct drm_gpu_scheduler *sched = entity->rq->sched;
+       struct dma_fence * fence = entity->dependency;
+       struct drm_sched_fence *s_fence;
+
+       if (fence->context == entity->fence_context ||
+            fence->context == entity->fence_context + 1) {
+                /*
+                 * Fence is a scheduled/finished fence from a job
+                 * which belongs to the same entity, we can ignore
+                 * fences from ourself
+                 */
+               dma_fence_put(entity->dependency);
+               return false;
+       }
+
+       s_fence = to_drm_sched_fence(fence);
+       if (s_fence && s_fence->sched == sched) {
+
+               /*
+                * Fence is from the same scheduler, only need to wait for
+                * it to be scheduled
+                */
+               fence = dma_fence_get(&s_fence->scheduled);
+               dma_fence_put(entity->dependency);
+               entity->dependency = fence;
+               if (!dma_fence_add_callback(fence, &entity->cb,
+                                           drm_sched_entity_clear_dep))
+                       return true;
+
+               /* Ignore it when it is already scheduled */
+               dma_fence_put(fence);
+               return false;
+       }
+
+       if (!dma_fence_add_callback(entity->dependency, &entity->cb,
+                                   drm_sched_entity_wakeup))
+               return true;
+
+       dma_fence_put(entity->dependency);
+       return false;
+}
+
+struct drm_sched_job *drm_sched_entity_pop_job(struct drm_sched_entity *entity)
+{
+       struct drm_gpu_scheduler *sched = entity->rq->sched;
+       struct drm_sched_job *sched_job = to_drm_sched_job(
+                                               spsc_queue_peek(&entity->job_queue));
+
+       if (!sched_job)
+               return NULL;
+
+       while ((entity->dependency = sched->ops->dependency(sched_job, entity))) {
+               if (drm_sched_entity_add_dependency_cb(entity)) {
+
+                       trace_drm_sched_job_wait_dep(sched_job, entity->dependency);
+                       return NULL;
+               }
+       }
+
+       /* skip jobs from entity that marked guilty */
+       if (entity->guilty && atomic_read(entity->guilty))
+               dma_fence_set_error(&sched_job->s_fence->finished, -ECANCELED);
+
+       dma_fence_put(entity->last_scheduled);
+       entity->last_scheduled = dma_fence_get(&sched_job->s_fence->finished);
+
+       spsc_queue_pop(&entity->job_queue);
+       return sched_job;
+}
+
+/**
+ * drm_sched_entity_select_rq - select a new rq for the entity
+ *
+ * @entity: scheduler entity
+ *
+ * Check all prerequisites and select a new rq for the entity for load
+ * balancing.
+ */
+void drm_sched_entity_select_rq(struct drm_sched_entity *entity)
+{
+       struct dma_fence *fence;
+       struct drm_sched_rq *rq;
+
+       if (!spsc_queue_count(&entity->job_queue) == 0 ||
+           entity->num_rq_list <= 1)
+               return;
+
+       fence = READ_ONCE(entity->last_scheduled);
+       if (fence && !dma_fence_is_signaled(fence))
+               return;
+
+       rq = drm_sched_entity_get_free_sched(entity);
+       spin_lock(&entity->rq_lock);
+       drm_sched_rq_remove_entity(entity->rq, entity);
+       entity->rq = rq;
+       spin_unlock(&entity->rq_lock);
+}
+
+/**
+ * drm_sched_entity_push_job - Submit a job to the entity's job queue
+ *
+ * @sched_job: job to submit
+ * @entity: scheduler entity
+ *
+ * Note: To guarantee that the order of insertion to queue matches
+ * the job's fence sequence number this function should be
+ * called with drm_sched_job_init under common lock.
+ *
+ * Returns 0 for success, negative error code otherwise.
+ */
+void drm_sched_entity_push_job(struct drm_sched_job *sched_job,
+                              struct drm_sched_entity *entity)
+{
+       bool first;
+
+       trace_drm_sched_job(sched_job, entity);
+       atomic_inc(&entity->rq->sched->num_jobs);
+       WRITE_ONCE(entity->last_user, current->group_leader);
+       first = spsc_queue_push(&entity->job_queue, &sched_job->queue_node);
+
+       /* first job wakes up scheduler */
+       if (first) {
+               /* Add the entity to the run queue */
+               spin_lock(&entity->rq_lock);
+               drm_sched_rq_add_entity(entity->rq, entity);
+               spin_unlock(&entity->rq_lock);
+               drm_sched_wakeup(entity->rq->sched);
+       }
+}
+EXPORT_SYMBOL(drm_sched_entity_push_job);
index 22c0f88f7d8f750535a11b3f9a90ef9af301f512..919ae572f7758eabb0f345e315aa367720fc67ff 100644 (file)
@@ -288,6 +288,21 @@ int drm_sched_init(struct drm_gpu_scheduler *sched,
                   uint32_t hw_submission, unsigned hang_limit, long timeout,
                   const char *name);
 void drm_sched_fini(struct drm_gpu_scheduler *sched);
+int drm_sched_job_init(struct drm_sched_job *job,
+                      struct drm_sched_entity *entity,
+                      void *owner);
+void drm_sched_wakeup(struct drm_gpu_scheduler *sched);
+void drm_sched_hw_job_reset(struct drm_gpu_scheduler *sched,
+                           struct drm_sched_job *job);
+void drm_sched_job_recovery(struct drm_gpu_scheduler *sched);
+bool drm_sched_dependency_optimized(struct dma_fence* fence,
+                                   struct drm_sched_entity *entity);
+void drm_sched_job_kickout(struct drm_sched_job *s_job);
+
+void drm_sched_rq_add_entity(struct drm_sched_rq *rq,
+                            struct drm_sched_entity *entity);
+void drm_sched_rq_remove_entity(struct drm_sched_rq *rq,
+                               struct drm_sched_entity *entity);
 
 int drm_sched_entity_init(struct drm_sched_entity *entity,
                          struct drm_sched_rq **rq_list,
@@ -296,22 +311,17 @@ int drm_sched_entity_init(struct drm_sched_entity *entity,
 long drm_sched_entity_flush(struct drm_sched_entity *entity, long timeout);
 void drm_sched_entity_fini(struct drm_sched_entity *entity);
 void drm_sched_entity_destroy(struct drm_sched_entity *entity);
+void drm_sched_entity_select_rq(struct drm_sched_entity *entity);
+struct drm_sched_job *drm_sched_entity_pop_job(struct drm_sched_entity *entity);
 void drm_sched_entity_push_job(struct drm_sched_job *sched_job,
                               struct drm_sched_entity *entity);
 void drm_sched_entity_set_priority(struct drm_sched_entity *entity,
                                   enum drm_sched_priority priority);
+bool drm_sched_entity_is_ready(struct drm_sched_entity *entity);
+
 struct drm_sched_fence *drm_sched_fence_create(
        struct drm_sched_entity *s_entity, void *owner);
 void drm_sched_fence_scheduled(struct drm_sched_fence *fence);
 void drm_sched_fence_finished(struct drm_sched_fence *fence);
-int drm_sched_job_init(struct drm_sched_job *job,
-                      struct drm_sched_entity *entity,
-                      void *owner);
-void drm_sched_hw_job_reset(struct drm_gpu_scheduler *sched,
-                           struct drm_sched_job *job);
-void drm_sched_job_recovery(struct drm_gpu_scheduler *sched);
-bool drm_sched_dependency_optimized(struct dma_fence* fence,
-                                   struct drm_sched_entity *entity);
-void drm_sched_job_kickout(struct drm_sched_job *s_job);
 
 #endif