drm/amdgpu/SRIOV:implement guilty job TDR for(V2)

1,TDR will kickout guilty job if it hang exceed the threshold
of the given one from kernel paramter "job_hang_limit", that
way a bad command stream will not infinitly cause GPU hang.

by default this threshold is 1 so a job will be kicked out
after it hang.

2,if a job timeout TDR routine will not reset all sched/ring,
instead if will only reset on the givn one which is indicated
by @job of amdgpu_sriov_gpu_reset, that way we don't need to
reset and recover each sched/ring if we already know which job
cause GPU hang.

3,unblock sriov_gpu_reset for AI family.

V2:
1:put kickout guilty job after sched parked.
2:since parking scheduler prior to kickout already occupies a
while, we can do last check on the in question job before
doing hw_reset.

TODO:
1:when a job is considered as guilty, we should mark some flag
in its fence status flag, and let UMD side aware that this
fence signaling is not due to job complete but job hang.

2:if gpu reset cause all video memory lost, we need introduce
a new policy to implement TDR, like drop all jobs not yet
signaled, and all IOCTL on this device will return ERROR
DEVICE_LOST.
this will be implemented later.

Signed-off-by: Monk Liu <Monk.Liu@amd.com>
Reviewed-by: Christian König <christian.koenig@amd.com>
Signed-off-by: Alex Deucher <alexander.deucher@amd.com>

diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu.h b/drivers/gpu/drm/amd/amdgpu/amdgpu.h
index e2cafbd690c0cc8a830845688b16f89cdf500b2b..a2dd218e35b9e07dcbe3ce3c20c94907a642b878 100644 (file)
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu.h
@@ -109,6 +109,7 @@ extern int amdgpu_prim_buf_per_se;
 extern int amdgpu_pos_buf_per_se;
 extern int amdgpu_cntl_sb_buf_per_se;
 extern int amdgpu_param_buf_per_se;
+extern int amdgpu_job_hang_limit;
 
 #define AMDGPU_DEFAULT_GTT_SIZE_MB             3072ULL /* 3GB by default */
 #define AMDGPU_WAIT_IDLE_TIMEOUT_IN_MS         3000

diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c
index 41c18700e2759d3cae2764229374d3a40350e12a..8b0f4864a88514800b64d8b65e0863657661af95 100644 (file)
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c
@@ -2617,7 +2617,7 @@ err:
  */
 int amdgpu_sriov_gpu_reset(struct amdgpu_device *adev, struct amdgpu_job *job)
 {
-       int i, r = 0;
+       int i, j, r = 0;
        int resched;
        struct amdgpu_bo *bo, *tmp;
        struct amdgpu_ring *ring;
@@ -2630,19 +2630,36 @@ int amdgpu_sriov_gpu_reset(struct amdgpu_device *adev, struct amdgpu_job *job)
        /* block TTM */
        resched = ttm_bo_lock_delayed_workqueue(&adev->mman.bdev);
 
-       /* block scheduler */
-       for (i = 0; i < AMDGPU_MAX_RINGS; ++i) {
-               ring = adev->rings[i];
+       /* we start from the ring trigger GPU hang */
+       j = job ? job->ring->idx : 0;
 
+       /* block scheduler */
+       for (i = j; i < j + AMDGPU_MAX_RINGS; ++i) {
+               ring = adev->rings[i % AMDGPU_MAX_RINGS];
                if (!ring || !ring->sched.thread)
                        continue;
 
                kthread_park(ring->sched.thread);
+
+               if (job && j != i)
+                       continue;
+
+               /* here give the last chance to check if fence signaled
+                * since we already pay some time on kthread_park */
+               if (job && dma_fence_is_signaled(&job->base.s_fence->finished)) {
+                       kthread_unpark(ring->sched.thread);
+                       goto give_up_reset;
+               }
+
+               if (amd_sched_invalidate_job(&job->base, amdgpu_job_hang_limit))
+                       amd_sched_job_kickout(&job->base);
+
+               /* only do job_reset on the hang ring if @job not NULL */
                amd_sched_hw_job_reset(&ring->sched);
-       }
 
-       /* after all hw jobs are reset, hw fence is meaningless, so force_completion */
-       amdgpu_fence_driver_force_completion(adev);
+               /* after all hw jobs are reset, hw fence is meaningless, so force_completion */
+               amdgpu_fence_driver_force_completion_ring(ring);
+       }
 
        /* request to take full control of GPU before re-initialization  */
        if (job)
@@ -2695,20 +2712,28 @@ int amdgpu_sriov_gpu_reset(struct amdgpu_device *adev, struct amdgpu_job *job)
        }
        dma_fence_put(fence);
 
-       for (i = 0; i < AMDGPU_MAX_RINGS; ++i) {
-               struct amdgpu_ring *ring = adev->rings[i];
+       for (i = j; i < j + AMDGPU_MAX_RINGS; ++i) {
+               ring = adev->rings[i % AMDGPU_MAX_RINGS];
                if (!ring || !ring->sched.thread)
                        continue;
 
+               if (job && j != i) {
+                       kthread_unpark(ring->sched.thread);
+                       continue;
+               }
+
                amd_sched_job_recovery(&ring->sched);
                kthread_unpark(ring->sched.thread);
        }
 
        drm_helper_resume_force_mode(adev->ddev);
+give_up_reset:
        ttm_bo_unlock_delayed_workqueue(&adev->mman.bdev, resched);
        if (r) {
                /* bad news, how to tell it to userspace ? */
                dev_info(adev->dev, "GPU reset failed\n");
+       } else {
+               dev_info(adev->dev, "GPU reset successed!\n");
        }
 
        adev->gfx.in_reset = false;

diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c
index 2b746e078b9a89e8ae9dc8069bcd01134024c165..7b07ac2c52b6aa24aeeb95afdf4145eb8ea5feea 100644 (file)
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c
@@ -111,6 +111,7 @@ int amdgpu_prim_buf_per_se = 0;
 int amdgpu_pos_buf_per_se = 0;
 int amdgpu_cntl_sb_buf_per_se = 0;
 int amdgpu_param_buf_per_se = 0;
+int amdgpu_job_hang_limit = 0;
 
 MODULE_PARM_DESC(vramlimit, "Restrict VRAM for testing, in megabytes");
 module_param_named(vramlimit, amdgpu_vram_limit, int, 0600);
@@ -233,6 +234,9 @@ module_param_named(cntl_sb_buf_per_se, amdgpu_cntl_sb_buf_per_se, int, 0444);
 MODULE_PARM_DESC(param_buf_per_se, "the size of Off-Chip Pramater Cache per Shader Engine (default depending on gfx)");
 module_param_named(param_buf_per_se, amdgpu_param_buf_per_se, int, 0444);
 
+MODULE_PARM_DESC(job_hang_limit, "how much time allow a job hang and not drop it (default 0)");
+module_param_named(job_hang_limit, amdgpu_job_hang_limit, int ,0444);
+
 
 static const struct pci_device_id pciidlist[] = {
 #ifdef  CONFIG_DRM_AMDGPU_SI

diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_fence.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_fence.c
index ef6c643115b82446eee2e6ab8b263933ee0a6d2d..333bad74906784f7ecc7a794c3d1d4e1dd915978 100644 (file)
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_fence.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_fence.c
@@ -541,6 +541,12 @@ void amdgpu_fence_driver_force_completion(struct amdgpu_device *adev)
        }
 }
 
+void amdgpu_fence_driver_force_completion_ring(struct amdgpu_ring *ring)
+{
+       if (ring)
+               amdgpu_fence_write(ring, ring->fence_drv.sync_seq);
+}
+
 /*
  * Common fence implementation
  */

diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_ring.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_ring.h
index 2b7b3c56d446ca25c9a6a92f1c4c260ae4f56f6e..fc7329b468f09e2ce24d286cb53ee3570d880adc 100644 (file)
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_ring.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_ring.h
@@ -76,6 +76,7 @@ struct amdgpu_fence_driver {
 int amdgpu_fence_driver_init(struct amdgpu_device *adev);
 void amdgpu_fence_driver_fini(struct amdgpu_device *adev);
 void amdgpu_fence_driver_force_completion(struct amdgpu_device *adev);
+void amdgpu_fence_driver_force_completion_ring(struct amdgpu_ring *ring);
 
 int amdgpu_fence_driver_init_ring(struct amdgpu_ring *ring,
                                  unsigned num_hw_submission);

diff --git a/drivers/gpu/drm/amd/scheduler/gpu_scheduler.c b/drivers/gpu/drm/amd/scheduler/gpu_scheduler.c
index fea96a765cf14698eaa2b9e6f30ea7a707245ea7..38cea6fb25a8b9221d64b43da04c4268a2c986b8 100644 (file)
--- a/drivers/gpu/drm/amd/scheduler/gpu_scheduler.c
+++ b/drivers/gpu/drm/amd/scheduler/gpu_scheduler.c
@@ -409,9 +409,18 @@ void amd_sched_hw_job_reset(struct amd_gpu_scheduler *sched)
                                              &s_job->s_fence->cb)) {
                        dma_fence_put(s_job->s_fence->parent);
                        s_job->s_fence->parent = NULL;
+                       atomic_dec(&sched->hw_rq_count);
                }
        }
-       atomic_set(&sched->hw_rq_count, 0);
+       spin_unlock(&sched->job_list_lock);
+}
+
+void amd_sched_job_kickout(struct amd_sched_job *s_job)
+{
+       struct amd_gpu_scheduler *sched = s_job->sched;
+
+       spin_lock(&sched->job_list_lock);
+       list_del_init(&s_job->node);
        spin_unlock(&sched->job_list_lock);
 }
 

diff --git a/drivers/gpu/drm/amd/scheduler/gpu_scheduler.h b/drivers/gpu/drm/amd/scheduler/gpu_scheduler.h
index 924d4a5899e140cb61766f2e4fde9223d36c9850..f9d8f28efd1619e9e8d8a5042982ca9d90818582 100644 (file)
--- a/drivers/gpu/drm/amd/scheduler/gpu_scheduler.h
+++ b/drivers/gpu/drm/amd/scheduler/gpu_scheduler.h
@@ -81,6 +81,7 @@ struct amd_sched_job {
        struct list_head                node;
        struct delayed_work             work_tdr;
        uint64_t                        id;
+       atomic_t karma;
 };
 
 extern const struct dma_fence_ops amd_sched_fence_ops_scheduled;
@@ -96,6 +97,11 @@ static inline struct amd_sched_fence *to_amd_sched_fence(struct dma_fence *f)
        return NULL;
 }
 
+static inline bool amd_sched_invalidate_job(struct amd_sched_job *s_job, int threshold)
+{
+       return (s_job && atomic_inc_return(&s_job->karma) > threshold);
+}
+
 /**
  * Define the backend operations called by the scheduler,
  * these functions should be implemented in driver side
@@ -160,4 +166,5 @@ void amd_sched_hw_job_reset(struct amd_gpu_scheduler *sched);
 void amd_sched_job_recovery(struct amd_gpu_scheduler *sched);
 bool amd_sched_dependency_optimized(struct dma_fence* fence,
                                    struct amd_sched_entity *entity);
+void amd_sched_job_kickout(struct amd_sched_job *s_job);
 #endif