gem/i915_gem_dmabuf.o \
gem/i915_gem_domain.o \
gem/i915_gem_execbuffer.o \
+ gem/i915_gem_fence.o \
gem/i915_gem_internal.o \
gem/i915_gem_object.o \
gem/i915_gem_mman.o \
{
struct clflush *clflush;
+ assert_object_held(obj);
+
/*
* Stolen memory is always coherent with the GPU as it is explicitly
* marked as wc by the system, or the system is cache-coherent.
true, I915_FENCE_TIMEOUT,
I915_FENCE_GFP);
- reservation_object_lock(obj->resv, NULL);
reservation_object_add_excl_fence(obj->resv, &clflush->dma);
- reservation_object_unlock(obj->resv);
i915_sw_fence_commit(&clflush->wait);
} else if (obj->mm.pages) {
static int i915_gem_begin_cpu_access(struct dma_buf *dma_buf, enum dma_data_direction direction)
{
struct drm_i915_gem_object *obj = dma_buf_to_obj(dma_buf);
- struct drm_device *dev = obj->base.dev;
bool write = (direction == DMA_BIDIRECTIONAL || direction == DMA_TO_DEVICE);
int err;
if (err)
return err;
- err = i915_mutex_lock_interruptible(dev);
+ err = i915_gem_object_lock_interruptible(obj);
if (err)
goto out;
err = i915_gem_object_set_to_cpu_domain(obj, write);
- mutex_unlock(&dev->struct_mutex);
+ i915_gem_object_unlock(obj);
out:
i915_gem_object_unpin_pages(obj);
static int i915_gem_end_cpu_access(struct dma_buf *dma_buf, enum dma_data_direction direction)
{
struct drm_i915_gem_object *obj = dma_buf_to_obj(dma_buf);
- struct drm_device *dev = obj->base.dev;
int err;
err = i915_gem_object_pin_pages(obj);
if (err)
return err;
- err = i915_mutex_lock_interruptible(dev);
+ err = i915_gem_object_lock_interruptible(obj);
if (err)
goto out;
err = i915_gem_object_set_to_gtt_domain(obj, false);
- mutex_unlock(&dev->struct_mutex);
+ i915_gem_object_unlock(obj);
out:
i915_gem_object_unpin_pages(obj);
if (!READ_ONCE(obj->pin_global))
return;
- mutex_lock(&obj->base.dev->struct_mutex);
+ i915_gem_object_lock(obj);
__i915_gem_object_flush_for_display(obj);
- mutex_unlock(&obj->base.dev->struct_mutex);
+ i915_gem_object_unlock(obj);
}
/**
{
int ret;
- lockdep_assert_held(&obj->base.dev->struct_mutex);
+ assert_object_held(obj);
ret = i915_gem_object_wait(obj,
I915_WAIT_INTERRUPTIBLE |
- I915_WAIT_LOCKED |
(write ? I915_WAIT_ALL : 0),
MAX_SCHEDULE_TIMEOUT);
if (ret)
{
int ret;
- lockdep_assert_held(&obj->base.dev->struct_mutex);
+ assert_object_held(obj);
ret = i915_gem_object_wait(obj,
I915_WAIT_INTERRUPTIBLE |
- I915_WAIT_LOCKED |
(write ? I915_WAIT_ALL : 0),
MAX_SCHEDULE_TIMEOUT);
if (ret)
struct i915_vma *vma;
int ret;
- lockdep_assert_held(&obj->base.dev->struct_mutex);
+ assert_object_held(obj);
if (obj->cache_level == cache_level)
return 0;
*/
ret = i915_gem_object_wait(obj,
I915_WAIT_INTERRUPTIBLE |
- I915_WAIT_LOCKED |
I915_WAIT_ALL,
MAX_SCHEDULE_TIMEOUT);
if (ret)
if (ret)
goto out;
- ret = i915_mutex_lock_interruptible(dev);
+ ret = mutex_lock_interruptible(&i915->drm.struct_mutex);
if (ret)
goto out;
- ret = i915_gem_object_set_cache_level(obj, level);
- mutex_unlock(&dev->struct_mutex);
+ ret = i915_gem_object_lock_interruptible(obj);
+ if (ret == 0) {
+ ret = i915_gem_object_set_cache_level(obj, level);
+ i915_gem_object_unlock(obj);
+ }
+ mutex_unlock(&i915->drm.struct_mutex);
out:
i915_gem_object_put(obj);
struct i915_vma *vma;
int ret;
- lockdep_assert_held(&obj->base.dev->struct_mutex);
+ assert_object_held(obj);
/* Mark the global pin early so that we account for the
* display coherency whilst setting up the cache domains.
void
i915_gem_object_unpin_from_display_plane(struct i915_vma *vma)
{
- lockdep_assert_held(&vma->vm->i915->drm.struct_mutex);
+ struct drm_i915_gem_object *obj = vma->obj;
+
+ assert_object_held(obj);
- if (WARN_ON(vma->obj->pin_global == 0))
+ if (WARN_ON(obj->pin_global == 0))
return;
- if (--vma->obj->pin_global == 0)
+ if (--obj->pin_global == 0)
vma->display_alignment = I915_GTT_MIN_ALIGNMENT;
/* Bump the LRU to try and avoid premature eviction whilst flipping */
- i915_gem_object_bump_inactive_ggtt(vma->obj);
+ i915_gem_object_bump_inactive_ggtt(obj);
i915_vma_unpin(vma);
}
{
int ret;
- lockdep_assert_held(&obj->base.dev->struct_mutex);
+ assert_object_held(obj);
ret = i915_gem_object_wait(obj,
I915_WAIT_INTERRUPTIBLE |
- I915_WAIT_LOCKED |
(write ? I915_WAIT_ALL : 0),
MAX_SCHEDULE_TIMEOUT);
if (ret)
if (err)
goto out;
- err = i915_mutex_lock_interruptible(dev);
+ err = i915_gem_object_lock_interruptible(obj);
if (err)
goto out_unpin;
/* And bump the LRU for this access */
i915_gem_object_bump_inactive_ggtt(obj);
- mutex_unlock(&dev->struct_mutex);
+ i915_gem_object_unlock(obj);
if (write_domain != 0)
intel_fb_obj_invalidate(obj,
{
int ret;
- lockdep_assert_held(&obj->base.dev->struct_mutex);
-
*needs_clflush = 0;
if (!i915_gem_object_has_struct_page(obj))
return -ENODEV;
+ ret = i915_gem_object_lock_interruptible(obj);
+ if (ret)
+ return ret;
+
ret = i915_gem_object_wait(obj,
- I915_WAIT_INTERRUPTIBLE |
- I915_WAIT_LOCKED,
+ I915_WAIT_INTERRUPTIBLE,
MAX_SCHEDULE_TIMEOUT);
if (ret)
- return ret;
+ goto err_unlock;
ret = i915_gem_object_pin_pages(obj);
if (ret)
- return ret;
+ goto err_unlock;
if (obj->cache_coherent & I915_BO_CACHE_COHERENT_FOR_READ ||
!static_cpu_has(X86_FEATURE_CLFLUSH)) {
err_unpin:
i915_gem_object_unpin_pages(obj);
+err_unlock:
+ i915_gem_object_unlock(obj);
return ret;
}
{
int ret;
- lockdep_assert_held(&obj->base.dev->struct_mutex);
-
*needs_clflush = 0;
if (!i915_gem_object_has_struct_page(obj))
return -ENODEV;
+ ret = i915_gem_object_lock_interruptible(obj);
+ if (ret)
+ return ret;
+
ret = i915_gem_object_wait(obj,
I915_WAIT_INTERRUPTIBLE |
- I915_WAIT_LOCKED |
I915_WAIT_ALL,
MAX_SCHEDULE_TIMEOUT);
if (ret)
- return ret;
+ goto err_unlock;
ret = i915_gem_object_pin_pages(obj);
if (ret)
- return ret;
+ goto err_unlock;
if (obj->cache_coherent & I915_BO_CACHE_COHERENT_FOR_WRITE ||
!static_cpu_has(X86_FEATURE_CLFLUSH)) {
err_unpin:
i915_gem_object_unpin_pages(obj);
+err_unlock:
+ i915_gem_object_unlock(obj);
return ret;
}
if (use_cpu_reloc(cache, obj))
return NULL;
+ i915_gem_object_lock(obj);
err = i915_gem_object_set_to_gtt_domain(obj, true);
+ i915_gem_object_unlock(obj);
if (err)
return ERR_PTR(err);
*addr = value;
}
+static int reloc_move_to_gpu(struct i915_request *rq, struct i915_vma *vma)
+{
+ struct drm_i915_gem_object *obj = vma->obj;
+ int err;
+
+ i915_vma_lock(vma);
+
+ if (obj->cache_dirty & ~obj->cache_coherent)
+ i915_gem_clflush_object(obj, 0);
+ obj->write_domain = 0;
+
+ err = i915_request_await_object(rq, vma->obj, true);
+ if (err == 0)
+ err = i915_vma_move_to_active(vma, rq, EXEC_OBJECT_WRITE);
+
+ i915_vma_unlock(vma);
+
+ return err;
+}
+
static int __reloc_gpu_alloc(struct i915_execbuffer *eb,
struct i915_vma *vma,
unsigned int len)
u32 *cmd;
int err;
- if (DBG_FORCE_RELOC == FORCE_GPU_RELOC) {
- obj = vma->obj;
- if (obj->cache_dirty & ~obj->cache_coherent)
- i915_gem_clflush_object(obj, 0);
- obj->write_domain = 0;
- }
-
- GEM_BUG_ON(vma->obj->write_domain & I915_GEM_DOMAIN_CPU);
-
obj = i915_gem_batch_pool_get(&eb->engine->batch_pool, PAGE_SIZE);
if (IS_ERR(obj))
return PTR_ERR(obj);
goto err_unpin;
}
- err = i915_request_await_object(rq, vma->obj, true);
+ err = reloc_move_to_gpu(rq, vma);
if (err)
goto err_request;
batch->node.start, PAGE_SIZE,
cache->gen > 5 ? 0 : I915_DISPATCH_SECURE);
if (err)
- goto err_request;
+ goto skip_request;
+ i915_vma_lock(batch);
GEM_BUG_ON(!reservation_object_test_signaled_rcu(batch->resv, true));
err = i915_vma_move_to_active(batch, rq, 0);
- if (err)
- goto skip_request;
-
- err = i915_vma_move_to_active(vma, rq, EXEC_OBJECT_WRITE);
+ i915_vma_unlock(batch);
if (err)
goto skip_request;
static int eb_move_to_gpu(struct i915_execbuffer *eb)
{
const unsigned int count = eb->buffer_count;
+ struct ww_acquire_ctx acquire;
unsigned int i;
- int err;
+ int err = 0;
+
+ ww_acquire_init(&acquire, &reservation_ww_class);
for (i = 0; i < count; i++) {
+ struct i915_vma *vma = eb->vma[i];
+
+ err = ww_mutex_lock_interruptible(&vma->resv->lock, &acquire);
+ if (!err)
+ continue;
+
+ GEM_BUG_ON(err == -EALREADY); /* No duplicate vma */
+
+ if (err == -EDEADLK) {
+ GEM_BUG_ON(i == 0);
+ do {
+ int j = i - 1;
+
+ ww_mutex_unlock(&eb->vma[j]->resv->lock);
+
+ swap(eb->flags[i], eb->flags[j]);
+ swap(eb->vma[i], eb->vma[j]);
+ eb->vma[i]->exec_flags = &eb->flags[i];
+ } while (--i);
+ GEM_BUG_ON(vma != eb->vma[0]);
+ vma->exec_flags = &eb->flags[0];
+
+ err = ww_mutex_lock_slow_interruptible(&vma->resv->lock,
+ &acquire);
+ }
+ if (err)
+ break;
+ }
+ ww_acquire_done(&acquire);
+
+ while (i--) {
unsigned int flags = eb->flags[i];
struct i915_vma *vma = eb->vma[i];
struct drm_i915_gem_object *obj = vma->obj;
+ assert_vma_held(vma);
+
if (flags & EXEC_OBJECT_CAPTURE) {
struct i915_capture_list *capture;
capture = kmalloc(sizeof(*capture), GFP_KERNEL);
- if (unlikely(!capture))
- return -ENOMEM;
-
- capture->next = eb->request->capture_list;
- capture->vma = eb->vma[i];
- eb->request->capture_list = capture;
+ if (capture) {
+ capture->next = eb->request->capture_list;
+ capture->vma = vma;
+ eb->request->capture_list = capture;
+ }
}
/*
flags &= ~EXEC_OBJECT_ASYNC;
}
- if (flags & EXEC_OBJECT_ASYNC)
- continue;
-
- err = i915_request_await_object
- (eb->request, obj, flags & EXEC_OBJECT_WRITE);
- if (err)
- return err;
- }
+ if (err == 0 && !(flags & EXEC_OBJECT_ASYNC)) {
+ err = i915_request_await_object
+ (eb->request, obj, flags & EXEC_OBJECT_WRITE);
+ }
- for (i = 0; i < count; i++) {
- unsigned int flags = eb->flags[i];
- struct i915_vma *vma = eb->vma[i];
+ if (err == 0)
+ err = i915_vma_move_to_active(vma, eb->request, flags);
- err = i915_vma_move_to_active(vma, eb->request, flags);
- if (unlikely(err)) {
- i915_request_skip(eb->request, err);
- return err;
- }
+ i915_vma_unlock(vma);
__eb_unreserve_vma(vma, flags);
vma->exec_flags = NULL;
if (unlikely(flags & __EXEC_OBJECT_HAS_REF))
i915_vma_put(vma);
}
+ ww_acquire_fini(&acquire);
+
+ if (unlikely(err))
+ goto err_skip;
+
eb->exec = NULL;
/* Unconditionally flush any chipset caches (for streaming writes). */
i915_gem_chipset_flush(eb->i915);
-
return 0;
+
+err_skip:
+ i915_request_skip(eb->request, err);
+ return err;
}
static bool i915_gem_check_execbuffer(struct drm_i915_gem_execbuffer2 *exec)
--- /dev/null
+/*
+ * SPDX-License-Identifier: MIT
+ *
+ * Copyright © 2019 Intel Corporation
+ */
+
+#include "i915_drv.h"
+#include "i915_gem_object.h"
+
+struct stub_fence {
+ struct dma_fence dma;
+ struct i915_sw_fence chain;
+};
+
+static int __i915_sw_fence_call
+stub_notify(struct i915_sw_fence *fence, enum i915_sw_fence_notify state)
+{
+ struct stub_fence *stub = container_of(fence, typeof(*stub), chain);
+
+ switch (state) {
+ case FENCE_COMPLETE:
+ dma_fence_signal(&stub->dma);
+ break;
+
+ case FENCE_FREE:
+ dma_fence_put(&stub->dma);
+ break;
+ }
+
+ return NOTIFY_DONE;
+}
+
+static const char *stub_driver_name(struct dma_fence *fence)
+{
+ return DRIVER_NAME;
+}
+
+static const char *stub_timeline_name(struct dma_fence *fence)
+{
+ return "object";
+}
+
+static void stub_release(struct dma_fence *fence)
+{
+ struct stub_fence *stub = container_of(fence, typeof(*stub), dma);
+
+ i915_sw_fence_fini(&stub->chain);
+
+ BUILD_BUG_ON(offsetof(typeof(*stub), dma));
+ dma_fence_free(&stub->dma);
+}
+
+static const struct dma_fence_ops stub_fence_ops = {
+ .get_driver_name = stub_driver_name,
+ .get_timeline_name = stub_timeline_name,
+ .release = stub_release,
+};
+
+struct dma_fence *
+i915_gem_object_lock_fence(struct drm_i915_gem_object *obj)
+{
+ struct stub_fence *stub;
+
+ assert_object_held(obj);
+
+ stub = kmalloc(sizeof(*stub), GFP_KERNEL);
+ if (!stub)
+ return NULL;
+
+ i915_sw_fence_init(&stub->chain, stub_notify);
+ dma_fence_init(&stub->dma, &stub_fence_ops, &stub->chain.wait.lock,
+ to_i915(obj->base.dev)->mm.unordered_timeline,
+ 0);
+
+ if (i915_sw_fence_await_reservation(&stub->chain,
+ obj->resv, NULL,
+ true, I915_FENCE_TIMEOUT,
+ I915_FENCE_GFP) < 0)
+ goto err;
+
+ reservation_object_add_excl_fence(obj->resv, &stub->dma);
+
+ return &stub->dma;
+
+err:
+ stub_release(&stub->dma);
+ return NULL;
+}
+
+void i915_gem_object_unlock_fence(struct drm_i915_gem_object *obj,
+ struct dma_fence *fence)
+{
+ struct stub_fence *stub = container_of(fence, typeof(*stub), dma);
+
+ i915_sw_fence_commit(&stub->chain);
+}
struct drm_i915_private *dev_priv = to_i915(obj->base.dev);
struct i915_vma *vma;
+ assert_object_held(obj);
+
if (!(obj->write_domain & flush_domains))
return;
__drm_gem_object_put(&obj->base);
}
+#define assert_object_held(obj) reservation_object_assert_held((obj)->resv)
+
static inline void i915_gem_object_lock(struct drm_i915_gem_object *obj)
{
reservation_object_lock(obj->resv, NULL);
}
+static inline int
+i915_gem_object_lock_interruptible(struct drm_i915_gem_object *obj)
+{
+ return reservation_object_lock_interruptible(obj->resv, NULL);
+}
+
static inline void i915_gem_object_unlock(struct drm_i915_gem_object *obj)
{
reservation_object_unlock(obj->resv);
}
+struct dma_fence *
+i915_gem_object_lock_fence(struct drm_i915_gem_object *obj);
+void i915_gem_object_unlock_fence(struct drm_i915_gem_object *obj,
+ struct dma_fence *fence);
+
static inline void
i915_gem_object_set_readonly(struct drm_i915_gem_object *obj)
{
i915_gem_object_finish_access(struct drm_i915_gem_object *obj)
{
i915_gem_object_unpin_pages(obj);
+ i915_gem_object_unlock(obj);
}
static inline struct intel_engine_cs *
* machine in an unusable condition.
*/
- mutex_lock(&i915->drm.struct_mutex);
for (phase = phases; *phase; phase++) {
- list_for_each_entry(obj, *phase, mm.link)
+ list_for_each_entry(obj, *phase, mm.link) {
+ i915_gem_object_lock(obj);
WARN_ON(i915_gem_object_set_to_gtt_domain(obj, false));
+ i915_gem_object_unlock(obj);
+ }
}
- mutex_unlock(&i915->drm.struct_mutex);
intel_uc_sanitize(i915);
i915_gem_sanitize(i915);
GEM_BUG_ON(!intel_engine_can_store_dword(engine));
- err = i915_gem_object_set_to_gtt_domain(vma->obj, true);
- if (err)
- return err;
-
batch = gpu_write_dw(vma, dword * sizeof(u32), value);
if (IS_ERR(batch))
return PTR_ERR(batch);
goto err_batch;
}
+ i915_vma_lock(batch);
err = i915_vma_move_to_active(batch, rq, 0);
+ i915_vma_unlock(batch);
if (err)
goto err_request;
i915_gem_object_set_active_reference(batch->obj);
- err = i915_vma_move_to_active(vma, rq, EXEC_OBJECT_WRITE);
+ i915_vma_lock(vma);
+ err = i915_gem_object_set_to_gtt_domain(vma->obj, false);
+ if (err == 0)
+ err = i915_vma_move_to_active(vma, rq, EXEC_OBJECT_WRITE);
+ i915_vma_unlock(vma);
if (err)
goto err_request;
u32 __iomem *map;
int err;
+ i915_gem_object_lock(obj);
err = i915_gem_object_set_to_gtt_domain(obj, true);
+ i915_gem_object_unlock(obj);
if (err)
return err;
u32 __iomem *map;
int err;
+ i915_gem_object_lock(obj);
err = i915_gem_object_set_to_gtt_domain(obj, false);
+ i915_gem_object_unlock(obj);
if (err)
return err;
u32 *map;
int err;
+ i915_gem_object_lock(obj);
err = i915_gem_object_set_to_wc_domain(obj, true);
+ i915_gem_object_unlock(obj);
if (err)
return err;
u32 *map;
int err;
+ i915_gem_object_lock(obj);
err = i915_gem_object_set_to_wc_domain(obj, false);
+ i915_gem_object_unlock(obj);
if (err)
return err;
u32 *cs;
int err;
+ i915_gem_object_lock(obj);
err = i915_gem_object_set_to_gtt_domain(obj, true);
+ i915_gem_object_unlock(obj);
if (err)
return err;
}
intel_ring_advance(rq, cs);
+ i915_vma_lock(vma);
err = i915_vma_move_to_active(vma, rq, EXEC_OBJECT_WRITE);
+ i915_vma_unlock(vma);
i915_vma_unpin(vma);
i915_request_add(rq);
i915_gem_object_flush_map(obj);
i915_gem_object_unpin_map(obj);
+ i915_gem_object_lock(obj);
err = i915_gem_object_set_to_gtt_domain(obj, false);
+ i915_gem_object_unlock(obj);
if (err)
goto err;
if (IS_ERR(vma))
return PTR_ERR(vma);
+ i915_gem_object_lock(obj);
err = i915_gem_object_set_to_gtt_domain(obj, false);
+ i915_gem_object_unlock(obj);
if (err)
return err;
if (err)
goto err_request;
+ i915_vma_lock(batch);
err = i915_vma_move_to_active(batch, rq, 0);
+ i915_vma_unlock(batch);
if (err)
goto skip_request;
+ i915_vma_lock(vma);
err = i915_vma_move_to_active(vma, rq, EXEC_OBJECT_WRITE);
+ i915_vma_unlock(vma);
if (err)
goto skip_request;
if (IS_ERR(vma))
return PTR_ERR(vma);
+ i915_gem_object_lock(obj);
err = i915_gem_object_set_to_gtt_domain(obj, false);
+ i915_gem_object_unlock(obj);
if (err)
return err;
if (err)
goto err_request;
+ i915_vma_lock(batch);
err = i915_vma_move_to_active(batch, rq, 0);
+ i915_vma_unlock(batch);
if (err)
goto skip_request;
+ i915_vma_lock(vma);
err = i915_vma_move_to_active(vma, rq, EXEC_OBJECT_WRITE);
+ i915_vma_unlock(vma);
if (err)
goto skip_request;
if (err)
goto err_request;
+ i915_vma_lock(vma);
err = i915_vma_move_to_active(vma, rq, 0);
+ i915_vma_unlock(vma);
if (err)
goto skip_request;
if (err)
goto err_request;
+ i915_vma_lock(vma);
err = i915_vma_move_to_active(vma, rq, EXEC_OBJECT_WRITE);
+ i915_vma_unlock(vma);
if (err)
goto skip_request;
i915_request_add(rq);
+ i915_gem_object_lock(obj);
err = i915_gem_object_set_to_cpu_domain(obj, false);
+ i915_gem_object_unlock(obj);
if (err)
goto err;
GEM_BUG_ON(view.partial.size > nreal);
cond_resched();
+ i915_gem_object_lock(obj);
err = i915_gem_object_set_to_gtt_domain(obj, true);
+ i915_gem_object_unlock(obj);
if (err) {
pr_err("Failed to flush to GTT write domain; err=%d\n",
err);
if (offset >= obj->base.size)
continue;
+ i915_gem_object_lock(obj);
i915_gem_object_flush_write_domain(obj, ~I915_GEM_DOMAIN_CPU);
+ i915_gem_object_unlock(obj);
p = i915_gem_object_get_page(obj, offset >> PAGE_SHIFT);
cpu = kmap(p) + offset_in_page(offset);
return PTR_ERR(rq);
}
+ i915_vma_lock(vma);
err = i915_vma_move_to_active(vma, rq, EXEC_OBJECT_WRITE);
+ i915_vma_unlock(vma);
i915_request_add(rq);
}
/* Make the object dirty so that put_pages must do copy back the data */
- mutex_lock(&i915->drm.struct_mutex);
+ i915_gem_object_lock(obj);
err = i915_gem_object_set_to_gtt_domain(obj, true);
- mutex_unlock(&i915->drm.struct_mutex);
+ i915_gem_object_unlock(obj);
if (err) {
pr_err("i915_gem_object_set_to_gtt_domain failed with err=%d\n",
err);
{
int err;
+ i915_vma_lock(vma);
err = i915_vma_move_to_active(vma, rq, flags);
+ i915_vma_unlock(vma);
if (err)
return err;
}
}
+ i915_vma_lock(arg.vma);
err = i915_vma_move_to_active(arg.vma, rq, flags);
+ i915_vma_unlock(arg.vma);
if (flags & EXEC_OBJECT_NEEDS_FENCE)
i915_vma_unpin_fence(arg.vma);
}
if (vma) {
+ i915_vma_lock(vma);
err = rq->engine->emit_bb_start(rq,
vma->node.start,
PAGE_SIZE, 0);
if (!err)
err = i915_vma_move_to_active(vma, rq, 0);
+ i915_vma_unlock(vma);
}
i915_request_add(rq);
goto err_pin;
}
+ i915_vma_lock(vma);
err = i915_vma_move_to_active(vma, rq, EXEC_OBJECT_WRITE);
+ i915_vma_unlock(vma);
if (err)
goto err_req;
return PTR_ERR(results);
err = 0;
+ i915_gem_object_lock(results);
igt_wedge_on_timeout(&wedge, ctx->i915, HZ / 5) /* a safety net! */
err = i915_gem_object_set_to_cpu_domain(results, false);
+ i915_gem_object_unlock(results);
if (i915_terminally_wedged(ctx->i915))
err = -EIO;
if (err)
if (err)
goto err_obj;
+ i915_gem_object_lock(obj);
err = i915_gem_object_set_to_wc_domain(obj, true);
+ i915_gem_object_unlock(obj);
if (err)
goto err_obj;
goto put_obj;
}
+ i915_gem_object_lock(obj);
ret = i915_gem_object_set_to_cpu_domain(obj, false);
+ i915_gem_object_unlock(obj);
if (ret) {
gvt_vgpu_err("failed to set shadow indirect ctx to CPU\n");
goto unmap_src;
}
ret = i915_gem_object_set_to_gtt_domain(bb->obj,
- false);
+ false);
if (ret)
goto err;
- i915_gem_object_finish_access(bb->obj);
- bb->accessing = false;
-
ret = i915_vma_move_to_active(bb->vma,
workload->req,
0);
if (ret)
goto err;
+
+ i915_gem_object_finish_access(bb->obj);
+ bb->accessing = false;
}
}
return 0;
void *dst, *src;
int ret;
- ret = i915_gem_object_prepare_read(src_obj, &src_needs_clflush);
+ ret = i915_gem_object_prepare_write(dst_obj, &dst_needs_clflush);
if (ret)
return ERR_PTR(ret);
- ret = i915_gem_object_prepare_write(dst_obj, &dst_needs_clflush);
- if (ret) {
- dst = ERR_PTR(ret);
- goto unpin_src;
- }
-
dst = i915_gem_object_pin_map(dst_obj, I915_MAP_FORCE_WB);
+ i915_gem_object_finish_access(dst_obj);
if (IS_ERR(dst))
- goto unpin_dst;
+ return dst;
+
+ ret = i915_gem_object_prepare_read(src_obj, &src_needs_clflush);
+ if (ret) {
+ i915_gem_object_unpin_map(dst_obj);
+ return ERR_PTR(ret);
+ }
src = ERR_PTR(-ENODEV);
if (src_needs_clflush &&
}
}
+ i915_gem_object_finish_access(src_obj);
+
/* dst_obj is returned with vmap pinned */
*needs_clflush_after = dst_needs_clflush & CLFLUSH_AFTER;
-unpin_dst:
- i915_gem_object_finish_access(dst_obj);
-unpin_src:
- i915_gem_object_finish_access(src_obj);
return dst;
}
{
struct i915_vma *vma;
LIST_HEAD(still_in_list);
- int ret;
+ int ret = 0;
lockdep_assert_held(&obj->base.dev->struct_mutex);
- /* Closed vma are removed from the obj->vma_list - but they may
- * still have an active binding on the object. To remove those we
- * must wait for all rendering to complete to the object (as unbinding
- * must anyway), and retire the requests.
- */
- ret = i915_gem_object_set_to_cpu_domain(obj, false);
- if (ret)
- return ret;
-
spin_lock(&obj->vma.lock);
while (!ret && (vma = list_first_entry_or_null(&obj->vma.list,
struct i915_vma,
unsigned int flags,
long timeout)
{
- struct i915_request *rq;
-
BUILD_BUG_ON(I915_WAIT_INTERRUPTIBLE != 0x1);
if (test_bit(DMA_FENCE_FLAG_SIGNALED_BIT, &fence->flags))
return timeout;
- if (!dma_fence_is_i915(fence))
- return dma_fence_wait_timeout(fence,
- flags & I915_WAIT_INTERRUPTIBLE,
- timeout);
+ if (dma_fence_is_i915(fence))
+ return i915_request_wait(to_request(fence), flags, timeout);
- rq = to_request(fence);
- if (i915_request_completed(rq))
- goto out;
-
- timeout = i915_request_wait(rq, flags, timeout);
-
-out:
- if (flags & I915_WAIT_LOCKED && i915_request_completed(rq))
- i915_request_retire_upto(rq);
-
- return timeout;
+ return dma_fence_wait_timeout(fence,
+ flags & I915_WAIT_INTERRUPTIBLE,
+ timeout);
}
static long
i915_gem_shmem_pread(struct drm_i915_gem_object *obj,
struct drm_i915_gem_pread *args)
{
- char __user *user_data;
- u64 remain;
unsigned int needs_clflush;
unsigned int idx, offset;
+ struct dma_fence *fence;
+ char __user *user_data;
+ u64 remain;
int ret;
- ret = mutex_lock_interruptible(&obj->base.dev->struct_mutex);
- if (ret)
- return ret;
-
ret = i915_gem_object_prepare_read(obj, &needs_clflush);
- mutex_unlock(&obj->base.dev->struct_mutex);
if (ret)
return ret;
+ fence = i915_gem_object_lock_fence(obj);
+ i915_gem_object_finish_access(obj);
+ if (!fence)
+ return -ENOMEM;
+
remain = args->size;
user_data = u64_to_user_ptr(args->data_ptr);
offset = offset_in_page(args->offset);
offset = 0;
}
- i915_gem_object_finish_access(obj);
+ i915_gem_object_unlock_fence(obj, fence);
return ret;
}
struct i915_ggtt *ggtt = &i915->ggtt;
intel_wakeref_t wakeref;
struct drm_mm_node node;
- struct i915_vma *vma;
+ struct dma_fence *fence;
void __user *user_data;
+ struct i915_vma *vma;
u64 remain, offset;
int ret;
GEM_BUG_ON(!node.allocated);
}
- ret = i915_gem_object_set_to_gtt_domain(obj, false);
+ mutex_unlock(&i915->drm.struct_mutex);
+
+ ret = i915_gem_object_lock_interruptible(obj);
if (ret)
goto out_unpin;
- mutex_unlock(&i915->drm.struct_mutex);
+ ret = i915_gem_object_set_to_gtt_domain(obj, false);
+ if (ret) {
+ i915_gem_object_unlock(obj);
+ goto out_unpin;
+ }
+
+ fence = i915_gem_object_lock_fence(obj);
+ i915_gem_object_unlock(obj);
+ if (!fence) {
+ ret = -ENOMEM;
+ goto out_unpin;
+ }
user_data = u64_to_user_ptr(args->data_ptr);
remain = args->size;
offset += page_length;
}
- mutex_lock(&i915->drm.struct_mutex);
+ i915_gem_object_unlock_fence(obj, fence);
out_unpin:
+ mutex_lock(&i915->drm.struct_mutex);
if (node.allocated) {
wmb();
ggtt->vm.clear_range(&ggtt->vm, node.start, node.size);
struct i915_ggtt *ggtt = &i915->ggtt;
intel_wakeref_t wakeref;
struct drm_mm_node node;
+ struct dma_fence *fence;
struct i915_vma *vma;
u64 remain, offset;
void __user *user_data;
GEM_BUG_ON(!node.allocated);
}
- ret = i915_gem_object_set_to_gtt_domain(obj, true);
+ mutex_unlock(&i915->drm.struct_mutex);
+
+ ret = i915_gem_object_lock_interruptible(obj);
if (ret)
goto out_unpin;
- mutex_unlock(&i915->drm.struct_mutex);
+ ret = i915_gem_object_set_to_gtt_domain(obj, true);
+ if (ret) {
+ i915_gem_object_unlock(obj);
+ goto out_unpin;
+ }
+
+ fence = i915_gem_object_lock_fence(obj);
+ i915_gem_object_unlock(obj);
+ if (!fence) {
+ ret = -ENOMEM;
+ goto out_unpin;
+ }
intel_fb_obj_invalidate(obj, ORIGIN_CPU);
}
intel_fb_obj_flush(obj, ORIGIN_CPU);
- mutex_lock(&i915->drm.struct_mutex);
+ i915_gem_object_unlock_fence(obj, fence);
out_unpin:
+ mutex_lock(&i915->drm.struct_mutex);
if (node.allocated) {
wmb();
ggtt->vm.clear_range(&ggtt->vm, node.start, node.size);
i915_gem_shmem_pwrite(struct drm_i915_gem_object *obj,
const struct drm_i915_gem_pwrite *args)
{
- struct drm_i915_private *i915 = to_i915(obj->base.dev);
- void __user *user_data;
- u64 remain;
unsigned int partial_cacheline_write;
unsigned int needs_clflush;
unsigned int offset, idx;
+ struct dma_fence *fence;
+ void __user *user_data;
+ u64 remain;
int ret;
- ret = mutex_lock_interruptible(&i915->drm.struct_mutex);
- if (ret)
- return ret;
-
ret = i915_gem_object_prepare_write(obj, &needs_clflush);
- mutex_unlock(&i915->drm.struct_mutex);
if (ret)
return ret;
+ fence = i915_gem_object_lock_fence(obj);
+ i915_gem_object_finish_access(obj);
+ if (!fence)
+ return -ENOMEM;
+
/* If we don't overwrite a cacheline completely we need to be
* careful to have up-to-date data by first clflushing. Don't
* overcomplicate things and flush the entire patch.
}
intel_fb_obj_flush(obj, ORIGIN_CPU);
- i915_gem_object_finish_access(obj);
+ i915_gem_object_unlock_fence(obj, fence);
+
return ret;
}
if (err)
goto err_active;
+ i915_gem_object_lock(state->obj);
err = i915_gem_object_set_to_cpu_domain(state->obj, false);
+ i915_gem_object_unlock(state->obj);
if (err)
goto err_active;
i915_gem_shrink(i915, -1UL, NULL, I915_SHRINK_UNBOUND);
i915_gem_drain_freed_objects(i915);
- mutex_lock(&i915->drm.struct_mutex);
for (phase = phases; *phase; phase++) {
- list_for_each_entry(obj, *phase, mm.link)
+ list_for_each_entry(obj, *phase, mm.link) {
+ i915_gem_object_lock(obj);
WARN_ON(i915_gem_object_set_to_cpu_domain(obj, true));
+ i915_gem_object_unlock(obj);
+ }
}
- mutex_unlock(&i915->drm.struct_mutex);
return 0;
}
WARN_ON(i915_vma_bind(vma,
obj ? obj->cache_level : 0,
PIN_UPDATE));
- if (obj)
+ if (obj) {
+ i915_gem_object_lock(obj);
WARN_ON(i915_gem_object_set_to_gtt_domain(obj, false));
+ i915_gem_object_unlock(obj);
+ }
lock:
mutex_lock(&ggtt->vm.mutex);
goto err_unpin;
}
+ i915_vma_lock(so.vma);
err = i915_vma_move_to_active(so.vma, rq, 0);
+ i915_vma_unlock(so.vma);
err_unpin:
i915_vma_unpin(so.vma);
err_vma:
{
lockdep_assert_held(&vma->vm->i915->drm.struct_mutex);
- GEM_BUG_ON(i915_vma_is_active(vma));
GEM_BUG_ON(i915_vma_is_pinned(vma));
if (i915_vma_is_closed(vma))
list_del(&vma->closed_link);
WARN_ON(i915_vma_unbind(vma));
+ GEM_BUG_ON(i915_vma_is_active(vma));
+
__i915_vma_destroy(vma);
}
* handle an error right now. Worst case should be missed
* synchronisation leading to rendering corruption.
*/
- reservation_object_lock(resv, NULL);
if (flags & EXEC_OBJECT_WRITE)
reservation_object_add_excl_fence(resv, &rq->fence);
else if (reservation_object_reserve_shared(resv, 1) == 0)
reservation_object_add_shared_fence(resv, &rq->fence);
- reservation_object_unlock(resv);
}
int i915_vma_move_to_active(struct i915_vma *vma,
{
struct drm_i915_gem_object *obj = vma->obj;
- lockdep_assert_held(&rq->i915->drm.struct_mutex);
+ assert_vma_held(vma);
+ assert_object_held(obj);
GEM_BUG_ON(!drm_mm_node_allocated(&vma->node));
/*
void i915_vma_reopen(struct i915_vma *vma);
void i915_vma_destroy(struct i915_vma *vma);
+#define assert_vma_held(vma) reservation_object_assert_held((vma)->resv)
+
+static inline void i915_vma_lock(struct i915_vma *vma)
+{
+ reservation_object_lock(vma->resv, NULL);
+}
+
+static inline void i915_vma_unlock(struct i915_vma *vma)
+{
+ reservation_object_unlock(vma->resv);
+}
+
int __i915_vma_do_pin(struct i915_vma *vma,
u64 size, u64 alignment, u64 flags);
static inline int __must_check
* pin/unpin/fence and not more.
*/
wakeref = intel_runtime_pm_get(dev_priv);
+ i915_gem_object_lock(obj);
atomic_inc(&dev_priv->gpu_error.pending_fb_pin);
err:
atomic_dec(&dev_priv->gpu_error.pending_fb_pin);
+ i915_gem_object_unlock(obj);
intel_runtime_pm_put(dev_priv, wakeref);
return vma;
}
{
lockdep_assert_held(&vma->vm->i915->drm.struct_mutex);
+ i915_gem_object_lock(vma->obj);
if (flags & PLANE_HAS_FENCE)
i915_vma_unpin_fence(vma);
i915_gem_object_unpin_from_display_plane(vma);
+ i915_gem_object_unlock(vma->obj);
+
i915_vma_put(vma);
}
static int guc_log_map(struct intel_guc_log *log)
{
- struct intel_guc *guc = log_to_guc(log);
- struct drm_i915_private *dev_priv = guc_to_i915(guc);
void *vaddr;
int ret;
if (!log->vma)
return -ENODEV;
- mutex_lock(&dev_priv->drm.struct_mutex);
+ i915_gem_object_lock(log->vma->obj);
ret = i915_gem_object_set_to_wc_domain(log->vma->obj, true);
- mutex_unlock(&dev_priv->drm.struct_mutex);
+ i915_gem_object_unlock(log->vma->obj);
if (ret)
return ret;
atomic_inc(&dev_priv->gpu_error.pending_fb_pin);
+ i915_gem_object_lock(new_bo);
vma = i915_gem_object_pin_to_display_plane(new_bo,
0, NULL, PIN_MAPPABLE);
+ i915_gem_object_unlock(new_bo);
if (IS_ERR(vma)) {
ret = PTR_ERR(vma);
goto out_pin_section;
static int get_registers(struct intel_overlay *overlay, bool use_phys)
{
+ struct drm_i915_private *i915 = overlay->i915;
struct drm_i915_gem_object *obj;
struct i915_vma *vma;
int err;
- obj = i915_gem_object_create_stolen(overlay->i915, PAGE_SIZE);
+ mutex_lock(&i915->drm.struct_mutex);
+
+ obj = i915_gem_object_create_stolen(i915, PAGE_SIZE);
if (obj == NULL)
- obj = i915_gem_object_create_internal(overlay->i915, PAGE_SIZE);
- if (IS_ERR(obj))
- return PTR_ERR(obj);
+ obj = i915_gem_object_create_internal(i915, PAGE_SIZE);
+ if (IS_ERR(obj)) {
+ err = PTR_ERR(obj);
+ goto err_unlock;
+ }
vma = i915_gem_object_ggtt_pin(obj, NULL, 0, 0, PIN_MAPPABLE);
if (IS_ERR(vma)) {
}
overlay->reg_bo = obj;
+ mutex_unlock(&i915->drm.struct_mutex);
return 0;
err_put_bo:
i915_gem_object_put(obj);
+err_unlock:
+ mutex_unlock(&i915->drm.struct_mutex);
return err;
}
INIT_ACTIVE_REQUEST(&overlay->last_flip);
- mutex_lock(&dev_priv->drm.struct_mutex);
-
ret = get_registers(overlay, OVERLAY_NEEDS_PHYSICAL(dev_priv));
if (ret)
goto out_free;
+ i915_gem_object_lock(overlay->reg_bo);
ret = i915_gem_object_set_to_gtt_domain(overlay->reg_bo, true);
+ i915_gem_object_unlock(overlay->reg_bo);
if (ret)
goto out_reg_bo;
- mutex_unlock(&dev_priv->drm.struct_mutex);
-
memset_io(overlay->regs, 0, sizeof(struct overlay_registers));
update_polyphase_filter(overlay->regs);
update_reg_attrs(overlay, overlay->regs);
out_reg_bo:
i915_gem_object_put(overlay->reg_bo);
out_free:
- mutex_unlock(&dev_priv->drm.struct_mutex);
kfree(overlay);
}
intel_uc_fw_type_repr(uc_fw->type),
intel_uc_fw_status_repr(uc_fw->load_status));
- intel_uc_fw_ggtt_bind(uc_fw);
-
/* Call custom loader */
+ intel_uc_fw_ggtt_bind(uc_fw);
err = xfer(uc_fw);
+ intel_uc_fw_ggtt_unbind(uc_fw);
if (err)
goto fail;
- intel_uc_fw_ggtt_unbind(uc_fw);
-
uc_fw->load_status = INTEL_UC_FIRMWARE_SUCCESS;
DRM_DEBUG_DRIVER("%s fw load %s\n",
intel_uc_fw_type_repr(uc_fw->type),
i915_gem_object_set_active_reference(batch->obj);
}
+ i915_vma_lock(batch);
err = i915_vma_move_to_active(batch, request[id], 0);
+ i915_vma_unlock(batch);
GEM_BUG_ON(err);
i915_request_get(request[id]);
GEM_BUG_ON(err);
request[id]->batch = batch;
+ i915_vma_lock(batch);
err = i915_vma_move_to_active(batch, request[id], 0);
+ i915_vma_unlock(batch);
GEM_BUG_ON(err);
i915_gem_object_set_active_reference(batch->obj);
unsigned int x, y;
int err;
+ i915_gem_object_lock(obj);
err = i915_gem_object_set_to_gtt_domain(obj, true);
+ i915_gem_object_unlock(obj);
if (err)
goto out;
{
int err;
+ i915_vma_lock(vma);
err = i915_vma_move_to_active(vma, rq, flags);
+ i915_vma_unlock(vma);
if (err)
return err;
projects / openwrt / staging / blogic.git / commitdiff