kref_put(&ctx->ref, i915_gem_context_free);
}
+static inline struct intel_timeline *
+i915_gem_context_lookup_timeline(struct i915_gem_context *ctx,
+ struct intel_engine_cs *engine)
+{
+ struct i915_address_space *vm;
+
+ vm = ctx->ppgtt ? &ctx->ppgtt->base : &ctx->i915->ggtt.base;
+ return &vm->timeline.engine[engine->id];
+}
+
static inline bool i915_gem_context_is_default(const struct i915_gem_context *c)
{
return c->user_handle == DEFAULT_CONTEXT_HANDLE;
* not need an engine->irq_seqno_barrier() before the seqno reads.
*/
list_for_each_entry(request, &engine->timeline->requests, link) {
- if (i915_gem_request_completed(request))
+ if (__i915_gem_request_completed(request))
continue;
- if (!i915_sw_fence_done(&request->submit))
- break;
-
return request;
}
{
struct drm_i915_gem_request *request;
struct i915_gem_context *incomplete_ctx;
+ struct intel_timeline *timeline;
bool ring_hung;
if (engine->irq_seqno_barrier)
list_for_each_entry_continue(request, &engine->timeline->requests, link)
if (request->ctx == incomplete_ctx)
reset_request(request);
+
+ timeline = i915_gem_context_lookup_timeline(incomplete_ctx, engine);
+ list_for_each_entry(request, &timeline->requests, link)
+ reset_request(request);
}
void i915_gem_reset(struct drm_i915_private *dev_priv)
return ctx;
if (USES_FULL_PPGTT(dev)) {
- struct i915_hw_ppgtt *ppgtt =
- i915_ppgtt_create(to_i915(dev), file_priv);
+ struct i915_hw_ppgtt *ppgtt;
+ ppgtt = i915_ppgtt_create(to_i915(dev), file_priv, ctx->name);
if (IS_ERR(ppgtt)) {
DRM_DEBUG_DRIVER("PPGTT setup failed (%ld)\n",
PTR_ERR(ppgtt));
#include "intel_drv.h"
#include "i915_trace.h"
-static bool
-gpu_is_idle(struct drm_i915_private *dev_priv)
+static bool ggtt_is_idle(struct drm_i915_private *dev_priv)
{
+ struct i915_ggtt *ggtt = &dev_priv->ggtt;
struct intel_engine_cs *engine;
enum intel_engine_id id;
for_each_engine(engine, dev_priv, id) {
- if (intel_engine_is_active(engine))
+ struct intel_timeline *tl;
+
+ tl = &ggtt->base.timeline.engine[engine->id];
+ if (i915_gem_active_isset(&tl->last_request))
return false;
}
if (!i915_is_ggtt(vm) || flags & PIN_NONBLOCK)
return -ENOSPC;
- if (gpu_is_idle(dev_priv)) {
+ if (ggtt_is_idle(dev_priv)) {
/* If we still have pending pageflip completions, drop
* back to userspace to give our workqueues time to
* acquire our locks and unpin the old scanouts.
}
static void i915_address_space_init(struct i915_address_space *vm,
- struct drm_i915_private *dev_priv)
+ struct drm_i915_private *dev_priv,
+ const char *name)
{
+ i915_gem_timeline_init(dev_priv, &vm->timeline, name);
drm_mm_init(&vm->mm, vm->start, vm->total);
INIT_LIST_HEAD(&vm->active_list);
INIT_LIST_HEAD(&vm->inactive_list);
static int i915_ppgtt_init(struct i915_hw_ppgtt *ppgtt,
struct drm_i915_private *dev_priv,
- struct drm_i915_file_private *file_priv)
+ struct drm_i915_file_private *file_priv,
+ const char *name)
{
int ret;
ret = __hw_ppgtt_init(ppgtt, dev_priv);
if (ret == 0) {
kref_init(&ppgtt->ref);
- i915_address_space_init(&ppgtt->base, dev_priv);
+ i915_address_space_init(&ppgtt->base, dev_priv, name);
ppgtt->base.file = file_priv;
}
struct i915_hw_ppgtt *
i915_ppgtt_create(struct drm_i915_private *dev_priv,
- struct drm_i915_file_private *fpriv)
+ struct drm_i915_file_private *fpriv,
+ const char *name)
{
struct i915_hw_ppgtt *ppgtt;
int ret;
if (!ppgtt)
return ERR_PTR(-ENOMEM);
- ret = i915_ppgtt_init(ppgtt, dev_priv, fpriv);
+ ret = i915_ppgtt_init(ppgtt, dev_priv, fpriv, name);
if (ret) {
kfree(ppgtt);
return ERR_PTR(ret);
WARN_ON(!list_empty(&ppgtt->base.inactive_list));
WARN_ON(!list_empty(&ppgtt->base.unbound_list));
+ i915_gem_timeline_fini(&ppgtt->base.timeline);
list_del(&ppgtt->base.global_link);
drm_mm_takedown(&ppgtt->base.mm);
/* Subtract the guard page before address space initialization to
* shrink the range used by drm_mm.
*/
+ mutex_lock(&dev_priv->drm.struct_mutex);
ggtt->base.total -= PAGE_SIZE;
- i915_address_space_init(&ggtt->base, dev_priv);
+ i915_address_space_init(&ggtt->base, dev_priv, "[global]");
ggtt->base.total += PAGE_SIZE;
if (!HAS_LLC(dev_priv))
ggtt->base.mm.color_adjust = i915_gtt_color_adjust;
+ mutex_unlock(&dev_priv->drm.struct_mutex);
if (!io_mapping_init_wc(&dev_priv->ggtt.mappable,
dev_priv->ggtt.mappable_base,
struct i915_address_space {
struct drm_mm mm;
+ struct i915_gem_timeline timeline;
struct drm_device *dev;
/* Every address space belongs to a struct file - except for the global
* GTT that is owned by the driver (and so @file is set to NULL). In
int i915_ppgtt_init_hw(struct drm_device *dev);
void i915_ppgtt_release(struct kref *kref);
struct i915_hw_ppgtt *i915_ppgtt_create(struct drm_i915_private *dev_priv,
- struct drm_i915_file_private *fpriv);
+ struct drm_i915_file_private *fpriv,
+ const char *name);
static inline void i915_ppgtt_get(struct i915_hw_ppgtt *ppgtt)
{
if (ppgtt)
static const char *i915_fence_get_timeline_name(struct dma_fence *fence)
{
- /* Timelines are bound by eviction to a VM. However, since
- * we only have a global seqno at the moment, we only have
- * a single timeline. Note that each timeline will have
- * multiple execution contexts (fence contexts) as we allow
- * engines within a single timeline to execute in parallel.
- */
return to_request(fence)->timeline->common->name;
}
return i915_wait_request(to_request(fence), interruptible, timeout);
}
-static void i915_fence_value_str(struct dma_fence *fence, char *str, int size)
-{
- snprintf(str, size, "%u", fence->seqno);
-}
-
-static void i915_fence_timeline_value_str(struct dma_fence *fence, char *str,
- int size)
-{
- snprintf(str, size, "%u",
- intel_engine_get_seqno(to_request(fence)->engine));
-}
-
static void i915_fence_release(struct dma_fence *fence)
{
struct drm_i915_gem_request *req = to_request(fence);
.signaled = i915_fence_signaled,
.wait = i915_fence_wait,
.release = i915_fence_release,
- .fence_value_str = i915_fence_value_str,
- .timeline_value_str = i915_fence_timeline_value_str,
};
int i915_gem_request_add_to_client(struct drm_i915_gem_request *req,
GEM_BUG_ON(!i915_gem_request_completed(request));
trace_i915_gem_request_retire(request);
+
+ spin_lock_irq(&request->engine->timeline->lock);
list_del_init(&request->link);
+ spin_unlock_irq(&request->engine->timeline->lock);
/* We know the GPU must have read the request to have
* sent us the seqno + interrupt, so use the position
return 0;
}
+static u32 __timeline_get_seqno(struct i915_gem_timeline *tl)
+{
+ /* next_seqno only incremented under a mutex */
+ return ++tl->next_seqno.counter;
+}
+
static u32 timeline_get_seqno(struct i915_gem_timeline *tl)
{
return atomic_inc_return(&tl->next_seqno);
container_of(fence, typeof(*request), submit);
struct intel_engine_cs *engine = request->engine;
struct intel_timeline *timeline;
+ unsigned long flags;
u32 seqno;
if (state != FENCE_COMPLETE)
return NOTIFY_DONE;
- /* Will be called from irq-context when using foreign DMA fences */
+ /* Transfer from per-context onto the global per-engine timeline */
+ timeline = engine->timeline;
+ GEM_BUG_ON(timeline == request->timeline);
- timeline = request->timeline;
+ /* Will be called from irq-context when using foreign DMA fences */
+ spin_lock_irqsave(&timeline->lock, flags);
- seqno = request->fence.seqno;
+ seqno = timeline_get_seqno(timeline->common);
GEM_BUG_ON(!seqno);
GEM_BUG_ON(i915_seqno_passed(intel_engine_get_seqno(engine), seqno));
request->ring->vaddr + request->postfix);
engine->submit_request(request);
+ spin_lock_nested(&request->timeline->lock, SINGLE_DEPTH_NESTING);
+ list_move_tail(&request->link, &timeline->requests);
+ spin_unlock(&request->timeline->lock);
+
+ spin_unlock_irqrestore(&timeline->lock, flags);
+
return NOTIFY_DONE;
}
/* Move the oldest request to the slab-cache (if not in use!) */
req = list_first_entry_or_null(&engine->timeline->requests,
typeof(*req), link);
- if (req && i915_gem_request_completed(req))
+ if (req && __i915_gem_request_completed(req))
i915_gem_request_retire(req);
/* Beware: Dragons be flying overhead.
goto err_unreserve;
}
- req->timeline = engine->timeline;
+ req->timeline = i915_gem_context_lookup_timeline(ctx, engine);
+ GEM_BUG_ON(req->timeline == engine->timeline);
spin_lock_init(&req->lock);
dma_fence_init(&req->fence,
&i915_fence_ops,
&req->lock,
req->timeline->fence_context,
- timeline_get_seqno(req->timeline->common));
+ __timeline_get_seqno(req->timeline->common));
i915_sw_fence_init(&req->submit, submit_notify);
i915_sw_fence_await_sw_fence(&request->submit, &prev->submit,
&request->submitq);
+ spin_lock_irq(&timeline->lock);
list_add_tail(&request->link, &timeline->requests);
+ spin_unlock_irq(&timeline->lock);
+
+ GEM_BUG_ON(i915_seqno_passed(timeline->last_submitted_seqno,
+ request->fence.seqno));
- timeline->last_pending_seqno = request->fence.seqno;
+ timeline->last_submitted_seqno = request->fence.seqno;
i915_gem_active_set(&timeline->last_request, request);
list_add_tail(&request->ring_link, &ring->request_list);
list_for_each_entry_safe(request, next,
&engine->timeline->requests, link) {
- if (!i915_gem_request_completed(request))
+ if (!__i915_gem_request_completed(request))
return;
i915_gem_request_retire(request);
tl->fence_context = fences++;
tl->common = timeline;
+ spin_lock_init(&tl->lock);
init_request_active(&tl->last_request, NULL);
INIT_LIST_HEAD(&tl->requests);
}
struct intel_timeline {
u64 fence_context;
u32 last_submitted_seqno;
- u32 last_pending_seqno;
+
+ spinlock_t lock;
/**
* List of breadcrumbs associated with GPU requests currently
unsigned int intel_kick_waiters(struct drm_i915_private *i915);
unsigned int intel_kick_signalers(struct drm_i915_private *i915);
-static inline bool intel_engine_is_active(struct intel_engine_cs *engine)
-{
- return i915_gem_active_isset(&engine->timeline->last_request);
-}
-
#endif /* _INTEL_RINGBUFFER_H_ */
OSDN Git Service
