* the batchbuffer in trusted mode, otherwise the ioctl is rejected.
*/
+struct eb_fence {
+ struct drm_syncobj *syncobj; /* Use with ptr_mask_bits() */
+ struct dma_fence *dma_fence;
+ u64 value;
+ struct dma_fence_chain *chain_fence;
+};
+
struct i915_execbuffer {
struct drm_i915_private *i915; /** i915 backpointer */
struct drm_file *file; /** per-file lookup tables and limits */
struct hlist_head *buckets; /** ht for relocation handles */
struct eb_vma_array *array;
- struct i915_eb_fence {
- struct drm_syncobj *syncobj; /* Use with ptr_mask_bits() */
- } *fences;
- u32 n_fences;
-
- u64 extension_flags; /** Available extensions parameters */
+ struct eb_fence *fences;
+ unsigned long num_fences;
};
static inline bool eb_use_cmdparser(const struct i915_execbuffer *eb)
}
static void
-__free_fence_array(struct i915_eb_fence *fences, unsigned int n)
+__free_fence_array(struct eb_fence *fences, unsigned int n)
{
- while (n--)
+ while (n--) {
drm_syncobj_put(ptr_mask_bits(fences[n].syncobj, 2));
+ dma_fence_put(fences[n].dma_fence);
+ kfree(fences[n].chain_fence);
+ }
kvfree(fences);
}
static int
-get_fence_array(struct drm_i915_gem_execbuffer2 *args,
- struct i915_execbuffer *eb)
+add_timeline_fence_array(struct i915_execbuffer *eb,
+ const struct drm_i915_gem_execbuffer_ext_timeline_fences *timeline_fences)
{
- const unsigned long nfences = args->num_cliprects;
- struct drm_i915_gem_exec_fence __user *user;
- struct i915_eb_fence *fences;
- unsigned long n;
- int err;
+ struct drm_i915_gem_exec_fence __user *user_fences;
+ u64 __user *user_values;
+ struct eb_fence *f;
+ u64 nfences;
+ int err = 0;
- if (!(args->flags & I915_EXEC_FENCE_ARRAY))
+ nfences = timeline_fences->fence_count;
+ if (!nfences)
return 0;
/* Check multiplication overflow for access_ok() and kvmalloc_array() */
BUILD_BUG_ON(sizeof(size_t) > sizeof(unsigned long));
if (nfences > min_t(unsigned long,
- ULONG_MAX / sizeof(*user),
- SIZE_MAX / sizeof(*fences)))
+ ULONG_MAX / sizeof(*user_fences),
+ SIZE_MAX / sizeof(*f)) - eb->num_fences)
return -EINVAL;
- user = u64_to_user_ptr(args->cliprects_ptr);
- if (!access_ok(user, nfences * sizeof(*user)))
+ user_fences = u64_to_user_ptr(timeline_fences->handles_ptr);
+ if (!access_ok(user_fences, nfences * sizeof(*user_fences)))
+ return -EFAULT;
+
+ user_values = u64_to_user_ptr(timeline_fences->values_ptr);
+ if (!access_ok(user_values, nfences * sizeof(*user_values)))
return -EFAULT;
- fences = kvmalloc_array(nfences, sizeof(*fences),
- __GFP_NOWARN | GFP_KERNEL);
- if (!fences)
+ f = krealloc(eb->fences,
+ (eb->num_fences + nfences) * sizeof(*f),
+ __GFP_NOWARN | GFP_KERNEL);
+ if (!f)
return -ENOMEM;
- for (n = 0; n < nfences; n++) {
- struct drm_i915_gem_exec_fence fence;
+ eb->fences = f;
+ f += eb->num_fences;
+
+ BUILD_BUG_ON(~(ARCH_KMALLOC_MINALIGN - 1) &
+ ~__I915_EXEC_FENCE_UNKNOWN_FLAGS);
+
+ while (nfences--) {
+ struct drm_i915_gem_exec_fence user_fence;
struct drm_syncobj *syncobj;
+ struct dma_fence *fence = NULL;
+ u64 point;
+
+ if (__copy_from_user(&user_fence,
+ user_fences++,
+ sizeof(user_fence)))
+ return -EFAULT;
+
+ if (user_fence.flags & __I915_EXEC_FENCE_UNKNOWN_FLAGS)
+ return -EINVAL;
+
+ if (__get_user(point, user_values++))
+ return -EFAULT;
+
+ syncobj = drm_syncobj_find(eb->file, user_fence.handle);
+ if (!syncobj) {
+ DRM_DEBUG("Invalid syncobj handle provided\n");
+ return -ENOENT;
+ }
+
+ fence = drm_syncobj_fence_get(syncobj);
- if (__copy_from_user(&fence, user++, sizeof(fence))) {
- err = -EFAULT;
- goto err;
+ if (!fence && user_fence.flags &&
+ !(user_fence.flags & I915_EXEC_FENCE_SIGNAL)) {
+ DRM_DEBUG("Syncobj handle has no fence\n");
+ drm_syncobj_put(syncobj);
+ return -EINVAL;
}
- if (fence.flags & __I915_EXEC_FENCE_UNKNOWN_FLAGS) {
- err = -EINVAL;
- goto err;
+ if (fence)
+ err = dma_fence_chain_find_seqno(&fence, point);
+
+ if (err && !(user_fence.flags & I915_EXEC_FENCE_SIGNAL)) {
+ DRM_DEBUG("Syncobj handle missing requested point %llu\n", point);
+ drm_syncobj_put(syncobj);
+ return err;
+ }
+
+ /*
+ * A point might have been signaled already and
+ * garbage collected from the timeline. In this case
+ * just ignore the point and carry on.
+ */
+ if (!fence && !(user_fence.flags & I915_EXEC_FENCE_SIGNAL)) {
+ drm_syncobj_put(syncobj);
+ continue;
+ }
+
+ /*
+ * For timeline syncobjs we need to preallocate chains for
+ * later signaling.
+ */
+ if (point != 0 && user_fence.flags & I915_EXEC_FENCE_SIGNAL) {
+ /*
+ * Waiting and signaling the same point (when point !=
+ * 0) would break the timeline.
+ */
+ if (user_fence.flags & I915_EXEC_FENCE_WAIT) {
+ DRM_DEBUG("Trying to wait & signal the same timeline point.\n");
+ dma_fence_put(fence);
+ drm_syncobj_put(syncobj);
+ return -EINVAL;
+ }
+
+ f->chain_fence =
+ kmalloc(sizeof(*f->chain_fence),
+ GFP_KERNEL);
+ if (!f->chain_fence) {
+ drm_syncobj_put(syncobj);
+ dma_fence_put(fence);
+ return -ENOMEM;
+ }
+ } else {
+ f->chain_fence = NULL;
}
- syncobj = drm_syncobj_find(eb->file, fence.handle);
+ f->syncobj = ptr_pack_bits(syncobj, user_fence.flags, 2);
+ f->dma_fence = fence;
+ f->value = point;
+ f++;
+ eb->num_fences++;
+ }
+
+ return 0;
+}
+
+static int add_fence_array(struct i915_execbuffer *eb)
+{
+ struct drm_i915_gem_execbuffer2 *args = eb->args;
+ struct drm_i915_gem_exec_fence __user *user;
+ unsigned long num_fences = args->num_cliprects;
+ struct eb_fence *f;
+
+ if (!(args->flags & I915_EXEC_FENCE_ARRAY))
+ return 0;
+
+ if (!num_fences)
+ return 0;
+
+ /* Check multiplication overflow for access_ok() and kvmalloc_array() */
+ BUILD_BUG_ON(sizeof(size_t) > sizeof(unsigned long));
+ if (num_fences > min_t(unsigned long,
+ ULONG_MAX / sizeof(*user),
+ SIZE_MAX / sizeof(*f) - eb->num_fences))
+ return -EINVAL;
+
+ user = u64_to_user_ptr(args->cliprects_ptr);
+ if (!access_ok(user, num_fences * sizeof(*user)))
+ return -EFAULT;
+
+ f = krealloc(eb->fences,
+ (eb->num_fences + num_fences) * sizeof(*f),
+ __GFP_NOWARN | GFP_KERNEL);
+ if (!f)
+ return -ENOMEM;
+
+ eb->fences = f;
+ f += eb->num_fences;
+ while (num_fences--) {
+ struct drm_i915_gem_exec_fence user_fence;
+ struct drm_syncobj *syncobj;
+ struct dma_fence *fence = NULL;
+
+ if (__copy_from_user(&user_fence, user++, sizeof(user_fence)))
+ return -EFAULT;
+
+ if (user_fence.flags & __I915_EXEC_FENCE_UNKNOWN_FLAGS)
+ return -EINVAL;
+
+ syncobj = drm_syncobj_find(eb->file, user_fence.handle);
if (!syncobj) {
DRM_DEBUG("Invalid syncobj handle provided\n");
- err = -ENOENT;
- goto err;
+ return -ENOENT;
+ }
+
+ if (user_fence.flags & I915_EXEC_FENCE_WAIT) {
+ fence = drm_syncobj_fence_get(syncobj);
+ if (!fence) {
+ DRM_DEBUG("Syncobj handle has no fence\n");
+ drm_syncobj_put(syncobj);
+ return -EINVAL;
+ }
}
BUILD_BUG_ON(~(ARCH_KMALLOC_MINALIGN - 1) &
~__I915_EXEC_FENCE_UNKNOWN_FLAGS);
- fences[n].syncobj = ptr_pack_bits(syncobj, fence.flags, 2);
+ f->syncobj = ptr_pack_bits(syncobj, user_fence.flags, 2);
+ f->dma_fence = fence;
+ f->value = 0;
+ f->chain_fence = NULL;
+ f++;
+ eb->num_fences++;
}
- eb->fences = fences;
- eb->n_fences = nfences;
-
return 0;
+}
-err:
- __free_fence_array(fences, n);
- return err;
+static void put_fence_array(struct eb_fence *fences, int num_fences)
+{
+ if (fences)
+ __free_fence_array(fences, num_fences);
}
static int
unsigned int n;
int err;
- for (n = 0; n < eb->n_fences; n++) {
+ for (n = 0; n < eb->num_fences; n++) {
struct drm_syncobj *syncobj;
- struct dma_fence *fence;
unsigned int flags;
syncobj = ptr_unpack_bits(eb->fences[n].syncobj, &flags, 2);
- if (!(flags & I915_EXEC_FENCE_WAIT))
- continue;
- fence = drm_syncobj_fence_get(syncobj);
- if (!fence)
- return -EINVAL;
+ if (!eb->fences[n].dma_fence)
+ continue;
- err = i915_request_await_dma_fence(eb->request, fence);
- dma_fence_put(fence);
+ err = i915_request_await_dma_fence(eb->request,
+ eb->fences[n].dma_fence);
if (err < 0)
return err;
}
return 0;
}
-static void
-signal_fence_array(struct i915_execbuffer *eb)
+static void signal_fence_array(const struct i915_execbuffer *eb)
{
struct dma_fence * const fence = &eb->request->fence;
unsigned int n;
- for (n = 0; n < eb->n_fences; n++) {
+ for (n = 0; n < eb->num_fences; n++) {
struct drm_syncobj *syncobj;
unsigned int flags;
if (!(flags & I915_EXEC_FENCE_SIGNAL))
continue;
- drm_syncobj_replace_fence(syncobj, fence);
+ if (eb->fences[n].chain_fence) {
+ drm_syncobj_add_point(syncobj,
+ eb->fences[n].chain_fence,
+ fence,
+ eb->fences[n].value);
+ /*
+ * The chain's ownership is transferred to the
+ * timeline.
+ */
+ eb->fences[n].chain_fence = NULL;
+ } else {
+ drm_syncobj_replace_fence(syncobj, fence);
+ }
}
}
+static int
+parse_timeline_fences(struct i915_user_extension __user *ext, void *data)
+{
+ struct i915_execbuffer *eb = data;
+ struct drm_i915_gem_execbuffer_ext_timeline_fences timeline_fences;
+
+ if (copy_from_user(&timeline_fences, ext, sizeof(timeline_fences)))
+ return -EFAULT;
+
+ return add_timeline_fence_array(eb, &timeline_fences);
+}
+
static void retire_requests(struct intel_timeline *tl, struct i915_request *end)
{
struct i915_request *rq, *rn;
}
static const i915_user_extension_fn execbuf_extensions[] = {
+ [DRM_I915_GEM_EXECBUFFER_EXT_TIMELINE_FENCES] = parse_timeline_fences,
};
static int
parse_execbuf2_extensions(struct drm_i915_gem_execbuffer2 *args,
struct i915_execbuffer *eb)
{
- eb->extension_flags = 0;
-
if (!(args->flags & I915_EXEC_USE_EXTENSIONS))
return 0;
eb.trampoline = NULL;
eb.fences = NULL;
- eb.n_fences = 0;
+ eb.num_fences = 0;
eb.batch_flags = 0;
if (args->flags & I915_EXEC_SECURE) {
if (args->flags & I915_EXEC_IS_PINNED)
eb.batch_flags |= I915_DISPATCH_PINNED;
+ err = parse_execbuf2_extensions(args, &eb);
+ if (err)
+ goto err_ext;
+
+ err = add_fence_array(&eb);
+ if (err)
+ goto err_ext;
+
#define IN_FENCES (I915_EXEC_FENCE_IN | I915_EXEC_FENCE_SUBMIT)
if (args->flags & IN_FENCES) {
if ((args->flags & IN_FENCES) == IN_FENCES)
return -EINVAL;
in_fence = sync_file_get_fence(lower_32_bits(args->rsvd2));
- if (!in_fence)
- return -EINVAL;
+ if (!in_fence) {
+ err = -EINVAL;
+ goto err_ext;
+ }
}
#undef IN_FENCES
}
}
- err = parse_execbuf2_extensions(args, &eb);
- if (err)
- goto err_out_fence;
-
- err = get_fence_array(args, &eb);
- if (err)
- goto err_arr_fence;
-
err = eb_create(&eb);
if (err)
- goto err_arr_fence;
+ goto err_out_fence;
GEM_BUG_ON(!eb.lut_size);
goto err_request;
}
- if (eb.n_fences) {
+ if (eb.fences) {
err = await_fence_array(&eb);
if (err)
goto err_request;
i915_request_get(eb.request);
eb_request_add(&eb);
- if (eb.n_fences)
+ if (eb.fences)
signal_fence_array(&eb);
if (out_fence) {
i915_gem_context_put(eb.gem_context);
err_destroy:
eb_destroy(&eb);
-err_arr_fence:
- __free_fence_array(eb.fences, eb.n_fences);
err_out_fence:
if (out_fence_fd != -1)
put_unused_fd(out_fence_fd);
err_in_fence:
dma_fence_put(in_fence);
+err_ext:
+ put_fence_array(eb.fences, eb.num_fences);
return err;
}
projects / platform / kernel / linux-rpi.git / commitdiff