drm/bridge: tpd12s015: Drop buggy __exit annotation for remove function
[platform/kernel/linux-starfive.git] / drivers / gpu / drm / drm_syncobj.c
1 /*
2  * Copyright 2017 Red Hat
3  * Parts ported from amdgpu (fence wait code).
4  * Copyright 2016 Advanced Micro Devices, Inc.
5  *
6  * Permission is hereby granted, free of charge, to any person obtaining a
7  * copy of this software and associated documentation files (the "Software"),
8  * to deal in the Software without restriction, including without limitation
9  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
10  * and/or sell copies of the Software, and to permit persons to whom the
11  * Software is furnished to do so, subject to the following conditions:
12  *
13  * The above copyright notice and this permission notice (including the next
14  * paragraph) shall be included in all copies or substantial portions of the
15  * Software.
16  *
17  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
18  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
19  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
20  * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
21  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
22  * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
23  * IN THE SOFTWARE.
24  *
25  * Authors:
26  *
27  */
28
29 /**
30  * DOC: Overview
31  *
32  * DRM synchronisation objects (syncobj, see struct &drm_syncobj) provide a
33  * container for a synchronization primitive which can be used by userspace
34  * to explicitly synchronize GPU commands, can be shared between userspace
35  * processes, and can be shared between different DRM drivers.
36  * Their primary use-case is to implement Vulkan fences and semaphores.
37  * The syncobj userspace API provides ioctls for several operations:
38  *
39  *  - Creation and destruction of syncobjs
40  *  - Import and export of syncobjs to/from a syncobj file descriptor
41  *  - Import and export a syncobj's underlying fence to/from a sync file
42  *  - Reset a syncobj (set its fence to NULL)
43  *  - Signal a syncobj (set a trivially signaled fence)
44  *  - Wait for a syncobj's fence to appear and be signaled
45  *
46  * The syncobj userspace API also provides operations to manipulate a syncobj
47  * in terms of a timeline of struct &dma_fence_chain rather than a single
48  * struct &dma_fence, through the following operations:
49  *
50  *   - Signal a given point on the timeline
51  *   - Wait for a given point to appear and/or be signaled
52  *   - Import and export from/to a given point of a timeline
53  *
54  * At it's core, a syncobj is simply a wrapper around a pointer to a struct
55  * &dma_fence which may be NULL.
56  * When a syncobj is first created, its pointer is either NULL or a pointer
57  * to an already signaled fence depending on whether the
58  * &DRM_SYNCOBJ_CREATE_SIGNALED flag is passed to
59  * &DRM_IOCTL_SYNCOBJ_CREATE.
60  *
61  * If the syncobj is considered as a binary (its state is either signaled or
62  * unsignaled) primitive, when GPU work is enqueued in a DRM driver to signal
63  * the syncobj, the syncobj's fence is replaced with a fence which will be
64  * signaled by the completion of that work.
65  * If the syncobj is considered as a timeline primitive, when GPU work is
66  * enqueued in a DRM driver to signal the a given point of the syncobj, a new
67  * struct &dma_fence_chain pointing to the DRM driver's fence and also
68  * pointing to the previous fence that was in the syncobj. The new struct
69  * &dma_fence_chain fence replace the syncobj's fence and will be signaled by
70  * completion of the DRM driver's work and also any work associated with the
71  * fence previously in the syncobj.
72  *
73  * When GPU work which waits on a syncobj is enqueued in a DRM driver, at the
74  * time the work is enqueued, it waits on the syncobj's fence before
75  * submitting the work to hardware. That fence is either :
76  *
77  *    - The syncobj's current fence if the syncobj is considered as a binary
78  *      primitive.
79  *    - The struct &dma_fence associated with a given point if the syncobj is
80  *      considered as a timeline primitive.
81  *
82  * If the syncobj's fence is NULL or not present in the syncobj's timeline,
83  * the enqueue operation is expected to fail.
84  *
85  * With binary syncobj, all manipulation of the syncobjs's fence happens in
86  * terms of the current fence at the time the ioctl is called by userspace
87  * regardless of whether that operation is an immediate host-side operation
88  * (signal or reset) or or an operation which is enqueued in some driver
89  * queue. &DRM_IOCTL_SYNCOBJ_RESET and &DRM_IOCTL_SYNCOBJ_SIGNAL can be used
90  * to manipulate a syncobj from the host by resetting its pointer to NULL or
91  * setting its pointer to a fence which is already signaled.
92  *
93  * With a timeline syncobj, all manipulation of the synobj's fence happens in
94  * terms of a u64 value referring to point in the timeline. See
95  * dma_fence_chain_find_seqno() to see how a given point is found in the
96  * timeline.
97  *
98  * Note that applications should be careful to always use timeline set of
99  * ioctl() when dealing with syncobj considered as timeline. Using a binary
100  * set of ioctl() with a syncobj considered as timeline could result incorrect
101  * synchronization. The use of binary syncobj is supported through the
102  * timeline set of ioctl() by using a point value of 0, this will reproduce
103  * the behavior of the binary set of ioctl() (for example replace the
104  * syncobj's fence when signaling).
105  *
106  *
107  * Host-side wait on syncobjs
108  * --------------------------
109  *
110  * &DRM_IOCTL_SYNCOBJ_WAIT takes an array of syncobj handles and does a
111  * host-side wait on all of the syncobj fences simultaneously.
112  * If &DRM_SYNCOBJ_WAIT_FLAGS_WAIT_ALL is set, the wait ioctl will wait on
113  * all of the syncobj fences to be signaled before it returns.
114  * Otherwise, it returns once at least one syncobj fence has been signaled
115  * and the index of a signaled fence is written back to the client.
116  *
117  * Unlike the enqueued GPU work dependencies which fail if they see a NULL
118  * fence in a syncobj, if &DRM_SYNCOBJ_WAIT_FLAGS_WAIT_FOR_SUBMIT is set,
119  * the host-side wait will first wait for the syncobj to receive a non-NULL
120  * fence and then wait on that fence.
121  * If &DRM_SYNCOBJ_WAIT_FLAGS_WAIT_FOR_SUBMIT is not set and any one of the
122  * syncobjs in the array has a NULL fence, -EINVAL will be returned.
123  * Assuming the syncobj starts off with a NULL fence, this allows a client
124  * to do a host wait in one thread (or process) which waits on GPU work
125  * submitted in another thread (or process) without having to manually
126  * synchronize between the two.
127  * This requirement is inherited from the Vulkan fence API.
128  *
129  * Similarly, &DRM_IOCTL_SYNCOBJ_TIMELINE_WAIT takes an array of syncobj
130  * handles as well as an array of u64 points and does a host-side wait on all
131  * of syncobj fences at the given points simultaneously.
132  *
133  * &DRM_IOCTL_SYNCOBJ_TIMELINE_WAIT also adds the ability to wait for a given
134  * fence to materialize on the timeline without waiting for the fence to be
135  * signaled by using the &DRM_SYNCOBJ_WAIT_FLAGS_WAIT_AVAILABLE flag. This
136  * requirement is inherited from the wait-before-signal behavior required by
137  * the Vulkan timeline semaphore API.
138  *
139  * Alternatively, &DRM_IOCTL_SYNCOBJ_EVENTFD can be used to wait without
140  * blocking: an eventfd will be signaled when the syncobj is. This is useful to
141  * integrate the wait in an event loop.
142  *
143  *
144  * Import/export of syncobjs
145  * -------------------------
146  *
147  * &DRM_IOCTL_SYNCOBJ_FD_TO_HANDLE and &DRM_IOCTL_SYNCOBJ_HANDLE_TO_FD
148  * provide two mechanisms for import/export of syncobjs.
149  *
150  * The first lets the client import or export an entire syncobj to a file
151  * descriptor.
152  * These fd's are opaque and have no other use case, except passing the
153  * syncobj between processes.
154  * All exported file descriptors and any syncobj handles created as a
155  * result of importing those file descriptors own a reference to the
156  * same underlying struct &drm_syncobj and the syncobj can be used
157  * persistently across all the processes with which it is shared.
158  * The syncobj is freed only once the last reference is dropped.
159  * Unlike dma-buf, importing a syncobj creates a new handle (with its own
160  * reference) for every import instead of de-duplicating.
161  * The primary use-case of this persistent import/export is for shared
162  * Vulkan fences and semaphores.
163  *
164  * The second import/export mechanism, which is indicated by
165  * &DRM_SYNCOBJ_FD_TO_HANDLE_FLAGS_IMPORT_SYNC_FILE or
166  * &DRM_SYNCOBJ_HANDLE_TO_FD_FLAGS_EXPORT_SYNC_FILE lets the client
167  * import/export the syncobj's current fence from/to a &sync_file.
168  * When a syncobj is exported to a sync file, that sync file wraps the
169  * sycnobj's fence at the time of export and any later signal or reset
170  * operations on the syncobj will not affect the exported sync file.
171  * When a sync file is imported into a syncobj, the syncobj's fence is set
172  * to the fence wrapped by that sync file.
173  * Because sync files are immutable, resetting or signaling the syncobj
174  * will not affect any sync files whose fences have been imported into the
175  * syncobj.
176  *
177  *
178  * Import/export of timeline points in timeline syncobjs
179  * -----------------------------------------------------
180  *
181  * &DRM_IOCTL_SYNCOBJ_TRANSFER provides a mechanism to transfer a struct
182  * &dma_fence_chain of a syncobj at a given u64 point to another u64 point
183  * into another syncobj.
184  *
185  * Note that if you want to transfer a struct &dma_fence_chain from a given
186  * point on a timeline syncobj from/into a binary syncobj, you can use the
187  * point 0 to mean take/replace the fence in the syncobj.
188  */
189
190 #include <linux/anon_inodes.h>
191 #include <linux/dma-fence-unwrap.h>
192 #include <linux/eventfd.h>
193 #include <linux/file.h>
194 #include <linux/fs.h>
195 #include <linux/sched/signal.h>
196 #include <linux/sync_file.h>
197 #include <linux/uaccess.h>
198
199 #include <drm/drm.h>
200 #include <drm/drm_drv.h>
201 #include <drm/drm_file.h>
202 #include <drm/drm_gem.h>
203 #include <drm/drm_print.h>
204 #include <drm/drm_syncobj.h>
205 #include <drm/drm_utils.h>
206
207 #include "drm_internal.h"
208
209 struct syncobj_wait_entry {
210         struct list_head node;
211         struct task_struct *task;
212         struct dma_fence *fence;
213         struct dma_fence_cb fence_cb;
214         u64    point;
215 };
216
217 static void syncobj_wait_syncobj_func(struct drm_syncobj *syncobj,
218                                       struct syncobj_wait_entry *wait);
219
220 struct syncobj_eventfd_entry {
221         struct list_head node;
222         struct dma_fence *fence;
223         struct dma_fence_cb fence_cb;
224         struct drm_syncobj *syncobj;
225         struct eventfd_ctx *ev_fd_ctx;
226         u64 point;
227         u32 flags;
228 };
229
230 static void
231 syncobj_eventfd_entry_func(struct drm_syncobj *syncobj,
232                            struct syncobj_eventfd_entry *entry);
233
234 /**
235  * drm_syncobj_find - lookup and reference a sync object.
236  * @file_private: drm file private pointer
237  * @handle: sync object handle to lookup.
238  *
239  * Returns a reference to the syncobj pointed to by handle or NULL. The
240  * reference must be released by calling drm_syncobj_put().
241  */
242 struct drm_syncobj *drm_syncobj_find(struct drm_file *file_private,
243                                      u32 handle)
244 {
245         struct drm_syncobj *syncobj;
246
247         spin_lock(&file_private->syncobj_table_lock);
248
249         /* Check if we currently have a reference on the object */
250         syncobj = idr_find(&file_private->syncobj_idr, handle);
251         if (syncobj)
252                 drm_syncobj_get(syncobj);
253
254         spin_unlock(&file_private->syncobj_table_lock);
255
256         return syncobj;
257 }
258 EXPORT_SYMBOL(drm_syncobj_find);
259
260 static void drm_syncobj_fence_add_wait(struct drm_syncobj *syncobj,
261                                        struct syncobj_wait_entry *wait)
262 {
263         struct dma_fence *fence;
264
265         if (wait->fence)
266                 return;
267
268         spin_lock(&syncobj->lock);
269         /* We've already tried once to get a fence and failed.  Now that we
270          * have the lock, try one more time just to be sure we don't add a
271          * callback when a fence has already been set.
272          */
273         fence = dma_fence_get(rcu_dereference_protected(syncobj->fence, 1));
274         if (!fence || dma_fence_chain_find_seqno(&fence, wait->point)) {
275                 dma_fence_put(fence);
276                 list_add_tail(&wait->node, &syncobj->cb_list);
277         } else if (!fence) {
278                 wait->fence = dma_fence_get_stub();
279         } else {
280                 wait->fence = fence;
281         }
282         spin_unlock(&syncobj->lock);
283 }
284
285 static void drm_syncobj_remove_wait(struct drm_syncobj *syncobj,
286                                     struct syncobj_wait_entry *wait)
287 {
288         if (!wait->node.next)
289                 return;
290
291         spin_lock(&syncobj->lock);
292         list_del_init(&wait->node);
293         spin_unlock(&syncobj->lock);
294 }
295
296 static void
297 syncobj_eventfd_entry_free(struct syncobj_eventfd_entry *entry)
298 {
299         eventfd_ctx_put(entry->ev_fd_ctx);
300         dma_fence_put(entry->fence);
301         /* This happens either inside the syncobj lock, or after the node has
302          * already been removed from the list.
303          */
304         list_del(&entry->node);
305         kfree(entry);
306 }
307
308 static void
309 drm_syncobj_add_eventfd(struct drm_syncobj *syncobj,
310                         struct syncobj_eventfd_entry *entry)
311 {
312         spin_lock(&syncobj->lock);
313         list_add_tail(&entry->node, &syncobj->ev_fd_list);
314         syncobj_eventfd_entry_func(syncobj, entry);
315         spin_unlock(&syncobj->lock);
316 }
317
318 /**
319  * drm_syncobj_add_point - add new timeline point to the syncobj
320  * @syncobj: sync object to add timeline point do
321  * @chain: chain node to use to add the point
322  * @fence: fence to encapsulate in the chain node
323  * @point: sequence number to use for the point
324  *
325  * Add the chain node as new timeline point to the syncobj.
326  */
327 void drm_syncobj_add_point(struct drm_syncobj *syncobj,
328                            struct dma_fence_chain *chain,
329                            struct dma_fence *fence,
330                            uint64_t point)
331 {
332         struct syncobj_wait_entry *wait_cur, *wait_tmp;
333         struct syncobj_eventfd_entry *ev_fd_cur, *ev_fd_tmp;
334         struct dma_fence *prev;
335
336         dma_fence_get(fence);
337
338         spin_lock(&syncobj->lock);
339
340         prev = drm_syncobj_fence_get(syncobj);
341         /* You are adding an unorder point to timeline, which could cause payload returned from query_ioctl is 0! */
342         if (prev && prev->seqno >= point)
343                 DRM_DEBUG("You are adding an unorder point to timeline!\n");
344         dma_fence_chain_init(chain, prev, fence, point);
345         rcu_assign_pointer(syncobj->fence, &chain->base);
346
347         list_for_each_entry_safe(wait_cur, wait_tmp, &syncobj->cb_list, node)
348                 syncobj_wait_syncobj_func(syncobj, wait_cur);
349         list_for_each_entry_safe(ev_fd_cur, ev_fd_tmp, &syncobj->ev_fd_list, node)
350                 syncobj_eventfd_entry_func(syncobj, ev_fd_cur);
351         spin_unlock(&syncobj->lock);
352
353         /* Walk the chain once to trigger garbage collection */
354         dma_fence_chain_for_each(fence, prev);
355         dma_fence_put(prev);
356 }
357 EXPORT_SYMBOL(drm_syncobj_add_point);
358
359 /**
360  * drm_syncobj_replace_fence - replace fence in a sync object.
361  * @syncobj: Sync object to replace fence in
362  * @fence: fence to install in sync file.
363  *
364  * This replaces the fence on a sync object.
365  */
366 void drm_syncobj_replace_fence(struct drm_syncobj *syncobj,
367                                struct dma_fence *fence)
368 {
369         struct dma_fence *old_fence;
370         struct syncobj_wait_entry *wait_cur, *wait_tmp;
371         struct syncobj_eventfd_entry *ev_fd_cur, *ev_fd_tmp;
372
373         if (fence)
374                 dma_fence_get(fence);
375
376         spin_lock(&syncobj->lock);
377
378         old_fence = rcu_dereference_protected(syncobj->fence,
379                                               lockdep_is_held(&syncobj->lock));
380         rcu_assign_pointer(syncobj->fence, fence);
381
382         if (fence != old_fence) {
383                 list_for_each_entry_safe(wait_cur, wait_tmp, &syncobj->cb_list, node)
384                         syncobj_wait_syncobj_func(syncobj, wait_cur);
385                 list_for_each_entry_safe(ev_fd_cur, ev_fd_tmp, &syncobj->ev_fd_list, node)
386                         syncobj_eventfd_entry_func(syncobj, ev_fd_cur);
387         }
388
389         spin_unlock(&syncobj->lock);
390
391         dma_fence_put(old_fence);
392 }
393 EXPORT_SYMBOL(drm_syncobj_replace_fence);
394
395 /**
396  * drm_syncobj_assign_null_handle - assign a stub fence to the sync object
397  * @syncobj: sync object to assign the fence on
398  *
399  * Assign a already signaled stub fence to the sync object.
400  */
401 static int drm_syncobj_assign_null_handle(struct drm_syncobj *syncobj)
402 {
403         struct dma_fence *fence = dma_fence_allocate_private_stub(ktime_get());
404
405         if (!fence)
406                 return -ENOMEM;
407
408         drm_syncobj_replace_fence(syncobj, fence);
409         dma_fence_put(fence);
410         return 0;
411 }
412
413 /* 5s default for wait submission */
414 #define DRM_SYNCOBJ_WAIT_FOR_SUBMIT_TIMEOUT 5000000000ULL
415 /**
416  * drm_syncobj_find_fence - lookup and reference the fence in a sync object
417  * @file_private: drm file private pointer
418  * @handle: sync object handle to lookup.
419  * @point: timeline point
420  * @flags: DRM_SYNCOBJ_WAIT_FLAGS_WAIT_FOR_SUBMIT or not
421  * @fence: out parameter for the fence
422  *
423  * This is just a convenience function that combines drm_syncobj_find() and
424  * drm_syncobj_fence_get().
425  *
426  * Returns 0 on success or a negative error value on failure. On success @fence
427  * contains a reference to the fence, which must be released by calling
428  * dma_fence_put().
429  */
430 int drm_syncobj_find_fence(struct drm_file *file_private,
431                            u32 handle, u64 point, u64 flags,
432                            struct dma_fence **fence)
433 {
434         struct drm_syncobj *syncobj = drm_syncobj_find(file_private, handle);
435         struct syncobj_wait_entry wait;
436         u64 timeout = nsecs_to_jiffies64(DRM_SYNCOBJ_WAIT_FOR_SUBMIT_TIMEOUT);
437         int ret;
438
439         if (!syncobj)
440                 return -ENOENT;
441
442         /* Waiting for userspace with locks help is illegal cause that can
443          * trivial deadlock with page faults for example. Make lockdep complain
444          * about it early on.
445          */
446         if (flags & DRM_SYNCOBJ_WAIT_FLAGS_WAIT_FOR_SUBMIT) {
447                 might_sleep();
448                 lockdep_assert_none_held_once();
449         }
450
451         *fence = drm_syncobj_fence_get(syncobj);
452
453         if (*fence) {
454                 ret = dma_fence_chain_find_seqno(fence, point);
455                 if (!ret) {
456                         /* If the requested seqno is already signaled
457                          * drm_syncobj_find_fence may return a NULL
458                          * fence. To make sure the recipient gets
459                          * signalled, use a new fence instead.
460                          */
461                         if (!*fence)
462                                 *fence = dma_fence_get_stub();
463
464                         goto out;
465                 }
466                 dma_fence_put(*fence);
467         } else {
468                 ret = -EINVAL;
469         }
470
471         if (!(flags & DRM_SYNCOBJ_WAIT_FLAGS_WAIT_FOR_SUBMIT))
472                 goto out;
473
474         memset(&wait, 0, sizeof(wait));
475         wait.task = current;
476         wait.point = point;
477         drm_syncobj_fence_add_wait(syncobj, &wait);
478
479         do {
480                 set_current_state(TASK_INTERRUPTIBLE);
481                 if (wait.fence) {
482                         ret = 0;
483                         break;
484                 }
485                 if (timeout == 0) {
486                         ret = -ETIME;
487                         break;
488                 }
489
490                 if (signal_pending(current)) {
491                         ret = -ERESTARTSYS;
492                         break;
493                 }
494
495                 timeout = schedule_timeout(timeout);
496         } while (1);
497
498         __set_current_state(TASK_RUNNING);
499         *fence = wait.fence;
500
501         if (wait.node.next)
502                 drm_syncobj_remove_wait(syncobj, &wait);
503
504 out:
505         drm_syncobj_put(syncobj);
506
507         return ret;
508 }
509 EXPORT_SYMBOL(drm_syncobj_find_fence);
510
511 /**
512  * drm_syncobj_free - free a sync object.
513  * @kref: kref to free.
514  *
515  * Only to be called from kref_put in drm_syncobj_put.
516  */
517 void drm_syncobj_free(struct kref *kref)
518 {
519         struct drm_syncobj *syncobj = container_of(kref,
520                                                    struct drm_syncobj,
521                                                    refcount);
522         struct syncobj_eventfd_entry *ev_fd_cur, *ev_fd_tmp;
523
524         drm_syncobj_replace_fence(syncobj, NULL);
525
526         list_for_each_entry_safe(ev_fd_cur, ev_fd_tmp, &syncobj->ev_fd_list, node)
527                 syncobj_eventfd_entry_free(ev_fd_cur);
528
529         kfree(syncobj);
530 }
531 EXPORT_SYMBOL(drm_syncobj_free);
532
533 /**
534  * drm_syncobj_create - create a new syncobj
535  * @out_syncobj: returned syncobj
536  * @flags: DRM_SYNCOBJ_* flags
537  * @fence: if non-NULL, the syncobj will represent this fence
538  *
539  * This is the first function to create a sync object. After creating, drivers
540  * probably want to make it available to userspace, either through
541  * drm_syncobj_get_handle() or drm_syncobj_get_fd().
542  *
543  * Returns 0 on success or a negative error value on failure.
544  */
545 int drm_syncobj_create(struct drm_syncobj **out_syncobj, uint32_t flags,
546                        struct dma_fence *fence)
547 {
548         int ret;
549         struct drm_syncobj *syncobj;
550
551         syncobj = kzalloc(sizeof(struct drm_syncobj), GFP_KERNEL);
552         if (!syncobj)
553                 return -ENOMEM;
554
555         kref_init(&syncobj->refcount);
556         INIT_LIST_HEAD(&syncobj->cb_list);
557         INIT_LIST_HEAD(&syncobj->ev_fd_list);
558         spin_lock_init(&syncobj->lock);
559
560         if (flags & DRM_SYNCOBJ_CREATE_SIGNALED) {
561                 ret = drm_syncobj_assign_null_handle(syncobj);
562                 if (ret < 0) {
563                         drm_syncobj_put(syncobj);
564                         return ret;
565                 }
566         }
567
568         if (fence)
569                 drm_syncobj_replace_fence(syncobj, fence);
570
571         *out_syncobj = syncobj;
572         return 0;
573 }
574 EXPORT_SYMBOL(drm_syncobj_create);
575
576 /**
577  * drm_syncobj_get_handle - get a handle from a syncobj
578  * @file_private: drm file private pointer
579  * @syncobj: Sync object to export
580  * @handle: out parameter with the new handle
581  *
582  * Exports a sync object created with drm_syncobj_create() as a handle on
583  * @file_private to userspace.
584  *
585  * Returns 0 on success or a negative error value on failure.
586  */
587 int drm_syncobj_get_handle(struct drm_file *file_private,
588                            struct drm_syncobj *syncobj, u32 *handle)
589 {
590         int ret;
591
592         /* take a reference to put in the idr */
593         drm_syncobj_get(syncobj);
594
595         idr_preload(GFP_KERNEL);
596         spin_lock(&file_private->syncobj_table_lock);
597         ret = idr_alloc(&file_private->syncobj_idr, syncobj, 1, 0, GFP_NOWAIT);
598         spin_unlock(&file_private->syncobj_table_lock);
599
600         idr_preload_end();
601
602         if (ret < 0) {
603                 drm_syncobj_put(syncobj);
604                 return ret;
605         }
606
607         *handle = ret;
608         return 0;
609 }
610 EXPORT_SYMBOL(drm_syncobj_get_handle);
611
612 static int drm_syncobj_create_as_handle(struct drm_file *file_private,
613                                         u32 *handle, uint32_t flags)
614 {
615         int ret;
616         struct drm_syncobj *syncobj;
617
618         ret = drm_syncobj_create(&syncobj, flags, NULL);
619         if (ret)
620                 return ret;
621
622         ret = drm_syncobj_get_handle(file_private, syncobj, handle);
623         drm_syncobj_put(syncobj);
624         return ret;
625 }
626
627 static int drm_syncobj_destroy(struct drm_file *file_private,
628                                u32 handle)
629 {
630         struct drm_syncobj *syncobj;
631
632         spin_lock(&file_private->syncobj_table_lock);
633         syncobj = idr_remove(&file_private->syncobj_idr, handle);
634         spin_unlock(&file_private->syncobj_table_lock);
635
636         if (!syncobj)
637                 return -EINVAL;
638
639         drm_syncobj_put(syncobj);
640         return 0;
641 }
642
643 static int drm_syncobj_file_release(struct inode *inode, struct file *file)
644 {
645         struct drm_syncobj *syncobj = file->private_data;
646
647         drm_syncobj_put(syncobj);
648         return 0;
649 }
650
651 static const struct file_operations drm_syncobj_file_fops = {
652         .release = drm_syncobj_file_release,
653 };
654
655 /**
656  * drm_syncobj_get_fd - get a file descriptor from a syncobj
657  * @syncobj: Sync object to export
658  * @p_fd: out parameter with the new file descriptor
659  *
660  * Exports a sync object created with drm_syncobj_create() as a file descriptor.
661  *
662  * Returns 0 on success or a negative error value on failure.
663  */
664 int drm_syncobj_get_fd(struct drm_syncobj *syncobj, int *p_fd)
665 {
666         struct file *file;
667         int fd;
668
669         fd = get_unused_fd_flags(O_CLOEXEC);
670         if (fd < 0)
671                 return fd;
672
673         file = anon_inode_getfile("syncobj_file",
674                                   &drm_syncobj_file_fops,
675                                   syncobj, 0);
676         if (IS_ERR(file)) {
677                 put_unused_fd(fd);
678                 return PTR_ERR(file);
679         }
680
681         drm_syncobj_get(syncobj);
682         fd_install(fd, file);
683
684         *p_fd = fd;
685         return 0;
686 }
687 EXPORT_SYMBOL(drm_syncobj_get_fd);
688
689 static int drm_syncobj_handle_to_fd(struct drm_file *file_private,
690                                     u32 handle, int *p_fd)
691 {
692         struct drm_syncobj *syncobj = drm_syncobj_find(file_private, handle);
693         int ret;
694
695         if (!syncobj)
696                 return -EINVAL;
697
698         ret = drm_syncobj_get_fd(syncobj, p_fd);
699         drm_syncobj_put(syncobj);
700         return ret;
701 }
702
703 static int drm_syncobj_fd_to_handle(struct drm_file *file_private,
704                                     int fd, u32 *handle)
705 {
706         struct drm_syncobj *syncobj;
707         struct fd f = fdget(fd);
708         int ret;
709
710         if (!f.file)
711                 return -EINVAL;
712
713         if (f.file->f_op != &drm_syncobj_file_fops) {
714                 fdput(f);
715                 return -EINVAL;
716         }
717
718         /* take a reference to put in the idr */
719         syncobj = f.file->private_data;
720         drm_syncobj_get(syncobj);
721
722         idr_preload(GFP_KERNEL);
723         spin_lock(&file_private->syncobj_table_lock);
724         ret = idr_alloc(&file_private->syncobj_idr, syncobj, 1, 0, GFP_NOWAIT);
725         spin_unlock(&file_private->syncobj_table_lock);
726         idr_preload_end();
727
728         if (ret > 0) {
729                 *handle = ret;
730                 ret = 0;
731         } else
732                 drm_syncobj_put(syncobj);
733
734         fdput(f);
735         return ret;
736 }
737
738 static int drm_syncobj_import_sync_file_fence(struct drm_file *file_private,
739                                               int fd, int handle)
740 {
741         struct dma_fence *fence = sync_file_get_fence(fd);
742         struct drm_syncobj *syncobj;
743
744         if (!fence)
745                 return -EINVAL;
746
747         syncobj = drm_syncobj_find(file_private, handle);
748         if (!syncobj) {
749                 dma_fence_put(fence);
750                 return -ENOENT;
751         }
752
753         drm_syncobj_replace_fence(syncobj, fence);
754         dma_fence_put(fence);
755         drm_syncobj_put(syncobj);
756         return 0;
757 }
758
759 static int drm_syncobj_export_sync_file(struct drm_file *file_private,
760                                         int handle, int *p_fd)
761 {
762         int ret;
763         struct dma_fence *fence;
764         struct sync_file *sync_file;
765         int fd = get_unused_fd_flags(O_CLOEXEC);
766
767         if (fd < 0)
768                 return fd;
769
770         ret = drm_syncobj_find_fence(file_private, handle, 0, 0, &fence);
771         if (ret)
772                 goto err_put_fd;
773
774         sync_file = sync_file_create(fence);
775
776         dma_fence_put(fence);
777
778         if (!sync_file) {
779                 ret = -EINVAL;
780                 goto err_put_fd;
781         }
782
783         fd_install(fd, sync_file->file);
784
785         *p_fd = fd;
786         return 0;
787 err_put_fd:
788         put_unused_fd(fd);
789         return ret;
790 }
791 /**
792  * drm_syncobj_open - initializes syncobj file-private structures at devnode open time
793  * @file_private: drm file-private structure to set up
794  *
795  * Called at device open time, sets up the structure for handling refcounting
796  * of sync objects.
797  */
798 void
799 drm_syncobj_open(struct drm_file *file_private)
800 {
801         idr_init_base(&file_private->syncobj_idr, 1);
802         spin_lock_init(&file_private->syncobj_table_lock);
803 }
804
805 static int
806 drm_syncobj_release_handle(int id, void *ptr, void *data)
807 {
808         struct drm_syncobj *syncobj = ptr;
809
810         drm_syncobj_put(syncobj);
811         return 0;
812 }
813
814 /**
815  * drm_syncobj_release - release file-private sync object resources
816  * @file_private: drm file-private structure to clean up
817  *
818  * Called at close time when the filp is going away.
819  *
820  * Releases any remaining references on objects by this filp.
821  */
822 void
823 drm_syncobj_release(struct drm_file *file_private)
824 {
825         idr_for_each(&file_private->syncobj_idr,
826                      &drm_syncobj_release_handle, file_private);
827         idr_destroy(&file_private->syncobj_idr);
828 }
829
830 int
831 drm_syncobj_create_ioctl(struct drm_device *dev, void *data,
832                          struct drm_file *file_private)
833 {
834         struct drm_syncobj_create *args = data;
835
836         if (!drm_core_check_feature(dev, DRIVER_SYNCOBJ))
837                 return -EOPNOTSUPP;
838
839         /* no valid flags yet */
840         if (args->flags & ~DRM_SYNCOBJ_CREATE_SIGNALED)
841                 return -EINVAL;
842
843         return drm_syncobj_create_as_handle(file_private,
844                                             &args->handle, args->flags);
845 }
846
847 int
848 drm_syncobj_destroy_ioctl(struct drm_device *dev, void *data,
849                           struct drm_file *file_private)
850 {
851         struct drm_syncobj_destroy *args = data;
852
853         if (!drm_core_check_feature(dev, DRIVER_SYNCOBJ))
854                 return -EOPNOTSUPP;
855
856         /* make sure padding is empty */
857         if (args->pad)
858                 return -EINVAL;
859         return drm_syncobj_destroy(file_private, args->handle);
860 }
861
862 int
863 drm_syncobj_handle_to_fd_ioctl(struct drm_device *dev, void *data,
864                                    struct drm_file *file_private)
865 {
866         struct drm_syncobj_handle *args = data;
867
868         if (!drm_core_check_feature(dev, DRIVER_SYNCOBJ))
869                 return -EOPNOTSUPP;
870
871         if (args->pad)
872                 return -EINVAL;
873
874         if (args->flags != 0 &&
875             args->flags != DRM_SYNCOBJ_HANDLE_TO_FD_FLAGS_EXPORT_SYNC_FILE)
876                 return -EINVAL;
877
878         if (args->flags & DRM_SYNCOBJ_HANDLE_TO_FD_FLAGS_EXPORT_SYNC_FILE)
879                 return drm_syncobj_export_sync_file(file_private, args->handle,
880                                                     &args->fd);
881
882         return drm_syncobj_handle_to_fd(file_private, args->handle,
883                                         &args->fd);
884 }
885
886 int
887 drm_syncobj_fd_to_handle_ioctl(struct drm_device *dev, void *data,
888                                    struct drm_file *file_private)
889 {
890         struct drm_syncobj_handle *args = data;
891
892         if (!drm_core_check_feature(dev, DRIVER_SYNCOBJ))
893                 return -EOPNOTSUPP;
894
895         if (args->pad)
896                 return -EINVAL;
897
898         if (args->flags != 0 &&
899             args->flags != DRM_SYNCOBJ_FD_TO_HANDLE_FLAGS_IMPORT_SYNC_FILE)
900                 return -EINVAL;
901
902         if (args->flags & DRM_SYNCOBJ_FD_TO_HANDLE_FLAGS_IMPORT_SYNC_FILE)
903                 return drm_syncobj_import_sync_file_fence(file_private,
904                                                           args->fd,
905                                                           args->handle);
906
907         return drm_syncobj_fd_to_handle(file_private, args->fd,
908                                         &args->handle);
909 }
910
911 static int drm_syncobj_transfer_to_timeline(struct drm_file *file_private,
912                                             struct drm_syncobj_transfer *args)
913 {
914         struct drm_syncobj *timeline_syncobj = NULL;
915         struct dma_fence *fence, *tmp;
916         struct dma_fence_chain *chain;
917         int ret;
918
919         timeline_syncobj = drm_syncobj_find(file_private, args->dst_handle);
920         if (!timeline_syncobj) {
921                 return -ENOENT;
922         }
923         ret = drm_syncobj_find_fence(file_private, args->src_handle,
924                                      args->src_point, args->flags,
925                                      &tmp);
926         if (ret)
927                 goto err_put_timeline;
928
929         fence = dma_fence_unwrap_merge(tmp);
930         dma_fence_put(tmp);
931         if (!fence) {
932                 ret = -ENOMEM;
933                 goto err_put_timeline;
934         }
935
936         chain = dma_fence_chain_alloc();
937         if (!chain) {
938                 ret = -ENOMEM;
939                 goto err_free_fence;
940         }
941
942         drm_syncobj_add_point(timeline_syncobj, chain, fence, args->dst_point);
943 err_free_fence:
944         dma_fence_put(fence);
945 err_put_timeline:
946         drm_syncobj_put(timeline_syncobj);
947
948         return ret;
949 }
950
951 static int
952 drm_syncobj_transfer_to_binary(struct drm_file *file_private,
953                                struct drm_syncobj_transfer *args)
954 {
955         struct drm_syncobj *binary_syncobj = NULL;
956         struct dma_fence *fence;
957         int ret;
958
959         binary_syncobj = drm_syncobj_find(file_private, args->dst_handle);
960         if (!binary_syncobj)
961                 return -ENOENT;
962         ret = drm_syncobj_find_fence(file_private, args->src_handle,
963                                      args->src_point, args->flags, &fence);
964         if (ret)
965                 goto err;
966         drm_syncobj_replace_fence(binary_syncobj, fence);
967         dma_fence_put(fence);
968 err:
969         drm_syncobj_put(binary_syncobj);
970
971         return ret;
972 }
973 int
974 drm_syncobj_transfer_ioctl(struct drm_device *dev, void *data,
975                            struct drm_file *file_private)
976 {
977         struct drm_syncobj_transfer *args = data;
978         int ret;
979
980         if (!drm_core_check_feature(dev, DRIVER_SYNCOBJ_TIMELINE))
981                 return -EOPNOTSUPP;
982
983         if (args->pad)
984                 return -EINVAL;
985
986         if (args->dst_point)
987                 ret = drm_syncobj_transfer_to_timeline(file_private, args);
988         else
989                 ret = drm_syncobj_transfer_to_binary(file_private, args);
990
991         return ret;
992 }
993
994 static void syncobj_wait_fence_func(struct dma_fence *fence,
995                                     struct dma_fence_cb *cb)
996 {
997         struct syncobj_wait_entry *wait =
998                 container_of(cb, struct syncobj_wait_entry, fence_cb);
999
1000         wake_up_process(wait->task);
1001 }
1002
1003 static void syncobj_wait_syncobj_func(struct drm_syncobj *syncobj,
1004                                       struct syncobj_wait_entry *wait)
1005 {
1006         struct dma_fence *fence;
1007
1008         /* This happens inside the syncobj lock */
1009         fence = rcu_dereference_protected(syncobj->fence,
1010                                           lockdep_is_held(&syncobj->lock));
1011         dma_fence_get(fence);
1012         if (!fence || dma_fence_chain_find_seqno(&fence, wait->point)) {
1013                 dma_fence_put(fence);
1014                 return;
1015         } else if (!fence) {
1016                 wait->fence = dma_fence_get_stub();
1017         } else {
1018                 wait->fence = fence;
1019         }
1020
1021         wake_up_process(wait->task);
1022         list_del_init(&wait->node);
1023 }
1024
1025 static signed long drm_syncobj_array_wait_timeout(struct drm_syncobj **syncobjs,
1026                                                   void __user *user_points,
1027                                                   uint32_t count,
1028                                                   uint32_t flags,
1029                                                   signed long timeout,
1030                                                   uint32_t *idx)
1031 {
1032         struct syncobj_wait_entry *entries;
1033         struct dma_fence *fence;
1034         uint64_t *points;
1035         uint32_t signaled_count, i;
1036
1037         if (flags & DRM_SYNCOBJ_WAIT_FLAGS_WAIT_FOR_SUBMIT)
1038                 lockdep_assert_none_held_once();
1039
1040         points = kmalloc_array(count, sizeof(*points), GFP_KERNEL);
1041         if (points == NULL)
1042                 return -ENOMEM;
1043
1044         if (!user_points) {
1045                 memset(points, 0, count * sizeof(uint64_t));
1046
1047         } else if (copy_from_user(points, user_points,
1048                                   sizeof(uint64_t) * count)) {
1049                 timeout = -EFAULT;
1050                 goto err_free_points;
1051         }
1052
1053         entries = kcalloc(count, sizeof(*entries), GFP_KERNEL);
1054         if (!entries) {
1055                 timeout = -ENOMEM;
1056                 goto err_free_points;
1057         }
1058         /* Walk the list of sync objects and initialize entries.  We do
1059          * this up-front so that we can properly return -EINVAL if there is
1060          * a syncobj with a missing fence and then never have the chance of
1061          * returning -EINVAL again.
1062          */
1063         signaled_count = 0;
1064         for (i = 0; i < count; ++i) {
1065                 struct dma_fence *fence;
1066
1067                 entries[i].task = current;
1068                 entries[i].point = points[i];
1069                 fence = drm_syncobj_fence_get(syncobjs[i]);
1070                 if (!fence || dma_fence_chain_find_seqno(&fence, points[i])) {
1071                         dma_fence_put(fence);
1072                         if (flags & (DRM_SYNCOBJ_WAIT_FLAGS_WAIT_FOR_SUBMIT |
1073                                      DRM_SYNCOBJ_WAIT_FLAGS_WAIT_AVAILABLE)) {
1074                                 continue;
1075                         } else {
1076                                 timeout = -EINVAL;
1077                                 goto cleanup_entries;
1078                         }
1079                 }
1080
1081                 if (fence)
1082                         entries[i].fence = fence;
1083                 else
1084                         entries[i].fence = dma_fence_get_stub();
1085
1086                 if ((flags & DRM_SYNCOBJ_WAIT_FLAGS_WAIT_AVAILABLE) ||
1087                     dma_fence_is_signaled(entries[i].fence)) {
1088                         if (signaled_count == 0 && idx)
1089                                 *idx = i;
1090                         signaled_count++;
1091                 }
1092         }
1093
1094         if (signaled_count == count ||
1095             (signaled_count > 0 &&
1096              !(flags & DRM_SYNCOBJ_WAIT_FLAGS_WAIT_ALL)))
1097                 goto cleanup_entries;
1098
1099         /* There's a very annoying laxness in the dma_fence API here, in
1100          * that backends are not required to automatically report when a
1101          * fence is signaled prior to fence->ops->enable_signaling() being
1102          * called.  So here if we fail to match signaled_count, we need to
1103          * fallthough and try a 0 timeout wait!
1104          */
1105
1106         if (flags & DRM_SYNCOBJ_WAIT_FLAGS_WAIT_FOR_SUBMIT) {
1107                 for (i = 0; i < count; ++i)
1108                         drm_syncobj_fence_add_wait(syncobjs[i], &entries[i]);
1109         }
1110
1111         do {
1112                 set_current_state(TASK_INTERRUPTIBLE);
1113
1114                 signaled_count = 0;
1115                 for (i = 0; i < count; ++i) {
1116                         fence = entries[i].fence;
1117                         if (!fence)
1118                                 continue;
1119
1120                         if ((flags & DRM_SYNCOBJ_WAIT_FLAGS_WAIT_AVAILABLE) ||
1121                             dma_fence_is_signaled(fence) ||
1122                             (!entries[i].fence_cb.func &&
1123                              dma_fence_add_callback(fence,
1124                                                     &entries[i].fence_cb,
1125                                                     syncobj_wait_fence_func))) {
1126                                 /* The fence has been signaled */
1127                                 if (flags & DRM_SYNCOBJ_WAIT_FLAGS_WAIT_ALL) {
1128                                         signaled_count++;
1129                                 } else {
1130                                         if (idx)
1131                                                 *idx = i;
1132                                         goto done_waiting;
1133                                 }
1134                         }
1135                 }
1136
1137                 if (signaled_count == count)
1138                         goto done_waiting;
1139
1140                 if (timeout == 0) {
1141                         timeout = -ETIME;
1142                         goto done_waiting;
1143                 }
1144
1145                 if (signal_pending(current)) {
1146                         timeout = -ERESTARTSYS;
1147                         goto done_waiting;
1148                 }
1149
1150                 timeout = schedule_timeout(timeout);
1151         } while (1);
1152
1153 done_waiting:
1154         __set_current_state(TASK_RUNNING);
1155
1156 cleanup_entries:
1157         for (i = 0; i < count; ++i) {
1158                 drm_syncobj_remove_wait(syncobjs[i], &entries[i]);
1159                 if (entries[i].fence_cb.func)
1160                         dma_fence_remove_callback(entries[i].fence,
1161                                                   &entries[i].fence_cb);
1162                 dma_fence_put(entries[i].fence);
1163         }
1164         kfree(entries);
1165
1166 err_free_points:
1167         kfree(points);
1168
1169         return timeout;
1170 }
1171
1172 /**
1173  * drm_timeout_abs_to_jiffies - calculate jiffies timeout from absolute value
1174  *
1175  * @timeout_nsec: timeout nsec component in ns, 0 for poll
1176  *
1177  * Calculate the timeout in jiffies from an absolute time in sec/nsec.
1178  */
1179 signed long drm_timeout_abs_to_jiffies(int64_t timeout_nsec)
1180 {
1181         ktime_t abs_timeout, now;
1182         u64 timeout_ns, timeout_jiffies64;
1183
1184         /* make 0 timeout means poll - absolute 0 doesn't seem valid */
1185         if (timeout_nsec == 0)
1186                 return 0;
1187
1188         abs_timeout = ns_to_ktime(timeout_nsec);
1189         now = ktime_get();
1190
1191         if (!ktime_after(abs_timeout, now))
1192                 return 0;
1193
1194         timeout_ns = ktime_to_ns(ktime_sub(abs_timeout, now));
1195
1196         timeout_jiffies64 = nsecs_to_jiffies64(timeout_ns);
1197         /*  clamp timeout to avoid infinite timeout */
1198         if (timeout_jiffies64 >= MAX_SCHEDULE_TIMEOUT - 1)
1199                 return MAX_SCHEDULE_TIMEOUT - 1;
1200
1201         return timeout_jiffies64 + 1;
1202 }
1203 EXPORT_SYMBOL(drm_timeout_abs_to_jiffies);
1204
1205 static int drm_syncobj_array_wait(struct drm_device *dev,
1206                                   struct drm_file *file_private,
1207                                   struct drm_syncobj_wait *wait,
1208                                   struct drm_syncobj_timeline_wait *timeline_wait,
1209                                   struct drm_syncobj **syncobjs, bool timeline)
1210 {
1211         signed long timeout = 0;
1212         uint32_t first = ~0;
1213
1214         if (!timeline) {
1215                 timeout = drm_timeout_abs_to_jiffies(wait->timeout_nsec);
1216                 timeout = drm_syncobj_array_wait_timeout(syncobjs,
1217                                                          NULL,
1218                                                          wait->count_handles,
1219                                                          wait->flags,
1220                                                          timeout, &first);
1221                 if (timeout < 0)
1222                         return timeout;
1223                 wait->first_signaled = first;
1224         } else {
1225                 timeout = drm_timeout_abs_to_jiffies(timeline_wait->timeout_nsec);
1226                 timeout = drm_syncobj_array_wait_timeout(syncobjs,
1227                                                          u64_to_user_ptr(timeline_wait->points),
1228                                                          timeline_wait->count_handles,
1229                                                          timeline_wait->flags,
1230                                                          timeout, &first);
1231                 if (timeout < 0)
1232                         return timeout;
1233                 timeline_wait->first_signaled = first;
1234         }
1235         return 0;
1236 }
1237
1238 static int drm_syncobj_array_find(struct drm_file *file_private,
1239                                   void __user *user_handles,
1240                                   uint32_t count_handles,
1241                                   struct drm_syncobj ***syncobjs_out)
1242 {
1243         uint32_t i, *handles;
1244         struct drm_syncobj **syncobjs;
1245         int ret;
1246
1247         handles = kmalloc_array(count_handles, sizeof(*handles), GFP_KERNEL);
1248         if (handles == NULL)
1249                 return -ENOMEM;
1250
1251         if (copy_from_user(handles, user_handles,
1252                            sizeof(uint32_t) * count_handles)) {
1253                 ret = -EFAULT;
1254                 goto err_free_handles;
1255         }
1256
1257         syncobjs = kmalloc_array(count_handles, sizeof(*syncobjs), GFP_KERNEL);
1258         if (syncobjs == NULL) {
1259                 ret = -ENOMEM;
1260                 goto err_free_handles;
1261         }
1262
1263         for (i = 0; i < count_handles; i++) {
1264                 syncobjs[i] = drm_syncobj_find(file_private, handles[i]);
1265                 if (!syncobjs[i]) {
1266                         ret = -ENOENT;
1267                         goto err_put_syncobjs;
1268                 }
1269         }
1270
1271         kfree(handles);
1272         *syncobjs_out = syncobjs;
1273         return 0;
1274
1275 err_put_syncobjs:
1276         while (i-- > 0)
1277                 drm_syncobj_put(syncobjs[i]);
1278         kfree(syncobjs);
1279 err_free_handles:
1280         kfree(handles);
1281
1282         return ret;
1283 }
1284
1285 static void drm_syncobj_array_free(struct drm_syncobj **syncobjs,
1286                                    uint32_t count)
1287 {
1288         uint32_t i;
1289
1290         for (i = 0; i < count; i++)
1291                 drm_syncobj_put(syncobjs[i]);
1292         kfree(syncobjs);
1293 }
1294
1295 int
1296 drm_syncobj_wait_ioctl(struct drm_device *dev, void *data,
1297                        struct drm_file *file_private)
1298 {
1299         struct drm_syncobj_wait *args = data;
1300         struct drm_syncobj **syncobjs;
1301         int ret = 0;
1302
1303         if (!drm_core_check_feature(dev, DRIVER_SYNCOBJ))
1304                 return -EOPNOTSUPP;
1305
1306         if (args->flags & ~(DRM_SYNCOBJ_WAIT_FLAGS_WAIT_ALL |
1307                             DRM_SYNCOBJ_WAIT_FLAGS_WAIT_FOR_SUBMIT))
1308                 return -EINVAL;
1309
1310         if (args->count_handles == 0)
1311                 return -EINVAL;
1312
1313         ret = drm_syncobj_array_find(file_private,
1314                                      u64_to_user_ptr(args->handles),
1315                                      args->count_handles,
1316                                      &syncobjs);
1317         if (ret < 0)
1318                 return ret;
1319
1320         ret = drm_syncobj_array_wait(dev, file_private,
1321                                      args, NULL, syncobjs, false);
1322
1323         drm_syncobj_array_free(syncobjs, args->count_handles);
1324
1325         return ret;
1326 }
1327
1328 int
1329 drm_syncobj_timeline_wait_ioctl(struct drm_device *dev, void *data,
1330                                 struct drm_file *file_private)
1331 {
1332         struct drm_syncobj_timeline_wait *args = data;
1333         struct drm_syncobj **syncobjs;
1334         int ret = 0;
1335
1336         if (!drm_core_check_feature(dev, DRIVER_SYNCOBJ_TIMELINE))
1337                 return -EOPNOTSUPP;
1338
1339         if (args->flags & ~(DRM_SYNCOBJ_WAIT_FLAGS_WAIT_ALL |
1340                             DRM_SYNCOBJ_WAIT_FLAGS_WAIT_FOR_SUBMIT |
1341                             DRM_SYNCOBJ_WAIT_FLAGS_WAIT_AVAILABLE))
1342                 return -EINVAL;
1343
1344         if (args->count_handles == 0)
1345                 return -EINVAL;
1346
1347         ret = drm_syncobj_array_find(file_private,
1348                                      u64_to_user_ptr(args->handles),
1349                                      args->count_handles,
1350                                      &syncobjs);
1351         if (ret < 0)
1352                 return ret;
1353
1354         ret = drm_syncobj_array_wait(dev, file_private,
1355                                      NULL, args, syncobjs, true);
1356
1357         drm_syncobj_array_free(syncobjs, args->count_handles);
1358
1359         return ret;
1360 }
1361
1362 static void syncobj_eventfd_entry_fence_func(struct dma_fence *fence,
1363                                              struct dma_fence_cb *cb)
1364 {
1365         struct syncobj_eventfd_entry *entry =
1366                 container_of(cb, struct syncobj_eventfd_entry, fence_cb);
1367
1368         eventfd_signal(entry->ev_fd_ctx, 1);
1369         syncobj_eventfd_entry_free(entry);
1370 }
1371
1372 static void
1373 syncobj_eventfd_entry_func(struct drm_syncobj *syncobj,
1374                            struct syncobj_eventfd_entry *entry)
1375 {
1376         int ret;
1377         struct dma_fence *fence;
1378
1379         /* This happens inside the syncobj lock */
1380         fence = dma_fence_get(rcu_dereference_protected(syncobj->fence, 1));
1381         ret = dma_fence_chain_find_seqno(&fence, entry->point);
1382         if (ret != 0 || !fence) {
1383                 dma_fence_put(fence);
1384                 return;
1385         }
1386
1387         list_del_init(&entry->node);
1388         entry->fence = fence;
1389
1390         if (entry->flags & DRM_SYNCOBJ_WAIT_FLAGS_WAIT_AVAILABLE) {
1391                 eventfd_signal(entry->ev_fd_ctx, 1);
1392                 syncobj_eventfd_entry_free(entry);
1393         } else {
1394                 ret = dma_fence_add_callback(fence, &entry->fence_cb,
1395                                              syncobj_eventfd_entry_fence_func);
1396                 if (ret == -ENOENT) {
1397                         eventfd_signal(entry->ev_fd_ctx, 1);
1398                         syncobj_eventfd_entry_free(entry);
1399                 }
1400         }
1401 }
1402
1403 int
1404 drm_syncobj_eventfd_ioctl(struct drm_device *dev, void *data,
1405                           struct drm_file *file_private)
1406 {
1407         struct drm_syncobj_eventfd *args = data;
1408         struct drm_syncobj *syncobj;
1409         struct eventfd_ctx *ev_fd_ctx;
1410         struct syncobj_eventfd_entry *entry;
1411
1412         if (!drm_core_check_feature(dev, DRIVER_SYNCOBJ_TIMELINE))
1413                 return -EOPNOTSUPP;
1414
1415         if (args->flags & ~DRM_SYNCOBJ_WAIT_FLAGS_WAIT_AVAILABLE)
1416                 return -EINVAL;
1417
1418         if (args->pad)
1419                 return -EINVAL;
1420
1421         syncobj = drm_syncobj_find(file_private, args->handle);
1422         if (!syncobj)
1423                 return -ENOENT;
1424
1425         ev_fd_ctx = eventfd_ctx_fdget(args->fd);
1426         if (IS_ERR(ev_fd_ctx))
1427                 return PTR_ERR(ev_fd_ctx);
1428
1429         entry = kzalloc(sizeof(*entry), GFP_KERNEL);
1430         if (!entry) {
1431                 eventfd_ctx_put(ev_fd_ctx);
1432                 return -ENOMEM;
1433         }
1434         entry->syncobj = syncobj;
1435         entry->ev_fd_ctx = ev_fd_ctx;
1436         entry->point = args->point;
1437         entry->flags = args->flags;
1438
1439         drm_syncobj_add_eventfd(syncobj, entry);
1440         drm_syncobj_put(syncobj);
1441
1442         return 0;
1443 }
1444
1445 int
1446 drm_syncobj_reset_ioctl(struct drm_device *dev, void *data,
1447                         struct drm_file *file_private)
1448 {
1449         struct drm_syncobj_array *args = data;
1450         struct drm_syncobj **syncobjs;
1451         uint32_t i;
1452         int ret;
1453
1454         if (!drm_core_check_feature(dev, DRIVER_SYNCOBJ))
1455                 return -EOPNOTSUPP;
1456
1457         if (args->pad != 0)
1458                 return -EINVAL;
1459
1460         if (args->count_handles == 0)
1461                 return -EINVAL;
1462
1463         ret = drm_syncobj_array_find(file_private,
1464                                      u64_to_user_ptr(args->handles),
1465                                      args->count_handles,
1466                                      &syncobjs);
1467         if (ret < 0)
1468                 return ret;
1469
1470         for (i = 0; i < args->count_handles; i++)
1471                 drm_syncobj_replace_fence(syncobjs[i], NULL);
1472
1473         drm_syncobj_array_free(syncobjs, args->count_handles);
1474
1475         return 0;
1476 }
1477
1478 int
1479 drm_syncobj_signal_ioctl(struct drm_device *dev, void *data,
1480                          struct drm_file *file_private)
1481 {
1482         struct drm_syncobj_array *args = data;
1483         struct drm_syncobj **syncobjs;
1484         uint32_t i;
1485         int ret;
1486
1487         if (!drm_core_check_feature(dev, DRIVER_SYNCOBJ))
1488                 return -EOPNOTSUPP;
1489
1490         if (args->pad != 0)
1491                 return -EINVAL;
1492
1493         if (args->count_handles == 0)
1494                 return -EINVAL;
1495
1496         ret = drm_syncobj_array_find(file_private,
1497                                      u64_to_user_ptr(args->handles),
1498                                      args->count_handles,
1499                                      &syncobjs);
1500         if (ret < 0)
1501                 return ret;
1502
1503         for (i = 0; i < args->count_handles; i++) {
1504                 ret = drm_syncobj_assign_null_handle(syncobjs[i]);
1505                 if (ret < 0)
1506                         break;
1507         }
1508
1509         drm_syncobj_array_free(syncobjs, args->count_handles);
1510
1511         return ret;
1512 }
1513
1514 int
1515 drm_syncobj_timeline_signal_ioctl(struct drm_device *dev, void *data,
1516                                   struct drm_file *file_private)
1517 {
1518         struct drm_syncobj_timeline_array *args = data;
1519         struct drm_syncobj **syncobjs;
1520         struct dma_fence_chain **chains;
1521         uint64_t *points;
1522         uint32_t i, j;
1523         int ret;
1524
1525         if (!drm_core_check_feature(dev, DRIVER_SYNCOBJ_TIMELINE))
1526                 return -EOPNOTSUPP;
1527
1528         if (args->flags != 0)
1529                 return -EINVAL;
1530
1531         if (args->count_handles == 0)
1532                 return -EINVAL;
1533
1534         ret = drm_syncobj_array_find(file_private,
1535                                      u64_to_user_ptr(args->handles),
1536                                      args->count_handles,
1537                                      &syncobjs);
1538         if (ret < 0)
1539                 return ret;
1540
1541         points = kmalloc_array(args->count_handles, sizeof(*points),
1542                                GFP_KERNEL);
1543         if (!points) {
1544                 ret = -ENOMEM;
1545                 goto out;
1546         }
1547         if (!u64_to_user_ptr(args->points)) {
1548                 memset(points, 0, args->count_handles * sizeof(uint64_t));
1549         } else if (copy_from_user(points, u64_to_user_ptr(args->points),
1550                                   sizeof(uint64_t) * args->count_handles)) {
1551                 ret = -EFAULT;
1552                 goto err_points;
1553         }
1554
1555         chains = kmalloc_array(args->count_handles, sizeof(void *), GFP_KERNEL);
1556         if (!chains) {
1557                 ret = -ENOMEM;
1558                 goto err_points;
1559         }
1560         for (i = 0; i < args->count_handles; i++) {
1561                 chains[i] = dma_fence_chain_alloc();
1562                 if (!chains[i]) {
1563                         for (j = 0; j < i; j++)
1564                                 dma_fence_chain_free(chains[j]);
1565                         ret = -ENOMEM;
1566                         goto err_chains;
1567                 }
1568         }
1569
1570         for (i = 0; i < args->count_handles; i++) {
1571                 struct dma_fence *fence = dma_fence_get_stub();
1572
1573                 drm_syncobj_add_point(syncobjs[i], chains[i],
1574                                       fence, points[i]);
1575                 dma_fence_put(fence);
1576         }
1577 err_chains:
1578         kfree(chains);
1579 err_points:
1580         kfree(points);
1581 out:
1582         drm_syncobj_array_free(syncobjs, args->count_handles);
1583
1584         return ret;
1585 }
1586
1587 int drm_syncobj_query_ioctl(struct drm_device *dev, void *data,
1588                             struct drm_file *file_private)
1589 {
1590         struct drm_syncobj_timeline_array *args = data;
1591         struct drm_syncobj **syncobjs;
1592         uint64_t __user *points = u64_to_user_ptr(args->points);
1593         uint32_t i;
1594         int ret;
1595
1596         if (!drm_core_check_feature(dev, DRIVER_SYNCOBJ_TIMELINE))
1597                 return -EOPNOTSUPP;
1598
1599         if (args->flags & ~DRM_SYNCOBJ_QUERY_FLAGS_LAST_SUBMITTED)
1600                 return -EINVAL;
1601
1602         if (args->count_handles == 0)
1603                 return -EINVAL;
1604
1605         ret = drm_syncobj_array_find(file_private,
1606                                      u64_to_user_ptr(args->handles),
1607                                      args->count_handles,
1608                                      &syncobjs);
1609         if (ret < 0)
1610                 return ret;
1611
1612         for (i = 0; i < args->count_handles; i++) {
1613                 struct dma_fence_chain *chain;
1614                 struct dma_fence *fence;
1615                 uint64_t point;
1616
1617                 fence = drm_syncobj_fence_get(syncobjs[i]);
1618                 chain = to_dma_fence_chain(fence);
1619                 if (chain) {
1620                         struct dma_fence *iter, *last_signaled =
1621                                 dma_fence_get(fence);
1622
1623                         if (args->flags &
1624                             DRM_SYNCOBJ_QUERY_FLAGS_LAST_SUBMITTED) {
1625                                 point = fence->seqno;
1626                         } else {
1627                                 dma_fence_chain_for_each(iter, fence) {
1628                                         if (iter->context != fence->context) {
1629                                                 dma_fence_put(iter);
1630                                                 /* It is most likely that timeline has
1631                                                 * unorder points. */
1632                                                 break;
1633                                         }
1634                                         dma_fence_put(last_signaled);
1635                                         last_signaled = dma_fence_get(iter);
1636                                 }
1637                                 point = dma_fence_is_signaled(last_signaled) ?
1638                                         last_signaled->seqno :
1639                                         to_dma_fence_chain(last_signaled)->prev_seqno;
1640                         }
1641                         dma_fence_put(last_signaled);
1642                 } else {
1643                         point = 0;
1644                 }
1645                 dma_fence_put(fence);
1646                 ret = copy_to_user(&points[i], &point, sizeof(uint64_t));
1647                 ret = ret ? -EFAULT : 0;
1648                 if (ret)
1649                         break;
1650         }
1651         drm_syncobj_array_free(syncobjs, args->count_handles);
1652
1653         return ret;
1654 }