drivers/gpu/vithar/kds/lib/kds/kds.c

   1 /*
   2  *
   3  * (C) COPYRIGHT 2012 ARM Limited. All rights reserved.
   4  *
   5  * This program is free software and is provided to you under the terms of the GNU General Public License version 2
   6  * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence.
   7  *
   8  * A copy of the licence is included with the program, and can also be obtained from Free Software
   9  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
  10  *
  11  */
  12
  13
  14
  15 #include <linux/slab.h>
  16 #include <linux/list.h>
  17 #include <linux/mutex.h>
  18 #include <linux/wait.h>
  19 #include <linux/sched.h>
  20 #include <linux/err.h>
  21 #include <linux/module.h>
  22 #include <linux/workqueue.h>
  23 #include "linux/kds.h"
  24
  25
  26 #define KDS_LINK_TRIGGERED (1u << 0)
  27 #define KDS_LINK_EXCLUSIVE (1u << 1)
  28
  29 #define KDS_IGNORED NULL
  30 #define KDS_INVALID (void*)-2
  31 #define KDS_RESOURCE (void*)-1
  32
  33 struct kds_resource_set
  34 {
  35         unsigned long         num_resources;
  36         unsigned long         pending;
  37         unsigned long         locked_resources;
  38         struct kds_callback * cb;
  39         void *                callback_parameter;
  40         void *                callback_extra_parameter;
  41         struct list_head      callback_link;
  42         struct work_struct    callback_work;
  43         struct kds_link       resources[0];
  44 };
  45
  46 static DEFINE_MUTEX(kds_lock);
  47
  48 int kds_callback_init(struct kds_callback * cb, int direct, kds_callback_fn user_cb)
  49 {
  50         int ret = 0;
  51
  52         cb->direct = direct;
  53         cb->user_cb = user_cb;
  54
  55         if (!direct)
  56         {
  57                 cb->wq = alloc_workqueue("kds", WQ_UNBOUND | WQ_HIGHPRI, WQ_UNBOUND_MAX_ACTIVE);
  58                 if (!cb->wq)
  59                         ret = -ENOMEM;
  60         }
  61         else
  62         {
  63                 cb->wq = NULL;
  64         }
  65
  66         return ret;
  67 }
  68 EXPORT_SYMBOL(kds_callback_init);
  69
  70 void kds_callback_term(struct kds_callback * cb)
  71 {
  72         if (!cb->direct)
  73         {
  74                 BUG_ON(!cb->wq);
  75                 destroy_workqueue(cb->wq);
  76         }
  77         else
  78         {
  79                 BUG_ON(cb->wq);
  80         }
  81 }
  82
  83 EXPORT_SYMBOL(kds_callback_term);
  84
  85 static void kds_do_user_callback(struct kds_resource_set * rset)
  86 {
  87         rset->cb->user_cb(rset->callback_parameter, rset->callback_extra_parameter);
  88 }
  89
  90 static void kds_queued_callback(struct work_struct * work)
  91 {
  92         struct kds_resource_set * rset;
  93         rset = container_of( work, struct kds_resource_set, callback_work);
  94
  95         kds_do_user_callback(rset);
  96 }
  97
  98 static void kds_callback_perform(struct kds_resource_set * rset)
  99 {
 100         if (rset->cb->direct)
 101                 kds_do_user_callback(rset);
 102         else
 103         {
 104                 int result;
 105                 result = queue_work(rset->cb->wq, &rset->callback_work);
 106                 /* if we got a 0 return it means we've triggered the same rset twice! */
 107                 BUG_ON(!result);
 108         }
 109 }
 110
 111 void kds_resource_init(struct kds_resource * res)
 112 {
 113         BUG_ON(!res);
 114         INIT_LIST_HEAD(&res->waiters.link);
 115         res->waiters.parent = KDS_RESOURCE;
 116 }
 117 EXPORT_SYMBOL(kds_resource_init);
 118
 119 void kds_resource_term(struct kds_resource * res)
 120 {
 121         BUG_ON(!res);
 122         BUG_ON(!list_empty(&res->waiters.link));
 123         res->waiters.parent = KDS_INVALID;
 124 }
 125 EXPORT_SYMBOL(kds_resource_term);
 126
 127 int kds_async_waitall(
 128                struct kds_resource_set ** pprset,
 129                unsigned long              flags,
 130                struct kds_callback *      cb,
 131                void *                     callback_parameter,
 132                void *                     callback_extra_parameter,
 133                int                        number_resources,
 134                unsigned long *            exclusive_access_bitmap,
 135                struct kds_resource **     resource_list)
 136 {
 137         struct kds_resource_set * rset = NULL;
 138         int i;
 139         int triggered;
 140         int err = -EFAULT;
 141
 142         BUG_ON(!pprset);
 143         BUG_ON(!resource_list);
 144         BUG_ON(!cb);
 145
 146         mutex_lock(&kds_lock);
 147
 148         if ((flags & KDS_FLAG_LOCKED_ACTION) == KDS_FLAG_LOCKED_FAIL)
 149         {
 150                 for (i = 0; i < number_resources; i++)
 151                 {
 152                         if (resource_list[i]->lock_count)
 153                         {
 154                                 err = -EBUSY;
 155                                 goto errout;
 156                         }
 157                 }
 158         }
 159
 160         rset = kmalloc(sizeof(*rset) + number_resources * sizeof(struct kds_link), GFP_KERNEL);
 161         if (!rset)
 162         {
 163                 err = -ENOMEM;
 164                 goto errout;
 165         }
 166
 167         rset->num_resources = number_resources;
 168         rset->pending = number_resources;
 169         rset->locked_resources = 0;
 170         rset->cb = cb;
 171         rset->callback_parameter = callback_parameter;
 172         rset->callback_extra_parameter = callback_extra_parameter;
 173         INIT_LIST_HEAD(&rset->callback_link);
 174         INIT_WORK(&rset->callback_work, kds_queued_callback);
 175
 176         for (i = 0; i < number_resources; i++)
 177         {
 178                 unsigned long link_state = 0;
 179
 180                 INIT_LIST_HEAD(&rset->resources[i].link);
 181                 rset->resources[i].parent = rset;
 182
 183                 if (test_bit(i, exclusive_access_bitmap))
 184                 {
 185                         link_state |= KDS_LINK_EXCLUSIVE;
 186                 }
 187
 188                 /* no-one else waiting? */
 189                 if (list_empty(&resource_list[i]->waiters.link))
 190                 {
 191                         link_state |= KDS_LINK_TRIGGERED;
 192                         rset->pending--;
 193                 }
 194                 /* Adding a non-exclusive and the current tail is a triggered non-exclusive? */
 195                 else if (((link_state & KDS_LINK_EXCLUSIVE) == 0) &&
 196                         (((list_entry(resource_list[i]->waiters.link.prev, struct kds_link, link)->state & (KDS_LINK_EXCLUSIVE | KDS_LINK_TRIGGERED)) == KDS_LINK_TRIGGERED)))
 197                 {
 198                         link_state |= KDS_LINK_TRIGGERED;
 199                         rset->pending--;
 200                 }
 201                 /* locked & ignore locked? */
 202                 else if ((resource_list[i]->lock_count) && ((flags & KDS_FLAG_LOCKED_ACTION) == KDS_FLAG_LOCKED_IGNORE) )
 203                 {
 204                         link_state |= KDS_LINK_TRIGGERED;
 205                         rset->pending--;
 206                         rset->resources[i].parent = KDS_IGNORED; /* to disable decrementing the pending count when we get the ignored resource */
 207                 }
 208                 rset->resources[i].state = link_state;
 209                 list_add_tail(&rset->resources[i].link, &resource_list[i]->waiters.link);
 210         }
 211
 212         triggered = (rset->pending == 0);
 213
 214         mutex_unlock(&kds_lock);
 215
 216         /* set the pointer before the callback is called so it sees it */
 217         *pprset = rset;
 218
 219         if (triggered)
 220         {
 221                 /* all resources obtained, trigger callback */
 222                 kds_callback_perform(rset);
 223         }
 224
 225         return 0;
 226
 227 errout:
 228         mutex_unlock(&kds_lock);
 229         return err;
 230 }
 231 EXPORT_SYMBOL(kds_async_waitall);
 232
 233 static void wake_up_sync_call(void * callback_parameter, void * callback_extra_parameter)
 234 {
 235         wait_queue_head_t * wait = (wait_queue_head_t*)callback_parameter;
 236         wake_up(wait);
 237 }
 238
 239 static struct kds_callback sync_cb =
 240 {
 241         wake_up_sync_call,
 242         1,
 243         NULL,
 244 };
 245
 246 struct kds_resource_set * kds_waitall(
 247                int                    number_resources,
 248                unsigned long *        exclusive_access_bitmap,
 249                struct kds_resource ** resource_list,
 250                unsigned long          jiffies_timeout)
 251 {
 252         struct kds_resource_set * rset;
 253         int i;
 254         int triggered = 0;
 255         DECLARE_WAIT_QUEUE_HEAD_ONSTACK(wake);
 256
 257         rset = kmalloc(sizeof(*rset) + number_resources * sizeof(struct kds_link), GFP_KERNEL);
 258         if (!rset)
 259                 return rset;
 260
 261         rset->num_resources = number_resources;
 262         rset->pending = number_resources;
 263         rset->locked_resources = 1;
 264         INIT_LIST_HEAD(&rset->callback_link);
 265
 266         mutex_lock(&kds_lock);
 267
 268         for (i = 0; i < number_resources; i++)
 269         {
 270                 unsigned long link_state = 0;
 271
 272                 if (likely(resource_list[i]->lock_count < ULONG_MAX))
 273                         resource_list[i]->lock_count++;
 274                 else
 275                         break;
 276
 277                 if (test_bit(i, exclusive_access_bitmap))
 278                 {
 279                         link_state |= KDS_LINK_EXCLUSIVE;
 280                 }
 281
 282                 if (list_empty(&resource_list[i]->waiters.link))
 283                 {
 284                         link_state |= KDS_LINK_TRIGGERED;
 285                         rset->pending--;
 286                 }
 287                 /* Adding a non-exclusive and the current tail is a triggered non-exclusive? */
 288                 else if (((link_state & KDS_LINK_EXCLUSIVE) == 0) &&
 289                         (((list_entry(resource_list[i]->waiters.link.prev, struct kds_link, link)->state & (KDS_LINK_EXCLUSIVE | KDS_LINK_TRIGGERED)) == KDS_LINK_TRIGGERED)))
 290                 {
 291                         link_state |= KDS_LINK_TRIGGERED;
 292                         rset->pending--;
 293                 }
 294
 295                 INIT_LIST_HEAD(&rset->resources[i].link);
 296                 rset->resources[i].parent = rset;
 297                 rset->resources[i].state = link_state;
 298                 list_add_tail(&rset->resources[i].link, &resource_list[i]->waiters.link);
 299         }
 300
 301         if (i < number_resources)
 302         {
 303                 /* an overflow was detected, roll back */
 304                 while (i--)
 305                 {
 306                         list_del(&rset->resources[i].link);
 307                         resource_list[i]->lock_count--;
 308                 }
 309                 mutex_unlock(&kds_lock);
 310                 kfree(rset);
 311                 return ERR_PTR(-EFAULT);
 312         }
 313
 314         if (rset->pending == 0)
 315                 triggered = 1;
 316         else
 317         {
 318                 rset->cb = &sync_cb;
 319                 rset->callback_parameter = &wake;
 320                 rset->callback_extra_parameter = NULL;
 321         }
 322
 323         mutex_unlock(&kds_lock);
 324
 325         if (!triggered)
 326         {
 327                 long wait_res;
 328                 if ( KDS_WAIT_BLOCKING == jiffies_timeout )
 329                 {
 330                         wait_res = wait_event_interruptible(wake, rset->pending == 0);
 331                 }
 332                 else
 333                 {
 334                         wait_res = wait_event_interruptible_timeout(wake, rset->pending == 0, jiffies_timeout);
 335                 }
 336                 if ((wait_res == -ERESTARTSYS) || (wait_res == 0))
 337                 {
 338                         /* use \a kds_resource_set_release to roll back */
 339                         kds_resource_set_release(&rset);
 340                         return ERR_PTR(wait_res);
 341                 }
 342         }
 343         return rset;
 344 }
 345 EXPORT_SYMBOL(kds_waitall);
 346
 347 void kds_resource_set_release(struct kds_resource_set ** pprset)
 348 {
 349         struct list_head triggered = LIST_HEAD_INIT(triggered);
 350         struct kds_resource_set * rset;
 351         struct kds_resource_set * it;
 352         int i;
 353
 354         BUG_ON(!pprset);
 355
 356         mutex_lock(&kds_lock);
 357
 358         rset = *pprset;
 359         if (!rset)
 360         {
 361                 /* caught a race between a cancelation
 362                  * and a completion, nothing to do */
 363                 mutex_unlock(&kds_lock);
 364                 return;
 365         }
 366
 367         /* clear user pointer so we'll be the only
 368          * thread handling the release */
 369         *pprset = NULL;
 370
 371         for (i = 0; i < rset->num_resources; i++)
 372         {
 373                 struct kds_resource * resource;
 374                 struct kds_link * it = NULL;
 375
 376                 /* fetch the previous entry on the linked list */
 377                 it = list_entry(rset->resources[i].link.prev, struct kds_link, link);
 378                 /* unlink ourself */
 379                 list_del(&rset->resources[i].link);
 380
 381                 /* any waiters? */
 382                 if (list_empty(&it->link))
 383                         continue;
 384
 385                 /* were we the head of the list? (head if prev is a resource) */
 386                 if (it->parent != KDS_RESOURCE)
 387                         continue;
 388
 389                 /* we were the head, find the kds_resource */
 390                 resource = container_of(it, struct kds_resource, waiters);
 391
 392                 if (rset->locked_resources)
 393                 {
 394                         resource->lock_count--;
 395                 }
 396
 397                 /* we know there is someone waiting from the any-waiters test above */
 398
 399                 /* find the head of the waiting list */
 400                 it = list_first_entry(&resource->waiters.link, struct kds_link, link);
 401
 402                 /* new exclusive owner? */
 403                 if (it->state & KDS_LINK_EXCLUSIVE)
 404                 {
 405                         /* link now triggered */
 406                         it->state |= KDS_LINK_TRIGGERED;
 407                         /* a parent to update? */
 408                         if (it->parent != KDS_IGNORED)
 409                         {
 410                                 if (0 == --it->parent->pending)
 411                                 {
 412                                         /* new owner now triggered, track for callback later */
 413                                         list_add(&it->parent->callback_link, &triggered);
 414                                 }
 415                         }
 416                 }
 417                 /* exclusive releasing ? */
 418                 else if (rset->resources[i].state & KDS_LINK_EXCLUSIVE)
 419                 {
 420                         /* trigger non-exclusive until end-of-list or first exclusive */
 421                         list_for_each_entry(it, &resource->waiters.link, link)
 422                         {
 423                                 /* exclusive found, stop triggering */
 424                                 if (it->state & KDS_LINK_EXCLUSIVE)
 425                                         break;
 426
 427                                 it->state |= KDS_LINK_TRIGGERED;
 428                                 /* a parent to update? */
 429                                 if (it->parent != KDS_IGNORED)
 430                                 {
 431                                         if (0 == --it->parent->pending)
 432                                         {
 433                                                 /* new owner now triggered, track for callback later */
 434                                                 list_add(&it->parent->callback_link, &triggered);
 435                                         }
 436                                 }
 437                         }
 438                 }
 439
 440         }
 441
 442         mutex_unlock(&kds_lock);
 443
 444         while (!list_empty(&triggered))
 445         {
 446                 it = list_first_entry(&triggered, struct kds_resource_set, callback_link);
 447                 list_del(&it->callback_link);
 448                 kds_callback_perform(it);
 449         }
 450
 451         cancel_work_sync(&rset->callback_work);
 452
 453         /* free the resource set */
 454         kfree(rset);
 455 }
 456 EXPORT_SYMBOL(kds_resource_set_release);
 457
 458 MODULE_LICENSE("GPL");
 459 MODULE_AUTHOR("ARM Ltd.");
 460 MODULE_VERSION("1.0");