drivers/gpu/arm/mali400/r4p0_rel0/common/mali_timeline.c

   1 /*
   2  * Copyright (C) 2010-2012 ARM Limited. All rights reserved.
   3  *
   4  * This program is free software and is provided to you under the terms of the GNU General Public License version 2
   5  * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence.
   6  *
   7  * A copy of the licence is included with the program, and can also be obtained from Free Software
   8  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
   9  */
  10
  11 #include "mali_timeline.h"
  12 #include "mali_kernel_common.h"
  13 #include "mali_osk_mali.h"
  14 #include "mali_scheduler.h"
  15 #include "mali_soft_job.h"
  16 #include "mali_timeline_fence_wait.h"
  17 #include "mali_timeline_sync_fence.h"
  18
  19 #define MALI_TIMELINE_SYSTEM_LOCKED(system) (mali_spinlock_reentrant_is_held((system)->spinlock, _mali_osk_get_tid()))
  20
  21 static mali_scheduler_mask mali_timeline_system_release_waiter(struct mali_timeline_system *system,
  22         struct mali_timeline_waiter *waiter);
  23
  24 #if defined(CONFIG_SYNC)
  25 /* Callback that is called when a sync fence a tracker is waiting on is signaled. */
  26 static void mali_timeline_sync_fence_callback(struct sync_fence *sync_fence, struct sync_fence_waiter *sync_fence_waiter)
  27 {
  28         struct mali_timeline_system  *system;
  29         struct mali_timeline_waiter  *waiter;
  30         struct mali_timeline_tracker *tracker;
  31         mali_scheduler_mask schedule_mask = MALI_SCHEDULER_MASK_EMPTY;
  32         u32 tid = _mali_osk_get_tid();
  33         mali_bool is_aborting = MALI_FALSE;
  34         int fence_status = sync_fence->status;
  35
  36         MALI_DEBUG_ASSERT_POINTER(sync_fence);
  37         MALI_DEBUG_ASSERT_POINTER(sync_fence_waiter);
  38
  39         tracker = _MALI_OSK_CONTAINER_OF(sync_fence_waiter, struct mali_timeline_tracker, sync_fence_waiter);
  40         MALI_DEBUG_ASSERT_POINTER(tracker);
  41
  42         system = tracker->system;
  43         MALI_DEBUG_ASSERT_POINTER(system);
  44         MALI_DEBUG_ASSERT_POINTER(system->session);
  45
  46         mali_spinlock_reentrant_wait(system->spinlock, tid);
  47
  48         is_aborting = system->session->is_aborting;
  49         if (!is_aborting && (0 > fence_status)) {
  50                 MALI_PRINT_ERROR(("Mali Timeline: sync fence fd %d signaled with error %d\n", tracker->fence.sync_fd, fence_status));
  51                 tracker->activation_error |= MALI_TIMELINE_ACTIVATION_ERROR_SYNC_BIT;
  52         }
  53
  54         waiter = tracker->waiter_sync;
  55         MALI_DEBUG_ASSERT_POINTER(waiter);
  56
  57         tracker->sync_fence = NULL;
  58         schedule_mask |= mali_timeline_system_release_waiter(system, waiter);
  59
  60         /* If aborting, wake up sleepers that are waiting for sync fence callbacks to complete. */
  61         if (is_aborting) {
  62                 _mali_osk_wait_queue_wake_up(system->wait_queue);
  63         }
  64
  65         mali_spinlock_reentrant_signal(system->spinlock, tid);
  66
  67         sync_fence_put(sync_fence);
  68
  69         if (!is_aborting) {
  70                 mali_scheduler_schedule_from_mask(schedule_mask, MALI_TRUE);
  71         }
  72 }
  73 #endif /* defined(CONFIG_SYNC) */
  74
  75 static mali_scheduler_mask mali_timeline_tracker_time_out(struct mali_timeline_tracker *tracker)
  76 {
  77         MALI_DEBUG_ASSERT_POINTER(tracker);
  78         MALI_DEBUG_ASSERT(MALI_TIMELINE_TRACKER_SOFT == tracker->type);
  79
  80         return mali_soft_job_system_timeout_job((struct mali_soft_job *) tracker->job);
  81 }
  82
  83 static void mali_timeline_timer_callback(void *data)
  84 {
  85         struct mali_timeline_system *system;
  86         struct mali_timeline_tracker *tracker;
  87         struct mali_timeline *timeline;
  88         mali_scheduler_mask schedule_mask = MALI_SCHEDULER_MASK_EMPTY;
  89         u32 tid = _mali_osk_get_tid();
  90
  91         timeline = (struct mali_timeline *) data;
  92         MALI_DEBUG_ASSERT_POINTER(timeline);
  93
  94         system = timeline->system;
  95         MALI_DEBUG_ASSERT_POINTER(system);
  96
  97         mali_spinlock_reentrant_wait(system->spinlock, tid);
  98
  99         if (!system->timer_enabled) {
 100                 mali_spinlock_reentrant_signal(system->spinlock, tid);
 101                 return;
 102         }
 103
 104         tracker = timeline->tracker_tail;
 105         timeline->timer_active = MALI_FALSE;
 106
 107         if (NULL != tracker && MALI_TRUE == tracker->timer_active) {
 108                 /* This is likely the delayed work that has been schedule out before cancelled. */
 109                 if (MALI_TIMELINE_TIMEOUT_HZ > (_mali_osk_time_tickcount() - tracker->os_tick_activate)) {
 110                         mali_spinlock_reentrant_signal(system->spinlock, tid);
 111                         return;
 112                 }
 113
 114                 schedule_mask = mali_timeline_tracker_time_out(tracker);
 115                 tracker->timer_active = MALI_FALSE;
 116         } else {
 117                 MALI_PRINT_ERROR(("Mali Timeline: Soft job timer callback without a waiting tracker.\n"));
 118         }
 119
 120         mali_spinlock_reentrant_signal(system->spinlock, tid);
 121
 122         mali_scheduler_schedule_from_mask(schedule_mask, MALI_FALSE);
 123 }
 124
 125 void mali_timeline_system_stop_timer(struct mali_timeline_system *system)
 126 {
 127         u32 i;
 128         u32 tid = _mali_osk_get_tid();
 129
 130         MALI_DEBUG_ASSERT_POINTER(system);
 131
 132         mali_spinlock_reentrant_wait(system->spinlock, tid);
 133         system->timer_enabled = MALI_FALSE;
 134         mali_spinlock_reentrant_signal(system->spinlock, tid);
 135
 136         for (i = 0; i < MALI_TIMELINE_MAX; ++i) {
 137                 struct mali_timeline *timeline = system->timelines[i];
 138
 139                 MALI_DEBUG_ASSERT_POINTER(timeline);
 140
 141                 if (NULL != timeline->delayed_work) {
 142                         _mali_osk_wq_delayed_cancel_work_sync(timeline->delayed_work);
 143                         timeline->timer_active = MALI_FALSE;
 144                 }
 145         }
 146 }
 147
 148 static void mali_timeline_destroy(struct mali_timeline *timeline)
 149 {
 150         MALI_DEBUG_ASSERT_POINTER(timeline);
 151         if (NULL != timeline) {
 152                 /* Assert that the timeline object has been properly cleaned up before destroying it. */
 153                 MALI_DEBUG_ASSERT(timeline->point_oldest == timeline->point_next);
 154                 MALI_DEBUG_ASSERT(NULL == timeline->tracker_head);
 155                 MALI_DEBUG_ASSERT(NULL == timeline->tracker_tail);
 156                 MALI_DEBUG_ASSERT(NULL == timeline->waiter_head);
 157                 MALI_DEBUG_ASSERT(NULL == timeline->waiter_tail);
 158                 MALI_DEBUG_ASSERT(NULL != timeline->system);
 159                 MALI_DEBUG_ASSERT(MALI_TIMELINE_MAX > timeline->id);
 160
 161 #if defined(CONFIG_SYNC)
 162                 if (NULL != timeline->sync_tl) {
 163                         sync_timeline_destroy(timeline->sync_tl);
 164                 }
 165 #endif /* defined(CONFIG_SYNC) */
 166
 167                 if (NULL != timeline->delayed_work) {
 168                         _mali_osk_wq_delayed_cancel_work_sync(timeline->delayed_work);
 169                         _mali_osk_wq_delayed_delete_work_nonflush(timeline->delayed_work);
 170                 }
 171
 172                 _mali_osk_free(timeline);
 173         }
 174 }
 175
 176 static struct mali_timeline *mali_timeline_create(struct mali_timeline_system *system, enum mali_timeline_id id)
 177 {
 178         struct mali_timeline *timeline;
 179
 180         MALI_DEBUG_ASSERT_POINTER(system);
 181         MALI_DEBUG_ASSERT(id < MALI_TIMELINE_MAX);
 182
 183         timeline = (struct mali_timeline *) _mali_osk_calloc(1, sizeof(struct mali_timeline));
 184         if (NULL == timeline) {
 185                 return NULL;
 186         }
 187
 188         /* Initially the timeline is empty. */
 189 #if defined(MALI_TIMELINE_DEBUG_START_POINT)
 190         /* Start the timeline a bit before wrapping when debugging. */
 191         timeline->point_next = UINT_MAX - MALI_TIMELINE_MAX_POINT_SPAN - 128;
 192 #else
 193         timeline->point_next = 1;
 194 #endif
 195         timeline->point_oldest = timeline->point_next;
 196
 197         /* The tracker and waiter lists will initially be empty. */
 198
 199         timeline->system = system;
 200         timeline->id = id;
 201
 202         timeline->delayed_work = _mali_osk_wq_delayed_create_work(mali_timeline_timer_callback, timeline);
 203         if (NULL == timeline->delayed_work) {
 204                 mali_timeline_destroy(timeline);
 205                 return NULL;
 206         }
 207
 208         timeline->timer_active = MALI_FALSE;
 209
 210 #if defined(CONFIG_SYNC)
 211         {
 212                 char timeline_name[32];
 213
 214                 switch (id) {
 215                 case MALI_TIMELINE_GP:
 216                         _mali_osk_snprintf(timeline_name, 32, "mali-%u-gp", _mali_osk_get_pid());
 217                         break;
 218                 case MALI_TIMELINE_PP:
 219                         _mali_osk_snprintf(timeline_name, 32, "mali-%u-pp", _mali_osk_get_pid());
 220                         break;
 221                 case MALI_TIMELINE_SOFT:
 222                         _mali_osk_snprintf(timeline_name, 32, "mali-%u-soft", _mali_osk_get_pid());
 223                         break;
 224                 default:
 225                         MALI_PRINT_ERROR(("Mali Timeline: Invalid timeline id %d\n", id));
 226                         mali_timeline_destroy(timeline);
 227                         return NULL;
 228                 }
 229
 230                 timeline->sync_tl = mali_sync_timeline_create(timeline_name);
 231                 if (NULL == timeline->sync_tl) {
 232                         mali_timeline_destroy(timeline);
 233                         return NULL;
 234                 }
 235         }
 236 #endif /* defined(CONFIG_SYNC) */
 237
 238         return timeline;
 239 }
 240
 241 static void mali_timeline_insert_tracker(struct mali_timeline *timeline, struct mali_timeline_tracker *tracker)
 242 {
 243         MALI_DEBUG_ASSERT_POINTER(timeline);
 244         MALI_DEBUG_ASSERT_POINTER(tracker);
 245
 246         if (mali_timeline_is_full(timeline)) {
 247                 /* Don't add tracker if timeline is full. */
 248                 tracker->point = MALI_TIMELINE_NO_POINT;
 249                 return;
 250         }
 251
 252         tracker->timeline = timeline;
 253         tracker->point    = timeline->point_next;
 254
 255         /* Find next available point. */
 256         timeline->point_next++;
 257         if (MALI_TIMELINE_NO_POINT == timeline->point_next) {
 258                 timeline->point_next++;
 259         }
 260
 261         MALI_DEBUG_ASSERT(!mali_timeline_is_empty(timeline));
 262
 263         /* Add tracker as new head on timeline's tracker list. */
 264         if (NULL == timeline->tracker_head) {
 265                 /* Tracker list is empty. */
 266                 MALI_DEBUG_ASSERT(NULL == timeline->tracker_tail);
 267
 268                 timeline->tracker_tail = tracker;
 269
 270                 MALI_DEBUG_ASSERT(NULL == tracker->timeline_next);
 271                 MALI_DEBUG_ASSERT(NULL == tracker->timeline_prev);
 272         } else {
 273                 MALI_DEBUG_ASSERT(NULL == timeline->tracker_head->timeline_next);
 274
 275                 tracker->timeline_prev = timeline->tracker_head;
 276                 timeline->tracker_head->timeline_next = tracker;
 277
 278                 MALI_DEBUG_ASSERT(NULL == tracker->timeline_next);
 279         }
 280         timeline->tracker_head = tracker;
 281
 282         MALI_DEBUG_ASSERT(NULL == timeline->tracker_head->timeline_next);
 283         MALI_DEBUG_ASSERT(NULL == timeline->tracker_tail->timeline_prev);
 284 }
 285
 286 /* Inserting the waiter object into the given timeline */
 287 static void mali_timeline_insert_waiter(struct mali_timeline *timeline, struct mali_timeline_waiter *waiter_new)
 288 {
 289         struct mali_timeline_waiter *waiter_prev;
 290         struct mali_timeline_waiter *waiter_next;
 291
 292         /* Waiter time must be between timeline head and tail, and there must
 293          * be less than MALI_TIMELINE_MAX_POINT_SPAN elements between */
 294         MALI_DEBUG_ASSERT(( waiter_new->point - timeline->point_oldest) < MALI_TIMELINE_MAX_POINT_SPAN);
 295         MALI_DEBUG_ASSERT((-waiter_new->point + timeline->point_next) < MALI_TIMELINE_MAX_POINT_SPAN);
 296
 297         /* Finding out where to put this waiter, in the linked waiter list of the given timeline **/
 298         waiter_prev = timeline->waiter_head; /* Insert new after  waiter_prev */
 299         waiter_next = NULL;                  /* Insert new before waiter_next */
 300
 301         /* Iterating backwards from head (newest) to tail (oldest) until we
 302          * find the correct spot to insert the new waiter */
 303         while (waiter_prev && mali_timeline_point_after(waiter_prev->point, waiter_new->point)) {
 304                 waiter_next = waiter_prev;
 305                 waiter_prev = waiter_prev->timeline_prev;
 306         }
 307
 308         if (NULL == waiter_prev && NULL == waiter_next) {
 309                 /* list is empty */
 310                 timeline->waiter_head = waiter_new;
 311                 timeline->waiter_tail = waiter_new;
 312         } else if (NULL == waiter_next) {
 313                 /* insert at head */
 314                 waiter_new->timeline_prev = timeline->waiter_head;
 315                 timeline->waiter_head->timeline_next = waiter_new;
 316                 timeline->waiter_head = waiter_new;
 317         } else if (NULL == waiter_prev) {
 318                 /* insert at tail */
 319                 waiter_new->timeline_next = timeline->waiter_tail;
 320                 timeline->waiter_tail->timeline_prev = waiter_new;
 321                 timeline->waiter_tail = waiter_new;
 322         } else {
 323                 /* insert between */
 324                 waiter_new->timeline_next = waiter_next;
 325                 waiter_new->timeline_prev = waiter_prev;
 326                 waiter_next->timeline_prev = waiter_new;
 327                 waiter_prev->timeline_next = waiter_new;
 328         }
 329 }
 330
 331 static void mali_timeline_update_delayed_work(struct mali_timeline *timeline)
 332 {
 333         struct mali_timeline_system *system;
 334         struct mali_timeline_tracker *oldest_tracker;
 335
 336         MALI_DEBUG_ASSERT_POINTER(timeline);
 337         MALI_DEBUG_ASSERT(MALI_TIMELINE_SOFT == timeline->id);
 338
 339         system = timeline->system;
 340         MALI_DEBUG_ASSERT_POINTER(system);
 341
 342         MALI_DEBUG_ASSERT(MALI_TIMELINE_SYSTEM_LOCKED(system));
 343
 344         /* Timer is disabled, early out. */
 345         if (!system->timer_enabled) return;
 346
 347         oldest_tracker = timeline->tracker_tail;
 348         if (NULL != oldest_tracker && 0 == oldest_tracker->trigger_ref_count) {
 349                 if (MALI_FALSE == oldest_tracker->timer_active) {
 350                         if (MALI_TRUE == timeline->timer_active) {
 351                                 _mali_osk_wq_delayed_cancel_work_async(timeline->delayed_work);
 352                         }
 353                         _mali_osk_wq_delayed_schedule_work(timeline->delayed_work, MALI_TIMELINE_TIMEOUT_HZ);
 354                         oldest_tracker->timer_active = MALI_TRUE;
 355                         timeline->timer_active = MALI_TRUE;
 356                 }
 357         } else if (MALI_TRUE == timeline->timer_active) {
 358                 _mali_osk_wq_delayed_cancel_work_async(timeline->delayed_work);
 359                 timeline->timer_active = MALI_FALSE;
 360         }
 361 }
 362
 363 static mali_scheduler_mask mali_timeline_update_oldest_point(struct mali_timeline *timeline)
 364 {
 365         mali_scheduler_mask schedule_mask = MALI_SCHEDULER_MASK_EMPTY;
 366
 367         MALI_DEBUG_ASSERT_POINTER(timeline);
 368
 369         MALI_DEBUG_CODE({
 370                 struct mali_timeline_system *system = timeline->system;
 371                 MALI_DEBUG_ASSERT_POINTER(system);
 372
 373                 MALI_DEBUG_ASSERT(MALI_TIMELINE_SYSTEM_LOCKED(system));
 374         });
 375
 376         if (NULL != timeline->tracker_tail) {
 377                 /* Set oldest point to oldest tracker's point */
 378                 timeline->point_oldest = timeline->tracker_tail->point;
 379         } else {
 380                 /* No trackers, mark point list as empty */
 381                 timeline->point_oldest = timeline->point_next;
 382         }
 383
 384         /* Release all waiters no longer on the timeline's point list.
 385          * Releasing a waiter can trigger this function to be called again, so
 386          * we do not store any pointers on stack. */
 387         while (NULL != timeline->waiter_tail) {
 388                 u32 waiter_time_relative;
 389                 u32 time_head_relative;
 390                 struct mali_timeline_waiter *waiter = timeline->waiter_tail;
 391
 392                 time_head_relative = timeline->point_next - timeline->point_oldest;
 393                 waiter_time_relative = waiter->point - timeline->point_oldest;
 394
 395                 if (waiter_time_relative < time_head_relative) {
 396                         /* This and all following waiters are on the point list, so we are done. */
 397                         break;
 398                 }
 399
 400                 /* Remove waiter from timeline's waiter list. */
 401                 if (NULL != waiter->timeline_next) {
 402                         waiter->timeline_next->timeline_prev = NULL;
 403                 } else {
 404                         /* This was the last waiter */
 405                         timeline->waiter_head = NULL;
 406                 }
 407                 timeline->waiter_tail = waiter->timeline_next;
 408
 409                 /* Release waiter.  This could activate a tracker, if this was
 410                  * the last waiter for the tracker. */
 411                 schedule_mask |= mali_timeline_system_release_waiter(timeline->system, waiter);
 412         }
 413
 414         return schedule_mask;
 415 }
 416
 417 void mali_timeline_tracker_init(struct mali_timeline_tracker *tracker,
 418                                 mali_timeline_tracker_type type,
 419                                 struct mali_timeline_fence *fence,
 420                                 void *job)
 421 {
 422         MALI_DEBUG_ASSERT_POINTER(tracker);
 423         MALI_DEBUG_ASSERT_POINTER(job);
 424
 425         MALI_DEBUG_ASSERT(MALI_TIMELINE_TRACKER_MAX > type);
 426
 427         /* Zero out all tracker members. */
 428         _mali_osk_memset(tracker, 0, sizeof(*tracker));
 429
 430         tracker->type = type;
 431         tracker->job = job;
 432         tracker->trigger_ref_count = 1;  /* Prevents any callback from trigging while adding it */
 433         tracker->os_tick_create = _mali_osk_time_tickcount();
 434         MALI_DEBUG_CODE(tracker->magic = MALI_TIMELINE_TRACKER_MAGIC);
 435
 436         tracker->activation_error = MALI_TIMELINE_ACTIVATION_ERROR_NONE;
 437
 438         /* Copy fence. */
 439         if (NULL != fence) {
 440                 _mali_osk_memcpy(&tracker->fence, fence, sizeof(struct mali_timeline_fence));
 441         }
 442 }
 443
 444 mali_scheduler_mask mali_timeline_tracker_release(struct mali_timeline_tracker *tracker)
 445 {
 446         struct mali_timeline *timeline;
 447         struct mali_timeline_system *system;
 448         struct mali_timeline_tracker *tracker_next, *tracker_prev;
 449         mali_scheduler_mask schedule_mask = MALI_SCHEDULER_MASK_EMPTY;
 450         u32 tid = _mali_osk_get_tid();
 451
 452         /* Upon entry a group lock will be held, but not a scheduler lock. */
 453         MALI_DEBUG_ASSERT_POINTER(tracker);
 454         MALI_DEBUG_ASSERT(MALI_TIMELINE_TRACKER_MAGIC == tracker->magic);
 455
 456         /* Tracker should have been triggered */
 457         MALI_DEBUG_ASSERT(0 == tracker->trigger_ref_count);
 458
 459         /* All waiters should have been released at this point */
 460         MALI_DEBUG_ASSERT(NULL == tracker->waiter_head);
 461         MALI_DEBUG_ASSERT(NULL == tracker->waiter_tail);
 462
 463         MALI_DEBUG_PRINT(3, ("Mali Timeline: releasing tracker for job 0x%08X\n", tracker->job));
 464
 465         timeline = tracker->timeline;
 466         if (NULL == timeline) {
 467                 /* Tracker was not on a timeline, there is nothing to release. */
 468                 return MALI_SCHEDULER_MASK_EMPTY;
 469         }
 470
 471         system = timeline->system;
 472         MALI_DEBUG_ASSERT_POINTER(system);
 473
 474         mali_spinlock_reentrant_wait(system->spinlock, tid);
 475
 476         /* Tracker should still be on timeline */
 477         MALI_DEBUG_ASSERT(!mali_timeline_is_empty(timeline));
 478         MALI_DEBUG_ASSERT( mali_timeline_is_point_on(timeline, tracker->point));
 479
 480         /* Tracker is no longer valid. */
 481         MALI_DEBUG_CODE(tracker->magic = 0);
 482
 483         tracker_next = tracker->timeline_next;
 484         tracker_prev = tracker->timeline_prev;
 485         tracker->timeline_next = NULL;
 486         tracker->timeline_prev = NULL;
 487
 488         /* Removing tracker from timeline's tracker list */
 489         if (NULL == tracker_next) {
 490                 /* This tracker was the head */
 491                 timeline->tracker_head = tracker_prev;
 492         } else {
 493                 tracker_next->timeline_prev = tracker_prev;
 494         }
 495
 496         if (NULL == tracker_prev) {
 497                 /* This tracker was the tail */
 498                 timeline->tracker_tail = tracker_next;
 499                 MALI_DEBUG_ASSERT(MALI_TIMELINE_SYSTEM_LOCKED(system));
 500                 /* Update the timeline's oldest time and release any waiters */
 501                 schedule_mask |= mali_timeline_update_oldest_point(timeline);
 502                 MALI_DEBUG_ASSERT(MALI_TIMELINE_SYSTEM_LOCKED(system));
 503         } else {
 504                 tracker_prev->timeline_next = tracker_next;
 505         }
 506
 507         MALI_DEBUG_ASSERT(MALI_TIMELINE_SYSTEM_LOCKED(system));
 508
 509         /* Update delayed work only when it is the soft job timeline */
 510         if (MALI_TIMELINE_SOFT == tracker->timeline->id) {
 511                 mali_timeline_update_delayed_work(tracker->timeline);
 512         }
 513
 514         mali_spinlock_reentrant_signal(system->spinlock, tid);
 515
 516         return schedule_mask;
 517 }
 518
 519 void mali_timeline_system_release_waiter_list(struct mali_timeline_system *system,
 520         struct mali_timeline_waiter *tail,
 521         struct mali_timeline_waiter *head)
 522 {
 523         MALI_DEBUG_ASSERT_POINTER(system);
 524         MALI_DEBUG_ASSERT_POINTER(head);
 525         MALI_DEBUG_ASSERT_POINTER(tail);
 526         MALI_DEBUG_ASSERT(MALI_TIMELINE_SYSTEM_LOCKED(system));
 527
 528         head->tracker_next = system->waiter_empty_list;
 529         system->waiter_empty_list = tail;
 530 }
 531
 532 static mali_scheduler_mask mali_timeline_tracker_activate(struct mali_timeline_tracker *tracker)
 533 {
 534         mali_scheduler_mask schedule_mask = MALI_SCHEDULER_MASK_EMPTY;
 535         struct mali_timeline_system *system;
 536         struct mali_timeline *timeline;
 537         u32 tid = _mali_osk_get_tid();
 538
 539         MALI_DEBUG_ASSERT_POINTER(tracker);
 540         MALI_DEBUG_ASSERT(MALI_TIMELINE_TRACKER_MAGIC == tracker->magic);
 541
 542         system = tracker->system;
 543         MALI_DEBUG_ASSERT_POINTER(system);
 544         MALI_DEBUG_ASSERT(MALI_TIMELINE_SYSTEM_LOCKED(system));
 545
 546         tracker->os_tick_activate = _mali_osk_time_tickcount();
 547
 548         if (NULL != tracker->waiter_head) {
 549                 mali_timeline_system_release_waiter_list(system, tracker->waiter_tail, tracker->waiter_head);
 550                 tracker->waiter_head = NULL;
 551                 tracker->waiter_tail = NULL;
 552         }
 553
 554         switch (tracker->type) {
 555         case MALI_TIMELINE_TRACKER_GP:
 556                 schedule_mask = mali_gp_scheduler_activate_job((struct mali_gp_job *) tracker->job);
 557                 break;
 558         case MALI_TIMELINE_TRACKER_PP:
 559                 schedule_mask = mali_pp_scheduler_activate_job((struct mali_pp_job *) tracker->job);
 560                 break;
 561         case MALI_TIMELINE_TRACKER_SOFT:
 562                 timeline = tracker->timeline;
 563                 MALI_DEBUG_ASSERT_POINTER(timeline);
 564
 565                 mali_soft_job_system_activate_job((struct mali_soft_job *) tracker->job);
 566
 567                 /* Start a soft timer to make sure the soft job be released in a limited time */
 568                 mali_spinlock_reentrant_wait(system->spinlock, tid);
 569                 mali_timeline_update_delayed_work(timeline);
 570                 mali_spinlock_reentrant_signal(system->spinlock, tid);
 571                 break;
 572         case MALI_TIMELINE_TRACKER_WAIT:
 573                 mali_timeline_fence_wait_activate((struct mali_timeline_fence_wait_tracker *) tracker->job);
 574                 break;
 575         case MALI_TIMELINE_TRACKER_SYNC:
 576 #if defined(CONFIG_SYNC)
 577                 mali_timeline_sync_fence_activate((struct mali_timeline_sync_fence_tracker *) tracker->job);
 578 #else
 579                 MALI_PRINT_ERROR(("Mali Timeline: sync tracker not supported\n", tracker->type));
 580 #endif /* defined(CONFIG_SYNC) */
 581                 break;
 582         default:
 583                 MALI_PRINT_ERROR(("Mali Timeline - Illegal tracker type: %d\n", tracker->type));
 584                 break;
 585         }
 586
 587         return schedule_mask;
 588 }
 589
 590 void mali_timeline_system_tracker_get(struct mali_timeline_system *system, struct mali_timeline_tracker *tracker)
 591 {
 592         u32 tid = _mali_osk_get_tid();
 593
 594         MALI_DEBUG_ASSERT_POINTER(tracker);
 595         MALI_DEBUG_ASSERT_POINTER(system);
 596
 597         mali_spinlock_reentrant_wait(system->spinlock, tid);
 598
 599         MALI_DEBUG_ASSERT(0 < tracker->trigger_ref_count);
 600         tracker->trigger_ref_count++;
 601
 602         mali_spinlock_reentrant_signal(system->spinlock, tid);
 603 }
 604
 605 mali_scheduler_mask mali_timeline_system_tracker_put(struct mali_timeline_system *system, struct mali_timeline_tracker *tracker, mali_timeline_activation_error activation_error)
 606 {
 607         u32 tid = _mali_osk_get_tid();
 608         mali_scheduler_mask schedule_mask = MALI_SCHEDULER_MASK_EMPTY;
 609
 610         MALI_DEBUG_ASSERT_POINTER(tracker);
 611         MALI_DEBUG_ASSERT_POINTER(system);
 612
 613         mali_spinlock_reentrant_wait(system->spinlock, tid);
 614
 615         MALI_DEBUG_ASSERT(0 < tracker->trigger_ref_count);
 616         tracker->trigger_ref_count--;
 617
 618         tracker->activation_error |= activation_error;
 619
 620         if (0 == tracker->trigger_ref_count) {
 621                 schedule_mask |= mali_timeline_tracker_activate(tracker);
 622                 tracker = NULL;
 623         }
 624
 625         mali_spinlock_reentrant_signal(system->spinlock, tid);
 626
 627         return schedule_mask;
 628 }
 629
 630 void mali_timeline_fence_copy_uk_fence(struct mali_timeline_fence *fence, _mali_uk_fence_t *uk_fence)
 631 {
 632         u32 i;
 633
 634         MALI_DEBUG_ASSERT_POINTER(fence);
 635         MALI_DEBUG_ASSERT_POINTER(uk_fence);
 636
 637         for (i = 0; i < MALI_TIMELINE_MAX; ++i) {
 638                 fence->points[i] = uk_fence->points[i];
 639         }
 640
 641         fence->sync_fd = uk_fence->sync_fd;
 642 }
 643
 644 struct mali_timeline_system *mali_timeline_system_create(struct mali_session_data *session)
 645 {
 646         u32 i;
 647         struct mali_timeline_system *system;
 648
 649         MALI_DEBUG_ASSERT_POINTER(session);
 650         MALI_DEBUG_PRINT(4, ("Mali Timeline: creating timeline system\n"));
 651
 652         system = (struct mali_timeline_system *) _mali_osk_calloc(1, sizeof(struct mali_timeline_system));
 653         if (NULL == system) {
 654                 return NULL;
 655         }
 656
 657         system->spinlock = mali_spinlock_reentrant_init(_MALI_OSK_LOCK_ORDER_TIMELINE_SYSTEM);
 658         if (NULL == system->spinlock) {
 659                 mali_timeline_system_destroy(system);
 660                 return NULL;
 661         }
 662
 663         for (i = 0; i < MALI_TIMELINE_MAX; ++i) {
 664                 system->timelines[i] = mali_timeline_create(system, (enum mali_timeline_id)i);
 665                 if (NULL == system->timelines[i]) {
 666                         mali_timeline_system_destroy(system);
 667                         return NULL;
 668                 }
 669         }
 670
 671 #if defined(CONFIG_SYNC)
 672         system->signaled_sync_tl = mali_sync_timeline_create("mali-always-signaled");
 673         if (NULL == system->signaled_sync_tl) {
 674                 mali_timeline_system_destroy(system);
 675                 return NULL;
 676         }
 677 #endif /* defined(CONFIG_SYNC) */
 678
 679         system->waiter_empty_list = NULL;
 680         system->session = session;
 681         system->timer_enabled = MALI_TRUE;
 682
 683         system->wait_queue = _mali_osk_wait_queue_init();
 684         if (NULL == system->wait_queue) {
 685                 mali_timeline_system_destroy(system);
 686                 return NULL;
 687         }
 688
 689         return system;
 690 }
 691
 692 #if defined(CONFIG_SYNC)
 693
 694 /**
 695  * Check if there are any trackers left on timeline.
 696  *
 697  * Used as a wait queue conditional.
 698  *
 699  * @param data Timeline.
 700  * @return MALI_TRUE if there are no trackers on timeline, MALI_FALSE if not.
 701  */
 702 static mali_bool mali_timeline_has_no_trackers(void *data)
 703 {
 704         struct mali_timeline *timeline = (struct mali_timeline *) data;
 705
 706         MALI_DEBUG_ASSERT_POINTER(timeline);
 707
 708         return mali_timeline_is_empty(timeline);
 709 }
 710
 711 /**
 712  * Cancel sync fence waiters waited upon by trackers on all timelines.
 713  *
 714  * Will return after all timelines have no trackers left.
 715  *
 716  * @param system Timeline system.
 717  */
 718 static void mali_timeline_cancel_sync_fence_waiters(struct mali_timeline_system *system)
 719 {
 720         u32 i;
 721         u32 tid = _mali_osk_get_tid();
 722         struct mali_timeline_tracker *tracker, *tracker_next;
 723         _MALI_OSK_LIST_HEAD_STATIC_INIT(tracker_list);
 724
 725         MALI_DEBUG_ASSERT_POINTER(system);
 726         MALI_DEBUG_ASSERT_POINTER(system->session);
 727         MALI_DEBUG_ASSERT(system->session->is_aborting);
 728
 729         mali_spinlock_reentrant_wait(system->spinlock, tid);
 730
 731         /* Cancel sync fence waiters. */
 732         for (i = 0; i < MALI_TIMELINE_MAX; ++i) {
 733                 struct mali_timeline *timeline = system->timelines[i];
 734
 735                 MALI_DEBUG_ASSERT_POINTER(timeline);
 736
 737                 tracker_next = timeline->tracker_tail;
 738                 while (NULL != tracker_next) {
 739                         tracker = tracker_next;
 740                         tracker_next = tracker->timeline_next;
 741
 742                         if (NULL == tracker->sync_fence) continue;
 743
 744                         MALI_DEBUG_PRINT(3, ("Mali Timeline: Cancelling sync fence wait for tracker 0x%08X.\n", tracker));
 745
 746                         /* Cancel sync fence waiter. */
 747                         if (0 == sync_fence_cancel_async(tracker->sync_fence, &tracker->sync_fence_waiter)) {
 748                                 /* Callback was not called, move tracker to local list. */
 749                                 _mali_osk_list_add(&tracker->sync_fence_cancel_list, &tracker_list);
 750                         }
 751                 }
 752         }
 753
 754         mali_spinlock_reentrant_signal(system->spinlock, tid);
 755
 756         /* Manually call sync fence callback in order to release waiter and trigger activation of tracker. */
 757         _MALI_OSK_LIST_FOREACHENTRY(tracker, tracker_next, &tracker_list, struct mali_timeline_tracker, sync_fence_cancel_list) {
 758                 mali_timeline_sync_fence_callback(tracker->sync_fence, &tracker->sync_fence_waiter);
 759         }
 760
 761         /* Sleep until all sync fence callbacks are done and all timelines are empty. */
 762         for (i = 0; i < MALI_TIMELINE_MAX; ++i) {
 763                 struct mali_timeline *timeline = system->timelines[i];
 764
 765                 MALI_DEBUG_ASSERT_POINTER(timeline);
 766
 767                 _mali_osk_wait_queue_wait_event(system->wait_queue, mali_timeline_has_no_trackers, (void *) timeline);
 768         }
 769 }
 770
 771 #endif /* defined(CONFIG_SYNC) */
 772
 773 void mali_timeline_system_abort(struct mali_timeline_system *system)
 774 {
 775         MALI_DEBUG_CODE(u32 tid = _mali_osk_get_tid(););
 776
 777         MALI_DEBUG_ASSERT_POINTER(system);
 778         MALI_DEBUG_ASSERT_POINTER(system->session);
 779         MALI_DEBUG_ASSERT(system->session->is_aborting);
 780
 781         MALI_DEBUG_PRINT(3, ("Mali Timeline: Aborting timeline system for session 0x%08X.\n", system->session));
 782
 783 #if defined(CONFIG_SYNC)
 784         mali_timeline_cancel_sync_fence_waiters(system);
 785 #endif /* defined(CONFIG_SYNC) */
 786
 787         /* Should not be any waiters or trackers left at this point. */
 788         MALI_DEBUG_CODE( {
 789                 u32 i;
 790                 mali_spinlock_reentrant_wait(system->spinlock, tid);
 791                 for (i = 0; i < MALI_TIMELINE_MAX; ++i)
 792                 {
 793                         struct mali_timeline *timeline = system->timelines[i];
 794                         MALI_DEBUG_ASSERT_POINTER(timeline);
 795                         MALI_DEBUG_ASSERT(timeline->point_oldest == timeline->point_next);
 796                         MALI_DEBUG_ASSERT(NULL == timeline->tracker_head);
 797                         MALI_DEBUG_ASSERT(NULL == timeline->tracker_tail);
 798                         MALI_DEBUG_ASSERT(NULL == timeline->waiter_head);
 799                         MALI_DEBUG_ASSERT(NULL == timeline->waiter_tail);
 800                 }
 801                 mali_spinlock_reentrant_signal(system->spinlock, tid);
 802         });
 803 }
 804
 805 void mali_timeline_system_destroy(struct mali_timeline_system *system)
 806 {
 807         u32 i;
 808         struct mali_timeline_waiter *waiter, *next;
 809
 810         MALI_DEBUG_ASSERT_POINTER(system);
 811         MALI_DEBUG_ASSERT_POINTER(system->session);
 812
 813         MALI_DEBUG_PRINT(4, ("Mali Timeline: destroying timeline system\n"));
 814
 815         if (NULL != system) {
 816                 /* There should be no waiters left on this queue. */
 817                 if (NULL != system->wait_queue) {
 818                         _mali_osk_wait_queue_term(system->wait_queue);
 819                         system->wait_queue = NULL;
 820                 }
 821
 822                 /* Free all waiters in empty list */
 823                 waiter = system->waiter_empty_list;
 824                 while (NULL != waiter) {
 825                         next = waiter->tracker_next;
 826                         _mali_osk_free(waiter);
 827                         waiter = next;
 828                 }
 829
 830 #if defined(CONFIG_SYNC)
 831                 if (NULL != system->signaled_sync_tl) {
 832                         sync_timeline_destroy(system->signaled_sync_tl);
 833                 }
 834 #endif /* defined(CONFIG_SYNC) */
 835
 836                 for (i = 0; i < MALI_TIMELINE_MAX; ++i) {
 837                         if (NULL != system->timelines[i]) {
 838                                 mali_timeline_destroy(system->timelines[i]);
 839                         }
 840                 }
 841                 if (NULL != system->spinlock) {
 842                         mali_spinlock_reentrant_term(system->spinlock);
 843                 }
 844
 845                 _mali_osk_free(system);
 846         }
 847 }
 848
 849 /**
 850  * Find how many waiters are needed for a given fence.
 851  *
 852  * @param fence The fence to check.
 853  * @return Number of waiters needed for fence.
 854  */
 855 static u32 mali_timeline_fence_num_waiters(struct mali_timeline_fence *fence)
 856 {
 857         u32 i, num_waiters = 0;
 858
 859         MALI_DEBUG_ASSERT_POINTER(fence);
 860
 861         for (i = 0; i < MALI_TIMELINE_MAX; ++i) {
 862                 if (MALI_TIMELINE_NO_POINT != fence->points[i]) {
 863                         ++num_waiters;
 864                 }
 865         }
 866
 867 #if defined(CONFIG_SYNC)
 868         if (-1 != fence->sync_fd) ++num_waiters;
 869 #endif /* defined(CONFIG_SYNC) */
 870
 871         return num_waiters;
 872 }
 873
 874 static struct mali_timeline_waiter *mali_timeline_system_get_zeroed_waiter(struct mali_timeline_system *system)
 875 {
 876         struct mali_timeline_waiter *waiter;
 877
 878         MALI_DEBUG_ASSERT_POINTER(system);
 879         MALI_DEBUG_ASSERT(MALI_TIMELINE_SYSTEM_LOCKED(system));
 880
 881         waiter = system->waiter_empty_list;
 882         if (NULL != waiter) {
 883                 /* Remove waiter from empty list and zero it */
 884                 system->waiter_empty_list = waiter->tracker_next;
 885                 _mali_osk_memset(waiter, 0, sizeof(*waiter));
 886         }
 887
 888         /* Return NULL if list was empty. */
 889         return waiter;
 890 }
 891
 892 static void mali_timeline_system_allocate_waiters(struct mali_timeline_system *system,
 893         struct mali_timeline_waiter **tail,
 894         struct mali_timeline_waiter **head,
 895         int max_num_waiters)
 896 {
 897         u32 i, tid = _mali_osk_get_tid();
 898         mali_bool do_alloc;
 899         struct mali_timeline_waiter *waiter;
 900
 901         MALI_DEBUG_ASSERT_POINTER(system);
 902         MALI_DEBUG_ASSERT_POINTER(tail);
 903         MALI_DEBUG_ASSERT_POINTER(head);
 904
 905         MALI_DEBUG_ASSERT(MALI_TIMELINE_SYSTEM_LOCKED(system));
 906
 907         *head = *tail = NULL;
 908         do_alloc = MALI_FALSE;
 909         i = 0;
 910         while (i < max_num_waiters) {
 911                 if (MALI_FALSE == do_alloc) {
 912                         waiter = mali_timeline_system_get_zeroed_waiter(system);
 913                         if (NULL == waiter) {
 914                                 do_alloc = MALI_TRUE;
 915                                 mali_spinlock_reentrant_signal(system->spinlock, tid);
 916                                 continue;
 917                         }
 918                 } else {
 919                         waiter = _mali_osk_calloc(1, sizeof(struct mali_timeline_waiter));
 920                         if (NULL == waiter) break;
 921                 }
 922                 ++i;
 923                 if (NULL == *tail) {
 924                         *tail = waiter;
 925                         *head = waiter;
 926                 } else {
 927                         (*head)->tracker_next = waiter;
 928                         *head = waiter;
 929                 }
 930         }
 931         if (MALI_TRUE == do_alloc) {
 932                 mali_spinlock_reentrant_wait(system->spinlock, tid);
 933         }
 934 }
 935
 936 /**
 937  * Create waiters for the given tracker. The tracker is activated when all waiters are release.
 938  *
 939  * @note Tracker can potentially be activated before this function returns.
 940  *
 941  * @param system Timeline system.
 942  * @param tracker Tracker we will create waiters for.
 943  * @param waiter_tail List of pre-allocated waiters.
 944  * @param waiter_head List of pre-allocated waiters.
 945  */
 946 static void mali_timeline_system_create_waiters_and_unlock(struct mali_timeline_system *system,
 947         struct mali_timeline_tracker *tracker,
 948         struct mali_timeline_waiter *waiter_tail,
 949         struct mali_timeline_waiter *waiter_head)
 950 {
 951         int i;
 952         u32 tid = _mali_osk_get_tid();
 953         mali_scheduler_mask schedule_mask = MALI_SCHEDULER_MASK_EMPTY;
 954 #if defined(CONFIG_SYNC)
 955         struct sync_fence *sync_fence = NULL;
 956 #endif /* defined(CONFIG_SYNC) */
 957
 958         MALI_DEBUG_ASSERT_POINTER(system);
 959         MALI_DEBUG_ASSERT_POINTER(tracker);
 960
 961         MALI_DEBUG_ASSERT(MALI_TIMELINE_SYSTEM_LOCKED(system));
 962
 963         MALI_DEBUG_ASSERT(NULL == tracker->waiter_head);
 964         MALI_DEBUG_ASSERT(NULL == tracker->waiter_tail);
 965         MALI_DEBUG_ASSERT(NULL != tracker->job);
 966
 967         /* Creating waiter object for all the timelines the fence is put on. Inserting this waiter
 968          * into the timelines sorted list of waiters */
 969         for (i = 0; i < MALI_TIMELINE_MAX; ++i) {
 970                 mali_timeline_point point;
 971                 struct mali_timeline *timeline;
 972                 struct mali_timeline_waiter *waiter;
 973
 974                 /* Get point on current timeline from tracker's fence. */
 975                 point = tracker->fence.points[i];
 976
 977                 if (likely(MALI_TIMELINE_NO_POINT == point)) {
 978                         /* Fence contains no point on this timeline so we don't need a waiter. */
 979                         continue;
 980                 }
 981
 982                 timeline = system->timelines[i];
 983                 MALI_DEBUG_ASSERT_POINTER(timeline);
 984
 985                 if (unlikely(!mali_timeline_is_point_valid(timeline, point))) {
 986                         MALI_PRINT_ERROR(("Mali Timeline: point %d is not valid (oldest=%d, next=%d)\n",
 987                                           point, timeline->point_oldest, timeline->point_next));
 988                         continue;
 989                 }
 990
 991                 if (likely(mali_timeline_is_point_released(timeline, point))) {
 992                         /* Tracker representing the point has been released so we don't need a
 993                          * waiter. */
 994                         continue;
 995                 }
 996
 997                 /* The point is on timeline. */
 998                 MALI_DEBUG_ASSERT(mali_timeline_is_point_on(timeline, point));
 999
1000                 /* Get a new zeroed waiter object. */
1001                 if (likely(NULL != waiter_tail)) {
1002                         waiter = waiter_tail;
1003                         waiter_tail = waiter_tail->tracker_next;
1004                 } else {
1005                         MALI_PRINT_ERROR(("Mali Timeline: failed to allocate memory for waiter\n"));
1006                         continue;
1007                 }
1008
1009                 /* Yanking the trigger ref count of the tracker. */
1010                 tracker->trigger_ref_count++;
1011
1012                 waiter->point   = point;
1013                 waiter->tracker = tracker;
1014
1015                 /* Insert waiter on tracker's singly-linked waiter list. */
1016                 if (NULL == tracker->waiter_head) {
1017                         /* list is empty */
1018                         MALI_DEBUG_ASSERT(NULL == tracker->waiter_tail);
1019                         tracker->waiter_tail = waiter;
1020                 } else {
1021                         tracker->waiter_head->tracker_next = waiter;
1022                 }
1023                 tracker->waiter_head = waiter;
1024
1025                 /* Add waiter to timeline. */
1026                 mali_timeline_insert_waiter(timeline, waiter);
1027         }
1028 #if defined(CONFIG_SYNC)
1029         if (-1 != tracker->fence.sync_fd) {
1030                 int ret;
1031                 struct mali_timeline_waiter *waiter;
1032
1033                 sync_fence = sync_fence_fdget(tracker->fence.sync_fd);
1034                 if (unlikely(NULL == sync_fence)) {
1035                         MALI_PRINT_ERROR(("Mali Timeline: failed to get sync fence from fd %d\n", tracker->fence.sync_fd));
1036                         goto exit;
1037                 }
1038
1039                 /* Check if we have a zeroed waiter object available. */
1040                 if (unlikely(NULL == waiter_tail)) {
1041                         MALI_PRINT_ERROR(("Mali Timeline: failed to allocate memory for waiter\n"));
1042                         goto exit;
1043                 }
1044
1045                 /* Start asynchronous wait that will release waiter when the fence is signaled. */
1046                 sync_fence_waiter_init(&tracker->sync_fence_waiter, mali_timeline_sync_fence_callback);
1047                 ret = sync_fence_wait_async(sync_fence, &tracker->sync_fence_waiter);
1048                 if (1 == ret) {
1049                         /* Fence already signaled, no waiter needed. */
1050                         goto exit;
1051                 } else if (0 != ret) {
1052                         MALI_PRINT_ERROR(("Mali Timeline: sync fence fd %d signaled with error %d\n", tracker->fence.sync_fd, ret));
1053                         tracker->activation_error |= MALI_TIMELINE_ACTIVATION_ERROR_SYNC_BIT;
1054                         goto exit;
1055                 }
1056
1057                 /* Grab new zeroed waiter object. */
1058                 waiter = waiter_tail;
1059                 waiter_tail = waiter_tail->tracker_next;
1060
1061                 /* Increase the trigger ref count of the tracker. */
1062                 tracker->trigger_ref_count++;
1063
1064                 waiter->point   = MALI_TIMELINE_NO_POINT;
1065                 waiter->tracker = tracker;
1066
1067                 /* Insert waiter on tracker's singly-linked waiter list. */
1068                 if (NULL == tracker->waiter_head) {
1069                         /* list is empty */
1070                         MALI_DEBUG_ASSERT(NULL == tracker->waiter_tail);
1071                         tracker->waiter_tail = waiter;
1072                 } else {
1073                         tracker->waiter_head->tracker_next = waiter;
1074                 }
1075                 tracker->waiter_head = waiter;
1076
1077                 /* Also store waiter in separate field for easy access by sync callback. */
1078                 tracker->waiter_sync = waiter;
1079
1080                 /* Store the sync fence in tracker so we can retrieve in abort session, if needed. */
1081                 tracker->sync_fence = sync_fence;
1082
1083                 sync_fence = NULL;
1084         }
1085 exit:
1086 #endif /* defined(CONFIG_SYNC) */
1087
1088         if (NULL != waiter_tail) {
1089                 mali_timeline_system_release_waiter_list(system, waiter_tail, waiter_head);
1090         }
1091
1092         /* Release the initial trigger ref count. */
1093         tracker->trigger_ref_count--;
1094
1095         /* If there were no waiters added to this tracker we activate immediately. */
1096         if (0 == tracker->trigger_ref_count) {
1097                 schedule_mask |= mali_timeline_tracker_activate(tracker);
1098         }
1099
1100         mali_spinlock_reentrant_signal(system->spinlock, tid);
1101
1102 #if defined(CONFIG_SYNC)
1103         if (NULL != sync_fence) {
1104                 sync_fence_put(sync_fence);
1105         }
1106 #endif /* defined(CONFIG_SYNC) */
1107
1108         mali_scheduler_schedule_from_mask(schedule_mask, MALI_FALSE);
1109 }
1110
1111 mali_timeline_point mali_timeline_system_add_tracker(struct mali_timeline_system *system,
1112         struct mali_timeline_tracker *tracker,
1113         enum mali_timeline_id timeline_id)
1114 {
1115         int num_waiters = 0;
1116         struct mali_timeline_waiter *waiter_tail, *waiter_head;
1117         u32 tid = _mali_osk_get_tid();
1118         mali_timeline_point point = MALI_TIMELINE_NO_POINT;
1119
1120         MALI_DEBUG_ASSERT_POINTER(system);
1121         MALI_DEBUG_ASSERT_POINTER(system->session);
1122         MALI_DEBUG_ASSERT_POINTER(tracker);
1123
1124         MALI_DEBUG_ASSERT(MALI_FALSE == system->session->is_aborting);
1125         MALI_DEBUG_ASSERT(MALI_TIMELINE_TRACKER_MAX > tracker->type);
1126         MALI_DEBUG_ASSERT(MALI_TIMELINE_TRACKER_MAGIC == tracker->magic);
1127
1128         MALI_DEBUG_PRINT(4, ("Mali Timeline: adding tracker for job %p, timeline: %d\n", tracker->job, timeline_id));
1129
1130         MALI_DEBUG_ASSERT(0 < tracker->trigger_ref_count);
1131         tracker->system = system;
1132
1133         mali_spinlock_reentrant_wait(system->spinlock, tid);
1134
1135         num_waiters = mali_timeline_fence_num_waiters(&tracker->fence);
1136
1137         /* Allocate waiters. */
1138         mali_timeline_system_allocate_waiters(system, &waiter_tail, &waiter_head, num_waiters);
1139         MALI_DEBUG_ASSERT(MALI_TIMELINE_SYSTEM_LOCKED(system));
1140
1141         /* Add tracker to timeline.  This will allocate a point for the tracker on the timeline. If
1142          * timeline ID is MALI_TIMELINE_NONE the tracker will NOT be added to a timeline and the
1143          * point will be MALI_TIMELINE_NO_POINT.
1144          *
1145          * NOTE: the tracker can fail to be added if the timeline is full.  If this happens, the
1146          * point will be MALI_TIMELINE_NO_POINT. */
1147         MALI_DEBUG_ASSERT(timeline_id < MALI_TIMELINE_MAX || timeline_id == MALI_TIMELINE_NONE);
1148         if (likely(timeline_id < MALI_TIMELINE_MAX)) {
1149                 struct mali_timeline *timeline = system->timelines[timeline_id];
1150                 mali_timeline_insert_tracker(timeline, tracker);
1151                 MALI_DEBUG_ASSERT(!mali_timeline_is_empty(timeline));
1152         }
1153
1154         point = tracker->point;
1155
1156         /* Create waiters for tracker based on supplied fence.  Each waiter will increase the
1157          * trigger ref count. */
1158         mali_timeline_system_create_waiters_and_unlock(system, tracker, waiter_tail, waiter_head);
1159         tracker = NULL;
1160
1161         /* At this point the tracker object might have been freed so we should no longer
1162          * access it. */
1163
1164
1165         /* The tracker will always be activated after calling add_tracker, even if NO_POINT is
1166          * returned. */
1167         return point;
1168 }
1169
1170 static mali_scheduler_mask mali_timeline_system_release_waiter(struct mali_timeline_system *system,
1171         struct mali_timeline_waiter *waiter)
1172 {
1173         struct mali_timeline_tracker *tracker;
1174         mali_scheduler_mask schedule_mask = MALI_SCHEDULER_MASK_EMPTY;
1175
1176         MALI_DEBUG_ASSERT_POINTER(system);
1177         MALI_DEBUG_ASSERT_POINTER(waiter);
1178
1179         MALI_DEBUG_ASSERT(MALI_TIMELINE_SYSTEM_LOCKED(system));
1180
1181         tracker = waiter->tracker;
1182         MALI_DEBUG_ASSERT_POINTER(tracker);
1183
1184         /* At this point the waiter has been removed from the timeline's waiter list, but it is
1185          * still on the tracker's waiter list.  All of the tracker's waiters will be released when
1186          * the tracker is activated. */
1187
1188         waiter->point   = MALI_TIMELINE_NO_POINT;
1189         waiter->tracker = NULL;
1190
1191         tracker->trigger_ref_count--;
1192         if (0 == tracker->trigger_ref_count) {
1193                 /* This was the last waiter; activate tracker */
1194                 schedule_mask |= mali_timeline_tracker_activate(tracker);
1195                 tracker = NULL;
1196         }
1197
1198         return schedule_mask;
1199 }
1200
1201 mali_timeline_point mali_timeline_system_get_latest_point(struct mali_timeline_system *system,
1202         enum mali_timeline_id timeline_id)
1203 {
1204         mali_timeline_point point;
1205         struct mali_timeline *timeline;
1206         u32 tid = _mali_osk_get_tid();
1207
1208         MALI_DEBUG_ASSERT_POINTER(system);
1209
1210         if (MALI_TIMELINE_MAX <= timeline_id) {
1211                 return MALI_TIMELINE_NO_POINT;
1212         }
1213
1214         mali_spinlock_reentrant_wait(system->spinlock, tid);
1215
1216         timeline = system->timelines[timeline_id];
1217         MALI_DEBUG_ASSERT_POINTER(timeline);
1218
1219         point = MALI_TIMELINE_NO_POINT;
1220         if (timeline->point_oldest != timeline->point_next) {
1221                 point = timeline->point_next - 1;
1222                 if (MALI_TIMELINE_NO_POINT == point) point--;
1223         }
1224
1225         mali_spinlock_reentrant_signal(system->spinlock, tid);
1226
1227         return point;
1228 }
1229
1230 #if defined(MALI_TIMELINE_DEBUG_FUNCTIONS)
1231
1232 static mali_bool is_waiting_on_timeline(struct mali_timeline_tracker *tracker, enum mali_timeline_id id)
1233 {
1234         struct mali_timeline *timeline;
1235         struct mali_timeline_system *system;
1236
1237         MALI_DEBUG_ASSERT_POINTER(tracker);
1238
1239         MALI_DEBUG_ASSERT_POINTER(tracker->timeline);
1240         timeline = tracker->timeline;
1241
1242         MALI_DEBUG_ASSERT_POINTER(timeline->system);
1243         system = timeline->system;
1244
1245         if (MALI_TIMELINE_MAX > id) {
1246                 return mali_timeline_is_point_on(system->timelines[id], tracker->fence.points[id]);
1247         } else {
1248                 MALI_DEBUG_ASSERT(MALI_TIMELINE_NONE == id);
1249                 return MALI_FALSE;
1250         }
1251 }
1252
1253 static const char *timeline_id_to_string(enum mali_timeline_id id)
1254 {
1255         switch (id) {
1256         case MALI_TIMELINE_GP:
1257                 return "  GP";
1258         case MALI_TIMELINE_PP:
1259                 return "  PP";
1260         case MALI_TIMELINE_SOFT:
1261                 return "SOFT";
1262         default:
1263                 return "NONE";
1264         }
1265 }
1266
1267 static const char *timeline_tracker_type_to_string(enum mali_timeline_tracker_type type)
1268 {
1269         switch (type) {
1270         case MALI_TIMELINE_TRACKER_GP:
1271                 return "  GP";
1272         case MALI_TIMELINE_TRACKER_PP:
1273                 return "  PP";
1274         case MALI_TIMELINE_TRACKER_SOFT:
1275                 return "SOFT";
1276         case MALI_TIMELINE_TRACKER_WAIT:
1277                 return "WAIT";
1278         case MALI_TIMELINE_TRACKER_SYNC:
1279                 return "SYNC";
1280         default:
1281                 return "INVALID";
1282         }
1283 }
1284
1285 mali_timeline_tracker_state mali_timeline_debug_get_tracker_state(struct mali_timeline_tracker *tracker)
1286 {
1287         struct mali_timeline *timeline = NULL;
1288
1289         MALI_DEBUG_ASSERT_POINTER(tracker);
1290         timeline = tracker->timeline;
1291
1292         if (0 != tracker->trigger_ref_count) {
1293                 return MALI_TIMELINE_TS_WAITING;
1294         }
1295
1296         if (timeline && (timeline->tracker_tail == tracker || NULL != tracker->timeline_prev)) {
1297                 return MALI_TIMELINE_TS_ACTIVE;
1298         }
1299
1300         if (timeline && (MALI_TIMELINE_NO_POINT == tracker->point)) {
1301                 return MALI_TIMELINE_TS_INIT;
1302         }
1303
1304         return MALI_TIMELINE_TS_FINISH;
1305 }
1306
1307 void mali_timeline_debug_print_tracker(struct mali_timeline_tracker *tracker)
1308 {
1309         const char *tracker_state = "IWAF";
1310
1311         MALI_DEBUG_ASSERT_POINTER(tracker);
1312
1313         if (0 != tracker->trigger_ref_count) {
1314                 MALI_PRINTF(("TL:  %s %u %c - ref_wait:%u [%s%u,%s%u,%s%u,%d]  (0x%08X)\n",
1315                              timeline_tracker_type_to_string(tracker->type), tracker->point,
1316                              *(tracker_state + mali_timeline_debug_get_tracker_state(tracker)),
1317                              tracker->trigger_ref_count,
1318                              is_waiting_on_timeline(tracker, MALI_TIMELINE_GP) ? "W" : " ", tracker->fence.points[0],
1319                              is_waiting_on_timeline(tracker, MALI_TIMELINE_PP) ? "W" : " ", tracker->fence.points[1],
1320                              is_waiting_on_timeline(tracker, MALI_TIMELINE_SOFT) ? "W" : " ", tracker->fence.points[2],
1321                              tracker->fence.sync_fd, tracker->job));
1322         } else {
1323                 MALI_PRINTF(("TL:  %s %u %c  (0x%08X)\n",
1324                              timeline_tracker_type_to_string(tracker->type), tracker->point,
1325                              *(tracker_state + mali_timeline_debug_get_tracker_state(tracker)),
1326                              tracker->job));
1327         }
1328 }
1329
1330 void mali_timeline_debug_print_timeline(struct mali_timeline *timeline)
1331 {
1332         struct mali_timeline_tracker *tracker = NULL;
1333         int i_max = 30;
1334
1335         MALI_DEBUG_ASSERT_POINTER(timeline);
1336
1337         tracker = timeline->tracker_tail;
1338         while (NULL != tracker && 0 < --i_max) {
1339                 mali_timeline_debug_print_tracker(tracker);
1340                 tracker = tracker->timeline_next;
1341         }
1342
1343         if (0 == i_max) {
1344                 MALI_PRINTF(("TL: Too many trackers in list to print\n"));
1345         }
1346 }
1347
1348 void mali_timeline_debug_print_system(struct mali_timeline_system *system)
1349 {
1350         int i;
1351         int num_printed = 0;
1352
1353         MALI_DEBUG_ASSERT_POINTER(system);
1354
1355         /* Print all timelines */
1356         for (i = 0; i < MALI_TIMELINE_MAX; ++i) {
1357                 struct mali_timeline *timeline = system->timelines[i];
1358
1359                 MALI_DEBUG_ASSERT_POINTER(timeline);
1360
1361                 if (NULL == timeline->tracker_head) continue;
1362
1363                 MALI_PRINTF(("TL: Timeline %s:\n",
1364                              timeline_id_to_string((enum mali_timeline_id)i)));
1365                 mali_timeline_debug_print_timeline(timeline);
1366                 num_printed++;
1367         }
1368
1369         if (0 == num_printed) {
1370                 MALI_PRINTF(("TL: All timelines empty\n"));
1371         }
1372 }
1373
1374 #endif /* defined(MALI_TIMELINE_DEBUG_FUNCTIONS) */