drivers/gpu/arm/mali400/r5p0_rel0/linux/mali_kernel_linux.c

   1 /**
   2  * Copyright (C) 2010-2014 ARM Limited. All rights reserved.
   3  *
   4  * This program is free software and is provided to you under the terms of the GNU General Public License version 2
   5  * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence.
   6  *
   7  * A copy of the licence is included with the program, and can also be obtained from Free Software
   8  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
   9  */
  10
  11 /**
  12  * @file mali_kernel_linux.c
  13  * Implementation of the Linux device driver entrypoints
  14  */
  15 #include <linux/module.h>   /* kernel module definitions */
  16 #include <linux/fs.h>       /* file system operations */
  17 #include <linux/cdev.h>     /* character device definitions */
  18 #include <linux/mm.h>       /* memory manager definitions */
  19 #include <linux/mali/mali_utgard_ioctl.h>
  20 #include <linux/version.h>
  21 #include <linux/device.h>
  22 #include "mali_kernel_license.h"
  23 #include <linux/platform_device.h>
  24 #include <linux/miscdevice.h>
  25 #include <linux/bug.h>
  26 #include <linux/of.h>
  27 #ifdef CONFIG_PM_RUNTIME
  28 #include <linux/pm_runtime.h>
  29 #endif
  30 #include <linux/mali/mali_utgard.h>
  31 #include "mali_kernel_common.h"
  32 #include "mali_session.h"
  33 #include "mali_kernel_core.h"
  34 #include "mali_osk.h"
  35 #include "mali_kernel_linux.h"
  36 #include "mali_ukk.h"
  37 #include "mali_ukk_wrappers.h"
  38 #include "mali_kernel_sysfs.h"
  39 #include "mali_pm.h"
  40 #include "mali_kernel_license.h"
  41 #include "mali_memory.h"
  42 #include "mali_memory_dma_buf.h"
  43 #if defined(CONFIG_MALI400_INTERNAL_PROFILING)
  44 #include "mali_profiling_internal.h"
  45 #endif
  46 #if defined(CONFIG_MALI400_PROFILING) && defined(CONFIG_MALI_DVFS)
  47 #include "mali_osk_profiling.h"
  48 #include "mali_dvfs_policy.h"
  49 static int is_first_resume = 1;
  50 /*Store the clk and vol for boot/insmod and mali_resume*/
  51 static struct mali_gpu_clk_item mali_gpu_clk[2];
  52 #endif
  53
  54 #include <linux/dma-mapping.h>
  55
  56 /* Streamline support for the Mali driver */
  57 #if defined(CONFIG_TRACEPOINTS) && defined(CONFIG_MALI400_PROFILING)
  58 /* Ask Linux to create the tracepoints */
  59 #define CREATE_TRACE_POINTS
  60 #include "mali_linux_trace.h"
  61
  62 EXPORT_TRACEPOINT_SYMBOL_GPL(mali_timeline_event);
  63 EXPORT_TRACEPOINT_SYMBOL_GPL(mali_hw_counter);
  64 EXPORT_TRACEPOINT_SYMBOL_GPL(mali_sw_counters);
  65 #endif /* CONFIG_TRACEPOINTS */
  66
  67 /* from the __malidrv_build_info.c file that is generated during build */
  68 extern const char *__malidrv_build_info(void);
  69
  70 /* Module parameter to control log level */
  71 int mali_debug_level = 2;
  72 module_param(mali_debug_level, int, S_IRUSR | S_IWUSR | S_IWGRP | S_IRGRP | S_IROTH); /* rw-rw-r-- */
  73 MODULE_PARM_DESC(mali_debug_level, "Higher number, more dmesg output");
  74
  75 int gpu_freq_cur = 0;
  76 module_param(gpu_freq_cur, int, S_IRUSR | S_IRGRP | S_IROTH); /* r-r-r-- */
  77 MODULE_PARM_DESC(gpu_freq_cur, "GPU gpu_freq_cur");
  78
  79 int gpu_freq_min_limit = -1;
  80 module_param(gpu_freq_min_limit, int, S_IRUSR | S_IWUSR | S_IWGRP | S_IRGRP | S_IROTH); /* rw-rw-r-- */
  81 MODULE_PARM_DESC(gpu_freq_min_limit, "GPU min_freq limit");
  82
  83 int gpu_freq_max_limit = -1;
  84 module_param(gpu_freq_max_limit, int, S_IRUSR | S_IWUSR | S_IWGRP | S_IRGRP | S_IROTH); /* rw-rw-r-- */
  85 MODULE_PARM_DESC(gpu_freq_max_limit, "GPU max_freq limit");
  86
  87 char* gpu_freq_list = NULL;
  88 module_param(gpu_freq_list, charp,S_IRUSR | S_IRGRP | S_IROTH ); /* r-r-r-- */
  89 MODULE_PARM_DESC(gpu_freq_list, "GPU freq_list");
  90
  91 int gpu_boost_level=0;
  92 module_param(gpu_boost_level, int, S_IRUSR | S_IRGRP | S_IROTH); /* r-r-r-- */
  93 MODULE_PARM_DESC(gpu_boost_level, "GPU gpu_boost_level");
  94
  95 int gpu_boost_sf_level = 0;
  96 module_param(gpu_boost_sf_level, int, S_IRUSR | S_IRGRP | S_IROTH); /* r-r-r-- */
  97 MODULE_PARM_DESC(gpu_boost_sf_level, "GPU gpu boost surfaceflinger level");
  98
  99 extern int mali_max_job_runtime;
 100 module_param(mali_max_job_runtime, int, S_IRUSR | S_IWUSR | S_IWGRP | S_IRGRP | S_IROTH);
 101 MODULE_PARM_DESC(mali_max_job_runtime, "Maximum allowed job runtime in msecs.\nJobs will be killed after this no matter what");
 102
 103 extern int mali_l2_max_reads;
 104 module_param(mali_l2_max_reads, int, S_IRUSR | S_IRGRP | S_IROTH);
 105 MODULE_PARM_DESC(mali_l2_max_reads, "Maximum reads for Mali L2 cache");
 106
 107 extern unsigned int mali_dedicated_mem_start;
 108 module_param(mali_dedicated_mem_start, uint, S_IRUSR | S_IRGRP | S_IROTH);
 109 MODULE_PARM_DESC(mali_dedicated_mem_start, "Physical start address of dedicated Mali GPU memory.");
 110
 111 extern unsigned int mali_dedicated_mem_size;
 112 module_param(mali_dedicated_mem_size, uint, S_IRUSR | S_IRGRP | S_IROTH);
 113 MODULE_PARM_DESC(mali_dedicated_mem_size, "Size of dedicated Mali GPU memory.");
 114
 115 extern unsigned int mali_shared_mem_size;
 116 module_param(mali_shared_mem_size, uint, S_IRUSR | S_IRGRP | S_IROTH);
 117 MODULE_PARM_DESC(mali_shared_mem_size, "Size of shared Mali GPU memory.");
 118
 119 #if defined(CONFIG_MALI400_PROFILING)
 120 extern int mali_boot_profiling;
 121 module_param(mali_boot_profiling, int, S_IRUSR | S_IRGRP | S_IROTH);
 122 MODULE_PARM_DESC(mali_boot_profiling, "Start profiling as a part of Mali driver initialization");
 123 #endif
 124
 125 extern int mali_max_pp_cores_group_1;
 126 module_param(mali_max_pp_cores_group_1, int, S_IRUSR | S_IRGRP | S_IROTH);
 127 MODULE_PARM_DESC(mali_max_pp_cores_group_1, "Limit the number of PP cores to use from first PP group.");
 128
 129 extern int mali_max_pp_cores_group_2;
 130 module_param(mali_max_pp_cores_group_2, int, S_IRUSR | S_IRGRP | S_IROTH);
 131 MODULE_PARM_DESC(mali_max_pp_cores_group_2, "Limit the number of PP cores to use from second PP group (Mali-450 only).");
 132
 133 #if defined(CONFIG_MALI_DVFS)
 134 /** the max fps the same as display vsync default 60, can set by module insert parameter */
 135 extern int mali_max_system_fps;
 136 module_param(mali_max_system_fps, int, S_IRUSR | S_IWUSR | S_IWGRP | S_IRGRP | S_IROTH);
 137 MODULE_PARM_DESC(mali_max_system_fps, "Max system fps the same as display VSYNC.");
 138
 139 /** a lower limit on their desired FPS default 58, can set by module insert parameter*/
 140 extern int mali_desired_fps;
 141 module_param(mali_desired_fps, int, S_IRUSR | S_IWUSR | S_IWGRP | S_IRGRP | S_IROTH);
 142 MODULE_PARM_DESC(mali_desired_fps, "A bit lower than max_system_fps which user desired fps");
 143 #endif
 144
 145 #if MALI_ENABLE_CPU_CYCLES
 146 #include <linux/cpumask.h>
 147 #include <linux/timer.h>
 148 #include <asm/smp.h>
 149 static struct timer_list mali_init_cpu_clock_timers[8];
 150 static u32 mali_cpu_clock_last_value[8] = {0,};
 151 #endif
 152
 153 /* Export symbols from common code: mali_user_settings.c */
 154 #include "mali_user_settings_db.h"
 155 EXPORT_SYMBOL(mali_set_user_setting);
 156 EXPORT_SYMBOL(mali_get_user_setting);
 157
 158 static char mali_dev_name[] = "mali"; /* should be const, but the functions we call requires non-cost */
 159
 160 /* This driver only supports one Mali device, and this variable stores this single platform device */
 161 struct platform_device *mali_platform_device = NULL;
 162
 163 /* This driver only supports one Mali device, and this variable stores the exposed misc device (/dev/mali) */
 164 static struct miscdevice mali_miscdevice = { 0, };
 165
 166 static int mali_miscdevice_register(struct platform_device *pdev);
 167 static void mali_miscdevice_unregister(void);
 168
 169 static int mali_open(struct inode *inode, struct file *filp);
 170 static int mali_release(struct inode *inode, struct file *filp);
 171 #ifdef HAVE_UNLOCKED_IOCTL
 172 static long mali_ioctl(struct file *filp, unsigned int cmd, unsigned long arg);
 173 #else
 174 static int mali_ioctl(struct inode *inode, struct file *filp, unsigned int cmd, unsigned long arg);
 175 #endif
 176
 177 static int mali_probe(struct platform_device *pdev);
 178 static int mali_remove(struct platform_device *pdev);
 179
 180 static int mali_driver_suspend_scheduler(struct device *dev);
 181 static int mali_driver_resume_scheduler(struct device *dev);
 182
 183 #ifdef CONFIG_PM_RUNTIME
 184 static int mali_driver_runtime_suspend(struct device *dev);
 185 static int mali_driver_runtime_resume(struct device *dev);
 186 static int mali_driver_runtime_idle(struct device *dev);
 187 #endif
 188
 189 extern void mali_platform_power_mode_change(int power_mode);
 190
 191 #if defined(MALI_FAKE_PLATFORM_DEVICE)
 192 #if defined(CONFIG_MALI_DT)
 193 extern int mali_platform_device_init(struct platform_device *device);
 194 extern int mali_platform_device_deinit(struct platform_device *device);
 195 #else
 196 extern int mali_platform_device_register(void);
 197 extern int mali_platform_device_unregister(void);
 198 #endif
 199 #endif
 200
 201 /* Linux power management operations provided by the Mali device driver */
 202 #if (LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 29))
 203 struct pm_ext_ops mali_dev_ext_pm_ops = {
 204         .base =
 205         {
 206                 .suspend = mali_driver_suspend_scheduler,
 207                 .resume = mali_driver_resume_scheduler,
 208                 .freeze = mali_driver_suspend_scheduler,
 209                 .thaw =   mali_driver_resume_scheduler,
 210         },
 211 };
 212 #else
 213 static const struct dev_pm_ops mali_dev_pm_ops = {
 214 #ifdef CONFIG_PM_RUNTIME
 215         .runtime_suspend = mali_driver_runtime_suspend,
 216         .runtime_resume = mali_driver_runtime_resume,
 217         .runtime_idle = mali_driver_runtime_idle,
 218 #endif
 219         .suspend = mali_driver_suspend_scheduler,
 220         .resume = mali_driver_resume_scheduler,
 221         .freeze = mali_driver_suspend_scheduler,
 222         .thaw = mali_driver_resume_scheduler,
 223         .poweroff = mali_driver_suspend_scheduler,
 224 };
 225 #endif
 226
 227 #ifdef CONFIG_MALI_DT
 228 struct of_device_id base_dt_ids[] = {
 229         {.compatible = "arm,mali-300"},
 230         {.compatible = "arm,mali-400"},
 231         {.compatible = "arm,mali-450"},
 232         {.compatible = "arm,mali-utgard"},
 233         {},
 234 };
 235
 236 MODULE_DEVICE_TABLE(of, base_dt_ids);
 237 #endif
 238
 239 /* The Mali device driver struct */
 240 static struct platform_driver mali_platform_driver = {
 241         .probe  = mali_probe,
 242         .remove = mali_remove,
 243 #if (LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 29))
 244         .pm = &mali_dev_ext_pm_ops,
 245 #endif
 246         .driver =
 247         {
 248                 .name   = MALI_GPU_NAME_UTGARD,
 249                 .owner  = THIS_MODULE,
 250                 .bus = &platform_bus_type,
 251 #if (LINUX_VERSION_CODE >= KERNEL_VERSION(2, 6, 29))
 252                 .pm = &mali_dev_pm_ops,
 253 #endif
 254 #ifdef CONFIG_MALI_DT
 255                 .of_match_table = of_match_ptr(base_dt_ids),
 256 #endif
 257         },
 258 };
 259
 260 /* Linux misc device operations (/dev/mali) */
 261 struct file_operations mali_fops = {
 262         .owner = THIS_MODULE,
 263         .open = mali_open,
 264         .release = mali_release,
 265 #ifdef HAVE_UNLOCKED_IOCTL
 266         .unlocked_ioctl = mali_ioctl,
 267 #else
 268         .ioctl = mali_ioctl,
 269 #endif
 270         .compat_ioctl = mali_ioctl,
 271         .mmap = mali_mmap
 272 };
 273
 274 #if MALI_ENABLE_CPU_CYCLES
 275 void mali_init_cpu_time_counters(int reset, int enable_divide_by_64)
 276 {
 277         /* The CPU assembly reference used is: ARM Architecture Reference Manual ARMv7-AR C.b */
 278         u32 write_value;
 279
 280         /* See B4.1.116 PMCNTENSET, Performance Monitors Count Enable Set register, VMSA */
 281         /* setting p15 c9 c12 1 to 0x8000000f==CPU_CYCLE_ENABLE |EVENT_3_ENABLE|EVENT_2_ENABLE|EVENT_1_ENABLE|EVENT_0_ENABLE */
 282         asm volatile("mcr p15, 0, %0, c9, c12, 1" :: "r"(0x8000000f));
 283
 284
 285         /* See B4.1.117 PMCR, Performance Monitors Control Register. Writing to p15, c9, c12, 0 */
 286         write_value = 1 << 0; /* Bit 0 set. Enable counters */
 287         if (reset) {
 288                 write_value |= 1 << 1; /* Reset event counters */
 289                 write_value |= 1 << 2; /* Reset cycle counter  */
 290         }
 291         if (enable_divide_by_64) {
 292                 write_value |= 1 << 3; /* Enable the Clock divider by 64 */
 293         }
 294         write_value |= 1 << 4; /* Export enable. Not needed */
 295         asm volatile("MCR p15, 0, %0, c9, c12, 0\t\n" :: "r"(write_value));
 296
 297         /* PMOVSR Overflow Flag Status Register - Clear Clock and Event overflows */
 298         asm volatile("MCR p15, 0, %0, c9, c12, 3\t\n" :: "r"(0x8000000f));
 299
 300
 301         /* See B4.1.124 PMUSERENR - setting p15 c9 c14 to 1" */
 302         /* User mode access to the Performance Monitors enabled. */
 303         /* Lets User space read cpu clock cycles */
 304         asm volatile("mcr p15, 0, %0, c9, c14, 0" :: "r"(1));
 305 }
 306
 307 /** A timer function that configures the cycle clock counter on current CPU.
 308  * The function \a mali_init_cpu_time_counters_on_all_cpus sets up this
 309  * function to trigger on all Cpus during module load.
 310  */
 311 static void mali_init_cpu_clock_timer_func(unsigned long data)
 312 {
 313         int reset_counters, enable_divide_clock_counter_by_64;
 314         int current_cpu = raw_smp_processor_id();
 315         unsigned int sample0;
 316         unsigned int sample1;
 317
 318         MALI_IGNORE(data);
 319
 320         reset_counters = 1;
 321         enable_divide_clock_counter_by_64 = 0;
 322         mali_init_cpu_time_counters(reset_counters, enable_divide_clock_counter_by_64);
 323
 324         sample0 = mali_get_cpu_cyclecount();
 325         sample1 = mali_get_cpu_cyclecount();
 326
 327         MALI_DEBUG_PRINT(3, ("Init Cpu %d cycle counter- First two samples: %08x %08x \n", current_cpu, sample0, sample1));
 328 }
 329
 330 /** A timer functions for storing current time on all cpus.
 331  * Used for checking if the clocks have similar values or if they are drifting.
 332  */
 333 static void mali_print_cpu_clock_timer_func(unsigned long data)
 334 {
 335         int current_cpu = raw_smp_processor_id();
 336         unsigned int sample0;
 337
 338         MALI_IGNORE(data);
 339         sample0 = mali_get_cpu_cyclecount();
 340         if (current_cpu < 8) {
 341                 mali_cpu_clock_last_value[current_cpu] = sample0;
 342         }
 343 }
 344
 345 /** Init the performance registers on all CPUs to count clock cycles.
 346  * For init \a print_only should be 0.
 347  * If \a print_only is 1, it will intead print the current clock value of all CPUs.
 348  */
 349 void mali_init_cpu_time_counters_on_all_cpus(int print_only)
 350 {
 351         int i = 0;
 352         int cpu_number;
 353         int jiffies_trigger;
 354         int jiffies_wait;
 355
 356         jiffies_wait = 2;
 357         jiffies_trigger = jiffies + jiffies_wait;
 358
 359         for (i = 0 ; i < 8 ; i++) {
 360                 init_timer(&mali_init_cpu_clock_timers[i]);
 361                 if (print_only) mali_init_cpu_clock_timers[i].function = mali_print_cpu_clock_timer_func;
 362                 else            mali_init_cpu_clock_timers[i].function = mali_init_cpu_clock_timer_func;
 363                 mali_init_cpu_clock_timers[i].expires = jiffies_trigger ;
 364         }
 365         cpu_number = cpumask_first(cpu_online_mask);
 366         for (i = 0 ; i < 8 ; i++) {
 367                 int next_cpu;
 368                 add_timer_on(&mali_init_cpu_clock_timers[i], cpu_number);
 369                 next_cpu = cpumask_next(cpu_number, cpu_online_mask);
 370                 if (next_cpu >= nr_cpu_ids) break;
 371                 cpu_number = next_cpu;
 372         }
 373
 374         while (jiffies_wait) jiffies_wait = schedule_timeout_uninterruptible(jiffies_wait);
 375
 376         for (i = 0 ; i < 8 ; i++) {
 377                 del_timer_sync(&mali_init_cpu_clock_timers[i]);
 378         }
 379
 380         if (print_only) {
 381                 if ((0 == mali_cpu_clock_last_value[2]) && (0 == mali_cpu_clock_last_value[3])) {
 382                         /* Diff can be printed if we want to check if the clocks are in sync
 383                         int diff = mali_cpu_clock_last_value[0] - mali_cpu_clock_last_value[1];*/
 384                         MALI_DEBUG_PRINT(2, ("CPU cycle counters readout all: %08x %08x\n", mali_cpu_clock_last_value[0], mali_cpu_clock_last_value[1]));
 385                 } else {
 386                         MALI_DEBUG_PRINT(2, ("CPU cycle counters readout all: %08x %08x %08x %08x\n", mali_cpu_clock_last_value[0], mali_cpu_clock_last_value[1], mali_cpu_clock_last_value[2], mali_cpu_clock_last_value[3]));
 387                 }
 388         }
 389 }
 390 #endif
 391
 392 int mali_module_init(void)
 393 {
 394         int err = 0;
 395
 396         MALI_DEBUG_PRINT(2, ("Inserting Mali v%d device driver. \n", _MALI_API_VERSION));
 397 //      MALI_DEBUG_PRINT(2, ("Compiled: %s, time: %s.\n", __DATE__, __TIME__));
 398         MALI_DEBUG_PRINT(2, ("Driver revision: %s\n", SVN_REV_STRING));
 399
 400 #if MALI_ENABLE_CPU_CYCLES
 401         mali_init_cpu_time_counters_on_all_cpus(0);
 402         MALI_DEBUG_PRINT(2, ("CPU cycle counter setup complete\n"));
 403         /* Printing the current cpu counters */
 404         mali_init_cpu_time_counters_on_all_cpus(1);
 405 #endif
 406
 407         /* Initialize module wide settings */
 408 #ifdef MALI_FAKE_PLATFORM_DEVICE
 409 #ifndef CONFIG_MALI_DT
 410         MALI_DEBUG_PRINT(2, ("mali_module_init() registering device\n"));
 411         err = mali_platform_device_register();
 412         if (0 != err) {
 413                 return err;
 414         }
 415 #endif
 416 #endif
 417
 418         MALI_DEBUG_PRINT(2, ("mali_module_init() registering driver\n"));
 419
 420         err = platform_driver_register(&mali_platform_driver);
 421
 422         if (0 != err) {
 423                 MALI_DEBUG_PRINT(2, ("mali_module_init() Failed to register driver (%d)\n", err));
 424 #ifdef MALI_FAKE_PLATFORM_DEVICE
 425 #ifndef CONFIG_MALI_DT
 426                 mali_platform_device_unregister();
 427 #endif
 428 #endif
 429                 mali_platform_device = NULL;
 430                 return err;
 431         }
 432
 433 #if defined(CONFIG_MALI400_INTERNAL_PROFILING)
 434         err = _mali_internal_profiling_init(mali_boot_profiling ? MALI_TRUE : MALI_FALSE);
 435         if (0 != err) {
 436                 /* No biggie if we wheren't able to initialize the profiling */
 437                 MALI_PRINT_ERROR(("Failed to initialize profiling, feature will be unavailable\n"));
 438         }
 439 #endif
 440
 441         /* Tracing the current frequency and voltage from boot/insmod*/
 442 #if defined(CONFIG_MALI400_PROFILING) && defined(CONFIG_MALI_DVFS)
 443         /* Just call mali_get_current_gpu_clk_item(),to record current clk info.*/
 444         mali_get_current_gpu_clk_item(&mali_gpu_clk[0]);
 445         _mali_osk_profiling_add_event(MALI_PROFILING_EVENT_TYPE_SINGLE |
 446                                           MALI_PROFILING_EVENT_CHANNEL_GPU |
 447                                           MALI_PROFILING_EVENT_REASON_SINGLE_GPU_FREQ_VOLT_CHANGE,
 448                                           mali_gpu_clk[0].clock,
 449                                           mali_gpu_clk[0].vol / 1000,
 450                                           0, 0, 0);
 451 #endif
 452
 453         MALI_PRINT(("Mali device driver loaded\n"));
 454
 455         return 0; /* Success */
 456 }
 457
 458 void mali_module_exit(void)
 459 {
 460         MALI_DEBUG_PRINT(2, ("Unloading Mali v%d device driver.\n", _MALI_API_VERSION));
 461
 462         MALI_DEBUG_PRINT(2, ("mali_module_exit() unregistering driver\n"));
 463
 464         platform_driver_unregister(&mali_platform_driver);
 465
 466 #if defined(MALI_FAKE_PLATFORM_DEVICE)
 467 #ifndef CONFIG_MALI_DT
 468         MALI_DEBUG_PRINT(2, ("mali_module_exit() unregistering device\n"));
 469         mali_platform_device_unregister();
 470 #endif
 471 #endif
 472
 473         /* Tracing the current frequency and voltage from rmmod*/
 474         _mali_osk_profiling_add_event(MALI_PROFILING_EVENT_TYPE_SINGLE |
 475                                       MALI_PROFILING_EVENT_CHANNEL_GPU |
 476                                       MALI_PROFILING_EVENT_REASON_SINGLE_GPU_FREQ_VOLT_CHANGE,
 477                                       0,
 478                                       0,
 479                                       0, 0, 0);
 480
 481 #if defined(CONFIG_MALI400_INTERNAL_PROFILING)
 482         _mali_internal_profiling_term();
 483 #endif
 484
 485         MALI_PRINT(("Mali device driver unloaded\n"));
 486 }
 487
 488 static int mali_probe(struct platform_device *pdev)
 489 {
 490         int err;
 491
 492         MALI_DEBUG_PRINT(2, ("mali_probe(): Called for platform device %s\n", pdev->name));
 493
 494         if (NULL != mali_platform_device) {
 495                 /* Already connected to a device, return error */
 496                 MALI_PRINT_ERROR(("mali_probe(): The Mali driver is already connected with a Mali device."));
 497                 return -EEXIST;
 498         }
 499
 500         mali_platform_device = pdev;
 501
 502 #ifdef CONFIG_64BIT
 503         static u64 mali_dma_mask = DMA_BIT_MASK(32);
 504         pdev->dev.dma_mask = &mali_dma_mask;
 505 #endif
 506
 507         err = mali_executor_initialize();
 508         if (_MALI_OSK_ERR_OK != err)
 509         {
 510                 MALI_PRINT_ERROR(("mali_executor_initialize failed.\n"));
 511                 return -EFAULT;
 512         }
 513
 514 #ifdef CONFIG_MALI_DT
 515         /* If we use DT to initialize our DDK, we have to prepare somethings. */
 516         err = mali_platform_device_init(mali_platform_device);
 517         if (0 != err) {
 518                 MALI_PRINT_ERROR(("mali_probe(): Failed to initialize platform device."));
 519                 return -EFAULT;
 520         }
 521 #endif
 522
 523         if (_MALI_OSK_ERR_OK == _mali_osk_wq_init()) {
 524                 /* Initialize the Mali GPU HW specified by pdev */
 525                 if (_MALI_OSK_ERR_OK == mali_initialize_subsystems()) {
 526                         /* Register a misc device (so we are accessible from user space) */
 527                         err = mali_miscdevice_register(pdev);
 528                         if (0 == err) {
 529                                 /* Setup sysfs entries */
 530                                 err = mali_sysfs_register(mali_dev_name);
 531
 532                                 if (0 == err) {
 533                                         MALI_DEBUG_PRINT(2, ("mali_probe(): Successfully initialized driver for platform device %s\n", pdev->name));
 534
 535                                         return 0;
 536                                 } else {
 537                                         MALI_PRINT_ERROR(("mali_probe(): failed to register sysfs entries"));
 538                                 }
 539                                 mali_miscdevice_unregister();
 540                         } else {
 541                                 MALI_PRINT_ERROR(("mali_probe(): failed to register Mali misc device."));
 542                         }
 543                         mali_terminate_subsystems();
 544                 } else {
 545                         MALI_PRINT_ERROR(("mali_probe(): Failed to initialize Mali device driver."));
 546                 }
 547                 _mali_osk_wq_term();
 548         }
 549
 550         mali_platform_device = NULL;
 551         return -EFAULT;
 552 }
 553
 554 static int mali_remove(struct platform_device *pdev)
 555 {
 556         MALI_DEBUG_PRINT(2, ("mali_remove() called for platform device %s\n", pdev->name));
 557         mali_sysfs_unregister();
 558         mali_miscdevice_unregister();
 559         mali_terminate_subsystems();
 560         _mali_osk_wq_term();
 561 #ifdef CONFIG_MALI_DT
 562         mali_platform_device_deinit(mali_platform_device);
 563 #endif
 564         mali_platform_device = NULL;
 565         return 0;
 566 }
 567
 568 static int mali_miscdevice_register(struct platform_device *pdev)
 569 {
 570         int err;
 571
 572         mali_miscdevice.minor = MISC_DYNAMIC_MINOR;
 573         mali_miscdevice.name = mali_dev_name;
 574         mali_miscdevice.fops = &mali_fops;
 575         mali_miscdevice.parent = get_device(&pdev->dev);
 576
 577         err = misc_register(&mali_miscdevice);
 578         if (0 != err) {
 579                 MALI_PRINT_ERROR(("Failed to register misc device, misc_register() returned %d\n", err));
 580         }
 581
 582         return err;
 583 }
 584
 585 static void mali_miscdevice_unregister(void)
 586 {
 587         misc_deregister(&mali_miscdevice);
 588 }
 589
 590 static int mali_driver_suspend_scheduler(struct device *dev)
 591 {
 592         mali_pm_os_suspend(MALI_TRUE);
 593
 594         if (pm_runtime_active(dev))
 595                 mali_platform_power_mode_change(2);
 596
 597         /* Tracing the frequency and voltage after mali is suspended */
 598         _mali_osk_profiling_add_event(MALI_PROFILING_EVENT_TYPE_SINGLE |
 599                                       MALI_PROFILING_EVENT_CHANNEL_GPU |
 600                                       MALI_PROFILING_EVENT_REASON_SINGLE_GPU_FREQ_VOLT_CHANGE,
 601                                       0,
 602                                       0,
 603                                       0, 0, 0);
 604         return 0;
 605 }
 606
 607 static int mali_driver_resume_scheduler(struct device *dev)
 608 {
 609         /* Tracing the frequency and voltage after mali is resumed */
 610 #if defined(CONFIG_MALI400_PROFILING) && defined(CONFIG_MALI_DVFS)
 611         /* Just call mali_get_current_gpu_clk_item() once,to record current clk info.*/
 612         if (is_first_resume == 1) {
 613                 mali_get_current_gpu_clk_item(&mali_gpu_clk[1]);
 614                 is_first_resume = 0;
 615         }
 616         _mali_osk_profiling_add_event(MALI_PROFILING_EVENT_TYPE_SINGLE |
 617                                       MALI_PROFILING_EVENT_CHANNEL_GPU |
 618                                       MALI_PROFILING_EVENT_REASON_SINGLE_GPU_FREQ_VOLT_CHANGE,
 619                                       mali_gpu_clk[1].clock,
 620                                       mali_gpu_clk[1].vol / 1000,
 621                                       0, 0, 0);
 622 #endif
 623
 624         if (pm_runtime_active(dev))
 625                 mali_platform_power_mode_change(0);
 626
 627         mali_pm_os_resume();
 628         return 0;
 629 }
 630
 631 #ifdef CONFIG_PM_RUNTIME
 632 static int mali_driver_runtime_suspend(struct device *dev)
 633 {
 634         if (MALI_TRUE == mali_pm_runtime_suspend()) {
 635                 mali_platform_power_mode_change(1);
 636
 637                 /* Tracing the frequency and voltage after mali is suspended */
 638                 _mali_osk_profiling_add_event(MALI_PROFILING_EVENT_TYPE_SINGLE |
 639                                               MALI_PROFILING_EVENT_CHANNEL_GPU |
 640                                               MALI_PROFILING_EVENT_REASON_SINGLE_GPU_FREQ_VOLT_CHANGE,
 641                                               0,
 642                                               0,
 643                                               0, 0, 0);
 644
 645                 return 0;
 646         } else {
 647                 return -EBUSY;
 648         }
 649 }
 650
 651 static int mali_driver_runtime_resume(struct device *dev)
 652 {
 653         /* Tracing the frequency and voltage after mali is resumed */
 654 #if defined(CONFIG_MALI400_PROFILING) && defined(CONFIG_MALI_DVFS)
 655         /* Just call mali_get_current_gpu_clk_item() once,to record current clk info.*/
 656         if (is_first_resume == 1) {
 657                 mali_get_current_gpu_clk_item(&mali_gpu_clk[1]);
 658                 is_first_resume = 0;
 659         }
 660         _mali_osk_profiling_add_event(MALI_PROFILING_EVENT_TYPE_SINGLE |
 661                                       MALI_PROFILING_EVENT_CHANNEL_GPU |
 662                                       MALI_PROFILING_EVENT_REASON_SINGLE_GPU_FREQ_VOLT_CHANGE,
 663                                       mali_gpu_clk[1].clock,
 664                                       mali_gpu_clk[1].vol / 1000,
 665                                       0, 0, 0);
 666 #endif
 667         mali_platform_power_mode_change(0);
 668
 669         mali_pm_runtime_resume();
 670         return 0;
 671 }
 672
 673 static int mali_driver_runtime_idle(struct device *dev)
 674 {
 675         MALI_DEBUG_PRINT(5, ("%s:%d \n",__FUNCTION__, __LINE__));
 676         pm_runtime_suspend(dev);
 677         return 0;
 678 }
 679 #endif
 680
 681 static int mali_open(struct inode *inode, struct file *filp)
 682 {
 683         struct mali_session_data *session_data;
 684         _mali_osk_errcode_t err;
 685
 686         /* input validation */
 687         if (mali_miscdevice.minor != iminor(inode)) {
 688                 MALI_PRINT_ERROR(("mali_open() Minor does not match\n"));
 689                 return -ENODEV;
 690         }
 691
 692         /* allocated struct to track this session */
 693         err = _mali_ukk_open((void **)&session_data);
 694         if (_MALI_OSK_ERR_OK != err) return map_errcode(err);
 695
 696         /* initialize file pointer */
 697         filp->f_pos = 0;
 698
 699         /* link in our session data */
 700         filp->private_data = (void *)session_data;
 701
 702         return 0;
 703 }
 704
 705 static int mali_release(struct inode *inode, struct file *filp)
 706 {
 707         _mali_osk_errcode_t err;
 708
 709         /* input validation */
 710         if (mali_miscdevice.minor != iminor(inode)) {
 711                 MALI_PRINT_ERROR(("mali_release() Minor does not match\n"));
 712                 return -ENODEV;
 713         }
 714
 715         err = _mali_ukk_close((void **)&filp->private_data);
 716         if (_MALI_OSK_ERR_OK != err) return map_errcode(err);
 717
 718         return 0;
 719 }
 720
 721 int map_errcode(_mali_osk_errcode_t err)
 722 {
 723         switch (err) {
 724         case _MALI_OSK_ERR_OK :
 725                 return 0;
 726         case _MALI_OSK_ERR_FAULT:
 727                 return -EFAULT;
 728         case _MALI_OSK_ERR_INVALID_FUNC:
 729                 return -ENOTTY;
 730         case _MALI_OSK_ERR_INVALID_ARGS:
 731                 return -EINVAL;
 732         case _MALI_OSK_ERR_NOMEM:
 733                 return -ENOMEM;
 734         case _MALI_OSK_ERR_TIMEOUT:
 735                 return -ETIMEDOUT;
 736         case _MALI_OSK_ERR_RESTARTSYSCALL:
 737                 return -ERESTARTSYS;
 738         case _MALI_OSK_ERR_ITEM_NOT_FOUND:
 739                 return -ENOENT;
 740         default:
 741                 return -EFAULT;
 742         }
 743 }
 744
 745 #ifdef HAVE_UNLOCKED_IOCTL
 746 static long mali_ioctl(struct file *filp, unsigned int cmd, unsigned long arg)
 747 #else
 748 static int mali_ioctl(struct inode *inode, struct file *filp, unsigned int cmd, unsigned long arg)
 749 #endif
 750 {
 751         int err;
 752         struct mali_session_data *session_data;
 753
 754 #ifndef HAVE_UNLOCKED_IOCTL
 755         /* inode not used */
 756         (void)inode;
 757 #endif
 758
 759         MALI_DEBUG_PRINT(7, ("Ioctl received 0x%08X 0x%08lX\n", cmd, arg));
 760
 761         session_data = (struct mali_session_data *)filp->private_data;
 762         if (NULL == session_data) {
 763                 MALI_DEBUG_PRINT(7, ("filp->private_data was NULL\n"));
 764                 return -ENOTTY;
 765         }
 766
 767         if (NULL == (void *)arg) {
 768                 MALI_DEBUG_PRINT(7, ("arg was NULL\n"));
 769                 return -ENOTTY;
 770         }
 771
 772         switch (cmd) {
 773         case MALI_IOC_WAIT_FOR_NOTIFICATION:
 774                 BUILD_BUG_ON(!IS_ALIGNED(sizeof(_mali_uk_wait_for_notification_s), sizeof(u64)));
 775                 err = wait_for_notification_wrapper(session_data, (_mali_uk_wait_for_notification_s __user *)arg);
 776                 break;
 777
 778         case MALI_IOC_GET_API_VERSION_V2:
 779                 BUILD_BUG_ON(!IS_ALIGNED(sizeof(_mali_uk_get_api_version_v2_s), sizeof(u64)));
 780                 err = get_api_version_v2_wrapper(session_data, (_mali_uk_get_api_version_v2_s __user *)arg);
 781                 break;
 782
 783         case MALI_IOC_GET_API_VERSION:
 784                 err = get_api_version_wrapper(session_data, (_mali_uk_get_api_version_s __user *)arg);
 785                 break;
 786
 787         case MALI_IOC_POST_NOTIFICATION:
 788                 BUILD_BUG_ON(!IS_ALIGNED(sizeof(_mali_uk_post_notification_s), sizeof(u64)));
 789                 err = post_notification_wrapper(session_data, (_mali_uk_post_notification_s __user *)arg);
 790                 break;
 791
 792         case MALI_IOC_GET_USER_SETTINGS:
 793                 BUILD_BUG_ON(!IS_ALIGNED(sizeof(_mali_uk_get_user_settings_s), sizeof(u64)));
 794                 err = get_user_settings_wrapper(session_data, (_mali_uk_get_user_settings_s __user *)arg);
 795                 break;
 796 #ifdef SPRD_GPU_BOOST
 797         case MALI_IOC_SET_GPU_BOOST_LEVEL:
 798                 err = get_user(session_data->level,(int __user *)arg);
 799                 break;
 800 #endif
 801         case MALI_IOC_REQUEST_HIGH_PRIORITY:
 802                 BUILD_BUG_ON(!IS_ALIGNED(sizeof(_mali_uk_request_high_priority_s), sizeof(u64)));
 803                 err = request_high_priority_wrapper(session_data, (_mali_uk_request_high_priority_s __user *)arg);
 804                 break;
 805
 806 #if defined(CONFIG_MALI400_PROFILING)
 807         case MALI_IOC_PROFILING_ADD_EVENT:
 808                 BUILD_BUG_ON(!IS_ALIGNED(sizeof(_mali_uk_profiling_add_event_s), sizeof(u64)));
 809                 err = profiling_add_event_wrapper(session_data, (_mali_uk_profiling_add_event_s __user *)arg);
 810                 break;
 811
 812         case MALI_IOC_PROFILING_REPORT_SW_COUNTERS:
 813                 BUILD_BUG_ON(!IS_ALIGNED(sizeof(_mali_uk_sw_counters_report_s), sizeof(u64)));
 814                 err = profiling_report_sw_counters_wrapper(session_data, (_mali_uk_sw_counters_report_s __user *)arg);
 815                 break;
 816
 817
 818         case MALI_IOC_PROFILING_MEMORY_USAGE_GET:
 819                 BUILD_BUG_ON(!IS_ALIGNED(sizeof(_mali_uk_profiling_memory_usage_get_s), sizeof(u64)));
 820                 err = profiling_memory_usage_get_wrapper(session_data, (_mali_uk_profiling_memory_usage_get_s __user *)arg);
 821                 break;
 822
 823 #else
 824
 825         case MALI_IOC_PROFILING_ADD_EVENT:          /* FALL-THROUGH */
 826         case MALI_IOC_PROFILING_REPORT_SW_COUNTERS: /* FALL-THROUGH */
 827         case MALI_IOC_PROFILING_MEMORY_USAGE_GET:   /* FALL-THROUGH */
 828                 MALI_DEBUG_PRINT(2, ("Profiling not supported\n"));
 829                 err = -ENOTTY;
 830                 break;
 831
 832 #endif
 833
 834         case MALI_IOC_MEM_WRITE_SAFE:
 835                 BUILD_BUG_ON(!IS_ALIGNED(sizeof(_mali_uk_mem_write_safe_s), sizeof(u64)));
 836                 err = mem_write_safe_wrapper(session_data, (_mali_uk_mem_write_safe_s __user *)arg);
 837                 break;
 838
 839         case MALI_IOC_MEM_MAP_EXT:
 840                 BUILD_BUG_ON(!IS_ALIGNED(sizeof(_mali_uk_map_external_mem_s), sizeof(u64)));
 841                 err = mem_map_ext_wrapper(session_data, (_mali_uk_map_external_mem_s __user *)arg);
 842                 break;
 843
 844         case MALI_IOC_MEM_UNMAP_EXT:
 845                 BUILD_BUG_ON(!IS_ALIGNED(sizeof(_mali_uk_unmap_external_mem_s), sizeof(u64)));
 846                 err = mem_unmap_ext_wrapper(session_data, (_mali_uk_unmap_external_mem_s __user *)arg);
 847                 break;
 848
 849         case MALI_IOC_MEM_QUERY_MMU_PAGE_TABLE_DUMP_SIZE:
 850                 BUILD_BUG_ON(!IS_ALIGNED(sizeof(_mali_uk_query_mmu_page_table_dump_size_s), sizeof(u64)));
 851                 err = mem_query_mmu_page_table_dump_size_wrapper(session_data, (_mali_uk_query_mmu_page_table_dump_size_s __user *)arg);
 852                 break;
 853
 854         case MALI_IOC_MEM_DUMP_MMU_PAGE_TABLE:
 855                 BUILD_BUG_ON(!IS_ALIGNED(sizeof(_mali_uk_dump_mmu_page_table_s), sizeof(u64)));
 856                 err = mem_dump_mmu_page_table_wrapper(session_data, (_mali_uk_dump_mmu_page_table_s __user *)arg);
 857                 break;
 858
 859 #if defined(CONFIG_MALI400_UMP)
 860
 861         case MALI_IOC_MEM_ATTACH_UMP:
 862                 BUILD_BUG_ON(!IS_ALIGNED(sizeof(_mali_uk_attach_ump_mem_s), sizeof(u64)));
 863                 err = mem_attach_ump_wrapper(session_data, (_mali_uk_attach_ump_mem_s __user *)arg);
 864                 break;
 865
 866         case MALI_IOC_MEM_RELEASE_UMP:
 867                 BUILD_BUG_ON(!IS_ALIGNED(sizeof(_mali_uk_release_ump_mem_s), sizeof(u64)));
 868                 err = mem_release_ump_wrapper(session_data, (_mali_uk_release_ump_mem_s __user *)arg);
 869                 break;
 870
 871 #else
 872
 873         case MALI_IOC_MEM_ATTACH_UMP:
 874         case MALI_IOC_MEM_RELEASE_UMP: /* FALL-THROUGH */
 875                 MALI_DEBUG_PRINT(2, ("UMP not supported\n"));
 876                 err = -ENOTTY;
 877                 break;
 878 #endif
 879
 880 #ifdef CONFIG_DMA_SHARED_BUFFER
 881         case MALI_IOC_MEM_ATTACH_DMA_BUF:
 882                 BUILD_BUG_ON(!IS_ALIGNED(sizeof(_mali_uk_attach_dma_buf_s), sizeof(u64)));
 883                 err = mali_attach_dma_buf(session_data, (_mali_uk_attach_dma_buf_s __user *)arg);
 884                 break;
 885
 886         case MALI_IOC_MEM_RELEASE_DMA_BUF:
 887                 BUILD_BUG_ON(!IS_ALIGNED(sizeof(_mali_uk_release_dma_buf_s), sizeof(u64)));
 888                 err = mali_release_dma_buf(session_data, (_mali_uk_release_dma_buf_s __user *)arg);
 889                 break;
 890
 891         case MALI_IOC_MEM_DMA_BUF_GET_SIZE:
 892                 BUILD_BUG_ON(!IS_ALIGNED(sizeof(_mali_uk_dma_buf_get_size_s), sizeof(u64)));
 893                 err = mali_dma_buf_get_size(session_data, (_mali_uk_dma_buf_get_size_s __user *)arg);
 894                 break;
 895 #else
 896
 897         case MALI_IOC_MEM_ATTACH_DMA_BUF:   /* FALL-THROUGH */
 898         case MALI_IOC_MEM_RELEASE_DMA_BUF:  /* FALL-THROUGH */
 899         case MALI_IOC_MEM_DMA_BUF_GET_SIZE: /* FALL-THROUGH */
 900                 MALI_DEBUG_PRINT(2, ("DMA-BUF not supported\n"));
 901                 err = -ENOTTY;
 902                 break;
 903 #endif
 904
 905 #if TIZEN_GLES_MEM_PROFILE
 906         case MALI_IOC_MEM_PROFILE_GLES_MEM:
 907                 BUILD_BUG_ON(!IS_ALIGNED(sizeof(_mali_uk_gles_mem_profiler_s),
 908                                 sizeof(u64)));
 909                 err = mem_profile_gles_mem(session_data,
 910                                 (_mali_uk_gles_mem_profiler_s __user *)arg);
 911                 break;
 912 #endif
 913
 914         case MALI_IOC_PP_START_JOB:
 915                 BUILD_BUG_ON(!IS_ALIGNED(sizeof(_mali_uk_pp_start_job_s), sizeof(u64)));
 916                 err = pp_start_job_wrapper(session_data, (_mali_uk_pp_start_job_s __user *)arg);
 917                 break;
 918
 919         case MALI_IOC_PP_AND_GP_START_JOB:
 920                 BUILD_BUG_ON(!IS_ALIGNED(sizeof(_mali_uk_pp_and_gp_start_job_s), sizeof(u64)));
 921                 err = pp_and_gp_start_job_wrapper(session_data, (_mali_uk_pp_and_gp_start_job_s __user *)arg);
 922                 break;
 923
 924         case MALI_IOC_PP_NUMBER_OF_CORES_GET:
 925                 BUILD_BUG_ON(!IS_ALIGNED(sizeof(_mali_uk_get_pp_number_of_cores_s), sizeof(u64)));
 926                 err = pp_get_number_of_cores_wrapper(session_data, (_mali_uk_get_pp_number_of_cores_s __user *)arg);
 927                 break;
 928
 929         case MALI_IOC_PP_CORE_VERSION_GET:
 930                 BUILD_BUG_ON(!IS_ALIGNED(sizeof(_mali_uk_get_pp_core_version_s), sizeof(u64)));
 931                 err = pp_get_core_version_wrapper(session_data, (_mali_uk_get_pp_core_version_s __user *)arg);
 932                 break;
 933
 934         case MALI_IOC_PP_DISABLE_WB:
 935                 BUILD_BUG_ON(!IS_ALIGNED(sizeof(_mali_uk_pp_disable_wb_s), sizeof(u64)));
 936                 err = pp_disable_wb_wrapper(session_data, (_mali_uk_pp_disable_wb_s __user *)arg);
 937                 break;
 938
 939         case MALI_IOC_GP2_START_JOB:
 940                 BUILD_BUG_ON(!IS_ALIGNED(sizeof(_mali_uk_gp_start_job_s), sizeof(u64)));
 941                 err = gp_start_job_wrapper(session_data, (_mali_uk_gp_start_job_s __user *)arg);
 942                 break;
 943
 944         case MALI_IOC_GP2_NUMBER_OF_CORES_GET:
 945                 BUILD_BUG_ON(!IS_ALIGNED(sizeof(_mali_uk_get_gp_number_of_cores_s), sizeof(u64)));
 946                 err = gp_get_number_of_cores_wrapper(session_data, (_mali_uk_get_gp_number_of_cores_s __user *)arg);
 947                 break;
 948
 949         case MALI_IOC_GP2_CORE_VERSION_GET:
 950                 BUILD_BUG_ON(!IS_ALIGNED(sizeof(_mali_uk_get_gp_core_version_s), sizeof(u64)));
 951                 err = gp_get_core_version_wrapper(session_data, (_mali_uk_get_gp_core_version_s __user *)arg);
 952                 break;
 953
 954         case MALI_IOC_GP2_SUSPEND_RESPONSE:
 955                 BUILD_BUG_ON(!IS_ALIGNED(sizeof(_mali_uk_gp_suspend_response_s), sizeof(u64)));
 956                 err = gp_suspend_response_wrapper(session_data, (_mali_uk_gp_suspend_response_s __user *)arg);
 957                 break;
 958
 959         case MALI_IOC_VSYNC_EVENT_REPORT:
 960                 BUILD_BUG_ON(!IS_ALIGNED(sizeof(_mali_uk_vsync_event_report_s), sizeof(u64)));
 961                 err = vsync_event_report_wrapper(session_data, (_mali_uk_vsync_event_report_s __user *)arg);
 962                 break;
 963
 964         case MALI_IOC_TIMELINE_GET_LATEST_POINT:
 965                 BUILD_BUG_ON(!IS_ALIGNED(sizeof(_mali_uk_timeline_get_latest_point_s), sizeof(u64)));
 966                 err = timeline_get_latest_point_wrapper(session_data, (_mali_uk_timeline_get_latest_point_s __user *)arg);
 967                 break;
 968         case MALI_IOC_TIMELINE_WAIT:
 969                 BUILD_BUG_ON(!IS_ALIGNED(sizeof(_mali_uk_timeline_wait_s), sizeof(u64)));
 970                 err = timeline_wait_wrapper(session_data, (_mali_uk_timeline_wait_s __user *)arg);
 971                 break;
 972         case MALI_IOC_TIMELINE_CREATE_SYNC_FENCE:
 973                 BUILD_BUG_ON(!IS_ALIGNED(sizeof(_mali_uk_timeline_create_sync_fence_s), sizeof(u64)));
 974                 err = timeline_create_sync_fence_wrapper(session_data, (_mali_uk_timeline_create_sync_fence_s __user *)arg);
 975                 break;
 976         case MALI_IOC_SOFT_JOB_START:
 977                 BUILD_BUG_ON(!IS_ALIGNED(sizeof(_mali_uk_soft_job_start_s), sizeof(u64)));
 978                 err = soft_job_start_wrapper(session_data, (_mali_uk_soft_job_start_s __user *)arg);
 979                 break;
 980         case MALI_IOC_SOFT_JOB_SIGNAL:
 981                 BUILD_BUG_ON(!IS_ALIGNED(sizeof(_mali_uk_soft_job_signal_s), sizeof(u64)));
 982                 err = soft_job_signal_wrapper(session_data, (_mali_uk_soft_job_signal_s __user *)arg);
 983                 break;
 984
 985         case MALI_IOC_MEM_INIT:
 986                 err = mem_init_wrapper(session_data, (_mali_uk_init_mem_s __user *)arg);
 987                 break;
 988         case MALI_IOC_MEM_TERM:
 989                 err = mem_term_wrapper(session_data, (_mali_uk_term_mem_s __user *)arg);
 990                 break;
 991
 992         default:
 993                 MALI_DEBUG_PRINT(2, ("No handler for ioctl 0x%08X 0x%08lX\n", cmd, arg));
 994                 err = -ENOTTY;
 995         };
 996
 997         return err;
 998 }
 999
1000
1001 module_init(mali_module_init);
1002 module_exit(mali_module_exit);
1003
1004 MODULE_LICENSE(MALI_KERNEL_LINUX_LICENSE);
1005 MODULE_AUTHOR("ARM Ltd.");
1006 MODULE_VERSION(SVN_REV_STRING);