drivers/tee/optee/smc_abi.c

   1 // SPDX-License-Identifier: GPL-2.0-only
   2 /*
   3  * Copyright (c) 2015-2021, Linaro Limited
   4  * Copyright (c) 2016, EPAM Systems
   5  */
   6
   7 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
   8
   9 #include <linux/arm-smccc.h>
  10 #include <linux/cpuhotplug.h>
  11 #include <linux/errno.h>
  12 #include <linux/firmware.h>
  13 #include <linux/interrupt.h>
  14 #include <linux/io.h>
  15 #include <linux/irqdomain.h>
  16 #include <linux/kernel.h>
  17 #include <linux/mm.h>
  18 #include <linux/module.h>
  19 #include <linux/of.h>
  20 #include <linux/of_irq.h>
  21 #include <linux/of_platform.h>
  22 #include <linux/platform_device.h>
  23 #include <linux/sched.h>
  24 #include <linux/slab.h>
  25 #include <linux/string.h>
  26 #include <linux/tee_drv.h>
  27 #include <linux/types.h>
  28 #include <linux/workqueue.h>
  29 #include "optee_private.h"
  30 #include "optee_smc.h"
  31 #include "optee_rpc_cmd.h"
  32 #include <linux/kmemleak.h>
  33 #define CREATE_TRACE_POINTS
  34 #include "optee_trace.h"
  35
  36 /*
  37  * This file implement the SMC ABI used when communicating with secure world
  38  * OP-TEE OS via raw SMCs.
  39  * This file is divided into the following sections:
  40  * 1. Convert between struct tee_param and struct optee_msg_param
  41  * 2. Low level support functions to register shared memory in secure world
  42  * 3. Dynamic shared memory pool based on alloc_pages()
  43  * 4. Do a normal scheduled call into secure world
  44  * 5. Asynchronous notification
  45  * 6. Driver initialization.
  46  */
  47
  48 /*
  49  * A typical OP-TEE private shm allocation is 224 bytes (argument struct
  50  * with 6 parameters, needed for open session). So with an alignment of 512
  51  * we'll waste a bit more than 50%. However, it's only expected that we'll
  52  * have a handful of these structs allocated at a time. Most memory will
  53  * be allocated aligned to the page size, So all in all this should scale
  54  * up and down quite well.
  55  */
  56 #define OPTEE_MIN_STATIC_POOL_ALIGN    9 /* 512 bytes aligned */
  57
  58 /* SMC ABI considers at most a single TEE firmware */
  59 static unsigned int pcpu_irq_num;
  60
  61 static int optee_cpuhp_enable_pcpu_irq(unsigned int cpu)
  62 {
  63         enable_percpu_irq(pcpu_irq_num, IRQ_TYPE_NONE);
  64
  65         return 0;
  66 }
  67
  68 static int optee_cpuhp_disable_pcpu_irq(unsigned int cpu)
  69 {
  70         disable_percpu_irq(pcpu_irq_num);
  71
  72         return 0;
  73 }
  74
  75 /*
  76  * 1. Convert between struct tee_param and struct optee_msg_param
  77  *
  78  * optee_from_msg_param() and optee_to_msg_param() are the main
  79  * functions.
  80  */
  81
  82 static int from_msg_param_tmp_mem(struct tee_param *p, u32 attr,
  83                                   const struct optee_msg_param *mp)
  84 {
  85         struct tee_shm *shm;
  86         phys_addr_t pa;
  87         int rc;
  88
  89         p->attr = TEE_IOCTL_PARAM_ATTR_TYPE_MEMREF_INPUT +
  90                   attr - OPTEE_MSG_ATTR_TYPE_TMEM_INPUT;
  91         p->u.memref.size = mp->u.tmem.size;
  92         shm = (struct tee_shm *)(unsigned long)mp->u.tmem.shm_ref;
  93         if (!shm) {
  94                 p->u.memref.shm_offs = 0;
  95                 p->u.memref.shm = NULL;
  96                 return 0;
  97         }
  98
  99         rc = tee_shm_get_pa(shm, 0, &pa);
 100         if (rc)
 101                 return rc;
 102
 103         p->u.memref.shm_offs = mp->u.tmem.buf_ptr - pa;
 104         p->u.memref.shm = shm;
 105
 106         return 0;
 107 }
 108
 109 static void from_msg_param_reg_mem(struct tee_param *p, u32 attr,
 110                                    const struct optee_msg_param *mp)
 111 {
 112         struct tee_shm *shm;
 113
 114         p->attr = TEE_IOCTL_PARAM_ATTR_TYPE_MEMREF_INPUT +
 115                   attr - OPTEE_MSG_ATTR_TYPE_RMEM_INPUT;
 116         p->u.memref.size = mp->u.rmem.size;
 117         shm = (struct tee_shm *)(unsigned long)mp->u.rmem.shm_ref;
 118
 119         if (shm) {
 120                 p->u.memref.shm_offs = mp->u.rmem.offs;
 121                 p->u.memref.shm = shm;
 122         } else {
 123                 p->u.memref.shm_offs = 0;
 124                 p->u.memref.shm = NULL;
 125         }
 126 }
 127
 128 /**
 129  * optee_from_msg_param() - convert from OPTEE_MSG parameters to
 130  *                          struct tee_param
 131  * @optee:      main service struct
 132  * @params:     subsystem internal parameter representation
 133  * @num_params: number of elements in the parameter arrays
 134  * @msg_params: OPTEE_MSG parameters
 135  * Returns 0 on success or <0 on failure
 136  */
 137 static int optee_from_msg_param(struct optee *optee, struct tee_param *params,
 138                                 size_t num_params,
 139                                 const struct optee_msg_param *msg_params)
 140 {
 141         int rc;
 142         size_t n;
 143
 144         for (n = 0; n < num_params; n++) {
 145                 struct tee_param *p = params + n;
 146                 const struct optee_msg_param *mp = msg_params + n;
 147                 u32 attr = mp->attr & OPTEE_MSG_ATTR_TYPE_MASK;
 148
 149                 switch (attr) {
 150                 case OPTEE_MSG_ATTR_TYPE_NONE:
 151                         p->attr = TEE_IOCTL_PARAM_ATTR_TYPE_NONE;
 152                         memset(&p->u, 0, sizeof(p->u));
 153                         break;
 154                 case OPTEE_MSG_ATTR_TYPE_VALUE_INPUT:
 155                 case OPTEE_MSG_ATTR_TYPE_VALUE_OUTPUT:
 156                 case OPTEE_MSG_ATTR_TYPE_VALUE_INOUT:
 157                         optee_from_msg_param_value(p, attr, mp);
 158                         break;
 159                 case OPTEE_MSG_ATTR_TYPE_TMEM_INPUT:
 160                 case OPTEE_MSG_ATTR_TYPE_TMEM_OUTPUT:
 161                 case OPTEE_MSG_ATTR_TYPE_TMEM_INOUT:
 162                         rc = from_msg_param_tmp_mem(p, attr, mp);
 163                         if (rc)
 164                                 return rc;
 165                         break;
 166                 case OPTEE_MSG_ATTR_TYPE_RMEM_INPUT:
 167                 case OPTEE_MSG_ATTR_TYPE_RMEM_OUTPUT:
 168                 case OPTEE_MSG_ATTR_TYPE_RMEM_INOUT:
 169                         from_msg_param_reg_mem(p, attr, mp);
 170                         break;
 171
 172                 default:
 173                         return -EINVAL;
 174                 }
 175         }
 176         return 0;
 177 }
 178
 179 static int to_msg_param_tmp_mem(struct optee_msg_param *mp,
 180                                 const struct tee_param *p)
 181 {
 182         int rc;
 183         phys_addr_t pa;
 184
 185         mp->attr = OPTEE_MSG_ATTR_TYPE_TMEM_INPUT + p->attr -
 186                    TEE_IOCTL_PARAM_ATTR_TYPE_MEMREF_INPUT;
 187
 188         mp->u.tmem.shm_ref = (unsigned long)p->u.memref.shm;
 189         mp->u.tmem.size = p->u.memref.size;
 190
 191         if (!p->u.memref.shm) {
 192                 mp->u.tmem.buf_ptr = 0;
 193                 return 0;
 194         }
 195
 196         rc = tee_shm_get_pa(p->u.memref.shm, p->u.memref.shm_offs, &pa);
 197         if (rc)
 198                 return rc;
 199
 200         mp->u.tmem.buf_ptr = pa;
 201         mp->attr |= OPTEE_MSG_ATTR_CACHE_PREDEFINED <<
 202                     OPTEE_MSG_ATTR_CACHE_SHIFT;
 203
 204         return 0;
 205 }
 206
 207 static int to_msg_param_reg_mem(struct optee_msg_param *mp,
 208                                 const struct tee_param *p)
 209 {
 210         mp->attr = OPTEE_MSG_ATTR_TYPE_RMEM_INPUT + p->attr -
 211                    TEE_IOCTL_PARAM_ATTR_TYPE_MEMREF_INPUT;
 212
 213         mp->u.rmem.shm_ref = (unsigned long)p->u.memref.shm;
 214         mp->u.rmem.size = p->u.memref.size;
 215         mp->u.rmem.offs = p->u.memref.shm_offs;
 216         return 0;
 217 }
 218
 219 /**
 220  * optee_to_msg_param() - convert from struct tee_params to OPTEE_MSG parameters
 221  * @optee:      main service struct
 222  * @msg_params: OPTEE_MSG parameters
 223  * @num_params: number of elements in the parameter arrays
 224  * @params:     subsystem itnernal parameter representation
 225  * Returns 0 on success or <0 on failure
 226  */
 227 static int optee_to_msg_param(struct optee *optee,
 228                               struct optee_msg_param *msg_params,
 229                               size_t num_params, const struct tee_param *params)
 230 {
 231         int rc;
 232         size_t n;
 233
 234         for (n = 0; n < num_params; n++) {
 235                 const struct tee_param *p = params + n;
 236                 struct optee_msg_param *mp = msg_params + n;
 237
 238                 switch (p->attr) {
 239                 case TEE_IOCTL_PARAM_ATTR_TYPE_NONE:
 240                         mp->attr = TEE_IOCTL_PARAM_ATTR_TYPE_NONE;
 241                         memset(&mp->u, 0, sizeof(mp->u));
 242                         break;
 243                 case TEE_IOCTL_PARAM_ATTR_TYPE_VALUE_INPUT:
 244                 case TEE_IOCTL_PARAM_ATTR_TYPE_VALUE_OUTPUT:
 245                 case TEE_IOCTL_PARAM_ATTR_TYPE_VALUE_INOUT:
 246                         optee_to_msg_param_value(mp, p);
 247                         break;
 248                 case TEE_IOCTL_PARAM_ATTR_TYPE_MEMREF_INPUT:
 249                 case TEE_IOCTL_PARAM_ATTR_TYPE_MEMREF_OUTPUT:
 250                 case TEE_IOCTL_PARAM_ATTR_TYPE_MEMREF_INOUT:
 251                         if (tee_shm_is_dynamic(p->u.memref.shm))
 252                                 rc = to_msg_param_reg_mem(mp, p);
 253                         else
 254                                 rc = to_msg_param_tmp_mem(mp, p);
 255                         if (rc)
 256                                 return rc;
 257                         break;
 258                 default:
 259                         return -EINVAL;
 260                 }
 261         }
 262         return 0;
 263 }
 264
 265 /*
 266  * 2. Low level support functions to register shared memory in secure world
 267  *
 268  * Functions to enable/disable shared memory caching in secure world, that
 269  * is, lazy freeing of previously allocated shared memory. Freeing is
 270  * performed when a request has been compled.
 271  *
 272  * Functions to register and unregister shared memory both for normal
 273  * clients and for tee-supplicant.
 274  */
 275
 276 /**
 277  * optee_enable_shm_cache() - Enables caching of some shared memory allocation
 278  *                            in OP-TEE
 279  * @optee:      main service struct
 280  */
 281 static void optee_enable_shm_cache(struct optee *optee)
 282 {
 283         struct optee_call_waiter w;
 284
 285         /* We need to retry until secure world isn't busy. */
 286         optee_cq_wait_init(&optee->call_queue, &w);
 287         while (true) {
 288                 struct arm_smccc_res res;
 289
 290                 optee->smc.invoke_fn(OPTEE_SMC_ENABLE_SHM_CACHE,
 291                                      0, 0, 0, 0, 0, 0, 0, &res);
 292                 if (res.a0 == OPTEE_SMC_RETURN_OK)
 293                         break;
 294                 optee_cq_wait_for_completion(&optee->call_queue, &w);
 295         }
 296         optee_cq_wait_final(&optee->call_queue, &w);
 297 }
 298
 299 /**
 300  * __optee_disable_shm_cache() - Disables caching of some shared memory
 301  *                               allocation in OP-TEE
 302  * @optee:      main service struct
 303  * @is_mapped:  true if the cached shared memory addresses were mapped by this
 304  *              kernel, are safe to dereference, and should be freed
 305  */
 306 static void __optee_disable_shm_cache(struct optee *optee, bool is_mapped)
 307 {
 308         struct optee_call_waiter w;
 309
 310         /* We need to retry until secure world isn't busy. */
 311         optee_cq_wait_init(&optee->call_queue, &w);
 312         while (true) {
 313                 union {
 314                         struct arm_smccc_res smccc;
 315                         struct optee_smc_disable_shm_cache_result result;
 316                 } res;
 317
 318                 optee->smc.invoke_fn(OPTEE_SMC_DISABLE_SHM_CACHE,
 319                                      0, 0, 0, 0, 0, 0, 0, &res.smccc);
 320                 if (res.result.status == OPTEE_SMC_RETURN_ENOTAVAIL)
 321                         break; /* All shm's freed */
 322                 if (res.result.status == OPTEE_SMC_RETURN_OK) {
 323                         struct tee_shm *shm;
 324
 325                         /*
 326                          * Shared memory references that were not mapped by
 327                          * this kernel must be ignored to prevent a crash.
 328                          */
 329                         if (!is_mapped)
 330                                 continue;
 331
 332                         shm = reg_pair_to_ptr(res.result.shm_upper32,
 333                                               res.result.shm_lower32);
 334                         tee_shm_free(shm);
 335                 } else {
 336                         optee_cq_wait_for_completion(&optee->call_queue, &w);
 337                 }
 338         }
 339         optee_cq_wait_final(&optee->call_queue, &w);
 340 }
 341
 342 /**
 343  * optee_disable_shm_cache() - Disables caching of mapped shared memory
 344  *                             allocations in OP-TEE
 345  * @optee:      main service struct
 346  */
 347 static void optee_disable_shm_cache(struct optee *optee)
 348 {
 349         return __optee_disable_shm_cache(optee, true);
 350 }
 351
 352 /**
 353  * optee_disable_unmapped_shm_cache() - Disables caching of shared memory
 354  *                                      allocations in OP-TEE which are not
 355  *                                      currently mapped
 356  * @optee:      main service struct
 357  */
 358 static void optee_disable_unmapped_shm_cache(struct optee *optee)
 359 {
 360         return __optee_disable_shm_cache(optee, false);
 361 }
 362
 363 #define PAGELIST_ENTRIES_PER_PAGE                               \
 364         ((OPTEE_MSG_NONCONTIG_PAGE_SIZE / sizeof(u64)) - 1)
 365
 366 /*
 367  * The final entry in each pagelist page is a pointer to the next
 368  * pagelist page.
 369  */
 370 static size_t get_pages_list_size(size_t num_entries)
 371 {
 372         int pages = DIV_ROUND_UP(num_entries, PAGELIST_ENTRIES_PER_PAGE);
 373
 374         return pages * OPTEE_MSG_NONCONTIG_PAGE_SIZE;
 375 }
 376
 377 static u64 *optee_allocate_pages_list(size_t num_entries)
 378 {
 379         return alloc_pages_exact(get_pages_list_size(num_entries), GFP_KERNEL);
 380 }
 381
 382 static void optee_free_pages_list(void *list, size_t num_entries)
 383 {
 384         free_pages_exact(list, get_pages_list_size(num_entries));
 385 }
 386
 387 /**
 388  * optee_fill_pages_list() - write list of user pages to given shared
 389  * buffer.
 390  *
 391  * @dst: page-aligned buffer where list of pages will be stored
 392  * @pages: array of pages that represents shared buffer
 393  * @num_pages: number of entries in @pages
 394  * @page_offset: offset of user buffer from page start
 395  *
 396  * @dst should be big enough to hold list of user page addresses and
 397  *      links to the next pages of buffer
 398  */
 399 static void optee_fill_pages_list(u64 *dst, struct page **pages, int num_pages,
 400                                   size_t page_offset)
 401 {
 402         int n = 0;
 403         phys_addr_t optee_page;
 404         /*
 405          * Refer to OPTEE_MSG_ATTR_NONCONTIG description in optee_msg.h
 406          * for details.
 407          */
 408         struct {
 409                 u64 pages_list[PAGELIST_ENTRIES_PER_PAGE];
 410                 u64 next_page_data;
 411         } *pages_data;
 412
 413         /*
 414          * Currently OP-TEE uses 4k page size and it does not looks
 415          * like this will change in the future.  On other hand, there are
 416          * no know ARM architectures with page size < 4k.
 417          * Thus the next built assert looks redundant. But the following
 418          * code heavily relies on this assumption, so it is better be
 419          * safe than sorry.
 420          */
 421         BUILD_BUG_ON(PAGE_SIZE < OPTEE_MSG_NONCONTIG_PAGE_SIZE);
 422
 423         pages_data = (void *)dst;
 424         /*
 425          * If linux page is bigger than 4k, and user buffer offset is
 426          * larger than 4k/8k/12k/etc this will skip first 4k pages,
 427          * because they bear no value data for OP-TEE.
 428          */
 429         optee_page = page_to_phys(*pages) +
 430                 round_down(page_offset, OPTEE_MSG_NONCONTIG_PAGE_SIZE);
 431
 432         while (true) {
 433                 pages_data->pages_list[n++] = optee_page;
 434
 435                 if (n == PAGELIST_ENTRIES_PER_PAGE) {
 436                         pages_data->next_page_data =
 437                                 virt_to_phys(pages_data + 1);
 438                         pages_data++;
 439                         n = 0;
 440                 }
 441
 442                 optee_page += OPTEE_MSG_NONCONTIG_PAGE_SIZE;
 443                 if (!(optee_page & ~PAGE_MASK)) {
 444                         if (!--num_pages)
 445                                 break;
 446                         pages++;
 447                         optee_page = page_to_phys(*pages);
 448                 }
 449         }
 450 }
 451
 452 static int optee_shm_register(struct tee_context *ctx, struct tee_shm *shm,
 453                               struct page **pages, size_t num_pages,
 454                               unsigned long start)
 455 {
 456         struct optee *optee = tee_get_drvdata(ctx->teedev);
 457         struct optee_msg_arg *msg_arg;
 458         struct tee_shm *shm_arg;
 459         u64 *pages_list;
 460         size_t sz;
 461         int rc;
 462
 463         if (!num_pages)
 464                 return -EINVAL;
 465
 466         rc = optee_check_mem_type(start, num_pages);
 467         if (rc)
 468                 return rc;
 469
 470         pages_list = optee_allocate_pages_list(num_pages);
 471         if (!pages_list)
 472                 return -ENOMEM;
 473
 474         /*
 475          * We're about to register shared memory we can't register shared
 476          * memory for this request or there's a catch-22.
 477          *
 478          * So in this we'll have to do the good old temporary private
 479          * allocation instead of using optee_get_msg_arg().
 480          */
 481         sz = optee_msg_arg_size(optee->rpc_param_count);
 482         shm_arg = tee_shm_alloc_priv_buf(ctx, sz);
 483         if (IS_ERR(shm_arg)) {
 484                 rc = PTR_ERR(shm_arg);
 485                 goto out;
 486         }
 487         msg_arg = tee_shm_get_va(shm_arg, 0);
 488         if (IS_ERR(msg_arg)) {
 489                 rc = PTR_ERR(msg_arg);
 490                 goto out;
 491         }
 492
 493         optee_fill_pages_list(pages_list, pages, num_pages,
 494                               tee_shm_get_page_offset(shm));
 495
 496         memset(msg_arg, 0, OPTEE_MSG_GET_ARG_SIZE(1));
 497         msg_arg->num_params = 1;
 498         msg_arg->cmd = OPTEE_MSG_CMD_REGISTER_SHM;
 499         msg_arg->params->attr = OPTEE_MSG_ATTR_TYPE_TMEM_OUTPUT |
 500                                 OPTEE_MSG_ATTR_NONCONTIG;
 501         msg_arg->params->u.tmem.shm_ref = (unsigned long)shm;
 502         msg_arg->params->u.tmem.size = tee_shm_get_size(shm);
 503         /*
 504          * In the least bits of msg_arg->params->u.tmem.buf_ptr we
 505          * store buffer offset from 4k page, as described in OP-TEE ABI.
 506          */
 507         msg_arg->params->u.tmem.buf_ptr = virt_to_phys(pages_list) |
 508           (tee_shm_get_page_offset(shm) & (OPTEE_MSG_NONCONTIG_PAGE_SIZE - 1));
 509
 510         if (optee->ops->do_call_with_arg(ctx, shm_arg, 0) ||
 511             msg_arg->ret != TEEC_SUCCESS)
 512                 rc = -EINVAL;
 513
 514         tee_shm_free(shm_arg);
 515 out:
 516         optee_free_pages_list(pages_list, num_pages);
 517         return rc;
 518 }
 519
 520 static int optee_shm_unregister(struct tee_context *ctx, struct tee_shm *shm)
 521 {
 522         struct optee *optee = tee_get_drvdata(ctx->teedev);
 523         struct optee_msg_arg *msg_arg;
 524         struct tee_shm *shm_arg;
 525         int rc = 0;
 526         size_t sz;
 527
 528         /*
 529          * We're about to unregister shared memory and we may not be able
 530          * register shared memory for this request in case we're called
 531          * from optee_shm_arg_cache_uninit().
 532          *
 533          * So in order to keep things simple in this function just as in
 534          * optee_shm_register() we'll use temporary private allocation
 535          * instead of using optee_get_msg_arg().
 536          */
 537         sz = optee_msg_arg_size(optee->rpc_param_count);
 538         shm_arg = tee_shm_alloc_priv_buf(ctx, sz);
 539         if (IS_ERR(shm_arg))
 540                 return PTR_ERR(shm_arg);
 541         msg_arg = tee_shm_get_va(shm_arg, 0);
 542         if (IS_ERR(msg_arg)) {
 543                 rc = PTR_ERR(msg_arg);
 544                 goto out;
 545         }
 546
 547         memset(msg_arg, 0, sz);
 548         msg_arg->num_params = 1;
 549         msg_arg->cmd = OPTEE_MSG_CMD_UNREGISTER_SHM;
 550         msg_arg->params[0].attr = OPTEE_MSG_ATTR_TYPE_RMEM_INPUT;
 551         msg_arg->params[0].u.rmem.shm_ref = (unsigned long)shm;
 552
 553         if (optee->ops->do_call_with_arg(ctx, shm_arg, 0) ||
 554             msg_arg->ret != TEEC_SUCCESS)
 555                 rc = -EINVAL;
 556 out:
 557         tee_shm_free(shm_arg);
 558         return rc;
 559 }
 560
 561 static int optee_shm_register_supp(struct tee_context *ctx, struct tee_shm *shm,
 562                                    struct page **pages, size_t num_pages,
 563                                    unsigned long start)
 564 {
 565         /*
 566          * We don't want to register supplicant memory in OP-TEE.
 567          * Instead information about it will be passed in RPC code.
 568          */
 569         return optee_check_mem_type(start, num_pages);
 570 }
 571
 572 static int optee_shm_unregister_supp(struct tee_context *ctx,
 573                                      struct tee_shm *shm)
 574 {
 575         return 0;
 576 }
 577
 578 /*
 579  * 3. Dynamic shared memory pool based on alloc_pages()
 580  *
 581  * Implements an OP-TEE specific shared memory pool which is used
 582  * when dynamic shared memory is supported by secure world.
 583  *
 584  * The main function is optee_shm_pool_alloc_pages().
 585  */
 586
 587 static int pool_op_alloc(struct tee_shm_pool *pool,
 588                          struct tee_shm *shm, size_t size, size_t align)
 589 {
 590         /*
 591          * Shared memory private to the OP-TEE driver doesn't need
 592          * to be registered with OP-TEE.
 593          */
 594         if (shm->flags & TEE_SHM_PRIV)
 595                 return optee_pool_op_alloc_helper(pool, shm, size, align, NULL);
 596
 597         return optee_pool_op_alloc_helper(pool, shm, size, align,
 598                                           optee_shm_register);
 599 }
 600
 601 static void pool_op_free(struct tee_shm_pool *pool,
 602                          struct tee_shm *shm)
 603 {
 604         if (!(shm->flags & TEE_SHM_PRIV))
 605                 optee_pool_op_free_helper(pool, shm, optee_shm_unregister);
 606         else
 607                 optee_pool_op_free_helper(pool, shm, NULL);
 608 }
 609
 610 static void pool_op_destroy_pool(struct tee_shm_pool *pool)
 611 {
 612         kfree(pool);
 613 }
 614
 615 static const struct tee_shm_pool_ops pool_ops = {
 616         .alloc = pool_op_alloc,
 617         .free = pool_op_free,
 618         .destroy_pool = pool_op_destroy_pool,
 619 };
 620
 621 /**
 622  * optee_shm_pool_alloc_pages() - create page-based allocator pool
 623  *
 624  * This pool is used when OP-TEE supports dymanic SHM. In this case
 625  * command buffers and such are allocated from kernel's own memory.
 626  */
 627 static struct tee_shm_pool *optee_shm_pool_alloc_pages(void)
 628 {
 629         struct tee_shm_pool *pool = kzalloc(sizeof(*pool), GFP_KERNEL);
 630
 631         if (!pool)
 632                 return ERR_PTR(-ENOMEM);
 633
 634         pool->ops = &pool_ops;
 635
 636         return pool;
 637 }
 638
 639 /*
 640  * 4. Do a normal scheduled call into secure world
 641  *
 642  * The function optee_smc_do_call_with_arg() performs a normal scheduled
 643  * call into secure world. During this call may normal world request help
 644  * from normal world using RPCs, Remote Procedure Calls. This includes
 645  * delivery of non-secure interrupts to for instance allow rescheduling of
 646  * the current task.
 647  */
 648
 649 static void handle_rpc_func_cmd_shm_free(struct tee_context *ctx,
 650                                          struct optee_msg_arg *arg)
 651 {
 652         struct tee_shm *shm;
 653
 654         arg->ret_origin = TEEC_ORIGIN_COMMS;
 655
 656         if (arg->num_params != 1 ||
 657             arg->params[0].attr != OPTEE_MSG_ATTR_TYPE_VALUE_INPUT) {
 658                 arg->ret = TEEC_ERROR_BAD_PARAMETERS;
 659                 return;
 660         }
 661
 662         shm = (struct tee_shm *)(unsigned long)arg->params[0].u.value.b;
 663         switch (arg->params[0].u.value.a) {
 664         case OPTEE_RPC_SHM_TYPE_APPL:
 665                 optee_rpc_cmd_free_suppl(ctx, shm);
 666                 break;
 667         case OPTEE_RPC_SHM_TYPE_KERNEL:
 668                 tee_shm_free(shm);
 669                 break;
 670         default:
 671                 arg->ret = TEEC_ERROR_BAD_PARAMETERS;
 672         }
 673         arg->ret = TEEC_SUCCESS;
 674 }
 675
 676 static void handle_rpc_func_cmd_shm_alloc(struct tee_context *ctx,
 677                                           struct optee *optee,
 678                                           struct optee_msg_arg *arg,
 679                                           struct optee_call_ctx *call_ctx)
 680 {
 681         phys_addr_t pa;
 682         struct tee_shm *shm;
 683         size_t sz;
 684         size_t n;
 685
 686         arg->ret_origin = TEEC_ORIGIN_COMMS;
 687
 688         if (!arg->num_params ||
 689             arg->params[0].attr != OPTEE_MSG_ATTR_TYPE_VALUE_INPUT) {
 690                 arg->ret = TEEC_ERROR_BAD_PARAMETERS;
 691                 return;
 692         }
 693
 694         for (n = 1; n < arg->num_params; n++) {
 695                 if (arg->params[n].attr != OPTEE_MSG_ATTR_TYPE_NONE) {
 696                         arg->ret = TEEC_ERROR_BAD_PARAMETERS;
 697                         return;
 698                 }
 699         }
 700
 701         sz = arg->params[0].u.value.b;
 702         switch (arg->params[0].u.value.a) {
 703         case OPTEE_RPC_SHM_TYPE_APPL:
 704                 shm = optee_rpc_cmd_alloc_suppl(ctx, sz);
 705                 break;
 706         case OPTEE_RPC_SHM_TYPE_KERNEL:
 707                 shm = tee_shm_alloc_priv_buf(optee->ctx, sz);
 708                 break;
 709         default:
 710                 arg->ret = TEEC_ERROR_BAD_PARAMETERS;
 711                 return;
 712         }
 713
 714         if (IS_ERR(shm)) {
 715                 arg->ret = TEEC_ERROR_OUT_OF_MEMORY;
 716                 return;
 717         }
 718
 719         if (tee_shm_get_pa(shm, 0, &pa)) {
 720                 arg->ret = TEEC_ERROR_BAD_PARAMETERS;
 721                 goto bad;
 722         }
 723
 724         sz = tee_shm_get_size(shm);
 725
 726         if (tee_shm_is_dynamic(shm)) {
 727                 struct page **pages;
 728                 u64 *pages_list;
 729                 size_t page_num;
 730
 731                 pages = tee_shm_get_pages(shm, &page_num);
 732                 if (!pages || !page_num) {
 733                         arg->ret = TEEC_ERROR_OUT_OF_MEMORY;
 734                         goto bad;
 735                 }
 736
 737                 pages_list = optee_allocate_pages_list(page_num);
 738                 if (!pages_list) {
 739                         arg->ret = TEEC_ERROR_OUT_OF_MEMORY;
 740                         goto bad;
 741                 }
 742
 743                 call_ctx->pages_list = pages_list;
 744                 call_ctx->num_entries = page_num;
 745
 746                 arg->params[0].attr = OPTEE_MSG_ATTR_TYPE_TMEM_OUTPUT |
 747                                       OPTEE_MSG_ATTR_NONCONTIG;
 748                 /*
 749                  * In the least bits of u.tmem.buf_ptr we store buffer offset
 750                  * from 4k page, as described in OP-TEE ABI.
 751                  */
 752                 arg->params[0].u.tmem.buf_ptr = virt_to_phys(pages_list) |
 753                         (tee_shm_get_page_offset(shm) &
 754                          (OPTEE_MSG_NONCONTIG_PAGE_SIZE - 1));
 755                 arg->params[0].u.tmem.size = tee_shm_get_size(shm);
 756                 arg->params[0].u.tmem.shm_ref = (unsigned long)shm;
 757
 758                 optee_fill_pages_list(pages_list, pages, page_num,
 759                                       tee_shm_get_page_offset(shm));
 760         } else {
 761                 arg->params[0].attr = OPTEE_MSG_ATTR_TYPE_TMEM_OUTPUT;
 762                 arg->params[0].u.tmem.buf_ptr = pa;
 763                 arg->params[0].u.tmem.size = sz;
 764                 arg->params[0].u.tmem.shm_ref = (unsigned long)shm;
 765         }
 766
 767         arg->ret = TEEC_SUCCESS;
 768         return;
 769 bad:
 770         tee_shm_free(shm);
 771 }
 772
 773 static void free_pages_list(struct optee_call_ctx *call_ctx)
 774 {
 775         if (call_ctx->pages_list) {
 776                 optee_free_pages_list(call_ctx->pages_list,
 777                                       call_ctx->num_entries);
 778                 call_ctx->pages_list = NULL;
 779                 call_ctx->num_entries = 0;
 780         }
 781 }
 782
 783 static void optee_rpc_finalize_call(struct optee_call_ctx *call_ctx)
 784 {
 785         free_pages_list(call_ctx);
 786 }
 787
 788 static void handle_rpc_func_cmd(struct tee_context *ctx, struct optee *optee,
 789                                 struct optee_msg_arg *arg,
 790                                 struct optee_call_ctx *call_ctx)
 791 {
 792
 793         switch (arg->cmd) {
 794         case OPTEE_RPC_CMD_SHM_ALLOC:
 795                 free_pages_list(call_ctx);
 796                 handle_rpc_func_cmd_shm_alloc(ctx, optee, arg, call_ctx);
 797                 break;
 798         case OPTEE_RPC_CMD_SHM_FREE:
 799                 handle_rpc_func_cmd_shm_free(ctx, arg);
 800                 break;
 801         default:
 802                 optee_rpc_cmd(ctx, optee, arg);
 803         }
 804 }
 805
 806 /**
 807  * optee_handle_rpc() - handle RPC from secure world
 808  * @ctx:        context doing the RPC
 809  * @param:      value of registers for the RPC
 810  * @call_ctx:   call context. Preserved during one OP-TEE invocation
 811  *
 812  * Result of RPC is written back into @param.
 813  */
 814 static void optee_handle_rpc(struct tee_context *ctx,
 815                              struct optee_msg_arg *rpc_arg,
 816                              struct optee_rpc_param *param,
 817                              struct optee_call_ctx *call_ctx)
 818 {
 819         struct tee_device *teedev = ctx->teedev;
 820         struct optee *optee = tee_get_drvdata(teedev);
 821         struct optee_msg_arg *arg;
 822         struct tee_shm *shm;
 823         phys_addr_t pa;
 824
 825         switch (OPTEE_SMC_RETURN_GET_RPC_FUNC(param->a0)) {
 826         case OPTEE_SMC_RPC_FUNC_ALLOC:
 827                 shm = tee_shm_alloc_priv_buf(optee->ctx, param->a1);
 828                 if (!IS_ERR(shm) && !tee_shm_get_pa(shm, 0, &pa)) {
 829                         reg_pair_from_64(&param->a1, &param->a2, pa);
 830                         reg_pair_from_64(&param->a4, &param->a5,
 831                                          (unsigned long)shm);
 832                 } else {
 833                         param->a1 = 0;
 834                         param->a2 = 0;
 835                         param->a4 = 0;
 836                         param->a5 = 0;
 837                 }
 838                 kmemleak_not_leak(shm);
 839                 break;
 840         case OPTEE_SMC_RPC_FUNC_FREE:
 841                 shm = reg_pair_to_ptr(param->a1, param->a2);
 842                 tee_shm_free(shm);
 843                 break;
 844         case OPTEE_SMC_RPC_FUNC_FOREIGN_INTR:
 845                 /*
 846                  * A foreign interrupt was raised while secure world was
 847                  * executing, since they are handled in Linux a dummy RPC is
 848                  * performed to let Linux take the interrupt through the normal
 849                  * vector.
 850                  */
 851                 break;
 852         case OPTEE_SMC_RPC_FUNC_CMD:
 853                 if (rpc_arg) {
 854                         arg = rpc_arg;
 855                 } else {
 856                         shm = reg_pair_to_ptr(param->a1, param->a2);
 857                         arg = tee_shm_get_va(shm, 0);
 858                         if (IS_ERR(arg)) {
 859                                 pr_err("%s: tee_shm_get_va %p failed\n",
 860                                        __func__, shm);
 861                                 break;
 862                         }
 863                 }
 864
 865                 handle_rpc_func_cmd(ctx, optee, arg, call_ctx);
 866                 break;
 867         default:
 868                 pr_warn("Unknown RPC func 0x%x\n",
 869                         (u32)OPTEE_SMC_RETURN_GET_RPC_FUNC(param->a0));
 870                 break;
 871         }
 872
 873         param->a0 = OPTEE_SMC_CALL_RETURN_FROM_RPC;
 874 }
 875
 876 /**
 877  * optee_smc_do_call_with_arg() - Do an SMC to OP-TEE in secure world
 878  * @ctx:        calling context
 879  * @shm:        shared memory holding the message to pass to secure world
 880  * @offs:       offset of the message in @shm
 881  *
 882  * Does and SMC to OP-TEE in secure world and handles eventual resulting
 883  * Remote Procedure Calls (RPC) from OP-TEE.
 884  *
 885  * Returns return code from secure world, 0 is OK
 886  */
 887 static int optee_smc_do_call_with_arg(struct tee_context *ctx,
 888                                       struct tee_shm *shm, u_int offs)
 889 {
 890         struct optee *optee = tee_get_drvdata(ctx->teedev);
 891         struct optee_call_waiter w;
 892         struct optee_rpc_param param = { };
 893         struct optee_call_ctx call_ctx = { };
 894         struct optee_msg_arg *rpc_arg = NULL;
 895         int rc;
 896
 897         if (optee->rpc_param_count) {
 898                 struct optee_msg_arg *arg;
 899                 unsigned int rpc_arg_offs;
 900
 901                 arg = tee_shm_get_va(shm, offs);
 902                 if (IS_ERR(arg))
 903                         return PTR_ERR(arg);
 904
 905                 rpc_arg_offs = OPTEE_MSG_GET_ARG_SIZE(arg->num_params);
 906                 rpc_arg = tee_shm_get_va(shm, offs + rpc_arg_offs);
 907                 if (IS_ERR(rpc_arg))
 908                         return PTR_ERR(rpc_arg);
 909         }
 910
 911         if  (rpc_arg && tee_shm_is_dynamic(shm)) {
 912                 param.a0 = OPTEE_SMC_CALL_WITH_REGD_ARG;
 913                 reg_pair_from_64(&param.a1, &param.a2, (u_long)shm);
 914                 param.a3 = offs;
 915         } else {
 916                 phys_addr_t parg;
 917
 918                 rc = tee_shm_get_pa(shm, offs, &parg);
 919                 if (rc)
 920                         return rc;
 921
 922                 if (rpc_arg)
 923                         param.a0 = OPTEE_SMC_CALL_WITH_RPC_ARG;
 924                 else
 925                         param.a0 = OPTEE_SMC_CALL_WITH_ARG;
 926                 reg_pair_from_64(&param.a1, &param.a2, parg);
 927         }
 928         /* Initialize waiter */
 929         optee_cq_wait_init(&optee->call_queue, &w);
 930         while (true) {
 931                 struct arm_smccc_res res;
 932
 933                 trace_optee_invoke_fn_begin(&param);
 934                 optee->smc.invoke_fn(param.a0, param.a1, param.a2, param.a3,
 935                                      param.a4, param.a5, param.a6, param.a7,
 936                                      &res);
 937                 trace_optee_invoke_fn_end(&param, &res);
 938
 939                 if (res.a0 == OPTEE_SMC_RETURN_ETHREAD_LIMIT) {
 940                         /*
 941                          * Out of threads in secure world, wait for a thread
 942                          * become available.
 943                          */
 944                         optee_cq_wait_for_completion(&optee->call_queue, &w);
 945                 } else if (OPTEE_SMC_RETURN_IS_RPC(res.a0)) {
 946                         cond_resched();
 947                         param.a0 = res.a0;
 948                         param.a1 = res.a1;
 949                         param.a2 = res.a2;
 950                         param.a3 = res.a3;
 951                         optee_handle_rpc(ctx, rpc_arg, &param, &call_ctx);
 952                 } else {
 953                         rc = res.a0;
 954                         break;
 955                 }
 956         }
 957
 958         optee_rpc_finalize_call(&call_ctx);
 959         /*
 960          * We're done with our thread in secure world, if there's any
 961          * thread waiters wake up one.
 962          */
 963         optee_cq_wait_final(&optee->call_queue, &w);
 964
 965         return rc;
 966 }
 967
 968 static int simple_call_with_arg(struct tee_context *ctx, u32 cmd)
 969 {
 970         struct optee_shm_arg_entry *entry;
 971         struct optee_msg_arg *msg_arg;
 972         struct tee_shm *shm;
 973         u_int offs;
 974
 975         msg_arg = optee_get_msg_arg(ctx, 0, &entry, &shm, &offs);
 976         if (IS_ERR(msg_arg))
 977                 return PTR_ERR(msg_arg);
 978
 979         msg_arg->cmd = cmd;
 980         optee_smc_do_call_with_arg(ctx, shm, offs);
 981
 982         optee_free_msg_arg(ctx, entry, offs);
 983         return 0;
 984 }
 985
 986 static int optee_smc_do_bottom_half(struct tee_context *ctx)
 987 {
 988         return simple_call_with_arg(ctx, OPTEE_MSG_CMD_DO_BOTTOM_HALF);
 989 }
 990
 991 static int optee_smc_stop_async_notif(struct tee_context *ctx)
 992 {
 993         return simple_call_with_arg(ctx, OPTEE_MSG_CMD_STOP_ASYNC_NOTIF);
 994 }
 995
 996 /*
 997  * 5. Asynchronous notification
 998  */
 999
1000 static u32 get_async_notif_value(optee_invoke_fn *invoke_fn, bool *value_valid,
1001                                  bool *value_pending)
1002 {
1003         struct arm_smccc_res res;
1004
1005         invoke_fn(OPTEE_SMC_GET_ASYNC_NOTIF_VALUE, 0, 0, 0, 0, 0, 0, 0, &res);
1006
1007         if (res.a0)
1008                 return 0;
1009         *value_valid = (res.a2 & OPTEE_SMC_ASYNC_NOTIF_VALUE_VALID);
1010         *value_pending = (res.a2 & OPTEE_SMC_ASYNC_NOTIF_VALUE_PENDING);
1011         return res.a1;
1012 }
1013
1014 static irqreturn_t irq_handler(struct optee *optee)
1015 {
1016         bool do_bottom_half = false;
1017         bool value_valid;
1018         bool value_pending;
1019         u32 value;
1020
1021         do {
1022                 value = get_async_notif_value(optee->smc.invoke_fn,
1023                                               &value_valid, &value_pending);
1024                 if (!value_valid)
1025                         break;
1026
1027                 if (value == OPTEE_SMC_ASYNC_NOTIF_VALUE_DO_BOTTOM_HALF)
1028                         do_bottom_half = true;
1029                 else
1030                         optee_notif_send(optee, value);
1031         } while (value_pending);
1032
1033         if (do_bottom_half)
1034                 return IRQ_WAKE_THREAD;
1035         return IRQ_HANDLED;
1036 }
1037
1038 static irqreturn_t notif_irq_handler(int irq, void *dev_id)
1039 {
1040         struct optee *optee = dev_id;
1041
1042         return irq_handler(optee);
1043 }
1044
1045 static irqreturn_t notif_irq_thread_fn(int irq, void *dev_id)
1046 {
1047         struct optee *optee = dev_id;
1048
1049         optee_smc_do_bottom_half(optee->ctx);
1050
1051         return IRQ_HANDLED;
1052 }
1053
1054 static int init_irq(struct optee *optee, u_int irq)
1055 {
1056         int rc;
1057
1058         rc = request_threaded_irq(irq, notif_irq_handler,
1059                                   notif_irq_thread_fn,
1060                                   0, "optee_notification", optee);
1061         if (rc)
1062                 return rc;
1063
1064         optee->smc.notif_irq = irq;
1065
1066         return 0;
1067 }
1068
1069 static irqreturn_t notif_pcpu_irq_handler(int irq, void *dev_id)
1070 {
1071         struct optee_pcpu *pcpu = dev_id;
1072         struct optee *optee = pcpu->optee;
1073
1074         if (irq_handler(optee) == IRQ_WAKE_THREAD)
1075                 queue_work(optee->smc.notif_pcpu_wq,
1076                            &optee->smc.notif_pcpu_work);
1077
1078         return IRQ_HANDLED;
1079 }
1080
1081 static void notif_pcpu_irq_work_fn(struct work_struct *work)
1082 {
1083         struct optee_smc *optee_smc = container_of(work, struct optee_smc,
1084                                                    notif_pcpu_work);
1085         struct optee *optee = container_of(optee_smc, struct optee, smc);
1086
1087         optee_smc_do_bottom_half(optee->ctx);
1088 }
1089
1090 static int init_pcpu_irq(struct optee *optee, u_int irq)
1091 {
1092         struct optee_pcpu __percpu *optee_pcpu;
1093         int cpu, rc;
1094
1095         optee_pcpu = alloc_percpu(struct optee_pcpu);
1096         if (!optee_pcpu)
1097                 return -ENOMEM;
1098
1099         for_each_present_cpu(cpu)
1100                 per_cpu_ptr(optee_pcpu, cpu)->optee = optee;
1101
1102         rc = request_percpu_irq(irq, notif_pcpu_irq_handler,
1103                                 "optee_pcpu_notification", optee_pcpu);
1104         if (rc)
1105                 goto err_free_pcpu;
1106
1107         INIT_WORK(&optee->smc.notif_pcpu_work, notif_pcpu_irq_work_fn);
1108         optee->smc.notif_pcpu_wq = create_workqueue("optee_pcpu_notification");
1109         if (!optee->smc.notif_pcpu_wq) {
1110                 rc = -EINVAL;
1111                 goto err_free_pcpu_irq;
1112         }
1113
1114         optee->smc.optee_pcpu = optee_pcpu;
1115         optee->smc.notif_irq = irq;
1116
1117         pcpu_irq_num = irq;
1118         rc = cpuhp_setup_state(CPUHP_AP_ONLINE_DYN, "optee/pcpu-notif:starting",
1119                                optee_cpuhp_enable_pcpu_irq,
1120                                optee_cpuhp_disable_pcpu_irq);
1121         if (!rc)
1122                 rc = -EINVAL;
1123         if (rc < 0)
1124                 goto err_free_pcpu_irq;
1125
1126         optee->smc.notif_cpuhp_state = rc;
1127
1128         return 0;
1129
1130 err_free_pcpu_irq:
1131         free_percpu_irq(irq, optee_pcpu);
1132 err_free_pcpu:
1133         free_percpu(optee_pcpu);
1134
1135         return rc;
1136 }
1137
1138 static int optee_smc_notif_init_irq(struct optee *optee, u_int irq)
1139 {
1140         if (irq_is_percpu_devid(irq))
1141                 return init_pcpu_irq(optee, irq);
1142         else
1143                 return init_irq(optee, irq);
1144 }
1145
1146 static void uninit_pcpu_irq(struct optee *optee)
1147 {
1148         cpuhp_remove_state(optee->smc.notif_cpuhp_state);
1149
1150         destroy_workqueue(optee->smc.notif_pcpu_wq);
1151
1152         free_percpu_irq(optee->smc.notif_irq, optee->smc.optee_pcpu);
1153         free_percpu(optee->smc.optee_pcpu);
1154 }
1155
1156 static void optee_smc_notif_uninit_irq(struct optee *optee)
1157 {
1158         if (optee->smc.sec_caps & OPTEE_SMC_SEC_CAP_ASYNC_NOTIF) {
1159                 optee_smc_stop_async_notif(optee->ctx);
1160                 if (optee->smc.notif_irq) {
1161                         if (irq_is_percpu_devid(optee->smc.notif_irq))
1162                                 uninit_pcpu_irq(optee);
1163                         else
1164                                 free_irq(optee->smc.notif_irq, optee);
1165
1166                         irq_dispose_mapping(optee->smc.notif_irq);
1167                 }
1168         }
1169 }
1170
1171 /*
1172  * 6. Driver initialization
1173  *
1174  * During driver initialization is secure world probed to find out which
1175  * features it supports so the driver can be initialized with a matching
1176  * configuration. This involves for instance support for dynamic shared
1177  * memory instead of a static memory carvout.
1178  */
1179
1180 static void optee_get_version(struct tee_device *teedev,
1181                               struct tee_ioctl_version_data *vers)
1182 {
1183         struct tee_ioctl_version_data v = {
1184                 .impl_id = TEE_IMPL_ID_OPTEE,
1185                 .impl_caps = TEE_OPTEE_CAP_TZ,
1186                 .gen_caps = TEE_GEN_CAP_GP,
1187         };
1188         struct optee *optee = tee_get_drvdata(teedev);
1189
1190         if (optee->smc.sec_caps & OPTEE_SMC_SEC_CAP_DYNAMIC_SHM)
1191                 v.gen_caps |= TEE_GEN_CAP_REG_MEM;
1192         if (optee->smc.sec_caps & OPTEE_SMC_SEC_CAP_MEMREF_NULL)
1193                 v.gen_caps |= TEE_GEN_CAP_MEMREF_NULL;
1194         *vers = v;
1195 }
1196
1197 static int optee_smc_open(struct tee_context *ctx)
1198 {
1199         struct optee *optee = tee_get_drvdata(ctx->teedev);
1200         u32 sec_caps = optee->smc.sec_caps;
1201
1202         return optee_open(ctx, sec_caps & OPTEE_SMC_SEC_CAP_MEMREF_NULL);
1203 }
1204
1205 static const struct tee_driver_ops optee_clnt_ops = {
1206         .get_version = optee_get_version,
1207         .open = optee_smc_open,
1208         .release = optee_release,
1209         .open_session = optee_open_session,
1210         .close_session = optee_close_session,
1211         .invoke_func = optee_invoke_func,
1212         .cancel_req = optee_cancel_req,
1213         .shm_register = optee_shm_register,
1214         .shm_unregister = optee_shm_unregister,
1215 };
1216
1217 static const struct tee_desc optee_clnt_desc = {
1218         .name = DRIVER_NAME "-clnt",
1219         .ops = &optee_clnt_ops,
1220         .owner = THIS_MODULE,
1221 };
1222
1223 static const struct tee_driver_ops optee_supp_ops = {
1224         .get_version = optee_get_version,
1225         .open = optee_smc_open,
1226         .release = optee_release_supp,
1227         .supp_recv = optee_supp_recv,
1228         .supp_send = optee_supp_send,
1229         .shm_register = optee_shm_register_supp,
1230         .shm_unregister = optee_shm_unregister_supp,
1231 };
1232
1233 static const struct tee_desc optee_supp_desc = {
1234         .name = DRIVER_NAME "-supp",
1235         .ops = &optee_supp_ops,
1236         .owner = THIS_MODULE,
1237         .flags = TEE_DESC_PRIVILEGED,
1238 };
1239
1240 static const struct optee_ops optee_ops = {
1241         .do_call_with_arg = optee_smc_do_call_with_arg,
1242         .to_msg_param = optee_to_msg_param,
1243         .from_msg_param = optee_from_msg_param,
1244 };
1245
1246 static int enable_async_notif(optee_invoke_fn *invoke_fn)
1247 {
1248         struct arm_smccc_res res;
1249
1250         invoke_fn(OPTEE_SMC_ENABLE_ASYNC_NOTIF, 0, 0, 0, 0, 0, 0, 0, &res);
1251
1252         if (res.a0)
1253                 return -EINVAL;
1254         return 0;
1255 }
1256
1257 static bool optee_msg_api_uid_is_optee_api(optee_invoke_fn *invoke_fn)
1258 {
1259         struct arm_smccc_res res;
1260
1261         invoke_fn(OPTEE_SMC_CALLS_UID, 0, 0, 0, 0, 0, 0, 0, &res);
1262
1263         if (res.a0 == OPTEE_MSG_UID_0 && res.a1 == OPTEE_MSG_UID_1 &&
1264             res.a2 == OPTEE_MSG_UID_2 && res.a3 == OPTEE_MSG_UID_3)
1265                 return true;
1266         return false;
1267 }
1268
1269 #ifdef CONFIG_OPTEE_INSECURE_LOAD_IMAGE
1270 static bool optee_msg_api_uid_is_optee_image_load(optee_invoke_fn *invoke_fn)
1271 {
1272         struct arm_smccc_res res;
1273
1274         invoke_fn(OPTEE_SMC_CALLS_UID, 0, 0, 0, 0, 0, 0, 0, &res);
1275
1276         if (res.a0 == OPTEE_MSG_IMAGE_LOAD_UID_0 &&
1277             res.a1 == OPTEE_MSG_IMAGE_LOAD_UID_1 &&
1278             res.a2 == OPTEE_MSG_IMAGE_LOAD_UID_2 &&
1279             res.a3 == OPTEE_MSG_IMAGE_LOAD_UID_3)
1280                 return true;
1281         return false;
1282 }
1283 #endif
1284
1285 static void optee_msg_get_os_revision(optee_invoke_fn *invoke_fn)
1286 {
1287         union {
1288                 struct arm_smccc_res smccc;
1289                 struct optee_smc_call_get_os_revision_result result;
1290         } res = {
1291                 .result = {
1292                         .build_id = 0
1293                 }
1294         };
1295
1296         invoke_fn(OPTEE_SMC_CALL_GET_OS_REVISION, 0, 0, 0, 0, 0, 0, 0,
1297                   &res.smccc);
1298
1299         if (res.result.build_id)
1300                 pr_info("revision %lu.%lu (%08lx)", res.result.major,
1301                         res.result.minor, res.result.build_id);
1302         else
1303                 pr_info("revision %lu.%lu", res.result.major, res.result.minor);
1304 }
1305
1306 static bool optee_msg_api_revision_is_compatible(optee_invoke_fn *invoke_fn)
1307 {
1308         union {
1309                 struct arm_smccc_res smccc;
1310                 struct optee_smc_calls_revision_result result;
1311         } res;
1312
1313         invoke_fn(OPTEE_SMC_CALLS_REVISION, 0, 0, 0, 0, 0, 0, 0, &res.smccc);
1314
1315         if (res.result.major == OPTEE_MSG_REVISION_MAJOR &&
1316             (int)res.result.minor >= OPTEE_MSG_REVISION_MINOR)
1317                 return true;
1318         return false;
1319 }
1320
1321 static bool optee_msg_exchange_capabilities(optee_invoke_fn *invoke_fn,
1322                                             u32 *sec_caps, u32 *max_notif_value,
1323                                             unsigned int *rpc_param_count)
1324 {
1325         union {
1326                 struct arm_smccc_res smccc;
1327                 struct optee_smc_exchange_capabilities_result result;
1328         } res;
1329         u32 a1 = 0;
1330
1331         /*
1332          * TODO This isn't enough to tell if it's UP system (from kernel
1333          * point of view) or not, is_smp() returns the information
1334          * needed, but can't be called directly from here.
1335          */
1336         if (!IS_ENABLED(CONFIG_SMP) || nr_cpu_ids == 1)
1337                 a1 |= OPTEE_SMC_NSEC_CAP_UNIPROCESSOR;
1338
1339         invoke_fn(OPTEE_SMC_EXCHANGE_CAPABILITIES, a1, 0, 0, 0, 0, 0, 0,
1340                   &res.smccc);
1341
1342         if (res.result.status != OPTEE_SMC_RETURN_OK)
1343                 return false;
1344
1345         *sec_caps = res.result.capabilities;
1346         if (*sec_caps & OPTEE_SMC_SEC_CAP_ASYNC_NOTIF)
1347                 *max_notif_value = res.result.max_notif_value;
1348         else
1349                 *max_notif_value = OPTEE_DEFAULT_MAX_NOTIF_VALUE;
1350         if (*sec_caps & OPTEE_SMC_SEC_CAP_RPC_ARG)
1351                 *rpc_param_count = (u8)res.result.data;
1352         else
1353                 *rpc_param_count = 0;
1354
1355         return true;
1356 }
1357
1358 static struct tee_shm_pool *
1359 optee_config_shm_memremap(optee_invoke_fn *invoke_fn, void **memremaped_shm)
1360 {
1361         union {
1362                 struct arm_smccc_res smccc;
1363                 struct optee_smc_get_shm_config_result result;
1364         } res;
1365         unsigned long vaddr;
1366         phys_addr_t paddr;
1367         size_t size;
1368         phys_addr_t begin;
1369         phys_addr_t end;
1370         void *va;
1371         void *rc;
1372
1373         invoke_fn(OPTEE_SMC_GET_SHM_CONFIG, 0, 0, 0, 0, 0, 0, 0, &res.smccc);
1374         if (res.result.status != OPTEE_SMC_RETURN_OK) {
1375                 pr_err("static shm service not available\n");
1376                 return ERR_PTR(-ENOENT);
1377         }
1378
1379         if (res.result.settings != OPTEE_SMC_SHM_CACHED) {
1380                 pr_err("only normal cached shared memory supported\n");
1381                 return ERR_PTR(-EINVAL);
1382         }
1383
1384         begin = roundup(res.result.start, PAGE_SIZE);
1385         end = rounddown(res.result.start + res.result.size, PAGE_SIZE);
1386         paddr = begin;
1387         size = end - begin;
1388
1389         va = memremap(paddr, size, MEMREMAP_WB);
1390         if (!va) {
1391                 pr_err("shared memory ioremap failed\n");
1392                 return ERR_PTR(-EINVAL);
1393         }
1394         vaddr = (unsigned long)va;
1395
1396         rc = tee_shm_pool_alloc_res_mem(vaddr, paddr, size,
1397                                         OPTEE_MIN_STATIC_POOL_ALIGN);
1398         if (IS_ERR(rc))
1399                 memunmap(va);
1400         else
1401                 *memremaped_shm = va;
1402
1403         return rc;
1404 }
1405
1406 /* Simple wrapper functions to be able to use a function pointer */
1407 static void optee_smccc_smc(unsigned long a0, unsigned long a1,
1408                             unsigned long a2, unsigned long a3,
1409                             unsigned long a4, unsigned long a5,
1410                             unsigned long a6, unsigned long a7,
1411                             struct arm_smccc_res *res)
1412 {
1413         arm_smccc_smc(a0, a1, a2, a3, a4, a5, a6, a7, res);
1414 }
1415
1416 static void optee_smccc_hvc(unsigned long a0, unsigned long a1,
1417                             unsigned long a2, unsigned long a3,
1418                             unsigned long a4, unsigned long a5,
1419                             unsigned long a6, unsigned long a7,
1420                             struct arm_smccc_res *res)
1421 {
1422         arm_smccc_hvc(a0, a1, a2, a3, a4, a5, a6, a7, res);
1423 }
1424
1425 static optee_invoke_fn *get_invoke_func(struct device *dev)
1426 {
1427         const char *method;
1428
1429         pr_info("probing for conduit method.\n");
1430
1431         if (device_property_read_string(dev, "method", &method)) {
1432                 pr_warn("missing \"method\" property\n");
1433                 return ERR_PTR(-ENXIO);
1434         }
1435
1436         if (!strcmp("hvc", method))
1437                 return optee_smccc_hvc;
1438         else if (!strcmp("smc", method))
1439                 return optee_smccc_smc;
1440
1441         pr_warn("invalid \"method\" property: %s\n", method);
1442         return ERR_PTR(-EINVAL);
1443 }
1444
1445 /* optee_remove - Device Removal Routine
1446  * @pdev: platform device information struct
1447  *
1448  * optee_remove is called by platform subsystem to alert the driver
1449  * that it should release the device
1450  */
1451 static int optee_smc_remove(struct platform_device *pdev)
1452 {
1453         struct optee *optee = platform_get_drvdata(pdev);
1454
1455         /*
1456          * Ask OP-TEE to free all cached shared memory objects to decrease
1457          * reference counters and also avoid wild pointers in secure world
1458          * into the old shared memory range.
1459          */
1460         if (!optee->rpc_param_count)
1461                 optee_disable_shm_cache(optee);
1462
1463         optee_smc_notif_uninit_irq(optee);
1464
1465         optee_remove_common(optee);
1466
1467         if (optee->smc.memremaped_shm)
1468                 memunmap(optee->smc.memremaped_shm);
1469
1470         kfree(optee);
1471
1472         return 0;
1473 }
1474
1475 /* optee_shutdown - Device Removal Routine
1476  * @pdev: platform device information struct
1477  *
1478  * platform_shutdown is called by the platform subsystem to alert
1479  * the driver that a shutdown, reboot, or kexec is happening and
1480  * device must be disabled.
1481  */
1482 static void optee_shutdown(struct platform_device *pdev)
1483 {
1484         struct optee *optee = platform_get_drvdata(pdev);
1485
1486         if (!optee->rpc_param_count)
1487                 optee_disable_shm_cache(optee);
1488 }
1489
1490 #ifdef CONFIG_OPTEE_INSECURE_LOAD_IMAGE
1491
1492 #define OPTEE_FW_IMAGE "optee/tee.bin"
1493
1494 static optee_invoke_fn *cpuhp_invoke_fn;
1495
1496 static int optee_cpuhp_probe(unsigned int cpu)
1497 {
1498         /*
1499          * Invoking a call on a CPU will cause OP-TEE to perform the required
1500          * setup for that CPU. Just invoke the call to get the UID since that
1501          * has no side effects.
1502          */
1503         if (optee_msg_api_uid_is_optee_api(cpuhp_invoke_fn))
1504                 return 0;
1505         else
1506                 return -EINVAL;
1507 }
1508
1509 static int optee_load_fw(struct platform_device *pdev,
1510                          optee_invoke_fn *invoke_fn)
1511 {
1512         const struct firmware *fw = NULL;
1513         struct arm_smccc_res res;
1514         phys_addr_t data_pa;
1515         u8 *data_buf = NULL;
1516         u64 data_size;
1517         u32 data_pa_high, data_pa_low;
1518         u32 data_size_high, data_size_low;
1519         int rc;
1520         int hp_state;
1521
1522         if (!optee_msg_api_uid_is_optee_image_load(invoke_fn))
1523                 return 0;
1524
1525         rc = request_firmware(&fw, OPTEE_FW_IMAGE, &pdev->dev);
1526         if (rc) {
1527                 /*
1528                  * The firmware in the rootfs will not be accessible until we
1529                  * are in the SYSTEM_RUNNING state, so return EPROBE_DEFER until
1530                  * that point.
1531                  */
1532                 if (system_state < SYSTEM_RUNNING)
1533                         return -EPROBE_DEFER;
1534                 goto fw_err;
1535         }
1536
1537         data_size = fw->size;
1538         /*
1539          * This uses the GFP_DMA flag to ensure we are allocated memory in the
1540          * 32-bit space since TF-A cannot map memory beyond the 32-bit boundary.
1541          */
1542         data_buf = kmalloc(fw->size, GFP_KERNEL | GFP_DMA);
1543         if (!data_buf) {
1544                 rc = -ENOMEM;
1545                 goto fw_err;
1546         }
1547         memcpy(data_buf, fw->data, fw->size);
1548         data_pa = virt_to_phys(data_buf);
1549         reg_pair_from_64(&data_pa_high, &data_pa_low, data_pa);
1550         reg_pair_from_64(&data_size_high, &data_size_low, data_size);
1551         goto fw_load;
1552
1553 fw_err:
1554         pr_warn("image loading failed\n");
1555         data_pa_high = 0;
1556         data_pa_low = 0;
1557         data_size_high = 0;
1558         data_size_low = 0;
1559
1560 fw_load:
1561         /*
1562          * Always invoke the SMC, even if loading the image fails, to indicate
1563          * to EL3 that we have passed the point where it should allow invoking
1564          * this SMC.
1565          */
1566         pr_warn("OP-TEE image loaded from kernel, this can be insecure");
1567         invoke_fn(OPTEE_SMC_CALL_LOAD_IMAGE, data_size_high, data_size_low,
1568                   data_pa_high, data_pa_low, 0, 0, 0, &res);
1569         if (!rc)
1570                 rc = res.a0;
1571         if (fw)
1572                 release_firmware(fw);
1573         kfree(data_buf);
1574
1575         if (!rc) {
1576                 /*
1577                  * We need to initialize OP-TEE on all other running cores as
1578                  * well. Any cores that aren't running yet will get initialized
1579                  * when they are brought up by the power management functions in
1580                  * TF-A which are registered by the OP-TEE SPD. Due to that we
1581                  * can un-register the callback right after registering it.
1582                  */
1583                 cpuhp_invoke_fn = invoke_fn;
1584                 hp_state = cpuhp_setup_state(CPUHP_AP_ONLINE_DYN, "optee:probe",
1585                                              optee_cpuhp_probe, NULL);
1586                 if (hp_state < 0) {
1587                         pr_warn("Failed with CPU hotplug setup for OP-TEE");
1588                         return -EINVAL;
1589                 }
1590                 cpuhp_remove_state(hp_state);
1591                 cpuhp_invoke_fn = NULL;
1592         }
1593
1594         return rc;
1595 }
1596 #else
1597 static inline int optee_load_fw(struct platform_device *pdev,
1598                                 optee_invoke_fn *invoke_fn)
1599 {
1600         return 0;
1601 }
1602 #endif
1603
1604 static int optee_probe(struct platform_device *pdev)
1605 {
1606         optee_invoke_fn *invoke_fn;
1607         struct tee_shm_pool *pool = ERR_PTR(-EINVAL);
1608         struct optee *optee = NULL;
1609         void *memremaped_shm = NULL;
1610         unsigned int rpc_param_count;
1611         struct tee_device *teedev;
1612         struct tee_context *ctx;
1613         u32 max_notif_value;
1614         u32 arg_cache_flags;
1615         u32 sec_caps;
1616         int rc;
1617
1618         invoke_fn = get_invoke_func(&pdev->dev);
1619         if (IS_ERR(invoke_fn))
1620                 return PTR_ERR(invoke_fn);
1621
1622         rc = optee_load_fw(pdev, invoke_fn);
1623         if (rc)
1624                 return rc;
1625
1626         if (!optee_msg_api_uid_is_optee_api(invoke_fn)) {
1627                 pr_warn("api uid mismatch\n");
1628                 return -EINVAL;
1629         }
1630
1631         optee_msg_get_os_revision(invoke_fn);
1632
1633         if (!optee_msg_api_revision_is_compatible(invoke_fn)) {
1634                 pr_warn("api revision mismatch\n");
1635                 return -EINVAL;
1636         }
1637
1638         if (!optee_msg_exchange_capabilities(invoke_fn, &sec_caps,
1639                                              &max_notif_value,
1640                                              &rpc_param_count)) {
1641                 pr_warn("capabilities mismatch\n");
1642                 return -EINVAL;
1643         }
1644
1645         /*
1646          * Try to use dynamic shared memory if possible
1647          */
1648         if (sec_caps & OPTEE_SMC_SEC_CAP_DYNAMIC_SHM) {
1649                 /*
1650                  * If we have OPTEE_SMC_SEC_CAP_RPC_ARG we can ask
1651                  * optee_get_msg_arg() to pre-register (by having
1652                  * OPTEE_SHM_ARG_ALLOC_PRIV cleared) the page used to pass
1653                  * an argument struct.
1654                  *
1655                  * With the page is pre-registered we can use a non-zero
1656                  * offset for argument struct, this is indicated with
1657                  * OPTEE_SHM_ARG_SHARED.
1658                  *
1659                  * This means that optee_smc_do_call_with_arg() will use
1660                  * OPTEE_SMC_CALL_WITH_REGD_ARG for pre-registered pages.
1661                  */
1662                 if (sec_caps & OPTEE_SMC_SEC_CAP_RPC_ARG)
1663                         arg_cache_flags = OPTEE_SHM_ARG_SHARED;
1664                 else
1665                         arg_cache_flags = OPTEE_SHM_ARG_ALLOC_PRIV;
1666
1667                 pool = optee_shm_pool_alloc_pages();
1668         }
1669
1670         /*
1671          * If dynamic shared memory is not available or failed - try static one
1672          */
1673         if (IS_ERR(pool) && (sec_caps & OPTEE_SMC_SEC_CAP_HAVE_RESERVED_SHM)) {
1674                 /*
1675                  * The static memory pool can use non-zero page offsets so
1676                  * let optee_get_msg_arg() know that with OPTEE_SHM_ARG_SHARED.
1677                  *
1678                  * optee_get_msg_arg() should not pre-register the
1679                  * allocated page used to pass an argument struct, this is
1680                  * indicated with OPTEE_SHM_ARG_ALLOC_PRIV.
1681                  *
1682                  * This means that optee_smc_do_call_with_arg() will use
1683                  * OPTEE_SMC_CALL_WITH_ARG if rpc_param_count is 0, else
1684                  * OPTEE_SMC_CALL_WITH_RPC_ARG.
1685                  */
1686                 arg_cache_flags = OPTEE_SHM_ARG_SHARED |
1687                                   OPTEE_SHM_ARG_ALLOC_PRIV;
1688                 pool = optee_config_shm_memremap(invoke_fn, &memremaped_shm);
1689         }
1690
1691         if (IS_ERR(pool))
1692                 return PTR_ERR(pool);
1693
1694         optee = kzalloc(sizeof(*optee), GFP_KERNEL);
1695         if (!optee) {
1696                 rc = -ENOMEM;
1697                 goto err_free_pool;
1698         }
1699
1700         optee->ops = &optee_ops;
1701         optee->smc.invoke_fn = invoke_fn;
1702         optee->smc.sec_caps = sec_caps;
1703         optee->rpc_param_count = rpc_param_count;
1704
1705         teedev = tee_device_alloc(&optee_clnt_desc, NULL, pool, optee);
1706         if (IS_ERR(teedev)) {
1707                 rc = PTR_ERR(teedev);
1708                 goto err_free_optee;
1709         }
1710         optee->teedev = teedev;
1711
1712         teedev = tee_device_alloc(&optee_supp_desc, NULL, pool, optee);
1713         if (IS_ERR(teedev)) {
1714                 rc = PTR_ERR(teedev);
1715                 goto err_unreg_teedev;
1716         }
1717         optee->supp_teedev = teedev;
1718
1719         rc = tee_device_register(optee->teedev);
1720         if (rc)
1721                 goto err_unreg_supp_teedev;
1722
1723         rc = tee_device_register(optee->supp_teedev);
1724         if (rc)
1725                 goto err_unreg_supp_teedev;
1726
1727         mutex_init(&optee->call_queue.mutex);
1728         INIT_LIST_HEAD(&optee->call_queue.waiters);
1729         optee_supp_init(&optee->supp);
1730         optee->smc.memremaped_shm = memremaped_shm;
1731         optee->pool = pool;
1732         optee_shm_arg_cache_init(optee, arg_cache_flags);
1733
1734         platform_set_drvdata(pdev, optee);
1735         ctx = teedev_open(optee->teedev);
1736         if (IS_ERR(ctx)) {
1737                 rc = PTR_ERR(ctx);
1738                 goto err_supp_uninit;
1739         }
1740         optee->ctx = ctx;
1741         rc = optee_notif_init(optee, max_notif_value);
1742         if (rc)
1743                 goto err_close_ctx;
1744
1745         if (sec_caps & OPTEE_SMC_SEC_CAP_ASYNC_NOTIF) {
1746                 unsigned int irq;
1747
1748                 rc = platform_get_irq(pdev, 0);
1749                 if (rc < 0) {
1750                         pr_err("platform_get_irq: ret %d\n", rc);
1751                         goto err_notif_uninit;
1752                 }
1753                 irq = rc;
1754
1755                 rc = optee_smc_notif_init_irq(optee, irq);
1756                 if (rc) {
1757                         irq_dispose_mapping(irq);
1758                         goto err_notif_uninit;
1759                 }
1760                 enable_async_notif(optee->smc.invoke_fn);
1761                 pr_info("Asynchronous notifications enabled\n");
1762         }
1763
1764         /*
1765          * Ensure that there are no pre-existing shm objects before enabling
1766          * the shm cache so that there's no chance of receiving an invalid
1767          * address during shutdown. This could occur, for example, if we're
1768          * kexec booting from an older kernel that did not properly cleanup the
1769          * shm cache.
1770          */
1771         optee_disable_unmapped_shm_cache(optee);
1772
1773         /*
1774          * Only enable the shm cache in case we're not able to pass the RPC
1775          * arg struct right after the normal arg struct.
1776          */
1777         if (!optee->rpc_param_count)
1778                 optee_enable_shm_cache(optee);
1779
1780         if (optee->smc.sec_caps & OPTEE_SMC_SEC_CAP_DYNAMIC_SHM)
1781                 pr_info("dynamic shared memory is enabled\n");
1782
1783         rc = optee_enumerate_devices(PTA_CMD_GET_DEVICES);
1784         if (rc)
1785                 goto err_disable_shm_cache;
1786
1787         pr_info("initialized driver\n");
1788         return 0;
1789
1790 err_disable_shm_cache:
1791         if (!optee->rpc_param_count)
1792                 optee_disable_shm_cache(optee);
1793         optee_smc_notif_uninit_irq(optee);
1794         optee_unregister_devices();
1795 err_notif_uninit:
1796         optee_notif_uninit(optee);
1797 err_close_ctx:
1798         teedev_close_context(ctx);
1799 err_supp_uninit:
1800         optee_shm_arg_cache_uninit(optee);
1801         optee_supp_uninit(&optee->supp);
1802         mutex_destroy(&optee->call_queue.mutex);
1803 err_unreg_supp_teedev:
1804         tee_device_unregister(optee->supp_teedev);
1805 err_unreg_teedev:
1806         tee_device_unregister(optee->teedev);
1807 err_free_optee:
1808         kfree(optee);
1809 err_free_pool:
1810         tee_shm_pool_free(pool);
1811         if (memremaped_shm)
1812                 memunmap(memremaped_shm);
1813         return rc;
1814 }
1815
1816 static const struct of_device_id optee_dt_match[] = {
1817         { .compatible = "linaro,optee-tz" },
1818         {},
1819 };
1820 MODULE_DEVICE_TABLE(of, optee_dt_match);
1821
1822 static struct platform_driver optee_driver = {
1823         .probe  = optee_probe,
1824         .remove = optee_smc_remove,
1825         .shutdown = optee_shutdown,
1826         .driver = {
1827                 .name = "optee",
1828                 .of_match_table = optee_dt_match,
1829         },
1830 };
1831
1832 int optee_smc_abi_register(void)
1833 {
1834         return platform_driver_register(&optee_driver);
1835 }
1836
1837 void optee_smc_abi_unregister(void)
1838 {
1839         platform_driver_unregister(&optee_driver);
1840 }