drivers/tee/optee/smc_abi.c

   1 // SPDX-License-Identifier: GPL-2.0-only
   2 /*
   3  * Copyright (c) 2015-2021, Linaro Limited
   4  * Copyright (c) 2016, EPAM Systems
   5  */
   6
   7 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
   8
   9 #include <linux/arm-smccc.h>
  10 #include <linux/cpuhotplug.h>
  11 #include <linux/errno.h>
  12 #include <linux/firmware.h>
  13 #include <linux/interrupt.h>
  14 #include <linux/io.h>
  15 #include <linux/irqdomain.h>
  16 #include <linux/kernel.h>
  17 #include <linux/mm.h>
  18 #include <linux/module.h>
  19 #include <linux/of.h>
  20 #include <linux/of_irq.h>
  21 #include <linux/of_platform.h>
  22 #include <linux/platform_device.h>
  23 #include <linux/sched.h>
  24 #include <linux/slab.h>
  25 #include <linux/string.h>
  26 #include <linux/tee_drv.h>
  27 #include <linux/types.h>
  28 #include <linux/workqueue.h>
  29 #include "optee_private.h"
  30 #include "optee_smc.h"
  31 #include "optee_rpc_cmd.h"
  32 #include <linux/kmemleak.h>
  33 #define CREATE_TRACE_POINTS
  34 #include "optee_trace.h"
  35
  36 /*
  37  * This file implement the SMC ABI used when communicating with secure world
  38  * OP-TEE OS via raw SMCs.
  39  * This file is divided into the following sections:
  40  * 1. Convert between struct tee_param and struct optee_msg_param
  41  * 2. Low level support functions to register shared memory in secure world
  42  * 3. Dynamic shared memory pool based on alloc_pages()
  43  * 4. Do a normal scheduled call into secure world
  44  * 5. Asynchronous notification
  45  * 6. Driver initialization.
  46  */
  47
  48 /*
  49  * A typical OP-TEE private shm allocation is 224 bytes (argument struct
  50  * with 6 parameters, needed for open session). So with an alignment of 512
  51  * we'll waste a bit more than 50%. However, it's only expected that we'll
  52  * have a handful of these structs allocated at a time. Most memory will
  53  * be allocated aligned to the page size, So all in all this should scale
  54  * up and down quite well.
  55  */
  56 #define OPTEE_MIN_STATIC_POOL_ALIGN    9 /* 512 bytes aligned */
  57
  58 /* SMC ABI considers at most a single TEE firmware */
  59 static unsigned int pcpu_irq_num;
  60
  61 static int optee_cpuhp_enable_pcpu_irq(unsigned int cpu)
  62 {
  63         enable_percpu_irq(pcpu_irq_num, IRQ_TYPE_NONE);
  64
  65         return 0;
  66 }
  67
  68 static int optee_cpuhp_disable_pcpu_irq(unsigned int cpu)
  69 {
  70         disable_percpu_irq(pcpu_irq_num);
  71
  72         return 0;
  73 }
  74
  75 /*
  76  * 1. Convert between struct tee_param and struct optee_msg_param
  77  *
  78  * optee_from_msg_param() and optee_to_msg_param() are the main
  79  * functions.
  80  */
  81
  82 static int from_msg_param_tmp_mem(struct tee_param *p, u32 attr,
  83                                   const struct optee_msg_param *mp)
  84 {
  85         struct tee_shm *shm;
  86         phys_addr_t pa;
  87         int rc;
  88
  89         p->attr = TEE_IOCTL_PARAM_ATTR_TYPE_MEMREF_INPUT +
  90                   attr - OPTEE_MSG_ATTR_TYPE_TMEM_INPUT;
  91         p->u.memref.size = mp->u.tmem.size;
  92         shm = (struct tee_shm *)(unsigned long)mp->u.tmem.shm_ref;
  93         if (!shm) {
  94                 p->u.memref.shm_offs = 0;
  95                 p->u.memref.shm = NULL;
  96                 return 0;
  97         }
  98
  99         rc = tee_shm_get_pa(shm, 0, &pa);
 100         if (rc)
 101                 return rc;
 102
 103         p->u.memref.shm_offs = mp->u.tmem.buf_ptr - pa;
 104         p->u.memref.shm = shm;
 105
 106         return 0;
 107 }
 108
 109 static void from_msg_param_reg_mem(struct tee_param *p, u32 attr,
 110                                    const struct optee_msg_param *mp)
 111 {
 112         struct tee_shm *shm;
 113
 114         p->attr = TEE_IOCTL_PARAM_ATTR_TYPE_MEMREF_INPUT +
 115                   attr - OPTEE_MSG_ATTR_TYPE_RMEM_INPUT;
 116         p->u.memref.size = mp->u.rmem.size;
 117         shm = (struct tee_shm *)(unsigned long)mp->u.rmem.shm_ref;
 118
 119         if (shm) {
 120                 p->u.memref.shm_offs = mp->u.rmem.offs;
 121                 p->u.memref.shm = shm;
 122         } else {
 123                 p->u.memref.shm_offs = 0;
 124                 p->u.memref.shm = NULL;
 125         }
 126 }
 127
 128 /**
 129  * optee_from_msg_param() - convert from OPTEE_MSG parameters to
 130  *                          struct tee_param
 131  * @optee:      main service struct
 132  * @params:     subsystem internal parameter representation
 133  * @num_params: number of elements in the parameter arrays
 134  * @msg_params: OPTEE_MSG parameters
 135  * Returns 0 on success or <0 on failure
 136  */
 137 static int optee_from_msg_param(struct optee *optee, struct tee_param *params,
 138                                 size_t num_params,
 139                                 const struct optee_msg_param *msg_params)
 140 {
 141         int rc;
 142         size_t n;
 143
 144         for (n = 0; n < num_params; n++) {
 145                 struct tee_param *p = params + n;
 146                 const struct optee_msg_param *mp = msg_params + n;
 147                 u32 attr = mp->attr & OPTEE_MSG_ATTR_TYPE_MASK;
 148
 149                 switch (attr) {
 150                 case OPTEE_MSG_ATTR_TYPE_NONE:
 151                         p->attr = TEE_IOCTL_PARAM_ATTR_TYPE_NONE;
 152                         memset(&p->u, 0, sizeof(p->u));
 153                         break;
 154                 case OPTEE_MSG_ATTR_TYPE_VALUE_INPUT:
 155                 case OPTEE_MSG_ATTR_TYPE_VALUE_OUTPUT:
 156                 case OPTEE_MSG_ATTR_TYPE_VALUE_INOUT:
 157                         optee_from_msg_param_value(p, attr, mp);
 158                         break;
 159                 case OPTEE_MSG_ATTR_TYPE_TMEM_INPUT:
 160                 case OPTEE_MSG_ATTR_TYPE_TMEM_OUTPUT:
 161                 case OPTEE_MSG_ATTR_TYPE_TMEM_INOUT:
 162                         rc = from_msg_param_tmp_mem(p, attr, mp);
 163                         if (rc)
 164                                 return rc;
 165                         break;
 166                 case OPTEE_MSG_ATTR_TYPE_RMEM_INPUT:
 167                 case OPTEE_MSG_ATTR_TYPE_RMEM_OUTPUT:
 168                 case OPTEE_MSG_ATTR_TYPE_RMEM_INOUT:
 169                         from_msg_param_reg_mem(p, attr, mp);
 170                         break;
 171
 172                 default:
 173                         return -EINVAL;
 174                 }
 175         }
 176         return 0;
 177 }
 178
 179 static int to_msg_param_tmp_mem(struct optee_msg_param *mp,
 180                                 const struct tee_param *p)
 181 {
 182         int rc;
 183         phys_addr_t pa;
 184
 185         mp->attr = OPTEE_MSG_ATTR_TYPE_TMEM_INPUT + p->attr -
 186                    TEE_IOCTL_PARAM_ATTR_TYPE_MEMREF_INPUT;
 187
 188         mp->u.tmem.shm_ref = (unsigned long)p->u.memref.shm;
 189         mp->u.tmem.size = p->u.memref.size;
 190
 191         if (!p->u.memref.shm) {
 192                 mp->u.tmem.buf_ptr = 0;
 193                 return 0;
 194         }
 195
 196         rc = tee_shm_get_pa(p->u.memref.shm, p->u.memref.shm_offs, &pa);
 197         if (rc)
 198                 return rc;
 199
 200         mp->u.tmem.buf_ptr = pa;
 201         mp->attr |= OPTEE_MSG_ATTR_CACHE_PREDEFINED <<
 202                     OPTEE_MSG_ATTR_CACHE_SHIFT;
 203
 204         return 0;
 205 }
 206
 207 static int to_msg_param_reg_mem(struct optee_msg_param *mp,
 208                                 const struct tee_param *p)
 209 {
 210         mp->attr = OPTEE_MSG_ATTR_TYPE_RMEM_INPUT + p->attr -
 211                    TEE_IOCTL_PARAM_ATTR_TYPE_MEMREF_INPUT;
 212
 213         mp->u.rmem.shm_ref = (unsigned long)p->u.memref.shm;
 214         mp->u.rmem.size = p->u.memref.size;
 215         mp->u.rmem.offs = p->u.memref.shm_offs;
 216         return 0;
 217 }
 218
 219 /**
 220  * optee_to_msg_param() - convert from struct tee_params to OPTEE_MSG parameters
 221  * @optee:      main service struct
 222  * @msg_params: OPTEE_MSG parameters
 223  * @num_params: number of elements in the parameter arrays
 224  * @params:     subsystem itnernal parameter representation
 225  * Returns 0 on success or <0 on failure
 226  */
 227 static int optee_to_msg_param(struct optee *optee,
 228                               struct optee_msg_param *msg_params,
 229                               size_t num_params, const struct tee_param *params)
 230 {
 231         int rc;
 232         size_t n;
 233
 234         for (n = 0; n < num_params; n++) {
 235                 const struct tee_param *p = params + n;
 236                 struct optee_msg_param *mp = msg_params + n;
 237
 238                 switch (p->attr) {
 239                 case TEE_IOCTL_PARAM_ATTR_TYPE_NONE:
 240                         mp->attr = TEE_IOCTL_PARAM_ATTR_TYPE_NONE;
 241                         memset(&mp->u, 0, sizeof(mp->u));
 242                         break;
 243                 case TEE_IOCTL_PARAM_ATTR_TYPE_VALUE_INPUT:
 244                 case TEE_IOCTL_PARAM_ATTR_TYPE_VALUE_OUTPUT:
 245                 case TEE_IOCTL_PARAM_ATTR_TYPE_VALUE_INOUT:
 246                         optee_to_msg_param_value(mp, p);
 247                         break;
 248                 case TEE_IOCTL_PARAM_ATTR_TYPE_MEMREF_INPUT:
 249                 case TEE_IOCTL_PARAM_ATTR_TYPE_MEMREF_OUTPUT:
 250                 case TEE_IOCTL_PARAM_ATTR_TYPE_MEMREF_INOUT:
 251                         if (tee_shm_is_dynamic(p->u.memref.shm))
 252                                 rc = to_msg_param_reg_mem(mp, p);
 253                         else
 254                                 rc = to_msg_param_tmp_mem(mp, p);
 255                         if (rc)
 256                                 return rc;
 257                         break;
 258                 default:
 259                         return -EINVAL;
 260                 }
 261         }
 262         return 0;
 263 }
 264
 265 /*
 266  * 2. Low level support functions to register shared memory in secure world
 267  *
 268  * Functions to enable/disable shared memory caching in secure world, that
 269  * is, lazy freeing of previously allocated shared memory. Freeing is
 270  * performed when a request has been compled.
 271  *
 272  * Functions to register and unregister shared memory both for normal
 273  * clients and for tee-supplicant.
 274  */
 275
 276 /**
 277  * optee_enable_shm_cache() - Enables caching of some shared memory allocation
 278  *                            in OP-TEE
 279  * @optee:      main service struct
 280  */
 281 static void optee_enable_shm_cache(struct optee *optee)
 282 {
 283         struct optee_call_waiter w;
 284
 285         /* We need to retry until secure world isn't busy. */
 286         optee_cq_wait_init(&optee->call_queue, &w);
 287         while (true) {
 288                 struct arm_smccc_res res;
 289
 290                 optee->smc.invoke_fn(OPTEE_SMC_ENABLE_SHM_CACHE,
 291                                      0, 0, 0, 0, 0, 0, 0, &res);
 292                 if (res.a0 == OPTEE_SMC_RETURN_OK)
 293                         break;
 294                 optee_cq_wait_for_completion(&optee->call_queue, &w);
 295         }
 296         optee_cq_wait_final(&optee->call_queue, &w);
 297 }
 298
 299 /**
 300  * __optee_disable_shm_cache() - Disables caching of some shared memory
 301  *                               allocation in OP-TEE
 302  * @optee:      main service struct
 303  * @is_mapped:  true if the cached shared memory addresses were mapped by this
 304  *              kernel, are safe to dereference, and should be freed
 305  */
 306 static void __optee_disable_shm_cache(struct optee *optee, bool is_mapped)
 307 {
 308         struct optee_call_waiter w;
 309
 310         /* We need to retry until secure world isn't busy. */
 311         optee_cq_wait_init(&optee->call_queue, &w);
 312         while (true) {
 313                 union {
 314                         struct arm_smccc_res smccc;
 315                         struct optee_smc_disable_shm_cache_result result;
 316                 } res;
 317
 318                 optee->smc.invoke_fn(OPTEE_SMC_DISABLE_SHM_CACHE,
 319                                      0, 0, 0, 0, 0, 0, 0, &res.smccc);
 320                 if (res.result.status == OPTEE_SMC_RETURN_ENOTAVAIL)
 321                         break; /* All shm's freed */
 322                 if (res.result.status == OPTEE_SMC_RETURN_OK) {
 323                         struct tee_shm *shm;
 324
 325                         /*
 326                          * Shared memory references that were not mapped by
 327                          * this kernel must be ignored to prevent a crash.
 328                          */
 329                         if (!is_mapped)
 330                                 continue;
 331
 332                         shm = reg_pair_to_ptr(res.result.shm_upper32,
 333                                               res.result.shm_lower32);
 334                         tee_shm_free(shm);
 335                 } else {
 336                         optee_cq_wait_for_completion(&optee->call_queue, &w);
 337                 }
 338         }
 339         optee_cq_wait_final(&optee->call_queue, &w);
 340 }
 341
 342 /**
 343  * optee_disable_shm_cache() - Disables caching of mapped shared memory
 344  *                             allocations in OP-TEE
 345  * @optee:      main service struct
 346  */
 347 static void optee_disable_shm_cache(struct optee *optee)
 348 {
 349         return __optee_disable_shm_cache(optee, true);
 350 }
 351
 352 /**
 353  * optee_disable_unmapped_shm_cache() - Disables caching of shared memory
 354  *                                      allocations in OP-TEE which are not
 355  *                                      currently mapped
 356  * @optee:      main service struct
 357  */
 358 static void optee_disable_unmapped_shm_cache(struct optee *optee)
 359 {
 360         return __optee_disable_shm_cache(optee, false);
 361 }
 362
 363 #define PAGELIST_ENTRIES_PER_PAGE                               \
 364         ((OPTEE_MSG_NONCONTIG_PAGE_SIZE / sizeof(u64)) - 1)
 365
 366 /*
 367  * The final entry in each pagelist page is a pointer to the next
 368  * pagelist page.
 369  */
 370 static size_t get_pages_list_size(size_t num_entries)
 371 {
 372         int pages = DIV_ROUND_UP(num_entries, PAGELIST_ENTRIES_PER_PAGE);
 373
 374         return pages * OPTEE_MSG_NONCONTIG_PAGE_SIZE;
 375 }
 376
 377 static u64 *optee_allocate_pages_list(size_t num_entries)
 378 {
 379         return alloc_pages_exact(get_pages_list_size(num_entries), GFP_KERNEL);
 380 }
 381
 382 static void optee_free_pages_list(void *list, size_t num_entries)
 383 {
 384         free_pages_exact(list, get_pages_list_size(num_entries));
 385 }
 386
 387 /**
 388  * optee_fill_pages_list() - write list of user pages to given shared
 389  * buffer.
 390  *
 391  * @dst: page-aligned buffer where list of pages will be stored
 392  * @pages: array of pages that represents shared buffer
 393  * @num_pages: number of entries in @pages
 394  * @page_offset: offset of user buffer from page start
 395  *
 396  * @dst should be big enough to hold list of user page addresses and
 397  *      links to the next pages of buffer
 398  */
 399 static void optee_fill_pages_list(u64 *dst, struct page **pages, int num_pages,
 400                                   size_t page_offset)
 401 {
 402         int n = 0;
 403         phys_addr_t optee_page;
 404         /*
 405          * Refer to OPTEE_MSG_ATTR_NONCONTIG description in optee_msg.h
 406          * for details.
 407          */
 408         struct {
 409                 u64 pages_list[PAGELIST_ENTRIES_PER_PAGE];
 410                 u64 next_page_data;
 411         } *pages_data;
 412
 413         /*
 414          * Currently OP-TEE uses 4k page size and it does not looks
 415          * like this will change in the future.  On other hand, there are
 416          * no know ARM architectures with page size < 4k.
 417          * Thus the next built assert looks redundant. But the following
 418          * code heavily relies on this assumption, so it is better be
 419          * safe than sorry.
 420          */
 421         BUILD_BUG_ON(PAGE_SIZE < OPTEE_MSG_NONCONTIG_PAGE_SIZE);
 422
 423         pages_data = (void *)dst;
 424         /*
 425          * If linux page is bigger than 4k, and user buffer offset is
 426          * larger than 4k/8k/12k/etc this will skip first 4k pages,
 427          * because they bear no value data for OP-TEE.
 428          */
 429         optee_page = page_to_phys(*pages) +
 430                 round_down(page_offset, OPTEE_MSG_NONCONTIG_PAGE_SIZE);
 431
 432         while (true) {
 433                 pages_data->pages_list[n++] = optee_page;
 434
 435                 if (n == PAGELIST_ENTRIES_PER_PAGE) {
 436                         pages_data->next_page_data =
 437                                 virt_to_phys(pages_data + 1);
 438                         pages_data++;
 439                         n = 0;
 440                 }
 441
 442                 optee_page += OPTEE_MSG_NONCONTIG_PAGE_SIZE;
 443                 if (!(optee_page & ~PAGE_MASK)) {
 444                         if (!--num_pages)
 445                                 break;
 446                         pages++;
 447                         optee_page = page_to_phys(*pages);
 448                 }
 449         }
 450 }
 451
 452 static int optee_shm_register(struct tee_context *ctx, struct tee_shm *shm,
 453                               struct page **pages, size_t num_pages,
 454                               unsigned long start)
 455 {
 456         struct optee *optee = tee_get_drvdata(ctx->teedev);
 457         struct optee_msg_arg *msg_arg;
 458         struct tee_shm *shm_arg;
 459         u64 *pages_list;
 460         size_t sz;
 461         int rc;
 462
 463         if (!num_pages)
 464                 return -EINVAL;
 465
 466         rc = optee_check_mem_type(start, num_pages);
 467         if (rc)
 468                 return rc;
 469
 470         pages_list = optee_allocate_pages_list(num_pages);
 471         if (!pages_list)
 472                 return -ENOMEM;
 473
 474         /*
 475          * We're about to register shared memory we can't register shared
 476          * memory for this request or there's a catch-22.
 477          *
 478          * So in this we'll have to do the good old temporary private
 479          * allocation instead of using optee_get_msg_arg().
 480          */
 481         sz = optee_msg_arg_size(optee->rpc_param_count);
 482         shm_arg = tee_shm_alloc_priv_buf(ctx, sz);
 483         if (IS_ERR(shm_arg)) {
 484                 rc = PTR_ERR(shm_arg);
 485                 goto out;
 486         }
 487         msg_arg = tee_shm_get_va(shm_arg, 0);
 488         if (IS_ERR(msg_arg)) {
 489                 rc = PTR_ERR(msg_arg);
 490                 goto out;
 491         }
 492
 493         optee_fill_pages_list(pages_list, pages, num_pages,
 494                               tee_shm_get_page_offset(shm));
 495
 496         memset(msg_arg, 0, OPTEE_MSG_GET_ARG_SIZE(1));
 497         msg_arg->num_params = 1;
 498         msg_arg->cmd = OPTEE_MSG_CMD_REGISTER_SHM;
 499         msg_arg->params->attr = OPTEE_MSG_ATTR_TYPE_TMEM_OUTPUT |
 500                                 OPTEE_MSG_ATTR_NONCONTIG;
 501         msg_arg->params->u.tmem.shm_ref = (unsigned long)shm;
 502         msg_arg->params->u.tmem.size = tee_shm_get_size(shm);
 503         /*
 504          * In the least bits of msg_arg->params->u.tmem.buf_ptr we
 505          * store buffer offset from 4k page, as described in OP-TEE ABI.
 506          */
 507         msg_arg->params->u.tmem.buf_ptr = virt_to_phys(pages_list) |
 508           (tee_shm_get_page_offset(shm) & (OPTEE_MSG_NONCONTIG_PAGE_SIZE - 1));
 509
 510         if (optee->ops->do_call_with_arg(ctx, shm_arg, 0) ||
 511             msg_arg->ret != TEEC_SUCCESS)
 512                 rc = -EINVAL;
 513
 514         tee_shm_free(shm_arg);
 515 out:
 516         optee_free_pages_list(pages_list, num_pages);
 517         return rc;
 518 }
 519
 520 static int optee_shm_unregister(struct tee_context *ctx, struct tee_shm *shm)
 521 {
 522         struct optee *optee = tee_get_drvdata(ctx->teedev);
 523         struct optee_msg_arg *msg_arg;
 524         struct tee_shm *shm_arg;
 525         int rc = 0;
 526         size_t sz;
 527
 528         /*
 529          * We're about to unregister shared memory and we may not be able
 530          * register shared memory for this request in case we're called
 531          * from optee_shm_arg_cache_uninit().
 532          *
 533          * So in order to keep things simple in this function just as in
 534          * optee_shm_register() we'll use temporary private allocation
 535          * instead of using optee_get_msg_arg().
 536          */
 537         sz = optee_msg_arg_size(optee->rpc_param_count);
 538         shm_arg = tee_shm_alloc_priv_buf(ctx, sz);
 539         if (IS_ERR(shm_arg))
 540                 return PTR_ERR(shm_arg);
 541         msg_arg = tee_shm_get_va(shm_arg, 0);
 542         if (IS_ERR(msg_arg)) {
 543                 rc = PTR_ERR(msg_arg);
 544                 goto out;
 545         }
 546
 547         memset(msg_arg, 0, sz);
 548         msg_arg->num_params = 1;
 549         msg_arg->cmd = OPTEE_MSG_CMD_UNREGISTER_SHM;
 550         msg_arg->params[0].attr = OPTEE_MSG_ATTR_TYPE_RMEM_INPUT;
 551         msg_arg->params[0].u.rmem.shm_ref = (unsigned long)shm;
 552
 553         if (optee->ops->do_call_with_arg(ctx, shm_arg, 0) ||
 554             msg_arg->ret != TEEC_SUCCESS)
 555                 rc = -EINVAL;
 556 out:
 557         tee_shm_free(shm_arg);
 558         return rc;
 559 }
 560
 561 static int optee_shm_register_supp(struct tee_context *ctx, struct tee_shm *shm,
 562                                    struct page **pages, size_t num_pages,
 563                                    unsigned long start)
 564 {
 565         /*
 566          * We don't want to register supplicant memory in OP-TEE.
 567          * Instead information about it will be passed in RPC code.
 568          */
 569         return optee_check_mem_type(start, num_pages);
 570 }
 571
 572 static int optee_shm_unregister_supp(struct tee_context *ctx,
 573                                      struct tee_shm *shm)
 574 {
 575         return 0;
 576 }
 577
 578 /*
 579  * 3. Dynamic shared memory pool based on alloc_pages()
 580  *
 581  * Implements an OP-TEE specific shared memory pool which is used
 582  * when dynamic shared memory is supported by secure world.
 583  *
 584  * The main function is optee_shm_pool_alloc_pages().
 585  */
 586
 587 static int pool_op_alloc(struct tee_shm_pool *pool,
 588                          struct tee_shm *shm, size_t size, size_t align)
 589 {
 590         /*
 591          * Shared memory private to the OP-TEE driver doesn't need
 592          * to be registered with OP-TEE.
 593          */
 594         if (shm->flags & TEE_SHM_PRIV)
 595                 return optee_pool_op_alloc_helper(pool, shm, size, align, NULL);
 596
 597         return optee_pool_op_alloc_helper(pool, shm, size, align,
 598                                           optee_shm_register);
 599 }
 600
 601 static void pool_op_free(struct tee_shm_pool *pool,
 602                          struct tee_shm *shm)
 603 {
 604         if (!(shm->flags & TEE_SHM_PRIV))
 605                 optee_pool_op_free_helper(pool, shm, optee_shm_unregister);
 606         else
 607                 optee_pool_op_free_helper(pool, shm, NULL);
 608 }
 609
 610 static void pool_op_destroy_pool(struct tee_shm_pool *pool)
 611 {
 612         kfree(pool);
 613 }
 614
 615 static const struct tee_shm_pool_ops pool_ops = {
 616         .alloc = pool_op_alloc,
 617         .free = pool_op_free,
 618         .destroy_pool = pool_op_destroy_pool,
 619 };
 620
 621 /**
 622  * optee_shm_pool_alloc_pages() - create page-based allocator pool
 623  *
 624  * This pool is used when OP-TEE supports dymanic SHM. In this case
 625  * command buffers and such are allocated from kernel's own memory.
 626  */
 627 static struct tee_shm_pool *optee_shm_pool_alloc_pages(void)
 628 {
 629         struct tee_shm_pool *pool = kzalloc(sizeof(*pool), GFP_KERNEL);
 630
 631         if (!pool)
 632                 return ERR_PTR(-ENOMEM);
 633
 634         pool->ops = &pool_ops;
 635
 636         return pool;
 637 }
 638
 639 /*
 640  * 4. Do a normal scheduled call into secure world
 641  *
 642  * The function optee_smc_do_call_with_arg() performs a normal scheduled
 643  * call into secure world. During this call may normal world request help
 644  * from normal world using RPCs, Remote Procedure Calls. This includes
 645  * delivery of non-secure interrupts to for instance allow rescheduling of
 646  * the current task.
 647  */
 648
 649 static void handle_rpc_func_cmd_shm_free(struct tee_context *ctx,
 650                                          struct optee_msg_arg *arg)
 651 {
 652         struct tee_shm *shm;
 653
 654         arg->ret_origin = TEEC_ORIGIN_COMMS;
 655
 656         if (arg->num_params != 1 ||
 657             arg->params[0].attr != OPTEE_MSG_ATTR_TYPE_VALUE_INPUT) {
 658                 arg->ret = TEEC_ERROR_BAD_PARAMETERS;
 659                 return;
 660         }
 661
 662         shm = (struct tee_shm *)(unsigned long)arg->params[0].u.value.b;
 663         switch (arg->params[0].u.value.a) {
 664         case OPTEE_RPC_SHM_TYPE_APPL:
 665                 optee_rpc_cmd_free_suppl(ctx, shm);
 666                 break;
 667         case OPTEE_RPC_SHM_TYPE_KERNEL:
 668                 tee_shm_free(shm);
 669                 break;
 670         default:
 671                 arg->ret = TEEC_ERROR_BAD_PARAMETERS;
 672         }
 673         arg->ret = TEEC_SUCCESS;
 674 }
 675
 676 static void handle_rpc_func_cmd_shm_alloc(struct tee_context *ctx,
 677                                           struct optee *optee,
 678                                           struct optee_msg_arg *arg,
 679                                           struct optee_call_ctx *call_ctx)
 680 {
 681         phys_addr_t pa;
 682         struct tee_shm *shm;
 683         size_t sz;
 684         size_t n;
 685
 686         arg->ret_origin = TEEC_ORIGIN_COMMS;
 687
 688         if (!arg->num_params ||
 689             arg->params[0].attr != OPTEE_MSG_ATTR_TYPE_VALUE_INPUT) {
 690                 arg->ret = TEEC_ERROR_BAD_PARAMETERS;
 691                 return;
 692         }
 693
 694         for (n = 1; n < arg->num_params; n++) {
 695                 if (arg->params[n].attr != OPTEE_MSG_ATTR_TYPE_NONE) {
 696                         arg->ret = TEEC_ERROR_BAD_PARAMETERS;
 697                         return;
 698                 }
 699         }
 700
 701         sz = arg->params[0].u.value.b;
 702         switch (arg->params[0].u.value.a) {
 703         case OPTEE_RPC_SHM_TYPE_APPL:
 704                 shm = optee_rpc_cmd_alloc_suppl(ctx, sz);
 705                 break;
 706         case OPTEE_RPC_SHM_TYPE_KERNEL:
 707                 shm = tee_shm_alloc_priv_buf(optee->ctx, sz);
 708                 break;
 709         default:
 710                 arg->ret = TEEC_ERROR_BAD_PARAMETERS;
 711                 return;
 712         }
 713
 714         if (IS_ERR(shm)) {
 715                 arg->ret = TEEC_ERROR_OUT_OF_MEMORY;
 716                 return;
 717         }
 718
 719         if (tee_shm_get_pa(shm, 0, &pa)) {
 720                 arg->ret = TEEC_ERROR_BAD_PARAMETERS;
 721                 goto bad;
 722         }
 723
 724         sz = tee_shm_get_size(shm);
 725
 726         if (tee_shm_is_dynamic(shm)) {
 727                 struct page **pages;
 728                 u64 *pages_list;
 729                 size_t page_num;
 730
 731                 pages = tee_shm_get_pages(shm, &page_num);
 732                 if (!pages || !page_num) {
 733                         arg->ret = TEEC_ERROR_OUT_OF_MEMORY;
 734                         goto bad;
 735                 }
 736
 737                 pages_list = optee_allocate_pages_list(page_num);
 738                 if (!pages_list) {
 739                         arg->ret = TEEC_ERROR_OUT_OF_MEMORY;
 740                         goto bad;
 741                 }
 742
 743                 call_ctx->pages_list = pages_list;
 744                 call_ctx->num_entries = page_num;
 745
 746                 arg->params[0].attr = OPTEE_MSG_ATTR_TYPE_TMEM_OUTPUT |
 747                                       OPTEE_MSG_ATTR_NONCONTIG;
 748                 /*
 749                  * In the least bits of u.tmem.buf_ptr we store buffer offset
 750                  * from 4k page, as described in OP-TEE ABI.
 751                  */
 752                 arg->params[0].u.tmem.buf_ptr = virt_to_phys(pages_list) |
 753                         (tee_shm_get_page_offset(shm) &
 754                          (OPTEE_MSG_NONCONTIG_PAGE_SIZE - 1));
 755                 arg->params[0].u.tmem.size = tee_shm_get_size(shm);
 756                 arg->params[0].u.tmem.shm_ref = (unsigned long)shm;
 757
 758                 optee_fill_pages_list(pages_list, pages, page_num,
 759                                       tee_shm_get_page_offset(shm));
 760         } else {
 761                 arg->params[0].attr = OPTEE_MSG_ATTR_TYPE_TMEM_OUTPUT;
 762                 arg->params[0].u.tmem.buf_ptr = pa;
 763                 arg->params[0].u.tmem.size = sz;
 764                 arg->params[0].u.tmem.shm_ref = (unsigned long)shm;
 765         }
 766
 767         arg->ret = TEEC_SUCCESS;
 768         return;
 769 bad:
 770         tee_shm_free(shm);
 771 }
 772
 773 static void free_pages_list(struct optee_call_ctx *call_ctx)
 774 {
 775         if (call_ctx->pages_list) {
 776                 optee_free_pages_list(call_ctx->pages_list,
 777                                       call_ctx->num_entries);
 778                 call_ctx->pages_list = NULL;
 779                 call_ctx->num_entries = 0;
 780         }
 781 }
 782
 783 static void optee_rpc_finalize_call(struct optee_call_ctx *call_ctx)
 784 {
 785         free_pages_list(call_ctx);
 786 }
 787
 788 static void handle_rpc_func_cmd(struct tee_context *ctx, struct optee *optee,
 789                                 struct optee_msg_arg *arg,
 790                                 struct optee_call_ctx *call_ctx)
 791 {
 792
 793         switch (arg->cmd) {
 794         case OPTEE_RPC_CMD_SHM_ALLOC:
 795                 free_pages_list(call_ctx);
 796                 handle_rpc_func_cmd_shm_alloc(ctx, optee, arg, call_ctx);
 797                 break;
 798         case OPTEE_RPC_CMD_SHM_FREE:
 799                 handle_rpc_func_cmd_shm_free(ctx, arg);
 800                 break;
 801         default:
 802                 optee_rpc_cmd(ctx, optee, arg);
 803         }
 804 }
 805
 806 /**
 807  * optee_handle_rpc() - handle RPC from secure world
 808  * @ctx:        context doing the RPC
 809  * @param:      value of registers for the RPC
 810  * @call_ctx:   call context. Preserved during one OP-TEE invocation
 811  *
 812  * Result of RPC is written back into @param.
 813  */
 814 static void optee_handle_rpc(struct tee_context *ctx,
 815                              struct optee_msg_arg *rpc_arg,
 816                              struct optee_rpc_param *param,
 817                              struct optee_call_ctx *call_ctx)
 818 {
 819         struct tee_device *teedev = ctx->teedev;
 820         struct optee *optee = tee_get_drvdata(teedev);
 821         struct optee_msg_arg *arg;
 822         struct tee_shm *shm;
 823         phys_addr_t pa;
 824
 825         switch (OPTEE_SMC_RETURN_GET_RPC_FUNC(param->a0)) {
 826         case OPTEE_SMC_RPC_FUNC_ALLOC:
 827                 shm = tee_shm_alloc_priv_buf(optee->ctx, param->a1);
 828                 if (!IS_ERR(shm) && !tee_shm_get_pa(shm, 0, &pa)) {
 829                         reg_pair_from_64(&param->a1, &param->a2, pa);
 830                         reg_pair_from_64(&param->a4, &param->a5,
 831                                          (unsigned long)shm);
 832                 } else {
 833                         param->a1 = 0;
 834                         param->a2 = 0;
 835                         param->a4 = 0;
 836                         param->a5 = 0;
 837                 }
 838                 kmemleak_not_leak(shm);
 839                 break;
 840         case OPTEE_SMC_RPC_FUNC_FREE:
 841                 shm = reg_pair_to_ptr(param->a1, param->a2);
 842                 tee_shm_free(shm);
 843                 break;
 844         case OPTEE_SMC_RPC_FUNC_FOREIGN_INTR:
 845                 /*
 846                  * A foreign interrupt was raised while secure world was
 847                  * executing, since they are handled in Linux a dummy RPC is
 848                  * performed to let Linux take the interrupt through the normal
 849                  * vector.
 850                  */
 851                 break;
 852         case OPTEE_SMC_RPC_FUNC_CMD:
 853                 if (rpc_arg) {
 854                         arg = rpc_arg;
 855                 } else {
 856                         shm = reg_pair_to_ptr(param->a1, param->a2);
 857                         arg = tee_shm_get_va(shm, 0);
 858                         if (IS_ERR(arg)) {
 859                                 pr_err("%s: tee_shm_get_va %p failed\n",
 860                                        __func__, shm);
 861                                 break;
 862                         }
 863                 }
 864
 865                 handle_rpc_func_cmd(ctx, optee, arg, call_ctx);
 866                 break;
 867         default:
 868                 pr_warn("Unknown RPC func 0x%x\n",
 869                         (u32)OPTEE_SMC_RETURN_GET_RPC_FUNC(param->a0));
 870                 break;
 871         }
 872
 873         param->a0 = OPTEE_SMC_CALL_RETURN_FROM_RPC;
 874 }
 875
 876 /**
 877  * optee_smc_do_call_with_arg() - Do an SMC to OP-TEE in secure world
 878  * @ctx:        calling context
 879  * @shm:        shared memory holding the message to pass to secure world
 880  * @offs:       offset of the message in @shm
 881  *
 882  * Does and SMC to OP-TEE in secure world and handles eventual resulting
 883  * Remote Procedure Calls (RPC) from OP-TEE.
 884  *
 885  * Returns return code from secure world, 0 is OK
 886  */
 887 static int optee_smc_do_call_with_arg(struct tee_context *ctx,
 888                                       struct tee_shm *shm, u_int offs)
 889 {
 890         struct optee *optee = tee_get_drvdata(ctx->teedev);
 891         struct optee_call_waiter w;
 892         struct optee_rpc_param param = { };
 893         struct optee_call_ctx call_ctx = { };
 894         struct optee_msg_arg *rpc_arg = NULL;
 895         int rc;
 896
 897         if (optee->rpc_param_count) {
 898                 struct optee_msg_arg *arg;
 899                 unsigned int rpc_arg_offs;
 900
 901                 arg = tee_shm_get_va(shm, offs);
 902                 if (IS_ERR(arg))
 903                         return PTR_ERR(arg);
 904
 905                 rpc_arg_offs = OPTEE_MSG_GET_ARG_SIZE(arg->num_params);
 906                 rpc_arg = tee_shm_get_va(shm, offs + rpc_arg_offs);
 907                 if (IS_ERR(rpc_arg))
 908                         return PTR_ERR(rpc_arg);
 909         }
 910
 911         if  (rpc_arg && tee_shm_is_dynamic(shm)) {
 912                 param.a0 = OPTEE_SMC_CALL_WITH_REGD_ARG;
 913                 reg_pair_from_64(&param.a1, &param.a2, (u_long)shm);
 914                 param.a3 = offs;
 915         } else {
 916                 phys_addr_t parg;
 917
 918                 rc = tee_shm_get_pa(shm, offs, &parg);
 919                 if (rc)
 920                         return rc;
 921
 922                 if (rpc_arg)
 923                         param.a0 = OPTEE_SMC_CALL_WITH_RPC_ARG;
 924                 else
 925                         param.a0 = OPTEE_SMC_CALL_WITH_ARG;
 926                 reg_pair_from_64(&param.a1, &param.a2, parg);
 927         }
 928         /* Initialize waiter */
 929         optee_cq_wait_init(&optee->call_queue, &w);
 930         while (true) {
 931                 struct arm_smccc_res res;
 932
 933                 trace_optee_invoke_fn_begin(&param);
 934                 optee->smc.invoke_fn(param.a0, param.a1, param.a2, param.a3,
 935                                      param.a4, param.a5, param.a6, param.a7,
 936                                      &res);
 937                 trace_optee_invoke_fn_end(&param, &res);
 938
 939                 if (res.a0 == OPTEE_SMC_RETURN_ETHREAD_LIMIT) {
 940                         /*
 941                          * Out of threads in secure world, wait for a thread
 942                          * become available.
 943                          */
 944                         optee_cq_wait_for_completion(&optee->call_queue, &w);
 945                 } else if (OPTEE_SMC_RETURN_IS_RPC(res.a0)) {
 946                         cond_resched();
 947                         param.a0 = res.a0;
 948                         param.a1 = res.a1;
 949                         param.a2 = res.a2;
 950                         param.a3 = res.a3;
 951                         optee_handle_rpc(ctx, rpc_arg, &param, &call_ctx);
 952                 } else {
 953                         rc = res.a0;
 954                         break;
 955                 }
 956         }
 957
 958         optee_rpc_finalize_call(&call_ctx);
 959         /*
 960          * We're done with our thread in secure world, if there's any
 961          * thread waiters wake up one.
 962          */
 963         optee_cq_wait_final(&optee->call_queue, &w);
 964
 965         return rc;
 966 }
 967
 968 static int simple_call_with_arg(struct tee_context *ctx, u32 cmd)
 969 {
 970         struct optee_shm_arg_entry *entry;
 971         struct optee_msg_arg *msg_arg;
 972         struct tee_shm *shm;
 973         u_int offs;
 974
 975         msg_arg = optee_get_msg_arg(ctx, 0, &entry, &shm, &offs);
 976         if (IS_ERR(msg_arg))
 977                 return PTR_ERR(msg_arg);
 978
 979         msg_arg->cmd = cmd;
 980         optee_smc_do_call_with_arg(ctx, shm, offs);
 981
 982         optee_free_msg_arg(ctx, entry, offs);
 983         return 0;
 984 }
 985
 986 static int optee_smc_do_bottom_half(struct tee_context *ctx)
 987 {
 988         return simple_call_with_arg(ctx, OPTEE_MSG_CMD_DO_BOTTOM_HALF);
 989 }
 990
 991 static int optee_smc_stop_async_notif(struct tee_context *ctx)
 992 {
 993         return simple_call_with_arg(ctx, OPTEE_MSG_CMD_STOP_ASYNC_NOTIF);
 994 }
 995
 996 /*
 997  * 5. Asynchronous notification
 998  */
 999
1000 static u32 get_async_notif_value(optee_invoke_fn *invoke_fn, bool *value_valid,
1001                                  bool *value_pending)
1002 {
1003         struct arm_smccc_res res;
1004
1005         invoke_fn(OPTEE_SMC_GET_ASYNC_NOTIF_VALUE, 0, 0, 0, 0, 0, 0, 0, &res);
1006
1007         if (res.a0) {
1008                 *value_valid = false;
1009                 return 0;
1010         }
1011         *value_valid = (res.a2 & OPTEE_SMC_ASYNC_NOTIF_VALUE_VALID);
1012         *value_pending = (res.a2 & OPTEE_SMC_ASYNC_NOTIF_VALUE_PENDING);
1013         return res.a1;
1014 }
1015
1016 static irqreturn_t irq_handler(struct optee *optee)
1017 {
1018         bool do_bottom_half = false;
1019         bool value_valid;
1020         bool value_pending;
1021         u32 value;
1022
1023         do {
1024                 value = get_async_notif_value(optee->smc.invoke_fn,
1025                                               &value_valid, &value_pending);
1026                 if (!value_valid)
1027                         break;
1028
1029                 if (value == OPTEE_SMC_ASYNC_NOTIF_VALUE_DO_BOTTOM_HALF)
1030                         do_bottom_half = true;
1031                 else
1032                         optee_notif_send(optee, value);
1033         } while (value_pending);
1034
1035         if (do_bottom_half)
1036                 return IRQ_WAKE_THREAD;
1037         return IRQ_HANDLED;
1038 }
1039
1040 static irqreturn_t notif_irq_handler(int irq, void *dev_id)
1041 {
1042         struct optee *optee = dev_id;
1043
1044         return irq_handler(optee);
1045 }
1046
1047 static irqreturn_t notif_irq_thread_fn(int irq, void *dev_id)
1048 {
1049         struct optee *optee = dev_id;
1050
1051         optee_smc_do_bottom_half(optee->ctx);
1052
1053         return IRQ_HANDLED;
1054 }
1055
1056 static int init_irq(struct optee *optee, u_int irq)
1057 {
1058         int rc;
1059
1060         rc = request_threaded_irq(irq, notif_irq_handler,
1061                                   notif_irq_thread_fn,
1062                                   0, "optee_notification", optee);
1063         if (rc)
1064                 return rc;
1065
1066         optee->smc.notif_irq = irq;
1067
1068         return 0;
1069 }
1070
1071 static irqreturn_t notif_pcpu_irq_handler(int irq, void *dev_id)
1072 {
1073         struct optee_pcpu *pcpu = dev_id;
1074         struct optee *optee = pcpu->optee;
1075
1076         if (irq_handler(optee) == IRQ_WAKE_THREAD)
1077                 queue_work(optee->smc.notif_pcpu_wq,
1078                            &optee->smc.notif_pcpu_work);
1079
1080         return IRQ_HANDLED;
1081 }
1082
1083 static void notif_pcpu_irq_work_fn(struct work_struct *work)
1084 {
1085         struct optee_smc *optee_smc = container_of(work, struct optee_smc,
1086                                                    notif_pcpu_work);
1087         struct optee *optee = container_of(optee_smc, struct optee, smc);
1088
1089         optee_smc_do_bottom_half(optee->ctx);
1090 }
1091
1092 static int init_pcpu_irq(struct optee *optee, u_int irq)
1093 {
1094         struct optee_pcpu __percpu *optee_pcpu;
1095         int cpu, rc;
1096
1097         optee_pcpu = alloc_percpu(struct optee_pcpu);
1098         if (!optee_pcpu)
1099                 return -ENOMEM;
1100
1101         for_each_present_cpu(cpu)
1102                 per_cpu_ptr(optee_pcpu, cpu)->optee = optee;
1103
1104         rc = request_percpu_irq(irq, notif_pcpu_irq_handler,
1105                                 "optee_pcpu_notification", optee_pcpu);
1106         if (rc)
1107                 goto err_free_pcpu;
1108
1109         INIT_WORK(&optee->smc.notif_pcpu_work, notif_pcpu_irq_work_fn);
1110         optee->smc.notif_pcpu_wq = create_workqueue("optee_pcpu_notification");
1111         if (!optee->smc.notif_pcpu_wq) {
1112                 rc = -EINVAL;
1113                 goto err_free_pcpu_irq;
1114         }
1115
1116         optee->smc.optee_pcpu = optee_pcpu;
1117         optee->smc.notif_irq = irq;
1118
1119         pcpu_irq_num = irq;
1120         rc = cpuhp_setup_state(CPUHP_AP_ONLINE_DYN, "optee/pcpu-notif:starting",
1121                                optee_cpuhp_enable_pcpu_irq,
1122                                optee_cpuhp_disable_pcpu_irq);
1123         if (!rc)
1124                 rc = -EINVAL;
1125         if (rc < 0)
1126                 goto err_free_pcpu_irq;
1127
1128         optee->smc.notif_cpuhp_state = rc;
1129
1130         return 0;
1131
1132 err_free_pcpu_irq:
1133         free_percpu_irq(irq, optee_pcpu);
1134 err_free_pcpu:
1135         free_percpu(optee_pcpu);
1136
1137         return rc;
1138 }
1139
1140 static int optee_smc_notif_init_irq(struct optee *optee, u_int irq)
1141 {
1142         if (irq_is_percpu_devid(irq))
1143                 return init_pcpu_irq(optee, irq);
1144         else
1145                 return init_irq(optee, irq);
1146 }
1147
1148 static void uninit_pcpu_irq(struct optee *optee)
1149 {
1150         cpuhp_remove_state(optee->smc.notif_cpuhp_state);
1151
1152         destroy_workqueue(optee->smc.notif_pcpu_wq);
1153
1154         free_percpu_irq(optee->smc.notif_irq, optee->smc.optee_pcpu);
1155         free_percpu(optee->smc.optee_pcpu);
1156 }
1157
1158 static void optee_smc_notif_uninit_irq(struct optee *optee)
1159 {
1160         if (optee->smc.sec_caps & OPTEE_SMC_SEC_CAP_ASYNC_NOTIF) {
1161                 optee_smc_stop_async_notif(optee->ctx);
1162                 if (optee->smc.notif_irq) {
1163                         if (irq_is_percpu_devid(optee->smc.notif_irq))
1164                                 uninit_pcpu_irq(optee);
1165                         else
1166                                 free_irq(optee->smc.notif_irq, optee);
1167
1168                         irq_dispose_mapping(optee->smc.notif_irq);
1169                 }
1170         }
1171 }
1172
1173 /*
1174  * 6. Driver initialization
1175  *
1176  * During driver initialization is secure world probed to find out which
1177  * features it supports so the driver can be initialized with a matching
1178  * configuration. This involves for instance support for dynamic shared
1179  * memory instead of a static memory carvout.
1180  */
1181
1182 static void optee_get_version(struct tee_device *teedev,
1183                               struct tee_ioctl_version_data *vers)
1184 {
1185         struct tee_ioctl_version_data v = {
1186                 .impl_id = TEE_IMPL_ID_OPTEE,
1187                 .impl_caps = TEE_OPTEE_CAP_TZ,
1188                 .gen_caps = TEE_GEN_CAP_GP,
1189         };
1190         struct optee *optee = tee_get_drvdata(teedev);
1191
1192         if (optee->smc.sec_caps & OPTEE_SMC_SEC_CAP_DYNAMIC_SHM)
1193                 v.gen_caps |= TEE_GEN_CAP_REG_MEM;
1194         if (optee->smc.sec_caps & OPTEE_SMC_SEC_CAP_MEMREF_NULL)
1195                 v.gen_caps |= TEE_GEN_CAP_MEMREF_NULL;
1196         *vers = v;
1197 }
1198
1199 static int optee_smc_open(struct tee_context *ctx)
1200 {
1201         struct optee *optee = tee_get_drvdata(ctx->teedev);
1202         u32 sec_caps = optee->smc.sec_caps;
1203
1204         return optee_open(ctx, sec_caps & OPTEE_SMC_SEC_CAP_MEMREF_NULL);
1205 }
1206
1207 static const struct tee_driver_ops optee_clnt_ops = {
1208         .get_version = optee_get_version,
1209         .open = optee_smc_open,
1210         .release = optee_release,
1211         .open_session = optee_open_session,
1212         .close_session = optee_close_session,
1213         .invoke_func = optee_invoke_func,
1214         .cancel_req = optee_cancel_req,
1215         .shm_register = optee_shm_register,
1216         .shm_unregister = optee_shm_unregister,
1217 };
1218
1219 static const struct tee_desc optee_clnt_desc = {
1220         .name = DRIVER_NAME "-clnt",
1221         .ops = &optee_clnt_ops,
1222         .owner = THIS_MODULE,
1223 };
1224
1225 static const struct tee_driver_ops optee_supp_ops = {
1226         .get_version = optee_get_version,
1227         .open = optee_smc_open,
1228         .release = optee_release_supp,
1229         .supp_recv = optee_supp_recv,
1230         .supp_send = optee_supp_send,
1231         .shm_register = optee_shm_register_supp,
1232         .shm_unregister = optee_shm_unregister_supp,
1233 };
1234
1235 static const struct tee_desc optee_supp_desc = {
1236         .name = DRIVER_NAME "-supp",
1237         .ops = &optee_supp_ops,
1238         .owner = THIS_MODULE,
1239         .flags = TEE_DESC_PRIVILEGED,
1240 };
1241
1242 static const struct optee_ops optee_ops = {
1243         .do_call_with_arg = optee_smc_do_call_with_arg,
1244         .to_msg_param = optee_to_msg_param,
1245         .from_msg_param = optee_from_msg_param,
1246 };
1247
1248 static int enable_async_notif(optee_invoke_fn *invoke_fn)
1249 {
1250         struct arm_smccc_res res;
1251
1252         invoke_fn(OPTEE_SMC_ENABLE_ASYNC_NOTIF, 0, 0, 0, 0, 0, 0, 0, &res);
1253
1254         if (res.a0)
1255                 return -EINVAL;
1256         return 0;
1257 }
1258
1259 static bool optee_msg_api_uid_is_optee_api(optee_invoke_fn *invoke_fn)
1260 {
1261         struct arm_smccc_res res;
1262
1263         invoke_fn(OPTEE_SMC_CALLS_UID, 0, 0, 0, 0, 0, 0, 0, &res);
1264
1265         if (res.a0 == OPTEE_MSG_UID_0 && res.a1 == OPTEE_MSG_UID_1 &&
1266             res.a2 == OPTEE_MSG_UID_2 && res.a3 == OPTEE_MSG_UID_3)
1267                 return true;
1268         return false;
1269 }
1270
1271 #ifdef CONFIG_OPTEE_INSECURE_LOAD_IMAGE
1272 static bool optee_msg_api_uid_is_optee_image_load(optee_invoke_fn *invoke_fn)
1273 {
1274         struct arm_smccc_res res;
1275
1276         invoke_fn(OPTEE_SMC_CALLS_UID, 0, 0, 0, 0, 0, 0, 0, &res);
1277
1278         if (res.a0 == OPTEE_MSG_IMAGE_LOAD_UID_0 &&
1279             res.a1 == OPTEE_MSG_IMAGE_LOAD_UID_1 &&
1280             res.a2 == OPTEE_MSG_IMAGE_LOAD_UID_2 &&
1281             res.a3 == OPTEE_MSG_IMAGE_LOAD_UID_3)
1282                 return true;
1283         return false;
1284 }
1285 #endif
1286
1287 static void optee_msg_get_os_revision(optee_invoke_fn *invoke_fn)
1288 {
1289         union {
1290                 struct arm_smccc_res smccc;
1291                 struct optee_smc_call_get_os_revision_result result;
1292         } res = {
1293                 .result = {
1294                         .build_id = 0
1295                 }
1296         };
1297
1298         invoke_fn(OPTEE_SMC_CALL_GET_OS_REVISION, 0, 0, 0, 0, 0, 0, 0,
1299                   &res.smccc);
1300
1301         if (res.result.build_id)
1302                 pr_info("revision %lu.%lu (%08lx)", res.result.major,
1303                         res.result.minor, res.result.build_id);
1304         else
1305                 pr_info("revision %lu.%lu", res.result.major, res.result.minor);
1306 }
1307
1308 static bool optee_msg_api_revision_is_compatible(optee_invoke_fn *invoke_fn)
1309 {
1310         union {
1311                 struct arm_smccc_res smccc;
1312                 struct optee_smc_calls_revision_result result;
1313         } res;
1314
1315         invoke_fn(OPTEE_SMC_CALLS_REVISION, 0, 0, 0, 0, 0, 0, 0, &res.smccc);
1316
1317         if (res.result.major == OPTEE_MSG_REVISION_MAJOR &&
1318             (int)res.result.minor >= OPTEE_MSG_REVISION_MINOR)
1319                 return true;
1320         return false;
1321 }
1322
1323 static bool optee_msg_exchange_capabilities(optee_invoke_fn *invoke_fn,
1324                                             u32 *sec_caps, u32 *max_notif_value,
1325                                             unsigned int *rpc_param_count)
1326 {
1327         union {
1328                 struct arm_smccc_res smccc;
1329                 struct optee_smc_exchange_capabilities_result result;
1330         } res;
1331         u32 a1 = 0;
1332
1333         /*
1334          * TODO This isn't enough to tell if it's UP system (from kernel
1335          * point of view) or not, is_smp() returns the information
1336          * needed, but can't be called directly from here.
1337          */
1338         if (!IS_ENABLED(CONFIG_SMP) || nr_cpu_ids == 1)
1339                 a1 |= OPTEE_SMC_NSEC_CAP_UNIPROCESSOR;
1340
1341         invoke_fn(OPTEE_SMC_EXCHANGE_CAPABILITIES, a1, 0, 0, 0, 0, 0, 0,
1342                   &res.smccc);
1343
1344         if (res.result.status != OPTEE_SMC_RETURN_OK)
1345                 return false;
1346
1347         *sec_caps = res.result.capabilities;
1348         if (*sec_caps & OPTEE_SMC_SEC_CAP_ASYNC_NOTIF)
1349                 *max_notif_value = res.result.max_notif_value;
1350         else
1351                 *max_notif_value = OPTEE_DEFAULT_MAX_NOTIF_VALUE;
1352         if (*sec_caps & OPTEE_SMC_SEC_CAP_RPC_ARG)
1353                 *rpc_param_count = (u8)res.result.data;
1354         else
1355                 *rpc_param_count = 0;
1356
1357         return true;
1358 }
1359
1360 static struct tee_shm_pool *
1361 optee_config_shm_memremap(optee_invoke_fn *invoke_fn, void **memremaped_shm)
1362 {
1363         union {
1364                 struct arm_smccc_res smccc;
1365                 struct optee_smc_get_shm_config_result result;
1366         } res;
1367         unsigned long vaddr;
1368         phys_addr_t paddr;
1369         size_t size;
1370         phys_addr_t begin;
1371         phys_addr_t end;
1372         void *va;
1373         void *rc;
1374
1375         invoke_fn(OPTEE_SMC_GET_SHM_CONFIG, 0, 0, 0, 0, 0, 0, 0, &res.smccc);
1376         if (res.result.status != OPTEE_SMC_RETURN_OK) {
1377                 pr_err("static shm service not available\n");
1378                 return ERR_PTR(-ENOENT);
1379         }
1380
1381         if (res.result.settings != OPTEE_SMC_SHM_CACHED) {
1382                 pr_err("only normal cached shared memory supported\n");
1383                 return ERR_PTR(-EINVAL);
1384         }
1385
1386         begin = roundup(res.result.start, PAGE_SIZE);
1387         end = rounddown(res.result.start + res.result.size, PAGE_SIZE);
1388         paddr = begin;
1389         size = end - begin;
1390
1391         va = memremap(paddr, size, MEMREMAP_WB);
1392         if (!va) {
1393                 pr_err("shared memory ioremap failed\n");
1394                 return ERR_PTR(-EINVAL);
1395         }
1396         vaddr = (unsigned long)va;
1397
1398         rc = tee_shm_pool_alloc_res_mem(vaddr, paddr, size,
1399                                         OPTEE_MIN_STATIC_POOL_ALIGN);
1400         if (IS_ERR(rc))
1401                 memunmap(va);
1402         else
1403                 *memremaped_shm = va;
1404
1405         return rc;
1406 }
1407
1408 /* Simple wrapper functions to be able to use a function pointer */
1409 static void optee_smccc_smc(unsigned long a0, unsigned long a1,
1410                             unsigned long a2, unsigned long a3,
1411                             unsigned long a4, unsigned long a5,
1412                             unsigned long a6, unsigned long a7,
1413                             struct arm_smccc_res *res)
1414 {
1415         arm_smccc_smc(a0, a1, a2, a3, a4, a5, a6, a7, res);
1416 }
1417
1418 static void optee_smccc_hvc(unsigned long a0, unsigned long a1,
1419                             unsigned long a2, unsigned long a3,
1420                             unsigned long a4, unsigned long a5,
1421                             unsigned long a6, unsigned long a7,
1422                             struct arm_smccc_res *res)
1423 {
1424         arm_smccc_hvc(a0, a1, a2, a3, a4, a5, a6, a7, res);
1425 }
1426
1427 static optee_invoke_fn *get_invoke_func(struct device *dev)
1428 {
1429         const char *method;
1430
1431         pr_info("probing for conduit method.\n");
1432
1433         if (device_property_read_string(dev, "method", &method)) {
1434                 pr_warn("missing \"method\" property\n");
1435                 return ERR_PTR(-ENXIO);
1436         }
1437
1438         if (!strcmp("hvc", method))
1439                 return optee_smccc_hvc;
1440         else if (!strcmp("smc", method))
1441                 return optee_smccc_smc;
1442
1443         pr_warn("invalid \"method\" property: %s\n", method);
1444         return ERR_PTR(-EINVAL);
1445 }
1446
1447 /* optee_remove - Device Removal Routine
1448  * @pdev: platform device information struct
1449  *
1450  * optee_remove is called by platform subsystem to alert the driver
1451  * that it should release the device
1452  */
1453 static int optee_smc_remove(struct platform_device *pdev)
1454 {
1455         struct optee *optee = platform_get_drvdata(pdev);
1456
1457         /*
1458          * Ask OP-TEE to free all cached shared memory objects to decrease
1459          * reference counters and also avoid wild pointers in secure world
1460          * into the old shared memory range.
1461          */
1462         if (!optee->rpc_param_count)
1463                 optee_disable_shm_cache(optee);
1464
1465         optee_smc_notif_uninit_irq(optee);
1466
1467         optee_remove_common(optee);
1468
1469         if (optee->smc.memremaped_shm)
1470                 memunmap(optee->smc.memremaped_shm);
1471
1472         kfree(optee);
1473
1474         return 0;
1475 }
1476
1477 /* optee_shutdown - Device Removal Routine
1478  * @pdev: platform device information struct
1479  *
1480  * platform_shutdown is called by the platform subsystem to alert
1481  * the driver that a shutdown, reboot, or kexec is happening and
1482  * device must be disabled.
1483  */
1484 static void optee_shutdown(struct platform_device *pdev)
1485 {
1486         struct optee *optee = platform_get_drvdata(pdev);
1487
1488         if (!optee->rpc_param_count)
1489                 optee_disable_shm_cache(optee);
1490 }
1491
1492 #ifdef CONFIG_OPTEE_INSECURE_LOAD_IMAGE
1493
1494 #define OPTEE_FW_IMAGE "optee/tee.bin"
1495
1496 static optee_invoke_fn *cpuhp_invoke_fn;
1497
1498 static int optee_cpuhp_probe(unsigned int cpu)
1499 {
1500         /*
1501          * Invoking a call on a CPU will cause OP-TEE to perform the required
1502          * setup for that CPU. Just invoke the call to get the UID since that
1503          * has no side effects.
1504          */
1505         if (optee_msg_api_uid_is_optee_api(cpuhp_invoke_fn))
1506                 return 0;
1507         else
1508                 return -EINVAL;
1509 }
1510
1511 static int optee_load_fw(struct platform_device *pdev,
1512                          optee_invoke_fn *invoke_fn)
1513 {
1514         const struct firmware *fw = NULL;
1515         struct arm_smccc_res res;
1516         phys_addr_t data_pa;
1517         u8 *data_buf = NULL;
1518         u64 data_size;
1519         u32 data_pa_high, data_pa_low;
1520         u32 data_size_high, data_size_low;
1521         int rc;
1522         int hp_state;
1523
1524         if (!optee_msg_api_uid_is_optee_image_load(invoke_fn))
1525                 return 0;
1526
1527         rc = request_firmware(&fw, OPTEE_FW_IMAGE, &pdev->dev);
1528         if (rc) {
1529                 /*
1530                  * The firmware in the rootfs will not be accessible until we
1531                  * are in the SYSTEM_RUNNING state, so return EPROBE_DEFER until
1532                  * that point.
1533                  */
1534                 if (system_state < SYSTEM_RUNNING)
1535                         return -EPROBE_DEFER;
1536                 goto fw_err;
1537         }
1538
1539         data_size = fw->size;
1540         /*
1541          * This uses the GFP_DMA flag to ensure we are allocated memory in the
1542          * 32-bit space since TF-A cannot map memory beyond the 32-bit boundary.
1543          */
1544         data_buf = kmalloc(fw->size, GFP_KERNEL | GFP_DMA);
1545         if (!data_buf) {
1546                 rc = -ENOMEM;
1547                 goto fw_err;
1548         }
1549         memcpy(data_buf, fw->data, fw->size);
1550         data_pa = virt_to_phys(data_buf);
1551         reg_pair_from_64(&data_pa_high, &data_pa_low, data_pa);
1552         reg_pair_from_64(&data_size_high, &data_size_low, data_size);
1553         goto fw_load;
1554
1555 fw_err:
1556         pr_warn("image loading failed\n");
1557         data_pa_high = 0;
1558         data_pa_low = 0;
1559         data_size_high = 0;
1560         data_size_low = 0;
1561
1562 fw_load:
1563         /*
1564          * Always invoke the SMC, even if loading the image fails, to indicate
1565          * to EL3 that we have passed the point where it should allow invoking
1566          * this SMC.
1567          */
1568         pr_warn("OP-TEE image loaded from kernel, this can be insecure");
1569         invoke_fn(OPTEE_SMC_CALL_LOAD_IMAGE, data_size_high, data_size_low,
1570                   data_pa_high, data_pa_low, 0, 0, 0, &res);
1571         if (!rc)
1572                 rc = res.a0;
1573         if (fw)
1574                 release_firmware(fw);
1575         kfree(data_buf);
1576
1577         if (!rc) {
1578                 /*
1579                  * We need to initialize OP-TEE on all other running cores as
1580                  * well. Any cores that aren't running yet will get initialized
1581                  * when they are brought up by the power management functions in
1582                  * TF-A which are registered by the OP-TEE SPD. Due to that we
1583                  * can un-register the callback right after registering it.
1584                  */
1585                 cpuhp_invoke_fn = invoke_fn;
1586                 hp_state = cpuhp_setup_state(CPUHP_AP_ONLINE_DYN, "optee:probe",
1587                                              optee_cpuhp_probe, NULL);
1588                 if (hp_state < 0) {
1589                         pr_warn("Failed with CPU hotplug setup for OP-TEE");
1590                         return -EINVAL;
1591                 }
1592                 cpuhp_remove_state(hp_state);
1593                 cpuhp_invoke_fn = NULL;
1594         }
1595
1596         return rc;
1597 }
1598 #else
1599 static inline int optee_load_fw(struct platform_device *pdev,
1600                                 optee_invoke_fn *invoke_fn)
1601 {
1602         return 0;
1603 }
1604 #endif
1605
1606 static int optee_probe(struct platform_device *pdev)
1607 {
1608         optee_invoke_fn *invoke_fn;
1609         struct tee_shm_pool *pool = ERR_PTR(-EINVAL);
1610         struct optee *optee = NULL;
1611         void *memremaped_shm = NULL;
1612         unsigned int rpc_param_count;
1613         struct tee_device *teedev;
1614         struct tee_context *ctx;
1615         u32 max_notif_value;
1616         u32 arg_cache_flags;
1617         u32 sec_caps;
1618         int rc;
1619
1620         invoke_fn = get_invoke_func(&pdev->dev);
1621         if (IS_ERR(invoke_fn))
1622                 return PTR_ERR(invoke_fn);
1623
1624         rc = optee_load_fw(pdev, invoke_fn);
1625         if (rc)
1626                 return rc;
1627
1628         if (!optee_msg_api_uid_is_optee_api(invoke_fn)) {
1629                 pr_warn("api uid mismatch\n");
1630                 return -EINVAL;
1631         }
1632
1633         optee_msg_get_os_revision(invoke_fn);
1634
1635         if (!optee_msg_api_revision_is_compatible(invoke_fn)) {
1636                 pr_warn("api revision mismatch\n");
1637                 return -EINVAL;
1638         }
1639
1640         if (!optee_msg_exchange_capabilities(invoke_fn, &sec_caps,
1641                                              &max_notif_value,
1642                                              &rpc_param_count)) {
1643                 pr_warn("capabilities mismatch\n");
1644                 return -EINVAL;
1645         }
1646
1647         /*
1648          * Try to use dynamic shared memory if possible
1649          */
1650         if (sec_caps & OPTEE_SMC_SEC_CAP_DYNAMIC_SHM) {
1651                 /*
1652                  * If we have OPTEE_SMC_SEC_CAP_RPC_ARG we can ask
1653                  * optee_get_msg_arg() to pre-register (by having
1654                  * OPTEE_SHM_ARG_ALLOC_PRIV cleared) the page used to pass
1655                  * an argument struct.
1656                  *
1657                  * With the page is pre-registered we can use a non-zero
1658                  * offset for argument struct, this is indicated with
1659                  * OPTEE_SHM_ARG_SHARED.
1660                  *
1661                  * This means that optee_smc_do_call_with_arg() will use
1662                  * OPTEE_SMC_CALL_WITH_REGD_ARG for pre-registered pages.
1663                  */
1664                 if (sec_caps & OPTEE_SMC_SEC_CAP_RPC_ARG)
1665                         arg_cache_flags = OPTEE_SHM_ARG_SHARED;
1666                 else
1667                         arg_cache_flags = OPTEE_SHM_ARG_ALLOC_PRIV;
1668
1669                 pool = optee_shm_pool_alloc_pages();
1670         }
1671
1672         /*
1673          * If dynamic shared memory is not available or failed - try static one
1674          */
1675         if (IS_ERR(pool) && (sec_caps & OPTEE_SMC_SEC_CAP_HAVE_RESERVED_SHM)) {
1676                 /*
1677                  * The static memory pool can use non-zero page offsets so
1678                  * let optee_get_msg_arg() know that with OPTEE_SHM_ARG_SHARED.
1679                  *
1680                  * optee_get_msg_arg() should not pre-register the
1681                  * allocated page used to pass an argument struct, this is
1682                  * indicated with OPTEE_SHM_ARG_ALLOC_PRIV.
1683                  *
1684                  * This means that optee_smc_do_call_with_arg() will use
1685                  * OPTEE_SMC_CALL_WITH_ARG if rpc_param_count is 0, else
1686                  * OPTEE_SMC_CALL_WITH_RPC_ARG.
1687                  */
1688                 arg_cache_flags = OPTEE_SHM_ARG_SHARED |
1689                                   OPTEE_SHM_ARG_ALLOC_PRIV;
1690                 pool = optee_config_shm_memremap(invoke_fn, &memremaped_shm);
1691         }
1692
1693         if (IS_ERR(pool))
1694                 return PTR_ERR(pool);
1695
1696         optee = kzalloc(sizeof(*optee), GFP_KERNEL);
1697         if (!optee) {
1698                 rc = -ENOMEM;
1699                 goto err_free_pool;
1700         }
1701
1702         optee->ops = &optee_ops;
1703         optee->smc.invoke_fn = invoke_fn;
1704         optee->smc.sec_caps = sec_caps;
1705         optee->rpc_param_count = rpc_param_count;
1706
1707         teedev = tee_device_alloc(&optee_clnt_desc, NULL, pool, optee);
1708         if (IS_ERR(teedev)) {
1709                 rc = PTR_ERR(teedev);
1710                 goto err_free_optee;
1711         }
1712         optee->teedev = teedev;
1713
1714         teedev = tee_device_alloc(&optee_supp_desc, NULL, pool, optee);
1715         if (IS_ERR(teedev)) {
1716                 rc = PTR_ERR(teedev);
1717                 goto err_unreg_teedev;
1718         }
1719         optee->supp_teedev = teedev;
1720
1721         rc = tee_device_register(optee->teedev);
1722         if (rc)
1723                 goto err_unreg_supp_teedev;
1724
1725         rc = tee_device_register(optee->supp_teedev);
1726         if (rc)
1727                 goto err_unreg_supp_teedev;
1728
1729         mutex_init(&optee->call_queue.mutex);
1730         INIT_LIST_HEAD(&optee->call_queue.waiters);
1731         optee_supp_init(&optee->supp);
1732         optee->smc.memremaped_shm = memremaped_shm;
1733         optee->pool = pool;
1734         optee_shm_arg_cache_init(optee, arg_cache_flags);
1735
1736         platform_set_drvdata(pdev, optee);
1737         ctx = teedev_open(optee->teedev);
1738         if (IS_ERR(ctx)) {
1739                 rc = PTR_ERR(ctx);
1740                 goto err_supp_uninit;
1741         }
1742         optee->ctx = ctx;
1743         rc = optee_notif_init(optee, max_notif_value);
1744         if (rc)
1745                 goto err_close_ctx;
1746
1747         if (sec_caps & OPTEE_SMC_SEC_CAP_ASYNC_NOTIF) {
1748                 unsigned int irq;
1749
1750                 rc = platform_get_irq(pdev, 0);
1751                 if (rc < 0) {
1752                         pr_err("platform_get_irq: ret %d\n", rc);
1753                         goto err_notif_uninit;
1754                 }
1755                 irq = rc;
1756
1757                 rc = optee_smc_notif_init_irq(optee, irq);
1758                 if (rc) {
1759                         irq_dispose_mapping(irq);
1760                         goto err_notif_uninit;
1761                 }
1762                 enable_async_notif(optee->smc.invoke_fn);
1763                 pr_info("Asynchronous notifications enabled\n");
1764         }
1765
1766         /*
1767          * Ensure that there are no pre-existing shm objects before enabling
1768          * the shm cache so that there's no chance of receiving an invalid
1769          * address during shutdown. This could occur, for example, if we're
1770          * kexec booting from an older kernel that did not properly cleanup the
1771          * shm cache.
1772          */
1773         optee_disable_unmapped_shm_cache(optee);
1774
1775         /*
1776          * Only enable the shm cache in case we're not able to pass the RPC
1777          * arg struct right after the normal arg struct.
1778          */
1779         if (!optee->rpc_param_count)
1780                 optee_enable_shm_cache(optee);
1781
1782         if (optee->smc.sec_caps & OPTEE_SMC_SEC_CAP_DYNAMIC_SHM)
1783                 pr_info("dynamic shared memory is enabled\n");
1784
1785         rc = optee_enumerate_devices(PTA_CMD_GET_DEVICES);
1786         if (rc)
1787                 goto err_disable_shm_cache;
1788
1789         pr_info("initialized driver\n");
1790         return 0;
1791
1792 err_disable_shm_cache:
1793         if (!optee->rpc_param_count)
1794                 optee_disable_shm_cache(optee);
1795         optee_smc_notif_uninit_irq(optee);
1796         optee_unregister_devices();
1797 err_notif_uninit:
1798         optee_notif_uninit(optee);
1799 err_close_ctx:
1800         teedev_close_context(ctx);
1801 err_supp_uninit:
1802         optee_shm_arg_cache_uninit(optee);
1803         optee_supp_uninit(&optee->supp);
1804         mutex_destroy(&optee->call_queue.mutex);
1805 err_unreg_supp_teedev:
1806         tee_device_unregister(optee->supp_teedev);
1807 err_unreg_teedev:
1808         tee_device_unregister(optee->teedev);
1809 err_free_optee:
1810         kfree(optee);
1811 err_free_pool:
1812         tee_shm_pool_free(pool);
1813         if (memremaped_shm)
1814                 memunmap(memremaped_shm);
1815         return rc;
1816 }
1817
1818 static const struct of_device_id optee_dt_match[] = {
1819         { .compatible = "linaro,optee-tz" },
1820         {},
1821 };
1822 MODULE_DEVICE_TABLE(of, optee_dt_match);
1823
1824 static struct platform_driver optee_driver = {
1825         .probe  = optee_probe,
1826         .remove = optee_smc_remove,
1827         .shutdown = optee_shutdown,
1828         .driver = {
1829                 .name = "optee",
1830                 .of_match_table = optee_dt_match,
1831         },
1832 };
1833
1834 int optee_smc_abi_register(void)
1835 {
1836         return platform_driver_register(&optee_driver);
1837 }
1838
1839 void optee_smc_abi_unregister(void)
1840 {
1841         platform_driver_unregister(&optee_driver);
1842 }