drivers/firmware/arm_sdei.c

   1 // SPDX-License-Identifier: GPL-2.0
   2 // Copyright (C) 2017 Arm Ltd.
   3 #define pr_fmt(fmt) "sdei: " fmt
   4
   5 #include <acpi/ghes.h>
   6 #include <linux/acpi.h>
   7 #include <linux/arm_sdei.h>
   8 #include <linux/arm-smccc.h>
   9 #include <linux/atomic.h>
  10 #include <linux/bitops.h>
  11 #include <linux/compiler.h>
  12 #include <linux/cpuhotplug.h>
  13 #include <linux/cpu.h>
  14 #include <linux/cpu_pm.h>
  15 #include <linux/errno.h>
  16 #include <linux/hardirq.h>
  17 #include <linux/kernel.h>
  18 #include <linux/kprobes.h>
  19 #include <linux/kvm_host.h>
  20 #include <linux/list.h>
  21 #include <linux/mutex.h>
  22 #include <linux/notifier.h>
  23 #include <linux/of.h>
  24 #include <linux/of_platform.h>
  25 #include <linux/percpu.h>
  26 #include <linux/platform_device.h>
  27 #include <linux/pm.h>
  28 #include <linux/ptrace.h>
  29 #include <linux/preempt.h>
  30 #include <linux/reboot.h>
  31 #include <linux/slab.h>
  32 #include <linux/smp.h>
  33 #include <linux/spinlock.h>
  34 #include <linux/uaccess.h>
  35
  36 /*
  37  * The call to use to reach the firmware.
  38  */
  39 static asmlinkage void (*sdei_firmware_call)(unsigned long function_id,
  40                       unsigned long arg0, unsigned long arg1,
  41                       unsigned long arg2, unsigned long arg3,
  42                       unsigned long arg4, struct arm_smccc_res *res);
  43
  44 /* entry point from firmware to arch asm code */
  45 static unsigned long sdei_entry_point;
  46
  47 struct sdei_event {
  48         /* These three are protected by the sdei_list_lock */
  49         struct list_head        list;
  50         bool                    reregister;
  51         bool                    reenable;
  52
  53         u32                     event_num;
  54         u8                      type;
  55         u8                      priority;
  56
  57         /* This pointer is handed to firmware as the event argument. */
  58         union {
  59                 /* Shared events */
  60                 struct sdei_registered_event *registered;
  61
  62                 /* CPU private events */
  63                 struct sdei_registered_event __percpu *private_registered;
  64         };
  65 };
  66
  67 /* Take the mutex for any API call or modification. Take the mutex first. */
  68 static DEFINE_MUTEX(sdei_events_lock);
  69
  70 /* and then hold this when modifying the list */
  71 static DEFINE_SPINLOCK(sdei_list_lock);
  72 static LIST_HEAD(sdei_list);
  73
  74 /* Private events are registered/enabled via IPI passing one of these */
  75 struct sdei_crosscall_args {
  76         struct sdei_event *event;
  77         atomic_t errors;
  78         int first_error;
  79 };
  80
  81 #define CROSSCALL_INIT(arg, event)      (arg.event = event, \
  82                                          arg.first_error = 0, \
  83                                          atomic_set(&arg.errors, 0))
  84
  85 static inline int sdei_do_cross_call(void *fn, struct sdei_event * event)
  86 {
  87         struct sdei_crosscall_args arg;
  88
  89         CROSSCALL_INIT(arg, event);
  90         on_each_cpu(fn, &arg, true);
  91
  92         return arg.first_error;
  93 }
  94
  95 static inline void
  96 sdei_cross_call_return(struct sdei_crosscall_args *arg, int err)
  97 {
  98         if (err && (atomic_inc_return(&arg->errors) == 1))
  99                 arg->first_error = err;
 100 }
 101
 102 static int sdei_to_linux_errno(unsigned long sdei_err)
 103 {
 104         switch (sdei_err) {
 105         case SDEI_NOT_SUPPORTED:
 106                 return -EOPNOTSUPP;
 107         case SDEI_INVALID_PARAMETERS:
 108                 return -EINVAL;
 109         case SDEI_DENIED:
 110                 return -EPERM;
 111         case SDEI_PENDING:
 112                 return -EINPROGRESS;
 113         case SDEI_OUT_OF_RESOURCE:
 114                 return -ENOMEM;
 115         }
 116
 117         return 0;
 118 }
 119
 120 static int invoke_sdei_fn(unsigned long function_id, unsigned long arg0,
 121                           unsigned long arg1, unsigned long arg2,
 122                           unsigned long arg3, unsigned long arg4,
 123                           u64 *result)
 124 {
 125         int err;
 126         struct arm_smccc_res res;
 127
 128         if (sdei_firmware_call) {
 129                 sdei_firmware_call(function_id, arg0, arg1, arg2, arg3, arg4,
 130                                    &res);
 131                 err = sdei_to_linux_errno(res.a0);
 132         } else {
 133                 /*
 134                  * !sdei_firmware_call means we failed to probe or called
 135                  * sdei_mark_interface_broken(). -EIO is not an error returned
 136                  * by sdei_to_linux_errno() and is used to suppress messages
 137                  * from this driver.
 138                  */
 139                 err = -EIO;
 140                 res.a0 = SDEI_NOT_SUPPORTED;
 141         }
 142
 143         if (result)
 144                 *result = res.a0;
 145
 146         return err;
 147 }
 148 NOKPROBE_SYMBOL(invoke_sdei_fn);
 149
 150 static struct sdei_event *sdei_event_find(u32 event_num)
 151 {
 152         struct sdei_event *e, *found = NULL;
 153
 154         lockdep_assert_held(&sdei_events_lock);
 155
 156         spin_lock(&sdei_list_lock);
 157         list_for_each_entry(e, &sdei_list, list) {
 158                 if (e->event_num == event_num) {
 159                         found = e;
 160                         break;
 161                 }
 162         }
 163         spin_unlock(&sdei_list_lock);
 164
 165         return found;
 166 }
 167
 168 int sdei_api_event_context(u32 query, u64 *result)
 169 {
 170         return invoke_sdei_fn(SDEI_1_0_FN_SDEI_EVENT_CONTEXT, query, 0, 0, 0, 0,
 171                               result);
 172 }
 173 NOKPROBE_SYMBOL(sdei_api_event_context);
 174
 175 static int sdei_api_event_get_info(u32 event, u32 info, u64 *result)
 176 {
 177         return invoke_sdei_fn(SDEI_1_0_FN_SDEI_EVENT_GET_INFO, event, info, 0,
 178                               0, 0, result);
 179 }
 180
 181 static struct sdei_event *sdei_event_create(u32 event_num,
 182                                             sdei_event_callback *cb,
 183                                             void *cb_arg)
 184 {
 185         int err;
 186         u64 result;
 187         struct sdei_event *event;
 188         struct sdei_registered_event *reg;
 189
 190         lockdep_assert_held(&sdei_events_lock);
 191
 192         event = kzalloc(sizeof(*event), GFP_KERNEL);
 193         if (!event) {
 194                 err = -ENOMEM;
 195                 goto fail;
 196         }
 197
 198         INIT_LIST_HEAD(&event->list);
 199         event->event_num = event_num;
 200
 201         err = sdei_api_event_get_info(event_num, SDEI_EVENT_INFO_EV_PRIORITY,
 202                                       &result);
 203         if (err)
 204                 goto fail;
 205         event->priority = result;
 206
 207         err = sdei_api_event_get_info(event_num, SDEI_EVENT_INFO_EV_TYPE,
 208                                       &result);
 209         if (err)
 210                 goto fail;
 211         event->type = result;
 212
 213         if (event->type == SDEI_EVENT_TYPE_SHARED) {
 214                 reg = kzalloc(sizeof(*reg), GFP_KERNEL);
 215                 if (!reg) {
 216                         err = -ENOMEM;
 217                         goto fail;
 218                 }
 219
 220                 reg->event_num = event->event_num;
 221                 reg->priority = event->priority;
 222
 223                 reg->callback = cb;
 224                 reg->callback_arg = cb_arg;
 225                 event->registered = reg;
 226         } else {
 227                 int cpu;
 228                 struct sdei_registered_event __percpu *regs;
 229
 230                 regs = alloc_percpu(struct sdei_registered_event);
 231                 if (!regs) {
 232                         err = -ENOMEM;
 233                         goto fail;
 234                 }
 235
 236                 for_each_possible_cpu(cpu) {
 237                         reg = per_cpu_ptr(regs, cpu);
 238
 239                         reg->event_num = event->event_num;
 240                         reg->priority = event->priority;
 241                         reg->callback = cb;
 242                         reg->callback_arg = cb_arg;
 243                 }
 244
 245                 event->private_registered = regs;
 246         }
 247
 248         spin_lock(&sdei_list_lock);
 249         list_add(&event->list, &sdei_list);
 250         spin_unlock(&sdei_list_lock);
 251
 252         return event;
 253
 254 fail:
 255         kfree(event);
 256         return ERR_PTR(err);
 257 }
 258
 259 static void sdei_event_destroy_llocked(struct sdei_event *event)
 260 {
 261         lockdep_assert_held(&sdei_events_lock);
 262         lockdep_assert_held(&sdei_list_lock);
 263
 264         list_del(&event->list);
 265
 266         if (event->type == SDEI_EVENT_TYPE_SHARED)
 267                 kfree(event->registered);
 268         else
 269                 free_percpu(event->private_registered);
 270
 271         kfree(event);
 272 }
 273
 274 static void sdei_event_destroy(struct sdei_event *event)
 275 {
 276         spin_lock(&sdei_list_lock);
 277         sdei_event_destroy_llocked(event);
 278         spin_unlock(&sdei_list_lock);
 279 }
 280
 281 static int sdei_api_get_version(u64 *version)
 282 {
 283         return invoke_sdei_fn(SDEI_1_0_FN_SDEI_VERSION, 0, 0, 0, 0, 0, version);
 284 }
 285
 286 int sdei_mask_local_cpu(void)
 287 {
 288         int err;
 289
 290         WARN_ON_ONCE(preemptible());
 291
 292         err = invoke_sdei_fn(SDEI_1_0_FN_SDEI_PE_MASK, 0, 0, 0, 0, 0, NULL);
 293         if (err && err != -EIO) {
 294                 pr_warn_once("failed to mask CPU[%u]: %d\n",
 295                               smp_processor_id(), err);
 296                 return err;
 297         }
 298
 299         return 0;
 300 }
 301
 302 static void _ipi_mask_cpu(void *ignored)
 303 {
 304         sdei_mask_local_cpu();
 305 }
 306
 307 int sdei_unmask_local_cpu(void)
 308 {
 309         int err;
 310
 311         WARN_ON_ONCE(preemptible());
 312
 313         err = invoke_sdei_fn(SDEI_1_0_FN_SDEI_PE_UNMASK, 0, 0, 0, 0, 0, NULL);
 314         if (err && err != -EIO) {
 315                 pr_warn_once("failed to unmask CPU[%u]: %d\n",
 316                              smp_processor_id(), err);
 317                 return err;
 318         }
 319
 320         return 0;
 321 }
 322
 323 static void _ipi_unmask_cpu(void *ignored)
 324 {
 325         sdei_unmask_local_cpu();
 326 }
 327
 328 static void _ipi_private_reset(void *ignored)
 329 {
 330         int err;
 331
 332         err = invoke_sdei_fn(SDEI_1_0_FN_SDEI_PRIVATE_RESET, 0, 0, 0, 0, 0,
 333                              NULL);
 334         if (err && err != -EIO)
 335                 pr_warn_once("failed to reset CPU[%u]: %d\n",
 336                              smp_processor_id(), err);
 337 }
 338
 339 static int sdei_api_shared_reset(void)
 340 {
 341         return invoke_sdei_fn(SDEI_1_0_FN_SDEI_SHARED_RESET, 0, 0, 0, 0, 0,
 342                               NULL);
 343 }
 344
 345 static void sdei_mark_interface_broken(void)
 346 {
 347         pr_err("disabling SDEI firmware interface\n");
 348         on_each_cpu(&_ipi_mask_cpu, NULL, true);
 349         sdei_firmware_call = NULL;
 350 }
 351
 352 static int sdei_platform_reset(void)
 353 {
 354         int err;
 355
 356         on_each_cpu(&_ipi_private_reset, NULL, true);
 357         err = sdei_api_shared_reset();
 358         if (err) {
 359                 pr_err("Failed to reset platform: %d\n", err);
 360                 sdei_mark_interface_broken();
 361         }
 362
 363         return err;
 364 }
 365
 366 static int sdei_api_event_enable(u32 event_num)
 367 {
 368         return invoke_sdei_fn(SDEI_1_0_FN_SDEI_EVENT_ENABLE, event_num, 0, 0, 0,
 369                               0, NULL);
 370 }
 371
 372 /* Called directly by the hotplug callbacks */
 373 static void _local_event_enable(void *data)
 374 {
 375         int err;
 376         struct sdei_crosscall_args *arg = data;
 377
 378         WARN_ON_ONCE(preemptible());
 379
 380         err = sdei_api_event_enable(arg->event->event_num);
 381
 382         sdei_cross_call_return(arg, err);
 383 }
 384
 385 int sdei_event_enable(u32 event_num)
 386 {
 387         int err = -EINVAL;
 388         struct sdei_event *event;
 389
 390         mutex_lock(&sdei_events_lock);
 391         event = sdei_event_find(event_num);
 392         if (!event) {
 393                 mutex_unlock(&sdei_events_lock);
 394                 return -ENOENT;
 395         }
 396
 397
 398         cpus_read_lock();
 399         if (event->type == SDEI_EVENT_TYPE_SHARED)
 400                 err = sdei_api_event_enable(event->event_num);
 401         else
 402                 err = sdei_do_cross_call(_local_event_enable, event);
 403
 404         if (!err) {
 405                 spin_lock(&sdei_list_lock);
 406                 event->reenable = true;
 407                 spin_unlock(&sdei_list_lock);
 408         }
 409         cpus_read_unlock();
 410         mutex_unlock(&sdei_events_lock);
 411
 412         return err;
 413 }
 414
 415 static int sdei_api_event_disable(u32 event_num)
 416 {
 417         return invoke_sdei_fn(SDEI_1_0_FN_SDEI_EVENT_DISABLE, event_num, 0, 0,
 418                               0, 0, NULL);
 419 }
 420
 421 static void _ipi_event_disable(void *data)
 422 {
 423         int err;
 424         struct sdei_crosscall_args *arg = data;
 425
 426         err = sdei_api_event_disable(arg->event->event_num);
 427
 428         sdei_cross_call_return(arg, err);
 429 }
 430
 431 int sdei_event_disable(u32 event_num)
 432 {
 433         int err = -EINVAL;
 434         struct sdei_event *event;
 435
 436         mutex_lock(&sdei_events_lock);
 437         event = sdei_event_find(event_num);
 438         if (!event) {
 439                 mutex_unlock(&sdei_events_lock);
 440                 return -ENOENT;
 441         }
 442
 443         spin_lock(&sdei_list_lock);
 444         event->reenable = false;
 445         spin_unlock(&sdei_list_lock);
 446
 447         if (event->type == SDEI_EVENT_TYPE_SHARED)
 448                 err = sdei_api_event_disable(event->event_num);
 449         else
 450                 err = sdei_do_cross_call(_ipi_event_disable, event);
 451         mutex_unlock(&sdei_events_lock);
 452
 453         return err;
 454 }
 455
 456 static int sdei_api_event_unregister(u32 event_num)
 457 {
 458         return invoke_sdei_fn(SDEI_1_0_FN_SDEI_EVENT_UNREGISTER, event_num, 0,
 459                               0, 0, 0, NULL);
 460 }
 461
 462 /* Called directly by the hotplug callbacks */
 463 static void _local_event_unregister(void *data)
 464 {
 465         int err;
 466         struct sdei_crosscall_args *arg = data;
 467
 468         WARN_ON_ONCE(preemptible());
 469
 470         err = sdei_api_event_unregister(arg->event->event_num);
 471
 472         sdei_cross_call_return(arg, err);
 473 }
 474
 475 static int _sdei_event_unregister(struct sdei_event *event)
 476 {
 477         lockdep_assert_held(&sdei_events_lock);
 478
 479         if (event->type == SDEI_EVENT_TYPE_SHARED)
 480                 return sdei_api_event_unregister(event->event_num);
 481
 482         return sdei_do_cross_call(_local_event_unregister, event);
 483 }
 484
 485 int sdei_event_unregister(u32 event_num)
 486 {
 487         int err;
 488         struct sdei_event *event;
 489
 490         WARN_ON(in_nmi());
 491
 492         mutex_lock(&sdei_events_lock);
 493         event = sdei_event_find(event_num);
 494         do {
 495                 if (!event) {
 496                         pr_warn("Event %u not registered\n", event_num);
 497                         err = -ENOENT;
 498                         break;
 499                 }
 500
 501                 spin_lock(&sdei_list_lock);
 502                 event->reregister = false;
 503                 event->reenable = false;
 504                 spin_unlock(&sdei_list_lock);
 505
 506                 err = _sdei_event_unregister(event);
 507                 if (err)
 508                         break;
 509
 510                 sdei_event_destroy(event);
 511         } while (0);
 512         mutex_unlock(&sdei_events_lock);
 513
 514         return err;
 515 }
 516
 517 /*
 518  * unregister events, but don't destroy them as they are re-registered by
 519  * sdei_reregister_shared().
 520  */
 521 static int sdei_unregister_shared(void)
 522 {
 523         int err = 0;
 524         struct sdei_event *event;
 525
 526         mutex_lock(&sdei_events_lock);
 527         spin_lock(&sdei_list_lock);
 528         list_for_each_entry(event, &sdei_list, list) {
 529                 if (event->type != SDEI_EVENT_TYPE_SHARED)
 530                         continue;
 531
 532                 err = _sdei_event_unregister(event);
 533                 if (err)
 534                         break;
 535         }
 536         spin_unlock(&sdei_list_lock);
 537         mutex_unlock(&sdei_events_lock);
 538
 539         return err;
 540 }
 541
 542 static int sdei_api_event_register(u32 event_num, unsigned long entry_point,
 543                                    void *arg, u64 flags, u64 affinity)
 544 {
 545         return invoke_sdei_fn(SDEI_1_0_FN_SDEI_EVENT_REGISTER, event_num,
 546                               (unsigned long)entry_point, (unsigned long)arg,
 547                               flags, affinity, NULL);
 548 }
 549
 550 /* Called directly by the hotplug callbacks */
 551 static void _local_event_register(void *data)
 552 {
 553         int err;
 554         struct sdei_registered_event *reg;
 555         struct sdei_crosscall_args *arg = data;
 556
 557         WARN_ON(preemptible());
 558
 559         reg = per_cpu_ptr(arg->event->private_registered, smp_processor_id());
 560         err = sdei_api_event_register(arg->event->event_num, sdei_entry_point,
 561                                       reg, 0, 0);
 562
 563         sdei_cross_call_return(arg, err);
 564 }
 565
 566 static int _sdei_event_register(struct sdei_event *event)
 567 {
 568         int err;
 569
 570         lockdep_assert_held(&sdei_events_lock);
 571
 572         if (event->type == SDEI_EVENT_TYPE_SHARED)
 573                 return sdei_api_event_register(event->event_num,
 574                                                sdei_entry_point,
 575                                                event->registered,
 576                                                SDEI_EVENT_REGISTER_RM_ANY, 0);
 577
 578         err = sdei_do_cross_call(_local_event_register, event);
 579         if (err)
 580                 sdei_do_cross_call(_local_event_unregister, event);
 581
 582         return err;
 583 }
 584
 585 int sdei_event_register(u32 event_num, sdei_event_callback *cb, void *arg)
 586 {
 587         int err;
 588         struct sdei_event *event;
 589
 590         WARN_ON(in_nmi());
 591
 592         mutex_lock(&sdei_events_lock);
 593         do {
 594                 if (sdei_event_find(event_num)) {
 595                         pr_warn("Event %u already registered\n", event_num);
 596                         err = -EBUSY;
 597                         break;
 598                 }
 599
 600                 event = sdei_event_create(event_num, cb, arg);
 601                 if (IS_ERR(event)) {
 602                         err = PTR_ERR(event);
 603                         pr_warn("Failed to create event %u: %d\n", event_num,
 604                                 err);
 605                         break;
 606                 }
 607
 608                 cpus_read_lock();
 609                 err = _sdei_event_register(event);
 610                 if (err) {
 611                         sdei_event_destroy(event);
 612                         pr_warn("Failed to register event %u: %d\n", event_num,
 613                                 err);
 614                 } else {
 615                         spin_lock(&sdei_list_lock);
 616                         event->reregister = true;
 617                         spin_unlock(&sdei_list_lock);
 618                 }
 619                 cpus_read_unlock();
 620         } while (0);
 621         mutex_unlock(&sdei_events_lock);
 622
 623         return err;
 624 }
 625
 626 static int sdei_reregister_event_llocked(struct sdei_event *event)
 627 {
 628         int err;
 629
 630         lockdep_assert_held(&sdei_events_lock);
 631         lockdep_assert_held(&sdei_list_lock);
 632
 633         err = _sdei_event_register(event);
 634         if (err) {
 635                 pr_err("Failed to re-register event %u\n", event->event_num);
 636                 sdei_event_destroy_llocked(event);
 637                 return err;
 638         }
 639
 640         if (event->reenable) {
 641                 if (event->type == SDEI_EVENT_TYPE_SHARED)
 642                         err = sdei_api_event_enable(event->event_num);
 643                 else
 644                         err = sdei_do_cross_call(_local_event_enable, event);
 645         }
 646
 647         if (err)
 648                 pr_err("Failed to re-enable event %u\n", event->event_num);
 649
 650         return err;
 651 }
 652
 653 static int sdei_reregister_shared(void)
 654 {
 655         int err = 0;
 656         struct sdei_event *event;
 657
 658         mutex_lock(&sdei_events_lock);
 659         spin_lock(&sdei_list_lock);
 660         list_for_each_entry(event, &sdei_list, list) {
 661                 if (event->type != SDEI_EVENT_TYPE_SHARED)
 662                         continue;
 663
 664                 if (event->reregister) {
 665                         err = sdei_reregister_event_llocked(event);
 666                         if (err)
 667                                 break;
 668                 }
 669         }
 670         spin_unlock(&sdei_list_lock);
 671         mutex_unlock(&sdei_events_lock);
 672
 673         return err;
 674 }
 675
 676 static int sdei_cpuhp_down(unsigned int cpu)
 677 {
 678         struct sdei_event *event;
 679         struct sdei_crosscall_args arg;
 680
 681         /* un-register private events */
 682         spin_lock(&sdei_list_lock);
 683         list_for_each_entry(event, &sdei_list, list) {
 684                 if (event->type == SDEI_EVENT_TYPE_SHARED)
 685                         continue;
 686
 687                 CROSSCALL_INIT(arg, event);
 688                 /* call the cross-call function locally... */
 689                 _local_event_unregister(&arg);
 690                 if (arg.first_error)
 691                         pr_err("Failed to unregister event %u: %d\n",
 692                                event->event_num, arg.first_error);
 693         }
 694         spin_unlock(&sdei_list_lock);
 695
 696         return sdei_mask_local_cpu();
 697 }
 698
 699 static int sdei_cpuhp_up(unsigned int cpu)
 700 {
 701         struct sdei_event *event;
 702         struct sdei_crosscall_args arg;
 703
 704         /* re-register/enable private events */
 705         spin_lock(&sdei_list_lock);
 706         list_for_each_entry(event, &sdei_list, list) {
 707                 if (event->type == SDEI_EVENT_TYPE_SHARED)
 708                         continue;
 709
 710                 if (event->reregister) {
 711                         CROSSCALL_INIT(arg, event);
 712                         /* call the cross-call function locally... */
 713                         _local_event_register(&arg);
 714                         if (arg.first_error)
 715                                 pr_err("Failed to re-register event %u: %d\n",
 716                                        event->event_num, arg.first_error);
 717                 }
 718
 719                 if (event->reenable) {
 720                         CROSSCALL_INIT(arg, event);
 721                         _local_event_enable(&arg);
 722                         if (arg.first_error)
 723                                 pr_err("Failed to re-enable event %u: %d\n",
 724                                        event->event_num, arg.first_error);
 725                 }
 726         }
 727         spin_unlock(&sdei_list_lock);
 728
 729         return sdei_unmask_local_cpu();
 730 }
 731
 732 /* When entering idle, mask/unmask events for this cpu */
 733 static int sdei_pm_notifier(struct notifier_block *nb, unsigned long action,
 734                             void *data)
 735 {
 736         int rv;
 737
 738         switch (action) {
 739         case CPU_PM_ENTER:
 740                 rv = sdei_mask_local_cpu();
 741                 break;
 742         case CPU_PM_EXIT:
 743         case CPU_PM_ENTER_FAILED:
 744                 rv = sdei_unmask_local_cpu();
 745                 break;
 746         default:
 747                 return NOTIFY_DONE;
 748         }
 749
 750         if (rv)
 751                 return notifier_from_errno(rv);
 752
 753         return NOTIFY_OK;
 754 }
 755
 756 static struct notifier_block sdei_pm_nb = {
 757         .notifier_call = sdei_pm_notifier,
 758 };
 759
 760 static int sdei_device_suspend(struct device *dev)
 761 {
 762         on_each_cpu(_ipi_mask_cpu, NULL, true);
 763
 764         return 0;
 765 }
 766
 767 static int sdei_device_resume(struct device *dev)
 768 {
 769         on_each_cpu(_ipi_unmask_cpu, NULL, true);
 770
 771         return 0;
 772 }
 773
 774 /*
 775  * We need all events to be reregistered when we resume from hibernate.
 776  *
 777  * The sequence is freeze->thaw. Reboot. freeze->restore. We unregister
 778  * events during freeze, then re-register and re-enable them during thaw
 779  * and restore.
 780  */
 781 static int sdei_device_freeze(struct device *dev)
 782 {
 783         int err;
 784
 785         /* unregister private events */
 786         cpuhp_remove_state(CPUHP_AP_ARM_SDEI_STARTING);
 787
 788         err = sdei_unregister_shared();
 789         if (err)
 790                 return err;
 791
 792         return 0;
 793 }
 794
 795 static int sdei_device_thaw(struct device *dev)
 796 {
 797         int err;
 798
 799         /* re-register shared events */
 800         err = sdei_reregister_shared();
 801         if (err) {
 802                 pr_warn("Failed to re-register shared events...\n");
 803                 sdei_mark_interface_broken();
 804                 return err;
 805         }
 806
 807         err = cpuhp_setup_state(CPUHP_AP_ARM_SDEI_STARTING, "SDEI",
 808                                 &sdei_cpuhp_up, &sdei_cpuhp_down);
 809         if (err)
 810                 pr_warn("Failed to re-register CPU hotplug notifier...\n");
 811
 812         return err;
 813 }
 814
 815 static int sdei_device_restore(struct device *dev)
 816 {
 817         int err;
 818
 819         err = sdei_platform_reset();
 820         if (err)
 821                 return err;
 822
 823         return sdei_device_thaw(dev);
 824 }
 825
 826 static const struct dev_pm_ops sdei_pm_ops = {
 827         .suspend = sdei_device_suspend,
 828         .resume = sdei_device_resume,
 829         .freeze = sdei_device_freeze,
 830         .thaw = sdei_device_thaw,
 831         .restore = sdei_device_restore,
 832 };
 833
 834 /*
 835  * Mask all CPUs and unregister all events on panic, reboot or kexec.
 836  */
 837 static int sdei_reboot_notifier(struct notifier_block *nb, unsigned long action,
 838                                 void *data)
 839 {
 840         /*
 841          * We are going to reset the interface, after this there is no point
 842          * doing work when we take CPUs offline.
 843          */
 844         cpuhp_remove_state(CPUHP_AP_ARM_SDEI_STARTING);
 845
 846         sdei_platform_reset();
 847
 848         return NOTIFY_OK;
 849 }
 850
 851 static struct notifier_block sdei_reboot_nb = {
 852         .notifier_call = sdei_reboot_notifier,
 853 };
 854
 855 static void sdei_smccc_smc(unsigned long function_id,
 856                            unsigned long arg0, unsigned long arg1,
 857                            unsigned long arg2, unsigned long arg3,
 858                            unsigned long arg4, struct arm_smccc_res *res)
 859 {
 860         arm_smccc_smc(function_id, arg0, arg1, arg2, arg3, arg4, 0, 0, res);
 861 }
 862 NOKPROBE_SYMBOL(sdei_smccc_smc);
 863
 864 static void sdei_smccc_hvc(unsigned long function_id,
 865                            unsigned long arg0, unsigned long arg1,
 866                            unsigned long arg2, unsigned long arg3,
 867                            unsigned long arg4, struct arm_smccc_res *res)
 868 {
 869         arm_smccc_hvc(function_id, arg0, arg1, arg2, arg3, arg4, 0, 0, res);
 870 }
 871 NOKPROBE_SYMBOL(sdei_smccc_hvc);
 872
 873 int sdei_register_ghes(struct ghes *ghes, sdei_event_callback *normal_cb,
 874                        sdei_event_callback *critical_cb)
 875 {
 876         int err;
 877         u64 result;
 878         u32 event_num;
 879         sdei_event_callback *cb;
 880
 881         if (!IS_ENABLED(CONFIG_ACPI_APEI_GHES))
 882                 return -EOPNOTSUPP;
 883
 884         event_num = ghes->generic->notify.vector;
 885         if (event_num == 0) {
 886                 /*
 887                  * Event 0 is reserved by the specification for
 888                  * SDEI_EVENT_SIGNAL.
 889                  */
 890                 return -EINVAL;
 891         }
 892
 893         err = sdei_api_event_get_info(event_num, SDEI_EVENT_INFO_EV_PRIORITY,
 894                                       &result);
 895         if (err)
 896                 return err;
 897
 898         if (result == SDEI_EVENT_PRIORITY_CRITICAL)
 899                 cb = critical_cb;
 900         else
 901                 cb = normal_cb;
 902
 903         err = sdei_event_register(event_num, cb, ghes);
 904         if (!err)
 905                 err = sdei_event_enable(event_num);
 906
 907         return err;
 908 }
 909
 910 int sdei_unregister_ghes(struct ghes *ghes)
 911 {
 912         int i;
 913         int err;
 914         u32 event_num = ghes->generic->notify.vector;
 915
 916         might_sleep();
 917
 918         if (!IS_ENABLED(CONFIG_ACPI_APEI_GHES))
 919                 return -EOPNOTSUPP;
 920
 921         /*
 922          * The event may be running on another CPU. Disable it
 923          * to stop new events, then try to unregister a few times.
 924          */
 925         err = sdei_event_disable(event_num);
 926         if (err)
 927                 return err;
 928
 929         for (i = 0; i < 3; i++) {
 930                 err = sdei_event_unregister(event_num);
 931                 if (err != -EINPROGRESS)
 932                         break;
 933
 934                 schedule();
 935         }
 936
 937         return err;
 938 }
 939
 940 static int sdei_get_conduit(struct platform_device *pdev)
 941 {
 942         const char *method;
 943         struct device_node *np = pdev->dev.of_node;
 944
 945         sdei_firmware_call = NULL;
 946         if (np) {
 947                 if (of_property_read_string(np, "method", &method)) {
 948                         pr_warn("missing \"method\" property\n");
 949                         return SMCCC_CONDUIT_NONE;
 950                 }
 951
 952                 if (!strcmp("hvc", method)) {
 953                         sdei_firmware_call = &sdei_smccc_hvc;
 954                         return SMCCC_CONDUIT_HVC;
 955                 } else if (!strcmp("smc", method)) {
 956                         sdei_firmware_call = &sdei_smccc_smc;
 957                         return SMCCC_CONDUIT_SMC;
 958                 }
 959
 960                 pr_warn("invalid \"method\" property: %s\n", method);
 961         } else if (!acpi_disabled) {
 962                 if (acpi_psci_use_hvc()) {
 963                         sdei_firmware_call = &sdei_smccc_hvc;
 964                         return SMCCC_CONDUIT_HVC;
 965                 } else {
 966                         sdei_firmware_call = &sdei_smccc_smc;
 967                         return SMCCC_CONDUIT_SMC;
 968                 }
 969         }
 970
 971         return SMCCC_CONDUIT_NONE;
 972 }
 973
 974 static int sdei_probe(struct platform_device *pdev)
 975 {
 976         int err;
 977         u64 ver = 0;
 978         int conduit;
 979
 980         conduit = sdei_get_conduit(pdev);
 981         if (!sdei_firmware_call)
 982                 return 0;
 983
 984         err = sdei_api_get_version(&ver);
 985         if (err == -EOPNOTSUPP)
 986                 pr_err("advertised but not implemented in platform firmware\n");
 987         if (err) {
 988                 pr_err("Failed to get SDEI version: %d\n", err);
 989                 sdei_mark_interface_broken();
 990                 return err;
 991         }
 992
 993         pr_info("SDEIv%d.%d (0x%x) detected in firmware.\n",
 994                 (int)SDEI_VERSION_MAJOR(ver), (int)SDEI_VERSION_MINOR(ver),
 995                 (int)SDEI_VERSION_VENDOR(ver));
 996
 997         if (SDEI_VERSION_MAJOR(ver) != 1) {
 998                 pr_warn("Conflicting SDEI version detected.\n");
 999                 sdei_mark_interface_broken();
1000                 return -EINVAL;
1001         }
1002
1003         err = sdei_platform_reset();
1004         if (err)
1005                 return err;
1006
1007         sdei_entry_point = sdei_arch_get_entry_point(conduit);
1008         if (!sdei_entry_point) {
1009                 /* Not supported due to hardware or boot configuration */
1010                 sdei_mark_interface_broken();
1011                 return 0;
1012         }
1013
1014         err = cpu_pm_register_notifier(&sdei_pm_nb);
1015         if (err) {
1016                 pr_warn("Failed to register CPU PM notifier...\n");
1017                 goto error;
1018         }
1019
1020         err = register_reboot_notifier(&sdei_reboot_nb);
1021         if (err) {
1022                 pr_warn("Failed to register reboot notifier...\n");
1023                 goto remove_cpupm;
1024         }
1025
1026         err = cpuhp_setup_state(CPUHP_AP_ARM_SDEI_STARTING, "SDEI",
1027                                 &sdei_cpuhp_up, &sdei_cpuhp_down);
1028         if (err) {
1029                 pr_warn("Failed to register CPU hotplug notifier...\n");
1030                 goto remove_reboot;
1031         }
1032
1033         return 0;
1034
1035 remove_reboot:
1036         unregister_reboot_notifier(&sdei_reboot_nb);
1037
1038 remove_cpupm:
1039         cpu_pm_unregister_notifier(&sdei_pm_nb);
1040
1041 error:
1042         sdei_mark_interface_broken();
1043         return err;
1044 }
1045
1046 static const struct of_device_id sdei_of_match[] = {
1047         { .compatible = "arm,sdei-1.0" },
1048         {}
1049 };
1050
1051 static struct platform_driver sdei_driver = {
1052         .driver         = {
1053                 .name                   = "sdei",
1054                 .pm                     = &sdei_pm_ops,
1055                 .of_match_table         = sdei_of_match,
1056         },
1057         .probe          = sdei_probe,
1058 };
1059
1060 static bool __init sdei_present_acpi(void)
1061 {
1062         acpi_status status;
1063         struct acpi_table_header *sdei_table_header;
1064
1065         if (acpi_disabled)
1066                 return false;
1067
1068         status = acpi_get_table(ACPI_SIG_SDEI, 0, &sdei_table_header);
1069         if (ACPI_FAILURE(status) && status != AE_NOT_FOUND) {
1070                 const char *msg = acpi_format_exception(status);
1071
1072                 pr_info("Failed to get ACPI:SDEI table, %s\n", msg);
1073         }
1074         if (ACPI_FAILURE(status))
1075                 return false;
1076
1077         acpi_put_table(sdei_table_header);
1078
1079         return true;
1080 }
1081
1082 static int __init sdei_init(void)
1083 {
1084         struct platform_device *pdev;
1085         int ret;
1086
1087         ret = platform_driver_register(&sdei_driver);
1088         if (ret || !sdei_present_acpi())
1089                 return ret;
1090
1091         pdev = platform_device_register_simple(sdei_driver.driver.name,
1092                                                0, NULL, 0);
1093         if (IS_ERR(pdev)) {
1094                 ret = PTR_ERR(pdev);
1095                 platform_driver_unregister(&sdei_driver);
1096                 pr_info("Failed to register ACPI:SDEI platform device %d\n",
1097                         ret);
1098         }
1099
1100         return ret;
1101 }
1102
1103 /*
1104  * On an ACPI system SDEI needs to be ready before HEST:GHES tries to register
1105  * its events. ACPI is initialised from a subsys_initcall(), GHES is initialised
1106  * by device_initcall(). We want to be called in the middle.
1107  */
1108 subsys_initcall_sync(sdei_init);
1109
1110 int sdei_event_handler(struct pt_regs *regs,
1111                        struct sdei_registered_event *arg)
1112 {
1113         int err;
1114         mm_segment_t orig_addr_limit;
1115         u32 event_num = arg->event_num;
1116
1117         /*
1118          * Save restore 'fs'.
1119          * The architecture's entry code save/restores 'fs' when taking an
1120          * exception from the kernel. This ensures addr_limit isn't inherited
1121          * if you interrupted something that allowed the uaccess routines to
1122          * access kernel memory.
1123          * Do the same here because this doesn't come via the same entry code.
1124         */
1125         orig_addr_limit = force_uaccess_begin();
1126
1127         err = arg->callback(event_num, regs, arg->callback_arg);
1128         if (err)
1129                 pr_err_ratelimited("event %u on CPU %u failed with error: %d\n",
1130                                    event_num, smp_processor_id(), err);
1131
1132         force_uaccess_end(orig_addr_limit);
1133
1134         return err;
1135 }
1136 NOKPROBE_SYMBOL(sdei_event_handler);