drivers/irqchip/irq-gic-v3-its.c

   1 // SPDX-License-Identifier: GPL-2.0-only
   2 /*
   3  * Copyright (C) 2013-2017 ARM Limited, All Rights Reserved.
   4  * Author: Marc Zyngier <marc.zyngier@arm.com>
   5  */
   6
   7 #include <linux/acpi.h>
   8 #include <linux/acpi_iort.h>
   9 #include <linux/bitfield.h>
  10 #include <linux/bitmap.h>
  11 #include <linux/cpu.h>
  12 #include <linux/crash_dump.h>
  13 #include <linux/delay.h>
  14 #include <linux/efi.h>
  15 #include <linux/interrupt.h>
  16 #include <linux/iommu.h>
  17 #include <linux/iopoll.h>
  18 #include <linux/irqdomain.h>
  19 #include <linux/list.h>
  20 #include <linux/log2.h>
  21 #include <linux/memblock.h>
  22 #include <linux/mm.h>
  23 #include <linux/msi.h>
  24 #include <linux/of.h>
  25 #include <linux/of_address.h>
  26 #include <linux/of_irq.h>
  27 #include <linux/of_pci.h>
  28 #include <linux/of_platform.h>
  29 #include <linux/percpu.h>
  30 #include <linux/slab.h>
  31 #include <linux/syscore_ops.h>
  32
  33 #include <linux/irqchip.h>
  34 #include <linux/irqchip/arm-gic-v3.h>
  35 #include <linux/irqchip/arm-gic-v4.h>
  36
  37 #include <asm/cputype.h>
  38 #include <asm/exception.h>
  39
  40 #include "irq-gic-common.h"
  41
  42 #define ITS_FLAGS_CMDQ_NEEDS_FLUSHING           (1ULL << 0)
  43 #define ITS_FLAGS_WORKAROUND_CAVIUM_22375       (1ULL << 1)
  44 #define ITS_FLAGS_WORKAROUND_CAVIUM_23144       (1ULL << 2)
  45 #define ITS_FLAGS_FORCE_NON_SHAREABLE           (1ULL << 3)
  46
  47 #define RD_LOCAL_LPI_ENABLED                    BIT(0)
  48 #define RD_LOCAL_PENDTABLE_PREALLOCATED         BIT(1)
  49 #define RD_LOCAL_MEMRESERVE_DONE                BIT(2)
  50
  51 static u32 lpi_id_bits;
  52
  53 /*
  54  * We allocate memory for PROPBASE to cover 2 ^ lpi_id_bits LPIs to
  55  * deal with (one configuration byte per interrupt). PENDBASE has to
  56  * be 64kB aligned (one bit per LPI, plus 8192 bits for SPI/PPI/SGI).
  57  */
  58 #define LPI_NRBITS              lpi_id_bits
  59 #define LPI_PROPBASE_SZ         ALIGN(BIT(LPI_NRBITS), SZ_64K)
  60 #define LPI_PENDBASE_SZ         ALIGN(BIT(LPI_NRBITS) / 8, SZ_64K)
  61
  62 #define LPI_PROP_DEFAULT_PRIO   GICD_INT_DEF_PRI
  63
  64 /*
  65  * Collection structure - just an ID, and a redistributor address to
  66  * ping. We use one per CPU as a bag of interrupts assigned to this
  67  * CPU.
  68  */
  69 struct its_collection {
  70         u64                     target_address;
  71         u16                     col_id;
  72 };
  73
  74 /*
  75  * The ITS_BASER structure - contains memory information, cached
  76  * value of BASER register configuration and ITS page size.
  77  */
  78 struct its_baser {
  79         void            *base;
  80         u64             val;
  81         u32             order;
  82         u32             psz;
  83 };
  84
  85 struct its_device;
  86
  87 /*
  88  * The ITS structure - contains most of the infrastructure, with the
  89  * top-level MSI domain, the command queue, the collections, and the
  90  * list of devices writing to it.
  91  *
  92  * dev_alloc_lock has to be taken for device allocations, while the
  93  * spinlock must be taken to parse data structures such as the device
  94  * list.
  95  */
  96 struct its_node {
  97         raw_spinlock_t          lock;
  98         struct mutex            dev_alloc_lock;
  99         struct list_head        entry;
 100         void __iomem            *base;
 101         void __iomem            *sgir_base;
 102         phys_addr_t             phys_base;
 103         struct its_cmd_block    *cmd_base;
 104         struct its_cmd_block    *cmd_write;
 105         struct its_baser        tables[GITS_BASER_NR_REGS];
 106         struct its_collection   *collections;
 107         struct fwnode_handle    *fwnode_handle;
 108         u64                     (*get_msi_base)(struct its_device *its_dev);
 109         u64                     typer;
 110         u64                     cbaser_save;
 111         u32                     ctlr_save;
 112         u32                     mpidr;
 113         struct list_head        its_device_list;
 114         u64                     flags;
 115         unsigned long           list_nr;
 116         int                     numa_node;
 117         unsigned int            msi_domain_flags;
 118         u32                     pre_its_base; /* for Socionext Synquacer */
 119         int                     vlpi_redist_offset;
 120 };
 121
 122 #define is_v4(its)              (!!((its)->typer & GITS_TYPER_VLPIS))
 123 #define is_v4_1(its)            (!!((its)->typer & GITS_TYPER_VMAPP))
 124 #define device_ids(its)         (FIELD_GET(GITS_TYPER_DEVBITS, (its)->typer) + 1)
 125
 126 #define ITS_ITT_ALIGN           SZ_256
 127
 128 /* The maximum number of VPEID bits supported by VLPI commands */
 129 #define ITS_MAX_VPEID_BITS                                              \
 130         ({                                                              \
 131                 int nvpeid = 16;                                        \
 132                 if (gic_rdists->has_rvpeid &&                           \
 133                     gic_rdists->gicd_typer2 & GICD_TYPER2_VIL)          \
 134                         nvpeid = 1 + (gic_rdists->gicd_typer2 &         \
 135                                       GICD_TYPER2_VID);                 \
 136                                                                         \
 137                 nvpeid;                                                 \
 138         })
 139 #define ITS_MAX_VPEID           (1 << (ITS_MAX_VPEID_BITS))
 140
 141 /* Convert page order to size in bytes */
 142 #define PAGE_ORDER_TO_SIZE(o)   (PAGE_SIZE << (o))
 143
 144 struct event_lpi_map {
 145         unsigned long           *lpi_map;
 146         u16                     *col_map;
 147         irq_hw_number_t         lpi_base;
 148         int                     nr_lpis;
 149         raw_spinlock_t          vlpi_lock;
 150         struct its_vm           *vm;
 151         struct its_vlpi_map     *vlpi_maps;
 152         int                     nr_vlpis;
 153 };
 154
 155 /*
 156  * The ITS view of a device - belongs to an ITS, owns an interrupt
 157  * translation table, and a list of interrupts.  If it some of its
 158  * LPIs are injected into a guest (GICv4), the event_map.vm field
 159  * indicates which one.
 160  */
 161 struct its_device {
 162         struct list_head        entry;
 163         struct its_node         *its;
 164         struct event_lpi_map    event_map;
 165         void                    *itt;
 166         u32                     nr_ites;
 167         u32                     device_id;
 168         bool                    shared;
 169 };
 170
 171 static struct {
 172         raw_spinlock_t          lock;
 173         struct its_device       *dev;
 174         struct its_vpe          **vpes;
 175         int                     next_victim;
 176 } vpe_proxy;
 177
 178 struct cpu_lpi_count {
 179         atomic_t        managed;
 180         atomic_t        unmanaged;
 181 };
 182
 183 static DEFINE_PER_CPU(struct cpu_lpi_count, cpu_lpi_count);
 184
 185 static LIST_HEAD(its_nodes);
 186 static DEFINE_RAW_SPINLOCK(its_lock);
 187 static struct rdists *gic_rdists;
 188 static struct irq_domain *its_parent;
 189
 190 static unsigned long its_list_map;
 191 static u16 vmovp_seq_num;
 192 static DEFINE_RAW_SPINLOCK(vmovp_lock);
 193
 194 static DEFINE_IDA(its_vpeid_ida);
 195
 196 #define gic_data_rdist()                (raw_cpu_ptr(gic_rdists->rdist))
 197 #define gic_data_rdist_cpu(cpu)         (per_cpu_ptr(gic_rdists->rdist, cpu))
 198 #define gic_data_rdist_rd_base()        (gic_data_rdist()->rd_base)
 199 #define gic_data_rdist_vlpi_base()      (gic_data_rdist_rd_base() + SZ_128K)
 200
 201 /*
 202  * Skip ITSs that have no vLPIs mapped, unless we're on GICv4.1, as we
 203  * always have vSGIs mapped.
 204  */
 205 static bool require_its_list_vmovp(struct its_vm *vm, struct its_node *its)
 206 {
 207         return (gic_rdists->has_rvpeid || vm->vlpi_count[its->list_nr]);
 208 }
 209
 210 static u16 get_its_list(struct its_vm *vm)
 211 {
 212         struct its_node *its;
 213         unsigned long its_list = 0;
 214
 215         list_for_each_entry(its, &its_nodes, entry) {
 216                 if (!is_v4(its))
 217                         continue;
 218
 219                 if (require_its_list_vmovp(vm, its))
 220                         __set_bit(its->list_nr, &its_list);
 221         }
 222
 223         return (u16)its_list;
 224 }
 225
 226 static inline u32 its_get_event_id(struct irq_data *d)
 227 {
 228         struct its_device *its_dev = irq_data_get_irq_chip_data(d);
 229         return d->hwirq - its_dev->event_map.lpi_base;
 230 }
 231
 232 static struct its_collection *dev_event_to_col(struct its_device *its_dev,
 233                                                u32 event)
 234 {
 235         struct its_node *its = its_dev->its;
 236
 237         return its->collections + its_dev->event_map.col_map[event];
 238 }
 239
 240 static struct its_vlpi_map *dev_event_to_vlpi_map(struct its_device *its_dev,
 241                                                u32 event)
 242 {
 243         if (WARN_ON_ONCE(event >= its_dev->event_map.nr_lpis))
 244                 return NULL;
 245
 246         return &its_dev->event_map.vlpi_maps[event];
 247 }
 248
 249 static struct its_vlpi_map *get_vlpi_map(struct irq_data *d)
 250 {
 251         if (irqd_is_forwarded_to_vcpu(d)) {
 252                 struct its_device *its_dev = irq_data_get_irq_chip_data(d);
 253                 u32 event = its_get_event_id(d);
 254
 255                 return dev_event_to_vlpi_map(its_dev, event);
 256         }
 257
 258         return NULL;
 259 }
 260
 261 static int vpe_to_cpuid_lock(struct its_vpe *vpe, unsigned long *flags)
 262 {
 263         raw_spin_lock_irqsave(&vpe->vpe_lock, *flags);
 264         return vpe->col_idx;
 265 }
 266
 267 static void vpe_to_cpuid_unlock(struct its_vpe *vpe, unsigned long flags)
 268 {
 269         raw_spin_unlock_irqrestore(&vpe->vpe_lock, flags);
 270 }
 271
 272 static struct irq_chip its_vpe_irq_chip;
 273
 274 static int irq_to_cpuid_lock(struct irq_data *d, unsigned long *flags)
 275 {
 276         struct its_vpe *vpe = NULL;
 277         int cpu;
 278
 279         if (d->chip == &its_vpe_irq_chip) {
 280                 vpe = irq_data_get_irq_chip_data(d);
 281         } else {
 282                 struct its_vlpi_map *map = get_vlpi_map(d);
 283                 if (map)
 284                         vpe = map->vpe;
 285         }
 286
 287         if (vpe) {
 288                 cpu = vpe_to_cpuid_lock(vpe, flags);
 289         } else {
 290                 /* Physical LPIs are already locked via the irq_desc lock */
 291                 struct its_device *its_dev = irq_data_get_irq_chip_data(d);
 292                 cpu = its_dev->event_map.col_map[its_get_event_id(d)];
 293                 /* Keep GCC quiet... */
 294                 *flags = 0;
 295         }
 296
 297         return cpu;
 298 }
 299
 300 static void irq_to_cpuid_unlock(struct irq_data *d, unsigned long flags)
 301 {
 302         struct its_vpe *vpe = NULL;
 303
 304         if (d->chip == &its_vpe_irq_chip) {
 305                 vpe = irq_data_get_irq_chip_data(d);
 306         } else {
 307                 struct its_vlpi_map *map = get_vlpi_map(d);
 308                 if (map)
 309                         vpe = map->vpe;
 310         }
 311
 312         if (vpe)
 313                 vpe_to_cpuid_unlock(vpe, flags);
 314 }
 315
 316 static struct its_collection *valid_col(struct its_collection *col)
 317 {
 318         if (WARN_ON_ONCE(col->target_address & GENMASK_ULL(15, 0)))
 319                 return NULL;
 320
 321         return col;
 322 }
 323
 324 static struct its_vpe *valid_vpe(struct its_node *its, struct its_vpe *vpe)
 325 {
 326         if (valid_col(its->collections + vpe->col_idx))
 327                 return vpe;
 328
 329         return NULL;
 330 }
 331
 332 /*
 333  * ITS command descriptors - parameters to be encoded in a command
 334  * block.
 335  */
 336 struct its_cmd_desc {
 337         union {
 338                 struct {
 339                         struct its_device *dev;
 340                         u32 event_id;
 341                 } its_inv_cmd;
 342
 343                 struct {
 344                         struct its_device *dev;
 345                         u32 event_id;
 346                 } its_clear_cmd;
 347
 348                 struct {
 349                         struct its_device *dev;
 350                         u32 event_id;
 351                 } its_int_cmd;
 352
 353                 struct {
 354                         struct its_device *dev;
 355                         int valid;
 356                 } its_mapd_cmd;
 357
 358                 struct {
 359                         struct its_collection *col;
 360                         int valid;
 361                 } its_mapc_cmd;
 362
 363                 struct {
 364                         struct its_device *dev;
 365                         u32 phys_id;
 366                         u32 event_id;
 367                 } its_mapti_cmd;
 368
 369                 struct {
 370                         struct its_device *dev;
 371                         struct its_collection *col;
 372                         u32 event_id;
 373                 } its_movi_cmd;
 374
 375                 struct {
 376                         struct its_device *dev;
 377                         u32 event_id;
 378                 } its_discard_cmd;
 379
 380                 struct {
 381                         struct its_collection *col;
 382                 } its_invall_cmd;
 383
 384                 struct {
 385                         struct its_vpe *vpe;
 386                 } its_vinvall_cmd;
 387
 388                 struct {
 389                         struct its_vpe *vpe;
 390                         struct its_collection *col;
 391                         bool valid;
 392                 } its_vmapp_cmd;
 393
 394                 struct {
 395                         struct its_vpe *vpe;
 396                         struct its_device *dev;
 397                         u32 virt_id;
 398                         u32 event_id;
 399                         bool db_enabled;
 400                 } its_vmapti_cmd;
 401
 402                 struct {
 403                         struct its_vpe *vpe;
 404                         struct its_device *dev;
 405                         u32 event_id;
 406                         bool db_enabled;
 407                 } its_vmovi_cmd;
 408
 409                 struct {
 410                         struct its_vpe *vpe;
 411                         struct its_collection *col;
 412                         u16 seq_num;
 413                         u16 its_list;
 414                 } its_vmovp_cmd;
 415
 416                 struct {
 417                         struct its_vpe *vpe;
 418                 } its_invdb_cmd;
 419
 420                 struct {
 421                         struct its_vpe *vpe;
 422                         u8 sgi;
 423                         u8 priority;
 424                         bool enable;
 425                         bool group;
 426                         bool clear;
 427                 } its_vsgi_cmd;
 428         };
 429 };
 430
 431 /*
 432  * The ITS command block, which is what the ITS actually parses.
 433  */
 434 struct its_cmd_block {
 435         union {
 436                 u64     raw_cmd[4];
 437                 __le64  raw_cmd_le[4];
 438         };
 439 };
 440
 441 #define ITS_CMD_QUEUE_SZ                SZ_64K
 442 #define ITS_CMD_QUEUE_NR_ENTRIES        (ITS_CMD_QUEUE_SZ / sizeof(struct its_cmd_block))
 443
 444 typedef struct its_collection *(*its_cmd_builder_t)(struct its_node *,
 445                                                     struct its_cmd_block *,
 446                                                     struct its_cmd_desc *);
 447
 448 typedef struct its_vpe *(*its_cmd_vbuilder_t)(struct its_node *,
 449                                               struct its_cmd_block *,
 450                                               struct its_cmd_desc *);
 451
 452 static void its_mask_encode(u64 *raw_cmd, u64 val, int h, int l)
 453 {
 454         u64 mask = GENMASK_ULL(h, l);
 455         *raw_cmd &= ~mask;
 456         *raw_cmd |= (val << l) & mask;
 457 }
 458
 459 static void its_encode_cmd(struct its_cmd_block *cmd, u8 cmd_nr)
 460 {
 461         its_mask_encode(&cmd->raw_cmd[0], cmd_nr, 7, 0);
 462 }
 463
 464 static void its_encode_devid(struct its_cmd_block *cmd, u32 devid)
 465 {
 466         its_mask_encode(&cmd->raw_cmd[0], devid, 63, 32);
 467 }
 468
 469 static void its_encode_event_id(struct its_cmd_block *cmd, u32 id)
 470 {
 471         its_mask_encode(&cmd->raw_cmd[1], id, 31, 0);
 472 }
 473
 474 static void its_encode_phys_id(struct its_cmd_block *cmd, u32 phys_id)
 475 {
 476         its_mask_encode(&cmd->raw_cmd[1], phys_id, 63, 32);
 477 }
 478
 479 static void its_encode_size(struct its_cmd_block *cmd, u8 size)
 480 {
 481         its_mask_encode(&cmd->raw_cmd[1], size, 4, 0);
 482 }
 483
 484 static void its_encode_itt(struct its_cmd_block *cmd, u64 itt_addr)
 485 {
 486         its_mask_encode(&cmd->raw_cmd[2], itt_addr >> 8, 51, 8);
 487 }
 488
 489 static void its_encode_valid(struct its_cmd_block *cmd, int valid)
 490 {
 491         its_mask_encode(&cmd->raw_cmd[2], !!valid, 63, 63);
 492 }
 493
 494 static void its_encode_target(struct its_cmd_block *cmd, u64 target_addr)
 495 {
 496         its_mask_encode(&cmd->raw_cmd[2], target_addr >> 16, 51, 16);
 497 }
 498
 499 static void its_encode_collection(struct its_cmd_block *cmd, u16 col)
 500 {
 501         its_mask_encode(&cmd->raw_cmd[2], col, 15, 0);
 502 }
 503
 504 static void its_encode_vpeid(struct its_cmd_block *cmd, u16 vpeid)
 505 {
 506         its_mask_encode(&cmd->raw_cmd[1], vpeid, 47, 32);
 507 }
 508
 509 static void its_encode_virt_id(struct its_cmd_block *cmd, u32 virt_id)
 510 {
 511         its_mask_encode(&cmd->raw_cmd[2], virt_id, 31, 0);
 512 }
 513
 514 static void its_encode_db_phys_id(struct its_cmd_block *cmd, u32 db_phys_id)
 515 {
 516         its_mask_encode(&cmd->raw_cmd[2], db_phys_id, 63, 32);
 517 }
 518
 519 static void its_encode_db_valid(struct its_cmd_block *cmd, bool db_valid)
 520 {
 521         its_mask_encode(&cmd->raw_cmd[2], db_valid, 0, 0);
 522 }
 523
 524 static void its_encode_seq_num(struct its_cmd_block *cmd, u16 seq_num)
 525 {
 526         its_mask_encode(&cmd->raw_cmd[0], seq_num, 47, 32);
 527 }
 528
 529 static void its_encode_its_list(struct its_cmd_block *cmd, u16 its_list)
 530 {
 531         its_mask_encode(&cmd->raw_cmd[1], its_list, 15, 0);
 532 }
 533
 534 static void its_encode_vpt_addr(struct its_cmd_block *cmd, u64 vpt_pa)
 535 {
 536         its_mask_encode(&cmd->raw_cmd[3], vpt_pa >> 16, 51, 16);
 537 }
 538
 539 static void its_encode_vpt_size(struct its_cmd_block *cmd, u8 vpt_size)
 540 {
 541         its_mask_encode(&cmd->raw_cmd[3], vpt_size, 4, 0);
 542 }
 543
 544 static void its_encode_vconf_addr(struct its_cmd_block *cmd, u64 vconf_pa)
 545 {
 546         its_mask_encode(&cmd->raw_cmd[0], vconf_pa >> 16, 51, 16);
 547 }
 548
 549 static void its_encode_alloc(struct its_cmd_block *cmd, bool alloc)
 550 {
 551         its_mask_encode(&cmd->raw_cmd[0], alloc, 8, 8);
 552 }
 553
 554 static void its_encode_ptz(struct its_cmd_block *cmd, bool ptz)
 555 {
 556         its_mask_encode(&cmd->raw_cmd[0], ptz, 9, 9);
 557 }
 558
 559 static void its_encode_vmapp_default_db(struct its_cmd_block *cmd,
 560                                         u32 vpe_db_lpi)
 561 {
 562         its_mask_encode(&cmd->raw_cmd[1], vpe_db_lpi, 31, 0);
 563 }
 564
 565 static void its_encode_vmovp_default_db(struct its_cmd_block *cmd,
 566                                         u32 vpe_db_lpi)
 567 {
 568         its_mask_encode(&cmd->raw_cmd[3], vpe_db_lpi, 31, 0);
 569 }
 570
 571 static void its_encode_db(struct its_cmd_block *cmd, bool db)
 572 {
 573         its_mask_encode(&cmd->raw_cmd[2], db, 63, 63);
 574 }
 575
 576 static void its_encode_sgi_intid(struct its_cmd_block *cmd, u8 sgi)
 577 {
 578         its_mask_encode(&cmd->raw_cmd[0], sgi, 35, 32);
 579 }
 580
 581 static void its_encode_sgi_priority(struct its_cmd_block *cmd, u8 prio)
 582 {
 583         its_mask_encode(&cmd->raw_cmd[0], prio >> 4, 23, 20);
 584 }
 585
 586 static void its_encode_sgi_group(struct its_cmd_block *cmd, bool grp)
 587 {
 588         its_mask_encode(&cmd->raw_cmd[0], grp, 10, 10);
 589 }
 590
 591 static void its_encode_sgi_clear(struct its_cmd_block *cmd, bool clr)
 592 {
 593         its_mask_encode(&cmd->raw_cmd[0], clr, 9, 9);
 594 }
 595
 596 static void its_encode_sgi_enable(struct its_cmd_block *cmd, bool en)
 597 {
 598         its_mask_encode(&cmd->raw_cmd[0], en, 8, 8);
 599 }
 600
 601 static inline void its_fixup_cmd(struct its_cmd_block *cmd)
 602 {
 603         /* Let's fixup BE commands */
 604         cmd->raw_cmd_le[0] = cpu_to_le64(cmd->raw_cmd[0]);
 605         cmd->raw_cmd_le[1] = cpu_to_le64(cmd->raw_cmd[1]);
 606         cmd->raw_cmd_le[2] = cpu_to_le64(cmd->raw_cmd[2]);
 607         cmd->raw_cmd_le[3] = cpu_to_le64(cmd->raw_cmd[3]);
 608 }
 609
 610 static struct its_collection *its_build_mapd_cmd(struct its_node *its,
 611                                                  struct its_cmd_block *cmd,
 612                                                  struct its_cmd_desc *desc)
 613 {
 614         unsigned long itt_addr;
 615         u8 size = ilog2(desc->its_mapd_cmd.dev->nr_ites);
 616
 617         itt_addr = virt_to_phys(desc->its_mapd_cmd.dev->itt);
 618         itt_addr = ALIGN(itt_addr, ITS_ITT_ALIGN);
 619
 620         its_encode_cmd(cmd, GITS_CMD_MAPD);
 621         its_encode_devid(cmd, desc->its_mapd_cmd.dev->device_id);
 622         its_encode_size(cmd, size - 1);
 623         its_encode_itt(cmd, itt_addr);
 624         its_encode_valid(cmd, desc->its_mapd_cmd.valid);
 625
 626         its_fixup_cmd(cmd);
 627
 628         return NULL;
 629 }
 630
 631 static struct its_collection *its_build_mapc_cmd(struct its_node *its,
 632                                                  struct its_cmd_block *cmd,
 633                                                  struct its_cmd_desc *desc)
 634 {
 635         its_encode_cmd(cmd, GITS_CMD_MAPC);
 636         its_encode_collection(cmd, desc->its_mapc_cmd.col->col_id);
 637         its_encode_target(cmd, desc->its_mapc_cmd.col->target_address);
 638         its_encode_valid(cmd, desc->its_mapc_cmd.valid);
 639
 640         its_fixup_cmd(cmd);
 641
 642         return desc->its_mapc_cmd.col;
 643 }
 644
 645 static struct its_collection *its_build_mapti_cmd(struct its_node *its,
 646                                                   struct its_cmd_block *cmd,
 647                                                   struct its_cmd_desc *desc)
 648 {
 649         struct its_collection *col;
 650
 651         col = dev_event_to_col(desc->its_mapti_cmd.dev,
 652                                desc->its_mapti_cmd.event_id);
 653
 654         its_encode_cmd(cmd, GITS_CMD_MAPTI);
 655         its_encode_devid(cmd, desc->its_mapti_cmd.dev->device_id);
 656         its_encode_event_id(cmd, desc->its_mapti_cmd.event_id);
 657         its_encode_phys_id(cmd, desc->its_mapti_cmd.phys_id);
 658         its_encode_collection(cmd, col->col_id);
 659
 660         its_fixup_cmd(cmd);
 661
 662         return valid_col(col);
 663 }
 664
 665 static struct its_collection *its_build_movi_cmd(struct its_node *its,
 666                                                  struct its_cmd_block *cmd,
 667                                                  struct its_cmd_desc *desc)
 668 {
 669         struct its_collection *col;
 670
 671         col = dev_event_to_col(desc->its_movi_cmd.dev,
 672                                desc->its_movi_cmd.event_id);
 673
 674         its_encode_cmd(cmd, GITS_CMD_MOVI);
 675         its_encode_devid(cmd, desc->its_movi_cmd.dev->device_id);
 676         its_encode_event_id(cmd, desc->its_movi_cmd.event_id);
 677         its_encode_collection(cmd, desc->its_movi_cmd.col->col_id);
 678
 679         its_fixup_cmd(cmd);
 680
 681         return valid_col(col);
 682 }
 683
 684 static struct its_collection *its_build_discard_cmd(struct its_node *its,
 685                                                     struct its_cmd_block *cmd,
 686                                                     struct its_cmd_desc *desc)
 687 {
 688         struct its_collection *col;
 689
 690         col = dev_event_to_col(desc->its_discard_cmd.dev,
 691                                desc->its_discard_cmd.event_id);
 692
 693         its_encode_cmd(cmd, GITS_CMD_DISCARD);
 694         its_encode_devid(cmd, desc->its_discard_cmd.dev->device_id);
 695         its_encode_event_id(cmd, desc->its_discard_cmd.event_id);
 696
 697         its_fixup_cmd(cmd);
 698
 699         return valid_col(col);
 700 }
 701
 702 static struct its_collection *its_build_inv_cmd(struct its_node *its,
 703                                                 struct its_cmd_block *cmd,
 704                                                 struct its_cmd_desc *desc)
 705 {
 706         struct its_collection *col;
 707
 708         col = dev_event_to_col(desc->its_inv_cmd.dev,
 709                                desc->its_inv_cmd.event_id);
 710
 711         its_encode_cmd(cmd, GITS_CMD_INV);
 712         its_encode_devid(cmd, desc->its_inv_cmd.dev->device_id);
 713         its_encode_event_id(cmd, desc->its_inv_cmd.event_id);
 714
 715         its_fixup_cmd(cmd);
 716
 717         return valid_col(col);
 718 }
 719
 720 static struct its_collection *its_build_int_cmd(struct its_node *its,
 721                                                 struct its_cmd_block *cmd,
 722                                                 struct its_cmd_desc *desc)
 723 {
 724         struct its_collection *col;
 725
 726         col = dev_event_to_col(desc->its_int_cmd.dev,
 727                                desc->its_int_cmd.event_id);
 728
 729         its_encode_cmd(cmd, GITS_CMD_INT);
 730         its_encode_devid(cmd, desc->its_int_cmd.dev->device_id);
 731         its_encode_event_id(cmd, desc->its_int_cmd.event_id);
 732
 733         its_fixup_cmd(cmd);
 734
 735         return valid_col(col);
 736 }
 737
 738 static struct its_collection *its_build_clear_cmd(struct its_node *its,
 739                                                   struct its_cmd_block *cmd,
 740                                                   struct its_cmd_desc *desc)
 741 {
 742         struct its_collection *col;
 743
 744         col = dev_event_to_col(desc->its_clear_cmd.dev,
 745                                desc->its_clear_cmd.event_id);
 746
 747         its_encode_cmd(cmd, GITS_CMD_CLEAR);
 748         its_encode_devid(cmd, desc->its_clear_cmd.dev->device_id);
 749         its_encode_event_id(cmd, desc->its_clear_cmd.event_id);
 750
 751         its_fixup_cmd(cmd);
 752
 753         return valid_col(col);
 754 }
 755
 756 static struct its_collection *its_build_invall_cmd(struct its_node *its,
 757                                                    struct its_cmd_block *cmd,
 758                                                    struct its_cmd_desc *desc)
 759 {
 760         its_encode_cmd(cmd, GITS_CMD_INVALL);
 761         its_encode_collection(cmd, desc->its_invall_cmd.col->col_id);
 762
 763         its_fixup_cmd(cmd);
 764
 765         return desc->its_invall_cmd.col;
 766 }
 767
 768 static struct its_vpe *its_build_vinvall_cmd(struct its_node *its,
 769                                              struct its_cmd_block *cmd,
 770                                              struct its_cmd_desc *desc)
 771 {
 772         its_encode_cmd(cmd, GITS_CMD_VINVALL);
 773         its_encode_vpeid(cmd, desc->its_vinvall_cmd.vpe->vpe_id);
 774
 775         its_fixup_cmd(cmd);
 776
 777         return valid_vpe(its, desc->its_vinvall_cmd.vpe);
 778 }
 779
 780 static struct its_vpe *its_build_vmapp_cmd(struct its_node *its,
 781                                            struct its_cmd_block *cmd,
 782                                            struct its_cmd_desc *desc)
 783 {
 784         unsigned long vpt_addr, vconf_addr;
 785         u64 target;
 786         bool alloc;
 787
 788         its_encode_cmd(cmd, GITS_CMD_VMAPP);
 789         its_encode_vpeid(cmd, desc->its_vmapp_cmd.vpe->vpe_id);
 790         its_encode_valid(cmd, desc->its_vmapp_cmd.valid);
 791
 792         if (!desc->its_vmapp_cmd.valid) {
 793                 if (is_v4_1(its)) {
 794                         alloc = !atomic_dec_return(&desc->its_vmapp_cmd.vpe->vmapp_count);
 795                         its_encode_alloc(cmd, alloc);
 796                 }
 797
 798                 goto out;
 799         }
 800
 801         vpt_addr = virt_to_phys(page_address(desc->its_vmapp_cmd.vpe->vpt_page));
 802         target = desc->its_vmapp_cmd.col->target_address + its->vlpi_redist_offset;
 803
 804         its_encode_target(cmd, target);
 805         its_encode_vpt_addr(cmd, vpt_addr);
 806         its_encode_vpt_size(cmd, LPI_NRBITS - 1);
 807
 808         if (!is_v4_1(its))
 809                 goto out;
 810
 811         vconf_addr = virt_to_phys(page_address(desc->its_vmapp_cmd.vpe->its_vm->vprop_page));
 812
 813         alloc = !atomic_fetch_inc(&desc->its_vmapp_cmd.vpe->vmapp_count);
 814
 815         its_encode_alloc(cmd, alloc);
 816
 817         /*
 818          * GICv4.1 provides a way to get the VLPI state, which needs the vPE
 819          * to be unmapped first, and in this case, we may remap the vPE
 820          * back while the VPT is not empty. So we can't assume that the
 821          * VPT is empty on map. This is why we never advertise PTZ.
 822          */
 823         its_encode_ptz(cmd, false);
 824         its_encode_vconf_addr(cmd, vconf_addr);
 825         its_encode_vmapp_default_db(cmd, desc->its_vmapp_cmd.vpe->vpe_db_lpi);
 826
 827 out:
 828         its_fixup_cmd(cmd);
 829
 830         return valid_vpe(its, desc->its_vmapp_cmd.vpe);
 831 }
 832
 833 static struct its_vpe *its_build_vmapti_cmd(struct its_node *its,
 834                                             struct its_cmd_block *cmd,
 835                                             struct its_cmd_desc *desc)
 836 {
 837         u32 db;
 838
 839         if (!is_v4_1(its) && desc->its_vmapti_cmd.db_enabled)
 840                 db = desc->its_vmapti_cmd.vpe->vpe_db_lpi;
 841         else
 842                 db = 1023;
 843
 844         its_encode_cmd(cmd, GITS_CMD_VMAPTI);
 845         its_encode_devid(cmd, desc->its_vmapti_cmd.dev->device_id);
 846         its_encode_vpeid(cmd, desc->its_vmapti_cmd.vpe->vpe_id);
 847         its_encode_event_id(cmd, desc->its_vmapti_cmd.event_id);
 848         its_encode_db_phys_id(cmd, db);
 849         its_encode_virt_id(cmd, desc->its_vmapti_cmd.virt_id);
 850
 851         its_fixup_cmd(cmd);
 852
 853         return valid_vpe(its, desc->its_vmapti_cmd.vpe);
 854 }
 855
 856 static struct its_vpe *its_build_vmovi_cmd(struct its_node *its,
 857                                            struct its_cmd_block *cmd,
 858                                            struct its_cmd_desc *desc)
 859 {
 860         u32 db;
 861
 862         if (!is_v4_1(its) && desc->its_vmovi_cmd.db_enabled)
 863                 db = desc->its_vmovi_cmd.vpe->vpe_db_lpi;
 864         else
 865                 db = 1023;
 866
 867         its_encode_cmd(cmd, GITS_CMD_VMOVI);
 868         its_encode_devid(cmd, desc->its_vmovi_cmd.dev->device_id);
 869         its_encode_vpeid(cmd, desc->its_vmovi_cmd.vpe->vpe_id);
 870         its_encode_event_id(cmd, desc->its_vmovi_cmd.event_id);
 871         its_encode_db_phys_id(cmd, db);
 872         its_encode_db_valid(cmd, true);
 873
 874         its_fixup_cmd(cmd);
 875
 876         return valid_vpe(its, desc->its_vmovi_cmd.vpe);
 877 }
 878
 879 static struct its_vpe *its_build_vmovp_cmd(struct its_node *its,
 880                                            struct its_cmd_block *cmd,
 881                                            struct its_cmd_desc *desc)
 882 {
 883         u64 target;
 884
 885         target = desc->its_vmovp_cmd.col->target_address + its->vlpi_redist_offset;
 886         its_encode_cmd(cmd, GITS_CMD_VMOVP);
 887         its_encode_seq_num(cmd, desc->its_vmovp_cmd.seq_num);
 888         its_encode_its_list(cmd, desc->its_vmovp_cmd.its_list);
 889         its_encode_vpeid(cmd, desc->its_vmovp_cmd.vpe->vpe_id);
 890         its_encode_target(cmd, target);
 891
 892         if (is_v4_1(its)) {
 893                 its_encode_db(cmd, true);
 894                 its_encode_vmovp_default_db(cmd, desc->its_vmovp_cmd.vpe->vpe_db_lpi);
 895         }
 896
 897         its_fixup_cmd(cmd);
 898
 899         return valid_vpe(its, desc->its_vmovp_cmd.vpe);
 900 }
 901
 902 static struct its_vpe *its_build_vinv_cmd(struct its_node *its,
 903                                           struct its_cmd_block *cmd,
 904                                           struct its_cmd_desc *desc)
 905 {
 906         struct its_vlpi_map *map;
 907
 908         map = dev_event_to_vlpi_map(desc->its_inv_cmd.dev,
 909                                     desc->its_inv_cmd.event_id);
 910
 911         its_encode_cmd(cmd, GITS_CMD_INV);
 912         its_encode_devid(cmd, desc->its_inv_cmd.dev->device_id);
 913         its_encode_event_id(cmd, desc->its_inv_cmd.event_id);
 914
 915         its_fixup_cmd(cmd);
 916
 917         return valid_vpe(its, map->vpe);
 918 }
 919
 920 static struct its_vpe *its_build_vint_cmd(struct its_node *its,
 921                                           struct its_cmd_block *cmd,
 922                                           struct its_cmd_desc *desc)
 923 {
 924         struct its_vlpi_map *map;
 925
 926         map = dev_event_to_vlpi_map(desc->its_int_cmd.dev,
 927                                     desc->its_int_cmd.event_id);
 928
 929         its_encode_cmd(cmd, GITS_CMD_INT);
 930         its_encode_devid(cmd, desc->its_int_cmd.dev->device_id);
 931         its_encode_event_id(cmd, desc->its_int_cmd.event_id);
 932
 933         its_fixup_cmd(cmd);
 934
 935         return valid_vpe(its, map->vpe);
 936 }
 937
 938 static struct its_vpe *its_build_vclear_cmd(struct its_node *its,
 939                                             struct its_cmd_block *cmd,
 940                                             struct its_cmd_desc *desc)
 941 {
 942         struct its_vlpi_map *map;
 943
 944         map = dev_event_to_vlpi_map(desc->its_clear_cmd.dev,
 945                                     desc->its_clear_cmd.event_id);
 946
 947         its_encode_cmd(cmd, GITS_CMD_CLEAR);
 948         its_encode_devid(cmd, desc->its_clear_cmd.dev->device_id);
 949         its_encode_event_id(cmd, desc->its_clear_cmd.event_id);
 950
 951         its_fixup_cmd(cmd);
 952
 953         return valid_vpe(its, map->vpe);
 954 }
 955
 956 static struct its_vpe *its_build_invdb_cmd(struct its_node *its,
 957                                            struct its_cmd_block *cmd,
 958                                            struct its_cmd_desc *desc)
 959 {
 960         if (WARN_ON(!is_v4_1(its)))
 961                 return NULL;
 962
 963         its_encode_cmd(cmd, GITS_CMD_INVDB);
 964         its_encode_vpeid(cmd, desc->its_invdb_cmd.vpe->vpe_id);
 965
 966         its_fixup_cmd(cmd);
 967
 968         return valid_vpe(its, desc->its_invdb_cmd.vpe);
 969 }
 970
 971 static struct its_vpe *its_build_vsgi_cmd(struct its_node *its,
 972                                           struct its_cmd_block *cmd,
 973                                           struct its_cmd_desc *desc)
 974 {
 975         if (WARN_ON(!is_v4_1(its)))
 976                 return NULL;
 977
 978         its_encode_cmd(cmd, GITS_CMD_VSGI);
 979         its_encode_vpeid(cmd, desc->its_vsgi_cmd.vpe->vpe_id);
 980         its_encode_sgi_intid(cmd, desc->its_vsgi_cmd.sgi);
 981         its_encode_sgi_priority(cmd, desc->its_vsgi_cmd.priority);
 982         its_encode_sgi_group(cmd, desc->its_vsgi_cmd.group);
 983         its_encode_sgi_clear(cmd, desc->its_vsgi_cmd.clear);
 984         its_encode_sgi_enable(cmd, desc->its_vsgi_cmd.enable);
 985
 986         its_fixup_cmd(cmd);
 987
 988         return valid_vpe(its, desc->its_vsgi_cmd.vpe);
 989 }
 990
 991 static u64 its_cmd_ptr_to_offset(struct its_node *its,
 992                                  struct its_cmd_block *ptr)
 993 {
 994         return (ptr - its->cmd_base) * sizeof(*ptr);
 995 }
 996
 997 static int its_queue_full(struct its_node *its)
 998 {
 999         int widx;
1000         int ridx;
1001
1002         widx = its->cmd_write - its->cmd_base;
1003         ridx = readl_relaxed(its->base + GITS_CREADR) / sizeof(struct its_cmd_block);
1004
1005         /* This is incredibly unlikely to happen, unless the ITS locks up. */
1006         if (((widx + 1) % ITS_CMD_QUEUE_NR_ENTRIES) == ridx)
1007                 return 1;
1008
1009         return 0;
1010 }
1011
1012 static struct its_cmd_block *its_allocate_entry(struct its_node *its)
1013 {
1014         struct its_cmd_block *cmd;
1015         u32 count = 1000000;    /* 1s! */
1016
1017         while (its_queue_full(its)) {
1018                 count--;
1019                 if (!count) {
1020                         pr_err_ratelimited("ITS queue not draining\n");
1021                         return NULL;
1022                 }
1023                 cpu_relax();
1024                 udelay(1);
1025         }
1026
1027         cmd = its->cmd_write++;
1028
1029         /* Handle queue wrapping */
1030         if (its->cmd_write == (its->cmd_base + ITS_CMD_QUEUE_NR_ENTRIES))
1031                 its->cmd_write = its->cmd_base;
1032
1033         /* Clear command  */
1034         cmd->raw_cmd[0] = 0;
1035         cmd->raw_cmd[1] = 0;
1036         cmd->raw_cmd[2] = 0;
1037         cmd->raw_cmd[3] = 0;
1038
1039         return cmd;
1040 }
1041
1042 static struct its_cmd_block *its_post_commands(struct its_node *its)
1043 {
1044         u64 wr = its_cmd_ptr_to_offset(its, its->cmd_write);
1045
1046         writel_relaxed(wr, its->base + GITS_CWRITER);
1047
1048         return its->cmd_write;
1049 }
1050
1051 static void its_flush_cmd(struct its_node *its, struct its_cmd_block *cmd)
1052 {
1053         /*
1054          * Make sure the commands written to memory are observable by
1055          * the ITS.
1056          */
1057         if (its->flags & ITS_FLAGS_CMDQ_NEEDS_FLUSHING)
1058                 gic_flush_dcache_to_poc(cmd, sizeof(*cmd));
1059         else
1060                 dsb(ishst);
1061 }
1062
1063 static int its_wait_for_range_completion(struct its_node *its,
1064                                          u64    prev_idx,
1065                                          struct its_cmd_block *to)
1066 {
1067         u64 rd_idx, to_idx, linear_idx;
1068         u32 count = 1000000;    /* 1s! */
1069
1070         /* Linearize to_idx if the command set has wrapped around */
1071         to_idx = its_cmd_ptr_to_offset(its, to);
1072         if (to_idx < prev_idx)
1073                 to_idx += ITS_CMD_QUEUE_SZ;
1074
1075         linear_idx = prev_idx;
1076
1077         while (1) {
1078                 s64 delta;
1079
1080                 rd_idx = readl_relaxed(its->base + GITS_CREADR);
1081
1082                 /*
1083                  * Compute the read pointer progress, taking the
1084                  * potential wrap-around into account.
1085                  */
1086                 delta = rd_idx - prev_idx;
1087                 if (rd_idx < prev_idx)
1088                         delta += ITS_CMD_QUEUE_SZ;
1089
1090                 linear_idx += delta;
1091                 if (linear_idx >= to_idx)
1092                         break;
1093
1094                 count--;
1095                 if (!count) {
1096                         pr_err_ratelimited("ITS queue timeout (%llu %llu)\n",
1097                                            to_idx, linear_idx);
1098                         return -1;
1099                 }
1100                 prev_idx = rd_idx;
1101                 cpu_relax();
1102                 udelay(1);
1103         }
1104
1105         return 0;
1106 }
1107
1108 /* Warning, macro hell follows */
1109 #define BUILD_SINGLE_CMD_FUNC(name, buildtype, synctype, buildfn)       \
1110 void name(struct its_node *its,                                         \
1111           buildtype builder,                                            \
1112           struct its_cmd_desc *desc)                                    \
1113 {                                                                       \
1114         struct its_cmd_block *cmd, *sync_cmd, *next_cmd;                \
1115         synctype *sync_obj;                                             \
1116         unsigned long flags;                                            \
1117         u64 rd_idx;                                                     \
1118                                                                         \
1119         raw_spin_lock_irqsave(&its->lock, flags);                       \
1120                                                                         \
1121         cmd = its_allocate_entry(its);                                  \
1122         if (!cmd) {             /* We're soooooo screewed... */         \
1123                 raw_spin_unlock_irqrestore(&its->lock, flags);          \
1124                 return;                                                 \
1125         }                                                               \
1126         sync_obj = builder(its, cmd, desc);                             \
1127         its_flush_cmd(its, cmd);                                        \
1128                                                                         \
1129         if (sync_obj) {                                                 \
1130                 sync_cmd = its_allocate_entry(its);                     \
1131                 if (!sync_cmd)                                          \
1132                         goto post;                                      \
1133                                                                         \
1134                 buildfn(its, sync_cmd, sync_obj);                       \
1135                 its_flush_cmd(its, sync_cmd);                           \
1136         }                                                               \
1137                                                                         \
1138 post:                                                                   \
1139         rd_idx = readl_relaxed(its->base + GITS_CREADR);                \
1140         next_cmd = its_post_commands(its);                              \
1141         raw_spin_unlock_irqrestore(&its->lock, flags);                  \
1142                                                                         \
1143         if (its_wait_for_range_completion(its, rd_idx, next_cmd))       \
1144                 pr_err_ratelimited("ITS cmd %ps failed\n", builder);    \
1145 }
1146
1147 static void its_build_sync_cmd(struct its_node *its,
1148                                struct its_cmd_block *sync_cmd,
1149                                struct its_collection *sync_col)
1150 {
1151         its_encode_cmd(sync_cmd, GITS_CMD_SYNC);
1152         its_encode_target(sync_cmd, sync_col->target_address);
1153
1154         its_fixup_cmd(sync_cmd);
1155 }
1156
1157 static BUILD_SINGLE_CMD_FUNC(its_send_single_command, its_cmd_builder_t,
1158                              struct its_collection, its_build_sync_cmd)
1159
1160 static void its_build_vsync_cmd(struct its_node *its,
1161                                 struct its_cmd_block *sync_cmd,
1162                                 struct its_vpe *sync_vpe)
1163 {
1164         its_encode_cmd(sync_cmd, GITS_CMD_VSYNC);
1165         its_encode_vpeid(sync_cmd, sync_vpe->vpe_id);
1166
1167         its_fixup_cmd(sync_cmd);
1168 }
1169
1170 static BUILD_SINGLE_CMD_FUNC(its_send_single_vcommand, its_cmd_vbuilder_t,
1171                              struct its_vpe, its_build_vsync_cmd)
1172
1173 static void its_send_int(struct its_device *dev, u32 event_id)
1174 {
1175         struct its_cmd_desc desc;
1176
1177         desc.its_int_cmd.dev = dev;
1178         desc.its_int_cmd.event_id = event_id;
1179
1180         its_send_single_command(dev->its, its_build_int_cmd, &desc);
1181 }
1182
1183 static void its_send_clear(struct its_device *dev, u32 event_id)
1184 {
1185         struct its_cmd_desc desc;
1186
1187         desc.its_clear_cmd.dev = dev;
1188         desc.its_clear_cmd.event_id = event_id;
1189
1190         its_send_single_command(dev->its, its_build_clear_cmd, &desc);
1191 }
1192
1193 static void its_send_inv(struct its_device *dev, u32 event_id)
1194 {
1195         struct its_cmd_desc desc;
1196
1197         desc.its_inv_cmd.dev = dev;
1198         desc.its_inv_cmd.event_id = event_id;
1199
1200         its_send_single_command(dev->its, its_build_inv_cmd, &desc);
1201 }
1202
1203 static void its_send_mapd(struct its_device *dev, int valid)
1204 {
1205         struct its_cmd_desc desc;
1206
1207         desc.its_mapd_cmd.dev = dev;
1208         desc.its_mapd_cmd.valid = !!valid;
1209
1210         its_send_single_command(dev->its, its_build_mapd_cmd, &desc);
1211 }
1212
1213 static void its_send_mapc(struct its_node *its, struct its_collection *col,
1214                           int valid)
1215 {
1216         struct its_cmd_desc desc;
1217
1218         desc.its_mapc_cmd.col = col;
1219         desc.its_mapc_cmd.valid = !!valid;
1220
1221         its_send_single_command(its, its_build_mapc_cmd, &desc);
1222 }
1223
1224 static void its_send_mapti(struct its_device *dev, u32 irq_id, u32 id)
1225 {
1226         struct its_cmd_desc desc;
1227
1228         desc.its_mapti_cmd.dev = dev;
1229         desc.its_mapti_cmd.phys_id = irq_id;
1230         desc.its_mapti_cmd.event_id = id;
1231
1232         its_send_single_command(dev->its, its_build_mapti_cmd, &desc);
1233 }
1234
1235 static void its_send_movi(struct its_device *dev,
1236                           struct its_collection *col, u32 id)
1237 {
1238         struct its_cmd_desc desc;
1239
1240         desc.its_movi_cmd.dev = dev;
1241         desc.its_movi_cmd.col = col;
1242         desc.its_movi_cmd.event_id = id;
1243
1244         its_send_single_command(dev->its, its_build_movi_cmd, &desc);
1245 }
1246
1247 static void its_send_discard(struct its_device *dev, u32 id)
1248 {
1249         struct its_cmd_desc desc;
1250
1251         desc.its_discard_cmd.dev = dev;
1252         desc.its_discard_cmd.event_id = id;
1253
1254         its_send_single_command(dev->its, its_build_discard_cmd, &desc);
1255 }
1256
1257 static void its_send_invall(struct its_node *its, struct its_collection *col)
1258 {
1259         struct its_cmd_desc desc;
1260
1261         desc.its_invall_cmd.col = col;
1262
1263         its_send_single_command(its, its_build_invall_cmd, &desc);
1264 }
1265
1266 static void its_send_vmapti(struct its_device *dev, u32 id)
1267 {
1268         struct its_vlpi_map *map = dev_event_to_vlpi_map(dev, id);
1269         struct its_cmd_desc desc;
1270
1271         desc.its_vmapti_cmd.vpe = map->vpe;
1272         desc.its_vmapti_cmd.dev = dev;
1273         desc.its_vmapti_cmd.virt_id = map->vintid;
1274         desc.its_vmapti_cmd.event_id = id;
1275         desc.its_vmapti_cmd.db_enabled = map->db_enabled;
1276
1277         its_send_single_vcommand(dev->its, its_build_vmapti_cmd, &desc);
1278 }
1279
1280 static void its_send_vmovi(struct its_device *dev, u32 id)
1281 {
1282         struct its_vlpi_map *map = dev_event_to_vlpi_map(dev, id);
1283         struct its_cmd_desc desc;
1284
1285         desc.its_vmovi_cmd.vpe = map->vpe;
1286         desc.its_vmovi_cmd.dev = dev;
1287         desc.its_vmovi_cmd.event_id = id;
1288         desc.its_vmovi_cmd.db_enabled = map->db_enabled;
1289
1290         its_send_single_vcommand(dev->its, its_build_vmovi_cmd, &desc);
1291 }
1292
1293 static void its_send_vmapp(struct its_node *its,
1294                            struct its_vpe *vpe, bool valid)
1295 {
1296         struct its_cmd_desc desc;
1297
1298         desc.its_vmapp_cmd.vpe = vpe;
1299         desc.its_vmapp_cmd.valid = valid;
1300         desc.its_vmapp_cmd.col = &its->collections[vpe->col_idx];
1301
1302         its_send_single_vcommand(its, its_build_vmapp_cmd, &desc);
1303 }
1304
1305 static void its_send_vmovp(struct its_vpe *vpe)
1306 {
1307         struct its_cmd_desc desc = {};
1308         struct its_node *its;
1309         unsigned long flags;
1310         int col_id = vpe->col_idx;
1311
1312         desc.its_vmovp_cmd.vpe = vpe;
1313
1314         if (!its_list_map) {
1315                 its = list_first_entry(&its_nodes, struct its_node, entry);
1316                 desc.its_vmovp_cmd.col = &its->collections[col_id];
1317                 its_send_single_vcommand(its, its_build_vmovp_cmd, &desc);
1318                 return;
1319         }
1320
1321         /*
1322          * Yet another marvel of the architecture. If using the
1323          * its_list "feature", we need to make sure that all ITSs
1324          * receive all VMOVP commands in the same order. The only way
1325          * to guarantee this is to make vmovp a serialization point.
1326          *
1327          * Wall <-- Head.
1328          */
1329         raw_spin_lock_irqsave(&vmovp_lock, flags);
1330
1331         desc.its_vmovp_cmd.seq_num = vmovp_seq_num++;
1332         desc.its_vmovp_cmd.its_list = get_its_list(vpe->its_vm);
1333
1334         /* Emit VMOVPs */
1335         list_for_each_entry(its, &its_nodes, entry) {
1336                 if (!is_v4(its))
1337                         continue;
1338
1339                 if (!require_its_list_vmovp(vpe->its_vm, its))
1340                         continue;
1341
1342                 desc.its_vmovp_cmd.col = &its->collections[col_id];
1343                 its_send_single_vcommand(its, its_build_vmovp_cmd, &desc);
1344         }
1345
1346         raw_spin_unlock_irqrestore(&vmovp_lock, flags);
1347 }
1348
1349 static void its_send_vinvall(struct its_node *its, struct its_vpe *vpe)
1350 {
1351         struct its_cmd_desc desc;
1352
1353         desc.its_vinvall_cmd.vpe = vpe;
1354         its_send_single_vcommand(its, its_build_vinvall_cmd, &desc);
1355 }
1356
1357 static void its_send_vinv(struct its_device *dev, u32 event_id)
1358 {
1359         struct its_cmd_desc desc;
1360
1361         /*
1362          * There is no real VINV command. This is just a normal INV,
1363          * with a VSYNC instead of a SYNC.
1364          */
1365         desc.its_inv_cmd.dev = dev;
1366         desc.its_inv_cmd.event_id = event_id;
1367
1368         its_send_single_vcommand(dev->its, its_build_vinv_cmd, &desc);
1369 }
1370
1371 static void its_send_vint(struct its_device *dev, u32 event_id)
1372 {
1373         struct its_cmd_desc desc;
1374
1375         /*
1376          * There is no real VINT command. This is just a normal INT,
1377          * with a VSYNC instead of a SYNC.
1378          */
1379         desc.its_int_cmd.dev = dev;
1380         desc.its_int_cmd.event_id = event_id;
1381
1382         its_send_single_vcommand(dev->its, its_build_vint_cmd, &desc);
1383 }
1384
1385 static void its_send_vclear(struct its_device *dev, u32 event_id)
1386 {
1387         struct its_cmd_desc desc;
1388
1389         /*
1390          * There is no real VCLEAR command. This is just a normal CLEAR,
1391          * with a VSYNC instead of a SYNC.
1392          */
1393         desc.its_clear_cmd.dev = dev;
1394         desc.its_clear_cmd.event_id = event_id;
1395
1396         its_send_single_vcommand(dev->its, its_build_vclear_cmd, &desc);
1397 }
1398
1399 static void its_send_invdb(struct its_node *its, struct its_vpe *vpe)
1400 {
1401         struct its_cmd_desc desc;
1402
1403         desc.its_invdb_cmd.vpe = vpe;
1404         its_send_single_vcommand(its, its_build_invdb_cmd, &desc);
1405 }
1406
1407 /*
1408  * irqchip functions - assumes MSI, mostly.
1409  */
1410 static void lpi_write_config(struct irq_data *d, u8 clr, u8 set)
1411 {
1412         struct its_vlpi_map *map = get_vlpi_map(d);
1413         irq_hw_number_t hwirq;
1414         void *va;
1415         u8 *cfg;
1416
1417         if (map) {
1418                 va = page_address(map->vm->vprop_page);
1419                 hwirq = map->vintid;
1420
1421                 /* Remember the updated property */
1422                 map->properties &= ~clr;
1423                 map->properties |= set | LPI_PROP_GROUP1;
1424         } else {
1425                 va = gic_rdists->prop_table_va;
1426                 hwirq = d->hwirq;
1427         }
1428
1429         cfg = va + hwirq - 8192;
1430         *cfg &= ~clr;
1431         *cfg |= set | LPI_PROP_GROUP1;
1432
1433         /*
1434          * Make the above write visible to the redistributors.
1435          * And yes, we're flushing exactly: One. Single. Byte.
1436          * Humpf...
1437          */
1438         if (gic_rdists->flags & RDIST_FLAGS_PROPBASE_NEEDS_FLUSHING)
1439                 gic_flush_dcache_to_poc(cfg, sizeof(*cfg));
1440         else
1441                 dsb(ishst);
1442 }
1443
1444 static void wait_for_syncr(void __iomem *rdbase)
1445 {
1446         while (readl_relaxed(rdbase + GICR_SYNCR) & 1)
1447                 cpu_relax();
1448 }
1449
1450 static void __direct_lpi_inv(struct irq_data *d, u64 val)
1451 {
1452         void __iomem *rdbase;
1453         unsigned long flags;
1454         int cpu;
1455
1456         /* Target the redistributor this LPI is currently routed to */
1457         cpu = irq_to_cpuid_lock(d, &flags);
1458         raw_spin_lock(&gic_data_rdist_cpu(cpu)->rd_lock);
1459
1460         rdbase = per_cpu_ptr(gic_rdists->rdist, cpu)->rd_base;
1461         gic_write_lpir(val, rdbase + GICR_INVLPIR);
1462         wait_for_syncr(rdbase);
1463
1464         raw_spin_unlock(&gic_data_rdist_cpu(cpu)->rd_lock);
1465         irq_to_cpuid_unlock(d, flags);
1466 }
1467
1468 static void direct_lpi_inv(struct irq_data *d)
1469 {
1470         struct its_vlpi_map *map = get_vlpi_map(d);
1471         u64 val;
1472
1473         if (map) {
1474                 struct its_device *its_dev = irq_data_get_irq_chip_data(d);
1475
1476                 WARN_ON(!is_v4_1(its_dev->its));
1477
1478                 val  = GICR_INVLPIR_V;
1479                 val |= FIELD_PREP(GICR_INVLPIR_VPEID, map->vpe->vpe_id);
1480                 val |= FIELD_PREP(GICR_INVLPIR_INTID, map->vintid);
1481         } else {
1482                 val = d->hwirq;
1483         }
1484
1485         __direct_lpi_inv(d, val);
1486 }
1487
1488 static void lpi_update_config(struct irq_data *d, u8 clr, u8 set)
1489 {
1490         struct its_device *its_dev = irq_data_get_irq_chip_data(d);
1491
1492         lpi_write_config(d, clr, set);
1493         if (gic_rdists->has_direct_lpi &&
1494             (is_v4_1(its_dev->its) || !irqd_is_forwarded_to_vcpu(d)))
1495                 direct_lpi_inv(d);
1496         else if (!irqd_is_forwarded_to_vcpu(d))
1497                 its_send_inv(its_dev, its_get_event_id(d));
1498         else
1499                 its_send_vinv(its_dev, its_get_event_id(d));
1500 }
1501
1502 static void its_vlpi_set_doorbell(struct irq_data *d, bool enable)
1503 {
1504         struct its_device *its_dev = irq_data_get_irq_chip_data(d);
1505         u32 event = its_get_event_id(d);
1506         struct its_vlpi_map *map;
1507
1508         /*
1509          * GICv4.1 does away with the per-LPI nonsense, nothing to do
1510          * here.
1511          */
1512         if (is_v4_1(its_dev->its))
1513                 return;
1514
1515         map = dev_event_to_vlpi_map(its_dev, event);
1516
1517         if (map->db_enabled == enable)
1518                 return;
1519
1520         map->db_enabled = enable;
1521
1522         /*
1523          * More fun with the architecture:
1524          *
1525          * Ideally, we'd issue a VMAPTI to set the doorbell to its LPI
1526          * value or to 1023, depending on the enable bit. But that
1527          * would be issuing a mapping for an /existing/ DevID+EventID
1528          * pair, which is UNPREDICTABLE. Instead, let's issue a VMOVI
1529          * to the /same/ vPE, using this opportunity to adjust the
1530          * doorbell. Mouahahahaha. We loves it, Precious.
1531          */
1532         its_send_vmovi(its_dev, event);
1533 }
1534
1535 static void its_mask_irq(struct irq_data *d)
1536 {
1537         if (irqd_is_forwarded_to_vcpu(d))
1538                 its_vlpi_set_doorbell(d, false);
1539
1540         lpi_update_config(d, LPI_PROP_ENABLED, 0);
1541 }
1542
1543 static void its_unmask_irq(struct irq_data *d)
1544 {
1545         if (irqd_is_forwarded_to_vcpu(d))
1546                 its_vlpi_set_doorbell(d, true);
1547
1548         lpi_update_config(d, 0, LPI_PROP_ENABLED);
1549 }
1550
1551 static __maybe_unused u32 its_read_lpi_count(struct irq_data *d, int cpu)
1552 {
1553         if (irqd_affinity_is_managed(d))
1554                 return atomic_read(&per_cpu_ptr(&cpu_lpi_count, cpu)->managed);
1555
1556         return atomic_read(&per_cpu_ptr(&cpu_lpi_count, cpu)->unmanaged);
1557 }
1558
1559 static void its_inc_lpi_count(struct irq_data *d, int cpu)
1560 {
1561         if (irqd_affinity_is_managed(d))
1562                 atomic_inc(&per_cpu_ptr(&cpu_lpi_count, cpu)->managed);
1563         else
1564                 atomic_inc(&per_cpu_ptr(&cpu_lpi_count, cpu)->unmanaged);
1565 }
1566
1567 static void its_dec_lpi_count(struct irq_data *d, int cpu)
1568 {
1569         if (irqd_affinity_is_managed(d))
1570                 atomic_dec(&per_cpu_ptr(&cpu_lpi_count, cpu)->managed);
1571         else
1572                 atomic_dec(&per_cpu_ptr(&cpu_lpi_count, cpu)->unmanaged);
1573 }
1574
1575 static unsigned int cpumask_pick_least_loaded(struct irq_data *d,
1576                                               const struct cpumask *cpu_mask)
1577 {
1578         unsigned int cpu = nr_cpu_ids, tmp;
1579         int count = S32_MAX;
1580
1581         for_each_cpu(tmp, cpu_mask) {
1582                 int this_count = its_read_lpi_count(d, tmp);
1583                 if (this_count < count) {
1584                         cpu = tmp;
1585                         count = this_count;
1586                 }
1587         }
1588
1589         return cpu;
1590 }
1591
1592 /*
1593  * As suggested by Thomas Gleixner in:
1594  * https://lore.kernel.org/r/87h80q2aoc.fsf@nanos.tec.linutronix.de
1595  */
1596 static int its_select_cpu(struct irq_data *d,
1597                           const struct cpumask *aff_mask)
1598 {
1599         struct its_device *its_dev = irq_data_get_irq_chip_data(d);
1600         static DEFINE_RAW_SPINLOCK(tmpmask_lock);
1601         static struct cpumask __tmpmask;
1602         struct cpumask *tmpmask;
1603         unsigned long flags;
1604         int cpu, node;
1605         node = its_dev->its->numa_node;
1606         tmpmask = &__tmpmask;
1607
1608         raw_spin_lock_irqsave(&tmpmask_lock, flags);
1609
1610         if (!irqd_affinity_is_managed(d)) {
1611                 /* First try the NUMA node */
1612                 if (node != NUMA_NO_NODE) {
1613                         /*
1614                          * Try the intersection of the affinity mask and the
1615                          * node mask (and the online mask, just to be safe).
1616                          */
1617                         cpumask_and(tmpmask, cpumask_of_node(node), aff_mask);
1618                         cpumask_and(tmpmask, tmpmask, cpu_online_mask);
1619
1620                         /*
1621                          * Ideally, we would check if the mask is empty, and
1622                          * try again on the full node here.
1623                          *
1624                          * But it turns out that the way ACPI describes the
1625                          * affinity for ITSs only deals about memory, and
1626                          * not target CPUs, so it cannot describe a single
1627                          * ITS placed next to two NUMA nodes.
1628                          *
1629                          * Instead, just fallback on the online mask. This
1630                          * diverges from Thomas' suggestion above.
1631                          */
1632                         cpu = cpumask_pick_least_loaded(d, tmpmask);
1633                         if (cpu < nr_cpu_ids)
1634                                 goto out;
1635
1636                         /* If we can't cross sockets, give up */
1637                         if ((its_dev->its->flags & ITS_FLAGS_WORKAROUND_CAVIUM_23144))
1638                                 goto out;
1639
1640                         /* If the above failed, expand the search */
1641                 }
1642
1643                 /* Try the intersection of the affinity and online masks */
1644                 cpumask_and(tmpmask, aff_mask, cpu_online_mask);
1645
1646                 /* If that doesn't fly, the online mask is the last resort */
1647                 if (cpumask_empty(tmpmask))
1648                         cpumask_copy(tmpmask, cpu_online_mask);
1649
1650                 cpu = cpumask_pick_least_loaded(d, tmpmask);
1651         } else {
1652                 cpumask_copy(tmpmask, aff_mask);
1653
1654                 /* If we cannot cross sockets, limit the search to that node */
1655                 if ((its_dev->its->flags & ITS_FLAGS_WORKAROUND_CAVIUM_23144) &&
1656                     node != NUMA_NO_NODE)
1657                         cpumask_and(tmpmask, tmpmask, cpumask_of_node(node));
1658
1659                 cpu = cpumask_pick_least_loaded(d, tmpmask);
1660         }
1661 out:
1662         raw_spin_unlock_irqrestore(&tmpmask_lock, flags);
1663
1664         pr_debug("IRQ%d -> %*pbl CPU%d\n", d->irq, cpumask_pr_args(aff_mask), cpu);
1665         return cpu;
1666 }
1667
1668 static int its_set_affinity(struct irq_data *d, const struct cpumask *mask_val,
1669                             bool force)
1670 {
1671         struct its_device *its_dev = irq_data_get_irq_chip_data(d);
1672         struct its_collection *target_col;
1673         u32 id = its_get_event_id(d);
1674         int cpu, prev_cpu;
1675
1676         /* A forwarded interrupt should use irq_set_vcpu_affinity */
1677         if (irqd_is_forwarded_to_vcpu(d))
1678                 return -EINVAL;
1679
1680         prev_cpu = its_dev->event_map.col_map[id];
1681         its_dec_lpi_count(d, prev_cpu);
1682
1683         if (!force)
1684                 cpu = its_select_cpu(d, mask_val);
1685         else
1686                 cpu = cpumask_pick_least_loaded(d, mask_val);
1687
1688         if (cpu < 0 || cpu >= nr_cpu_ids)
1689                 goto err;
1690
1691         /* don't set the affinity when the target cpu is same as current one */
1692         if (cpu != prev_cpu) {
1693                 target_col = &its_dev->its->collections[cpu];
1694                 its_send_movi(its_dev, target_col, id);
1695                 its_dev->event_map.col_map[id] = cpu;
1696                 irq_data_update_effective_affinity(d, cpumask_of(cpu));
1697         }
1698
1699         its_inc_lpi_count(d, cpu);
1700
1701         return IRQ_SET_MASK_OK_DONE;
1702
1703 err:
1704         its_inc_lpi_count(d, prev_cpu);
1705         return -EINVAL;
1706 }
1707
1708 static u64 its_irq_get_msi_base(struct its_device *its_dev)
1709 {
1710         struct its_node *its = its_dev->its;
1711
1712         return its->phys_base + GITS_TRANSLATER;
1713 }
1714
1715 static void its_irq_compose_msi_msg(struct irq_data *d, struct msi_msg *msg)
1716 {
1717         struct its_device *its_dev = irq_data_get_irq_chip_data(d);
1718         struct its_node *its;
1719         u64 addr;
1720
1721         its = its_dev->its;
1722         addr = its->get_msi_base(its_dev);
1723
1724         msg->address_lo         = lower_32_bits(addr);
1725         msg->address_hi         = upper_32_bits(addr);
1726         msg->data               = its_get_event_id(d);
1727
1728         iommu_dma_compose_msi_msg(irq_data_get_msi_desc(d), msg);
1729 }
1730
1731 static int its_irq_set_irqchip_state(struct irq_data *d,
1732                                      enum irqchip_irq_state which,
1733                                      bool state)
1734 {
1735         struct its_device *its_dev = irq_data_get_irq_chip_data(d);
1736         u32 event = its_get_event_id(d);
1737
1738         if (which != IRQCHIP_STATE_PENDING)
1739                 return -EINVAL;
1740
1741         if (irqd_is_forwarded_to_vcpu(d)) {
1742                 if (state)
1743                         its_send_vint(its_dev, event);
1744                 else
1745                         its_send_vclear(its_dev, event);
1746         } else {
1747                 if (state)
1748                         its_send_int(its_dev, event);
1749                 else
1750                         its_send_clear(its_dev, event);
1751         }
1752
1753         return 0;
1754 }
1755
1756 static int its_irq_retrigger(struct irq_data *d)
1757 {
1758         return !its_irq_set_irqchip_state(d, IRQCHIP_STATE_PENDING, true);
1759 }
1760
1761 /*
1762  * Two favourable cases:
1763  *
1764  * (a) Either we have a GICv4.1, and all vPEs have to be mapped at all times
1765  *     for vSGI delivery
1766  *
1767  * (b) Or the ITSs do not use a list map, meaning that VMOVP is cheap enough
1768  *     and we're better off mapping all VPEs always
1769  *
1770  * If neither (a) nor (b) is true, then we map vPEs on demand.
1771  *
1772  */
1773 static bool gic_requires_eager_mapping(void)
1774 {
1775         if (!its_list_map || gic_rdists->has_rvpeid)
1776                 return true;
1777
1778         return false;
1779 }
1780
1781 static void its_map_vm(struct its_node *its, struct its_vm *vm)
1782 {
1783         unsigned long flags;
1784
1785         if (gic_requires_eager_mapping())
1786                 return;
1787
1788         raw_spin_lock_irqsave(&vmovp_lock, flags);
1789
1790         /*
1791          * If the VM wasn't mapped yet, iterate over the vpes and get
1792          * them mapped now.
1793          */
1794         vm->vlpi_count[its->list_nr]++;
1795
1796         if (vm->vlpi_count[its->list_nr] == 1) {
1797                 int i;
1798
1799                 for (i = 0; i < vm->nr_vpes; i++) {
1800                         struct its_vpe *vpe = vm->vpes[i];
1801                         struct irq_data *d = irq_get_irq_data(vpe->irq);
1802
1803                         /* Map the VPE to the first possible CPU */
1804                         vpe->col_idx = cpumask_first(cpu_online_mask);
1805                         its_send_vmapp(its, vpe, true);
1806                         its_send_vinvall(its, vpe);
1807                         irq_data_update_effective_affinity(d, cpumask_of(vpe->col_idx));
1808                 }
1809         }
1810
1811         raw_spin_unlock_irqrestore(&vmovp_lock, flags);
1812 }
1813
1814 static void its_unmap_vm(struct its_node *its, struct its_vm *vm)
1815 {
1816         unsigned long flags;
1817
1818         /* Not using the ITS list? Everything is always mapped. */
1819         if (gic_requires_eager_mapping())
1820                 return;
1821
1822         raw_spin_lock_irqsave(&vmovp_lock, flags);
1823
1824         if (!--vm->vlpi_count[its->list_nr]) {
1825                 int i;
1826
1827                 for (i = 0; i < vm->nr_vpes; i++)
1828                         its_send_vmapp(its, vm->vpes[i], false);
1829         }
1830
1831         raw_spin_unlock_irqrestore(&vmovp_lock, flags);
1832 }
1833
1834 static int its_vlpi_map(struct irq_data *d, struct its_cmd_info *info)
1835 {
1836         struct its_device *its_dev = irq_data_get_irq_chip_data(d);
1837         u32 event = its_get_event_id(d);
1838         int ret = 0;
1839
1840         if (!info->map)
1841                 return -EINVAL;
1842
1843         raw_spin_lock(&its_dev->event_map.vlpi_lock);
1844
1845         if (!its_dev->event_map.vm) {
1846                 struct its_vlpi_map *maps;
1847
1848                 maps = kcalloc(its_dev->event_map.nr_lpis, sizeof(*maps),
1849                                GFP_ATOMIC);
1850                 if (!maps) {
1851                         ret = -ENOMEM;
1852                         goto out;
1853                 }
1854
1855                 its_dev->event_map.vm = info->map->vm;
1856                 its_dev->event_map.vlpi_maps = maps;
1857         } else if (its_dev->event_map.vm != info->map->vm) {
1858                 ret = -EINVAL;
1859                 goto out;
1860         }
1861
1862         /* Get our private copy of the mapping information */
1863         its_dev->event_map.vlpi_maps[event] = *info->map;
1864
1865         if (irqd_is_forwarded_to_vcpu(d)) {
1866                 /* Already mapped, move it around */
1867                 its_send_vmovi(its_dev, event);
1868         } else {
1869                 /* Ensure all the VPEs are mapped on this ITS */
1870                 its_map_vm(its_dev->its, info->map->vm);
1871
1872                 /*
1873                  * Flag the interrupt as forwarded so that we can
1874                  * start poking the virtual property table.
1875                  */
1876                 irqd_set_forwarded_to_vcpu(d);
1877
1878                 /* Write out the property to the prop table */
1879                 lpi_write_config(d, 0xff, info->map->properties);
1880
1881                 /* Drop the physical mapping */
1882                 its_send_discard(its_dev, event);
1883
1884                 /* and install the virtual one */
1885                 its_send_vmapti(its_dev, event);
1886
1887                 /* Increment the number of VLPIs */
1888                 its_dev->event_map.nr_vlpis++;
1889         }
1890
1891 out:
1892         raw_spin_unlock(&its_dev->event_map.vlpi_lock);
1893         return ret;
1894 }
1895
1896 static int its_vlpi_get(struct irq_data *d, struct its_cmd_info *info)
1897 {
1898         struct its_device *its_dev = irq_data_get_irq_chip_data(d);
1899         struct its_vlpi_map *map;
1900         int ret = 0;
1901
1902         raw_spin_lock(&its_dev->event_map.vlpi_lock);
1903
1904         map = get_vlpi_map(d);
1905
1906         if (!its_dev->event_map.vm || !map) {
1907                 ret = -EINVAL;
1908                 goto out;
1909         }
1910
1911         /* Copy our mapping information to the incoming request */
1912         *info->map = *map;
1913
1914 out:
1915         raw_spin_unlock(&its_dev->event_map.vlpi_lock);
1916         return ret;
1917 }
1918
1919 static int its_vlpi_unmap(struct irq_data *d)
1920 {
1921         struct its_device *its_dev = irq_data_get_irq_chip_data(d);
1922         u32 event = its_get_event_id(d);
1923         int ret = 0;
1924
1925         raw_spin_lock(&its_dev->event_map.vlpi_lock);
1926
1927         if (!its_dev->event_map.vm || !irqd_is_forwarded_to_vcpu(d)) {
1928                 ret = -EINVAL;
1929                 goto out;
1930         }
1931
1932         /* Drop the virtual mapping */
1933         its_send_discard(its_dev, event);
1934
1935         /* and restore the physical one */
1936         irqd_clr_forwarded_to_vcpu(d);
1937         its_send_mapti(its_dev, d->hwirq, event);
1938         lpi_update_config(d, 0xff, (LPI_PROP_DEFAULT_PRIO |
1939                                     LPI_PROP_ENABLED |
1940                                     LPI_PROP_GROUP1));
1941
1942         /* Potentially unmap the VM from this ITS */
1943         its_unmap_vm(its_dev->its, its_dev->event_map.vm);
1944
1945         /*
1946          * Drop the refcount and make the device available again if
1947          * this was the last VLPI.
1948          */
1949         if (!--its_dev->event_map.nr_vlpis) {
1950                 its_dev->event_map.vm = NULL;
1951                 kfree(its_dev->event_map.vlpi_maps);
1952         }
1953
1954 out:
1955         raw_spin_unlock(&its_dev->event_map.vlpi_lock);
1956         return ret;
1957 }
1958
1959 static int its_vlpi_prop_update(struct irq_data *d, struct its_cmd_info *info)
1960 {
1961         struct its_device *its_dev = irq_data_get_irq_chip_data(d);
1962
1963         if (!its_dev->event_map.vm || !irqd_is_forwarded_to_vcpu(d))
1964                 return -EINVAL;
1965
1966         if (info->cmd_type == PROP_UPDATE_AND_INV_VLPI)
1967                 lpi_update_config(d, 0xff, info->config);
1968         else
1969                 lpi_write_config(d, 0xff, info->config);
1970         its_vlpi_set_doorbell(d, !!(info->config & LPI_PROP_ENABLED));
1971
1972         return 0;
1973 }
1974
1975 static int its_irq_set_vcpu_affinity(struct irq_data *d, void *vcpu_info)
1976 {
1977         struct its_device *its_dev = irq_data_get_irq_chip_data(d);
1978         struct its_cmd_info *info = vcpu_info;
1979
1980         /* Need a v4 ITS */
1981         if (!is_v4(its_dev->its))
1982                 return -EINVAL;
1983
1984         /* Unmap request? */
1985         if (!info)
1986                 return its_vlpi_unmap(d);
1987
1988         switch (info->cmd_type) {
1989         case MAP_VLPI:
1990                 return its_vlpi_map(d, info);
1991
1992         case GET_VLPI:
1993                 return its_vlpi_get(d, info);
1994
1995         case PROP_UPDATE_VLPI:
1996         case PROP_UPDATE_AND_INV_VLPI:
1997                 return its_vlpi_prop_update(d, info);
1998
1999         default:
2000                 return -EINVAL;
2001         }
2002 }
2003
2004 static struct irq_chip its_irq_chip = {
2005         .name                   = "ITS",
2006         .irq_mask               = its_mask_irq,
2007         .irq_unmask             = its_unmask_irq,
2008         .irq_eoi                = irq_chip_eoi_parent,
2009         .irq_set_affinity       = its_set_affinity,
2010         .irq_compose_msi_msg    = its_irq_compose_msi_msg,
2011         .irq_set_irqchip_state  = its_irq_set_irqchip_state,
2012         .irq_retrigger          = its_irq_retrigger,
2013         .irq_set_vcpu_affinity  = its_irq_set_vcpu_affinity,
2014 };
2015
2016
2017 /*
2018  * How we allocate LPIs:
2019  *
2020  * lpi_range_list contains ranges of LPIs that are to available to
2021  * allocate from. To allocate LPIs, just pick the first range that
2022  * fits the required allocation, and reduce it by the required
2023  * amount. Once empty, remove the range from the list.
2024  *
2025  * To free a range of LPIs, add a free range to the list, sort it and
2026  * merge the result if the new range happens to be adjacent to an
2027  * already free block.
2028  *
2029  * The consequence of the above is that allocation is cost is low, but
2030  * freeing is expensive. We assumes that freeing rarely occurs.
2031  */
2032 #define ITS_MAX_LPI_NRBITS      16 /* 64K LPIs */
2033
2034 static DEFINE_MUTEX(lpi_range_lock);
2035 static LIST_HEAD(lpi_range_list);
2036
2037 struct lpi_range {
2038         struct list_head        entry;
2039         u32                     base_id;
2040         u32                     span;
2041 };
2042
2043 static struct lpi_range *mk_lpi_range(u32 base, u32 span)
2044 {
2045         struct lpi_range *range;
2046
2047         range = kmalloc(sizeof(*range), GFP_KERNEL);
2048         if (range) {
2049                 range->base_id = base;
2050                 range->span = span;
2051         }
2052
2053         return range;
2054 }
2055
2056 static int alloc_lpi_range(u32 nr_lpis, u32 *base)
2057 {
2058         struct lpi_range *range, *tmp;
2059         int err = -ENOSPC;
2060
2061         mutex_lock(&lpi_range_lock);
2062
2063         list_for_each_entry_safe(range, tmp, &lpi_range_list, entry) {
2064                 if (range->span >= nr_lpis) {
2065                         *base = range->base_id;
2066                         range->base_id += nr_lpis;
2067                         range->span -= nr_lpis;
2068
2069                         if (range->span == 0) {
2070                                 list_del(&range->entry);
2071                                 kfree(range);
2072                         }
2073
2074                         err = 0;
2075                         break;
2076                 }
2077         }
2078
2079         mutex_unlock(&lpi_range_lock);
2080
2081         pr_debug("ITS: alloc %u:%u\n", *base, nr_lpis);
2082         return err;
2083 }
2084
2085 static void merge_lpi_ranges(struct lpi_range *a, struct lpi_range *b)
2086 {
2087         if (&a->entry == &lpi_range_list || &b->entry == &lpi_range_list)
2088                 return;
2089         if (a->base_id + a->span != b->base_id)
2090                 return;
2091         b->base_id = a->base_id;
2092         b->span += a->span;
2093         list_del(&a->entry);
2094         kfree(a);
2095 }
2096
2097 static int free_lpi_range(u32 base, u32 nr_lpis)
2098 {
2099         struct lpi_range *new, *old;
2100
2101         new = mk_lpi_range(base, nr_lpis);
2102         if (!new)
2103                 return -ENOMEM;
2104
2105         mutex_lock(&lpi_range_lock);
2106
2107         list_for_each_entry_reverse(old, &lpi_range_list, entry) {
2108                 if (old->base_id < base)
2109                         break;
2110         }
2111         /*
2112          * old is the last element with ->base_id smaller than base,
2113          * so new goes right after it. If there are no elements with
2114          * ->base_id smaller than base, &old->entry ends up pointing
2115          * at the head of the list, and inserting new it the start of
2116          * the list is the right thing to do in that case as well.
2117          */
2118         list_add(&new->entry, &old->entry);
2119         /*
2120          * Now check if we can merge with the preceding and/or
2121          * following ranges.
2122          */
2123         merge_lpi_ranges(old, new);
2124         merge_lpi_ranges(new, list_next_entry(new, entry));
2125
2126         mutex_unlock(&lpi_range_lock);
2127         return 0;
2128 }
2129
2130 static int __init its_lpi_init(u32 id_bits)
2131 {
2132         u32 lpis = (1UL << id_bits) - 8192;
2133         u32 numlpis;
2134         int err;
2135
2136         numlpis = 1UL << GICD_TYPER_NUM_LPIS(gic_rdists->gicd_typer);
2137
2138         if (numlpis > 2 && !WARN_ON(numlpis > lpis)) {
2139                 lpis = numlpis;
2140                 pr_info("ITS: Using hypervisor restricted LPI range [%u]\n",
2141                         lpis);
2142         }
2143
2144         /*
2145          * Initializing the allocator is just the same as freeing the
2146          * full range of LPIs.
2147          */
2148         err = free_lpi_range(8192, lpis);
2149         pr_debug("ITS: Allocator initialized for %u LPIs\n", lpis);
2150         return err;
2151 }
2152
2153 static unsigned long *its_lpi_alloc(int nr_irqs, u32 *base, int *nr_ids)
2154 {
2155         unsigned long *bitmap = NULL;
2156         int err = 0;
2157
2158         do {
2159                 err = alloc_lpi_range(nr_irqs, base);
2160                 if (!err)
2161                         break;
2162
2163                 nr_irqs /= 2;
2164         } while (nr_irqs > 0);
2165
2166         if (!nr_irqs)
2167                 err = -ENOSPC;
2168
2169         if (err)
2170                 goto out;
2171
2172         bitmap = bitmap_zalloc(nr_irqs, GFP_ATOMIC);
2173         if (!bitmap)
2174                 goto out;
2175
2176         *nr_ids = nr_irqs;
2177
2178 out:
2179         if (!bitmap)
2180                 *base = *nr_ids = 0;
2181
2182         return bitmap;
2183 }
2184
2185 static void its_lpi_free(unsigned long *bitmap, u32 base, u32 nr_ids)
2186 {
2187         WARN_ON(free_lpi_range(base, nr_ids));
2188         bitmap_free(bitmap);
2189 }
2190
2191 static void gic_reset_prop_table(void *va)
2192 {
2193         /* Priority 0xa0, Group-1, disabled */
2194         memset(va, LPI_PROP_DEFAULT_PRIO | LPI_PROP_GROUP1, LPI_PROPBASE_SZ);
2195
2196         /* Make sure the GIC will observe the written configuration */
2197         gic_flush_dcache_to_poc(va, LPI_PROPBASE_SZ);
2198 }
2199
2200 static struct page *its_allocate_prop_table(gfp_t gfp_flags)
2201 {
2202         struct page *prop_page;
2203
2204         prop_page = alloc_pages(gfp_flags, get_order(LPI_PROPBASE_SZ));
2205         if (!prop_page)
2206                 return NULL;
2207
2208         gic_reset_prop_table(page_address(prop_page));
2209
2210         return prop_page;
2211 }
2212
2213 static void its_free_prop_table(struct page *prop_page)
2214 {
2215         free_pages((unsigned long)page_address(prop_page),
2216                    get_order(LPI_PROPBASE_SZ));
2217 }
2218
2219 static bool gic_check_reserved_range(phys_addr_t addr, unsigned long size)
2220 {
2221         phys_addr_t start, end, addr_end;
2222         u64 i;
2223
2224         /*
2225          * We don't bother checking for a kdump kernel as by
2226          * construction, the LPI tables are out of this kernel's
2227          * memory map.
2228          */
2229         if (is_kdump_kernel())
2230                 return true;
2231
2232         addr_end = addr + size - 1;
2233
2234         for_each_reserved_mem_range(i, &start, &end) {
2235                 if (addr >= start && addr_end <= end)
2236                         return true;
2237         }
2238
2239         /* Not found, not a good sign... */
2240         pr_warn("GICv3: Expected reserved range [%pa:%pa], not found\n",
2241                 &addr, &addr_end);
2242         add_taint(TAINT_CRAP, LOCKDEP_STILL_OK);
2243         return false;
2244 }
2245
2246 static int gic_reserve_range(phys_addr_t addr, unsigned long size)
2247 {
2248         if (efi_enabled(EFI_CONFIG_TABLES))
2249                 return efi_mem_reserve_persistent(addr, size);
2250
2251         return 0;
2252 }
2253
2254 static int __init its_setup_lpi_prop_table(void)
2255 {
2256         if (gic_rdists->flags & RDIST_FLAGS_RD_TABLES_PREALLOCATED) {
2257                 u64 val;
2258
2259                 val = gicr_read_propbaser(gic_data_rdist_rd_base() + GICR_PROPBASER);
2260                 lpi_id_bits = (val & GICR_PROPBASER_IDBITS_MASK) + 1;
2261
2262                 gic_rdists->prop_table_pa = val & GENMASK_ULL(51, 12);
2263                 gic_rdists->prop_table_va = memremap(gic_rdists->prop_table_pa,
2264                                                      LPI_PROPBASE_SZ,
2265                                                      MEMREMAP_WB);
2266                 gic_reset_prop_table(gic_rdists->prop_table_va);
2267         } else {
2268                 struct page *page;
2269
2270                 lpi_id_bits = min_t(u32,
2271                                     GICD_TYPER_ID_BITS(gic_rdists->gicd_typer),
2272                                     ITS_MAX_LPI_NRBITS);
2273                 page = its_allocate_prop_table(GFP_NOWAIT);
2274                 if (!page) {
2275                         pr_err("Failed to allocate PROPBASE\n");
2276                         return -ENOMEM;
2277                 }
2278
2279                 gic_rdists->prop_table_pa = page_to_phys(page);
2280                 gic_rdists->prop_table_va = page_address(page);
2281                 WARN_ON(gic_reserve_range(gic_rdists->prop_table_pa,
2282                                           LPI_PROPBASE_SZ));
2283         }
2284
2285         pr_info("GICv3: using LPI property table @%pa\n",
2286                 &gic_rdists->prop_table_pa);
2287
2288         return its_lpi_init(lpi_id_bits);
2289 }
2290
2291 static const char *its_base_type_string[] = {
2292         [GITS_BASER_TYPE_DEVICE]        = "Devices",
2293         [GITS_BASER_TYPE_VCPU]          = "Virtual CPUs",
2294         [GITS_BASER_TYPE_RESERVED3]     = "Reserved (3)",
2295         [GITS_BASER_TYPE_COLLECTION]    = "Interrupt Collections",
2296         [GITS_BASER_TYPE_RESERVED5]     = "Reserved (5)",
2297         [GITS_BASER_TYPE_RESERVED6]     = "Reserved (6)",
2298         [GITS_BASER_TYPE_RESERVED7]     = "Reserved (7)",
2299 };
2300
2301 static u64 its_read_baser(struct its_node *its, struct its_baser *baser)
2302 {
2303         u32 idx = baser - its->tables;
2304
2305         return gits_read_baser(its->base + GITS_BASER + (idx << 3));
2306 }
2307
2308 static void its_write_baser(struct its_node *its, struct its_baser *baser,
2309                             u64 val)
2310 {
2311         u32 idx = baser - its->tables;
2312
2313         gits_write_baser(val, its->base + GITS_BASER + (idx << 3));
2314         baser->val = its_read_baser(its, baser);
2315 }
2316
2317 static int its_setup_baser(struct its_node *its, struct its_baser *baser,
2318                            u64 cache, u64 shr, u32 order, bool indirect)
2319 {
2320         u64 val = its_read_baser(its, baser);
2321         u64 esz = GITS_BASER_ENTRY_SIZE(val);
2322         u64 type = GITS_BASER_TYPE(val);
2323         u64 baser_phys, tmp;
2324         u32 alloc_pages, psz;
2325         struct page *page;
2326         void *base;
2327
2328         psz = baser->psz;
2329         alloc_pages = (PAGE_ORDER_TO_SIZE(order) / psz);
2330         if (alloc_pages > GITS_BASER_PAGES_MAX) {
2331                 pr_warn("ITS@%pa: %s too large, reduce ITS pages %u->%u\n",
2332                         &its->phys_base, its_base_type_string[type],
2333                         alloc_pages, GITS_BASER_PAGES_MAX);
2334                 alloc_pages = GITS_BASER_PAGES_MAX;
2335                 order = get_order(GITS_BASER_PAGES_MAX * psz);
2336         }
2337
2338         page = alloc_pages_node(its->numa_node, GFP_KERNEL | __GFP_ZERO, order);
2339         if (!page)
2340                 return -ENOMEM;
2341
2342         base = (void *)page_address(page);
2343         baser_phys = virt_to_phys(base);
2344
2345         /* Check if the physical address of the memory is above 48bits */
2346         if (IS_ENABLED(CONFIG_ARM64_64K_PAGES) && (baser_phys >> 48)) {
2347
2348                 /* 52bit PA is supported only when PageSize=64K */
2349                 if (psz != SZ_64K) {
2350                         pr_err("ITS: no 52bit PA support when psz=%d\n", psz);
2351                         free_pages((unsigned long)base, order);
2352                         return -ENXIO;
2353                 }
2354
2355                 /* Convert 52bit PA to 48bit field */
2356                 baser_phys = GITS_BASER_PHYS_52_to_48(baser_phys);
2357         }
2358
2359 retry_baser:
2360         val = (baser_phys                                        |
2361                 (type << GITS_BASER_TYPE_SHIFT)                  |
2362                 ((esz - 1) << GITS_BASER_ENTRY_SIZE_SHIFT)       |
2363                 ((alloc_pages - 1) << GITS_BASER_PAGES_SHIFT)    |
2364                 cache                                            |
2365                 shr                                              |
2366                 GITS_BASER_VALID);
2367
2368         val |=  indirect ? GITS_BASER_INDIRECT : 0x0;
2369
2370         switch (psz) {
2371         case SZ_4K:
2372                 val |= GITS_BASER_PAGE_SIZE_4K;
2373                 break;
2374         case SZ_16K:
2375                 val |= GITS_BASER_PAGE_SIZE_16K;
2376                 break;
2377         case SZ_64K:
2378                 val |= GITS_BASER_PAGE_SIZE_64K;
2379                 break;
2380         }
2381
2382         its_write_baser(its, baser, val);
2383         tmp = baser->val;
2384
2385         if (its->flags & ITS_FLAGS_FORCE_NON_SHAREABLE)
2386                 tmp &= ~GITS_BASER_SHAREABILITY_MASK;
2387
2388         if ((val ^ tmp) & GITS_BASER_SHAREABILITY_MASK) {
2389                 /*
2390                  * Shareability didn't stick. Just use
2391                  * whatever the read reported, which is likely
2392                  * to be the only thing this redistributor
2393                  * supports. If that's zero, make it
2394                  * non-cacheable as well.
2395                  */
2396                 shr = tmp & GITS_BASER_SHAREABILITY_MASK;
2397                 if (!shr) {
2398                         cache = GITS_BASER_nC;
2399                         gic_flush_dcache_to_poc(base, PAGE_ORDER_TO_SIZE(order));
2400                 }
2401                 goto retry_baser;
2402         }
2403
2404         if (val != tmp) {
2405                 pr_err("ITS@%pa: %s doesn't stick: %llx %llx\n",
2406                        &its->phys_base, its_base_type_string[type],
2407                        val, tmp);
2408                 free_pages((unsigned long)base, order);
2409                 return -ENXIO;
2410         }
2411
2412         baser->order = order;
2413         baser->base = base;
2414         baser->psz = psz;
2415         tmp = indirect ? GITS_LVL1_ENTRY_SIZE : esz;
2416
2417         pr_info("ITS@%pa: allocated %d %s @%lx (%s, esz %d, psz %dK, shr %d)\n",
2418                 &its->phys_base, (int)(PAGE_ORDER_TO_SIZE(order) / (int)tmp),
2419                 its_base_type_string[type],
2420                 (unsigned long)virt_to_phys(base),
2421                 indirect ? "indirect" : "flat", (int)esz,
2422                 psz / SZ_1K, (int)shr >> GITS_BASER_SHAREABILITY_SHIFT);
2423
2424         return 0;
2425 }
2426
2427 static bool its_parse_indirect_baser(struct its_node *its,
2428                                      struct its_baser *baser,
2429                                      u32 *order, u32 ids)
2430 {
2431         u64 tmp = its_read_baser(its, baser);
2432         u64 type = GITS_BASER_TYPE(tmp);
2433         u64 esz = GITS_BASER_ENTRY_SIZE(tmp);
2434         u64 val = GITS_BASER_InnerShareable | GITS_BASER_RaWaWb;
2435         u32 new_order = *order;
2436         u32 psz = baser->psz;
2437         bool indirect = false;
2438
2439         /* No need to enable Indirection if memory requirement < (psz*2)bytes */
2440         if ((esz << ids) > (psz * 2)) {
2441                 /*
2442                  * Find out whether hw supports a single or two-level table by
2443                  * table by reading bit at offset '62' after writing '1' to it.
2444                  */
2445                 its_write_baser(its, baser, val | GITS_BASER_INDIRECT);
2446                 indirect = !!(baser->val & GITS_BASER_INDIRECT);
2447
2448                 if (indirect) {
2449                         /*
2450                          * The size of the lvl2 table is equal to ITS page size
2451                          * which is 'psz'. For computing lvl1 table size,
2452                          * subtract ID bits that sparse lvl2 table from 'ids'
2453                          * which is reported by ITS hardware times lvl1 table
2454                          * entry size.
2455                          */
2456                         ids -= ilog2(psz / (int)esz);
2457                         esz = GITS_LVL1_ENTRY_SIZE;
2458                 }
2459         }
2460
2461         /*
2462          * Allocate as many entries as required to fit the
2463          * range of device IDs that the ITS can grok... The ID
2464          * space being incredibly sparse, this results in a
2465          * massive waste of memory if two-level device table
2466          * feature is not supported by hardware.
2467          */
2468         new_order = max_t(u32, get_order(esz << ids), new_order);
2469         if (new_order > MAX_ORDER) {
2470                 new_order = MAX_ORDER;
2471                 ids = ilog2(PAGE_ORDER_TO_SIZE(new_order) / (int)esz);
2472                 pr_warn("ITS@%pa: %s Table too large, reduce ids %llu->%u\n",
2473                         &its->phys_base, its_base_type_string[type],
2474                         device_ids(its), ids);
2475         }
2476
2477         *order = new_order;
2478
2479         return indirect;
2480 }
2481
2482 static u32 compute_common_aff(u64 val)
2483 {
2484         u32 aff, clpiaff;
2485
2486         aff = FIELD_GET(GICR_TYPER_AFFINITY, val);
2487         clpiaff = FIELD_GET(GICR_TYPER_COMMON_LPI_AFF, val);
2488
2489         return aff & ~(GENMASK(31, 0) >> (clpiaff * 8));
2490 }
2491
2492 static u32 compute_its_aff(struct its_node *its)
2493 {
2494         u64 val;
2495         u32 svpet;
2496
2497         /*
2498          * Reencode the ITS SVPET and MPIDR as a GICR_TYPER, and compute
2499          * the resulting affinity. We then use that to see if this match
2500          * our own affinity.
2501          */
2502         svpet = FIELD_GET(GITS_TYPER_SVPET, its->typer);
2503         val  = FIELD_PREP(GICR_TYPER_COMMON_LPI_AFF, svpet);
2504         val |= FIELD_PREP(GICR_TYPER_AFFINITY, its->mpidr);
2505         return compute_common_aff(val);
2506 }
2507
2508 static struct its_node *find_sibling_its(struct its_node *cur_its)
2509 {
2510         struct its_node *its;
2511         u32 aff;
2512
2513         if (!FIELD_GET(GITS_TYPER_SVPET, cur_its->typer))
2514                 return NULL;
2515
2516         aff = compute_its_aff(cur_its);
2517
2518         list_for_each_entry(its, &its_nodes, entry) {
2519                 u64 baser;
2520
2521                 if (!is_v4_1(its) || its == cur_its)
2522                         continue;
2523
2524                 if (!FIELD_GET(GITS_TYPER_SVPET, its->typer))
2525                         continue;
2526
2527                 if (aff != compute_its_aff(its))
2528                         continue;
2529
2530                 /* GICv4.1 guarantees that the vPE table is GITS_BASER2 */
2531                 baser = its->tables[2].val;
2532                 if (!(baser & GITS_BASER_VALID))
2533                         continue;
2534
2535                 return its;
2536         }
2537
2538         return NULL;
2539 }
2540
2541 static void its_free_tables(struct its_node *its)
2542 {
2543         int i;
2544
2545         for (i = 0; i < GITS_BASER_NR_REGS; i++) {
2546                 if (its->tables[i].base) {
2547                         free_pages((unsigned long)its->tables[i].base,
2548                                    its->tables[i].order);
2549                         its->tables[i].base = NULL;
2550                 }
2551         }
2552 }
2553
2554 static int its_probe_baser_psz(struct its_node *its, struct its_baser *baser)
2555 {
2556         u64 psz = SZ_64K;
2557
2558         while (psz) {
2559                 u64 val, gpsz;
2560
2561                 val = its_read_baser(its, baser);
2562                 val &= ~GITS_BASER_PAGE_SIZE_MASK;
2563
2564                 switch (psz) {
2565                 case SZ_64K:
2566                         gpsz = GITS_BASER_PAGE_SIZE_64K;
2567                         break;
2568                 case SZ_16K:
2569                         gpsz = GITS_BASER_PAGE_SIZE_16K;
2570                         break;
2571                 case SZ_4K:
2572                 default:
2573                         gpsz = GITS_BASER_PAGE_SIZE_4K;
2574                         break;
2575                 }
2576
2577                 gpsz >>= GITS_BASER_PAGE_SIZE_SHIFT;
2578
2579                 val |= FIELD_PREP(GITS_BASER_PAGE_SIZE_MASK, gpsz);
2580                 its_write_baser(its, baser, val);
2581
2582                 if (FIELD_GET(GITS_BASER_PAGE_SIZE_MASK, baser->val) == gpsz)
2583                         break;
2584
2585                 switch (psz) {
2586                 case SZ_64K:
2587                         psz = SZ_16K;
2588                         break;
2589                 case SZ_16K:
2590                         psz = SZ_4K;
2591                         break;
2592                 case SZ_4K:
2593                 default:
2594                         return -1;
2595                 }
2596         }
2597
2598         baser->psz = psz;
2599         return 0;
2600 }
2601
2602 static int its_alloc_tables(struct its_node *its)
2603 {
2604         u64 shr = GITS_BASER_InnerShareable;
2605         u64 cache = GITS_BASER_RaWaWb;
2606         int err, i;
2607
2608         if (its->flags & ITS_FLAGS_WORKAROUND_CAVIUM_22375)
2609                 /* erratum 24313: ignore memory access type */
2610                 cache = GITS_BASER_nCnB;
2611
2612         for (i = 0; i < GITS_BASER_NR_REGS; i++) {
2613                 struct its_baser *baser = its->tables + i;
2614                 u64 val = its_read_baser(its, baser);
2615                 u64 type = GITS_BASER_TYPE(val);
2616                 bool indirect = false;
2617                 u32 order;
2618
2619                 if (type == GITS_BASER_TYPE_NONE)
2620                         continue;
2621
2622                 if (its_probe_baser_psz(its, baser)) {
2623                         its_free_tables(its);
2624                         return -ENXIO;
2625                 }
2626
2627                 order = get_order(baser->psz);
2628
2629                 switch (type) {
2630                 case GITS_BASER_TYPE_DEVICE:
2631                         indirect = its_parse_indirect_baser(its, baser, &order,
2632                                                             device_ids(its));
2633                         break;
2634
2635                 case GITS_BASER_TYPE_VCPU:
2636                         if (is_v4_1(its)) {
2637                                 struct its_node *sibling;
2638
2639                                 WARN_ON(i != 2);
2640                                 if ((sibling = find_sibling_its(its))) {
2641                                         *baser = sibling->tables[2];
2642                                         its_write_baser(its, baser, baser->val);
2643                                         continue;
2644                                 }
2645                         }
2646
2647                         indirect = its_parse_indirect_baser(its, baser, &order,
2648                                                             ITS_MAX_VPEID_BITS);
2649                         break;
2650                 }
2651
2652                 err = its_setup_baser(its, baser, cache, shr, order, indirect);
2653                 if (err < 0) {
2654                         its_free_tables(its);
2655                         return err;
2656                 }
2657
2658                 /* Update settings which will be used for next BASERn */
2659                 cache = baser->val & GITS_BASER_CACHEABILITY_MASK;
2660                 shr = baser->val & GITS_BASER_SHAREABILITY_MASK;
2661         }
2662
2663         return 0;
2664 }
2665
2666 static u64 inherit_vpe_l1_table_from_its(void)
2667 {
2668         struct its_node *its;
2669         u64 val;
2670         u32 aff;
2671
2672         val = gic_read_typer(gic_data_rdist_rd_base() + GICR_TYPER);
2673         aff = compute_common_aff(val);
2674
2675         list_for_each_entry(its, &its_nodes, entry) {
2676                 u64 baser, addr;
2677
2678                 if (!is_v4_1(its))
2679                         continue;
2680
2681                 if (!FIELD_GET(GITS_TYPER_SVPET, its->typer))
2682                         continue;
2683
2684                 if (aff != compute_its_aff(its))
2685                         continue;
2686
2687                 /* GICv4.1 guarantees that the vPE table is GITS_BASER2 */
2688                 baser = its->tables[2].val;
2689                 if (!(baser & GITS_BASER_VALID))
2690                         continue;
2691
2692                 /* We have a winner! */
2693                 gic_data_rdist()->vpe_l1_base = its->tables[2].base;
2694
2695                 val  = GICR_VPROPBASER_4_1_VALID;
2696                 if (baser & GITS_BASER_INDIRECT)
2697                         val |= GICR_VPROPBASER_4_1_INDIRECT;
2698                 val |= FIELD_PREP(GICR_VPROPBASER_4_1_PAGE_SIZE,
2699                                   FIELD_GET(GITS_BASER_PAGE_SIZE_MASK, baser));
2700                 switch (FIELD_GET(GITS_BASER_PAGE_SIZE_MASK, baser)) {
2701                 case GIC_PAGE_SIZE_64K:
2702                         addr = GITS_BASER_ADDR_48_to_52(baser);
2703                         break;
2704                 default:
2705                         addr = baser & GENMASK_ULL(47, 12);
2706                         break;
2707                 }
2708                 val |= FIELD_PREP(GICR_VPROPBASER_4_1_ADDR, addr >> 12);
2709                 val |= FIELD_PREP(GICR_VPROPBASER_SHAREABILITY_MASK,
2710                                   FIELD_GET(GITS_BASER_SHAREABILITY_MASK, baser));
2711                 val |= FIELD_PREP(GICR_VPROPBASER_INNER_CACHEABILITY_MASK,
2712                                   FIELD_GET(GITS_BASER_INNER_CACHEABILITY_MASK, baser));
2713                 val |= FIELD_PREP(GICR_VPROPBASER_4_1_SIZE, GITS_BASER_NR_PAGES(baser) - 1);
2714
2715                 return val;
2716         }
2717
2718         return 0;
2719 }
2720
2721 static u64 inherit_vpe_l1_table_from_rd(cpumask_t **mask)
2722 {
2723         u32 aff;
2724         u64 val;
2725         int cpu;
2726
2727         val = gic_read_typer(gic_data_rdist_rd_base() + GICR_TYPER);
2728         aff = compute_common_aff(val);
2729
2730         for_each_possible_cpu(cpu) {
2731                 void __iomem *base = gic_data_rdist_cpu(cpu)->rd_base;
2732
2733                 if (!base || cpu == smp_processor_id())
2734                         continue;
2735
2736                 val = gic_read_typer(base + GICR_TYPER);
2737                 if (aff != compute_common_aff(val))
2738                         continue;
2739
2740                 /*
2741                  * At this point, we have a victim. This particular CPU
2742                  * has already booted, and has an affinity that matches
2743                  * ours wrt CommonLPIAff. Let's use its own VPROPBASER.
2744                  * Make sure we don't write the Z bit in that case.
2745                  */
2746                 val = gicr_read_vpropbaser(base + SZ_128K + GICR_VPROPBASER);
2747                 val &= ~GICR_VPROPBASER_4_1_Z;
2748
2749                 gic_data_rdist()->vpe_l1_base = gic_data_rdist_cpu(cpu)->vpe_l1_base;
2750                 *mask = gic_data_rdist_cpu(cpu)->vpe_table_mask;
2751
2752                 return val;
2753         }
2754
2755         return 0;
2756 }
2757
2758 static bool allocate_vpe_l2_table(int cpu, u32 id)
2759 {
2760         void __iomem *base = gic_data_rdist_cpu(cpu)->rd_base;
2761         unsigned int psz, esz, idx, npg, gpsz;
2762         u64 val;
2763         struct page *page;
2764         __le64 *table;
2765
2766         if (!gic_rdists->has_rvpeid)
2767                 return true;
2768
2769         /* Skip non-present CPUs */
2770         if (!base)
2771                 return true;
2772
2773         val  = gicr_read_vpropbaser(base + SZ_128K + GICR_VPROPBASER);
2774
2775         esz  = FIELD_GET(GICR_VPROPBASER_4_1_ENTRY_SIZE, val) + 1;
2776         gpsz = FIELD_GET(GICR_VPROPBASER_4_1_PAGE_SIZE, val);
2777         npg  = FIELD_GET(GICR_VPROPBASER_4_1_SIZE, val) + 1;
2778
2779         switch (gpsz) {
2780         default:
2781                 WARN_ON(1);
2782                 fallthrough;
2783         case GIC_PAGE_SIZE_4K:
2784                 psz = SZ_4K;
2785                 break;
2786         case GIC_PAGE_SIZE_16K:
2787                 psz = SZ_16K;
2788                 break;
2789         case GIC_PAGE_SIZE_64K:
2790                 psz = SZ_64K;
2791                 break;
2792         }
2793
2794         /* Don't allow vpe_id that exceeds single, flat table limit */
2795         if (!(val & GICR_VPROPBASER_4_1_INDIRECT))
2796                 return (id < (npg * psz / (esz * SZ_8)));
2797
2798         /* Compute 1st level table index & check if that exceeds table limit */
2799         idx = id >> ilog2(psz / (esz * SZ_8));
2800         if (idx >= (npg * psz / GITS_LVL1_ENTRY_SIZE))
2801                 return false;
2802
2803         table = gic_data_rdist_cpu(cpu)->vpe_l1_base;
2804
2805         /* Allocate memory for 2nd level table */
2806         if (!table[idx]) {
2807                 page = alloc_pages(GFP_KERNEL | __GFP_ZERO, get_order(psz));
2808                 if (!page)
2809                         return false;
2810
2811                 /* Flush Lvl2 table to PoC if hw doesn't support coherency */
2812                 if (!(val & GICR_VPROPBASER_SHAREABILITY_MASK))
2813                         gic_flush_dcache_to_poc(page_address(page), psz);
2814
2815                 table[idx] = cpu_to_le64(page_to_phys(page) | GITS_BASER_VALID);
2816
2817                 /* Flush Lvl1 entry to PoC if hw doesn't support coherency */
2818                 if (!(val & GICR_VPROPBASER_SHAREABILITY_MASK))
2819                         gic_flush_dcache_to_poc(table + idx, GITS_LVL1_ENTRY_SIZE);
2820
2821                 /* Ensure updated table contents are visible to RD hardware */
2822                 dsb(sy);
2823         }
2824
2825         return true;
2826 }
2827
2828 static int allocate_vpe_l1_table(void)
2829 {
2830         void __iomem *vlpi_base = gic_data_rdist_vlpi_base();
2831         u64 val, gpsz, npg, pa;
2832         unsigned int psz = SZ_64K;
2833         unsigned int np, epp, esz;
2834         struct page *page;
2835
2836         if (!gic_rdists->has_rvpeid)
2837                 return 0;
2838
2839         /*
2840          * if VPENDBASER.Valid is set, disable any previously programmed
2841          * VPE by setting PendingLast while clearing Valid. This has the
2842          * effect of making sure no doorbell will be generated and we can
2843          * then safely clear VPROPBASER.Valid.
2844          */
2845         if (gicr_read_vpendbaser(vlpi_base + GICR_VPENDBASER) & GICR_VPENDBASER_Valid)
2846                 gicr_write_vpendbaser(GICR_VPENDBASER_PendingLast,
2847                                       vlpi_base + GICR_VPENDBASER);
2848
2849         /*
2850          * If we can inherit the configuration from another RD, let's do
2851          * so. Otherwise, we have to go through the allocation process. We
2852          * assume that all RDs have the exact same requirements, as
2853          * nothing will work otherwise.
2854          */
2855         val = inherit_vpe_l1_table_from_rd(&gic_data_rdist()->vpe_table_mask);
2856         if (val & GICR_VPROPBASER_4_1_VALID)
2857                 goto out;
2858
2859         gic_data_rdist()->vpe_table_mask = kzalloc(sizeof(cpumask_t), GFP_ATOMIC);
2860         if (!gic_data_rdist()->vpe_table_mask)
2861                 return -ENOMEM;
2862
2863         val = inherit_vpe_l1_table_from_its();
2864         if (val & GICR_VPROPBASER_4_1_VALID)
2865                 goto out;
2866
2867         /* First probe the page size */
2868         val = FIELD_PREP(GICR_VPROPBASER_4_1_PAGE_SIZE, GIC_PAGE_SIZE_64K);
2869         gicr_write_vpropbaser(val, vlpi_base + GICR_VPROPBASER);
2870         val = gicr_read_vpropbaser(vlpi_base + GICR_VPROPBASER);
2871         gpsz = FIELD_GET(GICR_VPROPBASER_4_1_PAGE_SIZE, val);
2872         esz = FIELD_GET(GICR_VPROPBASER_4_1_ENTRY_SIZE, val);
2873
2874         switch (gpsz) {
2875         default:
2876                 gpsz = GIC_PAGE_SIZE_4K;
2877                 fallthrough;
2878         case GIC_PAGE_SIZE_4K:
2879                 psz = SZ_4K;
2880                 break;
2881         case GIC_PAGE_SIZE_16K:
2882                 psz = SZ_16K;
2883                 break;
2884         case GIC_PAGE_SIZE_64K:
2885                 psz = SZ_64K;
2886                 break;
2887         }
2888
2889         /*
2890          * Start populating the register from scratch, including RO fields
2891          * (which we want to print in debug cases...)
2892          */
2893         val = 0;
2894         val |= FIELD_PREP(GICR_VPROPBASER_4_1_PAGE_SIZE, gpsz);
2895         val |= FIELD_PREP(GICR_VPROPBASER_4_1_ENTRY_SIZE, esz);
2896
2897         /* How many entries per GIC page? */
2898         esz++;
2899         epp = psz / (esz * SZ_8);
2900
2901         /*
2902          * If we need more than just a single L1 page, flag the table
2903          * as indirect and compute the number of required L1 pages.
2904          */
2905         if (epp < ITS_MAX_VPEID) {
2906                 int nl2;
2907
2908                 val |= GICR_VPROPBASER_4_1_INDIRECT;
2909
2910                 /* Number of L2 pages required to cover the VPEID space */
2911                 nl2 = DIV_ROUND_UP(ITS_MAX_VPEID, epp);
2912
2913                 /* Number of L1 pages to point to the L2 pages */
2914                 npg = DIV_ROUND_UP(nl2 * SZ_8, psz);
2915         } else {
2916                 npg = 1;
2917         }
2918
2919         val |= FIELD_PREP(GICR_VPROPBASER_4_1_SIZE, npg - 1);
2920
2921         /* Right, that's the number of CPU pages we need for L1 */
2922         np = DIV_ROUND_UP(npg * psz, PAGE_SIZE);
2923
2924         pr_debug("np = %d, npg = %lld, psz = %d, epp = %d, esz = %d\n",
2925                  np, npg, psz, epp, esz);
2926         page = alloc_pages(GFP_ATOMIC | __GFP_ZERO, get_order(np * PAGE_SIZE));
2927         if (!page)
2928                 return -ENOMEM;
2929
2930         gic_data_rdist()->vpe_l1_base = page_address(page);
2931         pa = virt_to_phys(page_address(page));
2932         WARN_ON(!IS_ALIGNED(pa, psz));
2933
2934         val |= FIELD_PREP(GICR_VPROPBASER_4_1_ADDR, pa >> 12);
2935         val |= GICR_VPROPBASER_RaWb;
2936         val |= GICR_VPROPBASER_InnerShareable;
2937         val |= GICR_VPROPBASER_4_1_Z;
2938         val |= GICR_VPROPBASER_4_1_VALID;
2939
2940 out:
2941         gicr_write_vpropbaser(val, vlpi_base + GICR_VPROPBASER);
2942         cpumask_set_cpu(smp_processor_id(), gic_data_rdist()->vpe_table_mask);
2943
2944         pr_debug("CPU%d: VPROPBASER = %llx %*pbl\n",
2945                  smp_processor_id(), val,
2946                  cpumask_pr_args(gic_data_rdist()->vpe_table_mask));
2947
2948         return 0;
2949 }
2950
2951 static int its_alloc_collections(struct its_node *its)
2952 {
2953         int i;
2954
2955         its->collections = kcalloc(nr_cpu_ids, sizeof(*its->collections),
2956                                    GFP_KERNEL);
2957         if (!its->collections)
2958                 return -ENOMEM;
2959
2960         for (i = 0; i < nr_cpu_ids; i++)
2961                 its->collections[i].target_address = ~0ULL;
2962
2963         return 0;
2964 }
2965
2966 static struct page *its_allocate_pending_table(gfp_t gfp_flags)
2967 {
2968         struct page *pend_page;
2969
2970         pend_page = alloc_pages(gfp_flags | __GFP_ZERO,
2971                                 get_order(LPI_PENDBASE_SZ));
2972         if (!pend_page)
2973                 return NULL;
2974
2975         /* Make sure the GIC will observe the zero-ed page */
2976         gic_flush_dcache_to_poc(page_address(pend_page), LPI_PENDBASE_SZ);
2977
2978         return pend_page;
2979 }
2980
2981 static void its_free_pending_table(struct page *pt)
2982 {
2983         free_pages((unsigned long)page_address(pt), get_order(LPI_PENDBASE_SZ));
2984 }
2985
2986 /*
2987  * Booting with kdump and LPIs enabled is generally fine. Any other
2988  * case is wrong in the absence of firmware/EFI support.
2989  */
2990 static bool enabled_lpis_allowed(void)
2991 {
2992         phys_addr_t addr;
2993         u64 val;
2994
2995         /* Check whether the property table is in a reserved region */
2996         val = gicr_read_propbaser(gic_data_rdist_rd_base() + GICR_PROPBASER);
2997         addr = val & GENMASK_ULL(51, 12);
2998
2999         return gic_check_reserved_range(addr, LPI_PROPBASE_SZ);
3000 }
3001
3002 static int __init allocate_lpi_tables(void)
3003 {
3004         u64 val;
3005         int err, cpu;
3006
3007         /*
3008          * If LPIs are enabled while we run this from the boot CPU,
3009          * flag the RD tables as pre-allocated if the stars do align.
3010          */
3011         val = readl_relaxed(gic_data_rdist_rd_base() + GICR_CTLR);
3012         if ((val & GICR_CTLR_ENABLE_LPIS) && enabled_lpis_allowed()) {
3013                 gic_rdists->flags |= (RDIST_FLAGS_RD_TABLES_PREALLOCATED |
3014                                       RDIST_FLAGS_PROPBASE_NEEDS_FLUSHING);
3015                 pr_info("GICv3: Using preallocated redistributor tables\n");
3016         }
3017
3018         err = its_setup_lpi_prop_table();
3019         if (err)
3020                 return err;
3021
3022         /*
3023          * We allocate all the pending tables anyway, as we may have a
3024          * mix of RDs that have had LPIs enabled, and some that
3025          * don't. We'll free the unused ones as each CPU comes online.
3026          */
3027         for_each_possible_cpu(cpu) {
3028                 struct page *pend_page;
3029
3030                 pend_page = its_allocate_pending_table(GFP_NOWAIT);
3031                 if (!pend_page) {
3032                         pr_err("Failed to allocate PENDBASE for CPU%d\n", cpu);
3033                         return -ENOMEM;
3034                 }
3035
3036                 gic_data_rdist_cpu(cpu)->pend_page = pend_page;
3037         }
3038
3039         return 0;
3040 }
3041
3042 static u64 read_vpend_dirty_clear(void __iomem *vlpi_base)
3043 {
3044         u32 count = 1000000;    /* 1s! */
3045         bool clean;
3046         u64 val;
3047
3048         do {
3049                 val = gicr_read_vpendbaser(vlpi_base + GICR_VPENDBASER);
3050                 clean = !(val & GICR_VPENDBASER_Dirty);
3051                 if (!clean) {
3052                         count--;
3053                         cpu_relax();
3054                         udelay(1);
3055                 }
3056         } while (!clean && count);
3057
3058         if (unlikely(!clean))
3059                 pr_err_ratelimited("ITS virtual pending table not cleaning\n");
3060
3061         return val;
3062 }
3063
3064 static u64 its_clear_vpend_valid(void __iomem *vlpi_base, u64 clr, u64 set)
3065 {
3066         u64 val;
3067
3068         /* Make sure we wait until the RD is done with the initial scan */
3069         val = read_vpend_dirty_clear(vlpi_base);
3070         val &= ~GICR_VPENDBASER_Valid;
3071         val &= ~clr;
3072         val |= set;
3073         gicr_write_vpendbaser(val, vlpi_base + GICR_VPENDBASER);
3074
3075         val = read_vpend_dirty_clear(vlpi_base);
3076         if (unlikely(val & GICR_VPENDBASER_Dirty))
3077                 val |= GICR_VPENDBASER_PendingLast;
3078
3079         return val;
3080 }
3081
3082 static void its_cpu_init_lpis(void)
3083 {
3084         void __iomem *rbase = gic_data_rdist_rd_base();
3085         struct page *pend_page;
3086         phys_addr_t paddr;
3087         u64 val, tmp;
3088
3089         if (gic_data_rdist()->flags & RD_LOCAL_LPI_ENABLED)
3090                 return;
3091
3092         val = readl_relaxed(rbase + GICR_CTLR);
3093         if ((gic_rdists->flags & RDIST_FLAGS_RD_TABLES_PREALLOCATED) &&
3094             (val & GICR_CTLR_ENABLE_LPIS)) {
3095                 /*
3096                  * Check that we get the same property table on all
3097                  * RDs. If we don't, this is hopeless.
3098                  */
3099                 paddr = gicr_read_propbaser(rbase + GICR_PROPBASER);
3100                 paddr &= GENMASK_ULL(51, 12);
3101                 if (WARN_ON(gic_rdists->prop_table_pa != paddr))
3102                         add_taint(TAINT_CRAP, LOCKDEP_STILL_OK);
3103
3104                 paddr = gicr_read_pendbaser(rbase + GICR_PENDBASER);
3105                 paddr &= GENMASK_ULL(51, 16);
3106
3107                 WARN_ON(!gic_check_reserved_range(paddr, LPI_PENDBASE_SZ));
3108                 gic_data_rdist()->flags |= RD_LOCAL_PENDTABLE_PREALLOCATED;
3109
3110                 goto out;
3111         }
3112
3113         pend_page = gic_data_rdist()->pend_page;
3114         paddr = page_to_phys(pend_page);
3115
3116         /* set PROPBASE */
3117         val = (gic_rdists->prop_table_pa |
3118                GICR_PROPBASER_InnerShareable |
3119                GICR_PROPBASER_RaWaWb |
3120                ((LPI_NRBITS - 1) & GICR_PROPBASER_IDBITS_MASK));
3121
3122         gicr_write_propbaser(val, rbase + GICR_PROPBASER);
3123         tmp = gicr_read_propbaser(rbase + GICR_PROPBASER);
3124
3125         if (gic_rdists->flags & RDIST_FLAGS_FORCE_NON_SHAREABLE)
3126                 tmp &= ~GICR_PROPBASER_SHAREABILITY_MASK;
3127
3128         if ((tmp ^ val) & GICR_PROPBASER_SHAREABILITY_MASK) {
3129                 if (!(tmp & GICR_PROPBASER_SHAREABILITY_MASK)) {
3130                         /*
3131                          * The HW reports non-shareable, we must
3132                          * remove the cacheability attributes as
3133                          * well.
3134                          */
3135                         val &= ~(GICR_PROPBASER_SHAREABILITY_MASK |
3136                                  GICR_PROPBASER_CACHEABILITY_MASK);
3137                         val |= GICR_PROPBASER_nC;
3138                         gicr_write_propbaser(val, rbase + GICR_PROPBASER);
3139                 }
3140                 pr_info_once("GIC: using cache flushing for LPI property table\n");
3141                 gic_rdists->flags |= RDIST_FLAGS_PROPBASE_NEEDS_FLUSHING;
3142         }
3143
3144         /* set PENDBASE */
3145         val = (page_to_phys(pend_page) |
3146                GICR_PENDBASER_InnerShareable |
3147                GICR_PENDBASER_RaWaWb);
3148
3149         gicr_write_pendbaser(val, rbase + GICR_PENDBASER);
3150         tmp = gicr_read_pendbaser(rbase + GICR_PENDBASER);
3151
3152         if (gic_rdists->flags & RDIST_FLAGS_FORCE_NON_SHAREABLE)
3153                 tmp &= ~GICR_PENDBASER_SHAREABILITY_MASK;
3154
3155         if (!(tmp & GICR_PENDBASER_SHAREABILITY_MASK)) {
3156                 /*
3157                  * The HW reports non-shareable, we must remove the
3158                  * cacheability attributes as well.
3159                  */
3160                 val &= ~(GICR_PENDBASER_SHAREABILITY_MASK |
3161                          GICR_PENDBASER_CACHEABILITY_MASK);
3162                 val |= GICR_PENDBASER_nC;
3163                 gicr_write_pendbaser(val, rbase + GICR_PENDBASER);
3164         }
3165
3166         /* Enable LPIs */
3167         val = readl_relaxed(rbase + GICR_CTLR);
3168         val |= GICR_CTLR_ENABLE_LPIS;
3169         writel_relaxed(val, rbase + GICR_CTLR);
3170
3171         if (gic_rdists->has_vlpis && !gic_rdists->has_rvpeid) {
3172                 void __iomem *vlpi_base = gic_data_rdist_vlpi_base();
3173
3174                 /*
3175                  * It's possible for CPU to receive VLPIs before it is
3176                  * scheduled as a vPE, especially for the first CPU, and the
3177                  * VLPI with INTID larger than 2^(IDbits+1) will be considered
3178                  * as out of range and dropped by GIC.
3179                  * So we initialize IDbits to known value to avoid VLPI drop.
3180                  */
3181                 val = (LPI_NRBITS - 1) & GICR_VPROPBASER_IDBITS_MASK;
3182                 pr_debug("GICv4: CPU%d: Init IDbits to 0x%llx for GICR_VPROPBASER\n",
3183                         smp_processor_id(), val);
3184                 gicr_write_vpropbaser(val, vlpi_base + GICR_VPROPBASER);
3185
3186                 /*
3187                  * Also clear Valid bit of GICR_VPENDBASER, in case some
3188                  * ancient programming gets left in and has possibility of
3189                  * corrupting memory.
3190                  */
3191                 val = its_clear_vpend_valid(vlpi_base, 0, 0);
3192         }
3193
3194         if (allocate_vpe_l1_table()) {
3195                 /*
3196                  * If the allocation has failed, we're in massive trouble.
3197                  * Disable direct injection, and pray that no VM was
3198                  * already running...
3199                  */
3200                 gic_rdists->has_rvpeid = false;
3201                 gic_rdists->has_vlpis = false;
3202         }
3203
3204         /* Make sure the GIC has seen the above */
3205         dsb(sy);
3206 out:
3207         gic_data_rdist()->flags |= RD_LOCAL_LPI_ENABLED;
3208         pr_info("GICv3: CPU%d: using %s LPI pending table @%pa\n",
3209                 smp_processor_id(),
3210                 gic_data_rdist()->flags & RD_LOCAL_PENDTABLE_PREALLOCATED ?
3211                 "reserved" : "allocated",
3212                 &paddr);
3213 }
3214
3215 static void its_cpu_init_collection(struct its_node *its)
3216 {
3217         int cpu = smp_processor_id();
3218         u64 target;
3219
3220         /* avoid cross node collections and its mapping */
3221         if (its->flags & ITS_FLAGS_WORKAROUND_CAVIUM_23144) {
3222                 struct device_node *cpu_node;
3223
3224                 cpu_node = of_get_cpu_node(cpu, NULL);
3225                 if (its->numa_node != NUMA_NO_NODE &&
3226                         its->numa_node != of_node_to_nid(cpu_node))
3227                         return;
3228         }
3229
3230         /*
3231          * We now have to bind each collection to its target
3232          * redistributor.
3233          */
3234         if (gic_read_typer(its->base + GITS_TYPER) & GITS_TYPER_PTA) {
3235                 /*
3236                  * This ITS wants the physical address of the
3237                  * redistributor.
3238                  */
3239                 target = gic_data_rdist()->phys_base;
3240         } else {
3241                 /* This ITS wants a linear CPU number. */
3242                 target = gic_read_typer(gic_data_rdist_rd_base() + GICR_TYPER);
3243                 target = GICR_TYPER_CPU_NUMBER(target) << 16;
3244         }
3245
3246         /* Perform collection mapping */
3247         its->collections[cpu].target_address = target;
3248         its->collections[cpu].col_id = cpu;
3249
3250         its_send_mapc(its, &its->collections[cpu], 1);
3251         its_send_invall(its, &its->collections[cpu]);
3252 }
3253
3254 static void its_cpu_init_collections(void)
3255 {
3256         struct its_node *its;
3257
3258         raw_spin_lock(&its_lock);
3259
3260         list_for_each_entry(its, &its_nodes, entry)
3261                 its_cpu_init_collection(its);
3262
3263         raw_spin_unlock(&its_lock);
3264 }
3265
3266 static struct its_device *its_find_device(struct its_node *its, u32 dev_id)
3267 {
3268         struct its_device *its_dev = NULL, *tmp;
3269         unsigned long flags;
3270
3271         raw_spin_lock_irqsave(&its->lock, flags);
3272
3273         list_for_each_entry(tmp, &its->its_device_list, entry) {
3274                 if (tmp->device_id == dev_id) {
3275                         its_dev = tmp;
3276                         break;
3277                 }
3278         }
3279
3280         raw_spin_unlock_irqrestore(&its->lock, flags);
3281
3282         return its_dev;
3283 }
3284
3285 static struct its_baser *its_get_baser(struct its_node *its, u32 type)
3286 {
3287         int i;
3288
3289         for (i = 0; i < GITS_BASER_NR_REGS; i++) {
3290                 if (GITS_BASER_TYPE(its->tables[i].val) == type)
3291                         return &its->tables[i];
3292         }
3293
3294         return NULL;
3295 }
3296
3297 static bool its_alloc_table_entry(struct its_node *its,
3298                                   struct its_baser *baser, u32 id)
3299 {
3300         struct page *page;
3301         u32 esz, idx;
3302         __le64 *table;
3303
3304         /* Don't allow device id that exceeds single, flat table limit */
3305         esz = GITS_BASER_ENTRY_SIZE(baser->val);
3306         if (!(baser->val & GITS_BASER_INDIRECT))
3307                 return (id < (PAGE_ORDER_TO_SIZE(baser->order) / esz));
3308
3309         /* Compute 1st level table index & check if that exceeds table limit */
3310         idx = id >> ilog2(baser->psz / esz);
3311         if (idx >= (PAGE_ORDER_TO_SIZE(baser->order) / GITS_LVL1_ENTRY_SIZE))
3312                 return false;
3313
3314         table = baser->base;
3315
3316         /* Allocate memory for 2nd level table */
3317         if (!table[idx]) {
3318                 page = alloc_pages_node(its->numa_node, GFP_KERNEL | __GFP_ZERO,
3319                                         get_order(baser->psz));
3320                 if (!page)
3321                         return false;
3322
3323                 /* Flush Lvl2 table to PoC if hw doesn't support coherency */
3324                 if (!(baser->val & GITS_BASER_SHAREABILITY_MASK))
3325                         gic_flush_dcache_to_poc(page_address(page), baser->psz);
3326
3327                 table[idx] = cpu_to_le64(page_to_phys(page) | GITS_BASER_VALID);
3328
3329                 /* Flush Lvl1 entry to PoC if hw doesn't support coherency */
3330                 if (!(baser->val & GITS_BASER_SHAREABILITY_MASK))
3331                         gic_flush_dcache_to_poc(table + idx, GITS_LVL1_ENTRY_SIZE);
3332
3333                 /* Ensure updated table contents are visible to ITS hardware */
3334                 dsb(sy);
3335         }
3336
3337         return true;
3338 }
3339
3340 static bool its_alloc_device_table(struct its_node *its, u32 dev_id)
3341 {
3342         struct its_baser *baser;
3343
3344         baser = its_get_baser(its, GITS_BASER_TYPE_DEVICE);
3345
3346         /* Don't allow device id that exceeds ITS hardware limit */
3347         if (!baser)
3348                 return (ilog2(dev_id) < device_ids(its));
3349
3350         return its_alloc_table_entry(its, baser, dev_id);
3351 }
3352
3353 static bool its_alloc_vpe_table(u32 vpe_id)
3354 {
3355         struct its_node *its;
3356         int cpu;
3357
3358         /*
3359          * Make sure the L2 tables are allocated on *all* v4 ITSs. We
3360          * could try and only do it on ITSs corresponding to devices
3361          * that have interrupts targeted at this VPE, but the
3362          * complexity becomes crazy (and you have tons of memory
3363          * anyway, right?).
3364          */
3365         list_for_each_entry(its, &its_nodes, entry) {
3366                 struct its_baser *baser;
3367
3368                 if (!is_v4(its))
3369                         continue;
3370
3371                 baser = its_get_baser(its, GITS_BASER_TYPE_VCPU);
3372                 if (!baser)
3373                         return false;
3374
3375                 if (!its_alloc_table_entry(its, baser, vpe_id))
3376                         return false;
3377         }
3378
3379         /* Non v4.1? No need to iterate RDs and go back early. */
3380         if (!gic_rdists->has_rvpeid)
3381                 return true;
3382
3383         /*
3384          * Make sure the L2 tables are allocated for all copies of
3385          * the L1 table on *all* v4.1 RDs.
3386          */
3387         for_each_possible_cpu(cpu) {
3388                 if (!allocate_vpe_l2_table(cpu, vpe_id))
3389                         return false;
3390         }
3391
3392         return true;
3393 }
3394
3395 static struct its_device *its_create_device(struct its_node *its, u32 dev_id,
3396                                             int nvecs, bool alloc_lpis)
3397 {
3398         struct its_device *dev;
3399         unsigned long *lpi_map = NULL;
3400         unsigned long flags;
3401         u16 *col_map = NULL;
3402         void *itt;
3403         int lpi_base;
3404         int nr_lpis;
3405         int nr_ites;
3406         int sz;
3407
3408         if (!its_alloc_device_table(its, dev_id))
3409                 return NULL;
3410
3411         if (WARN_ON(!is_power_of_2(nvecs)))
3412                 nvecs = roundup_pow_of_two(nvecs);
3413
3414         dev = kzalloc(sizeof(*dev), GFP_KERNEL);
3415         /*
3416          * Even if the device wants a single LPI, the ITT must be
3417          * sized as a power of two (and you need at least one bit...).
3418          */
3419         nr_ites = max(2, nvecs);
3420         sz = nr_ites * (FIELD_GET(GITS_TYPER_ITT_ENTRY_SIZE, its->typer) + 1);
3421         sz = max(sz, ITS_ITT_ALIGN) + ITS_ITT_ALIGN - 1;
3422         itt = kzalloc_node(sz, GFP_KERNEL, its->numa_node);
3423         if (alloc_lpis) {
3424                 lpi_map = its_lpi_alloc(nvecs, &lpi_base, &nr_lpis);
3425                 if (lpi_map)
3426                         col_map = kcalloc(nr_lpis, sizeof(*col_map),
3427                                           GFP_KERNEL);
3428         } else {
3429                 col_map = kcalloc(nr_ites, sizeof(*col_map), GFP_KERNEL);
3430                 nr_lpis = 0;
3431                 lpi_base = 0;
3432         }
3433
3434         if (!dev || !itt ||  !col_map || (!lpi_map && alloc_lpis)) {
3435                 kfree(dev);
3436                 kfree(itt);
3437                 bitmap_free(lpi_map);
3438                 kfree(col_map);
3439                 return NULL;
3440         }
3441
3442         gic_flush_dcache_to_poc(itt, sz);
3443
3444         dev->its = its;
3445         dev->itt = itt;
3446         dev->nr_ites = nr_ites;
3447         dev->event_map.lpi_map = lpi_map;
3448         dev->event_map.col_map = col_map;
3449         dev->event_map.lpi_base = lpi_base;
3450         dev->event_map.nr_lpis = nr_lpis;
3451         raw_spin_lock_init(&dev->event_map.vlpi_lock);
3452         dev->device_id = dev_id;
3453         INIT_LIST_HEAD(&dev->entry);
3454
3455         raw_spin_lock_irqsave(&its->lock, flags);
3456         list_add(&dev->entry, &its->its_device_list);
3457         raw_spin_unlock_irqrestore(&its->lock, flags);
3458
3459         /* Map device to its ITT */
3460         its_send_mapd(dev, 1);
3461
3462         return dev;
3463 }
3464
3465 static void its_free_device(struct its_device *its_dev)
3466 {
3467         unsigned long flags;
3468
3469         raw_spin_lock_irqsave(&its_dev->its->lock, flags);
3470         list_del(&its_dev->entry);
3471         raw_spin_unlock_irqrestore(&its_dev->its->lock, flags);
3472         kfree(its_dev->event_map.col_map);
3473         kfree(its_dev->itt);
3474         kfree(its_dev);
3475 }
3476
3477 static int its_alloc_device_irq(struct its_device *dev, int nvecs, irq_hw_number_t *hwirq)
3478 {
3479         int idx;
3480
3481         /* Find a free LPI region in lpi_map and allocate them. */
3482         idx = bitmap_find_free_region(dev->event_map.lpi_map,
3483                                       dev->event_map.nr_lpis,
3484                                       get_count_order(nvecs));
3485         if (idx < 0)
3486                 return -ENOSPC;
3487
3488         *hwirq = dev->event_map.lpi_base + idx;
3489
3490         return 0;
3491 }
3492
3493 static int its_msi_prepare(struct irq_domain *domain, struct device *dev,
3494                            int nvec, msi_alloc_info_t *info)
3495 {
3496         struct its_node *its;
3497         struct its_device *its_dev;
3498         struct msi_domain_info *msi_info;
3499         u32 dev_id;
3500         int err = 0;
3501
3502         /*
3503          * We ignore "dev" entirely, and rely on the dev_id that has
3504          * been passed via the scratchpad. This limits this domain's
3505          * usefulness to upper layers that definitely know that they
3506          * are built on top of the ITS.
3507          */
3508         dev_id = info->scratchpad[0].ul;
3509
3510         msi_info = msi_get_domain_info(domain);
3511         its = msi_info->data;
3512
3513         if (!gic_rdists->has_direct_lpi &&
3514             vpe_proxy.dev &&
3515             vpe_proxy.dev->its == its &&
3516             dev_id == vpe_proxy.dev->device_id) {
3517                 /* Bad luck. Get yourself a better implementation */
3518                 WARN_ONCE(1, "DevId %x clashes with GICv4 VPE proxy device\n",
3519                           dev_id);
3520                 return -EINVAL;
3521         }
3522
3523         mutex_lock(&its->dev_alloc_lock);
3524         its_dev = its_find_device(its, dev_id);
3525         if (its_dev) {
3526                 /*
3527                  * We already have seen this ID, probably through
3528                  * another alias (PCI bridge of some sort). No need to
3529                  * create the device.
3530                  */
3531                 its_dev->shared = true;
3532                 pr_debug("Reusing ITT for devID %x\n", dev_id);
3533                 goto out;
3534         }
3535
3536         its_dev = its_create_device(its, dev_id, nvec, true);
3537         if (!its_dev) {
3538                 err = -ENOMEM;
3539                 goto out;
3540         }
3541
3542         if (info->flags & MSI_ALLOC_FLAGS_PROXY_DEVICE)
3543                 its_dev->shared = true;
3544
3545         pr_debug("ITT %d entries, %d bits\n", nvec, ilog2(nvec));
3546 out:
3547         mutex_unlock(&its->dev_alloc_lock);
3548         info->scratchpad[0].ptr = its_dev;
3549         return err;
3550 }
3551
3552 static struct msi_domain_ops its_msi_domain_ops = {
3553         .msi_prepare    = its_msi_prepare,
3554 };
3555
3556 static int its_irq_gic_domain_alloc(struct irq_domain *domain,
3557                                     unsigned int virq,
3558                                     irq_hw_number_t hwirq)
3559 {
3560         struct irq_fwspec fwspec;
3561
3562         if (irq_domain_get_of_node(domain->parent)) {
3563                 fwspec.fwnode = domain->parent->fwnode;
3564                 fwspec.param_count = 3;
3565                 fwspec.param[0] = GIC_IRQ_TYPE_LPI;
3566                 fwspec.param[1] = hwirq;
3567                 fwspec.param[2] = IRQ_TYPE_EDGE_RISING;
3568         } else if (is_fwnode_irqchip(domain->parent->fwnode)) {
3569                 fwspec.fwnode = domain->parent->fwnode;
3570                 fwspec.param_count = 2;
3571                 fwspec.param[0] = hwirq;
3572                 fwspec.param[1] = IRQ_TYPE_EDGE_RISING;
3573         } else {
3574                 return -EINVAL;
3575         }
3576
3577         return irq_domain_alloc_irqs_parent(domain, virq, 1, &fwspec);
3578 }
3579
3580 static int its_irq_domain_alloc(struct irq_domain *domain, unsigned int virq,
3581                                 unsigned int nr_irqs, void *args)
3582 {
3583         msi_alloc_info_t *info = args;
3584         struct its_device *its_dev = info->scratchpad[0].ptr;
3585         struct its_node *its = its_dev->its;
3586         struct irq_data *irqd;
3587         irq_hw_number_t hwirq;
3588         int err;
3589         int i;
3590
3591         err = its_alloc_device_irq(its_dev, nr_irqs, &hwirq);
3592         if (err)
3593                 return err;
3594
3595         err = iommu_dma_prepare_msi(info->desc, its->get_msi_base(its_dev));
3596         if (err)
3597                 return err;
3598
3599         for (i = 0; i < nr_irqs; i++) {
3600                 err = its_irq_gic_domain_alloc(domain, virq + i, hwirq + i);
3601                 if (err)
3602                         return err;
3603
3604                 irq_domain_set_hwirq_and_chip(domain, virq + i,
3605                                               hwirq + i, &its_irq_chip, its_dev);
3606                 irqd = irq_get_irq_data(virq + i);
3607                 irqd_set_single_target(irqd);
3608                 irqd_set_affinity_on_activate(irqd);
3609                 irqd_set_resend_when_in_progress(irqd);
3610                 pr_debug("ID:%d pID:%d vID:%d\n",
3611                          (int)(hwirq + i - its_dev->event_map.lpi_base),
3612                          (int)(hwirq + i), virq + i);
3613         }
3614
3615         return 0;
3616 }
3617
3618 static int its_irq_domain_activate(struct irq_domain *domain,
3619                                    struct irq_data *d, bool reserve)
3620 {
3621         struct its_device *its_dev = irq_data_get_irq_chip_data(d);
3622         u32 event = its_get_event_id(d);
3623         int cpu;
3624
3625         cpu = its_select_cpu(d, cpu_online_mask);
3626         if (cpu < 0 || cpu >= nr_cpu_ids)
3627                 return -EINVAL;
3628
3629         its_inc_lpi_count(d, cpu);
3630         its_dev->event_map.col_map[event] = cpu;
3631         irq_data_update_effective_affinity(d, cpumask_of(cpu));
3632
3633         /* Map the GIC IRQ and event to the device */
3634         its_send_mapti(its_dev, d->hwirq, event);
3635         return 0;
3636 }
3637
3638 static void its_irq_domain_deactivate(struct irq_domain *domain,
3639                                       struct irq_data *d)
3640 {
3641         struct its_device *its_dev = irq_data_get_irq_chip_data(d);
3642         u32 event = its_get_event_id(d);
3643
3644         its_dec_lpi_count(d, its_dev->event_map.col_map[event]);
3645         /* Stop the delivery of interrupts */
3646         its_send_discard(its_dev, event);
3647 }
3648
3649 static void its_irq_domain_free(struct irq_domain *domain, unsigned int virq,
3650                                 unsigned int nr_irqs)
3651 {
3652         struct irq_data *d = irq_domain_get_irq_data(domain, virq);
3653         struct its_device *its_dev = irq_data_get_irq_chip_data(d);
3654         struct its_node *its = its_dev->its;
3655         int i;
3656
3657         bitmap_release_region(its_dev->event_map.lpi_map,
3658                               its_get_event_id(irq_domain_get_irq_data(domain, virq)),
3659                               get_count_order(nr_irqs));
3660
3661         for (i = 0; i < nr_irqs; i++) {
3662                 struct irq_data *data = irq_domain_get_irq_data(domain,
3663                                                                 virq + i);
3664                 /* Nuke the entry in the domain */
3665                 irq_domain_reset_irq_data(data);
3666         }
3667
3668         mutex_lock(&its->dev_alloc_lock);
3669
3670         /*
3671          * If all interrupts have been freed, start mopping the
3672          * floor. This is conditioned on the device not being shared.
3673          */
3674         if (!its_dev->shared &&
3675             bitmap_empty(its_dev->event_map.lpi_map,
3676                          its_dev->event_map.nr_lpis)) {
3677                 its_lpi_free(its_dev->event_map.lpi_map,
3678                              its_dev->event_map.lpi_base,
3679                              its_dev->event_map.nr_lpis);
3680
3681                 /* Unmap device/itt */
3682                 its_send_mapd(its_dev, 0);
3683                 its_free_device(its_dev);
3684         }
3685
3686         mutex_unlock(&its->dev_alloc_lock);
3687
3688         irq_domain_free_irqs_parent(domain, virq, nr_irqs);
3689 }
3690
3691 static const struct irq_domain_ops its_domain_ops = {
3692         .alloc                  = its_irq_domain_alloc,
3693         .free                   = its_irq_domain_free,
3694         .activate               = its_irq_domain_activate,
3695         .deactivate             = its_irq_domain_deactivate,
3696 };
3697
3698 /*
3699  * This is insane.
3700  *
3701  * If a GICv4.0 doesn't implement Direct LPIs (which is extremely
3702  * likely), the only way to perform an invalidate is to use a fake
3703  * device to issue an INV command, implying that the LPI has first
3704  * been mapped to some event on that device. Since this is not exactly
3705  * cheap, we try to keep that mapping around as long as possible, and
3706  * only issue an UNMAP if we're short on available slots.
3707  *
3708  * Broken by design(tm).
3709  *
3710  * GICv4.1, on the other hand, mandates that we're able to invalidate
3711  * by writing to a MMIO register. It doesn't implement the whole of
3712  * DirectLPI, but that's good enough. And most of the time, we don't
3713  * even have to invalidate anything, as the redistributor can be told
3714  * whether to generate a doorbell or not (we thus leave it enabled,
3715  * always).
3716  */
3717 static void its_vpe_db_proxy_unmap_locked(struct its_vpe *vpe)
3718 {
3719         /* GICv4.1 doesn't use a proxy, so nothing to do here */
3720         if (gic_rdists->has_rvpeid)
3721                 return;
3722
3723         /* Already unmapped? */
3724         if (vpe->vpe_proxy_event == -1)
3725                 return;
3726
3727         its_send_discard(vpe_proxy.dev, vpe->vpe_proxy_event);
3728         vpe_proxy.vpes[vpe->vpe_proxy_event] = NULL;
3729
3730         /*
3731          * We don't track empty slots at all, so let's move the
3732          * next_victim pointer if we can quickly reuse that slot
3733          * instead of nuking an existing entry. Not clear that this is
3734          * always a win though, and this might just generate a ripple
3735          * effect... Let's just hope VPEs don't migrate too often.
3736          */
3737         if (vpe_proxy.vpes[vpe_proxy.next_victim])
3738                 vpe_proxy.next_victim = vpe->vpe_proxy_event;
3739
3740         vpe->vpe_proxy_event = -1;
3741 }
3742
3743 static void its_vpe_db_proxy_unmap(struct its_vpe *vpe)
3744 {
3745         /* GICv4.1 doesn't use a proxy, so nothing to do here */
3746         if (gic_rdists->has_rvpeid)
3747                 return;
3748
3749         if (!gic_rdists->has_direct_lpi) {
3750                 unsigned long flags;
3751
3752                 raw_spin_lock_irqsave(&vpe_proxy.lock, flags);
3753                 its_vpe_db_proxy_unmap_locked(vpe);
3754                 raw_spin_unlock_irqrestore(&vpe_proxy.lock, flags);
3755         }
3756 }
3757
3758 static void its_vpe_db_proxy_map_locked(struct its_vpe *vpe)
3759 {
3760         /* GICv4.1 doesn't use a proxy, so nothing to do here */
3761         if (gic_rdists->has_rvpeid)
3762                 return;
3763
3764         /* Already mapped? */
3765         if (vpe->vpe_proxy_event != -1)
3766                 return;
3767
3768         /* This slot was already allocated. Kick the other VPE out. */
3769         if (vpe_proxy.vpes[vpe_proxy.next_victim])
3770                 its_vpe_db_proxy_unmap_locked(vpe_proxy.vpes[vpe_proxy.next_victim]);
3771
3772         /* Map the new VPE instead */
3773         vpe_proxy.vpes[vpe_proxy.next_victim] = vpe;
3774         vpe->vpe_proxy_event = vpe_proxy.next_victim;
3775         vpe_proxy.next_victim = (vpe_proxy.next_victim + 1) % vpe_proxy.dev->nr_ites;
3776
3777         vpe_proxy.dev->event_map.col_map[vpe->vpe_proxy_event] = vpe->col_idx;
3778         its_send_mapti(vpe_proxy.dev, vpe->vpe_db_lpi, vpe->vpe_proxy_event);
3779 }
3780
3781 static void its_vpe_db_proxy_move(struct its_vpe *vpe, int from, int to)
3782 {
3783         unsigned long flags;
3784         struct its_collection *target_col;
3785
3786         /* GICv4.1 doesn't use a proxy, so nothing to do here */
3787         if (gic_rdists->has_rvpeid)
3788                 return;
3789
3790         if (gic_rdists->has_direct_lpi) {
3791                 void __iomem *rdbase;
3792
3793                 rdbase = per_cpu_ptr(gic_rdists->rdist, from)->rd_base;
3794                 gic_write_lpir(vpe->vpe_db_lpi, rdbase + GICR_CLRLPIR);
3795                 wait_for_syncr(rdbase);
3796
3797                 return;
3798         }
3799
3800         raw_spin_lock_irqsave(&vpe_proxy.lock, flags);
3801
3802         its_vpe_db_proxy_map_locked(vpe);
3803
3804         target_col = &vpe_proxy.dev->its->collections[to];
3805         its_send_movi(vpe_proxy.dev, target_col, vpe->vpe_proxy_event);
3806         vpe_proxy.dev->event_map.col_map[vpe->vpe_proxy_event] = to;
3807
3808         raw_spin_unlock_irqrestore(&vpe_proxy.lock, flags);
3809 }
3810
3811 static int its_vpe_set_affinity(struct irq_data *d,
3812                                 const struct cpumask *mask_val,
3813                                 bool force)
3814 {
3815         struct its_vpe *vpe = irq_data_get_irq_chip_data(d);
3816         int from, cpu = cpumask_first(mask_val);
3817         unsigned long flags;
3818
3819         /*
3820          * Changing affinity is mega expensive, so let's be as lazy as
3821          * we can and only do it if we really have to. Also, if mapped
3822          * into the proxy device, we need to move the doorbell
3823          * interrupt to its new location.
3824          *
3825          * Another thing is that changing the affinity of a vPE affects
3826          * *other interrupts* such as all the vLPIs that are routed to
3827          * this vPE. This means that the irq_desc lock is not enough to
3828          * protect us, and that we must ensure nobody samples vpe->col_idx
3829          * during the update, hence the lock below which must also be
3830          * taken on any vLPI handling path that evaluates vpe->col_idx.
3831          */
3832         from = vpe_to_cpuid_lock(vpe, &flags);
3833         if (from == cpu)
3834                 goto out;
3835
3836         vpe->col_idx = cpu;
3837
3838         /*
3839          * GICv4.1 allows us to skip VMOVP if moving to a cpu whose RD
3840          * is sharing its VPE table with the current one.
3841          */
3842         if (gic_data_rdist_cpu(cpu)->vpe_table_mask &&
3843             cpumask_test_cpu(from, gic_data_rdist_cpu(cpu)->vpe_table_mask))
3844                 goto out;
3845
3846         its_send_vmovp(vpe);
3847         its_vpe_db_proxy_move(vpe, from, cpu);
3848
3849 out:
3850         irq_data_update_effective_affinity(d, cpumask_of(cpu));
3851         vpe_to_cpuid_unlock(vpe, flags);
3852
3853         return IRQ_SET_MASK_OK_DONE;
3854 }
3855
3856 static void its_wait_vpt_parse_complete(void)
3857 {
3858         void __iomem *vlpi_base = gic_data_rdist_vlpi_base();
3859         u64 val;
3860
3861         if (!gic_rdists->has_vpend_valid_dirty)
3862                 return;
3863
3864         WARN_ON_ONCE(readq_relaxed_poll_timeout_atomic(vlpi_base + GICR_VPENDBASER,
3865                                                        val,
3866                                                        !(val & GICR_VPENDBASER_Dirty),
3867                                                        1, 500));
3868 }
3869
3870 static void its_vpe_schedule(struct its_vpe *vpe)
3871 {
3872         void __iomem *vlpi_base = gic_data_rdist_vlpi_base();
3873         u64 val;
3874
3875         /* Schedule the VPE */
3876         val  = virt_to_phys(page_address(vpe->its_vm->vprop_page)) &
3877                 GENMASK_ULL(51, 12);
3878         val |= (LPI_NRBITS - 1) & GICR_VPROPBASER_IDBITS_MASK;
3879         val |= GICR_VPROPBASER_RaWb;
3880         val |= GICR_VPROPBASER_InnerShareable;
3881         gicr_write_vpropbaser(val, vlpi_base + GICR_VPROPBASER);
3882
3883         val  = virt_to_phys(page_address(vpe->vpt_page)) &
3884                 GENMASK_ULL(51, 16);
3885         val |= GICR_VPENDBASER_RaWaWb;
3886         val |= GICR_VPENDBASER_InnerShareable;
3887         /*
3888          * There is no good way of finding out if the pending table is
3889          * empty as we can race against the doorbell interrupt very
3890          * easily. So in the end, vpe->pending_last is only an
3891          * indication that the vcpu has something pending, not one
3892          * that the pending table is empty. A good implementation
3893          * would be able to read its coarse map pretty quickly anyway,
3894          * making this a tolerable issue.
3895          */
3896         val |= GICR_VPENDBASER_PendingLast;
3897         val |= vpe->idai ? GICR_VPENDBASER_IDAI : 0;
3898         val |= GICR_VPENDBASER_Valid;
3899         gicr_write_vpendbaser(val, vlpi_base + GICR_VPENDBASER);
3900 }
3901
3902 static void its_vpe_deschedule(struct its_vpe *vpe)
3903 {
3904         void __iomem *vlpi_base = gic_data_rdist_vlpi_base();
3905         u64 val;
3906
3907         val = its_clear_vpend_valid(vlpi_base, 0, 0);
3908
3909         vpe->idai = !!(val & GICR_VPENDBASER_IDAI);
3910         vpe->pending_last = !!(val & GICR_VPENDBASER_PendingLast);
3911 }
3912
3913 static void its_vpe_invall(struct its_vpe *vpe)
3914 {
3915         struct its_node *its;
3916
3917         list_for_each_entry(its, &its_nodes, entry) {
3918                 if (!is_v4(its))
3919                         continue;
3920
3921                 if (its_list_map && !vpe->its_vm->vlpi_count[its->list_nr])
3922                         continue;
3923
3924                 /*
3925                  * Sending a VINVALL to a single ITS is enough, as all
3926                  * we need is to reach the redistributors.
3927                  */
3928                 its_send_vinvall(its, vpe);
3929                 return;
3930         }
3931 }
3932
3933 static int its_vpe_set_vcpu_affinity(struct irq_data *d, void *vcpu_info)
3934 {
3935         struct its_vpe *vpe = irq_data_get_irq_chip_data(d);
3936         struct its_cmd_info *info = vcpu_info;
3937
3938         switch (info->cmd_type) {
3939         case SCHEDULE_VPE:
3940                 its_vpe_schedule(vpe);
3941                 return 0;
3942
3943         case DESCHEDULE_VPE:
3944                 its_vpe_deschedule(vpe);
3945                 return 0;
3946
3947         case COMMIT_VPE:
3948                 its_wait_vpt_parse_complete();
3949                 return 0;
3950
3951         case INVALL_VPE:
3952                 its_vpe_invall(vpe);
3953                 return 0;
3954
3955         default:
3956                 return -EINVAL;
3957         }
3958 }
3959
3960 static void its_vpe_send_cmd(struct its_vpe *vpe,
3961                              void (*cmd)(struct its_device *, u32))
3962 {
3963         unsigned long flags;
3964
3965         raw_spin_lock_irqsave(&vpe_proxy.lock, flags);
3966
3967         its_vpe_db_proxy_map_locked(vpe);
3968         cmd(vpe_proxy.dev, vpe->vpe_proxy_event);
3969
3970         raw_spin_unlock_irqrestore(&vpe_proxy.lock, flags);
3971 }
3972
3973 static void its_vpe_send_inv(struct irq_data *d)
3974 {
3975         struct its_vpe *vpe = irq_data_get_irq_chip_data(d);
3976
3977         if (gic_rdists->has_direct_lpi)
3978                 __direct_lpi_inv(d, d->parent_data->hwirq);
3979         else
3980                 its_vpe_send_cmd(vpe, its_send_inv);
3981 }
3982
3983 static void its_vpe_mask_irq(struct irq_data *d)
3984 {
3985         /*
3986          * We need to unmask the LPI, which is described by the parent
3987          * irq_data. Instead of calling into the parent (which won't
3988          * exactly do the right thing, let's simply use the
3989          * parent_data pointer. Yes, I'm naughty.
3990          */
3991         lpi_write_config(d->parent_data, LPI_PROP_ENABLED, 0);
3992         its_vpe_send_inv(d);
3993 }
3994
3995 static void its_vpe_unmask_irq(struct irq_data *d)
3996 {
3997         /* Same hack as above... */
3998         lpi_write_config(d->parent_data, 0, LPI_PROP_ENABLED);
3999         its_vpe_send_inv(d);
4000 }
4001
4002 static int its_vpe_set_irqchip_state(struct irq_data *d,
4003                                      enum irqchip_irq_state which,
4004                                      bool state)
4005 {
4006         struct its_vpe *vpe = irq_data_get_irq_chip_data(d);
4007
4008         if (which != IRQCHIP_STATE_PENDING)
4009                 return -EINVAL;
4010
4011         if (gic_rdists->has_direct_lpi) {
4012                 void __iomem *rdbase;
4013
4014                 rdbase = per_cpu_ptr(gic_rdists->rdist, vpe->col_idx)->rd_base;
4015                 if (state) {
4016                         gic_write_lpir(vpe->vpe_db_lpi, rdbase + GICR_SETLPIR);
4017                 } else {
4018                         gic_write_lpir(vpe->vpe_db_lpi, rdbase + GICR_CLRLPIR);
4019                         wait_for_syncr(rdbase);
4020                 }
4021         } else {
4022                 if (state)
4023                         its_vpe_send_cmd(vpe, its_send_int);
4024                 else
4025                         its_vpe_send_cmd(vpe, its_send_clear);
4026         }
4027
4028         return 0;
4029 }
4030
4031 static int its_vpe_retrigger(struct irq_data *d)
4032 {
4033         return !its_vpe_set_irqchip_state(d, IRQCHIP_STATE_PENDING, true);
4034 }
4035
4036 static struct irq_chip its_vpe_irq_chip = {
4037         .name                   = "GICv4-vpe",
4038         .irq_mask               = its_vpe_mask_irq,
4039         .irq_unmask             = its_vpe_unmask_irq,
4040         .irq_eoi                = irq_chip_eoi_parent,
4041         .irq_set_affinity       = its_vpe_set_affinity,
4042         .irq_retrigger          = its_vpe_retrigger,
4043         .irq_set_irqchip_state  = its_vpe_set_irqchip_state,
4044         .irq_set_vcpu_affinity  = its_vpe_set_vcpu_affinity,
4045 };
4046
4047 static struct its_node *find_4_1_its(void)
4048 {
4049         static struct its_node *its = NULL;
4050
4051         if (!its) {
4052                 list_for_each_entry(its, &its_nodes, entry) {
4053                         if (is_v4_1(its))
4054                                 return its;
4055                 }
4056
4057                 /* Oops? */
4058                 its = NULL;
4059         }
4060
4061         return its;
4062 }
4063
4064 static void its_vpe_4_1_send_inv(struct irq_data *d)
4065 {
4066         struct its_vpe *vpe = irq_data_get_irq_chip_data(d);
4067         struct its_node *its;
4068
4069         /*
4070          * GICv4.1 wants doorbells to be invalidated using the
4071          * INVDB command in order to be broadcast to all RDs. Send
4072          * it to the first valid ITS, and let the HW do its magic.
4073          */
4074         its = find_4_1_its();
4075         if (its)
4076                 its_send_invdb(its, vpe);
4077 }
4078
4079 static void its_vpe_4_1_mask_irq(struct irq_data *d)
4080 {
4081         lpi_write_config(d->parent_data, LPI_PROP_ENABLED, 0);
4082         its_vpe_4_1_send_inv(d);
4083 }
4084
4085 static void its_vpe_4_1_unmask_irq(struct irq_data *d)
4086 {
4087         lpi_write_config(d->parent_data, 0, LPI_PROP_ENABLED);
4088         its_vpe_4_1_send_inv(d);
4089 }
4090
4091 static void its_vpe_4_1_schedule(struct its_vpe *vpe,
4092                                  struct its_cmd_info *info)
4093 {
4094         void __iomem *vlpi_base = gic_data_rdist_vlpi_base();
4095         u64 val = 0;
4096
4097         /* Schedule the VPE */
4098         val |= GICR_VPENDBASER_Valid;
4099         val |= info->g0en ? GICR_VPENDBASER_4_1_VGRP0EN : 0;
4100         val |= info->g1en ? GICR_VPENDBASER_4_1_VGRP1EN : 0;
4101         val |= FIELD_PREP(GICR_VPENDBASER_4_1_VPEID, vpe->vpe_id);
4102
4103         gicr_write_vpendbaser(val, vlpi_base + GICR_VPENDBASER);
4104 }
4105
4106 static void its_vpe_4_1_deschedule(struct its_vpe *vpe,
4107                                    struct its_cmd_info *info)
4108 {
4109         void __iomem *vlpi_base = gic_data_rdist_vlpi_base();
4110         u64 val;
4111
4112         if (info->req_db) {
4113                 unsigned long flags;
4114
4115                 /*
4116                  * vPE is going to block: make the vPE non-resident with
4117                  * PendingLast clear and DB set. The GIC guarantees that if
4118                  * we read-back PendingLast clear, then a doorbell will be
4119                  * delivered when an interrupt comes.
4120                  *
4121                  * Note the locking to deal with the concurrent update of
4122                  * pending_last from the doorbell interrupt handler that can
4123                  * run concurrently.
4124                  */
4125                 raw_spin_lock_irqsave(&vpe->vpe_lock, flags);
4126                 val = its_clear_vpend_valid(vlpi_base,
4127                                             GICR_VPENDBASER_PendingLast,
4128                                             GICR_VPENDBASER_4_1_DB);
4129                 vpe->pending_last = !!(val & GICR_VPENDBASER_PendingLast);
4130                 raw_spin_unlock_irqrestore(&vpe->vpe_lock, flags);
4131         } else {
4132                 /*
4133                  * We're not blocking, so just make the vPE non-resident
4134                  * with PendingLast set, indicating that we'll be back.
4135                  */
4136                 val = its_clear_vpend_valid(vlpi_base,
4137                                             0,
4138                                             GICR_VPENDBASER_PendingLast);
4139                 vpe->pending_last = true;
4140         }
4141 }
4142
4143 static void its_vpe_4_1_invall(struct its_vpe *vpe)
4144 {
4145         void __iomem *rdbase;
4146         unsigned long flags;
4147         u64 val;
4148         int cpu;
4149
4150         val  = GICR_INVALLR_V;
4151         val |= FIELD_PREP(GICR_INVALLR_VPEID, vpe->vpe_id);
4152
4153         /* Target the redistributor this vPE is currently known on */
4154         cpu = vpe_to_cpuid_lock(vpe, &flags);
4155         raw_spin_lock(&gic_data_rdist_cpu(cpu)->rd_lock);
4156         rdbase = per_cpu_ptr(gic_rdists->rdist, cpu)->rd_base;
4157         gic_write_lpir(val, rdbase + GICR_INVALLR);
4158
4159         wait_for_syncr(rdbase);
4160         raw_spin_unlock(&gic_data_rdist_cpu(cpu)->rd_lock);
4161         vpe_to_cpuid_unlock(vpe, flags);
4162 }
4163
4164 static int its_vpe_4_1_set_vcpu_affinity(struct irq_data *d, void *vcpu_info)
4165 {
4166         struct its_vpe *vpe = irq_data_get_irq_chip_data(d);
4167         struct its_cmd_info *info = vcpu_info;
4168
4169         switch (info->cmd_type) {
4170         case SCHEDULE_VPE:
4171                 its_vpe_4_1_schedule(vpe, info);
4172                 return 0;
4173
4174         case DESCHEDULE_VPE:
4175                 its_vpe_4_1_deschedule(vpe, info);
4176                 return 0;
4177
4178         case COMMIT_VPE:
4179                 its_wait_vpt_parse_complete();
4180                 return 0;
4181
4182         case INVALL_VPE:
4183                 its_vpe_4_1_invall(vpe);
4184                 return 0;
4185
4186         default:
4187                 return -EINVAL;
4188         }
4189 }
4190
4191 static struct irq_chip its_vpe_4_1_irq_chip = {
4192         .name                   = "GICv4.1-vpe",
4193         .irq_mask               = its_vpe_4_1_mask_irq,
4194         .irq_unmask             = its_vpe_4_1_unmask_irq,
4195         .irq_eoi                = irq_chip_eoi_parent,
4196         .irq_set_affinity       = its_vpe_set_affinity,
4197         .irq_set_vcpu_affinity  = its_vpe_4_1_set_vcpu_affinity,
4198 };
4199
4200 static void its_configure_sgi(struct irq_data *d, bool clear)
4201 {
4202         struct its_vpe *vpe = irq_data_get_irq_chip_data(d);
4203         struct its_cmd_desc desc;
4204
4205         desc.its_vsgi_cmd.vpe = vpe;
4206         desc.its_vsgi_cmd.sgi = d->hwirq;
4207         desc.its_vsgi_cmd.priority = vpe->sgi_config[d->hwirq].priority;
4208         desc.its_vsgi_cmd.enable = vpe->sgi_config[d->hwirq].enabled;
4209         desc.its_vsgi_cmd.group = vpe->sgi_config[d->hwirq].group;
4210         desc.its_vsgi_cmd.clear = clear;
4211
4212         /*
4213          * GICv4.1 allows us to send VSGI commands to any ITS as long as the
4214          * destination VPE is mapped there. Since we map them eagerly at
4215          * activation time, we're pretty sure the first GICv4.1 ITS will do.
4216          */
4217         its_send_single_vcommand(find_4_1_its(), its_build_vsgi_cmd, &desc);
4218 }
4219
4220 static void its_sgi_mask_irq(struct irq_data *d)
4221 {
4222         struct its_vpe *vpe = irq_data_get_irq_chip_data(d);
4223
4224         vpe->sgi_config[d->hwirq].enabled = false;
4225         its_configure_sgi(d, false);
4226 }
4227
4228 static void its_sgi_unmask_irq(struct irq_data *d)
4229 {
4230         struct its_vpe *vpe = irq_data_get_irq_chip_data(d);
4231
4232         vpe->sgi_config[d->hwirq].enabled = true;
4233         its_configure_sgi(d, false);
4234 }
4235
4236 static int its_sgi_set_affinity(struct irq_data *d,
4237                                 const struct cpumask *mask_val,
4238                                 bool force)
4239 {
4240         /*
4241          * There is no notion of affinity for virtual SGIs, at least
4242          * not on the host (since they can only be targeting a vPE).
4243          * Tell the kernel we've done whatever it asked for.
4244          */
4245         irq_data_update_effective_affinity(d, mask_val);
4246         return IRQ_SET_MASK_OK;
4247 }
4248
4249 static int its_sgi_set_irqchip_state(struct irq_data *d,
4250                                      enum irqchip_irq_state which,
4251                                      bool state)
4252 {
4253         if (which != IRQCHIP_STATE_PENDING)
4254                 return -EINVAL;
4255
4256         if (state) {
4257                 struct its_vpe *vpe = irq_data_get_irq_chip_data(d);
4258                 struct its_node *its = find_4_1_its();
4259                 u64 val;
4260
4261                 val  = FIELD_PREP(GITS_SGIR_VPEID, vpe->vpe_id);
4262                 val |= FIELD_PREP(GITS_SGIR_VINTID, d->hwirq);
4263                 writeq_relaxed(val, its->sgir_base + GITS_SGIR - SZ_128K);
4264         } else {
4265                 its_configure_sgi(d, true);
4266         }
4267
4268         return 0;
4269 }
4270
4271 static int its_sgi_get_irqchip_state(struct irq_data *d,
4272                                      enum irqchip_irq_state which, bool *val)
4273 {
4274         struct its_vpe *vpe = irq_data_get_irq_chip_data(d);
4275         void __iomem *base;
4276         unsigned long flags;
4277         u32 count = 1000000;    /* 1s! */
4278         u32 status;
4279         int cpu;
4280
4281         if (which != IRQCHIP_STATE_PENDING)
4282                 return -EINVAL;
4283
4284         /*
4285          * Locking galore! We can race against two different events:
4286          *
4287          * - Concurrent vPE affinity change: we must make sure it cannot
4288          *   happen, or we'll talk to the wrong redistributor. This is
4289          *   identical to what happens with vLPIs.
4290          *
4291          * - Concurrent VSGIPENDR access: As it involves accessing two
4292          *   MMIO registers, this must be made atomic one way or another.
4293          */
4294         cpu = vpe_to_cpuid_lock(vpe, &flags);
4295         raw_spin_lock(&gic_data_rdist_cpu(cpu)->rd_lock);
4296         base = gic_data_rdist_cpu(cpu)->rd_base + SZ_128K;
4297         writel_relaxed(vpe->vpe_id, base + GICR_VSGIR);
4298         do {
4299                 status = readl_relaxed(base + GICR_VSGIPENDR);
4300                 if (!(status & GICR_VSGIPENDR_BUSY))
4301                         goto out;
4302
4303                 count--;
4304                 if (!count) {
4305                         pr_err_ratelimited("Unable to get SGI status\n");
4306                         goto out;
4307                 }
4308                 cpu_relax();
4309                 udelay(1);
4310         } while (count);
4311
4312 out:
4313         raw_spin_unlock(&gic_data_rdist_cpu(cpu)->rd_lock);
4314         vpe_to_cpuid_unlock(vpe, flags);
4315
4316         if (!count)
4317                 return -ENXIO;
4318
4319         *val = !!(status & (1 << d->hwirq));
4320
4321         return 0;
4322 }
4323
4324 static int its_sgi_set_vcpu_affinity(struct irq_data *d, void *vcpu_info)
4325 {
4326         struct its_vpe *vpe = irq_data_get_irq_chip_data(d);
4327         struct its_cmd_info *info = vcpu_info;
4328
4329         switch (info->cmd_type) {
4330         case PROP_UPDATE_VSGI:
4331                 vpe->sgi_config[d->hwirq].priority = info->priority;
4332                 vpe->sgi_config[d->hwirq].group = info->group;
4333                 its_configure_sgi(d, false);
4334                 return 0;
4335
4336         default:
4337                 return -EINVAL;
4338         }
4339 }
4340
4341 static struct irq_chip its_sgi_irq_chip = {
4342         .name                   = "GICv4.1-sgi",
4343         .irq_mask               = its_sgi_mask_irq,
4344         .irq_unmask             = its_sgi_unmask_irq,
4345         .irq_set_affinity       = its_sgi_set_affinity,
4346         .irq_set_irqchip_state  = its_sgi_set_irqchip_state,
4347         .irq_get_irqchip_state  = its_sgi_get_irqchip_state,
4348         .irq_set_vcpu_affinity  = its_sgi_set_vcpu_affinity,
4349 };
4350
4351 static int its_sgi_irq_domain_alloc(struct irq_domain *domain,
4352                                     unsigned int virq, unsigned int nr_irqs,
4353                                     void *args)
4354 {
4355         struct its_vpe *vpe = args;
4356         int i;
4357
4358         /* Yes, we do want 16 SGIs */
4359         WARN_ON(nr_irqs != 16);
4360
4361         for (i = 0; i < 16; i++) {
4362                 vpe->sgi_config[i].priority = 0;
4363                 vpe->sgi_config[i].enabled = false;
4364                 vpe->sgi_config[i].group = false;
4365
4366                 irq_domain_set_hwirq_and_chip(domain, virq + i, i,
4367                                               &its_sgi_irq_chip, vpe);
4368                 irq_set_status_flags(virq + i, IRQ_DISABLE_UNLAZY);
4369         }
4370
4371         return 0;
4372 }
4373
4374 static void its_sgi_irq_domain_free(struct irq_domain *domain,
4375                                     unsigned int virq,
4376                                     unsigned int nr_irqs)
4377 {
4378         /* Nothing to do */
4379 }
4380
4381 static int its_sgi_irq_domain_activate(struct irq_domain *domain,
4382                                        struct irq_data *d, bool reserve)
4383 {
4384         /* Write out the initial SGI configuration */
4385         its_configure_sgi(d, false);
4386         return 0;
4387 }
4388
4389 static void its_sgi_irq_domain_deactivate(struct irq_domain *domain,
4390                                           struct irq_data *d)
4391 {
4392         struct its_vpe *vpe = irq_data_get_irq_chip_data(d);
4393
4394         /*
4395          * The VSGI command is awkward:
4396          *
4397          * - To change the configuration, CLEAR must be set to false,
4398          *   leaving the pending bit unchanged.
4399          * - To clear the pending bit, CLEAR must be set to true, leaving
4400          *   the configuration unchanged.
4401          *
4402          * You just can't do both at once, hence the two commands below.
4403          */
4404         vpe->sgi_config[d->hwirq].enabled = false;
4405         its_configure_sgi(d, false);
4406         its_configure_sgi(d, true);
4407 }
4408
4409 static const struct irq_domain_ops its_sgi_domain_ops = {
4410         .alloc          = its_sgi_irq_domain_alloc,
4411         .free           = its_sgi_irq_domain_free,
4412         .activate       = its_sgi_irq_domain_activate,
4413         .deactivate     = its_sgi_irq_domain_deactivate,
4414 };
4415
4416 static int its_vpe_id_alloc(void)
4417 {
4418         return ida_simple_get(&its_vpeid_ida, 0, ITS_MAX_VPEID, GFP_KERNEL);
4419 }
4420
4421 static void its_vpe_id_free(u16 id)
4422 {
4423         ida_simple_remove(&its_vpeid_ida, id);
4424 }
4425
4426 static int its_vpe_init(struct its_vpe *vpe)
4427 {
4428         struct page *vpt_page;
4429         int vpe_id;
4430
4431         /* Allocate vpe_id */
4432         vpe_id = its_vpe_id_alloc();
4433         if (vpe_id < 0)
4434                 return vpe_id;
4435
4436         /* Allocate VPT */
4437         vpt_page = its_allocate_pending_table(GFP_KERNEL);
4438         if (!vpt_page) {
4439                 its_vpe_id_free(vpe_id);
4440                 return -ENOMEM;
4441         }
4442
4443         if (!its_alloc_vpe_table(vpe_id)) {
4444                 its_vpe_id_free(vpe_id);
4445                 its_free_pending_table(vpt_page);
4446                 return -ENOMEM;
4447         }
4448
4449         raw_spin_lock_init(&vpe->vpe_lock);
4450         vpe->vpe_id = vpe_id;
4451         vpe->vpt_page = vpt_page;
4452         if (gic_rdists->has_rvpeid)
4453                 atomic_set(&vpe->vmapp_count, 0);
4454         else
4455                 vpe->vpe_proxy_event = -1;
4456
4457         return 0;
4458 }
4459
4460 static void its_vpe_teardown(struct its_vpe *vpe)
4461 {
4462         its_vpe_db_proxy_unmap(vpe);
4463         its_vpe_id_free(vpe->vpe_id);
4464         its_free_pending_table(vpe->vpt_page);
4465 }
4466
4467 static void its_vpe_irq_domain_free(struct irq_domain *domain,
4468                                     unsigned int virq,
4469                                     unsigned int nr_irqs)
4470 {
4471         struct its_vm *vm = domain->host_data;
4472         int i;
4473
4474         irq_domain_free_irqs_parent(domain, virq, nr_irqs);
4475
4476         for (i = 0; i < nr_irqs; i++) {
4477                 struct irq_data *data = irq_domain_get_irq_data(domain,
4478                                                                 virq + i);
4479                 struct its_vpe *vpe = irq_data_get_irq_chip_data(data);
4480
4481                 BUG_ON(vm != vpe->its_vm);
4482
4483                 clear_bit(data->hwirq, vm->db_bitmap);
4484                 its_vpe_teardown(vpe);
4485                 irq_domain_reset_irq_data(data);
4486         }
4487
4488         if (bitmap_empty(vm->db_bitmap, vm->nr_db_lpis)) {
4489                 its_lpi_free(vm->db_bitmap, vm->db_lpi_base, vm->nr_db_lpis);
4490                 its_free_prop_table(vm->vprop_page);
4491         }
4492 }
4493
4494 static int its_vpe_irq_domain_alloc(struct irq_domain *domain, unsigned int virq,
4495                                     unsigned int nr_irqs, void *args)
4496 {
4497         struct irq_chip *irqchip = &its_vpe_irq_chip;
4498         struct its_vm *vm = args;
4499         unsigned long *bitmap;
4500         struct page *vprop_page;
4501         int base, nr_ids, i, err = 0;
4502
4503         BUG_ON(!vm);
4504
4505         bitmap = its_lpi_alloc(roundup_pow_of_two(nr_irqs), &base, &nr_ids);
4506         if (!bitmap)
4507                 return -ENOMEM;
4508
4509         if (nr_ids < nr_irqs) {
4510                 its_lpi_free(bitmap, base, nr_ids);
4511                 return -ENOMEM;
4512         }
4513
4514         vprop_page = its_allocate_prop_table(GFP_KERNEL);
4515         if (!vprop_page) {
4516                 its_lpi_free(bitmap, base, nr_ids);
4517                 return -ENOMEM;
4518         }
4519
4520         vm->db_bitmap = bitmap;
4521         vm->db_lpi_base = base;
4522         vm->nr_db_lpis = nr_ids;
4523         vm->vprop_page = vprop_page;
4524
4525         if (gic_rdists->has_rvpeid)
4526                 irqchip = &its_vpe_4_1_irq_chip;
4527
4528         for (i = 0; i < nr_irqs; i++) {
4529                 vm->vpes[i]->vpe_db_lpi = base + i;
4530                 err = its_vpe_init(vm->vpes[i]);
4531                 if (err)
4532                         break;
4533                 err = its_irq_gic_domain_alloc(domain, virq + i,
4534                                                vm->vpes[i]->vpe_db_lpi);
4535                 if (err)
4536                         break;
4537                 irq_domain_set_hwirq_and_chip(domain, virq + i, i,
4538                                               irqchip, vm->vpes[i]);
4539                 set_bit(i, bitmap);
4540                 irqd_set_resend_when_in_progress(irq_get_irq_data(virq + i));
4541         }
4542
4543         if (err) {
4544                 if (i > 0)
4545                         its_vpe_irq_domain_free(domain, virq, i);
4546
4547                 its_lpi_free(bitmap, base, nr_ids);
4548                 its_free_prop_table(vprop_page);
4549         }
4550
4551         return err;
4552 }
4553
4554 static int its_vpe_irq_domain_activate(struct irq_domain *domain,
4555                                        struct irq_data *d, bool reserve)
4556 {
4557         struct its_vpe *vpe = irq_data_get_irq_chip_data(d);
4558         struct its_node *its;
4559
4560         /*
4561          * If we use the list map, we issue VMAPP on demand... Unless
4562          * we're on a GICv4.1 and we eagerly map the VPE on all ITSs
4563          * so that VSGIs can work.
4564          */
4565         if (!gic_requires_eager_mapping())
4566                 return 0;
4567
4568         /* Map the VPE to the first possible CPU */
4569         vpe->col_idx = cpumask_first(cpu_online_mask);
4570
4571         list_for_each_entry(its, &its_nodes, entry) {
4572                 if (!is_v4(its))
4573                         continue;
4574
4575                 its_send_vmapp(its, vpe, true);
4576                 its_send_vinvall(its, vpe);
4577         }
4578
4579         irq_data_update_effective_affinity(d, cpumask_of(vpe->col_idx));
4580
4581         return 0;
4582 }
4583
4584 static void its_vpe_irq_domain_deactivate(struct irq_domain *domain,
4585                                           struct irq_data *d)
4586 {
4587         struct its_vpe *vpe = irq_data_get_irq_chip_data(d);
4588         struct its_node *its;
4589
4590         /*
4591          * If we use the list map on GICv4.0, we unmap the VPE once no
4592          * VLPIs are associated with the VM.
4593          */
4594         if (!gic_requires_eager_mapping())
4595                 return;
4596
4597         list_for_each_entry(its, &its_nodes, entry) {
4598                 if (!is_v4(its))
4599                         continue;
4600
4601                 its_send_vmapp(its, vpe, false);
4602         }
4603
4604         /*
4605          * There may be a direct read to the VPT after unmapping the
4606          * vPE, to guarantee the validity of this, we make the VPT
4607          * memory coherent with the CPU caches here.
4608          */
4609         if (find_4_1_its() && !atomic_read(&vpe->vmapp_count))
4610                 gic_flush_dcache_to_poc(page_address(vpe->vpt_page),
4611                                         LPI_PENDBASE_SZ);
4612 }
4613
4614 static const struct irq_domain_ops its_vpe_domain_ops = {
4615         .alloc                  = its_vpe_irq_domain_alloc,
4616         .free                   = its_vpe_irq_domain_free,
4617         .activate               = its_vpe_irq_domain_activate,
4618         .deactivate             = its_vpe_irq_domain_deactivate,
4619 };
4620
4621 static int its_force_quiescent(void __iomem *base)
4622 {
4623         u32 count = 1000000;    /* 1s */
4624         u32 val;
4625
4626         val = readl_relaxed(base + GITS_CTLR);
4627         /*
4628          * GIC architecture specification requires the ITS to be both
4629          * disabled and quiescent for writes to GITS_BASER<n> or
4630          * GITS_CBASER to not have UNPREDICTABLE results.
4631          */
4632         if ((val & GITS_CTLR_QUIESCENT) && !(val & GITS_CTLR_ENABLE))
4633                 return 0;
4634
4635         /* Disable the generation of all interrupts to this ITS */
4636         val &= ~(GITS_CTLR_ENABLE | GITS_CTLR_ImDe);
4637         writel_relaxed(val, base + GITS_CTLR);
4638
4639         /* Poll GITS_CTLR and wait until ITS becomes quiescent */
4640         while (1) {
4641                 val = readl_relaxed(base + GITS_CTLR);
4642                 if (val & GITS_CTLR_QUIESCENT)
4643                         return 0;
4644
4645                 count--;
4646                 if (!count)
4647                         return -EBUSY;
4648
4649                 cpu_relax();
4650                 udelay(1);
4651         }
4652 }
4653
4654 static bool __maybe_unused its_enable_quirk_cavium_22375(void *data)
4655 {
4656         struct its_node *its = data;
4657
4658         /* erratum 22375: only alloc 8MB table size (20 bits) */
4659         its->typer &= ~GITS_TYPER_DEVBITS;
4660         its->typer |= FIELD_PREP(GITS_TYPER_DEVBITS, 20 - 1);
4661         its->flags |= ITS_FLAGS_WORKAROUND_CAVIUM_22375;
4662
4663         return true;
4664 }
4665
4666 static bool __maybe_unused its_enable_quirk_cavium_23144(void *data)
4667 {
4668         struct its_node *its = data;
4669
4670         its->flags |= ITS_FLAGS_WORKAROUND_CAVIUM_23144;
4671
4672         return true;
4673 }
4674
4675 static bool __maybe_unused its_enable_quirk_qdf2400_e0065(void *data)
4676 {
4677         struct its_node *its = data;
4678
4679         /* On QDF2400, the size of the ITE is 16Bytes */
4680         its->typer &= ~GITS_TYPER_ITT_ENTRY_SIZE;
4681         its->typer |= FIELD_PREP(GITS_TYPER_ITT_ENTRY_SIZE, 16 - 1);
4682
4683         return true;
4684 }
4685
4686 static u64 its_irq_get_msi_base_pre_its(struct its_device *its_dev)
4687 {
4688         struct its_node *its = its_dev->its;
4689
4690         /*
4691          * The Socionext Synquacer SoC has a so-called 'pre-ITS',
4692          * which maps 32-bit writes targeted at a separate window of
4693          * size '4 << device_id_bits' onto writes to GITS_TRANSLATER
4694          * with device ID taken from bits [device_id_bits + 1:2] of
4695          * the window offset.
4696          */
4697         return its->pre_its_base + (its_dev->device_id << 2);
4698 }
4699
4700 static bool __maybe_unused its_enable_quirk_socionext_synquacer(void *data)
4701 {
4702         struct its_node *its = data;
4703         u32 pre_its_window[2];
4704         u32 ids;
4705
4706         if (!fwnode_property_read_u32_array(its->fwnode_handle,
4707                                            "socionext,synquacer-pre-its",
4708                                            pre_its_window,
4709                                            ARRAY_SIZE(pre_its_window))) {
4710
4711                 its->pre_its_base = pre_its_window[0];
4712                 its->get_msi_base = its_irq_get_msi_base_pre_its;
4713
4714                 ids = ilog2(pre_its_window[1]) - 2;
4715                 if (device_ids(its) > ids) {
4716                         its->typer &= ~GITS_TYPER_DEVBITS;
4717                         its->typer |= FIELD_PREP(GITS_TYPER_DEVBITS, ids - 1);
4718                 }
4719
4720                 /* the pre-ITS breaks isolation, so disable MSI remapping */
4721                 its->msi_domain_flags &= ~IRQ_DOMAIN_FLAG_ISOLATED_MSI;
4722                 return true;
4723         }
4724         return false;
4725 }
4726
4727 static bool __maybe_unused its_enable_quirk_hip07_161600802(void *data)
4728 {
4729         struct its_node *its = data;
4730
4731         /*
4732          * Hip07 insists on using the wrong address for the VLPI
4733          * page. Trick it into doing the right thing...
4734          */
4735         its->vlpi_redist_offset = SZ_128K;
4736         return true;
4737 }
4738
4739 static bool __maybe_unused its_enable_rk3588001(void *data)
4740 {
4741         struct its_node *its = data;
4742
4743         if (!of_machine_is_compatible("rockchip,rk3588") &&
4744             !of_machine_is_compatible("rockchip,rk3588s"))
4745                 return false;
4746
4747         its->flags |= ITS_FLAGS_FORCE_NON_SHAREABLE;
4748         gic_rdists->flags |= RDIST_FLAGS_FORCE_NON_SHAREABLE;
4749
4750         return true;
4751 }
4752
4753 static bool its_set_non_coherent(void *data)
4754 {
4755         struct its_node *its = data;
4756
4757         its->flags |= ITS_FLAGS_FORCE_NON_SHAREABLE;
4758         return true;
4759 }
4760
4761 static const struct gic_quirk its_quirks[] = {
4762 #ifdef CONFIG_CAVIUM_ERRATUM_22375
4763         {
4764                 .desc   = "ITS: Cavium errata 22375, 24313",
4765                 .iidr   = 0xa100034c,   /* ThunderX pass 1.x */
4766                 .mask   = 0xffff0fff,
4767                 .init   = its_enable_quirk_cavium_22375,
4768         },
4769 #endif
4770 #ifdef CONFIG_CAVIUM_ERRATUM_23144
4771         {
4772                 .desc   = "ITS: Cavium erratum 23144",
4773                 .iidr   = 0xa100034c,   /* ThunderX pass 1.x */
4774                 .mask   = 0xffff0fff,
4775                 .init   = its_enable_quirk_cavium_23144,
4776         },
4777 #endif
4778 #ifdef CONFIG_QCOM_QDF2400_ERRATUM_0065
4779         {
4780                 .desc   = "ITS: QDF2400 erratum 0065",
4781                 .iidr   = 0x00001070, /* QDF2400 ITS rev 1.x */
4782                 .mask   = 0xffffffff,
4783                 .init   = its_enable_quirk_qdf2400_e0065,
4784         },
4785 #endif
4786 #ifdef CONFIG_SOCIONEXT_SYNQUACER_PREITS
4787         {
4788                 /*
4789                  * The Socionext Synquacer SoC incorporates ARM's own GIC-500
4790                  * implementation, but with a 'pre-ITS' added that requires
4791                  * special handling in software.
4792                  */
4793                 .desc   = "ITS: Socionext Synquacer pre-ITS",
4794                 .iidr   = 0x0001143b,
4795                 .mask   = 0xffffffff,
4796                 .init   = its_enable_quirk_socionext_synquacer,
4797         },
4798 #endif
4799 #ifdef CONFIG_HISILICON_ERRATUM_161600802
4800         {
4801                 .desc   = "ITS: Hip07 erratum 161600802",
4802                 .iidr   = 0x00000004,
4803                 .mask   = 0xffffffff,
4804                 .init   = its_enable_quirk_hip07_161600802,
4805         },
4806 #endif
4807 #ifdef CONFIG_ROCKCHIP_ERRATUM_3588001
4808         {
4809                 .desc   = "ITS: Rockchip erratum RK3588001",
4810                 .iidr   = 0x0201743b,
4811                 .mask   = 0xffffffff,
4812                 .init   = its_enable_rk3588001,
4813         },
4814 #endif
4815         {
4816                 .desc   = "ITS: non-coherent attribute",
4817                 .property = "dma-noncoherent",
4818                 .init   = its_set_non_coherent,
4819         },
4820         {
4821         }
4822 };
4823
4824 static void its_enable_quirks(struct its_node *its)
4825 {
4826         u32 iidr = readl_relaxed(its->base + GITS_IIDR);
4827
4828         gic_enable_quirks(iidr, its_quirks, its);
4829
4830         if (is_of_node(its->fwnode_handle))
4831                 gic_enable_of_quirks(to_of_node(its->fwnode_handle),
4832                                      its_quirks, its);
4833 }
4834
4835 static int its_save_disable(void)
4836 {
4837         struct its_node *its;
4838         int err = 0;
4839
4840         raw_spin_lock(&its_lock);
4841         list_for_each_entry(its, &its_nodes, entry) {
4842                 void __iomem *base;
4843
4844                 base = its->base;
4845                 its->ctlr_save = readl_relaxed(base + GITS_CTLR);
4846                 err = its_force_quiescent(base);
4847                 if (err) {
4848                         pr_err("ITS@%pa: failed to quiesce: %d\n",
4849                                &its->phys_base, err);
4850                         writel_relaxed(its->ctlr_save, base + GITS_CTLR);
4851                         goto err;
4852                 }
4853
4854                 its->cbaser_save = gits_read_cbaser(base + GITS_CBASER);
4855         }
4856
4857 err:
4858         if (err) {
4859                 list_for_each_entry_continue_reverse(its, &its_nodes, entry) {
4860                         void __iomem *base;
4861
4862                         base = its->base;
4863                         writel_relaxed(its->ctlr_save, base + GITS_CTLR);
4864                 }
4865         }
4866         raw_spin_unlock(&its_lock);
4867
4868         return err;
4869 }
4870
4871 static void its_restore_enable(void)
4872 {
4873         struct its_node *its;
4874         int ret;
4875
4876         raw_spin_lock(&its_lock);
4877         list_for_each_entry(its, &its_nodes, entry) {
4878                 void __iomem *base;
4879                 int i;
4880
4881                 base = its->base;
4882
4883                 /*
4884                  * Make sure that the ITS is disabled. If it fails to quiesce,
4885                  * don't restore it since writing to CBASER or BASER<n>
4886                  * registers is undefined according to the GIC v3 ITS
4887                  * Specification.
4888                  *
4889                  * Firmware resuming with the ITS enabled is terminally broken.
4890                  */
4891                 WARN_ON(readl_relaxed(base + GITS_CTLR) & GITS_CTLR_ENABLE);
4892                 ret = its_force_quiescent(base);
4893                 if (ret) {
4894                         pr_err("ITS@%pa: failed to quiesce on resume: %d\n",
4895                                &its->phys_base, ret);
4896                         continue;
4897                 }
4898
4899                 gits_write_cbaser(its->cbaser_save, base + GITS_CBASER);
4900
4901                 /*
4902                  * Writing CBASER resets CREADR to 0, so make CWRITER and
4903                  * cmd_write line up with it.
4904                  */
4905                 its->cmd_write = its->cmd_base;
4906                 gits_write_cwriter(0, base + GITS_CWRITER);
4907
4908                 /* Restore GITS_BASER from the value cache. */
4909                 for (i = 0; i < GITS_BASER_NR_REGS; i++) {
4910                         struct its_baser *baser = &its->tables[i];
4911
4912                         if (!(baser->val & GITS_BASER_VALID))
4913                                 continue;
4914
4915                         its_write_baser(its, baser, baser->val);
4916                 }
4917                 writel_relaxed(its->ctlr_save, base + GITS_CTLR);
4918
4919                 /*
4920                  * Reinit the collection if it's stored in the ITS. This is
4921                  * indicated by the col_id being less than the HCC field.
4922                  * CID < HCC as specified in the GIC v3 Documentation.
4923                  */
4924                 if (its->collections[smp_processor_id()].col_id <
4925                     GITS_TYPER_HCC(gic_read_typer(base + GITS_TYPER)))
4926                         its_cpu_init_collection(its);
4927         }
4928         raw_spin_unlock(&its_lock);
4929 }
4930
4931 static struct syscore_ops its_syscore_ops = {
4932         .suspend = its_save_disable,
4933         .resume = its_restore_enable,
4934 };
4935
4936 static void __init __iomem *its_map_one(struct resource *res, int *err)
4937 {
4938         void __iomem *its_base;
4939         u32 val;
4940
4941         its_base = ioremap(res->start, SZ_64K);
4942         if (!its_base) {
4943                 pr_warn("ITS@%pa: Unable to map ITS registers\n", &res->start);
4944                 *err = -ENOMEM;
4945                 return NULL;
4946         }
4947
4948         val = readl_relaxed(its_base + GITS_PIDR2) & GIC_PIDR2_ARCH_MASK;
4949         if (val != 0x30 && val != 0x40) {
4950                 pr_warn("ITS@%pa: No ITS detected, giving up\n", &res->start);
4951                 *err = -ENODEV;
4952                 goto out_unmap;
4953         }
4954
4955         *err = its_force_quiescent(its_base);
4956         if (*err) {
4957                 pr_warn("ITS@%pa: Failed to quiesce, giving up\n", &res->start);
4958                 goto out_unmap;
4959         }
4960
4961         return its_base;
4962
4963 out_unmap:
4964         iounmap(its_base);
4965         return NULL;
4966 }
4967
4968 static int its_init_domain(struct its_node *its)
4969 {
4970         struct irq_domain *inner_domain;
4971         struct msi_domain_info *info;
4972
4973         info = kzalloc(sizeof(*info), GFP_KERNEL);
4974         if (!info)
4975                 return -ENOMEM;
4976
4977         info->ops = &its_msi_domain_ops;
4978         info->data = its;
4979
4980         inner_domain = irq_domain_create_hierarchy(its_parent,
4981                                                    its->msi_domain_flags, 0,
4982                                                    its->fwnode_handle, &its_domain_ops,
4983                                                    info);
4984         if (!inner_domain) {
4985                 kfree(info);
4986                 return -ENOMEM;
4987         }
4988
4989         irq_domain_update_bus_token(inner_domain, DOMAIN_BUS_NEXUS);
4990
4991         return 0;
4992 }
4993
4994 static int its_init_vpe_domain(void)
4995 {
4996         struct its_node *its;
4997         u32 devid;
4998         int entries;
4999
5000         if (gic_rdists->has_direct_lpi) {
5001                 pr_info("ITS: Using DirectLPI for VPE invalidation\n");
5002                 return 0;
5003         }
5004
5005         /* Any ITS will do, even if not v4 */
5006         its = list_first_entry(&its_nodes, struct its_node, entry);
5007
5008         entries = roundup_pow_of_two(nr_cpu_ids);
5009         vpe_proxy.vpes = kcalloc(entries, sizeof(*vpe_proxy.vpes),
5010                                  GFP_KERNEL);
5011         if (!vpe_proxy.vpes)
5012                 return -ENOMEM;
5013
5014         /* Use the last possible DevID */
5015         devid = GENMASK(device_ids(its) - 1, 0);
5016         vpe_proxy.dev = its_create_device(its, devid, entries, false);
5017         if (!vpe_proxy.dev) {
5018                 kfree(vpe_proxy.vpes);
5019                 pr_err("ITS: Can't allocate GICv4 proxy device\n");
5020                 return -ENOMEM;
5021         }
5022
5023         BUG_ON(entries > vpe_proxy.dev->nr_ites);
5024
5025         raw_spin_lock_init(&vpe_proxy.lock);
5026         vpe_proxy.next_victim = 0;
5027         pr_info("ITS: Allocated DevID %x as GICv4 proxy device (%d slots)\n",
5028                 devid, vpe_proxy.dev->nr_ites);
5029
5030         return 0;
5031 }
5032
5033 static int __init its_compute_its_list_map(struct its_node *its)
5034 {
5035         int its_number;
5036         u32 ctlr;
5037
5038         /*
5039          * This is assumed to be done early enough that we're
5040          * guaranteed to be single-threaded, hence no
5041          * locking. Should this change, we should address
5042          * this.
5043          */
5044         its_number = find_first_zero_bit(&its_list_map, GICv4_ITS_LIST_MAX);
5045         if (its_number >= GICv4_ITS_LIST_MAX) {
5046                 pr_err("ITS@%pa: No ITSList entry available!\n",
5047                        &its->phys_base);
5048                 return -EINVAL;
5049         }
5050
5051         ctlr = readl_relaxed(its->base + GITS_CTLR);
5052         ctlr &= ~GITS_CTLR_ITS_NUMBER;
5053         ctlr |= its_number << GITS_CTLR_ITS_NUMBER_SHIFT;
5054         writel_relaxed(ctlr, its->base + GITS_CTLR);
5055         ctlr = readl_relaxed(its->base + GITS_CTLR);
5056         if ((ctlr & GITS_CTLR_ITS_NUMBER) != (its_number << GITS_CTLR_ITS_NUMBER_SHIFT)) {
5057                 its_number = ctlr & GITS_CTLR_ITS_NUMBER;
5058                 its_number >>= GITS_CTLR_ITS_NUMBER_SHIFT;
5059         }
5060
5061         if (test_and_set_bit(its_number, &its_list_map)) {
5062                 pr_err("ITS@%pa: Duplicate ITSList entry %d\n",
5063                        &its->phys_base, its_number);
5064                 return -EINVAL;
5065         }
5066
5067         return its_number;
5068 }
5069
5070 static int __init its_probe_one(struct its_node *its)
5071 {
5072         u64 baser, tmp;
5073         struct page *page;
5074         u32 ctlr;
5075         int err;
5076
5077         if (is_v4(its)) {
5078                 if (!(its->typer & GITS_TYPER_VMOVP)) {
5079                         err = its_compute_its_list_map(its);
5080                         if (err < 0)
5081                                 goto out;
5082
5083                         its->list_nr = err;
5084
5085                         pr_info("ITS@%pa: Using ITS number %d\n",
5086                                 &its->phys_base, err);
5087                 } else {
5088                         pr_info("ITS@%pa: Single VMOVP capable\n", &its->phys_base);
5089                 }
5090
5091                 if (is_v4_1(its)) {
5092                         u32 svpet = FIELD_GET(GITS_TYPER_SVPET, its->typer);
5093
5094                         its->sgir_base = ioremap(its->phys_base + SZ_128K, SZ_64K);
5095                         if (!its->sgir_base) {
5096                                 err = -ENOMEM;
5097                                 goto out;
5098                         }
5099
5100                         its->mpidr = readl_relaxed(its->base + GITS_MPIDR);
5101
5102                         pr_info("ITS@%pa: Using GICv4.1 mode %08x %08x\n",
5103                                 &its->phys_base, its->mpidr, svpet);
5104                 }
5105         }
5106
5107         page = alloc_pages_node(its->numa_node, GFP_KERNEL | __GFP_ZERO,
5108                                 get_order(ITS_CMD_QUEUE_SZ));
5109         if (!page) {
5110                 err = -ENOMEM;
5111                 goto out_unmap_sgir;
5112         }
5113         its->cmd_base = (void *)page_address(page);
5114         its->cmd_write = its->cmd_base;
5115         its->get_msi_base = its_irq_get_msi_base;
5116         its->msi_domain_flags = IRQ_DOMAIN_FLAG_ISOLATED_MSI;
5117
5118         err = its_alloc_tables(its);
5119         if (err)
5120                 goto out_free_cmd;
5121
5122         err = its_alloc_collections(its);
5123         if (err)
5124                 goto out_free_tables;
5125
5126         baser = (virt_to_phys(its->cmd_base)    |
5127                  GITS_CBASER_RaWaWb             |
5128                  GITS_CBASER_InnerShareable     |
5129                  (ITS_CMD_QUEUE_SZ / SZ_4K - 1) |
5130                  GITS_CBASER_VALID);
5131
5132         gits_write_cbaser(baser, its->base + GITS_CBASER);
5133         tmp = gits_read_cbaser(its->base + GITS_CBASER);
5134
5135         if (its->flags & ITS_FLAGS_FORCE_NON_SHAREABLE)
5136                 tmp &= ~GITS_CBASER_SHAREABILITY_MASK;
5137
5138         if ((tmp ^ baser) & GITS_CBASER_SHAREABILITY_MASK) {
5139                 if (!(tmp & GITS_CBASER_SHAREABILITY_MASK)) {
5140                         /*
5141                          * The HW reports non-shareable, we must
5142                          * remove the cacheability attributes as
5143                          * well.
5144                          */
5145                         baser &= ~(GITS_CBASER_SHAREABILITY_MASK |
5146                                    GITS_CBASER_CACHEABILITY_MASK);
5147                         baser |= GITS_CBASER_nC;
5148                         gits_write_cbaser(baser, its->base + GITS_CBASER);
5149                 }
5150                 pr_info("ITS: using cache flushing for cmd queue\n");
5151                 its->flags |= ITS_FLAGS_CMDQ_NEEDS_FLUSHING;
5152         }
5153
5154         gits_write_cwriter(0, its->base + GITS_CWRITER);
5155         ctlr = readl_relaxed(its->base + GITS_CTLR);
5156         ctlr |= GITS_CTLR_ENABLE;
5157         if (is_v4(its))
5158                 ctlr |= GITS_CTLR_ImDe;
5159         writel_relaxed(ctlr, its->base + GITS_CTLR);
5160
5161         err = its_init_domain(its);
5162         if (err)
5163                 goto out_free_tables;
5164
5165         raw_spin_lock(&its_lock);
5166         list_add(&its->entry, &its_nodes);
5167         raw_spin_unlock(&its_lock);
5168
5169         return 0;
5170
5171 out_free_tables:
5172         its_free_tables(its);
5173 out_free_cmd:
5174         free_pages((unsigned long)its->cmd_base, get_order(ITS_CMD_QUEUE_SZ));
5175 out_unmap_sgir:
5176         if (its->sgir_base)
5177                 iounmap(its->sgir_base);
5178 out:
5179         pr_err("ITS@%pa: failed probing (%d)\n", &its->phys_base, err);
5180         return err;
5181 }
5182
5183 static bool gic_rdists_supports_plpis(void)
5184 {
5185         return !!(gic_read_typer(gic_data_rdist_rd_base() + GICR_TYPER) & GICR_TYPER_PLPIS);
5186 }
5187
5188 static int redist_disable_lpis(void)
5189 {
5190         void __iomem *rbase = gic_data_rdist_rd_base();
5191         u64 timeout = USEC_PER_SEC;
5192         u64 val;
5193
5194         if (!gic_rdists_supports_plpis()) {
5195                 pr_info("CPU%d: LPIs not supported\n", smp_processor_id());
5196                 return -ENXIO;
5197         }
5198
5199         val = readl_relaxed(rbase + GICR_CTLR);
5200         if (!(val & GICR_CTLR_ENABLE_LPIS))
5201                 return 0;
5202
5203         /*
5204          * If coming via a CPU hotplug event, we don't need to disable
5205          * LPIs before trying to re-enable them. They are already
5206          * configured and all is well in the world.
5207          *
5208          * If running with preallocated tables, there is nothing to do.
5209          */
5210         if ((gic_data_rdist()->flags & RD_LOCAL_LPI_ENABLED) ||
5211             (gic_rdists->flags & RDIST_FLAGS_RD_TABLES_PREALLOCATED))
5212                 return 0;
5213
5214         /*
5215          * From that point on, we only try to do some damage control.
5216          */
5217         pr_warn("GICv3: CPU%d: Booted with LPIs enabled, memory probably corrupted\n",
5218                 smp_processor_id());
5219         add_taint(TAINT_CRAP, LOCKDEP_STILL_OK);
5220
5221         /* Disable LPIs */
5222         val &= ~GICR_CTLR_ENABLE_LPIS;
5223         writel_relaxed(val, rbase + GICR_CTLR);
5224
5225         /* Make sure any change to GICR_CTLR is observable by the GIC */
5226         dsb(sy);
5227
5228         /*
5229          * Software must observe RWP==0 after clearing GICR_CTLR.EnableLPIs
5230          * from 1 to 0 before programming GICR_PEND{PROP}BASER registers.
5231          * Error out if we time out waiting for RWP to clear.
5232          */
5233         while (readl_relaxed(rbase + GICR_CTLR) & GICR_CTLR_RWP) {
5234                 if (!timeout) {
5235                         pr_err("CPU%d: Timeout while disabling LPIs\n",
5236                                smp_processor_id());
5237                         return -ETIMEDOUT;
5238                 }
5239                 udelay(1);
5240                 timeout--;
5241         }
5242
5243         /*
5244          * After it has been written to 1, it is IMPLEMENTATION
5245          * DEFINED whether GICR_CTLR.EnableLPI becomes RES1 or can be
5246          * cleared to 0. Error out if clearing the bit failed.
5247          */
5248         if (readl_relaxed(rbase + GICR_CTLR) & GICR_CTLR_ENABLE_LPIS) {
5249                 pr_err("CPU%d: Failed to disable LPIs\n", smp_processor_id());
5250                 return -EBUSY;
5251         }
5252
5253         return 0;
5254 }
5255
5256 int its_cpu_init(void)
5257 {
5258         if (!list_empty(&its_nodes)) {
5259                 int ret;
5260
5261                 ret = redist_disable_lpis();
5262                 if (ret)
5263                         return ret;
5264
5265                 its_cpu_init_lpis();
5266                 its_cpu_init_collections();
5267         }
5268
5269         return 0;
5270 }
5271
5272 static void rdist_memreserve_cpuhp_cleanup_workfn(struct work_struct *work)
5273 {
5274         cpuhp_remove_state_nocalls(gic_rdists->cpuhp_memreserve_state);
5275         gic_rdists->cpuhp_memreserve_state = CPUHP_INVALID;
5276 }
5277
5278 static DECLARE_WORK(rdist_memreserve_cpuhp_cleanup_work,
5279                     rdist_memreserve_cpuhp_cleanup_workfn);
5280
5281 static int its_cpu_memreserve_lpi(unsigned int cpu)
5282 {
5283         struct page *pend_page;
5284         int ret = 0;
5285
5286         /* This gets to run exactly once per CPU */
5287         if (gic_data_rdist()->flags & RD_LOCAL_MEMRESERVE_DONE)
5288                 return 0;
5289
5290         pend_page = gic_data_rdist()->pend_page;
5291         if (WARN_ON(!pend_page)) {
5292                 ret = -ENOMEM;
5293                 goto out;
5294         }
5295         /*
5296          * If the pending table was pre-programmed, free the memory we
5297          * preemptively allocated. Otherwise, reserve that memory for
5298          * later kexecs.
5299          */
5300         if (gic_data_rdist()->flags & RD_LOCAL_PENDTABLE_PREALLOCATED) {
5301                 its_free_pending_table(pend_page);
5302                 gic_data_rdist()->pend_page = NULL;
5303         } else {
5304                 phys_addr_t paddr = page_to_phys(pend_page);
5305                 WARN_ON(gic_reserve_range(paddr, LPI_PENDBASE_SZ));
5306         }
5307
5308 out:
5309         /* Last CPU being brought up gets to issue the cleanup */
5310         if (!IS_ENABLED(CONFIG_SMP) ||
5311             cpumask_equal(&cpus_booted_once_mask, cpu_possible_mask))
5312                 schedule_work(&rdist_memreserve_cpuhp_cleanup_work);
5313
5314         gic_data_rdist()->flags |= RD_LOCAL_MEMRESERVE_DONE;
5315         return ret;
5316 }
5317
5318 /* Mark all the BASER registers as invalid before they get reprogrammed */
5319 static int __init its_reset_one(struct resource *res)
5320 {
5321         void __iomem *its_base;
5322         int err, i;
5323
5324         its_base = its_map_one(res, &err);
5325         if (!its_base)
5326                 return err;
5327
5328         for (i = 0; i < GITS_BASER_NR_REGS; i++)
5329                 gits_write_baser(0, its_base + GITS_BASER + (i << 3));
5330
5331         iounmap(its_base);
5332         return 0;
5333 }
5334
5335 static const struct of_device_id its_device_id[] = {
5336         {       .compatible     = "arm,gic-v3-its",     },
5337         {},
5338 };
5339
5340 static struct its_node __init *its_node_init(struct resource *res,
5341                                              struct fwnode_handle *handle, int numa_node)
5342 {
5343         void __iomem *its_base;
5344         struct its_node *its;
5345         int err;
5346
5347         its_base = its_map_one(res, &err);
5348         if (!its_base)
5349                 return NULL;
5350
5351         pr_info("ITS %pR\n", res);
5352
5353         its = kzalloc(sizeof(*its), GFP_KERNEL);
5354         if (!its)
5355                 goto out_unmap;
5356
5357         raw_spin_lock_init(&its->lock);
5358         mutex_init(&its->dev_alloc_lock);
5359         INIT_LIST_HEAD(&its->entry);
5360         INIT_LIST_HEAD(&its->its_device_list);
5361
5362         its->typer = gic_read_typer(its_base + GITS_TYPER);
5363         its->base = its_base;
5364         its->phys_base = res->start;
5365
5366         its->numa_node = numa_node;
5367         its->fwnode_handle = handle;
5368
5369         return its;
5370
5371 out_unmap:
5372         iounmap(its_base);
5373         return NULL;
5374 }
5375
5376 static void its_node_destroy(struct its_node *its)
5377 {
5378         iounmap(its->base);
5379         kfree(its);
5380 }
5381
5382 static int __init its_of_probe(struct device_node *node)
5383 {
5384         struct device_node *np;
5385         struct resource res;
5386         int err;
5387
5388         /*
5389          * Make sure *all* the ITS are reset before we probe any, as
5390          * they may be sharing memory. If any of the ITS fails to
5391          * reset, don't even try to go any further, as this could
5392          * result in something even worse.
5393          */
5394         for (np = of_find_matching_node(node, its_device_id); np;
5395              np = of_find_matching_node(np, its_device_id)) {
5396                 if (!of_device_is_available(np) ||
5397                     !of_property_read_bool(np, "msi-controller") ||
5398                     of_address_to_resource(np, 0, &res))
5399                         continue;
5400
5401                 err = its_reset_one(&res);
5402                 if (err)
5403                         return err;
5404         }
5405
5406         for (np = of_find_matching_node(node, its_device_id); np;
5407              np = of_find_matching_node(np, its_device_id)) {
5408                 struct its_node *its;
5409
5410                 if (!of_device_is_available(np))
5411                         continue;
5412                 if (!of_property_read_bool(np, "msi-controller")) {
5413                         pr_warn("%pOF: no msi-controller property, ITS ignored\n",
5414                                 np);
5415                         continue;
5416                 }
5417
5418                 if (of_address_to_resource(np, 0, &res)) {
5419                         pr_warn("%pOF: no regs?\n", np);
5420                         continue;
5421                 }
5422
5423
5424                 its = its_node_init(&res, &np->fwnode, of_node_to_nid(np));
5425                 if (!its)
5426                         return -ENOMEM;
5427
5428                 its_enable_quirks(its);
5429                 err = its_probe_one(its);
5430                 if (err)  {
5431                         its_node_destroy(its);
5432                         return err;
5433                 }
5434         }
5435         return 0;
5436 }
5437
5438 #ifdef CONFIG_ACPI
5439
5440 #define ACPI_GICV3_ITS_MEM_SIZE (SZ_128K)
5441
5442 #ifdef CONFIG_ACPI_NUMA
5443 struct its_srat_map {
5444         /* numa node id */
5445         u32     numa_node;
5446         /* GIC ITS ID */
5447         u32     its_id;
5448 };
5449
5450 static struct its_srat_map *its_srat_maps __initdata;
5451 static int its_in_srat __initdata;
5452
5453 static int __init acpi_get_its_numa_node(u32 its_id)
5454 {
5455         int i;
5456
5457         for (i = 0; i < its_in_srat; i++) {
5458                 if (its_id == its_srat_maps[i].its_id)
5459                         return its_srat_maps[i].numa_node;
5460         }
5461         return NUMA_NO_NODE;
5462 }
5463
5464 static int __init gic_acpi_match_srat_its(union acpi_subtable_headers *header,
5465                                           const unsigned long end)
5466 {
5467         return 0;
5468 }
5469
5470 static int __init gic_acpi_parse_srat_its(union acpi_subtable_headers *header,
5471                          const unsigned long end)
5472 {
5473         int node;
5474         struct acpi_srat_gic_its_affinity *its_affinity;
5475
5476         its_affinity = (struct acpi_srat_gic_its_affinity *)header;
5477         if (!its_affinity)
5478                 return -EINVAL;
5479
5480         if (its_affinity->header.length < sizeof(*its_affinity)) {
5481                 pr_err("SRAT: Invalid header length %d in ITS affinity\n",
5482                         its_affinity->header.length);
5483                 return -EINVAL;
5484         }
5485
5486         /*
5487          * Note that in theory a new proximity node could be created by this
5488          * entry as it is an SRAT resource allocation structure.
5489          * We do not currently support doing so.
5490          */
5491         node = pxm_to_node(its_affinity->proximity_domain);
5492
5493         if (node == NUMA_NO_NODE || node >= MAX_NUMNODES) {
5494                 pr_err("SRAT: Invalid NUMA node %d in ITS affinity\n", node);
5495                 return 0;
5496         }
5497
5498         its_srat_maps[its_in_srat].numa_node = node;
5499         its_srat_maps[its_in_srat].its_id = its_affinity->its_id;
5500         its_in_srat++;
5501         pr_info("SRAT: PXM %d -> ITS %d -> Node %d\n",
5502                 its_affinity->proximity_domain, its_affinity->its_id, node);
5503
5504         return 0;
5505 }
5506
5507 static void __init acpi_table_parse_srat_its(void)
5508 {
5509         int count;
5510
5511         count = acpi_table_parse_entries(ACPI_SIG_SRAT,
5512                         sizeof(struct acpi_table_srat),
5513                         ACPI_SRAT_TYPE_GIC_ITS_AFFINITY,
5514                         gic_acpi_match_srat_its, 0);
5515         if (count <= 0)
5516                 return;
5517
5518         its_srat_maps = kmalloc_array(count, sizeof(struct its_srat_map),
5519                                       GFP_KERNEL);
5520         if (!its_srat_maps)
5521                 return;
5522
5523         acpi_table_parse_entries(ACPI_SIG_SRAT,
5524                         sizeof(struct acpi_table_srat),
5525                         ACPI_SRAT_TYPE_GIC_ITS_AFFINITY,
5526                         gic_acpi_parse_srat_its, 0);
5527 }
5528
5529 /* free the its_srat_maps after ITS probing */
5530 static void __init acpi_its_srat_maps_free(void)
5531 {
5532         kfree(its_srat_maps);
5533 }
5534 #else
5535 static void __init acpi_table_parse_srat_its(void)      { }
5536 static int __init acpi_get_its_numa_node(u32 its_id) { return NUMA_NO_NODE; }
5537 static void __init acpi_its_srat_maps_free(void) { }
5538 #endif
5539
5540 static int __init gic_acpi_parse_madt_its(union acpi_subtable_headers *header,
5541                                           const unsigned long end)
5542 {
5543         struct acpi_madt_generic_translator *its_entry;
5544         struct fwnode_handle *dom_handle;
5545         struct its_node *its;
5546         struct resource res;
5547         int err;
5548
5549         its_entry = (struct acpi_madt_generic_translator *)header;
5550         memset(&res, 0, sizeof(res));
5551         res.start = its_entry->base_address;
5552         res.end = its_entry->base_address + ACPI_GICV3_ITS_MEM_SIZE - 1;
5553         res.flags = IORESOURCE_MEM;
5554
5555         dom_handle = irq_domain_alloc_fwnode(&res.start);
5556         if (!dom_handle) {
5557                 pr_err("ITS@%pa: Unable to allocate GICv3 ITS domain token\n",
5558                        &res.start);
5559                 return -ENOMEM;
5560         }
5561
5562         err = iort_register_domain_token(its_entry->translation_id, res.start,
5563                                          dom_handle);
5564         if (err) {
5565                 pr_err("ITS@%pa: Unable to register GICv3 ITS domain token (ITS ID %d) to IORT\n",
5566                        &res.start, its_entry->translation_id);
5567                 goto dom_err;
5568         }
5569
5570         its = its_node_init(&res, dom_handle,
5571                             acpi_get_its_numa_node(its_entry->translation_id));
5572         if (!its) {
5573                 err = -ENOMEM;
5574                 goto node_err;
5575         }
5576
5577         err = its_probe_one(its);
5578         if (!err)
5579                 return 0;
5580
5581 node_err:
5582         iort_deregister_domain_token(its_entry->translation_id);
5583 dom_err:
5584         irq_domain_free_fwnode(dom_handle);
5585         return err;
5586 }
5587
5588 static int __init its_acpi_reset(union acpi_subtable_headers *header,
5589                                  const unsigned long end)
5590 {
5591         struct acpi_madt_generic_translator *its_entry;
5592         struct resource res;
5593
5594         its_entry = (struct acpi_madt_generic_translator *)header;
5595         res = (struct resource) {
5596                 .start  = its_entry->base_address,
5597                 .end    = its_entry->base_address + ACPI_GICV3_ITS_MEM_SIZE - 1,
5598                 .flags  = IORESOURCE_MEM,
5599         };
5600
5601         return its_reset_one(&res);
5602 }
5603
5604 static void __init its_acpi_probe(void)
5605 {
5606         acpi_table_parse_srat_its();
5607         /*
5608          * Make sure *all* the ITS are reset before we probe any, as
5609          * they may be sharing memory. If any of the ITS fails to
5610          * reset, don't even try to go any further, as this could
5611          * result in something even worse.
5612          */
5613         if (acpi_table_parse_madt(ACPI_MADT_TYPE_GENERIC_TRANSLATOR,
5614                                   its_acpi_reset, 0) > 0)
5615                 acpi_table_parse_madt(ACPI_MADT_TYPE_GENERIC_TRANSLATOR,
5616                                       gic_acpi_parse_madt_its, 0);
5617         acpi_its_srat_maps_free();
5618 }
5619 #else
5620 static void __init its_acpi_probe(void) { }
5621 #endif
5622
5623 int __init its_lpi_memreserve_init(void)
5624 {
5625         int state;
5626
5627         if (!efi_enabled(EFI_CONFIG_TABLES))
5628                 return 0;
5629
5630         if (list_empty(&its_nodes))
5631                 return 0;
5632
5633         gic_rdists->cpuhp_memreserve_state = CPUHP_INVALID;
5634         state = cpuhp_setup_state(CPUHP_AP_ONLINE_DYN,
5635                                   "irqchip/arm/gicv3/memreserve:online",
5636                                   its_cpu_memreserve_lpi,
5637                                   NULL);
5638         if (state < 0)
5639                 return state;
5640
5641         gic_rdists->cpuhp_memreserve_state = state;
5642
5643         return 0;
5644 }
5645
5646 int __init its_init(struct fwnode_handle *handle, struct rdists *rdists,
5647                     struct irq_domain *parent_domain)
5648 {
5649         struct device_node *of_node;
5650         struct its_node *its;
5651         bool has_v4 = false;
5652         bool has_v4_1 = false;
5653         int err;
5654
5655         gic_rdists = rdists;
5656
5657         its_parent = parent_domain;
5658         of_node = to_of_node(handle);
5659         if (of_node)
5660                 its_of_probe(of_node);
5661         else
5662                 its_acpi_probe();
5663
5664         if (list_empty(&its_nodes)) {
5665                 pr_warn("ITS: No ITS available, not enabling LPIs\n");
5666                 return -ENXIO;
5667         }
5668
5669         err = allocate_lpi_tables();
5670         if (err)
5671                 return err;
5672
5673         list_for_each_entry(its, &its_nodes, entry) {
5674                 has_v4 |= is_v4(its);
5675                 has_v4_1 |= is_v4_1(its);
5676         }
5677
5678         /* Don't bother with inconsistent systems */
5679         if (WARN_ON(!has_v4_1 && rdists->has_rvpeid))
5680                 rdists->has_rvpeid = false;
5681
5682         if (has_v4 & rdists->has_vlpis) {
5683                 const struct irq_domain_ops *sgi_ops;
5684
5685                 if (has_v4_1)
5686                         sgi_ops = &its_sgi_domain_ops;
5687                 else
5688                         sgi_ops = NULL;
5689
5690                 if (its_init_vpe_domain() ||
5691                     its_init_v4(parent_domain, &its_vpe_domain_ops, sgi_ops)) {
5692                         rdists->has_vlpis = false;
5693                         pr_err("ITS: Disabling GICv4 support\n");
5694                 }
5695         }
5696
5697         register_syscore_ops(&its_syscore_ops);
5698
5699         return 0;
5700 }