drivers/perf/alibaba_uncore_drw_pmu.c

   1 // SPDX-License-Identifier: GPL-2.0
   2 /*
   3  * Alibaba DDR Sub-System Driveway PMU driver
   4  *
   5  * Copyright (C) 2022 Alibaba Inc
   6  */
   7
   8 #define ALI_DRW_PMUNAME         "ali_drw"
   9 #define ALI_DRW_DRVNAME         ALI_DRW_PMUNAME "_pmu"
  10 #define pr_fmt(fmt)             ALI_DRW_DRVNAME ": " fmt
  11
  12 #include <linux/acpi.h>
  13 #include <linux/bitfield.h>
  14 #include <linux/bitmap.h>
  15 #include <linux/bitops.h>
  16 #include <linux/cpuhotplug.h>
  17 #include <linux/cpumask.h>
  18 #include <linux/device.h>
  19 #include <linux/errno.h>
  20 #include <linux/interrupt.h>
  21 #include <linux/irq.h>
  22 #include <linux/kernel.h>
  23 #include <linux/list.h>
  24 #include <linux/module.h>
  25 #include <linux/mutex.h>
  26 #include <linux/perf_event.h>
  27 #include <linux/platform_device.h>
  28 #include <linux/printk.h>
  29 #include <linux/rculist.h>
  30 #include <linux/refcount.h>
  31
  32
  33 #define ALI_DRW_PMU_COMMON_MAX_COUNTERS                 16
  34 #define ALI_DRW_PMU_TEST_SEL_COMMON_COUNTER_BASE        19
  35
  36 #define ALI_DRW_PMU_PA_SHIFT                    12
  37 #define ALI_DRW_PMU_CNT_INIT                    0x00000000
  38 #define ALI_DRW_CNT_MAX_PERIOD                  0xffffffff
  39 #define ALI_DRW_PMU_CYCLE_EVT_ID                0x80
  40
  41 #define ALI_DRW_PMU_CNT_CTRL                    0xC00
  42 #define ALI_DRW_PMU_CNT_RST                     BIT(2)
  43 #define ALI_DRW_PMU_CNT_STOP                    BIT(1)
  44 #define ALI_DRW_PMU_CNT_START                   BIT(0)
  45
  46 #define ALI_DRW_PMU_CNT_STATE                   0xC04
  47 #define ALI_DRW_PMU_TEST_CTRL                   0xC08
  48 #define ALI_DRW_PMU_CNT_PRELOAD                 0xC0C
  49
  50 #define ALI_DRW_PMU_CYCLE_CNT_HIGH_MASK         GENMASK(23, 0)
  51 #define ALI_DRW_PMU_CYCLE_CNT_LOW_MASK          GENMASK(31, 0)
  52 #define ALI_DRW_PMU_CYCLE_CNT_HIGH              0xC10
  53 #define ALI_DRW_PMU_CYCLE_CNT_LOW               0xC14
  54
  55 /* PMU EVENT SEL 0-3 are paired in 32-bit registers on a 4-byte stride */
  56 #define ALI_DRW_PMU_EVENT_SEL0                  0xC68
  57 /* counter 0-3 use sel0, counter 4-7 use sel1...*/
  58 #define ALI_DRW_PMU_EVENT_SELn(n) \
  59         (ALI_DRW_PMU_EVENT_SEL0 + (n / 4) * 0x4)
  60 #define ALI_DRW_PMCOM_CNT_EN                    BIT(7)
  61 #define ALI_DRW_PMCOM_CNT_EVENT_MASK            GENMASK(5, 0)
  62 #define ALI_DRW_PMCOM_CNT_EVENT_OFFSET(n) \
  63         (8 * (n % 4))
  64
  65 /* PMU COMMON COUNTER 0-15, are paired in 32-bit registers on a 4-byte stride */
  66 #define ALI_DRW_PMU_COMMON_COUNTER0             0xC78
  67 #define ALI_DRW_PMU_COMMON_COUNTERn(n) \
  68         (ALI_DRW_PMU_COMMON_COUNTER0 + 0x4 * (n))
  69
  70 #define ALI_DRW_PMU_OV_INTR_ENABLE_CTL          0xCB8
  71 #define ALI_DRW_PMU_OV_INTR_DISABLE_CTL         0xCBC
  72 #define ALI_DRW_PMU_OV_INTR_ENABLE_STATUS       0xCC0
  73 #define ALI_DRW_PMU_OV_INTR_CLR                 0xCC4
  74 #define ALI_DRW_PMU_OV_INTR_STATUS              0xCC8
  75 #define ALI_DRW_PMCOM_CNT_OV_INTR_MASK          GENMASK(23, 8)
  76 #define ALI_DRW_PMBW_CNT_OV_INTR_MASK           GENMASK(7, 0)
  77 #define ALI_DRW_PMU_OV_INTR_MASK                GENMASK_ULL(63, 0)
  78
  79 static int ali_drw_cpuhp_state_num;
  80
  81 static LIST_HEAD(ali_drw_pmu_irqs);
  82 static DEFINE_MUTEX(ali_drw_pmu_irqs_lock);
  83
  84 struct ali_drw_pmu_irq {
  85         struct hlist_node node;
  86         struct list_head irqs_node;
  87         struct list_head pmus_node;
  88         int irq_num;
  89         int cpu;
  90         refcount_t refcount;
  91 };
  92
  93 struct ali_drw_pmu {
  94         void __iomem *cfg_base;
  95         struct device *dev;
  96
  97         struct list_head pmus_node;
  98         struct ali_drw_pmu_irq *irq;
  99         int irq_num;
 100         int cpu;
 101         DECLARE_BITMAP(used_mask, ALI_DRW_PMU_COMMON_MAX_COUNTERS);
 102         struct perf_event *events[ALI_DRW_PMU_COMMON_MAX_COUNTERS];
 103         int evtids[ALI_DRW_PMU_COMMON_MAX_COUNTERS];
 104
 105         struct pmu pmu;
 106 };
 107
 108 #define to_ali_drw_pmu(p) (container_of(p, struct ali_drw_pmu, pmu))
 109
 110 #define DRW_CONFIG_EVENTID              GENMASK(7, 0)
 111 #define GET_DRW_EVENTID(event)  FIELD_GET(DRW_CONFIG_EVENTID, (event)->attr.config)
 112
 113 static ssize_t ali_drw_pmu_format_show(struct device *dev,
 114                                 struct device_attribute *attr, char *buf)
 115 {
 116         struct dev_ext_attribute *eattr;
 117
 118         eattr = container_of(attr, struct dev_ext_attribute, attr);
 119
 120         return sprintf(buf, "%s\n", (char *)eattr->var);
 121 }
 122
 123 /*
 124  * PMU event attributes
 125  */
 126 static ssize_t ali_drw_pmu_event_show(struct device *dev,
 127                                struct device_attribute *attr, char *page)
 128 {
 129         struct dev_ext_attribute *eattr;
 130
 131         eattr = container_of(attr, struct dev_ext_attribute, attr);
 132
 133         return sprintf(page, "config=0x%lx\n", (unsigned long)eattr->var);
 134 }
 135
 136 #define ALI_DRW_PMU_ATTR(_name, _func, _config)                            \
 137                 (&((struct dev_ext_attribute[]) {                               \
 138                                 { __ATTR(_name, 0444, _func, NULL), (void *)_config }   \
 139                 })[0].attr.attr)
 140
 141 #define ALI_DRW_PMU_FORMAT_ATTR(_name, _config)            \
 142         ALI_DRW_PMU_ATTR(_name, ali_drw_pmu_format_show, (void *)_config)
 143 #define ALI_DRW_PMU_EVENT_ATTR(_name, _config)             \
 144         ALI_DRW_PMU_ATTR(_name, ali_drw_pmu_event_show, (unsigned long)_config)
 145
 146 static struct attribute *ali_drw_pmu_events_attrs[] = {
 147         ALI_DRW_PMU_EVENT_ATTR(hif_rd_or_wr,                    0x0),
 148         ALI_DRW_PMU_EVENT_ATTR(hif_wr,                          0x1),
 149         ALI_DRW_PMU_EVENT_ATTR(hif_rd,                          0x2),
 150         ALI_DRW_PMU_EVENT_ATTR(hif_rmw,                         0x3),
 151         ALI_DRW_PMU_EVENT_ATTR(hif_hi_pri_rd,                   0x4),
 152         ALI_DRW_PMU_EVENT_ATTR(dfi_wr_data_cycles,              0x7),
 153         ALI_DRW_PMU_EVENT_ATTR(dfi_rd_data_cycles,              0x8),
 154         ALI_DRW_PMU_EVENT_ATTR(hpr_xact_when_critical,          0x9),
 155         ALI_DRW_PMU_EVENT_ATTR(lpr_xact_when_critical,          0xA),
 156         ALI_DRW_PMU_EVENT_ATTR(wr_xact_when_critical,           0xB),
 157         ALI_DRW_PMU_EVENT_ATTR(op_is_activate,                  0xC),
 158         ALI_DRW_PMU_EVENT_ATTR(op_is_rd_or_wr,                  0xD),
 159         ALI_DRW_PMU_EVENT_ATTR(op_is_rd_activate,               0xE),
 160         ALI_DRW_PMU_EVENT_ATTR(op_is_rd,                        0xF),
 161         ALI_DRW_PMU_EVENT_ATTR(op_is_wr,                        0x10),
 162         ALI_DRW_PMU_EVENT_ATTR(op_is_mwr,                       0x11),
 163         ALI_DRW_PMU_EVENT_ATTR(op_is_precharge,                 0x12),
 164         ALI_DRW_PMU_EVENT_ATTR(precharge_for_rdwr,              0x13),
 165         ALI_DRW_PMU_EVENT_ATTR(precharge_for_other,             0x14),
 166         ALI_DRW_PMU_EVENT_ATTR(rdwr_transitions,                0x15),
 167         ALI_DRW_PMU_EVENT_ATTR(write_combine,                   0x16),
 168         ALI_DRW_PMU_EVENT_ATTR(war_hazard,                      0x17),
 169         ALI_DRW_PMU_EVENT_ATTR(raw_hazard,                      0x18),
 170         ALI_DRW_PMU_EVENT_ATTR(waw_hazard,                      0x19),
 171         ALI_DRW_PMU_EVENT_ATTR(op_is_enter_selfref_rk0,         0x1A),
 172         ALI_DRW_PMU_EVENT_ATTR(op_is_enter_selfref_rk1,         0x1B),
 173         ALI_DRW_PMU_EVENT_ATTR(op_is_enter_selfref_rk2,         0x1C),
 174         ALI_DRW_PMU_EVENT_ATTR(op_is_enter_selfref_rk3,         0x1D),
 175         ALI_DRW_PMU_EVENT_ATTR(op_is_enter_powerdown_rk0,       0x1E),
 176         ALI_DRW_PMU_EVENT_ATTR(op_is_enter_powerdown_rk1,       0x1F),
 177         ALI_DRW_PMU_EVENT_ATTR(op_is_enter_powerdown_rk2,       0x20),
 178         ALI_DRW_PMU_EVENT_ATTR(op_is_enter_powerdown_rk3,       0x21),
 179         ALI_DRW_PMU_EVENT_ATTR(selfref_mode_rk0,                0x26),
 180         ALI_DRW_PMU_EVENT_ATTR(selfref_mode_rk1,                0x27),
 181         ALI_DRW_PMU_EVENT_ATTR(selfref_mode_rk2,                0x28),
 182         ALI_DRW_PMU_EVENT_ATTR(selfref_mode_rk3,                0x29),
 183         ALI_DRW_PMU_EVENT_ATTR(op_is_refresh,                   0x2A),
 184         ALI_DRW_PMU_EVENT_ATTR(op_is_crit_ref,                  0x2B),
 185         ALI_DRW_PMU_EVENT_ATTR(op_is_load_mode,                 0x2D),
 186         ALI_DRW_PMU_EVENT_ATTR(op_is_zqcl,                      0x2E),
 187         ALI_DRW_PMU_EVENT_ATTR(visible_window_limit_reached_rd, 0x30),
 188         ALI_DRW_PMU_EVENT_ATTR(visible_window_limit_reached_wr, 0x31),
 189         ALI_DRW_PMU_EVENT_ATTR(op_is_dqsosc_mpc,                0x34),
 190         ALI_DRW_PMU_EVENT_ATTR(op_is_dqsosc_mrr,                0x35),
 191         ALI_DRW_PMU_EVENT_ATTR(op_is_tcr_mrr,                   0x36),
 192         ALI_DRW_PMU_EVENT_ATTR(op_is_zqstart,                   0x37),
 193         ALI_DRW_PMU_EVENT_ATTR(op_is_zqlatch,                   0x38),
 194         ALI_DRW_PMU_EVENT_ATTR(chi_txreq,                       0x39),
 195         ALI_DRW_PMU_EVENT_ATTR(chi_txdat,                       0x3A),
 196         ALI_DRW_PMU_EVENT_ATTR(chi_rxdat,                       0x3B),
 197         ALI_DRW_PMU_EVENT_ATTR(chi_rxrsp,                       0x3C),
 198         ALI_DRW_PMU_EVENT_ATTR(tsz_vio,                         0x3D),
 199         ALI_DRW_PMU_EVENT_ATTR(cycle,                           0x80),
 200         NULL,
 201 };
 202
 203 static struct attribute_group ali_drw_pmu_events_attr_group = {
 204         .name = "events",
 205         .attrs = ali_drw_pmu_events_attrs,
 206 };
 207
 208 static struct attribute *ali_drw_pmu_format_attr[] = {
 209         ALI_DRW_PMU_FORMAT_ATTR(event, "config:0-7"),
 210         NULL,
 211 };
 212
 213 static const struct attribute_group ali_drw_pmu_format_group = {
 214         .name = "format",
 215         .attrs = ali_drw_pmu_format_attr,
 216 };
 217
 218 static ssize_t ali_drw_pmu_cpumask_show(struct device *dev,
 219                                         struct device_attribute *attr,
 220                                         char *buf)
 221 {
 222         struct ali_drw_pmu *drw_pmu = to_ali_drw_pmu(dev_get_drvdata(dev));
 223
 224         return cpumap_print_to_pagebuf(true, buf, cpumask_of(drw_pmu->cpu));
 225 }
 226
 227 static struct device_attribute ali_drw_pmu_cpumask_attr =
 228                 __ATTR(cpumask, 0444, ali_drw_pmu_cpumask_show, NULL);
 229
 230 static struct attribute *ali_drw_pmu_cpumask_attrs[] = {
 231         &ali_drw_pmu_cpumask_attr.attr,
 232         NULL,
 233 };
 234
 235 static const struct attribute_group ali_drw_pmu_cpumask_attr_group = {
 236         .attrs = ali_drw_pmu_cpumask_attrs,
 237 };
 238
 239 static const struct attribute_group *ali_drw_pmu_attr_groups[] = {
 240         &ali_drw_pmu_events_attr_group,
 241         &ali_drw_pmu_cpumask_attr_group,
 242         &ali_drw_pmu_format_group,
 243         NULL,
 244 };
 245
 246 /* find a counter for event, then in add func, hw.idx will equal to counter */
 247 static int ali_drw_get_counter_idx(struct perf_event *event)
 248 {
 249         struct ali_drw_pmu *drw_pmu = to_ali_drw_pmu(event->pmu);
 250         int idx;
 251
 252         for (idx = 0; idx < ALI_DRW_PMU_COMMON_MAX_COUNTERS; ++idx) {
 253                 if (!test_and_set_bit(idx, drw_pmu->used_mask))
 254                         return idx;
 255         }
 256
 257         /* The counters are all in use. */
 258         return -EBUSY;
 259 }
 260
 261 static u64 ali_drw_pmu_read_counter(struct perf_event *event)
 262 {
 263         struct ali_drw_pmu *drw_pmu = to_ali_drw_pmu(event->pmu);
 264         u64 cycle_high, cycle_low;
 265
 266         if (GET_DRW_EVENTID(event) == ALI_DRW_PMU_CYCLE_EVT_ID) {
 267                 cycle_high = readl(drw_pmu->cfg_base + ALI_DRW_PMU_CYCLE_CNT_HIGH);
 268                 cycle_high &= ALI_DRW_PMU_CYCLE_CNT_HIGH_MASK;
 269                 cycle_low = readl(drw_pmu->cfg_base + ALI_DRW_PMU_CYCLE_CNT_LOW);
 270                 cycle_low &= ALI_DRW_PMU_CYCLE_CNT_LOW_MASK;
 271                 return (cycle_high << 32 | cycle_low);
 272         }
 273
 274         return readl(drw_pmu->cfg_base +
 275                      ALI_DRW_PMU_COMMON_COUNTERn(event->hw.idx));
 276 }
 277
 278 static void ali_drw_pmu_event_update(struct perf_event *event)
 279 {
 280         struct hw_perf_event *hwc = &event->hw;
 281         u64 delta, prev, now;
 282
 283         do {
 284                 prev = local64_read(&hwc->prev_count);
 285                 now = ali_drw_pmu_read_counter(event);
 286         } while (local64_cmpxchg(&hwc->prev_count, prev, now) != prev);
 287
 288         /* handle overflow. */
 289         delta = now - prev;
 290         if (GET_DRW_EVENTID(event) == ALI_DRW_PMU_CYCLE_EVT_ID)
 291                 delta &= ALI_DRW_PMU_OV_INTR_MASK;
 292         else
 293                 delta &= ALI_DRW_CNT_MAX_PERIOD;
 294         local64_add(delta, &event->count);
 295 }
 296
 297 static void ali_drw_pmu_event_set_period(struct perf_event *event)
 298 {
 299         u64 pre_val;
 300         struct ali_drw_pmu *drw_pmu = to_ali_drw_pmu(event->pmu);
 301
 302         /* set a preload counter for test purpose */
 303         writel(ALI_DRW_PMU_TEST_SEL_COMMON_COUNTER_BASE + event->hw.idx,
 304                drw_pmu->cfg_base + ALI_DRW_PMU_TEST_CTRL);
 305
 306         /* set conunter initial value */
 307         pre_val = ALI_DRW_PMU_CNT_INIT;
 308         writel(pre_val, drw_pmu->cfg_base + ALI_DRW_PMU_CNT_PRELOAD);
 309         local64_set(&event->hw.prev_count, pre_val);
 310
 311         /* set sel mode to zero to start test */
 312         writel(0x0, drw_pmu->cfg_base + ALI_DRW_PMU_TEST_CTRL);
 313 }
 314
 315 static void ali_drw_pmu_enable_counter(struct perf_event *event)
 316 {
 317         u32 val, subval, reg, shift;
 318         int counter = event->hw.idx;
 319         struct ali_drw_pmu *drw_pmu = to_ali_drw_pmu(event->pmu);
 320
 321         reg = ALI_DRW_PMU_EVENT_SELn(counter);
 322         val = readl(drw_pmu->cfg_base + reg);
 323         subval = FIELD_PREP(ALI_DRW_PMCOM_CNT_EN, 1) |
 324                  FIELD_PREP(ALI_DRW_PMCOM_CNT_EVENT_MASK, drw_pmu->evtids[counter]);
 325
 326         shift = ALI_DRW_PMCOM_CNT_EVENT_OFFSET(counter);
 327         val &= ~(GENMASK(7, 0) << shift);
 328         val |= subval << shift;
 329
 330         writel(val, drw_pmu->cfg_base + reg);
 331 }
 332
 333 static void ali_drw_pmu_disable_counter(struct perf_event *event)
 334 {
 335         u32 val, reg, subval, shift;
 336         struct ali_drw_pmu *drw_pmu = to_ali_drw_pmu(event->pmu);
 337         int counter = event->hw.idx;
 338
 339         reg = ALI_DRW_PMU_EVENT_SELn(counter);
 340         val = readl(drw_pmu->cfg_base + reg);
 341         subval = FIELD_PREP(ALI_DRW_PMCOM_CNT_EN, 0) |
 342                  FIELD_PREP(ALI_DRW_PMCOM_CNT_EVENT_MASK, 0);
 343
 344         shift = ALI_DRW_PMCOM_CNT_EVENT_OFFSET(counter);
 345         val &= ~(GENMASK(7, 0) << shift);
 346         val |= subval << shift;
 347
 348         writel(val, drw_pmu->cfg_base + reg);
 349 }
 350
 351 static irqreturn_t ali_drw_pmu_isr(int irq_num, void *data)
 352 {
 353         struct ali_drw_pmu_irq *irq = data;
 354         struct ali_drw_pmu *drw_pmu;
 355         irqreturn_t ret = IRQ_NONE;
 356
 357         rcu_read_lock();
 358         list_for_each_entry_rcu(drw_pmu, &irq->pmus_node, pmus_node) {
 359                 unsigned long status, clr_status;
 360                 struct perf_event *event;
 361                 unsigned int idx;
 362
 363                 for (idx = 0; idx < ALI_DRW_PMU_COMMON_MAX_COUNTERS; idx++) {
 364                         event = drw_pmu->events[idx];
 365                         if (!event)
 366                                 continue;
 367                         ali_drw_pmu_disable_counter(event);
 368                 }
 369
 370                 /* common counter intr status */
 371                 status = readl(drw_pmu->cfg_base + ALI_DRW_PMU_OV_INTR_STATUS);
 372                 status = FIELD_GET(ALI_DRW_PMCOM_CNT_OV_INTR_MASK, status);
 373                 if (status) {
 374                         for_each_set_bit(idx, &status,
 375                                          ALI_DRW_PMU_COMMON_MAX_COUNTERS) {
 376                                 event = drw_pmu->events[idx];
 377                                 if (WARN_ON_ONCE(!event))
 378                                         continue;
 379                                 ali_drw_pmu_event_update(event);
 380                                 ali_drw_pmu_event_set_period(event);
 381                         }
 382
 383                         /* clear common counter intr status */
 384                         clr_status = FIELD_PREP(ALI_DRW_PMCOM_CNT_OV_INTR_MASK, 1);
 385                         writel(clr_status,
 386                                drw_pmu->cfg_base + ALI_DRW_PMU_OV_INTR_CLR);
 387                 }
 388
 389                 for (idx = 0; idx < ALI_DRW_PMU_COMMON_MAX_COUNTERS; idx++) {
 390                         event = drw_pmu->events[idx];
 391                         if (!event)
 392                                 continue;
 393                         if (!(event->hw.state & PERF_HES_STOPPED))
 394                                 ali_drw_pmu_enable_counter(event);
 395                 }
 396                 if (status)
 397                         ret = IRQ_HANDLED;
 398         }
 399         rcu_read_unlock();
 400         return ret;
 401 }
 402
 403 static struct ali_drw_pmu_irq *__ali_drw_pmu_init_irq(struct platform_device
 404                                                       *pdev, int irq_num)
 405 {
 406         int ret;
 407         struct ali_drw_pmu_irq *irq;
 408
 409         list_for_each_entry(irq, &ali_drw_pmu_irqs, irqs_node) {
 410                 if (irq->irq_num == irq_num
 411                     && refcount_inc_not_zero(&irq->refcount))
 412                         return irq;
 413         }
 414
 415         irq = kzalloc(sizeof(*irq), GFP_KERNEL);
 416         if (!irq)
 417                 return ERR_PTR(-ENOMEM);
 418
 419         INIT_LIST_HEAD(&irq->pmus_node);
 420
 421         /* Pick one CPU to be the preferred one to use */
 422         irq->cpu = smp_processor_id();
 423         refcount_set(&irq->refcount, 1);
 424
 425         /*
 426          * FIXME: one of DDRSS Driveway PMU overflow interrupt shares the same
 427          * irq number with MPAM ERR_IRQ. To register DDRSS PMU and MPAM drivers
 428          * successfully, add IRQF_SHARED flag. Howerer, PMU interrupt should not
 429          * share with other component.
 430          */
 431         ret = devm_request_irq(&pdev->dev, irq_num, ali_drw_pmu_isr,
 432                                IRQF_SHARED, dev_name(&pdev->dev), irq);
 433         if (ret < 0) {
 434                 dev_err(&pdev->dev,
 435                         "Fail to request IRQ:%d ret:%d\n", irq_num, ret);
 436                 goto out_free;
 437         }
 438
 439         ret = irq_set_affinity_hint(irq_num, cpumask_of(irq->cpu));
 440         if (ret)
 441                 goto out_free;
 442
 443         ret = cpuhp_state_add_instance_nocalls(ali_drw_cpuhp_state_num,
 444                                              &irq->node);
 445         if (ret)
 446                 goto out_free;
 447
 448         irq->irq_num = irq_num;
 449         list_add(&irq->irqs_node, &ali_drw_pmu_irqs);
 450
 451         return irq;
 452
 453 out_free:
 454         kfree(irq);
 455         return ERR_PTR(ret);
 456 }
 457
 458 static int ali_drw_pmu_init_irq(struct ali_drw_pmu *drw_pmu,
 459                                 struct platform_device *pdev)
 460 {
 461         int irq_num;
 462         struct ali_drw_pmu_irq *irq;
 463
 464         /* Read and init IRQ */
 465         irq_num = platform_get_irq(pdev, 0);
 466         if (irq_num < 0)
 467                 return irq_num;
 468
 469         mutex_lock(&ali_drw_pmu_irqs_lock);
 470         irq = __ali_drw_pmu_init_irq(pdev, irq_num);
 471         mutex_unlock(&ali_drw_pmu_irqs_lock);
 472
 473         if (IS_ERR(irq))
 474                 return PTR_ERR(irq);
 475
 476         drw_pmu->irq = irq;
 477
 478         mutex_lock(&ali_drw_pmu_irqs_lock);
 479         list_add_rcu(&drw_pmu->pmus_node, &irq->pmus_node);
 480         mutex_unlock(&ali_drw_pmu_irqs_lock);
 481
 482         return 0;
 483 }
 484
 485 static void ali_drw_pmu_uninit_irq(struct ali_drw_pmu *drw_pmu)
 486 {
 487         struct ali_drw_pmu_irq *irq = drw_pmu->irq;
 488
 489         mutex_lock(&ali_drw_pmu_irqs_lock);
 490         list_del_rcu(&drw_pmu->pmus_node);
 491
 492         if (!refcount_dec_and_test(&irq->refcount)) {
 493                 mutex_unlock(&ali_drw_pmu_irqs_lock);
 494                 return;
 495         }
 496
 497         list_del(&irq->irqs_node);
 498         mutex_unlock(&ali_drw_pmu_irqs_lock);
 499
 500         WARN_ON(irq_set_affinity_hint(irq->irq_num, NULL));
 501         cpuhp_state_remove_instance_nocalls(ali_drw_cpuhp_state_num,
 502                                             &irq->node);
 503         kfree(irq);
 504 }
 505
 506 static int ali_drw_pmu_event_init(struct perf_event *event)
 507 {
 508         struct ali_drw_pmu *drw_pmu = to_ali_drw_pmu(event->pmu);
 509         struct hw_perf_event *hwc = &event->hw;
 510         struct perf_event *sibling;
 511         struct device *dev = drw_pmu->pmu.dev;
 512
 513         if (event->attr.type != event->pmu->type)
 514                 return -ENOENT;
 515
 516         if (is_sampling_event(event)) {
 517                 dev_err(dev, "Sampling not supported!\n");
 518                 return -EOPNOTSUPP;
 519         }
 520
 521         if (event->attach_state & PERF_ATTACH_TASK) {
 522                 dev_err(dev, "Per-task counter cannot allocate!\n");
 523                 return -EOPNOTSUPP;
 524         }
 525
 526         event->cpu = drw_pmu->cpu;
 527         if (event->cpu < 0) {
 528                 dev_err(dev, "Per-task mode not supported!\n");
 529                 return -EOPNOTSUPP;
 530         }
 531
 532         if (event->group_leader != event &&
 533             !is_software_event(event->group_leader)) {
 534                 dev_err(dev, "driveway only allow one event!\n");
 535                 return -EINVAL;
 536         }
 537
 538         for_each_sibling_event(sibling, event->group_leader) {
 539                 if (sibling != event && !is_software_event(sibling)) {
 540                         dev_err(dev, "driveway event not allowed!\n");
 541                         return -EINVAL;
 542                 }
 543         }
 544
 545         /* reset all the pmu counters */
 546         writel(ALI_DRW_PMU_CNT_RST, drw_pmu->cfg_base + ALI_DRW_PMU_CNT_CTRL);
 547
 548         hwc->idx = -1;
 549
 550         return 0;
 551 }
 552
 553 static void ali_drw_pmu_start(struct perf_event *event, int flags)
 554 {
 555         struct ali_drw_pmu *drw_pmu = to_ali_drw_pmu(event->pmu);
 556
 557         event->hw.state = 0;
 558
 559         if (GET_DRW_EVENTID(event) == ALI_DRW_PMU_CYCLE_EVT_ID) {
 560                 writel(ALI_DRW_PMU_CNT_START,
 561                        drw_pmu->cfg_base + ALI_DRW_PMU_CNT_CTRL);
 562                 return;
 563         }
 564
 565         ali_drw_pmu_event_set_period(event);
 566         if (flags & PERF_EF_RELOAD) {
 567                 unsigned long prev_raw_count =
 568                     local64_read(&event->hw.prev_count);
 569                 writel(prev_raw_count,
 570                        drw_pmu->cfg_base + ALI_DRW_PMU_CNT_PRELOAD);
 571         }
 572
 573         ali_drw_pmu_enable_counter(event);
 574
 575         writel(ALI_DRW_PMU_CNT_START, drw_pmu->cfg_base + ALI_DRW_PMU_CNT_CTRL);
 576 }
 577
 578 static void ali_drw_pmu_stop(struct perf_event *event, int flags)
 579 {
 580         struct ali_drw_pmu *drw_pmu = to_ali_drw_pmu(event->pmu);
 581
 582         if (event->hw.state & PERF_HES_STOPPED)
 583                 return;
 584
 585         if (GET_DRW_EVENTID(event) != ALI_DRW_PMU_CYCLE_EVT_ID)
 586                 ali_drw_pmu_disable_counter(event);
 587
 588         writel(ALI_DRW_PMU_CNT_STOP, drw_pmu->cfg_base + ALI_DRW_PMU_CNT_CTRL);
 589
 590         ali_drw_pmu_event_update(event);
 591         event->hw.state |= PERF_HES_STOPPED | PERF_HES_UPTODATE;
 592 }
 593
 594 static int ali_drw_pmu_add(struct perf_event *event, int flags)
 595 {
 596         struct ali_drw_pmu *drw_pmu = to_ali_drw_pmu(event->pmu);
 597         struct hw_perf_event *hwc = &event->hw;
 598         int idx = -1;
 599         int evtid;
 600
 601         evtid = GET_DRW_EVENTID(event);
 602
 603         if (evtid != ALI_DRW_PMU_CYCLE_EVT_ID) {
 604                 idx = ali_drw_get_counter_idx(event);
 605                 if (idx < 0)
 606                         return idx;
 607                 drw_pmu->events[idx] = event;
 608                 drw_pmu->evtids[idx] = evtid;
 609         }
 610         hwc->idx = idx;
 611
 612         hwc->state = PERF_HES_STOPPED | PERF_HES_UPTODATE;
 613
 614         if (flags & PERF_EF_START)
 615                 ali_drw_pmu_start(event, PERF_EF_RELOAD);
 616
 617         /* Propagate our changes to the userspace mapping. */
 618         perf_event_update_userpage(event);
 619
 620         return 0;
 621 }
 622
 623 static void ali_drw_pmu_del(struct perf_event *event, int flags)
 624 {
 625         struct ali_drw_pmu *drw_pmu = to_ali_drw_pmu(event->pmu);
 626         struct hw_perf_event *hwc = &event->hw;
 627         int idx = hwc->idx;
 628
 629         ali_drw_pmu_stop(event, PERF_EF_UPDATE);
 630
 631         if (idx >= 0 && idx < ALI_DRW_PMU_COMMON_MAX_COUNTERS) {
 632                 drw_pmu->events[idx] = NULL;
 633                 drw_pmu->evtids[idx] = 0;
 634                 clear_bit(idx, drw_pmu->used_mask);
 635         }
 636
 637         perf_event_update_userpage(event);
 638 }
 639
 640 static void ali_drw_pmu_read(struct perf_event *event)
 641 {
 642         ali_drw_pmu_event_update(event);
 643 }
 644
 645 static int ali_drw_pmu_probe(struct platform_device *pdev)
 646 {
 647         struct ali_drw_pmu *drw_pmu;
 648         struct resource *res;
 649         char *name;
 650         int ret;
 651
 652         drw_pmu = devm_kzalloc(&pdev->dev, sizeof(*drw_pmu), GFP_KERNEL);
 653         if (!drw_pmu)
 654                 return -ENOMEM;
 655
 656         drw_pmu->dev = &pdev->dev;
 657         platform_set_drvdata(pdev, drw_pmu);
 658
 659         res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
 660         drw_pmu->cfg_base = devm_ioremap_resource(&pdev->dev, res);
 661         if (IS_ERR(drw_pmu->cfg_base))
 662                 return PTR_ERR(drw_pmu->cfg_base);
 663
 664         name = devm_kasprintf(drw_pmu->dev, GFP_KERNEL, "ali_drw_%llx",
 665                               (u64) (res->start >> ALI_DRW_PMU_PA_SHIFT));
 666         if (!name)
 667                 return -ENOMEM;
 668
 669         writel(ALI_DRW_PMU_CNT_RST, drw_pmu->cfg_base + ALI_DRW_PMU_CNT_CTRL);
 670
 671         /* enable the generation of interrupt by all common counters */
 672         writel(ALI_DRW_PMCOM_CNT_OV_INTR_MASK,
 673                drw_pmu->cfg_base + ALI_DRW_PMU_OV_INTR_ENABLE_CTL);
 674
 675         /* clearing interrupt status */
 676         writel(0xffffff, drw_pmu->cfg_base + ALI_DRW_PMU_OV_INTR_CLR);
 677
 678         drw_pmu->cpu = smp_processor_id();
 679
 680         ret = ali_drw_pmu_init_irq(drw_pmu, pdev);
 681         if (ret)
 682                 return ret;
 683
 684         drw_pmu->pmu = (struct pmu) {
 685                 .module         = THIS_MODULE,
 686                 .task_ctx_nr    = perf_invalid_context,
 687                 .event_init     = ali_drw_pmu_event_init,
 688                 .add            = ali_drw_pmu_add,
 689                 .del            = ali_drw_pmu_del,
 690                 .start          = ali_drw_pmu_start,
 691                 .stop           = ali_drw_pmu_stop,
 692                 .read           = ali_drw_pmu_read,
 693                 .attr_groups    = ali_drw_pmu_attr_groups,
 694                 .capabilities   = PERF_PMU_CAP_NO_EXCLUDE,
 695         };
 696
 697         ret = perf_pmu_register(&drw_pmu->pmu, name, -1);
 698         if (ret) {
 699                 dev_err(drw_pmu->dev, "DRW Driveway PMU PMU register failed!\n");
 700                 ali_drw_pmu_uninit_irq(drw_pmu);
 701         }
 702
 703         return ret;
 704 }
 705
 706 static int ali_drw_pmu_remove(struct platform_device *pdev)
 707 {
 708         struct ali_drw_pmu *drw_pmu = platform_get_drvdata(pdev);
 709
 710         /* disable the generation of interrupt by all common counters */
 711         writel(ALI_DRW_PMCOM_CNT_OV_INTR_MASK,
 712                drw_pmu->cfg_base + ALI_DRW_PMU_OV_INTR_DISABLE_CTL);
 713
 714         ali_drw_pmu_uninit_irq(drw_pmu);
 715         perf_pmu_unregister(&drw_pmu->pmu);
 716
 717         return 0;
 718 }
 719
 720 static int ali_drw_pmu_offline_cpu(unsigned int cpu, struct hlist_node *node)
 721 {
 722         struct ali_drw_pmu_irq *irq;
 723         struct ali_drw_pmu *drw_pmu;
 724         unsigned int target;
 725         int ret;
 726         cpumask_t node_online_cpus;
 727
 728         irq = hlist_entry_safe(node, struct ali_drw_pmu_irq, node);
 729         if (cpu != irq->cpu)
 730                 return 0;
 731
 732         ret = cpumask_and(&node_online_cpus,
 733                           cpumask_of_node(cpu_to_node(cpu)), cpu_online_mask);
 734         if (ret)
 735                 target = cpumask_any_but(&node_online_cpus, cpu);
 736         else
 737                 target = cpumask_any_but(cpu_online_mask, cpu);
 738
 739         if (target >= nr_cpu_ids)
 740                 return 0;
 741
 742         /* We're only reading, but this isn't the place to be involving RCU */
 743         mutex_lock(&ali_drw_pmu_irqs_lock);
 744         list_for_each_entry(drw_pmu, &irq->pmus_node, pmus_node)
 745                 perf_pmu_migrate_context(&drw_pmu->pmu, irq->cpu, target);
 746         mutex_unlock(&ali_drw_pmu_irqs_lock);
 747
 748         WARN_ON(irq_set_affinity_hint(irq->irq_num, cpumask_of(target)));
 749         irq->cpu = target;
 750
 751         return 0;
 752 }
 753
 754 /*
 755  * Due to historical reasons, the HID used in the production environment is
 756  * ARMHD700, so we leave ARMHD700 as Compatible ID.
 757  */
 758 static const struct acpi_device_id ali_drw_acpi_match[] = {
 759         {"BABA5000", 0},
 760         {"ARMHD700", 0},
 761         {}
 762 };
 763
 764 MODULE_DEVICE_TABLE(acpi, ali_drw_acpi_match);
 765
 766 static struct platform_driver ali_drw_pmu_driver = {
 767         .driver = {
 768                    .name = "ali_drw_pmu",
 769                    .acpi_match_table = ali_drw_acpi_match,
 770                    },
 771         .probe = ali_drw_pmu_probe,
 772         .remove = ali_drw_pmu_remove,
 773 };
 774
 775 static int __init ali_drw_pmu_init(void)
 776 {
 777         int ret;
 778
 779         ret = cpuhp_setup_state_multi(CPUHP_AP_ONLINE_DYN,
 780                                       "ali_drw_pmu:online",
 781                                       NULL, ali_drw_pmu_offline_cpu);
 782
 783         if (ret < 0) {
 784                 pr_err("DRW Driveway PMU: setup hotplug failed, ret = %d\n",
 785                        ret);
 786                 return ret;
 787         }
 788         ali_drw_cpuhp_state_num = ret;
 789
 790         ret = platform_driver_register(&ali_drw_pmu_driver);
 791         if (ret)
 792                 cpuhp_remove_multi_state(ali_drw_cpuhp_state_num);
 793
 794         return ret;
 795 }
 796
 797 static void __exit ali_drw_pmu_exit(void)
 798 {
 799         platform_driver_unregister(&ali_drw_pmu_driver);
 800         cpuhp_remove_multi_state(ali_drw_cpuhp_state_num);
 801 }
 802
 803 module_init(ali_drw_pmu_init);
 804 module_exit(ali_drw_pmu_exit);
 805
 806 MODULE_AUTHOR("Hongbo Yao <yaohongbo@linux.alibaba.com>");
 807 MODULE_AUTHOR("Neng Chen <nengchen@linux.alibaba.com>");
 808 MODULE_AUTHOR("Shuai Xue <xueshuai@linux.alibaba.com>");
 809 MODULE_DESCRIPTION("Alibaba DDR Sub-System Driveway PMU driver");
 810 MODULE_LICENSE("GPL v2");