drivers/cxl/core/region.c

   1 // SPDX-License-Identifier: GPL-2.0-only
   2 /* Copyright(c) 2022 Intel Corporation. All rights reserved. */
   3 #include <linux/memregion.h>
   4 #include <linux/genalloc.h>
   5 #include <linux/device.h>
   6 #include <linux/module.h>
   7 #include <linux/slab.h>
   8 #include <linux/uuid.h>
   9 #include <linux/sort.h>
  10 #include <linux/idr.h>
  11 #include <cxlmem.h>
  12 #include <cxl.h>
  13 #include "core.h"
  14
  15 /**
  16  * DOC: cxl core region
  17  *
  18  * CXL Regions represent mapped memory capacity in system physical address
  19  * space. Whereas the CXL Root Decoders identify the bounds of potential CXL
  20  * Memory ranges, Regions represent the active mapped capacity by the HDM
  21  * Decoder Capability structures throughout the Host Bridges, Switches, and
  22  * Endpoints in the topology.
  23  *
  24  * Region configuration has ordering constraints. UUID may be set at any time
  25  * but is only visible for persistent regions.
  26  * 1. Interleave granularity
  27  * 2. Interleave size
  28  * 3. Decoder targets
  29  */
  30
  31 static struct cxl_region *to_cxl_region(struct device *dev);
  32
  33 static ssize_t uuid_show(struct device *dev, struct device_attribute *attr,
  34                          char *buf)
  35 {
  36         struct cxl_region *cxlr = to_cxl_region(dev);
  37         struct cxl_region_params *p = &cxlr->params;
  38         ssize_t rc;
  39
  40         rc = down_read_interruptible(&cxl_region_rwsem);
  41         if (rc)
  42                 return rc;
  43         if (cxlr->mode != CXL_DECODER_PMEM)
  44                 rc = sysfs_emit(buf, "\n");
  45         else
  46                 rc = sysfs_emit(buf, "%pUb\n", &p->uuid);
  47         up_read(&cxl_region_rwsem);
  48
  49         return rc;
  50 }
  51
  52 static int is_dup(struct device *match, void *data)
  53 {
  54         struct cxl_region_params *p;
  55         struct cxl_region *cxlr;
  56         uuid_t *uuid = data;
  57
  58         if (!is_cxl_region(match))
  59                 return 0;
  60
  61         lockdep_assert_held(&cxl_region_rwsem);
  62         cxlr = to_cxl_region(match);
  63         p = &cxlr->params;
  64
  65         if (uuid_equal(&p->uuid, uuid)) {
  66                 dev_dbg(match, "already has uuid: %pUb\n", uuid);
  67                 return -EBUSY;
  68         }
  69
  70         return 0;
  71 }
  72
  73 static ssize_t uuid_store(struct device *dev, struct device_attribute *attr,
  74                           const char *buf, size_t len)
  75 {
  76         struct cxl_region *cxlr = to_cxl_region(dev);
  77         struct cxl_region_params *p = &cxlr->params;
  78         uuid_t temp;
  79         ssize_t rc;
  80
  81         if (len != UUID_STRING_LEN + 1)
  82                 return -EINVAL;
  83
  84         rc = uuid_parse(buf, &temp);
  85         if (rc)
  86                 return rc;
  87
  88         if (uuid_is_null(&temp))
  89                 return -EINVAL;
  90
  91         rc = down_write_killable(&cxl_region_rwsem);
  92         if (rc)
  93                 return rc;
  94
  95         if (uuid_equal(&p->uuid, &temp))
  96                 goto out;
  97
  98         rc = -EBUSY;
  99         if (p->state >= CXL_CONFIG_ACTIVE)
 100                 goto out;
 101
 102         rc = bus_for_each_dev(&cxl_bus_type, NULL, &temp, is_dup);
 103         if (rc < 0)
 104                 goto out;
 105
 106         uuid_copy(&p->uuid, &temp);
 107 out:
 108         up_write(&cxl_region_rwsem);
 109
 110         if (rc)
 111                 return rc;
 112         return len;
 113 }
 114 static DEVICE_ATTR_RW(uuid);
 115
 116 static struct cxl_region_ref *cxl_rr_load(struct cxl_port *port,
 117                                           struct cxl_region *cxlr)
 118 {
 119         return xa_load(&port->regions, (unsigned long)cxlr);
 120 }
 121
 122 static int cxl_region_invalidate_memregion(struct cxl_region *cxlr)
 123 {
 124         if (!cpu_cache_has_invalidate_memregion()) {
 125                 if (IS_ENABLED(CONFIG_CXL_REGION_INVALIDATION_TEST)) {
 126                         dev_warn_once(
 127                                 &cxlr->dev,
 128                                 "Bypassing cpu_cache_invalidate_memregion() for testing!\n");
 129                         return 0;
 130                 } else {
 131                         dev_err(&cxlr->dev,
 132                                 "Failed to synchronize CPU cache state\n");
 133                         return -ENXIO;
 134                 }
 135         }
 136
 137         cpu_cache_invalidate_memregion(IORES_DESC_CXL);
 138         return 0;
 139 }
 140
 141 static int cxl_region_decode_reset(struct cxl_region *cxlr, int count)
 142 {
 143         struct cxl_region_params *p = &cxlr->params;
 144         int i, rc = 0;
 145
 146         /*
 147          * Before region teardown attempt to flush, and if the flush
 148          * fails cancel the region teardown for data consistency
 149          * concerns
 150          */
 151         rc = cxl_region_invalidate_memregion(cxlr);
 152         if (rc)
 153                 return rc;
 154
 155         for (i = count - 1; i >= 0; i--) {
 156                 struct cxl_endpoint_decoder *cxled = p->targets[i];
 157                 struct cxl_memdev *cxlmd = cxled_to_memdev(cxled);
 158                 struct cxl_port *iter = cxled_to_port(cxled);
 159                 struct cxl_dev_state *cxlds = cxlmd->cxlds;
 160                 struct cxl_ep *ep;
 161
 162                 if (cxlds->rcd)
 163                         goto endpoint_reset;
 164
 165                 while (!is_cxl_root(to_cxl_port(iter->dev.parent)))
 166                         iter = to_cxl_port(iter->dev.parent);
 167
 168                 for (ep = cxl_ep_load(iter, cxlmd); iter;
 169                      iter = ep->next, ep = cxl_ep_load(iter, cxlmd)) {
 170                         struct cxl_region_ref *cxl_rr;
 171                         struct cxl_decoder *cxld;
 172
 173                         cxl_rr = cxl_rr_load(iter, cxlr);
 174                         cxld = cxl_rr->decoder;
 175                         if (cxld->reset)
 176                                 rc = cxld->reset(cxld);
 177                         if (rc)
 178                                 return rc;
 179                         set_bit(CXL_REGION_F_NEEDS_RESET, &cxlr->flags);
 180                 }
 181
 182 endpoint_reset:
 183                 rc = cxled->cxld.reset(&cxled->cxld);
 184                 if (rc)
 185                         return rc;
 186                 set_bit(CXL_REGION_F_NEEDS_RESET, &cxlr->flags);
 187         }
 188
 189         /* all decoders associated with this region have been torn down */
 190         clear_bit(CXL_REGION_F_NEEDS_RESET, &cxlr->flags);
 191
 192         return 0;
 193 }
 194
 195 static int commit_decoder(struct cxl_decoder *cxld)
 196 {
 197         struct cxl_switch_decoder *cxlsd = NULL;
 198
 199         if (cxld->commit)
 200                 return cxld->commit(cxld);
 201
 202         if (is_switch_decoder(&cxld->dev))
 203                 cxlsd = to_cxl_switch_decoder(&cxld->dev);
 204
 205         if (dev_WARN_ONCE(&cxld->dev, !cxlsd || cxlsd->nr_targets > 1,
 206                           "->commit() is required\n"))
 207                 return -ENXIO;
 208         return 0;
 209 }
 210
 211 static int cxl_region_decode_commit(struct cxl_region *cxlr)
 212 {
 213         struct cxl_region_params *p = &cxlr->params;
 214         int i, rc = 0;
 215
 216         for (i = 0; i < p->nr_targets; i++) {
 217                 struct cxl_endpoint_decoder *cxled = p->targets[i];
 218                 struct cxl_memdev *cxlmd = cxled_to_memdev(cxled);
 219                 struct cxl_region_ref *cxl_rr;
 220                 struct cxl_decoder *cxld;
 221                 struct cxl_port *iter;
 222                 struct cxl_ep *ep;
 223
 224                 /* commit bottom up */
 225                 for (iter = cxled_to_port(cxled); !is_cxl_root(iter);
 226                      iter = to_cxl_port(iter->dev.parent)) {
 227                         cxl_rr = cxl_rr_load(iter, cxlr);
 228                         cxld = cxl_rr->decoder;
 229                         rc = commit_decoder(cxld);
 230                         if (rc)
 231                                 break;
 232                 }
 233
 234                 if (rc) {
 235                         /* programming @iter failed, teardown */
 236                         for (ep = cxl_ep_load(iter, cxlmd); ep && iter;
 237                              iter = ep->next, ep = cxl_ep_load(iter, cxlmd)) {
 238                                 cxl_rr = cxl_rr_load(iter, cxlr);
 239                                 cxld = cxl_rr->decoder;
 240                                 if (cxld->reset)
 241                                         cxld->reset(cxld);
 242                         }
 243
 244                         cxled->cxld.reset(&cxled->cxld);
 245                         goto err;
 246                 }
 247         }
 248
 249         return 0;
 250
 251 err:
 252         /* undo the targets that were successfully committed */
 253         cxl_region_decode_reset(cxlr, i);
 254         return rc;
 255 }
 256
 257 static ssize_t commit_store(struct device *dev, struct device_attribute *attr,
 258                             const char *buf, size_t len)
 259 {
 260         struct cxl_region *cxlr = to_cxl_region(dev);
 261         struct cxl_region_params *p = &cxlr->params;
 262         bool commit;
 263         ssize_t rc;
 264
 265         rc = kstrtobool(buf, &commit);
 266         if (rc)
 267                 return rc;
 268
 269         rc = down_write_killable(&cxl_region_rwsem);
 270         if (rc)
 271                 return rc;
 272
 273         /* Already in the requested state? */
 274         if (commit && p->state >= CXL_CONFIG_COMMIT)
 275                 goto out;
 276         if (!commit && p->state < CXL_CONFIG_COMMIT)
 277                 goto out;
 278
 279         /* Not ready to commit? */
 280         if (commit && p->state < CXL_CONFIG_ACTIVE) {
 281                 rc = -ENXIO;
 282                 goto out;
 283         }
 284
 285         /*
 286          * Invalidate caches before region setup to drop any speculative
 287          * consumption of this address space
 288          */
 289         rc = cxl_region_invalidate_memregion(cxlr);
 290         if (rc)
 291                 goto out;
 292
 293         if (commit) {
 294                 rc = cxl_region_decode_commit(cxlr);
 295                 if (rc == 0)
 296                         p->state = CXL_CONFIG_COMMIT;
 297         } else {
 298                 p->state = CXL_CONFIG_RESET_PENDING;
 299                 up_write(&cxl_region_rwsem);
 300                 device_release_driver(&cxlr->dev);
 301                 down_write(&cxl_region_rwsem);
 302
 303                 /*
 304                  * The lock was dropped, so need to revalidate that the reset is
 305                  * still pending.
 306                  */
 307                 if (p->state == CXL_CONFIG_RESET_PENDING) {
 308                         rc = cxl_region_decode_reset(cxlr, p->interleave_ways);
 309                         /*
 310                          * Revert to committed since there may still be active
 311                          * decoders associated with this region, or move forward
 312                          * to active to mark the reset successful
 313                          */
 314                         if (rc)
 315                                 p->state = CXL_CONFIG_COMMIT;
 316                         else
 317                                 p->state = CXL_CONFIG_ACTIVE;
 318                 }
 319         }
 320
 321 out:
 322         up_write(&cxl_region_rwsem);
 323
 324         if (rc)
 325                 return rc;
 326         return len;
 327 }
 328
 329 static ssize_t commit_show(struct device *dev, struct device_attribute *attr,
 330                            char *buf)
 331 {
 332         struct cxl_region *cxlr = to_cxl_region(dev);
 333         struct cxl_region_params *p = &cxlr->params;
 334         ssize_t rc;
 335
 336         rc = down_read_interruptible(&cxl_region_rwsem);
 337         if (rc)
 338                 return rc;
 339         rc = sysfs_emit(buf, "%d\n", p->state >= CXL_CONFIG_COMMIT);
 340         up_read(&cxl_region_rwsem);
 341
 342         return rc;
 343 }
 344 static DEVICE_ATTR_RW(commit);
 345
 346 static umode_t cxl_region_visible(struct kobject *kobj, struct attribute *a,
 347                                   int n)
 348 {
 349         struct device *dev = kobj_to_dev(kobj);
 350         struct cxl_region *cxlr = to_cxl_region(dev);
 351
 352         /*
 353          * Support tooling that expects to find a 'uuid' attribute for all
 354          * regions regardless of mode.
 355          */
 356         if (a == &dev_attr_uuid.attr && cxlr->mode != CXL_DECODER_PMEM)
 357                 return 0444;
 358         return a->mode;
 359 }
 360
 361 static ssize_t interleave_ways_show(struct device *dev,
 362                                     struct device_attribute *attr, char *buf)
 363 {
 364         struct cxl_region *cxlr = to_cxl_region(dev);
 365         struct cxl_region_params *p = &cxlr->params;
 366         ssize_t rc;
 367
 368         rc = down_read_interruptible(&cxl_region_rwsem);
 369         if (rc)
 370                 return rc;
 371         rc = sysfs_emit(buf, "%d\n", p->interleave_ways);
 372         up_read(&cxl_region_rwsem);
 373
 374         return rc;
 375 }
 376
 377 static const struct attribute_group *get_cxl_region_target_group(void);
 378
 379 static ssize_t interleave_ways_store(struct device *dev,
 380                                      struct device_attribute *attr,
 381                                      const char *buf, size_t len)
 382 {
 383         struct cxl_root_decoder *cxlrd = to_cxl_root_decoder(dev->parent);
 384         struct cxl_decoder *cxld = &cxlrd->cxlsd.cxld;
 385         struct cxl_region *cxlr = to_cxl_region(dev);
 386         struct cxl_region_params *p = &cxlr->params;
 387         unsigned int val, save;
 388         int rc;
 389         u8 iw;
 390
 391         rc = kstrtouint(buf, 0, &val);
 392         if (rc)
 393                 return rc;
 394
 395         rc = ways_to_eiw(val, &iw);
 396         if (rc)
 397                 return rc;
 398
 399         /*
 400          * Even for x3, x9, and x12 interleaves the region interleave must be a
 401          * power of 2 multiple of the host bridge interleave.
 402          */
 403         if (!is_power_of_2(val / cxld->interleave_ways) ||
 404             (val % cxld->interleave_ways)) {
 405                 dev_dbg(&cxlr->dev, "invalid interleave: %d\n", val);
 406                 return -EINVAL;
 407         }
 408
 409         rc = down_write_killable(&cxl_region_rwsem);
 410         if (rc)
 411                 return rc;
 412         if (p->state >= CXL_CONFIG_INTERLEAVE_ACTIVE) {
 413                 rc = -EBUSY;
 414                 goto out;
 415         }
 416
 417         save = p->interleave_ways;
 418         p->interleave_ways = val;
 419         rc = sysfs_update_group(&cxlr->dev.kobj, get_cxl_region_target_group());
 420         if (rc)
 421                 p->interleave_ways = save;
 422 out:
 423         up_write(&cxl_region_rwsem);
 424         if (rc)
 425                 return rc;
 426         return len;
 427 }
 428 static DEVICE_ATTR_RW(interleave_ways);
 429
 430 static ssize_t interleave_granularity_show(struct device *dev,
 431                                            struct device_attribute *attr,
 432                                            char *buf)
 433 {
 434         struct cxl_region *cxlr = to_cxl_region(dev);
 435         struct cxl_region_params *p = &cxlr->params;
 436         ssize_t rc;
 437
 438         rc = down_read_interruptible(&cxl_region_rwsem);
 439         if (rc)
 440                 return rc;
 441         rc = sysfs_emit(buf, "%d\n", p->interleave_granularity);
 442         up_read(&cxl_region_rwsem);
 443
 444         return rc;
 445 }
 446
 447 static ssize_t interleave_granularity_store(struct device *dev,
 448                                             struct device_attribute *attr,
 449                                             const char *buf, size_t len)
 450 {
 451         struct cxl_root_decoder *cxlrd = to_cxl_root_decoder(dev->parent);
 452         struct cxl_decoder *cxld = &cxlrd->cxlsd.cxld;
 453         struct cxl_region *cxlr = to_cxl_region(dev);
 454         struct cxl_region_params *p = &cxlr->params;
 455         int rc, val;
 456         u16 ig;
 457
 458         rc = kstrtoint(buf, 0, &val);
 459         if (rc)
 460                 return rc;
 461
 462         rc = granularity_to_eig(val, &ig);
 463         if (rc)
 464                 return rc;
 465
 466         /*
 467          * When the host-bridge is interleaved, disallow region granularity !=
 468          * root granularity. Regions with a granularity less than the root
 469          * interleave result in needing multiple endpoints to support a single
 470          * slot in the interleave (possible to support in the future). Regions
 471          * with a granularity greater than the root interleave result in invalid
 472          * DPA translations (invalid to support).
 473          */
 474         if (cxld->interleave_ways > 1 && val != cxld->interleave_granularity)
 475                 return -EINVAL;
 476
 477         rc = down_write_killable(&cxl_region_rwsem);
 478         if (rc)
 479                 return rc;
 480         if (p->state >= CXL_CONFIG_INTERLEAVE_ACTIVE) {
 481                 rc = -EBUSY;
 482                 goto out;
 483         }
 484
 485         p->interleave_granularity = val;
 486 out:
 487         up_write(&cxl_region_rwsem);
 488         if (rc)
 489                 return rc;
 490         return len;
 491 }
 492 static DEVICE_ATTR_RW(interleave_granularity);
 493
 494 static ssize_t resource_show(struct device *dev, struct device_attribute *attr,
 495                              char *buf)
 496 {
 497         struct cxl_region *cxlr = to_cxl_region(dev);
 498         struct cxl_region_params *p = &cxlr->params;
 499         u64 resource = -1ULL;
 500         ssize_t rc;
 501
 502         rc = down_read_interruptible(&cxl_region_rwsem);
 503         if (rc)
 504                 return rc;
 505         if (p->res)
 506                 resource = p->res->start;
 507         rc = sysfs_emit(buf, "%#llx\n", resource);
 508         up_read(&cxl_region_rwsem);
 509
 510         return rc;
 511 }
 512 static DEVICE_ATTR_RO(resource);
 513
 514 static ssize_t mode_show(struct device *dev, struct device_attribute *attr,
 515                          char *buf)
 516 {
 517         struct cxl_region *cxlr = to_cxl_region(dev);
 518
 519         return sysfs_emit(buf, "%s\n", cxl_decoder_mode_name(cxlr->mode));
 520 }
 521 static DEVICE_ATTR_RO(mode);
 522
 523 static int alloc_hpa(struct cxl_region *cxlr, resource_size_t size)
 524 {
 525         struct cxl_root_decoder *cxlrd = to_cxl_root_decoder(cxlr->dev.parent);
 526         struct cxl_region_params *p = &cxlr->params;
 527         struct resource *res;
 528         u32 remainder = 0;
 529
 530         lockdep_assert_held_write(&cxl_region_rwsem);
 531
 532         /* Nothing to do... */
 533         if (p->res && resource_size(p->res) == size)
 534                 return 0;
 535
 536         /* To change size the old size must be freed first */
 537         if (p->res)
 538                 return -EBUSY;
 539
 540         if (p->state >= CXL_CONFIG_INTERLEAVE_ACTIVE)
 541                 return -EBUSY;
 542
 543         /* ways, granularity and uuid (if PMEM) need to be set before HPA */
 544         if (!p->interleave_ways || !p->interleave_granularity ||
 545             (cxlr->mode == CXL_DECODER_PMEM && uuid_is_null(&p->uuid)))
 546                 return -ENXIO;
 547
 548         div_u64_rem(size, SZ_256M * p->interleave_ways, &remainder);
 549         if (remainder)
 550                 return -EINVAL;
 551
 552         res = alloc_free_mem_region(cxlrd->res, size, SZ_256M,
 553                                     dev_name(&cxlr->dev));
 554         if (IS_ERR(res)) {
 555                 dev_dbg(&cxlr->dev, "failed to allocate HPA: %ld\n",
 556                         PTR_ERR(res));
 557                 return PTR_ERR(res);
 558         }
 559
 560         p->res = res;
 561         p->state = CXL_CONFIG_INTERLEAVE_ACTIVE;
 562
 563         return 0;
 564 }
 565
 566 static void cxl_region_iomem_release(struct cxl_region *cxlr)
 567 {
 568         struct cxl_region_params *p = &cxlr->params;
 569
 570         if (device_is_registered(&cxlr->dev))
 571                 lockdep_assert_held_write(&cxl_region_rwsem);
 572         if (p->res) {
 573                 /*
 574                  * Autodiscovered regions may not have been able to insert their
 575                  * resource.
 576                  */
 577                 if (p->res->parent)
 578                         remove_resource(p->res);
 579                 kfree(p->res);
 580                 p->res = NULL;
 581         }
 582 }
 583
 584 static int free_hpa(struct cxl_region *cxlr)
 585 {
 586         struct cxl_region_params *p = &cxlr->params;
 587
 588         lockdep_assert_held_write(&cxl_region_rwsem);
 589
 590         if (!p->res)
 591                 return 0;
 592
 593         if (p->state >= CXL_CONFIG_ACTIVE)
 594                 return -EBUSY;
 595
 596         cxl_region_iomem_release(cxlr);
 597         p->state = CXL_CONFIG_IDLE;
 598         return 0;
 599 }
 600
 601 static ssize_t size_store(struct device *dev, struct device_attribute *attr,
 602                           const char *buf, size_t len)
 603 {
 604         struct cxl_region *cxlr = to_cxl_region(dev);
 605         u64 val;
 606         int rc;
 607
 608         rc = kstrtou64(buf, 0, &val);
 609         if (rc)
 610                 return rc;
 611
 612         rc = down_write_killable(&cxl_region_rwsem);
 613         if (rc)
 614                 return rc;
 615
 616         if (val)
 617                 rc = alloc_hpa(cxlr, val);
 618         else
 619                 rc = free_hpa(cxlr);
 620         up_write(&cxl_region_rwsem);
 621
 622         if (rc)
 623                 return rc;
 624
 625         return len;
 626 }
 627
 628 static ssize_t size_show(struct device *dev, struct device_attribute *attr,
 629                          char *buf)
 630 {
 631         struct cxl_region *cxlr = to_cxl_region(dev);
 632         struct cxl_region_params *p = &cxlr->params;
 633         u64 size = 0;
 634         ssize_t rc;
 635
 636         rc = down_read_interruptible(&cxl_region_rwsem);
 637         if (rc)
 638                 return rc;
 639         if (p->res)
 640                 size = resource_size(p->res);
 641         rc = sysfs_emit(buf, "%#llx\n", size);
 642         up_read(&cxl_region_rwsem);
 643
 644         return rc;
 645 }
 646 static DEVICE_ATTR_RW(size);
 647
 648 static struct attribute *cxl_region_attrs[] = {
 649         &dev_attr_uuid.attr,
 650         &dev_attr_commit.attr,
 651         &dev_attr_interleave_ways.attr,
 652         &dev_attr_interleave_granularity.attr,
 653         &dev_attr_resource.attr,
 654         &dev_attr_size.attr,
 655         &dev_attr_mode.attr,
 656         NULL,
 657 };
 658
 659 static const struct attribute_group cxl_region_group = {
 660         .attrs = cxl_region_attrs,
 661         .is_visible = cxl_region_visible,
 662 };
 663
 664 static size_t show_targetN(struct cxl_region *cxlr, char *buf, int pos)
 665 {
 666         struct cxl_region_params *p = &cxlr->params;
 667         struct cxl_endpoint_decoder *cxled;
 668         int rc;
 669
 670         rc = down_read_interruptible(&cxl_region_rwsem);
 671         if (rc)
 672                 return rc;
 673
 674         if (pos >= p->interleave_ways) {
 675                 dev_dbg(&cxlr->dev, "position %d out of range %d\n", pos,
 676                         p->interleave_ways);
 677                 rc = -ENXIO;
 678                 goto out;
 679         }
 680
 681         cxled = p->targets[pos];
 682         if (!cxled)
 683                 rc = sysfs_emit(buf, "\n");
 684         else
 685                 rc = sysfs_emit(buf, "%s\n", dev_name(&cxled->cxld.dev));
 686 out:
 687         up_read(&cxl_region_rwsem);
 688
 689         return rc;
 690 }
 691
 692 static int match_free_decoder(struct device *dev, void *data)
 693 {
 694         struct cxl_decoder *cxld;
 695         int *id = data;
 696
 697         if (!is_switch_decoder(dev))
 698                 return 0;
 699
 700         cxld = to_cxl_decoder(dev);
 701
 702         /* enforce ordered allocation */
 703         if (cxld->id != *id)
 704                 return 0;
 705
 706         if (!cxld->region)
 707                 return 1;
 708
 709         (*id)++;
 710
 711         return 0;
 712 }
 713
 714 static int match_auto_decoder(struct device *dev, void *data)
 715 {
 716         struct cxl_region_params *p = data;
 717         struct cxl_decoder *cxld;
 718         struct range *r;
 719
 720         if (!is_switch_decoder(dev))
 721                 return 0;
 722
 723         cxld = to_cxl_decoder(dev);
 724         r = &cxld->hpa_range;
 725
 726         if (p->res && p->res->start == r->start && p->res->end == r->end)
 727                 return 1;
 728
 729         return 0;
 730 }
 731
 732 static struct cxl_decoder *cxl_region_find_decoder(struct cxl_port *port,
 733                                                    struct cxl_region *cxlr)
 734 {
 735         struct device *dev;
 736         int id = 0;
 737
 738         if (test_bit(CXL_REGION_F_AUTO, &cxlr->flags))
 739                 dev = device_find_child(&port->dev, &cxlr->params,
 740                                         match_auto_decoder);
 741         else
 742                 dev = device_find_child(&port->dev, &id, match_free_decoder);
 743         if (!dev)
 744                 return NULL;
 745         /*
 746          * This decoder is pinned registered as long as the endpoint decoder is
 747          * registered, and endpoint decoder unregistration holds the
 748          * cxl_region_rwsem over unregister events, so no need to hold on to
 749          * this extra reference.
 750          */
 751         put_device(dev);
 752         return to_cxl_decoder(dev);
 753 }
 754
 755 static struct cxl_region_ref *alloc_region_ref(struct cxl_port *port,
 756                                                struct cxl_region *cxlr)
 757 {
 758         struct cxl_region_params *p = &cxlr->params;
 759         struct cxl_region_ref *cxl_rr, *iter;
 760         unsigned long index;
 761         int rc;
 762
 763         xa_for_each(&port->regions, index, iter) {
 764                 struct cxl_region_params *ip = &iter->region->params;
 765
 766                 if (!ip->res)
 767                         continue;
 768
 769                 if (ip->res->start > p->res->start) {
 770                         dev_dbg(&cxlr->dev,
 771                                 "%s: HPA order violation %s:%pr vs %pr\n",
 772                                 dev_name(&port->dev),
 773                                 dev_name(&iter->region->dev), ip->res, p->res);
 774                         return ERR_PTR(-EBUSY);
 775                 }
 776         }
 777
 778         cxl_rr = kzalloc(sizeof(*cxl_rr), GFP_KERNEL);
 779         if (!cxl_rr)
 780                 return ERR_PTR(-ENOMEM);
 781         cxl_rr->port = port;
 782         cxl_rr->region = cxlr;
 783         cxl_rr->nr_targets = 1;
 784         xa_init(&cxl_rr->endpoints);
 785
 786         rc = xa_insert(&port->regions, (unsigned long)cxlr, cxl_rr, GFP_KERNEL);
 787         if (rc) {
 788                 dev_dbg(&cxlr->dev,
 789                         "%s: failed to track region reference: %d\n",
 790                         dev_name(&port->dev), rc);
 791                 kfree(cxl_rr);
 792                 return ERR_PTR(rc);
 793         }
 794
 795         return cxl_rr;
 796 }
 797
 798 static void cxl_rr_free_decoder(struct cxl_region_ref *cxl_rr)
 799 {
 800         struct cxl_region *cxlr = cxl_rr->region;
 801         struct cxl_decoder *cxld = cxl_rr->decoder;
 802
 803         if (!cxld)
 804                 return;
 805
 806         dev_WARN_ONCE(&cxlr->dev, cxld->region != cxlr, "region mismatch\n");
 807         if (cxld->region == cxlr) {
 808                 cxld->region = NULL;
 809                 put_device(&cxlr->dev);
 810         }
 811 }
 812
 813 static void free_region_ref(struct cxl_region_ref *cxl_rr)
 814 {
 815         struct cxl_port *port = cxl_rr->port;
 816         struct cxl_region *cxlr = cxl_rr->region;
 817
 818         cxl_rr_free_decoder(cxl_rr);
 819         xa_erase(&port->regions, (unsigned long)cxlr);
 820         xa_destroy(&cxl_rr->endpoints);
 821         kfree(cxl_rr);
 822 }
 823
 824 static int cxl_rr_ep_add(struct cxl_region_ref *cxl_rr,
 825                          struct cxl_endpoint_decoder *cxled)
 826 {
 827         int rc;
 828         struct cxl_port *port = cxl_rr->port;
 829         struct cxl_region *cxlr = cxl_rr->region;
 830         struct cxl_decoder *cxld = cxl_rr->decoder;
 831         struct cxl_ep *ep = cxl_ep_load(port, cxled_to_memdev(cxled));
 832
 833         if (ep) {
 834                 rc = xa_insert(&cxl_rr->endpoints, (unsigned long)cxled, ep,
 835                                GFP_KERNEL);
 836                 if (rc)
 837                         return rc;
 838         }
 839         cxl_rr->nr_eps++;
 840
 841         if (!cxld->region) {
 842                 cxld->region = cxlr;
 843                 get_device(&cxlr->dev);
 844         }
 845
 846         return 0;
 847 }
 848
 849 static int cxl_rr_alloc_decoder(struct cxl_port *port, struct cxl_region *cxlr,
 850                                 struct cxl_endpoint_decoder *cxled,
 851                                 struct cxl_region_ref *cxl_rr)
 852 {
 853         struct cxl_decoder *cxld;
 854
 855         if (port == cxled_to_port(cxled))
 856                 cxld = &cxled->cxld;
 857         else
 858                 cxld = cxl_region_find_decoder(port, cxlr);
 859         if (!cxld) {
 860                 dev_dbg(&cxlr->dev, "%s: no decoder available\n",
 861                         dev_name(&port->dev));
 862                 return -EBUSY;
 863         }
 864
 865         if (cxld->region) {
 866                 dev_dbg(&cxlr->dev, "%s: %s already attached to %s\n",
 867                         dev_name(&port->dev), dev_name(&cxld->dev),
 868                         dev_name(&cxld->region->dev));
 869                 return -EBUSY;
 870         }
 871
 872         /*
 873          * Endpoints should already match the region type, but backstop that
 874          * assumption with an assertion. Switch-decoders change mapping-type
 875          * based on what is mapped when they are assigned to a region.
 876          */
 877         dev_WARN_ONCE(&cxlr->dev,
 878                       port == cxled_to_port(cxled) &&
 879                               cxld->target_type != cxlr->type,
 880                       "%s:%s mismatch decoder type %d -> %d\n",
 881                       dev_name(&cxled_to_memdev(cxled)->dev),
 882                       dev_name(&cxld->dev), cxld->target_type, cxlr->type);
 883         cxld->target_type = cxlr->type;
 884         cxl_rr->decoder = cxld;
 885         return 0;
 886 }
 887
 888 /**
 889  * cxl_port_attach_region() - track a region's interest in a port by endpoint
 890  * @port: port to add a new region reference 'struct cxl_region_ref'
 891  * @cxlr: region to attach to @port
 892  * @cxled: endpoint decoder used to create or further pin a region reference
 893  * @pos: interleave position of @cxled in @cxlr
 894  *
 895  * The attach event is an opportunity to validate CXL decode setup
 896  * constraints and record metadata needed for programming HDM decoders,
 897  * in particular decoder target lists.
 898  *
 899  * The steps are:
 900  *
 901  * - validate that there are no other regions with a higher HPA already
 902  *   associated with @port
 903  * - establish a region reference if one is not already present
 904  *
 905  *   - additionally allocate a decoder instance that will host @cxlr on
 906  *     @port
 907  *
 908  * - pin the region reference by the endpoint
 909  * - account for how many entries in @port's target list are needed to
 910  *   cover all of the added endpoints.
 911  */
 912 static int cxl_port_attach_region(struct cxl_port *port,
 913                                   struct cxl_region *cxlr,
 914                                   struct cxl_endpoint_decoder *cxled, int pos)
 915 {
 916         struct cxl_memdev *cxlmd = cxled_to_memdev(cxled);
 917         struct cxl_ep *ep = cxl_ep_load(port, cxlmd);
 918         struct cxl_region_ref *cxl_rr;
 919         bool nr_targets_inc = false;
 920         struct cxl_decoder *cxld;
 921         unsigned long index;
 922         int rc = -EBUSY;
 923
 924         lockdep_assert_held_write(&cxl_region_rwsem);
 925
 926         cxl_rr = cxl_rr_load(port, cxlr);
 927         if (cxl_rr) {
 928                 struct cxl_ep *ep_iter;
 929                 int found = 0;
 930
 931                 /*
 932                  * Walk the existing endpoints that have been attached to
 933                  * @cxlr at @port and see if they share the same 'next' port
 934                  * in the downstream direction. I.e. endpoints that share common
 935                  * upstream switch.
 936                  */
 937                 xa_for_each(&cxl_rr->endpoints, index, ep_iter) {
 938                         if (ep_iter == ep)
 939                                 continue;
 940                         if (ep_iter->next == ep->next) {
 941                                 found++;
 942                                 break;
 943                         }
 944                 }
 945
 946                 /*
 947                  * New target port, or @port is an endpoint port that always
 948                  * accounts its own local decode as a target.
 949                  */
 950                 if (!found || !ep->next) {
 951                         cxl_rr->nr_targets++;
 952                         nr_targets_inc = true;
 953                 }
 954         } else {
 955                 cxl_rr = alloc_region_ref(port, cxlr);
 956                 if (IS_ERR(cxl_rr)) {
 957                         dev_dbg(&cxlr->dev,
 958                                 "%s: failed to allocate region reference\n",
 959                                 dev_name(&port->dev));
 960                         return PTR_ERR(cxl_rr);
 961                 }
 962                 nr_targets_inc = true;
 963
 964                 rc = cxl_rr_alloc_decoder(port, cxlr, cxled, cxl_rr);
 965                 if (rc)
 966                         goto out_erase;
 967         }
 968         cxld = cxl_rr->decoder;
 969
 970         rc = cxl_rr_ep_add(cxl_rr, cxled);
 971         if (rc) {
 972                 dev_dbg(&cxlr->dev,
 973                         "%s: failed to track endpoint %s:%s reference\n",
 974                         dev_name(&port->dev), dev_name(&cxlmd->dev),
 975                         dev_name(&cxld->dev));
 976                 goto out_erase;
 977         }
 978
 979         dev_dbg(&cxlr->dev,
 980                 "%s:%s %s add: %s:%s @ %d next: %s nr_eps: %d nr_targets: %d\n",
 981                 dev_name(port->uport_dev), dev_name(&port->dev),
 982                 dev_name(&cxld->dev), dev_name(&cxlmd->dev),
 983                 dev_name(&cxled->cxld.dev), pos,
 984                 ep ? ep->next ? dev_name(ep->next->uport_dev) :
 985                                       dev_name(&cxlmd->dev) :
 986                            "none",
 987                 cxl_rr->nr_eps, cxl_rr->nr_targets);
 988
 989         return 0;
 990 out_erase:
 991         if (nr_targets_inc)
 992                 cxl_rr->nr_targets--;
 993         if (cxl_rr->nr_eps == 0)
 994                 free_region_ref(cxl_rr);
 995         return rc;
 996 }
 997
 998 static void cxl_port_detach_region(struct cxl_port *port,
 999                                    struct cxl_region *cxlr,
1000                                    struct cxl_endpoint_decoder *cxled)
1001 {
1002         struct cxl_region_ref *cxl_rr;
1003         struct cxl_ep *ep = NULL;
1004
1005         lockdep_assert_held_write(&cxl_region_rwsem);
1006
1007         cxl_rr = cxl_rr_load(port, cxlr);
1008         if (!cxl_rr)
1009                 return;
1010
1011         /*
1012          * Endpoint ports do not carry cxl_ep references, and they
1013          * never target more than one endpoint by definition
1014          */
1015         if (cxl_rr->decoder == &cxled->cxld)
1016                 cxl_rr->nr_eps--;
1017         else
1018                 ep = xa_erase(&cxl_rr->endpoints, (unsigned long)cxled);
1019         if (ep) {
1020                 struct cxl_ep *ep_iter;
1021                 unsigned long index;
1022                 int found = 0;
1023
1024                 cxl_rr->nr_eps--;
1025                 xa_for_each(&cxl_rr->endpoints, index, ep_iter) {
1026                         if (ep_iter->next == ep->next) {
1027                                 found++;
1028                                 break;
1029                         }
1030                 }
1031                 if (!found)
1032                         cxl_rr->nr_targets--;
1033         }
1034
1035         if (cxl_rr->nr_eps == 0)
1036                 free_region_ref(cxl_rr);
1037 }
1038
1039 static int check_last_peer(struct cxl_endpoint_decoder *cxled,
1040                            struct cxl_ep *ep, struct cxl_region_ref *cxl_rr,
1041                            int distance)
1042 {
1043         struct cxl_memdev *cxlmd = cxled_to_memdev(cxled);
1044         struct cxl_region *cxlr = cxl_rr->region;
1045         struct cxl_region_params *p = &cxlr->params;
1046         struct cxl_endpoint_decoder *cxled_peer;
1047         struct cxl_port *port = cxl_rr->port;
1048         struct cxl_memdev *cxlmd_peer;
1049         struct cxl_ep *ep_peer;
1050         int pos = cxled->pos;
1051
1052         /*
1053          * If this position wants to share a dport with the last endpoint mapped
1054          * then that endpoint, at index 'position - distance', must also be
1055          * mapped by this dport.
1056          */
1057         if (pos < distance) {
1058                 dev_dbg(&cxlr->dev, "%s:%s: cannot host %s:%s at %d\n",
1059                         dev_name(port->uport_dev), dev_name(&port->dev),
1060                         dev_name(&cxlmd->dev), dev_name(&cxled->cxld.dev), pos);
1061                 return -ENXIO;
1062         }
1063         cxled_peer = p->targets[pos - distance];
1064         cxlmd_peer = cxled_to_memdev(cxled_peer);
1065         ep_peer = cxl_ep_load(port, cxlmd_peer);
1066         if (ep->dport != ep_peer->dport) {
1067                 dev_dbg(&cxlr->dev,
1068                         "%s:%s: %s:%s pos %d mismatched peer %s:%s\n",
1069                         dev_name(port->uport_dev), dev_name(&port->dev),
1070                         dev_name(&cxlmd->dev), dev_name(&cxled->cxld.dev), pos,
1071                         dev_name(&cxlmd_peer->dev),
1072                         dev_name(&cxled_peer->cxld.dev));
1073                 return -ENXIO;
1074         }
1075
1076         return 0;
1077 }
1078
1079 static int cxl_port_setup_targets(struct cxl_port *port,
1080                                   struct cxl_region *cxlr,
1081                                   struct cxl_endpoint_decoder *cxled)
1082 {
1083         struct cxl_root_decoder *cxlrd = to_cxl_root_decoder(cxlr->dev.parent);
1084         int parent_iw, parent_ig, ig, iw, rc, inc = 0, pos = cxled->pos;
1085         struct cxl_port *parent_port = to_cxl_port(port->dev.parent);
1086         struct cxl_region_ref *cxl_rr = cxl_rr_load(port, cxlr);
1087         struct cxl_memdev *cxlmd = cxled_to_memdev(cxled);
1088         struct cxl_ep *ep = cxl_ep_load(port, cxlmd);
1089         struct cxl_region_params *p = &cxlr->params;
1090         struct cxl_decoder *cxld = cxl_rr->decoder;
1091         struct cxl_switch_decoder *cxlsd;
1092         u16 eig, peig;
1093         u8 eiw, peiw;
1094
1095         /*
1096          * While root level decoders support x3, x6, x12, switch level
1097          * decoders only support powers of 2 up to x16.
1098          */
1099         if (!is_power_of_2(cxl_rr->nr_targets)) {
1100                 dev_dbg(&cxlr->dev, "%s:%s: invalid target count %d\n",
1101                         dev_name(port->uport_dev), dev_name(&port->dev),
1102                         cxl_rr->nr_targets);
1103                 return -EINVAL;
1104         }
1105
1106         cxlsd = to_cxl_switch_decoder(&cxld->dev);
1107         if (cxl_rr->nr_targets_set) {
1108                 int i, distance;
1109
1110                 /*
1111                  * Passthrough decoders impose no distance requirements between
1112                  * peers
1113                  */
1114                 if (cxl_rr->nr_targets == 1)
1115                         distance = 0;
1116                 else
1117                         distance = p->nr_targets / cxl_rr->nr_targets;
1118                 for (i = 0; i < cxl_rr->nr_targets_set; i++)
1119                         if (ep->dport == cxlsd->target[i]) {
1120                                 rc = check_last_peer(cxled, ep, cxl_rr,
1121                                                      distance);
1122                                 if (rc)
1123                                         return rc;
1124                                 goto out_target_set;
1125                         }
1126                 goto add_target;
1127         }
1128
1129         if (is_cxl_root(parent_port)) {
1130                 parent_ig = cxlrd->cxlsd.cxld.interleave_granularity;
1131                 parent_iw = cxlrd->cxlsd.cxld.interleave_ways;
1132                 /*
1133                  * For purposes of address bit routing, use power-of-2 math for
1134                  * switch ports.
1135                  */
1136                 if (!is_power_of_2(parent_iw))
1137                         parent_iw /= 3;
1138         } else {
1139                 struct cxl_region_ref *parent_rr;
1140                 struct cxl_decoder *parent_cxld;
1141
1142                 parent_rr = cxl_rr_load(parent_port, cxlr);
1143                 parent_cxld = parent_rr->decoder;
1144                 parent_ig = parent_cxld->interleave_granularity;
1145                 parent_iw = parent_cxld->interleave_ways;
1146         }
1147
1148         rc = granularity_to_eig(parent_ig, &peig);
1149         if (rc) {
1150                 dev_dbg(&cxlr->dev, "%s:%s: invalid parent granularity: %d\n",
1151                         dev_name(parent_port->uport_dev),
1152                         dev_name(&parent_port->dev), parent_ig);
1153                 return rc;
1154         }
1155
1156         rc = ways_to_eiw(parent_iw, &peiw);
1157         if (rc) {
1158                 dev_dbg(&cxlr->dev, "%s:%s: invalid parent interleave: %d\n",
1159                         dev_name(parent_port->uport_dev),
1160                         dev_name(&parent_port->dev), parent_iw);
1161                 return rc;
1162         }
1163
1164         iw = cxl_rr->nr_targets;
1165         rc = ways_to_eiw(iw, &eiw);
1166         if (rc) {
1167                 dev_dbg(&cxlr->dev, "%s:%s: invalid port interleave: %d\n",
1168                         dev_name(port->uport_dev), dev_name(&port->dev), iw);
1169                 return rc;
1170         }
1171
1172         /*
1173          * Interleave granularity is a multiple of @parent_port granularity.
1174          * Multiplier is the parent port interleave ways.
1175          */
1176         rc = granularity_to_eig(parent_ig * parent_iw, &eig);
1177         if (rc) {
1178                 dev_dbg(&cxlr->dev,
1179                         "%s: invalid granularity calculation (%d * %d)\n",
1180                         dev_name(&parent_port->dev), parent_ig, parent_iw);
1181                 return rc;
1182         }
1183
1184         rc = eig_to_granularity(eig, &ig);
1185         if (rc) {
1186                 dev_dbg(&cxlr->dev, "%s:%s: invalid interleave: %d\n",
1187                         dev_name(port->uport_dev), dev_name(&port->dev),
1188                         256 << eig);
1189                 return rc;
1190         }
1191
1192         if (iw > 8 || iw > cxlsd->nr_targets) {
1193                 dev_dbg(&cxlr->dev,
1194                         "%s:%s:%s: ways: %d overflows targets: %d\n",
1195                         dev_name(port->uport_dev), dev_name(&port->dev),
1196                         dev_name(&cxld->dev), iw, cxlsd->nr_targets);
1197                 return -ENXIO;
1198         }
1199
1200         if (test_bit(CXL_REGION_F_AUTO, &cxlr->flags)) {
1201                 if (cxld->interleave_ways != iw ||
1202                     cxld->interleave_granularity != ig ||
1203                     cxld->hpa_range.start != p->res->start ||
1204                     cxld->hpa_range.end != p->res->end ||
1205                     ((cxld->flags & CXL_DECODER_F_ENABLE) == 0)) {
1206                         dev_err(&cxlr->dev,
1207                                 "%s:%s %s expected iw: %d ig: %d %pr\n",
1208                                 dev_name(port->uport_dev), dev_name(&port->dev),
1209                                 __func__, iw, ig, p->res);
1210                         dev_err(&cxlr->dev,
1211                                 "%s:%s %s got iw: %d ig: %d state: %s %#llx:%#llx\n",
1212                                 dev_name(port->uport_dev), dev_name(&port->dev),
1213                                 __func__, cxld->interleave_ways,
1214                                 cxld->interleave_granularity,
1215                                 (cxld->flags & CXL_DECODER_F_ENABLE) ?
1216                                         "enabled" :
1217                                         "disabled",
1218                                 cxld->hpa_range.start, cxld->hpa_range.end);
1219                         return -ENXIO;
1220                 }
1221         } else {
1222                 cxld->interleave_ways = iw;
1223                 cxld->interleave_granularity = ig;
1224                 cxld->hpa_range = (struct range) {
1225                         .start = p->res->start,
1226                         .end = p->res->end,
1227                 };
1228         }
1229         dev_dbg(&cxlr->dev, "%s:%s iw: %d ig: %d\n", dev_name(port->uport_dev),
1230                 dev_name(&port->dev), iw, ig);
1231 add_target:
1232         if (cxl_rr->nr_targets_set == cxl_rr->nr_targets) {
1233                 dev_dbg(&cxlr->dev,
1234                         "%s:%s: targets full trying to add %s:%s at %d\n",
1235                         dev_name(port->uport_dev), dev_name(&port->dev),
1236                         dev_name(&cxlmd->dev), dev_name(&cxled->cxld.dev), pos);
1237                 return -ENXIO;
1238         }
1239         if (test_bit(CXL_REGION_F_AUTO, &cxlr->flags)) {
1240                 if (cxlsd->target[cxl_rr->nr_targets_set] != ep->dport) {
1241                         dev_dbg(&cxlr->dev, "%s:%s: %s expected %s at %d\n",
1242                                 dev_name(port->uport_dev), dev_name(&port->dev),
1243                                 dev_name(&cxlsd->cxld.dev),
1244                                 dev_name(ep->dport->dport_dev),
1245                                 cxl_rr->nr_targets_set);
1246                         return -ENXIO;
1247                 }
1248         } else
1249                 cxlsd->target[cxl_rr->nr_targets_set] = ep->dport;
1250         inc = 1;
1251 out_target_set:
1252         cxl_rr->nr_targets_set += inc;
1253         dev_dbg(&cxlr->dev, "%s:%s target[%d] = %s for %s:%s @ %d\n",
1254                 dev_name(port->uport_dev), dev_name(&port->dev),
1255                 cxl_rr->nr_targets_set - 1, dev_name(ep->dport->dport_dev),
1256                 dev_name(&cxlmd->dev), dev_name(&cxled->cxld.dev), pos);
1257
1258         return 0;
1259 }
1260
1261 static void cxl_port_reset_targets(struct cxl_port *port,
1262                                    struct cxl_region *cxlr)
1263 {
1264         struct cxl_region_ref *cxl_rr = cxl_rr_load(port, cxlr);
1265         struct cxl_decoder *cxld;
1266
1267         /*
1268          * After the last endpoint has been detached the entire cxl_rr may now
1269          * be gone.
1270          */
1271         if (!cxl_rr)
1272                 return;
1273         cxl_rr->nr_targets_set = 0;
1274
1275         cxld = cxl_rr->decoder;
1276         cxld->hpa_range = (struct range) {
1277                 .start = 0,
1278                 .end = -1,
1279         };
1280 }
1281
1282 static void cxl_region_teardown_targets(struct cxl_region *cxlr)
1283 {
1284         struct cxl_region_params *p = &cxlr->params;
1285         struct cxl_endpoint_decoder *cxled;
1286         struct cxl_dev_state *cxlds;
1287         struct cxl_memdev *cxlmd;
1288         struct cxl_port *iter;
1289         struct cxl_ep *ep;
1290         int i;
1291
1292         /*
1293          * In the auto-discovery case skip automatic teardown since the
1294          * address space is already active
1295          */
1296         if (test_bit(CXL_REGION_F_AUTO, &cxlr->flags))
1297                 return;
1298
1299         for (i = 0; i < p->nr_targets; i++) {
1300                 cxled = p->targets[i];
1301                 cxlmd = cxled_to_memdev(cxled);
1302                 cxlds = cxlmd->cxlds;
1303
1304                 if (cxlds->rcd)
1305                         continue;
1306
1307                 iter = cxled_to_port(cxled);
1308                 while (!is_cxl_root(to_cxl_port(iter->dev.parent)))
1309                         iter = to_cxl_port(iter->dev.parent);
1310
1311                 for (ep = cxl_ep_load(iter, cxlmd); iter;
1312                      iter = ep->next, ep = cxl_ep_load(iter, cxlmd))
1313                         cxl_port_reset_targets(iter, cxlr);
1314         }
1315 }
1316
1317 static int cxl_region_setup_targets(struct cxl_region *cxlr)
1318 {
1319         struct cxl_region_params *p = &cxlr->params;
1320         struct cxl_endpoint_decoder *cxled;
1321         struct cxl_dev_state *cxlds;
1322         int i, rc, rch = 0, vh = 0;
1323         struct cxl_memdev *cxlmd;
1324         struct cxl_port *iter;
1325         struct cxl_ep *ep;
1326
1327         for (i = 0; i < p->nr_targets; i++) {
1328                 cxled = p->targets[i];
1329                 cxlmd = cxled_to_memdev(cxled);
1330                 cxlds = cxlmd->cxlds;
1331
1332                 /* validate that all targets agree on topology */
1333                 if (!cxlds->rcd) {
1334                         vh++;
1335                 } else {
1336                         rch++;
1337                         continue;
1338                 }
1339
1340                 iter = cxled_to_port(cxled);
1341                 while (!is_cxl_root(to_cxl_port(iter->dev.parent)))
1342                         iter = to_cxl_port(iter->dev.parent);
1343
1344                 /*
1345                  * Descend the topology tree programming / validating
1346                  * targets while looking for conflicts.
1347                  */
1348                 for (ep = cxl_ep_load(iter, cxlmd); iter;
1349                      iter = ep->next, ep = cxl_ep_load(iter, cxlmd)) {
1350                         rc = cxl_port_setup_targets(iter, cxlr, cxled);
1351                         if (rc) {
1352                                 cxl_region_teardown_targets(cxlr);
1353                                 return rc;
1354                         }
1355                 }
1356         }
1357
1358         if (rch && vh) {
1359                 dev_err(&cxlr->dev, "mismatched CXL topologies detected\n");
1360                 cxl_region_teardown_targets(cxlr);
1361                 return -ENXIO;
1362         }
1363
1364         return 0;
1365 }
1366
1367 static int cxl_region_validate_position(struct cxl_region *cxlr,
1368                                         struct cxl_endpoint_decoder *cxled,
1369                                         int pos)
1370 {
1371         struct cxl_memdev *cxlmd = cxled_to_memdev(cxled);
1372         struct cxl_region_params *p = &cxlr->params;
1373         int i;
1374
1375         if (pos < 0 || pos >= p->interleave_ways) {
1376                 dev_dbg(&cxlr->dev, "position %d out of range %d\n", pos,
1377                         p->interleave_ways);
1378                 return -ENXIO;
1379         }
1380
1381         if (p->targets[pos] == cxled)
1382                 return 0;
1383
1384         if (p->targets[pos]) {
1385                 struct cxl_endpoint_decoder *cxled_target = p->targets[pos];
1386                 struct cxl_memdev *cxlmd_target = cxled_to_memdev(cxled_target);
1387
1388                 dev_dbg(&cxlr->dev, "position %d already assigned to %s:%s\n",
1389                         pos, dev_name(&cxlmd_target->dev),
1390                         dev_name(&cxled_target->cxld.dev));
1391                 return -EBUSY;
1392         }
1393
1394         for (i = 0; i < p->interleave_ways; i++) {
1395                 struct cxl_endpoint_decoder *cxled_target;
1396                 struct cxl_memdev *cxlmd_target;
1397
1398                 cxled_target = p->targets[i];
1399                 if (!cxled_target)
1400                         continue;
1401
1402                 cxlmd_target = cxled_to_memdev(cxled_target);
1403                 if (cxlmd_target == cxlmd) {
1404                         dev_dbg(&cxlr->dev,
1405                                 "%s already specified at position %d via: %s\n",
1406                                 dev_name(&cxlmd->dev), pos,
1407                                 dev_name(&cxled_target->cxld.dev));
1408                         return -EBUSY;
1409                 }
1410         }
1411
1412         return 0;
1413 }
1414
1415 static int cxl_region_attach_position(struct cxl_region *cxlr,
1416                                       struct cxl_root_decoder *cxlrd,
1417                                       struct cxl_endpoint_decoder *cxled,
1418                                       const struct cxl_dport *dport, int pos)
1419 {
1420         struct cxl_memdev *cxlmd = cxled_to_memdev(cxled);
1421         struct cxl_port *iter;
1422         int rc;
1423
1424         if (cxlrd->calc_hb(cxlrd, pos) != dport) {
1425                 dev_dbg(&cxlr->dev, "%s:%s invalid target position for %s\n",
1426                         dev_name(&cxlmd->dev), dev_name(&cxled->cxld.dev),
1427                         dev_name(&cxlrd->cxlsd.cxld.dev));
1428                 return -ENXIO;
1429         }
1430
1431         for (iter = cxled_to_port(cxled); !is_cxl_root(iter);
1432              iter = to_cxl_port(iter->dev.parent)) {
1433                 rc = cxl_port_attach_region(iter, cxlr, cxled, pos);
1434                 if (rc)
1435                         goto err;
1436         }
1437
1438         return 0;
1439
1440 err:
1441         for (iter = cxled_to_port(cxled); !is_cxl_root(iter);
1442              iter = to_cxl_port(iter->dev.parent))
1443                 cxl_port_detach_region(iter, cxlr, cxled);
1444         return rc;
1445 }
1446
1447 static int cxl_region_attach_auto(struct cxl_region *cxlr,
1448                                   struct cxl_endpoint_decoder *cxled, int pos)
1449 {
1450         struct cxl_region_params *p = &cxlr->params;
1451
1452         if (cxled->state != CXL_DECODER_STATE_AUTO) {
1453                 dev_err(&cxlr->dev,
1454                         "%s: unable to add decoder to autodetected region\n",
1455                         dev_name(&cxled->cxld.dev));
1456                 return -EINVAL;
1457         }
1458
1459         if (pos >= 0) {
1460                 dev_dbg(&cxlr->dev, "%s: expected auto position, not %d\n",
1461                         dev_name(&cxled->cxld.dev), pos);
1462                 return -EINVAL;
1463         }
1464
1465         if (p->nr_targets >= p->interleave_ways) {
1466                 dev_err(&cxlr->dev, "%s: no more target slots available\n",
1467                         dev_name(&cxled->cxld.dev));
1468                 return -ENXIO;
1469         }
1470
1471         /*
1472          * Temporarily record the endpoint decoder into the target array. Yes,
1473          * this means that userspace can view devices in the wrong position
1474          * before the region activates, and must be careful to understand when
1475          * it might be racing region autodiscovery.
1476          */
1477         pos = p->nr_targets;
1478         p->targets[pos] = cxled;
1479         cxled->pos = pos;
1480         p->nr_targets++;
1481
1482         return 0;
1483 }
1484
1485 static int cmp_interleave_pos(const void *a, const void *b)
1486 {
1487         struct cxl_endpoint_decoder *cxled_a = *(typeof(cxled_a) *)a;
1488         struct cxl_endpoint_decoder *cxled_b = *(typeof(cxled_b) *)b;
1489
1490         return cxled_a->pos - cxled_b->pos;
1491 }
1492
1493 static struct cxl_port *next_port(struct cxl_port *port)
1494 {
1495         if (!port->parent_dport)
1496                 return NULL;
1497         return port->parent_dport->port;
1498 }
1499
1500 static int match_switch_decoder_by_range(struct device *dev, void *data)
1501 {
1502         struct cxl_switch_decoder *cxlsd;
1503         struct range *r1, *r2 = data;
1504
1505         if (!is_switch_decoder(dev))
1506                 return 0;
1507
1508         cxlsd = to_cxl_switch_decoder(dev);
1509         r1 = &cxlsd->cxld.hpa_range;
1510
1511         if (is_root_decoder(dev))
1512                 return range_contains(r1, r2);
1513         return (r1->start == r2->start && r1->end == r2->end);
1514 }
1515
1516 static int find_pos_and_ways(struct cxl_port *port, struct range *range,
1517                              int *pos, int *ways)
1518 {
1519         struct cxl_switch_decoder *cxlsd;
1520         struct cxl_port *parent;
1521         struct device *dev;
1522         int rc = -ENXIO;
1523
1524         parent = next_port(port);
1525         if (!parent)
1526                 return rc;
1527
1528         dev = device_find_child(&parent->dev, range,
1529                                 match_switch_decoder_by_range);
1530         if (!dev) {
1531                 dev_err(port->uport_dev,
1532                         "failed to find decoder mapping %#llx-%#llx\n",
1533                         range->start, range->end);
1534                 return rc;
1535         }
1536         cxlsd = to_cxl_switch_decoder(dev);
1537         *ways = cxlsd->cxld.interleave_ways;
1538
1539         for (int i = 0; i < *ways; i++) {
1540                 if (cxlsd->target[i] == port->parent_dport) {
1541                         *pos = i;
1542                         rc = 0;
1543                         break;
1544                 }
1545         }
1546         put_device(dev);
1547
1548         return rc;
1549 }
1550
1551 /**
1552  * cxl_calc_interleave_pos() - calculate an endpoint position in a region
1553  * @cxled: endpoint decoder member of given region
1554  *
1555  * The endpoint position is calculated by traversing the topology from
1556  * the endpoint to the root decoder and iteratively applying this
1557  * calculation:
1558  *
1559  *    position = position * parent_ways + parent_pos;
1560  *
1561  * ...where @position is inferred from switch and root decoder target lists.
1562  *
1563  * Return: position >= 0 on success
1564  *         -ENXIO on failure
1565  */
1566 static int cxl_calc_interleave_pos(struct cxl_endpoint_decoder *cxled)
1567 {
1568         struct cxl_port *iter, *port = cxled_to_port(cxled);
1569         struct cxl_memdev *cxlmd = cxled_to_memdev(cxled);
1570         struct range *range = &cxled->cxld.hpa_range;
1571         int parent_ways = 0, parent_pos = 0, pos = 0;
1572         int rc;
1573
1574         /*
1575          * Example: the expected interleave order of the 4-way region shown
1576          * below is: mem0, mem2, mem1, mem3
1577          *
1578          *                root_port
1579          *                 /      \
1580          *      host_bridge_0    host_bridge_1
1581          *        |    |           |    |
1582          *       mem0 mem1        mem2 mem3
1583          *
1584          * In the example the calculator will iterate twice. The first iteration
1585          * uses the mem position in the host-bridge and the ways of the host-
1586          * bridge to generate the first, or local, position. The second
1587          * iteration uses the host-bridge position in the root_port and the ways
1588          * of the root_port to refine the position.
1589          *
1590          * A trace of the calculation per endpoint looks like this:
1591          * mem0: pos = 0 * 2 + 0    mem2: pos = 0 * 2 + 0
1592          *       pos = 0 * 2 + 0          pos = 0 * 2 + 1
1593          *       pos: 0                   pos: 1
1594          *
1595          * mem1: pos = 0 * 2 + 1    mem3: pos = 0 * 2 + 1
1596          *       pos = 1 * 2 + 0          pos = 1 * 2 + 1
1597          *       pos: 2                   pos = 3
1598          *
1599          * Note that while this example is simple, the method applies to more
1600          * complex topologies, including those with switches.
1601          */
1602
1603         /* Iterate from endpoint to root_port refining the position */
1604         for (iter = port; iter; iter = next_port(iter)) {
1605                 if (is_cxl_root(iter))
1606                         break;
1607
1608                 rc = find_pos_and_ways(iter, range, &parent_pos, &parent_ways);
1609                 if (rc)
1610                         return rc;
1611
1612                 pos = pos * parent_ways + parent_pos;
1613         }
1614
1615         dev_dbg(&cxlmd->dev,
1616                 "decoder:%s parent:%s port:%s range:%#llx-%#llx pos:%d\n",
1617                 dev_name(&cxled->cxld.dev), dev_name(cxlmd->dev.parent),
1618                 dev_name(&port->dev), range->start, range->end, pos);
1619
1620         return pos;
1621 }
1622
1623 static int cxl_region_sort_targets(struct cxl_region *cxlr)
1624 {
1625         struct cxl_region_params *p = &cxlr->params;
1626         int i, rc = 0;
1627
1628         for (i = 0; i < p->nr_targets; i++) {
1629                 struct cxl_endpoint_decoder *cxled = p->targets[i];
1630
1631                 cxled->pos = cxl_calc_interleave_pos(cxled);
1632                 /*
1633                  * Record that sorting failed, but still continue to calc
1634                  * cxled->pos so that follow-on code paths can reliably
1635                  * do p->targets[cxled->pos] to self-reference their entry.
1636                  */
1637                 if (cxled->pos < 0)
1638                         rc = -ENXIO;
1639         }
1640         /* Keep the cxlr target list in interleave position order */
1641         sort(p->targets, p->nr_targets, sizeof(p->targets[0]),
1642              cmp_interleave_pos, NULL);
1643
1644         dev_dbg(&cxlr->dev, "region sort %s\n", rc ? "failed" : "successful");
1645         return rc;
1646 }
1647
1648 static int cxl_region_attach(struct cxl_region *cxlr,
1649                              struct cxl_endpoint_decoder *cxled, int pos)
1650 {
1651         struct cxl_root_decoder *cxlrd = to_cxl_root_decoder(cxlr->dev.parent);
1652         struct cxl_memdev *cxlmd = cxled_to_memdev(cxled);
1653         struct cxl_region_params *p = &cxlr->params;
1654         struct cxl_port *ep_port, *root_port;
1655         struct cxl_dport *dport;
1656         int rc = -ENXIO;
1657
1658         if (cxled->mode != cxlr->mode) {
1659                 dev_dbg(&cxlr->dev, "%s region mode: %d mismatch: %d\n",
1660                         dev_name(&cxled->cxld.dev), cxlr->mode, cxled->mode);
1661                 return -EINVAL;
1662         }
1663
1664         if (cxled->mode == CXL_DECODER_DEAD) {
1665                 dev_dbg(&cxlr->dev, "%s dead\n", dev_name(&cxled->cxld.dev));
1666                 return -ENODEV;
1667         }
1668
1669         /* all full of members, or interleave config not established? */
1670         if (p->state > CXL_CONFIG_INTERLEAVE_ACTIVE) {
1671                 dev_dbg(&cxlr->dev, "region already active\n");
1672                 return -EBUSY;
1673         } else if (p->state < CXL_CONFIG_INTERLEAVE_ACTIVE) {
1674                 dev_dbg(&cxlr->dev, "interleave config missing\n");
1675                 return -ENXIO;
1676         }
1677
1678         if (p->nr_targets >= p->interleave_ways) {
1679                 dev_dbg(&cxlr->dev, "region already has %d endpoints\n",
1680                         p->nr_targets);
1681                 return -EINVAL;
1682         }
1683
1684         ep_port = cxled_to_port(cxled);
1685         root_port = cxlrd_to_port(cxlrd);
1686         dport = cxl_find_dport_by_dev(root_port, ep_port->host_bridge);
1687         if (!dport) {
1688                 dev_dbg(&cxlr->dev, "%s:%s invalid target for %s\n",
1689                         dev_name(&cxlmd->dev), dev_name(&cxled->cxld.dev),
1690                         dev_name(cxlr->dev.parent));
1691                 return -ENXIO;
1692         }
1693
1694         if (cxled->cxld.target_type != cxlr->type) {
1695                 dev_dbg(&cxlr->dev, "%s:%s type mismatch: %d vs %d\n",
1696                         dev_name(&cxlmd->dev), dev_name(&cxled->cxld.dev),
1697                         cxled->cxld.target_type, cxlr->type);
1698                 return -ENXIO;
1699         }
1700
1701         if (!cxled->dpa_res) {
1702                 dev_dbg(&cxlr->dev, "%s:%s: missing DPA allocation.\n",
1703                         dev_name(&cxlmd->dev), dev_name(&cxled->cxld.dev));
1704                 return -ENXIO;
1705         }
1706
1707         if (resource_size(cxled->dpa_res) * p->interleave_ways !=
1708             resource_size(p->res)) {
1709                 dev_dbg(&cxlr->dev,
1710                         "%s:%s: decoder-size-%#llx * ways-%d != region-size-%#llx\n",
1711                         dev_name(&cxlmd->dev), dev_name(&cxled->cxld.dev),
1712                         (u64)resource_size(cxled->dpa_res), p->interleave_ways,
1713                         (u64)resource_size(p->res));
1714                 return -EINVAL;
1715         }
1716
1717         if (test_bit(CXL_REGION_F_AUTO, &cxlr->flags)) {
1718                 int i;
1719
1720                 rc = cxl_region_attach_auto(cxlr, cxled, pos);
1721                 if (rc)
1722                         return rc;
1723
1724                 /* await more targets to arrive... */
1725                 if (p->nr_targets < p->interleave_ways)
1726                         return 0;
1727
1728                 /*
1729                  * All targets are here, which implies all PCI enumeration that
1730                  * affects this region has been completed. Walk the topology to
1731                  * sort the devices into their relative region decode position.
1732                  */
1733                 rc = cxl_region_sort_targets(cxlr);
1734                 if (rc)
1735                         return rc;
1736
1737                 for (i = 0; i < p->nr_targets; i++) {
1738                         cxled = p->targets[i];
1739                         ep_port = cxled_to_port(cxled);
1740                         dport = cxl_find_dport_by_dev(root_port,
1741                                                       ep_port->host_bridge);
1742                         rc = cxl_region_attach_position(cxlr, cxlrd, cxled,
1743                                                         dport, i);
1744                         if (rc)
1745                                 return rc;
1746                 }
1747
1748                 rc = cxl_region_setup_targets(cxlr);
1749                 if (rc)
1750                         return rc;
1751
1752                 /*
1753                  * If target setup succeeds in the autodiscovery case
1754                  * then the region is already committed.
1755                  */
1756                 p->state = CXL_CONFIG_COMMIT;
1757
1758                 return 0;
1759         }
1760
1761         rc = cxl_region_validate_position(cxlr, cxled, pos);
1762         if (rc)
1763                 return rc;
1764
1765         rc = cxl_region_attach_position(cxlr, cxlrd, cxled, dport, pos);
1766         if (rc)
1767                 return rc;
1768
1769         p->targets[pos] = cxled;
1770         cxled->pos = pos;
1771         p->nr_targets++;
1772
1773         if (p->nr_targets == p->interleave_ways) {
1774                 rc = cxl_region_setup_targets(cxlr);
1775                 if (rc)
1776                         return rc;
1777                 p->state = CXL_CONFIG_ACTIVE;
1778         }
1779
1780         cxled->cxld.interleave_ways = p->interleave_ways;
1781         cxled->cxld.interleave_granularity = p->interleave_granularity;
1782         cxled->cxld.hpa_range = (struct range) {
1783                 .start = p->res->start,
1784                 .end = p->res->end,
1785         };
1786
1787         if (p->nr_targets != p->interleave_ways)
1788                 return 0;
1789
1790         /*
1791          * Test the auto-discovery position calculator function
1792          * against this successfully created user-defined region.
1793          * A fail message here means that this interleave config
1794          * will fail when presented as CXL_REGION_F_AUTO.
1795          */
1796         for (int i = 0; i < p->nr_targets; i++) {
1797                 struct cxl_endpoint_decoder *cxled = p->targets[i];
1798                 int test_pos;
1799
1800                 test_pos = cxl_calc_interleave_pos(cxled);
1801                 dev_dbg(&cxled->cxld.dev,
1802                         "Test cxl_calc_interleave_pos(): %s test_pos:%d cxled->pos:%d\n",
1803                         (test_pos == cxled->pos) ? "success" : "fail",
1804                         test_pos, cxled->pos);
1805         }
1806
1807         return 0;
1808 }
1809
1810 static int cxl_region_detach(struct cxl_endpoint_decoder *cxled)
1811 {
1812         struct cxl_port *iter, *ep_port = cxled_to_port(cxled);
1813         struct cxl_region *cxlr = cxled->cxld.region;
1814         struct cxl_region_params *p;
1815         int rc = 0;
1816
1817         lockdep_assert_held_write(&cxl_region_rwsem);
1818
1819         if (!cxlr)
1820                 return 0;
1821
1822         p = &cxlr->params;
1823         get_device(&cxlr->dev);
1824
1825         if (p->state > CXL_CONFIG_ACTIVE) {
1826                 /*
1827                  * TODO: tear down all impacted regions if a device is
1828                  * removed out of order
1829                  */
1830                 rc = cxl_region_decode_reset(cxlr, p->interleave_ways);
1831                 if (rc)
1832                         goto out;
1833                 p->state = CXL_CONFIG_ACTIVE;
1834         }
1835
1836         for (iter = ep_port; !is_cxl_root(iter);
1837              iter = to_cxl_port(iter->dev.parent))
1838                 cxl_port_detach_region(iter, cxlr, cxled);
1839
1840         if (cxled->pos < 0 || cxled->pos >= p->interleave_ways ||
1841             p->targets[cxled->pos] != cxled) {
1842                 struct cxl_memdev *cxlmd = cxled_to_memdev(cxled);
1843
1844                 dev_WARN_ONCE(&cxlr->dev, 1, "expected %s:%s at position %d\n",
1845                               dev_name(&cxlmd->dev), dev_name(&cxled->cxld.dev),
1846                               cxled->pos);
1847                 goto out;
1848         }
1849
1850         if (p->state == CXL_CONFIG_ACTIVE) {
1851                 p->state = CXL_CONFIG_INTERLEAVE_ACTIVE;
1852                 cxl_region_teardown_targets(cxlr);
1853         }
1854         p->targets[cxled->pos] = NULL;
1855         p->nr_targets--;
1856         cxled->cxld.hpa_range = (struct range) {
1857                 .start = 0,
1858                 .end = -1,
1859         };
1860
1861         /* notify the region driver that one of its targets has departed */
1862         up_write(&cxl_region_rwsem);
1863         device_release_driver(&cxlr->dev);
1864         down_write(&cxl_region_rwsem);
1865 out:
1866         put_device(&cxlr->dev);
1867         return rc;
1868 }
1869
1870 void cxl_decoder_kill_region(struct cxl_endpoint_decoder *cxled)
1871 {
1872         down_write(&cxl_region_rwsem);
1873         cxled->mode = CXL_DECODER_DEAD;
1874         cxl_region_detach(cxled);
1875         up_write(&cxl_region_rwsem);
1876 }
1877
1878 static int attach_target(struct cxl_region *cxlr,
1879                          struct cxl_endpoint_decoder *cxled, int pos,
1880                          unsigned int state)
1881 {
1882         int rc = 0;
1883
1884         if (state == TASK_INTERRUPTIBLE)
1885                 rc = down_write_killable(&cxl_region_rwsem);
1886         else
1887                 down_write(&cxl_region_rwsem);
1888         if (rc)
1889                 return rc;
1890
1891         down_read(&cxl_dpa_rwsem);
1892         rc = cxl_region_attach(cxlr, cxled, pos);
1893         up_read(&cxl_dpa_rwsem);
1894         up_write(&cxl_region_rwsem);
1895         return rc;
1896 }
1897
1898 static int detach_target(struct cxl_region *cxlr, int pos)
1899 {
1900         struct cxl_region_params *p = &cxlr->params;
1901         int rc;
1902
1903         rc = down_write_killable(&cxl_region_rwsem);
1904         if (rc)
1905                 return rc;
1906
1907         if (pos >= p->interleave_ways) {
1908                 dev_dbg(&cxlr->dev, "position %d out of range %d\n", pos,
1909                         p->interleave_ways);
1910                 rc = -ENXIO;
1911                 goto out;
1912         }
1913
1914         if (!p->targets[pos]) {
1915                 rc = 0;
1916                 goto out;
1917         }
1918
1919         rc = cxl_region_detach(p->targets[pos]);
1920 out:
1921         up_write(&cxl_region_rwsem);
1922         return rc;
1923 }
1924
1925 static size_t store_targetN(struct cxl_region *cxlr, const char *buf, int pos,
1926                             size_t len)
1927 {
1928         int rc;
1929
1930         if (sysfs_streq(buf, "\n"))
1931                 rc = detach_target(cxlr, pos);
1932         else {
1933                 struct device *dev;
1934
1935                 dev = bus_find_device_by_name(&cxl_bus_type, NULL, buf);
1936                 if (!dev)
1937                         return -ENODEV;
1938
1939                 if (!is_endpoint_decoder(dev)) {
1940                         rc = -EINVAL;
1941                         goto out;
1942                 }
1943
1944                 rc = attach_target(cxlr, to_cxl_endpoint_decoder(dev), pos,
1945                                    TASK_INTERRUPTIBLE);
1946 out:
1947                 put_device(dev);
1948         }
1949
1950         if (rc < 0)
1951                 return rc;
1952         return len;
1953 }
1954
1955 #define TARGET_ATTR_RW(n)                                              \
1956 static ssize_t target##n##_show(                                       \
1957         struct device *dev, struct device_attribute *attr, char *buf)  \
1958 {                                                                      \
1959         return show_targetN(to_cxl_region(dev), buf, (n));             \
1960 }                                                                      \
1961 static ssize_t target##n##_store(struct device *dev,                   \
1962                                  struct device_attribute *attr,        \
1963                                  const char *buf, size_t len)          \
1964 {                                                                      \
1965         return store_targetN(to_cxl_region(dev), buf, (n), len);       \
1966 }                                                                      \
1967 static DEVICE_ATTR_RW(target##n)
1968
1969 TARGET_ATTR_RW(0);
1970 TARGET_ATTR_RW(1);
1971 TARGET_ATTR_RW(2);
1972 TARGET_ATTR_RW(3);
1973 TARGET_ATTR_RW(4);
1974 TARGET_ATTR_RW(5);
1975 TARGET_ATTR_RW(6);
1976 TARGET_ATTR_RW(7);
1977 TARGET_ATTR_RW(8);
1978 TARGET_ATTR_RW(9);
1979 TARGET_ATTR_RW(10);
1980 TARGET_ATTR_RW(11);
1981 TARGET_ATTR_RW(12);
1982 TARGET_ATTR_RW(13);
1983 TARGET_ATTR_RW(14);
1984 TARGET_ATTR_RW(15);
1985
1986 static struct attribute *target_attrs[] = {
1987         &dev_attr_target0.attr,
1988         &dev_attr_target1.attr,
1989         &dev_attr_target2.attr,
1990         &dev_attr_target3.attr,
1991         &dev_attr_target4.attr,
1992         &dev_attr_target5.attr,
1993         &dev_attr_target6.attr,
1994         &dev_attr_target7.attr,
1995         &dev_attr_target8.attr,
1996         &dev_attr_target9.attr,
1997         &dev_attr_target10.attr,
1998         &dev_attr_target11.attr,
1999         &dev_attr_target12.attr,
2000         &dev_attr_target13.attr,
2001         &dev_attr_target14.attr,
2002         &dev_attr_target15.attr,
2003         NULL,
2004 };
2005
2006 static umode_t cxl_region_target_visible(struct kobject *kobj,
2007                                          struct attribute *a, int n)
2008 {
2009         struct device *dev = kobj_to_dev(kobj);
2010         struct cxl_region *cxlr = to_cxl_region(dev);
2011         struct cxl_region_params *p = &cxlr->params;
2012
2013         if (n < p->interleave_ways)
2014                 return a->mode;
2015         return 0;
2016 }
2017
2018 static const struct attribute_group cxl_region_target_group = {
2019         .attrs = target_attrs,
2020         .is_visible = cxl_region_target_visible,
2021 };
2022
2023 static const struct attribute_group *get_cxl_region_target_group(void)
2024 {
2025         return &cxl_region_target_group;
2026 }
2027
2028 static const struct attribute_group *region_groups[] = {
2029         &cxl_base_attribute_group,
2030         &cxl_region_group,
2031         &cxl_region_target_group,
2032         NULL,
2033 };
2034
2035 static void cxl_region_release(struct device *dev)
2036 {
2037         struct cxl_root_decoder *cxlrd = to_cxl_root_decoder(dev->parent);
2038         struct cxl_region *cxlr = to_cxl_region(dev);
2039         int id = atomic_read(&cxlrd->region_id);
2040
2041         /*
2042          * Try to reuse the recently idled id rather than the cached
2043          * next id to prevent the region id space from increasing
2044          * unnecessarily.
2045          */
2046         if (cxlr->id < id)
2047                 if (atomic_try_cmpxchg(&cxlrd->region_id, &id, cxlr->id)) {
2048                         memregion_free(id);
2049                         goto out;
2050                 }
2051
2052         memregion_free(cxlr->id);
2053 out:
2054         put_device(dev->parent);
2055         kfree(cxlr);
2056 }
2057
2058 const struct device_type cxl_region_type = {
2059         .name = "cxl_region",
2060         .release = cxl_region_release,
2061         .groups = region_groups
2062 };
2063
2064 bool is_cxl_region(struct device *dev)
2065 {
2066         return dev->type == &cxl_region_type;
2067 }
2068 EXPORT_SYMBOL_NS_GPL(is_cxl_region, CXL);
2069
2070 static struct cxl_region *to_cxl_region(struct device *dev)
2071 {
2072         if (dev_WARN_ONCE(dev, dev->type != &cxl_region_type,
2073                           "not a cxl_region device\n"))
2074                 return NULL;
2075
2076         return container_of(dev, struct cxl_region, dev);
2077 }
2078
2079 static void unregister_region(void *dev)
2080 {
2081         struct cxl_region *cxlr = to_cxl_region(dev);
2082         struct cxl_region_params *p = &cxlr->params;
2083         int i;
2084
2085         device_del(dev);
2086
2087         /*
2088          * Now that region sysfs is shutdown, the parameter block is now
2089          * read-only, so no need to hold the region rwsem to access the
2090          * region parameters.
2091          */
2092         for (i = 0; i < p->interleave_ways; i++)
2093                 detach_target(cxlr, i);
2094
2095         cxl_region_iomem_release(cxlr);
2096         put_device(dev);
2097 }
2098
2099 static struct lock_class_key cxl_region_key;
2100
2101 static struct cxl_region *cxl_region_alloc(struct cxl_root_decoder *cxlrd, int id)
2102 {
2103         struct cxl_region *cxlr;
2104         struct device *dev;
2105
2106         cxlr = kzalloc(sizeof(*cxlr), GFP_KERNEL);
2107         if (!cxlr) {
2108                 memregion_free(id);
2109                 return ERR_PTR(-ENOMEM);
2110         }
2111
2112         dev = &cxlr->dev;
2113         device_initialize(dev);
2114         lockdep_set_class(&dev->mutex, &cxl_region_key);
2115         dev->parent = &cxlrd->cxlsd.cxld.dev;
2116         /*
2117          * Keep root decoder pinned through cxl_region_release to fixup
2118          * region id allocations
2119          */
2120         get_device(dev->parent);
2121         device_set_pm_not_required(dev);
2122         dev->bus = &cxl_bus_type;
2123         dev->type = &cxl_region_type;
2124         cxlr->id = id;
2125
2126         return cxlr;
2127 }
2128
2129 /**
2130  * devm_cxl_add_region - Adds a region to a decoder
2131  * @cxlrd: root decoder
2132  * @id: memregion id to create, or memregion_free() on failure
2133  * @mode: mode for the endpoint decoders of this region
2134  * @type: select whether this is an expander or accelerator (type-2 or type-3)
2135  *
2136  * This is the second step of region initialization. Regions exist within an
2137  * address space which is mapped by a @cxlrd.
2138  *
2139  * Return: 0 if the region was added to the @cxlrd, else returns negative error
2140  * code. The region will be named "regionZ" where Z is the unique region number.
2141  */
2142 static struct cxl_region *devm_cxl_add_region(struct cxl_root_decoder *cxlrd,
2143                                               int id,
2144                                               enum cxl_decoder_mode mode,
2145                                               enum cxl_decoder_type type)
2146 {
2147         struct cxl_port *port = to_cxl_port(cxlrd->cxlsd.cxld.dev.parent);
2148         struct cxl_region *cxlr;
2149         struct device *dev;
2150         int rc;
2151
2152         switch (mode) {
2153         case CXL_DECODER_RAM:
2154         case CXL_DECODER_PMEM:
2155                 break;
2156         default:
2157                 dev_err(&cxlrd->cxlsd.cxld.dev, "unsupported mode %d\n", mode);
2158                 return ERR_PTR(-EINVAL);
2159         }
2160
2161         cxlr = cxl_region_alloc(cxlrd, id);
2162         if (IS_ERR(cxlr))
2163                 return cxlr;
2164         cxlr->mode = mode;
2165         cxlr->type = type;
2166
2167         dev = &cxlr->dev;
2168         rc = dev_set_name(dev, "region%d", id);
2169         if (rc)
2170                 goto err;
2171
2172         rc = device_add(dev);
2173         if (rc)
2174                 goto err;
2175
2176         rc = devm_add_action_or_reset(port->uport_dev, unregister_region, cxlr);
2177         if (rc)
2178                 return ERR_PTR(rc);
2179
2180         dev_dbg(port->uport_dev, "%s: created %s\n",
2181                 dev_name(&cxlrd->cxlsd.cxld.dev), dev_name(dev));
2182         return cxlr;
2183
2184 err:
2185         put_device(dev);
2186         return ERR_PTR(rc);
2187 }
2188
2189 static ssize_t __create_region_show(struct cxl_root_decoder *cxlrd, char *buf)
2190 {
2191         return sysfs_emit(buf, "region%u\n", atomic_read(&cxlrd->region_id));
2192 }
2193
2194 static ssize_t create_pmem_region_show(struct device *dev,
2195                                        struct device_attribute *attr, char *buf)
2196 {
2197         return __create_region_show(to_cxl_root_decoder(dev), buf);
2198 }
2199
2200 static ssize_t create_ram_region_show(struct device *dev,
2201                                       struct device_attribute *attr, char *buf)
2202 {
2203         return __create_region_show(to_cxl_root_decoder(dev), buf);
2204 }
2205
2206 static struct cxl_region *__create_region(struct cxl_root_decoder *cxlrd,
2207                                           enum cxl_decoder_mode mode, int id)
2208 {
2209         int rc;
2210
2211         rc = memregion_alloc(GFP_KERNEL);
2212         if (rc < 0)
2213                 return ERR_PTR(rc);
2214
2215         if (atomic_cmpxchg(&cxlrd->region_id, id, rc) != id) {
2216                 memregion_free(rc);
2217                 return ERR_PTR(-EBUSY);
2218         }
2219
2220         return devm_cxl_add_region(cxlrd, id, mode, CXL_DECODER_HOSTONLYMEM);
2221 }
2222
2223 static ssize_t create_pmem_region_store(struct device *dev,
2224                                         struct device_attribute *attr,
2225                                         const char *buf, size_t len)
2226 {
2227         struct cxl_root_decoder *cxlrd = to_cxl_root_decoder(dev);
2228         struct cxl_region *cxlr;
2229         int rc, id;
2230
2231         rc = sscanf(buf, "region%d\n", &id);
2232         if (rc != 1)
2233                 return -EINVAL;
2234
2235         cxlr = __create_region(cxlrd, CXL_DECODER_PMEM, id);
2236         if (IS_ERR(cxlr))
2237                 return PTR_ERR(cxlr);
2238
2239         return len;
2240 }
2241 DEVICE_ATTR_RW(create_pmem_region);
2242
2243 static ssize_t create_ram_region_store(struct device *dev,
2244                                        struct device_attribute *attr,
2245                                        const char *buf, size_t len)
2246 {
2247         struct cxl_root_decoder *cxlrd = to_cxl_root_decoder(dev);
2248         struct cxl_region *cxlr;
2249         int rc, id;
2250
2251         rc = sscanf(buf, "region%d\n", &id);
2252         if (rc != 1)
2253                 return -EINVAL;
2254
2255         cxlr = __create_region(cxlrd, CXL_DECODER_RAM, id);
2256         if (IS_ERR(cxlr))
2257                 return PTR_ERR(cxlr);
2258
2259         return len;
2260 }
2261 DEVICE_ATTR_RW(create_ram_region);
2262
2263 static ssize_t region_show(struct device *dev, struct device_attribute *attr,
2264                            char *buf)
2265 {
2266         struct cxl_decoder *cxld = to_cxl_decoder(dev);
2267         ssize_t rc;
2268
2269         rc = down_read_interruptible(&cxl_region_rwsem);
2270         if (rc)
2271                 return rc;
2272
2273         if (cxld->region)
2274                 rc = sysfs_emit(buf, "%s\n", dev_name(&cxld->region->dev));
2275         else
2276                 rc = sysfs_emit(buf, "\n");
2277         up_read(&cxl_region_rwsem);
2278
2279         return rc;
2280 }
2281 DEVICE_ATTR_RO(region);
2282
2283 static struct cxl_region *
2284 cxl_find_region_by_name(struct cxl_root_decoder *cxlrd, const char *name)
2285 {
2286         struct cxl_decoder *cxld = &cxlrd->cxlsd.cxld;
2287         struct device *region_dev;
2288
2289         region_dev = device_find_child_by_name(&cxld->dev, name);
2290         if (!region_dev)
2291                 return ERR_PTR(-ENODEV);
2292
2293         return to_cxl_region(region_dev);
2294 }
2295
2296 static ssize_t delete_region_store(struct device *dev,
2297                                    struct device_attribute *attr,
2298                                    const char *buf, size_t len)
2299 {
2300         struct cxl_root_decoder *cxlrd = to_cxl_root_decoder(dev);
2301         struct cxl_port *port = to_cxl_port(dev->parent);
2302         struct cxl_region *cxlr;
2303
2304         cxlr = cxl_find_region_by_name(cxlrd, buf);
2305         if (IS_ERR(cxlr))
2306                 return PTR_ERR(cxlr);
2307
2308         devm_release_action(port->uport_dev, unregister_region, cxlr);
2309         put_device(&cxlr->dev);
2310
2311         return len;
2312 }
2313 DEVICE_ATTR_WO(delete_region);
2314
2315 static void cxl_pmem_region_release(struct device *dev)
2316 {
2317         struct cxl_pmem_region *cxlr_pmem = to_cxl_pmem_region(dev);
2318         int i;
2319
2320         for (i = 0; i < cxlr_pmem->nr_mappings; i++) {
2321                 struct cxl_memdev *cxlmd = cxlr_pmem->mapping[i].cxlmd;
2322
2323                 put_device(&cxlmd->dev);
2324         }
2325
2326         kfree(cxlr_pmem);
2327 }
2328
2329 static const struct attribute_group *cxl_pmem_region_attribute_groups[] = {
2330         &cxl_base_attribute_group,
2331         NULL,
2332 };
2333
2334 const struct device_type cxl_pmem_region_type = {
2335         .name = "cxl_pmem_region",
2336         .release = cxl_pmem_region_release,
2337         .groups = cxl_pmem_region_attribute_groups,
2338 };
2339
2340 bool is_cxl_pmem_region(struct device *dev)
2341 {
2342         return dev->type == &cxl_pmem_region_type;
2343 }
2344 EXPORT_SYMBOL_NS_GPL(is_cxl_pmem_region, CXL);
2345
2346 struct cxl_pmem_region *to_cxl_pmem_region(struct device *dev)
2347 {
2348         if (dev_WARN_ONCE(dev, !is_cxl_pmem_region(dev),
2349                           "not a cxl_pmem_region device\n"))
2350                 return NULL;
2351         return container_of(dev, struct cxl_pmem_region, dev);
2352 }
2353 EXPORT_SYMBOL_NS_GPL(to_cxl_pmem_region, CXL);
2354
2355 struct cxl_poison_context {
2356         struct cxl_port *port;
2357         enum cxl_decoder_mode mode;
2358         u64 offset;
2359 };
2360
2361 static int cxl_get_poison_unmapped(struct cxl_memdev *cxlmd,
2362                                    struct cxl_poison_context *ctx)
2363 {
2364         struct cxl_dev_state *cxlds = cxlmd->cxlds;
2365         u64 offset, length;
2366         int rc = 0;
2367
2368         /*
2369          * Collect poison for the remaining unmapped resources
2370          * after poison is collected by committed endpoints.
2371          *
2372          * Knowing that PMEM must always follow RAM, get poison
2373          * for unmapped resources based on the last decoder's mode:
2374          *      ram: scan remains of ram range, then any pmem range
2375          *      pmem: scan remains of pmem range
2376          */
2377
2378         if (ctx->mode == CXL_DECODER_RAM) {
2379                 offset = ctx->offset;
2380                 length = resource_size(&cxlds->ram_res) - offset;
2381                 rc = cxl_mem_get_poison(cxlmd, offset, length, NULL);
2382                 if (rc == -EFAULT)
2383                         rc = 0;
2384                 if (rc)
2385                         return rc;
2386         }
2387         if (ctx->mode == CXL_DECODER_PMEM) {
2388                 offset = ctx->offset;
2389                 length = resource_size(&cxlds->dpa_res) - offset;
2390                 if (!length)
2391                         return 0;
2392         } else if (resource_size(&cxlds->pmem_res)) {
2393                 offset = cxlds->pmem_res.start;
2394                 length = resource_size(&cxlds->pmem_res);
2395         } else {
2396                 return 0;
2397         }
2398
2399         return cxl_mem_get_poison(cxlmd, offset, length, NULL);
2400 }
2401
2402 static int poison_by_decoder(struct device *dev, void *arg)
2403 {
2404         struct cxl_poison_context *ctx = arg;
2405         struct cxl_endpoint_decoder *cxled;
2406         struct cxl_memdev *cxlmd;
2407         u64 offset, length;
2408         int rc = 0;
2409
2410         if (!is_endpoint_decoder(dev))
2411                 return rc;
2412
2413         cxled = to_cxl_endpoint_decoder(dev);
2414         if (!cxled->dpa_res || !resource_size(cxled->dpa_res))
2415                 return rc;
2416
2417         /*
2418          * Regions are only created with single mode decoders: pmem or ram.
2419          * Linux does not support mixed mode decoders. This means that
2420          * reading poison per endpoint decoder adheres to the requirement
2421          * that poison reads of pmem and ram must be separated.
2422          * CXL 3.0 Spec 8.2.9.8.4.1
2423          */
2424         if (cxled->mode == CXL_DECODER_MIXED) {
2425                 dev_dbg(dev, "poison list read unsupported in mixed mode\n");
2426                 return rc;
2427         }
2428
2429         cxlmd = cxled_to_memdev(cxled);
2430         if (cxled->skip) {
2431                 offset = cxled->dpa_res->start - cxled->skip;
2432                 length = cxled->skip;
2433                 rc = cxl_mem_get_poison(cxlmd, offset, length, NULL);
2434                 if (rc == -EFAULT && cxled->mode == CXL_DECODER_RAM)
2435                         rc = 0;
2436                 if (rc)
2437                         return rc;
2438         }
2439
2440         offset = cxled->dpa_res->start;
2441         length = cxled->dpa_res->end - offset + 1;
2442         rc = cxl_mem_get_poison(cxlmd, offset, length, cxled->cxld.region);
2443         if (rc == -EFAULT && cxled->mode == CXL_DECODER_RAM)
2444                 rc = 0;
2445         if (rc)
2446                 return rc;
2447
2448         /* Iterate until commit_end is reached */
2449         if (cxled->cxld.id == ctx->port->commit_end) {
2450                 ctx->offset = cxled->dpa_res->end + 1;
2451                 ctx->mode = cxled->mode;
2452                 return 1;
2453         }
2454
2455         return 0;
2456 }
2457
2458 int cxl_get_poison_by_endpoint(struct cxl_port *port)
2459 {
2460         struct cxl_poison_context ctx;
2461         int rc = 0;
2462
2463         rc = down_read_interruptible(&cxl_region_rwsem);
2464         if (rc)
2465                 return rc;
2466
2467         ctx = (struct cxl_poison_context) {
2468                 .port = port
2469         };
2470
2471         rc = device_for_each_child(&port->dev, &ctx, poison_by_decoder);
2472         if (rc == 1)
2473                 rc = cxl_get_poison_unmapped(to_cxl_memdev(port->uport_dev),
2474                                              &ctx);
2475
2476         up_read(&cxl_region_rwsem);
2477         return rc;
2478 }
2479
2480 static struct lock_class_key cxl_pmem_region_key;
2481
2482 static struct cxl_pmem_region *cxl_pmem_region_alloc(struct cxl_region *cxlr)
2483 {
2484         struct cxl_region_params *p = &cxlr->params;
2485         struct cxl_nvdimm_bridge *cxl_nvb;
2486         struct cxl_pmem_region *cxlr_pmem;
2487         struct device *dev;
2488         int i;
2489
2490         down_read(&cxl_region_rwsem);
2491         if (p->state != CXL_CONFIG_COMMIT) {
2492                 cxlr_pmem = ERR_PTR(-ENXIO);
2493                 goto out;
2494         }
2495
2496         cxlr_pmem = kzalloc(struct_size(cxlr_pmem, mapping, p->nr_targets),
2497                             GFP_KERNEL);
2498         if (!cxlr_pmem) {
2499                 cxlr_pmem = ERR_PTR(-ENOMEM);
2500                 goto out;
2501         }
2502
2503         cxlr_pmem->hpa_range.start = p->res->start;
2504         cxlr_pmem->hpa_range.end = p->res->end;
2505
2506         /* Snapshot the region configuration underneath the cxl_region_rwsem */
2507         cxlr_pmem->nr_mappings = p->nr_targets;
2508         for (i = 0; i < p->nr_targets; i++) {
2509                 struct cxl_endpoint_decoder *cxled = p->targets[i];
2510                 struct cxl_memdev *cxlmd = cxled_to_memdev(cxled);
2511                 struct cxl_pmem_region_mapping *m = &cxlr_pmem->mapping[i];
2512
2513                 /*
2514                  * Regions never span CXL root devices, so by definition the
2515                  * bridge for one device is the same for all.
2516                  */
2517                 if (i == 0) {
2518                         cxl_nvb = cxl_find_nvdimm_bridge(cxlmd);
2519                         if (!cxl_nvb) {
2520                                 cxlr_pmem = ERR_PTR(-ENODEV);
2521                                 goto out;
2522                         }
2523                         cxlr->cxl_nvb = cxl_nvb;
2524                 }
2525                 m->cxlmd = cxlmd;
2526                 get_device(&cxlmd->dev);
2527                 m->start = cxled->dpa_res->start;
2528                 m->size = resource_size(cxled->dpa_res);
2529                 m->position = i;
2530         }
2531
2532         dev = &cxlr_pmem->dev;
2533         cxlr_pmem->cxlr = cxlr;
2534         cxlr->cxlr_pmem = cxlr_pmem;
2535         device_initialize(dev);
2536         lockdep_set_class(&dev->mutex, &cxl_pmem_region_key);
2537         device_set_pm_not_required(dev);
2538         dev->parent = &cxlr->dev;
2539         dev->bus = &cxl_bus_type;
2540         dev->type = &cxl_pmem_region_type;
2541 out:
2542         up_read(&cxl_region_rwsem);
2543
2544         return cxlr_pmem;
2545 }
2546
2547 static void cxl_dax_region_release(struct device *dev)
2548 {
2549         struct cxl_dax_region *cxlr_dax = to_cxl_dax_region(dev);
2550
2551         kfree(cxlr_dax);
2552 }
2553
2554 static const struct attribute_group *cxl_dax_region_attribute_groups[] = {
2555         &cxl_base_attribute_group,
2556         NULL,
2557 };
2558
2559 const struct device_type cxl_dax_region_type = {
2560         .name = "cxl_dax_region",
2561         .release = cxl_dax_region_release,
2562         .groups = cxl_dax_region_attribute_groups,
2563 };
2564
2565 static bool is_cxl_dax_region(struct device *dev)
2566 {
2567         return dev->type == &cxl_dax_region_type;
2568 }
2569
2570 struct cxl_dax_region *to_cxl_dax_region(struct device *dev)
2571 {
2572         if (dev_WARN_ONCE(dev, !is_cxl_dax_region(dev),
2573                           "not a cxl_dax_region device\n"))
2574                 return NULL;
2575         return container_of(dev, struct cxl_dax_region, dev);
2576 }
2577 EXPORT_SYMBOL_NS_GPL(to_cxl_dax_region, CXL);
2578
2579 static struct lock_class_key cxl_dax_region_key;
2580
2581 static struct cxl_dax_region *cxl_dax_region_alloc(struct cxl_region *cxlr)
2582 {
2583         struct cxl_region_params *p = &cxlr->params;
2584         struct cxl_dax_region *cxlr_dax;
2585         struct device *dev;
2586
2587         down_read(&cxl_region_rwsem);
2588         if (p->state != CXL_CONFIG_COMMIT) {
2589                 cxlr_dax = ERR_PTR(-ENXIO);
2590                 goto out;
2591         }
2592
2593         cxlr_dax = kzalloc(sizeof(*cxlr_dax), GFP_KERNEL);
2594         if (!cxlr_dax) {
2595                 cxlr_dax = ERR_PTR(-ENOMEM);
2596                 goto out;
2597         }
2598
2599         cxlr_dax->hpa_range.start = p->res->start;
2600         cxlr_dax->hpa_range.end = p->res->end;
2601
2602         dev = &cxlr_dax->dev;
2603         cxlr_dax->cxlr = cxlr;
2604         device_initialize(dev);
2605         lockdep_set_class(&dev->mutex, &cxl_dax_region_key);
2606         device_set_pm_not_required(dev);
2607         dev->parent = &cxlr->dev;
2608         dev->bus = &cxl_bus_type;
2609         dev->type = &cxl_dax_region_type;
2610 out:
2611         up_read(&cxl_region_rwsem);
2612
2613         return cxlr_dax;
2614 }
2615
2616 static void cxlr_pmem_unregister(void *_cxlr_pmem)
2617 {
2618         struct cxl_pmem_region *cxlr_pmem = _cxlr_pmem;
2619         struct cxl_region *cxlr = cxlr_pmem->cxlr;
2620         struct cxl_nvdimm_bridge *cxl_nvb = cxlr->cxl_nvb;
2621
2622         /*
2623          * Either the bridge is in ->remove() context under the device_lock(),
2624          * or cxlr_release_nvdimm() is cancelling the bridge's release action
2625          * for @cxlr_pmem and doing it itself (while manually holding the bridge
2626          * lock).
2627          */
2628         device_lock_assert(&cxl_nvb->dev);
2629         cxlr->cxlr_pmem = NULL;
2630         cxlr_pmem->cxlr = NULL;
2631         device_unregister(&cxlr_pmem->dev);
2632 }
2633
2634 static void cxlr_release_nvdimm(void *_cxlr)
2635 {
2636         struct cxl_region *cxlr = _cxlr;
2637         struct cxl_nvdimm_bridge *cxl_nvb = cxlr->cxl_nvb;
2638
2639         device_lock(&cxl_nvb->dev);
2640         if (cxlr->cxlr_pmem)
2641                 devm_release_action(&cxl_nvb->dev, cxlr_pmem_unregister,
2642                                     cxlr->cxlr_pmem);
2643         device_unlock(&cxl_nvb->dev);
2644         cxlr->cxl_nvb = NULL;
2645         put_device(&cxl_nvb->dev);
2646 }
2647
2648 /**
2649  * devm_cxl_add_pmem_region() - add a cxl_region-to-nd_region bridge
2650  * @cxlr: parent CXL region for this pmem region bridge device
2651  *
2652  * Return: 0 on success negative error code on failure.
2653  */
2654 static int devm_cxl_add_pmem_region(struct cxl_region *cxlr)
2655 {
2656         struct cxl_pmem_region *cxlr_pmem;
2657         struct cxl_nvdimm_bridge *cxl_nvb;
2658         struct device *dev;
2659         int rc;
2660
2661         cxlr_pmem = cxl_pmem_region_alloc(cxlr);
2662         if (IS_ERR(cxlr_pmem))
2663                 return PTR_ERR(cxlr_pmem);
2664         cxl_nvb = cxlr->cxl_nvb;
2665
2666         dev = &cxlr_pmem->dev;
2667         rc = dev_set_name(dev, "pmem_region%d", cxlr->id);
2668         if (rc)
2669                 goto err;
2670
2671         rc = device_add(dev);
2672         if (rc)
2673                 goto err;
2674
2675         dev_dbg(&cxlr->dev, "%s: register %s\n", dev_name(dev->parent),
2676                 dev_name(dev));
2677
2678         device_lock(&cxl_nvb->dev);
2679         if (cxl_nvb->dev.driver)
2680                 rc = devm_add_action_or_reset(&cxl_nvb->dev,
2681                                               cxlr_pmem_unregister, cxlr_pmem);
2682         else
2683                 rc = -ENXIO;
2684         device_unlock(&cxl_nvb->dev);
2685
2686         if (rc)
2687                 goto err_bridge;
2688
2689         /* @cxlr carries a reference on @cxl_nvb until cxlr_release_nvdimm */
2690         return devm_add_action_or_reset(&cxlr->dev, cxlr_release_nvdimm, cxlr);
2691
2692 err:
2693         put_device(dev);
2694 err_bridge:
2695         put_device(&cxl_nvb->dev);
2696         cxlr->cxl_nvb = NULL;
2697         return rc;
2698 }
2699
2700 static void cxlr_dax_unregister(void *_cxlr_dax)
2701 {
2702         struct cxl_dax_region *cxlr_dax = _cxlr_dax;
2703
2704         device_unregister(&cxlr_dax->dev);
2705 }
2706
2707 static int devm_cxl_add_dax_region(struct cxl_region *cxlr)
2708 {
2709         struct cxl_dax_region *cxlr_dax;
2710         struct device *dev;
2711         int rc;
2712
2713         cxlr_dax = cxl_dax_region_alloc(cxlr);
2714         if (IS_ERR(cxlr_dax))
2715                 return PTR_ERR(cxlr_dax);
2716
2717         dev = &cxlr_dax->dev;
2718         rc = dev_set_name(dev, "dax_region%d", cxlr->id);
2719         if (rc)
2720                 goto err;
2721
2722         rc = device_add(dev);
2723         if (rc)
2724                 goto err;
2725
2726         dev_dbg(&cxlr->dev, "%s: register %s\n", dev_name(dev->parent),
2727                 dev_name(dev));
2728
2729         return devm_add_action_or_reset(&cxlr->dev, cxlr_dax_unregister,
2730                                         cxlr_dax);
2731 err:
2732         put_device(dev);
2733         return rc;
2734 }
2735
2736 static int match_root_decoder_by_range(struct device *dev, void *data)
2737 {
2738         struct range *r1, *r2 = data;
2739         struct cxl_root_decoder *cxlrd;
2740
2741         if (!is_root_decoder(dev))
2742                 return 0;
2743
2744         cxlrd = to_cxl_root_decoder(dev);
2745         r1 = &cxlrd->cxlsd.cxld.hpa_range;
2746         return range_contains(r1, r2);
2747 }
2748
2749 static int match_region_by_range(struct device *dev, void *data)
2750 {
2751         struct cxl_region_params *p;
2752         struct cxl_region *cxlr;
2753         struct range *r = data;
2754         int rc = 0;
2755
2756         if (!is_cxl_region(dev))
2757                 return 0;
2758
2759         cxlr = to_cxl_region(dev);
2760         p = &cxlr->params;
2761
2762         down_read(&cxl_region_rwsem);
2763         if (p->res && p->res->start == r->start && p->res->end == r->end)
2764                 rc = 1;
2765         up_read(&cxl_region_rwsem);
2766
2767         return rc;
2768 }
2769
2770 /* Establish an empty region covering the given HPA range */
2771 static struct cxl_region *construct_region(struct cxl_root_decoder *cxlrd,
2772                                            struct cxl_endpoint_decoder *cxled)
2773 {
2774         struct cxl_memdev *cxlmd = cxled_to_memdev(cxled);
2775         struct cxl_port *port = cxlrd_to_port(cxlrd);
2776         struct range *hpa = &cxled->cxld.hpa_range;
2777         struct cxl_region_params *p;
2778         struct cxl_region *cxlr;
2779         struct resource *res;
2780         int rc;
2781
2782         do {
2783                 cxlr = __create_region(cxlrd, cxled->mode,
2784                                        atomic_read(&cxlrd->region_id));
2785         } while (IS_ERR(cxlr) && PTR_ERR(cxlr) == -EBUSY);
2786
2787         if (IS_ERR(cxlr)) {
2788                 dev_err(cxlmd->dev.parent,
2789                         "%s:%s: %s failed assign region: %ld\n",
2790                         dev_name(&cxlmd->dev), dev_name(&cxled->cxld.dev),
2791                         __func__, PTR_ERR(cxlr));
2792                 return cxlr;
2793         }
2794
2795         down_write(&cxl_region_rwsem);
2796         p = &cxlr->params;
2797         if (p->state >= CXL_CONFIG_INTERLEAVE_ACTIVE) {
2798                 dev_err(cxlmd->dev.parent,
2799                         "%s:%s: %s autodiscovery interrupted\n",
2800                         dev_name(&cxlmd->dev), dev_name(&cxled->cxld.dev),
2801                         __func__);
2802                 rc = -EBUSY;
2803                 goto err;
2804         }
2805
2806         set_bit(CXL_REGION_F_AUTO, &cxlr->flags);
2807
2808         res = kmalloc(sizeof(*res), GFP_KERNEL);
2809         if (!res) {
2810                 rc = -ENOMEM;
2811                 goto err;
2812         }
2813
2814         *res = DEFINE_RES_MEM_NAMED(hpa->start, range_len(hpa),
2815                                     dev_name(&cxlr->dev));
2816         rc = insert_resource(cxlrd->res, res);
2817         if (rc) {
2818                 /*
2819                  * Platform-firmware may not have split resources like "System
2820                  * RAM" on CXL window boundaries see cxl_region_iomem_release()
2821                  */
2822                 dev_warn(cxlmd->dev.parent,
2823                          "%s:%s: %s %s cannot insert resource\n",
2824                          dev_name(&cxlmd->dev), dev_name(&cxled->cxld.dev),
2825                          __func__, dev_name(&cxlr->dev));
2826         }
2827
2828         p->res = res;
2829         p->interleave_ways = cxled->cxld.interleave_ways;
2830         p->interleave_granularity = cxled->cxld.interleave_granularity;
2831         p->state = CXL_CONFIG_INTERLEAVE_ACTIVE;
2832
2833         rc = sysfs_update_group(&cxlr->dev.kobj, get_cxl_region_target_group());
2834         if (rc)
2835                 goto err;
2836
2837         dev_dbg(cxlmd->dev.parent, "%s:%s: %s %s res: %pr iw: %d ig: %d\n",
2838                 dev_name(&cxlmd->dev), dev_name(&cxled->cxld.dev), __func__,
2839                 dev_name(&cxlr->dev), p->res, p->interleave_ways,
2840                 p->interleave_granularity);
2841
2842         /* ...to match put_device() in cxl_add_to_region() */
2843         get_device(&cxlr->dev);
2844         up_write(&cxl_region_rwsem);
2845
2846         return cxlr;
2847
2848 err:
2849         up_write(&cxl_region_rwsem);
2850         devm_release_action(port->uport_dev, unregister_region, cxlr);
2851         return ERR_PTR(rc);
2852 }
2853
2854 int cxl_add_to_region(struct cxl_port *root, struct cxl_endpoint_decoder *cxled)
2855 {
2856         struct cxl_memdev *cxlmd = cxled_to_memdev(cxled);
2857         struct range *hpa = &cxled->cxld.hpa_range;
2858         struct cxl_decoder *cxld = &cxled->cxld;
2859         struct device *cxlrd_dev, *region_dev;
2860         struct cxl_root_decoder *cxlrd;
2861         struct cxl_region_params *p;
2862         struct cxl_region *cxlr;
2863         bool attach = false;
2864         int rc;
2865
2866         cxlrd_dev = device_find_child(&root->dev, &cxld->hpa_range,
2867                                       match_root_decoder_by_range);
2868         if (!cxlrd_dev) {
2869                 dev_err(cxlmd->dev.parent,
2870                         "%s:%s no CXL window for range %#llx:%#llx\n",
2871                         dev_name(&cxlmd->dev), dev_name(&cxld->dev),
2872                         cxld->hpa_range.start, cxld->hpa_range.end);
2873                 return -ENXIO;
2874         }
2875
2876         cxlrd = to_cxl_root_decoder(cxlrd_dev);
2877
2878         /*
2879          * Ensure that if multiple threads race to construct_region() for @hpa
2880          * one does the construction and the others add to that.
2881          */
2882         mutex_lock(&cxlrd->range_lock);
2883         region_dev = device_find_child(&cxlrd->cxlsd.cxld.dev, hpa,
2884                                        match_region_by_range);
2885         if (!region_dev) {
2886                 cxlr = construct_region(cxlrd, cxled);
2887                 region_dev = &cxlr->dev;
2888         } else
2889                 cxlr = to_cxl_region(region_dev);
2890         mutex_unlock(&cxlrd->range_lock);
2891
2892         rc = PTR_ERR_OR_ZERO(cxlr);
2893         if (rc)
2894                 goto out;
2895
2896         attach_target(cxlr, cxled, -1, TASK_UNINTERRUPTIBLE);
2897
2898         down_read(&cxl_region_rwsem);
2899         p = &cxlr->params;
2900         attach = p->state == CXL_CONFIG_COMMIT;
2901         up_read(&cxl_region_rwsem);
2902
2903         if (attach) {
2904                 /*
2905                  * If device_attach() fails the range may still be active via
2906                  * the platform-firmware memory map, otherwise the driver for
2907                  * regions is local to this file, so driver matching can't fail.
2908                  */
2909                 if (device_attach(&cxlr->dev) < 0)
2910                         dev_err(&cxlr->dev, "failed to enable, range: %pr\n",
2911                                 p->res);
2912         }
2913
2914         put_device(region_dev);
2915 out:
2916         put_device(cxlrd_dev);
2917         return rc;
2918 }
2919 EXPORT_SYMBOL_NS_GPL(cxl_add_to_region, CXL);
2920
2921 static int is_system_ram(struct resource *res, void *arg)
2922 {
2923         struct cxl_region *cxlr = arg;
2924         struct cxl_region_params *p = &cxlr->params;
2925
2926         dev_dbg(&cxlr->dev, "%pr has System RAM: %pr\n", p->res, res);
2927         return 1;
2928 }
2929
2930 static int cxl_region_probe(struct device *dev)
2931 {
2932         struct cxl_region *cxlr = to_cxl_region(dev);
2933         struct cxl_region_params *p = &cxlr->params;
2934         int rc;
2935
2936         rc = down_read_interruptible(&cxl_region_rwsem);
2937         if (rc) {
2938                 dev_dbg(&cxlr->dev, "probe interrupted\n");
2939                 return rc;
2940         }
2941
2942         if (p->state < CXL_CONFIG_COMMIT) {
2943                 dev_dbg(&cxlr->dev, "config state: %d\n", p->state);
2944                 rc = -ENXIO;
2945                 goto out;
2946         }
2947
2948         if (test_bit(CXL_REGION_F_NEEDS_RESET, &cxlr->flags)) {
2949                 dev_err(&cxlr->dev,
2950                         "failed to activate, re-commit region and retry\n");
2951                 rc = -ENXIO;
2952                 goto out;
2953         }
2954
2955         /*
2956          * From this point on any path that changes the region's state away from
2957          * CXL_CONFIG_COMMIT is also responsible for releasing the driver.
2958          */
2959 out:
2960         up_read(&cxl_region_rwsem);
2961
2962         if (rc)
2963                 return rc;
2964
2965         switch (cxlr->mode) {
2966         case CXL_DECODER_PMEM:
2967                 return devm_cxl_add_pmem_region(cxlr);
2968         case CXL_DECODER_RAM:
2969                 /*
2970                  * The region can not be manged by CXL if any portion of
2971                  * it is already online as 'System RAM'
2972                  */
2973                 if (walk_iomem_res_desc(IORES_DESC_NONE,
2974                                         IORESOURCE_SYSTEM_RAM | IORESOURCE_BUSY,
2975                                         p->res->start, p->res->end, cxlr,
2976                                         is_system_ram) > 0)
2977                         return 0;
2978                 return devm_cxl_add_dax_region(cxlr);
2979         default:
2980                 dev_dbg(&cxlr->dev, "unsupported region mode: %d\n",
2981                         cxlr->mode);
2982                 return -ENXIO;
2983         }
2984 }
2985
2986 static struct cxl_driver cxl_region_driver = {
2987         .name = "cxl_region",
2988         .probe = cxl_region_probe,
2989         .id = CXL_DEVICE_REGION,
2990 };
2991
2992 int cxl_region_init(void)
2993 {
2994         return cxl_driver_register(&cxl_region_driver);
2995 }
2996
2997 void cxl_region_exit(void)
2998 {
2999         cxl_driver_unregister(&cxl_region_driver);
3000 }
3001
3002 MODULE_IMPORT_NS(CXL);
3003 MODULE_IMPORT_NS(DEVMEM);
3004 MODULE_ALIAS_CXL(CXL_DEVICE_REGION);