drivers/dax/bus.c

   1 // SPDX-License-Identifier: GPL-2.0
   2 /* Copyright(c) 2017-2018 Intel Corporation. All rights reserved. */
   3 #include <linux/memremap.h>
   4 #include <linux/device.h>
   5 #include <linux/mutex.h>
   6 #include <linux/list.h>
   7 #include <linux/slab.h>
   8 #include <linux/dax.h>
   9 #include "dax-private.h"
  10 #include "bus.h"
  11
  12 static struct class *dax_class;
  13
  14 static DEFINE_MUTEX(dax_bus_lock);
  15
  16 #define DAX_NAME_LEN 30
  17 struct dax_id {
  18         struct list_head list;
  19         char dev_name[DAX_NAME_LEN];
  20 };
  21
  22 static int dax_bus_uevent(struct device *dev, struct kobj_uevent_env *env)
  23 {
  24         /*
  25          * We only ever expect to handle device-dax instances, i.e. the
  26          * @type argument to MODULE_ALIAS_DAX_DEVICE() is always zero
  27          */
  28         return add_uevent_var(env, "MODALIAS=" DAX_DEVICE_MODALIAS_FMT, 0);
  29 }
  30
  31 static struct dax_device_driver *to_dax_drv(struct device_driver *drv)
  32 {
  33         return container_of(drv, struct dax_device_driver, drv);
  34 }
  35
  36 static struct dax_id *__dax_match_id(struct dax_device_driver *dax_drv,
  37                 const char *dev_name)
  38 {
  39         struct dax_id *dax_id;
  40
  41         lockdep_assert_held(&dax_bus_lock);
  42
  43         list_for_each_entry(dax_id, &dax_drv->ids, list)
  44                 if (sysfs_streq(dax_id->dev_name, dev_name))
  45                         return dax_id;
  46         return NULL;
  47 }
  48
  49 static int dax_match_id(struct dax_device_driver *dax_drv, struct device *dev)
  50 {
  51         int match;
  52
  53         mutex_lock(&dax_bus_lock);
  54         match = !!__dax_match_id(dax_drv, dev_name(dev));
  55         mutex_unlock(&dax_bus_lock);
  56
  57         return match;
  58 }
  59
  60 enum id_action {
  61         ID_REMOVE,
  62         ID_ADD,
  63 };
  64
  65 static ssize_t do_id_store(struct device_driver *drv, const char *buf,
  66                 size_t count, enum id_action action)
  67 {
  68         struct dax_device_driver *dax_drv = to_dax_drv(drv);
  69         unsigned int region_id, id;
  70         char devname[DAX_NAME_LEN];
  71         struct dax_id *dax_id;
  72         ssize_t rc = count;
  73         int fields;
  74
  75         fields = sscanf(buf, "dax%d.%d", &region_id, &id);
  76         if (fields != 2)
  77                 return -EINVAL;
  78         sprintf(devname, "dax%d.%d", region_id, id);
  79         if (!sysfs_streq(buf, devname))
  80                 return -EINVAL;
  81
  82         mutex_lock(&dax_bus_lock);
  83         dax_id = __dax_match_id(dax_drv, buf);
  84         if (!dax_id) {
  85                 if (action == ID_ADD) {
  86                         dax_id = kzalloc(sizeof(*dax_id), GFP_KERNEL);
  87                         if (dax_id) {
  88                                 strncpy(dax_id->dev_name, buf, DAX_NAME_LEN);
  89                                 list_add(&dax_id->list, &dax_drv->ids);
  90                         } else
  91                                 rc = -ENOMEM;
  92                 } else
  93                         /* nothing to remove */;
  94         } else if (action == ID_REMOVE) {
  95                 list_del(&dax_id->list);
  96                 kfree(dax_id);
  97         } else
  98                 /* dax_id already added */;
  99         mutex_unlock(&dax_bus_lock);
 100
 101         if (rc < 0)
 102                 return rc;
 103         if (action == ID_ADD)
 104                 rc = driver_attach(drv);
 105         if (rc)
 106                 return rc;
 107         return count;
 108 }
 109
 110 static ssize_t new_id_store(struct device_driver *drv, const char *buf,
 111                 size_t count)
 112 {
 113         return do_id_store(drv, buf, count, ID_ADD);
 114 }
 115 static DRIVER_ATTR_WO(new_id);
 116
 117 static ssize_t remove_id_store(struct device_driver *drv, const char *buf,
 118                 size_t count)
 119 {
 120         return do_id_store(drv, buf, count, ID_REMOVE);
 121 }
 122 static DRIVER_ATTR_WO(remove_id);
 123
 124 static struct attribute *dax_drv_attrs[] = {
 125         &driver_attr_new_id.attr,
 126         &driver_attr_remove_id.attr,
 127         NULL,
 128 };
 129 ATTRIBUTE_GROUPS(dax_drv);
 130
 131 static int dax_bus_match(struct device *dev, struct device_driver *drv);
 132
 133 static bool is_static(struct dax_region *dax_region)
 134 {
 135         return (dax_region->res.flags & IORESOURCE_DAX_STATIC) != 0;
 136 }
 137
 138 static int dax_bus_probe(struct device *dev)
 139 {
 140         struct dax_device_driver *dax_drv = to_dax_drv(dev->driver);
 141         struct dev_dax *dev_dax = to_dev_dax(dev);
 142         struct dax_region *dax_region = dev_dax->region;
 143         struct range *range = &dev_dax->range;
 144         int rc;
 145
 146         if (range_len(range) == 0 || dev_dax->id < 0)
 147                 return -ENXIO;
 148
 149         rc = dax_drv->probe(dev_dax);
 150
 151         if (rc || is_static(dax_region))
 152                 return rc;
 153
 154         /*
 155          * Track new seed creation only after successful probe of the
 156          * previous seed.
 157          */
 158         if (dax_region->seed == dev)
 159                 dax_region->seed = NULL;
 160
 161         return 0;
 162 }
 163
 164 static int dax_bus_remove(struct device *dev)
 165 {
 166         struct dax_device_driver *dax_drv = to_dax_drv(dev->driver);
 167         struct dev_dax *dev_dax = to_dev_dax(dev);
 168
 169         return dax_drv->remove(dev_dax);
 170 }
 171
 172 static struct bus_type dax_bus_type = {
 173         .name = "dax",
 174         .uevent = dax_bus_uevent,
 175         .match = dax_bus_match,
 176         .probe = dax_bus_probe,
 177         .remove = dax_bus_remove,
 178         .drv_groups = dax_drv_groups,
 179 };
 180
 181 static int dax_bus_match(struct device *dev, struct device_driver *drv)
 182 {
 183         struct dax_device_driver *dax_drv = to_dax_drv(drv);
 184
 185         /*
 186          * All but the 'device-dax' driver, which has 'match_always'
 187          * set, requires an exact id match.
 188          */
 189         if (dax_drv->match_always)
 190                 return 1;
 191
 192         return dax_match_id(dax_drv, dev);
 193 }
 194
 195 /*
 196  * Rely on the fact that drvdata is set before the attributes are
 197  * registered, and that the attributes are unregistered before drvdata
 198  * is cleared to assume that drvdata is always valid.
 199  */
 200 static ssize_t id_show(struct device *dev,
 201                 struct device_attribute *attr, char *buf)
 202 {
 203         struct dax_region *dax_region = dev_get_drvdata(dev);
 204
 205         return sprintf(buf, "%d\n", dax_region->id);
 206 }
 207 static DEVICE_ATTR_RO(id);
 208
 209 static ssize_t region_size_show(struct device *dev,
 210                 struct device_attribute *attr, char *buf)
 211 {
 212         struct dax_region *dax_region = dev_get_drvdata(dev);
 213
 214         return sprintf(buf, "%llu\n", (unsigned long long)
 215                         resource_size(&dax_region->res));
 216 }
 217 static struct device_attribute dev_attr_region_size = __ATTR(size, 0444,
 218                 region_size_show, NULL);
 219
 220 static ssize_t align_show(struct device *dev,
 221                 struct device_attribute *attr, char *buf)
 222 {
 223         struct dax_region *dax_region = dev_get_drvdata(dev);
 224
 225         return sprintf(buf, "%u\n", dax_region->align);
 226 }
 227 static DEVICE_ATTR_RO(align);
 228
 229 #define for_each_dax_region_resource(dax_region, res) \
 230         for (res = (dax_region)->res.child; res; res = res->sibling)
 231
 232 static unsigned long long dax_region_avail_size(struct dax_region *dax_region)
 233 {
 234         resource_size_t size = resource_size(&dax_region->res);
 235         struct resource *res;
 236
 237         device_lock_assert(dax_region->dev);
 238
 239         for_each_dax_region_resource(dax_region, res)
 240                 size -= resource_size(res);
 241         return size;
 242 }
 243
 244 static ssize_t available_size_show(struct device *dev,
 245                 struct device_attribute *attr, char *buf)
 246 {
 247         struct dax_region *dax_region = dev_get_drvdata(dev);
 248         unsigned long long size;
 249
 250         device_lock(dev);
 251         size = dax_region_avail_size(dax_region);
 252         device_unlock(dev);
 253
 254         return sprintf(buf, "%llu\n", size);
 255 }
 256 static DEVICE_ATTR_RO(available_size);
 257
 258 static ssize_t seed_show(struct device *dev,
 259                 struct device_attribute *attr, char *buf)
 260 {
 261         struct dax_region *dax_region = dev_get_drvdata(dev);
 262         struct device *seed;
 263         ssize_t rc;
 264
 265         if (is_static(dax_region))
 266                 return -EINVAL;
 267
 268         device_lock(dev);
 269         seed = dax_region->seed;
 270         rc = sprintf(buf, "%s\n", seed ? dev_name(seed) : "");
 271         device_unlock(dev);
 272
 273         return rc;
 274 }
 275 static DEVICE_ATTR_RO(seed);
 276
 277 static ssize_t create_show(struct device *dev,
 278                 struct device_attribute *attr, char *buf)
 279 {
 280         struct dax_region *dax_region = dev_get_drvdata(dev);
 281         struct device *youngest;
 282         ssize_t rc;
 283
 284         if (is_static(dax_region))
 285                 return -EINVAL;
 286
 287         device_lock(dev);
 288         youngest = dax_region->youngest;
 289         rc = sprintf(buf, "%s\n", youngest ? dev_name(youngest) : "");
 290         device_unlock(dev);
 291
 292         return rc;
 293 }
 294
 295 static ssize_t create_store(struct device *dev, struct device_attribute *attr,
 296                 const char *buf, size_t len)
 297 {
 298         struct dax_region *dax_region = dev_get_drvdata(dev);
 299         unsigned long long avail;
 300         ssize_t rc;
 301         int val;
 302
 303         if (is_static(dax_region))
 304                 return -EINVAL;
 305
 306         rc = kstrtoint(buf, 0, &val);
 307         if (rc)
 308                 return rc;
 309         if (val != 1)
 310                 return -EINVAL;
 311
 312         device_lock(dev);
 313         avail = dax_region_avail_size(dax_region);
 314         if (avail == 0)
 315                 rc = -ENOSPC;
 316         else {
 317                 struct dev_dax_data data = {
 318                         .dax_region = dax_region,
 319                         .size = 0,
 320                         .id = -1,
 321                 };
 322                 struct dev_dax *dev_dax = devm_create_dev_dax(&data);
 323
 324                 if (IS_ERR(dev_dax))
 325                         rc = PTR_ERR(dev_dax);
 326                 else {
 327                         /*
 328                          * In support of crafting multiple new devices
 329                          * simultaneously multiple seeds can be created,
 330                          * but only the first one that has not been
 331                          * successfully bound is tracked as the region
 332                          * seed.
 333                          */
 334                         if (!dax_region->seed)
 335                                 dax_region->seed = &dev_dax->dev;
 336                         dax_region->youngest = &dev_dax->dev;
 337                         rc = len;
 338                 }
 339         }
 340         device_unlock(dev);
 341
 342         return rc;
 343 }
 344 static DEVICE_ATTR_RW(create);
 345
 346 void kill_dev_dax(struct dev_dax *dev_dax)
 347 {
 348         struct dax_device *dax_dev = dev_dax->dax_dev;
 349         struct inode *inode = dax_inode(dax_dev);
 350
 351         kill_dax(dax_dev);
 352         unmap_mapping_range(inode->i_mapping, 0, 0, 1);
 353 }
 354 EXPORT_SYMBOL_GPL(kill_dev_dax);
 355
 356 static void free_dev_dax_range(struct dev_dax *dev_dax)
 357 {
 358         struct dax_region *dax_region = dev_dax->region;
 359         struct range *range = &dev_dax->range;
 360
 361         device_lock_assert(dax_region->dev);
 362         if (range_len(range))
 363                 __release_region(&dax_region->res, range->start,
 364                                 range_len(range));
 365 }
 366
 367 static void unregister_dev_dax(void *dev)
 368 {
 369         struct dev_dax *dev_dax = to_dev_dax(dev);
 370
 371         dev_dbg(dev, "%s\n", __func__);
 372
 373         kill_dev_dax(dev_dax);
 374         free_dev_dax_range(dev_dax);
 375         device_del(dev);
 376         put_device(dev);
 377 }
 378
 379 /* a return value >= 0 indicates this invocation invalidated the id */
 380 static int __free_dev_dax_id(struct dev_dax *dev_dax)
 381 {
 382         struct dax_region *dax_region = dev_dax->region;
 383         struct device *dev = &dev_dax->dev;
 384         int rc = dev_dax->id;
 385
 386         device_lock_assert(dev);
 387
 388         if (is_static(dax_region) || dev_dax->id < 0)
 389                 return -1;
 390         ida_free(&dax_region->ida, dev_dax->id);
 391         dev_dax->id = -1;
 392         return rc;
 393 }
 394
 395 static int free_dev_dax_id(struct dev_dax *dev_dax)
 396 {
 397         struct device *dev = &dev_dax->dev;
 398         int rc;
 399
 400         device_lock(dev);
 401         rc = __free_dev_dax_id(dev_dax);
 402         device_unlock(dev);
 403         return rc;
 404 }
 405
 406 static ssize_t delete_store(struct device *dev, struct device_attribute *attr,
 407                 const char *buf, size_t len)
 408 {
 409         struct dax_region *dax_region = dev_get_drvdata(dev);
 410         struct dev_dax *dev_dax;
 411         struct device *victim;
 412         bool do_del = false;
 413         int rc;
 414
 415         if (is_static(dax_region))
 416                 return -EINVAL;
 417
 418         victim = device_find_child_by_name(dax_region->dev, buf);
 419         if (!victim)
 420                 return -ENXIO;
 421
 422         device_lock(dev);
 423         device_lock(victim);
 424         dev_dax = to_dev_dax(victim);
 425         if (victim->driver || range_len(&dev_dax->range))
 426                 rc = -EBUSY;
 427         else {
 428                 /*
 429                  * Invalidate the device so it does not become active
 430                  * again, but always preserve device-id-0 so that
 431                  * /sys/bus/dax/ is guaranteed to be populated while any
 432                  * dax_region is registered.
 433                  */
 434                 if (dev_dax->id > 0) {
 435                         do_del = __free_dev_dax_id(dev_dax) >= 0;
 436                         rc = len;
 437                         if (dax_region->seed == victim)
 438                                 dax_region->seed = NULL;
 439                         if (dax_region->youngest == victim)
 440                                 dax_region->youngest = NULL;
 441                 } else
 442                         rc = -EBUSY;
 443         }
 444         device_unlock(victim);
 445
 446         /* won the race to invalidate the device, clean it up */
 447         if (do_del)
 448                 devm_release_action(dev, unregister_dev_dax, victim);
 449         device_unlock(dev);
 450         put_device(victim);
 451
 452         return rc;
 453 }
 454 static DEVICE_ATTR_WO(delete);
 455
 456 static umode_t dax_region_visible(struct kobject *kobj, struct attribute *a,
 457                 int n)
 458 {
 459         struct device *dev = container_of(kobj, struct device, kobj);
 460         struct dax_region *dax_region = dev_get_drvdata(dev);
 461
 462         if (is_static(dax_region))
 463                 if (a == &dev_attr_available_size.attr
 464                                 || a == &dev_attr_create.attr
 465                                 || a == &dev_attr_seed.attr
 466                                 || a == &dev_attr_delete.attr)
 467                         return 0;
 468         return a->mode;
 469 }
 470
 471 static struct attribute *dax_region_attributes[] = {
 472         &dev_attr_available_size.attr,
 473         &dev_attr_region_size.attr,
 474         &dev_attr_align.attr,
 475         &dev_attr_create.attr,
 476         &dev_attr_seed.attr,
 477         &dev_attr_delete.attr,
 478         &dev_attr_id.attr,
 479         NULL,
 480 };
 481
 482 static const struct attribute_group dax_region_attribute_group = {
 483         .name = "dax_region",
 484         .attrs = dax_region_attributes,
 485         .is_visible = dax_region_visible,
 486 };
 487
 488 static const struct attribute_group *dax_region_attribute_groups[] = {
 489         &dax_region_attribute_group,
 490         NULL,
 491 };
 492
 493 static void dax_region_free(struct kref *kref)
 494 {
 495         struct dax_region *dax_region;
 496
 497         dax_region = container_of(kref, struct dax_region, kref);
 498         kfree(dax_region);
 499 }
 500
 501 void dax_region_put(struct dax_region *dax_region)
 502 {
 503         kref_put(&dax_region->kref, dax_region_free);
 504 }
 505 EXPORT_SYMBOL_GPL(dax_region_put);
 506
 507 static void dax_region_unregister(void *region)
 508 {
 509         struct dax_region *dax_region = region;
 510
 511         sysfs_remove_groups(&dax_region->dev->kobj,
 512                         dax_region_attribute_groups);
 513         dax_region_put(dax_region);
 514 }
 515
 516 struct dax_region *alloc_dax_region(struct device *parent, int region_id,
 517                 struct resource *res, int target_node, unsigned int align,
 518                 unsigned long flags)
 519 {
 520         struct dax_region *dax_region;
 521
 522         /*
 523          * The DAX core assumes that it can store its private data in
 524          * parent->driver_data. This WARN is a reminder / safeguard for
 525          * developers of device-dax drivers.
 526          */
 527         if (dev_get_drvdata(parent)) {
 528                 dev_WARN(parent, "dax core failed to setup private data\n");
 529                 return NULL;
 530         }
 531
 532         if (!IS_ALIGNED(res->start, align)
 533                         || !IS_ALIGNED(resource_size(res), align))
 534                 return NULL;
 535
 536         dax_region = kzalloc(sizeof(*dax_region), GFP_KERNEL);
 537         if (!dax_region)
 538                 return NULL;
 539
 540         dev_set_drvdata(parent, dax_region);
 541         kref_init(&dax_region->kref);
 542         dax_region->id = region_id;
 543         dax_region->align = align;
 544         dax_region->dev = parent;
 545         dax_region->target_node = target_node;
 546         ida_init(&dax_region->ida);
 547         dax_region->res = (struct resource) {
 548                 .start = res->start,
 549                 .end = res->end,
 550                 .flags = IORESOURCE_MEM | flags,
 551         };
 552
 553         if (sysfs_create_groups(&parent->kobj, dax_region_attribute_groups)) {
 554                 kfree(dax_region);
 555                 return NULL;
 556         }
 557
 558         kref_get(&dax_region->kref);
 559         if (devm_add_action_or_reset(parent, dax_region_unregister, dax_region))
 560                 return NULL;
 561         return dax_region;
 562 }
 563 EXPORT_SYMBOL_GPL(alloc_dax_region);
 564
 565 static int alloc_dev_dax_range(struct dev_dax *dev_dax, resource_size_t size)
 566 {
 567         struct dax_region *dax_region = dev_dax->region;
 568         struct resource *res = &dax_region->res;
 569         struct device *dev = &dev_dax->dev;
 570         struct resource *alloc;
 571
 572         device_lock_assert(dax_region->dev);
 573
 574         /* handle the seed alloc special case */
 575         if (!size) {
 576                 dev_dax->range = (struct range) {
 577                         .start = res->start,
 578                         .end = res->start - 1,
 579                 };
 580                 return 0;
 581         }
 582
 583         /* TODO: handle multiple allocations per region */
 584         if (res->child)
 585                 return -ENOMEM;
 586
 587         alloc = __request_region(res, res->start, size, dev_name(dev), 0);
 588
 589         if (!alloc)
 590                 return -ENOMEM;
 591
 592         dev_dax->range = (struct range) {
 593                 .start = alloc->start,
 594                 .end = alloc->end,
 595         };
 596
 597         return 0;
 598 }
 599
 600 static ssize_t size_show(struct device *dev,
 601                 struct device_attribute *attr, char *buf)
 602 {
 603         struct dev_dax *dev_dax = to_dev_dax(dev);
 604         unsigned long long size = range_len(&dev_dax->range);
 605
 606         return sprintf(buf, "%llu\n", size);
 607 }
 608 static DEVICE_ATTR_RO(size);
 609
 610 static int dev_dax_target_node(struct dev_dax *dev_dax)
 611 {
 612         struct dax_region *dax_region = dev_dax->region;
 613
 614         return dax_region->target_node;
 615 }
 616
 617 static ssize_t target_node_show(struct device *dev,
 618                 struct device_attribute *attr, char *buf)
 619 {
 620         struct dev_dax *dev_dax = to_dev_dax(dev);
 621
 622         return sprintf(buf, "%d\n", dev_dax_target_node(dev_dax));
 623 }
 624 static DEVICE_ATTR_RO(target_node);
 625
 626 static ssize_t resource_show(struct device *dev,
 627                 struct device_attribute *attr, char *buf)
 628 {
 629         struct dev_dax *dev_dax = to_dev_dax(dev);
 630
 631         return sprintf(buf, "%#llx\n", dev_dax->range.start);
 632 }
 633 static DEVICE_ATTR(resource, 0400, resource_show, NULL);
 634
 635 static ssize_t modalias_show(struct device *dev, struct device_attribute *attr,
 636                 char *buf)
 637 {
 638         /*
 639          * We only ever expect to handle device-dax instances, i.e. the
 640          * @type argument to MODULE_ALIAS_DAX_DEVICE() is always zero
 641          */
 642         return sprintf(buf, DAX_DEVICE_MODALIAS_FMT "\n", 0);
 643 }
 644 static DEVICE_ATTR_RO(modalias);
 645
 646 static ssize_t numa_node_show(struct device *dev,
 647                 struct device_attribute *attr, char *buf)
 648 {
 649         return sprintf(buf, "%d\n", dev_to_node(dev));
 650 }
 651 static DEVICE_ATTR_RO(numa_node);
 652
 653 static umode_t dev_dax_visible(struct kobject *kobj, struct attribute *a, int n)
 654 {
 655         struct device *dev = container_of(kobj, struct device, kobj);
 656         struct dev_dax *dev_dax = to_dev_dax(dev);
 657
 658         if (a == &dev_attr_target_node.attr && dev_dax_target_node(dev_dax) < 0)
 659                 return 0;
 660         if (a == &dev_attr_numa_node.attr && !IS_ENABLED(CONFIG_NUMA))
 661                 return 0;
 662         return a->mode;
 663 }
 664
 665 static struct attribute *dev_dax_attributes[] = {
 666         &dev_attr_modalias.attr,
 667         &dev_attr_size.attr,
 668         &dev_attr_target_node.attr,
 669         &dev_attr_resource.attr,
 670         &dev_attr_numa_node.attr,
 671         NULL,
 672 };
 673
 674 static const struct attribute_group dev_dax_attribute_group = {
 675         .attrs = dev_dax_attributes,
 676         .is_visible = dev_dax_visible,
 677 };
 678
 679 static const struct attribute_group *dax_attribute_groups[] = {
 680         &dev_dax_attribute_group,
 681         NULL,
 682 };
 683
 684 static void dev_dax_release(struct device *dev)
 685 {
 686         struct dev_dax *dev_dax = to_dev_dax(dev);
 687         struct dax_region *dax_region = dev_dax->region;
 688         struct dax_device *dax_dev = dev_dax->dax_dev;
 689
 690         put_dax(dax_dev);
 691         free_dev_dax_id(dev_dax);
 692         dax_region_put(dax_region);
 693         kfree(dev_dax->pgmap);
 694         kfree(dev_dax);
 695 }
 696
 697 static const struct device_type dev_dax_type = {
 698         .release = dev_dax_release,
 699         .groups = dax_attribute_groups,
 700 };
 701
 702 struct dev_dax *devm_create_dev_dax(struct dev_dax_data *data)
 703 {
 704         struct dax_region *dax_region = data->dax_region;
 705         struct device *parent = dax_region->dev;
 706         struct dax_device *dax_dev;
 707         struct dev_dax *dev_dax;
 708         struct inode *inode;
 709         struct device *dev;
 710         int rc;
 711
 712         dev_dax = kzalloc(sizeof(*dev_dax), GFP_KERNEL);
 713         if (!dev_dax)
 714                 return ERR_PTR(-ENOMEM);
 715
 716         if (is_static(dax_region)) {
 717                 if (dev_WARN_ONCE(parent, data->id < 0,
 718                                 "dynamic id specified to static region\n")) {
 719                         rc = -EINVAL;
 720                         goto err_id;
 721                 }
 722
 723                 dev_dax->id = data->id;
 724         } else {
 725                 if (dev_WARN_ONCE(parent, data->id >= 0,
 726                                 "static id specified to dynamic region\n")) {
 727                         rc = -EINVAL;
 728                         goto err_id;
 729                 }
 730
 731                 rc = ida_alloc(&dax_region->ida, GFP_KERNEL);
 732                 if (rc < 0)
 733                         goto err_id;
 734                 dev_dax->id = rc;
 735         }
 736
 737         dev_dax->region = dax_region;
 738         dev = &dev_dax->dev;
 739         device_initialize(dev);
 740         dev_set_name(dev, "dax%d.%d", dax_region->id, dev_dax->id);
 741
 742         rc = alloc_dev_dax_range(dev_dax, data->size);
 743         if (rc)
 744                 goto err_range;
 745
 746         if (data->pgmap) {
 747                 dev_WARN_ONCE(parent, !is_static(dax_region),
 748                         "custom dev_pagemap requires a static dax_region\n");
 749
 750                 dev_dax->pgmap = kmemdup(data->pgmap,
 751                                 sizeof(struct dev_pagemap), GFP_KERNEL);
 752                 if (!dev_dax->pgmap) {
 753                         rc = -ENOMEM;
 754                         goto err_pgmap;
 755                 }
 756         }
 757
 758         /*
 759          * No 'host' or dax_operations since there is no access to this
 760          * device outside of mmap of the resulting character device.
 761          */
 762         dax_dev = alloc_dax(dev_dax, NULL, NULL, DAXDEV_F_SYNC);
 763         if (IS_ERR(dax_dev)) {
 764                 rc = PTR_ERR(dax_dev);
 765                 goto err_alloc_dax;
 766         }
 767
 768         /* a device_dax instance is dead while the driver is not attached */
 769         kill_dax(dax_dev);
 770
 771         /* from here on we're committed to teardown via dev_dax_release() */
 772         dev_dax->dax_dev = dax_dev;
 773         dev_dax->target_node = dax_region->target_node;
 774         kref_get(&dax_region->kref);
 775
 776         inode = dax_inode(dax_dev);
 777         dev->devt = inode->i_rdev;
 778         if (data->subsys == DEV_DAX_BUS)
 779                 dev->bus = &dax_bus_type;
 780         else
 781                 dev->class = dax_class;
 782         dev->parent = parent;
 783         dev->type = &dev_dax_type;
 784
 785         rc = device_add(dev);
 786         if (rc) {
 787                 kill_dev_dax(dev_dax);
 788                 put_device(dev);
 789                 return ERR_PTR(rc);
 790         }
 791
 792         rc = devm_add_action_or_reset(dax_region->dev, unregister_dev_dax, dev);
 793         if (rc)
 794                 return ERR_PTR(rc);
 795
 796         return dev_dax;
 797
 798 err_alloc_dax:
 799         kfree(dev_dax->pgmap);
 800 err_pgmap:
 801         free_dev_dax_range(dev_dax);
 802 err_range:
 803         free_dev_dax_id(dev_dax);
 804 err_id:
 805         kfree(dev_dax);
 806
 807         return ERR_PTR(rc);
 808 }
 809 EXPORT_SYMBOL_GPL(devm_create_dev_dax);
 810
 811 static int match_always_count;
 812
 813 int __dax_driver_register(struct dax_device_driver *dax_drv,
 814                 struct module *module, const char *mod_name)
 815 {
 816         struct device_driver *drv = &dax_drv->drv;
 817         int rc = 0;
 818
 819         INIT_LIST_HEAD(&dax_drv->ids);
 820         drv->owner = module;
 821         drv->name = mod_name;
 822         drv->mod_name = mod_name;
 823         drv->bus = &dax_bus_type;
 824
 825         /* there can only be one default driver */
 826         mutex_lock(&dax_bus_lock);
 827         match_always_count += dax_drv->match_always;
 828         if (match_always_count > 1) {
 829                 match_always_count--;
 830                 WARN_ON(1);
 831                 rc = -EINVAL;
 832         }
 833         mutex_unlock(&dax_bus_lock);
 834         if (rc)
 835                 return rc;
 836         return driver_register(drv);
 837 }
 838 EXPORT_SYMBOL_GPL(__dax_driver_register);
 839
 840 void dax_driver_unregister(struct dax_device_driver *dax_drv)
 841 {
 842         struct device_driver *drv = &dax_drv->drv;
 843         struct dax_id *dax_id, *_id;
 844
 845         mutex_lock(&dax_bus_lock);
 846         match_always_count -= dax_drv->match_always;
 847         list_for_each_entry_safe(dax_id, _id, &dax_drv->ids, list) {
 848                 list_del(&dax_id->list);
 849                 kfree(dax_id);
 850         }
 851         mutex_unlock(&dax_bus_lock);
 852         driver_unregister(drv);
 853 }
 854 EXPORT_SYMBOL_GPL(dax_driver_unregister);
 855
 856 int __init dax_bus_init(void)
 857 {
 858         int rc;
 859
 860         if (IS_ENABLED(CONFIG_DEV_DAX_PMEM_COMPAT)) {
 861                 dax_class = class_create(THIS_MODULE, "dax");
 862                 if (IS_ERR(dax_class))
 863                         return PTR_ERR(dax_class);
 864         }
 865
 866         rc = bus_register(&dax_bus_type);
 867         if (rc)
 868                 class_destroy(dax_class);
 869         return rc;
 870 }
 871
 872 void __exit dax_bus_exit(void)
 873 {
 874         bus_unregister(&dax_bus_type);
 875         class_destroy(dax_class);
 876 }