drivers/dax/bus.c

   1 // SPDX-License-Identifier: GPL-2.0
   2 /* Copyright(c) 2017-2018 Intel Corporation. All rights reserved. */
   3 #include <linux/memremap.h>
   4 #include <linux/device.h>
   5 #include <linux/mutex.h>
   6 #include <linux/list.h>
   7 #include <linux/slab.h>
   8 #include <linux/dax.h>
   9 #include <linux/io.h>
  10 #include "dax-private.h"
  11 #include "bus.h"
  12
  13 static struct class *dax_class;
  14
  15 static DEFINE_MUTEX(dax_bus_lock);
  16
  17 #define DAX_NAME_LEN 30
  18 struct dax_id {
  19         struct list_head list;
  20         char dev_name[DAX_NAME_LEN];
  21 };
  22
  23 static int dax_bus_uevent(struct device *dev, struct kobj_uevent_env *env)
  24 {
  25         /*
  26          * We only ever expect to handle device-dax instances, i.e. the
  27          * @type argument to MODULE_ALIAS_DAX_DEVICE() is always zero
  28          */
  29         return add_uevent_var(env, "MODALIAS=" DAX_DEVICE_MODALIAS_FMT, 0);
  30 }
  31
  32 static struct dax_device_driver *to_dax_drv(struct device_driver *drv)
  33 {
  34         return container_of(drv, struct dax_device_driver, drv);
  35 }
  36
  37 static struct dax_id *__dax_match_id(struct dax_device_driver *dax_drv,
  38                 const char *dev_name)
  39 {
  40         struct dax_id *dax_id;
  41
  42         lockdep_assert_held(&dax_bus_lock);
  43
  44         list_for_each_entry(dax_id, &dax_drv->ids, list)
  45                 if (sysfs_streq(dax_id->dev_name, dev_name))
  46                         return dax_id;
  47         return NULL;
  48 }
  49
  50 static int dax_match_id(struct dax_device_driver *dax_drv, struct device *dev)
  51 {
  52         int match;
  53
  54         mutex_lock(&dax_bus_lock);
  55         match = !!__dax_match_id(dax_drv, dev_name(dev));
  56         mutex_unlock(&dax_bus_lock);
  57
  58         return match;
  59 }
  60
  61 enum id_action {
  62         ID_REMOVE,
  63         ID_ADD,
  64 };
  65
  66 static ssize_t do_id_store(struct device_driver *drv, const char *buf,
  67                 size_t count, enum id_action action)
  68 {
  69         struct dax_device_driver *dax_drv = to_dax_drv(drv);
  70         unsigned int region_id, id;
  71         char devname[DAX_NAME_LEN];
  72         struct dax_id *dax_id;
  73         ssize_t rc = count;
  74         int fields;
  75
  76         fields = sscanf(buf, "dax%d.%d", &region_id, &id);
  77         if (fields != 2)
  78                 return -EINVAL;
  79         sprintf(devname, "dax%d.%d", region_id, id);
  80         if (!sysfs_streq(buf, devname))
  81                 return -EINVAL;
  82
  83         mutex_lock(&dax_bus_lock);
  84         dax_id = __dax_match_id(dax_drv, buf);
  85         if (!dax_id) {
  86                 if (action == ID_ADD) {
  87                         dax_id = kzalloc(sizeof(*dax_id), GFP_KERNEL);
  88                         if (dax_id) {
  89                                 strncpy(dax_id->dev_name, buf, DAX_NAME_LEN);
  90                                 list_add(&dax_id->list, &dax_drv->ids);
  91                         } else
  92                                 rc = -ENOMEM;
  93                 }
  94         } else if (action == ID_REMOVE) {
  95                 list_del(&dax_id->list);
  96                 kfree(dax_id);
  97         }
  98         mutex_unlock(&dax_bus_lock);
  99
 100         if (rc < 0)
 101                 return rc;
 102         if (action == ID_ADD)
 103                 rc = driver_attach(drv);
 104         if (rc)
 105                 return rc;
 106         return count;
 107 }
 108
 109 static ssize_t new_id_store(struct device_driver *drv, const char *buf,
 110                 size_t count)
 111 {
 112         return do_id_store(drv, buf, count, ID_ADD);
 113 }
 114 static DRIVER_ATTR_WO(new_id);
 115
 116 static ssize_t remove_id_store(struct device_driver *drv, const char *buf,
 117                 size_t count)
 118 {
 119         return do_id_store(drv, buf, count, ID_REMOVE);
 120 }
 121 static DRIVER_ATTR_WO(remove_id);
 122
 123 static struct attribute *dax_drv_attrs[] = {
 124         &driver_attr_new_id.attr,
 125         &driver_attr_remove_id.attr,
 126         NULL,
 127 };
 128 ATTRIBUTE_GROUPS(dax_drv);
 129
 130 static int dax_bus_match(struct device *dev, struct device_driver *drv);
 131
 132 static bool is_static(struct dax_region *dax_region)
 133 {
 134         return (dax_region->res.flags & IORESOURCE_DAX_STATIC) != 0;
 135 }
 136
 137 static u64 dev_dax_size(struct dev_dax *dev_dax)
 138 {
 139         u64 size = 0;
 140         int i;
 141
 142         device_lock_assert(&dev_dax->dev);
 143
 144         for (i = 0; i < dev_dax->nr_range; i++)
 145                 size += range_len(&dev_dax->ranges[i].range);
 146
 147         return size;
 148 }
 149
 150 static int dax_bus_probe(struct device *dev)
 151 {
 152         struct dax_device_driver *dax_drv = to_dax_drv(dev->driver);
 153         struct dev_dax *dev_dax = to_dev_dax(dev);
 154         struct dax_region *dax_region = dev_dax->region;
 155         int rc;
 156
 157         if (dev_dax_size(dev_dax) == 0 || dev_dax->id < 0)
 158                 return -ENXIO;
 159
 160         rc = dax_drv->probe(dev_dax);
 161
 162         if (rc || is_static(dax_region))
 163                 return rc;
 164
 165         /*
 166          * Track new seed creation only after successful probe of the
 167          * previous seed.
 168          */
 169         if (dax_region->seed == dev)
 170                 dax_region->seed = NULL;
 171
 172         return 0;
 173 }
 174
 175 static void dax_bus_remove(struct device *dev)
 176 {
 177         struct dax_device_driver *dax_drv = to_dax_drv(dev->driver);
 178         struct dev_dax *dev_dax = to_dev_dax(dev);
 179
 180         if (dax_drv->remove)
 181                 dax_drv->remove(dev_dax);
 182 }
 183
 184 static struct bus_type dax_bus_type = {
 185         .name = "dax",
 186         .uevent = dax_bus_uevent,
 187         .match = dax_bus_match,
 188         .probe = dax_bus_probe,
 189         .remove = dax_bus_remove,
 190         .drv_groups = dax_drv_groups,
 191 };
 192
 193 static int dax_bus_match(struct device *dev, struct device_driver *drv)
 194 {
 195         struct dax_device_driver *dax_drv = to_dax_drv(drv);
 196
 197         /*
 198          * All but the 'device-dax' driver, which has 'match_always'
 199          * set, requires an exact id match.
 200          */
 201         if (dax_drv->match_always)
 202                 return 1;
 203
 204         return dax_match_id(dax_drv, dev);
 205 }
 206
 207 /*
 208  * Rely on the fact that drvdata is set before the attributes are
 209  * registered, and that the attributes are unregistered before drvdata
 210  * is cleared to assume that drvdata is always valid.
 211  */
 212 static ssize_t id_show(struct device *dev,
 213                 struct device_attribute *attr, char *buf)
 214 {
 215         struct dax_region *dax_region = dev_get_drvdata(dev);
 216
 217         return sprintf(buf, "%d\n", dax_region->id);
 218 }
 219 static DEVICE_ATTR_RO(id);
 220
 221 static ssize_t region_size_show(struct device *dev,
 222                 struct device_attribute *attr, char *buf)
 223 {
 224         struct dax_region *dax_region = dev_get_drvdata(dev);
 225
 226         return sprintf(buf, "%llu\n", (unsigned long long)
 227                         resource_size(&dax_region->res));
 228 }
 229 static struct device_attribute dev_attr_region_size = __ATTR(size, 0444,
 230                 region_size_show, NULL);
 231
 232 static ssize_t region_align_show(struct device *dev,
 233                 struct device_attribute *attr, char *buf)
 234 {
 235         struct dax_region *dax_region = dev_get_drvdata(dev);
 236
 237         return sprintf(buf, "%u\n", dax_region->align);
 238 }
 239 static struct device_attribute dev_attr_region_align =
 240                 __ATTR(align, 0400, region_align_show, NULL);
 241
 242 #define for_each_dax_region_resource(dax_region, res) \
 243         for (res = (dax_region)->res.child; res; res = res->sibling)
 244
 245 static unsigned long long dax_region_avail_size(struct dax_region *dax_region)
 246 {
 247         resource_size_t size = resource_size(&dax_region->res);
 248         struct resource *res;
 249
 250         device_lock_assert(dax_region->dev);
 251
 252         for_each_dax_region_resource(dax_region, res)
 253                 size -= resource_size(res);
 254         return size;
 255 }
 256
 257 static ssize_t available_size_show(struct device *dev,
 258                 struct device_attribute *attr, char *buf)
 259 {
 260         struct dax_region *dax_region = dev_get_drvdata(dev);
 261         unsigned long long size;
 262
 263         device_lock(dev);
 264         size = dax_region_avail_size(dax_region);
 265         device_unlock(dev);
 266
 267         return sprintf(buf, "%llu\n", size);
 268 }
 269 static DEVICE_ATTR_RO(available_size);
 270
 271 static ssize_t seed_show(struct device *dev,
 272                 struct device_attribute *attr, char *buf)
 273 {
 274         struct dax_region *dax_region = dev_get_drvdata(dev);
 275         struct device *seed;
 276         ssize_t rc;
 277
 278         if (is_static(dax_region))
 279                 return -EINVAL;
 280
 281         device_lock(dev);
 282         seed = dax_region->seed;
 283         rc = sprintf(buf, "%s\n", seed ? dev_name(seed) : "");
 284         device_unlock(dev);
 285
 286         return rc;
 287 }
 288 static DEVICE_ATTR_RO(seed);
 289
 290 static ssize_t create_show(struct device *dev,
 291                 struct device_attribute *attr, char *buf)
 292 {
 293         struct dax_region *dax_region = dev_get_drvdata(dev);
 294         struct device *youngest;
 295         ssize_t rc;
 296
 297         if (is_static(dax_region))
 298                 return -EINVAL;
 299
 300         device_lock(dev);
 301         youngest = dax_region->youngest;
 302         rc = sprintf(buf, "%s\n", youngest ? dev_name(youngest) : "");
 303         device_unlock(dev);
 304
 305         return rc;
 306 }
 307
 308 static ssize_t create_store(struct device *dev, struct device_attribute *attr,
 309                 const char *buf, size_t len)
 310 {
 311         struct dax_region *dax_region = dev_get_drvdata(dev);
 312         unsigned long long avail;
 313         ssize_t rc;
 314         int val;
 315
 316         if (is_static(dax_region))
 317                 return -EINVAL;
 318
 319         rc = kstrtoint(buf, 0, &val);
 320         if (rc)
 321                 return rc;
 322         if (val != 1)
 323                 return -EINVAL;
 324
 325         device_lock(dev);
 326         avail = dax_region_avail_size(dax_region);
 327         if (avail == 0)
 328                 rc = -ENOSPC;
 329         else {
 330                 struct dev_dax_data data = {
 331                         .dax_region = dax_region,
 332                         .size = 0,
 333                         .id = -1,
 334                 };
 335                 struct dev_dax *dev_dax = devm_create_dev_dax(&data);
 336
 337                 if (IS_ERR(dev_dax))
 338                         rc = PTR_ERR(dev_dax);
 339                 else {
 340                         /*
 341                          * In support of crafting multiple new devices
 342                          * simultaneously multiple seeds can be created,
 343                          * but only the first one that has not been
 344                          * successfully bound is tracked as the region
 345                          * seed.
 346                          */
 347                         if (!dax_region->seed)
 348                                 dax_region->seed = &dev_dax->dev;
 349                         dax_region->youngest = &dev_dax->dev;
 350                         rc = len;
 351                 }
 352         }
 353         device_unlock(dev);
 354
 355         return rc;
 356 }
 357 static DEVICE_ATTR_RW(create);
 358
 359 void kill_dev_dax(struct dev_dax *dev_dax)
 360 {
 361         struct dax_device *dax_dev = dev_dax->dax_dev;
 362         struct inode *inode = dax_inode(dax_dev);
 363
 364         kill_dax(dax_dev);
 365         unmap_mapping_range(inode->i_mapping, 0, 0, 1);
 366 }
 367 EXPORT_SYMBOL_GPL(kill_dev_dax);
 368
 369 static void trim_dev_dax_range(struct dev_dax *dev_dax)
 370 {
 371         int i = dev_dax->nr_range - 1;
 372         struct range *range = &dev_dax->ranges[i].range;
 373         struct dax_region *dax_region = dev_dax->region;
 374
 375         device_lock_assert(dax_region->dev);
 376         dev_dbg(&dev_dax->dev, "delete range[%d]: %#llx:%#llx\n", i,
 377                 (unsigned long long)range->start,
 378                 (unsigned long long)range->end);
 379
 380         __release_region(&dax_region->res, range->start, range_len(range));
 381         if (--dev_dax->nr_range == 0) {
 382                 kfree(dev_dax->ranges);
 383                 dev_dax->ranges = NULL;
 384         }
 385 }
 386
 387 static void free_dev_dax_ranges(struct dev_dax *dev_dax)
 388 {
 389         while (dev_dax->nr_range)
 390                 trim_dev_dax_range(dev_dax);
 391 }
 392
 393 static void unregister_dev_dax(void *dev)
 394 {
 395         struct dev_dax *dev_dax = to_dev_dax(dev);
 396
 397         dev_dbg(dev, "%s\n", __func__);
 398
 399         kill_dev_dax(dev_dax);
 400         free_dev_dax_ranges(dev_dax);
 401         device_del(dev);
 402         put_device(dev);
 403 }
 404
 405 /* a return value >= 0 indicates this invocation invalidated the id */
 406 static int __free_dev_dax_id(struct dev_dax *dev_dax)
 407 {
 408         struct dax_region *dax_region = dev_dax->region;
 409         struct device *dev = &dev_dax->dev;
 410         int rc = dev_dax->id;
 411
 412         device_lock_assert(dev);
 413
 414         if (is_static(dax_region) || dev_dax->id < 0)
 415                 return -1;
 416         ida_free(&dax_region->ida, dev_dax->id);
 417         dev_dax->id = -1;
 418         return rc;
 419 }
 420
 421 static int free_dev_dax_id(struct dev_dax *dev_dax)
 422 {
 423         struct device *dev = &dev_dax->dev;
 424         int rc;
 425
 426         device_lock(dev);
 427         rc = __free_dev_dax_id(dev_dax);
 428         device_unlock(dev);
 429         return rc;
 430 }
 431
 432 static ssize_t delete_store(struct device *dev, struct device_attribute *attr,
 433                 const char *buf, size_t len)
 434 {
 435         struct dax_region *dax_region = dev_get_drvdata(dev);
 436         struct dev_dax *dev_dax;
 437         struct device *victim;
 438         bool do_del = false;
 439         int rc;
 440
 441         if (is_static(dax_region))
 442                 return -EINVAL;
 443
 444         victim = device_find_child_by_name(dax_region->dev, buf);
 445         if (!victim)
 446                 return -ENXIO;
 447
 448         device_lock(dev);
 449         device_lock(victim);
 450         dev_dax = to_dev_dax(victim);
 451         if (victim->driver || dev_dax_size(dev_dax))
 452                 rc = -EBUSY;
 453         else {
 454                 /*
 455                  * Invalidate the device so it does not become active
 456                  * again, but always preserve device-id-0 so that
 457                  * /sys/bus/dax/ is guaranteed to be populated while any
 458                  * dax_region is registered.
 459                  */
 460                 if (dev_dax->id > 0) {
 461                         do_del = __free_dev_dax_id(dev_dax) >= 0;
 462                         rc = len;
 463                         if (dax_region->seed == victim)
 464                                 dax_region->seed = NULL;
 465                         if (dax_region->youngest == victim)
 466                                 dax_region->youngest = NULL;
 467                 } else
 468                         rc = -EBUSY;
 469         }
 470         device_unlock(victim);
 471
 472         /* won the race to invalidate the device, clean it up */
 473         if (do_del)
 474                 devm_release_action(dev, unregister_dev_dax, victim);
 475         device_unlock(dev);
 476         put_device(victim);
 477
 478         return rc;
 479 }
 480 static DEVICE_ATTR_WO(delete);
 481
 482 static umode_t dax_region_visible(struct kobject *kobj, struct attribute *a,
 483                 int n)
 484 {
 485         struct device *dev = container_of(kobj, struct device, kobj);
 486         struct dax_region *dax_region = dev_get_drvdata(dev);
 487
 488         if (is_static(dax_region))
 489                 if (a == &dev_attr_available_size.attr
 490                                 || a == &dev_attr_create.attr
 491                                 || a == &dev_attr_seed.attr
 492                                 || a == &dev_attr_delete.attr)
 493                         return 0;
 494         return a->mode;
 495 }
 496
 497 static struct attribute *dax_region_attributes[] = {
 498         &dev_attr_available_size.attr,
 499         &dev_attr_region_size.attr,
 500         &dev_attr_region_align.attr,
 501         &dev_attr_create.attr,
 502         &dev_attr_seed.attr,
 503         &dev_attr_delete.attr,
 504         &dev_attr_id.attr,
 505         NULL,
 506 };
 507
 508 static const struct attribute_group dax_region_attribute_group = {
 509         .name = "dax_region",
 510         .attrs = dax_region_attributes,
 511         .is_visible = dax_region_visible,
 512 };
 513
 514 static const struct attribute_group *dax_region_attribute_groups[] = {
 515         &dax_region_attribute_group,
 516         NULL,
 517 };
 518
 519 static void dax_region_free(struct kref *kref)
 520 {
 521         struct dax_region *dax_region;
 522
 523         dax_region = container_of(kref, struct dax_region, kref);
 524         kfree(dax_region);
 525 }
 526
 527 void dax_region_put(struct dax_region *dax_region)
 528 {
 529         kref_put(&dax_region->kref, dax_region_free);
 530 }
 531 EXPORT_SYMBOL_GPL(dax_region_put);
 532
 533 static void dax_region_unregister(void *region)
 534 {
 535         struct dax_region *dax_region = region;
 536
 537         sysfs_remove_groups(&dax_region->dev->kobj,
 538                         dax_region_attribute_groups);
 539         dax_region_put(dax_region);
 540 }
 541
 542 struct dax_region *alloc_dax_region(struct device *parent, int region_id,
 543                 struct range *range, int target_node, unsigned int align,
 544                 unsigned long flags)
 545 {
 546         struct dax_region *dax_region;
 547
 548         /*
 549          * The DAX core assumes that it can store its private data in
 550          * parent->driver_data. This WARN is a reminder / safeguard for
 551          * developers of device-dax drivers.
 552          */
 553         if (dev_get_drvdata(parent)) {
 554                 dev_WARN(parent, "dax core failed to setup private data\n");
 555                 return NULL;
 556         }
 557
 558         if (!IS_ALIGNED(range->start, align)
 559                         || !IS_ALIGNED(range_len(range), align))
 560                 return NULL;
 561
 562         dax_region = kzalloc(sizeof(*dax_region), GFP_KERNEL);
 563         if (!dax_region)
 564                 return NULL;
 565
 566         dev_set_drvdata(parent, dax_region);
 567         kref_init(&dax_region->kref);
 568         dax_region->id = region_id;
 569         dax_region->align = align;
 570         dax_region->dev = parent;
 571         dax_region->target_node = target_node;
 572         ida_init(&dax_region->ida);
 573         dax_region->res = (struct resource) {
 574                 .start = range->start,
 575                 .end = range->end,
 576                 .flags = IORESOURCE_MEM | flags,
 577         };
 578
 579         if (sysfs_create_groups(&parent->kobj, dax_region_attribute_groups)) {
 580                 kfree(dax_region);
 581                 return NULL;
 582         }
 583
 584         kref_get(&dax_region->kref);
 585         if (devm_add_action_or_reset(parent, dax_region_unregister, dax_region))
 586                 return NULL;
 587         return dax_region;
 588 }
 589 EXPORT_SYMBOL_GPL(alloc_dax_region);
 590
 591 static void dax_mapping_release(struct device *dev)
 592 {
 593         struct dax_mapping *mapping = to_dax_mapping(dev);
 594         struct dev_dax *dev_dax = to_dev_dax(dev->parent);
 595
 596         ida_free(&dev_dax->ida, mapping->id);
 597         kfree(mapping);
 598 }
 599
 600 static void unregister_dax_mapping(void *data)
 601 {
 602         struct device *dev = data;
 603         struct dax_mapping *mapping = to_dax_mapping(dev);
 604         struct dev_dax *dev_dax = to_dev_dax(dev->parent);
 605         struct dax_region *dax_region = dev_dax->region;
 606
 607         dev_dbg(dev, "%s\n", __func__);
 608
 609         device_lock_assert(dax_region->dev);
 610
 611         dev_dax->ranges[mapping->range_id].mapping = NULL;
 612         mapping->range_id = -1;
 613
 614         device_del(dev);
 615         put_device(dev);
 616 }
 617
 618 static struct dev_dax_range *get_dax_range(struct device *dev)
 619 {
 620         struct dax_mapping *mapping = to_dax_mapping(dev);
 621         struct dev_dax *dev_dax = to_dev_dax(dev->parent);
 622         struct dax_region *dax_region = dev_dax->region;
 623
 624         device_lock(dax_region->dev);
 625         if (mapping->range_id < 0) {
 626                 device_unlock(dax_region->dev);
 627                 return NULL;
 628         }
 629
 630         return &dev_dax->ranges[mapping->range_id];
 631 }
 632
 633 static void put_dax_range(struct dev_dax_range *dax_range)
 634 {
 635         struct dax_mapping *mapping = dax_range->mapping;
 636         struct dev_dax *dev_dax = to_dev_dax(mapping->dev.parent);
 637         struct dax_region *dax_region = dev_dax->region;
 638
 639         device_unlock(dax_region->dev);
 640 }
 641
 642 static ssize_t start_show(struct device *dev,
 643                 struct device_attribute *attr, char *buf)
 644 {
 645         struct dev_dax_range *dax_range;
 646         ssize_t rc;
 647
 648         dax_range = get_dax_range(dev);
 649         if (!dax_range)
 650                 return -ENXIO;
 651         rc = sprintf(buf, "%#llx\n", dax_range->range.start);
 652         put_dax_range(dax_range);
 653
 654         return rc;
 655 }
 656 static DEVICE_ATTR(start, 0400, start_show, NULL);
 657
 658 static ssize_t end_show(struct device *dev,
 659                 struct device_attribute *attr, char *buf)
 660 {
 661         struct dev_dax_range *dax_range;
 662         ssize_t rc;
 663
 664         dax_range = get_dax_range(dev);
 665         if (!dax_range)
 666                 return -ENXIO;
 667         rc = sprintf(buf, "%#llx\n", dax_range->range.end);
 668         put_dax_range(dax_range);
 669
 670         return rc;
 671 }
 672 static DEVICE_ATTR(end, 0400, end_show, NULL);
 673
 674 static ssize_t pgoff_show(struct device *dev,
 675                 struct device_attribute *attr, char *buf)
 676 {
 677         struct dev_dax_range *dax_range;
 678         ssize_t rc;
 679
 680         dax_range = get_dax_range(dev);
 681         if (!dax_range)
 682                 return -ENXIO;
 683         rc = sprintf(buf, "%#lx\n", dax_range->pgoff);
 684         put_dax_range(dax_range);
 685
 686         return rc;
 687 }
 688 static DEVICE_ATTR(page_offset, 0400, pgoff_show, NULL);
 689
 690 static struct attribute *dax_mapping_attributes[] = {
 691         &dev_attr_start.attr,
 692         &dev_attr_end.attr,
 693         &dev_attr_page_offset.attr,
 694         NULL,
 695 };
 696
 697 static const struct attribute_group dax_mapping_attribute_group = {
 698         .attrs = dax_mapping_attributes,
 699 };
 700
 701 static const struct attribute_group *dax_mapping_attribute_groups[] = {
 702         &dax_mapping_attribute_group,
 703         NULL,
 704 };
 705
 706 static struct device_type dax_mapping_type = {
 707         .release = dax_mapping_release,
 708         .groups = dax_mapping_attribute_groups,
 709 };
 710
 711 static int devm_register_dax_mapping(struct dev_dax *dev_dax, int range_id)
 712 {
 713         struct dax_region *dax_region = dev_dax->region;
 714         struct dax_mapping *mapping;
 715         struct device *dev;
 716         int rc;
 717
 718         device_lock_assert(dax_region->dev);
 719
 720         if (dev_WARN_ONCE(&dev_dax->dev, !dax_region->dev->driver,
 721                                 "region disabled\n"))
 722                 return -ENXIO;
 723
 724         mapping = kzalloc(sizeof(*mapping), GFP_KERNEL);
 725         if (!mapping)
 726                 return -ENOMEM;
 727         mapping->range_id = range_id;
 728         mapping->id = ida_alloc(&dev_dax->ida, GFP_KERNEL);
 729         if (mapping->id < 0) {
 730                 kfree(mapping);
 731                 return -ENOMEM;
 732         }
 733         dev_dax->ranges[range_id].mapping = mapping;
 734         dev = &mapping->dev;
 735         device_initialize(dev);
 736         dev->parent = &dev_dax->dev;
 737         dev->type = &dax_mapping_type;
 738         dev_set_name(dev, "mapping%d", mapping->id);
 739         rc = device_add(dev);
 740         if (rc) {
 741                 put_device(dev);
 742                 return rc;
 743         }
 744
 745         rc = devm_add_action_or_reset(dax_region->dev, unregister_dax_mapping,
 746                         dev);
 747         if (rc)
 748                 return rc;
 749         return 0;
 750 }
 751
 752 static int alloc_dev_dax_range(struct dev_dax *dev_dax, u64 start,
 753                 resource_size_t size)
 754 {
 755         struct dax_region *dax_region = dev_dax->region;
 756         struct resource *res = &dax_region->res;
 757         struct device *dev = &dev_dax->dev;
 758         struct dev_dax_range *ranges;
 759         unsigned long pgoff = 0;
 760         struct resource *alloc;
 761         int i, rc;
 762
 763         device_lock_assert(dax_region->dev);
 764
 765         /* handle the seed alloc special case */
 766         if (!size) {
 767                 if (dev_WARN_ONCE(dev, dev_dax->nr_range,
 768                                         "0-size allocation must be first\n"))
 769                         return -EBUSY;
 770                 /* nr_range == 0 is elsewhere special cased as 0-size device */
 771                 return 0;
 772         }
 773
 774         alloc = __request_region(res, start, size, dev_name(dev), 0);
 775         if (!alloc)
 776                 return -ENOMEM;
 777
 778         ranges = krealloc(dev_dax->ranges, sizeof(*ranges)
 779                         * (dev_dax->nr_range + 1), GFP_KERNEL);
 780         if (!ranges) {
 781                 __release_region(res, alloc->start, resource_size(alloc));
 782                 return -ENOMEM;
 783         }
 784
 785         for (i = 0; i < dev_dax->nr_range; i++)
 786                 pgoff += PHYS_PFN(range_len(&ranges[i].range));
 787         dev_dax->ranges = ranges;
 788         ranges[dev_dax->nr_range++] = (struct dev_dax_range) {
 789                 .pgoff = pgoff,
 790                 .range = {
 791                         .start = alloc->start,
 792                         .end = alloc->end,
 793                 },
 794         };
 795
 796         dev_dbg(dev, "alloc range[%d]: %pa:%pa\n", dev_dax->nr_range - 1,
 797                         &alloc->start, &alloc->end);
 798         /*
 799          * A dev_dax instance must be registered before mapping device
 800          * children can be added. Defer to devm_create_dev_dax() to add
 801          * the initial mapping device.
 802          */
 803         if (!device_is_registered(&dev_dax->dev))
 804                 return 0;
 805
 806         rc = devm_register_dax_mapping(dev_dax, dev_dax->nr_range - 1);
 807         if (rc)
 808                 trim_dev_dax_range(dev_dax);
 809
 810         return rc;
 811 }
 812
 813 static int adjust_dev_dax_range(struct dev_dax *dev_dax, struct resource *res, resource_size_t size)
 814 {
 815         int last_range = dev_dax->nr_range - 1;
 816         struct dev_dax_range *dax_range = &dev_dax->ranges[last_range];
 817         struct dax_region *dax_region = dev_dax->region;
 818         bool is_shrink = resource_size(res) > size;
 819         struct range *range = &dax_range->range;
 820         struct device *dev = &dev_dax->dev;
 821         int rc;
 822
 823         device_lock_assert(dax_region->dev);
 824
 825         if (dev_WARN_ONCE(dev, !size, "deletion is handled by dev_dax_shrink\n"))
 826                 return -EINVAL;
 827
 828         rc = adjust_resource(res, range->start, size);
 829         if (rc)
 830                 return rc;
 831
 832         *range = (struct range) {
 833                 .start = range->start,
 834                 .end = range->start + size - 1,
 835         };
 836
 837         dev_dbg(dev, "%s range[%d]: %#llx:%#llx\n", is_shrink ? "shrink" : "extend",
 838                         last_range, (unsigned long long) range->start,
 839                         (unsigned long long) range->end);
 840
 841         return 0;
 842 }
 843
 844 static ssize_t size_show(struct device *dev,
 845                 struct device_attribute *attr, char *buf)
 846 {
 847         struct dev_dax *dev_dax = to_dev_dax(dev);
 848         unsigned long long size;
 849
 850         device_lock(dev);
 851         size = dev_dax_size(dev_dax);
 852         device_unlock(dev);
 853
 854         return sprintf(buf, "%llu\n", size);
 855 }
 856
 857 static bool alloc_is_aligned(struct dev_dax *dev_dax, resource_size_t size)
 858 {
 859         /*
 860          * The minimum mapping granularity for a device instance is a
 861          * single subsection, unless the arch says otherwise.
 862          */
 863         return IS_ALIGNED(size, max_t(unsigned long, dev_dax->align, memremap_compat_align()));
 864 }
 865
 866 static int dev_dax_shrink(struct dev_dax *dev_dax, resource_size_t size)
 867 {
 868         resource_size_t to_shrink = dev_dax_size(dev_dax) - size;
 869         struct dax_region *dax_region = dev_dax->region;
 870         struct device *dev = &dev_dax->dev;
 871         int i;
 872
 873         for (i = dev_dax->nr_range - 1; i >= 0; i--) {
 874                 struct range *range = &dev_dax->ranges[i].range;
 875                 struct dax_mapping *mapping = dev_dax->ranges[i].mapping;
 876                 struct resource *adjust = NULL, *res;
 877                 resource_size_t shrink;
 878
 879                 shrink = min_t(u64, to_shrink, range_len(range));
 880                 if (shrink >= range_len(range)) {
 881                         devm_release_action(dax_region->dev,
 882                                         unregister_dax_mapping, &mapping->dev);
 883                         trim_dev_dax_range(dev_dax);
 884                         to_shrink -= shrink;
 885                         if (!to_shrink)
 886                                 break;
 887                         continue;
 888                 }
 889
 890                 for_each_dax_region_resource(dax_region, res)
 891                         if (strcmp(res->name, dev_name(dev)) == 0
 892                                         && res->start == range->start) {
 893                                 adjust = res;
 894                                 break;
 895                         }
 896
 897                 if (dev_WARN_ONCE(dev, !adjust || i != dev_dax->nr_range - 1,
 898                                         "failed to find matching resource\n"))
 899                         return -ENXIO;
 900                 return adjust_dev_dax_range(dev_dax, adjust, range_len(range)
 901                                 - shrink);
 902         }
 903         return 0;
 904 }
 905
 906 /*
 907  * Only allow adjustments that preserve the relative pgoff of existing
 908  * allocations. I.e. the dev_dax->ranges array is ordered by increasing pgoff.
 909  */
 910 static bool adjust_ok(struct dev_dax *dev_dax, struct resource *res)
 911 {
 912         struct dev_dax_range *last;
 913         int i;
 914
 915         if (dev_dax->nr_range == 0)
 916                 return false;
 917         if (strcmp(res->name, dev_name(&dev_dax->dev)) != 0)
 918                 return false;
 919         last = &dev_dax->ranges[dev_dax->nr_range - 1];
 920         if (last->range.start != res->start || last->range.end != res->end)
 921                 return false;
 922         for (i = 0; i < dev_dax->nr_range - 1; i++) {
 923                 struct dev_dax_range *dax_range = &dev_dax->ranges[i];
 924
 925                 if (dax_range->pgoff > last->pgoff)
 926                         return false;
 927         }
 928
 929         return true;
 930 }
 931
 932 static ssize_t dev_dax_resize(struct dax_region *dax_region,
 933                 struct dev_dax *dev_dax, resource_size_t size)
 934 {
 935         resource_size_t avail = dax_region_avail_size(dax_region), to_alloc;
 936         resource_size_t dev_size = dev_dax_size(dev_dax);
 937         struct resource *region_res = &dax_region->res;
 938         struct device *dev = &dev_dax->dev;
 939         struct resource *res, *first;
 940         resource_size_t alloc = 0;
 941         int rc;
 942
 943         if (dev->driver)
 944                 return -EBUSY;
 945         if (size == dev_size)
 946                 return 0;
 947         if (size > dev_size && size - dev_size > avail)
 948                 return -ENOSPC;
 949         if (size < dev_size)
 950                 return dev_dax_shrink(dev_dax, size);
 951
 952         to_alloc = size - dev_size;
 953         if (dev_WARN_ONCE(dev, !alloc_is_aligned(dev_dax, to_alloc),
 954                         "resize of %pa misaligned\n", &to_alloc))
 955                 return -ENXIO;
 956
 957         /*
 958          * Expand the device into the unused portion of the region. This
 959          * may involve adjusting the end of an existing resource, or
 960          * allocating a new resource.
 961          */
 962 retry:
 963         first = region_res->child;
 964         if (!first)
 965                 return alloc_dev_dax_range(dev_dax, dax_region->res.start, to_alloc);
 966
 967         rc = -ENOSPC;
 968         for (res = first; res; res = res->sibling) {
 969                 struct resource *next = res->sibling;
 970
 971                 /* space at the beginning of the region */
 972                 if (res == first && res->start > dax_region->res.start) {
 973                         alloc = min(res->start - dax_region->res.start, to_alloc);
 974                         rc = alloc_dev_dax_range(dev_dax, dax_region->res.start, alloc);
 975                         break;
 976                 }
 977
 978                 alloc = 0;
 979                 /* space between allocations */
 980                 if (next && next->start > res->end + 1)
 981                         alloc = min(next->start - (res->end + 1), to_alloc);
 982
 983                 /* space at the end of the region */
 984                 if (!alloc && !next && res->end < region_res->end)
 985                         alloc = min(region_res->end - res->end, to_alloc);
 986
 987                 if (!alloc)
 988                         continue;
 989
 990                 if (adjust_ok(dev_dax, res)) {
 991                         rc = adjust_dev_dax_range(dev_dax, res, resource_size(res) + alloc);
 992                         break;
 993                 }
 994                 rc = alloc_dev_dax_range(dev_dax, res->end + 1, alloc);
 995                 break;
 996         }
 997         if (rc)
 998                 return rc;
 999         to_alloc -= alloc;
1000         if (to_alloc)
1001                 goto retry;
1002         return 0;
1003 }
1004
1005 static ssize_t size_store(struct device *dev, struct device_attribute *attr,
1006                 const char *buf, size_t len)
1007 {
1008         ssize_t rc;
1009         unsigned long long val;
1010         struct dev_dax *dev_dax = to_dev_dax(dev);
1011         struct dax_region *dax_region = dev_dax->region;
1012
1013         rc = kstrtoull(buf, 0, &val);
1014         if (rc)
1015                 return rc;
1016
1017         if (!alloc_is_aligned(dev_dax, val)) {
1018                 dev_dbg(dev, "%s: size: %lld misaligned\n", __func__, val);
1019                 return -EINVAL;
1020         }
1021
1022         device_lock(dax_region->dev);
1023         if (!dax_region->dev->driver) {
1024                 device_unlock(dax_region->dev);
1025                 return -ENXIO;
1026         }
1027         device_lock(dev);
1028         rc = dev_dax_resize(dax_region, dev_dax, val);
1029         device_unlock(dev);
1030         device_unlock(dax_region->dev);
1031
1032         return rc == 0 ? len : rc;
1033 }
1034 static DEVICE_ATTR_RW(size);
1035
1036 static ssize_t range_parse(const char *opt, size_t len, struct range *range)
1037 {
1038         unsigned long long addr = 0;
1039         char *start, *end, *str;
1040         ssize_t rc = -EINVAL;
1041
1042         str = kstrdup(opt, GFP_KERNEL);
1043         if (!str)
1044                 return rc;
1045
1046         end = str;
1047         start = strsep(&end, "-");
1048         if (!start || !end)
1049                 goto err;
1050
1051         rc = kstrtoull(start, 16, &addr);
1052         if (rc)
1053                 goto err;
1054         range->start = addr;
1055
1056         rc = kstrtoull(end, 16, &addr);
1057         if (rc)
1058                 goto err;
1059         range->end = addr;
1060
1061 err:
1062         kfree(str);
1063         return rc;
1064 }
1065
1066 static ssize_t mapping_store(struct device *dev, struct device_attribute *attr,
1067                 const char *buf, size_t len)
1068 {
1069         struct dev_dax *dev_dax = to_dev_dax(dev);
1070         struct dax_region *dax_region = dev_dax->region;
1071         size_t to_alloc;
1072         struct range r;
1073         ssize_t rc;
1074
1075         rc = range_parse(buf, len, &r);
1076         if (rc)
1077                 return rc;
1078
1079         rc = -ENXIO;
1080         device_lock(dax_region->dev);
1081         if (!dax_region->dev->driver) {
1082                 device_unlock(dax_region->dev);
1083                 return rc;
1084         }
1085         device_lock(dev);
1086
1087         to_alloc = range_len(&r);
1088         if (alloc_is_aligned(dev_dax, to_alloc))
1089                 rc = alloc_dev_dax_range(dev_dax, r.start, to_alloc);
1090         device_unlock(dev);
1091         device_unlock(dax_region->dev);
1092
1093         return rc == 0 ? len : rc;
1094 }
1095 static DEVICE_ATTR_WO(mapping);
1096
1097 static ssize_t align_show(struct device *dev,
1098                 struct device_attribute *attr, char *buf)
1099 {
1100         struct dev_dax *dev_dax = to_dev_dax(dev);
1101
1102         return sprintf(buf, "%d\n", dev_dax->align);
1103 }
1104
1105 static ssize_t dev_dax_validate_align(struct dev_dax *dev_dax)
1106 {
1107         struct device *dev = &dev_dax->dev;
1108         int i;
1109
1110         for (i = 0; i < dev_dax->nr_range; i++) {
1111                 size_t len = range_len(&dev_dax->ranges[i].range);
1112
1113                 if (!alloc_is_aligned(dev_dax, len)) {
1114                         dev_dbg(dev, "%s: align %u invalid for range %d\n",
1115                                 __func__, dev_dax->align, i);
1116                         return -EINVAL;
1117                 }
1118         }
1119
1120         return 0;
1121 }
1122
1123 static ssize_t align_store(struct device *dev, struct device_attribute *attr,
1124                 const char *buf, size_t len)
1125 {
1126         struct dev_dax *dev_dax = to_dev_dax(dev);
1127         struct dax_region *dax_region = dev_dax->region;
1128         unsigned long val, align_save;
1129         ssize_t rc;
1130
1131         rc = kstrtoul(buf, 0, &val);
1132         if (rc)
1133                 return -ENXIO;
1134
1135         if (!dax_align_valid(val))
1136                 return -EINVAL;
1137
1138         device_lock(dax_region->dev);
1139         if (!dax_region->dev->driver) {
1140                 device_unlock(dax_region->dev);
1141                 return -ENXIO;
1142         }
1143
1144         device_lock(dev);
1145         if (dev->driver) {
1146                 rc = -EBUSY;
1147                 goto out_unlock;
1148         }
1149
1150         align_save = dev_dax->align;
1151         dev_dax->align = val;
1152         rc = dev_dax_validate_align(dev_dax);
1153         if (rc)
1154                 dev_dax->align = align_save;
1155 out_unlock:
1156         device_unlock(dev);
1157         device_unlock(dax_region->dev);
1158         return rc == 0 ? len : rc;
1159 }
1160 static DEVICE_ATTR_RW(align);
1161
1162 static int dev_dax_target_node(struct dev_dax *dev_dax)
1163 {
1164         struct dax_region *dax_region = dev_dax->region;
1165
1166         return dax_region->target_node;
1167 }
1168
1169 static ssize_t target_node_show(struct device *dev,
1170                 struct device_attribute *attr, char *buf)
1171 {
1172         struct dev_dax *dev_dax = to_dev_dax(dev);
1173
1174         return sprintf(buf, "%d\n", dev_dax_target_node(dev_dax));
1175 }
1176 static DEVICE_ATTR_RO(target_node);
1177
1178 static ssize_t resource_show(struct device *dev,
1179                 struct device_attribute *attr, char *buf)
1180 {
1181         struct dev_dax *dev_dax = to_dev_dax(dev);
1182         struct dax_region *dax_region = dev_dax->region;
1183         unsigned long long start;
1184
1185         if (dev_dax->nr_range < 1)
1186                 start = dax_region->res.start;
1187         else
1188                 start = dev_dax->ranges[0].range.start;
1189
1190         return sprintf(buf, "%#llx\n", start);
1191 }
1192 static DEVICE_ATTR(resource, 0400, resource_show, NULL);
1193
1194 static ssize_t modalias_show(struct device *dev, struct device_attribute *attr,
1195                 char *buf)
1196 {
1197         /*
1198          * We only ever expect to handle device-dax instances, i.e. the
1199          * @type argument to MODULE_ALIAS_DAX_DEVICE() is always zero
1200          */
1201         return sprintf(buf, DAX_DEVICE_MODALIAS_FMT "\n", 0);
1202 }
1203 static DEVICE_ATTR_RO(modalias);
1204
1205 static ssize_t numa_node_show(struct device *dev,
1206                 struct device_attribute *attr, char *buf)
1207 {
1208         return sprintf(buf, "%d\n", dev_to_node(dev));
1209 }
1210 static DEVICE_ATTR_RO(numa_node);
1211
1212 static umode_t dev_dax_visible(struct kobject *kobj, struct attribute *a, int n)
1213 {
1214         struct device *dev = container_of(kobj, struct device, kobj);
1215         struct dev_dax *dev_dax = to_dev_dax(dev);
1216         struct dax_region *dax_region = dev_dax->region;
1217
1218         if (a == &dev_attr_target_node.attr && dev_dax_target_node(dev_dax) < 0)
1219                 return 0;
1220         if (a == &dev_attr_numa_node.attr && !IS_ENABLED(CONFIG_NUMA))
1221                 return 0;
1222         if (a == &dev_attr_mapping.attr && is_static(dax_region))
1223                 return 0;
1224         if ((a == &dev_attr_align.attr ||
1225              a == &dev_attr_size.attr) && is_static(dax_region))
1226                 return 0444;
1227         return a->mode;
1228 }
1229
1230 static struct attribute *dev_dax_attributes[] = {
1231         &dev_attr_modalias.attr,
1232         &dev_attr_size.attr,
1233         &dev_attr_mapping.attr,
1234         &dev_attr_target_node.attr,
1235         &dev_attr_align.attr,
1236         &dev_attr_resource.attr,
1237         &dev_attr_numa_node.attr,
1238         NULL,
1239 };
1240
1241 static const struct attribute_group dev_dax_attribute_group = {
1242         .attrs = dev_dax_attributes,
1243         .is_visible = dev_dax_visible,
1244 };
1245
1246 static const struct attribute_group *dax_attribute_groups[] = {
1247         &dev_dax_attribute_group,
1248         NULL,
1249 };
1250
1251 static void dev_dax_release(struct device *dev)
1252 {
1253         struct dev_dax *dev_dax = to_dev_dax(dev);
1254         struct dax_region *dax_region = dev_dax->region;
1255         struct dax_device *dax_dev = dev_dax->dax_dev;
1256
1257         put_dax(dax_dev);
1258         free_dev_dax_id(dev_dax);
1259         dax_region_put(dax_region);
1260         kfree(dev_dax->pgmap);
1261         kfree(dev_dax);
1262 }
1263
1264 static const struct device_type dev_dax_type = {
1265         .release = dev_dax_release,
1266         .groups = dax_attribute_groups,
1267 };
1268
1269 struct dev_dax *devm_create_dev_dax(struct dev_dax_data *data)
1270 {
1271         struct dax_region *dax_region = data->dax_region;
1272         struct device *parent = dax_region->dev;
1273         struct dax_device *dax_dev;
1274         struct dev_dax *dev_dax;
1275         struct inode *inode;
1276         struct device *dev;
1277         int rc;
1278
1279         dev_dax = kzalloc(sizeof(*dev_dax), GFP_KERNEL);
1280         if (!dev_dax)
1281                 return ERR_PTR(-ENOMEM);
1282
1283         if (is_static(dax_region)) {
1284                 if (dev_WARN_ONCE(parent, data->id < 0,
1285                                 "dynamic id specified to static region\n")) {
1286                         rc = -EINVAL;
1287                         goto err_id;
1288                 }
1289
1290                 dev_dax->id = data->id;
1291         } else {
1292                 if (dev_WARN_ONCE(parent, data->id >= 0,
1293                                 "static id specified to dynamic region\n")) {
1294                         rc = -EINVAL;
1295                         goto err_id;
1296                 }
1297
1298                 rc = ida_alloc(&dax_region->ida, GFP_KERNEL);
1299                 if (rc < 0)
1300                         goto err_id;
1301                 dev_dax->id = rc;
1302         }
1303
1304         dev_dax->region = dax_region;
1305         dev = &dev_dax->dev;
1306         device_initialize(dev);
1307         dev_set_name(dev, "dax%d.%d", dax_region->id, dev_dax->id);
1308
1309         rc = alloc_dev_dax_range(dev_dax, dax_region->res.start, data->size);
1310         if (rc)
1311                 goto err_range;
1312
1313         if (data->pgmap) {
1314                 dev_WARN_ONCE(parent, !is_static(dax_region),
1315                         "custom dev_pagemap requires a static dax_region\n");
1316
1317                 dev_dax->pgmap = kmemdup(data->pgmap,
1318                                 sizeof(struct dev_pagemap), GFP_KERNEL);
1319                 if (!dev_dax->pgmap) {
1320                         rc = -ENOMEM;
1321                         goto err_pgmap;
1322                 }
1323         }
1324
1325         /*
1326          * No 'host' or dax_operations since there is no access to this
1327          * device outside of mmap of the resulting character device.
1328          */
1329         dax_dev = alloc_dax(dev_dax, NULL, NULL, DAXDEV_F_SYNC);
1330         if (IS_ERR(dax_dev)) {
1331                 rc = PTR_ERR(dax_dev);
1332                 goto err_alloc_dax;
1333         }
1334
1335         /* a device_dax instance is dead while the driver is not attached */
1336         kill_dax(dax_dev);
1337
1338         dev_dax->dax_dev = dax_dev;
1339         dev_dax->target_node = dax_region->target_node;
1340         dev_dax->align = dax_region->align;
1341         ida_init(&dev_dax->ida);
1342         kref_get(&dax_region->kref);
1343
1344         inode = dax_inode(dax_dev);
1345         dev->devt = inode->i_rdev;
1346         if (data->subsys == DEV_DAX_BUS)
1347                 dev->bus = &dax_bus_type;
1348         else
1349                 dev->class = dax_class;
1350         dev->parent = parent;
1351         dev->type = &dev_dax_type;
1352
1353         rc = device_add(dev);
1354         if (rc) {
1355                 kill_dev_dax(dev_dax);
1356                 put_device(dev);
1357                 return ERR_PTR(rc);
1358         }
1359
1360         rc = devm_add_action_or_reset(dax_region->dev, unregister_dev_dax, dev);
1361         if (rc)
1362                 return ERR_PTR(rc);
1363
1364         /* register mapping device for the initial allocation range */
1365         if (dev_dax->nr_range && range_len(&dev_dax->ranges[0].range)) {
1366                 rc = devm_register_dax_mapping(dev_dax, 0);
1367                 if (rc)
1368                         return ERR_PTR(rc);
1369         }
1370
1371         return dev_dax;
1372
1373 err_alloc_dax:
1374         kfree(dev_dax->pgmap);
1375 err_pgmap:
1376         free_dev_dax_ranges(dev_dax);
1377 err_range:
1378         free_dev_dax_id(dev_dax);
1379 err_id:
1380         kfree(dev_dax);
1381
1382         return ERR_PTR(rc);
1383 }
1384 EXPORT_SYMBOL_GPL(devm_create_dev_dax);
1385
1386 static int match_always_count;
1387
1388 int __dax_driver_register(struct dax_device_driver *dax_drv,
1389                 struct module *module, const char *mod_name)
1390 {
1391         struct device_driver *drv = &dax_drv->drv;
1392         int rc = 0;
1393
1394         /*
1395          * dax_bus_probe() calls dax_drv->probe() unconditionally.
1396          * So better be safe than sorry and ensure it is provided.
1397          */
1398         if (!dax_drv->probe)
1399                 return -EINVAL;
1400
1401         INIT_LIST_HEAD(&dax_drv->ids);
1402         drv->owner = module;
1403         drv->name = mod_name;
1404         drv->mod_name = mod_name;
1405         drv->bus = &dax_bus_type;
1406
1407         /* there can only be one default driver */
1408         mutex_lock(&dax_bus_lock);
1409         match_always_count += dax_drv->match_always;
1410         if (match_always_count > 1) {
1411                 match_always_count--;
1412                 WARN_ON(1);
1413                 rc = -EINVAL;
1414         }
1415         mutex_unlock(&dax_bus_lock);
1416         if (rc)
1417                 return rc;
1418
1419         rc = driver_register(drv);
1420         if (rc && dax_drv->match_always) {
1421                 mutex_lock(&dax_bus_lock);
1422                 match_always_count -= dax_drv->match_always;
1423                 mutex_unlock(&dax_bus_lock);
1424         }
1425
1426         return rc;
1427 }
1428 EXPORT_SYMBOL_GPL(__dax_driver_register);
1429
1430 void dax_driver_unregister(struct dax_device_driver *dax_drv)
1431 {
1432         struct device_driver *drv = &dax_drv->drv;
1433         struct dax_id *dax_id, *_id;
1434
1435         mutex_lock(&dax_bus_lock);
1436         match_always_count -= dax_drv->match_always;
1437         list_for_each_entry_safe(dax_id, _id, &dax_drv->ids, list) {
1438                 list_del(&dax_id->list);
1439                 kfree(dax_id);
1440         }
1441         mutex_unlock(&dax_bus_lock);
1442         driver_unregister(drv);
1443 }
1444 EXPORT_SYMBOL_GPL(dax_driver_unregister);
1445
1446 int __init dax_bus_init(void)
1447 {
1448         int rc;
1449
1450         if (IS_ENABLED(CONFIG_DEV_DAX_PMEM_COMPAT)) {
1451                 dax_class = class_create(THIS_MODULE, "dax");
1452                 if (IS_ERR(dax_class))
1453                         return PTR_ERR(dax_class);
1454         }
1455
1456         rc = bus_register(&dax_bus_type);
1457         if (rc)
1458                 class_destroy(dax_class);
1459         return rc;
1460 }
1461
1462 void __exit dax_bus_exit(void)
1463 {
1464         bus_unregister(&dax_bus_type);
1465         class_destroy(dax_class);
1466 }