drivers/spi/spi.c

   1 /*
   2  * SPI init/core code
   3  *
   4  * Copyright (C) 2005 David Brownell
   5  * Copyright (C) 2008 Secret Lab Technologies Ltd.
   6  *
   7  * This program is free software; you can redistribute it and/or modify
   8  * it under the terms of the GNU General Public License as published by
   9  * the Free Software Foundation; either version 2 of the License, or
  10  * (at your option) any later version.
  11  *
  12  * This program is distributed in the hope that it will be useful,
  13  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  14  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  15  * GNU General Public License for more details.
  16  */
  17
  18 #include <linux/kernel.h>
  19 #include <linux/device.h>
  20 #include <linux/init.h>
  21 #include <linux/cache.h>
  22 #include <linux/dma-mapping.h>
  23 #include <linux/dmaengine.h>
  24 #include <linux/mutex.h>
  25 #include <linux/of_device.h>
  26 #include <linux/of_irq.h>
  27 #include <linux/clk/clk-conf.h>
  28 #include <linux/slab.h>
  29 #include <linux/mod_devicetable.h>
  30 #include <linux/spi/spi.h>
  31 #include <linux/of_gpio.h>
  32 #include <linux/pm_runtime.h>
  33 #include <linux/pm_domain.h>
  34 #include <linux/export.h>
  35 #include <linux/sched/rt.h>
  36 #include <linux/delay.h>
  37 #include <linux/kthread.h>
  38 #include <linux/ioport.h>
  39 #include <linux/acpi.h>
  40
  41 #define CREATE_TRACE_POINTS
  42 #include <trace/events/spi.h>
  43
  44 static void spidev_release(struct device *dev)
  45 {
  46         struct spi_device       *spi = to_spi_device(dev);
  47
  48         /* spi masters may cleanup for released devices */
  49         if (spi->master->cleanup)
  50                 spi->master->cleanup(spi);
  51
  52         spi_master_put(spi->master);
  53         kfree(spi);
  54 }
  55
  56 static ssize_t
  57 modalias_show(struct device *dev, struct device_attribute *a, char *buf)
  58 {
  59         const struct spi_device *spi = to_spi_device(dev);
  60         int len;
  61
  62         len = acpi_device_modalias(dev, buf, PAGE_SIZE - 1);
  63         if (len != -ENODEV)
  64                 return len;
  65
  66         return sprintf(buf, "%s%s\n", SPI_MODULE_PREFIX, spi->modalias);
  67 }
  68 static DEVICE_ATTR_RO(modalias);
  69
  70 #define SPI_STATISTICS_ATTRS(field, file)                               \
  71 static ssize_t spi_master_##field##_show(struct device *dev,            \
  72                                          struct device_attribute *attr, \
  73                                          char *buf)                     \
  74 {                                                                       \
  75         struct spi_master *master = container_of(dev,                   \
  76                                                  struct spi_master, dev); \
  77         return spi_statistics_##field##_show(&master->statistics, buf); \
  78 }                                                                       \
  79 static struct device_attribute dev_attr_spi_master_##field = {          \
  80         .attr = { .name = file, .mode = S_IRUGO },                      \
  81         .show = spi_master_##field##_show,                              \
  82 };                                                                      \
  83 static ssize_t spi_device_##field##_show(struct device *dev,            \
  84                                          struct device_attribute *attr, \
  85                                         char *buf)                      \
  86 {                                                                       \
  87         struct spi_device *spi = container_of(dev,                      \
  88                                               struct spi_device, dev);  \
  89         return spi_statistics_##field##_show(&spi->statistics, buf);    \
  90 }                                                                       \
  91 static struct device_attribute dev_attr_spi_device_##field = {          \
  92         .attr = { .name = file, .mode = S_IRUGO },                      \
  93         .show = spi_device_##field##_show,                              \
  94 }
  95
  96 #define SPI_STATISTICS_SHOW_NAME(name, file, field, format_string)      \
  97 static ssize_t spi_statistics_##name##_show(struct spi_statistics *stat, \
  98                                             char *buf)                  \
  99 {                                                                       \
 100         unsigned long flags;                                            \
 101         ssize_t len;                                                    \
 102         spin_lock_irqsave(&stat->lock, flags);                          \
 103         len = sprintf(buf, format_string, stat->field);                 \
 104         spin_unlock_irqrestore(&stat->lock, flags);                     \
 105         return len;                                                     \
 106 }                                                                       \
 107 SPI_STATISTICS_ATTRS(name, file)
 108
 109 #define SPI_STATISTICS_SHOW(field, format_string)                       \
 110         SPI_STATISTICS_SHOW_NAME(field, __stringify(field),             \
 111                                  field, format_string)
 112
 113 SPI_STATISTICS_SHOW(messages, "%lu");
 114 SPI_STATISTICS_SHOW(transfers, "%lu");
 115 SPI_STATISTICS_SHOW(errors, "%lu");
 116 SPI_STATISTICS_SHOW(timedout, "%lu");
 117
 118 SPI_STATISTICS_SHOW(spi_sync, "%lu");
 119 SPI_STATISTICS_SHOW(spi_sync_immediate, "%lu");
 120 SPI_STATISTICS_SHOW(spi_async, "%lu");
 121
 122 SPI_STATISTICS_SHOW(bytes, "%llu");
 123 SPI_STATISTICS_SHOW(bytes_rx, "%llu");
 124 SPI_STATISTICS_SHOW(bytes_tx, "%llu");
 125
 126 static struct attribute *spi_dev_attrs[] = {
 127         &dev_attr_modalias.attr,
 128         NULL,
 129 };
 130
 131 static const struct attribute_group spi_dev_group = {
 132         .attrs  = spi_dev_attrs,
 133 };
 134
 135 static struct attribute *spi_device_statistics_attrs[] = {
 136         &dev_attr_spi_device_messages.attr,
 137         &dev_attr_spi_device_transfers.attr,
 138         &dev_attr_spi_device_errors.attr,
 139         &dev_attr_spi_device_timedout.attr,
 140         &dev_attr_spi_device_spi_sync.attr,
 141         &dev_attr_spi_device_spi_sync_immediate.attr,
 142         &dev_attr_spi_device_spi_async.attr,
 143         &dev_attr_spi_device_bytes.attr,
 144         &dev_attr_spi_device_bytes_rx.attr,
 145         &dev_attr_spi_device_bytes_tx.attr,
 146         NULL,
 147 };
 148
 149 static const struct attribute_group spi_device_statistics_group = {
 150         .name  = "statistics",
 151         .attrs  = spi_device_statistics_attrs,
 152 };
 153
 154 static const struct attribute_group *spi_dev_groups[] = {
 155         &spi_dev_group,
 156         &spi_device_statistics_group,
 157         NULL,
 158 };
 159
 160 static struct attribute *spi_master_statistics_attrs[] = {
 161         &dev_attr_spi_master_messages.attr,
 162         &dev_attr_spi_master_transfers.attr,
 163         &dev_attr_spi_master_errors.attr,
 164         &dev_attr_spi_master_timedout.attr,
 165         &dev_attr_spi_master_spi_sync.attr,
 166         &dev_attr_spi_master_spi_sync_immediate.attr,
 167         &dev_attr_spi_master_spi_async.attr,
 168         &dev_attr_spi_master_bytes.attr,
 169         &dev_attr_spi_master_bytes_rx.attr,
 170         &dev_attr_spi_master_bytes_tx.attr,
 171         NULL,
 172 };
 173
 174 static const struct attribute_group spi_master_statistics_group = {
 175         .name  = "statistics",
 176         .attrs  = spi_master_statistics_attrs,
 177 };
 178
 179 static const struct attribute_group *spi_master_groups[] = {
 180         &spi_master_statistics_group,
 181         NULL,
 182 };
 183
 184 void spi_statistics_add_transfer_stats(struct spi_statistics *stats,
 185                                        struct spi_transfer *xfer,
 186                                        struct spi_master *master)
 187 {
 188         unsigned long flags;
 189
 190         spin_lock_irqsave(&stats->lock, flags);
 191
 192         stats->transfers++;
 193
 194         stats->bytes += xfer->len;
 195         if ((xfer->tx_buf) &&
 196             (xfer->tx_buf != master->dummy_tx))
 197                 stats->bytes_tx += xfer->len;
 198         if ((xfer->rx_buf) &&
 199             (xfer->rx_buf != master->dummy_rx))
 200                 stats->bytes_rx += xfer->len;
 201
 202         spin_unlock_irqrestore(&stats->lock, flags);
 203 }
 204 EXPORT_SYMBOL_GPL(spi_statistics_add_transfer_stats);
 205
 206 /* modalias support makes "modprobe $MODALIAS" new-style hotplug work,
 207  * and the sysfs version makes coldplug work too.
 208  */
 209
 210 static const struct spi_device_id *spi_match_id(const struct spi_device_id *id,
 211                                                 const struct spi_device *sdev)
 212 {
 213         while (id->name[0]) {
 214                 if (!strcmp(sdev->modalias, id->name))
 215                         return id;
 216                 id++;
 217         }
 218         return NULL;
 219 }
 220
 221 const struct spi_device_id *spi_get_device_id(const struct spi_device *sdev)
 222 {
 223         const struct spi_driver *sdrv = to_spi_driver(sdev->dev.driver);
 224
 225         return spi_match_id(sdrv->id_table, sdev);
 226 }
 227 EXPORT_SYMBOL_GPL(spi_get_device_id);
 228
 229 static int spi_match_device(struct device *dev, struct device_driver *drv)
 230 {
 231         const struct spi_device *spi = to_spi_device(dev);
 232         const struct spi_driver *sdrv = to_spi_driver(drv);
 233
 234         /* Attempt an OF style match */
 235         if (of_driver_match_device(dev, drv))
 236                 return 1;
 237
 238         /* Then try ACPI */
 239         if (acpi_driver_match_device(dev, drv))
 240                 return 1;
 241
 242         if (sdrv->id_table)
 243                 return !!spi_match_id(sdrv->id_table, spi);
 244
 245         return strcmp(spi->modalias, drv->name) == 0;
 246 }
 247
 248 static int spi_uevent(struct device *dev, struct kobj_uevent_env *env)
 249 {
 250         const struct spi_device         *spi = to_spi_device(dev);
 251         int rc;
 252
 253         rc = acpi_device_uevent_modalias(dev, env);
 254         if (rc != -ENODEV)
 255                 return rc;
 256
 257         add_uevent_var(env, "MODALIAS=%s%s", SPI_MODULE_PREFIX, spi->modalias);
 258         return 0;
 259 }
 260
 261 struct bus_type spi_bus_type = {
 262         .name           = "spi",
 263         .dev_groups     = spi_dev_groups,
 264         .match          = spi_match_device,
 265         .uevent         = spi_uevent,
 266 };
 267 EXPORT_SYMBOL_GPL(spi_bus_type);
 268
 269
 270 static int spi_drv_probe(struct device *dev)
 271 {
 272         const struct spi_driver         *sdrv = to_spi_driver(dev->driver);
 273         int ret;
 274
 275         ret = of_clk_set_defaults(dev->of_node, false);
 276         if (ret)
 277                 return ret;
 278
 279         ret = dev_pm_domain_attach(dev, true);
 280         if (ret != -EPROBE_DEFER) {
 281                 ret = sdrv->probe(to_spi_device(dev));
 282                 if (ret)
 283                         dev_pm_domain_detach(dev, true);
 284         }
 285
 286         return ret;
 287 }
 288
 289 static int spi_drv_remove(struct device *dev)
 290 {
 291         const struct spi_driver         *sdrv = to_spi_driver(dev->driver);
 292         int ret;
 293
 294         ret = sdrv->remove(to_spi_device(dev));
 295         dev_pm_domain_detach(dev, true);
 296
 297         return ret;
 298 }
 299
 300 static void spi_drv_shutdown(struct device *dev)
 301 {
 302         const struct spi_driver         *sdrv = to_spi_driver(dev->driver);
 303
 304         sdrv->shutdown(to_spi_device(dev));
 305 }
 306
 307 /**
 308  * __spi_register_driver - register a SPI driver
 309  * @sdrv: the driver to register
 310  * Context: can sleep
 311  */
 312 int __spi_register_driver(struct module *owner, struct spi_driver *sdrv)
 313 {
 314         sdrv->driver.owner = owner;
 315         sdrv->driver.bus = &spi_bus_type;
 316         if (sdrv->probe)
 317                 sdrv->driver.probe = spi_drv_probe;
 318         if (sdrv->remove)
 319                 sdrv->driver.remove = spi_drv_remove;
 320         if (sdrv->shutdown)
 321                 sdrv->driver.shutdown = spi_drv_shutdown;
 322         return driver_register(&sdrv->driver);
 323 }
 324 EXPORT_SYMBOL_GPL(__spi_register_driver);
 325
 326 /*-------------------------------------------------------------------------*/
 327
 328 /* SPI devices should normally not be created by SPI device drivers; that
 329  * would make them board-specific.  Similarly with SPI master drivers.
 330  * Device registration normally goes into like arch/.../mach.../board-YYY.c
 331  * with other readonly (flashable) information about mainboard devices.
 332  */
 333
 334 struct boardinfo {
 335         struct list_head        list;
 336         struct spi_board_info   board_info;
 337 };
 338
 339 static LIST_HEAD(board_list);
 340 static LIST_HEAD(spi_master_list);
 341
 342 /*
 343  * Used to protect add/del opertion for board_info list and
 344  * spi_master list, and their matching process
 345  */
 346 static DEFINE_MUTEX(board_lock);
 347
 348 /**
 349  * spi_alloc_device - Allocate a new SPI device
 350  * @master: Controller to which device is connected
 351  * Context: can sleep
 352  *
 353  * Allows a driver to allocate and initialize a spi_device without
 354  * registering it immediately.  This allows a driver to directly
 355  * fill the spi_device with device parameters before calling
 356  * spi_add_device() on it.
 357  *
 358  * Caller is responsible to call spi_add_device() on the returned
 359  * spi_device structure to add it to the SPI master.  If the caller
 360  * needs to discard the spi_device without adding it, then it should
 361  * call spi_dev_put() on it.
 362  *
 363  * Returns a pointer to the new device, or NULL.
 364  */
 365 struct spi_device *spi_alloc_device(struct spi_master *master)
 366 {
 367         struct spi_device       *spi;
 368
 369         if (!spi_master_get(master))
 370                 return NULL;
 371
 372         spi = kzalloc(sizeof(*spi), GFP_KERNEL);
 373         if (!spi) {
 374                 spi_master_put(master);
 375                 return NULL;
 376         }
 377
 378         spi->master = master;
 379         spi->dev.parent = &master->dev;
 380         spi->dev.bus = &spi_bus_type;
 381         spi->dev.release = spidev_release;
 382         spi->cs_gpio = -ENOENT;
 383
 384         spin_lock_init(&spi->statistics.lock);
 385
 386         device_initialize(&spi->dev);
 387         return spi;
 388 }
 389 EXPORT_SYMBOL_GPL(spi_alloc_device);
 390
 391 static void spi_dev_set_name(struct spi_device *spi)
 392 {
 393         struct acpi_device *adev = ACPI_COMPANION(&spi->dev);
 394
 395         if (adev) {
 396                 dev_set_name(&spi->dev, "spi-%s", acpi_dev_name(adev));
 397                 return;
 398         }
 399
 400         dev_set_name(&spi->dev, "%s.%u", dev_name(&spi->master->dev),
 401                      spi->chip_select);
 402 }
 403
 404 static int spi_dev_check(struct device *dev, void *data)
 405 {
 406         struct spi_device *spi = to_spi_device(dev);
 407         struct spi_device *new_spi = data;
 408
 409         if (spi->master == new_spi->master &&
 410             spi->chip_select == new_spi->chip_select)
 411                 return -EBUSY;
 412         return 0;
 413 }
 414
 415 /**
 416  * spi_add_device - Add spi_device allocated with spi_alloc_device
 417  * @spi: spi_device to register
 418  *
 419  * Companion function to spi_alloc_device.  Devices allocated with
 420  * spi_alloc_device can be added onto the spi bus with this function.
 421  *
 422  * Returns 0 on success; negative errno on failure
 423  */
 424 int spi_add_device(struct spi_device *spi)
 425 {
 426         static DEFINE_MUTEX(spi_add_lock);
 427         struct spi_master *master = spi->master;
 428         struct device *dev = master->dev.parent;
 429         int status;
 430
 431         /* Chipselects are numbered 0..max; validate. */
 432         if (spi->chip_select >= master->num_chipselect) {
 433                 dev_err(dev, "cs%d >= max %d\n",
 434                         spi->chip_select,
 435                         master->num_chipselect);
 436                 return -EINVAL;
 437         }
 438
 439         /* Set the bus ID string */
 440         spi_dev_set_name(spi);
 441
 442         /* We need to make sure there's no other device with this
 443          * chipselect **BEFORE** we call setup(), else we'll trash
 444          * its configuration.  Lock against concurrent add() calls.
 445          */
 446         mutex_lock(&spi_add_lock);
 447
 448         status = bus_for_each_dev(&spi_bus_type, NULL, spi, spi_dev_check);
 449         if (status) {
 450                 dev_err(dev, "chipselect %d already in use\n",
 451                                 spi->chip_select);
 452                 goto done;
 453         }
 454
 455         if (master->cs_gpios)
 456                 spi->cs_gpio = master->cs_gpios[spi->chip_select];
 457
 458         /* Drivers may modify this initial i/o setup, but will
 459          * normally rely on the device being setup.  Devices
 460          * using SPI_CS_HIGH can't coexist well otherwise...
 461          */
 462         status = spi_setup(spi);
 463         if (status < 0) {
 464                 dev_err(dev, "can't setup %s, status %d\n",
 465                                 dev_name(&spi->dev), status);
 466                 goto done;
 467         }
 468
 469         /* Device may be bound to an active driver when this returns */
 470         status = device_add(&spi->dev);
 471         if (status < 0)
 472                 dev_err(dev, "can't add %s, status %d\n",
 473                                 dev_name(&spi->dev), status);
 474         else
 475                 dev_dbg(dev, "registered child %s\n", dev_name(&spi->dev));
 476
 477 done:
 478         mutex_unlock(&spi_add_lock);
 479         return status;
 480 }
 481 EXPORT_SYMBOL_GPL(spi_add_device);
 482
 483 /**
 484  * spi_new_device - instantiate one new SPI device
 485  * @master: Controller to which device is connected
 486  * @chip: Describes the SPI device
 487  * Context: can sleep
 488  *
 489  * On typical mainboards, this is purely internal; and it's not needed
 490  * after board init creates the hard-wired devices.  Some development
 491  * platforms may not be able to use spi_register_board_info though, and
 492  * this is exported so that for example a USB or parport based adapter
 493  * driver could add devices (which it would learn about out-of-band).
 494  *
 495  * Returns the new device, or NULL.
 496  */
 497 struct spi_device *spi_new_device(struct spi_master *master,
 498                                   struct spi_board_info *chip)
 499 {
 500         struct spi_device       *proxy;
 501         int                     status;
 502
 503         /* NOTE:  caller did any chip->bus_num checks necessary.
 504          *
 505          * Also, unless we change the return value convention to use
 506          * error-or-pointer (not NULL-or-pointer), troubleshootability
 507          * suggests syslogged diagnostics are best here (ugh).
 508          */
 509
 510         proxy = spi_alloc_device(master);
 511         if (!proxy)
 512                 return NULL;
 513
 514         WARN_ON(strlen(chip->modalias) >= sizeof(proxy->modalias));
 515
 516         proxy->chip_select = chip->chip_select;
 517         proxy->max_speed_hz = chip->max_speed_hz;
 518         proxy->mode = chip->mode;
 519         proxy->irq = chip->irq;
 520         strlcpy(proxy->modalias, chip->modalias, sizeof(proxy->modalias));
 521         proxy->dev.platform_data = (void *) chip->platform_data;
 522         proxy->controller_data = chip->controller_data;
 523         proxy->controller_state = NULL;
 524
 525         status = spi_add_device(proxy);
 526         if (status < 0) {
 527                 spi_dev_put(proxy);
 528                 return NULL;
 529         }
 530
 531         return proxy;
 532 }
 533 EXPORT_SYMBOL_GPL(spi_new_device);
 534
 535 static void spi_match_master_to_boardinfo(struct spi_master *master,
 536                                 struct spi_board_info *bi)
 537 {
 538         struct spi_device *dev;
 539
 540         if (master->bus_num != bi->bus_num)
 541                 return;
 542
 543         dev = spi_new_device(master, bi);
 544         if (!dev)
 545                 dev_err(master->dev.parent, "can't create new device for %s\n",
 546                         bi->modalias);
 547 }
 548
 549 /**
 550  * spi_register_board_info - register SPI devices for a given board
 551  * @info: array of chip descriptors
 552  * @n: how many descriptors are provided
 553  * Context: can sleep
 554  *
 555  * Board-specific early init code calls this (probably during arch_initcall)
 556  * with segments of the SPI device table.  Any device nodes are created later,
 557  * after the relevant parent SPI controller (bus_num) is defined.  We keep
 558  * this table of devices forever, so that reloading a controller driver will
 559  * not make Linux forget about these hard-wired devices.
 560  *
 561  * Other code can also call this, e.g. a particular add-on board might provide
 562  * SPI devices through its expansion connector, so code initializing that board
 563  * would naturally declare its SPI devices.
 564  *
 565  * The board info passed can safely be __initdata ... but be careful of
 566  * any embedded pointers (platform_data, etc), they're copied as-is.
 567  */
 568 int spi_register_board_info(struct spi_board_info const *info, unsigned n)
 569 {
 570         struct boardinfo *bi;
 571         int i;
 572
 573         if (!n)
 574                 return -EINVAL;
 575
 576         bi = kzalloc(n * sizeof(*bi), GFP_KERNEL);
 577         if (!bi)
 578                 return -ENOMEM;
 579
 580         for (i = 0; i < n; i++, bi++, info++) {
 581                 struct spi_master *master;
 582
 583                 memcpy(&bi->board_info, info, sizeof(*info));
 584                 mutex_lock(&board_lock);
 585                 list_add_tail(&bi->list, &board_list);
 586                 list_for_each_entry(master, &spi_master_list, list)
 587                         spi_match_master_to_boardinfo(master, &bi->board_info);
 588                 mutex_unlock(&board_lock);
 589         }
 590
 591         return 0;
 592 }
 593
 594 /*-------------------------------------------------------------------------*/
 595
 596 static void spi_set_cs(struct spi_device *spi, bool enable)
 597 {
 598         if (spi->mode & SPI_CS_HIGH)
 599                 enable = !enable;
 600
 601         if (spi->cs_gpio >= 0)
 602                 gpio_set_value(spi->cs_gpio, !enable);
 603         else if (spi->master->set_cs)
 604                 spi->master->set_cs(spi, !enable);
 605 }
 606
 607 #ifdef CONFIG_HAS_DMA
 608 static int spi_map_buf(struct spi_master *master, struct device *dev,
 609                        struct sg_table *sgt, void *buf, size_t len,
 610                        enum dma_data_direction dir)
 611 {
 612         const bool vmalloced_buf = is_vmalloc_addr(buf);
 613         int desc_len;
 614         int sgs;
 615         struct page *vm_page;
 616         void *sg_buf;
 617         size_t min;
 618         int i, ret;
 619
 620         if (vmalloced_buf) {
 621                 desc_len = PAGE_SIZE;
 622                 sgs = DIV_ROUND_UP(len + offset_in_page(buf), desc_len);
 623         } else {
 624                 desc_len = master->max_dma_len;
 625                 sgs = DIV_ROUND_UP(len, desc_len);
 626         }
 627
 628         ret = sg_alloc_table(sgt, sgs, GFP_KERNEL);
 629         if (ret != 0)
 630                 return ret;
 631
 632         for (i = 0; i < sgs; i++) {
 633
 634                 if (vmalloced_buf) {
 635                         min = min_t(size_t,
 636                                     len, desc_len - offset_in_page(buf));
 637                         vm_page = vmalloc_to_page(buf);
 638                         if (!vm_page) {
 639                                 sg_free_table(sgt);
 640                                 return -ENOMEM;
 641                         }
 642                         sg_set_page(&sgt->sgl[i], vm_page,
 643                                     min, offset_in_page(buf));
 644                 } else {
 645                         min = min_t(size_t, len, desc_len);
 646                         sg_buf = buf;
 647                         sg_set_buf(&sgt->sgl[i], sg_buf, min);
 648                 }
 649
 650
 651                 buf += min;
 652                 len -= min;
 653         }
 654
 655         ret = dma_map_sg(dev, sgt->sgl, sgt->nents, dir);
 656         if (!ret)
 657                 ret = -ENOMEM;
 658         if (ret < 0) {
 659                 sg_free_table(sgt);
 660                 return ret;
 661         }
 662
 663         sgt->nents = ret;
 664
 665         return 0;
 666 }
 667
 668 static void spi_unmap_buf(struct spi_master *master, struct device *dev,
 669                           struct sg_table *sgt, enum dma_data_direction dir)
 670 {
 671         if (sgt->orig_nents) {
 672                 dma_unmap_sg(dev, sgt->sgl, sgt->orig_nents, dir);
 673                 sg_free_table(sgt);
 674         }
 675 }
 676
 677 static int __spi_map_msg(struct spi_master *master, struct spi_message *msg)
 678 {
 679         struct device *tx_dev, *rx_dev;
 680         struct spi_transfer *xfer;
 681         int ret;
 682
 683         if (!master->can_dma)
 684                 return 0;
 685
 686         if (master->dma_tx)
 687                 tx_dev = master->dma_tx->device->dev;
 688         else
 689                 tx_dev = &master->dev;
 690
 691         if (master->dma_rx)
 692                 rx_dev = master->dma_rx->device->dev;
 693         else
 694                 rx_dev = &master->dev;
 695
 696         list_for_each_entry(xfer, &msg->transfers, transfer_list) {
 697                 if (!master->can_dma(master, msg->spi, xfer))
 698                         continue;
 699
 700                 if (xfer->tx_buf != NULL) {
 701                         ret = spi_map_buf(master, tx_dev, &xfer->tx_sg,
 702                                           (void *)xfer->tx_buf, xfer->len,
 703                                           DMA_TO_DEVICE);
 704                         if (ret != 0)
 705                                 return ret;
 706                 }
 707
 708                 if (xfer->rx_buf != NULL) {
 709                         ret = spi_map_buf(master, rx_dev, &xfer->rx_sg,
 710                                           xfer->rx_buf, xfer->len,
 711                                           DMA_FROM_DEVICE);
 712                         if (ret != 0) {
 713                                 spi_unmap_buf(master, tx_dev, &xfer->tx_sg,
 714                                               DMA_TO_DEVICE);
 715                                 return ret;
 716                         }
 717                 }
 718         }
 719
 720         master->cur_msg_mapped = true;
 721
 722         return 0;
 723 }
 724
 725 static int __spi_unmap_msg(struct spi_master *master, struct spi_message *msg)
 726 {
 727         struct spi_transfer *xfer;
 728         struct device *tx_dev, *rx_dev;
 729
 730         if (!master->cur_msg_mapped || !master->can_dma)
 731                 return 0;
 732
 733         if (master->dma_tx)
 734                 tx_dev = master->dma_tx->device->dev;
 735         else
 736                 tx_dev = &master->dev;
 737
 738         if (master->dma_rx)
 739                 rx_dev = master->dma_rx->device->dev;
 740         else
 741                 rx_dev = &master->dev;
 742
 743         list_for_each_entry(xfer, &msg->transfers, transfer_list) {
 744                 if (!master->can_dma(master, msg->spi, xfer))
 745                         continue;
 746
 747                 spi_unmap_buf(master, rx_dev, &xfer->rx_sg, DMA_FROM_DEVICE);
 748                 spi_unmap_buf(master, tx_dev, &xfer->tx_sg, DMA_TO_DEVICE);
 749         }
 750
 751         return 0;
 752 }
 753 #else /* !CONFIG_HAS_DMA */
 754 static inline int __spi_map_msg(struct spi_master *master,
 755                                 struct spi_message *msg)
 756 {
 757         return 0;
 758 }
 759
 760 static inline int __spi_unmap_msg(struct spi_master *master,
 761                                   struct spi_message *msg)
 762 {
 763         return 0;
 764 }
 765 #endif /* !CONFIG_HAS_DMA */
 766
 767 static inline int spi_unmap_msg(struct spi_master *master,
 768                                 struct spi_message *msg)
 769 {
 770         struct spi_transfer *xfer;
 771
 772         list_for_each_entry(xfer, &msg->transfers, transfer_list) {
 773                 /*
 774                  * Restore the original value of tx_buf or rx_buf if they are
 775                  * NULL.
 776                  */
 777                 if (xfer->tx_buf == master->dummy_tx)
 778                         xfer->tx_buf = NULL;
 779                 if (xfer->rx_buf == master->dummy_rx)
 780                         xfer->rx_buf = NULL;
 781         }
 782
 783         return __spi_unmap_msg(master, msg);
 784 }
 785
 786 static int spi_map_msg(struct spi_master *master, struct spi_message *msg)
 787 {
 788         struct spi_transfer *xfer;
 789         void *tmp;
 790         unsigned int max_tx, max_rx;
 791
 792         if (master->flags & (SPI_MASTER_MUST_RX | SPI_MASTER_MUST_TX)) {
 793                 max_tx = 0;
 794                 max_rx = 0;
 795
 796                 list_for_each_entry(xfer, &msg->transfers, transfer_list) {
 797                         if ((master->flags & SPI_MASTER_MUST_TX) &&
 798                             !xfer->tx_buf)
 799                                 max_tx = max(xfer->len, max_tx);
 800                         if ((master->flags & SPI_MASTER_MUST_RX) &&
 801                             !xfer->rx_buf)
 802                                 max_rx = max(xfer->len, max_rx);
 803                 }
 804
 805                 if (max_tx) {
 806                         tmp = krealloc(master->dummy_tx, max_tx,
 807                                        GFP_KERNEL | GFP_DMA);
 808                         if (!tmp)
 809                                 return -ENOMEM;
 810                         master->dummy_tx = tmp;
 811                         memset(tmp, 0, max_tx);
 812                 }
 813
 814                 if (max_rx) {
 815                         tmp = krealloc(master->dummy_rx, max_rx,
 816                                        GFP_KERNEL | GFP_DMA);
 817                         if (!tmp)
 818                                 return -ENOMEM;
 819                         master->dummy_rx = tmp;
 820                 }
 821
 822                 if (max_tx || max_rx) {
 823                         list_for_each_entry(xfer, &msg->transfers,
 824                                             transfer_list) {
 825                                 if (!xfer->tx_buf)
 826                                         xfer->tx_buf = master->dummy_tx;
 827                                 if (!xfer->rx_buf)
 828                                         xfer->rx_buf = master->dummy_rx;
 829                         }
 830                 }
 831         }
 832
 833         return __spi_map_msg(master, msg);
 834 }
 835
 836 /*
 837  * spi_transfer_one_message - Default implementation of transfer_one_message()
 838  *
 839  * This is a standard implementation of transfer_one_message() for
 840  * drivers which impelment a transfer_one() operation.  It provides
 841  * standard handling of delays and chip select management.
 842  */
 843 static int spi_transfer_one_message(struct spi_master *master,
 844                                     struct spi_message *msg)
 845 {
 846         struct spi_transfer *xfer;
 847         bool keep_cs = false;
 848         int ret = 0;
 849         unsigned long ms = 1;
 850         struct spi_statistics *statm = &master->statistics;
 851         struct spi_statistics *stats = &msg->spi->statistics;
 852
 853         spi_set_cs(msg->spi, true);
 854
 855         SPI_STATISTICS_INCREMENT_FIELD(statm, messages);
 856         SPI_STATISTICS_INCREMENT_FIELD(stats, messages);
 857
 858         list_for_each_entry(xfer, &msg->transfers, transfer_list) {
 859                 trace_spi_transfer_start(msg, xfer);
 860
 861                 spi_statistics_add_transfer_stats(statm, xfer, master);
 862                 spi_statistics_add_transfer_stats(stats, xfer, master);
 863
 864                 if (xfer->tx_buf || xfer->rx_buf) {
 865                         reinit_completion(&master->xfer_completion);
 866
 867                         ret = master->transfer_one(master, msg->spi, xfer);
 868                         if (ret < 0) {
 869                                 SPI_STATISTICS_INCREMENT_FIELD(statm,
 870                                                                errors);
 871                                 SPI_STATISTICS_INCREMENT_FIELD(stats,
 872                                                                errors);
 873                                 dev_err(&msg->spi->dev,
 874                                         "SPI transfer failed: %d\n", ret);
 875                                 goto out;
 876                         }
 877
 878                         if (ret > 0) {
 879                                 ret = 0;
 880                                 ms = xfer->len * 8 * 1000 / xfer->speed_hz;
 881                                 ms += ms + 100; /* some tolerance */
 882
 883                                 ms = wait_for_completion_timeout(&master->xfer_completion,
 884                                                                  msecs_to_jiffies(ms));
 885                         }
 886
 887                         if (ms == 0) {
 888                                 SPI_STATISTICS_INCREMENT_FIELD(statm,
 889                                                                timedout);
 890                                 SPI_STATISTICS_INCREMENT_FIELD(stats,
 891                                                                timedout);
 892                                 dev_err(&msg->spi->dev,
 893                                         "SPI transfer timed out\n");
 894                                 msg->status = -ETIMEDOUT;
 895                         }
 896                 } else {
 897                         if (xfer->len)
 898                                 dev_err(&msg->spi->dev,
 899                                         "Bufferless transfer has length %u\n",
 900                                         xfer->len);
 901                 }
 902
 903                 trace_spi_transfer_stop(msg, xfer);
 904
 905                 if (msg->status != -EINPROGRESS)
 906                         goto out;
 907
 908                 if (xfer->delay_usecs)
 909                         udelay(xfer->delay_usecs);
 910
 911                 if (xfer->cs_change) {
 912                         if (list_is_last(&xfer->transfer_list,
 913                                          &msg->transfers)) {
 914                                 keep_cs = true;
 915                         } else {
 916                                 spi_set_cs(msg->spi, false);
 917                                 udelay(10);
 918                                 spi_set_cs(msg->spi, true);
 919                         }
 920                 }
 921
 922                 msg->actual_length += xfer->len;
 923         }
 924
 925 out:
 926         if (ret != 0 || !keep_cs)
 927                 spi_set_cs(msg->spi, false);
 928
 929         if (msg->status == -EINPROGRESS)
 930                 msg->status = ret;
 931
 932         if (msg->status && master->handle_err)
 933                 master->handle_err(master, msg);
 934
 935         spi_finalize_current_message(master);
 936
 937         return ret;
 938 }
 939
 940 /**
 941  * spi_finalize_current_transfer - report completion of a transfer
 942  * @master: the master reporting completion
 943  *
 944  * Called by SPI drivers using the core transfer_one_message()
 945  * implementation to notify it that the current interrupt driven
 946  * transfer has finished and the next one may be scheduled.
 947  */
 948 void spi_finalize_current_transfer(struct spi_master *master)
 949 {
 950         complete(&master->xfer_completion);
 951 }
 952 EXPORT_SYMBOL_GPL(spi_finalize_current_transfer);
 953
 954 /**
 955  * __spi_pump_messages - function which processes spi message queue
 956  * @master: master to process queue for
 957  * @in_kthread: true if we are in the context of the message pump thread
 958  *
 959  * This function checks if there is any spi message in the queue that
 960  * needs processing and if so call out to the driver to initialize hardware
 961  * and transfer each message.
 962  *
 963  * Note that it is called both from the kthread itself and also from
 964  * inside spi_sync(); the queue extraction handling at the top of the
 965  * function should deal with this safely.
 966  */
 967 static void __spi_pump_messages(struct spi_master *master, bool in_kthread)
 968 {
 969         unsigned long flags;
 970         bool was_busy = false;
 971         int ret;
 972
 973         /* Lock queue */
 974         spin_lock_irqsave(&master->queue_lock, flags);
 975
 976         /* Make sure we are not already running a message */
 977         if (master->cur_msg) {
 978                 spin_unlock_irqrestore(&master->queue_lock, flags);
 979                 return;
 980         }
 981
 982         /* If another context is idling the device then defer */
 983         if (master->idling) {
 984                 queue_kthread_work(&master->kworker, &master->pump_messages);
 985                 spin_unlock_irqrestore(&master->queue_lock, flags);
 986                 return;
 987         }
 988
 989         /* Check if the queue is idle */
 990         if (list_empty(&master->queue) || !master->running) {
 991                 if (!master->busy) {
 992                         spin_unlock_irqrestore(&master->queue_lock, flags);
 993                         return;
 994                 }
 995
 996                 /* Only do teardown in the thread */
 997                 if (!in_kthread) {
 998                         queue_kthread_work(&master->kworker,
 999                                            &master->pump_messages);
1000                         spin_unlock_irqrestore(&master->queue_lock, flags);
1001                         return;
1002                 }
1003
1004                 master->busy = false;
1005                 master->idling = true;
1006                 spin_unlock_irqrestore(&master->queue_lock, flags);
1007
1008                 kfree(master->dummy_rx);
1009                 master->dummy_rx = NULL;
1010                 kfree(master->dummy_tx);
1011                 master->dummy_tx = NULL;
1012                 if (master->unprepare_transfer_hardware &&
1013                     master->unprepare_transfer_hardware(master))
1014                         dev_err(&master->dev,
1015                                 "failed to unprepare transfer hardware\n");
1016                 if (master->auto_runtime_pm) {
1017                         pm_runtime_mark_last_busy(master->dev.parent);
1018                         pm_runtime_put_autosuspend(master->dev.parent);
1019                 }
1020                 trace_spi_master_idle(master);
1021
1022                 spin_lock_irqsave(&master->queue_lock, flags);
1023                 master->idling = false;
1024                 spin_unlock_irqrestore(&master->queue_lock, flags);
1025                 return;
1026         }
1027
1028         /* Extract head of queue */
1029         master->cur_msg =
1030                 list_first_entry(&master->queue, struct spi_message, queue);
1031
1032         list_del_init(&master->cur_msg->queue);
1033         if (master->busy)
1034                 was_busy = true;
1035         else
1036                 master->busy = true;
1037         spin_unlock_irqrestore(&master->queue_lock, flags);
1038
1039         if (!was_busy && master->auto_runtime_pm) {
1040                 ret = pm_runtime_get_sync(master->dev.parent);
1041                 if (ret < 0) {
1042                         dev_err(&master->dev, "Failed to power device: %d\n",
1043                                 ret);
1044                         return;
1045                 }
1046         }
1047
1048         if (!was_busy)
1049                 trace_spi_master_busy(master);
1050
1051         if (!was_busy && master->prepare_transfer_hardware) {
1052                 ret = master->prepare_transfer_hardware(master);
1053                 if (ret) {
1054                         dev_err(&master->dev,
1055                                 "failed to prepare transfer hardware\n");
1056
1057                         if (master->auto_runtime_pm)
1058                                 pm_runtime_put(master->dev.parent);
1059                         return;
1060                 }
1061         }
1062
1063         trace_spi_message_start(master->cur_msg);
1064
1065         if (master->prepare_message) {
1066                 ret = master->prepare_message(master, master->cur_msg);
1067                 if (ret) {
1068                         dev_err(&master->dev,
1069                                 "failed to prepare message: %d\n", ret);
1070                         master->cur_msg->status = ret;
1071                         spi_finalize_current_message(master);
1072                         return;
1073                 }
1074                 master->cur_msg_prepared = true;
1075         }
1076
1077         ret = spi_map_msg(master, master->cur_msg);
1078         if (ret) {
1079                 master->cur_msg->status = ret;
1080                 spi_finalize_current_message(master);
1081                 return;
1082         }
1083
1084         ret = master->transfer_one_message(master, master->cur_msg);
1085         if (ret) {
1086                 dev_err(&master->dev,
1087                         "failed to transfer one message from queue\n");
1088                 return;
1089         }
1090 }
1091
1092 /**
1093  * spi_pump_messages - kthread work function which processes spi message queue
1094  * @work: pointer to kthread work struct contained in the master struct
1095  */
1096 static void spi_pump_messages(struct kthread_work *work)
1097 {
1098         struct spi_master *master =
1099                 container_of(work, struct spi_master, pump_messages);
1100
1101         __spi_pump_messages(master, true);
1102 }
1103
1104 static int spi_init_queue(struct spi_master *master)
1105 {
1106         struct sched_param param = { .sched_priority = MAX_RT_PRIO - 1 };
1107
1108         master->running = false;
1109         master->busy = false;
1110
1111         init_kthread_worker(&master->kworker);
1112         master->kworker_task = kthread_run(kthread_worker_fn,
1113                                            &master->kworker, "%s",
1114                                            dev_name(&master->dev));
1115         if (IS_ERR(master->kworker_task)) {
1116                 dev_err(&master->dev, "failed to create message pump task\n");
1117                 return PTR_ERR(master->kworker_task);
1118         }
1119         init_kthread_work(&master->pump_messages, spi_pump_messages);
1120
1121         /*
1122          * Master config will indicate if this controller should run the
1123          * message pump with high (realtime) priority to reduce the transfer
1124          * latency on the bus by minimising the delay between a transfer
1125          * request and the scheduling of the message pump thread. Without this
1126          * setting the message pump thread will remain at default priority.
1127          */
1128         if (master->rt) {
1129                 dev_info(&master->dev,
1130                         "will run message pump with realtime priority\n");
1131                 sched_setscheduler(master->kworker_task, SCHED_FIFO, &param);
1132         }
1133
1134         return 0;
1135 }
1136
1137 /**
1138  * spi_get_next_queued_message() - called by driver to check for queued
1139  * messages
1140  * @master: the master to check for queued messages
1141  *
1142  * If there are more messages in the queue, the next message is returned from
1143  * this call.
1144  */
1145 struct spi_message *spi_get_next_queued_message(struct spi_master *master)
1146 {
1147         struct spi_message *next;
1148         unsigned long flags;
1149
1150         /* get a pointer to the next message, if any */
1151         spin_lock_irqsave(&master->queue_lock, flags);
1152         next = list_first_entry_or_null(&master->queue, struct spi_message,
1153                                         queue);
1154         spin_unlock_irqrestore(&master->queue_lock, flags);
1155
1156         return next;
1157 }
1158 EXPORT_SYMBOL_GPL(spi_get_next_queued_message);
1159
1160 /**
1161  * spi_finalize_current_message() - the current message is complete
1162  * @master: the master to return the message to
1163  *
1164  * Called by the driver to notify the core that the message in the front of the
1165  * queue is complete and can be removed from the queue.
1166  */
1167 void spi_finalize_current_message(struct spi_master *master)
1168 {
1169         struct spi_message *mesg;
1170         unsigned long flags;
1171         int ret;
1172
1173         spin_lock_irqsave(&master->queue_lock, flags);
1174         mesg = master->cur_msg;
1175         spin_unlock_irqrestore(&master->queue_lock, flags);
1176
1177         spi_unmap_msg(master, mesg);
1178
1179         if (master->cur_msg_prepared && master->unprepare_message) {
1180                 ret = master->unprepare_message(master, mesg);
1181                 if (ret) {
1182                         dev_err(&master->dev,
1183                                 "failed to unprepare message: %d\n", ret);
1184                 }
1185         }
1186
1187         spin_lock_irqsave(&master->queue_lock, flags);
1188         master->cur_msg = NULL;
1189         master->cur_msg_prepared = false;
1190         queue_kthread_work(&master->kworker, &master->pump_messages);
1191         spin_unlock_irqrestore(&master->queue_lock, flags);
1192
1193         trace_spi_message_done(mesg);
1194
1195         mesg->state = NULL;
1196         if (mesg->complete)
1197                 mesg->complete(mesg->context);
1198 }
1199 EXPORT_SYMBOL_GPL(spi_finalize_current_message);
1200
1201 static int spi_start_queue(struct spi_master *master)
1202 {
1203         unsigned long flags;
1204
1205         spin_lock_irqsave(&master->queue_lock, flags);
1206
1207         if (master->running || master->busy) {
1208                 spin_unlock_irqrestore(&master->queue_lock, flags);
1209                 return -EBUSY;
1210         }
1211
1212         master->running = true;
1213         master->cur_msg = NULL;
1214         spin_unlock_irqrestore(&master->queue_lock, flags);
1215
1216         queue_kthread_work(&master->kworker, &master->pump_messages);
1217
1218         return 0;
1219 }
1220
1221 static int spi_stop_queue(struct spi_master *master)
1222 {
1223         unsigned long flags;
1224         unsigned limit = 500;
1225         int ret = 0;
1226
1227         spin_lock_irqsave(&master->queue_lock, flags);
1228
1229         /*
1230          * This is a bit lame, but is optimized for the common execution path.
1231          * A wait_queue on the master->busy could be used, but then the common
1232          * execution path (pump_messages) would be required to call wake_up or
1233          * friends on every SPI message. Do this instead.
1234          */
1235         while ((!list_empty(&master->queue) || master->busy) && limit--) {
1236                 spin_unlock_irqrestore(&master->queue_lock, flags);
1237                 usleep_range(10000, 11000);
1238                 spin_lock_irqsave(&master->queue_lock, flags);
1239         }
1240
1241         if (!list_empty(&master->queue) || master->busy)
1242                 ret = -EBUSY;
1243         else
1244                 master->running = false;
1245
1246         spin_unlock_irqrestore(&master->queue_lock, flags);
1247
1248         if (ret) {
1249                 dev_warn(&master->dev,
1250                          "could not stop message queue\n");
1251                 return ret;
1252         }
1253         return ret;
1254 }
1255
1256 static int spi_destroy_queue(struct spi_master *master)
1257 {
1258         int ret;
1259
1260         ret = spi_stop_queue(master);
1261
1262         /*
1263          * flush_kthread_worker will block until all work is done.
1264          * If the reason that stop_queue timed out is that the work will never
1265          * finish, then it does no good to call flush/stop thread, so
1266          * return anyway.
1267          */
1268         if (ret) {
1269                 dev_err(&master->dev, "problem destroying queue\n");
1270                 return ret;
1271         }
1272
1273         flush_kthread_worker(&master->kworker);
1274         kthread_stop(master->kworker_task);
1275
1276         return 0;
1277 }
1278
1279 static int __spi_queued_transfer(struct spi_device *spi,
1280                                  struct spi_message *msg,
1281                                  bool need_pump)
1282 {
1283         struct spi_master *master = spi->master;
1284         unsigned long flags;
1285
1286         spin_lock_irqsave(&master->queue_lock, flags);
1287
1288         if (!master->running) {
1289                 spin_unlock_irqrestore(&master->queue_lock, flags);
1290                 return -ESHUTDOWN;
1291         }
1292         msg->actual_length = 0;
1293         msg->status = -EINPROGRESS;
1294
1295         list_add_tail(&msg->queue, &master->queue);
1296         if (!master->busy && need_pump)
1297                 queue_kthread_work(&master->kworker, &master->pump_messages);
1298
1299         spin_unlock_irqrestore(&master->queue_lock, flags);
1300         return 0;
1301 }
1302
1303 /**
1304  * spi_queued_transfer - transfer function for queued transfers
1305  * @spi: spi device which is requesting transfer
1306  * @msg: spi message which is to handled is queued to driver queue
1307  */
1308 static int spi_queued_transfer(struct spi_device *spi, struct spi_message *msg)
1309 {
1310         return __spi_queued_transfer(spi, msg, true);
1311 }
1312
1313 static int spi_master_initialize_queue(struct spi_master *master)
1314 {
1315         int ret;
1316
1317         master->transfer = spi_queued_transfer;
1318         if (!master->transfer_one_message)
1319                 master->transfer_one_message = spi_transfer_one_message;
1320
1321         /* Initialize and start queue */
1322         ret = spi_init_queue(master);
1323         if (ret) {
1324                 dev_err(&master->dev, "problem initializing queue\n");
1325                 goto err_init_queue;
1326         }
1327         master->queued = true;
1328         ret = spi_start_queue(master);
1329         if (ret) {
1330                 dev_err(&master->dev, "problem starting queue\n");
1331                 goto err_start_queue;
1332         }
1333
1334         return 0;
1335
1336 err_start_queue:
1337         spi_destroy_queue(master);
1338 err_init_queue:
1339         return ret;
1340 }
1341
1342 /*-------------------------------------------------------------------------*/
1343
1344 #if defined(CONFIG_OF)
1345 static struct spi_device *
1346 of_register_spi_device(struct spi_master *master, struct device_node *nc)
1347 {
1348         struct spi_device *spi;
1349         int rc;
1350         u32 value;
1351
1352         /* Alloc an spi_device */
1353         spi = spi_alloc_device(master);
1354         if (!spi) {
1355                 dev_err(&master->dev, "spi_device alloc error for %s\n",
1356                         nc->full_name);
1357                 rc = -ENOMEM;
1358                 goto err_out;
1359         }
1360
1361         /* Select device driver */
1362         rc = of_modalias_node(nc, spi->modalias,
1363                                 sizeof(spi->modalias));
1364         if (rc < 0) {
1365                 dev_err(&master->dev, "cannot find modalias for %s\n",
1366                         nc->full_name);
1367                 goto err_out;
1368         }
1369
1370         /* Device address */
1371         rc = of_property_read_u32(nc, "reg", &value);
1372         if (rc) {
1373                 dev_err(&master->dev, "%s has no valid 'reg' property (%d)\n",
1374                         nc->full_name, rc);
1375                 goto err_out;
1376         }
1377         spi->chip_select = value;
1378
1379         /* Mode (clock phase/polarity/etc.) */
1380         if (of_find_property(nc, "spi-cpha", NULL))
1381                 spi->mode |= SPI_CPHA;
1382         if (of_find_property(nc, "spi-cpol", NULL))
1383                 spi->mode |= SPI_CPOL;
1384         if (of_find_property(nc, "spi-cs-high", NULL))
1385                 spi->mode |= SPI_CS_HIGH;
1386         if (of_find_property(nc, "spi-3wire", NULL))
1387                 spi->mode |= SPI_3WIRE;
1388         if (of_find_property(nc, "spi-lsb-first", NULL))
1389                 spi->mode |= SPI_LSB_FIRST;
1390
1391         /* Device DUAL/QUAD mode */
1392         if (!of_property_read_u32(nc, "spi-tx-bus-width", &value)) {
1393                 switch (value) {
1394                 case 1:
1395                         break;
1396                 case 2:
1397                         spi->mode |= SPI_TX_DUAL;
1398                         break;
1399                 case 4:
1400                         spi->mode |= SPI_TX_QUAD;
1401                         break;
1402                 default:
1403                         dev_warn(&master->dev,
1404                                 "spi-tx-bus-width %d not supported\n",
1405                                 value);
1406                         break;
1407                 }
1408         }
1409
1410         if (!of_property_read_u32(nc, "spi-rx-bus-width", &value)) {
1411                 switch (value) {
1412                 case 1:
1413                         break;
1414                 case 2:
1415                         spi->mode |= SPI_RX_DUAL;
1416                         break;
1417                 case 4:
1418                         spi->mode |= SPI_RX_QUAD;
1419                         break;
1420                 default:
1421                         dev_warn(&master->dev,
1422                                 "spi-rx-bus-width %d not supported\n",
1423                                 value);
1424                         break;
1425                 }
1426         }
1427
1428         /* Device speed */
1429         rc = of_property_read_u32(nc, "spi-max-frequency", &value);
1430         if (rc) {
1431                 dev_err(&master->dev, "%s has no valid 'spi-max-frequency' property (%d)\n",
1432                         nc->full_name, rc);
1433                 goto err_out;
1434         }
1435         spi->max_speed_hz = value;
1436
1437         /* IRQ */
1438         spi->irq = irq_of_parse_and_map(nc, 0);
1439
1440         /* Store a pointer to the node in the device structure */
1441         of_node_get(nc);
1442         spi->dev.of_node = nc;
1443
1444         /* Register the new device */
1445         rc = spi_add_device(spi);
1446         if (rc) {
1447                 dev_err(&master->dev, "spi_device register error %s\n",
1448                         nc->full_name);
1449                 goto err_out;
1450         }
1451
1452         return spi;
1453
1454 err_out:
1455         spi_dev_put(spi);
1456         return ERR_PTR(rc);
1457 }
1458
1459 /**
1460  * of_register_spi_devices() - Register child devices onto the SPI bus
1461  * @master:     Pointer to spi_master device
1462  *
1463  * Registers an spi_device for each child node of master node which has a 'reg'
1464  * property.
1465  */
1466 static void of_register_spi_devices(struct spi_master *master)
1467 {
1468         struct spi_device *spi;
1469         struct device_node *nc;
1470
1471         if (!master->dev.of_node)
1472                 return;
1473
1474         for_each_available_child_of_node(master->dev.of_node, nc) {
1475                 spi = of_register_spi_device(master, nc);
1476                 if (IS_ERR(spi))
1477                         dev_warn(&master->dev, "Failed to create SPI device for %s\n",
1478                                 nc->full_name);
1479         }
1480 }
1481 #else
1482 static void of_register_spi_devices(struct spi_master *master) { }
1483 #endif
1484
1485 #ifdef CONFIG_ACPI
1486 static int acpi_spi_add_resource(struct acpi_resource *ares, void *data)
1487 {
1488         struct spi_device *spi = data;
1489
1490         if (ares->type == ACPI_RESOURCE_TYPE_SERIAL_BUS) {
1491                 struct acpi_resource_spi_serialbus *sb;
1492
1493                 sb = &ares->data.spi_serial_bus;
1494                 if (sb->type == ACPI_RESOURCE_SERIAL_TYPE_SPI) {
1495                         spi->chip_select = sb->device_selection;
1496                         spi->max_speed_hz = sb->connection_speed;
1497
1498                         if (sb->clock_phase == ACPI_SPI_SECOND_PHASE)
1499                                 spi->mode |= SPI_CPHA;
1500                         if (sb->clock_polarity == ACPI_SPI_START_HIGH)
1501                                 spi->mode |= SPI_CPOL;
1502                         if (sb->device_polarity == ACPI_SPI_ACTIVE_HIGH)
1503                                 spi->mode |= SPI_CS_HIGH;
1504                 }
1505         } else if (spi->irq < 0) {
1506                 struct resource r;
1507
1508                 if (acpi_dev_resource_interrupt(ares, 0, &r))
1509                         spi->irq = r.start;
1510         }
1511
1512         /* Always tell the ACPI core to skip this resource */
1513         return 1;
1514 }
1515
1516 static acpi_status acpi_spi_add_device(acpi_handle handle, u32 level,
1517                                        void *data, void **return_value)
1518 {
1519         struct spi_master *master = data;
1520         struct list_head resource_list;
1521         struct acpi_device *adev;
1522         struct spi_device *spi;
1523         int ret;
1524
1525         if (acpi_bus_get_device(handle, &adev))
1526                 return AE_OK;
1527         if (acpi_bus_get_status(adev) || !adev->status.present)
1528                 return AE_OK;
1529
1530         spi = spi_alloc_device(master);
1531         if (!spi) {
1532                 dev_err(&master->dev, "failed to allocate SPI device for %s\n",
1533                         dev_name(&adev->dev));
1534                 return AE_NO_MEMORY;
1535         }
1536
1537         ACPI_COMPANION_SET(&spi->dev, adev);
1538         spi->irq = -1;
1539
1540         INIT_LIST_HEAD(&resource_list);
1541         ret = acpi_dev_get_resources(adev, &resource_list,
1542                                      acpi_spi_add_resource, spi);
1543         acpi_dev_free_resource_list(&resource_list);
1544
1545         if (ret < 0 || !spi->max_speed_hz) {
1546                 spi_dev_put(spi);
1547                 return AE_OK;
1548         }
1549
1550         adev->power.flags.ignore_parent = true;
1551         strlcpy(spi->modalias, acpi_device_hid(adev), sizeof(spi->modalias));
1552         if (spi_add_device(spi)) {
1553                 adev->power.flags.ignore_parent = false;
1554                 dev_err(&master->dev, "failed to add SPI device %s from ACPI\n",
1555                         dev_name(&adev->dev));
1556                 spi_dev_put(spi);
1557         }
1558
1559         return AE_OK;
1560 }
1561
1562 static void acpi_register_spi_devices(struct spi_master *master)
1563 {
1564         acpi_status status;
1565         acpi_handle handle;
1566
1567         handle = ACPI_HANDLE(master->dev.parent);
1568         if (!handle)
1569                 return;
1570
1571         status = acpi_walk_namespace(ACPI_TYPE_DEVICE, handle, 1,
1572                                      acpi_spi_add_device, NULL,
1573                                      master, NULL);
1574         if (ACPI_FAILURE(status))
1575                 dev_warn(&master->dev, "failed to enumerate SPI slaves\n");
1576 }
1577 #else
1578 static inline void acpi_register_spi_devices(struct spi_master *master) {}
1579 #endif /* CONFIG_ACPI */
1580
1581 static void spi_master_release(struct device *dev)
1582 {
1583         struct spi_master *master;
1584
1585         master = container_of(dev, struct spi_master, dev);
1586         kfree(master);
1587 }
1588
1589 static struct class spi_master_class = {
1590         .name           = "spi_master",
1591         .owner          = THIS_MODULE,
1592         .dev_release    = spi_master_release,
1593         .dev_groups     = spi_master_groups,
1594 };
1595
1596
1597 /**
1598  * spi_alloc_master - allocate SPI master controller
1599  * @dev: the controller, possibly using the platform_bus
1600  * @size: how much zeroed driver-private data to allocate; the pointer to this
1601  *      memory is in the driver_data field of the returned device,
1602  *      accessible with spi_master_get_devdata().
1603  * Context: can sleep
1604  *
1605  * This call is used only by SPI master controller drivers, which are the
1606  * only ones directly touching chip registers.  It's how they allocate
1607  * an spi_master structure, prior to calling spi_register_master().
1608  *
1609  * This must be called from context that can sleep.  It returns the SPI
1610  * master structure on success, else NULL.
1611  *
1612  * The caller is responsible for assigning the bus number and initializing
1613  * the master's methods before calling spi_register_master(); and (after errors
1614  * adding the device) calling spi_master_put() and kfree() to prevent a memory
1615  * leak.
1616  */
1617 struct spi_master *spi_alloc_master(struct device *dev, unsigned size)
1618 {
1619         struct spi_master       *master;
1620
1621         if (!dev)
1622                 return NULL;
1623
1624         master = kzalloc(size + sizeof(*master), GFP_KERNEL);
1625         if (!master)
1626                 return NULL;
1627
1628         device_initialize(&master->dev);
1629         master->bus_num = -1;
1630         master->num_chipselect = 1;
1631         master->dev.class = &spi_master_class;
1632         master->dev.parent = get_device(dev);
1633         spi_master_set_devdata(master, &master[1]);
1634
1635         return master;
1636 }
1637 EXPORT_SYMBOL_GPL(spi_alloc_master);
1638
1639 #ifdef CONFIG_OF
1640 static int of_spi_register_master(struct spi_master *master)
1641 {
1642         int nb, i, *cs;
1643         struct device_node *np = master->dev.of_node;
1644
1645         if (!np)
1646                 return 0;
1647
1648         nb = of_gpio_named_count(np, "cs-gpios");
1649         master->num_chipselect = max_t(int, nb, master->num_chipselect);
1650
1651         /* Return error only for an incorrectly formed cs-gpios property */
1652         if (nb == 0 || nb == -ENOENT)
1653                 return 0;
1654         else if (nb < 0)
1655                 return nb;
1656
1657         cs = devm_kzalloc(&master->dev,
1658                           sizeof(int) * master->num_chipselect,
1659                           GFP_KERNEL);
1660         master->cs_gpios = cs;
1661
1662         if (!master->cs_gpios)
1663                 return -ENOMEM;
1664
1665         for (i = 0; i < master->num_chipselect; i++)
1666                 cs[i] = -ENOENT;
1667
1668         for (i = 0; i < nb; i++)
1669                 cs[i] = of_get_named_gpio(np, "cs-gpios", i);
1670
1671         return 0;
1672 }
1673 #else
1674 static int of_spi_register_master(struct spi_master *master)
1675 {
1676         return 0;
1677 }
1678 #endif
1679
1680 /**
1681  * spi_register_master - register SPI master controller
1682  * @master: initialized master, originally from spi_alloc_master()
1683  * Context: can sleep
1684  *
1685  * SPI master controllers connect to their drivers using some non-SPI bus,
1686  * such as the platform bus.  The final stage of probe() in that code
1687  * includes calling spi_register_master() to hook up to this SPI bus glue.
1688  *
1689  * SPI controllers use board specific (often SOC specific) bus numbers,
1690  * and board-specific addressing for SPI devices combines those numbers
1691  * with chip select numbers.  Since SPI does not directly support dynamic
1692  * device identification, boards need configuration tables telling which
1693  * chip is at which address.
1694  *
1695  * This must be called from context that can sleep.  It returns zero on
1696  * success, else a negative error code (dropping the master's refcount).
1697  * After a successful return, the caller is responsible for calling
1698  * spi_unregister_master().
1699  */
1700 int spi_register_master(struct spi_master *master)
1701 {
1702         static atomic_t         dyn_bus_id = ATOMIC_INIT((1<<15) - 1);
1703         struct device           *dev = master->dev.parent;
1704         struct boardinfo        *bi;
1705         int                     status = -ENODEV;
1706         int                     dynamic = 0;
1707
1708         if (!dev)
1709                 return -ENODEV;
1710
1711         status = of_spi_register_master(master);
1712         if (status)
1713                 return status;
1714
1715         /* even if it's just one always-selected device, there must
1716          * be at least one chipselect
1717          */
1718         if (master->num_chipselect == 0)
1719                 return -EINVAL;
1720
1721         if ((master->bus_num < 0) && master->dev.of_node)
1722                 master->bus_num = of_alias_get_id(master->dev.of_node, "spi");
1723
1724         /* convention:  dynamically assigned bus IDs count down from the max */
1725         if (master->bus_num < 0) {
1726                 /* FIXME switch to an IDR based scheme, something like
1727                  * I2C now uses, so we can't run out of "dynamic" IDs
1728                  */
1729                 master->bus_num = atomic_dec_return(&dyn_bus_id);
1730                 dynamic = 1;
1731         }
1732
1733         INIT_LIST_HEAD(&master->queue);
1734         spin_lock_init(&master->queue_lock);
1735         spin_lock_init(&master->bus_lock_spinlock);
1736         mutex_init(&master->bus_lock_mutex);
1737         master->bus_lock_flag = 0;
1738         init_completion(&master->xfer_completion);
1739         if (!master->max_dma_len)
1740                 master->max_dma_len = INT_MAX;
1741
1742         /* register the device, then userspace will see it.
1743          * registration fails if the bus ID is in use.
1744          */
1745         dev_set_name(&master->dev, "spi%u", master->bus_num);
1746         status = device_add(&master->dev);
1747         if (status < 0)
1748                 goto done;
1749         dev_dbg(dev, "registered master %s%s\n", dev_name(&master->dev),
1750                         dynamic ? " (dynamic)" : "");
1751
1752         /* If we're using a queued driver, start the queue */
1753         if (master->transfer)
1754                 dev_info(dev, "master is unqueued, this is deprecated\n");
1755         else {
1756                 status = spi_master_initialize_queue(master);
1757                 if (status) {
1758                         device_del(&master->dev);
1759                         goto done;
1760                 }
1761         }
1762         /* add statistics */
1763         spin_lock_init(&master->statistics.lock);
1764
1765         mutex_lock(&board_lock);
1766         list_add_tail(&master->list, &spi_master_list);
1767         list_for_each_entry(bi, &board_list, list)
1768                 spi_match_master_to_boardinfo(master, &bi->board_info);
1769         mutex_unlock(&board_lock);
1770
1771         /* Register devices from the device tree and ACPI */
1772         of_register_spi_devices(master);
1773         acpi_register_spi_devices(master);
1774 done:
1775         return status;
1776 }
1777 EXPORT_SYMBOL_GPL(spi_register_master);
1778
1779 static void devm_spi_unregister(struct device *dev, void *res)
1780 {
1781         spi_unregister_master(*(struct spi_master **)res);
1782 }
1783
1784 /**
1785  * dev_spi_register_master - register managed SPI master controller
1786  * @dev:    device managing SPI master
1787  * @master: initialized master, originally from spi_alloc_master()
1788  * Context: can sleep
1789  *
1790  * Register a SPI device as with spi_register_master() which will
1791  * automatically be unregister
1792  */
1793 int devm_spi_register_master(struct device *dev, struct spi_master *master)
1794 {
1795         struct spi_master **ptr;
1796         int ret;
1797
1798         ptr = devres_alloc(devm_spi_unregister, sizeof(*ptr), GFP_KERNEL);
1799         if (!ptr)
1800                 return -ENOMEM;
1801
1802         ret = spi_register_master(master);
1803         if (!ret) {
1804                 *ptr = master;
1805                 devres_add(dev, ptr);
1806         } else {
1807                 devres_free(ptr);
1808         }
1809
1810         return ret;
1811 }
1812 EXPORT_SYMBOL_GPL(devm_spi_register_master);
1813
1814 static int __unregister(struct device *dev, void *null)
1815 {
1816         spi_unregister_device(to_spi_device(dev));
1817         return 0;
1818 }
1819
1820 /**
1821  * spi_unregister_master - unregister SPI master controller
1822  * @master: the master being unregistered
1823  * Context: can sleep
1824  *
1825  * This call is used only by SPI master controller drivers, which are the
1826  * only ones directly touching chip registers.
1827  *
1828  * This must be called from context that can sleep.
1829  */
1830 void spi_unregister_master(struct spi_master *master)
1831 {
1832         int dummy;
1833
1834         if (master->queued) {
1835                 if (spi_destroy_queue(master))
1836                         dev_err(&master->dev, "queue remove failed\n");
1837         }
1838
1839         mutex_lock(&board_lock);
1840         list_del(&master->list);
1841         mutex_unlock(&board_lock);
1842
1843         dummy = device_for_each_child(&master->dev, NULL, __unregister);
1844         device_unregister(&master->dev);
1845 }
1846 EXPORT_SYMBOL_GPL(spi_unregister_master);
1847
1848 int spi_master_suspend(struct spi_master *master)
1849 {
1850         int ret;
1851
1852         /* Basically no-ops for non-queued masters */
1853         if (!master->queued)
1854                 return 0;
1855
1856         ret = spi_stop_queue(master);
1857         if (ret)
1858                 dev_err(&master->dev, "queue stop failed\n");
1859
1860         return ret;
1861 }
1862 EXPORT_SYMBOL_GPL(spi_master_suspend);
1863
1864 int spi_master_resume(struct spi_master *master)
1865 {
1866         int ret;
1867
1868         if (!master->queued)
1869                 return 0;
1870
1871         ret = spi_start_queue(master);
1872         if (ret)
1873                 dev_err(&master->dev, "queue restart failed\n");
1874
1875         return ret;
1876 }
1877 EXPORT_SYMBOL_GPL(spi_master_resume);
1878
1879 static int __spi_master_match(struct device *dev, const void *data)
1880 {
1881         struct spi_master *m;
1882         const u16 *bus_num = data;
1883
1884         m = container_of(dev, struct spi_master, dev);
1885         return m->bus_num == *bus_num;
1886 }
1887
1888 /**
1889  * spi_busnum_to_master - look up master associated with bus_num
1890  * @bus_num: the master's bus number
1891  * Context: can sleep
1892  *
1893  * This call may be used with devices that are registered after
1894  * arch init time.  It returns a refcounted pointer to the relevant
1895  * spi_master (which the caller must release), or NULL if there is
1896  * no such master registered.
1897  */
1898 struct spi_master *spi_busnum_to_master(u16 bus_num)
1899 {
1900         struct device           *dev;
1901         struct spi_master       *master = NULL;
1902
1903         dev = class_find_device(&spi_master_class, NULL, &bus_num,
1904                                 __spi_master_match);
1905         if (dev)
1906                 master = container_of(dev, struct spi_master, dev);
1907         /* reference got in class_find_device */
1908         return master;
1909 }
1910 EXPORT_SYMBOL_GPL(spi_busnum_to_master);
1911
1912
1913 /*-------------------------------------------------------------------------*/
1914
1915 /* Core methods for SPI master protocol drivers.  Some of the
1916  * other core methods are currently defined as inline functions.
1917  */
1918
1919 static int __spi_validate_bits_per_word(struct spi_master *master, u8 bits_per_word)
1920 {
1921         if (master->bits_per_word_mask) {
1922                 /* Only 32 bits fit in the mask */
1923                 if (bits_per_word > 32)
1924                         return -EINVAL;
1925                 if (!(master->bits_per_word_mask &
1926                                 SPI_BPW_MASK(bits_per_word)))
1927                         return -EINVAL;
1928         }
1929
1930         return 0;
1931 }
1932
1933 /**
1934  * spi_setup - setup SPI mode and clock rate
1935  * @spi: the device whose settings are being modified
1936  * Context: can sleep, and no requests are queued to the device
1937  *
1938  * SPI protocol drivers may need to update the transfer mode if the
1939  * device doesn't work with its default.  They may likewise need
1940  * to update clock rates or word sizes from initial values.  This function
1941  * changes those settings, and must be called from a context that can sleep.
1942  * Except for SPI_CS_HIGH, which takes effect immediately, the changes take
1943  * effect the next time the device is selected and data is transferred to
1944  * or from it.  When this function returns, the spi device is deselected.
1945  *
1946  * Note that this call will fail if the protocol driver specifies an option
1947  * that the underlying controller or its driver does not support.  For
1948  * example, not all hardware supports wire transfers using nine bit words,
1949  * LSB-first wire encoding, or active-high chipselects.
1950  */
1951 int spi_setup(struct spi_device *spi)
1952 {
1953         unsigned        bad_bits, ugly_bits;
1954         int             status = 0;
1955
1956         /* check mode to prevent that DUAL and QUAD set at the same time
1957          */
1958         if (((spi->mode & SPI_TX_DUAL) && (spi->mode & SPI_TX_QUAD)) ||
1959                 ((spi->mode & SPI_RX_DUAL) && (spi->mode & SPI_RX_QUAD))) {
1960                 dev_err(&spi->dev,
1961                 "setup: can not select dual and quad at the same time\n");
1962                 return -EINVAL;
1963         }
1964         /* if it is SPI_3WIRE mode, DUAL and QUAD should be forbidden
1965          */
1966         if ((spi->mode & SPI_3WIRE) && (spi->mode &
1967                 (SPI_TX_DUAL | SPI_TX_QUAD | SPI_RX_DUAL | SPI_RX_QUAD)))
1968                 return -EINVAL;
1969         /* help drivers fail *cleanly* when they need options
1970          * that aren't supported with their current master
1971          */
1972         bad_bits = spi->mode & ~spi->master->mode_bits;
1973         ugly_bits = bad_bits &
1974                     (SPI_TX_DUAL | SPI_TX_QUAD | SPI_RX_DUAL | SPI_RX_QUAD);
1975         if (ugly_bits) {
1976                 dev_warn(&spi->dev,
1977                          "setup: ignoring unsupported mode bits %x\n",
1978                          ugly_bits);
1979                 spi->mode &= ~ugly_bits;
1980                 bad_bits &= ~ugly_bits;
1981         }
1982         if (bad_bits) {
1983                 dev_err(&spi->dev, "setup: unsupported mode bits %x\n",
1984                         bad_bits);
1985                 return -EINVAL;
1986         }
1987
1988         if (!spi->bits_per_word)
1989                 spi->bits_per_word = 8;
1990
1991         if (__spi_validate_bits_per_word(spi->master, spi->bits_per_word))
1992                 return -EINVAL;
1993
1994         if (!spi->max_speed_hz)
1995                 spi->max_speed_hz = spi->master->max_speed_hz;
1996
1997         spi_set_cs(spi, false);
1998
1999         if (spi->master->setup)
2000                 status = spi->master->setup(spi);
2001
2002         dev_dbg(&spi->dev, "setup mode %d, %s%s%s%s%u bits/w, %u Hz max --> %d\n",
2003                         (int) (spi->mode & (SPI_CPOL | SPI_CPHA)),
2004                         (spi->mode & SPI_CS_HIGH) ? "cs_high, " : "",
2005                         (spi->mode & SPI_LSB_FIRST) ? "lsb, " : "",
2006                         (spi->mode & SPI_3WIRE) ? "3wire, " : "",
2007                         (spi->mode & SPI_LOOP) ? "loopback, " : "",
2008                         spi->bits_per_word, spi->max_speed_hz,
2009                         status);
2010
2011         return status;
2012 }
2013 EXPORT_SYMBOL_GPL(spi_setup);
2014
2015 static int __spi_validate(struct spi_device *spi, struct spi_message *message)
2016 {
2017         struct spi_master *master = spi->master;
2018         struct spi_transfer *xfer;
2019         int w_size;
2020
2021         if (list_empty(&message->transfers))
2022                 return -EINVAL;
2023
2024         /* Half-duplex links include original MicroWire, and ones with
2025          * only one data pin like SPI_3WIRE (switches direction) or where
2026          * either MOSI or MISO is missing.  They can also be caused by
2027          * software limitations.
2028          */
2029         if ((master->flags & SPI_MASTER_HALF_DUPLEX)
2030                         || (spi->mode & SPI_3WIRE)) {
2031                 unsigned flags = master->flags;
2032
2033                 list_for_each_entry(xfer, &message->transfers, transfer_list) {
2034                         if (xfer->rx_buf && xfer->tx_buf)
2035                                 return -EINVAL;
2036                         if ((flags & SPI_MASTER_NO_TX) && xfer->tx_buf)
2037                                 return -EINVAL;
2038                         if ((flags & SPI_MASTER_NO_RX) && xfer->rx_buf)
2039                                 return -EINVAL;
2040                 }
2041         }
2042
2043         /**
2044          * Set transfer bits_per_word and max speed as spi device default if
2045          * it is not set for this transfer.
2046          * Set transfer tx_nbits and rx_nbits as single transfer default
2047          * (SPI_NBITS_SINGLE) if it is not set for this transfer.
2048          */
2049         list_for_each_entry(xfer, &message->transfers, transfer_list) {
2050                 message->frame_length += xfer->len;
2051                 if (!xfer->bits_per_word)
2052                         xfer->bits_per_word = spi->bits_per_word;
2053
2054                 if (!xfer->speed_hz)
2055                         xfer->speed_hz = spi->max_speed_hz;
2056                 if (!xfer->speed_hz)
2057                         xfer->speed_hz = master->max_speed_hz;
2058
2059                 if (master->max_speed_hz &&
2060                     xfer->speed_hz > master->max_speed_hz)
2061                         xfer->speed_hz = master->max_speed_hz;
2062
2063                 if (__spi_validate_bits_per_word(master, xfer->bits_per_word))
2064                         return -EINVAL;
2065
2066                 /*
2067                  * SPI transfer length should be multiple of SPI word size
2068                  * where SPI word size should be power-of-two multiple
2069                  */
2070                 if (xfer->bits_per_word <= 8)
2071                         w_size = 1;
2072                 else if (xfer->bits_per_word <= 16)
2073                         w_size = 2;
2074                 else
2075                         w_size = 4;
2076
2077                 /* No partial transfers accepted */
2078                 if (xfer->len % w_size)
2079                         return -EINVAL;
2080
2081                 if (xfer->speed_hz && master->min_speed_hz &&
2082                     xfer->speed_hz < master->min_speed_hz)
2083                         return -EINVAL;
2084
2085                 if (xfer->tx_buf && !xfer->tx_nbits)
2086                         xfer->tx_nbits = SPI_NBITS_SINGLE;
2087                 if (xfer->rx_buf && !xfer->rx_nbits)
2088                         xfer->rx_nbits = SPI_NBITS_SINGLE;
2089                 /* check transfer tx/rx_nbits:
2090                  * 1. check the value matches one of single, dual and quad
2091                  * 2. check tx/rx_nbits match the mode in spi_device
2092                  */
2093                 if (xfer->tx_buf) {
2094                         if (xfer->tx_nbits != SPI_NBITS_SINGLE &&
2095                                 xfer->tx_nbits != SPI_NBITS_DUAL &&
2096                                 xfer->tx_nbits != SPI_NBITS_QUAD)
2097                                 return -EINVAL;
2098                         if ((xfer->tx_nbits == SPI_NBITS_DUAL) &&
2099                                 !(spi->mode & (SPI_TX_DUAL | SPI_TX_QUAD)))
2100                                 return -EINVAL;
2101                         if ((xfer->tx_nbits == SPI_NBITS_QUAD) &&
2102                                 !(spi->mode & SPI_TX_QUAD))
2103                                 return -EINVAL;
2104                 }
2105                 /* check transfer rx_nbits */
2106                 if (xfer->rx_buf) {
2107                         if (xfer->rx_nbits != SPI_NBITS_SINGLE &&
2108                                 xfer->rx_nbits != SPI_NBITS_DUAL &&
2109                                 xfer->rx_nbits != SPI_NBITS_QUAD)
2110                                 return -EINVAL;
2111                         if ((xfer->rx_nbits == SPI_NBITS_DUAL) &&
2112                                 !(spi->mode & (SPI_RX_DUAL | SPI_RX_QUAD)))
2113                                 return -EINVAL;
2114                         if ((xfer->rx_nbits == SPI_NBITS_QUAD) &&
2115                                 !(spi->mode & SPI_RX_QUAD))
2116                                 return -EINVAL;
2117                 }
2118         }
2119
2120         message->status = -EINPROGRESS;
2121
2122         return 0;
2123 }
2124
2125 static int __spi_async(struct spi_device *spi, struct spi_message *message)
2126 {
2127         struct spi_master *master = spi->master;
2128
2129         message->spi = spi;
2130
2131         SPI_STATISTICS_INCREMENT_FIELD(&master->statistics, spi_async);
2132         SPI_STATISTICS_INCREMENT_FIELD(&spi->statistics, spi_async);
2133
2134         trace_spi_message_submit(message);
2135
2136         return master->transfer(spi, message);
2137 }
2138
2139 /**
2140  * spi_async - asynchronous SPI transfer
2141  * @spi: device with which data will be exchanged
2142  * @message: describes the data transfers, including completion callback
2143  * Context: any (irqs may be blocked, etc)
2144  *
2145  * This call may be used in_irq and other contexts which can't sleep,
2146  * as well as from task contexts which can sleep.
2147  *
2148  * The completion callback is invoked in a context which can't sleep.
2149  * Before that invocation, the value of message->status is undefined.
2150  * When the callback is issued, message->status holds either zero (to
2151  * indicate complete success) or a negative error code.  After that
2152  * callback returns, the driver which issued the transfer request may
2153  * deallocate the associated memory; it's no longer in use by any SPI
2154  * core or controller driver code.
2155  *
2156  * Note that although all messages to a spi_device are handled in
2157  * FIFO order, messages may go to different devices in other orders.
2158  * Some device might be higher priority, or have various "hard" access
2159  * time requirements, for example.
2160  *
2161  * On detection of any fault during the transfer, processing of
2162  * the entire message is aborted, and the device is deselected.
2163  * Until returning from the associated message completion callback,
2164  * no other spi_message queued to that device will be processed.
2165  * (This rule applies equally to all the synchronous transfer calls,
2166  * which are wrappers around this core asynchronous primitive.)
2167  */
2168 int spi_async(struct spi_device *spi, struct spi_message *message)
2169 {
2170         struct spi_master *master = spi->master;
2171         int ret;
2172         unsigned long flags;
2173
2174         ret = __spi_validate(spi, message);
2175         if (ret != 0)
2176                 return ret;
2177
2178         spin_lock_irqsave(&master->bus_lock_spinlock, flags);
2179
2180         if (master->bus_lock_flag)
2181                 ret = -EBUSY;
2182         else
2183                 ret = __spi_async(spi, message);
2184
2185         spin_unlock_irqrestore(&master->bus_lock_spinlock, flags);
2186
2187         return ret;
2188 }
2189 EXPORT_SYMBOL_GPL(spi_async);
2190
2191 /**
2192  * spi_async_locked - version of spi_async with exclusive bus usage
2193  * @spi: device with which data will be exchanged
2194  * @message: describes the data transfers, including completion callback
2195  * Context: any (irqs may be blocked, etc)
2196  *
2197  * This call may be used in_irq and other contexts which can't sleep,
2198  * as well as from task contexts which can sleep.
2199  *
2200  * The completion callback is invoked in a context which can't sleep.
2201  * Before that invocation, the value of message->status is undefined.
2202  * When the callback is issued, message->status holds either zero (to
2203  * indicate complete success) or a negative error code.  After that
2204  * callback returns, the driver which issued the transfer request may
2205  * deallocate the associated memory; it's no longer in use by any SPI
2206  * core or controller driver code.
2207  *
2208  * Note that although all messages to a spi_device are handled in
2209  * FIFO order, messages may go to different devices in other orders.
2210  * Some device might be higher priority, or have various "hard" access
2211  * time requirements, for example.
2212  *
2213  * On detection of any fault during the transfer, processing of
2214  * the entire message is aborted, and the device is deselected.
2215  * Until returning from the associated message completion callback,
2216  * no other spi_message queued to that device will be processed.
2217  * (This rule applies equally to all the synchronous transfer calls,
2218  * which are wrappers around this core asynchronous primitive.)
2219  */
2220 int spi_async_locked(struct spi_device *spi, struct spi_message *message)
2221 {
2222         struct spi_master *master = spi->master;
2223         int ret;
2224         unsigned long flags;
2225
2226         ret = __spi_validate(spi, message);
2227         if (ret != 0)
2228                 return ret;
2229
2230         spin_lock_irqsave(&master->bus_lock_spinlock, flags);
2231
2232         ret = __spi_async(spi, message);
2233
2234         spin_unlock_irqrestore(&master->bus_lock_spinlock, flags);
2235
2236         return ret;
2237
2238 }
2239 EXPORT_SYMBOL_GPL(spi_async_locked);
2240
2241
2242 /*-------------------------------------------------------------------------*/
2243
2244 /* Utility methods for SPI master protocol drivers, layered on
2245  * top of the core.  Some other utility methods are defined as
2246  * inline functions.
2247  */
2248
2249 static void spi_complete(void *arg)
2250 {
2251         complete(arg);
2252 }
2253
2254 static int __spi_sync(struct spi_device *spi, struct spi_message *message,
2255                       int bus_locked)
2256 {
2257         DECLARE_COMPLETION_ONSTACK(done);
2258         int status;
2259         struct spi_master *master = spi->master;
2260         unsigned long flags;
2261
2262         status = __spi_validate(spi, message);
2263         if (status != 0)
2264                 return status;
2265
2266         message->complete = spi_complete;
2267         message->context = &done;
2268         message->spi = spi;
2269
2270         SPI_STATISTICS_INCREMENT_FIELD(&master->statistics, spi_sync);
2271         SPI_STATISTICS_INCREMENT_FIELD(&spi->statistics, spi_sync);
2272
2273         if (!bus_locked)
2274                 mutex_lock(&master->bus_lock_mutex);
2275
2276         /* If we're not using the legacy transfer method then we will
2277          * try to transfer in the calling context so special case.
2278          * This code would be less tricky if we could remove the
2279          * support for driver implemented message queues.
2280          */
2281         if (master->transfer == spi_queued_transfer) {
2282                 spin_lock_irqsave(&master->bus_lock_spinlock, flags);
2283
2284                 trace_spi_message_submit(message);
2285
2286                 status = __spi_queued_transfer(spi, message, false);
2287
2288                 spin_unlock_irqrestore(&master->bus_lock_spinlock, flags);
2289         } else {
2290                 status = spi_async_locked(spi, message);
2291         }
2292
2293         if (!bus_locked)
2294                 mutex_unlock(&master->bus_lock_mutex);
2295
2296         if (status == 0) {
2297                 /* Push out the messages in the calling context if we
2298                  * can.
2299                  */
2300                 if (master->transfer == spi_queued_transfer) {
2301                         SPI_STATISTICS_INCREMENT_FIELD(&master->statistics,
2302                                                        spi_sync_immediate);
2303                         SPI_STATISTICS_INCREMENT_FIELD(&spi->statistics,
2304                                                        spi_sync_immediate);
2305                         __spi_pump_messages(master, false);
2306                 }
2307
2308                 wait_for_completion(&done);
2309                 status = message->status;
2310         }
2311         message->context = NULL;
2312         return status;
2313 }
2314
2315 /**
2316  * spi_sync - blocking/synchronous SPI data transfers
2317  * @spi: device with which data will be exchanged
2318  * @message: describes the data transfers
2319  * Context: can sleep
2320  *
2321  * This call may only be used from a context that may sleep.  The sleep
2322  * is non-interruptible, and has no timeout.  Low-overhead controller
2323  * drivers may DMA directly into and out of the message buffers.
2324  *
2325  * Note that the SPI device's chip select is active during the message,
2326  * and then is normally disabled between messages.  Drivers for some
2327  * frequently-used devices may want to minimize costs of selecting a chip,
2328  * by leaving it selected in anticipation that the next message will go
2329  * to the same chip.  (That may increase power usage.)
2330  *
2331  * Also, the caller is guaranteeing that the memory associated with the
2332  * message will not be freed before this call returns.
2333  *
2334  * It returns zero on success, else a negative error code.
2335  */
2336 int spi_sync(struct spi_device *spi, struct spi_message *message)
2337 {
2338         return __spi_sync(spi, message, 0);
2339 }
2340 EXPORT_SYMBOL_GPL(spi_sync);
2341
2342 /**
2343  * spi_sync_locked - version of spi_sync with exclusive bus usage
2344  * @spi: device with which data will be exchanged
2345  * @message: describes the data transfers
2346  * Context: can sleep
2347  *
2348  * This call may only be used from a context that may sleep.  The sleep
2349  * is non-interruptible, and has no timeout.  Low-overhead controller
2350  * drivers may DMA directly into and out of the message buffers.
2351  *
2352  * This call should be used by drivers that require exclusive access to the
2353  * SPI bus. It has to be preceded by a spi_bus_lock call. The SPI bus must
2354  * be released by a spi_bus_unlock call when the exclusive access is over.
2355  *
2356  * It returns zero on success, else a negative error code.
2357  */
2358 int spi_sync_locked(struct spi_device *spi, struct spi_message *message)
2359 {
2360         return __spi_sync(spi, message, 1);
2361 }
2362 EXPORT_SYMBOL_GPL(spi_sync_locked);
2363
2364 /**
2365  * spi_bus_lock - obtain a lock for exclusive SPI bus usage
2366  * @master: SPI bus master that should be locked for exclusive bus access
2367  * Context: can sleep
2368  *
2369  * This call may only be used from a context that may sleep.  The sleep
2370  * is non-interruptible, and has no timeout.
2371  *
2372  * This call should be used by drivers that require exclusive access to the
2373  * SPI bus. The SPI bus must be released by a spi_bus_unlock call when the
2374  * exclusive access is over. Data transfer must be done by spi_sync_locked
2375  * and spi_async_locked calls when the SPI bus lock is held.
2376  *
2377  * It returns zero on success, else a negative error code.
2378  */
2379 int spi_bus_lock(struct spi_master *master)
2380 {
2381         unsigned long flags;
2382
2383         mutex_lock(&master->bus_lock_mutex);
2384
2385         spin_lock_irqsave(&master->bus_lock_spinlock, flags);
2386         master->bus_lock_flag = 1;
2387         spin_unlock_irqrestore(&master->bus_lock_spinlock, flags);
2388
2389         /* mutex remains locked until spi_bus_unlock is called */
2390
2391         return 0;
2392 }
2393 EXPORT_SYMBOL_GPL(spi_bus_lock);
2394
2395 /**
2396  * spi_bus_unlock - release the lock for exclusive SPI bus usage
2397  * @master: SPI bus master that was locked for exclusive bus access
2398  * Context: can sleep
2399  *
2400  * This call may only be used from a context that may sleep.  The sleep
2401  * is non-interruptible, and has no timeout.
2402  *
2403  * This call releases an SPI bus lock previously obtained by an spi_bus_lock
2404  * call.
2405  *
2406  * It returns zero on success, else a negative error code.
2407  */
2408 int spi_bus_unlock(struct spi_master *master)
2409 {
2410         master->bus_lock_flag = 0;
2411
2412         mutex_unlock(&master->bus_lock_mutex);
2413
2414         return 0;
2415 }
2416 EXPORT_SYMBOL_GPL(spi_bus_unlock);
2417
2418 /* portable code must never pass more than 32 bytes */
2419 #define SPI_BUFSIZ      max(32, SMP_CACHE_BYTES)
2420
2421 static u8       *buf;
2422
2423 /**
2424  * spi_write_then_read - SPI synchronous write followed by read
2425  * @spi: device with which data will be exchanged
2426  * @txbuf: data to be written (need not be dma-safe)
2427  * @n_tx: size of txbuf, in bytes
2428  * @rxbuf: buffer into which data will be read (need not be dma-safe)
2429  * @n_rx: size of rxbuf, in bytes
2430  * Context: can sleep
2431  *
2432  * This performs a half duplex MicroWire style transaction with the
2433  * device, sending txbuf and then reading rxbuf.  The return value
2434  * is zero for success, else a negative errno status code.
2435  * This call may only be used from a context that may sleep.
2436  *
2437  * Parameters to this routine are always copied using a small buffer;
2438  * portable code should never use this for more than 32 bytes.
2439  * Performance-sensitive or bulk transfer code should instead use
2440  * spi_{async,sync}() calls with dma-safe buffers.
2441  */
2442 int spi_write_then_read(struct spi_device *spi,
2443                 const void *txbuf, unsigned n_tx,
2444                 void *rxbuf, unsigned n_rx)
2445 {
2446         static DEFINE_MUTEX(lock);
2447
2448         int                     status;
2449         struct spi_message      message;
2450         struct spi_transfer     x[2];
2451         u8                      *local_buf;
2452
2453         /* Use preallocated DMA-safe buffer if we can.  We can't avoid
2454          * copying here, (as a pure convenience thing), but we can
2455          * keep heap costs out of the hot path unless someone else is
2456          * using the pre-allocated buffer or the transfer is too large.
2457          */
2458         if ((n_tx + n_rx) > SPI_BUFSIZ || !mutex_trylock(&lock)) {
2459                 local_buf = kmalloc(max((unsigned)SPI_BUFSIZ, n_tx + n_rx),
2460                                     GFP_KERNEL | GFP_DMA);
2461                 if (!local_buf)
2462                         return -ENOMEM;
2463         } else {
2464                 local_buf = buf;
2465         }
2466
2467         spi_message_init(&message);
2468         memset(x, 0, sizeof(x));
2469         if (n_tx) {
2470                 x[0].len = n_tx;
2471                 spi_message_add_tail(&x[0], &message);
2472         }
2473         if (n_rx) {
2474                 x[1].len = n_rx;
2475                 spi_message_add_tail(&x[1], &message);
2476         }
2477
2478         memcpy(local_buf, txbuf, n_tx);
2479         x[0].tx_buf = local_buf;
2480         x[1].rx_buf = local_buf + n_tx;
2481
2482         /* do the i/o */
2483         status = spi_sync(spi, &message);
2484         if (status == 0)
2485                 memcpy(rxbuf, x[1].rx_buf, n_rx);
2486
2487         if (x[0].tx_buf == buf)
2488                 mutex_unlock(&lock);
2489         else
2490                 kfree(local_buf);
2491
2492         return status;
2493 }
2494 EXPORT_SYMBOL_GPL(spi_write_then_read);
2495
2496 /*-------------------------------------------------------------------------*/
2497
2498 #if IS_ENABLED(CONFIG_OF_DYNAMIC)
2499 static int __spi_of_device_match(struct device *dev, void *data)
2500 {
2501         return dev->of_node == data;
2502 }
2503
2504 /* must call put_device() when done with returned spi_device device */
2505 static struct spi_device *of_find_spi_device_by_node(struct device_node *node)
2506 {
2507         struct device *dev = bus_find_device(&spi_bus_type, NULL, node,
2508                                                 __spi_of_device_match);
2509         return dev ? to_spi_device(dev) : NULL;
2510 }
2511
2512 static int __spi_of_master_match(struct device *dev, const void *data)
2513 {
2514         return dev->of_node == data;
2515 }
2516
2517 /* the spi masters are not using spi_bus, so we find it with another way */
2518 static struct spi_master *of_find_spi_master_by_node(struct device_node *node)
2519 {
2520         struct device *dev;
2521
2522         dev = class_find_device(&spi_master_class, NULL, node,
2523                                 __spi_of_master_match);
2524         if (!dev)
2525                 return NULL;
2526
2527         /* reference got in class_find_device */
2528         return container_of(dev, struct spi_master, dev);
2529 }
2530
2531 static int of_spi_notify(struct notifier_block *nb, unsigned long action,
2532                          void *arg)
2533 {
2534         struct of_reconfig_data *rd = arg;
2535         struct spi_master *master;
2536         struct spi_device *spi;
2537
2538         switch (of_reconfig_get_state_change(action, arg)) {
2539         case OF_RECONFIG_CHANGE_ADD:
2540                 master = of_find_spi_master_by_node(rd->dn->parent);
2541                 if (master == NULL)
2542                         return NOTIFY_OK;       /* not for us */
2543
2544                 spi = of_register_spi_device(master, rd->dn);
2545                 put_device(&master->dev);
2546
2547                 if (IS_ERR(spi)) {
2548                         pr_err("%s: failed to create for '%s'\n",
2549                                         __func__, rd->dn->full_name);
2550                         return notifier_from_errno(PTR_ERR(spi));
2551                 }
2552                 break;
2553
2554         case OF_RECONFIG_CHANGE_REMOVE:
2555                 /* find our device by node */
2556                 spi = of_find_spi_device_by_node(rd->dn);
2557                 if (spi == NULL)
2558                         return NOTIFY_OK;       /* no? not meant for us */
2559
2560                 /* unregister takes one ref away */
2561                 spi_unregister_device(spi);
2562
2563                 /* and put the reference of the find */
2564                 put_device(&spi->dev);
2565                 break;
2566         }
2567
2568         return NOTIFY_OK;
2569 }
2570
2571 static struct notifier_block spi_of_notifier = {
2572         .notifier_call = of_spi_notify,
2573 };
2574 #else /* IS_ENABLED(CONFIG_OF_DYNAMIC) */
2575 extern struct notifier_block spi_of_notifier;
2576 #endif /* IS_ENABLED(CONFIG_OF_DYNAMIC) */
2577
2578 static int __init spi_init(void)
2579 {
2580         int     status;
2581
2582         buf = kmalloc(SPI_BUFSIZ, GFP_KERNEL);
2583         if (!buf) {
2584                 status = -ENOMEM;
2585                 goto err0;
2586         }
2587
2588         status = bus_register(&spi_bus_type);
2589         if (status < 0)
2590                 goto err1;
2591
2592         status = class_register(&spi_master_class);
2593         if (status < 0)
2594                 goto err2;
2595
2596         if (IS_ENABLED(CONFIG_OF_DYNAMIC))
2597                 WARN_ON(of_reconfig_notifier_register(&spi_of_notifier));
2598
2599         return 0;
2600
2601 err2:
2602         bus_unregister(&spi_bus_type);
2603 err1:
2604         kfree(buf);
2605         buf = NULL;
2606 err0:
2607         return status;
2608 }
2609
2610 /* board_info is normally registered in arch_initcall(),
2611  * but even essential drivers wait till later
2612  *
2613  * REVISIT only boardinfo really needs static linking. the rest (device and
2614  * driver registration) _could_ be dynamically linked (modular) ... costs
2615  * include needing to have boardinfo data structures be much more public.
2616  */
2617 postcore_initcall(spi_init);
2618