lib/dma-debug.c

   1 /*
   2  * Copyright (C) 2008 Advanced Micro Devices, Inc.
   3  *
   4  * Author: Joerg Roedel <joerg.roedel@amd.com>
   5  *
   6  * This program is free software; you can redistribute it and/or modify it
   7  * under the terms of the GNU General Public License version 2 as published
   8  * by the Free Software Foundation.
   9  *
  10  * This program is distributed in the hope that it will be useful,
  11  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  12  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  13  * GNU General Public License for more details.
  14  *
  15  * You should have received a copy of the GNU General Public License
  16  * along with this program; if not, write to the Free Software
  17  * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307 USA
  18  */
  19
  20 #include <linux/scatterlist.h>
  21 #include <linux/dma-mapping.h>
  22 #include <linux/stacktrace.h>
  23 #include <linux/dma-debug.h>
  24 #include <linux/spinlock.h>
  25 #include <linux/debugfs.h>
  26 #include <linux/uaccess.h>
  27 #include <linux/export.h>
  28 #include <linux/device.h>
  29 #include <linux/types.h>
  30 #include <linux/sched.h>
  31 #include <linux/ctype.h>
  32 #include <linux/list.h>
  33 #include <linux/slab.h>
  34
  35 #include <asm/sections.h>
  36
  37 #define HASH_SIZE       1024ULL
  38 #define HASH_FN_SHIFT   13
  39 #define HASH_FN_MASK    (HASH_SIZE - 1)
  40
  41 enum {
  42         dma_debug_single,
  43         dma_debug_page,
  44         dma_debug_sg,
  45         dma_debug_coherent,
  46 };
  47
  48 #define DMA_DEBUG_STACKTRACE_ENTRIES 5
  49
  50 struct dma_debug_entry {
  51         struct list_head list;
  52         struct device    *dev;
  53         int              type;
  54         phys_addr_t      paddr;
  55         u64              dev_addr;
  56         u64              size;
  57         int              direction;
  58         int              sg_call_ents;
  59         int              sg_mapped_ents;
  60 #ifdef CONFIG_STACKTRACE
  61         struct           stack_trace stacktrace;
  62         unsigned long    st_entries[DMA_DEBUG_STACKTRACE_ENTRIES];
  63 #endif
  64 };
  65
  66 typedef bool (*match_fn)(struct dma_debug_entry *, struct dma_debug_entry *);
  67
  68 struct hash_bucket {
  69         struct list_head list;
  70         spinlock_t lock;
  71 } ____cacheline_aligned_in_smp;
  72
  73 /* Hash list to save the allocated dma addresses */
  74 static struct hash_bucket dma_entry_hash[HASH_SIZE];
  75 /* List of pre-allocated dma_debug_entry's */
  76 static LIST_HEAD(free_entries);
  77 /* Lock for the list above */
  78 static DEFINE_SPINLOCK(free_entries_lock);
  79
  80 /* Global disable flag - will be set in case of an error */
  81 static u32 global_disable __read_mostly;
  82
  83 /* Global error count */
  84 static u32 error_count;
  85
  86 /* Global error show enable*/
  87 static u32 show_all_errors __read_mostly;
  88 /* Number of errors to show */
  89 static u32 show_num_errors = 1;
  90
  91 static u32 num_free_entries;
  92 static u32 min_free_entries;
  93 static u32 nr_total_entries;
  94
  95 /* number of preallocated entries requested by kernel cmdline */
  96 static u32 req_entries;
  97
  98 /* debugfs dentry's for the stuff above */
  99 static struct dentry *dma_debug_dent        __read_mostly;
 100 static struct dentry *global_disable_dent   __read_mostly;
 101 static struct dentry *error_count_dent      __read_mostly;
 102 static struct dentry *show_all_errors_dent  __read_mostly;
 103 static struct dentry *show_num_errors_dent  __read_mostly;
 104 static struct dentry *num_free_entries_dent __read_mostly;
 105 static struct dentry *min_free_entries_dent __read_mostly;
 106 static struct dentry *filter_dent           __read_mostly;
 107
 108 /* per-driver filter related state */
 109
 110 #define NAME_MAX_LEN    64
 111
 112 static char                  current_driver_name[NAME_MAX_LEN] __read_mostly;
 113 static struct device_driver *current_driver                    __read_mostly;
 114
 115 static DEFINE_RWLOCK(driver_name_lock);
 116
 117 static const char *type2name[4] = { "single", "page",
 118                                     "scather-gather", "coherent" };
 119
 120 static const char *dir2name[4] = { "DMA_BIDIRECTIONAL", "DMA_TO_DEVICE",
 121                                    "DMA_FROM_DEVICE", "DMA_NONE" };
 122
 123 /*
 124  * The access to some variables in this macro is racy. We can't use atomic_t
 125  * here because all these variables are exported to debugfs. Some of them even
 126  * writeable. This is also the reason why a lock won't help much. But anyway,
 127  * the races are no big deal. Here is why:
 128  *
 129  *   error_count: the addition is racy, but the worst thing that can happen is
 130  *                that we don't count some errors
 131  *   show_num_errors: the subtraction is racy. Also no big deal because in
 132  *                    worst case this will result in one warning more in the
 133  *                    system log than the user configured. This variable is
 134  *                    writeable via debugfs.
 135  */
 136 static inline void dump_entry_trace(struct dma_debug_entry *entry)
 137 {
 138 #ifdef CONFIG_STACKTRACE
 139         if (entry) {
 140                 pr_warning("Mapped at:\n");
 141                 print_stack_trace(&entry->stacktrace, 0);
 142         }
 143 #endif
 144 }
 145
 146 static bool driver_filter(struct device *dev)
 147 {
 148         struct device_driver *drv;
 149         unsigned long flags;
 150         bool ret;
 151
 152         /* driver filter off */
 153         if (likely(!current_driver_name[0]))
 154                 return true;
 155
 156         /* driver filter on and initialized */
 157         if (current_driver && dev && dev->driver == current_driver)
 158                 return true;
 159
 160         /* driver filter on, but we can't filter on a NULL device... */
 161         if (!dev)
 162                 return false;
 163
 164         if (current_driver || !current_driver_name[0])
 165                 return false;
 166
 167         /* driver filter on but not yet initialized */
 168         drv = dev->driver;
 169         if (!drv)
 170                 return false;
 171
 172         /* lock to protect against change of current_driver_name */
 173         read_lock_irqsave(&driver_name_lock, flags);
 174
 175         ret = false;
 176         if (drv->name &&
 177             strncmp(current_driver_name, drv->name, NAME_MAX_LEN - 1) == 0) {
 178                 current_driver = drv;
 179                 ret = true;
 180         }
 181
 182         read_unlock_irqrestore(&driver_name_lock, flags);
 183
 184         return ret;
 185 }
 186
 187 #define err_printk(dev, entry, format, arg...) do {                     \
 188                 error_count += 1;                                       \
 189                 if (driver_filter(dev) &&                               \
 190                     (show_all_errors || show_num_errors > 0)) {         \
 191                         WARN(1, "%s %s: " format,                       \
 192                              dev ? dev_driver_string(dev) : "NULL",     \
 193                              dev ? dev_name(dev) : "NULL", ## arg);     \
 194                         dump_entry_trace(entry);                        \
 195                 }                                                       \
 196                 if (!show_all_errors && show_num_errors > 0)            \
 197                         show_num_errors -= 1;                           \
 198         } while (0);
 199
 200 /*
 201  * Hash related functions
 202  *
 203  * Every DMA-API request is saved into a struct dma_debug_entry. To
 204  * have quick access to these structs they are stored into a hash.
 205  */
 206 static int hash_fn(struct dma_debug_entry *entry)
 207 {
 208         /*
 209          * Hash function is based on the dma address.
 210          * We use bits 20-27 here as the index into the hash
 211          */
 212         return (entry->dev_addr >> HASH_FN_SHIFT) & HASH_FN_MASK;
 213 }
 214
 215 /*
 216  * Request exclusive access to a hash bucket for a given dma_debug_entry.
 217  */
 218 static struct hash_bucket *get_hash_bucket(struct dma_debug_entry *entry,
 219                                            unsigned long *flags)
 220 {
 221         int idx = hash_fn(entry);
 222         unsigned long __flags;
 223
 224         spin_lock_irqsave(&dma_entry_hash[idx].lock, __flags);
 225         *flags = __flags;
 226         return &dma_entry_hash[idx];
 227 }
 228
 229 /*
 230  * Give up exclusive access to the hash bucket
 231  */
 232 static void put_hash_bucket(struct hash_bucket *bucket,
 233                             unsigned long *flags)
 234 {
 235         unsigned long __flags = *flags;
 236
 237         spin_unlock_irqrestore(&bucket->lock, __flags);
 238 }
 239
 240 static bool exact_match(struct dma_debug_entry *a, struct dma_debug_entry *b)
 241 {
 242         return ((a->dev_addr == b->dev_addr) &&
 243                 (a->dev == b->dev)) ? true : false;
 244 }
 245
 246 static bool containing_match(struct dma_debug_entry *a,
 247                              struct dma_debug_entry *b)
 248 {
 249         if (a->dev != b->dev)
 250                 return false;
 251
 252         if ((b->dev_addr <= a->dev_addr) &&
 253             ((b->dev_addr + b->size) >= (a->dev_addr + a->size)))
 254                 return true;
 255
 256         return false;
 257 }
 258
 259 /*
 260  * Search a given entry in the hash bucket list
 261  */
 262 static struct dma_debug_entry *__hash_bucket_find(struct hash_bucket *bucket,
 263                                                   struct dma_debug_entry *ref,
 264                                                   match_fn match)
 265 {
 266         struct dma_debug_entry *entry, *ret = NULL;
 267         int matches = 0, match_lvl, last_lvl = -1;
 268
 269         list_for_each_entry(entry, &bucket->list, list) {
 270                 if (!match(ref, entry))
 271                         continue;
 272
 273                 /*
 274                  * Some drivers map the same physical address multiple
 275                  * times. Without a hardware IOMMU this results in the
 276                  * same device addresses being put into the dma-debug
 277                  * hash multiple times too. This can result in false
 278                  * positives being reported. Therefore we implement a
 279                  * best-fit algorithm here which returns the entry from
 280                  * the hash which fits best to the reference value
 281                  * instead of the first-fit.
 282                  */
 283                 matches += 1;
 284                 match_lvl = 0;
 285                 entry->size         == ref->size         ? ++match_lvl : 0;
 286                 entry->type         == ref->type         ? ++match_lvl : 0;
 287                 entry->direction    == ref->direction    ? ++match_lvl : 0;
 288                 entry->sg_call_ents == ref->sg_call_ents ? ++match_lvl : 0;
 289
 290                 if (match_lvl == 4) {
 291                         /* perfect-fit - return the result */
 292                         return entry;
 293                 } else if (match_lvl > last_lvl) {
 294                         /*
 295                          * We found an entry that fits better then the
 296                          * previous one or it is the 1st match.
 297                          */
 298                         last_lvl = match_lvl;
 299                         ret      = entry;
 300                 }
 301         }
 302
 303         /*
 304          * If we have multiple matches but no perfect-fit, just return
 305          * NULL.
 306          */
 307         ret = (matches == 1) ? ret : NULL;
 308
 309         return ret;
 310 }
 311
 312 static struct dma_debug_entry *bucket_find_exact(struct hash_bucket *bucket,
 313                                                  struct dma_debug_entry *ref)
 314 {
 315         return __hash_bucket_find(bucket, ref, exact_match);
 316 }
 317
 318 static struct dma_debug_entry *bucket_find_contain(struct hash_bucket **bucket,
 319                                                    struct dma_debug_entry *ref,
 320                                                    unsigned long *flags)
 321 {
 322
 323         unsigned int max_range = dma_get_max_seg_size(ref->dev);
 324         struct dma_debug_entry *entry, index = *ref;
 325         unsigned int range = 0;
 326
 327         while (range <= max_range) {
 328                 entry = __hash_bucket_find(*bucket, &index, containing_match);
 329
 330                 if (entry)
 331                         return entry;
 332
 333                 /*
 334                  * Nothing found, go back a hash bucket
 335                  */
 336                 put_hash_bucket(*bucket, flags);
 337                 range          += (1 << HASH_FN_SHIFT);
 338                 index.dev_addr -= (1 << HASH_FN_SHIFT);
 339                 *bucket = get_hash_bucket(&index, flags);
 340         }
 341
 342         return NULL;
 343 }
 344
 345 /*
 346  * Add an entry to a hash bucket
 347  */
 348 static void hash_bucket_add(struct hash_bucket *bucket,
 349                             struct dma_debug_entry *entry)
 350 {
 351         list_add_tail(&entry->list, &bucket->list);
 352 }
 353
 354 /*
 355  * Remove entry from a hash bucket list
 356  */
 357 static void hash_bucket_del(struct dma_debug_entry *entry)
 358 {
 359         list_del(&entry->list);
 360 }
 361
 362 /*
 363  * Dump mapping entries for debugging purposes
 364  */
 365 void debug_dma_dump_mappings(struct device *dev)
 366 {
 367         int idx;
 368
 369         for (idx = 0; idx < HASH_SIZE; idx++) {
 370                 struct hash_bucket *bucket = &dma_entry_hash[idx];
 371                 struct dma_debug_entry *entry;
 372                 unsigned long flags;
 373
 374                 spin_lock_irqsave(&bucket->lock, flags);
 375
 376                 list_for_each_entry(entry, &bucket->list, list) {
 377                         if (!dev || dev == entry->dev) {
 378                                 dev_info(entry->dev,
 379                                          "%s idx %d P=%Lx D=%Lx L=%Lx %s\n",
 380                                          type2name[entry->type], idx,
 381                                          (unsigned long long)entry->paddr,
 382                                          entry->dev_addr, entry->size,
 383                                          dir2name[entry->direction]);
 384                         }
 385                 }
 386
 387                 spin_unlock_irqrestore(&bucket->lock, flags);
 388         }
 389 }
 390 EXPORT_SYMBOL(debug_dma_dump_mappings);
 391
 392 /*
 393  * Wrapper function for adding an entry to the hash.
 394  * This function takes care of locking itself.
 395  */
 396 static void add_dma_entry(struct dma_debug_entry *entry)
 397 {
 398         struct hash_bucket *bucket;
 399         unsigned long flags;
 400
 401         bucket = get_hash_bucket(entry, &flags);
 402         hash_bucket_add(bucket, entry);
 403         put_hash_bucket(bucket, &flags);
 404 }
 405
 406 static struct dma_debug_entry *__dma_entry_alloc(void)
 407 {
 408         struct dma_debug_entry *entry;
 409
 410         entry = list_entry(free_entries.next, struct dma_debug_entry, list);
 411         list_del(&entry->list);
 412         memset(entry, 0, sizeof(*entry));
 413
 414         num_free_entries -= 1;
 415         if (num_free_entries < min_free_entries)
 416                 min_free_entries = num_free_entries;
 417
 418         return entry;
 419 }
 420
 421 /* struct dma_entry allocator
 422  *
 423  * The next two functions implement the allocator for
 424  * struct dma_debug_entries.
 425  */
 426 static struct dma_debug_entry *dma_entry_alloc(void)
 427 {
 428         struct dma_debug_entry *entry;
 429         unsigned long flags;
 430
 431         spin_lock_irqsave(&free_entries_lock, flags);
 432
 433         if (list_empty(&free_entries)) {
 434                 pr_err("DMA-API: debugging out of memory - disabling\n");
 435                 global_disable = true;
 436                 spin_unlock_irqrestore(&free_entries_lock, flags);
 437                 return NULL;
 438         }
 439
 440         entry = __dma_entry_alloc();
 441
 442         spin_unlock_irqrestore(&free_entries_lock, flags);
 443
 444 #ifdef CONFIG_STACKTRACE
 445         entry->stacktrace.max_entries = DMA_DEBUG_STACKTRACE_ENTRIES;
 446         entry->stacktrace.entries = entry->st_entries;
 447         entry->stacktrace.skip = 2;
 448         save_stack_trace(&entry->stacktrace);
 449 #endif
 450
 451         return entry;
 452 }
 453
 454 static void dma_entry_free(struct dma_debug_entry *entry)
 455 {
 456         unsigned long flags;
 457
 458         /*
 459          * add to beginning of the list - this way the entries are
 460          * more likely cache hot when they are reallocated.
 461          */
 462         spin_lock_irqsave(&free_entries_lock, flags);
 463         list_add(&entry->list, &free_entries);
 464         num_free_entries += 1;
 465         spin_unlock_irqrestore(&free_entries_lock, flags);
 466 }
 467
 468 int dma_debug_resize_entries(u32 num_entries)
 469 {
 470         int i, delta, ret = 0;
 471         unsigned long flags;
 472         struct dma_debug_entry *entry;
 473         LIST_HEAD(tmp);
 474
 475         spin_lock_irqsave(&free_entries_lock, flags);
 476
 477         if (nr_total_entries < num_entries) {
 478                 delta = num_entries - nr_total_entries;
 479
 480                 spin_unlock_irqrestore(&free_entries_lock, flags);
 481
 482                 for (i = 0; i < delta; i++) {
 483                         entry = kzalloc(sizeof(*entry), GFP_KERNEL);
 484                         if (!entry)
 485                                 break;
 486
 487                         list_add_tail(&entry->list, &tmp);
 488                 }
 489
 490                 spin_lock_irqsave(&free_entries_lock, flags);
 491
 492                 list_splice(&tmp, &free_entries);
 493                 nr_total_entries += i;
 494                 num_free_entries += i;
 495         } else {
 496                 delta = nr_total_entries - num_entries;
 497
 498                 for (i = 0; i < delta && !list_empty(&free_entries); i++) {
 499                         entry = __dma_entry_alloc();
 500                         kfree(entry);
 501                 }
 502
 503                 nr_total_entries -= i;
 504         }
 505
 506         if (nr_total_entries != num_entries)
 507                 ret = 1;
 508
 509         spin_unlock_irqrestore(&free_entries_lock, flags);
 510
 511         return ret;
 512 }
 513 EXPORT_SYMBOL(dma_debug_resize_entries);
 514
 515 /*
 516  * DMA-API debugging init code
 517  *
 518  * The init code does two things:
 519  *   1. Initialize core data structures
 520  *   2. Preallocate a given number of dma_debug_entry structs
 521  */
 522
 523 static int prealloc_memory(u32 num_entries)
 524 {
 525         struct dma_debug_entry *entry, *next_entry;
 526         int i;
 527
 528         for (i = 0; i < num_entries; ++i) {
 529                 entry = kzalloc(sizeof(*entry), GFP_KERNEL);
 530                 if (!entry)
 531                         goto out_err;
 532
 533                 list_add_tail(&entry->list, &free_entries);
 534         }
 535
 536         num_free_entries = num_entries;
 537         min_free_entries = num_entries;
 538
 539         pr_info("DMA-API: preallocated %d debug entries\n", num_entries);
 540
 541         return 0;
 542
 543 out_err:
 544
 545         list_for_each_entry_safe(entry, next_entry, &free_entries, list) {
 546                 list_del(&entry->list);
 547                 kfree(entry);
 548         }
 549
 550         return -ENOMEM;
 551 }
 552
 553 static ssize_t filter_read(struct file *file, char __user *user_buf,
 554                            size_t count, loff_t *ppos)
 555 {
 556         char buf[NAME_MAX_LEN + 1];
 557         unsigned long flags;
 558         int len;
 559
 560         if (!current_driver_name[0])
 561                 return 0;
 562
 563         /*
 564          * We can't copy to userspace directly because current_driver_name can
 565          * only be read under the driver_name_lock with irqs disabled. So
 566          * create a temporary copy first.
 567          */
 568         read_lock_irqsave(&driver_name_lock, flags);
 569         len = scnprintf(buf, NAME_MAX_LEN + 1, "%s\n", current_driver_name);
 570         read_unlock_irqrestore(&driver_name_lock, flags);
 571
 572         return simple_read_from_buffer(user_buf, count, ppos, buf, len);
 573 }
 574
 575 static ssize_t filter_write(struct file *file, const char __user *userbuf,
 576                             size_t count, loff_t *ppos)
 577 {
 578         char buf[NAME_MAX_LEN];
 579         unsigned long flags;
 580         size_t len;
 581         int i;
 582
 583         /*
 584          * We can't copy from userspace directly. Access to
 585          * current_driver_name is protected with a write_lock with irqs
 586          * disabled. Since copy_from_user can fault and may sleep we
 587          * need to copy to temporary buffer first
 588          */
 589         len = min(count, (size_t)(NAME_MAX_LEN - 1));
 590         if (copy_from_user(buf, userbuf, len))
 591                 return -EFAULT;
 592
 593         buf[len] = 0;
 594
 595         write_lock_irqsave(&driver_name_lock, flags);
 596
 597         /*
 598          * Now handle the string we got from userspace very carefully.
 599          * The rules are:
 600          *         - only use the first token we got
 601          *         - token delimiter is everything looking like a space
 602          *           character (' ', '\n', '\t' ...)
 603          *
 604          */
 605         if (!isalnum(buf[0])) {
 606                 /*
 607                  * If the first character userspace gave us is not
 608                  * alphanumerical then assume the filter should be
 609                  * switched off.
 610                  */
 611                 if (current_driver_name[0])
 612                         pr_info("DMA-API: switching off dma-debug driver filter\n");
 613                 current_driver_name[0] = 0;
 614                 current_driver = NULL;
 615                 goto out_unlock;
 616         }
 617
 618         /*
 619          * Now parse out the first token and use it as the name for the
 620          * driver to filter for.
 621          */
 622         for (i = 0; i < NAME_MAX_LEN - 1; ++i) {
 623                 current_driver_name[i] = buf[i];
 624                 if (isspace(buf[i]) || buf[i] == ' ' || buf[i] == 0)
 625                         break;
 626         }
 627         current_driver_name[i] = 0;
 628         current_driver = NULL;
 629
 630         pr_info("DMA-API: enable driver filter for driver [%s]\n",
 631                 current_driver_name);
 632
 633 out_unlock:
 634         write_unlock_irqrestore(&driver_name_lock, flags);
 635
 636         return count;
 637 }
 638
 639 static const struct file_operations filter_fops = {
 640         .read  = filter_read,
 641         .write = filter_write,
 642         .llseek = default_llseek,
 643 };
 644
 645 static int dma_debug_fs_init(void)
 646 {
 647         dma_debug_dent = debugfs_create_dir("dma-api", NULL);
 648         if (!dma_debug_dent) {
 649                 pr_err("DMA-API: can not create debugfs directory\n");
 650                 return -ENOMEM;
 651         }
 652
 653         global_disable_dent = debugfs_create_bool("disabled", 0444,
 654                         dma_debug_dent,
 655                         &global_disable);
 656         if (!global_disable_dent)
 657                 goto out_err;
 658
 659         error_count_dent = debugfs_create_u32("error_count", 0444,
 660                         dma_debug_dent, &error_count);
 661         if (!error_count_dent)
 662                 goto out_err;
 663
 664         show_all_errors_dent = debugfs_create_u32("all_errors", 0644,
 665                         dma_debug_dent,
 666                         &show_all_errors);
 667         if (!show_all_errors_dent)
 668                 goto out_err;
 669
 670         show_num_errors_dent = debugfs_create_u32("num_errors", 0644,
 671                         dma_debug_dent,
 672                         &show_num_errors);
 673         if (!show_num_errors_dent)
 674                 goto out_err;
 675
 676         num_free_entries_dent = debugfs_create_u32("num_free_entries", 0444,
 677                         dma_debug_dent,
 678                         &num_free_entries);
 679         if (!num_free_entries_dent)
 680                 goto out_err;
 681
 682         min_free_entries_dent = debugfs_create_u32("min_free_entries", 0444,
 683                         dma_debug_dent,
 684                         &min_free_entries);
 685         if (!min_free_entries_dent)
 686                 goto out_err;
 687
 688         filter_dent = debugfs_create_file("driver_filter", 0644,
 689                                           dma_debug_dent, NULL, &filter_fops);
 690         if (!filter_dent)
 691                 goto out_err;
 692
 693         return 0;
 694
 695 out_err:
 696         debugfs_remove_recursive(dma_debug_dent);
 697
 698         return -ENOMEM;
 699 }
 700
 701 static int device_dma_allocations(struct device *dev, struct dma_debug_entry **out_entry)
 702 {
 703         struct dma_debug_entry *entry;
 704         unsigned long flags;
 705         int count = 0, i;
 706
 707         local_irq_save(flags);
 708
 709         for (i = 0; i < HASH_SIZE; ++i) {
 710                 spin_lock(&dma_entry_hash[i].lock);
 711                 list_for_each_entry(entry, &dma_entry_hash[i].list, list) {
 712                         if (entry->dev == dev) {
 713                                 count += 1;
 714                                 *out_entry = entry;
 715                         }
 716                 }
 717                 spin_unlock(&dma_entry_hash[i].lock);
 718         }
 719
 720         local_irq_restore(flags);
 721
 722         return count;
 723 }
 724
 725 static int dma_debug_device_change(struct notifier_block *nb, unsigned long action, void *data)
 726 {
 727         struct device *dev = data;
 728         struct dma_debug_entry *uninitialized_var(entry);
 729         int count;
 730
 731         if (global_disable)
 732                 return 0;
 733
 734         switch (action) {
 735         case BUS_NOTIFY_UNBOUND_DRIVER:
 736                 count = device_dma_allocations(dev, &entry);
 737                 if (count == 0)
 738                         break;
 739                 err_printk(dev, entry, "DMA-API: device driver has pending "
 740                                 "DMA allocations while released from device "
 741                                 "[count=%d]\n"
 742                                 "One of leaked entries details: "
 743                                 "[device address=0x%016llx] [size=%llu bytes] "
 744                                 "[mapped with %s] [mapped as %s]\n",
 745                         count, entry->dev_addr, entry->size,
 746                         dir2name[entry->direction], type2name[entry->type]);
 747                 break;
 748         default:
 749                 break;
 750         }
 751
 752         return 0;
 753 }
 754
 755 void dma_debug_add_bus(struct bus_type *bus)
 756 {
 757         struct notifier_block *nb;
 758
 759         if (global_disable)
 760                 return;
 761
 762         nb = kzalloc(sizeof(struct notifier_block), GFP_KERNEL);
 763         if (nb == NULL) {
 764                 pr_err("dma_debug_add_bus: out of memory\n");
 765                 return;
 766         }
 767
 768         nb->notifier_call = dma_debug_device_change;
 769
 770         bus_register_notifier(bus, nb);
 771 }
 772
 773 /*
 774  * Let the architectures decide how many entries should be preallocated.
 775  */
 776 void dma_debug_init(u32 num_entries)
 777 {
 778         int i;
 779
 780         if (global_disable)
 781                 return;
 782
 783         for (i = 0; i < HASH_SIZE; ++i) {
 784                 INIT_LIST_HEAD(&dma_entry_hash[i].list);
 785                 spin_lock_init(&dma_entry_hash[i].lock);
 786         }
 787
 788         if (dma_debug_fs_init() != 0) {
 789                 pr_err("DMA-API: error creating debugfs entries - disabling\n");
 790                 global_disable = true;
 791
 792                 return;
 793         }
 794
 795         if (req_entries)
 796                 num_entries = req_entries;
 797
 798         if (prealloc_memory(num_entries) != 0) {
 799                 pr_err("DMA-API: debugging out of memory error - disabled\n");
 800                 global_disable = true;
 801
 802                 return;
 803         }
 804
 805         nr_total_entries = num_free_entries;
 806
 807         pr_info("DMA-API: debugging enabled by kernel config\n");
 808 }
 809
 810 static __init int dma_debug_cmdline(char *str)
 811 {
 812         if (!str)
 813                 return -EINVAL;
 814
 815         if (strncmp(str, "off", 3) == 0) {
 816                 pr_info("DMA-API: debugging disabled on kernel command line\n");
 817                 global_disable = true;
 818         }
 819
 820         return 0;
 821 }
 822
 823 static __init int dma_debug_entries_cmdline(char *str)
 824 {
 825         int res;
 826
 827         if (!str)
 828                 return -EINVAL;
 829
 830         res = get_option(&str, &req_entries);
 831
 832         if (!res)
 833                 req_entries = 0;
 834
 835         return 0;
 836 }
 837
 838 __setup("dma_debug=", dma_debug_cmdline);
 839 __setup("dma_debug_entries=", dma_debug_entries_cmdline);
 840
 841 static void check_unmap(struct dma_debug_entry *ref)
 842 {
 843         struct dma_debug_entry *entry;
 844         struct hash_bucket *bucket;
 845         unsigned long flags;
 846
 847         if (dma_mapping_error(ref->dev, ref->dev_addr)) {
 848                 err_printk(ref->dev, NULL, "DMA-API: device driver tries "
 849                            "to free an invalid DMA memory address\n");
 850                 return;
 851         }
 852
 853         bucket = get_hash_bucket(ref, &flags);
 854         entry = bucket_find_exact(bucket, ref);
 855
 856         if (!entry) {
 857                 err_printk(ref->dev, NULL, "DMA-API: device driver tries "
 858                            "to free DMA memory it has not allocated "
 859                            "[device address=0x%016llx] [size=%llu bytes]\n",
 860                            ref->dev_addr, ref->size);
 861                 goto out;
 862         }
 863
 864         if (ref->size != entry->size) {
 865                 err_printk(ref->dev, entry, "DMA-API: device driver frees "
 866                            "DMA memory with different size "
 867                            "[device address=0x%016llx] [map size=%llu bytes] "
 868                            "[unmap size=%llu bytes]\n",
 869                            ref->dev_addr, entry->size, ref->size);
 870         }
 871
 872         if (ref->type != entry->type) {
 873                 err_printk(ref->dev, entry, "DMA-API: device driver frees "
 874                            "DMA memory with wrong function "
 875                            "[device address=0x%016llx] [size=%llu bytes] "
 876                            "[mapped as %s] [unmapped as %s]\n",
 877                            ref->dev_addr, ref->size,
 878                            type2name[entry->type], type2name[ref->type]);
 879         } else if ((entry->type == dma_debug_coherent) &&
 880                    (ref->paddr != entry->paddr)) {
 881                 err_printk(ref->dev, entry, "DMA-API: device driver frees "
 882                            "DMA memory with different CPU address "
 883                            "[device address=0x%016llx] [size=%llu bytes] "
 884                            "[cpu alloc address=0x%016llx] "
 885                            "[cpu free address=0x%016llx]",
 886                            ref->dev_addr, ref->size,
 887                            (unsigned long long)entry->paddr,
 888                            (unsigned long long)ref->paddr);
 889         }
 890
 891         if (ref->sg_call_ents && ref->type == dma_debug_sg &&
 892             ref->sg_call_ents != entry->sg_call_ents) {
 893                 err_printk(ref->dev, entry, "DMA-API: device driver frees "
 894                            "DMA sg list with different entry count "
 895                            "[map count=%d] [unmap count=%d]\n",
 896                            entry->sg_call_ents, ref->sg_call_ents);
 897         }
 898
 899         /*
 900          * This may be no bug in reality - but most implementations of the
 901          * DMA API don't handle this properly, so check for it here
 902          */
 903         if (ref->direction != entry->direction) {
 904                 err_printk(ref->dev, entry, "DMA-API: device driver frees "
 905                            "DMA memory with different direction "
 906                            "[device address=0x%016llx] [size=%llu bytes] "
 907                            "[mapped with %s] [unmapped with %s]\n",
 908                            ref->dev_addr, ref->size,
 909                            dir2name[entry->direction],
 910                            dir2name[ref->direction]);
 911         }
 912
 913         hash_bucket_del(entry);
 914         dma_entry_free(entry);
 915
 916 out:
 917         put_hash_bucket(bucket, &flags);
 918 }
 919
 920 static void check_for_stack(struct device *dev, void *addr)
 921 {
 922         if (object_is_on_stack(addr))
 923                 err_printk(dev, NULL, "DMA-API: device driver maps memory from"
 924                                 "stack [addr=%p]\n", addr);
 925 }
 926
 927 static inline bool overlap(void *addr, unsigned long len, void *start, void *end)
 928 {
 929         unsigned long a1 = (unsigned long)addr;
 930         unsigned long b1 = a1 + len;
 931         unsigned long a2 = (unsigned long)start;
 932         unsigned long b2 = (unsigned long)end;
 933
 934         return !(b1 <= a2 || a1 >= b2);
 935 }
 936
 937 static void check_for_illegal_area(struct device *dev, void *addr, unsigned long len)
 938 {
 939         if (overlap(addr, len, _text, _etext) ||
 940             overlap(addr, len, __start_rodata, __end_rodata))
 941                 err_printk(dev, NULL, "DMA-API: device driver maps memory from kernel text or rodata [addr=%p] [len=%lu]\n", addr, len);
 942 }
 943
 944 static void check_sync(struct device *dev,
 945                        struct dma_debug_entry *ref,
 946                        bool to_cpu)
 947 {
 948         struct dma_debug_entry *entry;
 949         struct hash_bucket *bucket;
 950         unsigned long flags;
 951
 952         bucket = get_hash_bucket(ref, &flags);
 953
 954         entry = bucket_find_contain(&bucket, ref, &flags);
 955
 956         if (!entry) {
 957                 err_printk(dev, NULL, "DMA-API: device driver tries "
 958                                 "to sync DMA memory it has not allocated "
 959                                 "[device address=0x%016llx] [size=%llu bytes]\n",
 960                                 (unsigned long long)ref->dev_addr, ref->size);
 961                 goto out;
 962         }
 963
 964         if (ref->size > entry->size) {
 965                 err_printk(dev, entry, "DMA-API: device driver syncs"
 966                                 " DMA memory outside allocated range "
 967                                 "[device address=0x%016llx] "
 968                                 "[allocation size=%llu bytes] "
 969                                 "[sync offset+size=%llu]\n",
 970                                 entry->dev_addr, entry->size,
 971                                 ref->size);
 972         }
 973
 974         if (entry->direction == DMA_BIDIRECTIONAL)
 975                 goto out;
 976
 977         if (ref->direction != entry->direction) {
 978                 err_printk(dev, entry, "DMA-API: device driver syncs "
 979                                 "DMA memory with different direction "
 980                                 "[device address=0x%016llx] [size=%llu bytes] "
 981                                 "[mapped with %s] [synced with %s]\n",
 982                                 (unsigned long long)ref->dev_addr, entry->size,
 983                                 dir2name[entry->direction],
 984                                 dir2name[ref->direction]);
 985         }
 986
 987         if (to_cpu && !(entry->direction == DMA_FROM_DEVICE) &&
 988                       !(ref->direction == DMA_TO_DEVICE))
 989                 err_printk(dev, entry, "DMA-API: device driver syncs "
 990                                 "device read-only DMA memory for cpu "
 991                                 "[device address=0x%016llx] [size=%llu bytes] "
 992                                 "[mapped with %s] [synced with %s]\n",
 993                                 (unsigned long long)ref->dev_addr, entry->size,
 994                                 dir2name[entry->direction],
 995                                 dir2name[ref->direction]);
 996
 997         if (!to_cpu && !(entry->direction == DMA_TO_DEVICE) &&
 998                        !(ref->direction == DMA_FROM_DEVICE))
 999                 err_printk(dev, entry, "DMA-API: device driver syncs "
1000                                 "device write-only DMA memory to device "
1001                                 "[device address=0x%016llx] [size=%llu bytes] "
1002                                 "[mapped with %s] [synced with %s]\n",
1003                                 (unsigned long long)ref->dev_addr, entry->size,
1004                                 dir2name[entry->direction],
1005                                 dir2name[ref->direction]);
1006
1007 out:
1008         put_hash_bucket(bucket, &flags);
1009 }
1010
1011 void debug_dma_map_page(struct device *dev, struct page *page, size_t offset,
1012                         size_t size, int direction, dma_addr_t dma_addr,
1013                         bool map_single)
1014 {
1015         struct dma_debug_entry *entry;
1016
1017         if (unlikely(global_disable))
1018                 return;
1019
1020         if (unlikely(dma_mapping_error(dev, dma_addr)))
1021                 return;
1022
1023         entry = dma_entry_alloc();
1024         if (!entry)
1025                 return;
1026
1027         entry->dev       = dev;
1028         entry->type      = dma_debug_page;
1029         entry->paddr     = page_to_phys(page) + offset;
1030         entry->dev_addr  = dma_addr;
1031         entry->size      = size;
1032         entry->direction = direction;
1033
1034         if (map_single)
1035                 entry->type = dma_debug_single;
1036
1037         if (!PageHighMem(page)) {
1038                 void *addr = page_address(page) + offset;
1039
1040                 check_for_stack(dev, addr);
1041                 check_for_illegal_area(dev, addr, size);
1042         }
1043
1044         add_dma_entry(entry);
1045 }
1046 EXPORT_SYMBOL(debug_dma_map_page);
1047
1048 void debug_dma_unmap_page(struct device *dev, dma_addr_t addr,
1049                           size_t size, int direction, bool map_single)
1050 {
1051         struct dma_debug_entry ref = {
1052                 .type           = dma_debug_page,
1053                 .dev            = dev,
1054                 .dev_addr       = addr,
1055                 .size           = size,
1056                 .direction      = direction,
1057         };
1058
1059         if (unlikely(global_disable))
1060                 return;
1061
1062         if (map_single)
1063                 ref.type = dma_debug_single;
1064
1065         check_unmap(&ref);
1066 }
1067 EXPORT_SYMBOL(debug_dma_unmap_page);
1068
1069 void debug_dma_map_sg(struct device *dev, struct scatterlist *sg,
1070                       int nents, int mapped_ents, int direction)
1071 {
1072         struct dma_debug_entry *entry;
1073         struct scatterlist *s;
1074         int i;
1075
1076         if (unlikely(global_disable))
1077                 return;
1078
1079         for_each_sg(sg, s, mapped_ents, i) {
1080                 entry = dma_entry_alloc();
1081                 if (!entry)
1082                         return;
1083
1084                 entry->type           = dma_debug_sg;
1085                 entry->dev            = dev;
1086                 entry->paddr          = sg_phys(s);
1087                 entry->size           = sg_dma_len(s);
1088                 entry->dev_addr       = sg_dma_address(s);
1089                 entry->direction      = direction;
1090                 entry->sg_call_ents   = nents;
1091                 entry->sg_mapped_ents = mapped_ents;
1092
1093                 if (!PageHighMem(sg_page(s))) {
1094                         check_for_stack(dev, sg_virt(s));
1095                         check_for_illegal_area(dev, sg_virt(s), sg_dma_len(s));
1096                 }
1097
1098                 add_dma_entry(entry);
1099         }
1100 }
1101 EXPORT_SYMBOL(debug_dma_map_sg);
1102
1103 static int get_nr_mapped_entries(struct device *dev,
1104                                  struct dma_debug_entry *ref)
1105 {
1106         struct dma_debug_entry *entry;
1107         struct hash_bucket *bucket;
1108         unsigned long flags;
1109         int mapped_ents;
1110
1111         bucket       = get_hash_bucket(ref, &flags);
1112         entry        = bucket_find_exact(bucket, ref);
1113         mapped_ents  = 0;
1114
1115         if (entry)
1116                 mapped_ents = entry->sg_mapped_ents;
1117         put_hash_bucket(bucket, &flags);
1118
1119         return mapped_ents;
1120 }
1121
1122 void debug_dma_unmap_sg(struct device *dev, struct scatterlist *sglist,
1123                         int nelems, int dir)
1124 {
1125         struct scatterlist *s;
1126         int mapped_ents = 0, i;
1127
1128         if (unlikely(global_disable))
1129                 return;
1130
1131         for_each_sg(sglist, s, nelems, i) {
1132
1133                 struct dma_debug_entry ref = {
1134                         .type           = dma_debug_sg,
1135                         .dev            = dev,
1136                         .paddr          = sg_phys(s),
1137                         .dev_addr       = sg_dma_address(s),
1138                         .size           = sg_dma_len(s),
1139                         .direction      = dir,
1140                         .sg_call_ents   = nelems,
1141                 };
1142
1143                 if (mapped_ents && i >= mapped_ents)
1144                         break;
1145
1146                 if (!i)
1147                         mapped_ents = get_nr_mapped_entries(dev, &ref);
1148
1149                 check_unmap(&ref);
1150         }
1151 }
1152 EXPORT_SYMBOL(debug_dma_unmap_sg);
1153
1154 void debug_dma_alloc_coherent(struct device *dev, size_t size,
1155                               dma_addr_t dma_addr, void *virt)
1156 {
1157         struct dma_debug_entry *entry;
1158
1159         if (unlikely(global_disable))
1160                 return;
1161
1162         if (unlikely(virt == NULL))
1163                 return;
1164
1165         entry = dma_entry_alloc();
1166         if (!entry)
1167                 return;
1168
1169         entry->type      = dma_debug_coherent;
1170         entry->dev       = dev;
1171         entry->paddr     = virt_to_phys(virt);
1172         entry->size      = size;
1173         entry->dev_addr  = dma_addr;
1174         entry->direction = DMA_BIDIRECTIONAL;
1175
1176         add_dma_entry(entry);
1177 }
1178 EXPORT_SYMBOL(debug_dma_alloc_coherent);
1179
1180 void debug_dma_free_coherent(struct device *dev, size_t size,
1181                          void *virt, dma_addr_t addr)
1182 {
1183         struct dma_debug_entry ref = {
1184                 .type           = dma_debug_coherent,
1185                 .dev            = dev,
1186                 .paddr          = virt_to_phys(virt),
1187                 .dev_addr       = addr,
1188                 .size           = size,
1189                 .direction      = DMA_BIDIRECTIONAL,
1190         };
1191
1192         if (unlikely(global_disable))
1193                 return;
1194
1195         check_unmap(&ref);
1196 }
1197 EXPORT_SYMBOL(debug_dma_free_coherent);
1198
1199 void debug_dma_sync_single_for_cpu(struct device *dev, dma_addr_t dma_handle,
1200                                    size_t size, int direction)
1201 {
1202         struct dma_debug_entry ref;
1203
1204         if (unlikely(global_disable))
1205                 return;
1206
1207         ref.type         = dma_debug_single;
1208         ref.dev          = dev;
1209         ref.dev_addr     = dma_handle;
1210         ref.size         = size;
1211         ref.direction    = direction;
1212         ref.sg_call_ents = 0;
1213
1214         check_sync(dev, &ref, true);
1215 }
1216 EXPORT_SYMBOL(debug_dma_sync_single_for_cpu);
1217
1218 void debug_dma_sync_single_for_device(struct device *dev,
1219                                       dma_addr_t dma_handle, size_t size,
1220                                       int direction)
1221 {
1222         struct dma_debug_entry ref;
1223
1224         if (unlikely(global_disable))
1225                 return;
1226
1227         ref.type         = dma_debug_single;
1228         ref.dev          = dev;
1229         ref.dev_addr     = dma_handle;
1230         ref.size         = size;
1231         ref.direction    = direction;
1232         ref.sg_call_ents = 0;
1233
1234         check_sync(dev, &ref, false);
1235 }
1236 EXPORT_SYMBOL(debug_dma_sync_single_for_device);
1237
1238 void debug_dma_sync_single_range_for_cpu(struct device *dev,
1239                                          dma_addr_t dma_handle,
1240                                          unsigned long offset, size_t size,
1241                                          int direction)
1242 {
1243         struct dma_debug_entry ref;
1244
1245         if (unlikely(global_disable))
1246                 return;
1247
1248         ref.type         = dma_debug_single;
1249         ref.dev          = dev;
1250         ref.dev_addr     = dma_handle;
1251         ref.size         = offset + size;
1252         ref.direction    = direction;
1253         ref.sg_call_ents = 0;
1254
1255         check_sync(dev, &ref, true);
1256 }
1257 EXPORT_SYMBOL(debug_dma_sync_single_range_for_cpu);
1258
1259 void debug_dma_sync_single_range_for_device(struct device *dev,
1260                                             dma_addr_t dma_handle,
1261                                             unsigned long offset,
1262                                             size_t size, int direction)
1263 {
1264         struct dma_debug_entry ref;
1265
1266         if (unlikely(global_disable))
1267                 return;
1268
1269         ref.type         = dma_debug_single;
1270         ref.dev          = dev;
1271         ref.dev_addr     = dma_handle;
1272         ref.size         = offset + size;
1273         ref.direction    = direction;
1274         ref.sg_call_ents = 0;
1275
1276         check_sync(dev, &ref, false);
1277 }
1278 EXPORT_SYMBOL(debug_dma_sync_single_range_for_device);
1279
1280 void debug_dma_sync_sg_for_cpu(struct device *dev, struct scatterlist *sg,
1281                                int nelems, int direction)
1282 {
1283         struct scatterlist *s;
1284         int mapped_ents = 0, i;
1285
1286         if (unlikely(global_disable))
1287                 return;
1288
1289         for_each_sg(sg, s, nelems, i) {
1290
1291                 struct dma_debug_entry ref = {
1292                         .type           = dma_debug_sg,
1293                         .dev            = dev,
1294                         .paddr          = sg_phys(s),
1295                         .dev_addr       = sg_dma_address(s),
1296                         .size           = sg_dma_len(s),
1297                         .direction      = direction,
1298                         .sg_call_ents   = nelems,
1299                 };
1300
1301                 if (!i)
1302                         mapped_ents = get_nr_mapped_entries(dev, &ref);
1303
1304                 if (i >= mapped_ents)
1305                         break;
1306
1307                 check_sync(dev, &ref, true);
1308         }
1309 }
1310 EXPORT_SYMBOL(debug_dma_sync_sg_for_cpu);
1311
1312 void debug_dma_sync_sg_for_device(struct device *dev, struct scatterlist *sg,
1313                                   int nelems, int direction)
1314 {
1315         struct scatterlist *s;
1316         int mapped_ents = 0, i;
1317
1318         if (unlikely(global_disable))
1319                 return;
1320
1321         for_each_sg(sg, s, nelems, i) {
1322
1323                 struct dma_debug_entry ref = {
1324                         .type           = dma_debug_sg,
1325                         .dev            = dev,
1326                         .paddr          = sg_phys(s),
1327                         .dev_addr       = sg_dma_address(s),
1328                         .size           = sg_dma_len(s),
1329                         .direction      = direction,
1330                         .sg_call_ents   = nelems,
1331                 };
1332                 if (!i)
1333                         mapped_ents = get_nr_mapped_entries(dev, &ref);
1334
1335                 if (i >= mapped_ents)
1336                         break;
1337
1338                 check_sync(dev, &ref, false);
1339         }
1340 }
1341 EXPORT_SYMBOL(debug_dma_sync_sg_for_device);
1342
1343 static int __init dma_debug_driver_setup(char *str)
1344 {
1345         int i;
1346
1347         for (i = 0; i < NAME_MAX_LEN - 1; ++i, ++str) {
1348                 current_driver_name[i] = *str;
1349                 if (*str == 0)
1350                         break;
1351         }
1352
1353         if (current_driver_name[0])
1354                 pr_info("DMA-API: enable driver filter for driver [%s]\n",
1355                         current_driver_name);
1356
1357
1358         return 1;
1359 }
1360 __setup("dma_debug_driver=", dma_debug_driver_setup);