SAFE public projects git trees. - safe/jmp/linux-2.6/blob - lib/dma-debug.c

   1 /*
   2  * Copyright (C) 2008 Advanced Micro Devices, Inc.
   3  *
   4  * Author: Joerg Roedel <joerg.roedel@amd.com>
   5  *
   6  * This program is free software; you can redistribute it and/or modify it
   7  * under the terms of the GNU General Public License version 2 as published
   8  * by the Free Software Foundation.
   9  *
  10  * This program is distributed in the hope that it will be useful,
  11  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  12  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  13  * GNU General Public License for more details.
  14  *
  15  * You should have received a copy of the GNU General Public License
  16  * along with this program; if not, write to the Free Software
  17  * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307 USA
  18  */
  19
  20 #include <linux/scatterlist.h>
  21 #include <linux/dma-mapping.h>
  22 #include <linux/stacktrace.h>
  23 #include <linux/dma-debug.h>
  24 #include <linux/spinlock.h>
  25 #include <linux/debugfs.h>
  26 #include <linux/device.h>
  27 #include <linux/types.h>
  28 #include <linux/sched.h>
  29 #include <linux/list.h>
  30 #include <linux/slab.h>
  31
  32 #include <asm/sections.h>
  33
  34 #define HASH_SIZE       1024ULL
  35 #define HASH_FN_SHIFT   13
  36 #define HASH_FN_MASK    (HASH_SIZE - 1)
  37
  38 enum {
  39         dma_debug_single,
  40         dma_debug_page,
  41         dma_debug_sg,
  42         dma_debug_coherent,
  43 };
  44
  45 #define DMA_DEBUG_STACKTRACE_ENTRIES 5
  46
  47 struct dma_debug_entry {
  48         struct list_head list;
  49         struct device    *dev;
  50         int              type;
  51         phys_addr_t      paddr;
  52         u64              dev_addr;
  53         u64              size;
  54         int              direction;
  55         int              sg_call_ents;
  56         int              sg_mapped_ents;
  57 #ifdef CONFIG_STACKTRACE
  58         struct           stack_trace stacktrace;
  59         unsigned long    st_entries[DMA_DEBUG_STACKTRACE_ENTRIES];
  60 #endif
  61 };
  62
  63 struct hash_bucket {
  64         struct list_head list;
  65         spinlock_t lock;
  66 } ____cacheline_aligned_in_smp;
  67
  68 /* Hash list to save the allocated dma addresses */
  69 static struct hash_bucket dma_entry_hash[HASH_SIZE];
  70 /* List of pre-allocated dma_debug_entry's */
  71 static LIST_HEAD(free_entries);
  72 /* Lock for the list above */
  73 static DEFINE_SPINLOCK(free_entries_lock);
  74
  75 /* Global disable flag - will be set in case of an error */
  76 static bool global_disable __read_mostly;
  77
  78 /* Global error count */
  79 static u32 error_count;
  80
  81 /* Global error show enable*/
  82 static u32 show_all_errors __read_mostly;
  83 /* Number of errors to show */
  84 static u32 show_num_errors = 1;
  85
  86 static u32 num_free_entries;
  87 static u32 min_free_entries;
  88 static u32 nr_total_entries;
  89
  90 /* number of preallocated entries requested by kernel cmdline */
  91 static u32 req_entries;
  92
  93 /* debugfs dentry's for the stuff above */
  94 static struct dentry *dma_debug_dent        __read_mostly;
  95 static struct dentry *global_disable_dent   __read_mostly;
  96 static struct dentry *error_count_dent      __read_mostly;
  97 static struct dentry *show_all_errors_dent  __read_mostly;
  98 static struct dentry *show_num_errors_dent  __read_mostly;
  99 static struct dentry *num_free_entries_dent __read_mostly;
 100 static struct dentry *min_free_entries_dent __read_mostly;
 101
 102 static const char *type2name[4] = { "single", "page",
 103                                     "scather-gather", "coherent" };
 104
 105 static const char *dir2name[4] = { "DMA_BIDIRECTIONAL", "DMA_TO_DEVICE",
 106                                    "DMA_FROM_DEVICE", "DMA_NONE" };
 107
 108 /*
 109  * The access to some variables in this macro is racy. We can't use atomic_t
 110  * here because all these variables are exported to debugfs. Some of them even
 111  * writeable. This is also the reason why a lock won't help much. But anyway,
 112  * the races are no big deal. Here is why:
 113  *
 114  *   error_count: the addition is racy, but the worst thing that can happen is
 115  *                that we don't count some errors
 116  *   show_num_errors: the subtraction is racy. Also no big deal because in
 117  *                    worst case this will result in one warning more in the
 118  *                    system log than the user configured. This variable is
 119  *                    writeable via debugfs.
 120  */
 121 static inline void dump_entry_trace(struct dma_debug_entry *entry)
 122 {
 123 #ifdef CONFIG_STACKTRACE
 124         if (entry) {
 125                 printk(KERN_WARNING "Mapped at:\n");
 126                 print_stack_trace(&entry->stacktrace, 0);
 127         }
 128 #endif
 129 }
 130
 131 #define err_printk(dev, entry, format, arg...) do {             \
 132                 error_count += 1;                               \
 133                 if (show_all_errors || show_num_errors > 0) {   \
 134                         WARN(1, "%s %s: " format,               \
 135                              dev_driver_string(dev),            \
 136                              dev_name(dev) , ## arg);           \
 137                         dump_entry_trace(entry);                \
 138                 }                                               \
 139                 if (!show_all_errors && show_num_errors > 0)    \
 140                         show_num_errors -= 1;                   \
 141         } while (0);
 142
 143 /*
 144  * Hash related functions
 145  *
 146  * Every DMA-API request is saved into a struct dma_debug_entry. To
 147  * have quick access to these structs they are stored into a hash.
 148  */
 149 static int hash_fn(struct dma_debug_entry *entry)
 150 {
 151         /*
 152          * Hash function is based on the dma address.
 153          * We use bits 20-27 here as the index into the hash
 154          */
 155         return (entry->dev_addr >> HASH_FN_SHIFT) & HASH_FN_MASK;
 156 }
 157
 158 /*
 159  * Request exclusive access to a hash bucket for a given dma_debug_entry.
 160  */
 161 static struct hash_bucket *get_hash_bucket(struct dma_debug_entry *entry,
 162                                            unsigned long *flags)
 163 {
 164         int idx = hash_fn(entry);
 165         unsigned long __flags;
 166
 167         spin_lock_irqsave(&dma_entry_hash[idx].lock, __flags);
 168         *flags = __flags;
 169         return &dma_entry_hash[idx];
 170 }
 171
 172 /*
 173  * Give up exclusive access to the hash bucket
 174  */
 175 static void put_hash_bucket(struct hash_bucket *bucket,
 176                             unsigned long *flags)
 177 {
 178         unsigned long __flags = *flags;
 179
 180         spin_unlock_irqrestore(&bucket->lock, __flags);
 181 }
 182
 183 /*
 184  * Search a given entry in the hash bucket list
 185  */
 186 static struct dma_debug_entry *hash_bucket_find(struct hash_bucket *bucket,
 187                                                 struct dma_debug_entry *ref)
 188 {
 189         struct dma_debug_entry *entry, *ret = NULL;
 190         int matches = 0, match_lvl, last_lvl = 0;
 191
 192         list_for_each_entry(entry, &bucket->list, list) {
 193                 if ((entry->dev_addr != ref->dev_addr) ||
 194                     (entry->dev != ref->dev))
 195                         continue;
 196
 197                 /*
 198                  * Some drivers map the same physical address multiple
 199                  * times. Without a hardware IOMMU this results in the
 200                  * same device addresses being put into the dma-debug
 201                  * hash multiple times too. This can result in false
 202                  * positives being reported. Therfore we implement a
 203                  * best-fit algorithm here which returns the entry from
 204                  * the hash which fits best to the reference value
 205                  * instead of the first-fit.
 206                  */
 207                 matches += 1;
 208                 match_lvl = 0;
 209                 entry->size      == ref->size      ? ++match_lvl : match_lvl;
 210                 entry->type      == ref->type      ? ++match_lvl : match_lvl;
 211                 entry->direction == ref->direction ? ++match_lvl : match_lvl;
 212
 213                 if (match_lvl == 3) {
 214                         /* perfect-fit - return the result */
 215                         return entry;
 216                 } else if (match_lvl > last_lvl) {
 217                         /*
 218                          * We found an entry that fits better then the
 219                          * previous one
 220                          */
 221                         last_lvl = match_lvl;
 222                         ret      = entry;
 223                 }
 224         }
 225
 226         /*
 227          * If we have multiple matches but no perfect-fit, just return
 228          * NULL.
 229          */
 230         ret = (matches == 1) ? ret : NULL;
 231
 232         return ret;
 233 }
 234
 235 /*
 236  * Add an entry to a hash bucket
 237  */
 238 static void hash_bucket_add(struct hash_bucket *bucket,
 239                             struct dma_debug_entry *entry)
 240 {
 241         list_add_tail(&entry->list, &bucket->list);
 242 }
 243
 244 /*
 245  * Remove entry from a hash bucket list
 246  */
 247 static void hash_bucket_del(struct dma_debug_entry *entry)
 248 {
 249         list_del(&entry->list);
 250 }
 251
 252 /*
 253  * Dump mapping entries for debugging purposes
 254  */
 255 void debug_dma_dump_mappings(struct device *dev)
 256 {
 257         int idx;
 258
 259         for (idx = 0; idx < HASH_SIZE; idx++) {
 260                 struct hash_bucket *bucket = &dma_entry_hash[idx];
 261                 struct dma_debug_entry *entry;
 262                 unsigned long flags;
 263
 264                 spin_lock_irqsave(&bucket->lock, flags);
 265
 266                 list_for_each_entry(entry, &bucket->list, list) {
 267                         if (!dev || dev == entry->dev) {
 268                                 dev_info(entry->dev,
 269                                          "%s idx %d P=%Lx D=%Lx L=%Lx %s\n",
 270                                          type2name[entry->type], idx,
 271                                          (unsigned long long)entry->paddr,
 272                                          entry->dev_addr, entry->size,
 273                                          dir2name[entry->direction]);
 274                         }
 275                 }
 276
 277                 spin_unlock_irqrestore(&bucket->lock, flags);
 278         }
 279 }
 280 EXPORT_SYMBOL(debug_dma_dump_mappings);
 281
 282 /*
 283  * Wrapper function for adding an entry to the hash.
 284  * This function takes care of locking itself.
 285  */
 286 static void add_dma_entry(struct dma_debug_entry *entry)
 287 {
 288         struct hash_bucket *bucket;
 289         unsigned long flags;
 290
 291         bucket = get_hash_bucket(entry, &flags);
 292         hash_bucket_add(bucket, entry);
 293         put_hash_bucket(bucket, &flags);
 294 }
 295
 296 static struct dma_debug_entry *__dma_entry_alloc(void)
 297 {
 298         struct dma_debug_entry *entry;
 299
 300         entry = list_entry(free_entries.next, struct dma_debug_entry, list);
 301         list_del(&entry->list);
 302         memset(entry, 0, sizeof(*entry));
 303
 304         num_free_entries -= 1;
 305         if (num_free_entries < min_free_entries)
 306                 min_free_entries = num_free_entries;
 307
 308         return entry;
 309 }
 310
 311 /* struct dma_entry allocator
 312  *
 313  * The next two functions implement the allocator for
 314  * struct dma_debug_entries.
 315  */
 316 static struct dma_debug_entry *dma_entry_alloc(void)
 317 {
 318         struct dma_debug_entry *entry = NULL;
 319         unsigned long flags;
 320
 321         spin_lock_irqsave(&free_entries_lock, flags);
 322
 323         if (list_empty(&free_entries)) {
 324                 printk(KERN_ERR "DMA-API: debugging out of memory "
 325                                 "- disabling\n");
 326                 global_disable = true;
 327                 goto out;
 328         }
 329
 330         entry = __dma_entry_alloc();
 331
 332 #ifdef CONFIG_STACKTRACE
 333         entry->stacktrace.max_entries = DMA_DEBUG_STACKTRACE_ENTRIES;
 334         entry->stacktrace.entries = entry->st_entries;
 335         entry->stacktrace.skip = 2;
 336         save_stack_trace(&entry->stacktrace);
 337 #endif
 338
 339 out:
 340         spin_unlock_irqrestore(&free_entries_lock, flags);
 341
 342         return entry;
 343 }
 344
 345 static void dma_entry_free(struct dma_debug_entry *entry)
 346 {
 347         unsigned long flags;
 348
 349         /*
 350          * add to beginning of the list - this way the entries are
 351          * more likely cache hot when they are reallocated.
 352          */
 353         spin_lock_irqsave(&free_entries_lock, flags);
 354         list_add(&entry->list, &free_entries);
 355         num_free_entries += 1;
 356         spin_unlock_irqrestore(&free_entries_lock, flags);
 357 }
 358
 359 int dma_debug_resize_entries(u32 num_entries)
 360 {
 361         int i, delta, ret = 0;
 362         unsigned long flags;
 363         struct dma_debug_entry *entry;
 364         LIST_HEAD(tmp);
 365
 366         spin_lock_irqsave(&free_entries_lock, flags);
 367
 368         if (nr_total_entries < num_entries) {
 369                 delta = num_entries - nr_total_entries;
 370
 371                 spin_unlock_irqrestore(&free_entries_lock, flags);
 372
 373                 for (i = 0; i < delta; i++) {
 374                         entry = kzalloc(sizeof(*entry), GFP_KERNEL);
 375                         if (!entry)
 376                                 break;
 377
 378                         list_add_tail(&entry->list, &tmp);
 379                 }
 380
 381                 spin_lock_irqsave(&free_entries_lock, flags);
 382
 383                 list_splice(&tmp, &free_entries);
 384                 nr_total_entries += i;
 385                 num_free_entries += i;
 386         } else {
 387                 delta = nr_total_entries - num_entries;
 388
 389                 for (i = 0; i < delta && !list_empty(&free_entries); i++) {
 390                         entry = __dma_entry_alloc();
 391                         kfree(entry);
 392                 }
 393
 394                 nr_total_entries -= i;
 395         }
 396
 397         if (nr_total_entries != num_entries)
 398                 ret = 1;
 399
 400         spin_unlock_irqrestore(&free_entries_lock, flags);
 401
 402         return ret;
 403 }
 404 EXPORT_SYMBOL(dma_debug_resize_entries);
 405
 406 /*
 407  * DMA-API debugging init code
 408  *
 409  * The init code does two things:
 410  *   1. Initialize core data structures
 411  *   2. Preallocate a given number of dma_debug_entry structs
 412  */
 413
 414 static int prealloc_memory(u32 num_entries)
 415 {
 416         struct dma_debug_entry *entry, *next_entry;
 417         int i;
 418
 419         for (i = 0; i < num_entries; ++i) {
 420                 entry = kzalloc(sizeof(*entry), GFP_KERNEL);
 421                 if (!entry)
 422                         goto out_err;
 423
 424                 list_add_tail(&entry->list, &free_entries);
 425         }
 426
 427         num_free_entries = num_entries;
 428         min_free_entries = num_entries;
 429
 430         printk(KERN_INFO "DMA-API: preallocated %d debug entries\n",
 431                         num_entries);
 432
 433         return 0;
 434
 435 out_err:
 436
 437         list_for_each_entry_safe(entry, next_entry, &free_entries, list) {
 438                 list_del(&entry->list);
 439                 kfree(entry);
 440         }
 441
 442         return -ENOMEM;
 443 }
 444
 445 static int dma_debug_fs_init(void)
 446 {
 447         dma_debug_dent = debugfs_create_dir("dma-api", NULL);
 448         if (!dma_debug_dent) {
 449                 printk(KERN_ERR "DMA-API: can not create debugfs directory\n");
 450                 return -ENOMEM;
 451         }
 452
 453         global_disable_dent = debugfs_create_bool("disabled", 0444,
 454                         dma_debug_dent,
 455                         (u32 *)&global_disable);
 456         if (!global_disable_dent)
 457                 goto out_err;
 458
 459         error_count_dent = debugfs_create_u32("error_count", 0444,
 460                         dma_debug_dent, &error_count);
 461         if (!error_count_dent)
 462                 goto out_err;
 463
 464         show_all_errors_dent = debugfs_create_u32("all_errors", 0644,
 465                         dma_debug_dent,
 466                         &show_all_errors);
 467         if (!show_all_errors_dent)
 468                 goto out_err;
 469
 470         show_num_errors_dent = debugfs_create_u32("num_errors", 0644,
 471                         dma_debug_dent,
 472                         &show_num_errors);
 473         if (!show_num_errors_dent)
 474                 goto out_err;
 475
 476         num_free_entries_dent = debugfs_create_u32("num_free_entries", 0444,
 477                         dma_debug_dent,
 478                         &num_free_entries);
 479         if (!num_free_entries_dent)
 480                 goto out_err;
 481
 482         min_free_entries_dent = debugfs_create_u32("min_free_entries", 0444,
 483                         dma_debug_dent,
 484                         &min_free_entries);
 485         if (!min_free_entries_dent)
 486                 goto out_err;
 487
 488         return 0;
 489
 490 out_err:
 491         debugfs_remove_recursive(dma_debug_dent);
 492
 493         return -ENOMEM;
 494 }
 495
 496 void dma_debug_add_bus(struct bus_type *bus)
 497 {
 498         /* FIXME: register notifier */
 499 }
 500
 501 /*
 502  * Let the architectures decide how many entries should be preallocated.
 503  */
 504 void dma_debug_init(u32 num_entries)
 505 {
 506         int i;
 507
 508         if (global_disable)
 509                 return;
 510
 511         for (i = 0; i < HASH_SIZE; ++i) {
 512                 INIT_LIST_HEAD(&dma_entry_hash[i].list);
 513                 dma_entry_hash[i].lock = SPIN_LOCK_UNLOCKED;
 514         }
 515
 516         if (dma_debug_fs_init() != 0) {
 517                 printk(KERN_ERR "DMA-API: error creating debugfs entries "
 518                                 "- disabling\n");
 519                 global_disable = true;
 520
 521                 return;
 522         }
 523
 524         if (req_entries)
 525                 num_entries = req_entries;
 526
 527         if (prealloc_memory(num_entries) != 0) {
 528                 printk(KERN_ERR "DMA-API: debugging out of memory error "
 529                                 "- disabled\n");
 530                 global_disable = true;
 531
 532                 return;
 533         }
 534
 535         nr_total_entries = num_free_entries;
 536
 537         printk(KERN_INFO "DMA-API: debugging enabled by kernel config\n");
 538 }
 539
 540 static __init int dma_debug_cmdline(char *str)
 541 {
 542         if (!str)
 543                 return -EINVAL;
 544
 545         if (strncmp(str, "off", 3) == 0) {
 546                 printk(KERN_INFO "DMA-API: debugging disabled on kernel "
 547                                  "command line\n");
 548                 global_disable = true;
 549         }
 550
 551         return 0;
 552 }
 553
 554 static __init int dma_debug_entries_cmdline(char *str)
 555 {
 556         int res;
 557
 558         if (!str)
 559                 return -EINVAL;
 560
 561         res = get_option(&str, &req_entries);
 562
 563         if (!res)
 564                 req_entries = 0;
 565
 566         return 0;
 567 }
 568
 569 __setup("dma_debug=", dma_debug_cmdline);
 570 __setup("dma_debug_entries=", dma_debug_entries_cmdline);
 571
 572 static void check_unmap(struct dma_debug_entry *ref)
 573 {
 574         struct dma_debug_entry *entry;
 575         struct hash_bucket *bucket;
 576         unsigned long flags;
 577
 578         if (dma_mapping_error(ref->dev, ref->dev_addr)) {
 579                 err_printk(ref->dev, NULL, "DMA-API: device driver tries "
 580                            "to free an invalid DMA memory address\n");
 581                 return;
 582         }
 583
 584         bucket = get_hash_bucket(ref, &flags);
 585         entry = hash_bucket_find(bucket, ref);
 586
 587         if (!entry) {
 588                 err_printk(ref->dev, NULL, "DMA-API: device driver tries "
 589                            "to free DMA memory it has not allocated "
 590                            "[device address=0x%016llx] [size=%llu bytes]\n",
 591                            ref->dev_addr, ref->size);
 592                 goto out;
 593         }
 594
 595         if (ref->size != entry->size) {
 596                 err_printk(ref->dev, entry, "DMA-API: device driver frees "
 597                            "DMA memory with different size "
 598                            "[device address=0x%016llx] [map size=%llu bytes] "
 599                            "[unmap size=%llu bytes]\n",
 600                            ref->dev_addr, entry->size, ref->size);
 601         }
 602
 603         if (ref->type != entry->type) {
 604                 err_printk(ref->dev, entry, "DMA-API: device driver frees "
 605                            "DMA memory with wrong function "
 606                            "[device address=0x%016llx] [size=%llu bytes] "
 607                            "[mapped as %s] [unmapped as %s]\n",
 608                            ref->dev_addr, ref->size,
 609                            type2name[entry->type], type2name[ref->type]);
 610         } else if ((entry->type == dma_debug_coherent) &&
 611                    (ref->paddr != entry->paddr)) {
 612                 err_printk(ref->dev, entry, "DMA-API: device driver frees "
 613                            "DMA memory with different CPU address "
 614                            "[device address=0x%016llx] [size=%llu bytes] "
 615                            "[cpu alloc address=%p] [cpu free address=%p]",
 616                            ref->dev_addr, ref->size,
 617                            (void *)entry->paddr, (void *)ref->paddr);
 618         }
 619
 620         if (ref->sg_call_ents && ref->type == dma_debug_sg &&
 621             ref->sg_call_ents != entry->sg_call_ents) {
 622                 err_printk(ref->dev, entry, "DMA-API: device driver frees "
 623                            "DMA sg list with different entry count "
 624                            "[map count=%d] [unmap count=%d]\n",
 625                            entry->sg_call_ents, ref->sg_call_ents);
 626         }
 627
 628         /*
 629          * This may be no bug in reality - but most implementations of the
 630          * DMA API don't handle this properly, so check for it here
 631          */
 632         if (ref->direction != entry->direction) {
 633                 err_printk(ref->dev, entry, "DMA-API: device driver frees "
 634                            "DMA memory with different direction "
 635                            "[device address=0x%016llx] [size=%llu bytes] "
 636                            "[mapped with %s] [unmapped with %s]\n",
 637                            ref->dev_addr, ref->size,
 638                            dir2name[entry->direction],
 639                            dir2name[ref->direction]);
 640         }
 641
 642         hash_bucket_del(entry);
 643         dma_entry_free(entry);
 644
 645 out:
 646         put_hash_bucket(bucket, &flags);
 647 }
 648
 649 static void check_for_stack(struct device *dev, void *addr)
 650 {
 651         if (object_is_on_stack(addr))
 652                 err_printk(dev, NULL, "DMA-API: device driver maps memory from"
 653                                 "stack [addr=%p]\n", addr);
 654 }
 655
 656 static inline bool overlap(void *addr, u64 size, void *start, void *end)
 657 {
 658         void *addr2 = (char *)addr + size;
 659
 660         return ((addr >= start && addr < end) ||
 661                 (addr2 >= start && addr2 < end) ||
 662                 ((addr < start) && (addr2 >= end)));
 663 }
 664
 665 static void check_for_illegal_area(struct device *dev, void *addr, u64 size)
 666 {
 667         if (overlap(addr, size, _text, _etext) ||
 668             overlap(addr, size, __start_rodata, __end_rodata))
 669                 err_printk(dev, NULL, "DMA-API: device driver maps "
 670                                 "memory from kernel text or rodata "
 671                                 "[addr=%p] [size=%llu]\n", addr, size);
 672 }
 673
 674 static void check_sync(struct device *dev, dma_addr_t addr,
 675                        u64 size, u64 offset, int direction, bool to_cpu)
 676 {
 677         struct dma_debug_entry ref = {
 678                 .dev            = dev,
 679                 .dev_addr       = addr,
 680                 .size           = size,
 681                 .direction      = direction,
 682         };
 683         struct dma_debug_entry *entry;
 684         struct hash_bucket *bucket;
 685         unsigned long flags;
 686
 687         bucket = get_hash_bucket(&ref, &flags);
 688
 689         entry = hash_bucket_find(bucket, &ref);
 690
 691         if (!entry) {
 692                 err_printk(dev, NULL, "DMA-API: device driver tries "
 693                                 "to sync DMA memory it has not allocated "
 694                                 "[device address=0x%016llx] [size=%llu bytes]\n",
 695                                 (unsigned long long)addr, size);
 696                 goto out;
 697         }
 698
 699         if ((offset + size) > entry->size) {
 700                 err_printk(dev, entry, "DMA-API: device driver syncs"
 701                                 " DMA memory outside allocated range "
 702                                 "[device address=0x%016llx] "
 703                                 "[allocation size=%llu bytes] [sync offset=%llu] "
 704                                 "[sync size=%llu]\n", entry->dev_addr, entry->size,
 705                                 offset, size);
 706         }
 707
 708         if (direction != entry->direction) {
 709                 err_printk(dev, entry, "DMA-API: device driver syncs "
 710                                 "DMA memory with different direction "
 711                                 "[device address=0x%016llx] [size=%llu bytes] "
 712                                 "[mapped with %s] [synced with %s]\n",
 713                                 (unsigned long long)addr, entry->size,
 714                                 dir2name[entry->direction],
 715                                 dir2name[direction]);
 716         }
 717
 718         if (entry->direction == DMA_BIDIRECTIONAL)
 719                 goto out;
 720
 721         if (to_cpu && !(entry->direction == DMA_FROM_DEVICE) &&
 722                       !(direction == DMA_TO_DEVICE))
 723                 err_printk(dev, entry, "DMA-API: device driver syncs "
 724                                 "device read-only DMA memory for cpu "
 725                                 "[device address=0x%016llx] [size=%llu bytes] "
 726                                 "[mapped with %s] [synced with %s]\n",
 727                                 (unsigned long long)addr, entry->size,
 728                                 dir2name[entry->direction],
 729                                 dir2name[direction]);
 730
 731         if (!to_cpu && !(entry->direction == DMA_TO_DEVICE) &&
 732                        !(direction == DMA_FROM_DEVICE))
 733                 err_printk(dev, entry, "DMA-API: device driver syncs "
 734                                 "device write-only DMA memory to device "
 735                                 "[device address=0x%016llx] [size=%llu bytes] "
 736                                 "[mapped with %s] [synced with %s]\n",
 737                                 (unsigned long long)addr, entry->size,
 738                                 dir2name[entry->direction],
 739                                 dir2name[direction]);
 740
 741 out:
 742         put_hash_bucket(bucket, &flags);
 743
 744 }
 745
 746 void debug_dma_map_page(struct device *dev, struct page *page, size_t offset,
 747                         size_t size, int direction, dma_addr_t dma_addr,
 748                         bool map_single)
 749 {
 750         struct dma_debug_entry *entry;
 751
 752         if (unlikely(global_disable))
 753                 return;
 754
 755         if (unlikely(dma_mapping_error(dev, dma_addr)))
 756                 return;
 757
 758         entry = dma_entry_alloc();
 759         if (!entry)
 760                 return;
 761
 762         entry->dev       = dev;
 763         entry->type      = dma_debug_page;
 764         entry->paddr     = page_to_phys(page) + offset;
 765         entry->dev_addr  = dma_addr;
 766         entry->size      = size;
 767         entry->direction = direction;
 768
 769         if (map_single)
 770                 entry->type = dma_debug_single;
 771
 772         if (!PageHighMem(page)) {
 773                 void *addr = ((char *)page_address(page)) + offset;
 774                 check_for_stack(dev, addr);
 775                 check_for_illegal_area(dev, addr, size);
 776         }
 777
 778         add_dma_entry(entry);
 779 }
 780 EXPORT_SYMBOL(debug_dma_map_page);
 781
 782 void debug_dma_unmap_page(struct device *dev, dma_addr_t addr,
 783                           size_t size, int direction, bool map_single)
 784 {
 785         struct dma_debug_entry ref = {
 786                 .type           = dma_debug_page,
 787                 .dev            = dev,
 788                 .dev_addr       = addr,
 789                 .size           = size,
 790                 .direction      = direction,
 791         };
 792
 793         if (unlikely(global_disable))
 794                 return;
 795
 796         if (map_single)
 797                 ref.type = dma_debug_single;
 798
 799         check_unmap(&ref);
 800 }
 801 EXPORT_SYMBOL(debug_dma_unmap_page);
 802
 803 void debug_dma_map_sg(struct device *dev, struct scatterlist *sg,
 804                       int nents, int mapped_ents, int direction)
 805 {
 806         struct dma_debug_entry *entry;
 807         struct scatterlist *s;
 808         int i;
 809
 810         if (unlikely(global_disable))
 811                 return;
 812
 813         for_each_sg(sg, s, mapped_ents, i) {
 814                 entry = dma_entry_alloc();
 815                 if (!entry)
 816                         return;
 817
 818                 entry->type           = dma_debug_sg;
 819                 entry->dev            = dev;
 820                 entry->paddr          = sg_phys(s);
 821                 entry->size           = s->length;
 822                 entry->dev_addr       = s->dma_address;
 823                 entry->direction      = direction;
 824                 entry->sg_call_ents   = nents;
 825                 entry->sg_mapped_ents = mapped_ents;
 826
 827                 if (!PageHighMem(sg_page(s))) {
 828                         check_for_stack(dev, sg_virt(s));
 829                         check_for_illegal_area(dev, sg_virt(s), s->length);
 830                 }
 831
 832                 add_dma_entry(entry);
 833         }
 834 }
 835 EXPORT_SYMBOL(debug_dma_map_sg);
 836
 837 void debug_dma_unmap_sg(struct device *dev, struct scatterlist *sglist,
 838                         int nelems, int dir)
 839 {
 840         struct dma_debug_entry *entry;
 841         struct scatterlist *s;
 842         int mapped_ents = 0, i;
 843         unsigned long flags;
 844
 845         if (unlikely(global_disable))
 846                 return;
 847
 848         for_each_sg(sglist, s, nelems, i) {
 849
 850                 struct dma_debug_entry ref = {
 851                         .type           = dma_debug_sg,
 852                         .dev            = dev,
 853                         .paddr          = sg_phys(s),
 854                         .dev_addr       = s->dma_address,
 855                         .size           = s->length,
 856                         .direction      = dir,
 857                         .sg_call_ents   = 0,
 858                 };
 859
 860                 if (mapped_ents && i >= mapped_ents)
 861                         break;
 862
 863                 if (mapped_ents == 0) {
 864                         struct hash_bucket *bucket;
 865                         ref.sg_call_ents = nelems;
 866                         bucket = get_hash_bucket(&ref, &flags);
 867                         entry = hash_bucket_find(bucket, &ref);
 868                         if (entry)
 869                                 mapped_ents = entry->sg_mapped_ents;
 870                         put_hash_bucket(bucket, &flags);
 871                 }
 872
 873                 check_unmap(&ref);
 874         }
 875 }
 876 EXPORT_SYMBOL(debug_dma_unmap_sg);
 877
 878 void debug_dma_alloc_coherent(struct device *dev, size_t size,
 879                               dma_addr_t dma_addr, void *virt)
 880 {
 881         struct dma_debug_entry *entry;
 882
 883         if (unlikely(global_disable))
 884                 return;
 885
 886         if (unlikely(virt == NULL))
 887                 return;
 888
 889         entry = dma_entry_alloc();
 890         if (!entry)
 891                 return;
 892
 893         entry->type      = dma_debug_coherent;
 894         entry->dev       = dev;
 895         entry->paddr     = virt_to_phys(virt);
 896         entry->size      = size;
 897         entry->dev_addr  = dma_addr;
 898         entry->direction = DMA_BIDIRECTIONAL;
 899
 900         add_dma_entry(entry);
 901 }
 902 EXPORT_SYMBOL(debug_dma_alloc_coherent);
 903
 904 void debug_dma_free_coherent(struct device *dev, size_t size,
 905                          void *virt, dma_addr_t addr)
 906 {
 907         struct dma_debug_entry ref = {
 908                 .type           = dma_debug_coherent,
 909                 .dev            = dev,
 910                 .paddr          = virt_to_phys(virt),
 911                 .dev_addr       = addr,
 912                 .size           = size,
 913                 .direction      = DMA_BIDIRECTIONAL,
 914         };
 915
 916         if (unlikely(global_disable))
 917                 return;
 918
 919         check_unmap(&ref);
 920 }
 921 EXPORT_SYMBOL(debug_dma_free_coherent);
 922
 923 void debug_dma_sync_single_for_cpu(struct device *dev, dma_addr_t dma_handle,
 924                                    size_t size, int direction)
 925 {
 926         if (unlikely(global_disable))
 927                 return;
 928
 929         check_sync(dev, dma_handle, size, 0, direction, true);
 930 }
 931 EXPORT_SYMBOL(debug_dma_sync_single_for_cpu);
 932
 933 void debug_dma_sync_single_for_device(struct device *dev,
 934                                       dma_addr_t dma_handle, size_t size,
 935                                       int direction)
 936 {
 937         if (unlikely(global_disable))
 938                 return;
 939
 940         check_sync(dev, dma_handle, size, 0, direction, false);
 941 }
 942 EXPORT_SYMBOL(debug_dma_sync_single_for_device);
 943
 944 void debug_dma_sync_single_range_for_cpu(struct device *dev,
 945                                          dma_addr_t dma_handle,
 946                                          unsigned long offset, size_t size,
 947                                          int direction)
 948 {
 949         if (unlikely(global_disable))
 950                 return;
 951
 952         check_sync(dev, dma_handle, size, offset, direction, true);
 953 }
 954 EXPORT_SYMBOL(debug_dma_sync_single_range_for_cpu);
 955
 956 void debug_dma_sync_single_range_for_device(struct device *dev,
 957                                             dma_addr_t dma_handle,
 958                                             unsigned long offset,
 959                                             size_t size, int direction)
 960 {
 961         if (unlikely(global_disable))
 962                 return;
 963
 964         check_sync(dev, dma_handle, size, offset, direction, false);
 965 }
 966 EXPORT_SYMBOL(debug_dma_sync_single_range_for_device);
 967
 968 void debug_dma_sync_sg_for_cpu(struct device *dev, struct scatterlist *sg,
 969                                int nelems, int direction)
 970 {
 971         struct scatterlist *s;
 972         int i;
 973
 974         if (unlikely(global_disable))
 975                 return;
 976
 977         for_each_sg(sg, s, nelems, i) {
 978                 check_sync(dev, s->dma_address, s->dma_length, 0,
 979                                 direction, true);
 980         }
 981 }
 982 EXPORT_SYMBOL(debug_dma_sync_sg_for_cpu);
 983
 984 void debug_dma_sync_sg_for_device(struct device *dev, struct scatterlist *sg,
 985                                   int nelems, int direction)
 986 {
 987         struct scatterlist *s;
 988         int i;
 989
 990         if (unlikely(global_disable))
 991                 return;
 992
 993         for_each_sg(sg, s, nelems, i) {
 994                 check_sync(dev, s->dma_address, s->dma_length, 0,
 995                                 direction, false);
 996         }
 997 }
 998 EXPORT_SYMBOL(debug_dma_sync_sg_for_device);
 999