SAFE public projects git trees. - safe/jmp/linux-2.6/blob - lib/dma-debug.c

   1 /*
   2  * Copyright (C) 2008 Advanced Micro Devices, Inc.
   3  *
   4  * Author: Joerg Roedel <joerg.roedel@amd.com>
   5  *
   6  * This program is free software; you can redistribute it and/or modify it
   7  * under the terms of the GNU General Public License version 2 as published
   8  * by the Free Software Foundation.
   9  *
  10  * This program is distributed in the hope that it will be useful,
  11  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  12  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  13  * GNU General Public License for more details.
  14  *
  15  * You should have received a copy of the GNU General Public License
  16  * along with this program; if not, write to the Free Software
  17  * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307 USA
  18  */
  19
  20 #include <linux/dma-mapping.h>
  21 #include <linux/dma-debug.h>
  22 #include <linux/spinlock.h>
  23 #include <linux/debugfs.h>
  24 #include <linux/device.h>
  25 #include <linux/types.h>
  26 #include <linux/sched.h>
  27 #include <linux/list.h>
  28 #include <linux/slab.h>
  29
  30 #define HASH_SIZE       1024ULL
  31 #define HASH_FN_SHIFT   13
  32 #define HASH_FN_MASK    (HASH_SIZE - 1)
  33
  34 enum {
  35         dma_debug_single,
  36         dma_debug_page,
  37         dma_debug_sg,
  38         dma_debug_coherent,
  39 };
  40
  41 struct dma_debug_entry {
  42         struct list_head list;
  43         struct device    *dev;
  44         int              type;
  45         phys_addr_t      paddr;
  46         u64              dev_addr;
  47         u64              size;
  48         int              direction;
  49         int              sg_call_ents;
  50         int              sg_mapped_ents;
  51 };
  52
  53 struct hash_bucket {
  54         struct list_head list;
  55         spinlock_t lock;
  56 } ____cacheline_aligned_in_smp;
  57
  58 /* Hash list to save the allocated dma addresses */
  59 static struct hash_bucket dma_entry_hash[HASH_SIZE];
  60 /* List of pre-allocated dma_debug_entry's */
  61 static LIST_HEAD(free_entries);
  62 /* Lock for the list above */
  63 static DEFINE_SPINLOCK(free_entries_lock);
  64
  65 /* Global disable flag - will be set in case of an error */
  66 static bool global_disable __read_mostly;
  67
  68 /* Global error count */
  69 static u32 error_count;
  70
  71 /* Global error show enable*/
  72 static u32 show_all_errors __read_mostly;
  73 /* Number of errors to show */
  74 static u32 show_num_errors = 1;
  75
  76 static u32 num_free_entries;
  77 static u32 min_free_entries;
  78
  79 /* number of preallocated entries requested by kernel cmdline */
  80 static u32 req_entries;
  81
  82 /* debugfs dentry's for the stuff above */
  83 static struct dentry *dma_debug_dent        __read_mostly;
  84 static struct dentry *global_disable_dent   __read_mostly;
  85 static struct dentry *error_count_dent      __read_mostly;
  86 static struct dentry *show_all_errors_dent  __read_mostly;
  87 static struct dentry *show_num_errors_dent  __read_mostly;
  88 static struct dentry *num_free_entries_dent __read_mostly;
  89 static struct dentry *min_free_entries_dent __read_mostly;
  90
  91 static const char *type2name[4] = { "single", "page",
  92                                     "scather-gather", "coherent" };
  93
  94 static const char *dir2name[4] = { "DMA_BIDIRECTIONAL", "DMA_TO_DEVICE",
  95                                    "DMA_FROM_DEVICE", "DMA_NONE" };
  96
  97 /*
  98  * The access to some variables in this macro is racy. We can't use atomic_t
  99  * here because all these variables are exported to debugfs. Some of them even
 100  * writeable. This is also the reason why a lock won't help much. But anyway,
 101  * the races are no big deal. Here is why:
 102  *
 103  *   error_count: the addition is racy, but the worst thing that can happen is
 104  *                that we don't count some errors
 105  *   show_num_errors: the subtraction is racy. Also no big deal because in
 106  *                    worst case this will result in one warning more in the
 107  *                    system log than the user configured. This variable is
 108  *                    writeable via debugfs.
 109  */
 110 #define err_printk(dev, format, arg...) do {                    \
 111                 error_count += 1;                               \
 112                 if (show_all_errors || show_num_errors > 0) {   \
 113                         WARN(1, "%s %s: " format,               \
 114                              dev_driver_string(dev),            \
 115                              dev_name(dev) , ## arg);           \
 116                 }                                               \
 117                 if (!show_all_errors && show_num_errors > 0)    \
 118                         show_num_errors -= 1;                   \
 119         } while (0);
 120
 121 /*
 122  * Hash related functions
 123  *
 124  * Every DMA-API request is saved into a struct dma_debug_entry. To
 125  * have quick access to these structs they are stored into a hash.
 126  */
 127 static int hash_fn(struct dma_debug_entry *entry)
 128 {
 129         /*
 130          * Hash function is based on the dma address.
 131          * We use bits 20-27 here as the index into the hash
 132          */
 133         return (entry->dev_addr >> HASH_FN_SHIFT) & HASH_FN_MASK;
 134 }
 135
 136 /*
 137  * Request exclusive access to a hash bucket for a given dma_debug_entry.
 138  */
 139 static struct hash_bucket *get_hash_bucket(struct dma_debug_entry *entry,
 140                                            unsigned long *flags)
 141 {
 142         int idx = hash_fn(entry);
 143         unsigned long __flags;
 144
 145         spin_lock_irqsave(&dma_entry_hash[idx].lock, __flags);
 146         *flags = __flags;
 147         return &dma_entry_hash[idx];
 148 }
 149
 150 /*
 151  * Give up exclusive access to the hash bucket
 152  */
 153 static void put_hash_bucket(struct hash_bucket *bucket,
 154                             unsigned long *flags)
 155 {
 156         unsigned long __flags = *flags;
 157
 158         spin_unlock_irqrestore(&bucket->lock, __flags);
 159 }
 160
 161 /*
 162  * Search a given entry in the hash bucket list
 163  */
 164 static struct dma_debug_entry *hash_bucket_find(struct hash_bucket *bucket,
 165                                                 struct dma_debug_entry *ref)
 166 {
 167         struct dma_debug_entry *entry;
 168
 169         list_for_each_entry(entry, &bucket->list, list) {
 170                 if ((entry->dev_addr == ref->dev_addr) &&
 171                     (entry->dev == ref->dev))
 172                         return entry;
 173         }
 174
 175         return NULL;
 176 }
 177
 178 /*
 179  * Add an entry to a hash bucket
 180  */
 181 static void hash_bucket_add(struct hash_bucket *bucket,
 182                             struct dma_debug_entry *entry)
 183 {
 184         list_add_tail(&entry->list, &bucket->list);
 185 }
 186
 187 /*
 188  * Remove entry from a hash bucket list
 189  */
 190 static void hash_bucket_del(struct dma_debug_entry *entry)
 191 {
 192         list_del(&entry->list);
 193 }
 194
 195 /*
 196  * Wrapper function for adding an entry to the hash.
 197  * This function takes care of locking itself.
 198  */
 199 static void add_dma_entry(struct dma_debug_entry *entry)
 200 {
 201         struct hash_bucket *bucket;
 202         unsigned long flags;
 203
 204         bucket = get_hash_bucket(entry, &flags);
 205         hash_bucket_add(bucket, entry);
 206         put_hash_bucket(bucket, &flags);
 207 }
 208
 209 /* struct dma_entry allocator
 210  *
 211  * The next two functions implement the allocator for
 212  * struct dma_debug_entries.
 213  */
 214 static struct dma_debug_entry *dma_entry_alloc(void)
 215 {
 216         struct dma_debug_entry *entry = NULL;
 217         unsigned long flags;
 218
 219         spin_lock_irqsave(&free_entries_lock, flags);
 220
 221         if (list_empty(&free_entries)) {
 222                 printk(KERN_ERR "DMA-API: debugging out of memory "
 223                                 "- disabling\n");
 224                 global_disable = true;
 225                 goto out;
 226         }
 227
 228         entry = list_entry(free_entries.next, struct dma_debug_entry, list);
 229         list_del(&entry->list);
 230         memset(entry, 0, sizeof(*entry));
 231
 232         num_free_entries -= 1;
 233         if (num_free_entries < min_free_entries)
 234                 min_free_entries = num_free_entries;
 235
 236 out:
 237         spin_unlock_irqrestore(&free_entries_lock, flags);
 238
 239         return entry;
 240 }
 241
 242 static void dma_entry_free(struct dma_debug_entry *entry)
 243 {
 244         unsigned long flags;
 245
 246         /*
 247          * add to beginning of the list - this way the entries are
 248          * more likely cache hot when they are reallocated.
 249          */
 250         spin_lock_irqsave(&free_entries_lock, flags);
 251         list_add(&entry->list, &free_entries);
 252         num_free_entries += 1;
 253         spin_unlock_irqrestore(&free_entries_lock, flags);
 254 }
 255
 256 /*
 257  * DMA-API debugging init code
 258  *
 259  * The init code does two things:
 260  *   1. Initialize core data structures
 261  *   2. Preallocate a given number of dma_debug_entry structs
 262  */
 263
 264 static int prealloc_memory(u32 num_entries)
 265 {
 266         struct dma_debug_entry *entry, *next_entry;
 267         int i;
 268
 269         for (i = 0; i < num_entries; ++i) {
 270                 entry = kzalloc(sizeof(*entry), GFP_KERNEL);
 271                 if (!entry)
 272                         goto out_err;
 273
 274                 list_add_tail(&entry->list, &free_entries);
 275         }
 276
 277         num_free_entries = num_entries;
 278         min_free_entries = num_entries;
 279
 280         printk(KERN_INFO "DMA-API: preallocated %d debug entries\n",
 281                         num_entries);
 282
 283         return 0;
 284
 285 out_err:
 286
 287         list_for_each_entry_safe(entry, next_entry, &free_entries, list) {
 288                 list_del(&entry->list);
 289                 kfree(entry);
 290         }
 291
 292         return -ENOMEM;
 293 }
 294
 295 static int dma_debug_fs_init(void)
 296 {
 297         dma_debug_dent = debugfs_create_dir("dma-api", NULL);
 298         if (!dma_debug_dent) {
 299                 printk(KERN_ERR "DMA-API: can not create debugfs directory\n");
 300                 return -ENOMEM;
 301         }
 302
 303         global_disable_dent = debugfs_create_bool("disabled", 0444,
 304                         dma_debug_dent,
 305                         (u32 *)&global_disable);
 306         if (!global_disable_dent)
 307                 goto out_err;
 308
 309         error_count_dent = debugfs_create_u32("error_count", 0444,
 310                         dma_debug_dent, &error_count);
 311         if (!error_count_dent)
 312                 goto out_err;
 313
 314         show_all_errors_dent = debugfs_create_u32("all_errors", 0644,
 315                         dma_debug_dent,
 316                         &show_all_errors);
 317         if (!show_all_errors_dent)
 318                 goto out_err;
 319
 320         show_num_errors_dent = debugfs_create_u32("num_errors", 0644,
 321                         dma_debug_dent,
 322                         &show_num_errors);
 323         if (!show_num_errors_dent)
 324                 goto out_err;
 325
 326         num_free_entries_dent = debugfs_create_u32("num_free_entries", 0444,
 327                         dma_debug_dent,
 328                         &num_free_entries);
 329         if (!num_free_entries_dent)
 330                 goto out_err;
 331
 332         min_free_entries_dent = debugfs_create_u32("min_free_entries", 0444,
 333                         dma_debug_dent,
 334                         &min_free_entries);
 335         if (!min_free_entries_dent)
 336                 goto out_err;
 337
 338         return 0;
 339
 340 out_err:
 341         debugfs_remove_recursive(dma_debug_dent);
 342
 343         return -ENOMEM;
 344 }
 345
 346
 347 /*
 348  * Let the architectures decide how many entries should be preallocated.
 349  */
 350 void dma_debug_init(u32 num_entries)
 351 {
 352         int i;
 353
 354         if (global_disable)
 355                 return;
 356
 357         for (i = 0; i < HASH_SIZE; ++i) {
 358                 INIT_LIST_HEAD(&dma_entry_hash[i].list);
 359                 dma_entry_hash[i].lock = SPIN_LOCK_UNLOCKED;
 360         }
 361
 362         if (dma_debug_fs_init() != 0) {
 363                 printk(KERN_ERR "DMA-API: error creating debugfs entries "
 364                                 "- disabling\n");
 365                 global_disable = true;
 366
 367                 return;
 368         }
 369
 370         if (req_entries)
 371                 num_entries = req_entries;
 372
 373         if (prealloc_memory(num_entries) != 0) {
 374                 printk(KERN_ERR "DMA-API: debugging out of memory error "
 375                                 "- disabled\n");
 376                 global_disable = true;
 377
 378                 return;
 379         }
 380
 381         printk(KERN_INFO "DMA-API: debugging enabled by kernel config\n");
 382 }
 383
 384 static __init int dma_debug_cmdline(char *str)
 385 {
 386         if (!str)
 387                 return -EINVAL;
 388
 389         if (strncmp(str, "off", 3) == 0) {
 390                 printk(KERN_INFO "DMA-API: debugging disabled on kernel "
 391                                  "command line\n");
 392                 global_disable = true;
 393         }
 394
 395         return 0;
 396 }
 397
 398 static __init int dma_debug_entries_cmdline(char *str)
 399 {
 400         int res;
 401
 402         if (!str)
 403                 return -EINVAL;
 404
 405         res = get_option(&str, &req_entries);
 406
 407         if (!res)
 408                 req_entries = 0;
 409
 410         return 0;
 411 }
 412
 413 __setup("dma_debug=", dma_debug_cmdline);
 414 __setup("dma_debug_entries=", dma_debug_entries_cmdline);
 415
 416 static void check_unmap(struct dma_debug_entry *ref)
 417 {
 418         struct dma_debug_entry *entry;
 419         struct hash_bucket *bucket;
 420         unsigned long flags;
 421
 422         if (dma_mapping_error(ref->dev, ref->dev_addr))
 423                 return;
 424
 425         bucket = get_hash_bucket(ref, &flags);
 426         entry = hash_bucket_find(bucket, ref);
 427
 428         if (!entry) {
 429                 err_printk(ref->dev, "DMA-API: device driver tries "
 430                            "to free DMA memory it has not allocated "
 431                            "[device address=0x%016llx] [size=%llu bytes]\n",
 432                            ref->dev_addr, ref->size);
 433                 goto out;
 434         }
 435
 436         if (ref->size != entry->size) {
 437                 err_printk(ref->dev, "DMA-API: device driver frees "
 438                            "DMA memory with different size "
 439                            "[device address=0x%016llx] [map size=%llu bytes] "
 440                            "[unmap size=%llu bytes]\n",
 441                            ref->dev_addr, entry->size, ref->size);
 442         }
 443
 444         if (ref->type != entry->type) {
 445                 err_printk(ref->dev, "DMA-API: device driver frees "
 446                            "DMA memory with wrong function "
 447                            "[device address=0x%016llx] [size=%llu bytes] "
 448                            "[mapped as %s] [unmapped as %s]\n",
 449                            ref->dev_addr, ref->size,
 450                            type2name[entry->type], type2name[ref->type]);
 451         } else if ((entry->type == dma_debug_coherent) &&
 452                    (ref->paddr != entry->paddr)) {
 453                 err_printk(ref->dev, "DMA-API: device driver frees "
 454                            "DMA memory with different CPU address "
 455                            "[device address=0x%016llx] [size=%llu bytes] "
 456                            "[cpu alloc address=%p] [cpu free address=%p]",
 457                            ref->dev_addr, ref->size,
 458                            (void *)entry->paddr, (void *)ref->paddr);
 459         }
 460
 461         if (ref->sg_call_ents && ref->type == dma_debug_sg &&
 462             ref->sg_call_ents != entry->sg_call_ents) {
 463                 err_printk(ref->dev, "DMA-API: device driver frees "
 464                            "DMA sg list with different entry count "
 465                            "[map count=%d] [unmap count=%d]\n",
 466                            entry->sg_call_ents, ref->sg_call_ents);
 467         }
 468
 469         /*
 470          * This may be no bug in reality - but most implementations of the
 471          * DMA API don't handle this properly, so check for it here
 472          */
 473         if (ref->direction != entry->direction) {
 474                 err_printk(ref->dev, "DMA-API: device driver frees "
 475                            "DMA memory with different direction "
 476                            "[device address=0x%016llx] [size=%llu bytes] "
 477                            "[mapped with %s] [unmapped with %s]\n",
 478                            ref->dev_addr, ref->size,
 479                            dir2name[entry->direction],
 480                            dir2name[ref->direction]);
 481         }
 482
 483         hash_bucket_del(entry);
 484         dma_entry_free(entry);
 485
 486 out:
 487         put_hash_bucket(bucket, &flags);
 488 }
 489
 490 static void check_for_stack(struct device *dev, void *addr)
 491 {
 492         if (object_is_on_stack(addr))
 493                 err_printk(dev, "DMA-API: device driver maps memory from stack"
 494                                 " [addr=%p]\n", addr);
 495 }
 496
 497 static void check_sync(struct device *dev, dma_addr_t addr,
 498                        u64 size, u64 offset, int direction, bool to_cpu)
 499 {
 500         struct dma_debug_entry ref = {
 501                 .dev            = dev,
 502                 .dev_addr       = addr,
 503                 .size           = size,
 504                 .direction      = direction,
 505         };
 506         struct dma_debug_entry *entry;
 507         struct hash_bucket *bucket;
 508         unsigned long flags;
 509
 510         bucket = get_hash_bucket(&ref, &flags);
 511
 512         entry = hash_bucket_find(bucket, &ref);
 513
 514         if (!entry) {
 515                 err_printk(dev, "DMA-API: device driver tries "
 516                                 "to sync DMA memory it has not allocated "
 517                                 "[device address=0x%016llx] [size=%llu bytes]\n",
 518                                 addr, size);
 519                 goto out;
 520         }
 521
 522         if ((offset + size) > entry->size) {
 523                 err_printk(dev, "DMA-API: device driver syncs"
 524                                 " DMA memory outside allocated range "
 525                                 "[device address=0x%016llx] "
 526                                 "[allocation size=%llu bytes] [sync offset=%llu] "
 527                                 "[sync size=%llu]\n", entry->dev_addr, entry->size,
 528                                 offset, size);
 529         }
 530
 531         if (direction != entry->direction) {
 532                 err_printk(dev, "DMA-API: device driver syncs "
 533                                 "DMA memory with different direction "
 534                                 "[device address=0x%016llx] [size=%llu bytes] "
 535                                 "[mapped with %s] [synced with %s]\n",
 536                                 addr, entry->size,
 537                                 dir2name[entry->direction],
 538                                 dir2name[direction]);
 539         }
 540
 541         if (entry->direction == DMA_BIDIRECTIONAL)
 542                 goto out;
 543
 544         if (to_cpu && !(entry->direction == DMA_FROM_DEVICE) &&
 545                       !(direction == DMA_TO_DEVICE))
 546                 err_printk(dev, "DMA-API: device driver syncs "
 547                                 "device read-only DMA memory for cpu "
 548                                 "[device address=0x%016llx] [size=%llu bytes] "
 549                                 "[mapped with %s] [synced with %s]\n",
 550                                 addr, entry->size,
 551                                 dir2name[entry->direction],
 552                                 dir2name[direction]);
 553
 554         if (!to_cpu && !(entry->direction == DMA_TO_DEVICE) &&
 555                        !(direction == DMA_FROM_DEVICE))
 556                 err_printk(dev, "DMA-API: device driver syncs "
 557                                 "device write-only DMA memory to device "
 558                                 "[device address=0x%016llx] [size=%llu bytes] "
 559                                 "[mapped with %s] [synced with %s]\n",
 560                                 addr, entry->size,
 561                                 dir2name[entry->direction],
 562                                 dir2name[direction]);
 563
 564 out:
 565         put_hash_bucket(bucket, &flags);
 566
 567 }
 568