[SPARC64]: Add CONFIG_DEBUG_PAGEALLOC support.

The trick is that we do the kernel linear mapping TLB miss starting
with an instruction sequence like this:

	ba,pt		%xcc, kvmap_load
	 xor		%g2, %g4, %g5

succeeded by an instruction sequence which performs a full page table
walk starting at swapper_pg_dir.

We first take over the trap table from the firmware.  Then, using this
constant PTE generation for the linear mapping area above, we build
the kernel page tables for the linear mapping.

After this is setup, we patch that branch above into a "nop", which
will cause TLB misses to fall through to the full page table walk.

With this, the page unmapping for CONFIG_DEBUG_PAGEALLOC is trivial.

Signed-off-by: David S. Miller <davem@davemloft.net>

diff --git a/arch/sparc64/Kconfig.debug b/arch/sparc64/Kconfig.debug
index af0e941..fa06ea0 100644 (file)
--- a/arch/sparc64/Kconfig.debug
+++ b/arch/sparc64/Kconfig.debug
@@ -33,6 +33,14 @@ config DEBUG_BOOTMEM
        depends on DEBUG_KERNEL
        bool "Debug BOOTMEM initialization"
 
+config DEBUG_PAGEALLOC
+       bool "Page alloc debugging"
+       depends on DEBUG_KERNEL && !SOFTWARE_SUSPEND
+       help
+         Unmap pages from the kernel linear mapping after free_pages().
+         This results in a large slowdown, but helps to find certain types
+         of memory corruptions.
+
 config MCOUNT
        bool
        depends on STACK_DEBUG

diff --git a/arch/sparc64/kernel/head.S b/arch/sparc64/kernel/head.S
index 56af714..ecc748f 100644 (file)
--- a/arch/sparc64/kernel/head.S
+++ b/arch/sparc64/kernel/head.S
@@ -525,12 +525,6 @@ bootup_user_stack_end:
 
 #include "ttable.S"
 #include "systbls.S"
-
-       .align  1024
-       .globl  swapper_pg_dir
-swapper_pg_dir:
-       .word   0
-
 #include "ktlb.S"
 #include "etrap.S"
 #include "rtrap.S"

diff --git a/arch/sparc64/kernel/ktlb.S b/arch/sparc64/kernel/ktlb.S
index a591bc0..7796b37 100644 (file)
--- a/arch/sparc64/kernel/ktlb.S
+++ b/arch/sparc64/kernel/ktlb.S
@@ -132,9 +132,40 @@ kvmap_do_obp:
  */
        .align          32
 kvmap:
-       brlz,pt         %g4, kvmap_load
+       brgez,pn        %g4, kvmap_nonlinear
+        nop
+
+#ifdef CONFIG_DEBUG_PAGEALLOC
+       .globl          kvmap_linear_patch
+kvmap_linear_patch:
+#endif
+       ba,pt           %xcc, kvmap_load
         xor            %g2, %g4, %g5
 
+#ifdef CONFIG_DEBUG_PAGEALLOC
+       sethi           %hi(swapper_pg_dir), %g5
+       or              %g5, %lo(swapper_pg_dir), %g5
+       sllx            %g4, 64 - (PGDIR_SHIFT + PGDIR_BITS), %g6
+       srlx            %g6, 64 - PAGE_SHIFT, %g6
+       andn            %g6, 0x3, %g6
+       lduw            [%g5 + %g6], %g5
+       brz,pn          %g5, longpath
+        sllx           %g4, 64 - (PMD_SHIFT + PMD_BITS), %g6
+       srlx            %g6, 64 - PAGE_SHIFT, %g6
+       sllx            %g5, 11, %g5
+       andn            %g6, 0x3, %g6
+       lduwa           [%g5 + %g6] ASI_PHYS_USE_EC, %g5
+       brz,pn          %g5, longpath
+        sllx           %g4, 64 - PMD_SHIFT, %g6
+       srlx            %g6, 64 - PAGE_SHIFT, %g6
+       sllx            %g5, 11, %g5
+       andn            %g6, 0x7, %g6
+       ldxa            [%g5 + %g6] ASI_PHYS_USE_EC, %g5
+       brz,pn          %g5, longpath
+        nop
+       ba,a,pt         %xcc, kvmap_load
+#endif
+
 kvmap_nonlinear:
        sethi           %hi(MODULES_VADDR), %g5
        cmp             %g4, %g5

diff --git a/arch/sparc64/kernel/vmlinux.lds.S b/arch/sparc64/kernel/vmlinux.lds.S
index f47d0be..2af0cf0 100644 (file)
--- a/arch/sparc64/kernel/vmlinux.lds.S
+++ b/arch/sparc64/kernel/vmlinux.lds.S
@@ -9,8 +9,7 @@ ENTRY(_start)
 jiffies = jiffies_64;
 SECTIONS
 {
-  swapper_pmd_dir = 0x0000000000402000;
-  empty_pg_dir = 0x0000000000403000;
+  swapper_low_pmd_dir = 0x0000000000402000;
   . = 0x4000;
   .text 0x0000000000404000 :
   {

diff --git a/arch/sparc64/mm/init.c b/arch/sparc64/mm/init.c
index cf74737..8d72f8a 100644 (file)
--- a/arch/sparc64/mm/init.c
+++ b/arch/sparc64/mm/init.c
@@ -1332,15 +1332,114 @@ unsigned long __init bootmem_init(unsigned long *pages_avail)
        return end_pfn;
 }
 
+#ifdef CONFIG_DEBUG_PAGEALLOC
+static unsigned long kernel_map_range(unsigned long pstart, unsigned long pend, pgprot_t prot)
+{
+       unsigned long vstart = PAGE_OFFSET + pstart;
+       unsigned long vend = PAGE_OFFSET + pend;
+       unsigned long alloc_bytes = 0UL;
+
+       if ((vstart & ~PAGE_MASK) || (vend & ~PAGE_MASK)) {
+               prom_printf("kernel_map: Unaligned sp_banks[%lx:%lx]\n",
+                           vstart, vend);
+               prom_halt();
+       }
+
+       while (vstart < vend) {
+               unsigned long this_end, paddr = __pa(vstart);
+               pgd_t *pgd = pgd_offset_k(vstart);
+               pud_t *pud;
+               pmd_t *pmd;
+               pte_t *pte;
+
+               pud = pud_offset(pgd, vstart);
+               if (pud_none(*pud)) {
+                       pmd_t *new;
+
+                       new = __alloc_bootmem(PAGE_SIZE, PAGE_SIZE, PAGE_SIZE);
+                       alloc_bytes += PAGE_SIZE;
+                       pud_populate(&init_mm, pud, new);
+               }
+
+               pmd = pmd_offset(pud, vstart);
+               if (!pmd_present(*pmd)) {
+                       pte_t *new;
+
+                       new = __alloc_bootmem(PAGE_SIZE, PAGE_SIZE, PAGE_SIZE);
+                       alloc_bytes += PAGE_SIZE;
+                       pmd_populate_kernel(&init_mm, pmd, new);
+               }
+
+               pte = pte_offset_kernel(pmd, vstart);
+               this_end = (vstart + PMD_SIZE) & PMD_MASK;
+               if (this_end > vend)
+                       this_end = vend;
+
+               while (vstart < this_end) {
+                       pte_val(*pte) = (paddr | pgprot_val(prot));
+
+                       vstart += PAGE_SIZE;
+                       paddr += PAGE_SIZE;
+                       pte++;
+               }
+       }
+
+       return alloc_bytes;
+}
+
+extern struct linux_mlist_p1275 *prom_ptot_ptr;
+extern unsigned int kvmap_linear_patch[1];
+
+static void __init kernel_physical_mapping_init(void)
+{
+       struct linux_mlist_p1275 *p = prom_ptot_ptr;
+       unsigned long mem_alloced = 0UL;
+
+       while (p) {
+               unsigned long phys_start, phys_end;
+
+               phys_start = p->start_adr;
+               phys_end = phys_start + p->num_bytes;
+               mem_alloced += kernel_map_range(phys_start, phys_end,
+                                               PAGE_KERNEL);
+
+               p = p->theres_more;
+       }
+
+       printk("Allocated %ld bytes for kernel page tables.\n",
+              mem_alloced);
+
+       kvmap_linear_patch[0] = 0x01000000; /* nop */
+       flushi(&kvmap_linear_patch[0]);
+
+       __flush_tlb_all();
+}
+
+void kernel_map_pages(struct page *page, int numpages, int enable)
+{
+       unsigned long phys_start = page_to_pfn(page) << PAGE_SHIFT;
+       unsigned long phys_end = phys_start + (numpages * PAGE_SIZE);
+
+       kernel_map_range(phys_start, phys_end,
+                        (enable ? PAGE_KERNEL : __pgprot(0)));
+
+       /* we should perform an IPI and flush all tlbs,
+        * but that can deadlock->flush only current cpu.
+        */
+       __flush_tlb_kernel_range(PAGE_OFFSET + phys_start,
+                                PAGE_OFFSET + phys_end);
+}
+#endif
+
 /* paging_init() sets up the page tables */
 
 extern void cheetah_ecache_flush_init(void);
 
 static unsigned long last_valid_pfn;
+pgd_t swapper_pg_dir[2048];
 
 void __init paging_init(void)
 {
-       extern pmd_t swapper_pmd_dir[1024];
        unsigned long end_pfn, pages_avail, shift;
        unsigned long real_end;
 
@@ -1361,11 +1460,11 @@ void __init paging_init(void)
         */
        init_mm.pgd += ((shift) / (sizeof(pgd_t)));
        
-       memset(swapper_pmd_dir, 0, sizeof(swapper_pmd_dir));
+       memset(swapper_low_pmd_dir, 0, sizeof(swapper_low_pmd_dir));
 
        /* Now can init the kernel/bad page tables. */
        pud_set(pud_offset(&swapper_pg_dir[0], 0),
-               swapper_pmd_dir + (shift / sizeof(pgd_t)));
+               swapper_low_pmd_dir + (shift / sizeof(pgd_t)));
        
        swapper_pgd_zero = pgd_val(swapper_pg_dir[0]);
        
@@ -1390,6 +1489,10 @@ void __init paging_init(void)
        pages_avail = 0;
        last_valid_pfn = end_pfn = bootmem_init(&pages_avail);
 
+#ifdef CONFIG_DEBUG_PAGEALLOC
+       kernel_physical_mapping_init();
+#endif
+
        {
                unsigned long zones_size[MAX_NR_ZONES];
                unsigned long zholes_size[MAX_NR_ZONES];

diff --git a/include/asm-sparc64/cacheflush.h b/include/asm-sparc64/cacheflush.h
index ededd26..b3f6165 100644 (file)
--- a/include/asm-sparc64/cacheflush.h
+++ b/include/asm-sparc64/cacheflush.h
@@ -66,6 +66,11 @@ extern void flush_ptrace_access(struct vm_area_struct *, struct page *,
 #define flush_cache_vmap(start, end)           do { } while (0)
 #define flush_cache_vunmap(start, end)         do { } while (0)
 
+#ifdef CONFIG_DEBUG_PAGEALLOC
+/* internal debugging function */
+void kernel_map_pages(struct page *page, int numpages, int enable);
+#endif
+
 #endif /* !__ASSEMBLY__ */
 
 #endif /* _SPARC64_CACHEFLUSH_H */

diff --git a/include/asm-sparc64/pgtable.h b/include/asm-sparc64/pgtable.h
index a297f61..43cbb08 100644 (file)
--- a/include/asm-sparc64/pgtable.h
+++ b/include/asm-sparc64/pgtable.h
@@ -60,13 +60,13 @@
  * table can map
  */
 #define PMD_SHIFT      (PAGE_SHIFT + (PAGE_SHIFT-3))
-#define PMD_SIZE       (1UL << PMD_SHIFT)
+#define PMD_SIZE       (_AC(1,UL) << PMD_SHIFT)
 #define PMD_MASK       (~(PMD_SIZE-1))
 #define PMD_BITS       (PAGE_SHIFT - 2)
 
 /* PGDIR_SHIFT determines what a third-level page table entry can map */
 #define PGDIR_SHIFT    (PAGE_SHIFT + (PAGE_SHIFT-3) + PMD_BITS)
-#define PGDIR_SIZE     (1UL << PGDIR_SHIFT)
+#define PGDIR_SIZE     (_AC(1,UL) << PGDIR_SHIFT)
 #define PGDIR_MASK     (~(PGDIR_SIZE-1))
 #define PGDIR_BITS     (PAGE_SHIFT - 2)
 
@@ -336,7 +336,8 @@ static inline void set_pte_at(struct mm_struct *mm, unsigned long addr, pte_t *p
 #define pte_clear(mm,addr,ptep)                \
        set_pte_at((mm), (addr), (ptep), __pte(0UL))
 
-extern pgd_t swapper_pg_dir[1];
+extern pgd_t swapper_pg_dir[2048];
+extern pmd_t swapper_low_pmd_dir[2048];
 
 /* These do nothing with the way I have things setup. */
 #define mmu_lockarea(vaddr, len)               (vaddr)