struct hash_pte *htab_address;
unsigned long htab_size_bytes;
unsigned long htab_hash_mask;
+EXPORT_SYMBOL_GPL(htab_hash_mask);
int mmu_linear_psize = MMU_PAGE_4K;
int mmu_virtual_psize = MMU_PAGE_4K;
int mmu_vmalloc_psize = MMU_PAGE_4K;
int mmu_kernel_ssize = MMU_SEGSIZE_256M;
int mmu_highuser_ssize = MMU_SEGSIZE_256M;
u16 mmu_slb_size = 64;
+EXPORT_SYMBOL_GPL(mmu_slb_size);
#ifdef CONFIG_HUGETLB_PAGE
unsigned int HPAGE_SHIFT;
#endif
unsigned long tprot = prot;
/* Make kernel text executable */
- if (in_kernel_text(vaddr))
+ if (overlaps_kernel_text(vaddr, vaddr + step))
tprot &= ~HPTE_R_N;
hash = hpt_hash(va, shift, ssize);
return 0;
}
+#ifdef CONFIG_HUGETLB_PAGE
/* Scan for 16G memory blocks that have been set aside for huge pages
* and reserve those blocks for 16G huge pages.
*/
printk(KERN_INFO "Huge page(16GB) memory: "
"addr = 0x%lX size = 0x%lX pages = %d\n",
phys_addr, block_size, expected_pages);
- lmb_reserve(phys_addr, block_size * expected_pages);
- add_gpage(phys_addr, block_size, expected_pages);
+ if (phys_addr + (16 * GB) <= lmb_end_of_DRAM()) {
+ lmb_reserve(phys_addr, block_size * expected_pages);
+ add_gpage(phys_addr, block_size, expected_pages);
+ }
return 0;
}
+#endif /* CONFIG_HUGETLB_PAGE */
static void __init htab_init_page_sizes(void)
{
#ifdef CONFIG_HUGETLB_PAGE
/* Reserve 16G huge page memory sections for huge pages */
of_scan_flat_dt(htab_dt_scan_hugepage_blocks, NULL);
-
-/* Set default large page size. Currently, we pick 16M or 1M depending
- * on what is available
- */
- if (mmu_psize_defs[MMU_PAGE_16M].shift)
- HPAGE_SHIFT = mmu_psize_defs[MMU_PAGE_16M].shift;
- /* With 4k/4level pagetables, we can't (for now) cope with a
- * huge page size < PMD_SIZE */
- else if (mmu_psize_defs[MMU_PAGE_1M].shift)
- HPAGE_SHIFT = mmu_psize_defs[MMU_PAGE_1M].shift;
#endif /* CONFIG_HUGETLB_PAGE */
}
static unsigned long __init htab_get_table_size(void)
{
- unsigned long mem_size, rnd_mem_size, pteg_count;
+ unsigned long mem_size, rnd_mem_size, pteg_count, psize;
/* If hash size isn't already provided by the platform, we try to
* retrieve it from the device-tree. If it's not there neither, we
rnd_mem_size <<= 1;
/* # pages / 2 */
- pteg_count = max(rnd_mem_size >> (12 + 1), 1UL << 11);
+ psize = mmu_psize_defs[mmu_virtual_psize].shift;
+ pteg_count = max(rnd_mem_size >> (psize + 1), 1UL << 11);
return pteg_count << 7;
}
void create_section_mapping(unsigned long start, unsigned long end)
{
BUG_ON(htab_bolt_mapping(start, end, __pa(start),
- PAGE_KERNEL, mmu_linear_psize,
+ pgprot_val(PAGE_KERNEL), mmu_linear_psize,
mmu_kernel_ssize));
}
make_bl(htab_call_hpte_updatepp, ppc_md.hpte_updatepp);
}
-void __init htab_initialize(void)
+static void __init htab_initialize(void)
{
unsigned long table;
unsigned long pteg_count;
mtspr(SPRN_SDR1, _SDR1);
}
- prot = PAGE_KERNEL;
+ prot = pgprot_val(PAGE_KERNEL);
#ifdef CONFIG_DEBUG_PAGEALLOC
linear_map_hash_count = lmb_end_of_DRAM() >> PAGE_SHIFT;
#undef KB
#undef MB
-void htab_initialize_secondary(void)
+void __init early_init_mmu(void)
{
+ /* Setup initial STAB address in the PACA */
+ get_paca()->stab_real = __pa((u64)&initial_stab);
+ get_paca()->stab_addr = (u64)&initial_stab;
+
+ /* Initialize the MMU Hash table and create the linear mapping
+ * of memory. Has to be done before stab/slb initialization as
+ * this is currently where the page size encoding is obtained
+ */
+ htab_initialize();
+
+ /* Initialize stab / SLB management except on iSeries
+ */
+ if (cpu_has_feature(CPU_FTR_SLB))
+ slb_initialize();
+ else if (!firmware_has_feature(FW_FEATURE_ISERIES))
+ stab_initialize(get_paca()->stab_real);
+}
+
+#ifdef CONFIG_SMP
+void __cpuinit early_init_mmu_secondary(void)
+{
+ /* Initialize hash table for that CPU */
if (!firmware_has_feature(FW_FEATURE_LPAR))
mtspr(SPRN_SDR1, _SDR1);
+
+ /* Initialize STAB/SLB. We use a virtual address as it works
+ * in real mode on pSeries and we want a virutal address on
+ * iSeries anyway
+ */
+ if (cpu_has_feature(CPU_FTR_SLB))
+ slb_initialize();
+ else
+ stab_initialize(get_paca()->stab_addr);
}
+#endif /* CONFIG_SMP */
/*
* Called by asm hashtable.S for doing lazy icache flush
/* page is dirty */
if (!test_bit(PG_arch_1, &page->flags) && !PageReserved(page)) {
if (trap == 0x400) {
- __flush_dcache_icache(page_address(page));
+ flush_dcache_icache_page(page);
set_bit(PG_arch_1, &page->flags);
} else
pp |= HPTE_R_N;
* Result is 0: full permissions, _PAGE_RW: read-only,
* _PAGE_USER or _PAGE_USER|_PAGE_RW: no access.
*/
-static int subpage_protection(pgd_t *pgdir, unsigned long ea)
+static int subpage_protection(struct mm_struct *mm, unsigned long ea)
{
- struct subpage_prot_table *spt = pgd_subpage_prot(pgdir);
+ struct subpage_prot_table *spt = &mm->context.spt;
u32 spp = 0;
u32 **sbpm, *sbpp;
}
#else /* CONFIG_PPC_SUBPAGE_PROT */
-static inline int subpage_protection(pgd_t *pgdir, unsigned long ea)
+static inline int subpage_protection(struct mm_struct *mm, unsigned long ea)
{
return 0;
}
unsigned long vsid;
struct mm_struct *mm;
pte_t *ptep;
- cpumask_t tmp;
+ unsigned hugeshift;
+ const struct cpumask *tmp;
int rc, user_region = 0, local = 0;
int psize, ssize;
return 1;
/* Check CPU locality */
- tmp = cpumask_of_cpu(smp_processor_id());
- if (user_region && cpus_equal(mm->cpu_vm_mask, tmp))
+ tmp = cpumask_of(smp_processor_id());
+ if (user_region && cpumask_equal(mm_cpumask(mm), tmp))
local = 1;
-#ifdef CONFIG_HUGETLB_PAGE
- /* Handle hugepage regions */
- if (HPAGE_SHIFT && mmu_huge_psizes[psize]) {
- DBG_LOW(" -> huge page !\n");
- return hash_huge_page(mm, access, ea, vsid, local, trap);
- }
-#endif /* CONFIG_HUGETLB_PAGE */
-
#ifndef CONFIG_PPC_64K_PAGES
- /* If we use 4K pages and our psize is not 4K, then we are hitting
- * a special driver mapping, we need to align the address before
- * we fetch the PTE
+ /* If we use 4K pages and our psize is not 4K, then we might
+ * be hitting a special driver mapping, and need to align the
+ * address before we fetch the PTE.
+ *
+ * It could also be a hugepage mapping, in which case this is
+ * not necessary, but it's not harmful, either.
*/
if (psize != MMU_PAGE_4K)
ea &= ~((1ul << mmu_psize_defs[psize].shift) - 1);
#endif /* CONFIG_PPC_64K_PAGES */
/* Get PTE and page size from page tables */
- ptep = find_linux_pte(pgdir, ea);
+ ptep = find_linux_pte_or_hugepte(pgdir, ea, &hugeshift);
if (ptep == NULL || !pte_present(*ptep)) {
DBG_LOW(" no PTE !\n");
return 1;
}
+#ifdef CONFIG_HUGETLB_PAGE
+ if (hugeshift)
+ return __hash_page_huge(ea, access, vsid, ptep, trap, local,
+ ssize, hugeshift, psize);
+#endif /* CONFIG_HUGETLB_PAGE */
+
#ifndef CONFIG_PPC_64K_PAGES
DBG_LOW(" i-pte: %016lx\n", pte_val(*ptep));
#else
unsigned long vsid;
void *pgdir;
pte_t *ptep;
- cpumask_t mask;
unsigned long flags;
int local = 0;
int ssize;
local_irq_save(flags);
/* Is that local to this CPU ? */
- mask = cpumask_of_cpu(smp_processor_id());
- if (cpus_equal(mm->cpu_vm_mask, mask))
+ if (cpumask_equal(mm_cpumask(mm), cpumask_of(smp_processor_id())))
local = 1;
/* Hash it in */
git://ftp.safe.ca / safe / jmp / linux-2.6 / blobdiff