arch/powerpc/mm/tlb_nohash.c - kernel/windcharger - Git at Google

 /*
  * This file contains the routines for TLB flushing.
  * On machines where the MMU does not use a hash table to store virtual to
  * physical translations (ie, SW loaded TLBs or Book3E compilant processors,
  * this does -not- include 603 however which shares the implementation with
  * hash based processors)
  *
  *  -- BenH
  *
  * Copyright 2008,2009 Ben Herrenschmidt <benh@kernel.crashing.org>
  *                     IBM Corp.
  *
  *  Derived from arch/ppc/mm/init.c:
  *    Copyright (C) 1995-1996 Gary Thomas (gdt@linuxppc.org)
  *
  *  Modifications by Paul Mackerras (PowerMac) (paulus@cs.anu.edu.au)
  *  and Cort Dougan (PReP) (cort@cs.nmt.edu)
  *    Copyright (C) 1996 Paul Mackerras
  *
  *  Derived from "arch/i386/mm/init.c"
  *    Copyright (C) 1991, 1992, 1993, 1994  Linus Torvalds
  *
  *  This program is free software; you can redistribute it and/or
  *  modify it under the terms of the GNU General Public License
  *  as published by the Free Software Foundation; either version
  *  2 of the License, or (at your option) any later version.
  *
  */

 #include <linux/kernel.h>
 #include <linux/export.h>
 #include <linux/mm.h>
 #include <linux/init.h>
 #include <linux/highmem.h>
 #include <linux/pagemap.h>
 #include <linux/preempt.h>
 #include <linux/spinlock.h>
 #include <linux/memblock.h>
 #include <linux/of_fdt.h>
 #include <linux/hugetlb.h>

 #include <asm/tlbflush.h>
 #include <asm/tlb.h>
 #include <asm/code-patching.h>
 #include <asm/hugetlb.h>

 #include "mmu_decl.h"

 /*
  * This struct lists the sw-supported page sizes.  The hardawre MMU may support
  * other sizes not listed here.   The .ind field is only used on MMUs that have
  * indirect page table entries.
  */
 #ifdef CONFIG_PPC_BOOK3E_MMU
 #ifdef CONFIG_PPC_FSL_BOOK3E
 struct mmu_psize_def mmu_psize_defs[MMU_PAGE_COUNT] = {
 	[MMU_PAGE_4K] = {
 		.shift	= 12,
 		.enc	= BOOK3E_PAGESZ_4K,
 	},
 	[MMU_PAGE_4M] = {
 		.shift	= 22,
 		.enc	= BOOK3E_PAGESZ_4M,
 	},
 	[MMU_PAGE_16M] = {
 		.shift	= 24,
 		.enc	= BOOK3E_PAGESZ_16M,
 	},
 	[MMU_PAGE_64M] = {
 		.shift	= 26,
 		.enc	= BOOK3E_PAGESZ_64M,
 	},
 	[MMU_PAGE_256M] = {
 		.shift	= 28,
 		.enc	= BOOK3E_PAGESZ_256M,
 	},
 	[MMU_PAGE_1G] = {
 		.shift	= 30,
 		.enc	= BOOK3E_PAGESZ_1GB,
 	},
 };
 #else
 struct mmu_psize_def mmu_psize_defs[MMU_PAGE_COUNT] = {
 	[MMU_PAGE_4K] = {
 		.shift	= 12,
 		.ind	= 20,
 		.enc	= BOOK3E_PAGESZ_4K,
 	},
 	[MMU_PAGE_16K] = {
 		.shift	= 14,
 		.enc	= BOOK3E_PAGESZ_16K,
 	},
 	[MMU_PAGE_64K] = {
 		.shift	= 16,
 		.ind	= 28,
 		.enc	= BOOK3E_PAGESZ_64K,
 	},
 	[MMU_PAGE_1M] = {
 		.shift	= 20,
 		.enc	= BOOK3E_PAGESZ_1M,
 	},
 	[MMU_PAGE_16M] = {
 		.shift	= 24,
 		.ind	= 36,
 		.enc	= BOOK3E_PAGESZ_16M,
 	},
 	[MMU_PAGE_256M] = {
 		.shift	= 28,
 		.enc	= BOOK3E_PAGESZ_256M,
 	},
 	[MMU_PAGE_1G] = {
 		.shift	= 30,
 		.enc	= BOOK3E_PAGESZ_1GB,
 	},
 };
 #endif /* CONFIG_FSL_BOOKE */

 static inline int mmu_get_tsize(int psize)
 {
 	return mmu_psize_defs[psize].enc;
 }
 #else
 static inline int mmu_get_tsize(int psize)
 {
 	/* This isn't used on !Book3E for now */
 	return 0;
 }
 #endif /* CONFIG_PPC_BOOK3E_MMU */

 /* The variables below are currently only used on 64-bit Book3E
  * though this will probably be made common with other nohash
  * implementations at some point
  */
 #ifdef CONFIG_PPC64

 int mmu_linear_psize;		/* Page size used for the linear mapping */
 int mmu_pte_psize;		/* Page size used for PTE pages */
 int mmu_vmemmap_psize;		/* Page size used for the virtual mem map */
 int book3e_htw_enabled;		/* Is HW tablewalk enabled ? */
 unsigned long linear_map_top;	/* Top of linear mapping */

 #endif /* CONFIG_PPC64 */

 #ifdef CONFIG_PPC_FSL_BOOK3E
 /* next_tlbcam_idx is used to round-robin tlbcam entry assignment */
 DEFINE_PER_CPU(int, next_tlbcam_idx);
 EXPORT_PER_CPU_SYMBOL(next_tlbcam_idx);
 #endif

 /*
  * Base TLB flushing operations:
  *
  *  - flush_tlb_mm(mm) flushes the specified mm context TLB's
  *  - flush_tlb_page(vma, vmaddr) flushes one page
  *  - flush_tlb_range(vma, start, end) flushes a range of pages
  *  - flush_tlb_kernel_range(start, end) flushes kernel pages
  *
  *  - local_* variants of page and mm only apply to the current
  *    processor
  */

 /*
  * These are the base non-SMP variants of page and mm flushing
  */
 void local_flush_tlb_mm(struct mm_struct *mm)
 {
 	unsigned int pid;

 	preempt_disable();
 	pid = mm->context.id;
 	if (pid != MMU_NO_CONTEXT)
 		_tlbil_pid(pid);
 	preempt_enable();
 }
 EXPORT_SYMBOL(local_flush_tlb_mm);

 void __local_flush_tlb_page(struct mm_struct *mm, unsigned long vmaddr,
 			    int tsize, int ind)
 {
 	unsigned int pid;

 	preempt_disable();
 	pid = mm ? mm->context.id : 0;
 	if (pid != MMU_NO_CONTEXT)
 		_tlbil_va(vmaddr, pid, tsize, ind);
 	preempt_enable();
 }

 void local_flush_tlb_page(struct vm_area_struct *vma, unsigned long vmaddr)
 {
 	__local_flush_tlb_page(vma ? vma->vm_mm : NULL, vmaddr,
 			       mmu_get_tsize(mmu_virtual_psize), 0);
 }
 EXPORT_SYMBOL(local_flush_tlb_page);

 /*
  * And here are the SMP non-local implementations
  */
 #ifdef CONFIG_SMP

 static DEFINE_RAW_SPINLOCK(tlbivax_lock);

 static int mm_is_core_local(struct mm_struct *mm)
 {
 	return cpumask_subset(mm_cpumask(mm),
 			      topology_thread_cpumask(smp_processor_id()));
 }

 struct tlb_flush_param {
 	unsigned long addr;
 	unsigned int pid;
 	unsigned int tsize;
 	unsigned int ind;
 };

 static void do_flush_tlb_mm_ipi(void *param)
 {
 	struct tlb_flush_param *p = param;

 	_tlbil_pid(p ? p->pid : 0);
 }

 static void do_flush_tlb_page_ipi(void *param)
 {
 	struct tlb_flush_param *p = param;

 	_tlbil_va(p->addr, p->pid, p->tsize, p->ind);
 }


 /* Note on invalidations and PID:
  *
  * We snapshot the PID with preempt disabled. At this point, it can still
  * change either because:
  * - our context is being stolen (PID -> NO_CONTEXT) on another CPU
  * - we are invaliating some target that isn't currently running here
  *   and is concurrently acquiring a new PID on another CPU
  * - some other CPU is re-acquiring a lost PID for this mm
  * etc...
  *
  * However, this shouldn't be a problem as we only guarantee
  * invalidation of TLB entries present prior to this call, so we
  * don't care about the PID changing, and invalidating a stale PID
  * is generally harmless.
  */

 void flush_tlb_mm(struct mm_struct *mm)
 {
 	unsigned int pid;

 	preempt_disable();
 	pid = mm->context.id;
 	if (unlikely(pid == MMU_NO_CONTEXT))
 		goto no_context;
 	if (!mm_is_core_local(mm)) {
 		struct tlb_flush_param p = { .pid = pid };
 		/* Ignores smp_processor_id() even if set. */
 		smp_call_function_many(mm_cpumask(mm),
 				       do_flush_tlb_mm_ipi, &p, 1);
 	}
 	_tlbil_pid(pid);
  no_context:
 	preempt_enable();
 }
 EXPORT_SYMBOL(flush_tlb_mm);

 void __flush_tlb_page(struct mm_struct *mm, unsigned long vmaddr,
 		      int tsize, int ind)
 {
 	struct cpumask *cpu_mask;
 	unsigned int pid;

 	preempt_disable();
 	pid = mm ? mm->context.id : 0;
 	if (unlikely(pid == MMU_NO_CONTEXT))
 		goto bail;
 	cpu_mask = mm_cpumask(mm);
 	if (!mm_is_core_local(mm)) {
 		/* If broadcast tlbivax is supported, use it */
 		if (mmu_has_feature(MMU_FTR_USE_TLBIVAX_BCAST)) {
 			int lock = mmu_has_feature(MMU_FTR_LOCK_BCAST_INVAL);
 			if (lock)
 				raw_spin_lock(&tlbivax_lock);
 			_tlbivax_bcast(vmaddr, pid, tsize, ind);
 			if (lock)
 				raw_spin_unlock(&tlbivax_lock);
 			goto bail;
 		} else {
 			struct tlb_flush_param p = {
 				.pid = pid,
 				.addr = vmaddr,
 				.tsize = tsize,
 				.ind = ind,
 			};
 			/* Ignores smp_processor_id() even if set in cpu_mask */
 			smp_call_function_many(cpu_mask,
 					       do_flush_tlb_page_ipi, &p, 1);
 		}
 	}
 	_tlbil_va(vmaddr, pid, tsize, ind);
  bail:
 	preempt_enable();
 }

 void flush_tlb_page(struct vm_area_struct *vma, unsigned long vmaddr)
 {
 #ifdef CONFIG_HUGETLB_PAGE
 	if (is_vm_hugetlb_page(vma))
 		flush_hugetlb_page(vma, vmaddr);
 #endif

 	__flush_tlb_page(vma ? vma->vm_mm : NULL, vmaddr,
 			 mmu_get_tsize(mmu_virtual_psize), 0);
 }
 EXPORT_SYMBOL(flush_tlb_page);

 #endif /* CONFIG_SMP */

 #ifdef CONFIG_PPC_47x
 void __init early_init_mmu_47x(void)
 {
 #ifdef CONFIG_SMP
 	unsigned long root = of_get_flat_dt_root();
 	if (of_get_flat_dt_prop(root, "cooperative-partition", NULL))
 		mmu_clear_feature(MMU_FTR_USE_TLBIVAX_BCAST);
 #endif /* CONFIG_SMP */
 }
 #endif /* CONFIG_PPC_47x */

 /*
  * Flush kernel TLB entries in the given range
  */
 void flush_tlb_kernel_range(unsigned long start, unsigned long end)
 {
 #ifdef CONFIG_SMP
 	preempt_disable();
 	smp_call_function(do_flush_tlb_mm_ipi, NULL, 1);
 	_tlbil_pid(0);
 	preempt_enable();
 #else
 	_tlbil_pid(0);
 #endif
 }
 EXPORT_SYMBOL(flush_tlb_kernel_range);

 /*
  * Currently, for range flushing, we just do a full mm flush. This should
  * be optimized based on a threshold on the size of the range, since
  * some implementation can stack multiple tlbivax before a tlbsync but
  * for now, we keep it that way
  */
 void flush_tlb_range(struct vm_area_struct *vma, unsigned long start,
 		     unsigned long end)

 {
 	flush_tlb_mm(vma->vm_mm);
 }
 EXPORT_SYMBOL(flush_tlb_range);

 void tlb_flush(struct mmu_gather *tlb)
 {
 	flush_tlb_mm(tlb->mm);
 }

 /*
  * Below are functions specific to the 64-bit variant of Book3E though that
  * may change in the future
  */

 #ifdef CONFIG_PPC64

 /*
  * Handling of virtual linear page tables or indirect TLB entries
  * flushing when PTE pages are freed
  */
 void tlb_flush_pgtable(struct mmu_gather *tlb, unsigned long address)
 {
 	int tsize = mmu_psize_defs[mmu_pte_psize].enc;

 	if (book3e_htw_enabled) {
 		unsigned long start = address & PMD_MASK;
 		unsigned long end = address + PMD_SIZE;
 		unsigned long size = 1UL << mmu_psize_defs[mmu_pte_psize].shift;

 		/* This isn't the most optimal, ideally we would factor out the
 		 * while preempt & CPU mask mucking around, or even the IPI but
 		 * it will do for now
 		 */
 		while (start < end) {
 			__flush_tlb_page(tlb->mm, start, tsize, 1);
 			start += size;
 		}
 	} else {
 		unsigned long rmask = 0xf000000000000000ul;
 		unsigned long rid = (address & rmask) | 0x1000000000000000ul;
 		unsigned long vpte = address & ~rmask;

 #ifdef CONFIG_PPC_64K_PAGES
 		vpte = (vpte >> (PAGE_SHIFT - 4)) & ~0xfffful;
 #else
 		vpte = (vpte >> (PAGE_SHIFT - 3)) & ~0xffful;
 #endif
 		vpte |= rid;
 		__flush_tlb_page(tlb->mm, vpte, tsize, 0);
 	}
 }

 static void setup_page_sizes(void)
 {
 	unsigned int tlb0cfg;
 	unsigned int tlb0ps;
 	unsigned int eptcfg;
 	int i, psize;

 #ifdef CONFIG_PPC_FSL_BOOK3E
 	unsigned int mmucfg = mfspr(SPRN_MMUCFG);

 	if (((mmucfg & MMUCFG_MAVN) == MMUCFG_MAVN_V1) &&
 		(mmu_has_feature(MMU_FTR_TYPE_FSL_E))) {
 		unsigned int tlb1cfg = mfspr(SPRN_TLB1CFG);
 		unsigned int min_pg, max_pg;

 		min_pg = (tlb1cfg & TLBnCFG_MINSIZE) >> TLBnCFG_MINSIZE_SHIFT;
 		max_pg = (tlb1cfg & TLBnCFG_MAXSIZE) >> TLBnCFG_MAXSIZE_SHIFT;

 		for (psize = 0; psize < MMU_PAGE_COUNT; ++psize) {
 			struct mmu_psize_def *def;
 			unsigned int shift;

 			def = &mmu_psize_defs[psize];
 			shift = def->shift;

 			if (shift == 0)
 				continue;

 			/* adjust to be in terms of 4^shift Kb */
 			shift = (shift - 10) >> 1;

 			if ((shift >= min_pg) && (shift <= max_pg))
 				def->flags |= MMU_PAGE_SIZE_DIRECT;
 		}

 		goto no_indirect;
 	}
 #endif

 	tlb0cfg = mfspr(SPRN_TLB0CFG);
 	tlb0ps = mfspr(SPRN_TLB0PS);
 	eptcfg = mfspr(SPRN_EPTCFG);

 	/* Look for supported direct sizes */
 	for (psize = 0; psize < MMU_PAGE_COUNT; ++psize) {
 		struct mmu_psize_def *def = &mmu_psize_defs[psize];

 		if (tlb0ps & (1U << (def->shift - 10)))
 			def->flags |= MMU_PAGE_SIZE_DIRECT;
 	}

 	/* Indirect page sizes supported ? */
 	if ((tlb0cfg & TLBnCFG_IND) == 0)
 		goto no_indirect;

 	/* Now, we only deal with one IND page size for each
 	 * direct size. Hopefully all implementations today are
 	 * unambiguous, but we might want to be careful in the
 	 * future.
 	 */
 	for (i = 0; i < 3; i++) {
 		unsigned int ps, sps;

 		sps = eptcfg & 0x1f;
 		eptcfg >>= 5;
 		ps = eptcfg & 0x1f;
 		eptcfg >>= 5;
 		if (!ps || !sps)
 			continue;
 		for (psize = 0; psize < MMU_PAGE_COUNT; psize++) {
 			struct mmu_psize_def *def = &mmu_psize_defs[psize];

 			if (ps == (def->shift - 10))
 				def->flags |= MMU_PAGE_SIZE_INDIRECT;
 			if (sps == (def->shift - 10))
 				def->ind = ps + 10;
 		}
 	}
  no_indirect:

 	/* Cleanup array and print summary */
 	pr_info("MMU: Supported page sizes\n");
 	for (psize = 0; psize < MMU_PAGE_COUNT; ++psize) {
 		struct mmu_psize_def *def = &mmu_psize_defs[psize];
 		const char *__page_type_names[] = {
 			"unsupported",
 			"direct",
 			"indirect",
 			"direct & indirect"
 		};
 		if (def->flags == 0) {
 			def->shift = 0;
 			continue;
 		}
 		pr_info("  %8ld KB as %s\n", 1ul << (def->shift - 10),
 			__page_type_names[def->flags & 0x3]);
 	}
 }

 static void __patch_exception(int exc, unsigned long addr)
 {
 	extern unsigned int interrupt_base_book3e;
  	unsigned int *ibase = &interrupt_base_book3e;

 	/* Our exceptions vectors start with a NOP and -then- a branch
 	 * to deal with single stepping from userspace which stops on
 	 * the second instruction. Thus we need to patch the second
 	 * instruction of the exception, not the first one
 	 */

 	patch_branch(ibase + (exc / 4) + 1, addr, 0);
 }

 #define patch_exception(exc, name) do { \
 	extern unsigned int name; \
 	__patch_exception((exc), (unsigned long)&name); \
 } while (0)

 static void setup_mmu_htw(void)
 {
 	/* Check if HW tablewalk is present, and if yes, enable it by:
 	 *
 	 * - patching the TLB miss handlers to branch to the
 	 *   one dedicates to it
 	 *
 	 * - setting the global book3e_htw_enabled
        	 */
 	unsigned int tlb0cfg = mfspr(SPRN_TLB0CFG);

 	if ((tlb0cfg & TLBnCFG_IND) &&
 	    (tlb0cfg & TLBnCFG_PT)) {
 		patch_exception(0x1c0, exc_data_tlb_miss_htw_book3e);
 		patch_exception(0x1e0, exc_instruction_tlb_miss_htw_book3e);
 		book3e_htw_enabled = 1;
 	}
 	pr_info("MMU: Book3E HW tablewalk %s\n",
 		book3e_htw_enabled ? "enabled" : "not supported");
 }

 /*
  * Early initialization of the MMU TLB code
  */
 static void __early_init_mmu(int boot_cpu)
 {
 	unsigned int mas4;

 	/* XXX This will have to be decided at runtime, but right
 	 * now our boot and TLB miss code hard wires it. Ideally
 	 * we should find out a suitable page size and patch the
 	 * TLB miss code (either that or use the PACA to store
 	 * the value we want)
 	 */
 	mmu_linear_psize = MMU_PAGE_1G;

 	/* XXX This should be decided at runtime based on supported
 	 * page sizes in the TLB, but for now let's assume 16M is
 	 * always there and a good fit (which it probably is)
 	 */
 	mmu_vmemmap_psize = MMU_PAGE_16M;

 	/* XXX This code only checks for TLB 0 capabilities and doesn't
 	 *     check what page size combos are supported by the HW. It
 	 *     also doesn't handle the case where a separate array holds
 	 *     the IND entries from the array loaded by the PT.
 	 */
 	if (boot_cpu) {
 		/* Look for supported page sizes */
 		setup_page_sizes();

 		/* Look for HW tablewalk support */
 		setup_mmu_htw();
 	}

 	/* Set MAS4 based on page table setting */

 	mas4 = 0x4 << MAS4_WIMGED_SHIFT;
 	if (book3e_htw_enabled) {
 		mas4 |= mas4 | MAS4_INDD;
 #ifdef CONFIG_PPC_64K_PAGES
 		mas4 |=	BOOK3E_PAGESZ_256M << MAS4_TSIZED_SHIFT;
 		mmu_pte_psize = MMU_PAGE_256M;
 #else
 		mas4 |=	BOOK3E_PAGESZ_1M << MAS4_TSIZED_SHIFT;
 		mmu_pte_psize = MMU_PAGE_1M;
 #endif
 	} else {
 #ifdef CONFIG_PPC_64K_PAGES
 		mas4 |=	BOOK3E_PAGESZ_64K << MAS4_TSIZED_SHIFT;
 #else
 		mas4 |=	BOOK3E_PAGESZ_4K << MAS4_TSIZED_SHIFT;
 #endif
 		mmu_pte_psize = mmu_virtual_psize;
 	}
 	mtspr(SPRN_MAS4, mas4);

 	/* Set the global containing the top of the linear mapping
 	 * for use by the TLB miss code
 	 */
 	linear_map_top = memblock_end_of_DRAM();

 #ifdef CONFIG_PPC_FSL_BOOK3E
 	if (mmu_has_feature(MMU_FTR_TYPE_FSL_E)) {
 		unsigned int num_cams;

 		/* use a quarter of the TLBCAM for bolted linear map */
 		num_cams = (mfspr(SPRN_TLB1CFG) & TLBnCFG_N_ENTRY) / 4;
 		linear_map_top = map_mem_in_cams(linear_map_top, num_cams);

 		/* limit memory so we dont have linear faults */
 		memblock_enforce_memory_limit(linear_map_top);

 		patch_exception(0x1c0, exc_data_tlb_miss_bolted_book3e);
 		patch_exception(0x1e0, exc_instruction_tlb_miss_bolted_book3e);
 	}
 #endif

 	/* A sync won't hurt us after mucking around with
 	 * the MMU configuration
 	 */
 	mb();

 	memblock_set_current_limit(linear_map_top);
 }

 void __init early_init_mmu(void)
 {
 	__early_init_mmu(1);
 }

 void __cpuinit early_init_mmu_secondary(void)
 {
 	__early_init_mmu(0);
 }

 void setup_initial_memory_limit(phys_addr_t first_memblock_base,
 				phys_addr_t first_memblock_size)
 {
 	/* On non-FSL Embedded 64-bit, we adjust the RMA size to match
 	 * the bolted TLB entry. We know for now that only 1G
 	 * entries are supported though that may eventually
 	 * change.
 	 *
 	 * on FSL Embedded 64-bit, we adjust the RMA size to match the
 	 * first bolted TLB entry size.  We still limit max to 1G even if
 	 * the TLB could cover more.  This is due to what the early init
 	 * code is setup to do.
 	 *
 	 * We crop it to the size of the first MEMBLOCK to
 	 * avoid going over total available memory just in case...
 	 */
 #ifdef CONFIG_PPC_FSL_BOOK3E
 	if (mmu_has_feature(MMU_FTR_TYPE_FSL_E)) {
 		unsigned long linear_sz;
 		linear_sz = calc_cam_sz(first_memblock_size, PAGE_OFFSET,
 					first_memblock_base);
 		ppc64_rma_size = min_t(u64, linear_sz, 0x40000000);
 	} else
 #endif
 		ppc64_rma_size = min_t(u64, first_memblock_size, 0x40000000);

 	/* Finally limit subsequent allocations */
 	memblock_set_current_limit(first_memblock_base + ppc64_rma_size);
 }
 #else /* ! CONFIG_PPC64 */
 void __init early_init_mmu(void)
 {
 #ifdef CONFIG_PPC_47x
 	early_init_mmu_47x();
 #endif
 }
 #endif /* CONFIG_PPC64 */
	/*
	* This file contains the routines for TLB flushing.
	* On machines where the MMU does not use a hash table to store virtual to
	* physical translations (ie, SW loaded TLBs or Book3E compilant processors,
	* this does -not- include 603 however which shares the implementation with
	* hash based processors)
	*
	* -- BenH
	*
	* Copyright 2008,2009 Ben Herrenschmidt <benh@kernel.crashing.org>
	* IBM Corp.
	*
	* Derived from arch/ppc/mm/init.c:
	* Copyright (C) 1995-1996 Gary Thomas (gdt@linuxppc.org)
	*
	* Modifications by Paul Mackerras (PowerMac) (paulus@cs.anu.edu.au)
	* and Cort Dougan (PReP) (cort@cs.nmt.edu)
	* Copyright (C) 1996 Paul Mackerras
	*
	* Derived from "arch/i386/mm/init.c"
	* Copyright (C) 1991, 1992, 1993, 1994 Linus Torvalds
	*
	* This program is free software; you can redistribute it and/or
	* modify it under the terms of the GNU General Public License
	* as published by the Free Software Foundation; either version
	* 2 of the License, or (at your option) any later version.
	*
	*/

	#include <linux/kernel.h>
	#include <linux/export.h>
	#include <linux/mm.h>
	#include <linux/init.h>
	#include <linux/highmem.h>
	#include <linux/pagemap.h>
	#include <linux/preempt.h>
	#include <linux/spinlock.h>
	#include <linux/memblock.h>
	#include <linux/of_fdt.h>
	#include <linux/hugetlb.h>

	#include <asm/tlbflush.h>
	#include <asm/tlb.h>
	#include <asm/code-patching.h>
	#include <asm/hugetlb.h>

	#include "mmu_decl.h"

	/*
	* This struct lists the sw-supported page sizes. The hardawre MMU may support
	* other sizes not listed here. The .ind field is only used on MMUs that have
	* indirect page table entries.
	*/
	#ifdef CONFIG_PPC_BOOK3E_MMU
	#ifdef CONFIG_PPC_FSL_BOOK3E
	struct mmu_psize_def mmu_psize_defs[MMU_PAGE_COUNT] = {
	[MMU_PAGE_4K] = {
	.shift = 12,
	.enc = BOOK3E_PAGESZ_4K,
	},
	[MMU_PAGE_4M] = {
	.shift = 22,
	.enc = BOOK3E_PAGESZ_4M,
	},
	[MMU_PAGE_16M] = {
	.shift = 24,
	.enc = BOOK3E_PAGESZ_16M,
	},
	[MMU_PAGE_64M] = {
	.shift = 26,
	.enc = BOOK3E_PAGESZ_64M,
	},
	[MMU_PAGE_256M] = {
	.shift = 28,
	.enc = BOOK3E_PAGESZ_256M,
	},
	[MMU_PAGE_1G] = {
	.shift = 30,
	.enc = BOOK3E_PAGESZ_1GB,
	},
	};
	#else
	struct mmu_psize_def mmu_psize_defs[MMU_PAGE_COUNT] = {
	[MMU_PAGE_4K] = {
	.shift = 12,
	.ind = 20,
	.enc = BOOK3E_PAGESZ_4K,
	},
	[MMU_PAGE_16K] = {
	.shift = 14,
	.enc = BOOK3E_PAGESZ_16K,
	},
	[MMU_PAGE_64K] = {
	.shift = 16,
	.ind = 28,
	.enc = BOOK3E_PAGESZ_64K,
	},
	[MMU_PAGE_1M] = {
	.shift = 20,
	.enc = BOOK3E_PAGESZ_1M,
	},
	[MMU_PAGE_16M] = {
	.shift = 24,
	.ind = 36,
	.enc = BOOK3E_PAGESZ_16M,
	},
	[MMU_PAGE_256M] = {
	.shift = 28,
	.enc = BOOK3E_PAGESZ_256M,
	},
	[MMU_PAGE_1G] = {
	.shift = 30,
	.enc = BOOK3E_PAGESZ_1GB,
	},
	};
	#endif /* CONFIG_FSL_BOOKE */

	static inline int mmu_get_tsize(int psize)
	{
	return mmu_psize_defs[psize].enc;
	}
	#else
	static inline int mmu_get_tsize(int psize)
	{
	/* This isn't used on !Book3E for now */
	return 0;
	}
	#endif /* CONFIG_PPC_BOOK3E_MMU */

	/* The variables below are currently only used on 64-bit Book3E
	* though this will probably be made common with other nohash
	* implementations at some point
	*/
	#ifdef CONFIG_PPC64

	int mmu_linear_psize; /* Page size used for the linear mapping */
	int mmu_pte_psize; /* Page size used for PTE pages */
	int mmu_vmemmap_psize; /* Page size used for the virtual mem map */
	int book3e_htw_enabled; /* Is HW tablewalk enabled ? */
	unsigned long linear_map_top; /* Top of linear mapping */

	#endif /* CONFIG_PPC64 */

	#ifdef CONFIG_PPC_FSL_BOOK3E
	/* next_tlbcam_idx is used to round-robin tlbcam entry assignment */
	DEFINE_PER_CPU(int, next_tlbcam_idx);
	EXPORT_PER_CPU_SYMBOL(next_tlbcam_idx);
	#endif

	/*
	* Base TLB flushing operations:
	*
	* - flush_tlb_mm(mm) flushes the specified mm context TLB's
	* - flush_tlb_page(vma, vmaddr) flushes one page
	* - flush_tlb_range(vma, start, end) flushes a range of pages
	* - flush_tlb_kernel_range(start, end) flushes kernel pages
	*
	* - local_* variants of page and mm only apply to the current
	* processor
	*/

	/*
	* These are the base non-SMP variants of page and mm flushing
	*/
	void local_flush_tlb_mm(struct mm_struct *mm)
	{
	unsigned int pid;

	preempt_disable();
	pid = mm->context.id;
	if (pid != MMU_NO_CONTEXT)
	_tlbil_pid(pid);
	preempt_enable();
	}
	EXPORT_SYMBOL(local_flush_tlb_mm);

	void __local_flush_tlb_page(struct mm_struct *mm, unsigned long vmaddr,
	int tsize, int ind)
	{
	unsigned int pid;

	preempt_disable();
	pid = mm ? mm->context.id : 0;
	if (pid != MMU_NO_CONTEXT)
	_tlbil_va(vmaddr, pid, tsize, ind);
	preempt_enable();
	}

	void local_flush_tlb_page(struct vm_area_struct *vma, unsigned long vmaddr)
	{
	__local_flush_tlb_page(vma ? vma->vm_mm : NULL, vmaddr,
	mmu_get_tsize(mmu_virtual_psize), 0);
	}
	EXPORT_SYMBOL(local_flush_tlb_page);

	/*
	* And here are the SMP non-local implementations
	*/
	#ifdef CONFIG_SMP

	static DEFINE_RAW_SPINLOCK(tlbivax_lock);

	static int mm_is_core_local(struct mm_struct *mm)
	{
	return cpumask_subset(mm_cpumask(mm),
	topology_thread_cpumask(smp_processor_id()));
	}

	struct tlb_flush_param {
	unsigned long addr;
	unsigned int pid;
	unsigned int tsize;
	unsigned int ind;
	};

	static void do_flush_tlb_mm_ipi(void *param)
	{
	struct tlb_flush_param *p = param;

	_tlbil_pid(p ? p->pid : 0);
	}

	static void do_flush_tlb_page_ipi(void *param)
	{
	struct tlb_flush_param *p = param;

	_tlbil_va(p->addr, p->pid, p->tsize, p->ind);
	}


	/* Note on invalidations and PID:
	*
	* We snapshot the PID with preempt disabled. At this point, it can still
	* change either because:
	* - our context is being stolen (PID -> NO_CONTEXT) on another CPU
	* - we are invaliating some target that isn't currently running here
	* and is concurrently acquiring a new PID on another CPU
	* - some other CPU is re-acquiring a lost PID for this mm
	* etc...
	*
	* However, this shouldn't be a problem as we only guarantee
	* invalidation of TLB entries present prior to this call, so we
	* don't care about the PID changing, and invalidating a stale PID
	* is generally harmless.
	*/

	void flush_tlb_mm(struct mm_struct *mm)
	{
	unsigned int pid;

	preempt_disable();
	pid = mm->context.id;
	if (unlikely(pid == MMU_NO_CONTEXT))
	goto no_context;
	if (!mm_is_core_local(mm)) {
	struct tlb_flush_param p = { .pid = pid };
	/* Ignores smp_processor_id() even if set. */
	smp_call_function_many(mm_cpumask(mm),
	do_flush_tlb_mm_ipi, &p, 1);
	}
	_tlbil_pid(pid);
	no_context:
	preempt_enable();
	}
	EXPORT_SYMBOL(flush_tlb_mm);

	void __flush_tlb_page(struct mm_struct *mm, unsigned long vmaddr,
	int tsize, int ind)
	{
	struct cpumask *cpu_mask;
	unsigned int pid;

	preempt_disable();
	pid = mm ? mm->context.id : 0;
	if (unlikely(pid == MMU_NO_CONTEXT))
	goto bail;
	cpu_mask = mm_cpumask(mm);
	if (!mm_is_core_local(mm)) {
	/* If broadcast tlbivax is supported, use it */
	if (mmu_has_feature(MMU_FTR_USE_TLBIVAX_BCAST)) {
	int lock = mmu_has_feature(MMU_FTR_LOCK_BCAST_INVAL);
	if (lock)
	raw_spin_lock(&tlbivax_lock);
	_tlbivax_bcast(vmaddr, pid, tsize, ind);
	if (lock)
	raw_spin_unlock(&tlbivax_lock);
	goto bail;
	} else {
	struct tlb_flush_param p = {
	.pid = pid,
	.addr = vmaddr,
	.tsize = tsize,
	.ind = ind,
	};
	/* Ignores smp_processor_id() even if set in cpu_mask */
	smp_call_function_many(cpu_mask,
	do_flush_tlb_page_ipi, &p, 1);
	}
	}
	_tlbil_va(vmaddr, pid, tsize, ind);
	bail:
	preempt_enable();
	}

	void flush_tlb_page(struct vm_area_struct *vma, unsigned long vmaddr)
	{
	#ifdef CONFIG_HUGETLB_PAGE
	if (is_vm_hugetlb_page(vma))
	flush_hugetlb_page(vma, vmaddr);
	#endif

	__flush_tlb_page(vma ? vma->vm_mm : NULL, vmaddr,
	mmu_get_tsize(mmu_virtual_psize), 0);
	}
	EXPORT_SYMBOL(flush_tlb_page);

	#endif /* CONFIG_SMP */

	#ifdef CONFIG_PPC_47x
	void __init early_init_mmu_47x(void)
	{
	#ifdef CONFIG_SMP
	unsigned long root = of_get_flat_dt_root();
	if (of_get_flat_dt_prop(root, "cooperative-partition", NULL))
	mmu_clear_feature(MMU_FTR_USE_TLBIVAX_BCAST);
	#endif /* CONFIG_SMP */
	}
	#endif /* CONFIG_PPC_47x */

	/*
	* Flush kernel TLB entries in the given range
	*/
	void flush_tlb_kernel_range(unsigned long start, unsigned long end)
	{
	#ifdef CONFIG_SMP
	preempt_disable();
	smp_call_function(do_flush_tlb_mm_ipi, NULL, 1);
	_tlbil_pid(0);
	preempt_enable();
	#else
	_tlbil_pid(0);
	#endif
	}
	EXPORT_SYMBOL(flush_tlb_kernel_range);

	/*
	* Currently, for range flushing, we just do a full mm flush. This should
	* be optimized based on a threshold on the size of the range, since
	* some implementation can stack multiple tlbivax before a tlbsync but
	* for now, we keep it that way
	*/
	void flush_tlb_range(struct vm_area_struct *vma, unsigned long start,
	unsigned long end)

	{
	flush_tlb_mm(vma->vm_mm);
	}
	EXPORT_SYMBOL(flush_tlb_range);

	void tlb_flush(struct mmu_gather *tlb)
	{
	flush_tlb_mm(tlb->mm);
	}

	/*
	* Below are functions specific to the 64-bit variant of Book3E though that
	* may change in the future
	*/

	#ifdef CONFIG_PPC64

	/*
	* Handling of virtual linear page tables or indirect TLB entries
	* flushing when PTE pages are freed
	*/
	void tlb_flush_pgtable(struct mmu_gather *tlb, unsigned long address)
	{
	int tsize = mmu_psize_defs[mmu_pte_psize].enc;

	if (book3e_htw_enabled) {
	unsigned long start = address & PMD_MASK;
	unsigned long end = address + PMD_SIZE;
	unsigned long size = 1UL << mmu_psize_defs[mmu_pte_psize].shift;

	/* This isn't the most optimal, ideally we would factor out the
	* while preempt & CPU mask mucking around, or even the IPI but
	* it will do for now
	*/
	while (start < end) {
	__flush_tlb_page(tlb->mm, start, tsize, 1);
	start += size;
	}
	} else {
	unsigned long rmask = 0xf000000000000000ul;
	unsigned long rid = (address & rmask) \| 0x1000000000000000ul;
	unsigned long vpte = address & ~rmask;

	#ifdef CONFIG_PPC_64K_PAGES
	vpte = (vpte >> (PAGE_SHIFT - 4)) & ~0xfffful;
	#else
	vpte = (vpte >> (PAGE_SHIFT - 3)) & ~0xffful;
	#endif
	vpte \|= rid;
	__flush_tlb_page(tlb->mm, vpte, tsize, 0);
	}
	}

	static void setup_page_sizes(void)
	{
	unsigned int tlb0cfg;
	unsigned int tlb0ps;
	unsigned int eptcfg;
	int i, psize;

	#ifdef CONFIG_PPC_FSL_BOOK3E
	unsigned int mmucfg = mfspr(SPRN_MMUCFG);

	if (((mmucfg & MMUCFG_MAVN) == MMUCFG_MAVN_V1) &&
	(mmu_has_feature(MMU_FTR_TYPE_FSL_E))) {
	unsigned int tlb1cfg = mfspr(SPRN_TLB1CFG);
	unsigned int min_pg, max_pg;

	min_pg = (tlb1cfg & TLBnCFG_MINSIZE) >> TLBnCFG_MINSIZE_SHIFT;
	max_pg = (tlb1cfg & TLBnCFG_MAXSIZE) >> TLBnCFG_MAXSIZE_SHIFT;

	for (psize = 0; psize < MMU_PAGE_COUNT; ++psize) {
	struct mmu_psize_def *def;
	unsigned int shift;

	def = &mmu_psize_defs[psize];
	shift = def->shift;

	if (shift == 0)
	continue;

	/* adjust to be in terms of 4^shift Kb */
	shift = (shift - 10) >> 1;

	if ((shift >= min_pg) && (shift <= max_pg))
	def->flags \|= MMU_PAGE_SIZE_DIRECT;
	}

	goto no_indirect;
	}
	#endif

	tlb0cfg = mfspr(SPRN_TLB0CFG);
	tlb0ps = mfspr(SPRN_TLB0PS);
	eptcfg = mfspr(SPRN_EPTCFG);

	/* Look for supported direct sizes */
	for (psize = 0; psize < MMU_PAGE_COUNT; ++psize) {
	struct mmu_psize_def *def = &mmu_psize_defs[psize];

	if (tlb0ps & (1U << (def->shift - 10)))
	def->flags \|= MMU_PAGE_SIZE_DIRECT;
	}

	/* Indirect page sizes supported ? */
	if ((tlb0cfg & TLBnCFG_IND) == 0)
	goto no_indirect;

	/* Now, we only deal with one IND page size for each
	* direct size. Hopefully all implementations today are
	* unambiguous, but we might want to be careful in the
	* future.
	*/
	for (i = 0; i < 3; i++) {
	unsigned int ps, sps;

	sps = eptcfg & 0x1f;
	eptcfg >>= 5;
	ps = eptcfg & 0x1f;
	eptcfg >>= 5;
	if (!ps \|\| !sps)
	continue;
	for (psize = 0; psize < MMU_PAGE_COUNT; psize++) {
	struct mmu_psize_def *def = &mmu_psize_defs[psize];

	if (ps == (def->shift - 10))
	def->flags \|= MMU_PAGE_SIZE_INDIRECT;
	if (sps == (def->shift - 10))
	def->ind = ps + 10;
	}
	}
	no_indirect:

	/* Cleanup array and print summary */
	pr_info("MMU: Supported page sizes\n");
	for (psize = 0; psize < MMU_PAGE_COUNT; ++psize) {
	struct mmu_psize_def *def = &mmu_psize_defs[psize];
	const char *__page_type_names[] = {
	"unsupported",
	"direct",
	"indirect",
	"direct & indirect"
	};
	if (def->flags == 0) {
	def->shift = 0;
	continue;
	}
	pr_info(" %8ld KB as %s\n", 1ul << (def->shift - 10),
	__page_type_names[def->flags & 0x3]);
	}
	}

	static void __patch_exception(int exc, unsigned long addr)
	{
	extern unsigned int interrupt_base_book3e;
	unsigned int *ibase = &interrupt_base_book3e;

	/* Our exceptions vectors start with a NOP and -then- a branch
	* to deal with single stepping from userspace which stops on
	* the second instruction. Thus we need to patch the second
	* instruction of the exception, not the first one
	*/

	patch_branch(ibase + (exc / 4) + 1, addr, 0);
	}

	#define patch_exception(exc, name) do { \
	extern unsigned int name; \
	__patch_exception((exc), (unsigned long)&name); \
	} while (0)

	static void setup_mmu_htw(void)
	{
	/* Check if HW tablewalk is present, and if yes, enable it by:
	*
	* - patching the TLB miss handlers to branch to the
	* one dedicates to it
	*
	* - setting the global book3e_htw_enabled
	*/
	unsigned int tlb0cfg = mfspr(SPRN_TLB0CFG);

	if ((tlb0cfg & TLBnCFG_IND) &&
	(tlb0cfg & TLBnCFG_PT)) {
	patch_exception(0x1c0, exc_data_tlb_miss_htw_book3e);
	patch_exception(0x1e0, exc_instruction_tlb_miss_htw_book3e);
	book3e_htw_enabled = 1;
	}
	pr_info("MMU: Book3E HW tablewalk %s\n",
	book3e_htw_enabled ? "enabled" : "not supported");
	}

	/*
	* Early initialization of the MMU TLB code
	*/
	static void __early_init_mmu(int boot_cpu)
	{
	unsigned int mas4;

	/* XXX This will have to be decided at runtime, but right
	* now our boot and TLB miss code hard wires it. Ideally
	* we should find out a suitable page size and patch the
	* TLB miss code (either that or use the PACA to store
	* the value we want)
	*/
	mmu_linear_psize = MMU_PAGE_1G;

	/* XXX This should be decided at runtime based on supported
	* page sizes in the TLB, but for now let's assume 16M is
	* always there and a good fit (which it probably is)
	*/
	mmu_vmemmap_psize = MMU_PAGE_16M;

	/* XXX This code only checks for TLB 0 capabilities and doesn't
	* check what page size combos are supported by the HW. It
	* also doesn't handle the case where a separate array holds
	* the IND entries from the array loaded by the PT.
	*/
	if (boot_cpu) {
	/* Look for supported page sizes */
	setup_page_sizes();

	/* Look for HW tablewalk support */
	setup_mmu_htw();
	}

	/* Set MAS4 based on page table setting */

	mas4 = 0x4 << MAS4_WIMGED_SHIFT;
	if (book3e_htw_enabled) {
	mas4 \|= mas4 \| MAS4_INDD;
	#ifdef CONFIG_PPC_64K_PAGES
	mas4 \|= BOOK3E_PAGESZ_256M << MAS4_TSIZED_SHIFT;
	mmu_pte_psize = MMU_PAGE_256M;
	#else
	mas4 \|= BOOK3E_PAGESZ_1M << MAS4_TSIZED_SHIFT;
	mmu_pte_psize = MMU_PAGE_1M;
	#endif
	} else {
	#ifdef CONFIG_PPC_64K_PAGES
	mas4 \|= BOOK3E_PAGESZ_64K << MAS4_TSIZED_SHIFT;
	#else
	mas4 \|= BOOK3E_PAGESZ_4K << MAS4_TSIZED_SHIFT;
	#endif
	mmu_pte_psize = mmu_virtual_psize;
	}
	mtspr(SPRN_MAS4, mas4);

	/* Set the global containing the top of the linear mapping
	* for use by the TLB miss code
	*/
	linear_map_top = memblock_end_of_DRAM();

	#ifdef CONFIG_PPC_FSL_BOOK3E
	if (mmu_has_feature(MMU_FTR_TYPE_FSL_E)) {
	unsigned int num_cams;

	/* use a quarter of the TLBCAM for bolted linear map */
	num_cams = (mfspr(SPRN_TLB1CFG) & TLBnCFG_N_ENTRY) / 4;
	linear_map_top = map_mem_in_cams(linear_map_top, num_cams);

	/* limit memory so we dont have linear faults */
	memblock_enforce_memory_limit(linear_map_top);

	patch_exception(0x1c0, exc_data_tlb_miss_bolted_book3e);
	patch_exception(0x1e0, exc_instruction_tlb_miss_bolted_book3e);
	}
	#endif

	/* A sync won't hurt us after mucking around with
	* the MMU configuration
	*/
	mb();

	memblock_set_current_limit(linear_map_top);
	}

	void __init early_init_mmu(void)
	{
	__early_init_mmu(1);
	}

	void __cpuinit early_init_mmu_secondary(void)
	{
	__early_init_mmu(0);
	}

	void setup_initial_memory_limit(phys_addr_t first_memblock_base,
	phys_addr_t first_memblock_size)
	{
	/* On non-FSL Embedded 64-bit, we adjust the RMA size to match
	* the bolted TLB entry. We know for now that only 1G
	* entries are supported though that may eventually
	* change.
	*
	* on FSL Embedded 64-bit, we adjust the RMA size to match the
	* first bolted TLB entry size. We still limit max to 1G even if
	* the TLB could cover more. This is due to what the early init
	* code is setup to do.
	*
	* We crop it to the size of the first MEMBLOCK to
	* avoid going over total available memory just in case...
	*/
	#ifdef CONFIG_PPC_FSL_BOOK3E
	if (mmu_has_feature(MMU_FTR_TYPE_FSL_E)) {
	unsigned long linear_sz;
	linear_sz = calc_cam_sz(first_memblock_size, PAGE_OFFSET,
	first_memblock_base);
	ppc64_rma_size = min_t(u64, linear_sz, 0x40000000);
	} else
	#endif
	ppc64_rma_size = min_t(u64, first_memblock_size, 0x40000000);

	/* Finally limit subsequent allocations */
	memblock_set_current_limit(first_memblock_base + ppc64_rma_size);
	}
	#else /* ! CONFIG_PPC64 */
	void __init early_init_mmu(void)
	{
	#ifdef CONFIG_PPC_47x
	early_init_mmu_47x();
	#endif
	}
	#endif /* CONFIG_PPC64 */