diff options
Diffstat (limited to 'arch/x86/xen/mmu.c')
| -rw-r--r-- | arch/x86/xen/mmu.c | 2788 |
1 files changed, 2 insertions, 2786 deletions
diff --git a/arch/x86/xen/mmu.c b/arch/x86/xen/mmu.c index f226038a39ca..5e375a5e815f 100644 --- a/arch/x86/xen/mmu.c +++ b/arch/x86/xen/mmu.c @@ -1,84 +1,10 @@ -/* - * Xen mmu operations - * - * This file contains the various mmu fetch and update operations. - * The most important job they must perform is the mapping between the - * domain's pfn and the overall machine mfns. - * - * Xen allows guests to directly update the pagetable, in a controlled - * fashion. In other words, the guest modifies the same pagetable - * that the CPU actually uses, which eliminates the overhead of having - * a separate shadow pagetable. - * - * In order to allow this, it falls on the guest domain to map its - * notion of a "physical" pfn - which is just a domain-local linear - * address - into a real "machine address" which the CPU's MMU can - * use. - * - * A pgd_t/pmd_t/pte_t will typically contain an mfn, and so can be - * inserted directly into the pagetable. When creating a new - * pte/pmd/pgd, it converts the passed pfn into an mfn. Conversely, - * when reading the content back with __(pgd|pmd|pte)_val, it converts - * the mfn back into a pfn. - * - * The other constraint is that all pages which make up a pagetable - * must be mapped read-only in the guest. This prevents uncontrolled - * guest updates to the pagetable. Xen strictly enforces this, and - * will disallow any pagetable update which will end up mapping a - * pagetable page RW, and will disallow using any writable page as a - * pagetable. - * - * Naively, when loading %cr3 with the base of a new pagetable, Xen - * would need to validate the whole pagetable before going on. - * Naturally, this is quite slow. The solution is to "pin" a - * pagetable, which enforces all the constraints on the pagetable even - * when it is not actively in use. This menas that Xen can be assured - * that it is still valid when you do load it into %cr3, and doesn't - * need to revalidate it. - * - * Jeremy Fitzhardinge <jeremy@xensource.com>, XenSource Inc, 2007 - */ -#include <linux/sched/mm.h> -#include <linux/highmem.h> -#include <linux/debugfs.h> -#include <linux/bug.h> -#include <linux/vmalloc.h> -#include <linux/export.h> -#include <linux/init.h> -#include <linux/gfp.h> -#include <linux/memblock.h> -#include <linux/seq_file.h> -#include <linux/crash_dump.h> - -#include <trace/events/xen.h> - -#include <asm/pgtable.h> -#include <asm/tlbflush.h> -#include <asm/fixmap.h> -#include <asm/mmu_context.h> -#include <asm/setup.h> -#include <asm/paravirt.h> -#include <asm/e820/api.h> -#include <asm/linkage.h> -#include <asm/page.h> -#include <asm/init.h> -#include <asm/pat.h> -#include <asm/smp.h> - +#include <linux/pfn.h> +#include <asm/xen/page.h> #include <asm/xen/hypercall.h> -#include <asm/xen/hypervisor.h> - -#include <xen/xen.h> -#include <xen/page.h> -#include <xen/interface/xen.h> -#include <xen/interface/hvm/hvm_op.h> -#include <xen/interface/version.h> #include <xen/interface/memory.h> -#include <xen/hvc-console.h> #include "multicalls.h" #include "mmu.h" -#include "debugfs.h" /* * Protects atomic reservation decrease/increase against concurrent increases. @@ -86,45 +12,6 @@ */ DEFINE_SPINLOCK(xen_reservation_lock); -#ifdef CONFIG_X86_32 -/* - * Identity map, in addition to plain kernel map. This needs to be - * large enough to allocate page table pages to allocate the rest. - * Each page can map 2MB. - */ -#define LEVEL1_IDENT_ENTRIES (PTRS_PER_PTE * 4) -static RESERVE_BRK_ARRAY(pte_t, level1_ident_pgt, LEVEL1_IDENT_ENTRIES); -#endif -#ifdef CONFIG_X86_64 -/* l3 pud for userspace vsyscall mapping */ -static pud_t level3_user_vsyscall[PTRS_PER_PUD] __page_aligned_bss; -#endif /* CONFIG_X86_64 */ - -/* - * Note about cr3 (pagetable base) values: - * - * xen_cr3 contains the current logical cr3 value; it contains the - * last set cr3. This may not be the current effective cr3, because - * its update may be being lazily deferred. However, a vcpu looking - * at its own cr3 can use this value knowing that it everything will - * be self-consistent. - * - * xen_current_cr3 contains the actual vcpu cr3; it is set once the - * hypercall to set the vcpu cr3 is complete (so it may be a little - * out of date, but it will never be set early). If one vcpu is - * looking at another vcpu's cr3 value, it should use this variable. - */ -DEFINE_PER_CPU(unsigned long, xen_cr3); /* cr3 stored as physaddr */ -DEFINE_PER_CPU(unsigned long, xen_current_cr3); /* actual vcpu cr3 */ - -static phys_addr_t xen_pt_base, xen_pt_size __initdata; - -/* - * Just beyond the highest usermode address. STACK_TOP_MAX has a - * redzone above it, so round it up to a PGD boundary. - */ -#define USER_LIMIT ((STACK_TOP_MAX + PGDIR_SIZE - 1) & PGDIR_MASK) - unsigned long arbitrary_virt_to_mfn(void *vaddr) { xmaddr_t maddr = arbitrary_virt_to_machine(vaddr); @@ -155,1218 +42,6 @@ xmaddr_t arbitrary_virt_to_machine(void *vaddr) } EXPORT_SYMBOL_GPL(arbitrary_virt_to_machine); -void make_lowmem_page_readonly(void *vaddr) -{ - pte_t *pte, ptev; - unsigned long address = (unsigned long)vaddr; - unsigned int level; - - pte = lookup_address(address, &level); - if (pte == NULL) - return; /* vaddr missing */ - - ptev = pte_wrprotect(*pte); - - if (HYPERVISOR_update_va_mapping(address, ptev, 0)) - BUG(); -} - -void make_lowmem_page_readwrite(void *vaddr) -{ - pte_t *pte, ptev; - unsigned long address = (unsigned long)vaddr; - unsigned int level; - - pte = lookup_address(address, &level); - if (pte == NULL) - return; /* vaddr missing */ - - ptev = pte_mkwrite(*pte); - - if (HYPERVISOR_update_va_mapping(address, ptev, 0)) - BUG(); -} - - -static bool xen_page_pinned(void *ptr) -{ - struct page *page = virt_to_page(ptr); - - return PagePinned(page); -} - -void xen_set_domain_pte(pte_t *ptep, pte_t pteval, unsigned domid) -{ - struct multicall_space mcs; - struct mmu_update *u; - - trace_xen_mmu_set_domain_pte(ptep, pteval, domid); - - mcs = xen_mc_entry(sizeof(*u)); - u = mcs.args; - - /* ptep might be kmapped when using 32-bit HIGHPTE */ - u->ptr = virt_to_machine(ptep).maddr; - u->val = pte_val_ma(pteval); - - MULTI_mmu_update(mcs.mc, mcs.args, 1, NULL, domid); - - xen_mc_issue(PARAVIRT_LAZY_MMU); -} -EXPORT_SYMBOL_GPL(xen_set_domain_pte); - -static void xen_extend_mmu_update(const struct mmu_update *update) -{ - struct multicall_space mcs; - struct mmu_update *u; - - mcs = xen_mc_extend_args(__HYPERVISOR_mmu_update, sizeof(*u)); - - if (mcs.mc != NULL) { - mcs.mc->args[1]++; - } else { - mcs = __xen_mc_entry(sizeof(*u)); - MULTI_mmu_update(mcs.mc, mcs.args, 1, NULL, DOMID_SELF); - } - - u = mcs.args; - *u = *update; -} - -static void xen_extend_mmuext_op(const struct mmuext_op *op) -{ - struct multicall_space mcs; - struct mmuext_op *u; - - mcs = xen_mc_extend_args(__HYPERVISOR_mmuext_op, sizeof(*u)); - - if (mcs.mc != NULL) { - mcs.mc->args[1]++; - } else { - mcs = __xen_mc_entry(sizeof(*u)); - MULTI_mmuext_op(mcs.mc, mcs.args, 1, NULL, DOMID_SELF); - } - - u = mcs.args; - *u = *op; -} - -static void xen_set_pmd_hyper(pmd_t *ptr, pmd_t val) -{ - struct mmu_update u; - - preempt_disable(); - - xen_mc_batch(); - - /* ptr may be ioremapped for 64-bit pagetable setup */ - u.ptr = arbitrary_virt_to_machine(ptr).maddr; - u.val = pmd_val_ma(val); - xen_extend_mmu_update(&u); - - xen_mc_issue(PARAVIRT_LAZY_MMU); - - preempt_enable(); -} - -static void xen_set_pmd(pmd_t *ptr, pmd_t val) -{ - trace_xen_mmu_set_pmd(ptr, val); - - /* If page is not pinned, we can just update the entry - directly */ - if (!xen_page_pinned(ptr)) { - *ptr = val; - return; - } - - xen_set_pmd_hyper(ptr, val); -} - -/* - * Associate a virtual page frame with a given physical page frame - * and protection flags for that frame. - */ -void set_pte_mfn(unsigned long vaddr, unsigned long mfn, pgprot_t flags) -{ - set_pte_vaddr(vaddr, mfn_pte(mfn, flags)); -} - -static bool xen_batched_set_pte(pte_t *ptep, pte_t pteval) -{ - struct mmu_update u; - - if (paravirt_get_lazy_mode() != PARAVIRT_LAZY_MMU) - return false; - - xen_mc_batch(); - - u.ptr = virt_to_machine(ptep).maddr | MMU_NORMAL_PT_UPDATE; - u.val = pte_val_ma(pteval); - xen_extend_mmu_update(&u); - - xen_mc_issue(PARAVIRT_LAZY_MMU); - - return true; -} - -static inline void __xen_set_pte(pte_t *ptep, pte_t pteval) -{ - if (!xen_batched_set_pte(ptep, pteval)) { - /* - * Could call native_set_pte() here and trap and - * emulate the PTE write but with 32-bit guests this - * needs two traps (one for each of the two 32-bit - * words in the PTE) so do one hypercall directly - * instead. - */ - struct mmu_update u; - - u.ptr = virt_to_machine(ptep).maddr | MMU_NORMAL_PT_UPDATE; - u.val = pte_val_ma(pteval); - HYPERVISOR_mmu_update(&u, 1, NULL, DOMID_SELF); - } -} - -static void xen_set_pte(pte_t *ptep, pte_t pteval) -{ - trace_xen_mmu_set_pte(ptep, pteval); - __xen_set_pte(ptep, pteval); -} - -static void xen_set_pte_at(struct mm_struct *mm, unsigned long addr, - pte_t *ptep, pte_t pteval) -{ - trace_xen_mmu_set_pte_at(mm, addr, ptep, pteval); - __xen_set_pte(ptep, pteval); -} - -pte_t xen_ptep_modify_prot_start(struct mm_struct *mm, - unsigned long addr, pte_t *ptep) -{ - /* Just return the pte as-is. We preserve the bits on commit */ - trace_xen_mmu_ptep_modify_prot_start(mm, addr, ptep, *ptep); - return *ptep; -} - -void xen_ptep_modify_prot_commit(struct mm_struct *mm, unsigned long addr, - pte_t *ptep, pte_t pte) -{ - struct mmu_update u; - - trace_xen_mmu_ptep_modify_prot_commit(mm, addr, ptep, pte); - xen_mc_batch(); - - u.ptr = virt_to_machine(ptep).maddr | MMU_PT_UPDATE_PRESERVE_AD; - u.val = pte_val_ma(pte); - xen_extend_mmu_update(&u); - - xen_mc_issue(PARAVIRT_LAZY_MMU); -} - -/* Assume pteval_t is equivalent to all the other *val_t types. */ -static pteval_t pte_mfn_to_pfn(pteval_t val) -{ - if (val & _PAGE_PRESENT) { - unsigned long mfn = (val & PTE_PFN_MASK) >> PAGE_SHIFT; - unsigned long pfn = mfn_to_pfn(mfn); - - pteval_t flags = val & PTE_FLAGS_MASK; - if (unlikely(pfn == ~0)) - val = flags & ~_PAGE_PRESENT; - else - val = ((pteval_t)pfn << PAGE_SHIFT) | flags; - } - - return val; -} - -static pteval_t pte_pfn_to_mfn(pteval_t val) -{ - if (val & _PAGE_PRESENT) { - unsigned long pfn = (val & PTE_PFN_MASK) >> PAGE_SHIFT; - pteval_t flags = val & PTE_FLAGS_MASK; - unsigned long mfn; - - if (!xen_feature(XENFEAT_auto_translated_physmap)) - mfn = __pfn_to_mfn(pfn); - else - mfn = pfn; - /* - * If there's no mfn for the pfn, then just create an - * empty non-present pte. Unfortunately this loses - * information about the original pfn, so - * pte_mfn_to_pfn is asymmetric. - */ - if (unlikely(mfn == INVALID_P2M_ENTRY)) { - mfn = 0; - flags = 0; - } else - mfn &= ~(FOREIGN_FRAME_BIT | IDENTITY_FRAME_BIT); - val = ((pteval_t)mfn << PAGE_SHIFT) | flags; - } - - return val; -} - -__visible pteval_t xen_pte_val(pte_t pte) -{ - pteval_t pteval = pte.pte; - - return pte_mfn_to_pfn(pteval); -} -PV_CALLEE_SAVE_REGS_THUNK(xen_pte_val); - -__visible pgdval_t xen_pgd_val(pgd_t pgd) -{ - return pte_mfn_to_pfn(pgd.pgd); -} -PV_CALLEE_SAVE_REGS_THUNK(xen_pgd_val); - -__visible pte_t xen_make_pte(pteval_t pte) -{ - pte = pte_pfn_to_mfn(pte); - - return native_make_pte(pte); -} -PV_CALLEE_SAVE_REGS_THUNK(xen_make_pte); - -__visible pgd_t xen_make_pgd(pgdval_t pgd) -{ - pgd = pte_pfn_to_mfn(pgd); - return native_make_pgd(pgd); -} -PV_CALLEE_SAVE_REGS_THUNK(xen_make_pgd); - -__visible pmdval_t xen_pmd_val(pmd_t pmd) -{ - return pte_mfn_to_pfn(pmd.pmd); -} -PV_CALLEE_SAVE_REGS_THUNK(xen_pmd_val); - -static void xen_set_pud_hyper(pud_t *ptr, pud_t val) -{ - struct mmu_update u; - - preempt_disable(); - - xen_mc_batch(); - - /* ptr may be ioremapped for 64-bit pagetable setup */ - u.ptr = arbitrary_virt_to_machine(ptr).maddr; - u.val = pud_val_ma(val); - xen_extend_mmu_update(&u); - - xen_mc_issue(PARAVIRT_LAZY_MMU); - - preempt_enable(); -} - -static void xen_set_pud(pud_t *ptr, pud_t val) -{ - trace_xen_mmu_set_pud(ptr, val); - - /* If page is not pinned, we can just update the entry - directly */ - if (!xen_page_pinned(ptr)) { - *ptr = val; - return; - } - - xen_set_pud_hyper(ptr, val); -} - -#ifdef CONFIG_X86_PAE -static void xen_set_pte_atomic(pte_t *ptep, pte_t pte) -{ - trace_xen_mmu_set_pte_atomic(ptep, pte); - set_64bit((u64 *)ptep, native_pte_val(pte)); -} - -static void xen_pte_clear(struct mm_struct *mm, unsigned long addr, pte_t *ptep) -{ - trace_xen_mmu_pte_clear(mm, addr, ptep); - if (!xen_batched_set_pte(ptep, native_make_pte(0))) - native_pte_clear(mm, addr, ptep); -} - -static void xen_pmd_clear(pmd_t *pmdp) -{ - trace_xen_mmu_pmd_clear(pmdp); - set_pmd(pmdp, __pmd(0)); -} -#endif /* CONFIG_X86_PAE */ - -__visible pmd_t xen_make_pmd(pmdval_t pmd) -{ - pmd = pte_pfn_to_mfn(pmd); - return native_make_pmd(pmd); -} -PV_CALLEE_SAVE_REGS_THUNK(xen_make_pmd); - -#if CONFIG_PGTABLE_LEVELS == 4 -__visible pudval_t xen_pud_val(pud_t pud) -{ - return pte_mfn_to_pfn(pud.pud); -} -PV_CALLEE_SAVE_REGS_THUNK(xen_pud_val); - -__visible pud_t xen_make_pud(pudval_t pud) -{ - pud = pte_pfn_to_mfn(pud); - - return native_make_pud(pud); -} -PV_CALLEE_SAVE_REGS_THUNK(xen_make_pud); - -static pgd_t *xen_get_user_pgd(pgd_t *pgd) -{ - pgd_t *pgd_page = (pgd_t *)(((unsigned long)pgd) & PAGE_MASK); - unsigned offset = pgd - pgd_page; - pgd_t *user_ptr = NULL; - - if (offset < pgd_index(USER_LIMIT)) { - struct page *page = virt_to_page(pgd_page); - user_ptr = (pgd_t *)page->private; - if (user_ptr) - user_ptr += offset; - } - - return user_ptr; -} - -static void __xen_set_p4d_hyper(p4d_t *ptr, p4d_t val) -{ - struct mmu_update u; - - u.ptr = virt_to_machine(ptr).maddr; - u.val = p4d_val_ma(val); - xen_extend_mmu_update(&u); -} - -/* - * Raw hypercall-based set_p4d, intended for in early boot before - * there's a page structure. This implies: - * 1. The only existing pagetable is the kernel's - * 2. It is always pinned - * 3. It has no user pagetable attached to it - */ -static void __init xen_set_p4d_hyper(p4d_t *ptr, p4d_t val) -{ - preempt_disable(); - - xen_mc_batch(); - - __xen_set_p4d_hyper(ptr, val); - - xen_mc_issue(PARAVIRT_LAZY_MMU); - - preempt_enable(); -} - -static void xen_set_p4d(p4d_t *ptr, p4d_t val) -{ - pgd_t *user_ptr = xen_get_user_pgd((pgd_t *)ptr); - pgd_t pgd_val; - - trace_xen_mmu_set_p4d(ptr, (p4d_t *)user_ptr, val); - - /* If page is not pinned, we can just update the entry - directly */ - if (!xen_page_pinned(ptr)) { - *ptr = val; - if (user_ptr) { - WARN_ON(xen_page_pinned(user_ptr)); - pgd_val.pgd = p4d_val_ma(val); - *user_ptr = pgd_val; - } - return; - } - - /* If it's pinned, then we can at least batch the kernel and - user updates together. */ - xen_mc_batch(); - - __xen_set_p4d_hyper(ptr, val); - if (user_ptr) - __xen_set_p4d_hyper((p4d_t *)user_ptr, val); - - xen_mc_issue(PARAVIRT_LAZY_MMU); -} -#endif /* CONFIG_PGTABLE_LEVELS == 4 */ - -static int xen_pmd_walk(struct mm_struct *mm, pmd_t *pmd, - int (*func)(struct mm_struct *mm, struct page *, enum pt_level), - bool last, unsigned long limit) -{ - int i, nr, flush = 0; - - nr = last ? pmd_index(limit) + 1 : PTRS_PER_PMD; - for (i = 0; i < nr; i++) { - if (!pmd_none(pmd[i])) - flush |= (*func)(mm, pmd_page(pmd[i]), PT_PTE); - } - return flush; -} - -static int xen_pud_walk(struct mm_struct *mm, pud_t *pud, - int (*func)(struct mm_struct *mm, struct page *, enum pt_level), - bool last, unsigned long limit) -{ - int i, nr, flush = 0; - - nr = last ? pud_index(limit) + 1 : PTRS_PER_PUD; - for (i = 0; i < nr; i++) { - pmd_t *pmd; - - if (pud_none(pud[i])) - continue; - - pmd = pmd_offset(&pud[i], 0); - if (PTRS_PER_PMD > 1) - flush |= (*func)(mm, virt_to_page(pmd), PT_PMD); - flush |= xen_pmd_walk(mm, pmd, func, - last && i == nr - 1, limit); - } - return flush; -} - -static int xen_p4d_walk(struct mm_struct *mm, p4d_t *p4d, - int (*func)(struct mm_struct *mm, struct page *, enum pt_level), - bool last, unsigned long limit) -{ - int i, nr, flush = 0; - - nr = last ? p4d_index(limit) + 1 : PTRS_PER_P4D; - for (i = 0; i < nr; i++) { - pud_t *pud; - - if (p4d_none(p4d[i])) - continue; - - pud = pud_offset(&p4d[i], 0); - if (PTRS_PER_PUD > 1) - flush |= (*func)(mm, virt_to_page(pud), PT_PUD); - flush |= xen_pud_walk(mm, pud, func, - last && i == nr - 1, limit); - } - return flush; -} - -/* - * (Yet another) pagetable walker. This one is intended for pinning a - * pagetable. This means that it walks a pagetable and calls the - * callback function on each page it finds making up the page table, - * at every level. It walks the entire pagetable, but it only bothers - * pinning pte pages which are below limit. In the normal case this - * will be STACK_TOP_MAX, but at boot we need to pin up to - * FIXADDR_TOP. - * - * For 32-bit the important bit is that we don't pin beyond there, - * because then we start getting into Xen's ptes. - * - * For 64-bit, we must skip the Xen hole in the middle of the address - * space, just after the big x86-64 virtual hole. - */ -static int __xen_pgd_walk(struct mm_struct *mm, pgd_t *pgd, - int (*func)(struct mm_struct *mm, struct page *, - enum pt_level), - unsigned long limit) -{ - int i, nr, flush = 0; - unsigned hole_low, hole_high; - - /* The limit is the last byte to be touched */ - limit--; - BUG_ON(limit >= FIXADDR_TOP); - - if (xen_feature(XENFEAT_auto_translated_physmap)) - return 0; - - /* - * 64-bit has a great big hole in the middle of the address - * space, which contains the Xen mappings. On 32-bit these - * will end up making a zero-sized hole and so is a no-op. - */ - hole_low = pgd_index(USER_LIMIT); - hole_high = pgd_index(PAGE_OFFSET); - - nr = pgd_index(limit) + 1; - for (i = 0; i < nr; i++) { - p4d_t *p4d; - - if (i >= hole_low && i < hole_high) - continue; - - if (pgd_none(pgd[i])) - continue; - - p4d = p4d_offset(&pgd[i], 0); - if (PTRS_PER_P4D > 1) - flush |= (*func)(mm, virt_to_page(p4d), PT_P4D); - flush |= xen_p4d_walk(mm, p4d, func, i == nr - 1, limit); - } - - /* Do the top level last, so that the callbacks can use it as - a cue to do final things like tlb flushes. */ - flush |= (*func)(mm, virt_to_page(pgd), PT_PGD); - - return flush; -} - -static int xen_pgd_walk(struct mm_struct *mm, - int (*func)(struct mm_struct *mm, struct page *, - enum pt_level), - unsigned long limit) -{ - return __xen_pgd_walk(mm, mm->pgd, func, limit); -} - -/* If we're using split pte locks, then take the page's lock and - return a pointer to it. Otherwise return NULL. */ -static spinlock_t *xen_pte_lock(struct page *page, struct mm_struct *mm) -{ - spinlock_t *ptl = NULL; - -#if USE_SPLIT_PTE_PTLOCKS - ptl = ptlock_ptr(page); - spin_lock_nest_lock(ptl, &mm->page_table_lock); -#endif - - return ptl; -} - -static void xen_pte_unlock(void *v) -{ - spinlock_t *ptl = v; - spin_unlock(ptl); -} - -static void xen_do_pin(unsigned level, unsigned long pfn) -{ - struct mmuext_op op; - - op.cmd = level; - op.arg1.mfn = pfn_to_mfn(pfn); - - xen_extend_mmuext_op(&op); -} - -static int xen_pin_page(struct mm_struct *mm, struct page *page, - enum pt_level level) -{ - unsigned pgfl = TestSetPagePinned(page); - int flush; - - if (pgfl) - flush = 0; /* already pinned */ - else if (PageHighMem(page)) - /* kmaps need flushing if we found an unpinned - highpage */ - flush = 1; - else { - void *pt = lowmem_page_address(page); - unsigned long pfn = page_to_pfn(page); - struct multicall_space mcs = __xen_mc_entry(0); - spinlock_t *ptl; - - flush = 0; - - /* - * We need to hold the pagetable lock between the time - * we make the pagetable RO and when we actually pin - * it. If we don't, then other users may come in and - * attempt to update the pagetable by writing it, - * which will fail because the memory is RO but not - * pinned, so Xen won't do the trap'n'emulate. - * - * If we're using split pte locks, we can't hold the - * entire pagetable's worth of locks during the - * traverse, because we may wrap the preempt count (8 - * bits). The solution is to mark RO and pin each PTE - * page while holding the lock. This means the number - * of locks we end up holding is never more than a - * batch size (~32 entries, at present). - * - * If we're not using split pte locks, we needn't pin - * the PTE pages independently, because we're - * protected by the overall pagetable lock. - */ - ptl = NULL; - if (level == PT_PTE) - ptl = xen_pte_lock(page, mm); - - MULTI_update_va_mapping(mcs.mc, (unsigned long)pt, - pfn_pte(pfn, PAGE_KERNEL_RO), - level == PT_PGD ? UVMF_TLB_FLUSH : 0); - - if (ptl) { - xen_do_pin(MMUEXT_PIN_L1_TABLE, pfn); - - /* Queue a deferred unlock for when this batch - is completed. */ - xen_mc_callback(xen_pte_unlock, ptl); - } - } - - return flush; -} - -/* This is called just after a mm has been created, but it has not - been used yet. We need to make sure that its pagetable is all - read-only, and can be pinned. */ -static void __xen_pgd_pin(struct mm_struct *mm, pgd_t *pgd) -{ - trace_xen_mmu_pgd_pin(mm, pgd); - - xen_mc_batch(); - - if (__xen_pgd_walk(mm, pgd, xen_pin_page, USER_LIMIT)) { - /* re-enable interrupts for flushing */ - xen_mc_issue(0); - - kmap_flush_unused(); - - xen_mc_batch(); - } - -#ifdef CONFIG_X86_64 - { - pgd_t *user_pgd = xen_get_user_pgd(pgd); - - xen_do_pin(MMUEXT_PIN_L4_TABLE, PFN_DOWN(__pa(pgd))); - - if (user_pgd) { - xen_pin_page(mm, virt_to_page(user_pgd), PT_PGD); - xen_do_pin(MMUEXT_PIN_L4_TABLE, - PFN_DOWN(__pa(user_pgd))); - } - } -#else /* CONFIG_X86_32 */ -#ifdef CONFIG_X86_PAE - /* Need to make sure unshared kernel PMD is pinnable */ - xen_pin_page(mm, pgd_page(pgd[pgd_index(TASK_SIZE)]), - PT_PMD); -#endif - xen_do_pin(MMUEXT_PIN_L3_TABLE, PFN_DOWN(__pa(pgd))); -#endif /* CONFIG_X86_64 */ - xen_mc_issue(0); -} - -static void xen_pgd_pin(struct mm_struct *mm) -{ - __xen_pgd_pin(mm, mm->pgd); -} - -/* - * On save, we need to pin all pagetables to make sure they get their - * mfns turned into pfns. Search the list for any unpinned pgds and pin - * them (unpinned pgds are not currently in use, probably because the - * process is under construction or destruction). - * - * Expected to be called in stop_machine() ("equivalent to taking - * every spinlock in the system"), so the locking doesn't really - * matter all that much. - */ -void xen_mm_pin_all(void) -{ - struct page *page; - - spin_lock(&pgd_lock); - - list_for_each_entry(page, &pgd_list, lru) { - if (!PagePinned(page)) { - __xen_pgd_pin(&init_mm, (pgd_t *)page_address(page)); - SetPageSavePinned(page); - } - } - - spin_unlock(&pgd_lock); -} - -/* - * The init_mm pagetable is really pinned as soon as its created, but - * that's before we have page structures to store the bits. So do all - * the book-keeping now. - */ -static int __init xen_mark_pinned(struct mm_struct *mm, struct page *page, - enum pt_level level) -{ - SetPagePinned(page); - return 0; -} - -static void __init xen_mark_init_mm_pinned(void) -{ - xen_pgd_walk(&init_mm, xen_mark_pinned, FIXADDR_TOP); -} - -static int xen_unpin_page(struct mm_struct *mm, struct page *page, - enum pt_level level) -{ - unsigned pgfl = TestClearPagePinned(page); - - if (pgfl && !PageHighMem(page)) { - void *pt = lowmem_page_address(page); - unsigned long pfn = page_to_pfn(page); - spinlock_t *ptl = NULL; - struct multicall_space mcs; - - /* - * Do the converse to pin_page. If we're using split - * pte locks, we must be holding the lock for while - * the pte page is unpinned but still RO to prevent - * concurrent updates from seeing it in this - * partially-pinned state. - */ - if (level == PT_PTE) { - ptl = xen_pte_lock(page, mm); - - if (ptl) - xen_do_pin(MMUEXT_UNPIN_TABLE, pfn); - } - - mcs = __xen_mc_entry(0); - - MULTI_update_va_mapping(mcs.mc, (unsigned long)pt, - pfn_pte(pfn, PAGE_KERNEL), - level == PT_PGD ? UVMF_TLB_FLUSH : 0); - - if (ptl) { - /* unlock when batch completed */ - xen_mc_callback(xen_pte_unlock, ptl); - } - } - - return 0; /* never need to flush on unpin */ -} - -/* Release a pagetables pages back as normal RW */ -static void __xen_pgd_unpin(struct mm_struct *mm, pgd_t *pgd) -{ - trace_xen_mmu_pgd_unpin(mm, pgd); - - xen_mc_batch(); - - xen_do_pin(MMUEXT_UNPIN_TABLE, PFN_DOWN(__pa(pgd))); - -#ifdef CONFIG_X86_64 - { - pgd_t *user_pgd = xen_get_user_pgd(pgd); - - if (user_pgd) { - xen_do_pin(MMUEXT_UNPIN_TABLE, - PFN_DOWN(__pa(user_pgd))); - xen_unpin_page(mm, virt_to_page(user_pgd), PT_PGD); - } - } -#endif - -#ifdef CONFIG_X86_PAE - /* Need to make sure unshared kernel PMD is unpinned */ - xen_unpin_page(mm, pgd_page(pgd[pgd_index(TASK_SIZE)]), - PT_PMD); -#endif - - __xen_pgd_walk(mm, pgd, xen_unpin_page, USER_LIMIT); - - xen_mc_issue(0); -} - -static void xen_pgd_unpin(struct mm_struct *mm) -{ - __xen_pgd_unpin(mm, mm->pgd); -} - -/* - * On resume, undo any pinning done at save, so that the rest of the - * kernel doesn't see any unexpected pinned pagetables. - */ -void xen_mm_unpin_all(void) -{ - struct page *page; - - spin_lock(&pgd_lock); - - list_for_each_entry(page, &pgd_list, lru) { - if (PageSavePinned(page)) { - BUG_ON(!PagePinned(page)); - __xen_pgd_unpin(&init_mm, (pgd_t *)page_address(page)); - ClearPageSavePinned(page); - } - } - - spin_unlock(&pgd_lock); -} - -static void xen_activate_mm(struct mm_struct *prev, struct mm_struct *next) -{ - spin_lock(&next->page_table_lock); - xen_pgd_pin(next); - spin_unlock(&next->page_table_lock); -} - -static void xen_dup_mmap(struct mm_struct *oldmm, struct mm_struct *mm) -{ - spin_lock(&mm->page_table_lock); - xen_pgd_pin(mm); - spin_unlock(&mm->page_table_lock); -} - - -#ifdef CONFIG_SMP -/* Another cpu may still have their %cr3 pointing at the pagetable, so - we need to repoint it somewhere else before we can unpin it. */ -static void drop_other_mm_ref(void *info) -{ - struct mm_struct *mm = info; - struct mm_struct *active_mm; - - active_mm = this_cpu_read(cpu_tlbstate.active_mm); - - if (active_mm == mm && this_cpu_read(cpu_tlbstate.state) != TLBSTATE_OK) - leave_mm(smp_processor_id()); - - /* If this cpu still has a stale cr3 reference, then make sure - it has been flushed. */ - if (this_cpu_read(xen_current_cr3) == __pa(mm->pgd)) - load_cr3(swapper_pg_dir); -} - -static void xen_drop_mm_ref(struct mm_struct *mm) -{ - cpumask_var_t mask; - unsigned cpu; - - if (current->active_mm == mm) { - if (current->mm == mm) - load_cr3(swapper_pg_dir); - else - leave_mm(smp_processor_id()); - } - - /* Get the "official" set of cpus referring to our pagetable. */ - if (!alloc_cpumask_var(&mask, GFP_ATOMIC)) { - for_each_online_cpu(cpu) { - if (!cpumask_test_cpu(cpu, mm_cpumask(mm)) - && per_cpu(xen_current_cr3, cpu) != __pa(mm->pgd)) - continue; - smp_call_function_single(cpu, drop_other_mm_ref, mm, 1); - } - return; - } - cpumask_copy(mask, mm_cpumask(mm)); - - /* It's possible that a vcpu may have a stale reference to our - cr3, because its in lazy mode, and it hasn't yet flushed - its set of pending hypercalls yet. In this case, we can - look at its actual current cr3 value, and force it to flush - if needed. */ - for_each_online_cpu(cpu) { - if (per_cpu(xen_current_cr3, cpu) == __pa(mm->pgd)) - cpumask_set_cpu(cpu, mask); - } - - if (!cpumask_empty(mask)) - smp_call_function_many(mask, drop_other_mm_ref, mm, 1); - free_cpumask_var(mask); -} -#else -static void xen_drop_mm_ref(struct mm_struct *mm) -{ - if (current->active_mm == mm) - load_cr3(swapper_pg_dir); -} -#endif - -/* - * While a process runs, Xen pins its pagetables, which means that the - * hypervisor forces it to be read-only, and it controls all updates - * to it. This means that all pagetable updates have to go via the - * hypervisor, which is moderately expensive. - * - * Since we're pulling the pagetable down, we switch to use init_mm, - * unpin old process pagetable and mark it all read-write, which - * allows further operations on it to be simple memory accesses. - * - * The only subtle point is that another CPU may be still using the - * pagetable because of lazy tlb flushing. This means we need need to - * switch all CPUs off this pagetable before we can unpin it. - */ -static void xen_exit_mmap(struct mm_struct *mm) -{ - get_cpu(); /* make sure we don't move around */ - xen_drop_mm_ref(mm); - put_cpu(); - - spin_lock(&mm->page_table_lock); - - /* pgd may not be pinned in the error exit path of execve */ - if (xen_page_pinned(mm->pgd)) - xen_pgd_unpin(mm); - - spin_unlock(&mm->page_table_lock); -} - -static void xen_post_allocator_init(void); - -static void __init pin_pagetable_pfn(unsigned cmd, unsigned long pfn) -{ - struct mmuext_op op; - - op.cmd = cmd; - op.arg1.mfn = pfn_to_mfn(pfn); - if (HYPERVISOR_mmuext_op(&op, 1, NULL, DOMID_SELF)) - BUG(); -} - -#ifdef CONFIG_X86_64 -static void __init xen_cleanhighmap(unsigned long vaddr, - unsigned long vaddr_end) -{ - unsigned long kernel_end = roundup((unsigned long)_brk_end, PMD_SIZE) - 1; - pmd_t *pmd = level2_kernel_pgt + pmd_index(vaddr); - - /* NOTE: The loop is more greedy than the cleanup_highmap variant. - * We include the PMD passed in on _both_ boundaries. */ - for (; vaddr <= vaddr_end && (pmd < (level2_kernel_pgt + PTRS_PER_PMD)); - pmd++, vaddr += PMD_SIZE) { - if (pmd_none(*pmd)) - continue; - if (vaddr < (unsigned long) _text || vaddr > kernel_end) - set_pmd(pmd, __pmd(0)); - } - /* In case we did something silly, we should crash in this function - * instead of somewhere later and be confusing. */ - xen_mc_flush(); -} - -/* - * Make a page range writeable and free it. - */ -static void __init xen_free_ro_pages(unsigned long paddr, unsigned long size) -{ - void *vaddr = __va(paddr); - void *vaddr_end = vaddr + size; - - for (; vaddr < vaddr_end; vaddr += PAGE_SIZE) - make_lowmem_page_readwrite(vaddr); - - memblock_free(paddr, size); -} - -static void __init xen_cleanmfnmap_free_pgtbl(void *pgtbl, bool unpin) -{ - unsigned long pa = __pa(pgtbl) & PHYSICAL_PAGE_MASK; - - if (unpin) - pin_pagetable_pfn(MMUEXT_UNPIN_TABLE, PFN_DOWN(pa)); - ClearPagePinned(virt_to_page(__va(pa))); - xen_free_ro_pages(pa, PAGE_SIZE); -} - -static void __init xen_cleanmfnmap_pmd(pmd_t *pmd, bool unpin) -{ - unsigned long pa; - pte_t *pte_tbl; - int i; - - if (pmd_large(*pmd)) { - pa = pmd_val(*pmd) & PHYSICAL_PAGE_MASK; - xen_free_ro_pages(pa, PMD_SIZE); - return; - } - - pte_tbl = pte_offset_kernel(pmd, 0); - for (i = 0; i < PTRS_PER_PTE; i++) { - if (pte_none(pte_tbl[i])) - continue; - pa = pte_pfn(pte_tbl[i]) << PAGE_SHIFT; - xen_free_ro_pages(pa, PAGE_SIZE); - } - set_pmd(pmd, __pmd(0)); - xen_cleanmfnmap_free_pgtbl(pte_tbl, unpin); -} - -static void __init xen_cleanmfnmap_pud(pud_t *pud, bool unpin) -{ - unsigned long pa; - pmd_t *pmd_tbl; - int i; - - if (pud_large(*pud)) { - pa = pud_val(*pud) & PHYSICAL_PAGE_MASK; - xen_free_ro_pages(pa, PUD_SIZE); - return; - } - - pmd_tbl = pmd_offset(pud, 0); - for (i = 0; i < PTRS_PER_PMD; i++) { - if (pmd_none(pmd_tbl[i])) - continue; - xen_cleanmfnmap_pmd(pmd_tbl + i, unpin); - } - set_pud(pud, __pud(0)); - xen_cleanmfnmap_free_pgtbl(pmd_tbl, unpin); -} - -static void __init xen_cleanmfnmap_p4d(p4d_t *p4d, bool unpin) -{ - unsigned long pa; - pud_t *pud_tbl; - int i; - - if (p4d_large(*p4d)) { - pa = p4d_val(*p4d) & PHYSICAL_PAGE_MASK; - xen_free_ro_pages(pa, P4D_SIZE); - return; - } - - pud_tbl = pud_offset(p4d, 0); - for (i = 0; i < PTRS_PER_PUD; i++) { - if (pud_none(pud_tbl[i])) - continue; - xen_cleanmfnmap_pud(pud_tbl + i, unpin); - } - set_p4d(p4d, __p4d(0)); - xen_cleanmfnmap_free_pgtbl(pud_tbl, unpin); -} - -/* - * Since it is well isolated we can (and since it is perhaps large we should) - * also free the page tables mapping the initial P->M table.< |