/*
* PPC64 (POWER4) Huge TLB Page Support for Kernel.
*
* Copyright (C) 2003 David Gibson, IBM Corporation.
*
* Based on the IA-32 version:
* Copyright (C) 2002, Rohit Seth <rohit.seth@intel.com>
*/
#include <linux/init.h>
#include <linux/fs.h>
#include <linux/mm.h>
#include <linux/hugetlb.h>
#include <linux/pagemap.h>
#include <linux/smp_lock.h>
#include <linux/slab.h>
#include <linux/err.h>
#include <linux/sysctl.h>
#include <asm/mman.h>
#include <asm/pgalloc.h>
#include <asm/tlb.h>
#include <asm/tlbflush.h>
#include <asm/mmu_context.h>
#include <asm/machdep.h>
#include <asm/cputable.h>
#include <asm/tlb.h>
#include <linux/sysctl.h>
#define HUGEPGDIR_SHIFT (HPAGE_SHIFT + PAGE_SHIFT - 3)
#define HUGEPGDIR_SIZE (1UL << HUGEPGDIR_SHIFT)
#define HUGEPGDIR_MASK (~(HUGEPGDIR_SIZE-1))
#define HUGEPTE_INDEX_SIZE 9
#define HUGEPGD_INDEX_SIZE 10
#define PTRS_PER_HUGEPTE (1 << HUGEPTE_INDEX_SIZE)
#define PTRS_PER_HUGEPGD (1 << HUGEPGD_INDEX_SIZE)
static inline int hugepgd_index(unsigned long addr)
{
return (addr & ~REGION_MASK) >> HUGEPGDIR_SHIFT;
}
static pgd_t *hugepgd_offset(struct mm_struct *mm, unsigned long addr)
{
int index;
if (! mm->context.huge_pgdir)
return NULL;
index = hugepgd_index(addr);
BUG_ON(index >= PTRS_PER_HUGEPGD);
return mm->context.huge_pgdir + index;
}
static inline pte_t *hugepte_offset(pgd_t *dir, unsigned long addr)
{
int index;
if (pgd_none(*dir))
return NULL;
index = (addr >> HPAGE_SHIFT) % PTRS_PER_HUGEPTE;
return (pte_t *)pgd_page(*dir) + index;
}
static pgd_t *hugepgd_alloc(struct mm_struct *mm, unsigned long addr)
{
BUG_ON(! in_hugepage_area(mm->context, addr));
if (! mm->context.huge_pgdir) {
pgd_t *new;
spin_unlock(&mm->page_table_lock);
/* Don't use pgd_alloc(), because we want __GFP_REPEAT */
new = kmem_cache_alloc(zero_cache, GFP_KERNEL | __GFP_REPEAT);
BUG_ON(memcmp(new, empty_zero_page, PAGE_SIZE));
spin_lock(&mm->page_table_lock);
/*
* Because we dropped the lock, we should re-check the
* entry, as somebody else could have populated it..
*/
if (mm->context.huge_pgdir)
pgd_free(new);
else
mm->context.huge_pgdir = new;
}
return hugepgd_offset(mm, addr);
}
static pte_t *hugepte_alloc(struct mm_struct *mm, pgd_t *dir,
unsigned long addr)
{
if (! pgd_present(*dir)) {
pte_t *new;
spin_unlock(&mm->page_table_lock);
new = kmem_cache_alloc(zero_cache, GFP_KERNEL | __GFP_REPEAT);
BUG_ON(memcmp(new, empty_zero_page, PAGE_SIZE));
spin_lock(&mm->page_table_lock);
/*
* Because we dropped the lock, we should re-check the
* entry, as somebody else could have populated it..
*/
if (pgd_present(*dir)) {
if (new)
kmem_cache_free(zero_cache, new);
} else {
struct page *ptepage;
if (! new)
return NULL;
ptepage = virt_to_page(new);
ptepage->mapping = (void *) mm;
ptepage->index = addr & HUGEPGDIR_MASK;
pgd_populate(mm, dir, new);
}
}
return hugepte_offset(dir, addr);
}
static pte_t *huge_pte_offset(struct mm_struct *mm, unsigned long addr)
{
pgd_t *pgd;
BUG_ON(! in_hugepage_area(mm->context, addr));
pgd = hugepgd_offset(mm, addr);
if (! pgd)
return NULL;
return hugepte_offset(pgd, addr);
}
static pte_t *huge_pte_alloc(struct mm_struct *mm, unsigned long addr)
{
pgd_t *pgd;
BUG_ON(! in_hugepage_area(mm->context, addr));
pgd = hugepgd_alloc(mm, addr);
if (! pgd)
return NULL;
return hugepte_alloc(mm, pgd, addr);
}
static void set_huge_pte(struct mm_struct *mm, struct vm_area_struct *vma,
unsigned long addr, struct page *page,
pte_t *ptep, int write_access)
{
pte_t entry;
add_mm_counter(mm, rss, HPAGE_SIZE / PAGE_SIZE);
if (write_access) {
entry =
pte_mkwrite(pte_mkdirty(mk_pte(page, vma->vm_page_prot)));
} else {
entry = pte_wrprotect(mk_pte(page, vma->vm_page_prot));
}
entry = pte_mkyoung(entry);
entry = pte_mkhuge(entry);
set_pte_at(mm, addr, ptep, entry);
}
/*
* This function checks for proper alignment of input addr and len parameters.
*/
int is_aligned_hugepage_range(unsigned long addr, unsigned long len)
{
if (len & ~HPAGE_MASK)
return