diff options
author | Ralf Baechle <ralf@linux-mips.org> | 2012-01-11 15:41:47 +0100 |
---|---|---|
committer | Ralf Baechle <ralf@linux-mips.org> | 2012-01-11 15:41:47 +0100 |
commit | 39b741431af7f6f46b2e0e7f7f13ea2351fb4a5f (patch) | |
tree | 89355f4ae7bbb874537bb65f71ba0d19b3d468e1 /arch/mips | |
parent | 5b0ec2efb7d373faa7b1a7632c459b93895d45cd (diff) | |
parent | d7a887a73dec6c387b02a966a71aac767bbd9ce6 (diff) |
Merge branch 'next/generic' into mips-for-linux-next
Diffstat (limited to 'arch/mips')
51 files changed, 946 insertions, 155 deletions
diff --git a/arch/mips/alchemy/common/dbdma.c b/arch/mips/alchemy/common/dbdma.c index 0e63ee487d6d..9ba407b4fc97 100644 --- a/arch/mips/alchemy/common/dbdma.c +++ b/arch/mips/alchemy/common/dbdma.c @@ -1019,8 +1019,7 @@ static int __init dbdma_setup(unsigned int irq, dbdev_tab_t *idtable) dbdma_gptr->ddma_inten = 0xffff; au_sync(); - ret = request_irq(irq, dbdma_interrupt, IRQF_DISABLED, "dbdma", - (void *)dbdma_gptr); + ret = request_irq(irq, dbdma_interrupt, 0, "dbdma", (void *)dbdma_gptr); if (ret) printk(KERN_ERR "Cannot grab DBDMA interrupt!\n"); else { diff --git a/arch/mips/alchemy/common/time.c b/arch/mips/alchemy/common/time.c index d5da6adbf634..146a5fa80360 100644 --- a/arch/mips/alchemy/common/time.c +++ b/arch/mips/alchemy/common/time.c @@ -92,7 +92,7 @@ static struct clock_event_device au1x_rtcmatch2_clockdev = { static struct irqaction au1x_rtcmatch2_irqaction = { .handler = au1x_rtcmatch2_irq, - .flags = IRQF_DISABLED | IRQF_TIMER, + .flags = IRQF_TIMER, .name = "timer", .dev_id = &au1x_rtcmatch2_clockdev, }; diff --git a/arch/mips/alchemy/devboards/db1200/platform.c b/arch/mips/alchemy/devboards/db1200/platform.c index c61867c93c4a..78459c17c628 100644 --- a/arch/mips/alchemy/devboards/db1200/platform.c +++ b/arch/mips/alchemy/devboards/db1200/platform.c @@ -276,12 +276,12 @@ static int db1200_mmc_cd_setup(void *mmc_host, int en) if (en) { ret = request_irq(DB1200_SD0_INSERT_INT, db1200_mmc_cd, - IRQF_DISABLED, "sd_insert", mmc_host); + 0, "sd_insert", mmc_host); if (ret) goto out; ret = request_irq(DB1200_SD0_EJECT_INT, db1200_mmc_cd, - IRQF_DISABLED, "sd_eject", mmc_host); + 0, "sd_eject", mmc_host); if (ret) { free_irq(DB1200_SD0_INSERT_INT, mmc_host); goto out; diff --git a/arch/mips/cavium-octeon/setup.c b/arch/mips/cavium-octeon/setup.c index 853bac4f25b2..260b27367347 100644 --- a/arch/mips/cavium-octeon/setup.c +++ b/arch/mips/cavium-octeon/setup.c @@ -642,14 +642,6 @@ void __init plat_mem_setup(void) total = 0; - /* First add the init memory we will be returning. */ - memory = __pa_symbol(&__init_begin) & PAGE_MASK; - mem_alloc_size = (__pa_symbol(&__init_end) & PAGE_MASK) - memory; - if (mem_alloc_size > 0) { - add_memory_region(memory, mem_alloc_size, BOOT_MEM_RAM); - total += mem_alloc_size; - } - /* * The Mips memory init uses the first memory location for * some memory vectors. When SPARSEMEM is in use, it doesn't diff --git a/arch/mips/dec/setup.c b/arch/mips/dec/setup.c index f7b7ba6d5c45..b874accd878a 100644 --- a/arch/mips/dec/setup.c +++ b/arch/mips/dec/setup.c @@ -110,7 +110,6 @@ static struct irqaction fpuirq = { }; static struct irqaction busirq = { - .flags = IRQF_DISABLED, .name = "bus error", .flags = IRQF_NO_THREAD, }; diff --git a/arch/mips/include/asm/bootinfo.h b/arch/mips/include/asm/bootinfo.h index 35cd1bab69c3..7a51d879e6ca 100644 --- a/arch/mips/include/asm/bootinfo.h +++ b/arch/mips/include/asm/bootinfo.h @@ -86,6 +86,7 @@ extern unsigned long mips_machtype; #define BOOT_MEM_RAM 1 #define BOOT_MEM_ROM_DATA 2 #define BOOT_MEM_RESERVED 3 +#define BOOT_MEM_INIT_RAM 4 /* * A memory map that's built upon what was determined diff --git a/arch/mips/include/asm/gio_device.h b/arch/mips/include/asm/gio_device.h new file mode 100644 index 000000000000..5437c84664bf --- /dev/null +++ b/arch/mips/include/asm/gio_device.h @@ -0,0 +1,56 @@ +#include <linux/device.h> +#include <linux/mod_devicetable.h> + +struct gio_device_id { + __u8 id; +}; + +struct gio_device { + struct device dev; + struct resource resource; + unsigned int irq; + unsigned int slotno; + + const char *name; + struct gio_device_id id; + unsigned id32:1; + unsigned gio64:1; +}; +#define to_gio_device(d) container_of(d, struct gio_device, dev) + +struct gio_driver { + const char *name; + struct module *owner; + const struct gio_device_id *id_table; + + int (*probe)(struct gio_device *, const struct gio_device_id *); + void (*remove)(struct gio_device *); + int (*suspend)(struct gio_device *, pm_message_t); + int (*resume)(struct gio_device *); + void (*shutdown)(struct gio_device *); + + struct device_driver driver; +}; +#define to_gio_driver(drv) container_of(drv, struct gio_driver, driver) + +extern const struct gio_device_id *gio_match_device(const struct gio_device_id *, + const struct gio_device *); +extern struct gio_device *gio_dev_get(struct gio_device *); +extern void gio_dev_put(struct gio_device *); + +extern int gio_device_register(struct gio_device *); +extern void gio_device_unregister(struct gio_device *); +extern void gio_release_dev(struct device *); + +static inline void gio_device_free(struct gio_device *dev) +{ + gio_release_dev(&dev->dev); +} + +extern int gio_register_driver(struct gio_driver *); +extern void gio_unregister_driver(struct gio_driver *); + +#define gio_get_drvdata(_dev) drv_get_drvdata(&(_dev)->dev) +#define gio_set_drvdata(_dev, data) drv_set_drvdata(&(_dev)->dev, (data)) + +extern void gio_set_master(struct gio_device *); diff --git a/arch/mips/include/asm/hugetlb.h b/arch/mips/include/asm/hugetlb.h index c565b7c3f0b5..58d36889f09b 100644 --- a/arch/mips/include/asm/hugetlb.h +++ b/arch/mips/include/asm/hugetlb.h @@ -70,7 +70,7 @@ static inline pte_t huge_ptep_get_and_clear(struct mm_struct *mm, static inline void huge_ptep_clear_flush(struct vm_area_struct *vma, unsigned long addr, pte_t *ptep) { - flush_tlb_mm(vma->vm_mm); + flush_tlb_page(vma, addr & huge_page_mask(hstate_vma(vma))); } static inline int huge_pte_none(pte_t pte) diff --git a/arch/mips/include/asm/mach-generic/floppy.h b/arch/mips/include/asm/mach-generic/floppy.h index 001a8ce17c17..a38f4d43e5e5 100644 --- a/arch/mips/include/asm/mach-generic/floppy.h +++ b/arch/mips/include/asm/mach-generic/floppy.h @@ -98,7 +98,7 @@ static inline void fd_disable_irq(void) static inline int fd_request_irq(void) { return request_irq(FLOPPY_IRQ, floppy_interrupt, - IRQF_DISABLED, "floppy", NULL); + 0, "floppy", NULL); } static inline void fd_free_irq(void) diff --git a/arch/mips/include/asm/mach-jazz/floppy.h b/arch/mips/include/asm/mach-jazz/floppy.h index 56e9ca6ae426..88b5acb75145 100644 --- a/arch/mips/include/asm/mach-jazz/floppy.h +++ b/arch/mips/include/asm/mach-jazz/floppy.h @@ -90,7 +90,7 @@ static inline void fd_disable_irq(void) static inline int fd_request_irq(void) { return request_irq(FLOPPY_IRQ, floppy_interrupt, - IRQF_DISABLED, "floppy", NULL); + 0, "floppy", NULL); } static inline void fd_free_irq(void) diff --git a/arch/mips/include/asm/page.h b/arch/mips/include/asm/page.h index e59cd1ac09c2..d41790928c64 100644 --- a/arch/mips/include/asm/page.h +++ b/arch/mips/include/asm/page.h @@ -38,6 +38,14 @@ #define HPAGE_SIZE (_AC(1,UL) << HPAGE_SHIFT) #define HPAGE_MASK (~(HPAGE_SIZE - 1)) #define HUGETLB_PAGE_ORDER (HPAGE_SHIFT - PAGE_SHIFT) +#else /* !CONFIG_HUGETLB_PAGE */ +# ifndef BUILD_BUG +# define BUILD_BUG() do { extern void __build_bug(void); __build_bug(); } while (0) +# endif +#define HPAGE_SHIFT ({BUILD_BUG(); 0; }) +#define HPAGE_SIZE ({BUILD_BUG(); 0; }) +#define HPAGE_MASK ({BUILD_BUG(); 0; }) +#define HUGETLB_PAGE_ORDER ({BUILD_BUG(); 0; }) #endif /* CONFIG_HUGETLB_PAGE */ #ifndef __ASSEMBLY__ diff --git a/arch/mips/include/asm/pgtable-32.h b/arch/mips/include/asm/pgtable-32.h index 469390656c3f..5d56bb230345 100644 --- a/arch/mips/include/asm/pgtable-32.h +++ b/arch/mips/include/asm/pgtable-32.h @@ -19,17 +19,7 @@ #include <asm-generic/pgtable-nopmd.h> /* - * - add_temporary_entry() add a temporary TLB entry. We use TLB entries - * starting at the top and working down. This is for populating the - * TLB before trap_init() puts the TLB miss handler in place. It - * should be used only for entries matching the actual page tables, - * to prevent inconsistencies. - */ -extern int add_temporary_entry(unsigned long entrylo0, unsigned long entrylo1, - unsigned long entryhi, unsigned long pagemask); - - -/* Basically we have the same two-level (which is the logical three level + * Basically we have the same two-level (which is the logical three level * Linux page table layout folded) page tables as the i386. Some day * when we have proper page coloring support we can have a 1% quicker * tlb refill handling mechanism, but for now it is a bit slower but diff --git a/arch/mips/jazz/irq.c b/arch/mips/jazz/irq.c index 99ab86ffa4c0..f21868b28b24 100644 --- a/arch/mips/jazz/irq.c +++ b/arch/mips/jazz/irq.c @@ -134,7 +134,7 @@ static irqreturn_t r4030_timer_interrupt(int irq, void *dev_id) static struct irqaction r4030_timer_irqaction = { .handler = r4030_timer_interrupt, - .flags = IRQF_DISABLED | IRQF_TIMER, + .flags = IRQF_TIMER, .name = "R4030 timer", }; diff --git a/arch/mips/kernel/cevt-bcm1480.c b/arch/mips/kernel/cevt-bcm1480.c index 36c3898b76db..69bbfae183bc 100644 --- a/arch/mips/kernel/cevt-bcm1480.c +++ b/arch/mips/kernel/cevt-bcm1480.c @@ -145,7 +145,7 @@ void __cpuinit sb1480_clockevent_init(void) bcm1480_unmask_irq(cpu, irq); action->handler = sibyte_counter_handler; - action->flags = IRQF_DISABLED | IRQF_PERCPU | IRQF_TIMER; + action->flags = IRQF_PERCPU | IRQF_TIMER; action->name = name; action->dev_id = cd; diff --git a/arch/mips/kernel/cevt-ds1287.c b/arch/mips/kernel/cevt-ds1287.c index 939157e397b9..ed648cb5a69f 100644 --- a/arch/mips/kernel/cevt-ds1287.c +++ b/arch/mips/kernel/cevt-ds1287.c @@ -108,7 +108,7 @@ static irqreturn_t ds1287_interrupt(int irq, void *dev_id) static struct irqaction ds1287_irqaction = { .handler = ds1287_interrupt, - .flags = IRQF_DISABLED | IRQF_PERCPU | IRQF_TIMER, + .flags = IRQF_PERCPU | IRQF_TIMER, .name = "ds1287", }; diff --git a/arch/mips/kernel/cevt-gt641xx.c b/arch/mips/kernel/cevt-gt641xx.c index 339f3639b90e..831b47585b7c 100644 --- a/arch/mips/kernel/cevt-gt641xx.c +++ b/arch/mips/kernel/cevt-gt641xx.c @@ -114,7 +114,7 @@ static irqreturn_t gt641xx_timer0_interrupt(int irq, void *dev_id) static struct irqaction gt641xx_timer0_irqaction = { .handler = gt641xx_timer0_interrupt, - .flags = IRQF_DISABLED | IRQF_PERCPU | IRQF_TIMER, + .flags = IRQF_PERCPU | IRQF_TIMER, .name = "gt641xx_timer0", }; diff --git a/arch/mips/kernel/cevt-r4k.c b/arch/mips/kernel/cevt-r4k.c index e2d8e199be32..51095dd9599d 100644 --- a/arch/mips/kernel/cevt-r4k.c +++ b/arch/mips/kernel/cevt-r4k.c @@ -84,7 +84,7 @@ out: struct irqaction c0_compare_irqaction = { .handler = c0_compare_interrupt, - .flags = IRQF_DISABLED | IRQF_PERCPU | IRQF_TIMER, + .flags = IRQF_PERCPU | IRQF_TIMER, .name = "timer", }; diff --git a/arch/mips/kernel/cevt-sb1250.c b/arch/mips/kernel/cevt-sb1250.c index 590c54f28a81..e73439fd6850 100644 --- a/arch/mips/kernel/cevt-sb1250.c +++ b/arch/mips/kernel/cevt-sb1250.c @@ -144,7 +144,7 @@ void __cpuinit sb1250_clockevent_init(void) sb1250_unmask_irq(cpu, irq); action->handler = sibyte_counter_handler; - action->flags = IRQF_DISABLED | IRQF_PERCPU | IRQF_TIMER; + action->flags = IRQF_PERCPU | IRQF_TIMER; action->name = name; action->dev_id = cd; diff --git a/arch/mips/kernel/cevt-txx9.c b/arch/mips/kernel/cevt-txx9.c index f0ab92a1b057..e5c30b1d0860 100644 --- a/arch/mips/kernel/cevt-txx9.c +++ b/arch/mips/kernel/cevt-txx9.c @@ -146,7 +146,7 @@ static irqreturn_t txx9tmr_interrupt(int irq, void *dev_id) static struct irqaction txx9tmr_irq = { .handler = txx9tmr_interrupt, - .flags = IRQF_DISABLED | IRQF_PERCPU | IRQF_TIMER, + .flags = IRQF_PERCPU | IRQF_TIMER, .name = "txx9tmr", .dev_id = &txx9_clock_event_device, }; diff --git a/arch/mips/kernel/i8253.c b/arch/mips/kernel/i8253.c index 7047bff35ea5..c5bc344fc745 100644 --- a/arch/mips/kernel/i8253.c +++ b/arch/mips/kernel/i8253.c @@ -19,7 +19,7 @@ static irqreturn_t timer_interrupt(int irq, void *dev_id) static struct irqaction irq0 = { .handler = timer_interrupt, - .flags = IRQF_DISABLED | IRQF_NOBALANCING | IRQF_TIMER, + .flags = IRQF_NOBALANCING | IRQF_TIMER, .name = "timer" }; diff --git a/arch/mips/kernel/rtlx.c b/arch/mips/kernel/rtlx.c index 933166f44a6d..a9d801dec6b0 100644 --- a/arch/mips/kernel/rtlx.c +++ b/arch/mips/kernel/rtlx.c @@ -473,7 +473,6 @@ static const struct file_operations rtlx_fops = { static struct irqaction rtlx_irq = { .handler = rtlx_interrupt, - .flags = IRQF_DISABLED, .name = "RTLX", }; diff --git a/arch/mips/kernel/setup.c b/arch/mips/kernel/setup.c index 84af26ab2212..e86c2cf554aa 100644 --- a/arch/mips/kernel/setup.c +++ b/arch/mips/kernel/setup.c @@ -121,6 +121,9 @@ static void __init print_memory_map(void) case BOOT_MEM_RAM: printk(KERN_CONT "(usable)\n"); break; + case BOOT_MEM_INIT_RAM: + printk(KERN_CONT "(usable after init)\n"); + break; case BOOT_MEM_ROM_DATA: printk(KERN_CONT "(ROM data)\n"); break; @@ -361,15 +364,24 @@ static void __init bootmem_init(void) for (i = 0; i < boot_mem_map.nr_map; i++) { unsigned long start, end, size; + start = PFN_UP(boot_mem_map.map[i].addr); + end = PFN_DOWN(boot_mem_map.map[i].addr + + boot_mem_map.map[i].size); + /* * Reserve usable memory. */ - if (boot_mem_map.map[i].type != BOOT_MEM_RAM) + switch (boot_mem_map.map[i].type) { + case BOOT_MEM_RAM: + break; + case BOOT_MEM_INIT_RAM: + memory_present(0, start, end); continue; + default: + /* Not usable memory */ + continue; + } - start = PFN_UP(boot_mem_map.map[i].addr); - end = PFN_DOWN(boot_mem_map.map[i].addr - + boot_mem_map.map[i].size); /* * We are rounding up the start address of usable memory * and at the end of the usable range downwards. @@ -455,11 +467,33 @@ early_param("mem", early_parse_mem); static void __init arch_mem_init(char **cmdline_p) { + phys_t init_mem, init_end, init_size; + extern void plat_mem_setup(void); /* call board setup routine */ plat_mem_setup(); + init_mem = PFN_UP(__pa_symbol(&__init_begin)) << PAGE_SHIFT; + init_end = PFN_DOWN(__pa_symbol(&__init_end)) << PAGE_SHIFT; + init_size = init_end - init_mem; + if (init_size) { + /* Make sure it is in the boot_mem_map */ + int i, found; + found = 0; + for (i = 0; i < boot_mem_map.nr_map; i++) { + if (init_mem >= boot_mem_map.map[i].addr && + init_mem < (boot_mem_map.map[i].addr + + boot_mem_map.map[i].size)) { + found = 1; + break; + } + } + if (!found) + add_memory_region(init_mem, init_size, + BOOT_MEM_INIT_RAM); + } + pr_info("Determined physical RAM map:\n"); print_memory_map(); @@ -523,6 +557,7 @@ static void __init resource_init(void) res = alloc_bootmem(sizeof(struct resource)); switch (boot_mem_map.map[i].type) { case BOOT_MEM_RAM: + case BOOT_MEM_INIT_RAM: case BOOT_MEM_ROM_DATA: res->name = "System RAM"; break; diff --git a/arch/mips/kernel/smtc.c b/arch/mips/kernel/smtc.c index b1eea63cc988..0a42ff3ff6a1 100644 --- a/arch/mips/kernel/smtc.c +++ b/arch/mips/kernel/smtc.c @@ -1130,7 +1130,7 @@ static void ipi_irq_dispatch(void) static struct irqaction irq_ipi = { .handler = ipi_interrupt, - .flags = IRQF_DISABLED | IRQF_PERCPU, + .flags = IRQF_PERCPU, .name = "SMTC_IPI" }; diff --git a/arch/mips/lantiq/irq.c b/arch/mips/lantiq/irq.c index 51fa0a4ba556..d673731c538a 100644 --- a/arch/mips/lantiq/irq.c +++ b/arch/mips/lantiq/irq.c @@ -240,7 +240,6 @@ out: static struct irqaction cascade = { .handler = no_action, - .flags = IRQF_DISABLED, .name = "cascade", }; diff --git a/arch/mips/loongson/common/cs5536/cs5536_mfgpt.c b/arch/mips/loongson/common/cs5536/cs5536_mfgpt.c index 0cb1b9760e34..5d1f48fa1a52 100644 --- a/arch/mips/loongson/common/cs5536/cs5536_mfgpt.c +++ b/arch/mips/loongson/common/cs5536/cs5536_mfgpt.c @@ -111,7 +111,7 @@ static irqreturn_t timer_interrupt(int irq, void *dev_id) static struct irqaction irq5 = { .handler = timer_interrupt, - .flags = IRQF_DISABLED | IRQF_NOBALANCING | IRQF_TIMER, + .flags = IRQF_NOBALANCING | IRQF_TIMER, .name = "timer" }; diff --git a/arch/mips/mm/Makefile b/arch/mips/mm/Makefile index 4d8c1623eee2..3ca2a065cf76 100644 --- a/arch/mips/mm/Makefile +++ b/arch/mips/mm/Makefile @@ -3,8 +3,8 @@ # obj-y += cache.o dma-default.o extable.o fault.o \ - init.o mmap.o tlbex.o tlbex-fault.o uasm.o \ - page.o + gup.o init.o mmap.o page.o tlbex.o \ + tlbex-fault.o uasm.o obj-$(CONFIG_32BIT) += ioremap.o pgtable-32.o obj-$(CONFIG_64BIT) += pgtable-64.o diff --git a/arch/mips/mm/gup.c b/arch/mips/mm/gup.c new file mode 100644 index 000000000000..33aadbcf170b --- /dev/null +++ b/arch/mips/mm/gup.c @@ -0,0 +1,315 @@ +/* + * Lockless get_user_pages_fast for MIPS + * + * Copyright (C) 2008 Nick Piggin + * Copyright (C) 2008 Novell Inc. + * Copyright (C) 2011 Ralf Baechle + */ +#include <linux/sched.h> +#include <linux/mm.h> +#include <linux/vmstat.h> +#include <linux/highmem.h> +#include <linux/swap.h> +#include <linux/hugetlb.h> + +#include <asm/pgtable.h> + +static inline pte_t gup_get_pte(pte_t *ptep) +{ +#if defined(CONFIG_64BIT_PHYS_ADDR) && defined(CONFIG_CPU_MIPS32) + pte_t pte; + +retry: + pte.pte_low = ptep->pte_low; + smp_rmb(); + pte.pte_high = ptep->pte_high; + smp_rmb(); + if (unlikely(pte.pte_low != ptep->pte_low)) + goto retry; + + return pte; +#else + return ACCESS_ONCE(*ptep); +#endif +} + +static int gup_pte_range(pmd_t pmd, unsigned long addr, unsigned long end, + int write, struct page **pages, int *nr) +{ + pte_t *ptep = pte_offset_map(&pmd, addr); + do { + pte_t pte = gup_get_pte(ptep); + struct page *page; + + if (!pte_present(pte) || + pte_special(pte) || (write && !pte_write(pte))) { + pte_unmap(ptep); + return 0; + } + VM_BUG_ON(!pfn_valid(pte_pfn(pte))); + page = pte_page(pte); + get_page(page); + SetPageReferenced(page); + pages[*nr] = page; + (*nr)++; + + } while (ptep++, addr += PAGE_SIZE, addr != end); + + pte_unmap(ptep - 1); + return 1; +} + +static inline void get_head_page_multiple(struct page *page, int nr) +{ + VM_BUG_ON(page != compound_head(page)); + VM_BUG_ON(page_count(page) == 0); + atomic_add(nr, &page->_count); + SetPageReferenced(page); +} + +static int gup_huge_pmd(pmd_t pmd, unsigned long addr, unsigned long end, + int write, struct page **pages, int *nr) +{ + pte_t pte = *(pte_t *)&pmd; + struct page *head, *page; + int refs; + + if (write && !pte_write(pte)) + return 0; + /* hugepages are never "special" */ + VM_BUG_ON(pte_special(pte)); + VM_BUG_ON(!pfn_valid(pte_pfn(pte))); + + refs = 0; + head = pte_page(pte); + page = head + ((addr & ~PMD_MASK) >> PAGE_SHIFT); + do { + VM_BUG_ON(compound_head(page) != head); + pages[*nr] = page; + if (PageTail(page)) + get_huge_page_tail(page); + (*nr)++; + page++; + refs++; + } while (addr += PAGE_SIZE, addr != end); + + get_head_page_multiple(head, refs); + return 1; +} + +static int gup_pmd_range(pud_t pud, unsigned long addr, unsigned long end, + int write, struct page **pages, int *nr) +{ + unsigned long next; + pmd_t *pmdp; + + pmdp = pmd_offset(&pud, addr); + do { + pmd_t pmd = *pmdp; + + next = pmd_addr_end(addr, end); + /* + * The pmd_trans_splitting() check below explains why + * pmdp_splitting_flush has to flush the tlb, to stop + * this gup-fast code from running while we set the + * splitting bit in the pmd. Returning zero will take + * the slow path that will call wait_split_huge_page() + * if the pmd is still in splitting state. gup-fast + * can't because it has irq disabled and + * wait_split_huge_page() would never return as the + * tlb flush IPI wouldn't run. + */ + if (pmd_none(pmd) || pmd_trans_splitting(pmd)) + return 0; + if (unlikely(pmd_huge(pmd))) { + if (!gup_huge_pmd(pmd, addr, next, write, pages,nr)) + return 0; + } else { + if (!gup_pte_range(pmd, addr, next, write, pages,nr)) + return 0; + } + } while (pmdp++, addr = next, addr != end); + + return 1; +} + +static int gup_huge_pud(pud_t pud, unsigned long addr, unsigned long end, + int write, struct page **pages, int *nr) +{ + |