From 3748b2f15b06ea1861df39d5e9693dcd6e9542b1 Mon Sep 17 00:00:00 2001 From: KAMEZAWA Hiroyuki Date: Wed, 28 Mar 2012 14:42:39 -0700 Subject: procfs: fix /proc/statm bda7bad62bc4 ("procfs: speed up /proc/pid/stat, statm") broke /proc/statm - 'text' is printed twice by mistake. Signed-off-by: KAMEZAWA Hiroyuki Reported-by: Ulrich Drepper Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds --- fs/proc/array.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fs/proc/array.c b/fs/proc/array.c index fbb53c249086..f9bd395b3473 100644 --- a/fs/proc/array.c +++ b/fs/proc/array.c @@ -550,7 +550,7 @@ int proc_pid_statm(struct seq_file *m, struct pid_namespace *ns, seq_put_decimal_ull(m, ' ', shared); seq_put_decimal_ull(m, ' ', text); seq_put_decimal_ull(m, ' ', 0); - seq_put_decimal_ull(m, ' ', text); + seq_put_decimal_ull(m, ' ', data); seq_put_decimal_ull(m, ' ', 0); seq_putc(m, '\n'); -- cgit v1.2.3 From 623e3db9f9b7d6e7b2a99180f9cf0825c936ab7a Mon Sep 17 00:00:00 2001 From: Hugh Dickins Date: Wed, 28 Mar 2012 14:42:40 -0700 Subject: mm for fs: add truncate_pagecache_range() Holepunching filesystems ext4 and xfs are using truncate_inode_pages_range but forgetting to unmap pages first (ocfs2 remembers). This is not really a bug, since races already require truncate_inode_page() to handle that case once the page is locked; but it can be very inefficient if the file being punched happens to be mapped into many vmas. Provide a drop-in replacement truncate_pagecache_range() which does the unmapping pass first, handling the awkward mismatch between arguments to truncate_inode_pages_range() and arguments to unmap_mapping_range(). Note that holepunching does not unmap privately COWed pages in the range: POSIX requires that we do so when truncating, but it's hard to justify, difficult to implement without an i_size cutoff, and no filesystem is attempting to implement it. Signed-off-by: Hugh Dickins Cc: "Theodore Ts'o" Cc: Andreas Dilger Cc: Mark Fasheh Cc: Joel Becker Cc: Ben Myers Cc: Alex Elder Cc: Christoph Hellwig Cc: Dave Chinner Cc: Al Viro Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds --- include/linux/mm.h | 2 +- mm/truncate.c | 40 ++++++++++++++++++++++++++++++++++++++++ 2 files changed, 41 insertions(+), 1 deletion(-) diff --git a/include/linux/mm.h b/include/linux/mm.h index cf7982336103..630068184265 100644 --- a/include/linux/mm.h +++ b/include/linux/mm.h @@ -954,7 +954,7 @@ extern void truncate_pagecache(struct inode *inode, loff_t old, loff_t new); extern void truncate_setsize(struct inode *inode, loff_t newsize); extern int vmtruncate(struct inode *inode, loff_t offset); extern int vmtruncate_range(struct inode *inode, loff_t offset, loff_t end); - +void truncate_pagecache_range(struct inode *inode, loff_t offset, loff_t end); int truncate_inode_page(struct address_space *mapping, struct page *page); int generic_error_remove_page(struct address_space *mapping, struct page *page); diff --git a/mm/truncate.c b/mm/truncate.c index 18aded3a89fc..61a183b89df6 100644 --- a/mm/truncate.c +++ b/mm/truncate.c @@ -626,3 +626,43 @@ int vmtruncate_range(struct inode *inode, loff_t lstart, loff_t lend) return 0; } + +/** + * truncate_pagecache_range - unmap and remove pagecache that is hole-punched + * @inode: inode + * @lstart: offset of beginning of hole + * @lend: offset of last byte of hole + * + * This function should typically be called before the filesystem + * releases resources associated with the freed range (eg. deallocates + * blocks). This way, pagecache will always stay logically coherent + * with on-disk format, and the filesystem would not have to deal with + * situations such as writepage being called for a page that has already + * had its underlying blocks deallocated. + */ +void truncate_pagecache_range(struct inode *inode, loff_t lstart, loff_t lend) +{ + struct address_space *mapping = inode->i_mapping; + loff_t unmap_start = round_up(lstart, PAGE_SIZE); + loff_t unmap_end = round_down(1 + lend, PAGE_SIZE) - 1; + /* + * This rounding is currently just for example: unmap_mapping_range + * expands its hole outwards, whereas we want it to contract the hole + * inwards. However, existing callers of truncate_pagecache_range are + * doing their own page rounding first; and truncate_inode_pages_range + * currently BUGs if lend is not pagealigned-1 (it handles partial + * page at start of hole, but not partial page at end of hole). Note + * unmap_mapping_range allows holelen 0 for all, and we allow lend -1. + */ + + /* + * Unlike in truncate_pagecache, unmap_mapping_range is called only + * once (before truncating pagecache), and without "even_cows" flag: + * hole-punching should not remove private COWed pages from the hole. + */ + if ((u64)unmap_end > (u64)unmap_start) + unmap_mapping_range(mapping, unmap_start, + 1 + unmap_end - unmap_start, 0); + truncate_inode_pages_range(mapping, lstart, lend); +} +EXPORT_SYMBOL(truncate_pagecache_range); -- cgit v1.2.3 From 45f83cefe3a5f0476ac3f96382ebfdc3fe4caab2 Mon Sep 17 00:00:00 2001 From: Andrea Arcangeli Date: Wed, 28 Mar 2012 14:42:40 -0700 Subject: mm: thp: fix up pmd_trans_unstable() locations pmd_trans_unstable() should be called before pmd_offset_map() in the locations where the mmap_sem is held for reading. Signed-off-by: Andrea Arcangeli Cc: Mel Gorman Cc: Hugh Dickins Cc: Larry Woodman Cc: Ulrich Obergfell Cc: Rik van Riel Cc: Mark Salter Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds --- fs/proc/task_mmu.c | 5 ++--- mm/memcontrol.c | 4 ++++ 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/fs/proc/task_mmu.c b/fs/proc/task_mmu.c index 9694cc283511..c283832d411d 100644 --- a/fs/proc/task_mmu.c +++ b/fs/proc/task_mmu.c @@ -781,9 +781,6 @@ static int pagemap_pte_range(pmd_t *pmd, unsigned long addr, unsigned long end, int err = 0; pagemap_entry_t pme = make_pme(PM_NOT_PRESENT); - if (pmd_trans_unstable(pmd)) - return 0; - /* find the first VMA at or above 'addr' */ vma = find_vma(walk->mm, addr); spin_lock(&walk->mm->page_table_lock); @@ -802,6 +799,8 @@ static int pagemap_pte_range(pmd_t *pmd, unsigned long addr, unsigned long end, return err; } + if (pmd_trans_unstable(pmd)) + return 0; for (; addr != end; addr += PAGE_SIZE) { /* check to see if we've left 'vma' behind diff --git a/mm/memcontrol.c b/mm/memcontrol.c index b2ee6df0e9bb..7d698df4a067 100644 --- a/mm/memcontrol.c +++ b/mm/memcontrol.c @@ -5306,6 +5306,8 @@ static int mem_cgroup_count_precharge_pte_range(pmd_t *pmd, return 0; } + if (pmd_trans_unstable(pmd)) + return 0; pte = pte_offset_map_lock(vma->vm_mm, pmd, addr, &ptl); for (; addr != end; pte++, addr += PAGE_SIZE) if (get_mctgt_type(vma, addr, *pte, NULL)) @@ -5502,6 +5504,8 @@ static int mem_cgroup_move_charge_pte_range(pmd_t *pmd, return 0; } + if (pmd_trans_unstable(pmd)) + return 0; retry: pte = pte_offset_map_lock(vma->vm_mm, pmd, addr, &ptl); for (; addr != end; addr += PAGE_SIZE) { -- cgit v1.2.3 From 29fd66d289f2981e11c550f8b411a6d3d38be0cf Mon Sep 17 00:00:00 2001 From: David Rientjes Date: Wed, 28 Mar 2012 14:42:41 -0700 Subject: mm, coredump: fail allocations when coredumping instead of oom killing The size of coredump files is limited by RLIMIT_CORE, however, allocating large amounts of memory results in three negative consequences: - the coredumping process may be chosen for oom kill and quickly deplete all memory reserves in oom conditions preventing further progress from being made or tasks from exiting, - the coredumping process may cause other processes to be oom killed without fault of their own as the result of a SIGSEGV, for example, in the coredumping process, or - the coredumping process may result in a livelock while writing to the dump file if it needs memory to allocate while other threads are in the exit path waiting on the coredumper to complete. This is fixed by implying __GFP_NORETRY in the page allocator for coredumping processes when reclaim has failed so the allocations fail and the process continues to exit. Signed-off-by: David Rientjes Cc: Mel Gorman Cc: KAMEZAWA Hiroyuki Cc: Minchan Kim Cc: Oleg Nesterov Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds --- mm/page_alloc.c | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/mm/page_alloc.c b/mm/page_alloc.c index caea788628e4..c313afcc8e5a 100644 --- a/mm/page_alloc.c +++ b/mm/page_alloc.c @@ -2308,6 +2308,10 @@ rebalance: if ((gfp_mask & __GFP_FS) && !(gfp_mask & __GFP_NORETRY)) { if (oom_killer_disabled) goto nopage; + /* Coredumps can quickly deplete all memory reserves */ + if ((current->flags & PF_DUMPCORE) && + !(gfp_mask & __GFP_NOFAIL)) + goto nopage; page = __alloc_pages_may_oom(gfp_mask, order, zonelist, high_zoneidx, nodemask, preferred_zone, -- cgit v1.2.3 From d15cab975459fb6092eeba1be72c13621337784f Mon Sep 17 00:00:00 2001 From: Hugh Dickins Date: Wed, 28 Mar 2012 14:42:42 -0700 Subject: swapon: check validity of swap_flags Most system calls taking flags first check that the flags passed in are valid, and that helps userspace to detect when new flags are supported. But swapon never did so: start checking now, to help if we ever want to support more swap_flags in future. It's difficult to get stray bits set in an int, and swapon is not widely used, so this is most unlikely to break any userspace; but we can just revert if it turns out to do so. Signed-off-by: Hugh Dickins Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds --- include/linux/swap.h | 3 +++ mm/swapfile.c | 3 +++ 2 files changed, 6 insertions(+) diff --git a/include/linux/swap.h b/include/linux/swap.h index b86b5c20617d..8dc0ea7caf02 100644 --- a/include/linux/swap.h +++ b/include/linux/swap.h @@ -21,6 +21,9 @@ struct bio; #define SWAP_FLAG_PRIO_SHIFT 0 #define SWAP_FLAG_DISCARD 0x10000 /* discard swap cluster after use */ +#define SWAP_FLAGS_VALID (SWAP_FLAG_PRIO_MASK | SWAP_FLAG_PREFER | \ + SWAP_FLAG_DISCARD) + static inline int current_is_kswapd(void) { return current->flags & PF_KSWAPD; diff --git a/mm/swapfile.c b/mm/swapfile.c index dae42f380d6e..fafc26d1b1dc 100644 --- a/mm/swapfile.c +++ b/mm/swapfile.c @@ -2022,6 +2022,9 @@ SYSCALL_DEFINE2(swapon, const char __user *, specialfile, int, swap_flags) struct page *page = NULL; struct inode *inode = NULL; + if (swap_flags & ~SWAP_FLAGS_VALID) + return -EINVAL; + if (!capable(CAP_SYS_ADMIN)) return -EPERM; -- cgit v1.2.3 From 3fc498f165304dc913f1d13b5ac9ab4c758ee7ab Mon Sep 17 00:00:00 2001 From: Gilad Ben-Yossef Date: Wed, 28 Mar 2012 14:42:43 -0700 Subject: smp: introduce a generic on_each_cpu_mask() function We have lots of infrastructure in place to partition multi-core systems such that we have a group of CPUs that are dedicated to specific task: cgroups, scheduler and interrupt affinity, and cpuisol= boot parameter. Still, kernel code will at times interrupt all CPUs in the system via IPIs for various needs. These IPIs are useful and cannot be avoided altogether, but in certain cases it is possible to interrupt only specific CPUs that have useful work to do and not the entire system. This patch set, inspired by discussions with Peter Zijlstra and Frederic Weisbecker when testing the nohz task patch set, is a first stab at trying to explore doing this by locating the places where such global IPI calls are being made and turning the global IPI into an IPI for a specific group of CPUs. The purpose of the patch set is to get feedback if this is the right way to go for dealing with this issue and indeed, if the issue is even worth dealing with at all. Based on the feedback from this patch set I plan to offer further patches that address similar issue in other code paths. This patch creates an on_each_cpu_mask() and on_each_cpu_cond() infrastructure API (the former derived from existing arch specific versions in Tile and Arm) and uses them to turn several global IPI invocation to per CPU group invocations. Core kernel: on_each_cpu_mask() calls a function on processors specified by cpumask, which may or may not include the local processor. You must not call this function with disabled interrupts or from a hardware interrupt handler or from a bottom half handler. arch/arm: Note that the generic version is a little different then the Arm one: 1. It has the mask as first parameter 2. It calls the function on the calling CPU with interrupts disabled, but this should be OK since the function is called on the other CPUs with interrupts disabled anyway. arch/tile: The API is the same as the tile private one, but the generic version also calls the function on the with interrupts disabled in UP case This is OK since the function is called on the other CPUs with interrupts disabled. Signed-off-by: Gilad Ben-Yossef Reviewed-by: Christoph Lameter Acked-by: Chris Metcalf Acked-by: Peter Zijlstra Cc: Frederic Weisbecker Cc: Russell King Cc: Pekka Enberg Cc: Matt Mackall Cc: Rik van Riel Cc: Andi Kleen Cc: Sasha Levin Cc: Mel Gorman Cc: Alexander Viro Cc: Avi Kivity Acked-by: Michal Nazarewicz Cc: Kosaki Motohiro Cc: Milton Miller Cc: Russell King Acked-by: Peter Zijlstra Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds --- arch/arm/kernel/smp_tlb.c | 20 +++++--------------- arch/tile/include/asm/smp.h | 7 ------- arch/tile/kernel/smp.c | 19 ------------------- include/linux/smp.h | 22 ++++++++++++++++++++++ kernel/smp.c | 29 +++++++++++++++++++++++++++++ 5 files changed, 56 insertions(+), 41 deletions(-) diff --git a/arch/arm/kernel/smp_tlb.c b/arch/arm/kernel/smp_tlb.c index 7dcb35285be7..02c5d2ce23bf 100644 --- a/arch/arm/kernel/smp_tlb.c +++ b/arch/arm/kernel/smp_tlb.c @@ -13,18 +13,6 @@ #include #include -static void on_each_cpu_mask(void (*func)(void *), void *info, int wait, - const struct cpumask *mask) -{ - preempt_disable(); - - smp_call_function_many(mask, func, info, wait); - if (cpumask_test_cpu(smp_processor_id(), mask)) - func(info); - - preempt_enable(); -} - /**********************************************************************/ /* @@ -87,7 +75,7 @@ void flush_tlb_all(void) void flush_tlb_mm(struct mm_struct *mm) { if (tlb_ops_need_broadcast()) - on_each_cpu_mask(ipi_flush_tlb_mm, mm, 1, mm_cpumask(mm)); + on_each_cpu_mask(mm_cpumask(mm), ipi_flush_tlb_mm, mm, 1); else local_flush_tlb_mm(mm); } @@ -98,7 +86,8 @@ void flush_tlb_page(struct vm_area_struct *vma, unsigned long uaddr) struct tlb_args ta; ta.ta_vma = vma; ta.ta_start = uaddr; - on_each_cpu_mask(ipi_flush_tlb_page, &ta, 1, mm_cpumask(vma->vm_mm)); + on_each_cpu_mask(mm_cpumask(vma->vm_mm), ipi_flush_tlb_page, + &ta, 1); } else local_flush_tlb_page(vma, uaddr); } @@ -121,7 +110,8 @@ void flush_tlb_range(struct vm_area_struct *vma, ta.ta_vma = vma; ta.ta_start = start; ta.ta_end = end; - on_each_cpu_mask(ipi_flush_tlb_range, &ta, 1, mm_cpumask(vma->vm_mm)); + on_each_cpu_mask(mm_cpumask(vma->vm_mm), ipi_flush_tlb_range, + &ta, 1); } else local_flush_tlb_range(vma, start, end); } diff --git a/arch/tile/include/asm/smp.h b/arch/tile/include/asm/smp.h index 532124ae4b12..1aa759aeb5b3 100644 --- a/arch/tile/include/asm/smp.h +++ b/arch/tile/include/asm/smp.h @@ -43,10 +43,6 @@ void evaluate_message(int tag); /* Boot a secondary cpu */ void online_secondary(void); -/* Call a function on a specified set of CPUs (may include this one). */ -extern void on_each_cpu_mask(const struct cpumask *mask, - void (*func)(void *), void *info, bool wait); - /* Topology of the supervisor tile grid, and coordinates of boot processor */ extern HV_Topology smp_topology; @@ -91,9 +87,6 @@ void print_disabled_cpus(void); #else /* !CONFIG_SMP */ -#define on_each_cpu_mask(mask, func, info, wait) \ - do { if (cpumask_test_cpu(0, (mask))) func(info); } while (0) - #define smp_master_cpu 0 #define smp_height 1 #define smp_width 1 diff --git a/arch/tile/kernel/smp.c b/arch/tile/kernel/smp.c index c52224d5ed45..a44e103c5a63 100644 --- a/arch/tile/kernel/smp.c +++ b/arch/tile/kernel/smp.c @@ -87,25 +87,6 @@ void send_IPI_allbutself(int tag) send_IPI_many(&mask, tag); } - -/* - * Provide smp_call_function_mask, but also run function locally - * if specified in the mask. - */ -void on_each_cpu_mask(const struct cpumask *mask, void (*func)(void *), - void *info, bool wait) -{ - int cpu = get_cpu(); - smp_call_function_many(mask, func, info, wait); - if (cpumask_test_cpu(cpu, mask)) { - local_irq_disable(); - func(info); - local_irq_enable(); - } - put_cpu(); -} - - /* * Functions related to starting/stopping cpus. */ diff --git a/include/linux/smp.h b/include/linux/smp.h index 8cc38d3bab0c..d0adb7898d54 100644 --- a/include/linux/smp.h +++ b/include/linux/smp.h @@ -101,6 +101,13 @@ static inline void call_function_init(void) { } */ int on_each_cpu(smp_call_func_t func, void *info, int wait); +/* + * Call a function on processors specified by mask, which might include + * the local one. + */ +void on_each_cpu_mask(const struct cpumask *mask, smp_call_func_t func, + void *info, bool wait); + /* * Mark the boot cpu "online" so that it can call console drivers in * printk() and can access its per-cpu storage. @@ -132,6 +139,21 @@ static inline int up_smp_call_function(smp_call_func_t func, void *info) local_irq_enable(); \ 0; \ }) +/* + * Note we still need to test the mask even for UP + * because we actually can get an empty mask from + * code that on SMP might call us without the local + * CPU in the mask. + */ +#define on_each_cpu_mask(mask, func, info, wait) \ + do { \ + if (cpumask_test_cpu(0, (mask))) { \ + local_irq_disable(); \ + (func)(info); \ + local_irq_enable(); \ + } \ + } while (0) + static inline void smp_send_reschedule(int cpu) { } #define num_booting_cpus() 1 #define smp_prepare_boot_cpu() do {} while (0) diff --git a/kernel/smp.c b/kernel/smp.c index db197d60489b..a081e6ce0e0a 100644 --- a/kernel/smp.c +++ b/kernel/smp.c @@ -701,3 +701,32 @@ int on_each_cpu(void (*func) (void *info), void *info, int wait) return ret; } EXPORT_SYMBOL(on_each_cpu); + +/** + * on_each_cpu_mask(): Run a function on processors specified by + * cpumask, which may include the local processor. + * @mask: The set of cpus to run on (only runs on online subset). + * @func: The function to run. This must be fast and non-blocking. + * @info: An arbitrary pointer to pass to the function. + * @wait: If true, wait (atomically) until function has completed + * on other CPUs. + * + * If @wait is true, then returns once @func has returned. + * + * You must not call this function with disabled interrupts or + * from a hardware interrupt handler or from a bottom half handler. + */ +void on_each_cpu_mask(const struct cpumask *mask, smp_call_func_t func, + void *info, bool wait) +{ + int cpu = get_cpu(); + + smp_call_function_many(mask, func, info, wait); + if (cpumask_test_cpu(cpu, mask)) { + local_irq_disable(); + func(info); + local_irq_enable(); + } + put_cpu(); +} +EXPORT_SYMBOL(on_each_cpu_mask); -- cgit v1.2.3 From b3a7e98e024ffa9f7e4554dd720c508015c4a831 Mon Sep 17 00:00:00 2001 From: Gilad Ben-Yossef Date: Wed, 28 Mar 2012 14:42:43 -0700 Subject: smp: add func to IPI cpus based on parameter func Add the on_each_cpu_cond() function that wraps on_each_cpu_mask() and calculates the cpumask of cpus to IPI by calling a function supplied as a parameter in order to determine whether to IPI each specific cpu. The function works around allocation failure of cpumask variable in CONFIG_CPUMASK_OFFSTACK=y by itereating over cpus sending an IPI a time via smp_call_function_single(). The function is useful since it allows to seperate the specific code that decided in each case whether to IPI a specific cpu for a specific request from the common boilerplate code of handling creating the mask, handling failures etc. [akpm@linux-foundation.org: s/gfpflags/gfp_flags/] [akpm@linux-foundation.org: avoid double-evaluation of `info' (per Michal), parenthesise evaluation of `cond_func'] [akpm@linux-foundation.org: s/CPU/CPUs, use all 80 cols in comment] Signed-off-by: Gilad Ben-Yossef Cc: Chris Metcalf Cc: Christoph Lameter Acked-by: Peter Zijlstra Cc: Frederic Weisbecker Cc: Russell King Cc: Pekka Enberg Cc: Matt Mackall Cc: Sasha Levin Cc: Rik van Riel Cc: Andi Kleen Cc: Alexander Viro Cc: Avi Kivity Acked-by: Michal Nazarewicz Cc: Kosaki Motohiro Cc: Milton Miller Reviewed-by: "Srivatsa S. Bhat" Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds --- include/linux/smp.h | 24 +++++++++++++++++++++ kernel/smp.c | 61 +++++++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 85 insertions(+) diff --git a/include/linux/smp.h b/include/linux/smp.h index d0adb7898d54..10530d92c04b 100644 --- a/include/linux/smp.h +++ b/include/linux/smp.h @@ -108,6 +108,15 @@ int on_each_cpu(smp_call_func_t func, void *info, int wait); void on_each_cpu_mask(const struct cpumask *mask, smp_call_func_t func, void *info, bool wait); +/* + * Call a function on each processor for which the supplied function + * cond_func returns a positive value. This may include the local + * processor. + */ +void on_each_cpu_cond(bool (*cond_func)(int cpu, void *info), + smp_call_func_t func, void *info, bool wait, + gfp_t gfp_flags); + /* * Mark the boot cpu "online" so that it can call console drivers in * printk() and can access its per-cpu storage. @@ -153,6 +162,21 @@ static inline int up_smp_call_function(smp_call_func_t func, void *info) local_irq_enable(); \ } \ } while (0) +/* + * Preemption is disabled here to make sure the cond_func is called under the + * same condtions in UP and SMP. + */ +#define on_each_cpu_cond(cond_func, func, info, wait, gfp_flags)\ + do { \ + void *__info = (info); \ + preempt_disable(); \ + if ((cond_func)(0, __info)) { \ + local_irq_disable(); \ + (func)(__info); \ + local_irq_enable(); \ + } \ + preempt_enable(); \ + } while (0) static inline void smp_send_reschedule(int cpu) { } #define num_booting_cpus() 1 diff --git a/kernel/smp.c b/kernel/smp.c index a081e6ce0e0a..2f8b10ecf759 100644 --- a/kernel/smp.c +++ b/kernel/smp.c @@ -730,3 +730,64 @@ void on_each_cpu_mask(const struct cpumask *mask, smp_call_func_t func, put_cpu(); } EXPORT_SYMBOL(on_each_cpu_mask); + +/* + * on_each_cpu_cond(): Call a function on each processor for which + * the supplied function cond_func returns true, optionally waiting + * for all the required CPUs to finish. This may include the local + * processor. + * @cond_func: A callback function that is passed a cpu id and + * the the info parameter. The function is called + * with preemption disabled. The function should + * return a blooean value indicating whether to IPI + * the specified CPU. + * @func: The function to run on all applicable CPUs. + * This must be fast and non-blocking. + * @info: An arbitrary pointer to pass to both functions. + * @wait: If true, wait (atomically) until function has + * completed on other CPUs. + * @gfp_flags: GFP flags to use when allocating the cpumask + * used internally by the function. + * + * The function might sleep if the GFP flags indicates a non + * atomic allocation is allowed. + * + * Preemption is disabled to protect against CPUs going offline but not online. + * CPUs going online during the call will not be seen or sent an IPI. + * + * You must not call this function with disabled interrupts or + * from a hardware interrupt handler or from a bottom half handler. + */ +void on_each_cpu_cond(bool (*cond_func)(int cpu, void *info), + smp_call_func_t func, void *info, bool wait, + gfp_t gfp_flags) +{ + cpumask_var_t cpus; + int cpu, ret; + + might_sleep_if(gfp_flags & __GFP_WAIT); + + if (likely(zalloc_cpumask_var(&cpus, (gfp_flags|__GFP_NOWARN)))) { + preempt_disable(); + for_each_online_cpu(cpu) + if (cond_func(cpu, info)) + cpumask_set_cpu(cpu, cpus); + on_each_cpu_mask(cpus, func, info, wait); + preempt_enable(); + free_cpumask_var(cpus); + } else { + /* + * No free cpumask, bother. No matter, we'll + * just have to IPI them one by one. + */ + preempt_disable(); + for_each_online_cpu(cpu) + if (cond_func(cpu, info)) { + ret = smp_call_function_single(cpu, func, + info, wait); + WARN_ON_ONCE(!ret); + } + preempt_enable(); + } +} +EXPORT_SYMBOL(on_each_cpu_cond); -- cgit v1.2.3 From a8364d5555b2030d093cde0f07951628e55454e1 Mon Sep 17 00:00:00 2001 From: Gilad Ben-Yossef Date: Wed, 28 Mar 2012 14:42:44 -0700 Subject: slub: only IPI CPUs that have per cpu obj to flush flush_all() is called for each kmem_cache_destroy(). So every cache being destroyed dynamically ends up sending an IPI to each CPU in the system, regardless if the cache has ever been used there. For example, if you close the Infinband ipath driver char device file, the close file ops calls kmem_cache_destroy(). So running some infiniband config tool on one a single CPU dedicated to system tasks might interrupt the rest of the 127 CPUs dedicated to some CPU intensive or latency sensitive task. I suspect there is a good chance that every line in the output of "git grep kmem_cache_destroy linux/ | grep '\->'" has a similar scenario. This patch attempts to rectify this issue by sending an IPI to flush the per cpu objects back to the free lists only to CPUs that seem to have such objects. The check which CPU to IPI is racy but we don't care since asking a CPU without per cpu objects to flush does no damage and as far as I can tell the flush_all by itself is racy against allocs on remote CPUs anyway, so if you required the flush_all to be determinstic, you had to arrange for locking regardless. Without this patch the following artificial test case: $ cd /sys/kernel/slab $ for DIR in *; do cat $DIR/alloc_calls > /dev/null; done produces 166 IPIs on an cpuset isolated CPU. With it it produces none. The code path of memory allocation failure for CPUMASK_OFFSTACK=y config was tested using fault injection framework. Signed-off-by: Gilad Ben-Yossef Acked-by: Christoph Lameter Cc: Chris Metcalf Acked-by: Peter Zijlstra Cc: Frederic Weisbecker Cc: Russell King Cc: Pekka Enberg Cc: Matt Mackall Cc: Sasha Levin Cc: Rik van Riel Cc: Andi Kleen Cc: Mel Gorman Cc: Alexander Viro Cc: Avi Kivity Cc: Michal Nazarewicz Cc: Kosaki Motohiro Cc: Milton Miller Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds --- mm/slub.c | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/mm/slub.c b/mm/slub.c index f4a6229848fd..dcbb1926cb7f 100644 --- a/mm/slub.c +++ b/mm/slub.c @@ -2028,9 +2028,17 @@ static void flush_cpu_slab(void *d) __flush_cpu_slab(s, smp_processor_id()); } +static bool has_cpu_slab(int cpu, void *info) +{ + struct kmem_cache *s = info; + struct kmem_cache_cpu *c = per_cpu_ptr(s->cpu_slab, cpu); + + return !!(c->page); +} + static void flush_all(struct kmem_cache *s) { - on_each_cpu(flush_cpu_slab, s, 1); + on_each_cpu_cond(has_cpu_slab, flush_cpu_slab, s, 1, GFP_ATOMIC); } /* -- cgit v1.2.3 From 42be35d0390b966253136a285f507f5ad00fd9e8 Mon Sep 17 00:00:00 2001 From: Gilad Ben-Yossef Date: Wed, 28 Mar 2012 14:42:45 -0700 Subject: fs: only send IPI to invalidate LRU BH when needed In several code paths, such as when unmounting a file system (but not only) we send an IPI to ask each cpu to invalidate its local LRU BHs. For multi-cores systems that have many cpus that may not have any LRU BH because they are idle or because they have not performed any file system accesses since last invalidation (e.g. CPU crunching on high perfomance computing nodes that write results to shared memory or only using filesystems that do not use the bh layer.) This can lead to loss of performance each time someone switches the KVM (the virtual keyboard and screen type, not the hypervisor) if it has a USB storage stuck in. This patch attempts to only send an IPI to cpus that have LRU BH. Signed-off-by: Gilad Ben-Yossef Acked-by: Peter Zijlstra Cc: Alexander Viro Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds --- fs/buffer.c | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/fs/buffer.c b/fs/buffer.c index 70e2017edd70..36d66653b931 100644 --- a/fs/buffer.c +++ b/fs/buffer.c @@ -1384,10 +1384,23 @@ static void invalidate_bh_lru(void *arg) } put_cpu_var(bh_lrus); } + +static bool has_bh_in_lru(int cpu, void *dummy) +{ + struct bh_lru *b = per_cpu_ptr(&bh_lrus, cpu); + int i; + for (i = 0; i < BH_LRU_SIZE; i++) { + if (b->bhs[i]) + return 1; + } + + return 0; +} + void invalidate_bh_lrus(void) { - on_each_cpu(invalidate_bh_lru, NULL, 1); + on_each_cpu_cond(has_bh_in_lru, invalidate_bh_lru, NULL, 1, GFP_KERNEL); } EXPORT_SYMBOL_GPL(invalidate_bh_lrus); -- cgit v1.2.3 From 74046494ea68676d29ef6501a4bd950f08112a2c Mon Sep 17 00:00:00 2001 From: Gilad Ben-Yossef Date: Wed, 28 Mar 2012 14:42:45 -0700 Subject: mm: only IPI CPUs to drain local pages if they exist Calculate a cpumask of CPUs with per-cpu pages in any zone and only send an IPI requesting CPUs to drain these pages to the buddy allocator if they actually have pages when asked to flush. This patch saves 85%+ of IPIs asking to drain per-cpu pages in case of severe memory pressure that leads to OOM since in these cases multiple, possibly concurrent, allocation requests end up in the direct reclaim code path so when the per-cpu pages end up reclaimed on first allocation failure for most of the proceeding allocation attempts until the memory pressure is off (possibly via the OOM killer) there are no per-cpu pages on most CPUs (and there can easily be hundreds of them). This also has the side effect of shortening the average latency of direct reclaim by 1 or more order of magnitude since waiting for all the CPUs to ACK the IPI takes a long time. Tested by running "hackbench 400" on a 8 CPU x86 VM and observing the difference between the number of direct reclaim attempts that end up in drain_all_pages() and those were more then 1/2 of the online CPU had any per-cpu page in them, using the vmstat counters introduced in the next patch in the series and using proc/interrupts. In the test sceanrio, this was seen to save around 3600 global IPIs after trigerring an OOM on a concurrent workload: $ cat /proc/vmstat | tail -n 2 pcp_global_drain 0 pcp_global_ipi_saved 0 $ cat /proc/interrupts | grep CAL CAL: 1 2 1 2 2 2 2 2 Function call interrupts $ hackbench 400 [OOM messages snipped] $ cat /proc/vmstat | tail -n 2 pcp_global_drain 3647 pcp_global_ipi_saved 3642 $ cat /proc/interrupts | grep CAL CAL: 6 13 6 3 3 3 1 2 7 Function call interrupts Please note that if the global drain is removed from the direct reclaim path as a patch from Mel Gorman currently suggests this should be replaced with an on_each_cpu_cond invocation. Signed-off-by: Gilad Ben-Yossef Acked-by: Mel Gorman Cc: KOSAKI Motohiro Acked-by: Christoph Lameter Acked-by: Peter Zijlstra Cc: Pekka Enberg Cc: Rik van Riel Cc: Andi Kleen Acked-by: Michal Nazarewicz Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds --- mm/page_alloc.c | 40 ++++++++++++++++++++++++++++++++++++++-- 1 file changed, 38 insertions(+), 2 deletions(-) diff --git a/mm/page_alloc.c b/mm/page_alloc.c index c313afcc8e5a..a712fb9e04ce 100644 --- a/mm/page_alloc.c +++ b/mm/page_alloc.c @@ -1161,11 +1161,47 @@ void drain_local_pages(void *arg) } /* - * Spill all the per-cpu pages from all CPUs back into the buddy allocator + * Spill all the per-cpu pages from all CPUs back into the buddy allocator. + * + * Note that this code is protected against sending an IPI to an offline + * CPU but does not guarantee sending an IPI to newly hotplugged CPUs: + * on_each_cpu_mask() blocks hotplug and won't talk to offlined CPUs but + * nothing keeps CPUs from showing up after we populated the cpumask and + * before the call to on_each_cpu_mask(). */ void drain_all_pages(void) { - on_each_cpu(drain_local_pages, NULL, 1); + int cpu; + struct per_cpu_pageset *pcp; + struct zone *zone; + + /* + * Allocate in the BSS so we wont require allocation in + * direct reclaim path for CONFIG_CPUMASK_OFFSTACK=y + */ + static cpumask_t cpus_with_pcps; + + /* + * We don't care about racing with CPU hotplug event + * as offline notification will cause the notified + * cpu to drain that CPU pcps and on_each_cpu_mask + * disables preemption as part of its processing + */ + for_each_online_cpu(cpu) { + bool has_pcps = false; + for_each_populated_zone(zone) { + pcp = per_cpu_ptr(zone->pageset, cpu); + if (pcp->pcp.count) { + has_pcps = true; + break; + } + } + if (has_pcps) + cpumask_set_cpu(cpu, &cpus_with_pcps); + else + cpumask_clear_cpu(cpu, &cpus_with_pcps); + } + on_each_cpu_mask(&cpus_with_pcps, drain_local_pages, NULL, 1); } #ifdef CONFIG_HIBERNATION -- cgit v1.2.3 From 38b93780a5381961ad92d24ab9a12a964189a3a4 Mon Sep 17 00:00:00 2001 From: "Srivatsa S. Bhat" Date: Wed, 28 Mar 2012 14:42:46 -0700 Subject: lib/cpumask.c: remove __any_online_cpu() __any_online_cpu() is not optimal and also unnecessary. So, replace its use by faster cpumask_* operations. Signed-off-by: Srivatsa S. Bhat Cc: Eric Dumazet Cc: Venkatesh Pallipadi Cc: Rusty Russell Cc: KOSAKI Motohiro Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds --- include/linux/cpumask.h | 3 +-- lib/cpumask.c | 12 ------------ 2 files changed, 1 insertion(+), 14 deletions(-) diff --git a/include/linux/cpumask.h b/include/linux/cpumask.h index 7b9b75a529be..1ffdb9856bb9 100644 --- a/include/linux/cpumask.h +++ b/include/linux/cpumask.h @@ -810,11 +810,10 @@ static inline const struct cpumask *get_cpu_mask(unsigned int cpu) #else /* NR_CPUS > 1 */ int __first_cpu(const cpumask_t *srcp); int __next_cpu(int n, const cpumask_t *srcp); -int __any_online_cpu(const cpumask_t *mask); #define first_cpu(src) __first_cpu(&(src)) #define next_cpu(n, src) __next_cpu((n), &(src)) -#define any_online_cpu(mask) __any_online_cpu(&(mask)) +#define any_online_cpu(mask) cpumask_any_and(&mask, cpu_online_mask) #define for_each_cpu_mask(cpu, mask) \ for ((cpu) = -1; \ (cpu) = next_cpu((cpu), (mask)), \ diff --git a/lib/cpumask.c b/lib/cpumask.c index 0b660118ed91..402a54ac35cb 100644 --- a/lib/cpumask.c +++ b/lib/cpumask.c @@ -26,18 +26,6 @@ int __next_cpu_nr(int n, const cpumask_t *srcp) EXPORT_SYMBOL(__next_cpu_nr); #endif -int __any_online_cpu(const cpumask_t *mask) -{ - int cpu; - - for_each_cpu(cpu, mask) { - if (cpu_online(cpu)) - break; - } - return cpu; -} -EXPORT_SYMBOL(__any_online_cpu); - /** * cpumask_next_and - get the next cpu in *src1p & *src2p * @n: the cpu prior to the place to search (ie. return will be > @n) -- cgit v1.2.3 From 7d7f98488b203cbf78538698cf5d937f670d96d3 Mon Sep 17 00:00:00 2001 From: "Srivatsa S. Bhat" Date: Wed, 28 Mar 2012 14:42:46 -0700 Subject: arch/ia64: remove references to cpu_*_map This was marked as obsolete for quite a while now.. Now it is time to remove it altogether. And while doing this, get rid of first_cpu() as well. Also, remove the redundant setting of cpu_online_mask in smp_prepare_cpus() because the generic code would have already set cpu 0 in cpu_online_mask. Reported-by: Tony Luck Signed-off-by: Srivatsa S. Bhat Cc: Rusty Russell Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds --- arch/ia64/kernel/acpi.c | 6 +++--- arch/ia64/kernel/irq_ia64.c | 8 ++++---- arch/ia64/kernel/mca.c | 6 ++++-- arch/ia64/kernel/msi_ia64.c | 4 ++-- arch/ia64/kernel/setup.c | 2 +- arch/ia64/kernel/smp.c | 2 +- arch/ia64/kernel/smpboot.c | 19 +++++++------------ arch/ia64/kernel/topology.c | 3 ++- 8 files changed, 24 insertions(+), 26 deletions(-) diff --git a/arch/ia64/kernel/acpi.c b/arch/ia64/kernel/acpi.c index 2d801bfe16ac..19bb1eefffb4 100644 --- a/arch/ia64/kernel/acpi.c +++ b/arch/ia64/kernel/acpi.c @@ -844,7 +844,7 @@ early_param("additional_cpus", setup_additional_cpus); * are onlined, or offlined. The reason is per-cpu data-structures * are allocated by some modules at init time, and dont expect to * do this dynamically on cpu arrival/departure. - * cpu_present_map on the other hand can change dynamically. + * cpu_present_mask on the other hand can change dynamically. * In case when cpu_hotplug is not compiled, then we resort to current * behaviour, which is cpu_possible == cpu_present. * - Ashok Raj @@ -922,7 +922,7 @@ static int __cpuinit _acpi_map_lsapic(acpi_handle handle, int *pcpu) acpi_map_cpu2node(handle, cpu, physid); - cpu_set(cpu, cpu_present_map); + set_cpu_present(cpu, true); ia64_cpu_to_sapicid[cpu] = physid; acpi_processor_set_pdc(handle); @@ -941,7 +941,7 @@ EXPORT_SYMBOL(acpi_map_lsapic); int acpi_unmap_lsapic(int cpu) { ia64_cpu_to_sapicid[cpu] = -1; - cpu_clear(cpu, cpu_present_map); + set_cpu_present(cpu, false); #ifdef CONFIG_ACPI_NUMA /* NUMA specific cleanup's */ diff --git a/arch/ia64/kernel/irq_ia64.c b/arch/ia64/kernel/irq_ia64.c index 782c3a357f24..51da77226b29 100644 --- a/arch/ia64/kernel/irq_ia64.c +++ b/arch/ia64/kernel/irq_ia64.c @@ -118,7 +118,7 @@ static inline int find_unassigned_vector(cpumask_t domain) cpumask_t mask; int pos, vector; - cpus_and(mask, domain, cpu_online_map); + cpumask_and(&mask, &domain, cpu_online_mask); if (cpus_empty(mask)) return -EINVAL; @@ -141,7 +141,7 @@ static int __bind_irq_vector(int irq, int vector, cpumask_t domain) BUG_ON((unsigned)irq >= NR_IRQS); BUG_ON((unsigned)vector >= IA64_NUM_VECTORS); - cpus_and(mask, domain, cpu_online_map); + cpumask_and(&mask, &domain, cpu_online_mask); if (cpus_empty(mask)) return -EINVAL; if ((cfg->vector == vector) && cpus_equal(cfg->domain, domain)) @@ -179,7 +179,7 @@ static void __clear_irq_vector(int irq) BUG_ON(cfg->vector == IRQ_VECTOR_UNASSIGNED); vector = cfg->vector; domain = cfg->domain; - cpus_and(mask, cfg->domain, cpu_online_map); + cpumask_and(&mask, &cfg->domain, cpu_online_mask); for_each_cpu_mask(cpu, mask) per_cpu(vector_irq, cpu)[vector] = -1; cfg->vector = IRQ_VECTOR_UNASSIGNED; @@ -322,7 +322,7 @@ void irq_complete_move(unsigned irq) if (unlikely(cpu_isset(smp_processor_id(), cfg->old_domain))) return; - cpus_and(cleanup_mask, cfg->old_domain, cpu_online_map); + cpumask_and(&cleanup_mask, &cfg->old_domain, cpu_online_mask); cfg->move_cleanup_count = cpus_weight(cleanup_mask); for_each_cpu_mask(i, cleanup_mask) platform_send_ipi(i, IA64_IRQ_MOVE_VECTOR, IA64_IPI_DM_INT, 0); diff --git a/arch/ia64/kernel/mca.c b/arch/ia64/kernel/mca.c index 8192009cb924..26dbbd3c3053 100644 --- a/arch/ia64/kernel/mca.c +++ b/arch/ia64/kernel/mca.c @@ -1515,7 +1515,8 @@ static void ia64_mca_cmc_poll (unsigned long dummy) { /* Trigger a CMC interrupt cascade */ - platform_send_ipi(first_cpu(cpu_online_map), IA64_CMCP_VECTOR, IA64_IPI_DM_INT, 0); + platform_send_ipi(cpumask_first(cpu_online_mask), IA64_CMCP_VECTOR, + IA64_IPI_DM_INT, 0); } /* @@ -1591,7 +1592,8 @@ static void ia64_mca_cpe_poll (unsigned long dummy) { /* Trigger a CPE interrupt cascade */ - platform_send_ipi(first_cpu(cpu_online_map), IA64_CPEP_VECTOR, IA64_IPI_DM_INT, 0); + platform_send_ipi(cpumask_first(cpu_online_mask), IA64_CPEP_VECTOR, + IA64_IPI_DM_INT, 0); } #endif /* CONFIG_ACPI */ diff --git a/arch/ia64/kernel/msi_ia64.c b/arch/ia64/kernel/msi_ia64.c index 94e0db72d4a6..fb2f1e622877 100644 --- a/arch/ia64/kernel/msi_ia64.c +++ b/arch/ia64/kernel/msi_ia64.c @@ -57,7 +57,7 @@ int ia64_setup_msi_irq(struct pci_dev *pdev, struct msi_desc *desc) return irq; irq_set_msi_desc(irq, desc); - cpus_and(mask, irq_to_domain(irq), cpu_online_map); + cpumask_and(&mask, &(irq_to_domain(irq)), cpu_online_mask); dest_phys_id = cpu_physical_id(first_cpu(mask)); vector = irq_to_vector(irq); @@ -179,7 +179,7 @@ msi_compose_msg(struct pci_dev *pdev, unsigned int irq, struct msi_msg *msg) unsigned dest; cpumask_t mask; - cpus_and(mask, irq_to_domain(irq), cpu_online_map); + cpumask_and(&mask, &(irq_to_domain(irq)), cpu_online_mask); dest = cpu_physical_id(first_cpu(mask)); msg->address_hi = 0; diff --git a/arch/ia64/kernel/setup.c b/arch/ia64/kernel/setup.c index cd57d7312de0..4d1a5508a0ed 100644 --- a/arch/ia64/kernel/setup.c +++ b/arch/ia64/kernel/setup.c @@ -486,7 +486,7 @@ mark_bsp_online (void) { #ifdef CONFIG_SMP /* If we register an early console, allow CPU 0 to printk */ - cpu_set(smp_processor_id(), cpu_online_map); + set_cpu_online(smp_processor_id(), true); #endif } diff --git a/arch/ia64/kernel/smp.c b/arch/ia64/kernel/smp.c index 0bd537b4ea6b..855197981962 100644 --- a/arch/ia64/kernel/smp.c +++ b/arch/ia64/kernel/smp.c @@ -77,7 +77,7 @@ stop_this_cpu(void) /* * Remove this CPU: */ - cpu_clear(smp_processor_id(), cpu_online_map); + set_cpu_online(smp_processor_id(), false); max_xtp(); local_irq_disable(); cpu_halt(); diff --git a/arch/ia64/kernel/smpboot.c b/arch/ia64/kernel/smpboot.c index 559097986672..90916beddf07 100644 --- a/arch/ia64/kernel/smpboot.c +++ b/arch/ia64/kernel/smpboot.c @@ -401,7 +401,7 @@ smp_callin (void) /* Setup the per cpu irq handling data structures */ __setup_vector_irq(cpuid); notify_cpu_starting(cpuid); - cpu_set(cpuid, cpu_online_map); + set_cpu_online(cpuid, true); per_cpu(cpu_state, cpuid) = CPU_ONLINE; spin_unlock(&vector_lock); ipi_call_unlock_irq(); @@ -548,7 +548,7 @@ do_rest: if (!cpu_isset(cpu, cpu_callin_map)) { printk(KERN_ERR "Processor 0x%x/0x%x is stuck.\n", cpu, sapicid); ia64_cpu_to_sapicid[cpu] = -1; - cpu_clear(cpu, cpu_online_map); /* was set in smp_callin() */ + set_cpu_online(cpu, false); /* was set in smp_callin() */ return -EINVAL; } return 0; @@ -578,8 +578,7 @@ smp_build_cpu_map (void) } ia64_cpu_to_sapicid[0] = boot_cpu_id; - cpus_clear(cpu_present_map); - set_cpu_present(0, true); + init_cpu_present(cpumask_of(0)); set_cpu_possible(0, true); for (cpu = 1, i = 0; i < smp_boot_data.cpu_count; i++) { sapicid = smp_boot_data.cpu_phys_id[i]; @@ -606,10 +605,6 @@ smp_prepare_cpus (unsigned int max_cpus) smp_setup_percpu_timer(); - /* - * We have the boot CPU online for sure. - */ - cpu_set(0, cpu_online_map); cpu_set(0, cpu_callin_map); local_cpu_data->loops_per_jiffy = loops_per_jiffy; @@ -633,7 +628,7 @@ smp_prepare_cpus (unsigned int max_cpus) void __devinit smp_prepare_boot_cpu(void) { - cpu_set(smp_processor_id(), cpu_online_map); + set_cpu_online(smp_processor_id(), true); cpu_set(smp_processor_id(), cpu_callin_map); set_numa_node(cpu_to_node_map[smp_processor_id()]); per_cpu(cpu_state, smp_processor_id()) = CPU_ONLINE; @@ -690,7 +685,7 @@ int migrate_platform_irqs(unsigned int cpu) /* * Now re-target the CPEI to a different processor */ - new_cpei_cpu = any_online_cpu(cpu_online_map); + new_cpei_cpu = cpumask_any(cpu_online_mask); mask = cpumask_of(new_cpei_cpu); set_cpei_target_cpu(new_cpei_cpu); data = irq_get_irq_data(ia64_cpe_irq); @@ -732,10 +727,10 @@ int __cpu_disable(void) return -EBUSY; } - cpu_clear(cpu, cpu_online_map); + set_cpu_online(cpu, false); if (migrate_platform_irqs(cpu)) { - cpu_set(cpu, cpu_online_map); + set_cpu_online(cpu, true); return -EBUSY; } diff --git a/arch/ia64/kernel/topology.c b/arch/ia64/kernel/topology.c index 9deb21dbf629..c64460b9c704 100644 --- a/arch/ia64/kernel/topology.c +++ b/arch/ia64/kernel/topology.c @@ -220,7 +220,8 @@ static ssize_t show_shared_cpu_map(struct cache_info *this_leaf, char *buf) ssize_t len; cpumask_t shared_cpu_map; - cpus_and(shared_cpu_map, this_leaf->shared_cpu_map, cpu_online_map); + cpumask_and(&shared_cpu_map, + &this_leaf->shared_cpu_map, cpu_online_mask); len = cpumask_scnprintf(buf, NR_CPUS+1, &shared_cpu_map); len += sprintf(buf+len, "\n"); return len; -- cgit v1.2.3 From d034cfab4f7b9e768c5c1caaa56c5bd4805d2b92 Mon Sep 17 00:00:00 2001 From: Will Deacon Date: Wed, 28 Mar 2012 14:42:47 -0700 Subject: kexec: crash: don't save swapper_pg_dir for !CONFIG_MMU configurations nommu platforms don't have very interesting swapper_pg_dir pointers and usually just #define them to NULL, meaning that we can't include them in the vmcoreinfo on the kexec crash path. This patch only saves the swapper_pg_dir if we have an MMU. Signed-off-by: Will Deacon Reviewed-by: Simon Horman Cc: "Eric W. Biederman" Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds --- kernel/kexec.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/kernel/kexec.c b/kernel/kexec.c index a6a675cb9818..769e347c5196 100644 --- a/kernel/kexec.c +++ b/kernel/kexec.c @@ -1462,7 +1462,9 @@ static int __init crash_save_vmcoreinfo_init(void) VMCOREINFO_SYMBOL(init_uts_ns); VMCOREINFO_SYMBOL(node_online_map); +#ifdef CONFIG_MMU VMCOREINFO_SYMBOL(swapper_pg_dir); +#endif VMCOREINFO_SYMBOL(_stext); VMCOREINFO_SYMBOL(vmlist); -- cgit v1.2.3 From eaa3be6add6f327ab0a633e4fee8e6f2cc8c8a4c Mon Sep 17 00:00:00 2001 From: Zhenzhong Duan Date: Wed, 28 Mar 2012 14:42:47 -0700 Subject: kexec: add further check to crashkernel When using crashkernel=2M-256M, the kernel doesn't give any warning. This is misleading sometimes. Signed-off-by: Zhenzhong Duan Acked-by: "Eric W. Biederman" Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds --- kernel/kexec.c | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/kernel/kexec.c b/kernel/kexec.c index 769e347c5196..3288c9b29bae 100644 --- a/kernel/kexec.c +++ b/kernel/kexec.c @@ -1359,6 +1359,10 @@ static int __init parse_crashkernel_simple(char *cmdline, if (*cur == '@') *crash_base = memparse(cur+1, &cur); + else if (*cur != ' ' && *cur != '\0') { + pr_warning("crashkernel: unrecognized char\n"); + return -EINVAL; + } return 0; } -- cgit v1.2.3 From 09c71bfd8384278c42f56380365940508194cec0 Mon Sep 17 00:00:00 2001 From: Dave Young Date: Wed, 28 Mar 2012 14:42:47 -0700 Subject: kdump x86: fix total mem size calculation for reservation crashkernel reservation need know the total memory size. Current get_total_mem simply use max_pfn - min_low_pfn. It is wrong because it will including memory holes in the middle. Especially for kvm guest with memory > 0xe0000000, there's below in qemu code: qemu split memory as below: if (ram_size >= 0xe0000000 ) { above_4g_mem_size = ram_size - 0xe0000000; below_4g_mem_size = 0xe0000000; } else { below_4g_mem_size = ram_size; } So for 4G mem guest, seabios will insert a 512M usable region beyond of 4G. Thus in above case max_pfn - min_low_pfn will be more than original memsize. Fixing this issue by using memblock_phys_mem_size() to get the total memsize. Signed-off-by: Dave Young Reviewed-by: WANG Cong Reviewed-by: Simon Horman Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds --- arch/x86/kernel/setup.c | 11 +---------- 1 file changed, 1 insertion(+), 10 deletions(-) diff --git a/arch/x86/kernel/setup.c b/arch/x86/kernel/setup.c index 88638883176a..ab77aae4ad9b 100644 --- a/arch/x86/kernel/setup.c +++ b/arch/x86/kernel/setup.c @@ -509,15 +509,6 @@ static void __init memblock_x86_reserve_range_setup_data(void) #ifdef CONFIG_KEXEC -static inline unsigned long long get_total_mem(void) -{ - unsigned long long total; - - total = max_pfn - min_low_pfn; - - return total << PAGE_SHIFT; -} - /* * Keep the crash kernel below this limit. On 32 bits earlier kernels * would limit the kernel to the low 512 MiB due to mapping restrictions. @@ -536,7 +527,7 @@ static void __init reserve_crashkernel(void) unsigned long long crash_size, crash_base; int ret; - total_mem = get_total_mem(); + total_mem = memblock_phys_mem_size(); ret = parse_crashkernel(boot_command_line, total_mem, &crash_size, &crash_base); -- cgit v1.2.3 From b88e769368a88cf28e53db158b84eda096144bce Mon Sep 17 00:00:00 2001 From: Srinivas_Gowda Date: Wed, 28 Mar 2012 14:42:48 -0700 Subject: ipmi: decrease the IPMI message transaction time in interrupt mode Call the event handler immediately after starting the next message. This change considerably decreases the IPMI transaction time (cuts off ~9ms for a single ipmitool transaction). Signed-off-by: Srinivas_Gowda Signed-off-by: Corey Minyard Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds --- drivers/char/ipmi/ipmi_si_intf.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/drivers/char/ipmi/ipmi_si_intf.c b/drivers/char/ipmi/ipmi_si_intf.c index 50fcf9c04569..73ebbb1a3269 100644 --- a/drivers/char/ipmi/ipmi_si_intf.c +++ b/drivers/char/ipmi/ipmi_si_intf.c @@ -932,8 +932,10 @@ static void sender(void *send_info, spin_unlock_irqrestore(&smi_info->msg_lock, flags); spin_lock_irqsave(&smi_info->si_lock, flags); - if (smi_info->si_state == SI_NORMAL && smi_info->curr_msg == NULL) + if (smi_info->si_state == SI_NORMAL && smi_info->curr_msg == NULL) { start_next_msg(smi_info); + smi_event_handler(smi_info, 0); + } spin_unlock_irqrestore(&smi_info->si_lock, flags); } -- cgit v1.2.3 From 828dc9da50f9632bbc5bc9dfa510619d13135015 Mon Sep 17 00:00:00 2001 From: Matthew Garrett Date: Wed, 28 Mar 2012 14:42:48 -0700 Subject: ipmi: increase KCS timeouts We currently time out and retry KCS transactions after 1 second of waiting for IBF or OBF. This appears to be too short for some hardware. The IPMI spec says "All system software wait loops should include error timeouts. For simplicity, such timeouts are not shown explicitly in the flow diagrams. A five-second timeout or greater is recommended". Change the timeout to five seconds to satisfy the slow hardware. Signed-off-by: Matthew Garrett Signed-off-by: Corey Minyard Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds --- drivers/char/ipmi/ipmi_kcs_sm.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/drivers/char/ipmi/ipmi_kcs_sm.c b/drivers/char/ipmi/ipmi_kcs_sm.c index cf82fedae099..e53fc24c6af3 100644 --- a/drivers/char/ipmi/ipmi_kcs_sm.c +++ b/drivers/char/ipmi/ipmi_kcs_sm.c @@ -118,8 +118,8 @@ enum kcs_states { #define MAX_KCS_WRITE_SIZE IPMI_MAX_MSG_LENGTH /* Timeouts in microseconds. */ -#define IBF_RETRY_TIMEOUT 1000000 -#define OBF_RETRY_TIMEOUT 1000000 +#define IBF_RETRY_TIMEOUT 5000000 +#define OBF_RETRY_TIMEOUT 5000000 #define MAX_ERROR_RETRIES 10 #define ERROR0_OBF_WAIT_JIFFIES (2*HZ) -- cgit v1.2.3 From 7adf579c8babf62026e6aab1dee85e6b104d9936 Mon Sep 17 00:00:00 2001 From: Corey Minyard Date: Wed, 28 Mar 2012 14:42:49 -0700 Subject: ipmi: use a tasklet for handling received messages The IPMI driver would release a lock, deliver a message, then relock. This is obviously ugly, and this patch converts the message handler interface to use a tasklet to schedule work. This lets the receive handler be called from an interrupt handler with interrupts enabled. Signed-off-by: Corey Minyard Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds --- drivers/char/ipmi/ipmi_msghandler.c | 141 ++++++++++++++++++++++-------------- drivers/char/ipmi/ipmi_si_intf.c | 14 +--- 2 files changed, 88 insertions(+), 67 deletions(-) diff --git a/drivers/char/ipmi/ipmi_msghandler.c b/drivers/char/ipmi/ipmi_msghandler.c index 58c0e6387cf7..289ab506b79b 100644 --- a/drivers/char/ipmi/ipmi_msghandler.c +++ b/drivers/char/ipmi/ipmi_msghandler.c @@ -46,6 +46,7 @@ #include #include #include +#include #define PFX "IPMI message handler: " @@ -53,6 +54,8 @@ static struct ipmi_recv_msg *ipmi_alloc_recv_msg(void); static int ipmi_init_msghandler(void); +static void smi_recv_tasklet(unsigned long); +static void handle_new_recv_msgs(ipmi_smi_t intf); static int initialized; @@ -355,12 +358,15 @@ struct ipmi_smi { int curr_seq; /* - * Messages that were delayed for some reason (out of memory, - * for instance), will go in here to be processed later in a - * periodic timer interrupt. + * Messages queued for delivery. If delivery fails (out of memory + * for instance), They will stay in here to be processed later in a + * periodic timer interrupt. The tasklet is for handling received + * messages directly from the handler. */ spinlock_t waiting_msgs_lock; struct list_head waiting_msgs; + atomic_t watchdog_pretimeouts_to_deliver; + struct tasklet_struct recv_tasklet; /* * The list of command receivers that are registered for commands @@ -493,6 +499,8 @@ static void clean_up_interface_data(ipmi_smi_t intf) struct cmd_rcvr *rcvr, *rcvr2; struct list_head list; + tasklet_kill(&intf->recv_tasklet); + free_smi_msg_list(&intf->waiting_msgs); free_recv_msg_list(&intf->waiting_events); @@ -2792,6 +2800,9 @@ void ipmi_poll_interface(ipmi_user_t user) if (intf->handlers->poll) intf->handlers->poll(intf->send_info); + + /* In case something came in */ + handle_new_recv_msgs(intf); } EXPORT_SYMBOL(ipmi_poll_interface); @@ -2860,6 +2871,10 @@ int ipmi_register_smi(struct ipmi_smi_handlers *handlers, #endif spin_lock_init(&intf->waiting_msgs_lock); INIT_LIST_HEAD(&intf->waiting_msgs); + tasklet_init(&intf->recv_tasklet, + smi_recv_tasklet, + (unsigned long) intf); + atomic_set(&intf->watchdog_pretimeouts_to_deliver, 0); spin_lock_init(&intf->events_lock); INIT_LIST_HEAD(&intf->waiting_events); intf->waiting_events_count = 0; @@ -3622,11 +3637,11 @@ static int handle_bmc_rsp(ipmi_smi_t intf, } /* - * Handle a new message. Return 1 if the message should be requeued, + * Handle a received message. Return 1 if the message should be requeued, * 0 if the message should be freed, or -1 if the message should not * be freed or requeued. */ -static int handle_new_recv_msg(ipmi_smi_t intf, +static int handle_one_recv_msg(ipmi_smi_t intf, struct ipmi_smi_msg *msg) { int requeue; @@ -3784,12 +3799,72 @@ static int handle_new_recv_msg(ipmi_smi_t intf, return requeue; } +/* + * If there are messages in the queue or pretimeouts, handle them. + */ +static void handle_new_recv_msgs(ipmi_smi_t intf) +{ + struct ipmi_smi_msg *smi_msg; + unsigned long flags = 0; + int rv; + int run_to_completion = intf->run_to_completion; + + /* See if any waiting messages need to be processed. */ + if (!run_to_completion) + spin_lock_irqsave(&intf->waiting_msgs_lock, flags); + while (!list_empty(&intf->waiting_msgs)) { + smi_msg = list_entry(intf->waiting_msgs.next, + struct ipmi_smi_msg, link); + list_del(&smi_msg->link); + if (!run_to_completion) + spin_unlock_irqrestore(&intf->waiting_msgs_lock, flags); + rv = handle_one_recv_msg(intf, smi_msg); + if (!run_to_completion) + spin_lock_irqsave(&intf->waiting_msgs_lock, flags); + if (rv == 0) { + /* Message handled */ + ipmi_free_smi_msg(smi_msg); + } else if (rv < 0) { + /* Fatal error on the message, del but don't free. */ + } else { + /* + * To preserve message order, quit if we + * can't handle a message. + */ + list_add(&smi_msg->link, &intf->waiting_msgs); + break; + } + } + if (!run_to_completion) + spin_unlock_irqrestore(&intf->waiting_msgs_lock, flags); + + /* + * If the pretimout count is non-zero, decrement one from it and + * deliver pretimeouts to all the users. + */ + if (atomic_add_unless(&intf->watchdog_pretimeouts_to_deliver, -1, 0)) { + ipmi_user_t user; + + rcu_read_lock(); + list_for_each_entry_rcu(user, &intf->users, link) { + if (user->handler->ipmi_watchdog_pretimeout) + user->handler->ipmi_watchdog_pretimeout( + user->handler_data); + } + rcu_read_unlock(); + } +} + +static void smi_recv_tasklet(unsigned long val) +{ + handle_new_recv_msgs((ipmi_smi_t) val); +} + /* Handle a new message from the lower layer. */ void ipmi_smi_msg_received(ipmi_smi_t intf, struct ipmi_smi_msg *msg) { unsigned long flags = 0; /* keep us warning-free. */ - int rv; int run_to_completion; @@ -3843,31 +3918,11 @@ void ipmi_smi_msg_received(ipmi_smi_t intf, run_to_completion = intf->run_to_completion; if (!run_to_completion) spin_lock_irqsave(&intf->waiting_msgs_lock, flags); - if (!list_empty(&intf->waiting_msgs)) { - list_add_tail(&msg->link, &intf->waiting_msgs); - if (!run_to_completion) - spin_unlock_irqrestore(&intf->waiting_msgs_lock, flags); - goto out; - } + list_add_tail(&msg->link, &intf->waiting_msgs); if (!run_to_completion) spin_unlock_irqrestore(&intf->waiting_msgs_lock, flags); - rv = handle_new_recv_msg(intf, msg); - if (rv > 0) { - /* - * Could not handle the message now, just add it to a - * list to handle later. - */ - run_to_completion = intf->run_to_completion; - if (!run_to_completion) - spin_lock_irqsave(&intf->waiting_msgs_lock, flags); - list_add_tail(&msg->link, &intf->waiting_msgs); - if (!run_to_completion) - spin_unlock_irqrestore(&intf->waiting_msgs_lock, flags); - } else if (rv == 0) { - ipmi_free_smi_msg(msg); - } - + tasklet_schedule(&intf->recv_tasklet); out: return; } @@ -3875,16 +3930,8 @@ EXPORT_SYMBOL(ipmi_smi_msg_received); void ipmi_smi_watchdog_pretimeout(ipmi_smi_t intf) { - ipmi_user_t user; - - rcu_read_lock(); - list_for_each_entry_rcu(user, &intf->users, link) { - if (!user->handler->ipmi_watchdog_pretimeout) - continue; - - user->handler->ipmi_watchdog_pretimeout(user->handler_data); - } - rcu_read_unlock(); + atomic_set(&intf->watchdog_pretimeouts_to_deliver, 1); + tasklet_schedule(&intf->recv_tasklet); } EXPORT_SYMBOL(ipmi_smi_watchdog_pretimeout); @@ -3998,28 +4045,12 @@ static void ipmi_timeout_handler(long timeout_period) ipmi_smi_t intf; struct list_head timeouts; struct ipmi_recv_msg *msg, *msg2; - struct ipmi_smi_msg *smi_msg, *smi_msg2; unsigned long flags; int i; rcu_read_lock(); list_for_each_entry_rcu(intf, &ipmi_interfaces, link) { - /* See if any waiting messages need to be processed. */ - spin_lock_irqsave(&intf->waiting_msgs_lock, flags); - list_for_each_entry_safe(smi_msg, smi_msg2, - &intf->waiting_msgs, link) { - if (!handle_new_recv_msg(intf, smi_msg)) { - list_del(&smi_msg->link); - ipmi_free_smi_msg(smi_msg); - } else { - /* - * To preserve message order, quit if we - * can't handle a message. - */ - break; - } - } - spin_unlock_irqrestore(&intf->waiting_msgs_lock, flags); + tasklet_schedule(&intf->recv_tasklet); /* * Go through the seq table and find any messages that diff --git a/drivers/char/ipmi/ipmi_si_intf.c b/drivers/char/ipmi/ipmi_si_intf.c index 73ebbb1a3269..01e53cd105dd 100644 --- a/drivers/char/ipmi/ipmi_si_intf.c +++ b/drivers/char/ipmi/ipmi_si_intf.c @@ -320,16 +320,8 @@ static int register_xaction_notifier(struct notifier_block *nb) static void deliver_recv_msg(struct smi_info *smi_info, struct ipmi_smi_msg *msg) { - /* Deliver the message to the upper layer with the lock - released. */ - - if (smi_info->run_to_completion) { - ipmi_smi_msg_received(smi_info->intf, msg); - } else { - spin_unlock(&(smi_info->si_lock)); - ipmi_smi_msg_received(smi_info->intf, msg); - spin_lock(&(smi_info->si_lock)); - } + /* Deliver the message to the upper layer. */ + ipmi_smi_msg_received(smi_info->intf, msg); } static void return_hosed_msg(struct smi_info *smi_info, int cCode) @@ -481,9 +473,7 @@ static void handle_flags(struct smi_info *smi_info) start_clear_flags(smi_info); smi_info->msg_flags &= ~WDT_PRE_TIMEOUT_INT; - spin_unlock(&(smi_info->si_lock)); ipmi_smi_watchdog_pretimeout(smi_info->intf); - spin_lock(&(smi_info->si_lock)); } else if (smi_info->msg_flags & RECEIVE_MSG_AVAIL) { /* Messages available. */ smi_info->curr_msg = ipmi_alloc_smi_msg(); -- cgit v1.2.3 From 895dcfd1cab84d7e1c22af645a7f2f3c9bb5f24e Mon Sep 17 00:00:00 2001 From: Corey Minyard Date: Wed, 28 Mar 2012 14:42:49 -0700 Subject: ipmi: fix message handling during panics The part of the IPMI driver that delivered panic information to the event log and extended the watchdog timeout during a panic was not properly handling the messages. It used static messages to avoid allocation, but wasn't properly waiting for these, or wasn't properly handling the refcounts. Signed-off-by: Corey Minyard Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds --