diff options
author | Pekka Enberg <penberg@kernel.org> | 2013-05-07 09:19:47 +0300 |
---|---|---|
committer | Pekka Enberg <penberg@kernel.org> | 2013-05-07 09:19:47 +0300 |
commit | 69df2ac1288b456a95aceadafbf88cd891a577c8 (patch) | |
tree | 0f2e83a8c4bc826f12d3f3871ecc1d7be0c9e4e3 /mm/slab.c | |
parent | c1be5a5b1b355d40e6cf79cc979eb66dafa24ad1 (diff) | |
parent | 8a965b3baa89ffedc73c0fbc750006c631012ced (diff) |
Merge branch 'slab/next' into slab/for-linus
Diffstat (limited to 'mm/slab.c')
-rw-r--r-- | mm/slab.c | 790 |
1 files changed, 334 insertions, 456 deletions
diff --git a/mm/slab.c b/mm/slab.c index 856e4a192d25..a28562306d06 100644 --- a/mm/slab.c +++ b/mm/slab.c @@ -286,68 +286,27 @@ struct arraycache_init { }; /* - * The slab lists for all objects. - */ -struct kmem_list3 { - struct list_head slabs_partial; /* partial list first, better asm code */ - struct list_head slabs_full; - struct list_head slabs_free; - unsigned long free_objects; - unsigned int free_limit; - unsigned int colour_next; /* Per-node cache coloring */ - spinlock_t list_lock; - struct array_cache *shared; /* shared per node */ - struct array_cache **alien; /* on other nodes */ - unsigned long next_reap; /* updated without locking */ - int free_touched; /* updated without locking */ -}; - -/* * Need this for bootstrapping a per node allocator. */ #define NUM_INIT_LISTS (3 * MAX_NUMNODES) -static struct kmem_list3 __initdata initkmem_list3[NUM_INIT_LISTS]; +static struct kmem_cache_node __initdata init_kmem_cache_node[NUM_INIT_LISTS]; #define CACHE_CACHE 0 #define SIZE_AC MAX_NUMNODES -#define SIZE_L3 (2 * MAX_NUMNODES) +#define SIZE_NODE (2 * MAX_NUMNODES) static int drain_freelist(struct kmem_cache *cache, - struct kmem_list3 *l3, int tofree); + struct kmem_cache_node *n, int tofree); static void free_block(struct kmem_cache *cachep, void **objpp, int len, int node); static int enable_cpucache(struct kmem_cache *cachep, gfp_t gfp); static void cache_reap(struct work_struct *unused); -/* - * This function must be completely optimized away if a constant is passed to - * it. Mostly the same as what is in linux/slab.h except it returns an index. - */ -static __always_inline int index_of(const size_t size) -{ - extern void __bad_size(void); - - if (__builtin_constant_p(size)) { - int i = 0; - -#define CACHE(x) \ - if (size <=x) \ - return i; \ - else \ - i++; -#include <linux/kmalloc_sizes.h> -#undef CACHE - __bad_size(); - } else - __bad_size(); - return 0; -} - static int slab_early_init = 1; -#define INDEX_AC index_of(sizeof(struct arraycache_init)) -#define INDEX_L3 index_of(sizeof(struct kmem_list3)) +#define INDEX_AC kmalloc_index(sizeof(struct arraycache_init)) +#define INDEX_NODE kmalloc_index(sizeof(struct kmem_cache_node)) -static void kmem_list3_init(struct kmem_list3 *parent) +static void kmem_cache_node_init(struct kmem_cache_node *parent) { INIT_LIST_HEAD(&parent->slabs_full); INIT_LIST_HEAD(&parent->slabs_partial); @@ -363,7 +322,7 @@ static void kmem_list3_init(struct kmem_list3 *parent) #define MAKE_LIST(cachep, listp, slab, nodeid) \ do { \ INIT_LIST_HEAD(listp); \ - list_splice(&(cachep->nodelists[nodeid]->slab), listp); \ + list_splice(&(cachep->node[nodeid]->slab), listp); \ } while (0) #define MAKE_ALL_LISTS(cachep, ptr, nodeid) \ @@ -524,30 +483,6 @@ static inline unsigned int obj_to_index(const struct kmem_cache *cache, return reciprocal_divide(offset, cache->reciprocal_buffer_size); } -/* - * These are the default caches for kmalloc. Custom caches can have other sizes. - */ -struct cache_sizes malloc_sizes[] = { -#define CACHE(x) { .cs_size = (x) }, -#include <linux/kmalloc_sizes.h> - CACHE(ULONG_MAX) -#undef CACHE -}; -EXPORT_SYMBOL(malloc_sizes); - -/* Must match cache_sizes above. Out of line to keep cache footprint low. */ -struct cache_names { - char *name; - char *name_dma; -}; - -static struct cache_names __initdata cache_names[] = { -#define CACHE(x) { .name = "size-" #x, .name_dma = "size-" #x "(DMA)" }, -#include <linux/kmalloc_sizes.h> - {NULL,} -#undef CACHE -}; - static struct arraycache_init initarray_generic = { {0, BOOT_CPUCACHE_ENTRIES, 1, 0} }; @@ -586,15 +521,15 @@ static void slab_set_lock_classes(struct kmem_cache *cachep, int q) { struct array_cache **alc; - struct kmem_list3 *l3; + struct kmem_cache_node *n; int r; - l3 = cachep->nodelists[q]; - if (!l3) + n = cachep->node[q]; + if (!n) return; - lockdep_set_class(&l3->list_lock, l3_key); - alc = l3->alien; + lockdep_set_class(&n->list_lock, l3_key); + alc = n->alien; /* * FIXME: This check for BAD_ALIEN_MAGIC * should go away when common slab code is taught to @@ -625,28 +560,30 @@ static void slab_set_debugobj_lock_classes(struct kmem_cache *cachep) static void init_node_lock_keys(int q) { - struct cache_sizes *s = malloc_sizes; + int i; if (slab_state < UP) return; - for (s = malloc_sizes; s->cs_size != ULONG_MAX; s++) { - struct kmem_list3 *l3; + for (i = 1; i < PAGE_SHIFT + MAX_ORDER; i++) { + struct kmem_cache_node *n; + struct kmem_cache *cache = kmalloc_caches[i]; + + if (!cache) + continue; - l3 = s->cs_cachep->nodelists[q]; - if (!l3 || OFF_SLAB(s->cs_cachep)) + n = cache->node[q]; + if (!n || OFF_SLAB(cache)) continue; - slab_set_lock_classes(s->cs_cachep, &on_slab_l3_key, + slab_set_lock_classes(cache, &on_slab_l3_key, &on_slab_alc_key, q); } } static void on_slab_lock_classes_node(struct kmem_cache *cachep, int q) { - struct kmem_list3 *l3; - l3 = cachep->nodelists[q]; - if (!l3) + if (!cachep->node[q]) return; slab_set_lock_classes(cachep, &on_slab_l3_key, @@ -702,41 +639,6 @@ static inline struct array_cache *cpu_cache_get(struct kmem_cache *cachep) return cachep->array[smp_processor_id()]; } -static inline struct kmem_cache *__find_general_cachep(size_t size, - gfp_t gfpflags) -{ - struct cache_sizes *csizep = malloc_sizes; - -#if DEBUG - /* This happens if someone tries to call - * kmem_cache_create(), or __kmalloc(), before - * the generic caches are initialized. - */ - BUG_ON(malloc_sizes[INDEX_AC].cs_cachep == NULL); -#endif - if (!size) - return ZERO_SIZE_PTR; - - while (size > csizep->cs_size) - csizep++; - - /* - * Really subtle: The last entry with cs->cs_size==ULONG_MAX - * has cs_{dma,}cachep==NULL. Thus no special case - * for large kmalloc calls required. - */ -#ifdef CONFIG_ZONE_DMA - if (unlikely(gfpflags & GFP_DMA)) - return csizep->cs_dmacachep; -#endif - return csizep->cs_cachep; -} - -static struct kmem_cache *kmem_find_general_cachep(size_t size, gfp_t gfpflags) -{ - return __find_general_cachep(size, gfpflags); -} - static size_t slab_mgmt_size(size_t nr_objs, size_t align) { return ALIGN(sizeof(struct slab)+nr_objs*sizeof(kmem_bufctl_t), align); @@ -938,29 +840,29 @@ static inline bool is_slab_pfmemalloc(struct slab *slabp) static void recheck_pfmemalloc_active(struct kmem_cache *cachep, struct array_cache *ac) { - struct kmem_list3 *l3 = cachep->nodelists[numa_mem_id()]; + struct kmem_cache_node *n = cachep->node[numa_mem_id()]; struct slab *slabp; unsigned long flags; if (!pfmemalloc_active) return; - spin_lock_irqsave(&l3->list_lock, flags); - list_for_each_entry(slabp, &l3->slabs_full, list) + spin_lock_irqsave(&n->list_lock, flags); + list_for_each_entry(slabp, &n->slabs_full, list) if (is_slab_pfmemalloc(slabp)) goto out; - list_for_each_entry(slabp, &l3->slabs_partial, list) + list_for_each_entry(slabp, &n->slabs_partial, list) if (is_slab_pfmemalloc(slabp)) goto out; - list_for_each_entry(slabp, &l3->slabs_free, list) + list_for_each_entry(slabp, &n->slabs_free, list) if (is_slab_pfmemalloc(slabp)) goto out; pfmemalloc_active = false; out: - spin_unlock_irqrestore(&l3->list_lock, flags); + spin_unlock_irqrestore(&n->list_lock, flags); } static void *__ac_get_obj(struct kmem_cache *cachep, struct array_cache *ac, @@ -971,7 +873,7 @@ static void *__ac_get_obj(struct kmem_cache *cachep, struct array_cache *ac, /* Ensure the caller is allowed to use objects from PFMEMALLOC slab */ if (unlikely(is_obj_pfmemalloc(objp))) { - struct kmem_list3 *l3; + struct kmem_cache_node *n; if (gfp_pfmemalloc_allowed(flags)) { clear_obj_pfmemalloc(&objp); @@ -993,8 +895,8 @@ static void *__ac_get_obj(struct kmem_cache *cachep, struct array_cache *ac, * If there are empty slabs on the slabs_free list and we are * being forced to refill the cache, mark this one !pfmemalloc. */ - l3 = cachep->nodelists[numa_mem_id()]; - if (!list_empty(&l3->slabs_free) && force_refill) { + n = cachep->node[numa_mem_id()]; + if (!list_empty(&n->slabs_free) && force_refill) { struct slab *slabp = virt_to_slab(objp); ClearPageSlabPfmemalloc(virt_to_head_page(slabp->s_mem)); clear_obj_pfmemalloc(&objp); @@ -1071,7 +973,7 @@ static int transfer_objects(struct array_cache *to, #ifndef CONFIG_NUMA #define drain_alien_cache(cachep, alien) do { } while (0) -#define reap_alien(cachep, l3) do { } while (0) +#define reap_alien(cachep, n) do { } while (0) static inline struct array_cache **alloc_alien_cache(int node, int limit, gfp_t gfp) { @@ -1143,33 +1045,33 @@ static void free_alien_cache(struct array_cache **ac_ptr) static void __drain_alien_cache(struct kmem_cache *cachep, struct array_cache *ac, int node) { - struct kmem_list3 *rl3 = cachep->nodelists[node]; + struct kmem_cache_node *n = cachep->node[node]; if (ac->avail) { - spin_lock(&rl3->list_lock); + spin_lock(&n->list_lock); /* * Stuff objects into the remote nodes shared array first. * That way we could avoid the overhead of putting the objects * into the free lists and getting them back later. */ - if (rl3->shared) - transfer_objects(rl3->shared, ac, ac->limit); + if (n->shared) + transfer_objects(n->shared, ac, ac->limit); free_block(cachep, ac->entry, ac->avail, node); ac->avail = 0; - spin_unlock(&rl3->list_lock); + spin_unlock(&n->list_lock); } } /* * Called from cache_reap() to regularly drain alien caches round robin. */ -static void reap_alien(struct kmem_cache *cachep, struct kmem_list3 *l3) +static void reap_alien(struct kmem_cache *cachep, struct kmem_cache_node *n) { int node = __this_cpu_read(slab_reap_node); - if (l3->alien) { - struct array_cache *ac = l3->alien[node]; + if (n->alien) { + struct array_cache *ac = n->alien[node]; if (ac && ac->avail && spin_trylock_irq(&ac->lock)) { __drain_alien_cache(cachep, ac, node); @@ -1199,7 +1101,7 @@ static inline int cache_free_alien(struct kmem_cache *cachep, void *objp) { struct slab *slabp = virt_to_slab(objp); int nodeid = slabp->nodeid; - struct kmem_list3 *l3; + struct kmem_cache_node *n; struct array_cache *alien = NULL; int node; @@ -1212,10 +1114,10 @@ static inline int cache_free_alien(struct kmem_cache *cachep, void *objp) if (likely(slabp->nodeid == node)) return 0; - l3 = cachep->nodelists[node]; + n = cachep->node[node]; STATS_INC_NODEFREES(cachep); - if (l3->alien && l3->alien[nodeid]) { - alien = l3->alien[nodeid]; + if (n->alien && n->alien[nodeid]) { + alien = n->alien[nodeid]; spin_lock(&alien->lock); if (unlikely(alien->avail == alien->limit)) { STATS_INC_ACOVERFLOW(cachep); @@ -1224,28 +1126,28 @@ static inline int cache_free_alien(struct kmem_cache *cachep, void *objp) ac_put_obj(cachep, alien, objp); spin_unlock(&alien->lock); } else { - spin_lock(&(cachep->nodelists[nodeid])->list_lock); + spin_lock(&(cachep->node[nodeid])->list_lock); free_block(cachep, &objp, 1, nodeid); - spin_unlock(&(cachep->nodelists[nodeid])->list_lock); + spin_unlock(&(cachep->node[nodeid])->list_lock); } return 1; } #endif /* - * Allocates and initializes nodelists for a node on each slab cache, used for - * either memory or cpu hotplug. If memory is being hot-added, the kmem_list3 + * Allocates and initializes node for a node on each slab cache, used for + * either memory or cpu hotplug. If memory is being hot-added, the kmem_cache_node * will be allocated off-node since memory is not yet online for the new node. - * When hotplugging memory or a cpu, existing nodelists are not replaced if + * When hotplugging memory or a cpu, existing node are not replaced if * already in use. * * Must hold slab_mutex. */ -static int init_cache_nodelists_node(int node) +static int init_cache_node_node(int node) { struct kmem_cache *cachep; - struct kmem_list3 *l3; - const int memsize = sizeof(struct kmem_list3); + struct kmem_cache_node *n; + const int memsize = sizeof(struct kmem_cache_node); list_for_each_entry(cachep, &slab_caches, list) { /* @@ -1253,12 +1155,12 @@ static int init_cache_nodelists_node(int node) * begin anything. Make sure some other cpu on this * node has not already allocated this */ - if (!cachep->nodelists[node]) { - l3 = kmalloc_node(memsize, GFP_KERNEL, node); - if (!l3) + if (!cachep->node[node]) { + n = kmalloc_node(memsize, GFP_KERNEL, node); + if (!n) return -ENOMEM; - kmem_list3_init(l3); - l3->next_reap = jiffies + REAPTIMEOUT_LIST3 + + kmem_cache_node_init(n); + n->next_reap = jiffies + REAPTIMEOUT_LIST3 + ((unsigned long)cachep) % REAPTIMEOUT_LIST3; /* @@ -1266,14 +1168,14 @@ static int init_cache_nodelists_node(int node) * go. slab_mutex is sufficient * protection here. */ - cachep->nodelists[node] = l3; + cachep->node[node] = n; } - spin_lock_irq(&cachep->nodelists[node]->list_lock); - cachep->nodelists[node]->free_limit = + spin_lock_irq(&cachep->node[node]->list_lock); + cachep->node[node]->free_limit = (1 + nr_cpus_node(node)) * cachep->batchcount + cachep->num; - spin_unlock_irq(&cachep->nodelists[node]->list_lock); + spin_unlock_irq(&cachep->node[node]->list_lock); } return 0; } @@ -1281,7 +1183,7 @@ static int init_cache_nodelists_node(int node) static void __cpuinit cpuup_canceled(long cpu) { struct kmem_cache *cachep; - struct kmem_list3 *l3 = NULL; + struct kmem_cache_node *n = NULL; int node = cpu_to_mem(cpu); const struct cpumask *mask = cpumask_of_node(node); @@ -1293,34 +1195,34 @@ static void __cpuinit cpuup_canceled(long cpu) /* cpu is dead; no one can alloc from it. */ nc = cachep->array[cpu]; cachep->array[cpu] = NULL; - l3 = cachep->nodelists[node]; + n = cachep->node[node]; - if (!l3) + if (!n) goto free_array_cache; - spin_lock_irq(&l3->list_lock); + spin_lock_irq(&n->list_lock); - /* Free limit for this kmem_list3 */ - l3->free_limit -= cachep->batchcount; + /* Free limit for this kmem_cache_node */ + n->free_limit -= cachep->batchcount; if (nc) free_block(cachep, nc->entry, nc->avail, node); if (!cpumask_empty(mask)) { - spin_unlock_irq(&l3->list_lock); + spin_unlock_irq(&n->list_lock); goto free_array_cache; } - shared = l3->shared; + shared = n->shared; if (shared) { free_block(cachep, shared->entry, shared->avail, node); - l3->shared = NULL; + n->shared = NULL; } - alien = l3->alien; - l3->alien = NULL; + alien = n->alien; + n->alien = NULL; - spin_unlock_irq(&l3->list_lock); + spin_unlock_irq(&n->list_lock); kfree(shared); if (alien) { @@ -1336,17 +1238,17 @@ free_array_cache: * shrink each nodelist to its limit. */ list_for_each_entry(cachep, &slab_caches, list) { - l3 = cachep->nodelists[node]; - if (!l3) + n = cachep->node[node]; + if (!n) continue; - drain_freelist(cachep, l3, l3->free_objects); + drain_freelist(cachep, n, n->free_objects); } } static int __cpuinit cpuup_prepare(long cpu) { struct kmem_cache *cachep; - struct kmem_list3 *l3 = NULL; + struct kmem_cache_node *n = NULL; int node = cpu_to_mem(cpu); int err; @@ -1354,9 +1256,9 @@ static int __cpuinit cpuup_prepare(long cpu) * We need to do this right in the beginning since * alloc_arraycache's are going to use this list. * kmalloc_node allows us to add the slab to the right - * kmem_list3 and not this cpu's kmem_list3 + * kmem_cache_node and not this cpu's kmem_cache_node */ - err = init_cache_nodelists_node(node); + err = init_cache_node_node(node); if (err < 0) goto bad; @@ -1391,25 +1293,25 @@ static int __cpuinit cpuup_prepare(long cpu) } } cachep->array[cpu] = nc; - l3 = cachep->nodelists[node]; - BUG_ON(!l3); + n = cachep->node[node]; + BUG_ON(!n); - spin_lock_irq(&l3->list_lock); - if (!l3->shared) { + spin_lock_irq(&n->list_lock); + if (!n->shared) { /* * We are serialised from CPU_DEAD or * CPU_UP_CANCELLED by the cpucontrol lock */ - l3->shared = shared; + n->shared = shared; shared = NULL; } #ifdef CONFIG_NUMA - if (!l3->alien) { - l3->alien = alien; + if (!n->alien) { + n->alien = alien; alien = NULL; } #endif - spin_unlock_irq(&l3->list_lock); + spin_unlock_irq(&n->list_lock); kfree(shared); free_alien_cache(alien); if (cachep->flags & SLAB_DEBUG_OBJECTS) @@ -1464,9 +1366,9 @@ static int __cpuinit cpuup_callback(struct notifier_block *nfb, case CPU_DEAD_FROZEN: /* * Even if all the cpus of a node are down, we don't free the - * kmem_list3 of any cache. This to avoid a race between + * kmem_cache_node of any cache. This to avoid a race between * cpu_down, and a kmalloc allocation from another cpu for - * memory from the node of the cpu going down. The list3 + * memory from the node of the cpu going down. The node * structure is usually allocated from kmem_cache_create() and * gets destroyed at kmem_cache_destroy(). */ @@ -1494,22 +1396,22 @@ static struct notifier_block __cpuinitdata cpucache_notifier = { * * Must hold slab_mutex. */ -static int __meminit drain_cache_nodelists_node(int node) +static int __meminit drain_cache_node_node(int node) { struct kmem_cache *cachep; int ret = 0; list_for_each_entry(cachep, &slab_caches, list) { - struct kmem_list3 *l3; + struct kmem_cache_node *n; - l3 = cachep->nodelists[node]; - if (!l3) + n = cachep->node[node]; + if (!n) continue; - drain_freelist(cachep, l3, l3->free_objects); + drain_freelist(cachep, n, n->free_objects); - if (!list_empty(&l3->slabs_full) || - !list_empty(&l3->slabs_partial)) { + if (!list_empty(&n->slabs_full) || + !list_empty(&n->slabs_partial)) { ret = -EBUSY; break; } @@ -1531,12 +1433,12 @@ static int __meminit slab_memory_callback(struct notifier_block *self, switch (action) { case MEM_GOING_ONLINE: mutex_lock(&slab_mutex); - ret = init_cache_nodelists_node(nid); + ret = init_cache_node_node(nid); mutex_unlock(&slab_mutex); break; case MEM_GOING_OFFLINE: mutex_lock(&slab_mutex); - ret = drain_cache_nodelists_node(nid); + ret = drain_cache_node_node(nid); mutex_unlock(&slab_mutex); break; case MEM_ONLINE: @@ -1551,37 +1453,37 @@ out: #endif /* CONFIG_NUMA && CONFIG_MEMORY_HOTPLUG */ /* - * swap the static kmem_list3 with kmalloced memory + * swap the static kmem_cache_node with kmalloced memory */ -static void __init init_list(struct kmem_cache *cachep, struct kmem_list3 *list, +static void __init init_list(struct kmem_cache *cachep, struct kmem_cache_node *list, int nodeid) { - struct kmem_list3 *ptr; + struct kmem_cache_node *ptr; - ptr = kmalloc_node(sizeof(struct kmem_list3), GFP_NOWAIT, nodeid); + ptr = kmalloc_node(sizeof(struct kmem_cache_node), GFP_NOWAIT, nodeid); BUG_ON(!ptr); - memcpy(ptr, list, sizeof(struct kmem_list3)); + memcpy(ptr, list, sizeof(struct kmem_cache_node)); /* * Do not assume that spinlocks can be initialized via memcpy: */ spin_lock_init(&ptr->list_lock); MAKE_ALL_LISTS(cachep, ptr, nodeid); - cachep->nodelists[nodeid] = ptr; + cachep->node[nodeid] = ptr; } /* - * For setting up all the kmem_list3s for cache whose buffer_size is same as - * size of kmem_list3. + * For setting up all the kmem_cache_node for cache whose buffer_size is same as + * size of kmem_cache_node. */ -static void __init set_up_list3s(struct kmem_cache *cachep, int index) +static void __init set_up_node(struct kmem_cache *cachep, int index) { int node; for_each_online_node(node) { - cachep->nodelists[node] = &initkmem_list3[index + node]; - cachep->nodelists[node]->next_reap = jiffies + + cachep->node[node] = &init_kmem_cache_node[index + node]; + cachep->node[node]->next_reap = jiffies + REAPTIMEOUT_LIST3 + ((unsigned long)cachep) % REAPTIMEOUT_LIST3; } @@ -1589,11 +1491,11 @@ static void __init set_up_list3s(struct kmem_cache *cachep, int index) /* * The memory after the last cpu cache pointer is used for the - * the nodelists pointer. + * the node pointer. */ -static void setup_nodelists_pointer(struct kmem_cache *cachep) +static void setup_node_pointer(struct kmem_cache *cachep) { - cachep->nodelists = (struct kmem_list3 **)&cachep->array[nr_cpu_ids]; + cachep->node = (struct kmem_cache_node **)&cachep->array[nr_cpu_ids]; } /* @@ -1602,20 +1504,18 @@ static void setup_nodelists_pointer(struct kmem_cache *cachep) */ void __init kmem_cache_init(void) { - struct cache_sizes *sizes; - struct cache_names *names; int i; kmem_cache = &kmem_cache_boot; - setup_nodelists_pointer(kmem_cache); + setup_node_pointer(kmem_cache); if (num_possible_nodes() == 1) use_alien_caches = 0; for (i = 0; i < NUM_INIT_LISTS; i++) - kmem_list3_init(&initkmem_list3[i]); + kmem_cache_node_init(&init_kmem_cache_node[i]); - set_up_list3s(kmem_cache, CACHE_CACHE); + set_up_node(kmem_cache, CACHE_CACHE); /* * Fragmentation resistance on low memory - only use bigger @@ -1631,7 +1531,7 @@ void __init kmem_cache_init(void) * kmem_cache structures of all caches, except kmem_cache itself: * kmem_cache is statically allocated. * Initially an __init data area is used for the head array and the - * kmem_list3 structures, it's replaced with a kmalloc allocated + * kmem_cache_node structures, it's replaced with a kmalloc allocated * array at the end of the bootstrap. * 2) Create the first kmalloc cache. * The struct kmem_cache for the new cache is allocated normally. @@ -1640,7 +1540,7 @@ void __init kmem_cache_init(void) * head arrays. * 4) Replace the __init data head arrays for kmem_cache and the first * kmalloc cache with kmalloc allocated arrays. - * 5) Replace the __init data for kmem_list3 for kmem_cache and + * 5) Replace the __init data for kmem_cache_node for kmem_cache and * the other cache's with kmalloc allocated memory. * 6) Resize the head arrays of the kmalloc caches to their final sizes. */ @@ -1652,50 +1552,28 @@ void __init kmem_cache_init(void) */ create_boot_cache(kmem_cache, "kmem_cache", offsetof(struct kmem_cache, array[nr_cpu_ids]) + - nr_node_ids * sizeof(struct kmem_list3 *), + nr_node_ids * sizeof(struct kmem_cache_node *), SLAB_HWCACHE_ALIGN); list_add(&kmem_cache->list, &slab_caches); /* 2+3) create the kmalloc caches */ - sizes = malloc_sizes; - names = cache_names; /* * Initialize the caches that provide memory for the array cache and the - * kmem_list3 structures first. Without this, further allocations will + * kmem_cache_node structures first. Without this, further allocations will * bug. */ - sizes[INDEX_AC].cs_cachep = create_kmalloc_cache(names[INDEX_AC].name, - sizes[INDEX_AC].cs_size, ARCH_KMALLOC_FLAGS); + kmalloc_caches[INDEX_AC] = create_kmalloc_cache("kmalloc-ac", + kmalloc_size(INDEX_AC), ARCH_KMALLOC_FLAGS); - if (INDEX_AC != INDEX_L3) - sizes[INDEX_L3].cs_cachep = - create_kmalloc_cache(names[INDEX_L3].name, - sizes[INDEX_L3].cs_size, ARCH_KMALLOC_FLAGS); + if (INDEX_AC != INDEX_NODE) + kmalloc_caches[INDEX_NODE] = + create_kmalloc_cache("kmalloc-node", + kmalloc_size(INDEX_NODE), ARCH_KMALLOC_FLAGS); slab_early_init = 0; - while (sizes->cs_size != ULONG_MAX) { - /* - * For performance, all the general caches are L1 aligned. - * This should be particularly beneficial on SMP boxes, as it - * eliminates "false sharing". - * Note for systems short on memory removing the alignment will - * allow tighter packing of the smaller caches. - */ - if (!sizes->cs_cachep) - sizes->cs_cachep = create_kmalloc_cache(names->name, - sizes->cs_size, ARCH_KMALLOC_FLAGS); - -#ifdef CONFIG_ZONE_DMA - sizes->cs_dmacachep = create_kmalloc_cache( - names->name_dma, sizes->cs_size, - SLAB_CACHE_DMA|ARCH_KMALLOC_FLAGS); -#endif - sizes++; - names++; - } /* 4) Replace the bootstrap head arrays */ { struct array_cache *ptr; @@ -1713,36 +1591,35 @@ void __init kmem_cache_init(void) ptr = kmalloc(sizeof(struct arraycache_init), GFP_NOWAIT); - BUG_ON(cpu_cache_get(malloc_sizes[INDEX_AC].cs_cachep) + BUG_ON(cpu_cache_get(kmalloc_caches[INDEX_AC]) != &initarray_generic.cache); - memcpy(ptr, cpu_cache_get(malloc_sizes[INDEX_AC].cs_cachep), + memcpy(ptr, cpu_cache_get(kmalloc_caches[INDEX_AC]), sizeof(struct arraycache_init)); /* * Do not assume that spinlocks can be initialized via memcpy: */ spin_lock_init(&ptr->lock); - malloc_sizes[INDEX_AC].cs_cachep->array[smp_processor_id()] = - ptr; + kmalloc_caches[INDEX_AC]->array[smp_processor_id()] = ptr; } - /* 5) Replace the bootstrap kmem_list3's */ + /* 5) Replace the bootstrap kmem_cache_node */ { int nid; for_each_online_node(nid) { - init_list(kmem_cache, &initkmem_list3[CACHE_CACHE + nid], nid); + init_list(kmem_cache, &init_kmem_cache_node[CACHE_CACHE + nid], nid); - init_list(malloc_sizes[INDEX_AC].cs_cachep, - &initkmem_list3[SIZE_AC + nid], nid); + init_list(kmalloc_caches[INDEX_AC], + &init_kmem_cache_node[SIZE_AC + nid], nid); - if (INDEX_AC != INDEX_L3) { - init_list(malloc_sizes[INDEX_L3].cs_cachep, - &initkmem_list3[SIZE_L3 + nid], nid); + if (INDEX_AC != INDEX_NODE) { + init_list(kmalloc_caches[INDEX_NODE], + &init_kmem_cache_node[SIZE_NODE + nid], nid); } } } - slab_state = UP; + create_kmalloc_caches(ARCH_KMALLOC_FLAGS); } void __init kmem_cache_init_late(void) @@ -1773,7 +1650,7 @@ void __init kmem_cache_init_late(void) #ifdef CONFIG_NUMA /* * Register a memory hotplug callback that initializes and frees - * nodelists. + * node. */ hotplug_memory_notifier(slab_memory_callback, SLAB_CALLBACK_PRI); #endif @@ -1803,7 +1680,7 @@ __initcall(cpucache_init); static noinline void slab_out_of_memory(struct kmem_cache *cachep, gfp_t gfpflags, int nodeid) { - struct kmem_list3 *l3; + struct kmem_cache_node *n; struct slab *slabp; unsigned long flags; int node; @@ -1818,24 +1695,24 @@ slab_out_of_memory(struct kmem_cache *cachep, gfp_t gfpflags, int nodeid) unsigned long active_objs = 0, num_objs = 0, free_objects = 0; unsigned long active_slabs = 0, num_slabs = 0; - l3 = cachep->nodelists[node]; - if (!l3) + n = cachep->node[node]; + if (!n) continue; - spin_lock_irqsave(&l3->list_lock, flags); - list_for_each_entry(slabp, &l3->slabs_full, list) { + spin_lock_irqsave(&n->list_lock, flags); + list_for_each_entry(slabp, &n->slabs_full, list) { active_objs += cachep->num; active_slabs++; } - list_for_each_entry(slabp, &l3->slabs_partial, list) { + list_for_each_entry(slabp, &n->slabs_partial, list) { active_objs += slabp->inuse; active_slabs++; } - list_for_each_entry(slabp, &l3->slabs_free, list) + list_for_each_entry(slabp, &n->slabs_free, list) num_slabs++; - free_objects += l3->free_objects; - spin_unlock_irqrestore(&l3->list_lock, flags); + free_objects += n->free_objects; + spin_unlock_irqrestore(&n->list_lock, flags); num_slabs += active_slabs; num_objs = num_slabs * cachep->num; @@ -2260,7 +2137,7 @@ static int __init_refok setup_cpu_cache(struct kmem_cache *cachep, gfp_t gfp) if (slab_state == DOWN) { /* * Note: Creation of first cache (kmem_cache). - * The setup_list3s is taken care + * The setup_node is taken care * of by the caller of __kmem_cache_create */ cachep->array[smp_processor_id()] = &initarray_generic.cache; @@ -2274,13 +2151,13 @@ static int __init_refok setup_cpu_cache(struct kmem_cache *cachep, gfp_t gfp) cachep->array[smp_processor_id()] = &initarray_generic.cache; /* - * If the cache that's used by kmalloc(sizeof(kmem_list3)) is - * the second cache, then we need to set up all its list3s, + * If the cache that's used by kmalloc(sizeof(kmem_cache_node)) is + * the second cache, then we need to set up all its node/, * otherwise the creation of further caches will BUG(). */ - set_up_list3s(cachep, SIZE_AC); - if (INDEX_AC == INDEX_L3) - slab_state = PARTIAL_L3; + set_up_node(cachep, SIZE_AC); + if (INDEX_AC == INDEX_NODE) + slab_state = PARTIAL_NODE; else slab_state = PARTIAL_ARRAYCACHE; } else { @@ -2289,20 +2166,20 @@ static int __init_refok setup_cpu_cache(struct kmem_cache *cachep, gfp_t gfp) kmalloc(sizeof(struct arraycache_init), gfp); if (slab_state == PARTIAL_ARRAYCACHE) { - set_up_list3s(cachep, SIZE_L3); - slab_state = PARTIAL_L3; + set_up_node(cachep, SIZE_NODE); + slab_state = PARTIAL_NODE; } else { int node; for_each_online_node(node) { - cachep->nodelists[node] = - kmalloc_node(sizeof(struct kmem_list3), + cachep->node[node] = + kmalloc_node(sizeof(struct kmem_cache_node), gfp, node); - BUG_ON(!cachep->nodelists[node]); - kmem_list3_init(cachep->nodelists[node]); + BUG_ON(!cachep->node[node]); + kmem_cache_node_init(cachep->node[node]); } } } - cachep->nodelists[numa_mem_id()]->next_reap = + cachep->node[numa_mem_id()]->next_reap = jiffies + REAPTIMEOUT_LIST3 + ((unsigned long)cachep) % REAPTIMEOUT_LIST3; @@ -2405,7 +2282,7 @@ __kmem_cache_create (struct kmem_cache *cachep, unsigned long flags) else gfp = GFP_NOWAIT; - setup_nodelists_pointer(cachep); + setup_node_pointer(cachep); #if DEBUG /* @@ -2428,7 +2305,7 @@ __kmem_cache_create (struct kmem_cache *cachep, unsigned long flags) size += BYTES_PER_WORD; } #if FORCED_DEBUG && defined(CONFIG_DEBUG_PAGEALLOC) - if (size >= malloc_sizes[INDEX_L3 + 1].cs_size + if (size >= kmalloc_size(INDEX_NODE + 1) && cachep->object_size > cache_line_size() && ALIGN(size, cachep->align) < PAGE_SIZE) { cachep->obj_offset += PAGE_SIZE - ALIGN(size, cachep->align); @@ -2499,7 +2376,7 @@ __kmem_cache_create (struct kmem_cache *cachep, unsigned long flags) cachep->reciprocal_buffer_size = reciprocal_value(size); if (flags & CFLGS_OFF_SLAB) { - cachep->slabp_cache = kmem_find_general_cachep(slab_size, 0u); + cachep->slabp_cache = kmalloc_slab(slab_size, 0u); /* * This is a possibility for one of the malloc_sizes caches. * But since we go off slab only for object size greater than @@ -2545,7 +2422,7 @@ static void check_spinlock_acquired(struct kmem_cache *cachep) { #ifdef CONFIG_SMP check_irq_off(); - assert_spin_locked(&cachep->nodelists[numa_mem_id()]->list_lock); + assert_spin_locked(&cachep->node[numa_mem_id()]->list_lock); #endif } @@ -2553,7 +2430,7 @@ static void check_spinlock_acquired_node(struct kmem_cache *cachep, int node) { #ifdef CONFIG_SMP check_irq_off(); - assert_spin_locked(&cachep->nodelists[node]->list_lock); + assert_spin_locked(&cachep->node[node]->list_lock); #endif } @@ -2564,7 +2441,7 @@ static void check_spinlock_acquired_node(struct kmem_cache *cachep, int node) #define check_spinlock_acquired_node(x, y) do { } while(0) #endif -static void drain_array(struct kmem_cache *cachep, struct kmem_list3 *l3, +static void drain_array(struct kmem_cache *cachep, struct kmem_cache_node *n, struct array_cache *ac, int force, int node); @@ -2576,29 +2453,29 @@ static void do_drain(void *arg) check_irq_off(); ac = cpu_cache_get(cachep); - spin_lock(&cachep->nodelists[node]->list_lock); + spin_lock(&cachep->node[node]->list_lock); free_block(cachep, ac->entry, ac->avail, node); - spin_unlock(&cachep->nodelists[node]->list_lock); + spin_unlock(&cachep->node[node]->list_lock); ac->avail = 0; } static void drain_cpu_caches(struct kmem_cache *cachep) { - struct kmem_list3 *l3; + struct kmem_cache_node *n; int node; on_each_cpu(do_drain, cachep, 1); check_irq_on(); for_each_online_node(node) { - l3 = cachep->nodelists[node]; - if (l3 && l3->alien) - drain_alien_cache(cachep, l3->alien); + n = cachep->node[node]; + if (n && n->alien) + drain_alien_cache(cachep, n->alien); } for_each_online_node(node) { - l3 = cachep->nodelists[node]; - if (l3) - drain_array(cachep, l3, l3->shared, 1, node); + n = cachep->node[node]; + if (n) + drain_array(cachep, n, n->shared, 1, node); } } @@ -2609,19 +2486,19 @@ static void drain_cpu_caches(struct kmem_cache *cachep) * Returns the actual number of slabs released. */ static int drain_freelist(struct kmem_cache *cache, - struct kmem_list3 *l3, int tofree) + struct kmem_cache_node *n, int tofree) { struct list_head *p; int nr_freed; struct slab *slabp; nr_freed = 0; - while (nr_freed < tofree && !list_empty(&l3->slabs_free)) { + while (nr_freed < tofree && !list_empty(&n->slabs_free)) { - spin_lock_irq(&l3->list_lock); - p = l3->slabs_free.prev; - if (p == &l3->slabs_free) { - spin_unlock_irq(&l3->list_lock); + spin_lock_irq(&n->list_lock); + p = n->slabs_free.prev; + if (p == &n->slabs_free) { + spin_unlock_irq(&n->list_lock); goto out; } @@ -2634,8 +2511,8 @@ static int drain_freelist(struct kmem_cache *cache, * Safe to drop the lock. The slab is no longer linked * to the cache. */ - l3->free_objects -= cache->num; - spin_unlock_irq(&l3->list_lock); + n->free_objects -= cache->num; + spin_unlock_irq(&n->list_lock); slab_destroy(cache, slabp); nr_freed++; } @@ -2647,20 +2524,20 @@ out: static int __cache_shrink(struct kmem_cache *cachep) { int ret = 0, i = 0; - struct kmem_list3 *l3; + struct kmem_cache_node *n; drain_cpu_caches(cachep); check_irq_on(); for_each_online_node(i) { - l3 = cachep->nodelists[i]; - if (!l3) + n = cachep->node[i]; + if (!n |