1
0
Fork 0
mirror of https://github.com/followmsi/android_kernel_google_msm.git synced 2024-11-06 23:17:41 +00:00

[PATCH] NUMA slab locking fixes: irq disabling from cahep->spinlock to l3 lock

Earlier, we had to disable on chip interrupts while taking the
cachep->spinlock because, at cache_grow, on every addition of a slab to a slab
cache, we incremented colour_next which was protected by the cachep->spinlock,
and cache_grow could occur at interrupt context.  Since, now we protect the
per-node colour_next with the node's list_lock, we do not need to disable on
chip interrupts while taking the per-cache spinlock, but we just need to
disable interrupts when taking the per-node kmem_list3 list_lock.

Signed-off-by: Alok N Kataria <alokk@calsoftinc.com>
Signed-off-by: Ravikiran Thirumalai <kiran@scalex86.org>
Signed-off-by: Shai Fultheim <shai@scalex86.org>
Cc: Christoph Lameter <christoph@lameter.com>
Signed-off-by: Andrew Morton <akpm@osdl.org>
Signed-off-by: Linus Torvalds <torvalds@osdl.org>
This commit is contained in:
Ravikiran G Thirumalai 2006-02-04 23:27:58 -08:00 committed by Linus Torvalds
parent 2e1217cf96
commit ca3b9b9173

View file

@ -987,7 +987,7 @@ static int __devinit cpuup_callback(struct notifier_block *nfb,
cpumask_t mask; cpumask_t mask;
mask = node_to_cpumask(node); mask = node_to_cpumask(node);
spin_lock_irq(&cachep->spinlock); spin_lock(&cachep->spinlock);
/* cpu is dead; no one can alloc from it. */ /* cpu is dead; no one can alloc from it. */
nc = cachep->array[cpu]; nc = cachep->array[cpu];
cachep->array[cpu] = NULL; cachep->array[cpu] = NULL;
@ -996,7 +996,7 @@ static int __devinit cpuup_callback(struct notifier_block *nfb,
if (!l3) if (!l3)
goto unlock_cache; goto unlock_cache;
spin_lock(&l3->list_lock); spin_lock_irq(&l3->list_lock);
/* Free limit for this kmem_list3 */ /* Free limit for this kmem_list3 */
l3->free_limit -= cachep->batchcount; l3->free_limit -= cachep->batchcount;
@ -1004,7 +1004,7 @@ static int __devinit cpuup_callback(struct notifier_block *nfb,
free_block(cachep, nc->entry, nc->avail, node); free_block(cachep, nc->entry, nc->avail, node);
if (!cpus_empty(mask)) { if (!cpus_empty(mask)) {
spin_unlock(&l3->list_lock); spin_unlock_irq(&l3->list_lock);
goto unlock_cache; goto unlock_cache;
} }
@ -1023,13 +1023,13 @@ static int __devinit cpuup_callback(struct notifier_block *nfb,
/* free slabs belonging to this node */ /* free slabs belonging to this node */
if (__node_shrink(cachep, node)) { if (__node_shrink(cachep, node)) {
cachep->nodelists[node] = NULL; cachep->nodelists[node] = NULL;
spin_unlock(&l3->list_lock); spin_unlock_irq(&l3->list_lock);
kfree(l3); kfree(l3);
} else { } else {
spin_unlock(&l3->list_lock); spin_unlock_irq(&l3->list_lock);
} }
unlock_cache: unlock_cache:
spin_unlock_irq(&cachep->spinlock); spin_unlock(&cachep->spinlock);
kfree(nc); kfree(nc);
} }
mutex_unlock(&cache_chain_mutex); mutex_unlock(&cache_chain_mutex);
@ -2011,18 +2011,18 @@ static void drain_cpu_caches(struct kmem_cache *cachep)
smp_call_function_all_cpus(do_drain, cachep); smp_call_function_all_cpus(do_drain, cachep);
check_irq_on(); check_irq_on();
spin_lock_irq(&cachep->spinlock); spin_lock(&cachep->spinlock);
for_each_online_node(node) { for_each_online_node(node) {
l3 = cachep->nodelists[node]; l3 = cachep->nodelists[node];
if (l3) { if (l3) {
spin_lock(&l3->list_lock); spin_lock_irq(&l3->list_lock);
drain_array_locked(cachep, l3->shared, 1, node); drain_array_locked(cachep, l3->shared, 1, node);
spin_unlock(&l3->list_lock); spin_unlock_irq(&l3->list_lock);
if (l3->alien) if (l3->alien)
drain_alien_cache(cachep, l3); drain_alien_cache(cachep, l3);
} }
} }
spin_unlock_irq(&cachep->spinlock); spin_unlock(&cachep->spinlock);
} }
static int __node_shrink(struct kmem_cache *cachep, int node) static int __node_shrink(struct kmem_cache *cachep, int node)
@ -2338,7 +2338,6 @@ static int cache_grow(struct kmem_cache *cachep, gfp_t flags, int nodeid)
offset *= cachep->colour_off; offset *= cachep->colour_off;
check_irq_off();
if (local_flags & __GFP_WAIT) if (local_flags & __GFP_WAIT)
local_irq_enable(); local_irq_enable();
@ -2725,6 +2724,7 @@ static void *__cache_alloc_node(struct kmem_cache *cachep, gfp_t flags, int node
BUG_ON(!l3); BUG_ON(!l3);
retry: retry:
check_irq_off();
spin_lock(&l3->list_lock); spin_lock(&l3->list_lock);
entry = l3->slabs_partial.next; entry = l3->slabs_partial.next;
if (entry == &l3->slabs_partial) { if (entry == &l3->slabs_partial) {
@ -3304,11 +3304,11 @@ static int do_tune_cpucache(struct kmem_cache *cachep, int limit, int batchcount
smp_call_function_all_cpus(do_ccupdate_local, (void *)&new); smp_call_function_all_cpus(do_ccupdate_local, (void *)&new);
check_irq_on(); check_irq_on();
spin_lock_irq(&cachep->spinlock); spin_lock(&cachep->spinlock);
cachep->batchcount = batchcount; cachep->batchcount = batchcount;
cachep->limit = limit; cachep->limit = limit;
cachep->shared = shared; cachep->shared = shared;
spin_unlock_irq(&cachep->spinlock); spin_unlock(&cachep->spinlock);
for_each_online_cpu(i) { for_each_online_cpu(i) {
struct array_cache *ccold = new.new[i]; struct array_cache *ccold = new.new[i];
@ -3564,8 +3564,7 @@ static int s_show(struct seq_file *m, void *p)
int node; int node;
struct kmem_list3 *l3; struct kmem_list3 *l3;
check_irq_on(); spin_lock(&cachep->spinlock);
spin_lock_irq(&cachep->spinlock);
active_objs = 0; active_objs = 0;
num_slabs = 0; num_slabs = 0;
for_each_online_node(node) { for_each_online_node(node) {
@ -3573,7 +3572,8 @@ static int s_show(struct seq_file *m, void *p)
if (!l3) if (!l3)
continue; continue;
spin_lock(&l3->list_lock); check_irq_on();
spin_lock_irq(&l3->list_lock);
list_for_each(q, &l3->slabs_full) { list_for_each(q, &l3->slabs_full) {
slabp = list_entry(q, struct slab, list); slabp = list_entry(q, struct slab, list);
@ -3600,7 +3600,7 @@ static int s_show(struct seq_file *m, void *p)
free_objects += l3->free_objects; free_objects += l3->free_objects;
shared_avail += l3->shared->avail; shared_avail += l3->shared->avail;
spin_unlock(&l3->list_lock); spin_unlock_irq(&l3->list_lock);
} }
num_slabs += active_slabs; num_slabs += active_slabs;
num_objs = num_slabs * cachep->num; num_objs = num_slabs * cachep->num;
@ -3644,7 +3644,7 @@ static int s_show(struct seq_file *m, void *p)
} }
#endif #endif
seq_putc(m, '\n'); seq_putc(m, '\n');
spin_unlock_irq(&cachep->spinlock); spin_unlock(&cachep->spinlock);
return 0; return 0;
} }