mirror of
https://github.com/followmsi/android_kernel_google_msm.git
synced 2024-11-06 23:17:41 +00:00
[PATCH] NUMA slab locking fixes: irq disabling from cahep->spinlock to l3 lock
Earlier, we had to disable on chip interrupts while taking the cachep->spinlock because, at cache_grow, on every addition of a slab to a slab cache, we incremented colour_next which was protected by the cachep->spinlock, and cache_grow could occur at interrupt context. Since, now we protect the per-node colour_next with the node's list_lock, we do not need to disable on chip interrupts while taking the per-cache spinlock, but we just need to disable interrupts when taking the per-node kmem_list3 list_lock. Signed-off-by: Alok N Kataria <alokk@calsoftinc.com> Signed-off-by: Ravikiran Thirumalai <kiran@scalex86.org> Signed-off-by: Shai Fultheim <shai@scalex86.org> Cc: Christoph Lameter <christoph@lameter.com> Signed-off-by: Andrew Morton <akpm@osdl.org> Signed-off-by: Linus Torvalds <torvalds@osdl.org>
This commit is contained in:
parent
2e1217cf96
commit
ca3b9b9173
1 changed files with 18 additions and 18 deletions
36
mm/slab.c
36
mm/slab.c
|
@ -987,7 +987,7 @@ static int __devinit cpuup_callback(struct notifier_block *nfb,
|
||||||
cpumask_t mask;
|
cpumask_t mask;
|
||||||
|
|
||||||
mask = node_to_cpumask(node);
|
mask = node_to_cpumask(node);
|
||||||
spin_lock_irq(&cachep->spinlock);
|
spin_lock(&cachep->spinlock);
|
||||||
/* cpu is dead; no one can alloc from it. */
|
/* cpu is dead; no one can alloc from it. */
|
||||||
nc = cachep->array[cpu];
|
nc = cachep->array[cpu];
|
||||||
cachep->array[cpu] = NULL;
|
cachep->array[cpu] = NULL;
|
||||||
|
@ -996,7 +996,7 @@ static int __devinit cpuup_callback(struct notifier_block *nfb,
|
||||||
if (!l3)
|
if (!l3)
|
||||||
goto unlock_cache;
|
goto unlock_cache;
|
||||||
|
|
||||||
spin_lock(&l3->list_lock);
|
spin_lock_irq(&l3->list_lock);
|
||||||
|
|
||||||
/* Free limit for this kmem_list3 */
|
/* Free limit for this kmem_list3 */
|
||||||
l3->free_limit -= cachep->batchcount;
|
l3->free_limit -= cachep->batchcount;
|
||||||
|
@ -1004,7 +1004,7 @@ static int __devinit cpuup_callback(struct notifier_block *nfb,
|
||||||
free_block(cachep, nc->entry, nc->avail, node);
|
free_block(cachep, nc->entry, nc->avail, node);
|
||||||
|
|
||||||
if (!cpus_empty(mask)) {
|
if (!cpus_empty(mask)) {
|
||||||
spin_unlock(&l3->list_lock);
|
spin_unlock_irq(&l3->list_lock);
|
||||||
goto unlock_cache;
|
goto unlock_cache;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -1023,13 +1023,13 @@ static int __devinit cpuup_callback(struct notifier_block *nfb,
|
||||||
/* free slabs belonging to this node */
|
/* free slabs belonging to this node */
|
||||||
if (__node_shrink(cachep, node)) {
|
if (__node_shrink(cachep, node)) {
|
||||||
cachep->nodelists[node] = NULL;
|
cachep->nodelists[node] = NULL;
|
||||||
spin_unlock(&l3->list_lock);
|
spin_unlock_irq(&l3->list_lock);
|
||||||
kfree(l3);
|
kfree(l3);
|
||||||
} else {
|
} else {
|
||||||
spin_unlock(&l3->list_lock);
|
spin_unlock_irq(&l3->list_lock);
|
||||||
}
|
}
|
||||||
unlock_cache:
|
unlock_cache:
|
||||||
spin_unlock_irq(&cachep->spinlock);
|
spin_unlock(&cachep->spinlock);
|
||||||
kfree(nc);
|
kfree(nc);
|
||||||
}
|
}
|
||||||
mutex_unlock(&cache_chain_mutex);
|
mutex_unlock(&cache_chain_mutex);
|
||||||
|
@ -2011,18 +2011,18 @@ static void drain_cpu_caches(struct kmem_cache *cachep)
|
||||||
|
|
||||||
smp_call_function_all_cpus(do_drain, cachep);
|
smp_call_function_all_cpus(do_drain, cachep);
|
||||||
check_irq_on();
|
check_irq_on();
|
||||||
spin_lock_irq(&cachep->spinlock);
|
spin_lock(&cachep->spinlock);
|
||||||
for_each_online_node(node) {
|
for_each_online_node(node) {
|
||||||
l3 = cachep->nodelists[node];
|
l3 = cachep->nodelists[node];
|
||||||
if (l3) {
|
if (l3) {
|
||||||
spin_lock(&l3->list_lock);
|
spin_lock_irq(&l3->list_lock);
|
||||||
drain_array_locked(cachep, l3->shared, 1, node);
|
drain_array_locked(cachep, l3->shared, 1, node);
|
||||||
spin_unlock(&l3->list_lock);
|
spin_unlock_irq(&l3->list_lock);
|
||||||
if (l3->alien)
|
if (l3->alien)
|
||||||
drain_alien_cache(cachep, l3);
|
drain_alien_cache(cachep, l3);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
spin_unlock_irq(&cachep->spinlock);
|
spin_unlock(&cachep->spinlock);
|
||||||
}
|
}
|
||||||
|
|
||||||
static int __node_shrink(struct kmem_cache *cachep, int node)
|
static int __node_shrink(struct kmem_cache *cachep, int node)
|
||||||
|
@ -2338,7 +2338,6 @@ static int cache_grow(struct kmem_cache *cachep, gfp_t flags, int nodeid)
|
||||||
|
|
||||||
offset *= cachep->colour_off;
|
offset *= cachep->colour_off;
|
||||||
|
|
||||||
check_irq_off();
|
|
||||||
if (local_flags & __GFP_WAIT)
|
if (local_flags & __GFP_WAIT)
|
||||||
local_irq_enable();
|
local_irq_enable();
|
||||||
|
|
||||||
|
@ -2725,6 +2724,7 @@ static void *__cache_alloc_node(struct kmem_cache *cachep, gfp_t flags, int node
|
||||||
BUG_ON(!l3);
|
BUG_ON(!l3);
|
||||||
|
|
||||||
retry:
|
retry:
|
||||||
|
check_irq_off();
|
||||||
spin_lock(&l3->list_lock);
|
spin_lock(&l3->list_lock);
|
||||||
entry = l3->slabs_partial.next;
|
entry = l3->slabs_partial.next;
|
||||||
if (entry == &l3->slabs_partial) {
|
if (entry == &l3->slabs_partial) {
|
||||||
|
@ -3304,11 +3304,11 @@ static int do_tune_cpucache(struct kmem_cache *cachep, int limit, int batchcount
|
||||||
smp_call_function_all_cpus(do_ccupdate_local, (void *)&new);
|
smp_call_function_all_cpus(do_ccupdate_local, (void *)&new);
|
||||||
|
|
||||||
check_irq_on();
|
check_irq_on();
|
||||||
spin_lock_irq(&cachep->spinlock);
|
spin_lock(&cachep->spinlock);
|
||||||
cachep->batchcount = batchcount;
|
cachep->batchcount = batchcount;
|
||||||
cachep->limit = limit;
|
cachep->limit = limit;
|
||||||
cachep->shared = shared;
|
cachep->shared = shared;
|
||||||
spin_unlock_irq(&cachep->spinlock);
|
spin_unlock(&cachep->spinlock);
|
||||||
|
|
||||||
for_each_online_cpu(i) {
|
for_each_online_cpu(i) {
|
||||||
struct array_cache *ccold = new.new[i];
|
struct array_cache *ccold = new.new[i];
|
||||||
|
@ -3564,8 +3564,7 @@ static int s_show(struct seq_file *m, void *p)
|
||||||
int node;
|
int node;
|
||||||
struct kmem_list3 *l3;
|
struct kmem_list3 *l3;
|
||||||
|
|
||||||
check_irq_on();
|
spin_lock(&cachep->spinlock);
|
||||||
spin_lock_irq(&cachep->spinlock);
|
|
||||||
active_objs = 0;
|
active_objs = 0;
|
||||||
num_slabs = 0;
|
num_slabs = 0;
|
||||||
for_each_online_node(node) {
|
for_each_online_node(node) {
|
||||||
|
@ -3573,7 +3572,8 @@ static int s_show(struct seq_file *m, void *p)
|
||||||
if (!l3)
|
if (!l3)
|
||||||
continue;
|
continue;
|
||||||
|
|
||||||
spin_lock(&l3->list_lock);
|
check_irq_on();
|
||||||
|
spin_lock_irq(&l3->list_lock);
|
||||||
|
|
||||||
list_for_each(q, &l3->slabs_full) {
|
list_for_each(q, &l3->slabs_full) {
|
||||||
slabp = list_entry(q, struct slab, list);
|
slabp = list_entry(q, struct slab, list);
|
||||||
|
@ -3600,7 +3600,7 @@ static int s_show(struct seq_file *m, void *p)
|
||||||
free_objects += l3->free_objects;
|
free_objects += l3->free_objects;
|
||||||
shared_avail += l3->shared->avail;
|
shared_avail += l3->shared->avail;
|
||||||
|
|
||||||
spin_unlock(&l3->list_lock);
|
spin_unlock_irq(&l3->list_lock);
|
||||||
}
|
}
|
||||||
num_slabs += active_slabs;
|
num_slabs += active_slabs;
|
||||||
num_objs = num_slabs * cachep->num;
|
num_objs = num_slabs * cachep->num;
|
||||||
|
@ -3644,7 +3644,7 @@ static int s_show(struct seq_file *m, void *p)
|
||||||
}
|
}
|
||||||
#endif
|
#endif
|
||||||
seq_putc(m, '\n');
|
seq_putc(m, '\n');
|
||||||
spin_unlock_irq(&cachep->spinlock);
|
spin_unlock(&cachep->spinlock);
|
||||||
return 0;
|
return 0;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
Loading…
Reference in a new issue