rcu: Add a module parameter to force use of expedited RCU primitives

There have been some embedded applications that would benefit from
use of expedited grace-period primitives.  In some ways, this is
similar to synchronize_net() doing either a normal or an expedited
grace period depending on lock state, but with control outside of
the kernel.

This commit therefore adds rcu_expedited boot and sysfs parameters
that cause the kernel to substitute expedited primitives for the
normal grace-period primitives.

[ paulmck: Add trace/event/rcu.h to kernel/srcu.c to avoid build error.
	   Get rid of infinite loop through contention path.]

Signed-off-by: Antti P Miettinen <amiettinen@nvidia.com>
Signed-off-by: Paul E. McKenney <paulmck@linux.vnet.ibm.com>
This commit is contained in:
Antti P Miettinen 2012-10-05 09:59:15 +03:00 committed by Paul E. McKenney
parent ba49df4767
commit 3705b88db0
7 changed files with 48 additions and 7 deletions

View File

@ -141,6 +141,23 @@ static ssize_t fscaps_show(struct kobject *kobj,
}
KERNEL_ATTR_RO(fscaps);
int rcu_expedited;
static ssize_t rcu_expedited_show(struct kobject *kobj,
struct kobj_attribute *attr, char *buf)
{
return sprintf(buf, "%d\n", rcu_expedited);
}
static ssize_t rcu_expedited_store(struct kobject *kobj,
struct kobj_attribute *attr,
const char *buf, size_t count)
{
if (kstrtoint(buf, 0, &rcu_expedited))
return -EINVAL;
return count;
}
KERNEL_ATTR_RW(rcu_expedited);
/*
* Make /sys/kernel/notes give the raw contents of our kernel .notes section.
*/
@ -182,6 +199,7 @@ static struct attribute * kernel_attrs[] = {
&kexec_crash_size_attr.attr,
&vmcoreinfo_attr.attr,
#endif
&rcu_expedited_attr.attr,
NULL
};

View File

@ -109,4 +109,6 @@ static inline bool __rcu_reclaim(char *rn, struct rcu_head *head)
}
}
extern int rcu_expedited;
#endif /* __LINUX_RCU_H */

View File

@ -46,12 +46,15 @@
#include <linux/export.h>
#include <linux/hardirq.h>
#include <linux/delay.h>
#include <linux/module.h>
#define CREATE_TRACE_POINTS
#include <trace/events/rcu.h>
#include "rcu.h"
module_param(rcu_expedited, int, 0);
#ifdef CONFIG_PREEMPT_RCU
/*

View File

@ -706,7 +706,10 @@ void synchronize_rcu(void)
return;
/* Once we get past the fastpath checks, same code as rcu_barrier(). */
rcu_barrier();
if (rcu_expedited)
synchronize_rcu_expedited();
else
rcu_barrier();
}
EXPORT_SYMBOL_GPL(synchronize_rcu);

View File

@ -2224,7 +2224,10 @@ void synchronize_sched(void)
"Illegal synchronize_sched() in RCU-sched read-side critical section");
if (rcu_blocking_is_gp())
return;
wait_rcu_gp(call_rcu_sched);
if (rcu_expedited)
synchronize_sched_expedited();
else
wait_rcu_gp(call_rcu_sched);
}
EXPORT_SYMBOL_GPL(synchronize_sched);
@ -2245,7 +2248,10 @@ void synchronize_rcu_bh(void)
"Illegal synchronize_rcu_bh() in RCU-bh read-side critical section");
if (rcu_blocking_is_gp())
return;
wait_rcu_gp(call_rcu_bh);
if (rcu_expedited)
synchronize_rcu_bh_expedited();
else
wait_rcu_gp(call_rcu_bh);
}
EXPORT_SYMBOL_GPL(synchronize_rcu_bh);
@ -2328,7 +2334,7 @@ void synchronize_sched_expedited(void)
if (trycount++ < 10) {
udelay(trycount * num_online_cpus());
} else {
synchronize_sched();
wait_rcu_gp(call_rcu_sched);
return;
}

View File

@ -679,7 +679,10 @@ void synchronize_rcu(void)
"Illegal synchronize_rcu() in RCU read-side critical section");
if (!rcu_scheduler_active)
return;
wait_rcu_gp(call_rcu);
if (rcu_expedited)
synchronize_rcu_expedited();
else
wait_rcu_gp(call_rcu);
}
EXPORT_SYMBOL_GPL(synchronize_rcu);
@ -831,7 +834,7 @@ void synchronize_rcu_expedited(void)
udelay(trycount * num_online_cpus());
} else {
put_online_cpus();
synchronize_rcu();
wait_rcu_gp(call_rcu);
return;
}
}

View File

@ -34,6 +34,10 @@
#include <linux/delay.h>
#include <linux/srcu.h>
#include <trace/events/rcu.h>
#include "rcu.h"
/*
* Initialize an rcu_batch structure to empty.
*/
@ -464,7 +468,9 @@ static void __synchronize_srcu(struct srcu_struct *sp, int trycount)
*/
void synchronize_srcu(struct srcu_struct *sp)
{
__synchronize_srcu(sp, SYNCHRONIZE_SRCU_TRYCOUNT);
__synchronize_srcu(sp, rcu_expedited
? SYNCHRONIZE_SRCU_EXP_TRYCOUNT
: SYNCHRONIZE_SRCU_TRYCOUNT);
}
EXPORT_SYMBOL_GPL(synchronize_srcu);