Piteball/android_kernel_google_msm
1
0
Fork 0
mirror of https://github.com/followmsi/android_kernel_google_msm.git synced 2024-09-20 19:51:42 +00:00
Code Issues Projects Releases Wiki Activity

rcu: Move RCU_FAST_NO_HZ per-CPU variables to rcu_dynticks structure

Browse source 
The RCU_FAST_NO_HZ code relies on a number of per-CPU variables.
This works, but is hidden from someone scanning the data structures
in rcutree.h.  This commit therefore converts these per-CPU variables
to fields in the per-CPU rcu_dynticks structures.

Suggested-by: Peter Zijlstra <peterz@infradead.org>
Signed-off-by: Paul E. McKenney <paul.mckenney@linaro.org>
Signed-off-by: Paul E. McKenney <paulmck@linux.vnet.ibm.com>
Tested-by: Heiko Carstens <heiko.carstens@de.ibm.com>
Tested-by: Pascal Chapperon <pascal.chapperon@wanadoo.fr>
This commit is contained in:
Paul E. McKenney 2012-05-09 12:07:05 -07:00 committed by Paul E. McKenney
parent fd4b352687
commit 5955f7eecd
2 changed files with 58 additions and 55 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

14
kernel/rcutree.h
View file

@ -84,6 +84,20 @@ struct rcu_dynticks {
/* Process level is worth LLONG_MAX/2. */ /* Process level is worth LLONG_MAX/2. */
int dynticks_nmi_nesting; /* Track NMI nesting level. */ int dynticks_nmi_nesting; /* Track NMI nesting level. */
atomic_t dynticks; /* Even value for idle, else odd. */ atomic_t dynticks; /* Even value for idle, else odd. */
#ifdef CONFIG_RCU_FAST_NO_HZ
int dyntick_drain; /* Prepare-for-idle state variable. */
unsigned long dyntick_holdoff;
/* No retries for the jiffy of failure. */
struct timer_list idle_gp_timer;
/* Wake up CPU sleeping with callbacks. */
unsigned long idle_gp_timer_expires;
/* When to wake up CPU (for repost). */
bool idle_first_pass; /* First pass of attempt to go idle? */
unsigned long nonlazy_posted;
/* # times non-lazy CBs posted to CPU. */
unsigned long nonlazy_posted_snap;
/* idle-period nonlazy_posted snapshot. */
#endif /* #ifdef CONFIG_RCU_FAST_NO_HZ */
}; };
/* RCU's kthread states for tracing. */ /* RCU's kthread states for tracing. */

99
kernel/rcutree_plugin.h
View file

@ -1962,21 +1962,6 @@ static void rcu_idle_count_callbacks_posted(void)
#define RCU_IDLE_GP_DELAY 6 /* Roughly one grace period. */ #define RCU_IDLE_GP_DELAY 6 /* Roughly one grace period. */
#define RCU_IDLE_LAZY_GP_DELAY (6 * HZ) /* Roughly six seconds. */ #define RCU_IDLE_LAZY_GP_DELAY (6 * HZ) /* Roughly six seconds. */
/* Loop counter for rcu_prepare_for_idle(). */
static DEFINE_PER_CPU(int, rcu_dyntick_drain);
/* If rcu_dyntick_holdoff==jiffies, don't try to enter dyntick-idle mode. */
static DEFINE_PER_CPU(unsigned long, rcu_dyntick_holdoff);
/* Timer to awaken the CPU if it enters dyntick-idle mode with callbacks. */
static DEFINE_PER_CPU(struct timer_list, rcu_idle_gp_timer);
/* Scheduled expiry time for rcu_idle_gp_timer to allow reposting. */
static DEFINE_PER_CPU(unsigned long, rcu_idle_gp_timer_expires);
/* Enable special processing on first attempt to enter dyntick-idle mode. */
static DEFINE_PER_CPU(bool, rcu_idle_first_pass);
/* Running count of non-lazy callbacks posted, never decremented. */
static DEFINE_PER_CPU(unsigned long, rcu_nonlazy_posted);
/* Snapshot of rcu_nonlazy_posted to detect meaningful exits from idle. */
static DEFINE_PER_CPU(unsigned long, rcu_nonlazy_posted_snap);
/* /*
* Allow the CPU to enter dyntick-idle mode if either: (1) There are no * Allow the CPU to enter dyntick-idle mode if either: (1) There are no
* callbacks on this CPU, (2) this CPU has not yet attempted to enter * callbacks on this CPU, (2) this CPU has not yet attempted to enter
@ -1988,13 +1973,15 @@ static DEFINE_PER_CPU(unsigned long, rcu_nonlazy_posted_snap);
*/ */
int rcu_needs_cpu(int cpu) int rcu_needs_cpu(int cpu)
{ {
struct rcu_dynticks *rdtp = &per_cpu(rcu_dynticks, cpu);
/* Flag a new idle sojourn to the idle-entry state machine. */ /* Flag a new idle sojourn to the idle-entry state machine. */
per_cpu(rcu_idle_first_pass, cpu) = 1; rdtp->idle_first_pass = 1;
/* If no callbacks, RCU doesn't need the CPU. */ /* If no callbacks, RCU doesn't need the CPU. */
if (!rcu_cpu_has_callbacks(cpu)) if (!rcu_cpu_has_callbacks(cpu))
return 0; return 0;
/* Otherwise, RCU needs the CPU only if it recently tried and failed. */ /* Otherwise, RCU needs the CPU only if it recently tried and failed. */
return per_cpu(rcu_dyntick_holdoff, cpu) == jiffies; return rdtp->dyntick_holdoff == jiffies;
} }
/* /*
@ -2075,21 +2062,24 @@ static void rcu_idle_gp_timer_func(unsigned long cpu_in)
*/ */
static void rcu_prepare_for_idle_init(int cpu) static void rcu_prepare_for_idle_init(int cpu)
{ {
per_cpu(rcu_dyntick_holdoff, cpu) = jiffies - 1; struct rcu_dynticks *rdtp = &per_cpu(rcu_dynticks, cpu);
setup_timer(&per_cpu(rcu_idle_gp_timer, cpu),
rcu_idle_gp_timer_func, cpu); rdtp->dyntick_holdoff = jiffies - 1;
per_cpu(rcu_idle_gp_timer_expires, cpu) = jiffies - 1; setup_timer(&rdtp->idle_gp_timer, rcu_idle_gp_timer_func, cpu);
per_cpu(rcu_idle_first_pass, cpu) = 1; rdtp->idle_gp_timer_expires = jiffies - 1;
rdtp->idle_first_pass = 1;
} }
/* /*
* Clean up for exit from idle. Because we are exiting from idle, there * Clean up for exit from idle. Because we are exiting from idle, there
* is no longer any point to rcu_idle_gp_timer, so cancel it. This will * is no longer any point to ->idle_gp_timer, so cancel it. This will
* do nothing if this timer is not active, so just cancel it unconditionally. * do nothing if this timer is not active, so just cancel it unconditionally.
*/ */
static void rcu_cleanup_after_idle(int cpu) static void rcu_cleanup_after_idle(int cpu)
{ {
del_timer(&per_cpu(rcu_idle_gp_timer, cpu)); struct rcu_dynticks *rdtp = &per_cpu(rcu_dynticks, cpu);
del_timer(&rdtp->idle_gp_timer);
trace_rcu_prep_idle("Cleanup after idle"); trace_rcu_prep_idle("Cleanup after idle");
} }
@ -2108,42 +2098,41 @@ static void rcu_cleanup_after_idle(int cpu)
* Because it is not legal to invoke rcu_process_callbacks() with irqs * Because it is not legal to invoke rcu_process_callbacks() with irqs
* disabled, we do one pass of force_quiescent_state(), then do a * disabled, we do one pass of force_quiescent_state(), then do a
* invoke_rcu_core() to cause rcu_process_callbacks() to be invoked * invoke_rcu_core() to cause rcu_process_callbacks() to be invoked
* later. The per-cpu rcu_dyntick_drain variable controls the sequencing. * later. The ->dyntick_drain field controls the sequencing.
* *
* The caller must have disabled interrupts. * The caller must have disabled interrupts.
*/ */
static void rcu_prepare_for_idle(int cpu) static void rcu_prepare_for_idle(int cpu)
{ {
struct timer_list *tp; struct timer_list *tp;
struct rcu_dynticks *rdtp = &per_cpu(rcu_dynticks, cpu);
/* /*
* If this is an idle re-entry, for example, due to use of * If this is an idle re-entry, for example, due to use of
* RCU_NONIDLE() or the new idle-loop tracing API within the idle * RCU_NONIDLE() or the new idle-loop tracing API within the idle
* loop, then don't take any state-machine actions, unless the * loop, then don't take any state-machine actions, unless the
* momentary exit from idle queued additional non-lazy callbacks. * momentary exit from idle queued additional non-lazy callbacks.
* Instead, repost the rcu_idle_gp_timer if this CPU has callbacks * Instead, repost the ->idle_gp_timer if this CPU has callbacks
* pending. * pending.
*/ */
if (!per_cpu(rcu_idle_first_pass, cpu) && if (!rdtp->idle_first_pass &&
(per_cpu(rcu_nonlazy_posted, cpu) == (rdtp->nonlazy_posted == rdtp->nonlazy_posted_snap)) {
per_cpu(rcu_nonlazy_posted_snap, cpu))) {
if (rcu_cpu_has_callbacks(cpu)) { if (rcu_cpu_has_callbacks(cpu)) {
tp = &per_cpu(rcu_idle_gp_timer, cpu); tp = &rdtp->idle_gp_timer;
mod_timer_pinned(tp, per_cpu(rcu_idle_gp_timer_expires, cpu)); mod_timer_pinned(tp, rdtp->idle_gp_timer_expires);
} }
return; return;
} }
per_cpu(rcu_idle_first_pass, cpu) = 0; rdtp->idle_first_pass = 0;
per_cpu(rcu_nonlazy_posted_snap, cpu) = rdtp->nonlazy_posted_snap = rdtp->nonlazy_posted - 1;
per_cpu(rcu_nonlazy_posted, cpu) - 1;
/* /*
* If there are no callbacks on this CPU, enter dyntick-idle mode. * If there are no callbacks on this CPU, enter dyntick-idle mode.
* Also reset state to avoid prejudicing later attempts. * Also reset state to avoid prejudicing later attempts.
*/ */
if (!rcu_cpu_has_callbacks(cpu)) { if (!rcu_cpu_has_callbacks(cpu)) {
per_cpu(rcu_dyntick_holdoff, cpu) = jiffies - 1; rdtp->dyntick_holdoff = jiffies - 1;
per_cpu(rcu_dyntick_drain, cpu) = 0; rdtp->dyntick_drain = 0;
trace_rcu_prep_idle("No callbacks"); trace_rcu_prep_idle("No callbacks");
return; return;
} }
@ -2152,38 +2141,37 @@ static void rcu_prepare_for_idle(int cpu)
* If in holdoff mode, just return. We will presumably have * If in holdoff mode, just return. We will presumably have
* refrained from disabling the scheduling-clock tick. * refrained from disabling the scheduling-clock tick.
*/ */
if (per_cpu(rcu_dyntick_holdoff, cpu) == jiffies) { if (rdtp->dyntick_holdoff == jiffies) {
trace_rcu_prep_idle("In holdoff"); trace_rcu_prep_idle("In holdoff");
return; return;
} }
/* Check and update the rcu_dyntick_drain sequencing. */ /* Check and update the ->dyntick_drain sequencing. */
if (per_cpu(rcu_dyntick_drain, cpu) <= 0) { if (rdtp->dyntick_drain <= 0) {
/* First time through, initialize the counter. */ /* First time through, initialize the counter. */
per_cpu(rcu_dyntick_drain, cpu) = RCU_IDLE_FLUSHES; rdtp->dyntick_drain = RCU_IDLE_FLUSHES;
} else if (per_cpu(rcu_dyntick_drain, cpu) <= RCU_IDLE_OPT_FLUSHES && } else if (rdtp->dyntick_drain <= RCU_IDLE_OPT_FLUSHES &&
!rcu_pending(cpu) && !rcu_pending(cpu) &&
!local_softirq_pending()) { !local_softirq_pending()) {
/* Can we go dyntick-idle despite still having callbacks? */ /* Can we go dyntick-idle despite still having callbacks? */
per_cpu(rcu_dyntick_drain, cpu) = 0; rdtp->dyntick_drain = 0;
per_cpu(rcu_dyntick_holdoff, cpu) = jiffies; rdtp->dyntick_holdoff = jiffies;
if (rcu_cpu_has_nonlazy_callbacks(cpu)) { if (rcu_cpu_has_nonlazy_callbacks(cpu)) {
trace_rcu_prep_idle("Dyntick with callbacks"); trace_rcu_prep_idle("Dyntick with callbacks");
per_cpu(rcu_idle_gp_timer_expires, cpu) = rdtp->idle_gp_timer_expires =
jiffies + RCU_IDLE_GP_DELAY; jiffies + RCU_IDLE_GP_DELAY;
} else { } else {
per_cpu(rcu_idle_gp_timer_expires, cpu) = rdtp->idle_gp_timer_expires =
jiffies + RCU_IDLE_LAZY_GP_DELAY; jiffies + RCU_IDLE_LAZY_GP_DELAY;
trace_rcu_prep_idle("Dyntick with lazy callbacks"); trace_rcu_prep_idle("Dyntick with lazy callbacks");
} }
tp = &per_cpu(rcu_idle_gp_timer, cpu); tp = &rdtp->idle_gp_timer;
mod_timer_pinned(tp, per_cpu(rcu_idle_gp_timer_expires, cpu)); mod_timer_pinned(tp, rdtp->idle_gp_timer_expires);
per_cpu(rcu_nonlazy_posted_snap, cpu) = rdtp->nonlazy_posted_snap = rdtp->nonlazy_posted;
per_cpu(rcu_nonlazy_posted, cpu);
return; /* Nothing more to do immediately. */ return; /* Nothing more to do immediately. */
} else if (--per_cpu(rcu_dyntick_drain, cpu) <= 0) { } else if (--(rdtp->dyntick_drain) <= 0) {
/* We have hit the limit, so time to give up. */ /* We have hit the limit, so time to give up. */
per_cpu(rcu_dyntick_holdoff, cpu) = jiffies; rdtp->dyntick_holdoff = jiffies;
trace_rcu_prep_idle("Begin holdoff"); trace_rcu_prep_idle("Begin holdoff");
invoke_rcu_core(); /* Force the CPU out of dyntick-idle. */ invoke_rcu_core(); /* Force the CPU out of dyntick-idle. */
return; return;
@ -2229,7 +2217,7 @@ static void rcu_prepare_for_idle(int cpu)
*/ */
static void rcu_idle_count_callbacks_posted(void) static void rcu_idle_count_callbacks_posted(void)
{ {
__this_cpu_add(rcu_nonlazy_posted, 1); __this_cpu_add(rcu_dynticks.nonlazy_posted, 1);
} }
#endif /* #else #if !defined(CONFIG_RCU_FAST_NO_HZ) */ #endif /* #else #if !defined(CONFIG_RCU_FAST_NO_HZ) */
@ -2240,11 +2228,12 @@ static void rcu_idle_count_callbacks_posted(void)
static void print_cpu_stall_fast_no_hz(char *cp, int cpu) static void print_cpu_stall_fast_no_hz(char *cp, int cpu)
{ {
struct timer_list *tltp = &per_cpu(rcu_idle_gp_timer, cpu); struct rcu_dynticks *rdtp = &per_cpu(rcu_dynticks, cpu);
struct timer_list *tltp = &rdtp->idle_gp_timer;
sprintf(cp, "drain=%d %c timer=%lu", sprintf(cp, "drain=%d %c timer=%lu",
per_cpu(rcu_dyntick_drain, cpu), rdtp->dyntick_drain,
per_cpu(rcu_dyntick_holdoff, cpu) == jiffies ? 'H' : '.', rdtp->dyntick_holdoff == jiffies ? 'H' : '.',
timer_pending(tltp) ? tltp->expires - jiffies : -1); timer_pending(tltp) ? tltp->expires - jiffies : -1);
} }