Piteball
/
android_kernel_samsung_msm8976
mirror of https://github.com/team-infusion-developers/android_kernel_samsung_msm8976.git
1
1
Fork 0
Code Issues Projects Releases Wiki Activity

sched: Use an accessor to read the rq clock 

Read the runqueue clock through an accessor. This
prepares for adding a debugging infrastructure to
detect missing or redundant calls to update_rq_clock()
between a scheduler's entry and exit point.

Signed-off-by: Frederic Weisbecker <fweisbec@gmail.com>
Cc: Li Zhong <zhong@linux.vnet.ibm.com>
Cc: Steven Rostedt <rostedt@goodmis.org>
Cc: Paul Turner <pjt@google.com>
Cc: Mike Galbraith <efault@gmx.de>
Signed-off-by: Peter Zijlstra <peterz@infradead.org>
Link: http://lkml.kernel.org/r/1365724262-20142-6-git-send-email-fweisbec@gmail.com
Signed-off-by: Ingo Molnar <mingo@kernel.org>
[rameezmustafa@codeaurora.org: Port to msm-3.10]
Signed-off-by: Syed Rameez Mustafa <rameezmustafa@codeaurora.org>
Git-Commit: 78becc27097585c6aec7043834cadde950ae79f2
Git-Repo: git://git.kernel.org/cgit/linux/kernel/git/torvalds/linux.git
This commit is contained in:
Frederic Weisbecker 2013-04-12 01:51:02 +02:00 committed by Syed Rameez Mustafa
parent b970ef27c6
commit dbb0220237
6 changed files with 48 additions and 39 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
kernel/sched/core.c
View File

@ -733,7 +733,7 @@ void sched_avg_update(struct rq *rq)
{
s64 period = sched_avg_period();
while ((s64)(rq->clock - rq->age_stamp) > period) {
while ((s64)(rq_clock(rq) - rq->age_stamp) > period) {
/*
* Inline assembly required to prevent the compiler
* optimising this loop into a divmod call.
@ -3027,7 +3027,7 @@ ttwu_do_wakeup(struct rq *rq, struct task_struct *p, int wake_flags)
p->sched_class->task_woken(rq, p);
if (rq->idle_stamp) {
u64 delta = rq->clock - rq->idle_stamp;
u64 delta = rq_clock(rq) - rq->idle_stamp;
u64 max = 2*sysctl_sched_migration_cost;
if (delta > max)
@ -4480,7 +4480,7 @@ static u64 do_task_delta_exec(struct task_struct *p, struct rq *rq)
if (task_current(rq, p)) {
update_rq_clock(rq);
ns = rq->clock_task - p->se.exec_start;
ns = rq_clock_task(rq) - p->se.exec_start;
if ((s64)ns < 0)
ns = 0;
}

47
kernel/sched/fair.c
View File

@ -711,7 +711,7 @@ __update_curr(struct cfs_rq *cfs_rq, struct sched_entity *curr,
static void update_curr(struct cfs_rq *cfs_rq)
{
struct sched_entity *curr = cfs_rq->curr;
u64 now = rq_of(cfs_rq)->clock_task;
u64 now = rq_clock_task(rq_of(cfs_rq));
unsigned long delta_exec;
if (unlikely(!curr))
@ -747,8 +747,8 @@ update_stats_wait_start(struct cfs_rq *cfs_rq, struct sched_entity *se,
schedstat_set(se->statistics.wait_start,
migrating &&
likely(rq_of(cfs_rq)->clock > se->statistics.wait_start) ?
rq_of(cfs_rq)->clock - se->statistics.wait_start :
rq_of(cfs_rq)->clock);
rq_clock(rq_of(cfs_rq)) - se->statistics.wait_start :
rq_clock(rq_of(cfs_rq)));
}
/*
@ -849,16 +849,16 @@ update_stats_wait_end(struct cfs_rq *cfs_rq, struct sched_entity *se,
}
schedstat_set(se->statistics.wait_max, max(se->statistics.wait_max,
rq_of(cfs_rq)->clock - se->statistics.wait_start));
rq_clock(rq_of(cfs_rq)) - se->statistics.wait_start));
schedstat_set(se->statistics.wait_count, se->statistics.wait_count + 1);
schedstat_set(se->statistics.wait_sum, se->statistics.wait_sum +
rq_of(cfs_rq)->clock - se->statistics.wait_start);
rq_clock(rq_of(cfs_rq)) - se->statistics.wait_start);
#ifdef CONFIG_SCHEDSTATS
if (entity_is_task(se)) {
u64 delta;
struct sched_max_latency *max;
delta = rq_of(cfs_rq)->clock - se->statistics.wait_start;
delta = rq_clock(rq_of(cfs_rq)) - se->statistics.wait_start;
trace_sched_stat_wait(task_of(se), delta);
delta = delta >> 10;
@ -896,7 +896,7 @@ update_stats_curr_start(struct cfs_rq *cfs_rq, struct sched_entity *se)
/*
* We are starting a new run period:
*/
se->exec_start = rq_of(cfs_rq)->clock_task;
se->exec_start = rq_clock_task(rq_of(cfs_rq));
}
/**************************************************
@ -3421,7 +3421,7 @@ static void update_cfs_rq_blocked_load(struct cfs_rq *cfs_rq, int force_update)
static inline void update_rq_runnable_avg(struct rq *rq, int runnable)
{
__update_entity_runnable_avg(cpu_of(rq), rq->clock_task,
__update_entity_runnable_avg(cpu_of(rq), rq_clock_task(rq),
&rq->avg, runnable);
__update_tg_runnable_avg(&rq->avg, &rq->cfs);
}
@ -3437,7 +3437,7 @@ static inline void enqueue_entity_load_avg(struct cfs_rq *cfs_rq,
* accumulated while sleeping.
*/
if (unlikely(se->avg.decay_count <= 0)) {
se->avg.last_runnable_update = rq_of(cfs_rq)->clock_task;
se->avg.last_runnable_update = rq_clock_task(rq_of(cfs_rq));
if (se->avg.decay_count) {
/*
* In a wake-up migration we have to approximate the
@ -3675,7 +3675,7 @@ static void enqueue_sleeper(struct cfs_rq *cfs_rq, struct sched_entity *se)
tsk = task_of(se);
if (se->statistics.sleep_start) {
u64 delta = rq_of(cfs_rq)->clock - se->statistics.sleep_start;
u64 delta = rq_clock(rq_of(cfs_rq)) - se->statistics.sleep_start;
if ((s64)delta < 0)
delta = 0;
@ -3692,7 +3692,7 @@ static void enqueue_sleeper(struct cfs_rq *cfs_rq, struct sched_entity *se)
}
}
if (se->statistics.block_start) {
u64 delta = rq_of(cfs_rq)->clock - se->statistics.block_start;
u64 delta = rq_clock(rq_of(cfs_rq)) - se->statistics.block_start;
if ((s64)delta < 0)
delta = 0;
@ -3873,9 +3873,9 @@ dequeue_entity(struct cfs_rq *cfs_rq, struct sched_entity *se, int flags)
struct task_struct *tsk = task_of(se);
if (tsk->state & TASK_INTERRUPTIBLE)
se->statistics.sleep_start = rq_of(cfs_rq)->clock;
se->statistics.sleep_start = rq_clock(rq_of(cfs_rq));
if (tsk->state & TASK_UNINTERRUPTIBLE)
se->statistics.block_start = rq_of(cfs_rq)->clock;
se->statistics.block_start = rq_clock(rq_of(cfs_rq));
}
#endif
}
@ -4153,7 +4153,7 @@ static inline u64 cfs_rq_clock_task(struct cfs_rq *cfs_rq)
if (unlikely(cfs_rq->throttle_count))
return cfs_rq->throttled_clock_task;
return rq_of(cfs_rq)->clock_task - cfs_rq->throttled_clock_task_time;
return rq_clock_task(rq_of(cfs_rq)) - cfs_rq->throttled_clock_task_time;
}
/* returns 0 on failure to allocate runtime */
@ -4209,10 +4209,9 @@ static int assign_cfs_rq_runtime(struct cfs_rq *cfs_rq)
static void expire_cfs_rq_runtime(struct cfs_rq *cfs_rq)
{
struct cfs_bandwidth *cfs_b = tg_cfs_bandwidth(cfs_rq->tg);
struct rq *rq = rq_of(cfs_rq);
/* if the deadline is ahead of our clock, nothing to do */
if (likely((s64)(rq->clock - cfs_rq->runtime_expires) < 0))
if (likely((s64)(rq_clock(rq_of(cfs_rq)) - cfs_rq->runtime_expires) < 0))
return;
if (cfs_rq->runtime_remaining < 0)
@ -4328,7 +4327,7 @@ static int tg_unthrottle_up(struct task_group *tg, void *data)
#ifdef CONFIG_SMP
if (!cfs_rq->throttle_count) {
/* adjust cfs_rq_clock_task() */
cfs_rq->throttled_clock_task_time += rq->clock_task -
cfs_rq->throttled_clock_task_time += rq_clock_task(rq) -
cfs_rq->throttled_clock_task;
}
#endif
@ -4343,7 +4342,7 @@ static int tg_throttle_down(struct task_group *tg, void *data)
/* group is entering throttled state, stop time */
if (!cfs_rq->throttle_count)
cfs_rq->throttled_clock_task = rq->clock_task;
cfs_rq->throttled_clock_task = rq_clock_task(rq);
cfs_rq->throttle_count++;
return 0;
@ -4386,7 +4385,7 @@ static void throttle_cfs_rq(struct cfs_rq *cfs_rq)
}
cfs_rq->throttled = 1;
cfs_rq->throttled_clock = rq->clock;
cfs_rq->throttled_clock = rq_clock(rq);
raw_spin_lock(&cfs_b->lock);
list_add_tail_rcu(&cfs_rq->throttled_list, &cfs_b->throttled_cfs_rq);
if (!cfs_b->timer_active)
@ -4414,7 +4413,7 @@ void unthrottle_cfs_rq(struct cfs_rq *cfs_rq)
cfs_rq->throttled = 0;
raw_spin_lock(&cfs_b->lock);
cfs_b->throttled_time += rq->clock - cfs_rq->throttled_clock;
cfs_b->throttled_time += rq_clock(rq) - cfs_rq->throttled_clock;
list_del_rcu(&cfs_rq->throttled_list);
raw_spin_unlock(&cfs_b->lock);
@ -4845,7 +4844,7 @@ static void __maybe_unused unthrottle_offline_cfs_rqs(struct rq *rq)
#else /* CONFIG_CFS_BANDWIDTH */
static inline u64 cfs_rq_clock_task(struct cfs_rq *cfs_rq)
{
return rq_of(cfs_rq)->clock_task;
return rq_clock_task(rq_of(cfs_rq));
}
static void account_cfs_rq_runtime(struct cfs_rq *cfs_rq,
@ -6139,7 +6138,7 @@ int can_migrate_task(struct task_struct *p, struct lb_env *env)
* 2) too many balance attempts have failed.
*/
tsk_cache_hot = task_hot(p, env->src_rq->clock_task, env->sd);
tsk_cache_hot = task_hot(p, rq_clock_task(env->src_rq), env->sd);
if (!tsk_cache_hot ||
env->sd->nr_balance_failed > env->sd->cache_nice_tries) {
@ -6558,7 +6557,7 @@ static unsigned long scale_rt_power(int cpu)
age_stamp = ACCESS_ONCE(rq->age_stamp);
avg = ACCESS_ONCE(rq->rt_avg);
total = sched_avg_period() + (rq->clock - age_stamp);
total = sched_avg_period() + (rq_clock(rq) - age_stamp);
if (unlikely(total < avg)) {
/* Ensures that power won't end up being negative */
@ -7693,7 +7692,7 @@ void idle_balance(int this_cpu, struct rq *this_rq)
int balance_cpu = -1;
struct rq *balance_rq = NULL;
this_rq->idle_stamp = this_rq->clock;
this_rq->idle_stamp = rq_clock(this_rq);
if (this_rq->avg_idle < sysctl_sched_migration_cost)
return;

8
kernel/sched/rt.c
View File

@ -932,7 +932,7 @@ static void update_curr_rt(struct rq *rq)
if (curr->sched_class != &rt_sched_class)
return;
delta_exec = rq->clock_task - curr->se.exec_start;
delta_exec = rq_clock_task(rq) - curr->se.exec_start;
if (unlikely((s64)delta_exec <= 0))
return;
@ -942,7 +942,7 @@ static void update_curr_rt(struct rq *rq)
curr->se.sum_exec_runtime += delta_exec;
account_group_exec_runtime(curr, delta_exec);
curr->se.exec_start = rq->clock_task;
curr->se.exec_start = rq_clock_task(rq);
cpuacct_charge(curr, delta_exec);
sched_rt_avg_update(rq, delta_exec);
@ -1459,7 +1459,7 @@ static struct task_struct *_pick_next_task_rt(struct rq *rq)
update_rq_clock(rq);
}
p = rt_task_of(rt_se);
p->se.exec_start = rq->clock_task;
p->se.exec_start = rq_clock_task(rq);
return p;
}
@ -2170,7 +2170,7 @@ static void set_curr_task_rt(struct rq *rq)
{
struct task_struct *p = rq->curr;
p->se.exec_start = rq->clock_task;
p->se.exec_start = rq_clock_task(rq);
/* The running task is never eligible for pushing */
dequeue_pushable_task(rq, p);

10
kernel/sched/sched.h
View File

@ -600,6 +600,16 @@ DECLARE_PER_CPU(struct rq, runqueues);
#define cpu_curr(cpu) (cpu_rq(cpu)->curr)
#define raw_rq() (&__raw_get_cpu_var(runqueues))
static inline u64 rq_clock(struct rq *rq)
{
return rq->clock;
}
static inline u64 rq_clock_task(struct rq *rq)
{
return rq->clock_task;
}
#ifdef CONFIG_SMP
#define rcu_dereference_check_sched_domain(p) \

8
kernel/sched/stats.h
View File

@ -61,7 +61,7 @@ static inline void sched_info_reset_dequeued(struct task_struct *t)
*/
static inline void sched_info_dequeued(struct task_struct *t)
{
unsigned long long now = task_rq(t)->clock, delta = 0;
unsigned long long now = rq_clock(task_rq(t)), delta = 0;
if (unlikely(sched_info_on()))
if (t->sched_info.last_queued)
@ -79,7 +79,7 @@ static inline void sched_info_dequeued(struct task_struct *t)
*/
static void sched_info_arrive(struct task_struct *t)
{
unsigned long long now = task_rq(t)->clock, delta = 0;
unsigned long long now = rq_clock(task_rq(t)), delta = 0;
if (t->sched_info.last_queued)
delta = now - t->sched_info.last_queued;
@ -100,7 +100,7 @@ static inline void sched_info_queued(struct task_struct *t)
{
if (unlikely(sched_info_on()))
if (!t->sched_info.last_queued)
t->sched_info.last_queued = task_rq(t)->clock;
t->sched_info.last_queued = rq_clock(task_rq(t));
}
/*
@ -112,7 +112,7 @@ static inline void sched_info_queued(struct task_struct *t)
*/
static inline void sched_info_depart(struct task_struct *t)
{
unsigned long long delta = task_rq(t)->clock -
unsigned long long delta = rq_clock(task_rq(t)) -
t->sched_info.last_arrival;
rq_sched_info_depart(task_rq(t), delta);

8
kernel/sched/stop_task.c
View File

@ -52,7 +52,7 @@ static struct task_struct *pick_next_task_stop(struct rq *rq)
struct task_struct *stop = rq->stop;
if (stop && stop->on_rq) {
stop->se.exec_start = rq->clock_task;
stop->se.exec_start = rq_clock_task(rq);
return stop;
}
@ -83,7 +83,7 @@ static void put_prev_task_stop(struct rq *rq, struct task_struct *prev)
struct task_struct *curr = rq->curr;
u64 delta_exec;
delta_exec = rq->clock_task - curr->se.exec_start;
delta_exec = rq_clock_task(rq) - curr->se.exec_start;
if (unlikely((s64)delta_exec < 0))
delta_exec = 0;
@ -93,7 +93,7 @@ static void put_prev_task_stop(struct rq *rq, struct task_struct *prev)
curr->se.sum_exec_runtime += delta_exec;
account_group_exec_runtime(curr, delta_exec);
curr->se.exec_start = rq->clock_task;
curr->se.exec_start = rq_clock_task(rq);
cpuacct_charge(curr, delta_exec);
}
@ -105,7 +105,7 @@ static void set_curr_task_stop(struct rq *rq)
{
struct task_struct *stop = rq->stop;
stop->se.exec_start = rq->clock_task;
stop->se.exec_start = rq_clock_task(rq);
}
static void switched_to_stop(struct rq *rq, struct task_struct *p)