The following commit has been merged in the linux branch: commit 8633322c5fd5b2a986b279f88a7559d8409f7da3 Merge: 9532faeb293f5a5f0ff06f567de14e557698dbde 4a6cc4bd32e580722882115d4c8b964d732c11e4 Author: Linus Torvalds torvalds@linux-foundation.org Date: Thu Oct 29 09:19:29 2009 -0700
Merge branch 'for-linus' of git://git.kernel.org/pub/scm/linux/kernel/git/tj/percpu
* 'for-linus' of git://git.kernel.org/pub/scm/linux/kernel/git/tj/percpu: sched: move rq_weight data array out of .percpu percpu: allow pcpu_alloc() to be called with IRQs off
diff --combined kernel/sched.c index e886895,526d237..a455dca --- a/kernel/sched.c +++ b/kernel/sched.c @@@ -676,7 -676,6 +676,7 @@@ inline void update_rq_clock(struct rq *
/** * runqueue_is_locked + * @cpu: the processor in question. * * Returns true if the current cpu runqueue is locked. * This interface allows printk to be called with the runqueue lock @@@ -781,7 -780,7 +781,7 @@@ static int sched_feat_open(struct inod return single_open(filp, sched_feat_show, NULL); }
-static struct file_operations sched_feat_fops = { +static const struct file_operations sched_feat_fops = { .open = sched_feat_open, .write = sched_feat_write, .read = seq_read, @@@ -1564,11 -1563,7 +1564,7 @@@ static unsigned long cpu_avg_load_per_t
#ifdef CONFIG_FAIR_GROUP_SCHED
- struct update_shares_data { - unsigned long rq_weight[NR_CPUS]; - }; - - static DEFINE_PER_CPU(struct update_shares_data, update_shares_data); + static __read_mostly unsigned long *update_shares_data;
static void __set_se_shares(struct sched_entity *se, unsigned long shares);
@@@ -1578,12 -1573,12 +1574,12 @@@ static void update_group_shares_cpu(struct task_group *tg, int cpu, unsigned long sd_shares, unsigned long sd_rq_weight, - struct update_shares_data *usd) + unsigned long *usd_rq_weight) { unsigned long shares, rq_weight; int boost = 0;
- rq_weight = usd->rq_weight[cpu]; + rq_weight = usd_rq_weight[cpu]; if (!rq_weight) { boost = 1; rq_weight = NICE_0_LOAD; @@@ -1618,7 -1613,7 +1614,7 @@@ static int tg_shares_up(struct task_group *tg, void *data) { unsigned long weight, rq_weight = 0, shares = 0; - struct update_shares_data *usd; + unsigned long *usd_rq_weight; struct sched_domain *sd = data; unsigned long flags; int i; @@@ -1627,11 -1622,11 +1623,11 @@@ return 0;
local_irq_save(flags); - usd = &__get_cpu_var(update_shares_data); + usd_rq_weight = per_cpu_ptr(update_shares_data, smp_processor_id());
for_each_cpu(i, sched_domain_span(sd)) { weight = tg->cfs_rq[i]->load.weight; - usd->rq_weight[i] = weight; + usd_rq_weight[i] = weight;
/* * If there are currently no tasks on the cpu pretend there @@@ -1652,7 -1647,7 +1648,7 @@@ shares = tg->shares;
for_each_cpu(i, sched_domain_span(sd)) - update_group_shares_cpu(tg, i, shares, rq_weight, usd); + update_group_shares_cpu(tg, i, shares, rq_weight, usd_rq_weight);
local_irq_restore(flags);
@@@ -2312,7 -2307,7 +2308,7 @@@ static int try_to_wake_up(struct task_s { int cpu, orig_cpu, this_cpu, success = 0; unsigned long flags; - struct rq *rq; + struct rq *rq, *orig_rq;
if (!sched_feat(SYNC_WAKEUPS)) wake_flags &= ~WF_SYNC; @@@ -2320,7 -2315,7 +2316,7 @@@ this_cpu = get_cpu();
smp_wmb(); - rq = task_rq_lock(p, &flags); + rq = orig_rq = task_rq_lock(p, &flags); update_rq_clock(rq); if (!(p->state & state)) goto out; @@@ -2351,10 -2346,6 +2347,10 @@@ set_task_cpu(p, cpu);
rq = task_rq_lock(p, &flags); + + if (rq != orig_rq) + update_rq_clock(rq); + WARN_ON(p->state != TASK_WAKING); cpu = task_cpu(p);
@@@ -2520,17 -2511,22 +2516,17 @@@ void sched_fork(struct task_struct *p, __sched_fork(p);
/* * Revert to default priority/policy on fork if requested. */ if (unlikely(p->sched_reset_on_fork)) { - if (p->policy == SCHED_FIFO || p->policy == SCHED_RR) + if (p->policy == SCHED_FIFO || p->policy == SCHED_RR) { p->policy = SCHED_NORMAL; - - if (p->normal_prio < DEFAULT_PRIO) - p->prio = DEFAULT_PRIO; + p->normal_prio = p->static_prio; + }
if (PRIO_TO_NICE(p->static_prio) < 0) { p->static_prio = NICE_TO_PRIO(0); + p->normal_prio = p->static_prio; set_load_weight(p); }
@@@ -2541,11 -2537,6 +2537,11 @@@ p->sched_reset_on_fork = 0; }
+ /* + * Make sure we do not leak PI boosting priority to the child. + */ + p->prio = current->normal_prio; + if (!rt_prio(p->prio)) p->sched_class = &fair_sched_class;
@@@ -2586,6 -2577,8 +2582,6 @@@ void wake_up_new_task(struct task_struc BUG_ON(p->state != TASK_RUNNING); update_rq_clock(rq);
- p->prio = effective_prio(p); - if (!p->sched_class->task_new || !current->se.on_rq) { activate_task(rq, p, 0); } else { @@@ -3661,7 -3654,6 +3657,7 @@@ static void update_group_power(struct s
/** * update_sg_lb_stats - Update sched_group's statistics for load balancing. + * @sd: The sched_domain whose statistics are to be updated. * @group: sched_group whose statistics are to be updated. * @this_cpu: Cpu for which load balance is currently performed. * @idle: Idle status of this_cpu @@@ -6724,6 -6716,9 +6720,6 @@@ EXPORT_SYMBOL(yield) /* * This task is about to go to sleep on IO. Increment rq->nr_iowait so * that process accounting knows that this is a task in IO wait state. - * - * But don't do that if it is a deliberate, throttling IO wait (this task - * has set its backing_dev_info: the queue against which it should throttle) */ void __sched io_schedule(void) { @@@ -9407,6 -9402,10 +9403,10 @@@ void __init sched_init(void #endif /* CONFIG_USER_SCHED */ #endif /* CONFIG_GROUP_SCHED */
+ #if defined CONFIG_FAIR_GROUP_SCHED && defined CONFIG_SMP + update_shares_data = __alloc_percpu(nr_cpu_ids * sizeof(unsigned long), + __alignof__(unsigned long)); + #endif for_each_possible_cpu(i) { struct rq *rq;