debuggers.hg

changeset 21215:0387e6ce1540

credit2: Flexible cpu-to-schedule-spinlock mappings

Credit2 shares a runqueue between several cpus. Rather than have
double locking and dealing with the cpu-to-runqueue races, allow
the scheduler to redefine the sched_lock-to-cpu mapping.

Signed-off-by: George Dunlap <george.dunlap@eu.citrix.com>
author Keir Fraser <keir.fraser@citrix.com>
date Wed Apr 14 12:05:31 2010 +0100 (2010-04-14)
parents 5cd44772fd21
children dca0baecc909
files xen/arch/ia64/vmx/vmmu.c xen/common/sched_credit.c xen/common/schedule.c xen/include/xen/sched-if.h
line diff
     1.1 --- a/xen/arch/ia64/vmx/vmmu.c	Wed Apr 14 12:03:27 2010 +0100
     1.2 +++ b/xen/arch/ia64/vmx/vmmu.c	Wed Apr 14 12:05:31 2010 +0100
     1.3 @@ -394,7 +394,7 @@ static void ptc_ga_remote_func (void *va
     1.4      if (cpu != current->processor)
     1.5          return;
     1.6      local_irq_save(flags);
     1.7 -    if (!spin_trylock(&per_cpu(schedule_data, cpu).schedule_lock))
     1.8 +    if (!spin_trylock(per_cpu(schedule_data, cpu).schedule_lock))
     1.9          goto bail2;
    1.10      if (v->processor != cpu)
    1.11          goto bail1;
    1.12 @@ -416,7 +416,7 @@ static void ptc_ga_remote_func (void *va
    1.13      ia64_dv_serialize_data();
    1.14      args->vcpu = NULL;
    1.15  bail1:
    1.16 -    spin_unlock(&per_cpu(schedule_data, cpu).schedule_lock);
    1.17 +    spin_unlock(per_cpu(schedule_data, cpu).schedule_lock);
    1.18  bail2:
    1.19      local_irq_restore(flags);
    1.20  }
    1.21 @@ -446,7 +446,7 @@ IA64FAULT vmx_vcpu_ptc_ga(VCPU *vcpu, u6
    1.22          do {
    1.23              cpu = v->processor;
    1.24              if (cpu != current->processor) {
    1.25 -                spin_barrier(&per_cpu(schedule_data, cpu).schedule_lock);
    1.26 +                spin_barrier(per_cpu(schedule_data, cpu).schedule_lock);
    1.27                  /* Flush VHPT on remote processors. */
    1.28                  smp_call_function_single(cpu, &ptc_ga_remote_func, &args, 1);
    1.29              } else {
     2.1 --- a/xen/common/sched_credit.c	Wed Apr 14 12:03:27 2010 +0100
     2.2 +++ b/xen/common/sched_credit.c	Wed Apr 14 12:05:31 2010 +0100
     2.3 @@ -789,7 +789,7 @@ csched_runq_sort(unsigned int cpu)
     2.4  
     2.5      spc->runq_sort_last = sort_epoch;
     2.6  
     2.7 -    spin_lock_irqsave(&per_cpu(schedule_data, cpu).schedule_lock, flags);
     2.8 +    spin_lock_irqsave(per_cpu(schedule_data, cpu).schedule_lock, flags);
     2.9  
    2.10      runq = &spc->runq;
    2.11      elem = runq->next;
    2.12 @@ -814,7 +814,7 @@ csched_runq_sort(unsigned int cpu)
    2.13          elem = next;
    2.14      }
    2.15  
    2.16 -    spin_unlock_irqrestore(&per_cpu(schedule_data, cpu).schedule_lock, flags);
    2.17 +    spin_unlock_irqrestore(per_cpu(schedule_data, cpu).schedule_lock, flags);
    2.18  }
    2.19  
    2.20  static void
    2.21 @@ -1130,7 +1130,7 @@ csched_load_balance(int cpu, struct csch
    2.22           * cause a deadlock if the peer CPU is also load balancing and trying
    2.23           * to lock this CPU.
    2.24           */
    2.25 -        if ( !spin_trylock(&per_cpu(schedule_data, peer_cpu).schedule_lock) )
    2.26 +        if ( !spin_trylock(per_cpu(schedule_data, peer_cpu).schedule_lock) )
    2.27          {
    2.28              CSCHED_STAT_CRANK(steal_trylock_failed);
    2.29              continue;
    2.30 @@ -1140,7 +1140,7 @@ csched_load_balance(int cpu, struct csch
    2.31           * Any work over there to steal?
    2.32           */
    2.33          speer = csched_runq_steal(peer_cpu, cpu, snext->pri);
    2.34 -        spin_unlock(&per_cpu(schedule_data, peer_cpu).schedule_lock);
    2.35 +        spin_unlock(per_cpu(schedule_data, peer_cpu).schedule_lock);
    2.36          if ( speer != NULL )
    2.37              return speer;
    2.38      }
     3.1 --- a/xen/common/schedule.c	Wed Apr 14 12:03:27 2010 +0100
     3.2 +++ b/xen/common/schedule.c	Wed Apr 14 12:05:31 2010 +0100
     3.3 @@ -131,7 +131,7 @@ static inline void vcpu_runstate_change(
     3.4      s_time_t delta;
     3.5  
     3.6      ASSERT(v->runstate.state != new_state);
     3.7 -    ASSERT(spin_is_locked(&per_cpu(schedule_data,v->processor).schedule_lock));
     3.8 +    ASSERT(spin_is_locked(per_cpu(schedule_data,v->processor).schedule_lock));
     3.9  
    3.10      vcpu_urgent_count_update(v);
    3.11  
    3.12 @@ -340,7 +340,7 @@ static void vcpu_migrate(struct vcpu *v)
    3.13      /* Switch to new CPU, then unlock old CPU. */
    3.14      v->processor = new_cpu;
    3.15      spin_unlock_irqrestore(
    3.16 -        &per_cpu(schedule_data, old_cpu).schedule_lock, flags);
    3.17 +        per_cpu(schedule_data, old_cpu).schedule_lock, flags);
    3.18  
    3.19      /* Wake on new CPU. */
    3.20      vcpu_wake(v);
    3.21 @@ -808,7 +808,7 @@ static void schedule(void)
    3.22  
    3.23      sd = &this_cpu(schedule_data);
    3.24  
    3.25 -    spin_lock_irq(&sd->schedule_lock);
    3.26 +    spin_lock_irq(sd->schedule_lock);
    3.27  
    3.28      stop_timer(&sd->s_timer);
    3.29      
    3.30 @@ -824,7 +824,7 @@ static void schedule(void)
    3.31  
    3.32      if ( unlikely(prev == next) )
    3.33      {
    3.34 -        spin_unlock_irq(&sd->schedule_lock);
    3.35 +        spin_unlock_irq(sd->schedule_lock);
    3.36          trace_continue_running(next);
    3.37          return continue_running(prev);
    3.38      }
    3.39 @@ -862,7 +862,7 @@ static void schedule(void)
    3.40      ASSERT(!next->is_running);
    3.41      next->is_running = 1;
    3.42  
    3.43 -    spin_unlock_irq(&sd->schedule_lock);
    3.44 +    spin_unlock_irq(sd->schedule_lock);
    3.45  
    3.46      perfc_incr(sched_ctx);
    3.47  
    3.48 @@ -930,7 +930,9 @@ void __init scheduler_init(void)
    3.49  
    3.50      for_each_possible_cpu ( i )
    3.51      {
    3.52 -        spin_lock_init(&per_cpu(schedule_data, i).schedule_lock);
    3.53 +        spin_lock_init(&per_cpu(schedule_data, i)._lock);
    3.54 +        per_cpu(schedule_data, i).schedule_lock
    3.55 +            = &per_cpu(schedule_data, i)._lock;
    3.56          init_timer(&per_cpu(schedule_data, i).s_timer, s_timer_fn, NULL, i);
    3.57      }
    3.58  
    3.59 @@ -967,10 +969,10 @@ void dump_runq(unsigned char key)
    3.60  
    3.61      for_each_online_cpu ( i )
    3.62      {
    3.63 -        spin_lock(&per_cpu(schedule_data, i).schedule_lock);
    3.64 +        spin_lock(per_cpu(schedule_data, i).schedule_lock);
    3.65          printk("CPU[%02d] ", i);
    3.66          SCHED_OP(dump_cpu_state, i);
    3.67 -        spin_unlock(&per_cpu(schedule_data, i).schedule_lock);
    3.68 +        spin_unlock(per_cpu(schedule_data, i).schedule_lock);
    3.69      }
    3.70  
    3.71      local_irq_restore(flags);
     4.1 --- a/xen/include/xen/sched-if.h	Wed Apr 14 12:03:27 2010 +0100
     4.2 +++ b/xen/include/xen/sched-if.h	Wed Apr 14 12:05:31 2010 +0100
     4.3 @@ -10,8 +10,19 @@
     4.4  
     4.5  #include <xen/percpu.h>
     4.6  
     4.7 +/*
     4.8 + * In order to allow a scheduler to remap the lock->cpu mapping,
     4.9 + * we have a per-cpu pointer, along with a pre-allocated set of
    4.10 + * locks.  The generic schedule init code will point each schedule lock
    4.11 + * pointer to the schedule lock; if the scheduler wants to remap them,
    4.12 + * it can simply modify the schedule locks.
    4.13 + * 
    4.14 + * For cache betterness, keep the actual lock in the same cache area
    4.15 + * as the rest of the struct.  Just have the scheduler point to the
    4.16 + * one it wants (This may be the one right in front of it).*/
    4.17  struct schedule_data {
    4.18 -    spinlock_t          schedule_lock;  /* spinlock protecting curr        */
    4.19 +    spinlock_t         *schedule_lock,
    4.20 +                       _lock;
    4.21      struct vcpu        *curr;           /* current task                    */
    4.22      struct vcpu        *idle;           /* idle task for this cpu          */
    4.23      void               *sched_priv;
    4.24 @@ -27,11 +38,19 @@ static inline void vcpu_schedule_lock(st
    4.25  
    4.26      for ( ; ; )
    4.27      {
    4.28 +        /* NB: For schedulers with multiple cores per runqueue,
    4.29 +         * a vcpu may change processor w/o changing runqueues;
    4.30 +         * so we may release a lock only to grab it again.
    4.31 +         *
    4.32 +         * If that is measured to be an issue, then the check
    4.33 +         * should be changed to checking if the locks pointed to
    4.34 +         * by cpu and v->processor are still the same.
    4.35 +         */
    4.36          cpu = v->processor;
    4.37 -        spin_lock(&per_cpu(schedule_data, cpu).schedule_lock);
    4.38 +        spin_lock(per_cpu(schedule_data, cpu).schedule_lock);
    4.39          if ( likely(v->processor == cpu) )
    4.40              break;
    4.41 -        spin_unlock(&per_cpu(schedule_data, cpu).schedule_lock);
    4.42 +        spin_unlock(per_cpu(schedule_data, cpu).schedule_lock);
    4.43      }
    4.44  }
    4.45  
    4.46 @@ -42,7 +61,7 @@ static inline void vcpu_schedule_lock(st
    4.47  
    4.48  static inline void vcpu_schedule_unlock(struct vcpu *v)
    4.49  {
    4.50 -    spin_unlock(&per_cpu(schedule_data, v->processor).schedule_lock);
    4.51 +    spin_unlock(per_cpu(schedule_data, v->processor).schedule_lock);
    4.52  }
    4.53  
    4.54  #define vcpu_schedule_unlock_irq(v) \