debuggers.hg

view xen/common/cpupool.c @ 22478:aee9a8f63aae

cpupool: Simplify locking, use refcounts for cpupool liveness.

Signed-off-by: Keir Fraser <keir@xen.org>
author Keir Fraser <keir@xen.org>
date Fri Nov 26 10:07:57 2010 +0000 (2010-11-26)
parents dab8676e97ce
children
line source
1 /******************************************************************************
2 * cpupool.c
3 *
4 * Generic cpupool-handling functions.
5 *
6 * Cpupools are a feature to have configurable scheduling domains. Each
7 * cpupool runs an own scheduler on a dedicated set of physical cpus.
8 * A domain is bound to one cpupool at any time, but it can be moved to
9 * another cpupool.
10 *
11 * (C) 2009, Juergen Gross, Fujitsu Technology Solutions
12 */
14 #include <xen/lib.h>
15 #include <xen/init.h>
16 #include <xen/cpumask.h>
17 #include <xen/percpu.h>
18 #include <xen/sched.h>
19 #include <xen/sched-if.h>
20 #include <xen/cpu.h>
22 #define for_each_cpupool(ptr) \
23 for ((ptr) = &cpupool_list; *(ptr) != NULL; (ptr) = &((*(ptr))->next))
25 struct cpupool *cpupool0; /* Initial cpupool with Dom0 */
26 cpumask_t cpupool_free_cpus; /* cpus not in any cpupool */
28 static struct cpupool *cpupool_list; /* linked list, sorted by poolid */
30 static int cpupool_moving_cpu = -1;
31 static struct cpupool *cpupool_cpu_moving = NULL;
32 static cpumask_t cpupool_locked_cpus = CPU_MASK_NONE;
34 static DEFINE_SPINLOCK(cpupool_lock);
36 DEFINE_PER_CPU(struct cpupool *, cpupool);
38 #define cpupool_dprintk(x...) ((void)0)
40 static struct cpupool *alloc_cpupool_struct(void)
41 {
42 return xmalloc(struct cpupool);
43 }
45 static void free_cpupool_struct(struct cpupool *c)
46 {
47 xfree(c);
48 }
50 /*
51 * find a cpupool by it's id. to be called with cpupool lock held
52 * if exact is not specified, the first cpupool with an id larger or equal to
53 * the searched id is returned
54 * returns NULL if not found.
55 */
56 static struct cpupool *cpupool_find_by_id(int id, int exact)
57 {
58 struct cpupool **q;
60 ASSERT(spin_is_locked(&cpupool_lock));
62 for_each_cpupool(q)
63 if ( (*q)->cpupool_id >= id )
64 break;
66 return (!exact || ((*q)->cpupool_id == id)) ? *q : NULL;
67 }
69 static struct cpupool *__cpupool_get_by_id(int poolid, int exact)
70 {
71 struct cpupool *c;
72 spin_lock(&cpupool_lock);
73 c = cpupool_find_by_id(poolid, exact);
74 if ( c != NULL )
75 atomic_inc(&c->refcnt);
76 spin_unlock(&cpupool_lock);
77 return c;
78 }
80 struct cpupool *cpupool_get_by_id(int poolid)
81 {
82 return __cpupool_get_by_id(poolid, 1);
83 }
85 void cpupool_put(struct cpupool *pool)
86 {
87 if ( !atomic_dec_and_test(&pool->refcnt) )
88 return;
89 scheduler_free(pool->sched);
90 free_cpupool_struct(pool);
91 }
93 /*
94 * create a new cpupool with specified poolid and scheduler
95 * returns pointer to new cpupool structure if okay, NULL else
96 * possible failures:
97 * - no memory
98 * - poolid already used
99 * - unknown scheduler
100 */
101 static struct cpupool *cpupool_create(
102 int poolid, unsigned int sched_id, int *perr)
103 {
104 struct cpupool *c;
105 struct cpupool **q;
106 int last = 0;
108 *perr = -ENOMEM;
109 if ( (c = alloc_cpupool_struct()) == NULL )
110 return NULL;
111 memset(c, 0, sizeof(*c));
113 /* One reference for caller, one reference for cpupool_destroy(). */
114 atomic_set(&c->refcnt, 2);
116 cpupool_dprintk("cpupool_create(pool=%d,sched=%u)\n", poolid, sched_id);
118 spin_lock(&cpupool_lock);
120 for_each_cpupool(q)
121 {
122 last = (*q)->cpupool_id;
123 if ( (poolid != CPUPOOLID_NONE) && (last >= poolid) )
124 break;
125 }
126 if ( *q != NULL )
127 {
128 if ( (*q)->cpupool_id == poolid )
129 {
130 spin_unlock(&cpupool_lock);
131 free_cpupool_struct(c);
132 *perr = -EEXIST;
133 return NULL;
134 }
135 c->next = *q;
136 }
138 c->cpupool_id = (poolid == CPUPOOLID_NONE) ? (last + 1) : poolid;
139 if ( poolid == 0 )
140 {
141 c->sched = scheduler_get_default();
142 }
143 else
144 {
145 c->sched = scheduler_alloc(sched_id, perr);
146 if ( c->sched == NULL )
147 {
148 spin_unlock(&cpupool_lock);
149 free_cpupool_struct(c);
150 return NULL;
151 }
152 }
154 *q = c;
156 spin_unlock(&cpupool_lock);
158 cpupool_dprintk("Created cpupool %d with scheduler %s (%s)\n",
159 c->cpupool_id, c->sched->name, c->sched->opt_name);
161 *perr = 0;
162 return c;
163 }
164 /*
165 * destroys the given cpupool
166 * returns 0 on success, 1 else
167 * possible failures:
168 * - pool still in use
169 * - cpus still assigned to pool
170 * - pool not in list
171 */
172 static int cpupool_destroy(struct cpupool *c)
173 {
174 struct cpupool **q;
176 spin_lock(&cpupool_lock);
177 for_each_cpupool(q)
178 if ( *q == c )
179 break;
180 if ( *q != c )
181 {
182 spin_unlock(&cpupool_lock);
183 return -ENOENT;
184 }
185 if ( (c->n_dom != 0) || cpus_weight(c->cpu_valid) )
186 {
187 spin_unlock(&cpupool_lock);
188 return -EBUSY;
189 }
190 *q = c->next;
191 spin_unlock(&cpupool_lock);
193 cpupool_put(c);
195 cpupool_dprintk("cpupool_destroy(pool=%d)\n", c->cpupool_id);
196 return 0;
197 }
199 /*
200 * assign a specific cpu to a cpupool
201 * cpupool_lock must be held
202 */
203 static int cpupool_assign_cpu_locked(struct cpupool *c, unsigned int cpu)
204 {
205 if ( (cpupool_moving_cpu == cpu) && (c != cpupool_cpu_moving) )
206 return -EBUSY;
207 per_cpu(cpupool, cpu) = c;
208 schedule_cpu_switch(cpu, c);
209 cpu_clear(cpu, cpupool_free_cpus);
210 if (cpupool_moving_cpu == cpu)
211 {
212 cpupool_moving_cpu = -1;
213 cpupool_put(cpupool_cpu_moving);
214 cpupool_cpu_moving = NULL;
215 }
216 cpu_set(cpu, c->cpu_valid);
217 return 0;
218 }
220 static long cpupool_unassign_cpu_helper(void *info)
221 {
222 int cpu = cpupool_moving_cpu;
223 long ret;
225 cpupool_dprintk("cpupool_unassign_cpu(pool=%d,cpu=%d) ret %ld\n",
226 cpupool_id, cpu, ret);
228 spin_lock(&cpupool_lock);
229 ret = cpu_disable_scheduler(cpu);
230 cpu_set(cpu, cpupool_free_cpus);
231 if ( !ret )
232 {
233 schedule_cpu_switch(cpu, NULL);
234 per_cpu(cpupool, cpu) = NULL;
235 cpupool_moving_cpu = -1;
236 cpupool_put(cpupool_cpu_moving);
237 cpupool_cpu_moving = NULL;
238 }
239 spin_unlock(&cpupool_lock);
240 return ret;
241 }
243 /*
244 * unassign a specific cpu from a cpupool
245 * we must be sure not to run on the cpu to be unassigned! to achieve this
246 * the main functionality is performed via continue_hypercall_on_cpu on a
247 * specific cpu.
248 * if the cpu to be removed is the last one of the cpupool no active domain
249 * must be bound to the cpupool. dying domains are moved to cpupool0 as they
250 * might be zombies.
251 * possible failures:
252 * - last cpu and still active domains in cpupool
253 * - cpu just being unplugged
254 */
255 int cpupool_unassign_cpu(struct cpupool *c, unsigned int cpu)
256 {
257 int work_cpu;
258 int ret;
259 struct domain *d;
261 cpupool_dprintk("cpupool_unassign_cpu(pool=%d,cpu=%d)\n",
262 c->cpupool_id, cpu);
264 spin_lock(&cpupool_lock);
265 ret = -EBUSY;
266 if ( (cpupool_moving_cpu != -1) && (cpu != cpupool_moving_cpu) )
267 goto out;
268 if ( cpu_isset(cpu, cpupool_locked_cpus) )
269 goto out;
271 ret = 0;
272 if ( !cpu_isset(cpu, c->cpu_valid) && (cpu != cpupool_moving_cpu) )
273 goto out;
275 if ( (c->n_dom > 0) && (cpus_weight(c->cpu_valid) == 1) &&
276 (cpu != cpupool_moving_cpu) )
277 {
278 for_each_domain(d)
279 {
280 if ( d->cpupool != c )
281 continue;
282 if ( !d->is_dying )
283 {
284 ret = -EBUSY;
285 break;
286 }
287 c->n_dom--;
288 ret = sched_move_domain(d, cpupool0);
289 if ( ret )
290 {
291 c->n_dom++;
292 break;
293 }
294 cpupool0->n_dom++;
295 }
296 if ( ret )
297 goto out;
298 }
299 cpupool_moving_cpu = cpu;
300 atomic_inc(&c->refcnt);
301 cpupool_cpu_moving = c;
302 cpu_clear(cpu, c->cpu_valid);
303 spin_unlock(&cpupool_lock);
305 work_cpu = smp_processor_id();
306 if ( work_cpu == cpu )
307 {
308 work_cpu = first_cpu(cpupool0->cpu_valid);
309 if ( work_cpu == cpu )
310 work_cpu = next_cpu(cpu, cpupool0->cpu_valid);
311 }
312 return continue_hypercall_on_cpu(work_cpu, cpupool_unassign_cpu_helper, c);
314 out:
315 spin_unlock(&cpupool_lock);
316 cpupool_dprintk("cpupool_unassign_cpu(pool=%d,cpu=%d) ret %d\n",
317 cpupool_id, cpu, ret);
318 return ret;
319 }
321 /*
322 * add a new domain to a cpupool
323 * possible failures:
324 * - pool does not exist
325 * - no cpu assigned to pool
326 */
327 int cpupool_add_domain(struct domain *d, int poolid)
328 {
329 struct cpupool *c;
330 int rc = 1;
331 int n_dom;
333 if ( poolid == CPUPOOLID_NONE )
334 return 0;
335 spin_lock(&cpupool_lock);
336 c = cpupool_find_by_id(poolid, 1);
337 if ( (c != NULL) && cpus_weight(c->cpu_valid) )
338 {
339 c->n_dom++;
340 n_dom = c->n_dom;
341 d->cpupool = c;
342 rc = 0;
343 }
344 spin_unlock(&cpupool_lock);
345 cpupool_dprintk("cpupool_add_domain(dom=%d,pool=%d) n_dom %d rc %d\n",
346 d->domain_id, poolid, n_dom, rc);
347 return rc;
348 }
350 /*
351 * remove a domain from a cpupool
352 */
353 void cpupool_rm_domain(struct domain *d)
354 {
355 int cpupool_id;
356 int n_dom;
358 if ( d->cpupool == NULL )
359 return;
360 spin_lock(&cpupool_lock);
361 cpupool_id = d->cpupool->cpupool_id;
362 d->cpupool->n_dom--;
363 n_dom = d->cpupool->n_dom;
364 d->cpupool = NULL;
365 spin_unlock(&cpupool_lock);
366 cpupool_dprintk("cpupool_rm_domain(dom=%d,pool=%d) n_dom %d\n",
367 d->domain_id, cpupool_id, n_dom);
368 return;
369 }
371 /*
372 * called to add a new cpu to pool admin
373 * we add a hotplugged cpu to the cpupool0 to be able to add it to dom0
374 */
375 static void cpupool_cpu_add(unsigned int cpu)
376 {
377 spin_lock(&cpupool_lock);
378 cpu_clear(cpu, cpupool_locked_cpus);
379 cpu_set(cpu, cpupool_free_cpus);
380 cpupool_assign_cpu_locked(cpupool0, cpu);
381 spin_unlock(&cpupool_lock);
382 }
384 /*
385 * called to remove a cpu from pool admin
386 * the cpu to be removed is locked to avoid removing it from dom0
387 * returns failure if not in pool0
388 */
389 static int cpupool_cpu_remove(unsigned int cpu)
390 {
391 int ret = 0;
393 spin_lock(&cpupool_lock);
394 if ( !cpu_isset(cpu, cpupool0->cpu_valid))
395 ret = -EBUSY;
396 else
397 cpu_set(cpu, cpupool_locked_cpus);
398 spin_unlock(&cpupool_lock);
400 return ret;
401 }
403 /*
404 * do cpupool related sysctl operations
405 */
406 int cpupool_do_sysctl(struct xen_sysctl_cpupool_op *op)
407 {
408 int ret;
409 struct cpupool *c;
411 switch ( op->op )
412 {
414 case XEN_SYSCTL_CPUPOOL_OP_CREATE:
415 {
416 int poolid;
418 poolid = (op->cpupool_id == XEN_SYSCTL_CPUPOOL_PAR_ANY) ?
419 CPUPOOLID_NONE: op->cpupool_id;
420 c = cpupool_create(poolid, op->sched_id, &ret);
421 if ( c != NULL )
422 {
423 op->cpupool_id = c->cpupool_id;
424 cpupool_put(c);
425 }
426 }
427 break;
429 case XEN_SYSCTL_CPUPOOL_OP_DESTROY:
430 {
431 c = cpupool_get_by_id(op->cpupool_id);
432 ret = -ENOENT;
433 if ( c == NULL )
434 break;
435 ret = cpupool_destroy(c);
436 cpupool_put(c);
437 }
438 break;
440 case XEN_SYSCTL_CPUPOOL_OP_INFO:
441 {
442 c = __cpupool_get_by_id(op->cpupool_id, 0);
443 ret = -ENOENT;
444 if ( c == NULL )
445 break;
446 op->cpupool_id = c->cpupool_id;
447 op->sched_id = c->sched->sched_id;
448 op->n_dom = c->n_dom;
449 ret = cpumask_to_xenctl_cpumap(&op->cpumap, &c->cpu_valid);
450 cpupool_put(c);
451 }
452 break;
454 case XEN_SYSCTL_CPUPOOL_OP_ADDCPU:
455 {
456 unsigned cpu;
458 cpu = op->cpu;
459 cpupool_dprintk("cpupool_assign_cpu(pool=%d,cpu=%d)\n",
460 op->cpupool_id, cpu);
461 spin_lock(&cpupool_lock);
462 if ( cpu == XEN_SYSCTL_CPUPOOL_PAR_ANY )
463 cpu = first_cpu(cpupool_free_cpus);
464 ret = -EINVAL;
465 if ( cpu >= NR_CPUS )
466 goto addcpu_out;
467 ret = -EBUSY;
468 if ( !cpu_isset(cpu, cpupool_free_cpus) )
469 goto addcpu_out;
470 c = cpupool_find_by_id(op->cpupool_id, 0);
471 ret = -ENOENT;
472 if ( c == NULL )
473 goto addcpu_out;
474 ret = cpupool_assign_cpu_locked(c, cpu);
475 addcpu_out:
476 spin_unlock(&cpupool_lock);
477 cpupool_dprintk("cpupool_assign_cpu(pool=%d,cpu=%d) ret %d\n",
478 op->cpupool_id, cpu, ret);
479 }
480 break;
482 case XEN_SYSCTL_CPUPOOL_OP_RMCPU:
483 {
484 unsigned cpu;
486 c = __cpupool_get_by_id(op->cpupool_id, 0);
487 ret = -ENOENT;
488 if ( c == NULL )
489 break;
490 cpu = op->cpu;
491 if ( cpu == XEN_SYSCTL_CPUPOOL_PAR_ANY )
492 cpu = last_cpu(c->cpu_valid);
493 ret = (cpu < NR_CPUS) ? cpupool_unassign_cpu(c, cpu) : -EINVAL;
494 cpupool_put(c);
495 }
496 break;
498 case XEN_SYSCTL_CPUPOOL_OP_MOVEDOMAIN:
499 {
500 struct domain *d;
502 ret = -EINVAL;
503 if ( op->domid == 0 )
504 break;
505 ret = -ESRCH;
506 d = rcu_lock_domain_by_id(op->domid);
507 if ( d == NULL )
508 break;
509 if ( d->cpupool == NULL )
510 {
511 ret = -EINVAL;
512 rcu_unlock_domain(d);
513 break;
514 }
515 if ( op->cpupool_id == d->cpupool->cpupool_id )
516 {
517 ret = 0;
518 rcu_unlock_domain(d);
519 break;
520 }
521 cpupool_dprintk("cpupool move_domain(dom=%d)->pool=%d\n",
522 d->domain_id, op->cpupool_id);
523 ret = -ENOENT;
524 spin_lock(&cpupool_lock);
525 c = cpupool_find_by_id(op->cpupool_id, 1);
526 if ( (c != NULL) && cpus_weight(c->cpu_valid) )
527 {
528 d->cpupool->n_dom--;
529 ret = sched_move_domain(d, c);
530 if ( ret )
531 d->cpupool->n_dom++;
532 else
533 c->n_dom++;
534 }
535 spin_unlock(&cpupool_lock);
536 cpupool_dprintk("cpupool move_domain(dom=%d)->pool=%d ret %d\n",
537 d->domain_id, op->cpupool_id, ret);
538 rcu_unlock_domain(d);
539 }
540 break;
542 case XEN_SYSCTL_CPUPOOL_OP_FREEINFO:
543 {
544 ret = cpumask_to_xenctl_cpumap(
545 &op->cpumap, &cpupool_free_cpus);
546 }
547 break;
549 default:
550 ret = -ENOSYS;
551 break;
552 }
554 return ret;
555 }
557 void schedule_dump(struct cpupool *c);
559 void dump_runq(unsigned char key)
560 {
561 unsigned long flags;
562 s_time_t now = NOW();
563 struct cpupool **c;
565 spin_lock(&cpupool_lock);
566 local_irq_save(flags);
568 printk("sched_smt_power_savings: %s\n",
569 sched_smt_power_savings? "enabled":"disabled");
570 printk("NOW=0x%08X%08X\n", (u32)(now>>32), (u32)now);
572 printk("Idle cpupool:\n");
573 schedule_dump(NULL);
575 for_each_cpupool(c)
576 {
577 printk("Cpupool %d:\n", (*c)->cpupool_id);
578 schedule_dump(*c);
579 }
581 local_irq_restore(flags);
582 spin_unlock(&cpupool_lock);
583 }
585 static int cpu_callback(
586 struct notifier_block *nfb, unsigned long action, void *hcpu)
587 {
588 unsigned int cpu = (unsigned long)hcpu;
589 int rc = 0;
591 switch ( action )
592 {
593 case CPU_DOWN_FAILED:
594 case CPU_ONLINE:
595 cpupool_cpu_add(cpu);
596 break;
597 case CPU_DOWN_PREPARE:
598 rc = cpupool_cpu_remove(cpu);
599 break;
600 default:
601 break;
602 }
604 return !rc ? NOTIFY_DONE : notifier_from_errno(rc);
605 }
607 static struct notifier_block cpu_nfb = {
608 .notifier_call = cpu_callback
609 };
611 static int __init cpupool_presmp_init(void)
612 {
613 int err;
614 void *cpu = (void *)(long)smp_processor_id();
615 cpupool0 = cpupool_create(0, 0, &err);
616 BUG_ON(cpupool0 == NULL);
617 cpupool_put(cpupool0);
618 cpu_callback(&cpu_nfb, CPU_ONLINE, cpu);
619 register_cpu_notifier(&cpu_nfb);
620 return 0;
621 }
622 presmp_initcall(cpupool_presmp_init);
624 /*
625 * Local variables:
626 * mode: C
627 * c-set-style: "BSD"
628 * c-basic-offset: 4
629 * tab-width: 4
630 * indent-tabs-mode: nil
631 * End:
632 */