debuggers.hg

view xen/arch/x86/irq.c @ 22906:700ac6445812

Now add KDB to the non-kdb tree
author Mukesh Rathor
date Thu Feb 03 15:42:41 2011 -0800 (2011-02-03)
parents e7f44fb4ecc3
children
line source
1 /******************************************************************************
2 * arch/x86/irq.c
3 *
4 * Portions of this file are:
5 * Copyright (C) 1992, 1998 Linus Torvalds, Ingo Molnar
6 */
8 #include <xen/config.h>
9 #include <xen/init.h>
10 #include <xen/delay.h>
11 #include <xen/errno.h>
12 #include <xen/event.h>
13 #include <xen/irq.h>
14 #include <xen/perfc.h>
15 #include <xen/sched.h>
16 #include <xen/keyhandler.h>
17 #include <xen/compat.h>
18 #include <xen/iocap.h>
19 #include <xen/iommu.h>
20 #include <xen/trace.h>
21 #include <asm/msi.h>
22 #include <asm/current.h>
23 #include <asm/flushtlb.h>
24 #include <asm/mach-generic/mach_apic.h>
25 #include <public/physdev.h>
27 /* opt_noirqbalance: If true, software IRQ balancing/affinity is disabled. */
28 bool_t __read_mostly opt_noirqbalance = 0;
29 boolean_param("noirqbalance", opt_noirqbalance);
31 unsigned int __read_mostly nr_irqs_gsi = 16;
32 unsigned int __read_mostly nr_irqs;
33 integer_param("nr_irqs", nr_irqs);
35 u8 __read_mostly *irq_vector;
36 struct irq_desc __read_mostly *irq_desc = NULL;
38 int __read_mostly *irq_status = NULL;
39 #define IRQ_UNUSED (0)
40 #define IRQ_USED (1)
41 #define IRQ_RSVD (2)
43 #define IRQ_VECTOR_UNASSIGNED (0)
45 static DECLARE_BITMAP(used_vectors, NR_VECTORS);
47 struct irq_cfg __read_mostly *irq_cfg = NULL;
49 static DEFINE_SPINLOCK(vector_lock);
51 DEFINE_PER_CPU(vector_irq_t, vector_irq);
53 DEFINE_PER_CPU(struct cpu_user_regs *, __irq_regs);
55 static LIST_HEAD(irq_ratelimit_list);
56 static DEFINE_SPINLOCK(irq_ratelimit_lock);
57 static struct timer irq_ratelimit_timer;
59 /* irq_ratelimit: the max irq rate allowed in every 10ms, set 0 to disable */
60 static unsigned int __read_mostly irq_ratelimit_threshold = 10000;
61 integer_param("irq_ratelimit", irq_ratelimit_threshold);
63 /* Must be called when irq disabled */
64 void lock_vector_lock(void)
65 {
66 /* Used to the online set of cpus does not change
67 * during assign_irq_vector.
68 */
69 spin_lock(&vector_lock);
70 }
72 void unlock_vector_lock(void)
73 {
74 spin_unlock(&vector_lock);
75 }
77 static int __init __bind_irq_vector(int irq, int vector, cpumask_t cpu_mask)
78 {
79 cpumask_t online_mask;
80 int cpu;
81 struct irq_cfg *cfg = irq_cfg(irq);
83 BUG_ON((unsigned)irq >= nr_irqs);
84 BUG_ON((unsigned)vector >= NR_VECTORS);
86 cpus_and(online_mask, cpu_mask, cpu_online_map);
87 if (cpus_empty(online_mask))
88 return -EINVAL;
89 if ((cfg->vector == vector) && cpus_equal(cfg->cpu_mask, online_mask))
90 return 0;
91 if (cfg->vector != IRQ_VECTOR_UNASSIGNED)
92 return -EBUSY;
93 for_each_cpu_mask(cpu, online_mask)
94 per_cpu(vector_irq, cpu)[vector] = irq;
95 cfg->vector = vector;
96 cfg->cpu_mask = online_mask;
97 irq_status[irq] = IRQ_USED;
98 if (IO_APIC_IRQ(irq))
99 irq_vector[irq] = vector;
100 return 0;
101 }
103 int __init bind_irq_vector(int irq, int vector, cpumask_t cpu_mask)
104 {
105 unsigned long flags;
106 int ret;
108 spin_lock_irqsave(&vector_lock, flags);
109 ret = __bind_irq_vector(irq, vector, cpu_mask);
110 spin_unlock_irqrestore(&vector_lock, flags);
111 return ret;
112 }
114 static inline int find_unassigned_irq(void)
115 {
116 int irq;
118 for (irq = nr_irqs_gsi; irq < nr_irqs; irq++)
119 if (irq_status[irq] == IRQ_UNUSED)
120 return irq;
121 return -ENOSPC;
122 }
124 /*
125 * Dynamic irq allocate and deallocation for MSI
126 */
127 int create_irq(void)
128 {
129 unsigned long flags;
130 int irq, ret;
131 irq = -ENOSPC;
133 spin_lock_irqsave(&vector_lock, flags);
135 irq = find_unassigned_irq();
136 if (irq < 0)
137 goto out;
138 ret = __assign_irq_vector(irq, irq_cfg(irq), TARGET_CPUS);
139 if (ret < 0)
140 irq = ret;
141 out:
142 spin_unlock_irqrestore(&vector_lock, flags);
144 return irq;
145 }
147 static void dynamic_irq_cleanup(unsigned int irq)
148 {
149 struct irq_desc *desc = irq_to_desc(irq);
150 unsigned long flags;
151 struct irqaction *action;
153 spin_lock_irqsave(&desc->lock, flags);
154 desc->status |= IRQ_DISABLED;
155 desc->handler->shutdown(irq);
156 action = desc->action;
157 desc->action = NULL;
158 desc->depth = 1;
159 desc->msi_desc = NULL;
160 desc->handler = &no_irq_type;
161 cpus_setall(desc->affinity);
162 spin_unlock_irqrestore(&desc->lock, flags);
164 /* Wait to make sure it's not being used on another CPU */
165 do { smp_mb(); } while ( desc->status & IRQ_INPROGRESS );
167 if (action)
168 xfree(action);
169 }
171 static void init_one_irq_status(int irq);
173 static void __clear_irq_vector(int irq)
174 {
175 int cpu, vector;
176 cpumask_t tmp_mask;
177 struct irq_cfg *cfg = irq_cfg(irq);
179 BUG_ON(!cfg->vector);
181 vector = cfg->vector;
182 cpus_and(tmp_mask, cfg->cpu_mask, cpu_online_map);
184 for_each_cpu_mask(cpu, tmp_mask)
185 per_cpu(vector_irq, cpu)[vector] = -1;
187 cfg->vector = IRQ_VECTOR_UNASSIGNED;
188 cpus_clear(cfg->cpu_mask);
189 init_one_irq_status(irq);
191 if (likely(!cfg->move_in_progress))
192 return;
193 for_each_cpu_mask(cpu, tmp_mask) {
194 for (vector = FIRST_DYNAMIC_VECTOR; vector <= LAST_DYNAMIC_VECTOR;
195 vector++) {
196 if (per_cpu(vector_irq, cpu)[vector] != irq)
197 continue;
198 per_cpu(vector_irq, cpu)[vector] = -1;
199 break;
200 }
201 }
203 cfg->move_in_progress = 0;
204 }
206 void clear_irq_vector(int irq)
207 {
208 unsigned long flags;
210 spin_lock_irqsave(&vector_lock, flags);
211 __clear_irq_vector(irq);
212 spin_unlock_irqrestore(&vector_lock, flags);
213 }
215 void destroy_irq(unsigned int irq)
216 {
217 BUG_ON(!MSI_IRQ(irq));
218 dynamic_irq_cleanup(irq);
219 clear_irq_vector(irq);
220 }
222 int irq_to_vector(int irq)
223 {
224 int vector = -1;
225 struct irq_cfg *cfg;
227 BUG_ON(irq >= nr_irqs || irq < 0);
229 if (IO_APIC_IRQ(irq))
230 vector = irq_vector[irq];
231 else if(MSI_IRQ(irq)) {
232 cfg = irq_cfg(irq);
233 vector = cfg->vector;
234 } else
235 vector = LEGACY_VECTOR(irq);
237 return vector;
238 }
240 static void init_one_irq_desc(struct irq_desc *desc)
241 {
242 desc->status = IRQ_DISABLED;
243 desc->handler = &no_irq_type;
244 desc->action = NULL;
245 desc->depth = 1;
246 desc->msi_desc = NULL;
247 spin_lock_init(&desc->lock);
248 cpus_setall(desc->affinity);
249 INIT_LIST_HEAD(&desc->rl_link);
250 }
252 static void init_one_irq_status(int irq)
253 {
254 irq_status[irq] = IRQ_UNUSED;
255 }
257 static void init_one_irq_cfg(struct irq_cfg *cfg)
258 {
259 cfg->vector = IRQ_VECTOR_UNASSIGNED;
260 cpus_clear(cfg->cpu_mask);
261 cpus_clear(cfg->old_cpu_mask);
262 }
264 int init_irq_data(void)
265 {
266 struct irq_desc *desc;
267 struct irq_cfg *cfg;
268 int irq, vector;
270 for (vector = 0; vector < NR_VECTORS; ++vector)
271 this_cpu(vector_irq)[vector] = -1;
273 irq_desc = xmalloc_array(struct irq_desc, nr_irqs);
274 irq_cfg = xmalloc_array(struct irq_cfg, nr_irqs);
275 irq_status = xmalloc_array(int, nr_irqs);
276 irq_vector = xmalloc_array(u8, nr_irqs_gsi);
278 if ( !irq_desc || !irq_cfg || !irq_status ||! irq_vector )
279 return -ENOMEM;
281 memset(irq_desc, 0, nr_irqs * sizeof(*irq_desc));
282 memset(irq_cfg, 0, nr_irqs * sizeof(*irq_cfg));
283 memset(irq_status, 0, nr_irqs * sizeof(*irq_status));
284 memset(irq_vector, 0, nr_irqs_gsi * sizeof(*irq_vector));
286 for (irq = 0; irq < nr_irqs; irq++) {
287 desc = irq_to_desc(irq);
288 cfg = irq_cfg(irq);
289 desc->irq = irq;
290 desc->chip_data = cfg;
291 init_one_irq_desc(desc);
292 init_one_irq_cfg(cfg);
293 init_one_irq_status(irq);
294 }
296 /* Never allocate the hypercall vector or Linux/BSD fast-trap vector. */
297 set_bit(LEGACY_SYSCALL_VECTOR, used_vectors);
298 set_bit(HYPERCALL_VECTOR, used_vectors);
300 /* IRQ_MOVE_CLEANUP_VECTOR used for clean up vectors */
301 set_bit(IRQ_MOVE_CLEANUP_VECTOR, used_vectors);
303 return 0;
304 }
306 static void __do_IRQ_guest(int vector);
308 void no_action(int cpl, void *dev_id, struct cpu_user_regs *regs) { }
310 static void enable_none(unsigned int vector) { }
311 static unsigned int startup_none(unsigned int vector) { return 0; }
312 static void disable_none(unsigned int vector) { }
313 static void ack_none(unsigned int irq)
314 {
315 ack_bad_irq(irq);
316 }
318 #define shutdown_none disable_none
319 #define end_none enable_none
321 hw_irq_controller no_irq_type = {
322 "none",
323 startup_none,
324 shutdown_none,
325 enable_none,
326 disable_none,
327 ack_none,
328 end_none
329 };
331 int __assign_irq_vector(int irq, struct irq_cfg *cfg, const cpumask_t *mask)
332 {
333 /*
334 * NOTE! The local APIC isn't very good at handling
335 * multiple interrupts at the same interrupt level.
336 * As the interrupt level is determined by taking the
337 * vector number and shifting that right by 4, we
338 * want to spread these out a bit so that they don't
339 * all fall in the same interrupt level.
340 *
341 * Also, we've got to be careful not to trash gate
342 * 0x80, because int 0x80 is hm, kind of importantish. ;)
343 */
344 static int current_vector = FIRST_DYNAMIC_VECTOR, current_offset = 0;
345 unsigned int old_vector;
346 int cpu, err;
347 cpumask_t tmp_mask;
349 old_vector = irq_to_vector(irq);
350 if (old_vector) {
351 cpus_and(tmp_mask, *mask, cpu_online_map);
352 if (cpus_intersects(tmp_mask, cfg->cpu_mask)) {
353 cfg->vector = old_vector;
354 return 0;
355 }
356 }
358 if ((cfg->move_in_progress) || cfg->move_cleanup_count)
359 return -EAGAIN;
361 err = -ENOSPC;
362 for_each_cpu_mask(cpu, *mask) {
363 int new_cpu;
364 int vector, offset;
366 /* Only try and allocate irqs on cpus that are present. */
367 if (!cpu_online(cpu))
368 continue;
370 cpus_and(tmp_mask, *vector_allocation_cpumask(cpu), cpu_online_map);
372 vector = current_vector;
373 offset = current_offset;
374 next:
375 vector += 8;
376 if (vector > LAST_DYNAMIC_VECTOR) {
377 /* If out of vectors on large boxen, must share them. */
378 offset = (offset + 1) % 8;
379 vector = FIRST_DYNAMIC_VECTOR + offset;
380 }
381 if (unlikely(current_vector == vector))
382 continue;
384 if (test_bit(vector, used_vectors))
385 goto next;
387 for_each_cpu_mask(new_cpu, tmp_mask)
388 if (per_cpu(vector_irq, new_cpu)[vector] != -1)
389 goto next;
390 /* Found one! */
391 current_vector = vector;
392 current_offset = offset;
393 if (old_vector) {
394 cfg->move_in_progress = 1;
395 cpus_copy(cfg->old_cpu_mask, cfg->cpu_mask);
396 }
397 for_each_cpu_mask(new_cpu, tmp_mask)
398 per_cpu(vector_irq, new_cpu)[vector] = irq;
399 cfg->vector = vector;
400 cpus_copy(cfg->cpu_mask, tmp_mask);
402 irq_status[irq] = IRQ_USED;
403 if (IO_APIC_IRQ(irq))
404 irq_vector[irq] = vector;
405 err = 0;
406 break;
407 }
408 return err;
409 }
411 int assign_irq_vector(int irq)
412 {
413 int ret;
414 unsigned long flags;
415 struct irq_cfg *cfg = &irq_cfg[irq];
416 struct irq_desc *desc = irq_to_desc(irq);
418 BUG_ON(irq >= nr_irqs || irq <0);
420 spin_lock_irqsave(&vector_lock, flags);
421 ret = __assign_irq_vector(irq, cfg, TARGET_CPUS);
422 if (!ret) {
423 ret = cfg->vector;
424 cpus_copy(desc->affinity, cfg->cpu_mask);
425 }
426 spin_unlock_irqrestore(&vector_lock, flags);
427 return ret;
428 }
430 /*
431 * Initialize vector_irq on a new cpu. This function must be called
432 * with vector_lock held.
433 */
434 void __setup_vector_irq(int cpu)
435 {
436 int irq, vector;
437 struct irq_cfg *cfg;
439 /* Clear vector_irq */
440 for (vector = 0; vector < NR_VECTORS; ++vector)
441 per_cpu(vector_irq, cpu)[vector] = -1;
442 /* Mark the inuse vectors */
443 for (irq = 0; irq < nr_irqs; ++irq) {
444 cfg = irq_cfg(irq);
445 if (!cpu_isset(cpu, cfg->cpu_mask))
446 continue;
447 vector = irq_to_vector(irq);
448 per_cpu(vector_irq, cpu)[vector] = irq;
449 }
450 }
452 void move_masked_irq(int irq)
453 {
454 struct irq_desc *desc = irq_to_desc(irq);
456 if (likely(!(desc->status & IRQ_MOVE_PENDING)))
457 return;
459 desc->status &= ~IRQ_MOVE_PENDING;
461 if (unlikely(cpus_empty(desc->pending_mask)))
462 return;
464 if (!desc->handler->set_affinity)
465 return;
467 /*
468 * If there was a valid mask to work with, please do the disable,
469 * re-program, enable sequence. This is *not* particularly important for
470 * level triggered but in a edge trigger case, we might be setting rte when
471 * an active trigger is comming in. This could cause some ioapics to
472 * mal-function. Being paranoid i guess!
473 *
474 * For correct operation this depends on the caller masking the irqs.
475 */
476 if (likely(cpus_intersects(desc->pending_mask, cpu_online_map)))
477 desc->handler->set_affinity(irq, desc->pending_mask);
479 cpus_clear(desc->pending_mask);
480 }
482 void move_native_irq(int irq)
483 {
484 struct irq_desc *desc = irq_to_desc(irq);
486 if (likely(!(desc->status & IRQ_MOVE_PENDING)))
487 return;
489 if (unlikely(desc->status & IRQ_DISABLED))
490 return;
492 desc->handler->disable(irq);
493 move_masked_irq(irq);
494 desc->handler->enable(irq);
495 }
497 /* For re-setting irq interrupt affinity for specific irq */
498 void irq_set_affinity(struct irq_desc *desc, const cpumask_t *mask)
499 {
500 if (!desc->handler->set_affinity)
501 return;
503 ASSERT(spin_is_locked(&desc->lock));
504 desc->status &= ~IRQ_MOVE_PENDING;
505 wmb();
506 cpus_copy(desc->pending_mask, *mask);
507 wmb();
508 desc->status |= IRQ_MOVE_PENDING;
509 }
511 void pirq_set_affinity(struct domain *d, int pirq, const cpumask_t *mask)
512 {
513 unsigned long flags;
514 struct irq_desc *desc = domain_spin_lock_irq_desc(d, pirq, &flags);
516 if ( !desc )
517 return;
518 irq_set_affinity(desc, mask);
519 spin_unlock_irqrestore(&desc->lock, flags);
520 }
522 DEFINE_PER_CPU(unsigned int, irq_count);
524 asmlinkage void do_IRQ(struct cpu_user_regs *regs)
525 {
526 struct irqaction *action;
527 uint32_t tsc_in;
528 struct irq_desc *desc;
529 unsigned int vector = regs->entry_vector;
530 int irq = __get_cpu_var(vector_irq[vector]);
531 struct cpu_user_regs *old_regs = set_irq_regs(regs);
533 perfc_incr(irqs);
535 this_cpu(irq_count)++;
537 if (irq < 0) {
538 ack_APIC_irq();
539 printk("%s: %d.%d No irq handler for vector (irq %d)\n",
540 __func__, smp_processor_id(), vector, irq);
541 set_irq_regs(old_regs);
542 return;
543 }
545 irq_enter();
547 desc = irq_to_desc(irq);
549 spin_lock(&desc->lock);
550 desc->handler->ack(irq);
552 if ( likely(desc->status & IRQ_GUEST) )
553 {
554 if ( irq_ratelimit_timer.function && /* irq rate limiting enabled? */
555 unlikely(desc->rl_cnt++ >= irq_ratelimit_threshold) )
556 {
557 s_time_t now = NOW();
558 if ( now < (desc->rl_quantum_start + MILLISECS(10)) )
559 {
560 desc->handler->disable(irq);
561 /*
562 * If handler->disable doesn't actually mask the interrupt, a
563 * disabled irq still can fire. This check also avoids possible
564 * deadlocks if ratelimit_timer_fn runs at the same time.
565 */
566 if ( likely(list_empty(&desc->rl_link)) )
567 {
568 spin_lock(&irq_ratelimit_lock);
569 if ( list_empty(&irq_ratelimit_list) )
570 set_timer(&irq_ratelimit_timer, now + MILLISECS(10));
571 list_add(&desc->rl_link, &irq_ratelimit_list);
572 spin_unlock(&irq_ratelimit_lock);
573 }
574 goto out;
575 }
576 desc->rl_cnt = 0;
577 desc->rl_quantum_start = now;
578 }
580 tsc_in = tb_init_done ? get_cycles() : 0;
581 __do_IRQ_guest(irq);
582 TRACE_3D(TRC_TRACE_IRQ, irq, tsc_in, get_cycles());
583 goto out_no_end;
584 }
586 desc->status &= ~IRQ_REPLAY;
587 desc->status |= IRQ_PENDING;
589 /*
590 * Since we set PENDING, if another processor is handling a different
591 * instance of this same irq, the other processor will take care of it.
592 */
593 if ( desc->status & (IRQ_DISABLED | IRQ_INPROGRESS) )
594 goto out;
596 desc->status |= IRQ_INPROGRESS;
598 action = desc->action;
599 while ( desc->status & IRQ_PENDING )
600 {
601 desc->status &= ~IRQ_PENDING;
602 spin_unlock_irq(&desc->lock);
603 tsc_in = tb_init_done ? get_cycles() : 0;
604 action->handler(irq, action->dev_id, regs);
605 TRACE_3D(TRC_TRACE_IRQ, irq, tsc_in, get_cycles());
606 spin_lock_irq(&desc->lock);
607 }
609 desc->status &= ~IRQ_INPROGRESS;
611 out:
612 desc->handler->end(irq);
613 out_no_end:
614 spin_unlock(&desc->lock);
615 irq_exit();
616 set_irq_regs(old_regs);
617 }
619 static void irq_ratelimit_timer_fn(void *data)
620 {
621 struct irq_desc *desc, *tmp;
622 unsigned long flags;
624 spin_lock_irqsave(&irq_ratelimit_lock, flags);
626 list_for_each_entry_safe ( desc, tmp, &irq_ratelimit_list, rl_link )
627 {
628 spin_lock(&desc->lock);
629 desc->handler->enable(desc->irq);
630 list_del(&desc->rl_link);
631 INIT_LIST_HEAD(&desc->rl_link);
632 spin_unlock(&desc->lock);
633 }
635 spin_unlock_irqrestore(&irq_ratelimit_lock, flags);
636 }
638 static int __init irq_ratelimit_init(void)
639 {
640 if ( irq_ratelimit_threshold )
641 init_timer(&irq_ratelimit_timer, irq_ratelimit_timer_fn, NULL, 0);
642 return 0;
643 }
644 __initcall(irq_ratelimit_init);
646 int request_irq(unsigned int irq,
647 void (*handler)(int, void *, struct cpu_user_regs *),
648 unsigned long irqflags, const char * devname, void *dev_id)
649 {
650 struct irqaction * action;
651 int retval;
653 /*
654 * Sanity-check: shared interrupts must pass in a real dev-ID,
655 * otherwise we'll have trouble later trying to figure out
656 * which interrupt is which (messes up the interrupt freeing
657 * logic etc).
658 */
659 if (irq >= nr_irqs)
660 return -EINVAL;
661 if (!handler)
662 return -EINVAL;
664 action = xmalloc(struct irqaction);
665 if (!action)
666 return -ENOMEM;
668 action->handler = handler;
669 action->name = devname;
670 action->dev_id = dev_id;
671 action->free_on_release = 1;
673 retval = setup_irq(irq, action);
674 if (retval)
675 xfree(action);
677 return retval;
678 }
680 void release_irq(unsigned int irq)
681 {
682 struct irq_desc *desc;
683 unsigned long flags;
684 struct irqaction *action;
686 desc = irq_to_desc(irq);
688 spin_lock_irqsave(&desc->lock,flags);
689 action = desc->action;
690 desc->action = NULL;
691 desc->depth = 1;
692 desc->status |= IRQ_DISABLED;
693 desc->handler->shutdown(irq);
694 spin_unlock_irqrestore(&desc->lock,flags);
696 /* Wait to make sure it's not being used on another CPU */
697 do { smp_mb(); } while ( desc->status & IRQ_INPROGRESS );
699 if (action && action->free_on_release)
700 xfree(action);
701 }
703 int setup_irq(unsigned int irq, struct irqaction *new)
704 {
705 struct irq_desc *desc;
706 unsigned long flags;
708 desc = irq_to_desc(irq);
710 spin_lock_irqsave(&desc->lock,flags);
712 if ( desc->action != NULL )
713 {
714 spin_unlock_irqrestore(&desc->lock,flags);
715 return -EBUSY;
716 }
718 desc->action = new;
719 desc->depth = 0;
720 desc->status &= ~IRQ_DISABLED;
721 desc->handler->startup(irq);
723 spin_unlock_irqrestore(&desc->lock,flags);
725 return 0;
726 }
729 /*
730 * HANDLING OF GUEST-BOUND PHYSICAL IRQS
731 */
733 #define IRQ_MAX_GUESTS 7
734 typedef struct {
735 u8 nr_guests;
736 u8 in_flight;
737 u8 shareable;
738 u8 ack_type;
739 #define ACKTYPE_NONE 0 /* No final acknowledgement is required */
740 #define ACKTYPE_UNMASK 1 /* Unmask PIC hardware (from any CPU) */
741 #define ACKTYPE_EOI 2 /* EOI on the CPU that was interrupted */
742 cpumask_t cpu_eoi_map; /* CPUs that need to EOI this interrupt */
743 struct timer eoi_timer;
744 struct domain *guest[IRQ_MAX_GUESTS];
745 } irq_guest_action_t;
747 /*
748 * Stack of interrupts awaiting EOI on each CPU. These must be popped in
749 * order, as only the current highest-priority pending irq can be EOIed.
750 */
751 struct pending_eoi {
752 u32 ready:1; /* Ready for EOI now? */
753 u32 irq:23; /* irq of the vector */
754 u32 vector:8; /* vector awaiting EOI */
755 };
757 static DEFINE_PER_CPU(struct pending_eoi, pending_eoi[NR_DYNAMIC_VECTORS]);
758 #define pending_eoi_sp(p) ((p)[NR_DYNAMIC_VECTORS-1].vector)
760 bool_t cpu_has_pending_apic_eoi(void)
761 {
762 return (pending_eoi_sp(this_cpu(pending_eoi)) != 0);
763 }
765 static inline void set_pirq_eoi(struct domain *d, unsigned int irq)
766 {
767 if ( d->arch.pirq_eoi_map )
768 set_bit(irq, d->arch.pirq_eoi_map);
769 }
771 static inline void clear_pirq_eoi(struct domain *d, unsigned int irq)
772 {
773 if ( d->arch.pirq_eoi_map )
774 clear_bit(irq, d->arch.pirq_eoi_map);
775 }
777 static void _irq_guest_eoi(struct irq_desc *desc)
778 {
779 irq_guest_action_t *action = (irq_guest_action_t *)desc->action;
780 unsigned int i, irq = desc - irq_desc;
782 if ( !(desc->status & IRQ_GUEST_EOI_PENDING) )
783 return;
785 for ( i = 0; i < action->nr_guests; ++i )
786 clear_pirq_eoi(action->guest[i],
787 domain_irq_to_pirq(action->guest[i], irq));
789 desc->status &= ~(IRQ_INPROGRESS|IRQ_GUEST_EOI_PENDING);
790 desc->handler->enable(irq);
791 }
793 static void set_eoi_ready(void *data);
795 static void irq_guest_eoi_timer_fn(void *data)
796 {
797 struct irq_desc *desc = data;
798 unsigned int irq = desc - irq_desc;
799 irq_guest_action_t *action;
800 cpumask_t cpu_eoi_map;
801 unsigned long flags;
803 spin_lock_irqsave(&desc->lock, flags);
805 if ( !(desc->status & IRQ_GUEST) )
806 goto out;
808 action = (irq_guest_action_t *)desc->action;
810 if ( action->ack_type != ACKTYPE_NONE )
811 {
812 unsigned int i;
813 for ( i = 0; i < action->nr_guests; i++ )
814 {
815 struct domain *d = action->guest[i];
816 unsigned int pirq = domain_irq_to_pirq(d, irq);
817 if ( test_and_clear_bit(pirq, d->pirq_mask) )
818 action->in_flight--;
819 }
820 }
822 if ( action->in_flight != 0 )
823 goto out;
825 switch ( action->ack_type )
826 {
827 case ACKTYPE_UNMASK:
828 desc->handler->end(irq);
829 break;
830 case ACKTYPE_EOI:
831 cpu_eoi_map = action->cpu_eoi_map;
832 spin_unlock_irq(&desc->lock);
833 on_selected_cpus(&cpu_eoi_map, set_eoi_ready, desc, 0);
834 spin_lock_irq(&desc->lock);
835 break;
836 case ACKTYPE_NONE:
837 _irq_guest_eoi(desc);
838 break;
839 }
841 out:
842 spin_unlock_irqrestore(&desc->lock, flags);
843 }
845 static void __do_IRQ_guest(int irq)
846 {
847 struct irq_desc *desc = irq_to_desc(irq);
848 irq_guest_action_t *action = (irq_guest_action_t *)desc->action;
849 struct domain *d;
850 int i, sp, already_pending = 0;
851 struct pending_eoi *peoi = this_cpu(pending_eoi);
852 int vector = get_irq_regs()->entry_vector;
854 if ( unlikely(action->nr_guests == 0) )
855 {
856 /* An interrupt may slip through while freeing an ACKTYPE_EOI irq. */
857 ASSERT(action->ack_type == ACKTYPE_EOI);
858 ASSERT(desc->status & IRQ_DISABLED);
859 desc->handler->end(irq);
860 return;
861 }
863 if ( action->ack_type == ACKTYPE_EOI )
864 {
865 sp = pending_eoi_sp(peoi);
866 ASSERT((sp == 0) || (peoi[sp-1].vector < vector));
867 ASSERT(sp < (NR_DYNAMIC_VECTORS-1));
868 peoi[sp].irq = irq;
869 peoi[sp].vector = vector;
870 peoi[sp].ready = 0;
871 pending_eoi_sp(peoi) = sp+1;
872 cpu_set(smp_processor_id(), action->cpu_eoi_map);
873 }
875 for ( i = 0; i < action->nr_guests; i++ )
876 {
877 unsigned int pirq;
878 d = action->guest[i];
879 pirq = domain_irq_to_pirq(d, irq);
880 if ( (action->ack_type != ACKTYPE_NONE) &&
881 !test_and_set_bit(pirq, d->pirq_mask) )
882 action->in_flight++;
883 if ( hvm_do_IRQ_dpci(d, pirq) )
884 {
885 if ( action->ack_type == ACKTYPE_NONE )
886 {
887 already_pending += !!(desc->status & IRQ_INPROGRESS);
888 desc->status |= IRQ_INPROGRESS; /* cleared during hvm eoi */
889 }
890 }
891 else if ( send_guest_pirq(d, pirq) &&
892 (action->ack_type == ACKTYPE_NONE) )
893 {
894 already_pending++;
895 }
896 }
898 stop_timer(&action->eoi_timer);
900 if ( (action->ack_type == ACKTYPE_NONE) &&
901 (already_pending == action->nr_guests) )
902 {
903 desc->handler->disable(irq);
904 desc->status |= IRQ_GUEST_EOI_PENDING;
905 for ( i = 0; i < already_pending; ++i )
906 {
907 d = action->guest[i];
908 set_pirq_eoi(d, domain_irq_to_pirq(d, irq));
909 /*
910 * Could check here whether the guest unmasked the event by now
911 * (or perhaps just re-issue the send_guest_pirq()), and if it
912 * can now accept the event,
913 * - clear all the pirq_eoi bits we already set,
914 * - re-enable the vector, and
915 * - skip the timer setup below.
916 */
917 }
918 }
920 migrate_timer(&action->eoi_timer, smp_processor_id());
921 set_timer(&action->eoi_timer, NOW() + MILLISECS(1));
922 }
924 /*
925 * Retrieve Xen irq-descriptor corresponding to a domain-specific irq.
926 * The descriptor is returned locked. This function is safe against changes
927 * to the per-domain irq-to-vector mapping.
928 */
929 struct irq_desc *domain_spin_lock_irq_desc(
930 struct domain *d, int pirq, unsigned long *pflags)
931 {
932 int irq;
933 unsigned long flags;
934 struct irq_desc *desc;
936 for ( ; ; )
937 {
938 irq = domain_pirq_to_irq(d, pirq);
939 if ( irq <= 0 )
940 return NULL;
941 desc = irq_to_desc(irq);
942 spin_lock_irqsave(&desc->lock, flags);
943 if ( irq == domain_pirq_to_irq(d, pirq) )
944 break;
945 spin_unlock_irqrestore(&desc->lock, flags);
946 }
948 if ( pflags != NULL )
949 *pflags = flags;
950 return desc;
951 }
953 /* Flush all ready EOIs from the top of this CPU's pending-EOI stack. */
954 static void flush_ready_eoi(void)
955 {
956 struct pending_eoi *peoi = this_cpu(pending_eoi);
957 struct irq_desc *desc;
958 int irq, sp;
960 ASSERT(!local_irq_is_enabled());
962 sp = pending_eoi_sp(peoi);
964 while ( (--sp >= 0) && peoi[sp].ready )
965 {
966 irq = peoi[sp].irq;
967 ASSERT(irq > 0);
968 desc = irq_to_desc(irq);
969 spin_lock(&desc->lock);
970 desc->handler->end(irq);
971 spin_unlock(&desc->lock);
972 }
974 pending_eoi_sp(peoi) = sp+1;
975 }
977 static void __set_eoi_ready(struct irq_desc *desc)
978 {
979 irq_guest_action_t *action = (irq_guest_action_t *)desc->action;
980 struct pending_eoi *peoi = this_cpu(pending_eoi);
981 int irq, sp;
983 irq = desc - irq_desc;
985 if ( !(desc->status & IRQ_GUEST) ||
986 (action->in_flight != 0) ||
987 !cpu_test_and_clear(smp_processor_id(), action->cpu_eoi_map) )
988 return;
990 sp = pending_eoi_sp(peoi);
992 do {
993 ASSERT(sp > 0);
994 } while ( peoi[--sp].irq != irq );
995 ASSERT(!peoi[sp].ready);
996 peoi[sp].ready = 1;
997 }
999 /* Mark specified IRQ as ready-for-EOI (if it really is) and attempt to EOI. */
1000 static void set_eoi_ready(void *data)
1002 struct irq_desc *desc = data;
1004 ASSERT(!local_irq_is_enabled());
1006 spin_lock(&desc->lock);
1007 __set_eoi_ready(desc);
1008 spin_unlock(&desc->lock);
1010 flush_ready_eoi();
1013 static void __pirq_guest_eoi(struct domain *d, int pirq)
1015 struct irq_desc *desc;
1016 irq_guest_action_t *action;
1017 cpumask_t cpu_eoi_map;
1018 int irq;
1020 ASSERT(local_irq_is_enabled());
1021 desc = domain_spin_lock_irq_desc(d, pirq, NULL);
1022 if ( desc == NULL )
1023 return;
1025 action = (irq_guest_action_t *)desc->action;
1026 irq = desc - irq_desc;
1028 if ( action->ack_type == ACKTYPE_NONE )
1030 ASSERT(!test_bit(pirq, d->pirq_mask));
1031 stop_timer(&action->eoi_timer);
1032 _irq_guest_eoi(desc);
1035 if ( unlikely(!test_and_clear_bit(pirq, d->pirq_mask)) ||
1036 unlikely(--action->in_flight != 0) )
1038 spin_unlock_irq(&desc->lock);
1039 return;
1042 if ( action->ack_type == ACKTYPE_UNMASK )
1044 ASSERT(cpus_empty(action->cpu_eoi_map));
1045 desc->handler->end(irq);
1046 spin_unlock_irq(&desc->lock);
1047 return;
1050 ASSERT(action->ack_type == ACKTYPE_EOI);
1052 cpu_eoi_map = action->cpu_eoi_map;
1054 if ( cpu_test_and_clear(smp_processor_id(), cpu_eoi_map) )
1056 __set_eoi_ready(desc);
1057 spin_unlock(&desc->lock);
1058 flush_ready_eoi();
1059 local_irq_enable();
1061 else
1063 spin_unlock_irq(&desc->lock);
1066 if ( !cpus_empty(cpu_eoi_map) )
1067 on_selected_cpus(&cpu_eoi_map, set_eoi_ready, desc, 0);
1070 int pirq_guest_eoi(struct domain *d, int irq)
1072 if ( (irq < 0) || (irq >= d->nr_pirqs) )
1073 return -EINVAL;
1075 __pirq_guest_eoi(d, irq);
1077 return 0;
1080 int pirq_guest_unmask(struct domain *d)
1082 unsigned int irq, nr = d->nr_pirqs;
1084 for ( irq = find_first_bit(d->pirq_mask, nr);
1085 irq < nr;
1086 irq = find_next_bit(d->pirq_mask, nr, irq+1) )
1088 if ( !test_bit(d->pirq_to_evtchn[irq], &shared_info(d, evtchn_mask)) )
1089 __pirq_guest_eoi(d, irq);
1092 return 0;
1095 extern int ioapic_ack_new;
1096 static int pirq_acktype(struct domain *d, int pirq)
1098 struct irq_desc *desc;
1099 int irq;
1101 irq = domain_pirq_to_irq(d, pirq);
1102 if ( irq <= 0 )
1103 return ACKTYPE_NONE;
1105 desc = irq_to_desc(irq);
1107 if ( desc->handler == &no_irq_type )
1108 return ACKTYPE_NONE;
1110 /*
1111 * Edge-triggered IO-APIC and LAPIC interrupts need no final
1112 * acknowledgement: we ACK early during interrupt processing.
1113 */
1114 if ( !strcmp(desc->handler->typename, "IO-APIC-edge") ||
1115 !strcmp(desc->handler->typename, "local-APIC-edge") )
1116 return ACKTYPE_NONE;
1118 /*
1119 * MSIs are treated as edge-triggered interrupts, except
1120 * when there is no proper way to mask them.
1121 */
1122 if ( desc->handler == &pci_msi_type )
1123 return msi_maskable_irq(desc->msi_desc) ? ACKTYPE_NONE : ACKTYPE_EOI;
1125 /*
1126 * Level-triggered IO-APIC interrupts need to be acknowledged on the CPU
1127 * on which they were received. This is because we tickle the LAPIC to EOI.
1128 */
1129 if ( !strcmp(desc->handler->typename, "IO-APIC-level") )
1130 return ioapic_ack_new ? ACKTYPE_EOI : ACKTYPE_UNMASK;
1132 /* Legacy PIC interrupts can be acknowledged from any CPU. */
1133 if ( !strcmp(desc->handler->typename, "XT-PIC") )
1134 return ACKTYPE_UNMASK;
1136 printk("Unknown PIC type '%s' for IRQ %d\n", desc->handler->typename, irq);
1137 BUG();
1139 return 0;
1142 int pirq_shared(struct domain *d, int pirq)
1144 struct irq_desc *desc;
1145 irq_guest_action_t *action;
1146 unsigned long flags;
1147 int shared;
1149 desc = domain_spin_lock_irq_desc(d, pirq, &flags);
1150 if ( desc == NULL )
1151 return 0;
1153 action = (irq_guest_action_t *)desc->action;
1154 shared = ((desc->status & IRQ_GUEST) && (action->nr_guests > 1));
1156 spin_unlock_irqrestore(&desc->lock, flags);
1158 return shared;
1161 int pirq_guest_bind(struct vcpu *v, int pirq, int will_share)
1163 unsigned int irq;
1164 struct irq_desc *desc;
1165 irq_guest_action_t *action, *newaction = NULL;
1166 int rc = 0;
1167 cpumask_t cpumask = CPU_MASK_NONE;
1169 WARN_ON(!spin_is_locked(&v->domain->event_lock));
1170 BUG_ON(!local_irq_is_enabled());
1172 retry:
1173 desc = domain_spin_lock_irq_desc(v->domain, pirq, NULL);
1174 if ( desc == NULL )
1176 rc = -EINVAL;
1177 goto out;
1180 action = (irq_guest_action_t *)desc->action;
1181 irq = desc - irq_desc;
1183 if ( !(desc->status & IRQ_GUEST) )
1185 if ( desc->action != NULL )
1187 gdprintk(XENLOG_INFO,
1188 "Cannot bind IRQ %d to guest. In use by '%s'.\n",
1189 pirq, desc->action->name);
1190 rc = -EBUSY;
1191 goto unlock_out;
1194 if ( newaction == NULL )
1196 spin_unlock_irq(&desc->lock);
1197 if ( (newaction = xmalloc(irq_guest_action_t)) != NULL )
1198 goto retry;
1199 gdprintk(XENLOG_INFO,
1200 "Cannot bind IRQ %d to guest. Out of memory.\n",
1201 pirq);
1202 rc = -ENOMEM;
1203 goto out;
1206 action = newaction;
1207 desc->action = (struct irqaction *)action;
1208 newaction = NULL;
1210 action->nr_guests = 0;
1211 action->in_flight = 0;
1212 action->shareable = will_share;
1213 action->ack_type = pirq_acktype(v->domain, pirq);
1214 cpus_clear(action->cpu_eoi_map);
1215 init_timer(&action->eoi_timer, irq_guest_eoi_timer_fn, desc, 0);
1217 desc->depth = 0;
1218 desc->status |= IRQ_GUEST;
1219 desc->status &= ~IRQ_DISABLED;
1220 desc->handler->startup(irq);
1222 /* Attempt to bind the interrupt target to the correct CPU. */
1223 cpu_set(v->processor, cpumask);
1224 if ( !opt_noirqbalance && (desc->handler->set_affinity != NULL) )
1225 desc->handler->set_affinity(irq, cpumask);
1227 else if ( !will_share || !action->shareable )
1229 gdprintk(XENLOG_INFO, "Cannot bind IRQ %d to guest. %s.\n",
1230 pirq,
1231 will_share ?
1232 "Others do not share" :
1233 "Will not share with others");
1234 rc = -EBUSY;
1235 goto unlock_out;
1237 else if ( action->nr_guests == 0 )
1239 /*
1240 * Indicates that an ACKTYPE_EOI interrupt is being released.
1241 * Wait for that to happen before continuing.
1242 */
1243 ASSERT(action->ack_type == ACKTYPE_EOI);
1244 ASSERT(desc->status & IRQ_DISABLED);
1245 spin_unlock_irq(&desc->lock);
1246 cpu_relax();
1247 goto retry;
1250 if ( action->nr_guests == IRQ_MAX_GUESTS )
1252 gdprintk(XENLOG_INFO, "Cannot bind IRQ %d to guest. "
1253 "Already at max share.\n", pirq);
1254 rc = -EBUSY;
1255 goto unlock_out;
1258 action->guest[action->nr_guests++] = v->domain;
1260 if ( action->ack_type != ACKTYPE_NONE )
1261 set_pirq_eoi(v->domain, pirq);
1262 else
1263 clear_pirq_eoi(v->domain, pirq);
1265 unlock_out:
1266 spin_unlock_irq(&desc->lock);
1267 out:
1268 if ( newaction != NULL )
1269 xfree(newaction);
1270 return rc;
1273 static irq_guest_action_t *__pirq_guest_unbind(
1274 struct domain *d, int pirq, struct irq_desc *desc)
1276 unsigned int irq;
1277 irq_guest_action_t *action;
1278 cpumask_t cpu_eoi_map;
1279 int i;
1281 BUG_ON(!(desc->status & IRQ_GUEST));
1283 action = (irq_guest_action_t *)desc->action;
1284 irq = desc - irq_desc;
1286 if ( unlikely(action == NULL) )
1288 dprintk(XENLOG_G_WARNING, "dom%d: pirq %d: desc->action is NULL!\n",
1289 d->domain_id, pirq);
1290 return NULL;
1293 for ( i = 0; (i < action->nr_guests) && (action->guest[i] != d); i++ )
1294 continue;
1295 BUG_ON(i == action->nr_guests);
1296 memmove(&action->guest[i], &action->guest[i+1],
1297 (action->nr_guests-i-1) * sizeof(action->guest[0]));
1298 action->nr_guests--;
1300 switch ( action->ack_type )
1302 case ACKTYPE_UNMASK:
1303 if ( test_and_clear_bit(pirq, d->pirq_mask) &&
1304 (--action->in_flight == 0) )
1305 desc->handler->end(irq);
1306 break;
1307 case ACKTYPE_EOI:
1308 /* NB. If #guests == 0 then we clear the eoi_map later on. */
1309 if ( test_and_clear_bit(pirq, d->pirq_mask) &&
1310 (--action->in_flight == 0) &&
1311 (action->nr_guests != 0) )
1313 cpu_eoi_map = action->cpu_eoi_map;
1314 spin_unlock_irq(&desc->lock);
1315 on_selected_cpus(&cpu_eoi_map, set_eoi_ready, desc, 0);
1316 spin_lock_irq(&desc->lock);
1318 break;
1319 case ACKTYPE_NONE:
1320 stop_timer(&action->eoi_timer);
1321 _irq_guest_eoi(desc);
1322 break;
1325 /*
1326 * The guest cannot re-bind to this IRQ until this function returns. So,
1327 * when we have flushed this IRQ from pirq_mask, it should remain flushed.
1328 */
1329 BUG_ON(test_bit(pirq, d->pirq_mask));
1331 if ( action->nr_guests != 0 )
1332 return NULL;
1334 BUG_ON(action->in_flight != 0);
1336 /* Disabling IRQ before releasing the desc_lock avoids an IRQ storm. */
1337 desc->depth = 1;
1338 desc->status |= IRQ_DISABLED;
1339 desc->handler->disable(irq);
1341 /*
1342 * Mark any remaining pending EOIs as ready to flush.
1343 * NOTE: We will need to make this a stronger barrier if in future we allow
1344 * an interrupt vectors to be re-bound to a different PIC. In that case we
1345 * would need to flush all ready EOIs before returning as otherwise the
1346 * desc->handler could change and we would call the wrong 'end' hook.
1347 */
1348 cpu_eoi_map = action->cpu_eoi_map;
1349 if ( !cpus_empty(cpu_eoi_map) )
1351 BUG_ON(action->ack_type != ACKTYPE_EOI);
1352 spin_unlock_irq(&desc->lock);
1353 on_selected_cpus(&cpu_eoi_map, set_eoi_ready, desc, 1);
1354 spin_lock_irq(&desc->lock);
1357 BUG_ON(!cpus_empty(action->cpu_eoi_map));
1359 desc->action = NULL;
1360 desc->status &= ~(IRQ_GUEST|IRQ_GUEST_EOI_PENDING|IRQ_INPROGRESS);
1361 desc->handler->shutdown(irq);
1363 /* Caller frees the old guest descriptor block. */
1364 return action;
1367 void pirq_guest_unbind(struct domain *d, int pirq)
1369 irq_guest_action_t *oldaction = NULL;
1370 struct irq_desc *desc;
1371 int irq;
1373 WARN_ON(!spin_is_locked(&d->event_lock));
1375 BUG_ON(!local_irq_is_enabled());
1376 desc = domain_spin_lock_irq_desc(d, pirq, NULL);
1378 if ( desc == NULL )
1380 irq = -domain_pirq_to_irq(d, pirq);
1381 BUG_ON(irq <= 0);
1382 desc = irq_to_desc(irq);
1383 spin_lock_irq(&desc->lock);
1384 d->arch.pirq_irq[pirq] = d->arch.irq_pirq[irq] = 0;
1386 else
1388 oldaction = __pirq_guest_unbind(d, pirq, desc);
1391 spin_unlock_irq(&desc->lock);
1393 if ( oldaction != NULL )
1395 kill_timer(&oldaction->eoi_timer);
1396 xfree(oldaction);
1400 static int pirq_guest_force_unbind(struct domain *d, int irq)
1402 struct irq_desc *desc;
1403 irq_guest_action_t *action, *oldaction = NULL;
1404 int i, bound = 0;
1406 WARN_ON(!spin_is_locked(&d->event_lock));
1408 BUG_ON(!local_irq_is_enabled());
1409 desc = domain_spin_lock_irq_desc(d, irq, NULL);
1410 BUG_ON(desc == NULL);
1412 if ( !(desc->status & IRQ_GUEST) )
1413 goto out;
1415 action = (irq_guest_action_t *)desc->action;
1416 if ( unlikely(action == NULL) )
1418 dprintk(XENLOG_G_WARNING, "dom%d: pirq %d: desc->action is NULL!\n",
1419 d->domain_id, irq);
1420 goto out;
1423 for ( i = 0; (i < action->nr_guests) && (action->guest[i] != d); i++ )
1424 continue;
1425 if ( i == action->nr_guests )
1426 goto out;
1428 bound = 1;
1429 oldaction = __pirq_guest_unbind(d, irq, desc);
1431 out:
1432 spin_unlock_irq(&desc->lock);
1434 if ( oldaction != NULL )
1436 kill_timer(&oldaction->eoi_timer);
1437 xfree(oldaction);
1440 return bound;
1443 int get_free_pirq(struct domain *d, int type, int index)
1445 int i;
1447 ASSERT(spin_is_locked(&d->event_lock));
1449 if ( type == MAP_PIRQ_TYPE_GSI )
1451 for ( i = 16; i < nr_irqs_gsi; i++ )
1452 if ( !d->arch.pirq_irq[i] )
1454 if ( !is_hvm_domain(d) ||
1455 d->arch.pirq_emuirq[i] == IRQ_UNBOUND )
1456 break;
1458 if ( i == nr_irqs_gsi )
1459 return -ENOSPC;
1461 else
1463 for ( i = d->nr_pirqs - 1; i >= nr_irqs_gsi; i-- )
1464 if ( !d->arch.pirq_irq[i] )
1466 if ( !is_hvm_domain(d) ||
1467 d->arch.pirq_emuirq[i] == IRQ_UNBOUND )
1468 break;
1470 if ( i < nr_irqs_gsi )
1471 return -ENOSPC;
1474 return i;
1477 int map_domain_pirq(
1478 struct domain *d, int pirq, int irq, int type, void *data)
1480 int ret = 0;
1481 int old_irq, old_pirq;
1482 struct irq_desc *desc;
1483 unsigned long flags;
1484 struct msi_desc *msi_desc;
1485 struct pci_dev *pdev = NULL;
1487 ASSERT(spin_is_locked(&pcidevs_lock));
1488 ASSERT(spin_is_locked(&d->event_lock));
1490 if ( !IS_PRIV(current->domain) &&
1491 !(IS_PRIV_FOR(current->domain, d) &&
1492 irq_access_permitted(current->domain, pirq)))
1493 return -EPERM;
1495 if ( pirq < 0 || pirq >= d->nr_pirqs || irq < 0 || irq >= nr_irqs )
1497 dprintk(XENLOG_G_ERR, "dom%d: invalid pirq %d or irq %d\n",
1498 d->domain_id, pirq, irq);
1499 return -EINVAL;
1502 old_irq = domain_pirq_to_irq(d, pirq);
1503 old_pirq = domain_irq_to_pirq(d, irq);
1505 if ( (old_irq > 0 && (old_irq != irq) ) ||
1506 (old_pirq && (old_pirq != pirq)) )
1508 dprintk(XENLOG_G_WARNING, "dom%d: pirq %d or irq %d already mapped\n",
1509 d->domain_id, pirq, irq);
1510 return 0;
1513 ret = irq_permit_access(d, pirq);
1514 if ( ret )
1516 dprintk(XENLOG_G_ERR, "dom%d: could not permit access to irq %d\n",
1517 d->domain_id, pirq);
1518 return ret;
1521 desc = irq_to_desc(irq);
1523 if ( type == MAP_PIRQ_TYPE_MSI )
1525 struct msi_info *msi = (struct msi_info *)data;
1527 ret = -ENODEV;
1528 if ( !cpu_has_apic )
1529 goto done;
1531 pdev = pci_get_pdev(msi->bus, msi->devfn);
1532 ret = pci_enable_msi(msi, &msi_desc);
1533 if ( ret )
1534 goto done;
1536 spin_lock_irqsave(&desc->lock, flags);
1538 if ( desc->handler != &no_irq_type )
1539 dprintk(XENLOG_G_ERR, "dom%d: irq %d in use\n",
1540 d->domain_id, irq);
1541 desc->handler = &pci_msi_type;
1542 d->arch.pirq_irq[pirq] = irq;
1543 d->arch.irq_pirq[irq] = pirq;
1544 setup_msi_irq(pdev, msi_desc, irq);
1545 spin_unlock_irqrestore(&desc->lock, flags);
1546 } else
1548 spin_lock_irqsave(&desc->lock, flags);
1549 d->arch.pirq_irq[pirq] = irq;
1550 d->arch.irq_pirq[irq] = pirq;
1551 spin_unlock_irqrestore(&desc->lock, flags);
1554 done:
1555 return ret;
1558 /* The pirq should have been unbound before this call. */
1559 int unmap_domain_pirq(struct domain *d, int pirq)
1561 unsigned long flags;
1562 struct irq_desc *desc;
1563 int irq, ret = 0;
1564 bool_t forced_unbind;
1565 struct msi_desc *msi_desc = NULL;
1567 if ( (pirq < 0) || (pirq >= d->nr_pirqs) )
1568 return -EINVAL;
1570 ASSERT(spin_is_locked(&pcidevs_lock));
1571 ASSERT(spin_is_locked(&d->event_lock));
1573 irq = domain_pirq_to_irq(d, pirq);
1574 if ( irq <= 0 )
1576 dprintk(XENLOG_G_ERR, "dom%d: pirq %d not mapped\n",
1577 d->domain_id, pirq);
1578 ret = -EINVAL;
1579 goto done;
1582 forced_unbind = pirq_guest_force_unbind(d, pirq);
1583 if ( forced_unbind )
1584 dprintk(XENLOG_G_WARNING, "dom%d: forcing unbind of pirq %d\n",
1585 d->domain_id, pirq);
1587 desc = irq_to_desc(irq);
1589 if ( (msi_desc = desc->msi_desc) != NULL )
1590 pci_disable_msi(msi_desc);
1592 spin_lock_irqsave(&desc->lock, flags);
1594 BUG_ON(irq != domain_pirq_to_irq(d, pirq));
1596 if ( !forced_unbind )
1598 d->arch.pirq_irq[pirq] = 0;
1599 d->arch.irq_pirq[irq] = 0;
1601 else
1603 d->arch.pirq_irq[pirq] = -irq;
1604 d->arch.irq_pirq[irq] = -pirq;
1607 spin_unlock_irqrestore(&desc->lock, flags);
1608 if (msi_desc)
1609 msi_free_irq(msi_desc);
1611 ret = irq_deny_access(d, pirq);
1612 if ( ret )
1613 dprintk(XENLOG_G_ERR, "dom%d: could not deny access to irq %d\n",
1614 d->domain_id, pirq);
1616 if ( desc->handler == &pci_msi_type )
1617 desc->handler = &no_irq_type;
1619 done:
1620 return ret;
1623 void free_domain_pirqs(struct domain *d)
1625 int i;
1627 spin_lock(&pcidevs_lock);
1628 spin_lock(&d->event_lock);
1630 for ( i = 0; i < d->nr_pirqs; i++ )
1631 if ( d->arch.pirq_irq[i] > 0 )
1632 unmap_domain_pirq(d, i);
1634 spin_unlock(&d->event_lock);
1635 spin_unlock(&pcidevs_lock);
1638 extern void dump_ioapic_irq_info(void);
1640 static void dump_irqs(unsigned char key)
1642 int i, irq, pirq;
1643 struct irq_desc *desc;
1644 struct irq_cfg *cfg;
1645 irq_guest_action_t *action;
1646 struct domain *d;
1647 unsigned long flags;
1649 printk("Guest interrupt information:\n");
1651 for ( irq = 0; irq < nr_irqs; irq++ )
1654 desc = irq_to_desc(irq);
1655 cfg = desc->chip_data;
1657 if ( !desc->handler || desc->handler == &no_irq_type )
1658 continue;
1660 spin_lock_irqsave(&desc->lock, flags);
1662 cpumask_scnprintf(keyhandler_scratch, sizeof(keyhandler_scratch),
1663 desc->affinity);
1664 printk(" IRQ:%4d affinity:%s vec:%02x type=%-15s"
1665 " status=%08x ",
1666 irq, keyhandler_scratch, cfg->vector,
1667 desc->handler->typename, desc->status);
1669 if ( !(desc->status & IRQ_GUEST) )
1670 printk("mapped, unbound\n");
1671 else
1673 action = (irq_guest_action_t *)desc->action;
1675 printk("in-flight=%d domain-list=", action->in_flight);
1677 for ( i = 0; i < action->nr_guests; i++ )
1679 d = action->guest[i];
1680 pirq = domain_irq_to_pirq(d, irq);
1681 printk("%u:%3d(%c%c%c%c)",
1682 d->domain_id, pirq,
1683 (test_bit(d->pirq_to_evtchn[pirq],
1684 &shared_info(d, evtchn_pending)) ?
1685 'P' : '-'),
1686 (test_bit(d->pirq_to_evtchn[pirq] /
1687 BITS_PER_EVTCHN_WORD(d),
1688 &vcpu_info(d->vcpu[0], evtchn_pending_sel)) ?
1689 'S' : '-'),
1690 (test_bit(d->pirq_to_evtchn[pirq],
1691 &shared_info(d, evtchn_mask)) ?
1692 'M' : '-'),
1693 (test_bit(pirq, d->pirq_mask) ?
1694 'M' : '-'));
1695 if ( i != action->nr_guests )
1696 printk(",");
1699 printk("\n");
1702 spin_unlock_irqrestore(&desc->lock, flags);
1705 dump_ioapic_irq_info();
1708 static struct keyhandler dump_irqs_keyhandler = {
1709 .diagnostic = 1,
1710 .u.fn = dump_irqs,
1711 .desc = "dump interrupt bindings"
1712 };
1714 static int __init setup_dump_irqs(void)
1716 register_keyhandler('i', &dump_irqs_keyhandler);
1717 return 0;
1719 __initcall(setup_dump_irqs);
1721 /* A cpu has been removed from cpu_online_mask. Re-set irq affinities. */
1722 void fixup_irqs(void)
1724 unsigned int irq, sp;
1725 static int warned;
1726 struct irq_desc *desc;
1727 irq_guest_action_t *action;
1728 struct pending_eoi *peoi;
1730 for ( irq = 0; irq < nr_irqs; irq++ )
1732 int break_affinity = 0;
1733 int set_affinity = 1;
1734 cpumask_t affinity;
1736 if ( irq == 2 )
1737 continue;
1739 desc = irq_to_desc(irq);
1741 spin_lock(&desc->lock);
1743 affinity = desc->affinity;
1744 if ( !desc->action || cpus_subset(affinity, cpu_online_map) )
1746 spin_unlock(&desc->lock);
1747 continue;
1750 cpus_and(affinity, affinity, cpu_online_map);
1751 if ( cpus_empty(affinity) )
1753 break_affinity = 1;
1754 affinity = cpu_online_map;
1757 if ( desc->handler->disable )
1758 desc->handler->disable(irq);
1760 if ( desc->handler->set_affinity )
1761 desc->handler->set_affinity(irq, affinity);
1762 else if ( !(warned++) )
1763 set_affinity = 0;
1765 if ( desc->handler->enable )
1766 desc->handler->enable(irq);
1768 spin_unlock(&desc->lock);
1770 if ( break_affinity && set_affinity )
1771 printk("Broke affinity for irq %i\n", irq);
1772 else if ( !set_affinity )
1773 printk("Cannot set affinity for irq %i\n", irq);
1776 /* That doesn't seem sufficient. Give it 1ms. */
1777 local_irq_enable();
1778 mdelay(1);
1779 local_irq_disable();
1781 /* Clean up cpu_eoi_map of every interrupt to exclude this CPU. */
1782 for ( irq = 0; irq < nr_irqs; irq++ )
1784 desc = irq_to_desc(irq);
1785 if ( !(desc->status & IRQ_GUEST) )
1786 continue;
1787 action = (irq_guest_action_t *)desc->action;
1788 cpu_clear(smp_processor_id(), action->cpu_eoi_map);
1791 /* Flush the interrupt EOI stack. */
1792 peoi = this_cpu(pending_eoi);
1793 for ( sp = 0; sp < pending_eoi_sp(peoi); sp++ )
1794 peoi[sp].ready = 1;
1795 flush_ready_eoi();
1798 int map_domain_emuirq_pirq(struct domain *d, int pirq, int emuirq)
1800 int old_emuirq = IRQ_UNBOUND, old_pirq = IRQ_UNBOUND;
1802 ASSERT(spin_is_locked(&d->event_lock));
1804 if ( !is_hvm_domain(d) )
1805 return -EINVAL;
1807 if ( pirq < 0 || pirq >= d->nr_pirqs ||
1808 emuirq == IRQ_UNBOUND || emuirq >= (int) nr_irqs )
1810 dprintk(XENLOG_G_ERR, "dom%d: invalid pirq %d or emuirq %d\n",
1811 d->domain_id, pirq, emuirq);
1812 return -EINVAL;
1815 old_emuirq = domain_pirq_to_emuirq(d, pirq);
1816 if ( emuirq != IRQ_PT )
1817 old_pirq = domain_emuirq_to_pirq(d, emuirq);
1819 if ( (old_emuirq != IRQ_UNBOUND && (old_emuirq != emuirq) ) ||
1820 (old_pirq != IRQ_UNBOUND && (old_pirq != pirq)) )
1822 dprintk(XENLOG_G_WARNING, "dom%d: pirq %d or emuirq %d already mapped\n",
1823 d->domain_id, pirq, emuirq);
1824 return 0;
1827 d->arch.pirq_emuirq[pirq] = emuirq;
1828 /* do not store emuirq mappings for pt devices */
1829 if ( emuirq != IRQ_PT )
1830 d->arch.emuirq_pirq[emuirq] = pirq;
1832 return 0;
1835 int unmap_domain_pirq_emuirq(struct domain *d, int pirq)
1837 int emuirq, ret = 0;
1839 if ( !is_hvm_domain(d) )
1840 return -EINVAL;
1842 if ( (pirq < 0) || (pirq >= d->nr_pirqs) )
1843 return -EINVAL;
1845 ASSERT(spin_is_locked(&d->event_lock));
1847 emuirq = domain_pirq_to_emuirq(d, pirq);
1848 if ( emuirq == IRQ_UNBOUND )
1850 dprintk(XENLOG_G_ERR, "dom%d: pirq %d not mapped\n",
1851 d->domain_id, pirq);
1852 ret = -EINVAL;
1853 goto done;
1856 d->arch.pirq_emuirq[pirq] = IRQ_UNBOUND;
1857 if ( emuirq != IRQ_PT )
1858 d->arch.emuirq_pirq[emuirq] = IRQ_UNBOUND;
1860 done:
1861 return ret;
1864 int hvm_domain_use_pirq(struct domain *d, int pirq)
1866 int emuirq;
1868 if ( !is_hvm_domain(d) )
1869 return 0;
1871 emuirq = domain_pirq_to_emuirq(d, pirq);
1872 if ( emuirq != IRQ_UNBOUND && d->pirq_to_evtchn[pirq] != 0 )
1873 return 1;
1874 else
1875 return 0;
1878 #ifdef XEN_KDB_CONFIG
1879 void kdb_prnt_guest_mapped_irqs(void)
1881 int irq, j;
1882 char affstr[NR_CPUS/4+NR_CPUS/32+2]; /* courtesy dump_irqs() */
1884 kdbp("irq vec aff type domid:mapped-pirq pairs (all in decimal)\n");
1885 for (irq=0; irq < nr_irqs; irq++) {
1886 irq_desc_t *dp = irq_to_desc(irq);
1887 struct irq_cfg *cfgp = dp->chip_data;
1888 irq_guest_action_t *actp = (irq_guest_action_t *)dp->action;
1890 if (!dp->handler ||dp->handler==&no_irq_type || !(dp->status&IRQ_GUEST))
1891 continue;
1893 cpumask_scnprintf(affstr, sizeof(affstr), dp->affinity);
1894 kdbp("[%3ld] %3d %3s %-13s ", irq, cfgp->vector, affstr,
1895 dp->handler->typename);
1896 for (j=0; j < actp->nr_guests; j++)
1897 kdbp("%03d:%04d ", actp->guest[j]->domain_id,
1898 domain_irq_to_pirq(actp->guest[j], irq));
1899 kdbp("\n");
1902 #endif