debuggers.hg

view xen/drivers/passthrough/vtd/intremap.c @ 21979:add40eb47868

vt-d: Fix ioapic write order in io_apic_write_remap_rte

At the end of io_apic_write_remap_rte, it writes new entry (remapped
interrupt) to ioapic. But it writes low 32 bits before high 32 bits,
it unmasks interrupt before writing high 32 bits if 'mask' bit in low
32 bits is cleared. Thus it may result in issues. This patch fixes
this issue by writing high 32 bits before low 32 bits.

Signed-off-by: Jiang, Yunhong <yunhong.jiang@intel.com>
Signed-off-by: Weidong Han <weidong.han@intel.com>
author Keir Fraser <keir.fraser@citrix.com>
date Mon Aug 09 16:32:45 2010 +0100 (2010-08-09)
parents 34f612ed4184
children befd1814c0a2
line source
1 /*
2 * Copyright (c) 2006, Intel Corporation.
3 *
4 * This program is free software; you can redistribute it and/or modify it
5 * under the terms and conditions of the GNU General Public License,
6 * version 2, as published by the Free Software Foundation.
7 *
8 * This program is distributed in the hope it will be useful, but WITHOUT
9 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
10 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
11 * more details.
12 *
13 * You should have received a copy of the GNU General Public License along with
14 * this program; if not, write to the Free Software Foundation, Inc., 59 Temple
15 * Place - Suite 330, Boston, MA 02111-1307 USA.
16 *
17 * Copyright (C) Allen Kay <allen.m.kay@intel.com>
18 * Copyright (C) Xiaohui Xin <xiaohui.xin@intel.com>
19 */
21 #include <xen/irq.h>
22 #include <xen/sched.h>
23 #include <xen/iommu.h>
24 #include <asm/hvm/iommu.h>
25 #include <xen/time.h>
26 #include <xen/list.h>
27 #include <xen/pci.h>
28 #include <xen/pci_regs.h>
29 #include "iommu.h"
30 #include "dmar.h"
31 #include "vtd.h"
32 #include "extern.h"
34 #ifdef __ia64__
35 #define nr_ioapics iosapic_get_nr_iosapics()
36 #define nr_ioapic_registers(i) iosapic_get_nr_pins(i)
37 #else
38 #include <asm/apic.h>
39 #include <asm/io_apic.h>
40 #define nr_ioapic_registers(i) nr_ioapic_registers[i]
41 #endif
43 /*
44 * source validation type (SVT)
45 */
46 #define SVT_NO_VERIFY 0x0 /* no verification is required */
47 #define SVT_VERIFY_SID_SQ 0x1 /* verify using SID and SQ fiels */
48 #define SVT_VERIFY_BUS 0x2 /* verify bus of request-id */
50 /*
51 * source-id qualifier (SQ)
52 */
53 #define SQ_ALL_16 0x0 /* verify all 16 bits of request-id */
54 #define SQ_13_IGNORE_1 0x1 /* verify most significant 13 bits, ignore
55 * the third least significant bit
56 */
57 #define SQ_13_IGNORE_2 0x2 /* verify most significant 13 bits, ignore
58 * the second and third least significant bits
59 */
60 #define SQ_13_IGNORE_3 0x3 /* verify most significant 13 bits, ignore
61 * the least three significant bits
62 */
64 /* apic_pin_2_ir_idx[apicid][pin] = interrupt remapping table index */
65 static int **apic_pin_2_ir_idx;
67 static int init_apic_pin_2_ir_idx(void)
68 {
69 int *_apic_pin_2_ir_idx;
70 unsigned int nr_pins, i;
72 /* Here we shouldn't need to re-init when resuming from S3. */
73 if ( apic_pin_2_ir_idx != NULL )
74 return 0;
76 nr_pins = 0;
77 for ( i = 0; i < nr_ioapics; i++ )
78 nr_pins += nr_ioapic_registers(i);
80 _apic_pin_2_ir_idx = xmalloc_array(int, nr_pins);
81 apic_pin_2_ir_idx = xmalloc_array(int *, nr_ioapics);
82 if ( (_apic_pin_2_ir_idx == NULL) || (apic_pin_2_ir_idx == NULL) )
83 {
84 xfree(_apic_pin_2_ir_idx);
85 xfree(apic_pin_2_ir_idx);
86 return -ENOMEM;
87 }
89 for ( i = 0; i < nr_pins; i++ )
90 _apic_pin_2_ir_idx[i] = -1;
92 nr_pins = 0;
93 for ( i = 0; i < nr_ioapics; i++ )
94 {
95 apic_pin_2_ir_idx[i] = &_apic_pin_2_ir_idx[nr_pins];
96 nr_pins += nr_ioapic_registers(i);
97 }
99 return 0;
100 }
102 static u16 apicid_to_bdf(int apic_id)
103 {
104 struct acpi_drhd_unit *drhd = ioapic_to_drhd(apic_id);
105 struct acpi_ioapic_unit *acpi_ioapic_unit;
107 list_for_each_entry ( acpi_ioapic_unit, &drhd->ioapic_list, list )
108 if ( acpi_ioapic_unit->apic_id == apic_id )
109 return acpi_ioapic_unit->ioapic.info;
111 dprintk(XENLOG_ERR VTDPREFIX, "Didn't find the bdf for the apic_id!\n");
112 return 0;
113 }
115 static void set_ire_sid(struct iremap_entry *ire,
116 unsigned int svt, unsigned int sq, unsigned int sid)
117 {
118 ire->hi.svt = svt;
119 ire->hi.sq = sq;
120 ire->hi.sid = sid;
121 }
123 static void set_ioapic_source_id(int apic_id, struct iremap_entry *ire)
124 {
125 set_ire_sid(ire, SVT_VERIFY_SID_SQ, SQ_ALL_16,
126 apicid_to_bdf(apic_id));
127 }
129 int iommu_supports_eim(void)
130 {
131 struct acpi_drhd_unit *drhd;
132 int apic;
134 if ( !iommu_enabled || !iommu_qinval || !iommu_intremap )
135 return 0;
137 if ( list_empty(&acpi_drhd_units) )
138 {
139 dprintk(XENLOG_WARNING VTDPREFIX, "VT-d is not supported\n");
140 return 0;
141 }
143 /* We MUST have a DRHD unit for each IOAPIC. */
144 for ( apic = 0; apic < nr_ioapics; apic++ )
145 if ( !ioapic_to_drhd(IO_APIC_ID(apic)) )
146 {
147 dprintk(XENLOG_WARNING VTDPREFIX,
148 "There is not a DRHD for IOAPIC 0x%x (id: 0x%x)!\n",
149 apic, IO_APIC_ID(apic));
150 return 0;
151 }
153 for_each_drhd_unit ( drhd )
154 if ( !ecap_queued_inval(drhd->iommu->ecap) ||
155 !ecap_intr_remap(drhd->iommu->ecap) ||
156 !ecap_eim(drhd->iommu->ecap) )
157 return 0;
159 return 1;
160 }
162 /* Mark specified intr remap entry as free */
163 static void free_remap_entry(struct iommu *iommu, int index)
164 {
165 struct iremap_entry *iremap_entry = NULL, *iremap_entries;
166 struct ir_ctrl *ir_ctrl = iommu_ir_ctrl(iommu);
168 if ( index < 0 || index > IREMAP_ENTRY_NR - 1 )
169 return;
171 ASSERT( spin_is_locked(&ir_ctrl->iremap_lock) );
173 GET_IREMAP_ENTRY(ir_ctrl->iremap_maddr, index,
174 iremap_entries, iremap_entry);
176 memset(iremap_entry, 0, sizeof(struct iremap_entry));
177 iommu_flush_cache_entry(iremap_entry, sizeof(struct iremap_entry));
178 iommu_flush_iec_index(iommu, 0, index);
180 unmap_vtd_domain_page(iremap_entries);
181 ir_ctrl->iremap_num--;
182 }
184 /*
185 * Look for a free intr remap entry.
186 * Need hold iremap_lock, and setup returned entry before releasing lock.
187 */
188 static int alloc_remap_entry(struct iommu *iommu)
189 {
190 struct iremap_entry *iremap_entries = NULL;
191 struct ir_ctrl *ir_ctrl = iommu_ir_ctrl(iommu);
192 int i;
194 ASSERT( spin_is_locked(&ir_ctrl->iremap_lock) );
196 for ( i = 0; i < IREMAP_ENTRY_NR; i++ )
197 {
198 struct iremap_entry *p;
199 if ( i % (1 << IREMAP_ENTRY_ORDER) == 0 )
200 {
201 /* This entry across page boundry */
202 if ( iremap_entries )
203 unmap_vtd_domain_page(iremap_entries);
205 GET_IREMAP_ENTRY(ir_ctrl->iremap_maddr, i,
206 iremap_entries, p);
207 }
208 else
209 p = &iremap_entries[i % (1 << IREMAP_ENTRY_ORDER)];
211 if ( p->lo_val == 0 && p->hi_val == 0 ) /* a free entry */
212 break;
213 }
215 if ( iremap_entries )
216 unmap_vtd_domain_page(iremap_entries);
218 ir_ctrl->iremap_num++;
219 return i;
220 }
222 static int remap_entry_to_ioapic_rte(
223 struct iommu *iommu, int index, struct IO_xAPIC_route_entry *old_rte)
224 {
225 struct iremap_entry *iremap_entry = NULL, *iremap_entries;
226 unsigned long flags;
227 struct ir_ctrl *ir_ctrl = iommu_ir_ctrl(iommu);
229 if ( ir_ctrl == NULL )
230 {
231 dprintk(XENLOG_ERR VTDPREFIX,
232 "remap_entry_to_ioapic_rte: ir_ctl is not ready\n");
233 return -EFAULT;
234 }
236 if ( index < 0 || index > IREMAP_ENTRY_NR - 1 )
237 {
238 dprintk(XENLOG_ERR VTDPREFIX,
239 "%s: index (%d) for remap table is invalid !\n",
240 __func__, index);
241 return -EFAULT;
242 }
244 spin_lock_irqsave(&ir_ctrl->iremap_lock, flags);
246 GET_IREMAP_ENTRY(ir_ctrl->iremap_maddr, index,
247 iremap_entries, iremap_entry);
249 if ( iremap_entry->hi_val == 0 && iremap_entry->lo_val == 0 )
250 {
251 dprintk(XENLOG_ERR VTDPREFIX,
252 "%s: index (%d) get an empty entry!\n",
253 __func__, index);
254 return -EFAULT;
255 }
257 old_rte->vector = iremap_entry->lo.vector;
258 old_rte->delivery_mode = iremap_entry->lo.dlm;
259 old_rte->dest_mode = iremap_entry->lo.dm;
260 old_rte->trigger = iremap_entry->lo.tm;
261 old_rte->__reserved_2 = 0;
262 old_rte->dest.logical.__reserved_1 = 0;
263 old_rte->dest.logical.logical_dest = iremap_entry->lo.dst >> 8;
265 unmap_vtd_domain_page(iremap_entries);
266 spin_unlock_irqrestore(&ir_ctrl->iremap_lock, flags);
267 return 0;
268 }
270 static int ioapic_rte_to_remap_entry(struct iommu *iommu,
271 int apic, unsigned int ioapic_pin, struct IO_xAPIC_route_entry *old_rte,
272 unsigned int rte_upper, unsigned int value)
273 {
274 struct iremap_entry *iremap_entry = NULL, *iremap_entries;
275 struct iremap_entry new_ire;
276 struct IO_APIC_route_remap_entry *remap_rte;
277 struct IO_xAPIC_route_entry new_rte;
278 int index;
279 unsigned long flags;
280 struct ir_ctrl *ir_ctrl = iommu_ir_ctrl(iommu);
282 remap_rte = (struct IO_APIC_route_remap_entry *) old_rte;
283 spin_lock_irqsave(&ir_ctrl->iremap_lock, flags);
285 index = apic_pin_2_ir_idx[apic][ioapic_pin];
286 if ( index < 0 )
287 {
288 index = alloc_remap_entry(iommu);
289 apic_pin_2_ir_idx[apic][ioapic_pin] = index;
290 }
292 if ( index > IREMAP_ENTRY_NR - 1 )
293 {
294 dprintk(XENLOG_ERR VTDPREFIX,
295 "%s: intremap index (%d) is larger than"
296 " the maximum index (%d)!\n",
297 __func__, index, IREMAP_ENTRY_NR - 1);
298 spin_unlock_irqrestore(&ir_ctrl->iremap_lock, flags);
299 return -EFAULT;
300 }
302 GET_IREMAP_ENTRY(ir_ctrl->iremap_maddr, index,
303 iremap_entries, iremap_entry);
305 memcpy(&new_ire, iremap_entry, sizeof(struct iremap_entry));
307 if ( rte_upper )
308 {
309 #if defined(__i386__) || defined(__x86_64__)
310 if ( x2apic_enabled )
311 new_ire.lo.dst = value;
312 else
313 new_ire.lo.dst = (value >> 24) << 8;
314 #else /* __ia64__ */
315 new_ire.lo.dst = value >> 16;
316 #endif
317 }
318 else
319 {
320 *(((u32 *)&new_rte) + 0) = value;
321 new_ire.lo.fpd = 0;
322 new_ire.lo.dm = new_rte.dest_mode;
323 new_ire.lo.rh = 0;
324 new_ire.lo.tm = new_rte.trigger;
325 new_ire.lo.dlm = new_rte.delivery_mode;
326 new_ire.lo.avail = 0;
327 new_ire.lo.res_1 = 0;
328 new_ire.lo.vector = new_rte.vector;
329 new_ire.lo.res_2 = 0;
331 set_ioapic_source_id(IO_APIC_ID(apic), &new_ire);
332 new_ire.hi.res_1 = 0;
333 new_ire.lo.p = 1; /* finally, set present bit */
335 /* now construct new ioapic rte entry */
336 remap_rte->vector = new_rte.vector;
337 remap_rte->delivery_mode = 0; /* has to be 0 for remap format */
338 remap_rte->index_15 = (index >> 15) & 0x1;
339 remap_rte->index_0_14 = index & 0x7fff;
341 remap_rte->delivery_status = new_rte.delivery_status;
342 remap_rte->polarity = new_rte.polarity;
343 remap_rte->irr = new_rte.irr;
344 remap_rte->trigger = new_rte.trigger;
345 remap_rte->mask = new_rte.mask;
346 remap_rte->reserved = 0;
347 remap_rte->format = 1; /* indicate remap format */
348 }
350 memcpy(iremap_entry, &new_ire, sizeof(struct iremap_entry));
351 iommu_flush_cache_entry(iremap_entry, sizeof(struct iremap_entry));
352 iommu_flush_iec_index(iommu, 0, index);
353 invalidate_sync(iommu);
355 unmap_vtd_domain_page(iremap_entries);
356 spin_unlock_irqrestore(&ir_ctrl->iremap_lock, flags);
357 return 0;
358 }
360 unsigned int io_apic_read_remap_rte(
361 unsigned int apic, unsigned int reg)
362 {
363 unsigned int ioapic_pin = (reg - 0x10) / 2;
364 int index;
365 struct IO_xAPIC_route_entry old_rte = { 0 };
366 struct IO_APIC_route_remap_entry *remap_rte;
367 int rte_upper = (reg & 1) ? 1 : 0;
368 struct iommu *iommu = ioapic_to_iommu(IO_APIC_ID(apic));
369 struct ir_ctrl *ir_ctrl = iommu_ir_ctrl(iommu);
371 if ( !iommu || !ir_ctrl || ir_ctrl->iremap_maddr == 0 ||
372 (ir_ctrl->iremap_num == 0) ||
373 ( (index = apic_pin_2_ir_idx[apic][ioapic_pin]) < 0 ) )
374 {
375 *IO_APIC_BASE(apic) = reg;
376 return *(IO_APIC_BASE(apic)+4);
377 }
379 if ( rte_upper )
380 reg--;
382 /* read lower and upper 32-bits of rte entry */
383 *IO_APIC_BASE(apic) = reg;
384 *(((u32 *)&old_rte) + 0) = *(IO_APIC_BASE(apic)+4);
385 *IO_APIC_BASE(apic) = reg + 1;
386 *(((u32 *)&old_rte) + 1) = *(IO_APIC_BASE(apic)+4);
388 remap_rte = (struct IO_APIC_route_remap_entry *) &old_rte;
390 if ( remap_entry_to_ioapic_rte(iommu, index, &old_rte) )
391 {
392 *IO_APIC_BASE(apic) = rte_upper ? (reg + 1) : reg;
393 return *(IO_APIC_BASE(apic)+4);
394 }
396 if ( rte_upper )
397 return (*(((u32 *)&old_rte) + 1));
398 else
399 return (*(((u32 *)&old_rte) + 0));
400 }
402 void io_apic_write_remap_rte(
403 unsigned int apic, unsigned int reg, unsigned int value)
404 {
405 unsigned int ioapic_pin = (reg - 0x10) / 2;
406 struct IO_xAPIC_route_entry old_rte = { 0 };
407 struct IO_APIC_route_remap_entry *remap_rte;
408 unsigned int rte_upper = (reg & 1) ? 1 : 0;
409 struct iommu *iommu = ioapic_to_iommu(IO_APIC_ID(apic));
410 struct ir_ctrl *ir_ctrl = iommu_ir_ctrl(iommu);
411 int saved_mask;
413 if ( !iommu || !ir_ctrl || ir_ctrl->iremap_maddr == 0 )
414 {
415 *IO_APIC_BASE(apic) = reg;
416 *(IO_APIC_BASE(apic)+4) = value;
417 return;
418 }
420 if ( rte_upper )
421 reg--;
423 /* read both lower and upper 32-bits of rte entry */
424 *IO_APIC_BASE(apic) = reg;
425 *(((u32 *)&old_rte) + 0) = *(IO_APIC_BASE(apic)+4);
426 *IO_APIC_BASE(apic) = reg + 1;
427 *(((u32 *)&old_rte) + 1) = *(IO_APIC_BASE(apic)+4);
429 remap_rte = (struct IO_APIC_route_remap_entry *) &old_rte;
431 /* mask the interrupt while we change the intremap table */
432 saved_mask = remap_rte->mask;
433 remap_rte->mask = 1;
434 *IO_APIC_BASE(apic) = reg;
435 *(IO_APIC_BASE(apic)+4) = *(((int *)&old_rte)+0);
436 remap_rte->mask = saved_mask;
438 if ( ioapic_rte_to_remap_entry(iommu, apic, ioapic_pin,
439 &old_rte, rte_upper, value) )
440 {
441 *IO_APIC_BASE(apic) = rte_upper ? (reg + 1) : reg;
442 *(IO_APIC_BASE(apic)+4) = value;
443 return;
444 }
446 /* write new entry to ioapic */
447 *IO_APIC_BASE(apic) = reg + 1;
448 *(IO_APIC_BASE(apic)+4) = *(((u32 *)&old_rte)+1);
449 *IO_APIC_BASE(apic) = reg;
450 *(IO_APIC_BASE(apic)+4) = *(((u32 *)&old_rte)+0);
451 }
453 #if defined(__i386__) || defined(__x86_64__)
455 static void set_msi_source_id(struct pci_dev *pdev, struct iremap_entry *ire)
456 {
457 int type;
458 u8 bus, devfn, secbus;
459 int ret;
461 if ( !pdev || !ire )
462 return;
464 bus = pdev->bus;
465 devfn = pdev->devfn;
466 type = pdev_type(bus, devfn);
467 switch ( type )
468 {
469 case DEV_TYPE_PCIe_BRIDGE:
470 case DEV_TYPE_PCIe2PCI_BRIDGE:
471 case DEV_TYPE_LEGACY_PCI_BRIDGE:
472 break;
474 case DEV_TYPE_PCIe_ENDPOINT:
475 set_ire_sid(ire, SVT_VERIFY_SID_SQ, SQ_ALL_16, PCI_BDF2(bus, devfn));
476 break;
478 case DEV_TYPE_PCI:
479 ret = find_upstream_bridge(&bus, &devfn, &secbus);
480 if ( ret == 0 ) /* integrated PCI device */
481 {
482 set_ire_sid(ire, SVT_VERIFY_SID_SQ, SQ_ALL_16,
483 PCI_BDF2(bus, devfn));
484 }
485 else if ( ret == 1 ) /* find upstream bridge */
486 {
487 if ( pdev_type(bus, devfn) == DEV_TYPE_PCIe2PCI_BRIDGE )
488 set_ire_sid(ire, SVT_VERIFY_BUS, SQ_ALL_16,
489 (bus << 8) | pdev->bus);
490 else if ( pdev_type(bus, devfn) == DEV_TYPE_LEGACY_PCI_BRIDGE )
491 set_ire_sid(ire, SVT_VERIFY_BUS, SQ_ALL_16,
492 PCI_BDF2(bus, devfn));
493 }
494 break;
496 default:
497 dprintk(XENLOG_WARNING VTDPREFIX, "d%d: unknown(%u): bdf = %x:%x.%x\n",
498 pdev->domain->domain_id, type,
499 bus, PCI_SLOT(devfn), PCI_FUNC(devfn));
500 break;
501 }
502 }
504 static int remap_entry_to_msi_msg(
505 struct iommu *iommu, struct msi_msg *msg)
506 {
507 struct iremap_entry *iremap_entry = NULL, *iremap_entries;
508 struct msi_msg_remap_entry *remap_rte;
509 int index;
510 unsigned long flags;
511 struct ir_ctrl *ir_ctrl = iommu_ir_ctrl(iommu);
513 if ( ir_ctrl == NULL )
514 {
515 dprintk(XENLOG_ERR VTDPREFIX,
516 "remap_entry_to_msi_msg: ir_ctl == NULL");
517 return -EFAULT;
518 }
520 remap_rte = (struct msi_msg_remap_entry *) msg;
521 index = (remap_rte->address_lo.index_15 << 15) |
522 remap_rte->address_lo.index_0_14;
524 if ( index < 0 || index > IREMAP_ENTRY_NR - 1 )
525 {
526 dprintk(XENLOG_ERR VTDPREFIX,
527 "%s: index (%d) for remap table is invalid !\n",
528 __func__, index);
529 return -EFAULT;
530 }
532 spin_lock_irqsave(&ir_ctrl->iremap_lock, flags);
534 GET_IREMAP_ENTRY(ir_ctrl->iremap_maddr, index,
535 iremap_entries, iremap_entry);
537 if ( iremap_entry->hi_val == 0 && iremap_entry->lo_val == 0 )
538 {
539 dprintk(XENLOG_ERR VTDPREFIX,
540 "%s: index (%d) get an empty entry!\n",
541 __func__, index);
542 return -EFAULT;
543 }
545 msg->address_hi = MSI_ADDR_BASE_HI;
546 msg->address_lo =
547 MSI_ADDR_BASE_LO |
548 ((iremap_entry->lo.dm == 0) ?
549 MSI_ADDR_DESTMODE_PHYS:
550 MSI_ADDR_DESTMODE_LOGIC) |
551 ((iremap_entry->lo.dlm != dest_LowestPrio) ?
552 MSI_ADDR_REDIRECTION_CPU:
553 MSI_ADDR_REDIRECTION_LOWPRI);
554 if ( x2apic_enabled )
555 msg->dest32 = iremap_entry->lo.dst;
556 else
557 msg->address_lo |=
558 ((iremap_entry->lo.dst >> 8) & 0xff ) << MSI_ADDR_DEST_ID_SHIFT;
560 msg->data =
561 MSI_DATA_TRIGGER_EDGE |
562 MSI_DATA_LEVEL_ASSERT |
563 ((iremap_entry->lo.dlm != dest_LowestPrio) ?
564 MSI_DATA_DELIVERY_FIXED:
565 MSI_DATA_DELIVERY_LOWPRI) |
566 iremap_entry->lo.vector;
568 unmap_vtd_domain_page(iremap_entries);
569 spin_unlock_irqrestore(&ir_ctrl->iremap_lock, flags);
570 return 0;
571 }
573 static int msi_msg_to_remap_entry(
574 struct iommu *iommu, struct pci_dev *pdev,
575 struct msi_desc *msi_desc, struct msi_msg *msg)
576 {
577 struct iremap_entry *iremap_entry = NULL, *iremap_entries;
578 struct iremap_entry new_ire;
579 struct msi_msg_remap_entry *remap_rte;
580 int index;
581 unsigned long flags;
582 struct ir_ctrl *ir_ctrl = iommu_ir_ctrl(iommu);
584 remap_rte = (struct msi_msg_remap_entry *) msg;
585 spin_lock_irqsave(&ir_ctrl->iremap_lock, flags);
587 if ( msg == NULL )
588 {
589 /* Free specified unused IRTE */
590 free_remap_entry(iommu, msi_desc->remap_index);
591 spin_unlock_irqrestore(&ir_ctrl->iremap_lock, flags);
592 return 0;
593 }
595 if ( msi_desc->remap_index < 0 )
596 {
597 /*
598 * TODO: Multiple-vector MSI requires allocating multiple continuous
599 * entries and configuring addr/data of msi_msg in different way. So
600 * alloca_remap_entry will be changed if enabling multiple-vector MSI
601 * in future.
602 */
603 index = alloc_remap_entry(iommu);
604 msi_desc->remap_index = index;
605 }
606 else
607 index = msi_desc->remap_index;
609 if ( index > IREMAP_ENTRY_NR - 1 )
610 {
611 dprintk(XENLOG_ERR VTDPREFIX,
612 "%s: intremap index (%d) is larger than"
613 " the maximum index (%d)!\n",
614 __func__, index, IREMAP_ENTRY_NR - 1);
615 msi_desc->remap_index = -1;
616 spin_unlock_irqrestore(&ir_ctrl->iremap_lock, flags);
617 return -EFAULT;
618 }
620 GET_IREMAP_ENTRY(ir_ctrl->iremap_maddr, index,
621 iremap_entries, iremap_entry);
623 memcpy(&new_ire, iremap_entry, sizeof(struct iremap_entry));
625 /* Set interrupt remapping table entry */
626 new_ire.lo.fpd = 0;
627 new_ire.lo.dm = (msg->address_lo >> MSI_ADDR_DESTMODE_SHIFT) & 0x1;
628 new_ire.lo.rh = 0;
629 new_ire.lo.tm = (msg->data >> MSI_DATA_TRIGGER_SHIFT) & 0x1;
630 new_ire.lo.dlm = (msg->data >> MSI_DATA_DELIVERY_MODE_SHIFT) & 0x1;
631 new_ire.lo.avail = 0;
632 new_ire.lo.res_1 = 0;
633 new_ire.lo.vector = (msg->data >> MSI_DATA_VECTOR_SHIFT) &
634 MSI_DATA_VECTOR_MASK;
635 new_ire.lo.res_2 = 0;
636 if ( x2apic_enabled )
637 new_ire.lo.dst = msg->dest32;
638 else
639 new_ire.lo.dst = ((msg->address_lo >> MSI_ADDR_DEST_ID_SHIFT)
640 & 0xff) << 8;
642 set_msi_source_id(pdev, &new_ire);
643 new_ire.hi.res_1 = 0;
644 new_ire.lo.p = 1; /* finally, set present bit */
646 /* now construct new MSI/MSI-X rte entry */
647 remap_rte->address_lo.dontcare = 0;
648 remap_rte->address_lo.index_15 = (index >> 15) & 0x1;
649 remap_rte->address_lo.index_0_14 = index & 0x7fff;
650 remap_rte->address_lo.SHV = 1;
651 remap_rte->address_lo.format = 1;
653 remap_rte->address_hi = 0;
654 remap_rte->data = 0;
656 memcpy(iremap_entry, &new_ire, sizeof(struct iremap_entry));
657 iommu_flush_cache_entry(iremap_entry, sizeof(struct iremap_entry));
658 iommu_flush_iec_index(iommu, 0, index);
659 invalidate_sync(iommu);
661 unmap_vtd_domain_page(iremap_entries);
662 spin_unlock_irqrestore(&ir_ctrl->iremap_lock, flags);
663 return 0;
664 }
666 void msi_msg_read_remap_rte(
667 struct msi_desc *msi_desc, struct msi_msg *msg)
668 {
669 struct pci_dev *pdev = msi_desc->dev;
670 struct acpi_drhd_unit *drhd = NULL;
671 struct iommu *iommu = NULL;
672 struct ir_ctrl *ir_ctrl;
674 if ( (drhd = acpi_find_matched_drhd_unit(pdev)) == NULL )
675 return;
676 iommu = drhd->iommu;
678 ir_ctrl = iommu_ir_ctrl(iommu);
679 if ( !iommu || !ir_ctrl || ir_ctrl->iremap_maddr == 0 )
680 return;
682 remap_entry_to_msi_msg(iommu, msg);
683 }
685 void msi_msg_write_remap_rte(
686 struct msi_desc *msi_desc, struct msi_msg *msg)
687 {
688 struct pci_dev *pdev = msi_desc->dev;
689 struct acpi_drhd_unit *drhd = NULL;
690 struct iommu *iommu = NULL;
691 struct ir_ctrl *ir_ctrl;
693 if ( (drhd = acpi_find_matched_drhd_unit(pdev)) == NULL )
694 return;
695 iommu = drhd->iommu;
697 ir_ctrl = iommu_ir_ctrl(iommu);
698 if ( !iommu || !ir_ctrl || ir_ctrl->iremap_maddr == 0 )
699 return;
701 msi_msg_to_remap_entry(iommu, pdev, msi_desc, msg);
702 }
703 #elif defined(__ia64__)
704 void msi_msg_read_remap_rte(
705 struct msi_desc *msi_desc, struct msi_msg *msg)
706 {
707 /* TODO. */
708 }
710 void msi_msg_write_remap_rte(
711 struct msi_desc *msi_desc, struct msi_msg *msg)
712 {
713 /* TODO. */
714 }
715 #endif
717 int enable_intremap(struct iommu *iommu, int eim)
718 {
719 struct acpi_drhd_unit *drhd;
720 struct ir_ctrl *ir_ctrl;
721 u32 sts, gcmd;
722 unsigned long flags;
724 ASSERT(ecap_intr_remap(iommu->ecap) && iommu_intremap);
726 ir_ctrl = iommu_ir_ctrl(iommu);
727 sts = dmar_readl(iommu->reg, DMAR_GSTS_REG);
729 /* Return if already enabled by Xen */
730 if ( (sts & DMA_GSTS_IRES) && ir_ctrl->iremap_maddr )
731 return 0;
733 sts = dmar_readl(iommu->reg, DMAR_GSTS_REG);
734 if ( !(sts & DMA_GSTS_QIES) )
735 {
736 dprintk(XENLOG_ERR VTDPREFIX,
737 "Queued invalidation is not enabled, should not enable "
738 "interrupt remapping\n");
739 return -EINVAL;
740 }
742 if ( ir_ctrl->iremap_maddr == 0 )
743 {
744 drhd = iommu_to_drhd(iommu);
745 ir_ctrl->iremap_maddr = alloc_pgtable_maddr(drhd, IREMAP_ARCH_PAGE_NR);
746 if ( ir_ctrl->iremap_maddr == 0 )
747 {
748 dprintk(XENLOG_WARNING VTDPREFIX,
749 "Cannot allocate memory for ir_ctrl->iremap_maddr\n");
750 return -ENOMEM;
751 }
752 ir_ctrl->iremap_num = 0;
753 }
755 #ifdef CONFIG_X86
756 /* set extended interrupt mode bit */
757 ir_ctrl->iremap_maddr |=
758 eim ? (1 << IRTA_REG_EIME_SHIFT) : 0;
759 #endif
760 spin_lock_irqsave(&iommu->register_lock, flags);
762 /* set size of the interrupt remapping table */
763 ir_ctrl->iremap_maddr |= IRTA_REG_TABLE_SIZE;
764 dmar_writeq(iommu->reg, DMAR_IRTA_REG, ir_ctrl->iremap_maddr);
766 /* set SIRTP */
767 gcmd = dmar_readl(iommu->reg, DMAR_GSTS_REG);
768 gcmd |= DMA_GCMD_SIRTP;
769 dmar_writel(iommu->reg, DMAR_GCMD_REG, gcmd);
771 IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG, dmar_readl,
772 (sts & DMA_GSTS_SIRTPS), sts);
773 spin_unlock_irqrestore(&iommu->register_lock, flags);
775 /* After set SIRTP, must globally invalidate the interrupt entry cache */
776 iommu_flush_iec_global(iommu);
778 spin_lock_irqsave(&iommu->register_lock, flags);
779 /* enable interrupt remapping hardware */
780 gcmd |= DMA_GCMD_IRE;
781 dmar_writel(iommu->reg, DMAR_GCMD_REG, gcmd);
783 IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG, dmar_readl,
784 (sts & DMA_GSTS_IRES), sts);
785 spin_unlock_irqrestore(&iommu->register_lock, flags);
787 return init_apic_pin_2_ir_idx();
788 }
790 void disable_intremap(struct iommu *iommu)
791 {
792 u32 sts;
793 unsigned long flags;
795 if ( !ecap_intr_remap(iommu->ecap) )
796 return;
798 spin_lock_irqsave(&iommu->register_lock, flags);
799 sts = dmar_readl(iommu->reg, DMAR_GSTS_REG);
800 if ( !(sts & DMA_GSTS_IRES) )
801 goto out;
803 dmar_writel(iommu->reg, DMAR_GCMD_REG, sts & (~DMA_GCMD_IRE));
805 IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG, dmar_readl,
806 !(sts & DMA_GSTS_IRES), sts);
807 out:
808 spin_unlock_irqrestore(&iommu->register_lock, flags);
809 }
811 /*
812 * This function is used to enable Interrutp remapping when
813 * enable x2apic
814 */
815 int iommu_enable_IR(void)
816 {
817 struct acpi_drhd_unit *drhd;
818 struct iommu *iommu;
820 if ( !iommu_supports_eim() )
821 return -1;
823 for_each_drhd_unit ( drhd )
824 {
825 struct qi_ctrl *qi_ctrl = NULL;
827 iommu = drhd->iommu;
828 qi_ctrl = iommu_qi_ctrl(iommu);
830 /* Clear previous faults */
831 clear_fault_bits(iommu);
833 /*
834 * Disable interrupt remapping and queued invalidation if
835 * already enabled by BIOS
836 */
837 disable_intremap(iommu);
838 disable_qinval(iommu);
839 }
841 /* Enable queue invalidation */
842 for_each_drhd_unit ( drhd )
843 {
844 iommu = drhd->iommu;
845 if ( enable_qinval(iommu) != 0 )
846 {
847 dprintk(XENLOG_INFO VTDPREFIX,
848 "Failed to enable Queued Invalidation!\n");
849 return -1;
850 }
851 }
853 /* Enable interrupt remapping */
854 for_each_drhd_unit ( drhd )
855 {
856 iommu = drhd->iommu;
857 if ( enable_intremap(iommu, 1) )
858 {
859 dprintk(XENLOG_INFO VTDPREFIX,
860 "Failed to enable Interrupt Remapping!\n");
861 return -1;
862 }
863 }
865 return 0;
866 }
868 /*
869 * Check if interrupt remapping is enabled or not
870 * return 1: enabled
871 * return 0: not enabled
872 */
873 int intremap_enabled(void)
874 {
875 struct acpi_drhd_unit *drhd;
876 u32 sts;
878 for_each_drhd_unit ( drhd )
879 {
880 sts = dmar_readl(drhd->iommu->reg, DMAR_GSTS_REG);
881 if ( !(sts & DMA_GSTS_IRES) )
882 return 0;
883 }
885 return 1;
886 }