xcp-1.6-updates/xen-4.1.hg

view xen/drivers/passthrough/amd/iommu_init.c @ 23263:b01091775dd9

iommu: Move IOMMU faults handling into softirq for AMD-Vi.

Dealing with interrupts from AMD-Vi IOMMU(s) is deferred to a
softirq-tasklet, raised by the actual IRQ handler. To avoid more
interrupts being generated (because of further faults), they must be
masked in the IOMMU within the low level IRQ handler and enabled back
in the tasklet body. Notice that this may cause the log to overflow,
but none of the existing entry will be overwritten.

Signed-off-by: Dario Faggioli <dario.faggioli@citrix.com>
Committed-by: Keir Fraser <keir@xen.org>
xen-unstable changeset: 24527:028230eb2359
xen-unstable date: Fri Jan 20 10:20:32 2012 +0000
author Dario Faggioli <dario.faggioli@citrix.com>
date Thu Mar 08 10:05:04 2012 +0000 (2012-03-08)
parents 1092a143ef9d
children
line source
1 /*
2 * Copyright (C) 2007 Advanced Micro Devices, Inc.
3 * Author: Leo Duran <leo.duran@amd.com>
4 * Author: Wei Wang <wei.wang2@amd.com> - adapted to xen
5 *
6 * This program is free software; you can redistribute it and/or modify
7 * it under the terms of the GNU General Public License as published by
8 * the Free Software Foundation; either version 2 of the License, or
9 * (at your option) any later version.
10 *
11 * This program is distributed in the hope that it will be useful,
12 * but WITHOUT ANY WARRANTY; without even the implied warranty of
13 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14 * GNU General Public License for more details.
15 *
16 * You should have received a copy of the GNU General Public License
17 * along with this program; if not, write to the Free Software
18 * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
19 */
21 #include <xen/config.h>
22 #include <xen/errno.h>
23 #include <xen/pci.h>
24 #include <xen/pci_regs.h>
25 #include <asm/amd-iommu.h>
26 #include <asm/msi.h>
27 #include <asm/hvm/svm/amd-iommu-proto.h>
28 #include <asm-x86/fixmap.h>
29 #include <mach_apic.h>
31 static struct amd_iommu **irq_to_iommu;
32 static int nr_amd_iommus;
33 static long amd_iommu_cmd_buffer_entries = IOMMU_CMD_BUFFER_DEFAULT_ENTRIES;
34 static long amd_iommu_event_log_entries = IOMMU_EVENT_LOG_DEFAULT_ENTRIES;
36 static struct tasklet amd_iommu_irq_tasklet;
38 unsigned short ivrs_bdf_entries;
39 struct ivrs_mappings *ivrs_mappings;
40 struct list_head amd_iommu_head;
41 struct table_struct device_table;
43 static int __init map_iommu_mmio_region(struct amd_iommu *iommu)
44 {
45 unsigned long mfn;
47 if ( nr_amd_iommus > MAX_AMD_IOMMUS )
48 {
49 AMD_IOMMU_DEBUG("nr_amd_iommus %d > MAX_IOMMUS\n", nr_amd_iommus);
50 return -ENOMEM;
51 }
53 iommu->mmio_base = (void *)fix_to_virt(
54 FIX_IOMMU_MMIO_BASE_0 + nr_amd_iommus * MMIO_PAGES_PER_IOMMU);
55 mfn = (unsigned long)(iommu->mmio_base_phys >> PAGE_SHIFT);
56 map_pages_to_xen((unsigned long)iommu->mmio_base, mfn,
57 MMIO_PAGES_PER_IOMMU, PAGE_HYPERVISOR_NOCACHE);
59 memset(iommu->mmio_base, 0, IOMMU_MMIO_REGION_LENGTH);
61 return 0;
62 }
64 static void __init unmap_iommu_mmio_region(struct amd_iommu *iommu)
65 {
66 if ( iommu->mmio_base )
67 {
68 iounmap(iommu->mmio_base);
69 iommu->mmio_base = NULL;
70 }
71 }
73 static void register_iommu_dev_table_in_mmio_space(struct amd_iommu *iommu)
74 {
75 u64 addr_64, addr_lo, addr_hi;
76 u32 entry;
78 addr_64 = (u64)virt_to_maddr(iommu->dev_table.buffer);
79 addr_lo = addr_64 & DMA_32BIT_MASK;
80 addr_hi = addr_64 >> 32;
82 set_field_in_reg_u32((u32)addr_lo >> PAGE_SHIFT, 0,
83 IOMMU_DEV_TABLE_BASE_LOW_MASK,
84 IOMMU_DEV_TABLE_BASE_LOW_SHIFT, &entry);
85 set_field_in_reg_u32((iommu->dev_table.alloc_size / PAGE_SIZE) - 1,
86 entry, IOMMU_DEV_TABLE_SIZE_MASK,
87 IOMMU_DEV_TABLE_SIZE_SHIFT, &entry);
88 writel(entry, iommu->mmio_base + IOMMU_DEV_TABLE_BASE_LOW_OFFSET);
90 set_field_in_reg_u32((u32)addr_hi, 0,
91 IOMMU_DEV_TABLE_BASE_HIGH_MASK,
92 IOMMU_DEV_TABLE_BASE_HIGH_SHIFT, &entry);
93 writel(entry, iommu->mmio_base + IOMMU_DEV_TABLE_BASE_HIGH_OFFSET);
94 }
96 static void register_iommu_cmd_buffer_in_mmio_space(struct amd_iommu *iommu)
97 {
98 u64 addr_64, addr_lo, addr_hi;
99 u32 power_of2_entries;
100 u32 entry;
102 addr_64 = (u64)virt_to_maddr(iommu->cmd_buffer.buffer);
103 addr_lo = addr_64 & DMA_32BIT_MASK;
104 addr_hi = addr_64 >> 32;
106 set_field_in_reg_u32((u32)addr_lo >> PAGE_SHIFT, 0,
107 IOMMU_CMD_BUFFER_BASE_LOW_MASK,
108 IOMMU_CMD_BUFFER_BASE_LOW_SHIFT, &entry);
109 writel(entry, iommu->mmio_base + IOMMU_CMD_BUFFER_BASE_LOW_OFFSET);
111 power_of2_entries = get_order_from_bytes(iommu->cmd_buffer.alloc_size) +
112 IOMMU_CMD_BUFFER_POWER_OF2_ENTRIES_PER_PAGE;
114 set_field_in_reg_u32((u32)addr_hi, 0,
115 IOMMU_CMD_BUFFER_BASE_HIGH_MASK,
116 IOMMU_CMD_BUFFER_BASE_HIGH_SHIFT, &entry);
117 set_field_in_reg_u32(power_of2_entries, entry,
118 IOMMU_CMD_BUFFER_LENGTH_MASK,
119 IOMMU_CMD_BUFFER_LENGTH_SHIFT, &entry);
120 writel(entry, iommu->mmio_base+IOMMU_CMD_BUFFER_BASE_HIGH_OFFSET);
121 }
123 static void register_iommu_event_log_in_mmio_space(struct amd_iommu *iommu)
124 {
125 u64 addr_64, addr_lo, addr_hi;
126 u32 power_of2_entries;
127 u32 entry;
129 addr_64 = (u64)virt_to_maddr(iommu->event_log.buffer);
130 addr_lo = addr_64 & DMA_32BIT_MASK;
131 addr_hi = addr_64 >> 32;
133 set_field_in_reg_u32((u32)addr_lo >> PAGE_SHIFT, 0,
134 IOMMU_EVENT_LOG_BASE_LOW_MASK,
135 IOMMU_EVENT_LOG_BASE_LOW_SHIFT, &entry);
136 writel(entry, iommu->mmio_base + IOMMU_EVENT_LOG_BASE_LOW_OFFSET);
138 power_of2_entries = get_order_from_bytes(iommu->event_log.alloc_size) +
139 IOMMU_EVENT_LOG_POWER_OF2_ENTRIES_PER_PAGE;
141 set_field_in_reg_u32((u32)addr_hi, 0,
142 IOMMU_EVENT_LOG_BASE_HIGH_MASK,
143 IOMMU_EVENT_LOG_BASE_HIGH_SHIFT, &entry);
144 set_field_in_reg_u32(power_of2_entries, entry,
145 IOMMU_EVENT_LOG_LENGTH_MASK,
146 IOMMU_EVENT_LOG_LENGTH_SHIFT, &entry);
147 writel(entry, iommu->mmio_base+IOMMU_EVENT_LOG_BASE_HIGH_OFFSET);
148 }
150 static void set_iommu_translation_control(struct amd_iommu *iommu,
151 int enable)
152 {
153 u32 entry;
155 entry = readl(iommu->mmio_base + IOMMU_CONTROL_MMIO_OFFSET);
157 if ( enable )
158 {
159 set_field_in_reg_u32(iommu->ht_tunnel_support ? IOMMU_CONTROL_ENABLED :
160 IOMMU_CONTROL_DISABLED, entry,
161 IOMMU_CONTROL_HT_TUNNEL_TRANSLATION_MASK,
162 IOMMU_CONTROL_HT_TUNNEL_TRANSLATION_SHIFT, &entry);
163 set_field_in_reg_u32(iommu->isochronous ? IOMMU_CONTROL_ENABLED :
164 IOMMU_CONTROL_DISABLED, entry,
165 IOMMU_CONTROL_ISOCHRONOUS_MASK,
166 IOMMU_CONTROL_ISOCHRONOUS_SHIFT, &entry);
167 set_field_in_reg_u32(iommu->coherent ? IOMMU_CONTROL_ENABLED :
168 IOMMU_CONTROL_DISABLED, entry,
169 IOMMU_CONTROL_COHERENT_MASK,
170 IOMMU_CONTROL_COHERENT_SHIFT, &entry);
171 set_field_in_reg_u32(iommu->res_pass_pw ? IOMMU_CONTROL_ENABLED :
172 IOMMU_CONTROL_DISABLED, entry,
173 IOMMU_CONTROL_RESP_PASS_POSTED_WRITE_MASK,
174 IOMMU_CONTROL_RESP_PASS_POSTED_WRITE_SHIFT, &entry);
175 /* do not set PassPW bit */
176 set_field_in_reg_u32(IOMMU_CONTROL_DISABLED, entry,
177 IOMMU_CONTROL_PASS_POSTED_WRITE_MASK,
178 IOMMU_CONTROL_PASS_POSTED_WRITE_SHIFT, &entry);
179 }
180 set_field_in_reg_u32(enable ? IOMMU_CONTROL_ENABLED :
181 IOMMU_CONTROL_DISABLED, entry,
182 IOMMU_CONTROL_TRANSLATION_ENABLE_MASK,
183 IOMMU_CONTROL_TRANSLATION_ENABLE_SHIFT, &entry);
184 writel(entry, iommu->mmio_base+IOMMU_CONTROL_MMIO_OFFSET);
185 }
187 static void set_iommu_command_buffer_control(struct amd_iommu *iommu,
188 int enable)
189 {
190 u32 entry;
192 entry = readl(iommu->mmio_base + IOMMU_CONTROL_MMIO_OFFSET);
193 set_field_in_reg_u32(enable ? IOMMU_CONTROL_ENABLED :
194 IOMMU_CONTROL_DISABLED, entry,
195 IOMMU_CONTROL_COMMAND_BUFFER_ENABLE_MASK,
196 IOMMU_CONTROL_COMMAND_BUFFER_ENABLE_SHIFT, &entry);
198 /*reset head and tail pointer manually before enablement */
199 if ( enable == IOMMU_CONTROL_ENABLED )
200 {
201 writel(0x0, iommu->mmio_base + IOMMU_CMD_BUFFER_HEAD_OFFSET);
202 writel(0x0, iommu->mmio_base + IOMMU_CMD_BUFFER_TAIL_OFFSET);
203 }
205 writel(entry, iommu->mmio_base+IOMMU_CONTROL_MMIO_OFFSET);
206 }
208 static void register_iommu_exclusion_range(struct amd_iommu *iommu)
209 {
210 u64 addr_lo, addr_hi;
211 u32 entry;
213 addr_lo = iommu->exclusion_limit & DMA_32BIT_MASK;
214 addr_hi = iommu->exclusion_limit >> 32;
216 set_field_in_reg_u32((u32)addr_hi, 0,
217 IOMMU_EXCLUSION_LIMIT_HIGH_MASK,
218 IOMMU_EXCLUSION_LIMIT_HIGH_SHIFT, &entry);
219 writel(entry, iommu->mmio_base+IOMMU_EXCLUSION_LIMIT_HIGH_OFFSET);
221 set_field_in_reg_u32((u32)addr_lo >> PAGE_SHIFT, 0,
222 IOMMU_EXCLUSION_LIMIT_LOW_MASK,
223 IOMMU_EXCLUSION_LIMIT_LOW_SHIFT, &entry);
224 writel(entry, iommu->mmio_base+IOMMU_EXCLUSION_LIMIT_LOW_OFFSET);
226 addr_lo = iommu->exclusion_base & DMA_32BIT_MASK;
227 addr_hi = iommu->exclusion_base >> 32;
229 set_field_in_reg_u32((u32)addr_hi, 0,
230 IOMMU_EXCLUSION_BASE_HIGH_MASK,
231 IOMMU_EXCLUSION_BASE_HIGH_SHIFT, &entry);
232 writel(entry, iommu->mmio_base+IOMMU_EXCLUSION_BASE_HIGH_OFFSET);
234 set_field_in_reg_u32((u32)addr_lo >> PAGE_SHIFT, 0,
235 IOMMU_EXCLUSION_BASE_LOW_MASK,
236 IOMMU_EXCLUSION_BASE_LOW_SHIFT, &entry);
238 set_field_in_reg_u32(iommu->exclusion_allow_all, entry,
239 IOMMU_EXCLUSION_ALLOW_ALL_MASK,
240 IOMMU_EXCLUSION_ALLOW_ALL_SHIFT, &entry);
242 set_field_in_reg_u32(iommu->exclusion_enable, entry,
243 IOMMU_EXCLUSION_RANGE_ENABLE_MASK,
244 IOMMU_EXCLUSION_RANGE_ENABLE_SHIFT, &entry);
245 writel(entry, iommu->mmio_base+IOMMU_EXCLUSION_BASE_LOW_OFFSET);
246 }
248 static void set_iommu_event_log_control(struct amd_iommu *iommu,
249 int enable)
250 {
251 u32 entry;
253 entry = readl(iommu->mmio_base + IOMMU_CONTROL_MMIO_OFFSET);
254 set_field_in_reg_u32(enable ? IOMMU_CONTROL_ENABLED :
255 IOMMU_CONTROL_DISABLED, entry,
256 IOMMU_CONTROL_EVENT_LOG_ENABLE_MASK,
257 IOMMU_CONTROL_EVENT_LOG_ENABLE_SHIFT, &entry);
258 set_field_in_reg_u32(enable ? IOMMU_CONTROL_ENABLED :
259 IOMMU_CONTROL_DISABLED, entry,
260 IOMMU_CONTROL_EVENT_LOG_INT_MASK,
261 IOMMU_CONTROL_EVENT_LOG_INT_SHIFT, &entry);
262 set_field_in_reg_u32(IOMMU_CONTROL_DISABLED, entry,
263 IOMMU_CONTROL_COMP_WAIT_INT_MASK,
264 IOMMU_CONTROL_COMP_WAIT_INT_SHIFT, &entry);
266 /*reset head and tail pointer manually before enablement */
267 if ( enable == IOMMU_CONTROL_ENABLED )
268 {
269 writel(0x0, iommu->mmio_base + IOMMU_EVENT_LOG_HEAD_OFFSET);
270 writel(0x0, iommu->mmio_base + IOMMU_EVENT_LOG_TAIL_OFFSET);
271 }
272 writel(entry, iommu->mmio_base + IOMMU_CONTROL_MMIO_OFFSET);
273 }
275 static void parse_event_log_entry(u32 entry[]);
277 static int amd_iommu_read_event_log(struct amd_iommu *iommu)
278 {
279 u32 tail, head, *event_log;
281 BUG_ON( !iommu );
283 /* make sure there's an entry in the log */
284 tail = readl(iommu->mmio_base + IOMMU_EVENT_LOG_TAIL_OFFSET);
285 tail = get_field_from_reg_u32(tail,
286 IOMMU_EVENT_LOG_TAIL_MASK,
287 IOMMU_EVENT_LOG_TAIL_SHIFT);
289 while ( tail != iommu->event_log_head )
290 {
291 /* read event log entry */
292 event_log = (u32 *)(iommu->event_log.buffer +
293 (iommu->event_log_head *
294 IOMMU_EVENT_LOG_ENTRY_SIZE));
296 parse_event_log_entry(event_log);
298 if ( ++iommu->event_log_head == iommu->event_log.entries )
299 iommu->event_log_head = 0;
301 /* update head pointer */
302 set_field_in_reg_u32(iommu->event_log_head, 0,
303 IOMMU_EVENT_LOG_HEAD_MASK,
304 IOMMU_EVENT_LOG_HEAD_SHIFT, &head);
305 writel(head, iommu->mmio_base + IOMMU_EVENT_LOG_HEAD_OFFSET);
306 }
308 return 0;
309 }
311 static void amd_iommu_reset_event_log(struct amd_iommu *iommu)
312 {
313 u32 entry;
314 int log_run;
315 int loop_count = 1000;
317 /* wait until EventLogRun bit = 0 */
318 do {
319 entry = readl(iommu->mmio_base + IOMMU_STATUS_MMIO_OFFSET);
320 log_run = get_field_from_reg_u32(entry,
321 IOMMU_STATUS_EVENT_LOG_RUN_MASK,
322 IOMMU_STATUS_EVENT_LOG_RUN_SHIFT);
323 loop_count--;
324 } while ( log_run && loop_count );
326 if ( log_run )
327 {
328 AMD_IOMMU_DEBUG("Warning: EventLogRun bit is not cleared"
329 "before reset!\n");
330 return;
331 }
333 set_iommu_event_log_control(iommu, IOMMU_CONTROL_DISABLED);
335 /* read event log for debugging */
336 amd_iommu_read_event_log(iommu);
338 /*clear overflow bit */
339 set_field_in_reg_u32(IOMMU_CONTROL_DISABLED, entry,
340 IOMMU_STATUS_EVENT_OVERFLOW_MASK,
341 IOMMU_STATUS_EVENT_OVERFLOW_SHIFT, &entry);
342 writel(entry, iommu->mmio_base+IOMMU_STATUS_MMIO_OFFSET);
344 /*reset event log base address */
345 iommu->event_log_head = 0;
347 set_iommu_event_log_control(iommu, IOMMU_CONTROL_ENABLED);
348 }
350 static void iommu_msi_set_affinity(unsigned int irq, cpumask_t mask)
351 {
352 struct msi_msg msg;
353 unsigned int dest;
354 struct amd_iommu *iommu = irq_to_iommu[irq];
355 struct irq_desc *desc = irq_to_desc(irq);
356 struct irq_cfg *cfg = desc->chip_data;
357 u8 bus = (iommu->bdf >> 8) & 0xff;
358 u8 dev = PCI_SLOT(iommu->bdf & 0xff);
359 u8 func = PCI_FUNC(iommu->bdf & 0xff);
361 dest = set_desc_affinity(desc, &mask);
363 if ( dest == BAD_APICID )
364 {
365 dprintk(XENLOG_ERR, "Set iommu interrupt affinity error!\n");
366 return;
367 }
369 memset(&msg, 0, sizeof(msg));
370 msg.data = MSI_DATA_VECTOR(cfg->vector) & 0xff;
371 msg.data |= 1 << 14;
372 msg.data |= (INT_DELIVERY_MODE != dest_LowestPrio) ?
373 MSI_DATA_DELIVERY_FIXED:
374 MSI_DATA_DELIVERY_LOWPRI;
376 msg.address_hi =0;
377 msg.address_lo = (MSI_ADDRESS_HEADER << (MSI_ADDRESS_HEADER_SHIFT + 8));
378 msg.address_lo |= INT_DEST_MODE ? MSI_ADDR_DESTMODE_LOGIC:
379 MSI_ADDR_DESTMODE_PHYS;
380 msg.address_lo |= (INT_DELIVERY_MODE != dest_LowestPrio) ?
381 MSI_ADDR_REDIRECTION_CPU:
382 MSI_ADDR_REDIRECTION_LOWPRI;
383 msg.address_lo |= MSI_ADDR_DEST_ID(dest & 0xff);
385 pci_conf_write32(bus, dev, func,
386 iommu->msi_cap + PCI_MSI_DATA_64, msg.data);
387 pci_conf_write32(bus, dev, func,
388 iommu->msi_cap + PCI_MSI_ADDRESS_LO, msg.address_lo);
389 pci_conf_write32(bus, dev, func,
390 iommu->msi_cap + PCI_MSI_ADDRESS_HI, msg.address_hi);
392 }
394 static void amd_iommu_msi_enable(struct amd_iommu *iommu, int flag)
395 {
396 u16 control;
397 int bus = (iommu->bdf >> 8) & 0xff;
398 int dev = PCI_SLOT(iommu->bdf & 0xff);
399 int func = PCI_FUNC(iommu->bdf & 0xff);
401 control = pci_conf_read16(bus, dev, func,
402 iommu->msi_cap + PCI_MSI_FLAGS);
403 control &= ~(1);
404 if ( flag )
405 control |= flag;
406 pci_conf_write16(bus, dev, func,
407 iommu->msi_cap + PCI_MSI_FLAGS, control);
408 }
410 static void iommu_msi_unmask(unsigned int irq)
411 {
412 unsigned long flags;
413 struct amd_iommu *iommu = irq_to_iommu[irq];
415 /* FIXME: do not support mask bits at the moment */
416 if ( iommu->maskbit )
417 return;
419 spin_lock_irqsave(&iommu->lock, flags);
420 amd_iommu_msi_enable(iommu, IOMMU_CONTROL_ENABLED);
421 spin_unlock_irqrestore(&iommu->lock, flags);
422 }
424 static void iommu_msi_mask(unsigned int irq)
425 {
426 unsigned long flags;
427 struct amd_iommu *iommu = irq_to_iommu[irq];
428 struct irq_desc *desc = irq_to_desc(irq);
430 irq_complete_move(&desc);
432 /* FIXME: do not support mask bits at the moment */
433 if ( iommu->maskbit )
434 return;
436 spin_lock_irqsave(&iommu->lock, flags);
437 amd_iommu_msi_enable(iommu, IOMMU_CONTROL_DISABLED);
438 spin_unlock_irqrestore(&iommu->lock, flags);
439 }
441 static unsigned int iommu_msi_startup(unsigned int irq)
442 {
443 iommu_msi_unmask(irq);
444 return 0;
445 }
447 static void iommu_msi_end(unsigned int irq, u8 vector)
448 {
449 iommu_msi_unmask(irq);
450 ack_APIC_irq();
451 }
454 static hw_irq_controller iommu_msi_type = {
455 .typename = "AMD-IOMMU-MSI",
456 .startup = iommu_msi_startup,
457 .shutdown = iommu_msi_mask,
458 .enable = iommu_msi_unmask,
459 .disable = iommu_msi_mask,
460 .ack = iommu_msi_mask,
461 .end = iommu_msi_end,
462 .set_affinity = iommu_msi_set_affinity,
463 };
465 static void parse_event_log_entry(u32 entry[])
466 {
467 u16 domain_id, device_id, bdf, cword;
468 u32 code;
469 u64 *addr;
470 char * event_str[] = {"ILLEGAL_DEV_TABLE_ENTRY",
471 "IO_PAGE_FAULT",
472 "DEV_TABLE_HW_ERROR",
473 "PAGE_TABLE_HW_ERROR",
474 "ILLEGAL_COMMAND_ERROR",
475 "COMMAND_HW_ERROR",
476 "IOTLB_INV_TIMEOUT",
477 "INVALID_DEV_REQUEST"};
479 code = get_field_from_reg_u32(entry[1], IOMMU_EVENT_CODE_MASK,
480 IOMMU_EVENT_CODE_SHIFT);
482 if ( (code > IOMMU_EVENT_INVALID_DEV_REQUEST) ||
483 (code < IOMMU_EVENT_ILLEGAL_DEV_TABLE_ENTRY) )
484 {
485 AMD_IOMMU_DEBUG("Invalid event log entry!\n");
486 return;
487 }
489 if ( code == IOMMU_EVENT_IO_PAGE_FAULT )
490 {
491 device_id = get_field_from_reg_u32(entry[0],
492 IOMMU_EVENT_DEVICE_ID_MASK,
493 IOMMU_EVENT_DEVICE_ID_SHIFT);
494 domain_id = get_field_from_reg_u32(entry[1],
495 IOMMU_EVENT_DOMAIN_ID_MASK,
496 IOMMU_EVENT_DOMAIN_ID_SHIFT);
497 addr= (u64*) (entry + 2);
498 printk(XENLOG_ERR "AMD-Vi: "
499 "%s: domain = %d, device id = 0x%04x, "
500 "fault address = 0x%"PRIx64"\n",
501 event_str[code-1], domain_id, device_id, *addr);
503 /* Tell the device to stop DMAing; we can't rely on the guest to
504 * control it for us. */
505 for ( bdf = 0; bdf < ivrs_bdf_entries; bdf++ )
506 if ( get_dma_requestor_id(bdf) == device_id )
507 {
508 cword = pci_conf_read16(PCI_BUS(bdf), PCI_SLOT(bdf),
509 PCI_FUNC(bdf), PCI_COMMAND);
510 pci_conf_write16(PCI_BUS(bdf), PCI_SLOT(bdf),
511 PCI_FUNC(bdf), PCI_COMMAND,
512 cword & ~PCI_COMMAND_MASTER);
513 }
514 }
515 else
516 {
517 AMD_IOMMU_DEBUG("event 0x%08x 0x%08x 0x%08x 0x%08x\n", entry[0],
518 entry[1], entry[2], entry[3]);
519 }
520 }
522 static void do_amd_iommu_irq(unsigned long data)
523 {
524 struct amd_iommu *iommu;
526 if ( !iommu_found() )
527 {
528 AMD_IOMMU_DEBUG("no device found, something must be very wrong!\n");
529 return;
530 }
532 /*
533 * No matter from where the interrupt came from, check all the
534 * IOMMUs present in the system. This allows for having just one
535 * tasklet (instead of one per each IOMMUs).
536 */
537 for_each_amd_iommu ( iommu )
538 {
539 u32 entry;
540 unsigned long flags;
541 int of;
543 spin_lock_irqsave(&iommu->lock, flags);
544 amd_iommu_read_event_log(iommu);
546 /* check event overflow */
547 entry = readl(iommu->mmio_base + IOMMU_STATUS_MMIO_OFFSET);
548 of = get_field_from_reg_u32(entry,
549 IOMMU_STATUS_EVENT_OVERFLOW_MASK,
550 IOMMU_STATUS_EVENT_OVERFLOW_SHIFT);
552 /* reset event log if event overflow */
553 if ( of )
554 amd_iommu_reset_event_log(iommu);
556 /* reset interrupt status bit */
557 entry = readl(iommu->mmio_base + IOMMU_STATUS_MMIO_OFFSET);
558 set_field_in_reg_u32(IOMMU_CONTROL_ENABLED, entry,
559 IOMMU_STATUS_EVENT_LOG_INT_MASK,
560 IOMMU_STATUS_EVENT_LOG_INT_SHIFT, &entry);
561 writel(entry, iommu->mmio_base+IOMMU_STATUS_MMIO_OFFSET);
562 spin_unlock_irqrestore(&iommu->lock, flags);
563 }
564 }
566 static void amd_iommu_page_fault(int irq, void *dev_id,
567 struct cpu_user_regs *regs)
568 {
569 u32 entry;
570 unsigned long flags;
571 struct amd_iommu *iommu = dev_id;
573 spin_lock_irqsave(&iommu->lock, flags);
575 /* Silence interrupts from both event logging */
576 entry = readl(iommu->mmio_base + IOMMU_STATUS_MMIO_OFFSET);
577 set_field_in_reg_u32(IOMMU_CONTROL_DISABLED, entry,
578 IOMMU_STATUS_EVENT_LOG_INT_MASK,
579 IOMMU_STATUS_EVENT_LOG_INT_SHIFT, &entry);
580 writel(entry, iommu->mmio_base+IOMMU_STATUS_MMIO_OFFSET);
582 spin_unlock_irqrestore(&iommu->lock, flags);
584 /* It is the tasklet that will clear the logs and re-enable interrupts */
585 tasklet_schedule(&amd_iommu_irq_tasklet);
586 }
588 static int set_iommu_interrupt_handler(struct amd_iommu *iommu)
589 {
590 int irq, ret;
592 irq = create_irq();
593 if ( irq <= 0 )
594 {
595 dprintk(XENLOG_ERR, "IOMMU: no irqs\n");
596 return 0;
597 }
599 irq_desc[irq].handler = &iommu_msi_type;
600 irq_to_iommu[irq] = iommu;
601 ret = request_irq(irq, amd_iommu_page_fault, 0,
602 "amd_iommu", iommu);
603 if ( ret )
604 {
605 irq_desc[irq].handler = &no_irq_type;
606 irq_to_iommu[irq] = NULL;
607 destroy_irq(irq);
608 AMD_IOMMU_DEBUG("can't request irq\n");
609 return 0;
610 }
612 iommu->irq = irq;
613 return irq;
614 }
616 static void enable_iommu(struct amd_iommu *iommu)
617 {
618 unsigned long flags;
620 spin_lock_irqsave(&iommu->lock, flags);
622 if ( iommu->enabled )
623 {
624 spin_unlock_irqrestore(&iommu->lock, flags);
625 return;
626 }
628 register_iommu_dev_table_in_mmio_space(iommu);
629 register_iommu_cmd_buffer_in_mmio_space(iommu);
630 register_iommu_event_log_in_mmio_space(iommu);
631 register_iommu_exclusion_range(iommu);
633 iommu_msi_set_affinity(iommu->irq, cpu_online_map);
634 amd_iommu_msi_enable(iommu, IOMMU_CONTROL_ENABLED);
636 set_iommu_command_buffer_control(iommu, IOMMU_CONTROL_ENABLED);
637 set_iommu_event_log_control(iommu, IOMMU_CONTROL_ENABLED);
638 set_iommu_translation_control(iommu, IOMMU_CONTROL_ENABLED);
640 iommu->enabled = 1;
641 spin_unlock_irqrestore(&iommu->lock, flags);
643 }
645 static void __init deallocate_iommu_table_struct(
646 struct table_struct *table)
647 {
648 int order = 0;
649 if ( table->buffer )
650 {
651 order = get_order_from_bytes(table->alloc_size);
652 __free_amd_iommu_tables(table->buffer, order);
653 table->buffer = NULL;
654 }
655 }
657 static int __init allocate_iommu_table_struct(struct table_struct *table,
658 const char *name)
659 {
660 int order = 0;
661 if ( table->buffer == NULL )
662 {
663 order = get_order_from_bytes(table->alloc_size);
664 table->buffer = __alloc_amd_iommu_tables(order);
666 if ( table->buffer == NULL )
667 {
668 AMD_IOMMU_DEBUG("Error allocating %s\n", name);
669 return -ENOMEM;
670 }
671 memset(table->buffer, 0, PAGE_SIZE * (1UL << order));
672 }
673 return 0;
674 }
676 static int __init allocate_cmd_buffer(struct amd_iommu *iommu)
677 {
678 /* allocate 'command buffer' in power of 2 increments of 4K */
679 iommu->cmd_buffer_tail = 0;
680 iommu->cmd_buffer.alloc_size = PAGE_SIZE <<
681 get_order_from_bytes(
682 PAGE_ALIGN(amd_iommu_cmd_buffer_entries *
683 IOMMU_CMD_BUFFER_ENTRY_SIZE));
684 iommu->cmd_buffer.entries = iommu->cmd_buffer.alloc_size /
685 IOMMU_CMD_BUFFER_ENTRY_SIZE;
687 return (allocate_iommu_table_struct(&iommu->cmd_buffer, "Command Buffer"));
688 }
690 static int __init allocate_event_log(struct amd_iommu *iommu)
691 {
692 /* allocate 'event log' in power of 2 increments of 4K */
693 iommu->event_log_head = 0;
694 iommu->event_log.alloc_size = PAGE_SIZE <<
695 get_order_from_bytes(
696 PAGE_ALIGN(amd_iommu_event_log_entries *
697 IOMMU_EVENT_LOG_ENTRY_SIZE));
698 iommu->event_log.entries = iommu->event_log.alloc_size /
699 IOMMU_EVENT_LOG_ENTRY_SIZE;
701 return (allocate_iommu_table_struct(&iommu->event_log, "Event Log"));
702 }
704 static int __init amd_iommu_init_one(struct amd_iommu *iommu)
705 {
706 if ( allocate_cmd_buffer(iommu) != 0 )
707 goto error_out;
709 if ( allocate_event_log(iommu) != 0 )
710 goto error_out;
712 if ( map_iommu_mmio_region(iommu) != 0 )
713 goto error_out;
715 if ( set_iommu_interrupt_handler(iommu) == 0 )
716 goto error_out;
718 /* To make sure that device_table.buffer has been successfully allocated */
719 if ( device_table.buffer == NULL )
720 goto error_out;
722 iommu->dev_table.alloc_size = device_table.alloc_size;
723 iommu->dev_table.entries = device_table.entries;
724 iommu->dev_table.buffer = device_table.buffer;
726 enable_iommu(iommu);
727 printk("AMD-Vi: IOMMU %d Enabled.\n", nr_amd_iommus );
728 nr_amd_iommus++;
730 softirq_tasklet_init(&amd_iommu_irq_tasklet, do_amd_iommu_irq, 0);
732 return 0;
734 error_out:
735 return -ENODEV;
736 }
738 static void __init amd_iommu_init_cleanup(void)
739 {
740 struct amd_iommu *iommu, *next;
741 int bdf;
743 /* free amd iommu list */
744 list_for_each_entry_safe ( iommu, next, &amd_iommu_head, list )
745 {
746 list_del(&iommu->list);
747 if ( iommu->enabled )
748 {
749 deallocate_iommu_table_struct(&iommu->cmd_buffer);
750 deallocate_iommu_table_struct(&iommu->event_log);
751 unmap_iommu_mmio_region(iommu);
752 }
753 xfree(iommu);
754 }
756 /* free interrupt remapping table */
757 for ( bdf = 0; bdf < ivrs_bdf_entries; bdf++ )
758 {
759 if ( ivrs_mappings[bdf].intremap_table )
760 amd_iommu_free_intremap_table(bdf);
761 }
763 /* free device table */
764 deallocate_iommu_table_struct(&device_table);
766 /* free ivrs_mappings[] */
767 if ( ivrs_mappings )
768 {
769 xfree(ivrs_mappings);
770 ivrs_mappings = NULL;
771 }
773 /* free irq_to_iommu[] */
774 if ( irq_to_iommu )
775 {
776 xfree(irq_to_iommu);
777 irq_to_iommu = NULL;
778 }
780 iommu_enabled = 0;
781 iommu_passthrough = 0;
782 iommu_intremap = 0;
783 }
785 static int __init init_ivrs_mapping(void)
786 {
787 int bdf;
789 BUG_ON( !ivrs_bdf_entries );
791 ivrs_mappings = xmalloc_array( struct ivrs_mappings, ivrs_bdf_entries);
792 if ( ivrs_mappings == NULL )
793 {
794 AMD_IOMMU_DEBUG("Error allocating IVRS Mappings table\n");
795 return -ENOMEM;
796 }
797 memset(ivrs_mappings, 0, ivrs_bdf_entries * sizeof(struct ivrs_mappings));
799 /* assign default values for device entries */
800 for ( bdf = 0; bdf < ivrs_bdf_entries; bdf++ )
801 {
802 ivrs_mappings[bdf].dte_requestor_id = bdf;
803 ivrs_mappings[bdf].dte_sys_mgt_enable =
804 IOMMU_DEV_TABLE_SYS_MGT_MSG_FORWARDED;
805 ivrs_mappings[bdf].dte_allow_exclusion = IOMMU_CONTROL_DISABLED;
806 ivrs_mappings[bdf].unity_map_enable = IOMMU_CONTROL_DISABLED;
807 ivrs_mappings[bdf].iommu = NULL;
809 ivrs_mappings[bdf].intremap_table = NULL;
810 ivrs_mappings[bdf].dte_lint1_pass = IOMMU_CONTROL_DISABLED;
811 ivrs_mappings[bdf].dte_lint0_pass = IOMMU_CONTROL_DISABLED;
812 ivrs_mappings[bdf].dte_nmi_pass = IOMMU_CONTROL_DISABLED;
813 ivrs_mappings[bdf].dte_ext_int_pass = IOMMU_CONTROL_DISABLED;
814 ivrs_mappings[bdf].dte_init_pass = IOMMU_CONTROL_DISABLED;
816 if ( amd_iommu_perdev_intremap )
817 spin_lock_init(&ivrs_mappings[bdf].intremap_lock);
818 }
819 return 0;
820 }
822 static int __init amd_iommu_setup_device_table(void)
823 {
824 int bdf;
825 void *intr_tb, *dte;
826 int sys_mgt, dev_ex, lint1_pass, lint0_pass,
827 nmi_pass, ext_int_pass, init_pass;
829 BUG_ON( (ivrs_bdf_entries == 0) );
831 /* allocate 'device table' on a 4K boundary */
832 device_table.alloc_size = PAGE_SIZE <<
833 get_order_from_bytes(
834 PAGE_ALIGN(ivrs_bdf_entries *
835 IOMMU_DEV_TABLE_ENTRY_SIZE));
836 device_table.entries = device_table.alloc_size /
837 IOMMU_DEV_TABLE_ENTRY_SIZE;
839 if ( allocate_iommu_table_struct(&device_table, "Device Table") != 0 )
840 return -ENOMEM;
842 /* Add device table entries */
843 for ( bdf = 0; bdf < ivrs_bdf_entries; bdf++ )
844 {
845 intr_tb = ivrs_mappings[bdf].intremap_table;
847 if ( intr_tb )
848 {
849 sys_mgt = ivrs_mappings[bdf].dte_sys_mgt_enable;
850 dev_ex = ivrs_mappings[bdf].dte_allow_exclusion;
852 /* get interrupt remapping settings */
853 lint1_pass = ivrs_mappings[bdf].dte_lint1_pass;
854 lint0_pass = ivrs_mappings[bdf].dte_lint0_pass;
855 nmi_pass = ivrs_mappings[bdf].dte_nmi_pass;
856 ext_int_pass = ivrs_mappings[bdf].dte_ext_int_pass;
857 init_pass = ivrs_mappings[bdf].dte_init_pass;
859 /* add device table entry */
860 dte = device_table.buffer + (bdf * IOMMU_DEV_TABLE_ENTRY_SIZE);
861 amd_iommu_add_dev_table_entry(
862 dte, sys_mgt, dev_ex, lint1_pass, lint0_pass,
863 nmi_pass, ext_int_pass, init_pass);
865 amd_iommu_set_intremap_table(
866 dte, (u64)virt_to_maddr(intr_tb), iommu_intremap);
868 AMD_IOMMU_DEBUG("Add device table entry: device id = 0x%04x, "
869 "interupt table = 0x%"PRIx64"\n", bdf,
870 (u64)virt_to_maddr(intr_tb));
871 }
872 }
874 return 0;
875 }
877 int __init amd_iommu_init(void)
878 {
879 struct amd_iommu *iommu;
881 BUG_ON( !iommu_found() );
883 irq_to_iommu = xmalloc_array(struct amd_iommu *, nr_irqs);
884 if ( irq_to_iommu == NULL )
885 goto error_out;
886 memset(irq_to_iommu, 0, nr_irqs * sizeof(struct iommu*));
888 ivrs_bdf_entries = amd_iommu_get_ivrs_dev_entries();
890 if ( !ivrs_bdf_entries )
891 goto error_out;
893 if ( init_ivrs_mapping() != 0 )
894 goto error_out;
896 if ( amd_iommu_update_ivrs_mapping_acpi() != 0 )
897 goto error_out;
899 /* initialize io-apic interrupt remapping entries */
900 if ( amd_iommu_setup_ioapic_remapping() != 0 )
901 goto error_out;
903 /* allocate and initialize a global device table shared by all iommus */
904 if ( amd_iommu_setup_device_table() != 0 )
905 goto error_out;
907 /* per iommu initialization */
908 for_each_amd_iommu ( iommu )
909 if ( amd_iommu_init_one(iommu) != 0 )
910 goto error_out;
912 return 0;
914 error_out:
915 amd_iommu_init_cleanup();
916 return -ENODEV;
917 }
919 static void disable_iommu(struct amd_iommu *iommu)
920 {
921 unsigned long flags;
923 spin_lock_irqsave(&iommu->lock, flags);
925 if ( !iommu->enabled )
926 {
927 spin_unlock_irqrestore(&iommu->lock, flags);
928 return;
929 }
931 amd_iommu_msi_enable(iommu, IOMMU_CONTROL_DISABLED);
932 set_iommu_command_buffer_control(iommu, IOMMU_CONTROL_DISABLED);
933 set_iommu_event_log_control(iommu, IOMMU_CONTROL_DISABLED);
934 set_iommu_translation_control(iommu, IOMMU_CONTROL_DISABLED);
936 iommu->enabled = 0;
938 spin_unlock_irqrestore(&iommu->lock, flags);
940 }
942 static void invalidate_all_domain_pages(void)
943 {
944 struct domain *d;
945 for_each_domain( d )
946 invalidate_all_iommu_pages(d);
947 }
949 static void invalidate_all_devices(void)
950 {
951 int bdf, req_id;
952 unsigned long flags;
953 struct amd_iommu *iommu;
955 for ( bdf = 0; bdf < ivrs_bdf_entries; bdf++ )
956 {
957 iommu = find_iommu_for_device(bdf);
958 req_id = ivrs_mappings[bdf].dte_requestor_id;
959 if ( iommu )
960 {
961 spin_lock_irqsave(&iommu->lock, flags);
962 invalidate_dev_table_entry(iommu, req_id);
963 invalidate_interrupt_table(iommu, req_id);
964 flush_command_buffer(iommu);
965 spin_unlock_irqrestore(&iommu->lock, flags);
966 }
967 }
968 }
970 void amd_iommu_suspend(void)
971 {
972 struct amd_iommu *iommu;
974 for_each_amd_iommu ( iommu )
975 disable_iommu(iommu);
976 }
978 void amd_iommu_resume(void)
979 {
980 struct amd_iommu *iommu;
982 for_each_amd_iommu ( iommu )
983 {
984 /*
985 * To make sure that iommus have not been touched
986 * before re-enablement
987 */
988 disable_iommu(iommu);
989 enable_iommu(iommu);
990 }
992 /* flush all cache entries after iommu re-enabled */
993 invalidate_all_devices();
994 invalidate_all_domain_pages();
995 }