debuggers.hg

view xen/drivers/passthrough/vtd/iommu.c @ 21026:d9db3684f292

VT-d: Fix ia64 build for 20974:3b475d9ed6b5

This patch fixes the following error on ia64:
iommu.c: In function 'init_vtd_hw':
iommu.c:1831: error: 'nr_ioapics' undeclared (first use in this
function)

Signed-off-by: KUWAMURA Shin'ya <kuwa@jp.fujitsu.com>
author Keir Fraser <keir.fraser@citrix.com>
date Thu Feb 25 11:54:19 2010 +0000 (2010-02-25)
parents 3b475d9ed6b5
children c9f795ff78b1
line source
1 /*
2 * Copyright (c) 2006, Intel Corporation.
3 *
4 * This program is free software; you can redistribute it and/or modify it
5 * under the terms and conditions of the GNU General Public License,
6 * version 2, as published by the Free Software Foundation.
7 *
8 * This program is distributed in the hope it will be useful, but WITHOUT
9 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
10 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
11 * more details.
12 *
13 * You should have received a copy of the GNU General Public License along with
14 * this program; if not, write to the Free Software Foundation, Inc., 59 Temple
15 * Place - Suite 330, Boston, MA 02111-1307 USA.
16 *
17 * Copyright (C) Ashok Raj <ashok.raj@intel.com>
18 * Copyright (C) Shaohua Li <shaohua.li@intel.com>
19 * Copyright (C) Allen Kay <allen.m.kay@intel.com> - adapted to xen
20 */
22 #include <xen/irq.h>
23 #include <xen/sched.h>
24 #include <xen/xmalloc.h>
25 #include <xen/domain_page.h>
26 #include <xen/iommu.h>
27 #include <asm/hvm/iommu.h>
28 #include <xen/numa.h>
29 #include <xen/time.h>
30 #include <xen/pci.h>
31 #include <xen/pci_regs.h>
32 #include <xen/keyhandler.h>
33 #include <asm/msi.h>
34 #include <asm/irq.h>
35 #include <mach_apic.h>
36 #include "iommu.h"
37 #include "dmar.h"
38 #include "extern.h"
39 #include "vtd.h"
41 #ifdef __ia64__
42 #define nr_ioapics iosapic_get_nr_iosapics()
43 #endif
45 int nr_iommus;
46 static bool_t rwbf_quirk;
48 static void setup_dom0_devices(struct domain *d);
49 static void setup_dom0_rmrr(struct domain *d);
51 static int domain_iommu_domid(struct domain *d,
52 struct iommu *iommu)
53 {
54 unsigned long nr_dom, i;
56 nr_dom = cap_ndoms(iommu->cap);
57 i = find_first_bit(iommu->domid_bitmap, nr_dom);
58 while ( i < nr_dom )
59 {
60 if ( iommu->domid_map[i] == d->domain_id )
61 return i;
63 i = find_next_bit(iommu->domid_bitmap, nr_dom, i+1);
64 }
66 gdprintk(XENLOG_ERR VTDPREFIX,
67 "Cannot get valid iommu domid: domid=%d iommu->index=%d\n",
68 d->domain_id, iommu->index);
69 return -1;
70 }
72 #define DID_FIELD_WIDTH 16
73 #define DID_HIGH_OFFSET 8
74 static int context_set_domain_id(struct context_entry *context,
75 struct domain *d,
76 struct iommu *iommu)
77 {
78 unsigned long nr_dom, i;
79 int found = 0;
81 ASSERT(spin_is_locked(&iommu->lock));
83 nr_dom = cap_ndoms(iommu->cap);
84 i = find_first_bit(iommu->domid_bitmap, nr_dom);
85 while ( i < nr_dom )
86 {
87 if ( iommu->domid_map[i] == d->domain_id )
88 {
89 found = 1;
90 break;
91 }
92 i = find_next_bit(iommu->domid_bitmap, nr_dom, i+1);
93 }
95 if ( found == 0 )
96 {
97 i = find_first_zero_bit(iommu->domid_bitmap, nr_dom);
98 if ( i >= nr_dom )
99 {
100 gdprintk(XENLOG_ERR VTDPREFIX, "IOMMU: no free domain ids\n");
101 return -EFAULT;
102 }
103 iommu->domid_map[i] = d->domain_id;
104 }
106 set_bit(i, iommu->domid_bitmap);
107 context->hi |= (i & ((1 << DID_FIELD_WIDTH) - 1)) << DID_HIGH_OFFSET;
108 return 0;
109 }
111 static struct intel_iommu *alloc_intel_iommu(void)
112 {
113 struct intel_iommu *intel;
115 intel = xmalloc(struct intel_iommu);
116 if ( intel == NULL )
117 return NULL;
118 memset(intel, 0, sizeof(struct intel_iommu));
120 spin_lock_init(&intel->qi_ctrl.qinval_lock);
121 spin_lock_init(&intel->ir_ctrl.iremap_lock);
123 return intel;
124 }
126 static void free_intel_iommu(struct intel_iommu *intel)
127 {
128 xfree(intel);
129 }
131 struct qi_ctrl *iommu_qi_ctrl(struct iommu *iommu)
132 {
133 return iommu ? &iommu->intel->qi_ctrl : NULL;
134 }
136 struct ir_ctrl *iommu_ir_ctrl(struct iommu *iommu)
137 {
138 return iommu ? &iommu->intel->ir_ctrl : NULL;
139 }
141 struct iommu_flush *iommu_get_flush(struct iommu *iommu)
142 {
143 return iommu ? &iommu->intel->flush : NULL;
144 }
146 static unsigned int clflush_size;
147 static int iommus_incoherent;
148 static void __iommu_flush_cache(void *addr, unsigned int size)
149 {
150 int i;
152 if ( !iommus_incoherent )
153 return;
155 for ( i = 0; i < size; i += clflush_size )
156 cacheline_flush((char *)addr + i);
157 }
159 void iommu_flush_cache_entry(void *addr, unsigned int size)
160 {
161 __iommu_flush_cache(addr, size);
162 }
164 void iommu_flush_cache_page(void *addr, unsigned long npages)
165 {
166 __iommu_flush_cache(addr, PAGE_SIZE * npages);
167 }
169 /* Allocate page table, return its machine address */
170 u64 alloc_pgtable_maddr(struct acpi_drhd_unit *drhd, unsigned long npages)
171 {
172 struct acpi_rhsa_unit *rhsa;
173 struct page_info *pg, *cur_pg;
174 u64 *vaddr;
175 int node = -1, i;
177 rhsa = drhd_to_rhsa(drhd);
178 if ( rhsa )
179 node = pxm_to_node(rhsa->proximity_domain);
181 pg = alloc_domheap_pages(NULL, get_order_from_pages(npages),
182 (node == -1 ) ? 0 : MEMF_node(node));
183 if ( !pg )
184 return 0;
186 cur_pg = pg;
187 for ( i = 0; i < npages; i++ )
188 {
189 vaddr = __map_domain_page(cur_pg);
190 memset(vaddr, 0, PAGE_SIZE);
192 iommu_flush_cache_page(vaddr, 1);
193 unmap_domain_page(vaddr);
194 cur_pg++;
195 }
197 return page_to_maddr(pg);
198 }
200 void free_pgtable_maddr(u64 maddr)
201 {
202 if ( maddr != 0 )
203 free_domheap_page(maddr_to_page(maddr));
204 }
206 /* context entry handling */
207 static u64 bus_to_context_maddr(struct iommu *iommu, u8 bus)
208 {
209 struct acpi_drhd_unit *drhd;
210 struct root_entry *root, *root_entries;
211 u64 maddr;
213 ASSERT(spin_is_locked(&iommu->lock));
214 root_entries = (struct root_entry *)map_vtd_domain_page(iommu->root_maddr);
215 root = &root_entries[bus];
216 if ( !root_present(*root) )
217 {
218 drhd = iommu_to_drhd(iommu);
219 maddr = alloc_pgtable_maddr(drhd, 1);
220 if ( maddr == 0 )
221 {
222 unmap_vtd_domain_page(root_entries);
223 return 0;
224 }
225 set_root_value(*root, maddr);
226 set_root_present(*root);
227 iommu_flush_cache_entry(root, sizeof(struct root_entry));
228 }
229 maddr = (u64) get_context_addr(*root);
230 unmap_vtd_domain_page(root_entries);
231 return maddr;
232 }
234 static u64 addr_to_dma_page_maddr(struct domain *domain, u64 addr, int alloc)
235 {
236 struct acpi_drhd_unit *drhd;
237 struct pci_dev *pdev;
238 struct hvm_iommu *hd = domain_hvm_iommu(domain);
239 int addr_width = agaw_to_width(hd->agaw);
240 struct dma_pte *parent, *pte = NULL;
241 int level = agaw_to_level(hd->agaw);
242 int offset;
243 u64 pte_maddr = 0, maddr;
244 u64 *vaddr = NULL;
246 addr &= (((u64)1) << addr_width) - 1;
247 ASSERT(spin_is_locked(&hd->mapping_lock));
248 if ( hd->pgd_maddr == 0 )
249 {
250 /*
251 * just get any passthrough device in the domainr - assume user
252 * assigns only devices from same node to a given guest.
253 */
254 pdev = pci_get_pdev_by_domain(domain, -1, -1);
255 drhd = acpi_find_matched_drhd_unit(pdev);
256 if ( !alloc || ((hd->pgd_maddr = alloc_pgtable_maddr(drhd, 1)) == 0) )
257 goto out;
258 }
260 parent = (struct dma_pte *)map_vtd_domain_page(hd->pgd_maddr);
261 while ( level > 1 )
262 {
263 offset = address_level_offset(addr, level);
264 pte = &parent[offset];
266 if ( dma_pte_addr(*pte) == 0 )
267 {
268 if ( !alloc )
269 break;
271 pdev = pci_get_pdev_by_domain(domain, -1, -1);
272 drhd = acpi_find_matched_drhd_unit(pdev);
273 maddr = alloc_pgtable_maddr(drhd, 1);
274 if ( !maddr )
275 break;
277 dma_set_pte_addr(*pte, maddr);
278 vaddr = map_vtd_domain_page(maddr);
280 /*
281 * high level table always sets r/w, last level
282 * page table control read/write
283 */
284 dma_set_pte_readable(*pte);
285 dma_set_pte_writable(*pte);
286 iommu_flush_cache_entry(pte, sizeof(struct dma_pte));
287 }
288 else
289 {
290 vaddr = map_vtd_domain_page(pte->val);
291 }
293 if ( level == 2 )
294 {
295 pte_maddr = pte->val & PAGE_MASK_4K;
296 unmap_vtd_domain_page(vaddr);
297 break;
298 }
300 unmap_vtd_domain_page(parent);
301 parent = (struct dma_pte *)vaddr;
302 vaddr = NULL;
303 level--;
304 }
306 unmap_vtd_domain_page(parent);
307 out:
308 return pte_maddr;
309 }
311 static void iommu_flush_write_buffer(struct iommu *iommu)
312 {
313 u32 val;
314 unsigned long flags;
316 if ( !rwbf_quirk && !cap_rwbf(iommu->cap) )
317 return;
319 spin_lock_irqsave(&iommu->register_lock, flags);
320 val = dmar_readl(iommu->reg, DMAR_GSTS_REG);
321 dmar_writel(iommu->reg, DMAR_GCMD_REG, val | DMA_GCMD_WBF);
323 /* Make sure hardware complete it */
324 IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG, dmar_readl,
325 !(val & DMA_GSTS_WBFS), val);
327 spin_unlock_irqrestore(&iommu->register_lock, flags);
328 }
330 /* return value determine if we need a write buffer flush */
331 static int flush_context_reg(
332 void *_iommu,
333 u16 did, u16 source_id, u8 function_mask, u64 type,
334 int flush_non_present_entry)
335 {
336 struct iommu *iommu = (struct iommu *) _iommu;
337 u64 val = 0;
338 unsigned long flags;
340 /*
341 * In the non-present entry flush case, if hardware doesn't cache
342 * non-present entry we do nothing and if hardware cache non-present
343 * entry, we flush entries of domain 0 (the domain id is used to cache
344 * any non-present entries)
345 */
346 if ( flush_non_present_entry )
347 {
348 if ( !cap_caching_mode(iommu->cap) )
349 return 1;
350 else
351 did = 0;
352 }
354 /* use register invalidation */
355 switch ( type )
356 {
357 case DMA_CCMD_GLOBAL_INVL:
358 val = DMA_CCMD_GLOBAL_INVL;
359 break;
360 case DMA_CCMD_DOMAIN_INVL:
361 val = DMA_CCMD_DOMAIN_INVL|DMA_CCMD_DID(did);
362 break;
363 case DMA_CCMD_DEVICE_INVL:
364 val = DMA_CCMD_DEVICE_INVL|DMA_CCMD_DID(did)
365 |DMA_CCMD_SID(source_id)|DMA_CCMD_FM(function_mask);
366 break;
367 default:
368 BUG();
369 }
370 val |= DMA_CCMD_ICC;
372 spin_lock_irqsave(&iommu->register_lock, flags);
373 dmar_writeq(iommu->reg, DMAR_CCMD_REG, val);
375 /* Make sure hardware complete it */
376 IOMMU_WAIT_OP(iommu, DMAR_CCMD_REG, dmar_readq,
377 !(val & DMA_CCMD_ICC), val);
379 spin_unlock_irqrestore(&iommu->register_lock, flags);
380 /* flush context entry will implicitly flush write buffer */
381 return 0;
382 }
384 static int inline iommu_flush_context_global(
385 struct iommu *iommu, int flush_non_present_entry)
386 {
387 struct iommu_flush *flush = iommu_get_flush(iommu);
388 return flush->context(iommu, 0, 0, 0, DMA_CCMD_GLOBAL_INVL,
389 flush_non_present_entry);
390 }
392 static int inline iommu_flush_context_domain(
393 struct iommu *iommu, u16 did, int flush_non_present_entry)
394 {
395 struct iommu_flush *flush = iommu_get_flush(iommu);
396 return flush->context(iommu, did, 0, 0, DMA_CCMD_DOMAIN_INVL,
397 flush_non_present_entry);
398 }
400 static int inline iommu_flush_context_device(
401 struct iommu *iommu, u16 did, u16 source_id,
402 u8 function_mask, int flush_non_present_entry)
403 {
404 struct iommu_flush *flush = iommu_get_flush(iommu);
405 return flush->context(iommu, did, source_id, function_mask,
406 DMA_CCMD_DEVICE_INVL,
407 flush_non_present_entry);
408 }
410 /* return value determine if we need a write buffer flush */
411 static int flush_iotlb_reg(void *_iommu, u16 did,
412 u64 addr, unsigned int size_order, u64 type,
413 int flush_non_present_entry, int flush_dev_iotlb)
414 {
415 struct iommu *iommu = (struct iommu *) _iommu;
416 int tlb_offset = ecap_iotlb_offset(iommu->ecap);
417 u64 val = 0, val_iva = 0;
418 unsigned long flags;
420 /*
421 * In the non-present entry flush case, if hardware doesn't cache
422 * non-present entry we do nothing and if hardware cache non-present
423 * entry, we flush entries of domain 0 (the domain id is used to cache
424 * any non-present entries)
425 */
426 if ( flush_non_present_entry )
427 {
428 if ( !cap_caching_mode(iommu->cap) )
429 return 1;
430 else
431 did = 0;
432 }
434 /* use register invalidation */
435 switch ( type )
436 {
437 case DMA_TLB_GLOBAL_FLUSH:
438 /* global flush doesn't need set IVA_REG */
439 val = DMA_TLB_GLOBAL_FLUSH|DMA_TLB_IVT;
440 break;
441 case DMA_TLB_DSI_FLUSH:
442 val = DMA_TLB_DSI_FLUSH|DMA_TLB_IVT|DMA_TLB_DID(did);
443 break;
444 case DMA_TLB_PSI_FLUSH:
445 val = DMA_TLB_PSI_FLUSH|DMA_TLB_IVT|DMA_TLB_DID(did);
446 /* Note: always flush non-leaf currently */
447 val_iva = size_order | addr;
448 break;
449 default:
450 BUG();
451 }
452 /* Note: set drain read/write */
453 if ( cap_read_drain(iommu->cap) )
454 val |= DMA_TLB_READ_DRAIN;
455 if ( cap_write_drain(iommu->cap) )
456 val |= DMA_TLB_WRITE_DRAIN;
458 spin_lock_irqsave(&iommu->register_lock, flags);
459 /* Note: Only uses first TLB reg currently */
460 if ( val_iva )
461 dmar_writeq(iommu->reg, tlb_offset, val_iva);
462 dmar_writeq(iommu->reg, tlb_offset + 8, val);
464 /* Make sure hardware complete it */
465 IOMMU_WAIT_OP(iommu, (tlb_offset + 8), dmar_readq,
466 !(val & DMA_TLB_IVT), val);
467 spin_unlock_irqrestore(&iommu->register_lock, flags);
469 /* check IOTLB invalidation granularity */
470 if ( DMA_TLB_IAIG(val) == 0 )
471 dprintk(XENLOG_ERR VTDPREFIX, "IOMMU: flush IOTLB failed\n");
473 /* flush iotlb entry will implicitly flush write buffer */
474 return 0;
475 }
477 static int inline iommu_flush_iotlb_global(struct iommu *iommu,
478 int flush_non_present_entry, int flush_dev_iotlb)
479 {
480 struct iommu_flush *flush = iommu_get_flush(iommu);
481 return flush->iotlb(iommu, 0, 0, 0, DMA_TLB_GLOBAL_FLUSH,
482 flush_non_present_entry, flush_dev_iotlb);
483 }
485 static int inline iommu_flush_iotlb_dsi(struct iommu *iommu, u16 did,
486 int flush_non_present_entry, int flush_dev_iotlb)
487 {
488 struct iommu_flush *flush = iommu_get_flush(iommu);
489 return flush->iotlb(iommu, did, 0, 0, DMA_TLB_DSI_FLUSH,
490 flush_non_present_entry, flush_dev_iotlb);
491 }
493 static int inline get_alignment(u64 base, unsigned int size)
494 {
495 int t = 0;
496 u64 end;
498 end = base + size - 1;
499 while ( base != end )
500 {
501 t++;
502 base >>= 1;
503 end >>= 1;
504 }
505 return t;
506 }
508 static int inline iommu_flush_iotlb_psi(
509 struct iommu *iommu, u16 did, u64 addr, unsigned int pages,
510 int flush_non_present_entry, int flush_dev_iotlb)
511 {
512 unsigned int align;
513 struct iommu_flush *flush = iommu_get_flush(iommu);
515 ASSERT(!(addr & (~PAGE_MASK_4K)));
516 ASSERT(pages > 0);
518 /* Fallback to domain selective flush if no PSI support */
519 if ( !cap_pgsel_inv(iommu->cap) )
520 return iommu_flush_iotlb_dsi(iommu, did, flush_non_present_entry, flush_dev_iotlb);
522 /*
523 * PSI requires page size is 2 ^ x, and the base address is naturally
524 * aligned to the size
525 */
526 align = get_alignment(addr >> PAGE_SHIFT_4K, pages);
527 /* Fallback to domain selective flush if size is too big */
528 if ( align > cap_max_amask_val(iommu->cap) )
529 return iommu_flush_iotlb_dsi(iommu, did, flush_non_present_entry, flush_dev_iotlb);
531 addr >>= PAGE_SHIFT_4K + align;
532 addr <<= PAGE_SHIFT_4K + align;
534 return flush->iotlb(iommu, did, addr, align, DMA_TLB_PSI_FLUSH,
535 flush_non_present_entry, flush_dev_iotlb);
536 }
538 static void iommu_flush_all(void)
539 {
540 struct acpi_drhd_unit *drhd;
541 struct iommu *iommu;
542 int flush_dev_iotlb;
544 flush_all_cache();
545 for_each_drhd_unit ( drhd )
546 {
547 iommu = drhd->iommu;
548 iommu_flush_context_global(iommu, 0);
549 flush_dev_iotlb = find_ats_dev_drhd(iommu) ? 1 : 0;
550 iommu_flush_iotlb_global(iommu, 0, flush_dev_iotlb);
551 }
552 }
554 /* clear one page's page table */
555 static void dma_pte_clear_one(struct domain *domain, u64 addr)
556 {
557 struct hvm_iommu *hd = domain_hvm_iommu(domain);
558 struct acpi_drhd_unit *drhd;
559 struct iommu *iommu;
560 struct dma_pte *page = NULL, *pte = NULL;
561 u64 pg_maddr;
562 int flush_dev_iotlb;
563 int iommu_domid;
564 struct list_head *rmrr_list, *tmp;
565 struct mapped_rmrr *mrmrr;
567 spin_lock(&hd->mapping_lock);
568 /* get last level pte */
569 pg_maddr = addr_to_dma_page_maddr(domain, addr, 0);
570 if ( pg_maddr == 0 )
571 {
572 spin_unlock(&hd->mapping_lock);
573 return;
574 }
576 page = (struct dma_pte *)map_vtd_domain_page(pg_maddr);
577 pte = page + address_level_offset(addr, 1);
579 if ( !dma_pte_present(*pte) )
580 {
581 spin_unlock(&hd->mapping_lock);
582 unmap_vtd_domain_page(page);
583 return;
584 }
586 dma_clear_pte(*pte);
587 spin_unlock(&hd->mapping_lock);
588 iommu_flush_cache_entry(pte, sizeof(struct dma_pte));
590 /* No need pcidevs_lock here since do that on assign/deassign device*/
591 for_each_drhd_unit ( drhd )
592 {
593 iommu = drhd->iommu;
594 if ( test_bit(iommu->index, &hd->iommu_bitmap) )
595 {
596 flush_dev_iotlb = find_ats_dev_drhd(iommu) ? 1 : 0;
597 iommu_domid= domain_iommu_domid(domain, iommu);
598 if ( iommu_domid == -1 )
599 continue;
600 if ( iommu_flush_iotlb_psi(iommu, iommu_domid,
601 addr, 1, 0, flush_dev_iotlb) )
602 iommu_flush_write_buffer(iommu);
603 }
604 }
606 unmap_vtd_domain_page(page);
608 /* if the cleared address is between mapped RMRR region,
609 * remove the mapped RMRR
610 */
611 spin_lock(&pcidevs_lock);
612 list_for_each_safe ( rmrr_list, tmp, &hd->mapped_rmrrs )
613 {
614 mrmrr = list_entry(rmrr_list, struct mapped_rmrr, list);
615 if ( addr >= mrmrr->base && addr <= mrmrr->end )
616 {
617 list_del(&mrmrr->list);
618 xfree(mrmrr);
619 break;
620 }
621 }
622 spin_unlock(&pcidevs_lock);
623 }
625 static void iommu_free_pagetable(u64 pt_maddr, int level)
626 {
627 int i;
628 struct dma_pte *pt_vaddr, *pte;
629 int next_level = level - 1;
631 if ( pt_maddr == 0 )
632 return;
634 pt_vaddr = (struct dma_pte *)map_vtd_domain_page(pt_maddr);
636 for ( i = 0; i < PTE_NUM; i++ )
637 {
638 pte = &pt_vaddr[i];
639 if ( !dma_pte_present(*pte) )
640 continue;
642 if ( next_level >= 1 )
643 iommu_free_pagetable(dma_pte_addr(*pte), next_level);
645 dma_clear_pte(*pte);
646 iommu_flush_cache_entry(pte, sizeof(struct dma_pte));
647 }
649 unmap_vtd_domain_page(pt_vaddr);
650 free_pgtable_maddr(pt_maddr);
651 }
653 static int iommu_set_root_entry(struct iommu *iommu)
654 {
655 struct acpi_drhd_unit *drhd;
656 u32 sts;
657 unsigned long flags;
659 spin_lock(&iommu->lock);
661 if ( iommu->root_maddr == 0 )
662 {
663 drhd = iommu_to_drhd(iommu);
664 iommu->root_maddr = alloc_pgtable_maddr(drhd, 1);
665 }
667 if ( iommu->root_maddr == 0 )
668 {
669 spin_unlock(&iommu->lock);
670 return -ENOMEM;
671 }
673 spin_unlock(&iommu->lock);
674 spin_lock_irqsave(&iommu->register_lock, flags);
675 dmar_writeq(iommu->reg, DMAR_RTADDR_REG, iommu->root_maddr);
677 sts = dmar_readl(iommu->reg, DMAR_GSTS_REG);
678 dmar_writel(iommu->reg, DMAR_GCMD_REG, sts | DMA_GCMD_SRTP);
680 /* Make sure hardware complete it */
681 IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG, dmar_readl,
682 (sts & DMA_GSTS_RTPS), sts);
683 spin_unlock_irqrestore(&iommu->register_lock, flags);
685 return 0;
686 }
688 static void iommu_enable_translation(struct iommu *iommu)
689 {
690 u32 sts;
691 unsigned long flags;
693 dprintk(XENLOG_INFO VTDPREFIX,
694 "iommu_enable_translation: iommu->reg = %p\n", iommu->reg);
695 spin_lock_irqsave(&iommu->register_lock, flags);
696 sts = dmar_readl(iommu->reg, DMAR_GSTS_REG);
697 dmar_writel(iommu->reg, DMAR_GCMD_REG, sts | DMA_GCMD_TE);
699 /* Make sure hardware complete it */
700 IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG, dmar_readl,
701 (sts & DMA_GSTS_TES), sts);
702 spin_unlock_irqrestore(&iommu->register_lock, flags);
704 /* Disable PMRs when VT-d engine takes effect per spec definition */
705 disable_pmr(iommu);
706 }
708 static void iommu_disable_translation(struct iommu *iommu)
709 {
710 u32 sts;
711 unsigned long flags;
713 spin_lock_irqsave(&iommu->register_lock, flags);
714 sts = dmar_readl(iommu->reg, DMAR_GSTS_REG);
715 dmar_writel(iommu->reg, DMAR_GCMD_REG, sts & (~DMA_GCMD_TE));
717 /* Make sure hardware complete it */
718 IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG, dmar_readl,
719 !(sts & DMA_GSTS_TES), sts);
720 spin_unlock_irqrestore(&iommu->register_lock, flags);
721 }
723 enum faulttype {
724 DMA_REMAP,
725 INTR_REMAP,
726 UNKNOWN,
727 };
729 static const char *dma_remap_fault_reasons[] =
730 {
731 "Software",
732 "Present bit in root entry is clear",
733 "Present bit in context entry is clear",
734 "Invalid context entry",
735 "Access beyond MGAW",
736 "PTE Write access is not set",
737 "PTE Read access is not set",
738 "Next page table ptr is invalid",
739 "Root table address invalid",
740 "Context table ptr is invalid",
741 "non-zero reserved fields in RTP",
742 "non-zero reserved fields in CTP",
743 "non-zero reserved fields in PTE",
744 "Blocked a DMA translation request",
745 };
747 static const char *intr_remap_fault_reasons[] =
748 {
749 "Detected reserved fields in the decoded interrupt-remapped request",
750 "Interrupt index exceeded the interrupt-remapping table size",
751 "Present field in the IRTE entry is clear",
752 "Error accessing interrupt-remapping table pointed by IRTA_REG",
753 "Detected reserved fields in the IRTE entry",
754 "Blocked a compatibility format interrupt request",
755 "Blocked an interrupt request due to source-id verification failure",
756 };
758 static const char *iommu_get_fault_reason(u8 fault_reason, int *fault_type)
759 {
760 if ( fault_reason >= 0x20 && ( fault_reason < 0x20 +
761 ARRAY_SIZE(intr_remap_fault_reasons)) )
762 {
763 *fault_type = INTR_REMAP;
764 return intr_remap_fault_reasons[fault_reason - 0x20];
765 }
766 else if ( fault_reason < ARRAY_SIZE(dma_remap_fault_reasons) )
767 {
768 *fault_type = DMA_REMAP;
769 return dma_remap_fault_reasons[fault_reason];
770 }
771 else
772 {
773 *fault_type = UNKNOWN;
774 return "Unknown";
775 }
776 }
778 static struct iommu **irq_to_iommu;
779 static int iommu_page_fault_do_one(struct iommu *iommu, int type,
780 u8 fault_reason, u16 source_id, u64 addr)
781 {
782 const char *reason;
783 int fault_type;
784 reason = iommu_get_fault_reason(fault_reason, &fault_type);
786 if ( fault_type == DMA_REMAP )
787 {
788 dprintk(XENLOG_WARNING VTDPREFIX,
789 "DMAR:[%s] Request device [%02x:%02x.%d] "
790 "fault addr %"PRIx64", iommu reg = %p\n"
791 "DMAR:[fault reason %02xh] %s\n",
792 (type ? "DMA Read" : "DMA Write"),
793 (source_id >> 8), PCI_SLOT(source_id & 0xFF),
794 PCI_FUNC(source_id & 0xFF), addr, iommu->reg,
795 fault_reason, reason);
796 #ifndef __i386__ /* map_domain_page() cannot be used in this context */
797 print_vtd_entries(iommu, (source_id >> 8),
798 (source_id & 0xff), (addr >> PAGE_SHIFT));
799 #endif
800 }
801 else
802 dprintk(XENLOG_WARNING VTDPREFIX,
803 "INTR-REMAP: Request device [%02x:%02x.%d] "
804 "fault index %"PRIx64", iommu reg = %p\n"
805 "INTR-REMAP:[fault reason %02xh] %s\n",
806 (source_id >> 8), PCI_SLOT(source_id & 0xFF),
807 PCI_FUNC(source_id & 0xFF), addr >> 48, iommu->reg,
808 fault_reason, reason);
809 return 0;
811 }
813 static void iommu_fault_status(u32 fault_status)
814 {
815 if ( fault_status & DMA_FSTS_PFO )
816 dprintk(XENLOG_ERR VTDPREFIX,
817 "iommu_fault_status: Fault Overflow\n");
818 if ( fault_status & DMA_FSTS_PPF )
819 dprintk(XENLOG_ERR VTDPREFIX,
820 "iommu_fault_status: Primary Pending Fault\n");
821 if ( fault_status & DMA_FSTS_AFO )
822 dprintk(XENLOG_ERR VTDPREFIX,
823 "iommu_fault_status: Advanced Fault Overflow\n");
824 if ( fault_status & DMA_FSTS_APF )
825 dprintk(XENLOG_ERR VTDPREFIX,
826 "iommu_fault_status: Advanced Pending Fault\n");
827 if ( fault_status & DMA_FSTS_IQE )
828 dprintk(XENLOG_ERR VTDPREFIX,
829 "iommu_fault_status: Invalidation Queue Error\n");
830 if ( fault_status & DMA_FSTS_ICE )
831 dprintk(XENLOG_ERR VTDPREFIX,
832 "iommu_fault_status: Invalidation Completion Error\n");
833 if ( fault_status & DMA_FSTS_ITE )
834 dprintk(XENLOG_ERR VTDPREFIX,
835 "iommu_fault_status: Invalidation Time-out Error\n");
836 }
838 #define PRIMARY_FAULT_REG_LEN (16)
839 static void iommu_page_fault(int irq, void *dev_id,
840 struct cpu_user_regs *regs)
841 {
842 struct iommu *iommu = dev_id;
843 int reg, fault_index;
844 u32 fault_status;
845 unsigned long flags;
847 fault_status = dmar_readl(iommu->reg, DMAR_FSTS_REG);
849 iommu_fault_status(fault_status);
851 /* FIXME: ignore advanced fault log */
852 if ( !(fault_status & DMA_FSTS_PPF) )
853 goto clear_overflow;
855 fault_index = dma_fsts_fault_record_index(fault_status);
856 reg = cap_fault_reg_offset(iommu->cap);
857 while (1)
858 {
859 u8 fault_reason;
860 u16 source_id;
861 u32 data;
862 u64 guest_addr;
863 int type;
865 /* highest 32 bits */
866 spin_lock_irqsave(&iommu->register_lock, flags);
867 data = dmar_readl(iommu->reg, reg +
868 fault_index * PRIMARY_FAULT_REG_LEN + 12);
869 if ( !(data & DMA_FRCD_F) )
870 {
871 spin_unlock_irqrestore(&iommu->register_lock, flags);
872 break;
873 }
875 fault_reason = dma_frcd_fault_reason(data);
876 type = dma_frcd_type(data);
878 data = dmar_readl(iommu->reg, reg +
879 fault_index * PRIMARY_FAULT_REG_LEN + 8);
880 source_id = dma_frcd_source_id(data);
882 guest_addr = dmar_readq(iommu->reg, reg +
883 fault_index * PRIMARY_FAULT_REG_LEN);
884 guest_addr = dma_frcd_page_addr(guest_addr);
885 /* clear the fault */
886 dmar_writel(iommu->reg, reg +
887 fault_index * PRIMARY_FAULT_REG_LEN + 12, DMA_FRCD_F);
888 spin_unlock_irqrestore(&iommu->register_lock, flags);
890 iommu_page_fault_do_one(iommu, type, fault_reason,
891 source_id, guest_addr);
893 fault_index++;
894 if ( fault_index > cap_num_fault_regs(iommu->cap) )
895 fault_index = 0;
896 }
897 clear_overflow:
898 /* clear primary fault overflow */
899 fault_status = readl(iommu->reg + DMAR_FSTS_REG);
900 if ( fault_status & DMA_FSTS_PFO )
901 {
902 spin_lock_irqsave(&iommu->register_lock, flags);
903 dmar_writel(iommu->reg, DMAR_FSTS_REG, DMA_FSTS_PFO);
904 spin_unlock_irqrestore(&iommu->register_lock, flags);
905 }
906 }
908 static void dma_msi_unmask(unsigned int irq)
909 {
910 struct iommu *iommu = irq_to_iommu[irq];
911 unsigned long flags;
913 /* unmask it */
914 spin_lock_irqsave(&iommu->register_lock, flags);
915 dmar_writel(iommu->reg, DMAR_FECTL_REG, 0);
916 spin_unlock_irqrestore(&iommu->register_lock, flags);
917 }
919 static void dma_msi_mask(unsigned int irq)
920 {
921 unsigned long flags;
922 struct iommu *iommu = irq_to_iommu[irq];
923 struct irq_desc *desc = irq_to_desc(irq);
925 irq_complete_move(&desc);
927 /* mask it */
928 spin_lock_irqsave(&iommu->register_lock, flags);
929 dmar_writel(iommu->reg, DMAR_FECTL_REG, DMA_FECTL_IM);
930 spin_unlock_irqrestore(&iommu->register_lock, flags);
931 }
933 static unsigned int dma_msi_startup(unsigned int irq)
934 {
935 dma_msi_unmask(irq);
936 return 0;
937 }
939 static void dma_msi_end(unsigned int irq)
940 {
941 dma_msi_unmask(irq);
942 ack_APIC_irq();
943 }
945 static void dma_msi_set_affinity(unsigned int irq, cpumask_t mask)
946 {
947 struct msi_msg msg;
948 unsigned int dest;
949 unsigned long flags;
951 struct iommu *iommu = irq_to_iommu[irq];
952 struct irq_desc *desc = irq_to_desc(irq);
953 struct irq_cfg *cfg = desc->chip_data;
955 #ifdef CONFIG_X86
956 dest = set_desc_affinity(desc, mask);
957 if (dest == BAD_APICID){
958 dprintk(XENLOG_ERR VTDPREFIX, "Set iommu interrupt affinity error!\n");
959 return;
960 }
962 memset(&msg, 0, sizeof(msg));
963 msg.data = MSI_DATA_VECTOR(cfg->vector) & 0xff;
964 msg.data |= 1 << 14;
965 msg.data |= (INT_DELIVERY_MODE != dest_LowestPrio) ?
966 MSI_DATA_DELIVERY_FIXED:
967 MSI_DATA_DELIVERY_LOWPRI;
969 /* Follow MSI setting */
970 if (x2apic_enabled)
971 msg.address_hi = dest & 0xFFFFFF00;
972 msg.address_lo = (MSI_ADDRESS_HEADER << (MSI_ADDRESS_HEADER_SHIFT + 8));
973 msg.address_lo |= INT_DEST_MODE ? MSI_ADDR_DESTMODE_LOGIC:
974 MSI_ADDR_DESTMODE_PHYS;
975 msg.address_lo |= (INT_DELIVERY_MODE != dest_LowestPrio) ?
976 MSI_ADDR_REDIRECTION_CPU:
977 MSI_ADDR_REDIRECTION_LOWPRI;
978 msg.address_lo |= MSI_ADDR_DEST_ID(dest & 0xff);
979 #else
980 memset(&msg, 0, sizeof(msg));
981 msg.data = cfg->vector & 0xff;
982 msg.data |= 1 << 14;
983 msg.address_lo = (MSI_ADDRESS_HEADER << (MSI_ADDRESS_HEADER_SHIFT + 8));
984 msg.address_lo |= MSI_PHYSICAL_MODE << 2;
985 msg.address_lo |= MSI_REDIRECTION_HINT_MODE << 3;
986 dest = cpu_physical_id(first_cpu(mask));
987 msg.address_lo |= dest << MSI_TARGET_CPU_SHIFT;
988 #endif
990 spin_lock_irqsave(&iommu->register_lock, flags);
991 dmar_writel(iommu->reg, DMAR_FEDATA_REG, msg.data);
992 dmar_writel(iommu->reg, DMAR_FEADDR_REG, msg.address_lo);
993 dmar_writel(iommu->reg, DMAR_FEUADDR_REG, msg.address_hi);
994 spin_unlock_irqrestore(&iommu->register_lock, flags);
995 }
997 static hw_irq_controller dma_msi_type = {
998 .typename = "DMA_MSI",
999 .startup = dma_msi_startup,
1000 .shutdown = dma_msi_mask,
1001 .enable = dma_msi_unmask,
1002 .disable = dma_msi_mask,
1003 .ack = dma_msi_mask,
1004 .end = dma_msi_end,
1005 .set_affinity = dma_msi_set_affinity,
1006 };
1008 static int iommu_set_interrupt(struct iommu *iommu)
1010 int irq, ret;
1012 irq = create_irq();
1013 if ( irq <= 0 )
1015 dprintk(XENLOG_ERR VTDPREFIX, "IOMMU: no irq available!\n");
1016 return -EINVAL;
1019 irq_desc[irq].handler = &dma_msi_type;
1020 irq_to_iommu[irq] = iommu;
1021 #ifdef CONFIG_X86
1022 ret = request_irq(irq, iommu_page_fault, 0, "dmar", iommu);
1023 #else
1024 ret = request_irq_vector(irq, iommu_page_fault, 0, "dmar", iommu);
1025 #endif
1026 if ( ret )
1028 irq_desc[irq].handler = &no_irq_type;
1029 irq_to_iommu[irq] = NULL;
1030 destroy_irq(irq);
1031 dprintk(XENLOG_ERR VTDPREFIX, "IOMMU: can't request irq\n");
1032 return ret;
1035 return irq;
1038 static int iommu_alloc(struct acpi_drhd_unit *drhd)
1040 struct iommu *iommu;
1041 unsigned long sagaw, nr_dom;
1042 int agaw;
1044 if ( nr_iommus > MAX_IOMMUS )
1046 dprintk(XENLOG_ERR VTDPREFIX,
1047 "IOMMU: nr_iommus %d > MAX_IOMMUS\n", nr_iommus);
1048 return -ENOMEM;
1051 iommu = xmalloc(struct iommu);
1052 if ( iommu == NULL )
1053 return -ENOMEM;
1054 memset(iommu, 0, sizeof(struct iommu));
1056 iommu->irq = -1; /* No irq assigned yet. */
1058 iommu->intel = alloc_intel_iommu();
1059 if ( iommu->intel == NULL )
1061 xfree(iommu);
1062 return -ENOMEM;
1065 iommu->reg = map_to_nocache_virt(nr_iommus, drhd->address);
1066 iommu->index = nr_iommus++;
1068 iommu->cap = dmar_readq(iommu->reg, DMAR_CAP_REG);
1069 iommu->ecap = dmar_readq(iommu->reg, DMAR_ECAP_REG);
1071 dprintk(XENLOG_INFO VTDPREFIX,
1072 "drhd->address = %"PRIx64"\n", drhd->address);
1073 dprintk(XENLOG_INFO VTDPREFIX, "iommu->reg = %p\n", iommu->reg);
1075 /* Calculate number of pagetable levels: between 2 and 4. */
1076 sagaw = cap_sagaw(iommu->cap);
1077 for ( agaw = level_to_agaw(4); agaw >= 0; agaw-- )
1078 if ( test_bit(agaw, &sagaw) )
1079 break;
1080 if ( agaw < 0 )
1082 dprintk(XENLOG_ERR VTDPREFIX,
1083 "IOMMU: unsupported sagaw %lx\n", sagaw);
1084 xfree(iommu);
1085 return -ENODEV;
1087 iommu->nr_pt_levels = agaw_to_level(agaw);
1089 if ( !ecap_coherent(iommu->ecap) )
1090 iommus_incoherent = 1;
1092 /* allocate domain id bitmap */
1093 nr_dom = cap_ndoms(iommu->cap);
1094 iommu->domid_bitmap = xmalloc_array(unsigned long, BITS_TO_LONGS(nr_dom));
1095 if ( !iommu->domid_bitmap )
1096 return -ENOMEM ;
1097 memset(iommu->domid_bitmap, 0, nr_dom / 8);
1099 /*
1100 * if Caching mode is set, then invalid translations are tagged with
1101 * domain id 0, Hence reserve bit 0 for it
1102 */
1103 if ( cap_caching_mode(iommu->cap) )
1104 set_bit(0, iommu->domid_bitmap);
1106 iommu->domid_map = xmalloc_array(u16, nr_dom);
1107 if ( !iommu->domid_map )
1108 return -ENOMEM ;
1109 memset(iommu->domid_map, 0, nr_dom * sizeof(*iommu->domid_map));
1111 spin_lock_init(&iommu->lock);
1112 spin_lock_init(&iommu->register_lock);
1114 drhd->iommu = iommu;
1115 return 0;
1118 static void iommu_free(struct acpi_drhd_unit *drhd)
1120 struct iommu *iommu = drhd->iommu;
1122 if ( iommu == NULL )
1123 return;
1125 if ( iommu->root_maddr != 0 )
1127 free_pgtable_maddr(iommu->root_maddr);
1128 iommu->root_maddr = 0;
1131 if ( iommu->reg )
1132 iounmap(iommu->reg);
1134 xfree(iommu->domid_bitmap);
1135 xfree(iommu->domid_map);
1137 free_intel_iommu(iommu->intel);
1138 destroy_irq(iommu->irq);
1139 xfree(iommu);
1141 drhd->iommu = NULL;
1144 #define guestwidth_to_adjustwidth(gaw) ({ \
1145 int agaw, r = (gaw - 12) % 9; \
1146 agaw = (r == 0) ? gaw : (gaw + 9 - r); \
1147 if ( agaw > 64 ) \
1148 agaw = 64; \
1149 agaw; })
1151 static int intel_iommu_domain_init(struct domain *d)
1153 struct hvm_iommu *hd = domain_hvm_iommu(d);
1154 struct iommu *iommu;
1155 struct acpi_drhd_unit *drhd;
1157 hd->agaw = width_to_agaw(DEFAULT_DOMAIN_ADDRESS_WIDTH);
1159 if ( d->domain_id == 0 )
1161 /* Set up 1:1 page table for dom0 */
1162 iommu_set_dom0_mapping(d);
1164 setup_dom0_devices(d);
1165 setup_dom0_rmrr(d);
1167 iommu_flush_all();
1169 for_each_drhd_unit ( drhd )
1171 iommu = drhd->iommu;
1172 iommu_enable_translation(iommu);
1176 return 0;
1179 static int domain_context_mapping_one(
1180 struct domain *domain,
1181 struct iommu *iommu,
1182 u8 bus, u8 devfn)
1184 struct hvm_iommu *hd = domain_hvm_iommu(domain);
1185 struct context_entry *context, *context_entries;
1186 u64 maddr, pgd_maddr;
1187 struct pci_dev *pdev = NULL;
1188 int agaw;
1190 ASSERT(spin_is_locked(&pcidevs_lock));
1191 spin_lock(&iommu->lock);
1192 maddr = bus_to_context_maddr(iommu, bus);
1193 context_entries = (struct context_entry *)map_vtd_domain_page(maddr);
1194 context = &context_entries[devfn];
1196 if ( context_present(*context) )
1198 int res = 0;
1200 pdev = pci_get_pdev(bus, devfn);
1201 if (!pdev)
1202 res = -ENODEV;
1203 else if (pdev->domain != domain)
1204 res = -EINVAL;
1205 unmap_vtd_domain_page(context_entries);
1206 spin_unlock(&iommu->lock);
1207 return res;
1210 if ( iommu_passthrough && (domain->domain_id == 0) )
1212 context_set_translation_type(*context, CONTEXT_TT_PASS_THRU);
1213 agaw = level_to_agaw(iommu->nr_pt_levels);
1215 else
1217 spin_lock(&hd->mapping_lock);
1219 /* Ensure we have pagetables allocated down to leaf PTE. */
1220 if ( hd->pgd_maddr == 0 )
1222 addr_to_dma_page_maddr(domain, 0, 1);
1223 if ( hd->pgd_maddr == 0 )
1225 nomem:
1226 spin_unlock(&hd->mapping_lock);
1227 spin_unlock(&iommu->lock);
1228 unmap_vtd_domain_page(context_entries);
1229 return -ENOMEM;
1233 /* Skip top levels of page tables for 2- and 3-level DRHDs. */
1234 pgd_maddr = hd->pgd_maddr;
1235 for ( agaw = level_to_agaw(4);
1236 agaw != level_to_agaw(iommu->nr_pt_levels);
1237 agaw-- )
1239 struct dma_pte *p = map_vtd_domain_page(pgd_maddr);
1240 pgd_maddr = dma_pte_addr(*p);
1241 unmap_vtd_domain_page(p);
1242 if ( pgd_maddr == 0 )
1243 goto nomem;
1246 context_set_address_root(*context, pgd_maddr);
1247 if ( ats_enabled && ecap_dev_iotlb(iommu->ecap) )
1248 context_set_translation_type(*context, CONTEXT_TT_DEV_IOTLB);
1249 else
1250 context_set_translation_type(*context, CONTEXT_TT_MULTI_LEVEL);
1252 spin_unlock(&hd->mapping_lock);
1255 if ( context_set_domain_id(context, domain, iommu) )
1257 spin_unlock(&iommu->lock);
1258 return -EFAULT;
1261 context_set_address_width(*context, agaw);
1262 context_set_fault_enable(*context);
1263 context_set_present(*context);
1264 iommu_flush_cache_entry(context, sizeof(struct context_entry));
1265 spin_unlock(&iommu->lock);
1267 /* Context entry was previously non-present (with domid 0). */
1268 if ( iommu_flush_context_device(iommu, 0, (((u16)bus) << 8) | devfn,
1269 DMA_CCMD_MASK_NOBIT, 1) )
1270 iommu_flush_write_buffer(iommu);
1271 else
1273 int flush_dev_iotlb = find_ats_dev_drhd(iommu) ? 1 : 0;
1274 iommu_flush_iotlb_dsi(iommu, 0, 1, flush_dev_iotlb);
1277 set_bit(iommu->index, &hd->iommu_bitmap);
1279 unmap_vtd_domain_page(context_entries);
1281 return 0;
1284 static int domain_context_mapping(struct domain *domain, u8 bus, u8 devfn)
1286 struct acpi_drhd_unit *drhd;
1287 int ret = 0;
1288 u32 type;
1289 u8 secbus;
1290 struct pci_dev *pdev = pci_get_pdev(bus, devfn);
1292 drhd = acpi_find_matched_drhd_unit(pdev);
1293 if ( !drhd )
1294 return -ENODEV;
1296 ASSERT(spin_is_locked(&pcidevs_lock));
1298 type = pdev_type(bus, devfn);
1299 switch ( type )
1301 case DEV_TYPE_PCIe_BRIDGE:
1302 case DEV_TYPE_PCIe2PCI_BRIDGE:
1303 case DEV_TYPE_LEGACY_PCI_BRIDGE:
1304 break;
1306 case DEV_TYPE_PCIe_ENDPOINT:
1307 gdprintk(XENLOG_INFO VTDPREFIX,
1308 "domain_context_mapping:PCIe: bdf = %x:%x.%x\n",
1309 bus, PCI_SLOT(devfn), PCI_FUNC(devfn));
1310 ret = domain_context_mapping_one(domain, drhd->iommu, bus, devfn);
1311 break;
1313 case DEV_TYPE_PCI:
1314 gdprintk(XENLOG_INFO VTDPREFIX,
1315 "domain_context_mapping:PCI: bdf = %x:%x.%x\n",
1316 bus, PCI_SLOT(devfn), PCI_FUNC(devfn));
1318 ret = domain_context_mapping_one(domain, drhd->iommu, bus, devfn);
1319 if ( ret )
1320 break;
1322 if ( find_upstream_bridge(&bus, &devfn, &secbus) < 1 )
1323 break;
1325 /* PCIe to PCI/PCIx bridge */
1326 if ( pdev_type(bus, devfn) == DEV_TYPE_PCIe2PCI_BRIDGE )
1328 ret = domain_context_mapping_one(domain, drhd->iommu, bus, devfn);
1329 if ( ret )
1330 return ret;
1332 /*
1333 * Devices behind PCIe-to-PCI/PCIx bridge may generate
1334 * different requester-id. It may originate from devfn=0
1335 * on the secondary bus behind the bridge. Map that id
1336 * as well.
1337 */
1338 ret = domain_context_mapping_one(domain, drhd->iommu, secbus, 0);
1340 else /* Legacy PCI bridge */
1341 ret = domain_context_mapping_one(domain, drhd->iommu, bus, devfn);
1343 break;
1345 default:
1346 gdprintk(XENLOG_ERR VTDPREFIX,
1347 "domain_context_mapping:unknown type : bdf = %x:%x.%x\n",
1348 bus, PCI_SLOT(devfn), PCI_FUNC(devfn));
1349 ret = -EINVAL;
1350 break;
1353 return ret;
1356 static int domain_context_unmap_one(
1357 struct domain *domain,
1358 struct iommu *iommu,
1359 u8 bus, u8 devfn)
1361 struct context_entry *context, *context_entries;
1362 u64 maddr;
1363 int iommu_domid;
1365 ASSERT(spin_is_locked(&pcidevs_lock));
1366 spin_lock(&iommu->lock);
1368 maddr = bus_to_context_maddr(iommu, bus);
1369 context_entries = (struct context_entry *)map_vtd_domain_page(maddr);
1370 context = &context_entries[devfn];
1372 if ( !context_present(*context) )
1374 spin_unlock(&iommu->lock);
1375 unmap_vtd_domain_page(context_entries);
1376 return 0;
1379 context_clear_present(*context);
1380 context_clear_entry(*context);
1381 iommu_flush_cache_entry(context, sizeof(struct context_entry));
1383 iommu_domid= domain_iommu_domid(domain, iommu);
1384 if ( iommu_domid == -1 )
1386 spin_unlock(&iommu->lock);
1387 unmap_vtd_domain_page(context_entries);
1388 return -EINVAL;
1391 if ( iommu_flush_context_device(iommu, iommu_domid,
1392 (((u16)bus) << 8) | devfn,
1393 DMA_CCMD_MASK_NOBIT, 0) )
1394 iommu_flush_write_buffer(iommu);
1395 else
1397 int flush_dev_iotlb = find_ats_dev_drhd(iommu) ? 1 : 0;
1398 iommu_flush_iotlb_dsi(iommu, iommu_domid, 0, flush_dev_iotlb);
1401 spin_unlock(&iommu->lock);
1402 unmap_vtd_domain_page(context_entries);
1404 return 0;
1407 static int domain_context_unmap(struct domain *domain, u8 bus, u8 devfn)
1409 struct acpi_drhd_unit *drhd;
1410 struct iommu *iommu;
1411 int ret = 0;
1412 u32 type;
1413 u8 tmp_bus, tmp_devfn, secbus;
1414 struct pci_dev *pdev = pci_get_pdev(bus, devfn);
1415 int found = 0;
1417 BUG_ON(!pdev);
1419 drhd = acpi_find_matched_drhd_unit(pdev);
1420 if ( !drhd )
1421 return -ENODEV;
1422 iommu = drhd->iommu;
1424 type = pdev_type(bus, devfn);
1425 switch ( type )
1427 case DEV_TYPE_PCIe_BRIDGE:
1428 case DEV_TYPE_PCIe2PCI_BRIDGE:
1429 case DEV_TYPE_LEGACY_PCI_BRIDGE:
1430 goto out;
1432 case DEV_TYPE_PCIe_ENDPOINT:
1433 gdprintk(XENLOG_INFO VTDPREFIX,
1434 "domain_context_unmap:PCIe: bdf = %x:%x.%x\n",
1435 bus, PCI_SLOT(devfn), PCI_FUNC(devfn));
1436 ret = domain_context_unmap_one(domain, iommu, bus, devfn);
1437 break;
1439 case DEV_TYPE_PCI:
1440 gdprintk(XENLOG_INFO VTDPREFIX,
1441 "domain_context_unmap:PCI: bdf = %x:%x.%x\n",
1442 bus, PCI_SLOT(devfn), PCI_FUNC(devfn));
1443 ret = domain_context_unmap_one(domain, iommu, bus, devfn);
1444 if ( ret )
1445 break;
1447 tmp_bus = bus;
1448 tmp_devfn = devfn;
1449 if ( find_upstream_bridge(&tmp_bus, &tmp_devfn, &secbus) < 1 )
1450 break;
1452 /* PCIe to PCI/PCIx bridge */
1453 if ( pdev_type(tmp_bus, tmp_devfn) == DEV_TYPE_PCIe2PCI_BRIDGE )
1455 ret = domain_context_unmap_one(domain, iommu, tmp_bus, tmp_devfn);
1456 if ( ret )
1457 return ret;
1459 ret = domain_context_unmap_one(domain, iommu, secbus, 0);
1461 else /* Legacy PCI bridge */
1462 ret = domain_context_unmap_one(domain, iommu, tmp_bus, tmp_devfn);
1464 break;
1466 default:
1467 gdprintk(XENLOG_ERR VTDPREFIX,
1468 "domain_context_unmap:unknown type: bdf = %x:%x.%x\n",
1469 bus, PCI_SLOT(devfn), PCI_FUNC(devfn));
1470 ret = -EINVAL;
1471 goto out;
1474 /*
1475 * if no other devices under the same iommu owned by this domain,
1476 * clear iommu in iommu_bitmap and clear domain_id in domid_bitmp
1477 */
1478 for_each_pdev ( domain, pdev )
1480 if ( pdev->bus == bus && pdev->devfn == devfn )
1481 continue;
1483 drhd = acpi_find_matched_drhd_unit(pdev);
1484 if ( drhd && drhd->iommu == iommu )
1486 found = 1;
1487 break;
1491 if ( found == 0 )
1493 struct hvm_iommu *hd = domain_hvm_iommu(domain);
1494 int iommu_domid;
1496 clear_bit(iommu->index, &hd->iommu_bitmap);
1498 iommu_domid = domain_iommu_domid(domain, iommu);
1499 if ( iommu_domid == -1 )
1501 ret = -EINVAL;
1502 goto out;
1505 clear_bit(iommu_domid, iommu->domid_bitmap);
1506 iommu->domid_map[iommu_domid] = 0;
1509 out:
1510 return ret;
1513 static int reassign_device_ownership(
1514 struct domain *source,
1515 struct domain *target,
1516 u8 bus, u8 devfn)
1518 struct pci_dev *pdev;
1519 int ret;
1521 ASSERT(spin_is_locked(&pcidevs_lock));
1522 pdev = pci_get_pdev_by_domain(source, bus, devfn);
1524 if (!pdev)
1525 return -ENODEV;
1527 ret = domain_context_unmap(source, bus, devfn);
1528 if ( ret )
1529 return ret;
1531 ret = domain_context_mapping(target, bus, devfn);
1532 if ( ret )
1533 return ret;
1535 list_move(&pdev->domain_list, &target->arch.pdev_list);
1536 pdev->domain = target;
1538 return ret;
1541 void iommu_domain_teardown(struct domain *d)
1543 struct hvm_iommu *hd = domain_hvm_iommu(d);
1545 if ( list_empty(&acpi_drhd_units) )
1546 return;
1548 spin_lock(&hd->mapping_lock);
1549 iommu_free_pagetable(hd->pgd_maddr, agaw_to_level(hd->agaw));
1550 hd->pgd_maddr = 0;
1551 spin_unlock(&hd->mapping_lock);
1554 static int intel_iommu_map_page(
1555 struct domain *d, unsigned long gfn, unsigned long mfn)
1557 struct hvm_iommu *hd = domain_hvm_iommu(d);
1558 struct acpi_drhd_unit *drhd;
1559 struct iommu *iommu;
1560 struct dma_pte *page = NULL, *pte = NULL;
1561 u64 pg_maddr;
1562 int pte_present;
1563 int flush_dev_iotlb;
1564 int iommu_domid;
1566 /* do nothing if dom0 and iommu supports pass thru */
1567 if ( iommu_passthrough && (d->domain_id == 0) )
1568 return 0;
1570 spin_lock(&hd->mapping_lock);
1572 pg_maddr = addr_to_dma_page_maddr(d, (paddr_t)gfn << PAGE_SHIFT_4K, 1);
1573 if ( pg_maddr == 0 )
1575 spin_unlock(&hd->mapping_lock);
1576 return -ENOMEM;
1578 page = (struct dma_pte *)map_vtd_domain_page(pg_maddr);
1579 pte = page + (gfn & LEVEL_MASK);
1580 pte_present = dma_pte_present(*pte);
1581 dma_set_pte_addr(*pte, (paddr_t)mfn << PAGE_SHIFT_4K);
1582 dma_set_pte_prot(*pte, DMA_PTE_READ | DMA_PTE_WRITE);
1584 /* Set the SNP on leaf page table if Snoop Control available */
1585 if ( iommu_snoop )
1586 dma_set_pte_snp(*pte);
1588 iommu_flush_cache_entry(pte, sizeof(struct dma_pte));
1589 spin_unlock(&hd->mapping_lock);
1590 unmap_vtd_domain_page(page);
1592 /*
1593 * No need pcideves_lock here because we have flush
1594 * when assign/deassign device
1595 */
1596 for_each_drhd_unit ( drhd )
1598 iommu = drhd->iommu;
1600 if ( !test_bit(iommu->index, &hd->iommu_bitmap) )
1601 continue;
1603 flush_dev_iotlb = find_ats_dev_drhd(iommu) ? 1 : 0;
1604 iommu_domid= domain_iommu_domid(d, iommu);
1605 if ( iommu_domid == -1 )
1606 continue;
1607 if ( iommu_flush_iotlb_psi(iommu, iommu_domid,
1608 (paddr_t)gfn << PAGE_SHIFT_4K, 1,
1609 !pte_present, flush_dev_iotlb) )
1610 iommu_flush_write_buffer(iommu);
1613 return 0;
1616 static int intel_iommu_unmap_page(struct domain *d, unsigned long gfn)
1618 /* Do nothing if dom0 and iommu supports pass thru. */
1619 if ( iommu_passthrough && (d->domain_id == 0) )
1620 return 0;
1622 dma_pte_clear_one(d, (paddr_t)gfn << PAGE_SHIFT_4K);
1624 return 0;
1627 static int domain_rmrr_mapped(struct domain *d,
1628 struct acpi_rmrr_unit *rmrr)
1630 struct hvm_iommu *hd = domain_hvm_iommu(d);
1631 struct mapped_rmrr *mrmrr;
1633 list_for_each_entry( mrmrr, &hd->mapped_rmrrs, list )
1635 if ( mrmrr->base == rmrr->base_address &&
1636 mrmrr->end == rmrr->end_address )
1637 return 1;
1640 return 0;
1643 static int rmrr_identity_mapping(struct domain *d,
1644 struct acpi_rmrr_unit *rmrr)
1646 u64 base, end;
1647 unsigned long base_pfn, end_pfn;
1648 struct mapped_rmrr *mrmrr;
1649 struct hvm_iommu *hd = domain_hvm_iommu(d);
1651 ASSERT(spin_is_locked(&pcidevs_lock));
1652 ASSERT(rmrr->base_address < rmrr->end_address);
1654 if ( domain_rmrr_mapped(d, rmrr) )
1655 return 0;
1657 base = rmrr->base_address & PAGE_MASK_4K;
1658 base_pfn = base >> PAGE_SHIFT_4K;
1659 end = PAGE_ALIGN_4K(rmrr->end_address);
1660 end_pfn = end >> PAGE_SHIFT_4K;
1662 while ( base_pfn < end_pfn )
1664 if ( intel_iommu_map_page(d, base_pfn, base_pfn) )
1665 return -1;
1666 base_pfn++;
1669 mrmrr = xmalloc(struct mapped_rmrr);
1670 if ( !mrmrr )
1671 return -ENOMEM;
1672 mrmrr->base = rmrr->base_address;
1673 mrmrr->end = rmrr->end_address;
1674 list_add_tail(&mrmrr->list, &hd->mapped_rmrrs);
1676 return 0;
1679 static int intel_iommu_add_device(struct pci_dev *pdev)
1681 struct acpi_rmrr_unit *rmrr;
1682 u16 bdf;
1683 int ret, i;
1685 ASSERT(spin_is_locked(&pcidevs_lock));
1687 if ( !pdev->domain )
1688 return -EINVAL;
1690 ret = domain_context_mapping(pdev->domain, pdev->bus, pdev->devfn);
1691 if ( ret )
1693 gdprintk(XENLOG_ERR VTDPREFIX,
1694 "intel_iommu_add_device: context mapping failed\n");
1695 return ret;
1698 for_each_rmrr_device ( rmrr, bdf, i )
1700 if ( PCI_BUS(bdf) == pdev->bus && PCI_DEVFN2(bdf) == pdev->devfn )
1702 ret = rmrr_identity_mapping(pdev->domain, rmrr);
1703 if ( ret )
1704 gdprintk(XENLOG_ERR VTDPREFIX,
1705 "intel_iommu_add_device: RMRR mapping failed\n");
1709 return ret;
1712 static int intel_iommu_remove_device(struct pci_dev *pdev)
1714 struct acpi_rmrr_unit *rmrr;
1715 u16 bdf;
1716 int i;
1718 if ( !pdev->domain )
1719 return -EINVAL;
1721 /* If the device belongs to dom0, and it has RMRR, don't remove it
1722 * from dom0, because BIOS may use RMRR at booting time.
1723 */
1724 if ( pdev->domain->domain_id == 0 )
1726 for_each_rmrr_device ( rmrr, bdf, i )
1728 if ( PCI_BUS(bdf) == pdev->bus &&
1729 PCI_DEVFN2(bdf) == pdev->devfn )
1730 return 0;
1734 return domain_context_unmap(pdev->domain, pdev->bus, pdev->devfn);
1737 static void setup_dom0_devices(struct domain *d)
1739 struct pci_dev *pdev;
1740 int bus, devfn;
1742 spin_lock(&pcidevs_lock);
1743 for ( bus = 0; bus < 256; bus++ )
1745 for ( devfn = 0; devfn < 256; devfn++ )
1747 pdev = pci_get_pdev(bus, devfn);
1748 if ( !pdev )
1749 continue;
1751 pdev->domain = d;
1752 list_add(&pdev->domain_list, &d->arch.pdev_list);
1753 domain_context_mapping(d, pdev->bus, pdev->devfn);
1754 pci_enable_acs(pdev);
1755 if ( ats_device(0, pdev->bus, pdev->devfn) )
1756 enable_ats_device(0, pdev->bus, pdev->devfn);
1759 spin_unlock(&pcidevs_lock);
1762 void clear_fault_bits(struct iommu *iommu)
1764 u64 val;
1765 unsigned long flags;
1767 spin_lock_irqsave(&iommu->register_lock, flags);
1768 val = dmar_readq(
1769 iommu->reg,
1770 cap_fault_reg_offset(dmar_readq(iommu->reg,DMAR_CAP_REG))+0x8);
1771 dmar_writeq(
1772 iommu->reg,
1773 cap_fault_reg_offset(dmar_readq(iommu->reg,DMAR_CAP_REG))+8,
1774 val);
1775 dmar_writel(iommu->reg, DMAR_FSTS_REG, DMA_FSTS_FAULTS);
1776 spin_unlock_irqrestore(&iommu->register_lock, flags);
1779 static int init_vtd_hw(void)
1781 struct acpi_drhd_unit *drhd;
1782 struct iommu *iommu;
1783 struct iommu_flush *flush = NULL;
1784 int irq;
1785 int ret;
1786 unsigned long flags;
1787 struct irq_cfg *cfg;
1789 for_each_drhd_unit ( drhd )
1791 iommu = drhd->iommu;
1792 if ( iommu->irq < 0 )
1794 irq = iommu_set_interrupt(iommu);
1795 if ( irq < 0 )
1797 dprintk(XENLOG_ERR VTDPREFIX, "IOMMU: interrupt setup failed\n");
1798 return irq;
1800 iommu->irq = irq;
1803 cfg = irq_cfg(iommu->irq);
1804 dma_msi_set_affinity(iommu->irq, cfg->domain);
1806 clear_fault_bits(iommu);
1808 spin_lock_irqsave(&iommu->register_lock, flags);
1809 dmar_writel(iommu->reg, DMAR_FECTL_REG, 0);
1810 spin_unlock_irqrestore(&iommu->register_lock, flags);
1812 /* initialize flush functions */
1813 flush = iommu_get_flush(iommu);
1814 flush->context = flush_context_reg;
1815 flush->iotlb = flush_iotlb_reg;
1818 if ( iommu_qinval )
1820 for_each_drhd_unit ( drhd )
1822 iommu = drhd->iommu;
1823 if ( enable_qinval(iommu) != 0 )
1825 dprintk(XENLOG_INFO VTDPREFIX,
1826 "Failed to enable Queued Invalidation!\n");
1827 break;
1832 if ( iommu_intremap )
1834 int apic;
1835 for ( apic = 0; apic < nr_ioapics; apic++ )
1837 if ( ioapic_to_iommu(IO_APIC_ID(apic)) == NULL )
1839 iommu_intremap = 0;
1840 dprintk(XENLOG_ERR VTDPREFIX,
1841 "ioapic_to_iommu: ioapic 0x%x (id: 0x%x) is NULL! "
1842 "Will not try to enable Interrupt Remapping.\n",
1843 apic, IO_APIC_ID(apic));
1844 break;
1849 if ( iommu_intremap )
1851 for_each_drhd_unit ( drhd )
1853 iommu = drhd->iommu;
1854 if ( enable_intremap(iommu) != 0 )
1856 dprintk(XENLOG_INFO VTDPREFIX,
1857 "Failed to enable Interrupt Remapping!\n");
1858 break;
1863 for_each_drhd_unit ( drhd )
1865 iommu = drhd->iommu;
1866 ret = iommu_set_root_entry(iommu);
1867 if ( ret )
1869 dprintk(XENLOG_ERR VTDPREFIX, "IOMMU: set root entry failed\n");
1870 return -EIO;
1874 /*
1875 * After set root entry, must globally invalidate context cache, and
1876 * then globally invalidate IOTLB
1877 */
1878 iommu_flush_all();
1880 return 0;
1883 static void setup_dom0_rmrr(struct domain *d)
1885 struct acpi_rmrr_unit *rmrr;
1886 u16 bdf;
1887 int ret, i;
1889 spin_lock(&pcidevs_lock);
1890 for_each_rmrr_device ( rmrr, bdf, i )
1892 ret = rmrr_identity_mapping(d, rmrr);
1893 if ( ret )
1894 dprintk(XENLOG_ERR VTDPREFIX,
1895 "IOMMU: mapping reserved region failed\n");
1897 spin_unlock(&pcidevs_lock);
1900 static void platform_quirks(void)
1902 u32 id;
1904 /* Mobile 4 Series Chipset neglects to set RWBF capability. */
1905 id = pci_conf_read32(0, 0, 0, 0);
1906 if ( id == 0x2a408086 )
1908 dprintk(XENLOG_INFO VTDPREFIX, "DMAR: Forcing write-buffer flush\n");
1909 rwbf_quirk = 1;
1913 int intel_vtd_setup(void)
1915 struct acpi_drhd_unit *drhd;
1916 struct iommu *iommu;
1918 if ( list_empty(&acpi_drhd_units) )
1919 return -ENODEV;
1921 platform_quirks();
1923 clflush_size = get_cache_line_size();
1925 irq_to_iommu = xmalloc_array(struct iommu*, nr_irqs);
1926 BUG_ON(!irq_to_iommu);
1927 memset(irq_to_iommu, 0, nr_irqs * sizeof(struct iommu*));
1929 if(!irq_to_iommu)
1930 return -ENOMEM;
1932 /* We enable the following features only if they are supported by all VT-d
1933 * engines: Snoop Control, DMA passthrough, Queued Invalidation and
1934 * Interrupt Remapping.
1935 */
1936 for_each_drhd_unit ( drhd )
1938 if ( iommu_alloc(drhd) != 0 )
1939 goto error;
1941 iommu = drhd->iommu;
1943 if ( iommu_snoop && !ecap_snp_ctl(iommu->ecap) )
1944 iommu_snoop = 0;
1946 if ( iommu_passthrough && !ecap_pass_thru(iommu->ecap) )
1947 iommu_passthrough = 0;
1949 if ( iommu_qinval && !ecap_queued_inval(iommu->ecap) )
1950 iommu_qinval = 0;
1952 if ( iommu_intremap && !ecap_intr_remap(iommu->ecap) )
1953 iommu_intremap = 0;
1956 if ( !iommu_qinval && iommu_intremap )
1958 iommu_intremap = 0;
1959 dprintk(XENLOG_WARNING VTDPREFIX, "Interrupt Remapping disabled "
1960 "since Queued Invalidation isn't supported or enabled.\n");
1963 #define P(p,s) printk("Intel VT-d %s %ssupported.\n", s, (p)? "" : "not ")
1964 P(iommu_snoop, "Snoop Control");
1965 P(iommu_passthrough, "DMA Passthrough");
1966 P(iommu_qinval, "Queued Invalidation");
1967 P(iommu_intremap, "Interrupt Remapping");
1968 #undef P
1970 scan_pci_devices();
1972 if ( init_vtd_hw() )
1973 goto error;
1975 register_keyhandler('V', &dump_iommu_info_keyhandler);
1977 return 0;
1979 error:
1980 for_each_drhd_unit ( drhd )
1981 iommu_free(drhd);
1982 iommu_enabled = 0;
1983 iommu_snoop = 0;
1984 iommu_passthrough = 0;
1985 iommu_qinval = 0;
1986 iommu_intremap = 0;
1987 return -ENOMEM;
1990 /*
1991 * If the device isn't owned by dom0, it means it already
1992 * has been assigned to other domain, or it's not exist.
1993 */
1994 int device_assigned(u8 bus, u8 devfn)
1996 struct pci_dev *pdev;
1998 spin_lock(&pcidevs_lock);
1999 pdev = pci_get_pdev_by_domain(dom0, bus, devfn);
2000 if (!pdev)
2002 spin_unlock(&pcidevs_lock);
2003 return -1;
2006 spin_unlock(&pcidevs_lock);
2007 return 0;
2010 static int intel_iommu_assign_device(struct domain *d, u8 bus, u8 devfn)
2012 struct acpi_rmrr_unit *rmrr;
2013 int ret = 0, i;
2014 struct pci_dev *pdev;
2015 u16 bdf;
2017 if ( list_empty(&acpi_drhd_units) )
2018 return -ENODEV;
2020 ASSERT(spin_is_locked(&pcidevs_lock));
2021 pdev = pci_get_pdev(bus, devfn);
2022 if (!pdev)
2023 return -ENODEV;
2025 if (pdev->domain != dom0)
2027 gdprintk(XENLOG_ERR VTDPREFIX,
2028 "IOMMU: assign a assigned device\n");
2029 return -EBUSY;
2032 ret = reassign_device_ownership(dom0, d, bus, devfn);
2033 if ( ret )
2034 goto done;
2036 /* FIXME: Because USB RMRR conflicts with guest bios region,
2037 * ignore USB RMRR temporarily.
2038 */
2039 if ( is_usb_device(bus, devfn) )
2041 ret = 0;
2042 goto done;
2045 /* Setup rmrr identity mapping */
2046 for_each_rmrr_device( rmrr, bdf, i )
2048 if ( PCI_BUS(bdf) == bus && PCI_DEVFN2(bdf) == devfn )
2050 ret = rmrr_identity_mapping(d, rmrr);
2051 if ( ret )
2053 gdprintk(XENLOG_ERR VTDPREFIX,
2054 "IOMMU: mapping reserved region failed\n");
2055 goto done;
2060 done:
2061 return ret;
2064 static int intel_iommu_group_id(u8 bus, u8 devfn)
2066 u8 secbus;
2067 if ( find_upstream_bridge(&bus, &devfn, &secbus) < 0 )
2068 return -1;
2069 else
2070 return PCI_BDF2(bus, devfn);
2073 static u32 iommu_state[MAX_IOMMUS][MAX_IOMMU_REGS];
2074 static void vtd_suspend(void)
2076 struct acpi_drhd_unit *drhd;
2077 struct iommu *iommu;
2078 u32 i;
2080 if ( !iommu_enabled )
2081 return;
2083 iommu_flush_all();
2085 for_each_drhd_unit ( drhd )
2087 iommu = drhd->iommu;
2088 i = iommu->index;
2090 iommu_state[i][DMAR_FECTL_REG] =
2091 (u32) dmar_readl(iommu->reg, DMAR_FECTL_REG);
2092 iommu_state[i][DMAR_FEDATA_REG] =
2093 (u32) dmar_readl(iommu->reg, DMAR_FEDATA_REG);
2094 iommu_state[i][DMAR_FEADDR_REG] =
2095 (u32) dmar_readl(iommu->reg, DMAR_FEADDR_REG);
2096 iommu_state[i][DMAR_FEUADDR_REG] =
2097 (u32) dmar_readl(iommu->reg, DMAR_FEUADDR_REG);
2099 /* don't disable VT-d engine when force_iommu is set. */
2100 if ( force_iommu )
2101 continue;
2103 iommu_disable_translation(iommu);
2105 if ( iommu_intremap )
2106 disable_intremap(iommu);
2108 if ( iommu_qinval )
2109 disable_qinval(iommu);
2113 static void vtd_resume(void)
2115 struct acpi_drhd_unit *drhd;
2116 struct iommu *iommu;
2117 u32 i;
2118 unsigned long flags;
2120 if ( !iommu_enabled )
2121 return;
2123 if ( init_vtd_hw() != 0 && force_iommu )
2124 panic("IOMMU setup failed, crash Xen for security purpose!\n");
2126 for_each_drhd_unit ( drhd )
2128 iommu = drhd->iommu;
2129 i = iommu->index;
2131 spin_lock_irqsave(&iommu->register_lock, flags);
2132 dmar_writel(iommu->reg, DMAR_FECTL_REG,
2133 (u32) iommu_state[i][DMAR_FECTL_REG]);
2134 dmar_writel(iommu->reg, DMAR_FEDATA_REG,
2135 (u32) iommu_state[i][DMAR_FEDATA_REG]);
2136 dmar_writel(iommu->reg, DMAR_FEADDR_REG,
2137 (u32) iommu_state[i][DMAR_FEADDR_REG]);
2138 dmar_writel(iommu->reg, DMAR_FEUADDR_REG,
2139 (u32) iommu_state[i][DMAR_FEUADDR_REG]);
2140 spin_unlock_irqrestore(&iommu->register_lock, flags);
2142 iommu_enable_translation(iommu);
2146 const struct iommu_ops intel_iommu_ops = {
2147 .init = intel_iommu_domain_init,
2148 .add_device = intel_iommu_add_device,
2149 .remove_device = intel_iommu_remove_device,
2150 .assign_device = intel_iommu_assign_device,
2151 .teardown = iommu_domain_teardown,
2152 .map_page = intel_iommu_map_page,
2153 .unmap_page = intel_iommu_unmap_page,
2154 .reassign_device = reassign_device_ownership,
2155 .get_device_group_id = intel_iommu_group_id,
2156 .update_ire_from_apic = io_apic_write_remap_rte,
2157 .update_ire_from_msi = msi_msg_write_remap_rte,
2158 .read_apic_from_ire = io_apic_read_remap_rte,
2159 .read_msi_from_ire = msi_msg_read_remap_rte,
2160 .suspend = vtd_suspend,
2161 .resume = vtd_resume,
2162 };
2164 /*
2165 * Local variables:
2166 * mode: C
2167 * c-set-style: "BSD"
2168 * c-basic-offset: 4
2169 * tab-width: 4
2170 * indent-tabs-mode: nil
2171 * End:
2172 */