debuggers.hg

view xen/drivers/passthrough/vtd/iommu.c @ 20974:9a9ea52c3680

VT-d: get rid of duplicated definition

free_pgtable_maddr was implemented the same for x86 and IA64, so it's
not necessary to define it separately for x86 and IA64. This patch
moves free_pgtable_maddr definition to iommu.c to avoid duplicated
definition.

Signed-off-by: Weidong Han <weidong.han@intel.com>
author Keir Fraser <keir.fraser@citrix.com>
date Thu Feb 11 19:51:15 2010 +0000 (2010-02-11)
parents f909045c2284
children 3b475d9ed6b5
line source
1 /*
2 * Copyright (c) 2006, Intel Corporation.
3 *
4 * This program is free software; you can redistribute it and/or modify it
5 * under the terms and conditions of the GNU General Public License,
6 * version 2, as published by the Free Software Foundation.
7 *
8 * This program is distributed in the hope it will be useful, but WITHOUT
9 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
10 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
11 * more details.
12 *
13 * You should have received a copy of the GNU General Public License along with
14 * this program; if not, write to the Free Software Foundation, Inc., 59 Temple
15 * Place - Suite 330, Boston, MA 02111-1307 USA.
16 *
17 * Copyright (C) Ashok Raj <ashok.raj@intel.com>
18 * Copyright (C) Shaohua Li <shaohua.li@intel.com>
19 * Copyright (C) Allen Kay <allen.m.kay@intel.com> - adapted to xen
20 */
22 #include <xen/irq.h>
23 #include <xen/sched.h>
24 #include <xen/xmalloc.h>
25 #include <xen/domain_page.h>
26 #include <xen/iommu.h>
27 #include <asm/hvm/iommu.h>
28 #include <xen/numa.h>
29 #include <xen/time.h>
30 #include <xen/pci.h>
31 #include <xen/pci_regs.h>
32 #include <xen/keyhandler.h>
33 #include <asm/msi.h>
34 #include <asm/irq.h>
35 #include <mach_apic.h>
36 #include "iommu.h"
37 #include "dmar.h"
38 #include "extern.h"
39 #include "vtd.h"
41 int nr_iommus;
42 static bool_t rwbf_quirk;
44 static void setup_dom0_devices(struct domain *d);
45 static void setup_dom0_rmrr(struct domain *d);
47 static int domain_iommu_domid(struct domain *d,
48 struct iommu *iommu)
49 {
50 unsigned long nr_dom, i;
52 nr_dom = cap_ndoms(iommu->cap);
53 i = find_first_bit(iommu->domid_bitmap, nr_dom);
54 while ( i < nr_dom )
55 {
56 if ( iommu->domid_map[i] == d->domain_id )
57 return i;
59 i = find_next_bit(iommu->domid_bitmap, nr_dom, i+1);
60 }
62 gdprintk(XENLOG_ERR VTDPREFIX,
63 "Cannot get valid iommu domid: domid=%d iommu->index=%d\n",
64 d->domain_id, iommu->index);
65 return -1;
66 }
68 #define DID_FIELD_WIDTH 16
69 #define DID_HIGH_OFFSET 8
70 static int context_set_domain_id(struct context_entry *context,
71 struct domain *d,
72 struct iommu *iommu)
73 {
74 unsigned long nr_dom, i;
75 int found = 0;
77 ASSERT(spin_is_locked(&iommu->lock));
79 nr_dom = cap_ndoms(iommu->cap);
80 i = find_first_bit(iommu->domid_bitmap, nr_dom);
81 while ( i < nr_dom )
82 {
83 if ( iommu->domid_map[i] == d->domain_id )
84 {
85 found = 1;
86 break;
87 }
88 i = find_next_bit(iommu->domid_bitmap, nr_dom, i+1);
89 }
91 if ( found == 0 )
92 {
93 i = find_first_zero_bit(iommu->domid_bitmap, nr_dom);
94 if ( i >= nr_dom )
95 {
96 gdprintk(XENLOG_ERR VTDPREFIX, "IOMMU: no free domain ids\n");
97 return -EFAULT;
98 }
99 iommu->domid_map[i] = d->domain_id;
100 }
102 set_bit(i, iommu->domid_bitmap);
103 context->hi |= (i & ((1 << DID_FIELD_WIDTH) - 1)) << DID_HIGH_OFFSET;
104 return 0;
105 }
107 static struct intel_iommu *alloc_intel_iommu(void)
108 {
109 struct intel_iommu *intel;
111 intel = xmalloc(struct intel_iommu);
112 if ( intel == NULL )
113 return NULL;
114 memset(intel, 0, sizeof(struct intel_iommu));
116 spin_lock_init(&intel->qi_ctrl.qinval_lock);
117 spin_lock_init(&intel->ir_ctrl.iremap_lock);
119 return intel;
120 }
122 static void free_intel_iommu(struct intel_iommu *intel)
123 {
124 xfree(intel);
125 }
127 struct qi_ctrl *iommu_qi_ctrl(struct iommu *iommu)
128 {
129 return iommu ? &iommu->intel->qi_ctrl : NULL;
130 }
132 struct ir_ctrl *iommu_ir_ctrl(struct iommu *iommu)
133 {
134 return iommu ? &iommu->intel->ir_ctrl : NULL;
135 }
137 struct iommu_flush *iommu_get_flush(struct iommu *iommu)
138 {
139 return iommu ? &iommu->intel->flush : NULL;
140 }
142 static unsigned int clflush_size;
143 static int iommus_incoherent;
144 static void __iommu_flush_cache(void *addr, unsigned int size)
145 {
146 int i;
148 if ( !iommus_incoherent )
149 return;
151 for ( i = 0; i < size; i += clflush_size )
152 cacheline_flush((char *)addr + i);
153 }
155 void iommu_flush_cache_entry(void *addr, unsigned int size)
156 {
157 __iommu_flush_cache(addr, size);
158 }
160 void iommu_flush_cache_page(void *addr, unsigned long npages)
161 {
162 __iommu_flush_cache(addr, PAGE_SIZE * npages);
163 }
165 /* Allocate page table, return its machine address */
166 u64 alloc_pgtable_maddr(struct acpi_drhd_unit *drhd, unsigned long npages)
167 {
168 struct acpi_rhsa_unit *rhsa;
169 struct page_info *pg, *cur_pg;
170 u64 *vaddr;
171 int node = -1, i;
173 rhsa = drhd_to_rhsa(drhd);
174 if ( rhsa )
175 node = pxm_to_node(rhsa->proximity_domain);
177 pg = alloc_domheap_pages(NULL, get_order_from_pages(npages),
178 (node == -1 ) ? 0 : MEMF_node(node));
179 if ( !pg )
180 return 0;
182 cur_pg = pg;
183 for ( i = 0; i < npages; i++ )
184 {
185 vaddr = __map_domain_page(cur_pg);
186 memset(vaddr, 0, PAGE_SIZE);
188 iommu_flush_cache_page(vaddr, 1);
189 unmap_domain_page(vaddr);
190 cur_pg++;
191 }
193 return page_to_maddr(pg);
194 }
196 void free_pgtable_maddr(u64 maddr)
197 {
198 if ( maddr != 0 )
199 free_domheap_page(maddr_to_page(maddr));
200 }
202 /* context entry handling */
203 static u64 bus_to_context_maddr(struct iommu *iommu, u8 bus)
204 {
205 struct acpi_drhd_unit *drhd;
206 struct root_entry *root, *root_entries;
207 u64 maddr;
209 ASSERT(spin_is_locked(&iommu->lock));
210 root_entries = (struct root_entry *)map_vtd_domain_page(iommu->root_maddr);
211 root = &root_entries[bus];
212 if ( !root_present(*root) )
213 {
214 drhd = iommu_to_drhd(iommu);
215 maddr = alloc_pgtable_maddr(drhd, 1);
216 if ( maddr == 0 )
217 {
218 unmap_vtd_domain_page(root_entries);
219 return 0;
220 }
221 set_root_value(*root, maddr);
222 set_root_present(*root);
223 iommu_flush_cache_entry(root, sizeof(struct root_entry));
224 }
225 maddr = (u64) get_context_addr(*root);
226 unmap_vtd_domain_page(root_entries);
227 return maddr;
228 }
230 static u64 addr_to_dma_page_maddr(struct domain *domain, u64 addr, int alloc)
231 {
232 struct acpi_drhd_unit *drhd;
233 struct pci_dev *pdev;
234 struct hvm_iommu *hd = domain_hvm_iommu(domain);
235 int addr_width = agaw_to_width(hd->agaw);
236 struct dma_pte *parent, *pte = NULL;
237 int level = agaw_to_level(hd->agaw);
238 int offset;
239 u64 pte_maddr = 0, maddr;
240 u64 *vaddr = NULL;
242 addr &= (((u64)1) << addr_width) - 1;
243 ASSERT(spin_is_locked(&hd->mapping_lock));
244 if ( hd->pgd_maddr == 0 )
245 {
246 /*
247 * just get any passthrough device in the domainr - assume user
248 * assigns only devices from same node to a given guest.
249 */
250 pdev = pci_get_pdev_by_domain(domain, -1, -1);
251 drhd = acpi_find_matched_drhd_unit(pdev);
252 if ( !alloc || ((hd->pgd_maddr = alloc_pgtable_maddr(drhd, 1)) == 0) )
253 goto out;
254 }
256 parent = (struct dma_pte *)map_vtd_domain_page(hd->pgd_maddr);
257 while ( level > 1 )
258 {
259 offset = address_level_offset(addr, level);
260 pte = &parent[offset];
262 if ( dma_pte_addr(*pte) == 0 )
263 {
264 if ( !alloc )
265 break;
267 pdev = pci_get_pdev_by_domain(domain, -1, -1);
268 drhd = acpi_find_matched_drhd_unit(pdev);
269 maddr = alloc_pgtable_maddr(drhd, 1);
270 if ( !maddr )
271 break;
273 dma_set_pte_addr(*pte, maddr);
274 vaddr = map_vtd_domain_page(maddr);
276 /*
277 * high level table always sets r/w, last level
278 * page table control read/write
279 */
280 dma_set_pte_readable(*pte);
281 dma_set_pte_writable(*pte);
282 iommu_flush_cache_entry(pte, sizeof(struct dma_pte));
283 }
284 else
285 {
286 vaddr = map_vtd_domain_page(pte->val);
287 }
289 if ( level == 2 )
290 {
291 pte_maddr = pte->val & PAGE_MASK_4K;
292 unmap_vtd_domain_page(vaddr);
293 break;
294 }
296 unmap_vtd_domain_page(parent);
297 parent = (struct dma_pte *)vaddr;
298 vaddr = NULL;
299 level--;
300 }
302 unmap_vtd_domain_page(parent);
303 out:
304 return pte_maddr;
305 }
307 static void iommu_flush_write_buffer(struct iommu *iommu)
308 {
309 u32 val;
310 unsigned long flags;
312 if ( !rwbf_quirk && !cap_rwbf(iommu->cap) )
313 return;
315 spin_lock_irqsave(&iommu->register_lock, flags);
316 val = dmar_readl(iommu->reg, DMAR_GSTS_REG);
317 dmar_writel(iommu->reg, DMAR_GCMD_REG, val | DMA_GCMD_WBF);
319 /* Make sure hardware complete it */
320 IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG, dmar_readl,
321 !(val & DMA_GSTS_WBFS), val);
323 spin_unlock_irqrestore(&iommu->register_lock, flags);
324 }
326 /* return value determine if we need a write buffer flush */
327 static int flush_context_reg(
328 void *_iommu,
329 u16 did, u16 source_id, u8 function_mask, u64 type,
330 int flush_non_present_entry)
331 {
332 struct iommu *iommu = (struct iommu *) _iommu;
333 u64 val = 0;
334 unsigned long flags;
336 /*
337 * In the non-present entry flush case, if hardware doesn't cache
338 * non-present entry we do nothing and if hardware cache non-present
339 * entry, we flush entries of domain 0 (the domain id is used to cache
340 * any non-present entries)
341 */
342 if ( flush_non_present_entry )
343 {
344 if ( !cap_caching_mode(iommu->cap) )
345 return 1;
346 else
347 did = 0;
348 }
350 /* use register invalidation */
351 switch ( type )
352 {
353 case DMA_CCMD_GLOBAL_INVL:
354 val = DMA_CCMD_GLOBAL_INVL;
355 break;
356 case DMA_CCMD_DOMAIN_INVL:
357 val = DMA_CCMD_DOMAIN_INVL|DMA_CCMD_DID(did);
358 break;
359 case DMA_CCMD_DEVICE_INVL:
360 val = DMA_CCMD_DEVICE_INVL|DMA_CCMD_DID(did)
361 |DMA_CCMD_SID(source_id)|DMA_CCMD_FM(function_mask);
362 break;
363 default:
364 BUG();
365 }
366 val |= DMA_CCMD_ICC;
368 spin_lock_irqsave(&iommu->register_lock, flags);
369 dmar_writeq(iommu->reg, DMAR_CCMD_REG, val);
371 /* Make sure hardware complete it */
372 IOMMU_WAIT_OP(iommu, DMAR_CCMD_REG, dmar_readq,
373 !(val & DMA_CCMD_ICC), val);
375 spin_unlock_irqrestore(&iommu->register_lock, flags);
376 /* flush context entry will implicitly flush write buffer */
377 return 0;
378 }
380 static int inline iommu_flush_context_global(
381 struct iommu *iommu, int flush_non_present_entry)
382 {
383 struct iommu_flush *flush = iommu_get_flush(iommu);
384 return flush->context(iommu, 0, 0, 0, DMA_CCMD_GLOBAL_INVL,
385 flush_non_present_entry);
386 }
388 static int inline iommu_flush_context_domain(
389 struct iommu *iommu, u16 did, int flush_non_present_entry)
390 {
391 struct iommu_flush *flush = iommu_get_flush(iommu);
392 return flush->context(iommu, did, 0, 0, DMA_CCMD_DOMAIN_INVL,
393 flush_non_present_entry);
394 }
396 static int inline iommu_flush_context_device(
397 struct iommu *iommu, u16 did, u16 source_id,
398 u8 function_mask, int flush_non_present_entry)
399 {
400 struct iommu_flush *flush = iommu_get_flush(iommu);
401 return flush->context(iommu, did, source_id, function_mask,
402 DMA_CCMD_DEVICE_INVL,
403 flush_non_present_entry);
404 }
406 /* return value determine if we need a write buffer flush */
407 static int flush_iotlb_reg(void *_iommu, u16 did,
408 u64 addr, unsigned int size_order, u64 type,
409 int flush_non_present_entry, int flush_dev_iotlb)
410 {
411 struct iommu *iommu = (struct iommu *) _iommu;
412 int tlb_offset = ecap_iotlb_offset(iommu->ecap);
413 u64 val = 0, val_iva = 0;
414 unsigned long flags;
416 /*
417 * In the non-present entry flush case, if hardware doesn't cache
418 * non-present entry we do nothing and if hardware cache non-present
419 * entry, we flush entries of domain 0 (the domain id is used to cache
420 * any non-present entries)
421 */
422 if ( flush_non_present_entry )
423 {
424 if ( !cap_caching_mode(iommu->cap) )
425 return 1;
426 else
427 did = 0;
428 }
430 /* use register invalidation */
431 switch ( type )
432 {
433 case DMA_TLB_GLOBAL_FLUSH:
434 /* global flush doesn't need set IVA_REG */
435 val = DMA_TLB_GLOBAL_FLUSH|DMA_TLB_IVT;
436 break;
437 case DMA_TLB_DSI_FLUSH:
438 val = DMA_TLB_DSI_FLUSH|DMA_TLB_IVT|DMA_TLB_DID(did);
439 break;
440 case DMA_TLB_PSI_FLUSH:
441 val = DMA_TLB_PSI_FLUSH|DMA_TLB_IVT|DMA_TLB_DID(did);
442 /* Note: always flush non-leaf currently */
443 val_iva = size_order | addr;
444 break;
445 default:
446 BUG();
447 }
448 /* Note: set drain read/write */
449 if ( cap_read_drain(iommu->cap) )
450 val |= DMA_TLB_READ_DRAIN;
451 if ( cap_write_drain(iommu->cap) )
452 val |= DMA_TLB_WRITE_DRAIN;
454 spin_lock_irqsave(&iommu->register_lock, flags);
455 /* Note: Only uses first TLB reg currently */
456 if ( val_iva )
457 dmar_writeq(iommu->reg, tlb_offset, val_iva);
458 dmar_writeq(iommu->reg, tlb_offset + 8, val);
460 /* Make sure hardware complete it */
461 IOMMU_WAIT_OP(iommu, (tlb_offset + 8), dmar_readq,
462 !(val & DMA_TLB_IVT), val);
463 spin_unlock_irqrestore(&iommu->register_lock, flags);
465 /* check IOTLB invalidation granularity */
466 if ( DMA_TLB_IAIG(val) == 0 )
467 dprintk(XENLOG_ERR VTDPREFIX, "IOMMU: flush IOTLB failed\n");
469 /* flush iotlb entry will implicitly flush write buffer */
470 return 0;
471 }
473 static int inline iommu_flush_iotlb_global(struct iommu *iommu,
474 int flush_non_present_entry, int flush_dev_iotlb)
475 {
476 struct iommu_flush *flush = iommu_get_flush(iommu);
477 return flush->iotlb(iommu, 0, 0, 0, DMA_TLB_GLOBAL_FLUSH,
478 flush_non_present_entry, flush_dev_iotlb);
479 }
481 static int inline iommu_flush_iotlb_dsi(struct iommu *iommu, u16 did,
482 int flush_non_present_entry, int flush_dev_iotlb)
483 {
484 struct iommu_flush *flush = iommu_get_flush(iommu);
485 return flush->iotlb(iommu, did, 0, 0, DMA_TLB_DSI_FLUSH,
486 flush_non_present_entry, flush_dev_iotlb);
487 }
489 static int inline get_alignment(u64 base, unsigned int size)
490 {
491 int t = 0;
492 u64 end;
494 end = base + size - 1;
495 while ( base != end )
496 {
497 t++;
498 base >>= 1;
499 end >>= 1;
500 }
501 return t;
502 }
504 static int inline iommu_flush_iotlb_psi(
505 struct iommu *iommu, u16 did, u64 addr, unsigned int pages,
506 int flush_non_present_entry, int flush_dev_iotlb)
507 {
508 unsigned int align;
509 struct iommu_flush *flush = iommu_get_flush(iommu);
511 ASSERT(!(addr & (~PAGE_MASK_4K)));
512 ASSERT(pages > 0);
514 /* Fallback to domain selective flush if no PSI support */
515 if ( !cap_pgsel_inv(iommu->cap) )
516 return iommu_flush_iotlb_dsi(iommu, did, flush_non_present_entry, flush_dev_iotlb);
518 /*
519 * PSI requires page size is 2 ^ x, and the base address is naturally
520 * aligned to the size
521 */
522 align = get_alignment(addr >> PAGE_SHIFT_4K, pages);
523 /* Fallback to domain selective flush if size is too big */
524 if ( align > cap_max_amask_val(iommu->cap) )
525 return iommu_flush_iotlb_dsi(iommu, did, flush_non_present_entry, flush_dev_iotlb);
527 addr >>= PAGE_SHIFT_4K + align;
528 addr <<= PAGE_SHIFT_4K + align;
530 return flush->iotlb(iommu, did, addr, align, DMA_TLB_PSI_FLUSH,
531 flush_non_present_entry, flush_dev_iotlb);
532 }
534 static void iommu_flush_all(void)
535 {
536 struct acpi_drhd_unit *drhd;
537 struct iommu *iommu;
538 int flush_dev_iotlb;
540 flush_all_cache();
541 for_each_drhd_unit ( drhd )
542 {
543 iommu = drhd->iommu;
544 iommu_flush_context_global(iommu, 0);
545 flush_dev_iotlb = find_ats_dev_drhd(iommu) ? 1 : 0;
546 iommu_flush_iotlb_global(iommu, 0, flush_dev_iotlb);
547 }
548 }
550 /* clear one page's page table */
551 static void dma_pte_clear_one(struct domain *domain, u64 addr)
552 {
553 struct hvm_iommu *hd = domain_hvm_iommu(domain);
554 struct acpi_drhd_unit *drhd;
555 struct iommu *iommu;
556 struct dma_pte *page = NULL, *pte = NULL;
557 u64 pg_maddr;
558 int flush_dev_iotlb;
559 int iommu_domid;
560 struct list_head *rmrr_list, *tmp;
561 struct mapped_rmrr *mrmrr;
563 spin_lock(&hd->mapping_lock);
564 /* get last level pte */
565 pg_maddr = addr_to_dma_page_maddr(domain, addr, 0);
566 if ( pg_maddr == 0 )
567 {
568 spin_unlock(&hd->mapping_lock);
569 return;
570 }
572 page = (struct dma_pte *)map_vtd_domain_page(pg_maddr);
573 pte = page + address_level_offset(addr, 1);
575 if ( !dma_pte_present(*pte) )
576 {
577 spin_unlock(&hd->mapping_lock);
578 unmap_vtd_domain_page(page);
579 return;
580 }
582 dma_clear_pte(*pte);
583 spin_unlock(&hd->mapping_lock);
584 iommu_flush_cache_entry(pte, sizeof(struct dma_pte));
586 /* No need pcidevs_lock here since do that on assign/deassign device*/
587 for_each_drhd_unit ( drhd )
588 {
589 iommu = drhd->iommu;
590 if ( test_bit(iommu->index, &hd->iommu_bitmap) )
591 {
592 flush_dev_iotlb = find_ats_dev_drhd(iommu) ? 1 : 0;
593 iommu_domid= domain_iommu_domid(domain, iommu);
594 if ( iommu_domid == -1 )
595 continue;
596 if ( iommu_flush_iotlb_psi(iommu, iommu_domid,
597 addr, 1, 0, flush_dev_iotlb) )
598 iommu_flush_write_buffer(iommu);
599 }
600 }
602 unmap_vtd_domain_page(page);
604 /* if the cleared address is between mapped RMRR region,
605 * remove the mapped RMRR
606 */
607 spin_lock(&pcidevs_lock);
608 list_for_each_safe ( rmrr_list, tmp, &hd->mapped_rmrrs )
609 {
610 mrmrr = list_entry(rmrr_list, struct mapped_rmrr, list);
611 if ( addr >= mrmrr->base && addr <= mrmrr->end )
612 {
613 list_del(&mrmrr->list);
614 xfree(mrmrr);
615 break;
616 }
617 }
618 spin_unlock(&pcidevs_lock);
619 }
621 static void iommu_free_pagetable(u64 pt_maddr, int level)
622 {
623 int i;
624 struct dma_pte *pt_vaddr, *pte;
625 int next_level = level - 1;
627 if ( pt_maddr == 0 )
628 return;
630 pt_vaddr = (struct dma_pte *)map_vtd_domain_page(pt_maddr);
632 for ( i = 0; i < PTE_NUM; i++ )
633 {
634 pte = &pt_vaddr[i];
635 if ( !dma_pte_present(*pte) )
636 continue;
638 if ( next_level >= 1 )
639 iommu_free_pagetable(dma_pte_addr(*pte), next_level);
641 dma_clear_pte(*pte);
642 iommu_flush_cache_entry(pte, sizeof(struct dma_pte));
643 }
645 unmap_vtd_domain_page(pt_vaddr);
646 free_pgtable_maddr(pt_maddr);
647 }
649 static int iommu_set_root_entry(struct iommu *iommu)
650 {
651 struct acpi_drhd_unit *drhd;
652 u32 sts;
653 unsigned long flags;
655 spin_lock(&iommu->lock);
657 if ( iommu->root_maddr == 0 )
658 {
659 drhd = iommu_to_drhd(iommu);
660 iommu->root_maddr = alloc_pgtable_maddr(drhd, 1);
661 }
663 if ( iommu->root_maddr == 0 )
664 {
665 spin_unlock(&iommu->lock);
666 return -ENOMEM;
667 }
669 spin_unlock(&iommu->lock);
670 spin_lock_irqsave(&iommu->register_lock, flags);
671 dmar_writeq(iommu->reg, DMAR_RTADDR_REG, iommu->root_maddr);
673 sts = dmar_readl(iommu->reg, DMAR_GSTS_REG);
674 dmar_writel(iommu->reg, DMAR_GCMD_REG, sts | DMA_GCMD_SRTP);
676 /* Make sure hardware complete it */
677 IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG, dmar_readl,
678 (sts & DMA_GSTS_RTPS), sts);
679 spin_unlock_irqrestore(&iommu->register_lock, flags);
681 return 0;
682 }
684 static void iommu_enable_translation(struct iommu *iommu)
685 {
686 u32 sts;
687 unsigned long flags;
689 dprintk(XENLOG_INFO VTDPREFIX,
690 "iommu_enable_translation: iommu->reg = %p\n", iommu->reg);
691 spin_lock_irqsave(&iommu->register_lock, flags);
692 sts = dmar_readl(iommu->reg, DMAR_GSTS_REG);
693 dmar_writel(iommu->reg, DMAR_GCMD_REG, sts | DMA_GCMD_TE);
695 /* Make sure hardware complete it */
696 IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG, dmar_readl,
697 (sts & DMA_GSTS_TES), sts);
698 spin_unlock_irqrestore(&iommu->register_lock, flags);
700 /* Disable PMRs when VT-d engine takes effect per spec definition */
701 disable_pmr(iommu);
702 }
704 static void iommu_disable_translation(struct iommu *iommu)
705 {
706 u32 sts;
707 unsigned long flags;
709 spin_lock_irqsave(&iommu->register_lock, flags);
710 sts = dmar_readl(iommu->reg, DMAR_GSTS_REG);
711 dmar_writel(iommu->reg, DMAR_GCMD_REG, sts & (~DMA_GCMD_TE));
713 /* Make sure hardware complete it */
714 IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG, dmar_readl,
715 !(sts & DMA_GSTS_TES), sts);
716 spin_unlock_irqrestore(&iommu->register_lock, flags);
717 }
719 enum faulttype {
720 DMA_REMAP,
721 INTR_REMAP,
722 UNKNOWN,
723 };
725 static const char *dma_remap_fault_reasons[] =
726 {
727 "Software",
728 "Present bit in root entry is clear",
729 "Present bit in context entry is clear",
730 "Invalid context entry",
731 "Access beyond MGAW",
732 "PTE Write access is not set",
733 "PTE Read access is not set",
734 "Next page table ptr is invalid",
735 "Root table address invalid",
736 "Context table ptr is invalid",
737 "non-zero reserved fields in RTP",
738 "non-zero reserved fields in CTP",
739 "non-zero reserved fields in PTE",
740 "Blocked a DMA translation request",
741 };
743 static const char *intr_remap_fault_reasons[] =
744 {
745 "Detected reserved fields in the decoded interrupt-remapped request",
746 "Interrupt index exceeded the interrupt-remapping table size",
747 "Present field in the IRTE entry is clear",
748 "Error accessing interrupt-remapping table pointed by IRTA_REG",
749 "Detected reserved fields in the IRTE entry",
750 "Blocked a compatibility format interrupt request",
751 "Blocked an interrupt request due to source-id verification failure",
752 };
754 static const char *iommu_get_fault_reason(u8 fault_reason, int *fault_type)
755 {
756 if ( fault_reason >= 0x20 && ( fault_reason < 0x20 +
757 ARRAY_SIZE(intr_remap_fault_reasons)) )
758 {
759 *fault_type = INTR_REMAP;
760 return intr_remap_fault_reasons[fault_reason - 0x20];
761 }
762 else if ( fault_reason < ARRAY_SIZE(dma_remap_fault_reasons) )
763 {
764 *fault_type = DMA_REMAP;
765 return dma_remap_fault_reasons[fault_reason];
766 }
767 else
768 {
769 *fault_type = UNKNOWN;
770 return "Unknown";
771 }
772 }
774 static struct iommu **irq_to_iommu;
775 static int iommu_page_fault_do_one(struct iommu *iommu, int type,
776 u8 fault_reason, u16 source_id, u64 addr)
777 {
778 const char *reason;
779 int fault_type;
780 reason = iommu_get_fault_reason(fault_reason, &fault_type);
782 if ( fault_type == DMA_REMAP )
783 {
784 dprintk(XENLOG_WARNING VTDPREFIX,
785 "DMAR:[%s] Request device [%02x:%02x.%d] "
786 "fault addr %"PRIx64", iommu reg = %p\n"
787 "DMAR:[fault reason %02xh] %s\n",
788 (type ? "DMA Read" : "DMA Write"),
789 (source_id >> 8), PCI_SLOT(source_id & 0xFF),
790 PCI_FUNC(source_id & 0xFF), addr, iommu->reg,
791 fault_reason, reason);
792 #ifndef __i386__ /* map_domain_page() cannot be used in this context */
793 print_vtd_entries(iommu, (source_id >> 8),
794 (source_id & 0xff), (addr >> PAGE_SHIFT));
795 #endif
796 }
797 else
798 dprintk(XENLOG_WARNING VTDPREFIX,
799 "INTR-REMAP: Request device [%02x:%02x.%d] "
800 "fault index %"PRIx64", iommu reg = %p\n"
801 "INTR-REMAP:[fault reason %02xh] %s\n",
802 (source_id >> 8), PCI_SLOT(source_id & 0xFF),
803 PCI_FUNC(source_id & 0xFF), addr >> 48, iommu->reg,
804 fault_reason, reason);
805 return 0;
807 }
809 static void iommu_fault_status(u32 fault_status)
810 {
811 if ( fault_status & DMA_FSTS_PFO )
812 dprintk(XENLOG_ERR VTDPREFIX,
813 "iommu_fault_status: Fault Overflow\n");
814 if ( fault_status & DMA_FSTS_PPF )
815 dprintk(XENLOG_ERR VTDPREFIX,
816 "iommu_fault_status: Primary Pending Fault\n");
817 if ( fault_status & DMA_FSTS_AFO )
818 dprintk(XENLOG_ERR VTDPREFIX,
819 "iommu_fault_status: Advanced Fault Overflow\n");
820 if ( fault_status & DMA_FSTS_APF )
821 dprintk(XENLOG_ERR VTDPREFIX,
822 "iommu_fault_status: Advanced Pending Fault\n");
823 if ( fault_status & DMA_FSTS_IQE )
824 dprintk(XENLOG_ERR VTDPREFIX,
825 "iommu_fault_status: Invalidation Queue Error\n");
826 if ( fault_status & DMA_FSTS_ICE )
827 dprintk(XENLOG_ERR VTDPREFIX,
828 "iommu_fault_status: Invalidation Completion Error\n");
829 if ( fault_status & DMA_FSTS_ITE )
830 dprintk(XENLOG_ERR VTDPREFIX,
831 "iommu_fault_status: Invalidation Time-out Error\n");
832 }
834 #define PRIMARY_FAULT_REG_LEN (16)
835 static void iommu_page_fault(int irq, void *dev_id,
836 struct cpu_user_regs *regs)
837 {
838 struct iommu *iommu = dev_id;
839 int reg, fault_index;
840 u32 fault_status;
841 unsigned long flags;
843 fault_status = dmar_readl(iommu->reg, DMAR_FSTS_REG);
845 iommu_fault_status(fault_status);
847 /* FIXME: ignore advanced fault log */
848 if ( !(fault_status & DMA_FSTS_PPF) )
849 goto clear_overflow;
851 fault_index = dma_fsts_fault_record_index(fault_status);
852 reg = cap_fault_reg_offset(iommu->cap);
853 while (1)
854 {
855 u8 fault_reason;
856 u16 source_id;
857 u32 data;
858 u64 guest_addr;
859 int type;
861 /* highest 32 bits */
862 spin_lock_irqsave(&iommu->register_lock, flags);
863 data = dmar_readl(iommu->reg, reg +
864 fault_index * PRIMARY_FAULT_REG_LEN + 12);
865 if ( !(data & DMA_FRCD_F) )
866 {
867 spin_unlock_irqrestore(&iommu->register_lock, flags);
868 break;
869 }
871 fault_reason = dma_frcd_fault_reason(data);
872 type = dma_frcd_type(data);
874 data = dmar_readl(iommu->reg, reg +
875 fault_index * PRIMARY_FAULT_REG_LEN + 8);
876 source_id = dma_frcd_source_id(data);
878 guest_addr = dmar_readq(iommu->reg, reg +
879 fault_index * PRIMARY_FAULT_REG_LEN);
880 guest_addr = dma_frcd_page_addr(guest_addr);
881 /* clear the fault */
882 dmar_writel(iommu->reg, reg +
883 fault_index * PRIMARY_FAULT_REG_LEN + 12, DMA_FRCD_F);
884 spin_unlock_irqrestore(&iommu->register_lock, flags);
886 iommu_page_fault_do_one(iommu, type, fault_reason,
887 source_id, guest_addr);
889 fault_index++;
890 if ( fault_index > cap_num_fault_regs(iommu->cap) )
891 fault_index = 0;
892 }
893 clear_overflow:
894 /* clear primary fault overflow */
895 fault_status = readl(iommu->reg + DMAR_FSTS_REG);
896 if ( fault_status & DMA_FSTS_PFO )
897 {
898 spin_lock_irqsave(&iommu->register_lock, flags);
899 dmar_writel(iommu->reg, DMAR_FSTS_REG, DMA_FSTS_PFO);
900 spin_unlock_irqrestore(&iommu->register_lock, flags);
901 }
902 }
904 static void dma_msi_unmask(unsigned int irq)
905 {
906 struct iommu *iommu = irq_to_iommu[irq];
907 unsigned long flags;
909 /* unmask it */
910 spin_lock_irqsave(&iommu->register_lock, flags);
911 dmar_writel(iommu->reg, DMAR_FECTL_REG, 0);
912 spin_unlock_irqrestore(&iommu->register_lock, flags);
913 }
915 static void dma_msi_mask(unsigned int irq)
916 {
917 unsigned long flags;
918 struct iommu *iommu = irq_to_iommu[irq];
919 struct irq_desc *desc = irq_to_desc(irq);
921 irq_complete_move(&desc);
923 /* mask it */
924 spin_lock_irqsave(&iommu->register_lock, flags);
925 dmar_writel(iommu->reg, DMAR_FECTL_REG, DMA_FECTL_IM);
926 spin_unlock_irqrestore(&iommu->register_lock, flags);
927 }
929 static unsigned int dma_msi_startup(unsigned int irq)
930 {
931 dma_msi_unmask(irq);
932 return 0;
933 }
935 static void dma_msi_end(unsigned int irq)
936 {
937 dma_msi_unmask(irq);
938 ack_APIC_irq();
939 }
941 static void dma_msi_set_affinity(unsigned int irq, cpumask_t mask)
942 {
943 struct msi_msg msg;
944 unsigned int dest;
945 unsigned long flags;
947 struct iommu *iommu = irq_to_iommu[irq];
948 struct irq_desc *desc = irq_to_desc(irq);
949 struct irq_cfg *cfg = desc->chip_data;
951 #ifdef CONFIG_X86
952 dest = set_desc_affinity(desc, mask);
953 if (dest == BAD_APICID){
954 dprintk(XENLOG_ERR VTDPREFIX, "Set iommu interrupt affinity error!\n");
955 return;
956 }
958 memset(&msg, 0, sizeof(msg));
959 msg.data = MSI_DATA_VECTOR(cfg->vector) & 0xff;
960 msg.data |= 1 << 14;
961 msg.data |= (INT_DELIVERY_MODE != dest_LowestPrio) ?
962 MSI_DATA_DELIVERY_FIXED:
963 MSI_DATA_DELIVERY_LOWPRI;
965 /* Follow MSI setting */
966 if (x2apic_enabled)
967 msg.address_hi = dest & 0xFFFFFF00;
968 msg.address_lo = (MSI_ADDRESS_HEADER << (MSI_ADDRESS_HEADER_SHIFT + 8));
969 msg.address_lo |= INT_DEST_MODE ? MSI_ADDR_DESTMODE_LOGIC:
970 MSI_ADDR_DESTMODE_PHYS;
971 msg.address_lo |= (INT_DELIVERY_MODE != dest_LowestPrio) ?
972 MSI_ADDR_REDIRECTION_CPU:
973 MSI_ADDR_REDIRECTION_LOWPRI;
974 msg.address_lo |= MSI_ADDR_DEST_ID(dest & 0xff);
975 #else
976 memset(&msg, 0, sizeof(msg));
977 msg.data = cfg->vector & 0xff;
978 msg.data |= 1 << 14;
979 msg.address_lo = (MSI_ADDRESS_HEADER << (MSI_ADDRESS_HEADER_SHIFT + 8));
980 msg.address_lo |= MSI_PHYSICAL_MODE << 2;
981 msg.address_lo |= MSI_REDIRECTION_HINT_MODE << 3;
982 dest = cpu_physical_id(first_cpu(mask));
983 msg.address_lo |= dest << MSI_TARGET_CPU_SHIFT;
984 #endif
986 spin_lock_irqsave(&iommu->register_lock, flags);
987 dmar_writel(iommu->reg, DMAR_FEDATA_REG, msg.data);
988 dmar_writel(iommu->reg, DMAR_FEADDR_REG, msg.address_lo);
989 dmar_writel(iommu->reg, DMAR_FEUADDR_REG, msg.address_hi);
990 spin_unlock_irqrestore(&iommu->register_lock, flags);
991 }
993 static hw_irq_controller dma_msi_type = {
994 .typename = "DMA_MSI",
995 .startup = dma_msi_startup,
996 .shutdown = dma_msi_mask,
997 .enable = dma_msi_unmask,
998 .disable = dma_msi_mask,
999 .ack = dma_msi_mask,
1000 .end = dma_msi_end,
1001 .set_affinity = dma_msi_set_affinity,
1002 };
1004 static int iommu_set_interrupt(struct iommu *iommu)
1006 int irq, ret;
1008 irq = create_irq();
1009 if ( irq <= 0 )
1011 dprintk(XENLOG_ERR VTDPREFIX, "IOMMU: no irq available!\n");
1012 return -EINVAL;
1015 irq_desc[irq].handler = &dma_msi_type;
1016 irq_to_iommu[irq] = iommu;
1017 #ifdef CONFIG_X86
1018 ret = request_irq(irq, iommu_page_fault, 0, "dmar", iommu);
1019 #else
1020 ret = request_irq_vector(irq, iommu_page_fault, 0, "dmar", iommu);
1021 #endif
1022 if ( ret )
1024 irq_desc[irq].handler = &no_irq_type;
1025 irq_to_iommu[irq] = NULL;
1026 destroy_irq(irq);
1027 dprintk(XENLOG_ERR VTDPREFIX, "IOMMU: can't request irq\n");
1028 return ret;
1031 return irq;
1034 static int iommu_alloc(struct acpi_drhd_unit *drhd)
1036 struct iommu *iommu;
1037 unsigned long sagaw, nr_dom;
1038 int agaw;
1040 if ( nr_iommus > MAX_IOMMUS )
1042 dprintk(XENLOG_ERR VTDPREFIX,
1043 "IOMMU: nr_iommus %d > MAX_IOMMUS\n", nr_iommus);
1044 return -ENOMEM;
1047 iommu = xmalloc(struct iommu);
1048 if ( iommu == NULL )
1049 return -ENOMEM;
1050 memset(iommu, 0, sizeof(struct iommu));
1052 iommu->irq = -1; /* No irq assigned yet. */
1054 iommu->intel = alloc_intel_iommu();
1055 if ( iommu->intel == NULL )
1057 xfree(iommu);
1058 return -ENOMEM;
1061 iommu->reg = map_to_nocache_virt(nr_iommus, drhd->address);
1062 iommu->index = nr_iommus++;
1064 iommu->cap = dmar_readq(iommu->reg, DMAR_CAP_REG);
1065 iommu->ecap = dmar_readq(iommu->reg, DMAR_ECAP_REG);
1067 dprintk(XENLOG_INFO VTDPREFIX,
1068 "drhd->address = %"PRIx64"\n", drhd->address);
1069 dprintk(XENLOG_INFO VTDPREFIX, "iommu->reg = %p\n", iommu->reg);
1071 /* Calculate number of pagetable levels: between 2 and 4. */
1072 sagaw = cap_sagaw(iommu->cap);
1073 for ( agaw = level_to_agaw(4); agaw >= 0; agaw-- )
1074 if ( test_bit(agaw, &sagaw) )
1075 break;
1076 if ( agaw < 0 )
1078 dprintk(XENLOG_ERR VTDPREFIX,
1079 "IOMMU: unsupported sagaw %lx\n", sagaw);
1080 xfree(iommu);
1081 return -ENODEV;
1083 iommu->nr_pt_levels = agaw_to_level(agaw);
1085 if ( !ecap_coherent(iommu->ecap) )
1086 iommus_incoherent = 1;
1088 /* allocate domain id bitmap */
1089 nr_dom = cap_ndoms(iommu->cap);
1090 iommu->domid_bitmap = xmalloc_array(unsigned long, BITS_TO_LONGS(nr_dom));
1091 if ( !iommu->domid_bitmap )
1092 return -ENOMEM ;
1093 memset(iommu->domid_bitmap, 0, nr_dom / 8);
1095 /*
1096 * if Caching mode is set, then invalid translations are tagged with
1097 * domain id 0, Hence reserve bit 0 for it
1098 */
1099 if ( cap_caching_mode(iommu->cap) )
1100 set_bit(0, iommu->domid_bitmap);
1102 iommu->domid_map = xmalloc_array(u16, nr_dom);
1103 if ( !iommu->domid_map )
1104 return -ENOMEM ;
1105 memset(iommu->domid_map, 0, nr_dom * sizeof(*iommu->domid_map));
1107 spin_lock_init(&iommu->lock);
1108 spin_lock_init(&iommu->register_lock);
1110 drhd->iommu = iommu;
1111 return 0;
1114 static void iommu_free(struct acpi_drhd_unit *drhd)
1116 struct iommu *iommu = drhd->iommu;
1118 if ( iommu == NULL )
1119 return;
1121 if ( iommu->root_maddr != 0 )
1123 free_pgtable_maddr(iommu->root_maddr);
1124 iommu->root_maddr = 0;
1127 if ( iommu->reg )
1128 iounmap(iommu->reg);
1130 xfree(iommu->domid_bitmap);
1131 xfree(iommu->domid_map);
1133 free_intel_iommu(iommu->intel);
1134 destroy_irq(iommu->irq);
1135 xfree(iommu);
1137 drhd->iommu = NULL;
1140 #define guestwidth_to_adjustwidth(gaw) ({ \
1141 int agaw, r = (gaw - 12) % 9; \
1142 agaw = (r == 0) ? gaw : (gaw + 9 - r); \
1143 if ( agaw > 64 ) \
1144 agaw = 64; \
1145 agaw; })
1147 static int intel_iommu_domain_init(struct domain *d)
1149 struct hvm_iommu *hd = domain_hvm_iommu(d);
1150 struct iommu *iommu;
1151 struct acpi_drhd_unit *drhd;
1153 hd->agaw = width_to_agaw(DEFAULT_DOMAIN_ADDRESS_WIDTH);
1155 if ( d->domain_id == 0 )
1157 /* Set up 1:1 page table for dom0 */
1158 iommu_set_dom0_mapping(d);
1160 setup_dom0_devices(d);
1161 setup_dom0_rmrr(d);
1163 iommu_flush_all();
1165 for_each_drhd_unit ( drhd )
1167 iommu = drhd->iommu;
1168 iommu_enable_translation(iommu);
1172 return 0;
1175 static int domain_context_mapping_one(
1176 struct domain *domain,
1177 struct iommu *iommu,
1178 u8 bus, u8 devfn)
1180 struct hvm_iommu *hd = domain_hvm_iommu(domain);
1181 struct context_entry *context, *context_entries;
1182 u64 maddr, pgd_maddr;
1183 struct pci_dev *pdev = NULL;
1184 int agaw;
1186 ASSERT(spin_is_locked(&pcidevs_lock));
1187 spin_lock(&iommu->lock);
1188 maddr = bus_to_context_maddr(iommu, bus);
1189 context_entries = (struct context_entry *)map_vtd_domain_page(maddr);
1190 context = &context_entries[devfn];
1192 if ( context_present(*context) )
1194 int res = 0;
1196 pdev = pci_get_pdev(bus, devfn);
1197 if (!pdev)
1198 res = -ENODEV;
1199 else if (pdev->domain != domain)
1200 res = -EINVAL;
1201 unmap_vtd_domain_page(context_entries);
1202 spin_unlock(&iommu->lock);
1203 return res;
1206 if ( iommu_passthrough && (domain->domain_id == 0) )
1208 context_set_translation_type(*context, CONTEXT_TT_PASS_THRU);
1209 agaw = level_to_agaw(iommu->nr_pt_levels);
1211 else
1213 spin_lock(&hd->mapping_lock);
1215 /* Ensure we have pagetables allocated down to leaf PTE. */
1216 if ( hd->pgd_maddr == 0 )
1218 addr_to_dma_page_maddr(domain, 0, 1);
1219 if ( hd->pgd_maddr == 0 )
1221 nomem:
1222 spin_unlock(&hd->mapping_lock);
1223 spin_unlock(&iommu->lock);
1224 unmap_vtd_domain_page(context_entries);
1225 return -ENOMEM;
1229 /* Skip top levels of page tables for 2- and 3-level DRHDs. */
1230 pgd_maddr = hd->pgd_maddr;
1231 for ( agaw = level_to_agaw(4);
1232 agaw != level_to_agaw(iommu->nr_pt_levels);
1233 agaw-- )
1235 struct dma_pte *p = map_vtd_domain_page(pgd_maddr);
1236 pgd_maddr = dma_pte_addr(*p);
1237 unmap_vtd_domain_page(p);
1238 if ( pgd_maddr == 0 )
1239 goto nomem;
1242 context_set_address_root(*context, pgd_maddr);
1243 if ( ats_enabled && ecap_dev_iotlb(iommu->ecap) )
1244 context_set_translation_type(*context, CONTEXT_TT_DEV_IOTLB);
1245 else
1246 context_set_translation_type(*context, CONTEXT_TT_MULTI_LEVEL);
1248 spin_unlock(&hd->mapping_lock);
1251 if ( context_set_domain_id(context, domain, iommu) )
1253 spin_unlock(&iommu->lock);
1254 return -EFAULT;
1257 context_set_address_width(*context, agaw);
1258 context_set_fault_enable(*context);
1259 context_set_present(*context);
1260 iommu_flush_cache_entry(context, sizeof(struct context_entry));
1261 spin_unlock(&iommu->lock);
1263 /* Context entry was previously non-present (with domid 0). */
1264 if ( iommu_flush_context_device(iommu, 0, (((u16)bus) << 8) | devfn,
1265 DMA_CCMD_MASK_NOBIT, 1) )
1266 iommu_flush_write_buffer(iommu);
1267 else
1269 int flush_dev_iotlb = find_ats_dev_drhd(iommu) ? 1 : 0;
1270 iommu_flush_iotlb_dsi(iommu, 0, 1, flush_dev_iotlb);
1273 set_bit(iommu->index, &hd->iommu_bitmap);
1275 unmap_vtd_domain_page(context_entries);
1277 return 0;
1280 static int domain_context_mapping(struct domain *domain, u8 bus, u8 devfn)
1282 struct acpi_drhd_unit *drhd;
1283 int ret = 0;
1284 u32 type;
1285 u8 secbus;
1286 struct pci_dev *pdev = pci_get_pdev(bus, devfn);
1288 drhd = acpi_find_matched_drhd_unit(pdev);
1289 if ( !drhd )
1290 return -ENODEV;
1292 ASSERT(spin_is_locked(&pcidevs_lock));
1294 type = pdev_type(bus, devfn);
1295 switch ( type )
1297 case DEV_TYPE_PCIe_BRIDGE:
1298 case DEV_TYPE_PCIe2PCI_BRIDGE:
1299 case DEV_TYPE_LEGACY_PCI_BRIDGE:
1300 break;
1302 case DEV_TYPE_PCIe_ENDPOINT:
1303 gdprintk(XENLOG_INFO VTDPREFIX,
1304 "domain_context_mapping:PCIe: bdf = %x:%x.%x\n",
1305 bus, PCI_SLOT(devfn), PCI_FUNC(devfn));
1306 ret = domain_context_mapping_one(domain, drhd->iommu, bus, devfn);
1307 break;
1309 case DEV_TYPE_PCI:
1310 gdprintk(XENLOG_INFO VTDPREFIX,
1311 "domain_context_mapping:PCI: bdf = %x:%x.%x\n",
1312 bus, PCI_SLOT(devfn), PCI_FUNC(devfn));
1314 ret = domain_context_mapping_one(domain, drhd->iommu, bus, devfn);
1315 if ( ret )
1316 break;
1318 if ( find_upstream_bridge(&bus, &devfn, &secbus) < 1 )
1319 break;
1321 /* PCIe to PCI/PCIx bridge */
1322 if ( pdev_type(bus, devfn) == DEV_TYPE_PCIe2PCI_BRIDGE )
1324 ret = domain_context_mapping_one(domain, drhd->iommu, bus, devfn);
1325 if ( ret )
1326 return ret;
1328 /*
1329 * Devices behind PCIe-to-PCI/PCIx bridge may generate
1330 * different requester-id. It may originate from devfn=0
1331 * on the secondary bus behind the bridge. Map that id
1332 * as well.
1333 */
1334 ret = domain_context_mapping_one(domain, drhd->iommu, secbus, 0);
1336 else /* Legacy PCI bridge */
1337 ret = domain_context_mapping_one(domain, drhd->iommu, bus, devfn);
1339 break;
1341 default:
1342 gdprintk(XENLOG_ERR VTDPREFIX,
1343 "domain_context_mapping:unknown type : bdf = %x:%x.%x\n",
1344 bus, PCI_SLOT(devfn), PCI_FUNC(devfn));
1345 ret = -EINVAL;
1346 break;
1349 return ret;
1352 static int domain_context_unmap_one(
1353 struct domain *domain,
1354 struct iommu *iommu,
1355 u8 bus, u8 devfn)
1357 struct context_entry *context, *context_entries;
1358 u64 maddr;
1359 int iommu_domid;
1361 ASSERT(spin_is_locked(&pcidevs_lock));
1362 spin_lock(&iommu->lock);
1364 maddr = bus_to_context_maddr(iommu, bus);
1365 context_entries = (struct context_entry *)map_vtd_domain_page(maddr);
1366 context = &context_entries[devfn];
1368 if ( !context_present(*context) )
1370 spin_unlock(&iommu->lock);
1371 unmap_vtd_domain_page(context_entries);
1372 return 0;
1375 context_clear_present(*context);
1376 context_clear_entry(*context);
1377 iommu_flush_cache_entry(context, sizeof(struct context_entry));
1379 iommu_domid= domain_iommu_domid(domain, iommu);
1380 if ( iommu_domid == -1 )
1382 spin_unlock(&iommu->lock);
1383 unmap_vtd_domain_page(context_entries);
1384 return -EINVAL;
1387 if ( iommu_flush_context_device(iommu, iommu_domid,
1388 (((u16)bus) << 8) | devfn,
1389 DMA_CCMD_MASK_NOBIT, 0) )
1390 iommu_flush_write_buffer(iommu);
1391 else
1393 int flush_dev_iotlb = find_ats_dev_drhd(iommu) ? 1 : 0;
1394 iommu_flush_iotlb_dsi(iommu, iommu_domid, 0, flush_dev_iotlb);
1397 spin_unlock(&iommu->lock);
1398 unmap_vtd_domain_page(context_entries);
1400 return 0;
1403 static int domain_context_unmap(struct domain *domain, u8 bus, u8 devfn)
1405 struct acpi_drhd_unit *drhd;
1406 struct iommu *iommu;
1407 int ret = 0;
1408 u32 type;
1409 u8 tmp_bus, tmp_devfn, secbus;
1410 struct pci_dev *pdev = pci_get_pdev(bus, devfn);
1411 int found = 0;
1413 BUG_ON(!pdev);
1415 drhd = acpi_find_matched_drhd_unit(pdev);
1416 if ( !drhd )
1417 return -ENODEV;
1418 iommu = drhd->iommu;
1420 type = pdev_type(bus, devfn);
1421 switch ( type )
1423 case DEV_TYPE_PCIe_BRIDGE:
1424 case DEV_TYPE_PCIe2PCI_BRIDGE:
1425 case DEV_TYPE_LEGACY_PCI_BRIDGE:
1426 goto out;
1428 case DEV_TYPE_PCIe_ENDPOINT:
1429 gdprintk(XENLOG_INFO VTDPREFIX,
1430 "domain_context_unmap:PCIe: bdf = %x:%x.%x\n",
1431 bus, PCI_SLOT(devfn), PCI_FUNC(devfn));
1432 ret = domain_context_unmap_one(domain, iommu, bus, devfn);
1433 break;
1435 case DEV_TYPE_PCI:
1436 gdprintk(XENLOG_INFO VTDPREFIX,
1437 "domain_context_unmap:PCI: bdf = %x:%x.%x\n",
1438 bus, PCI_SLOT(devfn), PCI_FUNC(devfn));
1439 ret = domain_context_unmap_one(domain, iommu, bus, devfn);
1440 if ( ret )
1441 break;
1443 tmp_bus = bus;
1444 tmp_devfn = devfn;
1445 if ( find_upstream_bridge(&tmp_bus, &tmp_devfn, &secbus) < 1 )
1446 break;
1448 /* PCIe to PCI/PCIx bridge */
1449 if ( pdev_type(tmp_bus, tmp_devfn) == DEV_TYPE_PCIe2PCI_BRIDGE )
1451 ret = domain_context_unmap_one(domain, iommu, tmp_bus, tmp_devfn);
1452 if ( ret )
1453 return ret;
1455 ret = domain_context_unmap_one(domain, iommu, secbus, 0);
1457 else /* Legacy PCI bridge */
1458 ret = domain_context_unmap_one(domain, iommu, tmp_bus, tmp_devfn);
1460 break;
1462 default:
1463 gdprintk(XENLOG_ERR VTDPREFIX,
1464 "domain_context_unmap:unknown type: bdf = %x:%x.%x\n",
1465 bus, PCI_SLOT(devfn), PCI_FUNC(devfn));
1466 ret = -EINVAL;
1467 goto out;
1470 /*
1471 * if no other devices under the same iommu owned by this domain,
1472 * clear iommu in iommu_bitmap and clear domain_id in domid_bitmp
1473 */
1474 for_each_pdev ( domain, pdev )
1476 if ( pdev->bus == bus && pdev->devfn == devfn )
1477 continue;
1479 drhd = acpi_find_matched_drhd_unit(pdev);
1480 if ( drhd && drhd->iommu == iommu )
1482 found = 1;
1483 break;
1487 if ( found == 0 )
1489 struct hvm_iommu *hd = domain_hvm_iommu(domain);
1490 int iommu_domid;
1492 clear_bit(iommu->index, &hd->iommu_bitmap);
1494 iommu_domid = domain_iommu_domid(domain, iommu);
1495 if ( iommu_domid == -1 )
1497 ret = -EINVAL;
1498 goto out;
1501 clear_bit(iommu_domid, iommu->domid_bitmap);
1502 iommu->domid_map[iommu_domid] = 0;
1505 out:
1506 return ret;
1509 static int reassign_device_ownership(
1510 struct domain *source,
1511 struct domain *target,
1512 u8 bus, u8 devfn)
1514 struct pci_dev *pdev;
1515 int ret;
1517 ASSERT(spin_is_locked(&pcidevs_lock));
1518 pdev = pci_get_pdev_by_domain(source, bus, devfn);
1520 if (!pdev)
1521 return -ENODEV;
1523 ret = domain_context_unmap(source, bus, devfn);
1524 if ( ret )
1525 return ret;
1527 ret = domain_context_mapping(target, bus, devfn);
1528 if ( ret )
1529 return ret;
1531 list_move(&pdev->domain_list, &target->arch.pdev_list);
1532 pdev->domain = target;
1534 return ret;
1537 void iommu_domain_teardown(struct domain *d)
1539 struct hvm_iommu *hd = domain_hvm_iommu(d);
1541 if ( list_empty(&acpi_drhd_units) )
1542 return;
1544 spin_lock(&hd->mapping_lock);
1545 iommu_free_pagetable(hd->pgd_maddr, agaw_to_level(hd->agaw));
1546 hd->pgd_maddr = 0;
1547 spin_unlock(&hd->mapping_lock);
1550 static int intel_iommu_map_page(
1551 struct domain *d, unsigned long gfn, unsigned long mfn)
1553 struct hvm_iommu *hd = domain_hvm_iommu(d);
1554 struct acpi_drhd_unit *drhd;
1555 struct iommu *iommu;
1556 struct dma_pte *page = NULL, *pte = NULL;
1557 u64 pg_maddr;
1558 int pte_present;
1559 int flush_dev_iotlb;
1560 int iommu_domid;
1562 /* do nothing if dom0 and iommu supports pass thru */
1563 if ( iommu_passthrough && (d->domain_id == 0) )
1564 return 0;
1566 spin_lock(&hd->mapping_lock);
1568 pg_maddr = addr_to_dma_page_maddr(d, (paddr_t)gfn << PAGE_SHIFT_4K, 1);
1569 if ( pg_maddr == 0 )
1571 spin_unlock(&hd->mapping_lock);
1572 return -ENOMEM;
1574 page = (struct dma_pte *)map_vtd_domain_page(pg_maddr);
1575 pte = page + (gfn & LEVEL_MASK);
1576 pte_present = dma_pte_present(*pte);
1577 dma_set_pte_addr(*pte, (paddr_t)mfn << PAGE_SHIFT_4K);
1578 dma_set_pte_prot(*pte, DMA_PTE_READ | DMA_PTE_WRITE);
1580 /* Set the SNP on leaf page table if Snoop Control available */
1581 if ( iommu_snoop )
1582 dma_set_pte_snp(*pte);
1584 iommu_flush_cache_entry(pte, sizeof(struct dma_pte));
1585 spin_unlock(&hd->mapping_lock);
1586 unmap_vtd_domain_page(page);
1588 /*
1589 * No need pcideves_lock here because we have flush
1590 * when assign/deassign device
1591 */
1592 for_each_drhd_unit ( drhd )
1594 iommu = drhd->iommu;
1596 if ( !test_bit(iommu->index, &hd->iommu_bitmap) )
1597 continue;
1599 flush_dev_iotlb = find_ats_dev_drhd(iommu) ? 1 : 0;
1600 iommu_domid= domain_iommu_domid(d, iommu);
1601 if ( iommu_domid == -1 )
1602 continue;
1603 if ( iommu_flush_iotlb_psi(iommu, iommu_domid,
1604 (paddr_t)gfn << PAGE_SHIFT_4K, 1,
1605 !pte_present, flush_dev_iotlb) )
1606 iommu_flush_write_buffer(iommu);
1609 return 0;
1612 static int intel_iommu_unmap_page(struct domain *d, unsigned long gfn)
1614 /* Do nothing if dom0 and iommu supports pass thru. */
1615 if ( iommu_passthrough && (d->domain_id == 0) )
1616 return 0;
1618 dma_pte_clear_one(d, (paddr_t)gfn << PAGE_SHIFT_4K);
1620 return 0;
1623 static int domain_rmrr_mapped(struct domain *d,
1624 struct acpi_rmrr_unit *rmrr)
1626 struct hvm_iommu *hd = domain_hvm_iommu(d);
1627 struct mapped_rmrr *mrmrr;
1629 list_for_each_entry( mrmrr, &hd->mapped_rmrrs, list )
1631 if ( mrmrr->base == rmrr->base_address &&
1632 mrmrr->end == rmrr->end_address )
1633 return 1;
1636 return 0;
1639 static int rmrr_identity_mapping(struct domain *d,
1640 struct acpi_rmrr_unit *rmrr)
1642 u64 base, end;
1643 unsigned long base_pfn, end_pfn;
1644 struct mapped_rmrr *mrmrr;
1645 struct hvm_iommu *hd = domain_hvm_iommu(d);
1647 ASSERT(spin_is_locked(&pcidevs_lock));
1648 ASSERT(rmrr->base_address < rmrr->end_address);
1650 if ( domain_rmrr_mapped(d, rmrr) )
1651 return 0;
1653 base = rmrr->base_address & PAGE_MASK_4K;
1654 base_pfn = base >> PAGE_SHIFT_4K;
1655 end = PAGE_ALIGN_4K(rmrr->end_address);
1656 end_pfn = end >> PAGE_SHIFT_4K;
1658 while ( base_pfn < end_pfn )
1660 if ( intel_iommu_map_page(d, base_pfn, base_pfn) )
1661 return -1;
1662 base_pfn++;
1665 mrmrr = xmalloc(struct mapped_rmrr);
1666 if ( !mrmrr )
1667 return -ENOMEM;
1668 mrmrr->base = rmrr->base_address;
1669 mrmrr->end = rmrr->end_address;
1670 list_add_tail(&mrmrr->list, &hd->mapped_rmrrs);
1672 return 0;
1675 static int intel_iommu_add_device(struct pci_dev *pdev)
1677 struct acpi_rmrr_unit *rmrr;
1678 u16 bdf;
1679 int ret, i;
1681 ASSERT(spin_is_locked(&pcidevs_lock));
1683 if ( !pdev->domain )
1684 return -EINVAL;
1686 ret = domain_context_mapping(pdev->domain, pdev->bus, pdev->devfn);
1687 if ( ret )
1689 gdprintk(XENLOG_ERR VTDPREFIX,
1690 "intel_iommu_add_device: context mapping failed\n");
1691 return ret;
1694 for_each_rmrr_device ( rmrr, bdf, i )
1696 if ( PCI_BUS(bdf) == pdev->bus && PCI_DEVFN2(bdf) == pdev->devfn )
1698 ret = rmrr_identity_mapping(pdev->domain, rmrr);
1699 if ( ret )
1700 gdprintk(XENLOG_ERR VTDPREFIX,
1701 "intel_iommu_add_device: RMRR mapping failed\n");
1705 return ret;
1708 static int intel_iommu_remove_device(struct pci_dev *pdev)
1710 struct acpi_rmrr_unit *rmrr;
1711 u16 bdf;
1712 int i;
1714 if ( !pdev->domain )
1715 return -EINVAL;
1717 /* If the device belongs to dom0, and it has RMRR, don't remove it
1718 * from dom0, because BIOS may use RMRR at booting time.
1719 */
1720 if ( pdev->domain->domain_id == 0 )
1722 for_each_rmrr_device ( rmrr, bdf, i )
1724 if ( PCI_BUS(bdf) == pdev->bus &&
1725 PCI_DEVFN2(bdf) == pdev->devfn )
1726 return 0;
1730 return domain_context_unmap(pdev->domain, pdev->bus, pdev->devfn);
1733 static void setup_dom0_devices(struct domain *d)
1735 struct pci_dev *pdev;
1736 int bus, devfn;
1738 spin_lock(&pcidevs_lock);
1739 for ( bus = 0; bus < 256; bus++ )
1741 for ( devfn = 0; devfn < 256; devfn++ )
1743 pdev = pci_get_pdev(bus, devfn);
1744 if ( !pdev )
1745 continue;
1747 pdev->domain = d;
1748 list_add(&pdev->domain_list, &d->arch.pdev_list);
1749 domain_context_mapping(d, pdev->bus, pdev->devfn);
1750 pci_enable_acs(pdev);
1751 if ( ats_device(0, pdev->bus, pdev->devfn) )
1752 enable_ats_device(0, pdev->bus, pdev->devfn);
1755 spin_unlock(&pcidevs_lock);
1758 void clear_fault_bits(struct iommu *iommu)
1760 u64 val;
1761 unsigned long flags;
1763 spin_lock_irqsave(&iommu->register_lock, flags);
1764 val = dmar_readq(
1765 iommu->reg,
1766 cap_fault_reg_offset(dmar_readq(iommu->reg,DMAR_CAP_REG))+0x8);
1767 dmar_writeq(
1768 iommu->reg,
1769 cap_fault_reg_offset(dmar_readq(iommu->reg,DMAR_CAP_REG))+8,
1770 val);
1771 dmar_writel(iommu->reg, DMAR_FSTS_REG, DMA_FSTS_FAULTS);
1772 spin_unlock_irqrestore(&iommu->register_lock, flags);
1775 static int init_vtd_hw(void)
1777 struct acpi_drhd_unit *drhd;
1778 struct iommu *iommu;
1779 struct iommu_flush *flush = NULL;
1780 int irq;
1781 int ret;
1782 unsigned long flags;
1783 struct irq_cfg *cfg;
1785 for_each_drhd_unit ( drhd )
1787 iommu = drhd->iommu;
1788 if ( iommu->irq < 0 )
1790 irq = iommu_set_interrupt(iommu);
1791 if ( irq < 0 )
1793 dprintk(XENLOG_ERR VTDPREFIX, "IOMMU: interrupt setup failed\n");
1794 return irq;
1796 iommu->irq = irq;
1799 cfg = irq_cfg(iommu->irq);
1800 dma_msi_set_affinity(iommu->irq, cfg->domain);
1802 clear_fault_bits(iommu);
1804 spin_lock_irqsave(&iommu->register_lock, flags);
1805 dmar_writel(iommu->reg, DMAR_FECTL_REG, 0);
1806 spin_unlock_irqrestore(&iommu->register_lock, flags);
1808 /* initialize flush functions */
1809 flush = iommu_get_flush(iommu);
1810 flush->context = flush_context_reg;
1811 flush->iotlb = flush_iotlb_reg;
1814 if ( iommu_qinval )
1816 for_each_drhd_unit ( drhd )
1818 iommu = drhd->iommu;
1819 if ( enable_qinval(iommu) != 0 )
1821 dprintk(XENLOG_INFO VTDPREFIX,
1822 "Failed to enable Queued Invalidation!\n");
1823 break;
1828 if ( iommu_intremap )
1830 for_each_drhd_unit ( drhd )
1832 iommu = drhd->iommu;
1833 if ( enable_intremap(iommu) != 0 )
1835 dprintk(XENLOG_INFO VTDPREFIX,
1836 "Failed to enable Interrupt Remapping!\n");
1837 break;
1842 for_each_drhd_unit ( drhd )
1844 iommu = drhd->iommu;
1845 ret = iommu_set_root_entry(iommu);
1846 if ( ret )
1848 dprintk(XENLOG_ERR VTDPREFIX, "IOMMU: set root entry failed\n");
1849 return -EIO;
1853 /*
1854 * After set root entry, must globally invalidate context cache, and
1855 * then globally invalidate IOTLB
1856 */
1857 iommu_flush_all();
1859 return 0;
1862 static void setup_dom0_rmrr(struct domain *d)
1864 struct acpi_rmrr_unit *rmrr;
1865 u16 bdf;
1866 int ret, i;
1868 spin_lock(&pcidevs_lock);
1869 for_each_rmrr_device ( rmrr, bdf, i )
1871 ret = rmrr_identity_mapping(d, rmrr);
1872 if ( ret )
1873 dprintk(XENLOG_ERR VTDPREFIX,
1874 "IOMMU: mapping reserved region failed\n");
1876 spin_unlock(&pcidevs_lock);
1879 static void platform_quirks(void)
1881 u32 id;
1883 /* Mobile 4 Series Chipset neglects to set RWBF capability. */
1884 id = pci_conf_read32(0, 0, 0, 0);
1885 if ( id == 0x2a408086 )
1887 dprintk(XENLOG_INFO VTDPREFIX, "DMAR: Forcing write-buffer flush\n");
1888 rwbf_quirk = 1;
1892 int intel_vtd_setup(void)
1894 struct acpi_drhd_unit *drhd;
1895 struct iommu *iommu;
1897 if ( list_empty(&acpi_drhd_units) )
1898 return -ENODEV;
1900 platform_quirks();
1902 clflush_size = get_cache_line_size();
1904 irq_to_iommu = xmalloc_array(struct iommu*, nr_irqs);
1905 BUG_ON(!irq_to_iommu);
1906 memset(irq_to_iommu, 0, nr_irqs * sizeof(struct iommu*));
1908 if(!irq_to_iommu)
1909 return -ENOMEM;
1911 /* We enable the following features only if they are supported by all VT-d
1912 * engines: Snoop Control, DMA passthrough, Queued Invalidation and
1913 * Interrupt Remapping.
1914 */
1915 for_each_drhd_unit ( drhd )
1917 if ( iommu_alloc(drhd) != 0 )
1918 goto error;
1920 iommu = drhd->iommu;
1922 if ( iommu_snoop && !ecap_snp_ctl(iommu->ecap) )
1923 iommu_snoop = 0;
1925 if ( iommu_passthrough && !ecap_pass_thru(iommu->ecap) )
1926 iommu_passthrough = 0;
1928 if ( iommu_qinval && !ecap_queued_inval(iommu->ecap) )
1929 iommu_qinval = 0;
1931 if ( iommu_intremap && !ecap_intr_remap(iommu->ecap) )
1932 iommu_intremap = 0;
1935 if ( !iommu_qinval && iommu_intremap )
1937 iommu_intremap = 0;
1938 dprintk(XENLOG_WARNING VTDPREFIX, "Interrupt Remapping disabled "
1939 "since Queued Invalidation isn't supported or enabled.\n");
1942 #define P(p,s) printk("Intel VT-d %s %ssupported.\n", s, (p)? "" : "not ")
1943 P(iommu_snoop, "Snoop Control");
1944 P(iommu_passthrough, "DMA Passthrough");
1945 P(iommu_qinval, "Queued Invalidation");
1946 P(iommu_intremap, "Interrupt Remapping");
1947 #undef P
1949 scan_pci_devices();
1951 if ( init_vtd_hw() )
1952 goto error;
1954 register_keyhandler('V', &dump_iommu_info_keyhandler);
1956 return 0;
1958 error:
1959 for_each_drhd_unit ( drhd )
1960 iommu_free(drhd);
1961 iommu_enabled = 0;
1962 iommu_snoop = 0;
1963 iommu_passthrough = 0;
1964 iommu_qinval = 0;
1965 iommu_intremap = 0;
1966 return -ENOMEM;
1969 /*
1970 * If the device isn't owned by dom0, it means it already
1971 * has been assigned to other domain, or it's not exist.
1972 */
1973 int device_assigned(u8 bus, u8 devfn)
1975 struct pci_dev *pdev;
1977 spin_lock(&pcidevs_lock);
1978 pdev = pci_get_pdev_by_domain(dom0, bus, devfn);
1979 if (!pdev)
1981 spin_unlock(&pcidevs_lock);
1982 return -1;
1985 spin_unlock(&pcidevs_lock);
1986 return 0;
1989 static int intel_iommu_assign_device(struct domain *d, u8 bus, u8 devfn)
1991 struct acpi_rmrr_unit *rmrr;
1992 int ret = 0, i;
1993 struct pci_dev *pdev;
1994 u16 bdf;
1996 if ( list_empty(&acpi_drhd_units) )
1997 return -ENODEV;
1999 ASSERT(spin_is_locked(&pcidevs_lock));
2000 pdev = pci_get_pdev(bus, devfn);
2001 if (!pdev)
2002 return -ENODEV;
2004 if (pdev->domain != dom0)
2006 gdprintk(XENLOG_ERR VTDPREFIX,
2007 "IOMMU: assign a assigned device\n");
2008 return -EBUSY;
2011 ret = reassign_device_ownership(dom0, d, bus, devfn);
2012 if ( ret )
2013 goto done;
2015 /* FIXME: Because USB RMRR conflicts with guest bios region,
2016 * ignore USB RMRR temporarily.
2017 */
2018 if ( is_usb_device(bus, devfn) )
2020 ret = 0;
2021 goto done;
2024 /* Setup rmrr identity mapping */
2025 for_each_rmrr_device( rmrr, bdf, i )
2027 if ( PCI_BUS(bdf) == bus && PCI_DEVFN2(bdf) == devfn )
2029 ret = rmrr_identity_mapping(d, rmrr);
2030 if ( ret )
2032 gdprintk(XENLOG_ERR VTDPREFIX,
2033 "IOMMU: mapping reserved region failed\n");
2034 goto done;
2039 done:
2040 return ret;
2043 static int intel_iommu_group_id(u8 bus, u8 devfn)
2045 u8 secbus;
2046 if ( find_upstream_bridge(&bus, &devfn, &secbus) < 0 )
2047 return -1;
2048 else
2049 return PCI_BDF2(bus, devfn);
2052 static u32 iommu_state[MAX_IOMMUS][MAX_IOMMU_REGS];
2053 static void vtd_suspend(void)
2055 struct acpi_drhd_unit *drhd;
2056 struct iommu *iommu;
2057 u32 i;
2059 if ( !iommu_enabled )
2060 return;
2062 iommu_flush_all();
2064 for_each_drhd_unit ( drhd )
2066 iommu = drhd->iommu;
2067 i = iommu->index;
2069 iommu_state[i][DMAR_FECTL_REG] =
2070 (u32) dmar_readl(iommu->reg, DMAR_FECTL_REG);
2071 iommu_state[i][DMAR_FEDATA_REG] =
2072 (u32) dmar_readl(iommu->reg, DMAR_FEDATA_REG);
2073 iommu_state[i][DMAR_FEADDR_REG] =
2074 (u32) dmar_readl(iommu->reg, DMAR_FEADDR_REG);
2075 iommu_state[i][DMAR_FEUADDR_REG] =
2076 (u32) dmar_readl(iommu->reg, DMAR_FEUADDR_REG);
2078 /* don't disable VT-d engine when force_iommu is set. */
2079 if ( force_iommu )
2080 continue;
2082 iommu_disable_translation(iommu);
2084 if ( iommu_intremap )
2085 disable_intremap(iommu);
2087 if ( iommu_qinval )
2088 disable_qinval(iommu);
2092 static void vtd_resume(void)
2094 struct acpi_drhd_unit *drhd;
2095 struct iommu *iommu;
2096 u32 i;
2097 unsigned long flags;
2099 if ( !iommu_enabled )
2100 return;
2102 if ( init_vtd_hw() != 0 && force_iommu )
2103 panic("IOMMU setup failed, crash Xen for security purpose!\n");
2105 for_each_drhd_unit ( drhd )
2107 iommu = drhd->iommu;
2108 i = iommu->index;
2110 spin_lock_irqsave(&iommu->register_lock, flags);
2111 dmar_writel(iommu->reg, DMAR_FECTL_REG,
2112 (u32) iommu_state[i][DMAR_FECTL_REG]);
2113 dmar_writel(iommu->reg, DMAR_FEDATA_REG,
2114 (u32) iommu_state[i][DMAR_FEDATA_REG]);
2115 dmar_writel(iommu->reg, DMAR_FEADDR_REG,
2116 (u32) iommu_state[i][DMAR_FEADDR_REG]);
2117 dmar_writel(iommu->reg, DMAR_FEUADDR_REG,
2118 (u32) iommu_state[i][DMAR_FEUADDR_REG]);
2119 spin_unlock_irqrestore(&iommu->register_lock, flags);
2121 iommu_enable_translation(iommu);
2125 const struct iommu_ops intel_iommu_ops = {
2126 .init = intel_iommu_domain_init,
2127 .add_device = intel_iommu_add_device,
2128 .remove_device = intel_iommu_remove_device,
2129 .assign_device = intel_iommu_assign_device,
2130 .teardown = iommu_domain_teardown,
2131 .map_page = intel_iommu_map_page,
2132 .unmap_page = intel_iommu_unmap_page,
2133 .reassign_device = reassign_device_ownership,
2134 .get_device_group_id = intel_iommu_group_id,
2135 .update_ire_from_apic = io_apic_write_remap_rte,
2136 .update_ire_from_msi = msi_msg_write_remap_rte,
2137 .read_apic_from_ire = io_apic_read_remap_rte,
2138 .read_msi_from_ire = msi_msg_read_remap_rte,
2139 .suspend = vtd_suspend,
2140 .resume = vtd_resume,
2141 };
2143 /*
2144 * Local variables:
2145 * mode: C
2146 * c-set-style: "BSD"
2147 * c-basic-offset: 4
2148 * tab-width: 4
2149 * indent-tabs-mode: nil
2150 * End:
2151 */