debuggers.hg

view xen/arch/x86/hvm/vmx/vtd/intel-iommu.c @ 16366:b544448502a4

vt-d: Disable PMR on every vt-d engine.

Signed-off-by Gang Wei (Jimmy) <gang.wei@intel.com>
Signed-off-by Kevin Tian <kevin.tian@intel.com>
author Keir Fraser <keir@xensource.com>
date Wed Nov 07 09:13:48 2007 +0000 (2007-11-07)
parents 46a7c9a15b0f
children cb6675149af8
line source
1 /*
2 * Copyright (c) 2006, Intel Corporation.
3 *
4 * This program is free software; you can redistribute it and/or modify it
5 * under the terms and conditions of the GNU General Public License,
6 * version 2, as published by the Free Software Foundation.
7 *
8 * This program is distributed in the hope it will be useful, but WITHOUT
9 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
10 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
11 * more details.
12 *
13 * You should have received a copy of the GNU General Public License along with
14 * this program; if not, write to the Free Software Foundation, Inc., 59 Temple
15 * Place - Suite 330, Boston, MA 02111-1307 USA.
16 *
17 * Copyright (C) Ashok Raj <ashok.raj@intel.com>
18 * Copyright (C) Shaohua Li <shaohua.li@intel.com>
19 * Copyright (C) Allen Kay <allen.m.kay@intel.com> - adapted to xen
20 */
22 #include <xen/init.h>
23 #include <xen/irq.h>
24 #include <xen/spinlock.h>
25 #include <xen/sched.h>
26 #include <xen/xmalloc.h>
27 #include <xen/domain_page.h>
28 #include <asm/delay.h>
29 #include <asm/string.h>
30 #include <asm/mm.h>
31 #include <asm/iommu.h>
32 #include <asm/hvm/vmx/intel-iommu.h>
33 #include "dmar.h"
34 #include "pci-direct.h"
35 #include "pci_regs.h"
36 #include "msi.h"
38 #define VTDPREFIX
39 extern void print_iommu_regs(struct acpi_drhd_unit *drhd);
40 extern void print_vtd_entries(struct domain *d, int bus, int devfn,
41 unsigned long gmfn);
43 unsigned int x86_clflush_size;
44 void clflush_cache_range(void *adr, int size)
45 {
46 int i;
47 for ( i = 0; i < size; i += x86_clflush_size )
48 clflush(adr + i);
49 }
51 static void __iommu_flush_cache(struct iommu *iommu, void *addr, int size)
52 {
53 if ( !ecap_coherent(iommu->ecap) )
54 clflush_cache_range(addr, size);
55 }
57 #define iommu_flush_cache_entry(iommu, addr) \
58 __iommu_flush_cache(iommu, addr, 8)
59 #define iommu_flush_cache_page(iommu, addr) \
60 __iommu_flush_cache(iommu, addr, PAGE_SIZE_4K)
62 int nr_iommus;
63 /* context entry handling */
64 static struct context_entry * device_to_context_entry(struct iommu *iommu,
65 u8 bus, u8 devfn)
66 {
67 struct root_entry *root;
68 struct context_entry *context;
69 unsigned long phy_addr;
70 unsigned long flags;
72 spin_lock_irqsave(&iommu->lock, flags);
73 root = &iommu->root_entry[bus];
74 if ( !root_present(*root) )
75 {
76 phy_addr = (unsigned long) alloc_xenheap_page();
77 if ( !phy_addr )
78 {
79 spin_unlock_irqrestore(&iommu->lock, flags);
80 return NULL;
81 }
82 memset((void *) phy_addr, 0, PAGE_SIZE);
83 iommu_flush_cache_page(iommu, (void *)phy_addr);
84 phy_addr = virt_to_maddr((void *)phy_addr);
85 set_root_value(*root, phy_addr);
86 set_root_present(*root);
87 iommu_flush_cache_entry(iommu, root);
88 }
89 phy_addr = (unsigned long) get_context_addr(*root);
90 context = (struct context_entry *)maddr_to_virt(phy_addr);
91 spin_unlock_irqrestore(&iommu->lock, flags);
92 return &context[devfn];
93 }
95 static int device_context_mapped(struct iommu *iommu, u8 bus, u8 devfn)
96 {
97 struct root_entry *root;
98 struct context_entry *context;
99 unsigned long phy_addr;
100 int ret;
101 unsigned long flags;
103 spin_lock_irqsave(&iommu->lock, flags);
104 root = &iommu->root_entry[bus];
105 if ( !root_present(*root) )
106 {
107 ret = 0;
108 goto out;
109 }
110 phy_addr = get_context_addr(*root);
111 context = (struct context_entry *)maddr_to_virt(phy_addr);
112 ret = context_present(context[devfn]);
113 out:
114 spin_unlock_irqrestore(&iommu->lock, flags);
115 return ret;
116 }
118 /* page table handling */
119 #define LEVEL_STRIDE (9)
120 #define LEVEL_MASK ((1 << LEVEL_STRIDE) - 1)
121 #define agaw_to_level(val) ((val) + 2)
122 #define agaw_to_width(val) (30 + val * LEVEL_STRIDE)
123 #define width_to_agaw(w) ((w - 30)/LEVEL_STRIDE)
124 #define level_to_offset_bits(l) (12 + (l - 1) * LEVEL_STRIDE)
125 #define address_level_offset(addr, level) \
126 ((addr >> level_to_offset_bits(level)) & LEVEL_MASK)
127 #define level_mask(l) (((u64)(-1)) << level_to_offset_bits(l))
128 #define level_size(l) (1 << level_to_offset_bits(l))
129 #define align_to_level(addr, l) ((addr + level_size(l) - 1) & level_mask(l))
130 static struct page_info *addr_to_dma_page(struct domain *domain, u64 addr)
131 {
132 struct hvm_iommu *hd = domain_hvm_iommu(domain);
133 struct acpi_drhd_unit *drhd;
134 struct iommu *iommu;
135 int addr_width = agaw_to_width(hd->agaw);
136 struct dma_pte *parent, *pte = NULL, *pgd;
137 int level = agaw_to_level(hd->agaw);
138 int offset;
139 unsigned long flags;
140 struct page_info *pg = NULL;
141 u64 *vaddr = NULL;
143 drhd = list_entry(acpi_drhd_units.next, typeof(*drhd), list);
144 iommu = drhd->iommu;
146 addr &= (((u64)1) << addr_width) - 1;
147 spin_lock_irqsave(&hd->mapping_lock, flags);
148 if ( !hd->pgd )
149 {
150 pgd = (struct dma_pte *)alloc_xenheap_page();
151 if ( !pgd )
152 {
153 spin_unlock_irqrestore(&hd->mapping_lock, flags);
154 return NULL;
155 }
156 memset(pgd, 0, PAGE_SIZE);
157 hd->pgd = pgd;
158 }
160 parent = hd->pgd;
161 while ( level > 1 )
162 {
163 offset = address_level_offset(addr, level);
164 pte = &parent[offset];
166 if ( dma_pte_addr(*pte) == 0 )
167 {
168 pg = alloc_domheap_page(NULL);
169 vaddr = map_domain_page(page_to_mfn(pg));
170 if ( !vaddr )
171 {
172 spin_unlock_irqrestore(&hd->mapping_lock, flags);
173 return NULL;
174 }
175 memset(vaddr, 0, PAGE_SIZE);
176 iommu_flush_cache_page(iommu, vaddr);
178 dma_set_pte_addr(*pte, page_to_maddr(pg));
180 /*
181 * high level table always sets r/w, last level
182 * page table control read/write
183 */
184 dma_set_pte_readable(*pte);
185 dma_set_pte_writable(*pte);
186 iommu_flush_cache_entry(iommu, pte);
187 }
188 else
189 {
190 pg = maddr_to_page(pte->val);
191 vaddr = map_domain_page(page_to_mfn(pg));
192 if ( !vaddr )
193 {
194 spin_unlock_irqrestore(&hd->mapping_lock, flags);
195 return NULL;
196 }
197 }
199 if ( parent != hd->pgd )
200 unmap_domain_page(parent);
202 if ( level == 2 && vaddr )
203 {
204 unmap_domain_page(vaddr);
205 break;
206 }
208 parent = (struct dma_pte *)vaddr;
209 vaddr = NULL;
210 level--;
211 }
213 spin_unlock_irqrestore(&hd->mapping_lock, flags);
214 return pg;
215 }
217 /* return address's page at specific level */
218 static struct page_info *dma_addr_level_page(struct domain *domain,
219 u64 addr, int level)
220 {
221 struct hvm_iommu *hd = domain_hvm_iommu(domain);
222 struct dma_pte *parent, *pte = NULL;
223 int total = agaw_to_level(hd->agaw);
224 int offset;
225 struct page_info *pg = NULL;
227 parent = hd->pgd;
228 while ( level <= total )
229 {
230 offset = address_level_offset(addr, total);
231 pte = &parent[offset];
232 if ( dma_pte_addr(*pte) == 0 )
233 {
234 if ( parent != hd->pgd )
235 unmap_domain_page(parent);
236 break;
237 }
239 pg = maddr_to_page(pte->val);
240 if ( parent != hd->pgd )
241 unmap_domain_page(parent);
243 if ( level == total )
244 return pg;
246 parent = map_domain_page(page_to_mfn(pg));
247 total--;
248 }
250 return NULL;
251 }
253 static void iommu_flush_write_buffer(struct iommu *iommu)
254 {
255 u32 val;
256 unsigned long flag;
257 unsigned long start_time;
259 if ( !cap_rwbf(iommu->cap) )
260 return;
261 val = iommu->gcmd | DMA_GCMD_WBF;
263 spin_lock_irqsave(&iommu->register_lock, flag);
264 dmar_writel(iommu->reg, DMAR_GCMD_REG, val);
266 /* Make sure hardware complete it */
267 start_time = jiffies;
268 for ( ; ; )
269 {
270 val = dmar_readl(iommu->reg, DMAR_GSTS_REG);
271 if ( !(val & DMA_GSTS_WBFS) )
272 break;
273 if ( time_after(jiffies, start_time + DMAR_OPERATION_TIMEOUT) )
274 panic("DMAR hardware is malfunctional,"
275 " please disable IOMMU\n");
276 cpu_relax();
277 }
278 spin_unlock_irqrestore(&iommu->register_lock, flag);
279 }
281 /* return value determine if we need a write buffer flush */
282 static int __iommu_flush_context(
283 struct iommu *iommu,
284 u16 did, u16 source_id, u8 function_mask, u64 type,
285 int non_present_entry_flush)
286 {
287 u64 val = 0;
288 unsigned long flag;
289 unsigned long start_time;
291 /*
292 * In the non-present entry flush case, if hardware doesn't cache
293 * non-present entry we do nothing and if hardware cache non-present
294 * entry, we flush entries of domain 0 (the domain id is used to cache
295 * any non-present entries)
296 */
297 if ( non_present_entry_flush )
298 {
299 if ( !cap_caching_mode(iommu->cap) )
300 return 1;
301 else
302 did = 0;
303 }
305 /* use register invalidation */
306 switch ( type )
307 {
308 case DMA_CCMD_GLOBAL_INVL:
309 val = DMA_CCMD_GLOBAL_INVL;
310 break;
311 case DMA_CCMD_DOMAIN_INVL:
312 val = DMA_CCMD_DOMAIN_INVL|DMA_CCMD_DID(did);
313 break;
314 case DMA_CCMD_DEVICE_INVL:
315 val = DMA_CCMD_DEVICE_INVL|DMA_CCMD_DID(did)
316 |DMA_CCMD_SID(source_id)|DMA_CCMD_FM(function_mask);
317 break;
318 default:
319 BUG();
320 }
321 val |= DMA_CCMD_ICC;
323 spin_lock_irqsave(&iommu->register_lock, flag);
324 dmar_writeq(iommu->reg, DMAR_CCMD_REG, val);
326 /* Make sure hardware complete it */
327 start_time = jiffies;
328 for ( ; ; )
329 {
330 val = dmar_readq(iommu->reg, DMAR_CCMD_REG);
331 if ( !(val & DMA_CCMD_ICC) )
332 break;
333 if ( time_after(jiffies, start_time + DMAR_OPERATION_TIMEOUT) )
334 panic("DMAR hardware is malfunctional, please disable IOMMU\n");
335 cpu_relax();
336 }
337 spin_unlock_irqrestore(&iommu->register_lock, flag);
338 /* flush context entry will implictly flush write buffer */
339 return 0;
340 }
342 static int inline iommu_flush_context_global(
343 struct iommu *iommu, int non_present_entry_flush)
344 {
345 return __iommu_flush_context(iommu, 0, 0, 0, DMA_CCMD_GLOBAL_INVL,
346 non_present_entry_flush);
347 }
349 static int inline iommu_flush_context_domain(
350 struct iommu *iommu, u16 did, int non_present_entry_flush)
351 {
352 return __iommu_flush_context(iommu, did, 0, 0, DMA_CCMD_DOMAIN_INVL,
353 non_present_entry_flush);
354 }
356 static int inline iommu_flush_context_device(
357 struct iommu *iommu, u16 did, u16 source_id,
358 u8 function_mask, int non_present_entry_flush)
359 {
360 return __iommu_flush_context(iommu, did, source_id, function_mask,
361 DMA_CCMD_DEVICE_INVL,
362 non_present_entry_flush);
363 }
365 /* return value determine if we need a write buffer flush */
366 static int __iommu_flush_iotlb(struct iommu *iommu, u16 did,
367 u64 addr, unsigned int size_order, u64 type,
368 int non_present_entry_flush)
369 {
370 int tlb_offset = ecap_iotlb_offset(iommu->ecap);
371 u64 val = 0, val_iva = 0;
372 unsigned long flag;
373 unsigned long start_time;
375 /*
376 * In the non-present entry flush case, if hardware doesn't cache
377 * non-present entry we do nothing and if hardware cache non-present
378 * entry, we flush entries of domain 0 (the domain id is used to cache
379 * any non-present entries)
380 */
381 if ( non_present_entry_flush )
382 {
383 if ( !cap_caching_mode(iommu->cap) )
384 return 1;
385 else
386 did = 0;
387 }
389 /* use register invalidation */
390 switch ( type )
391 {
392 case DMA_TLB_GLOBAL_FLUSH:
393 /* global flush doesn't need set IVA_REG */
394 val = DMA_TLB_GLOBAL_FLUSH|DMA_TLB_IVT;
395 break;
396 case DMA_TLB_DSI_FLUSH:
397 val = DMA_TLB_DSI_FLUSH|DMA_TLB_IVT|DMA_TLB_DID(did);
398 break;
399 case DMA_TLB_PSI_FLUSH:
400 val = DMA_TLB_PSI_FLUSH|DMA_TLB_IVT|DMA_TLB_DID(did);
401 /* Note: always flush non-leaf currently */
402 val_iva = size_order | addr;
403 break;
404 default:
405 BUG();
406 }
407 /* Note: set drain read/write */
408 if ( cap_read_drain(iommu->cap) )
409 val |= DMA_TLB_READ_DRAIN;
410 if ( cap_write_drain(iommu->cap) )
411 val |= DMA_TLB_WRITE_DRAIN;
413 spin_lock_irqsave(&iommu->register_lock, flag);
414 /* Note: Only uses first TLB reg currently */
415 if ( val_iva )
416 dmar_writeq(iommu->reg, tlb_offset, val_iva);
417 dmar_writeq(iommu->reg, tlb_offset + 8, val);
419 /* Make sure hardware complete it */
420 start_time = jiffies;
421 for ( ; ; )
422 {
423 val = dmar_readq(iommu->reg, tlb_offset + 8);
424 if ( !(val & DMA_TLB_IVT) )
425 break;
426 if ( time_after(jiffies, start_time + DMAR_OPERATION_TIMEOUT) )
427 panic("DMAR hardware is malfunctional, please disable IOMMU\n");
428 cpu_relax();
429 }
430 spin_unlock_irqrestore(&iommu->register_lock, flag);
432 /* check IOTLB invalidation granularity */
433 if ( DMA_TLB_IAIG(val) == 0 )
434 printk(KERN_ERR VTDPREFIX "IOMMU: flush IOTLB failed\n");
435 if ( DMA_TLB_IAIG(val) != DMA_TLB_IIRG(type) )
436 printk(KERN_ERR VTDPREFIX "IOMMU: tlb flush request %x, actual %x\n",
437 (u32)DMA_TLB_IIRG(type), (u32)DMA_TLB_IAIG(val));
438 /* flush context entry will implictly flush write buffer */
439 return 0;
440 }
442 static int inline iommu_flush_iotlb_global(struct iommu *iommu,
443 int non_present_entry_flush)
444 {
445 return __iommu_flush_iotlb(iommu, 0, 0, 0, DMA_TLB_GLOBAL_FLUSH,
446 non_present_entry_flush);
447 }
449 static int inline iommu_flush_iotlb_dsi(struct iommu *iommu, u16 did,
450 int non_present_entry_flush)
451 {
452 return __iommu_flush_iotlb(iommu, did, 0, 0, DMA_TLB_DSI_FLUSH,
453 non_present_entry_flush);
454 }
456 static int inline get_alignment(u64 base, unsigned int size)
457 {
458 int t = 0;
459 u64 end;
461 end = base + size - 1;
462 while ( base != end )
463 {
464 t++;
465 base >>= 1;
466 end >>= 1;
467 }
468 return t;
469 }
471 static int inline iommu_flush_iotlb_psi(
472 struct iommu *iommu, u16 did,
473 u64 addr, unsigned int pages, int non_present_entry_flush)
474 {
475 unsigned int align;
477 BUG_ON(addr & (~PAGE_MASK_4K));
478 BUG_ON(pages == 0);
480 /* Fallback to domain selective flush if no PSI support */
481 if ( !cap_pgsel_inv(iommu->cap) )
482 return iommu_flush_iotlb_dsi(iommu, did,
483 non_present_entry_flush);
485 /*
486 * PSI requires page size is 2 ^ x, and the base address is naturally
487 * aligned to the size
488 */
489 align = get_alignment(addr >> PAGE_SHIFT_4K, pages);
490 /* Fallback to domain selective flush if size is too big */
491 if ( align > cap_max_amask_val(iommu->cap) )
492 return iommu_flush_iotlb_dsi(iommu, did,
493 non_present_entry_flush);
495 addr >>= PAGE_SHIFT_4K + align;
496 addr <<= PAGE_SHIFT_4K + align;
498 return __iommu_flush_iotlb(iommu, did, addr, align,
499 DMA_TLB_PSI_FLUSH, non_present_entry_flush);
500 }
502 void iommu_flush_all(void)
503 {
504 struct acpi_drhd_unit *drhd;
505 struct iommu *iommu;
506 int i = 0;
508 wbinvd();
509 for_each_drhd_unit ( drhd )
510 {
511 iommu = drhd->iommu;
512 iommu_flush_context_global(iommu, 0);
513 iommu_flush_iotlb_global(iommu, 0);
514 i++;
515 }
516 }
518 /* clear one page's page table */
519 static void dma_pte_clear_one(struct domain *domain, u64 addr)
520 {
521 struct acpi_drhd_unit *drhd;
522 struct iommu *iommu;
523 struct dma_pte *pte = NULL;
524 struct page_info *pg = NULL;
526 drhd = list_entry(acpi_drhd_units.next, typeof(*drhd), list);
528 /* get last level pte */
529 pg = dma_addr_level_page(domain, addr, 1);
530 if ( !pg )
531 return;
532 pte = (struct dma_pte *)map_domain_page(page_to_mfn(pg));
533 pte += address_level_offset(addr, 1);
534 if ( pte )
535 {
536 dma_clear_pte(*pte);
537 iommu_flush_cache_entry(drhd->iommu, pte);
539 for_each_drhd_unit ( drhd )
540 {
541 iommu = drhd->iommu;
542 if ( cap_caching_mode(iommu->cap) )
543 iommu_flush_iotlb_psi(iommu, domain->domain_id, addr, 1, 0);
544 else if (cap_rwbf(iommu->cap))
545 iommu_flush_write_buffer(iommu);
546 }
547 }
548 }
550 /* clear last level pte, a tlb flush should be followed */
551 static void dma_pte_clear_range(struct domain *domain, u64 start, u64 end)
552 {
553 struct hvm_iommu *hd = domain_hvm_iommu(domain);
554 int addr_width = agaw_to_width(hd->agaw);
556 start &= (((u64)1) << addr_width) - 1;
557 end &= (((u64)1) << addr_width) - 1;
558 /* in case it's partial page */
559 start = PAGE_ALIGN_4K(start);
560 end &= PAGE_MASK_4K;
562 /* we don't need lock here, nobody else touches the iova range */
563 while ( start < end )
564 {
565 dma_pte_clear_one(domain, start);
566 start += PAGE_SIZE_4K;
567 }
568 }
570 /* free page table pages. last level pte should already be cleared */
571 void dma_pte_free_pagetable(struct domain *domain, u64 start, u64 end)
572 {
573 struct acpi_drhd_unit *drhd;
574 struct hvm_iommu *hd = domain_hvm_iommu(domain);
575 struct iommu *iommu;
576 int addr_width = agaw_to_width(hd->agaw);
577 struct dma_pte *pte;
578 int total = agaw_to_level(hd->agaw);
579 int level;
580 u32 tmp;
581 struct page_info *pg = NULL;
583 drhd = list_entry(acpi_drhd_units.next, typeof(*drhd), list);
584 iommu = drhd->iommu;
586 start &= (((u64)1) << addr_width) - 1;
587 end &= (((u64)1) << addr_width) - 1;
589 /* we don't need lock here, nobody else touches the iova range */
590 level = 2;
591 while ( level <= total )
592 {
593 tmp = align_to_level(start, level);
594 if ( (tmp >= end) || ((tmp + level_size(level)) > end) )
595 return;
597 while ( tmp < end )
598 {
599 pg = dma_addr_level_page(domain, tmp, level);
600 if ( !pg )
601 return;
602 pte = (struct dma_pte *)map_domain_page(page_to_mfn(pg));
603 pte += address_level_offset(tmp, level);
604 dma_clear_pte(*pte);
605 iommu_flush_cache_entry(iommu, pte);
606 unmap_domain_page(pte);
607 free_domheap_page(pg);
609 tmp += level_size(level);
610 }
611 level++;
612 }
614 /* free pgd */
615 if ( start == 0 && end == ((((u64)1) << addr_width) - 1) )
616 {
617 free_xenheap_page((void *)hd->pgd);
618 hd->pgd = NULL;
619 }
620 }
622 /* iommu handling */
623 static int iommu_set_root_entry(struct iommu *iommu)
624 {
625 void *addr;
626 u32 cmd, sts;
627 struct root_entry *root;
628 unsigned long flags;
630 if ( iommu == NULL )
631 {
632 gdprintk(XENLOG_ERR VTDPREFIX,
633 "iommu_set_root_entry: iommu == NULL\n");
634 return -EINVAL;
635 }
637 if ( unlikely(!iommu->root_entry) )
638 {
639 root = (struct root_entry *)alloc_xenheap_page();
640 if ( root == NULL )
641 return -ENOMEM;
643 memset((u8*)root, 0, PAGE_SIZE);
644 iommu_flush_cache_page(iommu, root);
646 if ( cmpxchg((unsigned long *)&iommu->root_entry,
647 0, (unsigned long)root) != 0 )
648 free_xenheap_page((void *)root);
649 }
651 addr = iommu->root_entry;
653 spin_lock_irqsave(&iommu->register_lock, flags);
655 dmar_writeq(iommu->reg, DMAR_RTADDR_REG, virt_to_maddr(addr));
656 cmd = iommu->gcmd | DMA_GCMD_SRTP;
657 dmar_writel(iommu->reg, DMAR_GCMD_REG, cmd);
659 /* Make sure hardware complete it */
660 for ( ; ; )
661 {
662 sts = dmar_readl(iommu->reg, DMAR_GSTS_REG);
663 if ( sts & DMA_GSTS_RTPS )
664 break;
665 cpu_relax();
666 }
668 spin_unlock_irqrestore(&iommu->register_lock, flags);
670 return 0;
671 }
673 static int iommu_enable_translation(struct iommu *iommu)
674 {
675 u32 sts;
676 unsigned long flags;
678 dprintk(XENLOG_INFO VTDPREFIX,
679 "iommu_enable_translation: enabling vt-d translation\n");
680 spin_lock_irqsave(&iommu->register_lock, flags);
681 iommu->gcmd |= DMA_GCMD_TE;
682 dmar_writel(iommu->reg, DMAR_GCMD_REG, iommu->gcmd);
683 /* Make sure hardware complete it */
684 for ( ; ; )
685 {
686 sts = dmar_readl(iommu->reg, DMAR_GSTS_REG);
687 if ( sts & DMA_GSTS_TES )
688 break;
689 cpu_relax();
690 }
692 /* Disable PMRs when VT-d engine takes effect per spec definition */
693 disable_pmr(iommu);
694 spin_unlock_irqrestore(&iommu->register_lock, flags);
695 return 0;
696 }
698 int iommu_disable_translation(struct iommu *iommu)
699 {
700 u32 sts;
701 unsigned long flags;
703 spin_lock_irqsave(&iommu->register_lock, flags);
704 iommu->gcmd &= ~ DMA_GCMD_TE;
705 dmar_writel(iommu->reg, DMAR_GCMD_REG, iommu->gcmd);
707 /* Make sure hardware complete it */
708 for ( ; ; )
709 {
710 sts = dmar_readl(iommu->reg, DMAR_GSTS_REG);
711 if ( !(sts & DMA_GSTS_TES) )
712 break;
713 cpu_relax();
714 }
715 spin_unlock_irqrestore(&iommu->register_lock, flags);
716 return 0;
717 }
719 static struct iommu *vector_to_iommu[NR_VECTORS];
720 static int iommu_page_fault_do_one(struct iommu *iommu, int type,
721 u8 fault_reason, u16 source_id, u32 addr)
722 {
723 dprintk(XENLOG_WARNING VTDPREFIX,
724 "iommu_page_fault:%s: DEVICE %x:%x.%x addr %x REASON %x\n",
725 (type ? "DMA Read" : "DMA Write"),
726 (source_id >> 8), PCI_SLOT(source_id & 0xFF),
727 PCI_FUNC(source_id & 0xFF), addr, fault_reason);
729 print_vtd_entries(current->domain, (source_id >> 8),(source_id & 0xff),
730 (addr >> PAGE_SHIFT));
731 return 0;
732 }
734 #define PRIMARY_FAULT_REG_LEN (16)
735 static void iommu_page_fault(int vector, void *dev_id,
736 struct cpu_user_regs *regs)
737 {
738 struct iommu *iommu = dev_id;
739 int reg, fault_index;
740 u32 fault_status;
741 unsigned long flags;
743 dprintk(XENLOG_WARNING VTDPREFIX,
744 "iommu_page_fault: iommu->reg = %p\n", iommu->reg);
746 spin_lock_irqsave(&iommu->register_lock, flags);
747 fault_status = dmar_readl(iommu->reg, DMAR_FSTS_REG);
748 spin_unlock_irqrestore(&iommu->register_lock, flags);
750 /* FIXME: ignore advanced fault log */
751 if ( !(fault_status & DMA_FSTS_PPF) )
752 return;
753 fault_index = dma_fsts_fault_record_index(fault_status);
754 reg = cap_fault_reg_offset(iommu->cap);
755 for ( ; ; )
756 {
757 u8 fault_reason;
758 u16 source_id;
759 u32 guest_addr, data;
760 int type;
762 /* highest 32 bits */
763 spin_lock_irqsave(&iommu->register_lock, flags);
764 data = dmar_readl(iommu->reg, reg +
765 fault_index * PRIMARY_FAULT_REG_LEN + 12);
766 if ( !(data & DMA_FRCD_F) )
767 {
768 spin_unlock_irqrestore(&iommu->register_lock, flags);
769 break;
770 }
772 fault_reason = dma_frcd_fault_reason(data);
773 type = dma_frcd_type(data);
775 data = dmar_readl(iommu->reg, reg +
776 fault_index * PRIMARY_FAULT_REG_LEN + 8);
777 source_id = dma_frcd_source_id(data);
779 guest_addr = dmar_readq(iommu->reg, reg +
780 fault_index * PRIMARY_FAULT_REG_LEN);
781 guest_addr = dma_frcd_page_addr(guest_addr);
782 /* clear the fault */
783 dmar_writel(iommu->reg, reg +
784 fault_index * PRIMARY_FAULT_REG_LEN + 12, DMA_FRCD_F);
785 spin_unlock_irqrestore(&iommu->register_lock, flags);
787 iommu_page_fault_do_one(iommu, type, fault_reason,
788 source_id, guest_addr);
790 fault_index++;
791 if ( fault_index > cap_num_fault_regs(iommu->cap) )
792 fault_index = 0;
793 }
795 /* clear primary fault overflow */
796 if ( fault_status & DMA_FSTS_PFO )
797 {
798 spin_lock_irqsave(&iommu->register_lock, flags);
799 dmar_writel(iommu->reg, DMAR_FSTS_REG, DMA_FSTS_PFO);
800 spin_unlock_irqrestore(&iommu->register_lock, flags);
801 }
802 }
804 static void dma_msi_unmask(unsigned int vector)
805 {
806 struct iommu *iommu = vector_to_iommu[vector];
807 unsigned long flags;
809 /* unmask it */
810 spin_lock_irqsave(&iommu->register_lock, flags);
811 dmar_writel(iommu->reg, DMAR_FECTL_REG, 0);
812 spin_unlock_irqrestore(&iommu->register_lock, flags);
813 }
815 static void dma_msi_mask(unsigned int vector)
816 {
817 unsigned long flags;
818 struct iommu *iommu = vector_to_iommu[vector];
820 /* mask it */
821 spin_lock_irqsave(&iommu->register_lock, flags);
822 dmar_writel(iommu->reg, DMAR_FECTL_REG, DMA_FECTL_IM);
823 spin_unlock_irqrestore(&iommu->register_lock, flags);
824 }
826 static unsigned int dma_msi_startup(unsigned int vector)
827 {
828 dma_msi_unmask(vector);
829 return 0;
830 }
832 static void dma_msi_end(unsigned int vector)
833 {
834 dma_msi_unmask(vector);
835 ack_APIC_irq();
836 }
838 static void dma_msi_data_init(struct iommu *iommu, int vector)
839 {
840 u32 msi_data = 0;
841 unsigned long flags;
843 /* Fixed, edge, assert mode. Follow MSI setting */
844 msi_data |= vector & 0xff;
845 msi_data |= 1 << 14;
847 spin_lock_irqsave(&iommu->register_lock, flags);
848 dmar_writel(iommu->reg, DMAR_FEDATA_REG, msi_data);
849 spin_unlock_irqrestore(&iommu->register_lock, flags);
850 }
852 static void dma_msi_addr_init(struct iommu *iommu, int phy_cpu)
853 {
854 u64 msi_address;
855 unsigned long flags;
857 /* Physical, dedicated cpu. Follow MSI setting */
858 msi_address = (MSI_ADDRESS_HEADER << (MSI_ADDRESS_HEADER_SHIFT + 8));
859 msi_address |= MSI_PHYSICAL_MODE << 2;
860 msi_address |= MSI_REDIRECTION_HINT_MODE << 3;
861 msi_address |= phy_cpu << MSI_TARGET_CPU_SHIFT;
863 spin_lock_irqsave(&iommu->register_lock, flags);
864 dmar_writel(iommu->reg, DMAR_FEADDR_REG, (u32)msi_address);
865 dmar_writel(iommu->reg, DMAR_FEUADDR_REG, (u32)(msi_address >> 32));
866 spin_unlock_irqrestore(&iommu->register_lock, flags);
867 }
869 static void dma_msi_set_affinity(unsigned int vector, cpumask_t dest)
870 {
871 struct iommu *iommu = vector_to_iommu[vector];
872 dma_msi_addr_init(iommu, cpu_physical_id(first_cpu(dest)));
873 }
875 static struct hw_interrupt_type dma_msi_type = {
876 .typename = "DMA_MSI",
877 .startup = dma_msi_startup,
878 .shutdown = dma_msi_mask,
879 .enable = dma_msi_unmask,
880 .disable = dma_msi_mask,
881 .ack = dma_msi_mask,
882 .end = dma_msi_end,
883 .set_affinity = dma_msi_set_affinity,
884 };
886 int iommu_set_interrupt(struct iommu *iommu)
887 {
888 int vector, ret;
890 vector = assign_irq_vector(AUTO_ASSIGN);
891 vector_to_iommu[vector] = iommu;
893 /* VT-d fault is a MSI, make irq == vector */
894 irq_vector[vector] = vector;
895 vector_irq[vector] = vector;
897 if ( !vector )
898 {
899 gdprintk(XENLOG_ERR VTDPREFIX, "IOMMU: no vectors\n");
900 return -EINVAL;
901 }
903 irq_desc[vector].handler = &dma_msi_type;
904 ret = request_irq(vector, iommu_page_fault, 0, "dmar", iommu);
905 if ( ret )
906 gdprintk(XENLOG_ERR VTDPREFIX, "IOMMU: can't request irq\n");
907 return vector;
908 }
910 struct iommu *iommu_alloc(void *hw_data)
911 {
912 struct acpi_drhd_unit *drhd = (struct acpi_drhd_unit *) hw_data;
913 struct iommu *iommu;
915 if ( nr_iommus > MAX_IOMMUS )
916 {
917 gdprintk(XENLOG_ERR VTDPREFIX,
918 "IOMMU: nr_iommus %d > MAX_IOMMUS\n", nr_iommus);
919 return NULL;
920 }
922 iommu = xmalloc(struct iommu);
923 if ( !iommu )
924 return NULL;
925 memset(iommu, 0, sizeof(struct iommu));
927 set_fixmap_nocache(FIX_IOMMU_REGS_BASE_0 + nr_iommus, drhd->address);
928 iommu->reg = (void *) fix_to_virt(FIX_IOMMU_REGS_BASE_0 + nr_iommus);
929 dprintk(XENLOG_INFO VTDPREFIX,
930 "iommu_alloc: iommu->reg = %p drhd->address = %lx\n",
931 iommu->reg, drhd->address);
932 nr_iommus++;
934 if ( !iommu->reg )
935 {
936 printk(KERN_ERR VTDPREFIX "IOMMU: can't mapping the region\n");
937 goto error;
938 }
940 iommu->cap = dmar_readq(iommu->reg, DMAR_CAP_REG);
941 iommu->ecap = dmar_readq(iommu->reg, DMAR_ECAP_REG);
943 spin_lock_init(&iommu->lock);
944 spin_lock_init(&iommu->register_lock);
946 drhd->iommu = iommu;
947 return iommu;
948 error:
949 xfree(iommu);
950 return NULL;
951 }
953 static void free_iommu(struct iommu *iommu)
954 {
955 if ( !iommu )
956 return;
957 if ( iommu->root_entry )
958 free_xenheap_page((void *)iommu->root_entry);
959 if ( iommu->reg )
960 iounmap(iommu->reg);
961 free_irq(iommu->vector);
962 xfree(iommu);
963 }
965 #define guestwidth_to_adjustwidth(gaw) ({ \
966 int agaw, r = (gaw - 12) % 9; \
967 agaw = (r == 0) ? gaw : (gaw + 9 - r); \
968 if ( agaw > 64 ) \
969 agaw = 64; \
970 agaw; })
972 int iommu_domain_init(struct domain *domain)
973 {
974 struct hvm_iommu *hd = domain_hvm_iommu(domain);
975 struct iommu *iommu = NULL;
976 int guest_width = DEFAULT_DOMAIN_ADDRESS_WIDTH;
977 int adjust_width, agaw;
978 unsigned long sagaw;
979 struct acpi_drhd_unit *drhd;
981 spin_lock_init(&hd->mapping_lock);
982 spin_lock_init(&hd->iommu_list_lock);
983 INIT_LIST_HEAD(&hd->pdev_list);
984 INIT_LIST_HEAD(&hd->g2m_ioport_list);
986 if ( !vtd_enabled || list_empty(&acpi_drhd_units) )
987 return 0;
989 for_each_drhd_unit ( drhd )
990 iommu = drhd->iommu ? : iommu_alloc(drhd);
992 /* calculate AGAW */
993 if (guest_width > cap_mgaw(iommu->cap))
994 guest_width = cap_mgaw(iommu->cap);
995 adjust_width = guestwidth_to_adjustwidth(guest_width);
996 agaw = width_to_agaw(adjust_width);
997 /* FIXME: hardware doesn't support it, choose a bigger one? */
998 sagaw = cap_sagaw(iommu->cap);
999 if ( !test_bit(agaw, &sagaw) )
1001 gdprintk(XENLOG_ERR VTDPREFIX,
1002 "IOMMU: hardware doesn't support the agaw\n");
1003 agaw = find_next_bit(&sagaw, 5, agaw);
1004 if ( agaw >= 5 )
1005 return -ENODEV;
1007 hd->agaw = agaw;
1008 return 0;
1011 static int domain_context_mapping_one(
1012 struct domain *domain,
1013 struct iommu *iommu,
1014 u8 bus, u8 devfn)
1016 struct hvm_iommu *hd = domain_hvm_iommu(domain);
1017 struct context_entry *context;
1018 unsigned long flags;
1019 int ret = 0;
1021 context = device_to_context_entry(iommu, bus, devfn);
1022 if ( !context )
1024 gdprintk(XENLOG_INFO VTDPREFIX,
1025 "domain_context_mapping_one:context == NULL:"
1026 "bdf = %x:%x:%x\n",
1027 bus, PCI_SLOT(devfn), PCI_FUNC(devfn));
1028 return -ENOMEM;
1030 spin_lock_irqsave(&iommu->lock, flags);
1031 if ( context_present(*context) )
1033 spin_unlock_irqrestore(&iommu->lock, flags);
1034 gdprintk(XENLOG_INFO VTDPREFIX,
1035 "domain_context_mapping_one:context present:bdf=%x:%x:%x\n",
1036 bus, PCI_SLOT(devfn), PCI_FUNC(devfn));
1037 return 0;
1040 #ifdef VTD_DEBUG
1041 dprintk(XENLOG_INFO VTDPREFIX,
1042 "context_mapping_one_1-%x:%x:%x-*context = %lx %lx\n",
1043 bus, PCI_SLOT(devfn), PCI_FUNC(devfn), context->hi, context->lo);
1044 #endif
1046 /*
1047 * domain_id 0 is not valid on Intel's IOMMU, force domain_id to
1048 * be 1 based as required by intel's iommu hw.
1049 */
1050 context_set_domain_id(*context, domain->domain_id);
1051 context_set_address_width(*context, hd->agaw);
1053 if ( ecap_pass_thru(iommu->ecap) )
1055 context_set_translation_type(*context, CONTEXT_TT_PASS_THRU);
1057 else
1059 context_set_address_root(*context, virt_to_maddr(hd->pgd));
1060 context_set_translation_type(*context, CONTEXT_TT_MULTI_LEVEL);
1063 context_set_fault_enable(*context);
1064 context_set_present(*context);
1065 iommu_flush_cache_entry(iommu, context);
1067 #ifdef VTD_DEBUG
1068 dprintk(XENLOG_INFO VTDPREFIX,
1069 "context_mapping_one_2-%x:%x:%x-*context=%lx %lx hd->pgd = %p\n",
1070 bus, PCI_SLOT(devfn), PCI_FUNC(devfn),
1071 context->hi, context->lo, hd->pgd);
1072 #endif
1074 if ( iommu_flush_context_device(iommu, domain->domain_id,
1075 (((u16)bus) << 8) | devfn,
1076 DMA_CCMD_MASK_NOBIT, 1) )
1077 iommu_flush_write_buffer(iommu);
1078 else
1079 iommu_flush_iotlb_dsi(iommu, domain->domain_id, 0);
1080 spin_unlock_irqrestore(&iommu->lock, flags);
1081 return ret;
1084 static int __pci_find_next_cap(u8 bus, unsigned int devfn, u8 pos, int cap)
1086 u8 id;
1087 int ttl = 48;
1089 while ( ttl-- )
1091 pos = read_pci_config_byte(bus, PCI_SLOT(devfn), PCI_FUNC(devfn), pos);
1092 if ( pos < 0x40 )
1093 break;
1095 pos &= ~3;
1096 id = read_pci_config_byte(bus, PCI_SLOT(devfn), PCI_FUNC(devfn),
1097 pos + PCI_CAP_LIST_ID);
1099 if ( id == 0xff )
1100 break;
1101 if ( id == cap )
1102 return pos;
1104 pos += PCI_CAP_LIST_NEXT;
1106 return 0;
1109 #define PCI_BASE_CLASS_BRIDGE 0x06
1110 #define PCI_CLASS_BRIDGE_PCI 0x0604
1112 #define DEV_TYPE_PCIe_ENDPOINT 1
1113 #define DEV_TYPE_PCI_BRIDGE 2
1114 #define DEV_TYPE_PCI 3
1116 int pdev_type(struct pci_dev *dev)
1118 u16 class_device;
1119 u16 status;
1121 class_device = read_pci_config_16(dev->bus, PCI_SLOT(dev->devfn),
1122 PCI_FUNC(dev->devfn), PCI_CLASS_DEVICE);
1123 if ( class_device == PCI_CLASS_BRIDGE_PCI )
1124 return DEV_TYPE_PCI_BRIDGE;
1126 status = read_pci_config_16(dev->bus, PCI_SLOT(dev->devfn),
1127 PCI_FUNC(dev->devfn), PCI_STATUS);
1129 if ( !(status & PCI_STATUS_CAP_LIST) )
1130 return DEV_TYPE_PCI;
1132 if ( __pci_find_next_cap(dev->bus, dev->devfn,
1133 PCI_CAPABILITY_LIST, PCI_CAP_ID_EXP) )
1134 return DEV_TYPE_PCIe_ENDPOINT;
1136 return DEV_TYPE_PCI;
1139 #define MAX_BUSES 256
1140 struct pci_dev bus2bridge[MAX_BUSES];
1142 static int domain_context_mapping(
1143 struct domain *domain,
1144 struct iommu *iommu,
1145 struct pci_dev *pdev)
1147 int ret = 0;
1148 int dev, func, sec_bus, sub_bus;
1149 u32 type;
1151 type = pdev_type(pdev);
1152 switch ( type )
1154 case DEV_TYPE_PCI_BRIDGE:
1155 sec_bus = read_pci_config_byte(
1156 pdev->bus, PCI_SLOT(pdev->devfn),
1157 PCI_FUNC(pdev->devfn), PCI_SECONDARY_BUS);
1159 if ( bus2bridge[sec_bus].bus == 0 )
1161 bus2bridge[sec_bus].bus = pdev->bus;
1162 bus2bridge[sec_bus].devfn = pdev->devfn;
1165 sub_bus = read_pci_config_byte(
1166 pdev->bus, PCI_SLOT(pdev->devfn),
1167 PCI_FUNC(pdev->devfn), PCI_SUBORDINATE_BUS);
1169 if ( sec_bus != sub_bus )
1171 dprintk(XENLOG_INFO VTDPREFIX,
1172 "context_mapping: nested PCI bridge not supported\n");
1173 dprintk(XENLOG_INFO VTDPREFIX,
1174 " bdf = %x:%x:%x sec_bus = %x sub_bus = %x\n",
1175 pdev->bus, PCI_SLOT(pdev->devfn), PCI_FUNC(pdev->devfn),
1176 sec_bus, sub_bus);
1178 break;
1179 case DEV_TYPE_PCIe_ENDPOINT:
1180 gdprintk(XENLOG_INFO VTDPREFIX,
1181 "domain_context_mapping:PCIe : bdf = %x:%x:%x\n",
1182 pdev->bus, PCI_SLOT(pdev->devfn), PCI_FUNC(pdev->devfn));
1183 ret = domain_context_mapping_one(domain, iommu,
1184 (u8)(pdev->bus), (u8)(pdev->devfn));
1185 break;
1186 case DEV_TYPE_PCI:
1187 gdprintk(XENLOG_INFO VTDPREFIX,
1188 "domain_context_mapping:PCI: bdf = %x:%x:%x\n",
1189 pdev->bus, PCI_SLOT(pdev->devfn), PCI_FUNC(pdev->devfn));
1191 if ( pdev->bus == 0 )
1193 ret = domain_context_mapping_one(
1194 domain, iommu, (u8)(pdev->bus), (u8)(pdev->devfn));
1196 else
1198 if ( bus2bridge[pdev->bus].bus != 0 )
1199 gdprintk(XENLOG_ERR VTDPREFIX,
1200 "domain_context_mapping:bus2bridge"
1201 "[pdev->bus].bus != 0\n");
1203 ret = domain_context_mapping_one(
1204 domain, iommu,
1205 (u8)(bus2bridge[pdev->bus].bus),
1206 (u8)(bus2bridge[pdev->bus].devfn));
1208 /* now map everything behind the PCI bridge */
1209 for ( dev = 0; dev < 32; dev++ )
1211 for ( func = 0; func < 8; func++ )
1213 ret = domain_context_mapping_one(
1214 domain, iommu,
1215 pdev->bus, (u8)PCI_DEVFN(dev, func));
1216 if ( ret )
1217 return ret;
1221 break;
1222 default:
1223 gdprintk(XENLOG_ERR VTDPREFIX,
1224 "domain_context_mapping:unknown type : bdf = %x:%x:%x\n",
1225 pdev->bus, PCI_SLOT(pdev->devfn), PCI_FUNC(pdev->devfn));
1226 ret = -EINVAL;
1227 break;
1230 return ret;
1233 static int domain_context_unmap_one(
1234 struct domain *domain,
1235 struct iommu *iommu,
1236 u8 bus, u8 devfn)
1238 struct context_entry *context;
1239 unsigned long flags;
1241 context = device_to_context_entry(iommu, bus, devfn);
1242 if ( !context )
1244 gdprintk(XENLOG_INFO VTDPREFIX,
1245 "domain_context_unmap_one-%x:%x:%x- context == NULL:return\n",
1246 bus, PCI_SLOT(devfn), PCI_FUNC(devfn));
1247 return -ENOMEM;
1250 spin_lock_irqsave(&iommu->lock, flags);
1251 if ( !context_present(*context) )
1253 spin_unlock_irqrestore(&iommu->lock, flags);
1254 gdprintk(XENLOG_INFO VTDPREFIX,
1255 "domain_context_unmap_one-%x:%x:%x- "
1256 "context NOT present:return\n",
1257 bus, PCI_SLOT(devfn), PCI_FUNC(devfn));
1258 return 0;
1261 gdprintk(XENLOG_INFO VTDPREFIX,
1262 "domain_context_unmap_one_1:bdf = %x:%x:%x\n",
1263 bus, PCI_SLOT(devfn), PCI_FUNC(devfn));
1265 context_clear_present(*context);
1266 context_clear_entry(*context);
1267 iommu_flush_cache_entry(iommu, context);
1268 iommu_flush_context_global(iommu, 0);
1269 iommu_flush_iotlb_global(iommu, 0);
1270 spin_unlock_irqrestore(&iommu->lock, flags);
1272 gdprintk(XENLOG_INFO VTDPREFIX,
1273 "domain_context_unmap_one_2:bdf = %x:%x:%x\n",
1274 bus, PCI_SLOT(devfn), PCI_FUNC(devfn));
1276 return 0;
1279 static int domain_context_unmap(
1280 struct domain *domain,
1281 struct iommu *iommu,
1282 struct pci_dev *pdev)
1284 int ret = 0;
1285 int dev, func, sec_bus, sub_bus;
1286 u32 type;
1288 type = pdev_type(pdev);
1289 switch ( type )
1291 case DEV_TYPE_PCI_BRIDGE:
1292 sec_bus = read_pci_config_byte(
1293 pdev->bus, PCI_SLOT(pdev->devfn),
1294 PCI_FUNC(pdev->devfn), PCI_SECONDARY_BUS);
1295 sub_bus = read_pci_config_byte(
1296 pdev->bus, PCI_SLOT(pdev->devfn),
1297 PCI_FUNC(pdev->devfn), PCI_SUBORDINATE_BUS);
1299 gdprintk(XENLOG_INFO VTDPREFIX,
1300 "domain_context_unmap:BRIDGE:%x:%x:%x "
1301 "sec_bus=%x sub_bus=%x\n",
1302 pdev->bus, PCI_SLOT(pdev->devfn),
1303 PCI_FUNC(pdev->devfn), sec_bus, sub_bus);
1304 break;
1305 case DEV_TYPE_PCIe_ENDPOINT:
1306 gdprintk(XENLOG_INFO VTDPREFIX,
1307 "domain_context_unmap:PCIe : bdf = %x:%x:%x\n",
1308 pdev->bus, PCI_SLOT(pdev->devfn), PCI_FUNC(pdev->devfn));
1309 ret = domain_context_unmap_one(domain, iommu,
1310 (u8)(pdev->bus), (u8)(pdev->devfn));
1311 break;
1312 case DEV_TYPE_PCI:
1313 gdprintk(XENLOG_INFO VTDPREFIX,
1314 "domain_context_unmap:PCI: bdf = %x:%x:%x\n",
1315 pdev->bus, PCI_SLOT(pdev->devfn), PCI_FUNC(pdev->devfn));
1316 if ( pdev->bus == 0 )
1318 ret = domain_context_unmap_one(
1319 domain, iommu,
1320 (u8)(pdev->bus), (u8)(pdev->devfn));
1322 else
1324 if ( bus2bridge[pdev->bus].bus != 0 )
1325 gdprintk(XENLOG_INFO VTDPREFIX,
1326 "domain_context_mapping:"
1327 "bus2bridge[pdev->bus].bus != 0\n");
1329 ret = domain_context_unmap_one(domain, iommu,
1330 (u8)(bus2bridge[pdev->bus].bus),
1331 (u8)(bus2bridge[pdev->bus].devfn));
1333 /* now map everything behind the PCI bridge */
1334 for ( dev = 0; dev < 32; dev++ )
1336 for ( func = 0; func < 8; func++ )
1338 ret = domain_context_unmap_one(
1339 domain, iommu,
1340 pdev->bus, (u8)PCI_DEVFN(dev, func));
1341 if ( ret )
1342 return ret;
1346 break;
1347 default:
1348 gdprintk(XENLOG_ERR VTDPREFIX,
1349 "domain_context_unmap:unknown type: bdf = %x:%x:%x\n",
1350 pdev->bus, PCI_SLOT(pdev->devfn), PCI_FUNC(pdev->devfn));
1351 ret = -EINVAL;
1352 break;
1355 return ret;
1358 void reassign_device_ownership(
1359 struct domain *source,
1360 struct domain *target,
1361 u8 bus, u8 devfn)
1363 struct hvm_iommu *source_hd = domain_hvm_iommu(source);
1364 struct hvm_iommu *target_hd = domain_hvm_iommu(target);
1365 struct pci_dev *pdev;
1366 struct acpi_drhd_unit *drhd;
1367 struct iommu *iommu;
1368 int status;
1369 unsigned long flags;
1371 gdprintk(XENLOG_ERR VTDPREFIX,
1372 "reassign_device-%x:%x:%x- source = %d target = %d\n",
1373 bus, PCI_SLOT(devfn), PCI_FUNC(devfn),
1374 source->domain_id, target->domain_id);
1376 for_each_pdev( source, pdev )
1378 if ( (pdev->bus != bus) || (pdev->devfn != devfn) )
1379 continue;
1381 pdev->bus = bus;
1382 pdev->devfn = devfn;
1383 drhd = acpi_find_matched_drhd_unit(pdev);
1384 iommu = drhd->iommu;
1385 domain_context_unmap(source, iommu, pdev);
1387 /* Move pci device from the source domain to target domain. */
1388 spin_lock_irqsave(&source_hd->iommu_list_lock, flags);
1389 spin_lock_irqsave(&target_hd->iommu_list_lock, flags);
1390 list_move(&pdev->list, &target_hd->pdev_list);
1391 spin_unlock_irqrestore(&target_hd->iommu_list_lock, flags);
1392 spin_unlock_irqrestore(&source_hd->iommu_list_lock, flags);
1394 status = domain_context_mapping(target, iommu, pdev);
1395 if ( status != 0 )
1396 gdprintk(XENLOG_ERR VTDPREFIX, "domain_context_mapping failed\n");
1398 break;
1402 void return_devices_to_dom0(struct domain *d)
1404 struct hvm_iommu *hd = domain_hvm_iommu(d);
1405 struct pci_dev *pdev;
1407 while ( !list_empty(&hd->pdev_list) )
1409 pdev = list_entry(hd->pdev_list.next, typeof(*pdev), list);
1410 dprintk(XENLOG_INFO VTDPREFIX,
1411 "return_devices_to_dom0: bdf = %x:%x:%x\n",
1412 pdev->bus, PCI_SLOT(pdev->devfn), PCI_FUNC(pdev->devfn));
1413 reassign_device_ownership(d, dom0, pdev->bus, pdev->devfn);
1416 #ifdef VTD_DEBUG
1417 for_each_pdev ( dom0, pdev )
1418 dprintk(XENLOG_INFO VTDPREFIX,
1419 "return_devices_to_dom0:%x: bdf = %x:%x:%x\n",
1420 dom0->domain_id, pdev->bus,
1421 PCI_SLOT(pdev->devfn), PCI_FUNC(pdev->devfn));
1422 #endif
1425 void iommu_domain_teardown(struct domain *d)
1427 if ( list_empty(&acpi_drhd_units) )
1428 return;
1430 #if CONFIG_PAGING_LEVELS == 3
1432 struct hvm_iommu *hd = domain_hvm_iommu(d);
1433 int level = agaw_to_level(hd->agaw);
1434 struct dma_pte *pgd = NULL;
1436 switch ( level )
1438 case VTD_PAGE_TABLE_LEVEL_3:
1439 if ( hd->pgd )
1440 free_xenheap_page((void *)hd->pgd);
1441 break;
1442 case VTD_PAGE_TABLE_LEVEL_4:
1443 if ( hd->pgd )
1445 pgd = hd->pgd;
1446 if ( pgd[0].val != 0 )
1447 free_xenheap_page((void*)maddr_to_virt(
1448 dma_pte_addr(pgd[0])));
1450 free_xenheap_page((void *)hd->pgd);
1452 break;
1453 default:
1454 gdprintk(XENLOG_ERR VTDPREFIX,
1455 "Unsupported p2m table sharing level!\n");
1456 break;
1459 #endif
1460 return_devices_to_dom0(d);
1463 static int domain_context_mapped(struct domain *domain, struct pci_dev *pdev)
1465 struct acpi_drhd_unit *drhd;
1466 struct iommu *iommu;
1467 int ret;
1469 for_each_drhd_unit ( drhd )
1471 iommu = drhd->iommu;
1472 ret = device_context_mapped(iommu, pdev->bus, pdev->devfn);
1473 if ( ret )
1474 return ret;
1477 return 0;
1480 int iommu_map_page(struct domain *d, paddr_t gfn, paddr_t mfn)
1482 struct acpi_drhd_unit *drhd;
1483 struct iommu *iommu;
1484 struct dma_pte *pte = NULL;
1485 struct page_info *pg = NULL;
1487 drhd = list_entry(acpi_drhd_units.next, typeof(*drhd), list);
1488 iommu = drhd->iommu;
1490 /* do nothing if dom0 and iommu supports pass thru */
1491 if ( ecap_pass_thru(iommu->ecap) && (d->domain_id == 0) )
1492 return 0;
1494 pg = addr_to_dma_page(d, gfn << PAGE_SHIFT_4K);
1495 if ( !pg )
1496 return -ENOMEM;
1497 pte = (struct dma_pte *)map_domain_page(page_to_mfn(pg));
1498 pte += mfn & LEVEL_MASK;
1499 dma_set_pte_addr(*pte, mfn << PAGE_SHIFT_4K);
1500 dma_set_pte_prot(*pte, DMA_PTE_READ | DMA_PTE_WRITE);
1501 iommu_flush_cache_entry(iommu, pte);
1502 unmap_domain_page(pte);
1504 for_each_drhd_unit ( drhd )
1506 iommu = drhd->iommu;
1507 if ( cap_caching_mode(iommu->cap) )
1508 iommu_flush_iotlb_psi(iommu, d->domain_id,
1509 gfn << PAGE_SHIFT_4K, 1, 0);
1510 else if ( cap_rwbf(iommu->cap) )
1511 iommu_flush_write_buffer(iommu);
1514 return 0;
1517 int iommu_unmap_page(struct domain *d, dma_addr_t gfn)
1519 struct acpi_drhd_unit *drhd;
1520 struct iommu *iommu;
1522 drhd = list_entry(acpi_drhd_units.next, typeof(*drhd), list);
1523 iommu = drhd->iommu;
1525 /* do nothing if dom0 and iommu supports pass thru */
1526 if ( ecap_pass_thru(iommu->ecap) && (d->domain_id == 0) )
1527 return 0;
1529 dma_pte_clear_one(d, gfn << PAGE_SHIFT_4K);
1531 return 0;
1534 int iommu_page_mapping(struct domain *domain, dma_addr_t iova,
1535 void *hpa, size_t size, int prot)
1537 struct acpi_drhd_unit *drhd;
1538 struct iommu *iommu;
1539 unsigned long start_pfn, end_pfn;
1540 struct dma_pte *pte = NULL;
1541 int index;
1542 struct page_info *pg = NULL;
1544 drhd = list_entry(acpi_drhd_units.next, typeof(*drhd), list);
1545 iommu = drhd->iommu;
1546 if ( (prot & (DMA_PTE_READ|DMA_PTE_WRITE)) == 0 )
1547 return -EINVAL;
1548 iova = (iova >> PAGE_SHIFT_4K) << PAGE_SHIFT_4K;
1549 start_pfn = (unsigned long)(((unsigned long) hpa) >> PAGE_SHIFT_4K);
1550 end_pfn = (unsigned long)
1551 ((PAGE_ALIGN_4K(((unsigned long)hpa) + size)) >> PAGE_SHIFT_4K);
1552 index = 0;
1553 while ( start_pfn < end_pfn )
1555 pg = addr_to_dma_page(domain, iova + PAGE_SIZE_4K * index);
1556 if ( !pg )
1557 return -ENOMEM;
1558 pte = (struct dma_pte *)map_domain_page(page_to_mfn(pg));
1559 pte += start_pfn & LEVEL_MASK;
1560 dma_set_pte_addr(*pte, start_pfn << PAGE_SHIFT_4K);
1561 dma_set_pte_prot(*pte, prot);
1562 iommu_flush_cache_entry(iommu, pte);
1563 unmap_domain_page(pte);
1564 start_pfn++;
1565 index++;
1568 for_each_drhd_unit ( drhd )
1570 iommu = drhd->iommu;
1571 if ( cap_caching_mode(iommu->cap) )
1572 iommu_flush_iotlb_psi(iommu, domain->domain_id, iova, size, 0);
1573 else if ( cap_rwbf(iommu->cap) )
1574 iommu_flush_write_buffer(iommu);
1577 return 0;
1580 int iommu_page_unmapping(struct domain *domain, dma_addr_t addr, size_t size)
1582 dma_pte_clear_range(domain, addr, addr + size);
1584 return 0;
1587 void iommu_flush(struct domain *d, dma_addr_t gfn, u64 *p2m_entry)
1589 struct acpi_drhd_unit *drhd;
1590 struct iommu *iommu = NULL;
1591 struct dma_pte *pte = (struct dma_pte *) p2m_entry;
1593 for_each_drhd_unit ( drhd )
1595 iommu = drhd->iommu;
1596 if ( cap_caching_mode(iommu->cap) )
1597 iommu_flush_iotlb_psi(iommu, d->domain_id,
1598 gfn << PAGE_SHIFT_4K, 1, 0);
1599 else if ( cap_rwbf(iommu->cap) )
1600 iommu_flush_write_buffer(iommu);
1603 iommu_flush_cache_entry(iommu, pte);
1606 static int iommu_prepare_rmrr_dev(
1607 struct domain *d,
1608 struct acpi_rmrr_unit *rmrr,
1609 struct pci_dev *pdev)
1611 struct acpi_drhd_unit *drhd;
1612 unsigned long size;
1613 int ret;
1615 /* page table init */
1616 size = rmrr->end_address - rmrr->base_address + 1;
1617 ret = iommu_page_mapping(d, rmrr->base_address,
1618 (void *)rmrr->base_address, size,
1619 DMA_PTE_READ|DMA_PTE_WRITE);
1620 if ( ret )
1621 return ret;
1623 if ( domain_context_mapped(d, pdev) == 0 )
1625 drhd = acpi_find_matched_drhd_unit(pdev);
1626 ret = domain_context_mapping(d, drhd->iommu, pdev);
1627 if ( !ret )
1628 return 0;
1631 return ret;
1634 void __init setup_dom0_devices(void)
1636 struct hvm_iommu *hd = domain_hvm_iommu(dom0);
1637 struct acpi_drhd_unit *drhd;
1638 struct pci_dev *pdev;
1639 int bus, dev, func, ret;
1640 u32 l;
1642 #ifdef DEBUG_VTD_CONTEXT_ENTRY
1643 for ( bus = 0; bus < 256; bus++ )
1645 for ( dev = 0; dev < 32; dev++ )
1647 for ( func = 0; func < 8; func++ )
1649 struct context_entry *context;
1650 struct pci_dev device;
1652 device.bus = bus;
1653 device.devfn = PCI_DEVFN(dev, func);
1654 drhd = acpi_find_matched_drhd_unit(&device);
1655 context = device_to_context_entry(drhd->iommu,
1656 bus, PCI_DEVFN(dev, func));
1657 if ( (context->lo != 0) || (context->hi != 0) )
1658 dprintk(XENLOG_INFO VTDPREFIX,
1659 "setup_dom0_devices-%x:%x:%x- context not 0\n",
1660 bus, dev, func);
1664 #endif
1666 for ( bus = 0; bus < 256; bus++ )
1668 for ( dev = 0; dev < 32; dev++ )
1670 for ( func = 0; func < 8; func++ )
1672 l = read_pci_config(bus, dev, func, PCI_VENDOR_ID);
1673 /* some broken boards return 0 or ~0 if a slot is empty: */
1674 if ( (l == 0xffffffff) || (l == 0x00000000) ||
1675 (l == 0x0000ffff) || (l == 0xffff0000) )
1676 continue;
1677 pdev = xmalloc(struct pci_dev);
1678 pdev->bus = bus;
1679 pdev->devfn = PCI_DEVFN(dev, func);
1680 list_add_tail(&pdev->list, &hd->pdev_list);
1682 drhd = acpi_find_matched_drhd_unit(pdev);
1683 ret = domain_context_mapping(dom0, drhd->iommu, pdev);
1684 if ( ret != 0 )
1685 gdprintk(XENLOG_ERR VTDPREFIX,
1686 "domain_context_mapping failed\n");
1691 for_each_pdev ( dom0, pdev )
1692 dprintk(XENLOG_INFO VTDPREFIX,
1693 "setup_dom0_devices: bdf = %x:%x:%x\n",
1694 pdev->bus, PCI_SLOT(pdev->devfn), PCI_FUNC(pdev->devfn));
1697 void clear_fault_bit(struct iommu *iommu)
1699 u64 val;
1701 val = dmar_readq(
1702 iommu->reg,
1703 cap_fault_reg_offset(dmar_readq(iommu->reg,DMAR_CAP_REG))+0x8);
1704 dmar_writeq(
1705 iommu->reg,
1706 cap_fault_reg_offset(dmar_readq(iommu->reg,DMAR_CAP_REG))+8,
1707 val);
1708 dmar_writel(iommu->reg, DMAR_FSTS_REG, DMA_FSTS_PFO);
1711 static int init_vtd_hw(void)
1713 struct acpi_drhd_unit *drhd;
1714 struct iommu *iommu;
1715 int ret;
1717 for_each_drhd_unit ( drhd )
1719 iommu = drhd->iommu;
1720 ret = iommu_set_root_entry(iommu);
1721 if ( ret )
1723 gdprintk(XENLOG_ERR VTDPREFIX, "IOMMU: set root entry failed\n");
1724 return -EIO;
1728 return 0;
1731 static int enable_vtd_translation(void)
1733 struct acpi_drhd_unit *drhd;
1734 struct iommu *iommu;
1735 int vector = 0;
1737 for_each_drhd_unit ( drhd )
1739 iommu = drhd->iommu;
1740 vector = iommu_set_interrupt(iommu);
1741 dma_msi_data_init(iommu, vector);
1742 dma_msi_addr_init(iommu, cpu_physical_id(first_cpu(cpu_online_map)));
1743 iommu->vector = vector;
1744 clear_fault_bit(iommu);
1745 if ( iommu_enable_translation(iommu) )
1746 return -EIO;
1749 return 0;
1752 static void setup_dom0_rmrr(void)
1754 struct acpi_rmrr_unit *rmrr;
1755 struct pci_dev *pdev;
1756 int ret;
1758 for_each_rmrr_device ( rmrr, pdev )
1759 ret = iommu_prepare_rmrr_dev(dom0, rmrr, pdev);
1761 if ( ret )
1762 gdprintk(XENLOG_ERR VTDPREFIX,
1763 "IOMMU: mapping reserved region failed\n");
1765 end_for_each_rmrr_device ( rmrr, pdev )
1768 int iommu_setup(void)
1770 struct hvm_iommu *hd = domain_hvm_iommu(dom0);
1771 struct acpi_drhd_unit *drhd;
1772 struct iommu *iommu;
1773 unsigned long i;
1775 if ( !vtd_enabled )
1776 return 0;
1778 INIT_LIST_HEAD(&hd->pdev_list);
1780 /* start from scratch */
1781 iommu_flush_all();
1783 /* setup clflush size */
1784 x86_clflush_size = ((cpuid_ebx(1) >> 8) & 0xff) * 8;
1786 /*
1787 * allocate IO page directory page for the domain.
1788 */
1789 drhd = list_entry(acpi_drhd_units.next, typeof(*drhd), list);
1790 iommu = drhd->iommu;
1792 /* setup 1:1 page table for dom0 */
1793 for ( i = 0; i < max_page; i++ )
1794 iommu_map_page(dom0, i, i);
1796 if ( init_vtd_hw() )
1797 goto error;
1798 setup_dom0_devices();
1799 setup_dom0_rmrr();
1800 if ( enable_vtd_translation() )
1801 goto error;
1803 return 0;
1805 error:
1806 printk("iommu_setup() failed\n");
1807 for_each_drhd_unit ( drhd )
1809 iommu = drhd->iommu;
1810 free_iommu(iommu);
1812 return -EIO;
1815 /*
1816 * If the device isn't owned by dom0, it means it already
1817 * has been assigned to other domain, or it's not exist.
1818 */
1819 int device_assigned(u8 bus, u8 devfn)
1821 struct pci_dev *pdev;
1823 for_each_pdev( dom0, pdev )
1824 if ( (pdev->bus == bus ) && (pdev->devfn == devfn) )
1825 return 0;
1827 return 1;
1830 int assign_device(struct domain *d, u8 bus, u8 devfn)
1832 struct hvm_iommu *hd = domain_hvm_iommu(d);
1833 struct acpi_rmrr_unit *rmrr;
1834 struct pci_dev *pdev;
1835 int ret = 0;
1837 if ( list_empty(&acpi_drhd_units) )
1838 return ret;
1840 dprintk(XENLOG_INFO VTDPREFIX,
1841 "assign_device: bus = %x dev = %x func = %x\n",
1842 bus, PCI_SLOT(devfn), PCI_FUNC(devfn));
1844 reassign_device_ownership(dom0, d, bus, devfn);
1846 /* setup rmrr identify mapping just once per domain */
1847 if ( list_empty(&hd->pdev_list) )
1848 for_each_rmrr_device(rmrr, pdev)
1849 ret = iommu_prepare_rmrr_dev(d, rmrr, pdev);
1850 if ( ret )
1851 gdprintk(XENLOG_ERR VTDPREFIX,
1852 "IOMMU: mapping reserved region failed\n");
1853 end_for_each_rmrr_device(rmrr, pdev)
1854 return ret;
1857 void iommu_set_pgd(struct domain *d)
1859 struct hvm_iommu *hd = domain_hvm_iommu(d);
1860 unsigned long p2m_table;
1862 if ( hd->pgd )
1864 gdprintk(XENLOG_INFO VTDPREFIX,
1865 "iommu_set_pgd_1: hd->pgd = %p\n", hd->pgd);
1866 hd->pgd = NULL;
1868 p2m_table = mfn_x(pagetable_get_mfn(d->arch.phys_table));
1870 #if CONFIG_PAGING_LEVELS == 3
1871 if ( !hd->pgd )
1873 int level = agaw_to_level(hd->agaw);
1874 struct dma_pte *pmd = NULL;
1875 struct dma_pte *pgd = NULL;
1876 struct dma_pte *pte = NULL;
1877 l3_pgentry_t *l3e;
1878 unsigned long flags;
1879 int i;
1881 spin_lock_irqsave(&hd->mapping_lock, flags);
1882 if ( !hd->pgd )
1884 pgd = (struct dma_pte *)alloc_xenheap_page();
1885 memset((u8*)pgd, 0, PAGE_SIZE);
1886 if ( !hd->pgd )
1887 hd->pgd = pgd;
1888 else /* somebody is fast */
1889 free_xenheap_page((void *) pgd);
1892 l3e = map_domain_page(p2m_table);
1893 switch ( level )
1895 case VTD_PAGE_TABLE_LEVEL_3: /* Weybridge */
1896 /* We only support 8 entries for the PAE L3 p2m table */
1897 for ( i = 0; i < 8 ; i++ )
1899 /* Don't create new L2 entry, use ones from p2m table */
1900 pgd[i].val = l3e[i].l3 | _PAGE_PRESENT | _PAGE_RW;
1902 break;
1904 case VTD_PAGE_TABLE_LEVEL_4: /* Stoakley */
1905 /* We allocate one more page for the top vtd page table. */
1906 pmd = (struct dma_pte *)alloc_xenheap_page();
1907 memset((u8*)pmd, 0, PAGE_SIZE);
1908 pte = &pgd[0];
1909 dma_set_pte_addr(*pte, virt_to_maddr(pmd));
1910 dma_set_pte_readable(*pte);
1911 dma_set_pte_writable(*pte);
1913 for ( i = 0; i < 8; i++ )
1915 /* Don't create new L2 entry, use ones from p2m table */
1916 pmd[i].val = l3e[i].l3 | _PAGE_PRESENT | _PAGE_RW;
1918 break;
1919 default:
1920 gdprintk(XENLOG_ERR VTDPREFIX,
1921 "iommu_set_pgd:Unsupported p2m table sharing level!\n");
1922 break;
1924 unmap_domain_page(l3e);
1925 spin_unlock_irqrestore(&hd->mapping_lock, flags);
1927 #elif CONFIG_PAGING_LEVELS == 4
1928 if ( !hd->pgd )
1930 int level = agaw_to_level(hd->agaw);
1931 l3_pgentry_t *l3e;
1932 mfn_t pgd_mfn;
1934 switch ( level )
1936 case VTD_PAGE_TABLE_LEVEL_3:
1937 l3e = map_domain_page(p2m_table);
1938 if ( (l3e_get_flags(*l3e) & _PAGE_PRESENT) == 0 )
1940 gdprintk(XENLOG_ERR VTDPREFIX,
1941 "iommu_set_pgd: second level wasn't there\n");
1942 unmap_domain_page(l3e);
1943 return;
1945 pgd_mfn = _mfn(l3e_get_pfn(*l3e));
1946 unmap_domain_page(l3e);
1947 hd->pgd = maddr_to_virt(pagetable_get_paddr(
1948 pagetable_from_mfn(pgd_mfn)));
1949 break;
1951 case VTD_PAGE_TABLE_LEVEL_4:
1952 pgd_mfn = _mfn(p2m_table);
1953 hd->pgd = maddr_to_virt(pagetable_get_paddr(
1954 pagetable_from_mfn(pgd_mfn)));
1955 break;
1956 default:
1957 gdprintk(XENLOG_ERR VTDPREFIX,
1958 "iommu_set_pgd:Unsupported p2m table sharing level!\n");
1959 break;
1962 #endif
1963 gdprintk(XENLOG_INFO VTDPREFIX,
1964 "iommu_set_pgd: hd->pgd = %p\n", hd->pgd);
1968 u8 iommu_state[MAX_IOMMU_REGS * MAX_IOMMUS];
1969 int iommu_suspend(void)
1971 struct acpi_drhd_unit *drhd;
1972 struct iommu *iommu;
1973 int i = 0;
1975 iommu_flush_all();
1977 for_each_drhd_unit ( drhd )
1979 iommu = drhd->iommu;
1980 iommu_state[DMAR_RTADDR_REG * i] =
1981 (u64) dmar_readq(iommu->reg, DMAR_RTADDR_REG);
1982 iommu_state[DMAR_FECTL_REG * i] =
1983 (u32) dmar_readl(iommu->reg, DMAR_FECTL_REG);
1984 iommu_state[DMAR_FEDATA_REG * i] =
1985 (u32) dmar_readl(iommu->reg, DMAR_FEDATA_REG);
1986 iommu_state[DMAR_FEADDR_REG * i] =
1987 (u32) dmar_readl(iommu->reg, DMAR_FEADDR_REG);
1988 iommu_state[DMAR_FEUADDR_REG * i] =
1989 (u32) dmar_readl(iommu->reg, DMAR_FEUADDR_REG);
1990 iommu_state[DMAR_PLMBASE_REG * i] =
1991 (u32) dmar_readl(iommu->reg, DMAR_PLMBASE_REG);
1992 iommu_state[DMAR_PLMLIMIT_REG * i] =
1993 (u32) dmar_readl(iommu->reg, DMAR_PLMLIMIT_REG);
1994 iommu_state[DMAR_PHMBASE_REG * i] =
1995 (u64) dmar_readq(iommu->reg, DMAR_PHMBASE_REG);
1996 iommu_state[DMAR_PHMLIMIT_REG * i] =
1997 (u64) dmar_readq(iommu->reg, DMAR_PHMLIMIT_REG);
1998 i++;
2001 return 0;
2004 int iommu_resume(void)
2006 struct acpi_drhd_unit *drhd;
2007 struct iommu *iommu;
2008 int i = 0;
2010 iommu_flush_all();
2012 init_vtd_hw();
2013 for_each_drhd_unit ( drhd )
2015 iommu = drhd->iommu;
2016 dmar_writeq( iommu->reg, DMAR_RTADDR_REG,
2017 (u64) iommu_state[DMAR_RTADDR_REG * i]);
2018 dmar_writel(iommu->reg, DMAR_FECTL_REG,
2019 (u32) iommu_state[DMAR_FECTL_REG * i]);
2020 dmar_writel(iommu->reg, DMAR_FEDATA_REG,
2021 (u32) iommu_state[DMAR_FEDATA_REG * i]);
2022 dmar_writel(iommu->reg, DMAR_FEADDR_REG,
2023 (u32) iommu_state[DMAR_FEADDR_REG * i]);
2024 dmar_writel(iommu->reg, DMAR_FEUADDR_REG,
2025 (u32) iommu_state[DMAR_FEUADDR_REG * i]);
2026 dmar_writel(iommu->reg, DMAR_PLMBASE_REG,
2027 (u32) iommu_state[DMAR_PLMBASE_REG * i]);
2028 dmar_writel(iommu->reg, DMAR_PLMLIMIT_REG,
2029 (u32) iommu_state[DMAR_PLMLIMIT_REG * i]);
2030 dmar_writeq(iommu->reg, DMAR_PHMBASE_REG,
2031 (u64) iommu_state[DMAR_PHMBASE_REG * i]);
2032 dmar_writeq(iommu->reg, DMAR_PHMLIMIT_REG,
2033 (u64) iommu_state[DMAR_PHMLIMIT_REG * i]);
2035 if ( iommu_enable_translation(iommu) )
2036 return -EIO;
2037 i++;
2039 return 0;
2042 /*
2043 * Local variables:
2044 * mode: C
2045 * c-set-style: "BSD"
2046 * c-basic-offset: 4
2047 * tab-width: 4
2048 * indent-tabs-mode: nil
2049 * End:
2050 */