debuggers.hg

view xen/arch/x86/hvm/vmx/vtd/intel-iommu.c @ 16736:f983aa8e4b26

vt-d: Fix print_vtd_entries walk VTd mapping table.

DMA request to above guest physical memory will cause VTd fault, in
which print_vtd_entries() tries to walk VTd mapping table. However,
during walking, current Xen code didn't check if the PTE is valid and
may access to invalid memory address.

Signed-off-by: Xiaowei Yang <xiaowei.yang@intel.com>
author Keir Fraser <keir.fraser@citrix.com>
date Wed Jan 09 10:35:52 2008 +0000 (2008-01-09)
parents c0a9b8ab631c
children cc5bb500df5f
line source
1 /*
2 * Copyright (c) 2006, Intel Corporation.
3 *
4 * This program is free software; you can redistribute it and/or modify it
5 * under the terms and conditions of the GNU General Public License,
6 * version 2, as published by the Free Software Foundation.
7 *
8 * This program is distributed in the hope it will be useful, but WITHOUT
9 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
10 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
11 * more details.
12 *
13 * You should have received a copy of the GNU General Public License along with
14 * this program; if not, write to the Free Software Foundation, Inc., 59 Temple
15 * Place - Suite 330, Boston, MA 02111-1307 USA.
16 *
17 * Copyright (C) Ashok Raj <ashok.raj@intel.com>
18 * Copyright (C) Shaohua Li <shaohua.li@intel.com>
19 * Copyright (C) Allen Kay <allen.m.kay@intel.com> - adapted to xen
20 */
22 #include <xen/init.h>
23 #include <xen/irq.h>
24 #include <xen/spinlock.h>
25 #include <xen/sched.h>
26 #include <xen/xmalloc.h>
27 #include <xen/domain_page.h>
28 #include <asm/delay.h>
29 #include <asm/string.h>
30 #include <asm/mm.h>
31 #include <asm/iommu.h>
32 #include <asm/hvm/vmx/intel-iommu.h>
33 #include "dmar.h"
34 #include "pci-direct.h"
35 #include "pci_regs.h"
36 #include "msi.h"
38 #define domain_iommu_domid(d) ((d)->arch.hvm_domain.hvm_iommu.iommu_domid)
40 extern void print_iommu_regs(struct acpi_drhd_unit *drhd);
41 extern void print_vtd_entries(struct domain *d, int bus, int devfn,
42 unsigned long gmfn);
44 static spinlock_t domid_bitmap_lock; /* protect domain id bitmap */
45 static int domid_bitmap_size; /* domain id bitmap size in bit */
46 static void *domid_bitmap; /* iommu domain id bitmap */
48 #define DID_FIELD_WIDTH 16
49 #define DID_HIGH_OFFSET 8
50 static void context_set_domain_id(struct context_entry *context,
51 struct domain *d)
52 {
53 unsigned long flags;
54 domid_t iommu_domid = domain_iommu_domid(d);
56 if ( iommu_domid == 0 )
57 {
58 spin_lock_irqsave(&domid_bitmap_lock, flags);
59 iommu_domid = find_first_zero_bit(domid_bitmap, domid_bitmap_size);
60 set_bit(iommu_domid, domid_bitmap);
61 spin_unlock_irqrestore(&domid_bitmap_lock, flags);
62 d->arch.hvm_domain.hvm_iommu.iommu_domid = iommu_domid;
63 }
65 context->hi &= (1 << DID_HIGH_OFFSET) - 1;
66 context->hi |= iommu_domid << DID_HIGH_OFFSET;
67 }
69 static void iommu_domid_release(struct domain *d)
70 {
71 domid_t iommu_domid = domain_iommu_domid(d);
73 if ( iommu_domid != 0 )
74 {
75 d->arch.hvm_domain.hvm_iommu.iommu_domid = 0;
76 clear_bit(iommu_domid, domid_bitmap);
77 }
78 }
80 unsigned int x86_clflush_size;
81 void clflush_cache_range(void *adr, int size)
82 {
83 int i;
84 for ( i = 0; i < size; i += x86_clflush_size )
85 clflush(adr + i);
86 }
88 static void __iommu_flush_cache(struct iommu *iommu, void *addr, int size)
89 {
90 if ( !ecap_coherent(iommu->ecap) )
91 clflush_cache_range(addr, size);
92 }
94 #define iommu_flush_cache_entry(iommu, addr) \
95 __iommu_flush_cache(iommu, addr, 8)
96 #define iommu_flush_cache_page(iommu, addr) \
97 __iommu_flush_cache(iommu, addr, PAGE_SIZE_4K)
99 int nr_iommus;
100 /* context entry handling */
101 static struct context_entry * device_to_context_entry(struct iommu *iommu,
102 u8 bus, u8 devfn)
103 {
104 struct root_entry *root;
105 struct context_entry *context;
106 unsigned long phy_addr;
107 unsigned long flags;
109 spin_lock_irqsave(&iommu->lock, flags);
110 root = &iommu->root_entry[bus];
111 if ( !root_present(*root) )
112 {
113 phy_addr = (unsigned long) alloc_xenheap_page();
114 if ( !phy_addr )
115 {
116 spin_unlock_irqrestore(&iommu->lock, flags);
117 return NULL;
118 }
119 memset((void *) phy_addr, 0, PAGE_SIZE);
120 iommu_flush_cache_page(iommu, (void *)phy_addr);
121 phy_addr = virt_to_maddr((void *)phy_addr);
122 set_root_value(*root, phy_addr);
123 set_root_present(*root);
124 iommu_flush_cache_entry(iommu, root);
125 }
126 phy_addr = (unsigned long) get_context_addr(*root);
127 context = (struct context_entry *)maddr_to_virt(phy_addr);
128 spin_unlock_irqrestore(&iommu->lock, flags);
129 return &context[devfn];
130 }
132 static int device_context_mapped(struct iommu *iommu, u8 bus, u8 devfn)
133 {
134 struct root_entry *root;
135 struct context_entry *context;
136 unsigned long phy_addr;
137 int ret;
138 unsigned long flags;
140 spin_lock_irqsave(&iommu->lock, flags);
141 root = &iommu->root_entry[bus];
142 if ( !root_present(*root) )
143 {
144 ret = 0;
145 goto out;
146 }
147 phy_addr = get_context_addr(*root);
148 context = (struct context_entry *)maddr_to_virt(phy_addr);
149 ret = context_present(context[devfn]);
150 out:
151 spin_unlock_irqrestore(&iommu->lock, flags);
152 return ret;
153 }
155 static struct page_info *addr_to_dma_page(struct domain *domain, u64 addr)
156 {
157 struct hvm_iommu *hd = domain_hvm_iommu(domain);
158 struct acpi_drhd_unit *drhd;
159 struct iommu *iommu;
160 int addr_width = agaw_to_width(hd->agaw);
161 struct dma_pte *parent, *pte = NULL, *pgd;
162 int level = agaw_to_level(hd->agaw);
163 int offset;
164 unsigned long flags;
165 struct page_info *pg = NULL;
166 u64 *vaddr = NULL;
168 drhd = list_entry(acpi_drhd_units.next, typeof(*drhd), list);
169 iommu = drhd->iommu;
171 addr &= (((u64)1) << addr_width) - 1;
172 spin_lock_irqsave(&hd->mapping_lock, flags);
173 if ( !hd->pgd )
174 {
175 pgd = (struct dma_pte *)alloc_xenheap_page();
176 if ( !pgd )
177 {
178 spin_unlock_irqrestore(&hd->mapping_lock, flags);
179 return NULL;
180 }
181 memset(pgd, 0, PAGE_SIZE);
182 hd->pgd = pgd;
183 }
185 parent = hd->pgd;
186 while ( level > 1 )
187 {
188 offset = address_level_offset(addr, level);
189 pte = &parent[offset];
191 if ( dma_pte_addr(*pte) == 0 )
192 {
193 pg = alloc_domheap_page(NULL);
194 vaddr = map_domain_page(page_to_mfn(pg));
195 if ( !vaddr )
196 {
197 spin_unlock_irqrestore(&hd->mapping_lock, flags);
198 return NULL;
199 }
200 memset(vaddr, 0, PAGE_SIZE);
201 iommu_flush_cache_page(iommu, vaddr);
203 dma_set_pte_addr(*pte, page_to_maddr(pg));
205 /*
206 * high level table always sets r/w, last level
207 * page table control read/write
208 */
209 dma_set_pte_readable(*pte);
210 dma_set_pte_writable(*pte);
211 iommu_flush_cache_entry(iommu, pte);
212 }
213 else
214 {
215 pg = maddr_to_page(pte->val);
216 vaddr = map_domain_page(page_to_mfn(pg));
217 if ( !vaddr )
218 {
219 spin_unlock_irqrestore(&hd->mapping_lock, flags);
220 return NULL;
221 }
222 }
224 if ( parent != hd->pgd )
225 unmap_domain_page(parent);
227 if ( level == 2 && vaddr )
228 {
229 unmap_domain_page(vaddr);
230 break;
231 }
233 parent = (struct dma_pte *)vaddr;
234 vaddr = NULL;
235 level--;
236 }
238 spin_unlock_irqrestore(&hd->mapping_lock, flags);
239 return pg;
240 }
242 /* return address's page at specific level */
243 static struct page_info *dma_addr_level_page(struct domain *domain,
244 u64 addr, int level)
245 {
246 struct hvm_iommu *hd = domain_hvm_iommu(domain);
247 struct dma_pte *parent, *pte = NULL;
248 int total = agaw_to_level(hd->agaw);
249 int offset;
250 struct page_info *pg = NULL;
252 parent = hd->pgd;
253 while ( level <= total )
254 {
255 offset = address_level_offset(addr, total);
256 pte = &parent[offset];
257 if ( dma_pte_addr(*pte) == 0 )
258 {
259 if ( parent != hd->pgd )
260 unmap_domain_page(parent);
261 break;
262 }
264 pg = maddr_to_page(pte->val);
265 if ( parent != hd->pgd )
266 unmap_domain_page(parent);
268 if ( level == total )
269 return pg;
271 parent = map_domain_page(page_to_mfn(pg));
272 total--;
273 }
275 return NULL;
276 }
278 static void iommu_flush_write_buffer(struct iommu *iommu)
279 {
280 u32 val;
281 unsigned long flag;
282 unsigned long start_time;
284 if ( !cap_rwbf(iommu->cap) )
285 return;
286 val = iommu->gcmd | DMA_GCMD_WBF;
288 spin_lock_irqsave(&iommu->register_lock, flag);
289 dmar_writel(iommu->reg, DMAR_GCMD_REG, val);
291 /* Make sure hardware complete it */
292 start_time = jiffies;
293 for ( ; ; )
294 {
295 val = dmar_readl(iommu->reg, DMAR_GSTS_REG);
296 if ( !(val & DMA_GSTS_WBFS) )
297 break;
298 if ( time_after(jiffies, start_time + DMAR_OPERATION_TIMEOUT) )
299 panic("DMAR hardware is malfunctional,"
300 " please disable IOMMU\n");
301 cpu_relax();
302 }
303 spin_unlock_irqrestore(&iommu->register_lock, flag);
304 }
306 /* return value determine if we need a write buffer flush */
307 static int __iommu_flush_context(
308 struct iommu *iommu,
309 u16 did, u16 source_id, u8 function_mask, u64 type,
310 int non_present_entry_flush)
311 {
312 u64 val = 0;
313 unsigned long flag;
314 unsigned long start_time;
316 /*
317 * In the non-present entry flush case, if hardware doesn't cache
318 * non-present entry we do nothing and if hardware cache non-present
319 * entry, we flush entries of domain 0 (the domain id is used to cache
320 * any non-present entries)
321 */
322 if ( non_present_entry_flush )
323 {
324 if ( !cap_caching_mode(iommu->cap) )
325 return 1;
326 else
327 did = 0;
328 }
330 /* use register invalidation */
331 switch ( type )
332 {
333 case DMA_CCMD_GLOBAL_INVL:
334 val = DMA_CCMD_GLOBAL_INVL;
335 break;
336 case DMA_CCMD_DOMAIN_INVL:
337 val = DMA_CCMD_DOMAIN_INVL|DMA_CCMD_DID(did);
338 break;
339 case DMA_CCMD_DEVICE_INVL:
340 val = DMA_CCMD_DEVICE_INVL|DMA_CCMD_DID(did)
341 |DMA_CCMD_SID(source_id)|DMA_CCMD_FM(function_mask);
342 break;
343 default:
344 BUG();
345 }
346 val |= DMA_CCMD_ICC;
348 spin_lock_irqsave(&iommu->register_lock, flag);
349 dmar_writeq(iommu->reg, DMAR_CCMD_REG, val);
351 /* Make sure hardware complete it */
352 start_time = jiffies;
353 for ( ; ; )
354 {
355 val = dmar_readq(iommu->reg, DMAR_CCMD_REG);
356 if ( !(val & DMA_CCMD_ICC) )
357 break;
358 if ( time_after(jiffies, start_time + DMAR_OPERATION_TIMEOUT) )
359 panic("DMAR hardware is malfunctional, please disable IOMMU\n");
360 cpu_relax();
361 }
362 spin_unlock_irqrestore(&iommu->register_lock, flag);
363 /* flush context entry will implictly flush write buffer */
364 return 0;
365 }
367 static int inline iommu_flush_context_global(
368 struct iommu *iommu, int non_present_entry_flush)
369 {
370 return __iommu_flush_context(iommu, 0, 0, 0, DMA_CCMD_GLOBAL_INVL,
371 non_present_entry_flush);
372 }
374 static int inline iommu_flush_context_domain(
375 struct iommu *iommu, u16 did, int non_present_entry_flush)
376 {
377 return __iommu_flush_context(iommu, did, 0, 0, DMA_CCMD_DOMAIN_INVL,
378 non_present_entry_flush);
379 }
381 static int inline iommu_flush_context_device(
382 struct iommu *iommu, u16 did, u16 source_id,
383 u8 function_mask, int non_present_entry_flush)
384 {
385 return __iommu_flush_context(iommu, did, source_id, function_mask,
386 DMA_CCMD_DEVICE_INVL,
387 non_present_entry_flush);
388 }
390 /* return value determine if we need a write buffer flush */
391 static int __iommu_flush_iotlb(struct iommu *iommu, u16 did,
392 u64 addr, unsigned int size_order, u64 type,
393 int non_present_entry_flush)
394 {
395 int tlb_offset = ecap_iotlb_offset(iommu->ecap);
396 u64 val = 0, val_iva = 0;
397 unsigned long flag;
398 unsigned long start_time;
400 /*
401 * In the non-present entry flush case, if hardware doesn't cache
402 * non-present entry we do nothing and if hardware cache non-present
403 * entry, we flush entries of domain 0 (the domain id is used to cache
404 * any non-present entries)
405 */
406 if ( non_present_entry_flush )
407 {
408 if ( !cap_caching_mode(iommu->cap) )
409 return 1;
410 else
411 did = 0;
412 }
414 /* use register invalidation */
415 switch ( type )
416 {
417 case DMA_TLB_GLOBAL_FLUSH:
418 /* global flush doesn't need set IVA_REG */
419 val = DMA_TLB_GLOBAL_FLUSH|DMA_TLB_IVT;
420 break;
421 case DMA_TLB_DSI_FLUSH:
422 val = DMA_TLB_DSI_FLUSH|DMA_TLB_IVT|DMA_TLB_DID(did);
423 break;
424 case DMA_TLB_PSI_FLUSH:
425 val = DMA_TLB_PSI_FLUSH|DMA_TLB_IVT|DMA_TLB_DID(did);
426 /* Note: always flush non-leaf currently */
427 val_iva = size_order | addr;
428 break;
429 default:
430 BUG();
431 }
432 /* Note: set drain read/write */
433 if ( cap_read_drain(iommu->cap) )
434 val |= DMA_TLB_READ_DRAIN;
435 if ( cap_write_drain(iommu->cap) )
436 val |= DMA_TLB_WRITE_DRAIN;
438 spin_lock_irqsave(&iommu->register_lock, flag);
439 /* Note: Only uses first TLB reg currently */
440 if ( val_iva )
441 dmar_writeq(iommu->reg, tlb_offset, val_iva);
442 dmar_writeq(iommu->reg, tlb_offset + 8, val);
444 /* Make sure hardware complete it */
445 start_time = jiffies;
446 for ( ; ; )
447 {
448 val = dmar_readq(iommu->reg, tlb_offset + 8);
449 if ( !(val & DMA_TLB_IVT) )
450 break;
451 if ( time_after(jiffies, start_time + DMAR_OPERATION_TIMEOUT) )
452 panic("DMAR hardware is malfunctional, please disable IOMMU\n");
453 cpu_relax();
454 }
455 spin_unlock_irqrestore(&iommu->register_lock, flag);
457 /* check IOTLB invalidation granularity */
458 if ( DMA_TLB_IAIG(val) == 0 )
459 printk(KERN_ERR VTDPREFIX "IOMMU: flush IOTLB failed\n");
460 if ( DMA_TLB_IAIG(val) != DMA_TLB_IIRG(type) )
461 printk(KERN_ERR VTDPREFIX "IOMMU: tlb flush request %x, actual %x\n",
462 (u32)DMA_TLB_IIRG(type), (u32)DMA_TLB_IAIG(val));
463 /* flush context entry will implictly flush write buffer */
464 return 0;
465 }
467 static int inline iommu_flush_iotlb_global(struct iommu *iommu,
468 int non_present_entry_flush)
469 {
470 return __iommu_flush_iotlb(iommu, 0, 0, 0, DMA_TLB_GLOBAL_FLUSH,
471 non_present_entry_flush);
472 }
474 static int inline iommu_flush_iotlb_dsi(struct iommu *iommu, u16 did,
475 int non_present_entry_flush)
476 {
477 return __iommu_flush_iotlb(iommu, did, 0, 0, DMA_TLB_DSI_FLUSH,
478 non_present_entry_flush);
479 }
481 static int inline get_alignment(u64 base, unsigned int size)
482 {
483 int t = 0;
484 u64 end;
486 end = base + size - 1;
487 while ( base != end )
488 {
489 t++;
490 base >>= 1;
491 end >>= 1;
492 }
493 return t;
494 }
496 static int inline iommu_flush_iotlb_psi(
497 struct iommu *iommu, u16 did,
498 u64 addr, unsigned int pages, int non_present_entry_flush)
499 {
500 unsigned int align;
502 BUG_ON(addr & (~PAGE_MASK_4K));
503 BUG_ON(pages == 0);
505 /* Fallback to domain selective flush if no PSI support */
506 if ( !cap_pgsel_inv(iommu->cap) )
507 return iommu_flush_iotlb_dsi(iommu, did,
508 non_present_entry_flush);
510 /*
511 * PSI requires page size is 2 ^ x, and the base address is naturally
512 * aligned to the size
513 */
514 align = get_alignment(addr >> PAGE_SHIFT_4K, pages);
515 /* Fallback to domain selective flush if size is too big */
516 if ( align > cap_max_amask_val(iommu->cap) )
517 return iommu_flush_iotlb_dsi(iommu, did,
518 non_present_entry_flush);
520 addr >>= PAGE_SHIFT_4K + align;
521 addr <<= PAGE_SHIFT_4K + align;
523 return __iommu_flush_iotlb(iommu, did, addr, align,
524 DMA_TLB_PSI_FLUSH, non_present_entry_flush);
525 }
527 void iommu_flush_all(void)
528 {
529 struct acpi_drhd_unit *drhd;
530 struct iommu *iommu;
532 wbinvd();
533 for_each_drhd_unit ( drhd )
534 {
535 iommu = drhd->iommu;
536 iommu_flush_context_global(iommu, 0);
537 iommu_flush_iotlb_global(iommu, 0);
538 }
539 }
541 /* clear one page's page table */
542 static void dma_pte_clear_one(struct domain *domain, u64 addr)
543 {
544 struct acpi_drhd_unit *drhd;
545 struct iommu *iommu;
546 struct dma_pte *pte = NULL;
547 struct page_info *pg = NULL;
549 drhd = list_entry(acpi_drhd_units.next, typeof(*drhd), list);
551 /* get last level pte */
552 pg = dma_addr_level_page(domain, addr, 1);
553 if ( !pg )
554 return;
555 pte = (struct dma_pte *)map_domain_page(page_to_mfn(pg));
556 pte += address_level_offset(addr, 1);
557 if ( pte )
558 {
559 dma_clear_pte(*pte);
560 iommu_flush_cache_entry(drhd->iommu, pte);
562 for_each_drhd_unit ( drhd )
563 {
564 iommu = drhd->iommu;
565 if ( cap_caching_mode(iommu->cap) )
566 iommu_flush_iotlb_psi(iommu, domain_iommu_domid(domain),
567 addr, 1, 0);
568 else if (cap_rwbf(iommu->cap))
569 iommu_flush_write_buffer(iommu);
570 }
571 }
572 unmap_domain_page(pte);
573 }
575 /* clear last level pte, a tlb flush should be followed */
576 static void dma_pte_clear_range(struct domain *domain, u64 start, u64 end)
577 {
578 struct hvm_iommu *hd = domain_hvm_iommu(domain);
579 int addr_width = agaw_to_width(hd->agaw);
581 start &= (((u64)1) << addr_width) - 1;
582 end &= (((u64)1) << addr_width) - 1;
583 /* in case it's partial page */
584 start = PAGE_ALIGN_4K(start);
585 end &= PAGE_MASK_4K;
587 /* we don't need lock here, nobody else touches the iova range */
588 while ( start < end )
589 {
590 dma_pte_clear_one(domain, start);
591 start += PAGE_SIZE_4K;
592 }
593 }
595 /* free page table pages. last level pte should already be cleared */
596 void dma_pte_free_pagetable(struct domain *domain, u64 start, u64 end)
597 {
598 struct acpi_drhd_unit *drhd;
599 struct hvm_iommu *hd = domain_hvm_iommu(domain);
600 struct iommu *iommu;
601 int addr_width = agaw_to_width(hd->agaw);
602 struct dma_pte *pte;
603 int total = agaw_to_level(hd->agaw);
604 int level;
605 u32 tmp;
606 struct page_info *pg = NULL;
608 drhd = list_entry(acpi_drhd_units.next, typeof(*drhd), list);
609 iommu = drhd->iommu;
611 start &= (((u64)1) << addr_width) - 1;
612 end &= (((u64)1) << addr_width) - 1;
614 /* we don't need lock here, nobody else touches the iova range */
615 level = 2;
616 while ( level <= total )
617 {
618 tmp = align_to_level(start, level);
619 if ( (tmp >= end) || ((tmp + level_size(level)) > end) )
620 return;
622 while ( tmp < end )
623 {
624 pg = dma_addr_level_page(domain, tmp, level);
625 if ( !pg )
626 return;
627 pte = (struct dma_pte *)map_domain_page(page_to_mfn(pg));
628 pte += address_level_offset(tmp, level);
629 dma_clear_pte(*pte);
630 iommu_flush_cache_entry(iommu, pte);
631 unmap_domain_page(pte);
632 free_domheap_page(pg);
634 tmp += level_size(level);
635 }
636 level++;
637 }
639 /* free pgd */
640 if ( start == 0 && end == ((((u64)1) << addr_width) - 1) )
641 {
642 free_xenheap_page((void *)hd->pgd);
643 hd->pgd = NULL;
644 }
645 }
647 /* iommu handling */
648 static int iommu_set_root_entry(struct iommu *iommu)
649 {
650 void *addr;
651 u32 cmd, sts;
652 struct root_entry *root;
653 unsigned long flags;
655 if ( iommu == NULL )
656 {
657 gdprintk(XENLOG_ERR VTDPREFIX,
658 "iommu_set_root_entry: iommu == NULL\n");
659 return -EINVAL;
660 }
662 if ( unlikely(!iommu->root_entry) )
663 {
664 root = (struct root_entry *)alloc_xenheap_page();
665 if ( root == NULL )
666 return -ENOMEM;
668 memset((u8*)root, 0, PAGE_SIZE);
669 iommu_flush_cache_page(iommu, root);
671 if ( cmpxchg((unsigned long *)&iommu->root_entry,
672 0, (unsigned long)root) != 0 )
673 free_xenheap_page((void *)root);
674 }
676 addr = iommu->root_entry;
678 spin_lock_irqsave(&iommu->register_lock, flags);
680 dmar_writeq(iommu->reg, DMAR_RTADDR_REG, virt_to_maddr(addr));
681 cmd = iommu->gcmd | DMA_GCMD_SRTP;
682 dmar_writel(iommu->reg, DMAR_GCMD_REG, cmd);
684 /* Make sure hardware complete it */
685 for ( ; ; )
686 {
687 sts = dmar_readl(iommu->reg, DMAR_GSTS_REG);
688 if ( sts & DMA_GSTS_RTPS )
689 break;
690 cpu_relax();
691 }
693 spin_unlock_irqrestore(&iommu->register_lock, flags);
695 return 0;
696 }
698 static int iommu_enable_translation(struct iommu *iommu)
699 {
700 u32 sts;
701 unsigned long flags;
703 dprintk(XENLOG_INFO VTDPREFIX,
704 "iommu_enable_translation: enabling vt-d translation\n");
705 spin_lock_irqsave(&iommu->register_lock, flags);
706 iommu->gcmd |= DMA_GCMD_TE;
707 dmar_writel(iommu->reg, DMAR_GCMD_REG, iommu->gcmd);
708 /* Make sure hardware complete it */
709 for ( ; ; )
710 {
711 sts = dmar_readl(iommu->reg, DMAR_GSTS_REG);
712 if ( sts & DMA_GSTS_TES )
713 break;
714 cpu_relax();
715 }
717 /* Disable PMRs when VT-d engine takes effect per spec definition */
718 disable_pmr(iommu);
719 spin_unlock_irqrestore(&iommu->register_lock, flags);
720 return 0;
721 }
723 int iommu_disable_translation(struct iommu *iommu)
724 {
725 u32 sts;
726 unsigned long flags;
728 spin_lock_irqsave(&iommu->register_lock, flags);
729 iommu->gcmd &= ~ DMA_GCMD_TE;
730 dmar_writel(iommu->reg, DMAR_GCMD_REG, iommu->gcmd);
732 /* Make sure hardware complete it */
733 for ( ; ; )
734 {
735 sts = dmar_readl(iommu->reg, DMAR_GSTS_REG);
736 if ( !(sts & DMA_GSTS_TES) )
737 break;
738 cpu_relax();
739 }
740 spin_unlock_irqrestore(&iommu->register_lock, flags);
741 return 0;
742 }
744 static struct iommu *vector_to_iommu[NR_VECTORS];
745 static int iommu_page_fault_do_one(struct iommu *iommu, int type,
746 u8 fault_reason, u16 source_id, u32 addr)
747 {
748 dprintk(XENLOG_WARNING VTDPREFIX,
749 "iommu_page_fault:%s: DEVICE %x:%x.%x addr %x REASON %x\n",
750 (type ? "DMA Read" : "DMA Write"),
751 (source_id >> 8), PCI_SLOT(source_id & 0xFF),
752 PCI_FUNC(source_id & 0xFF), addr, fault_reason);
754 print_vtd_entries(current->domain, (source_id >> 8),(source_id & 0xff),
755 (addr >> PAGE_SHIFT));
756 return 0;
757 }
759 #define PRIMARY_FAULT_REG_LEN (16)
760 static void iommu_page_fault(int vector, void *dev_id,
761 struct cpu_user_regs *regs)
762 {
763 struct iommu *iommu = dev_id;
764 int reg, fault_index;
765 u32 fault_status;
766 unsigned long flags;
768 dprintk(XENLOG_WARNING VTDPREFIX,
769 "iommu_page_fault: iommu->reg = %p\n", iommu->reg);
771 spin_lock_irqsave(&iommu->register_lock, flags);
772 fault_status = dmar_readl(iommu->reg, DMAR_FSTS_REG);
773 spin_unlock_irqrestore(&iommu->register_lock, flags);
775 /* FIXME: ignore advanced fault log */
776 if ( !(fault_status & DMA_FSTS_PPF) )
777 return;
778 fault_index = dma_fsts_fault_record_index(fault_status);
779 reg = cap_fault_reg_offset(iommu->cap);
780 for ( ; ; )
781 {
782 u8 fault_reason;
783 u16 source_id;
784 u32 guest_addr, data;
785 int type;
787 /* highest 32 bits */
788 spin_lock_irqsave(&iommu->register_lock, flags);
789 data = dmar_readl(iommu->reg, reg +
790 fault_index * PRIMARY_FAULT_REG_LEN + 12);
791 if ( !(data & DMA_FRCD_F) )
792 {
793 spin_unlock_irqrestore(&iommu->register_lock, flags);
794 break;
795 }
797 fault_reason = dma_frcd_fault_reason(data);
798 type = dma_frcd_type(data);
800 data = dmar_readl(iommu->reg, reg +
801 fault_index * PRIMARY_FAULT_REG_LEN + 8);
802 source_id = dma_frcd_source_id(data);
804 guest_addr = dmar_readq(iommu->reg, reg +
805 fault_index * PRIMARY_FAULT_REG_LEN);
806 guest_addr = dma_frcd_page_addr(guest_addr);
807 /* clear the fault */
808 dmar_writel(iommu->reg, reg +
809 fault_index * PRIMARY_FAULT_REG_LEN + 12, DMA_FRCD_F);
810 spin_unlock_irqrestore(&iommu->register_lock, flags);
812 iommu_page_fault_do_one(iommu, type, fault_reason,
813 source_id, guest_addr);
815 fault_index++;
816 if ( fault_index > cap_num_fault_regs(iommu->cap) )
817 fault_index = 0;
818 }
820 /* clear primary fault overflow */
821 if ( fault_status & DMA_FSTS_PFO )
822 {
823 spin_lock_irqsave(&iommu->register_lock, flags);
824 dmar_writel(iommu->reg, DMAR_FSTS_REG, DMA_FSTS_PFO);
825 spin_unlock_irqrestore(&iommu->register_lock, flags);
826 }
827 }
829 static void dma_msi_unmask(unsigned int vector)
830 {
831 struct iommu *iommu = vector_to_iommu[vector];
832 unsigned long flags;
834 /* unmask it */
835 spin_lock_irqsave(&iommu->register_lock, flags);
836 dmar_writel(iommu->reg, DMAR_FECTL_REG, 0);
837 spin_unlock_irqrestore(&iommu->register_lock, flags);
838 }
840 static void dma_msi_mask(unsigned int vector)
841 {
842 unsigned long flags;
843 struct iommu *iommu = vector_to_iommu[vector];
845 /* mask it */
846 spin_lock_irqsave(&iommu->register_lock, flags);
847 dmar_writel(iommu->reg, DMAR_FECTL_REG, DMA_FECTL_IM);
848 spin_unlock_irqrestore(&iommu->register_lock, flags);
849 }
851 static unsigned int dma_msi_startup(unsigned int vector)
852 {
853 dma_msi_unmask(vector);
854 return 0;
855 }
857 static void dma_msi_end(unsigned int vector)
858 {
859 dma_msi_unmask(vector);
860 ack_APIC_irq();
861 }
863 static void dma_msi_data_init(struct iommu *iommu, int vector)
864 {
865 u32 msi_data = 0;
866 unsigned long flags;
868 /* Fixed, edge, assert mode. Follow MSI setting */
869 msi_data |= vector & 0xff;
870 msi_data |= 1 << 14;
872 spin_lock_irqsave(&iommu->register_lock, flags);
873 dmar_writel(iommu->reg, DMAR_FEDATA_REG, msi_data);
874 spin_unlock_irqrestore(&iommu->register_lock, flags);
875 }
877 static void dma_msi_addr_init(struct iommu *iommu, int phy_cpu)
878 {
879 u64 msi_address;
880 unsigned long flags;
882 /* Physical, dedicated cpu. Follow MSI setting */
883 msi_address = (MSI_ADDRESS_HEADER << (MSI_ADDRESS_HEADER_SHIFT + 8));
884 msi_address |= MSI_PHYSICAL_MODE << 2;
885 msi_address |= MSI_REDIRECTION_HINT_MODE << 3;
886 msi_address |= phy_cpu << MSI_TARGET_CPU_SHIFT;
888 spin_lock_irqsave(&iommu->register_lock, flags);
889 dmar_writel(iommu->reg, DMAR_FEADDR_REG, (u32)msi_address);
890 dmar_writel(iommu->reg, DMAR_FEUADDR_REG, (u32)(msi_address >> 32));
891 spin_unlock_irqrestore(&iommu->register_lock, flags);
892 }
894 static void dma_msi_set_affinity(unsigned int vector, cpumask_t dest)
895 {
896 struct iommu *iommu = vector_to_iommu[vector];
897 dma_msi_addr_init(iommu, cpu_physical_id(first_cpu(dest)));
898 }
900 static struct hw_interrupt_type dma_msi_type = {
901 .typename = "DMA_MSI",
902 .startup = dma_msi_startup,
903 .shutdown = dma_msi_mask,
904 .enable = dma_msi_unmask,
905 .disable = dma_msi_mask,
906 .ack = dma_msi_mask,
907 .end = dma_msi_end,
908 .set_affinity = dma_msi_set_affinity,
909 };
911 int iommu_set_interrupt(struct iommu *iommu)
912 {
913 int vector, ret;
915 vector = assign_irq_vector(AUTO_ASSIGN);
916 vector_to_iommu[vector] = iommu;
918 /* VT-d fault is a MSI, make irq == vector */
919 irq_vector[vector] = vector;
920 vector_irq[vector] = vector;
922 if ( !vector )
923 {
924 gdprintk(XENLOG_ERR VTDPREFIX, "IOMMU: no vectors\n");
925 return -EINVAL;
926 }
928 irq_desc[vector].handler = &dma_msi_type;
929 ret = request_irq(vector, iommu_page_fault, 0, "dmar", iommu);
930 if ( ret )
931 gdprintk(XENLOG_ERR VTDPREFIX, "IOMMU: can't request irq\n");
932 return vector;
933 }
935 struct iommu *iommu_alloc(void *hw_data)
936 {
937 struct acpi_drhd_unit *drhd = (struct acpi_drhd_unit *) hw_data;
938 struct iommu *iommu;
940 if ( nr_iommus > MAX_IOMMUS )
941 {
942 gdprintk(XENLOG_ERR VTDPREFIX,
943 "IOMMU: nr_iommus %d > MAX_IOMMUS\n", nr_iommus);
944 return NULL;
945 }
947 iommu = xmalloc(struct iommu);
948 if ( !iommu )
949 return NULL;
950 memset(iommu, 0, sizeof(struct iommu));
952 set_fixmap_nocache(FIX_IOMMU_REGS_BASE_0 + nr_iommus, drhd->address);
953 iommu->reg = (void *) fix_to_virt(FIX_IOMMU_REGS_BASE_0 + nr_iommus);
954 dprintk(XENLOG_INFO VTDPREFIX,
955 "iommu_alloc: iommu->reg = %p drhd->address = %lx\n",
956 iommu->reg, drhd->address);
957 nr_iommus++;
959 if ( !iommu->reg )
960 {
961 printk(KERN_ERR VTDPREFIX "IOMMU: can't mapping the region\n");
962 goto error;
963 }
965 iommu->cap = dmar_readq(iommu->reg, DMAR_CAP_REG);
966 iommu->ecap = dmar_readq(iommu->reg, DMAR_ECAP_REG);
968 spin_lock_init(&iommu->lock);
969 spin_lock_init(&iommu->register_lock);
971 drhd->iommu = iommu;
972 return iommu;
973 error:
974 xfree(iommu);
975 return NULL;
976 }
978 static void free_iommu(struct iommu *iommu)
979 {
980 if ( !iommu )
981 return;
982 if ( iommu->root_entry )
983 free_xenheap_page((void *)iommu->root_entry);
984 if ( iommu->reg )
985 iounmap(iommu->reg);
986 free_irq(iommu->vector);
987 xfree(iommu);
988 }
990 #define guestwidth_to_adjustwidth(gaw) ({ \
991 int agaw, r = (gaw - 12) % 9; \
992 agaw = (r == 0) ? gaw : (gaw + 9 - r); \
993 if ( agaw > 64 ) \
994 agaw = 64; \
995 agaw; })
997 int iommu_domain_init(struct domain *domain)
998 {
999 struct hvm_iommu *hd = domain_hvm_iommu(domain);
1000 struct iommu *iommu = NULL;
1001 int guest_width = DEFAULT_DOMAIN_ADDRESS_WIDTH;
1002 int adjust_width, agaw;
1003 unsigned long sagaw;
1004 struct acpi_drhd_unit *drhd;
1006 spin_lock_init(&hd->mapping_lock);
1007 spin_lock_init(&hd->iommu_list_lock);
1008 INIT_LIST_HEAD(&hd->pdev_list);
1009 INIT_LIST_HEAD(&hd->g2m_ioport_list);
1011 if ( !vtd_enabled || list_empty(&acpi_drhd_units) )
1012 return 0;
1014 for_each_drhd_unit ( drhd )
1015 iommu = drhd->iommu ? : iommu_alloc(drhd);
1017 /* calculate AGAW */
1018 if (guest_width > cap_mgaw(iommu->cap))
1019 guest_width = cap_mgaw(iommu->cap);
1020 adjust_width = guestwidth_to_adjustwidth(guest_width);
1021 agaw = width_to_agaw(adjust_width);
1022 /* FIXME: hardware doesn't support it, choose a bigger one? */
1023 sagaw = cap_sagaw(iommu->cap);
1024 if ( !test_bit(agaw, &sagaw) )
1026 gdprintk(XENLOG_ERR VTDPREFIX,
1027 "IOMMU: hardware doesn't support the agaw\n");
1028 agaw = find_next_bit(&sagaw, 5, agaw);
1029 if ( agaw >= 5 )
1030 return -ENODEV;
1032 hd->agaw = agaw;
1033 return 0;
1036 static int domain_context_mapping_one(
1037 struct domain *domain,
1038 struct iommu *iommu,
1039 u8 bus, u8 devfn)
1041 struct hvm_iommu *hd = domain_hvm_iommu(domain);
1042 struct context_entry *context;
1043 unsigned long flags;
1044 int ret = 0;
1046 context = device_to_context_entry(iommu, bus, devfn);
1047 if ( !context )
1049 gdprintk(XENLOG_ERR VTDPREFIX,
1050 "domain_context_mapping_one:context == NULL:"
1051 "bdf = %x:%x:%x\n",
1052 bus, PCI_SLOT(devfn), PCI_FUNC(devfn));
1053 return -ENOMEM;
1056 if ( context_present(*context) )
1058 gdprintk(XENLOG_WARNING VTDPREFIX,
1059 "domain_context_mapping_one:context present:bdf=%x:%x:%x\n",
1060 bus, PCI_SLOT(devfn), PCI_FUNC(devfn));
1061 return 0;
1064 spin_lock_irqsave(&iommu->lock, flags);
1065 /*
1066 * domain_id 0 is not valid on Intel's IOMMU, force domain_id to
1067 * be 1 based as required by intel's iommu hw.
1068 */
1069 context_set_domain_id(context, domain);
1070 context_set_address_width(*context, hd->agaw);
1072 if ( ecap_pass_thru(iommu->ecap) )
1073 context_set_translation_type(*context, CONTEXT_TT_PASS_THRU);
1074 else
1076 if ( !hd->pgd )
1078 struct dma_pte *pgd = (struct dma_pte *)alloc_xenheap_page();
1079 if ( !pgd )
1081 spin_unlock_irqrestore(&hd->mapping_lock, flags);
1082 return -ENOMEM;
1084 memset(pgd, 0, PAGE_SIZE);
1085 hd->pgd = pgd;
1088 context_set_address_root(*context, virt_to_maddr(hd->pgd));
1089 context_set_translation_type(*context, CONTEXT_TT_MULTI_LEVEL);
1092 context_set_fault_enable(*context);
1093 context_set_present(*context);
1094 iommu_flush_cache_entry(iommu, context);
1096 gdprintk(XENLOG_INFO VTDPREFIX,
1097 "domain_context_mapping_one-%x:%x:%x-*context=%"PRIx64":%"PRIx64
1098 " hd->pgd=%p\n",
1099 bus, PCI_SLOT(devfn), PCI_FUNC(devfn),
1100 context->hi, context->lo, hd->pgd);
1102 if ( iommu_flush_context_device(iommu, domain_iommu_domid(domain),
1103 (((u16)bus) << 8) | devfn,
1104 DMA_CCMD_MASK_NOBIT, 1) )
1105 iommu_flush_write_buffer(iommu);
1106 else
1107 iommu_flush_iotlb_dsi(iommu, domain_iommu_domid(domain), 0);
1108 spin_unlock_irqrestore(&iommu->lock, flags);
1109 return ret;
1112 static int __pci_find_next_cap(u8 bus, unsigned int devfn, u8 pos, int cap)
1114 u8 id;
1115 int ttl = 48;
1117 while ( ttl-- )
1119 pos = read_pci_config_byte(bus, PCI_SLOT(devfn), PCI_FUNC(devfn), pos);
1120 if ( pos < 0x40 )
1121 break;
1123 pos &= ~3;
1124 id = read_pci_config_byte(bus, PCI_SLOT(devfn), PCI_FUNC(devfn),
1125 pos + PCI_CAP_LIST_ID);
1127 if ( id == 0xff )
1128 break;
1129 if ( id == cap )
1130 return pos;
1132 pos += PCI_CAP_LIST_NEXT;
1134 return 0;
1137 #define PCI_BASE_CLASS_BRIDGE 0x06
1138 #define PCI_CLASS_BRIDGE_PCI 0x0604
1140 #define DEV_TYPE_PCIe_ENDPOINT 1
1141 #define DEV_TYPE_PCI_BRIDGE 2
1142 #define DEV_TYPE_PCI 3
1144 int pdev_type(struct pci_dev *dev)
1146 u16 class_device;
1147 u16 status;
1149 class_device = read_pci_config_16(dev->bus, PCI_SLOT(dev->devfn),
1150 PCI_FUNC(dev->devfn), PCI_CLASS_DEVICE);
1151 if ( class_device == PCI_CLASS_BRIDGE_PCI )
1152 return DEV_TYPE_PCI_BRIDGE;
1154 status = read_pci_config_16(dev->bus, PCI_SLOT(dev->devfn),
1155 PCI_FUNC(dev->devfn), PCI_STATUS);
1157 if ( !(status & PCI_STATUS_CAP_LIST) )
1158 return DEV_TYPE_PCI;
1160 if ( __pci_find_next_cap(dev->bus, dev->devfn,
1161 PCI_CAPABILITY_LIST, PCI_CAP_ID_EXP) )
1162 return DEV_TYPE_PCIe_ENDPOINT;
1164 return DEV_TYPE_PCI;
1167 #define MAX_BUSES 256
1168 struct pci_dev bus2bridge[MAX_BUSES];
1170 static int domain_context_mapping(
1171 struct domain *domain,
1172 struct iommu *iommu,
1173 struct pci_dev *pdev)
1175 int ret = 0;
1176 int dev, func, sec_bus, sub_bus;
1177 u32 type;
1179 type = pdev_type(pdev);
1180 switch ( type )
1182 case DEV_TYPE_PCI_BRIDGE:
1183 sec_bus = read_pci_config_byte(
1184 pdev->bus, PCI_SLOT(pdev->devfn),
1185 PCI_FUNC(pdev->devfn), PCI_SECONDARY_BUS);
1187 if ( bus2bridge[sec_bus].bus == 0 )
1189 bus2bridge[sec_bus].bus = pdev->bus;
1190 bus2bridge[sec_bus].devfn = pdev->devfn;
1193 sub_bus = read_pci_config_byte(
1194 pdev->bus, PCI_SLOT(pdev->devfn),
1195 PCI_FUNC(pdev->devfn), PCI_SUBORDINATE_BUS);
1197 if ( sec_bus != sub_bus )
1198 gdprintk(XENLOG_WARNING VTDPREFIX,
1199 "context_context_mapping: nested PCI bridge not "
1200 "supported: bdf = %x:%x:%x sec_bus = %x sub_bus = %x\n",
1201 pdev->bus, PCI_SLOT(pdev->devfn), PCI_FUNC(pdev->devfn),
1202 sec_bus, sub_bus);
1203 break;
1204 case DEV_TYPE_PCIe_ENDPOINT:
1205 gdprintk(XENLOG_INFO VTDPREFIX,
1206 "domain_context_mapping:PCIe : bdf = %x:%x:%x\n",
1207 pdev->bus, PCI_SLOT(pdev->devfn), PCI_FUNC(pdev->devfn));
1208 ret = domain_context_mapping_one(domain, iommu,
1209 (u8)(pdev->bus), (u8)(pdev->devfn));
1210 break;
1211 case DEV_TYPE_PCI:
1212 gdprintk(XENLOG_INFO VTDPREFIX,
1213 "domain_context_mapping:PCI: bdf = %x:%x:%x\n",
1214 pdev->bus, PCI_SLOT(pdev->devfn), PCI_FUNC(pdev->devfn));
1216 if ( pdev->bus == 0 )
1217 ret = domain_context_mapping_one(
1218 domain, iommu, (u8)(pdev->bus), (u8)(pdev->devfn));
1219 else
1221 if ( bus2bridge[pdev->bus].bus != 0 )
1222 gdprintk(XENLOG_WARNING VTDPREFIX,
1223 "domain_context_mapping:bus2bridge"
1224 "[%d].bus != 0\n", pdev->bus);
1226 ret = domain_context_mapping_one(
1227 domain, iommu,
1228 (u8)(bus2bridge[pdev->bus].bus),
1229 (u8)(bus2bridge[pdev->bus].devfn));
1231 /* now map everything behind the PCI bridge */
1232 for ( dev = 0; dev < 32; dev++ )
1234 for ( func = 0; func < 8; func++ )
1236 ret = domain_context_mapping_one(
1237 domain, iommu,
1238 pdev->bus, (u8)PCI_DEVFN(dev, func));
1239 if ( ret )
1240 return ret;
1244 break;
1245 default:
1246 gdprintk(XENLOG_ERR VTDPREFIX,
1247 "domain_context_mapping:unknown type : bdf = %x:%x:%x\n",
1248 pdev->bus, PCI_SLOT(pdev->devfn), PCI_FUNC(pdev->devfn));
1249 ret = -EINVAL;
1250 break;
1253 return ret;
1256 static int domain_context_unmap_one(
1257 struct domain *domain,
1258 struct iommu *iommu,
1259 u8 bus, u8 devfn)
1261 struct context_entry *context;
1262 unsigned long flags;
1264 context = device_to_context_entry(iommu, bus, devfn);
1265 if ( !context )
1267 gdprintk(XENLOG_ERR VTDPREFIX,
1268 "domain_context_unmap_one-%x:%x:%x- context == NULL:return\n",
1269 bus, PCI_SLOT(devfn), PCI_FUNC(devfn));
1270 return -ENOMEM;
1273 if ( !context_present(*context) )
1275 gdprintk(XENLOG_WARNING VTDPREFIX,
1276 "domain_context_unmap_one-%x:%x:%x- "
1277 "context NOT present:return\n",
1278 bus, PCI_SLOT(devfn), PCI_FUNC(devfn));
1279 return 0;
1282 gdprintk(XENLOG_INFO VTDPREFIX,
1283 "domain_context_unmap_one: bdf = %x:%x:%x\n",
1284 bus, PCI_SLOT(devfn), PCI_FUNC(devfn));
1286 spin_lock_irqsave(&iommu->lock, flags);
1287 context_clear_present(*context);
1288 context_clear_entry(*context);
1289 iommu_flush_cache_entry(iommu, context);
1290 iommu_flush_context_global(iommu, 0);
1291 iommu_flush_iotlb_global(iommu, 0);
1292 spin_unlock_irqrestore(&iommu->lock, flags);
1294 return 0;
1297 static int domain_context_unmap(
1298 struct domain *domain,
1299 struct iommu *iommu,
1300 struct pci_dev *pdev)
1302 int ret = 0;
1303 int dev, func, sec_bus, sub_bus;
1304 u32 type;
1306 type = pdev_type(pdev);
1307 switch ( type )
1309 case DEV_TYPE_PCI_BRIDGE:
1310 sec_bus = read_pci_config_byte(
1311 pdev->bus, PCI_SLOT(pdev->devfn),
1312 PCI_FUNC(pdev->devfn), PCI_SECONDARY_BUS);
1313 sub_bus = read_pci_config_byte(
1314 pdev->bus, PCI_SLOT(pdev->devfn),
1315 PCI_FUNC(pdev->devfn), PCI_SUBORDINATE_BUS);
1317 gdprintk(XENLOG_INFO VTDPREFIX,
1318 "domain_context_unmap:BRIDGE:%x:%x:%x "
1319 "sec_bus=%x sub_bus=%x\n",
1320 pdev->bus, PCI_SLOT(pdev->devfn),
1321 PCI_FUNC(pdev->devfn), sec_bus, sub_bus);
1322 break;
1323 case DEV_TYPE_PCIe_ENDPOINT:
1324 gdprintk(XENLOG_INFO VTDPREFIX,
1325 "domain_context_unmap:PCIe : bdf = %x:%x:%x\n",
1326 pdev->bus, PCI_SLOT(pdev->devfn), PCI_FUNC(pdev->devfn));
1327 ret = domain_context_unmap_one(domain, iommu,
1328 (u8)(pdev->bus), (u8)(pdev->devfn));
1329 break;
1330 case DEV_TYPE_PCI:
1331 gdprintk(XENLOG_INFO VTDPREFIX,
1332 "domain_context_unmap:PCI: bdf = %x:%x:%x\n",
1333 pdev->bus, PCI_SLOT(pdev->devfn), PCI_FUNC(pdev->devfn));
1334 if ( pdev->bus == 0 )
1335 ret = domain_context_unmap_one(
1336 domain, iommu,
1337 (u8)(pdev->bus), (u8)(pdev->devfn));
1338 else
1340 if ( bus2bridge[pdev->bus].bus != 0 )
1341 gdprintk(XENLOG_WARNING VTDPREFIX,
1342 "domain_context_unmap:"
1343 "bus2bridge[%d].bus != 0\n", pdev->bus);
1345 ret = domain_context_unmap_one(domain, iommu,
1346 (u8)(bus2bridge[pdev->bus].bus),
1347 (u8)(bus2bridge[pdev->bus].devfn));
1349 /* Unmap everything behind the PCI bridge */
1350 for ( dev = 0; dev < 32; dev++ )
1352 for ( func = 0; func < 8; func++ )
1354 ret = domain_context_unmap_one(
1355 domain, iommu,
1356 pdev->bus, (u8)PCI_DEVFN(dev, func));
1357 if ( ret )
1358 return ret;
1362 break;
1363 default:
1364 gdprintk(XENLOG_ERR VTDPREFIX,
1365 "domain_context_unmap:unknown type: bdf = %x:%x:%x\n",
1366 pdev->bus, PCI_SLOT(pdev->devfn), PCI_FUNC(pdev->devfn));
1367 ret = -EINVAL;
1368 break;
1371 return ret;
1374 void reassign_device_ownership(
1375 struct domain *source,
1376 struct domain *target,
1377 u8 bus, u8 devfn)
1379 struct hvm_iommu *source_hd = domain_hvm_iommu(source);
1380 struct hvm_iommu *target_hd = domain_hvm_iommu(target);
1381 struct pci_dev *pdev;
1382 struct acpi_drhd_unit *drhd;
1383 struct iommu *iommu;
1384 int status;
1385 unsigned long flags;
1387 gdprintk(XENLOG_INFO VTDPREFIX,
1388 "reassign_device-%x:%x:%x- source = %d target = %d\n",
1389 bus, PCI_SLOT(devfn), PCI_FUNC(devfn),
1390 source->domain_id, target->domain_id);
1392 for_each_pdev( source, pdev )
1394 if ( (pdev->bus != bus) || (pdev->devfn != devfn) )
1395 continue;
1397 drhd = acpi_find_matched_drhd_unit(pdev);
1398 iommu = drhd->iommu;
1399 domain_context_unmap(source, iommu, pdev);
1401 /* Move pci device from the source domain to target domain. */
1402 spin_lock_irqsave(&source_hd->iommu_list_lock, flags);
1403 spin_lock_irqsave(&target_hd->iommu_list_lock, flags);
1404 list_move(&pdev->list, &target_hd->pdev_list);
1405 spin_unlock_irqrestore(&target_hd->iommu_list_lock, flags);
1406 spin_unlock_irqrestore(&source_hd->iommu_list_lock, flags);
1408 status = domain_context_mapping(target, iommu, pdev);
1409 if ( status != 0 )
1410 gdprintk(XENLOG_ERR VTDPREFIX, "domain_context_mapping failed\n");
1412 break;
1416 void return_devices_to_dom0(struct domain *d)
1418 struct hvm_iommu *hd = domain_hvm_iommu(d);
1419 struct pci_dev *pdev;
1421 while ( !list_empty(&hd->pdev_list) )
1423 pdev = list_entry(hd->pdev_list.next, typeof(*pdev), list);
1424 dprintk(XENLOG_INFO VTDPREFIX,
1425 "return_devices_to_dom0: bdf = %x:%x:%x\n",
1426 pdev->bus, PCI_SLOT(pdev->devfn), PCI_FUNC(pdev->devfn));
1427 reassign_device_ownership(d, dom0, pdev->bus, pdev->devfn);
1430 #ifdef VTD_DEBUG
1431 for_each_pdev ( dom0, pdev )
1432 dprintk(XENLOG_INFO VTDPREFIX,
1433 "return_devices_to_dom0:%x: bdf = %x:%x:%x\n",
1434 dom0->domain_id, pdev->bus,
1435 PCI_SLOT(pdev->devfn), PCI_FUNC(pdev->devfn));
1436 #endif
1439 void iommu_domain_teardown(struct domain *d)
1441 if ( list_empty(&acpi_drhd_units) )
1442 return;
1444 iommu_domid_release(d);
1446 #if CONFIG_PAGING_LEVELS == 3
1448 struct hvm_iommu *hd = domain_hvm_iommu(d);
1449 int level = agaw_to_level(hd->agaw);
1450 struct dma_pte *pgd = NULL;
1452 switch ( level )
1454 case VTD_PAGE_TABLE_LEVEL_3:
1455 if ( hd->pgd )
1456 free_xenheap_page((void *)hd->pgd);
1457 break;
1458 case VTD_PAGE_TABLE_LEVEL_4:
1459 if ( hd->pgd )
1461 pgd = hd->pgd;
1462 if ( pgd[0].val != 0 )
1463 free_xenheap_page((void*)maddr_to_virt(
1464 dma_pte_addr(pgd[0])));
1466 free_xenheap_page((void *)hd->pgd);
1468 break;
1469 default:
1470 gdprintk(XENLOG_ERR VTDPREFIX,
1471 "Unsupported p2m table sharing level!\n");
1472 break;
1475 #endif
1476 return_devices_to_dom0(d);
1479 static int domain_context_mapped(struct pci_dev *pdev)
1481 struct acpi_drhd_unit *drhd;
1482 struct iommu *iommu;
1483 int ret;
1485 for_each_drhd_unit ( drhd )
1487 iommu = drhd->iommu;
1488 ret = device_context_mapped(iommu, pdev->bus, pdev->devfn);
1489 if ( ret )
1490 return ret;
1493 return 0;
1496 int iommu_map_page(struct domain *d, paddr_t gfn, paddr_t mfn)
1498 struct acpi_drhd_unit *drhd;
1499 struct iommu *iommu;
1500 struct dma_pte *pte = NULL;
1501 struct page_info *pg = NULL;
1503 drhd = list_entry(acpi_drhd_units.next, typeof(*drhd), list);
1504 iommu = drhd->iommu;
1506 /* do nothing if dom0 and iommu supports pass thru */
1507 if ( ecap_pass_thru(iommu->ecap) && (d->domain_id == 0) )
1508 return 0;
1510 pg = addr_to_dma_page(d, gfn << PAGE_SHIFT_4K);
1511 if ( !pg )
1512 return -ENOMEM;
1513 pte = (struct dma_pte *)map_domain_page(page_to_mfn(pg));
1514 pte += gfn & LEVEL_MASK;
1515 dma_set_pte_addr(*pte, mfn << PAGE_SHIFT_4K);
1516 dma_set_pte_prot(*pte, DMA_PTE_READ | DMA_PTE_WRITE);
1517 iommu_flush_cache_entry(iommu, pte);
1518 unmap_domain_page(pte);
1520 for_each_drhd_unit ( drhd )
1522 iommu = drhd->iommu;
1523 if ( cap_caching_mode(iommu->cap) )
1524 iommu_flush_iotlb_psi(iommu, domain_iommu_domid(d),
1525 gfn << PAGE_SHIFT_4K, 1, 0);
1526 else if ( cap_rwbf(iommu->cap) )
1527 iommu_flush_write_buffer(iommu);
1530 return 0;
1533 int iommu_unmap_page(struct domain *d, dma_addr_t gfn)
1535 struct acpi_drhd_unit *drhd;
1536 struct iommu *iommu;
1538 drhd = list_entry(acpi_drhd_units.next, typeof(*drhd), list);
1539 iommu = drhd->iommu;
1541 /* do nothing if dom0 and iommu supports pass thru */
1542 if ( ecap_pass_thru(iommu->ecap) && (d->domain_id == 0) )
1543 return 0;
1545 dma_pte_clear_one(d, gfn << PAGE_SHIFT_4K);
1547 return 0;
1550 int iommu_page_mapping(struct domain *domain, dma_addr_t iova,
1551 void *hpa, size_t size, int prot)
1553 struct acpi_drhd_unit *drhd;
1554 struct iommu *iommu;
1555 unsigned long start_pfn, end_pfn;
1556 struct dma_pte *pte = NULL;
1557 int index;
1558 struct page_info *pg = NULL;
1560 drhd = list_entry(acpi_drhd_units.next, typeof(*drhd), list);
1561 iommu = drhd->iommu;
1562 if ( (prot & (DMA_PTE_READ|DMA_PTE_WRITE)) == 0 )
1563 return -EINVAL;
1564 iova = (iova >> PAGE_SHIFT_4K) << PAGE_SHIFT_4K;
1565 start_pfn = (unsigned long)(((unsigned long) hpa) >> PAGE_SHIFT_4K);
1566 end_pfn = (unsigned long)
1567 ((PAGE_ALIGN_4K(((unsigned long)hpa) + size)) >> PAGE_SHIFT_4K);
1568 index = 0;
1569 while ( start_pfn < end_pfn )
1571 pg = addr_to_dma_page(domain, iova + PAGE_SIZE_4K * index);
1572 if ( !pg )
1573 return -ENOMEM;
1574 pte = (struct dma_pte *)map_domain_page(page_to_mfn(pg));
1575 pte += start_pfn & LEVEL_MASK;
1576 dma_set_pte_addr(*pte, start_pfn << PAGE_SHIFT_4K);
1577 dma_set_pte_prot(*pte, prot);
1578 iommu_flush_cache_entry(iommu, pte);
1579 unmap_domain_page(pte);
1580 start_pfn++;
1581 index++;
1584 for_each_drhd_unit ( drhd )
1586 iommu = drhd->iommu;
1587 if ( cap_caching_mode(iommu->cap) )
1588 iommu_flush_iotlb_psi(iommu, domain_iommu_domid(domain),
1589 iova, index, 0);
1590 else if ( cap_rwbf(iommu->cap) )
1591 iommu_flush_write_buffer(iommu);
1594 return 0;
1597 int iommu_page_unmapping(struct domain *domain, dma_addr_t addr, size_t size)
1599 dma_pte_clear_range(domain, addr, addr + size);
1601 return 0;
1604 void iommu_flush(struct domain *d, dma_addr_t gfn, u64 *p2m_entry)
1606 struct acpi_drhd_unit *drhd;
1607 struct iommu *iommu = NULL;
1608 struct dma_pte *pte = (struct dma_pte *) p2m_entry;
1610 for_each_drhd_unit ( drhd )
1612 iommu = drhd->iommu;
1613 if ( cap_caching_mode(iommu->cap) )
1614 iommu_flush_iotlb_psi(iommu, domain_iommu_domid(d),
1615 gfn << PAGE_SHIFT_4K, 1, 0);
1616 else if ( cap_rwbf(iommu->cap) )
1617 iommu_flush_write_buffer(iommu);
1620 iommu_flush_cache_entry(iommu, pte);
1623 static int iommu_prepare_rmrr_dev(
1624 struct domain *d,
1625 struct acpi_rmrr_unit *rmrr,
1626 struct pci_dev *pdev)
1628 struct acpi_drhd_unit *drhd;
1629 unsigned long size;
1630 int ret;
1632 /* page table init */
1633 size = rmrr->end_address - rmrr->base_address + 1;
1634 ret = iommu_page_mapping(d, rmrr->base_address,
1635 (void *)rmrr->base_address, size,
1636 DMA_PTE_READ|DMA_PTE_WRITE);
1637 if ( ret )
1638 return ret;
1640 if ( domain_context_mapped(pdev) == 0 )
1642 drhd = acpi_find_matched_drhd_unit(pdev);
1643 ret = domain_context_mapping(d, drhd->iommu, pdev);
1644 if ( !ret )
1645 return 0;
1648 return ret;
1651 void __init setup_dom0_devices(void)
1653 struct hvm_iommu *hd = domain_hvm_iommu(dom0);
1654 struct acpi_drhd_unit *drhd;
1655 struct pci_dev *pdev;
1656 int bus, dev, func, ret;
1657 u32 l;
1659 #ifdef DEBUG_VTD_CONTEXT_ENTRY
1660 for ( bus = 0; bus < 256; bus++ )
1662 for ( dev = 0; dev < 32; dev++ )
1664 for ( func = 0; func < 8; func++ )
1666 struct context_entry *context;
1667 struct pci_dev device;
1669 device.bus = bus;
1670 device.devfn = PCI_DEVFN(dev, func);
1671 drhd = acpi_find_matched_drhd_unit(&device);
1672 context = device_to_context_entry(drhd->iommu,
1673 bus, PCI_DEVFN(dev, func));
1674 if ( (context->lo != 0) || (context->hi != 0) )
1675 dprintk(XENLOG_INFO VTDPREFIX,
1676 "setup_dom0_devices-%x:%x:%x- context not 0\n",
1677 bus, dev, func);
1681 #endif
1683 for ( bus = 0; bus < 256; bus++ )
1685 for ( dev = 0; dev < 32; dev++ )
1687 for ( func = 0; func < 8; func++ )
1689 l = read_pci_config(bus, dev, func, PCI_VENDOR_ID);
1690 /* some broken boards return 0 or ~0 if a slot is empty: */
1691 if ( (l == 0xffffffff) || (l == 0x00000000) ||
1692 (l == 0x0000ffff) || (l == 0xffff0000) )
1693 continue;
1694 pdev = xmalloc(struct pci_dev);
1695 pdev->bus = bus;
1696 pdev->devfn = PCI_DEVFN(dev, func);
1697 list_add_tail(&pdev->list, &hd->pdev_list);
1699 drhd = acpi_find_matched_drhd_unit(pdev);
1700 ret = domain_context_mapping(dom0, drhd->iommu, pdev);
1701 if ( ret != 0 )
1702 gdprintk(XENLOG_ERR VTDPREFIX,
1703 "domain_context_mapping failed\n");
1708 for_each_pdev ( dom0, pdev )
1709 dprintk(XENLOG_INFO VTDPREFIX,
1710 "setup_dom0_devices: bdf = %x:%x:%x\n",
1711 pdev->bus, PCI_SLOT(pdev->devfn), PCI_FUNC(pdev->devfn));
1714 void clear_fault_bit(struct iommu *iommu)
1716 u64 val;
1718 val = dmar_readq(
1719 iommu->reg,
1720 cap_fault_reg_offset(dmar_readq(iommu->reg,DMAR_CAP_REG))+0x8);
1721 dmar_writeq(
1722 iommu->reg,
1723 cap_fault_reg_offset(dmar_readq(iommu->reg,DMAR_CAP_REG))+8,
1724 val);
1725 dmar_writel(iommu->reg, DMAR_FSTS_REG, DMA_FSTS_PFO);
1728 static int init_vtd_hw(void)
1730 struct acpi_drhd_unit *drhd;
1731 struct iommu *iommu;
1732 int ret;
1734 for_each_drhd_unit ( drhd )
1736 iommu = drhd->iommu;
1737 ret = iommu_set_root_entry(iommu);
1738 if ( ret )
1740 gdprintk(XENLOG_ERR VTDPREFIX, "IOMMU: set root entry failed\n");
1741 return -EIO;
1745 return 0;
1748 static int enable_vtd_translation(void)
1750 struct acpi_drhd_unit *drhd;
1751 struct iommu *iommu;
1752 int vector = 0;
1754 for_each_drhd_unit ( drhd )
1756 iommu = drhd->iommu;
1757 vector = iommu_set_interrupt(iommu);
1758 dma_msi_data_init(iommu, vector);
1759 dma_msi_addr_init(iommu, cpu_physical_id(first_cpu(cpu_online_map)));
1760 iommu->vector = vector;
1761 clear_fault_bit(iommu);
1762 if ( iommu_enable_translation(iommu) )
1763 return -EIO;
1766 return 0;
1769 static void setup_dom0_rmrr(void)
1771 struct acpi_rmrr_unit *rmrr;
1772 struct pci_dev *pdev;
1773 int ret;
1775 for_each_rmrr_device ( rmrr, pdev )
1776 ret = iommu_prepare_rmrr_dev(dom0, rmrr, pdev);
1777 if ( ret )
1778 gdprintk(XENLOG_ERR VTDPREFIX,
1779 "IOMMU: mapping reserved region failed\n");
1780 end_for_each_rmrr_device ( rmrr, pdev )
1783 int iommu_setup(void)
1785 struct hvm_iommu *hd = domain_hvm_iommu(dom0);
1786 struct acpi_drhd_unit *drhd;
1787 struct iommu *iommu;
1788 unsigned long i;
1790 if ( !vtd_enabled )
1791 return 0;
1793 spin_lock_init(&domid_bitmap_lock);
1794 INIT_LIST_HEAD(&hd->pdev_list);
1796 /* start from scratch */
1797 iommu_flush_all();
1799 /* setup clflush size */
1800 x86_clflush_size = ((cpuid_ebx(1) >> 8) & 0xff) * 8;
1802 /* Allocate IO page directory page for the domain. */
1803 drhd = list_entry(acpi_drhd_units.next, typeof(*drhd), list);
1804 iommu = drhd->iommu;
1806 /* Allocate domain id bitmap, and set bit 0 as reserved */
1807 domid_bitmap_size = cap_ndoms(iommu->cap);
1808 domid_bitmap = xmalloc_bytes(domid_bitmap_size / 8);
1809 if ( domid_bitmap == NULL )
1810 goto error;
1811 memset(domid_bitmap, 0, domid_bitmap_size / 8);
1812 set_bit(0, domid_bitmap);
1814 /* setup 1:1 page table for dom0 */
1815 for ( i = 0; i < max_page; i++ )
1816 iommu_map_page(dom0, i, i);
1818 if ( init_vtd_hw() )
1819 goto error;
1820 setup_dom0_devices();
1821 setup_dom0_rmrr();
1822 if ( enable_vtd_translation() )
1823 goto error;
1825 return 0;
1827 error:
1828 printk("iommu_setup() failed\n");
1829 for_each_drhd_unit ( drhd )
1831 iommu = drhd->iommu;
1832 free_iommu(iommu);
1834 return -EIO;
1837 /*
1838 * If the device isn't owned by dom0, it means it already
1839 * has been assigned to other domain, or it's not exist.
1840 */
1841 int device_assigned(u8 bus, u8 devfn)
1843 struct pci_dev *pdev;
1845 for_each_pdev( dom0, pdev )
1846 if ( (pdev->bus == bus ) && (pdev->devfn == devfn) )
1847 return 0;
1849 return 1;
1852 int assign_device(struct domain *d, u8 bus, u8 devfn)
1854 struct acpi_rmrr_unit *rmrr;
1855 struct pci_dev *pdev;
1856 int ret = 0;
1858 if ( list_empty(&acpi_drhd_units) )
1859 return ret;
1861 gdprintk(XENLOG_INFO VTDPREFIX,
1862 "assign_device: bus = %x dev = %x func = %x\n",
1863 bus, PCI_SLOT(devfn), PCI_FUNC(devfn));
1865 reassign_device_ownership(dom0, d, bus, devfn);
1867 /* Setup rmrr identify mapping */
1868 for_each_rmrr_device( rmrr, pdev )
1869 if ( pdev->bus == bus && pdev->devfn == devfn )
1871 ret = iommu_prepare_rmrr_dev(d, rmrr, pdev);
1872 if ( ret )
1874 gdprintk(XENLOG_ERR VTDPREFIX,
1875 "IOMMU: mapping reserved region failed\n");
1876 return ret;
1879 end_for_each_rmrr_device(rmrr, pdev)
1881 return ret;
1884 void iommu_set_pgd(struct domain *d)
1886 struct hvm_iommu *hd = domain_hvm_iommu(d);
1887 unsigned long p2m_table;
1889 if ( hd->pgd )
1891 gdprintk(XENLOG_INFO VTDPREFIX,
1892 "iommu_set_pgd_1: hd->pgd = %p\n", hd->pgd);
1893 hd->pgd = NULL;
1895 p2m_table = mfn_x(pagetable_get_mfn(d->arch.phys_table));
1897 #if CONFIG_PAGING_LEVELS == 3
1898 if ( !hd->pgd )
1900 int level = agaw_to_level(hd->agaw);
1901 struct dma_pte *pmd = NULL;
1902 struct dma_pte *pgd = NULL;
1903 struct dma_pte *pte = NULL;
1904 l3_pgentry_t *l3e;
1905 unsigned long flags;
1906 int i;
1908 spin_lock_irqsave(&hd->mapping_lock, flags);
1909 if ( !hd->pgd )
1911 pgd = (struct dma_pte *)alloc_xenheap_page();
1912 if ( !pgd )
1914 spin_unlock_irqrestore(&hd->mapping_lock, flags);
1915 gdprintk(XENLOG_ERR VTDPREFIX,
1916 "Allocate pgd memory failed!\n");
1917 return;
1919 memset(pgd, 0, PAGE_SIZE);
1920 hd->pgd = pgd;
1923 l3e = map_domain_page(p2m_table);
1924 switch ( level )
1926 case VTD_PAGE_TABLE_LEVEL_3: /* Weybridge */
1927 /* We only support 8 entries for the PAE L3 p2m table */
1928 for ( i = 0; i < 8 ; i++ )
1930 /* Don't create new L2 entry, use ones from p2m table */
1931 pgd[i].val = l3e[i].l3 | _PAGE_PRESENT | _PAGE_RW;
1933 break;
1935 case VTD_PAGE_TABLE_LEVEL_4: /* Stoakley */
1936 /* We allocate one more page for the top vtd page table. */
1937 pmd = (struct dma_pte *)alloc_xenheap_page();
1938 if ( !pmd )
1940 unmap_domain_page(l3e);
1941 spin_unlock_irqrestore(&hd->mapping_lock, flags);
1942 gdprintk(XENLOG_ERR VTDPREFIX,
1943 "Allocate pmd memory failed!\n");
1944 return;
1946 memset((u8*)pmd, 0, PAGE_SIZE);
1947 pte = &pgd[0];
1948 dma_set_pte_addr(*pte, virt_to_maddr(pmd));
1949 dma_set_pte_readable(*pte);
1950 dma_set_pte_writable(*pte);
1952 for ( i = 0; i < 8; i++ )
1954 /* Don't create new L2 entry, use ones from p2m table */
1955 pmd[i].val = l3e[i].l3 | _PAGE_PRESENT | _PAGE_RW;
1957 break;
1958 default:
1959 gdprintk(XENLOG_ERR VTDPREFIX,
1960 "iommu_set_pgd:Unsupported p2m table sharing level!\n");
1961 break;
1963 unmap_domain_page(l3e);
1964 spin_unlock_irqrestore(&hd->mapping_lock, flags);
1966 #elif CONFIG_PAGING_LEVELS == 4
1967 if ( !hd->pgd )
1969 int level = agaw_to_level(hd->agaw);
1970 l3_pgentry_t *l3e;
1971 mfn_t pgd_mfn;
1973 switch ( level )
1975 case VTD_PAGE_TABLE_LEVEL_3:
1976 l3e = map_domain_page(p2m_table);
1977 if ( (l3e_get_flags(*l3e) & _PAGE_PRESENT) == 0 )
1979 gdprintk(XENLOG_ERR VTDPREFIX,
1980 "iommu_set_pgd: second level wasn't there\n");
1981 unmap_domain_page(l3e);
1982 return;
1984 pgd_mfn = _mfn(l3e_get_pfn(*l3e));
1985 unmap_domain_page(l3e);
1986 hd->pgd = maddr_to_virt(pagetable_get_paddr(
1987 pagetable_from_mfn(pgd_mfn)));
1988 break;
1990 case VTD_PAGE_TABLE_LEVEL_4:
1991 pgd_mfn = _mfn(p2m_table);
1992 hd->pgd = maddr_to_virt(pagetable_get_paddr(
1993 pagetable_from_mfn(pgd_mfn)));
1994 break;
1995 default:
1996 gdprintk(XENLOG_ERR VTDPREFIX,
1997 "iommu_set_pgd:Unsupported p2m table sharing level!\n");
1998 break;
2001 #endif
2002 gdprintk(XENLOG_INFO VTDPREFIX,
2003 "iommu_set_pgd: hd->pgd = %p\n", hd->pgd);
2007 u8 iommu_state[MAX_IOMMU_REGS * MAX_IOMMUS];
2008 int iommu_suspend(void)
2010 struct acpi_drhd_unit *drhd;
2011 struct iommu *iommu;
2012 int i = 0;
2014 iommu_flush_all();
2016 for_each_drhd_unit ( drhd )
2018 iommu = drhd->iommu;
2019 iommu_state[DMAR_RTADDR_REG * i] =
2020 (u64) dmar_readq(iommu->reg, DMAR_RTADDR_REG);
2021 iommu_state[DMAR_FECTL_REG * i] =
2022 (u32) dmar_readl(iommu->reg, DMAR_FECTL_REG);
2023 iommu_state[DMAR_FEDATA_REG * i] =
2024 (u32) dmar_readl(iommu->reg, DMAR_FEDATA_REG);
2025 iommu_state[DMAR_FEADDR_REG * i] =
2026 (u32) dmar_readl(iommu->reg, DMAR_FEADDR_REG);
2027 iommu_state[DMAR_FEUADDR_REG * i] =
2028 (u32) dmar_readl(iommu->reg, DMAR_FEUADDR_REG);
2029 iommu_state[DMAR_PLMBASE_REG * i] =
2030 (u32) dmar_readl(iommu->reg, DMAR_PLMBASE_REG);
2031 iommu_state[DMAR_PLMLIMIT_REG * i] =
2032 (u32) dmar_readl(iommu->reg, DMAR_PLMLIMIT_REG);
2033 iommu_state[DMAR_PHMBASE_REG * i] =
2034 (u64) dmar_readq(iommu->reg, DMAR_PHMBASE_REG);
2035 iommu_state[DMAR_PHMLIMIT_REG * i] =
2036 (u64) dmar_readq(iommu->reg, DMAR_PHMLIMIT_REG);
2037 i++;
2040 return 0;
2043 int iommu_resume(void)
2045 struct acpi_drhd_unit *drhd;
2046 struct iommu *iommu;
2047 int i = 0;
2049 iommu_flush_all();
2051 init_vtd_hw();
2052 for_each_drhd_unit ( drhd )
2054 iommu = drhd->iommu;
2055 dmar_writeq( iommu->reg, DMAR_RTADDR_REG,
2056 (u64) iommu_state[DMAR_RTADDR_REG * i]);
2057 dmar_writel(iommu->reg, DMAR_FECTL_REG,
2058 (u32) iommu_state[DMAR_FECTL_REG * i]);
2059 dmar_writel(iommu->reg, DMAR_FEDATA_REG,
2060 (u32) iommu_state[DMAR_FEDATA_REG * i]);
2061 dmar_writel(iommu->reg, DMAR_FEADDR_REG,
2062 (u32) iommu_state[DMAR_FEADDR_REG * i]);
2063 dmar_writel(iommu->reg, DMAR_FEUADDR_REG,
2064 (u32) iommu_state[DMAR_FEUADDR_REG * i]);
2065 dmar_writel(iommu->reg, DMAR_PLMBASE_REG,
2066 (u32) iommu_state[DMAR_PLMBASE_REG * i]);
2067 dmar_writel(iommu->reg, DMAR_PLMLIMIT_REG,
2068 (u32) iommu_state[DMAR_PLMLIMIT_REG * i]);
2069 dmar_writeq(iommu->reg, DMAR_PHMBASE_REG,
2070 (u64) iommu_state[DMAR_PHMBASE_REG * i]);
2071 dmar_writeq(iommu->reg, DMAR_PHMLIMIT_REG,
2072 (u64) iommu_state[DMAR_PHMLIMIT_REG * i]);
2074 if ( iommu_enable_translation(iommu) )
2075 return -EIO;
2076 i++;
2078 return 0;
2081 /*
2082 * Local variables:
2083 * mode: C
2084 * c-set-style: "BSD"
2085 * c-basic-offset: 4
2086 * tab-width: 4
2087 * indent-tabs-mode: nil
2088 * End:
2089 */