debuggers.hg

view xen/arch/x86/hvm/vmx/vtd/intel-iommu.c @ 16710:c0a9b8ab631c

vt-d: Only setup rmrr identity mappings of the assigned devices

Current implementation setups identity mappings for all the rmrrs
whether the assigned device has rmrr or not. It's wasteful and not
necessary, because only USB controllers and UMA integarated graphics
devices have rmrr. This patch only setups rmrr identity mappings of
the assigned devices. If the assigned devices don't have rmrr, needn't
to setup rmrr identity mapping for guest.

BTW, this patch fixes the VT-d bug (Fail to boot smp Linux guest with
VT-d NIC assigned on IA32e platform).

Signed-off-by: Weidong Han <weidong.han@intel.com>
author Keir Fraser <keir.fraser@citrix.com>
date Fri Dec 28 15:26:27 2007 +0000 (2007-12-28)
parents 8ae3f083490a
children f983aa8e4b26
line source
1 /*
2 * Copyright (c) 2006, Intel Corporation.
3 *
4 * This program is free software; you can redistribute it and/or modify it
5 * under the terms and conditions of the GNU General Public License,
6 * version 2, as published by the Free Software Foundation.
7 *
8 * This program is distributed in the hope it will be useful, but WITHOUT
9 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
10 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
11 * more details.
12 *
13 * You should have received a copy of the GNU General Public License along with
14 * this program; if not, write to the Free Software Foundation, Inc., 59 Temple
15 * Place - Suite 330, Boston, MA 02111-1307 USA.
16 *
17 * Copyright (C) Ashok Raj <ashok.raj@intel.com>
18 * Copyright (C) Shaohua Li <shaohua.li@intel.com>
19 * Copyright (C) Allen Kay <allen.m.kay@intel.com> - adapted to xen
20 */
22 #include <xen/init.h>
23 #include <xen/irq.h>
24 #include <xen/spinlock.h>
25 #include <xen/sched.h>
26 #include <xen/xmalloc.h>
27 #include <xen/domain_page.h>
28 #include <asm/delay.h>
29 #include <asm/string.h>
30 #include <asm/mm.h>
31 #include <asm/iommu.h>
32 #include <asm/hvm/vmx/intel-iommu.h>
33 #include "dmar.h"
34 #include "pci-direct.h"
35 #include "pci_regs.h"
36 #include "msi.h"
38 #define domain_iommu_domid(d) ((d)->arch.hvm_domain.hvm_iommu.iommu_domid)
40 extern void print_iommu_regs(struct acpi_drhd_unit *drhd);
41 extern void print_vtd_entries(struct domain *d, int bus, int devfn,
42 unsigned long gmfn);
44 static spinlock_t domid_bitmap_lock; /* protect domain id bitmap */
45 static int domid_bitmap_size; /* domain id bitmap size in bit */
46 static void *domid_bitmap; /* iommu domain id bitmap */
48 #define DID_FIELD_WIDTH 16
49 #define DID_HIGH_OFFSET 8
50 static void context_set_domain_id(struct context_entry *context,
51 struct domain *d)
52 {
53 unsigned long flags;
54 domid_t iommu_domid = domain_iommu_domid(d);
56 if ( iommu_domid == 0 )
57 {
58 spin_lock_irqsave(&domid_bitmap_lock, flags);
59 iommu_domid = find_first_zero_bit(domid_bitmap, domid_bitmap_size);
60 set_bit(iommu_domid, domid_bitmap);
61 spin_unlock_irqrestore(&domid_bitmap_lock, flags);
62 d->arch.hvm_domain.hvm_iommu.iommu_domid = iommu_domid;
63 }
65 context->hi &= (1 << DID_HIGH_OFFSET) - 1;
66 context->hi |= iommu_domid << DID_HIGH_OFFSET;
67 }
69 static void iommu_domid_release(struct domain *d)
70 {
71 domid_t iommu_domid = domain_iommu_domid(d);
73 if ( iommu_domid != 0 )
74 {
75 d->arch.hvm_domain.hvm_iommu.iommu_domid = 0;
76 clear_bit(iommu_domid, domid_bitmap);
77 }
78 }
80 unsigned int x86_clflush_size;
81 void clflush_cache_range(void *adr, int size)
82 {
83 int i;
84 for ( i = 0; i < size; i += x86_clflush_size )
85 clflush(adr + i);
86 }
88 static void __iommu_flush_cache(struct iommu *iommu, void *addr, int size)
89 {
90 if ( !ecap_coherent(iommu->ecap) )
91 clflush_cache_range(addr, size);
92 }
94 #define iommu_flush_cache_entry(iommu, addr) \
95 __iommu_flush_cache(iommu, addr, 8)
96 #define iommu_flush_cache_page(iommu, addr) \
97 __iommu_flush_cache(iommu, addr, PAGE_SIZE_4K)
99 int nr_iommus;
100 /* context entry handling */
101 static struct context_entry * device_to_context_entry(struct iommu *iommu,
102 u8 bus, u8 devfn)
103 {
104 struct root_entry *root;
105 struct context_entry *context;
106 unsigned long phy_addr;
107 unsigned long flags;
109 spin_lock_irqsave(&iommu->lock, flags);
110 root = &iommu->root_entry[bus];
111 if ( !root_present(*root) )
112 {
113 phy_addr = (unsigned long) alloc_xenheap_page();
114 if ( !phy_addr )
115 {
116 spin_unlock_irqrestore(&iommu->lock, flags);
117 return NULL;
118 }
119 memset((void *) phy_addr, 0, PAGE_SIZE);
120 iommu_flush_cache_page(iommu, (void *)phy_addr);
121 phy_addr = virt_to_maddr((void *)phy_addr);
122 set_root_value(*root, phy_addr);
123 set_root_present(*root);
124 iommu_flush_cache_entry(iommu, root);
125 }
126 phy_addr = (unsigned long) get_context_addr(*root);
127 context = (struct context_entry *)maddr_to_virt(phy_addr);
128 spin_unlock_irqrestore(&iommu->lock, flags);
129 return &context[devfn];
130 }
132 static int device_context_mapped(struct iommu *iommu, u8 bus, u8 devfn)
133 {
134 struct root_entry *root;
135 struct context_entry *context;
136 unsigned long phy_addr;
137 int ret;
138 unsigned long flags;
140 spin_lock_irqsave(&iommu->lock, flags);
141 root = &iommu->root_entry[bus];
142 if ( !root_present(*root) )
143 {
144 ret = 0;
145 goto out;
146 }
147 phy_addr = get_context_addr(*root);
148 context = (struct context_entry *)maddr_to_virt(phy_addr);
149 ret = context_present(context[devfn]);
150 out:
151 spin_unlock_irqrestore(&iommu->lock, flags);
152 return ret;
153 }
155 static struct page_info *addr_to_dma_page(struct domain *domain, u64 addr)
156 {
157 struct hvm_iommu *hd = domain_hvm_iommu(domain);
158 struct acpi_drhd_unit *drhd;
159 struct iommu *iommu;
160 int addr_width = agaw_to_width(hd->agaw);
161 struct dma_pte *parent, *pte = NULL, *pgd;
162 int level = agaw_to_level(hd->agaw);
163 int offset;
164 unsigned long flags;
165 struct page_info *pg = NULL;
166 u64 *vaddr = NULL;
168 drhd = list_entry(acpi_drhd_units.next, typeof(*drhd), list);
169 iommu = drhd->iommu;
171 addr &= (((u64)1) << addr_width) - 1;
172 spin_lock_irqsave(&hd->mapping_lock, flags);
173 if ( !hd->pgd )
174 {
175 pgd = (struct dma_pte *)alloc_xenheap_page();
176 if ( !pgd )
177 {
178 spin_unlock_irqrestore(&hd->mapping_lock, flags);
179 return NULL;
180 }
181 memset(pgd, 0, PAGE_SIZE);
182 hd->pgd = pgd;
183 }
185 parent = hd->pgd;
186 while ( level > 1 )
187 {
188 offset = address_level_offset(addr, level);
189 pte = &parent[offset];
191 if ( dma_pte_addr(*pte) == 0 )
192 {
193 pg = alloc_domheap_page(NULL);
194 vaddr = map_domain_page(page_to_mfn(pg));
195 if ( !vaddr )
196 {
197 spin_unlock_irqrestore(&hd->mapping_lock, flags);
198 return NULL;
199 }
200 memset(vaddr, 0, PAGE_SIZE);
201 iommu_flush_cache_page(iommu, vaddr);
203 dma_set_pte_addr(*pte, page_to_maddr(pg));
205 /*
206 * high level table always sets r/w, last level
207 * page table control read/write
208 */
209 dma_set_pte_readable(*pte);
210 dma_set_pte_writable(*pte);
211 iommu_flush_cache_entry(iommu, pte);
212 }
213 else
214 {
215 pg = maddr_to_page(pte->val);
216 vaddr = map_domain_page(page_to_mfn(pg));
217 if ( !vaddr )
218 {
219 spin_unlock_irqrestore(&hd->mapping_lock, flags);
220 return NULL;
221 }
222 }
224 if ( parent != hd->pgd )
225 unmap_domain_page(parent);
227 if ( level == 2 && vaddr )
228 {
229 unmap_domain_page(vaddr);
230 break;
231 }
233 parent = (struct dma_pte *)vaddr;
234 vaddr = NULL;
235 level--;
236 }
238 spin_unlock_irqrestore(&hd->mapping_lock, flags);
239 return pg;
240 }
242 /* return address's page at specific level */
243 static struct page_info *dma_addr_level_page(struct domain *domain,
244 u64 addr, int level)
245 {
246 struct hvm_iommu *hd = domain_hvm_iommu(domain);
247 struct dma_pte *parent, *pte = NULL;
248 int total = agaw_to_level(hd->agaw);
249 int offset;
250 struct page_info *pg = NULL;
252 parent = hd->pgd;
253 while ( level <= total )
254 {
255 offset = address_level_offset(addr, total);
256 pte = &parent[offset];
257 if ( dma_pte_addr(*pte) == 0 )
258 {
259 if ( parent != hd->pgd )
260 unmap_domain_page(parent);
261 break;
262 }
264 pg = maddr_to_page(pte->val);
265 if ( parent != hd->pgd )
266 unmap_domain_page(parent);
268 if ( level == total )
269 return pg;
271 parent = map_domain_page(page_to_mfn(pg));
272 total--;
273 }
275 return NULL;
276 }
278 static void iommu_flush_write_buffer(struct iommu *iommu)
279 {
280 u32 val;
281 unsigned long flag;
282 unsigned long start_time;
284 if ( !cap_rwbf(iommu->cap) )
285 return;
286 val = iommu->gcmd | DMA_GCMD_WBF;
288 spin_lock_irqsave(&iommu->register_lock, flag);
289 dmar_writel(iommu->reg, DMAR_GCMD_REG, val);
291 /* Make sure hardware complete it */
292 start_time = jiffies;
293 for ( ; ; )
294 {
295 val = dmar_readl(iommu->reg, DMAR_GSTS_REG);
296 if ( !(val & DMA_GSTS_WBFS) )
297 break;
298 if ( time_after(jiffies, start_time + DMAR_OPERATION_TIMEOUT) )
299 panic("DMAR hardware is malfunctional,"
300 " please disable IOMMU\n");
301 cpu_relax();
302 }
303 spin_unlock_irqrestore(&iommu->register_lock, flag);
304 }
306 /* return value determine if we need a write buffer flush */
307 static int __iommu_flush_context(
308 struct iommu *iommu,
309 u16 did, u16 source_id, u8 function_mask, u64 type,
310 int non_present_entry_flush)
311 {
312 u64 val = 0;
313 unsigned long flag;
314 unsigned long start_time;
316 /*
317 * In the non-present entry flush case, if hardware doesn't cache
318 * non-present entry we do nothing and if hardware cache non-present
319 * entry, we flush entries of domain 0 (the domain id is used to cache
320 * any non-present entries)
321 */
322 if ( non_present_entry_flush )
323 {
324 if ( !cap_caching_mode(iommu->cap) )
325 return 1;
326 else
327 did = 0;
328 }
330 /* use register invalidation */
331 switch ( type )
332 {
333 case DMA_CCMD_GLOBAL_INVL:
334 val = DMA_CCMD_GLOBAL_INVL;
335 break;
336 case DMA_CCMD_DOMAIN_INVL:
337 val = DMA_CCMD_DOMAIN_INVL|DMA_CCMD_DID(did);
338 break;
339 case DMA_CCMD_DEVICE_INVL:
340 val = DMA_CCMD_DEVICE_INVL|DMA_CCMD_DID(did)
341 |DMA_CCMD_SID(source_id)|DMA_CCMD_FM(function_mask);
342 break;
343 default:
344 BUG();
345 }
346 val |= DMA_CCMD_ICC;
348 spin_lock_irqsave(&iommu->register_lock, flag);
349 dmar_writeq(iommu->reg, DMAR_CCMD_REG, val);
351 /* Make sure hardware complete it */
352 start_time = jiffies;
353 for ( ; ; )
354 {
355 val = dmar_readq(iommu->reg, DMAR_CCMD_REG);
356 if ( !(val & DMA_CCMD_ICC) )
357 break;
358 if ( time_after(jiffies, start_time + DMAR_OPERATION_TIMEOUT) )
359 panic("DMAR hardware is malfunctional, please disable IOMMU\n");
360 cpu_relax();
361 }
362 spin_unlock_irqrestore(&iommu->register_lock, flag);
363 /* flush context entry will implictly flush write buffer */
364 return 0;
365 }
367 static int inline iommu_flush_context_global(
368 struct iommu *iommu, int non_present_entry_flush)
369 {
370 return __iommu_flush_context(iommu, 0, 0, 0, DMA_CCMD_GLOBAL_INVL,
371 non_present_entry_flush);
372 }
374 static int inline iommu_flush_context_domain(
375 struct iommu *iommu, u16 did, int non_present_entry_flush)
376 {
377 return __iommu_flush_context(iommu, did, 0, 0, DMA_CCMD_DOMAIN_INVL,
378 non_present_entry_flush);
379 }
381 static int inline iommu_flush_context_device(
382 struct iommu *iommu, u16 did, u16 source_id,
383 u8 function_mask, int non_present_entry_flush)
384 {
385 return __iommu_flush_context(iommu, did, source_id, function_mask,
386 DMA_CCMD_DEVICE_INVL,
387 non_present_entry_flush);
388 }
390 /* return value determine if we need a write buffer flush */
391 static int __iommu_flush_iotlb(struct iommu *iommu, u16 did,
392 u64 addr, unsigned int size_order, u64 type,
393 int non_present_entry_flush)
394 {
395 int tlb_offset = ecap_iotlb_offset(iommu->ecap);
396 u64 val = 0, val_iva = 0;
397 unsigned long flag;
398 unsigned long start_time;
400 /*
401 * In the non-present entry flush case, if hardware doesn't cache
402 * non-present entry we do nothing and if hardware cache non-present
403 * entry, we flush entries of domain 0 (the domain id is used to cache
404 * any non-present entries)
405 */
406 if ( non_present_entry_flush )
407 {
408 if ( !cap_caching_mode(iommu->cap) )
409 return 1;
410 else
411 did = 0;
412 }
414 /* use register invalidation */
415 switch ( type )
416 {
417 case DMA_TLB_GLOBAL_FLUSH:
418 /* global flush doesn't need set IVA_REG */
419 val = DMA_TLB_GLOBAL_FLUSH|DMA_TLB_IVT;
420 break;
421 case DMA_TLB_DSI_FLUSH:
422 val = DMA_TLB_DSI_FLUSH|DMA_TLB_IVT|DMA_TLB_DID(did);
423 break;
424 case DMA_TLB_PSI_FLUSH:
425 val = DMA_TLB_PSI_FLUSH|DMA_TLB_IVT|DMA_TLB_DID(did);
426 /* Note: always flush non-leaf currently */
427 val_iva = size_order | addr;
428 break;
429 default:
430 BUG();
431 }
432 /* Note: set drain read/write */
433 if ( cap_read_drain(iommu->cap) )
434 val |= DMA_TLB_READ_DRAIN;
435 if ( cap_write_drain(iommu->cap) )
436 val |= DMA_TLB_WRITE_DRAIN;
438 spin_lock_irqsave(&iommu->register_lock, flag);
439 /* Note: Only uses first TLB reg currently */
440 if ( val_iva )
441 dmar_writeq(iommu->reg, tlb_offset, val_iva);
442 dmar_writeq(iommu->reg, tlb_offset + 8, val);
444 /* Make sure hardware complete it */
445 start_time = jiffies;
446 for ( ; ; )
447 {
448 val = dmar_readq(iommu->reg, tlb_offset + 8);
449 if ( !(val & DMA_TLB_IVT) )
450 break;
451 if ( time_after(jiffies, start_time + DMAR_OPERATION_TIMEOUT) )
452 panic("DMAR hardware is malfunctional, please disable IOMMU\n");
453 cpu_relax();
454 }
455 spin_unlock_irqrestore(&iommu->register_lock, flag);
457 /* check IOTLB invalidation granularity */
458 if ( DMA_TLB_IAIG(val) == 0 )
459 printk(KERN_ERR VTDPREFIX "IOMMU: flush IOTLB failed\n");
460 if ( DMA_TLB_IAIG(val) != DMA_TLB_IIRG(type) )
461 printk(KERN_ERR VTDPREFIX "IOMMU: tlb flush request %x, actual %x\n",
462 (u32)DMA_TLB_IIRG(type), (u32)DMA_TLB_IAIG(val));
463 /* flush context entry will implictly flush write buffer */
464 return 0;
465 }
467 static int inline iommu_flush_iotlb_global(struct iommu *iommu,
468 int non_present_entry_flush)
469 {
470 return __iommu_flush_iotlb(iommu, 0, 0, 0, DMA_TLB_GLOBAL_FLUSH,
471 non_present_entry_flush);
472 }
474 static int inline iommu_flush_iotlb_dsi(struct iommu *iommu, u16 did,
475 int non_present_entry_flush)
476 {
477 return __iommu_flush_iotlb(iommu, did, 0, 0, DMA_TLB_DSI_FLUSH,
478 non_present_entry_flush);
479 }
481 static int inline get_alignment(u64 base, unsigned int size)
482 {
483 int t = 0;
484 u64 end;
486 end = base + size - 1;
487 while ( base != end )
488 {
489 t++;
490 base >>= 1;
491 end >>= 1;
492 }
493 return t;
494 }
496 static int inline iommu_flush_iotlb_psi(
497 struct iommu *iommu, u16 did,
498 u64 addr, unsigned int pages, int non_present_entry_flush)
499 {
500 unsigned int align;
502 BUG_ON(addr & (~PAGE_MASK_4K));
503 BUG_ON(pages == 0);
505 /* Fallback to domain selective flush if no PSI support */
506 if ( !cap_pgsel_inv(iommu->cap) )
507 return iommu_flush_iotlb_dsi(iommu, did,
508 non_present_entry_flush);
510 /*
511 * PSI requires page size is 2 ^ x, and the base address is naturally
512 * aligned to the size
513 */
514 align = get_alignment(addr >> PAGE_SHIFT_4K, pages);
515 /* Fallback to domain selective flush if size is too big */
516 if ( align > cap_max_amask_val(iommu->cap) )
517 return iommu_flush_iotlb_dsi(iommu, did,
518 non_present_entry_flush);
520 addr >>= PAGE_SHIFT_4K + align;
521 addr <<= PAGE_SHIFT_4K + align;
523 return __iommu_flush_iotlb(iommu, did, addr, align,
524 DMA_TLB_PSI_FLUSH, non_present_entry_flush);
525 }
527 void iommu_flush_all(void)
528 {
529 struct acpi_drhd_unit *drhd;
530 struct iommu *iommu;
531 int i = 0;
533 wbinvd();
534 for_each_drhd_unit ( drhd )
535 {
536 iommu = drhd->iommu;
537 iommu_flush_context_global(iommu, 0);
538 iommu_flush_iotlb_global(iommu, 0);
539 i++;
540 }
541 }
543 /* clear one page's page table */
544 static void dma_pte_clear_one(struct domain *domain, u64 addr)
545 {
546 struct acpi_drhd_unit *drhd;
547 struct iommu *iommu;
548 struct dma_pte *pte = NULL;
549 struct page_info *pg = NULL;
551 drhd = list_entry(acpi_drhd_units.next, typeof(*drhd), list);
553 /* get last level pte */
554 pg = dma_addr_level_page(domain, addr, 1);
555 if ( !pg )
556 return;
557 pte = (struct dma_pte *)map_domain_page(page_to_mfn(pg));
558 pte += address_level_offset(addr, 1);
559 if ( pte )
560 {
561 dma_clear_pte(*pte);
562 iommu_flush_cache_entry(drhd->iommu, pte);
564 for_each_drhd_unit ( drhd )
565 {
566 iommu = drhd->iommu;
567 if ( cap_caching_mode(iommu->cap) )
568 iommu_flush_iotlb_psi(iommu, domain_iommu_domid(domain),
569 addr, 1, 0);
570 else if (cap_rwbf(iommu->cap))
571 iommu_flush_write_buffer(iommu);
572 }
573 }
574 unmap_domain_page(pte);
575 }
577 /* clear last level pte, a tlb flush should be followed */
578 static void dma_pte_clear_range(struct domain *domain, u64 start, u64 end)
579 {
580 struct hvm_iommu *hd = domain_hvm_iommu(domain);
581 int addr_width = agaw_to_width(hd->agaw);
583 start &= (((u64)1) << addr_width) - 1;
584 end &= (((u64)1) << addr_width) - 1;
585 /* in case it's partial page */
586 start = PAGE_ALIGN_4K(start);
587 end &= PAGE_MASK_4K;
589 /* we don't need lock here, nobody else touches the iova range */
590 while ( start < end )
591 {
592 dma_pte_clear_one(domain, start);
593 start += PAGE_SIZE_4K;
594 }
595 }
597 /* free page table pages. last level pte should already be cleared */
598 void dma_pte_free_pagetable(struct domain *domain, u64 start, u64 end)
599 {
600 struct acpi_drhd_unit *drhd;
601 struct hvm_iommu *hd = domain_hvm_iommu(domain);
602 struct iommu *iommu;
603 int addr_width = agaw_to_width(hd->agaw);
604 struct dma_pte *pte;
605 int total = agaw_to_level(hd->agaw);
606 int level;
607 u32 tmp;
608 struct page_info *pg = NULL;
610 drhd = list_entry(acpi_drhd_units.next, typeof(*drhd), list);
611 iommu = drhd->iommu;
613 start &= (((u64)1) << addr_width) - 1;
614 end &= (((u64)1) << addr_width) - 1;
616 /* we don't need lock here, nobody else touches the iova range */
617 level = 2;
618 while ( level <= total )
619 {
620 tmp = align_to_level(start, level);
621 if ( (tmp >= end) || ((tmp + level_size(level)) > end) )
622 return;
624 while ( tmp < end )
625 {
626 pg = dma_addr_level_page(domain, tmp, level);
627 if ( !pg )
628 return;
629 pte = (struct dma_pte *)map_domain_page(page_to_mfn(pg));
630 pte += address_level_offset(tmp, level);
631 dma_clear_pte(*pte);
632 iommu_flush_cache_entry(iommu, pte);
633 unmap_domain_page(pte);
634 free_domheap_page(pg);
636 tmp += level_size(level);
637 }
638 level++;
639 }
641 /* free pgd */
642 if ( start == 0 && end == ((((u64)1) << addr_width) - 1) )
643 {
644 free_xenheap_page((void *)hd->pgd);
645 hd->pgd = NULL;
646 }
647 }
649 /* iommu handling */
650 static int iommu_set_root_entry(struct iommu *iommu)
651 {
652 void *addr;
653 u32 cmd, sts;
654 struct root_entry *root;
655 unsigned long flags;
657 if ( iommu == NULL )
658 {
659 gdprintk(XENLOG_ERR VTDPREFIX,
660 "iommu_set_root_entry: iommu == NULL\n");
661 return -EINVAL;
662 }
664 if ( unlikely(!iommu->root_entry) )
665 {
666 root = (struct root_entry *)alloc_xenheap_page();
667 if ( root == NULL )
668 return -ENOMEM;
670 memset((u8*)root, 0, PAGE_SIZE);
671 iommu_flush_cache_page(iommu, root);
673 if ( cmpxchg((unsigned long *)&iommu->root_entry,
674 0, (unsigned long)root) != 0 )
675 free_xenheap_page((void *)root);
676 }
678 addr = iommu->root_entry;
680 spin_lock_irqsave(&iommu->register_lock, flags);
682 dmar_writeq(iommu->reg, DMAR_RTADDR_REG, virt_to_maddr(addr));
683 cmd = iommu->gcmd | DMA_GCMD_SRTP;
684 dmar_writel(iommu->reg, DMAR_GCMD_REG, cmd);
686 /* Make sure hardware complete it */
687 for ( ; ; )
688 {
689 sts = dmar_readl(iommu->reg, DMAR_GSTS_REG);
690 if ( sts & DMA_GSTS_RTPS )
691 break;
692 cpu_relax();
693 }
695 spin_unlock_irqrestore(&iommu->register_lock, flags);
697 return 0;
698 }
700 static int iommu_enable_translation(struct iommu *iommu)
701 {
702 u32 sts;
703 unsigned long flags;
705 dprintk(XENLOG_INFO VTDPREFIX,
706 "iommu_enable_translation: enabling vt-d translation\n");
707 spin_lock_irqsave(&iommu->register_lock, flags);
708 iommu->gcmd |= DMA_GCMD_TE;
709 dmar_writel(iommu->reg, DMAR_GCMD_REG, iommu->gcmd);
710 /* Make sure hardware complete it */
711 for ( ; ; )
712 {
713 sts = dmar_readl(iommu->reg, DMAR_GSTS_REG);
714 if ( sts & DMA_GSTS_TES )
715 break;
716 cpu_relax();
717 }
719 /* Disable PMRs when VT-d engine takes effect per spec definition */
720 disable_pmr(iommu);
721 spin_unlock_irqrestore(&iommu->register_lock, flags);
722 return 0;
723 }
725 int iommu_disable_translation(struct iommu *iommu)
726 {
727 u32 sts;
728 unsigned long flags;
730 spin_lock_irqsave(&iommu->register_lock, flags);
731 iommu->gcmd &= ~ DMA_GCMD_TE;
732 dmar_writel(iommu->reg, DMAR_GCMD_REG, iommu->gcmd);
734 /* Make sure hardware complete it */
735 for ( ; ; )
736 {
737 sts = dmar_readl(iommu->reg, DMAR_GSTS_REG);
738 if ( !(sts & DMA_GSTS_TES) )
739 break;
740 cpu_relax();
741 }
742 spin_unlock_irqrestore(&iommu->register_lock, flags);
743 return 0;
744 }
746 static struct iommu *vector_to_iommu[NR_VECTORS];
747 static int iommu_page_fault_do_one(struct iommu *iommu, int type,
748 u8 fault_reason, u16 source_id, u32 addr)
749 {
750 dprintk(XENLOG_WARNING VTDPREFIX,
751 "iommu_page_fault:%s: DEVICE %x:%x.%x addr %x REASON %x\n",
752 (type ? "DMA Read" : "DMA Write"),
753 (source_id >> 8), PCI_SLOT(source_id & 0xFF),
754 PCI_FUNC(source_id & 0xFF), addr, fault_reason);
756 print_vtd_entries(current->domain, (source_id >> 8),(source_id & 0xff),
757 (addr >> PAGE_SHIFT));
758 return 0;
759 }
761 #define PRIMARY_FAULT_REG_LEN (16)
762 static void iommu_page_fault(int vector, void *dev_id,
763 struct cpu_user_regs *regs)
764 {
765 struct iommu *iommu = dev_id;
766 int reg, fault_index;
767 u32 fault_status;
768 unsigned long flags;
770 dprintk(XENLOG_WARNING VTDPREFIX,
771 "iommu_page_fault: iommu->reg = %p\n", iommu->reg);
773 spin_lock_irqsave(&iommu->register_lock, flags);
774 fault_status = dmar_readl(iommu->reg, DMAR_FSTS_REG);
775 spin_unlock_irqrestore(&iommu->register_lock, flags);
777 /* FIXME: ignore advanced fault log */
778 if ( !(fault_status & DMA_FSTS_PPF) )
779 return;
780 fault_index = dma_fsts_fault_record_index(fault_status);
781 reg = cap_fault_reg_offset(iommu->cap);
782 for ( ; ; )
783 {
784 u8 fault_reason;
785 u16 source_id;
786 u32 guest_addr, data;
787 int type;
789 /* highest 32 bits */
790 spin_lock_irqsave(&iommu->register_lock, flags);
791 data = dmar_readl(iommu->reg, reg +
792 fault_index * PRIMARY_FAULT_REG_LEN + 12);
793 if ( !(data & DMA_FRCD_F) )
794 {
795 spin_unlock_irqrestore(&iommu->register_lock, flags);
796 break;
797 }
799 fault_reason = dma_frcd_fault_reason(data);
800 type = dma_frcd_type(data);
802 data = dmar_readl(iommu->reg, reg +
803 fault_index * PRIMARY_FAULT_REG_LEN + 8);
804 source_id = dma_frcd_source_id(data);
806 guest_addr = dmar_readq(iommu->reg, reg +
807 fault_index * PRIMARY_FAULT_REG_LEN);
808 guest_addr = dma_frcd_page_addr(guest_addr);
809 /* clear the fault */
810 dmar_writel(iommu->reg, reg +
811 fault_index * PRIMARY_FAULT_REG_LEN + 12, DMA_FRCD_F);
812 spin_unlock_irqrestore(&iommu->register_lock, flags);
814 iommu_page_fault_do_one(iommu, type, fault_reason,
815 source_id, guest_addr);
817 fault_index++;
818 if ( fault_index > cap_num_fault_regs(iommu->cap) )
819 fault_index = 0;
820 }
822 /* clear primary fault overflow */
823 if ( fault_status & DMA_FSTS_PFO )
824 {
825 spin_lock_irqsave(&iommu->register_lock, flags);
826 dmar_writel(iommu->reg, DMAR_FSTS_REG, DMA_FSTS_PFO);
827 spin_unlock_irqrestore(&iommu->register_lock, flags);
828 }
829 }
831 static void dma_msi_unmask(unsigned int vector)
832 {
833 struct iommu *iommu = vector_to_iommu[vector];
834 unsigned long flags;
836 /* unmask it */
837 spin_lock_irqsave(&iommu->register_lock, flags);
838 dmar_writel(iommu->reg, DMAR_FECTL_REG, 0);
839 spin_unlock_irqrestore(&iommu->register_lock, flags);
840 }
842 static void dma_msi_mask(unsigned int vector)
843 {
844 unsigned long flags;
845 struct iommu *iommu = vector_to_iommu[vector];
847 /* mask it */
848 spin_lock_irqsave(&iommu->register_lock, flags);
849 dmar_writel(iommu->reg, DMAR_FECTL_REG, DMA_FECTL_IM);
850 spin_unlock_irqrestore(&iommu->register_lock, flags);
851 }
853 static unsigned int dma_msi_startup(unsigned int vector)
854 {
855 dma_msi_unmask(vector);
856 return 0;
857 }
859 static void dma_msi_end(unsigned int vector)
860 {
861 dma_msi_unmask(vector);
862 ack_APIC_irq();
863 }
865 static void dma_msi_data_init(struct iommu *iommu, int vector)
866 {
867 u32 msi_data = 0;
868 unsigned long flags;
870 /* Fixed, edge, assert mode. Follow MSI setting */
871 msi_data |= vector & 0xff;
872 msi_data |= 1 << 14;
874 spin_lock_irqsave(&iommu->register_lock, flags);
875 dmar_writel(iommu->reg, DMAR_FEDATA_REG, msi_data);
876 spin_unlock_irqrestore(&iommu->register_lock, flags);
877 }
879 static void dma_msi_addr_init(struct iommu *iommu, int phy_cpu)
880 {
881 u64 msi_address;
882 unsigned long flags;
884 /* Physical, dedicated cpu. Follow MSI setting */
885 msi_address = (MSI_ADDRESS_HEADER << (MSI_ADDRESS_HEADER_SHIFT + 8));
886 msi_address |= MSI_PHYSICAL_MODE << 2;
887 msi_address |= MSI_REDIRECTION_HINT_MODE << 3;
888 msi_address |= phy_cpu << MSI_TARGET_CPU_SHIFT;
890 spin_lock_irqsave(&iommu->register_lock, flags);
891 dmar_writel(iommu->reg, DMAR_FEADDR_REG, (u32)msi_address);
892 dmar_writel(iommu->reg, DMAR_FEUADDR_REG, (u32)(msi_address >> 32));
893 spin_unlock_irqrestore(&iommu->register_lock, flags);
894 }
896 static void dma_msi_set_affinity(unsigned int vector, cpumask_t dest)
897 {
898 struct iommu *iommu = vector_to_iommu[vector];
899 dma_msi_addr_init(iommu, cpu_physical_id(first_cpu(dest)));
900 }
902 static struct hw_interrupt_type dma_msi_type = {
903 .typename = "DMA_MSI",
904 .startup = dma_msi_startup,
905 .shutdown = dma_msi_mask,
906 .enable = dma_msi_unmask,
907 .disable = dma_msi_mask,
908 .ack = dma_msi_mask,
909 .end = dma_msi_end,
910 .set_affinity = dma_msi_set_affinity,
911 };
913 int iommu_set_interrupt(struct iommu *iommu)
914 {
915 int vector, ret;
917 vector = assign_irq_vector(AUTO_ASSIGN);
918 vector_to_iommu[vector] = iommu;
920 /* VT-d fault is a MSI, make irq == vector */
921 irq_vector[vector] = vector;
922 vector_irq[vector] = vector;
924 if ( !vector )
925 {
926 gdprintk(XENLOG_ERR VTDPREFIX, "IOMMU: no vectors\n");
927 return -EINVAL;
928 }
930 irq_desc[vector].handler = &dma_msi_type;
931 ret = request_irq(vector, iommu_page_fault, 0, "dmar", iommu);
932 if ( ret )
933 gdprintk(XENLOG_ERR VTDPREFIX, "IOMMU: can't request irq\n");
934 return vector;
935 }
937 struct iommu *iommu_alloc(void *hw_data)
938 {
939 struct acpi_drhd_unit *drhd = (struct acpi_drhd_unit *) hw_data;
940 struct iommu *iommu;
942 if ( nr_iommus > MAX_IOMMUS )
943 {
944 gdprintk(XENLOG_ERR VTDPREFIX,
945 "IOMMU: nr_iommus %d > MAX_IOMMUS\n", nr_iommus);
946 return NULL;
947 }
949 iommu = xmalloc(struct iommu);
950 if ( !iommu )
951 return NULL;
952 memset(iommu, 0, sizeof(struct iommu));
954 set_fixmap_nocache(FIX_IOMMU_REGS_BASE_0 + nr_iommus, drhd->address);
955 iommu->reg = (void *) fix_to_virt(FIX_IOMMU_REGS_BASE_0 + nr_iommus);
956 dprintk(XENLOG_INFO VTDPREFIX,
957 "iommu_alloc: iommu->reg = %p drhd->address = %lx\n",
958 iommu->reg, drhd->address);
959 nr_iommus++;
961 if ( !iommu->reg )
962 {
963 printk(KERN_ERR VTDPREFIX "IOMMU: can't mapping the region\n");
964 goto error;
965 }
967 iommu->cap = dmar_readq(iommu->reg, DMAR_CAP_REG);
968 iommu->ecap = dmar_readq(iommu->reg, DMAR_ECAP_REG);
970 spin_lock_init(&iommu->lock);
971 spin_lock_init(&iommu->register_lock);
973 drhd->iommu = iommu;
974 return iommu;
975 error:
976 xfree(iommu);
977 return NULL;
978 }
980 static void free_iommu(struct iommu *iommu)
981 {
982 if ( !iommu )
983 return;
984 if ( iommu->root_entry )
985 free_xenheap_page((void *)iommu->root_entry);
986 if ( iommu->reg )
987 iounmap(iommu->reg);
988 free_irq(iommu->vector);
989 xfree(iommu);
990 }
992 #define guestwidth_to_adjustwidth(gaw) ({ \
993 int agaw, r = (gaw - 12) % 9; \
994 agaw = (r == 0) ? gaw : (gaw + 9 - r); \
995 if ( agaw > 64 ) \
996 agaw = 64; \
997 agaw; })
999 int iommu_domain_init(struct domain *domain)
1001 struct hvm_iommu *hd = domain_hvm_iommu(domain);
1002 struct iommu *iommu = NULL;
1003 int guest_width = DEFAULT_DOMAIN_ADDRESS_WIDTH;
1004 int adjust_width, agaw;
1005 unsigned long sagaw;
1006 struct acpi_drhd_unit *drhd;
1008 spin_lock_init(&hd->mapping_lock);
1009 spin_lock_init(&hd->iommu_list_lock);
1010 INIT_LIST_HEAD(&hd->pdev_list);
1011 INIT_LIST_HEAD(&hd->g2m_ioport_list);
1013 if ( !vtd_enabled || list_empty(&acpi_drhd_units) )
1014 return 0;
1016 for_each_drhd_unit ( drhd )
1017 iommu = drhd->iommu ? : iommu_alloc(drhd);
1019 /* calculate AGAW */
1020 if (guest_width > cap_mgaw(iommu->cap))
1021 guest_width = cap_mgaw(iommu->cap);
1022 adjust_width = guestwidth_to_adjustwidth(guest_width);
1023 agaw = width_to_agaw(adjust_width);
1024 /* FIXME: hardware doesn't support it, choose a bigger one? */
1025 sagaw = cap_sagaw(iommu->cap);
1026 if ( !test_bit(agaw, &sagaw) )
1028 gdprintk(XENLOG_ERR VTDPREFIX,
1029 "IOMMU: hardware doesn't support the agaw\n");
1030 agaw = find_next_bit(&sagaw, 5, agaw);
1031 if ( agaw >= 5 )
1032 return -ENODEV;
1034 hd->agaw = agaw;
1035 return 0;
1038 static int domain_context_mapping_one(
1039 struct domain *domain,
1040 struct iommu *iommu,
1041 u8 bus, u8 devfn)
1043 struct hvm_iommu *hd = domain_hvm_iommu(domain);
1044 struct context_entry *context;
1045 unsigned long flags;
1046 int ret = 0;
1048 context = device_to_context_entry(iommu, bus, devfn);
1049 if ( !context )
1051 gdprintk(XENLOG_ERR VTDPREFIX,
1052 "domain_context_mapping_one:context == NULL:"
1053 "bdf = %x:%x:%x\n",
1054 bus, PCI_SLOT(devfn), PCI_FUNC(devfn));
1055 return -ENOMEM;
1058 if ( context_present(*context) )
1060 gdprintk(XENLOG_WARNING VTDPREFIX,
1061 "domain_context_mapping_one:context present:bdf=%x:%x:%x\n",
1062 bus, PCI_SLOT(devfn), PCI_FUNC(devfn));
1063 return 0;
1066 spin_lock_irqsave(&iommu->lock, flags);
1067 /*
1068 * domain_id 0 is not valid on Intel's IOMMU, force domain_id to
1069 * be 1 based as required by intel's iommu hw.
1070 */
1071 context_set_domain_id(context, domain);
1072 context_set_address_width(*context, hd->agaw);
1074 if ( ecap_pass_thru(iommu->ecap) )
1075 context_set_translation_type(*context, CONTEXT_TT_PASS_THRU);
1076 else
1078 if ( !hd->pgd )
1080 struct dma_pte *pgd = (struct dma_pte *)alloc_xenheap_page();
1081 if ( !pgd )
1083 spin_unlock_irqrestore(&hd->mapping_lock, flags);
1084 return -ENOMEM;
1086 memset(pgd, 0, PAGE_SIZE);
1087 hd->pgd = pgd;
1090 context_set_address_root(*context, virt_to_maddr(hd->pgd));
1091 context_set_translation_type(*context, CONTEXT_TT_MULTI_LEVEL);
1094 context_set_fault_enable(*context);
1095 context_set_present(*context);
1096 iommu_flush_cache_entry(iommu, context);
1098 gdprintk(XENLOG_INFO VTDPREFIX,
1099 "domain_context_mapping_one-%x:%x:%x-*context=%"PRIx64":%"PRIx64
1100 " hd->pgd=%p\n",
1101 bus, PCI_SLOT(devfn), PCI_FUNC(devfn),
1102 context->hi, context->lo, hd->pgd);
1104 if ( iommu_flush_context_device(iommu, domain_iommu_domid(domain),
1105 (((u16)bus) << 8) | devfn,
1106 DMA_CCMD_MASK_NOBIT, 1) )
1107 iommu_flush_write_buffer(iommu);
1108 else
1109 iommu_flush_iotlb_dsi(iommu, domain_iommu_domid(domain), 0);
1110 spin_unlock_irqrestore(&iommu->lock, flags);
1111 return ret;
1114 static int __pci_find_next_cap(u8 bus, unsigned int devfn, u8 pos, int cap)
1116 u8 id;
1117 int ttl = 48;
1119 while ( ttl-- )
1121 pos = read_pci_config_byte(bus, PCI_SLOT(devfn), PCI_FUNC(devfn), pos);
1122 if ( pos < 0x40 )
1123 break;
1125 pos &= ~3;
1126 id = read_pci_config_byte(bus, PCI_SLOT(devfn), PCI_FUNC(devfn),
1127 pos + PCI_CAP_LIST_ID);
1129 if ( id == 0xff )
1130 break;
1131 if ( id == cap )
1132 return pos;
1134 pos += PCI_CAP_LIST_NEXT;
1136 return 0;
1139 #define PCI_BASE_CLASS_BRIDGE 0x06
1140 #define PCI_CLASS_BRIDGE_PCI 0x0604
1142 #define DEV_TYPE_PCIe_ENDPOINT 1
1143 #define DEV_TYPE_PCI_BRIDGE 2
1144 #define DEV_TYPE_PCI 3
1146 int pdev_type(struct pci_dev *dev)
1148 u16 class_device;
1149 u16 status;
1151 class_device = read_pci_config_16(dev->bus, PCI_SLOT(dev->devfn),
1152 PCI_FUNC(dev->devfn), PCI_CLASS_DEVICE);
1153 if ( class_device == PCI_CLASS_BRIDGE_PCI )
1154 return DEV_TYPE_PCI_BRIDGE;
1156 status = read_pci_config_16(dev->bus, PCI_SLOT(dev->devfn),
1157 PCI_FUNC(dev->devfn), PCI_STATUS);
1159 if ( !(status & PCI_STATUS_CAP_LIST) )
1160 return DEV_TYPE_PCI;
1162 if ( __pci_find_next_cap(dev->bus, dev->devfn,
1163 PCI_CAPABILITY_LIST, PCI_CAP_ID_EXP) )
1164 return DEV_TYPE_PCIe_ENDPOINT;
1166 return DEV_TYPE_PCI;
1169 #define MAX_BUSES 256
1170 struct pci_dev bus2bridge[MAX_BUSES];
1172 static int domain_context_mapping(
1173 struct domain *domain,
1174 struct iommu *iommu,
1175 struct pci_dev *pdev)
1177 int ret = 0;
1178 int dev, func, sec_bus, sub_bus;
1179 u32 type;
1181 type = pdev_type(pdev);
1182 switch ( type )
1184 case DEV_TYPE_PCI_BRIDGE:
1185 sec_bus = read_pci_config_byte(
1186 pdev->bus, PCI_SLOT(pdev->devfn),
1187 PCI_FUNC(pdev->devfn), PCI_SECONDARY_BUS);
1189 if ( bus2bridge[sec_bus].bus == 0 )
1191 bus2bridge[sec_bus].bus = pdev->bus;
1192 bus2bridge[sec_bus].devfn = pdev->devfn;
1195 sub_bus = read_pci_config_byte(
1196 pdev->bus, PCI_SLOT(pdev->devfn),
1197 PCI_FUNC(pdev->devfn), PCI_SUBORDINATE_BUS);
1199 if ( sec_bus != sub_bus )
1200 gdprintk(XENLOG_WARNING VTDPREFIX,
1201 "context_context_mapping: nested PCI bridge not "
1202 "supported: bdf = %x:%x:%x sec_bus = %x sub_bus = %x\n",
1203 pdev->bus, PCI_SLOT(pdev->devfn), PCI_FUNC(pdev->devfn),
1204 sec_bus, sub_bus);
1205 break;
1206 case DEV_TYPE_PCIe_ENDPOINT:
1207 gdprintk(XENLOG_INFO VTDPREFIX,
1208 "domain_context_mapping:PCIe : bdf = %x:%x:%x\n",
1209 pdev->bus, PCI_SLOT(pdev->devfn), PCI_FUNC(pdev->devfn));
1210 ret = domain_context_mapping_one(domain, iommu,
1211 (u8)(pdev->bus), (u8)(pdev->devfn));
1212 break;
1213 case DEV_TYPE_PCI:
1214 gdprintk(XENLOG_INFO VTDPREFIX,
1215 "domain_context_mapping:PCI: bdf = %x:%x:%x\n",
1216 pdev->bus, PCI_SLOT(pdev->devfn), PCI_FUNC(pdev->devfn));
1218 if ( pdev->bus == 0 )
1219 ret = domain_context_mapping_one(
1220 domain, iommu, (u8)(pdev->bus), (u8)(pdev->devfn));
1221 else
1223 if ( bus2bridge[pdev->bus].bus != 0 )
1224 gdprintk(XENLOG_WARNING VTDPREFIX,
1225 "domain_context_mapping:bus2bridge"
1226 "[%d].bus != 0\n", pdev->bus);
1228 ret = domain_context_mapping_one(
1229 domain, iommu,
1230 (u8)(bus2bridge[pdev->bus].bus),
1231 (u8)(bus2bridge[pdev->bus].devfn));
1233 /* now map everything behind the PCI bridge */
1234 for ( dev = 0; dev < 32; dev++ )
1236 for ( func = 0; func < 8; func++ )
1238 ret = domain_context_mapping_one(
1239 domain, iommu,
1240 pdev->bus, (u8)PCI_DEVFN(dev, func));
1241 if ( ret )
1242 return ret;
1246 break;
1247 default:
1248 gdprintk(XENLOG_ERR VTDPREFIX,
1249 "domain_context_mapping:unknown type : bdf = %x:%x:%x\n",
1250 pdev->bus, PCI_SLOT(pdev->devfn), PCI_FUNC(pdev->devfn));
1251 ret = -EINVAL;
1252 break;
1255 return ret;
1258 static int domain_context_unmap_one(
1259 struct domain *domain,
1260 struct iommu *iommu,
1261 u8 bus, u8 devfn)
1263 struct context_entry *context;
1264 unsigned long flags;
1266 context = device_to_context_entry(iommu, bus, devfn);
1267 if ( !context )
1269 gdprintk(XENLOG_ERR VTDPREFIX,
1270 "domain_context_unmap_one-%x:%x:%x- context == NULL:return\n",
1271 bus, PCI_SLOT(devfn), PCI_FUNC(devfn));
1272 return -ENOMEM;
1275 if ( !context_present(*context) )
1277 gdprintk(XENLOG_WARNING VTDPREFIX,
1278 "domain_context_unmap_one-%x:%x:%x- "
1279 "context NOT present:return\n",
1280 bus, PCI_SLOT(devfn), PCI_FUNC(devfn));
1281 return 0;
1284 gdprintk(XENLOG_INFO VTDPREFIX,
1285 "domain_context_unmap_one: bdf = %x:%x:%x\n",
1286 bus, PCI_SLOT(devfn), PCI_FUNC(devfn));
1288 spin_lock_irqsave(&iommu->lock, flags);
1289 context_clear_present(*context);
1290 context_clear_entry(*context);
1291 iommu_flush_cache_entry(iommu, context);
1292 iommu_flush_context_global(iommu, 0);
1293 iommu_flush_iotlb_global(iommu, 0);
1294 spin_unlock_irqrestore(&iommu->lock, flags);
1296 return 0;
1299 static int domain_context_unmap(
1300 struct domain *domain,
1301 struct iommu *iommu,
1302 struct pci_dev *pdev)
1304 int ret = 0;
1305 int dev, func, sec_bus, sub_bus;
1306 u32 type;
1308 type = pdev_type(pdev);
1309 switch ( type )
1311 case DEV_TYPE_PCI_BRIDGE:
1312 sec_bus = read_pci_config_byte(
1313 pdev->bus, PCI_SLOT(pdev->devfn),
1314 PCI_FUNC(pdev->devfn), PCI_SECONDARY_BUS);
1315 sub_bus = read_pci_config_byte(
1316 pdev->bus, PCI_SLOT(pdev->devfn),
1317 PCI_FUNC(pdev->devfn), PCI_SUBORDINATE_BUS);
1319 gdprintk(XENLOG_INFO VTDPREFIX,
1320 "domain_context_unmap:BRIDGE:%x:%x:%x "
1321 "sec_bus=%x sub_bus=%x\n",
1322 pdev->bus, PCI_SLOT(pdev->devfn),
1323 PCI_FUNC(pdev->devfn), sec_bus, sub_bus);
1324 break;
1325 case DEV_TYPE_PCIe_ENDPOINT:
1326 gdprintk(XENLOG_INFO VTDPREFIX,
1327 "domain_context_unmap:PCIe : bdf = %x:%x:%x\n",
1328 pdev->bus, PCI_SLOT(pdev->devfn), PCI_FUNC(pdev->devfn));
1329 ret = domain_context_unmap_one(domain, iommu,
1330 (u8)(pdev->bus), (u8)(pdev->devfn));
1331 break;
1332 case DEV_TYPE_PCI:
1333 gdprintk(XENLOG_INFO VTDPREFIX,
1334 "domain_context_unmap:PCI: bdf = %x:%x:%x\n",
1335 pdev->bus, PCI_SLOT(pdev->devfn), PCI_FUNC(pdev->devfn));
1336 if ( pdev->bus == 0 )
1337 ret = domain_context_unmap_one(
1338 domain, iommu,
1339 (u8)(pdev->bus), (u8)(pdev->devfn));
1340 else
1342 if ( bus2bridge[pdev->bus].bus != 0 )
1343 gdprintk(XENLOG_WARNING VTDPREFIX,
1344 "domain_context_unmap:"
1345 "bus2bridge[%d].bus != 0\n", pdev->bus);
1347 ret = domain_context_unmap_one(domain, iommu,
1348 (u8)(bus2bridge[pdev->bus].bus),
1349 (u8)(bus2bridge[pdev->bus].devfn));
1351 /* Unmap everything behind the PCI bridge */
1352 for ( dev = 0; dev < 32; dev++ )
1354 for ( func = 0; func < 8; func++ )
1356 ret = domain_context_unmap_one(
1357 domain, iommu,
1358 pdev->bus, (u8)PCI_DEVFN(dev, func));
1359 if ( ret )
1360 return ret;
1364 break;
1365 default:
1366 gdprintk(XENLOG_ERR VTDPREFIX,
1367 "domain_context_unmap:unknown type: bdf = %x:%x:%x\n",
1368 pdev->bus, PCI_SLOT(pdev->devfn), PCI_FUNC(pdev->devfn));
1369 ret = -EINVAL;
1370 break;
1373 return ret;
1376 void reassign_device_ownership(
1377 struct domain *source,
1378 struct domain *target,
1379 u8 bus, u8 devfn)
1381 struct hvm_iommu *source_hd = domain_hvm_iommu(source);
1382 struct hvm_iommu *target_hd = domain_hvm_iommu(target);
1383 struct pci_dev *pdev;
1384 struct acpi_drhd_unit *drhd;
1385 struct iommu *iommu;
1386 int status;
1387 unsigned long flags;
1389 gdprintk(XENLOG_INFO VTDPREFIX,
1390 "reassign_device-%x:%x:%x- source = %d target = %d\n",
1391 bus, PCI_SLOT(devfn), PCI_FUNC(devfn),
1392 source->domain_id, target->domain_id);
1394 for_each_pdev( source, pdev )
1396 if ( (pdev->bus != bus) || (pdev->devfn != devfn) )
1397 continue;
1399 drhd = acpi_find_matched_drhd_unit(pdev);
1400 iommu = drhd->iommu;
1401 domain_context_unmap(source, iommu, pdev);
1403 /* Move pci device from the source domain to target domain. */
1404 spin_lock_irqsave(&source_hd->iommu_list_lock, flags);
1405 spin_lock_irqsave(&target_hd->iommu_list_lock, flags);
1406 list_move(&pdev->list, &target_hd->pdev_list);
1407 spin_unlock_irqrestore(&target_hd->iommu_list_lock, flags);
1408 spin_unlock_irqrestore(&source_hd->iommu_list_lock, flags);
1410 status = domain_context_mapping(target, iommu, pdev);
1411 if ( status != 0 )
1412 gdprintk(XENLOG_ERR VTDPREFIX, "domain_context_mapping failed\n");
1414 break;
1418 void return_devices_to_dom0(struct domain *d)
1420 struct hvm_iommu *hd = domain_hvm_iommu(d);
1421 struct pci_dev *pdev;
1423 while ( !list_empty(&hd->pdev_list) )
1425 pdev = list_entry(hd->pdev_list.next, typeof(*pdev), list);
1426 dprintk(XENLOG_INFO VTDPREFIX,
1427 "return_devices_to_dom0: bdf = %x:%x:%x\n",
1428 pdev->bus, PCI_SLOT(pdev->devfn), PCI_FUNC(pdev->devfn));
1429 reassign_device_ownership(d, dom0, pdev->bus, pdev->devfn);
1432 #ifdef VTD_DEBUG
1433 for_each_pdev ( dom0, pdev )
1434 dprintk(XENLOG_INFO VTDPREFIX,
1435 "return_devices_to_dom0:%x: bdf = %x:%x:%x\n",
1436 dom0->domain_id, pdev->bus,
1437 PCI_SLOT(pdev->devfn), PCI_FUNC(pdev->devfn));
1438 #endif
1441 void iommu_domain_teardown(struct domain *d)
1443 if ( list_empty(&acpi_drhd_units) )
1444 return;
1446 iommu_domid_release(d);
1448 #if CONFIG_PAGING_LEVELS == 3
1450 struct hvm_iommu *hd = domain_hvm_iommu(d);
1451 int level = agaw_to_level(hd->agaw);
1452 struct dma_pte *pgd = NULL;
1454 switch ( level )
1456 case VTD_PAGE_TABLE_LEVEL_3:
1457 if ( hd->pgd )
1458 free_xenheap_page((void *)hd->pgd);
1459 break;
1460 case VTD_PAGE_TABLE_LEVEL_4:
1461 if ( hd->pgd )
1463 pgd = hd->pgd;
1464 if ( pgd[0].val != 0 )
1465 free_xenheap_page((void*)maddr_to_virt(
1466 dma_pte_addr(pgd[0])));
1468 free_xenheap_page((void *)hd->pgd);
1470 break;
1471 default:
1472 gdprintk(XENLOG_ERR VTDPREFIX,
1473 "Unsupported p2m table sharing level!\n");
1474 break;
1477 #endif
1478 return_devices_to_dom0(d);
1481 static int domain_context_mapped(struct pci_dev *pdev)
1483 struct acpi_drhd_unit *drhd;
1484 struct iommu *iommu;
1485 int ret;
1487 for_each_drhd_unit ( drhd )
1489 iommu = drhd->iommu;
1490 ret = device_context_mapped(iommu, pdev->bus, pdev->devfn);
1491 if ( ret )
1492 return ret;
1495 return 0;
1498 int iommu_map_page(struct domain *d, paddr_t gfn, paddr_t mfn)
1500 struct acpi_drhd_unit *drhd;
1501 struct iommu *iommu;
1502 struct dma_pte *pte = NULL;
1503 struct page_info *pg = NULL;
1505 drhd = list_entry(acpi_drhd_units.next, typeof(*drhd), list);
1506 iommu = drhd->iommu;
1508 /* do nothing if dom0 and iommu supports pass thru */
1509 if ( ecap_pass_thru(iommu->ecap) && (d->domain_id == 0) )
1510 return 0;
1512 pg = addr_to_dma_page(d, gfn << PAGE_SHIFT_4K);
1513 if ( !pg )
1514 return -ENOMEM;
1515 pte = (struct dma_pte *)map_domain_page(page_to_mfn(pg));
1516 pte += gfn & LEVEL_MASK;
1517 dma_set_pte_addr(*pte, mfn << PAGE_SHIFT_4K);
1518 dma_set_pte_prot(*pte, DMA_PTE_READ | DMA_PTE_WRITE);
1519 iommu_flush_cache_entry(iommu, pte);
1520 unmap_domain_page(pte);
1522 for_each_drhd_unit ( drhd )
1524 iommu = drhd->iommu;
1525 if ( cap_caching_mode(iommu->cap) )
1526 iommu_flush_iotlb_psi(iommu, domain_iommu_domid(d),
1527 gfn << PAGE_SHIFT_4K, 1, 0);
1528 else if ( cap_rwbf(iommu->cap) )
1529 iommu_flush_write_buffer(iommu);
1532 return 0;
1535 int iommu_unmap_page(struct domain *d, dma_addr_t gfn)
1537 struct acpi_drhd_unit *drhd;
1538 struct iommu *iommu;
1540 drhd = list_entry(acpi_drhd_units.next, typeof(*drhd), list);
1541 iommu = drhd->iommu;
1543 /* do nothing if dom0 and iommu supports pass thru */
1544 if ( ecap_pass_thru(iommu->ecap) && (d->domain_id == 0) )
1545 return 0;
1547 dma_pte_clear_one(d, gfn << PAGE_SHIFT_4K);
1549 return 0;
1552 int iommu_page_mapping(struct domain *domain, dma_addr_t iova,
1553 void *hpa, size_t size, int prot)
1555 struct acpi_drhd_unit *drhd;
1556 struct iommu *iommu;
1557 unsigned long start_pfn, end_pfn;
1558 struct dma_pte *pte = NULL;
1559 int index;
1560 struct page_info *pg = NULL;
1562 drhd = list_entry(acpi_drhd_units.next, typeof(*drhd), list);
1563 iommu = drhd->iommu;
1564 if ( (prot & (DMA_PTE_READ|DMA_PTE_WRITE)) == 0 )
1565 return -EINVAL;
1566 iova = (iova >> PAGE_SHIFT_4K) << PAGE_SHIFT_4K;
1567 start_pfn = (unsigned long)(((unsigned long) hpa) >> PAGE_SHIFT_4K);
1568 end_pfn = (unsigned long)
1569 ((PAGE_ALIGN_4K(((unsigned long)hpa) + size)) >> PAGE_SHIFT_4K);
1570 index = 0;
1571 while ( start_pfn < end_pfn )
1573 pg = addr_to_dma_page(domain, iova + PAGE_SIZE_4K * index);
1574 if ( !pg )
1575 return -ENOMEM;
1576 pte = (struct dma_pte *)map_domain_page(page_to_mfn(pg));
1577 pte += start_pfn & LEVEL_MASK;
1578 dma_set_pte_addr(*pte, start_pfn << PAGE_SHIFT_4K);
1579 dma_set_pte_prot(*pte, prot);
1580 iommu_flush_cache_entry(iommu, pte);
1581 unmap_domain_page(pte);
1582 start_pfn++;
1583 index++;
1586 for_each_drhd_unit ( drhd )
1588 iommu = drhd->iommu;
1589 if ( cap_caching_mode(iommu->cap) )
1590 iommu_flush_iotlb_psi(iommu, domain_iommu_domid(domain),
1591 iova, index, 0);
1592 else if ( cap_rwbf(iommu->cap) )
1593 iommu_flush_write_buffer(iommu);
1596 return 0;
1599 int iommu_page_unmapping(struct domain *domain, dma_addr_t addr, size_t size)
1601 dma_pte_clear_range(domain, addr, addr + size);
1603 return 0;
1606 void iommu_flush(struct domain *d, dma_addr_t gfn, u64 *p2m_entry)
1608 struct acpi_drhd_unit *drhd;
1609 struct iommu *iommu = NULL;
1610 struct dma_pte *pte = (struct dma_pte *) p2m_entry;
1612 for_each_drhd_unit ( drhd )
1614 iommu = drhd->iommu;
1615 if ( cap_caching_mode(iommu->cap) )
1616 iommu_flush_iotlb_psi(iommu, domain_iommu_domid(d),
1617 gfn << PAGE_SHIFT_4K, 1, 0);
1618 else if ( cap_rwbf(iommu->cap) )
1619 iommu_flush_write_buffer(iommu);
1622 iommu_flush_cache_entry(iommu, pte);
1625 static int iommu_prepare_rmrr_dev(
1626 struct domain *d,
1627 struct acpi_rmrr_unit *rmrr,
1628 struct pci_dev *pdev)
1630 struct acpi_drhd_unit *drhd;
1631 unsigned long size;
1632 int ret;
1634 /* page table init */
1635 size = rmrr->end_address - rmrr->base_address + 1;
1636 ret = iommu_page_mapping(d, rmrr->base_address,
1637 (void *)rmrr->base_address, size,
1638 DMA_PTE_READ|DMA_PTE_WRITE);
1639 if ( ret )
1640 return ret;
1642 if ( domain_context_mapped(pdev) == 0 )
1644 drhd = acpi_find_matched_drhd_unit(pdev);
1645 ret = domain_context_mapping(d, drhd->iommu, pdev);
1646 if ( !ret )
1647 return 0;
1650 return ret;
1653 void __init setup_dom0_devices(void)
1655 struct hvm_iommu *hd = domain_hvm_iommu(dom0);
1656 struct acpi_drhd_unit *drhd;
1657 struct pci_dev *pdev;
1658 int bus, dev, func, ret;
1659 u32 l;
1661 #ifdef DEBUG_VTD_CONTEXT_ENTRY
1662 for ( bus = 0; bus < 256; bus++ )
1664 for ( dev = 0; dev < 32; dev++ )
1666 for ( func = 0; func < 8; func++ )
1668 struct context_entry *context;
1669 struct pci_dev device;
1671 device.bus = bus;
1672 device.devfn = PCI_DEVFN(dev, func);
1673 drhd = acpi_find_matched_drhd_unit(&device);
1674 context = device_to_context_entry(drhd->iommu,
1675 bus, PCI_DEVFN(dev, func));
1676 if ( (context->lo != 0) || (context->hi != 0) )
1677 dprintk(XENLOG_INFO VTDPREFIX,
1678 "setup_dom0_devices-%x:%x:%x- context not 0\n",
1679 bus, dev, func);
1683 #endif
1685 for ( bus = 0; bus < 256; bus++ )
1687 for ( dev = 0; dev < 32; dev++ )
1689 for ( func = 0; func < 8; func++ )
1691 l = read_pci_config(bus, dev, func, PCI_VENDOR_ID);
1692 /* some broken boards return 0 or ~0 if a slot is empty: */
1693 if ( (l == 0xffffffff) || (l == 0x00000000) ||
1694 (l == 0x0000ffff) || (l == 0xffff0000) )
1695 continue;
1696 pdev = xmalloc(struct pci_dev);
1697 pdev->bus = bus;
1698 pdev->devfn = PCI_DEVFN(dev, func);
1699 list_add_tail(&pdev->list, &hd->pdev_list);
1701 drhd = acpi_find_matched_drhd_unit(pdev);
1702 ret = domain_context_mapping(dom0, drhd->iommu, pdev);
1703 if ( ret != 0 )
1704 gdprintk(XENLOG_ERR VTDPREFIX,
1705 "domain_context_mapping failed\n");
1710 for_each_pdev ( dom0, pdev )
1711 dprintk(XENLOG_INFO VTDPREFIX,
1712 "setup_dom0_devices: bdf = %x:%x:%x\n",
1713 pdev->bus, PCI_SLOT(pdev->devfn), PCI_FUNC(pdev->devfn));
1716 void clear_fault_bit(struct iommu *iommu)
1718 u64 val;
1720 val = dmar_readq(
1721 iommu->reg,
1722 cap_fault_reg_offset(dmar_readq(iommu->reg,DMAR_CAP_REG))+0x8);
1723 dmar_writeq(
1724 iommu->reg,
1725 cap_fault_reg_offset(dmar_readq(iommu->reg,DMAR_CAP_REG))+8,
1726 val);
1727 dmar_writel(iommu->reg, DMAR_FSTS_REG, DMA_FSTS_PFO);
1730 static int init_vtd_hw(void)
1732 struct acpi_drhd_unit *drhd;
1733 struct iommu *iommu;
1734 int ret;
1736 for_each_drhd_unit ( drhd )
1738 iommu = drhd->iommu;
1739 ret = iommu_set_root_entry(iommu);
1740 if ( ret )
1742 gdprintk(XENLOG_ERR VTDPREFIX, "IOMMU: set root entry failed\n");
1743 return -EIO;
1747 return 0;
1750 static int enable_vtd_translation(void)
1752 struct acpi_drhd_unit *drhd;
1753 struct iommu *iommu;
1754 int vector = 0;
1756 for_each_drhd_unit ( drhd )
1758 iommu = drhd->iommu;
1759 vector = iommu_set_interrupt(iommu);
1760 dma_msi_data_init(iommu, vector);
1761 dma_msi_addr_init(iommu, cpu_physical_id(first_cpu(cpu_online_map)));
1762 iommu->vector = vector;
1763 clear_fault_bit(iommu);
1764 if ( iommu_enable_translation(iommu) )
1765 return -EIO;
1768 return 0;
1771 static void setup_dom0_rmrr(void)
1773 struct acpi_rmrr_unit *rmrr;
1774 struct pci_dev *pdev;
1775 int ret;
1777 for_each_rmrr_device ( rmrr, pdev )
1778 ret = iommu_prepare_rmrr_dev(dom0, rmrr, pdev);
1779 if ( ret )
1780 gdprintk(XENLOG_ERR VTDPREFIX,
1781 "IOMMU: mapping reserved region failed\n");
1782 end_for_each_rmrr_device ( rmrr, pdev )
1785 int iommu_setup(void)
1787 struct hvm_iommu *hd = domain_hvm_iommu(dom0);
1788 struct acpi_drhd_unit *drhd;
1789 struct iommu *iommu;
1790 unsigned long i;
1792 if ( !vtd_enabled )
1793 return 0;
1795 spin_lock_init(&domid_bitmap_lock);
1796 INIT_LIST_HEAD(&hd->pdev_list);
1798 /* start from scratch */
1799 iommu_flush_all();
1801 /* setup clflush size */
1802 x86_clflush_size = ((cpuid_ebx(1) >> 8) & 0xff) * 8;
1804 /* Allocate IO page directory page for the domain. */
1805 drhd = list_entry(acpi_drhd_units.next, typeof(*drhd), list);
1806 iommu = drhd->iommu;
1808 /* Allocate domain id bitmap, and set bit 0 as reserved */
1809 domid_bitmap_size = cap_ndoms(iommu->cap);
1810 domid_bitmap = xmalloc_bytes(domid_bitmap_size / 8);
1811 if ( domid_bitmap == NULL )
1812 goto error;
1813 memset(domid_bitmap, 0, domid_bitmap_size / 8);
1814 set_bit(0, domid_bitmap);
1816 /* setup 1:1 page table for dom0 */
1817 for ( i = 0; i < max_page; i++ )
1818 iommu_map_page(dom0, i, i);
1820 if ( init_vtd_hw() )
1821 goto error;
1822 setup_dom0_devices();
1823 setup_dom0_rmrr();
1824 if ( enable_vtd_translation() )
1825 goto error;
1827 return 0;
1829 error:
1830 printk("iommu_setup() failed\n");
1831 for_each_drhd_unit ( drhd )
1833 iommu = drhd->iommu;
1834 free_iommu(iommu);
1836 return -EIO;
1839 /*
1840 * If the device isn't owned by dom0, it means it already
1841 * has been assigned to other domain, or it's not exist.
1842 */
1843 int device_assigned(u8 bus, u8 devfn)
1845 struct pci_dev *pdev;
1847 for_each_pdev( dom0, pdev )
1848 if ( (pdev->bus == bus ) && (pdev->devfn == devfn) )
1849 return 0;
1851 return 1;
1854 int assign_device(struct domain *d, u8 bus, u8 devfn)
1856 struct acpi_rmrr_unit *rmrr;
1857 struct pci_dev *pdev;
1858 int ret = 0;
1860 if ( list_empty(&acpi_drhd_units) )
1861 return ret;
1863 gdprintk(XENLOG_INFO VTDPREFIX,
1864 "assign_device: bus = %x dev = %x func = %x\n",
1865 bus, PCI_SLOT(devfn), PCI_FUNC(devfn));
1867 reassign_device_ownership(dom0, d, bus, devfn);
1869 /* Setup rmrr identify mapping */
1870 for_each_rmrr_device( rmrr, pdev )
1871 if ( pdev->bus == bus && pdev->devfn == devfn )
1873 ret = iommu_prepare_rmrr_dev(d, rmrr, pdev);
1874 if ( ret )
1876 gdprintk(XENLOG_ERR VTDPREFIX,
1877 "IOMMU: mapping reserved region failed\n");
1878 return ret;
1881 end_for_each_rmrr_device(rmrr, pdev)
1883 return ret;
1886 void iommu_set_pgd(struct domain *d)
1888 struct hvm_iommu *hd = domain_hvm_iommu(d);
1889 unsigned long p2m_table;
1891 if ( hd->pgd )
1893 gdprintk(XENLOG_INFO VTDPREFIX,
1894 "iommu_set_pgd_1: hd->pgd = %p\n", hd->pgd);
1895 hd->pgd = NULL;
1897 p2m_table = mfn_x(pagetable_get_mfn(d->arch.phys_table));
1899 #if CONFIG_PAGING_LEVELS == 3
1900 if ( !hd->pgd )
1902 int level = agaw_to_level(hd->agaw);
1903 struct dma_pte *pmd = NULL;
1904 struct dma_pte *pgd = NULL;
1905 struct dma_pte *pte = NULL;
1906 l3_pgentry_t *l3e;
1907 unsigned long flags;
1908 int i;
1910 spin_lock_irqsave(&hd->mapping_lock, flags);
1911 if ( !hd->pgd )
1913 pgd = (struct dma_pte *)alloc_xenheap_page();
1914 if ( !pgd )
1916 spin_unlock_irqrestore(&hd->mapping_lock, flags);
1917 gdprintk(XENLOG_ERR VTDPREFIX,
1918 "Allocate pgd memory failed!\n");
1919 return;
1921 memset(pgd, 0, PAGE_SIZE);
1922 hd->pgd = pgd;
1925 l3e = map_domain_page(p2m_table);
1926 switch ( level )
1928 case VTD_PAGE_TABLE_LEVEL_3: /* Weybridge */
1929 /* We only support 8 entries for the PAE L3 p2m table */
1930 for ( i = 0; i < 8 ; i++ )
1932 /* Don't create new L2 entry, use ones from p2m table */
1933 pgd[i].val = l3e[i].l3 | _PAGE_PRESENT | _PAGE_RW;
1935 break;
1937 case VTD_PAGE_TABLE_LEVEL_4: /* Stoakley */
1938 /* We allocate one more page for the top vtd page table. */
1939 pmd = (struct dma_pte *)alloc_xenheap_page();
1940 if ( !pmd )
1942 unmap_domain_page(l3e);
1943 spin_unlock_irqrestore(&hd->mapping_lock, flags);
1944 gdprintk(XENLOG_ERR VTDPREFIX,
1945 "Allocate pmd memory failed!\n");
1946 return;
1948 memset((u8*)pmd, 0, PAGE_SIZE);
1949 pte = &pgd[0];
1950 dma_set_pte_addr(*pte, virt_to_maddr(pmd));
1951 dma_set_pte_readable(*pte);
1952 dma_set_pte_writable(*pte);
1954 for ( i = 0; i < 8; i++ )
1956 /* Don't create new L2 entry, use ones from p2m table */
1957 pmd[i].val = l3e[i].l3 | _PAGE_PRESENT | _PAGE_RW;
1959 break;
1960 default:
1961 gdprintk(XENLOG_ERR VTDPREFIX,
1962 "iommu_set_pgd:Unsupported p2m table sharing level!\n");
1963 break;
1965 unmap_domain_page(l3e);
1966 spin_unlock_irqrestore(&hd->mapping_lock, flags);
1968 #elif CONFIG_PAGING_LEVELS == 4
1969 if ( !hd->pgd )
1971 int level = agaw_to_level(hd->agaw);
1972 l3_pgentry_t *l3e;
1973 mfn_t pgd_mfn;
1975 switch ( level )
1977 case VTD_PAGE_TABLE_LEVEL_3:
1978 l3e = map_domain_page(p2m_table);
1979 if ( (l3e_get_flags(*l3e) & _PAGE_PRESENT) == 0 )
1981 gdprintk(XENLOG_ERR VTDPREFIX,
1982 "iommu_set_pgd: second level wasn't there\n");
1983 unmap_domain_page(l3e);
1984 return;
1986 pgd_mfn = _mfn(l3e_get_pfn(*l3e));
1987 unmap_domain_page(l3e);
1988 hd->pgd = maddr_to_virt(pagetable_get_paddr(
1989 pagetable_from_mfn(pgd_mfn)));
1990 break;
1992 case VTD_PAGE_TABLE_LEVEL_4:
1993 pgd_mfn = _mfn(p2m_table);
1994 hd->pgd = maddr_to_virt(pagetable_get_paddr(
1995 pagetable_from_mfn(pgd_mfn)));
1996 break;
1997 default:
1998 gdprintk(XENLOG_ERR VTDPREFIX,
1999 "iommu_set_pgd:Unsupported p2m table sharing level!\n");
2000 break;
2003 #endif
2004 gdprintk(XENLOG_INFO VTDPREFIX,
2005 "iommu_set_pgd: hd->pgd = %p\n", hd->pgd);
2009 u8 iommu_state[MAX_IOMMU_REGS * MAX_IOMMUS];
2010 int iommu_suspend(void)
2012 struct acpi_drhd_unit *drhd;
2013 struct iommu *iommu;
2014 int i = 0;
2016 iommu_flush_all();
2018 for_each_drhd_unit ( drhd )
2020 iommu = drhd->iommu;
2021 iommu_state[DMAR_RTADDR_REG * i] =
2022 (u64) dmar_readq(iommu->reg, DMAR_RTADDR_REG);
2023 iommu_state[DMAR_FECTL_REG * i] =
2024 (u32) dmar_readl(iommu->reg, DMAR_FECTL_REG);
2025 iommu_state[DMAR_FEDATA_REG * i] =
2026 (u32) dmar_readl(iommu->reg, DMAR_FEDATA_REG);
2027 iommu_state[DMAR_FEADDR_REG * i] =
2028 (u32) dmar_readl(iommu->reg, DMAR_FEADDR_REG);
2029 iommu_state[DMAR_FEUADDR_REG * i] =
2030 (u32) dmar_readl(iommu->reg, DMAR_FEUADDR_REG);
2031 iommu_state[DMAR_PLMBASE_REG * i] =
2032 (u32) dmar_readl(iommu->reg, DMAR_PLMBASE_REG);
2033 iommu_state[DMAR_PLMLIMIT_REG * i] =
2034 (u32) dmar_readl(iommu->reg, DMAR_PLMLIMIT_REG);
2035 iommu_state[DMAR_PHMBASE_REG * i] =
2036 (u64) dmar_readq(iommu->reg, DMAR_PHMBASE_REG);
2037 iommu_state[DMAR_PHMLIMIT_REG * i] =
2038 (u64) dmar_readq(iommu->reg, DMAR_PHMLIMIT_REG);
2039 i++;
2042 return 0;
2045 int iommu_resume(void)
2047 struct acpi_drhd_unit *drhd;
2048 struct iommu *iommu;
2049 int i = 0;
2051 iommu_flush_all();
2053 init_vtd_hw();
2054 for_each_drhd_unit ( drhd )
2056 iommu = drhd->iommu;
2057 dmar_writeq( iommu->reg, DMAR_RTADDR_REG,
2058 (u64) iommu_state[DMAR_RTADDR_REG * i]);
2059 dmar_writel(iommu->reg, DMAR_FECTL_REG,
2060 (u32) iommu_state[DMAR_FECTL_REG * i]);
2061 dmar_writel(iommu->reg, DMAR_FEDATA_REG,
2062 (u32) iommu_state[DMAR_FEDATA_REG * i]);
2063 dmar_writel(iommu->reg, DMAR_FEADDR_REG,
2064 (u32) iommu_state[DMAR_FEADDR_REG * i]);
2065 dmar_writel(iommu->reg, DMAR_FEUADDR_REG,
2066 (u32) iommu_state[DMAR_FEUADDR_REG * i]);
2067 dmar_writel(iommu->reg, DMAR_PLMBASE_REG,
2068 (u32) iommu_state[DMAR_PLMBASE_REG * i]);
2069 dmar_writel(iommu->reg, DMAR_PLMLIMIT_REG,
2070 (u32) iommu_state[DMAR_PLMLIMIT_REG * i]);
2071 dmar_writeq(iommu->reg, DMAR_PHMBASE_REG,
2072 (u64) iommu_state[DMAR_PHMBASE_REG * i]);
2073 dmar_writeq(iommu->reg, DMAR_PHMLIMIT_REG,
2074 (u64) iommu_state[DMAR_PHMLIMIT_REG * i]);
2076 if ( iommu_enable_translation(iommu) )
2077 return -EIO;
2078 i++;
2080 return 0;
2083 /*
2084 * Local variables:
2085 * mode: C
2086 * c-set-style: "BSD"
2087 * c-basic-offset: 4
2088 * tab-width: 4
2089 * indent-tabs-mode: nil
2090 * End:
2091 */