1.1 --- a/xen/arch/x86/domain.c	Wed Jul 09 13:01:16 2008 +0100
     1.2 +++ b/xen/arch/x86/domain.c	Wed Jul 09 14:08:58 2008 +0100
     1.3 @@ -30,6 +30,7 @@
     1.4  #include <xen/percpu.h>
     1.5  #include <xen/compat.h>
     1.6  #include <xen/acpi.h>
     1.7 +#include <xen/pci.h>
     1.8  #include <asm/regs.h>
     1.9  #include <asm/mc146818rtc.h>
    1.10  #include <asm/system.h>
    1.11 @@ -473,6 +474,7 @@ void arch_domain_destroy(struct domain *
    1.12      if ( is_hvm_domain(d) )
    1.13          hvm_domain_destroy(d);
    1.14  
    1.15 +    pci_release_devices(d);
    1.16      if ( !is_idle_domain(d) )
    1.17          iommu_domain_destroy(d);
    1.18  

     2.1 --- a/xen/drivers/passthrough/amd/pci_amd_iommu.c	Wed Jul 09 13:01:16 2008 +0100
     2.2 +++ b/xen/drivers/passthrough/amd/pci_amd_iommu.c	Wed Jul 09 14:08:58 2008 +0100
     2.3 @@ -485,8 +485,6 @@ static void amd_iommu_disable_domain_dev
     2.4      }
     2.5  }
     2.6  
     2.7 -extern void pdev_flr(u8 bus, u8 devfn);
     2.8 -
     2.9  static int reassign_device( struct domain *source, struct domain *target,
    2.10                              u8 bus, u8 devfn)
    2.11  {
    2.12 @@ -498,6 +496,7 @@ static int reassign_device( struct domai
    2.13      if ( !pdev )
    2.14  	return -ENODEV;
    2.15  
    2.16 +    pdev_flr(pdev);
    2.17      bdf = (bus << 8) | devfn;
    2.18      /* supported device? */
    2.19      iommu = (bdf < ivrs_bdf_entries) ?
    2.20 @@ -545,26 +544,9 @@ static int amd_iommu_assign_device(struc
    2.21              ivrs_mappings[req_id].read_permission);
    2.22      }
    2.23  
    2.24 -    pdev_flr(bus, devfn);
    2.25      return reassign_device(dom0, d, bus, devfn);
    2.26  }
    2.27  
    2.28 -static void release_domain_devices(struct domain *d)
    2.29 -{
    2.30 -    struct pci_dev *pdev;
    2.31 -    u8 bus, devfn;
    2.32 -
    2.33 -    while ( (pdev = pci_lock_domain_pdev(d, -1, -1)) )
    2.34 -    {
    2.35 -        pdev_flr(pdev->bus, pdev->devfn);
    2.36 -	bus = pdev->bus; devfn = pdev->devfn;
    2.37 -	spin_unlock(&pdev->lock);
    2.38 -        amd_iov_info("release domain %d devices %x:%x.%x\n", d->domain_id,
    2.39 -		     bus, PCI_SLOT(devfn), PCI_FUNC(devfn));
    2.40 -        reassign_device(d, dom0, bus, devfn);
    2.41 -    }
    2.42 -}
    2.43 -
    2.44  static void deallocate_next_page_table(void *table, unsigned long index,
    2.45                                         int level)
    2.46  {
    2.47 @@ -618,13 +600,11 @@ static void deallocate_iommu_page_tables
    2.48  static void amd_iommu_domain_destroy(struct domain *d)
    2.49  {
    2.50      deallocate_iommu_page_tables(d);
    2.51 -    release_domain_devices(d);
    2.52  }
    2.53  
    2.54  static int amd_iommu_return_device(
    2.55      struct domain *s, struct domain *t, u8 bus, u8 devfn)
    2.56  {
    2.57 -    pdev_flr(bus, devfn);
    2.58      return reassign_device(s, t, bus, devfn);
    2.59  }
    2.60  

     3.1 --- a/xen/drivers/passthrough/pci.c	Wed Jul 09 13:01:16 2008 +0100
     3.2 +++ b/xen/drivers/passthrough/pci.c	Wed Jul 09 14:08:58 2008 +0100
     3.3 @@ -17,9 +17,11 @@
     3.4  
     3.5  #include <xen/sched.h>
     3.6  #include <xen/pci.h>
     3.7 +#include <xen/pci_regs.h>
     3.8  #include <xen/list.h>
     3.9  #include <xen/prefetch.h>
    3.10  #include <xen/iommu.h>
    3.11 +#include <xen/delay.h>
    3.12  #include <xen/keyhandler.h>
    3.13  
    3.14  
    3.15 @@ -145,6 +147,85 @@ int pci_remove_device(u8 bus, u8 devfn)
    3.16      return ret;
    3.17  }
    3.18  
    3.19 +void pci_release_devices(struct domain *d)
    3.20 +{
    3.21 +    struct pci_dev *pdev;
    3.22 +    u8 bus, devfn;
    3.23 +
    3.24 +    while ( (pdev = pci_lock_domain_pdev(d, -1, -1)) )
    3.25 +    {
    3.26 +        pci_cleanup_msi(pdev);
    3.27 +        bus = pdev->bus; devfn = pdev->devfn;
    3.28 +        spin_unlock(&pdev->lock);
    3.29 +        deassign_device(d, bus, devfn);
    3.30 +    }
    3.31 +}
    3.32 +
    3.33 +#define PCI_D3hot	        (3)
    3.34 +#define PCI_CONFIG_DWORD_SIZE   (64)
    3.35 +#define PCI_EXP_DEVCAP_FLR      (1 << 28)
    3.36 +#define PCI_EXP_DEVCTL_FLR      (1 << 15)
    3.37 +
    3.38 +void pdev_flr(struct pci_dev *pdev)
    3.39 +{
    3.40 +    u8 pos;
    3.41 +    u32 dev_cap, dev_status, pm_ctl;
    3.42 +    int flr = 0;
    3.43 +    u8 dev = PCI_SLOT(pdev->devfn);
    3.44 +    u8 func = PCI_FUNC(pdev->devfn);
    3.45 +
    3.46 +    pos = pci_find_cap_offset(pdev->bus, dev, func, PCI_CAP_ID_EXP);
    3.47 +    if ( pos != 0 )
    3.48 +    {
    3.49 +        dev_cap = pci_conf_read32(pdev->bus, dev, func, pos + PCI_EXP_DEVCAP);
    3.50 +        if ( dev_cap & PCI_EXP_DEVCAP_FLR )
    3.51 +        {
    3.52 +            pci_conf_write32(pdev->bus, dev, func,
    3.53 +                             pos + PCI_EXP_DEVCTL, PCI_EXP_DEVCTL_FLR);
    3.54 +            do {
    3.55 +                dev_status = pci_conf_read32(pdev->bus, dev, func,
    3.56 +                                             pos + PCI_EXP_DEVSTA);
    3.57 +            } while ( dev_status & PCI_EXP_DEVSTA_TRPND );
    3.58 +
    3.59 +            flr = 1;
    3.60 +        }
    3.61 +    }
    3.62 +
    3.63 +    /* If this device doesn't support function level reset,
    3.64 +     * program device from D0 t0 D3hot, and then return to D0
    3.65 +     * to implement function level reset
    3.66 +     */
    3.67 +    if ( flr == 0 )
    3.68 +    {
    3.69 +        pos = pci_find_cap_offset(pdev->bus, dev, func, PCI_CAP_ID_PM);
    3.70 +        if ( pos != 0 )
    3.71 +        {
    3.72 +            int i;
    3.73 +            u32 config[PCI_CONFIG_DWORD_SIZE];
    3.74 +            for ( i = 0; i < PCI_CONFIG_DWORD_SIZE; i++ )
    3.75 +                config[i] = pci_conf_read32(pdev->bus, dev, func, i*4);
    3.76 +
    3.77 +            /* Enter D3hot without soft reset */
    3.78 +            pm_ctl = pci_conf_read32(pdev->bus, dev, func, pos + PCI_PM_CTRL);
    3.79 +            pm_ctl |= PCI_PM_CTRL_NO_SOFT_RESET;
    3.80 +            pm_ctl &= ~PCI_PM_CTRL_STATE_MASK;
    3.81 +            pm_ctl |= PCI_D3hot;
    3.82 +            pci_conf_write32(pdev->bus, dev, func, pos + PCI_PM_CTRL, pm_ctl);
    3.83 +            mdelay(10);
    3.84 +
    3.85 +            /* From D3hot to D0 */
    3.86 +            pci_conf_write32(pdev->bus, dev, func, pos + PCI_PM_CTRL, 0);
    3.87 +            mdelay(10);
    3.88 +
    3.89 +            /* Write saved configurations to device */
    3.90 +            for ( i = 0; i < PCI_CONFIG_DWORD_SIZE; i++ )
    3.91 +                pci_conf_write32(pdev->bus, dev, func, i*4, config[i]);
    3.92 +
    3.93 +            flr = 1;
    3.94 +        }
    3.95 +    }
    3.96 +}
    3.97 +
    3.98  static void dump_pci_devices(unsigned char ch)
    3.99  {
   3.100      struct pci_dev *pdev;
   3.101 @@ -174,3 +255,14 @@ static int __init setup_dump_pcidevs(voi
   3.102      return 0;
   3.103  }
   3.104  __initcall(setup_dump_pcidevs);
   3.105 +
   3.106 +
   3.107 +
   3.108 +/*
   3.109 + * Local variables:
   3.110 + * mode: C
   3.111 + * c-set-style: "BSD"
   3.112 + * c-basic-offset: 4
   3.113 + * indent-tabs-mode: nil
   3.114 + * End:
   3.115 + */

     4.1 --- a/xen/drivers/passthrough/vtd/extern.h	Wed Jul 09 13:01:16 2008 +0100
     4.2 +++ b/xen/drivers/passthrough/vtd/extern.h	Wed Jul 09 14:08:58 2008 +0100
     4.3 @@ -28,7 +28,6 @@ extern struct ir_ctrl *ir_ctrl;
     4.4  
     4.5  void print_iommu_regs(struct acpi_drhd_unit *drhd);
     4.6  void print_vtd_entries(struct iommu *iommu, int bus, int devfn, u64 gmfn);
     4.7 -void pdev_flr(u8 bus, u8 devfn);
     4.8  
     4.9  int qinval_setup(struct iommu *iommu);
    4.10  int intremap_setup(struct iommu *iommu);

     5.1 --- a/xen/drivers/passthrough/vtd/iommu.c	Wed Jul 09 13:01:16 2008 +0100
     5.2 +++ b/xen/drivers/passthrough/vtd/iommu.c	Wed Jul 09 14:08:58 2008 +0100
     5.3 @@ -1366,7 +1366,7 @@ static int reassign_device_ownership(
     5.4      if ( !(pdev = pci_lock_domain_pdev(source, bus, devfn)) )
     5.5          return -ENODEV;
     5.6  
     5.7 -    pdev_flr(bus, devfn);
     5.8 +    pdev_flr(pdev);
     5.9      drhd = acpi_find_matched_drhd_unit(bus, devfn);
    5.10      pdev_iommu = drhd->iommu;
    5.11      domain_context_unmap(bus, devfn);
    5.12 @@ -1397,28 +1397,6 @@ static int reassign_device_ownership(
    5.13      return ret;
    5.14  }
    5.15  
    5.16 -void return_devices_to_dom0(struct domain *d)
    5.17 -{
    5.18 -    struct pci_dev *pdev;
    5.19 -
    5.20 -    while ( (pdev = pci_lock_domain_pdev(d, -1, -1)) )
    5.21 -    {
    5.22 -        pci_cleanup_msi(pdev);
    5.23 -        spin_unlock(&pdev->lock);
    5.24 -        reassign_device_ownership(d, dom0, pdev->bus, pdev->devfn);
    5.25 -    }
    5.26 -
    5.27 -#ifdef VTD_DEBUG
    5.28 -    read_lock(&pcidevs_lock);
    5.29 -    for_each_pdev ( dom0, pdev )
    5.30 -        dprintk(XENLOG_INFO VTDPREFIX,
    5.31 -                "return_devices_to_dom0:%x: bdf = %x:%x:%x\n",
    5.32 -                dom0->domain_id, pdev->bus,
    5.33 -                PCI_SLOT(pdev->devfn), PCI_FUNC(pdev->devfn));
    5.34 -    read_unlock(&pcidevs_lock);
    5.35 -#endif
    5.36 -}
    5.37 -
    5.38  void iommu_domain_teardown(struct domain *d)
    5.39  {
    5.40      struct hvm_iommu *hd = domain_hvm_iommu(d);
    5.41 @@ -1426,7 +1404,6 @@ void iommu_domain_teardown(struct domain
    5.42      if ( list_empty(&acpi_drhd_units) )
    5.43          return;
    5.44  
    5.45 -    return_devices_to_dom0(d);
    5.46      iommu_free_pagetable(hd->pgd_maddr, agaw_to_level(hd->agaw));
    5.47      hd->pgd_maddr = 0;
    5.48      iommu_domid_release(d);

     6.1 --- a/xen/drivers/passthrough/vtd/utils.c	Wed Jul 09 13:01:16 2008 +0100
     6.2 +++ b/xen/drivers/passthrough/vtd/utils.c	Wed Jul 09 14:08:58 2008 +0100
     6.3 @@ -94,71 +94,6 @@ void disable_pmr(struct iommu *iommu)
     6.4              "Disabled protected memory registers\n");
     6.5  }
     6.6  
     6.7 -#define PCI_D3hot   (3)
     6.8 -#define PCI_CONFIG_DWORD_SIZE   (64)
     6.9 -#define PCI_EXP_DEVCAP_FLR      (1 << 28)
    6.10 -#define PCI_EXP_DEVCTL_FLR      (1 << 15)
    6.11 -
    6.12 -void pdev_flr(u8 bus, u8 devfn)
    6.13 -{
    6.14 -    u8 pos;
    6.15 -    u32 dev_cap, dev_status, pm_ctl;
    6.16 -    int flr = 0;
    6.17 -    u8 dev = PCI_SLOT(devfn);
    6.18 -    u8 func = PCI_FUNC(devfn);
    6.19 -
    6.20 -    pos = pci_find_cap_offset(bus, dev, func, PCI_CAP_ID_EXP);
    6.21 -    if ( pos != 0 )
    6.22 -    {
    6.23 -        dev_cap = pci_conf_read32(bus, dev, func, pos + PCI_EXP_DEVCAP);
    6.24 -        if ( dev_cap & PCI_EXP_DEVCAP_FLR )
    6.25 -        {
    6.26 -            pci_conf_write32(bus, dev, func,
    6.27 -                             pos + PCI_EXP_DEVCTL, PCI_EXP_DEVCTL_FLR);
    6.28 -            do {
    6.29 -                dev_status = pci_conf_read32(bus, dev, func,
    6.30 -                                             pos + PCI_EXP_DEVSTA);
    6.31 -            } while ( dev_status & PCI_EXP_DEVSTA_TRPND );
    6.32 -
    6.33 -            flr = 1;
    6.34 -        }
    6.35 -    }
    6.36 -
    6.37 -    /* If this device doesn't support function level reset,
    6.38 -     * program device from D0 t0 D3hot, and then return to D0
    6.39 -     * to implement function level reset
    6.40 -     */
    6.41 -    if ( flr == 0 )
    6.42 -    {
    6.43 -        pos = pci_find_cap_offset(bus, dev, func, PCI_CAP_ID_PM);
    6.44 -        if ( pos != 0 )
    6.45 -        {
    6.46 -            int i;
    6.47 -            u32 config[PCI_CONFIG_DWORD_SIZE];
    6.48 -            for ( i = 0; i < PCI_CONFIG_DWORD_SIZE; i++ )
    6.49 -                config[i] = pci_conf_read32(bus, dev, func, i*4);
    6.50 -
    6.51 -            /* Enter D3hot without soft reset */
    6.52 -            pm_ctl = pci_conf_read32(bus, dev, func, pos + PCI_PM_CTRL);
    6.53 -            pm_ctl |= PCI_PM_CTRL_NO_SOFT_RESET;
    6.54 -            pm_ctl &= ~PCI_PM_CTRL_STATE_MASK;
    6.55 -            pm_ctl |= PCI_D3hot;
    6.56 -            pci_conf_write32(bus, dev, func, pos + PCI_PM_CTRL, pm_ctl);
    6.57 -            mdelay(10);
    6.58 -
    6.59 -            /* From D3hot to D0 */
    6.60 -            pci_conf_write32(bus, dev, func, pos + PCI_PM_CTRL, 0);
    6.61 -            mdelay(10);
    6.62 -
    6.63 -            /* Write saved configurations to device */
    6.64 -            for ( i = 0; i < PCI_CONFIG_DWORD_SIZE; i++ )
    6.65 -                pci_conf_write32(bus, dev, func, i*4, config[i]);
    6.66 -
    6.67 -            flr = 1;
    6.68 -        }
    6.69 -    }
    6.70 -}
    6.71 -
    6.72  void print_iommu_regs(struct acpi_drhd_unit *drhd)
    6.73  {
    6.74      struct iommu *iommu = drhd->iommu;

     7.1 --- a/xen/include/xen/pci.h	Wed Jul 09 13:01:16 2008 +0100
     7.2 +++ b/xen/include/xen/pci.h	Wed Jul 09 14:08:58 2008 +0100
     7.3 @@ -56,6 +56,8 @@ void free_pdev(struct pci_dev *pdev);
     7.4  struct pci_dev *pci_lock_pdev(int bus, int devfn);
     7.5  struct pci_dev *pci_lock_domain_pdev(struct domain *d, int bus, int devfn);
     7.6  
     7.7 +void pdev_flr(struct pci_dev *pdev);
     7.8 +void pci_release_devices(struct domain *d);
     7.9  int pci_add_device(u8 bus, u8 devfn);
    7.10  int pci_remove_device(u8 bus, u8 devfn);
    7.11