debuggers.hg

changeset 21997:1644b4efef8a

xl: pci multi-function passthrough v2

Implement PCI pass-through for multi-function devices. The supported BDF
notation is: BB:DD.* - therefore passing-through a subset of functions or
remapping the function numbers is not supported except for when passing
through a single function which will be a virtual function 0.

Letting qemu automatically select the virtual slot is not supported in
multi-function passthrough since the qemu-dm protocol can not actually
handle that case.

Also fix format error in xl pci-list-assignable.

Signed-off-by: Gianni Tedesco <gianni.tedesco@citrix.com>
Signed-off-by: Stefano Stabellini <stefano.stabellini@eu.citrix.com>
author Stefano Stabellini <sstabellini@xensource.com>
date Mon Aug 09 17:46:39 2010 +0100 (2010-08-09)
parents 36c9f6ea9782
children f45026ec8db5 6e621e039cde
files tools/libxl/libxl.h tools/libxl/libxl_pci.c tools/libxl/xl_cmdimpl.c
line diff
     1.1 --- a/tools/libxl/libxl.h	Mon Aug 09 17:44:01 2010 +0100
     1.2 +++ b/tools/libxl/libxl.h	Mon Aug 09 17:46:39 2010 +0100
     1.3 @@ -310,6 +310,8 @@ typedef struct {
     1.4      };
     1.5      unsigned int domain;
     1.6      unsigned int vdevfn;
     1.7 +#define LIBXL_PCI_FUNC_ALL (~0U)
     1.8 +    unsigned int vfunc_mask;
     1.9      bool msitranslate;
    1.10      bool power_mgmt;
    1.11  } libxl_device_pci;
     2.1 --- a/tools/libxl/libxl_pci.c	Mon Aug 09 17:44:01 2010 +0100
     2.2 +++ b/tools/libxl/libxl_pci.c	Mon Aug 09 17:46:39 2010 +0100
     2.3 @@ -136,8 +136,13 @@ int libxl_device_pci_parse_bdf(libxl_ctx
     2.4                      break;
     2.5                  }
     2.6                  *ptr = '\0';
     2.7 -                if ( hex_convert(tok, &func, 0x7) )
     2.8 -                    goto parse_error;
     2.9 +                if ( !strcmp(tok, "*") ) {
    2.10 +                    pcidev->vfunc_mask = LIBXL_PCI_FUNC_ALL;
    2.11 +                }else{
    2.12 +                    if ( hex_convert(tok, &func, 0x7) )
    2.13 +                        goto parse_error;
    2.14 +                    pcidev->vfunc_mask = (1 << 0);
    2.15 +                }
    2.16                  tok = ptr + 1;
    2.17              }
    2.18              break;
    2.19 @@ -187,7 +192,6 @@ int libxl_device_pci_parse_bdf(libxl_ctx
    2.20      return 0;
    2.21  
    2.22  parse_error:
    2.23 -    printf("parse error: %s\n", str);
    2.24      return ERROR_INVAL;
    2.25  }
    2.26  
    2.27 @@ -531,6 +535,55 @@ int libxl_device_pci_list_assignable(lib
    2.28      return 0;
    2.29  }
    2.30  
    2.31 +/* 
    2.32 + * This function checks that all functions of a device are bound to pciback
    2.33 + * driver. It also initialises a bit-mask of which function numbers are present
    2.34 + * on that device.
    2.35 +*/
    2.36 +static int pci_multifunction_check(libxl_ctx *ctx, libxl_device_pci *pcidev, unsigned int *func_mask)
    2.37 +{
    2.38 +    struct dirent *de;
    2.39 +    DIR *dir;
    2.40 +
    2.41 +    *func_mask = 0;
    2.42 +
    2.43 +    dir = opendir(SYSFS_PCI_DEV);
    2.44 +    if ( NULL == dir ) {
    2.45 +        XL_LOG_ERRNO(ctx, XL_LOG_ERROR, "Couldn't open %s", SYSFS_PCI_DEV);
    2.46 +        return -1;
    2.47 +    }
    2.48 +
    2.49 +    while( (de = readdir(dir)) ) {
    2.50 +        unsigned dom, bus, dev, func;
    2.51 +        struct stat st;
    2.52 +        char *path;
    2.53 +
    2.54 +        if ( sscanf(de->d_name, PCI_BDF, &dom, &bus, &dev, &func) != 4 )
    2.55 +            continue;
    2.56 +        if ( pcidev->domain != dom )
    2.57 +            continue;
    2.58 +        if ( pcidev->bus != bus )
    2.59 +            continue;
    2.60 +        if ( pcidev->dev != dev )
    2.61 +            continue;
    2.62 +
    2.63 +        path = libxl_sprintf(ctx, "%s/" PCI_BDF, SYSFS_PCIBACK_DRIVER, dom, bus, dev, func);
    2.64 +        if ( lstat(path, &st) ) {
    2.65 +            if ( errno == ENOENT )
    2.66 +                XL_LOG(ctx, XL_LOG_ERROR, PCI_BDF " is not assigned to pciback driver",
    2.67 +                       dom, bus, dev, func);
    2.68 +            else
    2.69 +                XL_LOG_ERRNO(ctx, XL_LOG_ERROR, "Couldn't lstat %s", path);
    2.70 +            closedir(dir);
    2.71 +            return -1;
    2.72 +        }
    2.73 +        (*func_mask) |= (1 << func);
    2.74 +    }
    2.75 +
    2.76 +    closedir(dir);
    2.77 +    return 0;
    2.78 +}
    2.79 +
    2.80  static int pci_ins_check(libxl_ctx *ctx, uint32_t domid, const char *state, void *priv)
    2.81  {
    2.82      char *orig_state = priv;
    2.83 @@ -652,8 +705,9 @@ out:
    2.84  
    2.85  int libxl_device_pci_add(libxl_ctx *ctx, uint32_t domid, libxl_device_pci *pcidev)
    2.86  {
    2.87 +    unsigned int orig_vdev, pfunc_mask;
    2.88      libxl_device_pci *assigned;
    2.89 -    int num_assigned, rc;
    2.90 +    int num_assigned, rc, i;
    2.91      int stubdomid = 0;
    2.92  
    2.93      rc = get_all_assigned_devices(ctx, &assigned, &num_assigned);
    2.94 @@ -679,10 +733,43 @@ int libxl_device_pci_add(libxl_ctx *ctx,
    2.95              return rc;
    2.96      }
    2.97  
    2.98 -    return do_pci_add(ctx, domid, pcidev);
    2.99 +    orig_vdev = pcidev->vdevfn & ~7U;
   2.100 +
   2.101 +    if ( pcidev->vfunc_mask == LIBXL_PCI_FUNC_ALL ) {
   2.102 +        if ( !(pcidev->vdevfn >> 3) ) {
   2.103 +            XL_LOG(ctx, XL_LOG_ERROR, "Must specify a v-slot for multi-function devices");
   2.104 +            return ERROR_INVAL;
   2.105 +        }
   2.106 +        if ( pci_multifunction_check(ctx, pcidev, &pfunc_mask) ) {
   2.107 +            return ERROR_FAIL;
   2.108 +        }
   2.109 +        pcidev->vfunc_mask &= pfunc_mask;
   2.110 +        /* so now vfunc_mask == pfunc_mask */
   2.111 +    }else{
   2.112 +        pfunc_mask = (1 << pcidev->func);
   2.113 +    }
   2.114 +
   2.115 +    for(i = 7; i >= 0; --i) {
   2.116 +        if ( (1 << i) & pfunc_mask ) {
   2.117 +            if ( pcidev->vfunc_mask == pfunc_mask ) {
   2.118 +                pcidev->func = i;
   2.119 +                pcidev->vdevfn = orig_vdev | i;
   2.120 +            }else{
   2.121 +                /* if not passing through multiple devices in a block make
   2.122 +                 * sure that virtual function number 0 is always used otherwise
   2.123 +                 * guest won't see the device
   2.124 +                 */
   2.125 +                pcidev->vdevfn = orig_vdev;
   2.126 +            }
   2.127 +            if ( do_pci_add(ctx, domid, pcidev) )
   2.128 +                rc = ERROR_FAIL;
   2.129 +        }
   2.130 +    }
   2.131 +
   2.132 +    return rc;
   2.133  }
   2.134  
   2.135 -int libxl_device_pci_remove(libxl_ctx *ctx, uint32_t domid, libxl_device_pci *pcidev)
   2.136 +static int do_pci_remove(libxl_ctx *ctx, uint32_t domid, libxl_device_pci *pcidev)
   2.137  {
   2.138      libxl_device_pci *assigned;
   2.139      char *path;
   2.140 @@ -711,10 +798,15 @@ int libxl_device_pci_remove(libxl_ctx *c
   2.141          libxl_xs_write(ctx, XBT_NULL, path, PCI_BDF, pcidev->domain,
   2.142                         pcidev->bus, pcidev->dev, pcidev->func);
   2.143          path = libxl_sprintf(ctx, "/local/domain/0/device-model/%d/command", domid);
   2.144 -        xs_write(ctx->xsh, XBT_NULL, path, "pci-rem", strlen("pci-rem"));
   2.145 -        if (libxl_wait_for_device_model(ctx, domid, "pci-removed", NULL, NULL) < 0) {
   2.146 -            XL_LOG(ctx, XL_LOG_ERROR, "Device Model didn't respond in time");
   2.147 -            return ERROR_FAIL;
   2.148 +
   2.149 +        /* Remove all functions at once atomically by only signalling
   2.150 +         * device-model for function 0 */
   2.151 +        if ( (pcidev->vdevfn & 0x7) == 0 ) {
   2.152 +            xs_write(ctx->xsh, XBT_NULL, path, "pci-rem", strlen("pci-rem"));
   2.153 +            if (libxl_wait_for_device_model(ctx, domid, "pci-removed", NULL, NULL) < 0) {
   2.154 +                XL_LOG(ctx, XL_LOG_ERROR, "Device Model didn't respond in time");
   2.155 +                return ERROR_FAIL;
   2.156 +            }
   2.157          }
   2.158          path = libxl_sprintf(ctx, "/local/domain/0/device-model/%d/state", domid);
   2.159          xs_write(ctx->xsh, XBT_NULL, path, state, strlen(state));
   2.160 @@ -769,7 +861,10 @@ skip1:
   2.161          fclose(f);
   2.162      }
   2.163  out:
   2.164 -    libxl_device_pci_reset(ctx, pcidev->domain, pcidev->bus, pcidev->dev, pcidev->func);
   2.165 +    /* don't do multiple resets while some functions are still passed through */
   2.166 +    if ( (pcidev->vdevfn & 0x7) == 0 ) {
   2.167 +        libxl_device_pci_reset(ctx, pcidev->domain, pcidev->bus, pcidev->dev, pcidev->func);
   2.168 +    }
   2.169  
   2.170      if (!libxl_is_stubdom(ctx, domid, NULL)) {
   2.171          rc = xc_deassign_device(ctx->xch, domid, pcidev->value);
   2.172 @@ -786,6 +881,38 @@ out:
   2.173      return 0;
   2.174  }
   2.175  
   2.176 +int libxl_device_pci_remove(libxl_ctx *ctx, uint32_t domid, libxl_device_pci *pcidev)
   2.177 +{
   2.178 +    unsigned int orig_vdev, pfunc_mask;
   2.179 +    int i, rc;
   2.180 +
   2.181 +    orig_vdev = pcidev->vdevfn & ~7U;
   2.182 +
   2.183 +    if ( pcidev->vfunc_mask == LIBXL_PCI_FUNC_ALL ) {
   2.184 +        if ( pci_multifunction_check(ctx, pcidev, &pfunc_mask) ) {
   2.185 +            return ERROR_FAIL;
   2.186 +        }
   2.187 +        pcidev->vfunc_mask &= pfunc_mask;
   2.188 +    }else{
   2.189 +        pfunc_mask = (1 << pcidev->func);
   2.190 +    }
   2.191 +
   2.192 +    for(i = 7; i >= 0; --i) {
   2.193 +        if ( (1 << i) & pfunc_mask ) {
   2.194 +            if ( pcidev->vfunc_mask == pfunc_mask ) {
   2.195 +                pcidev->func = i;
   2.196 +                pcidev->vdevfn = orig_vdev | i;
   2.197 +            }else{
   2.198 +                pcidev->vdevfn = orig_vdev;
   2.199 +            }
   2.200 +            if ( do_pci_remove(ctx, domid, pcidev) )
   2.201 +                rc = ERROR_FAIL;
   2.202 +        }
   2.203 +    }
   2.204 +
   2.205 +    return rc;
   2.206 +}
   2.207 +
   2.208  int libxl_device_pci_list_assigned(libxl_ctx *ctx, libxl_device_pci **list, uint32_t domid, int *num)
   2.209  {
   2.210      char *be_path, *num_devs, *xsdev, *xsvdevfn, *xsopts;
     3.1 --- a/tools/libxl/xl_cmdimpl.c	Mon Aug 09 17:44:01 2010 +0100
     3.2 +++ b/tools/libxl/xl_cmdimpl.c	Mon Aug 09 17:46:39 2010 +0100
     3.3 @@ -1931,7 +1931,7 @@ void pcilist_assignable(void)
     3.4      if ( libxl_device_pci_list_assignable(&ctx, &pcidevs, &num) )
     3.5          return;
     3.6      for (i = 0; i < num; i++) {
     3.7 -        printf("%04x:%02x:%02x:%01x\n",
     3.8 +        printf("%04x:%02x:%02x.%01x\n",
     3.9                  pcidevs[i].domain, pcidevs[i].bus, pcidevs[i].dev, pcidevs[i].func);
    3.10      }
    3.11      free(pcidevs);