debuggers.hg

view tools/libxl/libxl_pci.c @ 21951:6b9e44cf2651

Allow PCI passthrough to PV guest without IOMMU

Added a check which allows adding and removal of PCI devices for PV
guests in the absence of an IOMMU.

Signed-off-by: Mihir Nanavati <mihirn@cs.ubc.ca>
Signed-off-by: Stefano Stabellini <stefano.stabellini@eu.citrix.com>
author Mihir Nanavati <mihirn@cs.ubc.ca>
date Tue Aug 03 17:14:06 2010 +0100 (2010-08-03)
parents 9ced33b68bd8
children b8a651785471
line source
1 /*
2 * Copyright (C) 2009 Citrix Ltd.
3 * Author Vincent Hanquez <vincent.hanquez@eu.citrix.com>
4 * Author Stefano Stabellini <stefano.stabellini@eu.citrix.com>
5 *
6 * This program is free software; you can redistribute it and/or modify
7 * it under the terms of the GNU Lesser General Public License as published
8 * by the Free Software Foundation; version 2.1 only. with the special
9 * exception on linking described in file LICENSE.
10 *
11 * This program is distributed in the hope that it will be useful,
12 * but WITHOUT ANY WARRANTY; without even the implied warranty of
13 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14 * GNU Lesser General Public License for more details.
15 */
17 #include "libxl_osdeps.h"
19 #include <stdio.h>
20 #include <string.h>
21 #include <stdlib.h>
22 #include <sys/types.h>
23 #include <fcntl.h>
24 #include <sys/select.h>
25 #include <sys/mman.h>
26 #include <sys/wait.h>
27 #include <signal.h>
28 #include <unistd.h> /* for write, unlink and close */
29 #include <stdint.h>
30 #include <inttypes.h>
31 #include <dirent.h>
32 #include <assert.h>
34 #include "libxl.h"
35 #include "libxl_utils.h"
36 #include "libxl_internal.h"
37 #include "flexarray.h"
39 static int libxl_create_pci_backend(libxl_ctx *ctx, uint32_t domid, libxl_device_pci *pcidev, int num)
40 {
41 flexarray_t *front;
42 flexarray_t *back;
43 unsigned int boffset = 0;
44 unsigned int foffset = 0;
45 libxl_device device;
46 int i;
48 front = flexarray_make(16, 1);
49 if (!front)
50 return ERROR_NOMEM;
51 back = flexarray_make(16, 1);
52 if (!back)
53 return ERROR_NOMEM;
55 XL_LOG(ctx, XL_LOG_DEBUG, "Creating pci backend");
57 /* add pci device */
58 device.backend_devid = 0;
59 device.backend_domid = 0;
60 device.backend_kind = DEVICE_PCI;
61 device.devid = 0;
62 device.domid = domid;
63 device.kind = DEVICE_PCI;
65 flexarray_set(back, boffset++, "frontend-id");
66 flexarray_set(back, boffset++, libxl_sprintf(ctx, "%d", domid));
67 flexarray_set(back, boffset++, "online");
68 flexarray_set(back, boffset++, "1");
69 flexarray_set(back, boffset++, "state");
70 flexarray_set(back, boffset++, libxl_sprintf(ctx, "%d", 1));
71 flexarray_set(back, boffset++, "domain");
72 flexarray_set(back, boffset++, libxl_domid_to_name(ctx, domid));
73 for (i = 0; i < num; i++) {
74 flexarray_set(back, boffset++, libxl_sprintf(ctx, "key-%d", i));
75 flexarray_set(back, boffset++, libxl_sprintf(ctx, PCI_BDF, pcidev->domain, pcidev->bus, pcidev->dev, pcidev->func));
76 flexarray_set(back, boffset++, libxl_sprintf(ctx, "dev-%d", i));
77 flexarray_set(back, boffset++, libxl_sprintf(ctx, PCI_BDF, pcidev->domain, pcidev->bus, pcidev->dev, pcidev->func));
78 if (pcidev->vdevfn) {
79 flexarray_set(back, boffset++, libxl_sprintf(ctx, "vdevfn-%d", i));
80 flexarray_set(back, boffset++, libxl_sprintf(ctx, "%x", pcidev->vdevfn));
81 }
82 flexarray_set(back, boffset++, libxl_sprintf(ctx, "opts-%d", i));
83 flexarray_set(back, boffset++, libxl_sprintf(ctx, "msitranslate=%d,power_mgmt=%d", pcidev->msitranslate, pcidev->power_mgmt));
84 flexarray_set(back, boffset++, libxl_sprintf(ctx, "state-%d", i));
85 flexarray_set(back, boffset++, libxl_sprintf(ctx, "%d", 1));
86 }
87 flexarray_set(back, boffset++, "num_devs");
88 flexarray_set(back, boffset++, libxl_sprintf(ctx, "%d", num));
90 flexarray_set(front, foffset++, "backend-id");
91 flexarray_set(front, foffset++, libxl_sprintf(ctx, "%d", 0));
92 flexarray_set(front, foffset++, "state");
93 flexarray_set(front, foffset++, libxl_sprintf(ctx, "%d", 1));
95 libxl_device_generic_add(ctx, &device,
96 libxl_xs_kvs_of_flexarray(ctx, back, boffset),
97 libxl_xs_kvs_of_flexarray(ctx, front, foffset));
99 flexarray_free(back);
100 flexarray_free(front);
101 return 0;
102 }
104 static int libxl_device_pci_add_xenstore(libxl_ctx *ctx, uint32_t domid, libxl_device_pci *pcidev)
105 {
106 flexarray_t *back;
107 char *num_devs, *be_path;
108 int num = 0;
109 unsigned int boffset = 0;
110 xs_transaction_t t;
112 be_path = libxl_sprintf(ctx, "%s/backend/pci/%d/0", libxl_xs_get_dompath(ctx, 0), domid);
113 num_devs = libxl_xs_read(ctx, XBT_NULL, libxl_sprintf(ctx, "%s/num_devs", be_path));
114 if (!num_devs)
115 return libxl_create_pci_backend(ctx, domid, pcidev, 1);
117 if (!is_hvm(ctx, domid)) {
118 if (libxl_wait_for_backend(ctx, be_path, "4") < 0)
119 return ERROR_FAIL;
120 }
122 back = flexarray_make(16, 1);
123 if (!back)
124 return ERROR_NOMEM;
126 XL_LOG(ctx, XL_LOG_DEBUG, "Adding new pci device to xenstore");
127 num = atoi(num_devs);
128 flexarray_set(back, boffset++, libxl_sprintf(ctx, "key-%d", num));
129 flexarray_set(back, boffset++, libxl_sprintf(ctx, PCI_BDF, pcidev->domain, pcidev->bus, pcidev->dev, pcidev->func));
130 flexarray_set(back, boffset++, libxl_sprintf(ctx, "dev-%d", num));
131 flexarray_set(back, boffset++, libxl_sprintf(ctx, PCI_BDF, pcidev->domain, pcidev->bus, pcidev->dev, pcidev->func));
132 if (pcidev->vdevfn) {
133 flexarray_set(back, boffset++, libxl_sprintf(ctx, "vdevfn-%d", num));
134 flexarray_set(back, boffset++, libxl_sprintf(ctx, "%x", pcidev->vdevfn));
135 }
136 flexarray_set(back, boffset++, libxl_sprintf(ctx, "opts-%d", num));
137 flexarray_set(back, boffset++, libxl_sprintf(ctx, "msitranslate=%d,power_mgmt=%d", pcidev->msitranslate, pcidev->power_mgmt));
138 flexarray_set(back, boffset++, libxl_sprintf(ctx, "state-%d", num));
139 flexarray_set(back, boffset++, libxl_sprintf(ctx, "%d", 1));
140 flexarray_set(back, boffset++, "num_devs");
141 flexarray_set(back, boffset++, libxl_sprintf(ctx, "%d", num + 1));
142 flexarray_set(back, boffset++, "state");
143 flexarray_set(back, boffset++, libxl_sprintf(ctx, "%d", 7));
145 retry_transaction:
146 t = xs_transaction_start(ctx->xsh);
147 libxl_xs_writev(ctx, t, be_path,
148 libxl_xs_kvs_of_flexarray(ctx, back, boffset));
149 if (!xs_transaction_end(ctx->xsh, t, 0))
150 if (errno == EAGAIN)
151 goto retry_transaction;
153 flexarray_free(back);
154 return 0;
155 }
157 static int libxl_device_pci_remove_xenstore(libxl_ctx *ctx, uint32_t domid, libxl_device_pci *pcidev)
158 {
159 char *be_path, *num_devs_path, *num_devs, *xsdev, *tmp, *tmppath;
160 int num, i, j;
161 xs_transaction_t t;
162 unsigned int domain = 0, bus = 0, dev = 0, func = 0;
164 be_path = libxl_sprintf(ctx, "%s/backend/pci/%d/0", libxl_xs_get_dompath(ctx, 0), domid);
165 num_devs_path = libxl_sprintf(ctx, "%s/num_devs", be_path);
166 num_devs = libxl_xs_read(ctx, XBT_NULL, num_devs_path);
167 if (!num_devs)
168 return ERROR_INVAL;
169 num = atoi(num_devs);
171 if (!is_hvm(ctx, domid)) {
172 if (libxl_wait_for_backend(ctx, be_path, "4") < 0) {
173 XL_LOG(ctx, XL_LOG_DEBUG, "pci backend at %s is not ready", be_path);
174 return ERROR_FAIL;
175 }
176 }
178 for (i = 0; i < num; i++) {
179 xsdev = libxl_xs_read(ctx, XBT_NULL, libxl_sprintf(ctx, "%s/dev-%d", be_path, i));
180 sscanf(xsdev, PCI_BDF, &domain, &bus, &dev, &func);
181 if (domain == pcidev->domain && bus == pcidev->bus &&
182 pcidev->dev == dev && pcidev->func == func) {
183 break;
184 }
185 }
186 if (i == num) {
187 XL_LOG(ctx, XL_LOG_ERROR, "Couldn't find the device on xenstore");
188 return ERROR_INVAL;
189 }
191 retry_transaction:
192 t = xs_transaction_start(ctx->xsh);
193 xs_write(ctx->xsh, t, libxl_sprintf(ctx, "%s/state-%d", be_path, i), "5", strlen("5"));
194 xs_write(ctx->xsh, t, libxl_sprintf(ctx, "%s/state", be_path), "7", strlen("7"));
195 if (!xs_transaction_end(ctx->xsh, t, 0))
196 if (errno == EAGAIN)
197 goto retry_transaction;
199 if (!is_hvm(ctx, domid)) {
200 if (libxl_wait_for_backend(ctx, be_path, "4") < 0) {
201 XL_LOG(ctx, XL_LOG_DEBUG, "pci backend at %s is not ready", be_path);
202 return ERROR_FAIL;
203 }
204 }
206 retry_transaction2:
207 t = xs_transaction_start(ctx->xsh);
208 xs_rm(ctx->xsh, t, libxl_sprintf(ctx, "%s/state-%d", be_path, i));
209 xs_rm(ctx->xsh, t, libxl_sprintf(ctx, "%s/key-%d", be_path, i));
210 xs_rm(ctx->xsh, t, libxl_sprintf(ctx, "%s/dev-%d", be_path, i));
211 xs_rm(ctx->xsh, t, libxl_sprintf(ctx, "%s/vdev-%d", be_path, i));
212 xs_rm(ctx->xsh, t, libxl_sprintf(ctx, "%s/opts-%d", be_path, i));
213 xs_rm(ctx->xsh, t, libxl_sprintf(ctx, "%s/vdevfn-%d", be_path, i));
214 libxl_xs_write(ctx, t, num_devs_path, "%d", num - 1);
215 for (j = i + 1; j < num; j++) {
216 tmppath = libxl_sprintf(ctx, "%s/state-%d", be_path, j);
217 tmp = libxl_xs_read(ctx, t, tmppath);
218 xs_write(ctx->xsh, t, libxl_sprintf(ctx, "%s/state-%d", be_path, j - 1), tmp, strlen(tmp));
219 xs_rm(ctx->xsh, t, tmppath);
220 tmppath = libxl_sprintf(ctx, "%s/dev-%d", be_path, j);
221 tmp = libxl_xs_read(ctx, t, tmppath);
222 xs_write(ctx->xsh, t, libxl_sprintf(ctx, "%s/dev-%d", be_path, j - 1), tmp, strlen(tmp));
223 xs_rm(ctx->xsh, t, tmppath);
224 tmppath = libxl_sprintf(ctx, "%s/key-%d", be_path, j);
225 tmp = libxl_xs_read(ctx, t, tmppath);
226 xs_write(ctx->xsh, t, libxl_sprintf(ctx, "%s/key-%d", be_path, j - 1), tmp, strlen(tmp));
227 xs_rm(ctx->xsh, t, tmppath);
228 tmppath = libxl_sprintf(ctx, "%s/vdev-%d", be_path, j);
229 tmp = libxl_xs_read(ctx, t, tmppath);
230 if (tmp) {
231 xs_write(ctx->xsh, t, libxl_sprintf(ctx, "%s/vdev-%d", be_path, j - 1), tmp, strlen(tmp));
232 xs_rm(ctx->xsh, t, tmppath);
233 }
234 tmppath = libxl_sprintf(ctx, "%s/opts-%d", be_path, j);
235 tmp = libxl_xs_read(ctx, t, tmppath);
236 if (tmp) {
237 xs_write(ctx->xsh, t, libxl_sprintf(ctx, "%s/opts-%d", be_path, j - 1), tmp, strlen(tmp));
238 xs_rm(ctx->xsh, t, tmppath);
239 }
240 tmppath = libxl_sprintf(ctx, "%s/vdevfn-%d", be_path, j);
241 tmp = libxl_xs_read(ctx, t, tmppath);
242 if (tmp) {
243 xs_write(ctx->xsh, t, libxl_sprintf(ctx, "%s/vdevfn-%d", be_path, j - 1), tmp, strlen(tmp));
244 xs_rm(ctx->xsh, t, tmppath);
245 }
246 }
247 if (!xs_transaction_end(ctx->xsh, t, 0))
248 if (errno == EAGAIN)
249 goto retry_transaction2;
251 if (num == 1) {
252 char *fe_path = libxl_xs_read(ctx, XBT_NULL, libxl_sprintf(ctx, "%s/frontend", be_path));
253 libxl_device_destroy(ctx, be_path, 1);
254 xs_rm(ctx->xsh, XBT_NULL, be_path);
255 xs_rm(ctx->xsh, XBT_NULL, fe_path);
256 return 0;
257 }
259 return 0;
260 }
262 static int get_all_assigned_devices(libxl_ctx *ctx, libxl_device_pci **list, int *num)
263 {
264 libxl_device_pci *pcidevs = NULL;
265 char **domlist;
266 unsigned int nd = 0, i;
268 *list = NULL;
269 *num = 0;
271 domlist = libxl_xs_directory(ctx, XBT_NULL, "/local/domain", &nd);
272 for(i = 0; i < nd; i++) {
273 char *path, *num_devs;
275 path = libxl_sprintf(ctx, "/local/domain/0/backend/pci/%s/0/num_devs", domlist[i]);
276 num_devs = libxl_xs_read(ctx, XBT_NULL, path);
277 if ( num_devs ) {
278 int ndev = atoi(num_devs), j;
279 char *devpath, *bdf;
281 pcidevs = calloc(sizeof(*pcidevs), ndev);
282 for(j = (pcidevs) ? 0 : ndev; j < ndev; j++) {
283 devpath = libxl_sprintf(ctx, "/local/domain/0/backend/pci/%s/0/dev-%u",
284 domlist[i], j);
285 bdf = libxl_xs_read(ctx, XBT_NULL, devpath);
286 if ( bdf ) {
287 unsigned dom, bus, dev, func;
288 if ( sscanf(bdf, PCI_BDF, &dom, &bus, &dev, &func) != 4 )
289 continue;
291 libxl_device_pci_init(pcidevs + *num, dom, bus, dev, func, 0);
292 (*num)++;
293 }
294 }
295 }
296 }
298 if ( 0 == *num ) {
299 free(pcidevs);
300 pcidevs = NULL;
301 }else{
302 *list = pcidevs;
303 }
305 return 0;
306 }
308 static int is_assigned(libxl_device_pci *assigned, int num_assigned,
309 int dom, int bus, int dev, int func)
310 {
311 int i;
313 for(i = 0; i < num_assigned; i++) {
314 if ( assigned[i].domain != dom )
315 continue;
316 if ( assigned[i].bus != bus )
317 continue;
318 if ( assigned[i].dev != dev )
319 continue;
320 if ( assigned[i].func != func )
321 continue;
322 return 1;
323 }
325 return 0;
326 }
328 static int do_pci_add(libxl_ctx *ctx, uint32_t domid, libxl_device_pci *pcidev)
329 {
330 char *path;
331 char *state, *vdevfn;
332 int rc, hvm;
334 hvm = is_hvm(ctx, domid);
335 if (hvm) {
336 if (libxl_wait_for_device_model(ctx, domid, "running", NULL, NULL) < 0) {
337 return ERROR_FAIL;
338 }
339 path = libxl_sprintf(ctx, "/local/domain/0/device-model/%d/state", domid);
340 state = libxl_xs_read(ctx, XBT_NULL, path);
341 path = libxl_sprintf(ctx, "/local/domain/0/device-model/%d/parameter", domid);
342 if (pcidev->vdevfn)
343 libxl_xs_write(ctx, XBT_NULL, path, PCI_BDF_VDEVFN, pcidev->domain,
344 pcidev->bus, pcidev->dev, pcidev->func, pcidev->vdevfn);
345 else
346 libxl_xs_write(ctx, XBT_NULL, path, PCI_BDF, pcidev->domain,
347 pcidev->bus, pcidev->dev, pcidev->func);
348 path = libxl_sprintf(ctx, "/local/domain/0/device-model/%d/command", domid);
349 xs_write(ctx->xsh, XBT_NULL, path, "pci-ins", strlen("pci-ins"));
350 if (libxl_wait_for_device_model(ctx, domid, "pci-inserted", NULL, NULL) < 0)
351 XL_LOG(ctx, XL_LOG_ERROR, "Device Model didn't respond in time");
352 path = libxl_sprintf(ctx, "/local/domain/0/device-model/%d/parameter", domid);
353 vdevfn = libxl_xs_read(ctx, XBT_NULL, path);
354 sscanf(vdevfn + 2, "%x", &pcidev->vdevfn);
355 path = libxl_sprintf(ctx, "/local/domain/0/device-model/%d/state", domid);
356 xs_write(ctx->xsh, XBT_NULL, path, state, strlen(state));
357 } else {
358 char *sysfs_path = libxl_sprintf(ctx, SYSFS_PCI_DEV"/"PCI_BDF"/resource", pcidev->domain,
359 pcidev->bus, pcidev->dev, pcidev->func);
360 FILE *f = fopen(sysfs_path, "r");
361 unsigned long long start = 0, end = 0, flags = 0, size = 0;
362 int irq = 0;
363 int i;
365 if (f == NULL) {
366 XL_LOG_ERRNO(ctx, XL_LOG_ERROR, "Couldn't open %s", sysfs_path);
367 return ERROR_FAIL;
368 }
369 for (i = 0; i < PROC_PCI_NUM_RESOURCES; i++) {
370 if (fscanf(f, "0x%llx 0x%llx 0x%llx\n", &start, &end, &flags) != 3)
371 continue;
372 size = end - start + 1;
373 if (start) {
374 if (flags & PCI_BAR_IO) {
375 rc = xc_domain_ioport_permission(ctx->xch, domid, start, size, 1);
376 if (rc < 0) {
377 XL_LOG_ERRNOVAL(ctx, XL_LOG_ERROR, rc, "Error: xc_domain_ioport_permission error 0x%llx/0x%llx", start, size);
378 fclose(f);
379 return ERROR_FAIL;
380 }
381 } else {
382 rc = xc_domain_iomem_permission(ctx->xch, domid, start>>XC_PAGE_SHIFT,
383 (size+(XC_PAGE_SIZE-1))>>XC_PAGE_SHIFT, 1);
384 if (rc < 0) {
385 XL_LOG_ERRNOVAL(ctx, XL_LOG_ERROR, rc, "Error: xc_domain_iomem_permission error 0x%llx/0x%llx", start, size);
386 fclose(f);
387 return ERROR_FAIL;
388 }
389 }
390 }
391 }
392 fclose(f);
393 sysfs_path = libxl_sprintf(ctx, SYSFS_PCI_DEV"/"PCI_BDF"/irq", pcidev->domain,
394 pcidev->bus, pcidev->dev, pcidev->func);
395 f = fopen(sysfs_path, "r");
396 if (f == NULL) {
397 XL_LOG_ERRNO(ctx, XL_LOG_ERROR, "Couldn't open %s", sysfs_path);
398 goto out;
399 }
400 if ((fscanf(f, "%u", &irq) == 1) && irq) {
401 rc = xc_physdev_map_pirq(ctx->xch, domid, irq, &irq);
402 if (rc < 0) {
403 XL_LOG_ERRNOVAL(ctx, XL_LOG_ERROR, rc, "Error: xc_physdev_map_pirq irq=%d", irq);
404 fclose(f);
405 return ERROR_FAIL;
406 }
407 rc = xc_domain_irq_permission(ctx->xch, domid, irq, 1);
408 if (rc < 0) {
409 XL_LOG_ERRNOVAL(ctx, XL_LOG_ERROR, rc, "Error: xc_domain_irq_permission irq=%d", irq);
410 fclose(f);
411 return ERROR_FAIL;
412 }
413 }
414 fclose(f);
415 }
416 out:
417 if (!libxl_is_stubdom(ctx, domid, NULL)) {
418 rc = xc_assign_device(ctx->xch, domid, pcidev->value);
419 if (rc < 0 && (hvm || errno != ENOSYS)) {
420 XL_LOG_ERRNOVAL(ctx, XL_LOG_ERROR, rc, "xc_assign_device failed");
421 return ERROR_FAIL;
422 }
423 }
425 libxl_device_pci_add_xenstore(ctx, domid, pcidev);
426 return 0;
427 }
429 int libxl_device_pci_add(libxl_ctx *ctx, uint32_t domid, libxl_device_pci *pcidev)
430 {
431 libxl_device_pci *assigned;
432 int num_assigned, rc;
433 int stubdomid = 0;
435 rc = get_all_assigned_devices(ctx, &assigned, &num_assigned);
436 if ( rc ) {
437 XL_LOG(ctx, XL_LOG_ERROR, "cannot determine if device is assigned, refusing to continue");
438 return ERROR_FAIL;
439 }
440 if ( is_assigned(assigned, num_assigned, pcidev->domain,
441 pcidev->bus, pcidev->dev, pcidev->func) ) {
442 XL_LOG(ctx, XL_LOG_ERROR, "PCI device already attached to a domain");
443 free(assigned);
444 return ERROR_FAIL;
445 }
446 free(assigned);
448 libxl_device_pci_reset(ctx, pcidev->domain, pcidev->bus, pcidev->dev, pcidev->func);
450 stubdomid = libxl_get_stubdom_id(ctx, domid);
451 if (stubdomid != 0) {
452 libxl_device_pci pcidev_s = *pcidev;
453 rc = do_pci_add(ctx, stubdomid, &pcidev_s);
454 if ( rc )
455 return rc;
456 }
458 return do_pci_add(ctx, domid, pcidev);
459 }
461 int libxl_device_pci_remove(libxl_ctx *ctx, uint32_t domid, libxl_device_pci *pcidev)
462 {
463 char *path;
464 char *state;
465 int hvm, rc;
466 int stubdomid = 0;
468 /* TODO: check if the device can be detached */
469 libxl_device_pci_remove_xenstore(ctx, domid, pcidev);
471 hvm = is_hvm(ctx, domid);
472 if (hvm) {
473 if (libxl_wait_for_device_model(ctx, domid, "running", NULL, NULL) < 0) {
474 return ERROR_FAIL;
475 }
476 path = libxl_sprintf(ctx, "/local/domain/0/device-model/%d/state", domid);
477 state = libxl_xs_read(ctx, XBT_NULL, path);
478 path = libxl_sprintf(ctx, "/local/domain/0/device-model/%d/parameter", domid);
479 libxl_xs_write(ctx, XBT_NULL, path, PCI_BDF, pcidev->domain,
480 pcidev->bus, pcidev->dev, pcidev->func);
481 path = libxl_sprintf(ctx, "/local/domain/0/device-model/%d/command", domid);
482 xs_write(ctx->xsh, XBT_NULL, path, "pci-rem", strlen("pci-rem"));
483 if (libxl_wait_for_device_model(ctx, domid, "pci-removed", NULL, NULL) < 0) {
484 XL_LOG(ctx, XL_LOG_ERROR, "Device Model didn't respond in time");
485 return ERROR_FAIL;
486 }
487 path = libxl_sprintf(ctx, "/local/domain/0/device-model/%d/state", domid);
488 xs_write(ctx->xsh, XBT_NULL, path, state, strlen(state));
489 } else {
490 char *sysfs_path = libxl_sprintf(ctx, SYSFS_PCI_DEV"/"PCI_BDF"/resource", pcidev->domain,
491 pcidev->bus, pcidev->dev, pcidev->func);
492 FILE *f = fopen(sysfs_path, "r");
493 unsigned int start = 0, end = 0, flags = 0, size = 0;
494 int irq = 0;
495 int i;
497 if (f == NULL) {
498 XL_LOG_ERRNO(ctx, XL_LOG_ERROR, "Couldn't open %s", sysfs_path);
499 goto skip1;
500 }
501 for (i = 0; i < PROC_PCI_NUM_RESOURCES; i++) {
502 if (fscanf(f, "0x%x 0x%x 0x%x\n", &start, &end, &flags) != 3)
503 continue;
504 size = end - start + 1;
505 if (start) {
506 if (flags & PCI_BAR_IO) {
507 rc = xc_domain_ioport_permission(ctx->xch, domid, start, size, 0);
508 if (rc < 0)
509 XL_LOG_ERRNOVAL(ctx, XL_LOG_ERROR, rc, "xc_domain_ioport_permission error 0x%x/0x%x", start, size);
510 } else {
511 rc = xc_domain_iomem_permission(ctx->xch, domid, start>>XC_PAGE_SHIFT,
512 (size+(XC_PAGE_SIZE-1))>>XC_PAGE_SHIFT, 0);
513 if (rc < 0)
514 XL_LOG_ERRNOVAL(ctx, XL_LOG_ERROR, rc, "xc_domain_iomem_permission error 0x%x/0x%x", start, size);
515 }
516 }
517 }
518 fclose(f);
519 skip1:
520 sysfs_path = libxl_sprintf(ctx, SYSFS_PCI_DEV"/"PCI_BDF"/irq", pcidev->domain,
521 pcidev->bus, pcidev->dev, pcidev->func);
522 f = fopen(sysfs_path, "r");
523 if (f == NULL) {
524 XL_LOG_ERRNO(ctx, XL_LOG_ERROR, "Couldn't open %s", sysfs_path);
525 goto out;
526 }
527 if ((fscanf(f, "%u", &irq) == 1) && irq) {
528 rc = xc_physdev_unmap_pirq(ctx->xch, domid, irq);
529 if (rc < 0) {
530 XL_LOG_ERRNOVAL(ctx, XL_LOG_ERROR, rc, "xc_physdev_map_pirq irq=%d", irq);
531 }
532 rc = xc_domain_irq_permission(ctx->xch, domid, irq, 0);
533 if (rc < 0) {
534 XL_LOG_ERRNOVAL(ctx, XL_LOG_ERROR, rc, "xc_domain_irq_permission irq=%d", irq);
535 }
536 }
537 fclose(f);
538 }
539 out:
540 libxl_device_pci_reset(ctx, pcidev->domain, pcidev->bus, pcidev->dev, pcidev->func);
542 if (!libxl_is_stubdom(ctx, domid, NULL)) {
543 rc = xc_deassign_device(ctx->xch, domid, pcidev->value);
544 if (rc < 0 && (hvm || errno != ENOSYS))
545 XL_LOG_ERRNOVAL(ctx, XL_LOG_ERROR, rc, "xc_deassign_device failed");
546 }
548 stubdomid = libxl_get_stubdom_id(ctx, domid);
549 if (stubdomid != 0) {
550 libxl_device_pci pcidev_s = *pcidev;
551 libxl_device_pci_remove(ctx, stubdomid, &pcidev_s);
552 }
554 return 0;
555 }
557 static libxl_device_pci *scan_sys_pcidir(libxl_device_pci *assigned,
558 int num_assigned, const char *path, int *num)
559 {
560 libxl_device_pci *pcidevs = NULL, *new;
561 struct dirent *de;
562 DIR *dir;
564 dir = opendir(path);
565 if ( NULL == dir )
566 return pcidevs;
568 while( (de = readdir(dir)) ) {
569 unsigned dom, bus, dev, func;
570 if ( sscanf(de->d_name, PCI_BDF, &dom, &bus, &dev, &func) != 4 )
571 continue;
573 if ( is_assigned(assigned, num_assigned, dom, bus, dev, func) )
574 continue;
576 new = realloc(pcidevs, ((*num) + 1) * sizeof(*new));
577 if ( NULL == new )
578 continue;
580 pcidevs = new;
581 new = pcidevs + *num;
583 memset(new, 0, sizeof(*new));
584 libxl_device_pci_init(new, dom, bus, dev, func, 0);
585 (*num)++;
586 }
588 closedir(dir);
589 return pcidevs;
590 }
592 int libxl_device_pci_list_assignable(libxl_ctx *ctx, libxl_device_pci **list, int *num)
593 {
594 libxl_device_pci *pcidevs = NULL;
595 libxl_device_pci *assigned;
596 int num_assigned, rc;
598 *num = 0;
599 *list = NULL;
601 rc = get_all_assigned_devices(ctx, &assigned, &num_assigned);
602 if ( rc )
603 return rc;
605 pcidevs = scan_sys_pcidir(assigned, num_assigned,
606 SYSFS_PCIBACK_DRIVER, num);
608 free(assigned);
609 if ( *num )
610 *list = pcidevs;
611 return 0;
612 }
614 int libxl_device_pci_list_assigned(libxl_ctx *ctx, libxl_device_pci **list, uint32_t domid, int *num)
615 {
616 char *be_path, *num_devs, *xsdev, *xsvdevfn, *xsopts;
617 int n, i;
618 unsigned int domain = 0, bus = 0, dev = 0, func = 0, vdevfn = 0;
619 libxl_device_pci *pcidevs;
621 be_path = libxl_sprintf(ctx, "%s/backend/pci/%d/0", libxl_xs_get_dompath(ctx, 0), domid);
622 num_devs = libxl_xs_read(ctx, XBT_NULL, libxl_sprintf(ctx, "%s/num_devs", be_path));
623 if (!num_devs) {
624 *num = 0;
625 *list = NULL;
626 return ERROR_FAIL;
627 }
628 n = atoi(num_devs);
629 pcidevs = calloc(n, sizeof(libxl_device_pci));
630 *num = n;
632 for (i = 0; i < n; i++) {
633 xsdev = libxl_xs_read(ctx, XBT_NULL, libxl_sprintf(ctx, "%s/dev-%d", be_path, i));
634 sscanf(xsdev, PCI_BDF, &domain, &bus, &dev, &func);
635 xsvdevfn = libxl_xs_read(ctx, XBT_NULL, libxl_sprintf(ctx, "%s/vdevfn-%d", be_path, i));
636 if (xsvdevfn)
637 vdevfn = strtol(xsvdevfn, (char **) NULL, 16);
638 libxl_device_pci_init(pcidevs + i, domain, bus, dev, func, vdevfn);
639 xsopts = libxl_xs_read(ctx, XBT_NULL, libxl_sprintf(ctx, "%s/opts-%d", be_path, i));
640 if (xsopts) {
641 char *saveptr;
642 char *p = strtok_r(xsopts, ",=", &saveptr);
643 do {
644 while (*p == ' ')
645 p++;
646 if (!strcmp(p, "msitranslate")) {
647 p = strtok_r(NULL, ",=", &saveptr);
648 pcidevs[i].msitranslate = atoi(p);
649 } else if (!strcmp(p, "power_mgmt")) {
650 p = strtok_r(NULL, ",=", &saveptr);
651 pcidevs[i].power_mgmt = atoi(p);
652 }
653 } while ((p = strtok_r(NULL, ",=", &saveptr)) != NULL);
654 }
655 }
656 if ( *num )
657 *list = pcidevs;
658 return 0;
659 }
661 int libxl_device_pci_shutdown(libxl_ctx *ctx, uint32_t domid)
662 {
663 libxl_device_pci *pcidevs;
664 int num, i, rc;
666 rc = libxl_device_pci_list_assigned(ctx, &pcidevs, domid, &num);
667 if ( rc )
668 return rc;
669 for (i = 0; i < num; i++) {
670 if (libxl_device_pci_remove(ctx, domid, pcidevs + i) < 0)
671 return ERROR_FAIL;
672 }
673 free(pcidevs);
674 return 0;
675 }
677 int libxl_device_pci_init(libxl_device_pci *pcidev, unsigned int domain,
678 unsigned int bus, unsigned int dev,
679 unsigned int func, unsigned int vdevfn)
680 {
681 pcidev->domain = domain;
682 pcidev->bus = bus;
683 pcidev->dev = dev;
684 pcidev->func = func;
685 pcidev->vdevfn = vdevfn;
686 return 0;
687 }
689 int libxl_device_pci_reset(libxl_ctx *ctx, unsigned int domain, unsigned int bus,
690 unsigned int dev, unsigned int func)
691 {
692 char *reset = "/sys/bus/pci/drivers/pciback/do_flr";
693 int fd, rc;
695 fd = open(reset, O_WRONLY);
696 if (fd > 0) {
697 char *buf = libxl_sprintf(ctx, PCI_BDF, domain, bus, dev, func);
698 rc = write(fd, buf, strlen(buf));
699 if (rc < 0)
700 XL_LOG(ctx, XL_LOG_ERROR, "write to %s returned %d", reset, rc);
701 close(fd);
702 return rc < 0 ? rc : 0;
703 }
704 if (errno != ENOENT)
705 XL_LOG_ERRNO(ctx, XL_LOG_ERROR, "Failed to access pciback path %s", reset);
706 reset = libxl_sprintf(ctx, "/sys/bus/pci/devices/"PCI_BDF"/reset", domain, bus, dev, func);
707 fd = open(reset, O_WRONLY);
708 if (fd > 0) {
709 rc = write(fd, "1", 1);
710 if (rc < 0)
711 XL_LOG_ERRNO(ctx, XL_LOG_ERROR, "write to %s returned %d", reset, rc);
712 close(fd);
713 return rc < 0 ? rc : 0;
714 }
715 if (errno == ENOENT) {
716 XL_LOG(ctx, XL_LOG_ERROR, "The kernel doesn't support PCI device reset from sysfs");
717 } else {
718 XL_LOG_ERRNO(ctx, XL_LOG_ERROR, "Failed to access reset path %s", reset);
719 }
720 return -1;
721 }