debuggers.hg

view xen/common/grant_table.c @ 22906:700ac6445812

Now add KDB to the non-kdb tree
author Mukesh Rathor
date Thu Feb 03 15:42:41 2011 -0800 (2011-02-03)
parents 9a9bcf399856
children
line source
1 /******************************************************************************
2 * common/grant_table.c
3 *
4 * Mechanism for granting foreign access to page frames, and receiving
5 * page-ownership transfers.
6 *
7 * Copyright (c) 2005-2006 Christopher Clark
8 * Copyright (c) 2004 K A Fraser
9 * Copyright (c) 2005 Andrew Warfield
10 * Modifications by Geoffrey Lefebvre are (c) Intel Research Cambridge
11 *
12 * This program is free software; you can redistribute it and/or modify
13 * it under the terms of the GNU General Public License as published by
14 * the Free Software Foundation; either version 2 of the License, or
15 * (at your option) any later version.
16 *
17 * This program is distributed in the hope that it will be useful,
18 * but WITHOUT ANY WARRANTY; without even the implied warranty of
19 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
20 * GNU General Public License for more details.
21 *
22 * You should have received a copy of the GNU General Public License
23 * along with this program; if not, write to the Free Software
24 * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
25 */
27 #include <xen/config.h>
28 #include <xen/iocap.h>
29 #include <xen/lib.h>
30 #include <xen/sched.h>
31 #include <xen/mm.h>
32 #include <xen/event.h>
33 #include <xen/trace.h>
34 #include <xen/guest_access.h>
35 #include <xen/domain_page.h>
36 #include <xen/iommu.h>
37 #include <xen/paging.h>
38 #include <xen/keyhandler.h>
39 #include <xsm/xsm.h>
41 #ifndef max_nr_grant_frames
42 unsigned int max_nr_grant_frames = DEFAULT_MAX_NR_GRANT_FRAMES;
43 integer_param("gnttab_max_nr_frames", max_nr_grant_frames);
44 #endif
46 /* The maximum number of grant mappings is defined as a multiplier of the
47 * maximum number of grant table entries. This defines the multiplier used.
48 * Pretty arbitrary. [POLICY]
49 */
50 #define MAX_MAPTRACK_TO_GRANTS_RATIO 8
52 /*
53 * The first two members of a grant entry are updated as a combined pair.
54 * The following union allows that to happen in an endian-neutral fashion.
55 */
56 union grant_combo {
57 uint32_t word;
58 struct {
59 uint16_t flags;
60 domid_t domid;
61 } shorts;
62 };
64 /* Used to share code between unmap_grant_ref and unmap_and_replace. */
65 struct gnttab_unmap_common {
66 /* Input */
67 uint64_t host_addr;
68 uint64_t dev_bus_addr;
69 uint64_t new_addr;
70 grant_handle_t handle;
72 /* Return */
73 int16_t status;
75 /* Shared state beteen *_unmap and *_unmap_complete */
76 u16 flags;
77 unsigned long frame;
78 struct grant_mapping *map;
79 struct domain *rd;
80 };
82 /* Number of unmap operations that are done between each tlb flush */
83 #define GNTTAB_UNMAP_BATCH_SIZE 32
86 #define PIN_FAIL(_lbl, _rc, _f, _a...) \
87 do { \
88 gdprintk(XENLOG_WARNING, _f, ## _a ); \
89 rc = (_rc); \
90 goto _lbl; \
91 } while ( 0 )
93 #define MAPTRACK_PER_PAGE (PAGE_SIZE / sizeof(struct grant_mapping))
94 #define maptrack_entry(t, e) \
95 ((t)->maptrack[(e)/MAPTRACK_PER_PAGE][(e)%MAPTRACK_PER_PAGE])
97 static inline unsigned int
98 nr_maptrack_frames(struct grant_table *t)
99 {
100 return t->maptrack_limit / MAPTRACK_PER_PAGE;
101 }
103 static unsigned inline int max_nr_maptrack_frames(void)
104 {
105 return (max_nr_grant_frames * MAX_MAPTRACK_TO_GRANTS_RATIO);
106 }
108 #ifdef CONFIG_X86
109 #define gfn_to_mfn_private(_d, _gfn) ({ \
110 p2m_type_t __p2mt; \
111 unsigned long __x; \
112 __x = mfn_x(gfn_to_mfn_unshare(p2m_get_hostp2m(_d), _gfn, &__p2mt, 1)); \
113 if ( !p2m_is_valid(__p2mt) ) \
114 __x = INVALID_MFN; \
115 __x; })
116 #else
117 #define gfn_to_mfn_private(_d, _gfn) gmfn_to_mfn(_d, _gfn)
118 #endif
120 #define SHGNT_PER_PAGE_V1 (PAGE_SIZE / sizeof(grant_entry_v1_t))
121 #define shared_entry_v1(t, e) \
122 ((t)->shared_v1[(e)/SHGNT_PER_PAGE_V1][(e)%SHGNT_PER_PAGE_V1])
123 #define SHGNT_PER_PAGE_V2 (PAGE_SIZE / sizeof(grant_entry_v2_t))
124 #define shared_entry_v2(t, e) \
125 ((t)->shared_v2[(e)/SHGNT_PER_PAGE_V2][(e)%SHGNT_PER_PAGE_V2])
126 #define STGNT_PER_PAGE (PAGE_SIZE / sizeof(grant_status_t))
127 #define status_entry(t, e) \
128 ((t)->status[(e)/STGNT_PER_PAGE][(e)%STGNT_PER_PAGE])
129 static grant_entry_header_t *
130 shared_entry_header(struct grant_table *t, grant_ref_t ref)
131 {
132 ASSERT(t->gt_version != 0);
133 if (t->gt_version == 1)
134 return (grant_entry_header_t*)&shared_entry_v1(t, ref);
135 else
136 return &shared_entry_v2(t, ref).hdr;
137 }
138 #define ACGNT_PER_PAGE (PAGE_SIZE / sizeof(struct active_grant_entry))
139 #define active_entry(t, e) \
140 ((t)->active[(e)/ACGNT_PER_PAGE][(e)%ACGNT_PER_PAGE])
142 /* Check if the page has been paged out */
143 static int __get_paged_frame(unsigned long gfn, unsigned long *frame, int readonly, struct domain *rd)
144 {
145 int rc = GNTST_okay;
146 #if defined(P2M_PAGED_TYPES) || defined(P2M_SHARED_TYPES)
147 struct p2m_domain *p2m;
148 p2m_type_t p2mt;
149 mfn_t mfn;
151 p2m = p2m_get_hostp2m(rd);
152 if ( readonly )
153 mfn = gfn_to_mfn(p2m, gfn, &p2mt);
154 else
155 mfn = gfn_to_mfn_unshare(p2m, gfn, &p2mt, 1);
157 if ( p2m_is_valid(p2mt) ) {
158 *frame = mfn_x(mfn);
159 if ( p2m_is_paging(p2mt) )
160 {
161 p2m_mem_paging_populate(p2m, gfn);
162 rc = GNTST_eagain;
163 }
164 } else {
165 *frame = INVALID_MFN;
166 rc = GNTST_bad_page;
167 }
168 #else
169 *frame = readonly ? gmfn_to_mfn(rd, gfn) : gfn_to_mfn_private(rd, gfn);
170 #endif
172 return rc;
173 }
175 static inline int
176 __get_maptrack_handle(
177 struct grant_table *t)
178 {
179 unsigned int h;
180 if ( unlikely((h = t->maptrack_head) == (t->maptrack_limit - 1)) )
181 return -1;
182 t->maptrack_head = maptrack_entry(t, h).ref;
183 return h;
184 }
186 static inline void
187 put_maptrack_handle(
188 struct grant_table *t, int handle)
189 {
190 maptrack_entry(t, handle).ref = t->maptrack_head;
191 t->maptrack_head = handle;
192 }
194 static inline int
195 get_maptrack_handle(
196 struct grant_table *lgt)
197 {
198 int i;
199 grant_handle_t handle;
200 struct grant_mapping *new_mt;
201 unsigned int new_mt_limit, nr_frames;
203 if ( unlikely((handle = __get_maptrack_handle(lgt)) == -1) )
204 {
205 spin_lock(&lgt->lock);
207 if ( unlikely((handle = __get_maptrack_handle(lgt)) == -1) )
208 {
209 nr_frames = nr_maptrack_frames(lgt);
210 if ( nr_frames >= max_nr_maptrack_frames() )
211 {
212 spin_unlock(&lgt->lock);
213 return -1;
214 }
216 new_mt = alloc_xenheap_page();
217 if ( new_mt == NULL )
218 {
219 spin_unlock(&lgt->lock);
220 return -1;
221 }
223 clear_page(new_mt);
225 new_mt_limit = lgt->maptrack_limit + MAPTRACK_PER_PAGE;
227 for ( i = lgt->maptrack_limit; i < new_mt_limit; i++ )
228 {
229 new_mt[i % MAPTRACK_PER_PAGE].ref = i+1;
230 new_mt[i % MAPTRACK_PER_PAGE].flags = 0;
231 }
233 lgt->maptrack[nr_frames] = new_mt;
234 lgt->maptrack_limit = new_mt_limit;
236 gdprintk(XENLOG_INFO,
237 "Increased maptrack size to %u frames.\n", nr_frames + 1);
238 handle = __get_maptrack_handle(lgt);
239 }
241 spin_unlock(&lgt->lock);
242 }
243 return handle;
244 }
246 /* Number of grant table entries. Caller must hold d's grant table lock. */
247 static unsigned int nr_grant_entries(struct grant_table *gt)
248 {
249 ASSERT(gt->gt_version != 0);
250 if (gt->gt_version == 1)
251 return (nr_grant_frames(gt) << PAGE_SHIFT) / sizeof(grant_entry_v1_t);
252 else
253 return (nr_grant_frames(gt) << PAGE_SHIFT) / sizeof(grant_entry_v2_t);
254 }
256 static int _set_status_v1(domid_t domid,
257 int readonly,
258 int mapflag,
259 grant_entry_header_t *shah,
260 struct active_grant_entry *act)
261 {
262 int rc = GNTST_okay;
263 union grant_combo scombo, prev_scombo, new_scombo;
264 uint16_t mask = GTF_type_mask;
266 /*
267 * We bound the number of times we retry CMPXCHG on memory locations that
268 * we share with a guest OS. The reason is that the guest can modify that
269 * location at a higher rate than we can read-modify-CMPXCHG, so the guest
270 * could cause us to livelock. There are a few cases where it is valid for
271 * the guest to race our updates (e.g., to change the GTF_readonly flag),
272 * so we allow a few retries before failing.
273 */
274 int retries = 0;
276 /* if this is a grant mapping operation we should ensure GTF_sub_page
277 is not set */
278 if (mapflag)
279 mask |= GTF_sub_page;
281 scombo.word = *(u32 *)shah;
283 /*
284 * This loop attempts to set the access (reading/writing) flags
285 * in the grant table entry. It tries a cmpxchg on the field
286 * up to five times, and then fails under the assumption that
287 * the guest is misbehaving.
288 */
289 for ( ; ; )
290 {
291 /* If not already pinned, check the grant domid and type. */
292 if ( !act->pin &&
293 (((scombo.shorts.flags & mask) !=
294 GTF_permit_access) ||
295 (scombo.shorts.domid != domid)) )
296 PIN_FAIL(done, GNTST_general_error,
297 "Bad flags (%x) or dom (%d). (expected dom %d)\n",
298 scombo.shorts.flags, scombo.shorts.domid,
299 domid);
301 new_scombo = scombo;
302 new_scombo.shorts.flags |= GTF_reading;
304 if ( !readonly )
305 {
306 new_scombo.shorts.flags |= GTF_writing;
307 if ( unlikely(scombo.shorts.flags & GTF_readonly) )
308 PIN_FAIL(done, GNTST_general_error,
309 "Attempt to write-pin a r/o grant entry.\n");
310 }
312 prev_scombo.word = cmpxchg((u32 *)shah,
313 scombo.word, new_scombo.word);
314 if ( likely(prev_scombo.word == scombo.word) )
315 break;
317 if ( retries++ == 4 )
318 PIN_FAIL(done, GNTST_general_error,
319 "Shared grant entry is unstable.\n");
321 scombo = prev_scombo;
322 }
324 done:
325 return rc;
326 }
328 static int _set_status_v2(domid_t domid,
329 int readonly,
330 int mapflag,
331 grant_entry_header_t *shah,
332 struct active_grant_entry *act,
333 grant_status_t *status)
334 {
335 int rc = GNTST_okay;
336 union grant_combo scombo;
337 uint16_t flags = shah->flags;
338 domid_t id = shah->domid;
339 uint16_t mask = GTF_type_mask;
341 /* we read flags and domid in a single memory access.
342 this avoids the need for another memory barrier to
343 ensure access to these fields are not reordered */
344 scombo.word = *(u32 *)shah;
345 barrier(); /* but we still need to stop the compiler from turning
346 it back into two reads */
347 flags = scombo.shorts.flags;
348 id = scombo.shorts.domid;
350 /* if this is a grant mapping operation we should ensure GTF_sub_page
351 is not set */
352 if (mapflag)
353 mask |= GTF_sub_page;
355 /* If not already pinned, check the grant domid and type. */
356 if ( !act->pin &&
357 ( (((flags & mask) != GTF_permit_access) &&
358 ((flags & mask) != GTF_transitive)) ||
359 (id != domid)) )
360 PIN_FAIL(done, GNTST_general_error,
361 "Bad flags (%x) or dom (%d). (expected dom %d, flags %x)\n",
362 flags, id, domid, mask);
364 if ( readonly )
365 {
366 *status |= GTF_reading;
367 }
368 else
369 {
370 if ( unlikely(flags & GTF_readonly) )
371 PIN_FAIL(done, GNTST_general_error,
372 "Attempt to write-pin a r/o grant entry.\n");
373 *status |= GTF_reading | GTF_writing;
374 }
376 /* Make sure guest sees status update before checking if flags are
377 still valid */
378 mb();
380 scombo.word = *(u32 *)shah;
381 barrier();
382 flags = scombo.shorts.flags;
383 id = scombo.shorts.domid;
385 if ( !act->pin )
386 {
387 if ( (((flags & mask) != GTF_permit_access) &&
388 ((flags & mask) != GTF_transitive)) ||
389 (id != domid) ||
390 (!readonly && (flags & GTF_readonly)) )
391 {
392 gnttab_clear_flag(_GTF_reading | _GTF_writing, status);
393 PIN_FAIL(done, GNTST_general_error,
394 "Unstable flags (%x) or dom (%d). (expected dom %d) "
395 "(r/w: %d)\n",
396 flags, id, domid, !readonly);
397 }
398 }
399 else
400 {
401 if ( unlikely(flags & GTF_readonly) )
402 {
403 gnttab_clear_flag(_GTF_writing, status);
404 PIN_FAIL(done, GNTST_general_error,
405 "Unstable grant readonly flag\n");
406 }
407 }
409 done:
410 return rc;
411 }
414 static int _set_status(unsigned gt_version,
415 domid_t domid,
416 int readonly,
417 int mapflag,
418 grant_entry_header_t *shah,
419 struct active_grant_entry *act,
420 grant_status_t *status)
421 {
423 if (gt_version == 1)
424 return _set_status_v1(domid, readonly, mapflag, shah, act);
425 else
426 return _set_status_v2(domid, readonly, mapflag, shah, act, status);
427 }
429 static void mapcount(
430 struct domain *ld, unsigned long mfn,
431 unsigned int *wrc, unsigned int *rdc)
432 {
433 struct grant_table *gt = ld->grant_table;
434 struct grant_mapping *map;
435 grant_handle_t handle;
436 struct domain *rd;
438 *wrc = *rdc = 0;
440 for ( handle = 0; handle < gt->maptrack_limit; handle++ )
441 {
442 map = &maptrack_entry(gt, handle);
443 if ( !(map->flags & (GNTMAP_device_map|GNTMAP_host_map)) )
444 continue;
445 rd = rcu_lock_domain_by_id(map->domid);
446 if ( active_entry(rd->grant_table, map->ref).frame == mfn )
447 (map->flags & GNTMAP_readonly) ? (*rdc)++ : (*wrc)++;
448 rcu_unlock_domain(rd);
449 }
450 }
452 /*
453 * Returns 0 if TLB flush / invalidate required by caller.
454 * va will indicate the address to be invalidated.
455 *
456 * addr is _either_ a host virtual address, or the address of the pte to
457 * update, as indicated by the GNTMAP_contains_pte flag.
458 */
459 static void
460 __gnttab_map_grant_ref(
461 struct gnttab_map_grant_ref *op)
462 {
463 struct domain *ld, *rd, *owner;
464 struct vcpu *led;
465 int handle;
466 unsigned long frame = 0, nr_gets = 0;
467 struct page_info *pg;
468 int rc = GNTST_okay;
469 u32 old_pin;
470 u32 act_pin;
471 unsigned int cache_flags;
472 struct active_grant_entry *act;
473 struct grant_mapping *mt;
474 grant_entry_v1_t *sha1;
475 grant_entry_v2_t *sha2;
476 grant_entry_header_t *shah;
477 uint16_t *status;
479 led = current;
480 ld = led->domain;
482 if ( unlikely((op->flags & (GNTMAP_device_map|GNTMAP_host_map)) == 0) )
483 {
484 gdprintk(XENLOG_INFO, "Bad flags in grant map op (%x).\n", op->flags);
485 op->status = GNTST_bad_gntref;
486 return;
487 }
489 if ( unlikely(paging_mode_external(ld) &&
490 (op->flags & (GNTMAP_device_map|GNTMAP_application_map|
491 GNTMAP_contains_pte))) )
492 {
493 gdprintk(XENLOG_INFO, "No device mapping in HVM domain.\n");
494 op->status = GNTST_general_error;
495 return;
496 }
498 if ( unlikely((rd = rcu_lock_domain_by_id(op->dom)) == NULL) )
499 {
500 gdprintk(XENLOG_INFO, "Could not find domain %d\n", op->dom);
501 op->status = GNTST_bad_domain;
502 return;
503 }
505 rc = xsm_grant_mapref(ld, rd, op->flags);
506 if ( rc )
507 {
508 rcu_unlock_domain(rd);
509 op->status = GNTST_permission_denied;
510 return;
511 }
513 if ( unlikely((handle = get_maptrack_handle(ld->grant_table)) == -1) )
514 {
515 rcu_unlock_domain(rd);
516 gdprintk(XENLOG_INFO, "Failed to obtain maptrack handle.\n");
517 op->status = GNTST_no_device_space;
518 return;
519 }
521 spin_lock(&rd->grant_table->lock);
523 if ( rd->grant_table->gt_version == 0 )
524 PIN_FAIL(unlock_out, GNTST_general_error,
525 "remote grant table not yet set up");
527 /* Bounds check on the grant ref */
528 if ( unlikely(op->ref >= nr_grant_entries(rd->grant_table)))
529 PIN_FAIL(unlock_out, GNTST_bad_gntref, "Bad ref (%d).\n", op->ref);
531 act = &active_entry(rd->grant_table, op->ref);
532 shah = shared_entry_header(rd->grant_table, op->ref);
533 if (rd->grant_table->gt_version == 1) {
534 sha1 = &shared_entry_v1(rd->grant_table, op->ref);
535 sha2 = NULL;
536 status = &shah->flags;
537 } else {
538 sha2 = &shared_entry_v2(rd->grant_table, op->ref);
539 sha1 = NULL;
540 status = &status_entry(rd->grant_table, op->ref);
541 }
543 /* If already pinned, check the active domid and avoid refcnt overflow. */
544 if ( act->pin &&
545 ((act->domid != ld->domain_id) ||
546 (act->pin & 0x80808080U) != 0 ||
547 (act->is_sub_page)) )
548 PIN_FAIL(unlock_out, GNTST_general_error,
549 "Bad domain (%d != %d), or risk of counter overflow %08x, or subpage %d\n",
550 act->domid, ld->domain_id, act->pin, act->is_sub_page);
552 if ( !act->pin ||
553 (!(op->flags & GNTMAP_readonly) &&
554 !(act->pin & (GNTPIN_hstw_mask|GNTPIN_devw_mask))) )
555 {
556 if ( (rc = _set_status(rd->grant_table->gt_version,
557 ld->domain_id, op->flags & GNTMAP_readonly,
558 1, shah, act, status) ) != GNTST_okay )
559 goto unlock_out;
561 if ( !act->pin )
562 {
563 unsigned long gfn;
564 unsigned long frame;
566 gfn = sha1 ? sha1->frame : sha2->full_page.frame;
567 rc = __get_paged_frame(gfn, &frame, !!(op->flags & GNTMAP_readonly), rd);
568 if ( rc != GNTST_okay )
569 goto unlock_out;
570 act->gfn = gfn;
571 act->domid = ld->domain_id;
572 act->frame = frame;
573 act->start = 0;
574 act->length = PAGE_SIZE;
575 act->is_sub_page = 0;
576 }
577 }
579 old_pin = act->pin;
580 if ( op->flags & GNTMAP_device_map )
581 act->pin += (op->flags & GNTMAP_readonly) ?
582 GNTPIN_devr_inc : GNTPIN_devw_inc;
583 if ( op->flags & GNTMAP_host_map )
584 act->pin += (op->flags & GNTMAP_readonly) ?
585 GNTPIN_hstr_inc : GNTPIN_hstw_inc;
587 frame = act->frame;
588 act_pin = act->pin;
590 cache_flags = (shah->flags & (GTF_PAT | GTF_PWT | GTF_PCD) );
592 spin_unlock(&rd->grant_table->lock);
594 pg = mfn_valid(frame) ? mfn_to_page(frame) : NULL;
596 if ( !pg || (owner = page_get_owner_and_reference(pg)) == dom_io )
597 {
598 /* Only needed the reference to confirm dom_io ownership. */
599 if ( pg )
600 put_page(pg);
602 if ( paging_mode_external(ld) )
603 {
604 gdprintk(XENLOG_WARNING, "HVM guests can't grant map iomem\n");
605 rc = GNTST_general_error;
606 goto undo_out;
607 }
609 if ( !iomem_access_permitted(rd, frame, frame) )
610 {
611 gdprintk(XENLOG_WARNING,
612 "Iomem mapping not permitted %lx (domain %d)\n",
613 frame, rd->domain_id);
614 rc = GNTST_general_error;
615 goto undo_out;
616 }
618 rc = create_grant_host_mapping(
619 op->host_addr, frame, op->flags, cache_flags);
620 if ( rc != GNTST_okay )
621 goto undo_out;
622 }
623 else if ( owner == rd || owner == dom_cow )
624 {
625 if ( gnttab_host_mapping_get_page_type(op, ld, rd) &&
626 !get_page_type(pg, PGT_writable_page) )
627 goto could_not_pin;
629 nr_gets++;
630 if ( op->flags & GNTMAP_host_map )
631 {
632 rc = create_grant_host_mapping(op->host_addr, frame, op->flags, 0);
633 if ( rc != GNTST_okay )
634 goto undo_out;
636 if ( op->flags & GNTMAP_device_map )
637 {
638 nr_gets++;
639 (void)get_page(pg, rd);
640 if ( !(op->flags & GNTMAP_readonly) )
641 get_page_type(pg, PGT_writable_page);
642 }
643 }
644 }
645 else
646 {
647 could_not_pin:
648 if ( !rd->is_dying )
649 gdprintk(XENLOG_WARNING, "Could not pin grant frame %lx\n",
650 frame);
651 if ( owner != NULL )
652 put_page(pg);
653 rc = GNTST_general_error;
654 goto undo_out;
655 }
657 if ( !is_hvm_domain(ld) && need_iommu(ld) )
658 {
659 unsigned int wrc, rdc;
660 int err = 0;
661 /* Shouldn't happen, because you can't use iommu in a HVM domain. */
662 BUG_ON(paging_mode_translate(ld));
663 /* We're not translated, so we know that gmfns and mfns are
664 the same things, so the IOMMU entry is always 1-to-1. */
665 mapcount(ld, frame, &wrc, &rdc);
666 if ( (act_pin & (GNTPIN_hstw_mask|GNTPIN_devw_mask)) &&
667 !(old_pin & (GNTPIN_hstw_mask|GNTPIN_devw_mask)) )
668 {
669 if ( wrc == 0 )
670 err = iommu_map_page(ld, frame, frame,
671 IOMMUF_readable|IOMMUF_writable);
672 }
673 else if ( act_pin && !old_pin )
674 {
675 if ( (wrc + rdc) == 0 )
676 err = iommu_map_page(ld, frame, frame, IOMMUF_readable);
677 }
678 if ( err )
679 {
680 rc = GNTST_general_error;
681 goto undo_out;
682 }
683 }
685 TRACE_1D(TRC_MEM_PAGE_GRANT_MAP, op->dom);
687 mt = &maptrack_entry(ld->grant_table, handle);
688 mt->domid = op->dom;
689 mt->ref = op->ref;
690 mt->flags = op->flags;
692 op->dev_bus_addr = (u64)frame << PAGE_SHIFT;
693 op->handle = handle;
694 op->status = GNTST_okay;
696 rcu_unlock_domain(rd);
697 return;
699 undo_out:
700 if ( nr_gets > 1 )
701 {
702 if ( !(op->flags & GNTMAP_readonly) )
703 put_page_type(pg);
704 put_page(pg);
705 }
706 if ( nr_gets > 0 )
707 {
708 if ( gnttab_host_mapping_get_page_type(op, ld, rd) )
709 put_page_type(pg);
710 put_page(pg);
711 }
713 spin_lock(&rd->grant_table->lock);
715 act = &active_entry(rd->grant_table, op->ref);
716 shah = shared_entry_header(rd->grant_table, op->ref);
718 if ( op->flags & GNTMAP_device_map )
719 act->pin -= (op->flags & GNTMAP_readonly) ?
720 GNTPIN_devr_inc : GNTPIN_devw_inc;
721 if ( op->flags & GNTMAP_host_map )
722 act->pin -= (op->flags & GNTMAP_readonly) ?
723 GNTPIN_hstr_inc : GNTPIN_hstw_inc;
725 if ( !(op->flags & GNTMAP_readonly) &&
726 !(act->pin & (GNTPIN_hstw_mask|GNTPIN_devw_mask)) )
727 gnttab_clear_flag(_GTF_writing, status);
729 if ( !act->pin )
730 gnttab_clear_flag(_GTF_reading, status);
732 unlock_out:
733 spin_unlock(&rd->grant_table->lock);
734 op->status = rc;
735 put_maptrack_handle(ld->grant_table, handle);
736 rcu_unlock_domain(rd);
737 }
739 static long
740 gnttab_map_grant_ref(
741 XEN_GUEST_HANDLE(gnttab_map_grant_ref_t) uop, unsigned int count)
742 {
743 int i;
744 struct gnttab_map_grant_ref op;
746 for ( i = 0; i < count; i++ )
747 {
748 if (i && hypercall_preempt_check())
749 return i;
750 if ( unlikely(__copy_from_guest_offset(&op, uop, i, 1)) )
751 return -EFAULT;
752 __gnttab_map_grant_ref(&op);
753 if ( unlikely(__copy_to_guest_offset(uop, i, &op, 1)) )
754 return -EFAULT;
755 }
757 return 0;
758 }
760 static void
761 __gnttab_unmap_common(
762 struct gnttab_unmap_common *op)
763 {
764 domid_t dom;
765 struct domain *ld, *rd;
766 struct active_grant_entry *act;
767 s16 rc = 0;
768 u32 old_pin;
770 ld = current->domain;
772 op->frame = (unsigned long)(op->dev_bus_addr >> PAGE_SHIFT);
774 if ( unlikely(op->handle >= ld->grant_table->maptrack_limit) )
775 {
776 gdprintk(XENLOG_INFO, "Bad handle (%d).\n", op->handle);
777 op->status = GNTST_bad_handle;
778 return;
779 }
781 op->map = &maptrack_entry(ld->grant_table, op->handle);
783 if ( unlikely(!op->map->flags) )
784 {
785 gdprintk(XENLOG_INFO, "Zero flags for handle (%d).\n", op->handle);
786 op->status = GNTST_bad_handle;
787 return;
788 }
790 dom = op->map->domid;
791 op->flags = op->map->flags;
793 if ( unlikely((op->rd = rd = rcu_lock_domain_by_id(dom)) == NULL) )
794 {
795 /* This can happen when a grant is implicitly unmapped. */
796 gdprintk(XENLOG_INFO, "Could not find domain %d\n", dom);
797 domain_crash(ld); /* naughty... */
798 return;
799 }
801 rc = xsm_grant_unmapref(ld, rd);
802 if ( rc )
803 {
804 rcu_unlock_domain(rd);
805 op->status = GNTST_permission_denied;
806 return;
807 }
809 TRACE_1D(TRC_MEM_PAGE_GRANT_UNMAP, dom);
811 spin_lock(&rd->grant_table->lock);
813 act = &active_entry(rd->grant_table, op->map->ref);
814 old_pin = act->pin;
816 if ( op->frame == 0 )
817 {
818 op->frame = act->frame;
819 }
820 else
821 {
822 if ( unlikely(op->frame != act->frame) )
823 PIN_FAIL(unmap_out, GNTST_general_error,
824 "Bad frame number doesn't match gntref. (%lx != %lx)\n",
825 op->frame, act->frame);
826 if ( op->flags & GNTMAP_device_map )
827 {
828 ASSERT(act->pin & (GNTPIN_devw_mask | GNTPIN_devr_mask));
829 op->map->flags &= ~GNTMAP_device_map;
830 if ( op->flags & GNTMAP_readonly )
831 act->pin -= GNTPIN_devr_inc;
832 else
833 act->pin -= GNTPIN_devw_inc;
834 }
835 }
837 if ( (op->host_addr != 0) && (op->flags & GNTMAP_host_map) )
838 {
839 if ( (rc = replace_grant_host_mapping(op->host_addr,
840 op->frame, op->new_addr,
841 op->flags)) < 0 )
842 goto unmap_out;
844 ASSERT(act->pin & (GNTPIN_hstw_mask | GNTPIN_hstr_mask));
845 op->map->flags &= ~GNTMAP_host_map;
846 if ( op->flags & GNTMAP_readonly )
847 act->pin -= GNTPIN_hstr_inc;
848 else
849 act->pin -= GNTPIN_hstw_inc;
850 }
852 if ( !is_hvm_domain(ld) && need_iommu(ld) )
853 {
854 unsigned int wrc, rdc;
855 int err = 0;
856 BUG_ON(paging_mode_translate(ld));
857 mapcount(ld, op->frame, &wrc, &rdc);
858 if ( (wrc + rdc) == 0 )
859 err = iommu_unmap_page(ld, op->frame);
860 else if ( wrc == 0 )
861 err = iommu_map_page(ld, op->frame, op->frame, IOMMUF_readable);
862 if ( err )
863 {
864 rc = GNTST_general_error;
865 goto unmap_out;
866 }
867 }
869 /* If just unmapped a writable mapping, mark as dirtied */
870 if ( !(op->flags & GNTMAP_readonly) )
871 gnttab_mark_dirty(rd, op->frame);
873 unmap_out:
874 op->status = rc;
875 spin_unlock(&rd->grant_table->lock);
876 rcu_unlock_domain(rd);
877 }
879 static void
880 __gnttab_unmap_common_complete(struct gnttab_unmap_common *op)
881 {
882 struct domain *ld, *rd;
883 struct active_grant_entry *act;
884 grant_entry_header_t *sha;
885 struct page_info *pg;
886 uint16_t *status;
888 rd = op->rd;
890 if ( rd == NULL )
891 {
892 /*
893 * Suggests that __gntab_unmap_common failed in
894 * rcu_lock_domain_by_id() or earlier, and so we have nothing
895 * to complete
896 */
897 return;
898 }
900 ld = current->domain;
902 rcu_lock_domain(rd);
903 spin_lock(&rd->grant_table->lock);
905 if ( rd->grant_table->gt_version == 0 )
906 goto unmap_out;
908 act = &active_entry(rd->grant_table, op->map->ref);
909 sha = shared_entry_header(rd->grant_table, op->map->ref);
911 if ( rd->grant_table->gt_version == 1 )
912 status = &sha->flags;
913 else
914 status = &status_entry(rd->grant_table, op->map->ref);
916 if ( unlikely(op->frame != act->frame) )
917 {
918 /*
919 * Suggests that __gntab_unmap_common failed early and so
920 * nothing further to do
921 */
922 goto unmap_out;
923 }
925 pg = mfn_to_page(op->frame);
927 if ( op->flags & GNTMAP_device_map )
928 {
929 if ( !is_iomem_page(act->frame) )
930 {
931 if ( op->flags & GNTMAP_readonly )
932 put_page(pg);
933 else
934 put_page_and_type(pg);
935 }
936 }
938 if ( (op->host_addr != 0) && (op->flags & GNTMAP_host_map) )
939 {
940 if ( op->status != 0 )
941 {
942 /*
943 * Suggests that __gntab_unmap_common failed in
944 * replace_grant_host_mapping() so nothing further to do
945 */
946 goto unmap_out;
947 }
949 if ( !is_iomem_page(op->frame) )
950 {
951 if ( gnttab_host_mapping_get_page_type(op, ld, rd) )
952 put_page_type(pg);
953 put_page(pg);
954 }
955 }
957 if ( (op->map->flags & (GNTMAP_device_map|GNTMAP_host_map)) == 0 )
958 {
959 op->map->flags = 0;
960 put_maptrack_handle(ld->grant_table, op->handle);
961 }
963 if ( ((act->pin & (GNTPIN_devw_mask|GNTPIN_hstw_mask)) == 0) &&
964 !(op->flags & GNTMAP_readonly) )
965 gnttab_clear_flag(_GTF_writing, status);
967 if ( act->pin == 0 )
968 gnttab_clear_flag(_GTF_reading, status);
970 unmap_out:
971 spin_unlock(&rd->grant_table->lock);
972 rcu_unlock_domain(rd);
973 }
975 static void
976 __gnttab_unmap_grant_ref(
977 struct gnttab_unmap_grant_ref *op,
978 struct gnttab_unmap_common *common)
979 {
980 common->host_addr = op->host_addr;
981 common->dev_bus_addr = op->dev_bus_addr;
982 common->handle = op->handle;
984 /* Intialise these in case common contains old state */
985 common->new_addr = 0;
986 common->rd = NULL;
988 __gnttab_unmap_common(common);
989 op->status = common->status;
990 }
993 static long
994 gnttab_unmap_grant_ref(
995 XEN_GUEST_HANDLE(gnttab_unmap_grant_ref_t) uop, unsigned int count)
996 {
997 int i, c, partial_done, done = 0;
998 struct gnttab_unmap_grant_ref op;
999 struct gnttab_unmap_common common[GNTTAB_UNMAP_BATCH_SIZE];
1001 while ( count != 0 )
1003 c = min(count, (unsigned int)GNTTAB_UNMAP_BATCH_SIZE);
1004 partial_done = 0;
1006 for ( i = 0; i < c; i++ )
1008 if ( unlikely(__copy_from_guest_offset(&op, uop, done+i, 1)) )
1009 goto fault;
1010 __gnttab_unmap_grant_ref(&op, &(common[i]));
1011 ++partial_done;
1012 if ( unlikely(__copy_to_guest_offset(uop, done+i, &op, 1)) )
1013 goto fault;
1016 flush_tlb_mask(&current->domain->domain_dirty_cpumask);
1018 for ( i = 0; i < partial_done; i++ )
1019 __gnttab_unmap_common_complete(&(common[i]));
1021 count -= c;
1022 done += c;
1024 if (count && hypercall_preempt_check())
1025 return done;
1028 return 0;
1030 fault:
1031 flush_tlb_mask(&current->domain->domain_dirty_cpumask);
1033 for ( i = 0; i < partial_done; i++ )
1034 __gnttab_unmap_common_complete(&(common[i]));
1035 return -EFAULT;
1038 static void
1039 __gnttab_unmap_and_replace(
1040 struct gnttab_unmap_and_replace *op,
1041 struct gnttab_unmap_common *common)
1043 common->host_addr = op->host_addr;
1044 common->new_addr = op->new_addr;
1045 common->handle = op->handle;
1047 /* Intialise these in case common contains old state */
1048 common->dev_bus_addr = 0;
1049 common->rd = NULL;
1051 __gnttab_unmap_common(common);
1052 op->status = common->status;
1055 static long
1056 gnttab_unmap_and_replace(
1057 XEN_GUEST_HANDLE(gnttab_unmap_and_replace_t) uop, unsigned int count)
1059 int i, c, partial_done, done = 0;
1060 struct gnttab_unmap_and_replace op;
1061 struct gnttab_unmap_common common[GNTTAB_UNMAP_BATCH_SIZE];
1063 while ( count != 0 )
1065 c = min(count, (unsigned int)GNTTAB_UNMAP_BATCH_SIZE);
1066 partial_done = 0;
1068 for ( i = 0; i < c; i++ )
1070 if ( unlikely(__copy_from_guest_offset(&op, uop, done+i, 1)) )
1071 goto fault;
1072 __gnttab_unmap_and_replace(&op, &(common[i]));
1073 ++partial_done;
1074 if ( unlikely(__copy_to_guest_offset(uop, done+i, &op, 1)) )
1075 goto fault;
1078 flush_tlb_mask(&current->domain->domain_dirty_cpumask);
1080 for ( i = 0; i < partial_done; i++ )
1081 __gnttab_unmap_common_complete(&(common[i]));
1083 count -= c;
1084 done += c;
1086 if (count && hypercall_preempt_check())
1087 return done;
1090 return 0;
1092 fault:
1093 flush_tlb_mask(&current->domain->domain_dirty_cpumask);
1095 for ( i = 0; i < partial_done; i++ )
1096 __gnttab_unmap_common_complete(&(common[i]));
1097 return -EFAULT;
1100 static int
1101 gnttab_populate_status_frames(struct domain *d, struct grant_table *gt)
1103 unsigned i;
1104 unsigned req_status_frames;
1106 req_status_frames = grant_to_status_frames(gt->nr_grant_frames);
1107 for ( i = nr_status_frames(gt); i < req_status_frames; i++ )
1109 if ( (gt->status[i] = alloc_xenheap_page()) == NULL )
1110 goto status_alloc_failed;
1111 clear_page(gt->status[i]);
1113 /* Share the new status frames with the recipient domain */
1114 for ( i = nr_status_frames(gt); i < req_status_frames; i++ )
1115 gnttab_create_status_page(d, gt, i);
1117 gt->nr_status_frames = req_status_frames;
1119 return 0;
1121 status_alloc_failed:
1122 for ( i = nr_status_frames(gt); i < req_status_frames; i++ )
1124 free_xenheap_page(gt->status[i]);
1125 gt->status[i] = NULL;
1127 return -ENOMEM;
1130 static void
1131 gnttab_unpopulate_status_frames(struct domain *d, struct grant_table *gt)
1133 int i;
1135 for ( i = 0; i < nr_status_frames(gt); i++ )
1137 page_set_owner(virt_to_page(gt->status[i]), dom_xen);
1138 free_xenheap_page(gt->status[i]);
1139 gt->status[i] = NULL;
1141 gt->nr_status_frames = 0;
1144 int
1145 gnttab_grow_table(struct domain *d, unsigned int req_nr_frames)
1147 /* d's grant table lock must be held by the caller */
1149 struct grant_table *gt = d->grant_table;
1150 unsigned int i;
1152 ASSERT(req_nr_frames <= max_nr_grant_frames);
1154 gdprintk(XENLOG_INFO,
1155 "Expanding dom (%d) grant table from (%d) to (%d) frames.\n",
1156 d->domain_id, nr_grant_frames(gt), req_nr_frames);
1158 /* Active */
1159 for ( i = nr_active_grant_frames(gt);
1160 i < num_act_frames_from_sha_frames(req_nr_frames); i++ )
1162 if ( (gt->active[i] = alloc_xenheap_page()) == NULL )
1163 goto active_alloc_failed;
1164 clear_page(gt->active[i]);
1167 /* Shared */
1168 for ( i = nr_grant_frames(gt); i < req_nr_frames; i++ )
1170 if ( (gt->shared_raw[i] = alloc_xenheap_page()) == NULL )
1171 goto shared_alloc_failed;
1172 clear_page(gt->shared_raw[i]);
1175 /* Share the new shared frames with the recipient domain */
1176 for ( i = nr_grant_frames(gt); i < req_nr_frames; i++ )
1177 gnttab_create_shared_page(d, gt, i);
1179 gt->nr_grant_frames = req_nr_frames;
1181 /* Status pages - version 2 */
1182 if (gt->gt_version > 1)
1184 if ( gnttab_populate_status_frames(d, gt) )
1185 goto shared_alloc_failed;
1188 return 1;
1190 shared_alloc_failed:
1191 for ( i = nr_grant_frames(gt); i < req_nr_frames; i++ )
1193 free_xenheap_page(gt->shared_raw[i]);
1194 gt->shared_raw[i] = NULL;
1196 active_alloc_failed:
1197 for ( i = nr_active_grant_frames(gt);
1198 i < num_act_frames_from_sha_frames(req_nr_frames); i++ )
1200 free_xenheap_page(gt->active[i]);
1201 gt->active[i] = NULL;
1203 gdprintk(XENLOG_INFO, "Allocation failure when expanding grant table.\n");
1204 return 0;
1207 static long
1208 gnttab_setup_table(
1209 XEN_GUEST_HANDLE(gnttab_setup_table_t) uop, unsigned int count)
1211 struct gnttab_setup_table op;
1212 struct domain *d;
1213 int i;
1214 unsigned long gmfn;
1215 domid_t dom;
1217 if ( count != 1 )
1218 return -EINVAL;
1220 if ( unlikely(copy_from_guest(&op, uop, 1) != 0) )
1222 gdprintk(XENLOG_INFO, "Fault while reading gnttab_setup_table_t.\n");
1223 return -EFAULT;
1226 if ( unlikely(op.nr_frames > max_nr_grant_frames) )
1228 gdprintk(XENLOG_INFO, "Xen only supports up to %d grant-table frames"
1229 " per domain.\n",
1230 max_nr_grant_frames);
1231 op.status = GNTST_general_error;
1232 goto out1;
1235 dom = op.dom;
1236 if ( dom == DOMID_SELF )
1238 d = rcu_lock_current_domain();
1240 else
1242 if ( unlikely((d = rcu_lock_domain_by_id(dom)) == NULL) )
1244 gdprintk(XENLOG_INFO, "Bad domid %d.\n", dom);
1245 op.status = GNTST_bad_domain;
1246 goto out1;
1249 if ( unlikely(!IS_PRIV_FOR(current->domain, d)) )
1251 op.status = GNTST_permission_denied;
1252 goto out2;
1256 if ( xsm_grant_setup(current->domain, d) )
1258 op.status = GNTST_permission_denied;
1259 goto out2;
1262 spin_lock(&d->grant_table->lock);
1264 if ( d->grant_table->gt_version == 0 )
1265 d->grant_table->gt_version = 1;
1267 if ( (op.nr_frames > nr_grant_frames(d->grant_table) ||
1268 ( (d->grant_table->gt_version > 1 ) &&
1269 (grant_to_status_frames(op.nr_frames) >
1270 nr_status_frames(d->grant_table)) ) ) &&
1271 !gnttab_grow_table(d, op.nr_frames) )
1273 gdprintk(XENLOG_INFO,
1274 "Expand grant table to %d failed. Current: %d Max: %d.\n",
1275 op.nr_frames,
1276 nr_grant_frames(d->grant_table),
1277 max_nr_grant_frames);
1278 op.status = GNTST_general_error;
1279 goto out3;
1282 op.status = GNTST_okay;
1283 for ( i = 0; i < op.nr_frames; i++ )
1285 gmfn = gnttab_shared_gmfn(d, d->grant_table, i);
1286 /* Grant tables cannot be shared */
1287 BUG_ON(SHARED_M2P(gmfn));
1288 (void)copy_to_guest_offset(op.frame_list, i, &gmfn, 1);
1291 out3:
1292 spin_unlock(&d->grant_table->lock);
1293 out2:
1294 rcu_unlock_domain(d);
1295 out1:
1296 if ( unlikely(copy_to_guest(uop, &op, 1)) )
1297 return -EFAULT;
1299 return 0;
1302 static long
1303 gnttab_query_size(
1304 XEN_GUEST_HANDLE(gnttab_query_size_t) uop, unsigned int count)
1306 struct gnttab_query_size op;
1307 struct domain *d;
1308 domid_t dom;
1309 int rc;
1311 if ( count != 1 )
1312 return -EINVAL;
1314 if ( unlikely(copy_from_guest(&op, uop, 1) != 0) )
1316 gdprintk(XENLOG_INFO, "Fault while reading gnttab_query_size_t.\n");
1317 return -EFAULT;
1320 dom = op.dom;
1321 if ( dom == DOMID_SELF )
1323 d = rcu_lock_current_domain();
1325 else
1327 if ( unlikely((d = rcu_lock_domain_by_id(dom)) == NULL) )
1329 gdprintk(XENLOG_INFO, "Bad domid %d.\n", dom);
1330 op.status = GNTST_bad_domain;
1331 goto query_out;
1334 if ( unlikely(!IS_PRIV_FOR(current->domain, d)) )
1336 op.status = GNTST_permission_denied;
1337 goto query_out_unlock;
1341 rc = xsm_grant_query_size(current->domain, d);
1342 if ( rc )
1344 op.status = GNTST_permission_denied;
1345 goto query_out_unlock;
1348 spin_lock(&d->grant_table->lock);
1350 op.nr_frames = nr_grant_frames(d->grant_table);
1351 op.max_nr_frames = max_nr_grant_frames;
1352 op.status = GNTST_okay;
1354 spin_unlock(&d->grant_table->lock);
1357 query_out_unlock:
1358 rcu_unlock_domain(d);
1360 query_out:
1361 if ( unlikely(copy_to_guest(uop, &op, 1)) )
1362 return -EFAULT;
1364 return 0;
1367 /*
1368 * Check that the given grant reference (rd,ref) allows 'ld' to transfer
1369 * ownership of a page frame. If so, lock down the grant entry.
1370 */
1371 static int
1372 gnttab_prepare_for_transfer(
1373 struct domain *rd, struct domain *ld, grant_ref_t ref)
1375 struct grant_table *rgt;
1376 grant_entry_header_t *sha;
1377 union grant_combo scombo, prev_scombo, new_scombo;
1378 int retries = 0;
1380 if ( unlikely((rgt = rd->grant_table) == NULL) )
1382 gdprintk(XENLOG_INFO, "Dom %d has no grant table.\n", rd->domain_id);
1383 return 0;
1386 spin_lock(&rgt->lock);
1388 if ( rgt->gt_version == 0 )
1390 gdprintk(XENLOG_INFO,
1391 "Grant table not ready for transfer to domain(%d).\n",
1392 rd->domain_id);
1393 goto fail;
1396 if ( unlikely(ref >= nr_grant_entries(rd->grant_table)) )
1398 gdprintk(XENLOG_INFO,
1399 "Bad grant reference (%d) for transfer to domain(%d).\n",
1400 ref, rd->domain_id);
1401 goto fail;
1404 sha = shared_entry_header(rgt, ref);
1406 scombo.word = *(u32 *)&sha->flags;
1408 for ( ; ; )
1410 if ( unlikely(scombo.shorts.flags != GTF_accept_transfer) ||
1411 unlikely(scombo.shorts.domid != ld->domain_id) )
1413 gdprintk(XENLOG_INFO, "Bad flags (%x) or dom (%d). "
1414 "(NB. expected dom %d)\n",
1415 scombo.shorts.flags, scombo.shorts.domid,
1416 ld->domain_id);
1417 goto fail;
1420 new_scombo = scombo;
1421 new_scombo.shorts.flags |= GTF_transfer_committed;
1423 prev_scombo.word = cmpxchg((u32 *)&sha->flags,
1424 scombo.word, new_scombo.word);
1425 if ( likely(prev_scombo.word == scombo.word) )
1426 break;
1428 if ( retries++ == 4 )
1430 gdprintk(XENLOG_WARNING, "Shared grant entry is unstable.\n");
1431 goto fail;
1434 scombo = prev_scombo;
1437 spin_unlock(&rgt->lock);
1438 return 1;
1440 fail:
1441 spin_unlock(&rgt->lock);
1442 return 0;
1445 static long
1446 gnttab_transfer(
1447 XEN_GUEST_HANDLE(gnttab_transfer_t) uop, unsigned int count)
1449 struct domain *d = current->domain;
1450 struct domain *e;
1451 struct page_info *page;
1452 int i;
1453 struct gnttab_transfer gop;
1454 unsigned long mfn;
1455 unsigned int max_bitsize;
1457 for ( i = 0; i < count; i++ )
1459 if (i && hypercall_preempt_check())
1460 return i;
1462 /* Read from caller address space. */
1463 if ( unlikely(__copy_from_guest_offset(&gop, uop, i, 1)) )
1465 gdprintk(XENLOG_INFO, "gnttab_transfer: error reading req %d/%d\n",
1466 i, count);
1467 return -EFAULT;
1470 mfn = gfn_to_mfn_private(d, gop.mfn);
1472 /* Check the passed page frame for basic validity. */
1473 if ( unlikely(!mfn_valid(mfn)) )
1475 gdprintk(XENLOG_INFO, "gnttab_transfer: out-of-range %lx\n",
1476 (unsigned long)gop.mfn);
1477 gop.status = GNTST_bad_page;
1478 goto copyback;
1481 page = mfn_to_page(mfn);
1482 if ( unlikely(is_xen_heap_page(page)) )
1484 gdprintk(XENLOG_INFO, "gnttab_transfer: xen frame %lx\n",
1485 (unsigned long)gop.mfn);
1486 gop.status = GNTST_bad_page;
1487 goto copyback;
1490 if ( steal_page(d, page, 0) < 0 )
1492 gop.status = GNTST_bad_page;
1493 goto copyback;
1496 #ifndef __ia64__ /* IA64 implicitly replaces the old page in steal_page(). */
1497 guest_physmap_remove_page(d, gop.mfn, mfn, 0);
1498 #endif
1499 flush_tlb_mask(&d->domain_dirty_cpumask);
1501 /* Find the target domain. */
1502 if ( unlikely((e = rcu_lock_domain_by_id(gop.domid)) == NULL) )
1504 gdprintk(XENLOG_INFO, "gnttab_transfer: can't find domain %d\n",
1505 gop.domid);
1506 page->count_info &= ~(PGC_count_mask|PGC_allocated);
1507 free_domheap_page(page);
1508 gop.status = GNTST_bad_domain;
1509 goto copyback;
1512 if ( xsm_grant_transfer(d, e) )
1514 gop.status = GNTST_permission_denied;
1515 unlock_and_copyback:
1516 rcu_unlock_domain(e);
1517 page->count_info &= ~(PGC_count_mask|PGC_allocated);
1518 free_domheap_page(page);
1519 goto copyback;
1522 max_bitsize = domain_clamp_alloc_bitsize(
1523 e, BITS_PER_LONG+PAGE_SHIFT-1);
1524 if ( (1UL << (max_bitsize - PAGE_SHIFT)) <= mfn )
1526 struct page_info *new_page;
1527 void *sp, *dp;
1529 new_page = alloc_domheap_page(NULL, MEMF_bits(max_bitsize));
1530 if ( new_page == NULL )
1532 gop.status = GNTST_address_too_big;
1533 goto unlock_and_copyback;
1536 sp = map_domain_page(mfn);
1537 dp = __map_domain_page(new_page);
1538 memcpy(dp, sp, PAGE_SIZE);
1539 unmap_domain_page(dp);
1540 unmap_domain_page(sp);
1542 page->count_info &= ~(PGC_count_mask|PGC_allocated);
1543 free_domheap_page(page);
1544 page = new_page;
1547 spin_lock(&e->page_alloc_lock);
1549 /*
1550 * Check that 'e' will accept the page and has reservation
1551 * headroom. Also, a domain mustn't have PGC_allocated
1552 * pages when it is dying.
1553 */
1554 if ( unlikely(e->is_dying) ||
1555 unlikely(e->tot_pages >= e->max_pages) ||
1556 unlikely(!gnttab_prepare_for_transfer(e, d, gop.ref)) )
1558 if ( !e->is_dying )
1559 gdprintk(XENLOG_INFO, "gnttab_transfer: "
1560 "Transferee has no reservation "
1561 "headroom (%d,%d) or provided a bad grant ref (%08x) "
1562 "or is dying (%d)\n",
1563 e->tot_pages, e->max_pages, gop.ref, e->is_dying);
1564 spin_unlock(&e->page_alloc_lock);
1565 rcu_unlock_domain(e);
1566 page->count_info &= ~(PGC_count_mask|PGC_allocated);
1567 free_domheap_page(page);
1568 gop.status = GNTST_general_error;
1569 goto copyback;
1572 /* Okay, add the page to 'e'. */
1573 if ( unlikely(e->tot_pages++ == 0) )
1574 get_knownalive_domain(e);
1575 page_list_add_tail(page, &e->page_list);
1576 page_set_owner(page, e);
1578 spin_unlock(&e->page_alloc_lock);
1580 TRACE_1D(TRC_MEM_PAGE_GRANT_TRANSFER, e->domain_id);
1582 /* Tell the guest about its new page frame. */
1583 spin_lock(&e->grant_table->lock);
1585 if ( e->grant_table->gt_version == 1 )
1587 grant_entry_v1_t *sha = &shared_entry_v1(e->grant_table, gop.ref);
1588 guest_physmap_add_page(e, sha->frame, mfn, 0);
1589 sha->frame = mfn;
1591 else
1593 grant_entry_v2_t *sha = &shared_entry_v2(e->grant_table, gop.ref);
1594 guest_physmap_add_page(e, sha->full_page.frame, mfn, 0);
1595 sha->full_page.frame = mfn;
1597 wmb();
1598 shared_entry_header(e->grant_table, gop.ref)->flags |=
1599 GTF_transfer_completed;
1601 spin_unlock(&e->grant_table->lock);
1603 rcu_unlock_domain(e);
1605 gop.status = GNTST_okay;
1607 copyback:
1608 if ( unlikely(__copy_to_guest_offset(uop, i, &gop, 1)) )
1610 gdprintk(XENLOG_INFO, "gnttab_transfer: error writing resp "
1611 "%d/%d\n", i, count);
1612 return -EFAULT;
1616 return 0;
1619 /* Undo __acquire_grant_for_copy. Again, this has no effect on page
1620 type and reference counts. */
1621 static void
1622 __release_grant_for_copy(
1623 struct domain *rd, unsigned long gref, int readonly)
1625 grant_entry_header_t *sha;
1626 struct active_grant_entry *act;
1627 unsigned long r_frame;
1628 uint16_t *status;
1629 domid_t trans_domid;
1630 grant_ref_t trans_gref;
1631 int released_read;
1632 int released_write;
1633 struct domain *trans_dom;
1635 released_read = 0;
1636 released_write = 0;
1638 spin_lock(&rd->grant_table->lock);
1640 act = &active_entry(rd->grant_table, gref);
1641 sha = shared_entry_header(rd->grant_table, gref);
1642 r_frame = act->frame;
1644 if (rd->grant_table->gt_version == 1)
1646 status = &sha->flags;
1647 trans_domid = rd->domain_id;
1648 /* Shut the compiler up. This'll never be used, because
1649 trans_domid == rd->domain_id, but gcc doesn't know that. */
1650 trans_gref = 0x1234567;
1652 else
1654 status = &status_entry(rd->grant_table, gref);
1655 trans_domid = act->trans_dom;
1656 trans_gref = act->trans_gref;
1659 if ( readonly )
1661 act->pin -= GNTPIN_hstr_inc;
1663 else
1665 gnttab_mark_dirty(rd, r_frame);
1667 act->pin -= GNTPIN_hstw_inc;
1668 if ( !(act->pin & (GNTPIN_devw_mask|GNTPIN_hstw_mask)) )
1670 released_write = 1;
1671 gnttab_clear_flag(_GTF_writing, status);
1675 if ( !act->pin )
1677 gnttab_clear_flag(_GTF_reading, status);
1678 released_read = 1;
1681 spin_unlock(&rd->grant_table->lock);
1683 if ( trans_domid != rd->domain_id )
1685 if ( released_write || released_read )
1687 trans_dom = rcu_lock_domain_by_id(trans_domid);
1688 if ( trans_dom != NULL )
1690 /* Recursive calls, but they're tail calls, so it's
1691 okay. */
1692 if ( released_write )
1693 __release_grant_for_copy(trans_dom, trans_gref, 0);
1694 else if ( released_read )
1695 __release_grant_for_copy(trans_dom, trans_gref, 1);
1701 /* The status for a grant indicates that we're taking more access than
1702 the pin requires. Fix up the status to match the pin. Called
1703 under the domain's grant table lock. */
1704 /* Only safe on transitive grants. Even then, note that we don't
1705 attempt to drop any pin on the referent grant. */
1706 static void __fixup_status_for_pin(struct active_grant_entry *act,
1707 uint16_t *status)
1709 if ( !(act->pin & GNTPIN_hstw_mask) )
1710 *status &= ~_GTF_writing;
1712 if ( !(act->pin & GNTPIN_hstr_mask) )
1713 *status &= ~_GTF_reading;
1716 /* Grab a frame number from a grant entry and update the flags and pin
1717 count as appropriate. Note that this does *not* update the page
1718 type or reference counts, and does not check that the mfn is
1719 actually valid. */
1720 static int
1721 __acquire_grant_for_copy(
1722 struct domain *rd, unsigned long gref, struct domain *ld, int readonly,
1723 unsigned long *frame, unsigned *page_off, unsigned *length,
1724 unsigned allow_transitive, struct domain **owning_domain)
1726 grant_entry_v1_t *sha1;
1727 grant_entry_v2_t *sha2;
1728 grant_entry_header_t *shah;
1729 struct active_grant_entry *act;
1730 grant_status_t *status;
1731 uint32_t old_pin;
1732 domid_t trans_domid;
1733 grant_ref_t trans_gref;
1734 struct domain *rrd;
1735 unsigned long gfn;
1736 unsigned long grant_frame;
1737 unsigned trans_page_off;
1738 unsigned trans_length;
1739 int is_sub_page;
1740 struct domain *ignore;
1741 s16 rc = GNTST_okay;
1743 *owning_domain = NULL;
1745 spin_lock(&rd->grant_table->lock);
1747 if ( rd->grant_table->gt_version == 0 )
1748 PIN_FAIL(unlock_out, GNTST_general_error,
1749 "remote grant table not ready\n");
1751 if ( unlikely(gref >= nr_grant_entries(rd->grant_table)) )
1752 PIN_FAIL(unlock_out, GNTST_bad_gntref,
1753 "Bad grant reference %ld\n", gref);
1755 act = &active_entry(rd->grant_table, gref);
1756 shah = shared_entry_header(rd->grant_table, gref);
1757 if ( rd->grant_table->gt_version == 1 )
1759 sha1 = &shared_entry_v1(rd->grant_table, gref);
1760 sha2 = NULL;
1761 status = &shah->flags;
1763 else
1765 sha1 = NULL;
1766 sha2 = &shared_entry_v2(rd->grant_table, gref);
1767 status = &status_entry(rd->grant_table, gref);
1770 /* If already pinned, check the active domid and avoid refcnt overflow. */
1771 if ( act->pin &&
1772 ((act->domid != ld->domain_id) ||
1773 (act->pin & 0x80808080U) != 0) )
1774 PIN_FAIL(unlock_out, GNTST_general_error,
1775 "Bad domain (%d != %d), or risk of counter overflow %08x\n",
1776 act->domid, ld->domain_id, act->pin);
1778 old_pin = act->pin;
1779 if ( !act->pin ||
1780 (!readonly && !(act->pin & (GNTPIN_devw_mask|GNTPIN_hstw_mask))) )
1782 if ( (rc = _set_status(rd->grant_table->gt_version,
1783 ld->domain_id,
1784 readonly, 0, shah, act,
1785 status) ) != GNTST_okay )
1786 goto unlock_out;
1788 trans_domid = ld->domain_id;
1789 trans_gref = 0;
1790 if ( sha2 && (shah->flags & GTF_type_mask) == GTF_transitive )
1792 if ( !allow_transitive )
1793 PIN_FAIL(unlock_out, GNTST_general_error,
1794 "transitive grant when transitivity not allowed\n");
1796 trans_domid = sha2->transitive.trans_domid;
1797 trans_gref = sha2->transitive.gref;
1798 barrier(); /* Stop the compiler from re-loading
1799 trans_domid from shared memory */
1800 if ( trans_domid == rd->domain_id )
1801 PIN_FAIL(unlock_out, GNTST_general_error,
1802 "transitive grants cannot be self-referential\n");
1804 /* We allow the trans_domid == ld->domain_id case, which
1805 corresponds to a grant being issued by one domain, sent
1806 to another one, and then transitively granted back to
1807 the original domain. Allowing it is easy, and means
1808 that you don't need to go out of your way to avoid it
1809 in the guest. */
1811 rrd = rcu_lock_domain_by_id(trans_domid);
1812 if ( rrd == NULL )
1813 PIN_FAIL(unlock_out, GNTST_general_error,
1814 "transitive grant referenced bad domain %d\n",
1815 trans_domid);
1816 spin_unlock(&rd->grant_table->lock);
1818 rc = __acquire_grant_for_copy(rrd, trans_gref, rd,
1819 readonly, &grant_frame,
1820 &trans_page_off, &trans_length,
1821 0, &ignore);
1823 spin_lock(&rd->grant_table->lock);
1824 if ( rc != GNTST_okay ) {
1825 __fixup_status_for_pin(act, status);
1826 spin_unlock(&rd->grant_table->lock);
1827 return rc;
1830 /* We dropped the lock, so we have to check that nobody
1831 else tried to pin (or, for that matter, unpin) the
1832 reference in *this* domain. If they did, just give up
1833 and try again. */
1834 if ( act->pin != old_pin )
1836 __fixup_status_for_pin(act, status);
1837 spin_unlock(&rd->grant_table->lock);
1838 return __acquire_grant_for_copy(rd, gref, ld, readonly,
1839 frame, page_off, length,
1840 allow_transitive,
1841 owning_domain);
1844 /* The actual remote remote grant may or may not be a
1845 sub-page, but we always treat it as one because that
1846 blocks mappings of transitive grants. */
1847 is_sub_page = 1;
1848 *owning_domain = rrd;
1849 act->gfn = -1ul;
1851 else if ( sha1 )
1853 gfn = sha1->frame;
1854 rc = __get_paged_frame(gfn, &grant_frame, readonly, rd);
1855 if ( rc != GNTST_okay )
1856 goto unlock_out;
1857 act->gfn = gfn;
1858 is_sub_page = 0;
1859 trans_page_off = 0;
1860 trans_length = PAGE_SIZE;
1861 *owning_domain = rd;
1863 else if ( !(sha2->hdr.flags & GTF_sub_page) )
1865 gfn = sha2->full_page.frame;
1866 rc = __get_paged_frame(gfn, &grant_frame, readonly, rd);
1867 if ( rc != GNTST_okay )
1868 goto unlock_out;
1869 act->gfn = gfn;
1870 is_sub_page = 0;
1871 trans_page_off = 0;
1872 trans_length = PAGE_SIZE;
1873 *owning_domain = rd;
1875 else
1877 gfn = sha2->sub_page.frame;
1878 rc = __get_paged_frame(gfn, &grant_frame, readonly, rd);
1879 if ( rc != GNTST_okay )
1880 goto unlock_out;
1881 act->gfn = gfn;
1882 is_sub_page = 1;
1883 trans_page_off = sha2->sub_page.page_off;
1884 trans_length = sha2->sub_page.length;
1885 *owning_domain = rd;
1888 if ( !act->pin )
1890 act->domid = ld->domain_id;
1891 act->is_sub_page = is_sub_page;
1892 act->start = trans_page_off;
1893 act->length = trans_length;
1894 act->trans_dom = trans_domid;
1895 act->trans_gref = trans_gref;
1896 act->frame = grant_frame;
1899 else
1901 *owning_domain = rd;
1904 act->pin += readonly ? GNTPIN_hstr_inc : GNTPIN_hstw_inc;
1906 *page_off = act->start;
1907 *length = act->length;
1908 *frame = act->frame;
1910 unlock_out:
1911 spin_unlock(&rd->grant_table->lock);
1912 return rc;
1915 static void
1916 __gnttab_copy(
1917 struct gnttab_copy *op)
1919 struct domain *sd = NULL, *dd = NULL;
1920 struct domain *source_domain = NULL, *dest_domain = NULL;
1921 unsigned long s_frame, d_frame;
1922 char *sp, *dp;
1923 s16 rc = GNTST_okay;
1924 int have_d_grant = 0, have_s_grant = 0, have_s_ref = 0;
1925 int src_is_gref, dest_is_gref;
1927 if ( ((op->source.offset + op->len) > PAGE_SIZE) ||
1928 ((op->dest.offset + op->len) > PAGE_SIZE) )
1929 PIN_FAIL(error_out, GNTST_bad_copy_arg, "copy beyond page area.\n");
1931 src_is_gref = op->flags & GNTCOPY_source_gref;
1932 dest_is_gref = op->flags & GNTCOPY_dest_gref;
1934 if ( (op->source.domid != DOMID_SELF && !src_is_gref ) ||
1935 (op->dest.domid != DOMID_SELF && !dest_is_gref) )
1936 PIN_FAIL(error_out, GNTST_permission_denied,
1937 "only allow copy-by-mfn for DOMID_SELF.\n");
1939 if ( op->source.domid == DOMID_SELF )
1940 sd = rcu_lock_current_domain();
1941 else if ( (sd = rcu_lock_domain_by_id(op->source.domid)) == NULL )
1942 PIN_FAIL(error_out, GNTST_bad_domain,
1943 "couldn't find %d\n", op->source.domid);
1945 if ( op->dest.domid == DOMID_SELF )
1946 dd = rcu_lock_current_domain();
1947 else if ( (dd = rcu_lock_domain_by_id(op->dest.domid)) == NULL )
1948 PIN_FAIL(error_out, GNTST_bad_domain,
1949 "couldn't find %d\n", op->dest.domid);
1951 rc = xsm_grant_copy(sd, dd);
1952 if ( rc )
1954 rc = GNTST_permission_denied;
1955 goto error_out;
1958 if ( src_is_gref )
1960 unsigned source_off, source_len;
1961 rc = __acquire_grant_for_copy(sd, op->source.u.ref, current->domain, 1,
1962 &s_frame, &source_off, &source_len, 1,
1963 &source_domain);
1964 if ( rc != GNTST_okay )
1965 goto error_out;
1966 have_s_grant = 1;
1967 if ( op->source.offset < source_off ||
1968 op->len > source_len )
1969 PIN_FAIL(error_out, GNTST_general_error,
1970 "copy source out of bounds: %d < %d || %d > %d\n",
1971 op->source.offset, source_off,
1972 op->len, source_len);
1974 else
1976 #ifdef CONFIG_X86
1977 rc = __get_paged_frame(op->source.u.gmfn, &s_frame, 1, sd);
1978 if ( rc != GNTST_okay )
1979 goto error_out;
1980 #else
1981 s_frame = gmfn_to_mfn(sd, op->source.u.gmfn);
1982 #endif
1983 source_domain = sd;
1985 if ( unlikely(!mfn_valid(s_frame)) )
1986 PIN_FAIL(error_out, GNTST_general_error,
1987 "source frame %lx invalid.\n", s_frame);
1988 if ( !get_page(mfn_to_page(s_frame), source_domain) )
1990 if ( !sd->is_dying )
1991 gdprintk(XENLOG_WARNING, "Could not get src frame %lx\n", s_frame);
1992 rc = GNTST_general_error;
1993 goto error_out;
1995 have_s_ref = 1;
1997 if ( dest_is_gref )
1999 unsigned dest_off, dest_len;
2000 rc = __acquire_grant_for_copy(dd, op->dest.u.ref, current->domain, 0,
2001 &d_frame, &dest_off, &dest_len, 1,
2002 &dest_domain);
2003 if ( rc != GNTST_okay )
2004 goto error_out;
2005 have_d_grant = 1;
2006 if ( op->dest.offset < dest_off ||
2007 op->len > dest_len )
2008 PIN_FAIL(error_out, GNTST_general_error,
2009 "copy dest out of bounds: %d < %d || %d > %d\n",
2010 op->dest.offset, dest_off,
2011 op->len, dest_len);
2013 else
2015 #ifdef CONFIG_X86
2016 rc = __get_paged_frame(op->dest.u.gmfn, &d_frame, 0, dd);
2017 if ( rc != GNTST_okay )
2018 goto error_out;
2019 #else
2020 d_frame = gmfn_to_mfn(dd, op->dest.u.gmfn);
2021 #endif
2022 dest_domain = dd;
2024 if ( unlikely(!mfn_valid(d_frame)) )
2025 PIN_FAIL(error_out, GNTST_general_error,
2026 "destination frame %lx invalid.\n", d_frame);
2027 if ( !get_page_and_type(mfn_to_page(d_frame), dest_domain,
2028 PGT_writable_page) )
2030 if ( !dd->is_dying )
2031 gdprintk(XENLOG_WARNING, "Could not get dst frame %lx\n", d_frame);
2032 rc = GNTST_general_error;
2033 goto error_out;
2036 sp = map_domain_page(s_frame);
2037 dp = map_domain_page(d_frame);
2039 memcpy(dp + op->dest.offset, sp + op->source.offset, op->len);
2041 unmap_domain_page(dp);
2042 unmap_domain_page(sp);
2044 gnttab_mark_dirty(dd, d_frame);
2046 put_page_and_type(mfn_to_page(d_frame));
2047 error_out:
2048 if ( have_s_ref )
2049 put_page(mfn_to_page(s_frame));
2050 if ( have_s_grant )
2051 __release_grant_for_copy(sd, op->source.u.ref, 1);
2052 if ( have_d_grant )
2053 __release_grant_for_copy(dd, op->dest.u.ref, 0);
2054 if ( sd )
2055 rcu_unlock_domain(sd);
2056 if ( dd )
2057 rcu_unlock_domain(dd);
2058 op->status = rc;
2061 static long
2062 gnttab_copy(
2063 XEN_GUEST_HANDLE(gnttab_copy_t) uop, unsigned int count)
2065 int i;
2066 struct gnttab_copy op;
2068 for ( i = 0; i < count; i++ )
2070 if (i && hypercall_preempt_check())
2071 return i;
2072 if ( unlikely(__copy_from_guest_offset(&op, uop, i, 1)) )
2073 return -EFAULT;
2074 __gnttab_copy(&op);
2075 if ( unlikely(__copy_to_guest_offset(uop, i, &op, 1)) )
2076 return -EFAULT;
2078 return 0;
2081 static long
2082 gnttab_set_version(XEN_GUEST_HANDLE(gnttab_set_version_t uop))
2084 gnttab_set_version_t op;
2085 struct domain *d = current->domain;
2086 struct grant_table *gt = d->grant_table;
2087 struct active_grant_entry *act;
2088 long res;
2089 int i;
2091 if (copy_from_guest(&op, uop, 1))
2092 return -EFAULT;
2094 res = -EINVAL;
2095 if (op.version != 1 && op.version != 2)
2096 goto out;
2098 res = 0;
2099 if ( gt->gt_version == op.version )
2100 goto out;
2102 spin_lock(&gt->lock);
2103 /* Make sure that the grant table isn't currently in use when we
2104 change the version number. */
2105 /* (You need to change the version number for e.g. kexec.) */
2106 if ( gt->gt_version != 0 )
2108 for ( i = 0; i < nr_grant_entries(gt); i++ )
2110 act = &active_entry(gt, i);
2111 if ( act->pin != 0 )
2113 gdprintk(XENLOG_WARNING,
2114 "tried to change grant table version from %d to %d, but some grant entries still in use\n",
2115 gt->gt_version,
2116 op.version);
2117 res = -EBUSY;
2118 goto out_unlock;
2123 /* XXX: If we're going to version 2, we could maybe shrink the
2124 active grant table here. */
2126 if ( op.version == 2 && gt->gt_version < 2 )
2128 res = gnttab_populate_status_frames(d, gt);
2129 if ( res < 0)
2130 goto out_unlock;
2133 if ( op.version < 2 && gt->gt_version == 2 )
2134 gnttab_unpopulate_status_frames(d, gt);
2136 if ( op.version != gt->gt_version )
2138 /* Make sure there's no crud left over in the table from the
2139 old version. */
2140 for ( i = 0; i < nr_grant_frames(gt); i++ )
2141 memset(gt->shared_raw[i], 0, PAGE_SIZE);
2144 gt->gt_version = op.version;
2146 out_unlock:
2147 spin_unlock(&gt->lock);
2149 out:
2150 op.version = gt->gt_version;
2152 if (copy_to_guest(uop, &op, 1))
2153 res = -EFAULT;
2155 return res;
2158 static long
2159 gnttab_get_status_frames(XEN_GUEST_HANDLE(gnttab_get_status_frames_t) uop,
2160 int count)
2162 gnttab_get_status_frames_t op;
2163 struct domain *d;
2164 struct grant_table *gt;
2165 uint64_t gmfn;
2166 int i;
2167 int rc;
2169 if ( count != 1 )
2170 return -EINVAL;
2172 if ( unlikely(copy_from_guest(&op, uop, 1) != 0) )
2174 gdprintk(XENLOG_INFO,
2175 "Fault while reading gnttab_get_status_frames_t.\n");
2176 return -EFAULT;
2179 rc = rcu_lock_target_domain_by_id(op.dom, &d);
2180 if ( rc < 0 )
2182 if ( rc == -ESRCH )
2183 op.status = GNTST_bad_domain;
2184 else if ( rc == -EPERM )
2185 op.status = GNTST_permission_denied;
2186 else
2187 op.status = GNTST_general_error;
2188 goto out1;
2191 gt = d->grant_table;
2193 if ( unlikely(op.nr_frames > nr_status_frames(gt)) ) {
2194 gdprintk(XENLOG_INFO, "Guest requested addresses for %d grant status "
2195 "frames, but only %d are available.\n",
2196 op.nr_frames, nr_status_frames(gt));
2197 op.status = GNTST_general_error;
2198 goto out2;
2201 op.status = GNTST_okay;
2203 spin_lock(&gt->lock);
2205 for ( i = 0; i < op.nr_frames; i++ )
2207 gmfn = gnttab_status_gmfn(d, d->grant_table, i);
2208 if (copy_to_guest_offset(op.frame_list,
2209 i,
2210 &gmfn,
2211 1))
2212 op.status = GNTST_bad_virt_addr;
2215 spin_unlock(&gt->lock);
2216 out2:
2217 rcu_unlock_domain(d);
2218 out1:
2219 if ( unlikely(copy_to_guest(uop, &op, 1)) )
2220 return -EFAULT;
2222 return 0;
2225 static long
2226 gnttab_get_version(XEN_GUEST_HANDLE(gnttab_get_version_t uop))
2228 gnttab_get_version_t op;
2229 struct domain *d;
2231 if ( copy_from_guest(&op, uop, 1) )
2232 return -EFAULT;
2233 d = rcu_lock_domain_by_id(op.dom);
2234 if ( d == NULL )
2235 return -ESRCH;
2236 if ( !IS_PRIV_FOR(current->domain, d) )
2238 rcu_unlock_domain(d);
2239 return -EPERM;
2241 spin_lock(&d->grant_table->lock);
2242 op.version = d->grant_table->gt_version;
2243 spin_unlock(&d->grant_table->lock);
2245 if ( copy_to_guest(uop, &op, 1) )
2246 return -EFAULT;
2247 else
2248 return 0;
2251 long
2252 do_grant_table_op(
2253 unsigned int cmd, XEN_GUEST_HANDLE(void) uop, unsigned int count)
2255 long rc;
2256 struct domain *d = current->domain;
2258 if ( (int)count < 0 )
2259 return -EINVAL;
2261 domain_lock(d);
2263 rc = -EFAULT;
2264 switch ( cmd )
2266 case GNTTABOP_map_grant_ref:
2268 XEN_GUEST_HANDLE(gnttab_map_grant_ref_t) map =
2269 guest_handle_cast(uop, gnttab_map_grant_ref_t);
2270 if ( unlikely(!guest_handle_okay(map, count)) )
2271 goto out;
2272 rc = gnttab_map_grant_ref(map, count);
2273 if ( rc > 0 )
2275 guest_handle_add_offset(map, rc);
2276 uop = guest_handle_cast(map, void);
2278 break;
2280 case GNTTABOP_unmap_grant_ref:
2282 XEN_GUEST_HANDLE(gnttab_unmap_grant_ref_t) unmap =
2283 guest_handle_cast(uop, gnttab_unmap_grant_ref_t);
2284 if ( unlikely(!guest_handle_okay(unmap, count)) )
2285 goto out;
2286 rc = gnttab_unmap_grant_ref(unmap, count);
2287 if ( rc > 0 )
2289 guest_handle_add_offset(unmap, rc);
2290 uop = guest_handle_cast(unmap, void);
2292 break;
2294 case GNTTABOP_unmap_and_replace:
2296 XEN_GUEST_HANDLE(gnttab_unmap_and_replace_t) unmap =
2297 guest_handle_cast(uop, gnttab_unmap_and_replace_t);
2298 if ( unlikely(!guest_handle_okay(unmap, count)) )
2299 goto out;
2300 rc = -ENOSYS;
2301 if ( unlikely(!replace_grant_supported()) )
2302 goto out;
2303 rc = gnttab_unmap_and_replace(unmap, count);
2304 if ( rc > 0 )
2306 guest_handle_add_offset(unmap, rc);
2307 uop = guest_handle_cast(unmap, void);
2309 break;
2311 case GNTTABOP_setup_table:
2313 rc = gnttab_setup_table(
2314 guest_handle_cast(uop, gnttab_setup_table_t), count);
2315 ASSERT(rc <= 0);
2316 break;
2318 case GNTTABOP_transfer:
2320 XEN_GUEST_HANDLE(gnttab_transfer_t) transfer =
2321 guest_handle_cast(uop, gnttab_transfer_t);
2322 if ( unlikely(!guest_handle_okay(transfer, count)) )
2323 goto out;
2324 rc = gnttab_transfer(transfer, count);
2325 if ( rc > 0 )
2327 guest_handle_add_offset(transfer, rc);
2328 uop = guest_handle_cast(transfer, void);
2330 break;
2332 case GNTTABOP_copy:
2334 XEN_GUEST_HANDLE(gnttab_copy_t) copy =
2335 guest_handle_cast(uop, gnttab_copy_t);
2336 if ( unlikely(!guest_handle_okay(copy, count)) )
2337 goto out;
2338 rc = gnttab_copy(copy, count);
2339 if ( rc > 0 )
2341 guest_handle_add_offset(copy, rc);
2342 uop = guest_handle_cast(copy, void);
2344 break;
2346 case GNTTABOP_query_size:
2348 rc = gnttab_query_size(
2349 guest_handle_cast(uop, gnttab_query_size_t), count);
2350 ASSERT(rc <= 0);
2351 break;
2353 case GNTTABOP_set_version:
2355 rc = gnttab_set_version(guest_handle_cast(uop, gnttab_set_version_t));
2356 break;
2358 case GNTTABOP_get_status_frames:
2360 rc = gnttab_get_status_frames(
2361 guest_handle_cast(uop, gnttab_get_status_frames_t), count);
2362 break;
2364 case GNTTABOP_get_version:
2366 rc = gnttab_get_version(guest_handle_cast(uop, gnttab_get_version_t));
2367 break;
2369 default:
2370 rc = -ENOSYS;
2371 break;
2374 out:
2375 domain_unlock(d);
2377 if ( rc > 0 )
2379 ASSERT(rc < count);
2380 rc = hypercall_create_continuation(__HYPERVISOR_grant_table_op,
2381 "ihi", cmd, uop, count - rc);
2384 return rc;
2387 #ifdef CONFIG_COMPAT
2388 #include "compat/grant_table.c"
2389 #endif
2391 static unsigned int max_nr_active_grant_frames(void)
2393 return (((max_nr_grant_frames * (PAGE_SIZE / sizeof(grant_entry_v1_t))) +
2394 ((PAGE_SIZE / sizeof(struct active_grant_entry))-1))
2395 / (PAGE_SIZE / sizeof(struct active_grant_entry)));
2398 int
2399 grant_table_create(
2400 struct domain *d)
2402 struct grant_table *t;
2403 int i;
2405 if ( (t = xmalloc(struct grant_table)) == NULL )
2406 goto no_mem_0;
2408 /* Simple stuff. */
2409 memset(t, 0, sizeof(*t));
2410 spin_lock_init(&t->lock);
2411 t->nr_grant_frames = INITIAL_NR_GRANT_FRAMES;
2413 /* Active grant table. */
2414 if ( (t->active = xmalloc_array(struct active_grant_entry *,
2415 max_nr_active_grant_frames())) == NULL )
2416 goto no_mem_1;
2417 memset(t->active, 0, max_nr_active_grant_frames() * sizeof(t->active[0]));
2418 for ( i = 0;
2419 i < num_act_frames_from_sha_frames(INITIAL_NR_GRANT_FRAMES); i++ )
2421 if ( (t->active[i] = alloc_xenheap_page()) == NULL )
2422 goto no_mem_2;
2423 clear_page(t->active[i]);
2426 /* Tracking of mapped foreign frames table */
2427 if ( (t->maptrack = xmalloc_array(struct grant_mapping *,
2428 max_nr_maptrack_frames())) == NULL )
2429 goto no_mem_2;
2430 memset(t->maptrack, 0, max_nr_maptrack_frames() * sizeof(t->maptrack[0]));
2431 if ( (t->maptrack[0] = alloc_xenheap_page()) == NULL )
2432 goto no_mem_3;
2433 clear_page(t->maptrack[0]);
2434 t->maptrack_limit = PAGE_SIZE / sizeof(struct grant_mapping);
2435 for ( i = 0; i < t->maptrack_limit; i++ )
2436 t->maptrack[0][i].ref = i+1;
2438 /* Shared grant table. */
2439 if ( (t->shared_raw = xmalloc_array(void *, max_nr_grant_frames)) == NULL )
2440 goto no_mem_3;
2441 memset(t->shared_raw, 0, max_nr_grant_frames * sizeof(t->shared_raw[0]));
2442 for ( i = 0; i < INITIAL_NR_GRANT_FRAMES; i++ )
2444 if ( (t->shared_raw[i] = alloc_xenheap_page()) == NULL )
2445 goto no_mem_4;
2446 clear_page(t->shared_raw[i]);
2449 for ( i = 0; i < INITIAL_NR_GRANT_FRAMES; i++ )
2450 gnttab_create_shared_page(d, t, i);
2452 /* Status pages for grant table - for version 2 */
2453 t->status = xmalloc_array(grant_status_t *,
2454 grant_to_status_frames(max_nr_grant_frames));
2455 if ( t->status == NULL )
2456 goto no_mem_4;
2457 memset(t->status, 0,
2458 grant_to_status_frames(max_nr_grant_frames) * sizeof(t->status[0]));
2459 t->nr_status_frames = 0;
2461 /* Okay, install the structure. */
2462 d->grant_table = t;
2463 return 0;
2465 no_mem_4:
2466 for ( i = 0; i < INITIAL_NR_GRANT_FRAMES; i++ )
2467 free_xenheap_page(t->shared_raw[i]);
2468 xfree(t->shared_raw);
2469 no_mem_3:
2470 free_xenheap_page(t->maptrack[0]);
2471 xfree(t->maptrack);
2472 no_mem_2:
2473 for ( i = 0;
2474 i < num_act_frames_from_sha_frames(INITIAL_NR_GRANT_FRAMES); i++ )
2475 free_xenheap_page(t->active[i]);
2476 xfree(t->active);
2477 no_mem_1:
2478 xfree(t);
2479 no_mem_0:
2480 return -ENOMEM;
2483 void
2484 gnttab_release_mappings(
2485 struct domain *d)
2487 struct grant_table *gt = d->grant_table;
2488 struct grant_mapping *map;
2489 grant_ref_t ref;
2490 grant_handle_t handle;
2491 struct domain *rd;
2492 struct active_grant_entry *act;
2493 grant_entry_header_t *sha;
2494 uint16_t *status;
2495 struct page_info *pg;
2497 BUG_ON(!d->is_dying);
2499 for ( handle = 0; handle < gt->maptrack_limit; handle++ )
2501 map = &maptrack_entry(gt, handle);
2502 if ( !(map->flags & (GNTMAP_device_map|GNTMAP_host_map)) )
2503 continue;
2505 ref = map->ref;
2507 gdprintk(XENLOG_INFO, "Grant release (%hu) ref:(%hu) "
2508 "flags:(%x) dom:(%hu)\n",
2509 handle, ref, map->flags, map->domid);
2511 rd = rcu_lock_domain_by_id(map->domid);
2512 if ( rd == NULL )
2514 /* Nothing to clear up... */
2515 map->flags = 0;
2516 continue;
2519 spin_lock(&rd->grant_table->lock);
2521 act = &active_entry(rd->grant_table, ref);
2522 sha = shared_entry_header(rd->grant_table, ref);
2523 if (rd->grant_table->gt_version == 1)
2524 status = &sha->flags;
2525 else
2526 status = &status_entry(rd->grant_table, ref);
2528 pg = mfn_to_page(act->frame);
2530 if ( map->flags & GNTMAP_readonly )
2532 if ( map->flags & GNTMAP_device_map )
2534 BUG_ON(!(act->pin & GNTPIN_devr_mask));
2535 act->pin -= GNTPIN_devr_inc;
2536 if ( !is_iomem_page(act->frame) )
2537 put_page(pg);
2540 if ( map->flags & GNTMAP_host_map )
2542 BUG_ON(!(act->pin & GNTPIN_hstr_mask));
2543 act->pin -= GNTPIN_hstr_inc;
2544 if ( gnttab_release_host_mappings(d) &&
2545 !is_iomem_page(act->frame) )
2546 put_page(pg);
2549 else
2551 if ( map->flags & GNTMAP_device_map )
2553 BUG_ON(!(act->pin & GNTPIN_devw_mask));
2554 act->pin -= GNTPIN_devw_inc;
2555 if ( !is_iomem_page(act->frame) )
2556 put_page_and_type(pg);
2559 if ( map->flags & GNTMAP_host_map )
2561 BUG_ON(!(act->pin & GNTPIN_hstw_mask));
2562 act->pin -= GNTPIN_hstw_inc;
2563 if ( gnttab_release_host_mappings(d) &&
2564 !is_iomem_page(act->frame) )
2566 if ( gnttab_host_mapping_get_page_type(map, d, rd) )
2567 put_page_type(pg);
2568 put_page(pg);
2572 if ( (act->pin & (GNTPIN_devw_mask|GNTPIN_hstw_mask)) == 0 )
2573 gnttab_clear_flag(_GTF_writing, status);
2576 if ( act->pin == 0 )
2577 gnttab_clear_flag(_GTF_reading, status);
2579 spin_unlock(&rd->grant_table->lock);
2581 rcu_unlock_domain(rd);
2583 map->flags = 0;
2588 void
2589 grant_table_destroy(
2590 struct domain *d)
2592 struct grant_table *t = d->grant_table;
2593 int i;
2595 if ( t == NULL )
2596 return;
2598 for ( i = 0; i < nr_grant_frames(t); i++ )
2599 free_xenheap_page(t->shared_raw[i]);
2600 xfree(t->shared_raw);
2602 for ( i = 0; i < nr_maptrack_frames(t); i++ )
2603 free_xenheap_page(t->maptrack[i]);
2604 xfree(t->maptrack);
2606 for ( i = 0; i < nr_active_grant_frames(t); i++ )
2607 free_xenheap_page(t->active[i]);
2608 xfree(t->active);
2610 for ( i = 0; i < nr_status_frames(t); i++ )
2611 free_xenheap_page(t->status[i]);
2612 xfree(t->status);
2614 xfree(t);
2615 d->grant_table = NULL;
2618 void gnttab_usage_print(struct domain *rd)
2620 int first = 1;
2621 grant_ref_t ref;
2622 struct grant_table *gt = rd->grant_table;
2624 printk(" -------- active -------- -------- shared --------\n");
2625 printk("[ref] localdom mfn pin localdom gmfn flags\n");
2627 spin_lock(&gt->lock);
2629 if ( gt->gt_version == 0 )
2630 goto out;
2632 for ( ref = 0; ref != nr_grant_entries(gt); ref++ )
2634 struct active_grant_entry *act;
2635 struct grant_entry_header *sha;
2636 grant_entry_v1_t *sha1;
2637 grant_entry_v2_t *sha2;
2638 uint16_t status;
2639 uint64_t frame;
2641 act = &active_entry(gt, ref);
2642 if ( !act->pin )
2643 continue;
2645 sha = shared_entry_header(gt, ref);
2647 if ( gt->gt_version == 1 )
2649 sha1 = &shared_entry_v1(gt, ref);
2650 sha2 = NULL;
2651 status = sha->flags;
2652 frame = sha1->frame;
2654 else
2656 sha2 = &shared_entry_v2(gt, ref);
2657 sha1 = NULL;
2658 frame = sha2->full_page.frame;
2659 status = status_entry(gt, ref);
2662 if ( first )
2664 printk("grant-table for remote domain:%5d (v%d)\n",
2665 rd->domain_id, gt->gt_version);
2666 first = 0;
2669 /* [ddd] ddddd 0xXXXXXX 0xXXXXXXXX ddddd 0xXXXXXX 0xXX */
2670 printk("[%3d] %5d 0x%06lx 0x%08x %5d 0x%06"PRIx64" 0x%02x\n",
2671 ref, act->domid, act->frame, act->pin,
2672 sha->domid, frame, status);
2675 out:
2676 spin_unlock(&gt->lock);
2678 if ( first )
2679 printk("grant-table for remote domain:%5d ... "
2680 "no active grant table entries\n", rd->domain_id);
2683 static void gnttab_usage_print_all(unsigned char key)
2685 struct domain *d;
2686 printk("%s [ key '%c' pressed\n", __FUNCTION__, key);
2687 for_each_domain ( d )
2688 gnttab_usage_print(d);
2689 printk("%s ] done\n", __FUNCTION__);
2692 static struct keyhandler gnttab_usage_print_all_keyhandler = {
2693 .diagnostic = 1,
2694 .u.fn = gnttab_usage_print_all,
2695 .desc = "print grant table usage"
2696 };
2698 static int __init gnttab_usage_init(void)
2700 register_keyhandler('g', &gnttab_usage_print_all_keyhandler);
2701 return 0;
2703 __initcall(gnttab_usage_init);
2705 /*
2706 * Local variables:
2707 * mode: C
2708 * c-set-style: "BSD"
2709 * c-basic-offset: 4
2710 * tab-width: 4
2711 * indent-tabs-mode: nil
2712 * End:
2713 */