debuggers.hg

view xen/common/grant_table.c @ 4629:6375127fdf23

bitkeeper revision 1.1311.1.1 (426641eeBv97w6sl983zxeR4Dc3Utg)

Cleanup page table handling. Add macros to access page table
entries, fixup plenty of places in the code to use the page
table types instead of "unsigned long".

Signed-off-by: Gerd Knorr <kraxel@bytesex.org>
Signed-off-by: michael.fetterman@cl.cam.ac.uk
author mafetter@fleming.research
date Wed Apr 20 11:50:06 2005 +0000 (2005-04-20)
parents b4ebb22003b1
children 1803018b3b05
line source
1 /******************************************************************************
2 * common/grant_table.c
3 *
4 * Mechanism for granting foreign access to page frames, and receiving
5 * page-ownership transfers.
6 *
7 * Copyright (c) 2005 Christopher Clark
8 * Copyright (c) 2004 K A Fraser
9 *
10 * This program is free software; you can redistribute it and/or modify
11 * it under the terms of the GNU General Public License as published by
12 * the Free Software Foundation; either version 2 of the License, or
13 * (at your option) any later version.
14 *
15 * This program is distributed in the hope that it will be useful,
16 * but WITHOUT ANY WARRANTY; without even the implied warranty of
17 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
18 * GNU General Public License for more details.
19 *
20 * You should have received a copy of the GNU General Public License
21 * along with this program; if not, write to the Free Software
22 * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
23 */
25 #define GRANT_DEBUG 0
26 #define GRANT_DEBUG_VERBOSE 0
28 #include <xen/config.h>
29 #include <xen/sched.h>
30 #include <xen/shadow.h>
31 #include <xen/mm.h>
33 #define PIN_FAIL(_lbl, _rc, _f, _a...) \
34 do { \
35 DPRINTK( _f, ## _a ); \
36 rc = (_rc); \
37 goto _lbl; \
38 } while ( 0 )
40 static inline int
41 get_maptrack_handle(
42 grant_table_t *t)
43 {
44 unsigned int h;
45 if ( unlikely((h = t->maptrack_head) == t->maptrack_limit) )
46 return -1;
47 t->maptrack_head = t->maptrack[h].ref_and_flags >> MAPTRACK_REF_SHIFT;
48 t->map_count++;
49 return h;
50 }
52 static inline void
53 put_maptrack_handle(
54 grant_table_t *t, int handle)
55 {
56 t->maptrack[handle].ref_and_flags = t->maptrack_head << MAPTRACK_REF_SHIFT;
57 t->maptrack_head = handle;
58 t->map_count--;
59 }
61 static int
62 __gnttab_activate_grant_ref(
63 struct domain *mapping_d, /* IN */
64 struct exec_domain *mapping_ed,
65 struct domain *granting_d,
66 grant_ref_t ref,
67 u16 dev_hst_ro_flags,
68 unsigned long host_virt_addr,
69 unsigned long *pframe ) /* OUT */
70 {
71 domid_t sdom;
72 u16 sflags;
73 active_grant_entry_t *act;
74 grant_entry_t *sha;
75 s16 rc = 1;
76 unsigned long frame = 0;
77 int retries = 0;
79 /*
80 * Objectives of this function:
81 * . Make the record ( granting_d, ref ) active, if not already.
82 * . Update shared grant entry of owner, indicating frame is mapped.
83 * . Increment the owner act->pin reference counts.
84 * . get_page on shared frame if new mapping.
85 * . get_page_type if this is first RW mapping of frame.
86 * . Add PTE to virtual address space of mapping_d, if necessary.
87 * Returns:
88 * . -ve: error
89 * . 1: ok
90 * . 0: ok and TLB invalidate of host_virt_addr needed.
91 *
92 * On success, *pframe contains mfn.
93 */
95 /*
96 * We bound the number of times we retry CMPXCHG on memory locations that
97 * we share with a guest OS. The reason is that the guest can modify that
98 * location at a higher rate than we can read-modify-CMPXCHG, so the guest
99 * could cause us to livelock. There are a few cases where it is valid for
100 * the guest to race our updates (e.g., to change the GTF_readonly flag),
101 * so we allow a few retries before failing.
102 */
104 act = &granting_d->grant_table->active[ref];
105 sha = &granting_d->grant_table->shared[ref];
107 spin_lock(&granting_d->grant_table->lock);
109 if ( act->pin == 0 )
110 {
111 /* CASE 1: Activating a previously inactive entry. */
113 sflags = sha->flags;
114 sdom = sha->domid;
116 for ( ; ; )
117 {
118 u32 scombo, prev_scombo, new_scombo;
120 if ( unlikely((sflags & GTF_type_mask) != GTF_permit_access) ||
121 unlikely(sdom != mapping_d->id) )
122 PIN_FAIL(unlock_out, GNTST_general_error,
123 "Bad flags (%x) or dom (%d). (NB. expected dom %d)\n",
124 sflags, sdom, mapping_d->id);
126 /* Merge two 16-bit values into a 32-bit combined update. */
127 /* NB. Endianness! */
128 prev_scombo = scombo = ((u32)sdom << 16) | (u32)sflags;
130 new_scombo = scombo | GTF_reading;
131 if ( !(dev_hst_ro_flags & GNTMAP_readonly) )
132 {
133 new_scombo |= GTF_writing;
134 if ( unlikely(sflags & GTF_readonly) )
135 PIN_FAIL(unlock_out, GNTST_general_error,
136 "Attempt to write-pin a r/o grant entry.\n");
137 }
139 /* NB. prev_scombo is updated in place to seen value. */
140 if ( unlikely(cmpxchg_user((u32 *)&sha->flags,
141 prev_scombo,
142 new_scombo)) )
143 PIN_FAIL(unlock_out, GNTST_general_error,
144 "Fault while modifying shared flags and domid.\n");
146 /* Did the combined update work (did we see what we expected?). */
147 if ( likely(prev_scombo == scombo) )
148 break;
150 if ( retries++ == 4 )
151 PIN_FAIL(unlock_out, GNTST_general_error,
152 "Shared grant entry is unstable.\n");
154 /* Didn't see what we expected. Split out the seen flags & dom. */
155 /* NB. Endianness! */
156 sflags = (u16)prev_scombo;
157 sdom = (u16)(prev_scombo >> 16);
158 }
160 /* rmb(); */ /* not on x86 */
162 frame = __gpfn_to_mfn_foreign(granting_d, sha->frame);
164 if ( unlikely(!pfn_valid(frame)) ||
165 unlikely(!((dev_hst_ro_flags & GNTMAP_readonly) ?
166 get_page(&frame_table[frame], granting_d) :
167 get_page_and_type(&frame_table[frame], granting_d,
168 PGT_writable_page))) )
169 {
170 clear_bit(_GTF_writing, &sha->flags);
171 clear_bit(_GTF_reading, &sha->flags);
172 PIN_FAIL(unlock_out, GNTST_general_error,
173 "Could not pin the granted frame (%lx)!\n", frame);
174 }
176 if ( dev_hst_ro_flags & GNTMAP_device_map )
177 act->pin += (dev_hst_ro_flags & GNTMAP_readonly) ?
178 GNTPIN_devr_inc : GNTPIN_devw_inc;
179 if ( dev_hst_ro_flags & GNTMAP_host_map )
180 act->pin += (dev_hst_ro_flags & GNTMAP_readonly) ?
181 GNTPIN_hstr_inc : GNTPIN_hstw_inc;
182 act->domid = sdom;
183 act->frame = frame;
184 }
185 else
186 {
187 /* CASE 2: Active modications to an already active entry. */
189 /*
190 * A cheesy check for possible pin-count overflow.
191 * A more accurate check cannot be done with a single comparison.
192 */
193 if ( (act->pin & 0x80808080U) != 0 )
194 PIN_FAIL(unlock_out, ENOSPC, "Risk of counter overflow %08x\n", act->pin);
196 frame = act->frame;
198 if ( !(dev_hst_ro_flags & GNTMAP_readonly) &&
199 !((sflags = sha->flags) & GTF_writing) )
200 {
201 for ( ; ; )
202 {
203 u16 prev_sflags;
205 if ( unlikely(sflags & GTF_readonly) )
206 PIN_FAIL(unlock_out, GNTST_general_error,
207 "Attempt to write-pin a r/o grant entry.\n");
209 prev_sflags = sflags;
211 /* NB. prev_sflags is updated in place to seen value. */
212 if ( unlikely(cmpxchg_user(&sha->flags, prev_sflags,
213 prev_sflags | GTF_writing)) )
214 PIN_FAIL(unlock_out, GNTST_general_error,
215 "Fault while modifying shared flags.\n");
217 if ( likely(prev_sflags == sflags) )
218 break;
220 if ( retries++ == 4 )
221 PIN_FAIL(unlock_out, GNTST_general_error,
222 "Shared grant entry is unstable.\n");
224 sflags = prev_sflags;
225 }
227 if ( unlikely(!get_page_type(&frame_table[frame],
228 PGT_writable_page)) )
229 {
230 clear_bit(_GTF_writing, &sha->flags);
231 PIN_FAIL(unlock_out, GNTST_general_error,
232 "Attempt to write-pin a unwritable page.\n");
233 }
234 }
236 if ( dev_hst_ro_flags & GNTMAP_device_map )
237 act->pin += (dev_hst_ro_flags & GNTMAP_readonly) ?
238 GNTPIN_devr_inc : GNTPIN_devw_inc;
239 if ( dev_hst_ro_flags & GNTMAP_host_map )
240 act->pin += (dev_hst_ro_flags & GNTMAP_readonly) ?
241 GNTPIN_hstr_inc : GNTPIN_hstw_inc;
242 }
244 /* At this point:
245 * act->pin updated to reflect mapping.
246 * sha->flags updated to indicate to granting domain mapping done.
247 * frame contains the mfn.
248 */
250 spin_unlock(&granting_d->grant_table->lock);
252 if ( (host_virt_addr != 0) && (dev_hst_ro_flags & GNTMAP_host_map) )
253 {
254 /* Write update into the pagetable
255 */
256 l1_pgentry_t pte;
258 pte = l1e_create_pfn(frame, _PAGE_PRESENT | _PAGE_ACCESSED | _PAGE_DIRTY);
259 if ( !(dev_hst_ro_flags & GNTMAP_readonly) )
260 l1e_add_flags(&pte,_PAGE_RW);
261 rc = update_grant_va_mapping( host_virt_addr, pte,
262 mapping_d, mapping_ed );
264 /* IMPORTANT: (rc == 0) => must flush / invalidate entry in TLB.
265 * This is done in the outer gnttab_map_grant_ref.
266 */
268 if ( 0 > rc )
269 {
270 /* Abort. */
272 spin_lock(&granting_d->grant_table->lock);
274 if ( dev_hst_ro_flags & GNTMAP_readonly )
275 act->pin -= GNTPIN_hstr_inc;
276 else
277 {
278 act->pin -= GNTPIN_hstw_inc;
279 if ( (act->pin & (GNTPIN_hstw_mask|GNTPIN_devw_mask)) == 0 )
280 {
281 clear_bit(_GTF_writing, &sha->flags);
282 put_page_type(&frame_table[frame]);
283 }
284 }
285 if ( act->pin == 0 )
286 {
287 clear_bit(_GTF_reading, &sha->flags);
288 put_page(&frame_table[frame]);
289 }
291 spin_unlock(&granting_d->grant_table->lock);
292 }
294 }
295 *pframe = frame;
296 return rc;
298 unlock_out:
299 spin_unlock(&granting_d->grant_table->lock);
300 return rc;
301 }
303 static int
304 __gnttab_map_grant_ref(
305 gnttab_map_grant_ref_t *uop,
306 unsigned long *va)
307 {
308 domid_t dom;
309 grant_ref_t ref;
310 struct domain *ld, *rd;
311 struct exec_domain *led;
312 u16 dev_hst_ro_flags;
313 int handle;
314 unsigned long frame, host_virt_addr;
315 int rc;
317 /* Returns 0 if TLB flush / invalidate required by caller.
318 * va will indicate the address to be invalidated. */
320 led = current;
321 ld = led->domain;
323 /* Bitwise-OR avoids short-circuiting which screws control flow. */
324 if ( unlikely(__get_user(dom, &uop->dom) |
325 __get_user(ref, &uop->ref) |
326 __get_user(host_virt_addr, &uop->host_virt_addr) |
327 __get_user(dev_hst_ro_flags, &uop->flags)) )
328 {
329 DPRINTK("Fault while reading gnttab_map_grant_ref_t.\n");
330 return -EFAULT; /* don't set status */
331 }
334 if ( ((host_virt_addr != 0) || (dev_hst_ro_flags & GNTMAP_host_map) ) &&
335 unlikely(!__addr_ok(host_virt_addr)))
336 {
337 DPRINTK("Bad virtual address (%x) or flags (%x).\n",
338 host_virt_addr, dev_hst_ro_flags);
339 (void)__put_user(GNTST_bad_virt_addr, &uop->handle);
340 return GNTST_bad_gntref;
341 }
343 if ( unlikely(ref >= NR_GRANT_ENTRIES) ||
344 unlikely((dev_hst_ro_flags & (GNTMAP_device_map|GNTMAP_host_map)) ==
345 0) )
346 {
347 DPRINTK("Bad ref (%d) or flags (%x).\n", ref, dev_hst_ro_flags);
348 (void)__put_user(GNTST_bad_gntref, &uop->handle);
349 return GNTST_bad_gntref;
350 }
352 if ( unlikely((rd = find_domain_by_id(dom)) == NULL) ||
353 unlikely(ld == rd) )
354 {
355 if ( rd != NULL )
356 put_domain(rd);
357 DPRINTK("Could not find domain %d\n", dom);
358 (void)__put_user(GNTST_bad_domain, &uop->handle);
359 return GNTST_bad_domain;
360 }
362 /* get a maptrack handle */
363 if ( unlikely((handle = get_maptrack_handle(ld->grant_table)) == -1) )
364 {
365 int i;
366 grant_mapping_t *new_mt;
367 grant_table_t *lgt = ld->grant_table;
369 /* grow the maptrack table */
370 if ( (new_mt = (void *)alloc_xenheap_pages(lgt->maptrack_order + 1)) == NULL )
371 {
372 put_domain(rd);
373 DPRINTK("No more map handles available\n");
374 (void)__put_user(GNTST_no_device_space, &uop->handle);
375 return GNTST_no_device_space;
376 }
378 memcpy(new_mt, lgt->maptrack, PAGE_SIZE << lgt->maptrack_order);
379 for ( i = lgt->maptrack_limit; i < (lgt->maptrack_limit << 1); i++ )
380 new_mt[i].ref_and_flags = (i+1) << MAPTRACK_REF_SHIFT;
382 free_xenheap_pages((unsigned long)lgt->maptrack, lgt->maptrack_order);
383 lgt->maptrack = new_mt;
384 lgt->maptrack_order += 1;
385 lgt->maptrack_limit <<= 1;
387 printk("Doubled maptrack size\n");
388 handle = get_maptrack_handle(ld->grant_table);
389 }
391 #if GRANT_DEBUG_VERBOSE
392 DPRINTK("Mapping grant ref (%hu) for domain (%hu) with flags (%x)\n",
393 ref, dom, dev_hst_ro_flags);
394 #endif
396 if ( 0 <= ( rc = __gnttab_activate_grant_ref( ld, led, rd, ref,
397 dev_hst_ro_flags,
398 host_virt_addr, &frame)))
399 {
400 /* Only make the maptrack live _after_ writing the pte,
401 * in case we overwrite the same frame number, causing a
402 * maptrack walk to find it
403 */
404 ld->grant_table->maptrack[handle].domid = dom;
406 ld->grant_table->maptrack[handle].ref_and_flags
407 = (ref << MAPTRACK_REF_SHIFT) |
408 (dev_hst_ro_flags & MAPTRACK_GNTMAP_MASK);
410 (void)__put_user(frame, &uop->dev_bus_addr);
412 if ( dev_hst_ro_flags & GNTMAP_host_map )
413 *va = host_virt_addr;
415 (void)__put_user(handle, &uop->handle);
416 }
417 else
418 {
419 (void)__put_user(rc, &uop->handle);
420 put_maptrack_handle(ld->grant_table, handle);
421 }
423 put_domain(rd);
424 return rc;
425 }
427 static long
428 gnttab_map_grant_ref(
429 gnttab_map_grant_ref_t *uop, unsigned int count)
430 {
431 int i, flush = 0;
432 unsigned long va[8];
434 for ( i = 0; i < count; i++ )
435 if ( __gnttab_map_grant_ref(&uop[i],
436 &va[ (flush < 8 ? flush : 0) ] ) == 0)
437 flush++;
439 if ( flush != 0 )
440 {
441 if ( flush <= 8 )
442 for ( i = 0; i < flush; i++ )
443 flush_tlb_one_mask(current->domain->cpuset, va[i]);
444 else
445 local_flush_tlb();
446 }
448 return 0;
449 }
451 static int
452 __gnttab_unmap_grant_ref(
453 gnttab_unmap_grant_ref_t *uop,
454 unsigned long *va)
455 {
456 domid_t dom;
457 grant_ref_t ref;
458 u16 handle;
459 struct domain *ld, *rd;
461 active_grant_entry_t *act;
462 grant_entry_t *sha;
463 grant_mapping_t *map;
464 u16 flags;
465 s16 rc = 1;
466 unsigned long frame, virt;
468 ld = current->domain;
470 /* Bitwise-OR avoids short-circuiting which screws control flow. */
471 if ( unlikely(__get_user(virt, &uop->host_virt_addr) |
472 __get_user(frame, &uop->dev_bus_addr) |
473 __get_user(handle, &uop->handle)) )
474 {
475 DPRINTK("Fault while reading gnttab_unmap_grant_ref_t.\n");
476 return -EFAULT; /* don't set status */
477 }
479 map = &ld->grant_table->maptrack[handle];
481 if ( unlikely(handle >= ld->grant_table->maptrack_limit) ||
482 unlikely(!(map->ref_and_flags & MAPTRACK_GNTMAP_MASK)) )
483 {
484 DPRINTK("Bad handle (%d).\n", handle);
485 (void)__put_user(GNTST_bad_handle, &uop->status);
486 return GNTST_bad_handle;
487 }
489 dom = map->domid;
490 ref = map->ref_and_flags >> MAPTRACK_REF_SHIFT;
491 flags = map->ref_and_flags & MAPTRACK_GNTMAP_MASK;
493 if ( unlikely((rd = find_domain_by_id(dom)) == NULL) ||
494 unlikely(ld == rd) )
495 {
496 if ( rd != NULL )
497 put_domain(rd);
498 DPRINTK("Could not find domain %d\n", dom);
499 (void)__put_user(GNTST_bad_domain, &uop->status);
500 return GNTST_bad_domain;
501 }
502 #if GRANT_DEBUG_VERBOSE
503 DPRINTK("Unmapping grant ref (%hu) for domain (%hu) with handle (%hu)\n",
504 ref, dom, handle);
505 #endif
507 act = &rd->grant_table->active[ref];
508 sha = &rd->grant_table->shared[ref];
510 spin_lock(&rd->grant_table->lock);
512 if ( frame == 0 )
513 frame = act->frame;
514 else if ( frame == GNTUNMAP_DEV_FROM_VIRT )
515 {
516 if ( !( flags & GNTMAP_device_map ) )
517 PIN_FAIL(unmap_out, GNTST_bad_dev_addr,
518 "Bad frame number: frame not mapped for device access.\n");
519 frame = act->frame;
521 /* frame will be unmapped for device access below if virt addr ok */
522 }
523 else
524 {
525 if ( unlikely(frame != act->frame) )
526 PIN_FAIL(unmap_out, GNTST_general_error,
527 "Bad frame number doesn't match gntref.\n");
528 if ( flags & GNTMAP_device_map )
529 act->pin -= (flags & GNTMAP_readonly) ? GNTPIN_devr_inc
530 : GNTPIN_devw_inc;
532 map->ref_and_flags &= ~GNTMAP_device_map;
533 (void)__put_user(0, &uop->dev_bus_addr);
535 /* frame is now unmapped for device access */
536 }
538 if ( (virt != 0) &&
539 (flags & GNTMAP_host_map) &&
540 ((act->pin & (GNTPIN_hstw_mask | GNTPIN_hstr_mask)) > 0))
541 {
542 l1_pgentry_t *pl1e;
543 unsigned long _ol1e;
545 pl1e = &linear_pg_table[l1_linear_offset(virt)];
547 if ( unlikely(__get_user(_ol1e, (unsigned long *)pl1e) != 0) )
548 {
549 DPRINTK("Could not find PTE entry for address %x\n", virt);
550 rc = -EINVAL;
551 goto unmap_out;
552 }
554 /* check that the virtual address supplied is actually
555 * mapped to act->frame.
556 */
557 if ( unlikely((_ol1e >> PAGE_SHIFT) != frame ))
558 {
559 DPRINTK("PTE entry %x for address %x doesn't match frame %x\n",
560 _ol1e, virt, frame);
561 rc = -EINVAL;
562 goto unmap_out;
563 }
565 /* Delete pagetable entry
566 */
567 if ( unlikely(__put_user(0, (unsigned long *)pl1e)))
568 {
569 DPRINTK("Cannot delete PTE entry at %x for virtual address %x\n",
570 pl1e, virt);
571 rc = -EINVAL;
572 goto unmap_out;
573 }
575 map->ref_and_flags &= ~GNTMAP_host_map;
577 act->pin -= (flags & GNTMAP_readonly) ? GNTPIN_hstr_inc
578 : GNTPIN_hstw_inc;
580 if ( frame == GNTUNMAP_DEV_FROM_VIRT )
581 {
582 act->pin -= (flags & GNTMAP_readonly) ? GNTPIN_devr_inc
583 : GNTPIN_devw_inc;
585 map->ref_and_flags &= ~GNTMAP_device_map;
586 (void)__put_user(0, &uop->dev_bus_addr);
587 }
589 rc = 0;
590 *va = virt;
591 }
593 if ( (map->ref_and_flags & (GNTMAP_device_map|GNTMAP_host_map)) == 0)
594 {
595 map->ref_and_flags = 0;
596 put_maptrack_handle(ld->grant_table, handle);
597 }
599 /* If just unmapped a writable mapping, mark as dirtied */
600 if ( unlikely(shadow_mode_log_dirty(rd)) &&
601 !( flags & GNTMAP_readonly ) )
602 mark_dirty(rd, frame);
604 /* If the last writable mapping has been removed, put_page_type */
605 if ( ( (act->pin & (GNTPIN_devw_mask|GNTPIN_hstw_mask) ) == 0) &&
606 ( !( flags & GNTMAP_readonly ) ) )
607 {
608 clear_bit(_GTF_writing, &sha->flags);
609 put_page_type(&frame_table[frame]);
610 }
612 if ( act->pin == 0 )
613 {
614 clear_bit(_GTF_reading, &sha->flags);
615 put_page(&frame_table[frame]);
616 }
618 unmap_out:
619 (void)__put_user(rc, &uop->status);
620 spin_unlock(&rd->grant_table->lock);
621 put_domain(rd);
622 return rc;
623 }
625 static long
626 gnttab_unmap_grant_ref(
627 gnttab_unmap_grant_ref_t *uop, unsigned int count)
628 {
629 int i, flush = 0;
630 unsigned long va[8];
632 for ( i = 0; i < count; i++ )
633 if ( __gnttab_unmap_grant_ref(&uop[i],
634 &va[ (flush < 8 ? flush : 0) ] ) == 0)
635 flush++;
637 if ( flush != 0 )
638 {
639 if ( flush <= 8 )
640 for ( i = 0; i < flush; i++ )
641 flush_tlb_one_mask(current->domain->cpuset, va[i]);
642 else
643 local_flush_tlb();
644 }
646 return 0;
647 }
649 static long
650 gnttab_setup_table(
651 gnttab_setup_table_t *uop, unsigned int count)
652 {
653 gnttab_setup_table_t op;
654 struct domain *d;
655 int i;
657 if ( count != 1 )
658 return -EINVAL;
660 if ( unlikely(copy_from_user(&op, uop, sizeof(op)) != 0) )
661 {
662 DPRINTK("Fault while reading gnttab_setup_table_t.\n");
663 return -EFAULT;
664 }
666 if ( unlikely(op.nr_frames > NR_GRANT_FRAMES) )
667 {
668 DPRINTK("Xen only supports at most %d grant-table frames per domain.\n",
669 NR_GRANT_FRAMES);
670 (void)put_user(GNTST_general_error, &uop->status);
671 return 0;
672 }
674 if ( op.dom == DOMID_SELF )
675 {
676 op.dom = current->domain->id;
677 }
678 else if ( unlikely(!IS_PRIV(current->domain)) )
679 {
680 (void)put_user(GNTST_permission_denied, &uop->status);
681 return 0;
682 }
684 if ( unlikely((d = find_domain_by_id(op.dom)) == NULL) )
685 {
686 DPRINTK("Bad domid %d.\n", op.dom);
687 (void)put_user(GNTST_bad_domain, &uop->status);
688 return 0;
689 }
691 if ( op.nr_frames <= NR_GRANT_FRAMES )
692 {
693 ASSERT(d->grant_table != NULL);
694 (void)put_user(GNTST_okay, &uop->status);
696 for ( i = 0; i < op.nr_frames; i++ )
697 (void)put_user( (
698 virt_to_phys( (char*)(d->grant_table->shared)+(i*PAGE_SIZE) )
699 >> PAGE_SHIFT ), &uop->frame_list[i]);
700 }
702 put_domain(d);
703 return 0;
704 }
706 #if GRANT_DEBUG
707 static int
708 gnttab_dump_table(gnttab_dump_table_t *uop)
709 {
710 grant_table_t *gt;
711 gnttab_dump_table_t op;
712 struct domain *d;
713 u32 shared_mfn;
714 active_grant_entry_t *act;
715 grant_entry_t sha_copy;
716 grant_mapping_t *maptrack;
717 int i;
720 if ( unlikely(copy_from_user(&op, uop, sizeof(op)) != 0) )
721 {
722 DPRINTK("Fault while reading gnttab_dump_table_t.\n");
723 return -EFAULT;
724 }
726 if ( op.dom == DOMID_SELF )
727 {
728 op.dom = current->domain->id;
729 }
731 if ( unlikely((d = find_domain_by_id(op.dom)) == NULL) )
732 {
733 DPRINTK("Bad domid %d.\n", op.dom);
734 (void)put_user(GNTST_bad_domain, &uop->status);
735 return 0;
736 }
738 ASSERT(d->grant_table != NULL);
739 gt = d->grant_table;
740 (void)put_user(GNTST_okay, &uop->status);
742 shared_mfn = virt_to_phys(d->grant_table->shared);
744 DPRINTK("Grant table for dom (%hu) MFN (%x)\n",
745 op.dom, shared_mfn);
747 ASSERT(d->grant_table->active != NULL);
748 ASSERT(d->grant_table->shared != NULL);
749 ASSERT(d->grant_table->maptrack != NULL);
751 for ( i = 0; i < NR_GRANT_ENTRIES; i++ )
752 {
753 sha_copy = gt->shared[i];
755 if ( sha_copy.flags )
756 {
757 DPRINTK("Grant: dom (%hu) SHARED (%d) flags:(%hx) dom:(%hu) frame:(%lx)\n",
758 op.dom, i, sha_copy.flags, sha_copy.domid, sha_copy.frame);
759 }
760 }
762 spin_lock(&gt->lock);
764 for ( i = 0; i < NR_GRANT_ENTRIES; i++ )
765 {
766 act = &gt->active[i];
768 if ( act->pin )
769 {
770 DPRINTK("Grant: dom (%hu) ACTIVE (%d) pin:(%x) dom:(%hu) frame:(%lx)\n",
771 op.dom, i, act->pin, act->domid, act->frame);
772 }
773 }
775 for ( i = 0; i < gt->maptrack_limit; i++ )
776 {
777 maptrack = &gt->maptrack[i];
779 if ( maptrack->ref_and_flags & MAPTRACK_GNTMAP_MASK )
780 {
781 DPRINTK("Grant: dom (%hu) MAP (%d) ref:(%hu) flags:(%x) dom:(%hu)\n",
782 op.dom, i,
783 maptrack->ref_and_flags >> MAPTRACK_REF_SHIFT,
784 maptrack->ref_and_flags & MAPTRACK_GNTMAP_MASK,
785 maptrack->domid);
786 }
787 }
789 spin_unlock(&gt->lock);
791 put_domain(d);
792 return 0;
793 }
794 #endif
796 long
797 do_grant_table_op(
798 unsigned int cmd, void *uop, unsigned int count)
799 {
800 long rc;
802 if ( count > 512 )
803 return -EINVAL;
805 LOCK_BIGLOCK(current->domain);
807 rc = -EFAULT;
808 switch ( cmd )
809 {
810 case GNTTABOP_map_grant_ref:
811 if ( unlikely(!array_access_ok(
812 VERIFY_WRITE, uop, count, sizeof(gnttab_map_grant_ref_t))) )
813 goto out;
814 rc = gnttab_map_grant_ref((gnttab_map_grant_ref_t *)uop, count);
815 break;
816 case GNTTABOP_unmap_grant_ref:
817 if ( unlikely(!array_access_ok(
818 VERIFY_WRITE, uop, count, sizeof(gnttab_unmap_grant_ref_t))) )
819 goto out;
820 rc = gnttab_unmap_grant_ref((gnttab_unmap_grant_ref_t *)uop, count);
821 break;
822 case GNTTABOP_setup_table:
823 rc = gnttab_setup_table((gnttab_setup_table_t *)uop, count);
824 break;
825 #if GRANT_DEBUG
826 case GNTTABOP_dump_table:
827 rc = gnttab_dump_table((gnttab_dump_table_t *)uop);
828 break;
829 #endif
830 default:
831 rc = -ENOSYS;
832 break;
833 }
835 out:
836 UNLOCK_BIGLOCK(current->domain);
838 return rc;
839 }
841 int
842 gnttab_check_unmap(
843 struct domain *rd, struct domain *ld, unsigned long frame, int readonly)
844 {
845 /* Called when put_page is invoked on a page belonging to a foreign domain.
846 * Instead of decrementing the frame table ref count, locate the grant
847 * table entry, if any, and if found, decrement that count.
848 * Called a _lot_ at domain creation because pages mapped by priv domains
849 * also traverse this.
850 */
852 /* Note: if the same frame is mapped multiple times, and then one of
853 * the ptes is overwritten, which maptrack handle gets invalidated?
854 * Advice: don't do it.
855 */
857 unsigned int handle, ref, refcount;
858 grant_table_t *lgt, *rgt;
859 active_grant_entry_t *act;
860 grant_mapping_t *map;
861 int found = 0;
863 lgt = ld->grant_table;
865 #if GRANT_DEBUG_VERBOSE
866 if ( ld->id != 0 )
867 {
868 DPRINTK("Foreign unref rd(%d) ld(%d) frm(%x) flgs(%x).\n",
869 rd->id, ld->id, frame, readonly);
870 }
871 #endif
873 /* Fast exit if we're not mapping anything using grant tables */
874 if ( lgt->map_count == 0 )
875 return 0;
877 if ( get_domain(rd) == 0 )
878 {
879 DPRINTK("gnttab_check_unmap: couldn't get_domain rd(%d)\n", rd->id);
880 return 0;
881 }
883 rgt = rd->grant_table;
885 for ( handle = 0; handle < lgt->maptrack_limit; handle++ )
886 {
887 map = &lgt->maptrack[handle];
890 /* cwc22: if multiple grants of the same frame are disallowed,
891 * then the readonly check here can be changed to cause an early abort
892 * if we've matched on frame, but not on write permission.
893 */
894 if ( ( map->ref_and_flags & MAPTRACK_GNTMAP_MASK ) &&
895 ( readonly ? 1 : (!(map->ref_and_flags & GNTMAP_readonly))))
896 {
897 ref = (map->ref_and_flags >> MAPTRACK_REF_SHIFT);
898 act = &rgt->active[ref];
900 spin_lock(&rgt->lock);
902 if ( act->frame != frame )
903 {
904 spin_unlock(&rgt->lock);
905 continue;
906 }
908 refcount = act->pin & ( readonly ? GNTPIN_hstr_mask
909 : GNTPIN_hstw_mask );
910 if ( refcount == 0 )
911 {
912 spin_unlock(&rgt->lock);
913 continue;
914 }
916 /* gotcha */
917 DPRINTK("Grant unref rd(%d) ld(%d) frm(%x) flgs(%x).\n",
918 rd->id, ld->id, frame, readonly);
920 if ( readonly )
921 act->pin -= GNTPIN_hstr_inc;
922 else
923 {
924 act->pin -= GNTPIN_hstw_inc;
926 /* any more granted writable mappings? */
927 if ( (act->pin & (GNTPIN_hstw_mask|GNTPIN_devw_mask)) == 0 )
928 {
929 clear_bit(_GTF_writing, &rgt->shared[ref].flags);
930 put_page_type(&frame_table[frame]);
931 }
932 }
934 if ( act->pin == 0 )
935 {
936 clear_bit(_GTF_reading, &rgt->shared[ref].flags);
937 put_page(&frame_table[frame]);
938 }
939 spin_unlock(&rgt->lock);
941 clear_bit(GNTMAP_host_map, &map->ref_and_flags);
943 if ( !(map->ref_and_flags & GNTMAP_device_map) )
944 put_maptrack_handle(lgt, handle);
946 found = 1;
947 break;
948 }
949 }
950 put_domain(rd);
952 return found;
953 }
955 int
956 gnttab_prepare_for_transfer(
957 struct domain *rd, struct domain *ld, grant_ref_t ref)
958 {
959 grant_table_t *rgt;
960 grant_entry_t *sha;
961 domid_t sdom;
962 u16 sflags;
963 u32 scombo, prev_scombo;
964 int retries = 0;
965 unsigned long target_pfn;
967 DPRINTK("gnttab_prepare_for_transfer rd(%hu) ld(%hu) ref(%hu).\n",
968 rd->id, ld->id, ref);
970 if ( unlikely((rgt = rd->grant_table) == NULL) ||
971 unlikely(ref >= NR_GRANT_ENTRIES) )
972 {
973 DPRINTK("Dom %d has no g.t., or ref is bad (%d).\n", rd->id, ref);
974 return 0;
975 }
977 spin_lock(&rgt->lock);
979 sha = &rgt->shared[ref];
981 sflags = sha->flags;
982 sdom = sha->domid;
984 for ( ; ; )
985 {
986 target_pfn = sha->frame;
988 if ( unlikely(target_pfn >= max_page ) )
989 {
990 DPRINTK("Bad pfn (%x)\n", target_pfn);
991 goto fail;
992 }
994 if ( unlikely(sflags != GTF_accept_transfer) ||
995 unlikely(sdom != ld->id) )
996 {
997 DPRINTK("Bad flags (%x) or dom (%d). (NB. expected dom %d)\n",
998 sflags, sdom, ld->id);
999 goto fail;
1002 /* Merge two 16-bit values into a 32-bit combined update. */
1003 /* NB. Endianness! */
1004 prev_scombo = scombo = ((u32)sdom << 16) | (u32)sflags;
1006 /* NB. prev_scombo is updated in place to seen value. */
1007 if ( unlikely(cmpxchg_user((u32 *)&sha->flags, prev_scombo,
1008 prev_scombo | GTF_transfer_committed)) )
1010 DPRINTK("Fault while modifying shared flags and domid.\n");
1011 goto fail;
1014 /* Did the combined update work (did we see what we expected?). */
1015 if ( likely(prev_scombo == scombo) )
1016 break;
1018 if ( retries++ == 4 )
1020 DPRINTK("Shared grant entry is unstable.\n");
1021 goto fail;
1024 /* Didn't see what we expected. Split out the seen flags & dom. */
1025 /* NB. Endianness! */
1026 sflags = (u16)prev_scombo;
1027 sdom = (u16)(prev_scombo >> 16);
1030 spin_unlock(&rgt->lock);
1031 return 1;
1033 fail:
1034 spin_unlock(&rgt->lock);
1035 return 0;
1038 void
1039 gnttab_notify_transfer(
1040 struct domain *rd, struct domain *ld, grant_ref_t ref, unsigned long frame)
1042 grant_entry_t *sha;
1043 unsigned long pfn;
1045 DPRINTK("gnttab_notify_transfer rd(%hu) ld(%hu) ref(%hu).\n",
1046 rd->id, ld->id, ref);
1048 sha = &rd->grant_table->shared[ref];
1050 spin_lock(&rd->grant_table->lock);
1052 pfn = sha->frame;
1054 if ( unlikely(pfn >= max_page ) )
1055 DPRINTK("Bad pfn (%x)\n", pfn);
1056 else
1058 machine_to_phys_mapping[frame] = pfn;
1060 if ( unlikely(shadow_mode_log_dirty(ld)))
1061 mark_dirty(ld, frame);
1063 if (shadow_mode_translate(ld))
1064 __phys_to_machine_mapping[pfn] = frame;
1066 sha->frame = __mfn_to_gpfn(rd, frame);
1067 sha->domid = rd->id;
1068 wmb();
1069 sha->flags = ( GTF_accept_transfer | GTF_transfer_completed );
1071 spin_unlock(&rd->grant_table->lock);
1073 return;
1076 int
1077 grant_table_create(
1078 struct domain *d)
1080 grant_table_t *t;
1081 int i;
1083 if ( (t = xmalloc(grant_table_t)) == NULL )
1084 goto no_mem;
1086 /* Simple stuff. */
1087 memset(t, 0, sizeof(*t));
1088 spin_lock_init(&t->lock);
1090 /* Active grant table. */
1091 if ( (t->active = xmalloc_array(active_grant_entry_t, NR_GRANT_ENTRIES))
1092 == NULL )
1093 goto no_mem;
1094 memset(t->active, 0, sizeof(active_grant_entry_t) * NR_GRANT_ENTRIES);
1096 /* Tracking of mapped foreign frames table */
1097 if ( (t->maptrack = (void *)alloc_xenheap_page()) == NULL )
1098 goto no_mem;
1099 t->maptrack_order = 0;
1100 t->maptrack_limit = PAGE_SIZE / sizeof(grant_mapping_t);
1101 memset(t->maptrack, 0, PAGE_SIZE);
1102 for ( i = 0; i < t->maptrack_limit; i++ )
1103 t->maptrack[i].ref_and_flags = (i+1) << MAPTRACK_REF_SHIFT;
1105 /* Shared grant table. */
1106 if ( (t->shared = (void *)alloc_xenheap_pages(ORDER_GRANT_FRAMES)) == NULL )
1107 goto no_mem;
1108 memset(t->shared, 0, NR_GRANT_FRAMES * PAGE_SIZE);
1110 for ( i = 0; i < NR_GRANT_FRAMES; i++ )
1112 SHARE_PFN_WITH_DOMAIN(virt_to_page((char *)(t->shared)+(i*PAGE_SIZE)), d);
1113 machine_to_phys_mapping[ (virt_to_phys((char*)(t->shared)+(i*PAGE_SIZE))
1114 >> PAGE_SHIFT) ] = INVALID_M2P_ENTRY;
1117 /* Okay, install the structure. */
1118 wmb(); /* avoid races with lock-free access to d->grant_table */
1119 d->grant_table = t;
1120 return 0;
1122 no_mem:
1123 if ( t != NULL )
1125 if ( t->active != NULL )
1126 xfree(t->active);
1127 if ( t->maptrack != NULL )
1128 free_xenheap_page((unsigned long)t->maptrack);
1129 xfree(t);
1131 return -ENOMEM;
1134 void
1135 gnttab_release_dev_mappings(grant_table_t *gt)
1137 grant_mapping_t *map;
1138 domid_t dom;
1139 grant_ref_t ref;
1140 u16 handle;
1141 struct domain *ld, *rd;
1142 unsigned long frame;
1143 active_grant_entry_t *act;
1144 grant_entry_t *sha;
1146 ld = current->domain;
1148 for ( handle = 0; handle < gt->maptrack_limit; handle++ )
1150 map = &gt->maptrack[handle];
1152 if ( map->ref_and_flags & GNTMAP_device_map )
1154 dom = map->domid;
1155 ref = map->ref_and_flags >> MAPTRACK_REF_SHIFT;
1157 DPRINTK("Grant release (%hu) ref:(%hu) flags:(%x) dom:(%hu)\n",
1158 handle, ref,
1159 map->ref_and_flags & MAPTRACK_GNTMAP_MASK, dom);
1161 if ( unlikely((rd = find_domain_by_id(dom)) == NULL) ||
1162 unlikely(ld == rd) )
1164 if ( rd != NULL )
1165 put_domain(rd);
1167 printk(KERN_WARNING "Grant release: Could not find domain %d\n", dom);
1168 continue;
1171 act = &rd->grant_table->active[ref];
1172 sha = &rd->grant_table->shared[ref];
1174 spin_lock(&rd->grant_table->lock);
1176 if ( act->pin & (GNTPIN_devw_mask | GNTPIN_devr_mask) )
1178 frame = act->frame;
1180 if ( ( (act->pin & GNTPIN_hstw_mask) == 0 ) &&
1181 ( (act->pin & GNTPIN_devw_mask) > 0 ) )
1183 clear_bit(_GTF_writing, &sha->flags);
1184 put_page_type(&frame_table[frame]);
1187 act->pin &= ~(GNTPIN_devw_mask | GNTPIN_devr_mask);
1189 if ( act->pin == 0 )
1191 clear_bit(_GTF_reading, &sha->flags);
1192 map->ref_and_flags = 0;
1193 put_page(&frame_table[frame]);
1195 else
1196 map->ref_and_flags &= ~GNTMAP_device_map;
1199 spin_unlock(&rd->grant_table->lock);
1201 put_domain(rd);
1207 void
1208 grant_table_destroy(
1209 struct domain *d)
1211 grant_table_t *t;
1213 if ( (t = d->grant_table) != NULL )
1215 /* Free memory relating to this grant table. */
1216 d->grant_table = NULL;
1217 free_xenheap_pages((unsigned long)t->shared, ORDER_GRANT_FRAMES);
1218 free_xenheap_page((unsigned long)t->maptrack); //cwc22
1219 xfree(t->active);
1220 xfree(t);
1224 void
1225 grant_table_init(
1226 void)
1228 /* Nothing. */
1229 DPRINTK("Grant table init\n");
1232 /*
1233 * Local variables:
1234 * mode: C
1235 * c-set-style: "BSD"
1236 * c-basic-offset: 4
1237 * tab-width: 4
1238 * indent-tabs-mode: nil
1239 * End:
1240 */