debuggers.hg

annotate xen/include/asm-x86/shadow.h @ 3664:8472fafee3cf

bitkeeper revision 1.1159.212.74 (42015ef3sPQp8pjeJAck1wBtTAYL9g)

Interface to typed allocator is now just xmalloc/xmalloc_array/xfree.
_xmalloc/_xmalloc_array are dead (or, at least, non-API).
Signed-off-by: keir.fraser@cl.cam.ac.uk
author kaf24@scramble.cl.cam.ac.uk
date Wed Feb 02 23:14:59 2005 +0000 (2005-02-02)
parents 0ef6e8e6e85d
children 07d5c9548534
rev   line source
djm@1686 1 /* -*- Mode:C; c-basic-offset:4; tab-width:4 -*- */
djm@1686 2
djm@1686 3 #ifndef _XEN_SHADOW_H
djm@1686 4 #define _XEN_SHADOW_H
djm@1686 5
djm@1686 6 #include <xen/config.h>
djm@1686 7 #include <xen/types.h>
djm@1686 8 #include <xen/perfc.h>
djm@1686 9 #include <asm/processor.h>
djm@1686 10
djm@1686 11 /* Shadow PT flag bits in pfn_info */
djm@1686 12 #define PSH_shadowed (1<<31) /* page has a shadow. PFN points to shadow */
djm@1686 13 #define PSH_pfn_mask ((1<<21)-1)
djm@1686 14
djm@1686 15 /* Shadow PT operation mode : shadowmode variable in mm_struct */
djm@1686 16 #define SHM_test (1) /* just run domain on shadow PTs */
djm@1686 17 #define SHM_logdirty (2) /* log pages that are dirtied */
djm@1686 18 #define SHM_translate (3) /* lookup machine pages in translation table */
kaf24@2673 19 #define SHM_cow (4) /* copy on write all dirtied pages */
iap10@3328 20 #define SHM_full_32 (8) /* full virtualization for 32-bit */
djm@1686 21
djm@1686 22 #define shadow_linear_pg_table ((l1_pgentry_t *)SH_LINEAR_PT_VIRT_START)
kaf24@2673 23 #define shadow_linear_l2_table ((l2_pgentry_t *)(SH_LINEAR_PT_VIRT_START + \
kaf24@2673 24 (SH_LINEAR_PT_VIRT_START >> (L2_PAGETABLE_SHIFT - L1_PAGETABLE_SHIFT))))
djm@1686 25
kaf24@1787 26 #define shadow_mode(_d) ((_d)->mm.shadow_mode)
kaf24@1787 27 #define shadow_lock_init(_d) spin_lock_init(&(_d)->mm.shadow_lock)
kaf24@2766 28 #define shadow_lock(_m) spin_lock(&(_m)->shadow_lock)
kaf24@2766 29 #define shadow_unlock(_m) spin_unlock(&(_m)->shadow_lock)
kaf24@1787 30
djm@1686 31 extern void shadow_mode_init(void);
kaf24@1787 32 extern int shadow_mode_control(struct domain *p, dom0_shadow_control_t *sc);
kaf24@1787 33 extern int shadow_fault(unsigned long va, long error_code);
kaf24@2673 34 extern void shadow_l1_normal_pt_update(
kaf24@2673 35 unsigned long pa, unsigned long gpte,
kaf24@2673 36 unsigned long *prev_spfn_ptr, l1_pgentry_t **prev_spl1e_ptr);
kaf24@1787 37 extern void shadow_l2_normal_pt_update(unsigned long pa, unsigned long gpte);
kaf24@1787 38 extern void unshadow_table(unsigned long gpfn, unsigned int type);
kaf24@1787 39 extern int shadow_mode_enable(struct domain *p, unsigned int mode);
kaf24@1787 40
iap10@3328 41 #ifdef CONFIG_VMX
iap10@3328 42 extern void vmx_shadow_clear_state(struct mm_struct *);
iap10@3328 43 extern void vmx_shadow_invlpg(struct mm_struct *, unsigned long);
iap10@3328 44 #endif
iap10@3328 45
iap10@3328 46 #define __get_machine_to_phys(m, guest_gpfn, gpfn) \
iap10@3328 47 if ((m)->shadow_mode == SHM_full_32) \
iap10@3328 48 (guest_gpfn) = machine_to_phys_mapping[(gpfn)]; \
iap10@3328 49 else \
iap10@3328 50 (guest_gpfn) = (gpfn);
iap10@3328 51
iap10@3328 52 #define __get_phys_to_machine(m, host_gpfn, gpfn) \
iap10@3328 53 if ((m)->shadow_mode == SHM_full_32) \
iap10@3328 54 (host_gpfn) = phys_to_machine_mapping[(gpfn)]; \
iap10@3328 55 else \
iap10@3328 56 (host_gpfn) = (gpfn);
iap10@3328 57
kaf24@1787 58 extern void __shadow_mode_disable(struct domain *d);
kaf24@1787 59 static inline void shadow_mode_disable(struct domain *d)
kaf24@1787 60 {
cl349@2957 61 if ( shadow_mode(d->exec_domain[0]) )
kaf24@1787 62 __shadow_mode_disable(d);
kaf24@1787 63 }
kaf24@1787 64
djm@1686 65 extern unsigned long shadow_l2_table(
kaf24@2673 66 struct mm_struct *m, unsigned long gpfn);
iap10@3328 67
iap10@3328 68 static inline void shadow_invalidate(struct mm_struct *m) {
iap10@3328 69 if (m->shadow_mode != SHM_full_32)
iap10@3328 70 BUG();
iap10@3328 71 memset(m->shadow_vtable, 0, PAGE_SIZE);
iap10@3328 72 }
djm@1686 73
iap10@3328 74 #define SHADOW_DEBUG 0
djm@1686 75 #define SHADOW_HASH_DEBUG 0
djm@1686 76
djm@1686 77 struct shadow_status {
kaf24@2673 78 unsigned long pfn; /* Guest pfn. */
kaf24@2673 79 unsigned long spfn_and_flags; /* Shadow pfn plus flags. */
kaf24@2673 80 struct shadow_status *next; /* Pull-to-front list. */
djm@1686 81 };
djm@1686 82
kaf24@2673 83 #define shadow_ht_extra_size 128
kaf24@2673 84 #define shadow_ht_buckets 256
djm@1686 85
kaf24@2097 86 #ifdef VERBOSE
djm@1686 87 #define SH_LOG(_f, _a...) \
djm@1686 88 printk("DOM%u: (file=shadow.c, line=%d) " _f "\n", \
cl349@2957 89 current->domain->id , __LINE__ , ## _a )
djm@1686 90 #else
djm@1686 91 #define SH_LOG(_f, _a...)
djm@1686 92 #endif
djm@1686 93
djm@1686 94 #if SHADOW_DEBUG
djm@1686 95 #define SH_VLOG(_f, _a...) \
djm@1686 96 printk("DOM%u: (file=shadow.c, line=%d) " _f "\n", \
kaf24@2748 97 current->id , __LINE__ , ## _a )
djm@1686 98 #else
djm@1686 99 #define SH_VLOG(_f, _a...)
djm@1686 100 #endif
djm@1686 101
djm@1686 102 #if 0
djm@1686 103 #define SH_VVLOG(_f, _a...) \
djm@1686 104 printk("DOM%u: (file=shadow.c, line=%d) " _f "\n", \
kaf24@2748 105 current->id , __LINE__ , ## _a )
djm@1686 106 #else
iap10@3328 107 #define SH_VVLOG(_f, _a...)
djm@1686 108 #endif
djm@1686 109
iap10@3328 110 static inline void __shadow_get_pl2e(struct mm_struct *m,
iap10@3328 111 unsigned long va, unsigned long *sl2e)
iap10@3328 112 {
iap10@3328 113 if (m->shadow_mode == SHM_full_32) {
iap10@3328 114 *sl2e = l2_pgentry_val(m->shadow_vtable[va >> L2_PAGETABLE_SHIFT]);
iap10@3328 115 }
iap10@3328 116 else
iap10@3328 117 *sl2e = l2_pgentry_val(linear_l2_table[va >> L2_PAGETABLE_SHIFT]);
iap10@3328 118 }
iap10@3328 119
iap10@3328 120 static inline void __shadow_set_pl2e(struct mm_struct *m,
iap10@3328 121 unsigned long va, unsigned long value)
iap10@3328 122 {
iap10@3328 123 if (m->shadow_mode == SHM_full_32) {
iap10@3328 124 m->shadow_vtable[va >> L2_PAGETABLE_SHIFT] = mk_l2_pgentry(value);
iap10@3328 125 }
iap10@3328 126 else
iap10@3328 127 linear_l2_table[va >> L2_PAGETABLE_SHIFT] = mk_l2_pgentry(value);
iap10@3328 128 }
iap10@3328 129
iap10@3328 130 static inline void __guest_get_pl2e(struct mm_struct *m,
iap10@3328 131 unsigned long va, unsigned long *l2e)
iap10@3328 132 {
iap10@3328 133 if (m->shadow_mode == SHM_full_32) {
iap10@3328 134 *l2e = l2_pgentry_val(m->vpagetable[va >> L2_PAGETABLE_SHIFT]);
iap10@3328 135 }
iap10@3328 136 else
iap10@3328 137 *l2e = l2_pgentry_val(linear_l2_table[va >> L2_PAGETABLE_SHIFT]);
iap10@3328 138 }
iap10@3328 139
iap10@3328 140 static inline void __guest_set_pl2e(struct mm_struct *m,
iap10@3328 141 unsigned long va, unsigned long value)
iap10@3328 142 {
iap10@3328 143 if (m->shadow_mode == SHM_full_32) {
iap10@3328 144 unsigned long pfn;
iap10@3328 145
iap10@3328 146 pfn = phys_to_machine_mapping[value >> PAGE_SHIFT];
iap10@3328 147 m->guest_pl2e_cache[va >> L2_PAGETABLE_SHIFT] =
iap10@3328 148 mk_l2_pgentry((pfn << PAGE_SHIFT) | __PAGE_HYPERVISOR);
iap10@3328 149
iap10@3328 150 m->vpagetable[va >> L2_PAGETABLE_SHIFT] = mk_l2_pgentry(value);
iap10@3328 151 }
iap10@3328 152 else
iap10@3328 153 linear_l2_table[va >> L2_PAGETABLE_SHIFT] = mk_l2_pgentry(value);
iap10@3328 154
iap10@3328 155 }
djm@1686 156
djm@1686 157 /************************************************************************/
djm@1686 158
kaf24@2673 159 static inline int __mark_dirty( struct mm_struct *m, unsigned int mfn)
djm@1686 160 {
kaf24@2673 161 unsigned long pfn;
kaf24@2673 162 int rc = 0;
djm@1686 163
djm@1686 164 ASSERT(spin_is_locked(&m->shadow_lock));
kaf24@2673 165 ASSERT(m->shadow_dirty_bitmap != NULL);
djm@1686 166
djm@1686 167 pfn = machine_to_phys_mapping[mfn];
djm@1686 168
kaf24@2673 169 /*
kaf24@2673 170 * Values with the MSB set denote MFNs that aren't really part of the
kaf24@2673 171 * domain's pseudo-physical memory map (e.g., the shared info frame).
kaf24@2673 172 * Nothing to do here...
kaf24@2673 173 */
kaf24@2673 174 if ( unlikely(pfn & 0x80000000UL) )
kaf24@2673 175 return rc;
djm@1686 176
kaf24@2673 177 if ( likely(pfn < m->shadow_dirty_bitmap_size) )
djm@1686 178 {
kaf24@2673 179 /* N.B. Can use non-atomic TAS because protected by shadow_lock. */
kaf24@2673 180 if ( !__test_and_set_bit(pfn, m->shadow_dirty_bitmap) )
kaf24@1787 181 {
kaf24@1787 182 m->shadow_dirty_count++;
kaf24@1787 183 rc = 1;
kaf24@1787 184 }
djm@1686 185 }
kaf24@2673 186 #ifndef NDEBUG
kaf24@2673 187 else if ( mfn < max_page )
djm@1686 188 {
iap10@2778 189 SH_LOG("mark_dirty OOR! mfn=%x pfn=%lx max=%x (mm %p)",
kaf24@2673 190 mfn, pfn, m->shadow_dirty_bitmap_size, m );
kaf24@2673 191 SH_LOG("dom=%p caf=%08x taf=%08x\n",
kaf24@2673 192 frame_table[mfn].u.inuse.domain,
kaf24@2673 193 frame_table[mfn].count_info,
kaf24@2673 194 frame_table[mfn].u.inuse.type_info );
djm@1686 195 }
kaf24@2673 196 #endif
kaf24@1787 197
djm@1686 198 return rc;
djm@1686 199 }
djm@1686 200
djm@1686 201
kaf24@2673 202 static inline int mark_dirty(struct mm_struct *m, unsigned int mfn)
djm@1686 203 {
djm@1686 204 int rc;
kaf24@2416 205 shadow_lock(m);
kaf24@2673 206 rc = __mark_dirty(m, mfn);
kaf24@2416 207 shadow_unlock(m);
djm@1686 208 return rc;
djm@1686 209 }
djm@1686 210
djm@1686 211
djm@1686 212 /************************************************************************/
djm@1686 213
kaf24@1787 214 static inline void l1pte_write_fault(
kaf24@1787 215 struct mm_struct *m, unsigned long *gpte_p, unsigned long *spte_p)
djm@1686 216 {
djm@1686 217 unsigned long gpte = *gpte_p;
djm@1686 218 unsigned long spte = *spte_p;
djm@1686 219
kaf24@2673 220 ASSERT(gpte & _PAGE_RW);
kaf24@2673 221 gpte |= _PAGE_DIRTY | _PAGE_ACCESSED;
kaf24@2673 222
kaf24@1787 223 switch ( m->shadow_mode )
djm@1686 224 {
djm@1686 225 case SHM_test:
kaf24@2673 226 spte = gpte | _PAGE_RW;
djm@1686 227 break;
djm@1686 228
djm@1686 229 case SHM_logdirty:
kaf24@2673 230 spte = gpte | _PAGE_RW;
kaf24@2673 231 __mark_dirty(m, gpte >> PAGE_SHIFT);
iap10@3328 232
iap10@3328 233 case SHM_full_32:
iap10@3328 234 {
iap10@3328 235 unsigned long host_pfn, host_gpte;
iap10@3328 236
iap10@3328 237 host_pfn = phys_to_machine_mapping[gpte >> PAGE_SHIFT];
iap10@3328 238 host_gpte = (host_pfn << PAGE_SHIFT) | (gpte & ~PAGE_MASK);
iap10@3328 239 spte = host_gpte | _PAGE_RW;
iap10@3328 240 }
djm@1686 241 break;
djm@1686 242 }
djm@1686 243
iap10@3328 244 SH_VVLOG("updating spte=%lx gpte=%lx", spte, gpte);
djm@1686 245 *gpte_p = gpte;
djm@1686 246 *spte_p = spte;
djm@1686 247 }
djm@1686 248
kaf24@1787 249 static inline void l1pte_read_fault(
kaf24@1787 250 struct mm_struct *m, unsigned long *gpte_p, unsigned long *spte_p)
djm@1686 251 {
djm@1686 252 unsigned long gpte = *gpte_p;
djm@1686 253 unsigned long spte = *spte_p;
djm@1686 254
kaf24@2673 255 gpte |= _PAGE_ACCESSED;
kaf24@2673 256
kaf24@1787 257 switch ( m->shadow_mode )
djm@1686 258 {
djm@1686 259 case SHM_test:
kaf24@2673 260 spte = (gpte & _PAGE_DIRTY) ? gpte : (gpte & ~_PAGE_RW);
djm@1686 261 break;
djm@1686 262
djm@1686 263 case SHM_logdirty:
kaf24@2673 264 spte = gpte & ~_PAGE_RW;
djm@1686 265 break;
iap10@3328 266
iap10@3328 267 case SHM_full_32:
iap10@3328 268 {
iap10@3328 269 unsigned long host_pfn, host_gpte;
iap10@3328 270
iap10@3328 271 host_pfn = phys_to_machine_mapping[gpte >> PAGE_SHIFT];
iap10@3328 272 host_gpte = (host_pfn << PAGE_SHIFT) | (gpte & ~PAGE_MASK);
iap10@3328 273 spte = (host_gpte & _PAGE_DIRTY) ? host_gpte : (host_gpte & ~_PAGE_RW);
iap10@3328 274 }
iap10@3328 275 break;
iap10@3328 276
djm@1686 277 }
djm@1686 278
djm@1686 279 *gpte_p = gpte;
djm@1686 280 *spte_p = spte;
djm@1686 281 }
djm@1686 282
kaf24@2673 283 static inline void l1pte_propagate_from_guest(
kaf24@1787 284 struct mm_struct *m, unsigned long *gpte_p, unsigned long *spte_p)
djm@1686 285 {
djm@1686 286 unsigned long gpte = *gpte_p;
djm@1686 287 unsigned long spte = *spte_p;
djm@1686 288
kaf24@1787 289 switch ( m->shadow_mode )
djm@1686 290 {
djm@1686 291 case SHM_test:
djm@1686 292 spte = 0;
djm@1686 293 if ( (gpte & (_PAGE_PRESENT|_PAGE_ACCESSED) ) ==
djm@1686 294 (_PAGE_PRESENT|_PAGE_ACCESSED) )
kaf24@2673 295 spte = (gpte & _PAGE_DIRTY) ? gpte : (gpte & ~_PAGE_RW);
djm@1686 296 break;
djm@1686 297
djm@1686 298 case SHM_logdirty:
djm@1686 299 spte = 0;
djm@1686 300 if ( (gpte & (_PAGE_PRESENT|_PAGE_ACCESSED) ) ==
djm@1686 301 (_PAGE_PRESENT|_PAGE_ACCESSED) )
kaf24@2673 302 spte = gpte & ~_PAGE_RW;
djm@1686 303 break;
iap10@3328 304
iap10@3328 305 case SHM_full_32:
iap10@3328 306 {
iap10@3328 307 unsigned long host_pfn, host_gpte;
iap10@3607 308 spte = 0;
iap10@3607 309
iap10@3607 310 if (mmio_space(gpte & 0xFFFFF000)) {
iap10@3607 311 *spte_p = spte;
iap10@3607 312 return;
iap10@3607 313 }
iap10@3328 314
iap10@3328 315 host_pfn = phys_to_machine_mapping[gpte >> PAGE_SHIFT];
iap10@3328 316 host_gpte = (host_pfn << PAGE_SHIFT) | (gpte & ~PAGE_MASK);
iap10@3328 317
iap10@3328 318 if ( (host_gpte & (_PAGE_PRESENT|_PAGE_ACCESSED) ) ==
iap10@3328 319 (_PAGE_PRESENT|_PAGE_ACCESSED) )
iap10@3328 320 spte = (host_gpte & _PAGE_DIRTY) ? host_gpte : (host_gpte & ~_PAGE_RW);
iap10@3328 321 }
iap10@3328 322 break;
djm@1686 323 }
djm@1686 324
djm@1686 325 *gpte_p = gpte;
djm@1686 326 *spte_p = spte;
djm@1686 327 }
djm@1686 328
kaf24@1787 329 static inline void l2pde_general(
kaf24@2673 330 struct mm_struct *m,
kaf24@1787 331 unsigned long *gpde_p,
kaf24@1787 332 unsigned long *spde_p,
kaf24@1787 333 unsigned long sl1pfn)
djm@1686 334 {
djm@1686 335 unsigned long gpde = *gpde_p;
djm@1686 336 unsigned long spde = *spde_p;
djm@1686 337
djm@1686 338 spde = 0;
djm@1686 339
kaf24@2673 340 if ( sl1pfn != 0 )
djm@1686 341 {
kaf24@2673 342 spde = (gpde & ~PAGE_MASK) | (sl1pfn << PAGE_SHIFT) |
djm@1686 343 _PAGE_RW | _PAGE_ACCESSED | _PAGE_DIRTY;
kaf24@2673 344 gpde |= _PAGE_ACCESSED | _PAGE_DIRTY;
djm@1686 345
kaf24@2673 346 /* Detect linear p.t. mappings and write-protect them. */
kaf24@2673 347 if ( (frame_table[sl1pfn].u.inuse.type_info & PGT_type_mask) ==
iap10@3328 348 PGT_l2_page_table )
iap10@3328 349 {
iap10@3328 350 if (m->shadow_mode != SHM_full_32)
iap10@3328 351 spde = gpde & ~_PAGE_RW;
iap10@3328 352
iap10@3328 353 }
djm@1686 354 }
djm@1686 355
djm@1686 356 *gpde_p = gpde;
djm@1686 357 *spde_p = spde;
djm@1686 358 }
djm@1686 359
djm@1686 360 /*********************************************************************/
djm@1686 361
djm@1686 362 #if SHADOW_HASH_DEBUG
djm@1686 363 static void shadow_audit(struct mm_struct *m, int print)
djm@1686 364 {
kaf24@2673 365 int live = 0, free = 0, j = 0, abs;
djm@1686 366 struct shadow_status *a;
djm@1686 367
kaf24@2673 368 for ( j = 0; j < shadow_ht_buckets; j++ )
djm@1686 369 {
djm@1686 370 a = &m->shadow_ht[j];
kaf24@2673 371 if ( a->pfn ) { live++; ASSERT(a->spfn_and_flags & PSH_pfn_mask); }
kaf24@2673 372 ASSERT(a->pfn < 0x00100000UL);
kaf24@2673 373 a = a->next;
kaf24@2673 374 while ( a && (live < 9999) )
djm@1686 375 {
djm@1686 376 live++;
kaf24@2673 377 if ( (a->pfn == 0) || (a->spfn_and_flags == 0) )
djm@1686 378 {
djm@1686 379 printk("XXX live=%d pfn=%08lx sp=%08lx next=%p\n",
djm@1686 380 live, a->pfn, a->spfn_and_flags, a->next);
djm@1686 381 BUG();
djm@1686 382 }
kaf24@2673 383 ASSERT(a->pfn < 0x00100000UL);
kaf24@2673 384 ASSERT(a->spfn_and_flags & PSH_pfn_mask);
kaf24@2673 385 a = a->next;
djm@1686 386 }
kaf24@2673 387 ASSERT(live < 9999);
djm@1686 388 }
djm@1686 389
kaf24@2673 390 for ( a = m->shadow_ht_free; a != NULL; a = a->next )
kaf24@2673 391 free++;
djm@1686 392
kaf24@2673 393 if ( print)
kaf24@2673 394 printk("Xlive=%d free=%d\n",live,free);
djm@1686 395
kaf24@2673 396 abs = (perfc_value(shadow_l1_pages) + perfc_value(shadow_l2_pages)) - live;
kaf24@2673 397 if ( (abs < -1) || (abs > 1) )
djm@1686 398 {
djm@1686 399 printk("live=%d free=%d l1=%d l2=%d\n",live,free,
djm@1686 400 perfc_value(shadow_l1_pages), perfc_value(shadow_l2_pages) );
djm@1686 401 BUG();
djm@1686 402 }
djm@1686 403 }
djm@1686 404 #else
kaf24@2673 405 #define shadow_audit(p, print) ((void)0)
djm@1686 406 #endif
djm@1686 407
djm@1686 408
djm@1686 409
kaf24@2673 410 static inline struct shadow_status *hash_bucket(
kaf24@2673 411 struct mm_struct *m, unsigned int gpfn)
djm@1686 412 {
kaf24@2673 413 return &m->shadow_ht[gpfn % shadow_ht_buckets];
djm@1686 414 }
djm@1686 415
djm@1686 416
kaf24@2673 417 static inline unsigned long __shadow_status(
kaf24@2673 418 struct mm_struct *m, unsigned int gpfn)
djm@1686 419 {
kaf24@2673 420 struct shadow_status *p, *x, *head;
djm@1686 421
kaf24@2673 422 x = head = hash_bucket(m, gpfn);
kaf24@2673 423 p = NULL;
djm@1686 424
kaf24@2673 425 SH_VVLOG("lookup gpfn=%08x bucket=%p", gpfn, x);
kaf24@2673 426 shadow_audit(m, 0);
djm@1686 427
djm@1686 428 do
djm@1686 429 {
kaf24@2673 430 ASSERT(x->pfn || ((x == head) && (x->next == NULL)));
djm@1686 431
kaf24@2673 432 if ( x->pfn == gpfn )
kaf24@2673 433 {
kaf24@2673 434 /* Pull-to-front if 'x' isn't already the head item. */
kaf24@2673 435 if ( unlikely(x != head) )
kaf24@2673 436 {
kaf24@2673 437 /* Delete 'x' from list and reinsert immediately after head. */
kaf24@2673 438 p->next = x->next;
kaf24@2673 439 x->next = head->next;
kaf24@2673 440 head->next = x;
djm@1686 441
kaf24@2673 442 /* Swap 'x' contents with head contents. */
kaf24@2673 443 SWAP(head->pfn, x->pfn);
kaf24@2673 444 SWAP(head->spfn_and_flags, x->spfn_and_flags);
djm@1686 445 }
kaf24@2673 446
kaf24@2673 447 return head->spfn_and_flags;
djm@1686 448 }
kaf24@2673 449
kaf24@2673 450 p = x;
kaf24@2673 451 x = x->next;
djm@1686 452 }
kaf24@2673 453 while ( x != NULL );
djm@1686 454
djm@1686 455 return 0;
djm@1686 456 }
djm@1686 457
kaf24@2673 458 /*
kaf24@2673 459 * N.B. We can make this locking more fine grained (e.g., per shadow page) if
kaf24@2673 460 * it ever becomes a problem, but since we need a spin lock on the hash table
kaf24@2673 461 * anyway it's probably not worth being too clever.
kaf24@2673 462 */
kaf24@2673 463 static inline unsigned long get_shadow_status(
kaf24@2673 464 struct mm_struct *m, unsigned int gpfn )
djm@1686 465 {
djm@1686 466 unsigned long res;
djm@1686 467
kaf24@2673 468 ASSERT(m->shadow_mode);
djm@1686 469
kaf24@2673 470 /*
kaf24@2673 471 * If we get here we know that some sort of update has happened to the
kaf24@2673 472 * underlying page table page: either a PTE has been updated, or the page
kaf24@2673 473 * has changed type. If we're in log dirty mode, we should set the
kaf24@2673 474 * appropriate bit in the dirty bitmap.
kaf24@2673 475 * N.B. The VA update path doesn't use this and is handled independently.
kaf24@2673 476 */
kaf24@2673 477
kaf24@2416 478 shadow_lock(m);
djm@1686 479
kaf24@2673 480 if ( m->shadow_mode == SHM_logdirty )
djm@1686 481 __mark_dirty( m, gpfn );
djm@1686 482
kaf24@2673 483 if ( !(res = __shadow_status(m, gpfn)) )
kaf24@2416 484 shadow_unlock(m);
kaf24@2673 485
djm@1686 486 return res;
djm@1686 487 }
djm@1686 488
djm@1686 489
kaf24@2673 490 static inline void put_shadow_status(
kaf24@2673 491 struct mm_struct *m)
djm@1686 492 {
kaf24@2416 493 shadow_unlock(m);
djm@1686 494 }
djm@1686 495
djm@1686 496
kaf24@2673 497 static inline void delete_shadow_status(
kaf24@2673 498 struct mm_struct *m, unsigned int gpfn)
djm@1686 499 {
kaf24@2673 500 struct shadow_status *p, *x, *n, *head;
djm@1686 501
djm@1686 502 ASSERT(spin_is_locked(&m->shadow_lock));
kaf24@2673 503 ASSERT(gpfn != 0);
djm@1686 504
kaf24@2673 505 head = hash_bucket(m, gpfn);
djm@1686 506
iap10@3328 507 SH_VVLOG("delete gpfn=%08x bucket=%p", gpfn, head);
kaf24@2673 508 shadow_audit(m, 0);
djm@1686 509
kaf24@2673 510 /* Match on head item? */
kaf24@2673 511 if ( head->pfn == gpfn )
djm@1686 512 {
kaf24@2673 513 if ( (n = head->next) != NULL )
djm@1686 514 {
kaf24@2673 515 /* Overwrite head with contents of following node. */
kaf24@2673 516 head->pfn = n->pfn;
kaf24@2673 517 head->spfn_and_flags = n->spfn_and_flags;
djm@1686 518
kaf24@2673 519 /* Delete following node. */
kaf24@2673 520 head->next = n->next;
kaf24@2673 521
kaf24@2673 522 /* Add deleted node to the free list. */
kaf24@2673 523 n->pfn = 0;
kaf24@2673 524 n->spfn_and_flags = 0;
kaf24@2673 525 n->next = m->shadow_ht_free;
kaf24@2673 526 m->shadow_ht_free = n;
djm@1686 527 }
djm@1686 528 else
djm@1686 529 {
kaf24@2673 530 /* This bucket is now empty. Initialise the head node. */
kaf24@2673 531 head->pfn = 0;
kaf24@2673 532 head->spfn_and_flags = 0;
djm@1686 533 }
djm@1686 534
kaf24@2673 535 goto found;
djm@1686 536 }
djm@1686 537
kaf24@2673 538 p = head;
kaf24@2673 539 x = head->next;
djm@1686 540
djm@1686 541 do
djm@1686 542 {
kaf24@2673 543 if ( x->pfn == gpfn )
djm@1686 544 {
kaf24@2673 545 /* Delete matching node. */
kaf24@2673 546 p->next = x->next;
djm@1686 547
kaf24@2673 548 /* Add deleted node to the free list. */
kaf24@2673 549 x->pfn = 0;
kaf24@2673 550 x->spfn_and_flags = 0;
kaf24@2673 551 x->next = m->shadow_ht_free;
kaf24@2673 552 m->shadow_ht_free = x;
djm@1686 553
kaf24@2673 554 goto found;
djm@1686 555 }
djm@1686 556
kaf24@2673 557 p = x;
kaf24@2673 558 x = x->next;
djm@1686 559 }
kaf24@2673 560 while ( x != NULL );
djm@1686 561
kaf24@2673 562 /* If we got here, it wasn't in the list! */
djm@1686 563 BUG();
kaf24@2673 564
kaf24@2673 565 found:
kaf24@2673 566 shadow_audit(m, 0);
djm@1686 567 }
djm@1686 568
djm@1686 569
kaf24@2673 570 static inline void set_shadow_status(
kaf24@2673 571 struct mm_struct *m, unsigned int gpfn, unsigned long s)
djm@1686 572 {
kaf24@2673 573 struct shadow_status *x, *head, *extra;
djm@1686 574 int i;
djm@1686 575
djm@1686 576 ASSERT(spin_is_locked(&m->shadow_lock));
kaf24@2673 577 ASSERT(gpfn != 0);
kaf24@2673 578 ASSERT(s & PSH_shadowed);
djm@1686 579
kaf24@2673 580 x = head = hash_bucket(m, gpfn);
djm@1686 581
iap10@3328 582 SH_VVLOG("set gpfn=%08x s=%08lx bucket=%p(%p)", gpfn, s, x, x->next);
kaf24@2673 583 shadow_audit(m, 0);
djm@1686 584
kaf24@2673 585 /*
kaf24@2673 586 * STEP 1. If page is already in the table, update it in place.
kaf24@2673 587 */
djm@1686 588
djm@1686 589 do
djm@1686 590 {
kaf24@2673 591 if ( x->pfn == gpfn )
djm@1686 592 {
kaf24@2673 593 x->spfn_and_flags = s;
kaf24@2673 594 goto done;
djm@1686 595 }
djm@1686 596
kaf24@2673 597 x = x->next;
djm@1686 598 }
kaf24@2673 599 while ( x != NULL );
djm@1686 600
kaf24@2673 601 /*
kaf24@2673 602 * STEP 2. The page must be inserted into the table.
kaf24@2673 603 */
djm@1686 604
kaf24@2673 605 /* If the bucket is empty then insert the new page as the head item. */
kaf24@2673 606 if ( head->pfn == 0 )
djm@1686 607 {
kaf24@2673 608 head->pfn = gpfn;
kaf24@2673 609 head->spfn_and_flags = s;
kaf24@2673 610 ASSERT(head->next == NULL);
kaf24@2673 611 goto done;
djm@1686 612 }
djm@1686 613
kaf24@2673 614 /* We need to allocate a new node. Ensure the quicklist is non-empty. */
kaf24@2673 615 if ( unlikely(m->shadow_ht_free == NULL) )
djm@1686 616 {
kaf24@2673 617 SH_LOG("Allocate more shadow hashtable blocks.");
kaf24@2673 618
kaf24@3664 619 extra = (struct shadow_status *)xmalloc(
kaf24@3664 620 u8[sizeof(void *) + (shadow_ht_extra_size * sizeof(*x))]);
djm@1686 621
kaf24@2673 622 /* XXX Should be more graceful here. */
kaf24@2673 623 if ( extra == NULL )
kaf24@2673 624 BUG();
djm@1686 625
kaf24@2673 626 memset(extra, 0, sizeof(void *) + (shadow_ht_extra_size * sizeof(*x)));
djm@1686 627
kaf24@2673 628 /* Record the allocation block so it can be correctly freed later. */
djm@1686 629 m->shadow_extras_count++;
kaf24@2673 630 *((struct shadow_status **)&extra[shadow_ht_extra_size]) =
kaf24@2673 631 m->shadow_ht_extras;
kaf24@2673 632 m->shadow_ht_extras = &extra[0];
djm@1686 633
kaf24@2673 634 /* Thread a free chain through the newly-allocated nodes. */
kaf24@2673 635 for ( i = 0; i < (shadow_ht_extra_size - 1); i++ )
kaf24@2673 636 extra[i].next = &extra[i+1];
kaf24@2673 637 extra[i].next = NULL;
djm@1686 638
kaf24@2673 639 /* Add the new nodes to the free list. */
kaf24@2673 640 m->shadow_ht_free = &extra[0];
djm@1686 641 }
djm@1686 642
kaf24@2673 643 /* Allocate a new node from the quicklist. */
kaf24@2673 644 x = m->shadow_ht_free;
kaf24@2673 645 m->shadow_ht_free = x->next;
djm@1686 646
kaf24@2673 647 /* Initialise the new node and insert directly after the head item. */
kaf24@2673 648 x->pfn = gpfn;
kaf24@2673 649 x->spfn_and_flags = s;
kaf24@2673 650 x->next = head->next;
kaf24@2673 651 head->next = x;
djm@1686 652
kaf24@2673 653 done:
kaf24@2673 654 shadow_audit(m, 0);
djm@1686 655 }
iap10@3328 656
iap10@3328 657 #ifdef CONFIG_VMX
iap10@3328 658 #include <asm/domain_page.h>
djm@1686 659
iap10@3328 660 static inline void vmx_update_shadow_state(
iap10@3328 661 struct mm_struct *mm, unsigned long gpfn, unsigned long spfn)
iap10@3328 662 {
iap10@3328 663
iap10@3328 664 l2_pgentry_t *mpl2e = 0;
iap10@3328 665 l2_pgentry_t *gpl2e, *spl2e;
iap10@3328 666
iap10@3328 667 /* unmap the old mappings */
iap10@3328 668 if (mm->shadow_vtable)
iap10@3328 669 unmap_domain_mem(mm->shadow_vtable);
iap10@3328 670 if (mm->vpagetable)
iap10@3328 671 unmap_domain_mem(mm->vpagetable);
iap10@3328 672
iap10@3328 673 /* new mapping */
iap10@3328 674 mpl2e = (l2_pgentry_t *)
iap10@3328 675 map_domain_mem(pagetable_val(mm->monitor_table));
iap10@3328 676
iap10@3328 677 mpl2e[SH_LINEAR_PT_VIRT_START >> L2_PAGETABLE_SHIFT] =
iap10@3328 678 mk_l2_pgentry((spfn << PAGE_SHIFT) | __PAGE_HYPERVISOR);
iap10@3328 679 __flush_tlb_one(SH_LINEAR_PT_VIRT_START);
iap10@3328 680
iap10@3328 681 spl2e = (l2_pgentry_t *) map_domain_mem(spfn << PAGE_SHIFT);
iap10@3328 682 gpl2e = (l2_pgentry_t *) map_domain_mem(gpfn << PAGE_SHIFT);
iap10@3328 683 memset(spl2e, 0, ENTRIES_PER_L2_PAGETABLE * sizeof(l2_pgentry_t));
iap10@3328 684
iap10@3328 685 mm->shadow_table = mk_pagetable(spfn<<PAGE_SHIFT);
iap10@3328 686 mm->shadow_vtable = spl2e;
iap10@3328 687 mm->vpagetable = gpl2e; /* expect the guest did clean this up */
iap10@3328 688 unmap_domain_mem(mpl2e);
iap10@3328 689 }
iap10@3328 690
iap10@3328 691 static inline void __shadow_mk_pagetable( struct mm_struct *mm )
iap10@3328 692 {
iap10@3328 693 unsigned long gpfn = pagetable_val(mm->pagetable) >> PAGE_SHIFT;
iap10@3328 694 unsigned long spfn;
iap10@3328 695 SH_VLOG("0: __shadow_mk_pagetable(gpfn=%08lx\n", gpfn);
iap10@3328 696
iap10@3328 697 if (mm->shadow_mode == SHM_full_32)
iap10@3328 698 {
iap10@3328 699 unsigned long guest_gpfn;
iap10@3328 700 guest_gpfn = machine_to_phys_mapping[gpfn];
iap10@3328 701
iap10@3328 702 SH_VVLOG("__shadow_mk_pagetable(guest_gpfn=%08lx, gpfn=%08lx\n",
iap10@3328 703 guest_gpfn, gpfn);
iap10@3328 704
iap10@3607 705 spfn = __shadow_status(mm, guest_gpfn) & PSH_pfn_mask;
iap10@3328 706 if ( unlikely(spfn == 0) ) {
iap10@3328 707 spfn = shadow_l2_table(mm, gpfn);
iap10@3328 708 mm->shadow_table = mk_pagetable(spfn<<PAGE_SHIFT);
iap10@3328 709 } else {
iap10@3328 710 vmx_update_shadow_state(mm, gpfn, spfn);
iap10@3328 711 }
iap10@3328 712 } else {
iap10@3328 713 spfn = __shadow_status(mm, gpfn) & PSH_pfn_mask;
iap10@3328 714
iap10@3328 715 if ( unlikely(spfn == 0) ) {
iap10@3328 716 spfn = shadow_l2_table(mm, gpfn);
iap10@3328 717 }
iap10@3328 718 mm->shadow_table = mk_pagetable(spfn<<PAGE_SHIFT);
iap10@3328 719 }
iap10@3328 720 }
iap10@3328 721 #else
kaf24@2673 722 static inline void __shadow_mk_pagetable(struct mm_struct *mm)
djm@1686 723 {
kaf24@2673 724 unsigned long gpfn = pagetable_val(mm->pagetable) >> PAGE_SHIFT;
kaf24@2673 725 unsigned long spfn = __shadow_status(mm, gpfn);
djm@1686 726
kaf24@2673 727 if ( unlikely(spfn == 0) )
kaf24@2673 728 spfn = shadow_l2_table(mm, gpfn);
kaf24@2673 729
kaf24@2673 730 mm->shadow_table = mk_pagetable(spfn << PAGE_SHIFT);
djm@1686 731 }
iap10@3328 732 #endif /* CONFIG_VMX */
djm@1686 733
kaf24@2673 734 static inline void shadow_mk_pagetable(struct mm_struct *mm)
djm@1686 735 {
iap10@3328 736 if ( unlikely(mm->shadow_mode) )
iap10@3328 737 {
iap10@3328 738 SH_VVLOG("shadow_mk_pagetable( gptbase=%08lx, mode=%d )",
iap10@3328 739 pagetable_val(mm->pagetable), mm->shadow_mode );
djm@1686 740
iap10@3328 741 shadow_lock(mm);
iap10@3328 742 __shadow_mk_pagetable(mm);
iap10@3328 743 shadow_unlock(mm);
djm@1686 744
iap10@3328 745 SH_VVLOG("leaving shadow_mk_pagetable:\n");
iap10@3328 746
iap10@3328 747 SH_VVLOG("( gptbase=%08lx, mode=%d ) sh=%08lx",
iap10@3328 748 pagetable_val(mm->pagetable), mm->shadow_mode,
iap10@3328 749 pagetable_val(mm->shadow_table) );
iap10@3328 750
iap10@3328 751 }
djm@1686 752 }
djm@1686 753
djm@1686 754 #if SHADOW_DEBUG
djm@1686 755 extern int check_pagetable(struct mm_struct *m, pagetable_t pt, char *s);
djm@1686 756 #else
djm@1686 757 #define check_pagetable(m, pt, s) ((void)0)
djm@1686 758 #endif
djm@1686 759
djm@1686 760 #endif /* XEN_SHADOW_H */