debuggers.hg

annotate xen/common/page_alloc.c @ 3658:0ef6e8e6e85d

bitkeeper revision 1.1159.212.71 (4200f0afX_JumfbEHQex6TdFENULMQ)

Merge labyrinth.cl.cam.ac.uk:/auto/groups/xeno-xenod/BK/xen-unstable.bk
into labyrinth.cl.cam.ac.uk:/auto/groups/xeno/users/iap10/xeno-clone/xen-unstable.bk
author iap10@labyrinth.cl.cam.ac.uk
date Wed Feb 02 15:24:31 2005 +0000 (2005-02-02)
parents 6d98eb831816 552dd1f1c64c
children 677cb76cff18
rev   line source
iap10@274 1 /******************************************************************************
iap10@274 2 * page_alloc.c
iap10@274 3 *
kaf24@1249 4 * Simple buddy heap allocator for Xen.
iap10@274 5 *
kaf24@1971 6 * Copyright (c) 2002-2004 K A Fraser
kaf24@804 7 *
kaf24@804 8 * This program is free software; you can redistribute it and/or modify
kaf24@804 9 * it under the terms of the GNU General Public License as published by
kaf24@804 10 * the Free Software Foundation; either version 2 of the License, or
kaf24@804 11 * (at your option) any later version.
kaf24@804 12 *
kaf24@804 13 * This program is distributed in the hope that it will be useful,
kaf24@804 14 * but WITHOUT ANY WARRANTY; without even the implied warranty of
kaf24@804 15 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
kaf24@804 16 * GNU General Public License for more details.
kaf24@804 17 *
kaf24@804 18 * You should have received a copy of the GNU General Public License
kaf24@804 19 * along with this program; if not, write to the Free Software
kaf24@804 20 * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
iap10@274 21 */
iap10@274 22
kaf24@1248 23 #include <xen/config.h>
kaf24@1248 24 #include <xen/init.h>
kaf24@1248 25 #include <xen/types.h>
kaf24@1248 26 #include <xen/lib.h>
iap10@274 27 #include <asm/page.h>
kaf24@1248 28 #include <xen/spinlock.h>
kaf24@1248 29 #include <xen/slab.h>
kaf24@1979 30 #include <xen/irq.h>
kaf24@2535 31 #include <asm/domain_page.h>
iap10@274 32
kaf24@3372 33 /*
kaf24@3372 34 * Comma-separated list of hexadecimal page numbers containing bad bytes.
kaf24@3372 35 * e.g. 'badpage=0x3f45,0x8a321'.
kaf24@3372 36 */
kaf24@3372 37 static char opt_badpage[100] = "";
kaf24@3372 38 string_param("badpage", opt_badpage);
iap10@274 39
kaf24@3392 40 #define round_pgdown(_p) ((_p)&PAGE_MASK)
kaf24@3392 41 #define round_pgup(_p) (((_p)+(PAGE_SIZE-1))&PAGE_MASK)
kaf24@3392 42
iap10@274 43 /*********************
iap10@274 44 * ALLOCATION BITMAP
iap10@274 45 * One bit per page of memory. Bit set => page is allocated.
iap10@274 46 */
iap10@274 47
kaf24@2810 48 static unsigned long bitmap_size; /* in bytes */
iap10@274 49 static unsigned long *alloc_bitmap;
iap10@274 50 #define PAGES_PER_MAPWORD (sizeof(unsigned long) * 8)
iap10@274 51
kaf24@3620 52 #define allocated_in_map(_pn) \
kaf24@3620 53 ( !! (alloc_bitmap[(_pn)/PAGES_PER_MAPWORD] & \
kaf24@3620 54 (1UL<<((_pn)&(PAGES_PER_MAPWORD-1)))) )
iap10@274 55
iap10@274 56 /*
iap10@274 57 * Hint regarding bitwise arithmetic in map_{alloc,free}:
iap10@274 58 * -(1<<n) sets all bits >= n.
iap10@274 59 * (1<<n)-1 sets all bits < n.
iap10@274 60 * Variable names in map_{alloc,free}:
iap10@274 61 * *_idx == Index into `alloc_bitmap' array.
iap10@274 62 * *_off == Bit offset within an element of the `alloc_bitmap' array.
iap10@274 63 */
iap10@274 64
iap10@274 65 static void map_alloc(unsigned long first_page, unsigned long nr_pages)
iap10@274 66 {
iap10@274 67 unsigned long start_off, end_off, curr_idx, end_idx;
iap10@274 68
kaf24@1305 69 #ifndef NDEBUG
kaf24@1305 70 unsigned long i;
kaf24@1305 71 /* Check that the block isn't already allocated. */
kaf24@1305 72 for ( i = 0; i < nr_pages; i++ )
kaf24@1305 73 ASSERT(!allocated_in_map(first_page + i));
kaf24@1305 74 #endif
kaf24@1305 75
iap10@274 76 curr_idx = first_page / PAGES_PER_MAPWORD;
iap10@274 77 start_off = first_page & (PAGES_PER_MAPWORD-1);
iap10@274 78 end_idx = (first_page + nr_pages) / PAGES_PER_MAPWORD;
iap10@274 79 end_off = (first_page + nr_pages) & (PAGES_PER_MAPWORD-1);
iap10@274 80
iap10@274 81 if ( curr_idx == end_idx )
iap10@274 82 {
kaf24@3620 83 alloc_bitmap[curr_idx] |= ((1UL<<end_off)-1) & -(1UL<<start_off);
iap10@274 84 }
iap10@274 85 else
iap10@274 86 {
kaf24@3620 87 alloc_bitmap[curr_idx] |= -(1UL<<start_off);
kaf24@3620 88 while ( ++curr_idx < end_idx ) alloc_bitmap[curr_idx] = ~0UL;
kaf24@3620 89 alloc_bitmap[curr_idx] |= (1UL<<end_off)-1;
iap10@274 90 }
iap10@274 91 }
iap10@274 92
iap10@274 93
iap10@274 94 static void map_free(unsigned long first_page, unsigned long nr_pages)
iap10@274 95 {
iap10@274 96 unsigned long start_off, end_off, curr_idx, end_idx;
iap10@274 97
kaf24@1305 98 #ifndef NDEBUG
kaf24@1305 99 unsigned long i;
kaf24@1305 100 /* Check that the block isn't already freed. */
kaf24@1305 101 for ( i = 0; i < nr_pages; i++ )
kaf24@1305 102 ASSERT(allocated_in_map(first_page + i));
kaf24@1305 103 #endif
kaf24@1305 104
kaf24@3392 105 curr_idx = first_page / PAGES_PER_MAPWORD;
iap10@274 106 start_off = first_page & (PAGES_PER_MAPWORD-1);
iap10@274 107 end_idx = (first_page + nr_pages) / PAGES_PER_MAPWORD;
iap10@274 108 end_off = (first_page + nr_pages) & (PAGES_PER_MAPWORD-1);
iap10@274 109
iap10@274 110 if ( curr_idx == end_idx )
iap10@274 111 {
kaf24@3620 112 alloc_bitmap[curr_idx] &= -(1UL<<end_off) | ((1UL<<start_off)-1);
iap10@274 113 }
iap10@274 114 else
iap10@274 115 {
kaf24@3620 116 alloc_bitmap[curr_idx] &= (1UL<<start_off)-1;
iap10@274 117 while ( ++curr_idx != end_idx ) alloc_bitmap[curr_idx] = 0;
kaf24@3620 118 alloc_bitmap[curr_idx] &= -(1UL<<end_off);
iap10@274 119 }
iap10@274 120 }
iap10@274 121
iap10@274 122
iap10@274 123
iap10@274 124 /*************************
kaf24@3392 125 * BOOT-TIME ALLOCATOR
kaf24@3392 126 */
kaf24@3392 127
kaf24@3392 128 /* Initialise allocator to handle up to @max_page pages. */
kaf24@3392 129 unsigned long init_boot_allocator(unsigned long bitmap_start)
kaf24@3392 130 {
kaf24@3392 131 bitmap_start = round_pgup(bitmap_start);
kaf24@3392 132
kaf24@3392 133 /* Allocate space for the allocation bitmap. */
kaf24@3392 134 bitmap_size = max_page / 8;
kaf24@3392 135 bitmap_size = round_pgup(bitmap_size);
kaf24@3392 136 alloc_bitmap = (unsigned long *)phys_to_virt(bitmap_start);
kaf24@3392 137
kaf24@3392 138 /* All allocated by default. */
kaf24@3392 139 memset(alloc_bitmap, ~0, bitmap_size);
kaf24@3392 140
kaf24@3392 141 return bitmap_start + bitmap_size;
kaf24@3392 142 }
kaf24@3392 143
kaf24@3392 144 void init_boot_pages(unsigned long ps, unsigned long pe)
kaf24@3392 145 {
kaf24@3392 146 unsigned long bad_pfn;
kaf24@3392 147 char *p;
kaf24@3392 148
kaf24@3392 149 ps = round_pgup(ps);
kaf24@3392 150 pe = round_pgdown(pe);
kaf24@3392 151
kaf24@3392 152 map_free(ps >> PAGE_SHIFT, (pe - ps) >> PAGE_SHIFT);
kaf24@3392 153
kaf24@3392 154 /* Check new pages against the bad-page list. */
kaf24@3392 155 p = opt_badpage;
kaf24@3392 156 while ( *p != '\0' )
kaf24@3392 157 {
kaf24@3392 158 bad_pfn = simple_strtoul(p, &p, 0);
kaf24@3392 159
kaf24@3392 160 if ( *p == ',' )
kaf24@3392 161 p++;
kaf24@3392 162 else if ( *p != '\0' )
kaf24@3392 163 break;
kaf24@3392 164
kaf24@3392 165 if ( (bad_pfn < (bitmap_size*8)) && !allocated_in_map(bad_pfn) )
kaf24@3392 166 {
kaf24@3392 167 printk("Marking page %08lx as bad\n", bad_pfn);
kaf24@3392 168 map_alloc(bad_pfn, 1);
kaf24@3392 169 }
kaf24@3392 170 }
kaf24@3392 171 }
kaf24@3392 172
kaf24@3392 173 unsigned long alloc_boot_pages(unsigned long size, unsigned long align)
kaf24@3392 174 {
kaf24@3392 175 unsigned long pg, i;
kaf24@3392 176
kaf24@3392 177 size = round_pgup(size) >> PAGE_SHIFT;
kaf24@3392 178 align = round_pgup(align) >> PAGE_SHIFT;
kaf24@3392 179
kaf24@3392 180 for ( pg = 0; (pg + size) < (bitmap_size*PAGES_PER_MAPWORD); pg += align )
kaf24@3392 181 {
kaf24@3392 182 for ( i = 0; i < size; i++ )
kaf24@3392 183 if ( allocated_in_map(pg + i) )
kaf24@3392 184 break;
kaf24@3392 185
kaf24@3392 186 if ( i == size )
kaf24@3392 187 {
kaf24@3392 188 map_alloc(pg, size);
kaf24@3392 189 return pg << PAGE_SHIFT;
kaf24@3392 190 }
kaf24@3392 191 }
kaf24@3392 192
kaf24@3392 193 return 0;
kaf24@3392 194 }
kaf24@3392 195
kaf24@3392 196
kaf24@3392 197
kaf24@3392 198 /*************************
iap10@274 199 * BINARY BUDDY ALLOCATOR
iap10@274 200 */
iap10@274 201
kaf24@1974 202 #define MEMZONE_XEN 0
kaf24@1974 203 #define MEMZONE_DOM 1
kaf24@1974 204 #define NR_ZONES 2
kaf24@1974 205
kaf24@1974 206 /* Up to 2^10 pages can be allocated at once. */
kaf24@1974 207 #define MAX_ORDER 10
kaf24@3499 208 static struct list_head heap[NR_ZONES][MAX_ORDER+1];
kaf24@1974 209
kaf24@1974 210 static unsigned long avail[NR_ZONES];
iap10@274 211
kaf24@1974 212 static spinlock_t heap_lock = SPIN_LOCK_UNLOCKED;
iap10@274 213
kaf24@3392 214 void end_boot_allocator(void)
iap10@274 215 {
kaf24@3392 216 unsigned long i, j;
kaf24@3392 217 int curr_free = 0, next_free = 0;
kaf24@1974 218
kaf24@1974 219 memset(avail, 0, sizeof(avail));
iap10@274 220
kaf24@1974 221 for ( i = 0; i < NR_ZONES; i++ )
kaf24@3499 222 for ( j = 0; j <= MAX_ORDER; j++ )
kaf24@1974 223 INIT_LIST_HEAD(&heap[i][j]);
iap10@274 224
kaf24@3392 225 /* Pages that are free now go to the domain sub-allocator. */
kaf24@3392 226 for ( i = 0; i < max_page; i++ )
kaf24@2133 227 {
kaf24@3392 228 curr_free = next_free;
kaf24@3392 229 next_free = !allocated_in_map(i+1);
kaf24@3392 230 if ( next_free )
kaf24@3392 231 map_alloc(i+1, 1); /* prevent merging in free_heap_pages() */
kaf24@3392 232 if ( curr_free )
kaf24@3392 233 free_heap_pages(MEMZONE_DOM, pfn_to_page(i), 0);
kaf24@2133 234 }
kaf24@1974 235 }
kaf24@1974 236
kaf24@1974 237 /* Hand the specified arbitrary page range to the specified heap zone. */
kaf24@3499 238 void init_heap_pages(
kaf24@3499 239 unsigned int zone, struct pfn_info *pg, unsigned long nr_pages)
kaf24@1974 240 {
kaf24@3392 241 unsigned long i;
kaf24@1974 242
kaf24@3499 243 ASSERT(zone < NR_ZONES);
kaf24@3499 244
kaf24@2133 245 for ( i = 0; i < nr_pages; i++ )
kaf24@3392 246 free_heap_pages(zone, pg+i, 0);
kaf24@386 247 }
kaf24@386 248
kaf24@386 249
kaf24@1974 250 /* Allocate 2^@order contiguous pages. */
kaf24@3499 251 struct pfn_info *alloc_heap_pages(unsigned int zone, unsigned int order)
iap10@274 252 {
kaf24@1974 253 int i;
kaf24@1971 254 struct pfn_info *pg;
iap10@274 255
kaf24@3499 256 ASSERT(zone < NR_ZONES);
kaf24@3499 257
kaf24@3499 258 if ( unlikely(order > MAX_ORDER) )
kaf24@1979 259 return NULL;
kaf24@1979 260
kaf24@2844 261 spin_lock(&heap_lock);
iap10@274 262
iap10@274 263 /* Find smallest order which can satisfy the request. */
kaf24@3499 264 for ( i = order; i <= MAX_ORDER; i++ )
kaf24@1974 265 if ( !list_empty(&heap[zone][i]) )
kaf24@3499 266 goto found;
iap10@274 267
kaf24@3499 268 /* No suitable memory blocks. Fail the request. */
kaf24@3499 269 spin_unlock(&heap_lock);
kaf24@3499 270 return NULL;
kaf24@3499 271
kaf24@3499 272 found:
kaf24@1974 273 pg = list_entry(heap[zone][i].next, struct pfn_info, list);
kaf24@1971 274 list_del(&pg->list);
iap10@274 275
kaf24@1971 276 /* We may have to halve the chunk a number of times. */
iap10@274 277 while ( i != order )
iap10@274 278 {
kaf24@1971 279 PFN_ORDER(pg) = --i;
kaf24@1974 280 list_add_tail(&pg->list, &heap[zone][i]);
kaf24@1971 281 pg += 1 << i;
iap10@274 282 }
iap10@274 283
kaf24@1974 284 map_alloc(page_to_pfn(pg), 1 << order);
kaf24@1974 285 avail[zone] -= 1 << order;
iap10@274 286
kaf24@2844 287 spin_unlock(&heap_lock);
iap10@274 288
kaf24@1974 289 return pg;
iap10@274 290 }
iap10@274 291
iap10@274 292
kaf24@1974 293 /* Free 2^@order set of pages. */
kaf24@3499 294 void free_heap_pages(
kaf24@3499 295 unsigned int zone, struct pfn_info *pg, unsigned int order)
iap10@274 296 {
kaf24@1971 297 unsigned long mask;
iap10@274 298
kaf24@3499 299 ASSERT(zone < NR_ZONES);
kaf24@3499 300 ASSERT(order <= MAX_ORDER);
kaf24@3499 301
kaf24@2844 302 spin_lock(&heap_lock);
iap10@274 303
kaf24@1974 304 map_free(page_to_pfn(pg), 1 << order);
kaf24@1974 305 avail[zone] += 1 << order;
iap10@274 306
iap10@274 307 /* Merge chunks as far as possible. */
kaf24@1974 308 while ( order < MAX_ORDER )
iap10@274 309 {
kaf24@1971 310 mask = 1 << order;
kaf24@1971 311
kaf24@1971 312 if ( (page_to_pfn(pg) & mask) )
iap10@274 313 {
iap10@274 314 /* Merge with predecessor block? */
kaf24@1971 315 if ( allocated_in_map(page_to_pfn(pg)-mask) ||
kaf24@1971 316 (PFN_ORDER(pg-mask) != order) )
kaf24@1183 317 break;
kaf24@1971 318 list_del(&(pg-mask)->list);
kaf24@1971 319 pg -= mask;
iap10@274 320 }
iap10@274 321 else
iap10@274 322 {
iap10@274 323 /* Merge with successor block? */
kaf24@1971 324 if ( allocated_in_map(page_to_pfn(pg)+mask) ||
kaf24@1971 325 (PFN_ORDER(pg+mask) != order) )
kaf24@1183 326 break;
kaf24@1971 327 list_del(&(pg+mask)->list);
iap10@274 328 }
iap10@274 329
iap10@274 330 order++;
iap10@274 331 }
iap10@274 332
kaf24@1971 333 PFN_ORDER(pg) = order;
kaf24@1974 334 list_add_tail(&pg->list, &heap[zone][order]);
kaf24@1974 335
kaf24@2844 336 spin_unlock(&heap_lock);
kaf24@1974 337 }
kaf24@1974 338
kaf24@1974 339
kaf24@2810 340 /*
kaf24@2810 341 * Scrub all unallocated pages in all heap zones. This function is more
kaf24@2810 342 * convoluted than appears necessary because we do not want to continuously
kaf24@2810 343 * hold the lock or disable interrupts while scrubbing very large memory areas.
kaf24@2810 344 */
kaf24@2810 345 void scrub_heap_pages(void)
kaf24@2810 346 {
kaf24@2810 347 void *p;
kaf24@2810 348 unsigned long pfn, flags;
kaf24@2810 349
kaf24@2888 350 printk("Scrubbing Free RAM: ");
kaf24@2888 351
kaf24@2810 352 for ( pfn = 0; pfn < (bitmap_size * 8); pfn++ )
kaf24@2810 353 {
kaf24@2888 354 /* Every 100MB, print a progress dot and appease the watchdog. */
kaf24@2888 355 if ( (pfn % ((100*1024*1024)/PAGE_SIZE)) == 0 )
kaf24@2888 356 {
kaf24@2888 357 printk(".");
kaf24@2888 358 touch_nmi_watchdog();
kaf24@2888 359 }
kaf24@2888 360
kaf24@2810 361 /* Quick lock-free check. */
kaf24@2810 362 if ( allocated_in_map(pfn) )
kaf24@2810 363 continue;
kaf24@2810 364
kaf24@2810 365 spin_lock_irqsave(&heap_lock, flags);
kaf24@2810 366
kaf24@2810 367 /* Re-check page status with lock held. */
kaf24@2810 368 if ( !allocated_in_map(pfn) )
kaf24@2810 369 {
kaf24@2810 370 p = map_domain_mem(pfn << PAGE_SHIFT);
kaf24@2810 371 clear_page(p);
kaf24@2810 372 unmap_domain_mem(p);
kaf24@2810 373 }
kaf24@2810 374
kaf24@2810 375 spin_unlock_irqrestore(&heap_lock, flags);
kaf24@2810 376 }
kaf24@2888 377
kaf24@2888 378 printk("done.\n");
kaf24@2810 379 }
kaf24@2810 380
kaf24@2810 381
kaf24@1974 382
kaf24@1974 383 /*************************
kaf24@1974 384 * XEN-HEAP SUB-ALLOCATOR
kaf24@1974 385 */
kaf24@1974 386
kaf24@1974 387 void init_xenheap_pages(unsigned long ps, unsigned long pe)
kaf24@1974 388 {
kaf24@2844 389 unsigned long flags;
kaf24@2844 390
kaf24@1974 391 ps = round_pgup(ps);
kaf24@1974 392 pe = round_pgdown(pe);
kaf24@2844 393
kaf24@1974 394 memguard_guard_range(__va(ps), pe - ps);
kaf24@2844 395
kaf24@2844 396 local_irq_save(flags);
kaf24@1974 397 init_heap_pages(MEMZONE_XEN, phys_to_page(ps), (pe - ps) >> PAGE_SHIFT);
kaf24@2844 398 local_irq_restore(flags);
kaf24@1974 399 }
kaf24@1974 400
kaf24@2844 401
kaf24@3499 402 unsigned long alloc_xenheap_pages(unsigned int order)
kaf24@1974 403 {
kaf24@2844 404 unsigned long flags;
kaf24@1974 405 struct pfn_info *pg;
iap10@3654 406 int i, attempts = 0;
kaf24@1974 407
iap10@3654 408 retry:
kaf24@2844 409 local_irq_save(flags);
kaf24@2844 410 pg = alloc_heap_pages(MEMZONE_XEN, order);
kaf24@2844 411 local_irq_restore(flags);
kaf24@2844 412
kaf24@2844 413 if ( unlikely(pg == NULL) )
kaf24@1974 414 goto no_memory;
kaf24@2382 415
kaf24@1974 416 memguard_unguard_range(page_to_virt(pg), 1 << (order + PAGE_SHIFT));
kaf24@2382 417
kaf24@2382 418 for ( i = 0; i < (1 << order); i++ )
kaf24@2382 419 {
kaf24@2655 420 pg[i].count_info = 0;
kaf24@2384 421 pg[i].u.inuse.domain = NULL;
kaf24@2384 422 pg[i].u.inuse.type_info = 0;
kaf24@2382 423 }
kaf24@2382 424
kaf24@1974 425 return (unsigned long)page_to_virt(pg);
kaf24@1974 426
kaf24@1974 427 no_memory:
iap10@3654 428 if ( attempts++ < 8 )
iap10@3654 429 {
iap10@3654 430 xmem_cache_reap();
iap10@3654 431 goto retry;
iap10@3654 432 }
iap10@3654 433
kaf24@1974 434 printk("Cannot handle page request order %d!\n", order);
iap10@3654 435 dump_slabinfo();
kaf24@1974 436 return 0;
kaf24@1974 437 }
kaf24@1974 438
kaf24@2844 439
kaf24@3499 440 void free_xenheap_pages(unsigned long p, unsigned int order)
kaf24@1974 441 {
kaf24@2844 442 unsigned long flags;
kaf24@2844 443
kaf24@1974 444 memguard_guard_range((void *)p, 1 << (order + PAGE_SHIFT));
kaf24@2844 445
kaf24@2844 446 local_irq_save(flags);
kaf24@1974 447 free_heap_pages(MEMZONE_XEN, virt_to_page(p), order);
kaf24@2844 448 local_irq_restore(flags);
iap10@274 449 }
kaf24@1974 450
kaf24@1974 451
kaf24@1974 452
kaf24@1974 453 /*************************
kaf24@1974 454 * DOMAIN-HEAP SUB-ALLOCATOR
kaf24@1974 455 */
kaf24@1974 456
kaf24@1974 457 void init_domheap_pages(unsigned long ps, unsigned long pe)
kaf24@1974 458 {
kaf24@2844 459 ASSERT(!in_irq());
kaf24@2844 460
kaf24@1974 461 ps = round_pgup(ps);
kaf24@1974 462 pe = round_pgdown(pe);
kaf24@2844 463
kaf24@1974 464 init_heap_pages(MEMZONE_DOM, phys_to_page(ps), (pe - ps) >> PAGE_SHIFT);
kaf24@1974 465 }
kaf24@1974 466
kaf24@2844 467
kaf24@3499 468 struct pfn_info *alloc_domheap_pages(struct domain *d, unsigned int order)
kaf24@1974 469 {
kaf24@1979 470 struct pfn_info *pg;
kaf24@1979 471 unsigned long mask, flushed_mask, pfn_stamp, cpu_stamp;
kaf24@2382 472 int i, j;
kaf24@1979 473
kaf24@1979 474 ASSERT(!in_irq());
kaf24@1979 475
kaf24@1979 476 if ( unlikely((pg = alloc_heap_pages(MEMZONE_DOM, order)) == NULL) )
kaf24@1979 477 return NULL;
kaf24@1979 478
kaf24@1979 479 flushed_mask = 0;
kaf24@1979 480 for ( i = 0; i < (1 << order); i++ )
kaf24@1979 481 {
kaf24@1979 482 if ( (mask = (pg[i].u.free.cpu_mask & ~flushed_mask)) != 0 )
kaf24@1979 483 {
kaf24@1979 484 pfn_stamp = pg[i].tlbflush_timestamp;
kaf24@2382 485 for ( j = 0; (mask != 0) && (j < smp_num_cpus); j++ )
kaf24@1979 486 {
kaf24@3620 487 if ( mask & (1UL<<j) )
kaf24@1979 488 {
kaf24@2382 489 cpu_stamp = tlbflush_time[j];
kaf24@1979 490 if ( !NEED_FLUSH(cpu_stamp, pfn_stamp) )
kaf24@3620 491 mask &= ~(1UL<<j);
kaf24@1979 492 }
kaf24@1979 493 }
kaf24@1979 494
kaf24@1979 495 if ( unlikely(mask != 0) )
kaf24@1979 496 {
kaf24@1979 497 flush_tlb_mask(mask);
kaf24@1979 498 perfc_incrc(need_flush_tlb_flush);
kaf24@1979 499 flushed_mask |= mask;
kaf24@1979 500 }
kaf24@1979 501 }
kaf24@2382 502
kaf24@2655 503 pg[i].count_info = 0;
kaf24@2384 504 pg[i].u.inuse.domain = NULL;
kaf24@2384 505 pg[i].u.inuse.type_info = 0;
kaf24@1979 506 }
kaf24@1979 507
kaf24@1979 508 if ( d == NULL )
kaf24@1979 509 return pg;
kaf24@1979 510
kaf24@1979 511 spin_lock(&d->page_alloc_lock);
kaf24@1979 512
cl349@2957 513 if ( unlikely(test_bit(DF_DYING, &d->d_flags)) ||
kaf24@2466 514 unlikely((d->tot_pages + (1 << order)) > d->max_pages) )
kaf24@1979 515 {
kaf24@1979 516 DPRINTK("Over-allocation for domain %u: %u > %u\n",
kaf24@2748 517 d->id, d->tot_pages + (1 << order), d->max_pages);
kaf24@2466 518 DPRINTK("...or the domain is dying (%d)\n",
cl349@2957 519 !!test_bit(DF_DYING, &d->d_flags));
kaf24@1979 520 spin_unlock(&d->page_alloc_lock);
kaf24@1979 521 free_heap_pages(MEMZONE_DOM, pg, order);
kaf24@1979 522 return NULL;
kaf24@1979 523 }
kaf24@1979 524
kaf24@1979 525 if ( unlikely(d->tot_pages == 0) )
kaf24@2336 526 get_knownalive_domain(d);
kaf24@1979 527
kaf24@1979 528 d->tot_pages += 1 << order;
kaf24@1979 529
kaf24@1979 530 for ( i = 0; i < (1 << order); i++ )
kaf24@1979 531 {
kaf24@1979 532 pg[i].u.inuse.domain = d;
kaf24@1979 533 wmb(); /* Domain pointer must be visible before updating refcnt. */
kaf24@2384 534 pg[i].count_info |= PGC_allocated | 1;
kaf24@2083 535 list_add_tail(&pg[i].list, &d->page_list);
kaf24@1979 536 }
kaf24@1979 537
kaf24@1979 538 spin_unlock(&d->page_alloc_lock);
kaf24@1979 539
kaf24@1974 540 return pg;
kaf24@1974 541 }
kaf24@1974 542
kaf24@2844 543
kaf24@3499 544 void free_domheap_pages(struct pfn_info *pg, unsigned int order)
kaf24@1974 545 {
kaf24@1979 546 int i, drop_dom_ref;
kaf24@1979 547 struct domain *d = pg->u.inuse.domain;
cl349@2957 548 struct exec_domain *ed;
kaf24@2535 549 void *p;
cl349@2957 550 int cpu_mask = 0;
kaf24@1979 551
kaf24@2844 552 ASSERT(!in_irq());
kaf24@2844 553
kaf24@1979 554 if ( unlikely(IS_XEN_HEAP_FRAME(pg)) )
kaf24@1979 555 {
kaf24@2466 556 /* NB. May recursively lock from domain_relinquish_memory(). */
kaf24@1979 557 spin_lock_recursive(&d->page_alloc_lock);
kaf24@2382 558
kaf24@2360 559 for ( i = 0; i < (1 << order); i++ )
kaf24@2360 560 list_del(&pg[i].list);
kaf24@2382 561
kaf24@1979 562 d->xenheap_pages -= 1 << order;
kaf24@1979 563 drop_dom_ref = (d->xenheap_pages == 0);
kaf24@2382 564
kaf24@1979 565 spin_unlock_recursive(&d->page_alloc_lock);
kaf24@1979 566 }
kaf24@2329 567 else if ( likely(d != NULL) )
kaf24@1979 568 {
kaf24@1979 569 /* NB. May recursively lock from domain_relinquish_memory(). */
kaf24@1979 570 spin_lock_recursive(&d->page_alloc_lock);
kaf24@1979 571
cl349@2957 572 for_each_exec_domain(d, ed)
cl349@2957 573 cpu_mask |= 1 << ed->processor;
cl349@2957 574
kaf24@1979 575 for ( i = 0; i < (1 << order); i++ )
kaf24@1979 576 {
kaf24@2535 577 ASSERT((pg[i].u.inuse.type_info & PGT_count_mask) == 0);
kaf24@2790 578 pg[i].tlbflush_timestamp = tlbflush_current_time();
cl349@2957 579 pg[i].u.free.cpu_mask = cpu_mask;
kaf24@1979 580 list_del(&pg[i].list);
kaf24@2535 581
kaf24@2535 582 /*
kaf24@2535 583 * Normally we expect a domain to clear pages before freeing them,
kaf24@2535 584 * if it cares about the secrecy of their contents. However, after
kaf24@2535 585 * a domain has died we assume responsibility for erasure.
kaf24@2535 586 */
cl349@2957 587 if ( unlikely(test_bit(DF_DYING, &d->d_flags)) )
kaf24@2535 588 {
kaf24@2535 589 p = map_domain_mem(page_to_phys(&pg[i]));
kaf24@2535 590 clear_page(p);
kaf24@2535 591 unmap_domain_mem(p);
kaf24@2535 592 }
kaf24@1979 593 }
kaf24@1979 594
kaf24@1979 595 d->tot_pages -= 1 << order;
kaf24@1979 596 drop_dom_ref = (d->tot_pages == 0);
kaf24@1979 597
kaf24@1979 598 spin_unlock_recursive(&d->page_alloc_lock);
kaf24@1979 599
kaf24@1979 600 free_heap_pages(MEMZONE_DOM, pg, order);
kaf24@1979 601 }
kaf24@2329 602 else
kaf24@2329 603 {
kaf24@2329 604 /* Freeing an anonymous domain-heap page. */
kaf24@2329 605 free_heap_pages(MEMZONE_DOM, pg, order);
kaf24@2329 606 drop_dom_ref = 0;
kaf24@2329 607 }
kaf24@1979 608
kaf24@1979 609 if ( drop_dom_ref )
kaf24@1979 610 put_domain(d);
kaf24@1974 611 }
kaf24@1974 612
kaf24@2844 613
kaf24@1974 614 unsigned long avail_domheap_pages(void)
kaf24@1974 615 {
kaf24@1974 616 return avail[MEMZONE_DOM];
kaf24@1974 617 }