debuggers.hg

changeset 19951:ef38784f9f85

Do not use bitmap allocator after boot time.

Signed-off-by: Keir Fraser <keir.fraser@citrix.com>
author Keir Fraser <keir.fraser@citrix.com>
date Wed Jul 08 16:47:58 2009 +0100 (2009-07-08)
parents 721c14d7f60b
children d6c1d7992f43
files xen/common/page_alloc.c xen/include/asm-ia64/mm.h xen/include/asm-x86/mm.h
line diff
     1.1 --- a/xen/common/page_alloc.c	Wed Jul 08 14:22:00 2009 +0100
     1.2 +++ b/xen/common/page_alloc.c	Wed Jul 08 16:47:58 2009 +0100
     1.3 @@ -389,8 +389,7 @@ static struct page_info *alloc_heap_page
     1.4          page_list_add_tail(pg, &heap(node, zone, j));
     1.5          pg += 1 << j;
     1.6      }
     1.7 -    
     1.8 -    map_alloc(page_to_mfn(pg), request);
     1.9 +
    1.10      ASSERT(avail[node][zone] >= request);
    1.11      avail[node][zone] -= request;
    1.12  
    1.13 @@ -401,7 +400,8 @@ static struct page_info *alloc_heap_page
    1.14      for ( i = 0; i < (1 << order); i++ )
    1.15      {
    1.16          /* Reference count must continuously be zero for free pages. */
    1.17 -        BUG_ON(pg[i].count_info != 0);
    1.18 +        BUG_ON(pg[i].count_info != PGC_state_free);
    1.19 +        pg[i].count_info = PGC_state_inuse;
    1.20  
    1.21          if ( pg[i].u.free.need_tlbflush )
    1.22          {
    1.23 @@ -444,7 +444,7 @@ static int reserve_offlined_page(struct 
    1.24          struct page_info *pg;
    1.25          int next_order;
    1.26  
    1.27 -        if ( test_bit(_PGC_offlined, &cur_head->count_info) )
    1.28 +        if ( page_state_is(cur_head, offlined) )
    1.29          {
    1.30              cur_head++;
    1.31              continue;
    1.32 @@ -462,7 +462,7 @@ static int reserve_offlined_page(struct 
    1.33              for ( i = (1 << cur_order), pg = cur_head + (1 << cur_order );
    1.34                    i < (1 << next_order);
    1.35                    i++, pg++ )
    1.36 -                if ( test_bit(_PGC_offlined, &pg->count_info) )
    1.37 +                if ( page_state_is(pg, offlined) )
    1.38                      break;
    1.39              if ( i == ( 1 << next_order) )
    1.40              {
    1.41 @@ -483,13 +483,11 @@ static int reserve_offlined_page(struct 
    1.42  
    1.43      for ( cur_head = head; cur_head < head + ( 1UL << head_order); cur_head++ )
    1.44      {
    1.45 -        if ( !test_bit(_PGC_offlined, &cur_head->count_info) )
    1.46 +        if ( !page_state_is(cur_head, offlined) )
    1.47              continue;
    1.48  
    1.49          avail[node][zone]--;
    1.50  
    1.51 -        map_alloc(page_to_mfn(cur_head), 1);
    1.52 -
    1.53          page_list_add_tail(cur_head,
    1.54                             test_bit(_PGC_broken, &cur_head->count_info) ?
    1.55                             &page_broken_list : &page_offlined_list);
    1.56 @@ -525,14 +523,13 @@ static void free_heap_pages(
    1.57           *     in its pseudophysical address space).
    1.58           * In all the above cases there can be no guest mappings of this page.
    1.59           */
    1.60 -        ASSERT(!(pg[i].count_info & PGC_offlined));
    1.61 -        pg[i].count_info &= PGC_offlining | PGC_broken;
    1.62 -        if ( pg[i].count_info & PGC_offlining )
    1.63 -        {
    1.64 -            pg[i].count_info &= ~PGC_offlining;
    1.65 -            pg[i].count_info |= PGC_offlined;
    1.66 +        ASSERT(!page_state_is(&pg[i], offlined));
    1.67 +        pg[i].count_info =
    1.68 +            ((pg[i].count_info & PGC_broken) |
    1.69 +             (page_state_is(&pg[i], offlining)
    1.70 +              ? PGC_state_offlined : PGC_state_free));
    1.71 +        if ( page_state_is(&pg[i], offlined) )
    1.72              tainted = 1;
    1.73 -        }
    1.74  
    1.75          /* If a page has no owner it will need no safety TLB flush. */
    1.76          pg[i].u.free.need_tlbflush = (page_get_owner(&pg[i]) != NULL);
    1.77 @@ -542,7 +539,6 @@ static void free_heap_pages(
    1.78  
    1.79      spin_lock(&heap_lock);
    1.80  
    1.81 -    map_free(page_to_mfn(pg), 1 << order);
    1.82      avail[node][zone] += 1 << order;
    1.83  
    1.84      /* Merge chunks as far as possible. */
    1.85 @@ -553,7 +549,7 @@ static void free_heap_pages(
    1.86          if ( (page_to_mfn(pg) & mask) )
    1.87          {
    1.88              /* Merge with predecessor block? */
    1.89 -            if ( allocated_in_map(page_to_mfn(pg)-mask) ||
    1.90 +            if ( !page_state_is(pg-mask, free) ||
    1.91                   (PFN_ORDER(pg-mask) != order) )
    1.92                  break;
    1.93              pg -= mask;
    1.94 @@ -562,7 +558,7 @@ static void free_heap_pages(
    1.95          else
    1.96          {
    1.97              /* Merge with successor block? */
    1.98 -            if ( allocated_in_map(page_to_mfn(pg)+mask) ||
    1.99 +            if ( !page_state_is(pg+mask, free) ||
   1.100                   (PFN_ORDER(pg+mask) != order) )
   1.101                  break;
   1.102              page_list_del(pg + mask, &heap(node, zone, order));
   1.103 @@ -593,7 +589,6 @@ static void free_heap_pages(
   1.104   * Once a page is broken, it can't be assigned anymore
   1.105   * A page will be offlined only if it is free
   1.106   * return original count_info
   1.107 - *
   1.108   */
   1.109  static unsigned long mark_page_offline(struct page_info *pg, int broken)
   1.110  {
   1.111 @@ -605,26 +600,19 @@ static unsigned long mark_page_offline(s
   1.112      do {
   1.113          nx = x = y;
   1.114  
   1.115 -        if ( ((x & PGC_offlined_broken) == PGC_offlined_broken) )
   1.116 -            return y;
   1.117 -
   1.118 -        if ( x & PGC_offlined )
   1.119 +        if ( ((x & PGC_state) != PGC_state_offlined) &&
   1.120 +             ((x & PGC_state) != PGC_state_offlining) )
   1.121          {
   1.122 -            /* PGC_offlined means it is a free page. */
   1.123 -            if ( broken && !(nx & PGC_broken) )
   1.124 -                nx |= PGC_broken;
   1.125 -            else
   1.126 -                return y;
   1.127 -        }
   1.128 -        else
   1.129 -        {
   1.130 -            /* It is not offlined, not reserved page */
   1.131 -            nx |= (allocated_in_map(page_to_mfn(pg)) ?
   1.132 -                   PGC_offlining : PGC_offlined);
   1.133 +            nx &= ~PGC_state;
   1.134 +            nx |= (((x & PGC_state) == PGC_state_free)
   1.135 +                   ? PGC_state_offlined : PGC_state_offlining);
   1.136          }
   1.137  
   1.138          if ( broken )
   1.139              nx |= PGC_broken;
   1.140 +
   1.141 +        if ( x == nx )
   1.142 +            break;
   1.143      } while ( (y = cmpxchg(&pg->count_info, x, nx)) != x );
   1.144  
   1.145      return y;
   1.146 @@ -698,13 +686,13 @@ int offline_page(unsigned long mfn, int 
   1.147  
   1.148      old_info = mark_page_offline(pg, broken);
   1.149  
   1.150 -    if ( !allocated_in_map(mfn) )
   1.151 +    if ( page_state_is(pg, free) )
   1.152      {
   1.153          /* Free pages are reserve directly */
   1.154          reserve_heap_page(pg);
   1.155          *status = PG_OFFLINE_OFFLINED;
   1.156      }
   1.157 -    else if ( test_bit(_PGC_offlined, &pg->count_info) )
   1.158 +    else if ( page_state_is(pg, offlined) )
   1.159      {
   1.160          *status = PG_OFFLINE_OFFLINED;
   1.161      }
   1.162 @@ -749,8 +737,9 @@ int offline_page(unsigned long mfn, int 
   1.163   */
   1.164  unsigned int online_page(unsigned long mfn, uint32_t *status)
   1.165  {
   1.166 +    unsigned long x, nx, y;
   1.167      struct page_info *pg;
   1.168 -    int ret = 0, free = 0;
   1.169 +    int ret;
   1.170  
   1.171      if ( mfn > max_page )
   1.172      {
   1.173 @@ -760,30 +749,40 @@ unsigned int online_page(unsigned long m
   1.174  
   1.175      pg = mfn_to_page(mfn);
   1.176  
   1.177 -    *status = 0;
   1.178 -
   1.179      spin_lock(&heap_lock);
   1.180  
   1.181 -    if ( unlikely(is_page_broken(pg)) )
   1.182 -    {
   1.183 -        ret = -EINVAL;
   1.184 -        *status = PG_ONLINE_FAILED |PG_ONLINE_BROKEN;
   1.185 -    }
   1.186 -    else if ( pg->count_info & PGC_offlined )
   1.187 -    {
   1.188 -        clear_bit(_PGC_offlined, &pg->count_info);
   1.189 -        page_list_del(pg, &page_offlined_list);
   1.190 -        *status = PG_ONLINE_ONLINED;
   1.191 -        free = 1;
   1.192 -    }
   1.193 -    else if ( pg->count_info & PGC_offlining )
   1.194 -    {
   1.195 -        clear_bit(_PGC_offlining, &pg->count_info);
   1.196 -        *status = PG_ONLINE_ONLINED;
   1.197 -    }
   1.198 +    y = pg->count_info;
   1.199 +    do {
   1.200 +        ret = *status = 0;
   1.201 +
   1.202 +        if ( y & PGC_broken )
   1.203 +        {
   1.204 +            ret = -EINVAL;
   1.205 +            *status = PG_ONLINE_FAILED |PG_ONLINE_BROKEN;
   1.206 +            break;
   1.207 +        }
   1.208 +
   1.209 +        if ( (y & PGC_state) == PGC_state_offlined )
   1.210 +        {
   1.211 +            page_list_del(pg, &page_offlined_list);
   1.212 +            *status = PG_ONLINE_ONLINED;
   1.213 +        }
   1.214 +        else if ( (y & PGC_state) == PGC_state_offlining )
   1.215 +        {
   1.216 +            *status = PG_ONLINE_ONLINED;
   1.217 +        }
   1.218 +        else
   1.219 +        {
   1.220 +            break;
   1.221 +        }
   1.222 +
   1.223 +        x = y;
   1.224 +        nx = (x & ~PGC_state) | PGC_state_inuse;
   1.225 +    } while ( (y = cmpxchg(&pg->count_info, x, nx)) != x );
   1.226 +
   1.227      spin_unlock(&heap_lock);
   1.228  
   1.229 -    if ( free )
   1.230 +    if ( (y & PGC_state) == PGC_state_offlined )
   1.231          free_heap_pages(pg, 0);
   1.232  
   1.233      return ret;
   1.234 @@ -804,11 +803,11 @@ int query_page_offline(unsigned long mfn
   1.235  
   1.236      pg = mfn_to_page(mfn);
   1.237  
   1.238 -    if (pg->count_info & PGC_offlining)
   1.239 +    if ( page_state_is(pg, offlining) )
   1.240          *status |= PG_OFFLINE_STATUS_OFFLINE_PENDING;
   1.241 -    if (pg->count_info & PGC_broken)
   1.242 +    if ( pg->count_info & PGC_broken )
   1.243          *status |= PG_OFFLINE_STATUS_BROKEN;
   1.244 -    if (pg->count_info & PGC_offlined)
   1.245 +    if ( page_state_is(pg, offlined) )
   1.246          *status |= PG_OFFLINE_STATUS_OFFLINED;
   1.247  
   1.248      spin_unlock(&heap_lock);
   1.249 @@ -934,6 +933,7 @@ void __init end_boot_allocator(void)
   1.250  void __init scrub_heap_pages(void)
   1.251  {
   1.252      unsigned long mfn;
   1.253 +    struct page_info *pg;
   1.254  
   1.255      if ( !opt_bootscrub )
   1.256          return;
   1.257 @@ -944,8 +944,10 @@ void __init scrub_heap_pages(void)
   1.258      {
   1.259          process_pending_timers();
   1.260  
   1.261 +        pg = mfn_to_page(mfn);
   1.262 +
   1.263          /* Quick lock-free check. */
   1.264 -        if ( allocated_in_map(mfn) )
   1.265 +        if ( !page_state_is(pg, free) )
   1.266              continue;
   1.267  
   1.268          /* Every 100MB, print a progress dot. */
   1.269 @@ -955,8 +957,8 @@ void __init scrub_heap_pages(void)
   1.270          spin_lock(&heap_lock);
   1.271  
   1.272          /* Re-check page status with lock held. */
   1.273 -        if ( !allocated_in_map(mfn) )
   1.274 -            scrub_one_page(mfn_to_page(mfn));
   1.275 +        if ( page_state_is(pg, free) )
   1.276 +            scrub_one_page(pg);
   1.277  
   1.278          spin_unlock(&heap_lock);
   1.279      }
     2.1 --- a/xen/include/asm-ia64/mm.h	Wed Jul 08 14:22:00 2009 +0100
     2.2 +++ b/xen/include/asm-ia64/mm.h	Wed Jul 08 16:47:58 2009 +0100
     2.3 @@ -135,18 +135,14 @@ struct page_info
     2.4   /* Page is broken? */
     2.5  #define _PGC_broken       PG_shift(7)
     2.6  #define PGC_broken        PG_mask(1, 7)
     2.7 - /* Page is offline pending ? */
     2.8 -#define _PGC_offlining    PG_shift(8)
     2.9 -#define PGC_offlining     PG_mask(1, 8)
    2.10 - /* Page is offlined */
    2.11 -#define _PGC_offlined     PG_shift(9)
    2.12 -#define PGC_offlined      PG_mask(1, 9)
    2.13 -#define PGC_offlined_broken (PGC_offlined | PGC_broken)
    2.14  
    2.15 -#define is_page_offlining(page) ((page)->count_info & PGC_offlining)
    2.16 -#define is_page_offlined(page)  ((page)->count_info & PGC_offlined)
    2.17 -#define is_page_broken(page)    ((page)->count_info & PGC_broken)
    2.18 -#define is_page_online(page)    (!is_page_offlined(page))
    2.19 + /* Mutually-exclusive page states: { inuse, offlining, offlined, free }. */
    2.20 +#define PGC_state         PG_mask(3, 9)
    2.21 +#define PGC_state_inuse   PG_mask(0, 9)
    2.22 +#define PGC_state_offlining PG_mask(1, 9)
    2.23 +#define PGC_state_offlined PG_mask(2, 9)
    2.24 +#define PGC_state_free    PG_mask(3, 9)
    2.25 +#define page_state_is(pg, st) (((pg)->count_info&PGC_state) == PGC_state_##st)
    2.26  
    2.27   /* Count of references to this frame. */
    2.28  #define PGC_count_width   PG_shift(9)
     3.1 --- a/xen/include/asm-x86/mm.h	Wed Jul 08 14:22:00 2009 +0100
     3.2 +++ b/xen/include/asm-x86/mm.h	Wed Jul 08 16:47:58 2009 +0100
     3.3 @@ -199,25 +199,20 @@ struct page_info
     3.4  #define PGC_cacheattr_base PG_shift(6)
     3.5  #define PGC_cacheattr_mask PG_mask(7, 6)
     3.6   /* Page is broken? */
     3.7 -#define _PGC_broken         PG_shift(7)
     3.8 -#define PGC_broken          PG_mask(1, 7)
     3.9 - /* Page is offline pending ? */
    3.10 -#define _PGC_offlining      PG_shift(8)
    3.11 -#define PGC_offlining       PG_mask(1, 8)
    3.12 - /* Page is offlined */
    3.13 -#define _PGC_offlined       PG_shift(9)
    3.14 -#define PGC_offlined        PG_mask(1, 9)
    3.15 -#define PGC_offlined_broken (PGC_offlined | PGC_broken)
    3.16 +#define _PGC_broken       PG_shift(7)
    3.17 +#define PGC_broken        PG_mask(1, 7)
    3.18 + /* Mutually-exclusive page states: { inuse, offlining, offlined, free }. */
    3.19 +#define PGC_state         PG_mask(3, 9)
    3.20 +#define PGC_state_inuse   PG_mask(0, 9)
    3.21 +#define PGC_state_offlining PG_mask(1, 9)
    3.22 +#define PGC_state_offlined PG_mask(2, 9)
    3.23 +#define PGC_state_free    PG_mask(3, 9)
    3.24 +#define page_state_is(pg, st) (((pg)->count_info&PGC_state) == PGC_state_##st)
    3.25  
    3.26   /* Count of references to this frame. */
    3.27  #define PGC_count_width   PG_shift(9)
    3.28  #define PGC_count_mask    ((1UL<<PGC_count_width)-1)
    3.29  
    3.30 -#define is_page_offlining(page)  ((page)->count_info & PGC_offlining)
    3.31 -#define is_page_offlined(page)   ((page)->count_info & PGC_offlined)
    3.32 -#define is_page_broken(page)     ((page)->count_info & PGC_broken)
    3.33 -#define is_page_online(page)     (!is_page_offlined(page))
    3.34 -
    3.35  #if defined(__i386__)
    3.36  #define is_xen_heap_page(page) is_xen_heap_mfn(page_to_mfn(page))
    3.37  #define is_xen_heap_mfn(mfn) ({                         \