1.1 --- a/linux-2.6-xen-sparse/drivers/xen/balloon/balloon.c	Mon Sep 05 17:21:08 2005 +0000
     1.2 +++ b/linux-2.6-xen-sparse/drivers/xen/balloon/balloon.c	Mon Sep 05 17:51:55 2005 +0000
     1.3 @@ -58,6 +58,12 @@
     1.4  static struct proc_dir_entry *balloon_pde;
     1.5  
     1.6  static DECLARE_MUTEX(balloon_mutex);
     1.7 +
     1.8 +/*
     1.9 + * Protects atomic reservation decrease/increase against concurrent increases.
    1.10 + * Also protects non-atomic updates of current_pages and driver_pages, and
    1.11 + * balloon lists.
    1.12 + */
    1.13  spinlock_t balloon_lock = SPIN_LOCK_UNLOCKED;
    1.14  
    1.15  /* We aim for 'current allocation' == 'target allocation'. */
    1.16 @@ -157,6 +163,146 @@ static unsigned long current_target(void
    1.17  	return target;
    1.18  }
    1.19  
    1.20 +static int increase_reservation(unsigned long nr_pages)
    1.21 +{
    1.22 +	unsigned long *mfn_list, pfn, i, flags;
    1.23 +	struct page   *page;
    1.24 +	long           rc;
    1.25 +	struct xen_memory_reservation reservation = {
    1.26 +		.address_bits = 0,
    1.27 +		.extent_order = 0,
    1.28 +		.domid        = DOMID_SELF
    1.29 +	};
    1.30 +
    1.31 +	if (nr_pages > (PAGE_SIZE / sizeof(unsigned long)))
    1.32 +		nr_pages = PAGE_SIZE / sizeof(unsigned long);
    1.33 +
    1.34 +	mfn_list = (unsigned long *)__get_free_page(GFP_KERNEL);
    1.35 +	if (mfn_list == NULL)
    1.36 +		return -ENOMEM;
    1.37 +
    1.38 +	balloon_lock(flags);
    1.39 +
    1.40 +	reservation.extent_start = mfn_list;
    1.41 +	reservation.nr_extents   = nr_pages;
    1.42 +	rc = HYPERVISOR_memory_op(
    1.43 +		XENMEM_increase_reservation, &reservation);
    1.44 +	if (rc < nr_pages) {
    1.45 +		/* We hit the Xen hard limit: reprobe. */
    1.46 +		reservation.extent_start = mfn_list;
    1.47 +		reservation.nr_extents   = rc;
    1.48 +		BUG_ON(HYPERVISOR_memory_op(
    1.49 +			XENMEM_decrease_reservation,
    1.50 +			&reservation) != rc);
    1.51 +		hard_limit = current_pages + rc - driver_pages;
    1.52 +		goto out;
    1.53 +	}
    1.54 +
    1.55 +	for (i = 0; i < nr_pages; i++) {
    1.56 +		page = balloon_retrieve();
    1.57 +		BUG_ON(page == NULL);
    1.58 +
    1.59 +		pfn = page - mem_map;
    1.60 +		BUG_ON(phys_to_machine_mapping[pfn] != INVALID_P2M_ENTRY);
    1.61 +
    1.62 +		/* Update P->M and M->P tables. */
    1.63 +		phys_to_machine_mapping[pfn] = mfn_list[i];
    1.64 +		xen_machphys_update(mfn_list[i], pfn);
    1.65 +            
    1.66 +		/* Link back into the page tables if not highmem. */
    1.67 +		if (pfn < max_low_pfn)
    1.68 +			BUG_ON(HYPERVISOR_update_va_mapping(
    1.69 +				(unsigned long)__va(pfn << PAGE_SHIFT),
    1.70 +				pfn_pte_ma(mfn_list[i], PAGE_KERNEL),
    1.71 +				0));
    1.72 +
    1.73 +		/* Relinquish the page back to the allocator. */
    1.74 +		ClearPageReserved(page);
    1.75 +		set_page_count(page, 1);
    1.76 +		__free_page(page);
    1.77 +	}
    1.78 +
    1.79 +	current_pages += nr_pages;
    1.80 +
    1.81 + out:
    1.82 +	balloon_unlock(flags);
    1.83 +
    1.84 +	free_page((unsigned long)mfn_list);
    1.85 +
    1.86 +	return 0;
    1.87 +}
    1.88 +
    1.89 +static int decrease_reservation(unsigned long nr_pages)
    1.90 +{
    1.91 +	unsigned long *mfn_list, pfn, i, flags;
    1.92 +	struct page   *page;
    1.93 +	void          *v;
    1.94 +	int            need_sleep = 0;
    1.95 +	struct xen_memory_reservation reservation = {
    1.96 +		.address_bits = 0,
    1.97 +		.extent_order = 0,
    1.98 +		.domid        = DOMID_SELF
    1.99 +	};
   1.100 +
   1.101 +	if (nr_pages > (PAGE_SIZE / sizeof(unsigned long)))
   1.102 +		nr_pages = PAGE_SIZE / sizeof(unsigned long);
   1.103 +
   1.104 +	mfn_list = (unsigned long *)__get_free_page(GFP_KERNEL);
   1.105 +	if (mfn_list == NULL)
   1.106 +		return -ENOMEM;
   1.107 +
   1.108 +	for (i = 0; i < nr_pages; i++) {
   1.109 +		if ((page = alloc_page(GFP_HIGHUSER)) == NULL) {
   1.110 +			nr_pages = i;
   1.111 +			need_sleep = 1;
   1.112 +			break;
   1.113 +		}
   1.114 +
   1.115 +		pfn = page - mem_map;
   1.116 +		mfn_list[i] = phys_to_machine_mapping[pfn];
   1.117 +
   1.118 +		if (!PageHighMem(page)) {
   1.119 +			v = phys_to_virt(pfn << PAGE_SHIFT);
   1.120 +			scrub_pages(v, 1);
   1.121 +			BUG_ON(HYPERVISOR_update_va_mapping(
   1.122 +				(unsigned long)v, __pte_ma(0), 0));
   1.123 +		}
   1.124 +#ifdef CONFIG_XEN_SCRUB_PAGES
   1.125 +		else {
   1.126 +			v = kmap(page);
   1.127 +			scrub_pages(v, 1);
   1.128 +			kunmap(page);
   1.129 +		}
   1.130 +#endif
   1.131 +	}
   1.132 +
   1.133 +	/* Ensure that ballooned highmem pages don't have kmaps. */
   1.134 +	kmap_flush_unused();
   1.135 +	flush_tlb_all();
   1.136 +
   1.137 +	balloon_lock(flags);
   1.138 +
   1.139 +	/* No more mappings: invalidate P2M and add to balloon. */
   1.140 +	for (i = 0; i < nr_pages; i++) {
   1.141 +		pfn = mfn_to_pfn(mfn_list[i]);
   1.142 +		phys_to_machine_mapping[pfn] = INVALID_P2M_ENTRY;
   1.143 +		balloon_append(pfn_to_page(pfn));
   1.144 +	}
   1.145 +
   1.146 +	reservation.extent_start = mfn_list;
   1.147 +	reservation.nr_extents   = nr_pages;
   1.148 +	BUG_ON(HYPERVISOR_memory_op(
   1.149 +		XENMEM_decrease_reservation, &reservation) != nr_pages);
   1.150 +
   1.151 +	current_pages -= nr_pages;
   1.152 +
   1.153 +	balloon_unlock(flags);
   1.154 +
   1.155 +	free_page((unsigned long)mfn_list);
   1.156 +
   1.157 +	return need_sleep;
   1.158 +}
   1.159 +
   1.160  /*
   1.161   * We avoid multiple worker processes conflicting via the balloon mutex.
   1.162   * We may of course race updates of the target counts (which are protected
   1.163 @@ -165,123 +311,23 @@ static unsigned long current_target(void
   1.164   */
   1.165  static void balloon_process(void *unused)
   1.166  {
   1.167 -	unsigned long *mfn_list, pfn, i, flags;
   1.168 -	struct page   *page;
   1.169 -	long           credit, debt, rc;
   1.170 -	void          *v;
   1.171 -	struct xen_memory_reservation reservation = {
   1.172 -		.address_bits = 0,
   1.173 -		.extent_order = 0,
   1.174 -		.domid        = DOMID_SELF
   1.175 -	};
   1.176 +	int need_sleep = 0;
   1.177 +	long credit;
   1.178  
   1.179  	down(&balloon_mutex);
   1.180  
   1.181 - retry:
   1.182 -	mfn_list = NULL;
   1.183 -
   1.184 -	if ((credit = current_target() - current_pages) > 0) {
   1.185 -		mfn_list = vmalloc(credit * sizeof(*mfn_list));
   1.186 -		if (mfn_list == NULL)
   1.187 -			goto out;
   1.188 -
   1.189 -		balloon_lock(flags);
   1.190 -		reservation.extent_start = mfn_list;
   1.191 -		reservation.nr_extents   = credit;
   1.192 -		rc = HYPERVISOR_memory_op(
   1.193 -			XENMEM_increase_reservation, &reservation);
   1.194 -		balloon_unlock(flags);
   1.195 -		if (rc < credit) {
   1.196 -			/* We hit the Xen hard limit: reprobe. */
   1.197 -			reservation.extent_start = mfn_list;
   1.198 -			reservation.nr_extents   = rc;
   1.199 -			BUG_ON(HYPERVISOR_memory_op(
   1.200 -				XENMEM_decrease_reservation,
   1.201 -				&reservation) != rc);
   1.202 -			hard_limit = current_pages + rc - driver_pages;
   1.203 -			vfree(mfn_list);
   1.204 -			goto retry;
   1.205 -		}
   1.206 -
   1.207 -		for (i = 0; i < credit; i++) {
   1.208 -			page = balloon_retrieve();
   1.209 -			BUG_ON(page == NULL);
   1.210 -
   1.211 -			pfn = page - mem_map;
   1.212 -			if (phys_to_machine_mapping[pfn] != INVALID_P2M_ENTRY)
   1.213 -				BUG();
   1.214 -
   1.215 -			/* Update P->M and M->P tables. */
   1.216 -			phys_to_machine_mapping[pfn] = mfn_list[i];
   1.217 -			xen_machphys_update(mfn_list[i], pfn);
   1.218 -            
   1.219 -			/* Link back into the page tables if not highmem. */
   1.220 -			if (pfn < max_low_pfn)
   1.221 -				BUG_ON(HYPERVISOR_update_va_mapping(
   1.222 -					(unsigned long)__va(pfn << PAGE_SHIFT),
   1.223 -					pfn_pte_ma(mfn_list[i], PAGE_KERNEL),
   1.224 -					0));
   1.225 -
   1.226 -			/* Relinquish the page back to the allocator. */
   1.227 -			ClearPageReserved(page);
   1.228 -			set_page_count(page, 1);
   1.229 -			__free_page(page);
   1.230 -		}
   1.231 +	do {
   1.232 +		credit = current_target() - current_pages;
   1.233 +		if (credit > 0)
   1.234 +			need_sleep = (increase_reservation(credit) != 0);
   1.235 +		if (credit < 0)
   1.236 +			need_sleep = (decrease_reservation(-credit) != 0);
   1.237  
   1.238 -		current_pages += credit;
   1.239 -	} else if (credit < 0) {
   1.240 -		debt = -credit;
   1.241 -
   1.242 -		mfn_list = vmalloc(debt * sizeof(*mfn_list));
   1.243 -		if (mfn_list == NULL)
   1.244 -			goto out;
   1.245 -
   1.246 -		for (i = 0; i < debt; i++) {
   1.247 -			if ((page = alloc_page(GFP_HIGHUSER)) == NULL) {
   1.248 -				debt = i;
   1.249 -				break;
   1.250 -			}
   1.251 -
   1.252 -			pfn = page - mem_map;
   1.253 -			mfn_list[i] = phys_to_machine_mapping[pfn];
   1.254 -
   1.255 -			if (!PageHighMem(page)) {
   1.256 -				v = phys_to_virt(pfn << PAGE_SHIFT);
   1.257 -				scrub_pages(v, 1);
   1.258 -				BUG_ON(HYPERVISOR_update_va_mapping(
   1.259 -					(unsigned long)v, __pte_ma(0), 0));
   1.260 -			}
   1.261 -#ifdef CONFIG_XEN_SCRUB_PAGES
   1.262 -			else {
   1.263 -				v = kmap(page);
   1.264 -				scrub_pages(v, 1);
   1.265 -				kunmap(page);
   1.266 -			}
   1.267 +#ifndef CONFIG_PREEMPT
   1.268 +		if (need_resched())
   1.269 +			schedule();
   1.270  #endif
   1.271 -		}
   1.272 -
   1.273 -		/* Ensure that ballooned highmem pages don't have kmaps. */
   1.274 -		kmap_flush_unused();
   1.275 -		flush_tlb_all();
   1.276 -
   1.277 -		/* No more mappings: invalidate P2M and add to balloon. */
   1.278 -		for (i = 0; i < debt; i++) {
   1.279 -			pfn = mfn_to_pfn(mfn_list[i]);
   1.280 -			phys_to_machine_mapping[pfn] = INVALID_P2M_ENTRY;
   1.281 -			balloon_append(pfn_to_page(pfn));
   1.282 -		}
   1.283 -
   1.284 -		reservation.extent_start = mfn_list;
   1.285 -		reservation.nr_extents   = debt;
   1.286 -		BUG_ON(HYPERVISOR_memory_op(
   1.287 -			XENMEM_decrease_reservation, &reservation) != debt);
   1.288 -
   1.289 -		current_pages -= debt;
   1.290 -	}
   1.291 -
   1.292 - out:
   1.293 -	if (mfn_list != NULL)
   1.294 -		vfree(mfn_list);
   1.295 +	} while ((credit != 0) && !need_sleep);
   1.296  
   1.297  	/* Schedule more work if there is some still to be done. */
   1.298  	if (current_target() != current_pages)
   1.299 @@ -441,8 +487,9 @@ subsys_initcall(balloon_init);
   1.300  void balloon_update_driver_allowance(long delta)
   1.301  {
   1.302  	unsigned long flags;
   1.303 +
   1.304  	balloon_lock(flags);
   1.305 -	driver_pages += delta; /* non-atomic update */
   1.306 +	driver_pages += delta;
   1.307  	balloon_unlock(flags);
   1.308  }
   1.309  
   1.310 @@ -475,9 +522,10 @@ struct page *balloon_alloc_empty_page_ra
   1.311  
   1.312  	scrub_pages(vstart, 1 << order);
   1.313  
   1.314 -	balloon_lock(flags);
   1.315  	BUG_ON(generic_page_range(
   1.316  		&init_mm, vstart, PAGE_SIZE << order, dealloc_pte_fn, NULL));
   1.317 +
   1.318 +	balloon_lock(flags);
   1.319  	current_pages -= 1UL << order;
   1.320  	balloon_unlock(flags);
   1.321