debuggers.hg

annotate xen/common/memory.c @ 22848:6341fe0f4e5a

Added tag 4.1.0-rc2 for changeset 9dca60d88c63
author Keir Fraser <keir@xen.org>
date Tue Jan 25 14:06:55 2011 +0000 (2011-01-25)
parents c5b42971234a
children
rev   line source
kaf24@6506 1 /******************************************************************************
kaf24@6506 2 * memory.c
kaf24@6506 3 *
kaf24@6506 4 * Code to handle memory-related requests.
kaf24@6506 5 *
kaf24@6506 6 * Copyright (c) 2003-2004, B Dragovic
kaf24@6506 7 * Copyright (c) 2003-2005, K A Fraser
kaf24@6506 8 */
kaf24@6506 9
kaf24@6506 10 #include <xen/config.h>
kaf24@6506 11 #include <xen/types.h>
kaf24@6506 12 #include <xen/lib.h>
kaf24@6506 13 #include <xen/mm.h>
kaf24@6506 14 #include <xen/perfc.h>
kaf24@6506 15 #include <xen/sched.h>
kaf24@6506 16 #include <xen/event.h>
Tim@15666 17 #include <xen/paging.h>
kaf24@8498 18 #include <xen/iocap.h>
kaf24@9054 19 #include <xen/guest_access.h>
ack@13297 20 #include <xen/hypercall.h>
kaf24@11236 21 #include <xen/errno.h>
keir@20505 22 #include <xen/tmem.h>
keir@22502 23 #include <xen/tmem_xen.h>
kaf24@6506 24 #include <asm/current.h>
kaf24@6506 25 #include <asm/hardirq.h>
keir@20805 26 #ifdef CONFIG_X86
keir@20805 27 # include <asm/p2m.h>
keir@20805 28 #endif
keir@17421 29 #include <xen/numa.h>
kaf24@6506 30 #include <public/memory.h>
kfraser@15846 31 #include <xsm/xsm.h>
keir@20924 32 #include <xen/trace.h>
kaf24@6506 33
kfraser@12398 34 struct memop_args {
kfraser@12398 35 /* INPUT */
kfraser@12398 36 struct domain *domain; /* Domain to be affected. */
kfraser@12398 37 XEN_GUEST_HANDLE(xen_pfn_t) extent_list; /* List of extent base addrs. */
kfraser@12398 38 unsigned int nr_extents; /* Number of extents to allocate or free. */
kfraser@12398 39 unsigned int extent_order; /* Size of each extent. */
kfraser@12398 40 unsigned int memflags; /* Allocation flags. */
kfraser@12398 41
kfraser@12398 42 /* INPUT/OUTPUT */
kfraser@12398 43 unsigned int nr_done; /* Number of extents processed so far. */
kfraser@12398 44 int preempted; /* Was the hypercall preempted? */
kfraser@12398 45 };
kfraser@12398 46
kfraser@12398 47 static void increase_reservation(struct memop_args *a)
kaf24@6506 48 {
kaf24@8764 49 struct page_info *page;
kaf24@10314 50 unsigned long i;
kaf24@10314 51 xen_pfn_t mfn;
kfraser@12398 52 struct domain *d = a->domain;
kaf24@6506 53
kfraser@12398 54 if ( !guest_handle_is_null(a->extent_list) &&
keir@17892 55 !guest_handle_subrange_okay(a->extent_list, a->nr_done,
keir@17892 56 a->nr_extents-1) )
kfraser@12398 57 return;
kaf24@6506 58
keir@19807 59 if ( !multipage_allocation_permitted(current->domain, a->extent_order) )
kfraser@12398 60 return;
kaf24@6506 61
kfraser@12398 62 for ( i = a->nr_done; i < a->nr_extents; i++ )
kaf24@6506 63 {
kaf24@6506 64 if ( hypercall_preempt_check() )
kaf24@6627 65 {
kfraser@12398 66 a->preempted = 1;
kfraser@12398 67 goto out;
kaf24@6627 68 }
kaf24@6506 69
keir@18021 70 page = alloc_domheap_pages(d, a->extent_order, a->memflags);
kfraser@12398 71 if ( unlikely(page == NULL) )
kaf24@6506 72 {
kaf24@12062 73 gdprintk(XENLOG_INFO, "Could not allocate order=%d extent: "
kfraser@10398 74 "id=%d memflags=%x (%ld of %d)\n",
kfraser@12398 75 a->extent_order, d->domain_id, a->memflags,
kfraser@12398 76 i, a->nr_extents);
kfraser@12398 77 goto out;
kaf24@6506 78 }
kaf24@6506 79
kaf24@6506 80 /* Inform the domain of the new page's machine address. */
kfraser@12398 81 if ( !guest_handle_is_null(a->extent_list) )
kaf24@8882 82 {
kaf24@8882 83 mfn = page_to_mfn(page);
kfraser@12398 84 if ( unlikely(__copy_to_guest_offset(a->extent_list, i, &mfn, 1)) )
kfraser@12398 85 goto out;
kaf24@8882 86 }
kaf24@6506 87 }
kaf24@6506 88
kfraser@12398 89 out:
kfraser@12398 90 a->nr_done = i;
kaf24@6506 91 }
sos22@8726 92
kfraser@12398 93 static void populate_physmap(struct memop_args *a)
kaf24@8711 94 {
kaf24@8764 95 struct page_info *page;
kaf24@10314 96 unsigned long i, j;
kfraser@12398 97 xen_pfn_t gpfn, mfn;
kfraser@12398 98 struct domain *d = a->domain;
kaf24@8711 99
keir@17892 100 if ( !guest_handle_subrange_okay(a->extent_list, a->nr_done,
keir@17892 101 a->nr_extents-1) )
kfraser@12398 102 return;
kaf24@8711 103
keir@19807 104 if ( !multipage_allocation_permitted(current->domain, a->extent_order) )
kfraser@12398 105 return;
kaf24@8711 106
kfraser@12398 107 for ( i = a->nr_done; i < a->nr_extents; i++ )
kaf24@8711 108 {
kaf24@8711 109 if ( hypercall_preempt_check() )
kaf24@8711 110 {
kfraser@12398 111 a->preempted = 1;
sos22@8726 112 goto out;
kaf24@8711 113 }
kaf24@8711 114
kfraser@12398 115 if ( unlikely(__copy_from_guest_offset(&gpfn, a->extent_list, i, 1)) )
kaf24@8882 116 goto out;
kaf24@8882 117
keir@18996 118 if ( a->memflags & MEMF_populate_on_demand )
keir@18996 119 {
keir@18996 120 if ( guest_physmap_mark_populate_on_demand(d, gpfn,
keir@18996 121 a->extent_order) < 0 )
keir@18996 122 goto out;
keir@18996 123 }
keir@18996 124 else
kaf24@8711 125 {
keir@18996 126 page = alloc_domheap_pages(d, a->extent_order, a->memflags);
keir@18996 127 if ( unlikely(page == NULL) )
keir@18996 128 {
keir@20505 129 if ( !opt_tmem || (a->extent_order != 0) )
keir@20505 130 gdprintk(XENLOG_INFO, "Could not allocate order=%d extent:"
keir@20505 131 " id=%d memflags=%x (%ld of %d)\n",
keir@20505 132 a->extent_order, d->domain_id, a->memflags,
keir@20505 133 i, a->nr_extents);
keir@18996 134 goto out;
keir@18996 135 }
kaf24@8711 136
keir@18996 137 mfn = page_to_mfn(page);
keir@18996 138 guest_physmap_add_page(d, gpfn, mfn, a->extent_order);
kaf24@8711 139
keir@18996 140 if ( !paging_mode_translate(d) )
keir@18996 141 {
keir@18996 142 for ( j = 0; j < (1 << a->extent_order); j++ )
keir@18996 143 set_gpfn_from_mfn(mfn + j, gpfn + j);
kaf24@8711 144
keir@18996 145 /* Inform the domain of the new page's machine address. */
keir@18996 146 if ( unlikely(__copy_to_guest_offset(a->extent_list, i, &mfn, 1)) )
keir@18996 147 goto out;
keir@18996 148 }
sos22@8726 149 }
kaf24@8711 150 }
kaf24@8711 151
keir@18996 152 out:
kfraser@12398 153 a->nr_done = i;
kaf24@8711 154 }
cl349@9225 155
kfraser@12398 156 int guest_remove_page(struct domain *d, unsigned long gmfn)
cl349@9225 157 {
cl349@9225 158 struct page_info *page;
keir@20805 159 #ifdef CONFIG_X86
keir@20726 160 p2m_type_t p2mt;
keir@20805 161 #endif
cl349@9225 162 unsigned long mfn;
cl349@9225 163
keir@20805 164 #ifdef CONFIG_X86
keir@21986 165 mfn = mfn_x(gfn_to_mfn(p2m_get_hostp2m(d), gmfn, &p2mt));
keir@22751 166 if ( unlikely(p2m_is_paging(p2mt)) )
keir@22751 167 {
keir@22751 168 guest_physmap_remove_page(d, gmfn, mfn, 0);
keir@22751 169 p2m_mem_paging_drop_page(p2m_get_hostp2m(d), gmfn);
keir@22751 170 return 1;
keir@22751 171 }
keir@20805 172 #else
keir@20805 173 mfn = gmfn_to_mfn(d, gmfn);
keir@20805 174 #endif
cl349@9225 175 if ( unlikely(!mfn_valid(mfn)) )
cl349@9225 176 {
kaf24@12062 177 gdprintk(XENLOG_INFO, "Domain %u page number %lx invalid\n",
tdeegan@11189 178 d->domain_id, gmfn);
cl349@9225 179 return 0;
cl349@9225 180 }
cl349@9225 181
cl349@9225 182 page = mfn_to_page(mfn);
keir@20805 183 #ifdef CONFIG_X86
keir@20726 184 /* If gmfn is shared, just drop the guest reference (which may or may not
keir@20726 185 * free the page) */
keir@20726 186 if(p2m_is_shared(p2mt))
keir@20726 187 {
keir@20726 188 put_page_and_type(page);
keir@20726 189 guest_physmap_remove_page(d, gmfn, mfn, 0);
keir@20726 190 return 1;
keir@20726 191 }
keir@20726 192
keir@20805 193 #endif /* CONFIG_X86 */
cl349@9225 194 if ( unlikely(!get_page(page, d)) )
cl349@9225 195 {
kaf24@12062 196 gdprintk(XENLOG_INFO, "Bad page free for domain %u\n", d->domain_id);
cl349@9225 197 return 0;
cl349@9225 198 }
cl349@9225 199
cl349@9225 200 if ( test_and_clear_bit(_PGT_pinned, &page->u.inuse.type_info) )
cl349@9225 201 put_page_and_type(page);
cl349@9225 202
cl349@9225 203 if ( test_and_clear_bit(_PGC_allocated, &page->count_info) )
cl349@9225 204 put_page(page);
cl349@9225 205
keir@17683 206 guest_physmap_remove_page(d, gmfn, mfn, 0);
cl349@9225 207
cl349@9225 208 put_page(page);
cl349@9225 209
cl349@9225 210 return 1;
cl349@9225 211 }
cl349@9225 212
kfraser@12398 213 static void decrease_reservation(struct memop_args *a)
kaf24@6506 214 {
kaf24@10314 215 unsigned long i, j;
kaf24@10314 216 xen_pfn_t gmfn;
kaf24@6506 217
keir@17892 218 if ( !guest_handle_subrange_okay(a->extent_list, a->nr_done,
keir@17892 219 a->nr_extents-1) )
kfraser@12398 220 return;
kaf24@6506 221
kfraser@12398 222 for ( i = a->nr_done; i < a->nr_extents; i++ )
kaf24@6506 223 {
kaf24@6506 224 if ( hypercall_preempt_check() )
kaf24@6627 225 {
kfraser@12398 226 a->preempted = 1;
kfraser@12398 227 goto out;
kaf24@6627 228 }
kaf24@6506 229
kfraser@12398 230 if ( unlikely(__copy_from_guest_offset(&gmfn, a->extent_list, i, 1)) )
kfraser@12398 231 goto out;
kaf24@6506 232
keir@20924 233 if ( tb_init_done )
keir@20924 234 {
keir@20924 235 struct {
keir@20924 236 u64 gfn;
keir@20924 237 int d:16,order:16;
keir@20924 238 } t;
keir@20924 239
keir@20924 240 t.gfn = gmfn;
keir@20924 241 t.d = a->domain->domain_id;
keir@20924 242 t.order = a->extent_order;
keir@20924 243
keir@22230 244 __trace_var(TRC_MEM_DECREASE_RESERVATION, 0, sizeof(t), &t);
keir@20924 245 }
keir@20924 246
keir@18993 247 /* See if populate-on-demand wants to handle this */
keir@18993 248 if ( is_hvm_domain(a->domain)
keir@18993 249 && p2m_pod_decrease_reservation(a->domain, gmfn, a->extent_order) )
keir@18993 250 continue;
keir@18993 251
kfraser@12398 252 for ( j = 0; j < (1 << a->extent_order); j++ )
kfraser@12398 253 if ( !guest_remove_page(a->domain, gmfn + j) )
kfraser@12398 254 goto out;
kaf24@6506 255 }
kaf24@6506 256
kfraser@12398 257 out:
kfraser@12398 258 a->nr_done = i;
kaf24@6506 259 }
kaf24@6506 260
kfraser@12398 261 static long memory_exchange(XEN_GUEST_HANDLE(xen_memory_exchange_t) arg)
kfraser@10398 262 {
kfraser@10398 263 struct xen_memory_exchange exch;
keir@19170 264 PAGE_LIST_HEAD(in_chunk_list);
keir@19170 265 PAGE_LIST_HEAD(out_chunk_list);
kfraser@10398 266 unsigned long in_chunk_order, out_chunk_order;
kaf24@10465 267 xen_pfn_t gpfn, gmfn, mfn;
kfraser@10398 268 unsigned long i, j, k;
keir@21959 269 unsigned int memflags = 0;
kfraser@10398 270 long rc = 0;
kfraser@10398 271 struct domain *d;
kfraser@10398 272 struct page_info *page;
kfraser@10398 273
kfraser@10398 274 if ( copy_from_guest(&exch, arg, 1) )
kfraser@10398 275 return -EFAULT;
kfraser@10398 276
kfraser@10398 277 /* Various sanity checks. */
kfraser@10398 278 if ( (exch.nr_exchanged > exch.in.nr_extents) ||
kfraser@10398 279 /* Input and output domain identifiers match? */
kfraser@10398 280 (exch.in.domid != exch.out.domid) ||
kfraser@10398 281 /* Sizes of input and output lists do not overflow a long? */
kfraser@10398 282 ((~0UL >> exch.in.extent_order) < exch.in.nr_extents) ||
kfraser@10398 283 ((~0UL >> exch.out.extent_order) < exch.out.nr_extents) ||
kfraser@10398 284 /* Sizes of input and output lists match? */
kfraser@10398 285 ((exch.in.nr_extents << exch.in.extent_order) !=
kfraser@10398 286 (exch.out.nr_extents << exch.out.extent_order)) )
kfraser@10398 287 {
kfraser@10398 288 rc = -EINVAL;
kfraser@10398 289 goto fail_early;
kfraser@10398 290 }
kfraser@10398 291
kfraser@10398 292 /* Only privileged guests can allocate multi-page contiguous extents. */
keir@19807 293 if ( !multipage_allocation_permitted(current->domain,
keir@19807 294 exch.in.extent_order) ||
keir@19807 295 !multipage_allocation_permitted(current->domain,
keir@19807 296 exch.out.extent_order) )
kfraser@10398 297 {
kfraser@10398 298 rc = -EPERM;
kfraser@10398 299 goto fail_early;
kfraser@10398 300 }
kfraser@10398 301
kfraser@10398 302 if ( exch.in.extent_order <= exch.out.extent_order )
kfraser@10398 303 {
kfraser@10398 304 in_chunk_order = exch.out.extent_order - exch.in.extent_order;
kfraser@10398 305 out_chunk_order = 0;
kfraser@10398 306 }
kfraser@10398 307 else
kfraser@10398 308 {
kfraser@10398 309 in_chunk_order = 0;
kfraser@10398 310 out_chunk_order = exch.in.extent_order - exch.out.extent_order;
kfraser@10398 311 }
kfraser@10398 312
keir@19968 313 if ( likely(exch.in.domid == DOMID_SELF) )
keir@19968 314 {
keir@19968 315 d = rcu_lock_current_domain();
keir@19968 316 }
keir@19968 317 else
kfraser@10398 318 {
keir@19968 319 if ( (d = rcu_lock_domain_by_id(exch.in.domid)) == NULL )
keir@19968 320 goto fail_early;
keir@19968 321
keir@19968 322 if ( !IS_PRIV_FOR(current->domain, d) )
keir@19968 323 {
keir@19968 324 rcu_unlock_domain(d);
keir@19968 325 rc = -EPERM;
keir@19968 326 goto fail_early;
keir@19968 327 }
kfraser@10398 328 }
kfraser@10398 329
keir@16586 330 memflags |= MEMF_bits(domain_clamp_alloc_bitsize(
keir@18021 331 d,
keir@18021 332 XENMEMF_get_address_bits(exch.out.mem_flags) ? :
keir@18021 333 (BITS_PER_LONG+PAGE_SHIFT)));
keir@21959 334 memflags |= MEMF_node(XENMEMF_get_node(exch.out.mem_flags));
kfraser@11933 335
kfraser@12398 336 for ( i = (exch.nr_exchanged >> in_chunk_order);
kfraser@12398 337 i < (exch.in.nr_extents >> in_chunk_order);
kfraser@12398 338 i++ )
kfraser@10398 339 {
kfraser@10398 340 if ( hypercall_preempt_check() )
kfraser@10398 341 {
kfraser@12398 342 exch.nr_exchanged = i << in_chunk_order;
keir@19968 343 rcu_unlock_domain(d);
kfraser@10398 344 if ( copy_field_to_guest(arg, &exch, nr_exchanged) )
kfraser@10398 345 return -EFAULT;
kfraser@10398 346 return hypercall_create_continuation(
kfraser@10398 347 __HYPERVISOR_memory_op, "lh", XENMEM_exchange, arg);
kfraser@10398 348 }
kfraser@10398 349
kfraser@10398 350 /* Steal a chunk's worth of input pages from the domain. */
kfraser@10398 351 for ( j = 0; j < (1UL << in_chunk_order); j++ )
kfraser@10398 352 {
kfraser@10398 353 if ( unlikely(__copy_from_guest_offset(
kfraser@10398 354 &gmfn, exch.in.extent_start, (i<<in_chunk_order)+j, 1)) )
kfraser@10398 355 {
kfraser@10398 356 rc = -EFAULT;
kfraser@10398 357 goto fail;
kfraser@10398 358 }
kfraser@10398 359
kfraser@10398 360 for ( k = 0; k < (1UL << exch.in.extent_order); k++ )
kfraser@10398 361 {
keir@20805 362 #ifdef CONFIG_X86
keir@20726 363 p2m_type_t p2mt;
keir@20726 364
keir@20726 365 /* Shared pages cannot be exchanged */
keir@21986 366 mfn = mfn_x(gfn_to_mfn_unshare(p2m_get_hostp2m(d), gmfn + k, &p2mt, 0));
keir@20726 367 if ( p2m_is_shared(p2mt) )
keir@20726 368 {
keir@20726 369 rc = -ENOMEM;
keir@20726 370 goto fail;
keir@20726 371 }
keir@20805 372 #else /* !CONFIG_X86 */
keir@20805 373 mfn = gmfn_to_mfn(d, gmfn + k);
keir@20805 374 #endif
kfraser@10398 375 if ( unlikely(!mfn_valid(mfn)) )
kfraser@10398 376 {
kfraser@10398 377 rc = -EINVAL;
kfraser@10398 378 goto fail;
kfraser@10398 379 }
kfraser@10398 380
kfraser@10398 381 page = mfn_to_page(mfn);
kfraser@10398 382
kfraser@10398 383 if ( unlikely(steal_page(d, page, MEMF_no_refcount)) )
kfraser@10398 384 {
kfraser@10398 385 rc = -EINVAL;
kfraser@10398 386 goto fail;
kfraser@10398 387 }
kfraser@10398 388
keir@19170 389 page_list_add(page, &in_chunk_list);
kfraser@10398 390 }
kfraser@10398 391 }
kfraser@10398 392
kfraser@10398 393 /* Allocate a chunk's worth of anonymous output pages. */
kfraser@10398 394 for ( j = 0; j < (1UL << out_chunk_order); j++ )
kfraser@10398 395 {
keir@17421 396 page = alloc_domheap_pages(NULL, exch.out.extent_order, memflags);
kfraser@10398 397 if ( unlikely(page == NULL) )
kfraser@10398 398 {
kfraser@10398 399 rc = -ENOMEM;
kfraser@10398 400 goto fail;
kfraser@10398 401 }
kfraser@10398 402
keir@19170 403 page_list_add(page, &out_chunk_list);
kfraser@10398 404 }
kfraser@10398 405
kfraser@10398 406 /*
kfraser@10398 407 * Success! Beyond this point we cannot fail for this chunk.
kfraser@10398 408 */
kfraser@10398 409
kfraser@10398 410 /* Destroy final reference to each input page. */
keir@19170 411 while ( (page = page_list_remove_head(&in_chunk_list)) )
kfraser@10398 412 {
keir@20726 413 unsigned long gfn;
keir@20726 414
kfraser@10398 415 if ( !test_and_clear_bit(_PGC_allocated, &page->count_info) )
kfraser@10398 416 BUG();
kfraser@10398 417 mfn = page_to_mfn(page);
keir@20726 418 gfn = mfn_to_gmfn(d, mfn);
keir@20726 419 /* Pages were unshared above */
keir@20726 420 BUG_ON(SHARED_M2P(gfn));
keir@20726 421 guest_physmap_remove_page(d, gfn, mfn, 0);
kfraser@10398 422 put_page(page);
kfraser@10398 423 }
kfraser@10398 424
kfraser@10398 425 /* Assign each output page to the domain. */
kfraser@10398 426 j = 0;
keir@19170 427 while ( (page = page_list_remove_head(&out_chunk_list)) )
kfraser@10398 428 {
kfraser@10398 429 if ( assign_pages(d, page, exch.out.extent_order,
kfraser@10398 430 MEMF_no_refcount) )
keir@19968 431 {
keir@19968 432 unsigned long dec_count;
keir@19968 433 bool_t drop_dom_ref;
keir@19968 434
keir@19968 435 /*
keir@19968 436 * Pages in in_chunk_list is stolen without
keir@19968 437 * decreasing the tot_pages. If the domain is dying when
keir@19968 438 * assign pages, we need decrease the count. For those pages
keir@19968 439 * that has been assigned, it should be covered by
keir@19968 440 * domain_relinquish_resources().
keir@19968 441 */
keir@19968 442 dec_count = (((1UL << exch.in.extent_order) *
keir@19968 443 (1UL << in_chunk_order)) -
keir@19968 444 (j * (1UL << exch.out.extent_order)));
keir@19968 445
keir@19968 446 spin_lock(&d->page_alloc_lock);
keir@19968 447 d->tot_pages -= dec_count;
keir@19968 448 drop_dom_ref = (dec_count && !d->tot_pages);
keir@19968 449 spin_unlock(&d->page_alloc_lock);
keir@19968 450
keir@19968 451 if ( drop_dom_ref )
keir@19968 452 put_domain(d);
keir@19968 453
keir@19968 454 free_domheap_pages(page, exch.out.extent_order);
keir@19968 455 goto dying;
keir@19968 456 }
kfraser@10398 457
kfraser@10398 458 /* Note that we ignore errors accessing the output extent list. */
kfraser@10398 459 (void)__copy_from_guest_offset(
kfraser@10398 460 &gpfn, exch.out.extent_start, (i<<out_chunk_order)+j, 1);
kfraser@10398 461
kfraser@10398 462 mfn = page_to_mfn(page);
keir@17761 463 guest_physmap_add_page(d, gpfn, mfn, exch.out.extent_order);
keir@17761 464
keir@17761 465 if ( !paging_mode_translate(d) )
kfraser@10398 466 {
kfraser@10398 467 for ( k = 0; k < (1UL << exch.out.extent_order); k++ )
kfraser@10398 468 set_gpfn_from_mfn(mfn + k, gpfn + k);
kfraser@10398 469 (void)__copy_to_guest_offset(
kfraser@10398 470 exch.out.extent_start, (i<<out_chunk_order)+j, &mfn, 1);
kfraser@10398 471 }
kfraser@10398 472 j++;
kfraser@10398 473 }
keir@19968 474 BUG_ON( !(d->is_dying) && (j != (1UL << out_chunk_order)) );
kfraser@10398 475 }
kfraser@10398 476
kfraser@12398 477 exch.nr_exchanged = exch.in.nr_extents;
kfraser@10398 478 if ( copy_field_to_guest(arg, &exch, nr_exchanged) )
kfraser@10398 479 rc = -EFAULT;
keir@19968 480 rcu_unlock_domain(d);
kfraser@10398 481 return rc;
kfraser@10398 482
kfraser@10398 483 /*
kfraser@10398 484 * Failed a chunk! Free any partial chunk work. Tell caller how many
kfraser@10398 485 * chunks succeeded.
kfraser@10398 486 */
kfraser@10398 487 fail:
kfraser@10398 488 /* Reassign any input pages we managed to steal. */
keir@19170 489 while ( (page = page_list_remove_head(&in_chunk_list)) )
kfraser@10398 490 if ( assign_pages(d, page, 0, MEMF_no_refcount) )
kfraser@10398 491 BUG();
keir@19968 492 dying:
keir@19968 493 rcu_unlock_domain(d);
kfraser@10398 494 /* Free any output pages we managed to allocate. */
keir@19170 495 while ( (page = page_list_remove_head(&out_chunk_list)) )
kfraser@10398 496 free_domheap_pages(page, exch.out.extent_order);
kfraser@10398 497
kfraser@12398 498 exch.nr_exchanged = i << in_chunk_order;
kfraser@10398 499
kfraser@10398 500 fail_early:
kfraser@10398 501 if ( copy_field_to_guest(arg, &exch, nr_exchanged) )
kfraser@10398 502 rc = -EFAULT;
kfraser@10398 503 return rc;
kfraser@10398 504 }
kfraser@10398 505
kaf24@9904 506 long do_memory_op(unsigned long cmd, XEN_GUEST_HANDLE(void) arg)
kaf24@6506 507 {
kaf24@6506 508 struct domain *d;
kfraser@12398 509 int rc, op;
keir@18021 510 unsigned int address_bits;
keir@19078 511 unsigned long start_extent;
kaf24@6506 512 struct xen_memory_reservation reservation;
kfraser@12398 513 struct memop_args args;
kaf24@7981 514 domid_t domid;
kaf24@6506 515
ack@13297 516 op = cmd & MEMOP_CMD_MASK;
kaf24@6506 517
kaf24@6506 518 switch ( op )
kaf24@6506 519 {
kaf24@6506 520 case XENMEM_increase_reservation:
kaf24@6506 521 case XENMEM_decrease_reservation:
kaf24@8711 522 case XENMEM_populate_physmap:
ack@13297 523 start_extent = cmd >> MEMOP_EXTENT_SHIFT;
kfraser@10398 524
kaf24@9054 525 if ( copy_from_guest(&reservation, arg, 1) )
kfraser@10398 526 return start_extent;
kaf24@6506 527
kaf24@8894 528 /* Is size too large for us to encode a continuation? */
ack@13297 529 if ( reservation.nr_extents > (ULONG_MAX >> MEMOP_EXTENT_SHIFT) )
kfraser@10398 530 return start_extent;
kaf24@8894 531
keir@21131 532 if ( unlikely(start_extent >= reservation.nr_extents) )
kfraser@10398 533 return start_extent;
kaf24@9054 534
kfraser@12398 535 args.extent_list = reservation.extent_start;
kfraser@12398 536 args.nr_extents = reservation.nr_extents;
kfraser@12398 537 args.extent_order = reservation.extent_order;
kfraser@12398 538 args.nr_done = start_extent;
kfraser@12398 539 args.preempted = 0;
kfraser@12398 540 args.memflags = 0;
kaf24@6506 541
keir@18021 542 address_bits = XENMEMF_get_address_bits(reservation.mem_flags);
keir@18021 543 if ( (address_bits != 0) &&
keir@18021 544 (address_bits < (get_order_from_pages(max_page) + PAGE_SHIFT)) )
kaf24@6506 545 {
keir@18021 546 if ( address_bits <= PAGE_SHIFT )
kfraser@10398 547 return start_extent;
keir@18021 548 args.memflags = MEMF_bits(address_bits);
kaf24@6506 549 }
kaf24@6506 550
keir@18021 551 args.memflags |= MEMF_node(XENMEMF_get_node(reservation.mem_flags));
keir@21959 552 if ( reservation.mem_flags & XENMEMF_exact_node_request )
keir@21765 553 args.memflags |= MEMF_exact_node;
keir@18021 554
keir@18996 555 if ( op == XENMEM_populate_physmap
keir@18996 556 && (reservation.mem_flags & XENMEMF_populate_on_demand) )
keir@18996 557 args.memflags |= MEMF_populate_on_demand;
keir@18996 558
kaf24@6506 559 if ( likely(reservation.domid == DOMID_SELF) )
keir@17357 560 {
keir@17357 561 d = rcu_lock_current_domain();
keir@17357 562 }
keir@17357 563 else
keir@17357 564 {
keir@17357 565 if ( (d = rcu_lock_domain_by_id(reservation.domid)) == NULL )
keir@16894 566 return start_extent;
keir@17357 567 if ( !IS_PRIV_FOR(current->domain, d) )
keir@17357 568 {
keir@16894 569 rcu_unlock_domain(d);
keir@16894 570 return start_extent;
keir@16894 571 }
keir@16894 572 }
kfraser@12398 573 args.domain = d;
kaf24@6506 574
kfraser@15846 575 rc = xsm_memory_adjust_reservation(current->domain, d);
kfraser@15846 576 if ( rc )
kfraser@15846 577 {
keir@17357 578 rcu_unlock_domain(d);
kfraser@15846 579 return rc;
kfraser@15846 580 }
kfraser@15846 581
kaf24@8711 582 switch ( op )
kaf24@8711 583 {
kaf24@8711 584 case XENMEM_increase_reservation:
kfraser@12398 585 increase_reservation(&args);
kaf24@8711 586 break;
kaf24@8711 587 case XENMEM_decrease_reservation:
kfraser@12398 588 decrease_reservation(&args);
kaf24@8711 589 break;
kfraser@12398 590 default: /* XENMEM_populate_physmap */
kfraser@12398 591 populate_physmap(&args);
kaf24@8711 592 break;
kaf24@8711 593 }
kaf24@6506 594
keir@17357 595 rcu_unlock_domain(d);
kaf24@6506 596
kfraser@12398 597 rc = args.nr_done;
kaf24@6506 598
kfraser@12398 599 if ( args.preempted )
kaf24@9054 600 return hypercall_create_continuation(
kaf24@9054 601 __HYPERVISOR_memory_op, "lh",
ack@13297 602 op | (rc << MEMOP_EXTENT_SHIFT), arg);
kaf24@6627 603
kaf24@6506 604 break;
kaf24@6506 605
kfraser@10398 606 case XENMEM_exchange:
kfraser@10398 607 rc = memory_exchange(guest_handle_cast(arg, xen_memory_exchange_t));
kfraser@10398 608 break;
kfraser@10398 609
kaf24@6506 610 case XENMEM_maximum_ram_page:
kaf24@7981 611 rc = max_page;
kaf24@7981 612 break;
kaf24@7981 613
kaf24@7981 614 case XENMEM_current_reservation:
kaf24@7981 615 case XENMEM_maximum_reservation:
kfraser@14478 616 case XENMEM_maximum_gpfn:
kaf24@9054 617 if ( copy_from_guest(&domid, arg, 1) )
kaf24@6506 618 return -EFAULT;
kaf24@7981 619
keir@18604 620 rc = rcu_lock_target_domain_by_id(domid, &d);
keir@18604 621 if ( rc )
keir@18604 622 return rc;
kaf24@7981 623
kfraser@15846 624 rc = xsm_memory_stat_reservation(current->domain, d);
kfraser@15846 625 if ( rc )
kfraser@15846 626 {
keir@17357 627 rcu_unlock_domain(d);
kfraser@15846 628 return rc;
kfraser@15846 629 }
kfraser@15846 630
kfraser@14478 631 switch ( op )
kfraser@14478 632 {
kfraser@14478 633 case XENMEM_current_reservation:
kfraser@14478 634 rc = d->tot_pages;
kfraser@14478 635 break;
kfraser@14478 636 case XENMEM_maximum_reservation:
kfraser@14478 637 rc = d->max_pages;
kfraser@14478 638 break;
kfraser@14478 639 default:
kfraser@14478 640 ASSERT(op == XENMEM_maximum_gpfn);
kfraser@14478 641 rc = domain_get_maximum_gpfn(d);
kfraser@14478 642 break;
kfraser@14478 643 }
kaf24@7981 644
keir@17357 645 rcu_unlock_domain(d);
kaf24@7981 646
kaf24@6506 647 break;
kaf24@6506 648
kaf24@6506 649 default:
kaf24@8081 650 rc = arch_memory_op(op, arg);
kaf24@6506 651 break;
kaf24@6506 652 }
kaf24@6506 653
kaf24@6506 654 return rc;
kaf24@6506 655 }
kaf24@6506 656
kaf24@6506 657 /*
kaf24@6506 658 * Local variables:
kaf24@6506 659 * mode: C
kaf24@6506 660 * c-set-style: "BSD"
kaf24@6506 661 * c-basic-offset: 4
kaf24@6506 662 * tab-width: 4
kaf24@6506 663 * indent-tabs-mode: nil
kaf24@6506 664 * End:
kaf24@6506 665 */