rev |
line source |
kaf24@6506
|
1 /******************************************************************************
|
kaf24@6506
|
2 * memory.c
|
kaf24@6506
|
3 *
|
kaf24@6506
|
4 * Code to handle memory-related requests.
|
kaf24@6506
|
5 *
|
kaf24@6506
|
6 * Copyright (c) 2003-2004, B Dragovic
|
kaf24@6506
|
7 * Copyright (c) 2003-2005, K A Fraser
|
kaf24@6506
|
8 */
|
kaf24@6506
|
9
|
kaf24@6506
|
10 #include <xen/config.h>
|
kaf24@6506
|
11 #include <xen/types.h>
|
kaf24@6506
|
12 #include <xen/lib.h>
|
kaf24@6506
|
13 #include <xen/mm.h>
|
kaf24@6506
|
14 #include <xen/perfc.h>
|
kaf24@6506
|
15 #include <xen/sched.h>
|
kaf24@6506
|
16 #include <xen/event.h>
|
Tim@15666
|
17 #include <xen/paging.h>
|
kaf24@8498
|
18 #include <xen/iocap.h>
|
kaf24@9054
|
19 #include <xen/guest_access.h>
|
ack@13297
|
20 #include <xen/hypercall.h>
|
kaf24@11236
|
21 #include <xen/errno.h>
|
keir@20505
|
22 #include <xen/tmem.h>
|
keir@22502
|
23 #include <xen/tmem_xen.h>
|
kaf24@6506
|
24 #include <asm/current.h>
|
kaf24@6506
|
25 #include <asm/hardirq.h>
|
keir@20805
|
26 #ifdef CONFIG_X86
|
keir@20805
|
27 # include <asm/p2m.h>
|
keir@20805
|
28 #endif
|
keir@17421
|
29 #include <xen/numa.h>
|
kaf24@6506
|
30 #include <public/memory.h>
|
kfraser@15846
|
31 #include <xsm/xsm.h>
|
keir@20924
|
32 #include <xen/trace.h>
|
kaf24@6506
|
33
|
kfraser@12398
|
34 struct memop_args {
|
kfraser@12398
|
35 /* INPUT */
|
kfraser@12398
|
36 struct domain *domain; /* Domain to be affected. */
|
kfraser@12398
|
37 XEN_GUEST_HANDLE(xen_pfn_t) extent_list; /* List of extent base addrs. */
|
kfraser@12398
|
38 unsigned int nr_extents; /* Number of extents to allocate or free. */
|
kfraser@12398
|
39 unsigned int extent_order; /* Size of each extent. */
|
kfraser@12398
|
40 unsigned int memflags; /* Allocation flags. */
|
kfraser@12398
|
41
|
kfraser@12398
|
42 /* INPUT/OUTPUT */
|
kfraser@12398
|
43 unsigned int nr_done; /* Number of extents processed so far. */
|
kfraser@12398
|
44 int preempted; /* Was the hypercall preempted? */
|
kfraser@12398
|
45 };
|
kfraser@12398
|
46
|
kfraser@12398
|
47 static void increase_reservation(struct memop_args *a)
|
kaf24@6506
|
48 {
|
kaf24@8764
|
49 struct page_info *page;
|
kaf24@10314
|
50 unsigned long i;
|
kaf24@10314
|
51 xen_pfn_t mfn;
|
kfraser@12398
|
52 struct domain *d = a->domain;
|
kaf24@6506
|
53
|
kfraser@12398
|
54 if ( !guest_handle_is_null(a->extent_list) &&
|
keir@17892
|
55 !guest_handle_subrange_okay(a->extent_list, a->nr_done,
|
keir@17892
|
56 a->nr_extents-1) )
|
kfraser@12398
|
57 return;
|
kaf24@6506
|
58
|
keir@19807
|
59 if ( !multipage_allocation_permitted(current->domain, a->extent_order) )
|
kfraser@12398
|
60 return;
|
kaf24@6506
|
61
|
kfraser@12398
|
62 for ( i = a->nr_done; i < a->nr_extents; i++ )
|
kaf24@6506
|
63 {
|
kaf24@6506
|
64 if ( hypercall_preempt_check() )
|
kaf24@6627
|
65 {
|
kfraser@12398
|
66 a->preempted = 1;
|
kfraser@12398
|
67 goto out;
|
kaf24@6627
|
68 }
|
kaf24@6506
|
69
|
keir@18021
|
70 page = alloc_domheap_pages(d, a->extent_order, a->memflags);
|
kfraser@12398
|
71 if ( unlikely(page == NULL) )
|
kaf24@6506
|
72 {
|
kaf24@12062
|
73 gdprintk(XENLOG_INFO, "Could not allocate order=%d extent: "
|
kfraser@10398
|
74 "id=%d memflags=%x (%ld of %d)\n",
|
kfraser@12398
|
75 a->extent_order, d->domain_id, a->memflags,
|
kfraser@12398
|
76 i, a->nr_extents);
|
kfraser@12398
|
77 goto out;
|
kaf24@6506
|
78 }
|
kaf24@6506
|
79
|
kaf24@6506
|
80 /* Inform the domain of the new page's machine address. */
|
kfraser@12398
|
81 if ( !guest_handle_is_null(a->extent_list) )
|
kaf24@8882
|
82 {
|
kaf24@8882
|
83 mfn = page_to_mfn(page);
|
kfraser@12398
|
84 if ( unlikely(__copy_to_guest_offset(a->extent_list, i, &mfn, 1)) )
|
kfraser@12398
|
85 goto out;
|
kaf24@8882
|
86 }
|
kaf24@6506
|
87 }
|
kaf24@6506
|
88
|
kfraser@12398
|
89 out:
|
kfraser@12398
|
90 a->nr_done = i;
|
kaf24@6506
|
91 }
|
sos22@8726
|
92
|
kfraser@12398
|
93 static void populate_physmap(struct memop_args *a)
|
kaf24@8711
|
94 {
|
kaf24@8764
|
95 struct page_info *page;
|
kaf24@10314
|
96 unsigned long i, j;
|
kfraser@12398
|
97 xen_pfn_t gpfn, mfn;
|
kfraser@12398
|
98 struct domain *d = a->domain;
|
kaf24@8711
|
99
|
keir@17892
|
100 if ( !guest_handle_subrange_okay(a->extent_list, a->nr_done,
|
keir@17892
|
101 a->nr_extents-1) )
|
kfraser@12398
|
102 return;
|
kaf24@8711
|
103
|
keir@19807
|
104 if ( !multipage_allocation_permitted(current->domain, a->extent_order) )
|
kfraser@12398
|
105 return;
|
kaf24@8711
|
106
|
kfraser@12398
|
107 for ( i = a->nr_done; i < a->nr_extents; i++ )
|
kaf24@8711
|
108 {
|
kaf24@8711
|
109 if ( hypercall_preempt_check() )
|
kaf24@8711
|
110 {
|
kfraser@12398
|
111 a->preempted = 1;
|
sos22@8726
|
112 goto out;
|
kaf24@8711
|
113 }
|
kaf24@8711
|
114
|
kfraser@12398
|
115 if ( unlikely(__copy_from_guest_offset(&gpfn, a->extent_list, i, 1)) )
|
kaf24@8882
|
116 goto out;
|
kaf24@8882
|
117
|
keir@18996
|
118 if ( a->memflags & MEMF_populate_on_demand )
|
keir@18996
|
119 {
|
keir@18996
|
120 if ( guest_physmap_mark_populate_on_demand(d, gpfn,
|
keir@18996
|
121 a->extent_order) < 0 )
|
keir@18996
|
122 goto out;
|
keir@18996
|
123 }
|
keir@18996
|
124 else
|
kaf24@8711
|
125 {
|
keir@18996
|
126 page = alloc_domheap_pages(d, a->extent_order, a->memflags);
|
keir@18996
|
127 if ( unlikely(page == NULL) )
|
keir@18996
|
128 {
|
keir@20505
|
129 if ( !opt_tmem || (a->extent_order != 0) )
|
keir@20505
|
130 gdprintk(XENLOG_INFO, "Could not allocate order=%d extent:"
|
keir@20505
|
131 " id=%d memflags=%x (%ld of %d)\n",
|
keir@20505
|
132 a->extent_order, d->domain_id, a->memflags,
|
keir@20505
|
133 i, a->nr_extents);
|
keir@18996
|
134 goto out;
|
keir@18996
|
135 }
|
kaf24@8711
|
136
|
keir@18996
|
137 mfn = page_to_mfn(page);
|
keir@18996
|
138 guest_physmap_add_page(d, gpfn, mfn, a->extent_order);
|
kaf24@8711
|
139
|
keir@18996
|
140 if ( !paging_mode_translate(d) )
|
keir@18996
|
141 {
|
keir@18996
|
142 for ( j = 0; j < (1 << a->extent_order); j++ )
|
keir@18996
|
143 set_gpfn_from_mfn(mfn + j, gpfn + j);
|
kaf24@8711
|
144
|
keir@18996
|
145 /* Inform the domain of the new page's machine address. */
|
keir@18996
|
146 if ( unlikely(__copy_to_guest_offset(a->extent_list, i, &mfn, 1)) )
|
keir@18996
|
147 goto out;
|
keir@18996
|
148 }
|
sos22@8726
|
149 }
|
kaf24@8711
|
150 }
|
kaf24@8711
|
151
|
keir@18996
|
152 out:
|
kfraser@12398
|
153 a->nr_done = i;
|
kaf24@8711
|
154 }
|
cl349@9225
|
155
|
kfraser@12398
|
156 int guest_remove_page(struct domain *d, unsigned long gmfn)
|
cl349@9225
|
157 {
|
cl349@9225
|
158 struct page_info *page;
|
keir@20805
|
159 #ifdef CONFIG_X86
|
keir@20726
|
160 p2m_type_t p2mt;
|
keir@20805
|
161 #endif
|
cl349@9225
|
162 unsigned long mfn;
|
cl349@9225
|
163
|
keir@20805
|
164 #ifdef CONFIG_X86
|
keir@21986
|
165 mfn = mfn_x(gfn_to_mfn(p2m_get_hostp2m(d), gmfn, &p2mt));
|
keir@22751
|
166 if ( unlikely(p2m_is_paging(p2mt)) )
|
keir@22751
|
167 {
|
keir@22751
|
168 guest_physmap_remove_page(d, gmfn, mfn, 0);
|
keir@22751
|
169 p2m_mem_paging_drop_page(p2m_get_hostp2m(d), gmfn);
|
keir@22751
|
170 return 1;
|
keir@22751
|
171 }
|
keir@20805
|
172 #else
|
keir@20805
|
173 mfn = gmfn_to_mfn(d, gmfn);
|
keir@20805
|
174 #endif
|
cl349@9225
|
175 if ( unlikely(!mfn_valid(mfn)) )
|
cl349@9225
|
176 {
|
kaf24@12062
|
177 gdprintk(XENLOG_INFO, "Domain %u page number %lx invalid\n",
|
tdeegan@11189
|
178 d->domain_id, gmfn);
|
cl349@9225
|
179 return 0;
|
cl349@9225
|
180 }
|
cl349@9225
|
181
|
cl349@9225
|
182 page = mfn_to_page(mfn);
|
keir@20805
|
183 #ifdef CONFIG_X86
|
keir@20726
|
184 /* If gmfn is shared, just drop the guest reference (which may or may not
|
keir@20726
|
185 * free the page) */
|
keir@20726
|
186 if(p2m_is_shared(p2mt))
|
keir@20726
|
187 {
|
keir@20726
|
188 put_page_and_type(page);
|
keir@20726
|
189 guest_physmap_remove_page(d, gmfn, mfn, 0);
|
keir@20726
|
190 return 1;
|
keir@20726
|
191 }
|
keir@20726
|
192
|
keir@20805
|
193 #endif /* CONFIG_X86 */
|
cl349@9225
|
194 if ( unlikely(!get_page(page, d)) )
|
cl349@9225
|
195 {
|
kaf24@12062
|
196 gdprintk(XENLOG_INFO, "Bad page free for domain %u\n", d->domain_id);
|
cl349@9225
|
197 return 0;
|
cl349@9225
|
198 }
|
cl349@9225
|
199
|
cl349@9225
|
200 if ( test_and_clear_bit(_PGT_pinned, &page->u.inuse.type_info) )
|
cl349@9225
|
201 put_page_and_type(page);
|
cl349@9225
|
202
|
cl349@9225
|
203 if ( test_and_clear_bit(_PGC_allocated, &page->count_info) )
|
cl349@9225
|
204 put_page(page);
|
cl349@9225
|
205
|
keir@17683
|
206 guest_physmap_remove_page(d, gmfn, mfn, 0);
|
cl349@9225
|
207
|
cl349@9225
|
208 put_page(page);
|
cl349@9225
|
209
|
cl349@9225
|
210 return 1;
|
cl349@9225
|
211 }
|
cl349@9225
|
212
|
kfraser@12398
|
213 static void decrease_reservation(struct memop_args *a)
|
kaf24@6506
|
214 {
|
kaf24@10314
|
215 unsigned long i, j;
|
kaf24@10314
|
216 xen_pfn_t gmfn;
|
kaf24@6506
|
217
|
keir@17892
|
218 if ( !guest_handle_subrange_okay(a->extent_list, a->nr_done,
|
keir@17892
|
219 a->nr_extents-1) )
|
kfraser@12398
|
220 return;
|
kaf24@6506
|
221
|
kfraser@12398
|
222 for ( i = a->nr_done; i < a->nr_extents; i++ )
|
kaf24@6506
|
223 {
|
kaf24@6506
|
224 if ( hypercall_preempt_check() )
|
kaf24@6627
|
225 {
|
kfraser@12398
|
226 a->preempted = 1;
|
kfraser@12398
|
227 goto out;
|
kaf24@6627
|
228 }
|
kaf24@6506
|
229
|
kfraser@12398
|
230 if ( unlikely(__copy_from_guest_offset(&gmfn, a->extent_list, i, 1)) )
|
kfraser@12398
|
231 goto out;
|
kaf24@6506
|
232
|
keir@20924
|
233 if ( tb_init_done )
|
keir@20924
|
234 {
|
keir@20924
|
235 struct {
|
keir@20924
|
236 u64 gfn;
|
keir@20924
|
237 int d:16,order:16;
|
keir@20924
|
238 } t;
|
keir@20924
|
239
|
keir@20924
|
240 t.gfn = gmfn;
|
keir@20924
|
241 t.d = a->domain->domain_id;
|
keir@20924
|
242 t.order = a->extent_order;
|
keir@20924
|
243
|
keir@22230
|
244 __trace_var(TRC_MEM_DECREASE_RESERVATION, 0, sizeof(t), &t);
|
keir@20924
|
245 }
|
keir@20924
|
246
|
keir@18993
|
247 /* See if populate-on-demand wants to handle this */
|
keir@18993
|
248 if ( is_hvm_domain(a->domain)
|
keir@18993
|
249 && p2m_pod_decrease_reservation(a->domain, gmfn, a->extent_order) )
|
keir@18993
|
250 continue;
|
keir@18993
|
251
|
kfraser@12398
|
252 for ( j = 0; j < (1 << a->extent_order); j++ )
|
kfraser@12398
|
253 if ( !guest_remove_page(a->domain, gmfn + j) )
|
kfraser@12398
|
254 goto out;
|
kaf24@6506
|
255 }
|
kaf24@6506
|
256
|
kfraser@12398
|
257 out:
|
kfraser@12398
|
258 a->nr_done = i;
|
kaf24@6506
|
259 }
|
kaf24@6506
|
260
|
kfraser@12398
|
261 static long memory_exchange(XEN_GUEST_HANDLE(xen_memory_exchange_t) arg)
|
kfraser@10398
|
262 {
|
kfraser@10398
|
263 struct xen_memory_exchange exch;
|
keir@19170
|
264 PAGE_LIST_HEAD(in_chunk_list);
|
keir@19170
|
265 PAGE_LIST_HEAD(out_chunk_list);
|
kfraser@10398
|
266 unsigned long in_chunk_order, out_chunk_order;
|
kaf24@10465
|
267 xen_pfn_t gpfn, gmfn, mfn;
|
kfraser@10398
|
268 unsigned long i, j, k;
|
keir@21959
|
269 unsigned int memflags = 0;
|
kfraser@10398
|
270 long rc = 0;
|
kfraser@10398
|
271 struct domain *d;
|
kfraser@10398
|
272 struct page_info *page;
|
kfraser@10398
|
273
|
kfraser@10398
|
274 if ( copy_from_guest(&exch, arg, 1) )
|
kfraser@10398
|
275 return -EFAULT;
|
kfraser@10398
|
276
|
kfraser@10398
|
277 /* Various sanity checks. */
|
kfraser@10398
|
278 if ( (exch.nr_exchanged > exch.in.nr_extents) ||
|
kfraser@10398
|
279 /* Input and output domain identifiers match? */
|
kfraser@10398
|
280 (exch.in.domid != exch.out.domid) ||
|
kfraser@10398
|
281 /* Sizes of input and output lists do not overflow a long? */
|
kfraser@10398
|
282 ((~0UL >> exch.in.extent_order) < exch.in.nr_extents) ||
|
kfraser@10398
|
283 ((~0UL >> exch.out.extent_order) < exch.out.nr_extents) ||
|
kfraser@10398
|
284 /* Sizes of input and output lists match? */
|
kfraser@10398
|
285 ((exch.in.nr_extents << exch.in.extent_order) !=
|
kfraser@10398
|
286 (exch.out.nr_extents << exch.out.extent_order)) )
|
kfraser@10398
|
287 {
|
kfraser@10398
|
288 rc = -EINVAL;
|
kfraser@10398
|
289 goto fail_early;
|
kfraser@10398
|
290 }
|
kfraser@10398
|
291
|
kfraser@10398
|
292 /* Only privileged guests can allocate multi-page contiguous extents. */
|
keir@19807
|
293 if ( !multipage_allocation_permitted(current->domain,
|
keir@19807
|
294 exch.in.extent_order) ||
|
keir@19807
|
295 !multipage_allocation_permitted(current->domain,
|
keir@19807
|
296 exch.out.extent_order) )
|
kfraser@10398
|
297 {
|
kfraser@10398
|
298 rc = -EPERM;
|
kfraser@10398
|
299 goto fail_early;
|
kfraser@10398
|
300 }
|
kfraser@10398
|
301
|
kfraser@10398
|
302 if ( exch.in.extent_order <= exch.out.extent_order )
|
kfraser@10398
|
303 {
|
kfraser@10398
|
304 in_chunk_order = exch.out.extent_order - exch.in.extent_order;
|
kfraser@10398
|
305 out_chunk_order = 0;
|
kfraser@10398
|
306 }
|
kfraser@10398
|
307 else
|
kfraser@10398
|
308 {
|
kfraser@10398
|
309 in_chunk_order = 0;
|
kfraser@10398
|
310 out_chunk_order = exch.in.extent_order - exch.out.extent_order;
|
kfraser@10398
|
311 }
|
kfraser@10398
|
312
|
keir@19968
|
313 if ( likely(exch.in.domid == DOMID_SELF) )
|
keir@19968
|
314 {
|
keir@19968
|
315 d = rcu_lock_current_domain();
|
keir@19968
|
316 }
|
keir@19968
|
317 else
|
kfraser@10398
|
318 {
|
keir@19968
|
319 if ( (d = rcu_lock_domain_by_id(exch.in.domid)) == NULL )
|
keir@19968
|
320 goto fail_early;
|
keir@19968
|
321
|
keir@19968
|
322 if ( !IS_PRIV_FOR(current->domain, d) )
|
keir@19968
|
323 {
|
keir@19968
|
324 rcu_unlock_domain(d);
|
keir@19968
|
325 rc = -EPERM;
|
keir@19968
|
326 goto fail_early;
|
keir@19968
|
327 }
|
kfraser@10398
|
328 }
|
kfraser@10398
|
329
|
keir@16586
|
330 memflags |= MEMF_bits(domain_clamp_alloc_bitsize(
|
keir@18021
|
331 d,
|
keir@18021
|
332 XENMEMF_get_address_bits(exch.out.mem_flags) ? :
|
keir@18021
|
333 (BITS_PER_LONG+PAGE_SHIFT)));
|
keir@21959
|
334 memflags |= MEMF_node(XENMEMF_get_node(exch.out.mem_flags));
|
kfraser@11933
|
335
|
kfraser@12398
|
336 for ( i = (exch.nr_exchanged >> in_chunk_order);
|
kfraser@12398
|
337 i < (exch.in.nr_extents >> in_chunk_order);
|
kfraser@12398
|
338 i++ )
|
kfraser@10398
|
339 {
|
kfraser@10398
|
340 if ( hypercall_preempt_check() )
|
kfraser@10398
|
341 {
|
kfraser@12398
|
342 exch.nr_exchanged = i << in_chunk_order;
|
keir@19968
|
343 rcu_unlock_domain(d);
|
kfraser@10398
|
344 if ( copy_field_to_guest(arg, &exch, nr_exchanged) )
|
kfraser@10398
|
345 return -EFAULT;
|
kfraser@10398
|
346 return hypercall_create_continuation(
|
kfraser@10398
|
347 __HYPERVISOR_memory_op, "lh", XENMEM_exchange, arg);
|
kfraser@10398
|
348 }
|
kfraser@10398
|
349
|
kfraser@10398
|
350 /* Steal a chunk's worth of input pages from the domain. */
|
kfraser@10398
|
351 for ( j = 0; j < (1UL << in_chunk_order); j++ )
|
kfraser@10398
|
352 {
|
kfraser@10398
|
353 if ( unlikely(__copy_from_guest_offset(
|
kfraser@10398
|
354 &gmfn, exch.in.extent_start, (i<<in_chunk_order)+j, 1)) )
|
kfraser@10398
|
355 {
|
kfraser@10398
|
356 rc = -EFAULT;
|
kfraser@10398
|
357 goto fail;
|
kfraser@10398
|
358 }
|
kfraser@10398
|
359
|
kfraser@10398
|
360 for ( k = 0; k < (1UL << exch.in.extent_order); k++ )
|
kfraser@10398
|
361 {
|
keir@20805
|
362 #ifdef CONFIG_X86
|
keir@20726
|
363 p2m_type_t p2mt;
|
keir@20726
|
364
|
keir@20726
|
365 /* Shared pages cannot be exchanged */
|
keir@21986
|
366 mfn = mfn_x(gfn_to_mfn_unshare(p2m_get_hostp2m(d), gmfn + k, &p2mt, 0));
|
keir@20726
|
367 if ( p2m_is_shared(p2mt) )
|
keir@20726
|
368 {
|
keir@20726
|
369 rc = -ENOMEM;
|
keir@20726
|
370 goto fail;
|
keir@20726
|
371 }
|
keir@20805
|
372 #else /* !CONFIG_X86 */
|
keir@20805
|
373 mfn = gmfn_to_mfn(d, gmfn + k);
|
keir@20805
|
374 #endif
|
kfraser@10398
|
375 if ( unlikely(!mfn_valid(mfn)) )
|
kfraser@10398
|
376 {
|
kfraser@10398
|
377 rc = -EINVAL;
|
kfraser@10398
|
378 goto fail;
|
kfraser@10398
|
379 }
|
kfraser@10398
|
380
|
kfraser@10398
|
381 page = mfn_to_page(mfn);
|
kfraser@10398
|
382
|
kfraser@10398
|
383 if ( unlikely(steal_page(d, page, MEMF_no_refcount)) )
|
kfraser@10398
|
384 {
|
kfraser@10398
|
385 rc = -EINVAL;
|
kfraser@10398
|
386 goto fail;
|
kfraser@10398
|
387 }
|
kfraser@10398
|
388
|
keir@19170
|
389 page_list_add(page, &in_chunk_list);
|
kfraser@10398
|
390 }
|
kfraser@10398
|
391 }
|
kfraser@10398
|
392
|
kfraser@10398
|
393 /* Allocate a chunk's worth of anonymous output pages. */
|
kfraser@10398
|
394 for ( j = 0; j < (1UL << out_chunk_order); j++ )
|
kfraser@10398
|
395 {
|
keir@17421
|
396 page = alloc_domheap_pages(NULL, exch.out.extent_order, memflags);
|
kfraser@10398
|
397 if ( unlikely(page == NULL) )
|
kfraser@10398
|
398 {
|
kfraser@10398
|
399 rc = -ENOMEM;
|
kfraser@10398
|
400 goto fail;
|
kfraser@10398
|
401 }
|
kfraser@10398
|
402
|
keir@19170
|
403 page_list_add(page, &out_chunk_list);
|
kfraser@10398
|
404 }
|
kfraser@10398
|
405
|
kfraser@10398
|
406 /*
|
kfraser@10398
|
407 * Success! Beyond this point we cannot fail for this chunk.
|
kfraser@10398
|
408 */
|
kfraser@10398
|
409
|
kfraser@10398
|
410 /* Destroy final reference to each input page. */
|
keir@19170
|
411 while ( (page = page_list_remove_head(&in_chunk_list)) )
|
kfraser@10398
|
412 {
|
keir@20726
|
413 unsigned long gfn;
|
keir@20726
|
414
|
kfraser@10398
|
415 if ( !test_and_clear_bit(_PGC_allocated, &page->count_info) )
|
kfraser@10398
|
416 BUG();
|
kfraser@10398
|
417 mfn = page_to_mfn(page);
|
keir@20726
|
418 gfn = mfn_to_gmfn(d, mfn);
|
keir@20726
|
419 /* Pages were unshared above */
|
keir@20726
|
420 BUG_ON(SHARED_M2P(gfn));
|
keir@20726
|
421 guest_physmap_remove_page(d, gfn, mfn, 0);
|
kfraser@10398
|
422 put_page(page);
|
kfraser@10398
|
423 }
|
kfraser@10398
|
424
|
kfraser@10398
|
425 /* Assign each output page to the domain. */
|
kfraser@10398
|
426 j = 0;
|
keir@19170
|
427 while ( (page = page_list_remove_head(&out_chunk_list)) )
|
kfraser@10398
|
428 {
|
kfraser@10398
|
429 if ( assign_pages(d, page, exch.out.extent_order,
|
kfraser@10398
|
430 MEMF_no_refcount) )
|
keir@19968
|
431 {
|
keir@19968
|
432 unsigned long dec_count;
|
keir@19968
|
433 bool_t drop_dom_ref;
|
keir@19968
|
434
|
keir@19968
|
435 /*
|
keir@19968
|
436 * Pages in in_chunk_list is stolen without
|
keir@19968
|
437 * decreasing the tot_pages. If the domain is dying when
|
keir@19968
|
438 * assign pages, we need decrease the count. For those pages
|
keir@19968
|
439 * that has been assigned, it should be covered by
|
keir@19968
|
440 * domain_relinquish_resources().
|
keir@19968
|
441 */
|
keir@19968
|
442 dec_count = (((1UL << exch.in.extent_order) *
|
keir@19968
|
443 (1UL << in_chunk_order)) -
|
keir@19968
|
444 (j * (1UL << exch.out.extent_order)));
|
keir@19968
|
445
|
keir@19968
|
446 spin_lock(&d->page_alloc_lock);
|
keir@19968
|
447 d->tot_pages -= dec_count;
|
keir@19968
|
448 drop_dom_ref = (dec_count && !d->tot_pages);
|
keir@19968
|
449 spin_unlock(&d->page_alloc_lock);
|
keir@19968
|
450
|
keir@19968
|
451 if ( drop_dom_ref )
|
keir@19968
|
452 put_domain(d);
|
keir@19968
|
453
|
keir@19968
|
454 free_domheap_pages(page, exch.out.extent_order);
|
keir@19968
|
455 goto dying;
|
keir@19968
|
456 }
|
kfraser@10398
|
457
|
kfraser@10398
|
458 /* Note that we ignore errors accessing the output extent list. */
|
kfraser@10398
|
459 (void)__copy_from_guest_offset(
|
kfraser@10398
|
460 &gpfn, exch.out.extent_start, (i<<out_chunk_order)+j, 1);
|
kfraser@10398
|
461
|
kfraser@10398
|
462 mfn = page_to_mfn(page);
|
keir@17761
|
463 guest_physmap_add_page(d, gpfn, mfn, exch.out.extent_order);
|
keir@17761
|
464
|
keir@17761
|
465 if ( !paging_mode_translate(d) )
|
kfraser@10398
|
466 {
|
kfraser@10398
|
467 for ( k = 0; k < (1UL << exch.out.extent_order); k++ )
|
kfraser@10398
|
468 set_gpfn_from_mfn(mfn + k, gpfn + k);
|
kfraser@10398
|
469 (void)__copy_to_guest_offset(
|
kfraser@10398
|
470 exch.out.extent_start, (i<<out_chunk_order)+j, &mfn, 1);
|
kfraser@10398
|
471 }
|
kfraser@10398
|
472 j++;
|
kfraser@10398
|
473 }
|
keir@19968
|
474 BUG_ON( !(d->is_dying) && (j != (1UL << out_chunk_order)) );
|
kfraser@10398
|
475 }
|
kfraser@10398
|
476
|
kfraser@12398
|
477 exch.nr_exchanged = exch.in.nr_extents;
|
kfraser@10398
|
478 if ( copy_field_to_guest(arg, &exch, nr_exchanged) )
|
kfraser@10398
|
479 rc = -EFAULT;
|
keir@19968
|
480 rcu_unlock_domain(d);
|
kfraser@10398
|
481 return rc;
|
kfraser@10398
|
482
|
kfraser@10398
|
483 /*
|
kfraser@10398
|
484 * Failed a chunk! Free any partial chunk work. Tell caller how many
|
kfraser@10398
|
485 * chunks succeeded.
|
kfraser@10398
|
486 */
|
kfraser@10398
|
487 fail:
|
kfraser@10398
|
488 /* Reassign any input pages we managed to steal. */
|
keir@19170
|
489 while ( (page = page_list_remove_head(&in_chunk_list)) )
|
kfraser@10398
|
490 if ( assign_pages(d, page, 0, MEMF_no_refcount) )
|
kfraser@10398
|
491 BUG();
|
keir@19968
|
492 dying:
|
keir@19968
|
493 rcu_unlock_domain(d);
|
kfraser@10398
|
494 /* Free any output pages we managed to allocate. */
|
keir@19170
|
495 while ( (page = page_list_remove_head(&out_chunk_list)) )
|
kfraser@10398
|
496 free_domheap_pages(page, exch.out.extent_order);
|
kfraser@10398
|
497
|
kfraser@12398
|
498 exch.nr_exchanged = i << in_chunk_order;
|
kfraser@10398
|
499
|
kfraser@10398
|
500 fail_early:
|
kfraser@10398
|
501 if ( copy_field_to_guest(arg, &exch, nr_exchanged) )
|
kfraser@10398
|
502 rc = -EFAULT;
|
kfraser@10398
|
503 return rc;
|
kfraser@10398
|
504 }
|
kfraser@10398
|
505
|
kaf24@9904
|
506 long do_memory_op(unsigned long cmd, XEN_GUEST_HANDLE(void) arg)
|
kaf24@6506
|
507 {
|
kaf24@6506
|
508 struct domain *d;
|
kfraser@12398
|
509 int rc, op;
|
keir@18021
|
510 unsigned int address_bits;
|
keir@19078
|
511 unsigned long start_extent;
|
kaf24@6506
|
512 struct xen_memory_reservation reservation;
|
kfraser@12398
|
513 struct memop_args args;
|
kaf24@7981
|
514 domid_t domid;
|
kaf24@6506
|
515
|
ack@13297
|
516 op = cmd & MEMOP_CMD_MASK;
|
kaf24@6506
|
517
|
kaf24@6506
|
518 switch ( op )
|
kaf24@6506
|
519 {
|
kaf24@6506
|
520 case XENMEM_increase_reservation:
|
kaf24@6506
|
521 case XENMEM_decrease_reservation:
|
kaf24@8711
|
522 case XENMEM_populate_physmap:
|
ack@13297
|
523 start_extent = cmd >> MEMOP_EXTENT_SHIFT;
|
kfraser@10398
|
524
|
kaf24@9054
|
525 if ( copy_from_guest(&reservation, arg, 1) )
|
kfraser@10398
|
526 return start_extent;
|
kaf24@6506
|
527
|
kaf24@8894
|
528 /* Is size too large for us to encode a continuation? */
|
ack@13297
|
529 if ( reservation.nr_extents > (ULONG_MAX >> MEMOP_EXTENT_SHIFT) )
|
kfraser@10398
|
530 return start_extent;
|
kaf24@8894
|
531
|
keir@21131
|
532 if ( unlikely(start_extent >= reservation.nr_extents) )
|
kfraser@10398
|
533 return start_extent;
|
kaf24@9054
|
534
|
kfraser@12398
|
535 args.extent_list = reservation.extent_start;
|
kfraser@12398
|
536 args.nr_extents = reservation.nr_extents;
|
kfraser@12398
|
537 args.extent_order = reservation.extent_order;
|
kfraser@12398
|
538 args.nr_done = start_extent;
|
kfraser@12398
|
539 args.preempted = 0;
|
kfraser@12398
|
540 args.memflags = 0;
|
kaf24@6506
|
541
|
keir@18021
|
542 address_bits = XENMEMF_get_address_bits(reservation.mem_flags);
|
keir@18021
|
543 if ( (address_bits != 0) &&
|
keir@18021
|
544 (address_bits < (get_order_from_pages(max_page) + PAGE_SHIFT)) )
|
kaf24@6506
|
545 {
|
keir@18021
|
546 if ( address_bits <= PAGE_SHIFT )
|
kfraser@10398
|
547 return start_extent;
|
keir@18021
|
548 args.memflags = MEMF_bits(address_bits);
|
kaf24@6506
|
549 }
|
kaf24@6506
|
550
|
keir@18021
|
551 args.memflags |= MEMF_node(XENMEMF_get_node(reservation.mem_flags));
|
keir@21959
|
552 if ( reservation.mem_flags & XENMEMF_exact_node_request )
|
keir@21765
|
553 args.memflags |= MEMF_exact_node;
|
keir@18021
|
554
|
keir@18996
|
555 if ( op == XENMEM_populate_physmap
|
keir@18996
|
556 && (reservation.mem_flags & XENMEMF_populate_on_demand) )
|
keir@18996
|
557 args.memflags |= MEMF_populate_on_demand;
|
keir@18996
|
558
|
kaf24@6506
|
559 if ( likely(reservation.domid == DOMID_SELF) )
|
keir@17357
|
560 {
|
keir@17357
|
561 d = rcu_lock_current_domain();
|
keir@17357
|
562 }
|
keir@17357
|
563 else
|
keir@17357
|
564 {
|
keir@17357
|
565 if ( (d = rcu_lock_domain_by_id(reservation.domid)) == NULL )
|
keir@16894
|
566 return start_extent;
|
keir@17357
|
567 if ( !IS_PRIV_FOR(current->domain, d) )
|
keir@17357
|
568 {
|
keir@16894
|
569 rcu_unlock_domain(d);
|
keir@16894
|
570 return start_extent;
|
keir@16894
|
571 }
|
keir@16894
|
572 }
|
kfraser@12398
|
573 args.domain = d;
|
kaf24@6506
|
574
|
kfraser@15846
|
575 rc = xsm_memory_adjust_reservation(current->domain, d);
|
kfraser@15846
|
576 if ( rc )
|
kfraser@15846
|
577 {
|
keir@17357
|
578 rcu_unlock_domain(d);
|
kfraser@15846
|
579 return rc;
|
kfraser@15846
|
580 }
|
kfraser@15846
|
581
|
kaf24@8711
|
582 switch ( op )
|
kaf24@8711
|
583 {
|
kaf24@8711
|
584 case XENMEM_increase_reservation:
|
kfraser@12398
|
585 increase_reservation(&args);
|
kaf24@8711
|
586 break;
|
kaf24@8711
|
587 case XENMEM_decrease_reservation:
|
kfraser@12398
|
588 decrease_reservation(&args);
|
kaf24@8711
|
589 break;
|
kfraser@12398
|
590 default: /* XENMEM_populate_physmap */
|
kfraser@12398
|
591 populate_physmap(&args);
|
kaf24@8711
|
592 break;
|
kaf24@8711
|
593 }
|
kaf24@6506
|
594
|
keir@17357
|
595 rcu_unlock_domain(d);
|
kaf24@6506
|
596
|
kfraser@12398
|
597 rc = args.nr_done;
|
kaf24@6506
|
598
|
kfraser@12398
|
599 if ( args.preempted )
|
kaf24@9054
|
600 return hypercall_create_continuation(
|
kaf24@9054
|
601 __HYPERVISOR_memory_op, "lh",
|
ack@13297
|
602 op | (rc << MEMOP_EXTENT_SHIFT), arg);
|
kaf24@6627
|
603
|
kaf24@6506
|
604 break;
|
kaf24@6506
|
605
|
kfraser@10398
|
606 case XENMEM_exchange:
|
kfraser@10398
|
607 rc = memory_exchange(guest_handle_cast(arg, xen_memory_exchange_t));
|
kfraser@10398
|
608 break;
|
kfraser@10398
|
609
|
kaf24@6506
|
610 case XENMEM_maximum_ram_page:
|
kaf24@7981
|
611 rc = max_page;
|
kaf24@7981
|
612 break;
|
kaf24@7981
|
613
|
kaf24@7981
|
614 case XENMEM_current_reservation:
|
kaf24@7981
|
615 case XENMEM_maximum_reservation:
|
kfraser@14478
|
616 case XENMEM_maximum_gpfn:
|
kaf24@9054
|
617 if ( copy_from_guest(&domid, arg, 1) )
|
kaf24@6506
|
618 return -EFAULT;
|
kaf24@7981
|
619
|
keir@18604
|
620 rc = rcu_lock_target_domain_by_id(domid, &d);
|
keir@18604
|
621 if ( rc )
|
keir@18604
|
622 return rc;
|
kaf24@7981
|
623
|
kfraser@15846
|
624 rc = xsm_memory_stat_reservation(current->domain, d);
|
kfraser@15846
|
625 if ( rc )
|
kfraser@15846
|
626 {
|
keir@17357
|
627 rcu_unlock_domain(d);
|
kfraser@15846
|
628 return rc;
|
kfraser@15846
|
629 }
|
kfraser@15846
|
630
|
kfraser@14478
|
631 switch ( op )
|
kfraser@14478
|
632 {
|
kfraser@14478
|
633 case XENMEM_current_reservation:
|
kfraser@14478
|
634 rc = d->tot_pages;
|
kfraser@14478
|
635 break;
|
kfraser@14478
|
636 case XENMEM_maximum_reservation:
|
kfraser@14478
|
637 rc = d->max_pages;
|
kfraser@14478
|
638 break;
|
kfraser@14478
|
639 default:
|
kfraser@14478
|
640 ASSERT(op == XENMEM_maximum_gpfn);
|
kfraser@14478
|
641 rc = domain_get_maximum_gpfn(d);
|
kfraser@14478
|
642 break;
|
kfraser@14478
|
643 }
|
kaf24@7981
|
644
|
keir@17357
|
645 rcu_unlock_domain(d);
|
kaf24@7981
|
646
|
kaf24@6506
|
647 break;
|
kaf24@6506
|
648
|
kaf24@6506
|
649 default:
|
kaf24@8081
|
650 rc = arch_memory_op(op, arg);
|
kaf24@6506
|
651 break;
|
kaf24@6506
|
652 }
|
kaf24@6506
|
653
|
kaf24@6506
|
654 return rc;
|
kaf24@6506
|
655 }
|
kaf24@6506
|
656
|
kaf24@6506
|
657 /*
|
kaf24@6506
|
658 * Local variables:
|
kaf24@6506
|
659 * mode: C
|
kaf24@6506
|
660 * c-set-style: "BSD"
|
kaf24@6506
|
661 * c-basic-offset: 4
|
kaf24@6506
|
662 * tab-width: 4
|
kaf24@6506
|
663 * indent-tabs-mode: nil
|
kaf24@6506
|
664 * End:
|
kaf24@6506
|
665 */
|