debuggers.hg

view xen/common/tmem_xen.c @ 22829:051a1b1b8f8a

Disable tmem by default for 4.1 release.

Although one major source of order>0 allocations has been removed,
others still remain, so re-disable tmem until the issue can be fixed
properly.

Signed-off-by: Tim Deegan <Tim.Deegan@citrix.com>
author Keir Fraser <keir@xen.org>
date Wed Jan 19 18:24:26 2011 +0000 (2011-01-19)
parents e8acb9753ff1
children
line source
1 /******************************************************************************
2 * tmem-xen.c
3 *
4 * Xen-specific Transcendent memory
5 *
6 * Copyright (c) 2009, Dan Magenheimer, Oracle Corp.
7 */
9 #include <xen/tmem.h>
10 #include <xen/tmem_xen.h>
11 #include <xen/lzo.h> /* compression code */
12 #include <xen/paging.h>
13 #include <xen/domain_page.h>
14 #include <xen/cpu.h>
16 #define EXPORT /* indicates code other modules are dependent upon */
18 EXPORT bool_t __read_mostly opt_tmem = 0;
19 boolean_param("tmem", opt_tmem);
21 EXPORT bool_t __read_mostly opt_tmem_compress = 0;
22 boolean_param("tmem_compress", opt_tmem_compress);
24 EXPORT bool_t __read_mostly opt_tmem_dedup = 0;
25 boolean_param("tmem_dedup", opt_tmem_dedup);
27 EXPORT bool_t __read_mostly opt_tmem_tze = 0;
28 boolean_param("tmem_tze", opt_tmem_tze);
30 EXPORT bool_t __read_mostly opt_tmem_shared_auth = 0;
31 boolean_param("tmem_shared_auth", opt_tmem_shared_auth);
33 EXPORT int __read_mostly opt_tmem_lock = 0;
34 integer_param("tmem_lock", opt_tmem_lock);
36 EXPORT atomic_t freeable_page_count = ATOMIC_INIT(0);
38 #ifdef COMPARE_COPY_PAGE_SSE2
39 DECL_CYC_COUNTER(pg_copy1);
40 DECL_CYC_COUNTER(pg_copy2);
41 DECL_CYC_COUNTER(pg_copy3);
42 DECL_CYC_COUNTER(pg_copy4);
43 #else
44 DECL_CYC_COUNTER(pg_copy);
45 #endif
47 /* these are a concurrency bottleneck, could be percpu and dynamically
48 * allocated iff opt_tmem_compress */
49 #define LZO_WORKMEM_BYTES LZO1X_1_MEM_COMPRESS
50 #define LZO_DSTMEM_PAGES 2
51 static DEFINE_PER_CPU_READ_MOSTLY(unsigned char *, workmem);
52 static DEFINE_PER_CPU_READ_MOSTLY(unsigned char *, dstmem);
54 #ifdef COMPARE_COPY_PAGE_SSE2
55 #include <asm/flushtlb.h> /* REMOVE ME AFTER TEST */
56 #include <asm/page.h> /* REMOVE ME AFTER TEST */
57 #endif
58 void tmh_copy_page(char *to, char*from)
59 {
60 #ifdef COMPARE_COPY_PAGE_SSE2
61 DECL_LOCAL_CYC_COUNTER(pg_copy1);
62 DECL_LOCAL_CYC_COUNTER(pg_copy2);
63 DECL_LOCAL_CYC_COUNTER(pg_copy3);
64 DECL_LOCAL_CYC_COUNTER(pg_copy4);
65 *to = *from; /* don't measure TLB misses */
66 flush_area_local(to,FLUSH_CACHE|FLUSH_ORDER(0));
67 flush_area_local(from,FLUSH_CACHE|FLUSH_ORDER(0));
68 START_CYC_COUNTER(pg_copy1);
69 copy_page_sse2(to, from); /* cold cache */
70 END_CYC_COUNTER(pg_copy1);
71 START_CYC_COUNTER(pg_copy2);
72 copy_page_sse2(to, from); /* hot cache */
73 END_CYC_COUNTER(pg_copy2);
74 flush_area_local(to,FLUSH_CACHE|FLUSH_ORDER(0));
75 flush_area_local(from,FLUSH_CACHE|FLUSH_ORDER(0));
76 START_CYC_COUNTER(pg_copy3);
77 memcpy(to, from, PAGE_SIZE); /* cold cache */
78 END_CYC_COUNTER(pg_copy3);
79 START_CYC_COUNTER(pg_copy4);
80 memcpy(to, from, PAGE_SIZE); /* hot cache */
81 END_CYC_COUNTER(pg_copy4);
82 #else
83 DECL_LOCAL_CYC_COUNTER(pg_copy);
84 START_CYC_COUNTER(pg_copy);
85 memcpy(to, from, PAGE_SIZE);
86 END_CYC_COUNTER(pg_copy);
87 #endif
88 }
90 #ifdef __ia64__
91 static inline void *cli_get_page(tmem_cli_mfn_t cmfn, unsigned long *pcli_mfn,
92 pfp_t **pcli_pfp, bool_t cli_write)
93 {
94 ASSERT(0);
95 return NULL;
96 }
98 static inline void cli_put_page(void *cli_va, pfp_t *cli_pfp,
99 unsigned long cli_mfn, bool_t mark_dirty)
100 {
101 ASSERT(0);
102 }
103 #else
104 static inline void *cli_get_page(tmem_cli_mfn_t cmfn, unsigned long *pcli_mfn,
105 pfp_t **pcli_pfp, bool_t cli_write)
106 {
107 unsigned long cli_mfn;
108 p2m_type_t t;
109 struct page_info *page;
110 int ret;
112 cli_mfn = mfn_x(gfn_to_mfn(p2m_get_hostp2m(current->domain), cmfn, &t));
113 if ( t != p2m_ram_rw || !mfn_valid(cli_mfn) )
114 return NULL;
115 page = mfn_to_page(cli_mfn);
116 if ( cli_write )
117 ret = get_page_and_type(page, current->domain, PGT_writable_page);
118 else
119 ret = get_page(page, current->domain);
120 if ( !ret )
121 return NULL;
122 *pcli_mfn = cli_mfn;
123 *pcli_pfp = (pfp_t *)page;
124 return map_domain_page(cli_mfn);
125 }
127 static inline void cli_put_page(void *cli_va, pfp_t *cli_pfp,
128 unsigned long cli_mfn, bool_t mark_dirty)
129 {
130 if ( mark_dirty )
131 {
132 put_page_and_type((struct page_info *)cli_pfp);
133 paging_mark_dirty(current->domain,cli_mfn);
134 }
135 else
136 put_page((struct page_info *)cli_pfp);
137 unmap_domain_page(cli_va);
138 }
139 #endif
141 EXPORT int tmh_copy_from_client(pfp_t *pfp,
142 tmem_cli_mfn_t cmfn, pagesize_t tmem_offset,
143 pagesize_t pfn_offset, pagesize_t len, void *cli_va)
144 {
145 unsigned long tmem_mfn, cli_mfn = 0;
146 void *tmem_va;
147 pfp_t *cli_pfp = NULL;
148 bool_t tmemc = cli_va != NULL; /* if true, cli_va is control-op buffer */
150 ASSERT(pfp != NULL);
151 tmem_mfn = page_to_mfn(pfp);
152 tmem_va = map_domain_page(tmem_mfn);
153 if ( tmem_offset == 0 && pfn_offset == 0 && len == 0 )
154 {
155 memset(tmem_va, 0, PAGE_SIZE);
156 unmap_domain_page(tmem_va);
157 return 1;
158 }
159 if ( !tmemc )
160 {
161 cli_va = cli_get_page(cmfn, &cli_mfn, &cli_pfp, 0);
162 if ( cli_va == NULL )
163 return -EFAULT;
164 }
165 mb();
166 if (len == PAGE_SIZE && !tmem_offset && !pfn_offset)
167 tmh_copy_page(tmem_va, cli_va);
168 else if ( (tmem_offset+len <= PAGE_SIZE) &&
169 (pfn_offset+len <= PAGE_SIZE) )
170 memcpy((char *)tmem_va+tmem_offset,(char *)cli_va+pfn_offset,len);
171 if ( !tmemc )
172 cli_put_page(cli_va, cli_pfp, cli_mfn, 0);
173 unmap_domain_page(tmem_va);
174 return 1;
175 }
177 EXPORT int tmh_compress_from_client(tmem_cli_mfn_t cmfn,
178 void **out_va, size_t *out_len, void *cli_va)
179 {
180 int ret = 0;
181 unsigned char *dmem = this_cpu(dstmem);
182 unsigned char *wmem = this_cpu(workmem);
183 pfp_t *cli_pfp = NULL;
184 unsigned long cli_mfn = 0;
185 bool_t tmemc = cli_va != NULL; /* if true, cli_va is control-op buffer */
187 if ( dmem == NULL || wmem == NULL )
188 return 0; /* no buffer, so can't compress */
189 if ( !tmemc )
190 {
191 cli_va = cli_get_page(cmfn, &cli_mfn, &cli_pfp, 0);
192 if ( cli_va == NULL )
193 return -EFAULT;
194 }
195 mb();
196 ret = lzo1x_1_compress(cli_va, PAGE_SIZE, dmem, out_len, wmem);
197 ASSERT(ret == LZO_E_OK);
198 *out_va = dmem;
199 if ( !tmemc )
200 cli_put_page(cli_va, cli_pfp, cli_mfn, 0);
201 unmap_domain_page(cli_va);
202 return 1;
203 }
205 EXPORT int tmh_copy_to_client(tmem_cli_mfn_t cmfn, pfp_t *pfp,
206 pagesize_t tmem_offset, pagesize_t pfn_offset, pagesize_t len, void *cli_va)
207 {
208 unsigned long tmem_mfn, cli_mfn = 0;
209 void *tmem_va;
210 pfp_t *cli_pfp = NULL;
211 bool_t tmemc = cli_va != NULL; /* if true, cli_va is control-op buffer */
213 ASSERT(pfp != NULL);
214 if ( !tmemc )
215 {
216 cli_va = cli_get_page(cmfn, &cli_mfn, &cli_pfp, 1);
217 if ( cli_va == NULL )
218 return -EFAULT;
219 }
220 tmem_mfn = page_to_mfn(pfp);
221 tmem_va = map_domain_page(tmem_mfn);
222 if (len == PAGE_SIZE && !tmem_offset && !pfn_offset)
223 tmh_copy_page(cli_va, tmem_va);
224 else if ( (tmem_offset+len <= PAGE_SIZE) && (pfn_offset+len <= PAGE_SIZE) )
225 memcpy((char *)cli_va+pfn_offset,(char *)tmem_va+tmem_offset,len);
226 unmap_domain_page(tmem_va);
227 if ( !tmemc )
228 cli_put_page(cli_va, cli_pfp, cli_mfn, 1);
229 mb();
230 return 1;
231 }
233 EXPORT int tmh_decompress_to_client(tmem_cli_mfn_t cmfn, void *tmem_va,
234 size_t size, void *cli_va)
235 {
236 unsigned long cli_mfn = 0;
237 pfp_t *cli_pfp = NULL;
238 size_t out_len = PAGE_SIZE;
239 bool_t tmemc = cli_va != NULL; /* if true, cli_va is control-op buffer */
240 int ret;
242 if ( !tmemc )
243 {
244 cli_va = cli_get_page(cmfn, &cli_mfn, &cli_pfp, 1);
245 if ( cli_va == NULL )
246 return -EFAULT;
247 }
248 ret = lzo1x_decompress_safe(tmem_va, size, cli_va, &out_len);
249 ASSERT(ret == LZO_E_OK);
250 ASSERT(out_len == PAGE_SIZE);
251 if ( !tmemc )
252 cli_put_page(cli_va, cli_pfp, cli_mfn, 1);
253 mb();
254 return 1;
255 }
257 EXPORT int tmh_copy_tze_to_client(tmem_cli_mfn_t cmfn, void *tmem_va,
258 pagesize_t len)
259 {
260 void *cli_va;
261 unsigned long cli_mfn;
262 pfp_t *cli_pfp = NULL;
264 ASSERT(!(len & (sizeof(uint64_t)-1)));
265 ASSERT(len <= PAGE_SIZE);
266 ASSERT(len > 0 || tmem_va == NULL);
267 cli_va = cli_get_page(cmfn, &cli_mfn, &cli_pfp, 1);
268 if ( cli_va == NULL )
269 return -EFAULT;
270 if ( len > 0 )
271 memcpy((char *)cli_va,(char *)tmem_va,len);
272 if ( len < PAGE_SIZE )
273 memset((char *)cli_va+len,0,PAGE_SIZE-len);
274 cli_put_page(cli_va, cli_pfp, cli_mfn, 1);
275 mb();
276 return 1;
277 }
279 /****************** XEN-SPECIFIC MEMORY ALLOCATION ********************/
281 EXPORT struct xmem_pool *tmh_mempool = 0;
282 EXPORT unsigned int tmh_mempool_maxalloc = 0;
284 EXPORT DEFINE_SPINLOCK(tmh_page_list_lock);
285 EXPORT PAGE_LIST_HEAD(tmh_page_list);
286 EXPORT unsigned long tmh_page_list_pages = 0;
288 /* free anything on tmh_page_list to Xen's scrub list */
289 EXPORT void tmh_release_avail_pages_to_host(void)
290 {
291 spin_lock(&tmh_page_list_lock);
292 while ( !page_list_empty(&tmh_page_list) )
293 {
294 struct page_info *pg = page_list_remove_head(&tmh_page_list);
295 scrub_one_page(pg);
296 tmh_page_list_pages--;
297 free_domheap_page(pg);
298 }
299 ASSERT(tmh_page_list_pages == 0);
300 INIT_PAGE_LIST_HEAD(&tmh_page_list);
301 spin_unlock(&tmh_page_list_lock);
302 }
304 EXPORT void tmh_scrub_page(struct page_info *pi, unsigned int memflags)
305 {
306 if ( pi == NULL )
307 return;
308 if ( !(memflags & MEMF_tmem) )
309 scrub_one_page(pi);
310 }
312 #ifndef __i386__
313 static noinline void *tmh_mempool_page_get(unsigned long size)
314 {
315 struct page_info *pi;
317 ASSERT(size == PAGE_SIZE);
318 if ( (pi = tmh_alloc_page(NULL,0)) == NULL )
319 return NULL;
320 ASSERT(IS_VALID_PAGE(pi));
321 return page_to_virt(pi);
322 }
324 static void tmh_mempool_page_put(void *page_va)
325 {
326 ASSERT(IS_PAGE_ALIGNED(page_va));
327 tmh_free_page(virt_to_page(page_va));
328 }
330 static int __init tmh_mempool_init(void)
331 {
332 tmh_mempool = xmem_pool_create("tmem", tmh_mempool_page_get,
333 tmh_mempool_page_put, PAGE_SIZE, 0, PAGE_SIZE);
334 if ( tmh_mempool )
335 tmh_mempool_maxalloc = xmem_pool_maxalloc(tmh_mempool);
336 return tmh_mempool != NULL;
337 }
339 /* persistent pools are per-domain */
341 static void *tmh_persistent_pool_page_get(unsigned long size)
342 {
343 struct page_info *pi;
344 struct domain *d = current->domain;
346 ASSERT(size == PAGE_SIZE);
347 if ( (pi = _tmh_alloc_page_thispool(d)) == NULL )
348 return NULL;
349 ASSERT(IS_VALID_PAGE(pi));
350 return __map_domain_page(pi);
351 }
353 static void tmh_persistent_pool_page_put(void *page_va)
354 {
355 struct page_info *pi;
357 ASSERT(IS_PAGE_ALIGNED(page_va));
358 pi = virt_to_page(page_va);
359 ASSERT(IS_VALID_PAGE(pi));
360 _tmh_free_page_thispool(pi);
361 }
362 #endif
364 /****************** XEN-SPECIFIC CLIENT HANDLING ********************/
366 EXPORT tmh_client_t *tmh_client_init(cli_id_t cli_id)
367 {
368 tmh_client_t *tmh;
369 char name[5];
370 int i, shift;
372 if ( (tmh = xmalloc(tmh_client_t)) == NULL )
373 return NULL;
374 for (i = 0, shift = 12; i < 4; shift -=4, i++)
375 name[i] = (((unsigned short)cli_id >> shift) & 0xf) + '0';
376 name[4] = '\0';
377 #ifndef __i386__
378 tmh->persistent_pool = xmem_pool_create(name, tmh_persistent_pool_page_get,
379 tmh_persistent_pool_page_put, PAGE_SIZE, 0, PAGE_SIZE);
380 if ( tmh->persistent_pool == NULL )
381 {
382 xfree(tmh);
383 return NULL;
384 }
385 #endif
386 return tmh;
387 }
389 EXPORT void tmh_client_destroy(tmh_client_t *tmh)
390 {
391 ASSERT(tmh->domain->is_dying);
392 #ifndef __i386__
393 xmem_pool_destroy(tmh->persistent_pool);
394 #endif
395 tmh->domain = NULL;
396 }
398 /****************** XEN-SPECIFIC HOST INITIALIZATION ********************/
400 #ifndef __i386__
402 static int dstmem_order, workmem_order;
404 static int cpu_callback(
405 struct notifier_block *nfb, unsigned long action, void *hcpu)
406 {
407 unsigned int cpu = (unsigned long)hcpu;
409 switch ( action )
410 {
411 case CPU_UP_PREPARE: {
412 if ( per_cpu(dstmem, cpu) == NULL )
413 {
414 struct page_info *p = alloc_domheap_pages(0, dstmem_order, 0);
415 per_cpu(dstmem, cpu) = p ? page_to_virt(p) : NULL;
416 }
417 if ( per_cpu(workmem, cpu) == NULL )
418 {
419 struct page_info *p = alloc_domheap_pages(0, workmem_order, 0);
420 per_cpu(workmem, cpu) = p ? page_to_virt(p) : NULL;
421 }
422 break;
423 }
424 case CPU_DEAD:
425 case CPU_UP_CANCELED: {
426 if ( per_cpu(dstmem, cpu) != NULL )
427 {
428 struct page_info *p = virt_to_page(per_cpu(dstmem, cpu));
429 free_domheap_pages(p, dstmem_order);
430 per_cpu(dstmem, cpu) = NULL;
431 }
432 if ( per_cpu(workmem, cpu) != NULL )
433 {
434 struct page_info *p = virt_to_page(per_cpu(workmem, cpu));
435 free_domheap_pages(p, workmem_order);
436 per_cpu(workmem, cpu) = NULL;
437 }
438 break;
439 }
440 default:
441 break;
442 }
444 return NOTIFY_DONE;
445 }
447 static struct notifier_block cpu_nfb = {
448 .notifier_call = cpu_callback
449 };
451 EXPORT int __init tmh_init(void)
452 {
453 unsigned int cpu;
455 if ( !tmh_mempool_init() )
456 return 0;
458 dstmem_order = get_order_from_pages(LZO_DSTMEM_PAGES);
459 workmem_order = get_order_from_bytes(LZO1X_1_MEM_COMPRESS);
461 for_each_online_cpu ( cpu )
462 {
463 void *hcpu = (void *)(long)cpu;
464 cpu_callback(&cpu_nfb, CPU_UP_PREPARE, hcpu);
465 }
467 register_cpu_notifier(&cpu_nfb);
469 return 1;
470 }
472 #else
474 EXPORT int __init tmh_init(void)
475 {
476 return 1;
477 }
479 #endif