debuggers.hg

view xen/include/xen/tmem_xen.h @ 22906:700ac6445812

Now add KDB to the non-kdb tree
author Mukesh Rathor
date Thu Feb 03 15:42:41 2011 -0800 (2011-02-03)
parents e8acb9753ff1
children
line source
1 /******************************************************************************
2 * tmem_xen.h
3 *
4 * Xen-specific Transcendent memory
5 *
6 * Copyright (c) 2009, Dan Magenheimer, Oracle Corp.
7 */
9 #ifndef __XEN_TMEM_XEN_H__
10 #define __XEN_TMEM_XEN_H__
12 #include <xen/config.h>
13 #include <xen/mm.h> /* heap alloc/free */
14 #include <xen/xmalloc.h> /* xmalloc/xfree */
15 #include <xen/sched.h> /* struct domain */
16 #include <xen/guest_access.h> /* copy_from_guest */
17 #include <xen/hash.h> /* hash_long */
18 #include <xen/domain_page.h> /* __map_domain_page */
19 #include <public/tmem.h>
20 #ifdef CONFIG_COMPAT
21 #include <compat/tmem.h>
22 #endif
24 struct tmem_host_dependent_client {
25 struct domain *domain;
26 struct xmem_pool *persistent_pool;
27 };
28 typedef struct tmem_host_dependent_client tmh_client_t;
30 typedef uint32_t pagesize_t; /* like size_t, must handle largest PAGE_SIZE */
32 #define IS_PAGE_ALIGNED(addr) \
33 ((void *)((((unsigned long)addr + (PAGE_SIZE - 1)) & PAGE_MASK)) == addr)
34 #define IS_VALID_PAGE(_pi) ( mfn_valid(page_to_mfn(_pi)) )
36 extern struct xmem_pool *tmh_mempool;
37 extern unsigned int tmh_mempool_maxalloc;
38 extern struct page_list_head tmh_page_list;
39 extern spinlock_t tmh_page_list_lock;
40 extern unsigned long tmh_page_list_pages;
41 extern atomic_t freeable_page_count;
43 extern spinlock_t tmem_lock;
44 extern spinlock_t tmem_spinlock;
45 extern rwlock_t tmem_rwlock;
47 extern void tmh_copy_page(char *to, char*from);
48 extern int tmh_init(void);
49 #define tmh_hash hash_long
51 extern void tmh_release_avail_pages_to_host(void);
52 extern void tmh_scrub_page(struct page_info *pi, unsigned int memflags);
54 extern bool_t opt_tmem_compress;
55 static inline bool_t tmh_compression_enabled(void)
56 {
57 return opt_tmem_compress;
58 }
60 extern bool_t opt_tmem_dedup;
61 static inline bool_t tmh_dedup_enabled(void)
62 {
63 return opt_tmem_dedup;
64 }
66 extern bool_t opt_tmem_tze;
67 static inline bool_t tmh_tze_enabled(void)
68 {
69 return opt_tmem_tze;
70 }
72 static inline void tmh_tze_disable(void)
73 {
74 opt_tmem_tze = 0;
75 }
77 extern bool_t opt_tmem_shared_auth;
78 static inline bool_t tmh_shared_auth(void)
79 {
80 return opt_tmem_shared_auth;
81 }
83 extern bool_t opt_tmem;
84 static inline bool_t tmh_enabled(void)
85 {
86 return opt_tmem;
87 }
89 extern int opt_tmem_lock;
91 /*
92 * Memory free page list management
93 */
95 static inline struct page_info *tmh_page_list_get(void)
96 {
97 struct page_info *pi;
99 spin_lock(&tmh_page_list_lock);
100 if ( (pi = page_list_remove_head(&tmh_page_list)) != NULL )
101 tmh_page_list_pages--;
102 spin_unlock(&tmh_page_list_lock);
103 ASSERT((pi == NULL) || IS_VALID_PAGE(pi));
104 return pi;
105 }
107 static inline void tmh_page_list_put(struct page_info *pi)
108 {
109 ASSERT(IS_VALID_PAGE(pi));
110 spin_lock(&tmh_page_list_lock);
111 page_list_add(pi, &tmh_page_list);
112 tmh_page_list_pages++;
113 spin_unlock(&tmh_page_list_lock);
114 }
116 static inline unsigned long tmh_avail_pages(void)
117 {
118 return tmh_page_list_pages;
119 }
121 /*
122 * Memory allocation for persistent data
123 */
125 static inline bool_t domain_fully_allocated(struct domain *d)
126 {
127 return ( d->tot_pages >= d->max_pages );
128 }
129 #define tmh_client_memory_fully_allocated(_pool) \
130 domain_fully_allocated(_pool->client->tmh->domain)
132 static inline void *_tmh_alloc_subpage_thispool(struct xmem_pool *cmem_mempool,
133 size_t size, size_t align)
134 {
135 #if 0
136 if ( d->tot_pages >= d->max_pages )
137 return NULL;
138 #endif
139 #ifdef __i386__
140 return _xmalloc(size,align);
141 #else
142 ASSERT( size < tmh_mempool_maxalloc );
143 if ( cmem_mempool == NULL )
144 return NULL;
145 return xmem_pool_alloc(size, cmem_mempool);
146 #endif
147 }
148 #define tmh_alloc_subpage_thispool(_pool, _s, _a) \
149 _tmh_alloc_subpage_thispool(pool->client->tmh->persistent_pool, \
150 _s, _a)
152 static inline void _tmh_free_subpage_thispool(struct xmem_pool *cmem_mempool,
153 void *ptr, size_t size)
154 {
155 #ifdef __i386__
156 xfree(ptr);
157 #else
158 ASSERT( size < tmh_mempool_maxalloc );
159 ASSERT( cmem_mempool != NULL );
160 xmem_pool_free(ptr,cmem_mempool);
161 #endif
162 }
163 #define tmh_free_subpage_thispool(_pool, _p, _s) \
164 _tmh_free_subpage_thispool(_pool->client->tmh->persistent_pool, _p, _s)
166 static inline struct page_info *_tmh_alloc_page_thispool(struct domain *d)
167 {
168 struct page_info *pi;
170 /* note that this tot_pages check is not protected by d->page_alloc_lock,
171 * so may race and periodically fail in donate_page or alloc_domheap_pages
172 * That's OK... neither is a problem, though chatty if log_lvl is set */
173 if ( d->tot_pages >= d->max_pages )
174 return NULL;
176 if ( tmh_page_list_pages )
177 {
178 if ( (pi = tmh_page_list_get()) != NULL )
179 {
180 if ( donate_page(d,pi,0) == 0 )
181 goto out;
182 else
183 tmh_page_list_put(pi);
184 }
185 }
187 pi = alloc_domheap_pages(d,0,MEMF_tmem);
189 out:
190 ASSERT((pi == NULL) || IS_VALID_PAGE(pi));
191 return pi;
192 }
193 #define tmh_alloc_page_thispool(_pool) \
194 _tmh_alloc_page_thispool(_pool->client->tmh->domain)
196 static inline void _tmh_free_page_thispool(struct page_info *pi)
197 {
198 struct domain *d = page_get_owner(pi);
200 ASSERT(IS_VALID_PAGE(pi));
201 if ( (d == NULL) || steal_page(d,pi,0) == 0 )
202 tmh_page_list_put(pi);
203 else
204 {
205 scrub_one_page(pi);
206 ASSERT((pi->count_info & ~(PGC_allocated | 1)) == 0);
207 free_domheap_pages(pi,0);
208 }
209 }
210 #define tmh_free_page_thispool(_pool,_pg) \
211 _tmh_free_page_thispool(_pg)
213 /*
214 * Memory allocation for ephemeral (non-persistent) data
215 */
217 static inline void *tmh_alloc_subpage(void *pool, size_t size,
218 size_t align)
219 {
220 #ifdef __i386__
221 ASSERT( size < PAGE_SIZE );
222 return _xmalloc(size, align);
223 #else
224 ASSERT( size < tmh_mempool_maxalloc );
225 ASSERT( tmh_mempool != NULL );
226 return xmem_pool_alloc(size, tmh_mempool);
227 #endif
228 }
230 static inline void tmh_free_subpage(void *ptr, size_t size)
231 {
232 #ifdef __i386__
233 ASSERT( size < PAGE_SIZE );
234 xfree(ptr);
235 #else
236 ASSERT( size < tmh_mempool_maxalloc );
237 xmem_pool_free(ptr,tmh_mempool);
238 #endif
239 }
241 static inline struct page_info *tmh_alloc_page(void *pool, int no_heap)
242 {
243 struct page_info *pi = tmh_page_list_get();
245 if ( pi == NULL && !no_heap )
246 pi = alloc_domheap_pages(0,0,MEMF_tmem);
247 ASSERT((pi == NULL) || IS_VALID_PAGE(pi));
248 if ( pi != NULL && !no_heap )
249 atomic_inc(&freeable_page_count);
250 return pi;
251 }
253 static inline void tmh_free_page(struct page_info *pi)
254 {
255 ASSERT(IS_VALID_PAGE(pi));
256 tmh_page_list_put(pi);
257 atomic_dec(&freeable_page_count);
258 }
260 static inline unsigned int tmem_subpage_maxsize(void)
261 {
262 return tmh_mempool_maxalloc;
263 }
265 static inline unsigned long tmh_freeable_pages(void)
266 {
267 return tmh_avail_pages() + _atomic_read(freeable_page_count);
268 }
270 static inline unsigned long tmh_free_mb(void)
271 {
272 return (tmh_avail_pages() + total_free_pages()) >> (20 - PAGE_SHIFT);
273 }
275 /*
276 * Memory allocation for "infrastructure" data
277 */
279 static inline void *tmh_alloc_infra(size_t size, size_t align)
280 {
281 return _xmalloc(size,align);
282 }
284 static inline void tmh_free_infra(void *p)
285 {
286 return xfree(p);
287 }
289 #define tmh_lock_all opt_tmem_lock
290 #define tmh_called_from_tmem(_memflags) (_memflags & MEMF_tmem)
292 /* "Client" (==domain) abstraction */
294 struct client;
295 typedef domid_t cli_id_t;
296 typedef struct domain tmh_cli_ptr_t;
297 typedef struct page_info pfp_t;
299 extern tmh_client_t *tmh_client_init(cli_id_t);
300 extern void tmh_client_destroy(tmh_client_t *);
302 static inline struct client *tmh_client_from_cli_id(cli_id_t cli_id)
303 {
304 struct client *c;
305 struct domain *d = rcu_lock_domain_by_id(cli_id);
306 if (d == NULL)
307 return NULL;
308 c = (struct client *)(d->tmem);
309 rcu_unlock_domain(d);
310 return c;
311 }
313 static inline struct client *tmh_client_from_current(void)
314 {
315 return (struct client *)(current->domain->tmem);
316 }
318 #define tmh_client_is_dying(_client) (!!_client->tmh->domain->is_dying)
320 static inline cli_id_t tmh_get_cli_id_from_current(void)
321 {
322 return current->domain->domain_id;
323 }
325 static inline tmh_cli_ptr_t *tmh_get_cli_ptr_from_current(void)
326 {
327 return current->domain;
328 }
330 static inline bool_t tmh_set_client_from_id(
331 struct client *client, tmh_client_t *tmh, cli_id_t cli_id)
332 {
333 struct domain *d = rcu_lock_domain_by_id(cli_id);
334 bool_t rc = 0;
335 if ( d == NULL )
336 return 0;
337 if ( !d->is_dying )
338 {
339 d->tmem = client;
340 tmh->domain = d;
341 rc = 1;
342 }
343 rcu_unlock_domain(d);
344 return rc;
345 }
347 static inline bool_t tmh_current_is_privileged(void)
348 {
349 return IS_PRIV(current->domain);
350 }
352 static inline uint8_t tmh_get_first_byte(pfp_t *pfp)
353 {
354 void *p = __map_domain_page(pfp);
356 return (uint8_t)(*(char *)p);
357 }
359 static inline int tmh_page_cmp(pfp_t *pfp1, pfp_t *pfp2)
360 {
361 const uint64_t *p1 = (uint64_t *)__map_domain_page(pfp1);
362 const uint64_t *p2 = (uint64_t *)__map_domain_page(pfp2);
363 int i;
365 // FIXME: code in assembly?
366 ASSERT(p1 != NULL);
367 ASSERT(p2 != NULL);
368 for ( i = PAGE_SIZE/sizeof(uint64_t); i && *p1 == *p2; i--, *p1++, *p2++ );
369 if ( !i )
370 return 0;
371 if ( *p1 < *p2 )
372 return -1;
373 return 1;
374 }
376 static inline int tmh_pcd_cmp(void *va1, pagesize_t len1, void *va2, pagesize_t len2)
377 {
378 const char *p1 = (char *)va1;
379 const char *p2 = (char *)va2;
380 pagesize_t i;
382 ASSERT(len1 <= PAGE_SIZE);
383 ASSERT(len2 <= PAGE_SIZE);
384 if ( len1 < len2 )
385 return -1;
386 if ( len1 > len2 )
387 return 1;
388 ASSERT(len1 == len2);
389 for ( i = len2; i && *p1 == *p2; i--, *p1++, *p2++ );
390 if ( !i )
391 return 0;
392 if ( *p1 < *p2 )
393 return -1;
394 return 1;
395 }
397 static inline int tmh_tze_pfp_cmp(pfp_t *pfp1, pagesize_t pfp_len, void *tva, pagesize_t tze_len)
398 {
399 const uint64_t *p1 = (uint64_t *)__map_domain_page(pfp1);
400 const uint64_t *p2;
401 pagesize_t i;
403 if ( tze_len == PAGE_SIZE )
404 p2 = (uint64_t *)__map_domain_page((pfp_t *)tva);
405 else
406 p2 = (uint64_t *)tva;
407 ASSERT(pfp_len <= PAGE_SIZE);
408 ASSERT(!(pfp_len & (sizeof(uint64_t)-1)));
409 ASSERT(tze_len <= PAGE_SIZE);
410 ASSERT(!(tze_len & (sizeof(uint64_t)-1)));
411 if ( pfp_len < tze_len )
412 return -1;
413 if ( pfp_len > tze_len )
414 return 1;
415 ASSERT(pfp_len == tze_len);
416 for ( i = tze_len/sizeof(uint64_t); i && *p1 == *p2; i--, *p1++, *p2++ );
417 if ( !i )
418 return 0;
419 if ( *p1 < *p2 )
420 return -1;
421 return 1;
422 }
424 /* return the size of the data in the pfp, ignoring trailing zeroes and
425 * rounded up to the nearest multiple of 8 */
426 static inline pagesize_t tmh_tze_pfp_scan(pfp_t *pfp)
427 {
428 const uint64_t *p = (uint64_t *)__map_domain_page(pfp);
429 pagesize_t bytecount = PAGE_SIZE;
430 pagesize_t len = PAGE_SIZE/sizeof(uint64_t);
431 p += len;
432 while ( len-- && !*--p )
433 bytecount -= sizeof(uint64_t);
434 return bytecount;
435 }
437 static inline void tmh_tze_copy_from_pfp(void *tva, pfp_t *pfp, pagesize_t len)
438 {
439 uint64_t *p1 = (uint64_t *)tva;
440 const uint64_t *p2 = (uint64_t *)__map_domain_page(pfp);
442 pagesize_t i;
443 ASSERT(!(len & (sizeof(uint64_t)-1)));
444 for ( i = len/sizeof(uint64_t); i--; *p1++ = *p2++);
445 }
447 /* these typedefs are in the public/tmem.h interface
448 typedef XEN_GUEST_HANDLE(void) cli_mfn_t;
449 typedef XEN_GUEST_HANDLE(char) cli_va_t;
450 */
451 typedef XEN_GUEST_HANDLE(tmem_op_t) tmem_cli_op_t;
453 static inline int tmh_get_tmemop_from_client(tmem_op_t *op, tmem_cli_op_t uops)
454 {
455 #ifdef CONFIG_COMPAT
456 if ( is_hvm_vcpu(current) ?
457 hvm_guest_x86_mode(current) != 8 :
458 is_pv_32on64_vcpu(current) )
459 {
460 int rc;
461 enum XLAT_tmem_op_u u;
462 tmem_op_compat_t cop;
464 rc = copy_from_guest(&cop, guest_handle_cast(uops, void), 1);
465 if ( rc )
466 return rc;
467 switch ( cop.cmd )
468 {
469 case TMEM_NEW_POOL: u = XLAT_tmem_op_u_creat; break;
470 case TMEM_CONTROL: u = XLAT_tmem_op_u_ctrl; break;
471 case TMEM_AUTH: u = XLAT_tmem_op_u_creat; break;
472 case TMEM_RESTORE_NEW:u = XLAT_tmem_op_u_creat; break;
473 default: u = XLAT_tmem_op_u_gen ; break;
474 }
475 #define XLAT_tmem_op_HNDL_u_ctrl_buf(_d_, _s_) \
476 guest_from_compat_handle((_d_)->u.ctrl.buf, (_s_)->u.ctrl.buf)
477 XLAT_tmem_op(op, &cop);
478 #undef XLAT_tmem_op_HNDL_u_ctrl_buf
479 return 0;
480 }
481 #endif
482 return copy_from_guest(op, uops, 1);
483 }
485 static inline void tmh_copy_to_client_buf_offset(tmem_cli_va_t clibuf, int off,
486 char *tmembuf, int len)
487 {
488 copy_to_guest_offset(clibuf,off,tmembuf,len);
489 }
491 #define TMH_CLI_ID_NULL ((cli_id_t)((domid_t)-1L))
493 #define tmh_cli_id_str "domid"
494 #define tmh_client_str "domain"
496 extern int tmh_decompress_to_client(tmem_cli_mfn_t,void*,size_t,void*);
498 extern int tmh_compress_from_client(tmem_cli_mfn_t,void**,size_t *,void*);
500 extern int tmh_copy_from_client(pfp_t *pfp,
501 tmem_cli_mfn_t cmfn, pagesize_t tmem_offset,
502 pagesize_t pfn_offset, pagesize_t len, void *cva);
504 extern int tmh_copy_to_client(tmem_cli_mfn_t cmfn, pfp_t *pfp,
505 pagesize_t tmem_offset, pagesize_t pfn_offset, pagesize_t len, void *cva);
507 extern int tmh_copy_tze_to_client(tmem_cli_mfn_t cmfn, void *tmem_va, pagesize_t len);
510 #define TMEM_PERF
511 #ifdef TMEM_PERF
512 #define DECL_CYC_COUNTER(x) \
513 uint64_t x##_sum_cycles = 0, x##_count = 0; \
514 uint32_t x##_min_cycles = 0x7fffffff, x##_max_cycles = 0;
515 #define EXTERN_CYC_COUNTER(x) \
516 extern uint64_t x##_sum_cycles, x##_count; \
517 extern uint32_t x##_min_cycles, x##_max_cycles;
518 #define DECL_LOCAL_CYC_COUNTER(x) \
519 int64_t x##_start = 0
520 #define START_CYC_COUNTER(x) x##_start = get_cycles()
521 #define DUP_START_CYC_COUNTER(x,y) x##_start = y##_start
522 /* following might race, but since its advisory only, don't care */
523 #define END_CYC_COUNTER(x) \
524 do { \
525 x##_start = get_cycles() - x##_start; \
526 if (x##_start > 0 && x##_start < 1000000000) { \
527 x##_sum_cycles += x##_start; x##_count++; \
528 if ((uint32_t)x##_start < x##_min_cycles) x##_min_cycles = x##_start; \
529 if ((uint32_t)x##_start > x##_max_cycles) x##_max_cycles = x##_start; \
530 } \
531 } while (0)
532 #define END_CYC_COUNTER_CLI(x,y) \
533 do { \
534 x##_start = get_cycles() - x##_start; \
535 if (x##_start > 0 && x##_start < 1000000000) { \
536 x##_sum_cycles += x##_start; x##_count++; \
537 if ((uint32_t)x##_start < x##_min_cycles) x##_min_cycles = x##_start; \
538 if ((uint32_t)x##_start > x##_max_cycles) x##_max_cycles = x##_start; \
539 y->total_cycles += x##_start; \
540 } \
541 } while (0)
542 #define RESET_CYC_COUNTER(x) { x##_sum_cycles = 0, x##_count = 0; \
543 x##_min_cycles = 0x7fffffff, x##_max_cycles = 0; }
544 #define SCNPRINTF_CYC_COUNTER(buf,size,x,tag) \
545 scnprintf(buf,size, \
546 tag"n:%"PRIu64","tag"t:%"PRIu64","tag"x:%"PRId32","tag"m:%"PRId32",", \
547 x##_count,x##_sum_cycles,x##_max_cycles,x##_min_cycles)
548 #else
549 #define DECL_CYC_COUNTER(x)
550 #define EXTERN_CYC_COUNTER(x) \
551 extern uint64_t x##_sum_cycles, x##_count; \
552 extern uint32_t x##_min_cycles, x##_max_cycles;
553 #define DECL_LOCAL_CYC_COUNTER(x) do { } while (0)
554 #define START_CYC_COUNTER(x) do { } while (0)
555 #define DUP_START_CYC_COUNTER(x) do { } while (0)
556 #define END_CYC_COUNTER(x) do { } while (0)
557 #define SCNPRINTF_CYC_COUNTER(buf,size,x,tag) (0)
558 #define RESET_CYC_COUNTER(x) do { } while (0)
559 #endif
561 #endif /* __XEN_TMEM_XEN_H__ */