debuggers.hg

view xen/common/tmem_xen.c @ 20964:a3fa6d444b25

Fix domain reference leaks

Besides two unlikely/rarely hit ones in x86 code, the main offender
was tmh_client_from_cli_id(), which didn't even have a counterpart
(albeit it had a comment correctly saying that it causes d->refcnt to
get incremented). Unfortunately(?) this required a bit of code
restructuring (as I needed to change the code anyway, I also fixed
a couple os missing bounds checks which would sooner or later be
reported as security vulnerabilities), so I would hope Dan could give
it his blessing before it gets applied.

Signed-off-by: Jan Beulich <jbeulich@novell.com>
author Keir Fraser <keir.fraser@citrix.com>
date Wed Feb 10 09:18:43 2010 +0000 (2010-02-10)
parents 869884b93584
children 61372a4f4e76
line source
1 /******************************************************************************
2 * tmem-xen.c
3 *
4 * Xen-specific Transcendent memory
5 *
6 * Copyright (c) 2009, Dan Magenheimer, Oracle Corp.
7 */
9 #include <xen/tmem.h>
10 #include <xen/tmem_xen.h>
11 #include <xen/lzo.h> /* compression code */
12 #include <xen/paging.h>
13 #include <xen/domain_page.h>
15 #define EXPORT /* indicates code other modules are dependent upon */
17 EXPORT int opt_tmem = 0;
18 boolean_param("tmem", opt_tmem);
20 EXPORT int opt_tmem_compress = 0;
21 boolean_param("tmem_compress", opt_tmem_compress);
23 EXPORT int opt_tmem_shared_auth = 0;
24 boolean_param("tmem_shared_auth", opt_tmem_shared_auth);
26 EXPORT int opt_tmem_lock = 0;
27 integer_param("tmem_lock", opt_tmem_lock);
29 EXPORT atomic_t freeable_page_count = ATOMIC_INIT(0);
31 #ifdef COMPARE_COPY_PAGE_SSE2
32 DECL_CYC_COUNTER(pg_copy1);
33 DECL_CYC_COUNTER(pg_copy2);
34 DECL_CYC_COUNTER(pg_copy3);
35 DECL_CYC_COUNTER(pg_copy4);
36 #else
37 DECL_CYC_COUNTER(pg_copy);
38 #endif
40 /* these are a concurrency bottleneck, could be percpu and dynamically
41 * allocated iff opt_tmem_compress */
42 #define LZO_WORKMEM_BYTES LZO1X_1_MEM_COMPRESS
43 #define LZO_DSTMEM_PAGES 2
44 static DEFINE_PER_CPU_READ_MOSTLY(unsigned char *, workmem);
45 static DEFINE_PER_CPU_READ_MOSTLY(unsigned char *, dstmem);
47 #ifdef COMPARE_COPY_PAGE_SSE2
48 #include <asm/flushtlb.h> /* REMOVE ME AFTER TEST */
49 #include <asm/page.h> /* REMOVE ME AFTER TEST */
50 #endif
51 void tmh_copy_page(char *to, char*from)
52 {
53 #ifdef COMPARE_COPY_PAGE_SSE2
54 DECL_LOCAL_CYC_COUNTER(pg_copy1);
55 DECL_LOCAL_CYC_COUNTER(pg_copy2);
56 DECL_LOCAL_CYC_COUNTER(pg_copy3);
57 DECL_LOCAL_CYC_COUNTER(pg_copy4);
58 *to = *from; /* don't measure TLB misses */
59 flush_area_local(to,FLUSH_CACHE|FLUSH_ORDER(0));
60 flush_area_local(from,FLUSH_CACHE|FLUSH_ORDER(0));
61 START_CYC_COUNTER(pg_copy1);
62 copy_page_sse2(to, from); /* cold cache */
63 END_CYC_COUNTER(pg_copy1);
64 START_CYC_COUNTER(pg_copy2);
65 copy_page_sse2(to, from); /* hot cache */
66 END_CYC_COUNTER(pg_copy2);
67 flush_area_local(to,FLUSH_CACHE|FLUSH_ORDER(0));
68 flush_area_local(from,FLUSH_CACHE|FLUSH_ORDER(0));
69 START_CYC_COUNTER(pg_copy3);
70 memcpy(to, from, PAGE_SIZE); /* cold cache */
71 END_CYC_COUNTER(pg_copy3);
72 START_CYC_COUNTER(pg_copy4);
73 memcpy(to, from, PAGE_SIZE); /* hot cache */
74 END_CYC_COUNTER(pg_copy4);
75 #else
76 DECL_LOCAL_CYC_COUNTER(pg_copy);
77 START_CYC_COUNTER(pg_copy);
78 memcpy(to, from, PAGE_SIZE);
79 END_CYC_COUNTER(pg_copy);
80 #endif
81 }
83 #ifdef __ia64__
84 static inline void *cli_mfn_to_va(tmem_cli_mfn_t cmfn, unsigned long *pcli_mfn)
85 {
86 ASSERT(0);
87 return NULL;
88 }
89 #define paging_mark_dirty(_x,_y) do {} while(0)
90 #else
91 static inline void *cli_mfn_to_va(tmem_cli_mfn_t cmfn, unsigned long *pcli_mfn)
92 {
93 unsigned long cli_mfn;
94 p2m_type_t t;
96 cli_mfn = mfn_x(gfn_to_mfn(current->domain, cmfn, &t));
97 if (t != p2m_ram_rw)
98 return NULL;
99 if (pcli_mfn != NULL)
100 *pcli_mfn = cli_mfn;
101 return map_domain_page(cli_mfn);
102 }
103 #endif
105 EXPORT int tmh_copy_from_client(pfp_t *pfp,
106 tmem_cli_mfn_t cmfn, uint32_t tmem_offset,
107 uint32_t pfn_offset, uint32_t len, void *cli_va)
108 {
109 unsigned long tmem_mfn;
110 void *tmem_va;
112 ASSERT(pfp != NULL);
113 if ( tmem_offset || pfn_offset || len )
114 if ( (cli_va == NULL) && ((cli_va = cli_mfn_to_va(cmfn,NULL)) == NULL) )
115 return -EFAULT;
116 tmem_mfn = page_to_mfn(pfp);
117 tmem_va = map_domain_page(tmem_mfn);
118 mb();
119 if (!len && !tmem_offset && !pfn_offset)
120 memset(tmem_va, 0, PAGE_SIZE);
121 else if (len == PAGE_SIZE && !tmem_offset && !pfn_offset)
122 tmh_copy_page(tmem_va, cli_va);
123 else if ( (tmem_offset+len <= PAGE_SIZE) &&
124 (pfn_offset+len <= PAGE_SIZE) )
125 memcpy((char *)tmem_va+tmem_offset,(char *)cli_va+pfn_offset,len);
126 unmap_domain_page(cli_va);
127 unmap_domain_page(tmem_va);
128 return 1;
129 }
131 EXPORT int tmh_compress_from_client(tmem_cli_mfn_t cmfn,
132 void **out_va, size_t *out_len, void *cli_va)
133 {
134 int ret = 0;
135 unsigned char *dmem = this_cpu(dstmem);
136 unsigned char *wmem = this_cpu(workmem);
138 if ( (cli_va == NULL) && (cli_va = cli_mfn_to_va(cmfn,NULL)) == NULL)
139 return -EFAULT;
140 if ( dmem == NULL || wmem == NULL )
141 return 0; /* no buffer, so can't compress */
142 mb();
143 ret = lzo1x_1_compress(cli_va, PAGE_SIZE, dmem, out_len, wmem);
144 ASSERT(ret == LZO_E_OK);
145 *out_va = dmem;
146 unmap_domain_page(cli_va);
147 return 1;
148 }
150 EXPORT int tmh_copy_to_client(tmem_cli_mfn_t cmfn, pfp_t *pfp,
151 uint32_t tmem_offset, uint32_t pfn_offset, uint32_t len, void *cli_va)
152 {
153 unsigned long tmem_mfn, cli_mfn = 0;
154 int mark_dirty = 1;
155 void *tmem_va;
157 ASSERT(pfp != NULL);
158 if ( cli_va != NULL )
159 mark_dirty = 0;
160 else if ( (cli_va = cli_mfn_to_va(cmfn,&cli_mfn)) == NULL)
161 return -EFAULT;
162 tmem_mfn = page_to_mfn(pfp);
163 tmem_va = map_domain_page(tmem_mfn);
164 if (len == PAGE_SIZE && !tmem_offset && !pfn_offset)
165 tmh_copy_page(cli_va, tmem_va);
166 else if ( (tmem_offset+len <= PAGE_SIZE) && (pfn_offset+len <= PAGE_SIZE) )
167 memcpy((char *)cli_va+pfn_offset,(char *)tmem_va+tmem_offset,len);
168 unmap_domain_page(tmem_va);
169 if ( mark_dirty )
170 {
171 unmap_domain_page(cli_va);
172 paging_mark_dirty(current->domain,cli_mfn);
173 }
174 mb();
175 return 1;
176 }
178 EXPORT int tmh_decompress_to_client(tmem_cli_mfn_t cmfn, void *tmem_va,
179 size_t size, void *cli_va)
180 {
181 unsigned long cli_mfn = 0;
182 int mark_dirty = 1;
183 size_t out_len = PAGE_SIZE;
184 int ret;
186 if ( cli_va != NULL )
187 mark_dirty = 0;
188 else if ( (cli_va = cli_mfn_to_va(cmfn,&cli_mfn)) == NULL)
189 return -EFAULT;
190 ret = lzo1x_decompress_safe(tmem_va, size, cli_va, &out_len);
191 ASSERT(ret == LZO_E_OK);
192 ASSERT(out_len == PAGE_SIZE);
193 if ( mark_dirty )
194 {
195 unmap_domain_page(cli_va);
196 paging_mark_dirty(current->domain,cli_mfn);
197 }
198 mb();
199 return 1;
200 }
202 /****************** XEN-SPECIFIC MEMORY ALLOCATION ********************/
204 EXPORT struct xmem_pool *tmh_mempool = 0;
205 EXPORT unsigned int tmh_mempool_maxalloc = 0;
207 EXPORT DEFINE_SPINLOCK(tmh_page_list_lock);
208 EXPORT PAGE_LIST_HEAD(tmh_page_list);
209 EXPORT unsigned long tmh_page_list_pages = 0;
211 /* free anything on tmh_page_list to Xen's scrub list */
212 EXPORT void tmh_release_avail_pages_to_host(void)
213 {
214 spin_lock(&tmh_page_list_lock);
215 while ( !page_list_empty(&tmh_page_list) )
216 {
217 struct page_info *pg = page_list_remove_head(&tmh_page_list);
218 scrub_one_page(pg);
219 tmh_page_list_pages--;
220 free_domheap_page(pg);
221 }
222 ASSERT(tmh_page_list_pages == 0);
223 INIT_PAGE_LIST_HEAD(&tmh_page_list);
224 spin_unlock(&tmh_page_list_lock);
225 }
227 EXPORT void tmh_scrub_page(struct page_info *pi, unsigned int memflags)
228 {
229 if ( pi == NULL )
230 return;
231 if ( !(memflags & MEMF_tmem) )
232 scrub_one_page(pi);
233 }
235 #ifndef __i386__
236 static noinline void *tmh_mempool_page_get(unsigned long size)
237 {
238 struct page_info *pi;
240 ASSERT(size == PAGE_SIZE);
241 if ( (pi = tmh_alloc_page(NULL,0)) == NULL )
242 return NULL;
243 ASSERT(IS_VALID_PAGE(pi));
244 return page_to_virt(pi);
245 }
247 static void tmh_mempool_page_put(void *page_va)
248 {
249 ASSERT(IS_PAGE_ALIGNED(page_va));
250 tmh_free_page(virt_to_page(page_va));
251 }
253 static int tmh_mempool_init(void)
254 {
255 tmh_mempool = xmem_pool_create("tmem", tmh_mempool_page_get,
256 tmh_mempool_page_put, PAGE_SIZE, 0, PAGE_SIZE);
257 if ( tmh_mempool )
258 tmh_mempool_maxalloc = xmem_pool_maxalloc(tmh_mempool);
259 return tmh_mempool != NULL;
260 }
262 /* persistent pools are per-domain */
264 static void *tmh_persistent_pool_page_get(unsigned long size)
265 {
266 struct page_info *pi;
267 struct domain *d = current->domain;
269 ASSERT(size == PAGE_SIZE);
270 if ( (pi = _tmh_alloc_page_thispool(d)) == NULL )
271 return NULL;
272 ASSERT(IS_VALID_PAGE(pi));
273 return __map_domain_page(pi);
274 }
276 static void tmh_persistent_pool_page_put(void *page_va)
277 {
278 struct page_info *pi;
280 ASSERT(IS_PAGE_ALIGNED(page_va));
281 pi = virt_to_page(page_va);
282 ASSERT(IS_VALID_PAGE(pi));
283 _tmh_free_page_thispool(pi);
284 }
285 #endif
287 /****************** XEN-SPECIFIC CLIENT HANDLING ********************/
289 EXPORT tmh_client_t *tmh_client_init(cli_id_t cli_id)
290 {
291 tmh_client_t *tmh;
292 char name[5];
293 int i, shift;
295 if ( (tmh = xmalloc(tmh_client_t)) == NULL )
296 return NULL;
297 for (i = 0, shift = 12; i < 4; shift -=4, i++)
298 name[i] = (((unsigned short)cli_id >> shift) & 0xf) + '0';
299 name[4] = '\0';
300 #ifndef __i386__
301 tmh->persistent_pool = xmem_pool_create(name, tmh_persistent_pool_page_get,
302 tmh_persistent_pool_page_put, PAGE_SIZE, 0, PAGE_SIZE);
303 if ( tmh->persistent_pool == NULL )
304 {
305 xfree(tmh);
306 return NULL;
307 }
308 #endif
309 return tmh;
310 }
312 EXPORT void tmh_client_destroy(tmh_client_t *tmh)
313 {
314 #ifndef __i386__
315 xmem_pool_destroy(tmh->persistent_pool);
316 #endif
317 put_domain(tmh->domain);
318 tmh->domain = NULL;
319 }
321 /****************** XEN-SPECIFIC HOST INITIALIZATION ********************/
323 EXPORT int tmh_init(void)
324 {
325 #ifndef __i386__
326 int dstmem_order, workmem_order;
327 bool_t bad_alloc = 0;
328 struct page_info *pi;
329 unsigned char *p1, *p2;
330 int cpu;
332 if ( !tmh_mempool_init() )
333 return 0;
335 dstmem_order = get_order_from_pages(LZO_DSTMEM_PAGES);
336 workmem_order = get_order_from_bytes(LZO1X_1_MEM_COMPRESS);
337 for_each_possible_cpu ( cpu )
338 {
339 pi = alloc_domheap_pages(0,dstmem_order,0);
340 per_cpu(dstmem, cpu) = p1 = ((pi == NULL) ? NULL : page_to_virt(pi));
341 pi = alloc_domheap_pages(0,workmem_order,0);
342 per_cpu(workmem, cpu) = p2 = ((pi == NULL) ? NULL : page_to_virt(pi));
343 if ( (p1 == NULL) || (p2 == NULL) )
344 bad_alloc++;
345 }
346 if ( bad_alloc )
347 printk("tmem: can't allocate compression buffers for %d cpus\n",
348 bad_alloc);
349 #endif
350 return 1;
351 }