debuggers.hg

view xen/common/tmem_xen.c @ 19964:3952eaeb70b0

Introduce and use a per-CPU read-mostly sub-section

Since mixing data that only gets setup once and then (perhaps
frequently) gets read by remote CPUs with data that the local CPU may
modify (again, perhaps frequently) still causes undesirable cache
protocol related bus traffic, separate the former class of objects
from the latter.

These objects converted here are just picked based on their write-once
(or write-very-rarely) properties; perhaps some more adjustments may
be desirable subsequently. The primary users of the new sub-section
will result from the next patch.

Signed-off-by: Jan Beulich <jbeulich@novell.com>
author Keir Fraser <keir.fraser@citrix.com>
date Mon Jul 13 11:32:41 2009 +0100 (2009-07-13)
parents 8fecba36bc63
children 91407452cdb6
line source
1 /******************************************************************************
2 * tmem-xen.c
3 *
4 * Xen-specific Transcendent memory
5 *
6 * Copyright (c) 2009, Dan Magenheimer, Oracle Corp.
7 */
9 #include <xen/tmem.h>
10 #include <xen/tmem_xen.h>
11 #include <xen/lzo.h> /* compression code */
12 #include <xen/paging.h>
13 #include <xen/domain_page.h>
15 #define EXPORT /* indicates code other modules are dependent upon */
17 EXPORT int opt_tmem = 0;
18 boolean_param("tmem", opt_tmem);
20 EXPORT int opt_tmem_compress = 0;
21 boolean_param("tmem_compress", opt_tmem_compress);
23 EXPORT int opt_tmem_lock = 0;
24 integer_param("tmem_lock", opt_tmem_lock);
26 #ifdef COMPARE_COPY_PAGE_SSE2
27 DECL_CYC_COUNTER(pg_copy1);
28 DECL_CYC_COUNTER(pg_copy2);
29 DECL_CYC_COUNTER(pg_copy3);
30 DECL_CYC_COUNTER(pg_copy4);
31 #else
32 DECL_CYC_COUNTER(pg_copy);
33 #endif
35 /* these are a concurrency bottleneck, could be percpu and dynamically
36 * allocated iff opt_tmem_compress */
37 #define LZO_WORKMEM_BYTES LZO1X_1_MEM_COMPRESS
38 #define LZO_DSTMEM_PAGES 2
39 static DEFINE_PER_CPU_READ_MOSTLY(unsigned char *, workmem);
40 static DEFINE_PER_CPU_READ_MOSTLY(unsigned char *, dstmem);
42 #ifdef COMPARE_COPY_PAGE_SSE2
43 #include <asm/flushtlb.h> /* REMOVE ME AFTER TEST */
44 #include <asm/page.h> /* REMOVE ME AFTER TEST */
45 #endif
46 void tmh_copy_page(char *to, char*from)
47 {
48 #ifdef COMPARE_COPY_PAGE_SSE2
49 DECL_LOCAL_CYC_COUNTER(pg_copy1);
50 DECL_LOCAL_CYC_COUNTER(pg_copy2);
51 DECL_LOCAL_CYC_COUNTER(pg_copy3);
52 DECL_LOCAL_CYC_COUNTER(pg_copy4);
53 *to = *from; /* don't measure TLB misses */
54 flush_area_local(to,FLUSH_CACHE|FLUSH_ORDER(0));
55 flush_area_local(from,FLUSH_CACHE|FLUSH_ORDER(0));
56 START_CYC_COUNTER(pg_copy1);
57 copy_page_sse2(to, from); /* cold cache */
58 END_CYC_COUNTER(pg_copy1);
59 START_CYC_COUNTER(pg_copy2);
60 copy_page_sse2(to, from); /* hot cache */
61 END_CYC_COUNTER(pg_copy2);
62 flush_area_local(to,FLUSH_CACHE|FLUSH_ORDER(0));
63 flush_area_local(from,FLUSH_CACHE|FLUSH_ORDER(0));
64 START_CYC_COUNTER(pg_copy3);
65 memcpy(to, from, PAGE_SIZE); /* cold cache */
66 END_CYC_COUNTER(pg_copy3);
67 START_CYC_COUNTER(pg_copy4);
68 memcpy(to, from, PAGE_SIZE); /* hot cache */
69 END_CYC_COUNTER(pg_copy4);
70 #else
71 DECL_LOCAL_CYC_COUNTER(pg_copy);
72 START_CYC_COUNTER(pg_copy);
73 memcpy(to, from, PAGE_SIZE);
74 END_CYC_COUNTER(pg_copy);
75 #endif
76 }
78 #ifdef __ia64__
79 static inline void *cli_mfn_to_va(tmem_cli_mfn_t cmfn, unsigned long *pcli_mfn)
80 {
81 ASSERT(0);
82 }
83 #define paging_mark_dirty(_x,_y) do {} while(0)
84 #else
85 static inline void *cli_mfn_to_va(tmem_cli_mfn_t cmfn, unsigned long *pcli_mfn)
86 {
87 unsigned long cli_mfn;
88 p2m_type_t t;
90 cli_mfn = mfn_x(gfn_to_mfn(current->domain, cmfn, &t));
91 if (t != p2m_ram_rw)
92 return NULL;
93 if (pcli_mfn != NULL)
94 *pcli_mfn = cli_mfn;
95 return map_domain_page(cli_mfn);
96 }
97 #endif
99 EXPORT int tmh_copy_from_client(pfp_t *pfp,
100 tmem_cli_mfn_t cmfn, uint32_t tmem_offset,
101 uint32_t pfn_offset, uint32_t len)
102 {
103 unsigned long tmem_mfn;
104 void *tmem_va, *cli_va = NULL;
106 ASSERT(pfp != NULL);
107 if ( tmem_offset || pfn_offset || len )
108 if ( (cli_va = cli_mfn_to_va(cmfn,NULL)) == NULL)
109 return -EFAULT;
110 tmem_mfn = page_to_mfn(pfp);
111 tmem_va = map_domain_page(tmem_mfn);
112 mb();
113 if (!len && !tmem_offset && !pfn_offset)
114 memset(tmem_va, 0, PAGE_SIZE);
115 else if (len == PAGE_SIZE && !tmem_offset && !pfn_offset)
116 tmh_copy_page(tmem_va, cli_va);
117 else if ( (tmem_offset+len <= PAGE_SIZE) &&
118 (pfn_offset+len <= PAGE_SIZE) )
119 memcpy((char *)tmem_va+tmem_offset,(char *)cli_va+pfn_offset,len);
120 unmap_domain_page(cli_va);
121 unmap_domain_page(tmem_va);
122 return 1;
123 }
125 EXPORT int tmh_compress_from_client(tmem_cli_mfn_t cmfn,
126 void **out_va, size_t *out_len)
127 {
128 void *cli_va;
129 int ret = 0;
130 unsigned char *dmem = this_cpu(dstmem);
131 unsigned char *wmem = this_cpu(workmem);
133 if ( (cli_va = cli_mfn_to_va(cmfn,NULL)) == NULL)
134 return -EFAULT;
135 if ( dmem == NULL || wmem == NULL )
136 return 0; /* no buffer, so can't compress */
137 mb();
138 ret = lzo1x_1_compress(cli_va, PAGE_SIZE, dmem, out_len, wmem);
139 ASSERT(ret == LZO_E_OK);
140 *out_va = dmem;
141 unmap_domain_page(cli_va);
142 return 1;
143 }
145 EXPORT int tmh_copy_to_client(tmem_cli_mfn_t cmfn, pfp_t *pfp,
146 uint32_t tmem_offset, uint32_t pfn_offset, uint32_t len)
147 {
148 unsigned long tmem_mfn, cli_mfn;
149 void *tmem_va, *cli_va;
151 ASSERT(pfp != NULL);
152 if ( (cli_va = cli_mfn_to_va(cmfn,&cli_mfn)) == NULL)
153 return -EFAULT;
154 tmem_mfn = page_to_mfn(pfp);
155 tmem_va = map_domain_page(tmem_mfn);
156 if (len == PAGE_SIZE && !tmem_offset && !pfn_offset)
157 tmh_copy_page(cli_va, tmem_va);
158 else if ( (tmem_offset+len <= PAGE_SIZE) && (pfn_offset+len <= PAGE_SIZE) )
159 memcpy((char *)cli_va+pfn_offset,(char *)tmem_va+tmem_offset,len);
160 unmap_domain_page(tmem_va);
161 unmap_domain_page(cli_va);
162 paging_mark_dirty(current->domain,cli_mfn);
163 mb();
164 return 1;
165 }
167 EXPORT int tmh_decompress_to_client(tmem_cli_mfn_t cmfn, void *tmem_va, size_t size)
168 {
169 unsigned long cli_mfn;
170 void *cli_va;
171 size_t out_len = PAGE_SIZE;
172 int ret;
174 if ( (cli_va = cli_mfn_to_va(cmfn,&cli_mfn)) == NULL)
175 return -EFAULT;
176 ret = lzo1x_decompress_safe(tmem_va, size, cli_va, &out_len);
177 ASSERT(ret == LZO_E_OK);
178 ASSERT(out_len == PAGE_SIZE);
179 unmap_domain_page(cli_va);
180 paging_mark_dirty(current->domain,cli_mfn);
181 mb();
182 return 1;
183 }
185 /****************** XEN-SPECIFIC MEMORY ALLOCATION ********************/
187 EXPORT struct xmem_pool *tmh_mempool = 0;
188 EXPORT unsigned int tmh_mempool_maxalloc = 0;
190 EXPORT DEFINE_SPINLOCK(tmh_page_list_lock);
191 EXPORT PAGE_LIST_HEAD(tmh_page_list);
192 EXPORT unsigned long tmh_page_list_pages = 0;
194 /* free anything on tmh_page_list to Xen's scrub list */
195 EXPORT void tmh_release_avail_pages_to_host(void)
196 {
197 spin_lock(&tmh_page_list_lock);
198 while ( !page_list_empty(&tmh_page_list) )
199 {
200 struct page_info *pg = page_list_first(&tmh_page_list);
201 scrub_one_page(pg);
202 free_domheap_page(pg);
203 }
204 INIT_PAGE_LIST_HEAD(&tmh_page_list);
205 tmh_page_list_pages = 0;
206 spin_unlock(&tmh_page_list_lock);
207 }
209 EXPORT void tmh_scrub_page(struct page_info *pi, unsigned int memflags)
210 {
211 if ( pi == NULL )
212 return;
213 if ( !(memflags & MEMF_tmem) )
214 scrub_one_page(pi);
215 }
217 #ifndef __i386__
218 static noinline void *tmh_mempool_page_get(unsigned long size)
219 {
220 struct page_info *pi;
222 ASSERT(size == PAGE_SIZE);
223 if ( (pi = tmh_alloc_page(NULL,0)) == NULL )
224 return NULL;
225 ASSERT(IS_VALID_PAGE(pi));
226 return page_to_virt(pi);
227 }
229 static void tmh_mempool_page_put(void *page_va)
230 {
231 ASSERT(IS_PAGE_ALIGNED(page_va));
232 tmh_free_page(virt_to_page(page_va));
233 }
235 static int tmh_mempool_init(void)
236 {
237 tmh_mempool = xmem_pool_create("tmem", tmh_mempool_page_get,
238 tmh_mempool_page_put, PAGE_SIZE, 0, PAGE_SIZE);
239 if ( tmh_mempool )
240 tmh_mempool_maxalloc = xmem_pool_maxalloc(tmh_mempool);
241 return tmh_mempool != NULL;
242 }
244 /* persistent pools are per-domain */
246 static void *tmh_persistent_pool_page_get(unsigned long size)
247 {
248 struct page_info *pi;
249 struct domain *d = current->domain;
251 ASSERT(size == PAGE_SIZE);
252 if ( (pi = _tmh_alloc_page_thispool(d)) == NULL )
253 return NULL;
254 ASSERT(IS_VALID_PAGE(pi));
255 return map_domain_page(page_to_mfn(pi));
256 }
258 static void tmh_persistent_pool_page_put(void *page_va)
259 {
260 struct page_info *pi;
262 ASSERT(IS_PAGE_ALIGNED(page_va));
263 pi = virt_to_page(page_va);
264 ASSERT(IS_VALID_PAGE(pi));
265 _tmh_free_page_thispool(pi);
266 }
267 #endif
269 /****************** XEN-SPECIFIC CLIENT HANDLING ********************/
271 EXPORT tmh_client_t *tmh_client_init(void)
272 {
273 tmh_client_t *tmh;
274 char name[5];
275 domid_t domid = current->domain->domain_id;
276 int i, shift;
278 if ( (tmh = xmalloc(tmh_client_t)) == NULL )
279 return NULL;
280 for (i = 0, shift = 12; i < 4; shift -=4, i++)
281 name[i] = ((unsigned short)domid >> shift) & 0xf;
282 name[4] = '\0';
283 #ifndef __i386__
284 tmh->persistent_pool = xmem_pool_create(name, tmh_persistent_pool_page_get,
285 tmh_persistent_pool_page_put, PAGE_SIZE, 0, PAGE_SIZE);
286 if ( tmh->persistent_pool == NULL )
287 {
288 xfree(tmh);
289 return NULL;
290 }
291 #endif
292 tmh->domain = current->domain;
293 return tmh;
294 }
296 EXPORT void tmh_client_destroy(tmh_client_t *tmh)
297 {
298 #ifndef __i386__
299 xmem_pool_destroy(tmh->persistent_pool);
300 #endif
301 xfree(tmh);
302 }
304 /****************** XEN-SPECIFIC HOST INITIALIZATION ********************/
306 EXPORT int tmh_init(void)
307 {
308 #ifndef __i386__
309 int dstmem_order, workmem_order;
310 bool_t bad_alloc = 0;
311 struct page_info *pi;
312 unsigned char *p1, *p2;
313 int cpu;
315 if ( !tmh_mempool_init() )
316 return 0;
318 dstmem_order = get_order_from_pages(LZO_DSTMEM_PAGES);
319 workmem_order = get_order_from_bytes(LZO1X_1_MEM_COMPRESS);
320 for_each_cpu ( cpu )
321 {
322 pi = alloc_domheap_pages(0,dstmem_order,0);
323 per_cpu(dstmem, cpu) = p1 = ((pi == NULL) ? NULL : page_to_virt(pi));
324 pi = alloc_domheap_pages(0,workmem_order,0);
325 per_cpu(workmem, cpu) = p2 = ((pi == NULL) ? NULL : page_to_virt(pi));
326 if ( (p1 == NULL) || (p2 == NULL) )
327 bad_alloc++;
328 }
329 if ( bad_alloc )
330 printk("tmem: can't allocate compression buffers for %d cpus\n",
331 bad_alloc);
332 #endif
333 return 1;
334 }