/root/src/xen/xen/arch/x86/hvm/mtrr.c
Line | Count | Source (jump to first uncovered line) |
1 | | /* |
2 | | * mtrr.c: MTRR/PAT virtualization |
3 | | * |
4 | | * Copyright (c) 2007, Intel Corporation. |
5 | | * |
6 | | * This program is free software; you can redistribute it and/or modify it |
7 | | * under the terms and conditions of the GNU General Public License, |
8 | | * version 2, as published by the Free Software Foundation. |
9 | | * |
10 | | * This program is distributed in the hope it will be useful, but WITHOUT |
11 | | * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or |
12 | | * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for |
13 | | * more details. |
14 | | * |
15 | | * You should have received a copy of the GNU General Public License along with |
16 | | * this program; If not, see <http://www.gnu.org/licenses/>. |
17 | | */ |
18 | | |
19 | | #include <xen/domain_page.h> |
20 | | #include <asm/e820.h> |
21 | | #include <asm/iocap.h> |
22 | | #include <asm/paging.h> |
23 | | #include <asm/p2m.h> |
24 | | #include <asm/mtrr.h> |
25 | | #include <asm/hvm/support.h> |
26 | | #include <asm/hvm/cacheattr.h> |
27 | | #include <public/hvm/e820.h> |
28 | | |
29 | | /* Get page attribute fields (PAn) from PAT MSR. */ |
30 | 37 | #define pat_cr_2_paf(pat_cr,n) ((((uint64_t)pat_cr) >> ((n)<<3)) & 0xff) |
31 | | |
32 | | /* PAT entry to PTE flags (PAT, PCD, PWT bits). */ |
33 | | static const uint8_t pat_entry_2_pte_flags[8] = { |
34 | | 0, _PAGE_PWT, |
35 | | _PAGE_PCD, _PAGE_PCD | _PAGE_PWT, |
36 | | _PAGE_PAT, _PAGE_PAT | _PAGE_PWT, |
37 | | _PAGE_PAT | _PAGE_PCD, _PAGE_PAT | _PAGE_PCD | _PAGE_PWT }; |
38 | | |
39 | | /* Effective mm type lookup table, according to MTRR and PAT. */ |
40 | | static const uint8_t mm_type_tbl[MTRR_NUM_TYPES][PAT_TYPE_NUMS] = { |
41 | | #define RS MEMORY_NUM_TYPES |
42 | | #define UC MTRR_TYPE_UNCACHABLE |
43 | | #define WB MTRR_TYPE_WRBACK |
44 | | #define WC MTRR_TYPE_WRCOMB |
45 | | #define WP MTRR_TYPE_WRPROT |
46 | | #define WT MTRR_TYPE_WRTHROUGH |
47 | | |
48 | | /* PAT(UC, WC, RS, RS, WT, WP, WB, UC-) */ |
49 | | /* MTRR(UC) */ {UC, WC, RS, RS, UC, UC, UC, UC}, |
50 | | /* MTRR(WC) */ {UC, WC, RS, RS, UC, UC, WC, WC}, |
51 | | /* MTRR(RS) */ {RS, RS, RS, RS, RS, RS, RS, RS}, |
52 | | /* MTRR(RS) */ {RS, RS, RS, RS, RS, RS, RS, RS}, |
53 | | /* MTRR(WT) */ {UC, WC, RS, RS, WT, WP, WT, UC}, |
54 | | /* MTRR(WP) */ {UC, WC, RS, RS, WT, WP, WP, WC}, |
55 | | /* MTRR(WB) */ {UC, WC, RS, RS, WT, WP, WB, UC} |
56 | | |
57 | | #undef UC |
58 | | #undef WC |
59 | | #undef WT |
60 | | #undef WP |
61 | | #undef WB |
62 | | #undef RS |
63 | | }; |
64 | | |
65 | | /* |
66 | | * Reverse lookup table, to find a pat type according to MTRR and effective |
67 | | * memory type. This table is dynamically generated. |
68 | | */ |
69 | | static uint8_t __read_mostly mtrr_epat_tbl[MTRR_NUM_TYPES][MEMORY_NUM_TYPES] = |
70 | | { [0 ... MTRR_NUM_TYPES-1] = |
71 | | { [0 ... MEMORY_NUM_TYPES-1] = INVALID_MEM_TYPE } |
72 | | }; |
73 | | |
74 | | /* Lookup table for PAT entry of a given PAT value in host PAT. */ |
75 | | static uint8_t __read_mostly pat_entry_tbl[PAT_TYPE_NUMS] = |
76 | | { [0 ... PAT_TYPE_NUMS-1] = INVALID_MEM_TYPE }; |
77 | | |
78 | | bool_t is_var_mtrr_overlapped(const struct mtrr_state *m) |
79 | 24 | { |
80 | 24 | unsigned int seg, i; |
81 | 24 | unsigned int num_var_ranges = (uint8_t)m->mtrr_cap; |
82 | 24 | |
83 | 24 | for ( i = 0; i < num_var_ranges; i++ ) |
84 | 24 | { |
85 | 24 | uint64_t base1 = m->var_ranges[i].base >> PAGE_SHIFT; |
86 | 24 | uint64_t mask1 = m->var_ranges[i].mask >> PAGE_SHIFT; |
87 | 24 | |
88 | 24 | if ( !(m->var_ranges[i].mask & MTRR_PHYSMASK_VALID) ) |
89 | 0 | continue; |
90 | 24 | |
91 | 48 | for ( seg = i + 1; seg < num_var_ranges; seg ++ ) |
92 | 48 | { |
93 | 48 | uint64_t base2 = m->var_ranges[seg].base >> PAGE_SHIFT; |
94 | 48 | uint64_t mask2 = m->var_ranges[seg].mask >> PAGE_SHIFT; |
95 | 48 | |
96 | 48 | if ( !(m->var_ranges[seg].mask & MTRR_PHYSMASK_VALID) ) |
97 | 0 | continue; |
98 | 48 | |
99 | 48 | if ( (base1 & mask1 & mask2) == (base2 & mask2 & mask1) ) |
100 | 24 | { |
101 | 24 | /* MTRR is overlapped. */ |
102 | 24 | return 1; |
103 | 24 | } |
104 | 48 | } |
105 | 24 | } |
106 | 0 | return 0; |
107 | 24 | } |
108 | | |
109 | | static int __init hvm_mtrr_pat_init(void) |
110 | 1 | { |
111 | 1 | unsigned int i, j; |
112 | 1 | |
113 | 8 | for ( i = 0; i < MTRR_NUM_TYPES; i++ ) |
114 | 7 | { |
115 | 63 | for ( j = 0; j < PAT_TYPE_NUMS; j++ ) |
116 | 56 | { |
117 | 56 | unsigned int tmp = mm_type_tbl[i][j]; |
118 | 56 | |
119 | 56 | if ( tmp < MEMORY_NUM_TYPES ) |
120 | 30 | mtrr_epat_tbl[i][tmp] = j; |
121 | 56 | } |
122 | 7 | } |
123 | 1 | |
124 | 9 | for ( i = 0; i < PAT_TYPE_NUMS; i++ ) |
125 | 8 | { |
126 | 39 | for ( j = 0; j < PAT_TYPE_NUMS; j++ ) |
127 | 37 | { |
128 | 37 | if ( pat_cr_2_paf(host_pat, j) == i ) |
129 | 6 | { |
130 | 6 | pat_entry_tbl[i] = j; |
131 | 6 | break; |
132 | 6 | } |
133 | 37 | } |
134 | 8 | } |
135 | 1 | |
136 | 1 | return 0; |
137 | 1 | } |
138 | | __initcall(hvm_mtrr_pat_init); |
139 | | |
140 | | uint8_t pat_type_2_pte_flags(uint8_t pat_type) |
141 | 0 | { |
142 | 0 | unsigned int pat_entry = pat_entry_tbl[pat_type]; |
143 | 0 |
|
144 | 0 | /* |
145 | 0 | * INVALID_MEM_TYPE, means doesn't find the pat_entry in host PAT for a |
146 | 0 | * given pat_type. If host PAT covers all the PAT types, it can't happen. |
147 | 0 | */ |
148 | 0 | if ( unlikely(pat_entry == INVALID_MEM_TYPE) ) |
149 | 0 | pat_entry = pat_entry_tbl[PAT_TYPE_UNCACHABLE]; |
150 | 0 |
|
151 | 0 | return pat_entry_2_pte_flags[pat_entry]; |
152 | 0 | } |
153 | | |
154 | | int hvm_vcpu_cacheattr_init(struct vcpu *v) |
155 | 12 | { |
156 | 12 | struct mtrr_state *m = &v->arch.hvm_vcpu.mtrr; |
157 | 12 | |
158 | 12 | memset(m, 0, sizeof(*m)); |
159 | 12 | |
160 | 12 | m->var_ranges = xzalloc_array(struct mtrr_var_range, MTRR_VCNT); |
161 | 12 | if ( m->var_ranges == NULL ) |
162 | 0 | return -ENOMEM; |
163 | 12 | |
164 | 12 | m->mtrr_cap = (1u << 10) | (1u << 8) | MTRR_VCNT; |
165 | 12 | |
166 | 12 | v->arch.hvm_vcpu.pat_cr = |
167 | 12 | ((uint64_t)PAT_TYPE_WRBACK) | /* PAT0: WB */ |
168 | 12 | ((uint64_t)PAT_TYPE_WRTHROUGH << 8) | /* PAT1: WT */ |
169 | 12 | ((uint64_t)PAT_TYPE_UC_MINUS << 16) | /* PAT2: UC- */ |
170 | 12 | ((uint64_t)PAT_TYPE_UNCACHABLE << 24) | /* PAT3: UC */ |
171 | 12 | ((uint64_t)PAT_TYPE_WRBACK << 32) | /* PAT4: WB */ |
172 | 12 | ((uint64_t)PAT_TYPE_WRTHROUGH << 40) | /* PAT5: WT */ |
173 | 12 | ((uint64_t)PAT_TYPE_UC_MINUS << 48) | /* PAT6: UC- */ |
174 | 12 | ((uint64_t)PAT_TYPE_UNCACHABLE << 56); /* PAT7: UC */ |
175 | 12 | |
176 | 12 | return 0; |
177 | 12 | } |
178 | | |
179 | | void hvm_vcpu_cacheattr_destroy(struct vcpu *v) |
180 | 0 | { |
181 | 0 | xfree(v->arch.hvm_vcpu.mtrr.var_ranges); |
182 | 0 | } |
183 | | |
184 | | /* |
185 | | * Get MTRR memory type for physical address pa. |
186 | | * |
187 | | * May return a negative value when order > 0, indicating to the caller |
188 | | * that the respective mapping needs splitting. |
189 | | */ |
190 | | static int get_mtrr_type(const struct mtrr_state *m, |
191 | | paddr_t pa, unsigned int order) |
192 | 2.99M | { |
193 | 2.99M | uint8_t overlap_mtrr = 0; |
194 | 2.99M | uint8_t overlap_mtrr_pos = 0; |
195 | 2.99M | uint64_t mask = -(uint64_t)PAGE_SIZE << order; |
196 | 2.99M | unsigned int seg, num_var_ranges = m->mtrr_cap & 0xff; |
197 | 2.99M | |
198 | 2.99M | if ( unlikely(!(m->enabled & 0x2)) ) |
199 | 589k | return MTRR_TYPE_UNCACHABLE; |
200 | 2.99M | |
201 | 2.40M | pa &= mask; |
202 | 2.40M | if ( (pa < 0x100000) && (m->enabled & 1) ) |
203 | 1.93k | { |
204 | 1.93k | /* Fixed range MTRR takes effect. */ |
205 | 1.93k | uint32_t addr = (uint32_t)pa, index; |
206 | 1.93k | |
207 | 1.93k | if ( addr < 0x80000 ) |
208 | 1.66k | { |
209 | 1.66k | /* 0x00000 ... 0x7FFFF in 64k steps */ |
210 | 1.66k | if ( order > 4 ) |
211 | 0 | return -1; |
212 | 1.66k | seg = (addr >> 16); |
213 | 1.66k | return m->fixed_ranges[seg]; |
214 | 1.66k | } |
215 | 273 | else if ( addr < 0xc0000 ) |
216 | 273 | { |
217 | 273 | /* 0x80000 ... 0xBFFFF in 16k steps */ |
218 | 273 | if ( order > 2 ) |
219 | 0 | return -1; |
220 | 273 | seg = (addr - 0x80000) >> 14; |
221 | 273 | index = (seg >> 3) + 1; |
222 | 273 | seg &= 7; /* select 0-7 segments */ |
223 | 273 | return m->fixed_ranges[index*8 + seg]; |
224 | 273 | } |
225 | 273 | else |
226 | 0 | { |
227 | 0 | /* 0xC0000 ... 0xFFFFF in 4k steps */ |
228 | 0 | if ( order ) |
229 | 0 | return -1; |
230 | 0 | seg = (addr - 0xc0000) >> 12; |
231 | 0 | index = (seg >> 3) + 3; |
232 | 0 | seg &= 7; /* select 0-7 segments */ |
233 | 0 | return m->fixed_ranges[index*8 + seg]; |
234 | 0 | } |
235 | 1.93k | } |
236 | 2.40M | |
237 | 2.40M | /* Match with variable MTRRs. */ |
238 | 24.5M | for ( seg = 0; seg < num_var_ranges; seg++ ) |
239 | 22.1M | { |
240 | 22.1M | uint64_t phys_base = m->var_ranges[seg].base; |
241 | 22.1M | uint64_t phys_mask = m->var_ranges[seg].mask; |
242 | 22.1M | |
243 | 22.1M | if ( phys_mask & MTRR_PHYSMASK_VALID ) |
244 | 10.4M | { |
245 | 10.4M | phys_mask &= mask; |
246 | 10.4M | if ( (pa & phys_mask) == (phys_base & phys_mask) ) |
247 | 1.49M | { |
248 | 1.49M | if ( unlikely(m->overlapped) || order ) |
249 | 1.49M | { |
250 | 1.49M | overlap_mtrr |= 1 << (phys_base & MTRR_PHYSBASE_TYPE_MASK); |
251 | 1.49M | overlap_mtrr_pos = phys_base & MTRR_PHYSBASE_TYPE_MASK; |
252 | 1.49M | } |
253 | 1.49M | else |
254 | 0 | { |
255 | 0 | /* If no overlap, return the found one */ |
256 | 0 | return (phys_base & MTRR_PHYSBASE_TYPE_MASK); |
257 | 0 | } |
258 | 1.49M | } |
259 | 10.4M | } |
260 | 22.1M | } |
261 | 2.40M | |
262 | 2.40M | /* Not found? */ |
263 | 2.40M | if ( unlikely(overlap_mtrr == 0) ) |
264 | 906k | return m->def_type; |
265 | 2.40M | |
266 | 2.40M | /* One match, or multiple identical ones? */ |
267 | 1.49M | if ( likely(overlap_mtrr == (1 << overlap_mtrr_pos)) ) |
268 | 1.49M | return overlap_mtrr_pos; |
269 | 1.49M | |
270 | 0 | if ( order ) |
271 | 0 | return -1; |
272 | 0 |
|
273 | 0 | /* Two or more matches, one being UC? */ |
274 | 0 | if ( overlap_mtrr & (1 << MTRR_TYPE_UNCACHABLE) ) |
275 | 0 | return MTRR_TYPE_UNCACHABLE; |
276 | 0 |
|
277 | 0 | /* Two or more matches, all of them WT and WB? */ |
278 | 0 | if ( overlap_mtrr == |
279 | 0 | ((1 << MTRR_TYPE_WRTHROUGH) | (1 << MTRR_TYPE_WRBACK)) ) |
280 | 0 | return MTRR_TYPE_WRTHROUGH; |
281 | 0 |
|
282 | 0 | /* Behaviour is undefined, but return the last overlapped type. */ |
283 | 0 | return overlap_mtrr_pos; |
284 | 0 | } |
285 | | |
286 | | /* |
287 | | * return the memory type from PAT. |
288 | | * NOTE: valid only when paging is enabled. |
289 | | * Only 4K page PTE is supported now. |
290 | | */ |
291 | | static uint8_t page_pat_type(uint64_t pat_cr, uint32_t pte_flags) |
292 | 0 | { |
293 | 0 | int32_t pat_entry; |
294 | 0 |
|
295 | 0 | /* PCD/PWT -> bit 1/0 of PAT entry */ |
296 | 0 | pat_entry = ( pte_flags >> 3 ) & 0x3; |
297 | 0 | /* PAT bits as bit 2 of PAT entry */ |
298 | 0 | if ( pte_flags & _PAGE_PAT ) |
299 | 0 | pat_entry |= 4; |
300 | 0 |
|
301 | 0 | return (uint8_t)pat_cr_2_paf(pat_cr, pat_entry); |
302 | 0 | } |
303 | | |
304 | | /* |
305 | | * Effective memory type for leaf page. |
306 | | */ |
307 | | static uint8_t effective_mm_type(struct mtrr_state *m, |
308 | | uint64_t pat, |
309 | | paddr_t gpa, |
310 | | uint32_t pte_flags, |
311 | | uint8_t gmtrr_mtype) |
312 | 0 | { |
313 | 0 | uint8_t mtrr_mtype, pat_value, effective; |
314 | 0 | |
315 | 0 | /* if get_pat_flags() gives a dedicated MTRR type, |
316 | 0 | * just use it |
317 | 0 | */ |
318 | 0 | if ( gmtrr_mtype == NO_HARDCODE_MEM_TYPE ) |
319 | 0 | mtrr_mtype = get_mtrr_type(m, gpa, 0); |
320 | 0 | else |
321 | 0 | mtrr_mtype = gmtrr_mtype; |
322 | 0 |
|
323 | 0 | pat_value = page_pat_type(pat, pte_flags); |
324 | 0 |
|
325 | 0 | effective = mm_type_tbl[mtrr_mtype][pat_value]; |
326 | 0 |
|
327 | 0 | return effective; |
328 | 0 | } |
329 | | |
330 | | uint32_t get_pat_flags(struct vcpu *v, |
331 | | uint32_t gl1e_flags, |
332 | | paddr_t gpaddr, |
333 | | paddr_t spaddr, |
334 | | uint8_t gmtrr_mtype) |
335 | 0 | { |
336 | 0 | uint8_t guest_eff_mm_type; |
337 | 0 | uint8_t shadow_mtrr_type; |
338 | 0 | uint8_t pat_entry_value; |
339 | 0 | uint64_t pat = v->arch.hvm_vcpu.pat_cr; |
340 | 0 | struct mtrr_state *g = &v->arch.hvm_vcpu.mtrr; |
341 | 0 |
|
342 | 0 | /* 1. Get the effective memory type of guest physical address, |
343 | 0 | * with the pair of guest MTRR and PAT |
344 | 0 | */ |
345 | 0 | guest_eff_mm_type = effective_mm_type(g, pat, gpaddr, |
346 | 0 | gl1e_flags, gmtrr_mtype); |
347 | 0 | /* 2. Get the memory type of host physical address, with MTRR */ |
348 | 0 | shadow_mtrr_type = get_mtrr_type(&mtrr_state, spaddr, 0); |
349 | 0 |
|
350 | 0 | /* 3. Find the memory type in PAT, with host MTRR memory type |
351 | 0 | * and guest effective memory type. |
352 | 0 | */ |
353 | 0 | pat_entry_value = mtrr_epat_tbl[shadow_mtrr_type][guest_eff_mm_type]; |
354 | 0 | /* If conflit occurs(e.g host MTRR is UC, guest memory type is |
355 | 0 | * WB),set UC as effective memory. Here, returning PAT_TYPE_UNCACHABLE will |
356 | 0 | * always set effective memory as UC. |
357 | 0 | */ |
358 | 0 | if ( pat_entry_value == INVALID_MEM_TYPE ) |
359 | 0 | { |
360 | 0 | struct domain *d = v->domain; |
361 | 0 | p2m_type_t p2mt; |
362 | 0 | get_gfn_query_unlocked(d, paddr_to_pfn(gpaddr), &p2mt); |
363 | 0 | if (p2m_is_ram(p2mt)) |
364 | 0 | gdprintk(XENLOG_WARNING, |
365 | 0 | "Conflict occurs for a given guest l1e flags:%x " |
366 | 0 | "at %"PRIx64" (the effective mm type:%d), " |
367 | 0 | "because the host mtrr type is:%d\n", |
368 | 0 | gl1e_flags, (uint64_t)gpaddr, guest_eff_mm_type, |
369 | 0 | shadow_mtrr_type); |
370 | 0 | pat_entry_value = PAT_TYPE_UNCACHABLE; |
371 | 0 | } |
372 | 0 | /* 4. Get the pte flags */ |
373 | 0 | return pat_type_2_pte_flags(pat_entry_value); |
374 | 0 | } |
375 | | |
376 | | static inline bool_t valid_mtrr_type(uint8_t type) |
377 | 34 | { |
378 | 34 | switch ( type ) |
379 | 34 | { |
380 | 34 | case MTRR_TYPE_UNCACHABLE: |
381 | 34 | case MTRR_TYPE_WRBACK: |
382 | 34 | case MTRR_TYPE_WRCOMB: |
383 | 34 | case MTRR_TYPE_WRPROT: |
384 | 34 | case MTRR_TYPE_WRTHROUGH: |
385 | 34 | return 1; |
386 | 34 | } |
387 | 0 | return 0; |
388 | 34 | } |
389 | | |
390 | | bool_t mtrr_def_type_msr_set(struct domain *d, struct mtrr_state *m, |
391 | | uint64_t msr_content) |
392 | 34 | { |
393 | 34 | uint8_t def_type = msr_content & 0xff; |
394 | 34 | uint8_t enabled = (msr_content >> 10) & 0x3; |
395 | 34 | |
396 | 34 | if ( unlikely(!valid_mtrr_type(def_type)) ) |
397 | 0 | { |
398 | 0 | HVM_DBG_LOG(DBG_LEVEL_MSR, "invalid MTRR def type:%x\n", def_type); |
399 | 0 | return 0; |
400 | 0 | } |
401 | 34 | |
402 | 34 | if ( unlikely(msr_content && (msr_content & ~0xcffUL)) ) |
403 | 0 | { |
404 | 0 | HVM_DBG_LOG(DBG_LEVEL_MSR, "invalid msr content:%"PRIx64"\n", |
405 | 0 | msr_content); |
406 | 0 | return 0; |
407 | 0 | } |
408 | 34 | |
409 | 34 | if ( m->enabled != enabled || m->def_type != def_type ) |
410 | 23 | { |
411 | 23 | m->enabled = enabled; |
412 | 23 | m->def_type = def_type; |
413 | 23 | memory_type_changed(d); |
414 | 23 | } |
415 | 34 | |
416 | 34 | return 1; |
417 | 34 | } |
418 | | |
419 | | bool_t mtrr_fix_range_msr_set(struct domain *d, struct mtrr_state *m, |
420 | | uint32_t row, uint64_t msr_content) |
421 | 0 | { |
422 | 0 | uint64_t *fixed_range_base = (uint64_t *)m->fixed_ranges; |
423 | 0 |
|
424 | 0 | if ( fixed_range_base[row] != msr_content ) |
425 | 0 | { |
426 | 0 | uint8_t *range = (uint8_t*)&msr_content; |
427 | 0 | unsigned int i; |
428 | 0 |
|
429 | 0 | for ( i = 0; i < 8; i++ ) |
430 | 0 | if ( unlikely(!valid_mtrr_type(range[i])) ) |
431 | 0 | return 0; |
432 | 0 |
|
433 | 0 | fixed_range_base[row] = msr_content; |
434 | 0 | memory_type_changed(d); |
435 | 0 | } |
436 | 0 |
|
437 | 0 | return 1; |
438 | 0 | } |
439 | | |
440 | | bool_t mtrr_var_range_msr_set( |
441 | | struct domain *d, struct mtrr_state *m, uint32_t msr, uint64_t msr_content) |
442 | 176 | { |
443 | 176 | uint32_t index, phys_addr; |
444 | 176 | uint64_t msr_mask; |
445 | 176 | uint64_t *var_range_base = (uint64_t*)m->var_ranges; |
446 | 176 | |
447 | 176 | index = msr - MSR_IA32_MTRR_PHYSBASE(0); |
448 | 176 | if ( var_range_base[index] == msr_content ) |
449 | 176 | return 1; |
450 | 176 | |
451 | 0 | if ( unlikely(!valid_mtrr_type((uint8_t)msr_content)) ) |
452 | 0 | return 0; |
453 | 0 |
|
454 | 0 | if ( d == current->domain ) |
455 | 0 | phys_addr = d->arch.cpuid->extd.maxphysaddr; |
456 | 0 | else |
457 | 0 | phys_addr = paddr_bits; |
458 | 0 | msr_mask = ~((((uint64_t)1) << phys_addr) - 1); |
459 | 0 | msr_mask |= (index & 1) ? 0x7ffUL : 0xf00UL; |
460 | 0 | if ( unlikely(msr_content & msr_mask) ) |
461 | 0 | { |
462 | 0 | HVM_DBG_LOG(DBG_LEVEL_MSR, "invalid msr content:%"PRIx64"\n", |
463 | 0 | msr_content); |
464 | 0 | return 0; |
465 | 0 | } |
466 | 0 |
|
467 | 0 | var_range_base[index] = msr_content; |
468 | 0 |
|
469 | 0 | m->overlapped = is_var_mtrr_overlapped(m); |
470 | 0 |
|
471 | 0 | memory_type_changed(d); |
472 | 0 |
|
473 | 0 | return 1; |
474 | 0 | } |
475 | | |
476 | | bool_t mtrr_pat_not_equal(struct vcpu *vd, struct vcpu *vs) |
477 | 0 | { |
478 | 0 | struct mtrr_state *md = &vd->arch.hvm_vcpu.mtrr; |
479 | 0 | struct mtrr_state *ms = &vs->arch.hvm_vcpu.mtrr; |
480 | 0 | int32_t res; |
481 | 0 | uint8_t num_var_ranges = (uint8_t)md->mtrr_cap; |
482 | 0 |
|
483 | 0 | /* Test fixed ranges. */ |
484 | 0 | res = memcmp(md->fixed_ranges, ms->fixed_ranges, |
485 | 0 | NUM_FIXED_RANGES*sizeof(mtrr_type)); |
486 | 0 | if ( res ) |
487 | 0 | return 1; |
488 | 0 |
|
489 | 0 | /* Test var ranges. */ |
490 | 0 | res = memcmp(md->var_ranges, ms->var_ranges, |
491 | 0 | num_var_ranges*sizeof(struct mtrr_var_range)); |
492 | 0 | if ( res ) |
493 | 0 | return 1; |
494 | 0 |
|
495 | 0 | /* Test default type MSR. */ |
496 | 0 | if ( (md->def_type != ms->def_type) |
497 | 0 | && (md->enabled != ms->enabled) ) |
498 | 0 | return 1; |
499 | 0 |
|
500 | 0 | /* Test PAT. */ |
501 | 0 | if ( vd->arch.hvm_vcpu.pat_cr != vs->arch.hvm_vcpu.pat_cr ) |
502 | 0 | return 1; |
503 | 0 |
|
504 | 0 | return 0; |
505 | 0 | } |
506 | | |
507 | | struct hvm_mem_pinned_cacheattr_range { |
508 | | struct list_head list; |
509 | | uint64_t start, end; |
510 | | uint32_t type; |
511 | | struct rcu_head rcu; |
512 | | }; |
513 | | |
514 | | static DEFINE_RCU_READ_LOCK(pinned_cacheattr_rcu_lock); |
515 | | |
516 | | void hvm_init_cacheattr_region_list(struct domain *d) |
517 | 1 | { |
518 | 1 | INIT_LIST_HEAD(&d->arch.hvm_domain.pinned_cacheattr_ranges); |
519 | 1 | } |
520 | | |
521 | | void hvm_destroy_cacheattr_region_list(struct domain *d) |
522 | 0 | { |
523 | 0 | struct list_head *head = &d->arch.hvm_domain.pinned_cacheattr_ranges; |
524 | 0 | struct hvm_mem_pinned_cacheattr_range *range; |
525 | 0 |
|
526 | 0 | while ( !list_empty(head) ) |
527 | 0 | { |
528 | 0 | range = list_entry(head->next, |
529 | 0 | struct hvm_mem_pinned_cacheattr_range, |
530 | 0 | list); |
531 | 0 | list_del(&range->list); |
532 | 0 | xfree(range); |
533 | 0 | } |
534 | 0 | } |
535 | | |
536 | | int hvm_get_mem_pinned_cacheattr(struct domain *d, gfn_t gfn, |
537 | | unsigned int order) |
538 | 1.49M | { |
539 | 1.49M | struct hvm_mem_pinned_cacheattr_range *range; |
540 | 1.49M | uint64_t mask = ~(uint64_t)0 << order; |
541 | 1.49M | int rc = -ENXIO; |
542 | 1.49M | |
543 | 1.49M | ASSERT(is_hvm_domain(d)); |
544 | 1.49M | |
545 | 1.49M | rcu_read_lock(&pinned_cacheattr_rcu_lock); |
546 | 1.49M | list_for_each_entry_rcu ( range, |
547 | 1.49M | &d->arch.hvm_domain.pinned_cacheattr_ranges, |
548 | 1.49M | list ) |
549 | 0 | { |
550 | 0 | if ( ((gfn_x(gfn) & mask) >= range->start) && |
551 | 0 | ((gfn_x(gfn) | ~mask) <= range->end) ) |
552 | 0 | { |
553 | 0 | rc = range->type; |
554 | 0 | break; |
555 | 0 | } |
556 | 0 | if ( ((gfn_x(gfn) & mask) <= range->end) && |
557 | 0 | ((gfn_x(gfn) | ~mask) >= range->start) ) |
558 | 0 | { |
559 | 0 | rc = -EADDRNOTAVAIL; |
560 | 0 | break; |
561 | 0 | } |
562 | 0 | } |
563 | 1.49M | rcu_read_unlock(&pinned_cacheattr_rcu_lock); |
564 | 1.49M | |
565 | 1.49M | return rc; |
566 | 1.49M | } |
567 | | |
568 | | static void free_pinned_cacheattr_entry(struct rcu_head *rcu) |
569 | 0 | { |
570 | 0 | xfree(container_of(rcu, struct hvm_mem_pinned_cacheattr_range, rcu)); |
571 | 0 | } |
572 | | |
573 | | int hvm_set_mem_pinned_cacheattr(struct domain *d, uint64_t gfn_start, |
574 | | uint64_t gfn_end, uint32_t type) |
575 | 0 | { |
576 | 0 | struct hvm_mem_pinned_cacheattr_range *range; |
577 | 0 | int rc = 1; |
578 | 0 |
|
579 | 0 | if ( !is_hvm_domain(d) ) |
580 | 0 | return -EOPNOTSUPP; |
581 | 0 |
|
582 | 0 | if ( gfn_end < gfn_start || (gfn_start | gfn_end) >> paddr_bits ) |
583 | 0 | return -EINVAL; |
584 | 0 |
|
585 | 0 | switch ( type ) |
586 | 0 | { |
587 | 0 | case XEN_DOMCTL_DELETE_MEM_CACHEATTR: |
588 | 0 | /* Remove the requested range. */ |
589 | 0 | rcu_read_lock(&pinned_cacheattr_rcu_lock); |
590 | 0 | list_for_each_entry_rcu ( range, |
591 | 0 | &d->arch.hvm_domain.pinned_cacheattr_ranges, |
592 | 0 | list ) |
593 | 0 | if ( range->start == gfn_start && range->end == gfn_end ) |
594 | 0 | { |
595 | 0 | rcu_read_unlock(&pinned_cacheattr_rcu_lock); |
596 | 0 | list_del_rcu(&range->list); |
597 | 0 | type = range->type; |
598 | 0 | call_rcu(&range->rcu, free_pinned_cacheattr_entry); |
599 | 0 | p2m_memory_type_changed(d); |
600 | 0 | switch ( type ) |
601 | 0 | { |
602 | 0 | case PAT_TYPE_UC_MINUS: |
603 | 0 | /* |
604 | 0 | * For EPT we can also avoid the flush in this case; |
605 | 0 | * see epte_get_entry_emt(). |
606 | 0 | */ |
607 | 0 | if ( hap_enabled(d) && cpu_has_vmx ) |
608 | 0 | case PAT_TYPE_UNCACHABLE: |
609 | 0 | break; |
610 | 0 | /* fall through */ |
611 | 0 | default: |
612 | 0 | flush_all(FLUSH_CACHE); |
613 | 0 | break; |
614 | 0 | } |
615 | 0 | return 0; |
616 | 0 | } |
617 | 0 | rcu_read_unlock(&pinned_cacheattr_rcu_lock); |
618 | 0 | return -ENOENT; |
619 | 0 |
|
620 | 0 | case PAT_TYPE_UC_MINUS: |
621 | 0 | case PAT_TYPE_UNCACHABLE: |
622 | 0 | case PAT_TYPE_WRBACK: |
623 | 0 | case PAT_TYPE_WRCOMB: |
624 | 0 | case PAT_TYPE_WRPROT: |
625 | 0 | case PAT_TYPE_WRTHROUGH: |
626 | 0 | break; |
627 | 0 |
|
628 | 0 | default: |
629 | 0 | return -EINVAL; |
630 | 0 | } |
631 | 0 |
|
632 | 0 | rcu_read_lock(&pinned_cacheattr_rcu_lock); |
633 | 0 | list_for_each_entry_rcu ( range, |
634 | 0 | &d->arch.hvm_domain.pinned_cacheattr_ranges, |
635 | 0 | list ) |
636 | 0 | { |
637 | 0 | if ( range->start == gfn_start && range->end == gfn_end ) |
638 | 0 | { |
639 | 0 | range->type = type; |
640 | 0 | rc = 0; |
641 | 0 | break; |
642 | 0 | } |
643 | 0 | if ( range->start <= gfn_end && gfn_start <= range->end ) |
644 | 0 | { |
645 | 0 | rc = -EBUSY; |
646 | 0 | break; |
647 | 0 | } |
648 | 0 | } |
649 | 0 | rcu_read_unlock(&pinned_cacheattr_rcu_lock); |
650 | 0 | if ( rc <= 0 ) |
651 | 0 | return rc; |
652 | 0 |
|
653 | 0 | range = xzalloc(struct hvm_mem_pinned_cacheattr_range); |
654 | 0 | if ( range == NULL ) |
655 | 0 | return -ENOMEM; |
656 | 0 |
|
657 | 0 | range->start = gfn_start; |
658 | 0 | range->end = gfn_end; |
659 | 0 | range->type = type; |
660 | 0 |
|
661 | 0 | list_add_rcu(&range->list, &d->arch.hvm_domain.pinned_cacheattr_ranges); |
662 | 0 | p2m_memory_type_changed(d); |
663 | 0 | if ( type != PAT_TYPE_WRBACK ) |
664 | 0 | flush_all(FLUSH_CACHE); |
665 | 0 |
|
666 | 0 | return 0; |
667 | 0 | } |
668 | | |
669 | | static int hvm_save_mtrr_msr(struct domain *d, hvm_domain_context_t *h) |
670 | 0 | { |
671 | 0 | int i; |
672 | 0 | struct vcpu *v; |
673 | 0 | struct hvm_hw_mtrr hw_mtrr; |
674 | 0 | struct mtrr_state *mtrr_state; |
675 | 0 | /* save mtrr&pat */ |
676 | 0 | for_each_vcpu(d, v) |
677 | 0 | { |
678 | 0 | mtrr_state = &v->arch.hvm_vcpu.mtrr; |
679 | 0 |
|
680 | 0 | hvm_get_guest_pat(v, &hw_mtrr.msr_pat_cr); |
681 | 0 |
|
682 | 0 | hw_mtrr.msr_mtrr_def_type = mtrr_state->def_type |
683 | 0 | | (mtrr_state->enabled << 10); |
684 | 0 | hw_mtrr.msr_mtrr_cap = mtrr_state->mtrr_cap; |
685 | 0 |
|
686 | 0 | for ( i = 0; i < MTRR_VCNT; i++ ) |
687 | 0 | { |
688 | 0 | /* save physbase */ |
689 | 0 | hw_mtrr.msr_mtrr_var[i*2] = |
690 | 0 | ((uint64_t*)mtrr_state->var_ranges)[i*2]; |
691 | 0 | /* save physmask */ |
692 | 0 | hw_mtrr.msr_mtrr_var[i*2+1] = |
693 | 0 | ((uint64_t*)mtrr_state->var_ranges)[i*2+1]; |
694 | 0 | } |
695 | 0 |
|
696 | 0 | for ( i = 0; i < NUM_FIXED_MSR; i++ ) |
697 | 0 | hw_mtrr.msr_mtrr_fixed[i] = |
698 | 0 | ((uint64_t*)mtrr_state->fixed_ranges)[i]; |
699 | 0 |
|
700 | 0 | if ( hvm_save_entry(MTRR, v->vcpu_id, h, &hw_mtrr) != 0 ) |
701 | 0 | return 1; |
702 | 0 | } |
703 | 0 | return 0; |
704 | 0 | } |
705 | | |
706 | | static int hvm_load_mtrr_msr(struct domain *d, hvm_domain_context_t *h) |
707 | 0 | { |
708 | 0 | int vcpuid, i; |
709 | 0 | struct vcpu *v; |
710 | 0 | struct mtrr_state *mtrr_state; |
711 | 0 | struct hvm_hw_mtrr hw_mtrr; |
712 | 0 |
|
713 | 0 | vcpuid = hvm_load_instance(h); |
714 | 0 | if ( vcpuid >= d->max_vcpus || (v = d->vcpu[vcpuid]) == NULL ) |
715 | 0 | { |
716 | 0 | dprintk(XENLOG_G_ERR, "HVM restore: dom%d has no vcpu%u\n", |
717 | 0 | d->domain_id, vcpuid); |
718 | 0 | return -EINVAL; |
719 | 0 | } |
720 | 0 |
|
721 | 0 | if ( hvm_load_entry(MTRR, h, &hw_mtrr) != 0 ) |
722 | 0 | return -EINVAL; |
723 | 0 |
|
724 | 0 | mtrr_state = &v->arch.hvm_vcpu.mtrr; |
725 | 0 |
|
726 | 0 | hvm_set_guest_pat(v, hw_mtrr.msr_pat_cr); |
727 | 0 |
|
728 | 0 | mtrr_state->mtrr_cap = hw_mtrr.msr_mtrr_cap; |
729 | 0 |
|
730 | 0 | for ( i = 0; i < NUM_FIXED_MSR; i++ ) |
731 | 0 | mtrr_fix_range_msr_set(d, mtrr_state, i, hw_mtrr.msr_mtrr_fixed[i]); |
732 | 0 |
|
733 | 0 | for ( i = 0; i < MTRR_VCNT; i++ ) |
734 | 0 | { |
735 | 0 | mtrr_var_range_msr_set(d, mtrr_state, |
736 | 0 | MSR_IA32_MTRR_PHYSBASE(i), |
737 | 0 | hw_mtrr.msr_mtrr_var[i * 2]); |
738 | 0 | mtrr_var_range_msr_set(d, mtrr_state, |
739 | 0 | MSR_IA32_MTRR_PHYSMASK(i), |
740 | 0 | hw_mtrr.msr_mtrr_var[i * 2 + 1]); |
741 | 0 | } |
742 | 0 |
|
743 | 0 | mtrr_def_type_msr_set(d, mtrr_state, hw_mtrr.msr_mtrr_def_type); |
744 | 0 |
|
745 | 0 | return 0; |
746 | 0 | } |
747 | | |
748 | | HVM_REGISTER_SAVE_RESTORE(MTRR, hvm_save_mtrr_msr, hvm_load_mtrr_msr, |
749 | | 1, HVMSR_PER_VCPU); |
750 | | |
751 | | void memory_type_changed(struct domain *d) |
752 | 23 | { |
753 | 23 | if ( need_iommu(d) && d->vcpu && d->vcpu[0] ) |
754 | 23 | { |
755 | 23 | p2m_memory_type_changed(d); |
756 | 23 | flush_all(FLUSH_CACHE); |
757 | 23 | } |
758 | 23 | } |
759 | | |
760 | | int epte_get_entry_emt(struct domain *d, unsigned long gfn, mfn_t mfn, |
761 | | unsigned int order, uint8_t *ipat, bool_t direct_mmio) |
762 | 1.85M | { |
763 | 1.85M | int gmtrr_mtype, hmtrr_mtype; |
764 | 1.85M | struct vcpu *v = current; |
765 | 1.85M | |
766 | 1.85M | *ipat = 0; |
767 | 1.85M | |
768 | 1.85M | if ( v->domain != d ) |
769 | 632k | v = d->vcpu ? d->vcpu[0] : NULL; |
770 | 1.85M | |
771 | 1.85M | /* Mask, not add, for order so it works with INVALID_MFN on unmapping */ |
772 | 1.85M | if ( rangeset_overlaps_range(mmio_ro_ranges, mfn_x(mfn), |
773 | 1.85M | mfn_x(mfn) | ((1UL << order) - 1)) ) |
774 | 0 | { |
775 | 0 | if ( !order || rangeset_contains_range(mmio_ro_ranges, mfn_x(mfn), |
776 | 0 | mfn_x(mfn) | ((1UL << order) - 1)) ) |
777 | 0 | { |
778 | 0 | *ipat = 1; |
779 | 0 | return MTRR_TYPE_UNCACHABLE; |
780 | 0 | } |
781 | 0 | /* Force invalid memory type so resolve_misconfig() will split it */ |
782 | 0 | return -1; |
783 | 0 | } |
784 | 1.85M | |
785 | 1.85M | if ( direct_mmio ) |
786 | 362k | { |
787 | 362k | if ( (mfn_x(mfn) ^ d->arch.hvm_domain.vmx.apic_access_mfn) >> order ) |
788 | 362k | return MTRR_TYPE_UNCACHABLE; |
789 | 1 | if ( order ) |
790 | 0 | return -1; |
791 | 1 | *ipat = 1; |
792 | 1 | return MTRR_TYPE_WRBACK; |
793 | 1 | } |
794 | 1.85M | |
795 | 1.49M | if ( !mfn_valid(mfn) ) |
796 | 0 | { |
797 | 0 | *ipat = 1; |
798 | 0 | return MTRR_TYPE_UNCACHABLE; |
799 | 0 | } |
800 | 1.49M | |
801 | 1.49M | if ( !need_iommu(d) && !cache_flush_permitted(d) ) |
802 | 0 | { |
803 | 0 | *ipat = 1; |
804 | 0 | return MTRR_TYPE_WRBACK; |
805 | 0 | } |
806 | 1.49M | |
807 | 1.49M | gmtrr_mtype = hvm_get_mem_pinned_cacheattr(d, _gfn(gfn), order); |
808 | 1.49M | if ( gmtrr_mtype >= 0 ) |
809 | 0 | { |
810 | 0 | *ipat = 1; |
811 | 0 | return gmtrr_mtype != PAT_TYPE_UC_MINUS ? gmtrr_mtype |
812 | 0 | : MTRR_TYPE_UNCACHABLE; |
813 | 0 | } |
814 | 1.49M | if ( gmtrr_mtype == -EADDRNOTAVAIL ) |
815 | 0 | return -1; |
816 | 1.49M | |
817 | 1.49M | gmtrr_mtype = is_hvm_domain(d) && v ? |
818 | 1.49M | get_mtrr_type(&v->arch.hvm_vcpu.mtrr, |
819 | 1.49M | gfn << PAGE_SHIFT, order) : |
820 | 0 | MTRR_TYPE_WRBACK; |
821 | 1.49M | hmtrr_mtype = get_mtrr_type(&mtrr_state, mfn_x(mfn) << PAGE_SHIFT, order); |
822 | 1.49M | if ( gmtrr_mtype < 0 || hmtrr_mtype < 0 ) |
823 | 0 | return -1; |
824 | 1.49M | |
825 | 1.49M | /* If both types match we're fine. */ |
826 | 1.49M | if ( likely(gmtrr_mtype == hmtrr_mtype) ) |
827 | 889k | return hmtrr_mtype; |
828 | 1.49M | |
829 | 1.49M | /* If either type is UC, we have to go with that one. */ |
830 | 606k | if ( gmtrr_mtype == MTRR_TYPE_UNCACHABLE || |
831 | 0 | hmtrr_mtype == MTRR_TYPE_UNCACHABLE ) |
832 | 606k | return MTRR_TYPE_UNCACHABLE; |
833 | 606k | |
834 | 606k | /* If either type is WB, we have to go with the other one. */ |
835 | 0 | if ( gmtrr_mtype == MTRR_TYPE_WRBACK ) |
836 | 0 | return hmtrr_mtype; |
837 | 0 | if ( hmtrr_mtype == MTRR_TYPE_WRBACK ) |
838 | 0 | return gmtrr_mtype; |
839 | 0 |
|
840 | 0 | /* |
841 | 0 | * At this point we have disagreeing WC, WT, or WP types. The only |
842 | 0 | * combination that can be cleanly resolved is WT:WP. The ones involving |
843 | 0 | * WC need to be converted to UC, both due to the memory ordering |
844 | 0 | * differences and because WC disallows reads to be cached (WT and WP |
845 | 0 | * permit this), while WT and WP require writes to go straight to memory |
846 | 0 | * (WC can buffer them). |
847 | 0 | */ |
848 | 0 | if ( (gmtrr_mtype == MTRR_TYPE_WRTHROUGH && |
849 | 0 | hmtrr_mtype == MTRR_TYPE_WRPROT) || |
850 | 0 | (gmtrr_mtype == MTRR_TYPE_WRPROT && |
851 | 0 | hmtrr_mtype == MTRR_TYPE_WRTHROUGH) ) |
852 | 0 | return MTRR_TYPE_WRPROT; |
853 | 0 |
|
854 | 0 | return MTRR_TYPE_UNCACHABLE; |
855 | 0 | } |