/root/src/xen/xen/arch/x86/cpu/common.c
Line | Count | Source (jump to first uncovered line) |
1 | | #include <xen/init.h> |
2 | | #include <xen/string.h> |
3 | | #include <xen/delay.h> |
4 | | #include <xen/smp.h> |
5 | | #include <asm/current.h> |
6 | | #include <asm/processor.h> |
7 | | #include <asm/xstate.h> |
8 | | #include <asm/msr.h> |
9 | | #include <asm/io.h> |
10 | | #include <asm/mpspec.h> |
11 | | #include <asm/apic.h> |
12 | | #include <mach_apic.h> |
13 | | #include <asm/setup.h> |
14 | | #include <public/sysctl.h> /* for XEN_INVALID_{SOCKET,CORE}_ID */ |
15 | | |
16 | | #include "cpu.h" |
17 | | |
18 | | bool_t opt_arat = 1; |
19 | | boolean_param("arat", opt_arat); |
20 | | |
21 | | /* pku: Flag to enable Memory Protection Keys (default on). */ |
22 | | static bool_t opt_pku = 1; |
23 | | boolean_param("pku", opt_pku); |
24 | | |
25 | | unsigned int opt_cpuid_mask_ecx = ~0u; |
26 | | integer_param("cpuid_mask_ecx", opt_cpuid_mask_ecx); |
27 | | unsigned int opt_cpuid_mask_edx = ~0u; |
28 | | integer_param("cpuid_mask_edx", opt_cpuid_mask_edx); |
29 | | |
30 | | unsigned int opt_cpuid_mask_xsave_eax = ~0u; |
31 | | integer_param("cpuid_mask_xsave_eax", opt_cpuid_mask_xsave_eax); |
32 | | |
33 | | unsigned int opt_cpuid_mask_ext_ecx = ~0u; |
34 | | integer_param("cpuid_mask_ext_ecx", opt_cpuid_mask_ext_ecx); |
35 | | unsigned int opt_cpuid_mask_ext_edx = ~0u; |
36 | | integer_param("cpuid_mask_ext_edx", opt_cpuid_mask_ext_edx); |
37 | | |
38 | | unsigned int __initdata expected_levelling_cap; |
39 | | unsigned int __read_mostly levelling_caps; |
40 | | |
41 | | DEFINE_PER_CPU(struct cpuidmasks, cpuidmasks); |
42 | | struct cpuidmasks __read_mostly cpuidmask_defaults; |
43 | | |
44 | | const struct cpu_dev *__read_mostly cpu_devs[X86_VENDOR_NUM] = {}; |
45 | | |
46 | | unsigned int paddr_bits __read_mostly = 36; |
47 | | unsigned int hap_paddr_bits __read_mostly = 36; |
48 | | unsigned int vaddr_bits __read_mostly = VADDR_BITS; |
49 | | |
50 | | /* |
51 | | * Default host IA32_CR_PAT value to cover all memory types. |
52 | | * BIOS usually sets it to 0x07040600070406. |
53 | | */ |
54 | | u64 host_pat = 0x050100070406; |
55 | | |
56 | | static unsigned int cleared_caps[NCAPINTS]; |
57 | | static unsigned int forced_caps[NCAPINTS]; |
58 | | |
59 | | void __init setup_clear_cpu_cap(unsigned int cap) |
60 | 1 | { |
61 | 1 | const uint32_t *dfs; |
62 | 1 | unsigned int i; |
63 | 1 | |
64 | 1 | if (__test_and_set_bit(cap, cleared_caps)) |
65 | 0 | return; |
66 | 1 | |
67 | 1 | if (test_bit(cap, forced_caps)) |
68 | 0 | printk("%pS clearing previously forced feature %#x\n", |
69 | 0 | __builtin_return_address(0), cap); |
70 | 1 | |
71 | 1 | __clear_bit(cap, boot_cpu_data.x86_capability); |
72 | 1 | dfs = lookup_deep_deps(cap); |
73 | 1 | |
74 | 1 | if (!dfs) |
75 | 1 | return; |
76 | 1 | |
77 | 0 | for (i = 0; i < FSCAPINTS; ++i) { |
78 | 0 | cleared_caps[i] |= dfs[i]; |
79 | 0 | boot_cpu_data.x86_capability[i] &= ~dfs[i]; |
80 | 0 | if (!(forced_caps[i] & dfs[i])) |
81 | 0 | continue; |
82 | 0 | printk("%pS implicitly clearing previously forced feature(s) %u:%#x\n", |
83 | 0 | __builtin_return_address(0), |
84 | 0 | i, forced_caps[i] & dfs[i]); |
85 | 0 | } |
86 | 0 | } |
87 | | |
88 | | void __init setup_force_cpu_cap(unsigned int cap) |
89 | 2 | { |
90 | 2 | if (__test_and_set_bit(cap, forced_caps)) |
91 | 0 | return; |
92 | 2 | |
93 | 2 | if (test_bit(cap, cleared_caps)) { |
94 | 0 | printk("%pS tries to force previously cleared feature %#x\n", |
95 | 0 | __builtin_return_address(0), cap); |
96 | 0 | return; |
97 | 0 | } |
98 | 2 | |
99 | 2 | __set_bit(cap, boot_cpu_data.x86_capability); |
100 | 2 | } |
101 | | |
102 | | static void default_init(struct cpuinfo_x86 * c) |
103 | 0 | { |
104 | 0 | /* Not much we can do here... */ |
105 | 0 | /* Check if at least it has cpuid */ |
106 | 0 | BUG_ON(c->cpuid_level == -1); |
107 | 0 | __clear_bit(X86_FEATURE_SEP, c->x86_capability); |
108 | 0 | } |
109 | | |
110 | | static const struct cpu_dev default_cpu = { |
111 | | .c_init = default_init, |
112 | | .c_vendor = "Unknown", |
113 | | }; |
114 | | static const struct cpu_dev *this_cpu = &default_cpu; |
115 | | |
116 | | static void default_ctxt_switch_levelling(const struct vcpu *next) |
117 | 0 | { |
118 | 0 | /* Nop */ |
119 | 0 | } |
120 | | void (* __read_mostly ctxt_switch_levelling)(const struct vcpu *next) = |
121 | | default_ctxt_switch_levelling; |
122 | | |
123 | | bool_t opt_cpu_info; |
124 | | boolean_param("cpuinfo", opt_cpu_info); |
125 | | |
126 | | int get_model_name(struct cpuinfo_x86 *c) |
127 | 12 | { |
128 | 12 | unsigned int *v; |
129 | 12 | char *p, *q; |
130 | 12 | |
131 | 12 | if (c->extended_cpuid_level < 0x80000004) |
132 | 0 | return 0; |
133 | 12 | |
134 | 12 | v = (unsigned int *) c->x86_model_id; |
135 | 12 | cpuid(0x80000002, &v[0], &v[1], &v[2], &v[3]); |
136 | 12 | cpuid(0x80000003, &v[4], &v[5], &v[6], &v[7]); |
137 | 12 | cpuid(0x80000004, &v[8], &v[9], &v[10], &v[11]); |
138 | 12 | c->x86_model_id[48] = 0; |
139 | 12 | |
140 | 12 | /* Intel chips right-justify this string for some dumb reason; |
141 | 12 | undo that brain damage */ |
142 | 12 | p = q = &c->x86_model_id[0]; |
143 | 84 | while ( *p == ' ' ) |
144 | 72 | p++; |
145 | 12 | if ( p != q ) { |
146 | 504 | while ( *p ) |
147 | 492 | *q++ = *p++; |
148 | 108 | while ( q <= &c->x86_model_id[48] ) |
149 | 96 | *q++ = '\0'; /* Zero-pad the rest */ |
150 | 12 | } |
151 | 12 | |
152 | 12 | return 1; |
153 | 12 | } |
154 | | |
155 | | |
156 | | void display_cacheinfo(struct cpuinfo_x86 *c) |
157 | 0 | { |
158 | 0 | unsigned int dummy, ecx, edx, l2size; |
159 | 0 |
|
160 | 0 | if (c->extended_cpuid_level >= 0x80000005) { |
161 | 0 | cpuid(0x80000005, &dummy, &dummy, &ecx, &edx); |
162 | 0 | if (opt_cpu_info) |
163 | 0 | printk("CPU: L1 I cache %dK (%d bytes/line)," |
164 | 0 | " D cache %dK (%d bytes/line)\n", |
165 | 0 | edx>>24, edx&0xFF, ecx>>24, ecx&0xFF); |
166 | 0 | c->x86_cache_size=(ecx>>24)+(edx>>24); |
167 | 0 | } |
168 | 0 |
|
169 | 0 | if (c->extended_cpuid_level < 0x80000006) /* Some chips just has a large L1. */ |
170 | 0 | return; |
171 | 0 |
|
172 | 0 | ecx = cpuid_ecx(0x80000006); |
173 | 0 | l2size = ecx >> 16; |
174 | 0 | |
175 | 0 | c->x86_cache_size = l2size; |
176 | 0 |
|
177 | 0 | if (opt_cpu_info) |
178 | 0 | printk("CPU: L2 Cache: %dK (%d bytes/line)\n", |
179 | 0 | l2size, ecx & 0xFF); |
180 | 0 | } |
181 | | |
182 | | int get_cpu_vendor(uint32_t b, uint32_t c, uint32_t d, enum get_cpu_vendor mode) |
183 | 14 | { |
184 | 14 | int i; |
185 | 14 | static int printed; |
186 | 14 | |
187 | 14 | for (i = 0; i < X86_VENDOR_NUM; i++) { |
188 | 14 | if (cpu_devs[i]) { |
189 | 14 | struct { |
190 | 14 | uint32_t b, d, c; |
191 | 14 | } *ptr = (void *)cpu_devs[i]->c_ident; |
192 | 14 | |
193 | 14 | if (ptr->b == b && ptr->c == c && ptr->d == d) { |
194 | 14 | if (mode == gcv_host) |
195 | 13 | this_cpu = cpu_devs[i]; |
196 | 14 | return i; |
197 | 14 | } |
198 | 14 | } |
199 | 14 | } |
200 | 0 | if (mode == gcv_guest) |
201 | 0 | return X86_VENDOR_UNKNOWN; |
202 | 0 | if (!printed) { |
203 | 0 | printed++; |
204 | 0 | printk(KERN_ERR "CPU: Vendor unknown, using generic init.\n"); |
205 | 0 | printk(KERN_ERR "CPU: Your system may be unstable.\n"); |
206 | 0 | } |
207 | 0 | this_cpu = &default_cpu; |
208 | 0 |
|
209 | 0 | return X86_VENDOR_UNKNOWN; |
210 | 0 | } |
211 | | |
212 | | static inline u32 _phys_pkg_id(u32 cpuid_apic, int index_msb) |
213 | 49 | { |
214 | 49 | return cpuid_apic >> index_msb; |
215 | 49 | } |
216 | | |
217 | | /* |
218 | | * cpuid returns the value latched in the HW at reset, not the APIC ID |
219 | | * register's value. For any box whose BIOS changes APIC IDs, like |
220 | | * clustered APIC systems, we must use get_apic_id(). |
221 | | * |
222 | | * See Intel's IA-32 SW Dev's Manual Vol2 under CPUID. |
223 | | */ |
224 | | static inline u32 phys_pkg_id(u32 cpuid_apic, int index_msb) |
225 | 48 | { |
226 | 48 | return _phys_pkg_id(get_apic_id(), index_msb); |
227 | 48 | } |
228 | | |
229 | | /* Do minimum CPU detection early. |
230 | | Fields really needed: vendor, cpuid_level, family, model, mask, cache alignment. |
231 | | The others are not touched to avoid unwanted side effects. |
232 | | |
233 | | WARNING: this function is only called on the BP. Don't add code here |
234 | | that is supposed to run on all CPUs. */ |
235 | | static void __init early_cpu_detect(void) |
236 | 1 | { |
237 | 1 | struct cpuinfo_x86 *c = &boot_cpu_data; |
238 | 1 | u32 eax, ebx, ecx, edx; |
239 | 1 | |
240 | 1 | c->x86_cache_alignment = 32; |
241 | 1 | |
242 | 1 | /* Get vendor name */ |
243 | 1 | cpuid(0x00000000, &c->cpuid_level, &ebx, &ecx, &edx); |
244 | 1 | *(u32 *)&c->x86_vendor_id[0] = ebx; |
245 | 1 | *(u32 *)&c->x86_vendor_id[8] = ecx; |
246 | 1 | *(u32 *)&c->x86_vendor_id[4] = edx; |
247 | 1 | |
248 | 1 | c->x86_vendor = get_cpu_vendor(ebx, ecx, edx, gcv_host); |
249 | 1 | |
250 | 1 | cpuid(0x00000001, &eax, &ebx, &ecx, &edx); |
251 | 1 | c->x86 = get_cpu_family(eax, &c->x86_model, &c->x86_mask); |
252 | 1 | |
253 | 1 | edx &= ~cleared_caps[cpufeat_word(X86_FEATURE_FPU)]; |
254 | 1 | ecx &= ~cleared_caps[cpufeat_word(X86_FEATURE_SSE3)]; |
255 | 1 | if (edx & cpufeat_mask(X86_FEATURE_CLFLUSH)) |
256 | 1 | c->x86_cache_alignment = ((ebx >> 8) & 0xff) * 8; |
257 | 1 | /* Leaf 0x1 capabilities filled in early for Xen. */ |
258 | 1 | c->x86_capability[cpufeat_word(X86_FEATURE_FPU)] = edx; |
259 | 1 | c->x86_capability[cpufeat_word(X86_FEATURE_SSE3)] = ecx; |
260 | 1 | |
261 | 1 | printk(XENLOG_INFO |
262 | 1 | "CPU Vendor: %s, Family %u (%#x), Model %u (%#x), Stepping %u (raw %08x)\n", |
263 | 1 | this_cpu->c_vendor, c->x86, c->x86, |
264 | 1 | c->x86_model, c->x86_model, c->x86_mask, eax); |
265 | 1 | |
266 | 1 | eax = cpuid_eax(0x80000000); |
267 | 1 | if ((eax >> 16) == 0x8000 && eax >= 0x80000008) { |
268 | 1 | eax = cpuid_eax(0x80000008); |
269 | 1 | paddr_bits = eax & 0xff; |
270 | 1 | if (paddr_bits > PADDR_BITS) |
271 | 0 | paddr_bits = PADDR_BITS; |
272 | 1 | vaddr_bits = (eax >> 8) & 0xff; |
273 | 1 | if (vaddr_bits > VADDR_BITS) |
274 | 0 | vaddr_bits = VADDR_BITS; |
275 | 1 | hap_paddr_bits = ((eax >> 16) & 0xff) ?: paddr_bits; |
276 | 1 | if (hap_paddr_bits > PADDR_BITS) |
277 | 0 | hap_paddr_bits = PADDR_BITS; |
278 | 1 | } |
279 | 1 | |
280 | 1 | initialize_cpu_data(0); |
281 | 1 | } |
282 | | |
283 | | static void generic_identify(struct cpuinfo_x86 *c) |
284 | 12 | { |
285 | 12 | u32 eax, ebx, ecx, edx, tmp; |
286 | 12 | |
287 | 12 | /* Get vendor name */ |
288 | 12 | cpuid(0x00000000, &c->cpuid_level, &ebx, &ecx, &edx); |
289 | 12 | *(u32 *)&c->x86_vendor_id[0] = ebx; |
290 | 12 | *(u32 *)&c->x86_vendor_id[8] = ecx; |
291 | 12 | *(u32 *)&c->x86_vendor_id[4] = edx; |
292 | 12 | |
293 | 12 | c->x86_vendor = get_cpu_vendor(ebx, ecx, edx, gcv_host); |
294 | 12 | /* Initialize the standard set of capabilities */ |
295 | 12 | /* Note that the vendor-specific code below might override */ |
296 | 12 | |
297 | 12 | /* Model and family information. */ |
298 | 12 | cpuid(0x00000001, &eax, &ebx, &ecx, &edx); |
299 | 12 | c->x86 = get_cpu_family(eax, &c->x86_model, &c->x86_mask); |
300 | 12 | c->apicid = phys_pkg_id((ebx >> 24) & 0xFF, 0); |
301 | 12 | c->phys_proc_id = c->apicid; |
302 | 12 | |
303 | 12 | if (this_cpu->c_early_init) |
304 | 12 | this_cpu->c_early_init(c); |
305 | 12 | |
306 | 12 | /* c_early_init() may have adjusted cpuid levels/features. Reread. */ |
307 | 12 | c->cpuid_level = cpuid_eax(0); |
308 | 12 | cpuid(0x00000001, &eax, &ebx, &ecx, &edx); |
309 | 12 | c->x86_capability[cpufeat_word(X86_FEATURE_FPU)] = edx; |
310 | 12 | c->x86_capability[cpufeat_word(X86_FEATURE_SSE3)] = ecx; |
311 | 12 | |
312 | 12 | if ( cpu_has(c, X86_FEATURE_CLFLUSH) ) |
313 | 12 | c->x86_clflush_size = ((ebx >> 8) & 0xff) * 8; |
314 | 12 | |
315 | 12 | if ( (c->cpuid_level >= CPUID_PM_LEAF) && |
316 | 12 | (cpuid_ecx(CPUID_PM_LEAF) & CPUID6_ECX_APERFMPERF_CAPABILITY) ) |
317 | 12 | set_bit(X86_FEATURE_APERFMPERF, c->x86_capability); |
318 | 12 | |
319 | 12 | /* AMD-defined flags: level 0x80000001 */ |
320 | 12 | c->extended_cpuid_level = cpuid_eax(0x80000000); |
321 | 12 | if ((c->extended_cpuid_level >> 16) != 0x8000) |
322 | 0 | c->extended_cpuid_level = 0; |
323 | 12 | if (c->extended_cpuid_level > 0x80000000) |
324 | 12 | cpuid(0x80000001, &tmp, &tmp, |
325 | 12 | &c->x86_capability[cpufeat_word(X86_FEATURE_LAHF_LM)], |
326 | 12 | &c->x86_capability[cpufeat_word(X86_FEATURE_SYSCALL)]); |
327 | 12 | if (c == &boot_cpu_data) |
328 | 1 | bootsym(cpuid_ext_features) = |
329 | 1 | c->x86_capability[cpufeat_word(X86_FEATURE_NX)]; |
330 | 12 | |
331 | 12 | if (c->extended_cpuid_level >= 0x80000004) |
332 | 12 | get_model_name(c); /* Default name */ |
333 | 12 | if (c->extended_cpuid_level >= 0x80000007) |
334 | 12 | c->x86_capability[cpufeat_word(X86_FEATURE_ITSC)] |
335 | 12 | = cpuid_edx(0x80000007); |
336 | 12 | if (c->extended_cpuid_level >= 0x80000008) |
337 | 12 | c->x86_capability[cpufeat_word(X86_FEATURE_CLZERO)] |
338 | 12 | = cpuid_ebx(0x80000008); |
339 | 12 | |
340 | 12 | /* Intel-defined flags: level 0x00000007 */ |
341 | 12 | if ( c->cpuid_level >= 0x00000007 ) |
342 | 12 | cpuid_count(0x00000007, 0, &tmp, |
343 | 12 | &c->x86_capability[cpufeat_word(X86_FEATURE_FSGSBASE)], |
344 | 12 | &c->x86_capability[cpufeat_word(X86_FEATURE_PKU)], |
345 | 12 | &c->x86_capability[cpufeat_word(X86_FEATURE_AVX512_4VNNIW)]); |
346 | 12 | } |
347 | | |
348 | | /* |
349 | | * This does the hard work of actually picking apart the CPU stuff... |
350 | | */ |
351 | | void identify_cpu(struct cpuinfo_x86 *c) |
352 | 12 | { |
353 | 12 | int i; |
354 | 12 | |
355 | 12 | c->x86_cache_size = -1; |
356 | 12 | c->x86_vendor = X86_VENDOR_UNKNOWN; |
357 | 12 | c->cpuid_level = -1; /* CPUID not detected */ |
358 | 12 | c->x86_model = c->x86_mask = 0; /* So far unknown... */ |
359 | 12 | c->x86_vendor_id[0] = '\0'; /* Unset */ |
360 | 12 | c->x86_model_id[0] = '\0'; /* Unset */ |
361 | 12 | c->x86_max_cores = 1; |
362 | 12 | c->x86_num_siblings = 1; |
363 | 12 | c->x86_clflush_size = 0; |
364 | 12 | c->phys_proc_id = XEN_INVALID_SOCKET_ID; |
365 | 12 | c->cpu_core_id = XEN_INVALID_CORE_ID; |
366 | 12 | c->compute_unit_id = INVALID_CUID; |
367 | 12 | memset(&c->x86_capability, 0, sizeof c->x86_capability); |
368 | 12 | |
369 | 12 | generic_identify(c); |
370 | 12 | |
371 | 12 | #ifdef NOISY_CAPS |
372 | | printk(KERN_DEBUG "CPU: After vendor identify, caps:"); |
373 | | for (i = 0; i < NCAPINTS; i++) |
374 | | printk(" %08x", c->x86_capability[i]); |
375 | | printk("\n"); |
376 | | #endif |
377 | 12 | |
378 | 12 | /* |
379 | 12 | * Vendor-specific initialization. In this section we |
380 | 12 | * canonicalize the feature flags, meaning if there are |
381 | 12 | * features a certain CPU supports which CPUID doesn't |
382 | 12 | * tell us, CPUID claiming incorrect flags, or other bugs, |
383 | 12 | * we handle them here. |
384 | 12 | * |
385 | 12 | * At the end of this section, c->x86_capability better |
386 | 12 | * indicate the features this CPU genuinely supports! |
387 | 12 | */ |
388 | 12 | if (this_cpu->c_init) |
389 | 12 | this_cpu->c_init(c); |
390 | 12 | |
391 | 12 | |
392 | 12 | if ( !opt_pku ) |
393 | 0 | setup_clear_cpu_cap(X86_FEATURE_PKU); |
394 | 12 | |
395 | 12 | /* |
396 | 12 | * The vendor-specific functions might have changed features. Now |
397 | 12 | * we do "generic changes." |
398 | 12 | */ |
399 | 132 | for (i = 0; i < FSCAPINTS; ++i) |
400 | 120 | c->x86_capability[i] &= known_features[i]; |
401 | 12 | |
402 | 144 | for (i = 0 ; i < NCAPINTS ; ++i) { |
403 | 132 | c->x86_capability[i] |= forced_caps[i]; |
404 | 132 | c->x86_capability[i] &= ~cleared_caps[i]; |
405 | 132 | } |
406 | 12 | |
407 | 12 | /* If the model name is still unset, do table lookup. */ |
408 | 12 | if ( !c->x86_model_id[0] ) { |
409 | 0 | /* Last resort... */ |
410 | 0 | snprintf(c->x86_model_id, sizeof(c->x86_model_id), |
411 | 0 | "%02x/%02x", c->x86_vendor, c->x86_model); |
412 | 0 | } |
413 | 12 | |
414 | 12 | /* Now the feature flags better reflect actual CPU features! */ |
415 | 12 | |
416 | 12 | if ( cpu_has_xsave ) |
417 | 12 | xstate_init(c); |
418 | 12 | |
419 | 12 | #ifdef NOISY_CAPS |
420 | | printk(KERN_DEBUG "CPU: After all inits, caps:"); |
421 | | for (i = 0; i < NCAPINTS; i++) |
422 | | printk(" %08x", c->x86_capability[i]); |
423 | | printk("\n"); |
424 | | #endif |
425 | 12 | |
426 | 12 | /* |
427 | 12 | * On SMP, boot_cpu_data holds the common feature set between |
428 | 12 | * all CPUs; so make sure that we indicate which features are |
429 | 12 | * common between the CPUs. The first time this routine gets |
430 | 12 | * executed, c == &boot_cpu_data. |
431 | 12 | */ |
432 | 12 | if ( c != &boot_cpu_data ) { |
433 | 11 | /* AND the already accumulated flags with these */ |
434 | 132 | for ( i = 0 ; i < NCAPINTS ; i++ ) |
435 | 121 | boot_cpu_data.x86_capability[i] &= c->x86_capability[i]; |
436 | 11 | |
437 | 11 | mcheck_init(c, false); |
438 | 1 | } else { |
439 | 1 | mcheck_init(c, true); |
440 | 1 | |
441 | 1 | mtrr_bp_init(); |
442 | 1 | } |
443 | 12 | } |
444 | | |
445 | | /* leaf 0xb SMT level */ |
446 | 13 | #define SMT_LEVEL 0 |
447 | | |
448 | | /* leaf 0xb sub-leaf types */ |
449 | 0 | #define INVALID_TYPE 0 |
450 | 12 | #define SMT_TYPE 1 |
451 | 13 | #define CORE_TYPE 2 |
452 | | |
453 | 25 | #define LEAFB_SUBTYPE(ecx) (((ecx) >> 8) & 0xff) |
454 | 26 | #define BITS_SHIFT_NEXT_LEVEL(eax) ((eax) & 0x1f) |
455 | 24 | #define LEVEL_MAX_SIBLINGS(ebx) ((ebx) & 0xffff) |
456 | | |
457 | | /* |
458 | | * Check for extended topology enumeration cpuid leaf 0xb and if it |
459 | | * exists, use it for cpu topology detection. |
460 | | */ |
461 | | void detect_extended_topology(struct cpuinfo_x86 *c) |
462 | 12 | { |
463 | 12 | unsigned int eax, ebx, ecx, edx, sub_index; |
464 | 12 | unsigned int ht_mask_width, core_plus_mask_width; |
465 | 12 | unsigned int core_select_mask, core_level_siblings; |
466 | 12 | unsigned int initial_apicid; |
467 | 12 | |
468 | 12 | if ( c->cpuid_level < 0xb ) |
469 | 0 | return; |
470 | 12 | |
471 | 12 | cpuid_count(0xb, SMT_LEVEL, &eax, &ebx, &ecx, &edx); |
472 | 12 | |
473 | 12 | /* Check if the cpuid leaf 0xb is actually implemented */ |
474 | 12 | if ( ebx == 0 || (LEAFB_SUBTYPE(ecx) != SMT_TYPE) ) |
475 | 0 | return; |
476 | 12 | |
477 | 12 | __set_bit(X86_FEATURE_XTOPOLOGY, c->x86_capability); |
478 | 12 | |
479 | 12 | initial_apicid = edx; |
480 | 12 | |
481 | 12 | /* Populate HT related information from sub-leaf level 0 */ |
482 | 12 | core_level_siblings = c->x86_num_siblings = LEVEL_MAX_SIBLINGS(ebx); |
483 | 12 | core_plus_mask_width = ht_mask_width = BITS_SHIFT_NEXT_LEVEL(eax); |
484 | 12 | |
485 | 12 | sub_index = 1; |
486 | 12 | do { |
487 | 12 | cpuid_count(0xb, sub_index, &eax, &ebx, &ecx, &edx); |
488 | 12 | |
489 | 12 | /* Check for the Core type in the implemented sub leaves */ |
490 | 12 | if ( LEAFB_SUBTYPE(ecx) == CORE_TYPE ) { |
491 | 12 | core_level_siblings = LEVEL_MAX_SIBLINGS(ebx); |
492 | 12 | core_plus_mask_width = BITS_SHIFT_NEXT_LEVEL(eax); |
493 | 12 | break; |
494 | 12 | } |
495 | 12 | |
496 | 0 | sub_index++; |
497 | 0 | } while ( LEAFB_SUBTYPE(ecx) != INVALID_TYPE ); |
498 | 12 | |
499 | 12 | core_select_mask = (~(~0u << core_plus_mask_width)) >> ht_mask_width; |
500 | 12 | |
501 | 12 | c->cpu_core_id = phys_pkg_id(initial_apicid, ht_mask_width) |
502 | 12 | & core_select_mask; |
503 | 12 | c->phys_proc_id = phys_pkg_id(initial_apicid, core_plus_mask_width); |
504 | 12 | |
505 | 12 | c->apicid = phys_pkg_id(initial_apicid, 0); |
506 | 12 | c->x86_max_cores = (core_level_siblings / c->x86_num_siblings); |
507 | 12 | |
508 | 12 | if ( opt_cpu_info ) |
509 | 0 | { |
510 | 0 | printk("CPU: Physical Processor ID: %d\n", |
511 | 0 | c->phys_proc_id); |
512 | 0 | if ( c->x86_max_cores > 1 ) |
513 | 0 | printk("CPU: Processor Core ID: %d\n", |
514 | 0 | c->cpu_core_id); |
515 | 0 | } |
516 | 12 | } |
517 | | |
518 | | void detect_ht(struct cpuinfo_x86 *c) |
519 | 0 | { |
520 | 0 | u32 eax, ebx, ecx, edx; |
521 | 0 | int index_msb, core_bits; |
522 | 0 |
|
523 | 0 | if (!cpu_has(c, X86_FEATURE_HTT) || |
524 | 0 | cpu_has(c, X86_FEATURE_CMP_LEGACY) || |
525 | 0 | cpu_has(c, X86_FEATURE_XTOPOLOGY)) |
526 | 0 | return; |
527 | 0 |
|
528 | 0 | cpuid(1, &eax, &ebx, &ecx, &edx); |
529 | 0 | c->x86_num_siblings = (ebx & 0xff0000) >> 16; |
530 | 0 |
|
531 | 0 | if (c->x86_num_siblings == 1) { |
532 | 0 | printk(KERN_INFO "CPU: Hyper-Threading is disabled\n"); |
533 | 0 | } else if (c->x86_num_siblings > 1 ) { |
534 | 0 | index_msb = get_count_order(c->x86_num_siblings); |
535 | 0 | c->phys_proc_id = phys_pkg_id((ebx >> 24) & 0xFF, index_msb); |
536 | 0 |
|
537 | 0 | if (opt_cpu_info) |
538 | 0 | printk("CPU: Physical Processor ID: %d\n", |
539 | 0 | c->phys_proc_id); |
540 | 0 |
|
541 | 0 | c->x86_num_siblings = c->x86_num_siblings / c->x86_max_cores; |
542 | 0 |
|
543 | 0 | index_msb = get_count_order(c->x86_num_siblings) ; |
544 | 0 |
|
545 | 0 | core_bits = get_count_order(c->x86_max_cores); |
546 | 0 |
|
547 | 0 | c->cpu_core_id = phys_pkg_id((ebx >> 24) & 0xFF, index_msb) & |
548 | 0 | ((1 << core_bits) - 1); |
549 | 0 |
|
550 | 0 | if (opt_cpu_info && c->x86_max_cores > 1) |
551 | 0 | printk("CPU: Processor Core ID: %d\n", |
552 | 0 | c->cpu_core_id); |
553 | 0 | } |
554 | 0 | } |
555 | | |
556 | | unsigned int __init apicid_to_socket(unsigned int apicid) |
557 | 1 | { |
558 | 1 | unsigned int dummy; |
559 | 1 | |
560 | 1 | if (boot_cpu_has(X86_FEATURE_XTOPOLOGY)) { |
561 | 1 | unsigned int eax, ecx, sub_index = 1, core_plus_mask_width; |
562 | 1 | |
563 | 1 | cpuid_count(0xb, SMT_LEVEL, &eax, &dummy, &dummy, &dummy); |
564 | 1 | core_plus_mask_width = BITS_SHIFT_NEXT_LEVEL(eax); |
565 | 1 | do { |
566 | 1 | cpuid_count(0xb, sub_index, &eax, &dummy, &ecx, |
567 | 1 | &dummy); |
568 | 1 | |
569 | 1 | if (LEAFB_SUBTYPE(ecx) == CORE_TYPE) { |
570 | 1 | core_plus_mask_width = |
571 | 1 | BITS_SHIFT_NEXT_LEVEL(eax); |
572 | 1 | break; |
573 | 1 | } |
574 | 1 | |
575 | 0 | sub_index++; |
576 | 0 | } while (LEAFB_SUBTYPE(ecx) != INVALID_TYPE); |
577 | 1 | |
578 | 1 | return _phys_pkg_id(apicid, core_plus_mask_width); |
579 | 1 | } |
580 | 1 | |
581 | 0 | if (boot_cpu_has(X86_FEATURE_HTT) && |
582 | 0 | !boot_cpu_has(X86_FEATURE_CMP_LEGACY)) { |
583 | 0 | unsigned int num_siblings = (cpuid_ebx(1) & 0xff0000) >> 16; |
584 | 0 |
|
585 | 0 | if (num_siblings) |
586 | 0 | return _phys_pkg_id(apicid, |
587 | 0 | get_count_order(num_siblings)); |
588 | 0 | } |
589 | 0 |
|
590 | 0 | return apicid; |
591 | 0 | } |
592 | | |
593 | | void print_cpu_info(unsigned int cpu) |
594 | 12 | { |
595 | 12 | const struct cpuinfo_x86 *c = cpu_data + cpu; |
596 | 12 | const char *vendor = NULL; |
597 | 12 | |
598 | 12 | if (!opt_cpu_info) |
599 | 12 | return; |
600 | 12 | |
601 | 0 | printk("CPU%u: ", cpu); |
602 | 0 |
|
603 | 0 | if (c->x86_vendor < X86_VENDOR_NUM) |
604 | 0 | vendor = this_cpu->c_vendor; |
605 | 0 | else |
606 | 0 | vendor = c->x86_vendor_id; |
607 | 0 |
|
608 | 0 | if (vendor && strncmp(c->x86_model_id, vendor, strlen(vendor))) |
609 | 0 | printk("%s ", vendor); |
610 | 0 |
|
611 | 0 | if (!c->x86_model_id[0]) |
612 | 0 | printk("%d86", c->x86); |
613 | 0 | else |
614 | 0 | printk("%s", c->x86_model_id); |
615 | 0 |
|
616 | 0 | printk(" stepping %02x\n", c->x86_mask); |
617 | 0 | } |
618 | | |
619 | | static cpumask_t cpu_initialized; |
620 | | |
621 | | /* This is hacky. :) |
622 | | * We're emulating future behavior. |
623 | | * In the future, the cpu-specific init functions will be called implicitly |
624 | | * via the magic of initcalls. |
625 | | * They will insert themselves into the cpu_devs structure. |
626 | | * Then, when cpu_init() is called, we can just iterate over that array. |
627 | | */ |
628 | | |
629 | | void __init early_cpu_init(void) |
630 | 1 | { |
631 | 1 | intel_cpu_init(); |
632 | 1 | amd_init_cpu(); |
633 | 1 | centaur_init_cpu(); |
634 | 1 | early_cpu_detect(); |
635 | 1 | } |
636 | | |
637 | | /* |
638 | | * Sets up system tables and descriptors. |
639 | | * |
640 | | * - Sets up TSS with stack pointers, including ISTs |
641 | | * - Inserts TSS selector into regular and compat GDTs |
642 | | * - Loads GDT, IDT, TR then null LDT |
643 | | * - Sets up IST references in the IDT |
644 | | */ |
645 | | void load_system_tables(void) |
646 | 13 | { |
647 | 13 | unsigned int cpu = smp_processor_id(); |
648 | 13 | unsigned long stack_bottom = get_stack_bottom(), |
649 | 13 | stack_top = stack_bottom & ~(STACK_SIZE - 1); |
650 | 13 | |
651 | 13 | struct tss_struct *tss = &this_cpu(init_tss); |
652 | 13 | struct desc_struct *gdt = |
653 | 13 | this_cpu(gdt_table) - FIRST_RESERVED_GDT_ENTRY; |
654 | 13 | struct desc_struct *compat_gdt = |
655 | 13 | this_cpu(compat_gdt_table) - FIRST_RESERVED_GDT_ENTRY; |
656 | 13 | |
657 | 13 | const struct desc_ptr gdtr = { |
658 | 13 | .base = (unsigned long)gdt, |
659 | 13 | .limit = LAST_RESERVED_GDT_BYTE, |
660 | 13 | }; |
661 | 13 | const struct desc_ptr idtr = { |
662 | 13 | .base = (unsigned long)idt_tables[cpu], |
663 | 13 | .limit = (IDT_ENTRIES * sizeof(idt_entry_t)) - 1, |
664 | 13 | }; |
665 | 13 | |
666 | 13 | *tss = (struct tss_struct){ |
667 | 13 | /* Main stack for interrupts/exceptions. */ |
668 | 13 | .rsp0 = stack_bottom, |
669 | 13 | |
670 | 13 | /* Ring 1 and 2 stacks poisoned. */ |
671 | 13 | .rsp1 = 0x8600111111111111ul, |
672 | 13 | .rsp2 = 0x8600111111111111ul, |
673 | 13 | |
674 | 13 | /* |
675 | 13 | * MCE, NMI and Double Fault handlers get their own stacks. |
676 | 13 | * All others poisoned. |
677 | 13 | */ |
678 | 13 | .ist = { |
679 | 13 | [IST_MCE - 1] = stack_top + IST_MCE * PAGE_SIZE, |
680 | 13 | [IST_DF - 1] = stack_top + IST_DF * PAGE_SIZE, |
681 | 13 | [IST_NMI - 1] = stack_top + IST_NMI * PAGE_SIZE, |
682 | 13 | |
683 | 13 | [IST_MAX ... ARRAY_SIZE(tss->ist) - 1] = |
684 | 13 | 0x8600111111111111ul, |
685 | 13 | }, |
686 | 13 | |
687 | 13 | .bitmap = IOBMP_INVALID_OFFSET, |
688 | 13 | }; |
689 | 13 | |
690 | 13 | _set_tssldt_desc( |
691 | 13 | gdt + TSS_ENTRY, |
692 | 13 | (unsigned long)tss, |
693 | 13 | offsetof(struct tss_struct, __cacheline_filler) - 1, |
694 | 13 | SYS_DESC_tss_avail); |
695 | 13 | _set_tssldt_desc( |
696 | 13 | compat_gdt + TSS_ENTRY, |
697 | 13 | (unsigned long)tss, |
698 | 13 | offsetof(struct tss_struct, __cacheline_filler) - 1, |
699 | 13 | SYS_DESC_tss_busy); |
700 | 13 | |
701 | 13 | asm volatile ("lgdt %0" : : "m" (gdtr) ); |
702 | 13 | asm volatile ("lidt %0" : : "m" (idtr) ); |
703 | 13 | asm volatile ("ltr %w0" : : "rm" (TSS_ENTRY << 3) ); |
704 | 13 | asm volatile ("lldt %w0" : : "rm" (0) ); |
705 | 13 | |
706 | 13 | set_ist(&idt_tables[cpu][TRAP_double_fault], IST_DF); |
707 | 13 | set_ist(&idt_tables[cpu][TRAP_nmi], IST_NMI); |
708 | 13 | set_ist(&idt_tables[cpu][TRAP_machine_check], IST_MCE); |
709 | 13 | |
710 | 13 | /* |
711 | 13 | * Bottom-of-stack must be 16-byte aligned! |
712 | 13 | * |
713 | 13 | * Defer checks until exception support is sufficiently set up. |
714 | 13 | */ |
715 | 13 | BUILD_BUG_ON((sizeof(struct cpu_info) - |
716 | 13 | offsetof(struct cpu_info, guest_cpu_user_regs.es)) & 0xf); |
717 | 13 | BUG_ON(system_state != SYS_STATE_early_boot && (stack_bottom & 0xf)); |
718 | 13 | } |
719 | | |
720 | | /* |
721 | | * cpu_init() initializes state that is per-CPU. Some data is already |
722 | | * initialized (naturally) in the bootstrap process, such as the GDT |
723 | | * and IDT. We reload them nevertheless, this function acts as a |
724 | | * 'CPU state barrier', nothing should get across. |
725 | | */ |
726 | | void cpu_init(void) |
727 | 12 | { |
728 | 12 | int cpu = smp_processor_id(); |
729 | 12 | |
730 | 12 | if (cpumask_test_and_set_cpu(cpu, &cpu_initialized)) { |
731 | 0 | printk(KERN_WARNING "CPU#%d already initialized!\n", cpu); |
732 | 0 | for (;;) local_irq_enable(); |
733 | 0 | } |
734 | 12 | if (opt_cpu_info) |
735 | 0 | printk("Initializing CPU#%d\n", cpu); |
736 | 12 | |
737 | 12 | if (cpu_has_pat) |
738 | 12 | wrmsrl(MSR_IA32_CR_PAT, host_pat); |
739 | 12 | |
740 | 12 | /* Install correct page table. */ |
741 | 12 | write_ptbase(current); |
742 | 12 | |
743 | 12 | /* Ensure FPU gets initialised for each domain. */ |
744 | 12 | stts(); |
745 | 12 | |
746 | 12 | /* Clear all 6 debug registers: */ |
747 | 72 | #define CD(register) asm volatile ( "mov %0,%%db" #register : : "r"(0UL) ); |
748 | 12 | CD(0); CD(1); CD(2); CD(3); /* no db4 and db5 */; CD(6); CD(7); |
749 | 12 | #undef CD |
750 | 12 | } |
751 | | |
752 | | void cpu_uninit(unsigned int cpu) |
753 | 0 | { |
754 | 0 | cpumask_clear_cpu(cpu, &cpu_initialized); |
755 | 0 | } |
756 | | |
757 | | /* |
758 | | * x86_match_cpu - match the current CPU against an array of |
759 | | * x86_cpu_ids |
760 | | * @match: Pointer to array of x86_cpu_ids. Last entry terminated with |
761 | | * {}. |
762 | | * Return the entry if the current CPU matches the entries in the |
763 | | * passed x86_cpu_id match table. Otherwise NULL. The match table |
764 | | * contains vendor (X86_VENDOR_*), family, model and feature bits or |
765 | | * respective wildcard entries. |
766 | | * |
767 | | * A typical table entry would be to match a specific CPU |
768 | | * { X86_VENDOR_INTEL, 6, 0x12 } |
769 | | * or to match a specific CPU feature |
770 | | * { X86_FEATURE_MATCH(X86_FEATURE_FOOBAR) } |
771 | | * |
772 | | * This always matches against the boot cpu, assuming models and |
773 | | features are |
774 | | * consistent over all CPUs. |
775 | | */ |
776 | | const struct x86_cpu_id *x86_match_cpu(const struct x86_cpu_id table[]) |
777 | 1 | { |
778 | 1 | const struct x86_cpu_id *m; |
779 | 1 | const struct cpuinfo_x86 *c = &boot_cpu_data; |
780 | 1 | |
781 | 17 | for (m = table; m->vendor | m->family | m->model | m->feature; m++) { |
782 | 17 | if (c->x86_vendor != m->vendor) |
783 | 0 | continue; |
784 | 17 | if (c->x86 != m->family) |
785 | 0 | continue; |
786 | 17 | if (c->x86_model != m->model) |
787 | 16 | continue; |
788 | 1 | if (!cpu_has(c, m->feature)) |
789 | 0 | continue; |
790 | 1 | return m; |
791 | 1 | } |
792 | 0 | return NULL; |
793 | 1 | } |