/root/src/xen/xen/arch/x86/cpu/vpmu.c
Line | Count | Source (jump to first uncovered line) |
1 | | /* |
2 | | * vpmu.c: PMU virtualization for HVM domain. |
3 | | * |
4 | | * Copyright (c) 2007, Intel Corporation. |
5 | | * |
6 | | * This program is free software; you can redistribute it and/or modify it |
7 | | * under the terms and conditions of the GNU General Public License, |
8 | | * version 2, as published by the Free Software Foundation. |
9 | | * |
10 | | * This program is distributed in the hope it will be useful, but WITHOUT |
11 | | * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or |
12 | | * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for |
13 | | * more details. |
14 | | * |
15 | | * You should have received a copy of the GNU General Public License along with |
16 | | * this program; If not, see <http://www.gnu.org/licenses/>. |
17 | | * |
18 | | * Author: Haitao Shan <haitao.shan@intel.com> |
19 | | */ |
20 | | #include <xen/sched.h> |
21 | | #include <xen/xenoprof.h> |
22 | | #include <xen/event.h> |
23 | | #include <xen/guest_access.h> |
24 | | #include <xen/cpu.h> |
25 | | #include <asm/regs.h> |
26 | | #include <asm/types.h> |
27 | | #include <asm/msr.h> |
28 | | #include <asm/nmi.h> |
29 | | #include <asm/p2m.h> |
30 | | #include <asm/vpmu.h> |
31 | | #include <asm/hvm/support.h> |
32 | | #include <asm/hvm/vmx/vmx.h> |
33 | | #include <asm/hvm/vmx/vmcs.h> |
34 | | #include <asm/hvm/svm/svm.h> |
35 | | #include <asm/hvm/svm/vmcb.h> |
36 | | #include <asm/apic.h> |
37 | | #include <public/pmu.h> |
38 | | #include <xsm/xsm.h> |
39 | | |
40 | | #include <compat/pmu.h> |
41 | | CHECK_pmu_cntr_pair; |
42 | | CHECK_pmu_data; |
43 | | CHECK_pmu_params; |
44 | | |
45 | | /* |
46 | | * "vpmu" : vpmu generally enabled (all counters) |
47 | | * "vpmu=off" : vpmu generally disabled |
48 | | * "vpmu=bts" : vpmu enabled and Intel BTS feature switched on. |
49 | | * "vpmu=ipc" : vpmu enabled for IPC counters only (most restrictive) |
50 | | * "vpmu=arch" : vpmu enabled for predef arch counters only (restrictive) |
51 | | * flag combinations are allowed, eg, "vpmu=ipc,bts". |
52 | | */ |
53 | | static unsigned int __read_mostly opt_vpmu_enabled; |
54 | | unsigned int __read_mostly vpmu_mode = XENPMU_MODE_OFF; |
55 | | unsigned int __read_mostly vpmu_features = 0; |
56 | | static int parse_vpmu_params(const char *s); |
57 | | custom_param("vpmu", parse_vpmu_params); |
58 | | |
59 | | static DEFINE_SPINLOCK(vpmu_lock); |
60 | | static unsigned vpmu_count; |
61 | | |
62 | | static DEFINE_PER_CPU(struct vcpu *, last_vcpu); |
63 | | |
64 | | static int parse_vpmu_param(const char *s, unsigned int len) |
65 | 0 | { |
66 | 0 | if ( !*s || !len ) |
67 | 0 | return 0; |
68 | 0 | if ( !strncmp(s, "bts", len) ) |
69 | 0 | vpmu_features |= XENPMU_FEATURE_INTEL_BTS; |
70 | 0 | else if ( !strncmp(s, "ipc", len) ) |
71 | 0 | vpmu_features |= XENPMU_FEATURE_IPC_ONLY; |
72 | 0 | else if ( !strncmp(s, "arch", len) ) |
73 | 0 | vpmu_features |= XENPMU_FEATURE_ARCH_ONLY; |
74 | 0 | else |
75 | 0 | return 1; |
76 | 0 | return 0; |
77 | 0 | } |
78 | | |
79 | | static int __init parse_vpmu_params(const char *s) |
80 | 0 | { |
81 | 0 | const char *sep, *p = s; |
82 | 0 |
|
83 | 0 | switch ( parse_bool(s, NULL) ) |
84 | 0 | { |
85 | 0 | case 0: |
86 | 0 | break; |
87 | 0 | default: |
88 | 0 | for ( ; ; ) |
89 | 0 | { |
90 | 0 | sep = strchr(p, ','); |
91 | 0 | if ( sep == NULL ) |
92 | 0 | sep = strchr(p, 0); |
93 | 0 | if ( parse_vpmu_param(p, sep - p) ) |
94 | 0 | goto error; |
95 | 0 | if ( !*sep ) |
96 | 0 | /* reached end of flags */ |
97 | 0 | break; |
98 | 0 | p = sep + 1; |
99 | 0 | } |
100 | 0 | /* fall through */ |
101 | 0 | case 1: |
102 | 0 | /* Default VPMU mode */ |
103 | 0 | vpmu_mode = XENPMU_MODE_SELF; |
104 | 0 | opt_vpmu_enabled = 1; |
105 | 0 | break; |
106 | 0 | } |
107 | 0 | return 0; |
108 | 0 |
|
109 | 0 | error: |
110 | 0 | printk("VPMU: unknown flags: %s - vpmu disabled!\n", s); |
111 | 0 | return -EINVAL; |
112 | 0 | } |
113 | | |
114 | | void vpmu_lvtpc_update(uint32_t val) |
115 | 12 | { |
116 | 12 | struct vpmu_struct *vpmu; |
117 | 12 | struct vcpu *curr = current; |
118 | 12 | |
119 | 12 | if ( likely(vpmu_mode == XENPMU_MODE_OFF) ) |
120 | 12 | return; |
121 | 12 | |
122 | 0 | vpmu = vcpu_vpmu(curr); |
123 | 0 |
|
124 | 0 | vpmu->hw_lapic_lvtpc = PMU_APIC_VECTOR | (val & APIC_LVT_MASKED); |
125 | 0 |
|
126 | 0 | /* Postpone APIC updates for PV(H) guests if PMU interrupt is pending */ |
127 | 0 | if ( has_vlapic(curr->domain) || !vpmu->xenpmu_data || |
128 | 0 | !vpmu_is_set(vpmu, VPMU_CACHED) ) |
129 | 0 | apic_write(APIC_LVTPC, vpmu->hw_lapic_lvtpc); |
130 | 0 | } |
131 | | |
132 | | int vpmu_do_msr(unsigned int msr, uint64_t *msr_content, |
133 | | uint64_t supported, bool_t is_write) |
134 | 12 | { |
135 | 12 | struct vcpu *curr = current; |
136 | 12 | struct vpmu_struct *vpmu; |
137 | 12 | const struct arch_vpmu_ops *ops; |
138 | 12 | int ret = 0; |
139 | 12 | |
140 | 12 | /* |
141 | 12 | * Hide the PMU MSRs if vpmu is not configured, or the hardware domain is |
142 | 12 | * profiling the whole system. |
143 | 12 | */ |
144 | 12 | if ( likely(vpmu_mode == XENPMU_MODE_OFF) || |
145 | 0 | ((vpmu_mode & XENPMU_MODE_ALL) && |
146 | 0 | !is_hardware_domain(curr->domain)) ) |
147 | 12 | goto nop; |
148 | 12 | |
149 | 0 | vpmu = vcpu_vpmu(curr); |
150 | 0 | ops = vpmu->arch_vpmu_ops; |
151 | 0 | if ( !ops ) |
152 | 0 | goto nop; |
153 | 0 |
|
154 | 0 | if ( is_write && ops->do_wrmsr ) |
155 | 0 | ret = ops->do_wrmsr(msr, *msr_content, supported); |
156 | 0 | else if ( !is_write && ops->do_rdmsr ) |
157 | 0 | ret = ops->do_rdmsr(msr, msr_content); |
158 | 0 | else |
159 | 0 | goto nop; |
160 | 0 |
|
161 | 0 | /* |
162 | 0 | * We may have received a PMU interrupt while handling MSR access |
163 | 0 | * and since do_wr/rdmsr may load VPMU context we should save |
164 | 0 | * (and unload) it again. |
165 | 0 | */ |
166 | 0 | if ( !has_vlapic(curr->domain) && vpmu->xenpmu_data && |
167 | 0 | vpmu_is_set(vpmu, VPMU_CACHED) ) |
168 | 0 | { |
169 | 0 | vpmu_set(vpmu, VPMU_CONTEXT_SAVE); |
170 | 0 | ops->arch_vpmu_save(curr, 0); |
171 | 0 | vpmu_reset(vpmu, VPMU_CONTEXT_SAVE | VPMU_CONTEXT_LOADED); |
172 | 0 | } |
173 | 0 |
|
174 | 0 | return ret; |
175 | 0 |
|
176 | 12 | nop: |
177 | 12 | if ( !is_write && (msr != MSR_IA32_MISC_ENABLE) ) |
178 | 0 | *msr_content = 0; |
179 | 12 | |
180 | 12 | return 0; |
181 | 0 | } |
182 | | |
183 | | static inline struct vcpu *choose_hwdom_vcpu(void) |
184 | 0 | { |
185 | 0 | unsigned idx; |
186 | 0 |
|
187 | 0 | if ( hardware_domain->max_vcpus == 0 ) |
188 | 0 | return NULL; |
189 | 0 |
|
190 | 0 | idx = smp_processor_id() % hardware_domain->max_vcpus; |
191 | 0 |
|
192 | 0 | return hardware_domain->vcpu[idx]; |
193 | 0 | } |
194 | | |
195 | | void vpmu_do_interrupt(struct cpu_user_regs *regs) |
196 | 0 | { |
197 | 0 | struct vcpu *sampled = current, *sampling; |
198 | 0 | struct vpmu_struct *vpmu; |
199 | 0 | struct vlapic *vlapic; |
200 | 0 | u32 vlapic_lvtpc; |
201 | 0 |
|
202 | 0 | /* |
203 | 0 | * dom0 will handle interrupt for special domains (e.g. idle domain) or, |
204 | 0 | * in XENPMU_MODE_ALL, for everyone. |
205 | 0 | */ |
206 | 0 | if ( (vpmu_mode & XENPMU_MODE_ALL) || |
207 | 0 | (sampled->domain->domain_id >= DOMID_FIRST_RESERVED) ) |
208 | 0 | { |
209 | 0 | sampling = choose_hwdom_vcpu(); |
210 | 0 | if ( !sampling ) |
211 | 0 | return; |
212 | 0 | } |
213 | 0 | else |
214 | 0 | sampling = sampled; |
215 | 0 |
|
216 | 0 | vpmu = vcpu_vpmu(sampling); |
217 | 0 | if ( !vpmu->arch_vpmu_ops ) |
218 | 0 | return; |
219 | 0 |
|
220 | 0 | /* PV(H) guest */ |
221 | 0 | if ( !has_vlapic(sampling->domain) || (vpmu_mode & XENPMU_MODE_ALL) ) |
222 | 0 | { |
223 | 0 | const struct cpu_user_regs *cur_regs; |
224 | 0 | uint64_t *flags = &vpmu->xenpmu_data->pmu.pmu_flags; |
225 | 0 | domid_t domid; |
226 | 0 |
|
227 | 0 | if ( !vpmu->xenpmu_data ) |
228 | 0 | return; |
229 | 0 |
|
230 | 0 | if ( !(vpmu_mode & XENPMU_MODE_ALL) && |
231 | 0 | !vpmu->arch_vpmu_ops->do_interrupt(regs) ) |
232 | 0 | return; |
233 | 0 |
|
234 | 0 | if ( vpmu_is_set(vpmu, VPMU_CACHED) ) |
235 | 0 | return; |
236 | 0 |
|
237 | 0 | /* PV guest will be reading PMU MSRs from xenpmu_data */ |
238 | 0 | vpmu_set(vpmu, VPMU_CONTEXT_SAVE | VPMU_CONTEXT_LOADED); |
239 | 0 | vpmu->arch_vpmu_ops->arch_vpmu_save(sampling, 1); |
240 | 0 | vpmu_reset(vpmu, VPMU_CONTEXT_SAVE | VPMU_CONTEXT_LOADED); |
241 | 0 |
|
242 | 0 | if ( is_hvm_vcpu(sampled) ) |
243 | 0 | *flags = 0; |
244 | 0 | else |
245 | 0 | *flags = PMU_SAMPLE_PV; |
246 | 0 |
|
247 | 0 | if ( sampled == sampling ) |
248 | 0 | domid = DOMID_SELF; |
249 | 0 | else |
250 | 0 | domid = sampled->domain->domain_id; |
251 | 0 |
|
252 | 0 | /* Store appropriate registers in xenpmu_data */ |
253 | 0 | /* FIXME: 32-bit PVH should go here as well */ |
254 | 0 | if ( is_pv_32bit_vcpu(sampling) ) |
255 | 0 | { |
256 | 0 | /* |
257 | 0 | * 32-bit dom0 cannot process Xen's addresses (which are 64 bit) |
258 | 0 | * and therefore we treat it the same way as a non-privileged |
259 | 0 | * PV 32-bit domain. |
260 | 0 | */ |
261 | 0 | struct compat_pmu_regs *cmp; |
262 | 0 |
|
263 | 0 | cur_regs = guest_cpu_user_regs(); |
264 | 0 |
|
265 | 0 | cmp = (void *)&vpmu->xenpmu_data->pmu.r.regs; |
266 | 0 | cmp->ip = cur_regs->rip; |
267 | 0 | cmp->sp = cur_regs->rsp; |
268 | 0 | cmp->flags = cur_regs->rflags; |
269 | 0 | cmp->ss = cur_regs->ss; |
270 | 0 | cmp->cs = cur_regs->cs; |
271 | 0 | if ( (cmp->cs & 3) > 1 ) |
272 | 0 | *flags |= PMU_SAMPLE_USER; |
273 | 0 | } |
274 | 0 | else |
275 | 0 | { |
276 | 0 | struct xen_pmu_regs *r = &vpmu->xenpmu_data->pmu.r.regs; |
277 | 0 |
|
278 | 0 | if ( (vpmu_mode & XENPMU_MODE_SELF) ) |
279 | 0 | cur_regs = guest_cpu_user_regs(); |
280 | 0 | else if ( !guest_mode(regs) && |
281 | 0 | is_hardware_domain(sampling->domain) ) |
282 | 0 | { |
283 | 0 | cur_regs = regs; |
284 | 0 | domid = DOMID_XEN; |
285 | 0 | } |
286 | 0 | else |
287 | 0 | cur_regs = guest_cpu_user_regs(); |
288 | 0 |
|
289 | 0 | r->ip = cur_regs->rip; |
290 | 0 | r->sp = cur_regs->rsp; |
291 | 0 | r->flags = cur_regs->rflags; |
292 | 0 |
|
293 | 0 | if ( !is_hvm_vcpu(sampled) ) |
294 | 0 | { |
295 | 0 | r->ss = cur_regs->ss; |
296 | 0 | r->cs = cur_regs->cs; |
297 | 0 | if ( !(sampled->arch.flags & TF_kernel_mode) ) |
298 | 0 | *flags |= PMU_SAMPLE_USER; |
299 | 0 | } |
300 | 0 | else |
301 | 0 | { |
302 | 0 | struct segment_register seg; |
303 | 0 |
|
304 | 0 | hvm_get_segment_register(sampled, x86_seg_cs, &seg); |
305 | 0 | r->cs = seg.sel; |
306 | 0 | hvm_get_segment_register(sampled, x86_seg_ss, &seg); |
307 | 0 | r->ss = seg.sel; |
308 | 0 | r->cpl = seg.dpl; |
309 | 0 | if ( !(sampled->arch.hvm_vcpu.guest_cr[0] & X86_CR0_PE) ) |
310 | 0 | *flags |= PMU_SAMPLE_REAL; |
311 | 0 | } |
312 | 0 | } |
313 | 0 |
|
314 | 0 | vpmu->xenpmu_data->domain_id = domid; |
315 | 0 | vpmu->xenpmu_data->vcpu_id = sampled->vcpu_id; |
316 | 0 | if ( is_hardware_domain(sampling->domain) ) |
317 | 0 | vpmu->xenpmu_data->pcpu_id = smp_processor_id(); |
318 | 0 | else |
319 | 0 | vpmu->xenpmu_data->pcpu_id = sampled->vcpu_id; |
320 | 0 |
|
321 | 0 | vpmu->hw_lapic_lvtpc |= APIC_LVT_MASKED; |
322 | 0 | apic_write(APIC_LVTPC, vpmu->hw_lapic_lvtpc); |
323 | 0 | *flags |= PMU_CACHED; |
324 | 0 | vpmu_set(vpmu, VPMU_CACHED); |
325 | 0 |
|
326 | 0 | send_guest_vcpu_virq(sampling, VIRQ_XENPMU); |
327 | 0 |
|
328 | 0 | return; |
329 | 0 | } |
330 | 0 |
|
331 | 0 | /* HVM guests */ |
332 | 0 | vlapic = vcpu_vlapic(sampling); |
333 | 0 |
|
334 | 0 | /* We don't support (yet) HVM dom0 */ |
335 | 0 | ASSERT(sampling == sampled); |
336 | 0 |
|
337 | 0 | if ( !vpmu->arch_vpmu_ops->do_interrupt(regs) || |
338 | 0 | !is_vlapic_lvtpc_enabled(vlapic) ) |
339 | 0 | return; |
340 | 0 |
|
341 | 0 | vlapic_lvtpc = vlapic_get_reg(vlapic, APIC_LVTPC); |
342 | 0 |
|
343 | 0 | switch ( GET_APIC_DELIVERY_MODE(vlapic_lvtpc) ) |
344 | 0 | { |
345 | 0 | case APIC_MODE_FIXED: |
346 | 0 | vlapic_set_irq(vlapic, vlapic_lvtpc & APIC_VECTOR_MASK, 0); |
347 | 0 | break; |
348 | 0 | case APIC_MODE_NMI: |
349 | 0 | sampling->nmi_pending = 1; |
350 | 0 | break; |
351 | 0 | } |
352 | 0 | } |
353 | | |
354 | | static void vpmu_save_force(void *arg) |
355 | 0 | { |
356 | 0 | struct vcpu *v = (struct vcpu *)arg; |
357 | 0 | struct vpmu_struct *vpmu = vcpu_vpmu(v); |
358 | 0 |
|
359 | 0 | if ( !vpmu_is_set(vpmu, VPMU_CONTEXT_LOADED) ) |
360 | 0 | return; |
361 | 0 |
|
362 | 0 | vpmu_set(vpmu, VPMU_CONTEXT_SAVE); |
363 | 0 |
|
364 | 0 | if ( vpmu->arch_vpmu_ops ) |
365 | 0 | (void)vpmu->arch_vpmu_ops->arch_vpmu_save(v, 0); |
366 | 0 |
|
367 | 0 | vpmu_reset(vpmu, VPMU_CONTEXT_SAVE); |
368 | 0 |
|
369 | 0 | per_cpu(last_vcpu, smp_processor_id()) = NULL; |
370 | 0 | } |
371 | | |
372 | | void vpmu_save(struct vcpu *v) |
373 | 0 | { |
374 | 0 | struct vpmu_struct *vpmu = vcpu_vpmu(v); |
375 | 0 | int pcpu = smp_processor_id(); |
376 | 0 |
|
377 | 0 | if ( !vpmu_are_all_set(vpmu, VPMU_CONTEXT_ALLOCATED | VPMU_CONTEXT_LOADED) ) |
378 | 0 | return; |
379 | 0 |
|
380 | 0 | vpmu->last_pcpu = pcpu; |
381 | 0 | per_cpu(last_vcpu, pcpu) = v; |
382 | 0 |
|
383 | 0 | if ( vpmu->arch_vpmu_ops ) |
384 | 0 | if ( vpmu->arch_vpmu_ops->arch_vpmu_save(v, 0) ) |
385 | 0 | vpmu_reset(vpmu, VPMU_CONTEXT_LOADED); |
386 | 0 |
|
387 | 0 | apic_write(APIC_LVTPC, PMU_APIC_VECTOR | APIC_LVT_MASKED); |
388 | 0 | } |
389 | | |
390 | | int vpmu_load(struct vcpu *v, bool_t from_guest) |
391 | 0 | { |
392 | 0 | struct vpmu_struct *vpmu = vcpu_vpmu(v); |
393 | 0 | int pcpu = smp_processor_id(); |
394 | 0 | struct vcpu *prev = NULL; |
395 | 0 |
|
396 | 0 | if ( !vpmu_is_set(vpmu, VPMU_CONTEXT_ALLOCATED) ) |
397 | 0 | return 0; |
398 | 0 |
|
399 | 0 | /* First time this VCPU is running here */ |
400 | 0 | if ( vpmu->last_pcpu != pcpu ) |
401 | 0 | { |
402 | 0 | /* |
403 | 0 | * Get the context from last pcpu that we ran on. Note that if another |
404 | 0 | * VCPU is running there it must have saved this VPCU's context before |
405 | 0 | * startig to run (see below). |
406 | 0 | * There should be no race since remote pcpu will disable interrupts |
407 | 0 | * before saving the context. |
408 | 0 | */ |
409 | 0 | if ( vpmu_is_set(vpmu, VPMU_CONTEXT_LOADED) ) |
410 | 0 | { |
411 | 0 | on_selected_cpus(cpumask_of(vpmu->last_pcpu), |
412 | 0 | vpmu_save_force, (void *)v, 1); |
413 | 0 | vpmu_reset(vpmu, VPMU_CONTEXT_LOADED); |
414 | 0 | } |
415 | 0 | } |
416 | 0 |
|
417 | 0 | /* Prevent forced context save from remote CPU */ |
418 | 0 | local_irq_disable(); |
419 | 0 |
|
420 | 0 | prev = per_cpu(last_vcpu, pcpu); |
421 | 0 |
|
422 | 0 | if ( prev != v && prev ) |
423 | 0 | { |
424 | 0 | vpmu = vcpu_vpmu(prev); |
425 | 0 |
|
426 | 0 | /* Someone ran here before us */ |
427 | 0 | vpmu_save_force(prev); |
428 | 0 | vpmu_reset(vpmu, VPMU_CONTEXT_LOADED); |
429 | 0 |
|
430 | 0 | vpmu = vcpu_vpmu(v); |
431 | 0 | } |
432 | 0 |
|
433 | 0 | local_irq_enable(); |
434 | 0 |
|
435 | 0 | /* Only when PMU is counting, we load PMU context immediately. */ |
436 | 0 | if ( !vpmu_is_set(vpmu, VPMU_RUNNING) || |
437 | 0 | (!has_vlapic(vpmu_vcpu(vpmu)->domain) && |
438 | 0 | vpmu_is_set(vpmu, VPMU_CACHED)) ) |
439 | 0 | return 0; |
440 | 0 |
|
441 | 0 | if ( vpmu->arch_vpmu_ops && vpmu->arch_vpmu_ops->arch_vpmu_load ) |
442 | 0 | { |
443 | 0 | int ret; |
444 | 0 |
|
445 | 0 | apic_write(APIC_LVTPC, vpmu->hw_lapic_lvtpc); |
446 | 0 | /* Arch code needs to set VPMU_CONTEXT_LOADED */ |
447 | 0 | ret = vpmu->arch_vpmu_ops->arch_vpmu_load(v, from_guest); |
448 | 0 | if ( ret ) |
449 | 0 | { |
450 | 0 | apic_write(APIC_LVTPC, vpmu->hw_lapic_lvtpc | APIC_LVT_MASKED); |
451 | 0 | return ret; |
452 | 0 | } |
453 | 0 | } |
454 | 0 |
|
455 | 0 | return 0; |
456 | 0 | } |
457 | | |
458 | | static int vpmu_arch_initialise(struct vcpu *v) |
459 | 12 | { |
460 | 12 | struct vpmu_struct *vpmu = vcpu_vpmu(v); |
461 | 12 | uint8_t vendor = current_cpu_data.x86_vendor; |
462 | 12 | int ret; |
463 | 12 | |
464 | 12 | BUILD_BUG_ON(sizeof(struct xen_pmu_intel_ctxt) > XENPMU_CTXT_PAD_SZ); |
465 | 12 | BUILD_BUG_ON(sizeof(struct xen_pmu_amd_ctxt) > XENPMU_CTXT_PAD_SZ); |
466 | 12 | BUILD_BUG_ON(sizeof(struct xen_pmu_regs) > XENPMU_REGS_PAD_SZ); |
467 | 12 | BUILD_BUG_ON(sizeof(struct compat_pmu_regs) > XENPMU_REGS_PAD_SZ); |
468 | 12 | |
469 | 12 | ASSERT(!(vpmu->flags & ~VPMU_AVAILABLE) && !vpmu->context); |
470 | 12 | |
471 | 12 | if ( !vpmu_available(v) ) |
472 | 12 | return 0; |
473 | 12 | |
474 | 0 | switch ( vendor ) |
475 | 0 | { |
476 | 0 | case X86_VENDOR_AMD: |
477 | 0 | ret = svm_vpmu_initialise(v); |
478 | 0 | break; |
479 | 0 |
|
480 | 0 | case X86_VENDOR_INTEL: |
481 | 0 | ret = vmx_vpmu_initialise(v); |
482 | 0 | break; |
483 | 0 |
|
484 | 0 | default: |
485 | 0 | if ( vpmu_mode != XENPMU_MODE_OFF ) |
486 | 0 | { |
487 | 0 | printk(XENLOG_G_WARNING "VPMU: Unknown CPU vendor %d. " |
488 | 0 | "Disabling VPMU\n", vendor); |
489 | 0 | opt_vpmu_enabled = 0; |
490 | 0 | vpmu_mode = XENPMU_MODE_OFF; |
491 | 0 | } |
492 | 0 | return -EINVAL; |
493 | 0 | } |
494 | 0 |
|
495 | 0 | vpmu->hw_lapic_lvtpc = PMU_APIC_VECTOR | APIC_LVT_MASKED; |
496 | 0 |
|
497 | 0 | if ( ret ) |
498 | 0 | printk(XENLOG_G_WARNING "VPMU: Initialization failed for %pv\n", v); |
499 | 0 |
|
500 | 0 | return ret; |
501 | 0 | } |
502 | | |
503 | | static void get_vpmu(struct vcpu *v) |
504 | 12 | { |
505 | 12 | spin_lock(&vpmu_lock); |
506 | 12 | |
507 | 12 | /* |
508 | 12 | * Keep count of VPMUs in the system so that we won't try to change |
509 | 12 | * vpmu_mode while a guest might be using one. |
510 | 12 | * vpmu_mode can be safely updated while dom0's VPMUs are active and |
511 | 12 | * so we don't need to include it in the count. |
512 | 12 | */ |
513 | 12 | if ( !is_hardware_domain(v->domain) && |
514 | 0 | (vpmu_mode & (XENPMU_MODE_SELF | XENPMU_MODE_HV)) ) |
515 | 0 | { |
516 | 0 | vpmu_count++; |
517 | 0 | vpmu_set(vcpu_vpmu(v), VPMU_AVAILABLE); |
518 | 0 | } |
519 | 12 | else if ( is_hardware_domain(v->domain) && |
520 | 12 | (vpmu_mode != XENPMU_MODE_OFF) ) |
521 | 0 | vpmu_set(vcpu_vpmu(v), VPMU_AVAILABLE); |
522 | 12 | |
523 | 12 | spin_unlock(&vpmu_lock); |
524 | 12 | } |
525 | | |
526 | | static void put_vpmu(struct vcpu *v) |
527 | 0 | { |
528 | 0 | spin_lock(&vpmu_lock); |
529 | 0 |
|
530 | 0 | if ( !vpmu_available(v) ) |
531 | 0 | goto out; |
532 | 0 |
|
533 | 0 | if ( !is_hardware_domain(v->domain) && |
534 | 0 | (vpmu_mode & (XENPMU_MODE_SELF | XENPMU_MODE_HV)) ) |
535 | 0 | { |
536 | 0 | vpmu_count--; |
537 | 0 | vpmu_reset(vcpu_vpmu(v), VPMU_AVAILABLE); |
538 | 0 | } |
539 | 0 | else if ( is_hardware_domain(v->domain) && |
540 | 0 | (vpmu_mode != XENPMU_MODE_OFF) ) |
541 | 0 | vpmu_reset(vcpu_vpmu(v), VPMU_AVAILABLE); |
542 | 0 |
|
543 | 0 | out: |
544 | 0 | spin_unlock(&vpmu_lock); |
545 | 0 | } |
546 | | |
547 | | void vpmu_initialise(struct vcpu *v) |
548 | 12 | { |
549 | 12 | get_vpmu(v); |
550 | 12 | |
551 | 12 | /* |
552 | 12 | * Guests without LAPIC (i.e. PV) call vpmu_arch_initialise() |
553 | 12 | * from pvpmu_init(). |
554 | 12 | */ |
555 | 12 | if ( has_vlapic(v->domain) && vpmu_arch_initialise(v) ) |
556 | 0 | put_vpmu(v); |
557 | 12 | } |
558 | | |
559 | | static void vpmu_clear_last(void *arg) |
560 | 0 | { |
561 | 0 | if ( this_cpu(last_vcpu) == arg ) |
562 | 0 | this_cpu(last_vcpu) = NULL; |
563 | 0 | } |
564 | | |
565 | | static void vpmu_arch_destroy(struct vcpu *v) |
566 | 0 | { |
567 | 0 | struct vpmu_struct *vpmu = vcpu_vpmu(v); |
568 | 0 |
|
569 | 0 | if ( !vpmu_is_set(vpmu, VPMU_CONTEXT_ALLOCATED) ) |
570 | 0 | return; |
571 | 0 |
|
572 | 0 | /* |
573 | 0 | * Need to clear last_vcpu in case it points to v. |
574 | 0 | * We can check here non-atomically whether it is 'v' since |
575 | 0 | * last_vcpu can never become 'v' again at this point. |
576 | 0 | * We will test it again in vpmu_clear_last() with interrupts |
577 | 0 | * disabled to make sure we don't clear someone else. |
578 | 0 | */ |
579 | 0 | if ( cpu_online(vpmu->last_pcpu) && |
580 | 0 | per_cpu(last_vcpu, vpmu->last_pcpu) == v ) |
581 | 0 | on_selected_cpus(cpumask_of(vpmu->last_pcpu), |
582 | 0 | vpmu_clear_last, v, 1); |
583 | 0 |
|
584 | 0 | if ( vpmu->arch_vpmu_ops && vpmu->arch_vpmu_ops->arch_vpmu_destroy ) |
585 | 0 | { |
586 | 0 | /* |
587 | 0 | * Unload VPMU first if VPMU_CONTEXT_LOADED being set. |
588 | 0 | * This will stop counters. |
589 | 0 | */ |
590 | 0 | if ( vpmu_is_set(vpmu, VPMU_CONTEXT_LOADED) ) |
591 | 0 | on_selected_cpus(cpumask_of(vcpu_vpmu(v)->last_pcpu), |
592 | 0 | vpmu_save_force, v, 1); |
593 | 0 |
|
594 | 0 | vpmu->arch_vpmu_ops->arch_vpmu_destroy(v); |
595 | 0 | } |
596 | 0 | } |
597 | | |
598 | | void vpmu_destroy(struct vcpu *v) |
599 | 0 | { |
600 | 0 | vpmu_arch_destroy(v); |
601 | 0 |
|
602 | 0 | put_vpmu(v); |
603 | 0 | } |
604 | | |
605 | | static int pvpmu_init(struct domain *d, xen_pmu_params_t *params) |
606 | 0 | { |
607 | 0 | struct vcpu *v; |
608 | 0 | struct vpmu_struct *vpmu; |
609 | 0 | struct page_info *page; |
610 | 0 | uint64_t gfn = params->val; |
611 | 0 |
|
612 | 0 | if ( (params->vcpu >= d->max_vcpus) || (d->vcpu[params->vcpu] == NULL) ) |
613 | 0 | return -EINVAL; |
614 | 0 |
|
615 | 0 | v = d->vcpu[params->vcpu]; |
616 | 0 | vpmu = vcpu_vpmu(v); |
617 | 0 |
|
618 | 0 | if ( !vpmu_available(v) ) |
619 | 0 | return -ENOENT; |
620 | 0 |
|
621 | 0 | page = get_page_from_gfn(d, gfn, NULL, P2M_ALLOC); |
622 | 0 | if ( !page ) |
623 | 0 | return -EINVAL; |
624 | 0 |
|
625 | 0 | if ( !get_page_type(page, PGT_writable_page) ) |
626 | 0 | { |
627 | 0 | put_page(page); |
628 | 0 | return -EINVAL; |
629 | 0 | } |
630 | 0 |
|
631 | 0 | spin_lock(&vpmu->vpmu_lock); |
632 | 0 |
|
633 | 0 | if ( v->arch.vpmu.xenpmu_data ) |
634 | 0 | { |
635 | 0 | spin_unlock(&vpmu->vpmu_lock); |
636 | 0 | put_page_and_type(page); |
637 | 0 | return -EEXIST; |
638 | 0 | } |
639 | 0 |
|
640 | 0 | v->arch.vpmu.xenpmu_data = __map_domain_page_global(page); |
641 | 0 | if ( !v->arch.vpmu.xenpmu_data ) |
642 | 0 | { |
643 | 0 | spin_unlock(&vpmu->vpmu_lock); |
644 | 0 | put_page_and_type(page); |
645 | 0 | return -ENOMEM; |
646 | 0 | } |
647 | 0 |
|
648 | 0 | if ( vpmu_arch_initialise(v) ) |
649 | 0 | put_vpmu(v); |
650 | 0 |
|
651 | 0 | spin_unlock(&vpmu->vpmu_lock); |
652 | 0 |
|
653 | 0 | return 0; |
654 | 0 | } |
655 | | |
656 | | static void pvpmu_finish(struct domain *d, xen_pmu_params_t *params) |
657 | 0 | { |
658 | 0 | struct vcpu *v; |
659 | 0 | struct vpmu_struct *vpmu; |
660 | 0 | uint64_t mfn; |
661 | 0 | void *xenpmu_data; |
662 | 0 |
|
663 | 0 | if ( (params->vcpu >= d->max_vcpus) || (d->vcpu[params->vcpu] == NULL) ) |
664 | 0 | return; |
665 | 0 |
|
666 | 0 | v = d->vcpu[params->vcpu]; |
667 | 0 | if ( v != current ) |
668 | 0 | vcpu_pause(v); |
669 | 0 |
|
670 | 0 | vpmu = vcpu_vpmu(v); |
671 | 0 | spin_lock(&vpmu->vpmu_lock); |
672 | 0 |
|
673 | 0 | vpmu_arch_destroy(v); |
674 | 0 | xenpmu_data = vpmu->xenpmu_data; |
675 | 0 | vpmu->xenpmu_data = NULL; |
676 | 0 |
|
677 | 0 | spin_unlock(&vpmu->vpmu_lock); |
678 | 0 |
|
679 | 0 | if ( xenpmu_data ) |
680 | 0 | { |
681 | 0 | mfn = domain_page_map_to_mfn(xenpmu_data); |
682 | 0 | ASSERT(mfn_valid(_mfn(mfn))); |
683 | 0 | unmap_domain_page_global(xenpmu_data); |
684 | 0 | put_page_and_type(mfn_to_page(mfn)); |
685 | 0 | } |
686 | 0 |
|
687 | 0 | if ( v != current ) |
688 | 0 | vcpu_unpause(v); |
689 | 0 | } |
690 | | |
691 | | /* Dump some vpmu informations on console. Used in keyhandler dump_domains(). */ |
692 | | void vpmu_dump(struct vcpu *v) |
693 | 0 | { |
694 | 0 | struct vpmu_struct *vpmu = vcpu_vpmu(v); |
695 | 0 |
|
696 | 0 | if ( vpmu->arch_vpmu_ops && vpmu->arch_vpmu_ops->arch_vpmu_dump ) |
697 | 0 | vpmu->arch_vpmu_ops->arch_vpmu_dump(v); |
698 | 0 | } |
699 | | |
700 | | long do_xenpmu_op(unsigned int op, XEN_GUEST_HANDLE_PARAM(xen_pmu_params_t) arg) |
701 | 0 | { |
702 | 0 | int ret; |
703 | 0 | struct vcpu *curr; |
704 | 0 | struct xen_pmu_params pmu_params = {.val = 0}; |
705 | 0 | struct xen_pmu_data *xenpmu_data; |
706 | 0 | struct vpmu_struct *vpmu; |
707 | 0 |
|
708 | 0 | if ( !opt_vpmu_enabled || has_vlapic(current->domain) ) |
709 | 0 | return -EOPNOTSUPP; |
710 | 0 |
|
711 | 0 | ret = xsm_pmu_op(XSM_OTHER, current->domain, op); |
712 | 0 | if ( ret ) |
713 | 0 | return ret; |
714 | 0 |
|
715 | 0 | /* Check major version when parameters are specified */ |
716 | 0 | switch ( op ) |
717 | 0 | { |
718 | 0 | case XENPMU_mode_set: |
719 | 0 | case XENPMU_feature_set: |
720 | 0 | case XENPMU_init: |
721 | 0 | case XENPMU_finish: |
722 | 0 | if ( copy_from_guest(&pmu_params, arg, 1) ) |
723 | 0 | return -EFAULT; |
724 | 0 |
|
725 | 0 | if ( pmu_params.version.maj != XENPMU_VER_MAJ ) |
726 | 0 | return -EINVAL; |
727 | 0 | } |
728 | 0 |
|
729 | 0 | switch ( op ) |
730 | 0 | { |
731 | 0 | case XENPMU_mode_set: |
732 | 0 | { |
733 | 0 | if ( (pmu_params.val & |
734 | 0 | ~(XENPMU_MODE_SELF | XENPMU_MODE_HV | XENPMU_MODE_ALL)) || |
735 | 0 | (hweight64(pmu_params.val) > 1) ) |
736 | 0 | return -EINVAL; |
737 | 0 |
|
738 | 0 | /* 32-bit dom0 can only sample itself. */ |
739 | 0 | if ( is_pv_32bit_vcpu(current) && |
740 | 0 | (pmu_params.val & (XENPMU_MODE_HV | XENPMU_MODE_ALL)) ) |
741 | 0 | return -EINVAL; |
742 | 0 |
|
743 | 0 | spin_lock(&vpmu_lock); |
744 | 0 |
|
745 | 0 | /* |
746 | 0 | * We can always safely switch between XENPMU_MODE_SELF and |
747 | 0 | * XENPMU_MODE_HV while other VPMUs are active. |
748 | 0 | */ |
749 | 0 | if ( (vpmu_count == 0) || |
750 | 0 | ((vpmu_mode ^ pmu_params.val) == |
751 | 0 | (XENPMU_MODE_SELF | XENPMU_MODE_HV)) ) |
752 | 0 | vpmu_mode = pmu_params.val; |
753 | 0 | else if ( vpmu_mode != pmu_params.val ) |
754 | 0 | { |
755 | 0 | gprintk(XENLOG_WARNING, |
756 | 0 | "VPMU: Cannot change mode while active VPMUs exist\n"); |
757 | 0 | ret = -EBUSY; |
758 | 0 | } |
759 | 0 |
|
760 | 0 | spin_unlock(&vpmu_lock); |
761 | 0 |
|
762 | 0 | break; |
763 | 0 | } |
764 | 0 |
|
765 | 0 | case XENPMU_mode_get: |
766 | 0 | memset(&pmu_params, 0, sizeof(pmu_params)); |
767 | 0 | pmu_params.val = vpmu_mode; |
768 | 0 |
|
769 | 0 | pmu_params.version.maj = XENPMU_VER_MAJ; |
770 | 0 | pmu_params.version.min = XENPMU_VER_MIN; |
771 | 0 |
|
772 | 0 | if ( copy_to_guest(arg, &pmu_params, 1) ) |
773 | 0 | ret = -EFAULT; |
774 | 0 |
|
775 | 0 | break; |
776 | 0 |
|
777 | 0 | case XENPMU_feature_set: |
778 | 0 | if ( pmu_params.val & ~(XENPMU_FEATURE_INTEL_BTS | |
779 | 0 | XENPMU_FEATURE_IPC_ONLY | |
780 | 0 | XENPMU_FEATURE_ARCH_ONLY)) |
781 | 0 | return -EINVAL; |
782 | 0 |
|
783 | 0 | spin_lock(&vpmu_lock); |
784 | 0 |
|
785 | 0 | if ( (vpmu_count == 0) || (vpmu_features == pmu_params.val) ) |
786 | 0 | vpmu_features = pmu_params.val; |
787 | 0 | else |
788 | 0 | { |
789 | 0 | gprintk(XENLOG_WARNING, |
790 | 0 | "VPMU: Cannot change features while active VPMUs exist\n"); |
791 | 0 | ret = -EBUSY; |
792 | 0 | } |
793 | 0 |
|
794 | 0 | spin_unlock(&vpmu_lock); |
795 | 0 |
|
796 | 0 | break; |
797 | 0 |
|
798 | 0 | case XENPMU_feature_get: |
799 | 0 | pmu_params.val = vpmu_features; |
800 | 0 | if ( copy_field_to_guest(arg, &pmu_params, val) ) |
801 | 0 | ret = -EFAULT; |
802 | 0 |
|
803 | 0 | break; |
804 | 0 |
|
805 | 0 | case XENPMU_init: |
806 | 0 | ret = pvpmu_init(current->domain, &pmu_params); |
807 | 0 | break; |
808 | 0 |
|
809 | 0 | case XENPMU_finish: |
810 | 0 | pvpmu_finish(current->domain, &pmu_params); |
811 | 0 | break; |
812 | 0 |
|
813 | 0 | case XENPMU_lvtpc_set: |
814 | 0 | xenpmu_data = current->arch.vpmu.xenpmu_data; |
815 | 0 | if ( xenpmu_data != NULL ) |
816 | 0 | vpmu_lvtpc_update(xenpmu_data->pmu.l.lapic_lvtpc); |
817 | 0 | else |
818 | 0 | ret = -EINVAL; |
819 | 0 | break; |
820 | 0 |
|
821 | 0 | case XENPMU_flush: |
822 | 0 | curr = current; |
823 | 0 | vpmu = vcpu_vpmu(curr); |
824 | 0 | xenpmu_data = curr->arch.vpmu.xenpmu_data; |
825 | 0 | if ( xenpmu_data == NULL ) |
826 | 0 | return -EINVAL; |
827 | 0 | xenpmu_data->pmu.pmu_flags &= ~PMU_CACHED; |
828 | 0 | vpmu_reset(vpmu, VPMU_CACHED); |
829 | 0 | vpmu_lvtpc_update(xenpmu_data->pmu.l.lapic_lvtpc); |
830 | 0 | if ( vpmu_load(curr, 1) ) |
831 | 0 | { |
832 | 0 | xenpmu_data->pmu.pmu_flags |= PMU_CACHED; |
833 | 0 | vpmu_set(vpmu, VPMU_CACHED); |
834 | 0 | ret = -EIO; |
835 | 0 | } |
836 | 0 | break ; |
837 | 0 |
|
838 | 0 | default: |
839 | 0 | ret = -EINVAL; |
840 | 0 | } |
841 | 0 |
|
842 | 0 | return ret; |
843 | 0 | } |
844 | | |
845 | | static int cpu_callback( |
846 | | struct notifier_block *nfb, unsigned long action, void *hcpu) |
847 | 0 | { |
848 | 0 | unsigned int cpu = (unsigned long)hcpu; |
849 | 0 | struct vcpu *vcpu = per_cpu(last_vcpu, cpu); |
850 | 0 | struct vpmu_struct *vpmu; |
851 | 0 |
|
852 | 0 | if ( !vcpu ) |
853 | 0 | return NOTIFY_DONE; |
854 | 0 |
|
855 | 0 | vpmu = vcpu_vpmu(vcpu); |
856 | 0 | if ( !vpmu_is_set(vpmu, VPMU_CONTEXT_ALLOCATED) ) |
857 | 0 | return NOTIFY_DONE; |
858 | 0 |
|
859 | 0 | if ( action == CPU_DYING ) |
860 | 0 | { |
861 | 0 | vpmu_save_force(vcpu); |
862 | 0 | vpmu_reset(vpmu, VPMU_CONTEXT_LOADED); |
863 | 0 | } |
864 | 0 |
|
865 | 0 | return NOTIFY_DONE; |
866 | 0 | } |
867 | | |
868 | | static struct notifier_block cpu_nfb = { |
869 | | .notifier_call = cpu_callback |
870 | | }; |
871 | | |
872 | | static int __init vpmu_init(void) |
873 | 1 | { |
874 | 1 | int vendor = current_cpu_data.x86_vendor; |
875 | 1 | |
876 | 1 | if ( !opt_vpmu_enabled ) |
877 | 1 | { |
878 | 1 | printk(XENLOG_INFO "VPMU: disabled\n"); |
879 | 1 | return 0; |
880 | 1 | } |
881 | 1 | |
882 | 1 | /* NMI watchdog uses LVTPC and HW counter */ |
883 | 0 | if ( opt_watchdog && opt_vpmu_enabled ) |
884 | 0 | { |
885 | 0 | printk(XENLOG_WARNING "NMI watchdog is enabled. Turning VPMU off.\n"); |
886 | 0 | opt_vpmu_enabled = 0; |
887 | 0 | vpmu_mode = XENPMU_MODE_OFF; |
888 | 0 | return 0; |
889 | 0 | } |
890 | 0 |
|
891 | 0 | switch ( vendor ) |
892 | 0 | { |
893 | 0 | case X86_VENDOR_AMD: |
894 | 0 | if ( amd_vpmu_init() ) |
895 | 0 | vpmu_mode = XENPMU_MODE_OFF; |
896 | 0 | break; |
897 | 0 | case X86_VENDOR_INTEL: |
898 | 0 | if ( core2_vpmu_init() ) |
899 | 0 | vpmu_mode = XENPMU_MODE_OFF; |
900 | 0 | break; |
901 | 0 | default: |
902 | 0 | printk(XENLOG_WARNING "VPMU: Unknown CPU vendor: %d. " |
903 | 0 | "Turning VPMU off.\n", vendor); |
904 | 0 | vpmu_mode = XENPMU_MODE_OFF; |
905 | 0 | break; |
906 | 0 | } |
907 | 0 |
|
908 | 0 | if ( vpmu_mode != XENPMU_MODE_OFF ) |
909 | 0 | { |
910 | 0 | register_cpu_notifier(&cpu_nfb); |
911 | 0 | printk(XENLOG_INFO "VPMU: version " __stringify(XENPMU_VER_MAJ) "." |
912 | 0 | __stringify(XENPMU_VER_MIN) "\n"); |
913 | 0 | } |
914 | 0 | else |
915 | 0 | opt_vpmu_enabled = 0; |
916 | 0 |
|
917 | 0 | return 0; |
918 | 0 | } |
919 | | __initcall(vpmu_init); |