/root/src/xen/xen/arch/x86/apic.c
Line | Count | Source (jump to first uncovered line) |
1 | | /* |
2 | | * based on linux-2.6.17.13/arch/i386/kernel/apic.c |
3 | | * |
4 | | * Local APIC handling, local APIC timers |
5 | | * |
6 | | * (c) 1999, 2000 Ingo Molnar <mingo@redhat.com> |
7 | | * |
8 | | * Fixes |
9 | | * Maciej W. Rozycki : Bits for genuine 82489DX APICs; |
10 | | * thanks to Eric Gilmore |
11 | | * and Rolf G. Tews |
12 | | * for testing these extensively. |
13 | | * Maciej W. Rozycki : Various updates and fixes. |
14 | | * Mikael Pettersson : Power Management for UP-APIC. |
15 | | * Pavel Machek and |
16 | | * Mikael Pettersson : PM converted to driver model. |
17 | | */ |
18 | | |
19 | | #include <xen/perfc.h> |
20 | | #include <xen/errno.h> |
21 | | #include <xen/init.h> |
22 | | #include <xen/mm.h> |
23 | | #include <xen/sched.h> |
24 | | #include <xen/irq.h> |
25 | | #include <xen/delay.h> |
26 | | #include <xen/smp.h> |
27 | | #include <xen/softirq.h> |
28 | | #include <asm/mc146818rtc.h> |
29 | | #include <asm/msr.h> |
30 | | #include <asm/atomic.h> |
31 | | #include <asm/mpspec.h> |
32 | | #include <asm/flushtlb.h> |
33 | | #include <asm/hardirq.h> |
34 | | #include <asm/apic.h> |
35 | | #include <asm/io_apic.h> |
36 | | #include <mach_apic.h> |
37 | | #include <io_ports.h> |
38 | | #include <xen/kexec.h> |
39 | | |
40 | | static bool __read_mostly tdt_enabled; |
41 | | static bool __initdata tdt_enable = true; |
42 | | boolean_param("tdt", tdt_enable); |
43 | | |
44 | | static struct { |
45 | | int active; |
46 | | /* r/w apic fields */ |
47 | | unsigned int apic_id; |
48 | | unsigned int apic_taskpri; |
49 | | unsigned int apic_ldr; |
50 | | unsigned int apic_dfr; |
51 | | unsigned int apic_spiv; |
52 | | unsigned int apic_lvtt; |
53 | | unsigned int apic_lvtpc; |
54 | | unsigned int apic_lvtcmci; |
55 | | unsigned int apic_lvt0; |
56 | | unsigned int apic_lvt1; |
57 | | unsigned int apic_lvterr; |
58 | | unsigned int apic_tmict; |
59 | | unsigned int apic_tdcr; |
60 | | unsigned int apic_thmr; |
61 | | } apic_pm_state; |
62 | | |
63 | | /* |
64 | | * Knob to control our willingness to enable the local APIC. |
65 | | */ |
66 | | static s8 __initdata enable_local_apic; /* -1=force-disable, +1=force-enable */ |
67 | | |
68 | | /* |
69 | | * Debug level |
70 | | */ |
71 | | u8 __read_mostly apic_verbosity; |
72 | | |
73 | | static bool __initdata opt_x2apic = true; |
74 | | boolean_param("x2apic", opt_x2apic); |
75 | | |
76 | | /* |
77 | | * Bootstrap processor local APIC boot mode - so we can undo our changes |
78 | | * to the APIC state. |
79 | | */ |
80 | | static enum apic_mode apic_boot_mode = APIC_MODE_INVALID; |
81 | | |
82 | | bool __read_mostly x2apic_enabled; |
83 | | bool __read_mostly directed_eoi_enabled; |
84 | | |
85 | | static int modern_apic(void) |
86 | 1 | { |
87 | 1 | unsigned int lvr, version; |
88 | 1 | /* AMD systems use old APIC versions, so check the CPU */ |
89 | 1 | if (boot_cpu_data.x86_vendor == X86_VENDOR_AMD && |
90 | 0 | boot_cpu_data.x86 >= 0xf) |
91 | 0 | return 1; |
92 | 1 | lvr = apic_read(APIC_LVR); |
93 | 1 | version = GET_APIC_VERSION(lvr); |
94 | 1 | return version >= 0x14; |
95 | 1 | } |
96 | | |
97 | | /* |
98 | | * 'what should we do if we get a hw irq event on an illegal vector'. |
99 | | * each architecture has to answer this themselves. |
100 | | */ |
101 | | void ack_bad_irq(unsigned int irq) |
102 | 0 | { |
103 | 0 | printk("unexpected IRQ trap at irq %02x\n", irq); |
104 | 0 | /* |
105 | 0 | * Currently unexpected vectors happen only on SMP and APIC. |
106 | 0 | * We _must_ ack these because every local APIC has only N |
107 | 0 | * irq slots per priority level, and a 'hanging, unacked' IRQ |
108 | 0 | * holds up an irq slot - in excessive cases (when multiple |
109 | 0 | * unexpected vectors occur) that might lock up the APIC |
110 | 0 | * completely. |
111 | 0 | * But only ack when the APIC is enabled -AK |
112 | 0 | */ |
113 | 0 | if (cpu_has_apic) |
114 | 0 | ack_APIC_irq(); |
115 | 0 | } |
116 | | |
117 | | void __init apic_intr_init(void) |
118 | 1 | { |
119 | 1 | smp_intr_init(); |
120 | 1 | |
121 | 1 | /* self generated IPI for local APIC timer */ |
122 | 1 | set_direct_apic_vector(LOCAL_TIMER_VECTOR, apic_timer_interrupt); |
123 | 1 | |
124 | 1 | /* IPI vectors for APIC spurious and error interrupts */ |
125 | 1 | set_direct_apic_vector(SPURIOUS_APIC_VECTOR, spurious_interrupt); |
126 | 1 | set_direct_apic_vector(ERROR_APIC_VECTOR, error_interrupt); |
127 | 1 | |
128 | 1 | /* Performance Counters Interrupt */ |
129 | 1 | set_direct_apic_vector(PMU_APIC_VECTOR, pmu_apic_interrupt); |
130 | 1 | } |
131 | | |
132 | | /* Using APIC to generate smp_local_timer_interrupt? */ |
133 | | static bool __read_mostly using_apic_timer; |
134 | | |
135 | | static bool __read_mostly enabled_via_apicbase; |
136 | | |
137 | | int get_physical_broadcast(void) |
138 | 0 | { |
139 | 0 | if (modern_apic()) |
140 | 0 | return 0xff; |
141 | 0 | else |
142 | 0 | return 0xf; |
143 | 0 | } |
144 | | |
145 | | int get_maxlvt(void) |
146 | 24 | { |
147 | 24 | unsigned int v = apic_read(APIC_LVR); |
148 | 24 | |
149 | 24 | return GET_APIC_MAXLVT(v); |
150 | 24 | } |
151 | | |
152 | | void clear_local_APIC(void) |
153 | 0 | { |
154 | 0 | int maxlvt; |
155 | 0 | unsigned long v; |
156 | 0 |
|
157 | 0 | maxlvt = get_maxlvt(); |
158 | 0 |
|
159 | 0 | /* Work around AMD Erratum 411. This is a nice thing to do anyway. */ |
160 | 0 | apic_write(APIC_TMICT, 0); |
161 | 0 |
|
162 | 0 | /* |
163 | 0 | * Masking an LVT entry on a P6 can trigger a local APIC error |
164 | 0 | * if the vector is zero. Mask LVTERR first to prevent this. |
165 | 0 | */ |
166 | 0 | if (maxlvt >= 3) { |
167 | 0 | v = ERROR_APIC_VECTOR; /* any non-zero vector will do */ |
168 | 0 | apic_write(APIC_LVTERR, v | APIC_LVT_MASKED); |
169 | 0 | } |
170 | 0 | /* |
171 | 0 | * Careful: we have to set masks only first to deassert |
172 | 0 | * any level-triggered sources. |
173 | 0 | */ |
174 | 0 | v = apic_read(APIC_LVTT); |
175 | 0 | apic_write(APIC_LVTT, v | APIC_LVT_MASKED); |
176 | 0 | v = apic_read(APIC_LVT0); |
177 | 0 | apic_write(APIC_LVT0, v | APIC_LVT_MASKED); |
178 | 0 | v = apic_read(APIC_LVT1); |
179 | 0 | apic_write(APIC_LVT1, v | APIC_LVT_MASKED); |
180 | 0 | if (maxlvt >= 4) { |
181 | 0 | v = apic_read(APIC_LVTPC); |
182 | 0 | apic_write(APIC_LVTPC, v | APIC_LVT_MASKED); |
183 | 0 | } |
184 | 0 |
|
185 | 0 | /* lets not touch this if we didn't frob it */ |
186 | 0 | #ifdef CONFIG_X86_MCE_THERMAL |
187 | 0 | if (maxlvt >= 5) { |
188 | 0 | v = apic_read(APIC_LVTTHMR); |
189 | 0 | apic_write(APIC_LVTTHMR, v | APIC_LVT_MASKED); |
190 | 0 | } |
191 | 0 | #endif |
192 | 0 |
|
193 | 0 | if (maxlvt >= 6) { |
194 | 0 | v = apic_read(APIC_CMCI); |
195 | 0 | apic_write(APIC_CMCI, v | APIC_LVT_MASKED); |
196 | 0 | } |
197 | 0 | /* |
198 | 0 | * Clean APIC state for other OSs: |
199 | 0 | */ |
200 | 0 | apic_write(APIC_LVTT, APIC_LVT_MASKED); |
201 | 0 | apic_write(APIC_LVT0, APIC_LVT_MASKED); |
202 | 0 | apic_write(APIC_LVT1, APIC_LVT_MASKED); |
203 | 0 | if (maxlvt >= 3) |
204 | 0 | apic_write(APIC_LVTERR, APIC_LVT_MASKED); |
205 | 0 | if (maxlvt >= 4) |
206 | 0 | apic_write(APIC_LVTPC, APIC_LVT_MASKED); |
207 | 0 |
|
208 | 0 | #ifdef CONFIG_X86_MCE_THERMAL |
209 | 0 | if (maxlvt >= 5) |
210 | 0 | apic_write(APIC_LVTTHMR, APIC_LVT_MASKED); |
211 | 0 | #endif |
212 | 0 | if (maxlvt >= 6) |
213 | 0 | apic_write(APIC_CMCI, APIC_LVT_MASKED); |
214 | 0 |
|
215 | 0 | if (maxlvt > 3) /* Due to Pentium errata 3AP and 11AP. */ |
216 | 0 | apic_write(APIC_ESR, 0); |
217 | 0 | apic_read(APIC_ESR); |
218 | 0 | } |
219 | | |
220 | | void __init connect_bsp_APIC(void) |
221 | 1 | { |
222 | 1 | if (pic_mode) { |
223 | 0 | /* |
224 | 0 | * Do not trust the local APIC being empty at bootup. |
225 | 0 | */ |
226 | 0 | clear_local_APIC(); |
227 | 0 | /* |
228 | 0 | * PIC mode, enable APIC mode in the IMCR, i.e. |
229 | 0 | * connect BSP's local APIC to INT and NMI lines. |
230 | 0 | */ |
231 | 0 | apic_printk(APIC_VERBOSE, "leaving PIC mode, " |
232 | 0 | "enabling APIC mode.\n"); |
233 | 0 | outb(0x70, 0x22); |
234 | 0 | outb(0x01, 0x23); |
235 | 0 | } |
236 | 1 | enable_apic_mode(); |
237 | 1 | } |
238 | | |
239 | | void disconnect_bsp_APIC(int virt_wire_setup) |
240 | 0 | { |
241 | 0 | if (pic_mode) { |
242 | 0 | /* |
243 | 0 | * Put the board back into PIC mode (has an effect |
244 | 0 | * only on certain older boards). Note that APIC |
245 | 0 | * interrupts, including IPIs, won't work beyond |
246 | 0 | * this point! The only exception are INIT IPIs. |
247 | 0 | */ |
248 | 0 | apic_printk(APIC_VERBOSE, "disabling APIC mode, " |
249 | 0 | "entering PIC mode.\n"); |
250 | 0 | outb(0x70, 0x22); |
251 | 0 | outb(0x00, 0x23); |
252 | 0 | } |
253 | 0 | else { |
254 | 0 | /* Go back to Virtual Wire compatibility mode */ |
255 | 0 | unsigned long value; |
256 | 0 |
|
257 | 0 | /* For the spurious interrupt use vector F, and enable it */ |
258 | 0 | value = apic_read(APIC_SPIV); |
259 | 0 | value &= ~APIC_VECTOR_MASK; |
260 | 0 | value |= APIC_SPIV_APIC_ENABLED; |
261 | 0 | value |= 0xf; |
262 | 0 | apic_write(APIC_SPIV, value); |
263 | 0 |
|
264 | 0 | if (!virt_wire_setup) { |
265 | 0 | /* For LVT0 make it edge triggered, active high, external and enabled */ |
266 | 0 | value = apic_read(APIC_LVT0); |
267 | 0 | value &= ~(APIC_MODE_MASK | APIC_SEND_PENDING | |
268 | 0 | APIC_INPUT_POLARITY | APIC_LVT_REMOTE_IRR | |
269 | 0 | APIC_LVT_LEVEL_TRIGGER | APIC_LVT_MASKED ); |
270 | 0 | value |= APIC_LVT_REMOTE_IRR | APIC_SEND_PENDING; |
271 | 0 | value = SET_APIC_DELIVERY_MODE(value, APIC_MODE_EXTINT); |
272 | 0 | apic_write(APIC_LVT0, value); |
273 | 0 | } |
274 | 0 | else { |
275 | 0 | /* Disable LVT0 */ |
276 | 0 | apic_write(APIC_LVT0, APIC_LVT_MASKED); |
277 | 0 | } |
278 | 0 |
|
279 | 0 | /* For LVT1 make it edge triggered, active high, nmi and enabled */ |
280 | 0 | value = apic_read(APIC_LVT1); |
281 | 0 | value &= ~( |
282 | 0 | APIC_MODE_MASK | APIC_SEND_PENDING | |
283 | 0 | APIC_INPUT_POLARITY | APIC_LVT_REMOTE_IRR | |
284 | 0 | APIC_LVT_LEVEL_TRIGGER | APIC_LVT_MASKED); |
285 | 0 | value |= APIC_LVT_REMOTE_IRR | APIC_SEND_PENDING; |
286 | 0 | value = SET_APIC_DELIVERY_MODE(value, APIC_MODE_NMI); |
287 | 0 | apic_write(APIC_LVT1, value); |
288 | 0 | } |
289 | 0 | } |
290 | | |
291 | | void disable_local_APIC(void) |
292 | 0 | { |
293 | 0 | clear_local_APIC(); |
294 | 0 |
|
295 | 0 | /* |
296 | 0 | * Disable APIC (implies clearing of registers |
297 | 0 | * for 82489DX!). |
298 | 0 | */ |
299 | 0 | apic_write(APIC_SPIV, apic_read(APIC_SPIV) & ~APIC_SPIV_APIC_ENABLED); |
300 | 0 |
|
301 | 0 | if (enabled_via_apicbase) { |
302 | 0 | uint64_t msr_content; |
303 | 0 | rdmsrl(MSR_IA32_APICBASE, msr_content); |
304 | 0 | wrmsrl(MSR_IA32_APICBASE, msr_content & |
305 | 0 | ~(MSR_IA32_APICBASE_ENABLE|MSR_IA32_APICBASE_EXTD)); |
306 | 0 | } |
307 | 0 |
|
308 | 0 | if ( kexecing && (current_local_apic_mode() != apic_boot_mode) ) |
309 | 0 | { |
310 | 0 | uint64_t msr_content; |
311 | 0 | rdmsrl(MSR_IA32_APICBASE, msr_content); |
312 | 0 | msr_content &= ~(MSR_IA32_APICBASE_ENABLE|MSR_IA32_APICBASE_EXTD); |
313 | 0 | wrmsrl(MSR_IA32_APICBASE, msr_content); |
314 | 0 |
|
315 | 0 | switch ( apic_boot_mode ) |
316 | 0 | { |
317 | 0 | case APIC_MODE_DISABLED: |
318 | 0 | break; /* Nothing to do - we did this above */ |
319 | 0 | case APIC_MODE_XAPIC: |
320 | 0 | msr_content |= MSR_IA32_APICBASE_ENABLE; |
321 | 0 | wrmsrl(MSR_IA32_APICBASE, msr_content); |
322 | 0 | break; |
323 | 0 | case APIC_MODE_X2APIC: |
324 | 0 | msr_content |= MSR_IA32_APICBASE_ENABLE; |
325 | 0 | wrmsrl(MSR_IA32_APICBASE, msr_content); |
326 | 0 | msr_content |= MSR_IA32_APICBASE_EXTD; |
327 | 0 | wrmsrl(MSR_IA32_APICBASE, msr_content); |
328 | 0 | break; |
329 | 0 | default: |
330 | 0 | printk("Default case when reverting #%d lapic to boot state\n", |
331 | 0 | smp_processor_id()); |
332 | 0 | break; |
333 | 0 | } |
334 | 0 | } |
335 | 0 |
|
336 | 0 | } |
337 | | |
338 | | /* |
339 | | * This is to verify that we're looking at a real local APIC. |
340 | | * Check these against your board if the CPUs aren't getting |
341 | | * started for no apparent reason. |
342 | | */ |
343 | | int __init verify_local_APIC(void) |
344 | 1 | { |
345 | 1 | unsigned int reg0, reg1; |
346 | 1 | |
347 | 1 | /* |
348 | 1 | * The version register is read-only in a real APIC. |
349 | 1 | */ |
350 | 1 | reg0 = apic_read(APIC_LVR); |
351 | 1 | apic_printk(APIC_DEBUG, "Getting VERSION: %x\n", reg0); |
352 | 1 | |
353 | 1 | /* We don't try writing LVR in x2APIC mode since that incurs #GP. */ |
354 | 1 | if ( !x2apic_enabled ) |
355 | 0 | apic_write(APIC_LVR, reg0 ^ APIC_LVR_MASK); |
356 | 1 | reg1 = apic_read(APIC_LVR); |
357 | 1 | apic_printk(APIC_DEBUG, "Getting VERSION: %x\n", reg1); |
358 | 1 | |
359 | 1 | /* |
360 | 1 | * The two version reads above should print the same |
361 | 1 | * numbers. If the second one is different, then we |
362 | 1 | * poke at a non-APIC. |
363 | 1 | */ |
364 | 1 | if (reg1 != reg0) |
365 | 0 | return 0; |
366 | 1 | |
367 | 1 | /* |
368 | 1 | * Check if the version looks reasonably. |
369 | 1 | */ |
370 | 1 | reg1 = GET_APIC_VERSION(reg0); |
371 | 1 | if (reg1 == 0x00 || reg1 == 0xff) |
372 | 0 | return 0; |
373 | 1 | reg1 = get_maxlvt(); |
374 | 1 | if (reg1 < 0x02 || reg1 == 0xff) |
375 | 0 | return 0; |
376 | 1 | |
377 | 1 | /* |
378 | 1 | * Detecting directed EOI on BSP: |
379 | 1 | * If having directed EOI support in lapic, force to use ioapic_ack_old, |
380 | 1 | * and enable the directed EOI for intr handling. |
381 | 1 | */ |
382 | 1 | if ( reg0 & APIC_LVR_DIRECTED_EOI ) |
383 | 1 | { |
384 | 1 | if ( ioapic_ack_new && ioapic_ack_forced ) |
385 | 0 | printk("Not enabling directed EOI because ioapic_ack_new has been " |
386 | 0 | "forced on the command line\n"); |
387 | 1 | else |
388 | 1 | { |
389 | 1 | ioapic_ack_new = false; |
390 | 1 | directed_eoi_enabled = true; |
391 | 1 | printk("Enabled directed EOI with ioapic_ack_old on!\n"); |
392 | 1 | } |
393 | 1 | } |
394 | 1 | |
395 | 1 | /* |
396 | 1 | * The ID register is read/write in a real APIC. |
397 | 1 | */ |
398 | 1 | reg0 = apic_read(APIC_ID); |
399 | 1 | apic_printk(APIC_DEBUG, "Getting ID: %x\n", reg0); |
400 | 1 | |
401 | 1 | /* |
402 | 1 | * The next two are just to see if we have sane values. |
403 | 1 | * They're only really relevant if we're in Virtual Wire |
404 | 1 | * compatibility mode, but most boxes are anymore. |
405 | 1 | */ |
406 | 1 | reg0 = apic_read(APIC_LVT0); |
407 | 1 | apic_printk(APIC_DEBUG, "Getting LVT0: %x\n", reg0); |
408 | 1 | reg1 = apic_read(APIC_LVT1); |
409 | 1 | apic_printk(APIC_DEBUG, "Getting LVT1: %x\n", reg1); |
410 | 1 | |
411 | 1 | return 1; |
412 | 1 | } |
413 | | |
414 | | void __init sync_Arb_IDs(void) |
415 | 1 | { |
416 | 1 | /* Unsupported on P4 - see Intel Dev. Manual Vol. 3, Ch. 8.6.1 |
417 | 1 | And not needed on AMD */ |
418 | 1 | if (modern_apic()) |
419 | 1 | return; |
420 | 1 | /* |
421 | 1 | * Wait for idle. |
422 | 1 | */ |
423 | 0 | apic_wait_icr_idle(); |
424 | 0 |
|
425 | 0 | apic_printk(APIC_DEBUG, "Synchronizing Arb IDs.\n"); |
426 | 0 | apic_write(APIC_ICR, APIC_DEST_ALLINC | APIC_INT_LEVELTRIG | APIC_DM_INIT); |
427 | 0 | } |
428 | | |
429 | | /* |
430 | | * An initial setup of the virtual wire mode. |
431 | | */ |
432 | | void __init init_bsp_APIC(void) |
433 | 1 | { |
434 | 1 | unsigned long value; |
435 | 1 | |
436 | 1 | /* |
437 | 1 | * Don't do the setup now if we have a SMP BIOS as the |
438 | 1 | * through-I/O-APIC virtual wire mode might be active. |
439 | 1 | */ |
440 | 1 | if (smp_found_config || !cpu_has_apic) |
441 | 1 | return; |
442 | 1 | |
443 | 1 | /* |
444 | 1 | * Do not trust the local APIC being empty at bootup. |
445 | 1 | */ |
446 | 0 | clear_local_APIC(); |
447 | 0 | |
448 | 0 | /* |
449 | 0 | * Enable APIC. |
450 | 0 | */ |
451 | 0 | value = apic_read(APIC_SPIV); |
452 | 0 | value &= ~APIC_VECTOR_MASK; |
453 | 0 | value |= APIC_SPIV_APIC_ENABLED; |
454 | 0 | |
455 | 0 | /* This bit is reserved on P4/Xeon and should be cleared */ |
456 | 0 | if ((boot_cpu_data.x86_vendor == X86_VENDOR_INTEL) && (boot_cpu_data.x86 == 15)) |
457 | 0 | value &= ~APIC_SPIV_FOCUS_DISABLED; |
458 | 0 | else |
459 | 0 | value |= APIC_SPIV_FOCUS_DISABLED; |
460 | 0 | value |= SPURIOUS_APIC_VECTOR; |
461 | 0 | apic_write(APIC_SPIV, value); |
462 | 0 |
|
463 | 0 | /* |
464 | 0 | * Set up the virtual wire mode. |
465 | 0 | */ |
466 | 0 | apic_write(APIC_LVT0, APIC_DM_EXTINT); |
467 | 0 | apic_write(APIC_LVT1, APIC_DM_NMI); |
468 | 0 | } |
469 | | |
470 | | static void apic_pm_activate(void) |
471 | 12 | { |
472 | 12 | apic_pm_state.active = 1; |
473 | 12 | } |
474 | | |
475 | | static void __enable_x2apic(void) |
476 | 12 | { |
477 | 12 | uint64_t msr_content; |
478 | 12 | |
479 | 12 | rdmsrl(MSR_IA32_APICBASE, msr_content); |
480 | 12 | if ( !(msr_content & MSR_IA32_APICBASE_EXTD) ) |
481 | 12 | { |
482 | 12 | msr_content |= MSR_IA32_APICBASE_ENABLE | MSR_IA32_APICBASE_EXTD; |
483 | 12 | msr_content = (uint32_t)msr_content; |
484 | 12 | wrmsrl(MSR_IA32_APICBASE, msr_content); |
485 | 12 | } |
486 | 12 | } |
487 | | |
488 | | static void resume_x2apic(void) |
489 | 0 | { |
490 | 0 | struct IO_APIC_route_entry **ioapic_entries = NULL; |
491 | 0 |
|
492 | 0 | ASSERT(x2apic_enabled); |
493 | 0 |
|
494 | 0 | ioapic_entries = alloc_ioapic_entries(); |
495 | 0 | if ( !ioapic_entries ) |
496 | 0 | { |
497 | 0 | printk("Allocate ioapic_entries failed\n"); |
498 | 0 | goto out; |
499 | 0 | } |
500 | 0 |
|
501 | 0 | if ( save_IO_APIC_setup(ioapic_entries) ) |
502 | 0 | { |
503 | 0 | printk("Saving IO-APIC state failed\n"); |
504 | 0 | goto out; |
505 | 0 | } |
506 | 0 |
|
507 | 0 | mask_8259A(); |
508 | 0 | mask_IO_APIC_setup(ioapic_entries); |
509 | 0 |
|
510 | 0 | iommu_enable_x2apic_IR(); |
511 | 0 | __enable_x2apic(); |
512 | 0 |
|
513 | 0 | restore_IO_APIC_setup(ioapic_entries); |
514 | 0 | unmask_8259A(); |
515 | 0 |
|
516 | 0 | out: |
517 | 0 | if ( ioapic_entries ) |
518 | 0 | free_ioapic_entries(ioapic_entries); |
519 | 0 | } |
520 | | |
521 | | void setup_local_APIC(void) |
522 | 12 | { |
523 | 12 | unsigned long oldvalue, value, maxlvt; |
524 | 12 | int i, j; |
525 | 12 | |
526 | 12 | /* Pound the ESR really hard over the head with a big hammer - mbligh */ |
527 | 12 | if (esr_disable) { |
528 | 0 | apic_write(APIC_ESR, 0); |
529 | 0 | apic_write(APIC_ESR, 0); |
530 | 0 | apic_write(APIC_ESR, 0); |
531 | 0 | apic_write(APIC_ESR, 0); |
532 | 0 | } |
533 | 12 | |
534 | 12 | BUILD_BUG_ON((SPURIOUS_APIC_VECTOR & 0x0f) != 0x0f); |
535 | 12 | |
536 | 12 | /* |
537 | 12 | * Double-check whether this APIC is really registered. |
538 | 12 | */ |
539 | 12 | if (!apic_id_registered()) |
540 | 0 | BUG(); |
541 | 12 | |
542 | 12 | /* |
543 | 12 | * Intel recommends to set DFR, LDR and TPR before enabling |
544 | 12 | * an APIC. See e.g. "AP-388 82489DX User's Manual" (Intel |
545 | 12 | * document number 292116). So here it goes... |
546 | 12 | */ |
547 | 12 | init_apic_ldr(); |
548 | 12 | |
549 | 12 | /* |
550 | 12 | * Set Task Priority to reject any interrupts below FIRST_DYNAMIC_VECTOR. |
551 | 12 | */ |
552 | 12 | apic_write(APIC_TASKPRI, (FIRST_DYNAMIC_VECTOR & 0xF0) - 0x10); |
553 | 12 | |
554 | 12 | /* |
555 | 12 | * After a crash, we no longer service the interrupts and a pending |
556 | 12 | * interrupt from previous kernel might still have ISR bit set. |
557 | 12 | * |
558 | 12 | * Most probably by now CPU has serviced that pending interrupt and |
559 | 12 | * it might not have done the ack_APIC_irq() because it thought, |
560 | 12 | * interrupt came from i8259 as ExtInt. LAPIC did not get EOI so it |
561 | 12 | * does not clear the ISR bit and cpu thinks it has already serivced |
562 | 12 | * the interrupt. Hence a vector might get locked. It was noticed |
563 | 12 | * for timer irq (vector 0x31). Issue an extra EOI to clear ISR. |
564 | 12 | */ |
565 | 108 | for (i = APIC_ISR_NR - 1; i >= 0; i--) { |
566 | 96 | value = apic_read(APIC_ISR + i*0x10); |
567 | 3.16k | for (j = 31; j >= 0; j--) { |
568 | 3.07k | if (value & (1u << j)) |
569 | 0 | ack_APIC_irq(); |
570 | 3.07k | } |
571 | 96 | } |
572 | 12 | |
573 | 12 | /* |
574 | 12 | * Now that we are all set up, enable the APIC |
575 | 12 | */ |
576 | 12 | value = apic_read(APIC_SPIV); |
577 | 12 | value &= ~APIC_VECTOR_MASK; |
578 | 12 | /* |
579 | 12 | * Enable APIC |
580 | 12 | */ |
581 | 12 | value |= APIC_SPIV_APIC_ENABLED; |
582 | 12 | |
583 | 12 | /* |
584 | 12 | * Some unknown Intel IO/APIC (or APIC) errata is biting us with |
585 | 12 | * certain networking cards. If high frequency interrupts are |
586 | 12 | * happening on a particular IOAPIC pin, plus the IOAPIC routing |
587 | 12 | * entry is masked/unmasked at a high rate as well then sooner or |
588 | 12 | * later IOAPIC line gets 'stuck', no more interrupts are received |
589 | 12 | * from the device. If focus CPU is disabled then the hang goes |
590 | 12 | * away, oh well :-( |
591 | 12 | * |
592 | 12 | * [ This bug can be reproduced easily with a level-triggered |
593 | 12 | * PCI Ne2000 networking cards and PII/PIII processors, dual |
594 | 12 | * BX chipset. ] |
595 | 12 | */ |
596 | 12 | /* |
597 | 12 | * Actually disabling the focus CPU check just makes the hang less |
598 | 12 | * frequent as it makes the interrupt distributon model be more |
599 | 12 | * like LRU than MRU (the short-term load is more even across CPUs). |
600 | 12 | * See also the comment in end_level_ioapic_irq(). --macro |
601 | 12 | */ |
602 | 12 | #if 1 |
603 | 12 | /* Enable focus processor (bit==0) */ |
604 | 12 | value &= ~APIC_SPIV_FOCUS_DISABLED; |
605 | 12 | #else |
606 | | /* Disable focus processor (bit==1) */ |
607 | | value |= APIC_SPIV_FOCUS_DISABLED; |
608 | | #endif |
609 | 12 | /* |
610 | 12 | * Set spurious IRQ vector |
611 | 12 | */ |
612 | 12 | value |= SPURIOUS_APIC_VECTOR; |
613 | 12 | |
614 | 12 | /* |
615 | 12 | * Enable directed EOI |
616 | 12 | */ |
617 | 12 | if ( directed_eoi_enabled ) |
618 | 12 | { |
619 | 12 | value |= APIC_SPIV_DIRECTED_EOI; |
620 | 12 | apic_printk(APIC_VERBOSE, "Suppress EOI broadcast on CPU#%d\n", |
621 | 0 | smp_processor_id()); |
622 | 12 | } |
623 | 12 | |
624 | 12 | apic_write(APIC_SPIV, value); |
625 | 12 | |
626 | 12 | /* |
627 | 12 | * Set up LVT0, LVT1: |
628 | 12 | * |
629 | 12 | * set up through-local-APIC on the BP's LINT0. This is not |
630 | 12 | * strictly necessery in pure symmetric-IO mode, but sometimes |
631 | 12 | * we delegate interrupts to the 8259A. |
632 | 12 | */ |
633 | 12 | /* |
634 | 12 | * TODO: set up through-local-APIC from through-I/O-APIC? --macro |
635 | 12 | */ |
636 | 12 | value = apic_read(APIC_LVT0) & APIC_LVT_MASKED; |
637 | 12 | if (!smp_processor_id() && (pic_mode || !value)) { |
638 | 1 | value = APIC_DM_EXTINT; |
639 | 1 | apic_printk(APIC_VERBOSE, "enabled ExtINT on CPU#%d\n", |
640 | 0 | smp_processor_id()); |
641 | 11 | } else { |
642 | 11 | value = APIC_DM_EXTINT | APIC_LVT_MASKED; |
643 | 11 | apic_printk(APIC_VERBOSE, "masked ExtINT on CPU#%d\n", |
644 | 0 | smp_processor_id()); |
645 | 11 | } |
646 | 12 | apic_write(APIC_LVT0, value); |
647 | 12 | |
648 | 12 | /* |
649 | 12 | * only the BP should see the LINT1 NMI signal, obviously. |
650 | 12 | */ |
651 | 12 | if (!smp_processor_id()) |
652 | 1 | value = APIC_DM_NMI; |
653 | 12 | else |
654 | 11 | value = APIC_DM_NMI | APIC_LVT_MASKED; |
655 | 12 | apic_write(APIC_LVT1, value); |
656 | 12 | |
657 | 12 | if (!esr_disable) { |
658 | 12 | maxlvt = get_maxlvt(); |
659 | 12 | if (maxlvt > 3) /* Due to the Pentium erratum 3AP. */ |
660 | 12 | apic_write(APIC_ESR, 0); |
661 | 12 | oldvalue = apic_read(APIC_ESR); |
662 | 12 | |
663 | 12 | value = ERROR_APIC_VECTOR; // enables sending errors |
664 | 12 | apic_write(APIC_LVTERR, value); |
665 | 12 | /* |
666 | 12 | * spec says clear errors after enabling vector. |
667 | 12 | */ |
668 | 12 | if (maxlvt > 3) |
669 | 12 | apic_write(APIC_ESR, 0); |
670 | 12 | value = apic_read(APIC_ESR); |
671 | 12 | if (value != oldvalue) |
672 | 0 | apic_printk(APIC_VERBOSE, "ESR value before enabling " |
673 | 12 | "vector: %#lx after: %#lx\n", |
674 | 12 | oldvalue, value); |
675 | 0 | } else { |
676 | 0 | /* |
677 | 0 | * Something untraceble is creating bad interrupts on |
678 | 0 | * secondary quads ... for the moment, just leave the |
679 | 0 | * ESR disabled - we can't do anything useful with the |
680 | 0 | * errors anyway - mbligh |
681 | 0 | */ |
682 | 0 | printk("Leaving ESR disabled.\n"); |
683 | 0 | } |
684 | 12 | |
685 | 12 | if (nmi_watchdog == NMI_LOCAL_APIC) |
686 | 0 | setup_apic_nmi_watchdog(); |
687 | 12 | apic_pm_activate(); |
688 | 12 | } |
689 | | |
690 | | int lapic_suspend(void) |
691 | 0 | { |
692 | 0 | unsigned long flags; |
693 | 0 | int maxlvt = get_maxlvt(); |
694 | 0 | if (!apic_pm_state.active) |
695 | 0 | return 0; |
696 | 0 |
|
697 | 0 | apic_pm_state.apic_id = apic_read(APIC_ID); |
698 | 0 | apic_pm_state.apic_taskpri = apic_read(APIC_TASKPRI); |
699 | 0 | apic_pm_state.apic_ldr = apic_read(APIC_LDR); |
700 | 0 | apic_pm_state.apic_dfr = apic_read(APIC_DFR); |
701 | 0 | apic_pm_state.apic_spiv = apic_read(APIC_SPIV); |
702 | 0 | apic_pm_state.apic_lvtt = apic_read(APIC_LVTT); |
703 | 0 | if (maxlvt >= 4) |
704 | 0 | apic_pm_state.apic_lvtpc = apic_read(APIC_LVTPC); |
705 | 0 |
|
706 | 0 | if (maxlvt >= 6) { |
707 | 0 | apic_pm_state.apic_lvtcmci = apic_read(APIC_CMCI); |
708 | 0 | } |
709 | 0 |
|
710 | 0 | apic_pm_state.apic_lvt0 = apic_read(APIC_LVT0); |
711 | 0 | apic_pm_state.apic_lvt1 = apic_read(APIC_LVT1); |
712 | 0 | apic_pm_state.apic_lvterr = apic_read(APIC_LVTERR); |
713 | 0 | apic_pm_state.apic_tmict = apic_read(APIC_TMICT); |
714 | 0 | apic_pm_state.apic_tdcr = apic_read(APIC_TDCR); |
715 | 0 | if (maxlvt >= 5) |
716 | 0 | apic_pm_state.apic_thmr = apic_read(APIC_LVTTHMR); |
717 | 0 |
|
718 | 0 | local_irq_save(flags); |
719 | 0 | disable_local_APIC(); |
720 | 0 | iommu_disable_x2apic_IR(); |
721 | 0 | local_irq_restore(flags); |
722 | 0 | return 0; |
723 | 0 | } |
724 | | |
725 | | int lapic_resume(void) |
726 | 0 | { |
727 | 0 | uint64_t msr_content; |
728 | 0 | unsigned long flags; |
729 | 0 | int maxlvt; |
730 | 0 |
|
731 | 0 | if (!apic_pm_state.active) |
732 | 0 | return 0; |
733 | 0 |
|
734 | 0 | local_irq_save(flags); |
735 | 0 |
|
736 | 0 | /* |
737 | 0 | * Make sure the APICBASE points to the right address |
738 | 0 | * |
739 | 0 | * FIXME! This will be wrong if we ever support suspend on |
740 | 0 | * SMP! We'll need to do this as part of the CPU restore! |
741 | 0 | */ |
742 | 0 | if ( !x2apic_enabled ) |
743 | 0 | { |
744 | 0 | rdmsrl(MSR_IA32_APICBASE, msr_content); |
745 | 0 | msr_content &= ~MSR_IA32_APICBASE_BASE; |
746 | 0 | wrmsrl(MSR_IA32_APICBASE, |
747 | 0 | msr_content | MSR_IA32_APICBASE_ENABLE | mp_lapic_addr); |
748 | 0 | } |
749 | 0 | else |
750 | 0 | resume_x2apic(); |
751 | 0 |
|
752 | 0 | maxlvt = get_maxlvt(); |
753 | 0 | apic_write(APIC_LVTERR, ERROR_APIC_VECTOR | APIC_LVT_MASKED); |
754 | 0 | apic_write(APIC_ID, apic_pm_state.apic_id); |
755 | 0 | apic_write(APIC_DFR, apic_pm_state.apic_dfr); |
756 | 0 | apic_write(APIC_LDR, apic_pm_state.apic_ldr); |
757 | 0 | apic_write(APIC_TASKPRI, apic_pm_state.apic_taskpri); |
758 | 0 | apic_write(APIC_SPIV, apic_pm_state.apic_spiv); |
759 | 0 | apic_write(APIC_LVT0, apic_pm_state.apic_lvt0); |
760 | 0 | apic_write(APIC_LVT1, apic_pm_state.apic_lvt1); |
761 | 0 | if (maxlvt >= 5) |
762 | 0 | apic_write(APIC_LVTTHMR, apic_pm_state.apic_thmr); |
763 | 0 |
|
764 | 0 | if (maxlvt >= 6) { |
765 | 0 | apic_write(APIC_CMCI, apic_pm_state.apic_lvtcmci); |
766 | 0 | } |
767 | 0 |
|
768 | 0 | if (maxlvt >= 4) |
769 | 0 | apic_write(APIC_LVTPC, apic_pm_state.apic_lvtpc); |
770 | 0 | apic_write(APIC_LVTT, apic_pm_state.apic_lvtt); |
771 | 0 | apic_write(APIC_TDCR, apic_pm_state.apic_tdcr); |
772 | 0 | apic_write(APIC_TMICT, apic_pm_state.apic_tmict); |
773 | 0 | apic_write(APIC_ESR, 0); |
774 | 0 | apic_read(APIC_ESR); |
775 | 0 | apic_write(APIC_LVTERR, apic_pm_state.apic_lvterr); |
776 | 0 | apic_write(APIC_ESR, 0); |
777 | 0 | apic_read(APIC_ESR); |
778 | 0 | local_irq_restore(flags); |
779 | 0 | return 0; |
780 | 0 | } |
781 | | |
782 | | |
783 | | /* |
784 | | * Detect and enable local APICs on non-SMP boards. |
785 | | * Original code written by Keir Fraser. |
786 | | */ |
787 | | |
788 | | static int __init lapic_disable(const char *str) |
789 | 0 | { |
790 | 0 | enable_local_apic = -1; |
791 | 0 | setup_clear_cpu_cap(X86_FEATURE_APIC); |
792 | 0 | return 0; |
793 | 0 | } |
794 | | custom_param("nolapic", lapic_disable); |
795 | | boolean_param("lapic", enable_local_apic); |
796 | | |
797 | | static int __init apic_set_verbosity(const char *str) |
798 | 0 | { |
799 | 0 | if (strcmp("debug", str) == 0) |
800 | 0 | apic_verbosity = APIC_DEBUG; |
801 | 0 | else if (strcmp("verbose", str) == 0) |
802 | 0 | apic_verbosity = APIC_VERBOSE; |
803 | 0 | else |
804 | 0 | return -EINVAL; |
805 | 0 |
|
806 | 0 | return 0; |
807 | 0 | } |
808 | | custom_param("apic_verbosity", apic_set_verbosity); |
809 | | |
810 | | static int __init detect_init_APIC (void) |
811 | 0 | { |
812 | 0 | uint64_t msr_content; |
813 | 0 |
|
814 | 0 | /* Disabled by kernel option? */ |
815 | 0 | if (enable_local_apic < 0) |
816 | 0 | return -1; |
817 | 0 |
|
818 | 0 | if (rdmsr_safe(MSR_IA32_APICBASE, msr_content)) { |
819 | 0 | printk("No local APIC present\n"); |
820 | 0 | return -1; |
821 | 0 | } |
822 | 0 |
|
823 | 0 | if (!cpu_has_apic) { |
824 | 0 | /* |
825 | 0 | * Over-ride BIOS and try to enable the local |
826 | 0 | * APIC only if "lapic" specified. |
827 | 0 | */ |
828 | 0 | if (enable_local_apic <= 0) { |
829 | 0 | printk("Local APIC disabled by BIOS -- " |
830 | 0 | "you can enable it with \"lapic\"\n"); |
831 | 0 | return -1; |
832 | 0 | } |
833 | 0 | /* |
834 | 0 | * Some BIOSes disable the local APIC in the |
835 | 0 | * APIC_BASE MSR. This can only be done in |
836 | 0 | * software for Intel P6 or later and AMD K7 |
837 | 0 | * (Model > 1) or later. |
838 | 0 | */ |
839 | 0 | if (!(msr_content & MSR_IA32_APICBASE_ENABLE)) { |
840 | 0 | printk("Local APIC disabled by BIOS -- reenabling.\n"); |
841 | 0 | msr_content &= ~MSR_IA32_APICBASE_BASE; |
842 | 0 | msr_content |= MSR_IA32_APICBASE_ENABLE | APIC_DEFAULT_PHYS_BASE; |
843 | 0 | wrmsrl(MSR_IA32_APICBASE, msr_content); |
844 | 0 | enabled_via_apicbase = true; |
845 | 0 | } |
846 | 0 | } |
847 | 0 | /* |
848 | 0 | * The APIC feature bit should now be enabled |
849 | 0 | * in `cpuid' |
850 | 0 | */ |
851 | 0 | if (!(cpuid_edx(1) & cpufeat_mask(X86_FEATURE_APIC))) { |
852 | 0 | printk("Could not enable APIC!\n"); |
853 | 0 | return -1; |
854 | 0 | } |
855 | 0 |
|
856 | 0 | setup_force_cpu_cap(X86_FEATURE_APIC); |
857 | 0 | mp_lapic_addr = APIC_DEFAULT_PHYS_BASE; |
858 | 0 |
|
859 | 0 | /* The BIOS may have set up the APIC at some other address */ |
860 | 0 | if (msr_content & MSR_IA32_APICBASE_ENABLE) |
861 | 0 | mp_lapic_addr = msr_content & MSR_IA32_APICBASE_BASE; |
862 | 0 |
|
863 | 0 | if (nmi_watchdog != NMI_NONE) |
864 | 0 | nmi_watchdog = NMI_LOCAL_APIC; |
865 | 0 |
|
866 | 0 | printk("Found and enabled local APIC!\n"); |
867 | 0 |
|
868 | 0 | apic_pm_activate(); |
869 | 0 |
|
870 | 0 | return 0; |
871 | 0 | } |
872 | | |
873 | | void x2apic_ap_setup(void) |
874 | 11 | { |
875 | 11 | if ( x2apic_enabled ) |
876 | 11 | __enable_x2apic(); |
877 | 11 | } |
878 | | |
879 | | void __init x2apic_bsp_setup(void) |
880 | 1 | { |
881 | 1 | struct IO_APIC_route_entry **ioapic_entries = NULL; |
882 | 1 | |
883 | 1 | if ( !cpu_has_x2apic ) |
884 | 0 | return; |
885 | 1 | |
886 | 1 | if ( !opt_x2apic ) |
887 | 0 | { |
888 | 0 | if ( !x2apic_enabled ) |
889 | 0 | { |
890 | 0 | printk("Not enabling x2APIC: disabled by cmdline.\n"); |
891 | 0 | return; |
892 | 0 | } |
893 | 0 | printk("x2APIC: Already enabled by BIOS: Ignoring cmdline disable.\n"); |
894 | 0 | } |
895 | 1 | |
896 | 1 | if ( !iommu_supports_eim() ) |
897 | 0 | { |
898 | 0 | if ( !x2apic_enabled ) |
899 | 0 | { |
900 | 0 | printk("Not enabling x2APIC: depends on iommu_supports_eim.\n"); |
901 | 0 | return; |
902 | 0 | } |
903 | 0 | panic("x2APIC: already enabled by BIOS, but " |
904 | 0 | "iommu_supports_eim failed"); |
905 | 0 | } |
906 | 1 | |
907 | 1 | if ( (ioapic_entries = alloc_ioapic_entries()) == NULL ) |
908 | 0 | { |
909 | 0 | printk("Allocate ioapic_entries failed\n"); |
910 | 0 | goto out; |
911 | 0 | } |
912 | 1 | |
913 | 1 | if ( save_IO_APIC_setup(ioapic_entries) ) |
914 | 0 | { |
915 | 0 | printk("Saving IO-APIC state failed\n"); |
916 | 0 | goto out; |
917 | 0 | } |
918 | 1 | |
919 | 1 | mask_8259A(); |
920 | 1 | mask_IO_APIC_setup(ioapic_entries); |
921 | 1 | |
922 | 1 | switch ( iommu_enable_x2apic_IR() ) |
923 | 1 | { |
924 | 1 | case 0: |
925 | 1 | break; |
926 | 0 | case -ENXIO: /* ACPI_DMAR_X2APIC_OPT_OUT set */ |
927 | 0 | if ( !x2apic_enabled ) |
928 | 0 | { |
929 | 0 | printk("Not enabling x2APIC (upon firmware request)\n"); |
930 | 0 | goto restore_out; |
931 | 0 | } |
932 | 0 | /* fall through */ |
933 | 0 | default: |
934 | 0 | if ( x2apic_enabled ) |
935 | 0 | panic("Interrupt remapping could not be enabled while " |
936 | 0 | "x2APIC is already enabled by BIOS"); |
937 | 0 |
|
938 | 0 | printk(XENLOG_ERR |
939 | 0 | "Failed to enable Interrupt Remapping: Will not enable x2APIC.\n"); |
940 | 0 | goto restore_out; |
941 | 1 | } |
942 | 1 | |
943 | 1 | force_iommu = 1; |
944 | 1 | |
945 | 1 | genapic = apic_x2apic_probe(); |
946 | 1 | printk("Switched to APIC driver %s.\n", genapic->name); |
947 | 1 | |
948 | 1 | if ( !x2apic_enabled ) |
949 | 1 | { |
950 | 1 | x2apic_enabled = true; |
951 | 1 | __enable_x2apic(); |
952 | 1 | } |
953 | 1 | |
954 | 1 | restore_out: |
955 | 1 | restore_IO_APIC_setup(ioapic_entries); |
956 | 1 | unmask_8259A(); |
957 | 1 | |
958 | 1 | out: |
959 | 1 | if ( ioapic_entries ) |
960 | 1 | free_ioapic_entries(ioapic_entries); |
961 | 1 | } |
962 | | |
963 | | void __init init_apic_mappings(void) |
964 | 1 | { |
965 | 1 | unsigned long apic_phys; |
966 | 1 | |
967 | 1 | if ( x2apic_enabled ) |
968 | 0 | goto __next; |
969 | 1 | /* |
970 | 1 | * If no local APIC can be found then set up a fake all |
971 | 1 | * zeroes page to simulate the local APIC and another |
972 | 1 | * one for the IO-APIC. |
973 | 1 | */ |
974 | 1 | if (!smp_found_config && detect_init_APIC()) { |
975 | 0 | apic_phys = __pa(alloc_xenheap_page()); |
976 | 0 | clear_page(__va(apic_phys)); |
977 | 0 | } else |
978 | 1 | apic_phys = mp_lapic_addr; |
979 | 1 | |
980 | 1 | set_fixmap_nocache(FIX_APIC_BASE, apic_phys); |
981 | 1 | apic_printk(APIC_VERBOSE, "mapped APIC to %08Lx (%08lx)\n", APIC_BASE, |
982 | 0 | apic_phys); |
983 | 1 | |
984 | 1 | __next: |
985 | 1 | /* |
986 | 1 | * Fetch the APIC ID of the BSP in case we have a |
987 | 1 | * default configuration (or the MP table is broken). |
988 | 1 | */ |
989 | 1 | if (boot_cpu_physical_apicid == -1U) |
990 | 0 | boot_cpu_physical_apicid = get_apic_id(); |
991 | 1 | x86_cpu_to_apicid[0] = get_apic_id(); |
992 | 1 | |
993 | 1 | init_ioapic_mappings(); |
994 | 1 | } |
995 | | |
996 | | /***************************************************************************** |
997 | | * APIC calibration |
998 | | * |
999 | | * The APIC is programmed in bus cycles. |
1000 | | * Timeout values should specified in real time units. |
1001 | | * The "cheapest" time source is the cyclecounter. |
1002 | | * |
1003 | | * Thus, we need a mappings from: bus cycles <- cycle counter <- system time |
1004 | | * |
1005 | | * The calibration is currently a bit shoddy since it requires the external |
1006 | | * timer chip to generate periodic timer interupts. |
1007 | | *****************************************************************************/ |
1008 | | |
1009 | | /* used for system time scaling */ |
1010 | | static u32 __read_mostly bus_scale; /* scaling factor: ns -> bus cycles */ |
1011 | | |
1012 | | /* |
1013 | | * The timer chip is already set up at HZ interrupts per second here, |
1014 | | * but we do not accept timer interrupts yet. We only allow the BP |
1015 | | * to calibrate. |
1016 | | */ |
1017 | | static unsigned int __init get_8254_timer_count(void) |
1018 | 26.8k | { |
1019 | 26.8k | /*extern spinlock_t i8253_lock;*/ |
1020 | 26.8k | /*unsigned long flags;*/ |
1021 | 26.8k | |
1022 | 26.8k | unsigned int count; |
1023 | 26.8k | |
1024 | 26.8k | /*spin_lock_irqsave(&i8253_lock, flags);*/ |
1025 | 26.8k | |
1026 | 26.8k | outb_p(0x00, PIT_MODE); |
1027 | 26.8k | count = inb_p(PIT_CH0); |
1028 | 26.8k | count |= inb_p(PIT_CH0) << 8; |
1029 | 26.8k | |
1030 | 26.8k | /*spin_unlock_irqrestore(&i8253_lock, flags);*/ |
1031 | 26.8k | |
1032 | 26.8k | return count; |
1033 | 26.8k | } |
1034 | | |
1035 | | /* next tick in 8254 can be caught by catching timer wraparound */ |
1036 | | static void __init wait_8254_wraparound(void) |
1037 | 11 | { |
1038 | 11 | unsigned int curr_count, prev_count; |
1039 | 11 | |
1040 | 11 | curr_count = get_8254_timer_count(); |
1041 | 26.8k | do { |
1042 | 26.8k | prev_count = curr_count; |
1043 | 26.8k | curr_count = get_8254_timer_count(); |
1044 | 26.8k | |
1045 | 26.8k | /* workaround for broken Mercury/Neptune */ |
1046 | 26.8k | if (prev_count >= curr_count + 0x100) |
1047 | 0 | curr_count = get_8254_timer_count(); |
1048 | 26.8k | |
1049 | 26.8k | } while (prev_count >= curr_count); |
1050 | 11 | } |
1051 | | |
1052 | | /* |
1053 | | * This function sets up the local APIC timer, with a timeout of |
1054 | | * 'clocks' APIC bus clock. During calibration we actually call |
1055 | | * this function twice on the boot CPU, once with a bogus timeout |
1056 | | * value, second time for real. The other (noncalibrating) CPUs |
1057 | | * call this function only once, with the real, calibrated value. |
1058 | | * |
1059 | | * We do reads before writes even if unnecessary, to get around the |
1060 | | * P5 APIC double write bug. |
1061 | | */ |
1062 | | |
1063 | 15 | #define APIC_DIVISOR 1 |
1064 | | |
1065 | | static void __setup_APIC_LVTT(unsigned int clocks) |
1066 | 14 | { |
1067 | 14 | unsigned int lvtt_value, tmp_value; |
1068 | 14 | |
1069 | 14 | /* NB. Xen uses local APIC timer in one-shot mode. */ |
1070 | 14 | lvtt_value = /*APIC_TIMER_MODE_PERIODIC |*/ LOCAL_TIMER_VECTOR; |
1071 | 14 | |
1072 | 14 | if ( tdt_enabled ) |
1073 | 12 | { |
1074 | 12 | lvtt_value &= (~APIC_TIMER_MODE_MASK); |
1075 | 12 | lvtt_value |= APIC_TIMER_MODE_TSC_DEADLINE; |
1076 | 12 | } |
1077 | 14 | |
1078 | 14 | apic_write(APIC_LVTT, lvtt_value); |
1079 | 14 | |
1080 | 14 | tmp_value = apic_read(APIC_TDCR); |
1081 | 14 | apic_write(APIC_TDCR, tmp_value | APIC_TDR_DIV_1); |
1082 | 14 | |
1083 | 14 | apic_write(APIC_TMICT, clocks / APIC_DIVISOR); |
1084 | 14 | } |
1085 | | |
1086 | | static void setup_APIC_timer(void) |
1087 | 12 | { |
1088 | 12 | unsigned long flags; |
1089 | 12 | local_irq_save(flags); |
1090 | 12 | __setup_APIC_LVTT(0); |
1091 | 12 | local_irq_restore(flags); |
1092 | 12 | } |
1093 | | |
1094 | | /* |
1095 | | * In this function we calibrate APIC bus clocks to the external |
1096 | | * timer. Unfortunately we cannot use jiffies and the timer irq |
1097 | | * to calibrate, since some later bootup code depends on getting |
1098 | | * the first irq? Ugh. |
1099 | | * |
1100 | | * We want to do the calibration only once since we |
1101 | | * want to have local timer irqs syncron. CPUs connected |
1102 | | * by the same APIC bus have the very same bus frequency. |
1103 | | * And we want to have irqs off anyways, no accidental |
1104 | | * APIC irq that way. |
1105 | | */ |
1106 | | |
1107 | | static int __init calibrate_APIC_clock(void) |
1108 | 1 | { |
1109 | 1 | unsigned long long t1, t2; |
1110 | 1 | long tt1, tt2; |
1111 | 1 | long result; |
1112 | 1 | int i; |
1113 | 1 | unsigned long bus_freq; /* KAF: pointer-size avoids compile warns. */ |
1114 | 1 | u32 bus_cycle; /* length of one bus cycle in pico-seconds */ |
1115 | 1 | const int LOOPS = HZ/10; |
1116 | 1 | |
1117 | 1 | apic_printk(APIC_VERBOSE, "calibrating APIC timer ...\n"); |
1118 | 1 | |
1119 | 1 | /* |
1120 | 1 | * Put whatever arbitrary (but long enough) timeout |
1121 | 1 | * value into the APIC clock, we just want to get the |
1122 | 1 | * counter running for calibration. |
1123 | 1 | */ |
1124 | 1 | __setup_APIC_LVTT(1000000000); |
1125 | 1 | |
1126 | 1 | /* |
1127 | 1 | * The timer chip counts down to zero. Let's wait |
1128 | 1 | * for a wraparound to start exact measurement: |
1129 | 1 | * (the current tick might have been already half done) |
1130 | 1 | */ |
1131 | 1 | wait_8254_wraparound(); |
1132 | 1 | |
1133 | 1 | /* |
1134 | 1 | * We wrapped around just now. Let's start: |
1135 | 1 | */ |
1136 | 1 | t1 = rdtsc_ordered(); |
1137 | 1 | tt1 = apic_read(APIC_TMCCT); |
1138 | 1 | |
1139 | 1 | /* |
1140 | 1 | * Let's wait LOOPS wraprounds: |
1141 | 1 | */ |
1142 | 11 | for (i = 0; i < LOOPS; i++) |
1143 | 10 | wait_8254_wraparound(); |
1144 | 1 | |
1145 | 1 | tt2 = apic_read(APIC_TMCCT); |
1146 | 1 | t2 = rdtsc_ordered(); |
1147 | 1 | |
1148 | 1 | /* |
1149 | 1 | * The APIC bus clock counter is 32 bits only, it |
1150 | 1 | * might have overflown, but note that we use signed |
1151 | 1 | * longs, thus no extra care needed. |
1152 | 1 | * |
1153 | 1 | * underflown to be exact, as the timer counts down ;) |
1154 | 1 | */ |
1155 | 1 | |
1156 | 1 | result = (tt1-tt2)*APIC_DIVISOR/LOOPS; |
1157 | 1 | |
1158 | 1 | apic_printk(APIC_VERBOSE, "..... CPU clock speed is %ld.%04ld MHz.\n", |
1159 | 0 | ((long)(t2 - t1) / LOOPS) / (1000000 / HZ), |
1160 | 0 | ((long)(t2 - t1) / LOOPS) % (1000000 / HZ)); |
1161 | 1 | |
1162 | 1 | apic_printk(APIC_VERBOSE, "..... host bus clock speed is %ld.%04ld MHz.\n", |
1163 | 0 | result / (1000000 / HZ), result % (1000000 / HZ)); |
1164 | 1 | |
1165 | 1 | /* set up multipliers for accurate timer code */ |
1166 | 1 | bus_freq = result*HZ; |
1167 | 1 | bus_cycle = (u32) (1000000000000LL/bus_freq); /* in pico seconds */ |
1168 | 1 | bus_scale = (1000*262144)/bus_cycle; |
1169 | 1 | |
1170 | 1 | apic_printk(APIC_VERBOSE, "..... bus_scale = %#x\n", bus_scale); |
1171 | 1 | /* reset APIC to zero timeout value */ |
1172 | 1 | __setup_APIC_LVTT(0); |
1173 | 1 | |
1174 | 1 | return result; |
1175 | 1 | } |
1176 | | |
1177 | | void __init setup_boot_APIC_clock(void) |
1178 | 1 | { |
1179 | 1 | unsigned long flags; |
1180 | 1 | apic_printk(APIC_VERBOSE, "Using local APIC timer interrupts.\n"); |
1181 | 1 | using_apic_timer = true; |
1182 | 1 | |
1183 | 1 | local_irq_save(flags); |
1184 | 1 | |
1185 | 1 | calibrate_APIC_clock(); |
1186 | 1 | |
1187 | 1 | if ( tdt_enable && boot_cpu_has(X86_FEATURE_TSC_DEADLINE) ) |
1188 | 1 | { |
1189 | 1 | printk(KERN_DEBUG "TSC deadline timer enabled\n"); |
1190 | 1 | tdt_enabled = true; |
1191 | 1 | } |
1192 | 1 | |
1193 | 1 | setup_APIC_timer(); |
1194 | 1 | |
1195 | 1 | local_irq_restore(flags); |
1196 | 1 | } |
1197 | | |
1198 | | void setup_secondary_APIC_clock(void) |
1199 | 11 | { |
1200 | 11 | setup_APIC_timer(); |
1201 | 11 | } |
1202 | | |
1203 | | void disable_APIC_timer(void) |
1204 | 0 | { |
1205 | 0 | if (using_apic_timer) { |
1206 | 0 | unsigned long v; |
1207 | 0 |
|
1208 | 0 | /* Work around AMD Erratum 411. This is a nice thing to do anyway. */ |
1209 | 0 | apic_write(APIC_TMICT, 0); |
1210 | 0 |
|
1211 | 0 | v = apic_read(APIC_LVTT); |
1212 | 0 | apic_write(APIC_LVTT, v | APIC_LVT_MASKED); |
1213 | 0 | } |
1214 | 0 | } |
1215 | | |
1216 | | void enable_APIC_timer(void) |
1217 | 0 | { |
1218 | 0 | if (using_apic_timer) { |
1219 | 0 | unsigned long v; |
1220 | 0 | |
1221 | 0 | v = apic_read(APIC_LVTT); |
1222 | 0 | apic_write(APIC_LVTT, v & ~APIC_LVT_MASKED); |
1223 | 0 | } |
1224 | 0 | } |
1225 | | |
1226 | | #undef APIC_DIVISOR |
1227 | | |
1228 | | /* |
1229 | | * reprogram_timer: Reprogram the APIC timer. |
1230 | | * Timeout is a Xen system time (nanoseconds since boot); 0 disables the timer. |
1231 | | * Returns 1 on success; 0 if the timeout is too soon or is in the past. |
1232 | | */ |
1233 | | int reprogram_timer(s_time_t timeout) |
1234 | 3.57M | { |
1235 | 3.57M | s_time_t expire; |
1236 | 3.57M | u32 apic_tmict = 0; |
1237 | 3.57M | |
1238 | 3.57M | /* No local APIC: timer list is polled via the PIT interrupt. */ |
1239 | 3.57M | if ( !cpu_has_apic ) |
1240 | 0 | return 1; |
1241 | 3.57M | |
1242 | 3.57M | if ( tdt_enabled ) |
1243 | 3.64M | { |
1244 | 3.64M | wrmsrl(MSR_IA32_TSC_DEADLINE, timeout ? stime2tsc(timeout) : 0); |
1245 | 3.64M | return 1; |
1246 | 3.64M | } |
1247 | 3.57M | |
1248 | 18.4E | if ( timeout && ((expire = timeout - NOW()) > 0) ) |
1249 | 0 | apic_tmict = min_t(u64, (bus_scale * expire) >> 18, UINT_MAX); |
1250 | 18.4E | |
1251 | 18.4E | apic_write(APIC_TMICT, (unsigned long)apic_tmict); |
1252 | 18.4E | |
1253 | 0 | return apic_tmict || !timeout; |
1254 | 3.57M | } |
1255 | | |
1256 | | void apic_timer_interrupt(struct cpu_user_regs * regs) |
1257 | 25.8k | { |
1258 | 25.8k | ack_APIC_irq(); |
1259 | 25.8k | perfc_incr(apic_timer); |
1260 | 25.8k | raise_softirq(TIMER_SOFTIRQ); |
1261 | 25.8k | } |
1262 | | |
1263 | | static DEFINE_PER_CPU(bool, state_dump_pending); |
1264 | | |
1265 | | void smp_send_state_dump(unsigned int cpu) |
1266 | 0 | { |
1267 | 0 | /* We overload the spurious interrupt handler to handle the dump. */ |
1268 | 0 | per_cpu(state_dump_pending, cpu) = true; |
1269 | 0 | send_IPI_mask(cpumask_of(cpu), SPURIOUS_APIC_VECTOR); |
1270 | 0 | } |
1271 | | |
1272 | | /* |
1273 | | * Spurious interrupts should _never_ happen with our APIC/SMP architecture. |
1274 | | */ |
1275 | | void spurious_interrupt(struct cpu_user_regs *regs) |
1276 | 0 | { |
1277 | 0 | /* |
1278 | 0 | * Check if this is a vectored interrupt (most likely, as this is probably |
1279 | 0 | * a request to dump local CPU state). Vectored interrupts are ACKed; |
1280 | 0 | * spurious interrupts are not. |
1281 | 0 | */ |
1282 | 0 | if (apic_isr_read(SPURIOUS_APIC_VECTOR)) { |
1283 | 0 | ack_APIC_irq(); |
1284 | 0 | if (this_cpu(state_dump_pending)) { |
1285 | 0 | this_cpu(state_dump_pending) = false; |
1286 | 0 | dump_execstate(regs); |
1287 | 0 | goto out; |
1288 | 0 | } |
1289 | 0 | } |
1290 | 0 |
|
1291 | 0 | /* see sw-dev-man vol 3, chapter 7.4.13.5 */ |
1292 | 0 | printk(KERN_INFO "spurious APIC interrupt on CPU#%d, should " |
1293 | 0 | "never happen.\n", smp_processor_id()); |
1294 | 0 |
|
1295 | 0 | out: ; |
1296 | 0 | } |
1297 | | |
1298 | | /* |
1299 | | * This interrupt should never happen with our APIC/SMP architecture |
1300 | | */ |
1301 | | |
1302 | | void error_interrupt(struct cpu_user_regs *regs) |
1303 | 0 | { |
1304 | 0 | static const char *const esr_fields[] = { |
1305 | 0 | "Send CS error", |
1306 | 0 | "Receive CS error", |
1307 | 0 | "Send accept error", |
1308 | 0 | "Receive accept error", |
1309 | 0 | "Redirectable IPI", |
1310 | 0 | "Send illegal vector", |
1311 | 0 | "Received illegal vector", |
1312 | 0 | "Illegal register address", |
1313 | 0 | }; |
1314 | 0 | unsigned int v, v1; |
1315 | 0 | int i; |
1316 | 0 |
|
1317 | 0 | /* First tickle the hardware, only then report what went on. -- REW */ |
1318 | 0 | v = apic_read(APIC_ESR); |
1319 | 0 | apic_write(APIC_ESR, 0); |
1320 | 0 | v1 = apic_read(APIC_ESR); |
1321 | 0 | ack_APIC_irq(); |
1322 | 0 |
|
1323 | 0 | printk(XENLOG_DEBUG "APIC error on CPU%u: %02x(%02x)", |
1324 | 0 | smp_processor_id(), v , v1); |
1325 | 0 | for ( i = 7; i >= 0; --i ) |
1326 | 0 | if ( v1 & (1 << i) ) |
1327 | 0 | printk(", %s", esr_fields[i]); |
1328 | 0 | printk("\n"); |
1329 | 0 | } |
1330 | | |
1331 | | /* |
1332 | | * This interrupt handles performance counters interrupt |
1333 | | */ |
1334 | | |
1335 | | void pmu_apic_interrupt(struct cpu_user_regs *regs) |
1336 | 0 | { |
1337 | 0 | ack_APIC_irq(); |
1338 | 0 | vpmu_do_interrupt(regs); |
1339 | 0 | } |
1340 | | |
1341 | | /* |
1342 | | * This initializes the IO-APIC and APIC hardware if this is |
1343 | | * a UP kernel. |
1344 | | */ |
1345 | | int __init APIC_init_uniprocessor (void) |
1346 | 0 | { |
1347 | 0 | if (enable_local_apic < 0) |
1348 | 0 | setup_clear_cpu_cap(X86_FEATURE_APIC); |
1349 | 0 |
|
1350 | 0 | if (!smp_found_config && !cpu_has_apic) { |
1351 | 0 | skip_ioapic_setup = true; |
1352 | 0 | return -1; |
1353 | 0 | } |
1354 | 0 |
|
1355 | 0 | /* |
1356 | 0 | * Complain if the BIOS pretends there is one. |
1357 | 0 | */ |
1358 | 0 | if (!cpu_has_apic) { |
1359 | 0 | printk(KERN_ERR "BIOS bug, local APIC #%d not detected!...\n", |
1360 | 0 | boot_cpu_physical_apicid); |
1361 | 0 | skip_ioapic_setup = true; |
1362 | 0 | return -1; |
1363 | 0 | } |
1364 | 0 |
|
1365 | 0 | verify_local_APIC(); |
1366 | 0 |
|
1367 | 0 | connect_bsp_APIC(); |
1368 | 0 |
|
1369 | 0 | /* |
1370 | 0 | * Hack: In case of kdump, after a crash, kernel might be booting |
1371 | 0 | * on a cpu with non-zero lapic id. But boot_cpu_physical_apicid |
1372 | 0 | * might be zero if read from MP tables. Get it from LAPIC. |
1373 | 0 | */ |
1374 | 0 | #ifdef CONFIG_CRASH_DUMP |
1375 | | boot_cpu_physical_apicid = get_apic_id(); |
1376 | | #endif |
1377 | 0 | physids_clear(phys_cpu_present_map); |
1378 | 0 | physid_set(boot_cpu_physical_apicid, phys_cpu_present_map); |
1379 | 0 |
|
1380 | 0 | setup_local_APIC(); |
1381 | 0 |
|
1382 | 0 | if (nmi_watchdog == NMI_LOCAL_APIC) |
1383 | 0 | check_nmi_watchdog(); |
1384 | 0 |
|
1385 | 0 | if (smp_found_config) |
1386 | 0 | if (!skip_ioapic_setup && nr_ioapics) |
1387 | 0 | setup_IO_APIC(); |
1388 | 0 |
|
1389 | 0 | setup_boot_APIC_clock(); |
1390 | 0 |
|
1391 | 0 | return 0; |
1392 | 0 | } |
1393 | | |
1394 | | static const char * __init apic_mode_to_str(const enum apic_mode mode) |
1395 | 0 | { |
1396 | 0 | switch ( mode ) |
1397 | 0 | { |
1398 | 0 | case APIC_MODE_INVALID: |
1399 | 0 | return "invalid"; |
1400 | 0 | case APIC_MODE_DISABLED: |
1401 | 0 | return "disabled"; |
1402 | 0 | case APIC_MODE_XAPIC: |
1403 | 0 | return "xapic"; |
1404 | 0 | case APIC_MODE_X2APIC: |
1405 | 0 | return "x2apic"; |
1406 | 0 | default: |
1407 | 0 | return "unrecognised"; |
1408 | 0 | } |
1409 | 0 | } |
1410 | | |
1411 | | /* Needs to be called during startup. It records the state the BIOS |
1412 | | * leaves the local APIC so we can undo upon kexec. |
1413 | | */ |
1414 | | void __init record_boot_APIC_mode(void) |
1415 | 1 | { |
1416 | 1 | /* Sanity check - we should only ever run once, but could possibly |
1417 | 1 | * be called several times */ |
1418 | 1 | if ( APIC_MODE_INVALID != apic_boot_mode ) |
1419 | 0 | return; |
1420 | 1 | |
1421 | 1 | apic_boot_mode = current_local_apic_mode(); |
1422 | 1 | |
1423 | 1 | apic_printk(APIC_DEBUG, "APIC boot state is '%s'\n", |
1424 | 1 | apic_mode_to_str(apic_boot_mode)); |
1425 | 1 | } |
1426 | | |
1427 | | /* Look at the bits in MSR_IA32_APICBASE and work out which |
1428 | | * APIC mode we are in */ |
1429 | | enum apic_mode current_local_apic_mode(void) |
1430 | 1 | { |
1431 | 1 | u64 msr_contents; |
1432 | 1 | |
1433 | 1 | rdmsrl(MSR_IA32_APICBASE, msr_contents); |
1434 | 1 | |
1435 | 1 | /* Reading EXTD bit from the MSR is only valid if CPUID |
1436 | 1 | * says so, else reserved */ |
1437 | 1 | if ( boot_cpu_has(X86_FEATURE_X2APIC) |
1438 | 1 | && (msr_contents & MSR_IA32_APICBASE_EXTD) ) |
1439 | 0 | return APIC_MODE_X2APIC; |
1440 | 1 | |
1441 | 1 | /* EN bit should always be valid as long as we can read the MSR |
1442 | 1 | */ |
1443 | 1 | if ( msr_contents & MSR_IA32_APICBASE_ENABLE ) |
1444 | 1 | return APIC_MODE_XAPIC; |
1445 | 1 | |
1446 | 0 | return APIC_MODE_DISABLED; |
1447 | 1 | } |
1448 | | |
1449 | | |
1450 | | void check_for_unexpected_msi(unsigned int vector) |
1451 | 0 | { |
1452 | 0 | BUG_ON(apic_isr_read(vector)); |
1453 | 0 | } |