debuggers.hg

view xen/arch/x86/smp.c @ 3674:fb875591fd72

bitkeeper revision 1.1159.223.63 (42028527-fv-d9BM0_LRp8UKGP19gQ)

Fix NMI deferral.
Signed-off-by: keir.fraser@cl.cam.ac.uk
author kaf24@scramble.cl.cam.ac.uk
date Thu Feb 03 20:10:15 2005 +0000 (2005-02-03)
parents f65b65977b19
children 42bdac6c8985 f620c41a1fef 0dc3b8b8c298
line source
1 /*
2 * Intel SMP support routines.
3 *
4 * (c) 1995 Alan Cox, Building #3 <alan@redhat.com>
5 * (c) 1998-99, 2000 Ingo Molnar <mingo@redhat.com>
6 *
7 * This code is released under the GNU General Public License version 2 or
8 * later.
9 */
11 #include <xen/irq.h>
12 #include <xen/sched.h>
13 #include <xen/delay.h>
14 #include <xen/spinlock.h>
15 #include <asm/smp.h>
16 #include <asm/mc146818rtc.h>
17 #include <asm/flushtlb.h>
18 #include <asm/smpboot.h>
19 #include <asm/hardirq.h>
21 #ifdef CONFIG_SMP
23 /*
24 * Some notes on x86 processor bugs affecting SMP operation:
25 *
26 * Pentium, Pentium Pro, II, III (and all CPUs) have bugs.
27 * The Linux implications for SMP are handled as follows:
28 *
29 * Pentium III / [Xeon]
30 * None of the E1AP-E3AP errata are visible to the user.
31 *
32 * E1AP. see PII A1AP
33 * E2AP. see PII A2AP
34 * E3AP. see PII A3AP
35 *
36 * Pentium II / [Xeon]
37 * None of the A1AP-A3AP errata are visible to the user.
38 *
39 * A1AP. see PPro 1AP
40 * A2AP. see PPro 2AP
41 * A3AP. see PPro 7AP
42 *
43 * Pentium Pro
44 * None of 1AP-9AP errata are visible to the normal user,
45 * except occasional delivery of 'spurious interrupt' as trap #15.
46 * This is very rare and a non-problem.
47 *
48 * 1AP. Linux maps APIC as non-cacheable
49 * 2AP. worked around in hardware
50 * 3AP. fixed in C0 and above steppings microcode update.
51 * Linux does not use excessive STARTUP_IPIs.
52 * 4AP. worked around in hardware
53 * 5AP. symmetric IO mode (normal Linux operation) not affected.
54 * 'noapic' mode has vector 0xf filled out properly.
55 * 6AP. 'noapic' mode might be affected - fixed in later steppings
56 * 7AP. We do not assume writes to the LVT deassering IRQs
57 * 8AP. We do not enable low power mode (deep sleep) during MP bootup
58 * 9AP. We do not use mixed mode
59 */
61 /*
62 * the following functions deal with sending IPIs between CPUs.
63 *
64 * We use 'broadcast', CPU->CPU IPIs and self-IPIs too.
65 */
67 static inline int __prepare_ICR (unsigned int shortcut, int vector)
68 {
69 return APIC_DM_FIXED | shortcut | vector | APIC_DEST_LOGICAL;
70 }
72 static inline int __prepare_ICR2 (unsigned int mask)
73 {
74 return SET_APIC_DEST_FIELD(mask);
75 }
77 static inline void __send_IPI_shortcut(unsigned int shortcut, int vector)
78 {
79 /*
80 * Subtle. In the case of the 'never do double writes' workaround
81 * we have to lock out interrupts to be safe. As we don't care
82 * of the value read we use an atomic rmw access to avoid costly
83 * cli/sti. Otherwise we use an even cheaper single atomic write
84 * to the APIC.
85 */
86 unsigned int cfg;
88 /*
89 * Wait for idle.
90 */
91 apic_wait_icr_idle();
93 /*
94 * No need to touch the target chip field
95 */
96 cfg = __prepare_ICR(shortcut, vector);
98 /*
99 * Send the IPI. The write to APIC_ICR fires this off.
100 */
101 apic_write_around(APIC_ICR, cfg);
102 }
104 void send_IPI_self(int vector)
105 {
106 __send_IPI_shortcut(APIC_DEST_SELF, vector);
107 }
109 static inline void send_IPI_mask(int mask, int vector)
110 {
111 unsigned long cfg;
112 unsigned long flags;
114 __save_flags(flags);
115 __cli();
118 /*
119 * Wait for idle.
120 */
121 apic_wait_icr_idle();
123 /*
124 * prepare target chip field
125 */
126 cfg = __prepare_ICR2(mask);
127 apic_write_around(APIC_ICR2, cfg);
129 /*
130 * program the ICR
131 */
132 cfg = __prepare_ICR(0, vector);
134 /*
135 * Send the IPI. The write to APIC_ICR fires this off.
136 */
137 apic_write_around(APIC_ICR, cfg);
139 __restore_flags(flags);
140 }
142 static inline void send_IPI_allbutself(int vector)
143 {
144 /*
145 * if there are no other CPUs in the system then
146 * we get an APIC send error if we try to broadcast.
147 * thus we have to avoid sending IPIs in this case.
148 */
149 if (!(smp_num_cpus > 1))
150 return;
152 __send_IPI_shortcut(APIC_DEST_ALLBUT, vector);
153 }
155 /*
156 * ********* XEN NOTICE **********
157 * I've left the following comments lying around as they look liek they might
158 * be useful to get multiprocessor guest OSes going. However, I suspect the
159 * issues we face will be quite different so I've ripped out all the
160 * TLBSTATE logic (I didn't understand it anyway :-). These comments do
161 * not apply to Xen, therefore! -- Keir (8th Oct 2003).
162 */
163 /*
164 * Smarter SMP flushing macros.
165 * c/o Linus Torvalds.
166 *
167 * These mean you can really definitely utterly forget about
168 * writing to user space from interrupts. (Its not allowed anyway).
169 *
170 * Optimizations Manfred Spraul <manfred@colorfullife.com>
171 *
172 * The flush IPI assumes that a thread switch happens in this order:
173 * [cpu0: the cpu that switches]
174 * 1) switch_mm() either 1a) or 1b)
175 * 1a) thread switch to a different mm
176 * 1a1) clear_bit(cpu, &old_mm.cpu_vm_mask);
177 * Stop ipi delivery for the old mm. This is not synchronized with
178 * the other cpus, but smp_invalidate_interrupt ignore flush ipis
179 * for the wrong mm, and in the worst case we perform a superflous
180 * tlb flush.
181 * 1a2) set cpu_tlbstate to TLBSTATE_OK
182 * Now the smp_invalidate_interrupt won't call leave_mm if cpu0
183 * was in lazy tlb mode.
184 * 1a3) update cpu_tlbstate[].active_mm
185 * Now cpu0 accepts tlb flushes for the new mm.
186 * 1a4) set_bit(cpu, &new_mm.cpu_vm_mask);
187 * Now the other cpus will send tlb flush ipis.
188 * 1a4) change cr3.
189 * 1b) thread switch without mm change
190 * cpu_tlbstate[].active_mm is correct, cpu0 already handles
191 * flush ipis.
192 * 1b1) set cpu_tlbstate to TLBSTATE_OK
193 * 1b2) test_and_set the cpu bit in cpu_vm_mask.
194 * Atomically set the bit [other cpus will start sending flush ipis],
195 * and test the bit.
196 * 1b3) if the bit was 0: leave_mm was called, flush the tlb.
197 * 2) switch %%esp, ie current
198 *
199 * The interrupt must handle 2 special cases:
200 * - cr3 is changed before %%esp, ie. it cannot use current->{active_,}mm.
201 * - the cpu performs speculative tlb reads, i.e. even if the cpu only
202 * runs in kernel space, the cpu could load tlb entries for user space
203 * pages.
204 *
205 * The good news is that cpu_tlbstate is local to each cpu, no
206 * write/read ordering problems.
207 *
208 * TLB flush IPI:
209 *
210 * 1) Flush the tlb entries if the cpu uses the mm that's being flushed.
211 * 2) Leave the mm if we are in the lazy tlb mode.
212 */
214 static spinlock_t flush_lock = SPIN_LOCK_UNLOCKED;
215 static unsigned long flush_cpumask;
217 asmlinkage void smp_invalidate_interrupt(void)
218 {
219 ack_APIC_irq();
220 perfc_incrc(ipis);
221 local_flush_tlb();
222 clear_bit(smp_processor_id(), &flush_cpumask);
223 }
225 void flush_tlb_mask(unsigned long mask)
226 {
227 ASSERT(local_irq_is_enabled());
229 if ( mask & (1 << smp_processor_id()) )
230 {
231 local_flush_tlb();
232 mask &= ~(1 << smp_processor_id());
233 }
235 if ( mask != 0 )
236 {
237 spin_lock(&flush_lock);
239 flush_cpumask = mask;
240 send_IPI_mask(mask, INVALIDATE_TLB_VECTOR);
241 while ( flush_cpumask != 0 )
242 {
243 rep_nop();
244 barrier();
245 }
247 spin_unlock(&flush_lock);
248 }
249 }
251 /* Call with no locks held and interrupts enabled (e.g., softirq context). */
252 void new_tlbflush_clock_period(void)
253 {
254 ASSERT(local_irq_is_enabled());
256 /* Flush everyone else. We definitely flushed just before entry. */
257 if ( smp_num_cpus > 1 )
258 {
259 spin_lock(&flush_lock);
260 flush_cpumask = ((1 << smp_num_cpus) - 1) & ~(1 << smp_processor_id());
261 send_IPI_allbutself(INVALIDATE_TLB_VECTOR);
262 while ( flush_cpumask != 0 )
263 {
264 rep_nop();
265 barrier();
266 }
267 spin_unlock(&flush_lock);
268 }
270 /* No need for atomicity: we are the only possible updater. */
271 ASSERT(tlbflush_clock == 0);
272 tlbflush_clock++;
273 }
275 static void flush_tlb_all_pge_ipi(void* info)
276 {
277 __flush_tlb_pge();
278 }
280 void flush_tlb_all_pge(void)
281 {
282 smp_call_function (flush_tlb_all_pge_ipi,0,1,1);
283 __flush_tlb_pge();
284 }
286 void smp_send_event_check_mask(unsigned long cpu_mask)
287 {
288 cpu_mask &= ~(1<<smp_processor_id());
289 if ( cpu_mask != 0 )
290 send_IPI_mask(cpu_mask, EVENT_CHECK_VECTOR);
291 }
293 /*
294 * Structure and data for smp_call_function(). This is designed to minimise
295 * static memory requirements. It also looks cleaner.
296 */
297 static spinlock_t call_lock = SPIN_LOCK_UNLOCKED;
299 struct call_data_struct {
300 void (*func) (void *info);
301 void *info;
302 atomic_t started;
303 atomic_t finished;
304 int wait;
305 };
307 static struct call_data_struct * call_data;
309 /*
310 * this function sends a 'generic call function' IPI to all other CPUs
311 * in the system.
312 */
314 int smp_call_function (void (*func) (void *info), void *info, int nonatomic,
315 int wait)
316 /*
317 * [SUMMARY] Run a function on all other CPUs.
318 * <func> The function to run. This must be fast and non-blocking.
319 * <info> An arbitrary pointer to pass to the function.
320 * <nonatomic> currently unused.
321 * <wait> If true, wait (atomically) until function has completed on other CPUs.
322 * [RETURNS] 0 on success, else a negative status code. Does not return until
323 * remote CPUs are nearly ready to execute <<func>> or are or have executed.
324 *
325 * You must not call this function with disabled interrupts or from a
326 * hardware interrupt handler, or bottom halfs.
327 */
328 {
329 struct call_data_struct data;
330 int cpus = smp_num_cpus-1;
332 if (!cpus)
333 return 0;
335 data.func = func;
336 data.info = info;
337 atomic_set(&data.started, 0);
338 data.wait = wait;
339 if (wait)
340 atomic_set(&data.finished, 0);
342 ASSERT(local_irq_is_enabled());
344 spin_lock(&call_lock);
346 call_data = &data;
347 wmb();
348 /* Send a message to all other CPUs and wait for them to respond */
349 send_IPI_allbutself(CALL_FUNCTION_VECTOR);
351 /* Wait for response */
352 while (atomic_read(&data.started) != cpus)
353 barrier();
355 if (wait)
356 while (atomic_read(&data.finished) != cpus)
357 barrier();
359 spin_unlock(&call_lock);
361 return 0;
362 }
364 static void stop_this_cpu (void * dummy)
365 {
366 /*
367 * Remove this CPU:
368 */
369 clear_bit(smp_processor_id(), &cpu_online_map);
370 __cli();
371 disable_local_APIC();
372 for(;;) __asm__("hlt");
373 }
375 /*
376 * this function calls the 'stop' function on all other CPUs in the system.
377 */
379 void smp_send_stop(void)
380 {
381 smp_call_function(stop_this_cpu, NULL, 1, 0);
382 smp_num_cpus = 1;
384 __cli();
385 disable_local_APIC();
386 __sti();
387 }
389 /*
390 * Nothing to do, as all the work is done automatically when
391 * we return from the interrupt.
392 */
393 asmlinkage void smp_event_check_interrupt(void)
394 {
395 ack_APIC_irq();
396 perfc_incrc(ipis);
397 }
399 asmlinkage void smp_call_function_interrupt(void)
400 {
401 void (*func) (void *info) = call_data->func;
402 void *info = call_data->info;
403 int wait = call_data->wait;
405 ack_APIC_irq();
406 perfc_incrc(ipis);
408 /*
409 * Notify initiating CPU that I've grabbed the data and am
410 * about to execute the function
411 */
412 mb();
413 atomic_inc(&call_data->started);
414 /*
415 * At this point the info structure may be out of scope unless wait==1
416 */
417 (*func)(info);
418 if (wait) {
419 mb();
420 atomic_inc(&call_data->finished);
421 }
422 }
424 #endif /* CONFIG_SMP */