cl349@4087: #ifndef __ASM_SYSTEM_H cl349@4087: #define __ASM_SYSTEM_H cl349@4087: cl349@4087: #include cl349@4087: #include cl349@4087: #include kaf24@6760: #include cl349@4087: #include cl349@4087: #include cl349@4087: #include sos22@5466: #include cl349@4087: cl349@4087: #ifdef __KERNEL__ cl349@4087: cl349@4087: struct task_struct; /* one of the stranger aspects of C forward declarations.. */ cl349@4087: extern struct task_struct * FASTCALL(__switch_to(struct task_struct *prev, struct task_struct *next)); cl349@4087: cl349@4087: #define switch_to(prev,next,last) do { \ cl349@4087: unsigned long esi,edi; \ cl349@4087: asm volatile("pushfl\n\t" \ cl349@4087: "pushl %%ebp\n\t" \ cl349@4087: "movl %%esp,%0\n\t" /* save ESP */ \ cl349@4087: "movl %5,%%esp\n\t" /* restore ESP */ \ cl349@4087: "movl $1f,%1\n\t" /* save EIP */ \ cl349@4087: "pushl %6\n\t" /* restore EIP */ \ cl349@4087: "jmp __switch_to\n" \ cl349@4087: "1:\t" \ cl349@4087: "popl %%ebp\n\t" \ cl349@4087: "popfl" \ cl349@4087: :"=m" (prev->thread.esp),"=m" (prev->thread.eip), \ cl349@4087: "=a" (last),"=S" (esi),"=D" (edi) \ cl349@4087: :"m" (next->thread.esp),"m" (next->thread.eip), \ cl349@4087: "2" (prev), "d" (next)); \ cl349@4087: } while (0) cl349@4087: cl349@4087: #define _set_base(addr,base) do { unsigned long __pr; \ cl349@4087: __asm__ __volatile__ ("movw %%dx,%1\n\t" \ cl349@4087: "rorl $16,%%edx\n\t" \ cl349@4087: "movb %%dl,%2\n\t" \ cl349@4087: "movb %%dh,%3" \ cl349@4087: :"=&d" (__pr) \ cl349@4087: :"m" (*((addr)+2)), \ cl349@4087: "m" (*((addr)+4)), \ cl349@4087: "m" (*((addr)+7)), \ cl349@4087: "0" (base) \ cl349@4087: ); } while(0) cl349@4087: cl349@4087: #define _set_limit(addr,limit) do { unsigned long __lr; \ cl349@4087: __asm__ __volatile__ ("movw %%dx,%1\n\t" \ cl349@4087: "rorl $16,%%edx\n\t" \ cl349@4087: "movb %2,%%dh\n\t" \ cl349@4087: "andb $0xf0,%%dh\n\t" \ cl349@4087: "orb %%dh,%%dl\n\t" \ cl349@4087: "movb %%dl,%2" \ cl349@4087: :"=&d" (__lr) \ cl349@4087: :"m" (*(addr)), \ cl349@4087: "m" (*((addr)+6)), \ cl349@4087: "0" (limit) \ cl349@4087: ); } while(0) cl349@4087: cl349@4087: #define set_base(ldt,base) _set_base( ((char *)&(ldt)) , (base) ) cl349@4087: #define set_limit(ldt,limit) _set_limit( ((char *)&(ldt)) , ((limit)-1)>>12 ) cl349@4087: cl349@4087: static inline unsigned long _get_base(char * addr) cl349@4087: { cl349@4087: unsigned long __base; cl349@4087: __asm__("movb %3,%%dh\n\t" cl349@4087: "movb %2,%%dl\n\t" cl349@4087: "shll $16,%%edx\n\t" cl349@4087: "movw %1,%%dx" cl349@4087: :"=&d" (__base) cl349@4087: :"m" (*((addr)+2)), cl349@4087: "m" (*((addr)+4)), cl349@4087: "m" (*((addr)+7))); cl349@4087: return __base; cl349@4087: } cl349@4087: cl349@4087: #define get_base(ldt) _get_base( ((char *)&(ldt)) ) cl349@4087: cl349@4087: /* cl349@4087: * Load a segment. Fall back on loading the zero cl349@4087: * segment if something goes wrong.. cl349@4087: */ cl349@4087: #define loadsegment(seg,value) \ cl349@4087: asm volatile("\n" \ cl349@4087: "1:\t" \ vh249@5730: "mov %0,%%" #seg "\n" \ cl349@4087: "2:\n" \ cl349@4087: ".section .fixup,\"ax\"\n" \ cl349@4087: "3:\t" \ cl349@4087: "pushl $0\n\t" \ cl349@4087: "popl %%" #seg "\n\t" \ cl349@4087: "jmp 2b\n" \ cl349@4087: ".previous\n" \ cl349@4087: ".section __ex_table,\"a\"\n\t" \ cl349@4087: ".align 4\n\t" \ cl349@4087: ".long 1b,3b\n" \ cl349@4087: ".previous" \ vh249@5730: : :"m" (value)) cl349@4087: cl349@4087: /* cl349@4087: * Save a segment register away cl349@4087: */ cl349@4087: #define savesegment(seg, value) \ vh249@5730: asm volatile("mov %%" #seg ",%0":"=m" (value)) cl349@4087: cl349@4087: /* cl349@4087: * Clear and set 'TS' bit respectively cl349@4087: */ cl349@4112: #define clts() (HYPERVISOR_fpu_taskswitch(0)) kaf24@5367: #define read_cr0() ({ \ kaf24@5367: unsigned int __dummy; \ kaf24@5367: __asm__( \ kaf24@5367: "movl %%cr0,%0\n\t" \ kaf24@5367: :"=r" (__dummy)); \ kaf24@5367: __dummy; \ kaf24@5367: }) cl349@4087: #define write_cr0(x) \ kaf24@5367: __asm__("movl %0,%%cr0": :"r" (x)); kaf24@5367: kaf24@5367: #define read_cr4() ({ \ kaf24@5367: unsigned int __dummy; \ kaf24@5367: __asm__( \ kaf24@5367: "movl %%cr4,%0\n\t" \ kaf24@5367: :"=r" (__dummy)); \ kaf24@5367: __dummy; \ kaf24@5367: }) cl349@4087: #define write_cr4(x) \ kaf24@5367: __asm__("movl %0,%%cr4": :"r" (x)); cl349@4112: #define stts() (HYPERVISOR_fpu_taskswitch(1)) cl349@4087: cl349@4087: #endif /* __KERNEL__ */ cl349@4087: cl349@4112: #define wbinvd() \ cl349@4112: __asm__ __volatile__ ("wbinvd": : :"memory"); cl349@4087: cl349@4087: static inline unsigned long get_limit(unsigned long segment) cl349@4087: { cl349@4087: unsigned long __limit; cl349@4087: __asm__("lsll %1,%0" cl349@4087: :"=r" (__limit):"r" (segment)); cl349@4087: return __limit+1; cl349@4087: } cl349@4087: cl349@4087: #define nop() __asm__ __volatile__ ("nop") cl349@4087: cl349@4087: #define xchg(ptr,v) ((__typeof__(*(ptr)))__xchg((unsigned long)(v),(ptr),sizeof(*(ptr)))) cl349@4087: cl349@4087: #define tas(ptr) (xchg((ptr),1)) cl349@4087: cl349@4087: struct __xchg_dummy { unsigned long a[100]; }; cl349@4087: #define __xg(x) ((struct __xchg_dummy *)(x)) cl349@4087: cl349@4087: cl349@4087: /* cl349@4087: * The semantics of XCHGCMP8B are a bit strange, this is why cl349@4087: * there is a loop and the loading of %%eax and %%edx has to cl349@4087: * be inside. This inlines well in most cases, the cached cl349@4087: * cost is around ~38 cycles. (in the future we might want cl349@4087: * to do an SIMD/3DNOW!/MMX/FPU 64-bit store here, but that cl349@4087: * might have an implicit FPU-save as a cost, so it's not cl349@4087: * clear which path to go.) cl349@4087: * cl349@4087: * cmpxchg8b must be used with the lock prefix here to allow cl349@4087: * the instruction to be executed atomically, see page 3-102 cl349@4087: * of the instruction set reference 24319102.pdf. We need cl349@4087: * the reader side to see the coherent 64bit value. cl349@4087: */ cl349@4087: static inline void __set_64bit (unsigned long long * ptr, cl349@4087: unsigned int low, unsigned int high) cl349@4087: { cl349@4087: __asm__ __volatile__ ( cl349@4087: "\n1:\t" cl349@4087: "movl (%0), %%eax\n\t" cl349@4087: "movl 4(%0), %%edx\n\t" cl349@4087: "lock cmpxchg8b (%0)\n\t" cl349@4087: "jnz 1b" cl349@4087: : /* no outputs */ cl349@4087: : "D"(ptr), cl349@4087: "b"(low), cl349@4087: "c"(high) cl349@4087: : "ax","dx","memory"); cl349@4087: } cl349@4087: cl349@4087: static inline void __set_64bit_constant (unsigned long long *ptr, cl349@4087: unsigned long long value) cl349@4087: { cl349@4087: __set_64bit(ptr,(unsigned int)(value), (unsigned int)((value)>>32ULL)); cl349@4087: } cl349@4087: #define ll_low(x) *(((unsigned int*)&(x))+0) cl349@4087: #define ll_high(x) *(((unsigned int*)&(x))+1) cl349@4087: cl349@4087: static inline void __set_64bit_var (unsigned long long *ptr, cl349@4087: unsigned long long value) cl349@4087: { cl349@4087: __set_64bit(ptr,ll_low(value), ll_high(value)); cl349@4087: } cl349@4087: cl349@4087: #define set_64bit(ptr,value) \ cl349@4087: (__builtin_constant_p(value) ? \ cl349@4087: __set_64bit_constant(ptr, value) : \ cl349@4087: __set_64bit_var(ptr, value) ) cl349@4087: cl349@4087: #define _set_64bit(ptr,value) \ cl349@4087: (__builtin_constant_p(value) ? \ cl349@4087: __set_64bit(ptr, (unsigned int)(value), (unsigned int)((value)>>32ULL) ) : \ cl349@4087: __set_64bit(ptr, ll_low(value), ll_high(value)) ) cl349@4087: cl349@4087: /* cl349@4087: * Note: no "lock" prefix even on SMP: xchg always implies lock anyway cl349@4087: * Note 2: xchg has side effect, so that attribute volatile is necessary, cl349@4087: * but generally the primitive is invalid, *ptr is output argument. --ANK cl349@4087: */ cl349@4087: static inline unsigned long __xchg(unsigned long x, volatile void * ptr, int size) cl349@4087: { cl349@4087: switch (size) { cl349@4087: case 1: cl349@4087: __asm__ __volatile__("xchgb %b0,%1" cl349@4087: :"=q" (x) cl349@4087: :"m" (*__xg(ptr)), "0" (x) cl349@4087: :"memory"); cl349@4087: break; cl349@4087: case 2: cl349@4087: __asm__ __volatile__("xchgw %w0,%1" cl349@4087: :"=r" (x) cl349@4087: :"m" (*__xg(ptr)), "0" (x) cl349@4087: :"memory"); cl349@4087: break; cl349@4087: case 4: cl349@4087: __asm__ __volatile__("xchgl %0,%1" cl349@4087: :"=r" (x) cl349@4087: :"m" (*__xg(ptr)), "0" (x) cl349@4087: :"memory"); cl349@4087: break; cl349@4087: } cl349@4087: return x; cl349@4087: } cl349@4087: cl349@4087: /* cl349@4087: * Atomic compare and exchange. Compare OLD with MEM, if identical, cl349@4087: * store NEW in MEM. Return the initial value in MEM. Success is cl349@4087: * indicated by comparing RETURN with OLD. cl349@4087: */ cl349@4087: cl349@4087: #ifdef CONFIG_X86_CMPXCHG cl349@4087: #define __HAVE_ARCH_CMPXCHG 1 cl349@4087: #endif cl349@4087: cl349@4087: static inline unsigned long __cmpxchg(volatile void *ptr, unsigned long old, cl349@4087: unsigned long new, int size) cl349@4087: { cl349@4087: unsigned long prev; cl349@4087: switch (size) { cl349@4087: case 1: sos22@5466: __asm__ __volatile__(LOCK "cmpxchgb %b1,%2" cl349@4087: : "=a"(prev) cl349@4087: : "q"(new), "m"(*__xg(ptr)), "0"(old) cl349@4087: : "memory"); cl349@4087: return prev; cl349@4087: case 2: sos22@5466: __asm__ __volatile__(LOCK "cmpxchgw %w1,%2" cl349@4087: : "=a"(prev) cl349@4087: : "q"(new), "m"(*__xg(ptr)), "0"(old) cl349@4087: : "memory"); cl349@4087: return prev; cl349@4087: case 4: sos22@5466: __asm__ __volatile__(LOCK "cmpxchgl %1,%2" cl349@4087: : "=a"(prev) cl349@4087: : "q"(new), "m"(*__xg(ptr)), "0"(old) cl349@4087: : "memory"); cl349@4087: return prev; cl349@4087: } cl349@4087: return old; cl349@4087: } cl349@4087: cl349@4087: #define cmpxchg(ptr,o,n)\ cl349@4087: ((__typeof__(*(ptr)))__cmpxchg((ptr),(unsigned long)(o),\ cl349@4087: (unsigned long)(n),sizeof(*(ptr)))) cl349@4087: cl349@4087: #ifdef __KERNEL__ cl349@4087: struct alt_instr { cl349@4087: __u8 *instr; /* original instruction */ cl349@4087: __u8 *replacement; cl349@4087: __u8 cpuid; /* cpuid bit set for replacement */ cl349@4087: __u8 instrlen; /* length of original instruction */ cl349@4087: __u8 replacementlen; /* length of new instruction, <= instrlen */ cl349@4087: __u8 pad; cl349@4087: }; cl349@4087: #endif cl349@4087: cl349@4087: /* cl349@4087: * Alternative instructions for different CPU types or capabilities. cl349@4087: * cl349@4087: * This allows to use optimized instructions even on generic binary cl349@4087: * kernels. cl349@4087: * cl349@4087: * length of oldinstr must be longer or equal the length of newinstr cl349@4087: * It can be padded with nops as needed. cl349@4087: * cl349@4087: * For non barrier like inlines please define new variants cl349@4087: * without volatile and memory clobber. cl349@4087: */ cl349@4087: #define alternative(oldinstr, newinstr, feature) \ cl349@4087: asm volatile ("661:\n\t" oldinstr "\n662:\n" \ cl349@4087: ".section .altinstructions,\"a\"\n" \ cl349@4087: " .align 4\n" \ cl349@4087: " .long 661b\n" /* label */ \ cl349@4087: " .long 663f\n" /* new instruction */ \ cl349@4087: " .byte %c0\n" /* feature bit */ \ cl349@4087: " .byte 662b-661b\n" /* sourcelen */ \ cl349@4087: " .byte 664f-663f\n" /* replacementlen */ \ cl349@4087: ".previous\n" \ cl349@4087: ".section .altinstr_replacement,\"ax\"\n" \ cl349@4087: "663:\n\t" newinstr "\n664:\n" /* replacement */ \ cl349@4087: ".previous" :: "i" (feature) : "memory") cl349@4087: cl349@4087: /* cl349@4087: * Alternative inline assembly with input. cl349@4087: * cl349@4087: * Pecularities: cl349@4087: * No memory clobber here. cl349@4087: * Argument numbers start with 1. cl349@4087: * Best is to use constraints that are fixed size (like (%1) ... "r") cl349@4087: * If you use variable sized constraints like "m" or "g" in the cl349@4087: * replacement maake sure to pad to the worst case length. cl349@4087: */ cl349@4087: #define alternative_input(oldinstr, newinstr, feature, input...) \ cl349@4087: asm volatile ("661:\n\t" oldinstr "\n662:\n" \ cl349@4087: ".section .altinstructions,\"a\"\n" \ cl349@4087: " .align 4\n" \ cl349@4087: " .long 661b\n" /* label */ \ cl349@4087: " .long 663f\n" /* new instruction */ \ cl349@4087: " .byte %c0\n" /* feature bit */ \ cl349@4087: " .byte 662b-661b\n" /* sourcelen */ \ cl349@4087: " .byte 664f-663f\n" /* replacementlen */ \ cl349@4087: ".previous\n" \ cl349@4087: ".section .altinstr_replacement,\"ax\"\n" \ cl349@4087: "663:\n\t" newinstr "\n664:\n" /* replacement */ \ cl349@4087: ".previous" :: "i" (feature), ##input) cl349@4087: cl349@4087: /* cl349@4087: * Force strict CPU ordering. cl349@4087: * And yes, this is required on UP too when we're talking cl349@4087: * to devices. cl349@4087: * cl349@4087: * For now, "wmb()" doesn't actually do anything, as all cl349@4087: * Intel CPU's follow what Intel calls a *Processor Order*, cl349@4087: * in which all writes are seen in the program order even cl349@4087: * outside the CPU. cl349@4087: * cl349@4087: * I expect future Intel CPU's to have a weaker ordering, cl349@4087: * but I'd also expect them to finally get their act together cl349@4087: * and add some real memory barriers if so. cl349@4087: * cl349@4087: * Some non intel clones support out of order store. wmb() ceases to be a cl349@4087: * nop for these. cl349@4087: */ cl349@4087: cl349@4087: cl349@4087: /* cl349@4087: * Actually only lfence would be needed for mb() because all stores done cl349@4087: * by the kernel should be already ordered. But keep a full barrier for now. cl349@4087: */ cl349@4087: cl349@4087: #define mb() alternative("lock; addl $0,0(%%esp)", "mfence", X86_FEATURE_XMM2) cl349@4087: #define rmb() alternative("lock; addl $0,0(%%esp)", "lfence", X86_FEATURE_XMM2) cl349@4087: cl349@4087: /** cl349@4087: * read_barrier_depends - Flush all pending reads that subsequents reads cl349@4087: * depend on. cl349@4087: * cl349@4087: * No data-dependent reads from memory-like regions are ever reordered cl349@4087: * over this barrier. All reads preceding this primitive are guaranteed cl349@4087: * to access memory (but not necessarily other CPUs' caches) before any cl349@4087: * reads following this primitive that depend on the data return by cl349@4087: * any of the preceding reads. This primitive is much lighter weight than cl349@4087: * rmb() on most CPUs, and is never heavier weight than is cl349@4087: * rmb(). cl349@4087: * cl349@4087: * These ordering constraints are respected by both the local CPU cl349@4087: * and the compiler. cl349@4087: * cl349@4087: * Ordering is not guaranteed by anything other than these primitives, cl349@4087: * not even by data dependencies. See the documentation for cl349@4087: * memory_barrier() for examples and URLs to more information. cl349@4087: * cl349@4087: * For example, the following code would force ordering (the initial cl349@4087: * value of "a" is zero, "b" is one, and "p" is "&a"): cl349@4087: * cl349@4087: * cl349@4087: * CPU 0 CPU 1 cl349@4087: * cl349@4087: * b = 2; cl349@4087: * memory_barrier(); cl349@4087: * p = &b; q = p; cl349@4087: * read_barrier_depends(); cl349@4087: * d = *q; cl349@4087: * cl349@4087: * cl349@4087: * because the read of "*q" depends on the read of "p" and these cl349@4087: * two reads are separated by a read_barrier_depends(). However, cl349@4087: * the following code, with the same initial values for "a" and "b": cl349@4087: * cl349@4087: * cl349@4087: * CPU 0 CPU 1 cl349@4087: * cl349@4087: * a = 2; cl349@4087: * memory_barrier(); cl349@4087: * b = 3; y = b; cl349@4087: * read_barrier_depends(); cl349@4087: * x = a; cl349@4087: * cl349@4087: * cl349@4087: * does not enforce ordering, since there is no data dependency between cl349@4087: * the read of "a" and the read of "b". Therefore, on some CPUs, such cl349@4087: * as Alpha, "y" could be set to 3 and "x" to 0. Use rmb() cl349@4087: * in cases like thiswhere there are no data dependencies. cl349@4087: **/ cl349@4087: cl349@4087: #define read_barrier_depends() do { } while(0) cl349@4087: cl349@4087: #ifdef CONFIG_X86_OOSTORE cl349@4087: /* Actually there are no OOO store capable CPUs for now that do SSE, cl349@4087: but make it already an possibility. */ cl349@4087: #define wmb() alternative("lock; addl $0,0(%%esp)", "sfence", X86_FEATURE_XMM) cl349@4087: #else cl349@4087: #define wmb() __asm__ __volatile__ ("": : :"memory") cl349@4087: #endif cl349@4087: cl349@4087: #ifdef CONFIG_SMP sos22@5466: #define smp_wmb() wmb() sos22@5466: #if defined(CONFIG_SMP_ALTERNATIVES) && !defined(MODULE) sos22@5466: #define smp_alt_mb(instr) \ sos22@5466: __asm__ __volatile__("6667:\nnop\nnop\nnop\nnop\nnop\nnop\n6668:\n" \ sos22@5466: ".section __smp_alternatives,\"a\"\n" \ sos22@5466: ".long 6667b\n" \ sos22@5466: ".long 6673f\n" \ sos22@5466: ".previous\n" \ sos22@5466: ".section __smp_replacements,\"a\"\n" \ sos22@5466: "6673:.byte 6668b-6667b\n" \ sos22@5466: ".byte 6670f-6669f\n" \ sos22@5466: ".byte 6671f-6670f\n" \ sos22@5466: ".byte 0\n" \ sos22@5466: ".byte %c0\n" \ sos22@5466: "6669:lock;addl $0,0(%%esp)\n" \ sos22@5466: "6670:" instr "\n" \ sos22@5466: "6671:\n" \ sos22@5466: ".previous\n" \ sos22@5466: : \ sos22@5466: : "i" (X86_FEATURE_XMM2) \ sos22@5466: : "memory") sos22@5466: #define smp_rmb() smp_alt_mb("lfence") sos22@5466: #define smp_mb() smp_alt_mb("mfence") sos22@5466: #define set_mb(var, value) do { \ sos22@5466: unsigned long __set_mb_temp; \ sos22@5466: __asm__ __volatile__("6667:movl %1, %0\n6668:\n" \ sos22@5466: ".section __smp_alternatives,\"a\"\n" \ sos22@5466: ".long 6667b\n" \ sos22@5466: ".long 6673f\n" \ sos22@5466: ".previous\n" \ sos22@5466: ".section __smp_replacements,\"a\"\n" \ sos22@5466: "6673: .byte 6668b-6667b\n" \ sos22@5466: ".byte 6670f-6669f\n" \ sos22@5466: ".byte 0\n" \ sos22@5466: ".byte 6671f-6670f\n" \ sos22@5466: ".byte -1\n" \ sos22@5466: "6669: xchg %1, %0\n" \ sos22@5466: "6670:movl %1, %0\n" \ sos22@5466: "6671:\n" \ sos22@5466: ".previous\n" \ sos22@5466: : "=m" (var), "=r" (__set_mb_temp) \ sos22@5466: : "1" (value) \ sos22@5466: : "memory"); } while (0) sos22@5466: #else cl349@4087: #define smp_rmb() rmb() sos22@5466: #define smp_mb() mb() sos22@5466: #define set_mb(var, value) do { xchg(&var, value); } while (0) sos22@5466: #endif cl349@4087: #define smp_read_barrier_depends() read_barrier_depends() cl349@4087: #else cl349@4087: #define smp_mb() barrier() cl349@4087: #define smp_rmb() barrier() cl349@4087: #define smp_wmb() barrier() cl349@4087: #define smp_read_barrier_depends() do { } while(0) cl349@4087: #define set_mb(var, value) do { var = value; barrier(); } while (0) cl349@4087: #endif cl349@4087: cl349@4087: #define set_wmb(var, value) do { var = value; wmb(); } while (0) cl349@4087: cl349@4087: /* interrupt control.. */ cl349@4087: cl349@4087: /* cl349@4087: * The use of 'barrier' in the following reflects their use as local-lock cl349@4087: * operations. Reentrancy must be prevented (e.g., __cli()) /before/ following cl349@4684: * critical operations are executed. All critical operations must complete cl349@4087: * /before/ reentrancy is permitted (e.g., __sti()). Alpha architecture also cl349@4087: * includes these barriers, for example. cl349@4087: */ cl349@4087: cl349@4087: #define __cli() \ cl349@4087: do { \ cl349@4112: vcpu_info_t *_vcpu; \ cl349@4112: preempt_disable(); \ cl349@4112: _vcpu = &HYPERVISOR_shared_info->vcpu_data[smp_processor_id()]; \ cl349@4112: _vcpu->evtchn_upcall_mask = 1; \ cl349@4112: preempt_enable_no_resched(); \ cl349@4087: barrier(); \ cl349@4087: } while (0) cl349@4087: cl349@4087: #define __sti() \ cl349@4087: do { \ cl349@4112: vcpu_info_t *_vcpu; \ cl349@4087: barrier(); \ cl349@4112: preempt_disable(); \ cl349@4112: _vcpu = &HYPERVISOR_shared_info->vcpu_data[smp_processor_id()]; \ cl349@4112: _vcpu->evtchn_upcall_mask = 0; \ cl349@4087: barrier(); /* unmask then check (avoid races) */ \ cl349@4112: if ( unlikely(_vcpu->evtchn_upcall_pending) ) \ cl349@4112: force_evtchn_callback(); \ cl349@4112: preempt_enable(); \ cl349@4087: } while (0) cl349@4087: cl349@4087: #define __save_flags(x) \ cl349@4087: do { \ cl349@4112: vcpu_info_t *_vcpu; \ cl349@4112: _vcpu = &HYPERVISOR_shared_info->vcpu_data[smp_processor_id()]; \ cl349@4112: (x) = _vcpu->evtchn_upcall_mask; \ cl349@4087: } while (0) cl349@4087: cl349@4087: #define __restore_flags(x) \ cl349@4087: do { \ cl349@4112: vcpu_info_t *_vcpu; \ cl349@4087: barrier(); \ cl349@4112: preempt_disable(); \ cl349@4112: _vcpu = &HYPERVISOR_shared_info->vcpu_data[smp_processor_id()]; \ cl349@4112: if ((_vcpu->evtchn_upcall_mask = (x)) == 0) { \ cl349@4112: barrier(); /* unmask then check (avoid races) */ \ cl349@4112: if ( unlikely(_vcpu->evtchn_upcall_pending) ) \ cl349@4112: force_evtchn_callback(); \ cl349@4112: preempt_enable(); \ cl349@4112: } else \ cl349@4112: preempt_enable_no_resched(); \ cl349@4087: } while (0) cl349@4087: cl349@4112: #define safe_halt() ((void)0) cl349@4087: cl349@4087: #define __save_and_cli(x) \ cl349@4087: do { \ cl349@4112: vcpu_info_t *_vcpu; \ cl349@4112: preempt_disable(); \ cl349@4112: _vcpu = &HYPERVISOR_shared_info->vcpu_data[smp_processor_id()]; \ cl349@4112: (x) = _vcpu->evtchn_upcall_mask; \ cl349@4112: _vcpu->evtchn_upcall_mask = 1; \ cl349@4112: preempt_enable_no_resched(); \ cl349@4087: barrier(); \ cl349@4087: } while (0) cl349@4087: cl349@4087: #define local_irq_save(x) __save_and_cli(x) cl349@4087: #define local_irq_restore(x) __restore_flags(x) cl349@4087: #define local_save_flags(x) __save_flags(x) cl349@4087: #define local_irq_disable() __cli() cl349@4087: #define local_irq_enable() __sti() cl349@4087: kaf24@6551: /* Don't use smp_processor_id: this is called in debug versions of that fn. */ kaf24@6552: #ifdef CONFIG_SMP cl349@4112: #define irqs_disabled() \ kaf24@6551: HYPERVISOR_shared_info->vcpu_data[__smp_processor_id()].evtchn_upcall_mask kaf24@6552: #else kaf24@6552: #define irqs_disabled() \ kaf24@6552: HYPERVISOR_shared_info->vcpu_data[0].evtchn_upcall_mask kaf24@6552: #endif cl349@4087: cl349@4087: /* cl349@4087: * disable hlt during certain critical i/o operations cl349@4087: */ cl349@4087: #define HAVE_DISABLE_HLT cl349@4087: void disable_hlt(void); cl349@4087: void enable_hlt(void); cl349@4087: cl349@4087: extern int es7000_plat; cl349@4087: void cpu_idle_wait(void); cl349@4087: vh249@5730: extern unsigned long arch_align_stack(unsigned long sp); vh249@5730: cl349@4087: #endif