kaf24@1710: /*
kaf24@1710:  * Hypercall and fault low-level handling routines.
kaf24@1710:  *
kaf24@1710:  * Copyright (c) 2002-2004, K A Fraser
kaf24@1710:  * Copyright (c) 1991, 1992 Linus Torvalds
kaf24@3127:  * 
kaf24@3127:  * Calling back to a guest OS:
kaf24@3127:  * ===========================
kaf24@3127:  * 
kaf24@1710:  * First, we require that all callbacks (either via a supplied
kaf24@1710:  * interrupt-descriptor-table, or via the special event or failsafe callbacks
kaf24@1710:  * in the shared-info-structure) are to ring 1. This just makes life easier,
kaf24@1710:  * in that it means we don't have to do messy GDT/LDT lookups to find
kaf24@1710:  * out which the privilege-level of the return code-selector. That code
kaf24@1710:  * would just be a hassle to write, and would need to account for running
kaf24@1710:  * off the end of the GDT/LDT, for example. For all callbacks we check
kaf24@3127:  * that the provided return CS is not == __HYPERVISOR_{CS,DS}. Apart from that 
kaf24@3127:  * we're safe as don't allow a guest OS to install ring-0 privileges into the
kaf24@3127:  * GDT/LDT. It's up to the guest OS to ensure all returns via the IDT are to
kaf24@3127:  * ring 1. If not, we load incorrect SS/ESP values from the TSS (for ring 1
kaf24@3127:  * rather than the correct ring) and bad things are bound to ensue -- IRET is
kaf24@1710:  * likely to fault, and we may end up killing the domain (no harm can
kaf24@1710:  * come to Xen, though).
kaf24@1710:  *      
kaf24@1710:  * When doing a callback, we check if the return CS is in ring 0. If so,
kaf24@1710:  * callback is delayed until next return to ring != 0.
kaf24@1710:  * If return CS is in ring 1, then we create a callback frame
kaf24@1710:  * starting at return SS/ESP. The base of the frame does an intra-privilege
kaf24@1710:  * interrupt-return.
kaf24@1710:  * If return CS is in ring > 1, we create a callback frame starting
kaf24@1710:  * at SS/ESP taken from appropriate section of the current TSS. The base
kaf24@1710:  * of the frame does an inter-privilege interrupt-return.
kaf24@1710:  * 
kaf24@1710:  * Note that the "failsafe callback" uses a special stackframe:
kaf24@1710:  * { return_DS, return_ES, return_FS, return_GS, return_EIP,
kaf24@1710:  *   return_CS, return_EFLAGS[, return_ESP, return_SS] }
kaf24@1710:  * That is, original values for DS/ES/FS/GS are placed on stack rather than
kaf24@1710:  * in DS/ES/FS/GS themselves. Why? It saves us loading them, only to have them
kaf24@1710:  * saved/restored in guest OS. Furthermore, if we load them we may cause
kaf24@1710:  * a fault if they are invalid, which is a hassle to deal with. We avoid
kaf24@1710:  * that problem if we don't load them :-) This property allows us to use
kaf24@1710:  * the failsafe callback as a fallback: if we ever fault on loading DS/ES/FS/GS
kaf24@1710:  * on return to ring != 0, we can simply package it up as a return via
kaf24@1710:  * the failsafe callback, and let the guest OS sort it out (perhaps by
kaf24@1710:  * killing an application process). Note that we also do this for any
kaf24@1710:  * faulting IRET -- just let the guest OS handle it via the event
kaf24@1710:  * callback.
kaf24@1710:  *
kaf24@1710:  * We terminate a domain in the following cases:
kaf24@1710:  *  - creating a callback stack frame (due to bad ring-1 stack).
kaf24@1710:  *  - faulting IRET on entry to failsafe callback handler.
kaf24@1710:  * So, each domain must keep its ring-1 %ss/%esp and failsafe callback
kaf24@1710:  * handler in good order (absolutely no faults allowed!).
kaf24@1710:  */
kaf24@1710: 
kaf24@1710: #include <xen/config.h>
kaf24@1710: #include <xen/errno.h>
kaf24@2085: #include <xen/softirq.h>
kaf24@3314: #include <asm/asm_defns.h>
kaf24@2827: #include <public/xen.h>
kaf24@1710: 
kaf24@1710: #define GET_CURRENT(reg)   \
kaf24@3010:         movl $8192-4, reg; \
kaf24@1710:         orl  %esp, reg;    \
kaf24@1710:         andl $~3,reg;      \
kaf24@1710:         movl (reg),reg;
kaf24@1710: 
kaf24@1710:         ALIGN
kaf24@1710: restore_all_guest:
kaf24@3127:         testb $TF_failsafe_return,DOMAIN_thread_flags(%ebx)
kaf24@3127:         jnz  failsafe_callback
kaf24@3207:         testl $X86_EFLAGS_VM,XREGS_eflags(%esp)
kaf24@3207:         jnz  restore_all_vm86
kaf24@3127: FLT1:   movl XREGS_ds(%esp),%ds
kaf24@3127: FLT2:   movl XREGS_es(%esp),%es
kaf24@3127: FLT3:   movl XREGS_fs(%esp),%fs
kaf24@3127: FLT4:   movl XREGS_gs(%esp),%gs
kaf24@3207: restore_all_vm86:
kaf24@1710:         popl %ebx
kaf24@1710:         popl %ecx
kaf24@1710:         popl %edx
kaf24@1710:         popl %esi
kaf24@1710:         popl %edi
kaf24@1710:         popl %ebp
kaf24@1710:         popl %eax
kaf24@2954:         addl $4,%esp
kaf24@3127: FLT5:   iret
kaf24@3127: .section .fixup,"ax"
kaf24@3127: FIX5:   subl  $28,%esp
kaf24@3127:         pushl 28(%esp)                 # error_code/entry_vector
kaf24@3127:         movl  %eax,XREGS_eax+4(%esp)
kaf24@3127:         movl  %ebp,XREGS_ebp+4(%esp)
kaf24@3127:         movl  %edi,XREGS_edi+4(%esp)
kaf24@3127:         movl  %esi,XREGS_esi+4(%esp)
kaf24@3127:         movl  %edx,XREGS_edx+4(%esp)
kaf24@3127:         movl  %ecx,XREGS_ecx+4(%esp)
kaf24@3127:         movl  %ebx,XREGS_ebx+4(%esp)
kaf24@3127: FIX1:   SET_XEN_SEGMENTS(a)
kaf24@3127:         movl  %eax,%fs
kaf24@3127:         movl  %eax,%gs
kaf24@3127:         sti
kaf24@3127:         popl  %esi
kaf24@3127:         pushfl                         # EFLAGS
kaf24@3127:         movl  $__HYPERVISOR_CS,%eax
kaf24@3127:         pushl %eax                     # CS
kaf24@3127:         movl  $DBLFLT1,%eax
kaf24@3127:         pushl %eax                     # EIP
kaf24@3127:         pushl %esi                     # error_code/entry_vector
kaf24@3127:         jmp   error_code
kaf24@3127: DBLFLT1:GET_CURRENT(%ebx)
kaf24@3127:         jmp   test_all_events
kaf24@3127: DBLFIX1:GET_CURRENT(%ebx)
kaf24@3127:         testb $TF_failsafe_return,DOMAIN_thread_flags(%ebx)
kaf24@3127:         jnz   domain_crash             # cannot reenter failsafe code
kaf24@3127:         orb   $TF_failsafe_return,DOMAIN_thread_flags(%ebx)
kaf24@3127:         jmp   test_all_events          # will return via failsafe code
kaf24@3127: .previous
kaf24@3127: .section __pre_ex_table,"a"
kaf24@3127: 	.long FLT1,FIX1
kaf24@3127: 	.long FLT2,FIX1
kaf24@3127: 	.long FLT3,FIX1
kaf24@3127: 	.long FLT4,FIX1
kaf24@3127: 	.long FLT5,FIX5
kaf24@3127: .previous
kaf24@3127: .section __ex_table,"a"
kaf24@3127:         .long DBLFLT1,DBLFIX1
kaf24@3127: .previous
kaf24@3127: 
kaf24@3127: /* No special register assumptions */
kaf24@3127: failsafe_callback:
kaf24@3127:         GET_CURRENT(%ebx)
kaf24@3127:         andb $~TF_failsafe_return,DOMAIN_thread_flags(%ebx)
kaf24@3127:         leal DOMAIN_trap_bounce(%ebx),%edx
kaf24@3127:         movl DOMAIN_failsafe_addr(%ebx),%eax
kaf24@3127:         movl %eax,TRAPBOUNCE_eip(%edx)
kaf24@3127:         movl DOMAIN_failsafe_sel(%ebx),%eax
kaf24@3127:         movw %ax,TRAPBOUNCE_cs(%edx)
kaf24@3127:         movw $TBF_FAILSAFE,TRAPBOUNCE_flags(%edx)
kaf24@3127:         call create_bounce_frame
kaf24@3127:         popl %ebx
kaf24@3127:         popl %ecx
kaf24@3127:         popl %edx
kaf24@3127:         popl %esi
kaf24@3127:         popl %edi
kaf24@3127:         popl %ebp
kaf24@3127:         popl %eax
kaf24@3127:         addl $4,%esp
kaf24@3127: FLT6:   iret 
kaf24@3127: .section .fixup,"ax"
kaf24@3127: FIX6:   pushl %ebx
kaf24@3127:         GET_CURRENT(%ebx)
kaf24@3127:         orb   $TF_failsafe_return,DOMAIN_thread_flags(%ebx)
kaf24@3127:         pop   %ebx
kaf24@3127:         jmp   FIX5
kaf24@3127: .section __pre_ex_table,"a"
kaf24@3127: 	.long FLT6,FIX6
kaf24@3127: .previous
kaf24@1710: 
kaf24@1710:         ALIGN
kaf24@1710: restore_all_xen:
kaf24@1710: 	popl %ebx
kaf24@1710: 	popl %ecx
kaf24@1710: 	popl %edx
kaf24@1710: 	popl %esi
kaf24@1710: 	popl %edi
kaf24@1710: 	popl %ebp
kaf24@1710: 	popl %eax
kaf24@1710:         addl $4,%esp
kaf24@1710:         iret
kaf24@1710: 
kaf24@1710:         ALIGN
kaf24@1710: ENTRY(hypercall)
kaf24@3127:         subl $4,%esp
kaf24@2955: 	SAVE_ALL(b)
kaf24@2954:         sti
kaf24@2954:         GET_CURRENT(%ebx)
kaf24@1710: 	andl $(NR_hypercalls-1),%eax
kaf24@1710: 	call *SYMBOL_NAME(hypercall_table)(,%eax,4)
kaf24@1710: 
kaf24@1710: ret_from_hypercall:
kaf24@3127:         movl %eax,XREGS_eax(%esp)       # save the return value
kaf24@1710: 
kaf24@1710: test_all_events:
kaf24@1710:         xorl %ecx,%ecx
kaf24@1710:         notl %ecx
kaf24@1710:         cli                             # tests must not race interrupts
kaf24@1710: /*test_softirqs:*/  
kaf24@2954:         movl DOMAIN_processor(%ebx),%eax
kaf24@1710:         shl  $6,%eax                    # sizeof(irq_cpustat) == 64
kaf24@1710:         test %ecx,SYMBOL_NAME(irq_stat)(%eax,1)
kaf24@1710:         jnz  process_softirqs
kaf24@1710: /*test_guest_events:*/
kaf24@2954:         movl DOMAIN_shared_info(%ebx),%eax
kaf24@2954:         testb $0xFF,SHINFO_upcall_mask(%eax)
kaf24@1710:         jnz  restore_all_guest
kaf24@2954:         testb $0xFF,SHINFO_upcall_pending(%eax)
kaf24@1710:         jz   restore_all_guest
kaf24@1710: /*process_guest_events:*/
kaf24@3081:         leal DOMAIN_trap_bounce(%ebx),%edx
kaf24@2954:         movl DOMAIN_event_addr(%ebx),%eax
kaf24@3081:         movl %eax,TRAPBOUNCE_eip(%edx)
kaf24@2954:         movl DOMAIN_event_sel(%ebx),%eax
kaf24@3081:         movw %ax,TRAPBOUNCE_cs(%edx)
kaf24@3127:         movw $TBF_INTERRUPT,TRAPBOUNCE_flags(%edx)
kaf24@1710:         call create_bounce_frame
kaf24@3127:         movl DOMAIN_shared_info(%ebx),%eax
kaf24@3127:         movb $1,SHINFO_upcall_mask(%eax) # Upcalls are masked during delivery
kaf24@1710:         jmp  restore_all_guest
kaf24@1710: 
kaf24@1710:         ALIGN
kaf24@1710: process_softirqs:
kaf24@1710:         sti       
kaf24@1710:         call SYMBOL_NAME(do_softirq)
kaf24@1710:         jmp  test_all_events
kaf24@1710:                 
kaf24@2954: /* CREATE A BASIC EXCEPTION FRAME ON GUEST OS (RING-1) STACK:            */
kaf24@2954: /*   {EIP, CS, EFLAGS, [ESP, SS]}                                        */
kaf24@3081: /* %edx == trap_bounce, %ebx == task_struct                              */
kaf24@2954: /* %eax,%ecx are clobbered. %gs:%esi contain new XREGS_ss/XREGS_esp. */
kaf24@3207: create_bounce_frame:
kaf24@3207:         movl XREGS_eflags+4(%esp),%ecx
kaf24@3127:         movb XREGS_cs+4(%esp),%cl
kaf24@3207:         testl $(2|X86_EFLAGS_VM),%ecx
kaf24@3207:         jz   ring1 /* jump if returning to an existing ring-1 activation */
kaf24@1710:         /* obtain ss/esp from TSS -- no current ring-1 activations */
kaf24@2954:         movl DOMAIN_processor(%ebx),%eax
kaf24@1710:         /* next 4 lines multiply %eax by 8320, which is sizeof(tss_struct) */
kaf24@1710:         movl %eax, %ecx
kaf24@1710:         shll $7, %ecx
kaf24@1710:         shll $13, %eax
kaf24@1710:         addl %ecx,%eax
kaf24@1710:         addl $init_tss + 12,%eax
kaf24@1710:         movl (%eax),%esi /* tss->esp1 */
kaf24@3127: FLT7:   movl 4(%eax),%gs /* tss->ss1  */
kaf24@3207:         testl $X86_EFLAGS_VM,XREGS_eflags+4(%esp)
kaf24@3207:         jz   nvm86_1
kaf24@3207: 	subl $16,%esi       /* push ES/DS/FS/GS (VM86 stack frame) */
kaf24@3207: 	movl XREGS_es+4(%esp),%eax
kaf24@3207: FLT8:   movl %eax,%gs:(%esi)
kaf24@3207: 	movl XREGS_ds+4(%esp),%eax
kaf24@3207: FLT9:   movl %eax,%gs:4(%esi)
kaf24@3207: 	movl XREGS_fs+4(%esp),%eax
kaf24@3207: FLT10:  movl %eax,%gs:8(%esi)
kaf24@3207: 	movl XREGS_gs+4(%esp),%eax
kaf24@3207: FLT11:  movl %eax,%gs:12(%esi)
kaf24@3207: nvm86_1:subl $8,%esi        /* push SS/ESP (inter-priv iret) */
kaf24@2954:         movl XREGS_esp+4(%esp),%eax
kaf24@3207: FLT12:  movl %eax,%gs:(%esi) 
kaf24@2954:         movl XREGS_ss+4(%esp),%eax
kaf24@3207: FLT13:  movl %eax,%gs:4(%esi) 
kaf24@3207:         jmp 1f
kaf24@3207: ring1:  /* obtain ss/esp from oldss/oldesp -- a ring-1 activation exists */
kaf24@2954:         movl XREGS_esp+4(%esp),%esi
kaf24@3207: FLT14:  movl XREGS_ss+4(%esp),%gs 
kaf24@3207: 1:      /* Construct a stack frame: EFLAGS, CS/EIP */
kaf24@1710:         subl $12,%esi
kaf24@2954:         movl XREGS_eip+4(%esp),%eax
kaf24@3207: FLT15:  movl %eax,%gs:(%esi) 
kaf24@2954:         movl XREGS_cs+4(%esp),%eax
kaf24@3207: FLT16:  movl %eax,%gs:4(%esi) 
kaf24@2954:         movl XREGS_eflags+4(%esp),%eax
kaf24@3207: FLT17:  movl %eax,%gs:8(%esi)
kaf24@3127:         movb TRAPBOUNCE_flags(%edx),%cl
kaf24@3127:         test $TBF_EXCEPTION_ERRCODE,%cl
kaf24@3127:         jz   1f
kaf24@3127:         subl $4,%esi                    # push error_code onto guest frame
kaf24@3127:         movl TRAPBOUNCE_error_code(%edx),%eax
kaf24@3207: FLT18:  movl %eax,%gs:(%esi)
kaf24@3127:         testb $TBF_EXCEPTION_CR2,%cl
kaf24@3127:         jz   2f
kaf24@3127:         subl $4,%esi                    # push %cr2 onto guest frame
kaf24@3127:         movl TRAPBOUNCE_cr2(%edx),%eax
kaf24@3207: FLT19:  movl %eax,%gs:(%esi)
kaf24@3127: 1:      testb $TBF_FAILSAFE,%cl
kaf24@3127:         jz   2f
kaf24@3127:         subl $16,%esi                # add DS/ES/FS/GS to failsafe stack frame
kaf24@3207:         testl $X86_EFLAGS_VM,XREGS_eflags+4(%esp)
kaf24@3207:         jz   nvm86_2
kaf24@3207:         xorl %eax,%eax               # VM86: we write zero selector values
kaf24@3207: FLT20:  movl %eax,%gs:(%esi) 
kaf24@3207: FLT21:  movl %eax,%gs:4(%esi)
kaf24@3207: FLT22:  movl %eax,%gs:8(%esi) 
kaf24@3207: FLT23:  movl %eax,%gs:12(%esi)
kaf24@3207:         jmp  2f
kaf24@3207: nvm86_2:movl XREGS_ds+4(%esp),%eax   # non-VM86: write real selector values
kaf24@3207: FLT24:  movl %eax,%gs:(%esi) 
kaf24@3127:         movl XREGS_es+4(%esp),%eax
kaf24@3207: FLT25:  movl %eax,%gs:4(%esi)
kaf24@3127:         movl XREGS_fs+4(%esp),%eax
kaf24@3207: FLT26:  movl %eax,%gs:8(%esi) 
kaf24@3127:         movl XREGS_gs+4(%esp),%eax
kaf24@3207: FLT27:  movl %eax,%gs:12(%esi)
kaf24@3127: 2:      movb $0,TRAPBOUNCE_flags(%edx)
kaf24@3207:         testl $X86_EFLAGS_VM,XREGS_eflags+4(%esp)
kaf24@3207:         jz   nvm86_3
kaf24@3207:         xorl %eax,%eax      /* zero DS-GS, just as a real CPU would */
kaf24@3207:         movl %eax,XREGS_ds+4(%esp)
kaf24@3207:         movl %eax,XREGS_es+4(%esp)
kaf24@3207:         movl %eax,XREGS_fs+4(%esp)
kaf24@3207:         movl %eax,XREGS_gs+4(%esp)
kaf24@3207: nvm86_3:/* Rewrite our stack frame and return to ring 1. */
kaf24@1710:         /* IA32 Ref. Vol. 3: TF, VM, RF and NT flags are cleared on trap. */
kaf24@3127:         andl $0xfffcbeff,XREGS_eflags+4(%esp)
kaf24@2954:         movl %gs,XREGS_ss+4(%esp)
kaf24@2954:         movl %esi,XREGS_esp+4(%esp)
kaf24@3081:         movzwl TRAPBOUNCE_cs(%edx),%eax
kaf24@2954:         movl %eax,XREGS_cs+4(%esp)
kaf24@3081:         movl TRAPBOUNCE_eip(%edx),%eax
kaf24@2954:         movl %eax,XREGS_eip+4(%esp)
kaf24@1710:         ret
kaf24@3127: .section .fixup,"ax"
kaf24@3127: FIX7:   sti
kaf24@3127:         popl  %esi
kaf24@3127:         addl  $4,%esp                  # Discard create_b_frame return address
kaf24@3127:         pushfl                         # EFLAGS
kaf24@3127:         movl  $__HYPERVISOR_CS,%eax
kaf24@3127:         pushl %eax                     # CS
kaf24@3127:         movl  $DBLFLT2,%eax
kaf24@3127:         pushl %eax                     # EIP
kaf24@3127:         pushl %esi                     # error_code/entry_vector
kaf24@3127:         jmp   error_code
kaf24@3127: DBLFLT2:jmp   process_guest_exception_and_events
kaf24@3127: .previous
kaf24@3127: .section __pre_ex_table,"a"
kaf24@3207: 	.long  FLT7,FIX7 ,  FLT8,FIX7 ,  FLT9,FIX7 , FLT10,FIX7
kaf24@3207: 	.long FLT11,FIX7 , FLT12,FIX7 , FLT13,FIX7 , FLT14,FIX7
kaf24@3207: 	.long FLT15,FIX7 , FLT16,FIX7 , FLT17,FIX7 , FLT18,FIX7
kaf24@3207: 	.long FLT19,FIX7 , FLT20,FIX7 , FLT21,FIX7 , FLT22,FIX7
kaf24@3207: 	.long FLT23,FIX7 , FLT24,FIX7 , FLT25,FIX7 , FLT26,FIX7 , FLT27,FIX7
kaf24@3127: .previous
kaf24@1710: .section __ex_table,"a"
kaf24@3127:         .long DBLFLT2,domain_crash
kaf24@1710: .previous
kaf24@1710: 
kaf24@1710:         ALIGN
kaf24@3127: process_guest_exception_and_events:
kaf24@3081:         leal DOMAIN_trap_bounce(%ebx),%edx
kaf24@3127:         testb $TBF_EXCEPTION,TRAPBOUNCE_flags(%edx)
kaf24@1710:         jz   test_all_events
kaf24@3257:         cli  # create_bounce_frame needs CLI for pre-exceptions to work
kaf24@3127:         call create_bounce_frame
kaf24@1710:         jmp  test_all_events
kaf24@1710: 
kaf24@1710:         ALIGN
kaf24@1710: ENTRY(ret_from_intr)
kaf24@3207:         GET_CURRENT(%ebx)
kaf24@3207:         movl  XREGS_eflags(%esp),%eax
kaf24@3207:         movb  XREGS_cs(%esp),%al
kaf24@3207:         testl $(3|X86_EFLAGS_VM),%eax
kaf24@3207:         jnz   test_all_events
kaf24@3207:         jmp   restore_all_xen
kaf24@1710: 
kaf24@1710: ENTRY(divide_error)
kaf24@3127: 	pushl $TRAP_divide_error<<16
kaf24@1710: 	ALIGN
kaf24@1710: error_code:
kaf24@3127:         SAVE_ALL_NOSEGREGS(a)
kaf24@3127:         SET_XEN_SEGMENTS(a)
kaf24@3127:         testb $X86_EFLAGS_IF>>8,XREGS_eflags+1(%esp)
kaf24@3127:         jz    exception_with_ints_disabled
kaf24@3198: 1:      sti                             # re-enable interrupts
kaf24@3127:         xorl  %eax,%eax
kaf24@3127:         movw  XREGS_entry_vector(%esp),%ax
kaf24@3127:         movl  %esp,%edx
ach61@2843: 	pushl %edx			# push the xen_regs pointer
kaf24@1710: 	GET_CURRENT(%ebx)
kaf24@3127: 	call  *SYMBOL_NAME(exception_table)(,%eax,4)
kaf24@3127:         addl  $4,%esp
kaf24@3207:         movl  XREGS_eflags(%esp),%eax
kaf24@2954:         movb  XREGS_cs(%esp),%al
kaf24@3207:         testl $(3|X86_EFLAGS_VM),%eax
kaf24@3127: 	jz    restore_all_xen
kaf24@1710:         jmp   process_guest_exception_and_events
kaf24@1710: 
kaf24@3127: exception_with_ints_disabled:
kaf24@3207:         movl  XREGS_eflags(%esp),%eax
kaf24@3127:         movb  XREGS_cs(%esp),%al
kaf24@3207:         testl $(3|X86_EFLAGS_VM),%eax   # interrupts disabled outside Xen?
kaf24@3207:         jnz   1b                        # it really does happen!
kaf24@3207:                                         #  (e.g., DOM0 X server)
kaf24@3127:         pushl XREGS_eip(%esp)
kaf24@3127:         call  search_pre_exception_table
kaf24@3127:         addl  $4,%esp
kaf24@3127:         testl %eax,%eax                 # no fixup code for faulting EIP?
kaf24@3127:         jz    FATAL_exception_with_ints_disabled
kaf24@3127:         movl  %eax,XREGS_eip(%esp)
kaf24@3127:         movl  %esp,%esi
kaf24@3127:         subl  $4,%esp
kaf24@3127:         movl  %esp,%edi
kaf24@3127:         movl  $XREGS_kernel_sizeof/4,%ecx
kaf24@3127:         rep;  movsl                     # make room for error_code/entry_vector
kaf24@3127:         movl  XREGS_error_code(%esp),%eax # error_code/entry_vector
kaf24@3127:         movl  %eax,XREGS_kernel_sizeof(%esp)
kaf24@3127:         jmp   restore_all_xen           # return to fixup code
kaf24@3127: 
kaf24@3127: FATAL_exception_with_ints_disabled:
kaf24@3127:         xorl  %esi,%esi
kaf24@3127:         movw  XREGS_entry_vector(%esp),%si
kaf24@3127:         movl  %esp,%edx
kaf24@3127: 	pushl %edx			# push the xen_regs pointer
kaf24@3127:         pushl %esi                      # push the trapnr (entry vector)
kaf24@3127:         call  SYMBOL_NAME(fatal_trap)
kaf24@3127:         ud2
kaf24@3127:                                         
kaf24@1710: ENTRY(coprocessor_error)
kaf24@3127: 	pushl $TRAP_copro_error<<16
kaf24@1710: 	jmp error_code
kaf24@1710: 
kaf24@1710: ENTRY(simd_coprocessor_error)
kaf24@3127: 	pushl $TRAP_simd_error<<16
kaf24@1710: 	jmp error_code
kaf24@1710: 
kaf24@1710: ENTRY(device_not_available)
kaf24@3127: 	pushl $TRAP_no_device<<16
kaf24@1710:         jmp   error_code
kaf24@1710: 
kaf24@1710: ENTRY(debug)
kaf24@3127: 	pushl $TRAP_debug<<16
kaf24@1710: 	jmp error_code
kaf24@1710: 
kaf24@1710: ENTRY(int3)
kaf24@3127: 	pushl $TRAP_int3<<16
kaf24@1710: 	jmp error_code
kaf24@1710: 
kaf24@1710: ENTRY(overflow)
kaf24@3127: 	pushl $TRAP_overflow<<16
kaf24@1710: 	jmp error_code
kaf24@1710: 
kaf24@1710: ENTRY(bounds)
kaf24@3127: 	pushl $TRAP_bounds<<16
kaf24@1710: 	jmp error_code
kaf24@1710: 
kaf24@1710: ENTRY(invalid_op)
kaf24@3127: 	pushl $TRAP_invalid_op<<16
kaf24@1710: 	jmp error_code
kaf24@1710: 
kaf24@1710: ENTRY(coprocessor_segment_overrun)
kaf24@3127: 	pushl $TRAP_copro_seg<<16
kaf24@1710: 	jmp error_code
kaf24@1710: 
kaf24@1710: ENTRY(invalid_TSS)
kaf24@3127:         movw $TRAP_invalid_tss,2(%esp)
kaf24@1710: 	jmp error_code
kaf24@1710: 
kaf24@1710: ENTRY(segment_not_present)
kaf24@3127:         movw $TRAP_no_segment,2(%esp)
kaf24@1710: 	jmp error_code
kaf24@1710: 
kaf24@1710: ENTRY(stack_segment)
kaf24@3127:         movw $TRAP_stack_error,2(%esp)
kaf24@1710: 	jmp error_code
kaf24@1710: 
kaf24@1710: ENTRY(general_protection)
kaf24@3127:         movw $TRAP_gp_fault,2(%esp)
kaf24@1710: 	jmp error_code
kaf24@1710: 
kaf24@1710: ENTRY(alignment_check)
kaf24@3127:         movw $TRAP_alignment_check,2(%esp)
kaf24@1710: 	jmp error_code
kaf24@1710: 
kaf24@1710: ENTRY(page_fault)
kaf24@3127:         movw $TRAP_page_fault,2(%esp)
kaf24@1710: 	jmp error_code
kaf24@1710: 
kaf24@1710: ENTRY(machine_check)
kaf24@3127:         pushl $TRAP_machine_check<<16
kaf24@1710: 	jmp error_code
kaf24@1710: 
kaf24@1710: ENTRY(spurious_interrupt_bug)
kaf24@3127:         pushl $TRAP_spurious_int<<16
kaf24@1710: 	jmp error_code
kaf24@1710: 
kaf24@1710: ENTRY(nmi)
kaf24@1710:         # Save state but do not trash the segment registers!
kaf24@1710:         # We may otherwise be unable to reload them or copy them to ring 1. 
kaf24@1710: 	pushl %eax
kaf24@2955: 	SAVE_ALL_NOSEGREGS(a)
kaf24@1710: 
kaf24@2085:         # Check for hardware problems.
kaf24@1710:         inb   $0x61,%al
kaf24@1710:         testb $0x80,%al
kaf24@2080:         jne   nmi_parity_err
kaf24@1710:         testb $0x40,%al
kaf24@1710:         jne   nmi_io_err
kaf24@1710:         movl  %eax,%ebx
kaf24@1710:         
kaf24@1710:         # Okay, its almost a normal NMI tick. We can only process it if:
kaf24@1710:         #  A. We are the outermost Xen activation (in which case we have
kaf24@1710:         #     the selectors safely saved on our stack)
kaf24@1710:         #  B. DS-GS all contain sane Xen values.
kaf24@1710:         # In all other cases we bail without touching DS-GS, as we have
kaf24@1710:         # interrupted an enclosing Xen activation in tricky prologue or
kaf24@1710:         # epilogue code.
kaf24@3207:         movl  XREGS_eflags(%esp),%eax
kaf24@2954:         movb  XREGS_cs(%esp),%al
kaf24@3207:         testl $(3|X86_EFLAGS_VM),%eax
kaf24@3127:         jnz   do_watchdog_tick
kaf24@3373:         movl  %ds,%eax
kaf24@1710:         cmpw  $(__HYPERVISOR_DS),%ax
kaf24@2954:         jne   restore_all_xen
kaf24@3373:         movl  %es,%eax
kaf24@1710:         cmpw  $(__HYPERVISOR_DS),%ax
kaf24@2954:         jne   restore_all_xen
kaf24@1710: 
kaf24@1710: do_watchdog_tick:
kaf24@1710:         movl  $(__HYPERVISOR_DS),%edx
kaf24@1710:         movl  %edx,%ds
kaf24@1710:         movl  %edx,%es
kaf24@1710:         movl  %esp,%edx
kaf24@1710: 	pushl %ebx   # reason
kaf24@1710: 	pushl %edx   # regs
kaf24@1710:         call  SYMBOL_NAME(do_nmi)
kaf24@1710: 	addl  $8,%esp
kaf24@3207:         movl  XREGS_eflags(%esp),%eax
kaf24@2954:         movb  XREGS_cs(%esp),%al
kaf24@3207:         testl $(3|X86_EFLAGS_VM),%eax
kaf24@3127: 	jz    restore_all_xen
kaf24@1710:         GET_CURRENT(%ebx)
kaf24@1710:         jmp   restore_all_guest
kaf24@1710: 
kaf24@2085: nmi_parity_err:
kaf24@2085:         # Clear and disable the parity-error line
kaf24@2085:         andb $0xf,%al
kaf24@2085:         orb  $0x4,%al
kaf24@2085:         outb %al,$0x61
kaf24@2085:         cmpb $'i',%ss:SYMBOL_NAME(opt_nmi) # nmi=ignore
kaf24@2954:         je   restore_all_xen
kaf24@2085:         bts  $0,%ss:SYMBOL_NAME(nmi_softirq_reason)
kaf24@2085:         bts  $NMI_SOFTIRQ,%ss:SYMBOL_NAME(irq_stat)
kaf24@2085:         cmpb $'d',%ss:SYMBOL_NAME(opt_nmi) # nmi=dom0
kaf24@2954:         je   restore_all_xen
kaf24@2085:         movl $(__HYPERVISOR_DS),%edx       # nmi=fatal
kaf24@1710:         movl %edx,%ds
kaf24@1710:         movl %edx,%es
kaf24@2079:         movl %esp,%edx
kaf24@2079:         push %edx
kaf24@2079:         call SYMBOL_NAME(mem_parity_error)
kaf24@2085:         addl $4,%esp
kaf24@2085:         jmp  ret_from_intr
kaf24@2085:                 
kaf24@1710: nmi_io_err: 
kaf24@2085:         # Clear and disable the I/O-error line
kaf24@2085:         andb $0xf,%al
kaf24@2085:         orb  $0x8,%al
kaf24@2085:         outb %al,$0x61
kaf24@2085:         cmpb $'i',%ss:SYMBOL_NAME(opt_nmi) # nmi=ignore
kaf24@2954:         je   restore_all_xen
kaf24@2085:         bts  $1,%ss:SYMBOL_NAME(nmi_softirq_reason)
kaf24@2085:         bts  $NMI_SOFTIRQ,%ss:SYMBOL_NAME(irq_stat)
kaf24@2085:         cmpb $'d',%ss:SYMBOL_NAME(opt_nmi) # nmi=dom0
kaf24@2954:         je   restore_all_xen
kaf24@2085:         movl $(__HYPERVISOR_DS),%edx       # nmi=fatal
kaf24@1710:         movl %edx,%ds
kaf24@1710:         movl %edx,%es
kaf24@2079:         movl %esp,%edx
kaf24@2079:         push %edx
kaf24@2079:         call SYMBOL_NAME(io_check_error)                        
kaf24@2085:         addl $4,%esp
kaf24@2085:         jmp  ret_from_intr
kaf24@3207: 
kaf24@3207: 
kaf24@3207: ENTRY(setup_vm86_frame)
kaf24@3207:         # Copies the entire stack frame forwards by 16 bytes.
kaf24@3207:         .macro copy_vm86_words count=18
kaf24@3207:         .if \count
kaf24@3207:         pushl ((\count-1)*4)(%esp)
kaf24@3207:         popl  ((\count-1)*4)+16(%esp)
kaf24@3207:         copy_vm86_words "(\count-1)"
kaf24@3207:         .endif
kaf24@3207:         .endm
kaf24@3207:         copy_vm86_words
kaf24@3207:         addl $16,%esp
kaf24@3207:         ret
kaf24@3207: 
kaf24@3207: do_switch_vm86:
kaf24@3207:         # Discard the return address
kaf24@3207:         addl $4,%esp
kaf24@3207: 
kaf24@3218:         movl XREGS_eflags(%esp),%edx
kaf24@3218: 
kaf24@3207:         # GS:ESI == Ring-1 stack activation
kaf24@3207:         movl XREGS_esp(%esp),%esi
kaf24@3207: VFLT1:  movl XREGS_ss(%esp),%gs
kaf24@3207: 
kaf24@3207:         # ES:EDI == Ring-0 stack activation
kaf24@3207:         leal XREGS_eip(%esp),%edi
kaf24@3207: 
kaf24@3207:         # Restore the hypercall-number-clobbered EAX on our stack frame
kaf24@3207: VFLT2:  movl %gs:(%esi),%eax
kaf24@3207:         movl %eax,XREGS_eax(%esp)
kaf24@3207:         addl $4,%esi
kaf24@3207:         	
kaf24@3207:       	# Copy the VM86 activation from the ring-1 stack to the ring-0 stack
kaf24@3207:         movl $(XREGS_user_sizeof-XREGS_eip)/4,%ecx
kaf24@3207: VFLT3:  movl %gs:(%esi),%eax
kaf24@3207:         stosl
kaf24@3207:         addl $4,%esi
kaf24@3207:         loop VFLT3
kaf24@3207: 
kaf24@3207:         # Fix up EFLAGS
kaf24@3207:         andl $~X86_EFLAGS_IOPL,XREGS_eflags(%esp)
kaf24@3218:         andl $X86_EFLAGS_IOPL,%edx # Ignore attempts to change EFLAGS.IOPL
kaf24@3207:         jnz  1f
kaf24@3218:         orl  $X86_EFLAGS_IF,%edx   # EFLAGS.IOPL=0 => no messing with EFLAGS.IF
kaf24@3218: 1:      orl  $X86_EFLAGS_VM,%edx   # Force EFLAGS.VM
kaf24@3218:         orl  %edx,XREGS_eflags(%esp)
kaf24@3207:         
kaf24@3207:         jmp test_all_events
kaf24@3207: 
kaf24@3207: .section __ex_table,"a"
kaf24@3207:         .long VFLT1,domain_crash
kaf24@3207:         .long VFLT2,domain_crash
kaf24@3207:         .long VFLT3,domain_crash
kaf24@3207: .previous
kaf24@3207: 
kaf24@1710: .data
kaf24@3127: 
kaf24@3127: ENTRY(exception_table)
kaf24@3127:         .long SYMBOL_NAME(do_divide_error)
kaf24@3127:         .long SYMBOL_NAME(do_debug)
kaf24@3127:         .long 0 # nmi
kaf24@3127:         .long SYMBOL_NAME(do_int3)
kaf24@3127:         .long SYMBOL_NAME(do_overflow)
kaf24@3127:         .long SYMBOL_NAME(do_bounds)
kaf24@3127:         .long SYMBOL_NAME(do_invalid_op)
kaf24@3127:         .long SYMBOL_NAME(math_state_restore)
kaf24@3127:         .long 0 # double fault
kaf24@3127:         .long SYMBOL_NAME(do_coprocessor_segment_overrun)
kaf24@3127:         .long SYMBOL_NAME(do_invalid_TSS)
kaf24@3127:         .long SYMBOL_NAME(do_segment_not_present)
kaf24@3127:         .long SYMBOL_NAME(do_stack_segment)
kaf24@3127:         .long SYMBOL_NAME(do_general_protection)
kaf24@3127:         .long SYMBOL_NAME(do_page_fault)
kaf24@3127:         .long SYMBOL_NAME(do_spurious_interrupt_bug)
kaf24@3127:         .long SYMBOL_NAME(do_coprocessor_error)
kaf24@3127:         .long SYMBOL_NAME(do_alignment_check)
kaf24@3127:         .long SYMBOL_NAME(do_machine_check)
kaf24@3127:         .long SYMBOL_NAME(do_simd_coprocessor_error)
kaf24@3127: 
kaf24@1710: ENTRY(hypercall_table)
kaf24@1710:         .long SYMBOL_NAME(do_set_trap_table)     /*  0 */
kaf24@1710:         .long SYMBOL_NAME(do_mmu_update)
kaf24@1710:         .long SYMBOL_NAME(do_set_gdt)
kaf24@1710:         .long SYMBOL_NAME(do_stack_switch)
kaf24@1710:         .long SYMBOL_NAME(do_set_callbacks)
kaf24@1710:         .long SYMBOL_NAME(do_fpu_taskswitch)     /*  5 */
kaf24@1710:         .long SYMBOL_NAME(do_sched_op)
kaf24@1710:         .long SYMBOL_NAME(do_dom0_op)
kaf24@1710:         .long SYMBOL_NAME(do_set_debugreg)
kaf24@1710:         .long SYMBOL_NAME(do_get_debugreg)
kaf24@1710:         .long SYMBOL_NAME(do_update_descriptor)  /* 10 */
kaf24@1710:         .long SYMBOL_NAME(do_set_fast_trap)
kaf24@1710:         .long SYMBOL_NAME(do_dom_mem_op)
kaf24@1710:         .long SYMBOL_NAME(do_multicall)
kaf24@1710:         .long SYMBOL_NAME(do_update_va_mapping)
kaf24@1710:         .long SYMBOL_NAME(do_set_timer_op)       /* 15 */
kaf24@1710:         .long SYMBOL_NAME(do_event_channel_op)
kaf24@1710:         .long SYMBOL_NAME(do_xen_version)
kaf24@1710:         .long SYMBOL_NAME(do_console_io)
kaf24@1710:         .long SYMBOL_NAME(do_physdev_op)
kaf24@2375:         .long SYMBOL_NAME(do_grant_table_op)     /* 20 */
kaf24@2111:         .long SYMBOL_NAME(do_vm_assist)
kaf24@2375:         .long SYMBOL_NAME(do_update_va_mapping_otherdomain)
kaf24@3207:         .long SYMBOL_NAME(do_switch_vm86)
kaf24@1710:         .rept NR_hypercalls-((.-hypercall_table)/4)
kaf24@1710:         .long SYMBOL_NAME(do_ni_hypercall)
kaf24@1710:         .endr