1.1 --- a/xen/arch/x86/boot/x86_64.S	Sun Feb 06 18:49:34 2005 +0000
     1.2 +++ b/xen/arch/x86/boot/x86_64.S	Sun Feb 06 21:30:19 2005 +0000
     1.3 @@ -248,12 +248,3 @@ ENTRY(cpu0_stack)    # Initial stack is 
     1.4          .org 0x6000
     1.5  ENTRY(stext)
     1.6  ENTRY(_stext)
     1.7 -
     1.8 -.globl copy_from_user, copy_to_user, copy_user_generic
     1.9 -copy_from_user: 
    1.10 -copy_to_user:
    1.11 -copy_user_generic:
    1.12 -.globl __get_user_1, __get_user_4, __get_user_8
    1.13 -__get_user_1:
    1.14 -__get_user_4:
    1.15 -__get_user_8:

     2.1 --- a/xen/arch/x86/x86_32/usercopy.c	Sun Feb 06 18:49:34 2005 +0000
     2.2 +++ b/xen/arch/x86/x86_32/usercopy.c	Sun Feb 06 21:30:19 2005 +0000
     2.3 @@ -9,8 +9,6 @@
     2.4  #include <xen/mm.h>
     2.5  #include <asm/uaccess.h>
     2.6  
     2.7 -#define might_sleep() ((void)0)
     2.8 -
     2.9  static inline int __movsl_is_ok(unsigned long a1, unsigned long a2, unsigned long n)
    2.10  {
    2.11  #ifdef CONFIG_X86_INTEL_USERCOPY
    2.12 @@ -22,93 +20,6 @@ static inline int __movsl_is_ok(unsigned
    2.13  #define movsl_is_ok(a1,a2,n) \
    2.14  	__movsl_is_ok((unsigned long)(a1),(unsigned long)(a2),(n))
    2.15  
    2.16 -/*
    2.17 - * Copy a null terminated string from userspace.
    2.18 - */
    2.19 -
    2.20 -#define __do_strncpy_from_user(dst,src,count,res)			   \
    2.21 -do {									   \
    2.22 -	int __d0, __d1, __d2;						   \
    2.23 -	__asm__ __volatile__(						   \
    2.24 -		"	testl %1,%1\n"					   \
    2.25 -		"	jz 2f\n"					   \
    2.26 -		"0:	lodsb\n"					   \
    2.27 -		"	stosb\n"					   \
    2.28 -		"	testb %%al,%%al\n"				   \
    2.29 -		"	jz 1f\n"					   \
    2.30 -		"	decl %1\n"					   \
    2.31 -		"	jnz 0b\n"					   \
    2.32 -		"1:	subl %1,%0\n"					   \
    2.33 -		"2:\n"							   \
    2.34 -		".section .fixup,\"ax\"\n"				   \
    2.35 -		"3:	movl %5,%0\n"					   \
    2.36 -		"	jmp 2b\n"					   \
    2.37 -		".previous\n"						   \
    2.38 -		".section __ex_table,\"a\"\n"				   \
    2.39 -		"	.align 4\n"					   \
    2.40 -		"	.long 0b,3b\n"					   \
    2.41 -		".previous"						   \
    2.42 -		: "=d"(res), "=c"(count), "=&a" (__d0), "=&S" (__d1),	   \
    2.43 -		  "=&D" (__d2)						   \
    2.44 -		: "i"(-EFAULT), "0"(count), "1"(count), "3"(src), "4"(dst) \
    2.45 -		: "memory");						   \
    2.46 -} while (0)
    2.47 -
    2.48 -/**
    2.49 - * __strncpy_from_user: - Copy a NUL terminated string from userspace, with less checking.
    2.50 - * @dst:   Destination address, in kernel space.  This buffer must be at
    2.51 - *         least @count bytes long.
    2.52 - * @src:   Source address, in user space.
    2.53 - * @count: Maximum number of bytes to copy, including the trailing NUL.
    2.54 - * 
    2.55 - * Copies a NUL-terminated string from userspace to kernel space.
    2.56 - * Caller must check the specified block with access_ok() before calling
    2.57 - * this function.
    2.58 - *
    2.59 - * On success, returns the length of the string (not including the trailing
    2.60 - * NUL).
    2.61 - *
    2.62 - * If access to userspace fails, returns -EFAULT (some data may have been
    2.63 - * copied).
    2.64 - *
    2.65 - * If @count is smaller than the length of the string, copies @count bytes
    2.66 - * and returns @count.
    2.67 - */
    2.68 -long
    2.69 -__strncpy_from_user(char *dst, const char __user *src, long count)
    2.70 -{
    2.71 -	long res;
    2.72 -	__do_strncpy_from_user(dst, src, count, res);
    2.73 -	return res;
    2.74 -}
    2.75 -
    2.76 -/**
    2.77 - * strncpy_from_user: - Copy a NUL terminated string from userspace.
    2.78 - * @dst:   Destination address, in kernel space.  This buffer must be at
    2.79 - *         least @count bytes long.
    2.80 - * @src:   Source address, in user space.
    2.81 - * @count: Maximum number of bytes to copy, including the trailing NUL.
    2.82 - * 
    2.83 - * Copies a NUL-terminated string from userspace to kernel space.
    2.84 - *
    2.85 - * On success, returns the length of the string (not including the trailing
    2.86 - * NUL).
    2.87 - *
    2.88 - * If access to userspace fails, returns -EFAULT (some data may have been
    2.89 - * copied).
    2.90 - *
    2.91 - * If @count is smaller than the length of the string, copies @count bytes
    2.92 - * and returns @count.
    2.93 - */
    2.94 -long
    2.95 -strncpy_from_user(char *dst, const char __user *src, long count)
    2.96 -{
    2.97 -	long res = -EFAULT;
    2.98 -	if (access_ok(VERIFY_READ, src, 1))
    2.99 -		__do_strncpy_from_user(dst, src, count, res);
   2.100 -	return res;
   2.101 -}
   2.102 -
   2.103  
   2.104  /*
   2.105   * Zero Userspace
   2.106 @@ -148,7 +59,6 @@ do {									\
   2.107  unsigned long
   2.108  clear_user(void __user *to, unsigned long n)
   2.109  {
   2.110 -	might_sleep();
   2.111  	if (access_ok(VERIFY_WRITE, to, n))
   2.112  		__do_clear_user(to, n);
   2.113  	return n;
   2.114 @@ -172,49 +82,6 @@ unsigned long
   2.115  	return n;
   2.116  }
   2.117  
   2.118 -/**
   2.119 - * strlen_user: - Get the size of a string in user space.
   2.120 - * @s: The string to measure.
   2.121 - * @n: The maximum valid length
   2.122 - *
   2.123 - * Get the size of a NUL-terminated string in user space.
   2.124 - *
   2.125 - * Returns the size of the string INCLUDING the terminating NUL.
   2.126 - * On exception, returns 0.
   2.127 - * If the string is too long, returns a value greater than @n.
   2.128 - */
   2.129 -long strnlen_user(const char __user *s, long n)
   2.130 -{
   2.131 -	unsigned long mask = -__addr_ok(s);
   2.132 -	unsigned long res, tmp;
   2.133 -
   2.134 -	might_sleep();
   2.135 -
   2.136 -	__asm__ __volatile__(
   2.137 -		"	testl %0, %0\n"
   2.138 -		"	jz 3f\n"
   2.139 -		"	andl %0,%%ecx\n"
   2.140 -		"0:	repne; scasb\n"
   2.141 -		"	setne %%al\n"
   2.142 -		"	subl %%ecx,%0\n"
   2.143 -		"	addl %0,%%eax\n"
   2.144 -		"1:\n"
   2.145 -		".section .fixup,\"ax\"\n"
   2.146 -		"2:	xorl %%eax,%%eax\n"
   2.147 -		"	jmp 1b\n"
   2.148 -		"3:	movb $1,%%al\n"
   2.149 -		"	jmp 1b\n"
   2.150 -		".previous\n"
   2.151 -		".section __ex_table,\"a\"\n"
   2.152 -		"	.align 4\n"
   2.153 -		"	.long 0b,2b\n"
   2.154 -		".previous"
   2.155 -		:"=r" (n), "=D" (s), "=a" (res), "=c" (tmp)
   2.156 -		:"0" (n), "1" (s), "2" (0), "3" (mask)
   2.157 -		:"cc");
   2.158 -	return res & mask;
   2.159 -}
   2.160 -
   2.161  #ifdef CONFIG_X86_INTEL_USERCOPY
   2.162  static unsigned long
   2.163  __copy_user_intel(void __user *to, const void *from, unsigned long size)
   2.164 @@ -543,12 +410,10 @@ unsigned long
   2.165  unsigned long
   2.166  copy_to_user(void __user *to, const void *from, unsigned long n)
   2.167  {
   2.168 -	might_sleep();
   2.169  	if (access_ok(VERIFY_WRITE, to, n))
   2.170  		n = __copy_to_user(to, from, n);
   2.171  	return n;
   2.172  }
   2.173 -EXPORT_SYMBOL(copy_to_user);
   2.174  
   2.175  /**
   2.176   * copy_from_user: - Copy a block of data from user space.
   2.177 @@ -569,11 +434,9 @@ EXPORT_SYMBOL(copy_to_user);
   2.178  unsigned long
   2.179  copy_from_user(void *to, const void __user *from, unsigned long n)
   2.180  {
   2.181 -	might_sleep();
   2.182  	if (access_ok(VERIFY_READ, from, n))
   2.183  		n = __copy_from_user(to, from, n);
   2.184  	else
   2.185  		memset(to, 0, n);
   2.186  	return n;
   2.187  }
   2.188 -EXPORT_SYMBOL(copy_from_user);

     3.1 --- a/xen/arch/x86/x86_64/usercopy.c	Sun Feb 06 18:49:34 2005 +0000
     3.2 +++ b/xen/arch/x86/x86_64/usercopy.c	Sun Feb 06 21:30:19 2005 +0000
     3.3 @@ -8,55 +8,6 @@
     3.4  #include <asm/uaccess.h>
     3.5  
     3.6  /*
     3.7 - * Copy a null terminated string from userspace.
     3.8 - */
     3.9 -
    3.10 -#define __do_strncpy_from_user(dst,src,count,res)			   \
    3.11 -do {									   \
    3.12 -	long __d0, __d1, __d2;						   \
    3.13 -	__asm__ __volatile__(						   \
    3.14 -		"	testq %1,%1\n"					   \
    3.15 -		"	jz 2f\n"					   \
    3.16 -		"0:	lodsb\n"					   \
    3.17 -		"	stosb\n"					   \
    3.18 -		"	testb %%al,%%al\n"				   \
    3.19 -		"	jz 1f\n"					   \
    3.20 -		"	decq %1\n"					   \
    3.21 -		"	jnz 0b\n"					   \
    3.22 -		"1:	subq %1,%0\n"					   \
    3.23 -		"2:\n"							   \
    3.24 -		".section .fixup,\"ax\"\n"				   \
    3.25 -		"3:	movq %5,%0\n"					   \
    3.26 -		"	jmp 2b\n"					   \
    3.27 -		".previous\n"						   \
    3.28 -		".section __ex_table,\"a\"\n"				   \
    3.29 -		"	.align 8\n"					   \
    3.30 -		"	.quad 0b,3b\n"					   \
    3.31 -		".previous"						   \
    3.32 -		: "=r"(res), "=c"(count), "=&a" (__d0), "=&S" (__d1),	   \
    3.33 -		  "=&D" (__d2)						   \
    3.34 -		: "i"(-EFAULT), "0"(count), "1"(count), "3"(src), "4"(dst) \
    3.35 -		: "memory");						   \
    3.36 -} while (0)
    3.37 -
    3.38 -long
    3.39 -__strncpy_from_user(char *dst, const char *src, long count)
    3.40 -{
    3.41 -	long res;
    3.42 -	__do_strncpy_from_user(dst, src, count, res);
    3.43 -	return res;
    3.44 -}
    3.45 -
    3.46 -long
    3.47 -strncpy_from_user(char *dst, const char *src, long count)
    3.48 -{
    3.49 -	long res = -EFAULT;
    3.50 -	if (access_ok(VERIFY_READ, src, 1))
    3.51 -		__do_strncpy_from_user(dst, src, count, res);
    3.52 -	return res;
    3.53 -}
    3.54 -
    3.55 -/*
    3.56   * Zero Userspace
    3.57   */
    3.58  
    3.59 @@ -93,6 +44,86 @@ unsigned long __clear_user(void *addr, u
    3.60  	return size;
    3.61  }
    3.62  
    3.63 +unsigned long __copy_to_user_ll(void __user *to, const void *from, unsigned n)
    3.64 +{
    3.65 +	unsigned long __d0, __d1, __d2, __n = n;
    3.66 +	__asm__ __volatile__(
    3.67 +		"	cmpq  $15,%0\n"
    3.68 +		"	jbe  1f\n"
    3.69 +		"	mov  %1,%0\n"
    3.70 +		"	neg  %0\n"
    3.71 +		"	and  $7,%0\n"
    3.72 +		"	sub  %0,%3\n"
    3.73 +		"4:	rep; movsb\n" /* make 'to' address aligned */
    3.74 +		"	mov  %3,%0\n"
    3.75 +		"	shr  $3,%0\n"
    3.76 +		"	and  $7,%3\n"
    3.77 +		"	.align 2,0x90\n"
    3.78 +		"0:	rep; movsq\n" /* as many quadwords as possible... */
    3.79 +		"	mov  %3,%0\n"
    3.80 +		"1:	rep; movsb\n" /* ...remainder copied as bytes */
    3.81 +		"2:\n"
    3.82 +		".section .fixup,\"ax\"\n"
    3.83 +		"5:	add %3,%0\n"
    3.84 +		"	jmp 2b\n"
    3.85 +		"3:	lea 0(%3,%0,8),%0\n"
    3.86 +		"	jmp 2b\n"
    3.87 +		".previous\n"
    3.88 +		".section __ex_table,\"a\"\n"
    3.89 +		"	.align 8\n"
    3.90 +		"	.quad 4b,5b\n"
    3.91 +		"	.quad 0b,3b\n"
    3.92 +		"	.quad 1b,2b\n"
    3.93 +		".previous"
    3.94 +		: "=&c"(__n), "=&D" (__d0), "=&S" (__d1), "=r"(__d2)
    3.95 +		: "3"(__n), "0"(__n), "1"(to), "2"(from)
    3.96 +		: "memory");
    3.97 +	return (unsigned)__n;
    3.98 +}
    3.99 +
   3.100 +unsigned long
   3.101 +__copy_from_user_ll(void *to, const void __user *from, unsigned n)
   3.102 +{
   3.103 +	unsigned long __d0, __d1, __d2, __n = n;
   3.104 +	__asm__ __volatile__(
   3.105 +		"	cmp  $15,%0\n"
   3.106 +		"	jbe  1f\n"
   3.107 +		"	mov  %1,%0\n"
   3.108 +		"	neg  %0\n"
   3.109 +		"	and  $7,%0\n"
   3.110 +		"	sub  %0,%3\n"
   3.111 +		"4:	rep; movsb\n" /* make 'to' address aligned */
   3.112 +		"	mov  %3,%0\n"
   3.113 +		"	shr  $3,%0\n"
   3.114 +		"	and  $7,%3\n"
   3.115 +		"	.align 2,0x90\n"
   3.116 +		"0:	rep; movsq\n" /* as many quadwords as possible... */
   3.117 +		"	mov  %3,%0\n"
   3.118 +		"1:	rep; movsb\n" /* ...remainder copied as bytes */
   3.119 +		"2:\n"
   3.120 +		".section .fixup,\"ax\"\n"
   3.121 +		"5:	add %3,%0\n"
   3.122 +		"	jmp 6f\n"
   3.123 +		"3:	lea 0(%3,%0,8),%0\n"
   3.124 +		"6:	push %0\n"
   3.125 +		"	push %%rax\n"
   3.126 +		"	xor  %%rax,%%rax\n"
   3.127 +		"	rep; stosb\n"
   3.128 +		"	pop  %%rax\n"
   3.129 +		"	pop  %0\n"
   3.130 +		"	jmp 2b\n"
   3.131 +		".previous\n"
   3.132 +		".section __ex_table,\"a\"\n"
   3.133 +		"	.align 8\n"
   3.134 +		"	.quad 4b,5b\n"
   3.135 +		"	.quad 0b,3b\n"
   3.136 +		"	.quad 1b,6b\n"
   3.137 +		".previous"
   3.138 +		: "=&c"(__n), "=&D" (__d0), "=&S" (__d1), "=r"(__d2)
   3.139 +		: "3"(__n), "0"(__n), "1"(to), "2"(from)
   3.140 +		: "memory");
   3.141 +	return (unsigned)__n;
   3.142 +}
   3.143  
   3.144  unsigned long clear_user(void *to, unsigned long n)
   3.145  {
   3.146 @@ -101,36 +132,49 @@ unsigned long clear_user(void *to, unsig
   3.147  	return n;
   3.148  }
   3.149  
   3.150 -/*
   3.151 - * Return the size of a string (including the ending 0)
   3.152 +/**
   3.153 + * copy_to_user: - Copy a block of data into user space.
   3.154 + * @to:   Destination address, in user space.
   3.155 + * @from: Source address, in kernel space.
   3.156 + * @n:    Number of bytes to copy.
   3.157 + *
   3.158 + * Context: User context only.  This function may sleep.
   3.159   *
   3.160 - * Return 0 on exception, a value greater than N if too long
   3.161 + * Copy data from kernel space to user space.
   3.162 + *
   3.163 + * Returns number of bytes that could not be copied.
   3.164 + * On success, this will be zero.
   3.165   */
   3.166 -
   3.167 -long strnlen_user(const char *s, long n)
   3.168 +unsigned long
   3.169 +copy_to_user(void __user *to, const void *from, unsigned n)
   3.170  {
   3.171 -	unsigned long res = 0;
   3.172 -	char c;
   3.173 -
   3.174 -	if (!access_ok(VERIFY_READ, s, n))
   3.175 -		return 0;
   3.176 -
   3.177 -	while (1) {
   3.178 -		if (get_user(c, s))
   3.179 -			return 0;
   3.180 -		if (!c)
   3.181 -			return res+1;
   3.182 -		if (res>n)
   3.183 -			return n+1;
   3.184 -		res++;
   3.185 -		s++;
   3.186 -	}
   3.187 +	if (access_ok(VERIFY_WRITE, to, n))
   3.188 +		n = __copy_to_user(to, from, n);
   3.189 +	return n;
   3.190  }
   3.191  
   3.192 -unsigned long copy_in_user(void *to, const void *from, unsigned len)
   3.193 +/**
   3.194 + * copy_from_user: - Copy a block of data from user space.
   3.195 + * @to:   Destination address, in kernel space.
   3.196 + * @from: Source address, in user space.
   3.197 + * @n:    Number of bytes to copy.
   3.198 + *
   3.199 + * Context: User context only.  This function may sleep.
   3.200 + *
   3.201 + * Copy data from user space to kernel space.
   3.202 + *
   3.203 + * Returns number of bytes that could not be copied.
   3.204 + * On success, this will be zero.
   3.205 + *
   3.206 + * If some data could not be copied, this function will pad the copied
   3.207 + * data to the requested size using zero bytes.
   3.208 + */
   3.209 +unsigned long
   3.210 +copy_from_user(void *to, const void __user *from, unsigned n)
   3.211  {
   3.212 -	if (access_ok(VERIFY_WRITE, to, len) && access_ok(VERIFY_READ, from, len)) { 
   3.213 -		return copy_user_generic(to, from, len);
   3.214 -	} 
   3.215 -	return len;		
   3.216 +	if (access_ok(VERIFY_READ, from, n))
   3.217 +		n = __copy_from_user(to, from, n);
   3.218 +	else
   3.219 +		memset(to, 0, n);
   3.220 +	return n;
   3.221  }

     4.1 --- a/xen/common/dom_mem_ops.c	Sun Feb 06 18:49:34 2005 +0000
     4.2 +++ b/xen/common/dom_mem_ops.c	Sun Feb 06 21:30:19 2005 +0000
     4.3 @@ -35,7 +35,7 @@ static long
     4.4  alloc_dom_mem(struct domain *d, 
     4.5                unsigned long *extent_list, 
     4.6                unsigned long  start_extent,
     4.7 -              unsigned long  nr_extents,
     4.8 +              unsigned int   nr_extents,
     4.9                unsigned int   extent_order)
    4.10  {
    4.11      struct pfn_info *page;
    4.12 @@ -73,7 +73,7 @@ static long
    4.13  free_dom_mem(struct domain *d,
    4.14               unsigned long *extent_list, 
    4.15               unsigned long  start_extent,
    4.16 -             unsigned long  nr_extents,
    4.17 +             unsigned int   nr_extents,
    4.18               unsigned int   extent_order)
    4.19  {
    4.20      struct pfn_info *page;
    4.21 @@ -134,7 +134,7 @@ do_dom_mem_op(unsigned long  op,
    4.22      op           &= (1 << START_EXTENT_SHIFT) - 1;
    4.23  
    4.24      if ( unlikely(start_extent > nr_extents) || 
    4.25 -         unlikely(nr_extents > (~0UL >> START_EXTENT_SHIFT)) )
    4.26 +         unlikely(nr_extents > ~0U) ) /* can pack into a uint? */
    4.27          return -EINVAL;
    4.28  
    4.29      if ( likely(domid == DOMID_SELF) )
    4.30 @@ -150,11 +150,13 @@ do_dom_mem_op(unsigned long  op,
    4.31      {
    4.32      case MEMOP_increase_reservation:
    4.33          rc = alloc_dom_mem(
    4.34 -            d, extent_list, start_extent, nr_extents, extent_order);
    4.35 +            d, extent_list, start_extent, 
    4.36 +            (unsigned int)nr_extents, extent_order);
    4.37          break;
    4.38      case MEMOP_decrease_reservation:
    4.39          rc = free_dom_mem(
    4.40 -            d, extent_list, start_extent, nr_extents, extent_order);
    4.41 +            d, extent_list, start_extent, 
    4.42 +            (unsigned int)nr_extents, extent_order);
    4.43          break;
    4.44      default:
    4.45          rc = -ENOSYS;

     5.1 --- a/xen/include/asm-x86/x86_32/uaccess.h	Sun Feb 06 18:49:34 2005 +0000
     5.2 +++ b/xen/include/asm-x86/x86_32/uaccess.h	Sun Feb 06 21:30:19 2005 +0000
     5.3 @@ -10,9 +10,7 @@
     5.4  #include <xen/string.h>
     5.5  #include <xen/sched.h>
     5.6  
     5.7 -/* No user-pointer checking. */
     5.8  #define __user
     5.9 -#define __chk_user_ptr(_p) ((void)0)
    5.10  
    5.11  #define VERIFY_READ 0
    5.12  #define VERIFY_WRITE 1
    5.13 @@ -22,7 +20,7 @@
    5.14   */
    5.15  #ifdef CONFIG_X86_INTEL_USERCOPY
    5.16  extern struct movsl_mask {
    5.17 -	int mask;
    5.18 +    int mask;
    5.19  } __cacheline_aligned movsl_mask;
    5.20  #endif
    5.21  
    5.22 @@ -34,41 +32,22 @@ extern struct movsl_mask {
    5.23   *
    5.24   * This is equivalent to the following test:
    5.25   * (u33)addr + (u33)size >= (u33)HYPERVISOR_VIRT_START
    5.26 - *
    5.27 - * This needs 33-bit arithmetic. We have a carry...
    5.28   */
    5.29 -#define __range_ok(addr,size) ({ \
    5.30 +#define __range_not_ok(addr,size) ({ \
    5.31  	unsigned long flag,sum; \
    5.32 -	__chk_user_ptr(addr); \
    5.33  	asm("addl %3,%1 ; sbbl %0,%0; cmpl %1,%4; sbbl $0,%0" \
    5.34  		:"=&r" (flag), "=r" (sum) \
    5.35  		:"1" (addr),"g" ((int)(size)),"r" (HYPERVISOR_VIRT_START)); \
    5.36  	flag; })
    5.37  
    5.38 -/**
    5.39 - * access_ok: - Checks if a user space pointer is valid
    5.40 - * @type: Type of access: %VERIFY_READ or %VERIFY_WRITE.  Note that
    5.41 - *        %VERIFY_WRITE is a superset of %VERIFY_READ - if it is safe
    5.42 - *        to write to a block, it is always safe to read from it.
    5.43 - * @addr: User space pointer to start of block to check
    5.44 - * @size: Size of block to check
    5.45 - *
    5.46 - * Context: User context only.  This function may sleep.
    5.47 - *
    5.48 - * Checks if a pointer to a block of memory in user space is valid.
    5.49 - *
    5.50 - * Returns true (nonzero) if the memory block may be valid, false (zero)
    5.51 - * if it is definitely invalid.
    5.52 - *
    5.53 - * Note that, depending on architecture, this function probably just
    5.54 - * checks that the pointer is in the user space range - after calling
    5.55 - * this function, memory access functions may still return -EFAULT.
    5.56 - */
    5.57 -#define access_ok(type,addr,size) (likely(__range_ok(addr,size) == 0))
    5.58 +#define access_ok(type,addr,size) (likely(__range_not_ok(addr,size) == 0))
    5.59  
    5.60  #define array_access_ok(type,addr,count,size) \
    5.61      (likely(count < (~0UL/size)) && access_ok(type,addr,count*size))
    5.62  
    5.63 +extern long __get_user_bad(void);
    5.64 +extern void __put_user_bad(void);
    5.65 +
    5.66  /**
    5.67   * get_user: - Get a simple variable from user space.
    5.68   * @x:   Variable to store result.
    5.69 @@ -89,8 +68,6 @@ extern struct movsl_mask {
    5.70  #define get_user(x,ptr)	\
    5.71    __get_user_check((x),(ptr),sizeof(*(ptr)))
    5.72  
    5.73 -extern void __put_user_bad(void);
    5.74 -
    5.75  /**
    5.76   * put_user: - Write a simple value into user space.
    5.77   * @x:   Value to copy to user space.
    5.78 @@ -195,7 +172,6 @@ extern void __put_user_bad(void);
    5.79  #define __put_user_size(x,ptr,size,retval,errret)			\
    5.80  do {									\
    5.81  	retval = 0;							\
    5.82 -	__chk_user_ptr(ptr);						\
    5.83  	switch (size) {							\
    5.84  	case 1: __put_user_asm(x,ptr,retval,"b","b","iq",errret);break;	\
    5.85  	case 2: __put_user_asm(x,ptr,retval,"w","w","ir",errret);break; \
    5.86 @@ -259,12 +235,9 @@ struct __large_struct { unsigned long bu
    5.87  	__gu_err;							\
    5.88  })							
    5.89  
    5.90 -extern long __get_user_bad(void);
    5.91 -
    5.92  #define __get_user_size(x,ptr,size,retval,errret)			\
    5.93  do {									\
    5.94  	retval = 0;							\
    5.95 -	__chk_user_ptr(ptr);						\
    5.96  	switch (size) {							\
    5.97  	case 1: __get_user_asm(x,ptr,retval,"b","b","=q",errret);break;	\
    5.98  	case 2: __get_user_asm(x,ptr,retval,"w","w","=r",errret);break;	\
    5.99 @@ -317,22 +290,22 @@ unsigned long __copy_from_user_ll(void *
   5.100  static always_inline unsigned long
   5.101  __copy_to_user(void __user *to, const void *from, unsigned long n)
   5.102  {
   5.103 -	if (__builtin_constant_p(n)) {
   5.104 -		unsigned long ret;
   5.105 +    if (__builtin_constant_p(n)) {
   5.106 +        unsigned long ret;
   5.107  
   5.108 -		switch (n) {
   5.109 -		case 1:
   5.110 -			__put_user_size(*(u8 *)from, (u8 __user *)to, 1, ret, 1);
   5.111 -			return ret;
   5.112 -		case 2:
   5.113 -			__put_user_size(*(u16 *)from, (u16 __user *)to, 2, ret, 2);
   5.114 -			return ret;
   5.115 -		case 4:
   5.116 -			__put_user_size(*(u32 *)from, (u32 __user *)to, 4, ret, 4);
   5.117 -			return ret;
   5.118 -		}
   5.119 -	}
   5.120 -	return __copy_to_user_ll(to, from, n);
   5.121 +        switch (n) {
   5.122 +        case 1:
   5.123 +            __put_user_size(*(u8 *)from, (u8 __user *)to, 1, ret, 1);
   5.124 +            return ret;
   5.125 +        case 2:
   5.126 +            __put_user_size(*(u16 *)from, (u16 __user *)to, 2, ret, 2);
   5.127 +            return ret;
   5.128 +        case 4:
   5.129 +            __put_user_size(*(u32 *)from, (u32 __user *)to, 4, ret, 4);
   5.130 +            return ret;
   5.131 +        }
   5.132 +    }
   5.133 +    return __copy_to_user_ll(to, from, n);
   5.134  }
   5.135  
   5.136  /**
   5.137 @@ -355,47 +328,28 @@ static always_inline unsigned long
   5.138  static always_inline unsigned long
   5.139  __copy_from_user(void *to, const void __user *from, unsigned long n)
   5.140  {
   5.141 -	if (__builtin_constant_p(n)) {
   5.142 -		unsigned long ret;
   5.143 +    if (__builtin_constant_p(n)) {
   5.144 +        unsigned long ret;
   5.145  
   5.146 -		switch (n) {
   5.147 -		case 1:
   5.148 -			__get_user_size(*(u8 *)to, from, 1, ret, 1);
   5.149 -			return ret;
   5.150 -		case 2:
   5.151 -			__get_user_size(*(u16 *)to, from, 2, ret, 2);
   5.152 -			return ret;
   5.153 -		case 4:
   5.154 -			__get_user_size(*(u32 *)to, from, 4, ret, 4);
   5.155 -			return ret;
   5.156 -		}
   5.157 -	}
   5.158 -	return __copy_from_user_ll(to, from, n);
   5.159 +        switch (n) {
   5.160 +        case 1:
   5.161 +            __get_user_size(*(u8 *)to, from, 1, ret, 1);
   5.162 +            return ret;
   5.163 +        case 2:
   5.164 +            __get_user_size(*(u16 *)to, from, 2, ret, 2);
   5.165 +            return ret;
   5.166 +        case 4:
   5.167 +            __get_user_size(*(u32 *)to, from, 4, ret, 4);
   5.168 +            return ret;
   5.169 +        }
   5.170 +    }
   5.171 +    return __copy_from_user_ll(to, from, n);
   5.172  }
   5.173  
   5.174  unsigned long copy_to_user(void __user *to, const void *from, unsigned long n);
   5.175  unsigned long copy_from_user(void *to,
   5.176 -			const void __user *from, unsigned long n);
   5.177 -long strncpy_from_user(char *dst, const char __user *src, long count);
   5.178 -long __strncpy_from_user(char *dst, const char __user *src, long count);
   5.179 +                             const void __user *from, unsigned long n);
   5.180  
   5.181 -/**
   5.182 - * strlen_user: - Get the size of a string in user space.
   5.183 - * @str: The string to measure.
   5.184 - *
   5.185 - * Context: User context only.  This function may sleep.
   5.186 - *
   5.187 - * Get the size of a NUL-terminated string in user space.
   5.188 - *
   5.189 - * Returns the size of the string INCLUDING the terminating NUL.
   5.190 - * On exception, returns 0.
   5.191 - *
   5.192 - * If there is a limit on the length of a valid string, you may wish to
   5.193 - * consider using strnlen_user() instead.
   5.194 - */
   5.195 -#define strlen_user(str) strnlen_user(str, ~0UL >> 1)
   5.196 -
   5.197 -long strnlen_user(const char __user *str, long n);
   5.198  unsigned long clear_user(void __user *mem, unsigned long len);
   5.199  unsigned long __clear_user(void __user *mem, unsigned long len);
   5.200  

     6.1 --- a/xen/include/asm-x86/x86_64/uaccess.h	Sun Feb 06 18:49:34 2005 +0000
     6.2 +++ b/xen/include/asm-x86/x86_64/uaccess.h	Sun Feb 06 21:30:19 2005 +0000
     6.3 @@ -11,10 +11,7 @@
     6.4  #include <xen/prefetch.h>
     6.5  #include <asm/page.h>
     6.6  
     6.7 -/* No user-pointer checking. */
     6.8  #define __user
     6.9 -#define __force
    6.10 -#define __chk_user_ptr(_p) ((void)0)
    6.11  
    6.12  #define VERIFY_READ 0
    6.13  #define VERIFY_WRITE 1
    6.14 @@ -22,122 +19,149 @@
    6.15  #define __addr_ok(addr) ((unsigned long)(addr) < HYPERVISOR_VIRT_START)
    6.16  
    6.17  /*
    6.18 - * Uhhuh, this needs 65-bit arithmetic. We have a carry..
    6.19 + * Test whether a block of memory is a valid user space address.
    6.20 + * Returns 0 if the range is valid, nonzero otherwise.
    6.21 + *
    6.22 + * This is equivalent to the following test:
    6.23 + * ((u65)addr >= (u65)HYPERVISOR_VIRT_END) ?
    6.24 + * (((u65)addr + (u65)size) >= ((u65)1 << 64)) :
    6.25 + * (((u65)addr + (u65)size) >= ((u65)HYPERVISOR_VIRT_START))
    6.26   */
    6.27  #define __range_not_ok(addr,size) ({ \
    6.28 -	unsigned long flag,sum; \
    6.29 -	__chk_user_ptr(addr); \
    6.30 -	asm("# range_ok\n\r" \
    6.31 -		"addq %3,%1 ; sbbq %0,%0 ; cmpq %1,%4 ; sbbq $0,%0"  \
    6.32 -		:"=&r" (flag), "=r" (sum) \
    6.33 -		:"1" (addr),"g" ((long)(size)),"r" (HYPERVISOR_VIRT_START)); \
    6.34 -	flag; })
    6.35 +    unsigned long flag,sum; \
    6.36 +    if ((unsigned long)addr >= HYPERVISOR_VIRT_END) \
    6.37 +        asm("addq %3,%1 ; sbbq %0,%0" \
    6.38 +            :"=&r" (flag), "=r" (sum) \
    6.39 +            :"1" (addr),"g" ((long)(size))); \
    6.40 +    else \
    6.41 +        asm("addq %3,%1 ; sbbq %0,%0 ; cmpq %1,%4 ; sbbq $0,%0"  \
    6.42 +            :"=&r" (flag), "=r" (sum) \
    6.43 +            :"1" (addr),"g" ((long)(size)),"r" (HYPERVISOR_VIRT_START)); \
    6.44 +    flag; })
    6.45  
    6.46  #define access_ok(type, addr, size) (__range_not_ok(addr,size) == 0)
    6.47  
    6.48  #define array_access_ok(type,addr,count,size)                    \
    6.49      (likely(sizeof(count) <= 4) /* disallow 64-bit counts */ &&  \
    6.50 -     access_ok(type,addr,count*size))
    6.51 -
    6.52 -extern inline int verify_area(int type, const void __user * addr, unsigned long size)
    6.53 -{
    6.54 -	return access_ok(type,addr,size) ? 0 : -EFAULT;
    6.55 -}
    6.56 -
    6.57 -/*
    6.58 - * These are the main single-value transfer routines.  They automatically
    6.59 - * use the right size if we just have the right pointer type.
    6.60 - *
    6.61 - * This gets kind of ugly. We want to return _two_ values in "get_user()"
    6.62 - * and yet we don't want to do any pointers, because that is too much
    6.63 - * of a performance impact. Thus we have a few rather ugly macros here,
    6.64 - * and hide all the ugliness from the user.
    6.65 - *
    6.66 - * The "__xxx" versions of the user access functions are versions that
    6.67 - * do not verify the address space, that must have been done previously
    6.68 - * with a separate "access_ok()" call (this is used when we do multiple
    6.69 - * accesses to the same area of user memory).
    6.70 - */
    6.71 -
    6.72 -extern void __get_user_1(void);
    6.73 -extern void __get_user_2(void);
    6.74 -extern void __get_user_4(void);
    6.75 -extern void __get_user_8(void);
    6.76 +     access_ok(type,addr,(unsigned long)count*(unsigned long)size))
    6.77  
    6.78 -#define __get_user_x(size,ret,x,ptr) \
    6.79 -	__asm__ __volatile__("call __get_user_" #size \
    6.80 -		:"=a" (ret),"=d" (x) \
    6.81 -		:"0" (ptr) \
    6.82 -		:"rbx")
    6.83 -
    6.84 -/* Careful: we have to cast the result to the type of the pointer for sign reasons */
    6.85 -#define get_user(x,ptr)							\
    6.86 -({	long __val_gu;							\
    6.87 -	int __ret_gu; 							\
    6.88 -	__chk_user_ptr(ptr);						\
    6.89 -	switch(sizeof (*(ptr))) {					\
    6.90 -	case 1:  __get_user_x(1,__ret_gu,__val_gu,ptr); break;		\
    6.91 -	case 2:  __get_user_x(2,__ret_gu,__val_gu,ptr); break;		\
    6.92 -	case 4:  __get_user_x(4,__ret_gu,__val_gu,ptr); break;		\
    6.93 -	case 8:  __get_user_x(8,__ret_gu,__val_gu,ptr); break;		\
    6.94 -	default: __get_user_bad(); break;				\
    6.95 -	}								\
    6.96 -	(x) = (__typeof__(*(ptr)))__val_gu;				\
    6.97 -	__ret_gu;							\
    6.98 -})
    6.99 -
   6.100 -extern void __put_user_1(void);
   6.101 -extern void __put_user_2(void);
   6.102 -extern void __put_user_4(void);
   6.103 -extern void __put_user_8(void);
   6.104 -
   6.105 +extern long __get_user_bad(void);
   6.106  extern void __put_user_bad(void);
   6.107  
   6.108 -#define __put_user_x(size,ret,x,ptr)					\
   6.109 -	__asm__ __volatile__("call __put_user_" #size			\
   6.110 -		:"=a" (ret)						\
   6.111 -		:"0" (ptr),"d" (x)					\
   6.112 -		:"rbx")
   6.113 +/**
   6.114 + * get_user: - Get a simple variable from user space.
   6.115 + * @x:   Variable to store result.
   6.116 + * @ptr: Source address, in user space.
   6.117 + *
   6.118 + * Context: User context only.  This function may sleep.
   6.119 + *
   6.120 + * This macro copies a single simple variable from user space to kernel
   6.121 + * space.  It supports simple types like char and int, but not larger
   6.122 + * data types like structures or arrays.
   6.123 + *
   6.124 + * @ptr must have pointer-to-simple-variable type, and the result of
   6.125 + * dereferencing @ptr must be assignable to @x without a cast.
   6.126 + *
   6.127 + * Returns zero on success, or -EFAULT on error.
   6.128 + * On error, the variable @x is set to zero.
   6.129 + */
   6.130 +#define get_user(x,ptr)	\
   6.131 +  __get_user_check((x),(ptr),sizeof(*(ptr)))
   6.132  
   6.133 +/**
   6.134 + * put_user: - Write a simple value into user space.
   6.135 + * @x:   Value to copy to user space.
   6.136 + * @ptr: Destination address, in user space.
   6.137 + *
   6.138 + * Context: User context only.  This function may sleep.
   6.139 + *
   6.140 + * This macro copies a single simple value from kernel space to user
   6.141 + * space.  It supports simple types like char and int, but not larger
   6.142 + * data types like structures or arrays.
   6.143 + *
   6.144 + * @ptr must have pointer-to-simple-variable type, and @x must be assignable
   6.145 + * to the result of dereferencing @ptr.
   6.146 + *
   6.147 + * Returns zero on success, or -EFAULT on error.
   6.148 + */
   6.149  #define put_user(x,ptr)							\
   6.150    __put_user_check((__typeof__(*(ptr)))(x),(ptr),sizeof(*(ptr)))
   6.151  
   6.152 +
   6.153 +/**
   6.154 + * __get_user: - Get a simple variable from user space, with less checking.
   6.155 + * @x:   Variable to store result.
   6.156 + * @ptr: Source address, in user space.
   6.157 + *
   6.158 + * Context: User context only.  This function may sleep.
   6.159 + *
   6.160 + * This macro copies a single simple variable from user space to kernel
   6.161 + * space.  It supports simple types like char and int, but not larger
   6.162 + * data types like structures or arrays.
   6.163 + *
   6.164 + * @ptr must have pointer-to-simple-variable type, and the result of
   6.165 + * dereferencing @ptr must be assignable to @x without a cast.
   6.166 + *
   6.167 + * Caller must check the pointer with access_ok() before calling this
   6.168 + * function.
   6.169 + *
   6.170 + * Returns zero on success, or -EFAULT on error.
   6.171 + * On error, the variable @x is set to zero.
   6.172 + */
   6.173  #define __get_user(x,ptr) \
   6.174    __get_user_nocheck((x),(ptr),sizeof(*(ptr)))
   6.175 +
   6.176 +
   6.177 +/**
   6.178 + * __put_user: - Write a simple value into user space, with less checking.
   6.179 + * @x:   Value to copy to user space.
   6.180 + * @ptr: Destination address, in user space.
   6.181 + *
   6.182 + * Context: User context only.  This function may sleep.
   6.183 + *
   6.184 + * This macro copies a single simple value from kernel space to user
   6.185 + * space.  It supports simple types like char and int, but not larger
   6.186 + * data types like structures or arrays.
   6.187 + *
   6.188 + * @ptr must have pointer-to-simple-variable type, and @x must be assignable
   6.189 + * to the result of dereferencing @ptr.
   6.190 + *
   6.191 + * Caller must check the pointer with access_ok() before calling this
   6.192 + * function.
   6.193 + *
   6.194 + * Returns zero on success, or -EFAULT on error.
   6.195 + */
   6.196  #define __put_user(x,ptr) \
   6.197    __put_user_nocheck((__typeof__(*(ptr)))(x),(ptr),sizeof(*(ptr)))
   6.198  
   6.199 -#define __put_user_nocheck(x,ptr,size)			\
   6.200 -({							\
   6.201 -	int __pu_err;					\
   6.202 -	__put_user_size((x),(ptr),(size),__pu_err);	\
   6.203 -	__pu_err;					\
   6.204 +#define __put_user_nocheck(x,ptr,size)				\
   6.205 +({								\
   6.206 +	long __pu_err;						\
   6.207 +	__put_user_size((x),(ptr),(size),__pu_err,-EFAULT);	\
   6.208 +	__pu_err;						\
   6.209  })
   6.210  
   6.211 +#define __put_user_check(x,ptr,size)					\
   6.212 +({									\
   6.213 +	long __pu_err = -EFAULT;					\
   6.214 +	__typeof__(*(ptr)) __user *__pu_addr = (ptr);			\
   6.215 +	if (__addr_ok(__pu_addr))					\
   6.216 +		__put_user_size((x),__pu_addr,(size),__pu_err,-EFAULT);	\
   6.217 +	__pu_err;							\
   6.218 +})							
   6.219  
   6.220 -#define __put_user_check(x,ptr,size)			\
   6.221 -({							\
   6.222 -	int __pu_err = -EFAULT;				\
   6.223 -	__typeof__(*(ptr)) __user *__pu_addr = (ptr);	\
   6.224 -	if (likely(access_ok(VERIFY_WRITE,__pu_addr,size)))	\
   6.225 -		__put_user_size((x),__pu_addr,(size),__pu_err);	\
   6.226 -	__pu_err;					\
   6.227 -})
   6.228 -
   6.229 -#define __put_user_size(x,ptr,size,retval)				\
   6.230 +#define __put_user_size(x,ptr,size,retval,errret)			\
   6.231  do {									\
   6.232  	retval = 0;							\
   6.233 -	__chk_user_ptr(ptr);						\
   6.234  	switch (size) {							\
   6.235 -	  case 1: __put_user_asm(x,ptr,retval,"b","b","iq",-EFAULT); break;\
   6.236 -	  case 2: __put_user_asm(x,ptr,retval,"w","w","ir",-EFAULT); break;\
   6.237 -	  case 4: __put_user_asm(x,ptr,retval,"l","k","ir",-EFAULT); break;\
   6.238 -	  case 8: __put_user_asm(x,ptr,retval,"q","","ir",-EFAULT); break;\
   6.239 -	  default: __put_user_bad();					\
   6.240 +	case 1: __put_user_asm(x,ptr,retval,"b","b","iq",errret);break;	\
   6.241 +	case 2: __put_user_asm(x,ptr,retval,"w","w","ir",errret);break; \
   6.242 +	case 4: __put_user_asm(x,ptr,retval,"l","k","ir",errret);break;	\
   6.243 +	case 8: __put_user_asm(x,ptr,retval,"q","","ir",errret);break;	\
   6.244 +	default: __put_user_bad();					\
   6.245  	}								\
   6.246  } while (0)
   6.247  
   6.248 -/* FIXME: this hack is definitely wrong -AK */
   6.249  struct __large_struct { unsigned long buf[100]; };
   6.250  #define __m(x) (*(struct __large_struct *)(x))
   6.251  
   6.252 @@ -146,178 +170,139 @@ struct __large_struct { unsigned long bu
   6.253   * we do not write to any memory gcc knows about, so there are no
   6.254   * aliasing issues.
   6.255   */
   6.256 -#define __put_user_asm(x, addr, err, itype, rtype, ltype, errno)	\
   6.257 -	__asm__ __volatile__(					\
   6.258 -		"1:	mov"itype" %"rtype"1,%2\n"		\
   6.259 -		"2:\n"						\
   6.260 -		".section .fixup,\"ax\"\n"			\
   6.261 -		"3:	mov %3,%0\n"				\
   6.262 -		"	jmp 2b\n"				\
   6.263 -		".previous\n"					\
   6.264 -		".section __ex_table,\"a\"\n"			\
   6.265 -		"	.align 8\n"				\
   6.266 -		"	.quad 1b,3b\n"				\
   6.267 -		".previous"					\
   6.268 -		: "=r"(err)					\
   6.269 -		: ltype (x), "m"(__m(addr)), "i"(errno), "0"(err))
   6.270 -
   6.271 +#define __put_user_asm(x, addr, err, itype, rtype, ltype, errret)	\
   6.272 +	__asm__ __volatile__(						\
   6.273 +		"1:	mov"itype" %"rtype"1,%2\n"			\
   6.274 +		"2:\n"							\
   6.275 +		".section .fixup,\"ax\"\n"				\
   6.276 +		"3:	mov %3,%0\n"					\
   6.277 +		"	jmp 2b\n"					\
   6.278 +		".previous\n"						\
   6.279 +		".section __ex_table,\"a\"\n"				\
   6.280 +		"	.align 8\n"					\
   6.281 +		"	.quad 1b,3b\n"					\
   6.282 +		".previous"						\
   6.283 +		: "=r"(err)						\
   6.284 +		: ltype (x), "m"(__m(addr)), "i"(errret), "0"(err))
   6.285  
   6.286  #define __get_user_nocheck(x,ptr,size)				\
   6.287  ({								\
   6.288 -	int __gu_err;						\
   6.289 -	long __gu_val;						\
   6.290 -	__get_user_size(__gu_val,(ptr),(size),__gu_err);	\
   6.291 +	long __gu_err, __gu_val;				\
   6.292 +	__get_user_size(__gu_val,(ptr),(size),__gu_err,-EFAULT);\
   6.293  	(x) = (__typeof__(*(ptr)))__gu_val;			\
   6.294  	__gu_err;						\
   6.295  })
   6.296  
   6.297 -extern int __get_user_bad(void);
   6.298 +#define __get_user_check(x,ptr,size)					\
   6.299 +({									\
   6.300 +	long __gu_err, __gu_val;					\
   6.301 +	__typeof__(*(ptr)) __user *__gu_addr = (ptr);			\
   6.302 +	__get_user_size(__gu_val,__gu_addr,(size),__gu_err,-EFAULT);	\
   6.303 +	(x) = (__typeof__(*(ptr)))__gu_val;				\
   6.304 +	if (!__addr_ok(__gu_addr)) __gu_err = -EFAULT;			\
   6.305 +	__gu_err;							\
   6.306 +})							
   6.307  
   6.308 -#define __get_user_size(x,ptr,size,retval)				\
   6.309 +#define __get_user_size(x,ptr,size,retval,errret)			\
   6.310  do {									\
   6.311  	retval = 0;							\
   6.312 -	__chk_user_ptr(ptr);						\
   6.313  	switch (size) {							\
   6.314 -	  case 1: __get_user_asm(x,ptr,retval,"b","b","=q",-EFAULT); break;\
   6.315 -	  case 2: __get_user_asm(x,ptr,retval,"w","w","=r",-EFAULT); break;\
   6.316 -	  case 4: __get_user_asm(x,ptr,retval,"l","k","=r",-EFAULT); break;\
   6.317 -	  case 8: __get_user_asm(x,ptr,retval,"q","","=r",-EFAULT); break;\
   6.318 -	  default: (x) = __get_user_bad();				\
   6.319 +	case 1: __get_user_asm(x,ptr,retval,"b","b","=q",errret);break;	\
   6.320 +	case 2: __get_user_asm(x,ptr,retval,"w","w","=r",errret);break;	\
   6.321 +	case 4: __get_user_asm(x,ptr,retval,"l","k","=r",errret);break;	\
   6.322 +	case 8: __get_user_asm(x,ptr,retval,"q","","=r",errret); break;	\
   6.323 +	default: (x) = __get_user_bad();				\
   6.324  	}								\
   6.325  } while (0)
   6.326  
   6.327 -#define __get_user_asm(x, addr, err, itype, rtype, ltype, errno)	\
   6.328 -	__asm__ __volatile__(					\
   6.329 -		"1:	mov"itype" %2,%"rtype"1\n"		\
   6.330 -		"2:\n"						\
   6.331 -		".section .fixup,\"ax\"\n"			\
   6.332 -		"3:	mov %3,%0\n"				\
   6.333 -		"	xor"itype" %"rtype"1,%"rtype"1\n"	\
   6.334 -		"	jmp 2b\n"				\
   6.335 -		".previous\n"					\
   6.336 -		".section __ex_table,\"a\"\n"			\
   6.337 -		"	.align 8\n"				\
   6.338 -		"	.quad 1b,3b\n"				\
   6.339 -		".previous"					\
   6.340 -		: "=r"(err), ltype (x)				\
   6.341 -		: "m"(__m(addr)), "i"(errno), "0"(err))
   6.342 +#define __get_user_asm(x, addr, err, itype, rtype, ltype, errret)	\
   6.343 +	__asm__ __volatile__(						\
   6.344 +		"1:	mov"itype" %2,%"rtype"1\n"			\
   6.345 +		"2:\n"							\
   6.346 +		".section .fixup,\"ax\"\n"				\
   6.347 +		"3:	mov %3,%0\n"					\
   6.348 +		"	xor"itype" %"rtype"1,%"rtype"1\n"		\
   6.349 +		"	jmp 2b\n"					\
   6.350 +		".previous\n"						\
   6.351 +		".section __ex_table,\"a\"\n"				\
   6.352 +		"	.align 8\n"					\
   6.353 +		"	.quad 1b,3b\n"					\
   6.354 +		".previous"						\
   6.355 +		: "=r"(err), ltype (x)					\
   6.356 +		: "m"(__m(addr)), "i"(errret), "0"(err))
   6.357 +
   6.358  
   6.359  /*
   6.360   * Copy To/From Userspace
   6.361   */
   6.362  
   6.363  /* Handles exceptions in both to and from, but doesn't do access_ok */
   6.364 -extern unsigned long copy_user_generic(void *to, const void *from, unsigned len); 
   6.365 +unsigned long __copy_to_user_ll(void __user *to, const void *from, unsigned n);
   6.366 +unsigned long __copy_from_user_ll(void *to, const void __user *from, unsigned n);
   6.367  
   6.368 -extern unsigned long copy_to_user(void __user *to, const void *from, unsigned len); 
   6.369 -extern unsigned long copy_from_user(void *to, const void __user *from, unsigned len); 
   6.370 -extern unsigned long copy_in_user(void __user *to, const void __user *from, unsigned len); 
   6.371 +unsigned long copy_to_user(void __user *to, const void *from, unsigned len); 
   6.372 +unsigned long copy_from_user(void *to, const void __user *from, unsigned len); 
   6.373  
   6.374  static always_inline int __copy_from_user(void *dst, const void __user *src, unsigned size) 
   6.375  { 
   6.376 -       int ret = 0;
   6.377 -	if (!__builtin_constant_p(size))
   6.378 -		return copy_user_generic(dst,(__force void *)src,size);
   6.379 -	switch (size) { 
   6.380 -	case 1:__get_user_asm(*(u8*)dst,(u8 __user *)src,ret,"b","b","=q",1); 
   6.381 -		return ret;
   6.382 -	case 2:__get_user_asm(*(u16*)dst,(u16 __user *)src,ret,"w","w","=r",2);
   6.383 -		return ret;
   6.384 -	case 4:__get_user_asm(*(u32*)dst,(u32 __user *)src,ret,"l","k","=r",4);
   6.385 -		return ret;
   6.386 -	case 8:__get_user_asm(*(u64*)dst,(u64 __user *)src,ret,"q","","=r",8);
   6.387 -		return ret; 
   6.388 -	case 10:
   6.389 -	       	__get_user_asm(*(u64*)dst,(u64 __user *)src,ret,"q","","=r",16);
   6.390 -		if (unlikely(ret)) return ret;
   6.391 -		__get_user_asm(*(u16*)(8+(char*)dst),(u16 __user *)(8+(char __user *)src),ret,"w","w","=r",2);
   6.392 -		return ret; 
   6.393 -	case 16:
   6.394 -		__get_user_asm(*(u64*)dst,(u64 __user *)src,ret,"q","","=r",16);
   6.395 -		if (unlikely(ret)) return ret;
   6.396 -		__get_user_asm(*(u64*)(8+(char*)dst),(u64 __user *)(8+(char __user *)src),ret,"q","","=r",8);
   6.397 -		return ret; 
   6.398 -	default:
   6.399 -		return copy_user_generic(dst,(__force void *)src,size); 
   6.400 -	}
   6.401 +    int ret = 0;
   6.402 +    if (!__builtin_constant_p(size))
   6.403 +        return __copy_from_user_ll(dst,(void *)src,size);
   6.404 +    switch (size) { 
   6.405 +    case 1:__get_user_asm(*(u8*)dst,(u8 __user *)src,ret,"b","b","=q",1); 
   6.406 +        return ret;
   6.407 +    case 2:__get_user_asm(*(u16*)dst,(u16 __user *)src,ret,"w","w","=r",2);
   6.408 +        return ret;
   6.409 +    case 4:__get_user_asm(*(u32*)dst,(u32 __user *)src,ret,"l","k","=r",4);
   6.410 +        return ret;
   6.411 +    case 8:__get_user_asm(*(u64*)dst,(u64 __user *)src,ret,"q","","=r",8);
   6.412 +        return ret; 
   6.413 +    case 10:
   6.414 +        __get_user_asm(*(u64*)dst,(u64 __user *)src,ret,"q","","=r",16);
   6.415 +        if (unlikely(ret)) return ret;
   6.416 +        __get_user_asm(*(u16*)(8+(char*)dst),(u16 __user *)(8+(char __user *)src),ret,"w","w","=r",2);
   6.417 +        return ret; 
   6.418 +    case 16:
   6.419 +        __get_user_asm(*(u64*)dst,(u64 __user *)src,ret,"q","","=r",16);
   6.420 +        if (unlikely(ret)) return ret;
   6.421 +        __get_user_asm(*(u64*)(8+(char*)dst),(u64 __user *)(8+(char __user *)src),ret,"q","","=r",8);
   6.422 +        return ret; 
   6.423 +    default:
   6.424 +        return __copy_from_user_ll(dst,(void *)src,size); 
   6.425 +    }
   6.426  }	
   6.427  
   6.428  static always_inline int __copy_to_user(void __user *dst, const void *src, unsigned size) 
   6.429  { 
   6.430 -       int ret = 0;
   6.431 -	if (!__builtin_constant_p(size))
   6.432 -		return copy_user_generic((__force void *)dst,src,size);
   6.433 -	switch (size) { 
   6.434 -	case 1:__put_user_asm(*(u8*)src,(u8 __user *)dst,ret,"b","b","iq",1); 
   6.435 -		return ret;
   6.436 -	case 2:__put_user_asm(*(u16*)src,(u16 __user *)dst,ret,"w","w","ir",2);
   6.437 -		return ret;
   6.438 -	case 4:__put_user_asm(*(u32*)src,(u32 __user *)dst,ret,"l","k","ir",4);
   6.439 -		return ret;
   6.440 -	case 8:__put_user_asm(*(u64*)src,(u64 __user *)dst,ret,"q","","ir",8);
   6.441 -		return ret; 
   6.442 -	case 10:
   6.443 -		__put_user_asm(*(u64*)src,(u64 __user *)dst,ret,"q","","ir",10);
   6.444 -		if (unlikely(ret)) return ret;
   6.445 -		asm("":::"memory");
   6.446 -		__put_user_asm(4[(u16*)src],4+(u16 __user *)dst,ret,"w","w","ir",2);
   6.447 -		return ret; 
   6.448 -	case 16:
   6.449 -		__put_user_asm(*(u64*)src,(u64 __user *)dst,ret,"q","","ir",16);
   6.450 -		if (unlikely(ret)) return ret;
   6.451 -		asm("":::"memory");
   6.452 -		__put_user_asm(1[(u64*)src],1+(u64 __user *)dst,ret,"q","","ir",8);
   6.453 -		return ret; 
   6.454 -	default:
   6.455 -		return copy_user_generic((__force void *)dst,src,size); 
   6.456 -	}
   6.457 +    int ret = 0;
   6.458 +    if (!__builtin_constant_p(size))
   6.459 +        return __copy_to_user_ll((void *)dst,src,size);
   6.460 +    switch (size) { 
   6.461 +    case 1:__put_user_asm(*(u8*)src,(u8 __user *)dst,ret,"b","b","iq",1); 
   6.462 +        return ret;
   6.463 +    case 2:__put_user_asm(*(u16*)src,(u16 __user *)dst,ret,"w","w","ir",2);
   6.464 +        return ret;
   6.465 +    case 4:__put_user_asm(*(u32*)src,(u32 __user *)dst,ret,"l","k","ir",4);
   6.466 +        return ret;
   6.467 +    case 8:__put_user_asm(*(u64*)src,(u64 __user *)dst,ret,"q","","ir",8);
   6.468 +        return ret; 
   6.469 +    case 10:
   6.470 +        __put_user_asm(*(u64*)src,(u64 __user *)dst,ret,"q","","ir",10);
   6.471 +        if (unlikely(ret)) return ret;
   6.472 +        asm("":::"memory");
   6.473 +        __put_user_asm(4[(u16*)src],4+(u16 __user *)dst,ret,"w","w","ir",2);
   6.474 +        return ret; 
   6.475 +    case 16:
   6.476 +        __put_user_asm(*(u64*)src,(u64 __user *)dst,ret,"q","","ir",16);
   6.477 +        if (unlikely(ret)) return ret;
   6.478 +        asm("":::"memory");
   6.479 +        __put_user_asm(1[(u64*)src],1+(u64 __user *)dst,ret,"q","","ir",8);
   6.480 +        return ret; 
   6.481 +    default:
   6.482 +        return __copy_to_user_ll((void *)dst,src,size); 
   6.483 +    }
   6.484  }	
   6.485  
   6.486 -
   6.487 -static always_inline int __copy_in_user(void __user *dst, const void __user *src, unsigned size) 
   6.488 -{ 
   6.489 -       int ret = 0;
   6.490 -	if (!__builtin_constant_p(size))
   6.491 -		return copy_user_generic((__force void *)dst,(__force void *)src,size);
   6.492 -	switch (size) { 
   6.493 -	case 1: { 
   6.494 -		u8 tmp;
   6.495 -		__get_user_asm(tmp,(u8 __user *)src,ret,"b","b","=q",1); 
   6.496 -		if (likely(!ret))
   6.497 -			__put_user_asm(tmp,(u8 __user *)dst,ret,"b","b","iq",1); 
   6.498 -		return ret;
   6.499 -	}
   6.500 -	case 2: { 
   6.501 -		u16 tmp;
   6.502 -		__get_user_asm(tmp,(u16 __user *)src,ret,"w","w","=r",2); 
   6.503 -		if (likely(!ret))
   6.504 -			__put_user_asm(tmp,(u16 __user *)dst,ret,"w","w","ir",2); 
   6.505 -		return ret;
   6.506 -	}
   6.507 -
   6.508 -	case 4: { 
   6.509 -		u32 tmp;
   6.510 -		__get_user_asm(tmp,(u32 __user *)src,ret,"l","k","=r",4); 
   6.511 -		if (likely(!ret))
   6.512 -			__put_user_asm(tmp,(u32 __user *)dst,ret,"l","k","ir",4); 
   6.513 -		return ret;
   6.514 -	}
   6.515 -	case 8: { 
   6.516 -		u64 tmp;
   6.517 -		__get_user_asm(tmp,(u64 __user *)src,ret,"q","","=r",8); 
   6.518 -		if (likely(!ret))
   6.519 -			__put_user_asm(tmp,(u64 __user *)dst,ret,"q","","ir",8); 
   6.520 -		return ret;
   6.521 -	}
   6.522 -	default:
   6.523 -		return copy_user_generic((__force void *)dst,(__force void *)src,size); 
   6.524 -	}
   6.525 -}	
   6.526 -
   6.527 -long strncpy_from_user(char *dst, const char __user *src, long count);
   6.528 -long __strncpy_from_user(char *dst, const char __user *src, long count);
   6.529 -long strnlen_user(const char __user *str, long n);
   6.530 -long strlen_user(const char __user *str);
   6.531  unsigned long clear_user(void __user *mem, unsigned long len);
   6.532  unsigned long __clear_user(void __user *mem, unsigned long len);
   6.533