1.1 --- a/.rootkeys	Sat Feb 05 02:13:44 2005 +0000
     1.2 +++ b/.rootkeys	Sat Feb 05 08:07:49 2005 +0000
     1.3 @@ -935,7 +935,6 @@ 4064773cJ31vZt-zhbSoxqft1Jaw0w xen/commo
     1.4  40589968dD2D1aejwSOvrROg7fOvGQ xen/common/sched_bvt.c
     1.5  40589968be_t_n0-w6ggceW7h-sx0w xen/common/sched_rrobin.c
     1.6  3e397e6619PgAfBbw2XFbXkewvUWgw xen/common/schedule.c
     1.7 -3ddb79bdB9RNMnkQnUyZ5C9hhMSQQw xen/common/slab.c
     1.8  3ddb79bd0gVQYmL2zvuJnldvD0AGxQ xen/common/softirq.c
     1.9  3e7f358awXBC3Vw-wFRwPw18qL1khg xen/common/string.c
    1.10  403a3edbejm33XLTGMuinKEwQBrOIg xen/common/trace.c

     2.1 --- a/xen/common/slab.c	Sat Feb 05 02:13:44 2005 +0000
     2.2 +++ /dev/null	Thu Jan 01 00:00:00 1970 +0000
     2.3 @@ -1,1844 +0,0 @@
     2.4 -/*
     2.5 - * linux/mm/slab.c
     2.6 - * Written by Mark Hemment, 1996/97.
     2.7 - * (markhe@nextd.demon.co.uk)
     2.8 - *
     2.9 - * xmem_cache_destroy() + some cleanup - 1999 Andrea Arcangeli
    2.10 - *
    2.11 - * Major cleanup, different bufctl logic, per-cpu arrays
    2.12 - *	(c) 2000 Manfred Spraul
    2.13 - *
    2.14 - * An implementation of the Slab Allocator as described in outline in;
    2.15 - *	UNIX Internals: The New Frontiers by Uresh Vahalia
    2.16 - *	Pub: Prentice Hall	ISBN 0-13-101908-2
    2.17 - * or with a little more detail in;
    2.18 - *	The Slab Allocator: An Object-Caching Kernel Memory Allocator
    2.19 - *	Jeff Bonwick (Sun Microsystems).
    2.20 - *	Presented at: USENIX Summer 1994 Technical Conference
    2.21 - *
    2.22 - *
    2.23 - * The memory is organized in caches, one cache for each object type.
    2.24 - * (e.g. inode_cache, dentry_cache, buffer_head, vm_area_struct)
    2.25 - * Each cache consists out of many slabs (they are small (usually one
    2.26 - * page long) and always contiguous), and each slab contains multiple
    2.27 - * initialized objects.
    2.28 - *
    2.29 - * In order to reduce fragmentation, the slabs are sorted in 3 groups:
    2.30 - *   full slabs with 0 free objects
    2.31 - *   partial slabs
    2.32 - *   empty slabs with no allocated objects
    2.33 - *
    2.34 - * If partial slabs exist, then new allocations come from these slabs,
    2.35 - * otherwise from empty slabs or new slabs are allocated.
    2.36 - *
    2.37 - * xmem_cache_destroy() CAN CRASH if you try to allocate from the cache
    2.38 - * during xmem_cache_destroy(). The caller must prevent concurrent allocs.
    2.39 - *
    2.40 - * On SMP systems, each cache has a short per-cpu head array, most allocs
    2.41 - * and frees go into that array, and if that array overflows, then 1/2
    2.42 - * of the entries in the array are given back into the global cache.
    2.43 - * This reduces the number of spinlock operations.
    2.44 - *
    2.45 - * The c_cpuarray may not be read with enabled local interrupts.
    2.46 - *
    2.47 - * SMP synchronization:
    2.48 - *  constructors and destructors are called without any locking.
    2.49 - *  Several members in xmem_cache_t and slab_t never change, they
    2.50 - *	are accessed without any locking.
    2.51 - *  The per-cpu arrays are never accessed from the wrong cpu, no locking.
    2.52 - *  The non-constant members are protected with a per-cache irq spinlock.
    2.53 - */
    2.54 -
    2.55 -#include <xen/config.h>
    2.56 -#include <xen/init.h>
    2.57 -#include <xen/types.h>
    2.58 -#include <xen/lib.h>
    2.59 -#include <xen/slab.h>
    2.60 -#include <xen/list.h>
    2.61 -#include <xen/spinlock.h>
    2.62 -#include <xen/errno.h>
    2.63 -#include <xen/smp.h>
    2.64 -#include <xen/sched.h>
    2.65 -
    2.66 -/*
    2.67 - * DEBUG  - 1 for xmem_cache_create() to honour; SLAB_DEBUG_INITIAL,
    2.68 - *	    SLAB_RED_ZONE & SLAB_POISON.
    2.69 - *	    0 for faster, smaller code (especially in the critical paths).
    2.70 - *
    2.71 - * STATS  - 1 to collect stats for /proc/slabinfo.
    2.72 - *	    0 for faster, smaller code (especially in the critical paths).
    2.73 - *
    2.74 - * FORCED_DEBUG	- 1 enables SLAB_RED_ZONE and SLAB_POISON (if possible)
    2.75 - */
    2.76 -#ifdef CONFIG_DEBUG_SLAB
    2.77 -#define	DEBUG		1
    2.78 -#define	STATS		1
    2.79 -#define	FORCED_DEBUG	1
    2.80 -#else
    2.81 -#define	DEBUG		0
    2.82 -#define	STATS		0
    2.83 -#define	FORCED_DEBUG	0
    2.84 -#endif
    2.85 -
    2.86 -/*
    2.87 - * Parameters for xmem_cache_reap
    2.88 - */
    2.89 -#define REAP_SCANLEN	10
    2.90 -#define REAP_PERFECT	10
    2.91 -
    2.92 -/* Shouldn't this be in a header file somewhere? */
    2.93 -#define	BYTES_PER_WORD		sizeof(void *)
    2.94 -
    2.95 -/* Legal flag mask for xmem_cache_create(). */
    2.96 -#if DEBUG
    2.97 -#define CREATE_MASK	(SLAB_DEBUG_INITIAL | SLAB_RED_ZONE | \
    2.98 -			 SLAB_POISON | SLAB_HWCACHE_ALIGN | \
    2.99 -			 SLAB_NO_REAP)
   2.100 -#else
   2.101 -#define CREATE_MASK	(SLAB_HWCACHE_ALIGN | SLAB_NO_REAP)
   2.102 -#endif
   2.103 -
   2.104 -/*
   2.105 - * xmem_bufctl_t:
   2.106 - *
   2.107 - * Bufctl's are used for linking objs within a slab
   2.108 - * linked offsets.
   2.109 - *
   2.110 - * This implementaion relies on "struct page" for locating the cache &
   2.111 - * slab an object belongs to.
   2.112 - * This allows the bufctl structure to be small (one int), but limits
   2.113 - * the number of objects a slab (not a cache) can contain when off-slab
   2.114 - * bufctls are used. The limit is the size of the largest general cache
   2.115 - * that does not use off-slab slabs.
   2.116 - * For 32bit archs with 4 kB pages, is this 56.
   2.117 - * This is not serious, as it is only for large objects, when it is unwise
   2.118 - * to have too many per slab.
   2.119 - * Note: This limit can be raised by introducing a general cache whose size
   2.120 - * is less than 512 (PAGE_SIZE<<3), but greater than 256.
   2.121 - */
   2.122 -
   2.123 -#define BUFCTL_END	(((xmem_bufctl_t)(~0U))-0)
   2.124 -#define BUFCTL_FREE	(((xmem_bufctl_t)(~0U))-1)
   2.125 -#define	SLAB_LIMIT	(((xmem_bufctl_t)(~0U))-2)
   2.126 -
   2.127 -/* Max number of objs-per-slab for caches which use off-slab slabs.
   2.128 - * Needed to avoid a possible looping condition in xmem_cache_grow().
   2.129 - */
   2.130 -static unsigned long offslab_limit;
   2.131 -
   2.132 -/*
   2.133 - * slab_t
   2.134 - *
   2.135 - * Manages the objs in a slab. Placed either at the beginning of mem allocated
   2.136 - * for a slab, or allocated from an general cache.
   2.137 - * Slabs are chained into three list: fully used, partial, fully free slabs.
   2.138 - */
   2.139 -typedef struct slab_s {
   2.140 -    struct list_head list;
   2.141 -    unsigned long    colouroff;
   2.142 -    void            *s_mem;    /* including colour offset */
   2.143 -    unsigned int     inuse;    /* num of objs active in slab */
   2.144 -    xmem_bufctl_t    free;
   2.145 -} slab_t;
   2.146 -
   2.147 -#define slab_bufctl(slabp) \
   2.148 -	((xmem_bufctl_t *)(((slab_t*)slabp)+1))
   2.149 -
   2.150 -/*
   2.151 - * cpucache_t
   2.152 - *
   2.153 - * Per cpu structures
   2.154 - * The limit is stored in the per-cpu structure to reduce the data cache
   2.155 - * footprint.
   2.156 - */
   2.157 -typedef struct cpucache_s {
   2.158 -    unsigned int avail;
   2.159 -    unsigned int limit;
   2.160 -} cpucache_t;
   2.161 -
   2.162 -#define cc_entry(cpucache) \
   2.163 -	((void **)(((cpucache_t*)(cpucache))+1))
   2.164 -#define cc_data(cachep) \
   2.165 -	((cachep)->cpudata[smp_processor_id()])
   2.166 -/*
   2.167 - * xmem_cache_t
   2.168 - *
   2.169 - * manages a cache.
   2.170 - */
   2.171 -
   2.172 -#define CACHE_NAMELEN	20	/* max name length for a slab cache */
   2.173 -
   2.174 -struct xmem_cache_s {
   2.175 -/* 1) each alloc & free */
   2.176 -    /* full, partial first, then free */
   2.177 -    struct list_head	slabs_full;
   2.178 -    struct list_head	slabs_partial;
   2.179 -    struct list_head	slabs_free;
   2.180 -    unsigned int		objsize;
   2.181 -    unsigned int	 	flags;	/* constant flags */
   2.182 -    unsigned int		num;	/* # of objs per slab */
   2.183 -    spinlock_t		spinlock;
   2.184 -#ifdef CONFIG_SMP
   2.185 -    unsigned int		batchcount;
   2.186 -#endif
   2.187 -
   2.188 -/* 2) slab additions /removals */
   2.189 -    /* order of pgs per slab (2^n) */
   2.190 -    unsigned int		gfporder;
   2.191 -    size_t			colour;		/* cache colouring range */
   2.192 -    unsigned int		colour_off;	/* colour offset */
   2.193 -    unsigned int		colour_next;	/* cache colouring */
   2.194 -    xmem_cache_t		*slabp_cache;
   2.195 -    unsigned int		growing;
   2.196 -    unsigned int		dflags;		/* dynamic flags */
   2.197 -
   2.198 -    /* constructor func */
   2.199 -    void (*ctor)(void *, xmem_cache_t *, unsigned long);
   2.200 -
   2.201 -    /* de-constructor func */
   2.202 -    void (*dtor)(void *, xmem_cache_t *, unsigned long);
   2.203 -
   2.204 -    unsigned long		failures;
   2.205 -
   2.206 -/* 3) cache creation/removal */
   2.207 -    char			name[CACHE_NAMELEN];
   2.208 -    struct list_head	next;
   2.209 -#ifdef CONFIG_SMP
   2.210 -/* 4) per-cpu data */
   2.211 -    cpucache_t		*cpudata[NR_CPUS];
   2.212 -#endif
   2.213 -#if STATS
   2.214 -    unsigned long		num_active;
   2.215 -    unsigned long		num_allocations;
   2.216 -    unsigned long		high_mark;
   2.217 -    unsigned long		grown;
   2.218 -    unsigned long		reaped;
   2.219 -    unsigned long 		errors;
   2.220 -#ifdef CONFIG_SMP
   2.221 -    atomic_t		allochit;
   2.222 -    atomic_t		allocmiss;
   2.223 -    atomic_t		freehit;
   2.224 -    atomic_t		freemiss;
   2.225 -#endif
   2.226 -#endif
   2.227 -};
   2.228 -
   2.229 -/* internal c_flags */
   2.230 -#define	CFLGS_OFF_SLAB	0x010000UL	/* slab management in own cache */
   2.231 -#define	CFLGS_OPTIMIZE	0x020000UL	/* optimized slab lookup */
   2.232 -
   2.233 -/* c_dflags (dynamic flags). Need to hold the spinlock to access this member */
   2.234 -#define	DFLGS_GROWN	0x000001UL	/* don't reap a recently grown */
   2.235 -
   2.236 -#define	OFF_SLAB(x)	((x)->flags & CFLGS_OFF_SLAB)
   2.237 -#define	OPTIMIZE(x)	((x)->flags & CFLGS_OPTIMIZE)
   2.238 -#define	GROWN(x)	((x)->dlags & DFLGS_GROWN)
   2.239 -
   2.240 -#if STATS
   2.241 -#define	STATS_INC_ACTIVE(x)	((x)->num_active++)
   2.242 -#define	STATS_DEC_ACTIVE(x)	((x)->num_active--)
   2.243 -#define	STATS_INC_ALLOCED(x)	((x)->num_allocations++)
   2.244 -#define	STATS_INC_GROWN(x)	((x)->grown++)
   2.245 -#define	STATS_INC_REAPED(x)	((x)->reaped++)
   2.246 -#define	STATS_SET_HIGH(x)	do { if ((x)->num_active > (x)->high_mark) \
   2.247 -					(x)->high_mark = (x)->num_active; \
   2.248 -				} while (0)
   2.249 -#define	STATS_INC_ERR(x)	((x)->errors++)
   2.250 -#else
   2.251 -#define	STATS_INC_ACTIVE(x)	do { } while (0)
   2.252 -#define	STATS_DEC_ACTIVE(x)	do { } while (0)
   2.253 -#define	STATS_INC_ALLOCED(x)	do { } while (0)
   2.254 -#define	STATS_INC_GROWN(x)	do { } while (0)
   2.255 -#define	STATS_INC_REAPED(x)	do { } while (0)
   2.256 -#define	STATS_SET_HIGH(x)	do { } while (0)
   2.257 -#define	STATS_INC_ERR(x)	do { } while (0)
   2.258 -#endif
   2.259 -
   2.260 -#if STATS && defined(CONFIG_SMP)
   2.261 -#define STATS_INC_ALLOCHIT(x)	atomic_inc(&(x)->allochit)
   2.262 -#define STATS_INC_ALLOCMISS(x)	atomic_inc(&(x)->allocmiss)
   2.263 -#define STATS_INC_FREEHIT(x)	atomic_inc(&(x)->freehit)
   2.264 -#define STATS_INC_FREEMISS(x)	atomic_inc(&(x)->freemiss)
   2.265 -#else
   2.266 -#define STATS_INC_ALLOCHIT(x)	do { } while (0)
   2.267 -#define STATS_INC_ALLOCMISS(x)	do { } while (0)
   2.268 -#define STATS_INC_FREEHIT(x)	do { } while (0)
   2.269 -#define STATS_INC_FREEMISS(x)	do { } while (0)
   2.270 -#endif
   2.271 -
   2.272 -#if DEBUG
   2.273 -/* Magic nums for obj red zoning.
   2.274 - * Placed in the first word before and the first word after an obj.
   2.275 - */
   2.276 -#define	RED_MAGIC1	0x5A2CF071UL	/* when obj is active */
   2.277 -#define	RED_MAGIC2	0x170FC2A5UL	/* when obj is inactive */
   2.278 -
   2.279 -/* ...and for poisoning */
   2.280 -#define	POISON_BYTE	0x5a		/* byte value for poisoning */
   2.281 -#define	POISON_END	0xa5		/* end-byte of poisoning */
   2.282 -
   2.283 -#endif
   2.284 -
   2.285 -/* maximum size of an obj (in 2^order pages) */
   2.286 -#define	MAX_OBJ_ORDER	5	/* 32 pages */
   2.287 -
   2.288 -/*
   2.289 - * Do not go above this order unless 0 objects fit into the slab.
   2.290 - */
   2.291 -#define	BREAK_GFP_ORDER_HI	2
   2.292 -#define	BREAK_GFP_ORDER_LO	1
   2.293 -static int slab_break_gfp_order = BREAK_GFP_ORDER_LO;
   2.294 -
   2.295 -/*
   2.296 - * Absolute limit for the gfp order
   2.297 - */
   2.298 -#define	MAX_GFP_ORDER	5	/* 32 pages */
   2.299 -
   2.300 -
   2.301 -/* Macros for storing/retrieving the cachep and or slab from the
   2.302 - * global 'mem_map'. These are used to find the slab an obj belongs to.
   2.303 - * With xfree(), these are used to find the cache which an obj belongs to.
   2.304 - */
   2.305 -#define	SET_PAGE_CACHE(pg,x)  ((pg)->list.next = (struct list_head *)(x))
   2.306 -#define	GET_PAGE_CACHE(pg)    ((xmem_cache_t *)(pg)->list.next)
   2.307 -#define	SET_PAGE_SLAB(pg,x)   ((pg)->list.prev = (struct list_head *)(x))
   2.308 -#define	GET_PAGE_SLAB(pg)     ((slab_t *)(pg)->list.prev)
   2.309 -
   2.310 -/* Size description struct for general caches. */
   2.311 -typedef struct cache_sizes {
   2.312 -    size_t		 cs_size;
   2.313 -    xmem_cache_t	*cs_cachep;
   2.314 -} cache_sizes_t;
   2.315 -
   2.316 -static cache_sizes_t cache_sizes[] = {
   2.317 -    {    32,	NULL},
   2.318 -    {    64,	NULL},
   2.319 -    {   128,	NULL},
   2.320 -    {   256,	NULL},
   2.321 -    {   512,	NULL},
   2.322 -    {  1024,	NULL},
   2.323 -    {  2048,	NULL},
   2.324 -    {  4096,	NULL},
   2.325 -    {  8192,	NULL},
   2.326 -    { 16384,	NULL},
   2.327 -    { 32768,	NULL},
   2.328 -    { 65536,	NULL},
   2.329 -    {     0,	NULL}
   2.330 -};
   2.331 -
   2.332 -/* internal cache of cache description objs */
   2.333 -static xmem_cache_t cache_cache = {
   2.334 -    slabs_full:    LIST_HEAD_INIT(cache_cache.slabs_full),
   2.335 -    slabs_partial: LIST_HEAD_INIT(cache_cache.slabs_partial),
   2.336 -    slabs_free:    LIST_HEAD_INIT(cache_cache.slabs_free),
   2.337 -    objsize:       sizeof(xmem_cache_t),
   2.338 -    flags:         SLAB_NO_REAP,
   2.339 -    spinlock:      SPIN_LOCK_UNLOCKED,
   2.340 -    colour_off:    L1_CACHE_BYTES,
   2.341 -    name:          "xmem_cache"
   2.342 -};
   2.343 -
   2.344 -/* Guard access to the cache-chain. */
   2.345 -/* KAF: No semaphores, as we'll never wait around for I/O. */
   2.346 -static spinlock_t cache_chain_sem;
   2.347 -#define init_MUTEX(_m)   spin_lock_init(_m)
   2.348 -#define down(_m)         spin_lock_irqsave(_m,spin_flags)
   2.349 -#define up(_m)           spin_unlock_irqrestore(_m,spin_flags)
   2.350 -
   2.351 -/* Place maintainer for reaping. */
   2.352 -static xmem_cache_t *clock_searchp = &cache_cache;
   2.353 -
   2.354 -#define cache_chain (cache_cache.next)
   2.355 -
   2.356 -#ifdef CONFIG_SMP
   2.357 -/*
   2.358 - * chicken and egg problem: delay the per-cpu array allocation
   2.359 - * until the general caches are up.
   2.360 - */
   2.361 -static int g_cpucache_up;
   2.362 -
   2.363 -static void enable_cpucache (xmem_cache_t *cachep);
   2.364 -static void enable_all_cpucaches (void);
   2.365 -#endif
   2.366 -
   2.367 -/* Cal the num objs, wastage, and bytes left over for a given slab size. */
   2.368 -static void xmem_cache_estimate (unsigned long gfporder, size_t size,
   2.369 -                                 int flags, size_t *left_over, unsigned int *num)
   2.370 -{
   2.371 -    int i;
   2.372 -    size_t wastage = PAGE_SIZE<<gfporder;
   2.373 -    size_t extra = 0;
   2.374 -    size_t base = 0;
   2.375 -
   2.376 -    if (!(flags & CFLGS_OFF_SLAB)) {
   2.377 -        base = sizeof(slab_t);
   2.378 -        extra = sizeof(xmem_bufctl_t);
   2.379 -    }
   2.380 -    i = 0;
   2.381 -    while (i*size + L1_CACHE_ALIGN(base+i*extra) <= wastage)
   2.382 -        i++;
   2.383 -    if (i > 0)
   2.384 -        i--;
   2.385 -
   2.386 -    if (i > SLAB_LIMIT)
   2.387 -        i = SLAB_LIMIT;
   2.388 -
   2.389 -    *num = i;
   2.390 -    wastage -= i*size;
   2.391 -    wastage -= L1_CACHE_ALIGN(base+i*extra);
   2.392 -    *left_over = wastage;
   2.393 -}
   2.394 -
   2.395 -/* Initialisation - setup the `cache' cache. */
   2.396 -void __init xmem_cache_init(void)
   2.397 -{
   2.398 -    size_t left_over;
   2.399 -
   2.400 -    init_MUTEX(&cache_chain_sem);
   2.401 -    INIT_LIST_HEAD(&cache_chain);
   2.402 -
   2.403 -    xmem_cache_estimate(0, cache_cache.objsize, 0,
   2.404 -			&left_over, &cache_cache.num);
   2.405 -    if (!cache_cache.num)
   2.406 -        BUG();
   2.407 -
   2.408 -    cache_cache.colour = left_over/cache_cache.colour_off;
   2.409 -    cache_cache.colour_next = 0;
   2.410 -}
   2.411 -
   2.412 -
   2.413 -/* Initialisation - setup remaining internal and general caches.
   2.414 - * Called after the gfp() functions have been enabled, and before smp_init().
   2.415 - */
   2.416 -void __init xmem_cache_sizes_init(unsigned long num_physpages)
   2.417 -{
   2.418 -    cache_sizes_t *sizes = cache_sizes;
   2.419 -    char name[20];
   2.420 -    /*
   2.421 -     * Fragmentation resistance on low memory - only use bigger
   2.422 -     * page orders on machines with more than 32MB of memory.
   2.423 -     */
   2.424 -    if (num_physpages > (32 << 20) >> PAGE_SHIFT)
   2.425 -        slab_break_gfp_order = BREAK_GFP_ORDER_HI;
   2.426 -    do {
   2.427 -        /* For performance, all the general caches are L1 aligned.
   2.428 -         * This should be particularly beneficial on SMP boxes, as it
   2.429 -         * eliminates "false sharing".
   2.430 -         * Note for systems short on memory removing the alignment will
   2.431 -         * allow tighter packing of the smaller caches. */
   2.432 -        sprintf(name,"size-%Zd",sizes->cs_size);
   2.433 -        if (!(sizes->cs_cachep =
   2.434 -              xmem_cache_create(name, sizes->cs_size,
   2.435 -                                0, SLAB_HWCACHE_ALIGN, NULL, NULL))) {
   2.436 -            BUG();
   2.437 -        }
   2.438 -
   2.439 -        /* Inc off-slab bufctl limit until the ceiling is hit. */
   2.440 -        if (!(OFF_SLAB(sizes->cs_cachep))) {
   2.441 -            offslab_limit = sizes->cs_size-sizeof(slab_t);
   2.442 -            offslab_limit /= 2;
   2.443 -        }
   2.444 -        sizes++;
   2.445 -    } while (sizes->cs_size);
   2.446 -}
   2.447 -
   2.448 -int __init xmem_cpucache_init(void)
   2.449 -{
   2.450 -#ifdef CONFIG_SMP
   2.451 -    g_cpucache_up = 1;
   2.452 -    enable_all_cpucaches();
   2.453 -#endif
   2.454 -    return 0;
   2.455 -}
   2.456 -
   2.457 -/*__initcall(xmem_cpucache_init);*/
   2.458 -
   2.459 -/* Interface to system's page allocator. No need to hold the cache-lock.
   2.460 - */
   2.461 -static inline void *xmem_getpages(xmem_cache_t *cachep)
   2.462 -{
   2.463 -    void *addr;
   2.464 -
   2.465 -    addr = (void*) alloc_xenheap_pages(cachep->gfporder);
   2.466 -    /* Assume that now we have the pages no one else can legally
   2.467 -     * messes with the 'struct page's.
   2.468 -     * However vm_scan() might try to test the structure to see if
   2.469 -     * it is a named-page or buffer-page.  The members it tests are
   2.470 -     * of no interest here.....
   2.471 -     */
   2.472 -    return addr;
   2.473 -}
   2.474 -
   2.475 -/* Interface to system's page release. */
   2.476 -static inline void xmem_freepages (xmem_cache_t *cachep, void *addr)
   2.477 -{
   2.478 -    unsigned long i = (1<<cachep->gfporder);
   2.479 -    struct pfn_info *page = virt_to_page(addr);
   2.480 -
   2.481 -    /* free_xenheap_pages() does not clear the type bit - we do that.
   2.482 -     * The pages have been unlinked from their cache-slab,
   2.483 -     * but their 'struct page's might be accessed in
   2.484 -     * vm_scan(). Shouldn't be a worry.
   2.485 -     */
   2.486 -    while (i--) {
   2.487 -        PageClearSlab(page);
   2.488 -        page++;
   2.489 -    }
   2.490 -
   2.491 -    free_xenheap_pages((unsigned long)addr, cachep->gfporder);
   2.492 -}
   2.493 -
   2.494 -#if DEBUG
   2.495 -static inline void xmem_poison_obj (xmem_cache_t *cachep, void *addr)
   2.496 -{
   2.497 -    int size = cachep->objsize;
   2.498 -    if (cachep->flags & SLAB_RED_ZONE) {
   2.499 -        addr += BYTES_PER_WORD;
   2.500 -        size -= 2*BYTES_PER_WORD;
   2.501 -    }
   2.502 -    memset(addr, POISON_BYTE, size);
   2.503 -    *(unsigned char *)(addr+size-1) = POISON_END;
   2.504 -}
   2.505 -
   2.506 -static inline int xmem_check_poison_obj (xmem_cache_t *cachep, void *addr)
   2.507 -{
   2.508 -    int size = cachep->objsize;
   2.509 -    void *end;
   2.510 -    if (cachep->flags & SLAB_RED_ZONE) {
   2.511 -        addr += BYTES_PER_WORD;
   2.512 -        size -= 2*BYTES_PER_WORD;
   2.513 -    }
   2.514 -    end = memchr(addr, POISON_END, size);
   2.515 -    if (end != (addr+size-1))
   2.516 -        return 1;
   2.517 -    return 0;
   2.518 -}
   2.519 -#endif
   2.520 -
   2.521 -/* Destroy all the objs in a slab, and release the mem back to the system.
   2.522 - * Before calling the slab must have been unlinked from the cache.
   2.523 - * The cache-lock is not held/needed.
   2.524 - */
   2.525 -static void xmem_slab_destroy (xmem_cache_t *cachep, slab_t *slabp)
   2.526 -{
   2.527 -    if (cachep->dtor
   2.528 -#if DEBUG
   2.529 -        || cachep->flags & (SLAB_POISON | SLAB_RED_ZONE)
   2.530 -#endif
   2.531 -	) {
   2.532 -        int i;
   2.533 -        for (i = 0; i < cachep->num; i++) {
   2.534 -            void* objp = slabp->s_mem+cachep->objsize*i;
   2.535 -#if DEBUG
   2.536 -            if (cachep->flags & SLAB_RED_ZONE) {
   2.537 -                if (*((unsigned long*)(objp)) != RED_MAGIC1)
   2.538 -                    BUG();
   2.539 -                if (*((unsigned long*)(objp + cachep->objsize
   2.540 -                                       -BYTES_PER_WORD)) != RED_MAGIC1)
   2.541 -                    BUG();
   2.542 -                objp += BYTES_PER_WORD;
   2.543 -            }
   2.544 -#endif
   2.545 -            if (cachep->dtor)
   2.546 -                (cachep->dtor)(objp, cachep, 0);
   2.547 -#if DEBUG
   2.548 -            if (cachep->flags & SLAB_RED_ZONE) {
   2.549 -                objp -= BYTES_PER_WORD;
   2.550 -            }	
   2.551 -            if ((cachep->flags & SLAB_POISON)  &&
   2.552 -                xmem_check_poison_obj(cachep, objp))
   2.553 -                BUG();
   2.554 -#endif
   2.555 -        }
   2.556 -    }
   2.557 -
   2.558 -    xmem_freepages(cachep, slabp->s_mem-slabp->colouroff);
   2.559 -    if (OFF_SLAB(cachep))
   2.560 -        xmem_cache_free(cachep->slabp_cache, slabp);
   2.561 -}
   2.562 -
   2.563 -/**
   2.564 - * xmem_cache_create - Create a cache.
   2.565 - * @name: A string which is used in /proc/slabinfo to identify this cache.
   2.566 - * @size: The size of objects to be created in this cache.
   2.567 - * @offset: The offset to use within the page.
   2.568 - * @flags: SLAB flags
   2.569 - * @ctor: A constructor for the objects.
   2.570 - * @dtor: A destructor for the objects.
   2.571 - *
   2.572 - * Returns a ptr to the cache on success, NULL on failure.
   2.573 - * Cannot be called within a int, but can be interrupted.
   2.574 - * The @ctor is run when new pages are allocated by the cache
   2.575 - * and the @dtor is run before the pages are handed back.
   2.576 - * The flags are
   2.577 - *
   2.578 - * %SLAB_POISON - Poison the slab with a known test pattern (a5a5a5a5)
   2.579 - * to catch references to uninitialised memory.
   2.580 - *
   2.581 - * %SLAB_RED_ZONE - Insert `Red' zones around the allocated memory to check
   2.582 - * for buffer overruns.
   2.583 - *
   2.584 - * %SLAB_NO_REAP - Don't automatically reap this cache when we're under
   2.585 - * memory pressure.
   2.586 - *
   2.587 - * %SLAB_HWCACHE_ALIGN - Align the objects in this cache to a hardware
   2.588 - * cacheline.  This can be beneficial if you're counting cycles as closely
   2.589 - * as davem.
   2.590 - */
   2.591 -xmem_cache_t *
   2.592 -xmem_cache_create (const char *name, size_t size, size_t offset,
   2.593 -                   unsigned long flags,
   2.594 -                   void (*ctor)(void*, xmem_cache_t *, unsigned long),
   2.595 -                   void (*dtor)(void*, xmem_cache_t *, unsigned long))
   2.596 -{
   2.597 -    const char *func_nm = KERN_ERR "xmem_create: ";
   2.598 -    size_t left_over, align, slab_size;
   2.599 -    xmem_cache_t *cachep = NULL;
   2.600 -    unsigned long spin_flags;
   2.601 -
   2.602 -    /*
   2.603 -     * Sanity checks... these are all serious usage bugs.
   2.604 -     */
   2.605 -    if ((!name) ||
   2.606 -        ((strlen(name) >= CACHE_NAMELEN - 1)) ||
   2.607 -        (size < BYTES_PER_WORD) ||
   2.608 -        (size > (1<<MAX_OBJ_ORDER)*PAGE_SIZE) ||
   2.609 -        (dtor && !ctor) ||
   2.610 -        (offset < 0 || offset > size))
   2.611 -        BUG();
   2.612 -
   2.613 -#if DEBUG
   2.614 -    if ((flags & SLAB_DEBUG_INITIAL) && !ctor) {
   2.615 -        /* No constructor, but inital state check requested */
   2.616 -        printk("%sNo con, but init state check requested - %s\n",
   2.617 -               func_nm, name);
   2.618 -        flags &= ~SLAB_DEBUG_INITIAL;
   2.619 -    }
   2.620 -
   2.621 -    if ((flags & SLAB_POISON) && ctor) {
   2.622 -        /* request for poisoning, but we can't do that with a constructor */
   2.623 -        printk("%sPoisoning requested, but con given - %s\n",
   2.624 -               func_nm, name);
   2.625 -        flags &= ~SLAB_POISON;
   2.626 -    }
   2.627 -#if FORCED_DEBUG
   2.628 -    if (size < (PAGE_SIZE>>3))
   2.629 -        /*
   2.630 -         * do not red zone large object, causes severe
   2.631 -         * fragmentation.
   2.632 -         */
   2.633 -        flags |= SLAB_RED_ZONE;
   2.634 -    if (!ctor)
   2.635 -        flags |= SLAB_POISON;
   2.636 -#endif
   2.637 -#endif
   2.638 -
   2.639 -    /*
   2.640 -     * Always checks flags, a caller might be expecting debug
   2.641 -     * support which isn't available.
   2.642 -     */
   2.643 -    if (flags & ~CREATE_MASK)
   2.644 -        BUG();
   2.645 -
   2.646 -    /* Get cache's description obj. */
   2.647 -    cachep = (xmem_cache_t *)xmem_cache_alloc(&cache_cache);
   2.648 -    if (!cachep)
   2.649 -        goto opps;
   2.650 -    memset(cachep, 0, sizeof(xmem_cache_t));
   2.651 -
   2.652 -    /* Check that size is in terms of words.  This is needed to avoid
   2.653 -     * unaligned accesses for some archs when redzoning is used, and makes
   2.654 -     * sure any on-slab bufctl's are also correctly aligned.
   2.655 -     */
   2.656 -    if (size & (BYTES_PER_WORD-1)) {
   2.657 -        size += (BYTES_PER_WORD-1);
   2.658 -        size &= ~(BYTES_PER_WORD-1);
   2.659 -        printk("%sForcing size word alignment - %s\n", func_nm, name);
   2.660 -    }
   2.661 -	
   2.662 -#if DEBUG
   2.663 -    if (flags & SLAB_RED_ZONE) {
   2.664 -        /*
   2.665 -         * There is no point trying to honour cache alignment
   2.666 -         * when redzoning.
   2.667 -         */
   2.668 -        flags &= ~SLAB_HWCACHE_ALIGN;
   2.669 -        size += 2*BYTES_PER_WORD;	/* words for redzone */
   2.670 -    }
   2.671 -#endif
   2.672 -    align = BYTES_PER_WORD;
   2.673 -    if (flags & SLAB_HWCACHE_ALIGN)
   2.674 -        align = L1_CACHE_BYTES;
   2.675 -
   2.676 -    /* Determine if the slab management is 'on' or 'off' slab. */
   2.677 -    if (size >= (PAGE_SIZE>>3))
   2.678 -        /*
   2.679 -         * Size is large, assume best to place the slab management obj
   2.680 -         * off-slab (should allow better packing of objs).
   2.681 -         */
   2.682 -        flags |= CFLGS_OFF_SLAB;
   2.683 -
   2.684 -    if (flags & SLAB_HWCACHE_ALIGN) {
   2.685 -        /* Need to adjust size so that objs are cache aligned. */
   2.686 -        /* Small obj size, can get at least two per cache line. */
   2.687 -        /* FIXME: only power of 2 supported, was better */
   2.688 -        while (size < align/2)
   2.689 -            align /= 2;
   2.690 -        size = (size+align-1)&(~(align-1));
   2.691 -    }
   2.692 -
   2.693 -    /* Cal size (in pages) of slabs, and the num of objs per slab.
   2.694 -     * This could be made much more intelligent.  For now, try to avoid
   2.695 -     * using high page-orders for slabs.  When the gfp() funcs are more
   2.696 -     * friendly towards high-order requests, this should be changed.
   2.697 -     */
   2.698 -    do {
   2.699 -        unsigned int break_flag = 0;
   2.700 -    cal_wastage:
   2.701 -        xmem_cache_estimate(cachep->gfporder, size, flags,
   2.702 -                            &left_over, &cachep->num);
   2.703 -        if (break_flag)
   2.704 -            break;
   2.705 -        if (cachep->gfporder >= MAX_GFP_ORDER)
   2.706 -            break;
   2.707 -        if (!cachep->num)
   2.708 -            goto next;
   2.709 -        if (flags & CFLGS_OFF_SLAB && cachep->num > offslab_limit) {
   2.710 -            /* Oops, this num of objs will cause problems. */
   2.711 -            cachep->gfporder--;
   2.712 -            break_flag++;
   2.713 -            goto cal_wastage;
   2.714 -        }
   2.715 -
   2.716 -        /*
   2.717 -         * Large num of objs is good, but v. large slabs are currently
   2.718 -         * bad for the gfp()s.
   2.719 -         */
   2.720 -        if (cachep->gfporder >= slab_break_gfp_order)
   2.721 -            break;
   2.722 -
   2.723 -        if ((left_over*8) <= (PAGE_SIZE<<cachep->gfporder))
   2.724 -            break;	/* Acceptable internal fragmentation. */
   2.725 -    next:
   2.726 -        cachep->gfporder++;
   2.727 -    } while (1);
   2.728 -
   2.729 -    if (!cachep->num) {
   2.730 -        printk("xmem_cache_create: couldn't create cache %s.\n", name);
   2.731 -        xmem_cache_free(&cache_cache, cachep);
   2.732 -        cachep = NULL;
   2.733 -        goto opps;
   2.734 -    }
   2.735 -    slab_size = L1_CACHE_ALIGN(cachep->num*sizeof(xmem_bufctl_t) + 
   2.736 -                               sizeof(slab_t));
   2.737 -
   2.738 -    /*
   2.739 -     * If the slab has been placed off-slab, and we have enough space then
   2.740 -     * move it on-slab. This is at the expense of any extra colouring.
   2.741 -     */
   2.742 -    if (flags & CFLGS_OFF_SLAB && left_over >= slab_size) {
   2.743 -        flags &= ~CFLGS_OFF_SLAB;
   2.744 -        left_over -= slab_size;
   2.745 -    }
   2.746 -
   2.747 -    /* Offset must be a multiple of the alignment. */
   2.748 -    offset += (align-1);
   2.749 -    offset &= ~(align-1);
   2.750 -    if (!offset)
   2.751 -        offset = L1_CACHE_BYTES;
   2.752 -    cachep->colour_off = offset;
   2.753 -    cachep->colour = left_over/offset;
   2.754 -
   2.755 -    /* init remaining fields */
   2.756 -    if (!cachep->gfporder && !(flags & CFLGS_OFF_SLAB))
   2.757 -        flags |= CFLGS_OPTIMIZE;
   2.758 -
   2.759 -    cachep->flags = flags;
   2.760 -    spin_lock_init(&cachep->spinlock);
   2.761 -    cachep->objsize = size;
   2.762 -    INIT_LIST_HEAD(&cachep->slabs_full);
   2.763 -    INIT_LIST_HEAD(&cachep->slabs_partial);
   2.764 -    INIT_LIST_HEAD(&cachep->slabs_free);
   2.765 -
   2.766 -    if (flags & CFLGS_OFF_SLAB)
   2.767 -        cachep->slabp_cache = xmem_find_general_cachep(slab_size);
   2.768 -    cachep->ctor = ctor;
   2.769 -    cachep->dtor = dtor;
   2.770 -    /* Copy name over so we don't have problems with unloaded modules */
   2.771 -    strcpy(cachep->name, name);
   2.772 -
   2.773 -#ifdef CONFIG_SMP
   2.774 -    if (g_cpucache_up)
   2.775 -        enable_cpucache(cachep);
   2.776 -#endif
   2.777 -    /* Need the semaphore to access the chain. */
   2.778 -    down(&cache_chain_sem);
   2.779 -    {
   2.780 -	xmem_cache_t *pc;
   2.781 -
   2.782 -        list_for_each_entry(pc, &cache_chain, next) {
   2.783 -            /* The name field is constant - no lock needed. */
   2.784 -            if (!strcmp(pc->name, name))
   2.785 -                BUG();
   2.786 -        }
   2.787 -    }
   2.788 -
   2.789 -    /* There is no reason to lock our new cache before we
   2.790 -     * link it in - no one knows about it yet...
   2.791 -     */
   2.792 -    list_add(&cachep->next, &cache_chain);
   2.793 -    up(&cache_chain_sem);
   2.794 - opps:
   2.795 -    return cachep;
   2.796 -}
   2.797 -
   2.798 -
   2.799 -#if DEBUG
   2.800 -/*
   2.801 - * This check if the xmem_cache_t pointer is chained in the cache_cache
   2.802 - * list. -arca
   2.803 - */
   2.804 -static int is_chained_xmem_cache(xmem_cache_t * cachep)
   2.805 -{
   2.806 -    xmem_cache_t *pc;
   2.807 -    int ret = 0;
   2.808 -    unsigned long spin_flags;
   2.809 -
   2.810 -    /* Find the cache in the chain of caches. */
   2.811 -    down(&cache_chain_sem);
   2.812 -    list_for_each_entry(pc, &cache_chain, next) {
   2.813 -        if (pc == &cachep) {
   2.814 -            ret = 1;
   2.815 -            break;
   2.816 -        }
   2.817 -    }
   2.818 -    up(&cache_chain_sem);
   2.819 -
   2.820 -    return ret;
   2.821 -}
   2.822 -#else
   2.823 -#define is_chained_xmem_cache(x) 1
   2.824 -#endif
   2.825 -
   2.826 -#ifdef CONFIG_SMP
   2.827 -/*
   2.828 - * Waits for all CPUs to execute func().
   2.829 - */
   2.830 -static void smp_call_function_all_cpus(void (*func) (void *arg), void *arg)
   2.831 -{
   2.832 -    local_irq_disable();
   2.833 -    func(arg);
   2.834 -    local_irq_enable();
   2.835 -
   2.836 -    if (smp_call_function(func, arg, 1, 1))
   2.837 -        BUG();
   2.838 -}
   2.839 -typedef struct ccupdate_struct_s
   2.840 -{
   2.841 -    xmem_cache_t *cachep;
   2.842 -    cpucache_t *new[NR_CPUS];
   2.843 -} ccupdate_struct_t;
   2.844 -
   2.845 -static void do_ccupdate_local(void *info)
   2.846 -{
   2.847 -    ccupdate_struct_t *new = (ccupdate_struct_t *)info;
   2.848 -    cpucache_t *old = cc_data(new->cachep);
   2.849 -	
   2.850 -    cc_data(new->cachep) = new->new[smp_processor_id()];
   2.851 -    new->new[smp_processor_id()] = old;
   2.852 -}
   2.853 -
   2.854 -static void free_block (xmem_cache_t* cachep, void** objpp, int len);
   2.855 -
   2.856 -static void drain_cpu_caches(xmem_cache_t *cachep)
   2.857 -{
   2.858 -    ccupdate_struct_t new;
   2.859 -    int i;
   2.860 -    unsigned long spin_flags;
   2.861 -
   2.862 -    memset(&new.new,0,sizeof(new.new));
   2.863 -
   2.864 -    new.cachep = cachep;
   2.865 -
   2.866 -    down(&cache_chain_sem);
   2.867 -    smp_call_function_all_cpus(do_ccupdate_local, (void *)&new);
   2.868 -
   2.869 -    for (i = 0; i < smp_num_cpus; i++) {
   2.870 -        cpucache_t* ccold = new.new[cpu_logical_map(i)];
   2.871 -        if (!ccold || (ccold->avail == 0))
   2.872 -            continue;
   2.873 -        local_irq_disable();
   2.874 -        free_block(cachep, cc_entry(ccold), ccold->avail);
   2.875 -        local_irq_enable();
   2.876 -        ccold->avail = 0;
   2.877 -    }
   2.878 -    smp_call_function_all_cpus(do_ccupdate_local, (void *)&new);
   2.879 -    up(&cache_chain_sem);
   2.880 -}
   2.881 -
   2.882 -#else
   2.883 -#define drain_cpu_caches(cachep)	do { } while (0)
   2.884 -#endif
   2.885 -
   2.886 -static int __xmem_cache_shrink(xmem_cache_t *cachep)
   2.887 -{
   2.888 -    slab_t *slabp;
   2.889 -    int ret;
   2.890 -
   2.891 -    drain_cpu_caches(cachep);
   2.892 -
   2.893 -    spin_lock_irq(&cachep->spinlock);
   2.894 -
   2.895 -    /* If the cache is growing, stop shrinking. */
   2.896 -    while (!cachep->growing) {
   2.897 -        struct list_head *p;
   2.898 -
   2.899 -        p = cachep->slabs_free.prev;
   2.900 -        if (p == &cachep->slabs_free)
   2.901 -            break;
   2.902 -
   2.903 -        slabp = list_entry(cachep->slabs_free.prev, slab_t, list);
   2.904 -#if DEBUG
   2.905 -        if (slabp->inuse)
   2.906 -            BUG();
   2.907 -#endif
   2.908 -        list_del(&slabp->list);
   2.909 -
   2.910 -        spin_unlock_irq(&cachep->spinlock);
   2.911 -        xmem_slab_destroy(cachep, slabp);
   2.912 -        spin_lock_irq(&cachep->spinlock);
   2.913 -    }
   2.914 -    ret = (!list_empty(&cachep->slabs_full) || 
   2.915 -           !list_empty(&cachep->slabs_partial));
   2.916 -    spin_unlock_irq(&cachep->spinlock);
   2.917 -    return ret;
   2.918 -}
   2.919 -
   2.920 -/**
   2.921 - * xmem_cache_shrink - Shrink a cache.
   2.922 - * @cachep: The cache to shrink.
   2.923 - *
   2.924 - * Releases as many slabs as possible for a cache.
   2.925 - * To help debugging, a zero exit status indicates all slabs were released.
   2.926 - */
   2.927 -int xmem_cache_shrink(xmem_cache_t *cachep)
   2.928 -{
   2.929 -    if (!cachep || !is_chained_xmem_cache(cachep))
   2.930 -        BUG();
   2.931 -
   2.932 -    return __xmem_cache_shrink(cachep);
   2.933 -}
   2.934 -
   2.935 -/**
   2.936 - * xmem_cache_destroy - delete a cache
   2.937 - * @cachep: the cache to destroy
   2.938 - *
   2.939 - * Remove a xmem_cache_t object from the slab cache.
   2.940 - * Returns 0 on success.
   2.941 - *
   2.942 - * It is expected this function will be called by a module when it is
   2.943 - * unloaded.  This will remove the cache completely, and avoid a duplicate
   2.944 - * cache being allocated each time a module is loaded and unloaded, if the
   2.945 - * module doesn't have persistent in-kernel storage across loads and unloads.
   2.946 - *
   2.947 - * The caller must guarantee that noone will allocate memory from the cache
   2.948 - * during the xmem_cache_destroy().
   2.949 - */
   2.950 -int xmem_cache_destroy (xmem_cache_t * cachep)
   2.951 -{
   2.952 -    unsigned long spin_flags;
   2.953 -
   2.954 -    if (!cachep || cachep->growing)
   2.955 -        BUG();
   2.956 -
   2.957 -    /* Find the cache in the chain of caches. */
   2.958 -    down(&cache_chain_sem);
   2.959 -    /* the chain is never empty, cache_cache is never destroyed */
   2.960 -    if (clock_searchp == cachep)
   2.961 -        clock_searchp = list_entry(cachep->next.next,
   2.962 -                                   xmem_cache_t, next);
   2.963 -    list_del(&cachep->next);
   2.964 -    up(&cache_chain_sem);
   2.965 -
   2.966 -    if (__xmem_cache_shrink(cachep)) {
   2.967 -        printk(KERN_ERR "xmem_cache_destroy: Can't free all objects %p\n",
   2.968 -               cachep);
   2.969 -        down(&cache_chain_sem);
   2.970 -        list_add(&cachep->next,&cache_chain);
   2.971 -        up(&cache_chain_sem);
   2.972 -        return 1;
   2.973 -    }
   2.974 -#ifdef CONFIG_SMP
   2.975 -    {
   2.976 -        int i;
   2.977 -        for (i = 0; i < NR_CPUS; i++)
   2.978 -            xfree(cachep->cpudata[i]);
   2.979 -    }
   2.980 -#endif
   2.981 -    xmem_cache_free(&cache_cache, cachep);
   2.982 -
   2.983 -    return 0;
   2.984 -}
   2.985 -
   2.986 -/* Get the memory for a slab management obj. */
   2.987 -static inline slab_t *xmem_cache_slabmgmt(xmem_cache_t *cachep,
   2.988 -                                          void *objp, int colour_off, 
   2.989 -                                          int local_flags)
   2.990 -{
   2.991 -    slab_t *slabp;
   2.992 -	
   2.993 -    if (OFF_SLAB(cachep)) {
   2.994 -        /* Slab management obj is off-slab. */
   2.995 -        slabp = xmem_cache_alloc(cachep->slabp_cache);
   2.996 -        if (!slabp)
   2.997 -            return NULL;
   2.998 -    } else {
   2.999 -        /* FIXME: change to
  2.1000 -           slabp = objp
  2.1001 -           * if you enable OPTIMIZE
  2.1002 -           */
  2.1003 -        slabp = objp+colour_off;
  2.1004 -        colour_off += L1_CACHE_ALIGN(cachep->num *
  2.1005 -                                     sizeof(xmem_bufctl_t) + sizeof(slab_t));
  2.1006 -    }
  2.1007 -    slabp->inuse = 0;
  2.1008 -    slabp->colouroff = colour_off;
  2.1009 -    slabp->s_mem = objp+colour_off;
  2.1010 -
  2.1011 -    return slabp;
  2.1012 -}
  2.1013 -
  2.1014 -static inline void xmem_cache_init_objs(xmem_cache_t *cachep,
  2.1015 -                                         slab_t *slabp,
  2.1016 -                                        unsigned long ctor_flags)
  2.1017 -{
  2.1018 -    int i;
  2.1019 -
  2.1020 -    for (i = 0; i < cachep->num; i++) {
  2.1021 -        void* objp = slabp->s_mem+cachep->objsize*i;
  2.1022 -#if DEBUG
  2.1023 -        if (cachep->flags & SLAB_RED_ZONE) {
  2.1024 -            *((unsigned long*)(objp)) = RED_MAGIC1;
  2.1025 -            *((unsigned long*)(objp + cachep->objsize -
  2.1026 -                               BYTES_PER_WORD)) = RED_MAGIC1;
  2.1027 -            objp += BYTES_PER_WORD;
  2.1028 -        }
  2.1029 -#endif
  2.1030 -
  2.1031 -        /*
  2.1032 -         * Constructors are not allowed to allocate memory from
  2.1033 -         * the same cache which they are a constructor for.
  2.1034 -         * Otherwise, deadlock. They must also be threaded.
  2.1035 -         */
  2.1036 -        if (cachep->ctor)
  2.1037 -            cachep->ctor(objp, cachep, ctor_flags);
  2.1038 -#if DEBUG
  2.1039 -        if (cachep->flags & SLAB_RED_ZONE)
  2.1040 -            objp -= BYTES_PER_WORD;
  2.1041 -        if (cachep->flags & SLAB_POISON)
  2.1042 -            /* need to poison the objs */
  2.1043 -            xmem_poison_obj(cachep, objp);
  2.1044 -        if (cachep->flags & SLAB_RED_ZONE) {
  2.1045 -            if (*((unsigned long*)(objp)) != RED_MAGIC1)
  2.1046 -                BUG();
  2.1047 -            if (*((unsigned long*)(objp + cachep->objsize -
  2.1048 -                                   BYTES_PER_WORD)) != RED_MAGIC1)
  2.1049 -                BUG();
  2.1050 -        }
  2.1051 -#endif
  2.1052 -        slab_bufctl(slabp)[i] = i+1;
  2.1053 -    }
  2.1054 -    slab_bufctl(slabp)[i-1] = BUFCTL_END;
  2.1055 -    slabp->free = 0;
  2.1056 -}
  2.1057 -
  2.1058 -/*
  2.1059 - * Grow (by 1) the number of slabs within a cache.  This is called by
  2.1060 - * xmem_cache_alloc() when there are no active objs left in a cache.
  2.1061 - */
  2.1062 -static int xmem_cache_grow(xmem_cache_t * cachep)
  2.1063 -{
  2.1064 -    slab_t	*slabp;
  2.1065 -    struct pfn_info	*page; unsigned int i;
  2.1066 -    void		*objp;
  2.1067 -    size_t		 offset;
  2.1068 -    unsigned long	 ctor_flags;
  2.1069 -    unsigned long	 save_flags;
  2.1070 -
  2.1071 -    ctor_flags = SLAB_CTOR_CONSTRUCTOR;
  2.1072 -
  2.1073 -    /* About to mess with non-constant members - lock. */
  2.1074 -    spin_lock_irqsave(&cachep->spinlock, save_flags);
  2.1075 -
  2.1076 -    /* Get colour for the slab, and cal the next value. */
  2.1077 -    offset = cachep->colour_next;
  2.1078 -    cachep->colour_next++;
  2.1079 -    if (cachep->colour_next >= cachep->colour)
  2.1080 -        cachep->colour_next = 0;
  2.1081 -    offset *= cachep->colour_off;
  2.1082 -    cachep->dflags |= DFLGS_GROWN;
  2.1083 -
  2.1084 -    cachep->growing++;
  2.1085 -    spin_unlock_irqrestore(&cachep->spinlock, save_flags);
  2.1086 -
  2.1087 -    /* A series of memory allocations for a new slab.
  2.1088 -     * Neither the cache-chain semaphore, or cache-lock, are
  2.1089 -     * held, but the incrementing c_growing prevents this
  2.1090 -     * cache from being reaped or shrunk.
  2.1091 -     * Note: The cache could be selected in for reaping in
  2.1092 -     * xmem_cache_reap(), but when the final test is made the
  2.1093 -     * growing value will be seen.
  2.1094 -     */
  2.1095 -
  2.1096 -    /* Get mem for the objs. */
  2.1097 -    if (!(objp = xmem_getpages(cachep)))
  2.1098 -        goto failed;
  2.1099 -
  2.1100 -    /* Get slab management. */
  2.1101 -    if (!(slabp = xmem_cache_slabmgmt(cachep, objp, offset, 0)))
  2.1102 -        goto opps1;
  2.1103 -
  2.1104 -    /* Nasty!!!!!! I hope this is OK. */
  2.1105 -    i = 1 << cachep->gfporder;
  2.1106 -    page = virt_to_page(objp);
  2.1107 -    do {
  2.1108 -        SET_PAGE_CACHE(page, cachep);
  2.1109 -        SET_PAGE_SLAB(page, slabp);
  2.1110 -        PageSetSlab(page);
  2.1111 -        page++;
  2.1112 -    } while (--i);
  2.1113 -
  2.1114 -    xmem_cache_init_objs(cachep, slabp, ctor_flags);
  2.1115 -
  2.1116 -    spin_lock_irqsave(&cachep->spinlock, save_flags);
  2.1117 -    cachep->growing--;
  2.1118 -
  2.1119 -    /* Make slab active. */
  2.1120 -    list_add_tail(&slabp->list, &cachep->slabs_free);
  2.1121 -    STATS_INC_GROWN(cachep);
  2.1122 -    cachep->failures = 0;
  2.1123 -
  2.1124 -    spin_unlock_irqrestore(&cachep->spinlock, save_flags);
  2.1125 -    return 1;
  2.1126 - opps1:
  2.1127 -    xmem_freepages(cachep, objp);
  2.1128 - failed:
  2.1129 -    spin_lock_irqsave(&cachep->spinlock, save_flags);
  2.1130 -    cachep->growing--;
  2.1131 -    spin_unlock_irqrestore(&cachep->spinlock, save_flags);
  2.1132 -    return 0;
  2.1133 -}
  2.1134 -
  2.1135 -/*
  2.1136 - * Perform extra freeing checks:
  2.1137 - * - detect double free
  2.1138 - * - detect bad pointers.
  2.1139 - * Called with the cache-lock held.
  2.1140 - */
  2.1141 -
  2.1142 -#if DEBUG
  2.1143 -static int xmem_extra_free_checks (xmem_cache_t * cachep,
  2.1144 -                                   slab_t *slabp, void * objp)
  2.1145 -{
  2.1146 -    int i;
  2.1147 -    unsigned int objnr = (objp-slabp->s_mem)/cachep->objsize;
  2.1148 -
  2.1149 -    if (objnr >= cachep->num)
  2.1150 -        BUG();
  2.1151 -    if (objp != slabp->s_mem + objnr*cachep->objsize)
  2.1152 -        BUG();
  2.1153 -
  2.1154 -    /* Check slab's freelist to see if this obj is there. */
  2.1155 -    for (i = slabp->free; i != BUFCTL_END; i = slab_bufctl(slabp)[i]) {
  2.1156 -        if (i == objnr)
  2.1157 -            BUG();
  2.1158 -    }
  2.1159 -    return 0;
  2.1160 -}
  2.1161 -#endif
  2.1162 -
  2.1163 -static inline void * xmem_cache_alloc_one_tail (xmem_cache_t *cachep,
  2.1164 -						slab_t *slabp)
  2.1165 -{
  2.1166 -    void *objp;
  2.1167 -
  2.1168 -    STATS_INC_ALLOCED(cachep);
  2.1169 -    STATS_INC_ACTIVE(cachep);
  2.1170 -    STATS_SET_HIGH(cachep);
  2.1171 -
  2.1172 -    /* get obj pointer */
  2.1173 -    slabp->inuse++;
  2.1174 -    objp = slabp->s_mem + slabp->free*cachep->objsize;
  2.1175 -    slabp->free=slab_bufctl(slabp)[slabp->free];
  2.1176 -
  2.1177 -    if (unlikely(slabp->free == BUFCTL_END)) {
  2.1178 -        list_del(&slabp->list);
  2.1179 -        list_add(&slabp->list, &cachep->slabs_full);
  2.1180 -    }
  2.1181 -#if DEBUG
  2.1182 -    if (cachep->flags & SLAB_POISON)
  2.1183 -        if (xmem_check_poison_obj(cachep, objp))
  2.1184 -            BUG();
  2.1185 -    if (cachep->flags & SLAB_RED_ZONE) {
  2.1186 -        /* Set alloc red-zone, and check old one. */
  2.1187 -        if (xchg((unsigned long *)objp, RED_MAGIC2) !=
  2.1188 -            RED_MAGIC1)
  2.1189 -            BUG();
  2.1190 -        if (xchg((unsigned long *)(objp+cachep->objsize -
  2.1191 -                                   BYTES_PER_WORD), RED_MAGIC2) != RED_MAGIC1)
  2.1192 -            BUG();
  2.1193 -        objp += BYTES_PER_WORD;
  2.1194 -    }
  2.1195 -#endif
  2.1196 -    return objp;
  2.1197 -}
  2.1198 -
  2.1199 -/*
  2.1200 - * Returns a ptr to an obj in the given cache.
  2.1201 - * caller must guarantee synchronization
  2.1202 - * #define for the goto optimization 8-)
  2.1203 - */
  2.1204 -#define xmem_cache_alloc_one(cachep)				\
  2.1205 -({								\
  2.1206 -	struct list_head * slabs_partial, * entry;		\
  2.1207 -	slab_t *slabp;						\
  2.1208 -								\
  2.1209 -	slabs_partial = &(cachep)->slabs_partial;		\
  2.1210 -	entry = slabs_partial->next;				\
  2.1211 -	if (unlikely(entry == slabs_partial)) {			\
  2.1212 -		struct list_head * slabs_free;			\
  2.1213 -		slabs_free = &(cachep)->slabs_free;		\
  2.1214 -		entry = slabs_free->next;			\
  2.1215 -		if (unlikely(entry == slabs_free))		\
  2.1216 -			goto alloc_new_slab;			\
  2.1217 -		list_del(entry);				\
  2.1218 -		list_add(entry, slabs_partial);			\
  2.1219 -	}							\
  2.1220 -								\
  2.1221 -	slabp = list_entry(entry, slab_t, list);		\
  2.1222 -	xmem_cache_alloc_one_tail(cachep, slabp);		\
  2.1223 -})
  2.1224 -
  2.1225 -#ifdef CONFIG_SMP
  2.1226 -void* xmem_cache_alloc_batch(xmem_cache_t* cachep)
  2.1227 -{
  2.1228 -    int batchcount = cachep->batchcount;
  2.1229 -    cpucache_t* cc = cc_data(cachep);
  2.1230 -
  2.1231 -    spin_lock(&cachep->spinlock);
  2.1232 -    while (batchcount--) {
  2.1233 -        struct list_head * slabs_partial, * entry;
  2.1234 -        slab_t *slabp;
  2.1235 -        /* Get slab alloc is to come from. */
  2.1236 -        slabs_partial = &(cachep)->slabs_partial;
  2.1237 -        entry = slabs_partial->next;
  2.1238 -        if (unlikely(entry == slabs_partial)) {
  2.1239 -            struct list_head * slabs_free;
  2.1240 -            slabs_free = &(cachep)->slabs_free;
  2.1241 -            entry = slabs_free->next;
  2.1242 -            if (unlikely(entry == slabs_free))
  2.1243 -                break;
  2.1244 -            list_del(entry);
  2.1245 -            list_add(entry, slabs_partial);
  2.1246 -        }
  2.1247 -
  2.1248 -        slabp = list_entry(entry, slab_t, list);
  2.1249 -        cc_entry(cc)[cc->avail++] =
  2.1250 -            xmem_cache_alloc_one_tail(cachep, slabp);
  2.1251 -    }
  2.1252 -    spin_unlock(&cachep->spinlock);
  2.1253 -
  2.1254 -    if (cc->avail)
  2.1255 -        return cc_entry(cc)[--cc->avail];
  2.1256 -    return NULL;
  2.1257 -}
  2.1258 -#endif
  2.1259 -
  2.1260 -static inline void *__xmem_cache_alloc(xmem_cache_t *cachep)
  2.1261 -{
  2.1262 -    unsigned long flags;
  2.1263 -    void* objp;
  2.1264 -
  2.1265 - try_again:
  2.1266 -    local_irq_save(flags);
  2.1267 -#ifdef CONFIG_SMP
  2.1268 -    {
  2.1269 -        cpucache_t *cc = cc_data(cachep);
  2.1270 -
  2.1271 -        if (cc) {
  2.1272 -            if (cc->avail) {
  2.1273 -                STATS_INC_ALLOCHIT(cachep);
  2.1274 -                objp = cc_entry(cc)[--cc->avail];
  2.1275 -            } else {
  2.1276 -                STATS_INC_ALLOCMISS(cachep);
  2.1277 -                objp = xmem_cache_alloc_batch(cachep);
  2.1278 -                if (!objp)
  2.1279 -                    goto alloc_new_slab_nolock;
  2.1280 -            }
  2.1281 -        } else {
  2.1282 -            spin_lock(&cachep->spinlock);
  2.1283 -            objp = xmem_cache_alloc_one(cachep);
  2.1284 -            spin_unlock(&cachep->spinlock);
  2.1285 -        }
  2.1286 -    }
  2.1287 -#else
  2.1288 -    objp = xmem_cache_alloc_one(cachep);
  2.1289 -#endif
  2.1290 -    local_irq_restore(flags);
  2.1291 -    return objp;
  2.1292 - alloc_new_slab:
  2.1293 -#ifdef CONFIG_SMP
  2.1294 -    spin_unlock(&cachep->spinlock);
  2.1295 - alloc_new_slab_nolock:
  2.1296 -#endif
  2.1297 -    local_irq_restore(flags);
  2.1298 -    if (xmem_cache_grow(cachep))
  2.1299 -        /* Someone may have stolen our objs.  Doesn't matter, we'll
  2.1300 -         * just come back here again.
  2.1301 -         */
  2.1302 -        goto try_again;
  2.1303 -    return NULL;
  2.1304 -}
  2.1305 -
  2.1306 -/*
  2.1307 - * Release an obj back to its cache. If the obj has a constructed
  2.1308 - * state, it should be in this state _before_ it is released.
  2.1309 - * - caller is responsible for the synchronization
  2.1310 - */
  2.1311 -
  2.1312 -#if DEBUG
  2.1313 -# define CHECK_NR(pg)						\
  2.1314 -	do {							\
  2.1315 -		if (!VALID_PAGE(pg)) {				\
  2.1316 -			printk(KERN_ERR "xfree: out of range ptr %lxh.\n", \
  2.1317 -				(unsigned long)objp);		\
  2.1318 -			BUG();					\
  2.1319 -		} \
  2.1320 -	} while (0)
  2.1321 -# define CHECK_PAGE(page)					\
  2.1322 -	do {							\
  2.1323 -		CHECK_NR(page);					\
  2.1324 -		if (!PageSlab(page)) {				\
  2.1325 -			printk(KERN_ERR "xfree: bad ptr %lxh.\n", \
  2.1326 -				(unsigned long)objp);		\
  2.1327 -			BUG();					\
  2.1328 -		}						\
  2.1329 -	} while (0)
  2.1330 -
  2.1331 -#else
  2.1332 -# define CHECK_PAGE(pg)	do { } while (0)
  2.1333 -#endif
  2.1334 -
  2.1335 -static inline void xmem_cache_free_one(xmem_cache_t *cachep, void *objp)
  2.1336 -{
  2.1337 -    slab_t* slabp;
  2.1338 -
  2.1339 -    CHECK_PAGE(virt_to_page(objp));
  2.1340 -    /* reduces memory footprint
  2.1341 -     *
  2.1342 -     if (OPTIMIZE(cachep))
  2.1343 -     slabp = (void*)((unsigned long)objp&(~(PAGE_SIZE-1)));
  2.1344 -     else
  2.1345 -    */
  2.1346 -    slabp = GET_PAGE_SLAB(virt_to_page(objp));
  2.1347 -
  2.1348 -#if DEBUG
  2.1349 -    if (cachep->flags & SLAB_DEBUG_INITIAL)
  2.1350 -        /* Need to call the slab's constructor so the
  2.1351 -         * caller can perform a verify of its state (debugging).
  2.1352 -         * Called without the cache-lock held.
  2.1353 -         */
  2.1354 -        cachep->ctor(objp, cachep, SLAB_CTOR_CONSTRUCTOR|SLAB_CTOR_VERIFY);
  2.1355 -
  2.1356 -    if (cachep->flags & SLAB_RED_ZONE) {
  2.1357 -        objp -= BYTES_PER_WORD;
  2.1358 -        if (xchg((unsigned long *)objp, RED_MAGIC1) != RED_MAGIC2)
  2.1359 -            /* Either write before start, or a double free. */
  2.1360 -            BUG();
  2.1361 -        if (xchg((unsigned long *)(objp+cachep->objsize -
  2.1362 -                                   BYTES_PER_WORD), RED_MAGIC1) != RED_MAGIC2)
  2.1363 -            /* Either write past end, or a double free. */
  2.1364 -            BUG();
  2.1365 -    }
  2.1366 -    if (cachep->flags & SLAB_POISON)
  2.1367 -        xmem_poison_obj(cachep, objp);
  2.1368 -    if (xmem_extra_free_checks(cachep, slabp, objp))
  2.1369 -        return;
  2.1370 -#endif
  2.1371 -    {
  2.1372 -        unsigned int objnr = (objp-slabp->s_mem)/cachep->objsize;
  2.1373 -
  2.1374 -        slab_bufctl(slabp)[objnr] = slabp->free;
  2.1375 -        slabp->free = objnr;
  2.1376 -    }
  2.1377 -    STATS_DEC_ACTIVE(cachep);
  2.1378 -	
  2.1379 -    /* fixup slab chains */
  2.1380 -    {
  2.1381 -        int inuse = slabp->inuse;
  2.1382 -        if (unlikely(!--slabp->inuse)) {
  2.1383 -            /* Was partial or full, now empty. */
  2.1384 -            list_del(&slabp->list);
  2.1385 -            list_add(&slabp->list, &cachep->slabs_free);
  2.1386 -        } else if (unlikely(inuse == cachep->num)) {
  2.1387 -            /* Was full. */
  2.1388 -            list_del(&slabp->list);
  2.1389 -            list_add(&slabp->list, &cachep->slabs_partial);
  2.1390 -        }
  2.1391 -    }
  2.1392 -}
  2.1393 -
  2.1394 -#ifdef CONFIG_SMP
  2.1395 -static inline void __free_block (xmem_cache_t* cachep,
  2.1396 -                                 void** objpp, int len)
  2.1397 -{
  2.1398 -    for ( ; len > 0; len--, objpp++)
  2.1399 -        xmem_cache_free_one(cachep, *objpp);
  2.1400 -}
  2.1401 -
  2.1402 -static void free_block (xmem_cache_t* cachep, void** objpp, int len)
  2.1403 -{
  2.1404 -    spin_lock(&cachep->spinlock);
  2.1405 -    __free_block(cachep, objpp, len);
  2.1406 -    spin_unlock(&cachep->spinlock);
  2.1407 -}
  2.1408 -#endif
  2.1409 -
  2.1410 -/*
  2.1411 - * __xmem_cache_free
  2.1412 - * called with disabled ints
  2.1413 - */
  2.1414 -static inline void __xmem_cache_free (xmem_cache_t *cachep, void* objp)
  2.1415 -{
  2.1416 -#ifdef CONFIG_SMP
  2.1417 -    cpucache_t *cc = cc_data(cachep);
  2.1418 -
  2.1419 -    CHECK_PAGE(virt_to_page(objp));
  2.1420 -    if (cc) {
  2.1421 -        int batchcount;
  2.1422 -        if (cc->avail < cc->limit) {
  2.1423 -            STATS_INC_FREEHIT(cachep);
  2.1424 -            cc_entry(cc)[cc->avail++] = objp;
  2.1425 -            return;
  2.1426 -        }
  2.1427 -        STATS_INC_FREEMISS(cachep);
  2.1428 -        batchcount = cachep->batchcount;
  2.1429 -        cc->avail -= batchcount;
  2.1430 -        free_block(cachep,
  2.1431 -                   &cc_entry(cc)[cc->avail],batchcount);
  2.1432 -        cc_entry(cc)[cc->avail++] = objp;
  2.1433 -        return;
  2.1434 -    } else {
  2.1435 -        free_block(cachep, &objp, 1);
  2.1436 -    }
  2.1437 -#else
  2.1438 -    xmem_cache_free_one(cachep, objp);
  2.1439 -#endif
  2.1440 -}
  2.1441 -
  2.1442 -/**
  2.1443 - * xmem_cache_alloc - Allocate an object
  2.1444 - * @cachep: The cache to allocate from.
  2.1445 - *
  2.1446 - * Allocate an object from this cache.  The flags are only relevant
  2.1447 - * if the cache has no available objects.
  2.1448 - */
  2.1449 -void *xmem_cache_alloc(xmem_cache_t *cachep)
  2.1450 -{
  2.1451 -    return __xmem_cache_alloc(cachep);
  2.1452 -}
  2.1453 -
  2.1454 -/**
  2.1455 - * _xmalloc - allocate memory
  2.1456 - * @size: how many bytes of memory are required.
  2.1457 - */
  2.1458 -void *_xmalloc(size_t size)
  2.1459 -{
  2.1460 -    cache_sizes_t *csizep = cache_sizes;
  2.1461 -
  2.1462 -    for (; csizep->cs_size; csizep++) {
  2.1463 -        if (size > csizep->cs_size)
  2.1464 -            continue;
  2.1465 -        return __xmem_cache_alloc(csizep->cs_cachep);
  2.1466 -    }
  2.1467 -    return NULL;
  2.1468 -}
  2.1469 -
  2.1470 -/**
  2.1471 - * xmem_cache_free - Deallocate an object
  2.1472 - * @cachep: The cache the allocation was from.
  2.1473 - * @objp: The previously allocated object.
  2.1474 - *
  2.1475 - * Free an object which was previously allocated from this
  2.1476 - * cache.
  2.1477 - */
  2.1478 -void xmem_cache_free (xmem_cache_t *cachep, void *objp)
  2.1479 -{
  2.1480 -    unsigned long flags;
  2.1481 -#if DEBUG
  2.1482 -    CHECK_PAGE(virt_to_page(objp));
  2.1483 -    if (cachep != GET_PAGE_CACHE(virt_to_page(objp)))
  2.1484 -        BUG();
  2.1485 -#endif
  2.1486 -
  2.1487 -    local_irq_save(flags);
  2.1488 -    __xmem_cache_free(cachep, objp);
  2.1489 -    local_irq_restore(flags);
  2.1490 -}
  2.1491 -
  2.1492 -/**
  2.1493 - * xfree - free previously allocated memory
  2.1494 - * @objp: pointer returned by xmalloc.
  2.1495 - *
  2.1496 - * Don't free memory not originally allocated by xmalloc()
  2.1497 - * or you will run into trouble.
  2.1498 - */
  2.1499 -void xfree (const void *objp)
  2.1500 -{
  2.1501 -    xmem_cache_t *c;
  2.1502 -    unsigned long flags;
  2.1503 -
  2.1504 -    if (!objp)
  2.1505 -        return;
  2.1506 -    local_irq_save(flags);
  2.1507 -    CHECK_PAGE(virt_to_page(objp));
  2.1508 -    c = GET_PAGE_CACHE(virt_to_page(objp));
  2.1509 -    __xmem_cache_free(c, (void*)objp);
  2.1510 -    local_irq_restore(flags);
  2.1511 -}
  2.1512 -
  2.1513 -xmem_cache_t *xmem_find_general_cachep(size_t size)
  2.1514 -{
  2.1515 -    cache_sizes_t *csizep = cache_sizes;
  2.1516 -
  2.1517 -    /* This function could be moved to the header file, and
  2.1518 -     * made inline so consumers can quickly determine what
  2.1519 -     * cache pointer they require.
  2.1520 -     */
  2.1521 -    for ( ; csizep->cs_size; csizep++) {
  2.1522 -        if (size > csizep->cs_size)
  2.1523 -            continue;
  2.1524 -        break;
  2.1525 -    }
  2.1526 -    return csizep->cs_cachep;
  2.1527 -}
  2.1528 -
  2.1529 -#ifdef CONFIG_SMP
  2.1530 -
  2.1531 -/* called with cache_chain_sem acquired.  */
  2.1532 -static int xmem_tune_cpucache (xmem_cache_t* cachep, int limit, int batchcount)
  2.1533 -{
  2.1534 -    ccupdate_struct_t new;
  2.1535 -    int i;
  2.1536 -
  2.1537 -    /*
  2.1538 -     * These are admin-provided, so we are more graceful.
  2.1539 -     */
  2.1540 -    if (limit < 0)
  2.1541 -        return -EINVAL;
  2.1542 -    if (batchcount < 0)
  2.1543 -        return -EINVAL;
  2.1544 -    if (batchcount > limit)
  2.1545 -        return -EINVAL;
  2.1546 -    if (limit != 0 && !batchcount)
  2.1547 -        return -EINVAL;
  2.1548 -
  2.1549 -    memset(&new.new,0,sizeof(new.new));
  2.1550 -    if (limit) {
  2.1551 -        for (i = 0; i< smp_num_cpus; i++) {
  2.1552 -            cpucache_t* ccnew;
  2.1553 -
  2.1554 -            ccnew = _xmalloc(sizeof(void*)*limit+sizeof(cpucache_t));
  2.1555 -            if (!ccnew)
  2.1556 -                goto oom;
  2.1557 -            ccnew->limit = limit;
  2.1558 -            ccnew->avail = 0;
  2.1559 -            new.new[cpu_logical_map(i)] = ccnew;
  2.1560 -        }
  2.1561 -    }
  2.1562 -    new.cachep = cachep;
  2.1563 -    spin_lock_irq(&cachep->spinlock);
  2.1564 -    cachep->batchcount = batchcount;
  2.1565 -    spin_unlock_irq(&cachep->spinlock);
  2.1566 -
  2.1567 -    smp_call_function_all_cpus(do_ccupdate_local, (void *)&new);
  2.1568 -
  2.1569 -    for (i = 0; i < smp_num_cpus; i++) {
  2.1570 -        cpucache_t* ccold = new.new[cpu_logical_map(i)];
  2.1571 -        if (!ccold)
  2.1572 -            continue;
  2.1573 -        local_irq_disable();
  2.1574 -        free_block(cachep, cc_entry(ccold), ccold->avail);
  2.1575 -        local_irq_enable();
  2.1576 -        xfree(ccold);
  2.1577 -    }
  2.1578 -    return 0;
  2.1579 - oom:
  2.1580 -    for (i--; i >= 0; i--)
  2.1581 -        xfree(new.new[cpu_logical_map(i)]);
  2.1582 -    return -ENOMEM;
  2.1583 -}
  2.1584 -
  2.1585 -static void enable_cpucache (xmem_cache_t *cachep)
  2.1586 -{
  2.1587 -    int err;
  2.1588 -    int limit;
  2.1589 -
  2.1590 -    /* FIXME: optimize */
  2.1591 -    if (cachep->objsize > PAGE_SIZE)
  2.1592 -        return;
  2.1593 -    if (cachep->objsize > 1024)
  2.1594 -        limit = 60;
  2.1595 -    else if (cachep->objsize > 256)
  2.1596 -        limit = 124;
  2.1597 -    else
  2.1598 -        limit = 252;
  2.1599 -
  2.1600 -    err = xmem_tune_cpucache(cachep, limit, limit/2);
  2.1601 -    if (err)
  2.1602 -        printk(KERN_ERR "enable_cpucache failed for %s, error %d.\n",
  2.1603 -               cachep->name, -err);
  2.1604 -}
  2.1605 -
  2.1606 -static void enable_all_cpucaches (void)
  2.1607 -{
  2.1608 -    struct list_head* p;
  2.1609 -    unsigned long spin_flags;
  2.1610 -
  2.1611 -    down(&cache_chain_sem);
  2.1612 -
  2.1613 -    p = &cache_cache.next;
  2.1614 -    do {
  2.1615 -        xmem_cache_t* cachep = list_entry(p, xmem_cache_t, next);
  2.1616 -
  2.1617 -        enable_cpucache(cachep);
  2.1618 -        p = cachep->next.next;
  2.1619 -    } while (p != &cache_cache.next);
  2.1620 -
  2.1621 -    up(&cache_chain_sem);
  2.1622 -}
  2.1623 -#endif
  2.1624 -
  2.1625 -/**
  2.1626 - * xmem_cache_reap - Reclaim memory from caches.
  2.1627 - */
  2.1628 -int xmem_cache_reap(void)
  2.1629 -{
  2.1630 -    slab_t *slabp;
  2.1631 -    xmem_cache_t *searchp;
  2.1632 -    xmem_cache_t *best_cachep;
  2.1633 -    unsigned int best_pages;
  2.1634 -    unsigned int best_len;
  2.1635 -    unsigned int scan;
  2.1636 -    int ret = 0;
  2.1637 -    unsigned long spin_flags;
  2.1638 -
  2.1639 -    down(&cache_chain_sem);
  2.1640 -
  2.1641 -    scan = REAP_SCANLEN;
  2.1642 -    best_len = 0;
  2.1643 -    best_pages = 0;
  2.1644 -    best_cachep = NULL;
  2.1645 -    searchp = clock_searchp;
  2.1646 -    do {
  2.1647 -        unsigned int pages;
  2.1648 -        struct list_head* p;
  2.1649 -        unsigned int full_free;
  2.1650 -
  2.1651 -        /* It's safe to test this without holding the cache-lock. */
  2.1652 -        if (searchp->flags & SLAB_NO_REAP)
  2.1653 -            goto next;
  2.1654 -        spin_lock_irq(&searchp->spinlock);
  2.1655 -        if (searchp->growing)
  2.1656 -            goto next_unlock;
  2.1657 -        if (searchp->dflags & DFLGS_GROWN) {
  2.1658 -            searchp->dflags &= ~DFLGS_GROWN;
  2.1659 -            goto next_unlock;
  2.1660 -        }
  2.1661 -#ifdef CONFIG_SMP
  2.1662 -        {
  2.1663 -            cpucache_t *cc = cc_data(searchp);
  2.1664 -            if (cc && cc->avail) {
  2.1665 -                __free_block(searchp, cc_entry(cc), cc->avail);
  2.1666 -                cc->avail = 0;
  2.1667 -            }
  2.1668 -        }
  2.1669 -#endif
  2.1670 -
  2.1671 -        full_free = 0;
  2.1672 -        p = searchp->slabs_free.next;
  2.1673 -        while (p != &searchp->slabs_free) {
  2.1674 -            slabp = list_entry(p, slab_t, list);
  2.1675 -#if DEBUG
  2.1676 -            if (slabp->inuse)
  2.1677 -                BUG();
  2.1678 -#endif
  2.1679 -            full_free++;
  2.1680 -            p = p->next;
  2.1681 -        }
  2.1682 -
  2.1683 -        /*
  2.1684 -         * Try to avoid slabs with constructors and/or
  2.1685 -         * more than one page per slab (as it can be difficult
  2.1686 -         * to get high orders from gfp()).
  2.1687 -         */
  2.1688 -        pages = full_free * (1<<searchp->gfporder);
  2.1689 -        if (searchp->ctor)
  2.1690 -            pages = (pages*4+1)/5;
  2.1691 -        if (searchp->gfporder)
  2.1692 -            pages = (pages*4+1)/5;
  2.1693 -        if (pages > best_pages) {
  2.1694 -            best_cachep = searchp;
  2.1695 -            best_len = full_free;
  2.1696 -            best_pages = pages;
  2.1697 -            if (pages >= REAP_PERFECT) {
  2.1698 -                clock_searchp = list_entry(searchp->next.next,
  2.1699 -                                           xmem_cache_t,next);
  2.1700 -                goto perfect;
  2.1701 -            }
  2.1702 -        }
  2.1703 -    next_unlock:
  2.1704 -        spin_unlock_irq(&searchp->spinlock);
  2.1705 -    next:
  2.1706 -        searchp = list_entry(searchp->next.next,xmem_cache_t,next);
  2.1707 -    } while (--scan && searchp != clock_searchp);
  2.1708 -
  2.1709 -    clock_searchp = searchp;
  2.1710 -
  2.1711 -    if (!best_cachep)
  2.1712 -        /* couldn't find anything to reap */
  2.1713 -        goto out;
  2.1714 -
  2.1715 -    spin_lock_irq(&best_cachep->spinlock);
  2.1716 - perfect:
  2.1717 -    /* free only 50% of the free slabs */
  2.1718 -    best_len = (best_len + 1)/2;
  2.1719 -    for (scan = 0; scan < best_len; scan++) {
  2.1720 -        struct list_head *p;
  2.1721 -
  2.1722 -        if (best_cachep->growing)
  2.1723 -            break;
  2.1724 -        p = best_cachep->slabs_free.prev;
  2.1725 -        if (p == &best_cachep->slabs_free)
  2.1726 -            break;
  2.1727 -        slabp = list_entry(p,slab_t,list);
  2.1728 -#if DEBUG
  2.1729 -        if (slabp->inuse)
  2.1730 -            BUG();
  2.1731 -#endif
  2.1732 -        list_del(&slabp->list);
  2.1733 -        STATS_INC_REAPED(best_cachep);
  2.1734 -
  2.1735 -        /* Safe to drop the lock. The slab is no longer linked to the
  2.1736 -         * cache.
  2.1737 -         */
  2.1738 -        spin_unlock_irq(&best_cachep->spinlock);
  2.1739 -        xmem_slab_destroy(best_cachep, slabp);
  2.1740 -        spin_lock_irq(&best_cachep->spinlock);
  2.1741 -    }
  2.1742 -    spin_unlock_irq(&best_cachep->spinlock);
  2.1743 -    ret = scan * (1 << best_cachep->gfporder);
  2.1744 - out:
  2.1745 -    up(&cache_chain_sem);
  2.1746 -    return ret;
  2.1747 -}
  2.1748 -
  2.1749 -void dump_slabinfo()
  2.1750 -{
  2.1751 -    struct list_head *p;
  2.1752 -    unsigned long spin_flags;
  2.1753 -
  2.1754 -    /* Output format version, so at least we can change it without _too_
  2.1755 -     * many complaints.
  2.1756 -     */
  2.1757 -    printk( "slabinfo - version: 1.1"
  2.1758 -#if STATS
  2.1759 -            " (statistics)"
  2.1760 -#endif
  2.1761 -#ifdef CONFIG_SMP
  2.1762 -            " (SMP)"
  2.1763 -#endif
  2.1764 -            "\n");
  2.1765 -    down(&cache_chain_sem);
  2.1766 -    p = &cache_cache.next;
  2.1767 -    do {
  2.1768 -        xmem_cache_t	*cachep;
  2.1769 -        slab_t		*slabp;
  2.1770 -        unsigned long	active_objs;
  2.1771 -        unsigned long	num_objs;
  2.1772 -        unsigned long	active_slabs = 0;
  2.1773 -        unsigned long	num_slabs;
  2.1774 -        cachep = list_entry(p, xmem_cache_t, next);
  2.1775 -
  2.1776 -        spin_lock_irq(&cachep->spinlock);
  2.1777 -        active_objs = 0;
  2.1778 -        num_slabs = 0;
  2.1779 -        list_for_each_entry(slabp, &cachep->slabs_full, list) {
  2.1780 -            if (slabp->inuse != cachep->num)
  2.1781 -                BUG();
  2.1782 -            active_objs += cachep->num;
  2.1783 -            active_slabs++;
  2.1784 -        }
  2.1785 -        list_for_each_entry(slabp, &cachep->slabs_partial, list) {
  2.1786 -            if (slabp->inuse == cachep->num || !slabp->inuse)
  2.1787 -                BUG();
  2.1788 -            active_objs += slabp->inuse;
  2.1789 -            active_slabs++;
  2.1790 -        }
  2.1791 -        list_for_each_entry(slabp, &cachep->slabs_free, list) {
  2.1792 -            if (slabp->inuse)
  2.1793 -                BUG();
  2.1794 -            num_slabs++;
  2.1795 -        }
  2.1796 -        num_slabs+=active_slabs;
  2.1797 -        num_objs = num_slabs*cachep->num;
  2.1798 -
  2.1799 -        printk("%-17s %6lu %6lu %6u %4lu %4lu %4u",
  2.1800 -               cachep->name, active_objs, num_objs, cachep->objsize,
  2.1801 -               active_slabs, num_slabs, (1<<cachep->gfporder));
  2.1802 -
  2.1803 -#if STATS
  2.1804 -        {
  2.1805 -            unsigned long errors = cachep->errors;
  2.1806 -            unsigned long high = cachep->high_mark;
  2.1807 -            unsigned long grown = cachep->grown;
  2.1808 -            unsigned long reaped = cachep->reaped;
  2.1809 -            unsigned long allocs = cachep->num_allocations;
  2.1810 -
  2.1811 -            printk(" : %6lu %7lu %5lu %4lu %4lu",
  2.1812 -                   high, allocs, grown, reaped, errors);
  2.1813 -        }
  2.1814 -#endif
  2.1815 -#ifdef CONFIG_SMP
  2.1816 -        {
  2.1817 -            unsigned int batchcount = cachep->batchcount;
  2.1818 -            unsigned int limit;
  2.1819 -
  2.1820 -            if (cc_data(cachep))
  2.1821 -                limit = cc_data(cachep)->limit;
  2.1822 -            else
  2.1823 -                limit = 0;
  2.1824 -            printk(" : %4u %4u",
  2.1825 -                   limit, batchcount);
  2.1826 -        }
  2.1827 -#endif
  2.1828 -#if STATS && defined(CONFIG_SMP)
  2.1829 -        {
  2.1830 -            unsigned long allochit = atomic_read(&cachep->allochit);
  2.1831 -            unsigned long allocmiss = atomic_read(&cachep->allocmiss);
  2.1832 -            unsigned long freehit = atomic_read(&cachep->freehit);
  2.1833 -            unsigned long freemiss = atomic_read(&cachep->freemiss);
  2.1834 -            printk(" : %6lu %6lu %6lu %6lu",
  2.1835 -                   allochit, allocmiss, freehit, freemiss);
  2.1836 -        }
  2.1837 -#endif
  2.1838 -        printk("\n");
  2.1839 -        spin_unlock_irq(&cachep->spinlock);
  2.1840 -
  2.1841 -        p = cachep->next.next;
  2.1842 -    } while (p != &cache_cache.next);
  2.1843 -
  2.1844 -    up(&cache_chain_sem);
  2.1845 -
  2.1846 -    return;
  2.1847 -}