3b0efdfa1e
Define a struct that describes common fields used in all slab allocators. A slab allocator either uses the common definition (like SLOB) or is required to provide members of kmem_cache with the definition given. After that it will be possible to share code that only operates on those fields of kmem_cache. The patch basically takes the slob definition of kmem cache and uses the field namees for the other allocators. It also standardizes the names used for basic object lengths in allocators: object_size Struct size specified at kmem_cache_create. Basically the payload expected to be used by the subsystem. size The size of memory allocator for each object. This size is larger than object_size and includes padding, alignment and extra metadata for each object (f.e. for debugging and rcu). Signed-off-by: Christoph Lameter <cl@linux.com> Signed-off-by: Pekka Enberg <penberg@kernel.org>
217 lines
5 KiB
C
217 lines
5 KiB
C
#ifndef _LINUX_SLAB_DEF_H
|
|
#define _LINUX_SLAB_DEF_H
|
|
|
|
/*
|
|
* Definitions unique to the original Linux SLAB allocator.
|
|
*
|
|
* What we provide here is a way to optimize the frequent kmalloc
|
|
* calls in the kernel by selecting the appropriate general cache
|
|
* if kmalloc was called with a size that can be established at
|
|
* compile time.
|
|
*/
|
|
|
|
#include <linux/init.h>
|
|
#include <asm/page.h> /* kmalloc_sizes.h needs PAGE_SIZE */
|
|
#include <asm/cache.h> /* kmalloc_sizes.h needs L1_CACHE_BYTES */
|
|
#include <linux/compiler.h>
|
|
|
|
/*
|
|
* struct kmem_cache
|
|
*
|
|
* manages a cache.
|
|
*/
|
|
|
|
struct kmem_cache {
|
|
/* 1) Cache tunables. Protected by cache_chain_mutex */
|
|
unsigned int batchcount;
|
|
unsigned int limit;
|
|
unsigned int shared;
|
|
|
|
unsigned int size;
|
|
u32 reciprocal_buffer_size;
|
|
/* 2) touched by every alloc & free from the backend */
|
|
|
|
unsigned int flags; /* constant flags */
|
|
unsigned int num; /* # of objs per slab */
|
|
|
|
/* 3) cache_grow/shrink */
|
|
/* order of pgs per slab (2^n) */
|
|
unsigned int gfporder;
|
|
|
|
/* force GFP flags, e.g. GFP_DMA */
|
|
gfp_t gfpflags;
|
|
|
|
size_t colour; /* cache colouring range */
|
|
unsigned int colour_off; /* colour offset */
|
|
struct kmem_cache *slabp_cache;
|
|
unsigned int slab_size;
|
|
unsigned int dflags; /* dynamic flags */
|
|
|
|
/* constructor func */
|
|
void (*ctor)(void *obj);
|
|
|
|
/* 4) cache creation/removal */
|
|
const char *name;
|
|
struct list_head list;
|
|
int refcount;
|
|
int object_size;
|
|
int align;
|
|
|
|
/* 5) statistics */
|
|
#ifdef CONFIG_DEBUG_SLAB
|
|
unsigned long num_active;
|
|
unsigned long num_allocations;
|
|
unsigned long high_mark;
|
|
unsigned long grown;
|
|
unsigned long reaped;
|
|
unsigned long errors;
|
|
unsigned long max_freeable;
|
|
unsigned long node_allocs;
|
|
unsigned long node_frees;
|
|
unsigned long node_overflow;
|
|
atomic_t allochit;
|
|
atomic_t allocmiss;
|
|
atomic_t freehit;
|
|
atomic_t freemiss;
|
|
|
|
/*
|
|
* If debugging is enabled, then the allocator can add additional
|
|
* fields and/or padding to every object. size contains the total
|
|
* object size including these internal fields, the following two
|
|
* variables contain the offset to the user object and its size.
|
|
*/
|
|
int obj_offset;
|
|
#endif /* CONFIG_DEBUG_SLAB */
|
|
|
|
/* 6) per-cpu/per-node data, touched during every alloc/free */
|
|
/*
|
|
* We put array[] at the end of kmem_cache, because we want to size
|
|
* this array to nr_cpu_ids slots instead of NR_CPUS
|
|
* (see kmem_cache_init())
|
|
* We still use [NR_CPUS] and not [1] or [0] because cache_cache
|
|
* is statically defined, so we reserve the max number of cpus.
|
|
*/
|
|
struct kmem_list3 **nodelists;
|
|
struct array_cache *array[NR_CPUS];
|
|
/*
|
|
* Do not add fields after array[]
|
|
*/
|
|
};
|
|
|
|
/* Size description struct for general caches. */
|
|
struct cache_sizes {
|
|
size_t cs_size;
|
|
struct kmem_cache *cs_cachep;
|
|
#ifdef CONFIG_ZONE_DMA
|
|
struct kmem_cache *cs_dmacachep;
|
|
#endif
|
|
};
|
|
extern struct cache_sizes malloc_sizes[];
|
|
|
|
void *kmem_cache_alloc(struct kmem_cache *, gfp_t);
|
|
void *__kmalloc(size_t size, gfp_t flags);
|
|
|
|
#ifdef CONFIG_TRACING
|
|
extern void *kmem_cache_alloc_trace(size_t size,
|
|
struct kmem_cache *cachep, gfp_t flags);
|
|
extern size_t slab_buffer_size(struct kmem_cache *cachep);
|
|
#else
|
|
static __always_inline void *
|
|
kmem_cache_alloc_trace(size_t size, struct kmem_cache *cachep, gfp_t flags)
|
|
{
|
|
return kmem_cache_alloc(cachep, flags);
|
|
}
|
|
static inline size_t slab_buffer_size(struct kmem_cache *cachep)
|
|
{
|
|
return 0;
|
|
}
|
|
#endif
|
|
|
|
static __always_inline void *kmalloc(size_t size, gfp_t flags)
|
|
{
|
|
struct kmem_cache *cachep;
|
|
void *ret;
|
|
|
|
if (__builtin_constant_p(size)) {
|
|
int i = 0;
|
|
|
|
if (!size)
|
|
return ZERO_SIZE_PTR;
|
|
|
|
#define CACHE(x) \
|
|
if (size <= x) \
|
|
goto found; \
|
|
else \
|
|
i++;
|
|
#include <linux/kmalloc_sizes.h>
|
|
#undef CACHE
|
|
return NULL;
|
|
found:
|
|
#ifdef CONFIG_ZONE_DMA
|
|
if (flags & GFP_DMA)
|
|
cachep = malloc_sizes[i].cs_dmacachep;
|
|
else
|
|
#endif
|
|
cachep = malloc_sizes[i].cs_cachep;
|
|
|
|
ret = kmem_cache_alloc_trace(size, cachep, flags);
|
|
|
|
return ret;
|
|
}
|
|
return __kmalloc(size, flags);
|
|
}
|
|
|
|
#ifdef CONFIG_NUMA
|
|
extern void *__kmalloc_node(size_t size, gfp_t flags, int node);
|
|
extern void *kmem_cache_alloc_node(struct kmem_cache *, gfp_t flags, int node);
|
|
|
|
#ifdef CONFIG_TRACING
|
|
extern void *kmem_cache_alloc_node_trace(size_t size,
|
|
struct kmem_cache *cachep,
|
|
gfp_t flags,
|
|
int nodeid);
|
|
#else
|
|
static __always_inline void *
|
|
kmem_cache_alloc_node_trace(size_t size,
|
|
struct kmem_cache *cachep,
|
|
gfp_t flags,
|
|
int nodeid)
|
|
{
|
|
return kmem_cache_alloc_node(cachep, flags, nodeid);
|
|
}
|
|
#endif
|
|
|
|
static __always_inline void *kmalloc_node(size_t size, gfp_t flags, int node)
|
|
{
|
|
struct kmem_cache *cachep;
|
|
|
|
if (__builtin_constant_p(size)) {
|
|
int i = 0;
|
|
|
|
if (!size)
|
|
return ZERO_SIZE_PTR;
|
|
|
|
#define CACHE(x) \
|
|
if (size <= x) \
|
|
goto found; \
|
|
else \
|
|
i++;
|
|
#include <linux/kmalloc_sizes.h>
|
|
#undef CACHE
|
|
return NULL;
|
|
found:
|
|
#ifdef CONFIG_ZONE_DMA
|
|
if (flags & GFP_DMA)
|
|
cachep = malloc_sizes[i].cs_dmacachep;
|
|
else
|
|
#endif
|
|
cachep = malloc_sizes[i].cs_cachep;
|
|
|
|
return kmem_cache_alloc_node_trace(size, cachep, flags, node);
|
|
}
|
|
return __kmalloc_node(size, flags, node);
|
|
}
|
|
|
|
#endif /* CONFIG_NUMA */
|
|
|
|
#endif /* _LINUX_SLAB_DEF_H */
|