]>
Commit | Line | Data |
---|---|---|
10cef602 MM |
1 | /* |
2 | * SLOB Allocator: Simple List Of Blocks | |
3 | * | |
4 | * Matt Mackall <[email protected]> 12/30/03 | |
5 | * | |
6 | * How SLOB works: | |
7 | * | |
8 | * The core of SLOB is a traditional K&R style heap allocator, with | |
9 | * support for returning aligned objects. The granularity of this | |
10 | * allocator is 8 bytes on x86, though it's perhaps possible to reduce | |
11 | * this to 4 if it's deemed worth the effort. The slob heap is a | |
12 | * singly-linked list of pages from __get_free_page, grown on demand | |
13 | * and allocation from the heap is currently first-fit. | |
14 | * | |
15 | * Above this is an implementation of kmalloc/kfree. Blocks returned | |
16 | * from kmalloc are 8-byte aligned and prepended with a 8-byte header. | |
17 | * If kmalloc is asked for objects of PAGE_SIZE or larger, it calls | |
18 | * __get_free_pages directly so that it can return page-aligned blocks | |
19 | * and keeps a linked list of such pages and their orders. These | |
20 | * objects are detected in kfree() by their page alignment. | |
21 | * | |
22 | * SLAB is emulated on top of SLOB by simply calling constructors and | |
23 | * destructors for every SLAB allocation. Objects are returned with | |
5af60839 | 24 | * the 8-byte alignment unless the SLAB_HWCACHE_ALIGN flag is |
10cef602 MM |
25 | * set, in which case the low-level allocator will fragment blocks to |
26 | * create the proper alignment. Again, objects of page-size or greater | |
27 | * are allocated by calling __get_free_pages. As SLAB objects know | |
28 | * their size, no separate size bookkeeping is necessary and there is | |
29 | * essentially no allocation space overhead. | |
30 | */ | |
31 | ||
10cef602 MM |
32 | #include <linux/slab.h> |
33 | #include <linux/mm.h> | |
34 | #include <linux/cache.h> | |
35 | #include <linux/init.h> | |
36 | #include <linux/module.h> | |
37 | #include <linux/timer.h> | |
38 | ||
39 | struct slob_block { | |
40 | int units; | |
41 | struct slob_block *next; | |
42 | }; | |
43 | typedef struct slob_block slob_t; | |
44 | ||
45 | #define SLOB_UNIT sizeof(slob_t) | |
46 | #define SLOB_UNITS(size) (((size) + SLOB_UNIT - 1)/SLOB_UNIT) | |
47 | #define SLOB_ALIGN L1_CACHE_BYTES | |
48 | ||
49 | struct bigblock { | |
50 | int order; | |
51 | void *pages; | |
52 | struct bigblock *next; | |
53 | }; | |
54 | typedef struct bigblock bigblock_t; | |
55 | ||
56 | static slob_t arena = { .next = &arena, .units = 1 }; | |
57 | static slob_t *slobfree = &arena; | |
58 | static bigblock_t *bigblocks; | |
59 | static DEFINE_SPINLOCK(slob_lock); | |
60 | static DEFINE_SPINLOCK(block_lock); | |
61 | ||
62 | static void slob_free(void *b, int size); | |
bcb4ddb4 DG |
63 | static void slob_timer_cbk(void); |
64 | ||
10cef602 MM |
65 | |
66 | static void *slob_alloc(size_t size, gfp_t gfp, int align) | |
67 | { | |
68 | slob_t *prev, *cur, *aligned = 0; | |
69 | int delta = 0, units = SLOB_UNITS(size); | |
70 | unsigned long flags; | |
71 | ||
72 | spin_lock_irqsave(&slob_lock, flags); | |
73 | prev = slobfree; | |
74 | for (cur = prev->next; ; prev = cur, cur = cur->next) { | |
75 | if (align) { | |
76 | aligned = (slob_t *)ALIGN((unsigned long)cur, align); | |
77 | delta = aligned - cur; | |
78 | } | |
79 | if (cur->units >= units + delta) { /* room enough? */ | |
80 | if (delta) { /* need to fragment head to align? */ | |
81 | aligned->units = cur->units - delta; | |
82 | aligned->next = cur->next; | |
83 | cur->next = aligned; | |
84 | cur->units = delta; | |
85 | prev = cur; | |
86 | cur = aligned; | |
87 | } | |
88 | ||
89 | if (cur->units == units) /* exact fit? */ | |
90 | prev->next = cur->next; /* unlink */ | |
91 | else { /* fragment */ | |
92 | prev->next = cur + units; | |
93 | prev->next->units = cur->units - units; | |
94 | prev->next->next = cur->next; | |
95 | cur->units = units; | |
96 | } | |
97 | ||
98 | slobfree = prev; | |
99 | spin_unlock_irqrestore(&slob_lock, flags); | |
100 | return cur; | |
101 | } | |
102 | if (cur == slobfree) { | |
103 | spin_unlock_irqrestore(&slob_lock, flags); | |
104 | ||
105 | if (size == PAGE_SIZE) /* trying to shrink arena? */ | |
106 | return 0; | |
107 | ||
108 | cur = (slob_t *)__get_free_page(gfp); | |
109 | if (!cur) | |
110 | return 0; | |
111 | ||
112 | slob_free(cur, PAGE_SIZE); | |
113 | spin_lock_irqsave(&slob_lock, flags); | |
114 | cur = slobfree; | |
115 | } | |
116 | } | |
117 | } | |
118 | ||
119 | static void slob_free(void *block, int size) | |
120 | { | |
121 | slob_t *cur, *b = (slob_t *)block; | |
122 | unsigned long flags; | |
123 | ||
124 | if (!block) | |
125 | return; | |
126 | ||
127 | if (size) | |
128 | b->units = SLOB_UNITS(size); | |
129 | ||
130 | /* Find reinsertion point */ | |
131 | spin_lock_irqsave(&slob_lock, flags); | |
132 | for (cur = slobfree; !(b > cur && b < cur->next); cur = cur->next) | |
133 | if (cur >= cur->next && (b > cur || b < cur->next)) | |
134 | break; | |
135 | ||
136 | if (b + b->units == cur->next) { | |
137 | b->units += cur->next->units; | |
138 | b->next = cur->next->next; | |
139 | } else | |
140 | b->next = cur->next; | |
141 | ||
142 | if (cur + cur->units == b) { | |
143 | cur->units += b->units; | |
144 | cur->next = b->next; | |
145 | } else | |
146 | cur->next = b; | |
147 | ||
148 | slobfree = cur; | |
149 | ||
150 | spin_unlock_irqrestore(&slob_lock, flags); | |
151 | } | |
152 | ||
2e892f43 | 153 | void *__kmalloc(size_t size, gfp_t gfp) |
10cef602 MM |
154 | { |
155 | slob_t *m; | |
156 | bigblock_t *bb; | |
157 | unsigned long flags; | |
158 | ||
159 | if (size < PAGE_SIZE - SLOB_UNIT) { | |
160 | m = slob_alloc(size + SLOB_UNIT, gfp, 0); | |
161 | return m ? (void *)(m + 1) : 0; | |
162 | } | |
163 | ||
164 | bb = slob_alloc(sizeof(bigblock_t), gfp, 0); | |
165 | if (!bb) | |
166 | return 0; | |
167 | ||
4ab688c5 | 168 | bb->order = get_order(size); |
10cef602 MM |
169 | bb->pages = (void *)__get_free_pages(gfp, bb->order); |
170 | ||
171 | if (bb->pages) { | |
172 | spin_lock_irqsave(&block_lock, flags); | |
173 | bb->next = bigblocks; | |
174 | bigblocks = bb; | |
175 | spin_unlock_irqrestore(&block_lock, flags); | |
176 | return bb->pages; | |
177 | } | |
178 | ||
179 | slob_free(bb, sizeof(bigblock_t)); | |
180 | return 0; | |
181 | } | |
2e892f43 | 182 | EXPORT_SYMBOL(__kmalloc); |
10cef602 | 183 | |
fd76bab2 PE |
184 | /** |
185 | * krealloc - reallocate memory. The contents will remain unchanged. | |
186 | * | |
187 | * @p: object to reallocate memory for. | |
188 | * @new_size: how many bytes of memory are required. | |
189 | * @flags: the type of memory to allocate. | |
190 | * | |
191 | * The contents of the object pointed to are preserved up to the | |
192 | * lesser of the new and old sizes. If @p is %NULL, krealloc() | |
193 | * behaves exactly like kmalloc(). If @size is 0 and @p is not a | |
194 | * %NULL pointer, the object pointed to is freed. | |
195 | */ | |
196 | void *krealloc(const void *p, size_t new_size, gfp_t flags) | |
197 | { | |
198 | void *ret; | |
199 | ||
200 | if (unlikely(!p)) | |
201 | return kmalloc_track_caller(new_size, flags); | |
202 | ||
203 | if (unlikely(!new_size)) { | |
204 | kfree(p); | |
205 | return NULL; | |
206 | } | |
207 | ||
208 | ret = kmalloc_track_caller(new_size, flags); | |
209 | if (ret) { | |
210 | memcpy(ret, p, min(new_size, ksize(p))); | |
211 | kfree(p); | |
212 | } | |
213 | return ret; | |
214 | } | |
215 | EXPORT_SYMBOL(krealloc); | |
216 | ||
10cef602 MM |
217 | void kfree(const void *block) |
218 | { | |
219 | bigblock_t *bb, **last = &bigblocks; | |
220 | unsigned long flags; | |
221 | ||
222 | if (!block) | |
223 | return; | |
224 | ||
225 | if (!((unsigned long)block & (PAGE_SIZE-1))) { | |
226 | /* might be on the big block list */ | |
227 | spin_lock_irqsave(&block_lock, flags); | |
228 | for (bb = bigblocks; bb; last = &bb->next, bb = bb->next) { | |
229 | if (bb->pages == block) { | |
230 | *last = bb->next; | |
231 | spin_unlock_irqrestore(&block_lock, flags); | |
232 | free_pages((unsigned long)block, bb->order); | |
233 | slob_free(bb, sizeof(bigblock_t)); | |
234 | return; | |
235 | } | |
236 | } | |
237 | spin_unlock_irqrestore(&block_lock, flags); | |
238 | } | |
239 | ||
240 | slob_free((slob_t *)block - 1, 0); | |
241 | return; | |
242 | } | |
243 | ||
244 | EXPORT_SYMBOL(kfree); | |
245 | ||
fd76bab2 | 246 | size_t ksize(const void *block) |
10cef602 MM |
247 | { |
248 | bigblock_t *bb; | |
249 | unsigned long flags; | |
250 | ||
251 | if (!block) | |
252 | return 0; | |
253 | ||
254 | if (!((unsigned long)block & (PAGE_SIZE-1))) { | |
255 | spin_lock_irqsave(&block_lock, flags); | |
256 | for (bb = bigblocks; bb; bb = bb->next) | |
257 | if (bb->pages == block) { | |
258 | spin_unlock_irqrestore(&slob_lock, flags); | |
259 | return PAGE_SIZE << bb->order; | |
260 | } | |
261 | spin_unlock_irqrestore(&block_lock, flags); | |
262 | } | |
263 | ||
264 | return ((slob_t *)block - 1)->units * SLOB_UNIT; | |
265 | } | |
266 | ||
267 | struct kmem_cache { | |
268 | unsigned int size, align; | |
269 | const char *name; | |
270 | void (*ctor)(void *, struct kmem_cache *, unsigned long); | |
271 | void (*dtor)(void *, struct kmem_cache *, unsigned long); | |
272 | }; | |
273 | ||
274 | struct kmem_cache *kmem_cache_create(const char *name, size_t size, | |
275 | size_t align, unsigned long flags, | |
276 | void (*ctor)(void*, struct kmem_cache *, unsigned long), | |
277 | void (*dtor)(void*, struct kmem_cache *, unsigned long)) | |
278 | { | |
279 | struct kmem_cache *c; | |
280 | ||
281 | c = slob_alloc(sizeof(struct kmem_cache), flags, 0); | |
282 | ||
283 | if (c) { | |
284 | c->name = name; | |
285 | c->size = size; | |
286 | c->ctor = ctor; | |
287 | c->dtor = dtor; | |
288 | /* ignore alignment unless it's forced */ | |
5af60839 | 289 | c->align = (flags & SLAB_HWCACHE_ALIGN) ? SLOB_ALIGN : 0; |
10cef602 MM |
290 | if (c->align < align) |
291 | c->align = align; | |
bc0055ae AM |
292 | } else if (flags & SLAB_PANIC) |
293 | panic("Cannot create slab cache %s\n", name); | |
10cef602 MM |
294 | |
295 | return c; | |
296 | } | |
297 | EXPORT_SYMBOL(kmem_cache_create); | |
298 | ||
133d205a | 299 | void kmem_cache_destroy(struct kmem_cache *c) |
10cef602 MM |
300 | { |
301 | slob_free(c, sizeof(struct kmem_cache)); | |
10cef602 MM |
302 | } |
303 | EXPORT_SYMBOL(kmem_cache_destroy); | |
304 | ||
305 | void *kmem_cache_alloc(struct kmem_cache *c, gfp_t flags) | |
306 | { | |
307 | void *b; | |
308 | ||
309 | if (c->size < PAGE_SIZE) | |
310 | b = slob_alloc(c->size, flags, c->align); | |
311 | else | |
4ab688c5 | 312 | b = (void *)__get_free_pages(flags, get_order(c->size)); |
10cef602 MM |
313 | |
314 | if (c->ctor) | |
315 | c->ctor(b, c, SLAB_CTOR_CONSTRUCTOR); | |
316 | ||
317 | return b; | |
318 | } | |
319 | EXPORT_SYMBOL(kmem_cache_alloc); | |
320 | ||
a8c0f9a4 PE |
321 | void *kmem_cache_zalloc(struct kmem_cache *c, gfp_t flags) |
322 | { | |
323 | void *ret = kmem_cache_alloc(c, flags); | |
324 | if (ret) | |
325 | memset(ret, 0, c->size); | |
326 | ||
327 | return ret; | |
328 | } | |
329 | EXPORT_SYMBOL(kmem_cache_zalloc); | |
330 | ||
10cef602 MM |
331 | void kmem_cache_free(struct kmem_cache *c, void *b) |
332 | { | |
333 | if (c->dtor) | |
334 | c->dtor(b, c, 0); | |
335 | ||
336 | if (c->size < PAGE_SIZE) | |
337 | slob_free(b, c->size); | |
338 | else | |
4ab688c5 | 339 | free_pages((unsigned long)b, get_order(c->size)); |
10cef602 MM |
340 | } |
341 | EXPORT_SYMBOL(kmem_cache_free); | |
342 | ||
343 | unsigned int kmem_cache_size(struct kmem_cache *c) | |
344 | { | |
345 | return c->size; | |
346 | } | |
347 | EXPORT_SYMBOL(kmem_cache_size); | |
348 | ||
349 | const char *kmem_cache_name(struct kmem_cache *c) | |
350 | { | |
351 | return c->name; | |
352 | } | |
353 | EXPORT_SYMBOL(kmem_cache_name); | |
354 | ||
355 | static struct timer_list slob_timer = TIMER_INITIALIZER( | |
bcb4ddb4 | 356 | (void (*)(unsigned long))slob_timer_cbk, 0, 0); |
10cef602 | 357 | |
2e892f43 CL |
358 | int kmem_cache_shrink(struct kmem_cache *d) |
359 | { | |
360 | return 0; | |
361 | } | |
362 | EXPORT_SYMBOL(kmem_cache_shrink); | |
363 | ||
55935a34 | 364 | int kmem_ptr_validate(struct kmem_cache *a, const void *b) |
2e892f43 CL |
365 | { |
366 | return 0; | |
367 | } | |
368 | ||
bcb4ddb4 DG |
369 | void __init kmem_cache_init(void) |
370 | { | |
371 | slob_timer_cbk(); | |
372 | } | |
373 | ||
374 | static void slob_timer_cbk(void) | |
10cef602 MM |
375 | { |
376 | void *p = slob_alloc(PAGE_SIZE, 0, PAGE_SIZE-1); | |
377 | ||
378 | if (p) | |
379 | free_page((unsigned long)p); | |
380 | ||
381 | mod_timer(&slob_timer, jiffies + HZ); | |
382 | } |