2 * Copyright © 2010 Intel Corporation
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
11 * The above copyright notice and this permission notice (including the next
12 * paragraph) shall be included in all copies or substantial portions of the
15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
18 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
20 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
21 * DEALINGS IN THE SOFTWARE.
31 /* Some versions of MinGW are missing _vscprintf's declaration, although they
32 * still provide the symbol in the import library. */
34 _CRTIMP int _vscprintf(const char *format, va_list argptr);
41 #define va_copy(dest, src) __va_copy((dest), (src))
43 #define va_copy(dest, src) (dest) = (src)
47 #define CANARY 0x5A1106
49 /* Align the header's size so that ralloc() allocations will return with the
50 * same alignment as a libc malloc would have (8 on 32-bit GLIBC, 16 on
51 * 64-bit), avoiding performance penalities on x86 and alignment faults on
57 #elif defined(__LP64__)
58 __attribute__((aligned(16)))
60 __attribute__((aligned(8)))
65 /* A canary value used to determine whether a pointer is ralloc'd. */
69 struct ralloc_header *parent;
71 /* The first child (head of a linked list) */
72 struct ralloc_header *child;
74 /* Linked list of siblings */
75 struct ralloc_header *prev;
76 struct ralloc_header *next;
78 void (*destructor)(void *);
81 typedef struct ralloc_header ralloc_header;
83 static void unlink_block(ralloc_header *info);
84 static void unsafe_free(ralloc_header *info);
86 static ralloc_header *
87 get_header(const void *ptr)
89 ralloc_header *info = (ralloc_header *) (((char *) ptr) -
90 sizeof(ralloc_header));
92 assert(info->canary == CANARY);
97 #define PTR_FROM_HEADER(info) (((char *) info) + sizeof(ralloc_header))
100 add_child(ralloc_header *parent, ralloc_header *info)
102 if (parent != NULL) {
103 info->parent = parent;
104 info->next = parent->child;
105 parent->child = info;
107 if (info->next != NULL)
108 info->next->prev = info;
113 ralloc_context(const void *ctx)
115 return ralloc_size(ctx, 0);
119 ralloc_size(const void *ctx, size_t size)
121 void *block = malloc(size + sizeof(ralloc_header));
123 ralloc_header *parent;
125 if (unlikely(block == NULL))
128 info = (ralloc_header *) block;
129 /* measurements have shown that calloc is slower (because of
130 * the multiplication overflow checking?), so clear things
137 info->destructor = NULL;
139 parent = ctx != NULL ? get_header(ctx) : NULL;
141 add_child(parent, info);
144 info->canary = CANARY;
147 return PTR_FROM_HEADER(info);
151 rzalloc_size(const void *ctx, size_t size)
153 void *ptr = ralloc_size(ctx, size);
156 memset(ptr, 0, size);
161 /* helper function - assumes ptr != NULL */
163 resize(void *ptr, size_t size)
165 ralloc_header *child, *old, *info;
167 old = get_header(ptr);
168 info = realloc(old, size + sizeof(ralloc_header));
173 /* Update parent and sibling's links to the reallocated node. */
174 if (info != old && info->parent != NULL) {
175 if (info->parent->child == old)
176 info->parent->child = info;
178 if (info->prev != NULL)
179 info->prev->next = info;
181 if (info->next != NULL)
182 info->next->prev = info;
185 /* Update child->parent links for all children */
186 for (child = info->child; child != NULL; child = child->next)
187 child->parent = info;
189 return PTR_FROM_HEADER(info);
193 reralloc_size(const void *ctx, void *ptr, size_t size)
195 if (unlikely(ptr == NULL))
196 return ralloc_size(ctx, size);
198 assert(ralloc_parent(ptr) == ctx);
199 return resize(ptr, size);
203 ralloc_array_size(const void *ctx, size_t size, unsigned count)
205 if (count > SIZE_MAX/size)
208 return ralloc_size(ctx, size * count);
212 rzalloc_array_size(const void *ctx, size_t size, unsigned count)
214 if (count > SIZE_MAX/size)
217 return rzalloc_size(ctx, size * count);
221 reralloc_array_size(const void *ctx, void *ptr, size_t size, unsigned count)
223 if (count > SIZE_MAX/size)
226 return reralloc_size(ctx, ptr, size * count);
230 ralloc_free(void *ptr)
237 info = get_header(ptr);
243 unlink_block(ralloc_header *info)
245 /* Unlink from parent & siblings */
246 if (info->parent != NULL) {
247 if (info->parent->child == info)
248 info->parent->child = info->next;
250 if (info->prev != NULL)
251 info->prev->next = info->next;
253 if (info->next != NULL)
254 info->next->prev = info->prev;
262 unsafe_free(ralloc_header *info)
264 /* Recursively free any children...don't waste time unlinking them. */
266 while (info->child != NULL) {
268 info->child = temp->next;
272 /* Free the block itself. Call the destructor first, if any. */
273 if (info->destructor != NULL)
274 info->destructor(PTR_FROM_HEADER(info));
280 ralloc_steal(const void *new_ctx, void *ptr)
282 ralloc_header *info, *parent;
284 if (unlikely(ptr == NULL))
287 info = get_header(ptr);
288 parent = new_ctx ? get_header(new_ctx) : NULL;
292 add_child(parent, info);
296 ralloc_adopt(const void *new_ctx, void *old_ctx)
298 ralloc_header *new_info, *old_info, *child;
300 if (unlikely(old_ctx == NULL))
303 old_info = get_header(old_ctx);
304 new_info = get_header(new_ctx);
306 /* If there are no children, bail. */
307 if (unlikely(old_info->child == NULL))
310 /* Set all the children's parent to new_ctx; get a pointer to the last child. */
311 for (child = old_info->child; child->next != NULL; child = child->next) {
312 child->parent = new_info;
314 child->parent = new_info;
316 /* Connect the two lists together; parent them to new_ctx; make old_ctx empty. */
317 child->next = new_info->child;
319 child->next->prev = child;
320 new_info->child = old_info->child;
321 old_info->child = NULL;
325 ralloc_parent(const void *ptr)
329 if (unlikely(ptr == NULL))
332 info = get_header(ptr);
333 return info->parent ? PTR_FROM_HEADER(info->parent) : NULL;
337 ralloc_set_destructor(const void *ptr, void(*destructor)(void *))
339 ralloc_header *info = get_header(ptr);
340 info->destructor = destructor;
344 ralloc_strdup(const void *ctx, const char *str)
349 if (unlikely(str == NULL))
353 ptr = ralloc_array(ctx, char, n + 1);
360 ralloc_strndup(const void *ctx, const char *str, size_t max)
365 if (unlikely(str == NULL))
368 n = strnlen(str, max);
369 ptr = ralloc_array(ctx, char, n + 1);
375 /* helper routine for strcat/strncat - n is the exact amount to copy */
377 cat(char **dest, const char *str, size_t n)
380 size_t existing_length;
381 assert(dest != NULL && *dest != NULL);
383 existing_length = strlen(*dest);
384 both = resize(*dest, existing_length + n + 1);
385 if (unlikely(both == NULL))
388 memcpy(both + existing_length, str, n);
389 both[existing_length + n] = '\0';
397 ralloc_strcat(char **dest, const char *str)
399 return cat(dest, str, strlen(str));
403 ralloc_strncat(char **dest, const char *str, size_t n)
405 return cat(dest, str, strnlen(str, n));
409 ralloc_str_append(char **dest, const char *str,
410 size_t existing_length, size_t str_size)
413 assert(dest != NULL && *dest != NULL);
415 both = resize(*dest, existing_length + str_size + 1);
416 if (unlikely(both == NULL))
419 memcpy(both + existing_length, str, str_size);
420 both[existing_length + str_size] = '\0';
428 ralloc_asprintf(const void *ctx, const char *fmt, ...)
433 ptr = ralloc_vasprintf(ctx, fmt, args);
438 /* Return the length of the string that would be generated by a printf-style
439 * format and argument list, not including the \0 byte.
442 printf_length(const char *fmt, va_list untouched_args)
447 /* Make a copy of the va_list so the original caller can still use it */
449 va_copy(args, untouched_args);
452 /* We need to use _vcsprintf to calculate the size as vsnprintf returns -1
453 * if the number of characters to write is greater than count.
455 size = _vscprintf(fmt, args);
458 size = vsnprintf(&junk, 1, fmt, args);
468 ralloc_vasprintf(const void *ctx, const char *fmt, va_list args)
470 size_t size = printf_length(fmt, args) + 1;
472 char *ptr = ralloc_size(ctx, size);
474 vsnprintf(ptr, size, fmt, args);
480 ralloc_asprintf_append(char **str, const char *fmt, ...)
485 success = ralloc_vasprintf_append(str, fmt, args);
491 ralloc_vasprintf_append(char **str, const char *fmt, va_list args)
493 size_t existing_length;
495 existing_length = *str ? strlen(*str) : 0;
496 return ralloc_vasprintf_rewrite_tail(str, &existing_length, fmt, args);
500 ralloc_asprintf_rewrite_tail(char **str, size_t *start, const char *fmt, ...)
505 success = ralloc_vasprintf_rewrite_tail(str, start, fmt, args);
511 ralloc_vasprintf_rewrite_tail(char **str, size_t *start, const char *fmt,
519 if (unlikely(*str == NULL)) {
520 // Assuming a NULL context is probably bad, but it's expected behavior.
521 *str = ralloc_vasprintf(NULL, fmt, args);
522 *start = strlen(*str);
526 new_length = printf_length(fmt, args);
528 ptr = resize(*str, *start + new_length + 1);
529 if (unlikely(ptr == NULL))
532 vsnprintf(ptr + *start, new_length + 1, fmt, args);
534 *start += new_length;
538 /***************************************************************************
539 * Linear allocator for short-lived allocations.
540 ***************************************************************************
542 * The allocator consists of a parent node (2K buffer), which requires
543 * a ralloc parent, and child nodes (allocations). Child nodes can't be freed
544 * directly, because the parent doesn't track them. You have to release
545 * the parent node in order to release all its children.
547 * The allocator uses a fixed-sized buffer with a monotonically increasing
548 * offset after each allocation. If the buffer is all used, another buffer
549 * is allocated, sharing the same ralloc parent, so all buffers are at
550 * the same level in the ralloc hierarchy.
552 * The linear parent node is always the first buffer and keeps track of all
556 #define MIN_LINEAR_BUFSIZE 2048
557 #define SUBALLOC_ALIGNMENT 8
558 #define LMAGIC 0x87b9c7d3
563 #elif defined(__LP64__)
564 __attribute__((aligned(16)))
566 __attribute__((aligned(8)))
570 unsigned magic; /* for debugging */
572 unsigned offset; /* points to the first unused byte in the buffer */
573 unsigned size; /* size of the buffer */
574 void *ralloc_parent; /* new buffers will use this */
575 struct linear_header *next; /* next buffer if we have more */
576 struct linear_header *latest; /* the only buffer that has free space */
578 /* After this structure, the buffer begins.
579 * Each suballocation consists of linear_size_chunk as its header followed
580 * by the suballocation, so it goes:
582 * - linear_size_chunk
584 * - linear_size_chunk
588 * linear_size_chunk is only needed by linear_realloc.
592 struct linear_size_chunk {
593 unsigned size; /* for realloc */
597 typedef struct linear_header linear_header;
598 typedef struct linear_size_chunk linear_size_chunk;
600 #define LINEAR_PARENT_TO_HEADER(parent) \
602 ((char*)(parent) - sizeof(linear_size_chunk) - sizeof(linear_header))
604 /* Allocate the linear buffer with its header. */
605 static linear_header *
606 create_linear_node(void *ralloc_ctx, unsigned min_size)
610 min_size += sizeof(linear_size_chunk);
612 if (likely(min_size < MIN_LINEAR_BUFSIZE))
613 min_size = MIN_LINEAR_BUFSIZE;
615 node = ralloc_size(ralloc_ctx, sizeof(linear_header) + min_size);
620 node->magic = LMAGIC;
623 node->size = min_size;
624 node->ralloc_parent = ralloc_ctx;
631 linear_alloc_child(void *parent, unsigned size)
633 linear_header *first = LINEAR_PARENT_TO_HEADER(parent);
634 linear_header *latest = first->latest;
635 linear_header *new_node;
636 linear_size_chunk *ptr;
640 assert(first->magic == LMAGIC);
642 assert(!latest->next);
644 size = ALIGN_POT(size, SUBALLOC_ALIGNMENT);
645 full_size = sizeof(linear_size_chunk) + size;
647 if (unlikely(latest->offset + full_size > latest->size)) {
648 /* allocate a new node */
649 new_node = create_linear_node(latest->ralloc_parent, size);
650 if (unlikely(!new_node))
653 first->latest = new_node;
654 latest->latest = new_node;
655 latest->next = new_node;
659 ptr = (linear_size_chunk *)((char*)&latest[1] + latest->offset);
661 latest->offset += full_size;
663 assert((uintptr_t)&ptr[1] % SUBALLOC_ALIGNMENT == 0);
668 linear_alloc_parent(void *ralloc_ctx, unsigned size)
672 if (unlikely(!ralloc_ctx))
675 size = ALIGN_POT(size, SUBALLOC_ALIGNMENT);
677 node = create_linear_node(ralloc_ctx, size);
681 return linear_alloc_child((char*)node +
682 sizeof(linear_header) +
683 sizeof(linear_size_chunk), size);
687 linear_zalloc_child(void *parent, unsigned size)
689 void *ptr = linear_alloc_child(parent, size);
692 memset(ptr, 0, size);
697 linear_zalloc_parent(void *parent, unsigned size)
699 void *ptr = linear_alloc_parent(parent, size);
702 memset(ptr, 0, size);
707 linear_free_parent(void *ptr)
714 node = LINEAR_PARENT_TO_HEADER(ptr);
716 assert(node->magic == LMAGIC);
728 ralloc_steal_linear_parent(void *new_ralloc_ctx, void *ptr)
735 node = LINEAR_PARENT_TO_HEADER(ptr);
737 assert(node->magic == LMAGIC);
741 ralloc_steal(new_ralloc_ctx, node);
742 node->ralloc_parent = new_ralloc_ctx;
748 ralloc_parent_of_linear_parent(void *ptr)
750 linear_header *node = LINEAR_PARENT_TO_HEADER(ptr);
752 assert(node->magic == LMAGIC);
754 return node->ralloc_parent;
758 linear_realloc(void *parent, void *old, unsigned new_size)
760 unsigned old_size = 0;
761 ralloc_header *new_ptr;
763 new_ptr = linear_alloc_child(parent, new_size);
768 old_size = ((linear_size_chunk*)old)[-1].size;
770 if (likely(new_ptr && old_size))
771 memcpy(new_ptr, old, MIN2(old_size, new_size));
776 /* All code below is pretty much copied from ralloc and only the alloc
777 * calls are different.
781 linear_strdup(void *parent, const char *str)
790 ptr = linear_alloc_child(parent, n + 1);
800 linear_asprintf(void *parent, const char *fmt, ...)
805 ptr = linear_vasprintf(parent, fmt, args);
811 linear_vasprintf(void *parent, const char *fmt, va_list args)
813 unsigned size = printf_length(fmt, args) + 1;
815 char *ptr = linear_alloc_child(parent, size);
817 vsnprintf(ptr, size, fmt, args);
823 linear_asprintf_append(void *parent, char **str, const char *fmt, ...)
828 success = linear_vasprintf_append(parent, str, fmt, args);
834 linear_vasprintf_append(void *parent, char **str, const char *fmt, va_list args)
836 size_t existing_length;
838 existing_length = *str ? strlen(*str) : 0;
839 return linear_vasprintf_rewrite_tail(parent, str, &existing_length, fmt, args);
843 linear_asprintf_rewrite_tail(void *parent, char **str, size_t *start,
844 const char *fmt, ...)
849 success = linear_vasprintf_rewrite_tail(parent, str, start, fmt, args);
855 linear_vasprintf_rewrite_tail(void *parent, char **str, size_t *start,
856 const char *fmt, va_list args)
863 if (unlikely(*str == NULL)) {
864 *str = linear_vasprintf(parent, fmt, args);
865 *start = strlen(*str);
869 new_length = printf_length(fmt, args);
871 ptr = linear_realloc(parent, *str, *start + new_length + 1);
872 if (unlikely(ptr == NULL))
875 vsnprintf(ptr + *start, new_length + 1, fmt, args);
877 *start += new_length;
881 /* helper routine for strcat/strncat - n is the exact amount to copy */
883 linear_cat(void *parent, char **dest, const char *str, unsigned n)
886 unsigned existing_length;
887 assert(dest != NULL && *dest != NULL);
889 existing_length = strlen(*dest);
890 both = linear_realloc(parent, *dest, existing_length + n + 1);
891 if (unlikely(both == NULL))
894 memcpy(both + existing_length, str, n);
895 both[existing_length + n] = '\0';
902 linear_strcat(void *parent, char **dest, const char *str)
904 return linear_cat(parent, dest, str, strlen(str));