#include "alloc/Heap.h"
#include "alloc/HeapInternal.h"
#include "alloc/DdmHeap.h"
+#include "alloc/DlMalloc.h"
#include "alloc/HeapSource.h"
#define DEFAULT_HEAP_ID 1
((u1)((((kind) & 0x7) << 3) | ((solidity) & 0x7)))
struct HeapChunkContext {
+ void* startOfNextMemoryChunk;
u1 *buf;
u1 *p;
u1 *pieceLenField;
ctx->pieceLenField = NULL;
}
-static void heap_chunk_callback(const void *chunkptr, size_t chunklen,
- const void *userptr, size_t userlen, void *arg)
-{
- HeapChunkContext *ctx = (HeapChunkContext *)arg;
- u1 state;
-
- UNUSED_PARAMETER(userlen);
-
- assert((chunklen & (ALLOCATION_UNIT_SIZE-1)) == 0);
-
+static void append_chunk(HeapChunkContext *ctx, u1 state, void* ptr, size_t length) {
/* Make sure there's enough room left in the buffer.
* We need to use two bytes for every fractional 256
- * allocation units used by the chunk.
+ * allocation units used by the chunk and 17 bytes for
+ * any header.
*/
{
- size_t needed = (((chunklen/ALLOCATION_UNIT_SIZE + 255) / 256) * 2);
+ size_t needed = (((length/ALLOCATION_UNIT_SIZE + 255) / 256) * 2) + 17;
size_t bytesLeft = ctx->bufLen - (size_t)(ctx->p - ctx->buf);
if (bytesLeft < needed) {
flush_hpsg_chunk(ctx);
}
-
bytesLeft = ctx->bufLen - (size_t)(ctx->p - ctx->buf);
if (bytesLeft < needed) {
- ALOGW("chunk is too big to transmit (chunklen=%zd, %zd bytes)",
- chunklen, needed);
+ ALOGW("chunk is too big to transmit (length=%zd, %zd bytes)",
+ length, needed);
return;
}
}
-
-//TODO: notice when there's a gap and start a new heap, or at least a new range.
if (ctx->needHeader) {
/*
* Start a new HPSx chunk.
*ctx->p++ = 8;
/* [u4]: virtual address of segment start */
- set4BE(ctx->p, (uintptr_t)chunkptr); ctx->p += 4;
+ set4BE(ctx->p, (uintptr_t)ptr); ctx->p += 4;
/* [u4]: offset of this piece (relative to the virtual address) */
set4BE(ctx->p, 0); ctx->p += 4;
ctx->needHeader = false;
}
+ /* Write out the chunk description.
+ */
+ length /= ALLOCATION_UNIT_SIZE; // convert to allocation units
+ ctx->totalAllocationUnits += length;
+ while (length > 256) {
+ *ctx->p++ = state | HPSG_PARTIAL;
+ *ctx->p++ = 255; // length - 1
+ length -= 256;
+ }
+ *ctx->p++ = state;
+ *ctx->p++ = length - 1;
+}
+
+/*
+ * Called by dlmalloc_inspect_all. If used_bytes != 0 then start is
+ * the start of a malloc-ed piece of memory of size used_bytes. If
+ * start is 0 then start is the beginning of any free space not
+ * including dlmalloc's book keeping and end the start of the next
+ * dlmalloc chunk. Regions purely containing book keeping don't
+ * callback.
+ */
+static void heap_chunk_callback(void* start, void* end, size_t used_bytes,
+ void* arg)
+{
+ u1 state;
+ HeapChunkContext *ctx = (HeapChunkContext *)arg;
+ UNUSED_PARAMETER(end);
- /* Determine the type of this chunk.
+ if (used_bytes == 0) {
+ if (start == NULL) {
+ // Reset for start of new heap.
+ ctx->startOfNextMemoryChunk = NULL;
+ flush_hpsg_chunk(ctx);
+ }
+ // Only process in use memory so that free region information
+ // also includes dlmalloc book keeping.
+ return;
+ }
+
+ /* If we're looking at the native heap, we'll just return
+ * (SOLIDITY_HARD, KIND_NATIVE) for all allocated chunks
*/
- if (userptr == NULL) {
- /* It's a free chunk.
- */
- state = HPSG_STATE(SOLIDITY_FREE, 0);
- } else {
- const Object *obj = (const Object *)userptr;
- /* If we're looking at the native heap, we'll just return
- * (SOLIDITY_HARD, KIND_NATIVE) for all allocated chunks
- */
- bool native = ctx->type == CHUNK_TYPE("NHSG");
+ bool native = ctx->type == CHUNK_TYPE("NHSG");
+
+ if (ctx->startOfNextMemoryChunk != NULL) {
+ // Transmit any pending free memory. Native free memory of
+ // over kMaxFreeLen could be because of the use of mmaps, so
+ // don't report. If not free memory then start a new segment.
+ bool flush = true;
+ if (start > ctx->startOfNextMemoryChunk) {
+ const size_t kMaxFreeLen = 2 * SYSTEM_PAGE_SIZE;
+ void* freeStart = ctx->startOfNextMemoryChunk;
+ void* freeEnd = start;
+ size_t freeLen = (char*)freeEnd - (char*)freeStart;
+ if (!native || freeLen < kMaxFreeLen) {
+ append_chunk(ctx, HPSG_STATE(SOLIDITY_FREE, 0),
+ freeStart, freeLen);
+ flush = false;
+ }
+ }
+ if (flush) {
+ ctx->startOfNextMemoryChunk = NULL;
+ flush_hpsg_chunk(ctx);
+ }
+ }
+ const Object *obj = (const Object *)start;
- /* It's an allocated chunk. Figure out what it is.
- */
+ /* It's an allocated chunk. Figure out what it is.
+ */
//TODO: if ctx.merge, see if this chunk is different from the last chunk.
// If it's the same, we should combine them.
- if (!native && dvmIsValidObject(obj)) {
- ClassObject *clazz = obj->clazz;
- if (clazz == NULL) {
- /* The object was probably just created
- * but hasn't been initialized yet.
- */
- state = HPSG_STATE(SOLIDITY_HARD, KIND_OBJECT);
- } else if (dvmIsTheClassClass(clazz)) {
- state = HPSG_STATE(SOLIDITY_HARD, KIND_CLASS_OBJECT);
- } else if (IS_CLASS_FLAG_SET(clazz, CLASS_ISARRAY)) {
- if (IS_CLASS_FLAG_SET(clazz, CLASS_ISOBJECTARRAY)) {
+ if (!native && dvmIsValidObject(obj)) {
+ ClassObject *clazz = obj->clazz;
+ if (clazz == NULL) {
+ /* The object was probably just created
+ * but hasn't been initialized yet.
+ */
+ state = HPSG_STATE(SOLIDITY_HARD, KIND_OBJECT);
+ } else if (dvmIsTheClassClass(clazz)) {
+ state = HPSG_STATE(SOLIDITY_HARD, KIND_CLASS_OBJECT);
+ } else if (IS_CLASS_FLAG_SET(clazz, CLASS_ISARRAY)) {
+ if (IS_CLASS_FLAG_SET(clazz, CLASS_ISOBJECTARRAY)) {
+ state = HPSG_STATE(SOLIDITY_HARD, KIND_ARRAY_4);
+ } else {
+ switch (clazz->elementClass->primitiveType) {
+ case PRIM_BOOLEAN:
+ case PRIM_BYTE:
+ state = HPSG_STATE(SOLIDITY_HARD, KIND_ARRAY_1);
+ break;
+ case PRIM_CHAR:
+ case PRIM_SHORT:
+ state = HPSG_STATE(SOLIDITY_HARD, KIND_ARRAY_2);
+ break;
+ case PRIM_INT:
+ case PRIM_FLOAT:
state = HPSG_STATE(SOLIDITY_HARD, KIND_ARRAY_4);
- } else {
- switch (clazz->elementClass->primitiveType) {
- case PRIM_BOOLEAN:
- case PRIM_BYTE:
- state = HPSG_STATE(SOLIDITY_HARD, KIND_ARRAY_1);
- break;
- case PRIM_CHAR:
- case PRIM_SHORT:
- state = HPSG_STATE(SOLIDITY_HARD, KIND_ARRAY_2);
- break;
- case PRIM_INT:
- case PRIM_FLOAT:
- state = HPSG_STATE(SOLIDITY_HARD, KIND_ARRAY_4);
- break;
- case PRIM_DOUBLE:
- case PRIM_LONG:
- state = HPSG_STATE(SOLIDITY_HARD, KIND_ARRAY_8);
- break;
- default:
- assert(!"Unknown GC heap object type");
- state = HPSG_STATE(SOLIDITY_HARD, KIND_UNKNOWN);
- break;
- }
+ break;
+ case PRIM_DOUBLE:
+ case PRIM_LONG:
+ state = HPSG_STATE(SOLIDITY_HARD, KIND_ARRAY_8);
+ break;
+ default:
+ assert(!"Unknown GC heap object type");
+ state = HPSG_STATE(SOLIDITY_HARD, KIND_UNKNOWN);
+ break;
}
- } else {
- state = HPSG_STATE(SOLIDITY_HARD, KIND_OBJECT);
}
} else {
- obj = NULL; // it's not actually an object
- state = HPSG_STATE(SOLIDITY_HARD, KIND_NATIVE);
+ state = HPSG_STATE(SOLIDITY_HARD, KIND_OBJECT);
}
+ } else {
+ obj = NULL; // it's not actually an object
+ state = HPSG_STATE(SOLIDITY_HARD, KIND_NATIVE);
}
-
- /* Write out the chunk description.
- */
- chunklen /= ALLOCATION_UNIT_SIZE; // convert to allocation units
- ctx->totalAllocationUnits += chunklen;
- while (chunklen > 256) {
- *ctx->p++ = state | HPSG_PARTIAL;
- *ctx->p++ = 255; // length - 1
- chunklen -= 256;
- }
- *ctx->p++ = state;
- *ctx->p++ = chunklen - 1;
+ append_chunk(ctx, state, start, used_bytes + HEAP_SOURCE_CHUNK_OVERHEAD);
+ ctx->startOfNextMemoryChunk =
+ (char*)start + used_bytes + HEAP_SOURCE_CHUNK_OVERHEAD;
}
enum HpsgWhen {
*/
#define HPSx_CHUNK_SIZE (16384 - 16)
-extern "C" void dlmalloc_walk_heap(void(*)(const void*, size_t, const void*, size_t, void*),void*);
-
static void walkHeap(bool merge, bool native)
{
HeapChunkContext ctx;
ctx.p = ctx.buf;
ctx.needHeader = true;
if (native) {
- dlmalloc_walk_heap(heap_chunk_callback, (void *)&ctx);
+ dlmalloc_inspect_all(heap_chunk_callback, (void*)&ctx);
} else {
dvmHeapSourceWalk(heap_chunk_callback, (void *)&ctx);
}
* limitations under the License.
*/
-#include <cutils/mspace.h>
#include <stdint.h>
#include <sys/mman.h>
#include <errno.h>
#define SIZE_MAX UINT_MAX // TODO: get SIZE_MAX from stdint.h
#include "Dalvik.h"
+#include "alloc/DlMalloc.h"
#include "alloc/Heap.h"
#include "alloc/HeapInternal.h"
#include "alloc/HeapSource.h"
#include "alloc/HeapBitmap.h"
#include "alloc/HeapBitmapInlines.h"
-// TODO: find a real header file for these.
-extern "C" int dlmalloc_trim(size_t);
-extern "C" void dlmalloc_walk_free_pages(void(*)(void*, void*, void*), void*);
-
static void snapIdealFootprint();
static void setIdealFootprint(size_t max);
static size_t getMaximumSize(const HeapSource *hs);
* The highest address of this heap, exclusive.
*/
char *limit;
+
+ /*
+ * If the heap has an mspace, the current high water mark in
+ * allocations requested via dvmHeapSourceMorecore.
+ */
+ char *brk;
};
struct HeapSource {
if (isSoftLimited(hs)) {
return hs->softLimit;
} else {
- return mspace_max_allowed_footprint(hs2heap(hs)->msp);
+ return mspace_footprint_limit(hs2heap(hs)->msp);
}
}
{
assert(heap->bytesAllocated < mspace_footprint(heap->msp));
- heap->bytesAllocated += mspace_usable_size(heap->msp, ptr) +
+ heap->bytesAllocated += mspace_usable_size(ptr) +
HEAP_SOURCE_CHUNK_OVERHEAD;
heap->objectsAllocated++;
HeapSource* hs = gDvm.gcHeap->heapSource;
static void countFree(Heap *heap, const void *ptr, size_t *numBytes)
{
- size_t delta = mspace_usable_size(heap->msp, ptr) + HEAP_SOURCE_CHUNK_OVERHEAD;
+ size_t delta = mspace_usable_size(ptr) + HEAP_SOURCE_CHUNK_OVERHEAD;
assert(delta > 0);
if (delta < heap->bytesAllocated) {
heap->bytesAllocated -= delta;
static HeapSource *gHs = NULL;
-static mspace createMspace(void *base, size_t startSize, size_t maximumSize)
+static mspace createMspace(void* begin, size_t morecoreStart, size_t startingSize)
{
- /* Create an unlocked dlmalloc mspace to use as
- * a heap source.
- *
- * We start off reserving startSize / 2 bytes but
- * letting the heap grow to startSize. This saves
- * memory in the case where a process uses even less
- * than the starting size.
- */
- LOGV_HEAP("Creating VM heap of size %zu", startSize);
+ // Clear errno to allow strerror on error.
errno = 0;
-
- mspace msp = create_contiguous_mspace_with_base(startSize/2,
- maximumSize, /*locked=*/false, base);
+ // Allow access to inital pages that will hold mspace.
+ mprotect(begin, morecoreStart, PROT_READ | PROT_WRITE);
+ // Create mspace using our backing storage starting at begin and with a footprint of
+ // morecoreStart. Don't use an internal dlmalloc lock. When morecoreStart bytes of memory are
+ // exhausted morecore will be called.
+ mspace msp = create_mspace_with_base(begin, morecoreStart, false /*locked*/);
if (msp != NULL) {
- /* Don't let the heap grow past the starting size without
- * our intervention.
- */
- mspace_set_max_allowed_footprint(msp, startSize);
+ // Do not allow morecore requests to succeed beyond the starting size of the heap.
+ mspace_set_footprint_limit(msp, startingSize);
} else {
- /* There's no guarantee that errno has meaning when the call
- * fails, but it often does.
- */
- LOGE_HEAP("Can't create VM heap of size (%zu,%zu): %s",
- startSize/2, maximumSize, strerror(errno));
+ ALOGE("create_mspace_with_base failed %s", strerror(errno));
}
-
return msp;
}
/*
+ * Service request from DlMalloc to increase heap size.
+ */
+void* dvmHeapSourceMorecore(void* mspace, intptr_t increment)
+{
+ Heap* heap = NULL;
+ for (size_t i = 0; i < gHs->numHeaps; i++) {
+ if (gHs->heaps[i].msp == mspace) {
+ heap = &gHs->heaps[i];
+ break;
+ }
+ }
+ if (heap == NULL) {
+ ALOGE("Failed to find heap for mspace %p", mspace);
+ dvmAbort();
+ }
+ char* original_brk = heap->brk;
+ if (increment != 0) {
+ char* new_brk = original_brk + increment;
+ if (increment > 0) {
+ // Should never be asked to increase the allocation beyond the capacity of the space.
+ // Enforced by mspace_set_footprint_limit.
+ assert(new_brk <= heap->limit);
+ mprotect(original_brk, increment, PROT_READ | PROT_WRITE);
+ } else {
+ // Should never be asked for negative footprint (ie before base).
+ assert(original_brk + increment > heap->base);
+ // Advise we don't need the pages and protect them.
+ size_t size = -increment;
+ madvise(new_brk, size, MADV_DONTNEED);
+ mprotect(new_brk, size, PROT_NONE);
+ }
+ // Update brk.
+ heap->brk = new_brk;
+ }
+ return original_brk;
+}
+
+const size_t kInitialMorecoreStart = SYSTEM_PAGE_SIZE;
+/*
* Add the initial heap. Returns false if the initial heap was
* already added to the heap source.
*/
hs->heaps[0].maximumSize = maximumSize;
hs->heaps[0].concurrentStartBytes = SIZE_MAX;
hs->heaps[0].base = hs->heapBase;
- hs->heaps[0].limit = hs->heapBase + hs->heaps[0].maximumSize;
+ hs->heaps[0].limit = hs->heapBase + maximumSize;
+ hs->heaps[0].brk = hs->heapBase + kInitialMorecoreStart;
hs->numHeaps = 1;
return true;
}
* Heap storage comes from a common virtual memory reservation.
* The new heap will start on the page after the old heap.
*/
- void *sbrk0 = contiguous_mspace_sbrk0(hs->heaps[0].msp);
- char *base = (char *)ALIGN_UP_TO_PAGE_SIZE(sbrk0);
+ char *base = hs->heaps[0].brk;
size_t overhead = base - hs->heaps[0].base;
assert(((size_t)hs->heaps[0].base & (SYSTEM_PAGE_SIZE - 1)) == 0);
overhead, hs->maximumSize);
return false;
}
-
+ size_t morecoreStart = SYSTEM_PAGE_SIZE;
heap.maximumSize = hs->growthLimit - overhead;
heap.concurrentStartBytes = HEAP_MIN_FREE - CONCURRENT_START;
heap.base = base;
heap.limit = heap.base + heap.maximumSize;
- heap.msp = createMspace(base, HEAP_MIN_FREE, hs->maximumSize - overhead);
+ heap.brk = heap.base + morecoreStart;
+ heap.msp = createMspace(base, morecoreStart, HEAP_MIN_FREE);
if (heap.msp == NULL) {
return false;
}
*/
hs->heaps[0].maximumSize = overhead;
hs->heaps[0].limit = base;
- mspace msp = hs->heaps[0].msp;
- mspace_set_max_allowed_footprint(msp, mspace_footprint(msp));
+ mspace_set_footprint_limit(hs->heaps[0].msp, overhead);
/* Put the new heap in the list, at heaps[0].
* Shift existing heaps down.
/* Create an unlocked dlmalloc mspace to use as
* a heap source.
*/
- msp = createMspace(base, startSize, maximumSize);
+ msp = createMspace(base, kInitialMorecoreStart, startSize);
if (msp == NULL) {
goto fail;
}
value = mspace_footprint(heap->msp);
break;
case HS_ALLOWED_FOOTPRINT:
- value = mspace_max_allowed_footprint(heap->msp);
+ value = mspace_footprint_limit(heap->msp);
break;
case HS_BYTES_ALLOCATED:
value = heap->bytesAllocated;
*/
size_t max = heap->maximumSize;
- mspace_set_max_allowed_footprint(heap->msp, max);
+ mspace_set_footprint_limit(heap->msp, max);
void* ptr = dvmHeapSourceAlloc(n);
/* Shrink back down as small as possible. Our caller may
* readjust max_allowed to a more appropriate value.
*/
- mspace_set_max_allowed_footprint(heap->msp,
- mspace_footprint(heap->msp));
+ mspace_set_footprint_limit(heap->msp,
+ mspace_footprint(heap->msp));
return ptr;
}
// mspace_free, but on the other heaps we only do some
// accounting.
if (heap == gHs->heaps) {
- // mspace_merge_objects takes two allocated objects, and
- // if the second immediately follows the first, will merge
- // them, returning a larger object occupying the same
- // memory. This is a local operation, and doesn't require
- // dlmalloc to manipulate any freelists. It's pretty
- // inexpensive compared to free().
-
- // ptrs is an array of objects all in memory order, and if
- // client code has been allocating lots of short-lived
- // objects, this is likely to contain runs of objects all
- // now garbage, and thus highly amenable to this optimization.
-
- // Unroll the 0th iteration around the loop below,
- // countFree ptrs[0] and initializing merged.
- assert(ptrs[0] != NULL);
- assert(ptr2heap(gHs, ptrs[0]) == heap);
- countFree(heap, ptrs[0], &numBytes);
- void *merged = ptrs[0];
- for (size_t i = 1; i < numPtrs; i++) {
- assert(merged != NULL);
+ // Count freed objects.
+ for (size_t i = 0; i < numPtrs; i++) {
assert(ptrs[i] != NULL);
- assert((intptr_t)merged < (intptr_t)ptrs[i]);
assert(ptr2heap(gHs, ptrs[i]) == heap);
countFree(heap, ptrs[i], &numBytes);
- // Try to merge. If it works, merged now includes the
- // memory of ptrs[i]. If it doesn't, free merged, and
- // see if ptrs[i] starts a new run of adjacent
- // objects to merge.
- if (mspace_merge_objects(msp, merged, ptrs[i]) == NULL) {
- mspace_free(msp, merged);
- merged = ptrs[i];
- }
}
- assert(merged != NULL);
- mspace_free(msp, merged);
+ // Bulk free ptrs.
+ mspace_bulk_free(msp, ptrs, numPtrs);
} else {
// This is not an 'active heap'. Only do the accounting.
for (size_t i = 0; i < numPtrs; i++) {
Heap* heap = ptr2heap(gHs, ptr);
if (heap != NULL) {
- return mspace_usable_size(heap->msp, ptr);
+ return mspace_usable_size(ptr);
}
return 0;
}
if (softLimit < currentHeapSize) {
/* Don't let the heap grow any more, and impose a soft limit.
*/
- mspace_set_max_allowed_footprint(msp, currentHeapSize);
+ mspace_set_footprint_limit(msp, currentHeapSize);
hs->softLimit = softLimit;
} else {
/* Let the heap grow to the requested max, and remove any
* soft limit, if set.
*/
- mspace_set_max_allowed_footprint(msp, softLimit);
+ mspace_set_footprint_limit(msp, softLimit);
hs->softLimit = SIZE_MAX;
}
}
* Return free pages to the system.
* TODO: move this somewhere else, especially the native heap part.
*/
-static void releasePagesInRange(void *start, void *end, void *nbytes)
+static void releasePagesInRange(void* start, void* end, size_t used_bytes,
+ void* releasedBytes)
{
- /* Linux requires that the madvise() start address is page-aligned.
- * We also align the end address.
- */
- start = (void *)ALIGN_UP_TO_PAGE_SIZE(start);
- end = (void *)((size_t)end & ~(SYSTEM_PAGE_SIZE - 1));
- if (start < end) {
- size_t length = (char *)end - (char *)start;
- madvise(start, length, MADV_DONTNEED);
- *(size_t *)nbytes += length;
+ if (used_bytes == 0) {
+ /*
+ * We have a range of memory we can try to madvise()
+ * back. Linux requires that the madvise() start address is
+ * page-aligned. We also align the end address.
+ */
+ start = (void *)ALIGN_UP_TO_PAGE_SIZE(start);
+ end = (void *)((size_t)end & ~(SYSTEM_PAGE_SIZE - 1));
+ if (end > start) {
+ size_t length = (char *)end - (char *)start;
+ madvise(start, length, MADV_DONTNEED);
+ *(size_t *)releasedBytes += length;
+ }
}
}
for (size_t i = 0; i < hs->numHeaps; i++) {
Heap *heap = &hs->heaps[i];
- /* Return the wilderness chunk to the system.
- */
+ /* Return the wilderness chunk to the system. */
mspace_trim(heap->msp, 0);
- /* Return any whole free pages to the system.
- */
- mspace_walk_free_pages(heap->msp, releasePagesInRange, &heapBytes);
+ /* Return any whole free pages to the system. */
+ mspace_inspect_all(heap->msp, releasePagesInRange, &heapBytes);
}
- /* Same for the native heap.
- */
+ /* Same for the native heap. */
dlmalloc_trim(0);
size_t nativeBytes = 0;
- dlmalloc_walk_free_pages(releasePagesInRange, &nativeBytes);
+ dlmalloc_inspect_all(releasePagesInRange, &nativeBytes);
LOGD_HEAP("madvised %zd (GC) + %zd (native) = %zd total bytes",
heapBytes, nativeBytes, heapBytes + nativeBytes);
* Walks over the heap source and passes every allocated and
* free chunk to the callback.
*/
-void dvmHeapSourceWalk(void(*callback)(const void *chunkptr, size_t chunklen,
- const void *userptr, size_t userlen,
- void *arg),
+void dvmHeapSourceWalk(void(*callback)(void* start, void* end,
+ size_t used_bytes, void* arg),
void *arg)
{
HS_BOILERPLATE();
//TODO: do this in address order
HeapSource *hs = gHs;
for (size_t i = hs->numHeaps; i > 0; --i) {
- mspace_walk_heap(hs->heaps[i-1].msp, callback, arg);
+ mspace_inspect_all(hs->heaps[i-1].msp, callback, arg);
+ callback(NULL, NULL, 0, arg); // Indicate end of a heap.
}
}