Export malloc_iterate and friends

Export malloc_iterate, malloc_enable, and malloc_disable to be used by
libmemunreachable.

Bug: 27208635
(cherry picked from commit 869691c6f7691dd5bf343617aa185c32e0d3d160)

Change-Id: Ifa44421a84ac3be6a57fac28c4da501febe88c28

diff --git a/libc/bionic/jemalloc.h b/libc/bionic/jemalloc.h
index 98ea0ee..fceb323 100644 (file)
--- a/libc/bionic/jemalloc.h
+++ b/libc/bionic/jemalloc.h
@@ -26,6 +26,10 @@
 __BEGIN_DECLS
 
 struct mallinfo je_mallinfo();
+int je_iterate(uintptr_t, size_t, void (*)(uintptr_t, size_t, void*), void*);
+void je_malloc_disable();
+void je_malloc_enable();
+int je_mallctl(const char *name, void *oldp, size_t *oldlenp, void *newp, size_t newlen);
 void* je_memalign_round_up_boundary(size_t, size_t);
 void* je_pvalloc(size_t);
 

diff --git a/libc/bionic/malloc_common.cpp b/libc/bionic/malloc_common.cpp
index 56f1c10..b991b9a 100644 (file)
--- a/libc/bionic/malloc_common.cpp
+++ b/libc/bionic/malloc_common.cpp
@@ -40,6 +40,8 @@
 //   free_malloc_leak_info: Frees the data allocated by the call to
 //                          get_malloc_leak_info.
 
+#include <pthread.h>
+
 #include <private/bionic_config.h>
 #include <private/bionic_globals.h>
 #include <private/bionic_malloc_dispatch.h>
@@ -63,6 +65,9 @@ static constexpr MallocDispatch __libc_malloc_default_dispatch
 #if defined(HAVE_DEPRECATED_MALLOC_FUNCS)
     Malloc(valloc),
 #endif
+    Malloc(iterate),
+    Malloc(malloc_disable),
+    Malloc(malloc_enable),
   };
 
 // In a VM process, this is set to 1 after fork()ing out of zygote.
@@ -159,7 +164,6 @@ extern "C" void* valloc(size_t bytes) {
 #if !defined(LIBC_STATIC)
 
 #include <dlfcn.h>
-#include <pthread.h>
 #include <stdio.h>
 #include <stdlib.h>
 
@@ -215,6 +219,7 @@ extern "C" void free_malloc_leak_info(uint8_t* info) {
   }
   g_debug_free_malloc_leak_info_func(info);
 }
+
 // =============================================================================
 
 template<typename FunctionType>
@@ -262,6 +267,18 @@ static bool InitMalloc(void* malloc_impl_handler, MallocDispatch* table, const c
                                          prefix, "realloc")) {
     return false;
   }
+  if (!InitMallocFunction<MallocIterate>(malloc_impl_handler, &table->iterate,
+                                         prefix, "iterate")) {
+    return false;
+  }
+  if (!InitMallocFunction<MallocMallocDisable>(malloc_impl_handler, &table->malloc_disable,
+                                         prefix, "malloc_disable")) {
+    return false;
+  }
+  if (!InitMallocFunction<MallocMallocEnable>(malloc_impl_handler, &table->malloc_enable,
+                                         prefix, "malloc_enable")) {
+    return false;
+  }
 #if defined(HAVE_DEPRECATED_MALLOC_FUNCS)
   if (!InitMallocFunction<MallocPvalloc>(malloc_impl_handler, &table->pvalloc,
                                          prefix, "pvalloc")) {
@@ -385,3 +402,37 @@ __LIBC_HIDDEN__ void __libc_init_malloc(libc_globals* globals) {
   malloc_init_impl(globals);
 }
 #endif  // !LIBC_STATIC
+
+// =============================================================================
+// Exported for use by libmemunreachable.
+// =============================================================================
+
+// Calls callback for every allocation in the anonymous heap mapping
+// [base, base+size).  Must be called between malloc_disable and malloc_enable.
+extern "C" int malloc_iterate(uintptr_t base, size_t size,
+    void (*callback)(uintptr_t base, size_t size, void* arg), void* arg) {
+  auto _iterate = __libc_globals->malloc_dispatch.iterate;
+  if (__predict_false(_iterate != nullptr)) {
+    return _iterate(base, size, callback, arg);
+  }
+  return Malloc(iterate)(base, size, callback, arg);
+}
+
+// Disable calls to malloc so malloc_iterate gets a consistent view of
+// allocated memory.
+extern "C" void malloc_disable() {
+  auto _malloc_disable = __libc_globals->malloc_dispatch.malloc_disable;
+  if (__predict_false(_malloc_disable != nullptr)) {
+    return _malloc_disable();
+  }
+  return Malloc(malloc_disable)();
+}
+
+// Re-enable calls to malloc after a previous call to malloc_disable.
+extern "C" void malloc_enable() {
+  auto _malloc_enable = __libc_globals->malloc_dispatch.malloc_enable;
+  if (__predict_false(_malloc_enable != nullptr)) {
+    return _malloc_enable();
+  }
+  return Malloc(malloc_enable)();
+}

diff --git a/libc/libc.arm.brillo.map b/libc/libc.arm.brillo.map
index 80a88bd..1da85ea 100644 (file)
--- a/libc/libc.arm.brillo.map
+++ b/libc/libc.arm.brillo.map
@@ -627,6 +627,9 @@ LIBC {
     madvise;
     mallinfo;
     malloc;
+    malloc_disable;
+    malloc_enable;
+    malloc_iterate;
     malloc_info;
     malloc_usable_size;
     mbrlen;

diff --git a/libc/libc.arm.map b/libc/libc.arm.map
index 18e51d4..bad3fec 100644 (file)
--- a/libc/libc.arm.map
+++ b/libc/libc.arm.map
@@ -627,6 +627,9 @@ LIBC {
     madvise;
     mallinfo;
     malloc;
+    malloc_disable;
+    malloc_enable;
+    malloc_iterate;
     malloc_info;
     malloc_usable_size;
     mbrlen;

diff --git a/libc/libc.arm64.map b/libc/libc.arm64.map
index e1ccd4e..c7aca4f 100644 (file)
--- a/libc/libc.arm64.map
+++ b/libc/libc.arm64.map
@@ -552,6 +552,9 @@ LIBC {
     madvise;
     mallinfo;
     malloc;
+    malloc_disable;
+    malloc_enable;
+    malloc_iterate;
     malloc_info;
     malloc_usable_size;
     mbrlen;

diff --git a/libc/libc.map.txt b/libc/libc.map.txt
index b87a5e3..1f4da93 100644 (file)
--- a/libc/libc.map.txt
+++ b/libc/libc.map.txt
@@ -630,6 +630,9 @@ LIBC {
     madvise;
     mallinfo;
     malloc;
+    malloc_disable;
+    malloc_enable;
+    malloc_iterate;
     malloc_info;
     malloc_usable_size;
     mbrlen;

diff --git a/libc/libc.mips.brillo.map b/libc/libc.mips.brillo.map
index 57dfd50..3562ebf 100644 (file)
--- a/libc/libc.mips.brillo.map
+++ b/libc/libc.mips.brillo.map
@@ -625,6 +625,9 @@ LIBC {
     madvise;
     mallinfo;
     malloc;
+    malloc_disable;
+    malloc_enable;
+    malloc_iterate;
     malloc_info;
     malloc_usable_size;
     mbrlen;

diff --git a/libc/libc.mips.map b/libc/libc.mips.map
index ed72fd4..1f7b9aa 100644 (file)
--- a/libc/libc.mips.map
+++ b/libc/libc.mips.map
@@ -625,6 +625,9 @@ LIBC {
     madvise;
     mallinfo;
     malloc;
+    malloc_disable;
+    malloc_enable;
+    malloc_iterate;
     malloc_info;
     malloc_usable_size;
     mbrlen;

diff --git a/libc/libc.mips64.map b/libc/libc.mips64.map
index e1ccd4e..c7aca4f 100644 (file)
--- a/libc/libc.mips64.map
+++ b/libc/libc.mips64.map
@@ -552,6 +552,9 @@ LIBC {
     madvise;
     mallinfo;
     malloc;
+    malloc_disable;
+    malloc_enable;
+    malloc_iterate;
     malloc_info;
     malloc_usable_size;
     mbrlen;

diff --git a/libc/libc.x86.brillo.map b/libc/libc.x86.brillo.map
index 3dcf1ea..c57a072 100644 (file)
--- a/libc/libc.x86.brillo.map
+++ b/libc/libc.x86.brillo.map
@@ -624,6 +624,9 @@ LIBC {
     madvise;
     mallinfo;
     malloc;
+    malloc_disable;
+    malloc_enable;
+    malloc_iterate;
     malloc_info;
     malloc_usable_size;
     mbrlen;

diff --git a/libc/libc.x86.map b/libc/libc.x86.map
index 1e44e2a..56d6609 100644 (file)
--- a/libc/libc.x86.map
+++ b/libc/libc.x86.map
@@ -624,6 +624,9 @@ LIBC {
     madvise;
     mallinfo;
     malloc;
+    malloc_disable;
+    malloc_enable;
+    malloc_iterate;
     malloc_info;
     malloc_usable_size;
     mbrlen;

diff --git a/libc/libc.x86_64.map b/libc/libc.x86_64.map
index e1ccd4e..c7aca4f 100644 (file)
--- a/libc/libc.x86_64.map
+++ b/libc/libc.x86_64.map
@@ -552,6 +552,9 @@ LIBC {
     madvise;
     mallinfo;
     malloc;
+    malloc_disable;
+    malloc_enable;
+    malloc_iterate;
     malloc_info;
     malloc_usable_size;
     mbrlen;

diff --git a/libc/malloc_debug/TrackData.cpp b/libc/malloc_debug/TrackData.cpp
index e5260a9..c9828d0 100644 (file)
--- a/libc/malloc_debug/TrackData.cpp
+++ b/libc/malloc_debug/TrackData.cpp
@@ -44,7 +44,7 @@
 #include "malloc_debug.h"
 #include "TrackData.h"
 
-void TrackData::GetList(std::vector<Header*>* list) {
+void TrackData::GetList(std::vector<const Header*>* list) {
   ScopedDisableDebugCalls disable;
 
   for (const auto& header : headers_) {
@@ -52,13 +52,13 @@ void TrackData::GetList(std::vector<Header*>* list) {
   }
 
   // Sort by the size of the allocation.
-  std::sort(list->begin(), list->end(), [](Header* a, Header* b) {
+  std::sort(list->begin(), list->end(), [](const Header* a, const Header* b) {
     if (a->size == b->size) return a < b;
     return a->size > b->size;
   });
 }
 
-void TrackData::Add(Header* header, bool backtrace_found) {
+void TrackData::Add(const Header* header, bool backtrace_found) {
   ScopedDisableDebugCalls disable;
 
   pthread_mutex_lock(&mutex_);
@@ -69,7 +69,7 @@ void TrackData::Add(Header* header, bool backtrace_found) {
   pthread_mutex_unlock(&mutex_);
 }
 
-void TrackData::Remove(Header* header, bool backtrace_found) {
+void TrackData::Remove(const Header* header, bool backtrace_found) {
   ScopedDisableDebugCalls disable;
 
   pthread_mutex_lock(&mutex_);
@@ -80,10 +80,19 @@ void TrackData::Remove(Header* header, bool backtrace_found) {
   pthread_mutex_unlock(&mutex_);
 }
 
+bool TrackData::Contains(const Header* header) {
+  ScopedDisableDebugCalls disable;
+
+  pthread_mutex_lock(&mutex_);
+  bool found = headers_.count(header);
+  pthread_mutex_unlock(&mutex_);
+  return found;
+}
+
 void TrackData::DisplayLeaks(DebugData& debug) {
   ScopedDisableDebugCalls disable;
 
-  std::vector<Header*> list;
+  std::vector<const Header*> list;
   GetList(&list);
 
   size_t track_count = 0;
@@ -117,7 +126,7 @@ void TrackData::GetInfo(DebugData& debug, uint8_t** info, size_t* overall_size,
   }
   *overall_size = *info_size * total_backtrace_allocs_;
 
-  std::vector<Header*> list;
+  std::vector<const Header*> list;
   GetList(&list);
 
   uint8_t* data = *info;

diff --git a/libc/malloc_debug/TrackData.h b/libc/malloc_debug/TrackData.h
index dcf0ede..1234316 100644 (file)
--- a/libc/malloc_debug/TrackData.h
+++ b/libc/malloc_debug/TrackData.h
@@ -47,11 +47,13 @@ class TrackData {
   TrackData() = default;
   virtual ~TrackData() = default;
 
-  void GetList(std::vector<Header*>* list);
+  void GetList(std::vector<const Header*>* list);
 
-  void Add(Header* header, bool backtrace_found);
+  void Add(const Header* header, bool backtrace_found);
 
-  void Remove(Header* header, bool backtrace_found);
+  void Remove(const Header* header, bool backtrace_found);
+
+  bool Contains(const Header *header);
 
   void GetInfo(DebugData& debug, uint8_t** info, size_t* overall_size,
                size_t* info_size, size_t* total_memory, size_t* backtrace_size);
@@ -64,7 +66,7 @@ class TrackData {
 
  private:
   pthread_mutex_t mutex_ = PTHREAD_MUTEX_INITIALIZER;
-  std::unordered_set<Header*> headers_;
+  std::unordered_set<const Header*> headers_;
   size_t total_backtrace_allocs_ = 0;
 
   DISALLOW_COPY_AND_ASSIGN(TrackData);

diff --git a/libc/malloc_debug/exported32.map b/libc/malloc_debug/exported32.map
index aef8818..8f994f1 100644 (file)
--- a/libc/malloc_debug/exported32.map
+++ b/libc/malloc_debug/exported32.map
@@ -6,8 +6,11 @@ LIBC_MALLOC_DEBUG {
     debug_free_malloc_leak_info;
     debug_get_malloc_leak_info;
     debug_initialize;
+    debug_iterate;
     debug_mallinfo;
     debug_malloc;
+    debug_malloc_disable;
+    debug_malloc_enable;
     debug_malloc_usable_size;
     debug_memalign;
     debug_posix_memalign;

diff --git a/libc/malloc_debug/exported64.map b/libc/malloc_debug/exported64.map
index 70fd581..8c47449 100644 (file)
--- a/libc/malloc_debug/exported64.map
+++ b/libc/malloc_debug/exported64.map
@@ -6,8 +6,11 @@ LIBC_MALLOC_DEBUG {
     debug_free_malloc_leak_info;
     debug_get_malloc_leak_info;
     debug_initialize;
+    debug_iterate;
     debug_mallinfo;
     debug_malloc;
+    debug_malloc_disable;
+    debug_malloc_enable;
     debug_malloc_usable_size;
     debug_memalign;
     debug_posix_memalign;

diff --git a/libc/malloc_debug/malloc_debug.cpp b/libc/malloc_debug/malloc_debug.cpp
index dcc6048..5b6e415 100644 (file)
--- a/libc/malloc_debug/malloc_debug.cpp
+++ b/libc/malloc_debug/malloc_debug.cpp
@@ -76,6 +76,10 @@ void* debug_realloc(void* pointer, size_t bytes);
 void* debug_calloc(size_t nmemb, size_t bytes);
 struct mallinfo debug_mallinfo();
 int debug_posix_memalign(void** memptr, size_t alignment, size_t size);
+int debug_iterate(uintptr_t base, size_t size,
+    void (*callback)(uintptr_t base, size_t size, void* arg), void* arg);
+void debug_malloc_disable();
+void debug_malloc_enable();
 
 #if defined(HAVE_DEPRECATED_MALLOC_FUNCS)
 void* debug_pvalloc(size_t bytes);
@@ -569,6 +573,49 @@ int debug_posix_memalign(void** memptr, size_t alignment, size_t size) {
   return (*memptr != nullptr) ? 0 : ENOMEM;
 }
 
+int debug_iterate(uintptr_t base, size_t size,
+    void (*callback)(uintptr_t base, size_t size, void* arg), void* arg) {
+  // Can't allocate, malloc is disabled
+  // Manual capture of the arguments to pass to the lambda below as void* arg
+  struct iterate_ctx {
+    decltype(callback) callback;
+    decltype(arg) arg;
+  } ctx = { callback, arg };
+
+  return g_dispatch->iterate(base, size,
+      [](uintptr_t base, size_t size, void* arg) {
+        const iterate_ctx* ctx = reinterpret_cast<iterate_ctx*>(arg);
+        const void* pointer = reinterpret_cast<void*>(base);
+        if (g_debug->need_header()) {
+          const Header* header = reinterpret_cast<const Header*>(pointer);
+          if (g_debug->config().options & TRACK_ALLOCS) {
+            if (g_debug->track->Contains(header)) {
+              // Return just the body of the allocation if we're sure the header exists
+              ctx->callback(reinterpret_cast<uintptr_t>(g_debug->GetPointer(header)),
+                  header->real_size(), ctx->arg);
+              return;
+            }
+          }
+        }
+        // Fall back to returning the whole allocation
+        ctx->callback(base, size, ctx->arg);
+      }, &ctx);
+}
+
+void debug_malloc_disable() {
+  g_dispatch->malloc_disable();
+  if (g_debug->track) {
+    g_debug->track->PrepareFork();
+  }
+}
+
+void debug_malloc_enable() {
+  if (g_debug->track) {
+    g_debug->track->PostForkParent();
+  }
+  g_dispatch->malloc_enable();
+}
+
 #if defined(HAVE_DEPRECATED_MALLOC_FUNCS)
 void* debug_pvalloc(size_t bytes) {
   if (DebugCallsDisabled()) {

diff --git a/libc/malloc_debug/tests/malloc_debug_unit_tests.cpp b/libc/malloc_debug/tests/malloc_debug_unit_tests.cpp
index b316e8a..28729e8 100644 (file)
--- a/libc/malloc_debug/tests/malloc_debug_unit_tests.cpp
+++ b/libc/malloc_debug/tests/malloc_debug_unit_tests.cpp
@@ -121,6 +121,9 @@ MallocDispatch MallocDebugTest::dispatch = {
 #if defined(HAVE_DEPRECATED_MALLOC_FUNCS)
   nullptr,
 #endif
+  nullptr,
+  nullptr,
+  nullptr,
 };
 
 void VerifyAllocCalls() {

diff --git a/libc/private/bionic_malloc_dispatch.h b/libc/private/bionic_malloc_dispatch.h
index 5dcd37a..02a092f 100644 (file)
--- a/libc/private/bionic_malloc_dispatch.h
+++ b/libc/private/bionic_malloc_dispatch.h
@@ -30,6 +30,7 @@
 #define _PRIVATE_BIONIC_MALLOC_DISPATCH_H
 
 #include <stddef.h>
+#include <stdint.h>
 #include <private/bionic_config.h>
 
 // Entry in malloc dispatch table.
@@ -41,6 +42,10 @@ typedef size_t (*MallocMallocUsableSize)(const void*);
 typedef void* (*MallocMemalign)(size_t, size_t);
 typedef int (*MallocPosixMemalign)(void**, size_t, size_t);
 typedef void* (*MallocRealloc)(void*, size_t);
+typedef int (*MallocIterate)(uintptr_t, size_t, void (*)(uintptr_t, size_t, void*), void*);
+typedef void (*MallocMallocDisable)();
+typedef void (*MallocMallocEnable)();
+
 #if defined(HAVE_DEPRECATED_MALLOC_FUNCS)
 typedef void* (*MallocPvalloc)(size_t);
 typedef void* (*MallocValloc)(size_t);
@@ -61,6 +66,9 @@ struct MallocDispatch {
 #if defined(HAVE_DEPRECATED_MALLOC_FUNCS)
   MallocValloc valloc;
 #endif
+  MallocIterate iterate;
+  MallocMallocDisable malloc_disable;
+  MallocMallocEnable malloc_enable;
 } __attribute__((aligned(32)));
 
 #endif