From b89fb12b587cba78a3c18caedd9378aba10870fd Mon Sep 17 00:00:00 2001
From: Kostya Serebryany <kcc@google.com>
Date: Wed, 30 Nov 2016 22:39:35 +0000
Subject: [PATCH] [libFuzzer] extend -rss_limit_mb to crash instantly on a
 single malloc that exceeds the limit

git-svn-id: https://llvm.org/svn/llvm-project/llvm/trunk@288281 91177308-0d34-0410-b5e6-96231b3b80d8
---
 lib/Fuzzer/FuzzerInternal.h                        |  2 ++
 lib/Fuzzer/FuzzerLoop.cpp                          | 16 +++++++++++++
 lib/Fuzzer/test/CMakeLists.txt                     |  1 +
 .../test/OutOfMemorySingleLargeMallocTest.cpp      | 28 ++++++++++++++++++++++
 lib/Fuzzer/test/fuzzer-oom-with-profile.test       |  4 ++--
 lib/Fuzzer/test/fuzzer-oom.test                    |  8 +++++--
 6 files changed, 55 insertions(+), 4 deletions(-)
 create mode 100644 lib/Fuzzer/test/OutOfMemorySingleLargeMallocTest.cpp

diff --git a/lib/Fuzzer/FuzzerInternal.h b/lib/Fuzzer/FuzzerInternal.h
index 02e806c65a3..ad42d7f6b73 100644
--- a/lib/Fuzzer/FuzzerInternal.h
+++ b/lib/Fuzzer/FuzzerInternal.h
@@ -105,6 +105,8 @@ public:
   void TryDetectingAMemoryLeak(const uint8_t *Data, size_t Size,
                                bool DuringInitialCorpusExecution);
 
+  void HandleMalloc(size_t Size);
+
 private:
   void AlarmCallback();
   void CrashCallback();
diff --git a/lib/Fuzzer/FuzzerLoop.cpp b/lib/Fuzzer/FuzzerLoop.cpp
index 63cb5c8bf1b..5f02d738798 100644
--- a/lib/Fuzzer/FuzzerLoop.cpp
+++ b/lib/Fuzzer/FuzzerLoop.cpp
@@ -140,6 +140,7 @@ static MallocFreeTracer AllocTracer;
 
 void MallocHook(const volatile void *ptr, size_t size) {
   size_t N = AllocTracer.Mallocs++;
+  F->HandleMalloc(size);
   if (int TraceLevel = AllocTracer.TraceLevel) {
     Printf("MALLOC[%zd] %p %zd\n", N, ptr, size);
     if (TraceLevel >= 2 && EF)
@@ -155,6 +156,21 @@ void FreeHook(const volatile void *ptr) {
   }
 }
 
+// Crash on a single malloc that exceeds the rss limit.
+void Fuzzer::HandleMalloc(size_t Size) {
+  if ((Size >> 20) < (size_t)Options.RssLimitMb)
+    return;
+  Printf("==%d== ERROR: libFuzzer: out-of-memory (malloc(%zd))\n", GetPid(),
+         Size);
+  Printf("   To change the out-of-memory limit use -rss_limit_mb=<N>\n\n");
+  if (EF->__sanitizer_print_stack_trace)
+    EF->__sanitizer_print_stack_trace();
+  DumpCurrentUnit("oom-");
+  Printf("SUMMARY: libFuzzer: out-of-memory\n");
+  PrintFinalStats();
+  _Exit(Options.ErrorExitCode); // Stop right now.
+}
+
 Fuzzer::Fuzzer(UserCallback CB, InputCorpus &Corpus, MutationDispatcher &MD,
                FuzzingOptions Options)
     : CB(CB), Corpus(Corpus), MD(MD), Options(Options) {
diff --git a/lib/Fuzzer/test/CMakeLists.txt b/lib/Fuzzer/test/CMakeLists.txt
index 091b38825f6..a664e944585 100644
--- a/lib/Fuzzer/test/CMakeLists.txt
+++ b/lib/Fuzzer/test/CMakeLists.txt
@@ -85,6 +85,7 @@ set(Tests
   NthRunCrashTest
   OneHugeAllocTest
   OutOfMemoryTest
+  OutOfMemorySingleLargeMallocTest
   RepeatedMemcmp
   RepeatedBytesTest
   SimpleCmpTest
diff --git a/lib/Fuzzer/test/OutOfMemorySingleLargeMallocTest.cpp b/lib/Fuzzer/test/OutOfMemorySingleLargeMallocTest.cpp
new file mode 100644
index 00000000000..5d95c428e68
--- /dev/null
+++ b/lib/Fuzzer/test/OutOfMemorySingleLargeMallocTest.cpp
@@ -0,0 +1,28 @@
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+
+// Tests OOM handling.
+#include <assert.h>
+#include <cstdint>
+#include <cstdlib>
+#include <cstddef>
+#include <cstring>
+#include <iostream>
+#include <unistd.h>
+
+static volatile char *SinkPtr;
+
+extern "C" int LLVMFuzzerTestOneInput(const uint8_t *Data, size_t Size) {
+  if (Size > 0 && Data[0] == 'H') {
+    if (Size > 1 && Data[1] == 'i') {
+      if (Size > 2 && Data[2] == '!') {
+          size_t kSize = 0xff000000U;
+          char *p = new char[kSize];
+          SinkPtr = p;
+          delete [] p;
+      }
+    }
+  }
+  return 0;
+}
+
diff --git a/lib/Fuzzer/test/fuzzer-oom-with-profile.test b/lib/Fuzzer/test/fuzzer-oom-with-profile.test
index 391fd4bb0ff..2b2b0b9d5da 100644
--- a/lib/Fuzzer/test/fuzzer-oom-with-profile.test
+++ b/lib/Fuzzer/test/fuzzer-oom-with-profile.test
@@ -1,6 +1,6 @@
 REQUIRES: linux
-RUN: not LLVMFuzzer-OutOfMemoryTest -rss_limit_mb=10 2>&1 | FileCheck %s
-CHECK: ERROR: libFuzzer: out-of-memory (used: {{.*}}; limit: 10Mb)
+RUN: not LLVMFuzzer-OutOfMemoryTest -rss_limit_mb=300 2>&1 | FileCheck %s
+CHECK: ERROR: libFuzzer: out-of-memory (used: {{.*}}; limit: 300Mb)
 CHECK: Live Heap Allocations
 CHECK: Test unit written to ./oom-
 SUMMARY: libFuzzer: out-of-memory
diff --git a/lib/Fuzzer/test/fuzzer-oom.test b/lib/Fuzzer/test/fuzzer-oom.test
index 4cdff2142fd..7bfd37632d8 100644
--- a/lib/Fuzzer/test/fuzzer-oom.test
+++ b/lib/Fuzzer/test/fuzzer-oom.test
@@ -1,4 +1,8 @@
-RUN: not LLVMFuzzer-OutOfMemoryTest -rss_limit_mb=10 2>&1 | FileCheck %s
-CHECK: ERROR: libFuzzer: out-of-memory (used: {{.*}}; limit: 10Mb)
+RUN: not LLVMFuzzer-OutOfMemoryTest -rss_limit_mb=300 2>&1 | FileCheck %s
+CHECK: ERROR: libFuzzer: out-of-memory (used: {{.*}}; limit: 300Mb)
 CHECK: Test unit written to ./oom-
 SUMMARY: libFuzzer: out-of-memory
+
+RUN: not LLVMFuzzer-OutOfMemorySingleLargeMallocTest 2>&1 | FileCheck %s --check-prefix=SINGLE_LARGE_MALLOC
+SINGLE_LARGE_MALLOC: libFuzzer: out-of-memory (malloc(42{{.*}}))
+SINGLE_LARGE_MALLOC: in LLVMFuzzerTestOneInput
-- 
2.11.0