From 5187ebbe875b67efe7e7ca4dff799beec13719c0 Mon Sep 17 00:00:00 2001 From: Matt Morehouse Date: Thu, 20 Jul 2017 20:43:39 +0000 Subject: [PATCH] Generate error reports when a fuzz target exits. Summary: Implements https://github.com/google/sanitizers/issues/835. Flush stdout before exiting in test cases. Since the atexit hook is used for exit reports, pending prints to stdout can be lost if they aren't flushed before calling exit(). Expect tests to have non-zero exit code if exit() is called. Reviewers: vitalybuka, kcc Reviewed By: kcc Subscribers: eraman, llvm-commits, hiraditya Differential Revision: https://reviews.llvm.org/D35602 git-svn-id: https://llvm.org/svn/llvm-project/llvm/trunk@308669 91177308-0d34-0410-b5e6-96231b3b80d8 --- lib/Fuzzer/FuzzerDriver.cpp | 3 +++ lib/Fuzzer/FuzzerInternal.h | 2 ++ lib/Fuzzer/FuzzerLoop.cpp | 18 ++++++++++++++++++ lib/Fuzzer/test/AbsNegAndConstant64Test.cpp | 1 + lib/Fuzzer/test/AbsNegAndConstantTest.cpp | 1 + lib/Fuzzer/test/BufferOverflowOnInput.cpp | 3 ++- lib/Fuzzer/test/CustomCrossOverTest.cpp | 3 ++- lib/Fuzzer/test/CustomMutatorTest.cpp | 3 ++- lib/Fuzzer/test/NthRunCrashTest.cpp | 3 ++- lib/Fuzzer/test/RepeatedBytesTest.cpp | 4 +++- lib/Fuzzer/test/SimpleDictionaryTest.cpp | 3 ++- lib/Fuzzer/test/SimpleTest.cpp | 3 ++- lib/Fuzzer/test/SimpleThreadedTest.cpp | 3 ++- lib/Fuzzer/test/exit-report.test | 5 +++++ lib/Fuzzer/test/fuzzer-flags.test | 8 ++++---- lib/Fuzzer/test/fuzzer-printcovpcs.test | 2 +- lib/Fuzzer/test/fuzzer.test | 6 +++--- lib/Fuzzer/test/inline-8bit-counters.test | 2 +- lib/Fuzzer/test/repeated-bytes.test | 2 +- lib/Fuzzer/test/trace-pc.test | 2 +- lib/Fuzzer/test/ulimit.test | 2 +- 21 files changed, 59 insertions(+), 20 deletions(-) create mode 100644 lib/Fuzzer/test/exit-report.test diff --git a/lib/Fuzzer/FuzzerDriver.cpp b/lib/Fuzzer/FuzzerDriver.cpp index fd8cab38a7b..36d9422bc28 100644 --- a/lib/Fuzzer/FuzzerDriver.cpp +++ b/lib/Fuzzer/FuzzerDriver.cpp @@ -20,6 +20,7 @@ #include #include #include +#include #include #include #include @@ -642,6 +643,8 @@ int FuzzerDriver(int *argc, char ***argv, UserCallback Callback) { Options.HandleXfsz = Flags.handle_xfsz; SetSignalHandler(Options); + std::atexit(Fuzzer::StaticExitCallback); + if (Flags.minimize_crash) return MinimizeCrashInput(Args, Options); diff --git a/lib/Fuzzer/FuzzerInternal.h b/lib/Fuzzer/FuzzerInternal.h index 3fc3fe004ce..8602818a420 100644 --- a/lib/Fuzzer/FuzzerInternal.h +++ b/lib/Fuzzer/FuzzerInternal.h @@ -60,6 +60,7 @@ public: static void StaticAlarmCallback(); static void StaticCrashSignalCallback(); + static void StaticExitCallback(); static void StaticInterruptCallback(); static void StaticFileSizeExceedCallback(); @@ -91,6 +92,7 @@ public: private: void AlarmCallback(); void CrashCallback(); + void ExitCallback(); void CrashOnOverwrittenData(); void InterruptCallback(); void MutateAndTestOne(); diff --git a/lib/Fuzzer/FuzzerLoop.cpp b/lib/Fuzzer/FuzzerLoop.cpp index ba4ba80db00..b9e70b6dadd 100644 --- a/lib/Fuzzer/FuzzerLoop.cpp +++ b/lib/Fuzzer/FuzzerLoop.cpp @@ -175,6 +175,11 @@ void Fuzzer::StaticCrashSignalCallback() { F->CrashCallback(); } +void Fuzzer::StaticExitCallback() { + assert(F); + F->ExitCallback(); +} + void Fuzzer::StaticInterruptCallback() { assert(F); F->InterruptCallback(); @@ -198,6 +203,19 @@ void Fuzzer::CrashCallback() { _Exit(Options.ErrorExitCode); // Stop right now. } +void Fuzzer::ExitCallback() { + if (!RunningCB) + return; // This exit did not come from the user callback + Printf("==%lu== ERROR: libFuzzer: fuzz target exited\n", GetPid()); + if (EF->__sanitizer_print_stack_trace) + EF->__sanitizer_print_stack_trace(); + Printf("SUMMARY: libFuzzer: fuzz target exited\n"); + DumpCurrentUnit("crash-"); + PrintFinalStats(); + _Exit(Options.ErrorExitCode); +} + + void Fuzzer::InterruptCallback() { Printf("==%lu== libFuzzer: run interrupted; exiting\n", GetPid()); PrintFinalStats(); diff --git a/lib/Fuzzer/test/AbsNegAndConstant64Test.cpp b/lib/Fuzzer/test/AbsNegAndConstant64Test.cpp index dfb6007b797..abeb784e9a1 100644 --- a/lib/Fuzzer/test/AbsNegAndConstant64Test.cpp +++ b/lib/Fuzzer/test/AbsNegAndConstant64Test.cpp @@ -16,6 +16,7 @@ extern "C" int LLVMFuzzerTestOneInput(const uint8_t *Data, size_t Size) { memcpy(&y, Data + sizeof(x), sizeof(y)); if (llabs(x) < 0 && y == 0xbaddcafedeadbeefULL) { printf("BINGO; Found the target, exiting; x = 0x%lx y 0x%lx\n", x, y); + fflush(stdout); exit(1); } return 0; diff --git a/lib/Fuzzer/test/AbsNegAndConstantTest.cpp b/lib/Fuzzer/test/AbsNegAndConstantTest.cpp index e9d983ff1eb..049db0a60c3 100644 --- a/lib/Fuzzer/test/AbsNegAndConstantTest.cpp +++ b/lib/Fuzzer/test/AbsNegAndConstantTest.cpp @@ -16,6 +16,7 @@ extern "C" int LLVMFuzzerTestOneInput(const uint8_t *Data, size_t Size) { memcpy(&y, Data + sizeof(x), sizeof(y)); if (abs(x) < 0 && y == 0xbaddcafe) { printf("BINGO; Found the target, exiting; x = 0x%x y 0x%x\n", x, y); + fflush(stdout); exit(1); } return 0; diff --git a/lib/Fuzzer/test/BufferOverflowOnInput.cpp b/lib/Fuzzer/test/BufferOverflowOnInput.cpp index 75e1fb90a19..159da92d4e9 100644 --- a/lib/Fuzzer/test/BufferOverflowOnInput.cpp +++ b/lib/Fuzzer/test/BufferOverflowOnInput.cpp @@ -7,6 +7,7 @@ #include #include #include +#include static volatile bool SeedLargeBuffer; @@ -15,7 +16,7 @@ extern "C" int LLVMFuzzerTestOneInput(const uint8_t *Data, size_t Size) { if (Size >= 4) SeedLargeBuffer = true; if (Size == 3 && SeedLargeBuffer && Data[3]) { - std::cout << "Woops, reading Data[3] w/o crashing\n"; + std::cout << "Woops, reading Data[3] w/o crashing\n" << std::flush; exit(1); } return 0; diff --git a/lib/Fuzzer/test/CustomCrossOverTest.cpp b/lib/Fuzzer/test/CustomCrossOverTest.cpp index b624088b902..58059c9c1eb 100644 --- a/lib/Fuzzer/test/CustomCrossOverTest.cpp +++ b/lib/Fuzzer/test/CustomCrossOverTest.cpp @@ -7,6 +7,7 @@ #include #include #include +#include #include #include @@ -26,7 +27,7 @@ extern "C" int LLVMFuzzerTestOneInput(const uint8_t *Data, size_t Size) { if (Size && Data[0] == 'a') sink--; if (Str.find(Target) != std::string::npos) { - std::cout << "BINGO; Found the target, exiting\n"; + std::cout << "BINGO; Found the target, exiting\n" << std::flush; exit(1); } return 0; diff --git a/lib/Fuzzer/test/CustomMutatorTest.cpp b/lib/Fuzzer/test/CustomMutatorTest.cpp index 521d7f506b4..b2adb94082d 100644 --- a/lib/Fuzzer/test/CustomMutatorTest.cpp +++ b/lib/Fuzzer/test/CustomMutatorTest.cpp @@ -7,6 +7,7 @@ #include #include #include +#include #include "FuzzerInterface.h" @@ -19,7 +20,7 @@ extern "C" int LLVMFuzzerTestOneInput(const uint8_t *Data, size_t Size) { if (Size > 1 && Data[1] == 'i') { Sink = 2; if (Size > 2 && Data[2] == '!') { - std::cout << "BINGO; Found the target, exiting\n"; + std::cout << "BINGO; Found the target, exiting\n" << std::flush; exit(1); } } diff --git a/lib/Fuzzer/test/NthRunCrashTest.cpp b/lib/Fuzzer/test/NthRunCrashTest.cpp index da5fbd33e96..26cdc8f17ad 100644 --- a/lib/Fuzzer/test/NthRunCrashTest.cpp +++ b/lib/Fuzzer/test/NthRunCrashTest.cpp @@ -5,12 +5,13 @@ #include #include #include +#include static int Counter; extern "C" int LLVMFuzzerTestOneInput(const uint8_t *Data, size_t Size) { if (Counter++ == 1000) { - std::cout << "BINGO; Found the target, exiting\n"; + std::cout << "BINGO; Found the target, exiting\n" << std::flush; exit(1); } return 0; diff --git a/lib/Fuzzer/test/RepeatedBytesTest.cpp b/lib/Fuzzer/test/RepeatedBytesTest.cpp index 14222f28474..31868cf8c54 100644 --- a/lib/Fuzzer/test/RepeatedBytesTest.cpp +++ b/lib/Fuzzer/test/RepeatedBytesTest.cpp @@ -7,6 +7,7 @@ #include #include #include +#include extern "C" int LLVMFuzzerTestOneInput(const uint8_t *Data, size_t Size) { assert(Data); @@ -21,7 +22,8 @@ extern "C" int LLVMFuzzerTestOneInput(const uint8_t *Data, size_t Size) { MaxA = GT * CurA + (!GT) * MaxA; } if (MaxA >= 20) { - std::cout << "BINGO; Found the target (Max: " << MaxA << "), exiting\n"; + std::cout << "BINGO; Found the target (Max: " << MaxA << "), exiting\n" + << std::flush; exit(0); } return 0; diff --git a/lib/Fuzzer/test/SimpleDictionaryTest.cpp b/lib/Fuzzer/test/SimpleDictionaryTest.cpp index a1cd2004722..ffa2e4137bd 100644 --- a/lib/Fuzzer/test/SimpleDictionaryTest.cpp +++ b/lib/Fuzzer/test/SimpleDictionaryTest.cpp @@ -10,6 +10,7 @@ #include #include #include +#include static volatile int Zero = 0; @@ -21,7 +22,7 @@ extern "C" int LLVMFuzzerTestOneInput(const uint8_t *Data, size_t Size) { if (Expected[i] + Zero == Data[i]) Match++; if (Match == strlen(Expected)) { - std::cout << "BINGO; Found the target, exiting\n"; + std::cout << "BINGO; Found the target, exiting\n" << std::flush; exit(1); } return 0; diff --git a/lib/Fuzzer/test/SimpleTest.cpp b/lib/Fuzzer/test/SimpleTest.cpp index a8b4988dff1..3882a842b88 100644 --- a/lib/Fuzzer/test/SimpleTest.cpp +++ b/lib/Fuzzer/test/SimpleTest.cpp @@ -7,6 +7,7 @@ #include #include #include +#include static volatile int Sink; @@ -17,7 +18,7 @@ extern "C" int LLVMFuzzerTestOneInput(const uint8_t *Data, size_t Size) { if (Size > 1 && Data[1] == 'i') { Sink = 2; if (Size > 2 && Data[2] == '!') { - std::cout << "BINGO; Found the target, exiting\n"; + std::cout << "BINGO; Found the target, exiting\n" << std::flush; exit(0); } } diff --git a/lib/Fuzzer/test/SimpleThreadedTest.cpp b/lib/Fuzzer/test/SimpleThreadedTest.cpp index 1abdc3fc6d6..deeae756a82 100644 --- a/lib/Fuzzer/test/SimpleThreadedTest.cpp +++ b/lib/Fuzzer/test/SimpleThreadedTest.cpp @@ -7,12 +7,13 @@ #include #include #include +#include #include extern "C" int LLVMFuzzerTestOneInput(const uint8_t *Data, size_t Size) { auto C = [&] { if (Size >= 2 && Data[0] == 'H') { - std::cout << "BINGO; Found the target, exiting\n"; + std::cout << "BINGO; Found the target, exiting\n" << std::flush; abort(); } }; diff --git a/lib/Fuzzer/test/exit-report.test b/lib/Fuzzer/test/exit-report.test new file mode 100644 index 00000000000..26e8cfc9155 --- /dev/null +++ b/lib/Fuzzer/test/exit-report.test @@ -0,0 +1,5 @@ +RUN: not LLVMFuzzer-SimpleTest 2>&1 | FileCheck %s + +CHECK: ERROR: libFuzzer: fuzz target exited +CHECK: SUMMARY: libFuzzer: fuzz target exited +CHECK: Test unit written to diff --git a/lib/Fuzzer/test/fuzzer-flags.test b/lib/Fuzzer/test/fuzzer-flags.test index 61ce07797b4..b9bf2f8e4cc 100644 --- a/lib/Fuzzer/test/fuzzer-flags.test +++ b/lib/Fuzzer/test/fuzzer-flags.test @@ -1,18 +1,18 @@ -RUN: LLVMFuzzer-FlagsTest -foo_bar=1 2>&1 | FileCheck %s --check-prefix=FOO_BAR +RUN: not LLVMFuzzer-FlagsTest -foo_bar=1 2>&1 | FileCheck %s --check-prefix=FOO_BAR FOO_BAR: WARNING: unrecognized flag '-foo_bar=1'; use -help=1 to list all flags FOO_BAR: BINGO -RUN: LLVMFuzzer-FlagsTest -runs=10 --max_len=100 2>&1 | FileCheck %s --check-prefix=DASH_DASH +RUN: not LLVMFuzzer-FlagsTest -runs=10 --max_len=100 2>&1 | FileCheck %s --check-prefix=DASH_DASH DASH_DASH: WARNING: did you mean '-max_len=100' (single dash)? DASH_DASH: INFO: A corpus is not provided, starting from an empty corpus RUN: LLVMFuzzer-FlagsTest -help=1 2>&1 | FileCheck %s --check-prefix=NO_INTERNAL NO_INTERNAL-NOT: internal flag -RUN: LLVMFuzzer-FlagsTest --foo-bar -runs=10 -ignore_remaining_args=1 --baz -help=1 test 2>&1 | FileCheck %s --check-prefix=PASSTHRU +RUN: not LLVMFuzzer-FlagsTest --foo-bar -runs=10 -ignore_remaining_args=1 --baz -help=1 test 2>&1 | FileCheck %s --check-prefix=PASSTHRU PASSTHRU: BINGO --foo-bar --baz -help=1 test RUN: mkdir -p %t/T0 %t/T1 RUN: touch %t/T1/empty -RUN: LLVMFuzzer-FlagsTest --foo-bar -merge=1 %t/T0 %t/T1 -ignore_remaining_args=1 --baz -help=1 test 2>&1 | FileCheck %s --check-prefix=PASSTHRU-MERGE +RUN: not LLVMFuzzer-FlagsTest --foo-bar -merge=1 %t/T0 %t/T1 -ignore_remaining_args=1 --baz -help=1 test 2>&1 | FileCheck %s --check-prefix=PASSTHRU-MERGE PASSTHRU-MERGE: BINGO --foo-bar --baz -help=1 test diff --git a/lib/Fuzzer/test/fuzzer-printcovpcs.test b/lib/Fuzzer/test/fuzzer-printcovpcs.test index e4c6f0ed1df..77a2e7d147c 100644 --- a/lib/Fuzzer/test/fuzzer-printcovpcs.test +++ b/lib/Fuzzer/test/fuzzer-printcovpcs.test @@ -1,4 +1,4 @@ -RUN: LLVMFuzzer-SimpleTest -print_pcs=1 -seed=1 2>&1 | FileCheck %s --check-prefix=PCS +RUN: not LLVMFuzzer-SimpleTest -print_pcs=1 -seed=1 2>&1 | FileCheck %s --check-prefix=PCS PCS-NOT: NEW_PC PCS:INITED PCS:NEW_PC: {{0x[a-f0-9]+}} diff --git a/lib/Fuzzer/test/fuzzer.test b/lib/Fuzzer/test/fuzzer.test index ff46d32b387..a3b7d93603f 100644 --- a/lib/Fuzzer/test/fuzzer.test +++ b/lib/Fuzzer/test/fuzzer.test @@ -1,10 +1,10 @@ CHECK: BINGO Done1000000: Done 1000000 runs in -RUN: LLVMFuzzer-SimpleTest 2>&1 | FileCheck %s +RUN: not LLVMFuzzer-SimpleTest 2>&1 | FileCheck %s # only_ascii mode. Will perform some minimal self-validation. -RUN: LLVMFuzzer-SimpleTest -only_ascii=1 2>&1 +RUN: not LLVMFuzzer-SimpleTest -only_ascii=1 2>&1 RUN: LLVMFuzzer-SimpleCmpTest -max_total_time=1 -use_cmp=0 2>&1 | FileCheck %s --check-prefix=MaxTotalTime MaxTotalTime: Done {{.*}} runs in {{.}} second(s) @@ -47,7 +47,7 @@ RUN: not LLVMFuzzer-DSOTest 2>&1 | FileCheck %s --check-prefix=DSO DSO: INFO: Loaded 3 modules DSO: BINGO -RUN: LLVMFuzzer-SimpleTest -exit_on_src_pos=SimpleTest.cpp:17 2>&1 | FileCheck %s --check-prefix=EXIT_ON_SRC_POS +RUN: LLVMFuzzer-SimpleTest -exit_on_src_pos=SimpleTest.cpp:18 2>&1 | FileCheck %s --check-prefix=EXIT_ON_SRC_POS RUN: LLVMFuzzer-ShrinkControlFlowTest -exit_on_src_pos=ShrinkControlFlowTest.cpp:23 2>&1 | FileCheck %s --check-prefix=EXIT_ON_SRC_POS EXIT_ON_SRC_POS: INFO: found line matching '{{.*}}', exiting. diff --git a/lib/Fuzzer/test/inline-8bit-counters.test b/lib/Fuzzer/test/inline-8bit-counters.test index 8747af81451..ff84dd119c1 100644 --- a/lib/Fuzzer/test/inline-8bit-counters.test +++ b/lib/Fuzzer/test/inline-8bit-counters.test @@ -1,4 +1,4 @@ REQUIRES: linux CHECK: INFO: Loaded 1 modules with {{.*}} inline 8-bit counters CHECK: BINGO -RUN: LLVMFuzzer-SimpleTest-Inline8bitCounters -runs=1000000 -seed=1 2>&1 | FileCheck %s +RUN: not LLVMFuzzer-SimpleTest-Inline8bitCounters -runs=1000000 -seed=1 2>&1 | FileCheck %s diff --git a/lib/Fuzzer/test/repeated-bytes.test b/lib/Fuzzer/test/repeated-bytes.test index 71394087939..fc72142f15d 100644 --- a/lib/Fuzzer/test/repeated-bytes.test +++ b/lib/Fuzzer/test/repeated-bytes.test @@ -1,2 +1,2 @@ CHECK: BINGO -RUN: LLVMFuzzer-RepeatedBytesTest -seed=1 -runs=1000000 2>&1 | FileCheck %s +RUN: not LLVMFuzzer-RepeatedBytesTest -seed=1 -runs=1000000 2>&1 | FileCheck %s diff --git a/lib/Fuzzer/test/trace-pc.test b/lib/Fuzzer/test/trace-pc.test index 3709677b71b..93bb2c06154 100644 --- a/lib/Fuzzer/test/trace-pc.test +++ b/lib/Fuzzer/test/trace-pc.test @@ -1,2 +1,2 @@ CHECK: BINGO -RUN: LLVMFuzzer-SimpleTest-TracePC -runs=100000 -seed=1 2>&1 | FileCheck %s +RUN: not LLVMFuzzer-SimpleTest-TracePC -runs=100000 -seed=1 2>&1 | FileCheck %s diff --git a/lib/Fuzzer/test/ulimit.test b/lib/Fuzzer/test/ulimit.test index c2faca13f72..ca4e5ffce15 100644 --- a/lib/Fuzzer/test/ulimit.test +++ b/lib/Fuzzer/test/ulimit.test @@ -1,4 +1,4 @@ REQUIRES: posix RUN: ulimit -s 1000 -RUN: LLVMFuzzer-SimpleTest +RUN: not LLVMFuzzer-SimpleTest -- 2.11.0