From 6d7729d6ae8b2ac3800e92092d61390ce4e3b6d7 Mon Sep 17 00:00:00 2001
From: Dmitry Petrochenko <dmitry.petrochenko@intel.com>
Date: Wed, 1 Oct 2014 13:31:58 +0700
Subject: [PATCH] Fix OOM throwing if it happens in finalizer reference

The Class::Alloc should return null if OOM happened during
adding finalizer reference, even if finalizable object is
allocated succesfully.

Change-Id: I66c1cdda50228bf1302839785ce4d4889b676f5b
Signed-off-by: Dmitry Petrochenko <dmitry.petrochenko@intel.com>
Signed-off-by: Serguei Katkov <serguei.i.katkov@intel.com>
---
 runtime/mirror/class-inl.h       |  4 ++++
 test/080-oom-throw/src/Main.java | 50 ++++++++++++++++++++++++++++++++++++----
 2 files changed, 50 insertions(+), 4 deletions(-)

diff --git a/runtime/mirror/class-inl.h b/runtime/mirror/class-inl.h
index 3d3ae166a..661de68e5 100644
--- a/runtime/mirror/class-inl.h
+++ b/runtime/mirror/class-inl.h
@@ -553,6 +553,10 @@ inline Object* Class::Alloc(Thread* self, gc::AllocatorType allocator_type) {
                                                              allocator_type, VoidFunctor());
   if (add_finalizer && LIKELY(obj != nullptr)) {
     heap->AddFinalizerReference(self, &obj);
+    if (UNLIKELY(self->IsExceptionPending())) {
+      // Failed to allocate finalizer reference, it means that whole allocation failed
+      obj = nullptr;
+    }
   }
   return obj;
 }
diff --git a/test/080-oom-throw/src/Main.java b/test/080-oom-throw/src/Main.java
index c93f8bbc5..63c521521 100644
--- a/test/080-oom-throw/src/Main.java
+++ b/test/080-oom-throw/src/Main.java
@@ -15,13 +15,15 @@
  */
 
 public class Main {
+    static char [][] holder;
+
     static class ArrayMemEater {
         static boolean sawOome;
 
         static void blowup(char[][] holder) {
             try {
                 for (int i = 0; i < holder.length; ++i) {
-                    holder[i] = new char[1024 * 1024];
+                    holder[i] = new char[1022 * 1024];
                 }
             } catch (OutOfMemoryError oome) {
                 ArrayMemEater.sawOome = true;
@@ -50,8 +52,30 @@ public class Main {
         }
     }
 
-    static boolean triggerArrayOOM() {
-        ArrayMemEater.blowup(new char[128 * 1024][]);
+    static class InstanceFinalizerMemEater {
+        static boolean sawOome;
+        static InstanceFinalizerMemEater hook;
+
+        InstanceFinalizerMemEater next;
+
+        static InstanceFinalizerMemEater allocate() {
+            try {
+                return new InstanceFinalizerMemEater();
+            } catch (OutOfMemoryError e) {
+                InstanceFinalizerMemEater.sawOome = true;
+                return null;
+            }
+        }
+
+        static void confuseCompilerOptimization(InstanceFinalizerMemEater instance) {
+            hook = instance;
+        }
+
+        protected void finalize() {}
+    }
+
+    static boolean triggerArrayOOM(char[][] holder) {
+        ArrayMemEater.blowup(holder);
         return ArrayMemEater.sawOome;
     }
 
@@ -67,11 +91,29 @@ public class Main {
         return InstanceMemEater.sawOome;
     }
 
+    static boolean triggerInstanceFinalizerOOM() {
+        InstanceFinalizerMemEater memEater = InstanceFinalizerMemEater.allocate();
+        InstanceFinalizerMemEater lastMemEater = memEater;
+        do {
+            lastMemEater.next = InstanceFinalizerMemEater.allocate();
+            lastMemEater = lastMemEater.next;
+        } while (lastMemEater != null);
+        memEater.confuseCompilerOptimization(memEater);
+        InstanceFinalizerMemEater.hook = null;
+        return InstanceFinalizerMemEater.sawOome;
+    }
+
     public static void main(String[] args) {
-        if (triggerArrayOOM()) {
+        // Keep holder alive to make instance OOM happen faster
+        holder = new char[128 * 1024][];
+        if (triggerArrayOOM(holder)) {
             System.out.println("NEW_ARRAY correctly threw OOME");
         }
 
+        if (!triggerInstanceFinalizerOOM()) {
+            System.out.println("NEW_INSTANCE (finalize) did not threw OOME");
+        }
+
         if (triggerInstanceOOM()) {
             System.out.println("NEW_INSTANCE correctly threw OOME");
         }
-- 
2.11.0