[ManagedStatic] Reimplement double-checked locking with std::atomic.

This gets rid of the memory fence in the hot path (dereferencing the
ManagedStatic), trading for an extra mutex lock in the cold path (when
the ManagedStatic was uninitialized). Since this only happens on the
first accesses it shouldn't matter much. On strict architectures like
x86 this removes any atomic instructions from the hot path.

Also remove the tsan annotations, tsan knows how standard atomics work
so they should be unnecessary now.

git-svn-id: https://llvm.org/svn/llvm-project/llvm/trunk@274131 91177308-0d34-0410-b5e6-96231b3b80d8

diff --git a/include/llvm/Support/ManagedStatic.h b/include/llvm/Support/ManagedStatic.h
index 3a34bdb..ec8154b 100644 (file)
--- a/include/llvm/Support/ManagedStatic.h
+++ b/include/llvm/Support/ManagedStatic.h
@@ -14,9 +14,9 @@
 #ifndef LLVM_SUPPORT_MANAGEDSTATIC_H
 #define LLVM_SUPPORT_MANAGEDSTATIC_H
 
-#include "llvm/Support/Atomic.h"
 #include "llvm/Support/Compiler.h"
-#include "llvm/Support/Threading.h"
+#include <atomic>
+#include <cstddef>
 
 namespace llvm {
 
@@ -41,7 +41,7 @@ class ManagedStaticBase {
 protected:
   // This should only be used as a static variable, which guarantees that this
   // will be zero initialized.
-  mutable void *Ptr;
+  mutable std::atomic<void *> Ptr;
   mutable void (*DeleterFn)(void*);
   mutable const ManagedStaticBase *Next;
 
@@ -61,40 +61,26 @@ public:
 template<class C>
 class ManagedStatic : public ManagedStaticBase {
 public:
-
   // Accessors.
   C &operator*() {
-    void* tmp = Ptr;
-    if (llvm_is_multithreaded()) sys::MemoryFence();
-    if (!tmp) RegisterManagedStatic(object_creator<C>, object_deleter<C>::call);
-    TsanHappensAfter(this);
+    void *Tmp = Ptr.load(std::memory_order_acquire);
+    if (!Tmp)
+      RegisterManagedStatic(object_creator<C>, object_deleter<C>::call);
 
-    return *static_cast<C*>(Ptr);
+    return *static_cast<C *>(Ptr.load(std::memory_order_relaxed));
   }
-  C *operator->() {
-    void* tmp = Ptr;
-    if (llvm_is_multithreaded()) sys::MemoryFence();
-    if (!tmp) RegisterManagedStatic(object_creator<C>, object_deleter<C>::call);
-    TsanHappensAfter(this);
 
-    return static_cast<C*>(Ptr);
-  }
+  C *operator->() { return &**this; }
+
   const C &operator*() const {
-    void* tmp = Ptr;
-    if (llvm_is_multithreaded()) sys::MemoryFence();
-    if (!tmp) RegisterManagedStatic(object_creator<C>, object_deleter<C>::call);
-    TsanHappensAfter(this);
+    void *Tmp = Ptr.load(std::memory_order_acquire);
+    if (!Tmp)
+      RegisterManagedStatic(object_creator<C>, object_deleter<C>::call);
 
-    return *static_cast<C*>(Ptr);
+    return *static_cast<C *>(Ptr.load(std::memory_order_relaxed));
   }
-  const C *operator->() const {
-    void* tmp = Ptr;
-    if (llvm_is_multithreaded()) sys::MemoryFence();
-    if (!tmp) RegisterManagedStatic(object_creator<C>, object_deleter<C>::call);
-    TsanHappensAfter(this);
 
-    return static_cast<C*>(Ptr);
-  }
+  const C *operator->() const { return &**this; }
 };
 
 /// llvm_shutdown - Deallocate and destroy all ManagedStatic variables.

diff --git a/lib/Support/ManagedStatic.cpp b/lib/Support/ManagedStatic.cpp
index bc4fe95..7dd3131 100644 (file)
--- a/lib/Support/ManagedStatic.cpp
+++ b/lib/Support/ManagedStatic.cpp
@@ -13,7 +13,6 @@
 
 #include "llvm/Support/ManagedStatic.h"
 #include "llvm/Config/config.h"
-#include "llvm/Support/Atomic.h"
 #include "llvm/Support/Mutex.h"
 #include "llvm/Support/MutexGuard.h"
 #include "llvm/Support/Threading.h"
@@ -42,18 +41,10 @@ void ManagedStaticBase::RegisterManagedStatic(void *(*Creator)(),
   if (llvm_is_multithreaded()) {
     MutexGuard Lock(*getManagedStaticMutex());
 
-    if (!Ptr) {
-      void* tmp = Creator();
+    if (!Ptr.load(std::memory_order_relaxed)) {
+      void *Tmp = Creator();
 
-      TsanHappensBefore(this);
-      sys::MemoryFence();
-
-      // This write is racy against the first read in the ManagedStatic
-      // accessors. The race is benign because it does a second read after a
-      // memory fence, at which point it isn't possible to get a partial value.
-      TsanIgnoreWritesBegin();
-      Ptr = tmp;
-      TsanIgnoreWritesEnd();
+      Ptr.store(Tmp, std::memory_order_release);
       DeleterFn = Deleter;
       
       // Add to list of managed statics.