Import compiler-rt r182741.vendor/compiler-rt/compiler-rt-r182741

1 files changed, 121 insertions, 22 deletions

diff --git a/lib/tsan/rtl/tsan_mman.cc b/lib/tsan/rtl/tsan_mman.cc
index 82f7105d60db..b6671b1abf09 100644
--- a/lib/tsan/rtl/tsan_mman.cc
+++ b/lib/tsan/rtl/tsan_mman.cc
@@ -29,6 +29,41 @@ extern "C" void WEAK __tsan_free_hook(void *ptr) {
 
 namespace __tsan {
 
+COMPILER_CHECK(sizeof(MBlock) == 16);
+
+void MBlock::Lock() {
+  atomic_uintptr_t *a = reinterpret_cast<atomic_uintptr_t*>(this);
+  uptr v = atomic_load(a, memory_order_relaxed);
+  for (int iter = 0;; iter++) {
+    if (v & 1) {
+      if (iter < 10)
+        proc_yield(20);
+      else
+        internal_sched_yield();
+      v = atomic_load(a, memory_order_relaxed);
+      continue;
+    }
+    if (atomic_compare_exchange_weak(a, &v, v | 1, memory_order_acquire))
+      break;
+  }
+}
+
+void MBlock::Unlock() {
+  atomic_uintptr_t *a = reinterpret_cast<atomic_uintptr_t*>(this);
+  uptr v = atomic_load(a, memory_order_relaxed);
+  DCHECK(v & 1);
+  atomic_store(a, v & ~1, memory_order_relaxed);
+}
+
+struct MapUnmapCallback {
+  void OnMap(uptr p, uptr size) const { }
+  void OnUnmap(uptr p, uptr size) const {
+    // We are about to unmap a chunk of user memory.
+    // Mark the corresponding shadow memory as not needed.
+    DontNeedShadowFor(p, size);
+  }
+};
+
 static char allocator_placeholder[sizeof(Allocator)] ALIGNED(64);
 Allocator *allocator() {
   return reinterpret_cast<Allocator*>(&allocator_placeholder);
@@ -38,8 +73,16 @@ void InitializeAllocator() {
   allocator()->Init();
 }
 
-void AlloctorThreadFinish(ThreadState *thr) {
-  allocator()->SwallowCache(&thr->alloc_cache);
+void AllocatorThreadStart(ThreadState *thr) {
+  allocator()->InitCache(&thr->alloc_cache);
+}
+
+void AllocatorThreadFinish(ThreadState *thr) {
+  allocator()->DestroyCache(&thr->alloc_cache);
+}
+
+void AllocatorPrintStats() {
+  allocator()->PrintStats();
 }
 
 static void SignalUnsafeCall(ThreadState *thr, uptr pc) {
@@ -48,7 +91,7 @@ static void SignalUnsafeCall(ThreadState *thr, uptr pc) {
   Context *ctx = CTX();
   StackTrace stack;
   stack.ObtainCurrent(thr, pc);
-  Lock l(&ctx->thread_mtx);
+  ThreadRegistryLock l(ctx->thread_registry);
   ScopedReport rep(ReportTypeSignalUnsafe);
   if (!IsFiredSuppression(ctx, rep, stack)) {
     rep.AddStack(&stack);
@@ -58,17 +101,15 @@ static void SignalUnsafeCall(ThreadState *thr, uptr pc) {
 
 void *user_alloc(ThreadState *thr, uptr pc, uptr sz, uptr align) {
   CHECK_GT(thr->in_rtl, 0);
+  if ((sz >= (1ull << 40)) || (align >= (1ull << 40)))
+    return 0;
   void *p = allocator()->Allocate(&thr->alloc_cache, sz, align);
   if (p == 0)
     return 0;
   MBlock *b = new(allocator()->GetMetaData(p)) MBlock;
-  b->size = sz;
-  b->head = 0;
-  b->alloc_tid = thr->unique_id;
-  b->alloc_stack_id = CurrentStackId(thr, pc);
-  if (CTX() && CTX()->initialized) {
+  b->Init(sz, thr->tid, CurrentStackId(thr, pc));
+  if (CTX() && CTX()->initialized)
     MemoryRangeImitateWrite(thr, pc, (uptr)p, sz);
-  }
   DPrintf("#%d: alloc(%zu) = %p\n", thr->tid, sz, p);
   SignalUnsafeCall(thr, pc);
   return p;
@@ -79,9 +120,9 @@ void user_free(ThreadState *thr, uptr pc, void *p) {
   CHECK_NE(p, (void*)0);
   DPrintf("#%d: free(%p)\n", thr->tid, p);
   MBlock *b = (MBlock*)allocator()->GetMetaData(p);
-  if (b->head)   {
-    Lock l(&b->mtx);
-    for (SyncVar *s = b->head; s;) {
+  if (b->ListHead()) {
+    MBlock::ScopedLock l(b);
+    for (SyncVar *s = b->ListHead(); s;) {
       SyncVar *res = s;
       s = s->next;
       StatInc(thr, StatSyncDestroyed);
@@ -89,12 +130,10 @@ void user_free(ThreadState *thr, uptr pc, void *p) {
       res->mtx.Unlock();
       DestroyAndFree(res);
     }
-    b->head = 0;
+    b->ListReset();
   }
-  if (CTX() && CTX()->initialized && thr->in_rtl == 1) {
-    MemoryRangeFreed(thr, pc, (uptr)p, b->size);
-  }
-  b->~MBlock();
+  if (CTX() && CTX()->initialized && thr->in_rtl == 1)
+    MemoryRangeFreed(thr, pc, (uptr)p, b->Size());
   allocator()->Deallocate(&thr->alloc_cache, p);
   SignalUnsafeCall(thr, pc);
 }
@@ -110,20 +149,29 @@ void *user_realloc(ThreadState *thr, uptr pc, void *p, uptr sz) {
       return 0;
     if (p) {
       MBlock *b = user_mblock(thr, p);
-      internal_memcpy(p2, p, min(b->size, sz));
+      CHECK_NE(b, 0);
+      internal_memcpy(p2, p, min(b->Size(), sz));
     }
   }
-  if (p) {
+  if (p)
     user_free(thr, pc, p);
-  }
   return p2;
 }
 
+uptr user_alloc_usable_size(ThreadState *thr, uptr pc, void *p) {
+  CHECK_GT(thr->in_rtl, 0);
+  if (p == 0)
+    return 0;
+  MBlock *b = (MBlock*)allocator()->GetMetaData(p);
+  return b ? b->Size() : 0;
+}
+
 MBlock *user_mblock(ThreadState *thr, void *p) {
-  CHECK_NE(p, (void*)0);
+  CHECK_NE(p, 0);
   Allocator *a = allocator();
   void *b = a->GetBlockBegin(p);
-  CHECK_NE(b, 0);
+  if (b == 0)
+    return 0;
   return (MBlock*)a->GetMetaData(b);
 }
 
@@ -164,3 +212,54 @@ void internal_free(void *p) {
 }
 
 }  // namespace __tsan
+
+using namespace __tsan;
+
+extern "C" {
+uptr __tsan_get_current_allocated_bytes() {
+  u64 stats[AllocatorStatCount];
+  allocator()->GetStats(stats);
+  u64 m = stats[AllocatorStatMalloced];
+  u64 f = stats[AllocatorStatFreed];
+  return m >= f ? m - f : 1;
+}
+
+uptr __tsan_get_heap_size() {
+  u64 stats[AllocatorStatCount];
+  allocator()->GetStats(stats);
+  u64 m = stats[AllocatorStatMmapped];
+  u64 f = stats[AllocatorStatUnmapped];
+  return m >= f ? m - f : 1;
+}
+
+uptr __tsan_get_free_bytes() {
+  return 1;
+}
+
+uptr __tsan_get_unmapped_bytes() {
+  return 1;
+}
+
+uptr __tsan_get_estimated_allocated_size(uptr size) {
+  return size;
+}
+
+bool __tsan_get_ownership(void *p) {
+  return allocator()->GetBlockBegin(p) != 0;
+}
+
+uptr __tsan_get_allocated_size(void *p) {
+  if (p == 0)
+    return 0;
+  p = allocator()->GetBlockBegin(p);
+  if (p == 0)
+    return 0;
+  MBlock *b = (MBlock*)allocator()->GetMetaData(p);
+  return b->Size();
+}
+
+void __tsan_on_thread_idle() {
+  ThreadState *thr = cur_thread();
+  allocator()->SwallowCache(&thr->alloc_cache);
+}
+}  // extern "C"