diff options
author | Dimitry Andric <dim@FreeBSD.org> | 2017-07-13 19:25:48 +0000 |
---|---|---|
committer | Dimitry Andric <dim@FreeBSD.org> | 2017-07-13 19:25:48 +0000 |
commit | 1992b790c2c12b7850bdf86662b67302052ec2fe (patch) | |
tree | 623c69b5fbf527bba17ecb9431ae5189871cecd4 /lib/tsan/rtl | |
parent | 50aa32eff79f252ab05a0c0a589cf2ca37cd9923 (diff) | |
download | src-1992b790c2c12b7850bdf86662b67302052ec2fe.tar.gz src-1992b790c2c12b7850bdf86662b67302052ec2fe.zip |
Vendor import of compiler-rt trunk r307894:vendor/compiler-rt/compiler-rt-trunk-r307894
Notes
Notes:
svn path=/vendor/compiler-rt/dist/; revision=320961
svn path=/vendor/compiler-rt/compiler-rt-trunk-r307894/; revision=320962; tag=vendor/compiler-rt/compiler-rt-trunk-r307894
Diffstat (limited to 'lib/tsan/rtl')
-rw-r--r-- | lib/tsan/rtl/tsan_clock.cc | 119 | ||||
-rw-r--r-- | lib/tsan/rtl/tsan_clock.h | 4 | ||||
-rw-r--r-- | lib/tsan/rtl/tsan_dense_alloc.h | 11 | ||||
-rw-r--r-- | lib/tsan/rtl/tsan_interceptors.cc | 29 | ||||
-rw-r--r-- | lib/tsan/rtl/tsan_interceptors_mac.cc | 7 | ||||
-rw-r--r-- | lib/tsan/rtl/tsan_interface_ann.cc | 16 | ||||
-rw-r--r-- | lib/tsan/rtl/tsan_interface_atomic.cc | 16 | ||||
-rw-r--r-- | lib/tsan/rtl/tsan_mman.cc | 2 | ||||
-rw-r--r-- | lib/tsan/rtl/tsan_platform.h | 49 | ||||
-rw-r--r-- | lib/tsan/rtl/tsan_platform_linux.cc | 17 | ||||
-rw-r--r-- | lib/tsan/rtl/tsan_platform_mac.cc | 8 | ||||
-rw-r--r-- | lib/tsan/rtl/tsan_platform_posix.cc | 27 | ||||
-rw-r--r-- | lib/tsan/rtl/tsan_rtl.cc | 3 | ||||
-rw-r--r-- | lib/tsan/rtl/tsan_rtl_aarch64.S | 127 | ||||
-rw-r--r-- | lib/tsan/rtl/tsan_rtl_amd64.S | 6 | ||||
-rw-r--r-- | lib/tsan/rtl/tsan_rtl_mutex.cc | 12 | ||||
-rw-r--r-- | lib/tsan/rtl/tsan_rtl_report.cc | 2 | ||||
-rw-r--r-- | lib/tsan/rtl/tsan_rtl_thread.cc | 4 | ||||
-rw-r--r-- | lib/tsan/rtl/tsan_stat.cc | 5 | ||||
-rw-r--r-- | lib/tsan/rtl/tsan_stat.h | 5 | ||||
-rw-r--r-- | lib/tsan/rtl/tsan_sync.cc | 4 |
21 files changed, 311 insertions, 162 deletions
diff --git a/lib/tsan/rtl/tsan_clock.cc b/lib/tsan/rtl/tsan_clock.cc index 32435adfdf33..9ee9104283f8 100644 --- a/lib/tsan/rtl/tsan_clock.cc +++ b/lib/tsan/rtl/tsan_clock.cc @@ -101,6 +101,9 @@ ThreadClock::ThreadClock(unsigned tid, unsigned reused) clk_[tid_].reused = reused_; } +void ThreadClock::ResetCached(ClockCache *c) { +} + void ThreadClock::acquire(ClockCache *c, const SyncClock *src) { DCHECK_LE(nclk_, kMaxTid); DCHECK_LE(src->size_, kMaxTid); @@ -116,9 +119,7 @@ void ThreadClock::acquire(ClockCache *c, const SyncClock *src) { // Check if we've already acquired src after the last release operation on src bool acquired = false; if (nclk > tid_) { - CPP_STAT_INC(StatClockAcquireLarge); if (src->elem(tid_).reused == reused_) { - CPP_STAT_INC(StatClockAcquireRepeat); for (unsigned i = 0; i < kDirtyTids; i++) { unsigned tid = src->dirty_tids_[i]; if (tid != kInvalidTid) { @@ -266,11 +267,11 @@ void ThreadClock::UpdateCurrentThread(SyncClock *dst) const { for (unsigned i = 0; i < kDirtyTids; i++) { if (dst->dirty_tids_[i] == tid_) { - CPP_STAT_INC(StatClockReleaseFast1); + CPP_STAT_INC(StatClockReleaseFast); return; } if (dst->dirty_tids_[i] == kInvalidTid) { - CPP_STAT_INC(StatClockReleaseFast2); + CPP_STAT_INC(StatClockReleaseFast); dst->dirty_tids_[i] = tid_; return; } @@ -297,56 +298,9 @@ bool ThreadClock::IsAlreadyAcquired(const SyncClock *src) const { return true; } -void SyncClock::Resize(ClockCache *c, uptr nclk) { - CPP_STAT_INC(StatClockReleaseResize); - if (RoundUpTo(nclk, ClockBlock::kClockCount) <= - RoundUpTo(size_, ClockBlock::kClockCount)) { - // Growing within the same block. - // Memory is already allocated, just increase the size. - size_ = nclk; - return; - } - if (nclk <= ClockBlock::kClockCount) { - // Grow from 0 to one-level table. - CHECK_EQ(size_, 0); - CHECK_EQ(tab_, 0); - CHECK_EQ(tab_idx_, 0); - size_ = nclk; - tab_idx_ = ctx->clock_alloc.Alloc(c); - tab_ = ctx->clock_alloc.Map(tab_idx_); - internal_memset(tab_, 0, sizeof(*tab_)); - return; - } - // Growing two-level table. - if (size_ == 0) { - // Allocate first level table. - tab_idx_ = ctx->clock_alloc.Alloc(c); - tab_ = ctx->clock_alloc.Map(tab_idx_); - internal_memset(tab_, 0, sizeof(*tab_)); - } else if (size_ <= ClockBlock::kClockCount) { - // Transform one-level table to two-level table. - u32 old = tab_idx_; - tab_idx_ = ctx->clock_alloc.Alloc(c); - tab_ = ctx->clock_alloc.Map(tab_idx_); - internal_memset(tab_, 0, sizeof(*tab_)); - tab_->table[0] = old; - } - // At this point we have first level table allocated. - // Add second level tables as necessary. - for (uptr i = RoundUpTo(size_, ClockBlock::kClockCount); - i < nclk; i += ClockBlock::kClockCount) { - u32 idx = ctx->clock_alloc.Alloc(c); - ClockBlock *cb = ctx->clock_alloc.Map(idx); - internal_memset(cb, 0, sizeof(*cb)); - CHECK_EQ(tab_->table[i/ClockBlock::kClockCount], 0); - tab_->table[i/ClockBlock::kClockCount] = idx; - } - size_ = nclk; -} - // Sets a single element in the vector clock. // This function is called only from weird places like AcquireGlobal. -void ThreadClock::set(unsigned tid, u64 v) { +void ThreadClock::set(ClockCache *c, unsigned tid, u64 v) { DCHECK_LT(tid, kMaxTid); DCHECK_GE(v, clk_[tid].epoch); clk_[tid].epoch = v; @@ -366,14 +320,8 @@ void ThreadClock::DebugDump(int(*printf)(const char *s, ...)) { tid_, reused_, last_acquire_); } -SyncClock::SyncClock() - : release_store_tid_(kInvalidTid) - , release_store_reused_() - , tab_() - , tab_idx_() - , size_() { - for (uptr i = 0; i < kDirtyTids; i++) - dirty_tids_[i] = kInvalidTid; +SyncClock::SyncClock() { + ResetImpl(); } SyncClock::~SyncClock() { @@ -395,6 +343,10 @@ void SyncClock::Reset(ClockCache *c) { ctx->clock_alloc.Free(c, tab_->table[i / ClockBlock::kClockCount]); ctx->clock_alloc.Free(c, tab_idx_); } + ResetImpl(); +} + +void SyncClock::ResetImpl() { tab_ = 0; tab_idx_ = 0; size_ = 0; @@ -404,6 +356,53 @@ void SyncClock::Reset(ClockCache *c) { dirty_tids_[i] = kInvalidTid; } +void SyncClock::Resize(ClockCache *c, uptr nclk) { + CPP_STAT_INC(StatClockReleaseResize); + if (RoundUpTo(nclk, ClockBlock::kClockCount) <= + RoundUpTo(size_, ClockBlock::kClockCount)) { + // Growing within the same block. + // Memory is already allocated, just increase the size. + size_ = nclk; + return; + } + if (nclk <= ClockBlock::kClockCount) { + // Grow from 0 to one-level table. + CHECK_EQ(size_, 0); + CHECK_EQ(tab_, 0); + CHECK_EQ(tab_idx_, 0); + size_ = nclk; + tab_idx_ = ctx->clock_alloc.Alloc(c); + tab_ = ctx->clock_alloc.Map(tab_idx_); + internal_memset(tab_, 0, sizeof(*tab_)); + return; + } + // Growing two-level table. + if (size_ == 0) { + // Allocate first level table. + tab_idx_ = ctx->clock_alloc.Alloc(c); + tab_ = ctx->clock_alloc.Map(tab_idx_); + internal_memset(tab_, 0, sizeof(*tab_)); + } else if (size_ <= ClockBlock::kClockCount) { + // Transform one-level table to two-level table. + u32 old = tab_idx_; + tab_idx_ = ctx->clock_alloc.Alloc(c); + tab_ = ctx->clock_alloc.Map(tab_idx_); + internal_memset(tab_, 0, sizeof(*tab_)); + tab_->table[0] = old; + } + // At this point we have first level table allocated. + // Add second level tables as necessary. + for (uptr i = RoundUpTo(size_, ClockBlock::kClockCount); + i < nclk; i += ClockBlock::kClockCount) { + u32 idx = ctx->clock_alloc.Alloc(c); + ClockBlock *cb = ctx->clock_alloc.Map(idx); + internal_memset(cb, 0, sizeof(*cb)); + CHECK_EQ(tab_->table[i/ClockBlock::kClockCount], 0); + tab_->table[i/ClockBlock::kClockCount] = idx; + } + size_ = nclk; +} + ClockElem &SyncClock::elem(unsigned tid) const { DCHECK_LT(tid, size_); if (size_ <= ClockBlock::kClockCount) diff --git a/lib/tsan/rtl/tsan_clock.h b/lib/tsan/rtl/tsan_clock.h index 4e352cb81d11..378b550fd11b 100644 --- a/lib/tsan/rtl/tsan_clock.h +++ b/lib/tsan/rtl/tsan_clock.h @@ -74,6 +74,7 @@ class SyncClock { u32 tab_idx_; u32 size_; + void ResetImpl(); ClockElem &elem(unsigned tid) const; }; @@ -89,7 +90,7 @@ struct ThreadClock { return clk_[tid].epoch; } - void set(unsigned tid, u64 v); + void set(ClockCache *c, unsigned tid, u64 v); void set(u64 v) { DCHECK_GE(v, clk_[tid_].epoch); @@ -108,6 +109,7 @@ struct ThreadClock { void release(ClockCache *c, SyncClock *dst) const; void acq_rel(ClockCache *c, SyncClock *dst); void ReleaseStore(ClockCache *c, SyncClock *dst) const; + void ResetCached(ClockCache *c); void DebugReset(); void DebugDump(int(*printf)(const char *s, ...)); diff --git a/lib/tsan/rtl/tsan_dense_alloc.h b/lib/tsan/rtl/tsan_dense_alloc.h index e9815c90a953..16dbdf391085 100644 --- a/lib/tsan/rtl/tsan_dense_alloc.h +++ b/lib/tsan/rtl/tsan_dense_alloc.h @@ -39,7 +39,7 @@ class DenseSlabAlloc { typedef DenseSlabAllocCache Cache; typedef typename Cache::IndexT IndexT; - DenseSlabAlloc() { + explicit DenseSlabAlloc(const char *name) { // Check that kL1Size and kL2Size are sane. CHECK_EQ(kL1Size & (kL1Size - 1), 0); CHECK_EQ(kL2Size & (kL2Size - 1), 0); @@ -49,6 +49,7 @@ class DenseSlabAlloc { internal_memset(map_, 0, sizeof(map_)); freelist_ = 0; fillpos_ = 0; + name_ = name; } ~DenseSlabAlloc() { @@ -96,15 +97,19 @@ class DenseSlabAlloc { SpinMutex mtx_; IndexT freelist_; uptr fillpos_; + const char *name_; void Refill(Cache *c) { SpinMutexLock lock(&mtx_); if (freelist_ == 0) { if (fillpos_ == kL1Size) { - Printf("ThreadSanitizer: DenseSlabAllocator overflow. Dying.\n"); + Printf("ThreadSanitizer: %s overflow (%zu*%zu). Dying.\n", + name_, kL1Size, kL2Size); Die(); } - T *batch = (T*)MmapOrDie(kL2Size * sizeof(T), "DenseSlabAllocator"); + VPrintf(2, "ThreadSanitizer: growing %s: %zu out of %zu*%zu\n", + name_, fillpos_, kL1Size, kL2Size); + T *batch = (T*)MmapOrDie(kL2Size * sizeof(T), name_); // Reserve 0 as invalid index. IndexT start = fillpos_ == 0 ? 1 : 0; for (IndexT i = start; i < kL2Size; i++) { diff --git a/lib/tsan/rtl/tsan_interceptors.cc b/lib/tsan/rtl/tsan_interceptors.cc index 334cc326daf6..001123f4941e 100644 --- a/lib/tsan/rtl/tsan_interceptors.cc +++ b/lib/tsan/rtl/tsan_interceptors.cc @@ -14,6 +14,7 @@ //===----------------------------------------------------------------------===// #include "sanitizer_common/sanitizer_atomic.h" +#include "sanitizer_common/sanitizer_errno.h" #include "sanitizer_common/sanitizer_libc.h" #include "sanitizer_common/sanitizer_linux.h" #include "sanitizer_common/sanitizer_platform_limits_posix.h" @@ -34,13 +35,11 @@ using namespace __tsan; // NOLINT #if SANITIZER_FREEBSD || SANITIZER_MAC -#define __errno_location __error #define stdout __stdoutp #define stderr __stderrp #endif #if SANITIZER_ANDROID -#define __errno_location __errno #define mallopt(a, b) #endif @@ -84,7 +83,6 @@ DECLARE_REAL_AND_INTERCEPTOR(void *, malloc, uptr size) DECLARE_REAL_AND_INTERCEPTOR(void, free, void *ptr) extern "C" void *pthread_self(); extern "C" void _exit(int status); -extern "C" int *__errno_location(); extern "C" int fileno_unlocked(void *stream); extern "C" int dirfd(void *dirp); #if !SANITIZER_FREEBSD && !SANITIZER_ANDROID @@ -98,9 +96,6 @@ const int PTHREAD_MUTEX_RECURSIVE_NP = 1; const int PTHREAD_MUTEX_RECURSIVE = 2; const int PTHREAD_MUTEX_RECURSIVE_NP = 2; #endif -const int EINVAL = 22; -const int EBUSY = 16; -const int EOWNERDEAD = 130; #if !SANITIZER_FREEBSD && !SANITIZER_MAC const int EPOLL_CTL_ADD = 1; #endif @@ -130,8 +125,6 @@ typedef long long_t; // NOLINT # define F_TLOCK 2 /* Test and lock a region for exclusive use. */ # define F_TEST 3 /* Test a region for other processes locks. */ -#define errno (*__errno_location()) - typedef void (*sighandler_t)(int sig); typedef void (*sigactionhandler_t)(int sig, my_siginfo_t *siginfo, void *uctx); @@ -268,7 +261,7 @@ ScopedInterceptor::~ScopedInterceptor() { void ScopedInterceptor::EnableIgnores() { if (ignoring_) { - ThreadIgnoreBegin(thr_, pc_, false); + ThreadIgnoreBegin(thr_, pc_, /*save_stack=*/false); if (flags()->ignore_noninstrumented_modules) thr_->suppress_reports++; if (in_ignored_lib_) { DCHECK(!thr_->in_ignored_lib); @@ -466,8 +459,14 @@ static void SetJmp(ThreadState *thr, uptr sp, uptr mangled_sp) { static void LongJmp(ThreadState *thr, uptr *env) { #ifdef __powerpc__ uptr mangled_sp = env[0]; -#elif SANITIZER_FREEBSD || SANITIZER_MAC +#elif SANITIZER_FREEBSD uptr mangled_sp = env[2]; +#elif SANITIZER_MAC +# ifdef __aarch64__ + uptr mangled_sp = env[13]; +# else + uptr mangled_sp = env[2]; +# endif #elif defined(SANITIZER_LINUX) # ifdef __aarch64__ uptr mangled_sp = env[13]; @@ -665,7 +664,7 @@ static bool fix_mmap_addr(void **addr, long_t sz, int flags) { if (*addr) { if (!IsAppMem((uptr)*addr) || !IsAppMem((uptr)*addr + sz - 1)) { if (flags & MAP_FIXED) { - errno = EINVAL; + errno = errno_EINVAL; return false; } else { *addr = 0; @@ -1122,7 +1121,7 @@ TSAN_INTERCEPTOR(int, pthread_mutex_init, void *m, void *a) { TSAN_INTERCEPTOR(int, pthread_mutex_destroy, void *m) { SCOPED_TSAN_INTERCEPTOR(pthread_mutex_destroy, m); int res = REAL(pthread_mutex_destroy)(m); - if (res == 0 || res == EBUSY) { + if (res == 0 || res == errno_EBUSY) { MutexDestroy(thr, pc, (uptr)m); } return res; @@ -1131,9 +1130,9 @@ TSAN_INTERCEPTOR(int, pthread_mutex_destroy, void *m) { TSAN_INTERCEPTOR(int, pthread_mutex_trylock, void *m) { SCOPED_TSAN_INTERCEPTOR(pthread_mutex_trylock, m); int res = REAL(pthread_mutex_trylock)(m); - if (res == EOWNERDEAD) + if (res == errno_EOWNERDEAD) MutexRepair(thr, pc, (uptr)m); - if (res == 0 || res == EOWNERDEAD) + if (res == 0 || res == errno_EOWNERDEAD) MutexPostLock(thr, pc, (uptr)m, MutexFlagTryLock); return res; } @@ -1311,7 +1310,7 @@ TSAN_INTERCEPTOR(int, pthread_barrier_wait, void *b) { TSAN_INTERCEPTOR(int, pthread_once, void *o, void (*f)()) { SCOPED_INTERCEPTOR_RAW(pthread_once, o, f); if (o == 0 || f == 0) - return EINVAL; + return errno_EINVAL; atomic_uint32_t *a; if (!SANITIZER_MAC) a = static_cast<atomic_uint32_t*>(o); diff --git a/lib/tsan/rtl/tsan_interceptors_mac.cc b/lib/tsan/rtl/tsan_interceptors_mac.cc index f6bf8a0e586b..4f1079467331 100644 --- a/lib/tsan/rtl/tsan_interceptors_mac.cc +++ b/lib/tsan/rtl/tsan_interceptors_mac.cc @@ -21,7 +21,10 @@ #include "tsan_interface_ann.h" #include <libkern/OSAtomic.h> + +#if defined(__has_include) && __has_include(<xpc/xpc.h>) #include <xpc/xpc.h> +#endif // #if defined(__has_include) && __has_include(<xpc/xpc.h>) typedef long long_t; // NOLINT @@ -235,6 +238,8 @@ TSAN_INTERCEPTOR(void, os_lock_unlock, void *lock) { REAL(os_lock_unlock)(lock); } +#if defined(__has_include) && __has_include(<xpc/xpc.h>) + TSAN_INTERCEPTOR(void, xpc_connection_set_event_handler, xpc_connection_t connection, xpc_handler_t handler) { SCOPED_TSAN_INTERCEPTOR(xpc_connection_set_event_handler, connection, @@ -287,6 +292,8 @@ TSAN_INTERCEPTOR(void, xpc_connection_cancel, xpc_connection_t connection) { REAL(xpc_connection_cancel)(connection); } +#endif // #if defined(__has_include) && __has_include(<xpc/xpc.h>) + // On macOS, libc++ is always linked dynamically, so intercepting works the // usual way. #define STDCXX_INTERCEPTOR TSAN_INTERCEPTOR diff --git a/lib/tsan/rtl/tsan_interface_ann.cc b/lib/tsan/rtl/tsan_interface_ann.cc index 45ec45bbdbbe..f68a0468de53 100644 --- a/lib/tsan/rtl/tsan_interface_ann.cc +++ b/lib/tsan/rtl/tsan_interface_ann.cc @@ -483,8 +483,8 @@ void __tsan_mutex_pre_lock(void *m, unsigned flagz) { else MutexPreLock(thr, pc, (uptr)m); } - ThreadIgnoreBegin(thr, pc, false); - ThreadIgnoreSyncBegin(thr, pc, false); + ThreadIgnoreBegin(thr, pc, /*save_stack=*/false); + ThreadIgnoreSyncBegin(thr, pc, /*save_stack=*/false); } INTERFACE_ATTRIBUTE @@ -510,8 +510,8 @@ int __tsan_mutex_pre_unlock(void *m, unsigned flagz) { } else { ret = MutexUnlock(thr, pc, (uptr)m, flagz); } - ThreadIgnoreBegin(thr, pc, false); - ThreadIgnoreSyncBegin(thr, pc, false); + ThreadIgnoreBegin(thr, pc, /*save_stack=*/false); + ThreadIgnoreSyncBegin(thr, pc, /*save_stack=*/false); return ret; } @@ -525,8 +525,8 @@ void __tsan_mutex_post_unlock(void *m, unsigned flagz) { INTERFACE_ATTRIBUTE void __tsan_mutex_pre_signal(void *addr, unsigned flagz) { SCOPED_ANNOTATION(__tsan_mutex_pre_signal); - ThreadIgnoreBegin(thr, pc, false); - ThreadIgnoreSyncBegin(thr, pc, false); + ThreadIgnoreBegin(thr, pc, /*save_stack=*/false); + ThreadIgnoreSyncBegin(thr, pc, /*save_stack=*/false); } INTERFACE_ATTRIBUTE @@ -547,7 +547,7 @@ void __tsan_mutex_pre_divert(void *addr, unsigned flagz) { INTERFACE_ATTRIBUTE void __tsan_mutex_post_divert(void *addr, unsigned flagz) { SCOPED_ANNOTATION(__tsan_mutex_post_divert); - ThreadIgnoreBegin(thr, pc, false); - ThreadIgnoreSyncBegin(thr, pc, false); + ThreadIgnoreBegin(thr, pc, /*save_stack=*/false); + ThreadIgnoreSyncBegin(thr, pc, /*save_stack=*/false); } } // extern "C" diff --git a/lib/tsan/rtl/tsan_interface_atomic.cc b/lib/tsan/rtl/tsan_interface_atomic.cc index b22d5c1ecef8..d334394f5303 100644 --- a/lib/tsan/rtl/tsan_interface_atomic.cc +++ b/lib/tsan/rtl/tsan_interface_atomic.cc @@ -220,8 +220,7 @@ static a128 NoTsanAtomicLoad(const volatile a128 *a, morder mo) { #endif template<typename T> -static T AtomicLoad(ThreadState *thr, uptr pc, const volatile T *a, - morder mo) { +static T AtomicLoad(ThreadState *thr, uptr pc, const volatile T *a, morder mo) { CHECK(IsLoadOrder(mo)); // This fast-path is critical for performance. // Assume the access is atomic. @@ -229,10 +228,17 @@ static T AtomicLoad(ThreadState *thr, uptr pc, const volatile T *a, MemoryReadAtomic(thr, pc, (uptr)a, SizeLog<T>()); return NoTsanAtomicLoad(a, mo); } - SyncVar *s = ctx->metamap.GetOrCreateAndLock(thr, pc, (uptr)a, false); - AcquireImpl(thr, pc, &s->clock); + // Don't create sync object if it does not exist yet. For example, an atomic + // pointer is initialized to nullptr and then periodically acquire-loaded. T v = NoTsanAtomicLoad(a, mo); - s->mtx.ReadUnlock(); + SyncVar *s = ctx->metamap.GetIfExistsAndLock((uptr)a, false); + if (s) { + AcquireImpl(thr, pc, &s->clock); + // Re-read under sync mutex because we need a consistent snapshot + // of the value and the clock we acquire. + v = NoTsanAtomicLoad(a, mo); + s->mtx.ReadUnlock(); + } MemoryReadAtomic(thr, pc, (uptr)a, SizeLog<T>()); return v; } diff --git a/lib/tsan/rtl/tsan_mman.cc b/lib/tsan/rtl/tsan_mman.cc index 7169d5b02c04..1434cf688ce9 100644 --- a/lib/tsan/rtl/tsan_mman.cc +++ b/lib/tsan/rtl/tsan_mman.cc @@ -294,6 +294,8 @@ uptr __sanitizer_get_allocated_size(const void *p) { void __tsan_on_thread_idle() { ThreadState *thr = cur_thread(); + thr->clock.ResetCached(&thr->proc()->clock_cache); + thr->last_sleep_clock.ResetCached(&thr->proc()->clock_cache); allocator()->SwallowCache(&thr->proc()->alloc_cache); internal_allocator()->SwallowCache(&thr->proc()->internal_alloc_cache); ctx->metamap.OnProcIdle(thr->proc()); diff --git a/lib/tsan/rtl/tsan_platform.h b/lib/tsan/rtl/tsan_platform.h index 60d9b9d8c452..bea1daba3952 100644 --- a/lib/tsan/rtl/tsan_platform.h +++ b/lib/tsan/rtl/tsan_platform.h @@ -100,6 +100,37 @@ struct Mapping { }; #define TSAN_MID_APP_RANGE 1 +#elif defined(__aarch64__) && defined(__APPLE__) +/* +C/C++ on Darwin/iOS/ARM64 (36-bit VMA, 64 GB VM) +0000 0000 00 - 0100 0000 00: - (4 GB) +0100 0000 00 - 0200 0000 00: main binary, modules, thread stacks (4 GB) +0200 0000 00 - 0300 0000 00: heap (4 GB) +0300 0000 00 - 0400 0000 00: - (4 GB) +0400 0000 00 - 0c00 0000 00: shadow memory (32 GB) +0c00 0000 00 - 0d00 0000 00: - (4 GB) +0d00 0000 00 - 0e00 0000 00: metainfo (4 GB) +0e00 0000 00 - 0f00 0000 00: - (4 GB) +0f00 0000 00 - 1000 0000 00: traces (4 GB) +*/ +struct Mapping { + static const uptr kLoAppMemBeg = 0x0100000000ull; + static const uptr kLoAppMemEnd = 0x0200000000ull; + static const uptr kHeapMemBeg = 0x0200000000ull; + static const uptr kHeapMemEnd = 0x0300000000ull; + static const uptr kShadowBeg = 0x0400000000ull; + static const uptr kShadowEnd = 0x0c00000000ull; + static const uptr kMetaShadowBeg = 0x0d00000000ull; + static const uptr kMetaShadowEnd = 0x0e00000000ull; + static const uptr kTraceMemBeg = 0x0f00000000ull; + static const uptr kTraceMemEnd = 0x1000000000ull; + static const uptr kHiAppMemBeg = 0x1000000000ull; + static const uptr kHiAppMemEnd = 0x1000000000ull; + static const uptr kAppMemMsk = 0x0ull; + static const uptr kAppMemXor = 0x0ull; + static const uptr kVdsoBeg = 0x7000000000000000ull; +}; + #elif defined(__aarch64__) // AArch64 supports multiple VMA which leads to multiple address transformation // functions. To support these multiple VMAS transformations and mappings TSAN @@ -389,7 +420,7 @@ uptr MappingImpl(void) { template<int Type> uptr MappingArchImpl(void) { -#ifdef __aarch64__ +#if defined(__aarch64__) && !defined(__APPLE__) switch (vmaSize) { case 39: return MappingImpl<Mapping39, Type>(); case 42: return MappingImpl<Mapping42, Type>(); @@ -542,7 +573,7 @@ bool IsAppMemImpl(uptr mem) { ALWAYS_INLINE bool IsAppMem(uptr mem) { -#ifdef __aarch64__ +#if defined(__aarch64__) && !defined(__APPLE__) switch (vmaSize) { case 39: return IsAppMemImpl<Mapping39>(mem); case 42: return IsAppMemImpl<Mapping42>(mem); @@ -569,7 +600,7 @@ bool IsShadowMemImpl(uptr mem) { ALWAYS_INLINE bool IsShadowMem(uptr mem) { -#ifdef __aarch64__ +#if defined(__aarch64__) && !defined(__APPLE__) switch (vmaSize) { case 39: return IsShadowMemImpl<Mapping39>(mem); case 42: return IsShadowMemImpl<Mapping42>(mem); @@ -596,7 +627,7 @@ bool IsMetaMemImpl(uptr mem) { ALWAYS_INLINE bool IsMetaMem(uptr mem) { -#ifdef __aarch64__ +#if defined(__aarch64__) && !defined(__APPLE__) switch (vmaSize) { case 39: return IsMetaMemImpl<Mapping39>(mem); case 42: return IsMetaMemImpl<Mapping42>(mem); @@ -633,7 +664,7 @@ uptr MemToShadowImpl(uptr x) { ALWAYS_INLINE uptr MemToShadow(uptr x) { -#ifdef __aarch64__ +#if defined(__aarch64__) && !defined(__APPLE__) switch (vmaSize) { case 39: return MemToShadowImpl<Mapping39>(x); case 42: return MemToShadowImpl<Mapping42>(x); @@ -672,7 +703,7 @@ u32 *MemToMetaImpl(uptr x) { ALWAYS_INLINE u32 *MemToMeta(uptr x) { -#ifdef __aarch64__ +#if defined(__aarch64__) && !defined(__APPLE__) switch (vmaSize) { case 39: return MemToMetaImpl<Mapping39>(x); case 42: return MemToMetaImpl<Mapping42>(x); @@ -724,7 +755,7 @@ uptr ShadowToMemImpl(uptr s) { ALWAYS_INLINE uptr ShadowToMem(uptr s) { -#ifdef __aarch64__ +#if defined(__aarch64__) && !defined(__APPLE__) switch (vmaSize) { case 39: return ShadowToMemImpl<Mapping39>(s); case 42: return ShadowToMemImpl<Mapping42>(s); @@ -759,7 +790,7 @@ uptr GetThreadTraceImpl(int tid) { ALWAYS_INLINE uptr GetThreadTrace(int tid) { -#ifdef __aarch64__ +#if defined(__aarch64__) && !defined(__APPLE__) switch (vmaSize) { case 39: return GetThreadTraceImpl<Mapping39>(tid); case 42: return GetThreadTraceImpl<Mapping42>(tid); @@ -789,7 +820,7 @@ uptr GetThreadTraceHeaderImpl(int tid) { ALWAYS_INLINE uptr GetThreadTraceHeader(int tid) { -#ifdef __aarch64__ +#if defined(__aarch64__) && !defined(__APPLE__) switch (vmaSize) { case 39: return GetThreadTraceHeaderImpl<Mapping39>(tid); case 42: return GetThreadTraceHeaderImpl<Mapping42>(tid); diff --git a/lib/tsan/rtl/tsan_platform_linux.cc b/lib/tsan/rtl/tsan_platform_linux.cc index d05c0e701e72..0ba01babe69a 100644 --- a/lib/tsan/rtl/tsan_platform_linux.cc +++ b/lib/tsan/rtl/tsan_platform_linux.cc @@ -47,7 +47,6 @@ #include <sys/resource.h> #include <sys/stat.h> #include <unistd.h> -#include <errno.h> #include <sched.h> #include <dlfcn.h> #if SANITIZER_LINUX @@ -182,17 +181,15 @@ static void MapRodata() { } // Map the file into shadow of .rodata sections. MemoryMappingLayout proc_maps(/*cache_enabled*/true); - uptr start, end, offset, prot; // Reusing the buffer 'name'. - while (proc_maps.Next(&start, &end, &offset, name, ARRAY_SIZE(name), &prot)) { - if (name[0] != 0 && name[0] != '[' - && (prot & MemoryMappingLayout::kProtectionRead) - && (prot & MemoryMappingLayout::kProtectionExecute) - && !(prot & MemoryMappingLayout::kProtectionWrite) - && IsAppMem(start)) { + MemoryMappedSegment segment(name, ARRAY_SIZE(name)); + while (proc_maps.Next(&segment)) { + if (segment.filename[0] != 0 && segment.filename[0] != '[' && + segment.IsReadable() && segment.IsExecutable() && + !segment.IsWritable() && IsAppMem(segment.start)) { // Assume it's .rodata - char *shadow_start = (char*)MemToShadow(start); - char *shadow_end = (char*)MemToShadow(end); + char *shadow_start = (char *)MemToShadow(segment.start); + char *shadow_end = (char *)MemToShadow(segment.end); for (char *p = shadow_start; p < shadow_end; p += marker.size()) { internal_mmap(p, Min<uptr>(marker.size(), shadow_end - p), PROT_READ, MAP_PRIVATE | MAP_FIXED, fd, 0); diff --git a/lib/tsan/rtl/tsan_platform_mac.cc b/lib/tsan/rtl/tsan_platform_mac.cc index a82bcd01bbf4..73a656ffca5e 100644 --- a/lib/tsan/rtl/tsan_platform_mac.cc +++ b/lib/tsan/rtl/tsan_platform_mac.cc @@ -230,6 +230,14 @@ static void my_pthread_introspection_hook(unsigned int event, pthread_t thread, #endif void InitializePlatformEarly() { +#if defined(__aarch64__) + uptr max_vm = GetMaxVirtualAddress() + 1; + if (max_vm != Mapping::kHiAppMemEnd) { + Printf("ThreadSanitizer: unsupported vm address limit %p, expected %p.\n", + max_vm, Mapping::kHiAppMemEnd); + Die(); + } +#endif } void InitializePlatform() { diff --git a/lib/tsan/rtl/tsan_platform_posix.cc b/lib/tsan/rtl/tsan_platform_posix.cc index 0732c83d689d..e4f90a811c35 100644 --- a/lib/tsan/rtl/tsan_platform_posix.cc +++ b/lib/tsan/rtl/tsan_platform_posix.cc @@ -46,6 +46,9 @@ void InitializeShadowMemory() { #elif defined(__mips64) const uptr kMadviseRangeBeg = 0xff00000000ull; const uptr kMadviseRangeSize = 0x0100000000ull; +#elif defined(__aarch64__) && defined(__APPLE__) + uptr kMadviseRangeBeg = LoAppMemBeg(); + uptr kMadviseRangeSize = LoAppMemEnd() - LoAppMemBeg(); #elif defined(__aarch64__) uptr kMadviseRangeBeg = 0; uptr kMadviseRangeSize = 0; @@ -115,21 +118,24 @@ static void ProtectRange(uptr beg, uptr end) { void CheckAndProtect() { // Ensure that the binary is indeed compiled with -pie. MemoryMappingLayout proc_maps(true); - uptr p, end, prot; - while (proc_maps.Next(&p, &end, 0, 0, 0, &prot)) { - if (IsAppMem(p)) + MemoryMappedSegment segment; + while (proc_maps.Next(&segment)) { + if (IsAppMem(segment.start)) continue; + if (segment.start >= HeapMemEnd() && segment.start < HeapEnd()) continue; + if (segment.protection == 0) // Zero page or mprotected. continue; - if (p >= HeapMemEnd() && - p < HeapEnd()) - continue; - if (prot == 0) // Zero page or mprotected. - continue; - if (p >= VdsoBeg()) // vdso + if (segment.start >= VdsoBeg()) // vdso break; - Printf("FATAL: ThreadSanitizer: unexpected memory mapping %p-%p\n", p, end); + Printf("FATAL: ThreadSanitizer: unexpected memory mapping %p-%p\n", + segment.start, segment.end); Die(); } +#if defined(__aarch64__) && defined(__APPLE__) + ProtectRange(HeapMemEnd(), ShadowBeg()); + ProtectRange(ShadowEnd(), MetaShadowBeg()); + ProtectRange(MetaShadowEnd(), TraceMemBeg()); +#else ProtectRange(LoAppMemEnd(), ShadowBeg()); ProtectRange(ShadowEnd(), MetaShadowBeg()); #ifdef TSAN_MID_APP_RANGE @@ -143,6 +149,7 @@ void CheckAndProtect() { ProtectRange(TraceMemBeg(), TraceMemEnd()); ProtectRange(TraceMemEnd(), HeapMemBeg()); ProtectRange(HeapEnd(), HiAppMemBeg()); +#endif } #endif diff --git a/lib/tsan/rtl/tsan_rtl.cc b/lib/tsan/rtl/tsan_rtl.cc index fa60f3247c38..a01525302b02 100644 --- a/lib/tsan/rtl/tsan_rtl.cc +++ b/lib/tsan/rtl/tsan_rtl.cc @@ -104,7 +104,8 @@ Context::Context() , racy_stacks(MBlockRacyStacks) , racy_addresses(MBlockRacyAddresses) , fired_suppressions_mtx(MutexTypeFired, StatMtxFired) - , fired_suppressions(8) { + , fired_suppressions(8) + , clock_alloc("clock allocator") { } // The objects are allocated in TLS, so one may rely on zero-initialization. diff --git a/lib/tsan/rtl/tsan_rtl_aarch64.S b/lib/tsan/rtl/tsan_rtl_aarch64.S index ef06f0444ae4..61171d635c18 100644 --- a/lib/tsan/rtl/tsan_rtl_aarch64.S +++ b/lib/tsan/rtl/tsan_rtl_aarch64.S @@ -1,13 +1,46 @@ +// The content of this file is AArch64-only: +#if defined(__aarch64__) + #include "sanitizer_common/sanitizer_asm.h" +#if !defined(__APPLE__) .section .bss .type __tsan_pointer_chk_guard, %object -.size __tsan_pointer_chk_guard, 8 +ASM_SIZE(ASM_TSAN_SYMBOL_INTERCEPTOR(__tsan_pointer_chk_guard)) __tsan_pointer_chk_guard: .zero 8 +#endif + +#if defined(__APPLE__) +.align 2 + +.section __DATA,__nl_symbol_ptr,non_lazy_symbol_pointers +.long _setjmp$non_lazy_ptr +_setjmp$non_lazy_ptr: +.indirect_symbol _setjmp +.long 0 + +.section __DATA,__nl_symbol_ptr,non_lazy_symbol_pointers +.long __setjmp$non_lazy_ptr +__setjmp$non_lazy_ptr: +.indirect_symbol __setjmp +.long 0 + +.section __DATA,__nl_symbol_ptr,non_lazy_symbol_pointers +.long _sigsetjmp$non_lazy_ptr +_sigsetjmp$non_lazy_ptr: +.indirect_symbol _sigsetjmp +.long 0 +#endif +#if !defined(__APPLE__) .section .text +#else +.section __TEXT,__text +.align 3 +#endif +#if !defined(__APPLE__) // GLIBC mangles the function pointers in jmp_buf (used in {set,long}*jmp // functions) by XORing them with a random guard pointer. For AArch64 it is a // global variable rather than a TCB one (as for x86_64/powerpc) and althought @@ -16,9 +49,9 @@ __tsan_pointer_chk_guard: // not stable). So InitializeGuardPtr obtains the pointer guard value by // issuing a setjmp and checking the resulting pointers values against the // original ones. -.hidden _Z18InitializeGuardPtrv +ASM_HIDDEN(_Z18InitializeGuardPtrv) .global _Z18InitializeGuardPtrv -.type _Z18InitializeGuardPtrv, @function +ASM_TYPE_FUNCTION(ASM_TSAN_SYMBOL_INTERCEPTOR(_Z18InitializeGuardPtrv)) _Z18InitializeGuardPtrv: CFI_STARTPROC // Allocates a jmp_buf for the setjmp call. @@ -55,12 +88,14 @@ _Z18InitializeGuardPtrv: CFI_DEF_CFA (31, 0) ret CFI_ENDPROC -.size _Z18InitializeGuardPtrv, .-_Z18InitializeGuardPtrv +ASM_SIZE(ASM_TSAN_SYMBOL_INTERCEPTOR(_Z18InitializeGuardPtrv)) +#endif -.hidden __tsan_setjmp +ASM_HIDDEN(__tsan_setjmp) .comm _ZN14__interception11real_setjmpE,8,8 -.type setjmp, @function -setjmp: +.globl ASM_TSAN_SYMBOL_INTERCEPTOR(setjmp) +ASM_TYPE_FUNCTION(ASM_TSAN_SYMBOL_INTERCEPTOR(setjmp)) +ASM_TSAN_SYMBOL_INTERCEPTOR(setjmp): CFI_STARTPROC // save env parameters for function call @@ -78,14 +113,19 @@ setjmp: CFI_OFFSET (19, -16) mov x19, x0 +#if !defined(__APPLE__) // SP pointer mangling (see glibc setjmp) adrp x2, __tsan_pointer_chk_guard ldr x2, [x2, #:lo12:__tsan_pointer_chk_guard] add x0, x29, 32 eor x1, x2, x0 +#else + add x0, x29, 32 + mov x1, x0 +#endif // call tsan interceptor - bl __tsan_setjmp + bl ASM_TSAN_SYMBOL(__tsan_setjmp) // restore env parameter mov x0, x19 @@ -96,18 +136,24 @@ setjmp: CFI_DEF_CFA (31, 0) // tail jump to libc setjmp +#if !defined(__APPLE__) adrp x1, :got:_ZN14__interception11real_setjmpE ldr x1, [x1, #:got_lo12:_ZN14__interception11real_setjmpE] ldr x1, [x1] +#else + adrp x1, _setjmp$non_lazy_ptr@page + add x1, x1, _setjmp$non_lazy_ptr@pageoff + ldr x1, [x1] +#endif br x1 CFI_ENDPROC -.size setjmp, .-setjmp +ASM_SIZE(ASM_TSAN_SYMBOL_INTERCEPTOR(setjmp)) .comm _ZN14__interception12real__setjmpE,8,8 -.globl _setjmp -.type _setjmp, @function -_setjmp: +.globl ASM_TSAN_SYMBOL_INTERCEPTOR(_setjmp) +ASM_TYPE_FUNCTION(ASM_TSAN_SYMBOL_INTERCEPTOR(_setjmp)) +ASM_TSAN_SYMBOL_INTERCEPTOR(_setjmp): CFI_STARTPROC // save env parameters for function call @@ -125,14 +171,19 @@ _setjmp: CFI_OFFSET (19, -16) mov x19, x0 +#if !defined(__APPLE__) // SP pointer mangling (see glibc setjmp) adrp x2, __tsan_pointer_chk_guard ldr x2, [x2, #:lo12:__tsan_pointer_chk_guard] add x0, x29, 32 eor x1, x2, x0 +#else + add x0, x29, 32 + mov x1, x0 +#endif // call tsan interceptor - bl __tsan_setjmp + bl ASM_TSAN_SYMBOL(__tsan_setjmp) // Restore jmp_buf parameter mov x0, x19 @@ -143,18 +194,24 @@ _setjmp: CFI_DEF_CFA (31, 0) // tail jump to libc setjmp +#if !defined(__APPLE__) adrp x1, :got:_ZN14__interception12real__setjmpE ldr x1, [x1, #:got_lo12:_ZN14__interception12real__setjmpE] ldr x1, [x1] +#else + adrp x1, __setjmp$non_lazy_ptr@page + add x1, x1, __setjmp$non_lazy_ptr@pageoff + ldr x1, [x1] +#endif br x1 CFI_ENDPROC -.size _setjmp, .-_setjmp +ASM_SIZE(ASM_TSAN_SYMBOL_INTERCEPTOR(_setjmp)) .comm _ZN14__interception14real_sigsetjmpE,8,8 -.globl sigsetjmp -.type sigsetjmp, @function -sigsetjmp: +.globl ASM_TSAN_SYMBOL_INTERCEPTOR(sigsetjmp) +ASM_TYPE_FUNCTION(ASM_TSAN_SYMBOL_INTERCEPTOR(sigsetjmp)) +ASM_TSAN_SYMBOL_INTERCEPTOR(sigsetjmp): CFI_STARTPROC // save env parameters for function call @@ -174,14 +231,19 @@ sigsetjmp: mov w20, w1 mov x19, x0 +#if !defined(__APPLE__) // SP pointer mangling (see glibc setjmp) adrp x2, __tsan_pointer_chk_guard ldr x2, [x2, #:lo12:__tsan_pointer_chk_guard] add x0, x29, 32 eor x1, x2, x0 +#else + add x0, x29, 32 + mov x1, x0 +#endif // call tsan interceptor - bl __tsan_setjmp + bl ASM_TSAN_SYMBOL(__tsan_setjmp) // restore env parameter mov w1, w20 @@ -195,17 +257,24 @@ sigsetjmp: CFI_DEF_CFA (31, 0) // tail jump to libc sigsetjmp +#if !defined(__APPLE__) adrp x2, :got:_ZN14__interception14real_sigsetjmpE ldr x2, [x2, #:got_lo12:_ZN14__interception14real_sigsetjmpE] ldr x2, [x2] +#else + adrp x2, _sigsetjmp$non_lazy_ptr@page + add x2, x2, _sigsetjmp$non_lazy_ptr@pageoff + ldr x2, [x2] +#endif br x2 CFI_ENDPROC -.size sigsetjmp, .-sigsetjmp +ASM_SIZE(ASM_TSAN_SYMBOL_INTERCEPTOR(sigsetjmp)) +#if !defined(__APPLE__) .comm _ZN14__interception16real___sigsetjmpE,8,8 -.globl __sigsetjmp -.type __sigsetjmp, @function -__sigsetjmp: +.globl ASM_TSAN_SYMBOL_INTERCEPTOR(__sigsetjmp) +ASM_TYPE_FUNCTION(ASM_TSAN_SYMBOL_INTERCEPTOR(__sigsetjmp)) +ASM_TSAN_SYMBOL_INTERCEPTOR(__sigsetjmp): CFI_STARTPROC // save env parameters for function call @@ -225,14 +294,16 @@ __sigsetjmp: mov w20, w1 mov x19, x0 +#if !defined(__APPLE__) // SP pointer mangling (see glibc setjmp) adrp x2, __tsan_pointer_chk_guard ldr x2, [x2, #:lo12:__tsan_pointer_chk_guard] add x0, x29, 32 eor x1, x2, x0 +#endif // call tsan interceptor - bl __tsan_setjmp + bl ASM_TSAN_SYMBOL(__tsan_setjmp) mov w1, w20 mov x0, x19 @@ -245,14 +316,22 @@ __sigsetjmp: CFI_DEF_CFA (31, 0) // tail jump to libc __sigsetjmp +#if !defined(__APPLE__) adrp x2, :got:_ZN14__interception16real___sigsetjmpE ldr x2, [x2, #:got_lo12:_ZN14__interception16real___sigsetjmpE] ldr x2, [x2] +#else + adrp x2, ASM_TSAN_SYMBOL(__sigsetjmp)@page + add x2, x2, ASM_TSAN_SYMBOL(__sigsetjmp)@pageoff +#endif br x2 CFI_ENDPROC -.size __sigsetjmp, .-__sigsetjmp +ASM_SIZE(ASM_TSAN_SYMBOL_INTERCEPTOR(__sigsetjmp)) +#endif #if defined(__linux__) /* We do not need executable stack. */ .section .note.GNU-stack,"",@progbits #endif + +#endif diff --git a/lib/tsan/rtl/tsan_rtl_amd64.S b/lib/tsan/rtl/tsan_rtl_amd64.S index caa832375e52..98947fd2a1ba 100644 --- a/lib/tsan/rtl/tsan_rtl_amd64.S +++ b/lib/tsan/rtl/tsan_rtl_amd64.S @@ -1,4 +1,8 @@ +// The content of this file is x86_64-only: +#if defined(__x86_64__) + #include "sanitizer_common/sanitizer_asm.h" + #if !defined(__APPLE__) .section .text #else @@ -357,3 +361,5 @@ ASM_SIZE(ASM_TSAN_SYMBOL_INTERCEPTOR(__sigsetjmp)) /* We do not need executable stack. */ .section .note.GNU-stack,"",@progbits #endif + +#endif diff --git a/lib/tsan/rtl/tsan_rtl_mutex.cc b/lib/tsan/rtl/tsan_rtl_mutex.cc index 54938f37e243..2f85811620f1 100644 --- a/lib/tsan/rtl/tsan_rtl_mutex.cc +++ b/lib/tsan/rtl/tsan_rtl_mutex.cc @@ -413,10 +413,10 @@ void Acquire(ThreadState *thr, uptr pc, uptr addr) { static void UpdateClockCallback(ThreadContextBase *tctx_base, void *arg) { ThreadState *thr = reinterpret_cast<ThreadState*>(arg); ThreadContext *tctx = static_cast<ThreadContext*>(tctx_base); + u64 epoch = tctx->epoch1; if (tctx->status == ThreadStatusRunning) - thr->clock.set(tctx->tid, tctx->thr->fast_state.epoch()); - else - thr->clock.set(tctx->tid, tctx->epoch1); + epoch = tctx->thr->fast_state.epoch(); + thr->clock.set(&thr->proc()->clock_cache, tctx->tid, epoch); } void AcquireGlobal(ThreadState *thr, uptr pc) { @@ -456,10 +456,10 @@ void ReleaseStore(ThreadState *thr, uptr pc, uptr addr) { static void UpdateSleepClockCallback(ThreadContextBase *tctx_base, void *arg) { ThreadState *thr = reinterpret_cast<ThreadState*>(arg); ThreadContext *tctx = static_cast<ThreadContext*>(tctx_base); + u64 epoch = tctx->epoch1; if (tctx->status == ThreadStatusRunning) - thr->last_sleep_clock.set(tctx->tid, tctx->thr->fast_state.epoch()); - else - thr->last_sleep_clock.set(tctx->tid, tctx->epoch1); + epoch = tctx->thr->fast_state.epoch(); + thr->last_sleep_clock.set(&thr->proc()->clock_cache, tctx->tid, epoch); } void AfterSleep(ThreadState *thr, uptr pc) { diff --git a/lib/tsan/rtl/tsan_rtl_report.cc b/lib/tsan/rtl/tsan_rtl_report.cc index 68b9f50308ea..85a982941ed7 100644 --- a/lib/tsan/rtl/tsan_rtl_report.cc +++ b/lib/tsan/rtl/tsan_rtl_report.cc @@ -314,7 +314,7 @@ void ScopedReport::AddLocation(uptr addr, uptr size) { return; #if !SANITIZER_GO int fd = -1; - int creat_tid = -1; + int creat_tid = kInvalidTid; u32 creat_stack = 0; if (FdLocation(addr, &fd, &creat_tid, &creat_stack)) { ReportLocation *loc = ReportLocation::New(ReportLocationFD); diff --git a/lib/tsan/rtl/tsan_rtl_thread.cc b/lib/tsan/rtl/tsan_rtl_thread.cc index 67eebf5d0c38..83fab082afe3 100644 --- a/lib/tsan/rtl/tsan_rtl_thread.cc +++ b/lib/tsan/rtl/tsan_rtl_thread.cc @@ -142,6 +142,10 @@ void ThreadContext::OnFinished() { if (common_flags()->detect_deadlocks) ctx->dd->DestroyLogicalThread(thr->dd_lt); + thr->clock.ResetCached(&thr->proc()->clock_cache); +#if !SANITIZER_GO + thr->last_sleep_clock.ResetCached(&thr->proc()->clock_cache); +#endif thr->~ThreadState(); #if TSAN_COLLECT_STATS StatAggregate(ctx->stat, thr->stat); diff --git a/lib/tsan/rtl/tsan_stat.cc b/lib/tsan/rtl/tsan_stat.cc index 2ee688bf5771..18c83d5c6dac 100644 --- a/lib/tsan/rtl/tsan_stat.cc +++ b/lib/tsan/rtl/tsan_stat.cc @@ -75,14 +75,11 @@ void StatOutput(u64 *stat) { name[StatClockAcquire] = "Clock acquire "; name[StatClockAcquireEmpty] = " empty clock "; name[StatClockAcquireFastRelease] = " fast from release-store "; - name[StatClockAcquireLarge] = " contains my tid "; - name[StatClockAcquireRepeat] = " repeated (fast) "; name[StatClockAcquireFull] = " full (slow) "; name[StatClockAcquiredSomething] = " acquired something "; name[StatClockRelease] = "Clock release "; name[StatClockReleaseResize] = " resize "; - name[StatClockReleaseFast1] = " fast1 "; - name[StatClockReleaseFast2] = " fast2 "; + name[StatClockReleaseFast] = " fast "; name[StatClockReleaseSlow] = " dirty overflow (slow) "; name[StatClockReleaseFull] = " full (slow) "; name[StatClockReleaseAcquired] = " was acquired "; diff --git a/lib/tsan/rtl/tsan_stat.h b/lib/tsan/rtl/tsan_stat.h index 7d2791ebbfcc..42d6a2b63532 100644 --- a/lib/tsan/rtl/tsan_stat.h +++ b/lib/tsan/rtl/tsan_stat.h @@ -74,15 +74,12 @@ enum StatType { StatClockAcquire, StatClockAcquireEmpty, StatClockAcquireFastRelease, - StatClockAcquireLarge, - StatClockAcquireRepeat, StatClockAcquireFull, StatClockAcquiredSomething, // Clocks - release. StatClockRelease, StatClockReleaseResize, - StatClockReleaseFast1, - StatClockReleaseFast2, + StatClockReleaseFast, StatClockReleaseSlow, StatClockReleaseFull, StatClockReleaseAcquired, diff --git a/lib/tsan/rtl/tsan_sync.cc b/lib/tsan/rtl/tsan_sync.cc index 4cc3cb89c34f..44ae558fa1b2 100644 --- a/lib/tsan/rtl/tsan_sync.cc +++ b/lib/tsan/rtl/tsan_sync.cc @@ -53,7 +53,9 @@ void SyncVar::Reset(Processor *proc) { } } -MetaMap::MetaMap() { +MetaMap::MetaMap() + : block_alloc_("heap block allocator") + , sync_alloc_("sync allocator") { atomic_store(&uid_gen_, 0, memory_order_relaxed); } |