aboutsummaryrefslogtreecommitdiff
path: root/lib/tsan
diff options
context:
space:
mode:
authorDimitry Andric <dim@FreeBSD.org>2017-07-13 19:25:48 +0000
committerDimitry Andric <dim@FreeBSD.org>2017-07-13 19:25:48 +0000
commit1992b790c2c12b7850bdf86662b67302052ec2fe (patch)
tree623c69b5fbf527bba17ecb9431ae5189871cecd4 /lib/tsan
parent50aa32eff79f252ab05a0c0a589cf2ca37cd9923 (diff)
downloadsrc-1992b790c2c12b7850bdf86662b67302052ec2fe.tar.gz
src-1992b790c2c12b7850bdf86662b67302052ec2fe.zip
Vendor import of compiler-rt trunk r307894:vendor/compiler-rt/compiler-rt-trunk-r307894
Notes
Notes: svn path=/vendor/compiler-rt/dist/; revision=320961 svn path=/vendor/compiler-rt/compiler-rt-trunk-r307894/; revision=320962; tag=vendor/compiler-rt/compiler-rt-trunk-r307894
Diffstat (limited to 'lib/tsan')
-rw-r--r--lib/tsan/CMakeLists.txt2
-rwxr-xr-xlib/tsan/check_analyze.sh4
-rw-r--r--lib/tsan/dd/dd_interceptors.cc19
-rwxr-xr-xlib/tsan/go/buildgo.sh2
-rw-r--r--lib/tsan/rtl/tsan_clock.cc119
-rw-r--r--lib/tsan/rtl/tsan_clock.h4
-rw-r--r--lib/tsan/rtl/tsan_dense_alloc.h11
-rw-r--r--lib/tsan/rtl/tsan_interceptors.cc29
-rw-r--r--lib/tsan/rtl/tsan_interceptors_mac.cc7
-rw-r--r--lib/tsan/rtl/tsan_interface_ann.cc16
-rw-r--r--lib/tsan/rtl/tsan_interface_atomic.cc16
-rw-r--r--lib/tsan/rtl/tsan_mman.cc2
-rw-r--r--lib/tsan/rtl/tsan_platform.h49
-rw-r--r--lib/tsan/rtl/tsan_platform_linux.cc17
-rw-r--r--lib/tsan/rtl/tsan_platform_mac.cc8
-rw-r--r--lib/tsan/rtl/tsan_platform_posix.cc27
-rw-r--r--lib/tsan/rtl/tsan_rtl.cc3
-rw-r--r--lib/tsan/rtl/tsan_rtl_aarch64.S127
-rw-r--r--lib/tsan/rtl/tsan_rtl_amd64.S6
-rw-r--r--lib/tsan/rtl/tsan_rtl_mutex.cc12
-rw-r--r--lib/tsan/rtl/tsan_rtl_report.cc2
-rw-r--r--lib/tsan/rtl/tsan_rtl_thread.cc4
-rw-r--r--lib/tsan/rtl/tsan_stat.cc5
-rw-r--r--lib/tsan/rtl/tsan_stat.h5
-rw-r--r--lib/tsan/rtl/tsan_sync.cc4
-rw-r--r--lib/tsan/tests/CMakeLists.txt1
-rw-r--r--lib/tsan/tests/unit/tsan_clock_test.cc62
27 files changed, 375 insertions, 188 deletions
diff --git a/lib/tsan/CMakeLists.txt b/lib/tsan/CMakeLists.txt
index 195ecb5dfe8a..193158c54e62 100644
--- a/lib/tsan/CMakeLists.txt
+++ b/lib/tsan/CMakeLists.txt
@@ -100,7 +100,7 @@ set(TSAN_RUNTIME_LIBRARIES)
add_compiler_rt_component(tsan)
if(APPLE)
- set(TSAN_ASM_SOURCES rtl/tsan_rtl_amd64.S)
+ set(TSAN_ASM_SOURCES rtl/tsan_rtl_amd64.S rtl/tsan_rtl_aarch64.S)
# Xcode will try to compile this file as C ('clang -x c'), and that will fail.
if (${CMAKE_GENERATOR} STREQUAL "Xcode")
enable_language(ASM)
diff --git a/lib/tsan/check_analyze.sh b/lib/tsan/check_analyze.sh
index 22eb444198a4..54dd1b0232dc 100755
--- a/lib/tsan/check_analyze.sh
+++ b/lib/tsan/check_analyze.sh
@@ -34,8 +34,8 @@ done
for f in read1 read2 read4 read8; do
check $f rsp 1
- check $f push 4
- check $f pop 4
+ check $f push 3
+ check $f pop 3
done
for f in func_entry func_exit; do
diff --git a/lib/tsan/dd/dd_interceptors.cc b/lib/tsan/dd/dd_interceptors.cc
index 97c72dd2b7fd..a39218f0454b 100644
--- a/lib/tsan/dd/dd_interceptors.cc
+++ b/lib/tsan/dd/dd_interceptors.cc
@@ -270,20 +270,19 @@ namespace __dsan {
static void InitDataSeg() {
MemoryMappingLayout proc_maps(true);
- uptr start, end, offset;
char name[128];
+ MemoryMappedSegment segment(name, ARRAY_SIZE(name));
bool prev_is_data = false;
- while (proc_maps.Next(&start, &end, &offset, name, ARRAY_SIZE(name),
- /*protection*/ 0)) {
- bool is_data = offset != 0 && name[0] != 0;
+ while (proc_maps.Next(&segment)) {
+ bool is_data = segment.offset != 0 && segment.filename[0] != 0;
// BSS may get merged with [heap] in /proc/self/maps. This is not very
// reliable.
- bool is_bss = offset == 0 &&
- (name[0] == 0 || internal_strcmp(name, "[heap]") == 0) && prev_is_data;
- if (g_data_start == 0 && is_data)
- g_data_start = start;
- if (is_bss)
- g_data_end = end;
+ bool is_bss = segment.offset == 0 &&
+ (segment.filename[0] == 0 ||
+ internal_strcmp(segment.filename, "[heap]") == 0) &&
+ prev_is_data;
+ if (g_data_start == 0 && is_data) g_data_start = segment.start;
+ if (is_bss) g_data_end = segment.end;
prev_is_data = is_data;
}
VPrintf(1, "guessed data_start=%p data_end=%p\n", g_data_start, g_data_end);
diff --git a/lib/tsan/go/buildgo.sh b/lib/tsan/go/buildgo.sh
index 59176809eae0..617dd9e11d27 100755
--- a/lib/tsan/go/buildgo.sh
+++ b/lib/tsan/go/buildgo.sh
@@ -69,7 +69,7 @@ elif [ "`uname -a | grep FreeBSD`" != "" ]; then
"
elif [ "`uname -a | grep Darwin`" != "" ]; then
SUFFIX="darwin_amd64"
- OSCFLAGS="-fPIC -Wno-unused-const-variable -Wno-unknown-warning-option -mmacosx-version-min=10.7"
+ OSCFLAGS="-fPIC -Wno-unused-const-variable -Wno-unknown-warning-option -isysroot $(xcodebuild -version -sdk macosx Path) -mmacosx-version-min=10.7"
OSLDFLAGS="-lpthread -fPIC -fpie -mmacosx-version-min=10.7"
SRCS="
$SRCS
diff --git a/lib/tsan/rtl/tsan_clock.cc b/lib/tsan/rtl/tsan_clock.cc
index 32435adfdf33..9ee9104283f8 100644
--- a/lib/tsan/rtl/tsan_clock.cc
+++ b/lib/tsan/rtl/tsan_clock.cc
@@ -101,6 +101,9 @@ ThreadClock::ThreadClock(unsigned tid, unsigned reused)
clk_[tid_].reused = reused_;
}
+void ThreadClock::ResetCached(ClockCache *c) {
+}
+
void ThreadClock::acquire(ClockCache *c, const SyncClock *src) {
DCHECK_LE(nclk_, kMaxTid);
DCHECK_LE(src->size_, kMaxTid);
@@ -116,9 +119,7 @@ void ThreadClock::acquire(ClockCache *c, const SyncClock *src) {
// Check if we've already acquired src after the last release operation on src
bool acquired = false;
if (nclk > tid_) {
- CPP_STAT_INC(StatClockAcquireLarge);
if (src->elem(tid_).reused == reused_) {
- CPP_STAT_INC(StatClockAcquireRepeat);
for (unsigned i = 0; i < kDirtyTids; i++) {
unsigned tid = src->dirty_tids_[i];
if (tid != kInvalidTid) {
@@ -266,11 +267,11 @@ void ThreadClock::UpdateCurrentThread(SyncClock *dst) const {
for (unsigned i = 0; i < kDirtyTids; i++) {
if (dst->dirty_tids_[i] == tid_) {
- CPP_STAT_INC(StatClockReleaseFast1);
+ CPP_STAT_INC(StatClockReleaseFast);
return;
}
if (dst->dirty_tids_[i] == kInvalidTid) {
- CPP_STAT_INC(StatClockReleaseFast2);
+ CPP_STAT_INC(StatClockReleaseFast);
dst->dirty_tids_[i] = tid_;
return;
}
@@ -297,56 +298,9 @@ bool ThreadClock::IsAlreadyAcquired(const SyncClock *src) const {
return true;
}
-void SyncClock::Resize(ClockCache *c, uptr nclk) {
- CPP_STAT_INC(StatClockReleaseResize);
- if (RoundUpTo(nclk, ClockBlock::kClockCount) <=
- RoundUpTo(size_, ClockBlock::kClockCount)) {
- // Growing within the same block.
- // Memory is already allocated, just increase the size.
- size_ = nclk;
- return;
- }
- if (nclk <= ClockBlock::kClockCount) {
- // Grow from 0 to one-level table.
- CHECK_EQ(size_, 0);
- CHECK_EQ(tab_, 0);
- CHECK_EQ(tab_idx_, 0);
- size_ = nclk;
- tab_idx_ = ctx->clock_alloc.Alloc(c);
- tab_ = ctx->clock_alloc.Map(tab_idx_);
- internal_memset(tab_, 0, sizeof(*tab_));
- return;
- }
- // Growing two-level table.
- if (size_ == 0) {
- // Allocate first level table.
- tab_idx_ = ctx->clock_alloc.Alloc(c);
- tab_ = ctx->clock_alloc.Map(tab_idx_);
- internal_memset(tab_, 0, sizeof(*tab_));
- } else if (size_ <= ClockBlock::kClockCount) {
- // Transform one-level table to two-level table.
- u32 old = tab_idx_;
- tab_idx_ = ctx->clock_alloc.Alloc(c);
- tab_ = ctx->clock_alloc.Map(tab_idx_);
- internal_memset(tab_, 0, sizeof(*tab_));
- tab_->table[0] = old;
- }
- // At this point we have first level table allocated.
- // Add second level tables as necessary.
- for (uptr i = RoundUpTo(size_, ClockBlock::kClockCount);
- i < nclk; i += ClockBlock::kClockCount) {
- u32 idx = ctx->clock_alloc.Alloc(c);
- ClockBlock *cb = ctx->clock_alloc.Map(idx);
- internal_memset(cb, 0, sizeof(*cb));
- CHECK_EQ(tab_->table[i/ClockBlock::kClockCount], 0);
- tab_->table[i/ClockBlock::kClockCount] = idx;
- }
- size_ = nclk;
-}
-
// Sets a single element in the vector clock.
// This function is called only from weird places like AcquireGlobal.
-void ThreadClock::set(unsigned tid, u64 v) {
+void ThreadClock::set(ClockCache *c, unsigned tid, u64 v) {
DCHECK_LT(tid, kMaxTid);
DCHECK_GE(v, clk_[tid].epoch);
clk_[tid].epoch = v;
@@ -366,14 +320,8 @@ void ThreadClock::DebugDump(int(*printf)(const char *s, ...)) {
tid_, reused_, last_acquire_);
}
-SyncClock::SyncClock()
- : release_store_tid_(kInvalidTid)
- , release_store_reused_()
- , tab_()
- , tab_idx_()
- , size_() {
- for (uptr i = 0; i < kDirtyTids; i++)
- dirty_tids_[i] = kInvalidTid;
+SyncClock::SyncClock() {
+ ResetImpl();
}
SyncClock::~SyncClock() {
@@ -395,6 +343,10 @@ void SyncClock::Reset(ClockCache *c) {
ctx->clock_alloc.Free(c, tab_->table[i / ClockBlock::kClockCount]);
ctx->clock_alloc.Free(c, tab_idx_);
}
+ ResetImpl();
+}
+
+void SyncClock::ResetImpl() {
tab_ = 0;
tab_idx_ = 0;
size_ = 0;
@@ -404,6 +356,53 @@ void SyncClock::Reset(ClockCache *c) {
dirty_tids_[i] = kInvalidTid;
}
+void SyncClock::Resize(ClockCache *c, uptr nclk) {
+ CPP_STAT_INC(StatClockReleaseResize);
+ if (RoundUpTo(nclk, ClockBlock::kClockCount) <=
+ RoundUpTo(size_, ClockBlock::kClockCount)) {
+ // Growing within the same block.
+ // Memory is already allocated, just increase the size.
+ size_ = nclk;
+ return;
+ }
+ if (nclk <= ClockBlock::kClockCount) {
+ // Grow from 0 to one-level table.
+ CHECK_EQ(size_, 0);
+ CHECK_EQ(tab_, 0);
+ CHECK_EQ(tab_idx_, 0);
+ size_ = nclk;
+ tab_idx_ = ctx->clock_alloc.Alloc(c);
+ tab_ = ctx->clock_alloc.Map(tab_idx_);
+ internal_memset(tab_, 0, sizeof(*tab_));
+ return;
+ }
+ // Growing two-level table.
+ if (size_ == 0) {
+ // Allocate first level table.
+ tab_idx_ = ctx->clock_alloc.Alloc(c);
+ tab_ = ctx->clock_alloc.Map(tab_idx_);
+ internal_memset(tab_, 0, sizeof(*tab_));
+ } else if (size_ <= ClockBlock::kClockCount) {
+ // Transform one-level table to two-level table.
+ u32 old = tab_idx_;
+ tab_idx_ = ctx->clock_alloc.Alloc(c);
+ tab_ = ctx->clock_alloc.Map(tab_idx_);
+ internal_memset(tab_, 0, sizeof(*tab_));
+ tab_->table[0] = old;
+ }
+ // At this point we have first level table allocated.
+ // Add second level tables as necessary.
+ for (uptr i = RoundUpTo(size_, ClockBlock::kClockCount);
+ i < nclk; i += ClockBlock::kClockCount) {
+ u32 idx = ctx->clock_alloc.Alloc(c);
+ ClockBlock *cb = ctx->clock_alloc.Map(idx);
+ internal_memset(cb, 0, sizeof(*cb));
+ CHECK_EQ(tab_->table[i/ClockBlock::kClockCount], 0);
+ tab_->table[i/ClockBlock::kClockCount] = idx;
+ }
+ size_ = nclk;
+}
+
ClockElem &SyncClock::elem(unsigned tid) const {
DCHECK_LT(tid, size_);
if (size_ <= ClockBlock::kClockCount)
diff --git a/lib/tsan/rtl/tsan_clock.h b/lib/tsan/rtl/tsan_clock.h
index 4e352cb81d11..378b550fd11b 100644
--- a/lib/tsan/rtl/tsan_clock.h
+++ b/lib/tsan/rtl/tsan_clock.h
@@ -74,6 +74,7 @@ class SyncClock {
u32 tab_idx_;
u32 size_;
+ void ResetImpl();
ClockElem &elem(unsigned tid) const;
};
@@ -89,7 +90,7 @@ struct ThreadClock {
return clk_[tid].epoch;
}
- void set(unsigned tid, u64 v);
+ void set(ClockCache *c, unsigned tid, u64 v);
void set(u64 v) {
DCHECK_GE(v, clk_[tid_].epoch);
@@ -108,6 +109,7 @@ struct ThreadClock {
void release(ClockCache *c, SyncClock *dst) const;
void acq_rel(ClockCache *c, SyncClock *dst);
void ReleaseStore(ClockCache *c, SyncClock *dst) const;
+ void ResetCached(ClockCache *c);
void DebugReset();
void DebugDump(int(*printf)(const char *s, ...));
diff --git a/lib/tsan/rtl/tsan_dense_alloc.h b/lib/tsan/rtl/tsan_dense_alloc.h
index e9815c90a953..16dbdf391085 100644
--- a/lib/tsan/rtl/tsan_dense_alloc.h
+++ b/lib/tsan/rtl/tsan_dense_alloc.h
@@ -39,7 +39,7 @@ class DenseSlabAlloc {
typedef DenseSlabAllocCache Cache;
typedef typename Cache::IndexT IndexT;
- DenseSlabAlloc() {
+ explicit DenseSlabAlloc(const char *name) {
// Check that kL1Size and kL2Size are sane.
CHECK_EQ(kL1Size & (kL1Size - 1), 0);
CHECK_EQ(kL2Size & (kL2Size - 1), 0);
@@ -49,6 +49,7 @@ class DenseSlabAlloc {
internal_memset(map_, 0, sizeof(map_));
freelist_ = 0;
fillpos_ = 0;
+ name_ = name;
}
~DenseSlabAlloc() {
@@ -96,15 +97,19 @@ class DenseSlabAlloc {
SpinMutex mtx_;
IndexT freelist_;
uptr fillpos_;
+ const char *name_;
void Refill(Cache *c) {
SpinMutexLock lock(&mtx_);
if (freelist_ == 0) {
if (fillpos_ == kL1Size) {
- Printf("ThreadSanitizer: DenseSlabAllocator overflow. Dying.\n");
+ Printf("ThreadSanitizer: %s overflow (%zu*%zu). Dying.\n",
+ name_, kL1Size, kL2Size);
Die();
}
- T *batch = (T*)MmapOrDie(kL2Size * sizeof(T), "DenseSlabAllocator");
+ VPrintf(2, "ThreadSanitizer: growing %s: %zu out of %zu*%zu\n",
+ name_, fillpos_, kL1Size, kL2Size);
+ T *batch = (T*)MmapOrDie(kL2Size * sizeof(T), name_);
// Reserve 0 as invalid index.
IndexT start = fillpos_ == 0 ? 1 : 0;
for (IndexT i = start; i < kL2Size; i++) {
diff --git a/lib/tsan/rtl/tsan_interceptors.cc b/lib/tsan/rtl/tsan_interceptors.cc
index 334cc326daf6..001123f4941e 100644
--- a/lib/tsan/rtl/tsan_interceptors.cc
+++ b/lib/tsan/rtl/tsan_interceptors.cc
@@ -14,6 +14,7 @@
//===----------------------------------------------------------------------===//
#include "sanitizer_common/sanitizer_atomic.h"
+#include "sanitizer_common/sanitizer_errno.h"
#include "sanitizer_common/sanitizer_libc.h"
#include "sanitizer_common/sanitizer_linux.h"
#include "sanitizer_common/sanitizer_platform_limits_posix.h"
@@ -34,13 +35,11 @@
using namespace __tsan; // NOLINT
#if SANITIZER_FREEBSD || SANITIZER_MAC
-#define __errno_location __error
#define stdout __stdoutp
#define stderr __stderrp
#endif
#if SANITIZER_ANDROID
-#define __errno_location __errno
#define mallopt(a, b)
#endif
@@ -84,7 +83,6 @@ DECLARE_REAL_AND_INTERCEPTOR(void *, malloc, uptr size)
DECLARE_REAL_AND_INTERCEPTOR(void, free, void *ptr)
extern "C" void *pthread_self();
extern "C" void _exit(int status);
-extern "C" int *__errno_location();
extern "C" int fileno_unlocked(void *stream);
extern "C" int dirfd(void *dirp);
#if !SANITIZER_FREEBSD && !SANITIZER_ANDROID
@@ -98,9 +96,6 @@ const int PTHREAD_MUTEX_RECURSIVE_NP = 1;
const int PTHREAD_MUTEX_RECURSIVE = 2;
const int PTHREAD_MUTEX_RECURSIVE_NP = 2;
#endif
-const int EINVAL = 22;
-const int EBUSY = 16;
-const int EOWNERDEAD = 130;
#if !SANITIZER_FREEBSD && !SANITIZER_MAC
const int EPOLL_CTL_ADD = 1;
#endif
@@ -130,8 +125,6 @@ typedef long long_t; // NOLINT
# define F_TLOCK 2 /* Test and lock a region for exclusive use. */
# define F_TEST 3 /* Test a region for other processes locks. */
-#define errno (*__errno_location())
-
typedef void (*sighandler_t)(int sig);
typedef void (*sigactionhandler_t)(int sig, my_siginfo_t *siginfo, void *uctx);
@@ -268,7 +261,7 @@ ScopedInterceptor::~ScopedInterceptor() {
void ScopedInterceptor::EnableIgnores() {
if (ignoring_) {
- ThreadIgnoreBegin(thr_, pc_, false);
+ ThreadIgnoreBegin(thr_, pc_, /*save_stack=*/false);
if (flags()->ignore_noninstrumented_modules) thr_->suppress_reports++;
if (in_ignored_lib_) {
DCHECK(!thr_->in_ignored_lib);
@@ -466,8 +459,14 @@ static void SetJmp(ThreadState *thr, uptr sp, uptr mangled_sp) {
static void LongJmp(ThreadState *thr, uptr *env) {
#ifdef __powerpc__
uptr mangled_sp = env[0];
-#elif SANITIZER_FREEBSD || SANITIZER_MAC
+#elif SANITIZER_FREEBSD
uptr mangled_sp = env[2];
+#elif SANITIZER_MAC
+# ifdef __aarch64__
+ uptr mangled_sp = env[13];
+# else
+ uptr mangled_sp = env[2];
+# endif
#elif defined(SANITIZER_LINUX)
# ifdef __aarch64__
uptr mangled_sp = env[13];
@@ -665,7 +664,7 @@ static bool fix_mmap_addr(void **addr, long_t sz, int flags) {
if (*addr) {
if (!IsAppMem((uptr)*addr) || !IsAppMem((uptr)*addr + sz - 1)) {
if (flags & MAP_FIXED) {
- errno = EINVAL;
+ errno = errno_EINVAL;
return false;
} else {
*addr = 0;
@@ -1122,7 +1121,7 @@ TSAN_INTERCEPTOR(int, pthread_mutex_init, void *m, void *a) {
TSAN_INTERCEPTOR(int, pthread_mutex_destroy, void *m) {
SCOPED_TSAN_INTERCEPTOR(pthread_mutex_destroy, m);
int res = REAL(pthread_mutex_destroy)(m);
- if (res == 0 || res == EBUSY) {
+ if (res == 0 || res == errno_EBUSY) {
MutexDestroy(thr, pc, (uptr)m);
}
return res;
@@ -1131,9 +1130,9 @@ TSAN_INTERCEPTOR(int, pthread_mutex_destroy, void *m) {
TSAN_INTERCEPTOR(int, pthread_mutex_trylock, void *m) {
SCOPED_TSAN_INTERCEPTOR(pthread_mutex_trylock, m);
int res = REAL(pthread_mutex_trylock)(m);
- if (res == EOWNERDEAD)
+ if (res == errno_EOWNERDEAD)
MutexRepair(thr, pc, (uptr)m);
- if (res == 0 || res == EOWNERDEAD)
+ if (res == 0 || res == errno_EOWNERDEAD)
MutexPostLock(thr, pc, (uptr)m, MutexFlagTryLock);
return res;
}
@@ -1311,7 +1310,7 @@ TSAN_INTERCEPTOR(int, pthread_barrier_wait, void *b) {
TSAN_INTERCEPTOR(int, pthread_once, void *o, void (*f)()) {
SCOPED_INTERCEPTOR_RAW(pthread_once, o, f);
if (o == 0 || f == 0)
- return EINVAL;
+ return errno_EINVAL;
atomic_uint32_t *a;
if (!SANITIZER_MAC)
a = static_cast<atomic_uint32_t*>(o);
diff --git a/lib/tsan/rtl/tsan_interceptors_mac.cc b/lib/tsan/rtl/tsan_interceptors_mac.cc
index f6bf8a0e586b..4f1079467331 100644
--- a/lib/tsan/rtl/tsan_interceptors_mac.cc
+++ b/lib/tsan/rtl/tsan_interceptors_mac.cc
@@ -21,7 +21,10 @@
#include "tsan_interface_ann.h"
#include <libkern/OSAtomic.h>
+
+#if defined(__has_include) && __has_include(<xpc/xpc.h>)
#include <xpc/xpc.h>
+#endif // #if defined(__has_include) && __has_include(<xpc/xpc.h>)
typedef long long_t; // NOLINT
@@ -235,6 +238,8 @@ TSAN_INTERCEPTOR(void, os_lock_unlock, void *lock) {
REAL(os_lock_unlock)(lock);
}
+#if defined(__has_include) && __has_include(<xpc/xpc.h>)
+
TSAN_INTERCEPTOR(void, xpc_connection_set_event_handler,
xpc_connection_t connection, xpc_handler_t handler) {
SCOPED_TSAN_INTERCEPTOR(xpc_connection_set_event_handler, connection,
@@ -287,6 +292,8 @@ TSAN_INTERCEPTOR(void, xpc_connection_cancel, xpc_connection_t connection) {
REAL(xpc_connection_cancel)(connection);
}
+#endif // #if defined(__has_include) && __has_include(<xpc/xpc.h>)
+
// On macOS, libc++ is always linked dynamically, so intercepting works the
// usual way.
#define STDCXX_INTERCEPTOR TSAN_INTERCEPTOR
diff --git a/lib/tsan/rtl/tsan_interface_ann.cc b/lib/tsan/rtl/tsan_interface_ann.cc
index 45ec45bbdbbe..f68a0468de53 100644
--- a/lib/tsan/rtl/tsan_interface_ann.cc
+++ b/lib/tsan/rtl/tsan_interface_ann.cc
@@ -483,8 +483,8 @@ void __tsan_mutex_pre_lock(void *m, unsigned flagz) {
else
MutexPreLock(thr, pc, (uptr)m);
}
- ThreadIgnoreBegin(thr, pc, false);
- ThreadIgnoreSyncBegin(thr, pc, false);
+ ThreadIgnoreBegin(thr, pc, /*save_stack=*/false);
+ ThreadIgnoreSyncBegin(thr, pc, /*save_stack=*/false);
}
INTERFACE_ATTRIBUTE
@@ -510,8 +510,8 @@ int __tsan_mutex_pre_unlock(void *m, unsigned flagz) {
} else {
ret = MutexUnlock(thr, pc, (uptr)m, flagz);
}
- ThreadIgnoreBegin(thr, pc, false);
- ThreadIgnoreSyncBegin(thr, pc, false);
+ ThreadIgnoreBegin(thr, pc, /*save_stack=*/false);
+ ThreadIgnoreSyncBegin(thr, pc, /*save_stack=*/false);
return ret;
}
@@ -525,8 +525,8 @@ void __tsan_mutex_post_unlock(void *m, unsigned flagz) {
INTERFACE_ATTRIBUTE
void __tsan_mutex_pre_signal(void *addr, unsigned flagz) {
SCOPED_ANNOTATION(__tsan_mutex_pre_signal);
- ThreadIgnoreBegin(thr, pc, false);
- ThreadIgnoreSyncBegin(thr, pc, false);
+ ThreadIgnoreBegin(thr, pc, /*save_stack=*/false);
+ ThreadIgnoreSyncBegin(thr, pc, /*save_stack=*/false);
}
INTERFACE_ATTRIBUTE
@@ -547,7 +547,7 @@ void __tsan_mutex_pre_divert(void *addr, unsigned flagz) {
INTERFACE_ATTRIBUTE
void __tsan_mutex_post_divert(void *addr, unsigned flagz) {
SCOPED_ANNOTATION(__tsan_mutex_post_divert);
- ThreadIgnoreBegin(thr, pc, false);
- ThreadIgnoreSyncBegin(thr, pc, false);
+ ThreadIgnoreBegin(thr, pc, /*save_stack=*/false);
+ ThreadIgnoreSyncBegin(thr, pc, /*save_stack=*/false);
}
} // extern "C"
diff --git a/lib/tsan/rtl/tsan_interface_atomic.cc b/lib/tsan/rtl/tsan_interface_atomic.cc
index b22d5c1ecef8..d334394f5303 100644
--- a/lib/tsan/rtl/tsan_interface_atomic.cc
+++ b/lib/tsan/rtl/tsan_interface_atomic.cc
@@ -220,8 +220,7 @@ static a128 NoTsanAtomicLoad(const volatile a128 *a, morder mo) {
#endif
template<typename T>
-static T AtomicLoad(ThreadState *thr, uptr pc, const volatile T *a,
- morder mo) {
+static T AtomicLoad(ThreadState *thr, uptr pc, const volatile T *a, morder mo) {
CHECK(IsLoadOrder(mo));
// This fast-path is critical for performance.
// Assume the access is atomic.
@@ -229,10 +228,17 @@ static T AtomicLoad(ThreadState *thr, uptr pc, const volatile T *a,
MemoryReadAtomic(thr, pc, (uptr)a, SizeLog<T>());
return NoTsanAtomicLoad(a, mo);
}
- SyncVar *s = ctx->metamap.GetOrCreateAndLock(thr, pc, (uptr)a, false);
- AcquireImpl(thr, pc, &s->clock);
+ // Don't create sync object if it does not exist yet. For example, an atomic
+ // pointer is initialized to nullptr and then periodically acquire-loaded.
T v = NoTsanAtomicLoad(a, mo);
- s->mtx.ReadUnlock();
+ SyncVar *s = ctx->metamap.GetIfExistsAndLock((uptr)a, false);
+ if (s) {
+ AcquireImpl(thr, pc, &s->clock);
+ // Re-read under sync mutex because we need a consistent snapshot
+ // of the value and the clock we acquire.
+ v = NoTsanAtomicLoad(a, mo);
+ s->mtx.ReadUnlock();
+ }
MemoryReadAtomic(thr, pc, (uptr)a, SizeLog<T>());
return v;
}
diff --git a/lib/tsan/rtl/tsan_mman.cc b/lib/tsan/rtl/tsan_mman.cc
index 7169d5b02c04..1434cf688ce9 100644
--- a/lib/tsan/rtl/tsan_mman.cc
+++ b/lib/tsan/rtl/tsan_mman.cc
@@ -294,6 +294,8 @@ uptr __sanitizer_get_allocated_size(const void *p) {
void __tsan_on_thread_idle() {
ThreadState *thr = cur_thread();
+ thr->clock.ResetCached(&thr->proc()->clock_cache);
+ thr->last_sleep_clock.ResetCached(&thr->proc()->clock_cache);
allocator()->SwallowCache(&thr->proc()->alloc_cache);
internal_allocator()->SwallowCache(&thr->proc()->internal_alloc_cache);
ctx->metamap.OnProcIdle(thr->proc());
diff --git a/lib/tsan/rtl/tsan_platform.h b/lib/tsan/rtl/tsan_platform.h
index 60d9b9d8c452..bea1daba3952 100644
--- a/lib/tsan/rtl/tsan_platform.h
+++ b/lib/tsan/rtl/tsan_platform.h
@@ -100,6 +100,37 @@ struct Mapping {
};
#define TSAN_MID_APP_RANGE 1
+#elif defined(__aarch64__) && defined(__APPLE__)
+/*
+C/C++ on Darwin/iOS/ARM64 (36-bit VMA, 64 GB VM)
+0000 0000 00 - 0100 0000 00: - (4 GB)
+0100 0000 00 - 0200 0000 00: main binary, modules, thread stacks (4 GB)
+0200 0000 00 - 0300 0000 00: heap (4 GB)
+0300 0000 00 - 0400 0000 00: - (4 GB)
+0400 0000 00 - 0c00 0000 00: shadow memory (32 GB)
+0c00 0000 00 - 0d00 0000 00: - (4 GB)
+0d00 0000 00 - 0e00 0000 00: metainfo (4 GB)
+0e00 0000 00 - 0f00 0000 00: - (4 GB)
+0f00 0000 00 - 1000 0000 00: traces (4 GB)
+*/
+struct Mapping {
+ static const uptr kLoAppMemBeg = 0x0100000000ull;
+ static const uptr kLoAppMemEnd = 0x0200000000ull;
+ static const uptr kHeapMemBeg = 0x0200000000ull;
+ static const uptr kHeapMemEnd = 0x0300000000ull;
+ static const uptr kShadowBeg = 0x0400000000ull;
+ static const uptr kShadowEnd = 0x0c00000000ull;
+ static const uptr kMetaShadowBeg = 0x0d00000000ull;
+ static const uptr kMetaShadowEnd = 0x0e00000000ull;
+ static const uptr kTraceMemBeg = 0x0f00000000ull;
+ static const uptr kTraceMemEnd = 0x1000000000ull;
+ static const uptr kHiAppMemBeg = 0x1000000000ull;
+ static const uptr kHiAppMemEnd = 0x1000000000ull;
+ static const uptr kAppMemMsk = 0x0ull;
+ static const uptr kAppMemXor = 0x0ull;
+ static const uptr kVdsoBeg = 0x7000000000000000ull;
+};
+
#elif defined(__aarch64__)
// AArch64 supports multiple VMA which leads to multiple address transformation
// functions. To support these multiple VMAS transformations and mappings TSAN
@@ -389,7 +420,7 @@ uptr MappingImpl(void) {
template<int Type>
uptr MappingArchImpl(void) {
-#ifdef __aarch64__
+#if defined(__aarch64__) && !defined(__APPLE__)
switch (vmaSize) {
case 39: return MappingImpl<Mapping39, Type>();
case 42: return MappingImpl<Mapping42, Type>();
@@ -542,7 +573,7 @@ bool IsAppMemImpl(uptr mem) {
ALWAYS_INLINE
bool IsAppMem(uptr mem) {
-#ifdef __aarch64__
+#if defined(__aarch64__) && !defined(__APPLE__)
switch (vmaSize) {
case 39: return IsAppMemImpl<Mapping39>(mem);
case 42: return IsAppMemImpl<Mapping42>(mem);
@@ -569,7 +600,7 @@ bool IsShadowMemImpl(uptr mem) {
ALWAYS_INLINE
bool IsShadowMem(uptr mem) {
-#ifdef __aarch64__
+#if defined(__aarch64__) && !defined(__APPLE__)
switch (vmaSize) {
case 39: return IsShadowMemImpl<Mapping39>(mem);
case 42: return IsShadowMemImpl<Mapping42>(mem);
@@ -596,7 +627,7 @@ bool IsMetaMemImpl(uptr mem) {
ALWAYS_INLINE
bool IsMetaMem(uptr mem) {
-#ifdef __aarch64__
+#if defined(__aarch64__) && !defined(__APPLE__)
switch (vmaSize) {
case 39: return IsMetaMemImpl<Mapping39>(mem);
case 42: return IsMetaMemImpl<Mapping42>(mem);
@@ -633,7 +664,7 @@ uptr MemToShadowImpl(uptr x) {
ALWAYS_INLINE
uptr MemToShadow(uptr x) {
-#ifdef __aarch64__
+#if defined(__aarch64__) && !defined(__APPLE__)
switch (vmaSize) {
case 39: return MemToShadowImpl<Mapping39>(x);
case 42: return MemToShadowImpl<Mapping42>(x);
@@ -672,7 +703,7 @@ u32 *MemToMetaImpl(uptr x) {
ALWAYS_INLINE
u32 *MemToMeta(uptr x) {
-#ifdef __aarch64__
+#if defined(__aarch64__) && !defined(__APPLE__)
switch (vmaSize) {
case 39: return MemToMetaImpl<Mapping39>(x);
case 42: return MemToMetaImpl<Mapping42>(x);
@@ -724,7 +755,7 @@ uptr ShadowToMemImpl(uptr s) {
ALWAYS_INLINE
uptr ShadowToMem(uptr s) {
-#ifdef __aarch64__
+#if defined(__aarch64__) && !defined(__APPLE__)
switch (vmaSize) {
case 39: return ShadowToMemImpl<Mapping39>(s);
case 42: return ShadowToMemImpl<Mapping42>(s);
@@ -759,7 +790,7 @@ uptr GetThreadTraceImpl(int tid) {
ALWAYS_INLINE
uptr GetThreadTrace(int tid) {
-#ifdef __aarch64__
+#if defined(__aarch64__) && !defined(__APPLE__)
switch (vmaSize) {
case 39: return GetThreadTraceImpl<Mapping39>(tid);
case 42: return GetThreadTraceImpl<Mapping42>(tid);
@@ -789,7 +820,7 @@ uptr GetThreadTraceHeaderImpl(int tid) {
ALWAYS_INLINE
uptr GetThreadTraceHeader(int tid) {
-#ifdef __aarch64__
+#if defined(__aarch64__) && !defined(__APPLE__)
switch (vmaSize) {
case 39: return GetThreadTraceHeaderImpl<Mapping39>(tid);
case 42: return GetThreadTraceHeaderImpl<Mapping42>(tid);
diff --git a/lib/tsan/rtl/tsan_platform_linux.cc b/lib/tsan/rtl/tsan_platform_linux.cc
index d05c0e701e72..0ba01babe69a 100644
--- a/lib/tsan/rtl/tsan_platform_linux.cc
+++ b/lib/tsan/rtl/tsan_platform_linux.cc
@@ -47,7 +47,6 @@
#include <sys/resource.h>
#include <sys/stat.h>
#include <unistd.h>
-#include <errno.h>
#include <sched.h>
#include <dlfcn.h>
#if SANITIZER_LINUX
@@ -182,17 +181,15 @@ static void MapRodata() {
}
// Map the file into shadow of .rodata sections.
MemoryMappingLayout proc_maps(/*cache_enabled*/true);
- uptr start, end, offset, prot;
// Reusing the buffer 'name'.
- while (proc_maps.Next(&start, &end, &offset, name, ARRAY_SIZE(name), &prot)) {
- if (name[0] != 0 && name[0] != '['
- && (prot & MemoryMappingLayout::kProtectionRead)
- && (prot & MemoryMappingLayout::kProtectionExecute)
- && !(prot & MemoryMappingLayout::kProtectionWrite)
- && IsAppMem(start)) {
+ MemoryMappedSegment segment(name, ARRAY_SIZE(name));
+ while (proc_maps.Next(&segment)) {
+ if (segment.filename[0] != 0 && segment.filename[0] != '[' &&
+ segment.IsReadable() && segment.IsExecutable() &&
+ !segment.IsWritable() && IsAppMem(segment.start)) {
// Assume it's .rodata
- char *shadow_start = (char*)MemToShadow(start);
- char *shadow_end = (char*)MemToShadow(end);
+ char *shadow_start = (char *)MemToShadow(segment.start);
+ char *shadow_end = (char *)MemToShadow(segment.end);
for (char *p = shadow_start; p < shadow_end; p += marker.size()) {
internal_mmap(p, Min<uptr>(marker.size(), shadow_end - p),
PROT_READ, MAP_PRIVATE | MAP_FIXED, fd, 0);
diff --git a/lib/tsan/rtl/tsan_platform_mac.cc b/lib/tsan/rtl/tsan_platform_mac.cc
index a82bcd01bbf4..73a656ffca5e 100644
--- a/lib/tsan/rtl/tsan_platform_mac.cc
+++ b/lib/tsan/rtl/tsan_platform_mac.cc
@@ -230,6 +230,14 @@ static void my_pthread_introspection_hook(unsigned int event, pthread_t thread,
#endif
void InitializePlatformEarly() {
+#if defined(__aarch64__)
+ uptr max_vm = GetMaxVirtualAddress() + 1;
+ if (max_vm != Mapping::kHiAppMemEnd) {
+ Printf("ThreadSanitizer: unsupported vm address limit %p, expected %p.\n",
+ max_vm, Mapping::kHiAppMemEnd);
+ Die();
+ }
+#endif
}
void InitializePlatform() {
diff --git a/lib/tsan/rtl/tsan_platform_posix.cc b/lib/tsan/rtl/tsan_platform_posix.cc
index 0732c83d689d..e4f90a811c35 100644
--- a/lib/tsan/rtl/tsan_platform_posix.cc
+++ b/lib/tsan/rtl/tsan_platform_posix.cc
@@ -46,6 +46,9 @@ void InitializeShadowMemory() {
#elif defined(__mips64)
const uptr kMadviseRangeBeg = 0xff00000000ull;
const uptr kMadviseRangeSize = 0x0100000000ull;
+#elif defined(__aarch64__) && defined(__APPLE__)
+ uptr kMadviseRangeBeg = LoAppMemBeg();
+ uptr kMadviseRangeSize = LoAppMemEnd() - LoAppMemBeg();
#elif defined(__aarch64__)
uptr kMadviseRangeBeg = 0;
uptr kMadviseRangeSize = 0;
@@ -115,21 +118,24 @@ static void ProtectRange(uptr beg, uptr end) {
void CheckAndProtect() {
// Ensure that the binary is indeed compiled with -pie.
MemoryMappingLayout proc_maps(true);
- uptr p, end, prot;
- while (proc_maps.Next(&p, &end, 0, 0, 0, &prot)) {
- if (IsAppMem(p))
+ MemoryMappedSegment segment;
+ while (proc_maps.Next(&segment)) {
+ if (IsAppMem(segment.start)) continue;
+ if (segment.start >= HeapMemEnd() && segment.start < HeapEnd()) continue;
+ if (segment.protection == 0) // Zero page or mprotected.
continue;
- if (p >= HeapMemEnd() &&
- p < HeapEnd())
- continue;
- if (prot == 0) // Zero page or mprotected.
- continue;
- if (p >= VdsoBeg()) // vdso
+ if (segment.start >= VdsoBeg()) // vdso
break;
- Printf("FATAL: ThreadSanitizer: unexpected memory mapping %p-%p\n", p, end);
+ Printf("FATAL: ThreadSanitizer: unexpected memory mapping %p-%p\n",
+ segment.start, segment.end);
Die();
}
+#if defined(__aarch64__) && defined(__APPLE__)
+ ProtectRange(HeapMemEnd(), ShadowBeg());
+ ProtectRange(ShadowEnd(), MetaShadowBeg());
+ ProtectRange(MetaShadowEnd(), TraceMemBeg());
+#else
ProtectRange(LoAppMemEnd(), ShadowBeg());
ProtectRange(ShadowEnd(), MetaShadowBeg());
#ifdef TSAN_MID_APP_RANGE
@@ -143,6 +149,7 @@ void CheckAndProtect() {
ProtectRange(TraceMemBeg(), TraceMemEnd());
ProtectRange(TraceMemEnd(), HeapMemBeg());
ProtectRange(HeapEnd(), HiAppMemBeg());
+#endif
}
#endif
diff --git a/lib/tsan/rtl/tsan_rtl.cc b/lib/tsan/rtl/tsan_rtl.cc
index fa60f3247c38..a01525302b02 100644
--- a/lib/tsan/rtl/tsan_rtl.cc
+++ b/lib/tsan/rtl/tsan_rtl.cc
@@ -104,7 +104,8 @@ Context::Context()
, racy_stacks(MBlockRacyStacks)
, racy_addresses(MBlockRacyAddresses)
, fired_suppressions_mtx(MutexTypeFired, StatMtxFired)
- , fired_suppressions(8) {
+ , fired_suppressions(8)
+ , clock_alloc("clock allocator") {
}
// The objects are allocated in TLS, so one may rely on zero-initialization.
diff --git a/lib/tsan/rtl/tsan_rtl_aarch64.S b/lib/tsan/rtl/tsan_rtl_aarch64.S
index ef06f0444ae4..61171d635c18 100644
--- a/lib/tsan/rtl/tsan_rtl_aarch64.S
+++ b/lib/tsan/rtl/tsan_rtl_aarch64.S
@@ -1,13 +1,46 @@
+// The content of this file is AArch64-only:
+#if defined(__aarch64__)
+
#include "sanitizer_common/sanitizer_asm.h"
+#if !defined(__APPLE__)
.section .bss
.type __tsan_pointer_chk_guard, %object
-.size __tsan_pointer_chk_guard, 8
+ASM_SIZE(ASM_TSAN_SYMBOL_INTERCEPTOR(__tsan_pointer_chk_guard))
__tsan_pointer_chk_guard:
.zero 8
+#endif
+
+#if defined(__APPLE__)
+.align 2
+
+.section __DATA,__nl_symbol_ptr,non_lazy_symbol_pointers
+.long _setjmp$non_lazy_ptr
+_setjmp$non_lazy_ptr:
+.indirect_symbol _setjmp
+.long 0
+
+.section __DATA,__nl_symbol_ptr,non_lazy_symbol_pointers
+.long __setjmp$non_lazy_ptr
+__setjmp$non_lazy_ptr:
+.indirect_symbol __setjmp
+.long 0
+
+.section __DATA,__nl_symbol_ptr,non_lazy_symbol_pointers
+.long _sigsetjmp$non_lazy_ptr
+_sigsetjmp$non_lazy_ptr:
+.indirect_symbol _sigsetjmp
+.long 0
+#endif
+#if !defined(__APPLE__)
.section .text
+#else
+.section __TEXT,__text
+.align 3
+#endif
+#if !defined(__APPLE__)
// GLIBC mangles the function pointers in jmp_buf (used in {set,long}*jmp
// functions) by XORing them with a random guard pointer. For AArch64 it is a
// global variable rather than a TCB one (as for x86_64/powerpc) and althought
@@ -16,9 +49,9 @@ __tsan_pointer_chk_guard:
// not stable). So InitializeGuardPtr obtains the pointer guard value by
// issuing a setjmp and checking the resulting pointers values against the
// original ones.
-.hidden _Z18InitializeGuardPtrv
+ASM_HIDDEN(_Z18InitializeGuardPtrv)
.global _Z18InitializeGuardPtrv
-.type _Z18InitializeGuardPtrv, @function
+ASM_TYPE_FUNCTION(ASM_TSAN_SYMBOL_INTERCEPTOR(_Z18InitializeGuardPtrv))
_Z18InitializeGuardPtrv:
CFI_STARTPROC
// Allocates a jmp_buf for the setjmp call.
@@ -55,12 +88,14 @@ _Z18InitializeGuardPtrv:
CFI_DEF_CFA (31, 0)
ret
CFI_ENDPROC
-.size _Z18InitializeGuardPtrv, .-_Z18InitializeGuardPtrv
+ASM_SIZE(ASM_TSAN_SYMBOL_INTERCEPTOR(_Z18InitializeGuardPtrv))
+#endif
-.hidden __tsan_setjmp
+ASM_HIDDEN(__tsan_setjmp)
.comm _ZN14__interception11real_setjmpE,8,8
-.type setjmp, @function
-setjmp:
+.globl ASM_TSAN_SYMBOL_INTERCEPTOR(setjmp)
+ASM_TYPE_FUNCTION(ASM_TSAN_SYMBOL_INTERCEPTOR(setjmp))
+ASM_TSAN_SYMBOL_INTERCEPTOR(setjmp):
CFI_STARTPROC
// save env parameters for function call
@@ -78,14 +113,19 @@ setjmp:
CFI_OFFSET (19, -16)
mov x19, x0
+#if !defined(__APPLE__)
// SP pointer mangling (see glibc setjmp)
adrp x2, __tsan_pointer_chk_guard
ldr x2, [x2, #:lo12:__tsan_pointer_chk_guard]
add x0, x29, 32
eor x1, x2, x0
+#else
+ add x0, x29, 32
+ mov x1, x0
+#endif
// call tsan interceptor
- bl __tsan_setjmp
+ bl ASM_TSAN_SYMBOL(__tsan_setjmp)
// restore env parameter
mov x0, x19
@@ -96,18 +136,24 @@ setjmp:
CFI_DEF_CFA (31, 0)
// tail jump to libc setjmp
+#if !defined(__APPLE__)
adrp x1, :got:_ZN14__interception11real_setjmpE
ldr x1, [x1, #:got_lo12:_ZN14__interception11real_setjmpE]
ldr x1, [x1]
+#else
+ adrp x1, _setjmp$non_lazy_ptr@page
+ add x1, x1, _setjmp$non_lazy_ptr@pageoff
+ ldr x1, [x1]
+#endif
br x1
CFI_ENDPROC
-.size setjmp, .-setjmp
+ASM_SIZE(ASM_TSAN_SYMBOL_INTERCEPTOR(setjmp))
.comm _ZN14__interception12real__setjmpE,8,8
-.globl _setjmp
-.type _setjmp, @function
-_setjmp:
+.globl ASM_TSAN_SYMBOL_INTERCEPTOR(_setjmp)
+ASM_TYPE_FUNCTION(ASM_TSAN_SYMBOL_INTERCEPTOR(_setjmp))
+ASM_TSAN_SYMBOL_INTERCEPTOR(_setjmp):
CFI_STARTPROC
// save env parameters for function call
@@ -125,14 +171,19 @@ _setjmp:
CFI_OFFSET (19, -16)
mov x19, x0
+#if !defined(__APPLE__)
// SP pointer mangling (see glibc setjmp)
adrp x2, __tsan_pointer_chk_guard
ldr x2, [x2, #:lo12:__tsan_pointer_chk_guard]
add x0, x29, 32
eor x1, x2, x0
+#else
+ add x0, x29, 32
+ mov x1, x0
+#endif
// call tsan interceptor
- bl __tsan_setjmp
+ bl ASM_TSAN_SYMBOL(__tsan_setjmp)
// Restore jmp_buf parameter
mov x0, x19
@@ -143,18 +194,24 @@ _setjmp:
CFI_DEF_CFA (31, 0)
// tail jump to libc setjmp
+#if !defined(__APPLE__)
adrp x1, :got:_ZN14__interception12real__setjmpE
ldr x1, [x1, #:got_lo12:_ZN14__interception12real__setjmpE]
ldr x1, [x1]
+#else
+ adrp x1, __setjmp$non_lazy_ptr@page
+ add x1, x1, __setjmp$non_lazy_ptr@pageoff
+ ldr x1, [x1]
+#endif
br x1
CFI_ENDPROC
-.size _setjmp, .-_setjmp
+ASM_SIZE(ASM_TSAN_SYMBOL_INTERCEPTOR(_setjmp))
.comm _ZN14__interception14real_sigsetjmpE,8,8
-.globl sigsetjmp
-.type sigsetjmp, @function
-sigsetjmp:
+.globl ASM_TSAN_SYMBOL_INTERCEPTOR(sigsetjmp)
+ASM_TYPE_FUNCTION(ASM_TSAN_SYMBOL_INTERCEPTOR(sigsetjmp))
+ASM_TSAN_SYMBOL_INTERCEPTOR(sigsetjmp):
CFI_STARTPROC
// save env parameters for function call
@@ -174,14 +231,19 @@ sigsetjmp:
mov w20, w1
mov x19, x0
+#if !defined(__APPLE__)
// SP pointer mangling (see glibc setjmp)
adrp x2, __tsan_pointer_chk_guard
ldr x2, [x2, #:lo12:__tsan_pointer_chk_guard]
add x0, x29, 32
eor x1, x2, x0
+#else
+ add x0, x29, 32
+ mov x1, x0
+#endif
// call tsan interceptor
- bl __tsan_setjmp
+ bl ASM_TSAN_SYMBOL(__tsan_setjmp)
// restore env parameter
mov w1, w20
@@ -195,17 +257,24 @@ sigsetjmp:
CFI_DEF_CFA (31, 0)
// tail jump to libc sigsetjmp
+#if !defined(__APPLE__)
adrp x2, :got:_ZN14__interception14real_sigsetjmpE
ldr x2, [x2, #:got_lo12:_ZN14__interception14real_sigsetjmpE]
ldr x2, [x2]
+#else
+ adrp x2, _sigsetjmp$non_lazy_ptr@page
+ add x2, x2, _sigsetjmp$non_lazy_ptr@pageoff
+ ldr x2, [x2]
+#endif
br x2
CFI_ENDPROC
-.size sigsetjmp, .-sigsetjmp
+ASM_SIZE(ASM_TSAN_SYMBOL_INTERCEPTOR(sigsetjmp))
+#if !defined(__APPLE__)
.comm _ZN14__interception16real___sigsetjmpE,8,8
-.globl __sigsetjmp
-.type __sigsetjmp, @function
-__sigsetjmp:
+.globl ASM_TSAN_SYMBOL_INTERCEPTOR(__sigsetjmp)
+ASM_TYPE_FUNCTION(ASM_TSAN_SYMBOL_INTERCEPTOR(__sigsetjmp))
+ASM_TSAN_SYMBOL_INTERCEPTOR(__sigsetjmp):
CFI_STARTPROC
// save env parameters for function call
@@ -225,14 +294,16 @@ __sigsetjmp:
mov w20, w1
mov x19, x0
+#if !defined(__APPLE__)
// SP pointer mangling (see glibc setjmp)
adrp x2, __tsan_pointer_chk_guard
ldr x2, [x2, #:lo12:__tsan_pointer_chk_guard]
add x0, x29, 32
eor x1, x2, x0
+#endif
// call tsan interceptor
- bl __tsan_setjmp
+ bl ASM_TSAN_SYMBOL(__tsan_setjmp)
mov w1, w20
mov x0, x19
@@ -245,14 +316,22 @@ __sigsetjmp:
CFI_DEF_CFA (31, 0)
// tail jump to libc __sigsetjmp
+#if !defined(__APPLE__)
adrp x2, :got:_ZN14__interception16real___sigsetjmpE
ldr x2, [x2, #:got_lo12:_ZN14__interception16real___sigsetjmpE]
ldr x2, [x2]
+#else
+ adrp x2, ASM_TSAN_SYMBOL(__sigsetjmp)@page
+ add x2, x2, ASM_TSAN_SYMBOL(__sigsetjmp)@pageoff
+#endif
br x2
CFI_ENDPROC
-.size __sigsetjmp, .-__sigsetjmp
+ASM_SIZE(ASM_TSAN_SYMBOL_INTERCEPTOR(__sigsetjmp))
+#endif
#if defined(__linux__)
/* We do not need executable stack. */
.section .note.GNU-stack,"",@progbits
#endif
+
+#endif
diff --git a/lib/tsan/rtl/tsan_rtl_amd64.S b/lib/tsan/rtl/tsan_rtl_amd64.S
index caa832375e52..98947fd2a1ba 100644
--- a/lib/tsan/rtl/tsan_rtl_amd64.S
+++ b/lib/tsan/rtl/tsan_rtl_amd64.S
@@ -1,4 +1,8 @@
+// The content of this file is x86_64-only:
+#if defined(__x86_64__)
+
#include "sanitizer_common/sanitizer_asm.h"
+
#if !defined(__APPLE__)
.section .text
#else
@@ -357,3 +361,5 @@ ASM_SIZE(ASM_TSAN_SYMBOL_INTERCEPTOR(__sigsetjmp))
/* We do not need executable stack. */
.section .note.GNU-stack,"",@progbits
#endif
+
+#endif
diff --git a/lib/tsan/rtl/tsan_rtl_mutex.cc b/lib/tsan/rtl/tsan_rtl_mutex.cc
index 54938f37e243..2f85811620f1 100644
--- a/lib/tsan/rtl/tsan_rtl_mutex.cc
+++ b/lib/tsan/rtl/tsan_rtl_mutex.cc
@@ -413,10 +413,10 @@ void Acquire(ThreadState *thr, uptr pc, uptr addr) {
static void UpdateClockCallback(ThreadContextBase *tctx_base, void *arg) {
ThreadState *thr = reinterpret_cast<ThreadState*>(arg);
ThreadContext *tctx = static_cast<ThreadContext*>(tctx_base);
+ u64 epoch = tctx->epoch1;
if (tctx->status == ThreadStatusRunning)
- thr->clock.set(tctx->tid, tctx->thr->fast_state.epoch());
- else
- thr->clock.set(tctx->tid, tctx->epoch1);
+ epoch = tctx->thr->fast_state.epoch();
+ thr->clock.set(&thr->proc()->clock_cache, tctx->tid, epoch);
}
void AcquireGlobal(ThreadState *thr, uptr pc) {
@@ -456,10 +456,10 @@ void ReleaseStore(ThreadState *thr, uptr pc, uptr addr) {
static void UpdateSleepClockCallback(ThreadContextBase *tctx_base, void *arg) {
ThreadState *thr = reinterpret_cast<ThreadState*>(arg);
ThreadContext *tctx = static_cast<ThreadContext*>(tctx_base);
+ u64 epoch = tctx->epoch1;
if (tctx->status == ThreadStatusRunning)
- thr->last_sleep_clock.set(tctx->tid, tctx->thr->fast_state.epoch());
- else
- thr->last_sleep_clock.set(tctx->tid, tctx->epoch1);
+ epoch = tctx->thr->fast_state.epoch();
+ thr->last_sleep_clock.set(&thr->proc()->clock_cache, tctx->tid, epoch);
}
void AfterSleep(ThreadState *thr, uptr pc) {
diff --git a/lib/tsan/rtl/tsan_rtl_report.cc b/lib/tsan/rtl/tsan_rtl_report.cc
index 68b9f50308ea..85a982941ed7 100644
--- a/lib/tsan/rtl/tsan_rtl_report.cc
+++ b/lib/tsan/rtl/tsan_rtl_report.cc
@@ -314,7 +314,7 @@ void ScopedReport::AddLocation(uptr addr, uptr size) {
return;
#if !SANITIZER_GO
int fd = -1;
- int creat_tid = -1;
+ int creat_tid = kInvalidTid;
u32 creat_stack = 0;
if (FdLocation(addr, &fd, &creat_tid, &creat_stack)) {
ReportLocation *loc = ReportLocation::New(ReportLocationFD);
diff --git a/lib/tsan/rtl/tsan_rtl_thread.cc b/lib/tsan/rtl/tsan_rtl_thread.cc
index 67eebf5d0c38..83fab082afe3 100644
--- a/lib/tsan/rtl/tsan_rtl_thread.cc
+++ b/lib/tsan/rtl/tsan_rtl_thread.cc
@@ -142,6 +142,10 @@ void ThreadContext::OnFinished() {
if (common_flags()->detect_deadlocks)
ctx->dd->DestroyLogicalThread(thr->dd_lt);
+ thr->clock.ResetCached(&thr->proc()->clock_cache);
+#if !SANITIZER_GO
+ thr->last_sleep_clock.ResetCached(&thr->proc()->clock_cache);
+#endif
thr->~ThreadState();
#if TSAN_COLLECT_STATS
StatAggregate(ctx->stat, thr->stat);
diff --git a/lib/tsan/rtl/tsan_stat.cc b/lib/tsan/rtl/tsan_stat.cc
index 2ee688bf5771..18c83d5c6dac 100644
--- a/lib/tsan/rtl/tsan_stat.cc
+++ b/lib/tsan/rtl/tsan_stat.cc
@@ -75,14 +75,11 @@ void StatOutput(u64 *stat) {
name[StatClockAcquire] = "Clock acquire ";
name[StatClockAcquireEmpty] = " empty clock ";
name[StatClockAcquireFastRelease] = " fast from release-store ";
- name[StatClockAcquireLarge] = " contains my tid ";
- name[StatClockAcquireRepeat] = " repeated (fast) ";
name[StatClockAcquireFull] = " full (slow) ";
name[StatClockAcquiredSomething] = " acquired something ";
name[StatClockRelease] = "Clock release ";
name[StatClockReleaseResize] = " resize ";
- name[StatClockReleaseFast1] = " fast1 ";
- name[StatClockReleaseFast2] = " fast2 ";
+ name[StatClockReleaseFast] = " fast ";
name[StatClockReleaseSlow] = " dirty overflow (slow) ";
name[StatClockReleaseFull] = " full (slow) ";
name[StatClockReleaseAcquired] = " was acquired ";
diff --git a/lib/tsan/rtl/tsan_stat.h b/lib/tsan/rtl/tsan_stat.h
index 7d2791ebbfcc..42d6a2b63532 100644
--- a/lib/tsan/rtl/tsan_stat.h
+++ b/lib/tsan/rtl/tsan_stat.h
@@ -74,15 +74,12 @@ enum StatType {
StatClockAcquire,
StatClockAcquireEmpty,
StatClockAcquireFastRelease,
- StatClockAcquireLarge,
- StatClockAcquireRepeat,
StatClockAcquireFull,
StatClockAcquiredSomething,
// Clocks - release.
StatClockRelease,
StatClockReleaseResize,
- StatClockReleaseFast1,
- StatClockReleaseFast2,
+ StatClockReleaseFast,
StatClockReleaseSlow,
StatClockReleaseFull,
StatClockReleaseAcquired,
diff --git a/lib/tsan/rtl/tsan_sync.cc b/lib/tsan/rtl/tsan_sync.cc
index 4cc3cb89c34f..44ae558fa1b2 100644
--- a/lib/tsan/rtl/tsan_sync.cc
+++ b/lib/tsan/rtl/tsan_sync.cc
@@ -53,7 +53,9 @@ void SyncVar::Reset(Processor *proc) {
}
}
-MetaMap::MetaMap() {
+MetaMap::MetaMap()
+ : block_alloc_("heap block allocator")
+ , sync_alloc_("sync allocator") {
atomic_store(&uid_gen_, 0, memory_order_relaxed);
}
diff --git a/lib/tsan/tests/CMakeLists.txt b/lib/tsan/tests/CMakeLists.txt
index ca43a928d1b8..f8aec6854bb3 100644
--- a/lib/tsan/tests/CMakeLists.txt
+++ b/lib/tsan/tests/CMakeLists.txt
@@ -15,6 +15,7 @@ set(TSAN_UNITTEST_CFLAGS
if(APPLE)
list(APPEND TSAN_UNITTEST_CFLAGS ${DARWIN_osx_CFLAGS})
+ list(APPEND TSAN_UNITTEST_LINKFLAGS ${DARWIN_osx_LINKFLAGS})
endif()
set(TSAN_RTL_HEADERS)
diff --git a/lib/tsan/tests/unit/tsan_clock_test.cc b/lib/tsan/tests/unit/tsan_clock_test.cc
index 83e25fb5a933..73104dd6b9d4 100644
--- a/lib/tsan/tests/unit/tsan_clock_test.cc
+++ b/lib/tsan/tests/unit/tsan_clock_test.cc
@@ -26,13 +26,13 @@ TEST(Clock, VectorBasic) {
clk.tick();
ASSERT_EQ(clk.size(), 1U);
ASSERT_EQ(clk.get(0), 1U);
- clk.set(3, clk.get(3) + 1);
+ clk.set(&cache, 3, clk.get(3) + 1);
ASSERT_EQ(clk.size(), 4U);
ASSERT_EQ(clk.get(0), 1U);
ASSERT_EQ(clk.get(1), 0U);
ASSERT_EQ(clk.get(2), 0U);
ASSERT_EQ(clk.get(3), 1U);
- clk.set(3, clk.get(3) + 1);
+ clk.set(&cache, 3, clk.get(3) + 1);
ASSERT_EQ(clk.get(3), 2U);
}
@@ -86,24 +86,26 @@ TEST(Clock, RepeatedAcquire) {
TEST(Clock, ManyThreads) {
SyncClock chunked;
- for (unsigned i = 0; i < 100; i++) {
+ for (unsigned i = 0; i < 200; i++) {
ThreadClock vector(0);
vector.tick();
- vector.set(i, 1);
+ vector.set(&cache, i, i + 1);
vector.release(&cache, &chunked);
ASSERT_EQ(i + 1, chunked.size());
vector.acquire(&cache, &chunked);
ASSERT_EQ(i + 1, vector.size());
}
- for (unsigned i = 0; i < 100; i++)
- ASSERT_EQ(1U, chunked.get(i));
+ for (unsigned i = 0; i < 200; i++) {
+ printf("i=%d\n", i);
+ ASSERT_EQ(i + 1, chunked.get(i));
+ }
ThreadClock vector(1);
vector.acquire(&cache, &chunked);
- ASSERT_EQ(100U, vector.size());
- for (unsigned i = 0; i < 100; i++)
- ASSERT_EQ(1U, vector.get(i));
+ ASSERT_EQ(200U, vector.size());
+ for (unsigned i = 0; i < 200; i++)
+ ASSERT_EQ(i + 1, vector.get(i));
chunked.Reset(&cache);
}
@@ -151,7 +153,7 @@ TEST(Clock, Growth) {
{
ThreadClock vector(10);
vector.tick();
- vector.set(5, 42);
+ vector.set(&cache, 5, 42);
SyncClock sync;
vector.release(&cache, &sync);
ASSERT_EQ(sync.size(), 11U);
@@ -180,8 +182,8 @@ TEST(Clock, Growth) {
{
ThreadClock vector(100);
vector.tick();
- vector.set(5, 42);
- vector.set(90, 84);
+ vector.set(&cache, 5, 42);
+ vector.set(&cache, 90, 84);
SyncClock sync;
vector.release(&cache, &sync);
ASSERT_EQ(sync.size(), 101U);
@@ -212,6 +214,42 @@ TEST(Clock, Growth) {
}
}
+TEST(Clock, Growth2) {
+ // Test clock growth for every pair of sizes:
+ const uptr sizes[] = {0, 1, 2, 30, 61, 62, 63, 64, 65, 66, 100, 124, 125, 126,
+ 127, 128, 129, 130, 188, 189, 190, 191, 192, 193, 254, 255};
+ const uptr n = sizeof(sizes) / sizeof(sizes[0]);
+ for (uptr fi = 0; fi < n; fi++) {
+ for (uptr ti = fi + 1; ti < n; ti++) {
+ const uptr from = sizes[fi];
+ const uptr to = sizes[ti];
+ SyncClock sync;
+ ThreadClock vector(0);
+ for (uptr i = 0; i < from; i++)
+ vector.set(&cache, i, i + 1);
+ if (from != 0)
+ vector.release(&cache, &sync);
+ ASSERT_EQ(sync.size(), from);
+ for (uptr i = 0; i < from; i++)
+ ASSERT_EQ(sync.get(i), i + 1);
+ for (uptr i = 0; i < to; i++)
+ vector.set(&cache, i, i + 1);
+ vector.release(&cache, &sync);
+ ASSERT_EQ(sync.size(), to);
+ for (uptr i = 0; i < to; i++)
+ ASSERT_EQ(sync.get(i), i + 1);
+ vector.set(&cache, to + 1, to + 1);
+ vector.release(&cache, &sync);
+ ASSERT_EQ(sync.size(), to + 2);
+ for (uptr i = 0; i < to; i++)
+ ASSERT_EQ(sync.get(i), i + 1);
+ ASSERT_EQ(sync.get(to), 0U);
+ ASSERT_EQ(sync.get(to + 1), to + 1);
+ sync.Reset(&cache);
+ }
+ }
+}
+
const uptr kThreads = 4;
const uptr kClocks = 4;