diff options
Diffstat (limited to 'lib/tsan/rtl/tsan_interceptors.cc')
| -rw-r--r-- | lib/tsan/rtl/tsan_interceptors.cc | 931 |
1 files changed, 596 insertions, 335 deletions
diff --git a/lib/tsan/rtl/tsan_interceptors.cc b/lib/tsan/rtl/tsan_interceptors.cc index a962250568b7..be58ca92cf91 100644 --- a/lib/tsan/rtl/tsan_interceptors.cc +++ b/lib/tsan/rtl/tsan_interceptors.cc @@ -1,4 +1,4 @@ -//===-- tsan_interceptors_linux.cc ----------------------------------------===// +//===-- tsan_interceptors.cc ----------------------------------------------===// // // The LLVM Compiler Infrastructure // @@ -9,16 +9,20 @@ // // This file is a part of ThreadSanitizer (TSan), a race detector. // +// FIXME: move as many interceptors as possible into +// sanitizer_common/sanitizer_common_interceptors.h //===----------------------------------------------------------------------===// -#include "interception/interception.h" #include "sanitizer_common/sanitizer_atomic.h" #include "sanitizer_common/sanitizer_libc.h" #include "sanitizer_common/sanitizer_placement_new.h" -#include "tsan_rtl.h" +#include "sanitizer_common/sanitizer_stacktrace.h" +#include "interception/interception.h" #include "tsan_interface.h" #include "tsan_platform.h" +#include "tsan_rtl.h" #include "tsan_mman.h" +#include "tsan_fd.h" using namespace __tsan; // NOLINT @@ -51,7 +55,7 @@ extern "C" void *pthread_self(); extern "C" void _exit(int status); extern "C" int __cxa_atexit(void (*func)(void *arg), void *arg, void *dso); extern "C" int *__errno_location(); -extern "C" int usleep(unsigned usec); +extern "C" int fileno_unlocked(void *stream); const int PTHREAD_MUTEX_RECURSIVE = 1; const int PTHREAD_MUTEX_RECURSIVE_NP = 1; const int kPthreadAttrSize = 56; @@ -69,6 +73,12 @@ const int PTHREAD_BARRIER_SERIAL_THREAD = -1; const int MAP_FIXED = 0x10; typedef long long_t; // NOLINT +// From /usr/include/unistd.h +# define F_ULOCK 0 /* Unlock a previously locked region. */ +# define F_LOCK 1 /* Lock a region for exclusive use. */ +# define F_TLOCK 2 /* Test and lock a region for exclusive use. */ +# define F_TEST 3 /* Test a region for other processes locks. */ + typedef void (*sighandler_t)(int sig); #define errno (*__errno_location()) @@ -94,6 +104,10 @@ const sighandler_t SIG_ERR = (sighandler_t)-1; const int SA_SIGINFO = 4; const int SIG_SETMASK = 2; +namespace std { +struct nothrow_t {}; +} // namespace std + static sigaction_t sigactions[kSigCount]; namespace __tsan { @@ -105,6 +119,7 @@ struct SignalDesc { }; struct SignalContext { + int in_blocking_func; int int_signal_send; int pending_signal_count; SignalDesc pending_signals[kSigCount]; @@ -115,10 +130,8 @@ static SignalContext *SigCtx(ThreadState *thr) { SignalContext *ctx = (SignalContext*)thr->signal_ctx; if (ctx == 0 && thr->is_alive) { ScopedInRtl in_rtl; - ctx = (SignalContext*)internal_alloc( - MBlockSignal, sizeof(*ctx)); - MemoryResetRange(thr, 0, (uptr)ctx, sizeof(*ctx)); - internal_memset(ctx, 0, sizeof(*ctx)); + ctx = (SignalContext*)MmapOrDie(sizeof(*ctx), "SignalContext"); + MemoryResetRange(thr, (uptr)&SigCtx, (uptr)ctx, sizeof(*ctx)); thr->signal_ctx = ctx; } return ctx; @@ -126,60 +139,55 @@ static SignalContext *SigCtx(ThreadState *thr) { static unsigned g_thread_finalize_key; -static void process_pending_signals(ThreadState *thr); - class ScopedInterceptor { public: - ScopedInterceptor(ThreadState *thr, const char *fname, uptr pc) - : thr_(thr) - , in_rtl_(thr->in_rtl) { - if (thr_->in_rtl == 0) { - Initialize(thr); - FuncEntry(thr, pc); - thr_->in_rtl++; - DPrintf("#%d: intercept %s()\n", thr_->tid, fname); - } else { - thr_->in_rtl++; - } - } - - ~ScopedInterceptor() { - thr_->in_rtl--; - if (thr_->in_rtl == 0) { - FuncExit(thr_); - process_pending_signals(thr_); - } - CHECK_EQ(in_rtl_, thr_->in_rtl); - } - + ScopedInterceptor(ThreadState *thr, const char *fname, uptr pc); + ~ScopedInterceptor(); private: ThreadState *const thr_; const int in_rtl_; }; +ScopedInterceptor::ScopedInterceptor(ThreadState *thr, const char *fname, + uptr pc) + : thr_(thr) + , in_rtl_(thr->in_rtl) { + if (thr_->in_rtl == 0) { + Initialize(thr); + FuncEntry(thr, pc); + thr_->in_rtl++; + DPrintf("#%d: intercept %s()\n", thr_->tid, fname); + } else { + thr_->in_rtl++; + } +} + +ScopedInterceptor::~ScopedInterceptor() { + thr_->in_rtl--; + if (thr_->in_rtl == 0) { + FuncExit(thr_); + ProcessPendingSignals(thr_); + } + CHECK_EQ(in_rtl_, thr_->in_rtl); +} + #define SCOPED_INTERCEPTOR_RAW(func, ...) \ ThreadState *thr = cur_thread(); \ StatInc(thr, StatInterceptor); \ StatInc(thr, StatInt_##func); \ - ScopedInterceptor si(thr, #func, \ - (__sanitizer::uptr)__builtin_return_address(0)); \ - const uptr pc = (uptr)&func; \ + const uptr caller_pc = GET_CALLER_PC(); \ + ScopedInterceptor si(thr, #func, caller_pc); \ + const uptr pc = __sanitizer::StackTrace::GetPreviousInstructionPc( \ + __sanitizer::StackTrace::GetCurrentPc()); \ (void)pc; \ /**/ #define SCOPED_TSAN_INTERCEPTOR(func, ...) \ SCOPED_INTERCEPTOR_RAW(func, __VA_ARGS__); \ - if (thr->in_rtl > 1) \ - return REAL(func)(__VA_ARGS__); \ -/**/ - -#define SCOPED_INTERCEPTOR_LIBC(func, ...) \ - ThreadState *thr = cur_thread(); \ - StatInc(thr, StatInterceptor); \ - StatInc(thr, StatInt_##func); \ - ScopedInterceptor si(thr, #func, callpc); \ - const uptr pc = (uptr)&func; \ - (void)pc; \ + if (REAL(func) == 0) { \ + Printf("FATAL: ThreadSanitizer: failed to intercept %s\n", #func); \ + Die(); \ + } \ if (thr->in_rtl > 1) \ return REAL(func)(__VA_ARGS__); \ /**/ @@ -187,30 +195,40 @@ class ScopedInterceptor { #define TSAN_INTERCEPTOR(ret, func, ...) INTERCEPTOR(ret, func, __VA_ARGS__) #define TSAN_INTERCEPT(func) INTERCEPT_FUNCTION(func) -// May be overriden by front-end. -extern "C" void WEAK __tsan_malloc_hook(void *ptr, uptr size) { - (void)ptr; - (void)size; -} +#define BLOCK_REAL(name) (BlockingCall(thr), REAL(name)) -extern "C" void WEAK __tsan_free_hook(void *ptr) { - (void)ptr; +struct BlockingCall { + explicit BlockingCall(ThreadState *thr) + : ctx(SigCtx(thr)) { + ctx->in_blocking_func++; + } + + ~BlockingCall() { + ctx->in_blocking_func--; + } + + SignalContext *ctx; +}; + +TSAN_INTERCEPTOR(unsigned, sleep, unsigned sec) { + SCOPED_TSAN_INTERCEPTOR(sleep, sec); + unsigned res = BLOCK_REAL(sleep)(sec); + AfterSleep(thr, pc); + return res; } -static void invoke_malloc_hook(void *ptr, uptr size) { - Context *ctx = CTX(); - ThreadState *thr = cur_thread(); - if (ctx == 0 || !ctx->initialized || thr->in_rtl) - return; - __tsan_malloc_hook(ptr, size); +TSAN_INTERCEPTOR(int, usleep, long_t usec) { + SCOPED_TSAN_INTERCEPTOR(usleep, usec); + int res = BLOCK_REAL(usleep)(usec); + AfterSleep(thr, pc); + return res; } -static void invoke_free_hook(void *ptr) { - Context *ctx = CTX(); - ThreadState *thr = cur_thread(); - if (ctx == 0 || !ctx->initialized || thr->in_rtl) - return; - __tsan_free_hook(ptr); +TSAN_INTERCEPTOR(int, nanosleep, void *req, void *rem) { + SCOPED_TSAN_INTERCEPTOR(nanosleep, req, rem); + int res = BLOCK_REAL(nanosleep)(req, rem); + AfterSleep(thr, pc); + return res; } class AtExitContext { @@ -269,7 +287,6 @@ static void finalize(void *arg) { { ScopedInRtl in_rtl; DestroyAndFree(atexit_ctx); - usleep(flags()->atexit_sleep_ms * 1000); } int status = Finalize(cur_thread()); if (status) @@ -279,45 +296,20 @@ static void finalize(void *arg) { TSAN_INTERCEPTOR(int, atexit, void (*f)()) { SCOPED_TSAN_INTERCEPTOR(atexit, f); return atexit_ctx->atexit(thr, pc, f); - return 0; } TSAN_INTERCEPTOR(void, longjmp, void *env, int val) { SCOPED_TSAN_INTERCEPTOR(longjmp, env, val); - TsanPrintf("ThreadSanitizer: longjmp() is not supported\n"); + Printf("ThreadSanitizer: longjmp() is not supported\n"); Die(); } TSAN_INTERCEPTOR(void, siglongjmp, void *env, int val) { SCOPED_TSAN_INTERCEPTOR(siglongjmp, env, val); - TsanPrintf("ThreadSanitizer: siglongjmp() is not supported\n"); + Printf("ThreadSanitizer: siglongjmp() is not supported\n"); Die(); } -static uptr fd2addr(int fd) { - (void)fd; - static u64 addr; - return (uptr)&addr; -} - -static uptr epollfd2addr(int fd) { - (void)fd; - static u64 addr; - return (uptr)&addr; -} - -static uptr file2addr(char *path) { - (void)path; - static u64 addr; - return (uptr)&addr; -} - -static uptr dir2addr(char *path) { - (void)path; - static u64 addr; - return (uptr)&addr; -} - TSAN_INTERCEPTOR(void*, malloc, uptr size) { void *p = 0; { @@ -328,12 +320,17 @@ TSAN_INTERCEPTOR(void*, malloc, uptr size) { return p; } +TSAN_INTERCEPTOR(void*, __libc_memalign, uptr align, uptr sz) { + SCOPED_TSAN_INTERCEPTOR(__libc_memalign, align, sz); + return user_alloc(thr, pc, sz, align); +} + TSAN_INTERCEPTOR(void*, calloc, uptr size, uptr n) { void *p = 0; { SCOPED_INTERCEPTOR_RAW(calloc, size, n); p = user_alloc(thr, pc, n * size); - internal_memset(p, 0, n * size); + if (p) internal_memset(p, 0, n * size); } invoke_malloc_hook(p, n * size); return p; @@ -366,6 +363,47 @@ TSAN_INTERCEPTOR(void, cfree, void *p) { user_free(thr, pc, p); } +#define OPERATOR_NEW_BODY(mangled_name) \ + void *p = 0; \ + { \ + SCOPED_INTERCEPTOR_RAW(mangled_name, size); \ + p = user_alloc(thr, pc, size); \ + } \ + invoke_malloc_hook(p, size); \ + return p; + +void *operator new(__sanitizer::uptr size) { + OPERATOR_NEW_BODY(_Znwm); +} +void *operator new[](__sanitizer::uptr size) { + OPERATOR_NEW_BODY(_Znam); +} +void *operator new(__sanitizer::uptr size, std::nothrow_t const&) { + OPERATOR_NEW_BODY(_ZnwmRKSt9nothrow_t); +} +void *operator new[](__sanitizer::uptr size, std::nothrow_t const&) { + OPERATOR_NEW_BODY(_ZnamRKSt9nothrow_t); +} + +#define OPERATOR_DELETE_BODY(mangled_name) \ + if (ptr == 0) return; \ + invoke_free_hook(ptr); \ + SCOPED_INTERCEPTOR_RAW(mangled_name, ptr); \ + user_free(thr, pc, ptr); + +void operator delete(void *ptr) { + OPERATOR_DELETE_BODY(_ZdlPv); +} +void operator delete[](void *ptr) { + OPERATOR_DELETE_BODY(_ZdlPvRKSt9nothrow_t); +} +void operator delete(void *ptr, std::nothrow_t const&) { + OPERATOR_DELETE_BODY(_ZdaPv); +} +void operator delete[](void *ptr, std::nothrow_t const&) { + OPERATOR_DELETE_BODY(_ZdaPvRKSt9nothrow_t); +} + TSAN_INTERCEPTOR(uptr, strlen, const char *s) { SCOPED_TSAN_INTERCEPTOR(strlen, s); uptr len = internal_strlen(s); @@ -536,137 +574,61 @@ TSAN_INTERCEPTOR(int, munmap, void *addr, long_t sz) { return res; } -#ifdef __LP64__ - -// void *operator new(size_t) -TSAN_INTERCEPTOR(void*, _Znwm, uptr sz) { - void *p = 0; - { - SCOPED_TSAN_INTERCEPTOR(_Znwm, sz); - p = user_alloc(thr, pc, sz); - } - invoke_malloc_hook(p, sz); - return p; -} - -// void *operator new(size_t, nothrow_t) -TSAN_INTERCEPTOR(void*, _ZnwmRKSt9nothrow_t, uptr sz) { - void *p = 0; - { - SCOPED_TSAN_INTERCEPTOR(_ZnwmRKSt9nothrow_t, sz); - p = user_alloc(thr, pc, sz); - } - invoke_malloc_hook(p, sz); - return p; -} - -// void *operator new[](size_t) -TSAN_INTERCEPTOR(void*, _Znam, uptr sz) { - void *p = 0; - { - SCOPED_TSAN_INTERCEPTOR(_Znam, sz); - p = user_alloc(thr, pc, sz); - } - invoke_malloc_hook(p, sz); - return p; -} - -// void *operator new[](size_t, nothrow_t) -TSAN_INTERCEPTOR(void*, _ZnamRKSt9nothrow_t, uptr sz) { - void *p = 0; - { - SCOPED_TSAN_INTERCEPTOR(_ZnamRKSt9nothrow_t, sz); - p = user_alloc(thr, pc, sz); - } - invoke_malloc_hook(p, sz); - return p; -} - -#else -#error "Not implemented" -#endif - -// void operator delete(void*) -TSAN_INTERCEPTOR(void, _ZdlPv, void *p) { - if (p == 0) - return; - invoke_free_hook(p); - SCOPED_TSAN_INTERCEPTOR(_ZdlPv, p); - user_free(thr, pc, p); -} - -// void operator delete(void*, nothrow_t) -TSAN_INTERCEPTOR(void, _ZdlPvRKSt9nothrow_t, void *p) { - if (p == 0) - return; - invoke_free_hook(p); - SCOPED_TSAN_INTERCEPTOR(_ZdlPvRKSt9nothrow_t, p); - user_free(thr, pc, p); -} - -// void operator delete[](void*) -TSAN_INTERCEPTOR(void, _ZdaPv, void *p) { - if (p == 0) - return; - invoke_free_hook(p); - SCOPED_TSAN_INTERCEPTOR(_ZdaPv, p); - user_free(thr, pc, p); -} - -// void operator delete[](void*, nothrow_t) -TSAN_INTERCEPTOR(void, _ZdaPvRKSt9nothrow_t, void *p) { - if (p == 0) - return; - invoke_free_hook(p); - SCOPED_TSAN_INTERCEPTOR(_ZdaPvRKSt9nothrow_t, p); - user_free(thr, pc, p); -} - TSAN_INTERCEPTOR(void*, memalign, uptr align, uptr sz) { SCOPED_TSAN_INTERCEPTOR(memalign, align, sz); - return user_alloc_aligned(thr, pc, sz, align); + return user_alloc(thr, pc, sz, align); } TSAN_INTERCEPTOR(void*, valloc, uptr sz) { SCOPED_TSAN_INTERCEPTOR(valloc, sz); - return user_alloc_aligned(thr, pc, sz, kPageSize); + return user_alloc(thr, pc, sz, GetPageSizeCached()); } TSAN_INTERCEPTOR(void*, pvalloc, uptr sz) { SCOPED_TSAN_INTERCEPTOR(pvalloc, sz); - sz = RoundUp(sz, kPageSize); - return user_alloc_aligned(thr, pc, sz, kPageSize); + sz = RoundUp(sz, GetPageSizeCached()); + return user_alloc(thr, pc, sz, GetPageSizeCached()); } TSAN_INTERCEPTOR(int, posix_memalign, void **memptr, uptr align, uptr sz) { SCOPED_TSAN_INTERCEPTOR(posix_memalign, memptr, align, sz); - *memptr = user_alloc_aligned(thr, pc, sz, align); + *memptr = user_alloc(thr, pc, sz, align); return 0; } // Used in thread-safe function static initialization. -TSAN_INTERCEPTOR(int, __cxa_guard_acquire, char *m) { - SCOPED_TSAN_INTERCEPTOR(__cxa_guard_acquire, m); - int res = REAL(__cxa_guard_acquire)(m); - if (res) { - // This thread does the init. - } else { - Acquire(thr, pc, (uptr)m); +extern "C" int INTERFACE_ATTRIBUTE __cxa_guard_acquire(atomic_uint32_t *g) { + SCOPED_INTERCEPTOR_RAW(__cxa_guard_acquire, g); + for (;;) { + u32 cmp = atomic_load(g, memory_order_acquire); + if (cmp == 0) { + if (atomic_compare_exchange_strong(g, &cmp, 1<<16, memory_order_relaxed)) + return 1; + } else if (cmp == 1) { + Acquire(thr, pc, (uptr)g); + return 0; + } else { + internal_sched_yield(); + } } - return res; } -TSAN_INTERCEPTOR(void, __cxa_guard_release, char *m) { - SCOPED_TSAN_INTERCEPTOR(__cxa_guard_release, m); - Release(thr, pc, (uptr)m); - REAL(__cxa_guard_release)(m); +extern "C" void INTERFACE_ATTRIBUTE __cxa_guard_release(atomic_uint32_t *g) { + SCOPED_INTERCEPTOR_RAW(__cxa_guard_release, g); + Release(thr, pc, (uptr)g); + atomic_store(g, 1, memory_order_release); +} + +extern "C" void INTERFACE_ATTRIBUTE __cxa_guard_abort(atomic_uint32_t *g) { + SCOPED_INTERCEPTOR_RAW(__cxa_guard_abort, g); + atomic_store(g, 0, memory_order_relaxed); } static void thread_finalize(void *v) { uptr iter = (uptr)v; if (iter > 1) { if (pthread_setspecific(g_thread_finalize_key, (void*)(iter - 1))) { - TsanPrintf("ThreadSanitizer: failed to set thread key\n"); + Printf("ThreadSanitizer: failed to set thread key\n"); Die(); } return; @@ -678,7 +640,7 @@ static void thread_finalize(void *v) { SignalContext *sctx = thr->signal_ctx; if (sctx) { thr->signal_ctx = 0; - internal_free(sctx); + UnmapOrDie(sctx, sizeof(*sctx)); } } } @@ -699,13 +661,13 @@ extern "C" void *__tsan_thread_start_func(void *arg) { ThreadState *thr = cur_thread(); ScopedInRtl in_rtl; if (pthread_setspecific(g_thread_finalize_key, (void*)4)) { - TsanPrintf("ThreadSanitizer: failed to set thread key\n"); + Printf("ThreadSanitizer: failed to set thread key\n"); Die(); } while ((tid = atomic_load(&p->tid, memory_order_acquire)) == 0) pthread_yield(); atomic_store(&p->tid, 0, memory_order_release); - ThreadStart(thr, tid); + ThreadStart(thr, tid, GetTid()); CHECK_EQ(thr->in_rtl, 1); } void *res = callback(param); @@ -740,7 +702,7 @@ TSAN_INTERCEPTOR(int, pthread_create, atomic_store(&p.tid, 0, memory_order_relaxed); int res = REAL(pthread_create)(th, attr, __tsan_thread_start_func, &p); if (res == 0) { - int tid = ThreadCreate(cur_thread(), pc, *(uptr*)th, detached); + int tid = ThreadCreate(thr, pc, *(uptr*)th, detached); CHECK_NE(tid, 0); atomic_store(&p.tid, tid, memory_order_release); while (atomic_load(&p.tid, memory_order_acquire) != 0) @@ -754,9 +716,9 @@ TSAN_INTERCEPTOR(int, pthread_create, TSAN_INTERCEPTOR(int, pthread_join, void *th, void **ret) { SCOPED_TSAN_INTERCEPTOR(pthread_join, th, ret); int tid = ThreadTid(thr, pc, (uptr)th); - int res = REAL(pthread_join)(th, ret); + int res = BLOCK_REAL(pthread_join)(th, ret); if (res == 0) { - ThreadJoin(cur_thread(), pc, tid); + ThreadJoin(thr, pc, tid); } return res; } @@ -766,7 +728,7 @@ TSAN_INTERCEPTOR(int, pthread_detach, void *th) { int tid = ThreadTid(thr, pc, (uptr)th); int res = REAL(pthread_detach)(th); if (res == 0) { - ThreadDetach(cur_thread(), pc, tid); + ThreadDetach(thr, pc, tid); } return res; } @@ -782,7 +744,7 @@ TSAN_INTERCEPTOR(int, pthread_mutex_init, void *m, void *a) { recursive = (type == PTHREAD_MUTEX_RECURSIVE || type == PTHREAD_MUTEX_RECURSIVE_NP); } - MutexCreate(cur_thread(), pc, (uptr)m, false, recursive); + MutexCreate(thr, pc, (uptr)m, false, recursive, false); } return res; } @@ -791,7 +753,7 @@ TSAN_INTERCEPTOR(int, pthread_mutex_destroy, void *m) { SCOPED_TSAN_INTERCEPTOR(pthread_mutex_destroy, m); int res = REAL(pthread_mutex_destroy)(m); if (res == 0 || res == EBUSY) { - MutexDestroy(cur_thread(), pc, (uptr)m); + MutexDestroy(thr, pc, (uptr)m); } return res; } @@ -800,7 +762,7 @@ TSAN_INTERCEPTOR(int, pthread_mutex_lock, void *m) { SCOPED_TSAN_INTERCEPTOR(pthread_mutex_lock, m); int res = REAL(pthread_mutex_lock)(m); if (res == 0) { - MutexLock(cur_thread(), pc, (uptr)m); + MutexLock(thr, pc, (uptr)m); } return res; } @@ -809,7 +771,7 @@ TSAN_INTERCEPTOR(int, pthread_mutex_trylock, void *m) { SCOPED_TSAN_INTERCEPTOR(pthread_mutex_trylock, m); int res = REAL(pthread_mutex_trylock)(m); if (res == 0) { - MutexLock(cur_thread(), pc, (uptr)m); + MutexLock(thr, pc, (uptr)m); } return res; } @@ -818,14 +780,14 @@ TSAN_INTERCEPTOR(int, pthread_mutex_timedlock, void *m, void *abstime) { SCOPED_TSAN_INTERCEPTOR(pthread_mutex_timedlock, m, abstime); int res = REAL(pthread_mutex_timedlock)(m, abstime); if (res == 0) { - MutexLock(cur_thread(), pc, (uptr)m); + MutexLock(thr, pc, (uptr)m); } return res; } TSAN_INTERCEPTOR(int, pthread_mutex_unlock, void *m) { SCOPED_TSAN_INTERCEPTOR(pthread_mutex_unlock, m); - MutexUnlock(cur_thread(), pc, (uptr)m); + MutexUnlock(thr, pc, (uptr)m); int res = REAL(pthread_mutex_unlock)(m); return res; } @@ -834,7 +796,7 @@ TSAN_INTERCEPTOR(int, pthread_spin_init, void *m, int pshared) { SCOPED_TSAN_INTERCEPTOR(pthread_spin_init, m, pshared); int res = REAL(pthread_spin_init)(m, pshared); if (res == 0) { - MutexCreate(cur_thread(), pc, (uptr)m, false, false); + MutexCreate(thr, pc, (uptr)m, false, false, false); } return res; } @@ -843,7 +805,7 @@ TSAN_INTERCEPTOR(int, pthread_spin_destroy, void *m) { SCOPED_TSAN_INTERCEPTOR(pthread_spin_destroy, m); int res = REAL(pthread_spin_destroy)(m); if (res == 0) { - MutexDestroy(cur_thread(), pc, (uptr)m); + MutexDestroy(thr, pc, (uptr)m); } return res; } @@ -852,7 +814,7 @@ TSAN_INTERCEPTOR(int, pthread_spin_lock, void *m) { SCOPED_TSAN_INTERCEPTOR(pthread_spin_lock, m); int res = REAL(pthread_spin_lock)(m); if (res == 0) { - MutexLock(cur_thread(), pc, (uptr)m); + MutexLock(thr, pc, (uptr)m); } return res; } @@ -861,14 +823,14 @@ TSAN_INTERCEPTOR(int, pthread_spin_trylock, void *m) { SCOPED_TSAN_INTERCEPTOR(pthread_spin_trylock, m); int res = REAL(pthread_spin_trylock)(m); if (res == 0) { - MutexLock(cur_thread(), pc, (uptr)m); + MutexLock(thr, pc, (uptr)m); } return res; } TSAN_INTERCEPTOR(int, pthread_spin_unlock, void *m) { SCOPED_TSAN_INTERCEPTOR(pthread_spin_unlock, m); - MutexUnlock(cur_thread(), pc, (uptr)m); + MutexUnlock(thr, pc, (uptr)m); int res = REAL(pthread_spin_unlock)(m); return res; } @@ -877,7 +839,7 @@ TSAN_INTERCEPTOR(int, pthread_rwlock_init, void *m, void *a) { SCOPED_TSAN_INTERCEPTOR(pthread_rwlock_init, m, a); int res = REAL(pthread_rwlock_init)(m, a); if (res == 0) { - MutexCreate(cur_thread(), pc, (uptr)m, true, false); + MutexCreate(thr, pc, (uptr)m, true, false, false); } return res; } @@ -886,7 +848,7 @@ TSAN_INTERCEPTOR(int, pthread_rwlock_destroy, void *m) { SCOPED_TSAN_INTERCEPTOR(pthread_rwlock_destroy, m); int res = REAL(pthread_rwlock_destroy)(m); if (res == 0) { - MutexDestroy(cur_thread(), pc, (uptr)m); + MutexDestroy(thr, pc, (uptr)m); } return res; } @@ -895,7 +857,7 @@ TSAN_INTERCEPTOR(int, pthread_rwlock_rdlock, void *m) { SCOPED_TSAN_INTERCEPTOR(pthread_rwlock_rdlock, m); int res = REAL(pthread_rwlock_rdlock)(m); if (res == 0) { - MutexReadLock(cur_thread(), pc, (uptr)m); + MutexReadLock(thr, pc, (uptr)m); } return res; } @@ -904,7 +866,7 @@ TSAN_INTERCEPTOR(int, pthread_rwlock_tryrdlock, void *m) { SCOPED_TSAN_INTERCEPTOR(pthread_rwlock_tryrdlock, m); int res = REAL(pthread_rwlock_tryrdlock)(m); if (res == 0) { - MutexReadLock(cur_thread(), pc, (uptr)m); + MutexReadLock(thr, pc, (uptr)m); } return res; } @@ -913,7 +875,7 @@ TSAN_INTERCEPTOR(int, pthread_rwlock_timedrdlock, void *m, void *abstime) { SCOPED_TSAN_INTERCEPTOR(pthread_rwlock_timedrdlock, m, abstime); int res = REAL(pthread_rwlock_timedrdlock)(m, abstime); if (res == 0) { - MutexReadLock(cur_thread(), pc, (uptr)m); + MutexReadLock(thr, pc, (uptr)m); } return res; } @@ -922,7 +884,7 @@ TSAN_INTERCEPTOR(int, pthread_rwlock_wrlock, void *m) { SCOPED_TSAN_INTERCEPTOR(pthread_rwlock_wrlock, m); int res = REAL(pthread_rwlock_wrlock)(m); if (res == 0) { - MutexLock(cur_thread(), pc, (uptr)m); + MutexLock(thr, pc, (uptr)m); } return res; } @@ -931,7 +893,7 @@ TSAN_INTERCEPTOR(int, pthread_rwlock_trywrlock, void *m) { SCOPED_TSAN_INTERCEPTOR(pthread_rwlock_trywrlock, m); int res = REAL(pthread_rwlock_trywrlock)(m); if (res == 0) { - MutexLock(cur_thread(), pc, (uptr)m); + MutexLock(thr, pc, (uptr)m); } return res; } @@ -940,23 +902,27 @@ TSAN_INTERCEPTOR(int, pthread_rwlock_timedwrlock, void *m, void *abstime) { SCOPED_TSAN_INTERCEPTOR(pthread_rwlock_timedwrlock, m, abstime); int res = REAL(pthread_rwlock_timedwrlock)(m, abstime); if (res == 0) { - MutexLock(cur_thread(), pc, (uptr)m); + MutexLock(thr, pc, (uptr)m); } return res; } TSAN_INTERCEPTOR(int, pthread_rwlock_unlock, void *m) { SCOPED_TSAN_INTERCEPTOR(pthread_rwlock_unlock, m); - MutexReadOrWriteUnlock(cur_thread(), pc, (uptr)m); + MutexReadOrWriteUnlock(thr, pc, (uptr)m); int res = REAL(pthread_rwlock_unlock)(m); return res; } +// libpthread.so contains several versions of pthread_cond_init symbol. +// When we just dlsym() it, we get the wrong (old) version. +/* TSAN_INTERCEPTOR(int, pthread_cond_init, void *c, void *a) { SCOPED_TSAN_INTERCEPTOR(pthread_cond_init, c, a); int res = REAL(pthread_cond_init)(c, a); return res; } +*/ TSAN_INTERCEPTOR(int, pthread_cond_destroy, void *c) { SCOPED_TSAN_INTERCEPTOR(pthread_cond_destroy, c); @@ -978,17 +944,17 @@ TSAN_INTERCEPTOR(int, pthread_cond_broadcast, void *c) { TSAN_INTERCEPTOR(int, pthread_cond_wait, void *c, void *m) { SCOPED_TSAN_INTERCEPTOR(pthread_cond_wait, c, m); - MutexUnlock(cur_thread(), pc, (uptr)m); + MutexUnlock(thr, pc, (uptr)m); int res = REAL(pthread_cond_wait)(c, m); - MutexLock(cur_thread(), pc, (uptr)m); + MutexLock(thr, pc, (uptr)m); return res; } TSAN_INTERCEPTOR(int, pthread_cond_timedwait, void *c, void *m, void *abstime) { SCOPED_TSAN_INTERCEPTOR(pthread_cond_timedwait, c, m, abstime); - MutexUnlock(cur_thread(), pc, (uptr)m); + MutexUnlock(thr, pc, (uptr)m); int res = REAL(pthread_cond_timedwait)(c, m, abstime); - MutexLock(cur_thread(), pc, (uptr)m); + MutexLock(thr, pc, (uptr)m); return res; } @@ -1008,12 +974,12 @@ TSAN_INTERCEPTOR(int, pthread_barrier_destroy, void *b) { TSAN_INTERCEPTOR(int, pthread_barrier_wait, void *b) { SCOPED_TSAN_INTERCEPTOR(pthread_barrier_wait, b); - Release(cur_thread(), pc, (uptr)b); + Release(thr, pc, (uptr)b); MemoryRead1Byte(thr, pc, (uptr)b); int res = REAL(pthread_barrier_wait)(b); MemoryRead1Byte(thr, pc, (uptr)b); if (res == 0 || res == PTHREAD_BARRIER_SERIAL_THREAD) { - Acquire(cur_thread(), pc, (uptr)b); + Acquire(thr, pc, (uptr)b); } return res; } @@ -1031,14 +997,14 @@ TSAN_INTERCEPTOR(int, pthread_once, void *o, void (*f)()) { (*f)(); CHECK_EQ(thr->in_rtl, 0); thr->in_rtl = old_in_rtl; - Release(cur_thread(), pc, (uptr)o); + Release(thr, pc, (uptr)o); atomic_store(a, 2, memory_order_release); } else { while (v != 2) { pthread_yield(); v = atomic_load(a, memory_order_acquire); } - Acquire(cur_thread(), pc, (uptr)o); + Acquire(thr, pc, (uptr)o); } return 0; } @@ -1057,34 +1023,34 @@ TSAN_INTERCEPTOR(int, sem_destroy, void *s) { TSAN_INTERCEPTOR(int, sem_wait, void *s) { SCOPED_TSAN_INTERCEPTOR(sem_wait, s); - int res = REAL(sem_wait)(s); + int res = BLOCK_REAL(sem_wait)(s); if (res == 0) { - Acquire(cur_thread(), pc, (uptr)s); + Acquire(thr, pc, (uptr)s); } return res; } TSAN_INTERCEPTOR(int, sem_trywait, void *s) { SCOPED_TSAN_INTERCEPTOR(sem_trywait, s); - int res = REAL(sem_trywait)(s); + int res = BLOCK_REAL(sem_trywait)(s); if (res == 0) { - Acquire(cur_thread(), pc, (uptr)s); + Acquire(thr, pc, (uptr)s); } return res; } TSAN_INTERCEPTOR(int, sem_timedwait, void *s, void *abstime) { SCOPED_TSAN_INTERCEPTOR(sem_timedwait, s, abstime); - int res = REAL(sem_timedwait)(s, abstime); + int res = BLOCK_REAL(sem_timedwait)(s, abstime); if (res == 0) { - Acquire(cur_thread(), pc, (uptr)s); + Acquire(thr, pc, (uptr)s); } return res; } TSAN_INTERCEPTOR(int, sem_post, void *s) { SCOPED_TSAN_INTERCEPTOR(sem_post, s); - Release(cur_thread(), pc, (uptr)s); + Release(thr, pc, (uptr)s); int res = REAL(sem_post)(s); return res; } @@ -1093,43 +1059,193 @@ TSAN_INTERCEPTOR(int, sem_getvalue, void *s, int *sval) { SCOPED_TSAN_INTERCEPTOR(sem_getvalue, s, sval); int res = REAL(sem_getvalue)(s, sval); if (res == 0) { - Acquire(cur_thread(), pc, (uptr)s); + Acquire(thr, pc, (uptr)s); } return res; } -TSAN_INTERCEPTOR(long_t, read, int fd, void *buf, long_t sz) { - SCOPED_TSAN_INTERCEPTOR(read, fd, buf, sz); - int res = REAL(read)(fd, buf, sz); - if (res >= 0) { - Acquire(cur_thread(), pc, fd2addr(fd)); - } +TSAN_INTERCEPTOR(int, open, const char *name, int flags, int mode) { + SCOPED_TSAN_INTERCEPTOR(open, name, flags, mode); + int fd = REAL(open)(name, flags, mode); + if (fd >= 0) + FdFileCreate(thr, pc, fd); + return fd; +} + +TSAN_INTERCEPTOR(int, open64, const char *name, int flags, int mode) { + SCOPED_TSAN_INTERCEPTOR(open64, name, flags, mode); + int fd = REAL(open64)(name, flags, mode); + if (fd >= 0) + FdFileCreate(thr, pc, fd); + return fd; +} + +TSAN_INTERCEPTOR(int, creat, const char *name, int mode) { + SCOPED_TSAN_INTERCEPTOR(creat, name, mode); + int fd = REAL(creat)(name, mode); + if (fd >= 0) + FdFileCreate(thr, pc, fd); + return fd; +} + +TSAN_INTERCEPTOR(int, creat64, const char *name, int mode) { + SCOPED_TSAN_INTERCEPTOR(creat64, name, mode); + int fd = REAL(creat64)(name, mode); + if (fd >= 0) + FdFileCreate(thr, pc, fd); + return fd; +} + +TSAN_INTERCEPTOR(int, dup, int oldfd) { + SCOPED_TSAN_INTERCEPTOR(dup, oldfd); + int newfd = REAL(dup)(oldfd); + if (oldfd >= 0 && newfd >= 0 && newfd != oldfd) + FdDup(thr, pc, oldfd, newfd); + return newfd; +} + +TSAN_INTERCEPTOR(int, dup2, int oldfd, int newfd) { + SCOPED_TSAN_INTERCEPTOR(dup2, oldfd, newfd); + int newfd2 = REAL(dup2)(oldfd, newfd); + if (oldfd >= 0 && newfd2 >= 0 && newfd2 != oldfd) + FdDup(thr, pc, oldfd, newfd2); + return newfd2; +} + +TSAN_INTERCEPTOR(int, dup3, int oldfd, int newfd, int flags) { + SCOPED_TSAN_INTERCEPTOR(dup3, oldfd, newfd, flags); + int newfd2 = REAL(dup3)(oldfd, newfd, flags); + if (oldfd >= 0 && newfd2 >= 0 && newfd2 != oldfd) + FdDup(thr, pc, oldfd, newfd2); + return newfd2; +} + +TSAN_INTERCEPTOR(int, eventfd, unsigned initval, int flags) { + SCOPED_TSAN_INTERCEPTOR(eventfd, initval, flags); + int fd = REAL(eventfd)(initval, flags); + if (fd >= 0) + FdEventCreate(thr, pc, fd); + return fd; +} + +TSAN_INTERCEPTOR(int, signalfd, int fd, void *mask, int flags) { + SCOPED_TSAN_INTERCEPTOR(signalfd, fd, mask, flags); + if (fd >= 0) + FdClose(thr, pc, fd); + fd = REAL(signalfd)(fd, mask, flags); + if (fd >= 0) + FdSignalCreate(thr, pc, fd); + return fd; +} + +TSAN_INTERCEPTOR(int, inotify_init, int fake) { + SCOPED_TSAN_INTERCEPTOR(inotify_init, fake); + int fd = REAL(inotify_init)(fake); + if (fd >= 0) + FdInotifyCreate(thr, pc, fd); + return fd; +} + +TSAN_INTERCEPTOR(int, inotify_init1, int flags) { + SCOPED_TSAN_INTERCEPTOR(inotify_init1, flags); + int fd = REAL(inotify_init1)(flags); + if (fd >= 0) + FdInotifyCreate(thr, pc, fd); + return fd; +} + +TSAN_INTERCEPTOR(int, socket, int domain, int type, int protocol) { + SCOPED_TSAN_INTERCEPTOR(socket, domain, type, protocol); + int fd = REAL(socket)(domain, type, protocol); + if (fd >= 0) + FdSocketCreate(thr, pc, fd); + return fd; +} + +TSAN_INTERCEPTOR(int, socketpair, int domain, int type, int protocol, int *fd) { + SCOPED_TSAN_INTERCEPTOR(socketpair, domain, type, protocol, fd); + int res = REAL(socketpair)(domain, type, protocol, fd); + if (res == 0 && fd[0] >= 0 && fd[1] >= 0) + FdPipeCreate(thr, pc, fd[0], fd[1]); return res; } -TSAN_INTERCEPTOR(long_t, pread, int fd, void *buf, long_t sz, unsigned off) { - SCOPED_TSAN_INTERCEPTOR(pread, fd, buf, sz, off); - int res = REAL(pread)(fd, buf, sz, off); - if (res >= 0) { - Acquire(cur_thread(), pc, fd2addr(fd)); - } +TSAN_INTERCEPTOR(int, connect, int fd, void *addr, unsigned addrlen) { + SCOPED_TSAN_INTERCEPTOR(connect, fd, addr, addrlen); + FdSocketConnecting(thr, pc, fd); + int res = REAL(connect)(fd, addr, addrlen); + if (res == 0 && fd >= 0) + FdSocketConnect(thr, pc, fd); return res; } -TSAN_INTERCEPTOR(long_t, pread64, int fd, void *buf, long_t sz, u64 off) { - SCOPED_TSAN_INTERCEPTOR(pread64, fd, buf, sz, off); - int res = REAL(pread64)(fd, buf, sz, off); - if (res >= 0) { - Acquire(cur_thread(), pc, fd2addr(fd)); - } +TSAN_INTERCEPTOR(int, accept, int fd, void *addr, unsigned *addrlen) { + SCOPED_TSAN_INTERCEPTOR(accept, fd, addr, addrlen); + int fd2 = REAL(accept)(fd, addr, addrlen); + if (fd >= 0 && fd2 >= 0) + FdSocketAccept(thr, pc, fd, fd2); + return fd2; +} + +TSAN_INTERCEPTOR(int, accept4, int fd, void *addr, unsigned *addrlen, int f) { + SCOPED_TSAN_INTERCEPTOR(accept4, fd, addr, addrlen, f); + int fd2 = REAL(accept4)(fd, addr, addrlen, f); + if (fd >= 0 && fd2 >= 0) + FdSocketAccept(thr, pc, fd, fd2); + return fd2; +} + +TSAN_INTERCEPTOR(int, epoll_create, int size) { + SCOPED_TSAN_INTERCEPTOR(epoll_create, size); + int fd = REAL(epoll_create)(size); + if (fd >= 0) + FdPollCreate(thr, pc, fd); + return fd; +} + +TSAN_INTERCEPTOR(int, epoll_create1, int flags) { + SCOPED_TSAN_INTERCEPTOR(epoll_create1, flags); + int fd = REAL(epoll_create1)(flags); + if (fd >= 0) + FdPollCreate(thr, pc, fd); + return fd; +} + +TSAN_INTERCEPTOR(int, close, int fd) { + SCOPED_TSAN_INTERCEPTOR(close, fd); + if (fd >= 0) + FdClose(thr, pc, fd); + return REAL(close)(fd); +} + +TSAN_INTERCEPTOR(int, __close, int fd) { + SCOPED_TSAN_INTERCEPTOR(__close, fd); + if (fd >= 0) + FdClose(thr, pc, fd); + return REAL(__close)(fd); +} + +TSAN_INTERCEPTOR(int, pipe, int *pipefd) { + SCOPED_TSAN_INTERCEPTOR(pipe, pipefd); + int res = REAL(pipe)(pipefd); + if (res == 0 && pipefd[0] >= 0 && pipefd[1] >= 0) + FdPipeCreate(thr, pc, pipefd[0], pipefd[1]); + return res; +} + +TSAN_INTERCEPTOR(int, pipe2, int *pipefd, int flags) { + SCOPED_TSAN_INTERCEPTOR(pipe2, pipefd, flags); + int res = REAL(pipe2)(pipefd, flags); + if (res == 0 && pipefd[0] >= 0 && pipefd[1] >= 0) + FdPipeCreate(thr, pc, pipefd[0], pipefd[1]); return res; } TSAN_INTERCEPTOR(long_t, readv, int fd, void *vec, int cnt) { SCOPED_TSAN_INTERCEPTOR(readv, fd, vec, cnt); int res = REAL(readv)(fd, vec, cnt); - if (res >= 0) { - Acquire(cur_thread(), pc, fd2addr(fd)); + if (res >= 0 && fd >= 0) { + FdAcquire(thr, pc, fd); } return res; } @@ -1137,57 +1253,40 @@ TSAN_INTERCEPTOR(long_t, readv, int fd, void *vec, int cnt) { TSAN_INTERCEPTOR(long_t, preadv64, int fd, void *vec, int cnt, u64 off) { SCOPED_TSAN_INTERCEPTOR(preadv64, fd, vec, cnt, off); int res = REAL(preadv64)(fd, vec, cnt, off); - if (res >= 0) { - Acquire(cur_thread(), pc, fd2addr(fd)); + if (res >= 0 && fd >= 0) { + FdAcquire(thr, pc, fd); } return res; } -TSAN_INTERCEPTOR(long_t, write, int fd, void *buf, long_t sz) { - SCOPED_TSAN_INTERCEPTOR(write, fd, buf, sz); - Release(cur_thread(), pc, fd2addr(fd)); - int res = REAL(write)(fd, buf, sz); - return res; -} - -TSAN_INTERCEPTOR(long_t, pwrite, int fd, void *buf, long_t sz, unsigned off) { - SCOPED_TSAN_INTERCEPTOR(pwrite, fd, buf, sz, off); - Release(cur_thread(), pc, fd2addr(fd)); - int res = REAL(pwrite)(fd, buf, sz, off); - return res; -} - -TSAN_INTERCEPTOR(long_t, pwrite64, int fd, void *buf, long_t sz, unsigned off) { - SCOPED_TSAN_INTERCEPTOR(pwrite64, fd, buf, sz, off); - Release(cur_thread(), pc, fd2addr(fd)); - int res = REAL(pwrite64)(fd, buf, sz, off); - return res; -} - TSAN_INTERCEPTOR(long_t, writev, int fd, void *vec, int cnt) { SCOPED_TSAN_INTERCEPTOR(writev, fd, vec, cnt); - Release(cur_thread(), pc, fd2addr(fd)); + if (fd >= 0) + FdRelease(thr, pc, fd); int res = REAL(writev)(fd, vec, cnt); return res; } TSAN_INTERCEPTOR(long_t, pwritev64, int fd, void *vec, int cnt, u64 off) { SCOPED_TSAN_INTERCEPTOR(pwritev64, fd, vec, cnt, off); - Release(cur_thread(), pc, fd2addr(fd)); + if (fd >= 0) + FdRelease(thr, pc, fd); int res = REAL(pwritev64)(fd, vec, cnt, off); return res; } TSAN_INTERCEPTOR(long_t, send, int fd, void *buf, long_t len, int flags) { SCOPED_TSAN_INTERCEPTOR(send, fd, buf, len, flags); - Release(cur_thread(), pc, fd2addr(fd)); + if (fd >= 0) + FdRelease(thr, pc, fd); int res = REAL(send)(fd, buf, len, flags); return res; } TSAN_INTERCEPTOR(long_t, sendmsg, int fd, void *msg, int flags) { SCOPED_TSAN_INTERCEPTOR(sendmsg, fd, msg, flags); - Release(cur_thread(), pc, fd2addr(fd)); + if (fd >= 0) + FdRelease(thr, pc, fd); int res = REAL(sendmsg)(fd, msg, flags); return res; } @@ -1195,8 +1294,8 @@ TSAN_INTERCEPTOR(long_t, sendmsg, int fd, void *msg, int flags) { TSAN_INTERCEPTOR(long_t, recv, int fd, void *buf, long_t len, int flags) { SCOPED_TSAN_INTERCEPTOR(recv, fd, buf, len, flags); int res = REAL(recv)(fd, buf, len, flags); - if (res >= 0) { - Acquire(cur_thread(), pc, fd2addr(fd)); + if (res >= 0 && fd >= 0) { + FdAcquire(thr, pc, fd); } return res; } @@ -1204,15 +1303,15 @@ TSAN_INTERCEPTOR(long_t, recv, int fd, void *buf, long_t len, int flags) { TSAN_INTERCEPTOR(long_t, recvmsg, int fd, void *msg, int flags) { SCOPED_TSAN_INTERCEPTOR(recvmsg, fd, msg, flags); int res = REAL(recvmsg)(fd, msg, flags); - if (res >= 0) { - Acquire(cur_thread(), pc, fd2addr(fd)); + if (res >= 0 && fd >= 0) { + FdAcquire(thr, pc, fd); } return res; } TSAN_INTERCEPTOR(int, unlink, char *path) { SCOPED_TSAN_INTERCEPTOR(unlink, path); - Release(cur_thread(), pc, file2addr(path)); + Release(thr, pc, File2addr(path)); int res = REAL(unlink)(path); return res; } @@ -1220,19 +1319,57 @@ TSAN_INTERCEPTOR(int, unlink, char *path) { TSAN_INTERCEPTOR(void*, fopen, char *path, char *mode) { SCOPED_TSAN_INTERCEPTOR(fopen, path, mode); void *res = REAL(fopen)(path, mode); - Acquire(cur_thread(), pc, file2addr(path)); + Acquire(thr, pc, File2addr(path)); + if (res) { + int fd = fileno_unlocked(res); + if (fd >= 0) + FdFileCreate(thr, pc, fd); + } return res; } +TSAN_INTERCEPTOR(void*, freopen, char *path, char *mode, void *stream) { + SCOPED_TSAN_INTERCEPTOR(freopen, path, mode, stream); + if (stream) { + int fd = fileno_unlocked(stream); + if (fd >= 0) + FdClose(thr, pc, fd); + } + void *res = REAL(freopen)(path, mode, stream); + Acquire(thr, pc, File2addr(path)); + if (res) { + int fd = fileno_unlocked(res); + if (fd >= 0) + FdFileCreate(thr, pc, fd); + } + return res; +} + +TSAN_INTERCEPTOR(int, fclose, void *stream) { + { + SCOPED_TSAN_INTERCEPTOR(fclose, stream); + if (stream) { + int fd = fileno_unlocked(stream); + if (fd >= 0) + FdClose(thr, pc, fd); + } + } + return REAL(fclose)(stream); +} + TSAN_INTERCEPTOR(uptr, fread, void *ptr, uptr size, uptr nmemb, void *f) { - SCOPED_TSAN_INTERCEPTOR(fread, ptr, size, nmemb, f); - MemoryAccessRange(thr, pc, (uptr)ptr, size * nmemb, true); + { + SCOPED_TSAN_INTERCEPTOR(fread, ptr, size, nmemb, f); + MemoryAccessRange(thr, pc, (uptr)ptr, size * nmemb, true); + } return REAL(fread)(ptr, size, nmemb, f); } TSAN_INTERCEPTOR(uptr, fwrite, const void *p, uptr size, uptr nmemb, void *f) { - SCOPED_TSAN_INTERCEPTOR(fwrite, p, size, nmemb, f); - MemoryAccessRange(thr, pc, (uptr)p, size * nmemb, false); + { + SCOPED_TSAN_INTERCEPTOR(fwrite, p, size, nmemb, f); + MemoryAccessRange(thr, pc, (uptr)p, size * nmemb, false); + } return REAL(fwrite)(p, size, nmemb, f); } @@ -1244,7 +1381,7 @@ TSAN_INTERCEPTOR(int, puts, const char *s) { TSAN_INTERCEPTOR(int, rmdir, char *path) { SCOPED_TSAN_INTERCEPTOR(rmdir, path); - Release(cur_thread(), pc, dir2addr(path)); + Release(thr, pc, Dir2addr(path)); int res = REAL(rmdir)(path); return res; } @@ -1252,28 +1389,37 @@ TSAN_INTERCEPTOR(int, rmdir, char *path) { TSAN_INTERCEPTOR(void*, opendir, char *path) { SCOPED_TSAN_INTERCEPTOR(opendir, path); void *res = REAL(opendir)(path); - Acquire(cur_thread(), pc, dir2addr(path)); + if (res != 0) + Acquire(thr, pc, Dir2addr(path)); return res; } TSAN_INTERCEPTOR(int, epoll_ctl, int epfd, int op, int fd, void *ev) { SCOPED_TSAN_INTERCEPTOR(epoll_ctl, epfd, op, fd, ev); - if (op == EPOLL_CTL_ADD) { - Release(cur_thread(), pc, epollfd2addr(epfd)); + if (op == EPOLL_CTL_ADD && epfd >= 0) { + FdRelease(thr, pc, epfd); } int res = REAL(epoll_ctl)(epfd, op, fd, ev); + if (fd >= 0) + FdAccess(thr, pc, fd); return res; } TSAN_INTERCEPTOR(int, epoll_wait, int epfd, void *ev, int cnt, int timeout) { SCOPED_TSAN_INTERCEPTOR(epoll_wait, epfd, ev, cnt, timeout); - int res = REAL(epoll_wait)(epfd, ev, cnt, timeout); - if (res > 0) { - Acquire(cur_thread(), pc, epollfd2addr(epfd)); + int res = BLOCK_REAL(epoll_wait)(epfd, ev, cnt, timeout); + if (res > 0 && epfd >= 0) { + FdAcquire(thr, pc, epfd); } return res; } +TSAN_INTERCEPTOR(int, poll, void *fds, long_t nfds, int timeout) { + SCOPED_TSAN_INTERCEPTOR(poll, fds, nfds, timeout); + int res = BLOCK_REAL(poll)(fds, nfds, timeout); + return res; +} + static void ALWAYS_INLINE rtl_generic_sighandler(bool sigact, int sig, my_siginfo_t *info, void *ctx) { ThreadState *thr = cur_thread(); @@ -1281,7 +1427,12 @@ static void ALWAYS_INLINE rtl_generic_sighandler(bool sigact, int sig, // Don't mess with synchronous signals. if (sig == SIGSEGV || sig == SIGBUS || sig == SIGILL || sig == SIGABRT || sig == SIGFPE || sig == SIGPIPE || - (sctx && sig == sctx->int_signal_send)) { + // If we are sending signal to ourselves, we must process it now. + (sctx && sig == sctx->int_signal_send) || + // If we are in blocking function, we can safely process it now + // (but check if we are in a recursive interceptor, + // i.e. pthread_join()->munmap()). + (sctx && sctx->in_blocking_func == 1 && thr->in_rtl == 1)) { CHECK(thr->in_rtl == 0 || thr->in_rtl == 1); int in_rtl = thr->in_rtl; thr->in_rtl = 0; @@ -1340,11 +1491,11 @@ TSAN_INTERCEPTOR(int, sigaction, int sig, sigaction_t *act, sigaction_t *old) { } TSAN_INTERCEPTOR(sighandler_t, signal, int sig, sighandler_t h) { - sigaction_t act = {}; + sigaction_t act; act.sa_handler = h; REAL(memset)(&act.sa_mask, -1, sizeof(act.sa_mask)); act.sa_flags = 0; - sigaction_t old = {}; + sigaction_t old; int res = sigaction(sig, &act, &old); if (res) return SIG_ERR; @@ -1395,11 +1546,89 @@ TSAN_INTERCEPTOR(int, pthread_kill, void *tid, int sig) { return res; } -static void process_pending_signals(ThreadState *thr) { +TSAN_INTERCEPTOR(int, gettimeofday, void *tv, void *tz) { + SCOPED_TSAN_INTERCEPTOR(gettimeofday, tv, tz); + // It's intercepted merely to process pending signals. + return REAL(gettimeofday)(tv, tz); +} + +// Linux kernel has a bug that leads to kernel deadlock if a process +// maps TBs of memory and then calls mlock(). +static void MlockIsUnsupported() { + static atomic_uint8_t printed; + if (atomic_exchange(&printed, 1, memory_order_relaxed)) + return; + Printf("INFO: ThreadSanitizer ignores mlock/mlockall/munlock/munlockall\n"); +} + +TSAN_INTERCEPTOR(int, mlock, const void *addr, uptr len) { + MlockIsUnsupported(); + return 0; +} + +TSAN_INTERCEPTOR(int, munlock, const void *addr, uptr len) { + MlockIsUnsupported(); + return 0; +} + +TSAN_INTERCEPTOR(int, mlockall, int flags) { + MlockIsUnsupported(); + return 0; +} + +TSAN_INTERCEPTOR(int, munlockall, void) { + MlockIsUnsupported(); + return 0; +} + +TSAN_INTERCEPTOR(int, fork, int fake) { + SCOPED_TSAN_INTERCEPTOR(fork, fake); + // It's intercepted merely to process pending signals. + int pid = REAL(fork)(fake); + if (pid == 0) { + // child + FdOnFork(thr, pc); + } else if (pid > 0) { + // parent + } + return pid; +} + +struct TsanInterceptorContext { + ThreadState *thr; + const uptr caller_pc; + const uptr pc; +}; + +#define COMMON_INTERCEPTOR_WRITE_RANGE(ctx, ptr, size) \ + MemoryAccessRange(((TsanInterceptorContext*)ctx)->thr, \ + ((TsanInterceptorContext*)ctx)->pc, \ + (uptr)ptr, size, true) +#define COMMON_INTERCEPTOR_READ_RANGE(ctx, ptr, size) \ + MemoryAccessRange(((TsanInterceptorContext*)ctx)->thr, \ + ((TsanInterceptorContext*)ctx)->pc, \ + (uptr)ptr, size, false) +#define COMMON_INTERCEPTOR_ENTER(ctx, func, ...) \ + SCOPED_TSAN_INTERCEPTOR(func, __VA_ARGS__) \ + TsanInterceptorContext _ctx = {thr, caller_pc, pc}; \ + ctx = (void*)&_ctx; \ + (void)ctx; +#define COMMON_INTERCEPTOR_FD_ACQUIRE(ctx, fd) \ + FdAcquire(((TsanInterceptorContext*)ctx)->thr, pc, fd) +#define COMMON_INTERCEPTOR_FD_RELEASE(ctx, fd) \ + FdRelease(((TsanInterceptorContext*)ctx)->thr, pc, fd) +#define COMMON_INTERCEPTOR_SET_THREAD_NAME(ctx, name) \ + ThreadSetName(((TsanInterceptorContext*)ctx)->thr, name) +#include "sanitizer_common/sanitizer_common_interceptors.inc" + +namespace __tsan { + +void ProcessPendingSignals(ThreadState *thr) { CHECK_EQ(thr->in_rtl, 0); SignalContext *sctx = SigCtx(thr); if (sctx == 0 || sctx->pending_signal_count == 0 || thr->in_signal_handler) return; + Context *ctx = CTX(); thr->in_signal_handler = true; sctx->pending_signal_count = 0; // These are too big for stack. @@ -1419,16 +1648,19 @@ static void process_pending_signals(ThreadState *thr) { sigactions[sig].sa_sigaction(sig, &signal->siginfo, &signal->ctx); else sigactions[sig].sa_handler(sig); - if (errno != 0) { + if (flags()->report_bugs && errno != 0) { ScopedInRtl in_rtl; - StackTrace stack; + __tsan::StackTrace stack; uptr pc = signal->sigaction ? (uptr)sigactions[sig].sa_sigaction : (uptr)sigactions[sig].sa_handler; stack.Init(&pc, 1); + Lock l(&ctx->thread_mtx); ScopedReport rep(ReportTypeErrnoInSignal); - rep.AddStack(&stack); - OutputReport(rep, rep.GetReport()->stacks[0]); + if (!IsFiredSuppression(ctx, rep, stack)) { + rep.AddStack(&stack); + OutputReport(ctx, rep, rep.GetReport()->stacks[0]); + } } errno = saved_errno; } @@ -1439,7 +1671,10 @@ static void process_pending_signals(ThreadState *thr) { thr->in_signal_handler = false; } -namespace __tsan { +static void unreachable() { + Printf("FATAL: ThreadSanitizer: unreachable called\n"); + Die(); +} void InitializeInterceptors() { CHECK_GT(cur_thread()->in_rtl, 0); @@ -1449,10 +1684,13 @@ void InitializeInterceptors() { REAL(memcpy) = internal_memcpy; REAL(memcmp) = internal_memcmp; + SANITIZER_COMMON_INTERCEPTORS_INIT; + TSAN_INTERCEPT(longjmp); TSAN_INTERCEPT(siglongjmp); TSAN_INTERCEPT(malloc); + TSAN_INTERCEPT(__libc_memalign); TSAN_INTERCEPT(calloc); TSAN_INTERCEPT(realloc); TSAN_INTERCEPT(free); @@ -1465,15 +1703,6 @@ void InitializeInterceptors() { TSAN_INTERCEPT(pvalloc); TSAN_INTERCEPT(posix_memalign); - TSAN_INTERCEPT(_Znwm); - TSAN_INTERCEPT(_ZnwmRKSt9nothrow_t); - TSAN_INTERCEPT(_Znam); - TSAN_INTERCEPT(_ZnamRKSt9nothrow_t); - TSAN_INTERCEPT(_ZdlPv); - TSAN_INTERCEPT(_ZdlPvRKSt9nothrow_t); - TSAN_INTERCEPT(_ZdaPv); - TSAN_INTERCEPT(_ZdaPvRKSt9nothrow_t); - TSAN_INTERCEPT(strlen); TSAN_INTERCEPT(memset); TSAN_INTERCEPT(memcpy); @@ -1490,9 +1719,6 @@ void InitializeInterceptors() { TSAN_INTERCEPT(strncpy); TSAN_INTERCEPT(strstr); - TSAN_INTERCEPT(__cxa_guard_acquire); - TSAN_INTERCEPT(__cxa_guard_release); - TSAN_INTERCEPT(pthread_create); TSAN_INTERCEPT(pthread_join); TSAN_INTERCEPT(pthread_detach); @@ -1520,7 +1746,7 @@ void InitializeInterceptors() { TSAN_INTERCEPT(pthread_rwlock_timedwrlock); TSAN_INTERCEPT(pthread_rwlock_unlock); - TSAN_INTERCEPT(pthread_cond_init); + // TSAN_INTERCEPT(pthread_cond_init); TSAN_INTERCEPT(pthread_cond_destroy); TSAN_INTERCEPT(pthread_cond_signal); TSAN_INTERCEPT(pthread_cond_broadcast); @@ -1541,14 +1767,30 @@ void InitializeInterceptors() { TSAN_INTERCEPT(sem_post); TSAN_INTERCEPT(sem_getvalue); - TSAN_INTERCEPT(read); - TSAN_INTERCEPT(pread); - TSAN_INTERCEPT(pread64); + TSAN_INTERCEPT(open); + TSAN_INTERCEPT(open64); + TSAN_INTERCEPT(creat); + TSAN_INTERCEPT(creat64); + TSAN_INTERCEPT(dup); + TSAN_INTERCEPT(dup2); + TSAN_INTERCEPT(dup3); + TSAN_INTERCEPT(eventfd); + TSAN_INTERCEPT(signalfd); + TSAN_INTERCEPT(inotify_init); + TSAN_INTERCEPT(inotify_init1); + TSAN_INTERCEPT(socket); + TSAN_INTERCEPT(socketpair); + TSAN_INTERCEPT(connect); + TSAN_INTERCEPT(accept); + TSAN_INTERCEPT(accept4); + TSAN_INTERCEPT(epoll_create); + TSAN_INTERCEPT(epoll_create1); + TSAN_INTERCEPT(close); + TSAN_INTERCEPT(pipe); + TSAN_INTERCEPT(pipe2); + TSAN_INTERCEPT(readv); TSAN_INTERCEPT(preadv64); - TSAN_INTERCEPT(write); - TSAN_INTERCEPT(pwrite); - TSAN_INTERCEPT(pwrite64); TSAN_INTERCEPT(writev); TSAN_INTERCEPT(pwritev64); TSAN_INTERCEPT(send); @@ -1558,6 +1800,8 @@ void InitializeInterceptors() { TSAN_INTERCEPT(unlink); TSAN_INTERCEPT(fopen); + TSAN_INTERCEPT(freopen); + TSAN_INTERCEPT(fclose); TSAN_INTERCEPT(fread); TSAN_INTERCEPT(fwrite); TSAN_INTERCEPT(puts); @@ -1566,25 +1810,42 @@ void InitializeInterceptors() { TSAN_INTERCEPT(epoll_ctl); TSAN_INTERCEPT(epoll_wait); + TSAN_INTERCEPT(poll); TSAN_INTERCEPT(sigaction); TSAN_INTERCEPT(signal); TSAN_INTERCEPT(raise); TSAN_INTERCEPT(kill); TSAN_INTERCEPT(pthread_kill); + TSAN_INTERCEPT(sleep); + TSAN_INTERCEPT(usleep); + TSAN_INTERCEPT(nanosleep); + TSAN_INTERCEPT(gettimeofday); + TSAN_INTERCEPT(mlock); + TSAN_INTERCEPT(munlock); + TSAN_INTERCEPT(mlockall); + TSAN_INTERCEPT(munlockall); + + TSAN_INTERCEPT(fork); + + // Need to setup it, because interceptors check that the function is resolved. + // But atexit is emitted directly into the module, so can't be resolved. + REAL(atexit) = (int(*)(void(*)()))unreachable; atexit_ctx = new(internal_alloc(MBlockAtExit, sizeof(AtExitContext))) AtExitContext(); if (__cxa_atexit(&finalize, 0, 0)) { - TsanPrintf("ThreadSanitizer: failed to setup atexit callback\n"); + Printf("ThreadSanitizer: failed to setup atexit callback\n"); Die(); } if (pthread_key_create(&g_thread_finalize_key, &thread_finalize)) { - TsanPrintf("ThreadSanitizer: failed to create thread key\n"); + Printf("ThreadSanitizer: failed to create thread key\n"); Die(); } + + FdInit(); } void internal_start_thread(void(*func)(void *arg), void *arg) { |