Import compiler-rt trunk r228651.vendor/compiler-rt/compiler-rt-r228651

https://llvm.org/svn/llvm-project/compiler-rt/trunk@228651

22 files changed, 343 insertions, 320 deletions

diff --git a/lib/tsan/rtl/Makefile.old b/lib/tsan/rtl/Makefile.old
index 79c761ce3f4e..150b376343be 100644
--- a/lib/tsan/rtl/Makefile.old
+++ b/lib/tsan/rtl/Makefile.old
@@ -1,4 +1,4 @@
-CXXFLAGS = -std=c++11 -fPIE -g -Wall -Werror -fno-builtin -msse3 -DTSAN_DEBUG=$(DEBUG) -DSANITIZER_DEBUG=$(DEBUG)
+CXXFLAGS = -std=c++11 -fPIE -g -Wall -Werror -fno-builtin -msse3 -DSANITIZER_DEBUG=$(DEBUG)
 CLANG=clang
 ifeq ($(DEBUG), 0)
   CXXFLAGS += -O3
diff --git a/lib/tsan/rtl/tsan_clock.cc b/lib/tsan/rtl/tsan_clock.cc
index f2b39a182b39..59e3de435f1b 100644
--- a/lib/tsan/rtl/tsan_clock.cc
+++ b/lib/tsan/rtl/tsan_clock.cc
@@ -104,8 +104,8 @@ ThreadClock::ThreadClock(unsigned tid, unsigned reused)
 }
 
 void ThreadClock::acquire(ClockCache *c, const SyncClock *src) {
-  DCHECK(nclk_ <= kMaxTid);
-  DCHECK(src->size_ <= kMaxTid);
+  DCHECK_LE(nclk_, kMaxTid);
+  DCHECK_LE(src->size_, kMaxTid);
   CPP_STAT_INC(StatClockAcquire);
 
   // Check if it's empty -> no need to do anything.
@@ -215,8 +215,8 @@ void ThreadClock::release(ClockCache *c, SyncClock *dst) const {
 }
 
 void ThreadClock::ReleaseStore(ClockCache *c, SyncClock *dst) const {
-  DCHECK(nclk_ <= kMaxTid);
-  DCHECK(dst->size_ <= kMaxTid);
+  DCHECK_LE(nclk_, kMaxTid);
+  DCHECK_LE(dst->size_, kMaxTid);
   CPP_STAT_INC(StatClockStore);
 
   // Check if we need to resize dst.
diff --git a/lib/tsan/rtl/tsan_defs.h b/lib/tsan/rtl/tsan_defs.h
index 7ed3796b5012..f19aee999332 100644
--- a/lib/tsan/rtl/tsan_defs.h
+++ b/lib/tsan/rtl/tsan_defs.h
@@ -18,10 +18,6 @@
 #include "sanitizer_common/sanitizer_libc.h"
 #include "tsan_stat.h"
 
-#ifndef TSAN_DEBUG
-#define TSAN_DEBUG 0
-#endif  // TSAN_DEBUG
-
 namespace __tsan {
 
 #ifdef SANITIZER_GO
@@ -44,18 +40,8 @@ const int kClkBits = 42;
 const unsigned kMaxTidReuse = (1 << (64 - kClkBits)) - 1;
 const uptr kShadowStackSize = 64 * 1024;
 
-#ifdef TSAN_SHADOW_COUNT
-# if TSAN_SHADOW_COUNT == 2 \
-  || TSAN_SHADOW_COUNT == 4 || TSAN_SHADOW_COUNT == 8
-const uptr kShadowCnt = TSAN_SHADOW_COUNT;
-# else
-#   error "TSAN_SHADOW_COUNT must be one of 2,4,8"
-# endif
-#else
 // Count of shadow values in a shadow cell.
-#define TSAN_SHADOW_COUNT 4
 const uptr kShadowCnt = 4;
-#endif
 
 // That many user bytes are mapped onto a single shadow cell.
 const uptr kShadowCell = 8;
@@ -88,7 +74,7 @@ const bool kCollectStats = false;
 // The following "build consistency" machinery ensures that all source files
 // are built in the same configuration. Inconsistent builds lead to
 // hard to debug crashes.
-#if TSAN_DEBUG
+#if SANITIZER_DEBUG
 void build_consistency_debug();
 #else
 void build_consistency_release();
@@ -100,18 +86,8 @@ void build_consistency_stats();
 void build_consistency_nostats();
 #endif
 
-#if TSAN_SHADOW_COUNT == 1
-void build_consistency_shadow1();
-#elif TSAN_SHADOW_COUNT == 2
-void build_consistency_shadow2();
-#elif TSAN_SHADOW_COUNT == 4
-void build_consistency_shadow4();
-#else
-void build_consistency_shadow8();
-#endif
-
 static inline void USED build_consistency() {
-#if TSAN_DEBUG
+#if SANITIZER_DEBUG
   build_consistency_debug();
 #else
   build_consistency_release();
@@ -121,15 +97,6 @@ static inline void USED build_consistency() {
 #else
   build_consistency_nostats();
 #endif
-#if TSAN_SHADOW_COUNT == 1
-  build_consistency_shadow1();
-#elif TSAN_SHADOW_COUNT == 2
-  build_consistency_shadow2();
-#elif TSAN_SHADOW_COUNT == 4
-  build_consistency_shadow4();
-#else
-  build_consistency_shadow8();
-#endif
 }
 
 template<typename T>
diff --git a/lib/tsan/rtl/tsan_flags.cc b/lib/tsan/rtl/tsan_flags.cc
index 5dc331f59469..fed3de8db2ed 100644
--- a/lib/tsan/rtl/tsan_flags.cc
+++ b/lib/tsan/rtl/tsan_flags.cc
@@ -12,6 +12,7 @@
 //===----------------------------------------------------------------------===//
 
 #include "sanitizer_common/sanitizer_flags.h"
+#include "sanitizer_common/sanitizer_flag_parser.h"
 #include "sanitizer_common/sanitizer_libc.h"
 #include "tsan_flags.h"
 #include "tsan_rtl.h"
@@ -33,80 +34,44 @@ const char *WEAK __tsan_default_options() {
 }
 #endif
 
-static void ParseFlags(Flags *f, const char *env) {
-  ParseFlag(env, &f->enable_annotations, "enable_annotations", "");
-  ParseFlag(env, &f->suppress_equal_stacks, "suppress_equal_stacks", "");
-  ParseFlag(env, &f->suppress_equal_addresses, "suppress_equal_addresses", "");
-  ParseFlag(env, &f->report_bugs, "report_bugs", "");
-  ParseFlag(env, &f->report_thread_leaks, "report_thread_leaks", "");
-  ParseFlag(env, &f->report_destroy_locked, "report_destroy_locked", "");
-  ParseFlag(env, &f->report_mutex_bugs, "report_mutex_bugs", "");
-  ParseFlag(env, &f->report_signal_unsafe, "report_signal_unsafe", "");
-  ParseFlag(env, &f->report_atomic_races, "report_atomic_races", "");
-  ParseFlag(env, &f->force_seq_cst_atomics, "force_seq_cst_atomics", "");
-  ParseFlag(env, &f->print_benign, "print_benign", "");
-  ParseFlag(env, &f->exitcode, "exitcode", "");
-  ParseFlag(env, &f->halt_on_error, "halt_on_error", "");
-  ParseFlag(env, &f->atexit_sleep_ms, "atexit_sleep_ms", "");
-  ParseFlag(env, &f->profile_memory, "profile_memory", "");
-  ParseFlag(env, &f->flush_memory_ms, "flush_memory_ms", "");
-  ParseFlag(env, &f->flush_symbolizer_ms, "flush_symbolizer_ms", "");
-  ParseFlag(env, &f->memory_limit_mb, "memory_limit_mb", "");
-  ParseFlag(env, &f->stop_on_start, "stop_on_start", "");
-  ParseFlag(env, &f->running_on_valgrind, "running_on_valgrind", "");
-  ParseFlag(env, &f->history_size, "history_size", "");
-  ParseFlag(env, &f->io_sync, "io_sync", "");
-  ParseFlag(env, &f->die_after_fork, "die_after_fork", "");
-
+void Flags::SetDefaults() {
+#define TSAN_FLAG(Type, Name, DefaultValue, Description) Name = DefaultValue;
+#include "tsan_flags.inc"
+#undef TSAN_FLAG
   // DDFlags
-  ParseFlag(env, &f->second_deadlock_stack, "second_deadlock_stack", "");
+  second_deadlock_stack = false;
 }
 
-void InitializeFlags(Flags *f, const char *env) {
-  internal_memset(f, 0, sizeof(*f));
-
-  // Default values.
-  f->enable_annotations = true;
-  f->suppress_equal_stacks = true;
-  f->suppress_equal_addresses = true;
-  f->report_bugs = true;
-  f->report_thread_leaks = true;
-  f->report_destroy_locked = true;
-  f->report_mutex_bugs = true;
-  f->report_signal_unsafe = true;
-  f->report_atomic_races = true;
-  f->force_seq_cst_atomics = false;
-  f->print_benign = false;
-  f->exitcode = 66;
-  f->halt_on_error = false;
-  f->atexit_sleep_ms = 1000;
-  f->profile_memory = "";
-  f->flush_memory_ms = 0;
-  f->flush_symbolizer_ms = 5000;
-  f->memory_limit_mb = 0;
-  f->stop_on_start = false;
-  f->running_on_valgrind = false;
-  f->history_size = kGoMode ? 1 : 2;  // There are a lot of goroutines in Go.
-  f->io_sync = 1;
-  f->die_after_fork = true;
-
-  // DDFlags
-  f->second_deadlock_stack = false;
+void RegisterTsanFlags(FlagParser *parser, Flags *f) {
+#define TSAN_FLAG(Type, Name, DefaultValue, Description) \
+  RegisterFlag(parser, #Name, Description, &f->Name);
+#include "tsan_flags.inc"
+#undef TSAN_FLAG
+}
 
-  CommonFlags *cf = common_flags();
-  SetCommonFlagsDefaults(cf);
-  // Override some common flags defaults.
-  cf->allow_addr2line = true;
-  cf->detect_deadlocks = true;
-  cf->print_suppressions = false;
-  cf->stack_trace_format = "    #%n %f %S %M";
+void InitializeFlags(Flags *f, const char *env) {
+  FlagParser parser;
+  RegisterTsanFlags(&parser, f);
+  RegisterCommonFlags(&parser);
+
+  f->SetDefaults();
+
+  SetCommonFlagsDefaults();
+  {
+    // Override some common flags defaults.
+    CommonFlags cf;
+    cf.CopyFrom(*common_flags());
+    cf.allow_addr2line = true;
+    cf.detect_deadlocks = true;
+    cf.print_suppressions = false;
+    cf.stack_trace_format = "    #%n %f %S %M";
+    OverrideCommonFlags(cf);
+  }
 
   // Let a frontend override.
-  ParseFlags(f, __tsan_default_options());
-  ParseCommonFlagsFromString(cf, __tsan_default_options());
+  parser.ParseString(__tsan_default_options());
   // Override from command line.
-  ParseFlags(f, env);
-  ParseCommonFlagsFromString(cf, env);
+  parser.ParseString(env);
 
   // Sanity check.
   if (!f->report_bugs) {
@@ -115,7 +80,11 @@ void InitializeFlags(Flags *f, const char *env) {
     f->report_signal_unsafe = false;
   }
 
-  if (cf->help) PrintFlagDescriptions();
+  SetVerbosity(common_flags()->verbosity);
+
+  if (Verbosity()) ReportUnrecognizedFlags();
+
+  if (common_flags()->help) parser.PrintFlagDescriptions();
 
   if (f->history_size < 0 || f->history_size > 7) {
     Printf("ThreadSanitizer: incorrect value for history_size"
diff --git a/lib/tsan/rtl/tsan_flags.h b/lib/tsan/rtl/tsan_flags.h
index 621ca139236f..e2f6b3c9f021 100644
--- a/lib/tsan/rtl/tsan_flags.h
+++ b/lib/tsan/rtl/tsan_flags.h
@@ -20,65 +20,12 @@
 namespace __tsan {
 
 struct Flags : DDFlags {
-  // Enable dynamic annotations, otherwise they are no-ops.
-  bool enable_annotations;
-  // Suppress a race report if we've already output another race report
-  // with the same stack.
-  bool suppress_equal_stacks;
-  // Suppress a race report if we've already output another race report
-  // on the same address.
-  bool suppress_equal_addresses;
-  // Turns off bug reporting entirely (useful for benchmarking).
-  bool report_bugs;
-  // Report thread leaks at exit?
-  bool report_thread_leaks;
-  // Report destruction of a locked mutex?
-  bool report_destroy_locked;
-  // Report incorrect usages of mutexes and mutex annotations?
-  bool report_mutex_bugs;
-  // Report violations of async signal-safety
-  // (e.g. malloc() call from a signal handler).
-  bool report_signal_unsafe;
-  // Report races between atomic and plain memory accesses.
-  bool report_atomic_races;
-  // If set, all atomics are effectively sequentially consistent (seq_cst),
-  // regardless of what user actually specified.
-  bool force_seq_cst_atomics;
-  // Print matched "benign" races at exit.
-  bool print_benign;
-  // Override exit status if something was reported.
-  int exitcode;
-  // Exit after first reported error.
-  bool halt_on_error;
-  // Sleep in main thread before exiting for that many ms
-  // (useful to catch "at exit" races).
-  int atexit_sleep_ms;
-  // If set, periodically write memory profile to that file.
-  const char *profile_memory;
-  // Flush shadow memory every X ms.
-  int flush_memory_ms;
-  // Flush symbolizer caches every X ms.
-  int flush_symbolizer_ms;
-  // Resident memory limit in MB to aim at.
-  // If the process consumes more memory, then TSan will flush shadow memory.
-  int memory_limit_mb;
-  // Stops on start until __tsan_resume() is called (for debugging).
-  bool stop_on_start;
-  // Controls whether RunningOnValgrind() returns true or false.
-  bool running_on_valgrind;
-  // Per-thread history size, controls how many previous memory accesses
-  // are remembered per thread.  Possible values are [0..7].
-  // history_size=0 amounts to 32K memory accesses.  Each next value doubles
-  // the amount of memory accesses, up to history_size=7 that amounts to
-  // 4M memory accesses.  The default value is 2 (128K memory accesses).
-  int history_size;
-  // Controls level of synchronization implied by IO operations.
-  // 0 - no synchronization
-  // 1 - reasonable level of synchronization (write->read)
-  // 2 - global synchronization of all IO operations
-  int io_sync;
-  // Die after multi-threaded fork if the child creates new threads.
-  bool die_after_fork;
+#define TSAN_FLAG(Type, Name, DefaultValue, Description) Type Name;
+#include "tsan_flags.inc"
+#undef TSAN_FLAG
+
+  void SetDefaults();
+  void ParseFromString(const char *str);
 };
 
 Flags *flags();
diff --git a/lib/tsan/rtl/tsan_flags.inc b/lib/tsan/rtl/tsan_flags.inc
new file mode 100644
index 000000000000..2925f38482a2
--- /dev/null
+++ b/lib/tsan/rtl/tsan_flags.inc
@@ -0,0 +1,78 @@
+//===-- tsan_flags.inc ------------------------------------------*- C++ -*-===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+//
+// TSan runtime flags.
+//
+//===----------------------------------------------------------------------===//
+#ifndef TSAN_FLAG
+# error "Define TSAN_FLAG prior to including this file!"
+#endif
+
+// TSAN_FLAG(Type, Name, DefaultValue, Description)
+// See COMMON_FLAG in sanitizer_flags.inc for more details.
+
+TSAN_FLAG(bool, enable_annotations, true,
+          "Enable dynamic annotations, otherwise they are no-ops.")
+// Suppress a race report if we've already output another race report
+// with the same stack.
+TSAN_FLAG(bool, suppress_equal_stacks, true,
+          "Suppress a race report if we've already output another race report "
+          "with the same stack.")
+TSAN_FLAG(bool, suppress_equal_addresses, true,
+          "Suppress a race report if we've already output another race report "
+          "on the same address.")
+
+TSAN_FLAG(bool, report_bugs, true,
+          "Turns off bug reporting entirely (useful for benchmarking).")
+TSAN_FLAG(bool, report_thread_leaks, true, "Report thread leaks at exit?")
+TSAN_FLAG(bool, report_destroy_locked, true,
+          "Report destruction of a locked mutex?")
+TSAN_FLAG(bool, report_mutex_bugs, true,
+          "Report incorrect usages of mutexes and mutex annotations?")
+TSAN_FLAG(bool, report_signal_unsafe, true,
+          "Report violations of async signal-safety "
+          "(e.g. malloc() call from a signal handler).")
+TSAN_FLAG(bool, report_atomic_races, true,
+          "Report races between atomic and plain memory accesses.")
+TSAN_FLAG(
+    bool, force_seq_cst_atomics, false,
+    "If set, all atomics are effectively sequentially consistent (seq_cst), "
+    "regardless of what user actually specified.")
+TSAN_FLAG(bool, print_benign, false, "Print matched \"benign\" races at exit.")
+TSAN_FLAG(int, exitcode, 66, "Override exit status if something was reported.")
+TSAN_FLAG(bool, halt_on_error, false, "Exit after first reported error.")
+TSAN_FLAG(int, atexit_sleep_ms, 1000,
+          "Sleep in main thread before exiting for that many ms "
+          "(useful to catch \"at exit\" races).")
+TSAN_FLAG(const char *, profile_memory, "",
+          "If set, periodically write memory profile to that file.")
+TSAN_FLAG(int, flush_memory_ms, 0, "Flush shadow memory every X ms.")
+TSAN_FLAG(int, flush_symbolizer_ms, 5000, "Flush symbolizer caches every X ms.")
+TSAN_FLAG(
+    int, memory_limit_mb, 0,
+    "Resident memory limit in MB to aim at."
+    "If the process consumes more memory, then TSan will flush shadow memory.")
+TSAN_FLAG(bool, stop_on_start, false,
+          "Stops on start until __tsan_resume() is called (for debugging).")
+TSAN_FLAG(bool, running_on_valgrind, false,
+          "Controls whether RunningOnValgrind() returns true or false.")
+TSAN_FLAG(
+    int, history_size, kGoMode ? 1 : 2, // There are a lot of goroutines in Go.
+    "Per-thread history size, controls how many previous memory accesses "
+    "are remembered per thread.  Possible values are [0..7]. "
+    "history_size=0 amounts to 32K memory accesses.  Each next value doubles "
+    "the amount of memory accesses, up to history_size=7 that amounts to "
+    "4M memory accesses.  The default value is 2 (128K memory accesses).")
+TSAN_FLAG(int, io_sync, 1,
+          "Controls level of synchronization implied by IO operations. "
+          "0 - no synchronization "
+          "1 - reasonable level of synchronization (write->read)"
+          "2 - global synchronization of all IO operations.")
+TSAN_FLAG(bool, die_after_fork, true,
+          "Die after multi-threaded fork if the child creates new threads.")
diff --git a/lib/tsan/rtl/tsan_interceptors.cc b/lib/tsan/rtl/tsan_interceptors.cc
index 5bede0ec7d0c..9a6401167bc1 100644
--- a/lib/tsan/rtl/tsan_interceptors.cc
+++ b/lib/tsan/rtl/tsan_interceptors.cc
@@ -72,6 +72,7 @@ extern "C" void *__libc_malloc(uptr size);
 extern "C" void *__libc_calloc(uptr size, uptr n);
 extern "C" void *__libc_realloc(void *ptr, uptr size);
 extern "C" void __libc_free(void *ptr);
+extern "C" int dirfd(void *dirp);
 #if !SANITIZER_FREEBSD
 extern "C" int mallopt(int param, int value);
 #endif
@@ -101,14 +102,15 @@ typedef long long_t;  // NOLINT
 # define F_TLOCK 2      /* Test and lock a region for exclusive use.  */
 # define F_TEST  3      /* Test a region for other processes locks.  */
 
-typedef void (*sighandler_t)(int sig);
-
 #define errno (*__errno_location())
 
+typedef void (*sighandler_t)(int sig);
+typedef void (*sigactionhandler_t)(int sig, my_siginfo_t *siginfo, void *uctx);
+
 struct sigaction_t {
   union {
     sighandler_t sa_handler;
-    void (*sa_sigaction)(int sig, my_siginfo_t *siginfo, void *uctx);
+    sigactionhandler_t sa_sigaction;
   };
 #if SANITIZER_FREEBSD
   int sa_flags;
@@ -505,14 +507,10 @@ TSAN_INTERCEPTOR(void*, __libc_memalign, uptr align, uptr sz) {
 TSAN_INTERCEPTOR(void*, calloc, uptr size, uptr n) {
   if (cur_thread()->in_symbolizer)
     return __libc_calloc(size, n);
-  if (__sanitizer::CallocShouldReturnNullDueToOverflow(size, n))
-    return AllocatorReturnNull();
   void *p = 0;
   {
     SCOPED_INTERCEPTOR_RAW(calloc, size, n);
-    p = user_alloc(thr, pc, n * size);
-    if (p)
-      internal_memset(p, 0, n * size);
+    p = user_calloc(thr, pc, size, n);
   }
   invoke_malloc_hook(p, n * size);
   return p;
@@ -952,6 +950,8 @@ TSAN_INTERCEPTOR(int, pthread_join, void *th, void **ret) {
   return res;
 }
 
+DEFINE_REAL_PTHREAD_FUNCTIONS
+
 TSAN_INTERCEPTOR(int, pthread_detach, void *th) {
   SCOPED_TSAN_INTERCEPTOR(pthread_detach, th);
   int tid = ThreadTid(thr, pc, (uptr)th);
@@ -1826,12 +1826,11 @@ TSAN_INTERCEPTOR(int, rmdir, char *path) {
   return res;
 }
 
-TSAN_INTERCEPTOR(void*, opendir, char *path) {
-  SCOPED_TSAN_INTERCEPTOR(opendir, path);
-  void *res = REAL(opendir)(path);
-  if (res != 0)
-    Acquire(thr, pc, Dir2addr(path));
-  return res;
+TSAN_INTERCEPTOR(int, closedir, void *dirp) {
+  SCOPED_TSAN_INTERCEPTOR(closedir, dirp);
+  int fd = dirfd(dirp);
+  FdClose(thr, pc, fd);
+  return REAL(closedir)(dirp);
 }
 
 #if !SANITIZER_FREEBSD
@@ -1875,15 +1874,18 @@ static void CallUserSignalHandler(ThreadState *thr, bool sync, bool acquire,
   // Ensure that the handler does not spoil errno.
   const int saved_errno = errno;
   errno = 99;
-  // Need to remember pc before the call, because the handler can reset it.
-  uptr pc = sigact ?
+  // This code races with sigaction. Be careful to not read sa_sigaction twice.
+  // Also need to remember pc for reporting before the call,
+  // because the handler can reset it.
+  volatile uptr pc = sigact ?
      (uptr)sigactions[sig].sa_sigaction :
      (uptr)sigactions[sig].sa_handler;
-  pc += 1;  // return address is expected, OutputReport() will undo this
-  if (sigact)
-    sigactions[sig].sa_sigaction(sig, info, uctx);
-  else
-    sigactions[sig].sa_handler(sig);
+  if (pc != (uptr)SIG_DFL && pc != (uptr)SIG_IGN) {
+    if (sigact)
+      ((sigactionhandler_t)pc)(sig, info, uctx);
+    else
+      ((sighandler_t)pc)(sig);
+  }
   // We do not detect errno spoiling for SIGTERM,
   // because some SIGTERM handlers do spoil errno but reraise SIGTERM,
   // tsan reports false positive in such case.
@@ -1893,7 +1895,9 @@ static void CallUserSignalHandler(ThreadState *thr, bool sync, bool acquire,
   // signal; and it looks too fragile to intercept all ways to reraise a signal.
   if (flags()->report_bugs && !sync && sig != SIGTERM && errno != 99) {
     VarSizeStackTrace stack;
-    ObtainCurrentStack(thr, pc, &stack);
+    // Add 1 to pc because return address is expected,
+    // OutputReport() will undo this.
+    ObtainCurrentStack(thr, pc + 1, &stack);
     ThreadRegistryLock l(ctx->thread_registry);
     ScopedReport rep(ReportTypeErrnoInSignal);
     if (!IsFiredSuppression(ctx, rep, stack)) {
@@ -1919,11 +1923,8 @@ void ProcessPendingSignals(ThreadState *thr) {
     SignalDesc *signal = &sctx->pending_signals[sig];
     if (signal->armed) {
       signal->armed = false;
-      if (sigactions[sig].sa_handler != SIG_DFL
-          && sigactions[sig].sa_handler != SIG_IGN) {
-        CallUserSignalHandler(thr, false, true, signal->sigaction,
-            sig, &signal->siginfo, &signal->ctx);
-      }
+      CallUserSignalHandler(thr, false, true, signal->sigaction, sig,
+          &signal->siginfo, &signal->ctx);
     }
   }
   pthread_sigmask(SIG_SETMASK, &oldset, 0);
@@ -2005,7 +2006,19 @@ TSAN_INTERCEPTOR(int, sigaction, int sig, sigaction_t *act, sigaction_t *old) {
     internal_memcpy(old, &sigactions[sig], sizeof(*old));
   if (act == 0)
     return 0;
-  internal_memcpy(&sigactions[sig], act, sizeof(*act));
+  // Copy act into sigactions[sig].
+  // Can't use struct copy, because compiler can emit call to memcpy.
+  // Can't use internal_memcpy, because it copies byte-by-byte,
+  // and signal handler reads the sa_handler concurrently. It it can read
+  // some bytes from old value and some bytes from new value.
+  // Use volatile to prevent insertion of memcpy.
+  sigactions[sig].sa_handler = *(volatile sighandler_t*)&act->sa_handler;
+  sigactions[sig].sa_flags = *(volatile int*)&act->sa_flags;
+  internal_memcpy(&sigactions[sig].sa_mask, &act->sa_mask,
+      sizeof(sigactions[sig].sa_mask));
+#if !SANITIZER_FREEBSD
+  sigactions[sig].sa_restorer = act->sa_restorer;
+#endif
   sigaction_t newact;
   internal_memcpy(&newact, act, sizeof(newact));
   REAL(sigfillset)(&newact.sa_mask);
@@ -2171,6 +2184,16 @@ static void HandleRecvmsg(ThreadState *thr, uptr pc,
 #undef SANITIZER_INTERCEPT_FGETPWENT
 #undef SANITIZER_INTERCEPT_GETPWNAM_AND_FRIENDS
 #undef SANITIZER_INTERCEPT_GETPWNAM_R_AND_FRIENDS
+// __tls_get_addr can be called with mis-aligned stack due to:
+// https://gcc.gnu.org/bugzilla/show_bug.cgi?id=58066
+// There are two potential issues:
+// 1. Sanitizer code contains a MOVDQA spill (it does not seem to be the case
+// right now). or 2. ProcessPendingSignal calls user handler which contains
+// MOVDQA spill (this happens right now).
+// Since the interceptor only initializes memory for msan, the simplest solution
+// is to disable the interceptor in tsan (other sanitizers do not call
+// signal handlers from COMMON_INTERCEPTOR_ENTER).
+#undef SANITIZER_INTERCEPT_TLS_GET_ADDR
 
 #define COMMON_INTERCEPT_FUNCTION(name) INTERCEPT_FUNCTION(name)
 
@@ -2209,12 +2232,15 @@ static void HandleRecvmsg(ThreadState *thr, uptr pc,
     if (fd >= 0) FdClose(thr, pc, fd);           \
   }
 
-#define COMMON_INTERCEPTOR_LIBRARY_LOADED(filename, res)  \
+#define COMMON_INTERCEPTOR_LIBRARY_LOADED(filename, handle) \
   libignore()->OnLibraryLoaded(filename)
 
 #define COMMON_INTERCEPTOR_LIBRARY_UNLOADED() \
   libignore()->OnLibraryUnloaded()
 
+#define COMMON_INTERCEPTOR_DIR_ACQUIRE(ctx, path) \
+  Acquire(((TsanInterceptorContext *) ctx)->thr, pc, Dir2addr(path))
+
 #define COMMON_INTERCEPTOR_FD_ACQUIRE(ctx, fd) \
   FdAcquire(((TsanInterceptorContext *) ctx)->thr, pc, fd)
 
@@ -2530,7 +2556,7 @@ void InitializeInterceptors() {
   TSAN_INTERCEPT(abort);
   TSAN_INTERCEPT(puts);
   TSAN_INTERCEPT(rmdir);
-  TSAN_INTERCEPT(opendir);
+  TSAN_INTERCEPT(closedir);
 
   TSAN_MAYBE_INTERCEPT_EPOLL_CTL;
   TSAN_MAYBE_INTERCEPT_EPOLL_WAIT;
@@ -2569,19 +2595,4 @@ void InitializeInterceptors() {
   FdInit();
 }
 
-void *internal_start_thread(void(*func)(void *arg), void *arg) {
-  // Start the thread with signals blocked, otherwise it can steal user signals.
-  __sanitizer_sigset_t set, old;
-  internal_sigfillset(&set);
-  internal_sigprocmask(SIG_SETMASK, &set, &old);
-  void *th;
-  REAL(pthread_create)(&th, 0, (void*(*)(void *arg))func, arg);
-  internal_sigprocmask(SIG_SETMASK, &old, 0);
-  return th;
-}
-
-void internal_join_thread(void *th) {
-  REAL(pthread_join)(th, 0);
-}
-
 }  // namespace __tsan
diff --git a/lib/tsan/rtl/tsan_interface.cc b/lib/tsan/rtl/tsan_interface.cc
index 9de3808e79ff..9bc9a696363d 100644
--- a/lib/tsan/rtl/tsan_interface.cc
+++ b/lib/tsan/rtl/tsan_interface.cc
@@ -38,56 +38,79 @@ void __tsan_write16(void *addr) {
   MemoryWrite(cur_thread(), CALLERPC, (uptr)addr + 8, kSizeLog8);
 }
 
-u16 __tsan_unaligned_read2(const uu16 *addr) {
+// __tsan_unaligned_read/write calls are emitted by compiler.
+
+void __tsan_unaligned_read2(const void *addr) {
   UnalignedMemoryAccess(cur_thread(), CALLERPC, (uptr)addr, 2, false, false);
-  return *addr;
 }
 
-u32 __tsan_unaligned_read4(const uu32 *addr) {
+void __tsan_unaligned_read4(const void *addr) {
   UnalignedMemoryAccess(cur_thread(), CALLERPC, (uptr)addr, 4, false, false);
-  return *addr;
 }
 
-u64 __tsan_unaligned_read8(const uu64 *addr) {
+void __tsan_unaligned_read8(const void *addr) {
   UnalignedMemoryAccess(cur_thread(), CALLERPC, (uptr)addr, 8, false, false);
-  return *addr;
 }
 
-void __tsan_unaligned_write2(uu16 *addr, u16 v) {
+void __tsan_unaligned_read16(const void *addr) {
+  UnalignedMemoryAccess(cur_thread(), CALLERPC, (uptr)addr, 16, false, false);
+}
+
+void __tsan_unaligned_write2(void *addr) {
   UnalignedMemoryAccess(cur_thread(), CALLERPC, (uptr)addr, 2, true, false);
-  *addr = v;
 }
 
-void __tsan_unaligned_write4(uu32 *addr, u32 v) {
+void __tsan_unaligned_write4(void *addr) {
   UnalignedMemoryAccess(cur_thread(), CALLERPC, (uptr)addr, 4, true, false);
-  *addr = v;
 }
 
-void __tsan_unaligned_write8(uu64 *addr, u64 v) {
+void __tsan_unaligned_write8(void *addr) {
   UnalignedMemoryAccess(cur_thread(), CALLERPC, (uptr)addr, 8, true, false);
-  *addr = v;
 }
 
+void __tsan_unaligned_write16(void *addr) {
+  UnalignedMemoryAccess(cur_thread(), CALLERPC, (uptr)addr, 16, true, false);
+}
+
+// __sanitizer_unaligned_load/store are for user instrumentation.
+
 extern "C" {
 SANITIZER_INTERFACE_ATTRIBUTE
-uint16_t __sanitizer_unaligned_load16(void *addr)
-    ALIAS("__tsan_unaligned_read2");
+u16 __sanitizer_unaligned_load16(const uu16 *addr) {
+  __tsan_unaligned_read2(addr);
+  return *addr;
+}
+
 SANITIZER_INTERFACE_ATTRIBUTE
-uint32_t __sanitizer_unaligned_load32(void *addr)
-    ALIAS("__tsan_unaligned_read4");
+u32 __sanitizer_unaligned_load32(const uu32 *addr) {
+  __tsan_unaligned_read4(addr);
+  return *addr;
+}
+
 SANITIZER_INTERFACE_ATTRIBUTE
-uint64_t __sanitizer_unaligned_load64(void *addr)
-    ALIAS("__tsan_unaligned_read8");
+u64 __sanitizer_unaligned_load64(const uu64 *addr) {
+  __tsan_unaligned_read8(addr);
+  return *addr;
+}
+
 SANITIZER_INTERFACE_ATTRIBUTE
-void __sanitizer_unaligned_store16(void *addr, uint16_t v)
-    ALIAS("__tsan_unaligned_write2");
+void __sanitizer_unaligned_store16(uu16 *addr, u16 v) {
+  __tsan_unaligned_write2(addr);
+  *addr = v;
+}
+
 SANITIZER_INTERFACE_ATTRIBUTE
-void __sanitizer_unaligned_store32(void *addr, uint32_t v)
-    ALIAS("__tsan_unaligned_write4");
+void __sanitizer_unaligned_store32(uu32 *addr, u32 v) {
+  __tsan_unaligned_write4(addr);
+  *addr = v;
+}
+
 SANITIZER_INTERFACE_ATTRIBUTE
-void __sanitizer_unaligned_store64(void *addr, uint64_t v)
-    ALIAS("__tsan_unaligned_write8");
+void __sanitizer_unaligned_store64(uu64 *addr, u64 v) {
+  __tsan_unaligned_write8(addr);
+  *addr = v;
 }
+}  // extern "C"
 
 void __tsan_acquire(void *addr) {
   Acquire(cur_thread(), CALLERPC, (uptr)addr);
diff --git a/lib/tsan/rtl/tsan_interface.h b/lib/tsan/rtl/tsan_interface.h
index 70450697d480..a05e6f0f6d09 100644
--- a/lib/tsan/rtl/tsan_interface.h
+++ b/lib/tsan/rtl/tsan_interface.h
@@ -41,12 +41,15 @@ SANITIZER_INTERFACE_ATTRIBUTE void __tsan_write4(void *addr);
 SANITIZER_INTERFACE_ATTRIBUTE void __tsan_write8(void *addr);
 SANITIZER_INTERFACE_ATTRIBUTE void __tsan_write16(void *addr);
 
-SANITIZER_INTERFACE_ATTRIBUTE u16 __tsan_unaligned_read2(const uu16 *addr);
-SANITIZER_INTERFACE_ATTRIBUTE u32 __tsan_unaligned_read4(const uu32 *addr);
-SANITIZER_INTERFACE_ATTRIBUTE u64 __tsan_unaligned_read8(const uu64 *addr);
-SANITIZER_INTERFACE_ATTRIBUTE void __tsan_unaligned_write2(uu16 *addr, u16 v);
-SANITIZER_INTERFACE_ATTRIBUTE void __tsan_unaligned_write4(uu32 *addr, u32 v);
-SANITIZER_INTERFACE_ATTRIBUTE void __tsan_unaligned_write8(uu64 *addr, u64 v);
+SANITIZER_INTERFACE_ATTRIBUTE void __tsan_unaligned_read2(const void *addr);
+SANITIZER_INTERFACE_ATTRIBUTE void __tsan_unaligned_read4(const void *addr);
+SANITIZER_INTERFACE_ATTRIBUTE void __tsan_unaligned_read8(const void *addr);
+SANITIZER_INTERFACE_ATTRIBUTE void __tsan_unaligned_read16(const void *addr);
+
+SANITIZER_INTERFACE_ATTRIBUTE void __tsan_unaligned_write2(void *addr);
+SANITIZER_INTERFACE_ATTRIBUTE void __tsan_unaligned_write4(void *addr);
+SANITIZER_INTERFACE_ATTRIBUTE void __tsan_unaligned_write8(void *addr);
+SANITIZER_INTERFACE_ATTRIBUTE void __tsan_unaligned_write16(void *addr);
 
 SANITIZER_INTERFACE_ATTRIBUTE void __tsan_vptr_read(void **vptr_p);
 SANITIZER_INTERFACE_ATTRIBUTE
diff --git a/lib/tsan/rtl/tsan_interface_java.cc b/lib/tsan/rtl/tsan_interface_java.cc
index 8615349f657f..0aea63d11671 100644
--- a/lib/tsan/rtl/tsan_interface_java.cc
+++ b/lib/tsan/rtl/tsan_interface_java.cc
@@ -219,3 +219,33 @@ int __tsan_java_mutex_unlock_rec(jptr addr) {
 
   return MutexUnlock(thr, pc, addr, true);
 }
+
+void __tsan_java_acquire(jptr addr) {
+  SCOPED_JAVA_FUNC(__tsan_java_acquire);
+  DPrintf("#%d: java_acquire(%p)\n", thr->tid, addr);
+  CHECK_NE(jctx, 0);
+  CHECK_GE(addr, jctx->heap_begin);
+  CHECK_LT(addr, jctx->heap_begin + jctx->heap_size);
+
+  Acquire(thr, caller_pc, addr);
+}
+
+void __tsan_java_release(jptr addr) {
+  SCOPED_JAVA_FUNC(__tsan_java_release);
+  DPrintf("#%d: java_release(%p)\n", thr->tid, addr);
+  CHECK_NE(jctx, 0);
+  CHECK_GE(addr, jctx->heap_begin);
+  CHECK_LT(addr, jctx->heap_begin + jctx->heap_size);
+
+  Release(thr, caller_pc, addr);
+}
+
+void __tsan_java_release_store(jptr addr) {
+  SCOPED_JAVA_FUNC(__tsan_java_release);
+  DPrintf("#%d: java_release_store(%p)\n", thr->tid, addr);
+  CHECK_NE(jctx, 0);
+  CHECK_GE(addr, jctx->heap_begin);
+  CHECK_LT(addr, jctx->heap_begin + jctx->heap_size);
+
+  ReleaseStore(thr, caller_pc, addr);
+}
diff --git a/lib/tsan/rtl/tsan_interface_java.h b/lib/tsan/rtl/tsan_interface_java.h
index 1f793df712de..30153a1d8505 100644
--- a/lib/tsan/rtl/tsan_interface_java.h
+++ b/lib/tsan/rtl/tsan_interface_java.h
@@ -79,6 +79,14 @@ void __tsan_java_mutex_lock_rec(jptr addr, int rec) INTERFACE_ATTRIBUTE;
 // the same recursion level.
 int __tsan_java_mutex_unlock_rec(jptr addr) INTERFACE_ATTRIBUTE;
 
+// Raw acquire/release primitives.
+// Can be used to establish happens-before edges on volatile/final fields,
+// in atomic operations, etc. release_store is the same as release, but it
+// breaks release sequence on addr (see C++ standard 1.10/7 for details).
+void __tsan_java_acquire(jptr addr) INTERFACE_ATTRIBUTE;
+void __tsan_java_release(jptr addr) INTERFACE_ATTRIBUTE;
+void __tsan_java_release_store(jptr addr) INTERFACE_ATTRIBUTE;
+
 #ifdef __cplusplus
 }  // extern "C"
 #endif
diff --git a/lib/tsan/rtl/tsan_mman.cc b/lib/tsan/rtl/tsan_mman.cc
index 285bdb34d91d..ebb3f77fb992 100644
--- a/lib/tsan/rtl/tsan_mman.cc
+++ b/lib/tsan/rtl/tsan_mman.cc
@@ -45,7 +45,7 @@ Allocator *allocator() {
 }
 
 void InitializeAllocator() {
-  allocator()->Init();
+  allocator()->Init(common_flags()->allocator_may_return_null);
 }
 
 void AllocatorThreadStart(ThreadState *thr) {
@@ -78,7 +78,7 @@ static void SignalUnsafeCall(ThreadState *thr, uptr pc) {
 
 void *user_alloc(ThreadState *thr, uptr pc, uptr sz, uptr align, bool signal) {
   if ((sz >= (1ull << 40)) || (align >= (1ull << 40)))
-    return AllocatorReturnNull();
+    return allocator()->ReturnNullOrDie();
   void *p = allocator()->Allocate(&thr->alloc_cache, sz, align);
   if (p == 0)
     return 0;
@@ -89,6 +89,15 @@ void *user_alloc(ThreadState *thr, uptr pc, uptr sz, uptr align, bool signal) {
   return p;
 }
 
+void *user_calloc(ThreadState *thr, uptr pc, uptr size, uptr n) {
+  if (CallocShouldReturnNullDueToOverflow(size, n))
+    return allocator()->ReturnNullOrDie();
+  void *p = user_alloc(thr, pc, n * size);
+  if (p)
+    internal_memset(p, 0, n * size);
+  return p;
+}
+
 void user_free(ThreadState *thr, uptr pc, void *p, bool signal) {
   if (ctx && ctx->initialized)
     OnUserFree(thr, pc, (uptr)p, true);
diff --git a/lib/tsan/rtl/tsan_mman.h b/lib/tsan/rtl/tsan_mman.h
index 7d41fa864a26..5ff956d827f6 100644
--- a/lib/tsan/rtl/tsan_mman.h
+++ b/lib/tsan/rtl/tsan_mman.h
@@ -27,6 +27,7 @@ void AllocatorPrintStats();
 // For user allocations.
 void *user_alloc(ThreadState *thr, uptr pc, uptr sz,
                  uptr align = kDefaultAlignment, bool signal = true);
+void *user_calloc(ThreadState *thr, uptr pc, uptr sz, uptr n);
 // Does not accept NULL.
 void user_free(ThreadState *thr, uptr pc, void *p, bool signal = true);
 void *user_realloc(ThreadState *thr, uptr pc, void *p, uptr sz);
diff --git a/lib/tsan/rtl/tsan_mutex.cc b/lib/tsan/rtl/tsan_mutex.cc
index 9ea9bae21b50..dc5a462a8081 100644
--- a/lib/tsan/rtl/tsan_mutex.cc
+++ b/lib/tsan/rtl/tsan_mutex.cc
@@ -25,7 +25,7 @@ namespace __tsan {
 // then Report mutex can be locked while under Threads mutex.
 // The leaf mutexes can be locked under any other mutexes.
 // Recursive locking is not supported.
-#if TSAN_DEBUG && !SANITIZER_GO
+#if SANITIZER_DEBUG && !SANITIZER_GO
 const MutexType MutexTypeLeaf = (MutexType)-1;
 static MutexType CanLockTab[MutexTypeCount][MutexTypeCount] = {
   /*0  MutexTypeInvalid*/     {},
@@ -47,7 +47,7 @@ static bool CanLockAdj[MutexTypeCount][MutexTypeCount];
 #endif
 
 void InitializeMutex() {
-#if TSAN_DEBUG && !SANITIZER_GO
+#if SANITIZER_DEBUG && !SANITIZER_GO
   // Build the "can lock" adjacency matrix.
   // If [i][j]==true, then one can lock mutex j while under mutex i.
   const int N = MutexTypeCount;
@@ -128,7 +128,7 @@ InternalDeadlockDetector::InternalDeadlockDetector() {
   // Rely on zero initialization because some mutexes can be locked before ctor.
 }
 
-#if TSAN_DEBUG && !SANITIZER_GO
+#if SANITIZER_DEBUG && !SANITIZER_GO
 void InternalDeadlockDetector::Lock(MutexType t) {
   // Printf("LOCK %d @%zu\n", t, seq_ + 1);
   CHECK_GT(t, MutexTypeInvalid);
@@ -170,7 +170,7 @@ void InternalDeadlockDetector::CheckNoLocks() {
 #endif
 
 void CheckNoLocks(ThreadState *thr) {
-#if TSAN_DEBUG && !SANITIZER_GO
+#if SANITIZER_DEBUG && !SANITIZER_GO
   thr->internal_deadlock_detector.CheckNoLocks();
 #endif
 }
@@ -208,7 +208,7 @@ class Backoff {
 Mutex::Mutex(MutexType type, StatType stat_type) {
   CHECK_GT(type, MutexTypeInvalid);
   CHECK_LT(type, MutexTypeCount);
-#if TSAN_DEBUG
+#if SANITIZER_DEBUG
   type_ = type;
 #endif
 #if TSAN_COLLECT_STATS
@@ -222,7 +222,7 @@ Mutex::~Mutex() {
 }
 
 void Mutex::Lock() {
-#if TSAN_DEBUG && !SANITIZER_GO
+#if SANITIZER_DEBUG && !SANITIZER_GO
   cur_thread()->internal_deadlock_detector.Lock(type_);
 #endif
   uptr cmp = kUnlocked;
@@ -247,13 +247,13 @@ void Mutex::Unlock() {
   uptr prev = atomic_fetch_sub(&state_, kWriteLock, memory_order_release);
   (void)prev;
   DCHECK_NE(prev & kWriteLock, 0);
-#if TSAN_DEBUG && !SANITIZER_GO
+#if SANITIZER_DEBUG && !SANITIZER_GO
   cur_thread()->internal_deadlock_detector.Unlock(type_);
 #endif
 }
 
 void Mutex::ReadLock() {
-#if TSAN_DEBUG && !SANITIZER_GO
+#if SANITIZER_DEBUG && !SANITIZER_GO
   cur_thread()->internal_deadlock_detector.Lock(type_);
 #endif
   uptr prev = atomic_fetch_add(&state_, kReadLock, memory_order_acquire);
@@ -275,7 +275,7 @@ void Mutex::ReadUnlock() {
   (void)prev;
   DCHECK_EQ(prev & kWriteLock, 0);
   DCHECK_GT(prev & ~kWriteLock, 0);
-#if TSAN_DEBUG && !SANITIZER_GO
+#if SANITIZER_DEBUG && !SANITIZER_GO
   cur_thread()->internal_deadlock_detector.Unlock(type_);
 #endif
 }
diff --git a/lib/tsan/rtl/tsan_mutex.h b/lib/tsan/rtl/tsan_mutex.h
index 7bb1c48fcac8..88fad57c78a0 100644
--- a/lib/tsan/rtl/tsan_mutex.h
+++ b/lib/tsan/rtl/tsan_mutex.h
@@ -52,7 +52,7 @@ class Mutex {
 
  private:
   atomic_uintptr_t state_;
-#if TSAN_DEBUG
+#if SANITIZER_DEBUG
   MutexType type_;
 #endif
 #if TSAN_COLLECT_STATS
diff --git a/lib/tsan/rtl/tsan_platform.h b/lib/tsan/rtl/tsan_platform.h
index 270a7519dd0a..03f95694d26c 100644
--- a/lib/tsan/rtl/tsan_platform.h
+++ b/lib/tsan/rtl/tsan_platform.h
@@ -252,9 +252,6 @@ void InitializePlatform();
 void FlushShadowMemory();
 void WriteMemoryProfile(char *buf, uptr buf_size, uptr nthread, uptr nlive);
 
-void *internal_start_thread(void(*func)(void*), void *arg);
-void internal_join_thread(void *th);
-
 // Says whether the addr relates to a global var.
 // Guesses with high probability, may yield both false positives and negatives.
 bool IsGlobalVar(uptr addr);
diff --git a/lib/tsan/rtl/tsan_platform_linux.cc b/lib/tsan/rtl/tsan_platform_linux.cc
index 4dcfa558529c..7bc28db296ec 100644
--- a/lib/tsan/rtl/tsan_platform_linux.cc
+++ b/lib/tsan/rtl/tsan_platform_linux.cc
@@ -215,10 +215,11 @@ void InitializeShadowMemory() {
   // Frequently a thread uses only a small part of stack and similarly
   // a program uses a small part of large mmap. On some programs
   // we see 20% memory usage reduction without huge pages for this range.
-#ifdef MADV_NOHUGEPAGE
-  madvise((void*)MemToShadow(0x7f0000000000ULL),
-      0x10000000000ULL * kShadowMultiplier, MADV_NOHUGEPAGE);
-#endif
+  // FIXME: don't use constants here.
+  NoHugePagesInRegion(MemToShadow(0x7f0000000000ULL),
+                      0x10000000000ULL * kShadowMultiplier);
+  if (common_flags()->use_madv_dontdump)
+    DontDumpShadowMemory(kShadowBeg, kShadowEnd - kShadowBeg);
   DPrintf("memory shadow: %zx-%zx (%zuGB)\n",
       kShadowBeg, kShadowEnd,
       (kShadowEnd - kShadowBeg) >> 30);
@@ -232,6 +233,8 @@ void InitializeShadowMemory() {
                "to link with -pie (%p, %p).\n", meta, kMetaShadowBeg);
     Die();
   }
+  if (common_flags()->use_madv_dontdump)
+    DontDumpShadowMemory(meta, meta_size);
   DPrintf("meta shadow: %zx-%zx (%zuGB)\n",
       meta, meta + meta_size, meta_size >> 30);
 
diff --git a/lib/tsan/rtl/tsan_platform_mac.cc b/lib/tsan/rtl/tsan_platform_mac.cc
index 15b9f9d2cb19..63f1748e13ce 100644
--- a/lib/tsan/rtl/tsan_platform_mac.cc
+++ b/lib/tsan/rtl/tsan_platform_mac.cc
@@ -60,6 +60,8 @@ void InitializeShadowMemory() {
            "to link with -pie.\n");
     Die();
   }
+  if (common_flags()->use_madv_dontdump)
+    DontDumpShadowMemory(kShadowBeg, kShadowEnd - kShadowBeg);
   DPrintf("kShadow %zx-%zx (%zuGB)\n",
       kShadowBeg, kShadowEnd,
       (kShadowEnd - kShadowBeg) >> 30);
diff --git a/lib/tsan/rtl/tsan_rtl.cc b/lib/tsan/rtl/tsan_rtl.cc
index 7cb7008e2980..b3320aad8038 100644
--- a/lib/tsan/rtl/tsan_rtl.cc
+++ b/lib/tsan/rtl/tsan_rtl.cc
@@ -365,8 +365,7 @@ int Finalize(ThreadState *thr) {
   ctx->report_mtx.Unlock();
 
 #ifndef SANITIZER_GO
-  if (common_flags()->verbosity)
-    AllocatorPrintStats();
+  if (Verbosity()) AllocatorPrintStats();
 #endif
 
   ThreadFinalize(thr);
@@ -565,43 +564,26 @@ void MemoryAccessImpl1(ThreadState *thr, uptr addr,
   // it's just not worth it (performance- and complexity-wise).
 
   Shadow old(0);
-  if (kShadowCnt == 1) {
-    int idx = 0;
-#include "tsan_update_shadow_word_inl.h"
-  } else if (kShadowCnt == 2) {
-    int idx = 0;
-#include "tsan_update_shadow_word_inl.h"
-    idx = 1;
-#include "tsan_update_shadow_word_inl.h"
-  } else if (kShadowCnt == 4) {
-    int idx = 0;
-#include "tsan_update_shadow_word_inl.h"
-    idx = 1;
-#include "tsan_update_shadow_word_inl.h"
-    idx = 2;
-#include "tsan_update_shadow_word_inl.h"
-    idx = 3;
-#include "tsan_update_shadow_word_inl.h"
-  } else if (kShadowCnt == 8) {
-    int idx = 0;
-#include "tsan_update_shadow_word_inl.h"
-    idx = 1;
-#include "tsan_update_shadow_word_inl.h"
-    idx = 2;
-#include "tsan_update_shadow_word_inl.h"
-    idx = 3;
+
+  // It release mode we manually unroll the loop,
+  // because empirically gcc generates better code this way.
+  // However, we can't afford unrolling in debug mode, because the function
+  // consumes almost 4K of stack. Gtest gives only 4K of stack to death test
+  // threads, which is not enough for the unrolled loop.
+#if SANITIZER_DEBUG
+  for (int idx = 0; idx < 4; idx++) {
 #include "tsan_update_shadow_word_inl.h"
-    idx = 4;
+  }
+#else
+  int idx = 0;
 #include "tsan_update_shadow_word_inl.h"
-    idx = 5;
+  idx = 1;
 #include "tsan_update_shadow_word_inl.h"
-    idx = 6;
+  idx = 2;
 #include "tsan_update_shadow_word_inl.h"
-    idx = 7;
+  idx = 3;
 #include "tsan_update_shadow_word_inl.h"
-  } else {
-    CHECK(false);
-  }
+#endif
 
   // we did not find any races and had already stored
   // the current access info, so we are done
@@ -652,7 +634,7 @@ bool ContainsSameAccessSlow(u64 *s, u64 a, u64 sync_epoch, bool is_write) {
   return false;
 }
 
-#if defined(__SSE3__) && TSAN_SHADOW_COUNT == 4
+#if defined(__SSE3__)
 #define SHUF(v0, v1, i0, i1, i2, i3) _mm_castps_si128(_mm_shuffle_ps( \
     _mm_castsi128_ps(v0), _mm_castsi128_ps(v1), \
     (i0)*1 + (i1)*4 + (i2)*16 + (i3)*64))
@@ -712,11 +694,12 @@ bool ContainsSameAccessFast(u64 *s, u64 a, u64 sync_epoch, bool is_write) {
 
 ALWAYS_INLINE
 bool ContainsSameAccess(u64 *s, u64 a, u64 sync_epoch, bool is_write) {
-#if defined(__SSE3__) && TSAN_SHADOW_COUNT == 4
+#if defined(__SSE3__)
   bool res = ContainsSameAccessFast(s, a, sync_epoch, is_write);
   // NOTE: this check can fail if the shadow is concurrently mutated
-  // by other threads.
-  DCHECK_EQ(res, ContainsSameAccessSlow(s, a, sync_epoch, is_write));
+  // by other threads. But it still can be useful if you modify
+  // ContainsSameAccessFast and want to ensure that it's not completely broken.
+  // DCHECK_EQ(res, ContainsSameAccessSlow(s, a, sync_epoch, is_write));
   return res;
 #else
   return ContainsSameAccessSlow(s, a, sync_epoch, is_write);
@@ -733,7 +716,7 @@ void MemoryAccess(ThreadState *thr, uptr pc, uptr addr,
       (int)(1 << kAccessSizeLog), kAccessIsWrite, shadow_mem,
       (uptr)shadow_mem[0], (uptr)shadow_mem[1],
       (uptr)shadow_mem[2], (uptr)shadow_mem[3]);
-#if TSAN_DEBUG
+#if SANITIZER_DEBUG
   if (!IsAppMem(addr)) {
     Printf("Access to non app mem %zx\n", addr);
     DCHECK(IsAppMem(addr));
@@ -990,7 +973,7 @@ bool MD5Hash::operator==(const MD5Hash &other) const {
   return hash[0] == other.hash[0] && hash[1] == other.hash[1];
 }
 
-#if TSAN_DEBUG
+#if SANITIZER_DEBUG
 void build_consistency_debug() {}
 #else
 void build_consistency_release() {}
@@ -1002,16 +985,6 @@ void build_consistency_stats() {}
 void build_consistency_nostats() {}
 #endif
 
-#if TSAN_SHADOW_COUNT == 1
-void build_consistency_shadow1() {}
-#elif TSAN_SHADOW_COUNT == 2
-void build_consistency_shadow2() {}
-#elif TSAN_SHADOW_COUNT == 4
-void build_consistency_shadow4() {}
-#else
-void build_consistency_shadow8() {}
-#endif
-
 }  // namespace __tsan
 
 #ifndef SANITIZER_GO
diff --git a/lib/tsan/rtl/tsan_rtl.h b/lib/tsan/rtl/tsan_rtl.h
index 8d886875159b..768e8307fcc8 100644
--- a/lib/tsan/rtl/tsan_rtl.h
+++ b/lib/tsan/rtl/tsan_rtl.h
@@ -685,7 +685,7 @@ void AcquireReleaseImpl(ThreadState *thr, uptr pc, SyncClock *c);
 // The trick is that the call preserves all registers and the compiler
 // does not treat it as a call.
 // If it does not work for you, use normal call.
-#if TSAN_DEBUG == 0
+#if !SANITIZER_DEBUG && defined(__x86_64__)
 // The caller may not create the stack frame for itself at all,
 // so we create a reserve stack frame for it (1024b must be enough).
 #define HACKY_CALL(f) \
diff --git a/lib/tsan/rtl/tsan_rtl_report.cc b/lib/tsan/rtl/tsan_rtl_report.cc
index 0481b23b7be0..d1621454242c 100644
--- a/lib/tsan/rtl/tsan_rtl_report.cc
+++ b/lib/tsan/rtl/tsan_rtl_report.cc
@@ -87,7 +87,7 @@ static void StackStripMain(SymbolizedStack *frames) {
     // can actually happen if we do not instrument some code,
     // so it's only a debug print. However we must try hard to not miss it
     // due to our fault.
-    DPrintf("Bottom stack frame of stack %zx is missed\n", stack->info.address);
+    DPrintf("Bottom stack frame is missed\n");
   }
 #else
   // The last frame always point into runtime (gosched0, goexit0, runtime.main).
@@ -251,7 +251,8 @@ ThreadContext *IsThreadStackOrTls(uptr addr, bool *is_stack) {
 
 void ScopedReport::AddThread(int unique_tid, bool suppressable) {
 #ifndef SANITIZER_GO
-  AddThread(FindThreadByUidLocked(unique_tid), suppressable);
+  if (const ThreadContext *tctx = FindThreadByUidLocked(unique_tid))
+    AddThread(tctx, suppressable);
 #endif
 }
 
@@ -397,7 +398,7 @@ void RestoreStack(int tid, const u64 epoch, VarSizeStackTrace *stk,
   InternalScopedBuffer<uptr> stack(kShadowStackSize);
   for (uptr i = 0; i < hdr->stack0.size; i++) {
     stack[i] = hdr->stack0.trace[i];
-    DPrintf2("  #%02lu: pc=%zx\n", i, stack[i]);
+    DPrintf2("  #%02zu: pc=%zx\n", i, stack[i]);
   }
   if (mset)
     *mset = hdr->mset0;
diff --git a/lib/tsan/rtl/tsan_rtl_thread.cc b/lib/tsan/rtl/tsan_rtl_thread.cc
index 7b7b27c024f6..e026217ed171 100644
--- a/lib/tsan/rtl/tsan_rtl_thread.cc
+++ b/lib/tsan/rtl/tsan_rtl_thread.cc
@@ -111,12 +111,13 @@ void ThreadContext::OnStarted(void *arg) {
     thr->dd_pt = ctx->dd->CreatePhysicalThread();
     thr->dd_lt = ctx->dd->CreateLogicalThread(unique_id);
   }
+  thr->fast_state.SetHistorySize(flags()->history_size);
+  // Commit switch to the new part of the trace.
+  // TraceAddEvent will reset stack0/mset0 in the new part for us.
+  TraceAddEvent(thr, thr->fast_state, EventTypeMop, 0);
+
   thr->fast_synch_epoch = epoch0;
   AcquireImpl(thr, 0, &sync);
-  thr->fast_state.SetHistorySize(flags()->history_size);
-  const uptr trace = (epoch0 / kTracePartSize) % TraceParts();
-  Trace *thr_trace = ThreadTrace(thr->tid);
-  thr_trace->headers[trace].epoch0 = epoch0;
   StatInc(thr, StatSyncAcquire);
   sync.Reset(&thr->clock_cache);
   DPrintf("#%d: ThreadStart epoch=%zu stk_addr=%zx stk_size=%zx "
@@ -329,7 +330,7 @@ void MemoryAccessRange(ThreadState *thr, uptr pc, uptr addr,
       thr->tid, (void*)pc, (void*)addr,
       (int)size, is_write);
 
-#if TSAN_DEBUG
+#if SANITIZER_DEBUG
   if (!IsAppMem(addr)) {
     Printf("Access to non app mem %zx\n", addr);
     DCHECK(IsAppMem(addr));