aboutsummaryrefslogtreecommitdiff
path: root/lib
diff options
context:
space:
mode:
authorDimitry Andric <dim@FreeBSD.org>2017-07-13 19:25:48 +0000
committerDimitry Andric <dim@FreeBSD.org>2017-07-13 19:25:48 +0000
commit1992b790c2c12b7850bdf86662b67302052ec2fe (patch)
tree623c69b5fbf527bba17ecb9431ae5189871cecd4 /lib
parent50aa32eff79f252ab05a0c0a589cf2ca37cd9923 (diff)
downloadsrc-1992b790c2c12b7850bdf86662b67302052ec2fe.tar.gz
src-1992b790c2c12b7850bdf86662b67302052ec2fe.zip
Vendor import of compiler-rt trunk r307894:vendor/compiler-rt/compiler-rt-trunk-r307894
Notes
Notes: svn path=/vendor/compiler-rt/dist/; revision=320961 svn path=/vendor/compiler-rt/compiler-rt-trunk-r307894/; revision=320962; tag=vendor/compiler-rt/compiler-rt-trunk-r307894
Diffstat (limited to 'lib')
-rw-r--r--lib/asan/asan_errors.cc7
-rw-r--r--lib/asan/asan_internal.h1
-rw-r--r--lib/asan/asan_linux.cc11
-rw-r--r--lib/asan/asan_mac.cc23
-rw-r--r--lib/asan/asan_new_delete.cc2
-rw-r--r--lib/asan/asan_posix.cc2
-rw-r--r--lib/asan/asan_rtl.cc10
-rw-r--r--lib/asan/asan_thread.cc1
-rw-r--r--lib/asan/asan_win.cc12
-rwxr-xr-xlib/asan/scripts/asan_device_setup2
-rw-r--r--lib/builtins/CMakeLists.txt13
-rw-r--r--lib/builtins/cpu_model.c523
-rw-r--r--lib/builtins/int_util.c10
-rw-r--r--lib/esan/working_set.cpp19
-rw-r--r--lib/lsan/lsan_common.cc16
-rw-r--r--lib/lsan/lsan_common.h2
-rw-r--r--lib/lsan/lsan_common_mac.cc2
-rw-r--r--lib/msan/msan_interceptors.cc11
-rw-r--r--lib/sanitizer_common/CMakeLists.txt4
-rw-r--r--lib/sanitizer_common/sanitizer_common.h3
-rw-r--r--lib/sanitizer_common/sanitizer_common_interceptors.inc1
-rw-r--r--lib/sanitizer_common/sanitizer_errno.cc35
-rw-r--r--lib/sanitizer_common/sanitizer_errno.h35
-rw-r--r--lib/sanitizer_common/sanitizer_errno_codes.h34
-rw-r--r--lib/sanitizer_common/sanitizer_linux.cc79
-rw-r--r--lib/sanitizer_common/sanitizer_linux_libcdep.cc19
-rw-r--r--lib/sanitizer_common/sanitizer_mac.cc58
-rw-r--r--lib/sanitizer_common/sanitizer_mac.h2
-rw-r--r--lib/sanitizer_common/sanitizer_mac_libcdep.cc30
-rw-r--r--lib/sanitizer_common/sanitizer_platform_limits_posix.cc9
-rw-r--r--lib/sanitizer_common/sanitizer_platform_limits_posix.h3
-rw-r--r--lib/sanitizer_common/sanitizer_posix.cc119
-rw-r--r--lib/sanitizer_common/sanitizer_procmaps.h41
-rw-r--r--lib/sanitizer_common/sanitizer_procmaps_common.cc14
-rw-r--r--lib/sanitizer_common/sanitizer_procmaps_freebsd.cc33
-rw-r--r--lib/sanitizer_common/sanitizer_procmaps_linux.cc43
-rw-r--r--lib/sanitizer_common/sanitizer_procmaps_mac.cc191
-rw-r--r--lib/sanitizer_common/sanitizer_stacktrace_libcdep.cc3
-rw-r--r--lib/sanitizer_common/sanitizer_win.cc3
-rw-r--r--lib/scudo/scudo_allocator.cpp16
-rw-r--r--lib/scudo/scudo_tls.h2
-rw-r--r--lib/scudo/scudo_utils.cpp36
-rw-r--r--lib/scudo/scudo_utils.h57
-rw-r--r--lib/tsan/CMakeLists.txt2
-rwxr-xr-xlib/tsan/check_analyze.sh4
-rw-r--r--lib/tsan/dd/dd_interceptors.cc19
-rwxr-xr-xlib/tsan/go/buildgo.sh2
-rw-r--r--lib/tsan/rtl/tsan_clock.cc119
-rw-r--r--lib/tsan/rtl/tsan_clock.h4
-rw-r--r--lib/tsan/rtl/tsan_dense_alloc.h11
-rw-r--r--lib/tsan/rtl/tsan_interceptors.cc29
-rw-r--r--lib/tsan/rtl/tsan_interceptors_mac.cc7
-rw-r--r--lib/tsan/rtl/tsan_interface_ann.cc16
-rw-r--r--lib/tsan/rtl/tsan_interface_atomic.cc16
-rw-r--r--lib/tsan/rtl/tsan_mman.cc2
-rw-r--r--lib/tsan/rtl/tsan_platform.h49
-rw-r--r--lib/tsan/rtl/tsan_platform_linux.cc17
-rw-r--r--lib/tsan/rtl/tsan_platform_mac.cc8
-rw-r--r--lib/tsan/rtl/tsan_platform_posix.cc27
-rw-r--r--lib/tsan/rtl/tsan_rtl.cc3
-rw-r--r--lib/tsan/rtl/tsan_rtl_aarch64.S127
-rw-r--r--lib/tsan/rtl/tsan_rtl_amd64.S6
-rw-r--r--lib/tsan/rtl/tsan_rtl_mutex.cc12
-rw-r--r--lib/tsan/rtl/tsan_rtl_report.cc2
-rw-r--r--lib/tsan/rtl/tsan_rtl_thread.cc4
-rw-r--r--lib/tsan/rtl/tsan_stat.cc5
-rw-r--r--lib/tsan/rtl/tsan_stat.h5
-rw-r--r--lib/tsan/rtl/tsan_sync.cc4
-rw-r--r--lib/tsan/tests/CMakeLists.txt1
-rw-r--r--lib/tsan/tests/unit/tsan_clock_test.cc62
70 files changed, 1132 insertions, 968 deletions
diff --git a/lib/asan/asan_errors.cc b/lib/asan/asan_errors.cc
index 57490ad180b5..b7a38eb7cece 100644
--- a/lib/asan/asan_errors.cc
+++ b/lib/asan/asan_errors.cc
@@ -61,10 +61,9 @@ static void MaybeDumpRegisters(void *context) {
static void MaybeReportNonExecRegion(uptr pc) {
#if SANITIZER_FREEBSD || SANITIZER_LINUX
MemoryMappingLayout proc_maps(/*cache_enabled*/ true);
- uptr start, end, protection;
- while (proc_maps.Next(&start, &end, nullptr, nullptr, 0, &protection)) {
- if (pc >= start && pc < end &&
- !(protection & MemoryMappingLayout::kProtectionExecute))
+ MemoryMappedSegment segment;
+ while (proc_maps.Next(&segment)) {
+ if (pc >= segment.start && pc < segment.end && !segment.IsExecutable())
Report("Hint: PC is at a non-executable region. Maybe a wild jump?\n");
}
#endif
diff --git a/lib/asan/asan_internal.h b/lib/asan/asan_internal.h
index 3b70695249e4..f09bbd83af25 100644
--- a/lib/asan/asan_internal.h
+++ b/lib/asan/asan_internal.h
@@ -75,6 +75,7 @@ void NORETURN ShowStatsAndAbort();
void ReplaceSystemMalloc();
// asan_linux.cc / asan_mac.cc / asan_win.cc
+uptr FindDynamicShadowStart();
void *AsanDoesNotSupportStaticLinkage();
void AsanCheckDynamicRTPrereqs();
void AsanCheckIncompatibleRT();
diff --git a/lib/asan/asan_linux.cc b/lib/asan/asan_linux.cc
index 50ef84c39a66..6d47ba432a61 100644
--- a/lib/asan/asan_linux.cc
+++ b/lib/asan/asan_linux.cc
@@ -77,6 +77,11 @@ void *AsanDoesNotSupportStaticLinkage() {
return &_DYNAMIC; // defined in link.h
}
+uptr FindDynamicShadowStart() {
+ UNREACHABLE("FindDynamicShadowStart is not available");
+ return 0;
+}
+
void AsanApplyToGlobals(globals_op_fptr op, const void *needle) {
UNIMPLEMENTED();
}
@@ -140,9 +145,9 @@ void AsanCheckIncompatibleRT() {
// system libraries, causing crashes later in ASan initialization.
MemoryMappingLayout proc_maps(/*cache_enabled*/true);
char filename[128];
- while (proc_maps.Next(nullptr, nullptr, nullptr, filename,
- sizeof(filename), nullptr)) {
- if (IsDynamicRTName(filename)) {
+ MemoryMappedSegment segment(filename, sizeof(filename));
+ while (proc_maps.Next(&segment)) {
+ if (IsDynamicRTName(segment.filename)) {
Report("Your application is linked against "
"incompatible ASan runtimes.\n");
Die();
diff --git a/lib/asan/asan_mac.cc b/lib/asan/asan_mac.cc
index 3c93b26d9bf6..b7af1a58664c 100644
--- a/lib/asan/asan_mac.cc
+++ b/lib/asan/asan_mac.cc
@@ -55,6 +55,29 @@ void *AsanDoesNotSupportStaticLinkage() {
return 0;
}
+uptr FindDynamicShadowStart() {
+ uptr granularity = GetMmapGranularity();
+ uptr alignment = 8 * granularity;
+ uptr left_padding = granularity;
+ uptr space_size = kHighShadowEnd + left_padding;
+
+ uptr largest_gap_found = 0;
+ uptr shadow_start = FindAvailableMemoryRange(space_size, alignment,
+ granularity, &largest_gap_found);
+ // If the shadow doesn't fit, restrict the address space to make it fit.
+ if (shadow_start == 0) {
+ uptr new_max_vm = RoundDownTo(largest_gap_found << SHADOW_SCALE, alignment);
+ RestrictMemoryToMaxAddress(new_max_vm);
+ kHighMemEnd = new_max_vm - 1;
+ space_size = kHighShadowEnd + left_padding;
+ shadow_start =
+ FindAvailableMemoryRange(space_size, alignment, granularity, nullptr);
+ }
+ CHECK_NE((uptr)0, shadow_start);
+ CHECK(IsAligned(shadow_start, alignment));
+ return shadow_start;
+}
+
// No-op. Mac does not support static linkage anyway.
void AsanCheckDynamicRTPrereqs() {}
diff --git a/lib/asan/asan_new_delete.cc b/lib/asan/asan_new_delete.cc
index 942b169d920c..e68c7f3e2400 100644
--- a/lib/asan/asan_new_delete.cc
+++ b/lib/asan/asan_new_delete.cc
@@ -26,7 +26,7 @@
// VS2015 dynamic CRT (MD) work.
#if SANITIZER_WINDOWS
#define CXX_OPERATOR_ATTRIBUTE
-#define COMMENT_EXPORT(sym) __pragma(comment(linker, "/export:"##sym))
+#define COMMENT_EXPORT(sym) __pragma(comment(linker, "/export:" sym))
#ifdef _WIN64
COMMENT_EXPORT("??2@YAPEAX_K@Z") // operator new
COMMENT_EXPORT("??2@YAPEAX_KAEBUnothrow_t@std@@@Z") // operator new nothrow
diff --git a/lib/asan/asan_posix.cc b/lib/asan/asan_posix.cc
index 68fde9139232..added746ace8 100644
--- a/lib/asan/asan_posix.cc
+++ b/lib/asan/asan_posix.cc
@@ -59,7 +59,7 @@ void AsanOnDeadlySignal(int signo, void *siginfo, void *context) {
// lis r0,-10000
// stdux r1,r1,r0 # store sp to [sp-10000] and update sp by -10000
// If the store faults then sp will not have been updated, so test above
- // will not work, becase the fault address will be more than just "slightly"
+ // will not work, because the fault address will be more than just "slightly"
// below sp.
if (!IsStackAccess && IsAccessibleMemoryRange(sig.pc, 4)) {
u32 inst = *(unsigned *)sig.pc;
diff --git a/lib/asan/asan_rtl.cc b/lib/asan/asan_rtl.cc
index d9d7d7e4f13d..5ae3568ae04a 100644
--- a/lib/asan/asan_rtl.cc
+++ b/lib/asan/asan_rtl.cc
@@ -438,15 +438,7 @@ static void InitializeShadowMemory() {
if (shadow_start == kDefaultShadowSentinel) {
__asan_shadow_memory_dynamic_address = 0;
CHECK_EQ(0, kLowShadowBeg);
-
- uptr granularity = GetMmapGranularity();
- uptr alignment = 8 * granularity;
- uptr left_padding = granularity;
- uptr space_size = kHighShadowEnd + left_padding;
-
- shadow_start = FindAvailableMemoryRange(space_size, alignment, granularity);
- CHECK_NE((uptr)0, shadow_start);
- CHECK(IsAligned(shadow_start, alignment));
+ shadow_start = FindDynamicShadowStart();
}
// Update the shadow memory address (potentially) used by instrumentation.
__asan_shadow_memory_dynamic_address = shadow_start;
diff --git a/lib/asan/asan_thread.cc b/lib/asan/asan_thread.cc
index 714496d5c606..b1a0d9a3b37f 100644
--- a/lib/asan/asan_thread.cc
+++ b/lib/asan/asan_thread.cc
@@ -200,7 +200,6 @@ FakeStack *AsanThread::AsyncSignalSafeLazyInitFakeStack() {
uptr stack_size = this->stack_size();
if (stack_size == 0) // stack_size is not yet available, don't use FakeStack.
return nullptr;
- CHECK_LE(stack_size, 0x10000000);
uptr old_val = 0;
// fake_stack_ has 3 states:
// 0 -- not initialized
diff --git a/lib/asan/asan_win.cc b/lib/asan/asan_win.cc
index 26db32465da8..8a839d913f95 100644
--- a/lib/asan/asan_win.cc
+++ b/lib/asan/asan_win.cc
@@ -217,6 +217,18 @@ void *AsanDoesNotSupportStaticLinkage() {
return 0;
}
+uptr FindDynamicShadowStart() {
+ uptr granularity = GetMmapGranularity();
+ uptr alignment = 8 * granularity;
+ uptr left_padding = granularity;
+ uptr space_size = kHighShadowEnd + left_padding;
+ uptr shadow_start =
+ FindAvailableMemoryRange(space_size, alignment, granularity, nullptr);
+ CHECK_NE((uptr)0, shadow_start);
+ CHECK(IsAligned(shadow_start, alignment));
+ return shadow_start;
+}
+
void AsanCheckDynamicRTPrereqs() {}
void AsanCheckIncompatibleRT() {}
diff --git a/lib/asan/scripts/asan_device_setup b/lib/asan/scripts/asan_device_setup
index 79ac2f916569..5a4f7c47cc21 100755
--- a/lib/asan/scripts/asan_device_setup
+++ b/lib/asan/scripts/asan_device_setup
@@ -52,7 +52,7 @@ function adb_remount {
local STORAGE=`$ADB shell mount | grep /system | cut -d ' ' -f1`
if [ "$STORAGE" != "" ]; then
echo Remounting $STORAGE at /system
- $ADB shell su -c "mount -o remount,rw $STORAGE /system"
+ $ADB shell su -c "mount -o rw,remount $STORAGE /system"
else
echo Failed to get storage device name for "/system" mount point
fi
diff --git a/lib/builtins/CMakeLists.txt b/lib/builtins/CMakeLists.txt
index 6556e7ac6513..f0d3f50714c1 100644
--- a/lib/builtins/CMakeLists.txt
+++ b/lib/builtins/CMakeLists.txt
@@ -44,7 +44,6 @@ set(GENERIC_SOURCES
ashrti3.c
bswapdi2.c
bswapsi2.c
- clear_cache.c
clzdi2.c
clzsi2.c
clzti2.c
@@ -68,7 +67,6 @@ set(GENERIC_SOURCES
divti3.c
divtf3.c
divxc3.c
- eprintf.c
extendsfdf2.c
extendhfsf2.c
ffsdi2.c
@@ -191,11 +189,12 @@ option(COMPILER_RT_EXCLUDE_ATOMIC_BUILTIN
"Skip the atomic builtin (this may be needed if system headers are unavailable)"
Off)
-if(NOT COMPILER_RT_BAREMETAL_BUILD)
+if(NOT FUCHSIA AND NOT COMPILER_RT_BAREMETAL_BUILD)
set(GENERIC_SOURCES
${GENERIC_SOURCES}
emutls.c
- enable_execute_stack.c)
+ enable_execute_stack.c
+ eprintf.c)
endif()
if(COMPILER_RT_HAS_ATOMIC_KEYWORD AND NOT COMPILER_RT_EXCLUDE_ATOMIC_BUILTIN)
@@ -221,6 +220,12 @@ if (HAVE_UNWIND_H)
gcc_personality_v0.c)
endif ()
+if (NOT FUCHSIA)
+ set(GENERIC_SOURCES
+ ${GENERIC_SOURCES}
+ clear_cache.c)
+endif()
+
if (NOT MSVC)
set(x86_64_SOURCES
x86_64/chkstk.S
diff --git a/lib/builtins/cpu_model.c b/lib/builtins/cpu_model.c
index 5ff6baf43876..c6b30eda0a77 100644
--- a/lib/builtins/cpu_model.c
+++ b/lib/builtins/cpu_model.c
@@ -44,29 +44,16 @@ enum ProcessorVendors {
};
enum ProcessorTypes {
- INTEL_ATOM = 1,
+ INTEL_BONNELL = 1,
INTEL_CORE2,
INTEL_COREI7,
AMDFAM10H,
AMDFAM15H,
- INTEL_i386,
- INTEL_i486,
- INTEL_PENTIUM,
- INTEL_PENTIUM_PRO,
- INTEL_PENTIUM_II,
- INTEL_PENTIUM_III,
- INTEL_PENTIUM_IV,
- INTEL_PENTIUM_M,
- INTEL_CORE_DUO,
- INTEL_XEONPHI,
- INTEL_X86_64,
- INTEL_NOCONA,
- INTEL_PRESCOTT,
- AMD_i486,
- AMDPENTIUM,
- AMDATHLON,
- AMDFAM14H,
- AMDFAM16H,
+ INTEL_SILVERMONT,
+ INTEL_KNL,
+ AMD_BTVER1,
+ AMD_BTVER2,
+ AMDFAM17H,
CPU_TYPE_MAX
};
@@ -79,32 +66,14 @@ enum ProcessorSubtypes {
AMDFAM10H_ISTANBUL,
AMDFAM15H_BDVER1,
AMDFAM15H_BDVER2,
- INTEL_PENTIUM_MMX,
- INTEL_CORE2_65,
- INTEL_CORE2_45,
+ AMDFAM15H_BDVER3,
+ AMDFAM15H_BDVER4,
+ AMDFAM17H_ZNVER1,
INTEL_COREI7_IVYBRIDGE,
INTEL_COREI7_HASWELL,
INTEL_COREI7_BROADWELL,
INTEL_COREI7_SKYLAKE,
INTEL_COREI7_SKYLAKE_AVX512,
- INTEL_ATOM_BONNELL,
- INTEL_ATOM_SILVERMONT,
- INTEL_KNIGHTS_LANDING,
- AMDPENTIUM_K6,
- AMDPENTIUM_K62,
- AMDPENTIUM_K63,
- AMDPENTIUM_GEODE,
- AMDATHLON_TBIRD,
- AMDATHLON_MP,
- AMDATHLON_XP,
- AMDATHLON_K8SSE3,
- AMDATHLON_OPTERON,
- AMDATHLON_FX,
- AMDATHLON_64,
- AMD_BTVER1,
- AMD_BTVER2,
- AMDFAM15H_BDVER3,
- AMDFAM15H_BDVER4,
CPU_SUBTYPE_MAX
};
@@ -120,11 +89,26 @@ enum ProcessorFeatures {
FEATURE_SSE4_2,
FEATURE_AVX,
FEATURE_AVX2,
- FEATURE_AVX512,
- FEATURE_AVX512SAVE,
- FEATURE_MOVBE,
- FEATURE_ADX,
- FEATURE_EM64T
+ FEATURE_SSE4_A,
+ FEATURE_FMA4,
+ FEATURE_XOP,
+ FEATURE_FMA,
+ FEATURE_AVX512F,
+ FEATURE_BMI,
+ FEATURE_BMI2,
+ FEATURE_AES,
+ FEATURE_PCLMUL,
+ FEATURE_AVX512VL,
+ FEATURE_AVX512BW,
+ FEATURE_AVX512DQ,
+ FEATURE_AVX512CD,
+ FEATURE_AVX512ER,
+ FEATURE_AVX512PF,
+ FEATURE_AVX512VBMI,
+ FEATURE_AVX512IFMA,
+ FEATURE_AVX5124VNNIW,
+ FEATURE_AVX5124FMAPS,
+ FEATURE_AVX512VPOPCNTDQ
};
// The check below for i386 was copied from clang's cpuid.h (__get_cpuid_max).
@@ -164,26 +148,27 @@ static bool isCpuIdSupported() {
/// getX86CpuIDAndInfo - Execute the specified cpuid and return the 4 values in
/// the specified arguments. If we can't run cpuid on the host, return true.
-static void getX86CpuIDAndInfo(unsigned value, unsigned *rEAX, unsigned *rEBX,
+static bool getX86CpuIDAndInfo(unsigned value, unsigned *rEAX, unsigned *rEBX,
unsigned *rECX, unsigned *rEDX) {
#if defined(__GNUC__) || defined(__clang__)
#if defined(__x86_64__)
- // gcc doesn't know cpuid would clobber ebx/rbx. Preseve it manually.
+ // gcc doesn't know cpuid would clobber ebx/rbx. Preserve it manually.
+ // FIXME: should we save this for Clang?
__asm__("movq\t%%rbx, %%rsi\n\t"
"cpuid\n\t"
"xchgq\t%%rbx, %%rsi\n\t"
: "=a"(*rEAX), "=S"(*rEBX), "=c"(*rECX), "=d"(*rEDX)
: "a"(value));
+ return false;
#elif defined(__i386__)
__asm__("movl\t%%ebx, %%esi\n\t"
"cpuid\n\t"
"xchgl\t%%ebx, %%esi\n\t"
: "=a"(*rEAX), "=S"(*rEBX), "=c"(*rECX), "=d"(*rEDX)
: "a"(value));
-// pedantic #else returns to appease -Wunreachable-code (so we don't generate
-// postprocessed code that looks like "return true; return false;")
+ return false;
#else
- assert(0 && "This method is defined only for x86.");
+ return true;
#endif
#elif defined(_MSC_VER)
// The MSVC intrinsic is portable across x86 and x64.
@@ -193,15 +178,16 @@ static void getX86CpuIDAndInfo(unsigned value, unsigned *rEAX, unsigned *rEBX,
*rEBX = registers[1];
*rECX = registers[2];
*rEDX = registers[3];
+ return false;
#else
- assert(0 && "This method is defined only for GNUC, Clang or MSVC.");
+ return true;
#endif
}
/// getX86CpuIDAndInfoEx - Execute the specified cpuid with subleaf and return
/// the 4 values in the specified arguments. If we can't run cpuid on the host,
/// return true.
-static void getX86CpuIDAndInfoEx(unsigned value, unsigned subleaf,
+static bool getX86CpuIDAndInfoEx(unsigned value, unsigned subleaf,
unsigned *rEAX, unsigned *rEBX, unsigned *rECX,
unsigned *rEDX) {
#if defined(__x86_64__) || defined(_M_X64)
@@ -213,6 +199,7 @@ static void getX86CpuIDAndInfoEx(unsigned value, unsigned subleaf,
"xchgq\t%%rbx, %%rsi\n\t"
: "=a"(*rEAX), "=S"(*rEBX), "=c"(*rECX), "=d"(*rEDX)
: "a"(value), "c"(subleaf));
+ return false;
#elif defined(_MSC_VER)
int registers[4];
__cpuidex(registers, value, subleaf);
@@ -220,8 +207,9 @@ static void getX86CpuIDAndInfoEx(unsigned value, unsigned subleaf,
*rEBX = registers[1];
*rECX = registers[2];
*rEDX = registers[3];
+ return false;
#else
- assert(0 && "This method is defined only for GNUC, Clang or MSVC.");
+ return true;
#endif
#elif defined(__i386__) || defined(_M_IX86)
#if defined(__GNUC__) || defined(__clang__)
@@ -230,6 +218,7 @@ static void getX86CpuIDAndInfoEx(unsigned value, unsigned subleaf,
"xchgl\t%%ebx, %%esi\n\t"
: "=a"(*rEAX), "=S"(*rEBX), "=c"(*rECX), "=d"(*rEDX)
: "a"(value), "c"(subleaf));
+ return false;
#elif defined(_MSC_VER)
__asm {
mov eax,value
@@ -244,11 +233,12 @@ static void getX86CpuIDAndInfoEx(unsigned value, unsigned subleaf,
mov esi,rEDX
mov dword ptr [esi],edx
}
+ return false;
#else
- assert(0 && "This method is defined only for GNUC, Clang or MSVC.");
+ return true;
#endif
#else
- assert(0 && "This method is defined only for x86.");
+ return true;
#endif
}
@@ -283,84 +273,15 @@ static void detectX86FamilyModel(unsigned EAX, unsigned *Family,
}
}
-static void getIntelProcessorTypeAndSubtype(unsigned int Family,
- unsigned int Model,
- unsigned int Brand_id,
- unsigned int Features,
- unsigned *Type, unsigned *Subtype) {
+static void
+getIntelProcessorTypeAndSubtype(unsigned Family, unsigned Model,
+ unsigned Brand_id, unsigned Features,
+ unsigned *Type, unsigned *Subtype) {
if (Brand_id != 0)
return;
switch (Family) {
- case 3:
- *Type = INTEL_i386;
- break;
- case 4:
- switch (Model) {
- case 0: // Intel486 DX processors
- case 1: // Intel486 DX processors
- case 2: // Intel486 SX processors
- case 3: // Intel487 processors, IntelDX2 OverDrive processors,
- // IntelDX2 processors
- case 4: // Intel486 SL processor
- case 5: // IntelSX2 processors
- case 7: // Write-Back Enhanced IntelDX2 processors
- case 8: // IntelDX4 OverDrive processors, IntelDX4 processors
- default:
- *Type = INTEL_i486;
- break;
- }
- case 5:
- switch (Model) {
- case 1: // Pentium OverDrive processor for Pentium processor (60, 66),
- // Pentium processors (60, 66)
- case 2: // Pentium OverDrive processor for Pentium processor (75, 90,
- // 100, 120, 133), Pentium processors (75, 90, 100, 120, 133,
- // 150, 166, 200)
- case 3: // Pentium OverDrive processors for Intel486 processor-based
- // systems
- *Type = INTEL_PENTIUM;
- break;
- case 4: // Pentium OverDrive processor with MMX technology for Pentium
- // processor (75, 90, 100, 120, 133), Pentium processor with
- // MMX technology (166, 200)
- *Type = INTEL_PENTIUM;
- *Subtype = INTEL_PENTIUM_MMX;
- break;
- default:
- *Type = INTEL_PENTIUM;
- break;
- }
case 6:
switch (Model) {
- case 0x01: // Pentium Pro processor
- *Type = INTEL_PENTIUM_PRO;
- break;
- case 0x03: // Intel Pentium II OverDrive processor, Pentium II processor,
- // model 03
- case 0x05: // Pentium II processor, model 05, Pentium II Xeon processor,
- // model 05, and Intel Celeron processor, model 05
- case 0x06: // Celeron processor, model 06
- *Type = INTEL_PENTIUM_II;
- break;
- case 0x07: // Pentium III processor, model 07, and Pentium III Xeon
- // processor, model 07
- case 0x08: // Pentium III processor, model 08, Pentium III Xeon processor,
- // model 08, and Celeron processor, model 08
- case 0x0a: // Pentium III Xeon processor, model 0Ah
- case 0x0b: // Pentium III processor, model 0Bh
- *Type = INTEL_PENTIUM_III;
- break;
- case 0x09: // Intel Pentium M processor, Intel Celeron M processor model 09.
- case 0x0d: // Intel Pentium M processor, Intel Celeron M processor, model
- // 0Dh. All processors are manufactured using the 90 nm process.
- case 0x15: // Intel EP80579 Integrated Processor and Intel EP80579
- // Integrated Processor with Intel QuickAssist Technology
- *Type = INTEL_PENTIUM_M;
- break;
- case 0x0e: // Intel Core Duo processor, Intel Core Solo processor, model
- // 0Eh. All processors are manufactured using the 65 nm process.
- *Type = INTEL_CORE_DUO;
- break; // yonah
case 0x0f: // Intel Core 2 Duo processor, Intel Core 2 Duo mobile
// processor, Intel Core 2 Quad processor, Intel Core 2 Quad
// mobile processor, Intel Core 2 Extreme processor, Intel
@@ -368,9 +289,6 @@ static void getIntelProcessorTypeAndSubtype(unsigned int Family,
// 0Fh. All processors are manufactured using the 65 nm process.
case 0x16: // Intel Celeron processor model 16h. All processors are
// manufactured using the 65 nm process
- *Type = INTEL_CORE2; // "core2"
- *Subtype = INTEL_CORE2_65;
- break;
case 0x17: // Intel Core 2 Extreme processor, Intel Xeon processor, model
// 17h. All processors are manufactured using the 45 nm process.
//
@@ -378,14 +296,13 @@ static void getIntelProcessorTypeAndSubtype(unsigned int Family,
case 0x1d: // Intel Xeon processor MP. All processors are manufactured using
// the 45 nm process.
*Type = INTEL_CORE2; // "penryn"
- *Subtype = INTEL_CORE2_45;
break;
case 0x1a: // Intel Core i7 processor and Intel Xeon processor. All
// processors are manufactured using the 45 nm process.
case 0x1e: // Intel(R) Core(TM) i7 CPU 870 @ 2.93GHz.
// As found in a Summer 2010 model iMac.
case 0x1f:
- case 0x2e: // Nehalem EX
+ case 0x2e: // Nehalem EX
*Type = INTEL_COREI7; // "nehalem"
*Subtype = INTEL_COREI7_NEHALEM;
break;
@@ -403,7 +320,7 @@ static void getIntelProcessorTypeAndSubtype(unsigned int Family,
*Subtype = INTEL_COREI7_SANDYBRIDGE;
break;
case 0x3a:
- case 0x3e: // Ivy Bridge EP
+ case 0x3e: // Ivy Bridge EP
*Type = INTEL_COREI7; // "ivybridge"
*Subtype = INTEL_COREI7_IVYBRIDGE;
break;
@@ -427,22 +344,26 @@ static void getIntelProcessorTypeAndSubtype(unsigned int Family,
break;
// Skylake:
- case 0x4e:
- *Type = INTEL_COREI7; // "skylake-avx512"
- *Subtype = INTEL_COREI7_SKYLAKE_AVX512;
- break;
- case 0x5e:
+ case 0x4e: // Skylake mobile
+ case 0x5e: // Skylake desktop
+ case 0x8e: // Kaby Lake mobile
+ case 0x9e: // Kaby Lake desktop
*Type = INTEL_COREI7; // "skylake"
*Subtype = INTEL_COREI7_SKYLAKE;
break;
+ // Skylake Xeon:
+ case 0x55:
+ *Type = INTEL_COREI7;
+ *Subtype = INTEL_COREI7_SKYLAKE_AVX512; // "skylake-avx512"
+ break;
+
case 0x1c: // Most 45 nm Intel Atom processors
case 0x26: // 45 nm Atom Lincroft
case 0x27: // 32 nm Atom Medfield
case 0x35: // 32 nm Atom Midview
case 0x36: // 32 nm Atom Midview
- *Type = INTEL_ATOM;
- *Subtype = INTEL_ATOM_BONNELL;
+ *Type = INTEL_BONNELL;
break; // "bonnell"
// Atom Silvermont codes from the Intel software optimization guide.
@@ -452,185 +373,29 @@ static void getIntelProcessorTypeAndSubtype(unsigned int Family,
case 0x5a:
case 0x5d:
case 0x4c: // really airmont
- *Type = INTEL_ATOM;
- *Subtype = INTEL_ATOM_SILVERMONT;
+ *Type = INTEL_SILVERMONT;
break; // "silvermont"
case 0x57:
- *Type = INTEL_XEONPHI; // knl
- *Subtype = INTEL_KNIGHTS_LANDING;
- break;
-
- default: // Unknown family 6 CPU, try to guess.
- if (Features & (1 << FEATURE_AVX512)) {
- *Type = INTEL_XEONPHI; // knl
- *Subtype = INTEL_KNIGHTS_LANDING;
- break;
- }
- if (Features & (1 << FEATURE_ADX)) {
- *Type = INTEL_COREI7;
- *Subtype = INTEL_COREI7_BROADWELL;
- break;
- }
- if (Features & (1 << FEATURE_AVX2)) {
- *Type = INTEL_COREI7;
- *Subtype = INTEL_COREI7_HASWELL;
- break;
- }
- if (Features & (1 << FEATURE_AVX)) {
- *Type = INTEL_COREI7;
- *Subtype = INTEL_COREI7_SANDYBRIDGE;
- break;
- }
- if (Features & (1 << FEATURE_SSE4_2)) {
- if (Features & (1 << FEATURE_MOVBE)) {
- *Type = INTEL_ATOM;
- *Subtype = INTEL_ATOM_SILVERMONT;
- } else {
- *Type = INTEL_COREI7;
- *Subtype = INTEL_COREI7_NEHALEM;
- }
- break;
- }
- if (Features & (1 << FEATURE_SSE4_1)) {
- *Type = INTEL_CORE2; // "penryn"
- *Subtype = INTEL_CORE2_45;
- break;
- }
- if (Features & (1 << FEATURE_SSSE3)) {
- if (Features & (1 << FEATURE_MOVBE)) {
- *Type = INTEL_ATOM;
- *Subtype = INTEL_ATOM_BONNELL; // "bonnell"
- } else {
- *Type = INTEL_CORE2; // "core2"
- *Subtype = INTEL_CORE2_65;
- }
- break;
- }
- if (Features & (1 << FEATURE_EM64T)) {
- *Type = INTEL_X86_64;
- break; // x86-64
- }
- if (Features & (1 << FEATURE_SSE2)) {
- *Type = INTEL_PENTIUM_M;
- break;
- }
- if (Features & (1 << FEATURE_SSE)) {
- *Type = INTEL_PENTIUM_III;
- break;
- }
- if (Features & (1 << FEATURE_MMX)) {
- *Type = INTEL_PENTIUM_II;
- break;
- }
- *Type = INTEL_PENTIUM_PRO;
- break;
- }
- case 15: {
- switch (Model) {
- case 0: // Pentium 4 processor, Intel Xeon processor. All processors are
- // model 00h and manufactured using the 0.18 micron process.
- case 1: // Pentium 4 processor, Intel Xeon processor, Intel Xeon
- // processor MP, and Intel Celeron processor. All processors are
- // model 01h and manufactured using the 0.18 micron process.
- case 2: // Pentium 4 processor, Mobile Intel Pentium 4 processor - M,
- // Intel Xeon processor, Intel Xeon processor MP, Intel Celeron
- // processor, and Mobile Intel Celeron processor. All processors
- // are model 02h and manufactured using the 0.13 micron process.
- *Type =
- ((Features & (1 << FEATURE_EM64T)) ? INTEL_X86_64 : INTEL_PENTIUM_IV);
+ *Type = INTEL_KNL; // knl
break;
- case 3: // Pentium 4 processor, Intel Xeon processor, Intel Celeron D
- // processor. All processors are model 03h and manufactured using
- // the 90 nm process.
- case 4: // Pentium 4 processor, Pentium 4 processor Extreme Edition,
- // Pentium D processor, Intel Xeon processor, Intel Xeon
- // processor MP, Intel Celeron D processor. All processors are
- // model 04h and manufactured using the 90 nm process.
- case 6: // Pentium 4 processor, Pentium D processor, Pentium processor
- // Extreme Edition, Intel Xeon processor, Intel Xeon processor
- // MP, Intel Celeron D processor. All processors are model 06h
- // and manufactured using the 65 nm process.
- *Type =
- ((Features & (1 << FEATURE_EM64T)) ? INTEL_NOCONA : INTEL_PRESCOTT);
- break;
-
- default:
- *Type =
- ((Features & (1 << FEATURE_EM64T)) ? INTEL_X86_64 : INTEL_PENTIUM_IV);
+ default: // Unknown family 6 CPU.
break;
+ break;
}
- }
default:
- break; /*"generic"*/
+ break; // Unknown.
}
}
-static void getAMDProcessorTypeAndSubtype(unsigned int Family,
- unsigned int Model,
- unsigned int Features, unsigned *Type,
+static void getAMDProcessorTypeAndSubtype(unsigned Family, unsigned Model,
+ unsigned Features, unsigned *Type,
unsigned *Subtype) {
// FIXME: this poorly matches the generated SubtargetFeatureKV table. There
// appears to be no way to generate the wide variety of AMD-specific targets
// from the information returned from CPUID.
switch (Family) {
- case 4:
- *Type = AMD_i486;
- case 5:
- *Type = AMDPENTIUM;
- switch (Model) {
- case 6:
- case 7:
- *Subtype = AMDPENTIUM_K6;
- break; // "k6"
- case 8:
- *Subtype = AMDPENTIUM_K62;
- break; // "k6-2"
- case 9:
- case 13:
- *Subtype = AMDPENTIUM_K63;
- break; // "k6-3"
- case 10:
- *Subtype = AMDPENTIUM_GEODE;
- break; // "geode"
- default:
- break;
- }
- case 6:
- *Type = AMDATHLON;
- switch (Model) {
- case 4:
- *Subtype = AMDATHLON_TBIRD;
- break; // "athlon-tbird"
- case 6:
- case 7:
- case 8:
- *Subtype = AMDATHLON_MP;
- break; // "athlon-mp"
- case 10:
- *Subtype = AMDATHLON_XP;
- break; // "athlon-xp"
- default:
- break;
- }
- case 15:
- *Type = AMDATHLON;
- if (Features & (1 << FEATURE_SSE3)) {
- *Subtype = AMDATHLON_K8SSE3;
- break; // "k8-sse3"
- }
- switch (Model) {
- case 1:
- *Subtype = AMDATHLON_OPTERON;
- break; // "opteron"
- case 5:
- *Subtype = AMDATHLON_FX;
- break; // "athlon-fx"; also opteron
- default:
- *Subtype = AMDATHLON_64;
- break; // "athlon64"
- }
case 16:
*Type = AMDFAM10H; // "amdfam10"
switch (Model) {
@@ -643,23 +408,16 @@ static void getAMDProcessorTypeAndSubtype(unsigned int Family,
case 8:
*Subtype = AMDFAM10H_ISTANBUL;
break;
- default:
- break;
}
+ break;
case 20:
- *Type = AMDFAM14H;
- *Subtype = AMD_BTVER1;
+ *Type = AMD_BTVER1;
break; // "btver1";
case 21:
*Type = AMDFAM15H;
- if (!(Features &
- (1 << FEATURE_AVX))) { // If no AVX support, provide a sane fallback.
- *Subtype = AMD_BTVER1;
- break; // "btver1"
- }
- if (Model >= 0x50 && Model <= 0x6f) {
+ if (Model >= 0x60 && Model <= 0x7f) {
*Subtype = AMDFAM15H_BDVER4;
- break; // "bdver4"; 50h-6Fh: Excavator
+ break; // "bdver4"; 60h-7Fh: Excavator
}
if (Model >= 0x30 && Model <= 0x3f) {
*Subtype = AMDFAM15H_BDVER3;
@@ -675,31 +433,47 @@ static void getAMDProcessorTypeAndSubtype(unsigned int Family,
}
break;
case 22:
- *Type = AMDFAM16H;
- if (!(Features &
- (1 << FEATURE_AVX))) { // If no AVX support provide a sane fallback.
- *Subtype = AMD_BTVER1;
- break; // "btver1";
- }
- *Subtype = AMD_BTVER2;
+ *Type = AMD_BTVER2;
break; // "btver2"
+ case 23:
+ *Type = AMDFAM17H;
+ *Subtype = AMDFAM17H_ZNVER1;
+ break;
default:
break; // "generic"
}
}
-static unsigned getAvailableFeatures(unsigned int ECX, unsigned int EDX,
- unsigned MaxLeaf) {
+static void getAvailableFeatures(unsigned ECX, unsigned EDX, unsigned MaxLeaf,
+ unsigned *FeaturesOut) {
unsigned Features = 0;
- unsigned int EAX, EBX;
- Features |= (((EDX >> 23) & 1) << FEATURE_MMX);
- Features |= (((EDX >> 25) & 1) << FEATURE_SSE);
- Features |= (((EDX >> 26) & 1) << FEATURE_SSE2);
- Features |= (((ECX >> 0) & 1) << FEATURE_SSE3);
- Features |= (((ECX >> 9) & 1) << FEATURE_SSSE3);
- Features |= (((ECX >> 19) & 1) << FEATURE_SSE4_1);
- Features |= (((ECX >> 20) & 1) << FEATURE_SSE4_2);
- Features |= (((ECX >> 22) & 1) << FEATURE_MOVBE);
+ unsigned EAX, EBX;
+
+ if ((EDX >> 15) & 1)
+ Features |= 1 << FEATURE_CMOV;
+ if ((EDX >> 23) & 1)
+ Features |= 1 << FEATURE_MMX;
+ if ((EDX >> 25) & 1)
+ Features |= 1 << FEATURE_SSE;
+ if ((EDX >> 26) & 1)
+ Features |= 1 << FEATURE_SSE2;
+
+ if ((ECX >> 0) & 1)
+ Features |= 1 << FEATURE_SSE3;
+ if ((ECX >> 1) & 1)
+ Features |= 1 << FEATURE_PCLMUL;
+ if ((ECX >> 9) & 1)
+ Features |= 1 << FEATURE_SSSE3;
+ if ((ECX >> 12) & 1)
+ Features |= 1 << FEATURE_FMA;
+ if ((ECX >> 19) & 1)
+ Features |= 1 << FEATURE_SSE4_1;
+ if ((ECX >> 20) & 1)
+ Features |= 1 << FEATURE_SSE4_2;
+ if ((ECX >> 23) & 1)
+ Features |= 1 << FEATURE_POPCNT;
+ if ((ECX >> 25) & 1)
+ Features |= 1 << FEATURE_AES;
// If CPUID indicates support for XSAVE, XRESTORE and AVX, and XGETBV
// indicates that the AVX registers will be saved and restored on context
@@ -708,20 +482,59 @@ static unsigned getAvailableFeatures(unsigned int ECX, unsigned int EDX,
bool HasAVX = ((ECX & AVXBits) == AVXBits) && !getX86XCR0(&EAX, &EDX) &&
((EAX & 0x6) == 0x6);
bool HasAVX512Save = HasAVX && ((EAX & 0xe0) == 0xe0);
- bool HasLeaf7 = MaxLeaf >= 0x7;
- getX86CpuIDAndInfoEx(0x7, 0x0, &EAX, &EBX, &ECX, &EDX);
- bool HasADX = HasLeaf7 && ((EBX >> 19) & 1);
- bool HasAVX2 = HasAVX && HasLeaf7 && (EBX & 0x20);
- bool HasAVX512 = HasLeaf7 && HasAVX512Save && ((EBX >> 16) & 1);
- Features |= (HasAVX << FEATURE_AVX);
- Features |= (HasAVX2 << FEATURE_AVX2);
- Features |= (HasAVX512 << FEATURE_AVX512);
- Features |= (HasAVX512Save << FEATURE_AVX512SAVE);
- Features |= (HasADX << FEATURE_ADX);
-
- getX86CpuIDAndInfo(0x80000001, &EAX, &EBX, &ECX, &EDX);
- Features |= (((EDX >> 29) & 0x1) << FEATURE_EM64T);
- return Features;
+
+ if (HasAVX)
+ Features |= 1 << FEATURE_AVX;
+
+ bool HasLeaf7 =
+ MaxLeaf >= 0x7 && !getX86CpuIDAndInfoEx(0x7, 0x0, &EAX, &EBX, &ECX, &EDX);
+
+ if (HasLeaf7 && ((EBX >> 3) & 1))
+ Features |= 1 << FEATURE_BMI;
+ if (HasLeaf7 && ((EBX >> 5) & 1) && HasAVX)
+ Features |= 1 << FEATURE_AVX2;
+ if (HasLeaf7 && ((EBX >> 9) & 1))
+ Features |= 1 << FEATURE_BMI2;
+ if (HasLeaf7 && ((EBX >> 16) & 1) && HasAVX512Save)
+ Features |= 1 << FEATURE_AVX512F;
+ if (HasLeaf7 && ((EBX >> 17) & 1) && HasAVX512Save)
+ Features |= 1 << FEATURE_AVX512DQ;
+ if (HasLeaf7 && ((EBX >> 21) & 1) && HasAVX512Save)
+ Features |= 1 << FEATURE_AVX512IFMA;
+ if (HasLeaf7 && ((EBX >> 26) & 1) && HasAVX512Save)
+ Features |= 1 << FEATURE_AVX512PF;
+ if (HasLeaf7 && ((EBX >> 27) & 1) && HasAVX512Save)
+ Features |= 1 << FEATURE_AVX512ER;
+ if (HasLeaf7 && ((EBX >> 28) & 1) && HasAVX512Save)
+ Features |= 1 << FEATURE_AVX512CD;
+ if (HasLeaf7 && ((EBX >> 30) & 1) && HasAVX512Save)
+ Features |= 1 << FEATURE_AVX512BW;
+ if (HasLeaf7 && ((EBX >> 31) & 1) && HasAVX512Save)
+ Features |= 1 << FEATURE_AVX512VL;
+
+ if (HasLeaf7 && ((ECX >> 1) & 1) && HasAVX512Save)
+ Features |= 1 << FEATURE_AVX512VBMI;
+ if (HasLeaf7 && ((ECX >> 14) & 1) && HasAVX512Save)
+ Features |= 1 << FEATURE_AVX512VPOPCNTDQ;
+
+ if (HasLeaf7 && ((EDX >> 2) & 1) && HasAVX512Save)
+ Features |= 1 << FEATURE_AVX5124VNNIW;
+ if (HasLeaf7 && ((EDX >> 3) & 1) && HasAVX512Save)
+ Features |= 1 << FEATURE_AVX5124FMAPS;
+
+ unsigned MaxExtLevel;
+ getX86CpuIDAndInfo(0x80000000, &MaxExtLevel, &EBX, &ECX, &EDX);
+
+ bool HasExtLeaf1 = MaxExtLevel >= 0x80000001 &&
+ !getX86CpuIDAndInfo(0x80000001, &EAX, &EBX, &ECX, &EDX);
+ if (HasExtLeaf1 && ((ECX >> 6) & 1))
+ Features |= 1 << FEATURE_SSE4_A;
+ if (HasExtLeaf1 && ((ECX >> 11) & 1))
+ Features |= 1 << FEATURE_XOP;
+ if (HasExtLeaf1 && ((ECX >> 16) & 1))
+ Features |= 1 << FEATURE_FMA4;
+
+ *FeaturesOut = Features;
}
#if defined(HAVE_INIT_PRIORITY)
@@ -751,11 +564,11 @@ struct __processor_model {
int CONSTRUCTOR_ATTRIBUTE
__cpu_indicator_init(void) {
- unsigned int EAX, EBX, ECX, EDX;
- unsigned int MaxLeaf = 5;
- unsigned int Vendor;
- unsigned int Model, Family, Brand_id;
- unsigned int Features = 0;
+ unsigned EAX, EBX, ECX, EDX;
+ unsigned MaxLeaf = 5;
+ unsigned Vendor;
+ unsigned Model, Family, Brand_id;
+ unsigned Features = 0;
/* This function needs to run just once. */
if (__cpu_model.__cpu_vendor)
@@ -765,9 +578,7 @@ __cpu_indicator_init(void) {
return -1;
/* Assume cpuid insn present. Run in level 0 to get vendor id. */
- getX86CpuIDAndInfo(0, &MaxLeaf, &Vendor, &ECX, &EDX);
-
- if (MaxLeaf < 1) {
+ if (getX86CpuIDAndInfo(0, &MaxLeaf, &Vendor, &ECX, &EDX) || MaxLeaf < 1) {
__cpu_model.__cpu_vendor = VENDOR_OTHER;
return -1;
}
@@ -776,7 +587,7 @@ __cpu_indicator_init(void) {
Brand_id = EBX & 0xff;
/* Find available features. */
- Features = getAvailableFeatures(ECX, EDX, MaxLeaf);
+ getAvailableFeatures(ECX, EDX, MaxLeaf, &Features);
__cpu_model.__cpu_features[0] = Features;
if (Vendor == SIG_INTEL) {
diff --git a/lib/builtins/int_util.c b/lib/builtins/int_util.c
index 420d1e237aae..de87410dbca2 100644
--- a/lib/builtins/int_util.c
+++ b/lib/builtins/int_util.c
@@ -45,6 +45,16 @@ void compilerrt_abort_impl(const char *file, int line, const char *function) {
__assert_rtn(function, file, line, "libcompiler_rt abort");
}
+#elif __Fuchsia__
+
+#ifndef _WIN32
+__attribute__((weak))
+__attribute__((visibility("hidden")))
+#endif
+void compilerrt_abort_impl(const char *file, int line, const char *function) {
+ __builtin_trap();
+}
+
#else
/* Get the system definition of abort() */
diff --git a/lib/esan/working_set.cpp b/lib/esan/working_set.cpp
index f39111993c33..e56902c8f32a 100644
--- a/lib/esan/working_set.cpp
+++ b/lib/esan/working_set.cpp
@@ -160,15 +160,16 @@ static u32 countAndClearShadowValues(u32 BitIdx, uptr ShadowStart,
static u32 computeWorkingSizeAndReset(u32 BitIdx) {
u32 WorkingSetSize = 0;
MemoryMappingLayout MemIter(true/*cache*/);
- uptr Start, End, Prot;
- while (MemIter.Next(&Start, &End, nullptr/*offs*/, nullptr/*file*/,
- 0/*file size*/, &Prot)) {
- VPrintf(4, "%s: considering %p-%p app=%d shadow=%d prot=%u\n",
- __FUNCTION__, Start, End, Prot, isAppMem(Start),
- isShadowMem(Start));
- if (isShadowMem(Start) && (Prot & MemoryMappingLayout::kProtectionWrite)) {
- VPrintf(3, "%s: walking %p-%p\n", __FUNCTION__, Start, End);
- WorkingSetSize += countAndClearShadowValues(BitIdx, Start, End);
+ MemoryMappedSegment Segment;
+ while (MemIter.Next(&Segment)) {
+ VPrintf(4, "%s: considering %p-%p app=%d shadow=%d prot=%u\n", __FUNCTION__,
+ Segment.start, Segment.end, Segment.protection,
+ isAppMem(Segment.start), isShadowMem(Segment.start));
+ if (isShadowMem(Segment.start) && Segment.IsWritable()) {
+ VPrintf(3, "%s: walking %p-%p\n", __FUNCTION__, Segment.start,
+ Segment.end);
+ WorkingSetSize +=
+ countAndClearShadowValues(BitIdx, Segment.start, Segment.end);
}
}
return WorkingSetSize;
diff --git a/lib/lsan/lsan_common.cc b/lib/lsan/lsan_common.cc
index a5ffc6835f5f..4ffa91568cc8 100644
--- a/lib/lsan/lsan_common.cc
+++ b/lib/lsan/lsan_common.cc
@@ -74,6 +74,10 @@ static const char kStdSuppressions[] =
// definition.
"leak:*pthread_exit*\n"
#endif // SANITIZER_SUPPRESS_LEAK_ON_PTHREAD_EXIT
+#if SANITIZER_MAC
+ // For Darwin and os_log/os_trace: https://reviews.llvm.org/D35173
+ "leak:*_os_trace*\n"
+#endif
// TLS leak in some glibc versions, described in
// https://sourceware.org/bugzilla/show_bug.cgi?id=12650.
"leak:*tls_get_addr*\n";
@@ -301,11 +305,10 @@ static void ProcessThreads(SuspendedThreadsList const &suspended_threads,
}
void ScanRootRegion(Frontier *frontier, const RootRegion &root_region,
- uptr region_begin, uptr region_end, uptr prot) {
+ uptr region_begin, uptr region_end, bool is_readable) {
uptr intersection_begin = Max(root_region.begin, region_begin);
uptr intersection_end = Min(region_end, root_region.begin + root_region.size);
if (intersection_begin >= intersection_end) return;
- bool is_readable = prot & MemoryMappingLayout::kProtectionRead;
LOG_POINTERS("Root region %p-%p intersects with mapped region %p-%p (%s)\n",
root_region.begin, root_region.begin + root_region.size,
region_begin, region_end,
@@ -318,11 +321,10 @@ void ScanRootRegion(Frontier *frontier, const RootRegion &root_region,
static void ProcessRootRegion(Frontier *frontier,
const RootRegion &root_region) {
MemoryMappingLayout proc_maps(/*cache_enabled*/ true);
- uptr begin, end, prot;
- while (proc_maps.Next(&begin, &end,
- /*offset*/ nullptr, /*filename*/ nullptr,
- /*filename_size*/ 0, &prot)) {
- ScanRootRegion(frontier, root_region, begin, end, prot);
+ MemoryMappedSegment segment;
+ while (proc_maps.Next(&segment)) {
+ ScanRootRegion(frontier, root_region, segment.start, segment.end,
+ segment.IsReadable());
}
}
diff --git a/lib/lsan/lsan_common.h b/lib/lsan/lsan_common.h
index beb31d6f40e4..d93ac1b10919 100644
--- a/lib/lsan/lsan_common.h
+++ b/lib/lsan/lsan_common.h
@@ -127,7 +127,7 @@ struct RootRegion {
InternalMmapVector<RootRegion> const *GetRootRegions();
void ScanRootRegion(Frontier *frontier, RootRegion const &region,
- uptr region_begin, uptr region_end, uptr prot);
+ uptr region_begin, uptr region_end, bool is_readable);
// Run stoptheworld while holding any platform-specific locks.
void DoStopTheWorld(StopTheWorldCallback callback, void* argument);
diff --git a/lib/lsan/lsan_common_mac.cc b/lib/lsan/lsan_common_mac.cc
index adde3a1b4035..f87c6b7e0425 100644
--- a/lib/lsan/lsan_common_mac.cc
+++ b/lib/lsan/lsan_common_mac.cc
@@ -156,7 +156,7 @@ void ProcessPlatformSpecificAllocations(Frontier *frontier) {
if (flags()->use_root_regions) {
for (uptr i = 0; i < root_regions->size(); i++) {
ScanRootRegion(frontier, (*root_regions)[i], address, end_address,
- info.protection);
+ info.protection & kProtectionRead);
}
}
diff --git a/lib/msan/msan_interceptors.cc b/lib/msan/msan_interceptors.cc
index ce8444a3bb2f..069777c7f5e7 100644
--- a/lib/msan/msan_interceptors.cc
+++ b/lib/msan/msan_interceptors.cc
@@ -27,6 +27,7 @@
#include "sanitizer_common/sanitizer_allocator_internal.h"
#include "sanitizer_common/sanitizer_atomic.h"
#include "sanitizer_common/sanitizer_common.h"
+#include "sanitizer_common/sanitizer_errno.h"
#include "sanitizer_common/sanitizer_stackdepot.h"
#include "sanitizer_common/sanitizer_libc.h"
#include "sanitizer_common/sanitizer_linux.h"
@@ -48,15 +49,9 @@ DECLARE_REAL(SIZE_T, strnlen, const char *s, SIZE_T maxlen)
DECLARE_REAL(void *, memcpy, void *dest, const void *src, uptr n)
DECLARE_REAL(void *, memset, void *dest, int c, uptr n)
-#if SANITIZER_FREEBSD
-#define __errno_location __error
-#endif
-
// True if this is a nested interceptor.
static THREADLOCAL int in_interceptor_scope;
-extern "C" int *__errno_location(void);
-
struct InterceptorScope {
InterceptorScope() { ++in_interceptor_scope; }
~InterceptorScope() { --in_interceptor_scope; }
@@ -915,7 +910,7 @@ INTERCEPTOR(void *, mmap, void *addr, SIZE_T length, int prot, int flags,
ENSURE_MSAN_INITED();
if (addr && !MEM_IS_APP(addr)) {
if (flags & map_fixed) {
- *__errno_location() = errno_EINVAL;
+ errno = errno_EINVAL;
return (void *)-1;
} else {
addr = nullptr;
@@ -933,7 +928,7 @@ INTERCEPTOR(void *, mmap64, void *addr, SIZE_T length, int prot, int flags,
ENSURE_MSAN_INITED();
if (addr && !MEM_IS_APP(addr)) {
if (flags & map_fixed) {
- *__errno_location() = errno_EINVAL;
+ errno = errno_EINVAL;
return (void *)-1;
} else {
addr = nullptr;
diff --git a/lib/sanitizer_common/CMakeLists.txt b/lib/sanitizer_common/CMakeLists.txt
index bf8459ef5e91..a17bd1299a2c 100644
--- a/lib/sanitizer_common/CMakeLists.txt
+++ b/lib/sanitizer_common/CMakeLists.txt
@@ -6,6 +6,7 @@ set(SANITIZER_SOURCES_NOTERMINATION
sanitizer_common.cc
sanitizer_deadlock_detector1.cc
sanitizer_deadlock_detector2.cc
+ sanitizer_errno.cc
sanitizer_flags.cc
sanitizer_flag_parser.cc
sanitizer_libc.cc
@@ -57,6 +58,7 @@ set(SANITIZER_LIBCDEP_SOURCES
sanitizer_coverage_libcdep_new.cc
sanitizer_coverage_win_sections.cc
sanitizer_linux_libcdep.cc
+ sanitizer_mac_libcdep.cc
sanitizer_posix_libcdep.cc
sanitizer_stacktrace_libcdep.cc
sanitizer_stoptheworld_linux_libcdep.cc
@@ -92,6 +94,8 @@ set(SANITIZER_HEADERS
sanitizer_common_syscalls.inc
sanitizer_deadlock_detector.h
sanitizer_deadlock_detector_interface.h
+ sanitizer_errno.h
+ sanitizer_errno_codes.h
sanitizer_flag_parser.h
sanitizer_flags.h
sanitizer_flags.inc
diff --git a/lib/sanitizer_common/sanitizer_common.h b/lib/sanitizer_common/sanitizer_common.h
index d44c71513896..89aae579856a 100644
--- a/lib/sanitizer_common/sanitizer_common.h
+++ b/lib/sanitizer_common/sanitizer_common.h
@@ -107,7 +107,8 @@ bool MprotectNoAccess(uptr addr, uptr size);
bool MprotectReadOnly(uptr addr, uptr size);
// Find an available address space.
-uptr FindAvailableMemoryRange(uptr size, uptr alignment, uptr left_padding);
+uptr FindAvailableMemoryRange(uptr size, uptr alignment, uptr left_padding,
+ uptr *largest_gap_found);
// Used to check if we can map shadow memory to a fixed location.
bool MemoryRangeIsAvailable(uptr range_start, uptr range_end);
diff --git a/lib/sanitizer_common/sanitizer_common_interceptors.inc b/lib/sanitizer_common/sanitizer_common_interceptors.inc
index 459530aa95ba..8607bf44902d 100644
--- a/lib/sanitizer_common/sanitizer_common_interceptors.inc
+++ b/lib/sanitizer_common/sanitizer_common_interceptors.inc
@@ -40,6 +40,7 @@
#include "interception/interception.h"
#include "sanitizer_addrhashmap.h"
+#include "sanitizer_errno.h"
#include "sanitizer_placement_new.h"
#include "sanitizer_platform_interceptors.h"
#include "sanitizer_tls_get_addr.h"
diff --git a/lib/sanitizer_common/sanitizer_errno.cc b/lib/sanitizer_common/sanitizer_errno.cc
new file mode 100644
index 000000000000..a6f9fc6125eb
--- /dev/null
+++ b/lib/sanitizer_common/sanitizer_errno.cc
@@ -0,0 +1,35 @@
+//===-- sanitizer_errno.cc --------------------------------------*- C++ -*-===//
+//
+// The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+//
+// This file is shared between sanitizers run-time libraries.
+//
+// Defines errno to avoid including errno.h and its dependencies into other
+// files (e.g. interceptors are not supposed to include any system headers).
+//
+//===----------------------------------------------------------------------===//
+
+#include "sanitizer_errno_codes.h"
+#include "sanitizer_internal_defs.h"
+
+#include <errno.h>
+
+namespace __sanitizer {
+
+COMPILER_CHECK(errno_ENOMEM == ENOMEM);
+COMPILER_CHECK(errno_EBUSY == EBUSY);
+COMPILER_CHECK(errno_EINVAL == EINVAL);
+
+// EOWNERDEAD is not present in some older platforms.
+#if defined(EOWNERDEAD)
+extern const int errno_EOWNERDEAD = EOWNERDEAD;
+#else
+extern const int errno_EOWNERDEAD = -1;
+#endif
+
+} // namespace __sanitizer
diff --git a/lib/sanitizer_common/sanitizer_errno.h b/lib/sanitizer_common/sanitizer_errno.h
new file mode 100644
index 000000000000..c405307ba8ec
--- /dev/null
+++ b/lib/sanitizer_common/sanitizer_errno.h
@@ -0,0 +1,35 @@
+//===-- sanitizer_errno.h ---------------------------------------*- C++ -*-===//
+//
+// The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+//
+// This file is shared between sanitizers run-time libraries.
+//
+// Defines errno to avoid including errno.h and its dependencies into sensitive
+// files (e.g. interceptors are not supposed to include any system headers).
+// It's ok to use errno.h directly when your file already depend on other system
+// includes though.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef SANITIZER_ERRNO_H
+#define SANITIZER_ERRNO_H
+
+#include "sanitizer_errno_codes.h"
+#include "sanitizer_platform.h"
+
+#if SANITIZER_FREEBSD || SANITIZER_MAC
+# define __errno_location __error
+#elif SANITIZER_ANDROID
+# define __errno_location __errno
+#endif
+
+extern "C" int *__errno_location();
+
+#define errno (*__errno_location())
+
+#endif // SANITIZER_ERRNO_H
diff --git a/lib/sanitizer_common/sanitizer_errno_codes.h b/lib/sanitizer_common/sanitizer_errno_codes.h
new file mode 100644
index 000000000000..dba774c5b6c5
--- /dev/null
+++ b/lib/sanitizer_common/sanitizer_errno_codes.h
@@ -0,0 +1,34 @@
+//===-- sanitizer_errno_codes.h ---------------------------------*- C++ -*-===//
+//
+// The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+//
+// This file is shared between sanitizers run-time libraries.
+//
+// Defines errno codes to avoid including errno.h and its dependencies into
+// sensitive files (e.g. interceptors are not supposed to include any system
+// headers).
+// It's ok to use errno.h directly when your file already depend on other system
+// includes though.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef SANITIZER_ERRNO_CODES_H
+#define SANITIZER_ERRNO_CODES_H
+
+namespace __sanitizer {
+
+#define errno_ENOMEM 12
+#define errno_EBUSY 16
+#define errno_EINVAL 22
+
+// Those might not present or their value differ on different platforms.
+extern const int errno_EOWNERDEAD;
+
+} // namespace __sanitizer
+
+#endif // SANITIZER_ERRNO_CODES_H
diff --git a/lib/sanitizer_common/sanitizer_linux.cc b/lib/sanitizer_common/sanitizer_linux.cc
index a94a63c7f16d..a79a2a155db9 100644
--- a/lib/sanitizer_common/sanitizer_linux.cc
+++ b/lib/sanitizer_common/sanitizer_linux.cc
@@ -59,6 +59,14 @@
#include <ucontext.h>
#include <unistd.h>
+#if SANITIZER_LINUX
+#include <sys/utsname.h>
+#endif
+
+#if SANITIZER_LINUX && !SANITIZER_ANDROID
+#include <sys/personality.h>
+#endif
+
#if SANITIZER_FREEBSD
#include <sys/exec.h>
#include <sys/sysctl.h>
@@ -209,7 +217,6 @@ static void stat64_to_stat(struct stat64 *in, struct stat *out) {
out->st_atime = in->st_atime;
out->st_mtime = in->st_mtime;
out->st_ctime = in->st_ctime;
- out->st_ino = in->st_ino;
}
#endif
@@ -229,7 +236,6 @@ static void kernel_stat_to_stat(struct kernel_stat *in, struct stat *out) {
out->st_atime = in->st_atime_nsec;
out->st_mtime = in->st_mtime_nsec;
out->st_ctime = in->st_ctime_nsec;
- out->st_ino = in->st_ino;
}
#endif
@@ -815,6 +821,72 @@ bool ThreadLister::GetDirectoryEntries() {
return true;
}
+#if SANITIZER_WORDSIZE == 32
+// Take care of unusable kernel area in top gigabyte.
+static uptr GetKernelAreaSize() {
+#if SANITIZER_LINUX && !SANITIZER_X32
+ const uptr gbyte = 1UL << 30;
+
+ // Firstly check if there are writable segments
+ // mapped to top gigabyte (e.g. stack).
+ MemoryMappingLayout proc_maps(/*cache_enabled*/true);
+ MemoryMappedSegment segment;
+ while (proc_maps.Next(&segment)) {
+ if ((segment.end >= 3 * gbyte) && segment.IsWritable()) return 0;
+ }
+
+#if !SANITIZER_ANDROID
+ // Even if nothing is mapped, top Gb may still be accessible
+ // if we are running on 64-bit kernel.
+ // Uname may report misleading results if personality type
+ // is modified (e.g. under schroot) so check this as well.
+ struct utsname uname_info;
+ int pers = personality(0xffffffffUL);
+ if (!(pers & PER_MASK)
+ && uname(&uname_info) == 0
+ && internal_strstr(uname_info.machine, "64"))
+ return 0;
+#endif // SANITIZER_ANDROID
+
+ // Top gigabyte is reserved for kernel.
+ return gbyte;
+#else
+ return 0;
+#endif // SANITIZER_LINUX && !SANITIZER_X32
+}
+#endif // SANITIZER_WORDSIZE == 32
+
+uptr GetMaxVirtualAddress() {
+#if SANITIZER_WORDSIZE == 64
+# if defined(__powerpc64__) || defined(__aarch64__)
+ // On PowerPC64 we have two different address space layouts: 44- and 46-bit.
+ // We somehow need to figure out which one we are using now and choose
+ // one of 0x00000fffffffffffUL and 0x00003fffffffffffUL.
+ // Note that with 'ulimit -s unlimited' the stack is moved away from the top
+ // of the address space, so simply checking the stack address is not enough.
+ // This should (does) work for both PowerPC64 Endian modes.
+ // Similarly, aarch64 has multiple address space layouts: 39, 42 and 47-bit.
+ return (1ULL << (MostSignificantSetBitIndex(GET_CURRENT_FRAME()) + 1)) - 1;
+# elif defined(__mips64)
+ return (1ULL << 40) - 1; // 0x000000ffffffffffUL;
+# elif defined(__s390x__)
+ return (1ULL << 53) - 1; // 0x001fffffffffffffUL;
+# else
+ return (1ULL << 47) - 1; // 0x00007fffffffffffUL;
+# endif
+#else // SANITIZER_WORDSIZE == 32
+# if defined(__s390__)
+ return (1ULL << 31) - 1; // 0x7fffffff;
+# else
+ uptr res = (1ULL << 32) - 1; // 0xffffffff;
+ if (!common_flags()->full_address_space)
+ res -= GetKernelAreaSize();
+ CHECK_LT(reinterpret_cast<uptr>(&res), res);
+ return res;
+# endif
+#endif // SANITIZER_WORDSIZE
+}
+
uptr GetPageSize() {
// Android post-M sysconf(_SC_PAGESIZE) crashes if called from .preinit_array.
#if SANITIZER_ANDROID
@@ -1599,7 +1671,8 @@ void CheckNoDeepBind(const char *filename, int flag) {
#endif
}
-uptr FindAvailableMemoryRange(uptr size, uptr alignment, uptr left_padding) {
+uptr FindAvailableMemoryRange(uptr size, uptr alignment, uptr left_padding,
+ uptr *largest_gap_found) {
UNREACHABLE("FindAvailableMemoryRange is not available");
return 0;
}
diff --git a/lib/sanitizer_common/sanitizer_linux_libcdep.cc b/lib/sanitizer_common/sanitizer_linux_libcdep.cc
index b9a48a1e496b..52196db12731 100644
--- a/lib/sanitizer_common/sanitizer_linux_libcdep.cc
+++ b/lib/sanitizer_common/sanitizer_linux_libcdep.cc
@@ -81,28 +81,25 @@ void GetThreadStackTopAndBottom(bool at_initialization, uptr *stack_top,
// Find the mapping that contains a stack variable.
MemoryMappingLayout proc_maps(/*cache_enabled*/true);
- uptr start, end, offset;
+ MemoryMappedSegment segment;
uptr prev_end = 0;
- while (proc_maps.Next(&start, &end, &offset, nullptr, 0,
- /* protection */nullptr)) {
- if ((uptr)&rl < end)
- break;
- prev_end = end;
+ while (proc_maps.Next(&segment)) {
+ if ((uptr)&rl < segment.end) break;
+ prev_end = segment.end;
}
- CHECK((uptr)&rl >= start && (uptr)&rl < end);
+ CHECK((uptr)&rl >= segment.start && (uptr)&rl < segment.end);
// Get stacksize from rlimit, but clip it so that it does not overlap
// with other mappings.
uptr stacksize = rl.rlim_cur;
- if (stacksize > end - prev_end)
- stacksize = end - prev_end;
+ if (stacksize > segment.end - prev_end) stacksize = segment.end - prev_end;
// When running with unlimited stack size, we still want to set some limit.
// The unlimited stack size is caused by 'ulimit -s unlimited'.
// Also, for some reason, GNU make spawns subprocesses with unlimited stack.
if (stacksize > kMaxThreadStackSize)
stacksize = kMaxThreadStackSize;
- *stack_top = end;
- *stack_bottom = end - stacksize;
+ *stack_top = segment.end;
+ *stack_bottom = segment.end - stacksize;
return;
}
pthread_attr_t attr;
diff --git a/lib/sanitizer_common/sanitizer_mac.cc b/lib/sanitizer_common/sanitizer_mac.cc
index b48238106dd9..8df01815f9f7 100644
--- a/lib/sanitizer_common/sanitizer_mac.cc
+++ b/lib/sanitizer_common/sanitizer_mac.cc
@@ -191,7 +191,8 @@ void internal_sigfillset(__sanitizer_sigset_t *set) { sigfillset(set); }
uptr internal_sigprocmask(int how, __sanitizer_sigset_t *set,
__sanitizer_sigset_t *oldset) {
- return sigprocmask(how, set, oldset);
+ // Don't use sigprocmask here, because it affects all threads.
+ return pthread_sigmask(how, set, oldset);
}
// Doesn't call pthread_atfork() handlers (but not available on 10.6).
@@ -799,9 +800,48 @@ char **GetArgv() {
return *_NSGetArgv();
}
+#if defined(__aarch64__) && SANITIZER_IOS && !SANITIZER_IOSSIM
+// The task_vm_info struct is normally provided by the macOS SDK, but we need
+// fields only available in 10.12+. Declare the struct manually to be able to
+// build against older SDKs.
+struct __sanitizer_task_vm_info {
+ uptr _unused[(SANITIZER_WORDSIZE == 32) ? 20 : 19];
+ uptr min_address;
+ uptr max_address;
+};
+
+uptr GetTaskInfoMaxAddress() {
+ __sanitizer_task_vm_info vm_info = {{0}, 0, 0};
+ mach_msg_type_number_t count = sizeof(vm_info) / sizeof(int);
+ int err = task_info(mach_task_self(), TASK_VM_INFO, (int *)&vm_info, &count);
+ if (err == 0) {
+ return vm_info.max_address - 1;
+ } else {
+ // xnu cannot provide vm address limit
+ return 0x200000000 - 1;
+ }
+}
+#endif
+
+uptr GetMaxVirtualAddress() {
+#if SANITIZER_WORDSIZE == 64
+# if defined(__aarch64__) && SANITIZER_IOS && !SANITIZER_IOSSIM
+ // Get the maximum VM address
+ static uptr max_vm = GetTaskInfoMaxAddress();
+ CHECK(max_vm);
+ return max_vm;
+# else
+ return (1ULL << 47) - 1; // 0x00007fffffffffffUL;
+# endif
+#else // SANITIZER_WORDSIZE == 32
+ return (1ULL << 32) - 1; // 0xffffffff;
+#endif // SANITIZER_WORDSIZE
+}
+
uptr FindAvailableMemoryRange(uptr shadow_size,
uptr alignment,
- uptr left_padding) {
+ uptr left_padding,
+ uptr *largest_gap_found) {
typedef vm_region_submap_short_info_data_64_t RegionInfo;
enum { kRegionInfoSize = VM_REGION_SUBMAP_SHORT_INFO_COUNT_64 };
// Start searching for available memory region past PAGEZERO, which is
@@ -812,6 +852,7 @@ uptr FindAvailableMemoryRange(uptr shadow_size,
mach_vm_address_t address = start_address;
mach_vm_address_t free_begin = start_address;
kern_return_t kr = KERN_SUCCESS;
+ if (largest_gap_found) *largest_gap_found = 0;
while (kr == KERN_SUCCESS) {
mach_vm_size_t vmsize = 0;
natural_t depth = 0;
@@ -821,10 +862,15 @@ uptr FindAvailableMemoryRange(uptr shadow_size,
(vm_region_info_t)&vminfo, &count);
if (free_begin != address) {
// We found a free region [free_begin..address-1].
- uptr shadow_address = RoundUpTo((uptr)free_begin + left_padding,
- alignment);
- if (shadow_address + shadow_size < (uptr)address) {
- return shadow_address;
+ uptr gap_start = RoundUpTo((uptr)free_begin + left_padding, alignment);
+ uptr gap_end = RoundDownTo((uptr)address, alignment);
+ uptr gap_size = gap_end > gap_start ? gap_end - gap_start : 0;
+ if (shadow_size < gap_size) {
+ return gap_start;
+ }
+
+ if (largest_gap_found && *largest_gap_found < gap_size) {
+ *largest_gap_found = gap_size;
}
}
// Move to the next region.
diff --git a/lib/sanitizer_common/sanitizer_mac.h b/lib/sanitizer_common/sanitizer_mac.h
index 636d9bfeac8c..3f1c68c8610a 100644
--- a/lib/sanitizer_common/sanitizer_mac.h
+++ b/lib/sanitizer_common/sanitizer_mac.h
@@ -36,6 +36,8 @@ MacosVersion GetMacosVersion();
char **GetEnviron();
+void RestrictMemoryToMaxAddress(uptr max_address);
+
} // namespace __sanitizer
extern "C" {
diff --git a/lib/sanitizer_common/sanitizer_mac_libcdep.cc b/lib/sanitizer_common/sanitizer_mac_libcdep.cc
new file mode 100644
index 000000000000..c95daa9372ad
--- /dev/null
+++ b/lib/sanitizer_common/sanitizer_mac_libcdep.cc
@@ -0,0 +1,30 @@
+//===-- sanitizer_mac_libcdep.cc ------------------------------------------===//
+//
+// The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+//
+// This file is shared between various sanitizers' runtime libraries and
+// implements OSX-specific functions.
+//===----------------------------------------------------------------------===//
+
+#include "sanitizer_platform.h"
+#if SANITIZER_MAC
+#include "sanitizer_mac.h"
+
+#include <sys/mman.h>
+
+namespace __sanitizer {
+
+void RestrictMemoryToMaxAddress(uptr max_address) {
+ uptr size_to_mmap = GetMaxVirtualAddress() + 1 - max_address;
+ void *res = MmapFixedNoAccess(max_address, size_to_mmap, "high gap");
+ CHECK(res != MAP_FAILED);
+}
+
+} // namespace __sanitizer
+
+#endif // SANITIZER_MAC
diff --git a/lib/sanitizer_common/sanitizer_platform_limits_posix.cc b/lib/sanitizer_common/sanitizer_platform_limits_posix.cc
index 683f019d70c3..83f4fd22f623 100644
--- a/lib/sanitizer_common/sanitizer_platform_limits_posix.cc
+++ b/lib/sanitizer_common/sanitizer_platform_limits_posix.cc
@@ -25,7 +25,6 @@
#endif
#include <arpa/inet.h>
#include <dirent.h>
-#include <errno.h>
#include <grp.h>
#include <limits.h>
#include <net/if.h>
@@ -931,14 +930,6 @@ unsigned struct_ElfW_Phdr_sz = sizeof(Elf_Phdr);
unsigned IOCTL_SNDCTL_DSP_GETOSPACE = SNDCTL_DSP_GETOSPACE;
#endif // (SANITIZER_LINUX || SANITIZER_FREEBSD) && !SANITIZER_ANDROID
- const int errno_EINVAL = EINVAL;
-// EOWNERDEAD is not present in some older platforms.
-#if defined(EOWNERDEAD)
- const int errno_EOWNERDEAD = EOWNERDEAD;
-#else
- const int errno_EOWNERDEAD = -1;
-#endif
-
const int si_SEGV_MAPERR = SEGV_MAPERR;
const int si_SEGV_ACCERR = SEGV_ACCERR;
} // namespace __sanitizer
diff --git a/lib/sanitizer_common/sanitizer_platform_limits_posix.h b/lib/sanitizer_common/sanitizer_platform_limits_posix.h
index 24ffcd7d94f4..63dcd2a6d683 100644
--- a/lib/sanitizer_common/sanitizer_platform_limits_posix.h
+++ b/lib/sanitizer_common/sanitizer_platform_limits_posix.h
@@ -1464,9 +1464,6 @@ struct __sanitizer_cookie_io_functions_t {
extern unsigned IOCTL_PIO_SCRNMAP;
#endif
- extern const int errno_EINVAL;
- extern const int errno_EOWNERDEAD;
-
extern const int si_SEGV_MAPERR;
extern const int si_SEGV_ACCERR;
} // namespace __sanitizer
diff --git a/lib/sanitizer_common/sanitizer_posix.cc b/lib/sanitizer_common/sanitizer_posix.cc
index 63f1bf713b24..8d3128ae199d 100644
--- a/lib/sanitizer_common/sanitizer_posix.cc
+++ b/lib/sanitizer_common/sanitizer_posix.cc
@@ -27,14 +27,6 @@
#include <signal.h>
#include <sys/mman.h>
-#if SANITIZER_LINUX
-#include <sys/utsname.h>
-#endif
-
-#if SANITIZER_LINUX && !SANITIZER_ANDROID
-#include <sys/personality.h>
-#endif
-
#if SANITIZER_FREEBSD
// The MAP_NORESERVE define has been removed in FreeBSD 11.x, and even before
// that, it was never implemented. So just define it to zero.
@@ -49,80 +41,6 @@ uptr GetMmapGranularity() {
return GetPageSize();
}
-#if SANITIZER_WORDSIZE == 32
-// Take care of unusable kernel area in top gigabyte.
-static uptr GetKernelAreaSize() {
-#if SANITIZER_LINUX && !SANITIZER_X32
- const uptr gbyte = 1UL << 30;
-
- // Firstly check if there are writable segments
- // mapped to top gigabyte (e.g. stack).
- MemoryMappingLayout proc_maps(/*cache_enabled*/true);
- uptr end, prot;
- while (proc_maps.Next(/*start*/nullptr, &end,
- /*offset*/nullptr, /*filename*/nullptr,
- /*filename_size*/0, &prot)) {
- if ((end >= 3 * gbyte)
- && (prot & MemoryMappingLayout::kProtectionWrite) != 0)
- return 0;
- }
-
-#if !SANITIZER_ANDROID
- // Even if nothing is mapped, top Gb may still be accessible
- // if we are running on 64-bit kernel.
- // Uname may report misleading results if personality type
- // is modified (e.g. under schroot) so check this as well.
- struct utsname uname_info;
- int pers = personality(0xffffffffUL);
- if (!(pers & PER_MASK)
- && uname(&uname_info) == 0
- && internal_strstr(uname_info.machine, "64"))
- return 0;
-#endif // SANITIZER_ANDROID
-
- // Top gigabyte is reserved for kernel.
- return gbyte;
-#else
- return 0;
-#endif // SANITIZER_LINUX && !SANITIZER_X32
-}
-#endif // SANITIZER_WORDSIZE == 32
-
-uptr GetMaxVirtualAddress() {
-#if SANITIZER_WORDSIZE == 64
-# if defined(__aarch64__) && SANITIZER_IOS && !SANITIZER_IOSSIM
- // Ideally, we would derive the upper bound from MACH_VM_MAX_ADDRESS. The
- // upper bound can change depending on the device.
- return 0x200000000 - 1;
-# elif defined(__powerpc64__) || defined(__aarch64__)
- // On PowerPC64 we have two different address space layouts: 44- and 46-bit.
- // We somehow need to figure out which one we are using now and choose
- // one of 0x00000fffffffffffUL and 0x00003fffffffffffUL.
- // Note that with 'ulimit -s unlimited' the stack is moved away from the top
- // of the address space, so simply checking the stack address is not enough.
- // This should (does) work for both PowerPC64 Endian modes.
- // Similarly, aarch64 has multiple address space layouts: 39, 42 and 47-bit.
- return (1ULL << (MostSignificantSetBitIndex(GET_CURRENT_FRAME()) + 1)) - 1;
-# elif defined(__mips64)
- return (1ULL << 40) - 1; // 0x000000ffffffffffUL;
-# elif defined(__s390x__)
- return (1ULL << 53) - 1; // 0x001fffffffffffffUL;
-# else
- return (1ULL << 47) - 1; // 0x00007fffffffffffUL;
-# endif
-#else // SANITIZER_WORDSIZE == 32
-# if defined(__s390__)
- return (1ULL << 31) - 1; // 0x7fffffff;
-# else
- uptr res = (1ULL << 32) - 1; // 0xffffffff;
- if (!common_flags()->full_address_space)
- res -= GetKernelAreaSize();
- CHECK_LT(reinterpret_cast<uptr>(&res), res);
- return res;
-# endif
-#endif // SANITIZER_WORDSIZE
-}
-
void *MmapOrDie(uptr size, const char *mem_type, bool raw_report) {
size = RoundUpTo(size, GetPageSizeCached());
uptr res = internal_mmap(nullptr, size,
@@ -162,7 +80,7 @@ void *MmapOrDieOnFatalError(uptr size, const char *mem_type) {
}
// We want to map a chunk of address space aligned to 'alignment'.
-// We do it by maping a bit more and then unmaping redundant pieces.
+// We do it by mapping a bit more and then unmapping redundant pieces.
// We probably can do it with fewer syscalls in some OS-dependent way.
void *MmapAlignedOrDieOnFatalError(uptr size, uptr alignment,
const char *mem_type) {
@@ -313,13 +231,12 @@ static inline bool IntervalsAreSeparate(uptr start1, uptr end1,
// memory).
bool MemoryRangeIsAvailable(uptr range_start, uptr range_end) {
MemoryMappingLayout proc_maps(/*cache_enabled*/true);
- uptr start, end;
- while (proc_maps.Next(&start, &end,
- /*offset*/nullptr, /*filename*/nullptr,
- /*filename_size*/0, /*protection*/nullptr)) {
- if (start == end) continue; // Empty range.
- CHECK_NE(0, end);
- if (!IntervalsAreSeparate(start, end - 1, range_start, range_end))
+ MemoryMappedSegment segment;
+ while (proc_maps.Next(&segment)) {
+ if (segment.start == segment.end) continue; // Empty range.
+ CHECK_NE(0, segment.end);
+ if (!IntervalsAreSeparate(segment.start, segment.end - 1, range_start,
+ range_end))
return false;
}
return true;
@@ -327,13 +244,13 @@ bool MemoryRangeIsAvailable(uptr range_start, uptr range_end) {
void DumpProcessMap() {
MemoryMappingLayout proc_maps(/*cache_enabled*/true);
- uptr start, end;
const sptr kBufSize = 4095;
char *filename = (char*)MmapOrDie(kBufSize, __func__);
+ MemoryMappedSegment segment(filename, kBufSize);
Report("Process memory map follows:\n");
- while (proc_maps.Next(&start, &end, /* file_offset */nullptr,
- filename, kBufSize, /* protection */nullptr)) {
- Printf("\t%p-%p\t%s\n", (void*)start, (void*)end, filename);
+ while (proc_maps.Next(&segment)) {
+ Printf("\t%p-%p\t%s\n", (void *)segment.start, (void *)segment.end,
+ segment.filename);
}
Report("End of process memory map.\n");
UnmapOrDie(filename, kBufSize);
@@ -363,14 +280,14 @@ void ReportFile::Write(const char *buffer, uptr length) {
}
bool GetCodeRangeForFile(const char *module, uptr *start, uptr *end) {
- uptr s, e, off, prot;
- InternalScopedString buff(kMaxPathLength);
MemoryMappingLayout proc_maps(/*cache_enabled*/false);
- while (proc_maps.Next(&s, &e, &off, buff.data(), buff.size(), &prot)) {
- if ((prot & MemoryMappingLayout::kProtectionExecute) != 0
- && internal_strcmp(module, buff.data()) == 0) {
- *start = s;
- *end = e;
+ InternalScopedString buff(kMaxPathLength);
+ MemoryMappedSegment segment(buff.data(), kMaxPathLength);
+ while (proc_maps.Next(&segment)) {
+ if (segment.IsExecutable() &&
+ internal_strcmp(module, segment.filename) == 0) {
+ *start = segment.start;
+ *end = segment.end;
return true;
}
}
diff --git a/lib/sanitizer_common/sanitizer_procmaps.h b/lib/sanitizer_common/sanitizer_procmaps.h
index 5aad6b959ad4..06d072b4dc77 100644
--- a/lib/sanitizer_common/sanitizer_procmaps.h
+++ b/lib/sanitizer_common/sanitizer_procmaps.h
@@ -31,13 +31,37 @@ struct ProcSelfMapsBuff {
void ReadProcMaps(ProcSelfMapsBuff *proc_maps);
#endif // SANITIZER_FREEBSD || SANITIZER_LINUX
+// Memory protection masks.
+static const uptr kProtectionRead = 1;
+static const uptr kProtectionWrite = 2;
+static const uptr kProtectionExecute = 4;
+static const uptr kProtectionShared = 8;
+
+struct MemoryMappedSegment {
+ MemoryMappedSegment(char *buff = nullptr, uptr size = 0)
+ : filename(buff), filename_size(size) {}
+ ~MemoryMappedSegment() {}
+
+ bool IsReadable() { return protection & kProtectionRead; }
+ bool IsWritable() { return protection & kProtectionWrite; }
+ bool IsExecutable() { return protection & kProtectionExecute; }
+ bool IsShared() { return protection & kProtectionShared; }
+
+ uptr start;
+ uptr end;
+ uptr offset;
+ char *filename; // owned by caller
+ uptr filename_size;
+ uptr protection;
+ ModuleArch arch;
+ u8 uuid[kModuleUUIDSize];
+};
+
class MemoryMappingLayout {
public:
explicit MemoryMappingLayout(bool cache_enabled);
~MemoryMappingLayout();
- bool Next(uptr *start, uptr *end, uptr *offset, char filename[],
- uptr filename_size, uptr *protection, ModuleArch *arch = nullptr,
- u8 *uuid = nullptr);
+ bool Next(MemoryMappedSegment *segment);
void Reset();
// In some cases, e.g. when running under a sandbox on Linux, ASan is unable
// to obtain the memory mappings. It should fall back to pre-cached data
@@ -47,12 +71,6 @@ class MemoryMappingLayout {
// Adds all mapped objects into a vector.
void DumpListOfModules(InternalMmapVector<LoadedModule> *modules);
- // Memory protection masks.
- static const uptr kProtectionRead = 1;
- static const uptr kProtectionWrite = 2;
- static const uptr kProtectionExecute = 4;
- static const uptr kProtectionShared = 8;
-
private:
void LoadFromCache();
@@ -67,10 +85,7 @@ class MemoryMappingLayout {
static StaticSpinMutex cache_lock_; // protects cached_proc_self_maps_.
# elif SANITIZER_MAC
template <u32 kLCSegment, typename SegmentCommand>
- bool NextSegmentLoad(uptr *start, uptr *end, uptr *offset, char filename[],
- uptr filename_size, ModuleArch *arch, u8 *uuid,
- uptr *protection);
- void GetSegmentAddrRange(uptr *start, uptr *end, uptr vmaddr, uptr vmsize);
+ bool NextSegmentLoad(MemoryMappedSegment *segment);
int current_image_;
u32 current_magic_;
u32 current_filetype_;
diff --git a/lib/sanitizer_common/sanitizer_procmaps_common.cc b/lib/sanitizer_common/sanitizer_procmaps_common.cc
index c583f42f25d8..b95f301a437d 100644
--- a/lib/sanitizer_common/sanitizer_procmaps_common.cc
+++ b/lib/sanitizer_common/sanitizer_procmaps_common.cc
@@ -119,12 +119,10 @@ void MemoryMappingLayout::LoadFromCache() {
void MemoryMappingLayout::DumpListOfModules(
InternalMmapVector<LoadedModule> *modules) {
Reset();
- uptr cur_beg, cur_end, cur_offset, prot;
InternalScopedString module_name(kMaxPathLength);
- for (uptr i = 0; Next(&cur_beg, &cur_end, &cur_offset, module_name.data(),
- module_name.size(), &prot);
- i++) {
- const char *cur_name = module_name.data();
+ MemoryMappedSegment segment(module_name.data(), module_name.size());
+ for (uptr i = 0; Next(&segment); i++) {
+ const char *cur_name = segment.filename;
if (cur_name[0] == '\0')
continue;
// Don't subtract 'cur_beg' from the first entry:
@@ -138,11 +136,11 @@ void MemoryMappingLayout::DumpListOfModules(
// mapped high at address space (in particular, higher than
// shadow memory of the tool), so the module can't be the
// first entry.
- uptr base_address = (i ? cur_beg : 0) - cur_offset;
+ uptr base_address = (i ? segment.start : 0) - segment.offset;
LoadedModule cur_module;
cur_module.set(cur_name, base_address);
- cur_module.addAddressRange(cur_beg, cur_end, prot & kProtectionExecute,
- prot & kProtectionWrite);
+ cur_module.addAddressRange(segment.start, segment.end,
+ segment.IsExecutable(), segment.IsWritable());
modules->push_back(cur_module);
}
}
diff --git a/lib/sanitizer_common/sanitizer_procmaps_freebsd.cc b/lib/sanitizer_common/sanitizer_procmaps_freebsd.cc
index 30216456330e..f0cdbeb4483a 100644
--- a/lib/sanitizer_common/sanitizer_procmaps_freebsd.cc
+++ b/lib/sanitizer_common/sanitizer_procmaps_freebsd.cc
@@ -48,36 +48,27 @@ void ReadProcMaps(ProcSelfMapsBuff *proc_maps) {
proc_maps->len = Size;
}
-bool MemoryMappingLayout::Next(uptr *start, uptr *end, uptr *offset,
- char filename[], uptr filename_size,
- uptr *protection, ModuleArch *arch, u8 *uuid) {
- CHECK(!arch && "not implemented");
- CHECK(!uuid && "not implemented");
+bool MemoryMappingLayout::Next(MemoryMappedSegment *segment) {
char *last = proc_self_maps_.data + proc_self_maps_.len;
if (current_ >= last) return false;
- uptr dummy;
- if (!start) start = &dummy;
- if (!end) end = &dummy;
- if (!offset) offset = &dummy;
- if (!protection) protection = &dummy;
struct kinfo_vmentry *VmEntry = (struct kinfo_vmentry*)current_;
- *start = (uptr)VmEntry->kve_start;
- *end = (uptr)VmEntry->kve_end;
- *offset = (uptr)VmEntry->kve_offset;
+ segment->start = (uptr)VmEntry->kve_start;
+ segment->end = (uptr)VmEntry->kve_end;
+ segment->offset = (uptr)VmEntry->kve_offset;
- *protection = 0;
+ segment->protection = 0;
if ((VmEntry->kve_protection & KVME_PROT_READ) != 0)
- *protection |= kProtectionRead;
+ segment->protection |= kProtectionRead;
if ((VmEntry->kve_protection & KVME_PROT_WRITE) != 0)
- *protection |= kProtectionWrite;
+ segment->protection |= kProtectionWrite;
if ((VmEntry->kve_protection & KVME_PROT_EXEC) != 0)
- *protection |= kProtectionExecute;
+ segment->protection |= kProtectionExecute;
- if (filename != NULL && filename_size > 0) {
- internal_snprintf(filename,
- Min(filename_size, (uptr)PATH_MAX),
- "%s", VmEntry->kve_path);
+ if (segment->filename != NULL && segment->filename_size > 0) {
+ internal_snprintf(segment->filename,
+ Min(segment->filename_size, (uptr)PATH_MAX), "%s",
+ VmEntry->kve_path);
}
current_ += VmEntry->kve_structsize;
diff --git a/lib/sanitizer_common/sanitizer_procmaps_linux.cc b/lib/sanitizer_common/sanitizer_procmaps_linux.cc
index 7e4a44be95b6..1bcad2bf70e6 100644
--- a/lib/sanitizer_common/sanitizer_procmaps_linux.cc
+++ b/lib/sanitizer_common/sanitizer_procmaps_linux.cc
@@ -26,41 +26,28 @@ static bool IsOneOf(char c, char c1, char c2) {
return c == c1 || c == c2;
}
-bool MemoryMappingLayout::Next(uptr *start, uptr *end, uptr *offset,
- char filename[], uptr filename_size,
- uptr *protection, ModuleArch *arch, u8 *uuid) {
- CHECK(!arch && "not implemented");
- CHECK(!uuid && "not implemented");
+bool MemoryMappingLayout::Next(MemoryMappedSegment *segment) {
char *last = proc_self_maps_.data + proc_self_maps_.len;
if (current_ >= last) return false;
- uptr dummy;
- if (!start) start = &dummy;
- if (!end) end = &dummy;
- if (!offset) offset = &dummy;
- if (!protection) protection = &dummy;
char *next_line = (char*)internal_memchr(current_, '\n', last - current_);
if (next_line == 0)
next_line = last;
// Example: 08048000-08056000 r-xp 00000000 03:0c 64593 /foo/bar
- *start = ParseHex(&current_);
+ segment->start = ParseHex(&current_);
CHECK_EQ(*current_++, '-');
- *end = ParseHex(&current_);
+ segment->end = ParseHex(&current_);
CHECK_EQ(*current_++, ' ');
CHECK(IsOneOf(*current_, '-', 'r'));
- *protection = 0;
- if (*current_++ == 'r')
- *protection |= kProtectionRead;
+ segment->protection = 0;
+ if (*current_++ == 'r') segment->protection |= kProtectionRead;
CHECK(IsOneOf(*current_, '-', 'w'));
- if (*current_++ == 'w')
- *protection |= kProtectionWrite;
+ if (*current_++ == 'w') segment->protection |= kProtectionWrite;
CHECK(IsOneOf(*current_, '-', 'x'));
- if (*current_++ == 'x')
- *protection |= kProtectionExecute;
+ if (*current_++ == 'x') segment->protection |= kProtectionExecute;
CHECK(IsOneOf(*current_, 's', 'p'));
- if (*current_++ == 's')
- *protection |= kProtectionShared;
+ if (*current_++ == 's') segment->protection |= kProtectionShared;
CHECK_EQ(*current_++, ' ');
- *offset = ParseHex(&current_);
+ segment->offset = ParseHex(&current_);
CHECK_EQ(*current_++, ' ');
ParseHex(&current_);
CHECK_EQ(*current_++, ':');
@@ -75,14 +62,12 @@ bool MemoryMappingLayout::Next(uptr *start, uptr *end, uptr *offset,
while (current_ < next_line && *current_ == ' ')
current_++;
// Fill in the filename.
- uptr i = 0;
- while (current_ < next_line) {
- if (filename && i < filename_size - 1)
- filename[i++] = *current_;
- current_++;
+ if (segment->filename) {
+ uptr len = Min((uptr)(next_line - current_), segment->filename_size - 1);
+ internal_strncpy(segment->filename, current_, len);
+ segment->filename[len] = 0;
}
- if (filename && i < filename_size)
- filename[i] = 0;
+
current_ = next_line + 1;
return true;
}
diff --git a/lib/sanitizer_common/sanitizer_procmaps_mac.cc b/lib/sanitizer_common/sanitizer_procmaps_mac.cc
index 131017458d4c..560451a16d90 100644
--- a/lib/sanitizer_common/sanitizer_procmaps_mac.cc
+++ b/lib/sanitizer_common/sanitizer_procmaps_mac.cc
@@ -88,6 +88,48 @@ void MemoryMappingLayout::LoadFromCache() {
// No-op on Mac for now.
}
+// _dyld_get_image_header() and related APIs don't report dyld itself.
+// We work around this by manually recursing through the memory map
+// until we hit a Mach header matching dyld instead. These recurse
+// calls are expensive, but the first memory map generation occurs
+// early in the process, when dyld is one of the only images loaded,
+// so it will be hit after only a few iterations.
+static mach_header *get_dyld_image_header() {
+ mach_port_name_t port;
+ if (task_for_pid(mach_task_self(), internal_getpid(), &port) !=
+ KERN_SUCCESS) {
+ return nullptr;
+ }
+
+ unsigned depth = 1;
+ vm_size_t size = 0;
+ vm_address_t address = 0;
+ kern_return_t err = KERN_SUCCESS;
+ mach_msg_type_number_t count = VM_REGION_SUBMAP_INFO_COUNT_64;
+
+ while (true) {
+ struct vm_region_submap_info_64 info;
+ err = vm_region_recurse_64(port, &address, &size, &depth,
+ (vm_region_info_t)&info, &count);
+ if (err != KERN_SUCCESS) return nullptr;
+
+ if (size >= sizeof(mach_header) && info.protection & kProtectionRead) {
+ mach_header *hdr = (mach_header *)address;
+ if ((hdr->magic == MH_MAGIC || hdr->magic == MH_MAGIC_64) &&
+ hdr->filetype == MH_DYLINKER) {
+ return hdr;
+ }
+ }
+ address += size;
+ }
+}
+
+const mach_header *get_dyld_hdr() {
+ if (!dyld_hdr) dyld_hdr = get_dyld_image_header();
+
+ return dyld_hdr;
+}
+
// Next and NextSegmentLoad were inspired by base/sysinfo.cc in
// Google Perftools, https://github.com/gperftools/gperftools.
@@ -96,40 +138,39 @@ void MemoryMappingLayout::LoadFromCache() {
// segment.
// Note that the segment addresses are not necessarily sorted.
template <u32 kLCSegment, typename SegmentCommand>
-bool MemoryMappingLayout::NextSegmentLoad(uptr *start, uptr *end, uptr *offset,
- char filename[], uptr filename_size,
- ModuleArch *arch, u8 *uuid,
- uptr *protection) {
+bool MemoryMappingLayout::NextSegmentLoad(MemoryMappedSegment *segment) {
const char *lc = current_load_cmd_addr_;
current_load_cmd_addr_ += ((const load_command *)lc)->cmdsize;
if (((const load_command *)lc)->cmd == kLCSegment) {
const SegmentCommand* sc = (const SegmentCommand *)lc;
- GetSegmentAddrRange(start, end, sc->vmaddr, sc->vmsize);
- if (protection) {
- // Return the initial protection.
- *protection = sc->initprot;
- }
- if (offset) {
- if (current_filetype_ == /*MH_EXECUTE*/ 0x2) {
- *offset = sc->vmaddr;
- } else {
- *offset = sc->fileoff;
- }
- }
- if (filename) {
- if (current_image_ == kDyldImageIdx) {
- internal_strncpy(filename, kDyldPath, filename_size);
- } else {
- internal_strncpy(filename, _dyld_get_image_name(current_image_),
- filename_size);
- }
- }
- if (arch) {
- *arch = current_arch_;
+
+ if (current_image_ == kDyldImageIdx) {
+ // vmaddr is masked with 0xfffff because on macOS versions < 10.12,
+ // it contains an absolute address rather than an offset for dyld.
+ // To make matters even more complicated, this absolute address
+ // isn't actually the absolute segment address, but the offset portion
+ // of the address is accurate when combined with the dyld base address,
+ // and the mask will give just this offset.
+ segment->start = (sc->vmaddr & 0xfffff) + (uptr)get_dyld_hdr();
+ segment->end = (sc->vmaddr & 0xfffff) + sc->vmsize + (uptr)get_dyld_hdr();
+ } else {
+ const sptr dlloff = _dyld_get_image_vmaddr_slide(current_image_);
+ segment->start = sc->vmaddr + dlloff;
+ segment->end = sc->vmaddr + sc->vmsize + dlloff;
}
- if (uuid) {
- internal_memcpy(uuid, current_uuid_, kModuleUUIDSize);
+
+ // Return the initial protection.
+ segment->protection = sc->initprot;
+ segment->offset =
+ (current_filetype_ == /*MH_EXECUTE*/ 0x2) ? sc->vmaddr : sc->fileoff;
+ if (segment->filename) {
+ const char *src = (current_image_ == kDyldImageIdx)
+ ? kDyldPath
+ : _dyld_get_image_name(current_image_);
+ internal_strncpy(segment->filename, src, segment->filename_size);
}
+ segment->arch = current_arch_;
+ internal_memcpy(segment->uuid, current_uuid_, kModuleUUIDSize);
return true;
}
return false;
@@ -190,70 +231,7 @@ static bool IsModuleInstrumented(const load_command *first_lc) {
return false;
}
-// _dyld_get_image_header() and related APIs don't report dyld itself.
-// We work around this by manually recursing through the memory map
-// until we hit a Mach header matching dyld instead. These recurse
-// calls are expensive, but the first memory map generation occurs
-// early in the process, when dyld is one of the only images loaded,
-// so it will be hit after only a few iterations.
-static mach_header *get_dyld_image_header() {
- mach_port_name_t port;
- if (task_for_pid(mach_task_self(), internal_getpid(), &port) !=
- KERN_SUCCESS) {
- return nullptr;
- }
-
- unsigned depth = 1;
- vm_size_t size = 0;
- vm_address_t address = 0;
- kern_return_t err = KERN_SUCCESS;
- mach_msg_type_number_t count = VM_REGION_SUBMAP_INFO_COUNT_64;
-
- while (true) {
- struct vm_region_submap_info_64 info;
- err = vm_region_recurse_64(port, &address, &size, &depth,
- (vm_region_info_t)&info, &count);
- if (err != KERN_SUCCESS) return nullptr;
-
- if (size >= sizeof(mach_header) &&
- info.protection & MemoryMappingLayout::kProtectionRead) {
- mach_header *hdr = (mach_header *)address;
- if ((hdr->magic == MH_MAGIC || hdr->magic == MH_MAGIC_64) &&
- hdr->filetype == MH_DYLINKER) {
- return hdr;
- }
- }
- address += size;
- }
-}
-
-const mach_header *get_dyld_hdr() {
- if (!dyld_hdr) dyld_hdr = get_dyld_image_header();
-
- return dyld_hdr;
-}
-
-void MemoryMappingLayout::GetSegmentAddrRange(uptr *start, uptr *end,
- uptr vmaddr, uptr vmsize) {
- if (current_image_ == kDyldImageIdx) {
- // vmaddr is masked with 0xfffff because on macOS versions < 10.12,
- // it contains an absolute address rather than an offset for dyld.
- // To make matters even more complicated, this absolute address
- // isn't actually the absolute segment address, but the offset portion
- // of the address is accurate when combined with the dyld base address,
- // and the mask will give just this offset.
- if (start) *start = (vmaddr & 0xfffff) + (uptr)get_dyld_hdr();
- if (end) *end = (vmaddr & 0xfffff) + vmsize + (uptr)get_dyld_hdr();
- } else {
- const sptr dlloff = _dyld_get_image_vmaddr_slide(current_image_);
- if (start) *start = vmaddr + dlloff;
- if (end) *end = vmaddr + vmsize + dlloff;
- }
-}
-
-bool MemoryMappingLayout::Next(uptr *start, uptr *end, uptr *offset,
- char filename[], uptr filename_size,
- uptr *protection, ModuleArch *arch, u8 *uuid) {
+bool MemoryMappingLayout::Next(MemoryMappedSegment *segment) {
for (; current_image_ >= kDyldImageIdx; current_image_--) {
const mach_header *hdr = (current_image_ == kDyldImageIdx)
? get_dyld_hdr()
@@ -291,16 +269,13 @@ bool MemoryMappingLayout::Next(uptr *start, uptr *end, uptr *offset,
#ifdef MH_MAGIC_64
case MH_MAGIC_64: {
if (NextSegmentLoad<LC_SEGMENT_64, struct segment_command_64>(
- start, end, offset, filename, filename_size, arch, uuid,
- protection))
+ segment))
return true;
break;
}
#endif
case MH_MAGIC: {
- if (NextSegmentLoad<LC_SEGMENT, struct segment_command>(
- start, end, offset, filename, filename_size, arch, uuid,
- protection))
+ if (NextSegmentLoad<LC_SEGMENT, struct segment_command>(segment))
return true;
break;
}
@@ -315,28 +290,22 @@ bool MemoryMappingLayout::Next(uptr *start, uptr *end, uptr *offset,
void MemoryMappingLayout::DumpListOfModules(
InternalMmapVector<LoadedModule> *modules) {
Reset();
- uptr cur_beg, cur_end, prot;
- ModuleArch cur_arch;
- u8 cur_uuid[kModuleUUIDSize];
InternalScopedString module_name(kMaxPathLength);
- for (uptr i = 0; Next(&cur_beg, &cur_end, 0, module_name.data(),
- module_name.size(), &prot, &cur_arch, &cur_uuid[0]);
- i++) {
- const char *cur_name = module_name.data();
- if (cur_name[0] == '\0')
- continue;
+ MemoryMappedSegment segment(module_name.data(), kMaxPathLength);
+ for (uptr i = 0; Next(&segment); i++) {
+ if (segment.filename[0] == '\0') continue;
LoadedModule *cur_module = nullptr;
if (!modules->empty() &&
- 0 == internal_strcmp(cur_name, modules->back().full_name())) {
+ 0 == internal_strcmp(segment.filename, modules->back().full_name())) {
cur_module = &modules->back();
} else {
modules->push_back(LoadedModule());
cur_module = &modules->back();
- cur_module->set(cur_name, cur_beg, cur_arch, cur_uuid,
- current_instrumented_);
+ cur_module->set(segment.filename, segment.start, segment.arch,
+ segment.uuid, current_instrumented_);
}
- cur_module->addAddressRange(cur_beg, cur_end, prot & kProtectionExecute,
- prot & kProtectionWrite);
+ cur_module->addAddressRange(segment.start, segment.end,
+ segment.IsExecutable(), segment.IsWritable());
}
}
diff --git a/lib/sanitizer_common/sanitizer_stacktrace_libcdep.cc b/lib/sanitizer_common/sanitizer_stacktrace_libcdep.cc
index 36c98d057bd3..747a4a701728 100644
--- a/lib/sanitizer_common/sanitizer_stacktrace_libcdep.cc
+++ b/lib/sanitizer_common/sanitizer_stacktrace_libcdep.cc
@@ -43,7 +43,8 @@ void StackTrace::Print() const {
if (dedup_frames-- > 0) {
if (dedup_token.length())
dedup_token.append("--");
- dedup_token.append(cur->info.function);
+ if (cur->info.function != nullptr)
+ dedup_token.append(cur->info.function);
}
}
frames->ClearAll();
diff --git a/lib/sanitizer_common/sanitizer_win.cc b/lib/sanitizer_common/sanitizer_win.cc
index 89d9cf61c3e4..de01e8d119a1 100644
--- a/lib/sanitizer_common/sanitizer_win.cc
+++ b/lib/sanitizer_common/sanitizer_win.cc
@@ -291,7 +291,8 @@ void DontDumpShadowMemory(uptr addr, uptr length) {
// FIXME: add madvise-analog when we move to 64-bits.
}
-uptr FindAvailableMemoryRange(uptr size, uptr alignment, uptr left_padding) {
+uptr FindAvailableMemoryRange(uptr size, uptr alignment, uptr left_padding,
+ uptr *largest_gap_found) {
uptr address = 0;
while (true) {
MEMORY_BASIC_INFORMATION info;
diff --git a/lib/scudo/scudo_allocator.cpp b/lib/scudo/scudo_allocator.cpp
index 00fa192181ad..ec9132f90a4f 100644
--- a/lib/scudo/scudo_allocator.cpp
+++ b/lib/scudo/scudo_allocator.cpp
@@ -264,7 +264,7 @@ ScudoQuarantineCache *getQuarantineCache(ScudoThreadContext *ThreadContext) {
ScudoQuarantineCache *>(ThreadContext->QuarantineCachePlaceHolder);
}
-Xorshift128Plus *getPrng(ScudoThreadContext *ThreadContext) {
+ScudoPrng *getPrng(ScudoThreadContext *ThreadContext) {
return &ThreadContext->Prng;
}
@@ -283,7 +283,7 @@ struct ScudoAllocator {
StaticSpinMutex FallbackMutex;
AllocatorCache FallbackAllocatorCache;
ScudoQuarantineCache FallbackQuarantineCache;
- Xorshift128Plus FallbackPrng;
+ ScudoPrng FallbackPrng;
bool DeallocationTypeMismatch;
bool ZeroContents;
@@ -333,8 +333,8 @@ struct ScudoAllocator {
static_cast<uptr>(Options.QuarantineSizeMb) << 20,
static_cast<uptr>(Options.ThreadLocalQuarantineSizeKb) << 10);
BackendAllocator.InitCache(&FallbackAllocatorCache);
- FallbackPrng.initFromURandom();
- Cookie = FallbackPrng.getNext();
+ FallbackPrng.init();
+ Cookie = FallbackPrng.getU64();
}
// Helper function that checks for a valid Scudo chunk. nullptr isn't.
@@ -373,19 +373,19 @@ struct ScudoAllocator {
bool FromPrimary = PrimaryAllocator::CanAllocate(AlignedSize, MinAlignment);
void *Ptr;
- uptr Salt;
+ u8 Salt;
uptr AllocationSize = FromPrimary ? AlignedSize : NeededSize;
uptr AllocationAlignment = FromPrimary ? MinAlignment : Alignment;
ScudoThreadContext *ThreadContext = getThreadContextAndLock();
if (LIKELY(ThreadContext)) {
- Salt = getPrng(ThreadContext)->getNext();
+ Salt = getPrng(ThreadContext)->getU8();
Ptr = BackendAllocator.Allocate(getAllocatorCache(ThreadContext),
AllocationSize, AllocationAlignment,
FromPrimary);
ThreadContext->unlock();
} else {
SpinMutexLock l(&FallbackMutex);
- Salt = FallbackPrng.getNext();
+ Salt = FallbackPrng.getU8();
Ptr = BackendAllocator.Allocate(&FallbackAllocatorCache, AllocationSize,
AllocationAlignment, FromPrimary);
}
@@ -612,7 +612,7 @@ static void initScudoInternal(const AllocatorOptions &Options) {
void ScudoThreadContext::init() {
getBackendAllocator().InitCache(&Cache);
- Prng.initFromURandom();
+ Prng.init();
memset(QuarantineCachePlaceHolder, 0, sizeof(QuarantineCachePlaceHolder));
}
diff --git a/lib/scudo/scudo_tls.h b/lib/scudo/scudo_tls.h
index f6039bebec44..20c49204cf13 100644
--- a/lib/scudo/scudo_tls.h
+++ b/lib/scudo/scudo_tls.h
@@ -30,7 +30,7 @@ namespace __scudo {
struct ALIGNED(64) ScudoThreadContext : public ScudoThreadContextPlatform {
AllocatorCache Cache;
- Xorshift128Plus Prng;
+ ScudoPrng Prng;
uptr QuarantineCachePlaceHolder[4];
void init();
void commitBack();
diff --git a/lib/scudo/scudo_utils.cpp b/lib/scudo/scudo_utils.cpp
index 31c391946c15..f7903ff34c73 100644
--- a/lib/scudo/scudo_utils.cpp
+++ b/lib/scudo/scudo_utils.cpp
@@ -123,40 +123,4 @@ bool testCPUFeature(CPUFeature Feature) {
}
#endif // defined(__x86_64__) || defined(__i386__)
-// readRetry will attempt to read Count bytes from the Fd specified, and if
-// interrupted will retry to read additional bytes to reach Count.
-static ssize_t readRetry(int Fd, u8 *Buffer, size_t Count) {
- ssize_t AmountRead = 0;
- while (static_cast<size_t>(AmountRead) < Count) {
- ssize_t Result = read(Fd, Buffer + AmountRead, Count - AmountRead);
- if (Result > 0)
- AmountRead += Result;
- else if (!Result)
- break;
- else if (errno != EINTR) {
- AmountRead = -1;
- break;
- }
- }
- return AmountRead;
-}
-
-static void fillRandom(u8 *Data, ssize_t Size) {
- int Fd = open("/dev/urandom", O_RDONLY);
- if (Fd < 0) {
- dieWithMessage("ERROR: failed to open /dev/urandom.\n");
- }
- bool Success = readRetry(Fd, Data, Size) == Size;
- close(Fd);
- if (!Success) {
- dieWithMessage("ERROR: failed to read enough data from /dev/urandom.\n");
- }
-}
-
-// Seeds the xorshift state with /dev/urandom.
-// TODO(kostyak): investigate using getrandom() if available.
-void Xorshift128Plus::initFromURandom() {
- fillRandom(reinterpret_cast<u8 *>(State), sizeof(State));
-}
-
} // namespace __scudo
diff --git a/lib/scudo/scudo_utils.h b/lib/scudo/scudo_utils.h
index 7198476f42cf..6c6c9d893404 100644
--- a/lib/scudo/scudo_utils.h
+++ b/lib/scudo/scudo_utils.h
@@ -36,23 +36,58 @@ enum CPUFeature {
};
bool testCPUFeature(CPUFeature feature);
-// Tiny PRNG based on https://en.wikipedia.org/wiki/Xorshift#xorshift.2B
-// The state (128 bits) will be stored in thread local storage.
-struct Xorshift128Plus {
+INLINE u64 rotl(const u64 X, int K) {
+ return (X << K) | (X >> (64 - K));
+}
+
+// XoRoShiRo128+ PRNG (http://xoroshiro.di.unimi.it/).
+struct XoRoShiRo128Plus {
public:
- void initFromURandom();
- u64 getNext() {
- u64 x = State[0];
- const u64 y = State[1];
- State[0] = y;
- x ^= x << 23;
- State[1] = x ^ y ^ (x >> 17) ^ (y >> 26);
- return State[1] + y;
+ void init() {
+ if (UNLIKELY(!GetRandom(reinterpret_cast<void *>(State), sizeof(State)))) {
+ // Early processes (eg: init) do not have /dev/urandom yet, but we still
+ // have to provide them with some degree of entropy. Not having a secure
+ // seed is not as problematic for them, as they are less likely to be
+ // the target of heap based vulnerabilities exploitation attempts.
+ State[0] = NanoTime();
+ State[1] = 0;
+ }
+ fillCache();
}
+ u8 getU8() {
+ if (UNLIKELY(isCacheEmpty()))
+ fillCache();
+ const u8 Result = static_cast<u8>(CachedBytes & 0xff);
+ CachedBytes >>= 8;
+ CachedBytesAvailable--;
+ return Result;
+ }
+ u64 getU64() { return next(); }
+
private:
+ u8 CachedBytesAvailable;
+ u64 CachedBytes;
u64 State[2];
+ u64 next() {
+ const u64 S0 = State[0];
+ u64 S1 = State[1];
+ const u64 Result = S0 + S1;
+ S1 ^= S0;
+ State[0] = rotl(S0, 55) ^ S1 ^ (S1 << 14);
+ State[1] = rotl(S1, 36);
+ return Result;
+ }
+ bool isCacheEmpty() {
+ return CachedBytesAvailable == 0;
+ }
+ void fillCache() {
+ CachedBytes = next();
+ CachedBytesAvailable = sizeof(CachedBytes);
+ }
};
+typedef XoRoShiRo128Plus ScudoPrng;
+
} // namespace __scudo
#endif // SCUDO_UTILS_H_
diff --git a/lib/tsan/CMakeLists.txt b/lib/tsan/CMakeLists.txt
index 195ecb5dfe8a..193158c54e62 100644
--- a/lib/tsan/CMakeLists.txt
+++ b/lib/tsan/CMakeLists.txt
@@ -100,7 +100,7 @@ set(TSAN_RUNTIME_LIBRARIES)
add_compiler_rt_component(tsan)
if(APPLE)
- set(TSAN_ASM_SOURCES rtl/tsan_rtl_amd64.S)
+ set(TSAN_ASM_SOURCES rtl/tsan_rtl_amd64.S rtl/tsan_rtl_aarch64.S)
# Xcode will try to compile this file as C ('clang -x c'), and that will fail.
if (${CMAKE_GENERATOR} STREQUAL "Xcode")
enable_language(ASM)
diff --git a/lib/tsan/check_analyze.sh b/lib/tsan/check_analyze.sh
index 22eb444198a4..54dd1b0232dc 100755
--- a/lib/tsan/check_analyze.sh
+++ b/lib/tsan/check_analyze.sh
@@ -34,8 +34,8 @@ done
for f in read1 read2 read4 read8; do
check $f rsp 1
- check $f push 4
- check $f pop 4
+ check $f push 3
+ check $f pop 3
done
for f in func_entry func_exit; do
diff --git a/lib/tsan/dd/dd_interceptors.cc b/lib/tsan/dd/dd_interceptors.cc
index 97c72dd2b7fd..a39218f0454b 100644
--- a/lib/tsan/dd/dd_interceptors.cc
+++ b/lib/tsan/dd/dd_interceptors.cc
@@ -270,20 +270,19 @@ namespace __dsan {
static void InitDataSeg() {
MemoryMappingLayout proc_maps(true);
- uptr start, end, offset;
char name[128];
+ MemoryMappedSegment segment(name, ARRAY_SIZE(name));
bool prev_is_data = false;
- while (proc_maps.Next(&start, &end, &offset, name, ARRAY_SIZE(name),
- /*protection*/ 0)) {
- bool is_data = offset != 0 && name[0] != 0;
+ while (proc_maps.Next(&segment)) {
+ bool is_data = segment.offset != 0 && segment.filename[0] != 0;
// BSS may get merged with [heap] in /proc/self/maps. This is not very
// reliable.
- bool is_bss = offset == 0 &&
- (name[0] == 0 || internal_strcmp(name, "[heap]") == 0) && prev_is_data;
- if (g_data_start == 0 && is_data)
- g_data_start = start;
- if (is_bss)
- g_data_end = end;
+ bool is_bss = segment.offset == 0 &&
+ (segment.filename[0] == 0 ||
+ internal_strcmp(segment.filename, "[heap]") == 0) &&
+ prev_is_data;
+ if (g_data_start == 0 && is_data) g_data_start = segment.start;
+ if (is_bss) g_data_end = segment.end;
prev_is_data = is_data;
}
VPrintf(1, "guessed data_start=%p data_end=%p\n", g_data_start, g_data_end);
diff --git a/lib/tsan/go/buildgo.sh b/lib/tsan/go/buildgo.sh
index 59176809eae0..617dd9e11d27 100755
--- a/lib/tsan/go/buildgo.sh
+++ b/lib/tsan/go/buildgo.sh
@@ -69,7 +69,7 @@ elif [ "`uname -a | grep FreeBSD`" != "" ]; then
"
elif [ "`uname -a | grep Darwin`" != "" ]; then
SUFFIX="darwin_amd64"
- OSCFLAGS="-fPIC -Wno-unused-const-variable -Wno-unknown-warning-option -mmacosx-version-min=10.7"
+ OSCFLAGS="-fPIC -Wno-unused-const-variable -Wno-unknown-warning-option -isysroot $(xcodebuild -version -sdk macosx Path) -mmacosx-version-min=10.7"
OSLDFLAGS="-lpthread -fPIC -fpie -mmacosx-version-min=10.7"
SRCS="
$SRCS
diff --git a/lib/tsan/rtl/tsan_clock.cc b/lib/tsan/rtl/tsan_clock.cc
index 32435adfdf33..9ee9104283f8 100644
--- a/lib/tsan/rtl/tsan_clock.cc
+++ b/lib/tsan/rtl/tsan_clock.cc
@@ -101,6 +101,9 @@ ThreadClock::ThreadClock(unsigned tid, unsigned reused)
clk_[tid_].reused = reused_;
}
+void ThreadClock::ResetCached(ClockCache *c) {
+}
+
void ThreadClock::acquire(ClockCache *c, const SyncClock *src) {
DCHECK_LE(nclk_, kMaxTid);
DCHECK_LE(src->size_, kMaxTid);
@@ -116,9 +119,7 @@ void ThreadClock::acquire(ClockCache *c, const SyncClock *src) {
// Check if we've already acquired src after the last release operation on src
bool acquired = false;
if (nclk > tid_) {
- CPP_STAT_INC(StatClockAcquireLarge);
if (src->elem(tid_).reused == reused_) {
- CPP_STAT_INC(StatClockAcquireRepeat);
for (unsigned i = 0; i < kDirtyTids; i++) {
unsigned tid = src->dirty_tids_[i];
if (tid != kInvalidTid) {
@@ -266,11 +267,11 @@ void ThreadClock::UpdateCurrentThread(SyncClock *dst) const {
for (unsigned i = 0; i < kDirtyTids; i++) {
if (dst->dirty_tids_[i] == tid_) {
- CPP_STAT_INC(StatClockReleaseFast1);
+ CPP_STAT_INC(StatClockReleaseFast);
return;
}
if (dst->dirty_tids_[i] == kInvalidTid) {
- CPP_STAT_INC(StatClockReleaseFast2);
+ CPP_STAT_INC(StatClockReleaseFast);
dst->dirty_tids_[i] = tid_;
return;
}
@@ -297,56 +298,9 @@ bool ThreadClock::IsAlreadyAcquired(const SyncClock *src) const {
return true;
}
-void SyncClock::Resize(ClockCache *c, uptr nclk) {
- CPP_STAT_INC(StatClockReleaseResize);
- if (RoundUpTo(nclk, ClockBlock::kClockCount) <=
- RoundUpTo(size_, ClockBlock::kClockCount)) {
- // Growing within the same block.
- // Memory is already allocated, just increase the size.
- size_ = nclk;
- return;
- }
- if (nclk <= ClockBlock::kClockCount) {
- // Grow from 0 to one-level table.
- CHECK_EQ(size_, 0);
- CHECK_EQ(tab_, 0);
- CHECK_EQ(tab_idx_, 0);
- size_ = nclk;
- tab_idx_ = ctx->clock_alloc.Alloc(c);
- tab_ = ctx->clock_alloc.Map(tab_idx_);
- internal_memset(tab_, 0, sizeof(*tab_));
- return;
- }
- // Growing two-level table.
- if (size_ == 0) {
- // Allocate first level table.
- tab_idx_ = ctx->clock_alloc.Alloc(c);
- tab_ = ctx->clock_alloc.Map(tab_idx_);
- internal_memset(tab_, 0, sizeof(*tab_));
- } else if (size_ <= ClockBlock::kClockCount) {
- // Transform one-level table to two-level table.
- u32 old = tab_idx_;
- tab_idx_ = ctx->clock_alloc.Alloc(c);
- tab_ = ctx->clock_alloc.Map(tab_idx_);
- internal_memset(tab_, 0, sizeof(*tab_));
- tab_->table[0] = old;
- }
- // At this point we have first level table allocated.
- // Add second level tables as necessary.
- for (uptr i = RoundUpTo(size_, ClockBlock::kClockCount);
- i < nclk; i += ClockBlock::kClockCount) {
- u32 idx = ctx->clock_alloc.Alloc(c);
- ClockBlock *cb = ctx->clock_alloc.Map(idx);
- internal_memset(cb, 0, sizeof(*cb));
- CHECK_EQ(tab_->table[i/ClockBlock::kClockCount], 0);
- tab_->table[i/ClockBlock::kClockCount] = idx;
- }
- size_ = nclk;
-}
-
// Sets a single element in the vector clock.
// This function is called only from weird places like AcquireGlobal.
-void ThreadClock::set(unsigned tid, u64 v) {
+void ThreadClock::set(ClockCache *c, unsigned tid, u64 v) {
DCHECK_LT(tid, kMaxTid);
DCHECK_GE(v, clk_[tid].epoch);
clk_[tid].epoch = v;
@@ -366,14 +320,8 @@ void ThreadClock::DebugDump(int(*printf)(const char *s, ...)) {
tid_, reused_, last_acquire_);
}
-SyncClock::SyncClock()
- : release_store_tid_(kInvalidTid)
- , release_store_reused_()
- , tab_()
- , tab_idx_()
- , size_() {
- for (uptr i = 0; i < kDirtyTids; i++)
- dirty_tids_[i] = kInvalidTid;
+SyncClock::SyncClock() {
+ ResetImpl();
}
SyncClock::~SyncClock() {
@@ -395,6 +343,10 @@ void SyncClock::Reset(ClockCache *c) {
ctx->clock_alloc.Free(c, tab_->table[i / ClockBlock::kClockCount]);
ctx->clock_alloc.Free(c, tab_idx_);
}
+ ResetImpl();
+}
+
+void SyncClock::ResetImpl() {
tab_ = 0;
tab_idx_ = 0;
size_ = 0;
@@ -404,6 +356,53 @@ void SyncClock::Reset(ClockCache *c) {
dirty_tids_[i] = kInvalidTid;
}
+void SyncClock::Resize(ClockCache *c, uptr nclk) {
+ CPP_STAT_INC(StatClockReleaseResize);
+ if (RoundUpTo(nclk, ClockBlock::kClockCount) <=
+ RoundUpTo(size_, ClockBlock::kClockCount)) {
+ // Growing within the same block.
+ // Memory is already allocated, just increase the size.
+ size_ = nclk;
+ return;
+ }
+ if (nclk <= ClockBlock::kClockCount) {
+ // Grow from 0 to one-level table.
+ CHECK_EQ(size_, 0);
+ CHECK_EQ(tab_, 0);
+ CHECK_EQ(tab_idx_, 0);
+ size_ = nclk;
+ tab_idx_ = ctx->clock_alloc.Alloc(c);
+ tab_ = ctx->clock_alloc.Map(tab_idx_);
+ internal_memset(tab_, 0, sizeof(*tab_));
+ return;
+ }
+ // Growing two-level table.
+ if (size_ == 0) {
+ // Allocate first level table.
+ tab_idx_ = ctx->clock_alloc.Alloc(c);
+ tab_ = ctx->clock_alloc.Map(tab_idx_);
+ internal_memset(tab_, 0, sizeof(*tab_));
+ } else if (size_ <= ClockBlock::kClockCount) {
+ // Transform one-level table to two-level table.
+ u32 old = tab_idx_;
+ tab_idx_ = ctx->clock_alloc.Alloc(c);
+ tab_ = ctx->clock_alloc.Map(tab_idx_);
+ internal_memset(tab_, 0, sizeof(*tab_));
+ tab_->table[0] = old;
+ }
+ // At this point we have first level table allocated.
+ // Add second level tables as necessary.
+ for (uptr i = RoundUpTo(size_, ClockBlock::kClockCount);
+ i < nclk; i += ClockBlock::kClockCount) {
+ u32 idx = ctx->clock_alloc.Alloc(c);
+ ClockBlock *cb = ctx->clock_alloc.Map(idx);
+ internal_memset(cb, 0, sizeof(*cb));
+ CHECK_EQ(tab_->table[i/ClockBlock::kClockCount], 0);
+ tab_->table[i/ClockBlock::kClockCount] = idx;
+ }
+ size_ = nclk;
+}
+
ClockElem &SyncClock::elem(unsigned tid) const {
DCHECK_LT(tid, size_);
if (size_ <= ClockBlock::kClockCount)
diff --git a/lib/tsan/rtl/tsan_clock.h b/lib/tsan/rtl/tsan_clock.h
index 4e352cb81d11..378b550fd11b 100644
--- a/lib/tsan/rtl/tsan_clock.h
+++ b/lib/tsan/rtl/tsan_clock.h
@@ -74,6 +74,7 @@ class SyncClock {
u32 tab_idx_;
u32 size_;
+ void ResetImpl();
ClockElem &elem(unsigned tid) const;
};
@@ -89,7 +90,7 @@ struct ThreadClock {
return clk_[tid].epoch;
}
- void set(unsigned tid, u64 v);
+ void set(ClockCache *c, unsigned tid, u64 v);
void set(u64 v) {
DCHECK_GE(v, clk_[tid_].epoch);
@@ -108,6 +109,7 @@ struct ThreadClock {
void release(ClockCache *c, SyncClock *dst) const;
void acq_rel(ClockCache *c, SyncClock *dst);
void ReleaseStore(ClockCache *c, SyncClock *dst) const;
+ void ResetCached(ClockCache *c);
void DebugReset();
void DebugDump(int(*printf)(const char *s, ...));
diff --git a/lib/tsan/rtl/tsan_dense_alloc.h b/lib/tsan/rtl/tsan_dense_alloc.h
index e9815c90a953..16dbdf391085 100644
--- a/lib/tsan/rtl/tsan_dense_alloc.h
+++ b/lib/tsan/rtl/tsan_dense_alloc.h
@@ -39,7 +39,7 @@ class DenseSlabAlloc {
typedef DenseSlabAllocCache Cache;
typedef typename Cache::IndexT IndexT;
- DenseSlabAlloc() {
+ explicit DenseSlabAlloc(const char *name) {
// Check that kL1Size and kL2Size are sane.
CHECK_EQ(kL1Size & (kL1Size - 1), 0);
CHECK_EQ(kL2Size & (kL2Size - 1), 0);
@@ -49,6 +49,7 @@ class DenseSlabAlloc {
internal_memset(map_, 0, sizeof(map_));
freelist_ = 0;
fillpos_ = 0;
+ name_ = name;
}
~DenseSlabAlloc() {
@@ -96,15 +97,19 @@ class DenseSlabAlloc {
SpinMutex mtx_;
IndexT freelist_;
uptr fillpos_;
+ const char *name_;
void Refill(Cache *c) {
SpinMutexLock lock(&mtx_);
if (freelist_ == 0) {
if (fillpos_ == kL1Size) {
- Printf("ThreadSanitizer: DenseSlabAllocator overflow. Dying.\n");
+ Printf("ThreadSanitizer: %s overflow (%zu*%zu). Dying.\n",
+ name_, kL1Size, kL2Size);
Die();
}
- T *batch = (T*)MmapOrDie(kL2Size * sizeof(T), "DenseSlabAllocator");
+ VPrintf(2, "ThreadSanitizer: growing %s: %zu out of %zu*%zu\n",
+ name_, fillpos_, kL1Size, kL2Size);
+ T *batch = (T*)MmapOrDie(kL2Size * sizeof(T), name_);
// Reserve 0 as invalid index.
IndexT start = fillpos_ == 0 ? 1 : 0;
for (IndexT i = start; i < kL2Size; i++) {
diff --git a/lib/tsan/rtl/tsan_interceptors.cc b/lib/tsan/rtl/tsan_interceptors.cc
index 334cc326daf6..001123f4941e 100644
--- a/lib/tsan/rtl/tsan_interceptors.cc
+++ b/lib/tsan/rtl/tsan_interceptors.cc
@@ -14,6 +14,7 @@
//===----------------------------------------------------------------------===//
#include "sanitizer_common/sanitizer_atomic.h"
+#include "sanitizer_common/sanitizer_errno.h"
#include "sanitizer_common/sanitizer_libc.h"
#include "sanitizer_common/sanitizer_linux.h"
#include "sanitizer_common/sanitizer_platform_limits_posix.h"
@@ -34,13 +35,11 @@
using namespace __tsan; // NOLINT
#if SANITIZER_FREEBSD || SANITIZER_MAC
-#define __errno_location __error
#define stdout __stdoutp
#define stderr __stderrp
#endif
#if SANITIZER_ANDROID
-#define __errno_location __errno
#define mallopt(a, b)
#endif
@@ -84,7 +83,6 @@ DECLARE_REAL_AND_INTERCEPTOR(void *, malloc, uptr size)
DECLARE_REAL_AND_INTERCEPTOR(void, free, void *ptr)
extern "C" void *pthread_self();
extern "C" void _exit(int status);
-extern "C" int *__errno_location();
extern "C" int fileno_unlocked(void *stream);
extern "C" int dirfd(void *dirp);
#if !SANITIZER_FREEBSD && !SANITIZER_ANDROID
@@ -98,9 +96,6 @@ const int PTHREAD_MUTEX_RECURSIVE_NP = 1;
const int PTHREAD_MUTEX_RECURSIVE = 2;
const int PTHREAD_MUTEX_RECURSIVE_NP = 2;
#endif
-const int EINVAL = 22;
-const int EBUSY = 16;
-const int EOWNERDEAD = 130;
#if !SANITIZER_FREEBSD && !SANITIZER_MAC
const int EPOLL_CTL_ADD = 1;
#endif
@@ -130,8 +125,6 @@ typedef long long_t; // NOLINT
# define F_TLOCK 2 /* Test and lock a region for exclusive use. */
# define F_TEST 3 /* Test a region for other processes locks. */
-#define errno (*__errno_location())
-
typedef void (*sighandler_t)(int sig);
typedef void (*sigactionhandler_t)(int sig, my_siginfo_t *siginfo, void *uctx);
@@ -268,7 +261,7 @@ ScopedInterceptor::~ScopedInterceptor() {
void ScopedInterceptor::EnableIgnores() {
if (ignoring_) {
- ThreadIgnoreBegin(thr_, pc_, false);
+ ThreadIgnoreBegin(thr_, pc_, /*save_stack=*/false);
if (flags()->ignore_noninstrumented_modules) thr_->suppress_reports++;
if (in_ignored_lib_) {
DCHECK(!thr_->in_ignored_lib);
@@ -466,8 +459,14 @@ static void SetJmp(ThreadState *thr, uptr sp, uptr mangled_sp) {
static void LongJmp(ThreadState *thr, uptr *env) {
#ifdef __powerpc__
uptr mangled_sp = env[0];
-#elif SANITIZER_FREEBSD || SANITIZER_MAC
+#elif SANITIZER_FREEBSD
uptr mangled_sp = env[2];
+#elif SANITIZER_MAC
+# ifdef __aarch64__
+ uptr mangled_sp = env[13];
+# else
+ uptr mangled_sp = env[2];
+# endif
#elif defined(SANITIZER_LINUX)
# ifdef __aarch64__
uptr mangled_sp = env[13];
@@ -665,7 +664,7 @@ static bool fix_mmap_addr(void **addr, long_t sz, int flags) {
if (*addr) {
if (!IsAppMem((uptr)*addr) || !IsAppMem((uptr)*addr + sz - 1)) {
if (flags & MAP_FIXED) {
- errno = EINVAL;
+ errno = errno_EINVAL;
return false;
} else {
*addr = 0;
@@ -1122,7 +1121,7 @@ TSAN_INTERCEPTOR(int, pthread_mutex_init, void *m, void *a) {
TSAN_INTERCEPTOR(int, pthread_mutex_destroy, void *m) {
SCOPED_TSAN_INTERCEPTOR(pthread_mutex_destroy, m);
int res = REAL(pthread_mutex_destroy)(m);
- if (res == 0 || res == EBUSY) {
+ if (res == 0 || res == errno_EBUSY) {
MutexDestroy(thr, pc, (uptr)m);
}
return res;
@@ -1131,9 +1130,9 @@ TSAN_INTERCEPTOR(int, pthread_mutex_destroy, void *m) {
TSAN_INTERCEPTOR(int, pthread_mutex_trylock, void *m) {
SCOPED_TSAN_INTERCEPTOR(pthread_mutex_trylock, m);
int res = REAL(pthread_mutex_trylock)(m);
- if (res == EOWNERDEAD)
+ if (res == errno_EOWNERDEAD)
MutexRepair(thr, pc, (uptr)m);
- if (res == 0 || res == EOWNERDEAD)
+ if (res == 0 || res == errno_EOWNERDEAD)
MutexPostLock(thr, pc, (uptr)m, MutexFlagTryLock);
return res;
}
@@ -1311,7 +1310,7 @@ TSAN_INTERCEPTOR(int, pthread_barrier_wait, void *b) {
TSAN_INTERCEPTOR(int, pthread_once, void *o, void (*f)()) {
SCOPED_INTERCEPTOR_RAW(pthread_once, o, f);
if (o == 0 || f == 0)
- return EINVAL;
+ return errno_EINVAL;
atomic_uint32_t *a;
if (!SANITIZER_MAC)
a = static_cast<atomic_uint32_t*>(o);
diff --git a/lib/tsan/rtl/tsan_interceptors_mac.cc b/lib/tsan/rtl/tsan_interceptors_mac.cc
index f6bf8a0e586b..4f1079467331 100644
--- a/lib/tsan/rtl/tsan_interceptors_mac.cc
+++ b/lib/tsan/rtl/tsan_interceptors_mac.cc
@@ -21,7 +21,10 @@
#include "tsan_interface_ann.h"
#include <libkern/OSAtomic.h>
+
+#if defined(__has_include) && __has_include(<xpc/xpc.h>)
#include <xpc/xpc.h>
+#endif // #if defined(__has_include) && __has_include(<xpc/xpc.h>)
typedef long long_t; // NOLINT
@@ -235,6 +238,8 @@ TSAN_INTERCEPTOR(void, os_lock_unlock, void *lock) {
REAL(os_lock_unlock)(lock);
}
+#if defined(__has_include) && __has_include(<xpc/xpc.h>)
+
TSAN_INTERCEPTOR(void, xpc_connection_set_event_handler,
xpc_connection_t connection, xpc_handler_t handler) {
SCOPED_TSAN_INTERCEPTOR(xpc_connection_set_event_handler, connection,
@@ -287,6 +292,8 @@ TSAN_INTERCEPTOR(void, xpc_connection_cancel, xpc_connection_t connection) {
REAL(xpc_connection_cancel)(connection);
}
+#endif // #if defined(__has_include) && __has_include(<xpc/xpc.h>)
+
// On macOS, libc++ is always linked dynamically, so intercepting works the
// usual way.
#define STDCXX_INTERCEPTOR TSAN_INTERCEPTOR
diff --git a/lib/tsan/rtl/tsan_interface_ann.cc b/lib/tsan/rtl/tsan_interface_ann.cc
index 45ec45bbdbbe..f68a0468de53 100644
--- a/lib/tsan/rtl/tsan_interface_ann.cc
+++ b/lib/tsan/rtl/tsan_interface_ann.cc
@@ -483,8 +483,8 @@ void __tsan_mutex_pre_lock(void *m, unsigned flagz) {
else
MutexPreLock(thr, pc, (uptr)m);
}
- ThreadIgnoreBegin(thr, pc, false);
- ThreadIgnoreSyncBegin(thr, pc, false);
+ ThreadIgnoreBegin(thr, pc, /*save_stack=*/false);
+ ThreadIgnoreSyncBegin(thr, pc, /*save_stack=*/false);
}
INTERFACE_ATTRIBUTE
@@ -510,8 +510,8 @@ int __tsan_mutex_pre_unlock(void *m, unsigned flagz) {
} else {
ret = MutexUnlock(thr, pc, (uptr)m, flagz);
}
- ThreadIgnoreBegin(thr, pc, false);
- ThreadIgnoreSyncBegin(thr, pc, false);
+ ThreadIgnoreBegin(thr, pc, /*save_stack=*/false);
+ ThreadIgnoreSyncBegin(thr, pc, /*save_stack=*/false);
return ret;
}
@@ -525,8 +525,8 @@ void __tsan_mutex_post_unlock(void *m, unsigned flagz) {
INTERFACE_ATTRIBUTE
void __tsan_mutex_pre_signal(void *addr, unsigned flagz) {
SCOPED_ANNOTATION(__tsan_mutex_pre_signal);
- ThreadIgnoreBegin(thr, pc, false);
- ThreadIgnoreSyncBegin(thr, pc, false);
+ ThreadIgnoreBegin(thr, pc, /*save_stack=*/false);
+ ThreadIgnoreSyncBegin(thr, pc, /*save_stack=*/false);
}
INTERFACE_ATTRIBUTE
@@ -547,7 +547,7 @@ void __tsan_mutex_pre_divert(void *addr, unsigned flagz) {
INTERFACE_ATTRIBUTE
void __tsan_mutex_post_divert(void *addr, unsigned flagz) {
SCOPED_ANNOTATION(__tsan_mutex_post_divert);
- ThreadIgnoreBegin(thr, pc, false);
- ThreadIgnoreSyncBegin(thr, pc, false);
+ ThreadIgnoreBegin(thr, pc, /*save_stack=*/false);
+ ThreadIgnoreSyncBegin(thr, pc, /*save_stack=*/false);
}
} // extern "C"
diff --git a/lib/tsan/rtl/tsan_interface_atomic.cc b/lib/tsan/rtl/tsan_interface_atomic.cc
index b22d5c1ecef8..d334394f5303 100644
--- a/lib/tsan/rtl/tsan_interface_atomic.cc
+++ b/lib/tsan/rtl/tsan_interface_atomic.cc
@@ -220,8 +220,7 @@ static a128 NoTsanAtomicLoad(const volatile a128 *a, morder mo) {
#endif
template<typename T>
-static T AtomicLoad(ThreadState *thr, uptr pc, const volatile T *a,
- morder mo) {
+static T AtomicLoad(ThreadState *thr, uptr pc, const volatile T *a, morder mo) {
CHECK(IsLoadOrder(mo));
// This fast-path is critical for performance.
// Assume the access is atomic.
@@ -229,10 +228,17 @@ static T AtomicLoad(ThreadState *thr, uptr pc, const volatile T *a,
MemoryReadAtomic(thr, pc, (uptr)a, SizeLog<T>());
return NoTsanAtomicLoad(a, mo);
}
- SyncVar *s = ctx->metamap.GetOrCreateAndLock(thr, pc, (uptr)a, false);
- AcquireImpl(thr, pc, &s->clock);
+ // Don't create sync object if it does not exist yet. For example, an atomic
+ // pointer is initialized to nullptr and then periodically acquire-loaded.
T v = NoTsanAtomicLoad(a, mo);
- s->mtx.ReadUnlock();
+ SyncVar *s = ctx->metamap.GetIfExistsAndLock((uptr)a, false);
+ if (s) {
+ AcquireImpl(thr, pc, &s->clock);
+ // Re-read under sync mutex because we need a consistent snapshot
+ // of the value and the clock we acquire.
+ v = NoTsanAtomicLoad(a, mo);
+ s->mtx.ReadUnlock();
+ }
MemoryReadAtomic(thr, pc, (uptr)a, SizeLog<T>());
return v;
}
diff --git a/lib/tsan/rtl/tsan_mman.cc b/lib/tsan/rtl/tsan_mman.cc
index 7169d5b02c04..1434cf688ce9 100644
--- a/lib/tsan/rtl/tsan_mman.cc
+++ b/lib/tsan/rtl/tsan_mman.cc
@@ -294,6 +294,8 @@ uptr __sanitizer_get_allocated_size(const void *p) {
void __tsan_on_thread_idle() {
ThreadState *thr = cur_thread();
+ thr->clock.ResetCached(&thr->proc()->clock_cache);
+ thr->last_sleep_clock.ResetCached(&thr->proc()->clock_cache);
allocator()->SwallowCache(&thr->proc()->alloc_cache);
internal_allocator()->SwallowCache(&thr->proc()->internal_alloc_cache);
ctx->metamap.OnProcIdle(thr->proc());
diff --git a/lib/tsan/rtl/tsan_platform.h b/lib/tsan/rtl/tsan_platform.h
index 60d9b9d8c452..bea1daba3952 100644
--- a/lib/tsan/rtl/tsan_platform.h
+++ b/lib/tsan/rtl/tsan_platform.h
@@ -100,6 +100,37 @@ struct Mapping {
};
#define TSAN_MID_APP_RANGE 1
+#elif defined(__aarch64__) && defined(__APPLE__)
+/*
+C/C++ on Darwin/iOS/ARM64 (36-bit VMA, 64 GB VM)
+0000 0000 00 - 0100 0000 00: - (4 GB)
+0100 0000 00 - 0200 0000 00: main binary, modules, thread stacks (4 GB)
+0200 0000 00 - 0300 0000 00: heap (4 GB)
+0300 0000 00 - 0400 0000 00: - (4 GB)
+0400 0000 00 - 0c00 0000 00: shadow memory (32 GB)
+0c00 0000 00 - 0d00 0000 00: - (4 GB)
+0d00 0000 00 - 0e00 0000 00: metainfo (4 GB)
+0e00 0000 00 - 0f00 0000 00: - (4 GB)
+0f00 0000 00 - 1000 0000 00: traces (4 GB)
+*/
+struct Mapping {
+ static const uptr kLoAppMemBeg = 0x0100000000ull;
+ static const uptr kLoAppMemEnd = 0x0200000000ull;
+ static const uptr kHeapMemBeg = 0x0200000000ull;
+ static const uptr kHeapMemEnd = 0x0300000000ull;
+ static const uptr kShadowBeg = 0x0400000000ull;
+ static const uptr kShadowEnd = 0x0c00000000ull;
+ static const uptr kMetaShadowBeg = 0x0d00000000ull;
+ static const uptr kMetaShadowEnd = 0x0e00000000ull;
+ static const uptr kTraceMemBeg = 0x0f00000000ull;
+ static const uptr kTraceMemEnd = 0x1000000000ull;
+ static const uptr kHiAppMemBeg = 0x1000000000ull;
+ static const uptr kHiAppMemEnd = 0x1000000000ull;
+ static const uptr kAppMemMsk = 0x0ull;
+ static const uptr kAppMemXor = 0x0ull;
+ static const uptr kVdsoBeg = 0x7000000000000000ull;
+};
+
#elif defined(__aarch64__)
// AArch64 supports multiple VMA which leads to multiple address transformation
// functions. To support these multiple VMAS transformations and mappings TSAN
@@ -389,7 +420,7 @@ uptr MappingImpl(void) {
template<int Type>
uptr MappingArchImpl(void) {
-#ifdef __aarch64__
+#if defined(__aarch64__) && !defined(__APPLE__)
switch (vmaSize) {
case 39: return MappingImpl<Mapping39, Type>();
case 42: return MappingImpl<Mapping42, Type>();
@@ -542,7 +573,7 @@ bool IsAppMemImpl(uptr mem) {
ALWAYS_INLINE
bool IsAppMem(uptr mem) {
-#ifdef __aarch64__
+#if defined(__aarch64__) && !defined(__APPLE__)
switch (vmaSize) {
case 39: return IsAppMemImpl<Mapping39>(mem);
case 42: return IsAppMemImpl<Mapping42>(mem);
@@ -569,7 +600,7 @@ bool IsShadowMemImpl(uptr mem) {
ALWAYS_INLINE
bool IsShadowMem(uptr mem) {
-#ifdef __aarch64__
+#if defined(__aarch64__) && !defined(__APPLE__)
switch (vmaSize) {
case 39: return IsShadowMemImpl<Mapping39>(mem);
case 42: return IsShadowMemImpl<Mapping42>(mem);
@@ -596,7 +627,7 @@ bool IsMetaMemImpl(uptr mem) {
ALWAYS_INLINE
bool IsMetaMem(uptr mem) {
-#ifdef __aarch64__
+#if defined(__aarch64__) && !defined(__APPLE__)
switch (vmaSize) {
case 39: return IsMetaMemImpl<Mapping39>(mem);
case 42: return IsMetaMemImpl<Mapping42>(mem);
@@ -633,7 +664,7 @@ uptr MemToShadowImpl(uptr x) {
ALWAYS_INLINE
uptr MemToShadow(uptr x) {
-#ifdef __aarch64__
+#if defined(__aarch64__) && !defined(__APPLE__)
switch (vmaSize) {
case 39: return MemToShadowImpl<Mapping39>(x);
case 42: return MemToShadowImpl<Mapping42>(x);
@@ -672,7 +703,7 @@ u32 *MemToMetaImpl(uptr x) {
ALWAYS_INLINE
u32 *MemToMeta(uptr x) {
-#ifdef __aarch64__
+#if defined(__aarch64__) && !defined(__APPLE__)
switch (vmaSize) {
case 39: return MemToMetaImpl<Mapping39>(x);
case 42: return MemToMetaImpl<Mapping42>(x);
@@ -724,7 +755,7 @@ uptr ShadowToMemImpl(uptr s) {
ALWAYS_INLINE
uptr ShadowToMem(uptr s) {
-#ifdef __aarch64__
+#if defined(__aarch64__) && !defined(__APPLE__)
switch (vmaSize) {
case 39: return ShadowToMemImpl<Mapping39>(s);
case 42: return ShadowToMemImpl<Mapping42>(s);
@@ -759,7 +790,7 @@ uptr GetThreadTraceImpl(int tid) {
ALWAYS_INLINE
uptr GetThreadTrace(int tid) {
-#ifdef __aarch64__
+#if defined(__aarch64__) && !defined(__APPLE__)
switch (vmaSize) {
case 39: return GetThreadTraceImpl<Mapping39>(tid);
case 42: return GetThreadTraceImpl<Mapping42>(tid);
@@ -789,7 +820,7 @@ uptr GetThreadTraceHeaderImpl(int tid) {
ALWAYS_INLINE
uptr GetThreadTraceHeader(int tid) {
-#ifdef __aarch64__
+#if defined(__aarch64__) && !defined(__APPLE__)
switch (vmaSize) {
case 39: return GetThreadTraceHeaderImpl<Mapping39>(tid);
case 42: return GetThreadTraceHeaderImpl<Mapping42>(tid);
diff --git a/lib/tsan/rtl/tsan_platform_linux.cc b/lib/tsan/rtl/tsan_platform_linux.cc
index d05c0e701e72..0ba01babe69a 100644
--- a/lib/tsan/rtl/tsan_platform_linux.cc
+++ b/lib/tsan/rtl/tsan_platform_linux.cc
@@ -47,7 +47,6 @@
#include <sys/resource.h>
#include <sys/stat.h>
#include <unistd.h>
-#include <errno.h>
#include <sched.h>
#include <dlfcn.h>
#if SANITIZER_LINUX
@@ -182,17 +181,15 @@ static void MapRodata() {
}
// Map the file into shadow of .rodata sections.
MemoryMappingLayout proc_maps(/*cache_enabled*/true);
- uptr start, end, offset, prot;
// Reusing the buffer 'name'.
- while (proc_maps.Next(&start, &end, &offset, name, ARRAY_SIZE(name), &prot)) {
- if (name[0] != 0 && name[0] != '['
- && (prot & MemoryMappingLayout::kProtectionRead)
- && (prot & MemoryMappingLayout::kProtectionExecute)
- && !(prot & MemoryMappingLayout::kProtectionWrite)
- && IsAppMem(start)) {
+ MemoryMappedSegment segment(name, ARRAY_SIZE(name));
+ while (proc_maps.Next(&segment)) {
+ if (segment.filename[0] != 0 && segment.filename[0] != '[' &&
+ segment.IsReadable() && segment.IsExecutable() &&
+ !segment.IsWritable() && IsAppMem(segment.start)) {
// Assume it's .rodata
- char *shadow_start = (char*)MemToShadow(start);
- char *shadow_end = (char*)MemToShadow(end);
+ char *shadow_start = (char *)MemToShadow(segment.start);
+ char *shadow_end = (char *)MemToShadow(segment.end);
for (char *p = shadow_start; p < shadow_end; p += marker.size()) {
internal_mmap(p, Min<uptr>(marker.size(), shadow_end - p),
PROT_READ, MAP_PRIVATE | MAP_FIXED, fd, 0);
diff --git a/lib/tsan/rtl/tsan_platform_mac.cc b/lib/tsan/rtl/tsan_platform_mac.cc
index a82bcd01bbf4..73a656ffca5e 100644
--- a/lib/tsan/rtl/tsan_platform_mac.cc
+++ b/lib/tsan/rtl/tsan_platform_mac.cc
@@ -230,6 +230,14 @@ static void my_pthread_introspection_hook(unsigned int event, pthread_t thread,
#endif
void InitializePlatformEarly() {
+#if defined(__aarch64__)
+ uptr max_vm = GetMaxVirtualAddress() + 1;
+ if (max_vm != Mapping::kHiAppMemEnd) {
+ Printf("ThreadSanitizer: unsupported vm address limit %p, expected %p.\n",
+ max_vm, Mapping::kHiAppMemEnd);
+ Die();
+ }
+#endif
}
void InitializePlatform() {
diff --git a/lib/tsan/rtl/tsan_platform_posix.cc b/lib/tsan/rtl/tsan_platform_posix.cc
index 0732c83d689d..e4f90a811c35 100644
--- a/lib/tsan/rtl/tsan_platform_posix.cc
+++ b/lib/tsan/rtl/tsan_platform_posix.cc
@@ -46,6 +46,9 @@ void InitializeShadowMemory() {
#elif defined(__mips64)
const uptr kMadviseRangeBeg = 0xff00000000ull;
const uptr kMadviseRangeSize = 0x0100000000ull;
+#elif defined(__aarch64__) && defined(__APPLE__)
+ uptr kMadviseRangeBeg = LoAppMemBeg();
+ uptr kMadviseRangeSize = LoAppMemEnd() - LoAppMemBeg();
#elif defined(__aarch64__)
uptr kMadviseRangeBeg = 0;
uptr kMadviseRangeSize = 0;
@@ -115,21 +118,24 @@ static void ProtectRange(uptr beg, uptr end) {
void CheckAndProtect() {
// Ensure that the binary is indeed compiled with -pie.
MemoryMappingLayout proc_maps(true);
- uptr p, end, prot;
- while (proc_maps.Next(&p, &end, 0, 0, 0, &prot)) {
- if (IsAppMem(p))
+ MemoryMappedSegment segment;
+ while (proc_maps.Next(&segment)) {
+ if (IsAppMem(segment.start)) continue;
+ if (segment.start >= HeapMemEnd() && segment.start < HeapEnd()) continue;
+ if (segment.protection == 0) // Zero page or mprotected.
continue;
- if (p >= HeapMemEnd() &&
- p < HeapEnd())
- continue;
- if (prot == 0) // Zero page or mprotected.
- continue;
- if (p >= VdsoBeg()) // vdso
+ if (segment.start >= VdsoBeg()) // vdso
break;
- Printf("FATAL: ThreadSanitizer: unexpected memory mapping %p-%p\n", p, end);
+ Printf("FATAL: ThreadSanitizer: unexpected memory mapping %p-%p\n",
+ segment.start, segment.end);
Die();
}
+#if defined(__aarch64__) && defined(__APPLE__)
+ ProtectRange(HeapMemEnd(), ShadowBeg());
+ ProtectRange(ShadowEnd(), MetaShadowBeg());
+ ProtectRange(MetaShadowEnd(), TraceMemBeg());
+#else
ProtectRange(LoAppMemEnd(), ShadowBeg());
ProtectRange(ShadowEnd(), MetaShadowBeg());
#ifdef TSAN_MID_APP_RANGE
@@ -143,6 +149,7 @@ void CheckAndProtect() {
ProtectRange(TraceMemBeg(), TraceMemEnd());
ProtectRange(TraceMemEnd(), HeapMemBeg());
ProtectRange(HeapEnd(), HiAppMemBeg());
+#endif
}
#endif
diff --git a/lib/tsan/rtl/tsan_rtl.cc b/lib/tsan/rtl/tsan_rtl.cc
index fa60f3247c38..a01525302b02 100644
--- a/lib/tsan/rtl/tsan_rtl.cc
+++ b/lib/tsan/rtl/tsan_rtl.cc
@@ -104,7 +104,8 @@ Context::Context()
, racy_stacks(MBlockRacyStacks)
, racy_addresses(MBlockRacyAddresses)
, fired_suppressions_mtx(MutexTypeFired, StatMtxFired)
- , fired_suppressions(8) {
+ , fired_suppressions(8)
+ , clock_alloc("clock allocator") {
}
// The objects are allocated in TLS, so one may rely on zero-initialization.
diff --git a/lib/tsan/rtl/tsan_rtl_aarch64.S b/lib/tsan/rtl/tsan_rtl_aarch64.S
index ef06f0444ae4..61171d635c18 100644
--- a/lib/tsan/rtl/tsan_rtl_aarch64.S
+++ b/lib/tsan/rtl/tsan_rtl_aarch64.S
@@ -1,13 +1,46 @@
+// The content of this file is AArch64-only:
+#if defined(__aarch64__)
+
#include "sanitizer_common/sanitizer_asm.h"
+#if !defined(__APPLE__)
.section .bss
.type __tsan_pointer_chk_guard, %object
-.size __tsan_pointer_chk_guard, 8
+ASM_SIZE(ASM_TSAN_SYMBOL_INTERCEPTOR(__tsan_pointer_chk_guard))
__tsan_pointer_chk_guard:
.zero 8
+#endif
+
+#if defined(__APPLE__)
+.align 2
+
+.section __DATA,__nl_symbol_ptr,non_lazy_symbol_pointers
+.long _setjmp$non_lazy_ptr
+_setjmp$non_lazy_ptr:
+.indirect_symbol _setjmp
+.long 0
+
+.section __DATA,__nl_symbol_ptr,non_lazy_symbol_pointers
+.long __setjmp$non_lazy_ptr
+__setjmp$non_lazy_ptr:
+.indirect_symbol __setjmp
+.long 0
+
+.section __DATA,__nl_symbol_ptr,non_lazy_symbol_pointers
+.long _sigsetjmp$non_lazy_ptr
+_sigsetjmp$non_lazy_ptr:
+.indirect_symbol _sigsetjmp
+.long 0
+#endif
+#if !defined(__APPLE__)
.section .text
+#else
+.section __TEXT,__text
+.align 3
+#endif
+#if !defined(__APPLE__)
// GLIBC mangles the function pointers in jmp_buf (used in {set,long}*jmp
// functions) by XORing them with a random guard pointer. For AArch64 it is a
// global variable rather than a TCB one (as for x86_64/powerpc) and althought
@@ -16,9 +49,9 @@ __tsan_pointer_chk_guard:
// not stable). So InitializeGuardPtr obtains the pointer guard value by
// issuing a setjmp and checking the resulting pointers values against the
// original ones.
-.hidden _Z18InitializeGuardPtrv
+ASM_HIDDEN(_Z18InitializeGuardPtrv)
.global _Z18InitializeGuardPtrv
-.type _Z18InitializeGuardPtrv, @function
+ASM_TYPE_FUNCTION(ASM_TSAN_SYMBOL_INTERCEPTOR(_Z18InitializeGuardPtrv))
_Z18InitializeGuardPtrv:
CFI_STARTPROC
// Allocates a jmp_buf for the setjmp call.
@@ -55,12 +88,14 @@ _Z18InitializeGuardPtrv:
CFI_DEF_CFA (31, 0)
ret
CFI_ENDPROC
-.size _Z18InitializeGuardPtrv, .-_Z18InitializeGuardPtrv
+ASM_SIZE(ASM_TSAN_SYMBOL_INTERCEPTOR(_Z18InitializeGuardPtrv))
+#endif
-.hidden __tsan_setjmp
+ASM_HIDDEN(__tsan_setjmp)
.comm _ZN14__interception11real_setjmpE,8,8
-.type setjmp, @function
-setjmp:
+.globl ASM_TSAN_SYMBOL_INTERCEPTOR(setjmp)
+ASM_TYPE_FUNCTION(ASM_TSAN_SYMBOL_INTERCEPTOR(setjmp))
+ASM_TSAN_SYMBOL_INTERCEPTOR(setjmp):
CFI_STARTPROC
// save env parameters for function call
@@ -78,14 +113,19 @@ setjmp:
CFI_OFFSET (19, -16)
mov x19, x0
+#if !defined(__APPLE__)
// SP pointer mangling (see glibc setjmp)
adrp x2, __tsan_pointer_chk_guard
ldr x2, [x2, #:lo12:__tsan_pointer_chk_guard]
add x0, x29, 32
eor x1, x2, x0
+#else
+ add x0, x29, 32
+ mov x1, x0
+#endif
// call tsan interceptor
- bl __tsan_setjmp
+ bl ASM_TSAN_SYMBOL(__tsan_setjmp)
// restore env parameter
mov x0, x19
@@ -96,18 +136,24 @@ setjmp:
CFI_DEF_CFA (31, 0)
// tail jump to libc setjmp
+#if !defined(__APPLE__)
adrp x1, :got:_ZN14__interception11real_setjmpE
ldr x1, [x1, #:got_lo12:_ZN14__interception11real_setjmpE]
ldr x1, [x1]
+#else
+ adrp x1, _setjmp$non_lazy_ptr@page
+ add x1, x1, _setjmp$non_lazy_ptr@pageoff
+ ldr x1, [x1]
+#endif
br x1
CFI_ENDPROC
-.size setjmp, .-setjmp
+ASM_SIZE(ASM_TSAN_SYMBOL_INTERCEPTOR(setjmp))
.comm _ZN14__interception12real__setjmpE,8,8
-.globl _setjmp
-.type _setjmp, @function
-_setjmp:
+.globl ASM_TSAN_SYMBOL_INTERCEPTOR(_setjmp)
+ASM_TYPE_FUNCTION(ASM_TSAN_SYMBOL_INTERCEPTOR(_setjmp))
+ASM_TSAN_SYMBOL_INTERCEPTOR(_setjmp):
CFI_STARTPROC
// save env parameters for function call
@@ -125,14 +171,19 @@ _setjmp:
CFI_OFFSET (19, -16)
mov x19, x0
+#if !defined(__APPLE__)
// SP pointer mangling (see glibc setjmp)
adrp x2, __tsan_pointer_chk_guard
ldr x2, [x2, #:lo12:__tsan_pointer_chk_guard]
add x0, x29, 32
eor x1, x2, x0
+#else
+ add x0, x29, 32
+ mov x1, x0
+#endif
// call tsan interceptor
- bl __tsan_setjmp
+ bl ASM_TSAN_SYMBOL(__tsan_setjmp)
// Restore jmp_buf parameter
mov x0, x19
@@ -143,18 +194,24 @@ _setjmp:
CFI_DEF_CFA (31, 0)
// tail jump to libc setjmp
+#if !defined(__APPLE__)
adrp x1, :got:_ZN14__interception12real__setjmpE
ldr x1, [x1, #:got_lo12:_ZN14__interception12real__setjmpE]
ldr x1, [x1]
+#else
+ adrp x1, __setjmp$non_lazy_ptr@page
+ add x1, x1, __setjmp$non_lazy_ptr@pageoff
+ ldr x1, [x1]
+#endif
br x1
CFI_ENDPROC
-.size _setjmp, .-_setjmp
+ASM_SIZE(ASM_TSAN_SYMBOL_INTERCEPTOR(_setjmp))
.comm _ZN14__interception14real_sigsetjmpE,8,8
-.globl sigsetjmp
-.type sigsetjmp, @function
-sigsetjmp:
+.globl ASM_TSAN_SYMBOL_INTERCEPTOR(sigsetjmp)
+ASM_TYPE_FUNCTION(ASM_TSAN_SYMBOL_INTERCEPTOR(sigsetjmp))
+ASM_TSAN_SYMBOL_INTERCEPTOR(sigsetjmp):
CFI_STARTPROC
// save env parameters for function call
@@ -174,14 +231,19 @@ sigsetjmp:
mov w20, w1
mov x19, x0
+#if !defined(__APPLE__)
// SP pointer mangling (see glibc setjmp)
adrp x2, __tsan_pointer_chk_guard
ldr x2, [x2, #:lo12:__tsan_pointer_chk_guard]
add x0, x29, 32
eor x1, x2, x0
+#else
+ add x0, x29, 32
+ mov x1, x0
+#endif
// call tsan interceptor
- bl __tsan_setjmp
+ bl ASM_TSAN_SYMBOL(__tsan_setjmp)
// restore env parameter
mov w1, w20
@@ -195,17 +257,24 @@ sigsetjmp:
CFI_DEF_CFA (31, 0)
// tail jump to libc sigsetjmp
+#if !defined(__APPLE__)
adrp x2, :got:_ZN14__interception14real_sigsetjmpE
ldr x2, [x2, #:got_lo12:_ZN14__interception14real_sigsetjmpE]
ldr x2, [x2]
+#else
+ adrp x2, _sigsetjmp$non_lazy_ptr@page
+ add x2, x2, _sigsetjmp$non_lazy_ptr@pageoff
+ ldr x2, [x2]
+#endif
br x2
CFI_ENDPROC
-.size sigsetjmp, .-sigsetjmp
+ASM_SIZE(ASM_TSAN_SYMBOL_INTERCEPTOR(sigsetjmp))
+#if !defined(__APPLE__)
.comm _ZN14__interception16real___sigsetjmpE,8,8
-.globl __sigsetjmp
-.type __sigsetjmp, @function
-__sigsetjmp:
+.globl ASM_TSAN_SYMBOL_INTERCEPTOR(__sigsetjmp)
+ASM_TYPE_FUNCTION(ASM_TSAN_SYMBOL_INTERCEPTOR(__sigsetjmp))
+ASM_TSAN_SYMBOL_INTERCEPTOR(__sigsetjmp):
CFI_STARTPROC
// save env parameters for function call
@@ -225,14 +294,16 @@ __sigsetjmp:
mov w20, w1
mov x19, x0
+#if !defined(__APPLE__)
// SP pointer mangling (see glibc setjmp)
adrp x2, __tsan_pointer_chk_guard
ldr x2, [x2, #:lo12:__tsan_pointer_chk_guard]
add x0, x29, 32
eor x1, x2, x0
+#endif
// call tsan interceptor
- bl __tsan_setjmp
+ bl ASM_TSAN_SYMBOL(__tsan_setjmp)
mov w1, w20
mov x0, x19
@@ -245,14 +316,22 @@ __sigsetjmp:
CFI_DEF_CFA (31, 0)
// tail jump to libc __sigsetjmp
+#if !defined(__APPLE__)
adrp x2, :got:_ZN14__interception16real___sigsetjmpE
ldr x2, [x2, #:got_lo12:_ZN14__interception16real___sigsetjmpE]
ldr x2, [x2]
+#else
+ adrp x2, ASM_TSAN_SYMBOL(__sigsetjmp)@page
+ add x2, x2, ASM_TSAN_SYMBOL(__sigsetjmp)@pageoff
+#endif
br x2
CFI_ENDPROC
-.size __sigsetjmp, .-__sigsetjmp
+ASM_SIZE(ASM_TSAN_SYMBOL_INTERCEPTOR(__sigsetjmp))
+#endif
#if defined(__linux__)
/* We do not need executable stack. */
.section .note.GNU-stack,"",@progbits
#endif
+
+#endif
diff --git a/lib/tsan/rtl/tsan_rtl_amd64.S b/lib/tsan/rtl/tsan_rtl_amd64.S
index caa832375e52..98947fd2a1ba 100644
--- a/lib/tsan/rtl/tsan_rtl_amd64.S
+++ b/lib/tsan/rtl/tsan_rtl_amd64.S
@@ -1,4 +1,8 @@
+// The content of this file is x86_64-only:
+#if defined(__x86_64__)
+
#include "sanitizer_common/sanitizer_asm.h"
+
#if !defined(__APPLE__)
.section .text
#else
@@ -357,3 +361,5 @@ ASM_SIZE(ASM_TSAN_SYMBOL_INTERCEPTOR(__sigsetjmp))
/* We do not need executable stack. */
.section .note.GNU-stack,"",@progbits
#endif
+
+#endif
diff --git a/lib/tsan/rtl/tsan_rtl_mutex.cc b/lib/tsan/rtl/tsan_rtl_mutex.cc
index 54938f37e243..2f85811620f1 100644
--- a/lib/tsan/rtl/tsan_rtl_mutex.cc
+++ b/lib/tsan/rtl/tsan_rtl_mutex.cc
@@ -413,10 +413,10 @@ void Acquire(ThreadState *thr, uptr pc, uptr addr) {
static void UpdateClockCallback(ThreadContextBase *tctx_base, void *arg) {
ThreadState *thr = reinterpret_cast<ThreadState*>(arg);
ThreadContext *tctx = static_cast<ThreadContext*>(tctx_base);
+ u64 epoch = tctx->epoch1;
if (tctx->status == ThreadStatusRunning)
- thr->clock.set(tctx->tid, tctx->thr->fast_state.epoch());
- else
- thr->clock.set(tctx->tid, tctx->epoch1);
+ epoch = tctx->thr->fast_state.epoch();
+ thr->clock.set(&thr->proc()->clock_cache, tctx->tid, epoch);
}
void AcquireGlobal(ThreadState *thr, uptr pc) {
@@ -456,10 +456,10 @@ void ReleaseStore(ThreadState *thr, uptr pc, uptr addr) {
static void UpdateSleepClockCallback(ThreadContextBase *tctx_base, void *arg) {
ThreadState *thr = reinterpret_cast<ThreadState*>(arg);
ThreadContext *tctx = static_cast<ThreadContext*>(tctx_base);
+ u64 epoch = tctx->epoch1;
if (tctx->status == ThreadStatusRunning)
- thr->last_sleep_clock.set(tctx->tid, tctx->thr->fast_state.epoch());
- else
- thr->last_sleep_clock.set(tctx->tid, tctx->epoch1);
+ epoch = tctx->thr->fast_state.epoch();
+ thr->last_sleep_clock.set(&thr->proc()->clock_cache, tctx->tid, epoch);
}
void AfterSleep(ThreadState *thr, uptr pc) {
diff --git a/lib/tsan/rtl/tsan_rtl_report.cc b/lib/tsan/rtl/tsan_rtl_report.cc
index 68b9f50308ea..85a982941ed7 100644
--- a/lib/tsan/rtl/tsan_rtl_report.cc
+++ b/lib/tsan/rtl/tsan_rtl_report.cc
@@ -314,7 +314,7 @@ void ScopedReport::AddLocation(uptr addr, uptr size) {
return;
#if !SANITIZER_GO
int fd = -1;
- int creat_tid = -1;
+ int creat_tid = kInvalidTid;
u32 creat_stack = 0;
if (FdLocation(addr, &fd, &creat_tid, &creat_stack)) {
ReportLocation *loc = ReportLocation::New(ReportLocationFD);
diff --git a/lib/tsan/rtl/tsan_rtl_thread.cc b/lib/tsan/rtl/tsan_rtl_thread.cc
index 67eebf5d0c38..83fab082afe3 100644
--- a/lib/tsan/rtl/tsan_rtl_thread.cc
+++ b/lib/tsan/rtl/tsan_rtl_thread.cc
@@ -142,6 +142,10 @@ void ThreadContext::OnFinished() {
if (common_flags()->detect_deadlocks)
ctx->dd->DestroyLogicalThread(thr->dd_lt);
+ thr->clock.ResetCached(&thr->proc()->clock_cache);
+#if !SANITIZER_GO
+ thr->last_sleep_clock.ResetCached(&thr->proc()->clock_cache);
+#endif
thr->~ThreadState();
#if TSAN_COLLECT_STATS
StatAggregate(ctx->stat, thr->stat);
diff --git a/lib/tsan/rtl/tsan_stat.cc b/lib/tsan/rtl/tsan_stat.cc
index 2ee688bf5771..18c83d5c6dac 100644
--- a/lib/tsan/rtl/tsan_stat.cc
+++ b/lib/tsan/rtl/tsan_stat.cc
@@ -75,14 +75,11 @@ void StatOutput(u64 *stat) {
name[StatClockAcquire] = "Clock acquire ";
name[StatClockAcquireEmpty] = " empty clock ";
name[StatClockAcquireFastRelease] = " fast from release-store ";
- name[StatClockAcquireLarge] = " contains my tid ";
- name[StatClockAcquireRepeat] = " repeated (fast) ";
name[StatClockAcquireFull] = " full (slow) ";
name[StatClockAcquiredSomething] = " acquired something ";
name[StatClockRelease] = "Clock release ";
name[StatClockReleaseResize] = " resize ";
- name[StatClockReleaseFast1] = " fast1 ";
- name[StatClockReleaseFast2] = " fast2 ";
+ name[StatClockReleaseFast] = " fast ";
name[StatClockReleaseSlow] = " dirty overflow (slow) ";
name[StatClockReleaseFull] = " full (slow) ";
name[StatClockReleaseAcquired] = " was acquired ";
diff --git a/lib/tsan/rtl/tsan_stat.h b/lib/tsan/rtl/tsan_stat.h
index 7d2791ebbfcc..42d6a2b63532 100644
--- a/lib/tsan/rtl/tsan_stat.h
+++ b/lib/tsan/rtl/tsan_stat.h
@@ -74,15 +74,12 @@ enum StatType {
StatClockAcquire,
StatClockAcquireEmpty,
StatClockAcquireFastRelease,
- StatClockAcquireLarge,
- StatClockAcquireRepeat,
StatClockAcquireFull,
StatClockAcquiredSomething,
// Clocks - release.
StatClockRelease,
StatClockReleaseResize,
- StatClockReleaseFast1,
- StatClockReleaseFast2,
+ StatClockReleaseFast,
StatClockReleaseSlow,
StatClockReleaseFull,
StatClockReleaseAcquired,
diff --git a/lib/tsan/rtl/tsan_sync.cc b/lib/tsan/rtl/tsan_sync.cc
index 4cc3cb89c34f..44ae558fa1b2 100644
--- a/lib/tsan/rtl/tsan_sync.cc
+++ b/lib/tsan/rtl/tsan_sync.cc
@@ -53,7 +53,9 @@ void SyncVar::Reset(Processor *proc) {
}
}
-MetaMap::MetaMap() {
+MetaMap::MetaMap()
+ : block_alloc_("heap block allocator")
+ , sync_alloc_("sync allocator") {
atomic_store(&uid_gen_, 0, memory_order_relaxed);
}
diff --git a/lib/tsan/tests/CMakeLists.txt b/lib/tsan/tests/CMakeLists.txt
index ca43a928d1b8..f8aec6854bb3 100644
--- a/lib/tsan/tests/CMakeLists.txt
+++ b/lib/tsan/tests/CMakeLists.txt
@@ -15,6 +15,7 @@ set(TSAN_UNITTEST_CFLAGS
if(APPLE)
list(APPEND TSAN_UNITTEST_CFLAGS ${DARWIN_osx_CFLAGS})
+ list(APPEND TSAN_UNITTEST_LINKFLAGS ${DARWIN_osx_LINKFLAGS})
endif()
set(TSAN_RTL_HEADERS)
diff --git a/lib/tsan/tests/unit/tsan_clock_test.cc b/lib/tsan/tests/unit/tsan_clock_test.cc
index 83e25fb5a933..73104dd6b9d4 100644
--- a/lib/tsan/tests/unit/tsan_clock_test.cc
+++ b/lib/tsan/tests/unit/tsan_clock_test.cc
@@ -26,13 +26,13 @@ TEST(Clock, VectorBasic) {
clk.tick();
ASSERT_EQ(clk.size(), 1U);
ASSERT_EQ(clk.get(0), 1U);
- clk.set(3, clk.get(3) + 1);
+ clk.set(&cache, 3, clk.get(3) + 1);
ASSERT_EQ(clk.size(), 4U);
ASSERT_EQ(clk.get(0), 1U);
ASSERT_EQ(clk.get(1), 0U);
ASSERT_EQ(clk.get(2), 0U);
ASSERT_EQ(clk.get(3), 1U);
- clk.set(3, clk.get(3) + 1);
+ clk.set(&cache, 3, clk.get(3) + 1);
ASSERT_EQ(clk.get(3), 2U);
}
@@ -86,24 +86,26 @@ TEST(Clock, RepeatedAcquire) {
TEST(Clock, ManyThreads) {
SyncClock chunked;
- for (unsigned i = 0; i < 100; i++) {
+ for (unsigned i = 0; i < 200; i++) {
ThreadClock vector(0);
vector.tick();
- vector.set(i, 1);
+ vector.set(&cache, i, i + 1);
vector.release(&cache, &chunked);
ASSERT_EQ(i + 1, chunked.size());
vector.acquire(&cache, &chunked);
ASSERT_EQ(i + 1, vector.size());
}
- for (unsigned i = 0; i < 100; i++)
- ASSERT_EQ(1U, chunked.get(i));
+ for (unsigned i = 0; i < 200; i++) {
+ printf("i=%d\n", i);
+ ASSERT_EQ(i + 1, chunked.get(i));
+ }
ThreadClock vector(1);
vector.acquire(&cache, &chunked);
- ASSERT_EQ(100U, vector.size());
- for (unsigned i = 0; i < 100; i++)
- ASSERT_EQ(1U, vector.get(i));
+ ASSERT_EQ(200U, vector.size());
+ for (unsigned i = 0; i < 200; i++)
+ ASSERT_EQ(i + 1, vector.get(i));
chunked.Reset(&cache);
}
@@ -151,7 +153,7 @@ TEST(Clock, Growth) {
{
ThreadClock vector(10);
vector.tick();
- vector.set(5, 42);
+ vector.set(&cache, 5, 42);
SyncClock sync;
vector.release(&cache, &sync);
ASSERT_EQ(sync.size(), 11U);
@@ -180,8 +182,8 @@ TEST(Clock, Growth) {
{
ThreadClock vector(100);
vector.tick();
- vector.set(5, 42);
- vector.set(90, 84);
+ vector.set(&cache, 5, 42);
+ vector.set(&cache, 90, 84);
SyncClock sync;
vector.release(&cache, &sync);
ASSERT_EQ(sync.size(), 101U);
@@ -212,6 +214,42 @@ TEST(Clock, Growth) {
}
}
+TEST(Clock, Growth2) {
+ // Test clock growth for every pair of sizes:
+ const uptr sizes[] = {0, 1, 2, 30, 61, 62, 63, 64, 65, 66, 100, 124, 125, 126,
+ 127, 128, 129, 130, 188, 189, 190, 191, 192, 193, 254, 255};
+ const uptr n = sizeof(sizes) / sizeof(sizes[0]);
+ for (uptr fi = 0; fi < n; fi++) {
+ for (uptr ti = fi + 1; ti < n; ti++) {
+ const uptr from = sizes[fi];
+ const uptr to = sizes[ti];
+ SyncClock sync;
+ ThreadClock vector(0);
+ for (uptr i = 0; i < from; i++)
+ vector.set(&cache, i, i + 1);
+ if (from != 0)
+ vector.release(&cache, &sync);
+ ASSERT_EQ(sync.size(), from);
+ for (uptr i = 0; i < from; i++)
+ ASSERT_EQ(sync.get(i), i + 1);
+ for (uptr i = 0; i < to; i++)
+ vector.set(&cache, i, i + 1);
+ vector.release(&cache, &sync);
+ ASSERT_EQ(sync.size(), to);
+ for (uptr i = 0; i < to; i++)
+ ASSERT_EQ(sync.get(i), i + 1);
+ vector.set(&cache, to + 1, to + 1);
+ vector.release(&cache, &sync);
+ ASSERT_EQ(sync.size(), to + 2);
+ for (uptr i = 0; i < to; i++)
+ ASSERT_EQ(sync.get(i), i + 1);
+ ASSERT_EQ(sync.get(to), 0U);
+ ASSERT_EQ(sync.get(to + 1), to + 1);
+ sync.Reset(&cache);
+ }
+ }
+}
+
const uptr kThreads = 4;
const uptr kClocks = 4;