diff options
Diffstat (limited to 'lib')
329 files changed, 12260 insertions, 4089 deletions
diff --git a/lib/CMakeLists.txt b/lib/CMakeLists.txt index 009c59f4d68e..4bc6f7a2d576 100644 --- a/lib/CMakeLists.txt +++ b/lib/CMakeLists.txt @@ -4,39 +4,47 @@ include(AddCompilerRT) include(SanitizerUtils) -if(COMPILER_RT_HAS_INTERCEPTION) - add_subdirectory(interception) +if(COMPILER_RT_BUILD_BUILTINS) + add_subdirectory(builtins) endif() -if(COMPILER_RT_HAS_SANITIZER_COMMON) - add_subdirectory(sanitizer_common) - add_subdirectory(lsan) - add_subdirectory(ubsan) -endif() +if(COMPILER_RT_BUILD_SANITIZERS) + if(COMPILER_RT_HAS_INTERCEPTION) + add_subdirectory(interception) + endif() -if(COMPILER_RT_HAS_ASAN) - add_subdirectory(asan) -endif() + if(COMPILER_RT_HAS_SANITIZER_COMMON) + add_subdirectory(sanitizer_common) + add_subdirectory(lsan) + add_subdirectory(ubsan) + endif() -add_subdirectory(builtins) + if(COMPILER_RT_HAS_ASAN) + add_subdirectory(asan) + endif() -if(COMPILER_RT_HAS_DFSAN) - add_subdirectory(dfsan) -endif() + if(COMPILER_RT_HAS_DFSAN) + add_subdirectory(dfsan) + endif() -if(COMPILER_RT_HAS_MSAN) - add_subdirectory(msan) -endif() + if(COMPILER_RT_HAS_MSAN) + add_subdirectory(msan) + endif() -if(COMPILER_RT_HAS_PROFILE) - add_subdirectory(profile) -endif() + if(COMPILER_RT_HAS_PROFILE) + add_subdirectory(profile) + endif() -if(COMPILER_RT_HAS_TSAN) - add_subdirectory(tsan) - add_subdirectory(tsan/dd) -endif() + if(COMPILER_RT_HAS_TSAN) + add_subdirectory(tsan) + add_subdirectory(tsan/dd) + endif() + + if(COMPILER_RT_HAS_SAFESTACK) + add_subdirectory(safestack) + endif() -if(COMPILER_RT_HAS_SAFESTACK) - add_subdirectory(safestack) + if(COMPILER_RT_HAS_CFI) + add_subdirectory(cfi) + endif() endif() diff --git a/lib/asan/.clang-format b/lib/asan/.clang-format new file mode 100644 index 000000000000..f6cb8ad931f5 --- /dev/null +++ b/lib/asan/.clang-format @@ -0,0 +1 @@ +BasedOnStyle: Google diff --git a/lib/asan/CMakeLists.txt b/lib/asan/CMakeLists.txt index 28611a8a4659..6716f48b22bd 100644 --- a/lib/asan/CMakeLists.txt +++ b/lib/asan/CMakeLists.txt @@ -70,18 +70,18 @@ append_list_if(COMPILER_RT_HAS_LIBRT rt ASAN_DYNAMIC_LIBS) append_list_if(COMPILER_RT_HAS_LIBM m ASAN_DYNAMIC_LIBS) append_list_if(COMPILER_RT_HAS_LIBPTHREAD pthread ASAN_DYNAMIC_LIBS) append_list_if(COMPILER_RT_HAS_LIBSTDCXX stdc++ ASAN_DYNAMIC_LIBS) - -append_list_if(ANDROID log ASAN_DYNAMIC_LIBS) +append_list_if(COMPILER_RT_HAS_LIBLOG log ASAN_DYNAMIC_LIBS) # Compile ASan sources into an object library. -if(APPLE) - add_compiler_rt_object_libraries(RTAsan - OS ${SANITIZER_COMMON_SUPPORTED_OS} - ARCHS ${ASAN_SUPPORTED_ARCH} - SOURCES ${ASAN_SOURCES} ${ASAN_CXX_SOURCES} - CFLAGS ${ASAN_DYNAMIC_CFLAGS} - DEFS ${ASAN_DYNAMIC_DEFINITIONS}) -else() + +add_compiler_rt_object_libraries(RTAsan_dynamic + OS ${SANITIZER_COMMON_SUPPORTED_OS} + ARCHS ${ASAN_SUPPORTED_ARCH} + SOURCES ${ASAN_SOURCES} ${ASAN_CXX_SOURCES} + CFLAGS ${ASAN_DYNAMIC_CFLAGS} + DEFS ${ASAN_DYNAMIC_DEFINITIONS}) + +if(NOT APPLE) add_compiler_rt_object_libraries(RTAsan ARCHS ${ASAN_SUPPORTED_ARCH} SOURCES ${ASAN_SOURCES} CFLAGS ${ASAN_CFLAGS} @@ -94,11 +94,6 @@ else() ARCHS ${ASAN_SUPPORTED_ARCH} SOURCES ${ASAN_PREINIT_SOURCES} CFLAGS ${ASAN_CFLAGS} DEFS ${ASAN_COMMON_DEFINITIONS}) - add_compiler_rt_object_libraries(RTAsan_dynamic - ARCHS ${ASAN_SUPPORTED_ARCH} - SOURCES ${ASAN_SOURCES} ${ASAN_CXX_SOURCES} - CFLAGS ${ASAN_DYNAMIC_CFLAGS} - DEFS ${ASAN_DYNAMIC_DEFINITIONS}) file(WRITE ${CMAKE_CURRENT_BINARY_DIR}/dummy.cc "") add_compiler_rt_object_libraries(RTAsan_dynamic_version_script_dummy @@ -111,49 +106,57 @@ endif() # Build ASan runtimes shipped with Clang. add_custom_target(asan) if(APPLE) - foreach (os ${SANITIZER_COMMON_SUPPORTED_OS}) - add_compiler_rt_darwin_dynamic_runtime(clang_rt.asan_${os}_dynamic ${os} - ARCHS ${ASAN_SUPPORTED_ARCH} - SOURCES $<TARGET_OBJECTS:RTAsan.${os}> - $<TARGET_OBJECTS:RTInterception.${os}> - $<TARGET_OBJECTS:RTSanitizerCommon.${os}> - $<TARGET_OBJECTS:RTLSanCommon.${os}> - $<TARGET_OBJECTS:RTUbsan.${os}> - CFLAGS ${ASAN_DYNAMIC_CFLAGS} - DEFS ${ASAN_DYNAMIC_DEFINITIONS}) - add_dependencies(asan clang_rt.asan_${os}_dynamic) - endforeach() + add_compiler_rt_runtime(clang_rt.asan + SHARED + OS ${SANITIZER_COMMON_SUPPORTED_OS} + ARCHS ${ASAN_SUPPORTED_ARCH} + OBJECT_LIBS RTAsan_dynamic + RTInterception + RTSanitizerCommon + RTSanitizerCommonLibc + RTLSanCommon + RTUbsan + CFLAGS ${ASAN_DYNAMIC_CFLAGS} + DEFS ${ASAN_DYNAMIC_DEFINITIONS} + PARENT_TARGET asan) else() # Build separate libraries for each target. - foreach(arch ${ASAN_SUPPORTED_ARCH}) - set(ASAN_COMMON_RUNTIME_OBJECTS - $<TARGET_OBJECTS:RTInterception.${arch}> - $<TARGET_OBJECTS:RTSanitizerCommon.${arch}> - $<TARGET_OBJECTS:RTSanitizerCommonLibc.${arch}> - $<TARGET_OBJECTS:RTLSanCommon.${arch}> - $<TARGET_OBJECTS:RTUbsan.${arch}>) - - add_compiler_rt_runtime(clang_rt.asan-${arch} ${arch} STATIC - SOURCES $<TARGET_OBJECTS:RTAsan_preinit.${arch}> - $<TARGET_OBJECTS:RTAsan.${arch}> - ${ASAN_COMMON_RUNTIME_OBJECTS} + + set(ASAN_COMMON_RUNTIME_OBJECT_LIBS + RTInterception + RTSanitizerCommon + RTSanitizerCommonLibc + RTLSanCommon + RTUbsan) + + add_compiler_rt_runtime(clang_rt.asan + STATIC + ARCHS ${ASAN_SUPPORTED_ARCH} + OBJECT_LIBS RTAsan_preinit + RTAsan + ${ASAN_COMMON_RUNTIME_OBJECT_LIBS} CFLAGS ${ASAN_CFLAGS} - DEFS ${ASAN_COMMON_DEFINITIONS}) - add_dependencies(asan clang_rt.asan-${arch}) + DEFS ${ASAN_COMMON_DEFINITIONS} + PARENT_TARGET asan) - add_compiler_rt_runtime(clang_rt.asan_cxx-${arch} ${arch} STATIC - SOURCES $<TARGET_OBJECTS:RTAsan_cxx.${arch}> - $<TARGET_OBJECTS:RTUbsan_cxx.${arch}> + add_compiler_rt_runtime(clang_rt.asan_cxx + STATIC + ARCHS ${ASAN_SUPPORTED_ARCH} + OBJECT_LIBS RTAsan_cxx + RTUbsan_cxx CFLAGS ${ASAN_CFLAGS} - DEFS ${ASAN_COMMON_DEFINITIONS}) - add_dependencies(asan clang_rt.asan_cxx-${arch}) + DEFS ${ASAN_COMMON_DEFINITIONS} + PARENT_TARGET asan) - add_compiler_rt_runtime(clang_rt.asan-preinit-${arch} ${arch} STATIC - SOURCES $<TARGET_OBJECTS:RTAsan_preinit.${arch}> + add_compiler_rt_runtime(clang_rt.asan-preinit + STATIC + ARCHS ${ASAN_SUPPORTED_ARCH} + OBJECT_LIBS RTAsan_preinit CFLAGS ${ASAN_CFLAGS} - DEFS ${ASAN_COMMON_DEFINITIONS}) - add_dependencies(asan clang_rt.asan-preinit-${arch}) + DEFS ${ASAN_COMMON_DEFINITIONS} + PARENT_TARGET asan) + foreach(arch ${ASAN_SUPPORTED_ARCH}) if (UNIX AND NOT ${arch} MATCHES "i386|i686") add_sanitizer_rt_version_list(clang_rt.asan-dynamic-${arch} LIBS clang_rt.asan-${arch} clang_rt.asan_cxx-${arch} @@ -168,48 +171,50 @@ else() set(VERSION_SCRIPT_FLAG) endif() - if (WIN32) - set(SHARED_ASAN_NAME clang_rt.asan_dynamic-${arch}${COMPILER_RT_OS_SUFFIX}) - else() - set(SHARED_ASAN_NAME clang_rt.asan-${arch}${COMPILER_RT_OS_SUFFIX}) - endif() - add_compiler_rt_runtime(clang_rt.asan-dynamic-${arch} ${arch} SHARED - OUTPUT_NAME ${SHARED_ASAN_NAME} - SOURCES $<TARGET_OBJECTS:RTAsan_dynamic.${arch}> + add_compiler_rt_runtime(clang_rt.asan + SHARED + ARCHS ${arch} + OBJECT_LIBS ${ASAN_COMMON_RUNTIME_OBJECT_LIBS} + RTAsan_dynamic # The only purpose of RTAsan_dynamic_version_script_dummy is to carry # a dependency of the shared runtime on the version script. With CMake # 3.1 or later it can be replaced with a straightforward # add_dependencies(clang_rt.asan-dynamic-${arch} clang_rt.asan-dynamic-${arch}-version-list) - $<TARGET_OBJECTS:RTAsan_dynamic_version_script_dummy.${arch}> - $<TARGET_OBJECTS:RTUbsan_cxx.${arch}> - ${ASAN_COMMON_RUNTIME_OBJECTS} + RTAsan_dynamic_version_script_dummy + RTUbsan_cxx CFLAGS ${ASAN_DYNAMIC_CFLAGS} LINKFLAGS ${ASAN_DYNAMIC_LINK_FLAGS} ${VERSION_SCRIPT_FLAG} - DEFS ${ASAN_DYNAMIC_DEFINITIONS}) - target_link_libraries(clang_rt.asan-dynamic-${arch} ${ASAN_DYNAMIC_LIBS}) - add_dependencies(asan clang_rt.asan-dynamic-${arch}) + LINK_LIBS ${ASAN_DYNAMIC_LIBS} + DEFS ${ASAN_DYNAMIC_DEFINITIONS} + PARENT_TARGET asan) if (UNIX AND NOT ${arch} MATCHES "i386|i686") - add_sanitizer_rt_symbols(clang_rt.asan_cxx-${arch}) + add_sanitizer_rt_symbols(clang_rt.asan_cxx + ARCHS ${arch}) add_dependencies(asan clang_rt.asan_cxx-${arch}-symbols) - add_sanitizer_rt_symbols(clang_rt.asan-${arch} asan.syms.extra) + add_sanitizer_rt_symbols(clang_rt.asan + ARCHS ${arch} + EXTRA asan.syms.extra) add_dependencies(asan clang_rt.asan-${arch}-symbols) endif() if (WIN32) - add_compiler_rt_runtime(clang_rt.asan_dll_thunk-${arch} ${arch} STATIC + add_compiler_rt_runtime(clang_rt.asan_dll_thunk + STATIC + ARCHS ${arch} SOURCES asan_win_dll_thunk.cc $<TARGET_OBJECTS:RTInterception.${arch}> CFLAGS ${ASAN_CFLAGS} -DASAN_DLL_THUNK - DEFS ${ASAN_COMMON_DEFINITIONS}) - add_dependencies(asan clang_rt.asan_dll_thunk-${arch}) - add_compiler_rt_runtime(clang_rt.asan_dynamic_runtime_thunk-${arch} ${arch} + DEFS ${ASAN_COMMON_DEFINITIONS} + PARENT_TARGET asan) + add_compiler_rt_runtime(clang_rt.asan_dynamic_runtime_thunk STATIC + ARCHS ${arch} SOURCES asan_win_dynamic_runtime_thunk.cc CFLAGS ${ASAN_CFLAGS} -DASAN_DYNAMIC_RUNTIME_THUNK -Zl - DEFS ${ASAN_COMMON_DEFINITIONS}) - add_dependencies(asan clang_rt.asan_dynamic_runtime_thunk-${arch}) + DEFS ${ASAN_COMMON_DEFINITIONS} + PARENT_TARGET asan) endif() endforeach() endif() diff --git a/lib/asan/README.txt b/lib/asan/README.txt index 8cc9bb17b59d..bb6ff42c5cde 100644 --- a/lib/asan/README.txt +++ b/lib/asan/README.txt @@ -23,4 +23,4 @@ from the root of your CMake build tree: make check-asan For more instructions see: -http://code.google.com/p/address-sanitizer/wiki/HowToBuild +https://github.com/google/sanitizers/wiki/AddressSanitizerHowToBuild diff --git a/lib/asan/asan_activation.cc b/lib/asan/asan_activation.cc index 3bc01984898d..9df3b977ea1b 100644 --- a/lib/asan/asan_activation.cc +++ b/lib/asan/asan_activation.cc @@ -38,7 +38,7 @@ static struct AsanDeactivatedFlags { #undef ASAN_ACTIVATION_FLAG #undef COMMON_ACTIVATION_FLAG - RegisterIncludeFlag(parser, cf); + RegisterIncludeFlags(parser, cf); } void OverrideFromActivationFlags() { @@ -61,11 +61,6 @@ static struct AsanDeactivatedFlags { parser.ParseString(env); } - // Override from getprop asan.options. - char buf[100]; - GetExtraActivationFlags(buf, sizeof(buf)); - parser.ParseString(buf); - SetVerbosity(cf.verbosity); if (Verbosity()) ReportUnrecognizedFlags(); @@ -124,6 +119,8 @@ void AsanActivate() { if (!asan_is_deactivated) return; VReport(1, "Activating ASan\n"); + UpdateProcessName(); + asan_deactivated_flags.OverrideFromActivationFlags(); SetCanPoisonMemory(asan_deactivated_flags.poison_heap); diff --git a/lib/asan/asan_allocator.cc b/lib/asan/asan_allocator.cc index 2df9a510bd9a..56f184a36651 100644 --- a/lib/asan/asan_allocator.cc +++ b/lib/asan/asan_allocator.cc @@ -14,8 +14,8 @@ // with ThreadSanitizer and MemorySanitizer. // //===----------------------------------------------------------------------===// -#include "asan_allocator.h" +#include "asan_allocator.h" #include "asan_mapping.h" #include "asan_poisoning.h" #include "asan_report.h" @@ -541,7 +541,7 @@ struct Allocator { u8 chunk_state = m->chunk_state; if (chunk_state != CHUNK_ALLOCATED) ReportInvalidFree(old_ptr, chunk_state, stack); - CHECK_NE(REAL(memcpy), (void*)0); + CHECK_NE(REAL(memcpy), nullptr); uptr memcpy_size = Min(new_size, m->UsedSize()); // If realloc() races with free(), we may start copying freed memory. // However, we will report racy double-free later anyway. @@ -579,7 +579,7 @@ struct Allocator { // Assumes alloc_beg == allocator.GetBlockBegin(alloc_beg). AsanChunk *GetAsanChunk(void *alloc_beg) { - if (!alloc_beg) return 0; + if (!alloc_beg) return nullptr; if (!allocator.FromPrimary(alloc_beg)) { uptr *meta = reinterpret_cast<uptr *>(allocator.GetMetaData(alloc_beg)); AsanChunk *m = reinterpret_cast<AsanChunk *>(meta[1]); @@ -619,7 +619,7 @@ struct Allocator { // The address is in the chunk's left redzone, so maybe it is actually // a right buffer overflow from the other chunk to the left. // Search a bit to the left to see if there is another chunk. - AsanChunk *m2 = 0; + AsanChunk *m2 = nullptr; for (uptr l = 1; l < GetPageSizeCached(); l++) { m2 = GetAsanChunkByAddr(addr - l); if (m2 == m1) continue; // Still the same chunk. @@ -653,7 +653,7 @@ static AsanAllocator &get_allocator() { } bool AsanChunkView::IsValid() { - return chunk_ != 0 && chunk_->chunk_state != CHUNK_AVAILABLE; + return chunk_ && chunk_->chunk_state != CHUNK_AVAILABLE; } uptr AsanChunkView::Beg() { return chunk_->Beg(); } uptr AsanChunkView::End() { return Beg() + UsedSize(); } @@ -723,11 +723,11 @@ void *asan_calloc(uptr nmemb, uptr size, BufferedStackTrace *stack) { } void *asan_realloc(void *p, uptr size, BufferedStackTrace *stack) { - if (p == 0) + if (!p) return instance.Allocate(size, 8, stack, FROM_MALLOC, true); if (size == 0) { instance.Deallocate(p, 0, stack, FROM_MALLOC); - return 0; + return nullptr; } return instance.Reallocate(p, size, stack); } @@ -755,7 +755,7 @@ int asan_posix_memalign(void **memptr, uptr alignment, uptr size, } uptr asan_malloc_usable_size(void *ptr, uptr pc, uptr bp) { - if (ptr == 0) return 0; + if (!ptr) return 0; uptr usable_size = instance.AllocationSize(reinterpret_cast<uptr>(ptr)); if (flags()->check_malloc_usable_size && (usable_size == 0)) { GET_STACK_TRACE_FATAL(pc, bp); @@ -780,7 +780,7 @@ void AsanSoftRssLimitExceededCallback(bool exceeded) { instance.allocator.SetRssLimitIsExceeded(exceeded); } -} // namespace __asan +} // namespace __asan // --- Implementation of LSan-specific functions --- {{{1 namespace __lsan { @@ -881,7 +881,7 @@ int __sanitizer_get_ownership(const void *p) { } uptr __sanitizer_get_allocated_size(const void *p) { - if (p == 0) return 0; + if (!p) return 0; uptr ptr = reinterpret_cast<uptr>(p); uptr allocated_size = instance.AllocationSize(ptr); // Die if p is not malloced or if it is already freed. @@ -904,5 +904,5 @@ SANITIZER_INTERFACE_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE void __sanitizer_free_hook(void *ptr) { (void)ptr; } -} // extern "C" +} // extern "C" #endif diff --git a/lib/asan/asan_allocator.h b/lib/asan/asan_allocator.h index 5ccd00c97bab..e3d53330cd2f 100644 --- a/lib/asan/asan_allocator.h +++ b/lib/asan/asan_allocator.h @@ -114,6 +114,11 @@ struct AsanMapUnmapCallback { # if defined(__powerpc64__) const uptr kAllocatorSpace = 0xa0000000000ULL; const uptr kAllocatorSize = 0x20000000000ULL; // 2T. +# elif defined(__aarch64__) +// AArch64/SANITIZIER_CAN_USER_ALLOCATOR64 is only for 42-bit VMA +// so no need to different values for different VMA. +const uptr kAllocatorSpace = 0x10000000000ULL; +const uptr kAllocatorSize = 0x10000000000ULL; // 3T. # else const uptr kAllocatorSpace = 0x600000000000ULL; const uptr kAllocatorSize = 0x40000000000ULL; // 4T. diff --git a/lib/asan/asan_debugging.cc b/lib/asan/asan_debugging.cc index 6fc5b690de99..7c3a8a73bd4e 100644 --- a/lib/asan/asan_debugging.cc +++ b/lib/asan/asan_debugging.cc @@ -108,14 +108,14 @@ static uptr AsanGetStack(uptr addr, uptr *trace, u32 size, u32 *thread_id, return 0; } -} // namespace __asan +} // namespace __asan using namespace __asan; SANITIZER_INTERFACE_ATTRIBUTE const char *__asan_locate_address(uptr addr, char *name, uptr name_size, uptr *region_address, uptr *region_size) { - AddressDescription descr = { name, name_size, 0, 0, 0 }; + AddressDescription descr = { name, name_size, 0, 0, nullptr }; AsanLocateAddress(addr, &descr); if (region_address) *region_address = descr.region_address; if (region_size) *region_size = descr.region_size; diff --git a/lib/asan/asan_fake_stack.cc b/lib/asan/asan_fake_stack.cc index d20641155b88..91fdf0aa1dca 100644 --- a/lib/asan/asan_fake_stack.cc +++ b/lib/asan/asan_fake_stack.cc @@ -11,6 +11,7 @@ // // FakeStack is used to detect use-after-return bugs. //===----------------------------------------------------------------------===// + #include "asan_allocator.h" #include "asan_poisoning.h" #include "asan_thread.h" @@ -32,7 +33,8 @@ ALWAYS_INLINE void SetShadow(uptr ptr, uptr size, uptr class_id, u64 magic) { if (class_id <= 6) { for (uptr i = 0; i < (1U << class_id); i++) { shadow[i] = magic; - SanitizerBreakOptimization(0); // Make sure this does not become memset. + // Make sure this does not become memset. + SanitizerBreakOptimization(nullptr); } } else { // The size class is too big, it's cheaper to poison only size bytes. @@ -80,7 +82,9 @@ void FakeStack::PoisonAll(u8 magic) { magic); } +#if !defined(_MSC_VER) || defined(__clang__) ALWAYS_INLINE USED +#endif FakeFrame *FakeStack::Allocate(uptr stack_size_log, uptr class_id, uptr real_stack) { CHECK_LT(class_id, kNumberOfSizeClasses); @@ -106,7 +110,7 @@ FakeFrame *FakeStack::Allocate(uptr stack_size_log, uptr class_id, *SavedFlagPtr(reinterpret_cast<uptr>(res), class_id) = &flags[pos]; return res; } - return 0; // We are out of fake stack. + return nullptr; // We are out of fake stack. } uptr FakeStack::AddrIsInFakeStack(uptr ptr, uptr *frame_beg, uptr *frame_end) { @@ -183,7 +187,7 @@ void SetTLSFakeStack(FakeStack *fs) { } static FakeStack *GetFakeStack() { AsanThread *t = GetCurrentThread(); - if (!t) return 0; + if (!t) return nullptr; return t->fake_stack(); } @@ -191,7 +195,7 @@ static FakeStack *GetFakeStackFast() { if (FakeStack *fs = GetTLSFakeStack()) return fs; if (!__asan_option_detect_stack_use_after_return) - return 0; + return nullptr; return GetFakeStack(); } @@ -212,7 +216,7 @@ ALWAYS_INLINE void OnFree(uptr ptr, uptr class_id, uptr size) { SetShadow(ptr, size, class_id, kMagic8); } -} // namespace __asan +} // namespace __asan // ---------------------- Interface ---------------- {{{1 using namespace __asan; @@ -245,13 +249,13 @@ SANITIZER_INTERFACE_ATTRIBUTE void *__asan_addr_is_in_fake_stack(void *fake_stack, void *addr, void **beg, void **end) { FakeStack *fs = reinterpret_cast<FakeStack*>(fake_stack); - if (!fs) return 0; + if (!fs) return nullptr; uptr frame_beg, frame_end; FakeFrame *frame = reinterpret_cast<FakeFrame *>(fs->AddrIsInFakeStack( reinterpret_cast<uptr>(addr), &frame_beg, &frame_end)); - if (!frame) return 0; + if (!frame) return nullptr; if (frame->magic != kCurrentStackFrameMagic) - return 0; + return nullptr; if (beg) *beg = reinterpret_cast<void*>(frame_beg); if (end) *end = reinterpret_cast<void*>(frame_end); return reinterpret_cast<void*>(frame->real_stack); @@ -276,4 +280,4 @@ void __asan_allocas_unpoison(uptr top, uptr bottom) { REAL(memset)(reinterpret_cast<void*>(MemToShadow(top)), 0, (bottom - top) / SHADOW_GRANULARITY); } -} // extern "C" +} // extern "C" diff --git a/lib/asan/asan_flags.cc b/lib/asan/asan_flags.cc index e8ea549b62e3..363ee67e77c6 100644 --- a/lib/asan/asan_flags.cc +++ b/lib/asan/asan_flags.cc @@ -65,6 +65,7 @@ void InitializeFlags() { cf.external_symbolizer_path = GetEnv("ASAN_SYMBOLIZER_PATH"); cf.malloc_context_size = kDefaultMallocContextSize; cf.intercept_tls_get_addr = true; + cf.exitcode = 1; OverrideCommonFlags(cf); } Flags *f = flags(); @@ -115,14 +116,6 @@ void InitializeFlags() { ubsan_parser.ParseString(GetEnv("UBSAN_OPTIONS")); #endif - // Let activation flags override current settings. On Android they come - // from a system property. On other platforms this is no-op. - if (!flags()->start_deactivated) { - char buf[100]; - GetExtraActivationFlags(buf, sizeof(buf)); - asan_parser.ParseString(buf); - } - SetVerbosity(common_flags()->verbosity); // TODO(eugenis): dump all flags at verbosity>=2? diff --git a/lib/asan/asan_flags.inc b/lib/asan/asan_flags.inc index 53a8a4039e7e..5e69242fb8e9 100644 --- a/lib/asan/asan_flags.inc +++ b/lib/asan/asan_flags.inc @@ -44,9 +44,6 @@ ASAN_FLAG( "to find more errors.") ASAN_FLAG(bool, replace_intrin, true, "If set, uses custom wrappers for memset/memcpy/memmove intinsics.") -ASAN_FLAG(bool, mac_ignore_invalid_free, false, - "Ignore invalid free() calls to work around some bugs. Used on OS X " - "only.") ASAN_FLAG(bool, detect_stack_use_after_return, false, "Enables stack-use-after-return checking at run-time.") ASAN_FLAG(int, min_uar_stack_size_log, 16, // We can't do smaller anyway. @@ -62,8 +59,6 @@ ASAN_FLAG( "bytes that will be filled with malloc_fill_byte on malloc.") ASAN_FLAG(int, malloc_fill_byte, 0xbe, "Value used to fill the newly allocated memory.") -ASAN_FLAG(int, exitcode, ASAN_DEFAULT_FAILURE_EXITCODE, - "Override the program exit status if the tool found an error.") ASAN_FLAG(bool, allow_user_poisoning, true, "If set, user may manually mark memory regions as poisoned or " "unpoisoned.") @@ -77,10 +72,7 @@ ASAN_FLAG(bool, check_malloc_usable_size, true, "295.*.") ASAN_FLAG(bool, unmap_shadow_on_exit, false, "If set, explicitly unmaps the (huge) shadow at exit.") -ASAN_FLAG( - bool, abort_on_error, false, - "If set, the tool calls abort() instead of _exit() after printing the " - "error report.") +ASAN_FLAG(bool, protect_shadow_gap, true, "If set, mprotect the shadow gap") ASAN_FLAG(bool, print_stats, false, "Print various statistics after printing an error message or if " "atexit=1.") @@ -104,8 +96,8 @@ ASAN_FLAG(bool, poison_array_cookie, true, "Poison (or not) the array cookie after operator new[].") // Turn off alloc/dealloc mismatch checker on Mac and Windows for now. -// https://code.google.com/p/address-sanitizer/issues/detail?id=131 -// https://code.google.com/p/address-sanitizer/issues/detail?id=309 +// https://github.com/google/sanitizers/issues/131 +// https://github.com/google/sanitizers/issues/309 // TODO(glider,timurrrr): Fix known issues and enable this back. ASAN_FLAG(bool, alloc_dealloc_mismatch, (SANITIZER_MAC == 0) && (SANITIZER_WINDOWS == 0), @@ -113,9 +105,6 @@ ASAN_FLAG(bool, alloc_dealloc_mismatch, ASAN_FLAG(bool, new_delete_type_mismatch, true, "Report errors on mismatch betwen size of new and delete.") -ASAN_FLAG(bool, strict_memcmp, true, - "If true, assume that memcmp(p1, p2, n) always reads n bytes before " - "comparing p1 and p2.") ASAN_FLAG( bool, strict_init_order, false, "If true, assume that dynamic initializers can never access globals from " @@ -134,8 +123,8 @@ ASAN_FLAG( "The bigger the value the harder we try.") ASAN_FLAG( bool, detect_container_overflow, true, - "If true, honor the container overflow annotations. " - "See https://code.google.com/p/address-sanitizer/wiki/ContainerOverflow") + "If true, honor the container overflow annotations. See " + "https://github.com/google/sanitizers/wiki/AddressSanitizerContainerOverflow") ASAN_FLAG(int, detect_odr_violation, 2, "If >=2, detect violation of One-Definition-Rule (ODR); " "If ==1, detect ODR-violation only if the two variables " @@ -143,3 +132,6 @@ ASAN_FLAG(int, detect_odr_violation, 2, ASAN_FLAG(bool, dump_instruction_bytes, false, "If true, dump 16 bytes starting at the instruction that caused SEGV") ASAN_FLAG(const char *, suppressions, "", "Suppressions file name.") +ASAN_FLAG(bool, halt_on_error, true, + "Crash the program after printing the first error report " + "(WARNING: USE AT YOUR OWN RISK!)") diff --git a/lib/asan/asan_globals.cc b/lib/asan/asan_globals.cc index c34b1d3cedf2..eb9f1bfefec2 100644 --- a/lib/asan/asan_globals.cc +++ b/lib/asan/asan_globals.cc @@ -11,6 +11,7 @@ // // Handle globals. //===----------------------------------------------------------------------===// + #include "asan_interceptors.h" #include "asan_internal.h" #include "asan_mapping.h" @@ -167,7 +168,7 @@ static void RegisterGlobal(const Global *g) { l->next = list_of_all_globals; list_of_all_globals = l; if (g->has_dynamic_init) { - if (dynamic_init_globals == 0) { + if (!dynamic_init_globals) { dynamic_init_globals = new(allocator_for_globals) VectorOfGlobals(kDynamicInitGlobalsInitialCapacity); } @@ -206,7 +207,7 @@ void StopInitOrderChecking() { } } -} // namespace __asan +} // namespace __asan // ---------------------- Interface ---------------- {{{1 using namespace __asan; // NOLINT diff --git a/lib/asan/asan_init_version.h b/lib/asan/asan_init_version.h index 6cf57c4aa2a8..bc8a622f5bb1 100644 --- a/lib/asan/asan_init_version.h +++ b/lib/asan/asan_init_version.h @@ -27,8 +27,8 @@ extern "C" { // v3=>v4: added '__asan_global_source_location' to __asan_global. // v4=>v5: changed the semantics and format of __asan_stack_malloc_ and // __asan_stack_free_ functions. - #define __asan_init __asan_init_v5 - #define __asan_init_name "__asan_init_v5" + // v5=>v6: changed the name of the version check symbol + #define __asan_version_mismatch_check __asan_version_mismatch_check_v6 } #endif // ASAN_INIT_VERSION_H diff --git a/lib/asan/asan_interceptors.cc b/lib/asan/asan_interceptors.cc index d8b48d391ab8..d9a0c71a002d 100644 --- a/lib/asan/asan_interceptors.cc +++ b/lib/asan/asan_interceptors.cc @@ -11,8 +11,8 @@ // // Intercept various libc functions. //===----------------------------------------------------------------------===// -#include "asan_interceptors.h" +#include "asan_interceptors.h" #include "asan_allocator.h" #include "asan_internal.h" #include "asan_mapping.h" @@ -27,6 +27,12 @@ #include "sanitizer_common/sanitizer_posix.h" #endif +#if defined(__i386) && SANITIZER_LINUX +#define ASAN_PTHREAD_CREATE_VERSION "GLIBC_2.1" +#elif defined(__mips__) && SANITIZER_LINUX +#define ASAN_PTHREAD_CREATE_VERSION "GLIBC_2.2" +#endif + namespace __asan { // Return true if we can quickly decide that the region is unpoisoned. @@ -69,7 +75,7 @@ struct AsanInterceptorContext { } \ if (!suppressed) { \ GET_CURRENT_PC_BP_SP; \ - __asan_report_error(pc, bp, sp, __bad, isWrite, __size, 0); \ + ReportGenericError(pc, bp, sp, __bad, isWrite, __size, 0, false);\ } \ } \ } while (0) @@ -105,7 +111,7 @@ static inline bool RangesOverlap(const char *offset1, uptr length1, static inline uptr MaybeRealStrnlen(const char *s, uptr maxlen) { #if ASAN_INTERCEPT_STRNLEN - if (REAL(strnlen) != 0) { + if (REAL(strnlen)) { return REAL(strnlen)(s, maxlen); } #endif @@ -123,7 +129,7 @@ int OnExit() { return 0; } -} // namespace __asan +} // namespace __asan // ---------------------- Wrappers ---------------- {{{1 using namespace __asan; // NOLINT @@ -172,7 +178,7 @@ DECLARE_REAL_AND_INTERCEPTOR(void, free, void *) } while (false) #define COMMON_INTERCEPTOR_BLOCK_REAL(name) REAL(name) // Strict init-order checking is dlopen-hostile: -// https://code.google.com/p/address-sanitizer/issues/detail?id=178 +// https://github.com/google/sanitizers/issues/178 #define COMMON_INTERCEPTOR_ON_DLOPEN(filename, flag) \ if (flags()->strict_init_order) { \ StopInitOrderChecking(); \ @@ -216,7 +222,7 @@ static thread_return_t THREAD_CALLING_CONV asan_thread_start(void *arg) { ThreadStartParam *param = reinterpret_cast<ThreadStartParam *>(arg); AsanThread *t = nullptr; while ((t = reinterpret_cast<AsanThread *>( - atomic_load(¶m->t, memory_order_acquire))) == 0) + atomic_load(¶m->t, memory_order_acquire))) == nullptr) internal_sched_yield(); SetCurrentThread(t); return t->ThreadStart(GetTid(), ¶m->is_registered); @@ -231,7 +237,7 @@ INTERCEPTOR(int, pthread_create, void *thread, StopInitOrderChecking(); GET_STACK_TRACE_THREAD; int detached = 0; - if (attr != 0) + if (attr) REAL(pthread_attr_getdetachstate)(attr, &detached); ThreadStartParam param; atomic_store(¶m.t, 0, memory_order_relaxed); @@ -270,14 +276,14 @@ INTERCEPTOR(void*, bsd_signal, int signum, void *handler) { } return 0; } -#else +#endif + INTERCEPTOR(void*, signal, int signum, void *handler) { if (!IsDeadlySignal(signum) || common_flags()->allow_user_segv_handler) { return REAL(signal)(signum, handler); } - return 0; + return nullptr; } -#endif INTERCEPTOR(int, sigaction, int signum, const struct sigaction *act, struct sigaction *oldact) { @@ -292,7 +298,7 @@ int real_sigaction(int signum, const void *act, void *oldact) { return REAL(sigaction)(signum, (const struct sigaction *)act, (struct sigaction *)oldact); } -} // namespace __sanitizer +} // namespace __sanitizer #elif SANITIZER_POSIX // We need to have defined REAL(sigaction) on posix systems. @@ -363,40 +369,6 @@ INTERCEPTOR(void, __cxa_throw, void *a, void *b, void *c) { } #endif -static inline int CharCmp(unsigned char c1, unsigned char c2) { - return (c1 == c2) ? 0 : (c1 < c2) ? -1 : 1; -} - -INTERCEPTOR(int, memcmp, const void *a1, const void *a2, uptr size) { - void *ctx; - ASAN_INTERCEPTOR_ENTER(ctx, memcmp); - if (UNLIKELY(!asan_inited)) return internal_memcmp(a1, a2, size); - ENSURE_ASAN_INITED(); - if (flags()->replace_intrin) { - if (flags()->strict_memcmp) { - // Check the entire regions even if the first bytes of the buffers are - // different. - ASAN_READ_RANGE(ctx, a1, size); - ASAN_READ_RANGE(ctx, a2, size); - // Fallthrough to REAL(memcmp) below. - } else { - unsigned char c1 = 0, c2 = 0; - const unsigned char *s1 = (const unsigned char*)a1; - const unsigned char *s2 = (const unsigned char*)a2; - uptr i; - for (i = 0; i < size; i++) { - c1 = s1[i]; - c2 = s2[i]; - if (c1 != c2) break; - } - ASAN_READ_RANGE(ctx, s1, Min(i + 1, size)); - ASAN_READ_RANGE(ctx, s2, Min(i + 1, size)); - return CharCmp(c1, c2); - } - } - return REAL(memcmp(a1, a2, size)); -} - // memcpy is called during __asan_init() from the internals of printf(...). // We do not treat memcpy with to==from as a bug. // See http://llvm.org/bugs/show_bug.cgi?id=11763. @@ -743,7 +715,7 @@ INTERCEPTOR(int, __cxa_atexit, void (*func)(void *), void *arg, #endif ENSURE_ASAN_INITED(); int res = REAL(__cxa_atexit)(func, arg, dso_handle); - REAL(__cxa_atexit)(AtCxaAtexit, 0, 0); + REAL(__cxa_atexit)(AtCxaAtexit, nullptr, nullptr); return res; } #endif // ASAN_INTERCEPT___CXA_ATEXIT @@ -767,7 +739,6 @@ void InitializeAsanInterceptors() { InitializeCommonInterceptors(); // Intercept mem* functions. - ASAN_INTERCEPT_FUNC(memcmp); ASAN_INTERCEPT_FUNC(memmove); ASAN_INTERCEPT_FUNC(memset); if (PLATFORM_HAS_DIFFERENT_MEMCPY_AND_MEMMOVE) { @@ -806,9 +777,8 @@ void InitializeAsanInterceptors() { ASAN_INTERCEPT_FUNC(sigaction); #if SANITIZER_ANDROID ASAN_INTERCEPT_FUNC(bsd_signal); -#else - ASAN_INTERCEPT_FUNC(signal); #endif + ASAN_INTERCEPT_FUNC(signal); #endif #if ASAN_INTERCEPT_SWAPCONTEXT ASAN_INTERCEPT_FUNC(swapcontext); @@ -827,7 +797,11 @@ void InitializeAsanInterceptors() { // Intercept threading-related functions #if ASAN_INTERCEPT_PTHREAD_CREATE +#if defined(ASAN_PTHREAD_CREATE_VERSION) + ASAN_INTERCEPT_FUNC_VER(pthread_create, ASAN_PTHREAD_CREATE_VERSION); +#else ASAN_INTERCEPT_FUNC(pthread_create); +#endif ASAN_INTERCEPT_FUNC(pthread_join); #endif @@ -845,4 +819,4 @@ void InitializeAsanInterceptors() { VReport(1, "AddressSanitizer: libc interceptors initialized\n"); } -} // namespace __asan +} // namespace __asan diff --git a/lib/asan/asan_interceptors.h b/lib/asan/asan_interceptors.h index 488ada78ab8b..279c5f38451f 100644 --- a/lib/asan/asan_interceptors.h +++ b/lib/asan/asan_interceptors.h @@ -98,6 +98,12 @@ DECLARE_REAL(int, sigaction, int signum, const struct sigaction *act, if ((!INTERCEPT_FUNCTION(name) || !REAL(name))) \ VReport(1, "AddressSanitizer: failed to intercept '" #name "'\n"); \ } while (0) +#define ASAN_INTERCEPT_FUNC_VER(name, ver) \ + do { \ + if ((!INTERCEPT_FUNCTION_VER(name, ver) || !REAL(name))) \ + VReport( \ + 1, "AddressSanitizer: failed to intercept '" #name "@@" #ver "'\n"); \ + } while (0) #else // OS X interceptors don't need to be initialized with INTERCEPT_FUNCTION. #define ASAN_INTERCEPT_FUNC(name) diff --git a/lib/asan/asan_interface_internal.h b/lib/asan/asan_interface_internal.h index ad8ebcd91ad9..9efddcbd42b2 100644 --- a/lib/asan/asan_interface_internal.h +++ b/lib/asan/asan_interface_internal.h @@ -27,10 +27,14 @@ using __sanitizer::uptr; extern "C" { // This function should be called at the very beginning of the process, // before any instrumented code is executed and before any call to malloc. - // Please note that __asan_init is a macro that is replaced with - // __asan_init_vXXX at compile-time. SANITIZER_INTERFACE_ATTRIBUTE void __asan_init(); + // This function exists purely to get a linker/loader error when using + // incompatible versions of instrumentation and runtime library. Please note + // that __asan_version_mismatch_check is a macro that is replaced with + // __asan_version_mismatch_check_vXXX at compile-time. + SANITIZER_INTERFACE_ATTRIBUTE void __asan_version_mismatch_check(); + // This structure is used to describe the source location of a place where // global was defined. struct __asan_global_source_location { @@ -131,8 +135,6 @@ extern "C" { uptr addr, int is_write, uptr access_size, u32 exp); SANITIZER_INTERFACE_ATTRIBUTE - int __asan_set_error_exit_code(int exit_code); - SANITIZER_INTERFACE_ATTRIBUTE void __asan_set_death_callback(void (*callback)(void)); SANITIZER_INTERFACE_ATTRIBUTE void __asan_set_error_report_callback(void (*callback)(const char*)); @@ -165,6 +167,19 @@ extern "C" { SANITIZER_INTERFACE_ATTRIBUTE void __asan_loadN(uptr p, uptr size); SANITIZER_INTERFACE_ATTRIBUTE void __asan_storeN(uptr p, uptr size); + SANITIZER_INTERFACE_ATTRIBUTE void __asan_load1_noabort(uptr p); + SANITIZER_INTERFACE_ATTRIBUTE void __asan_load2_noabort(uptr p); + SANITIZER_INTERFACE_ATTRIBUTE void __asan_load4_noabort(uptr p); + SANITIZER_INTERFACE_ATTRIBUTE void __asan_load8_noabort(uptr p); + SANITIZER_INTERFACE_ATTRIBUTE void __asan_load16_noabort(uptr p); + SANITIZER_INTERFACE_ATTRIBUTE void __asan_store1_noabort(uptr p); + SANITIZER_INTERFACE_ATTRIBUTE void __asan_store2_noabort(uptr p); + SANITIZER_INTERFACE_ATTRIBUTE void __asan_store4_noabort(uptr p); + SANITIZER_INTERFACE_ATTRIBUTE void __asan_store8_noabort(uptr p); + SANITIZER_INTERFACE_ATTRIBUTE void __asan_store16_noabort(uptr p); + SANITIZER_INTERFACE_ATTRIBUTE void __asan_loadN_noabort(uptr p, uptr size); + SANITIZER_INTERFACE_ATTRIBUTE void __asan_storeN_noabort(uptr p, uptr size); + SANITIZER_INTERFACE_ATTRIBUTE void __asan_exp_load1(uptr p, u32 exp); SANITIZER_INTERFACE_ATTRIBUTE void __asan_exp_load2(uptr p, u32 exp); SANITIZER_INTERFACE_ATTRIBUTE void __asan_exp_load4(uptr p, u32 exp); diff --git a/lib/asan/asan_internal.h b/lib/asan/asan_internal.h index 107e16ee31b9..0ef0d0eb5263 100644 --- a/lib/asan/asan_internal.h +++ b/lib/asan/asan_internal.h @@ -21,8 +21,6 @@ #include "sanitizer_common/sanitizer_stacktrace.h" #include "sanitizer_common/sanitizer_libc.h" -#define ASAN_DEFAULT_FAILURE_EXITCODE 1 - #if __has_feature(address_sanitizer) || defined(__SANITIZE_ADDRESS__) # error "The AddressSanitizer run-time should not be" " instrumented by AddressSanitizer" @@ -75,12 +73,9 @@ void *AsanDoesNotSupportStaticLinkage(); void AsanCheckDynamicRTPrereqs(); void AsanCheckIncompatibleRT(); -void AsanOnSIGSEGV(int, void *siginfo, void *context); +void AsanOnDeadlySignal(int, void *siginfo, void *context); -void DisableReexec(); -void MaybeReexec(); void ReadContextStack(void *context, uptr *stack, uptr *ssize); -void AsanPlatformThreadInit(); void StopInitOrderChecking(); // Wrapper for TLS/TSD. diff --git a/lib/asan/asan_linux.cc b/lib/asan/asan_linux.cc index 9580fc7c06d4..e26b400562df 100644 --- a/lib/asan/asan_linux.cc +++ b/lib/asan/asan_linux.cc @@ -70,14 +70,6 @@ namespace __asan { void InitializePlatformInterceptors() {} -void DisableReexec() { - // No need to re-exec on Linux. -} - -void MaybeReexec() { - // No need to re-exec on Linux. -} - void *AsanDoesNotSupportStaticLinkage() { // This will fail to link with -static. return &_DYNAMIC; // defined in link.h @@ -117,7 +109,7 @@ void AsanCheckDynamicRTPrereqs() { return; // Ensure that dynamic RT is the first DSO in the list - const char *first_dso_name = 0; + const char *first_dso_name = nullptr; dl_iterate_phdr(FindFirstDSOCallback, &first_dso_name); if (first_dso_name && !IsDynamicRTName(first_dso_name)) { Report("ASan runtime does not come first in initial library list; " @@ -142,7 +134,8 @@ void AsanCheckIncompatibleRT() { // system libraries, causing crashes later in ASan initialization. MemoryMappingLayout proc_maps(/*cache_enabled*/true); char filename[128]; - while (proc_maps.Next(0, 0, 0, filename, sizeof(filename), 0)) { + while (proc_maps.Next(nullptr, nullptr, nullptr, filename, + sizeof(filename), nullptr)) { if (IsDynamicRTName(filename)) { Report("Your application is linked against " "incompatible ASan runtimes.\n"); @@ -155,11 +148,7 @@ void AsanCheckIncompatibleRT() { } } } -#endif // SANITIZER_ANDROID - -void AsanPlatformThreadInit() { - // Nothing here for now. -} +#endif // SANITIZER_ANDROID #if !SANITIZER_ANDROID void ReadContextStack(void *context, uptr *stack, uptr *ssize) { @@ -177,6 +166,6 @@ void *AsanDlSymNext(const char *sym) { return dlsym(RTLD_NEXT, sym); } -} // namespace __asan +} // namespace __asan -#endif // SANITIZER_FREEBSD || SANITIZER_LINUX +#endif // SANITIZER_FREEBSD || SANITIZER_LINUX diff --git a/lib/asan/asan_mac.cc b/lib/asan/asan_mac.cc index 3e028378df28..f00d98f8e5e6 100644 --- a/lib/asan/asan_mac.cc +++ b/lib/asan/asan_mac.cc @@ -24,26 +24,17 @@ #include "sanitizer_common/sanitizer_libc.h" #include "sanitizer_common/sanitizer_mac.h" -#if !SANITIZER_IOS -#include <crt_externs.h> // for _NSGetArgv and _NSGetEnviron -#else -extern "C" { - extern char ***_NSGetArgv(void); -} -#endif - -#include <dlfcn.h> // for dladdr() +#include <fcntl.h> +#include <libkern/OSAtomic.h> #include <mach-o/dyld.h> #include <mach-o/loader.h> +#include <pthread.h> +#include <stdlib.h> // for free() #include <sys/mman.h> #include <sys/resource.h> #include <sys/sysctl.h> #include <sys/ucontext.h> -#include <fcntl.h> -#include <pthread.h> -#include <stdlib.h> // for free() #include <unistd.h> -#include <libkern/OSAtomic.h> namespace __asan { @@ -52,187 +43,12 @@ void InitializePlatformInterceptors() {} bool PlatformHasDifferentMemcpyAndMemmove() { // On OS X 10.7 memcpy() and memmove() are both resolved // into memmove$VARIANT$sse42. - // See also http://code.google.com/p/address-sanitizer/issues/detail?id=34. + // See also https://github.com/google/sanitizers/issues/34. // TODO(glider): need to check dynamically that memcpy() and memmove() are // actually the same function. return GetMacosVersion() == MACOS_VERSION_SNOW_LEOPARD; } -extern "C" -void __asan_init(); - -static const char kDyldInsertLibraries[] = "DYLD_INSERT_LIBRARIES"; -LowLevelAllocator allocator_for_env; - -// Change the value of the env var |name|, leaking the original value. -// If |name_value| is NULL, the variable is deleted from the environment, -// otherwise the corresponding "NAME=value" string is replaced with -// |name_value|. -void LeakyResetEnv(const char *name, const char *name_value) { - char **env = GetEnviron(); - uptr name_len = internal_strlen(name); - while (*env != 0) { - uptr len = internal_strlen(*env); - if (len > name_len) { - const char *p = *env; - if (!internal_memcmp(p, name, name_len) && p[name_len] == '=') { - // Match. - if (name_value) { - // Replace the old value with the new one. - *env = const_cast<char*>(name_value); - } else { - // Shift the subsequent pointers back. - char **del = env; - do { - del[0] = del[1]; - } while (*del++); - } - } - } - env++; - } -} - -static bool reexec_disabled = false; - -void DisableReexec() { - reexec_disabled = true; -} - -bool DyldNeedsEnvVariable() { -// If running on OS X 10.11+ or iOS 9.0+, dyld will interpose even if -// DYLD_INSERT_LIBRARIES is not set. - -#if SANITIZER_IOSSIM - // GetMacosVersion will not work for the simulator, whose kernel version - // is tied to the host. Use a weak linking hack for the simulator. - // This API was introduced in the same version of the OS as the dyld - // optimization. - - // Check for presence of a symbol that is available on OS X 10.11+, iOS 9.0+. - return (dlsym(RTLD_NEXT, "mach_memory_info") == nullptr); -#else - return (GetMacosVersion() <= MACOS_VERSION_YOSEMITE); -#endif -} - -void MaybeReexec() { - if (reexec_disabled) return; - - // Make sure the dynamic ASan runtime library is preloaded so that the - // wrappers work. If it is not, set DYLD_INSERT_LIBRARIES and re-exec - // ourselves. - Dl_info info; - CHECK(dladdr((void*)((uptr)__asan_init), &info)); - char *dyld_insert_libraries = - const_cast<char*>(GetEnv(kDyldInsertLibraries)); - uptr old_env_len = dyld_insert_libraries ? - internal_strlen(dyld_insert_libraries) : 0; - uptr fname_len = internal_strlen(info.dli_fname); - const char *dylib_name = StripModuleName(info.dli_fname); - uptr dylib_name_len = internal_strlen(dylib_name); - - bool lib_is_in_env = - dyld_insert_libraries && REAL(strstr)(dyld_insert_libraries, dylib_name); - if (DyldNeedsEnvVariable() && !lib_is_in_env) { - // DYLD_INSERT_LIBRARIES is not set or does not contain the runtime - // library. - char program_name[1024]; - uint32_t buf_size = sizeof(program_name); - _NSGetExecutablePath(program_name, &buf_size); - char *new_env = const_cast<char*>(info.dli_fname); - if (dyld_insert_libraries) { - // Append the runtime dylib name to the existing value of - // DYLD_INSERT_LIBRARIES. - new_env = (char*)allocator_for_env.Allocate(old_env_len + fname_len + 2); - internal_strncpy(new_env, dyld_insert_libraries, old_env_len); - new_env[old_env_len] = ':'; - // Copy fname_len and add a trailing zero. - internal_strncpy(new_env + old_env_len + 1, info.dli_fname, - fname_len + 1); - // Ok to use setenv() since the wrappers don't depend on the value of - // asan_inited. - setenv(kDyldInsertLibraries, new_env, /*overwrite*/1); - } else { - // Set DYLD_INSERT_LIBRARIES equal to the runtime dylib name. - setenv(kDyldInsertLibraries, info.dli_fname, /*overwrite*/0); - } - VReport(1, "exec()-ing the program with\n"); - VReport(1, "%s=%s\n", kDyldInsertLibraries, new_env); - VReport(1, "to enable ASan wrappers.\n"); - execv(program_name, *_NSGetArgv()); - - // We get here only if execv() failed. - Report("ERROR: The process is launched without DYLD_INSERT_LIBRARIES, " - "which is required for ASan to work. ASan tried to set the " - "environment variable and re-execute itself, but execv() failed, " - "possibly because of sandbox restrictions. Make sure to launch the " - "executable with:\n%s=%s\n", kDyldInsertLibraries, new_env); - CHECK("execv failed" && 0); - } - - if (!lib_is_in_env) - return; - - // DYLD_INSERT_LIBRARIES is set and contains the runtime library. Let's remove - // the dylib from the environment variable, because interceptors are installed - // and we don't want our children to inherit the variable. - - uptr env_name_len = internal_strlen(kDyldInsertLibraries); - // Allocate memory to hold the previous env var name, its value, the '=' - // sign and the '\0' char. - char *new_env = (char*)allocator_for_env.Allocate( - old_env_len + 2 + env_name_len); - CHECK(new_env); - internal_memset(new_env, '\0', old_env_len + 2 + env_name_len); - internal_strncpy(new_env, kDyldInsertLibraries, env_name_len); - new_env[env_name_len] = '='; - char *new_env_pos = new_env + env_name_len + 1; - - // Iterate over colon-separated pieces of |dyld_insert_libraries|. - char *piece_start = dyld_insert_libraries; - char *piece_end = NULL; - char *old_env_end = dyld_insert_libraries + old_env_len; - do { - if (piece_start[0] == ':') piece_start++; - piece_end = REAL(strchr)(piece_start, ':'); - if (!piece_end) piece_end = dyld_insert_libraries + old_env_len; - if ((uptr)(piece_start - dyld_insert_libraries) > old_env_len) break; - uptr piece_len = piece_end - piece_start; - - char *filename_start = - (char *)internal_memrchr(piece_start, '/', piece_len); - uptr filename_len = piece_len; - if (filename_start) { - filename_start += 1; - filename_len = piece_len - (filename_start - piece_start); - } else { - filename_start = piece_start; - } - - // If the current piece isn't the runtime library name, - // append it to new_env. - if ((dylib_name_len != filename_len) || - (internal_memcmp(filename_start, dylib_name, dylib_name_len) != 0)) { - if (new_env_pos != new_env + env_name_len + 1) { - new_env_pos[0] = ':'; - new_env_pos++; - } - internal_strncpy(new_env_pos, piece_start, piece_len); - new_env_pos += piece_len; - } - // Move on to the next piece. - piece_start = piece_end; - } while (piece_start < old_env_end); - - // Can't use setenv() here, because it requires the allocator to be - // initialized. - // FIXME: instead of filtering DYLD_INSERT_LIBRARIES here, do it in - // a separate function called after InitializeAllocator(). - if (new_env_pos == new_env + env_name_len + 1) new_env = NULL; - LeakyResetEnv(kDyldInsertLibraries, new_env); -} - // No-op. Mac does not support static linkage anyway. void *AsanDoesNotSupportStaticLinkage() { return 0; @@ -244,9 +60,6 @@ void AsanCheckDynamicRTPrereqs() {} // No-op. Mac does not support static linkage anyway. void AsanCheckIncompatibleRT() {} -void AsanPlatformThreadInit() { -} - void ReadContextStack(void *context, uptr *stack, uptr *ssize) { UNIMPLEMENTED(); } diff --git a/lib/asan/asan_malloc_linux.cc b/lib/asan/asan_malloc_linux.cc index 46a6a9db4a81..d5089f9f7b36 100644 --- a/lib/asan/asan_malloc_linux.cc +++ b/lib/asan/asan_malloc_linux.cc @@ -26,13 +26,25 @@ // ---------------------- Replacement functions ---------------- {{{1 using namespace __asan; // NOLINT +static const uptr kCallocPoolSize = 1024; +static uptr calloc_memory_for_dlsym[kCallocPoolSize]; + +static bool IsInCallocPool(const void *ptr) { + sptr off = (sptr)ptr - (sptr)calloc_memory_for_dlsym; + return 0 <= off && off < (sptr)kCallocPoolSize; +} + INTERCEPTOR(void, free, void *ptr) { GET_STACK_TRACE_FREE; + if (UNLIKELY(IsInCallocPool(ptr))) + return; asan_free(ptr, &stack, FROM_MALLOC); } INTERCEPTOR(void, cfree, void *ptr) { GET_STACK_TRACE_FREE; + if (UNLIKELY(IsInCallocPool(ptr))) + return; asan_free(ptr, &stack, FROM_MALLOC); } @@ -44,8 +56,6 @@ INTERCEPTOR(void*, malloc, uptr size) { INTERCEPTOR(void*, calloc, uptr nmemb, uptr size) { if (UNLIKELY(!asan_inited)) { // Hack: dlsym calls calloc before REAL(calloc) is retrieved from dlsym. - const uptr kCallocPoolSize = 1024; - static uptr calloc_memory_for_dlsym[kCallocPoolSize]; static uptr allocated; uptr size_in_words = ((nmemb * size) + kWordSize - 1) / kWordSize; void *mem = (void*)&calloc_memory_for_dlsym[allocated]; @@ -59,6 +69,13 @@ INTERCEPTOR(void*, calloc, uptr nmemb, uptr size) { INTERCEPTOR(void*, realloc, void *ptr, uptr size) { GET_STACK_TRACE_MALLOC; + if (UNLIKELY(IsInCallocPool(ptr))) { + uptr offset = (uptr)ptr - (uptr)calloc_memory_for_dlsym; + uptr copy_size = Min(size, kCallocPoolSize - offset); + void *new_ptr = asan_malloc(size, &stack); + internal_memcpy(new_ptr, ptr, copy_size); + return new_ptr; + } return asan_realloc(ptr, size, &stack); } diff --git a/lib/asan/asan_malloc_mac.cc b/lib/asan/asan_malloc_mac.cc index d7a6307c9bdc..744728d40df5 100644 --- a/lib/asan/asan_malloc_mac.cc +++ b/lib/asan/asan_malloc_mac.cc @@ -15,348 +15,47 @@ #include "sanitizer_common/sanitizer_platform.h" #if SANITIZER_MAC -#include <AvailabilityMacros.h> -#include <CoreFoundation/CFBase.h> -#include <dlfcn.h> -#include <malloc/malloc.h> -#include <sys/mman.h> - -#include "asan_allocator.h" #include "asan_interceptors.h" -#include "asan_internal.h" #include "asan_report.h" #include "asan_stack.h" #include "asan_stats.h" -#include "sanitizer_common/sanitizer_mac.h" - -// Similar code is used in Google Perftools, -// http://code.google.com/p/google-perftools. - -// ---------------------- Replacement functions ---------------- {{{1 -using namespace __asan; // NOLINT - -// TODO(glider): do we need both zones? -static malloc_zone_t *system_malloc_zone = 0; -static malloc_zone_t asan_zone; - -INTERCEPTOR(malloc_zone_t *, malloc_create_zone, - vm_size_t start_size, unsigned zone_flags) { - ENSURE_ASAN_INITED(); - GET_STACK_TRACE_MALLOC; - uptr page_size = GetPageSizeCached(); - uptr allocated_size = RoundUpTo(sizeof(asan_zone), page_size); - malloc_zone_t *new_zone = - (malloc_zone_t*)asan_memalign(page_size, allocated_size, - &stack, FROM_MALLOC); - internal_memcpy(new_zone, &asan_zone, sizeof(asan_zone)); - new_zone->zone_name = NULL; // The name will be changed anyway. - if (GetMacosVersion() >= MACOS_VERSION_LION) { - // Prevent the client app from overwriting the zone contents. - // Library functions that need to modify the zone will set PROT_WRITE on it. - // This matches the behavior of malloc_create_zone() on OSX 10.7 and higher. - mprotect(new_zone, allocated_size, PROT_READ); - } - return new_zone; -} - -INTERCEPTOR(malloc_zone_t *, malloc_default_zone, void) { - ENSURE_ASAN_INITED(); - return &asan_zone; -} - -INTERCEPTOR(malloc_zone_t *, malloc_default_purgeable_zone, void) { - // FIXME: ASan should support purgeable allocations. - // https://code.google.com/p/address-sanitizer/issues/detail?id=139 - ENSURE_ASAN_INITED(); - return &asan_zone; -} - -INTERCEPTOR(void, malloc_make_purgeable, void *ptr) { - // FIXME: ASan should support purgeable allocations. Ignoring them is fine - // for now. - ENSURE_ASAN_INITED(); -} - -INTERCEPTOR(int, malloc_make_nonpurgeable, void *ptr) { - // FIXME: ASan should support purgeable allocations. Ignoring them is fine - // for now. - ENSURE_ASAN_INITED(); - // Must return 0 if the contents were not purged since the last call to - // malloc_make_purgeable(). - return 0; -} - -INTERCEPTOR(void, malloc_set_zone_name, malloc_zone_t *zone, const char *name) { - ENSURE_ASAN_INITED(); - // Allocate |strlen("asan-") + 1 + internal_strlen(name)| bytes. - size_t buflen = 6 + (name ? internal_strlen(name) : 0); - InternalScopedString new_name(buflen); - if (name && zone->introspect == asan_zone.introspect) { - new_name.append("asan-%s", name); - name = new_name.data(); - } - - // Call the system malloc's implementation for both external and our zones, - // since that appropriately changes VM region protections on the zone. - REAL(malloc_set_zone_name)(zone, name); -} - -INTERCEPTOR(void *, malloc, size_t size) { - ENSURE_ASAN_INITED(); - GET_STACK_TRACE_MALLOC; - void *res = asan_malloc(size, &stack); - return res; -} -INTERCEPTOR(void, free, void *ptr) { - ENSURE_ASAN_INITED(); - if (!ptr) return; - GET_STACK_TRACE_FREE; +using namespace __asan; +#define COMMON_MALLOC_ZONE_NAME "asan" +#define COMMON_MALLOC_ENTER() ENSURE_ASAN_INITED() +#define COMMON_MALLOC_SANITIZER_INITIALIZED asan_inited +#define COMMON_MALLOC_FORCE_LOCK() asan_mz_force_lock() +#define COMMON_MALLOC_FORCE_UNLOCK() asan_mz_force_unlock() +#define COMMON_MALLOC_MEMALIGN(alignment, size) \ + GET_STACK_TRACE_MALLOC; \ + void *p = asan_memalign(alignment, size, &stack, FROM_MALLOC) +#define COMMON_MALLOC_MALLOC(size) \ + GET_STACK_TRACE_MALLOC; \ + void *p = asan_malloc(size, &stack) +#define COMMON_MALLOC_REALLOC(ptr, size) \ + GET_STACK_TRACE_MALLOC; \ + void *p = asan_realloc(ptr, size, &stack); +#define COMMON_MALLOC_CALLOC(count, size) \ + GET_STACK_TRACE_MALLOC; \ + void *p = asan_calloc(count, size, &stack); +#define COMMON_MALLOC_VALLOC(size) \ + GET_STACK_TRACE_MALLOC; \ + void *p = asan_memalign(GetPageSizeCached(), size, &stack, FROM_MALLOC); +#define COMMON_MALLOC_FREE(ptr) \ + GET_STACK_TRACE_FREE; \ asan_free(ptr, &stack, FROM_MALLOC); -} - -INTERCEPTOR(void *, realloc, void *ptr, size_t size) { - ENSURE_ASAN_INITED(); - GET_STACK_TRACE_MALLOC; - return asan_realloc(ptr, size, &stack); -} - -INTERCEPTOR(void *, calloc, size_t nmemb, size_t size) { - ENSURE_ASAN_INITED(); - GET_STACK_TRACE_MALLOC; - return asan_calloc(nmemb, size, &stack); -} - -INTERCEPTOR(void *, valloc, size_t size) { - ENSURE_ASAN_INITED(); - GET_STACK_TRACE_MALLOC; - return asan_memalign(GetPageSizeCached(), size, &stack, FROM_MALLOC); -} - -INTERCEPTOR(size_t, malloc_good_size, size_t size) { - ENSURE_ASAN_INITED(); - return asan_zone.introspect->good_size(&asan_zone, size); -} - -INTERCEPTOR(int, posix_memalign, void **memptr, size_t alignment, size_t size) { - ENSURE_ASAN_INITED(); - CHECK(memptr); - GET_STACK_TRACE_MALLOC; - void *result = asan_memalign(alignment, size, &stack, FROM_MALLOC); - if (result) { - *memptr = result; - return 0; - } - return -1; -} - -namespace { - -// TODO(glider): the __asan_mz_* functions should be united with the Linux -// wrappers, as they are basically copied from there. -extern "C" -SANITIZER_INTERFACE_ATTRIBUTE -size_t __asan_mz_size(malloc_zone_t* zone, const void* ptr) { - return asan_mz_size(ptr); -} - -extern "C" -SANITIZER_INTERFACE_ATTRIBUTE -void *__asan_mz_malloc(malloc_zone_t *zone, uptr size) { - if (UNLIKELY(!asan_inited)) { - CHECK(system_malloc_zone); - return malloc_zone_malloc(system_malloc_zone, size); - } - GET_STACK_TRACE_MALLOC; - return asan_malloc(size, &stack); -} - -extern "C" -SANITIZER_INTERFACE_ATTRIBUTE -void *__asan_mz_calloc(malloc_zone_t *zone, size_t nmemb, size_t size) { - if (UNLIKELY(!asan_inited)) { - // Hack: dlsym calls calloc before REAL(calloc) is retrieved from dlsym. - const size_t kCallocPoolSize = 1024; - static uptr calloc_memory_for_dlsym[kCallocPoolSize]; - static size_t allocated; - size_t size_in_words = ((nmemb * size) + kWordSize - 1) / kWordSize; - void *mem = (void*)&calloc_memory_for_dlsym[allocated]; - allocated += size_in_words; - CHECK(allocated < kCallocPoolSize); - return mem; - } - GET_STACK_TRACE_MALLOC; - return asan_calloc(nmemb, size, &stack); -} - -extern "C" -SANITIZER_INTERFACE_ATTRIBUTE -void *__asan_mz_valloc(malloc_zone_t *zone, size_t size) { - if (UNLIKELY(!asan_inited)) { - CHECK(system_malloc_zone); - return malloc_zone_valloc(system_malloc_zone, size); - } - GET_STACK_TRACE_MALLOC; - return asan_memalign(GetPageSizeCached(), size, &stack, FROM_MALLOC); -} - -#define GET_ZONE_FOR_PTR(ptr) \ - malloc_zone_t *zone_ptr = malloc_zone_from_ptr(ptr); \ - const char *zone_name = (zone_ptr == 0) ? 0 : zone_ptr->zone_name - -void ALWAYS_INLINE free_common(void *context, void *ptr) { - if (!ptr) return; - GET_STACK_TRACE_FREE; - // FIXME: need to retire this flag. - if (!flags()->mac_ignore_invalid_free) { - asan_free(ptr, &stack, FROM_MALLOC); - } else { - GET_ZONE_FOR_PTR(ptr); - WarnMacFreeUnallocated((uptr)ptr, (uptr)zone_ptr, zone_name, &stack); - return; - } -} - -// TODO(glider): the allocation callbacks need to be refactored. -extern "C" -SANITIZER_INTERFACE_ATTRIBUTE -void __asan_mz_free(malloc_zone_t *zone, void *ptr) { - free_common(zone, ptr); -} - -extern "C" -SANITIZER_INTERFACE_ATTRIBUTE -void *__asan_mz_realloc(malloc_zone_t *zone, void *ptr, size_t size) { - if (!ptr) { - GET_STACK_TRACE_MALLOC; - return asan_malloc(size, &stack); - } else { - if (asan_mz_size(ptr)) { - GET_STACK_TRACE_MALLOC; - return asan_realloc(ptr, size, &stack); - } else { - // We can't recover from reallocating an unknown address, because - // this would require reading at most |size| bytes from - // potentially unaccessible memory. - GET_STACK_TRACE_FREE; - GET_ZONE_FOR_PTR(ptr); - ReportMacMzReallocUnknown((uptr)ptr, (uptr)zone_ptr, zone_name, &stack); - } - } -} - -extern "C" -SANITIZER_INTERFACE_ATTRIBUTE -void __asan_mz_destroy(malloc_zone_t* zone) { - // A no-op -- we will not be destroyed! - Report("__asan_mz_destroy() called -- ignoring\n"); -} - -extern "C" -SANITIZER_INTERFACE_ATTRIBUTE -void *__asan_mz_memalign(malloc_zone_t *zone, size_t align, size_t size) { - if (UNLIKELY(!asan_inited)) { - CHECK(system_malloc_zone); - return malloc_zone_memalign(system_malloc_zone, align, size); - } - GET_STACK_TRACE_MALLOC; - return asan_memalign(align, size, &stack, FROM_MALLOC); -} - -// This function is currently unused, and we build with -Werror. -#if 0 -void __asan_mz_free_definite_size( - malloc_zone_t* zone, void *ptr, size_t size) { - // TODO(glider): check that |size| is valid. - UNIMPLEMENTED(); -} -#endif - -kern_return_t mi_enumerator(task_t task, void *, - unsigned type_mask, vm_address_t zone_address, - memory_reader_t reader, - vm_range_recorder_t recorder) { - // Should enumerate all the pointers we have. Seems like a lot of work. - return KERN_FAILURE; -} - -size_t mi_good_size(malloc_zone_t *zone, size_t size) { - // I think it's always safe to return size, but we maybe could do better. - return size; -} - -boolean_t mi_check(malloc_zone_t *zone) { - UNIMPLEMENTED(); -} - -void mi_print(malloc_zone_t *zone, boolean_t verbose) { - UNIMPLEMENTED(); -} - -void mi_log(malloc_zone_t *zone, void *address) { - // I don't think we support anything like this -} - -void mi_force_lock(malloc_zone_t *zone) { - asan_mz_force_lock(); -} - -void mi_force_unlock(malloc_zone_t *zone) { - asan_mz_force_unlock(); -} - -void mi_statistics(malloc_zone_t *zone, malloc_statistics_t *stats) { - AsanMallocStats malloc_stats; - FillMallocStatistics(&malloc_stats); - CHECK(sizeof(malloc_statistics_t) == sizeof(AsanMallocStats)); +#define COMMON_MALLOC_SIZE(ptr) \ + uptr size = asan_mz_size(ptr); +#define COMMON_MALLOC_FILL_STATS(zone, stats) \ + AsanMallocStats malloc_stats; \ + FillMallocStatistics(&malloc_stats); \ + CHECK(sizeof(malloc_statistics_t) == sizeof(AsanMallocStats)); \ internal_memcpy(stats, &malloc_stats, sizeof(malloc_statistics_t)); -} - -boolean_t mi_zone_locked(malloc_zone_t *zone) { - // UNIMPLEMENTED(); - return false; -} - -} // unnamed namespace - -namespace __asan { +#define COMMON_MALLOC_REPORT_UNKNOWN_REALLOC(ptr, zone_ptr, zone_name) \ + GET_STACK_TRACE_FREE; \ + ReportMacMzReallocUnknown((uptr)ptr, (uptr)zone_ptr, zone_name, &stack); +#define COMMON_MALLOC_NAMESPACE __asan -void ReplaceSystemMalloc() { - static malloc_introspection_t asan_introspection; - // Ok to use internal_memset, these places are not performance-critical. - internal_memset(&asan_introspection, 0, sizeof(asan_introspection)); +#include "sanitizer_common/sanitizer_malloc_mac.inc" - asan_introspection.enumerator = &mi_enumerator; - asan_introspection.good_size = &mi_good_size; - asan_introspection.check = &mi_check; - asan_introspection.print = &mi_print; - asan_introspection.log = &mi_log; - asan_introspection.force_lock = &mi_force_lock; - asan_introspection.force_unlock = &mi_force_unlock; - asan_introspection.statistics = &mi_statistics; - asan_introspection.zone_locked = &mi_zone_locked; - - internal_memset(&asan_zone, 0, sizeof(malloc_zone_t)); - - // Use version 6 for OSX >= 10.6. - asan_zone.version = 6; - asan_zone.zone_name = "asan"; - asan_zone.size = &__asan_mz_size; - asan_zone.malloc = &__asan_mz_malloc; - asan_zone.calloc = &__asan_mz_calloc; - asan_zone.valloc = &__asan_mz_valloc; - asan_zone.free = &__asan_mz_free; - asan_zone.realloc = &__asan_mz_realloc; - asan_zone.destroy = &__asan_mz_destroy; - asan_zone.batch_malloc = 0; - asan_zone.batch_free = 0; - asan_zone.free_definite_size = 0; - asan_zone.memalign = &__asan_mz_memalign; - asan_zone.introspect = &asan_introspection; - - // Register the ASan zone. - malloc_zone_register(&asan_zone); -} -} // namespace __asan - -#endif // SANITIZER_MAC +#endif diff --git a/lib/asan/asan_mapping.h b/lib/asan/asan_mapping.h index f9e1a527de18..8fe347c8bad0 100644 --- a/lib/asan/asan_mapping.h +++ b/lib/asan/asan_mapping.h @@ -17,7 +17,7 @@ #include "asan_internal.h" // The full explanation of the memory mapping could be found here: -// http://code.google.com/p/address-sanitizer/wiki/AddressSanitizerAlgorithm +// https://github.com/google/sanitizers/wiki/AddressSanitizerAlgorithm // // Typical shadow mapping on Linux/x86_64 with SHADOW_OFFSET == 0x00007fff8000: // || `[0x10007fff8000, 0x7fffffffffff]` || HighMem || @@ -73,6 +73,20 @@ // || `[0x2000000000, 0x23ffffffff]` || LowShadow || // || `[0x0000000000, 0x1fffffffff]` || LowMem || // +// Default Linux/AArch64 (39-bit VMA) mapping: +// || `[0x2000000000, 0x7fffffffff]` || highmem || +// || `[0x1400000000, 0x1fffffffff]` || highshadow || +// || `[0x1200000000, 0x13ffffffff]` || shadowgap || +// || `[0x1000000000, 0x11ffffffff]` || lowshadow || +// || `[0x0000000000, 0x0fffffffff]` || lowmem || +// +// Default Linux/AArch64 (42-bit VMA) mapping: +// || `[0x10000000000, 0x3ffffffffff]` || highmem || +// || `[0x0a000000000, 0x0ffffffffff]` || highshadow || +// || `[0x09000000000, 0x09fffffffff]` || shadowgap || +// || `[0x08000000000, 0x08fffffffff]` || lowshadow || +// || `[0x00000000000, 0x07fffffffff]` || lowmem || +// // Shadow mapping on FreeBSD/x86-64 with SHADOW_OFFSET == 0x400000000000: // || `[0x500000000000, 0x7fffffffffff]` || HighMem || // || `[0x4a0000000000, 0x4fffffffffff]` || HighShadow || @@ -113,11 +127,12 @@ static const u64 kFreeBSD_ShadowOffset64 = 1ULL << 46; // 0x400000000000 static const u64 kWindowsShadowOffset32 = 3ULL << 28; // 0x30000000 #define SHADOW_SCALE kDefaultShadowScale -#if SANITIZER_ANDROID -# define SHADOW_OFFSET (0) -#else -# if SANITIZER_WORDSIZE == 32 -# if defined(__mips__) + + +#if SANITIZER_WORDSIZE == 32 +# if SANITIZER_ANDROID +# define SHADOW_OFFSET (0) +# elif defined(__mips__) # define SHADOW_OFFSET kMIPS32_ShadowOffset32 # elif SANITIZER_FREEBSD # define SHADOW_OFFSET kFreeBSD_ShadowOffset32 @@ -130,7 +145,7 @@ static const u64 kWindowsShadowOffset32 = 3ULL << 28; // 0x30000000 # else # define SHADOW_OFFSET kDefaultShadowOffset32 # endif -# else +#else # if defined(__aarch64__) # define SHADOW_OFFSET kAArch64_ShadowOffset64 # elif defined(__powerpc64__) @@ -148,7 +163,6 @@ static const u64 kWindowsShadowOffset32 = 3ULL << 28; // 0x30000000 # else # define SHADOW_OFFSET kDefaultShort64bitShadowOffset # endif -# endif #endif #define SHADOW_GRANULARITY (1ULL << SHADOW_SCALE) @@ -171,7 +185,8 @@ static const u64 kWindowsShadowOffset32 = 3ULL << 28; // 0x30000000 // With the zero shadow base we can not actually map pages starting from 0. // This constant is somewhat arbitrary. -#define kZeroBaseShadowStart (1 << 18) +#define kZeroBaseShadowStart 0 +#define kZeroBaseMaxShadowStart (1 << 18) #define kShadowGapBeg (kLowShadowEnd ? kLowShadowEnd + 1 \ : kZeroBaseShadowStart) diff --git a/lib/asan/asan_new_delete.cc b/lib/asan/asan_new_delete.cc index e48bdaf03dd3..b5ba13ef4055 100644 --- a/lib/asan/asan_new_delete.cc +++ b/lib/asan/asan_new_delete.cc @@ -30,7 +30,7 @@ using namespace __asan; // NOLINT // This code has issues on OSX. -// See https://code.google.com/p/address-sanitizer/issues/detail?id=131. +// See https://github.com/google/sanitizers/issues/131. // Fake std::nothrow_t to avoid including <new>. namespace std { @@ -90,11 +90,11 @@ INTERCEPTOR(void *, _ZnamRKSt9nothrow_t, size_t size, std::nothrow_t const&) { #if !SANITIZER_MAC CXX_OPERATOR_ATTRIBUTE -void operator delete(void *ptr) throw() { +void operator delete(void *ptr) NOEXCEPT { OPERATOR_DELETE_BODY(FROM_NEW); } CXX_OPERATOR_ATTRIBUTE -void operator delete[](void *ptr) throw() { +void operator delete[](void *ptr) NOEXCEPT { OPERATOR_DELETE_BODY(FROM_NEW_BR); } CXX_OPERATOR_ATTRIBUTE @@ -106,12 +106,12 @@ void operator delete[](void *ptr, std::nothrow_t const&) { OPERATOR_DELETE_BODY(FROM_NEW_BR); } CXX_OPERATOR_ATTRIBUTE -void operator delete(void *ptr, size_t size) throw() { +void operator delete(void *ptr, size_t size) NOEXCEPT { GET_STACK_TRACE_FREE; asan_sized_free(ptr, size, &stack, FROM_NEW); } CXX_OPERATOR_ATTRIBUTE -void operator delete[](void *ptr, size_t size) throw() { +void operator delete[](void *ptr, size_t size) NOEXCEPT { GET_STACK_TRACE_FREE; asan_sized_free(ptr, size, &stack, FROM_NEW_BR); } diff --git a/lib/asan/asan_poisoning.cc b/lib/asan/asan_poisoning.cc index 569d359aa425..f77ab8780bb7 100644 --- a/lib/asan/asan_poisoning.cc +++ b/lib/asan/asan_poisoning.cc @@ -102,7 +102,7 @@ using namespace __asan; // NOLINT // that user program (un)poisons the memory it owns. It poisons memory // conservatively, and unpoisons progressively to make sure asan shadow // mapping invariant is preserved (see detailed mapping description here: -// http://code.google.com/p/address-sanitizer/wiki/AddressSanitizerAlgorithm). +// https://github.com/google/sanitizers/wiki/AddressSanitizerAlgorithm). // // * if user asks to poison region [left, right), the program poisons // at least [left, AlignDown(right)). @@ -354,7 +354,7 @@ void __sanitizer_annotate_contiguous_container(const void *beg_p, // Make a quick sanity check that we are indeed in this state. // // FIXME: Two of these three checks are disabled until we fix - // https://code.google.com/p/address-sanitizer/issues/detail?id=258. + // https://github.com/google/sanitizers/issues/258. // if (d1 != d2) // CHECK_EQ(*(u8*)MemToShadow(d1), old_mid - d1); if (a + granularity <= d1) @@ -375,10 +375,10 @@ void __sanitizer_annotate_contiguous_container(const void *beg_p, } } -int __sanitizer_verify_contiguous_container(const void *beg_p, - const void *mid_p, - const void *end_p) { - if (!flags()->detect_container_overflow) return 1; +const void *__sanitizer_contiguous_container_find_bad_address( + const void *beg_p, const void *mid_p, const void *end_p) { + if (!flags()->detect_container_overflow) + return nullptr; uptr beg = reinterpret_cast<uptr>(beg_p); uptr end = reinterpret_cast<uptr>(end_p); uptr mid = reinterpret_cast<uptr>(mid_p); @@ -395,17 +395,24 @@ int __sanitizer_verify_contiguous_container(const void *beg_p, uptr r3_end = end; for (uptr i = r1_beg; i < r1_end; i++) if (AddressIsPoisoned(i)) - return 0; + return reinterpret_cast<const void *>(i); for (uptr i = r2_beg; i < mid; i++) if (AddressIsPoisoned(i)) - return 0; + return reinterpret_cast<const void *>(i); for (uptr i = mid; i < r2_end; i++) if (!AddressIsPoisoned(i)) - return 0; + return reinterpret_cast<const void *>(i); for (uptr i = r3_beg; i < r3_end; i++) if (!AddressIsPoisoned(i)) - return 0; - return 1; + return reinterpret_cast<const void *>(i); + return nullptr; +} + +int __sanitizer_verify_contiguous_container(const void *beg_p, + const void *mid_p, + const void *end_p) { + return __sanitizer_contiguous_container_find_bad_address(beg_p, mid_p, + end_p) == nullptr; } extern "C" SANITIZER_INTERFACE_ATTRIBUTE diff --git a/lib/asan/asan_posix.cc b/lib/asan/asan_posix.cc index 2e857f6f624c..9e01bcd091bf 100644 --- a/lib/asan/asan_posix.cc +++ b/lib/asan/asan_posix.cc @@ -33,11 +33,11 @@ namespace __asan { -void AsanOnSIGSEGV(int, void *siginfo, void *context) { +void AsanOnDeadlySignal(int signo, void *siginfo, void *context) { ScopedDeadlySignal signal_scope(GetCurrentThread()); int code = (int)((siginfo_t*)siginfo)->si_code; // Write the first message using the bullet-proof write. - if (13 != internal_write(2, "ASAN:SIGSEGV\n", 13)) Die(); + if (18 != internal_write(2, "ASAN:DEADLYSIGNAL\n", 18)) Die(); SignalContext sig = SignalContext::Create(siginfo, context); // Access at a reasonable offset above SP, or slightly below it (to account @@ -75,8 +75,12 @@ void AsanOnSIGSEGV(int, void *siginfo, void *context) { // unaligned memory access. if (IsStackAccess && (code == si_SEGV_MAPERR || code == si_SEGV_ACCERR)) ReportStackOverflow(sig); + else if (signo == SIGFPE) + ReportDeadlySignal("FPE", sig); + else if (signo == SIGILL) + ReportDeadlySignal("ILL", sig); else - ReportSIGSEGV("SEGV", sig); + ReportDeadlySignal("SEGV", sig); } // ---------------------- TSD ---------------- {{{1 diff --git a/lib/asan/asan_report.cc b/lib/asan/asan_report.cc index c1681e644464..0fb60846c3b4 100644 --- a/lib/asan/asan_report.cc +++ b/lib/asan/asan_report.cc @@ -11,6 +11,7 @@ // // This file contains error reporting code. //===----------------------------------------------------------------------===// + #include "asan_flags.h" #include "asan_internal.h" #include "asan_mapping.h" @@ -27,9 +28,11 @@ namespace __asan { // -------------------- User-specified callbacks ----------------- {{{1 static void (*error_report_callback)(const char*); -static char *error_message_buffer = 0; +static char *error_message_buffer = nullptr; static uptr error_message_buffer_pos = 0; -static uptr error_message_buffer_size = 0; +static BlockingMutex error_message_buf_mutex(LINKER_INITIALIZED); +static const unsigned kAsanBuggyPcPoolSize = 25; +static __sanitizer::atomic_uintptr_t AsanBuggyPcPool[kAsanBuggyPcPoolSize]; struct ReportData { uptr pc; @@ -45,16 +48,20 @@ static bool report_happened = false; static ReportData report_data = {}; void AppendToErrorMessageBuffer(const char *buffer) { - if (error_message_buffer) { - uptr length = internal_strlen(buffer); - CHECK_GE(error_message_buffer_size, error_message_buffer_pos); - uptr remaining = error_message_buffer_size - error_message_buffer_pos; - internal_strncpy(error_message_buffer + error_message_buffer_pos, - buffer, remaining); - error_message_buffer[error_message_buffer_size - 1] = '\0'; - // FIXME: reallocate the buffer instead of truncating the message. - error_message_buffer_pos += Min(remaining, length); + BlockingMutexLock l(&error_message_buf_mutex); + if (!error_message_buffer) { + error_message_buffer = + (char*)MmapOrDieQuietly(kErrorMessageBufferSize, __func__); + error_message_buffer_pos = 0; } + uptr length = internal_strlen(buffer); + RAW_CHECK(kErrorMessageBufferSize >= error_message_buffer_pos); + uptr remaining = kErrorMessageBufferSize - error_message_buffer_pos; + internal_strncpy(error_message_buffer + error_message_buffer_pos, + buffer, remaining); + error_message_buffer[kErrorMessageBufferSize - 1] = '\0'; + // FIXME: reallocate the buffer instead of truncating the message. + error_message_buffer_pos += Min(remaining, length); } // ---------------------- Decorator ------------------------------ {{{1 @@ -373,7 +380,7 @@ static void PrintAccessAndVarIntersection(const StackVarDescr &var, uptr addr, uptr next_var_beg) { uptr var_end = var.beg + var.size; uptr addr_end = addr + access_size; - const char *pos_descr = 0; + const char *pos_descr = nullptr; // If the variable [var.beg, var_end) is the nearest variable to the // current memory access, indicate it in the log. if (addr >= var.beg) { @@ -544,7 +551,7 @@ void DescribeHeapAddress(uptr addr, uptr access_size) { StackTrace alloc_stack = chunk.GetAllocStack(); char tname[128]; Decorator d; - AsanThreadContext *free_thread = 0; + AsanThreadContext *free_thread = nullptr; if (chunk.FreeTid() != kInvalidTid) { free_thread = GetThreadContextByTidLocked(chunk.FreeTid()); Printf("%sfreed by thread T%d%s here:%s\n", d.Allocation(), @@ -621,26 +628,93 @@ void DescribeThread(AsanThreadContext *context) { // immediately after printing error report. class ScopedInErrorReport { public: - explicit ScopedInErrorReport(ReportData *report = nullptr) { - static atomic_uint32_t num_calls; - static u32 reporting_thread_tid; - if (atomic_fetch_add(&num_calls, 1, memory_order_relaxed) != 0) { + explicit ScopedInErrorReport(ReportData *report = nullptr, + bool fatal = false) { + halt_on_error_ = fatal || flags()->halt_on_error; + + if (lock_.TryLock()) { + StartReporting(report); + return; + } + + // ASan found two bugs in different threads simultaneously. + + u32 current_tid = GetCurrentTidOrInvalid(); + if (reporting_thread_tid_ == current_tid || + reporting_thread_tid_ == kInvalidTid) { + // This is either asynch signal or nested error during error reporting. + // Fail simple to avoid deadlocks in Report(). + + // Can't use Report() here because of potential deadlocks + // in nested signal handlers. + const char msg[] = "AddressSanitizer: nested bug in the same thread, " + "aborting.\n"; + WriteToFile(kStderrFd, msg, sizeof(msg)); + + internal__exit(common_flags()->exitcode); + } + + if (halt_on_error_) { // Do not print more than one report, otherwise they will mix up. // Error reporting functions shouldn't return at this situation, as - // they are defined as no-return. + // they are effectively no-returns. + Report("AddressSanitizer: while reporting a bug found another one. " - "Ignoring.\n"); - u32 current_tid = GetCurrentTidOrInvalid(); - if (current_tid != reporting_thread_tid) { - // ASan found two bugs in different threads simultaneously. Sleep - // long enough to make sure that the thread which started to print - // an error report will finish doing it. - SleepForSeconds(Max(100, flags()->sleep_before_dying + 1)); - } + "Ignoring.\n"); + + // Sleep long enough to make sure that the thread which started + // to print an error report will finish doing it. + SleepForSeconds(Max(100, flags()->sleep_before_dying + 1)); + // If we're still not dead for some reason, use raw _exit() instead of // Die() to bypass any additional checks. - internal__exit(flags()->exitcode); + internal__exit(common_flags()->exitcode); + } else { + // The other thread will eventually finish reporting + // so it's safe to wait + lock_.Lock(); + } + + StartReporting(report); + } + + ~ScopedInErrorReport() { + // Make sure the current thread is announced. + DescribeThread(GetCurrentThread()); + // We may want to grab this lock again when printing stats. + asanThreadRegistry().Unlock(); + // Print memory stats. + if (flags()->print_stats) + __asan_print_accumulated_stats(); + + // Copy the message buffer so that we could start logging without holding a + // lock that gets aquired during printing. + InternalScopedBuffer<char> buffer_copy(kErrorMessageBufferSize); + { + BlockingMutexLock l(&error_message_buf_mutex); + internal_memcpy(buffer_copy.data(), + error_message_buffer, kErrorMessageBufferSize); + } + + // Remove color sequences since logs cannot print them. + RemoveANSIEscapeSequencesFromString(buffer_copy.data()); + + LogFullErrorReport(buffer_copy.data()); + + if (error_report_callback) { + error_report_callback(buffer_copy.data()); } + CommonSanitizerReportMutex.Unlock(); + reporting_thread_tid_ = kInvalidTid; + lock_.Unlock(); + if (halt_on_error_) { + Report("ABORTING\n"); + Die(); + } + } + + private: + void StartReporting(ReportData *report) { if (report) report_data = *report; report_happened = true; ASAN_ON_ERROR(); @@ -650,27 +724,19 @@ class ScopedInErrorReport { // recursive reports. asanThreadRegistry().Lock(); CommonSanitizerReportMutex.Lock(); - reporting_thread_tid = GetCurrentTidOrInvalid(); + reporting_thread_tid_ = GetCurrentTidOrInvalid(); Printf("====================================================" "=============\n"); } - // Destructor is NORETURN, as functions that report errors are. - NORETURN ~ScopedInErrorReport() { - // Make sure the current thread is announced. - DescribeThread(GetCurrentThread()); - // We may want to grab this lock again when printing stats. - asanThreadRegistry().Unlock(); - // Print memory stats. - if (flags()->print_stats) - __asan_print_accumulated_stats(); - if (error_report_callback) { - error_report_callback(error_message_buffer); - } - Report("ABORTING\n"); - Die(); - } + + static StaticSpinMutex lock_; + static u32 reporting_thread_tid_; + bool halt_on_error_; }; +StaticSpinMutex ScopedInErrorReport::lock_; +u32 ScopedInErrorReport::reporting_thread_tid_; + void ReportStackOverflow(const SignalContext &sig) { ScopedInErrorReport in_report; Decorator d; @@ -686,8 +752,8 @@ void ReportStackOverflow(const SignalContext &sig) { ReportErrorSummary("stack-overflow", &stack); } -void ReportSIGSEGV(const char *description, const SignalContext &sig) { - ScopedInErrorReport in_report; +void ReportDeadlySignal(const char *description, const SignalContext &sig) { + ScopedInErrorReport in_report(/*report*/nullptr, /*fatal*/true); Decorator d; Printf("%s", d.Warning()); Report( @@ -703,7 +769,7 @@ void ReportSIGSEGV(const char *description, const SignalContext &sig) { stack.Print(); MaybeDumpInstructionBytes(sig.pc); Printf("AddressSanitizer can not provide additional info.\n"); - ReportErrorSummary("SEGV", &stack); + ReportErrorSummary(description, &stack); } void ReportDoubleFree(uptr addr, BufferedStackTrace *free_stack) { @@ -744,7 +810,7 @@ void ReportNewDeleteSizeMismatch(uptr addr, uptr delete_size, stack.Print(); DescribeHeapAddress(addr, 1); ReportErrorSummary("new-delete-type-mismatch", &stack); - Report("HINT: if you don't care about these warnings you may set " + Report("HINT: if you don't care about these errors you may set " "ASAN_OPTIONS=new_delete_type_mismatch=0\n"); } @@ -784,7 +850,7 @@ void ReportAllocTypeMismatch(uptr addr, BufferedStackTrace *free_stack, stack.Print(); DescribeHeapAddress(addr, 1); ReportErrorSummary("alloc-dealloc-mismatch", &stack); - Report("HINT: if you don't care about these warnings you may set " + Report("HINT: if you don't care about these errors you may set " "ASAN_OPTIONS=alloc_dealloc_mismatch=0\n"); } @@ -886,7 +952,7 @@ void ReportODRViolation(const __asan_global *g1, u32 stack_id1, Printf(" [2]:\n"); StackDepotGet(stack_id2).Print(); } - Report("HINT: if you don't care about these warnings you may set " + Report("HINT: if you don't care about these errors you may set " "ASAN_OPTIONS=detect_odr_violation=0\n"); InternalScopedString error_msg(256); error_msg.append("odr-violation: global '%s' at %s", @@ -925,17 +991,6 @@ static INLINE void CheckForInvalidPointerPair(void *p1, void *p2) { } // ----------------------- Mac-specific reports ----------------- {{{1 -void WarnMacFreeUnallocated(uptr addr, uptr zone_ptr, const char *zone_name, - BufferedStackTrace *stack) { - // Just print a warning here. - Printf("free_common(%p) -- attempting to free unallocated memory.\n" - "AddressSanitizer is ignoring this error on Mac OS now.\n", - addr); - PrintZoneForPointer(addr, zone_ptr, zone_name); - stack->Print(); - DescribeHeapAddress(addr, 1); -} - void ReportMacMzReallocUnknown(uptr addr, uptr zone_ptr, const char *zone_name, BufferedStackTrace *stack) { ScopedInErrorReport in_report; @@ -947,24 +1002,23 @@ void ReportMacMzReallocUnknown(uptr addr, uptr zone_ptr, const char *zone_name, DescribeHeapAddress(addr, 1); } -void ReportMacCfReallocUnknown(uptr addr, uptr zone_ptr, const char *zone_name, - BufferedStackTrace *stack) { - ScopedInErrorReport in_report; - Printf("cf_realloc(%p) -- attempting to realloc unallocated memory.\n" - "This is an unrecoverable problem, exiting now.\n", - addr); - PrintZoneForPointer(addr, zone_ptr, zone_name); - stack->Print(); - DescribeHeapAddress(addr, 1); +// -------------- SuppressErrorReport -------------- {{{1 +// Avoid error reports duplicating for ASan recover mode. +static bool SuppressErrorReport(uptr pc) { + if (!common_flags()->suppress_equal_pcs) return false; + for (unsigned i = 0; i < kAsanBuggyPcPoolSize; i++) { + uptr cmp = atomic_load_relaxed(&AsanBuggyPcPool[i]); + if (cmp == 0 && atomic_compare_exchange_strong(&AsanBuggyPcPool[i], &cmp, + pc, memory_order_relaxed)) + return false; + if (cmp == pc) return true; + } + Die(); } -} // namespace __asan - -// --------------------------- Interface --------------------- {{{1 -using namespace __asan; // NOLINT - -void __asan_report_error(uptr pc, uptr bp, uptr sp, uptr addr, int is_write, - uptr access_size, u32 exp) { +void ReportGenericError(uptr pc, uptr bp, uptr sp, uptr addr, bool is_write, + uptr access_size, u32 exp, bool fatal) { + if (!fatal && SuppressErrorReport(pc)) return; ENABLE_FRAME_POINTER; // Optimization experiments. @@ -1033,7 +1087,7 @@ void __asan_report_error(uptr pc, uptr bp, uptr sp, uptr addr, int is_write, ReportData report = { pc, sp, bp, addr, (bool)is_write, access_size, bug_descr }; - ScopedInErrorReport in_report(&report); + ScopedInErrorReport in_report(&report, fatal); Decorator d; Printf("%s", d.Warning()); @@ -1059,14 +1113,21 @@ void __asan_report_error(uptr pc, uptr bp, uptr sp, uptr addr, int is_write, PrintShadowMemoryForAddress(addr); } +} // namespace __asan + +// --------------------------- Interface --------------------- {{{1 +using namespace __asan; // NOLINT + +void __asan_report_error(uptr pc, uptr bp, uptr sp, uptr addr, int is_write, + uptr access_size, u32 exp) { + ENABLE_FRAME_POINTER; + bool fatal = flags()->halt_on_error; + ReportGenericError(pc, bp, sp, addr, is_write, access_size, exp, fatal); +} + void NOINLINE __asan_set_error_report_callback(void (*callback)(const char*)) { + BlockingMutexLock l(&error_message_buf_mutex); error_report_callback = callback; - if (callback) { - error_message_buffer_size = 1 << 16; - error_message_buffer = - (char*)MmapOrDie(error_message_buffer_size, __func__); - error_message_buffer_pos = 0; - } } void __asan_describe_address(uptr addr) { @@ -1117,7 +1178,7 @@ SANITIZER_INTERFACE_ATTRIBUTE void __sanitizer_ptr_cmp(void *a, void *b) { CheckForInvalidPointerPair(a, b); } -} // extern "C" +} // extern "C" #if !SANITIZER_SUPPORTS_WEAK_HOOKS // Provide default implementation of __asan_on_error that does nothing diff --git a/lib/asan/asan_report.h b/lib/asan/asan_report.h index e2786b0f260c..559b8adfd51d 100644 --- a/lib/asan/asan_report.h +++ b/lib/asan/asan_report.h @@ -49,44 +49,39 @@ bool DescribeAddressIfStack(uptr addr, uptr access_size); void DescribeThread(AsanThreadContext *context); // Different kinds of error reports. -void NORETURN ReportStackOverflow(const SignalContext &sig); -void NORETURN ReportSIGSEGV(const char *description, const SignalContext &sig); -void NORETURN ReportNewDeleteSizeMismatch(uptr addr, uptr delete_size, - BufferedStackTrace *free_stack); -void NORETURN ReportDoubleFree(uptr addr, BufferedStackTrace *free_stack); -void NORETURN ReportFreeNotMalloced(uptr addr, BufferedStackTrace *free_stack); -void NORETURN ReportAllocTypeMismatch(uptr addr, BufferedStackTrace *free_stack, - AllocType alloc_type, - AllocType dealloc_type); -void NORETURN - ReportMallocUsableSizeNotOwned(uptr addr, BufferedStackTrace *stack); -void NORETURN - ReportSanitizerGetAllocatedSizeNotOwned(uptr addr, - BufferedStackTrace *stack); -void NORETURN - ReportStringFunctionMemoryRangesOverlap(const char *function, - const char *offset1, uptr length1, - const char *offset2, uptr length2, - BufferedStackTrace *stack); -void NORETURN ReportStringFunctionSizeOverflow(uptr offset, uptr size, - BufferedStackTrace *stack); -void NORETURN - ReportBadParamsToAnnotateContiguousContainer(uptr beg, uptr end, - uptr old_mid, uptr new_mid, - BufferedStackTrace *stack); +void ReportGenericError(uptr pc, uptr bp, uptr sp, uptr addr, bool is_write, + uptr access_size, u32 exp, bool fatal); +void ReportStackOverflow(const SignalContext &sig); +void ReportDeadlySignal(const char *description, const SignalContext &sig); +void ReportNewDeleteSizeMismatch(uptr addr, uptr delete_size, + BufferedStackTrace *free_stack); +void ReportDoubleFree(uptr addr, BufferedStackTrace *free_stack); +void ReportFreeNotMalloced(uptr addr, BufferedStackTrace *free_stack); +void ReportAllocTypeMismatch(uptr addr, BufferedStackTrace *free_stack, + AllocType alloc_type, + AllocType dealloc_type); +void ReportMallocUsableSizeNotOwned(uptr addr, BufferedStackTrace *stack); +void ReportSanitizerGetAllocatedSizeNotOwned(uptr addr, + BufferedStackTrace *stack); +void ReportStringFunctionMemoryRangesOverlap(const char *function, + const char *offset1, uptr length1, + const char *offset2, uptr length2, + BufferedStackTrace *stack); +void ReportStringFunctionSizeOverflow(uptr offset, uptr size, + BufferedStackTrace *stack); +void ReportBadParamsToAnnotateContiguousContainer(uptr beg, uptr end, + uptr old_mid, uptr new_mid, + BufferedStackTrace *stack); -void NORETURN -ReportODRViolation(const __asan_global *g1, u32 stack_id1, - const __asan_global *g2, u32 stack_id2); +void ReportODRViolation(const __asan_global *g1, u32 stack_id1, + const __asan_global *g2, u32 stack_id2); // Mac-specific errors and warnings. -void WarnMacFreeUnallocated(uptr addr, uptr zone_ptr, const char *zone_name, - BufferedStackTrace *stack); -void NORETURN ReportMacMzReallocUnknown(uptr addr, uptr zone_ptr, - const char *zone_name, - BufferedStackTrace *stack); -void NORETURN ReportMacCfReallocUnknown(uptr addr, uptr zone_ptr, - const char *zone_name, - BufferedStackTrace *stack); +void ReportMacMzReallocUnknown(uptr addr, uptr zone_ptr, + const char *zone_name, + BufferedStackTrace *stack); +void ReportMacCfReallocUnknown(uptr addr, uptr zone_ptr, + const char *zone_name, + BufferedStackTrace *stack); } // namespace __asan diff --git a/lib/asan/asan_rtl.cc b/lib/asan/asan_rtl.cc index a8d92b915a9a..7b8b5dd9be1b 100644 --- a/lib/asan/asan_rtl.cc +++ b/lib/asan/asan_rtl.cc @@ -11,6 +11,7 @@ // // Main file of the ASan run-time library. //===----------------------------------------------------------------------===// + #include "asan_activation.h" #include "asan_allocator.h" #include "asan_interceptors.h" @@ -56,11 +57,6 @@ static void AsanDie() { UnmapOrDie((void*)kLowShadowBeg, kHighShadowEnd - kLowShadowBeg); } } - if (common_flags()->coverage) - __sanitizer_cov_dump(); - if (flags()->abort_on_error) - Abort(); - internal__exit(flags()->exitcode); } static void AsanCheckFailed(const char *file, int line, const char *cond, @@ -117,13 +113,18 @@ static void OnLowLevelAllocate(uptr ptr, uptr size) { extern "C" NOINLINE INTERFACE_ATTRIBUTE \ void __asan_report_ ## type ## size(uptr addr) { \ GET_CALLER_PC_BP_SP; \ - __asan_report_error(pc, bp, sp, addr, is_write, size, 0); \ + ReportGenericError(pc, bp, sp, addr, is_write, size, 0, true); \ } \ extern "C" NOINLINE INTERFACE_ATTRIBUTE \ void __asan_report_exp_ ## type ## size(uptr addr, u32 exp) { \ GET_CALLER_PC_BP_SP; \ - __asan_report_error(pc, bp, sp, addr, is_write, size, exp); \ -} + ReportGenericError(pc, bp, sp, addr, is_write, size, exp, true); \ +} \ +extern "C" NOINLINE INTERFACE_ATTRIBUTE \ +void __asan_report_ ## type ## size ## _noabort(uptr addr) { \ + GET_CALLER_PC_BP_SP; \ + ReportGenericError(pc, bp, sp, addr, is_write, size, 0, false); \ +} \ ASAN_REPORT_ERROR(load, false, 1) ASAN_REPORT_ERROR(load, false, 2) @@ -136,22 +137,27 @@ ASAN_REPORT_ERROR(store, true, 4) ASAN_REPORT_ERROR(store, true, 8) ASAN_REPORT_ERROR(store, true, 16) -#define ASAN_REPORT_ERROR_N(type, is_write) \ -extern "C" NOINLINE INTERFACE_ATTRIBUTE \ -void __asan_report_ ## type ## _n(uptr addr, uptr size) { \ - GET_CALLER_PC_BP_SP; \ - __asan_report_error(pc, bp, sp, addr, is_write, size, 0); \ -} \ -extern "C" NOINLINE INTERFACE_ATTRIBUTE \ +#define ASAN_REPORT_ERROR_N(type, is_write) \ +extern "C" NOINLINE INTERFACE_ATTRIBUTE \ +void __asan_report_ ## type ## _n(uptr addr, uptr size) { \ + GET_CALLER_PC_BP_SP; \ + ReportGenericError(pc, bp, sp, addr, is_write, size, 0, true); \ +} \ +extern "C" NOINLINE INTERFACE_ATTRIBUTE \ void __asan_report_exp_ ## type ## _n(uptr addr, uptr size, u32 exp) { \ GET_CALLER_PC_BP_SP; \ - __asan_report_error(pc, bp, sp, addr, is_write, size, exp); \ -} + ReportGenericError(pc, bp, sp, addr, is_write, size, exp, true); \ +} \ +extern "C" NOINLINE INTERFACE_ATTRIBUTE \ +void __asan_report_ ## type ## _n_noabort(uptr addr, uptr size) { \ + GET_CALLER_PC_BP_SP; \ + ReportGenericError(pc, bp, sp, addr, is_write, size, 0, false); \ +} \ ASAN_REPORT_ERROR_N(load, false) ASAN_REPORT_ERROR_N(store, true) -#define ASAN_MEMORY_ACCESS_CALLBACK_BODY(type, is_write, size, exp_arg) \ +#define ASAN_MEMORY_ACCESS_CALLBACK_BODY(type, is_write, size, exp_arg, fatal) \ uptr sp = MEM_TO_SHADOW(addr); \ uptr s = size <= SHADOW_GRANULARITY ? *reinterpret_cast<u8 *>(sp) \ : *reinterpret_cast<u16 *>(sp); \ @@ -163,7 +169,8 @@ ASAN_REPORT_ERROR_N(store, true) *__asan_test_only_reported_buggy_pointer = addr; \ } else { \ GET_CALLER_PC_BP_SP; \ - __asan_report_error(pc, bp, sp, addr, is_write, size, exp_arg); \ + ReportGenericError(pc, bp, sp, addr, is_write, size, exp_arg, \ + fatal); \ } \ } \ } @@ -171,12 +178,16 @@ ASAN_REPORT_ERROR_N(store, true) #define ASAN_MEMORY_ACCESS_CALLBACK(type, is_write, size) \ extern "C" NOINLINE INTERFACE_ATTRIBUTE \ void __asan_##type##size(uptr addr) { \ - ASAN_MEMORY_ACCESS_CALLBACK_BODY(type, is_write, size, 0) \ + ASAN_MEMORY_ACCESS_CALLBACK_BODY(type, is_write, size, 0, true) \ } \ extern "C" NOINLINE INTERFACE_ATTRIBUTE \ void __asan_exp_##type##size(uptr addr, u32 exp) { \ - ASAN_MEMORY_ACCESS_CALLBACK_BODY(type, is_write, size, exp) \ - } + ASAN_MEMORY_ACCESS_CALLBACK_BODY(type, is_write, size, exp, true) \ + } \ + extern "C" NOINLINE INTERFACE_ATTRIBUTE \ + void __asan_##type##size ## _noabort(uptr addr) { \ + ASAN_MEMORY_ACCESS_CALLBACK_BODY(type, is_write, size, 0, false) \ + } \ ASAN_MEMORY_ACCESS_CALLBACK(load, false, 1) ASAN_MEMORY_ACCESS_CALLBACK(load, false, 2) @@ -194,7 +205,7 @@ NOINLINE INTERFACE_ATTRIBUTE void __asan_loadN(uptr addr, uptr size) { if (__asan_region_is_poisoned(addr, size)) { GET_CALLER_PC_BP_SP; - __asan_report_error(pc, bp, sp, addr, false, size, 0); + ReportGenericError(pc, bp, sp, addr, false, size, 0, true); } } @@ -203,7 +214,16 @@ NOINLINE INTERFACE_ATTRIBUTE void __asan_exp_loadN(uptr addr, uptr size, u32 exp) { if (__asan_region_is_poisoned(addr, size)) { GET_CALLER_PC_BP_SP; - __asan_report_error(pc, bp, sp, addr, false, size, exp); + ReportGenericError(pc, bp, sp, addr, false, size, exp, true); + } +} + +extern "C" +NOINLINE INTERFACE_ATTRIBUTE +void __asan_loadN_noabort(uptr addr, uptr size) { + if (__asan_region_is_poisoned(addr, size)) { + GET_CALLER_PC_BP_SP; + ReportGenericError(pc, bp, sp, addr, false, size, 0, false); } } @@ -212,7 +232,7 @@ NOINLINE INTERFACE_ATTRIBUTE void __asan_storeN(uptr addr, uptr size) { if (__asan_region_is_poisoned(addr, size)) { GET_CALLER_PC_BP_SP; - __asan_report_error(pc, bp, sp, addr, true, size, 0); + ReportGenericError(pc, bp, sp, addr, true, size, 0, true); } } @@ -221,7 +241,16 @@ NOINLINE INTERFACE_ATTRIBUTE void __asan_exp_storeN(uptr addr, uptr size, u32 exp) { if (__asan_region_is_poisoned(addr, size)) { GET_CALLER_PC_BP_SP; - __asan_report_error(pc, bp, sp, addr, true, size, exp); + ReportGenericError(pc, bp, sp, addr, true, size, exp, true); + } +} + +extern "C" +NOINLINE INTERFACE_ATTRIBUTE +void __asan_storeN_noabort(uptr addr, uptr size) { + if (__asan_region_is_poisoned(addr, size)) { + GET_CALLER_PC_BP_SP; + ReportGenericError(pc, bp, sp, addr, true, size, 0, false); } } @@ -259,16 +288,15 @@ static NOINLINE void force_interface_symbols() { case 22: __asan_report_exp_store8(0, 0); break; case 23: __asan_report_exp_store16(0, 0); break; case 24: __asan_report_exp_store_n(0, 0, 0); break; - case 25: __asan_register_globals(0, 0); break; - case 26: __asan_unregister_globals(0, 0); break; - case 27: __asan_set_death_callback(0); break; - case 28: __asan_set_error_report_callback(0); break; + case 25: __asan_register_globals(nullptr, 0); break; + case 26: __asan_unregister_globals(nullptr, 0); break; + case 27: __asan_set_death_callback(nullptr); break; + case 28: __asan_set_error_report_callback(nullptr); break; case 29: __asan_handle_no_return(); break; - case 30: __asan_address_is_poisoned(0); break; - case 31: __asan_poison_memory_region(0, 0); break; - case 32: __asan_unpoison_memory_region(0, 0); break; - case 33: __asan_set_error_exit_code(0); break; - case 34: __asan_before_dynamic_init(0); break; + case 30: __asan_address_is_poisoned(nullptr); break; + case 31: __asan_poison_memory_region(nullptr, 0); break; + case 32: __asan_unpoison_memory_region(nullptr, 0); break; + case 34: __asan_before_dynamic_init(nullptr); break; case 35: __asan_after_dynamic_init(); break; case 36: __asan_poison_stack_memory(0, 0); break; case 37: __asan_unpoison_stack_memory(0, 0); break; @@ -298,9 +326,25 @@ static void InitializeHighMemEnd() { } static void ProtectGap(uptr addr, uptr size) { + if (!flags()->protect_shadow_gap) + return; void *res = MmapNoAccess(addr, size, "shadow gap"); if (addr == (uptr)res) return; + // A few pages at the start of the address space can not be protected. + // But we really want to protect as much as possible, to prevent this memory + // being returned as a result of a non-FIXED mmap(). + if (addr == kZeroBaseShadowStart) { + uptr step = GetPageSizeCached(); + while (size > step && addr < kZeroBaseMaxShadowStart) { + addr += step; + size -= step; + void *res = MmapNoAccess(addr, size, "shadow gap"); + if (addr == (uptr)res) + return; + } + } + Report("ERROR: Failed to protect the shadow gap. " "ASan cannot proceed correctly. ABORTING.\n"); DumpProcessMap(); @@ -363,12 +407,12 @@ static void AsanInitInternal() { CHECK(!asan_init_is_running && "ASan init calls itself!"); asan_init_is_running = true; + CacheBinaryName(); + // Initialize flags. This must be done early, because most of the // initialization steps look at flags(). InitializeFlags(); - CacheBinaryName(); - AsanCheckIncompatibleRT(); AsanCheckDynamicRTPrereqs(); @@ -381,7 +425,7 @@ static void AsanInitInternal() { AsanDoesNotSupportStaticLinkage(); // Install tool-specific callbacks in sanitizer_common. - SetDieCallback(AsanDie); + AddDieCallback(AsanDie); SetCheckFailedCallback(AsanCheckFailed); SetPrintfAndReportCallback(AppendToErrorMessageBuffer); @@ -457,7 +501,7 @@ static void AsanInitInternal() { } AsanTSDInit(PlatformTSDDtor); - InstallDeadlySignalHandlers(AsanOnSIGSEGV); + InstallDeadlySignalHandlers(AsanOnDeadlySignal); AllocatorOptions allocator_options; allocator_options.SetFrom(flags(), common_flags()); @@ -531,24 +575,26 @@ public: // NOLINT static AsanInitializer asan_initializer; #endif // ASAN_DYNAMIC -} // namespace __asan +} // namespace __asan // ---------------------- Interface ---------------- {{{1 using namespace __asan; // NOLINT -int NOINLINE __asan_set_error_exit_code(int exit_code) { - int old = flags()->exitcode; - flags()->exitcode = exit_code; - return old; -} - void NOINLINE __asan_handle_no_return() { int local_stack; AsanThread *curr_thread = GetCurrentThread(); - CHECK(curr_thread); uptr PageSize = GetPageSizeCached(); - uptr top = curr_thread->stack_top(); - uptr bottom = ((uptr)&local_stack - PageSize) & ~(PageSize-1); + uptr top, bottom; + if (curr_thread) { + top = curr_thread->stack_top(); + bottom = ((uptr)&local_stack - PageSize) & ~(PageSize - 1); + } else { + // If we haven't seen this thread, try asking the OS for stack bounds. + uptr tls_addr, tls_size, stack_size; + GetThreadStackAndTls(/*main=*/false, &bottom, &stack_size, &tls_addr, + &tls_size); + top = bottom + stack_size; + } static const uptr kMaxExpectedCleanupSize = 64 << 20; // 64M if (top - bottom > kMaxExpectedCleanupSize) { static bool reported_warning = false; @@ -559,12 +605,12 @@ void NOINLINE __asan_handle_no_return() { "stack top: %p; bottom %p; size: %p (%zd)\n" "False positive error reports may follow\n" "For details see " - "http://code.google.com/p/address-sanitizer/issues/detail?id=189\n", + "https://github.com/google/sanitizers/issues/189\n", top, bottom, top - bottom, top - bottom); return; } PoisonShadow(bottom, top - bottom, 0); - if (curr_thread->has_fake_stack()) + if (curr_thread && curr_thread->has_fake_stack()) curr_thread->fake_stack()->HandleNoReturn(); } @@ -578,3 +624,7 @@ void __asan_init() { AsanActivate(); AsanInitInternal(); } + +void __asan_version_mismatch_check() { + // Do nothing. +} diff --git a/lib/asan/asan_stack.h b/lib/asan/asan_stack.h index 122967a152f8..5c5181509801 100644 --- a/lib/asan/asan_stack.h +++ b/lib/asan/asan_stack.h @@ -11,6 +11,7 @@ // // ASan-private header for asan_stack.cc. //===----------------------------------------------------------------------===// + #ifndef ASAN_STACK_H #define ASAN_STACK_H @@ -48,15 +49,15 @@ void GetStackTraceWithPcBpAndContext(BufferedStackTrace *stack, uptr max_depth, uptr stack_bottom = t->stack_bottom(); ScopedUnwinding unwind_scope(t); stack->Unwind(max_depth, pc, bp, context, stack_top, stack_bottom, fast); - } else if (t == 0 && !fast) { + } else if (!t && !fast) { /* If GetCurrentThread() has failed, try to do slow unwind anyways. */ stack->Unwind(max_depth, pc, bp, context, 0, 0, false); } } -#endif // SANITIZER_WINDOWS +#endif // SANITIZER_WINDOWS } -} // namespace __asan +} // namespace __asan // NOTE: A Rule of thumb is to retrieve stack trace in the interceptors // as early as possible (in functions exposed to the user), as we generally @@ -115,4 +116,4 @@ void GetStackTraceWithPcBpAndContext(BufferedStackTrace *stack, uptr max_depth, stack.Print(); \ } -#endif // ASAN_STACK_H +#endif // ASAN_STACK_H diff --git a/lib/asan/asan_thread.cc b/lib/asan/asan_thread.cc index 9af5706d86d0..69813546f551 100644 --- a/lib/asan/asan_thread.cc +++ b/lib/asan/asan_thread.cc @@ -42,7 +42,7 @@ void AsanThreadContext::OnCreated(void *arg) { void AsanThreadContext::OnFinished() { // Drop the link to the AsanThread object. - thread = 0; + thread = nullptr; } // MIPS requires aligned address @@ -125,7 +125,7 @@ void AsanThread::Destroy() { FakeStack *AsanThread::AsyncSignalSafeLazyInitFakeStack() { uptr stack_size = this->stack_size(); if (stack_size == 0) // stack_size is not yet available, don't use FakeStack. - return 0; + return nullptr; uptr old_val = 0; // fake_stack_ has 3 states: // 0 -- not initialized @@ -146,11 +146,11 @@ FakeStack *AsanThread::AsyncSignalSafeLazyInitFakeStack() { SetTLSFakeStack(fake_stack_); return fake_stack_; } - return 0; + return nullptr; } void AsanThread::Init() { - fake_stack_ = 0; // Will be initialized lazily if needed. + fake_stack_ = nullptr; // Will be initialized lazily if needed. CHECK_EQ(this->stack_size(), 0U); SetThreadStackAndTls(); CHECK_GT(this->stack_size(), 0U); @@ -161,13 +161,12 @@ void AsanThread::Init() { VReport(1, "T%d: stack [%p,%p) size 0x%zx; local=%p\n", tid(), (void *)stack_bottom_, (void *)stack_top_, stack_top_ - stack_bottom_, &local); - AsanPlatformThreadInit(); } thread_return_t AsanThread::ThreadStart( uptr os_id, atomic_uintptr_t *signal_thread_is_registered) { Init(); - asanThreadRegistry().StartThread(tid(), os_id, 0); + asanThreadRegistry().StartThread(tid(), os_id, nullptr); if (signal_thread_is_registered) atomic_store(signal_thread_is_registered, 1, memory_order_release); @@ -277,7 +276,7 @@ AsanThread *GetCurrentThread() { return tctx->thread; } } - return 0; + return nullptr; } return context->thread; } @@ -302,7 +301,7 @@ AsanThread *FindThreadByStackAddress(uptr addr) { AsanThreadContext *tctx = static_cast<AsanThreadContext *>( asanThreadRegistry().FindThreadContextLocked(ThreadStackContainsAddress, (void *)addr)); - return tctx ? tctx->thread : 0; + return tctx ? tctx->thread : nullptr; } void EnsureMainThreadIDIsCorrect() { @@ -315,10 +314,10 @@ void EnsureMainThreadIDIsCorrect() { __asan::AsanThread *GetAsanThreadByOsIDLocked(uptr os_id) { __asan::AsanThreadContext *context = static_cast<__asan::AsanThreadContext *>( __asan::asanThreadRegistry().FindThreadContextByOsIDLocked(os_id)); - if (!context) return 0; + if (!context) return nullptr; return context->thread; } -} // namespace __asan +} // namespace __asan // --- Implementation of LSan-specific functions --- {{{1 namespace __lsan { @@ -355,4 +354,4 @@ void UnlockThreadRegistry() { void EnsureMainThreadIDIsCorrect() { __asan::EnsureMainThreadIDIsCorrect(); } -} // namespace __lsan +} // namespace __lsan diff --git a/lib/asan/asan_thread.h b/lib/asan/asan_thread.h index 50acfc42d6a2..ac35711f5794 100644 --- a/lib/asan/asan_thread.h +++ b/lib/asan/asan_thread.h @@ -11,6 +11,7 @@ // // ASan-private header for asan_thread.cc. //===----------------------------------------------------------------------===// + #ifndef ASAN_THREAD_H #define ASAN_THREAD_H @@ -36,7 +37,7 @@ class AsanThreadContext : public ThreadContextBase { explicit AsanThreadContext(int tid) : ThreadContextBase(tid), announced(false), destructor_iterations(GetPthreadDestructorIterations()), stack_id(0), - thread(0) {} + thread(nullptr) {} bool announced; u8 destructor_iterations; u32 stack_id; @@ -84,8 +85,8 @@ class AsanThread { void DeleteFakeStack(int tid) { if (!fake_stack_) return; FakeStack *t = fake_stack_; - fake_stack_ = 0; - SetTLSFakeStack(0); + fake_stack_ = nullptr; + SetTLSFakeStack(nullptr); t->Destroy(tid); } @@ -95,7 +96,7 @@ class AsanThread { FakeStack *fake_stack() { if (!__asan_option_detect_stack_use_after_return) - return 0; + return nullptr; if (!has_fake_stack()) return AsyncSignalSafeLazyInitFakeStack(); return fake_stack_; @@ -179,6 +180,6 @@ AsanThread *FindThreadByStackAddress(uptr addr); // Used to handle fork(). void EnsureMainThreadIDIsCorrect(); -} // namespace __asan +} // namespace __asan -#endif // ASAN_THREAD_H +#endif // ASAN_THREAD_H diff --git a/lib/asan/asan_win.cc b/lib/asan/asan_win.cc index addb3d40a696..92bd893d10ef 100644 --- a/lib/asan/asan_win.cc +++ b/lib/asan/asan_win.cc @@ -14,9 +14,9 @@ #include "sanitizer_common/sanitizer_platform.h" #if SANITIZER_WINDOWS +#define WIN32_LEAN_AND_MEAN #include <windows.h> -#include <dbghelp.h> #include <stdlib.h> #include "asan_interceptors.h" @@ -175,14 +175,6 @@ void PlatformTSDDtor(void *tsd) { // }}} // ---------------------- Various stuff ---------------- {{{ -void DisableReexec() { - // No need to re-exec on Windows. -} - -void MaybeReexec() { - // No need to re-exec on Windows. -} - void *AsanDoesNotSupportStaticLinkage() { #if defined(_DEBUG) #error Please build the runtime with a non-debug CRT: /MD or /MT @@ -194,15 +186,11 @@ void AsanCheckDynamicRTPrereqs() {} void AsanCheckIncompatibleRT() {} -void AsanPlatformThreadInit() { - // Nothing here for now. -} - void ReadContextStack(void *context, uptr *stack, uptr *ssize) { UNIMPLEMENTED(); } -void AsanOnSIGSEGV(int, void *siginfo, void *context) { +void AsanOnDeadlySignal(int, void *siginfo, void *context) { UNIMPLEMENTED(); } @@ -219,7 +207,7 @@ static long WINAPI SEHHandler(EXCEPTION_POINTERS *info) { ? "access-violation" : "in-page-error"; SignalContext sig = SignalContext::Create(exception_record, context); - ReportSIGSEGV(description, sig); + ReportDeadlySignal(description, sig); } // FIXME: Handle EXCEPTION_STACK_OVERFLOW here. @@ -257,7 +245,7 @@ int __asan_set_seh_filter() { // Put a pointer to __asan_set_seh_filter at the end of the global list // of C initializers, after the default EH is set by the CRT. #pragma section(".CRT$XIZ", long, read) // NOLINT -static __declspec(allocate(".CRT$XIZ")) +__declspec(allocate(".CRT$XIZ")) int (*__intercept_seh)() = __asan_set_seh_filter; #endif // }}} diff --git a/lib/asan/asan_win_dll_thunk.cc b/lib/asan/asan_win_dll_thunk.cc index b77f18168ae5..308196d307cc 100644 --- a/lib/asan/asan_win_dll_thunk.cc +++ b/lib/asan/asan_win_dll_thunk.cc @@ -12,8 +12,7 @@ // This file defines a family of thunks that should be statically linked into // the DLLs that have ASan instrumentation in order to delegate the calls to the // shared runtime that lives in the main binary. -// See https://code.google.com/p/address-sanitizer/issues/detail?id=209 for the -// details. +// See https://github.com/google/sanitizers/issues/209 for the details. //===----------------------------------------------------------------------===// // Only compile this code when buidling asan_dll_thunk.lib @@ -30,8 +29,9 @@ void *__stdcall GetProcAddress(void *module, const char *proc_name); void abort(); } -static void *getRealProcAddressOrDie(const char *name) { - void *ret = GetProcAddress(GetModuleHandleA(0), name); +static uptr getRealProcAddressOrDie(const char *name) { + uptr ret = + __interception::InternalGetProcAddress((void *)GetModuleHandleA(0), name); if (!ret) abort(); return ret; @@ -62,13 +62,12 @@ struct FunctionInterceptor<0> { }; #define INTERCEPT_WHEN_POSSIBLE(main_function, dll_function) \ - template<> struct FunctionInterceptor<__LINE__> { \ + template <> struct FunctionInterceptor<__LINE__> { \ static void Execute() { \ - void *wrapper = getRealProcAddressOrDie(main_function); \ - if (!__interception::OverrideFunction((uptr)dll_function, \ - (uptr)wrapper, 0)) \ + uptr wrapper = getRealProcAddressOrDie(main_function); \ + if (!__interception::OverrideFunction((uptr)dll_function, wrapper, 0)) \ abort(); \ - FunctionInterceptor<__LINE__-1>::Execute(); \ + FunctionInterceptor<__LINE__ - 1>::Execute(); \ } \ }; @@ -210,7 +209,7 @@ extern "C" { // __asan_init is expected to be called by only one thread. if (fn) return; - fn = (fntype)getRealProcAddressOrDie(__asan_init_name); + fn = (fntype)getRealProcAddressOrDie("__asan_init"); fn(); __asan_option_detect_stack_use_after_return = (__asan_should_detect_stack_use_after_return() != 0); @@ -219,6 +218,10 @@ extern "C" { } } +extern "C" void __asan_version_mismatch_check() { + // Do nothing. +} + INTERFACE_FUNCTION(__asan_handle_no_return) INTERFACE_FUNCTION(__asan_report_store1) @@ -253,6 +256,9 @@ INTERFACE_FUNCTION(__asan_memcpy); INTERFACE_FUNCTION(__asan_memset); INTERFACE_FUNCTION(__asan_memmove); +INTERFACE_FUNCTION(__asan_alloca_poison); +INTERFACE_FUNCTION(__asan_allocas_unpoison); + INTERFACE_FUNCTION(__asan_register_globals) INTERFACE_FUNCTION(__asan_unregister_globals) @@ -296,6 +302,7 @@ INTERFACE_FUNCTION(__asan_stack_free_10) // FIXME: we might want to have a sanitizer_win_dll_thunk? INTERFACE_FUNCTION(__sanitizer_annotate_contiguous_container) +INTERFACE_FUNCTION(__sanitizer_contiguous_container_find_bad_address) INTERFACE_FUNCTION(__sanitizer_cov) INTERFACE_FUNCTION(__sanitizer_cov_dump) INTERFACE_FUNCTION(__sanitizer_cov_indir_call16) @@ -304,6 +311,7 @@ INTERFACE_FUNCTION(__sanitizer_cov_module_init) INTERFACE_FUNCTION(__sanitizer_cov_trace_basic_block) INTERFACE_FUNCTION(__sanitizer_cov_trace_func_enter) INTERFACE_FUNCTION(__sanitizer_cov_trace_cmp) +INTERFACE_FUNCTION(__sanitizer_cov_trace_switch) INTERFACE_FUNCTION(__sanitizer_cov_with_check) INTERFACE_FUNCTION(__sanitizer_get_allocated_size) INTERFACE_FUNCTION(__sanitizer_get_coverage_guards) @@ -312,6 +320,7 @@ INTERFACE_FUNCTION(__sanitizer_get_estimated_allocated_size) INTERFACE_FUNCTION(__sanitizer_get_free_bytes) INTERFACE_FUNCTION(__sanitizer_get_heap_size) INTERFACE_FUNCTION(__sanitizer_get_ownership) +INTERFACE_FUNCTION(__sanitizer_get_total_unique_caller_callee_pairs) INTERFACE_FUNCTION(__sanitizer_get_total_unique_coverage) INTERFACE_FUNCTION(__sanitizer_get_unmapped_bytes) INTERFACE_FUNCTION(__sanitizer_maybe_open_cov_file) diff --git a/lib/asan/asan_win_dynamic_runtime_thunk.cc b/lib/asan/asan_win_dynamic_runtime_thunk.cc index d59f9f5768a0..73e5207bb334 100644 --- a/lib/asan/asan_win_dynamic_runtime_thunk.cc +++ b/lib/asan/asan_win_dynamic_runtime_thunk.cc @@ -24,6 +24,7 @@ // Using #ifdef rather than relying on Makefiles etc. // simplifies the build procedure. #ifdef ASAN_DYNAMIC_RUNTIME_THUNK +#define WIN32_LEAN_AND_MEAN #include <windows.h> // First, declare CRT sections we'll be using in this file @@ -58,6 +59,7 @@ int __asan_option_detect_stack_use_after_return = // using atexit() that calls a small subset of C terminators // where LLVM global_dtors is placed. Fingers crossed, no other C terminators // are there. +extern "C" int __cdecl atexit(void (__cdecl *f)(void)); extern "C" void __cdecl _initterm(void *a, void *b); namespace { diff --git a/lib/asan/scripts/asan_device_setup b/lib/asan/scripts/asan_device_setup index 104e07b722ca..6cb7b94c2197 100755 --- a/lib/asan/scripts/asan_device_setup +++ b/lib/asan/scripts/asan_device_setup @@ -88,19 +88,25 @@ function adb_pull { fi } -function get_device_arch { # OUTVAR +function get_device_arch { # OUT OUT64 local _outvar=$1 + local _outvar64=$2 local _ABI=$(adb_shell getprop ro.product.cpu.abi) local _ARCH= + local _ARCH64= if [[ $_ABI == x86* ]]; then _ARCH=i686 elif [[ $_ABI == armeabi* ]]; then _ARCH=arm + elif [[ $_ABI == arm64-v8a* ]]; then + _ARCH=arm + _ARCH64=aarch64 else echo "Unrecognized device ABI: $_ABI" exit 1 fi eval $_outvar=\$_ARCH + eval $_outvar64=\$_ARCH64 } while [[ $# > 0 ]]; do @@ -167,22 +173,33 @@ adb_wait_for_device adb_remount adb_wait_for_device -get_device_arch ARCH +get_device_arch ARCH ARCH64 echo "Target architecture: $ARCH" ASAN_RT="libclang_rt.asan-$ARCH-android.so" +if [[ -n $ARCH64 ]]; then + echo "Target architecture: $ARCH64" + ASAN_RT64="libclang_rt.asan-$ARCH64-android.so" +fi if [[ x$revert == xyes ]]; then echo '>> Uninstalling ASan' if ! adb_shell ls -l /system/bin/app_process | grep -o '\->.*app_process' >&/dev/null; then - echo '>> Pre-L device detected.' - adb_shell mv /system/bin/app_process.real /system/bin/app_process - adb_shell rm /system/bin/asanwrapper + echo '>> Pre-L device detected.' + adb_shell mv /system/bin/app_process.real /system/bin/app_process + adb_shell rm /system/bin/asanwrapper + elif ! adb_shell ls -l /system/bin/app_process64.real | grep -o 'No such file or directory' >&/dev/null; then + # 64-bit installation. + adb_shell mv /system/bin/app_process32.real /system/bin/app_process32 + adb_shell mv /system/bin/app_process64.real /system/bin/app_process64 + adb_shell rm /system/bin/asanwrapper + adb_shell rm /system/bin/asanwrapper64 else - adb_shell rm /system/bin/app_process.wrap - adb_shell rm /system/bin/asanwrapper - adb_shell rm /system/bin/app_process - adb_shell ln -s /system/bin/app_process32 /system/bin/app_process + # 32-bit installation. + adb_shell rm /system/bin/app_process.wrap + adb_shell rm /system/bin/asanwrapper + adb_shell rm /system/bin/app_process + adb_shell ln -s /system/bin/app_process32 /system/bin/app_process fi echo '>> Restarting shell' @@ -205,8 +222,13 @@ elif [[ -f "$HERE/$ASAN_RT" ]]; then ASAN_RT_PATH="$HERE" elif [[ $(basename "$HERE") == "bin" ]]; then # We could be in the toolchain's base directory. - # Consider ../lib, ../lib/asan, ../lib/linux and ../lib/clang/$VERSION/lib/linux. - P=$(ls "$HERE"/../lib/"$ASAN_RT" "$HERE"/../lib/asan/"$ASAN_RT" "$HERE"/../lib/linux/"$ASAN_RT" "$HERE"/../lib/clang/*/lib/linux/"$ASAN_RT" 2>/dev/null | sort | tail -1) + # Consider ../lib, ../lib/asan, ../lib/linux, + # ../lib/clang/$VERSION/lib/linux, and ../lib64/clang/$VERSION/lib/linux. + P=$(ls "$HERE"/../lib/"$ASAN_RT" \ + "$HERE"/../lib/asan/"$ASAN_RT" \ + "$HERE"/../lib/linux/"$ASAN_RT" \ + "$HERE"/../lib/clang/*/lib/linux/"$ASAN_RT" \ + "$HERE"/../lib64/clang/*/lib/linux/"$ASAN_RT" 2>/dev/null | sort | tail -1) if [[ -n "$P" ]]; then ASAN_RT_PATH="$(dirname "$P")" fi @@ -217,6 +239,13 @@ if [[ -z "$ASAN_RT_PATH" || ! -f "$ASAN_RT_PATH/$ASAN_RT" ]]; then exit 1 fi +if [[ -n "$ASAN_RT64" ]]; then + if [[ -z "$ASAN_RT_PATH" || ! -f "$ASAN_RT_PATH/$ASAN_RT64" ]]; then + echo ">> ASan runtime library not found" + exit 1 + fi +fi + TMPDIRBASE=$(mktemp -d) TMPDIROLD="$TMPDIRBASE/old" TMPDIR="$TMPDIRBASE/new" @@ -241,12 +270,24 @@ if ! adb_shell ls -l /system/bin/app_process | grep -o '\->.*app_process' >&/dev fi echo '>> Copying files from the device' -adb_pull /system/bin/app_process.wrap "$TMPDIROLD" || true -adb_pull /system/bin/asanwrapper "$TMPDIROLD" || true -adb_pull /system/lib/"$ASAN_RT" "$TMPDIROLD" || true +if [[ -n "$ASAN_RT64" ]]; then + adb_pull /system/lib/"$ASAN_RT" "$TMPDIROLD" || true + adb_pull /system/lib64/"$ASAN_RT64" "$TMPDIROLD" || true + adb_pull /system/bin/app_process32 "$TMPDIROLD" || true + adb_pull /system/bin/app_process32.real "$TMPDIROLD" || true + adb_pull /system/bin/app_process64 "$TMPDIROLD" || true + adb_pull /system/bin/app_process64.real "$TMPDIROLD" || true + adb_pull /system/bin/asanwrapper "$TMPDIROLD" || true + adb_pull /system/bin/asanwrapper64 "$TMPDIROLD" || true +else + adb_pull /system/lib/"$ASAN_RT" "$TMPDIROLD" || true + adb_pull /system/bin/app_process32 "$TMPDIROLD" || true + adb_pull /system/bin/app_process.wrap "$TMPDIROLD" || true + adb_pull /system/bin/asanwrapper "$TMPDIROLD" || true +fi cp -r "$TMPDIROLD" "$TMPDIR" -if [[ -f "$TMPDIR/app_process.wrap" ]]; then +if [[ -f "$TMPDIR/app_process.wrap" || -f "$TMPDIR/app_process64.real" ]]; then echo ">> Previous installation detected" else echo ">> New installation" @@ -255,10 +296,27 @@ fi echo '>> Generating wrappers' cp "$ASAN_RT_PATH/$ASAN_RT" "$TMPDIR/" +if [[ -n "$ASAN_RT64" ]]; then + cp "$ASAN_RT_PATH/$ASAN_RT64" "$TMPDIR/" +fi # FIXME: alloc_dealloc_mismatch=0 prevents a failure in libdvm startup, # which may or may not be a real bug (probably not). -ASAN_OPTIONS=start_deactivated=1,alloc_dealloc_mismatch=0 +ASAN_OPTIONS=start_deactivated=1,alloc_dealloc_mismatch=0,malloc_context_size=0 + +function generate_zygote_wrapper { # from, to, asan_rt + local _from=$1 + local _to=$2 + local _asan_rt=$3 + cat <<EOF >"$TMPDIR/$_from" +#!/system/bin/sh-from-zygote +ASAN_OPTIONS=$ASAN_OPTIONS \\ +ASAN_ACTIVATION_OPTIONS=include_if_exists=/data/local/tmp/asan.options.%b \\ +LD_PRELOAD=\$LD_PRELOAD:$_asan_rt \\ +exec $_to \$@ + +EOF +} # On Android-L not allowing user segv handler breaks some applications. if [[ PRE_L -eq 0 ]]; then @@ -270,13 +328,19 @@ if [[ x$extra_options != x ]] ; then fi # Zygote wrapper. -cat <<EOF >"$TMPDIR/app_process.wrap" -#!/system/bin/sh-from-zygote -ASAN_OPTIONS=$ASAN_OPTIONS \\ -LD_PRELOAD=\$LD_PRELOAD:$ASAN_RT \\ -exec /system/bin/app_process32 \$@ - -EOF +if [[ -f "$TMPDIR/app_process64" ]]; then + # A 64-bit device. + if [[ ! -f "$TMPDIR/app_process64.real" ]]; then + # New installation. + mv "$TMPDIR/app_process32" "$TMPDIR/app_process32.real" + mv "$TMPDIR/app_process64" "$TMPDIR/app_process64.real" + fi + generate_zygote_wrapper "app_process32" "/system/bin/app_process32.real" "$ASAN_RT" + generate_zygote_wrapper "app_process64" "/system/bin/app_process64.real" "$ASAN_RT64" +else + # A 32-bit device. + generate_zygote_wrapper "app_process.wrap" "/system/bin/app_process32" "$ASAN_RT" +fi # General command-line tool wrapper (use for anything that's not started as # zygote). @@ -287,25 +351,33 @@ exec \$@ EOF -if ! ( cd "$TMPDIRBASE" && diff -qr old/ new/ ) ; then - echo '>> Pushing files to the device' - adb_push "$TMPDIR/$ASAN_RT" /system/lib/ - adb_push "$TMPDIR/app_process.wrap" /system/bin - adb_push "$TMPDIR/asanwrapper" /system/bin - - adb_shell rm /system/bin/app_process - adb_shell ln -s /system/bin/app_process.wrap /system/bin/app_process - - adb_shell chown root.shell \ - /system/lib/"$ASAN_RT" \ - /system/bin/app_process.wrap \ - /system/bin/asanwrapper - adb_shell chmod 644 \ - /system/lib/"$ASAN_RT" - adb_shell chmod 755 \ - /system/bin/app_process.wrap \ - /system/bin/asanwrapper +if [[ -n "$ASAN_RT64" ]]; then + cat <<EOF >"$TMPDIR/asanwrapper64" +#!/system/bin/sh +LD_PRELOAD=$ASAN_RT64 \\ +exec \$@ + +EOF +fi + +function install { # from, to, chmod, chcon + local _from=$1 + local _to=$2 + local _mode=$3 + local _context=$4 + local _basename="$(basename "$_from")" + echo "Installing $_to/$_basename $_mode $_context" + adb_push "$_from" "$_to/$_basename" + adb_shell chown root.shell "$_to/$_basename" + if [[ -n "$_mode" ]]; then + adb_shell chmod "$_mode" "$_to/$_basename" + fi + if [[ -n "$_context" ]]; then + adb_shell chcon "$_context" "$_to/$_basename" + fi +} +if ! ( cd "$TMPDIRBASE" && diff -qr old/ new/ ) ; then # Make SELinux happy by keeping app_process wrapper and the shell # it runs on in zygote domain. ENFORCING=0 @@ -316,17 +388,35 @@ if ! ( cd "$TMPDIRBASE" && diff -qr old/ new/ ) ; then adb_shell setenforce 0 fi - adb_shell cp /system/bin/sh /system/bin/sh-from-zygote - if [[ PRE_L -eq 1 ]]; then CTX=u:object_r:system_file:s0 else CTX=u:object_r:zygote_exec:s0 fi - adb_shell chcon $CTX \ - /system/bin/sh-from-zygote \ - /system/bin/app_process.wrap \ - /system/bin/app_process32 + + echo '>> Pushing files to the device' + + if [[ -n "$ASAN_RT64" ]]; then + install "$TMPDIR/$ASAN_RT" /system/lib 644 + install "$TMPDIR/$ASAN_RT64" /system/lib64 644 + install "$TMPDIR/app_process32" /system/bin 755 $CTX + install "$TMPDIR/app_process32.real" /system/bin 755 $CTX + install "$TMPDIR/app_process64" /system/bin 755 $CTX + install "$TMPDIR/app_process64.real" /system/bin 755 $CTX + install "$TMPDIR/asanwrapper" /system/bin 755 + install "$TMPDIR/asanwrapper64" /system/bin 755 + else + install "$TMPDIR/$ASAN_RT" /system/lib 644 + install "$TMPDIR/app_process32" /system/bin 755 $CTX + install "$TMPDIR/app_process.wrap" /system/bin 755 $CTX + install "$TMPDIR/asanwrapper" /system/bin 755 $CTX + + adb_shell rm /system/bin/app_process + adb_shell ln -s /system/bin/app_process.wrap /system/bin/app_process + fi + + adb_shell cp /system/bin/sh /system/bin/sh-from-zygote + adb_shell chcon $CTX /system/bin/sh-from-zygote if [ $ENFORCING == 1 ]; then adb_shell setenforce 1 diff --git a/lib/asan/scripts/asan_symbolize.py b/lib/asan/scripts/asan_symbolize.py index b9d3ad3ad2fe..8e6fb61f7bf7 100755 --- a/lib/asan/scripts/asan_symbolize.py +++ b/lib/asan/scripts/asan_symbolize.py @@ -77,7 +77,7 @@ class LLVMSymbolizer(Symbolizer): cmd = [self.symbolizer_path, '--use-symbol-table=true', '--demangle=%s' % demangle, - '--functions=short', + '--functions=linkage', '--inlining=true', '--default-arch=%s' % self.default_arch] if self.system == 'Darwin': @@ -135,12 +135,13 @@ class Addr2LineSymbolizer(Symbolizer): super(Addr2LineSymbolizer, self).__init__() self.binary = binary self.pipe = self.open_addr2line() + self.output_terminator = -1 def open_addr2line(self): addr2line_tool = 'addr2line' if binutils_prefix: addr2line_tool = binutils_prefix + addr2line_tool - cmd = [addr2line_tool, '-f'] + cmd = [addr2line_tool, '-fi'] if demangle: cmd += ['--demangle'] cmd += ['-e', self.binary] @@ -153,16 +154,23 @@ class Addr2LineSymbolizer(Symbolizer): """Overrides Symbolizer.symbolize.""" if self.binary != binary: return None + lines = [] try: print >> self.pipe.stdin, offset - function_name = self.pipe.stdout.readline().rstrip() - file_name = self.pipe.stdout.readline().rstrip() + print >> self.pipe.stdin, self.output_terminator + is_first_frame = True + while True: + function_name = self.pipe.stdout.readline().rstrip() + file_name = self.pipe.stdout.readline().rstrip() + if is_first_frame: + is_first_frame = False + elif function_name in ['', '??']: + assert file_name == function_name + break + lines.append((function_name, file_name)); except Exception: - function_name = '' - file_name = '' - file_name = fix_filename(file_name) - return ['%s in %s %s' % (addr, function_name, file_name)] - + lines.append(('??', '??:0')) + return ['%s in %s %s' % (addr, function, fix_filename(file)) for (function, file) in lines] class UnbufferedLineConverter(object): """ @@ -263,7 +271,7 @@ def BreakpadSymbolizerFactory(binary): def SystemSymbolizerFactory(system, addr, binary): if system == 'Darwin': return DarwinSymbolizer(addr, binary) - elif system == 'Linux': + elif system == 'Linux' or system == 'FreeBSD': return Addr2LineSymbolizer(binary) diff --git a/lib/asan/tests/CMakeLists.txt b/lib/asan/tests/CMakeLists.txt index afdd2adf0887..7a8d8f7f106b 100644 --- a/lib/asan/tests/CMakeLists.txt +++ b/lib/asan/tests/CMakeLists.txt @@ -106,7 +106,7 @@ append_list_if(COMPILER_RT_HAS_LIBPTHREAD -pthread # TODO(eugenis): move all -l flags above to _LIBS? set(ASAN_UNITTEST_NOINST_LIBS) -append_list_if(ANDROID log ASAN_UNITTEST_NOINST_LIBS) +append_list_if(COMPILER_RT_HAS_LIBLOG log ASAN_UNITTEST_NOINST_LIBS) # NDK r10 requires -latomic almost always. append_list_if(ANDROID atomic ASAN_UNITTEST_NOINST_LIBS) @@ -217,9 +217,10 @@ macro(add_asan_tests_for_arch_and_kind arch kind) set(ASAN_TEST_RUNTIME RTAsanTest.${arch}${kind}) if(APPLE) set(ASAN_TEST_RUNTIME_OBJECTS - $<TARGET_OBJECTS:RTAsan.osx> + $<TARGET_OBJECTS:RTAsan_dynamic.osx> $<TARGET_OBJECTS:RTInterception.osx> $<TARGET_OBJECTS:RTSanitizerCommon.osx> + $<TARGET_OBJECTS:RTSanitizerCommonLibc.osx> $<TARGET_OBJECTS:RTLSanCommon.osx> $<TARGET_OBJECTS:RTUbsan.osx>) else() @@ -261,7 +262,11 @@ macro(add_asan_tests_for_arch_and_kind arch kind) endmacro() if(COMPILER_RT_CAN_EXECUTE_TESTS AND NOT ANDROID) - foreach(arch ${ASAN_SUPPORTED_ARCH}) + set(ASAN_TEST_ARCH ${ASAN_SUPPORTED_ARCH}) + if(APPLE) + darwin_filter_host_archs(ASAN_SUPPORTED_ARCH ASAN_TEST_ARCH) + endif() + foreach(arch ${ASAN_TEST_ARCH}) add_asan_tests_for_arch_and_kind(${arch} "-inline") add_asan_tests_for_arch_and_kind(${arch} "-with-calls" -mllvm -asan-instrumentation-with-call-threshold=0) diff --git a/lib/asan/tests/asan_asm_test.cc b/lib/asan/tests/asan_asm_test.cc index 200de2c137a5..09af5c386079 100644 --- a/lib/asan/tests/asan_asm_test.cc +++ b/lib/asan/tests/asan_asm_test.cc @@ -14,7 +14,10 @@ #if defined(__linux__) -#if defined(__x86_64__) || (defined(__i386__) && defined(__SSE2__)) +// Assembly instrumentation is broken on x86 Android (x86 + PIC + shared runtime +// library). See https://github.com/google/sanitizers/issues/353 +#if defined(__x86_64__) || \ + (defined(__i386__) && defined(__SSE2__) && !defined(__ANDROID__)) #include <emmintrin.h> @@ -70,7 +73,7 @@ DECLARE_ASM_REP_MOVS(U8, "movsq"); #endif // defined(__x86_64__) -#if defined(__i386__) && defined(__SSE2__) +#if defined(__i386__) && defined(__SSE2__) && !defined(__ANDROID__) namespace { @@ -108,7 +111,8 @@ template<> Type asm_read<Type>(Type *ptr) { \ #endif // defined(__i386__) && defined(__SSE2__) -#if defined(__x86_64__) || (defined(__i386__) && defined(__SSE2__)) +#if defined(__x86_64__) || \ + (defined(__i386__) && defined(__SSE2__) && !defined(__ANDROID__)) namespace { diff --git a/lib/asan/tests/asan_interface_test.cc b/lib/asan/tests/asan_interface_test.cc index a34c8528eae0..f5bfb8046b0a 100644 --- a/lib/asan/tests/asan_interface_test.cc +++ b/lib/asan/tests/asan_interface_test.cc @@ -140,16 +140,6 @@ static void DoDoubleFree() { delete Ident(x); } -TEST(AddressSanitizerInterface, ExitCode) { - int original_exit_code = __asan_set_error_exit_code(7); - EXPECT_EXIT(DoDoubleFree(), ::testing::ExitedWithCode(7), ""); - EXPECT_EQ(7, __asan_set_error_exit_code(8)); - EXPECT_EXIT(DoDoubleFree(), ::testing::ExitedWithCode(8), ""); - EXPECT_EQ(8, __asan_set_error_exit_code(original_exit_code)); - EXPECT_EXIT(DoDoubleFree(), - ::testing::ExitedWithCode(original_exit_code), ""); -} - static void MyDeathCallback() { fprintf(stderr, "MyDeathCallback\n"); fflush(0); // On Windows, stderr doesn't flush on crash. diff --git a/lib/asan/tests/asan_mac_test.cc b/lib/asan/tests/asan_mac_test.cc index cabdfd711ea2..dfa6d7596d74 100644 --- a/lib/asan/tests/asan_mac_test.cc +++ b/lib/asan/tests/asan_mac_test.cc @@ -216,12 +216,12 @@ TEST(AddressSanitizerMac, NSObjectOOB) { // Make sure that correct pointer is passed to free() when deallocating a // NSURL object. -// See http://code.google.com/p/address-sanitizer/issues/detail?id=70. +// See https://github.com/google/sanitizers/issues/70. TEST(AddressSanitizerMac, NSURLDeallocation) { TestNSURLDeallocation(); } -// See http://code.google.com/p/address-sanitizer/issues/detail?id=109. +// See https://github.com/google/sanitizers/issues/109. TEST(AddressSanitizerMac, Mstats) { malloc_statistics_t stats1, stats2; malloc_zone_statistics(/*all zones*/NULL, &stats1); diff --git a/lib/asan/tests/asan_noinst_test.cc b/lib/asan/tests/asan_noinst_test.cc index 6a428fbbc2b9..5f5354f92caf 100644 --- a/lib/asan/tests/asan_noinst_test.cc +++ b/lib/asan/tests/asan_noinst_test.cc @@ -34,7 +34,7 @@ // Make sure __asan_init is called before any test case is run. struct AsanInitCaller { AsanInitCaller() { - __asan::DisableReexec(); + DisableReexec(); __asan_init(); } }; diff --git a/lib/asan/tests/asan_test.cc b/lib/asan/tests/asan_test.cc index 07d59e09a72f..71fb27a0ca11 100644 --- a/lib/asan/tests/asan_test.cc +++ b/lib/asan/tests/asan_test.cc @@ -250,12 +250,12 @@ TEST(AddressSanitizer, BitFieldNegativeTest) { #if ASAN_NEEDS_SEGV namespace { -const char kUnknownCrash[] = "AddressSanitizer: SEGV on unknown address"; +const char kSEGVCrash[] = "AddressSanitizer: SEGV on unknown address"; const char kOverriddenHandler[] = "ASan signal handler has been overridden\n"; TEST(AddressSanitizer, WildAddressTest) { char *c = (char*)0x123; - EXPECT_DEATH(*c = 0, kUnknownCrash); + EXPECT_DEATH(*c = 0, kSEGVCrash); } void my_sigaction_sighandler(int, siginfo_t*, void*) { @@ -279,10 +279,10 @@ TEST(AddressSanitizer, SignalTest) { EXPECT_EQ(0, sigaction(SIGBUS, &sigact, 0)); #endif char *c = (char*)0x123; - EXPECT_DEATH(*c = 0, kUnknownCrash); + EXPECT_DEATH(*c = 0, kSEGVCrash); // ... and signal(). EXPECT_EQ(0, signal(SIGSEGV, my_signal_sighandler)); - EXPECT_DEATH(*c = 0, kUnknownCrash); + EXPECT_DEATH(*c = 0, kSEGVCrash); } } // namespace #endif @@ -335,6 +335,8 @@ void *ManyThreadsWorker(void *a) { return 0; } +#if !defined(__aarch64__) +// FIXME: Infinite loop in AArch64 (PR24389). TEST(AddressSanitizer, ManyThreadsTest) { const size_t kNumThreads = (SANITIZER_WORDSIZE == 32 || ASAN_AVOID_EXPENSIVE_TESTS) ? 30 : 1000; @@ -346,6 +348,7 @@ TEST(AddressSanitizer, ManyThreadsTest) { PTHREAD_JOIN(t[i], 0); } } +#endif TEST(AddressSanitizer, ReallocTest) { const int kMinElem = 5; @@ -607,7 +610,7 @@ NOINLINE void BuiltinLongJmpFunc1(jmp_buf buf) { } // Does not work on Power and ARM: -// https://code.google.com/p/address-sanitizer/issues/detail?id=185 +// https://github.com/google/sanitizers/issues/185 TEST(AddressSanitizer, BuiltinLongJmpTest) { static jmp_buf buf; if (!__builtin_setjmp((void**)buf)) { @@ -1153,9 +1156,9 @@ TEST(AddressSanitizer, AttributeNoSanitizeAddressTest) { // The new/delete/etc mismatch checks don't work on Android, // as calls to new/delete go through malloc/free. // OS X support is tracked here: -// https://code.google.com/p/address-sanitizer/issues/detail?id=131 +// https://github.com/google/sanitizers/issues/131 // Windows support is tracked here: -// https://code.google.com/p/address-sanitizer/issues/detail?id=309 +// https://github.com/google/sanitizers/issues/309 #if !defined(__ANDROID__) && \ !defined(__APPLE__) && \ !defined(_WIN32) @@ -1252,7 +1255,7 @@ TEST(AddressSanitizer, DISABLED_DemoTooMuchMemoryTest) { } } -// http://code.google.com/p/address-sanitizer/issues/detail?id=66 +// https://github.com/google/sanitizers/issues/66 TEST(AddressSanitizer, BufferOverflowAfterManyFrees) { for (int i = 0; i < 1000000; i++) { delete [] (Ident(new char [8644])); diff --git a/lib/asan/tests/asan_test_main.cc b/lib/asan/tests/asan_test_main.cc index 1746c5f4837b..cdaf801d914b 100644 --- a/lib/asan/tests/asan_test_main.cc +++ b/lib/asan/tests/asan_test_main.cc @@ -11,6 +11,20 @@ // //===----------------------------------------------------------------------===// #include "asan_test_utils.h" +#include "sanitizer_common/sanitizer_platform.h" + +// Default ASAN_OPTIONS for the unit tests. Let's turn symbolication off to +// speed up testing (unit tests don't use it anyway). +extern "C" const char* __asan_default_options() { +#if SANITIZER_MAC + // On Darwin, we default to `abort_on_error=1`, which would make tests run + // much slower. Let's override this and run lit tests with 'abort_on_error=0'. + // Also, make sure we do not overwhelm the syslog while testing. + return "symbolize=false:abort_on_error=0:log_to_syslog=0"; +#else + return "symbolize=false"; +#endif +} int main(int argc, char **argv) { testing::GTEST_FLAG(death_test_style) = "threadsafe"; diff --git a/lib/builtins/CMakeLists.txt b/lib/builtins/CMakeLists.txt index 98d518a83e2c..5ffad1d47b17 100644 --- a/lib/builtins/CMakeLists.txt +++ b/lib/builtins/CMakeLists.txt @@ -2,6 +2,9 @@ # generic implementations of the core runtime library along with optimized # architecture-specific code in various subdirectories. +# TODO: Need to add a mechanism for logging errors when builtin source files are +# added to a sub-directory and not this CMakeLists file. + set(GENERIC_SOURCES absvdi2.c absvsi2.c @@ -38,6 +41,7 @@ set(GENERIC_SOURCES divsc3.c divsf3.c divsi3.c + divtc3.c divti3.c divtf3.c divxc3.c @@ -139,59 +143,102 @@ set(GENERIC_SOURCES umodsi3.c umodti3.c) +if(APPLE) + set(GENERIC_SOURCES + ${GENERIC_SOURCES} + atomic_flag_clear.c + atomic_flag_clear_explicit.c + atomic_flag_test_and_set.c + atomic_flag_test_and_set_explicit.c + atomic_signal_fence.c + atomic_thread_fence.c) +endif() + +if(NOT WIN32 OR MINGW) + set(GENERIC_SOURCES + ${GENERIC_SOURCES} + emutls.c) +endif() + if (HAVE_UNWIND_H) set(GENERIC_SOURCES ${GENERIC_SOURCES} gcc_personality_v0.c) endif () -set(x86_64_SOURCES - x86_64/floatdidf.c - x86_64/floatdisf.c - x86_64/floatdixf.c - x86_64/floatundidf.S - x86_64/floatundisf.S - x86_64/floatundixf.S - ${GENERIC_SOURCES}) - -if(WIN32) +if (NOT MSVC) set(x86_64_SOURCES - ${x86_64_SOURCES} - x86_64/chkstk.S) -endif() + x86_64/chkstk.S + x86_64/chkstk2.S + x86_64/floatdidf.c + x86_64/floatdisf.c + x86_64/floatdixf.c + x86_64/floatundidf.S + x86_64/floatundisf.S + x86_64/floatundixf.S + ${GENERIC_SOURCES}) + set(x86_64h_SOURCES ${x86_64_SOURCES}) -set(i386_SOURCES - i386/ashldi3.S - i386/ashrdi3.S - i386/divdi3.S - i386/floatdidf.S - i386/floatdisf.S - i386/floatdixf.S - i386/floatundidf.S - i386/floatundisf.S - i386/floatundixf.S - i386/lshrdi3.S - i386/moddi3.S - i386/muldi3.S - i386/udivdi3.S - i386/umoddi3.S - ${GENERIC_SOURCES}) + if (WIN32) + set(x86_64_SOURCES + ${x86_64_SOURCES} + x86_64/chkstk.S + x86_64/chkstk2.S) + endif() -if(WIN32) set(i386_SOURCES - ${i386_SOURCES} - i386/chkstk.S) -endif() + i386/ashldi3.S + i386/ashrdi3.S + i386/chkstk.S + i386/chkstk2.S + i386/divdi3.S + i386/floatdidf.S + i386/floatdisf.S + i386/floatdixf.S + i386/floatundidf.S + i386/floatundisf.S + i386/floatundixf.S + i386/lshrdi3.S + i386/moddi3.S + i386/muldi3.S + i386/udivdi3.S + i386/umoddi3.S + ${GENERIC_SOURCES}) -set(i686_SOURCES - ${i386_SOURCES}) + if (WIN32) + set(i386_SOURCES + ${i386_SOURCES} + i386/chkstk.S + i386/chkstk2.S) + endif() + + set(i686_SOURCES + ${i386_SOURCES}) +else () # MSVC + # Use C versions of functions when building on MSVC + # MSVC's assembler takes Intel syntax, not AT&T syntax + set(x86_64_SOURCES + x86_64/floatdidf.c + x86_64/floatdisf.c + x86_64/floatdixf.c + ${GENERIC_SOURCES}) + set(x86_64h_SOURCES ${x86_64_SOURCES}) + set(i386_SOURCES ${GENERIC_SOURCES}) + set(i686_SOURCES ${i386_SOURCES}) +endif () # if (NOT MSVC) set(arm_SOURCES arm/adddf3vfp.S arm/addsf3vfp.S + arm/aeabi_cdcmp.S + arm/aeabi_cdcmpeq_check_nan.c + arm/aeabi_cfcmp.S + arm/aeabi_cfcmpeq_check_nan.c arm/aeabi_dcmp.S arm/aeabi_div0.c + arm/aeabi_drsub.c arm/aeabi_fcmp.S + arm/aeabi_frsub.c arm/aeabi_idivmod.S arm/aeabi_ldivmod.S arm/aeabi_memcmp.S @@ -202,6 +249,8 @@ set(arm_SOURCES arm/aeabi_uldivmod.S arm/bswapdi2.S arm/bswapsi2.S + arm/clzdi2.S + arm/clzsi2.S arm/comparesf2.S arm/divdf3vfp.S arm/divmodsi4.S @@ -270,10 +319,50 @@ set(arm_SOURCES arm/unordsf2vfp.S ${GENERIC_SOURCES}) +set(aarch64_SOURCES + comparetf2.c + extenddftf2.c + extendsftf2.c + fixtfdi.c + fixtfsi.c + fixtfti.c + fixunstfdi.c + fixunstfsi.c + fixunstfti.c + floatditf.c + floatsitf.c + floatunditf.c + floatunsitf.c + multc3.c + trunctfdf2.c + trunctfsf2.c + ${GENERIC_SOURCES}) + +set(armhf_SOURCES ${arm_SOURCES}) +set(armv7_SOURCES ${arm_SOURCES}) +set(armv7s_SOURCES ${arm_SOURCES}) +set(arm64_SOURCES ${aarch64_SOURCES}) + +# macho_embedded archs +set(armv6m_SOURCES ${GENERIC_SOURCES}) +set(armv7m_SOURCES ${arm_SOURCES}) +set(armv7em_SOURCES ${arm_SOURCES}) + +set(mips_SOURCES ${GENERIC_SOURCES}) +set(mipsel_SOURCES ${mips_SOURCES}) +set(mips64_SOURCES ${mips_SOURCES}) +set(mips64el_SOURCES ${mips_SOURCES}) + add_custom_target(builtins) -if (NOT WIN32 OR MINGW) - foreach (arch x86_64 i386 i686 arm) +if (APPLE) + add_subdirectory(Darwin-excludes) + add_subdirectory(macho_embedded) + darwin_add_builtin_libraries(${BUILTIN_SUPPORTED_OS}) +elseif (NOT WIN32 OR MINGW) + append_string_if(COMPILER_RT_HAS_STD_C99_FLAG -std=c99 maybe_stdc99) + + foreach (arch ${BUILTIN_SUPPORTED_ARCH}) if (CAN_TARGET_${arch}) # Filter out generic versions of routines that are re-implemented in # architecture specific manner. This prevents multiple definitions of the @@ -286,11 +375,12 @@ if (NOT WIN32 OR MINGW) endif () endforeach () - set_source_files_properties(${${arch}_SOURCES} PROPERTIES LANGUAGE C) - add_compiler_rt_runtime(clang_rt.builtins-${arch} ${arch} STATIC + add_compiler_rt_runtime(clang_rt.builtins + STATIC + ARCHS ${arch} SOURCES ${${arch}_SOURCES} - CFLAGS "-std=c99") - add_dependencies(builtins clang_rt.builtins-${arch}) + CFLAGS ${maybe_stdc99} + PARENT_TARGET builtins) endif () endforeach () endif () diff --git a/lib/builtins/Darwin-excludes/10.4-x86_64.txt b/lib/builtins/Darwin-excludes/10.4-x86_64.txt new file mode 100644 index 000000000000..f2ee7fef0c63 --- /dev/null +++ b/lib/builtins/Darwin-excludes/10.4-x86_64.txt @@ -0,0 +1,35 @@ +absvti2 +addvti3 +ashlti3 +ashrti3 +clzti2 +cmpti2 +ctzti2 +divti3 +ffsti2 +fixdfti +fixsfti +fixunsdfti +fixunssfti +fixunsxfti +fixxfti +floattidf +floattisf +floattixf +floatuntidf +floatuntisf +floatuntixf +lshrti3 +modti3 +muloti4 +multi3 +mulvti3 +negti2 +negvti2 +parityti2 +popcountti2 +subvti3 +ucmpti2 +udivmodti4 +udivti3 +umodti3 diff --git a/lib/builtins/Darwin-excludes/10.4.txt b/lib/builtins/Darwin-excludes/10.4.txt new file mode 100644 index 000000000000..70d3644f271c --- /dev/null +++ b/lib/builtins/Darwin-excludes/10.4.txt @@ -0,0 +1,96 @@ +apple_versioning +absvdi2 +absvsi2 +adddf3 +addsf3 +addvdi3 +addvsi3 +ashldi3 +ashrdi3 +clear_cache +clzdi2 +clzsi2 +cmpdi2 +ctzdi2 +ctzsi2 +divdc3 +divdf3 +divdi3 +divmoddi4 +divmodsi4 +divsc3 +divsf3 +divsi3 +divxc3 +enable_execute_stack +comparedf2 +comparesf2 +extendhfsf2 +extendsfdf2 +ffsdi2 +fixdfdi +fixdfsi +fixsfdi +fixsfsi +fixunsdfdi +fixunsdfsi +fixunssfdi +fixunssfsi +fixunsxfdi +fixunsxfsi +fixxfdi +floatdidf +floatdisf +floatdixf +floatsidf +floatsisf +floatunsidf +floatunsisf +gcc_personality_v0 +gnu_f2h_ieee +gnu_h2f_ieee +lshrdi3 +moddi3 +modsi3 +muldc3 +muldf3 +muldi3 +mulodi4 +mulosi4 +mulsc3 +mulsf3 +mulvdi3 +mulvsi3 +mulxc3 +negdf2 +negdi2 +negsf2 +negvdi2 +negvsi2 +paritydi2 +paritysi2 +popcountdi2 +popcountsi2 +powidf2 +powisf2 +powixf2 +subdf3 +subsf3 +subvdi3 +subvsi3 +truncdfhf2 +truncdfsf2 +truncsfhf2 +ucmpdi2 +udivdi3 +udivmoddi4 +udivmodsi4 +udivsi3 +umoddi3 +umodsi3 +atomic_flag_clear +atomic_flag_clear_explicit +atomic_flag_test_and_set +atomic_flag_test_and_set_explicit +atomic_signal_fence +atomic_thread_fence
\ No newline at end of file diff --git a/lib/builtins/Darwin-excludes/CMakeLists.txt b/lib/builtins/Darwin-excludes/CMakeLists.txt new file mode 100644 index 000000000000..266e42215243 --- /dev/null +++ b/lib/builtins/Darwin-excludes/CMakeLists.txt @@ -0,0 +1,4 @@ +file(GLOB filter_files ${CMAKE_CURRENT_SOURCE_DIR}/*.txt) +foreach(filter_file ${filter_files}) + set_property(DIRECTORY APPEND PROPERTY CMAKE_CONFIGURE_DEPENDS ${filter_file}) +endforeach() diff --git a/lib/builtins/Darwin-excludes/README.TXT b/lib/builtins/Darwin-excludes/README.TXT new file mode 100644 index 000000000000..173eccca6dec --- /dev/null +++ b/lib/builtins/Darwin-excludes/README.TXT @@ -0,0 +1,11 @@ +This folder contains list of symbols that should be excluded from the builtin +libraries for Darwin. There are two reasons symbols are excluded: + +(1) They aren't supported on Darwin +(2) They are contained within the OS on the minimum supported target + +The builtin libraries must contain all symbols not provided by the lowest +supported target OS. Meaning if minimum deployment target is iOS 6, all builtins +not included in the ios6-<arch>.txt files need to be included. The one catch is +that this is per-architecture. Since iOS 6 doesn't support arm64, when supporting +iOS 6, the minimum deployment target for arm64 binaries is iOS 7. diff --git a/lib/builtins/Darwin-excludes/ios-armv7.txt b/lib/builtins/Darwin-excludes/ios-armv7.txt new file mode 100644 index 000000000000..6aa542f7fe4a --- /dev/null +++ b/lib/builtins/Darwin-excludes/ios-armv7.txt @@ -0,0 +1,57 @@ +absvti2 +addtf3 +addvti3 +aeabi_cdcmp +aeabi_cdcmpeq_check_nan +aeabi_cfcmp +aeabi_cfcmpeq_check_nan +aeabi_dcmp +aeabi_div0 +aeabi_drsub +aeabi_fcmp +aeabi_frsub +aeabi_idivmod +aeabi_ldivmod +aeabi_memcmp +aeabi_memcpy +aeabi_memmove +aeabi_memset +aeabi_uidivmod +aeabi_uldivmod +ashlti3 +ashrti3 +clzti2 +cmpti2 +ctzti2 +divtf3 +divti3 +ffsti2 +fixdfti +fixsfti +fixunsdfti +fixunssfti +fixunsxfti +fixxfti +floattidf +floattisf +floattixf +floatuntidf +floatuntisf +floatuntixf +lshrti3 +modti3 +multf3 +multi3 +mulvti3 +negti2 +negvti2 +parityti2 +popcountti2 +powitf2 +subtf3 +subvti3 +trampoline_setup +ucmpti2 +udivmodti4 +udivti3 +umodti3 diff --git a/lib/builtins/Darwin-excludes/ios-armv7s.txt b/lib/builtins/Darwin-excludes/ios-armv7s.txt new file mode 100644 index 000000000000..28167aa4c5db --- /dev/null +++ b/lib/builtins/Darwin-excludes/ios-armv7s.txt @@ -0,0 +1,57 @@ +absvti2 +addtf3 +addvti3 +aeabi_cdcmp +aeabi_cdcmpeq_check_nan +aeabi_cfcmp +aeabi_cfcmpeq_check_nan +aeabi_dcmp +aeabi_div0 +aeabi_drsub +aeabi_fcmp +aeabi_frsub +aeabi_idivmod +aeabi_ldivmod +aeabi_memcmp +aeabi_memcpy +aeabi_memmove +aeabi_memset +aeabi_uidivmod +aeabi_uldivmod +ashlti3 +ashrti3 +clzti2 +cmpti2 +ctzti2 +divtf3 +divti3 +ffsti2 +fixdfti +fixsfti +fixunsdfti +fixunssfti +fixunsxfti +fixxfti +floattidf +floattisf +floattixf +floatuntidf +floatuntisf +floatuntixf +lshrti3 +modti3 +multf +multi3 +mulvti3 +negti2 +negvti2 +parityti2 +popcountti2 +powitf2 +subtf3 +subvti3 +trampoline_setup +ucmpti2 +udivmodti4 +udivti3 +umodti3 diff --git a/lib/builtins/Darwin-excludes/ios.txt b/lib/builtins/Darwin-excludes/ios.txt new file mode 100644 index 000000000000..5db24000a174 --- /dev/null +++ b/lib/builtins/Darwin-excludes/ios.txt @@ -0,0 +1 @@ +apple_versioning diff --git a/lib/builtins/Darwin-excludes/ios6-armv7.txt b/lib/builtins/Darwin-excludes/ios6-armv7.txt new file mode 100644 index 000000000000..b01fa711a357 --- /dev/null +++ b/lib/builtins/Darwin-excludes/ios6-armv7.txt @@ -0,0 +1,120 @@ +absvdi2 +absvsi2 +adddf3 +adddf3vfp +addsf3 +addsf3vfp +addvdi3 +addvsi3 +ashldi3 +ashrdi3 +bswapdi2 +bswapsi2 +clzdi2 +clzsi2 +cmpdi2 +ctzdi2 +ctzsi2 +divdc3 +divdf3 +divdf3vfp +divdi3 +divmodsi4 +divsc3 +divsf3 +divsf3vfp +divsi3 +eqdf2 +eqdf2vfp +eqsf2 +eqsf2vfp +extendsfdf2 +extendsfdf2vfp +ffsdi2 +fixdfdi +fixdfsi +fixdfsivfp +fixsfdi +fixsfsi +fixsfsivfp +fixunsdfdi +fixunsdfsi +fixunsdfsivfp +fixunssfdi +fixunssfsi +fixunssfsivfp +floatdidf +floatdisf +floatsidf +floatsidfvfp +floatsisf +floatsisfvfp +floatundidf +floatundisf +floatunsidf +floatunsisf +floatunssidfvfp +floatunssisfvfp +gcc_personality_sj0 +gedf2 +gedf2vfp +gesf2 +gesf2vfp +gtdf2 +gtdf2vfp +gtsf2 +gtsf2vfp +ledf2 +ledf2vfp +lesf2 +lesf2vfp +lshrdi3 +ltdf2 +ltdf2vfp +ltsf2 +ltsf2vfp +moddi3 +modsi3 +muldc3 +muldf3 +muldf3vfp +muldi3 +mulodi4 +mulosi4 +mulsc3 +mulsf3 +mulsf3vfp +mulvdi3 +mulvsi3 +nedf2 +nedf2vfp +negdi2 +negvdi2 +negvsi2 +nesf2 +nesf2vfp +paritydi2 +paritysi2 +popcountdi2 +popcountsi2 +powidf2 +powisf2 +subdf3 +subdf3vfp +subsf3 +subsf3vfp +subvdi3 +subvsi3 +truncdfsf2 +truncdfsf2vfp +ucmpdi2 +udivdi3 +udivmoddi4 +udivmodsi4 +udivsi3 +umoddi3 +umodsi3 +unorddf2 +unorddf2vfp +unordsf2 +unordsf2vfp diff --git a/lib/builtins/Darwin-excludes/ios6-armv7s.txt b/lib/builtins/Darwin-excludes/ios6-armv7s.txt new file mode 100644 index 000000000000..b01fa711a357 --- /dev/null +++ b/lib/builtins/Darwin-excludes/ios6-armv7s.txt @@ -0,0 +1,120 @@ +absvdi2 +absvsi2 +adddf3 +adddf3vfp +addsf3 +addsf3vfp +addvdi3 +addvsi3 +ashldi3 +ashrdi3 +bswapdi2 +bswapsi2 +clzdi2 +clzsi2 +cmpdi2 +ctzdi2 +ctzsi2 +divdc3 +divdf3 +divdf3vfp +divdi3 +divmodsi4 +divsc3 +divsf3 +divsf3vfp +divsi3 +eqdf2 +eqdf2vfp +eqsf2 +eqsf2vfp +extendsfdf2 +extendsfdf2vfp +ffsdi2 +fixdfdi +fixdfsi +fixdfsivfp +fixsfdi +fixsfsi +fixsfsivfp +fixunsdfdi +fixunsdfsi +fixunsdfsivfp +fixunssfdi +fixunssfsi +fixunssfsivfp +floatdidf +floatdisf +floatsidf +floatsidfvfp +floatsisf +floatsisfvfp +floatundidf +floatundisf +floatunsidf +floatunsisf +floatunssidfvfp +floatunssisfvfp +gcc_personality_sj0 +gedf2 +gedf2vfp +gesf2 +gesf2vfp +gtdf2 +gtdf2vfp +gtsf2 +gtsf2vfp +ledf2 +ledf2vfp +lesf2 +lesf2vfp +lshrdi3 +ltdf2 +ltdf2vfp +ltsf2 +ltsf2vfp +moddi3 +modsi3 +muldc3 +muldf3 +muldf3vfp +muldi3 +mulodi4 +mulosi4 +mulsc3 +mulsf3 +mulsf3vfp +mulvdi3 +mulvsi3 +nedf2 +nedf2vfp +negdi2 +negvdi2 +negvsi2 +nesf2 +nesf2vfp +paritydi2 +paritysi2 +popcountdi2 +popcountsi2 +powidf2 +powisf2 +subdf3 +subdf3vfp +subsf3 +subsf3vfp +subvdi3 +subvsi3 +truncdfsf2 +truncdfsf2vfp +ucmpdi2 +udivdi3 +udivmoddi4 +udivmodsi4 +udivsi3 +umoddi3 +umodsi3 +unorddf2 +unorddf2vfp +unordsf2 +unordsf2vfp diff --git a/lib/builtins/Darwin-excludes/ios7-arm64.txt b/lib/builtins/Darwin-excludes/ios7-arm64.txt new file mode 100644 index 000000000000..5e4caf9e9fb7 --- /dev/null +++ b/lib/builtins/Darwin-excludes/ios7-arm64.txt @@ -0,0 +1,16 @@ +clzti2 +divti3 +fixdfti +fixsfti +fixunsdfti +floattidf +floattisf +floatuntidf +floatuntisf +gcc_personality_v0 +modti3 +powidf2 +powisf2 +udivmodti4 +udivti3 +umodti3 diff --git a/lib/builtins/Darwin-excludes/iossim-i386.txt b/lib/builtins/Darwin-excludes/iossim-i386.txt new file mode 100644 index 000000000000..60c0e2d65056 --- /dev/null +++ b/lib/builtins/Darwin-excludes/iossim-i386.txt @@ -0,0 +1,82 @@ +absvti2 +addtf3 +addvti3 +ashlti3 +ashrti3 +clzti2 +cmpti2 +ctzti2 +divti3 +divtf3 +ffsti2 +fixdfti +fixsfti +fixunsdfti +fixunssfti +fixunsxfti +fixxfti +floattidf +floattisf +floattixf +floatuntidf +floatuntisf +floatuntixf +lshrti3 +modti3 +muloti4 +multi3 +multf3 +mulvti3 +negti2 +negvti2 +parityti2 +popcountti2 +powitf2 +subvti3 +subtf3 +trampoline_setup +ucmpti2 +udivmodti4 +udivti3 +umodti3 +absvti2 +addtf3 +addvti3 +ashlti3 +ashrti3 +clzti2 +cmpti2 +ctzti2 +divti3 +divtf3 +ffsti2 +fixdfti +fixsfti +fixunsdfti +fixunssfti +fixunsxfti +fixxfti +floattidf +floattisf +floattixf +floatuntidf +floatuntisf +floatuntixf +lshrti3 +modti3 +muloti4 +multi3 +multf3 +mulvti3 +negti2 +negvti2 +parityti2 +popcountti2 +powitf2 +subvti3 +subtf3 +trampoline_setup +ucmpti2 +udivmodti4 +udivti3 +umodti3 diff --git a/lib/builtins/Darwin-excludes/iossim-x86_64.txt b/lib/builtins/Darwin-excludes/iossim-x86_64.txt new file mode 100644 index 000000000000..de1574e6ce3d --- /dev/null +++ b/lib/builtins/Darwin-excludes/iossim-x86_64.txt @@ -0,0 +1,12 @@ +addtf3 +divtf3 +multf3 +powitf2 +subtf3 +trampoline_setup +addtf3 +divtf3 +multf3 +powitf2 +subtf3 +trampoline_setup diff --git a/lib/builtins/Darwin-excludes/iossim.txt b/lib/builtins/Darwin-excludes/iossim.txt new file mode 100644 index 000000000000..5db24000a174 --- /dev/null +++ b/lib/builtins/Darwin-excludes/iossim.txt @@ -0,0 +1 @@ +apple_versioning diff --git a/lib/builtins/Darwin-excludes/osx-i386.txt b/lib/builtins/Darwin-excludes/osx-i386.txt new file mode 100644 index 000000000000..60c0e2d65056 --- /dev/null +++ b/lib/builtins/Darwin-excludes/osx-i386.txt @@ -0,0 +1,82 @@ +absvti2 +addtf3 +addvti3 +ashlti3 +ashrti3 +clzti2 +cmpti2 +ctzti2 +divti3 +divtf3 +ffsti2 +fixdfti +fixsfti +fixunsdfti +fixunssfti +fixunsxfti +fixxfti +floattidf +floattisf +floattixf +floatuntidf +floatuntisf +floatuntixf +lshrti3 +modti3 +muloti4 +multi3 +multf3 +mulvti3 +negti2 +negvti2 +parityti2 +popcountti2 +powitf2 +subvti3 +subtf3 +trampoline_setup +ucmpti2 +udivmodti4 +udivti3 +umodti3 +absvti2 +addtf3 +addvti3 +ashlti3 +ashrti3 +clzti2 +cmpti2 +ctzti2 +divti3 +divtf3 +ffsti2 +fixdfti +fixsfti +fixunsdfti +fixunssfti +fixunsxfti +fixxfti +floattidf +floattisf +floattixf +floatuntidf +floatuntisf +floatuntixf +lshrti3 +modti3 +muloti4 +multi3 +multf3 +mulvti3 +negti2 +negvti2 +parityti2 +popcountti2 +powitf2 +subvti3 +subtf3 +trampoline_setup +ucmpti2 +udivmodti4 +udivti3 +umodti3 diff --git a/lib/builtins/Darwin-excludes/osx-x86_64.txt b/lib/builtins/Darwin-excludes/osx-x86_64.txt new file mode 100644 index 000000000000..de1574e6ce3d --- /dev/null +++ b/lib/builtins/Darwin-excludes/osx-x86_64.txt @@ -0,0 +1,12 @@ +addtf3 +divtf3 +multf3 +powitf2 +subtf3 +trampoline_setup +addtf3 +divtf3 +multf3 +powitf2 +subtf3 +trampoline_setup diff --git a/lib/builtins/Darwin-excludes/osx.txt b/lib/builtins/Darwin-excludes/osx.txt new file mode 100644 index 000000000000..5db24000a174 --- /dev/null +++ b/lib/builtins/Darwin-excludes/osx.txt @@ -0,0 +1 @@ +apple_versioning diff --git a/lib/builtins/README.txt b/lib/builtins/README.txt index 1c08e7415e64..ad36e4e5279a 100644 --- a/lib/builtins/README.txt +++ b/lib/builtins/README.txt @@ -220,7 +220,9 @@ _Unwind_Reason_Code __gcc_personality_v0(int version, _Unwind_Action actions, // for use with some implementations of assert() in <assert.h> void __eprintf(const char* format, const char* assertion_expression, const char* line, const char* file); - + +// for systems with emulated thread local storage +void* __emutls_get_address(struct __emutls_control*); // Power PC specific functions diff --git a/lib/builtins/arm/aeabi_cdcmp.S b/lib/builtins/arm/aeabi_cdcmp.S new file mode 100644 index 000000000000..036a6f542f79 --- /dev/null +++ b/lib/builtins/arm/aeabi_cdcmp.S @@ -0,0 +1,96 @@ +//===-- aeabi_cdcmp.S - EABI cdcmp* implementation ------------------------===// +// +// The LLVM Compiler Infrastructure +// +// This file is dual licensed under the MIT and the University of Illinois Open +// Source Licenses. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// + +#include "../assembly.h" + +#if __BYTE_ORDER__ != __ORDER_LITTLE_ENDIAN__ +#error big endian support not implemented +#endif + +#define APSR_Z (1 << 30) +#define APSR_C (1 << 29) + +// void __aeabi_cdcmpeq(double a, double b) { +// if (isnan(a) || isnan(b)) { +// Z = 0; C = 1; +// } else { +// __aeabi_cdcmple(a, b); +// } +// } + + .syntax unified + .p2align 2 +DEFINE_COMPILERRT_FUNCTION(__aeabi_cdcmpeq) + push {r0-r3, lr} + bl __aeabi_cdcmpeq_check_nan + cmp r0, #1 + pop {r0-r3, lr} + + // NaN has been ruled out, so __aeabi_cdcmple can't trap + bne __aeabi_cdcmple + + msr CPSR_f, #APSR_C + JMP(lr) +END_COMPILERRT_FUNCTION(__aeabi_cdcmpeq) + + +// void __aeabi_cdcmple(double a, double b) { +// if (__aeabi_dcmplt(a, b)) { +// Z = 0; C = 0; +// } else if (__aeabi_dcmpeq(a, b)) { +// Z = 1; C = 1; +// } else { +// Z = 0; C = 1; +// } +// } + + .syntax unified + .p2align 2 +DEFINE_COMPILERRT_FUNCTION(__aeabi_cdcmple) + // Per the RTABI, this function must preserve r0-r11. + // Save lr in the same instruction for compactness + push {r0-r3, lr} + + bl __aeabi_dcmplt + cmp r0, #1 + moveq ip, #0 + beq 1f + + ldm sp, {r0-r3} + bl __aeabi_dcmpeq + cmp r0, #1 + moveq ip, #(APSR_C | APSR_Z) + movne ip, #(APSR_C) + +1: + msr CPSR_f, ip + pop {r0-r3} + POP_PC() +END_COMPILERRT_FUNCTION(__aeabi_cdcmple) + +// int __aeabi_cdrcmple(double a, double b) { +// return __aeabi_cdcmple(b, a); +// } + + .syntax unified + .p2align 2 +DEFINE_COMPILERRT_FUNCTION(__aeabi_cdrcmple) + // Swap r0 and r2 + mov ip, r0 + mov r0, r2 + mov r2, ip + + // Swap r1 and r3 + mov ip, r1 + mov r1, r3 + mov r3, ip + + b __aeabi_cdcmple +END_COMPILERRT_FUNCTION(__aeabi_cdrcmple) + diff --git a/lib/builtins/arm/aeabi_cdcmpeq_check_nan.c b/lib/builtins/arm/aeabi_cdcmpeq_check_nan.c new file mode 100644 index 000000000000..577f6b2c5535 --- /dev/null +++ b/lib/builtins/arm/aeabi_cdcmpeq_check_nan.c @@ -0,0 +1,16 @@ +//===-- lib/arm/aeabi_cdcmpeq_helper.c - Helper for cdcmpeq ---------------===// +// +// The LLVM Compiler Infrastructure +// +// This file is dual licensed under the MIT and the University of Illinois Open +// Source Licenses. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// + +#include <stdint.h> + +__attribute__((pcs("aapcs"))) +__attribute__((visibility("hidden"))) +int __aeabi_cdcmpeq_check_nan(double a, double b) { + return __builtin_isnan(a) || __builtin_isnan(b); +} diff --git a/lib/builtins/arm/aeabi_cfcmp.S b/lib/builtins/arm/aeabi_cfcmp.S new file mode 100644 index 000000000000..43594e5c3936 --- /dev/null +++ b/lib/builtins/arm/aeabi_cfcmp.S @@ -0,0 +1,91 @@ +//===-- aeabi_cfcmp.S - EABI cfcmp* implementation ------------------------===// +// +// The LLVM Compiler Infrastructure +// +// This file is dual licensed under the MIT and the University of Illinois Open +// Source Licenses. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// + +#include "../assembly.h" + +#if __BYTE_ORDER__ != __ORDER_LITTLE_ENDIAN__ +#error big endian support not implemented +#endif + +#define APSR_Z (1 << 30) +#define APSR_C (1 << 29) + +// void __aeabi_cfcmpeq(float a, float b) { +// if (isnan(a) || isnan(b)) { +// Z = 0; C = 1; +// } else { +// __aeabi_cfcmple(a, b); +// } +// } + + .syntax unified + .p2align 2 +DEFINE_COMPILERRT_FUNCTION(__aeabi_cfcmpeq) + push {r0-r3, lr} + bl __aeabi_cfcmpeq_check_nan + cmp r0, #1 + pop {r0-r3, lr} + + // NaN has been ruled out, so __aeabi_cfcmple can't trap + bne __aeabi_cfcmple + + msr CPSR_f, #APSR_C + JMP(lr) +END_COMPILERRT_FUNCTION(__aeabi_cfcmpeq) + + +// void __aeabi_cfcmple(float a, float b) { +// if (__aeabi_fcmplt(a, b)) { +// Z = 0; C = 0; +// } else if (__aeabi_fcmpeq(a, b)) { +// Z = 1; C = 1; +// } else { +// Z = 0; C = 1; +// } +// } + + .syntax unified + .p2align 2 +DEFINE_COMPILERRT_FUNCTION(__aeabi_cfcmple) + // Per the RTABI, this function must preserve r0-r11. + // Save lr in the same instruction for compactness + push {r0-r3, lr} + + bl __aeabi_fcmplt + cmp r0, #1 + moveq ip, #0 + beq 1f + + ldm sp, {r0-r3} + bl __aeabi_fcmpeq + cmp r0, #1 + moveq ip, #(APSR_C | APSR_Z) + movne ip, #(APSR_C) + +1: + msr CPSR_f, ip + pop {r0-r3} + POP_PC() +END_COMPILERRT_FUNCTION(__aeabi_cfcmple) + +// int __aeabi_cfrcmple(float a, float b) { +// return __aeabi_cfcmple(b, a); +// } + + .syntax unified + .p2align 2 +DEFINE_COMPILERRT_FUNCTION(__aeabi_cfrcmple) + // Swap r0 and r1 + mov ip, r0 + mov r0, r1 + mov r1, ip + + b __aeabi_cfcmple +END_COMPILERRT_FUNCTION(__aeabi_cfrcmple) + diff --git a/lib/builtins/arm/aeabi_cfcmpeq_check_nan.c b/lib/builtins/arm/aeabi_cfcmpeq_check_nan.c new file mode 100644 index 000000000000..992e31fbd8d6 --- /dev/null +++ b/lib/builtins/arm/aeabi_cfcmpeq_check_nan.c @@ -0,0 +1,16 @@ +//===-- lib/arm/aeabi_cfcmpeq_helper.c - Helper for cdcmpeq ---------------===// +// +// The LLVM Compiler Infrastructure +// +// This file is dual licensed under the MIT and the University of Illinois Open +// Source Licenses. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// + +#include <stdint.h> + +__attribute__((pcs("aapcs"))) +__attribute__((visibility("hidden"))) +int __aeabi_cfcmpeq_check_nan(float a, float b) { + return __builtin_isnan(a) || __builtin_isnan(b); +} diff --git a/lib/builtins/arm/aeabi_drsub.c b/lib/builtins/arm/aeabi_drsub.c new file mode 100644 index 000000000000..fc17d5a4cc76 --- /dev/null +++ b/lib/builtins/arm/aeabi_drsub.c @@ -0,0 +1,19 @@ +//===-- lib/arm/aeabi_drsub.c - Double-precision subtraction --------------===// +// +// The LLVM Compiler Infrastructure +// +// This file is dual licensed under the MIT and the University of Illinois Open +// Source Licenses. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// + +#define DOUBLE_PRECISION +#include "../fp_lib.h" + +COMPILER_RT_ABI fp_t +__aeabi_dsub(fp_t, fp_t); + +COMPILER_RT_ABI fp_t +__aeabi_drsub(fp_t a, fp_t b) { + return __aeabi_dsub(b, a); +} diff --git a/lib/builtins/arm/aeabi_frsub.c b/lib/builtins/arm/aeabi_frsub.c new file mode 100644 index 000000000000..64258dc7e070 --- /dev/null +++ b/lib/builtins/arm/aeabi_frsub.c @@ -0,0 +1,19 @@ +//===-- lib/arm/aeabi_frsub.c - Single-precision subtraction --------------===// +// +// The LLVM Compiler Infrastructure +// +// This file is dual licensed under the MIT and the University of Illinois Open +// Source Licenses. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// + +#define SINGLE_PRECISION +#include "../fp_lib.h" + +COMPILER_RT_ABI fp_t +__aeabi_fsub(fp_t, fp_t); + +COMPILER_RT_ABI fp_t +__aeabi_frsub(fp_t a, fp_t b) { + return __aeabi_fsub(b, a); +} diff --git a/lib/builtins/assembly.h b/lib/builtins/assembly.h index 8bb0ddc106bd..c28970534cc4 100644 --- a/lib/builtins/assembly.h +++ b/lib/builtins/assembly.h @@ -73,6 +73,15 @@ #define JMPc(r, c) mov##c pc, r #endif +// pop {pc} can't switch Thumb mode on ARMv4T +#if __ARM_ARCH >= 5 +#define POP_PC() pop {pc} +#else +#define POP_PC() \ + pop {ip}; \ + JMP(ip) +#endif + #if __ARM_ARCH_ISA_THUMB == 2 #define IT(cond) it cond #define ITT(cond) itt cond diff --git a/lib/builtins/atomic.c b/lib/builtins/atomic.c index 35c8837dcecf..f1ddc3e0c522 100644 --- a/lib/builtins/atomic.c +++ b/lib/builtins/atomic.c @@ -56,13 +56,13 @@ static const long SPINLOCK_MASK = SPINLOCK_COUNT - 1; #include <machine/atomic.h> #include <sys/umtx.h> typedef struct _usem Lock; -inline static void unlock(Lock *l) { +__inline static void unlock(Lock *l) { __c11_atomic_store((_Atomic(uint32_t)*)&l->_count, 1, __ATOMIC_RELEASE); __c11_atomic_thread_fence(__ATOMIC_SEQ_CST); if (l->_has_waiters) _umtx_op(l, UMTX_OP_SEM_WAKE, 1, 0, 0); } -inline static void lock(Lock *l) { +__inline static void lock(Lock *l) { uint32_t old = 1; while (!__c11_atomic_compare_exchange_weak((_Atomic(uint32_t)*)&l->_count, &old, 0, __ATOMIC_ACQUIRE, __ATOMIC_RELAXED)) { @@ -76,12 +76,12 @@ static Lock locks[SPINLOCK_COUNT] = { [0 ... SPINLOCK_COUNT-1] = {0,1,0} }; #elif defined(__APPLE__) #include <libkern/OSAtomic.h> typedef OSSpinLock Lock; -inline static void unlock(Lock *l) { +__inline static void unlock(Lock *l) { OSSpinLockUnlock(l); } /// Locks a lock. In the current implementation, this is potentially /// unbounded in the contended case. -inline static void lock(Lock *l) { +__inline static void lock(Lock *l) { OSSpinLockLock(l); } static Lock locks[SPINLOCK_COUNT]; // initialized to OS_SPINLOCK_INIT which is 0 @@ -89,12 +89,12 @@ static Lock locks[SPINLOCK_COUNT]; // initialized to OS_SPINLOCK_INIT which is 0 #else typedef _Atomic(uintptr_t) Lock; /// Unlock a lock. This is a release operation. -inline static void unlock(Lock *l) { +__inline static void unlock(Lock *l) { __c11_atomic_store(l, 0, __ATOMIC_RELEASE); } /// Locks a lock. In the current implementation, this is potentially /// unbounded in the contended case. -inline static void lock(Lock *l) { +__inline static void lock(Lock *l) { uintptr_t old = 0; while (!__c11_atomic_compare_exchange_weak(l, &old, 1, __ATOMIC_ACQUIRE, __ATOMIC_RELAXED)) @@ -106,7 +106,7 @@ static Lock locks[SPINLOCK_COUNT]; /// Returns a lock to use for a given pointer. -static inline Lock *lock_for_pointer(void *ptr) { +static __inline Lock *lock_for_pointer(void *ptr) { intptr_t hash = (intptr_t)ptr; // Disregard the lowest 4 bits. We want all values that may be part of the // same memory operation to hash to the same value and therefore use the same diff --git a/lib/builtins/atomic_flag_clear.c b/lib/builtins/atomic_flag_clear.c index 15984cd5267d..da912af64312 100644 --- a/lib/builtins/atomic_flag_clear.c +++ b/lib/builtins/atomic_flag_clear.c @@ -12,8 +12,16 @@ *===------------------------------------------------------------------------=== */ +#ifndef __has_include +#define __has_include(inc) 0 +#endif + +#if __has_include(<stdatomic.h>) + #include <stdatomic.h> #undef atomic_flag_clear void atomic_flag_clear(volatile atomic_flag *object) { - return __c11_atomic_store(&(object)->_Value, 0, __ATOMIC_SEQ_CST); + __c11_atomic_store(&(object)->_Value, 0, __ATOMIC_SEQ_CST); } + +#endif diff --git a/lib/builtins/atomic_flag_clear_explicit.c b/lib/builtins/atomic_flag_clear_explicit.c index 0f7859c2cd82..1059b787f169 100644 --- a/lib/builtins/atomic_flag_clear_explicit.c +++ b/lib/builtins/atomic_flag_clear_explicit.c @@ -12,9 +12,17 @@ *===------------------------------------------------------------------------=== */ +#ifndef __has_include +#define __has_include(inc) 0 +#endif + +#if __has_include(<stdatomic.h>) + #include <stdatomic.h> #undef atomic_flag_clear_explicit void atomic_flag_clear_explicit(volatile atomic_flag *object, memory_order order) { - return __c11_atomic_store(&(object)->_Value, 0, order); + __c11_atomic_store(&(object)->_Value, 0, order); } + +#endif diff --git a/lib/builtins/atomic_flag_test_and_set.c b/lib/builtins/atomic_flag_test_and_set.c index 07209fc02d5e..e8811d39ef25 100644 --- a/lib/builtins/atomic_flag_test_and_set.c +++ b/lib/builtins/atomic_flag_test_and_set.c @@ -12,8 +12,16 @@ *===------------------------------------------------------------------------=== */ +#ifndef __has_include +#define __has_include(inc) 0 +#endif + +#if __has_include(<stdatomic.h>) + #include <stdatomic.h> #undef atomic_flag_test_and_set _Bool atomic_flag_test_and_set(volatile atomic_flag *object) { return __c11_atomic_exchange(&(object)->_Value, 1, __ATOMIC_SEQ_CST); } + +#endif diff --git a/lib/builtins/atomic_flag_test_and_set_explicit.c b/lib/builtins/atomic_flag_test_and_set_explicit.c index eaa5be08df46..5c8c2df90543 100644 --- a/lib/builtins/atomic_flag_test_and_set_explicit.c +++ b/lib/builtins/atomic_flag_test_and_set_explicit.c @@ -12,9 +12,17 @@ *===------------------------------------------------------------------------=== */ +#ifndef __has_include +#define __has_include(inc) 0 +#endif + +#if __has_include(<stdatomic.h>) + #include <stdatomic.h> #undef atomic_flag_test_and_set_explicit _Bool atomic_flag_test_and_set_explicit(volatile atomic_flag *object, memory_order order) { return __c11_atomic_exchange(&(object)->_Value, 1, order); } + +#endif diff --git a/lib/builtins/atomic_signal_fence.c b/lib/builtins/atomic_signal_fence.c index ad292d2f1c72..9ccc2ae60ad8 100644 --- a/lib/builtins/atomic_signal_fence.c +++ b/lib/builtins/atomic_signal_fence.c @@ -12,8 +12,16 @@ *===------------------------------------------------------------------------=== */ +#ifndef __has_include +#define __has_include(inc) 0 +#endif + +#if __has_include(<stdatomic.h>) + #include <stdatomic.h> #undef atomic_signal_fence void atomic_signal_fence(memory_order order) { __c11_atomic_signal_fence(order); } + +#endif diff --git a/lib/builtins/atomic_thread_fence.c b/lib/builtins/atomic_thread_fence.c index 71f698c9de75..d22560151bc8 100644 --- a/lib/builtins/atomic_thread_fence.c +++ b/lib/builtins/atomic_thread_fence.c @@ -12,8 +12,16 @@ *===------------------------------------------------------------------------=== */ +#ifndef __has_include +#define __has_include(inc) 0 +#endif + +#if __has_include(<stdatomic.h>) + #include <stdatomic.h> #undef atomic_thread_fence void atomic_thread_fence(memory_order order) { __c11_atomic_thread_fence(order); } + +#endif diff --git a/lib/builtins/comparedf2.c b/lib/builtins/comparedf2.c index 64eea1249055..9e29752231e9 100644 --- a/lib/builtins/comparedf2.c +++ b/lib/builtins/comparedf2.c @@ -80,6 +80,11 @@ __ledf2(fp_t a, fp_t b) { } } +#if defined(__ELF__) +// Alias for libgcc compatibility +FNALIAS(__cmpdf2, __ledf2); +#endif + enum GE_RESULT { GE_LESS = -1, GE_EQUAL = 0, diff --git a/lib/builtins/comparesf2.c b/lib/builtins/comparesf2.c index 442289c1004e..1fd50636abaf 100644 --- a/lib/builtins/comparesf2.c +++ b/lib/builtins/comparesf2.c @@ -80,6 +80,11 @@ __lesf2(fp_t a, fp_t b) { } } +#if defined(__ELF__) +// Alias for libgcc compatibility +FNALIAS(__cmpsf2, __lesf2); +#endif + enum GE_RESULT { GE_LESS = -1, GE_EQUAL = 0, diff --git a/lib/builtins/comparetf2.c b/lib/builtins/comparetf2.c index a6436de89e76..c0ad8ed0aecd 100644 --- a/lib/builtins/comparetf2.c +++ b/lib/builtins/comparetf2.c @@ -79,6 +79,11 @@ COMPILER_RT_ABI enum LE_RESULT __letf2(fp_t a, fp_t b) { } } +#if defined(__ELF__) +// Alias for libgcc compatibility +FNALIAS(__cmptf2, __letf2); +#endif + enum GE_RESULT { GE_LESS = -1, GE_EQUAL = 0, diff --git a/lib/builtins/divdc3.c b/lib/builtins/divdc3.c index 7de78c8711e1..3c88390b5e77 100644 --- a/lib/builtins/divdc3.c +++ b/lib/builtins/divdc3.c @@ -17,7 +17,7 @@ /* Returns: the quotient of (a + ib) / (c + id) */ -COMPILER_RT_ABI double _Complex +COMPILER_RT_ABI Dcomplex __divdc3(double __a, double __b, double __c, double __d) { int __ilogbw = 0; @@ -29,31 +29,31 @@ __divdc3(double __a, double __b, double __c, double __d) __d = crt_scalbn(__d, -__ilogbw); } double __denom = __c * __c + __d * __d; - double _Complex z; - __real__ z = crt_scalbn((__a * __c + __b * __d) / __denom, -__ilogbw); - __imag__ z = crt_scalbn((__b * __c - __a * __d) / __denom, -__ilogbw); - if (crt_isnan(__real__ z) && crt_isnan(__imag__ z)) + Dcomplex z; + COMPLEX_REAL(z) = crt_scalbn((__a * __c + __b * __d) / __denom, -__ilogbw); + COMPLEX_IMAGINARY(z) = crt_scalbn((__b * __c - __a * __d) / __denom, -__ilogbw); + if (crt_isnan(COMPLEX_REAL(z)) && crt_isnan(COMPLEX_IMAGINARY(z))) { if ((__denom == 0.0) && (!crt_isnan(__a) || !crt_isnan(__b))) { - __real__ z = crt_copysign(CRT_INFINITY, __c) * __a; - __imag__ z = crt_copysign(CRT_INFINITY, __c) * __b; + COMPLEX_REAL(z) = crt_copysign(CRT_INFINITY, __c) * __a; + COMPLEX_IMAGINARY(z) = crt_copysign(CRT_INFINITY, __c) * __b; } else if ((crt_isinf(__a) || crt_isinf(__b)) && crt_isfinite(__c) && crt_isfinite(__d)) { __a = crt_copysign(crt_isinf(__a) ? 1.0 : 0.0, __a); __b = crt_copysign(crt_isinf(__b) ? 1.0 : 0.0, __b); - __real__ z = CRT_INFINITY * (__a * __c + __b * __d); - __imag__ z = CRT_INFINITY * (__b * __c - __a * __d); + COMPLEX_REAL(z) = CRT_INFINITY * (__a * __c + __b * __d); + COMPLEX_IMAGINARY(z) = CRT_INFINITY * (__b * __c - __a * __d); } else if (crt_isinf(__logbw) && __logbw > 0.0 && crt_isfinite(__a) && crt_isfinite(__b)) { __c = crt_copysign(crt_isinf(__c) ? 1.0 : 0.0, __c); __d = crt_copysign(crt_isinf(__d) ? 1.0 : 0.0, __d); - __real__ z = 0.0 * (__a * __c + __b * __d); - __imag__ z = 0.0 * (__b * __c - __a * __d); + COMPLEX_REAL(z) = 0.0 * (__a * __c + __b * __d); + COMPLEX_IMAGINARY(z) = 0.0 * (__b * __c - __a * __d); } } return z; diff --git a/lib/builtins/divsc3.c b/lib/builtins/divsc3.c index 710d5320803f..42a48315e66d 100644 --- a/lib/builtins/divsc3.c +++ b/lib/builtins/divsc3.c @@ -17,7 +17,7 @@ /* Returns: the quotient of (a + ib) / (c + id) */ -COMPILER_RT_ABI float _Complex +COMPILER_RT_ABI Fcomplex __divsc3(float __a, float __b, float __c, float __d) { int __ilogbw = 0; @@ -29,31 +29,31 @@ __divsc3(float __a, float __b, float __c, float __d) __d = crt_scalbnf(__d, -__ilogbw); } float __denom = __c * __c + __d * __d; - float _Complex z; - __real__ z = crt_scalbnf((__a * __c + __b * __d) / __denom, -__ilogbw); - __imag__ z = crt_scalbnf((__b * __c - __a * __d) / __denom, -__ilogbw); - if (crt_isnan(__real__ z) && crt_isnan(__imag__ z)) + Fcomplex z; + COMPLEX_REAL(z) = crt_scalbnf((__a * __c + __b * __d) / __denom, -__ilogbw); + COMPLEX_IMAGINARY(z) = crt_scalbnf((__b * __c - __a * __d) / __denom, -__ilogbw); + if (crt_isnan(COMPLEX_REAL(z)) && crt_isnan(COMPLEX_IMAGINARY(z))) { if ((__denom == 0) && (!crt_isnan(__a) || !crt_isnan(__b))) { - __real__ z = crt_copysignf(CRT_INFINITY, __c) * __a; - __imag__ z = crt_copysignf(CRT_INFINITY, __c) * __b; + COMPLEX_REAL(z) = crt_copysignf(CRT_INFINITY, __c) * __a; + COMPLEX_IMAGINARY(z) = crt_copysignf(CRT_INFINITY, __c) * __b; } else if ((crt_isinf(__a) || crt_isinf(__b)) && crt_isfinite(__c) && crt_isfinite(__d)) { __a = crt_copysignf(crt_isinf(__a) ? 1 : 0, __a); __b = crt_copysignf(crt_isinf(__b) ? 1 : 0, __b); - __real__ z = CRT_INFINITY * (__a * __c + __b * __d); - __imag__ z = CRT_INFINITY * (__b * __c - __a * __d); + COMPLEX_REAL(z) = CRT_INFINITY * (__a * __c + __b * __d); + COMPLEX_IMAGINARY(z) = CRT_INFINITY * (__b * __c - __a * __d); } else if (crt_isinf(__logbw) && __logbw > 0 && crt_isfinite(__a) && crt_isfinite(__b)) { __c = crt_copysignf(crt_isinf(__c) ? 1 : 0, __c); __d = crt_copysignf(crt_isinf(__d) ? 1 : 0, __d); - __real__ z = 0 * (__a * __c + __b * __d); - __imag__ z = 0 * (__b * __c - __a * __d); + COMPLEX_REAL(z) = 0 * (__a * __c + __b * __d); + COMPLEX_IMAGINARY(z) = 0 * (__b * __c - __a * __d); } } return z; diff --git a/lib/builtins/divtc3.c b/lib/builtins/divtc3.c new file mode 100644 index 000000000000..04693df471ff --- /dev/null +++ b/lib/builtins/divtc3.c @@ -0,0 +1,60 @@ +/*===-- divtc3.c - Implement __divtc3 -------------------------------------=== + * + * The LLVM Compiler Infrastructure + * + * This file is dual licensed under the MIT and the University of Illinois Open + * Source Licenses. See LICENSE.TXT for details. + * + * ===----------------------------------------------------------------------=== + * + * This file implements __divtc3 for the compiler_rt library. + * + *===----------------------------------------------------------------------=== + */ + +#include "int_lib.h" +#include "int_math.h" + +/* Returns: the quotient of (a + ib) / (c + id) */ + +COMPILER_RT_ABI long double _Complex +__divtc3(long double __a, long double __b, long double __c, long double __d) +{ + int __ilogbw = 0; + long double __logbw = crt_logbl(crt_fmaxl(crt_fabsl(__c), crt_fabsl(__d))); + if (crt_isfinite(__logbw)) + { + __ilogbw = (int)__logbw; + __c = crt_scalbnl(__c, -__ilogbw); + __d = crt_scalbnl(__d, -__ilogbw); + } + long double __denom = __c * __c + __d * __d; + long double _Complex z; + __real__ z = crt_scalbnl((__a * __c + __b * __d) / __denom, -__ilogbw); + __imag__ z = crt_scalbnl((__b * __c - __a * __d) / __denom, -__ilogbw); + if (crt_isnan(__real__ z) && crt_isnan(__imag__ z)) + { + if ((__denom == 0.0) && (!crt_isnan(__a) || !crt_isnan(__b))) + { + __real__ z = crt_copysignl(CRT_INFINITY, __c) * __a; + __imag__ z = crt_copysignl(CRT_INFINITY, __c) * __b; + } + else if ((crt_isinf(__a) || crt_isinf(__b)) && + crt_isfinite(__c) && crt_isfinite(__d)) + { + __a = crt_copysignl(crt_isinf(__a) ? 1.0 : 0.0, __a); + __b = crt_copysignl(crt_isinf(__b) ? 1.0 : 0.0, __b); + __real__ z = CRT_INFINITY * (__a * __c + __b * __d); + __imag__ z = CRT_INFINITY * (__b * __c - __a * __d); + } + else if (crt_isinf(__logbw) && __logbw > 0.0 && + crt_isfinite(__a) && crt_isfinite(__b)) + { + __c = crt_copysignl(crt_isinf(__c) ? 1.0 : 0.0, __c); + __d = crt_copysignl(crt_isinf(__d) ? 1.0 : 0.0, __d); + __real__ z = 0.0 * (__a * __c + __b * __d); + __imag__ z = 0.0 * (__b * __c - __a * __d); + } + } + return z; +} diff --git a/lib/builtins/divxc3.c b/lib/builtins/divxc3.c index 175ae3cf4aee..6f49280e5f61 100644 --- a/lib/builtins/divxc3.c +++ b/lib/builtins/divxc3.c @@ -18,7 +18,7 @@ /* Returns: the quotient of (a + ib) / (c + id) */ -COMPILER_RT_ABI long double _Complex +COMPILER_RT_ABI Lcomplex __divxc3(long double __a, long double __b, long double __c, long double __d) { int __ilogbw = 0; @@ -30,31 +30,31 @@ __divxc3(long double __a, long double __b, long double __c, long double __d) __d = crt_scalbnl(__d, -__ilogbw); } long double __denom = __c * __c + __d * __d; - long double _Complex z; - __real__ z = crt_scalbnl((__a * __c + __b * __d) / __denom, -__ilogbw); - __imag__ z = crt_scalbnl((__b * __c - __a * __d) / __denom, -__ilogbw); - if (crt_isnan(__real__ z) && crt_isnan(__imag__ z)) + Lcomplex z; + COMPLEX_REAL(z) = crt_scalbnl((__a * __c + __b * __d) / __denom, -__ilogbw); + COMPLEX_IMAGINARY(z) = crt_scalbnl((__b * __c - __a * __d) / __denom, -__ilogbw); + if (crt_isnan(COMPLEX_REAL(z)) && crt_isnan(COMPLEX_IMAGINARY(z))) { if ((__denom == 0) && (!crt_isnan(__a) || !crt_isnan(__b))) { - __real__ z = crt_copysignl(CRT_INFINITY, __c) * __a; - __imag__ z = crt_copysignl(CRT_INFINITY, __c) * __b; + COMPLEX_REAL(z) = crt_copysignl(CRT_INFINITY, __c) * __a; + COMPLEX_IMAGINARY(z) = crt_copysignl(CRT_INFINITY, __c) * __b; } else if ((crt_isinf(__a) || crt_isinf(__b)) && crt_isfinite(__c) && crt_isfinite(__d)) { __a = crt_copysignl(crt_isinf(__a) ? 1 : 0, __a); __b = crt_copysignl(crt_isinf(__b) ? 1 : 0, __b); - __real__ z = CRT_INFINITY * (__a * __c + __b * __d); - __imag__ z = CRT_INFINITY * (__b * __c - __a * __d); + COMPLEX_REAL(z) = CRT_INFINITY * (__a * __c + __b * __d); + COMPLEX_IMAGINARY(z) = CRT_INFINITY * (__b * __c - __a * __d); } else if (crt_isinf(__logbw) && __logbw > 0 && crt_isfinite(__a) && crt_isfinite(__b)) { __c = crt_copysignl(crt_isinf(__c) ? 1 : 0, __c); __d = crt_copysignl(crt_isinf(__d) ? 1 : 0, __d); - __real__ z = 0 * (__a * __c + __b * __d); - __imag__ z = 0 * (__b * __c - __a * __d); + COMPLEX_REAL(z) = 0 * (__a * __c + __b * __d); + COMPLEX_IMAGINARY(z) = 0 * (__b * __c - __a * __d); } } return z; diff --git a/lib/builtins/emutls.c b/lib/builtins/emutls.c new file mode 100644 index 000000000000..09e79568bd56 --- /dev/null +++ b/lib/builtins/emutls.c @@ -0,0 +1,183 @@ +/* ===---------- emutls.c - Implements __emutls_get_address ---------------=== + * + * The LLVM Compiler Infrastructure + * + * This file is dual licensed under the MIT and the University of Illinois Open + * Source Licenses. See LICENSE.TXT for details. + * + * ===----------------------------------------------------------------------=== + */ +#include <pthread.h> +#include <stdint.h> +#include <stdlib.h> +#include <string.h> + +#include "int_lib.h" +#include "int_util.h" + +/* Default is not to use posix_memalign, so systems like Android + * can use thread local data without heavier POSIX memory allocators. + */ +#ifndef EMUTLS_USE_POSIX_MEMALIGN +#define EMUTLS_USE_POSIX_MEMALIGN 0 +#endif + +/* For every TLS variable xyz, + * there is one __emutls_control variable named __emutls_v.xyz. + * If xyz has non-zero initial value, __emutls_v.xyz's "value" + * will point to __emutls_t.xyz, which has the initial value. + */ +typedef struct __emutls_control { + size_t size; /* size of the object in bytes */ + size_t align; /* alignment of the object in bytes */ + union { + uintptr_t index; /* data[index-1] is the object address */ + void* address; /* object address, when in single thread env */ + } object; + void* value; /* null or non-zero initial value for the object */ +} __emutls_control; + +static __inline void *emutls_memalign_alloc(size_t align, size_t size) { + void *base; +#if EMUTLS_USE_POSIX_MEMALIGN + if (posix_memalign(&base, align, size) != 0) + abort(); +#else + #define EXTRA_ALIGN_PTR_BYTES (align - 1 + sizeof(void*)) + char* object; + if ((object = malloc(EXTRA_ALIGN_PTR_BYTES + size)) == NULL) + abort(); + base = (void*)(((uintptr_t)(object + EXTRA_ALIGN_PTR_BYTES)) + & ~(uintptr_t)(align - 1)); + + ((void**)base)[-1] = object; +#endif + return base; +} + +static __inline void emutls_memalign_free(void *base) { +#if EMUTLS_USE_POSIX_MEMALIGN + free(base); +#else + /* The mallocated address is in ((void**)base)[-1] */ + free(((void**)base)[-1]); +#endif +} + +/* Emulated TLS objects are always allocated at run-time. */ +static __inline void *emutls_allocate_object(__emutls_control *control) { + /* Use standard C types, check with gcc's emutls.o. */ + typedef unsigned int gcc_word __attribute__((mode(word))); + typedef unsigned int gcc_pointer __attribute__((mode(pointer))); + COMPILE_TIME_ASSERT(sizeof(size_t) == sizeof(gcc_word)); + COMPILE_TIME_ASSERT(sizeof(uintptr_t) == sizeof(gcc_pointer)); + COMPILE_TIME_ASSERT(sizeof(uintptr_t) == sizeof(void*)); + + size_t size = control->size; + size_t align = control->align; + if (align < sizeof(void*)) + align = sizeof(void*); + /* Make sure that align is power of 2. */ + if ((align & (align - 1)) != 0) + abort(); + + void* base = emutls_memalign_alloc(align, size); + if (control->value) + memcpy(base, control->value, size); + else + memset(base, 0, size); + return base; +} + +static pthread_mutex_t emutls_mutex = PTHREAD_MUTEX_INITIALIZER; + +static size_t emutls_num_object = 0; /* number of allocated TLS objects */ + +typedef struct emutls_address_array { + uintptr_t size; /* number of elements in the 'data' array */ + void* data[]; +} emutls_address_array; + +static pthread_key_t emutls_pthread_key; + +static void emutls_key_destructor(void* ptr) { + emutls_address_array* array = (emutls_address_array*)ptr; + uintptr_t i; + for (i = 0; i < array->size; ++i) { + if (array->data[i]) + emutls_memalign_free(array->data[i]); + } + free(ptr); +} + +static void emutls_init(void) { + if (pthread_key_create(&emutls_pthread_key, emutls_key_destructor) != 0) + abort(); +} + +/* Returns control->object.index; set index if not allocated yet. */ +static __inline uintptr_t emutls_get_index(__emutls_control *control) { + uintptr_t index = __atomic_load_n(&control->object.index, __ATOMIC_ACQUIRE); + if (!index) { + static pthread_once_t once = PTHREAD_ONCE_INIT; + pthread_once(&once, emutls_init); + pthread_mutex_lock(&emutls_mutex); + index = control->object.index; + if (!index) { + index = ++emutls_num_object; + __atomic_store_n(&control->object.index, index, __ATOMIC_RELEASE); + } + pthread_mutex_unlock(&emutls_mutex); + } + return index; +} + +/* Updates newly allocated thread local emutls_address_array. */ +static __inline void emutls_check_array_set_size(emutls_address_array *array, + uintptr_t size) { + if (array == NULL) + abort(); + array->size = size; + pthread_setspecific(emutls_pthread_key, (void*)array); +} + +/* Returns the new 'data' array size, number of elements, + * which must be no smaller than the given index. + */ +static __inline uintptr_t emutls_new_data_array_size(uintptr_t index) { + /* Need to allocate emutls_address_array with one extra slot + * to store the data array size. + * Round up the emutls_address_array size to multiple of 16. + */ + return ((index + 1 + 15) & ~((uintptr_t)15)) - 1; +} + +/* Returns the thread local emutls_address_array. + * Extends its size if necessary to hold address at index. + */ +static __inline emutls_address_array * +emutls_get_address_array(uintptr_t index) { + emutls_address_array* array = pthread_getspecific(emutls_pthread_key); + if (array == NULL) { + uintptr_t new_size = emutls_new_data_array_size(index); + array = calloc(new_size + 1, sizeof(void*)); + emutls_check_array_set_size(array, new_size); + } else if (index > array->size) { + uintptr_t orig_size = array->size; + uintptr_t new_size = emutls_new_data_array_size(index); + array = realloc(array, (new_size + 1) * sizeof(void*)); + if (array) + memset(array->data + orig_size, 0, + (new_size - orig_size) * sizeof(void*)); + emutls_check_array_set_size(array, new_size); + } + return array; +} + +void* __emutls_get_address(__emutls_control* control) { + uintptr_t index = emutls_get_index(control); + emutls_address_array* array = emutls_get_address_array(index); + if (array->data[index - 1] == NULL) + array->data[index - 1] = emutls_allocate_object(control); + return array->data[index - 1]; +} diff --git a/lib/builtins/enable_execute_stack.c b/lib/builtins/enable_execute_stack.c index 23e494051adf..0dc3482c4467 100644 --- a/lib/builtins/enable_execute_stack.c +++ b/lib/builtins/enable_execute_stack.c @@ -21,8 +21,8 @@ #define HAVE_SYSCONF 1 #ifdef _WIN32 -#include <windef.h> -#include <winbase.h> +#define WIN32_LEAN_AND_MEAN +#include <Windows.h> #else #ifndef __APPLE__ #include <unistd.h> diff --git a/lib/builtins/extendhfsf2.c b/lib/builtins/extendhfsf2.c index 7524e2ea7ed6..27115a48c184 100644 --- a/lib/builtins/extendhfsf2.c +++ b/lib/builtins/extendhfsf2.c @@ -12,9 +12,11 @@ #define DST_SINGLE #include "fp_extend_impl.inc" +ARM_EABI_FNALIAS(h2f, extendhfsf2) + // Use a forwarding definition and noinline to implement a poor man's alias, // as there isn't a good cross-platform way of defining one. -COMPILER_RT_ABI __attribute__((noinline)) float __extendhfsf2(uint16_t a) { +COMPILER_RT_ABI NOINLINE float __extendhfsf2(uint16_t a) { return __extendXfYf2__(a); } diff --git a/lib/builtins/fixunsdfdi.c b/lib/builtins/fixunsdfdi.c index 2e0d87eacf05..4b0bc9e1d051 100644 --- a/lib/builtins/fixunsdfdi.c +++ b/lib/builtins/fixunsdfdi.c @@ -22,8 +22,8 @@ COMPILER_RT_ABI du_int __fixunsdfdi(double a) { if (a <= 0.0) return 0; - su_int high = a/0x1p32f; - su_int low = a - (double)high*0x1p32f; + su_int high = a / 4294967296.f; /* a / 0x1p32f; */ + su_int low = a - (double)high * 4294967296.f; /* high * 0x1p32f; */ return ((du_int)high << 32) | low; } diff --git a/lib/builtins/fixunssfdi.c b/lib/builtins/fixunssfdi.c index 5a154e82cff4..f8ebab854f95 100644 --- a/lib/builtins/fixunssfdi.c +++ b/lib/builtins/fixunssfdi.c @@ -23,8 +23,8 @@ __fixunssfdi(float a) { if (a <= 0.0f) return 0; double da = a; - su_int high = da/0x1p32f; - su_int low = da - (double)high*0x1p32f; + su_int high = da / 4294967296.f; /* da / 0x1p32f; */ + su_int low = da - (double)high * 4294967296.f; /* high * 0x1p32f; */ return ((du_int)high << 32) | low; } diff --git a/lib/builtins/floatdidf.c b/lib/builtins/floatdidf.c index e53fa2580f6e..a300c9f312d2 100644 --- a/lib/builtins/floatdidf.c +++ b/lib/builtins/floatdidf.c @@ -32,8 +32,8 @@ ARM_EABI_FNALIAS(l2d, floatdidf) COMPILER_RT_ABI double __floatdidf(di_int a) { - static const double twop52 = 0x1.0p52; - static const double twop32 = 0x1.0p32; + static const double twop52 = 4503599627370496.0; // 0x1.0p52 + static const double twop32 = 4294967296.0; // 0x1.0p32 union { int64_t x; double d; } low = { .d = twop52 }; diff --git a/lib/builtins/floatditf.c b/lib/builtins/floatditf.c new file mode 100644 index 000000000000..cd51dd8aade4 --- /dev/null +++ b/lib/builtins/floatditf.c @@ -0,0 +1,50 @@ +//===-- lib/floatditf.c - integer -> quad-precision conversion ----*- C -*-===// +// +// The LLVM Compiler Infrastructure +// +// This file is dual licensed under the MIT and the University of Illinois Open +// Source Licenses. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// +// +// This file implements di_int to quad-precision conversion for the +// compiler-rt library in the IEEE-754 default round-to-nearest, ties-to-even +// mode. +// +//===----------------------------------------------------------------------===// + +#define QUAD_PRECISION +#include "fp_lib.h" + +#if defined(CRT_HAS_128BIT) && defined(CRT_LDBL_128BIT) +COMPILER_RT_ABI fp_t __floatditf(di_int a) { + + const int aWidth = sizeof a * CHAR_BIT; + + // Handle zero as a special case to protect clz + if (a == 0) + return fromRep(0); + + // All other cases begin by extracting the sign and absolute value of a + rep_t sign = 0; + du_int aAbs = (du_int)a; + if (a < 0) { + sign = signBit; + aAbs = ~(du_int)a + 1U; + } + + // Exponent of (fp_t)a is the width of abs(a). + const int exponent = (aWidth - 1) - __builtin_clzll(aAbs); + rep_t result; + + // Shift a into the significand field, rounding if it is a right-shift + const int shift = significandBits - exponent; + result = (rep_t)aAbs << shift ^ implicitBit; + + // Insert the exponent + result += (rep_t)(exponent + exponentBias) << significandBits; + // Insert the sign bit and return + return fromRep(result | sign); +} + +#endif diff --git a/lib/builtins/floatsitf.c b/lib/builtins/floatsitf.c index 85346933f81e..f0abca363b5e 100644 --- a/lib/builtins/floatsitf.c +++ b/lib/builtins/floatsitf.c @@ -30,16 +30,14 @@ COMPILER_RT_ABI fp_t __floatsitf(int a) { unsigned aAbs = (unsigned)a; if (a < 0) { sign = signBit; - aAbs += 0x80000000; + aAbs = ~(unsigned)a + 1U; } // Exponent of (fp_t)a is the width of abs(a). - const int exponent = (aWidth - 1) - __builtin_clz(a); + const int exponent = (aWidth - 1) - __builtin_clz(aAbs); rep_t result; - // Shift a into the significand field and clear the implicit bit. Extra - // cast to unsigned int is necessary to get the correct behavior for - // the input INT_MIN. + // Shift a into the significand field and clear the implicit bit. const int shift = significandBits - exponent; result = (rep_t)aAbs << shift ^ implicitBit; diff --git a/lib/builtins/floatundidf.c b/lib/builtins/floatundidf.c index 73b8bac1c1a1..67aa86e5e5b8 100644 --- a/lib/builtins/floatundidf.c +++ b/lib/builtins/floatundidf.c @@ -32,9 +32,9 @@ ARM_EABI_FNALIAS(ul2d, floatundidf) COMPILER_RT_ABI double __floatundidf(du_int a) { - static const double twop52 = 0x1.0p52; - static const double twop84 = 0x1.0p84; - static const double twop84_plus_twop52 = 0x1.00000001p84; + static const double twop52 = 4503599627370496.0; // 0x1.0p52 + static const double twop84 = 19342813113834066795298816.0; // 0x1.0p84 + static const double twop84_plus_twop52 = 19342813118337666422669312.0; // 0x1.00000001p84 union { uint64_t x; double d; } high = { .d = twop84 }; union { uint64_t x; double d; } low = { .d = twop52 }; diff --git a/lib/builtins/floatunditf.c b/lib/builtins/floatunditf.c new file mode 100644 index 000000000000..8098e95e82bc --- /dev/null +++ b/lib/builtins/floatunditf.c @@ -0,0 +1,40 @@ +//===-- lib/floatunditf.c - uint -> quad-precision conversion -----*- C -*-===// +// +// The LLVM Compiler Infrastructure +// +// This file is dual licensed under the MIT and the University of Illinois Open +// Source Licenses. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// +// +// This file implements du_int to quad-precision conversion for the +// compiler-rt library in the IEEE-754 default round-to-nearest, ties-to-even +// mode. +// +//===----------------------------------------------------------------------===// + +#define QUAD_PRECISION +#include "fp_lib.h" + +#if defined(CRT_HAS_128BIT) && defined(CRT_LDBL_128BIT) +COMPILER_RT_ABI fp_t __floatunditf(du_int a) { + + const int aWidth = sizeof a * CHAR_BIT; + + // Handle zero as a special case to protect clz + if (a == 0) return fromRep(0); + + // Exponent of (fp_t)a is the width of abs(a). + const int exponent = (aWidth - 1) - __builtin_clzll(a); + rep_t result; + + // Shift a into the significand field and clear the implicit bit. + const int shift = significandBits - exponent; + result = (rep_t)a << shift ^ implicitBit; + + // Insert the exponent + result += (rep_t)(exponent + exponentBias) << significandBits; + return fromRep(result); +} + +#endif diff --git a/lib/builtins/fp_add_impl.inc b/lib/builtins/fp_add_impl.inc index 5741889728cd..b47be1b648e6 100644 --- a/lib/builtins/fp_add_impl.inc +++ b/lib/builtins/fp_add_impl.inc @@ -14,7 +14,7 @@ #include "fp_lib.h" -static inline fp_t __addXf3__(fp_t a, fp_t b) { +static __inline fp_t __addXf3__(fp_t a, fp_t b) { rep_t aRep = toRep(a); rep_t bRep = toRep(b); const rep_t aAbs = aRep & absMask; diff --git a/lib/builtins/fp_extend.h b/lib/builtins/fp_extend.h index 5c2b92310df1..6d95a0680709 100644 --- a/lib/builtins/fp_extend.h +++ b/lib/builtins/fp_extend.h @@ -28,7 +28,7 @@ typedef double src_t; typedef uint64_t src_rep_t; #define SRC_REP_C UINT64_C static const int srcSigBits = 52; -static inline int src_rep_t_clz(src_rep_t a) { +static __inline int src_rep_t_clz(src_rep_t a) { #if defined __LP64__ return __builtin_clzl(a); #else @@ -75,12 +75,12 @@ static const int dstSigBits = 112; // End of specialization parameters. Two helper routines for conversion to and // from the representation of floating-point data as integer values follow. -static inline src_rep_t srcToRep(src_t x) { +static __inline src_rep_t srcToRep(src_t x) { const union { src_t f; src_rep_t i; } rep = {.f = x}; return rep.i; } -static inline dst_t dstFromRep(dst_rep_t x) { +static __inline dst_t dstFromRep(dst_rep_t x) { const union { dst_t f; dst_rep_t i; } rep = {.i = x}; return rep.f; } diff --git a/lib/builtins/fp_extend_impl.inc b/lib/builtins/fp_extend_impl.inc index edcfa8d2329d..b785cc7687ad 100644 --- a/lib/builtins/fp_extend_impl.inc +++ b/lib/builtins/fp_extend_impl.inc @@ -38,7 +38,7 @@ #include "fp_extend.h" -static inline dst_t __extendXfYf2__(src_t a) { +static __inline dst_t __extendXfYf2__(src_t a) { // Various constants whose values follow from the type parameters. // Any reasonable optimizer will fold and propagate all of these. const int srcBits = sizeof(src_t)*CHAR_BIT; diff --git a/lib/builtins/fp_fixint_impl.inc b/lib/builtins/fp_fixint_impl.inc index 035e87ca10e0..da70d4d39301 100644 --- a/lib/builtins/fp_fixint_impl.inc +++ b/lib/builtins/fp_fixint_impl.inc @@ -14,7 +14,7 @@ #include "fp_lib.h" -static inline fixint_t __fixint(fp_t a) { +static __inline fixint_t __fixint(fp_t a) { const fixint_t fixint_max = (fixint_t)((~(fixuint_t)0) / 2); const fixint_t fixint_min = -fixint_max - 1; // Break a into sign, exponent, significand diff --git a/lib/builtins/fp_fixuint_impl.inc b/lib/builtins/fp_fixuint_impl.inc index 5fefab0e2d8a..d68ccf27a79c 100644 --- a/lib/builtins/fp_fixuint_impl.inc +++ b/lib/builtins/fp_fixuint_impl.inc @@ -14,7 +14,7 @@ #include "fp_lib.h" -static inline fixuint_t __fixuint(fp_t a) { +static __inline fixuint_t __fixuint(fp_t a) { // Break a into sign, exponent, significand const rep_t aRep = toRep(a); const rep_t aAbs = aRep & absMask; @@ -27,7 +27,7 @@ static inline fixuint_t __fixuint(fp_t a) { return 0; // If the value is too large for the integer type, saturate. - if ((unsigned)exponent > sizeof(fixuint_t) * CHAR_BIT) + if ((unsigned)exponent >= sizeof(fixuint_t) * CHAR_BIT) return ~(fixuint_t)0; // If 0 <= exponent < significandBits, right shift to get the result. diff --git a/lib/builtins/fp_lib.h b/lib/builtins/fp_lib.h index faebb99ecd5e..223fb980aaed 100644 --- a/lib/builtins/fp_lib.h +++ b/lib/builtins/fp_lib.h @@ -46,12 +46,12 @@ typedef float fp_t; #define REP_C UINT32_C #define significandBits 23 -static inline int rep_clz(rep_t a) { +static __inline int rep_clz(rep_t a) { return __builtin_clz(a); } // 32x32 --> 64 bit multiply -static inline void wideMultiply(rep_t a, rep_t b, rep_t *hi, rep_t *lo) { +static __inline void wideMultiply(rep_t a, rep_t b, rep_t *hi, rep_t *lo) { const uint64_t product = (uint64_t)a*b; *hi = product >> 32; *lo = product; @@ -66,7 +66,7 @@ typedef double fp_t; #define REP_C UINT64_C #define significandBits 52 -static inline int rep_clz(rep_t a) { +static __inline int rep_clz(rep_t a) { #if defined __LP64__ return __builtin_clzl(a); #else @@ -83,7 +83,7 @@ static inline int rep_clz(rep_t a) { // 64x64 -> 128 wide multiply for platforms that don't have such an operation; // many 64-bit platforms have this operation, but they tend to have hardware // floating-point, so we don't bother with a special case for them here. -static inline void wideMultiply(rep_t a, rep_t b, rep_t *hi, rep_t *lo) { +static __inline void wideMultiply(rep_t a, rep_t b, rep_t *hi, rep_t *lo) { // Each of the component 32x32 -> 64 products const uint64_t plolo = loWord(a) * loWord(b); const uint64_t plohi = loWord(a) * hiWord(b); @@ -112,7 +112,7 @@ typedef long double fp_t; // 128-bit integer, we let the constant be casted to 128-bit integer #define significandBits 112 -static inline int rep_clz(rep_t a) { +static __inline int rep_clz(rep_t a) { const union { __uint128_t ll; @@ -148,7 +148,7 @@ static inline int rep_clz(rep_t a) { // 128x128 -> 256 wide multiply for platforms that don't have such an operation; // many 64-bit platforms have this operation, but they tend to have hardware // floating-point, so we don't bother with a special case for them here. -static inline void wideMultiply(rep_t a, rep_t b, rep_t *hi, rep_t *lo) { +static __inline void wideMultiply(rep_t a, rep_t b, rep_t *hi, rep_t *lo) { const uint64_t product11 = Word_1(a) * Word_1(b); const uint64_t product12 = Word_1(a) * Word_2(b); @@ -228,28 +228,28 @@ static inline void wideMultiply(rep_t a, rep_t b, rep_t *hi, rep_t *lo) { #define quietBit (implicitBit >> 1) #define qnanRep (exponentMask | quietBit) -static inline rep_t toRep(fp_t x) { +static __inline rep_t toRep(fp_t x) { const union { fp_t f; rep_t i; } rep = {.f = x}; return rep.i; } -static inline fp_t fromRep(rep_t x) { +static __inline fp_t fromRep(rep_t x) { const union { fp_t f; rep_t i; } rep = {.i = x}; return rep.f; } -static inline int normalize(rep_t *significand) { +static __inline int normalize(rep_t *significand) { const int shift = rep_clz(*significand) - rep_clz(implicitBit); *significand <<= shift; return 1 - shift; } -static inline void wideLeftShift(rep_t *hi, rep_t *lo, int count) { +static __inline void wideLeftShift(rep_t *hi, rep_t *lo, int count) { *hi = *hi << count | *lo >> (typeWidth - count); *lo = *lo << count; } -static inline void wideRightShiftWithSticky(rep_t *hi, rep_t *lo, unsigned int count) { +static __inline void wideRightShiftWithSticky(rep_t *hi, rep_t *lo, unsigned int count) { if (count < typeWidth) { const bool sticky = *lo << (typeWidth - count); *lo = *hi << (typeWidth - count) | *lo >> count | sticky; diff --git a/lib/builtins/fp_mul_impl.inc b/lib/builtins/fp_mul_impl.inc index ca8a0bb98b10..b34aa1b8f544 100644 --- a/lib/builtins/fp_mul_impl.inc +++ b/lib/builtins/fp_mul_impl.inc @@ -14,7 +14,7 @@ #include "fp_lib.h" -static inline fp_t __mulXf3__(fp_t a, fp_t b) { +static __inline fp_t __mulXf3__(fp_t a, fp_t b) { const unsigned int aExponent = toRep(a) >> significandBits & maxExponent; const unsigned int bExponent = toRep(b) >> significandBits & maxExponent; const rep_t productSign = (toRep(a) ^ toRep(b)) & signBit; diff --git a/lib/builtins/fp_trunc.h b/lib/builtins/fp_trunc.h index 373ba1b0411d..d5e79bb5b863 100644 --- a/lib/builtins/fp_trunc.h +++ b/lib/builtins/fp_trunc.h @@ -63,12 +63,12 @@ static const int dstSigBits = 10; // End of specialization parameters. Two helper routines for conversion to and // from the representation of floating-point data as integer values follow. -static inline src_rep_t srcToRep(src_t x) { +static __inline src_rep_t srcToRep(src_t x) { const union { src_t f; src_rep_t i; } rep = {.f = x}; return rep.i; } -static inline dst_t dstFromRep(dst_rep_t x) { +static __inline dst_t dstFromRep(dst_rep_t x) { const union { dst_t f; dst_rep_t i; } rep = {.i = x}; return rep.f; } diff --git a/lib/builtins/fp_trunc_impl.inc b/lib/builtins/fp_trunc_impl.inc index 372e8d6014dd..d88ae060913f 100644 --- a/lib/builtins/fp_trunc_impl.inc +++ b/lib/builtins/fp_trunc_impl.inc @@ -39,7 +39,7 @@ #include "fp_trunc.h" -static inline dst_t __truncXfYf2__(src_t a) { +static __inline dst_t __truncXfYf2__(src_t a) { // Various constants whose values follow from the type parameters. // Any reasonable optimizer will fold and propagate all of these. const int srcBits = sizeof(src_t)*CHAR_BIT; diff --git a/lib/builtins/gcc_personality_v0.c b/lib/builtins/gcc_personality_v0.c index 4b95cfd43b05..ed544d30b809 100644 --- a/lib/builtins/gcc_personality_v0.c +++ b/lib/builtins/gcc_personality_v0.c @@ -141,7 +141,8 @@ static uintptr_t readEncodedPointer(const uint8_t** data, uint8_t encoding) * throw through a C function compiled with -fexceptions. */ #if __USING_SJLJ_EXCEPTIONS__ -// the setjump-longjump based exceptions personality routine has a different name +/* the setjump-longjump based exceptions personality routine has a + * different name */ COMPILER_RT_ABI _Unwind_Reason_Code __gcc_personality_sj0(int version, _Unwind_Action actions, uint64_t exceptionClass, struct _Unwind_Exception* exceptionObject, @@ -194,15 +195,15 @@ __gcc_personality_v0(int version, _Unwind_Action actions, * Set Instruction Pointer to so we re-enter function * at landing pad. The landing pad is created by the compiler * to take two parameters in registers. - */ - _Unwind_SetGR(context, __builtin_eh_return_data_regno(0), - (uintptr_t)exceptionObject); + */ + _Unwind_SetGR(context, __builtin_eh_return_data_regno(0), + (uintptr_t)exceptionObject); _Unwind_SetGR(context, __builtin_eh_return_data_regno(1), 0); - _Unwind_SetIP(context, funcStart+landingPad); + _Unwind_SetIP(context, (funcStart + landingPad)); return _URC_INSTALL_CONTEXT; } } - + /* No landing pad found, continue unwinding. */ return _URC_CONTINUE_UNWIND; } diff --git a/lib/builtins/i386/chkstk.S b/lib/builtins/i386/chkstk.S index 3733d722ef19..b59974868f21 100644 --- a/lib/builtins/i386/chkstk.S +++ b/lib/builtins/i386/chkstk.S @@ -19,13 +19,13 @@ DEFINE_COMPILERRT_FUNCTION(__chkstk_ms) jb 1f 2: sub $0x1000,%ecx - orl $0,(%ecx) + test %ecx,(%ecx) sub $0x1000,%eax cmp $0x1000,%eax ja 2b 1: sub %eax,%ecx - orl $0,(%ecx) + test %ecx,(%ecx) pop %eax pop %ecx ret diff --git a/lib/builtins/i386/chkstk2.S b/lib/builtins/i386/chkstk2.S new file mode 100644 index 000000000000..7d65bb088928 --- /dev/null +++ b/lib/builtins/i386/chkstk2.S @@ -0,0 +1,40 @@ +// This file is dual licensed under the MIT and the University of Illinois Open +// Source Licenses. See LICENSE.TXT for details. + +#include "../assembly.h" + +#ifdef __i386__ + +// _chkstk (_alloca) routine - probe stack between %esp and (%esp-%eax) in 4k increments, +// then decrement %esp by %eax. Preserves all registers except %esp and flags. +// This routine is windows specific +// http://msdn.microsoft.com/en-us/library/ms648426.aspx + +.text +.balign 4 +DEFINE_COMPILERRT_FUNCTION(_alloca) // _chkstk and _alloca are the same function +DEFINE_COMPILERRT_FUNCTION(__chkstk) + push %ecx + cmp $0x1000,%eax + lea 8(%esp),%ecx // esp before calling this routine -> ecx + jb 1f +2: + sub $0x1000,%ecx + test %ecx,(%ecx) + sub $0x1000,%eax + cmp $0x1000,%eax + ja 2b +1: + sub %eax,%ecx + test %ecx,(%ecx) + + lea 4(%esp),%eax // load pointer to the return address into eax + mov %ecx,%esp // install the new top of stack pointer into esp + mov -4(%eax),%ecx // restore ecx + push (%eax) // push return address onto the stack + sub %esp,%eax // restore the original value in eax + ret +END_COMPILERRT_FUNCTION(__chkstk) +END_COMPILERRT_FUNCTION(_alloca) + +#endif // __i386__ diff --git a/lib/builtins/int_lib.h b/lib/builtins/int_lib.h index bca5d81d4414..e66cda3fffb4 100644 --- a/lib/builtins/int_lib.h +++ b/lib/builtins/int_lib.h @@ -20,6 +20,13 @@ /* Assumption: Right shift of signed negative is arithmetic shift. */ /* Assumption: Endianness is little or big (not mixed). */ +#if defined(__ELF__) +#define FNALIAS(alias_name, original_name) \ + void alias_name() __attribute__((alias(#original_name))) +#else +#define FNALIAS(alias, name) _Pragma("GCC error(\"alias unsupported on this file format\")") +#endif + /* ABI macro definitions */ #if __ARM_EABI__ @@ -28,13 +35,25 @@ # define COMPILER_RT_ABI __attribute__((pcs("aapcs"))) #else # define ARM_EABI_FNALIAS(aeabi_name, name) -# if defined(__arm__) && defined(_WIN32) +# if defined(__arm__) && defined(_WIN32) && (!defined(_MSC_VER) || defined(__clang__)) # define COMPILER_RT_ABI __attribute__((pcs("aapcs"))) # else # define COMPILER_RT_ABI # endif #endif +#ifdef _MSC_VER +#define ALWAYS_INLINE __forceinline +#define NOINLINE __declspec(noinline) +#define NORETURN __declspec(noreturn) +#define UNUSED +#else +#define ALWAYS_INLINE __attribute__((always_inline)) +#define NOINLINE __attribute__((noinline)) +#define NORETURN __attribute__((noreturn)) +#define UNUSED __attribute__((unused)) +#endif + #if defined(__NetBSD__) && (defined(_KERNEL) || defined(_STANDALONE)) /* * Kernel and boot environment can't use normal headers, @@ -71,4 +90,44 @@ COMPILER_RT_ABI si_int __clzti2(ti_int a); COMPILER_RT_ABI tu_int __udivmodti4(tu_int a, tu_int b, tu_int* rem); #endif +/* Definitions for builtins unavailable on MSVC */ +#if defined(_MSC_VER) && !defined(__clang__) +#include <intrin.h> + +uint32_t __inline __builtin_ctz(uint32_t value) { + uint32_t trailing_zero = 0; + if (_BitScanForward(&trailing_zero, value)) + return trailing_zero; + return 32; +} + +uint32_t __inline __builtin_clz(uint32_t value) { + uint32_t leading_zero = 0; + if (_BitScanReverse(&leading_zero, value)) + return 31 - leading_zero; + return 32; +} + +#if defined(_M_ARM) || defined(_M_X64) +uint32_t __inline __builtin_clzll(uint64_t value) { + uint32_t leading_zero = 0; + if (_BitScanReverse64(&leading_zero, value)) + return 63 - leading_zero; + return 64; +} +#else +uint32_t __inline __builtin_clzll(uint64_t value) { + if (value == 0) + return 64; + uint32_t msh = (uint32_t)(value >> 32); + uint32_t lsh = (uint32_t)(value & 0xFFFFFFFF); + if (msh != 0) + return __builtin_clz(msh); + return 32 + __builtin_clz(lsh); +} +#endif + +#define __builtin_clzl __builtin_clzll +#endif /* defined(_MSC_VER) && !defined(__clang__) */ + #endif /* INT_LIB_H */ diff --git a/lib/builtins/int_math.h b/lib/builtins/int_math.h index d6b4bdae162b..fc81fb7f0220 100644 --- a/lib/builtins/int_math.h +++ b/lib/builtins/int_math.h @@ -25,43 +25,90 @@ # define __has_builtin(x) 0 #endif -#define CRT_INFINITY __builtin_huge_valf() +#if defined(_MSC_VER) && !defined(__clang__) +#include <math.h> +#include <stdlib.h> +#include <ymath.h> +#endif -#define crt_isinf(x) __builtin_isinf((x)) -#define crt_isnan(x) __builtin_isnan((x)) +#if defined(_MSC_VER) && !defined(__clang__) +#define CRT_INFINITY INFINITY +#else +#define CRT_INFINITY __builtin_huge_valf() +#endif +#if defined(_MSC_VER) && !defined(__clang__) +#define crt_isfinite(x) _finite((x)) +#define crt_isinf(x) !_finite((x)) +#define crt_isnan(x) _isnan((x)) +#else /* Define crt_isfinite in terms of the builtin if available, otherwise provide * an alternate version in terms of our other functions. This supports some * versions of GCC which didn't have __builtin_isfinite. */ #if __has_builtin(__builtin_isfinite) # define crt_isfinite(x) __builtin_isfinite((x)) -#else +#elif defined(__GNUC__) # define crt_isfinite(x) \ __extension__(({ \ __typeof((x)) x_ = (x); \ !crt_isinf(x_) && !crt_isnan(x_); \ })) -#endif +#else +# error "Do not know how to check for infinity" +#endif /* __has_builtin(__builtin_isfinite) */ +#define crt_isinf(x) __builtin_isinf((x)) +#define crt_isnan(x) __builtin_isnan((x)) +#endif /* _MSC_VER */ +#if defined(_MSC_VER) && !defined(__clang__) +#define crt_copysign(x, y) copysign((x), (y)) +#define crt_copysignf(x, y) copysignf((x), (y)) +#define crt_copysignl(x, y) copysignl((x), (y)) +#else #define crt_copysign(x, y) __builtin_copysign((x), (y)) #define crt_copysignf(x, y) __builtin_copysignf((x), (y)) #define crt_copysignl(x, y) __builtin_copysignl((x), (y)) +#endif +#if defined(_MSC_VER) && !defined(__clang__) +#define crt_fabs(x) fabs((x)) +#define crt_fabsf(x) fabsf((x)) +#define crt_fabsl(x) fabs((x)) +#else #define crt_fabs(x) __builtin_fabs((x)) #define crt_fabsf(x) __builtin_fabsf((x)) #define crt_fabsl(x) __builtin_fabsl((x)) +#endif +#if defined(_MSC_VER) && !defined(__clang__) +#define crt_fmax(x, y) __max((x), (y)) +#define crt_fmaxf(x, y) __max((x), (y)) +#define crt_fmaxl(x, y) __max((x), (y)) +#else #define crt_fmax(x, y) __builtin_fmax((x), (y)) #define crt_fmaxf(x, y) __builtin_fmaxf((x), (y)) #define crt_fmaxl(x, y) __builtin_fmaxl((x), (y)) +#endif +#if defined(_MSC_VER) && !defined(__clang__) +#define crt_logb(x) logb((x)) +#define crt_logbf(x) logbf((x)) +#define crt_logbl(x) logbl((x)) +#else #define crt_logb(x) __builtin_logb((x)) #define crt_logbf(x) __builtin_logbf((x)) #define crt_logbl(x) __builtin_logbl((x)) +#endif +#if defined(_MSC_VER) && !defined(__clang__) +#define crt_scalbn(x, y) scalbn((x), (y)) +#define crt_scalbnf(x, y) scalbnf((x), (y)) +#define crt_scalbnl(x, y) scalbnl((x), (y)) +#else #define crt_scalbn(x, y) __builtin_scalbn((x), (y)) #define crt_scalbnf(x, y) __builtin_scalbnf((x), (y)) #define crt_scalbnl(x, y) __builtin_scalbnl((x), (y)) +#endif #endif /* INT_MATH_H */ diff --git a/lib/builtins/int_types.h b/lib/builtins/int_types.h index aedae14b2046..2dad43bc7389 100644 --- a/lib/builtins/int_types.h +++ b/lib/builtins/int_types.h @@ -20,6 +20,10 @@ #include "int_endianness.h" +/* si_int is defined in Linux sysroot's asm-generic/siginfo.h */ +#ifdef si_int +#undef si_int +#endif typedef int si_int; typedef unsigned su_int; @@ -95,14 +99,14 @@ typedef union }s; } utwords; -static inline ti_int make_ti(di_int h, di_int l) { +static __inline ti_int make_ti(di_int h, di_int l) { twords r; r.s.high = h; r.s.low = l; return r.all; } -static inline tu_int make_tu(du_int h, du_int l) { +static __inline tu_int make_tu(du_int h, du_int l) { utwords r; r.s.high = h; r.s.low = l; @@ -140,5 +144,22 @@ typedef union long double f; } long_double_bits; +#if __STDC_VERSION__ >= 199901L +typedef float _Complex Fcomplex; +typedef double _Complex Dcomplex; +typedef long double _Complex Lcomplex; + +#define COMPLEX_REAL(x) __real__(x) +#define COMPLEX_IMAGINARY(x) __imag__(x) +#else +typedef struct { float real, imaginary; } Fcomplex; + +typedef struct { double real, imaginary; } Dcomplex; + +typedef struct { long double real, imaginary; } Lcomplex; + +#define COMPLEX_REAL(x) (x).real +#define COMPLEX_IMAGINARY(x) (x).imaginary +#endif #endif /* INT_TYPES_H */ diff --git a/lib/builtins/int_util.c b/lib/builtins/int_util.c index 323e46179e6c..420d1e237aae 100644 --- a/lib/builtins/int_util.c +++ b/lib/builtins/int_util.c @@ -8,8 +8,8 @@ * ===----------------------------------------------------------------------=== */ -#include "int_util.h" #include "int_lib.h" +#include "int_util.h" /* NOTE: The definitions in this file are declared weak because we clients to be * able to arbitrarily package individual functions into separate .a files. If @@ -23,7 +23,7 @@ #ifdef KERNEL_USE -extern void panic(const char *, ...) __attribute__((noreturn)); +NORETURN extern void panic(const char *, ...); #ifndef _WIN32 __attribute__((visibility("hidden"))) #endif @@ -34,8 +34,8 @@ void compilerrt_abort_impl(const char *file, int line, const char *function) { #elif __APPLE__ /* from libSystem.dylib */ -extern void __assert_rtn(const char *func, const char *file, - int line, const char * message) __attribute__((noreturn)); +NORETURN extern void __assert_rtn(const char *func, const char *file, int line, + const char *message); #ifndef _WIN32 __attribute__((weak)) diff --git a/lib/builtins/int_util.h b/lib/builtins/int_util.h index a9b595db8d0f..a7b20ed66244 100644 --- a/lib/builtins/int_util.h +++ b/lib/builtins/int_util.h @@ -20,10 +20,14 @@ #define INT_UTIL_H /** \brief Trigger a program abort (or panic for kernel code). */ -#define compilerrt_abort() compilerrt_abort_impl(__FILE__, __LINE__, \ - __func__) +#define compilerrt_abort() compilerrt_abort_impl(__FILE__, __LINE__, __func__) -void compilerrt_abort_impl(const char *file, int line, - const char *function) __attribute__((noreturn)); +NORETURN void compilerrt_abort_impl(const char *file, int line, + const char *function); + +#define COMPILE_TIME_ASSERT(expr) COMPILE_TIME_ASSERT1(expr, __COUNTER__) +#define COMPILE_TIME_ASSERT1(expr, cnt) COMPILE_TIME_ASSERT2(expr, cnt) +#define COMPILE_TIME_ASSERT2(expr, cnt) \ + typedef char ct_assert_##cnt[(expr) ? 1 : -1] UNUSED #endif /* INT_UTIL_H */ diff --git a/lib/builtins/macho_embedded/CMakeLists.txt b/lib/builtins/macho_embedded/CMakeLists.txt new file mode 100644 index 000000000000..266e42215243 --- /dev/null +++ b/lib/builtins/macho_embedded/CMakeLists.txt @@ -0,0 +1,4 @@ +file(GLOB filter_files ${CMAKE_CURRENT_SOURCE_DIR}/*.txt) +foreach(filter_file ${filter_files}) + set_property(DIRECTORY APPEND PROPERTY CMAKE_CONFIGURE_DEPENDS ${filter_file}) +endforeach() diff --git a/lib/builtins/macho_embedded/arm.txt b/lib/builtins/macho_embedded/arm.txt new file mode 100644 index 000000000000..4b1683a6baef --- /dev/null +++ b/lib/builtins/macho_embedded/arm.txt @@ -0,0 +1,16 @@ +aeabi_cdcmpeq +aeabi_cdrcmple +aeabi_cfcmpeq +aeabi_cfrcmple +aeabi_dcmpeq +aeabi_dcmpge +aeabi_dcmpgt +aeabi_dcmple +aeabi_dcmplt +aeabi_drsub +aeabi_fcmpeq +aeabi_fcmpge +aeabi_fcmpgt +aeabi_fcmple +aeabi_fcmplt +aeabi_frsub diff --git a/lib/builtins/macho_embedded/common.txt b/lib/builtins/macho_embedded/common.txt new file mode 100644 index 000000000000..6ac85a771fcb --- /dev/null +++ b/lib/builtins/macho_embedded/common.txt @@ -0,0 +1,92 @@ +absvdi2 +absvsi2 +addvdi3 +addvsi3 +ashldi3 +ashrdi3 +clzdi2 +clzsi2 +cmpdi2 +ctzdi2 +ctzsi2 +divdc3 +divdi3 +divsc3 +divmodsi4 +udivmodsi4 +do_global_dtors +ffsdi2 +fixdfdi +fixsfdi +fixunsdfdi +fixunsdfsi +fixunssfdi +fixunssfsi +floatdidf +floatdisf +floatundidf +floatundisf +gcc_bcmp +lshrdi3 +moddi3 +muldc3 +muldi3 +mulsc3 +mulvdi3 +mulvsi3 +negdi2 +negvdi2 +negvsi2 +paritydi2 +paritysi2 +popcountdi2 +popcountsi2 +powidf2 +powisf2 +subvdi3 +subvsi3 +ucmpdi2 +udiv_w_sdiv +udivdi3 +udivmoddi4 +umoddi3 +adddf3 +addsf3 +cmpdf2 +cmpsf2 +div0 +divdf3 +divsf3 +divsi3 +extendsfdf2 +extendhfsf2 +ffssi2 +fixdfsi +fixsfsi +floatsidf +floatsisf +floatunsidf +floatunsisf +comparedf2 +comparesf2 +modsi3 +muldf3 +mulsf3 +negdf2 +negsf2 +subdf3 +subsf3 +truncdfhf2 +truncdfsf2 +truncsfhf2 +udivsi3 +umodsi3 +unorddf2 +unordsf2 +atomic_flag_clear +atomic_flag_clear_explicit +atomic_flag_test_and_set +atomic_flag_test_and_set_explicit +atomic_signal_fence +atomic_thread_fence +int_util diff --git a/lib/builtins/macho_embedded/i386.txt b/lib/builtins/macho_embedded/i386.txt new file mode 100644 index 000000000000..b92e44bb35ae --- /dev/null +++ b/lib/builtins/macho_embedded/i386.txt @@ -0,0 +1,7 @@ +i686.get_pc_thunk.eax +i686.get_pc_thunk.ebp +i686.get_pc_thunk.ebx +i686.get_pc_thunk.ecx +i686.get_pc_thunk.edi +i686.get_pc_thunk.edx +i686.get_pc_thunk.esi diff --git a/lib/builtins/macho_embedded/thumb2-64.txt b/lib/builtins/macho_embedded/thumb2-64.txt new file mode 100644 index 000000000000..1c72fb1c3c64 --- /dev/null +++ b/lib/builtins/macho_embedded/thumb2-64.txt @@ -0,0 +1,10 @@ +sync_fetch_and_add_8 +sync_fetch_and_sub_8 +sync_fetch_and_and_8 +sync_fetch_and_or_8 +sync_fetch_and_xor_8 +sync_fetch_and_nand_8 +sync_fetch_and_max_8 +sync_fetch_and_umax_8 +sync_fetch_and_min_8 +sync_fetch_and_umin_8 diff --git a/lib/builtins/macho_embedded/thumb2.txt b/lib/builtins/macho_embedded/thumb2.txt new file mode 100644 index 000000000000..6add5ecd2dc7 --- /dev/null +++ b/lib/builtins/macho_embedded/thumb2.txt @@ -0,0 +1,14 @@ +switch16 +switch32 +switch8 +switchu8 +sync_fetch_and_add_4 +sync_fetch_and_sub_4 +sync_fetch_and_and_4 +sync_fetch_and_or_4 +sync_fetch_and_xor_4 +sync_fetch_and_nand_4 +sync_fetch_and_max_4 +sync_fetch_and_umax_4 +sync_fetch_and_min_4 +sync_fetch_and_umin_4 diff --git a/lib/builtins/muldc3.c b/lib/builtins/muldc3.c index 3bfae2c52224..16d8e98390a3 100644 --- a/lib/builtins/muldc3.c +++ b/lib/builtins/muldc3.c @@ -17,17 +17,17 @@ /* Returns: the product of a + ib and c + id */ -COMPILER_RT_ABI double _Complex +COMPILER_RT_ABI Dcomplex __muldc3(double __a, double __b, double __c, double __d) { double __ac = __a * __c; double __bd = __b * __d; double __ad = __a * __d; double __bc = __b * __c; - double _Complex z; - __real__ z = __ac - __bd; - __imag__ z = __ad + __bc; - if (crt_isnan(__real__ z) && crt_isnan(__imag__ z)) + Dcomplex z; + COMPLEX_REAL(z) = __ac - __bd; + COMPLEX_IMAGINARY(z) = __ad + __bc; + if (crt_isnan(COMPLEX_REAL(z)) && crt_isnan(COMPLEX_IMAGINARY(z))) { int __recalc = 0; if (crt_isinf(__a) || crt_isinf(__b)) @@ -65,8 +65,8 @@ __muldc3(double __a, double __b, double __c, double __d) } if (__recalc) { - __real__ z = CRT_INFINITY * (__a * __c - __b * __d); - __imag__ z = CRT_INFINITY * (__a * __d + __b * __c); + COMPLEX_REAL(z) = CRT_INFINITY * (__a * __c - __b * __d); + COMPLEX_IMAGINARY(z) = CRT_INFINITY * (__a * __d + __b * __c); } } return z; diff --git a/lib/builtins/mulsc3.c b/lib/builtins/mulsc3.c index 29d46c63a799..c89cfd247a15 100644 --- a/lib/builtins/mulsc3.c +++ b/lib/builtins/mulsc3.c @@ -17,17 +17,17 @@ /* Returns: the product of a + ib and c + id */ -COMPILER_RT_ABI float _Complex +COMPILER_RT_ABI Fcomplex __mulsc3(float __a, float __b, float __c, float __d) { float __ac = __a * __c; float __bd = __b * __d; float __ad = __a * __d; float __bc = __b * __c; - float _Complex z; - __real__ z = __ac - __bd; - __imag__ z = __ad + __bc; - if (crt_isnan(__real__ z) && crt_isnan(__imag__ z)) + Fcomplex z; + COMPLEX_REAL(z) = __ac - __bd; + COMPLEX_IMAGINARY(z) = __ad + __bc; + if (crt_isnan(COMPLEX_REAL(z)) && crt_isnan(COMPLEX_IMAGINARY(z))) { int __recalc = 0; if (crt_isinf(__a) || crt_isinf(__b)) @@ -65,8 +65,8 @@ __mulsc3(float __a, float __b, float __c, float __d) } if (__recalc) { - __real__ z = CRT_INFINITY * (__a * __c - __b * __d); - __imag__ z = CRT_INFINITY * (__a * __d + __b * __c); + COMPLEX_REAL(z) = CRT_INFINITY * (__a * __c - __b * __d); + COMPLEX_IMAGINARY(z) = CRT_INFINITY * (__a * __d + __b * __c); } } return z; diff --git a/lib/builtins/multc3.c b/lib/builtins/multc3.c new file mode 100644 index 000000000000..0518bc2569f1 --- /dev/null +++ b/lib/builtins/multc3.c @@ -0,0 +1,68 @@ +/* ===-- multc3.c - Implement __multc3 -------------------------------------=== + * + * The LLVM Compiler Infrastructure + * + * This file is dual licensed under the MIT and the University of Illinois Open + * Source Licenses. See LICENSE.TXT for details. + * + * ===----------------------------------------------------------------------=== + * + * This file implements __multc3 for the compiler_rt library. + * + * ===----------------------------------------------------------------------=== + */ + +#include "int_lib.h" +#include "int_math.h" + +/* Returns: the product of a + ib and c + id */ + +COMPILER_RT_ABI long double _Complex +__multc3(long double a, long double b, long double c, long double d) +{ + long double ac = a * c; + long double bd = b * d; + long double ad = a * d; + long double bc = b * c; + long double _Complex z; + __real__ z = ac - bd; + __imag__ z = ad + bc; + if (crt_isnan(__real__ z) && crt_isnan(__imag__ z)) { + int recalc = 0; + if (crt_isinf(a) || crt_isinf(b)) { + a = crt_copysignl(crt_isinf(a) ? 1 : 0, a); + b = crt_copysignl(crt_isinf(b) ? 1 : 0, b); + if (crt_isnan(c)) + c = crt_copysignl(0, c); + if (crt_isnan(d)) + d = crt_copysignl(0, d); + recalc = 1; + } + if (crt_isinf(c) || crt_isinf(d)) { + c = crt_copysignl(crt_isinf(c) ? 1 : 0, c); + d = crt_copysignl(crt_isinf(d) ? 1 : 0, d); + if (crt_isnan(a)) + a = crt_copysignl(0, a); + if (crt_isnan(b)) + b = crt_copysignl(0, b); + recalc = 1; + } + if (!recalc && (crt_isinf(ac) || crt_isinf(bd) || + crt_isinf(ad) || crt_isinf(bc))) { + if (crt_isnan(a)) + a = crt_copysignl(0, a); + if (crt_isnan(b)) + b = crt_copysignl(0, b); + if (crt_isnan(c)) + c = crt_copysignl(0, c); + if (crt_isnan(d)) + d = crt_copysignl(0, d); + recalc = 1; + } + if (recalc) { + __real__ z = CRT_INFINITY * (a * c - b * d); + __imag__ z = CRT_INFINITY * (a * d + b * c); + } + } + return z; +} diff --git a/lib/builtins/mulxc3.c b/lib/builtins/mulxc3.c index 161fd0ce0dd4..ba3221691821 100644 --- a/lib/builtins/mulxc3.c +++ b/lib/builtins/mulxc3.c @@ -19,17 +19,17 @@ /* Returns: the product of a + ib and c + id */ -COMPILER_RT_ABI long double _Complex +COMPILER_RT_ABI Lcomplex __mulxc3(long double __a, long double __b, long double __c, long double __d) { long double __ac = __a * __c; long double __bd = __b * __d; long double __ad = __a * __d; long double __bc = __b * __c; - long double _Complex z; - __real__ z = __ac - __bd; - __imag__ z = __ad + __bc; - if (crt_isnan(__real__ z) && crt_isnan(__imag__ z)) + Lcomplex z; + COMPLEX_REAL(z) = __ac - __bd; + COMPLEX_IMAGINARY(z) = __ad + __bc; + if (crt_isnan(COMPLEX_REAL(z)) && crt_isnan(COMPLEX_IMAGINARY(z))) { int __recalc = 0; if (crt_isinf(__a) || crt_isinf(__b)) @@ -67,8 +67,8 @@ __mulxc3(long double __a, long double __b, long double __c, long double __d) } if (__recalc) { - __real__ z = CRT_INFINITY * (__a * __c - __b * __d); - __imag__ z = CRT_INFINITY * (__a * __d + __b * __c); + COMPLEX_REAL(z) = CRT_INFINITY * (__a * __c - __b * __d); + COMPLEX_IMAGINARY(z) = CRT_INFINITY * (__a * __d + __b * __c); } } return z; diff --git a/lib/builtins/ppc/DD.h b/lib/builtins/ppc/DD.h index fc3e41cbe07e..3e5f9e58c138 100644 --- a/lib/builtins/ppc/DD.h +++ b/lib/builtins/ppc/DD.h @@ -1,5 +1,5 @@ -#ifndef __DD_HEADER -#define __DD_HEADER +#ifndef COMPILERRT_DD_HEADER +#define COMPILERRT_DD_HEADER #include "../int_lib.h" @@ -9,7 +9,7 @@ typedef union { double hi; double lo; }s; -}DD; +} DD; typedef union { double d; @@ -19,28 +19,27 @@ typedef union { #define LOWORDER(xy,xHi,xLo,yHi,yLo) \ (((((xHi)*(yHi) - (xy)) + (xHi)*(yLo)) + (xLo)*(yHi)) + (xLo)*(yLo)) -static inline double __attribute__((always_inline)) -local_fabs(double x) -{ - doublebits result = { .d = x }; - result.x &= UINT64_C(0x7fffffffffffffff); - return result.d; +static __inline ALWAYS_INLINE double local_fabs(double x) { + doublebits result = {.d = x}; + result.x &= UINT64_C(0x7fffffffffffffff); + return result.d; } -static inline double __attribute__((always_inline)) -high26bits(double x) -{ - doublebits result = { .d = x }; - result.x &= UINT64_C(0xfffffffff8000000); - return result.d; +static __inline ALWAYS_INLINE double high26bits(double x) { + doublebits result = {.d = x}; + result.x &= UINT64_C(0xfffffffff8000000); + return result.d; } -static inline int __attribute__((always_inline)) -different_sign(double x, double y) -{ - doublebits xsignbit = { .d = x }, ysignbit = { .d = y }; - int result = (int)(xsignbit.x >> 63) ^ (int)(ysignbit.x >> 63); - return result; +static __inline ALWAYS_INLINE int different_sign(double x, double y) { + doublebits xsignbit = {.d = x}, ysignbit = {.d = y}; + int result = (int)(xsignbit.x >> 63) ^ (int)(ysignbit.x >> 63); + return result; } -#endif /* __DD_HEADER */ +long double __gcc_qadd(long double, long double); +long double __gcc_qsub(long double, long double); +long double __gcc_qmul(long double, long double); +long double __gcc_qdiv(long double, long double); + +#endif /* COMPILERRT_DD_HEADER */ diff --git a/lib/builtins/ppc/divtc3.c b/lib/builtins/ppc/divtc3.c index 299128186312..8ec41c528ab9 100644 --- a/lib/builtins/ppc/divtc3.c +++ b/lib/builtins/ppc/divtc3.c @@ -14,11 +14,6 @@ (x).s.lo = 0.0; \ } -long double __gcc_qadd(long double, long double); -long double __gcc_qsub(long double, long double); -long double __gcc_qmul(long double, long double); -long double __gcc_qdiv(long double, long double); - long double _Complex __divtc3(long double a, long double b, long double c, long double d) { diff --git a/lib/builtins/ppc/multc3.c b/lib/builtins/ppc/multc3.c index 738b65a83b03..9dd79c975dde 100644 --- a/lib/builtins/ppc/multc3.c +++ b/lib/builtins/ppc/multc3.c @@ -17,10 +17,6 @@ } \ } -long double __gcc_qadd(long double, long double); -long double __gcc_qsub(long double, long double); -long double __gcc_qmul(long double, long double); - long double _Complex __multc3(long double a, long double b, long double c, long double d) { diff --git a/lib/builtins/subdf3.c b/lib/builtins/subdf3.c index 089e062415b7..7a79e5e7765d 100644 --- a/lib/builtins/subdf3.c +++ b/lib/builtins/subdf3.c @@ -23,4 +23,3 @@ __subdf3(fp_t a, fp_t b) { return __adddf3(a, fromRep(toRep(b) ^ signBit)); } -/* FIXME: rsub for ARM EABI */ diff --git a/lib/builtins/subsf3.c b/lib/builtins/subsf3.c index 47f5e5e46ea8..c3b85144af48 100644 --- a/lib/builtins/subsf3.c +++ b/lib/builtins/subsf3.c @@ -23,4 +23,3 @@ __subsf3(fp_t a, fp_t b) { return __addsf3(a, fromRep(toRep(b) ^ signBit)); } -/* FIXME: rsub for ARM EABI */ diff --git a/lib/builtins/truncdfhf2.c b/lib/builtins/truncdfhf2.c index 0852df369625..17195cd9e799 100644 --- a/lib/builtins/truncdfhf2.c +++ b/lib/builtins/truncdfhf2.c @@ -11,6 +11,8 @@ #define DST_HALF #include "fp_trunc_impl.inc" +ARM_EABI_FNALIAS(d2h, truncdfhf2) + COMPILER_RT_ABI uint16_t __truncdfhf2(double a) { return __truncXfYf2__(a); } diff --git a/lib/builtins/truncsfhf2.c b/lib/builtins/truncsfhf2.c index 381e590c342f..9d61895bfd88 100644 --- a/lib/builtins/truncsfhf2.c +++ b/lib/builtins/truncsfhf2.c @@ -11,9 +11,11 @@ #define DST_HALF #include "fp_trunc_impl.inc" +ARM_EABI_FNALIAS(f2h, truncsfhf2) + // Use a forwarding definition and noinline to implement a poor man's alias, // as there isn't a good cross-platform way of defining one. -COMPILER_RT_ABI __attribute__((noinline)) uint16_t __truncsfhf2(float a) { +COMPILER_RT_ABI NOINLINE uint16_t __truncsfhf2(float a) { return __truncXfYf2__(a); } diff --git a/lib/builtins/x86_64/chkstk.S b/lib/builtins/x86_64/chkstk.S index 5759e84498c6..4149ac63d9d0 100644 --- a/lib/builtins/x86_64/chkstk.S +++ b/lib/builtins/x86_64/chkstk.S @@ -24,13 +24,13 @@ DEFINE_COMPILERRT_FUNCTION(___chkstk_ms) jb 1f 2: sub $0x1000,%rcx - orl $0,(%rcx) + test %rcx,(%rcx) sub $0x1000,%rax cmp $0x1000,%rax ja 2b 1: sub %rax,%rcx - orl $0,(%rcx) + test %rcx,(%rcx) pop %rax pop %rcx ret diff --git a/lib/builtins/x86_64/chkstk2.S b/lib/builtins/x86_64/chkstk2.S new file mode 100644 index 000000000000..ac1eb920e0e8 --- /dev/null +++ b/lib/builtins/x86_64/chkstk2.S @@ -0,0 +1,42 @@ +// This file is dual licensed under the MIT and the University of Illinois Open +// Source Licenses. See LICENSE.TXT for details. + +#include "../assembly.h" + +#ifdef __x86_64__ + +// _chkstk (_alloca) routine - probe stack between %rsp and (%rsp-%rax) in 4k increments, +// then decrement %rsp by %rax. Preserves all registers except %rsp and flags. +// This routine is windows specific +// http://msdn.microsoft.com/en-us/library/ms648426.aspx + +.text +.balign 4 +DEFINE_COMPILERRT_FUNCTION(__alloca) + mov %rcx,%rax // x64 _alloca is a normal function with parameter in rcx + // fallthrough +DEFINE_COMPILERRT_FUNCTION(___chkstk) + push %rcx + cmp $0x1000,%rax + lea 16(%rsp),%rcx // rsp before calling this routine -> rcx + jb 1f +2: + sub $0x1000,%rcx + test %rcx,(%rcx) + sub $0x1000,%rax + cmp $0x1000,%rax + ja 2b +1: + sub %rax,%rcx + test %rcx,(%rcx) + + lea 8(%rsp),%rax // load pointer to the return address into rax + mov %rcx,%rsp // install the new top of stack pointer into rsp + mov -8(%rax),%rcx // restore rcx + push (%rax) // push return address onto the stack + sub %rsp,%rax // restore the original value in rax + ret +END_COMPILERRT_FUNCTION(___chkstk) +END_COMPILERRT_FUNCTION(__alloca) + +#endif // __x86_64__ diff --git a/lib/cfi/CMakeLists.txt b/lib/cfi/CMakeLists.txt new file mode 100644 index 000000000000..24e51814cdab --- /dev/null +++ b/lib/cfi/CMakeLists.txt @@ -0,0 +1,40 @@ +add_custom_target(cfi) + +set(CFI_SOURCES cfi.cc) + +include_directories(..) + +set(CFI_CFLAGS + ${SANITIZER_COMMON_CFLAGS} +) + +set(CFI_DIAG_CFLAGS + -DCFI_ENABLE_DIAG=1 +) + +foreach(arch ${CFI_SUPPORTED_ARCH}) + add_compiler_rt_runtime(clang_rt.cfi + STATIC + ARCHS ${arch} + SOURCES ${CFI_SOURCES} + OBJECT_LIBS RTInterception + RTSanitizerCommon + RTSanitizerCommonLibc + CFLAGS ${CFI_CFLAGS} + PARENT_TARGET cfi) + add_compiler_rt_runtime(clang_rt.cfi_diag + STATIC + ARCHS ${arch} + SOURCES ${CFI_SOURCES} + OBJECT_LIBS RTInterception + RTSanitizerCommon + RTSanitizerCommonLibc + RTUbsan + RTUbsan_cxx + CFLAGS ${CFI_CFLAGS} ${CFI_DIAG_CFLAGS} + PARENT_TARGET cfi) +endforeach() + +add_compiler_rt_resource_file(cfi_blacklist cfi_blacklist.txt) +add_dependencies(cfi cfi_blacklist) +add_dependencies(compiler-rt cfi) diff --git a/lib/cfi/cfi.cc b/lib/cfi/cfi.cc new file mode 100644 index 000000000000..0e2a09190699 --- /dev/null +++ b/lib/cfi/cfi.cc @@ -0,0 +1,271 @@ +//===-------- cfi.cc ------------------------------------------------------===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// +// +// This file implements the runtime support for the cross-DSO CFI. +// +//===----------------------------------------------------------------------===// + +// FIXME: Intercept dlopen/dlclose. +// FIXME: Support diagnostic mode. +// FIXME: Harden: +// * mprotect shadow, use mremap for updates +// * something else equally important + +#include <assert.h> +#include <elf.h> +#include <link.h> +#include <string.h> + +typedef ElfW(Phdr) Elf_Phdr; +typedef ElfW(Ehdr) Elf_Ehdr; + +#include "interception/interception.h" +#include "sanitizer_common/sanitizer_common.h" +#include "sanitizer_common/sanitizer_flag_parser.h" +#include "ubsan/ubsan_init.h" +#include "ubsan/ubsan_flags.h" + +static uptr __cfi_shadow; +static constexpr uptr kShadowGranularity = 12; +static constexpr uptr kShadowAlign = 1UL << kShadowGranularity; // 4096 + +static constexpr uint16_t kInvalidShadow = 0; +static constexpr uint16_t kUncheckedShadow = 0xFFFFU; + +static uint16_t *mem_to_shadow(uptr x) { + return (uint16_t *)(__cfi_shadow + ((x >> kShadowGranularity) << 1)); +} + +typedef int (*CFICheckFn)(uptr, void *); + +class ShadowValue { + uptr addr; + uint16_t v; + explicit ShadowValue(uptr addr, uint16_t v) : addr(addr), v(v) {} + +public: + bool is_invalid() const { return v == kInvalidShadow; } + + bool is_unchecked() const { return v == kUncheckedShadow; } + + CFICheckFn get_cfi_check() const { + assert(!is_invalid() && !is_unchecked()); + uptr aligned_addr = addr & ~(kShadowAlign - 1); + uptr p = aligned_addr - (((uptr)v - 1) << kShadowGranularity); + return reinterpret_cast<CFICheckFn>(p); + } + + // Load a shadow valud for the given application memory address. + static const ShadowValue load(uptr addr) { + return ShadowValue(addr, *mem_to_shadow(addr)); + } +}; + +static void fill_shadow_constant(uptr begin, uptr end, uint16_t v) { + assert(v == kInvalidShadow || v == kUncheckedShadow); + uint16_t *shadow_begin = mem_to_shadow(begin); + uint16_t *shadow_end = mem_to_shadow(end - 1) + 1; + memset(shadow_begin, v, (shadow_end - shadow_begin) * sizeof(*shadow_begin)); +} + +static void fill_shadow(uptr begin, uptr end, uptr cfi_check) { + assert((cfi_check & (kShadowAlign - 1)) == 0); + + // Don't fill anything below cfi_check. We can not represent those addresses + // in the shadow, and must make sure at codegen to place all valid call + // targets above cfi_check. + uptr p = Max(begin, cfi_check); + uint16_t *s = mem_to_shadow(p); + uint16_t *s_end = mem_to_shadow(end - 1) + 1; + uint16_t sv = ((p - cfi_check) >> kShadowGranularity) + 1; + for (; s < s_end; s++, sv++) + *s = sv; + + // Sanity checks. + uptr q = p & ~(kShadowAlign - 1); + for (; q < end; q += kShadowAlign) { + assert((uptr)ShadowValue::load(q).get_cfi_check() == cfi_check); + assert((uptr)ShadowValue::load(q + kShadowAlign / 2).get_cfi_check() == + cfi_check); + assert((uptr)ShadowValue::load(q + kShadowAlign - 1).get_cfi_check() == + cfi_check); + } +} + +// This is a workaround for a glibc bug: +// https://sourceware.org/bugzilla/show_bug.cgi?id=15199 +// Other platforms can, hopefully, just do +// dlopen(RTLD_NOLOAD | RTLD_LAZY) +// dlsym("__cfi_check"). +static uptr find_cfi_check_in_dso(dl_phdr_info *info) { + const ElfW(Dyn) *dynamic = nullptr; + for (int i = 0; i < info->dlpi_phnum; ++i) { + if (info->dlpi_phdr[i].p_type == PT_DYNAMIC) { + dynamic = + (const ElfW(Dyn) *)(info->dlpi_addr + info->dlpi_phdr[i].p_vaddr); + break; + } + } + if (!dynamic) return 0; + uptr strtab = 0, symtab = 0; + for (const ElfW(Dyn) *p = dynamic; p->d_tag != PT_NULL; ++p) { + if (p->d_tag == DT_SYMTAB) + symtab = p->d_un.d_ptr; + else if (p->d_tag == DT_STRTAB) + strtab = p->d_un.d_ptr; + } + + if (symtab > strtab) { + VReport(1, "Can not handle: symtab > strtab (%p > %zx)\n", symtab, strtab); + return 0; + } + + // Verify that strtab and symtab are inside of the same LOAD segment. + // This excludes VDSO, which has (very high) bogus strtab and symtab pointers. + int phdr_idx; + for (phdr_idx = 0; phdr_idx < info->dlpi_phnum; phdr_idx++) { + const Elf_Phdr *phdr = &info->dlpi_phdr[phdr_idx]; + if (phdr->p_type == PT_LOAD) { + uptr beg = info->dlpi_addr + phdr->p_vaddr; + uptr end = beg + phdr->p_memsz; + if (strtab >= beg && strtab < end && symtab >= beg && symtab < end) + break; + } + } + if (phdr_idx == info->dlpi_phnum) { + // Nope, either different segments or just bogus pointers. + // Can not handle this. + VReport(1, "Can not handle: symtab %p, strtab %zx\n", symtab, strtab); + return 0; + } + + for (const ElfW(Sym) *p = (const ElfW(Sym) *)symtab; (ElfW(Addr))p < strtab; + ++p) { + char *name = (char*)(strtab + p->st_name); + if (strcmp(name, "__cfi_check") == 0) { + assert(p->st_info == ELF32_ST_INFO(STB_GLOBAL, STT_FUNC)); + uptr addr = info->dlpi_addr + p->st_value; + return addr; + } + } + return 0; +} + +static int dl_iterate_phdr_cb(dl_phdr_info *info, size_t size, void *data) { + uptr cfi_check = find_cfi_check_in_dso(info); + if (cfi_check) + VReport(1, "Module '%s' __cfi_check %zx\n", info->dlpi_name, cfi_check); + + for (int i = 0; i < info->dlpi_phnum; i++) { + const Elf_Phdr *phdr = &info->dlpi_phdr[i]; + if (phdr->p_type == PT_LOAD) { + // Jump tables are in the executable segment. + // VTables are in the non-executable one. + // Need to fill shadow for both. + // FIXME: reject writable if vtables are in the r/o segment. Depend on + // PT_RELRO? + uptr cur_beg = info->dlpi_addr + phdr->p_vaddr; + uptr cur_end = cur_beg + phdr->p_memsz; + if (cfi_check) { + VReport(1, " %zx .. %zx\n", cur_beg, cur_end); + fill_shadow(cur_beg, cur_end, cfi_check ? cfi_check : (uptr)(-1)); + } else { + fill_shadow_constant(cur_beg, cur_end, kUncheckedShadow); + } + } + } + return 0; +} + +// Fill shadow for the initial libraries. +static void init_shadow() { + dl_iterate_phdr(dl_iterate_phdr_cb, nullptr); +} + +extern "C" SANITIZER_INTERFACE_ATTRIBUTE +void __cfi_slowpath(uptr CallSiteTypeId, void *Ptr) { + uptr Addr = (uptr)Ptr; + VReport(3, "__cfi_slowpath: %zx, %p\n", CallSiteTypeId, Ptr); + ShadowValue sv = ShadowValue::load(Addr); + if (sv.is_invalid()) { + VReport(2, "CFI: invalid memory region for a function pointer (shadow==0): %p\n", Ptr); + Die(); + } + if (sv.is_unchecked()) { + VReport(2, "CFI: unchecked call (shadow=FFFF): %p\n", Ptr); + return; + } + CFICheckFn cfi_check = sv.get_cfi_check(); + VReport(2, "__cfi_check at %p\n", cfi_check); + cfi_check(CallSiteTypeId, Ptr); +} + +static void InitializeFlags() { + SetCommonFlagsDefaults(); +#ifdef CFI_ENABLE_DIAG + __ubsan::Flags *uf = __ubsan::flags(); + uf->SetDefaults(); +#endif + + FlagParser cfi_parser; + RegisterCommonFlags(&cfi_parser); + cfi_parser.ParseString(GetEnv("CFI_OPTIONS")); + +#ifdef CFI_ENABLE_DIAG + FlagParser ubsan_parser; + __ubsan::RegisterUbsanFlags(&ubsan_parser, uf); + RegisterCommonFlags(&ubsan_parser); + + const char *ubsan_default_options = __ubsan::MaybeCallUbsanDefaultOptions(); + ubsan_parser.ParseString(ubsan_default_options); + ubsan_parser.ParseString(GetEnv("UBSAN_OPTIONS")); +#endif + + SetVerbosity(common_flags()->verbosity); + + if (Verbosity()) ReportUnrecognizedFlags(); + + if (common_flags()->help) { + cfi_parser.PrintFlagDescriptions(); + } +} + +extern "C" SANITIZER_INTERFACE_ATTRIBUTE +#if !SANITIZER_CAN_USE_PREINIT_ARRAY +// On ELF platforms, the constructor is invoked using .preinit_array (see below) +__attribute__((constructor(0))) +#endif +void __cfi_init() { + SanitizerToolName = "CFI"; + InitializeFlags(); + + uptr vma = GetMaxVirtualAddress(); + // Shadow is 2 -> 2**kShadowGranularity. + uptr shadow_size = (vma >> (kShadowGranularity - 1)) + 1; + VReport(1, "CFI: VMA size %zx, shadow size %zx\n", vma, shadow_size); + void *shadow = MmapNoReserveOrDie(shadow_size, "CFI shadow"); + VReport(1, "CFI: shadow at %zx .. %zx\n", shadow, + reinterpret_cast<uptr>(shadow) + shadow_size); + __cfi_shadow = (uptr)shadow; + init_shadow(); + +#ifdef CFI_ENABLE_DIAG + __ubsan::InitAsPlugin(); +#endif +} + +#if SANITIZER_CAN_USE_PREINIT_ARRAY +// On ELF platforms, run cfi initialization before any other constructors. +// On other platforms we use the constructor attribute to arrange to run our +// initialization early. +extern "C" { +__attribute__((section(".preinit_array"), + used)) void (*__cfi_preinit)(void) = __cfi_init; +} +#endif diff --git a/lib/cfi/cfi_blacklist.txt b/lib/cfi/cfi_blacklist.txt new file mode 100644 index 000000000000..1f0eeb355617 --- /dev/null +++ b/lib/cfi/cfi_blacklist.txt @@ -0,0 +1,26 @@ +# Standard library types. +type:std::* + +# The stdext namespace contains Microsoft standard library extensions. +type:stdext::* + +# Types with a uuid attribute, i.e. COM types. +type:attr:uuid + +# STL allocators (T *allocator<T *>::allocate(size_type, const void*)). +# The type signature mandates a cast from uninitialized void* to T*. +# size_type can either be unsigned int (j) or unsigned long (m). +fun:*8allocateEjPKv +fun:*8allocateEmPKv + +# std::get_temporary_buffer, likewise (libstdc++, libc++). +fun:_ZSt20get_temporary_buffer* +fun:_ZNSt3__120get_temporary_buffer* + +# STL address-of magic (libstdc++, libc++). +fun:*__addressof* +fun:_ZNSt3__19addressof* + +# Windows C++ stdlib headers that contain bad unrelated casts. +src:*xmemory0 +src:*xstddef diff --git a/lib/dfsan/.clang-format b/lib/dfsan/.clang-format new file mode 100644 index 000000000000..f6cb8ad931f5 --- /dev/null +++ b/lib/dfsan/.clang-format @@ -0,0 +1 @@ +BasedOnStyle: Google diff --git a/lib/dfsan/CMakeLists.txt b/lib/dfsan/CMakeLists.txt index 24ea876f210d..19a7909d0429 100644 --- a/lib/dfsan/CMakeLists.txt +++ b/lib/dfsan/CMakeLists.txt @@ -15,20 +15,19 @@ add_custom_target(dfsan) foreach(arch ${DFSAN_SUPPORTED_ARCH}) set(DFSAN_CFLAGS ${DFSAN_COMMON_CFLAGS}) append_list_if(COMPILER_RT_HAS_FPIE_FLAG -fPIE DFSAN_CFLAGS) - add_compiler_rt_runtime(clang_rt.dfsan-${arch} ${arch} STATIC + add_compiler_rt_runtime(clang_rt.dfsan + STATIC + ARCHS ${arch} SOURCES ${DFSAN_RTL_SOURCES} $<TARGET_OBJECTS:RTInterception.${arch}> $<TARGET_OBJECTS:RTSanitizerCommon.${arch}> $<TARGET_OBJECTS:RTSanitizerCommonLibc.${arch}> - CFLAGS ${DFSAN_CFLAGS}) - set(DFSAN_NOLIBC_CFLAGS ${DFSAN_COMMON_CFLAGS} -DDFSAN_NOLIBC) - add_compiler_rt_runtime(clang_rt.dfsan-libc-${arch} ${arch} STATIC - SOURCES ${DFSAN_RTL_SOURCES} - $<TARGET_OBJECTS:RTSanitizerCommon.${arch}> - CFLAGS ${DFSAN_NOLIBC_CFLAGS}) - add_sanitizer_rt_symbols(clang_rt.dfsan-${arch} dfsan.syms.extra) + CFLAGS ${DFSAN_CFLAGS} + PARENT_TARGET dfsan) + add_sanitizer_rt_symbols(clang_rt.dfsan + ARCHS ${arch} + EXTRA dfsan.syms.extra) add_dependencies(dfsan - clang_rt.dfsan-${arch} clang_rt.dfsan-${arch}-symbols) endforeach() diff --git a/lib/dfsan/dfsan.cc b/lib/dfsan/dfsan.cc index d2e137e129c1..7285f202d060 100644 --- a/lib/dfsan/dfsan.cc +++ b/lib/dfsan/dfsan.cc @@ -42,6 +42,8 @@ Flags __dfsan::flags_data; SANITIZER_INTERFACE_ATTRIBUTE THREADLOCAL dfsan_label __dfsan_retval_tls; SANITIZER_INTERFACE_ATTRIBUTE THREADLOCAL dfsan_label __dfsan_arg_tls[64]; +SANITIZER_INTERFACE_ATTRIBUTE uptr __dfsan_shadow_ptr_mask; + // On Linux/x86_64, memory is laid out as follows: // // +--------------------+ 0x800000000000 (top of memory) @@ -80,24 +82,52 @@ SANITIZER_INTERFACE_ATTRIBUTE THREADLOCAL dfsan_label __dfsan_arg_tls[64]; // | reserved by kernel | // +--------------------+ 0x0000000000 +// On Linux/AArch64 (39-bit VMA), memory is laid out as follow: +// +// +--------------------+ 0x8000000000 (top of memory) +// | application memory | +// +--------------------+ 0x7000008000 (kAppAddr) +// | | +// | unused | +// | | +// +--------------------+ 0x1200000000 (kUnusedAddr) +// | union table | +// +--------------------+ 0x1000000000 (kUnionTableAddr) +// | shadow memory | +// +--------------------+ 0x0000010000 (kShadowAddr) +// | reserved by kernel | +// +--------------------+ 0x0000000000 + +// On Linux/AArch64 (42-bit VMA), memory is laid out as follow: +// +// +--------------------+ 0x40000000000 (top of memory) +// | application memory | +// +--------------------+ 0x3ff00008000 (kAppAddr) +// | | +// | unused | +// | | +// +--------------------+ 0x1200000000 (kUnusedAddr) +// | union table | +// +--------------------+ 0x8000000000 (kUnionTableAddr) +// | shadow memory | +// +--------------------+ 0x0000010000 (kShadowAddr) +// | reserved by kernel | +// +--------------------+ 0x0000000000 + typedef atomic_dfsan_label dfsan_union_table_t[kNumLabels][kNumLabels]; -#if defined(__x86_64__) -static const uptr kShadowAddr = 0x10000; -static const uptr kUnionTableAddr = 0x200000000000; -static const uptr kUnusedAddr = kUnionTableAddr + sizeof(dfsan_union_table_t); -static const uptr kAppAddr = 0x700000008000; -#elif defined(__mips64) -static const uptr kShadowAddr = 0x10000; -static const uptr kUnionTableAddr = 0x2000000000; -static const uptr kUnusedAddr = kUnionTableAddr + sizeof(dfsan_union_table_t); -static const uptr kAppAddr = 0xF000008000; -#else -# error "DFSan not supported for this platform!" +#ifdef DFSAN_RUNTIME_VMA +// Runtime detected VMA size. +int __dfsan::vmaSize; #endif +static uptr UnusedAddr() { + return MappingArchImpl<MAPPING_UNION_TABLE_ADDR>() + + sizeof(dfsan_union_table_t); +} + static atomic_dfsan_label *union_table(dfsan_label l1, dfsan_label l2) { - return &(*(dfsan_union_table_t *) kUnionTableAddr)[l1][l2]; + return &(*(dfsan_union_table_t *) UnionTableAddr())[l1][l2]; } // Checks we do not run out of labels. @@ -325,10 +355,30 @@ static void RegisterDfsanFlags(FlagParser *parser, Flags *f) { } static void InitializeFlags() { + SetCommonFlagsDefaults(); + flags().SetDefaults(); + FlagParser parser; + RegisterCommonFlags(&parser); RegisterDfsanFlags(&parser, &flags()); - flags().SetDefaults(); parser.ParseString(GetEnv("DFSAN_OPTIONS")); + SetVerbosity(common_flags()->verbosity); + if (Verbosity()) ReportUnrecognizedFlags(); + if (common_flags()->help) parser.PrintFlagDescriptions(); +} + +static void InitializePlatformEarly() { +#ifdef DFSAN_RUNTIME_VMA + __dfsan::vmaSize = + (MostSignificantSetBitIndex(GET_CURRENT_FRAME()) + 1); + if (__dfsan::vmaSize == 39 || __dfsan::vmaSize == 42) { + __dfsan_shadow_ptr_mask = ShadowMask(); + } else { + Printf("FATAL: DataFlowSanitizer: unsupported VMA range\n"); + Printf("FATAL: Found %d - Supported 39 and 42\n", __dfsan::vmaSize); + Die(); + } +#endif } static void dfsan_fini() { @@ -347,12 +397,12 @@ static void dfsan_fini() { } } -#ifdef DFSAN_NOLIBC -extern "C" void dfsan_init() { -#else static void dfsan_init(int argc, char **argv, char **envp) { -#endif - MmapFixedNoReserve(kShadowAddr, kUnusedAddr - kShadowAddr); + InitializeFlags(); + + InitializePlatformEarly(); + + MmapFixedNoReserve(ShadowAddr(), UnusedAddr() - ShadowAddr()); // Protect the region of memory we don't use, to preserve the one-to-one // mapping from application to shadow memory. But if ASLR is disabled, Linux @@ -360,21 +410,20 @@ static void dfsan_init(int argc, char **argv, char **envp) { // works so long as the program doesn't use too much memory. We support this // case by disabling memory protection when ASLR is disabled. uptr init_addr = (uptr)&dfsan_init; - if (!(init_addr >= kUnusedAddr && init_addr < kAppAddr)) - MmapNoAccess(kUnusedAddr, kAppAddr - kUnusedAddr); + if (!(init_addr >= UnusedAddr() && init_addr < AppAddr())) + MmapNoAccess(UnusedAddr(), AppAddr() - UnusedAddr()); - InitializeFlags(); InitializeInterceptors(); // Register the fini callback to run when the program terminates successfully // or it is killed by the runtime. Atexit(dfsan_fini); - SetDieCallback(dfsan_fini); + AddDieCallback(dfsan_fini); __dfsan_label_info[kInitializingLabel].desc = "<init label>"; } -#if !defined(DFSAN_NOLIBC) && SANITIZER_CAN_USE_PREINIT_ARRAY +#if SANITIZER_CAN_USE_PREINIT_ARRAY __attribute__((section(".preinit_array"), used)) static void (*dfsan_init_ptr)(int, char **, char **) = dfsan_init; #endif diff --git a/lib/dfsan/dfsan.h b/lib/dfsan/dfsan.h index ceba3533a233..81f949e3019e 100644 --- a/lib/dfsan/dfsan.h +++ b/lib/dfsan/dfsan.h @@ -16,6 +16,7 @@ #define DFSAN_H #include "sanitizer_common/sanitizer_internal_defs.h" +#include "dfsan_platform.h" // Copy declarations from public sanitizer/dfsan_interface.h header here. typedef u16 dfsan_label; @@ -44,11 +45,7 @@ namespace __dfsan { void InitializeInterceptors(); inline dfsan_label *shadow_for(void *ptr) { -#if defined(__x86_64__) - return (dfsan_label *) ((((uptr) ptr) & ~0x700000000000) << 1); -#elif defined(__mips64) - return (dfsan_label *) ((((uptr) ptr) & ~0xF000000000) << 1); -#endif + return (dfsan_label *) ((((uptr) ptr) & ShadowMask()) << 1); } inline const dfsan_label *shadow_for(const void *ptr) { diff --git a/lib/dfsan/dfsan_custom.cc b/lib/dfsan/dfsan_custom.cc index c58b471db53c..e0cd16ab695c 100644 --- a/lib/dfsan/dfsan_custom.cc +++ b/lib/dfsan/dfsan_custom.cc @@ -43,6 +43,14 @@ using namespace __dfsan; +#define CALL_WEAK_INTERCEPTOR_HOOK(f, ...) \ + do { \ + if (f) \ + f(__VA_ARGS__); \ + } while (false) +#define DECLARE_WEAK_INTERCEPTOR_HOOK(f, ...) \ +SANITIZER_INTERFACE_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE void f(__VA_ARGS__); + extern "C" { SANITIZER_INTERFACE_ATTRIBUTE int __dfsw_stat(const char *path, struct stat *buf, dfsan_label path_label, @@ -77,25 +85,23 @@ SANITIZER_INTERFACE_ATTRIBUTE char *__dfsw_strchr(const char *s, int c, *ret_label = dfsan_union(dfsan_read_label(s, i + 1), dfsan_union(s_label, c_label)); } - return s[i] == 0 ? 0 : const_cast<char *>(s+i); + return s[i] == 0 ? nullptr : const_cast<char *>(s+i); } } } -SANITIZER_INTERFACE_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE -void -dfsan_weak_hook_memcmp(uptr caller_pc, const void *s1, const void *s2, size_t n, - dfsan_label s1_label, dfsan_label s2_label, - dfsan_label n_label); +DECLARE_WEAK_INTERCEPTOR_HOOK(dfsan_weak_hook_memcmp, uptr caller_pc, + const void *s1, const void *s2, size_t n, + dfsan_label s1_label, dfsan_label s2_label, + dfsan_label n_label) SANITIZER_INTERFACE_ATTRIBUTE int __dfsw_memcmp(const void *s1, const void *s2, size_t n, dfsan_label s1_label, dfsan_label s2_label, dfsan_label n_label, dfsan_label *ret_label) { - if (dfsan_weak_hook_memcmp) - dfsan_weak_hook_memcmp(GET_CALLER_PC(), s1, s2, n, s1_label, s2_label, - n_label); + CALL_WEAK_INTERCEPTOR_HOOK(dfsan_weak_hook_memcmp, GET_CALLER_PC(), s1, s2, n, + s1_label, s2_label, n_label); const char *cs1 = (const char *) s1, *cs2 = (const char *) s2; for (size_t i = 0; i != n; ++i) { if (cs1[i] != cs2[i]) { @@ -118,10 +124,16 @@ SANITIZER_INTERFACE_ATTRIBUTE int __dfsw_memcmp(const void *s1, const void *s2, return 0; } +DECLARE_WEAK_INTERCEPTOR_HOOK(dfsan_weak_hook_strcmp, uptr caller_pc, + const char *s1, const char *s2, + dfsan_label s1_label, dfsan_label s2_label) + SANITIZER_INTERFACE_ATTRIBUTE int __dfsw_strcmp(const char *s1, const char *s2, dfsan_label s1_label, dfsan_label s2_label, dfsan_label *ret_label) { + CALL_WEAK_INTERCEPTOR_HOOK(dfsan_weak_hook_strcmp, GET_CALLER_PC(), s1, s2, + s1_label, s2_label); for (size_t i = 0;; ++i) { if (s1[i] != s2[i] || s1[i] == 0 || s2[i] == 0) { if (flags().strict_data_dependencies) { @@ -153,6 +165,11 @@ __dfsw_strcasecmp(const char *s1, const char *s2, dfsan_label s1_label, return 0; } +DECLARE_WEAK_INTERCEPTOR_HOOK(dfsan_weak_hook_strncmp, uptr caller_pc, + const char *s1, const char *s2, size_t n, + dfsan_label s1_label, dfsan_label s2_label, + dfsan_label n_label) + SANITIZER_INTERFACE_ATTRIBUTE int __dfsw_strncmp(const char *s1, const char *s2, size_t n, dfsan_label s1_label, dfsan_label s2_label, @@ -163,6 +180,9 @@ SANITIZER_INTERFACE_ATTRIBUTE int __dfsw_strncmp(const char *s1, const char *s2, return 0; } + CALL_WEAK_INTERCEPTOR_HOOK(dfsan_weak_hook_strncmp, GET_CALLER_PC(), s1, s2, + n, s1_label, s2_label, n_label); + for (size_t i = 0;; ++i) { if (s1[i] != s2[i] || s1[i] == 0 || s2[i] == 0 || i == n - 1) { if (flags().strict_data_dependencies) { @@ -828,8 +848,8 @@ typedef void (*write_trampoline_t)( // Calls to dfsan_set_write_callback() set the values in this struct. // Calls to the custom version of write() read (and invoke) them. static struct { - write_trampoline_t write_callback_trampoline = NULL; - void *write_callback = NULL; + write_trampoline_t write_callback_trampoline = nullptr; + void *write_callback = nullptr; } write_callback_info; SANITIZER_INTERFACE_ATTRIBUTE void @@ -846,7 +866,7 @@ SANITIZER_INTERFACE_ATTRIBUTE int __dfsw_write(int fd, const void *buf, size_t count, dfsan_label fd_label, dfsan_label buf_label, dfsan_label count_label, dfsan_label *ret_label) { - if (write_callback_info.write_callback != NULL) { + if (write_callback_info.write_callback) { write_callback_info.write_callback_trampoline( write_callback_info.write_callback, fd, buf, count, @@ -856,7 +876,7 @@ __dfsw_write(int fd, const void *buf, size_t count, *ret_label = 0; return write(fd, buf, count); } -} +} // namespace __dfsan // Type used to extract a dfsan_label with va_arg() typedef int dfsan_label_va; @@ -1112,4 +1132,4 @@ int __dfsw_snprintf(char *str, size_t size, const char *format, va_end(ap); return ret; } -} +} // extern "C" diff --git a/lib/dfsan/dfsan_platform.h b/lib/dfsan/dfsan_platform.h new file mode 100644 index 000000000000..f1d9f108e908 --- /dev/null +++ b/lib/dfsan/dfsan_platform.h @@ -0,0 +1,107 @@ +//===-- dfsan_platform.h ----------------------------------------*- C++ -*-===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// +// +// This file is a part of DataFlowSanitizer. +// +// Platform specific information for DFSan. +//===----------------------------------------------------------------------===// + +#ifndef DFSAN_PLATFORM_H +#define DFSAN_PLATFORM_H + +namespace __dfsan { + +#if defined(__x86_64__) +struct Mapping { + static const uptr kShadowAddr = 0x10000; + static const uptr kUnionTableAddr = 0x200000000000; + static const uptr kAppAddr = 0x700000008000; + static const uptr kShadowMask = ~0x700000000000; +}; +#elif defined(__mips64) +struct Mapping { + static const uptr kShadowAddr = 0x10000; + static const uptr kUnionTableAddr = 0x2000000000; + static const uptr kAppAddr = 0xF000008000; + static const uptr kShadowMask = ~0xF000000000; +}; +#elif defined(__aarch64__) +struct Mapping39 { + static const uptr kShadowAddr = 0x10000; + static const uptr kUnionTableAddr = 0x1000000000; + static const uptr kAppAddr = 0x7000008000; + static const uptr kShadowMask = ~0x7800000000; +}; + +struct Mapping42 { + static const uptr kShadowAddr = 0x10000; + static const uptr kUnionTableAddr = 0x8000000000; + static const uptr kAppAddr = 0x3ff00008000; + static const uptr kShadowMask = ~0x3c000000000; +}; + +extern int vmaSize; +# define DFSAN_RUNTIME_VMA 1 +#else +# error "DFSan not supported for this platform!" +#endif + +enum MappingType { + MAPPING_SHADOW_ADDR, + MAPPING_UNION_TABLE_ADDR, + MAPPING_APP_ADDR, + MAPPING_SHADOW_MASK +}; + +template<typename Mapping, int Type> +uptr MappingImpl(void) { + switch (Type) { + case MAPPING_SHADOW_ADDR: return Mapping::kShadowAddr; + case MAPPING_UNION_TABLE_ADDR: return Mapping::kUnionTableAddr; + case MAPPING_APP_ADDR: return Mapping::kAppAddr; + case MAPPING_SHADOW_MASK: return Mapping::kShadowMask; + } +} + +template<int Type> +uptr MappingArchImpl(void) { +#ifdef __aarch64__ + if (vmaSize == 39) + return MappingImpl<Mapping39, Type>(); + else + return MappingImpl<Mapping42, Type>(); + DCHECK(0); +#else + return MappingImpl<Mapping, Type>(); +#endif +} + +ALWAYS_INLINE +uptr ShadowAddr() { + return MappingArchImpl<MAPPING_SHADOW_ADDR>(); +} + +ALWAYS_INLINE +uptr UnionTableAddr() { + return MappingArchImpl<MAPPING_UNION_TABLE_ADDR>(); +} + +ALWAYS_INLINE +uptr AppAddr() { + return MappingArchImpl<MAPPING_APP_ADDR>(); +} + +ALWAYS_INLINE +uptr ShadowMask() { + return MappingArchImpl<MAPPING_SHADOW_MASK>(); +} + +} // namespace __dfsan + +#endif diff --git a/lib/dfsan/done_abilist.txt b/lib/dfsan/done_abilist.txt index e6c077ff1208..7ca8aeba32fe 100644 --- a/lib/dfsan/done_abilist.txt +++ b/lib/dfsan/done_abilist.txt @@ -266,10 +266,41 @@ fun:reflect.makeFuncStub=discard # Replaces __sanitizer_cov_trace_cmp with __dfsw___sanitizer_cov_trace_cmp fun:__sanitizer_cov_trace_cmp=custom fun:__sanitizer_cov_trace_cmp=uninstrumented +# Similar for __sanitizer_cov_trace_switch +fun:__sanitizer_cov_trace_switch=custom +fun:__sanitizer_cov_trace_switch=uninstrumented # Ignores all other __sanitizer callbacks. -fun:__sanitizer_*=uninstrumented -fun:__sanitizer_*=discard +fun:__sanitizer_cov=uninstrumented +fun:__sanitizer_cov=discard +fun:__sanitizer_cov_module_init=uninstrumented +fun:__sanitizer_cov_module_init=discard +fun:__sanitizer_cov_with_check=uninstrumented +fun:__sanitizer_cov_with_check=discard +fun:__sanitizer_cov_indir_call16=uninstrumented +fun:__sanitizer_cov_indir_call16=discard +fun:__sanitizer_cov_indir_call16=uninstrumented +fun:__sanitizer_cov_indir_call16=discard +fun:__sanitizer_reset_coverage=uninstrumented +fun:__sanitizer_reset_coverage=discard +fun:__sanitizer_set_death_callback=uninstrumented +fun:__sanitizer_set_death_callback=discard +fun:__sanitizer_get_coverage_guards=uninstrumented +fun:__sanitizer_get_coverage_guards=discard +fun:__sanitizer_get_number_of_counters=uninstrumented +fun:__sanitizer_get_number_of_counters=discard +fun:__sanitizer_update_counter_bitset_and_clear_counters=uninstrumented +fun:__sanitizer_update_counter_bitset_and_clear_counters=discard +fun:__sanitizer_get_total_unique_coverage=uninstrumented +fun:__sanitizer_get_total_unique_coverage=discard +fun:__sanitizer_get_total_unique_coverage=uninstrumented +fun:__sanitizer_get_total_unique_coverage=discard +fun:__sanitizer_update_counter_bitset_and_clear_counters=uninstrumented +fun:__sanitizer_update_counter_bitset_and_clear_counters=discard + +# Ignores the dfsan wrappers. +fun:__dfsw_*=uninstrumented +fun:__dfsw_*=discard # Don't add extra parameters to the Fuzzer callback. fun:LLVMFuzzerTestOneInput=uninstrumented diff --git a/lib/interception/.clang-format b/lib/interception/.clang-format new file mode 100644 index 000000000000..f6cb8ad931f5 --- /dev/null +++ b/lib/interception/.clang-format @@ -0,0 +1 @@ +BasedOnStyle: Google diff --git a/lib/interception/interception_linux.h b/lib/interception/interception_linux.h index d3f774bede9f..27a66c882041 100644 --- a/lib/interception/interception_linux.h +++ b/lib/interception/interception_linux.h @@ -35,12 +35,12 @@ void *GetFuncAddrVer(const char *func_name, const char *ver); (::__interception::uptr) & WRAP(func)) #if !defined(__ANDROID__) // android does not have dlvsym -# define INTERCEPT_FUNCTION_VER_LINUX_OR_FREEBSD(func, symver) \ - ::__interception::real_##func = (func##_f)(unsigned long) \ - ::__interception::GetFuncAddrVer(#func, symver) +#define INTERCEPT_FUNCTION_VER_LINUX_OR_FREEBSD(func, symver) \ + (::__interception::real_##func = (func##_f)( \ + unsigned long)::__interception::GetFuncAddrVer(#func, symver)) #else -# define INTERCEPT_FUNCTION_VER_LINUX_OR_FREEBSD(func, symver) \ - INTERCEPT_FUNCTION_LINUX_OR_FREEBSD(func) +#define INTERCEPT_FUNCTION_VER_LINUX_OR_FREEBSD(func, symver) \ + INTERCEPT_FUNCTION_LINUX_OR_FREEBSD(func) #endif // !defined(__ANDROID__) #endif // INTERCEPTION_LINUX_H diff --git a/lib/interception/interception_win.cc b/lib/interception/interception_win.cc index 19cf184948b9..4c04c83b982b 100644 --- a/lib/interception/interception_win.cc +++ b/lib/interception/interception_win.cc @@ -15,6 +15,7 @@ #ifdef _WIN32 #include "interception.h" +#define WIN32_LEAN_AND_MEAN #include <windows.h> namespace __interception { @@ -182,7 +183,7 @@ bool OverrideFunction(uptr old_func, uptr new_func, uptr *orig_old_func) { return true; } -static const void **InterestingDLLsAvailable() { +static void **InterestingDLLsAvailable() { const char *InterestingDLLs[] = { "kernel32.dll", "msvcr110.dll", // VS2012 @@ -198,14 +199,65 @@ static const void **InterestingDLLsAvailable() { result[j++] = (void *)h; } } - return (const void **)&result[0]; + return &result[0]; +} + +namespace { +// Utility for reading loaded PE images. +template <typename T> class RVAPtr { + public: + RVAPtr(void *module, uptr rva) + : ptr_(reinterpret_cast<T *>(reinterpret_cast<char *>(module) + rva)) {} + operator T *() { return ptr_; } + T *operator->() { return ptr_; } + T *operator++() { return ++ptr_; } + + private: + T *ptr_; +}; +} // namespace + +// Internal implementation of GetProcAddress. At least since Windows 8, +// GetProcAddress appears to initialize DLLs before returning function pointers +// into them. This is problematic for the sanitizers, because they typically +// want to intercept malloc *before* MSVCRT initializes. Our internal +// implementation walks the export list manually without doing initialization. +uptr InternalGetProcAddress(void *module, const char *func_name) { + // Check that the module header is full and present. + RVAPtr<IMAGE_DOS_HEADER> dos_stub(module, 0); + RVAPtr<IMAGE_NT_HEADERS> headers(module, dos_stub->e_lfanew); + if (!module || dos_stub->e_magic != IMAGE_DOS_SIGNATURE || // "MZ" + headers->Signature != IMAGE_NT_SIGNATURE || // "PE\0\0" + headers->FileHeader.SizeOfOptionalHeader < + sizeof(IMAGE_OPTIONAL_HEADER)) { + return 0; + } + + IMAGE_DATA_DIRECTORY *export_directory = + &headers->OptionalHeader.DataDirectory[IMAGE_DIRECTORY_ENTRY_EXPORT]; + RVAPtr<IMAGE_EXPORT_DIRECTORY> exports(module, + export_directory->VirtualAddress); + RVAPtr<DWORD> functions(module, exports->AddressOfFunctions); + RVAPtr<DWORD> names(module, exports->AddressOfNames); + RVAPtr<WORD> ordinals(module, exports->AddressOfNameOrdinals); + + for (DWORD i = 0; i < exports->NumberOfNames; i++) { + RVAPtr<char> name(module, names[i]); + if (!strcmp(func_name, name)) { + DWORD index = ordinals[i]; + RVAPtr<char> func(module, functions[index]); + return (uptr)(char *)func; + } + } + + return 0; } static bool GetFunctionAddressInDLLs(const char *func_name, uptr *func_addr) { *func_addr = 0; - const void **DLLs = InterestingDLLsAvailable(); + void **DLLs = InterestingDLLsAvailable(); for (size_t i = 0; *func_addr == 0 && DLLs[i]; ++i) - *func_addr = (uptr)GetProcAddress((HMODULE)DLLs[i], func_name); + *func_addr = InternalGetProcAddress(DLLs[i], func_name); return (*func_addr != 0); } diff --git a/lib/interception/interception_win.h b/lib/interception/interception_win.h index ba768a7233f9..96c4a0c0f5a3 100644 --- a/lib/interception/interception_win.h +++ b/lib/interception/interception_win.h @@ -30,6 +30,10 @@ bool OverrideFunction(uptr old_func, uptr new_func, uptr *orig_old_func = 0); // Overrides a function in a system DLL or DLL CRT by its exported name. bool OverrideFunction(const char *name, uptr new_func, uptr *orig_old_func = 0); + +// Windows-only replacement for GetProcAddress. Useful for some sanitizers. +uptr InternalGetProcAddress(void *module, const char *func_name); + } // namespace __interception #if defined(INTERCEPTION_DYNAMIC_CRT) diff --git a/lib/lsan/.clang-format b/lib/lsan/.clang-format new file mode 100644 index 000000000000..f6cb8ad931f5 --- /dev/null +++ b/lib/lsan/.clang-format @@ -0,0 +1 @@ +BasedOnStyle: Google diff --git a/lib/lsan/CMakeLists.txt b/lib/lsan/CMakeLists.txt index 37f794e2e11b..20e40932165c 100644 --- a/lib/lsan/CMakeLists.txt +++ b/lib/lsan/CMakeLists.txt @@ -26,14 +26,16 @@ add_compiler_rt_object_libraries(RTLSanCommon if(COMPILER_RT_HAS_LSAN) foreach(arch ${LSAN_SUPPORTED_ARCH}) - add_compiler_rt_runtime(clang_rt.lsan-${arch} ${arch} STATIC + add_compiler_rt_runtime(clang_rt.lsan + STATIC + ARCHS ${arch} SOURCES ${LSAN_SOURCES} $<TARGET_OBJECTS:RTInterception.${arch}> $<TARGET_OBJECTS:RTSanitizerCommon.${arch}> $<TARGET_OBJECTS:RTSanitizerCommonLibc.${arch}> $<TARGET_OBJECTS:RTLSanCommon.${arch}> - CFLAGS ${LSAN_CFLAGS}) - add_dependencies(lsan clang_rt.lsan-${arch}) + CFLAGS ${LSAN_CFLAGS} + PARENT_TARGET lsan) endforeach() endif() diff --git a/lib/lsan/lsan.cc b/lib/lsan/lsan.cc index 6018f7bf6f49..f3e6ad7c9cba 100644 --- a/lib/lsan/lsan.cc +++ b/lib/lsan/lsan.cc @@ -44,6 +44,7 @@ static void InitializeFlags() { cf.external_symbolizer_path = GetEnv("LSAN_SYMBOLIZER_PATH"); cf.malloc_context_size = 30; cf.detect_leaks = true; + cf.exitcode = 23; OverrideCommonFlags(cf); } @@ -69,6 +70,7 @@ extern "C" void __lsan_init() { return; lsan_init_is_running = true; SanitizerToolName = "LeakSanitizer"; + CacheBinaryName(); InitializeFlags(); InitCommonLsan(); InitializeAllocator(); diff --git a/lib/lsan/lsan_allocator.cc b/lib/lsan/lsan_allocator.cc index 67125dbb3e45..0a3678132ae1 100644 --- a/lib/lsan/lsan_allocator.cc +++ b/lib/lsan/lsan_allocator.cc @@ -26,13 +26,13 @@ extern "C" void *memset(void *ptr, int value, uptr num); namespace __lsan { struct ChunkMetadata { - bool allocated : 8; // Must be first. + u8 allocated : 8; // Must be first. ChunkTag tag : 2; uptr requested_size : 54; u32 stack_trace_id; }; -#if defined(__mips64) +#if defined(__mips64) || defined(__aarch64__) static const uptr kMaxAllowedMallocSize = 4UL << 30; static const uptr kRegionSizeLog = 20; static const uptr kNumRegions = SANITIZER_MMAP_RANGE_SIZE >> kRegionSizeLog; @@ -91,7 +91,7 @@ void *Allocate(const StackTrace &stack, uptr size, uptr alignment, size = 1; if (size > kMaxAllowedMallocSize) { Report("WARNING: LeakSanitizer failed to allocate %zu bytes\n", size); - return 0; + return nullptr; } void *p = allocator.Allocate(&cache, size, alignment, false); // Do not rely on the allocator to clear the memory (it's slow). @@ -114,7 +114,7 @@ void *Reallocate(const StackTrace &stack, void *p, uptr new_size, if (new_size > kMaxAllowedMallocSize) { Report("WARNING: LeakSanitizer failed to allocate %zu bytes\n", new_size); allocator.Deallocate(&cache, p); - return 0; + return nullptr; } p = allocator.Reallocate(&cache, p, new_size, alignment); RegisterAllocation(stack, p, new_size); @@ -212,7 +212,7 @@ IgnoreObjectResult IgnoreObjectLocked(const void *p) { return kIgnoreObjectInvalid; } } -} // namespace __lsan +} // namespace __lsan using namespace __lsan; @@ -241,10 +241,10 @@ SANITIZER_INTERFACE_ATTRIBUTE uptr __sanitizer_get_estimated_allocated_size(uptr size) { return size; } SANITIZER_INTERFACE_ATTRIBUTE -int __sanitizer_get_ownership(const void *p) { return Metadata(p) != 0; } +int __sanitizer_get_ownership(const void *p) { return Metadata(p) != nullptr; } SANITIZER_INTERFACE_ATTRIBUTE uptr __sanitizer_get_allocated_size(const void *p) { return GetMallocUsableSize(p); } -} // extern "C" +} // extern "C" diff --git a/lib/lsan/lsan_common.cc b/lib/lsan/lsan_common.cc index 0ffba505cc70..1cffac44395c 100644 --- a/lib/lsan/lsan_common.cc +++ b/lib/lsan/lsan_common.cc @@ -119,6 +119,10 @@ static inline bool CanBeAHeapPointer(uptr p) { return ((p >> 47) == 0); #elif defined(__mips64) return ((p >> 40) == 0); +#elif defined(__aarch64__) + unsigned runtimeVMA = + (MostSignificantSetBitIndex(GET_CURRENT_FRAME()) + 1); + return ((p >> runtimeVMA) == 0); #else return true; #endif @@ -243,8 +247,8 @@ static void ProcessRootRegion(Frontier *frontier, uptr root_begin, MemoryMappingLayout proc_maps(/*cache_enabled*/true); uptr begin, end, prot; while (proc_maps.Next(&begin, &end, - /*offset*/ 0, /*filename*/ 0, /*filename_size*/ 0, - &prot)) { + /*offset*/ nullptr, /*filename*/ nullptr, + /*filename_size*/ 0, &prot)) { uptr intersection_begin = Max(root_begin, begin); uptr intersection_end = Min(end, root_end); if (intersection_begin >= intersection_end) continue; @@ -375,8 +379,8 @@ static void PrintMatchedSuppressions() { Printf("Suppressions used:\n"); Printf(" count bytes template\n"); for (uptr i = 0; i < matched.size(); i++) - Printf("%7zu %10zu %s\n", static_cast<uptr>(matched[i]->hit_count), - matched[i]->weight, matched[i]->templ); + Printf("%7zu %10zu %s\n", static_cast<uptr>(atomic_load_relaxed( + &matched[i]->hit_count)), matched[i]->weight, matched[i]->templ); Printf("%s\n\n", line); } @@ -444,10 +448,8 @@ void DoLeakCheck() { if (!have_leaks) { return; } - if (flags()->exitcode) { - if (common_flags()->coverage) - __sanitizer_cov_dump(); - internal__exit(flags()->exitcode); + if (common_flags()->exitcode) { + Die(); } } @@ -486,7 +488,7 @@ static Suppression *GetSuppressionForStack(u32 stack_trace_id) { StackTrace::GetPreviousInstructionPc(stack.trace[i])); if (s) return s; } - return 0; + return nullptr; } ///// LeakReport implementation. ///// @@ -600,7 +602,8 @@ void LeakReport::ApplySuppressions() { Suppression *s = GetSuppressionForStack(leaks_[i].stack_trace_id); if (s) { s->weight += leaks_[i].total_size; - s->hit_count += leaks_[i].hit_count; + atomic_store_relaxed(&s->hit_count, atomic_load_relaxed(&s->hit_count) + + leaks_[i].hit_count); leaks_[i].is_suppressed = true; } } @@ -613,8 +616,8 @@ uptr LeakReport::UnsuppressedLeakCount() { return result; } -} // namespace __lsan -#endif // CAN_SANITIZE_LEAKS +} // namespace __lsan +#endif // CAN_SANITIZE_LEAKS using namespace __lsan; // NOLINT @@ -635,7 +638,7 @@ void __lsan_ignore_object(const void *p) { "heap object at %p is already being ignored\n", p); if (res == kIgnoreObjectSuccess) VReport(1, "__lsan_ignore_object(): ignoring heap object at %p\n", p); -#endif // CAN_SANITIZE_LEAKS +#endif // CAN_SANITIZE_LEAKS } SANITIZER_INTERFACE_ATTRIBUTE @@ -646,7 +649,7 @@ void __lsan_register_root_region(const void *begin, uptr size) { RootRegion region = {begin, size}; root_regions->push_back(region); VReport(1, "Registered root region at %p of size %llu\n", begin, size); -#endif // CAN_SANITIZE_LEAKS +#endif // CAN_SANITIZE_LEAKS } SANITIZER_INTERFACE_ATTRIBUTE @@ -673,7 +676,7 @@ void __lsan_unregister_root_region(const void *begin, uptr size) { begin, size); Die(); } -#endif // CAN_SANITIZE_LEAKS +#endif // CAN_SANITIZE_LEAKS } SANITIZER_INTERFACE_ATTRIBUTE @@ -699,7 +702,7 @@ void __lsan_do_leak_check() { #if CAN_SANITIZE_LEAKS if (common_flags()->detect_leaks) __lsan::DoLeakCheck(); -#endif // CAN_SANITIZE_LEAKS +#endif // CAN_SANITIZE_LEAKS } SANITIZER_INTERFACE_ATTRIBUTE @@ -707,7 +710,7 @@ int __lsan_do_recoverable_leak_check() { #if CAN_SANITIZE_LEAKS if (common_flags()->detect_leaks) return __lsan::DoRecoverableLeakCheck(); -#endif // CAN_SANITIZE_LEAKS +#endif // CAN_SANITIZE_LEAKS return 0; } @@ -717,4 +720,4 @@ int __lsan_is_turned_off() { return 0; } #endif -} // extern "C" +} // extern "C" diff --git a/lib/lsan/lsan_common.h b/lib/lsan/lsan_common.h index 4f9d24fb3ab9..0dfd0d4c9890 100644 --- a/lib/lsan/lsan_common.h +++ b/lib/lsan/lsan_common.h @@ -22,8 +22,8 @@ #include "sanitizer_common/sanitizer_stoptheworld.h" #include "sanitizer_common/sanitizer_symbolizer.h" -#if SANITIZER_LINUX && (defined(__x86_64__) || defined(__mips64)) \ - && (SANITIZER_WORDSIZE == 64) +#if (SANITIZER_LINUX && !SANITIZER_ANDROID) && (SANITIZER_WORDSIZE == 64) \ + && (defined(__x86_64__) || defined(__mips64) || defined(__aarch64__)) #define CAN_SANITIZE_LEAKS 1 #else #define CAN_SANITIZE_LEAKS 0 diff --git a/lib/lsan/lsan_common_linux.cc b/lib/lsan/lsan_common_linux.cc index 2955343e1f0b..1dc0561dab71 100644 --- a/lib/lsan/lsan_common_linux.cc +++ b/lib/lsan/lsan_common_linux.cc @@ -29,7 +29,7 @@ static const char kLinkerName[] = "ld"; // We request 2 modules matching "ld", so we can print a warning if there's more // than one match. But only the first one is actually used. static char linker_placeholder[2 * sizeof(LoadedModule)] ALIGNED(64); -static LoadedModule *linker = 0; +static LoadedModule *linker = nullptr; static bool IsLinker(const char* full_name) { return LibraryNameIs(full_name, kLinkerName); @@ -49,7 +49,7 @@ void InitializePlatformSpecificModules() { else if (num_matches > 1) VReport(1, "LeakSanitizer: Multiple modules match \"%s\". " "TLS will not be handled correctly.\n", kLinkerName); - linker = 0; + linker = nullptr; } static int ProcessGlobalRegionsCallback(struct dl_phdr_info *info, size_t size, @@ -174,5 +174,6 @@ void DoStopTheWorld(StopTheWorldCallback callback, void *argument) { dl_iterate_phdr(DoStopTheWorldCallback, ¶m); } -} // namespace __lsan -#endif // CAN_SANITIZE_LEAKS && SANITIZER_LINUX +} // namespace __lsan + +#endif // CAN_SANITIZE_LEAKS && SANITIZER_LINUX diff --git a/lib/lsan/lsan_flags.inc b/lib/lsan/lsan_flags.inc index b19b3452b2fc..c405005deed5 100644 --- a/lib/lsan/lsan_flags.inc +++ b/lib/lsan/lsan_flags.inc @@ -24,8 +24,6 @@ LSAN_FLAG( "Aggregate two objects into one leak if this many stack frames match. If " "zero, the entire stack trace must match.") LSAN_FLAG(int, max_leaks, 0, "The number of leaks reported.") -LSAN_FLAG(int, exitcode, 23, - "If nonzero kill the process with this exit code upon finding leaks.") // Flags controlling the root set of reachable memory. LSAN_FLAG(bool, use_globals, true, diff --git a/lib/lsan/lsan_interceptors.cc b/lib/lsan/lsan_interceptors.cc index 61a92154d95e..be0d0ddc282e 100644 --- a/lib/lsan/lsan_interceptors.cc +++ b/lib/lsan/lsan_interceptors.cc @@ -71,7 +71,7 @@ INTERCEPTOR(void*, calloc, uptr nmemb, uptr size) { CHECK(allocated < kCallocPoolSize); return mem; } - if (CallocShouldReturnNullDueToOverflow(size, nmemb)) return 0; + if (CallocShouldReturnNullDueToOverflow(size, nmemb)) return nullptr; ENSURE_LSAN_INITED; GET_STACK_TRACE_MALLOC; size *= nmemb; @@ -164,9 +164,9 @@ void *operator new[](uptr size, std::nothrow_t const&) { OPERATOR_NEW_BODY; } Deallocate(ptr); INTERCEPTOR_ATTRIBUTE -void operator delete(void *ptr) throw() { OPERATOR_DELETE_BODY; } +void operator delete(void *ptr) NOEXCEPT { OPERATOR_DELETE_BODY; } INTERCEPTOR_ATTRIBUTE -void operator delete[](void *ptr) throw() { OPERATOR_DELETE_BODY; } +void operator delete[](void *ptr) NOEXCEPT { OPERATOR_DELETE_BODY; } INTERCEPTOR_ATTRIBUTE void operator delete(void *ptr, std::nothrow_t const&) { OPERATOR_DELETE_BODY; } INTERCEPTOR_ATTRIBUTE @@ -226,7 +226,7 @@ INTERCEPTOR(int, pthread_create, void *th, void *attr, ENSURE_LSAN_INITED; EnsureMainThreadIDIsCorrect(); __sanitizer_pthread_attr_t myattr; - if (attr == 0) { + if (!attr) { pthread_attr_init(&myattr); attr = &myattr; } @@ -284,4 +284,4 @@ void InitializeInterceptors() { } } -} // namespace __lsan +} // namespace __lsan diff --git a/lib/lsan/lsan_thread.cc b/lib/lsan/lsan_thread.cc index 0f8efc093b56..10ac2c9f499d 100644 --- a/lib/lsan/lsan_thread.cc +++ b/lib/lsan/lsan_thread.cc @@ -79,7 +79,7 @@ void ThreadContext::OnFinished() { u32 ThreadCreate(u32 parent_tid, uptr user_id, bool detached) { return thread_registry->CreateThread(user_id, detached, parent_tid, - /* arg */ 0); + /* arg */ nullptr); } void ThreadStart(u32 tid, uptr os_id) { @@ -99,9 +99,9 @@ void ThreadFinish() { } ThreadContext *CurrentThreadContext() { - if (!thread_registry) return 0; + if (!thread_registry) return nullptr; if (GetCurrentThread() == kInvalidTid) - return 0; + return nullptr; // No lock needed when getting current thread. return (ThreadContext *)thread_registry->GetThreadLocked(GetCurrentThread()); } @@ -120,7 +120,7 @@ u32 ThreadTid(uptr uid) { void ThreadJoin(u32 tid) { CHECK_NE(tid, kInvalidTid); - thread_registry->JoinThread(tid, /* arg */0); + thread_registry->JoinThread(tid, /* arg */nullptr); } void EnsureMainThreadIDIsCorrect() { @@ -157,4 +157,4 @@ void UnlockThreadRegistry() { thread_registry->Unlock(); } -} // namespace __lsan +} // namespace __lsan diff --git a/lib/msan/.clang-format b/lib/msan/.clang-format new file mode 100644 index 000000000000..f6cb8ad931f5 --- /dev/null +++ b/lib/msan/.clang-format @@ -0,0 +1 @@ +BasedOnStyle: Google diff --git a/lib/msan/CMakeLists.txt b/lib/msan/CMakeLists.txt index de5980e5644b..1b48def46280 100644 --- a/lib/msan/CMakeLists.txt +++ b/lib/msan/CMakeLists.txt @@ -27,24 +27,32 @@ set(MSAN_RUNTIME_LIBRARIES) # Static runtime library. add_custom_target(msan) foreach(arch ${MSAN_SUPPORTED_ARCH}) - add_compiler_rt_runtime(clang_rt.msan-${arch} ${arch} STATIC + add_compiler_rt_runtime(clang_rt.msan + STATIC + ARCHS ${arch} SOURCES ${MSAN_RTL_SOURCES} $<TARGET_OBJECTS:RTInterception.${arch}> $<TARGET_OBJECTS:RTSanitizerCommon.${arch}> $<TARGET_OBJECTS:RTSanitizerCommonLibc.${arch}> $<TARGET_OBJECTS:RTUbsan.${arch}> - CFLAGS ${MSAN_RTL_CFLAGS}) - add_compiler_rt_runtime(clang_rt.msan_cxx-${arch} ${arch} STATIC + CFLAGS ${MSAN_RTL_CFLAGS} + PARENT_TARGET msan) + add_compiler_rt_runtime(clang_rt.msan_cxx + STATIC + ARCHS ${arch} SOURCES ${MSAN_RTL_CXX_SOURCES} $<TARGET_OBJECTS:RTUbsan_cxx.${arch}> - CFLAGS ${MSAN_RTL_CFLAGS}) - add_dependencies(msan clang_rt.msan-${arch} - clang_rt.msan_cxx-${arch}) + CFLAGS ${MSAN_RTL_CFLAGS} + PARENT_TARGET msan) list(APPEND MSAN_RUNTIME_LIBRARIES clang_rt.msan-${arch} clang_rt.msan_cxx-${arch}) if(UNIX) - add_sanitizer_rt_symbols(clang_rt.msan-${arch} msan.syms.extra) - add_sanitizer_rt_symbols(clang_rt.msan_cxx-${arch} msan.syms.extra) + add_sanitizer_rt_symbols(clang_rt.msan + ARCHS ${arch} + EXTRA msan.syms.extra) + add_sanitizer_rt_symbols(clang_rt.msan_cxx + ARCHS ${arch} + EXTRA msan.syms.extra) add_dependencies(msan clang_rt.msan-${arch}-symbols clang_rt.msan_cxx-${arch}-symbols) endif() diff --git a/lib/msan/msan.cc b/lib/msan/msan.cc index 163d59dabfa8..9949db4c13a0 100644 --- a/lib/msan/msan.cc +++ b/lib/msan/msan.cc @@ -55,7 +55,7 @@ SANITIZER_INTERFACE_ATTRIBUTE THREADLOCAL u32 __msan_retval_origin_tls; SANITIZER_INTERFACE_ATTRIBUTE -THREADLOCAL u64 __msan_va_arg_tls[kMsanParamTlsSize / sizeof(u64)]; +ALIGNED(16) THREADLOCAL u64 __msan_va_arg_tls[kMsanParamTlsSize / sizeof(u64)]; SANITIZER_INTERFACE_ATTRIBUTE THREADLOCAL u64 __msan_va_arg_overflow_size_tls; @@ -90,8 +90,6 @@ bool msan_init_is_running; int msan_report_count = 0; -void (*death_callback)(void); - // Array of stack origins. // FIXME: make it resizable. static const uptr kNumStackOriginDescrs = 1024 * 1024; @@ -145,6 +143,7 @@ static void InitializeFlags() { // FIXME: test and enable. cf.check_printf = false; cf.intercept_tls_get_addr = true; + cf.exitcode = 77; OverrideCommonFlags(cf); } @@ -185,11 +184,18 @@ static void InitializeFlags() { if (common_flags()->help) parser.PrintFlagDescriptions(); - // Check flag values: - if (f->exit_code < 0 || f->exit_code > 127) { - Printf("Exit code not in [0, 128) range: %d\n", f->exit_code); - Die(); + // Check if deprecated exit_code MSan flag is set. + if (f->exit_code != -1) { + if (Verbosity()) + Printf("MSAN_OPTIONS=exit_code is deprecated! " + "Please use MSAN_OPTIONS=exitcode instead.\n"); + CommonFlags cf; + cf.CopyFrom(*common_flags()); + cf.exitcode = f->exit_code; + OverrideCommonFlags(cf); } + + // Check flag values: if (f->origin_history_size < 0 || f->origin_history_size > Origin::kMaxDepth) { Printf( @@ -217,9 +223,9 @@ void GetStackTrace(BufferedStackTrace *stack, uptr max_s, uptr pc, uptr bp, if (!t || !StackTrace::WillUseFastUnwind(request_fast_unwind)) { // Block reports from our interceptors during _Unwind_Backtrace. SymbolizerScope sym_scope; - return stack->Unwind(max_s, pc, bp, 0, 0, 0, request_fast_unwind); + return stack->Unwind(max_s, pc, bp, nullptr, 0, 0, request_fast_unwind); } - stack->Unwind(max_s, pc, bp, 0, t->stack_top(), t->stack_bottom(), + stack->Unwind(max_s, pc, bp, nullptr, t->stack_top(), t->stack_bottom(), request_fast_unwind); } @@ -299,7 +305,7 @@ u32 ChainOrigin(u32 id, StackTrace *stack) { return chained.raw_id(); } -} // namespace __msan +} // namespace __msan // Interface. @@ -369,11 +375,11 @@ void __msan_init() { msan_init_is_running = 1; SanitizerToolName = "MemorySanitizer"; - SetDieCallback(MsanDie); InitTlsSize(); - InitializeFlags(); CacheBinaryName(); + InitializeFlags(); + __sanitizer_set_report_path(common_flags()->log_path); InitializeInterceptors(); @@ -407,7 +413,9 @@ void __msan_init() { MsanTSDInit(MsanTSDDtor); - MsanThread *main_thread = MsanThread::Create(0, 0); + MsanAllocatorInit(); + + MsanThread *main_thread = MsanThread::Create(nullptr, nullptr); SetCurrentThread(main_thread); main_thread->ThreadStart(); @@ -421,10 +429,6 @@ void __msan_init() { msan_inited = 1; } -void __msan_set_exit_code(int exit_code) { - flags()->exit_code = exit_code; -} - void __msan_set_keep_going(int keep_going) { flags()->halt_on_error = !keep_going; } @@ -511,7 +515,7 @@ void __msan_partial_poison(const void* data, void* shadow, uptr size) { internal_memcpy((void*)MEM_TO_SHADOW((uptr)data), shadow, size); } -void __msan_load_unpoisoned(void *src, uptr size, void *dst) { +void __msan_load_unpoisoned(const void *src, uptr size, void *dst) { internal_memcpy(dst, src, size); __msan_unpoison(dst, size); } @@ -619,7 +623,7 @@ void __sanitizer_unaligned_store64(uu64 *p, u64 x) { } void __msan_set_death_callback(void (*callback)(void)) { - death_callback = callback; + SetUserDieCallback(callback); } #if !SANITIZER_SUPPORTS_WEAK_HOOKS @@ -635,4 +639,4 @@ void __sanitizer_print_stack_trace() { GET_FATAL_STACK_TRACE_PC_BP(StackTrace::GetCurrentPc(), GET_CURRENT_FRAME()); stack.Print(); } -} // extern "C" +} // extern "C" diff --git a/lib/msan/msan.h b/lib/msan/msan.h index cd8bc19f51ef..2079a592b7b9 100644 --- a/lib/msan/msan.h +++ b/lib/msan/msan.h @@ -52,6 +52,61 @@ const MappingDesc kMemoryLayout[] = { #define MEM_TO_SHADOW(mem) (((uptr)(mem)) & ~0x4000000000ULL) #define SHADOW_TO_ORIGIN(shadow) (((uptr)(shadow)) + 0x002000000000) +#elif SANITIZER_LINUX && defined(__aarch64__) + +// The mapping describes both 39-bits and 42-bits. AArch64 maps: +// - 0x00000000000-0x00010000000: 39/42-bits program own segments +// - 0x05500000000-0x05600000000: 39-bits PIE program segments +// - 0x07f80000000-0x07fffffffff: 39-bits libraries segments +// - 0x2aa00000000-0x2ab00000000: 42-bits PIE program segments +// - 0x3ff00000000-0x3ffffffffff: 42-bits libraries segments +// It is fragmented in multiples segments to increase the memory available +// on 42-bits (12.21% of total VMA available for 42-bits and 13.28 for +// 39 bits). +const MappingDesc kMemoryLayout[] = { + {0x00000000000ULL, 0x01000000000ULL, MappingDesc::INVALID, "invalid"}, + {0x01000000000ULL, 0x02000000000ULL, MappingDesc::SHADOW, "shadow-2"}, + {0x02000000000ULL, 0x03000000000ULL, MappingDesc::ORIGIN, "origin-2"}, + {0x03000000000ULL, 0x04000000000ULL, MappingDesc::SHADOW, "shadow-1"}, + {0x04000000000ULL, 0x05000000000ULL, MappingDesc::ORIGIN, "origin-1"}, + {0x05000000000ULL, 0x06000000000ULL, MappingDesc::APP, "app-1"}, + {0x06000000000ULL, 0x07000000000ULL, MappingDesc::INVALID, "invalid"}, + {0x07000000000ULL, 0x08000000000ULL, MappingDesc::APP, "app-2"}, + {0x08000000000ULL, 0x09000000000ULL, MappingDesc::INVALID, "invalid"}, + // The mappings below are used only for 42-bits VMA. + {0x09000000000ULL, 0x0A000000000ULL, MappingDesc::SHADOW, "shadow-3"}, + {0x0A000000000ULL, 0x0B000000000ULL, MappingDesc::ORIGIN, "origin-3"}, + {0x0B000000000ULL, 0x0F000000000ULL, MappingDesc::INVALID, "invalid"}, + {0x0F000000000ULL, 0x10000000000ULL, MappingDesc::APP, "app-3"}, + {0x10000000000ULL, 0x11000000000ULL, MappingDesc::INVALID, "invalid"}, + {0x11000000000ULL, 0x12000000000ULL, MappingDesc::APP, "app-4"}, + {0x12000000000ULL, 0x17000000000ULL, MappingDesc::INVALID, "invalid"}, + {0x17000000000ULL, 0x18000000000ULL, MappingDesc::SHADOW, "shadow-4"}, + {0x18000000000ULL, 0x19000000000ULL, MappingDesc::ORIGIN, "origin-4"}, + {0x19000000000ULL, 0x20000000000ULL, MappingDesc::INVALID, "invalid"}, + {0x20000000000ULL, 0x21000000000ULL, MappingDesc::APP, "app-5"}, + {0x21000000000ULL, 0x26000000000ULL, MappingDesc::INVALID, "invalid"}, + {0x26000000000ULL, 0x27000000000ULL, MappingDesc::SHADOW, "shadow-5"}, + {0x27000000000ULL, 0x28000000000ULL, MappingDesc::ORIGIN, "origin-5"}, + {0x28000000000ULL, 0x29000000000ULL, MappingDesc::SHADOW, "shadow-7"}, + {0x29000000000ULL, 0x2A000000000ULL, MappingDesc::ORIGIN, "origin-7"}, + {0x2A000000000ULL, 0x2B000000000ULL, MappingDesc::APP, "app-6"}, + {0x2B000000000ULL, 0x2C000000000ULL, MappingDesc::INVALID, "invalid"}, + {0x2C000000000ULL, 0x2D000000000ULL, MappingDesc::SHADOW, "shadow-6"}, + {0x2D000000000ULL, 0x2E000000000ULL, MappingDesc::ORIGIN, "origin-6"}, + {0x2E000000000ULL, 0x2F000000000ULL, MappingDesc::APP, "app-7"}, + {0x2F000000000ULL, 0x39000000000ULL, MappingDesc::INVALID, "invalid"}, + {0x39000000000ULL, 0x3A000000000ULL, MappingDesc::SHADOW, "shadow-9"}, + {0x3A000000000ULL, 0x3B000000000ULL, MappingDesc::ORIGIN, "origin-9"}, + {0x3B000000000ULL, 0x3C000000000ULL, MappingDesc::APP, "app-8"}, + {0x3C000000000ULL, 0x3D000000000ULL, MappingDesc::INVALID, "invalid"}, + {0x3D000000000ULL, 0x3E000000000ULL, MappingDesc::SHADOW, "shadow-8"}, + {0x3E000000000ULL, 0x3F000000000ULL, MappingDesc::ORIGIN, "origin-8"}, + {0x3F000000000ULL, 0x40000000000ULL, MappingDesc::APP, "app-9"}, +}; +# define MEM_TO_SHADOW(mem) ((uptr)mem ^ 0x6000000000ULL) +# define SHADOW_TO_ORIGIN(shadow) (((uptr)(shadow)) + 0x1000000000ULL) + #elif SANITIZER_LINUX && defined(__powerpc64__) const MappingDesc kMemoryLayout[] = { @@ -94,6 +149,7 @@ const MappingDesc kMemoryLayout[] = { #elif SANITIZER_LINUX && SANITIZER_WORDSIZE == 64 +#ifdef MSAN_LINUX_X86_64_OLD_MAPPING // Requries PIE binary and ASLR enabled. // Main thread stack and DSOs at 0x7f0000000000 (sometimes 0x7e0000000000). // Heap at 0x600000000000. @@ -105,6 +161,28 @@ const MappingDesc kMemoryLayout[] = { #define MEM_TO_SHADOW(mem) (((uptr)(mem)) & ~0x400000000000ULL) #define SHADOW_TO_ORIGIN(mem) (((uptr)(mem)) + 0x200000000000ULL) +#else // MSAN_LINUX_X86_64_OLD_MAPPING +// All of the following configurations are supported. +// ASLR disabled: main executable and DSOs at 0x555550000000 +// PIE and ASLR: main executable and DSOs at 0x7f0000000000 +// non-PIE: main executable below 0x100000000, DSOs at 0x7f0000000000 +// Heap at 0x700000000000. +const MappingDesc kMemoryLayout[] = { + {0x000000000000ULL, 0x010000000000ULL, MappingDesc::APP, "app-1"}, + {0x010000000000ULL, 0x100000000000ULL, MappingDesc::SHADOW, "shadow-2"}, + {0x100000000000ULL, 0x110000000000ULL, MappingDesc::INVALID, "invalid"}, + {0x110000000000ULL, 0x200000000000ULL, MappingDesc::ORIGIN, "origin-2"}, + {0x200000000000ULL, 0x300000000000ULL, MappingDesc::SHADOW, "shadow-3"}, + {0x300000000000ULL, 0x400000000000ULL, MappingDesc::ORIGIN, "origin-3"}, + {0x400000000000ULL, 0x500000000000ULL, MappingDesc::INVALID, "invalid"}, + {0x500000000000ULL, 0x510000000000ULL, MappingDesc::SHADOW, "shadow-1"}, + {0x510000000000ULL, 0x600000000000ULL, MappingDesc::APP, "app-2"}, + {0x600000000000ULL, 0x610000000000ULL, MappingDesc::ORIGIN, "origin-1"}, + {0x610000000000ULL, 0x700000000000ULL, MappingDesc::INVALID, "invalid"}, + {0x700000000000ULL, 0x800000000000ULL, MappingDesc::APP, "app-3"}}; +#define MEM_TO_SHADOW(mem) (((uptr)(mem)) ^ 0x500000000000ULL) +#define SHADOW_TO_ORIGIN(mem) (((uptr)(mem)) + 0x100000000000ULL) +#endif // MSAN_LINUX_X86_64_OLD_MAPPING #else #error "Unsupported platform" @@ -148,6 +226,7 @@ bool InitShadow(bool init_origins); char *GetProcSelfMaps(); void InitializeInterceptors(); +void MsanAllocatorInit(); void MsanAllocatorThreadFinish(); void *MsanCalloc(StackTrace *stack, uptr nmemb, uptr size); void *MsanReallocate(StackTrace *stack, void *oldp, uptr size, @@ -167,7 +246,6 @@ struct SymbolizerScope { ~SymbolizerScope() { ExitSymbolizer(); } }; -void MsanDie(); void PrintWarning(uptr pc, uptr bp); void PrintWarningWithOrigin(uptr pc, uptr bp, u32 origin); @@ -224,8 +302,6 @@ class ScopedThreadLocalStateBackup { u64 va_arg_overflow_size_tls; }; -extern void (*death_callback)(void); - void MsanTSDInit(void (*destructor)(void *tsd)); void *MsanTSDGet(); void MsanTSDSet(void *tsd); diff --git a/lib/msan/msan_allocator.cc b/lib/msan/msan_allocator.cc index 6df35664279f..b7d394729bfc 100644 --- a/lib/msan/msan_allocator.cc +++ b/lib/msan/msan_allocator.cc @@ -49,15 +49,21 @@ struct MsanMapUnmapCallback { typedef SizeClassAllocator32<0, SANITIZER_MMAP_RANGE_SIZE, sizeof(Metadata), SizeClassMap, kRegionSizeLog, ByteMap, MsanMapUnmapCallback> PrimaryAllocator; + #elif defined(__x86_64__) +#if SANITIZER_LINUX && !defined(MSAN_LINUX_X86_64_OLD_MAPPING) + static const uptr kAllocatorSpace = 0x700000000000ULL; +#else static const uptr kAllocatorSpace = 0x600000000000ULL; - static const uptr kAllocatorSize = 0x80000000000; // 8T. +#endif + static const uptr kAllocatorSize = 0x80000000000; // 8T. static const uptr kMetadataSize = sizeof(Metadata); static const uptr kMaxAllowedMallocSize = 8UL << 30; typedef SizeClassAllocator64<kAllocatorSpace, kAllocatorSize, kMetadataSize, DefaultSizeClassMap, MsanMapUnmapCallback> PrimaryAllocator; + #elif defined(__powerpc64__) static const uptr kAllocatorSpace = 0x300000000000; static const uptr kAllocatorSize = 0x020000000000; // 2T @@ -67,6 +73,16 @@ struct MsanMapUnmapCallback { typedef SizeClassAllocator64<kAllocatorSpace, kAllocatorSize, kMetadataSize, DefaultSizeClassMap, MsanMapUnmapCallback> PrimaryAllocator; +#elif defined(__aarch64__) + static const uptr kMaxAllowedMallocSize = 2UL << 30; // 2G + static const uptr kRegionSizeLog = 20; + static const uptr kNumRegions = SANITIZER_MMAP_RANGE_SIZE >> kRegionSizeLog; + typedef TwoLevelByteMap<(kNumRegions >> 12), 1 << 12> ByteMap; + typedef CompactSizeClassMap SizeClassMap; + + typedef SizeClassAllocator32<0, SANITIZER_MMAP_RANGE_SIZE, sizeof(Metadata), + SizeClassMap, kRegionSizeLog, ByteMap, + MsanMapUnmapCallback> PrimaryAllocator; #endif typedef SizeClassAllocatorLocalCache<PrimaryAllocator> AllocatorCache; typedef LargeMmapAllocator<MsanMapUnmapCallback> SecondaryAllocator; @@ -77,12 +93,7 @@ static Allocator allocator; static AllocatorCache fallback_allocator_cache; static SpinMutex fallback_mutex; -static int inited = 0; - -static inline void Init() { - if (inited) return; - __msan_init(); - inited = true; // this must happen before any threads are created. +void MsanAllocatorInit() { allocator.Init(common_flags()->allocator_may_return_null); } @@ -98,7 +109,6 @@ void MsanThreadLocalMallocStorage::CommitBack() { static void *MsanAllocate(StackTrace *stack, uptr size, uptr alignment, bool zeroise) { - Init(); if (size > kMaxAllowedMallocSize) { Report("WARNING: MemorySanitizer failed to allocate %p bytes\n", (void *)size); @@ -133,7 +143,6 @@ static void *MsanAllocate(StackTrace *stack, uptr size, uptr alignment, void MsanDeallocate(StackTrace *stack, void *p) { CHECK(p); - Init(); MSAN_FREE_HOOK(p); Metadata *meta = reinterpret_cast<Metadata *>(allocator.GetMetaData(p)); uptr size = meta->requested_size; @@ -160,10 +169,9 @@ void MsanDeallocate(StackTrace *stack, void *p) { } void *MsanCalloc(StackTrace *stack, uptr nmemb, uptr size) { - Init(); if (CallocShouldReturnNullDueToOverflow(size, nmemb)) return allocator.ReturnNullOrDie(); - return MsanReallocate(stack, 0, nmemb * size, sizeof(u64), true); + return MsanReallocate(stack, nullptr, nmemb * size, sizeof(u64), true); } void *MsanReallocate(StackTrace *stack, void *old_p, uptr new_size, @@ -172,7 +180,7 @@ void *MsanReallocate(StackTrace *stack, void *old_p, uptr new_size, return MsanAllocate(stack, new_size, alignment, zeroise); if (!new_size) { MsanDeallocate(stack, old_p); - return 0; + return nullptr; } Metadata *meta = reinterpret_cast<Metadata*>(allocator.GetMetaData(old_p)); uptr old_size = meta->requested_size; @@ -202,14 +210,14 @@ void *MsanReallocate(StackTrace *stack, void *old_p, uptr new_size, } static uptr AllocationSize(const void *p) { - if (p == 0) return 0; + if (!p) return 0; const void *beg = allocator.GetBlockBegin(p); if (beg != p) return 0; Metadata *b = (Metadata *)allocator.GetMetaData(p); return b->requested_size; } -} // namespace __msan +} // namespace __msan using namespace __msan; diff --git a/lib/msan/msan_chained_origin_depot.cc b/lib/msan/msan_chained_origin_depot.cc index c21e8e82746a..e2796fd46464 100644 --- a/lib/msan/msan_chained_origin_depot.cc +++ b/lib/msan/msan_chained_origin_depot.cc @@ -28,12 +28,15 @@ struct ChainedOriginDepotNode { u32 prev_id; typedef ChainedOriginDepotDesc args_type; + bool eq(u32 hash, const args_type &args) const { return here_id == args.here_id && prev_id == args.prev_id; } + static uptr storage_size(const args_type &args) { return sizeof(ChainedOriginDepotNode); } + /* This is murmur2 hash for the 64->32 bit case. It does not behave all that well because the keys have a very biased distribution (I've seen 7-element buckets with the table only 14% full). @@ -76,19 +79,22 @@ struct ChainedOriginDepotNode { here_id = args.here_id; prev_id = args.prev_id; } + args_type load() const { args_type ret = {here_id, prev_id}; return ret; } + struct Handle { ChainedOriginDepotNode *node_; - Handle() : node_(0) {} + Handle() : node_(nullptr) {} explicit Handle(ChainedOriginDepotNode *node) : node_(node) {} bool valid() { return node_; } u32 id() { return node_->id; } int here_id() { return node_->here_id; } int prev_id() { return node_->prev_id; } }; + Handle get_handle() { return Handle(this); } typedef Handle handle_type; @@ -123,4 +129,4 @@ void ChainedOriginDepotUnlockAll() { chainedOriginDepot.UnlockAll(); } -} // namespace __msan +} // namespace __msan diff --git a/lib/msan/msan_flags.inc b/lib/msan/msan_flags.inc index cb58ffc4aba7..a7ff6c586071 100644 --- a/lib/msan/msan_flags.inc +++ b/lib/msan/msan_flags.inc @@ -17,13 +17,15 @@ // MSAN_FLAG(Type, Name, DefaultValue, Description) // See COMMON_FLAG in sanitizer_flags.inc for more details. -MSAN_FLAG(int, exit_code, 77, "") +MSAN_FLAG(int, exit_code, -1, + "DEPRECATED. Use exitcode from common flags instead.") MSAN_FLAG(int, origin_history_size, Origin::kMaxDepth, "") MSAN_FLAG(int, origin_history_per_stack_limit, 20000, "") MSAN_FLAG(bool, poison_heap_with_zeroes, false, "") MSAN_FLAG(bool, poison_stack_with_zeroes, false, "") MSAN_FLAG(bool, poison_in_malloc, true, "") MSAN_FLAG(bool, poison_in_free, true, "") +MSAN_FLAG(bool, poison_in_dtor, false, "") MSAN_FLAG(bool, report_umrs, true, "") MSAN_FLAG(bool, wrap_signals, true, "") MSAN_FLAG(bool, print_stats, false, "") diff --git a/lib/msan/msan_interceptors.cc b/lib/msan/msan_interceptors.cc index 6d5a056a3bb3..fc28e080f262 100644 --- a/lib/msan/msan_interceptors.cc +++ b/lib/msan/msan_interceptors.cc @@ -166,7 +166,7 @@ INTERCEPTOR(int, posix_memalign, void **memptr, SIZE_T alignment, SIZE_T size) { GET_MALLOC_STACK_TRACE; CHECK_EQ(alignment & (alignment - 1), 0); CHECK_NE(memptr, 0); - *memptr = MsanReallocate(&stack, 0, size, alignment, false); + *memptr = MsanReallocate(&stack, nullptr, size, alignment, false); CHECK_NE(*memptr, 0); __msan_unpoison(memptr, sizeof(*memptr)); return 0; @@ -176,7 +176,7 @@ INTERCEPTOR(int, posix_memalign, void **memptr, SIZE_T alignment, SIZE_T size) { INTERCEPTOR(void *, memalign, SIZE_T boundary, SIZE_T size) { GET_MALLOC_STACK_TRACE; CHECK_EQ(boundary & (boundary - 1), 0); - void *ptr = MsanReallocate(&stack, 0, size, boundary, false); + void *ptr = MsanReallocate(&stack, nullptr, size, boundary, false); return ptr; } #define MSAN_MAYBE_INTERCEPT_MEMALIGN INTERCEPT_FUNCTION(memalign) @@ -187,21 +187,21 @@ INTERCEPTOR(void *, memalign, SIZE_T boundary, SIZE_T size) { INTERCEPTOR(void *, aligned_alloc, SIZE_T boundary, SIZE_T size) { GET_MALLOC_STACK_TRACE; CHECK_EQ(boundary & (boundary - 1), 0); - void *ptr = MsanReallocate(&stack, 0, size, boundary, false); + void *ptr = MsanReallocate(&stack, nullptr, size, boundary, false); return ptr; } INTERCEPTOR(void *, __libc_memalign, SIZE_T boundary, SIZE_T size) { GET_MALLOC_STACK_TRACE; CHECK_EQ(boundary & (boundary - 1), 0); - void *ptr = MsanReallocate(&stack, 0, size, boundary, false); + void *ptr = MsanReallocate(&stack, nullptr, size, boundary, false); DTLS_on_libc_memalign(ptr, size * boundary); return ptr; } INTERCEPTOR(void *, valloc, SIZE_T size) { GET_MALLOC_STACK_TRACE; - void *ptr = MsanReallocate(&stack, 0, size, GetPageSizeCached(), false); + void *ptr = MsanReallocate(&stack, nullptr, size, GetPageSizeCached(), false); return ptr; } @@ -214,7 +214,7 @@ INTERCEPTOR(void *, pvalloc, SIZE_T size) { // pvalloc(0) should allocate one page. size = PageSize; } - void *ptr = MsanReallocate(&stack, 0, size, PageSize, false); + void *ptr = MsanReallocate(&stack, nullptr, size, PageSize, false); return ptr; } #define MSAN_MAYBE_INTERCEPT_PVALLOC INTERCEPT_FUNCTION(pvalloc) @@ -224,14 +224,14 @@ INTERCEPTOR(void *, pvalloc, SIZE_T size) { INTERCEPTOR(void, free, void *ptr) { GET_MALLOC_STACK_TRACE; - if (ptr == 0) return; + if (!ptr) return; MsanDeallocate(&stack, ptr); } #if !SANITIZER_FREEBSD INTERCEPTOR(void, cfree, void *ptr) { GET_MALLOC_STACK_TRACE; - if (ptr == 0) return; + if (!ptr) return; MsanDeallocate(&stack, ptr); } #define MSAN_MAYBE_INTERCEPT_CFREE INTERCEPT_FUNCTION(cfree) @@ -245,9 +245,15 @@ INTERCEPTOR(uptr, malloc_usable_size, void *ptr) { #if !SANITIZER_FREEBSD // This function actually returns a struct by value, but we can't unpoison a -// temporary! The following is equivalent on all supported platforms, and we -// have a test to confirm that. +// temporary! The following is equivalent on all supported platforms but +// aarch64 (which uses a different register for sret value). We have a test +// to confirm that. INTERCEPTOR(void, mallinfo, __sanitizer_mallinfo *sret) { +#ifdef __aarch64__ + uptr r8; + asm volatile("mov %0,x8" : "=r" (r8)); + sret = reinterpret_cast<__sanitizer_mallinfo*>(r8); +#endif REAL(memset)(sret, 0, sizeof(*sret)); __msan_unpoison(sret, sizeof(*sret)); } @@ -994,7 +1000,7 @@ INTERCEPTOR(void *, realloc, void *ptr, SIZE_T size) { INTERCEPTOR(void *, malloc, SIZE_T size) { GET_MALLOC_STACK_TRACE; - return MsanReallocate(&stack, 0, size, sizeof(u64), false); + return MsanReallocate(&stack, nullptr, size, sizeof(u64), false); } void __msan_allocated_memory(const void *data, uptr size) { @@ -1005,6 +1011,19 @@ void __msan_allocated_memory(const void *data, uptr size) { } } +void __msan_copy_shadow(void *dest, const void *src, uptr n) { + GET_STORE_STACK_TRACE; + MoveShadowAndOrigin(dest, src, n, &stack); +} + +void __sanitizer_dtor_callback(const void *data, uptr size) { + GET_MALLOC_STACK_TRACE; + if (flags()->poison_in_dtor) { + stack.tag = STACK_TRACE_TAG_POISON; + PoisonMemory(data, size, &stack); + } +} + INTERCEPTOR(void *, mmap, void *addr, SIZE_T length, int prot, int flags, int fd, OFF_T offset) { if (msan_init_is_running) @@ -1015,7 +1034,7 @@ INTERCEPTOR(void *, mmap, void *addr, SIZE_T length, int prot, int flags, *__errno_location() = errno_EINVAL; return (void *)-1; } else { - addr = 0; + addr = nullptr; } } void *res = REAL(mmap)(addr, length, prot, flags, fd, offset); @@ -1033,7 +1052,7 @@ INTERCEPTOR(void *, mmap64, void *addr, SIZE_T length, int prot, int flags, *__errno_location() = errno_EINVAL; return (void *)-1; } else { - addr = 0; + addr = nullptr; } } void *res = REAL(mmap64)(addr, length, prot, flags, fd, offset); @@ -1069,7 +1088,7 @@ INTERCEPTOR(int, dladdr, void *addr, dlinfo *info) { INTERCEPTOR(char *, dlerror, int fake) { ENSURE_MSAN_INITED(); char *res = REAL(dlerror)(fake); - if (res != 0) __msan_unpoison(res, REAL(strlen)(res) + 1); + if (res) __msan_unpoison(res, REAL(strlen)(res) + 1); return res; } @@ -1084,6 +1103,8 @@ static int msan_dl_iterate_phdr_cb(__sanitizer_dl_phdr_info *info, SIZE_T size, void *data) { if (info) { __msan_unpoison(info, size); + if (info->dlpi_phdr && info->dlpi_phnum) + __msan_unpoison(info->dlpi_phdr, struct_ElfW_Phdr_sz * info->dlpi_phnum); if (info->dlpi_name) __msan_unpoison(info->dlpi_name, REAL(strlen)(info->dlpi_name) + 1); } @@ -1164,7 +1185,7 @@ INTERCEPTOR(int, sigaction, int signo, const __sanitizer_sigaction *act, CHECK_LT(signo, kMaxSignals); uptr old_cb = atomic_load(&sigactions[signo], memory_order_relaxed); __sanitizer_sigaction new_act; - __sanitizer_sigaction *pnew_act = act ? &new_act : 0; + __sanitizer_sigaction *pnew_act = act ? &new_act : nullptr; if (act) { REAL(memcpy)(pnew_act, act, sizeof(__sanitizer_sigaction)); uptr cb = (uptr)pnew_act->sigaction; @@ -1221,7 +1242,7 @@ INTERCEPTOR(int, pthread_create, void *th, void *attr, void *(*callback)(void*), void * param) { ENSURE_MSAN_INITED(); // for GetTlsSize() __sanitizer_pthread_attr_t myattr; - if (attr == 0) { + if (!attr) { pthread_attr_init(&myattr); attr = &myattr; } @@ -1327,6 +1348,28 @@ INTERCEPTOR(int, fork, void) { return pid; } +INTERCEPTOR(int, openpty, int *amaster, int *aslave, char *name, + const void *termp, const void *winp) { + ENSURE_MSAN_INITED(); + InterceptorScope interceptor_scope; + int res = REAL(openpty)(amaster, aslave, name, termp, winp); + if (!res) { + __msan_unpoison(amaster, sizeof(*amaster)); + __msan_unpoison(aslave, sizeof(*aslave)); + } + return res; +} + +INTERCEPTOR(int, forkpty, int *amaster, char *name, const void *termp, + const void *winp) { + ENSURE_MSAN_INITED(); + InterceptorScope interceptor_scope; + int res = REAL(forkpty)(amaster, name, termp, winp); + if (res != -1) + __msan_unpoison(amaster, sizeof(*amaster)); + return res; +} + struct MSanInterceptorContext { bool in_interceptor_scope; }; @@ -1338,7 +1381,7 @@ int OnExit() { return 0; } -} // namespace __msan +} // namespace __msan // A version of CHECK_UNPOISONED using a saved scope value. Used in common // interceptors. @@ -1391,10 +1434,11 @@ int OnExit() { } while (false) // FIXME #define COMMON_INTERCEPTOR_BLOCK_REAL(name) REAL(name) #define COMMON_INTERCEPTOR_ON_EXIT(ctx) OnExit() -#define COMMON_INTERCEPTOR_LIBRARY_LOADED(filename, handle) \ - do { \ - link_map *map = GET_LINK_MAP_BY_DLOPEN_HANDLE((handle)); \ - if (map) ForEachMappedRegion(map, __msan_unpoison); \ +#define COMMON_INTERCEPTOR_LIBRARY_LOADED(filename, handle) \ + do { \ + link_map *map = GET_LINK_MAP_BY_DLOPEN_HANDLE((handle)); \ + if (filename && map) \ + ForEachMappedRegion(map, __msan_unpoison); \ } while (false) #define COMMON_INTERCEPTOR_GET_TLS_RANGE(begin, end) \ @@ -1591,7 +1635,9 @@ void InitializeInterceptors() { INTERCEPT_FUNCTION(__cxa_atexit); INTERCEPT_FUNCTION(shmat); INTERCEPT_FUNCTION(fork); + INTERCEPT_FUNCTION(openpty); + INTERCEPT_FUNCTION(forkpty); inited = 1; } -} // namespace __msan +} // namespace __msan diff --git a/lib/msan/msan_interface_internal.h b/lib/msan/msan_interface_internal.h index f4d37d96c5b5..c1e02ce72bf4 100644 --- a/lib/msan/msan_interface_internal.h +++ b/lib/msan/msan_interface_internal.h @@ -27,7 +27,7 @@ SANITIZER_INTERFACE_ATTRIBUTE void __msan_init(); // Print a warning and maybe return. -// This function can die based on flags()->exit_code. +// This function can die based on common_flags()->exitcode. SANITIZER_INTERFACE_ATTRIBUTE void __msan_warning(); @@ -106,10 +106,6 @@ int __msan_origin_is_descendant_or_same(u32 this_id, u32 prev_id); SANITIZER_INTERFACE_ATTRIBUTE void __msan_clear_on_return(); -// Default: -1 (don't exit on error). -SANITIZER_INTERFACE_ATTRIBUTE -void __msan_set_exit_code(int exit_code); - SANITIZER_INTERFACE_ATTRIBUTE void __msan_set_keep_going(int keep_going); @@ -140,6 +136,11 @@ void __msan_partial_poison(const void* data, void* shadow, uptr size); SANITIZER_INTERFACE_ATTRIBUTE void __msan_allocated_memory(const void* data, uptr size); +// Tell MSan about newly destroyed memory. Memory will be marked +// uninitialized. +SANITIZER_INTERFACE_ATTRIBUTE +void __sanitizer_dtor_callback(const void* data, uptr size); + SANITIZER_INTERFACE_ATTRIBUTE u16 __sanitizer_unaligned_load16(const uu16 *p); @@ -160,6 +161,9 @@ void __sanitizer_unaligned_store64(uu64 *p, u64 x); SANITIZER_INTERFACE_ATTRIBUTE void __msan_set_death_callback(void (*callback)(void)); + +SANITIZER_INTERFACE_ATTRIBUTE +void __msan_copy_shadow(void *dst, const void *src, uptr size); } // extern "C" #endif // MSAN_INTERFACE_INTERNAL_H diff --git a/lib/msan/msan_linux.cc b/lib/msan/msan_linux.cc index 7025ef6c812d..ab3be91fcf8d 100644 --- a/lib/msan/msan_linux.cc +++ b/lib/msan/msan_linux.cc @@ -56,7 +56,7 @@ static bool CheckMemoryRangeAvailability(uptr beg, uptr size) { static bool ProtectMemoryRange(uptr beg, uptr size, const char *name) { if (size > 0) { void *addr = MmapNoAccess(beg, size, name); - if (beg == 0 && addr != 0) { + if (beg == 0 && addr) { // Depending on the kernel configuration, we may not be able to protect // the page at address zero. uptr gap = 16 * GetPageSizeCached(); @@ -119,12 +119,18 @@ bool InitShadow(bool init_origins) { return false; } + const uptr maxVirtualAddress = GetMaxVirtualAddress(); + for (unsigned i = 0; i < kMemoryLayoutSize; ++i) { uptr start = kMemoryLayout[i].start; uptr end = kMemoryLayout[i].end; uptr size= end - start; MappingDesc::Type type = kMemoryLayout[i].type; + // Check if the segment should be mapped based on platform constraints. + if (start >= maxVirtualAddress) + continue; + bool map = type == MappingDesc::SHADOW || (init_origins && type == MappingDesc::ORIGIN); bool protect = type == MappingDesc::INVALID || @@ -151,20 +157,13 @@ bool InitShadow(bool init_origins) { return true; } -void MsanDie() { - if (common_flags()->coverage) - __sanitizer_cov_dump(); - if (death_callback) - death_callback(); - internal__exit(flags()->exit_code); -} - static void MsanAtExit(void) { if (flags()->print_stats && (flags()->atexit || msan_report_count > 0)) ReportStats(); if (msan_report_count > 0) { ReportAtExitStatistics(); - if (flags()->exit_code) _exit(flags()->exit_code); + if (common_flags()->exitcode) + internal__exit(common_flags()->exitcode); } } @@ -211,6 +210,6 @@ void MsanTSDDtor(void *tsd) { MsanThread::TSDDtor(tsd); } -} // namespace __msan +} // namespace __msan -#endif // SANITIZER_FREEBSD || SANITIZER_LINUX +#endif // SANITIZER_FREEBSD || SANITIZER_LINUX diff --git a/lib/msan/msan_new_delete.cc b/lib/msan/msan_new_delete.cc index c8bc0651b507..540100316693 100644 --- a/lib/msan/msan_new_delete.cc +++ b/lib/msan/msan_new_delete.cc @@ -45,9 +45,9 @@ void *operator new[](size_t size, std::nothrow_t const&) { OPERATOR_NEW_BODY; } if (ptr) MsanDeallocate(&stack, ptr) INTERCEPTOR_ATTRIBUTE -void operator delete(void *ptr) throw() { OPERATOR_DELETE_BODY; } +void operator delete(void *ptr) NOEXCEPT { OPERATOR_DELETE_BODY; } INTERCEPTOR_ATTRIBUTE -void operator delete[](void *ptr) throw() { OPERATOR_DELETE_BODY; } +void operator delete[](void *ptr) NOEXCEPT { OPERATOR_DELETE_BODY; } INTERCEPTOR_ATTRIBUTE void operator delete(void *ptr, std::nothrow_t const&) { OPERATOR_DELETE_BODY; } INTERCEPTOR_ATTRIBUTE diff --git a/lib/msan/msan_thread.h b/lib/msan/msan_thread.h index bc605b89a505..ed22e67edd50 100644 --- a/lib/msan/msan_thread.h +++ b/lib/msan/msan_thread.h @@ -32,7 +32,7 @@ class MsanThread { uptr stack_bottom() { return stack_bottom_; } uptr tls_begin() { return tls_begin_; } uptr tls_end() { return tls_end_; } - bool IsMainThread() { return start_routine_ == 0; } + bool IsMainThread() { return start_routine_ == nullptr; } bool AddrIsInStack(uptr addr) { return addr >= stack_bottom_ && addr < stack_top_; diff --git a/lib/msan/tests/CMakeLists.txt b/lib/msan/tests/CMakeLists.txt index bf16a16bcf20..087b1afbd5b3 100644 --- a/lib/msan/tests/CMakeLists.txt +++ b/lib/msan/tests/CMakeLists.txt @@ -18,13 +18,13 @@ set(MSAN_UNITTEST_HEADERS ../../../include/sanitizer/msan_interface.h ) set(MSAN_UNITTEST_COMMON_CFLAGS - -I${COMPILER_RT_LIBCXX_PATH}/include + -nostdinc++ + -isystem ${COMPILER_RT_LIBCXX_PATH}/include ${COMPILER_RT_TEST_CFLAGS} ${COMPILER_RT_GTEST_CFLAGS} -I${COMPILER_RT_SOURCE_DIR}/include -I${COMPILER_RT_SOURCE_DIR}/lib -I${COMPILER_RT_SOURCE_DIR}/lib/msan - -stdlib=libc++ -g -O2 -fno-exceptions @@ -44,15 +44,13 @@ set(MSAN_UNITTEST_INSTRUMENTED_CFLAGS ) set(MSAN_UNITTEST_LINK_FLAGS -fsanitize=memory + # Don't need -stdlib=libc++ because we explicitly list libc++.so in the linker + # inputs. # FIXME: we build libcxx without cxxabi and need libstdc++ to provide it. -lstdc++ ) append_list_if(COMPILER_RT_HAS_LIBDL -ldl MSAN_UNITTEST_LINK_FLAGS) -set(MSAN_LOADABLE_LINK_FLAGS - -fsanitize=memory - -shared -) # Compile source for the given architecture, using compiler # options in ${ARGN}, and add it to the object list. @@ -90,12 +88,6 @@ set_target_properties(MsanUnitTests PROPERTIES FOLDER "MSan unit tests") # Adds MSan unit tests and benchmarks for architecture. macro(add_msan_tests_for_arch arch kind) - set(LIBCXX_PREFIX ${CMAKE_CURRENT_BINARY_DIR}/../libcxx_msan${kind}) - add_custom_libcxx(libcxx_msan${kind} ${LIBCXX_PREFIX} - DEPS ${MSAN_RUNTIME_LIBRARIES} - CFLAGS ${MSAN_LIBCXX_CFLAGS} ${ARGN}) - set(MSAN_LIBCXX_SO ${LIBCXX_PREFIX}/lib/libc++.so) - # Build gtest instrumented with MSan. set(MSAN_INST_GTEST) msan_compile(MSAN_INST_GTEST ${COMPILER_RT_GTEST_SOURCE} ${arch} "${kind}" @@ -111,7 +103,7 @@ macro(add_msan_tests_for_arch arch kind) # Instrumented loadable module objects. set(MSAN_INST_LOADABLE_OBJECTS) msan_compile(MSAN_INST_LOADABLE_OBJECTS ${MSAN_LOADABLE_SOURCE} ${arch} "${kind}" - ${MSAN_UNITTEST_INSTRUMENTED_CFLAGS} ${ARGN}) + ${MSAN_UNITTEST_INSTRUMENTED_CFLAGS} "-fPIC" ${ARGN}) # Instrumented loadable library tests. set(MSAN_LOADABLE_SO) @@ -120,24 +112,31 @@ macro(add_msan_tests_for_arch arch kind) DEPS ${MSAN_INST_LOADABLE_OBJECTS}) set(MSAN_TEST_OBJECTS ${MSAN_INST_TEST_OBJECTS} ${MSAN_INST_GTEST}) - set(MSAN_TEST_DEPS ${MSAN_TEST_OBJECTS} libcxx_msan${kind} + set(MSAN_TEST_DEPS ${MSAN_TEST_OBJECTS} libcxx_msan_${arch} ${MSAN_LOADABLE_SO}) if(NOT COMPILER_RT_STANDALONE_BUILD) list(APPEND MSAN_TEST_DEPS msan) endif() get_target_flags_for_arch(${arch} TARGET_LINK_FLAGS) add_compiler_rt_test(MsanUnitTests "Msan-${arch}${kind}-Test" ${arch} - OBJECTS ${MSAN_TEST_OBJECTS} ${MSAN_LIBCXX_SO} - DEPS ${MSAN_TEST_DEPS} - LINK_FLAGS ${MSAN_UNITTEST_LINK_FLAGS} - ${TARGET_LINK_FLAGS} - "-Wl,-rpath=${CMAKE_CURRENT_BINARY_DIR}" - "-Wl,-rpath=${LIBCXX_PREFIX}/lib") + OBJECTS ${MSAN_TEST_OBJECTS} ${MSAN_LIBCXX_SO} + DEPS ${MSAN_TEST_DEPS} + LINK_FLAGS ${MSAN_UNITTEST_LINK_FLAGS} + ${TARGET_LINK_FLAGS} + "-Wl,-rpath=${CMAKE_CURRENT_BINARY_DIR}" + "-Wl,-rpath=${LIBCXX_PREFIX}/lib") endmacro() # We should only build MSan unit tests if we can build instrumented libcxx. if(COMPILER_RT_CAN_EXECUTE_TESTS AND COMPILER_RT_HAS_LIBCXX_SOURCES) foreach(arch ${MSAN_SUPPORTED_ARCH}) + get_target_flags_for_arch(${arch} TARGET_CFLAGS) + set(LIBCXX_PREFIX ${CMAKE_CURRENT_BINARY_DIR}/../libcxx_msan_${arch}) + add_custom_libcxx(libcxx_msan_${arch} ${LIBCXX_PREFIX} + DEPS ${MSAN_RUNTIME_LIBRARIES} + CFLAGS ${MSAN_LIBCXX_CFLAGS} ${TARGET_CFLAGS}) + set(MSAN_LIBCXX_SO ${LIBCXX_PREFIX}/lib/libc++.so) + add_msan_tests_for_arch(${arch} "") add_msan_tests_for_arch(${arch} "-with-call" -mllvm -msan-instrumentation-with-call-threshold=0) diff --git a/lib/msan/tests/msan_test.cc b/lib/msan/tests/msan_test.cc index 00dd20a3d775..b7162b3c081b 100644 --- a/lib/msan/tests/msan_test.cc +++ b/lib/msan/tests/msan_test.cc @@ -1883,7 +1883,7 @@ TEST(MemorySanitizer, swprintf) { ASSERT_EQ(buff[1], '2'); ASSERT_EQ(buff[2], '3'); ASSERT_EQ(buff[6], '7'); - ASSERT_EQ(buff[7], 0); + ASSERT_EQ(buff[7], L'\0'); EXPECT_POISONED(buff[8]); } @@ -1952,6 +1952,16 @@ TEST(MemorySanitizer, wcsnrtombs) { EXPECT_POISONED(buff[2]); } +TEST(MemorySanitizer, wcrtomb) { + wchar_t x = L'a'; + char buff[10]; + mbstate_t mbs; + memset(&mbs, 0, sizeof(mbs)); + size_t res = wcrtomb(buff, x, &mbs); + EXPECT_EQ(res, (size_t)1); + EXPECT_EQ(buff[0], 'a'); +} + TEST(MemorySanitizer, wmemset) { wchar_t x[25]; break_optimization(x); @@ -2876,6 +2886,8 @@ static void GetPathToLoadable(char *buf, size_t sz) { static const char basename[] = "libmsan_loadable.mips64.so"; #elif defined(__mips64) static const char basename[] = "libmsan_loadable.mips64el.so"; +#elif defined(__aarch64__) + static const char basename[] = "libmsan_loadable.aarch64.so"; #endif int res = snprintf(buf, sz, "%.*s/%s", (int)dir_len, program_path, basename); @@ -2982,6 +2994,14 @@ static void *SmallStackThread_threadfn(void* data) { return 0; } +#ifdef PTHREAD_STACK_MIN +# define SMALLSTACKSIZE PTHREAD_STACK_MIN +# define SMALLPRESTACKSIZE PTHREAD_STACK_MIN +#else +# define SMALLSTACKSIZE 64 * 1024 +# define SMALLPRESTACKSIZE 16 * 1024 +#endif + TEST(MemorySanitizer, SmallStackThread) { pthread_attr_t attr; pthread_t t; @@ -2989,7 +3009,7 @@ TEST(MemorySanitizer, SmallStackThread) { int res; res = pthread_attr_init(&attr); ASSERT_EQ(0, res); - res = pthread_attr_setstacksize(&attr, 64 * 1024); + res = pthread_attr_setstacksize(&attr, SMALLSTACKSIZE); ASSERT_EQ(0, res); res = pthread_create(&t, &attr, SmallStackThread_threadfn, NULL); ASSERT_EQ(0, res); @@ -3006,7 +3026,7 @@ TEST(MemorySanitizer, SmallPreAllocatedStackThread) { res = pthread_attr_init(&attr); ASSERT_EQ(0, res); void *stack; - const size_t kStackSize = 16 * 1024; + const size_t kStackSize = SMALLPRESTACKSIZE; res = posix_memalign(&stack, 4096, kStackSize); ASSERT_EQ(0, res); res = pthread_attr_setstack(&attr, stack, kStackSize); diff --git a/lib/profile/CMakeLists.txt b/lib/profile/CMakeLists.txt index d03409fc45b7..1b10ade0eee6 100644 --- a/lib/profile/CMakeLists.txt +++ b/lib/profile/CMakeLists.txt @@ -1,27 +1,71 @@ + +CHECK_CXX_SOURCE_COMPILES(" +#ifdef _MSC_VER +#include <Intrin.h> /* Workaround for PR19898. */ +#include <windows.h> +#endif +int main() { +#ifdef _MSC_VER + volatile LONG val = 1; + MemoryBarrier(); + InterlockedCompareExchange(&val, 0, 1); + InterlockedIncrement(&val); + InterlockedDecrement(&val); +#else + volatile unsigned long val = 1; + __sync_synchronize(); + __sync_val_compare_and_swap(&val, 1, 0); + __sync_add_and_fetch(&val, 1); + __sync_sub_and_fetch(&val, 1); +#endif + return 0; + } +" COMPILER_RT_TARGET_HAS_ATOMICS) + add_custom_target(profile) set(PROFILE_SOURCES GCDAProfiling.c InstrProfiling.c + InstrProfilingValue.c InstrProfilingBuffer.c InstrProfilingFile.c + InstrProfilingWriter.c InstrProfilingPlatformDarwin.c + InstrProfilingPlatformLinux.c InstrProfilingPlatformOther.c InstrProfilingRuntime.cc InstrProfilingUtil.c) +if(UNIX) + set(EXTRA_FLAGS + -fPIC + -Wno-pedantic) +else() + set(EXTRA_FLAGS + -fPIC) +endif() + +if(COMPILER_RT_TARGET_HAS_ATOMICS) + set(EXTRA_FLAGS + ${EXTRA_FLAGS} + -DCOMPILER_RT_HAS_ATOMICS=1) +endif() + if(APPLE) - add_compiler_rt_osx_static_runtime(clang_rt.profile_osx + add_compiler_rt_runtime(clang_rt.profile + STATIC + OS ${PROFILE_SUPPORTED_OS} ARCHS ${PROFILE_SUPPORTED_ARCH} - SOURCES ${PROFILE_SOURCES}) - add_dependencies(profile clang_rt.profile_osx) + SOURCES ${PROFILE_SOURCES} + PARENT_TARGET profile) else() - foreach(arch ${PROFILE_SUPPORTED_ARCH}) - add_compiler_rt_runtime(clang_rt.profile-${arch} ${arch} STATIC - CFLAGS -fPIC - SOURCES ${PROFILE_SOURCES}) - add_dependencies(profile clang_rt.profile-${arch}) - endforeach() + add_compiler_rt_runtime(clang_rt.profile + STATIC + ARCHS ${PROFILE_SUPPORTED_ARCH} + CFLAGS ${EXTRA_FLAGS} + SOURCES ${PROFILE_SOURCES} + PARENT_TARGET profile) endif() add_dependencies(compiler-rt profile) diff --git a/lib/profile/InstrProfData.inc b/lib/profile/InstrProfData.inc new file mode 100644 index 000000000000..48dae506cabb --- /dev/null +++ b/lib/profile/InstrProfData.inc @@ -0,0 +1,735 @@ +/*===-- InstrProfData.inc - instr profiling runtime structures -----------=== *\ +|* +|* The LLVM Compiler Infrastructure +|* +|* This file is distributed under the University of Illinois Open Source +|* License. See LICENSE.TXT for details. +|* +\*===----------------------------------------------------------------------===*/ +/* + * This is the master file that defines all the data structure, signature, + * constant literals that are shared across profiling runtime library, + * compiler (instrumentation), and host tools (reader/writer). The entities + * defined in this file affect the profile runtime ABI, the raw profile format, + * or both. + * + * The file has two identical copies. The master copy lives in LLVM and + * the other one sits in compiler-rt/lib/profile directory. To make changes + * in this file, first modify the master copy and copy it over to compiler-rt. + * Testing of any change in this file can start only after the two copies are + * synced up. + * + * The first part of the file includes macros that defines types, names, and + * initializers for the member fields of the core data structures. The field + * declarations for one structure is enabled by defining the field activation + * macro associated with that structure. Only one field activation record + * can be defined at one time and the rest definitions will be filtered out by + * the preprocessor. + * + * Examples of how the template is used to instantiate structure definition: + * 1. To declare a structure: + * + * struct ProfData { + * #define INSTR_PROF_DATA(Type, LLVMType, Name, Initializer) \ + * Type Name; + * #include "llvm/ProfileData/InstrProfData.inc" + * }; + * + * 2. To construct LLVM type arrays for the struct type: + * + * Type *DataTypes[] = { + * #define INSTR_PROF_DATA(Type, LLVMType, Name, Initializer) \ + * LLVMType, + * #include "llvm/ProfileData/InstrProfData.inc" + * }; + * + * 4. To construct constant array for the initializers: + * #define INSTR_PROF_DATA(Type, LLVMType, Name, Initializer) \ + * Initializer, + * Constant *ConstantVals[] = { + * #include "llvm/ProfileData/InstrProfData.inc" + * }; + * + * + * The second part of the file includes definitions all other entities that + * are related to runtime ABI and format. When no field activation macro is + * defined, this file can be included to introduce the definitions. + * +\*===----------------------------------------------------------------------===*/ + +/* INSTR_PROF_DATA start. */ +/* Definition of member fields of the per-function control structure. */ +#ifndef INSTR_PROF_DATA +#define INSTR_PROF_DATA(Type, LLVMType, Name, Initializer) +#else +#define INSTR_PROF_DATA_DEFINED +#endif + +INSTR_PROF_DATA(const uint32_t, llvm::Type::getInt32Ty(Ctx), NameSize, \ + ConstantInt::get(llvm::Type::getInt32Ty(Ctx), \ + NamePtr->getType()->getPointerElementType()->getArrayNumElements())) +INSTR_PROF_DATA(const uint32_t, llvm::Type::getInt32Ty(Ctx), NumCounters, \ + ConstantInt::get(llvm::Type::getInt32Ty(Ctx), NumCounters)) +INSTR_PROF_DATA(const uint64_t, llvm::Type::getInt64Ty(Ctx), FuncHash, \ + ConstantInt::get(llvm::Type::getInt64Ty(Ctx), \ + Inc->getHash()->getZExtValue())) +INSTR_PROF_DATA(const IntPtrT, llvm::Type::getInt8PtrTy(Ctx), NamePtr, \ + ConstantExpr::getBitCast(NamePtr, llvm::Type::getInt8PtrTy(Ctx))) +INSTR_PROF_DATA(const IntPtrT, llvm::Type::getInt64PtrTy(Ctx), CounterPtr, \ + ConstantExpr::getBitCast(CounterPtr, \ + llvm::Type::getInt64PtrTy(Ctx))) +INSTR_PROF_DATA(const IntPtrT, llvm::Type::getInt8PtrTy(Ctx), FunctionPointer, \ + FunctionAddr) +INSTR_PROF_DATA(IntPtrT, llvm::Type::getInt8PtrTy(Ctx), Values, \ + ConstantPointerNull::get(Int8PtrTy)) +INSTR_PROF_DATA(const uint16_t, Int16ArrayTy, NumValueSites[IPVK_Last+1], \ + ConstantArray::get(Int16ArrayTy, Int16ArrayVals)) +#undef INSTR_PROF_DATA +/* INSTR_PROF_DATA end. */ + +/* INSTR_PROF_RAW_HEADER start */ +/* Definition of member fields of the raw profile header data structure. */ +#ifndef INSTR_PROF_RAW_HEADER +#define INSTR_PROF_RAW_HEADER(Type, Name, Initializer) +#else +#define INSTR_PROF_DATA_DEFINED +#endif +INSTR_PROF_RAW_HEADER(uint64_t, Magic, __llvm_profile_get_magic()) +INSTR_PROF_RAW_HEADER(uint64_t, Version, __llvm_profile_get_version()) +INSTR_PROF_RAW_HEADER(uint64_t, DataSize, DataSize) +INSTR_PROF_RAW_HEADER(uint64_t, CountersSize, CountersSize) +INSTR_PROF_RAW_HEADER(uint64_t, NamesSize, NamesSize) +INSTR_PROF_RAW_HEADER(uint64_t, CountersDelta, (uintptr_t)CountersBegin) +INSTR_PROF_RAW_HEADER(uint64_t, NamesDelta, (uintptr_t)NamesBegin) +INSTR_PROF_RAW_HEADER(uint64_t, ValueKindLast, IPVK_Last) +INSTR_PROF_RAW_HEADER(uint64_t, ValueDataSize, ValueDataSize) +INSTR_PROF_RAW_HEADER(uint64_t, ValueDataDelta, (uintptr_t)ValueDataBegin) +#undef INSTR_PROF_RAW_HEADER +/* INSTR_PROF_RAW_HEADER end */ + +/* VALUE_PROF_FUNC_PARAM start */ +/* Definition of parameter types of the runtime API used to do value profiling + * for a given value site. + */ +#ifndef VALUE_PROF_FUNC_PARAM +#define VALUE_PROF_FUNC_PARAM(ArgType, ArgName, ArgLLVMType) +#define INSTR_PROF_COMMA +#else +#define INSTR_PROF_DATA_DEFINED +#define INSTR_PROF_COMMA , +#endif +VALUE_PROF_FUNC_PARAM(uint64_t, TargetValue, Type::getInt64Ty(Ctx)) \ + INSTR_PROF_COMMA +VALUE_PROF_FUNC_PARAM(void *, Data, Type::getInt8PtrTy(Ctx)) INSTR_PROF_COMMA +VALUE_PROF_FUNC_PARAM(uint32_t, CounterIndex, Type::getInt32Ty(Ctx)) +#undef VALUE_PROF_FUNC_PARAM +#undef INSTR_PROF_COMMA +/* VALUE_PROF_FUNC_PARAM end */ + +/* VALUE_PROF_KIND start */ +#ifndef VALUE_PROF_KIND +#define VALUE_PROF_KIND(Enumerator, Value) +#else +#define INSTR_PROF_DATA_DEFINED +#endif +VALUE_PROF_KIND(IPVK_IndirectCallTarget, 0) +/* These two kinds must be the last to be + * declared. This is to make sure the string + * array created with the template can be + * indexed with the kind value. + */ +VALUE_PROF_KIND(IPVK_First, IPVK_IndirectCallTarget) +VALUE_PROF_KIND(IPVK_Last, IPVK_IndirectCallTarget) + +#undef VALUE_PROF_KIND +/* VALUE_PROF_KIND end */ + +/* COVMAP_FUNC_RECORD start */ +/* Definition of member fields of the function record structure in coverage + * map. + */ +#ifndef COVMAP_FUNC_RECORD +#define COVMAP_FUNC_RECORD(Type, LLVMType, Name, Initializer) +#else +#define INSTR_PROF_DATA_DEFINED +#endif +COVMAP_FUNC_RECORD(const IntPtrT, llvm::Type::getInt8PtrTy(Ctx), \ + NamePtr, llvm::ConstantExpr::getBitCast(NamePtr, \ + llvm::Type::getInt8PtrTy(Ctx))) +COVMAP_FUNC_RECORD(const uint32_t, llvm::Type::getInt32Ty(Ctx), NameSize, \ + llvm::ConstantInt::get(llvm::Type::getInt32Ty(Ctx),\ + NameValue.size())) +COVMAP_FUNC_RECORD(const uint32_t, llvm::Type::getInt32Ty(Ctx), DataSize, \ + llvm::ConstantInt::get(llvm::Type::getInt32Ty(Ctx),\ + CoverageMapping.size())) +COVMAP_FUNC_RECORD(const uint64_t, llvm::Type::getInt64Ty(Ctx), FuncHash, \ + llvm::ConstantInt::get(llvm::Type::getInt64Ty(Ctx), FuncHash)) +#undef COVMAP_FUNC_RECORD +/* COVMAP_FUNC_RECORD end. */ + + +#ifdef INSTR_PROF_VALUE_PROF_DATA +#define INSTR_PROF_DATA_DEFINED + +/*! + * This is the header of the data structure that defines the on-disk + * layout of the value profile data of a particular kind for one function. + */ +typedef struct ValueProfRecord { + /* The kind of the value profile record. */ + uint32_t Kind; + /* + * The number of value profile sites. It is guaranteed to be non-zero; + * otherwise the record for this kind won't be emitted. + */ + uint32_t NumValueSites; + /* + * The first element of the array that stores the number of profiled + * values for each value site. The size of the array is NumValueSites. + * Since NumValueSites is greater than zero, there is at least one + * element in the array. + */ + uint8_t SiteCountArray[1]; + + /* + * The fake declaration is for documentation purpose only. + * Align the start of next field to be on 8 byte boundaries. + uint8_t Padding[X]; + */ + + /* The array of value profile data. The size of the array is the sum + * of all elements in SiteCountArray[]. + InstrProfValueData ValueData[]; + */ + +#ifdef __cplusplus + /*! + * \brief Return the number of value sites. + */ + uint32_t getNumValueSites() const { return NumValueSites; } + /*! + * \brief Read data from this record and save it to Record. + */ + void deserializeTo(InstrProfRecord &Record, + InstrProfRecord::ValueMapType *VMap); + /* + * In-place byte swap: + * Do byte swap for this instance. \c Old is the original order before + * the swap, and \c New is the New byte order. + */ + void swapBytes(support::endianness Old, support::endianness New); +#endif +} ValueProfRecord; + +/*! + * Per-function header/control data structure for value profiling + * data in indexed format. + */ +typedef struct ValueProfData { + /* + * Total size in bytes including this field. It must be a multiple + * of sizeof(uint64_t). + */ + uint32_t TotalSize; + /* + *The number of value profile kinds that has value profile data. + * In this implementation, a value profile kind is considered to + * have profile data if the number of value profile sites for the + * kind is not zero. More aggressively, the implementation can + * choose to check the actual data value: if none of the value sites + * has any profiled values, the kind can be skipped. + */ + uint32_t NumValueKinds; + + /* + * Following are a sequence of variable length records. The prefix/header + * of each record is defined by ValueProfRecord type. The number of + * records is NumValueKinds. + * ValueProfRecord Record_1; + * ValueProfRecord Record_N; + */ + +#if __cplusplus + /*! + * Return the total size in bytes of the on-disk value profile data + * given the data stored in Record. + */ + static uint32_t getSize(const InstrProfRecord &Record); + /*! + * Return a pointer to \c ValueProfData instance ready to be streamed. + */ + static std::unique_ptr<ValueProfData> + serializeFrom(const InstrProfRecord &Record); + /*! + * Check the integrity of the record. Return the error code when + * an error is detected, otherwise return instrprof_error::success. + */ + instrprof_error checkIntegrity(); + /*! + * Return a pointer to \c ValueProfileData instance ready to be read. + * All data in the instance are properly byte swapped. The input + * data is assumed to be in little endian order. + */ + static ErrorOr<std::unique_ptr<ValueProfData>> + getValueProfData(const unsigned char *SrcBuffer, + const unsigned char *const SrcBufferEnd, + support::endianness SrcDataEndianness); + /*! + * Swap byte order from \c Endianness order to host byte order. + */ + void swapBytesToHost(support::endianness Endianness); + /*! + * Swap byte order from host byte order to \c Endianness order. + */ + void swapBytesFromHost(support::endianness Endianness); + /*! + * Return the total size of \c ValueProfileData. + */ + uint32_t getSize() const { return TotalSize; } + /*! + * Read data from this data and save it to \c Record. + */ + void deserializeTo(InstrProfRecord &Record, + InstrProfRecord::ValueMapType *VMap); + void operator delete(void *ptr) { ::operator delete(ptr); } +#endif +} ValueProfData; + +/* + * The closure is designed to abstact away two types of value profile data: + * - InstrProfRecord which is the primary data structure used to + * represent profile data in host tools (reader, writer, and profile-use) + * - value profile runtime data structure suitable to be used by C + * runtime library. + * + * Both sources of data need to serialize to disk/memory-buffer in common + * format: ValueProfData. The abstraction allows compiler-rt's raw profiler + * writer to share the same format and code with indexed profile writer. + * + * For documentation of the member methods below, refer to corresponding methods + * in class InstrProfRecord. + */ +typedef struct ValueProfRecordClosure { + const void *Record; + uint32_t (*GetNumValueKinds)(const void *Record); + uint32_t (*GetNumValueSites)(const void *Record, uint32_t VKind); + uint32_t (*GetNumValueData)(const void *Record, uint32_t VKind); + uint32_t (*GetNumValueDataForSite)(const void *R, uint32_t VK, uint32_t S); + + /* + * After extracting the value profile data from the value profile record, + * this method is used to map the in-memory value to on-disk value. If + * the method is null, value will be written out untranslated. + */ + uint64_t (*RemapValueData)(uint32_t, uint64_t Value); + void (*GetValueForSite)(const void *R, InstrProfValueData *Dst, uint32_t K, + uint32_t S, uint64_t (*Mapper)(uint32_t, uint64_t)); + ValueProfData *(*AllocValueProfData)(size_t TotalSizeInBytes); +} ValueProfRecordClosure; + +/* + * A wrapper struct that represents value profile runtime data. + * Like InstrProfRecord class which is used by profiling host tools, + * ValueProfRuntimeRecord also implements the abstract intefaces defined in + * ValueProfRecordClosure so that the runtime data can be serialized using + * shared C implementation. In this structure, NumValueSites and Nodes + * members are the primary fields while other fields hold the derived + * information for fast implementation of closure interfaces. + */ +typedef struct ValueProfRuntimeRecord { + /* Number of sites for each value profile kind. */ + const uint16_t *NumValueSites; + /* An array of linked-list headers. The size of of the array is the + * total number of value profile sites : sum(NumValueSites[*])). Each + * linked-list stores the values profiled for a value profile site. */ + ValueProfNode **Nodes; + + /* Total number of value profile kinds which have at least one + * value profile sites. */ + uint32_t NumValueKinds; + /* An array recording the number of values tracked at each site. + * The size of the array is TotalNumValueSites. */ + uint8_t *SiteCountArray[IPVK_Last + 1]; + ValueProfNode **NodesKind[IPVK_Last + 1]; +} ValueProfRuntimeRecord; + +/* Forward declarations of C interfaces. */ +int initializeValueProfRuntimeRecord(ValueProfRuntimeRecord *RuntimeRecord, + const uint16_t *NumValueSites, + ValueProfNode **Nodes); +void finalizeValueProfRuntimeRecord(ValueProfRuntimeRecord *RuntimeRecord); +uint32_t getValueProfDataSizeRT(const ValueProfRuntimeRecord *Record); +ValueProfData * +serializeValueProfDataFromRT(const ValueProfRuntimeRecord *Record, + ValueProfData *Dst); +uint32_t getNumValueKindsRT(const void *R); + +#undef INSTR_PROF_VALUE_PROF_DATA +#endif /* INSTR_PROF_VALUE_PROF_DATA */ + + +#ifdef INSTR_PROF_COMMON_API_IMPL +#define INSTR_PROF_DATA_DEFINED +#ifdef __cplusplus +#define INSTR_PROF_INLINE inline +#else +#define INSTR_PROF_INLINE +#endif + +#ifndef offsetof +#define offsetof(TYPE, MEMBER) ((size_t) &((TYPE *)0)->MEMBER) +#endif + +/*! + * \brief Return the \c ValueProfRecord header size including the + * padding bytes. + */ +INSTR_PROF_INLINE +uint32_t getValueProfRecordHeaderSize(uint32_t NumValueSites) { + uint32_t Size = offsetof(ValueProfRecord, SiteCountArray) + + sizeof(uint8_t) * NumValueSites; + /* Round the size to multiple of 8 bytes. */ + Size = (Size + 7) & ~7; + return Size; +} + +/*! + * \brief Return the total size of the value profile record including the + * header and the value data. + */ +INSTR_PROF_INLINE +uint32_t getValueProfRecordSize(uint32_t NumValueSites, + uint32_t NumValueData) { + return getValueProfRecordHeaderSize(NumValueSites) + + sizeof(InstrProfValueData) * NumValueData; +} + +/*! + * \brief Return the pointer to the start of value data array. + */ +INSTR_PROF_INLINE +InstrProfValueData *getValueProfRecordValueData(ValueProfRecord *This) { + return (InstrProfValueData *)((char *)This + getValueProfRecordHeaderSize( + This->NumValueSites)); +} + +/*! + * \brief Return the total number of value data for \c This record. + */ +INSTR_PROF_INLINE +uint32_t getValueProfRecordNumValueData(ValueProfRecord *This) { + uint32_t NumValueData = 0; + uint32_t I; + for (I = 0; I < This->NumValueSites; I++) + NumValueData += This->SiteCountArray[I]; + return NumValueData; +} + +/*! + * \brief Use this method to advance to the next \c This \c ValueProfRecord. + */ +INSTR_PROF_INLINE +ValueProfRecord *getValueProfRecordNext(ValueProfRecord *This) { + uint32_t NumValueData = getValueProfRecordNumValueData(This); + return (ValueProfRecord *)((char *)This + + getValueProfRecordSize(This->NumValueSites, + NumValueData)); +} + +/*! + * \brief Return the first \c ValueProfRecord instance. + */ +INSTR_PROF_INLINE +ValueProfRecord *getFirstValueProfRecord(ValueProfData *This) { + return (ValueProfRecord *)((char *)This + sizeof(ValueProfData)); +} + +/* Closure based interfaces. */ + +/*! + * Return the total size in bytes of the on-disk value profile data + * given the data stored in Record. + */ +uint32_t getValueProfDataSize(ValueProfRecordClosure *Closure) { + uint32_t Kind; + uint32_t TotalSize = sizeof(ValueProfData); + const void *Record = Closure->Record; + uint32_t NumValueKinds = Closure->GetNumValueKinds(Record); + if (NumValueKinds == 0) + return TotalSize; + + for (Kind = IPVK_First; Kind <= IPVK_Last; Kind++) { + uint32_t NumValueSites = Closure->GetNumValueSites(Record, Kind); + if (!NumValueSites) + continue; + TotalSize += getValueProfRecordSize(NumValueSites, + Closure->GetNumValueData(Record, Kind)); + } + return TotalSize; +} + +/*! + * Extract value profile data of a function for the profile kind \c ValueKind + * from the \c Closure and serialize the data into \c This record instance. + */ +void serializeValueProfRecordFrom(ValueProfRecord *This, + ValueProfRecordClosure *Closure, + uint32_t ValueKind, uint32_t NumValueSites) { + uint32_t S; + const void *Record = Closure->Record; + This->Kind = ValueKind; + This->NumValueSites = NumValueSites; + InstrProfValueData *DstVD = getValueProfRecordValueData(This); + + for (S = 0; S < NumValueSites; S++) { + uint32_t ND = Closure->GetNumValueDataForSite(Record, ValueKind, S); + This->SiteCountArray[S] = ND; + Closure->GetValueForSite(Record, DstVD, ValueKind, S, + Closure->RemapValueData); + DstVD += ND; + } +} + +/*! + * Extract value profile data of a function from the \c Closure + * and serialize the data into \c DstData if it is not NULL or heap + * memory allocated by the \c Closure's allocator method. + */ +ValueProfData *serializeValueProfDataFrom(ValueProfRecordClosure *Closure, + ValueProfData *DstData) { + uint32_t Kind; + uint32_t TotalSize = getValueProfDataSize(Closure); + + ValueProfData *VPD = + DstData ? DstData : Closure->AllocValueProfData(TotalSize); + + VPD->TotalSize = TotalSize; + VPD->NumValueKinds = Closure->GetNumValueKinds(Closure->Record); + ValueProfRecord *VR = getFirstValueProfRecord(VPD); + for (Kind = IPVK_First; Kind <= IPVK_Last; Kind++) { + uint32_t NumValueSites = Closure->GetNumValueSites(Closure->Record, Kind); + if (!NumValueSites) + continue; + serializeValueProfRecordFrom(VR, Closure, Kind, NumValueSites); + VR = getValueProfRecordNext(VR); + } + return VPD; +} + +/* + * The value profiler runtime library stores the value profile data + * for a given function in \c NumValueSites and \c Nodes structures. + * \c ValueProfRuntimeRecord class is used to encapsulate the runtime + * profile data and provides fast interfaces to retrieve the profile + * information. This interface is used to initialize the runtime record + * and pre-compute the information needed for efficient implementation + * of callbacks required by ValueProfRecordClosure class. + */ +int initializeValueProfRuntimeRecord(ValueProfRuntimeRecord *RuntimeRecord, + const uint16_t *NumValueSites, + ValueProfNode **Nodes) { + unsigned I, J, S = 0, NumValueKinds = 0; + RuntimeRecord->NumValueSites = NumValueSites; + RuntimeRecord->Nodes = Nodes; + for (I = 0; I <= IPVK_Last; I++) { + uint16_t N = NumValueSites[I]; + if (!N) { + RuntimeRecord->SiteCountArray[I] = 0; + continue; + } + NumValueKinds++; + RuntimeRecord->SiteCountArray[I] = (uint8_t *)calloc(N, 1); + if (!RuntimeRecord->SiteCountArray[I]) + return 1; + RuntimeRecord->NodesKind[I] = Nodes ? &Nodes[S] : NULL; + for (J = 0; J < N; J++) { + /* Compute value count for each site. */ + uint32_t C = 0; + ValueProfNode *Site = Nodes ? RuntimeRecord->NodesKind[I][J] : NULL; + while (Site) { + C++; + Site = Site->Next; + } + if (C > UCHAR_MAX) + C = UCHAR_MAX; + RuntimeRecord->SiteCountArray[I][J] = C; + } + S += N; + } + RuntimeRecord->NumValueKinds = NumValueKinds; + return 0; +} + +void finalizeValueProfRuntimeRecord(ValueProfRuntimeRecord *RuntimeRecord) { + unsigned I; + for (I = 0; I <= IPVK_Last; I++) { + if (RuntimeRecord->SiteCountArray[I]) + free(RuntimeRecord->SiteCountArray[I]); + } +} + +/* ValueProfRecordClosure Interface implementation for + * ValueProfDataRuntimeRecord. */ +uint32_t getNumValueKindsRT(const void *R) { + return ((const ValueProfRuntimeRecord *)R)->NumValueKinds; +} + +uint32_t getNumValueSitesRT(const void *R, uint32_t VK) { + return ((const ValueProfRuntimeRecord *)R)->NumValueSites[VK]; +} + +uint32_t getNumValueDataForSiteRT(const void *R, uint32_t VK, uint32_t S) { + const ValueProfRuntimeRecord *Record = (const ValueProfRuntimeRecord *)R; + return Record->SiteCountArray[VK][S]; +} + +uint32_t getNumValueDataRT(const void *R, uint32_t VK) { + unsigned I, S = 0; + const ValueProfRuntimeRecord *Record = (const ValueProfRuntimeRecord *)R; + if (Record->SiteCountArray[VK] == 0) + return 0; + for (I = 0; I < Record->NumValueSites[VK]; I++) + S += Record->SiteCountArray[VK][I]; + return S; +} + +void getValueForSiteRT(const void *R, InstrProfValueData *Dst, uint32_t VK, + uint32_t S, uint64_t (*Mapper)(uint32_t, uint64_t)) { + unsigned I, N = 0; + const ValueProfRuntimeRecord *Record = (const ValueProfRuntimeRecord *)R; + N = getNumValueDataForSiteRT(R, VK, S); + if (N == 0) + return; + ValueProfNode *VNode = Record->NodesKind[VK][S]; + for (I = 0; I < N; I++) { + Dst[I] = VNode->VData; + VNode = VNode->Next; + } +} + +ValueProfData *allocValueProfDataRT(size_t TotalSizeInBytes) { + return (ValueProfData *)calloc(TotalSizeInBytes, 1); +} + +static ValueProfRecordClosure RTRecordClosure = {0, + getNumValueKindsRT, + getNumValueSitesRT, + getNumValueDataRT, + getNumValueDataForSiteRT, + 0, + getValueForSiteRT, + allocValueProfDataRT}; + +/* + * Return the size of ValueProfData structure to store data + * recorded in the runtime record. + */ +uint32_t getValueProfDataSizeRT(const ValueProfRuntimeRecord *Record) { + RTRecordClosure.Record = Record; + return getValueProfDataSize(&RTRecordClosure); +} + +/* + * Return a ValueProfData instance that stores the data collected + * from runtime. If \c DstData is provided by the caller, the value + * profile data will be store in *DstData and DstData is returned, + * otherwise the method will allocate space for the value data and + * return pointer to the newly allocated space. + */ +ValueProfData * +serializeValueProfDataFromRT(const ValueProfRuntimeRecord *Record, + ValueProfData *DstData) { + RTRecordClosure.Record = Record; + return serializeValueProfDataFrom(&RTRecordClosure, DstData); +} + + +#undef INSTR_PROF_COMMON_API_IMPL +#endif /* INSTR_PROF_COMMON_API_IMPL */ + +/*============================================================================*/ + + +#ifndef INSTR_PROF_DATA_DEFINED + +#ifndef INSTR_PROF_DATA_INC_ +#define INSTR_PROF_DATA_INC_ + +/* Helper macros. */ +#define INSTR_PROF_SIMPLE_QUOTE(x) #x +#define INSTR_PROF_QUOTE(x) INSTR_PROF_SIMPLE_QUOTE(x) +#define INSTR_PROF_SIMPLE_CONCAT(x,y) x ## y +#define INSTR_PROF_CONCAT(x,y) INSTR_PROF_SIMPLE_CONCAT(x,y) + +/* Magic number to detect file format and endianness. + * Use 255 at one end, since no UTF-8 file can use that character. Avoid 0, + * so that utilities, like strings, don't grab it as a string. 129 is also + * invalid UTF-8, and high enough to be interesting. + * Use "lprofr" in the centre to stand for "LLVM Profile Raw", or "lprofR" + * for 32-bit platforms. + */ +#define INSTR_PROF_RAW_MAGIC_64 (uint64_t)255 << 56 | (uint64_t)'l' << 48 | \ + (uint64_t)'p' << 40 | (uint64_t)'r' << 32 | (uint64_t)'o' << 24 | \ + (uint64_t)'f' << 16 | (uint64_t)'r' << 8 | (uint64_t)129 +#define INSTR_PROF_RAW_MAGIC_32 (uint64_t)255 << 56 | (uint64_t)'l' << 48 | \ + (uint64_t)'p' << 40 | (uint64_t)'r' << 32 | (uint64_t)'o' << 24 | \ + (uint64_t)'f' << 16 | (uint64_t)'R' << 8 | (uint64_t)129 + +/* Raw profile format version. */ +#define INSTR_PROF_RAW_VERSION 2 + +/* Runtime section names and name strings. */ +#define INSTR_PROF_DATA_SECT_NAME __llvm_prf_data +#define INSTR_PROF_NAME_SECT_NAME __llvm_prf_names +#define INSTR_PROF_CNTS_SECT_NAME __llvm_prf_cnts + +#define INSTR_PROF_DATA_SECT_NAME_STR \ + INSTR_PROF_QUOTE(INSTR_PROF_DATA_SECT_NAME) +#define INSTR_PROF_NAME_SECT_NAME_STR \ + INSTR_PROF_QUOTE(INSTR_PROF_NAME_SECT_NAME) +#define INSTR_PROF_CNTS_SECT_NAME_STR \ + INSTR_PROF_QUOTE(INSTR_PROF_CNTS_SECT_NAME) + +/* Macros to define start/stop section symbol for a given + * section on Linux. For instance + * INSTR_PROF_SECT_START(INSTR_PROF_DATA_SECT_NAME) will + * expand to __start___llvm_prof_data + */ +#define INSTR_PROF_SECT_START(Sect) \ + INSTR_PROF_CONCAT(__start_,Sect) +#define INSTR_PROF_SECT_STOP(Sect) \ + INSTR_PROF_CONCAT(__stop_,Sect) + +/* Value Profiling API linkage name. */ +#define INSTR_PROF_VALUE_PROF_FUNC __llvm_profile_instrument_target +#define INSTR_PROF_VALUE_PROF_FUNC_STR \ + INSTR_PROF_QUOTE(INSTR_PROF_VALUE_PROF_FUNC) + +/* InstrProfile per-function control data alignment. */ +#define INSTR_PROF_DATA_ALIGNMENT 8 + +/* The data structure that represents a tracked value by the + * value profiler. + */ +typedef struct InstrProfValueData { + /* Profiled value. */ + uint64_t Value; + /* Number of times the value appears in the training run. */ + uint64_t Count; +} InstrProfValueData; + +/* This is an internal data structure used by value profiler. It + * is defined here to allow serialization code sharing by LLVM + * to be used in unit test. + */ +typedef struct ValueProfNode { + InstrProfValueData VData; + struct ValueProfNode *Next; +} ValueProfNode; + +#endif /* INSTR_PROF_DATA_INC_ */ + +#else +#undef INSTR_PROF_DATA_DEFINED +#endif + diff --git a/lib/profile/InstrProfiling.c b/lib/profile/InstrProfiling.c index 8d010df28f18..58778aeec16a 100644 --- a/lib/profile/InstrProfiling.c +++ b/lib/profile/InstrProfiling.c @@ -8,41 +8,61 @@ \*===----------------------------------------------------------------------===*/ #include "InstrProfiling.h" +#include "InstrProfilingInternal.h" +#include <limits.h> +#include <stdio.h> +#include <stdlib.h> #include <string.h> +#define INSTR_PROF_VALUE_PROF_DATA +#include "InstrProfData.inc" -__attribute__((visibility("hidden"))) -uint64_t __llvm_profile_get_magic(void) { - /* Magic number to detect file format and endianness. - * - * Use 255 at one end, since no UTF-8 file can use that character. Avoid 0, - * so that utilities, like strings, don't grab it as a string. 129 is also - * invalid UTF-8, and high enough to be interesting. - * - * Use "lprofr" in the centre to stand for "LLVM Profile Raw", or "lprofR" - * for 32-bit platforms. - */ - unsigned char R = sizeof(void *) == sizeof(uint64_t) ? 'r' : 'R'; - return - (uint64_t)255 << 56 | - (uint64_t)'l' << 48 | - (uint64_t)'p' << 40 | - (uint64_t)'r' << 32 | - (uint64_t)'o' << 24 | - (uint64_t)'f' << 16 | - (uint64_t) R << 8 | - (uint64_t)129; +char *(*GetEnvHook)(const char *) = 0; + +COMPILER_RT_VISIBILITY uint64_t __llvm_profile_get_magic(void) { + return sizeof(void *) == sizeof(uint64_t) ? (INSTR_PROF_RAW_MAGIC_64) + : (INSTR_PROF_RAW_MAGIC_32); +} + +/* Return the number of bytes needed to add to SizeInBytes to make it + * the result a multiple of 8. + */ +COMPILER_RT_VISIBILITY uint8_t +__llvm_profile_get_num_padding_bytes(uint64_t SizeInBytes) { + return 7 & (sizeof(uint64_t) - SizeInBytes % sizeof(uint64_t)); } -__attribute__((visibility("hidden"))) -uint64_t __llvm_profile_get_version(void) { - /* This should be bumped any time the output format changes. */ - return 1; +COMPILER_RT_VISIBILITY uint64_t __llvm_profile_get_version(void) { + return INSTR_PROF_RAW_VERSION; } -__attribute__((visibility("hidden"))) -void __llvm_profile_reset_counters(void) { +COMPILER_RT_VISIBILITY void __llvm_profile_reset_counters(void) { uint64_t *I = __llvm_profile_begin_counters(); uint64_t *E = __llvm_profile_end_counters(); - memset(I, 0, sizeof(uint64_t)*(E - I)); + memset(I, 0, sizeof(uint64_t) * (E - I)); + + const __llvm_profile_data *DataBegin = __llvm_profile_begin_data(); + const __llvm_profile_data *DataEnd = __llvm_profile_end_data(); + const __llvm_profile_data *DI; + for (DI = DataBegin; DI != DataEnd; ++DI) { + uint64_t CurrentVSiteCount = 0; + uint32_t VKI, i; + if (!DI->Values) + continue; + + ValueProfNode **ValueCounters = (ValueProfNode **)DI->Values; + + for (VKI = IPVK_First; VKI <= IPVK_Last; ++VKI) + CurrentVSiteCount += DI->NumValueSites[VKI]; + + for (i = 0; i < CurrentVSiteCount; ++i) { + ValueProfNode *CurrentVNode = ValueCounters[i]; + + while (CurrentVNode) { + CurrentVNode->VData.Count = 0; + CurrentVNode = CurrentVNode->Next; + } + } + } } + diff --git a/lib/profile/InstrProfiling.h b/lib/profile/InstrProfiling.h index 3778a88893e6..d27ca569d535 100644 --- a/lib/profile/InstrProfiling.h +++ b/lib/profile/InstrProfiling.h @@ -10,32 +10,31 @@ #ifndef PROFILE_INSTRPROFILING_H_ #define PROFILE_INSTRPROFILING_H_ -#if defined(__FreeBSD__) && defined(__i386__) - -/* System headers define 'size_t' incorrectly on x64 FreeBSD (prior to - * FreeBSD 10, r232261) when compiled in 32-bit mode. - */ -#define PRIu64 "llu" -typedef unsigned int uint32_t; -typedef unsigned long long uint64_t; -typedef uint32_t uintptr_t; - -#else /* defined(__FreeBSD__) && defined(__i386__) */ - -#include <inttypes.h> -#include <stdint.h> - -#endif /* defined(__FreeBSD__) && defined(__i386__) */ +#include "InstrProfilingPort.h" +#include "InstrProfData.inc" + +enum ValueKind { +#define VALUE_PROF_KIND(Enumerator, Value) Enumerator = Value, +#include "InstrProfData.inc" +}; + +typedef void *IntPtrT; +typedef struct COMPILER_RT_ALIGNAS(INSTR_PROF_DATA_ALIGNMENT) + __llvm_profile_data { +#define INSTR_PROF_DATA(Type, LLVMType, Name, Initializer) Type Name; +#include "InstrProfData.inc" +} __llvm_profile_data; -#define PROFILE_HEADER_SIZE 7 +typedef struct __llvm_profile_header { +#define INSTR_PROF_RAW_HEADER(Type, Name, Initializer) Type Name; +#include "InstrProfData.inc" +} __llvm_profile_header; -typedef struct __llvm_profile_data { - const uint32_t NameSize; - const uint32_t NumCounters; - const uint64_t FuncHash; - const char *const Name; - uint64_t *const Counters; -} __llvm_profile_data; +/*! + * \brief Get number of bytes necessary to pad the argument to eight + * byte boundary. + */ +uint8_t __llvm_profile_get_num_padding_bytes(uint64_t SizeInBytes); /*! * \brief Get required size for profile buffer. @@ -58,9 +57,37 @@ uint64_t *__llvm_profile_begin_counters(void); uint64_t *__llvm_profile_end_counters(void); /*! + * \brief Clear profile counters to zero. + * + */ +void __llvm_profile_reset_counters(void); + +/*! + * \brief Counts the number of times a target value is seen. + * + * Records the target value for the CounterIndex if not seen before. Otherwise, + * increments the counter associated w/ the target value. + * void __llvm_profile_instrument_target(uint64_t TargetValue, void *Data, + * uint32_t CounterIndex); + */ +void INSTR_PROF_VALUE_PROF_FUNC( +#define VALUE_PROF_FUNC_PARAM(ArgType, ArgName, ArgLLVMType) ArgType ArgName +#include "InstrProfData.inc" +); + +/*! + * \brief Prepares the value profiling data for output. + * + * Returns an array of pointers to value profile data. + */ +struct ValueProfData; +struct ValueProfData **__llvm_profile_gather_value_data(uint64_t *Size); + +/*! * \brief Write instrumentation data to the current file. * - * Writes to the file with the last name given to \a __llvm_profile_set_filename(), + * Writes to the file with the last name given to \a * + * __llvm_profile_set_filename(), * or if it hasn't been called, the \c LLVM_PROFILE_FILE environment variable, * or if that's not set, the last name given to * \a __llvm_profile_override_default_filename(), or if that's not set, diff --git a/lib/profile/InstrProfilingBuffer.c b/lib/profile/InstrProfilingBuffer.c index 3c429c8a85ea..4227ca6b66ea 100644 --- a/lib/profile/InstrProfilingBuffer.c +++ b/lib/profile/InstrProfilingBuffer.c @@ -10,9 +10,7 @@ #include "InstrProfiling.h" #include "InstrProfilingInternal.h" -#include <string.h> - -__attribute__((visibility("hidden"))) +COMPILER_RT_VISIBILITY uint64_t __llvm_profile_get_size_for_buffer(void) { const __llvm_profile_data *DataBegin = __llvm_profile_begin_data(); const __llvm_profile_data *DataEnd = __llvm_profile_end_data(); @@ -27,78 +25,28 @@ uint64_t __llvm_profile_get_size_for_buffer(void) { #define PROFILE_RANGE_SIZE(Range) (Range##End - Range##Begin) -__attribute__((visibility("hidden"))) +COMPILER_RT_VISIBILITY uint64_t __llvm_profile_get_size_for_buffer_internal( - const __llvm_profile_data *DataBegin, - const __llvm_profile_data *DataEnd, const uint64_t *CountersBegin, - const uint64_t *CountersEnd, const char *NamesBegin, - const char *NamesEnd) { + const __llvm_profile_data *DataBegin, const __llvm_profile_data *DataEnd, + const uint64_t *CountersBegin, const uint64_t *CountersEnd, + const char *NamesBegin, const char *NamesEnd) { /* Match logic in __llvm_profile_write_buffer(). */ const uint64_t NamesSize = PROFILE_RANGE_SIZE(Names) * sizeof(char); - const uint64_t Padding = sizeof(uint64_t) - NamesSize % sizeof(uint64_t); - return sizeof(uint64_t) * PROFILE_HEADER_SIZE + - PROFILE_RANGE_SIZE(Data) * sizeof(__llvm_profile_data) + - PROFILE_RANGE_SIZE(Counters) * sizeof(uint64_t) + - NamesSize + Padding; + const uint8_t Padding = __llvm_profile_get_num_padding_bytes(NamesSize); + return sizeof(__llvm_profile_header) + + PROFILE_RANGE_SIZE(Data) * sizeof(__llvm_profile_data) + + PROFILE_RANGE_SIZE(Counters) * sizeof(uint64_t) + NamesSize + Padding; } -__attribute__((visibility("hidden"))) -int __llvm_profile_write_buffer(char *Buffer) { - /* Match logic in __llvm_profile_get_size_for_buffer(). - * Match logic in __llvm_profile_write_file(). - */ - const __llvm_profile_data *DataBegin = __llvm_profile_begin_data(); - const __llvm_profile_data *DataEnd = __llvm_profile_end_data(); - const uint64_t *CountersBegin = __llvm_profile_begin_counters(); - const uint64_t *CountersEnd = __llvm_profile_end_counters(); - const char *NamesBegin = __llvm_profile_begin_names(); - const char *NamesEnd = __llvm_profile_end_names(); - - return __llvm_profile_write_buffer_internal(Buffer, DataBegin, DataEnd, - CountersBegin, CountersEnd, - NamesBegin, NamesEnd); +COMPILER_RT_VISIBILITY int __llvm_profile_write_buffer(char *Buffer) { + return llvmWriteProfData(llvmBufferWriter, Buffer, 0, 0); } -__attribute__((visibility("hidden"))) -int __llvm_profile_write_buffer_internal( +COMPILER_RT_VISIBILITY int __llvm_profile_write_buffer_internal( char *Buffer, const __llvm_profile_data *DataBegin, const __llvm_profile_data *DataEnd, const uint64_t *CountersBegin, const uint64_t *CountersEnd, const char *NamesBegin, const char *NamesEnd) { - /* Match logic in __llvm_profile_get_size_for_buffer(). - * Match logic in __llvm_profile_write_file(). - */ - - /* Calculate size of sections. */ - const uint64_t DataSize = DataEnd - DataBegin; - const uint64_t CountersSize = CountersEnd - CountersBegin; - const uint64_t NamesSize = NamesEnd - NamesBegin; - const uint64_t Padding = sizeof(uint64_t) - NamesSize % sizeof(uint64_t); - - /* Enough zeroes for padding. */ - const char Zeroes[sizeof(uint64_t)] = {0}; - - /* Create the header. */ - uint64_t Header[PROFILE_HEADER_SIZE]; - Header[0] = __llvm_profile_get_magic(); - Header[1] = __llvm_profile_get_version(); - Header[2] = DataSize; - Header[3] = CountersSize; - Header[4] = NamesSize; - Header[5] = (uintptr_t)CountersBegin; - Header[6] = (uintptr_t)NamesBegin; - - /* Write the data. */ -#define UPDATE_memcpy(Data, Size) \ - do { \ - memcpy(Buffer, Data, Size); \ - Buffer += Size; \ - } while (0) - UPDATE_memcpy(Header, PROFILE_HEADER_SIZE * sizeof(uint64_t)); - UPDATE_memcpy(DataBegin, DataSize * sizeof(__llvm_profile_data)); - UPDATE_memcpy(CountersBegin, CountersSize * sizeof(uint64_t)); - UPDATE_memcpy(NamesBegin, NamesSize * sizeof(char)); - UPDATE_memcpy(Zeroes, Padding * sizeof(char)); -#undef UPDATE_memcpy - - return 0; + return llvmWriteProfDataImpl(llvmBufferWriter, Buffer, DataBegin, DataEnd, + CountersBegin, CountersEnd, 0, 0, NamesBegin, + NamesEnd); } diff --git a/lib/profile/InstrProfilingFile.c b/lib/profile/InstrProfilingFile.c index 68e8c7b07871..4ea7fbf9738a 100644 --- a/lib/profile/InstrProfilingFile.c +++ b/lib/profile/InstrProfilingFile.c @@ -8,6 +8,7 @@ \*===----------------------------------------------------------------------===*/ #include "InstrProfiling.h" +#include "InstrProfilingInternal.h" #include "InstrProfilingUtil.h" #include <errno.h> #include <stdio.h> @@ -16,47 +17,39 @@ #define UNCONST(ptr) ((void *)(uintptr_t)(ptr)) -static int writeFile(FILE *File) { - /* Match logic in __llvm_profile_write_buffer(). */ - const __llvm_profile_data *DataBegin = __llvm_profile_begin_data(); - const __llvm_profile_data *DataEnd = __llvm_profile_end_data(); - const uint64_t *CountersBegin = __llvm_profile_begin_counters(); - const uint64_t *CountersEnd = __llvm_profile_end_counters(); - const char *NamesBegin = __llvm_profile_begin_names(); - const char *NamesEnd = __llvm_profile_end_names(); - - /* Calculate size of sections. */ - const uint64_t DataSize = DataEnd - DataBegin; - const uint64_t CountersSize = CountersEnd - CountersBegin; - const uint64_t NamesSize = NamesEnd - NamesBegin; - const uint64_t Padding = sizeof(uint64_t) - NamesSize % sizeof(uint64_t); - - /* Enough zeroes for padding. */ - const char Zeroes[sizeof(uint64_t)] = {0}; - - /* Create the header. */ - uint64_t Header[PROFILE_HEADER_SIZE]; - Header[0] = __llvm_profile_get_magic(); - Header[1] = __llvm_profile_get_version(); - Header[2] = DataSize; - Header[3] = CountersSize; - Header[4] = NamesSize; - Header[5] = (uintptr_t)CountersBegin; - Header[6] = (uintptr_t)NamesBegin; - - /* Write the data. */ -#define CHECK_fwrite(Data, Size, Length, File) \ - do { if (fwrite(Data, Size, Length, File) != Length) return -1; } while (0) - CHECK_fwrite(Header, sizeof(uint64_t), PROFILE_HEADER_SIZE, File); - CHECK_fwrite(DataBegin, sizeof(__llvm_profile_data), DataSize, File); - CHECK_fwrite(CountersBegin, sizeof(uint64_t), CountersSize, File); - CHECK_fwrite(NamesBegin, sizeof(char), NamesSize, File); - CHECK_fwrite(Zeroes, sizeof(char), Padding, File); -#undef CHECK_fwrite - +/* Return 1 if there is an error, otherwise return 0. */ +static uint32_t fileWriter(ProfDataIOVec *IOVecs, uint32_t NumIOVecs, + void **WriterCtx) { + uint32_t I; + FILE *File = (FILE *)*WriterCtx; + for (I = 0; I < NumIOVecs; I++) { + if (fwrite(IOVecs[I].Data, IOVecs[I].ElmSize, IOVecs[I].NumElm, File) != + IOVecs[I].NumElm) + return 1; + } return 0; } +COMPILER_RT_VISIBILITY ProfBufferIO * +llvmCreateBufferIOInternal(void *File, uint32_t BufferSz) { + CallocHook = calloc; + FreeHook = free; + return llvmCreateBufferIO(fileWriter, File, BufferSz); +} + +static int writeFile(FILE *File) { + const char *BufferSzStr = 0; + uint64_t ValueDataSize = 0; + struct ValueProfData **ValueDataArray = + __llvm_profile_gather_value_data(&ValueDataSize); + FreeHook = &free; + CallocHook = &calloc; + BufferSzStr = getenv("LLVM_VP_BUFFER_SIZE"); + if (BufferSzStr && BufferSzStr[0]) + VPBufferSize = atoi(BufferSzStr); + return llvmWriteProfData(fileWriter, File, ValueDataArray, ValueDataSize); +} + static int writeFileWithName(const char *OutputName) { int RetVal; FILE *OutputFile; @@ -64,7 +57,7 @@ static int writeFileWithName(const char *OutputName) { return -1; /* Append to the file to support profiling multiple shared objects. */ - OutputFile = fopen(OutputName, "a"); + OutputFile = fopen(OutputName, "ab"); if (!OutputFile) return -1; @@ -74,8 +67,8 @@ static int writeFileWithName(const char *OutputName) { return RetVal; } -__attribute__((weak)) int __llvm_profile_OwnsFilename = 0; -__attribute__((weak)) const char *__llvm_profile_CurrentFilename = NULL; +COMPILER_RT_WEAK int __llvm_profile_OwnsFilename = 0; +COMPILER_RT_WEAK const char *__llvm_profile_CurrentFilename = NULL; static void truncateCurrentFile(void) { const char *Filename; @@ -182,7 +175,7 @@ static void setFilenameAutomatically(void) { resetFilenameToDefault(); } -__attribute__((visibility("hidden"))) +COMPILER_RT_VISIBILITY void __llvm_profile_initialize_file(void) { /* Check if the filename has been initialized. */ if (__llvm_profile_CurrentFilename) @@ -192,12 +185,12 @@ void __llvm_profile_initialize_file(void) { setFilenameAutomatically(); } -__attribute__((visibility("hidden"))) +COMPILER_RT_VISIBILITY void __llvm_profile_set_filename(const char *Filename) { setFilenamePossiblyWithPid(Filename); } -__attribute__((visibility("hidden"))) +COMPILER_RT_VISIBILITY void __llvm_profile_override_default_filename(const char *Filename) { /* If the env var is set, skip setting filename from argument. */ const char *Env_Filename = getenv("LLVM_PROFILE_FILE"); @@ -206,27 +199,28 @@ void __llvm_profile_override_default_filename(const char *Filename) { setFilenamePossiblyWithPid(Filename); } -__attribute__((visibility("hidden"))) +COMPILER_RT_VISIBILITY int __llvm_profile_write_file(void) { int rc; + GetEnvHook = &getenv; /* Check the filename. */ - if (!__llvm_profile_CurrentFilename) + if (!__llvm_profile_CurrentFilename) { + PROF_ERR("LLVM Profile: Failed to write file : %s\n", "Filename not set"); return -1; + } /* Write the file. */ rc = writeFileWithName(__llvm_profile_CurrentFilename); - if (rc && getenv("LLVM_PROFILE_VERBOSE_ERRORS")) - fprintf(stderr, "LLVM Profile: Failed to write file \"%s\": %s\n", + if (rc) + PROF_ERR("LLVM Profile: Failed to write file \"%s\": %s\n", __llvm_profile_CurrentFilename, strerror(errno)); return rc; } -static void writeFileWithoutReturn(void) { - __llvm_profile_write_file(); -} +static void writeFileWithoutReturn(void) { __llvm_profile_write_file(); } -__attribute__((visibility("hidden"))) +COMPILER_RT_VISIBILITY int __llvm_profile_register_write_file_atexit(void) { static int HasBeenRegistered = 0; diff --git a/lib/profile/InstrProfilingInternal.h b/lib/profile/InstrProfilingInternal.h index ede39cd9d713..4aab78ea509c 100644 --- a/lib/profile/InstrProfilingInternal.h +++ b/lib/profile/InstrProfilingInternal.h @@ -11,6 +11,7 @@ #define PROFILE_INSTRPROFILING_INTERNALH_ #include "InstrProfiling.h" +#include "stddef.h" /*! * \brief Write instrumentation data to the given buffer, given explicit @@ -37,4 +38,81 @@ int __llvm_profile_write_buffer_internal( const __llvm_profile_data *DataEnd, const uint64_t *CountersBegin, const uint64_t *CountersEnd, const char *NamesBegin, const char *NamesEnd); +/*! + * The data structure describing the data to be written by the + * low level writer callback function. + */ +typedef struct ProfDataIOVec { + const void *Data; + size_t ElmSize; + size_t NumElm; +} ProfDataIOVec; + +typedef uint32_t (*WriterCallback)(ProfDataIOVec *, uint32_t NumIOVecs, + void **WriterCtx); + +/*! + * The data structure for buffered IO of profile data. + */ +typedef struct ProfBufferIO { + /* File handle. */ + void *File; + /* Low level IO callback. */ + WriterCallback FileWriter; + /* The start of the buffer. */ + uint8_t *BufferStart; + /* Total size of the buffer. */ + uint32_t BufferSz; + /* Current byte offset from the start of the buffer. */ + uint32_t CurOffset; +} ProfBufferIO; + +/* The creator interface used by testing. */ +ProfBufferIO *llvmCreateBufferIOInternal(void *File, uint32_t DefaultBufferSz); +/*! + * This is the interface to create a handle for buffered IO. + */ +ProfBufferIO *llvmCreateBufferIO(WriterCallback FileWriter, void *File, + uint32_t DefaultBufferSz); +/*! + * The interface to destroy the bufferIO handle and reclaim + * the memory. + */ +void llvmDeleteBufferIO(ProfBufferIO *BufferIO); + +/*! + * This is the interface to write \c Data of \c Size bytes through + * \c BufferIO. Returns 0 if successful, otherwise return -1. + */ +int llvmBufferIOWrite(ProfBufferIO *BufferIO, const uint8_t *Data, + uint32_t Size); +/*! + * The interface to flush the remaining data in the buffer. + * through the low level writer callback. + */ +int llvmBufferIOFlush(ProfBufferIO *BufferIO); + +/* The low level interface to write data into a buffer. It is used as the + * callback by other high level writer methods such as buffered IO writer + * and profile data writer. */ +uint32_t llvmBufferWriter(ProfDataIOVec *IOVecs, uint32_t NumIOVecs, + void **WriterCtx); + +int llvmWriteProfData(WriterCallback Writer, void *WriterCtx, + struct ValueProfData **ValueDataArray, + const uint64_t ValueDataSize); +int llvmWriteProfDataImpl(WriterCallback Writer, void *WriterCtx, + const __llvm_profile_data *DataBegin, + const __llvm_profile_data *DataEnd, + const uint64_t *CountersBegin, + const uint64_t *CountersEnd, + struct ValueProfData **ValueDataBeginArray, + const uint64_t ValueDataSize, const char *NamesBegin, + const char *NamesEnd); + +extern char *(*GetEnvHook)(const char *); +extern void (*FreeHook)(void *); +extern void* (*CallocHook)(size_t, size_t); +extern uint32_t VPBufferSize; + #endif diff --git a/lib/profile/InstrProfilingPlatformDarwin.c b/lib/profile/InstrProfilingPlatformDarwin.c index 02299cc4630c..30ddbd2e4982 100644 --- a/lib/profile/InstrProfilingPlatformDarwin.c +++ b/lib/profile/InstrProfilingPlatformDarwin.c @@ -11,33 +11,36 @@ #if defined(__APPLE__) /* Use linker magic to find the bounds of the Data section. */ -__attribute__((visibility("hidden"))) -extern __llvm_profile_data DataStart __asm("section$start$__DATA$__llvm_prf_data"); -__attribute__((visibility("hidden"))) -extern __llvm_profile_data DataEnd __asm("section$end$__DATA$__llvm_prf_data"); -__attribute__((visibility("hidden"))) -extern char NamesStart __asm("section$start$__DATA$__llvm_prf_names"); -__attribute__((visibility("hidden"))) -extern char NamesEnd __asm("section$end$__DATA$__llvm_prf_names"); -__attribute__((visibility("hidden"))) -extern uint64_t CountersStart __asm("section$start$__DATA$__llvm_prf_cnts"); -__attribute__((visibility("hidden"))) -extern uint64_t CountersEnd __asm("section$end$__DATA$__llvm_prf_cnts"); +COMPILER_RT_VISIBILITY +extern __llvm_profile_data + DataStart __asm("section$start$__DATA$" INSTR_PROF_DATA_SECT_NAME_STR); +COMPILER_RT_VISIBILITY +extern __llvm_profile_data + DataEnd __asm("section$end$__DATA$" INSTR_PROF_DATA_SECT_NAME_STR); +COMPILER_RT_VISIBILITY +extern char + NamesStart __asm("section$start$__DATA$" INSTR_PROF_NAME_SECT_NAME_STR); +COMPILER_RT_VISIBILITY +extern char NamesEnd __asm("section$end$__DATA$" INSTR_PROF_NAME_SECT_NAME_STR); +COMPILER_RT_VISIBILITY +extern uint64_t + CountersStart __asm("section$start$__DATA$" INSTR_PROF_CNTS_SECT_NAME_STR); +COMPILER_RT_VISIBILITY +extern uint64_t + CountersEnd __asm("section$end$__DATA$" INSTR_PROF_CNTS_SECT_NAME_STR); -__attribute__((visibility("hidden"))) +COMPILER_RT_VISIBILITY const __llvm_profile_data *__llvm_profile_begin_data(void) { return &DataStart; } -__attribute__((visibility("hidden"))) -const __llvm_profile_data *__llvm_profile_end_data(void) { - return &DataEnd; -} -__attribute__((visibility("hidden"))) +COMPILER_RT_VISIBILITY +const __llvm_profile_data *__llvm_profile_end_data(void) { return &DataEnd; } +COMPILER_RT_VISIBILITY const char *__llvm_profile_begin_names(void) { return &NamesStart; } -__attribute__((visibility("hidden"))) +COMPILER_RT_VISIBILITY const char *__llvm_profile_end_names(void) { return &NamesEnd; } -__attribute__((visibility("hidden"))) +COMPILER_RT_VISIBILITY uint64_t *__llvm_profile_begin_counters(void) { return &CountersStart; } -__attribute__((visibility("hidden"))) +COMPILER_RT_VISIBILITY uint64_t *__llvm_profile_end_counters(void) { return &CountersEnd; } #endif diff --git a/lib/profile/InstrProfilingPlatformLinux.c b/lib/profile/InstrProfilingPlatformLinux.c new file mode 100644 index 000000000000..7843f47caa1b --- /dev/null +++ b/lib/profile/InstrProfilingPlatformLinux.c @@ -0,0 +1,59 @@ +/*===- InstrProfilingPlatformLinux.c - Profile data Linux platform ------===*\ +|* +|* The LLVM Compiler Infrastructure +|* +|* This file is distributed under the University of Illinois Open Source +|* License. See LICENSE.TXT for details. +|* +\*===----------------------------------------------------------------------===*/ + +#include "InstrProfiling.h" + +#if defined(__linux__) || defined(__FreeBSD__) +#include <stdlib.h> + +#define PROF_DATA_START INSTR_PROF_SECT_START(INSTR_PROF_DATA_SECT_NAME) +#define PROF_DATA_STOP INSTR_PROF_SECT_STOP(INSTR_PROF_DATA_SECT_NAME) +#define PROF_NAME_START INSTR_PROF_SECT_START(INSTR_PROF_NAME_SECT_NAME) +#define PROF_NAME_STOP INSTR_PROF_SECT_STOP(INSTR_PROF_NAME_SECT_NAME) +#define PROF_CNTS_START INSTR_PROF_SECT_START(INSTR_PROF_CNTS_SECT_NAME) +#define PROF_CNTS_STOP INSTR_PROF_SECT_STOP(INSTR_PROF_CNTS_SECT_NAME) + +/* Declare section start and stop symbols for various sections + * generated by compiler instrumentation. + */ +extern __llvm_profile_data PROF_DATA_START COMPILER_RT_VISIBILITY; +extern __llvm_profile_data PROF_DATA_STOP COMPILER_RT_VISIBILITY; +extern uint64_t PROF_CNTS_START COMPILER_RT_VISIBILITY; +extern uint64_t PROF_CNTS_STOP COMPILER_RT_VISIBILITY; +extern char PROF_NAME_START COMPILER_RT_VISIBILITY; +extern char PROF_NAME_STOP COMPILER_RT_VISIBILITY; + +/* Add dummy data to ensure the section is always created. */ +__llvm_profile_data + __prof_data_sect_data[0] COMPILER_RT_SECTION(INSTR_PROF_DATA_SECT_NAME_STR); +uint64_t + __prof_cnts_sect_data[0] COMPILER_RT_SECTION(INSTR_PROF_CNTS_SECT_NAME_STR); +char __prof_nms_sect_data[0] COMPILER_RT_SECTION(INSTR_PROF_NAME_SECT_NAME_STR); + +COMPILER_RT_VISIBILITY const __llvm_profile_data * +__llvm_profile_begin_data(void) { + return &PROF_DATA_START; +} +COMPILER_RT_VISIBILITY const __llvm_profile_data * +__llvm_profile_end_data(void) { + return &PROF_DATA_STOP; +} +COMPILER_RT_VISIBILITY const char *__llvm_profile_begin_names(void) { + return &PROF_NAME_START; +} +COMPILER_RT_VISIBILITY const char *__llvm_profile_end_names(void) { + return &PROF_NAME_STOP; +} +COMPILER_RT_VISIBILITY uint64_t *__llvm_profile_begin_counters(void) { + return &PROF_CNTS_START; +} +COMPILER_RT_VISIBILITY uint64_t *__llvm_profile_end_counters(void) { + return &PROF_CNTS_STOP; +} +#endif diff --git a/lib/profile/InstrProfilingPlatformOther.c b/lib/profile/InstrProfilingPlatformOther.c index 548d6a396b76..58ceb3458a0a 100644 --- a/lib/profile/InstrProfilingPlatformOther.c +++ b/lib/profile/InstrProfilingPlatformOther.c @@ -9,7 +9,7 @@ #include "InstrProfiling.h" -#if !defined(__APPLE__) +#if !defined(__APPLE__) && !defined(__linux__) && !defined(__FreeBSD__) #include <stdlib.h> static const __llvm_profile_data *DataFirst = NULL; @@ -26,49 +26,43 @@ static uint64_t *CountersLast = NULL; * calls are only required (and only emitted) on targets where we haven't * implemented linker magic to find the bounds of the sections. */ -__attribute__((visibility("hidden"))) +COMPILER_RT_VISIBILITY void __llvm_profile_register_function(void *Data_) { /* TODO: Only emit this function if we can't use linker magic. */ - const __llvm_profile_data *Data = (__llvm_profile_data*)Data_; + const __llvm_profile_data *Data = (__llvm_profile_data *)Data_; if (!DataFirst) { DataFirst = Data; DataLast = Data + 1; - NamesFirst = Data->Name; - NamesLast = Data->Name + Data->NameSize; - CountersFirst = Data->Counters; - CountersLast = Data->Counters + Data->NumCounters; + NamesFirst = Data->NamePtr; + NamesLast = (const char *)Data->NamePtr + Data->NameSize; + CountersFirst = Data->CounterPtr; + CountersLast = (uint64_t *)Data->CounterPtr + Data->NumCounters; return; } -#define UPDATE_FIRST(First, New) \ - First = New < First ? New : First +#define UPDATE_FIRST(First, New) First = New < First ? New : First UPDATE_FIRST(DataFirst, Data); - UPDATE_FIRST(NamesFirst, Data->Name); - UPDATE_FIRST(CountersFirst, Data->Counters); + UPDATE_FIRST(NamesFirst, (const char *)Data->NamePtr); + UPDATE_FIRST(CountersFirst, (uint64_t *)Data->CounterPtr); #undef UPDATE_FIRST -#define UPDATE_LAST(Last, New) \ - Last = New > Last ? New : Last +#define UPDATE_LAST(Last, New) Last = New > Last ? New : Last UPDATE_LAST(DataLast, Data + 1); - UPDATE_LAST(NamesLast, Data->Name + Data->NameSize); - UPDATE_LAST(CountersLast, Data->Counters + Data->NumCounters); + UPDATE_LAST(NamesLast, (const char *)Data->NamePtr + Data->NameSize); + UPDATE_LAST(CountersLast, (uint64_t *)Data->CounterPtr + Data->NumCounters); #undef UPDATE_LAST } -__attribute__((visibility("hidden"))) -const __llvm_profile_data *__llvm_profile_begin_data(void) { - return DataFirst; -} -__attribute__((visibility("hidden"))) -const __llvm_profile_data *__llvm_profile_end_data(void) { - return DataLast; -} -__attribute__((visibility("hidden"))) +COMPILER_RT_VISIBILITY +const __llvm_profile_data *__llvm_profile_begin_data(void) { return DataFirst; } +COMPILER_RT_VISIBILITY +const __llvm_profile_data *__llvm_profile_end_data(void) { return DataLast; } +COMPILER_RT_VISIBILITY const char *__llvm_profile_begin_names(void) { return NamesFirst; } -__attribute__((visibility("hidden"))) +COMPILER_RT_VISIBILITY const char *__llvm_profile_end_names(void) { return NamesLast; } -__attribute__((visibility("hidden"))) +COMPILER_RT_VISIBILITY uint64_t *__llvm_profile_begin_counters(void) { return CountersFirst; } -__attribute__((visibility("hidden"))) +COMPILER_RT_VISIBILITY uint64_t *__llvm_profile_end_counters(void) { return CountersLast; } #endif diff --git a/lib/profile/InstrProfilingPort.h b/lib/profile/InstrProfilingPort.h new file mode 100644 index 000000000000..da4f18fcbb46 --- /dev/null +++ b/lib/profile/InstrProfilingPort.h @@ -0,0 +1,76 @@ +/*===- InstrProfilingPort.h- Support library for PGO instrumentation ------===*\ +|* +|* The LLVM Compiler Infrastructure +|* +|* This file is distributed under the University of Illinois Open Source +|* License. See LICENSE.TXT for details. +|* +\*===----------------------------------------------------------------------===*/ + +#ifndef PROFILE_INSTRPROFILING_PORT_H_ +#define PROFILE_INSTRPROFILING_PORT_H_ + +#ifdef _MSC_VER +#define COMPILER_RT_ALIGNAS(x) __declspec(align(x)) +#define COMPILER_RT_VISIBILITY +#define COMPILER_RT_WEAK __declspec(selectany) +#elif __GNUC__ +#define COMPILER_RT_ALIGNAS(x) __attribute__((aligned(x))) +#define COMPILER_RT_VISIBILITY __attribute__((visibility("hidden"))) +#define COMPILER_RT_WEAK __attribute__((weak)) +#endif + +#define COMPILER_RT_SECTION(Sect) __attribute__((section(Sect))) + +#if COMPILER_RT_HAS_ATOMICS == 1 +#ifdef _MSC_VER +#include <windows.h> +#if defined(_WIN64) +#define COMPILER_RT_BOOL_CMPXCHG(Ptr, OldV, NewV) \ + (InterlockedCompareExchange64((LONGLONG volatile *)Ptr, (LONGLONG)NewV, \ + (LONGLONG)OldV) == (LONGLONG)OldV) +#else +#define COMPILER_RT_BOOL_CMPXCHG(Ptr, OldV, NewV) \ + (InterlockedCompareExchange((LONG volatile *)Ptr, (LONG)NewV, (LONG)OldV) == \ + (LONG)OldV) +#endif +#else +#define COMPILER_RT_BOOL_CMPXCHG(Ptr, OldV, NewV) \ + __sync_bool_compare_and_swap(Ptr, OldV, NewV) +#endif +#else +#define COMPILER_RT_BOOL_CMPXCHG(Ptr, OldV, NewV) \ + BoolCmpXchg((void **)Ptr, OldV, NewV) +#endif + +#define PROF_ERR(Format, ...) \ + if (GetEnvHook && GetEnvHook("LLVM_PROFILE_VERBOSE_ERRORS")) \ + fprintf(stderr, Format, __VA_ARGS__); + +#if defined(__FreeBSD__) && defined(__i386__) + +/* System headers define 'size_t' incorrectly on x64 FreeBSD (prior to + * FreeBSD 10, r232261) when compiled in 32-bit mode. + */ +#define PRIu64 "llu" +typedef unsigned char uint8_t; +typedef unsigned short uint16_t; +typedef unsigned int uint32_t; +typedef unsigned long long uint64_t; +typedef uint32_t uintptr_t; +#elif defined(__FreeBSD__) && defined(__x86_64__) +#define PRIu64 "lu" +typedef unsigned char uint8_t; +typedef unsigned short uint16_t; +typedef unsigned int uint32_t; +typedef unsigned long long uint64_t; +typedef unsigned long int uintptr_t; + +#else /* defined(__FreeBSD__) && defined(__i386__) */ + +#include <inttypes.h> +#include <stdint.h> + +#endif /* defined(__FreeBSD__) && defined(__i386__) */ + +#endif /* PROFILE_INSTRPROFILING_PORT_H_ */ diff --git a/lib/profile/InstrProfilingRuntime.cc b/lib/profile/InstrProfilingRuntime.cc index 081ecb29e987..12ad9f1573f4 100644 --- a/lib/profile/InstrProfilingRuntime.cc +++ b/lib/profile/InstrProfilingRuntime.cc @@ -11,8 +11,7 @@ extern "C" { #include "InstrProfiling.h" -__attribute__((visibility("hidden"))) int __llvm_profile_runtime; - +COMPILER_RT_VISIBILITY int __llvm_profile_runtime; } namespace { diff --git a/lib/profile/InstrProfilingUtil.c b/lib/profile/InstrProfilingUtil.c index e146dfca83c8..6f0443d3bb5d 100644 --- a/lib/profile/InstrProfilingUtil.c +++ b/lib/profile/InstrProfilingUtil.c @@ -8,6 +8,7 @@ \*===----------------------------------------------------------------------===*/ #include "InstrProfilingUtil.h" +#include "InstrProfiling.h" #ifdef _WIN32 #include <direct.h> @@ -18,7 +19,7 @@ int mkdir(const char*, unsigned short); #include <sys/types.h> #endif -__attribute__((visibility("hidden"))) +COMPILER_RT_VISIBILITY void __llvm_profile_recursive_mkdir(char *path) { int i; diff --git a/lib/profile/InstrProfilingValue.c b/lib/profile/InstrProfilingValue.c new file mode 100644 index 000000000000..39b4da446a81 --- /dev/null +++ b/lib/profile/InstrProfilingValue.c @@ -0,0 +1,180 @@ +/*===- InstrProfilingValue.c - Support library for PGO instrumentation ----===*\ +|* +|* The LLVM Compiler Infrastructure +|* +|* This file is distributed under the University of Illinois Open Source +|* License. See LICENSE.TXT for details. +|* +\*===----------------------------------------------------------------------===*/ + +#include "InstrProfiling.h" +#include "InstrProfilingInternal.h" +#include <limits.h> +#include <stdio.h> +#include <stdlib.h> +#include <string.h> +#define INSTR_PROF_VALUE_PROF_DATA +#define INSTR_PROF_COMMON_API_IMPL +#include "InstrProfData.inc" + +#define PROF_OOM(Msg) PROF_ERR(Msg ":%s\n", "Out of memory"); +#define PROF_OOM_RETURN(Msg) \ + { \ + PROF_OOM(Msg) \ + free(ValueDataArray); \ + return NULL; \ + } + +#if COMPILER_RT_HAS_ATOMICS != 1 +COMPILER_RT_VISIBILITY +uint32_t BoolCmpXchg(void **Ptr, void *OldV, void *NewV) { + void *R = *Ptr; + if (R == OldV) { + *Ptr = NewV; + return 1; + } + return 0; +} +#endif + +/* This method is only used in value profiler mock testing. */ +COMPILER_RT_VISIBILITY void +__llvm_profile_set_num_value_sites(__llvm_profile_data *Data, + uint32_t ValueKind, uint16_t NumValueSites) { + *((uint16_t *)&Data->NumValueSites[ValueKind]) = NumValueSites; +} + +/* This method is only used in value profiler mock testing. */ +COMPILER_RT_VISIBILITY const __llvm_profile_data * +__llvm_profile_iterate_data(const __llvm_profile_data *Data) { + return Data + 1; +} + +/* This method is only used in value profiler mock testing. */ +COMPILER_RT_VISIBILITY void * +__llvm_get_function_addr(const __llvm_profile_data *Data) { + return Data->FunctionPointer; +} + +/* Allocate an array that holds the pointers to the linked lists of + * value profile counter nodes. The number of element of the array + * is the total number of value profile sites instrumented. Returns + * 0 if allocation fails. + */ + +static int allocateValueProfileCounters(__llvm_profile_data *Data) { + uint64_t NumVSites = 0; + uint32_t VKI; + for (VKI = IPVK_First; VKI <= IPVK_Last; ++VKI) + NumVSites += Data->NumValueSites[VKI]; + + ValueProfNode **Mem = + (ValueProfNode **)calloc(NumVSites, sizeof(ValueProfNode *)); + if (!Mem) + return 0; + if (!COMPILER_RT_BOOL_CMPXCHG(&Data->Values, 0, Mem)) { + free(Mem); + return 0; + } + return 1; +} + +COMPILER_RT_VISIBILITY void +__llvm_profile_instrument_target(uint64_t TargetValue, void *Data, + uint32_t CounterIndex) { + + __llvm_profile_data *PData = (__llvm_profile_data *)Data; + if (!PData) + return; + + if (!PData->Values) { + if (!allocateValueProfileCounters(PData)) + return; + } + + ValueProfNode **ValueCounters = (ValueProfNode **)PData->Values; + ValueProfNode *PrevVNode = NULL; + ValueProfNode *CurrentVNode = ValueCounters[CounterIndex]; + + uint8_t VDataCount = 0; + while (CurrentVNode) { + if (TargetValue == CurrentVNode->VData.Value) { + CurrentVNode->VData.Count++; + return; + } + PrevVNode = CurrentVNode; + CurrentVNode = CurrentVNode->Next; + ++VDataCount; + } + + if (VDataCount >= UCHAR_MAX) + return; + + CurrentVNode = (ValueProfNode *)calloc(1, sizeof(ValueProfNode)); + if (!CurrentVNode) + return; + + CurrentVNode->VData.Value = TargetValue; + CurrentVNode->VData.Count++; + + uint32_t Success = 0; + if (!ValueCounters[CounterIndex]) + Success = + COMPILER_RT_BOOL_CMPXCHG(&ValueCounters[CounterIndex], 0, CurrentVNode); + else if (PrevVNode && !PrevVNode->Next) + Success = COMPILER_RT_BOOL_CMPXCHG(&(PrevVNode->Next), 0, CurrentVNode); + + if (!Success) { + free(CurrentVNode); + return; + } +} + +COMPILER_RT_VISIBILITY ValueProfData ** +__llvm_profile_gather_value_data(uint64_t *ValueDataSize) { + size_t S = 0; + __llvm_profile_data *I; + ValueProfData **ValueDataArray; + + const __llvm_profile_data *DataEnd = __llvm_profile_end_data(); + const __llvm_profile_data *DataBegin = __llvm_profile_begin_data(); + + if (!ValueDataSize) + return NULL; + + ValueDataArray = + (ValueProfData **)calloc(DataEnd - DataBegin, sizeof(void *)); + if (!ValueDataArray) + PROF_OOM_RETURN("Failed to write value profile data "); + + /* + * Compute the total Size of the buffer to hold ValueProfData + * structures for functions with value profile data. + */ + for (I = (__llvm_profile_data *)DataBegin; I != DataEnd; ++I) { + ValueProfRuntimeRecord R; + if (initializeValueProfRuntimeRecord(&R, I->NumValueSites, I->Values)) + PROF_OOM_RETURN("Failed to write value profile data "); + + /* Compute the size of ValueProfData from this runtime record. */ + if (getNumValueKindsRT(&R) != 0) { + ValueProfData *VD = NULL; + uint32_t VS = getValueProfDataSizeRT(&R); + VD = (ValueProfData *)calloc(VS, sizeof(uint8_t)); + if (!VD) + PROF_OOM_RETURN("Failed to write value profile data "); + serializeValueProfDataFromRT(&R, VD); + ValueDataArray[I - DataBegin] = VD; + S += VS; + } + finalizeValueProfRuntimeRecord(&R); + } + + if (!S) { + free(ValueDataArray); + ValueDataArray = NULL; + } + + *ValueDataSize = S; + return ValueDataArray; +} diff --git a/lib/profile/InstrProfilingWriter.c b/lib/profile/InstrProfilingWriter.c new file mode 100644 index 000000000000..a07bc538ed4b --- /dev/null +++ b/lib/profile/InstrProfilingWriter.c @@ -0,0 +1,175 @@ +/*===- InstrProfilingWriter.c - Write instrumentation to a file or buffer -===*\ +|* +|* The LLVM Compiler Infrastructure +|* +|* This file is distributed under the University of Illinois Open Source +|* License. See LICENSE.TXT for details. +|* +\*===----------------------------------------------------------------------===*/ + +#include "InstrProfiling.h" +#include "InstrProfilingInternal.h" +#include <string.h> + +#define INSTR_PROF_VALUE_PROF_DATA +#include "InstrProfData.inc" +void (*FreeHook)(void *) = NULL; +void* (*CallocHook)(size_t, size_t) = NULL; +uint32_t VPBufferSize = 0; + +/* The buffer writer is reponsponsible in keeping writer state + * across the call. + */ +COMPILER_RT_VISIBILITY uint32_t llvmBufferWriter(ProfDataIOVec *IOVecs, + uint32_t NumIOVecs, + void **WriterCtx) { + uint32_t I; + char **Buffer = (char **)WriterCtx; + for (I = 0; I < NumIOVecs; I++) { + size_t Length = IOVecs[I].ElmSize * IOVecs[I].NumElm; + memcpy(*Buffer, IOVecs[I].Data, Length); + *Buffer += Length; + } + return 0; +} + +static void llvmInitBufferIO(ProfBufferIO *BufferIO, WriterCallback FileWriter, + void *File, uint8_t *Buffer, uint32_t BufferSz) { + BufferIO->File = File; + BufferIO->FileWriter = FileWriter; + BufferIO->BufferStart = Buffer; + BufferIO->BufferSz = BufferSz; + BufferIO->CurOffset = 0; +} + +COMPILER_RT_VISIBILITY ProfBufferIO * +llvmCreateBufferIO(WriterCallback FileWriter, void *File, uint32_t BufferSz) { + ProfBufferIO *BufferIO = (ProfBufferIO *)CallocHook(1, sizeof(ProfBufferIO)); + uint8_t *Buffer = (uint8_t *)CallocHook(1, BufferSz); + if (!Buffer) { + FreeHook(BufferIO); + return 0; + } + llvmInitBufferIO(BufferIO, FileWriter, File, Buffer, BufferSz); + return BufferIO; +} + +COMPILER_RT_VISIBILITY void llvmDeleteBufferIO(ProfBufferIO *BufferIO) { + FreeHook(BufferIO->BufferStart); + FreeHook(BufferIO); +} + +COMPILER_RT_VISIBILITY int +llvmBufferIOWrite(ProfBufferIO *BufferIO, const uint8_t *Data, uint32_t Size) { + /* Buffer is not large enough, it is time to flush. */ + if (Size + BufferIO->CurOffset > BufferIO->BufferSz) { + if (llvmBufferIOFlush(BufferIO) != 0) + return -1; + } + /* Special case, bypass the buffer completely. */ + ProfDataIOVec IO[] = {{Data, sizeof(uint8_t), Size}}; + if (Size > BufferIO->BufferSz) { + if (BufferIO->FileWriter(IO, 1, &BufferIO->File)) + return -1; + } else { + /* Write the data to buffer */ + uint8_t *Buffer = BufferIO->BufferStart + BufferIO->CurOffset; + llvmBufferWriter(IO, 1, (void **)&Buffer); + BufferIO->CurOffset = Buffer - BufferIO->BufferStart; + } + return 0; +} + +COMPILER_RT_VISIBILITY int llvmBufferIOFlush(ProfBufferIO *BufferIO) { + if (BufferIO->CurOffset) { + ProfDataIOVec IO[] = { + {BufferIO->BufferStart, sizeof(uint8_t), BufferIO->CurOffset}}; + if (BufferIO->FileWriter(IO, 1, &BufferIO->File)) + return -1; + BufferIO->CurOffset = 0; + } + return 0; +} + +COMPILER_RT_VISIBILITY int llvmWriteProfData(WriterCallback Writer, + void *WriterCtx, + ValueProfData **ValueDataArray, + const uint64_t ValueDataSize) { + /* Match logic in __llvm_profile_write_buffer(). */ + const __llvm_profile_data *DataBegin = __llvm_profile_begin_data(); + const __llvm_profile_data *DataEnd = __llvm_profile_end_data(); + const uint64_t *CountersBegin = __llvm_profile_begin_counters(); + const uint64_t *CountersEnd = __llvm_profile_end_counters(); + const char *NamesBegin = __llvm_profile_begin_names(); + const char *NamesEnd = __llvm_profile_end_names(); + return llvmWriteProfDataImpl(Writer, WriterCtx, DataBegin, DataEnd, + CountersBegin, CountersEnd, ValueDataArray, + ValueDataSize, NamesBegin, NamesEnd); +} + +#define VP_BUFFER_SIZE 8 * 1024 +static int writeValueProfData(WriterCallback Writer, void *WriterCtx, + ValueProfData **ValueDataBegin, + uint64_t NumVData) { + ProfBufferIO *BufferIO; + uint32_t I = 0, BufferSz; + + if (!ValueDataBegin) + return 0; + + BufferSz = VPBufferSize ? VPBufferSize : VP_BUFFER_SIZE; + BufferIO = llvmCreateBufferIO(Writer, WriterCtx, BufferSz); + + for (I = 0; I < NumVData; I++) { + ValueProfData *CurVData = ValueDataBegin[I]; + if (!CurVData) + continue; + if (llvmBufferIOWrite(BufferIO, (const uint8_t *)CurVData, + CurVData->TotalSize) != 0) + return -1; + } + + if (llvmBufferIOFlush(BufferIO) != 0) + return -1; + llvmDeleteBufferIO(BufferIO); + + return 0; +} + +COMPILER_RT_VISIBILITY int llvmWriteProfDataImpl( + WriterCallback Writer, void *WriterCtx, + const __llvm_profile_data *DataBegin, const __llvm_profile_data *DataEnd, + const uint64_t *CountersBegin, const uint64_t *CountersEnd, + ValueProfData **ValueDataBegin, const uint64_t ValueDataSize, + const char *NamesBegin, const char *NamesEnd) { + + /* Calculate size of sections. */ + const uint64_t DataSize = DataEnd - DataBegin; + const uint64_t CountersSize = CountersEnd - CountersBegin; + const uint64_t NamesSize = NamesEnd - NamesBegin; + const uint64_t Padding = __llvm_profile_get_num_padding_bytes(NamesSize); + + /* Enough zeroes for padding. */ + const char Zeroes[sizeof(uint64_t)] = {0}; + + /* Create the header. */ + __llvm_profile_header Header; + + if (!DataSize) + return 0; + + /* Initialize header struture. */ +#define INSTR_PROF_RAW_HEADER(Type, Name, Init) Header.Name = Init; +#include "InstrProfData.inc" + + /* Write the data. */ + ProfDataIOVec IOVec[] = {{&Header, sizeof(__llvm_profile_header), 1}, + {DataBegin, sizeof(__llvm_profile_data), DataSize}, + {CountersBegin, sizeof(uint64_t), CountersSize}, + {NamesBegin, sizeof(uint8_t), NamesSize}, + {Zeroes, sizeof(uint8_t), Padding}}; + if (Writer(IOVec, sizeof(IOVec) / sizeof(*IOVec), &WriterCtx)) + return -1; + + return writeValueProfData(Writer, WriterCtx, ValueDataBegin, DataSize); +} diff --git a/lib/safestack/.clang-format b/lib/safestack/.clang-format new file mode 100644 index 000000000000..f6cb8ad931f5 --- /dev/null +++ b/lib/safestack/.clang-format @@ -0,0 +1 @@ +BasedOnStyle: Google diff --git a/lib/safestack/CMakeLists.txt b/lib/safestack/CMakeLists.txt index 1c15d079dbb5..9c11bb6f7e61 100644 --- a/lib/safestack/CMakeLists.txt +++ b/lib/safestack/CMakeLists.txt @@ -8,21 +8,27 @@ set(SAFESTACK_CFLAGS ${SANITIZER_COMMON_CFLAGS}) if(APPLE) # Build universal binary on APPLE. - add_compiler_rt_osx_static_runtime(clang_rt.safestack_osx - ARCH ${SAFESTACK_SUPPORTED_ARCH} + add_compiler_rt_runtime(clang_rt.safestack + STATIC + OS osx + ARCHS ${SAFESTACK_SUPPORTED_ARCH} SOURCES ${SAFESTACK_SOURCES} $<TARGET_OBJECTS:RTInterception.osx> $<TARGET_OBJECTS:RTSanitizerCommon.osx> - CFLAGS ${SAFESTACK_CFLAGS}) - add_dependencies(safestack clang_rt.safestack_osx) + $<TARGET_OBJECTS:RTSanitizerCommonNoLibc.osx> + CFLAGS ${SAFESTACK_CFLAGS} + PARENT_TARGET safestack) else() # Otherwise, build separate libraries for each target. foreach(arch ${SAFESTACK_SUPPORTED_ARCH}) - add_compiler_rt_runtime(clang_rt.safestack-${arch} ${arch} STATIC + add_compiler_rt_runtime(clang_rt.safestack + STATIC + ARCHS ${arch} SOURCES ${SAFESTACK_SOURCES} $<TARGET_OBJECTS:RTInterception.${arch}> $<TARGET_OBJECTS:RTSanitizerCommon.${arch}> - CFLAGS ${SAFESTACK_CFLAGS}) - add_dependencies(safestack clang_rt.safestack-${arch}) + $<TARGET_OBJECTS:RTSanitizerCommonNoLibc.${arch}> + CFLAGS ${SAFESTACK_CFLAGS} + PARENT_TARGET safestack) endforeach() endif() diff --git a/lib/safestack/safestack.cc b/lib/safestack/safestack.cc index 504bd3cd0d99..92c24b35d6d0 100644 --- a/lib/safestack/safestack.cc +++ b/lib/safestack/safestack.cc @@ -18,6 +18,7 @@ #include <pthread.h> #include <stddef.h> #include <stdint.h> +#include <unistd.h> #include <sys/resource.h> #include <sys/types.h> #include <sys/user.h> @@ -68,6 +69,9 @@ const unsigned kStackAlign = 16; /// size rlimit is set to infinity. const unsigned kDefaultUnsafeStackSize = 0x2800000; +/// Runtime page size obtained through sysconf +static unsigned pageSize; + // TODO: To make accessing the unsafe stack pointer faster, we plan to // eventually store it directly in the thread control block data structure on // platforms where this structure is pointed to by %fs or %gs. This is exactly @@ -171,7 +175,7 @@ INTERCEPTOR(int, pthread_create, pthread_t *thread, size_t size = 0; size_t guard = 0; - if (attr != NULL) { + if (attr) { pthread_attr_getstacksize(attr, &size); pthread_attr_getguardsize(attr, &guard); } else { @@ -185,7 +189,7 @@ INTERCEPTOR(int, pthread_create, pthread_t *thread, CHECK_NE(size, 0); CHECK_EQ((size & (kStackAlign - 1)), 0); - CHECK_EQ((guard & (PAGE_SIZE - 1)), 0); + CHECK_EQ((guard & (pageSize - 1)), 0); void *addr = unsafe_stack_alloc(size, guard); struct tinfo *tinfo = @@ -217,6 +221,7 @@ void __safestack_init() { void *addr = unsafe_stack_alloc(size, guard); unsafe_stack_setup(addr, size, guard); + pageSize = sysconf(_SC_PAGESIZE); // Initialize pthread interceptors for thread allocation INTERCEPT_FUNCTION(pthread_create); diff --git a/lib/sanitizer_common/.clang-format b/lib/sanitizer_common/.clang-format new file mode 100644 index 000000000000..f6cb8ad931f5 --- /dev/null +++ b/lib/sanitizer_common/.clang-format @@ -0,0 +1 @@ +BasedOnStyle: Google diff --git a/lib/sanitizer_common/CMakeLists.txt b/lib/sanitizer_common/CMakeLists.txt index f604c9f201d4..6a20f025507a 100644 --- a/lib/sanitizer_common/CMakeLists.txt +++ b/lib/sanitizer_common/CMakeLists.txt @@ -33,6 +33,12 @@ set(SANITIZER_SOURCES sanitizer_thread_registry.cc sanitizer_win.cc) +# Libc functions stubs. These sources should be linked instead of +# SANITIZER_LIBCDEP_SOURCES when sanitizer_common library must not depend on +# libc. +set(SANITIZER_NOLIBC_SOURCES + sanitizer_common_nolibc.cc) + set(SANITIZER_LIBCDEP_SOURCES sanitizer_common_libcdep.cc sanitizer_coverage_libcdep.cc @@ -43,7 +49,6 @@ set(SANITIZER_LIBCDEP_SOURCES sanitizer_stoptheworld_linux_libcdep.cc sanitizer_symbolizer_libcdep.cc sanitizer_symbolizer_posix_libcdep.cc - sanitizer_symbolizer_process_libcdep.cc sanitizer_unwind_linux_libcdep.cc) # Explicitly list all sanitizer_common headers. Not all of these are @@ -97,9 +102,9 @@ set(SANITIZER_HEADERS sanitizer_symbolizer_internal.h sanitizer_symbolizer_libbacktrace.h sanitizer_symbolizer_mac.h - sanitizer_symbolizer_win.h sanitizer_syscall_generic.inc sanitizer_syscall_linux_x86_64.inc + sanitizer_syscall_linux_aarch64.inc sanitizer_thread_registry.h) set(SANITIZER_COMMON_DEFINITIONS) @@ -124,38 +129,28 @@ append_list_if(SANITIZER_LIMIT_FRAME_SIZE -Wframe-larger-than=570 append_list_if(COMPILER_RT_HAS_WGLOBAL_CONSTRUCTORS_FLAG -Wglobal-constructors SANITIZER_CFLAGS) -add_custom_target(sanitizer_common) -set(SANITIZER_RUNTIME_LIBRARIES) if(APPLE) - # Build universal binary on APPLE. - - add_compiler_rt_object_libraries(RTSanitizerCommon - OS ${SANITIZER_COMMON_SUPPORTED_OS} - ARCHS ${SANITIZER_COMMON_SUPPORTED_ARCH} - SOURCES ${SANITIZER_SOURCES} ${SANITIZER_LIBCDEP_SOURCES} - CFLAGS ${SANITIZER_CFLAGS} - DEFS ${SANITIZER_COMMON_DEFINITIONS}) - foreach(os ${SANITIZER_COMMON_SUPPORTED_OS}) - list(APPEND SANITIZER_RUNTIME_LIBRARIES RTSanitizerCommon.${os}) - endforeach() -else() - # Otherwise, build separate libraries for each target. - - add_compiler_rt_object_libraries(RTSanitizerCommon - ARCHS ${SANITIZER_COMMON_SUPPORTED_ARCH} - SOURCES ${SANITIZER_SOURCES} CFLAGS ${SANITIZER_CFLAGS} - DEFS ${SANITIZER_COMMON_DEFINITIONS}) - add_compiler_rt_object_libraries(RTSanitizerCommonLibc - ARCHS ${SANITIZER_COMMON_SUPPORTED_ARCH} - SOURCES ${SANITIZER_LIBCDEP_SOURCES} CFLAGS ${SANITIZER_CFLAGS} - DEFS ${SANITIZER_COMMON_DEFINITIONS}) - foreach(arch ${SANITIZER_COMMON_SUPPORTED_ARCH}) - list(APPEND SANITIZER_RUNTIME_LIBRARIES RTSanitizerCommon.${arch} - RTSanitizerCommonLibc.${arch}) - endforeach() + set(OS_OPTION OS ${SANITIZER_COMMON_SUPPORTED_OS}) endif() -add_dependencies(compiler-rt sanitizer_common) +add_compiler_rt_object_libraries(RTSanitizerCommon + ${OS_OPTION} + ARCHS ${SANITIZER_COMMON_SUPPORTED_ARCH} + SOURCES ${SANITIZER_SOURCES} + CFLAGS ${SANITIZER_CFLAGS} + DEFS ${SANITIZER_COMMON_DEFINITIONS}) +add_compiler_rt_object_libraries(RTSanitizerCommonNoLibc + ${OS_OPTION} + ARCHS ${SANITIZER_COMMON_SUPPORTED_ARCH} + SOURCES ${SANITIZER_NOLIBC_SOURCES} + CFLAGS ${SANITIZER_CFLAGS} + DEFS ${SANITIZER_COMMON_DEFINITIONS}) +add_compiler_rt_object_libraries(RTSanitizerCommonLibc + ${OS_OPTION} + ARCHS ${SANITIZER_COMMON_SUPPORTED_ARCH} + SOURCES ${SANITIZER_LIBCDEP_SOURCES} + CFLAGS ${SANITIZER_CFLAGS} + DEFS ${SANITIZER_COMMON_DEFINITIONS}) # Unit tests for common sanitizer runtime. if(COMPILER_RT_INCLUDE_TESTS) diff --git a/lib/sanitizer_common/Makefile.mk b/lib/sanitizer_common/Makefile.mk index da83c2d6b3b7..5bb20d076e81 100644 --- a/lib/sanitizer_common/Makefile.mk +++ b/lib/sanitizer_common/Makefile.mk @@ -11,6 +11,8 @@ ModuleName := sanitizer_common SubDirs := Sources := $(foreach file,$(wildcard $(Dir)/*.cc),$(notdir $(file))) +NolibcSources := $(foreach file,$(wildcard $(Dir)/*_nolibc.cc),$(notdir $(file))) +Sources := $(filter-out $(NolibcSources),$(Sources)) ObjNames := $(Sources:%.cc=%.o) Implementation := Generic diff --git a/lib/sanitizer_common/sanitizer_addrhashmap.h b/lib/sanitizer_common/sanitizer_addrhashmap.h index acf4ff020939..e55fc4f95a9a 100644 --- a/lib/sanitizer_common/sanitizer_addrhashmap.h +++ b/lib/sanitizer_common/sanitizer_addrhashmap.h @@ -143,7 +143,7 @@ bool AddrHashMap<T, kSize>::Handle::created() const { template<typename T, uptr kSize> bool AddrHashMap<T, kSize>::Handle::exists() const { - return cell_ != 0; + return cell_ != nullptr; } template<typename T, uptr kSize> @@ -160,7 +160,7 @@ void AddrHashMap<T, kSize>::acquire(Handle *h) { h->created_ = false; h->addidx_ = -1U; h->bucket_ = b; - h->cell_ = 0; + h->cell_ = nullptr; // If we want to remove the element, we need exclusive access to the bucket, // so skip the lock-free phase. @@ -250,7 +250,7 @@ void AddrHashMap<T, kSize>::acquire(Handle *h) { } // Store in the add cells. - if (add == 0) { + if (!add) { // Allocate a new add array. const uptr kInitSize = 64; add = (AddBucket*)InternalAlloc(kInitSize); @@ -282,7 +282,7 @@ void AddrHashMap<T, kSize>::acquire(Handle *h) { template<typename T, uptr kSize> void AddrHashMap<T, kSize>::release(Handle *h) { - if (h->cell_ == 0) + if (!h->cell_) return; Bucket *b = h->bucket_; Cell *c = h->cell_; diff --git a/lib/sanitizer_common/sanitizer_allocator.cc b/lib/sanitizer_common/sanitizer_allocator.cc index 03b3e83153de..538e2db95d4e 100644 --- a/lib/sanitizer_common/sanitizer_allocator.cc +++ b/lib/sanitizer_common/sanitizer_allocator.cc @@ -11,6 +11,7 @@ // run-time libraries. // This allocator is used inside run-times. //===----------------------------------------------------------------------===// + #include "sanitizer_allocator.h" #include "sanitizer_allocator_internal.h" #include "sanitizer_common.h" @@ -44,7 +45,7 @@ InternalAllocator *internal_allocator() { return 0; } -#else // SANITIZER_GO +#else // SANITIZER_GO static ALIGNED(64) char internal_alloc_placeholder[sizeof(InternalAllocator)]; static atomic_uint8_t internal_allocator_initialized; @@ -77,29 +78,29 @@ static void *RawInternalAlloc(uptr size, InternalAllocatorCache *cache) { } static void RawInternalFree(void *ptr, InternalAllocatorCache *cache) { - if (cache == 0) { + if (!cache) { SpinMutexLock l(&internal_allocator_cache_mu); return internal_allocator()->Deallocate(&internal_allocator_cache, ptr); } internal_allocator()->Deallocate(cache, ptr); } -#endif // SANITIZER_GO +#endif // SANITIZER_GO const u64 kBlockMagic = 0x6A6CB03ABCEBC041ull; void *InternalAlloc(uptr size, InternalAllocatorCache *cache) { if (size + sizeof(u64) < size) - return 0; + return nullptr; void *p = RawInternalAlloc(size + sizeof(u64), cache); - if (p == 0) - return 0; + if (!p) + return nullptr; ((u64*)p)[0] = kBlockMagic; return (char*)p + sizeof(u64); } void InternalFree(void *addr, InternalAllocatorCache *cache) { - if (addr == 0) + if (!addr) return; addr = (char*)addr - sizeof(u64); CHECK_EQ(kBlockMagic, ((u64*)addr)[0]); @@ -147,4 +148,4 @@ void NORETURN ReportAllocatorCannotReturnNull() { Die(); } -} // namespace __sanitizer +} // namespace __sanitizer diff --git a/lib/sanitizer_common/sanitizer_allocator.h b/lib/sanitizer_common/sanitizer_allocator.h index deaffef7150d..44d6fce3b291 100644 --- a/lib/sanitizer_common/sanitizer_allocator.h +++ b/lib/sanitizer_common/sanitizer_allocator.h @@ -347,7 +347,7 @@ class SizeClassAllocator64 { CHECK_LT(class_id, kNumClasses); RegionInfo *region = GetRegionInfo(class_id); Batch *b = region->free_list.Pop(); - if (b == 0) + if (!b) b = PopulateFreeList(stat, c, class_id, region); region->n_allocated += b->count; return b; @@ -371,16 +371,16 @@ class SizeClassAllocator64 { void *GetBlockBegin(const void *p) { uptr class_id = GetSizeClass(p); uptr size = SizeClassMap::Size(class_id); - if (!size) return 0; + if (!size) return nullptr; uptr chunk_idx = GetChunkIdx((uptr)p, size); uptr reg_beg = (uptr)p & ~(kRegionSize - 1); uptr beg = chunk_idx * size; uptr next_beg = beg + size; - if (class_id >= kNumClasses) return 0; + if (class_id >= kNumClasses) return nullptr; RegionInfo *region = GetRegionInfo(class_id); if (region->mapped_user >= next_beg) return reinterpret_cast<void*>(reg_beg + beg); - return 0; + return nullptr; } static uptr GetActuallyAllocatedSize(void *p) { @@ -609,6 +609,7 @@ class TwoLevelByteMap { internal_memset(map1_, 0, sizeof(map1_)); mu_.Init(); } + void TestOnlyUnmap() { for (uptr i = 0; i < kSize1; i++) { u8 *p = Get(i); @@ -822,6 +823,10 @@ class SizeClassAllocator32 { void PrintStats() { } + static uptr AdditionalSize() { + return 0; + } + typedef SizeClassMap SizeClassMapT; static const uptr kNumClasses = SizeClassMap::kNumClasses; @@ -868,9 +873,9 @@ class SizeClassAllocator32 { uptr reg = AllocateRegion(stat, class_id); uptr n_chunks = kRegionSize / (size + kMetadataSize); uptr max_count = SizeClassMap::MaxCached(class_id); - Batch *b = 0; + Batch *b = nullptr; for (uptr i = reg; i < reg + n_chunks * size; i += size) { - if (b == 0) { + if (!b) { if (SizeClassMap::SizeClassRequiresSeparateTransferBatch(class_id)) b = (Batch*)c->Allocate(this, SizeClassMap::ClassID(sizeof(Batch))); else @@ -881,7 +886,7 @@ class SizeClassAllocator32 { if (b->count == max_count) { CHECK_GT(b->count, 0); sci->free_list.push_back(b); - b = 0; + b = nullptr; } } if (b) { @@ -1061,7 +1066,7 @@ class LargeMmapAllocator { void *ReturnNullOrDie() { if (atomic_load(&may_return_null_, memory_order_acquire)) - return 0; + return nullptr; ReportAllocatorCannotReturnNull(); } @@ -1101,7 +1106,7 @@ class LargeMmapAllocator { } bool PointerIsMine(const void *p) { - return GetBlockBegin(p) != 0; + return GetBlockBegin(p) != nullptr; } uptr GetActuallyAllocatedSize(void *p) { @@ -1130,13 +1135,13 @@ class LargeMmapAllocator { nearest_chunk = ch; } if (!nearest_chunk) - return 0; + return nullptr; Header *h = reinterpret_cast<Header *>(nearest_chunk); CHECK_GE(nearest_chunk, h->map_beg); CHECK_LT(nearest_chunk, h->map_beg + h->map_size); CHECK_LE(nearest_chunk, p); if (h->map_beg + h->map_size <= p) - return 0; + return nullptr; return GetUser(h); } @@ -1146,7 +1151,7 @@ class LargeMmapAllocator { mutex_.CheckLocked(); uptr p = reinterpret_cast<uptr>(ptr); uptr n = n_chunks_; - if (!n) return 0; + if (!n) return nullptr; if (!chunks_sorted_) { // Do one-time sort. chunks_sorted_ is reset in Allocate/Deallocate. SortArray(reinterpret_cast<uptr*>(chunks_), n); @@ -1158,7 +1163,7 @@ class LargeMmapAllocator { chunks_[n - 1]->map_size; } if (p < min_mmap_ || p >= max_mmap_) - return 0; + return nullptr; uptr beg = 0, end = n - 1; // This loop is a log(n) lower_bound. It does not check for the exact match // to avoid expensive cache-thrashing loads. @@ -1179,7 +1184,7 @@ class LargeMmapAllocator { Header *h = chunks_[beg]; if (h->map_beg + h->map_size <= p || p < h->map_beg) - return 0; + return nullptr; return GetUser(h); } @@ -1308,7 +1313,7 @@ class CombinedAllocator { void *ReturnNullOrDie() { if (MayReturnNull()) - return 0; + return nullptr; ReportAllocatorCannotReturnNull(); } @@ -1340,7 +1345,7 @@ class CombinedAllocator { return Allocate(cache, new_size, alignment); if (!new_size) { Deallocate(cache, p); - return 0; + return nullptr; } CHECK(PointerIsMine(p)); uptr old_size = GetActuallyAllocatedSize(p); @@ -1445,7 +1450,6 @@ class CombinedAllocator { // Returns true if calloc(size, n) should return 0 due to overflow in size*n. bool CallocShouldReturnNullDueToOverflow(uptr size, uptr n); -} // namespace __sanitizer - -#endif // SANITIZER_ALLOCATOR_H +} // namespace __sanitizer +#endif // SANITIZER_ALLOCATOR_H diff --git a/lib/sanitizer_common/sanitizer_allocator_internal.h b/lib/sanitizer_common/sanitizer_allocator_internal.h index 9b9cfd0b5931..3dcfccd7cba3 100644 --- a/lib/sanitizer_common/sanitizer_allocator_internal.h +++ b/lib/sanitizer_common/sanitizer_allocator_internal.h @@ -1,4 +1,4 @@ -//===-- sanitizer_allocator_internal.h -------------------------- C++ -----===// +//===-- sanitizer_allocator_internal.h --------------------------*- C++ -*-===// // // The LLVM Compiler Infrastructure // @@ -45,19 +45,19 @@ typedef SizeClassAllocatorLocalCache<PrimaryInternalAllocator> typedef CombinedAllocator<PrimaryInternalAllocator, InternalAllocatorCache, LargeMmapAllocator<> > InternalAllocator; -void *InternalAlloc(uptr size, InternalAllocatorCache *cache = 0); -void InternalFree(void *p, InternalAllocatorCache *cache = 0); +void *InternalAlloc(uptr size, InternalAllocatorCache *cache = nullptr); +void InternalFree(void *p, InternalAllocatorCache *cache = nullptr); InternalAllocator *internal_allocator(); enum InternalAllocEnum { INTERNAL_ALLOC }; -} // namespace __sanitizer +} // namespace __sanitizer inline void *operator new(__sanitizer::operator_new_size_type size, InternalAllocEnum) { return InternalAlloc(size); } -#endif // SANITIZER_ALLOCATOR_INTERNAL_H +#endif // SANITIZER_ALLOCATOR_INTERNAL_H diff --git a/lib/sanitizer_common/sanitizer_asm.h b/lib/sanitizer_common/sanitizer_asm.h index 906012a96f11..47c2b12a2049 100644 --- a/lib/sanitizer_common/sanitizer_asm.h +++ b/lib/sanitizer_common/sanitizer_asm.h @@ -23,8 +23,11 @@ # define CFI_STARTPROC .cfi_startproc # define CFI_ENDPROC .cfi_endproc # define CFI_ADJUST_CFA_OFFSET(n) .cfi_adjust_cfa_offset n +# define CFI_DEF_CFA_OFFSET(n) .cfi_def_cfa_offset n # define CFI_REL_OFFSET(reg, n) .cfi_rel_offset reg, n +# define CFI_OFFSET(reg, n) .cfi_offset reg, n # define CFI_DEF_CFA_REGISTER(reg) .cfi_def_cfa_register reg +# define CFI_DEF_CFA(reg, n) .cfi_def_cfa reg, n # define CFI_RESTORE(reg) .cfi_restore reg #else // No CFI @@ -32,9 +35,24 @@ # define CFI_STARTPROC # define CFI_ENDPROC # define CFI_ADJUST_CFA_OFFSET(n) +# define CFI_DEF_CFA_OFFSET(n) # define CFI_REL_OFFSET(reg, n) +# define CFI_OFFSET(reg, n) # define CFI_DEF_CFA_REGISTER(reg) +# define CFI_DEF_CFA(reg, n) # define CFI_RESTORE(reg) #endif - +#if !defined(__APPLE__) +# define ASM_HIDDEN(symbol) .hidden symbol +# define ASM_TYPE_FUNCTION(symbol) .type symbol, @function +# define ASM_SIZE(symbol) .size symbol, .-symbol +# define ASM_TSAN_SYMBOL(symbol) symbol +# define ASM_TSAN_SYMBOL_INTERCEPTOR(symbol) symbol +#else +# define ASM_HIDDEN(symbol) +# define ASM_TYPE_FUNCTION(symbol) +# define ASM_SIZE(symbol) +# define ASM_TSAN_SYMBOL(symbol) _##symbol +# define ASM_TSAN_SYMBOL_INTERCEPTOR(symbol) _wrap_##symbol +#endif diff --git a/lib/sanitizer_common/sanitizer_atomic.h b/lib/sanitizer_common/sanitizer_atomic.h index 7e3374aadd0c..b26693e24f8d 100644 --- a/lib/sanitizer_common/sanitizer_atomic.h +++ b/lib/sanitizer_common/sanitizer_atomic.h @@ -63,4 +63,20 @@ struct atomic_uintptr_t { # error "Unsupported compiler" #endif +namespace __sanitizer { + +// Clutter-reducing helpers. + +template<typename T> +INLINE typename T::Type atomic_load_relaxed(const volatile T *a) { + return atomic_load(a, memory_order_relaxed); +} + +template<typename T> +INLINE void atomic_store_relaxed(volatile T *a, typename T::Type v) { + atomic_store(a, v, memory_order_relaxed); +} + +} // namespace __sanitizer + #endif // SANITIZER_ATOMIC_H diff --git a/lib/sanitizer_common/sanitizer_common.cc b/lib/sanitizer_common/sanitizer_common.cc index d14e98824d99..9b41a3aa0af9 100644 --- a/lib/sanitizer_common/sanitizer_common.cc +++ b/lib/sanitizer_common/sanitizer_common.cc @@ -57,7 +57,7 @@ void ReportFile::ReopenIfNecessary() { CloseFile(fd); } - const char *exe_name = GetBinaryBasename(); + const char *exe_name = GetProcessName(); if (common_flags()->log_exe_name && exe_name) { internal_snprintf(full_path, kMaxPathLength, "%s.%s.%zu", path_prefix, exe_name, pid); @@ -105,24 +105,47 @@ uptr stoptheworld_tracer_pid = 0; // writing to the same log file. uptr stoptheworld_tracer_ppid = 0; -static DieCallbackType InternalDieCallback, UserDieCallback; -void SetDieCallback(DieCallbackType callback) { - InternalDieCallback = callback; +static const int kMaxNumOfInternalDieCallbacks = 5; +static DieCallbackType InternalDieCallbacks[kMaxNumOfInternalDieCallbacks]; + +bool AddDieCallback(DieCallbackType callback) { + for (int i = 0; i < kMaxNumOfInternalDieCallbacks; i++) { + if (InternalDieCallbacks[i] == nullptr) { + InternalDieCallbacks[i] = callback; + return true; + } + } + return false; } -void SetUserDieCallback(DieCallbackType callback) { - UserDieCallback = callback; + +bool RemoveDieCallback(DieCallbackType callback) { + for (int i = 0; i < kMaxNumOfInternalDieCallbacks; i++) { + if (InternalDieCallbacks[i] == callback) { + internal_memmove(&InternalDieCallbacks[i], &InternalDieCallbacks[i + 1], + sizeof(InternalDieCallbacks[0]) * + (kMaxNumOfInternalDieCallbacks - i - 1)); + InternalDieCallbacks[kMaxNumOfInternalDieCallbacks - 1] = nullptr; + return true; + } + } + return false; } -DieCallbackType GetDieCallback() { - return InternalDieCallback; +static DieCallbackType UserDieCallback; +void SetUserDieCallback(DieCallbackType callback) { + UserDieCallback = callback; } void NORETURN Die() { if (UserDieCallback) UserDieCallback(); - if (InternalDieCallback) - InternalDieCallback(); - internal__exit(1); + for (int i = kMaxNumOfInternalDieCallbacks - 1; i >= 0; i--) { + if (InternalDieCallbacks[i]) + InternalDieCallbacks[i](); + } + if (common_flags()->abort_on_error) + Abort(); + internal__exit(common_flags()->exitcode); } static CheckFailedCallbackType CheckFailedCallback; @@ -140,40 +163,60 @@ void NORETURN CheckFailed(const char *file, int line, const char *cond, Die(); } -uptr ReadFileToBuffer(const char *file_name, char **buff, uptr *buff_size, - uptr max_len, error_t *errno_p) { +void NORETURN ReportMmapFailureAndDie(uptr size, const char *mem_type, + const char *mmap_type, error_t err, + bool raw_report) { + static int recursion_count; + if (raw_report || recursion_count) { + // If raw report is requested or we went into recursion, just die. + // The Report() and CHECK calls below may call mmap recursively and fail. + RawWrite("ERROR: Failed to mmap\n"); + Die(); + } + recursion_count++; + Report("ERROR: %s failed to " + "%s 0x%zx (%zd) bytes of %s (error code: %d)\n", + SanitizerToolName, mmap_type, size, size, mem_type, err); +#ifndef SANITIZER_GO + DumpProcessMap(); +#endif + UNREACHABLE("unable to mmap"); +} + +bool ReadFileToBuffer(const char *file_name, char **buff, uptr *buff_size, + uptr *read_len, uptr max_len, error_t *errno_p) { uptr PageSize = GetPageSizeCached(); uptr kMinFileLen = PageSize; - uptr read_len = 0; - *buff = 0; + *buff = nullptr; *buff_size = 0; + *read_len = 0; // The files we usually open are not seekable, so try different buffer sizes. for (uptr size = kMinFileLen; size <= max_len; size *= 2) { fd_t fd = OpenFile(file_name, RdOnly, errno_p); - if (fd == kInvalidFd) return 0; + if (fd == kInvalidFd) return false; UnmapOrDie(*buff, *buff_size); *buff = (char*)MmapOrDie(size, __func__); *buff_size = size; + *read_len = 0; // Read up to one page at a time. - read_len = 0; bool reached_eof = false; - while (read_len + PageSize <= size) { + while (*read_len + PageSize <= size) { uptr just_read; - if (!ReadFromFile(fd, *buff + read_len, PageSize, &just_read, errno_p)) { + if (!ReadFromFile(fd, *buff + *read_len, PageSize, &just_read, errno_p)) { UnmapOrDie(*buff, *buff_size); - return 0; + return false; } if (just_read == 0) { reached_eof = true; break; } - read_len += just_read; + *read_len += just_read; } CloseFile(fd); if (reached_eof) // We've read the whole file. break; } - return read_len; + return true; } typedef bool UptrComparisonFunction(const uptr &a, const uptr &b); @@ -210,8 +253,8 @@ void *MmapAlignedOrDie(uptr size, uptr alignment, const char *mem_type) { const char *StripPathPrefix(const char *filepath, const char *strip_path_prefix) { - if (filepath == 0) return 0; - if (strip_path_prefix == 0) return filepath; + if (!filepath) return nullptr; + if (!strip_path_prefix) return filepath; const char *res = filepath; if (const char *pos = internal_strstr(filepath, strip_path_prefix)) res = pos + internal_strlen(strip_path_prefix); @@ -221,8 +264,8 @@ const char *StripPathPrefix(const char *filepath, } const char *StripModuleName(const char *module) { - if (module == 0) - return 0; + if (!module) + return nullptr; if (SANITIZER_WINDOWS) { // On Windows, both slash and backslash are possible. // Pick the one that goes last. @@ -255,6 +298,40 @@ void ReportErrorSummary(const char *error_type, const AddressInfo &info) { } #endif +// Removes the ANSI escape sequences from the input string (in-place). +void RemoveANSIEscapeSequencesFromString(char *str) { + if (!str) + return; + + // We are going to remove the escape sequences in place. + char *s = str; + char *z = str; + while (*s != '\0') { + CHECK_GE(s, z); + // Skip over ANSI escape sequences with pointer 's'. + if (*s == '\033' && *(s + 1) == '[') { + s = internal_strchrnul(s, 'm'); + if (*s == '\0') { + break; + } + s++; + continue; + } + // 's' now points at a character we want to keep. Copy over the buffer + // content if the escape sequence has been perviously skipped andadvance + // both pointers. + if (s != z) + *z = *s; + + // If we have not seen an escape sequence, just advance both pointers. + z++; + s++; + } + + // Null terminate the string. + *z = '\0'; +} + void LoadedModule::set(const char *module_name, uptr base_address) { clear(); full_name_ = internal_strdup(module_name); @@ -303,7 +380,7 @@ void DecreaseTotalMmap(uptr size) { } bool TemplateMatch(const char *templ, const char *str) { - if (str == 0 || str[0] == 0) + if ((!str) || str[0] == 0) return false; bool start = false; if (templ && templ[0] == '^') { @@ -324,9 +401,9 @@ bool TemplateMatch(const char *templ, const char *str) { return false; char *tpos = (char*)internal_strchr(templ, '*'); char *tpos1 = (char*)internal_strchr(templ, '$'); - if (tpos == 0 || (tpos1 && tpos1 < tpos)) + if ((!tpos) || (tpos1 && tpos1 < tpos)) tpos = tpos1; - if (tpos != 0) + if (tpos) tpos[0] = 0; const char *str0 = str; const char *spos = internal_strstr(str, templ); @@ -334,7 +411,7 @@ bool TemplateMatch(const char *templ, const char *str) { templ = tpos; if (tpos) tpos[0] = tpos == tpos1 ? '$' : '*'; - if (spos == 0) + if (!spos) return false; if (start && spos != str0) return false; @@ -344,11 +421,52 @@ bool TemplateMatch(const char *templ, const char *str) { return true; } +static const char kPathSeparator = SANITIZER_WINDOWS ? ';' : ':'; + +char *FindPathToBinary(const char *name) { + const char *path = GetEnv("PATH"); + if (!path) + return nullptr; + uptr name_len = internal_strlen(name); + InternalScopedBuffer<char> buffer(kMaxPathLength); + const char *beg = path; + while (true) { + const char *end = internal_strchrnul(beg, kPathSeparator); + uptr prefix_len = end - beg; + if (prefix_len + name_len + 2 <= kMaxPathLength) { + internal_memcpy(buffer.data(), beg, prefix_len); + buffer[prefix_len] = '/'; + internal_memcpy(&buffer[prefix_len + 1], name, name_len); + buffer[prefix_len + 1 + name_len] = '\0'; + if (FileExists(buffer.data())) + return internal_strdup(buffer.data()); + } + if (*end == '\0') break; + beg = end + 1; + } + return nullptr; +} + static char binary_name_cache_str[kMaxPathLength]; -static const char *binary_basename_cache_str; +static char process_name_cache_str[kMaxPathLength]; + +const char *GetProcessName() { + return process_name_cache_str; +} + +static uptr ReadProcessName(/*out*/ char *buf, uptr buf_len) { + ReadLongProcessName(buf, buf_len); + char *s = const_cast<char *>(StripModuleName(buf)); + uptr len = internal_strlen(s); + if (s != buf) { + internal_memmove(buf, s, len); + buf[len] = '\0'; + } + return len; +} -const char *GetBinaryBasename() { - return binary_basename_cache_str; +void UpdateProcessName() { + ReadProcessName(process_name_cache_str, sizeof(process_name_cache_str)); } // Call once to make sure that binary_name_cache_str is initialized @@ -356,7 +474,7 @@ void CacheBinaryName() { if (binary_name_cache_str[0] != '\0') return; ReadBinaryName(binary_name_cache_str, sizeof(binary_name_cache_str)); - binary_basename_cache_str = StripModuleName(binary_name_cache_str); + ReadProcessName(process_name_cache_str, sizeof(process_name_cache_str)); } uptr ReadBinaryNameCached(/*out*/char *buf, uptr buf_len) { @@ -370,7 +488,7 @@ uptr ReadBinaryNameCached(/*out*/char *buf, uptr buf_len) { return name_len; } -} // namespace __sanitizer +} // namespace __sanitizer using namespace __sanitizer; // NOLINT @@ -387,4 +505,4 @@ SANITIZER_INTERFACE_ATTRIBUTE void __sanitizer_set_death_callback(void (*callback)(void)) { SetUserDieCallback(callback); } -} // extern "C" +} // extern "C" diff --git a/lib/sanitizer_common/sanitizer_common.h b/lib/sanitizer_common/sanitizer_common.h index 2c5a8dbe1238..0585f6b15b87 100644 --- a/lib/sanitizer_common/sanitizer_common.h +++ b/lib/sanitizer_common/sanitizer_common.h @@ -49,6 +49,8 @@ static const uptr kMaxNumberOfModules = 1 << 14; const uptr kMaxThreadStackSize = 1 << 30; // 1Gb +static const uptr kErrorMessageBufferSize = 1 << 16; + // Denotes fake PC values that come from JIT/JAVA/etc. // For such PC values __tsan_symbolize_external() will be called. const u64 kExternalPCBit = 1ULL << 60; @@ -76,7 +78,10 @@ void GetThreadStackAndTls(bool main, uptr *stk_addr, uptr *stk_size, uptr *tls_addr, uptr *tls_size); // Memory management -void *MmapOrDie(uptr size, const char *mem_type); +void *MmapOrDie(uptr size, const char *mem_type, bool raw_report = false); +INLINE void *MmapOrDieQuietly(uptr size, const char *mem_type) { + return MmapOrDie(size, mem_type, /*raw_report*/ true); +} void UnmapOrDie(void *addr, uptr size); void *MmapFixedNoReserve(uptr fixed_addr, uptr size, const char *name = nullptr); @@ -97,6 +102,8 @@ void DecreaseTotalMmap(uptr size); uptr GetRSS(); void NoHugePagesInRegion(uptr addr, uptr length); void DontDumpShadowMemory(uptr addr, uptr length); +// Check if the built VMA size matches the runtime one. +void CheckVMASize(); // InternalScopedBuffer can be used instead of large stack arrays to // keep frame size low. @@ -160,6 +167,7 @@ void SetLowLevelAllocateCallback(LowLevelAllocateCallback callback); // IO void RawWrite(const char *buffer); bool ColorizeReports(); +void RemoveANSIEscapeSequencesFromString(char *buffer); void Printf(const char *format, ...); void Report(const char *format, ...); void SetPrintfAndReportCallback(void (*callback)(const char *)); @@ -224,14 +232,23 @@ bool WriteToFile(fd_t fd, const void *buff, uptr buff_size, bool RenameFile(const char *oldpath, const char *newpath, error_t *error_p = nullptr); +// Scoped file handle closer. +struct FileCloser { + explicit FileCloser(fd_t fd) : fd(fd) {} + ~FileCloser() { CloseFile(fd); } + fd_t fd; +}; + bool SupportsColoredOutput(fd_t fd); // Opens the file 'file_name" and reads up to 'max_len' bytes. // The resulting buffer is mmaped and stored in '*buff'. -// The size of the mmaped region is stored in '*buff_size', -// Returns the number of read bytes or 0 if file can not be opened. -uptr ReadFileToBuffer(const char *file_name, char **buff, uptr *buff_size, - uptr max_len, error_t *errno_p = nullptr); +// The size of the mmaped region is stored in '*buff_size'. +// The total number of read bytes is stored in '*read_len'. +// Returns true if file was successfully opened and read. +bool ReadFileToBuffer(const char *file_name, char **buff, uptr *buff_size, + uptr *read_len, uptr max_len = 1 << 26, + error_t *errno_p = nullptr); // Maps given file to virtual memory, and returns pointer to it // (or NULL if mapping fails). Stores the size of mmaped region // in '*buff_size'. @@ -249,7 +266,9 @@ const char *StripModuleName(const char *module); // OS uptr ReadBinaryName(/*out*/char *buf, uptr buf_len); uptr ReadBinaryNameCached(/*out*/char *buf, uptr buf_len); -const char *GetBinaryBasename(); +uptr ReadLongProcessName(/*out*/ char *buf, uptr buf_len); +const char *GetProcessName(); +void UpdateProcessName(); void CacheBinaryName(); void DisableCoreDumperIfNecessary(); void DumpProcessMap(); @@ -295,6 +314,9 @@ void NORETURN Abort(); void NORETURN Die(); void NORETURN CheckFailed(const char *file, int line, const char *cond, u64 v1, u64 v2); +void NORETURN ReportMmapFailureAndDie(uptr size, const char *mem_type, + const char *mmap_type, error_t err, + bool raw_report = false); // Set the name of the current thread to 'name', return true on succees. // The name may be truncated to a system-dependent limit. @@ -306,9 +328,16 @@ bool SanitizerGetThreadName(char *name, int max_len); // Specific tools may override behavior of "Die" and "CheckFailed" functions // to do tool-specific job. typedef void (*DieCallbackType)(void); -void SetDieCallback(DieCallbackType); -void SetUserDieCallback(DieCallbackType); -DieCallbackType GetDieCallback(); + +// It's possible to add several callbacks that would be run when "Die" is +// called. The callbacks will be run in the opposite order. The tools are +// strongly recommended to setup all callbacks during initialization, when there +// is only a single thread. +bool AddDieCallback(DieCallbackType callback); +bool RemoveDieCallback(DieCallbackType callback); + +void SetUserDieCallback(DieCallbackType callback); + typedef void (*CheckFailedCallbackType)(const char *, int, const char *, u64, u64); void SetCheckFailedCallback(CheckFailedCallbackType callback); @@ -400,7 +429,7 @@ INLINE uptr RoundUpToPowerOfTwo(uptr size) { } INLINE uptr RoundUpTo(uptr size, uptr boundary) { - CHECK(IsPowerOfTwo(boundary)); + RAW_CHECK(IsPowerOfTwo(boundary)); return (size + boundary - 1) & ~(boundary - 1); } @@ -626,17 +655,34 @@ enum AndroidApiLevel { ANDROID_POST_LOLLIPOP = 23 }; -#if SANITIZER_ANDROID +void WriteToSyslog(const char *buffer); + +#if SANITIZER_MAC +void LogFullErrorReport(const char *buffer); +#else +INLINE void LogFullErrorReport(const char *buffer) {} +#endif + +#if SANITIZER_LINUX || SANITIZER_MAC +void WriteOneLineToSyslog(const char *s); +#else +INLINE void WriteOneLineToSyslog(const char *s) {} +#endif + +#if SANITIZER_LINUX // Initialize Android logging. Any writes before this are silently lost. void AndroidLogInit(); -void AndroidLogWrite(const char *buffer); -void GetExtraActivationFlags(char *buf, uptr size); +bool ShouldLogAfterPrintf(); +#else +INLINE void AndroidLogInit() {} +INLINE bool ShouldLogAfterPrintf() { return false; } +#endif + +#if SANITIZER_ANDROID void SanitizerInitializeUnwinder(); AndroidApiLevel AndroidGetApiLevel(); #else -INLINE void AndroidLogInit() {} INLINE void AndroidLogWrite(const char *buffer_unused) {} -INLINE void GetExtraActivationFlags(char *buf, uptr size) { *buf = '\0'; } INLINE void SanitizerInitializeUnwinder() {} INLINE AndroidApiLevel AndroidGetApiLevel() { return ANDROID_NOT_ANDROID; } #endif @@ -685,6 +731,9 @@ struct SignalContext { void GetPcSpBp(void *context, uptr *pc, uptr *sp, uptr *bp); +void DisableReexec(); +void MaybeReexec(); + } // namespace __sanitizer inline void *operator new(__sanitizer::operator_new_size_type size, diff --git a/lib/sanitizer_common/sanitizer_common_interceptors.inc b/lib/sanitizer_common/sanitizer_common_interceptors.inc index a7772b7394a5..4639ddc92c6c 100644 --- a/lib/sanitizer_common/sanitizer_common_interceptors.inc +++ b/lib/sanitizer_common/sanitizer_common_interceptors.inc @@ -31,6 +31,7 @@ // COMMON_INTERCEPTOR_HANDLE_RECVMSG // COMMON_INTERCEPTOR_NOTHING_IS_INITIALIZED //===----------------------------------------------------------------------===// + #include "interception/interception.h" #include "sanitizer_addrhashmap.h" #include "sanitizer_placement_new.h" @@ -39,6 +40,22 @@ #include <stdarg.h> +#if SANITIZER_INTERCEPTOR_HOOKS +#define CALL_WEAK_INTERCEPTOR_HOOK(f, ...) \ + do { \ + if (f) \ + f(__VA_ARGS__); \ + } while (false); +#define DECLARE_WEAK_INTERCEPTOR_HOOK(f, ...) \ + extern "C" { \ + SANITIZER_INTERFACE_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE void f(__VA_ARGS__); \ + } // extern "C" +#else +#define DECLARE_WEAK_INTERCEPTOR_HOOK(f, ...) +#define CALL_WEAK_INTERCEPTOR_HOOK(f, ...) + +#endif // SANITIZER_INTERCEPTOR_HOOKS + #if SANITIZER_WINDOWS && !defined(va_copy) #define va_copy(dst, src) ((dst) = (src)) #endif // _WIN32 @@ -118,6 +135,14 @@ #define COMMON_INTERCEPTOR_GET_TLS_RANGE(begin, end) *begin = *end = 0; #endif +#ifndef COMMON_INTERCEPTOR_ACQUIRE +#define COMMON_INTERCEPTOR_ACQUIRE(ctx, u) {} +#endif + +#ifndef COMMON_INTERCEPTOR_RELEASE +#define COMMON_INTERCEPTOR_RELEASE(ctx, u) {} +#endif + struct FileMetadata { // For open_memstream(). char **addr; @@ -188,9 +213,14 @@ static inline int CharCmpX(unsigned char c1, unsigned char c2) { return (c1 == c2) ? 0 : (c1 < c2) ? -1 : 1; } +DECLARE_WEAK_INTERCEPTOR_HOOK(__sanitizer_weak_hook_strcmp, uptr called_pc, + const char *s1, const char *s2) + INTERCEPTOR(int, strcmp, const char *s1, const char *s2) { void *ctx; COMMON_INTERCEPTOR_ENTER(ctx, strcmp, s1, s2); + CALL_WEAK_INTERCEPTOR_HOOK(__sanitizer_weak_hook_strcmp, GET_CALLER_PC(), s1, + s2); unsigned char c1, c2; uptr i; for (i = 0;; i++) { @@ -203,11 +233,16 @@ INTERCEPTOR(int, strcmp, const char *s1, const char *s2) { return CharCmpX(c1, c2); } +DECLARE_WEAK_INTERCEPTOR_HOOK(__sanitizer_weak_hook_strncmp, uptr called_pc, + const char *s1, const char *s2, uptr n) + INTERCEPTOR(int, strncmp, const char *s1, const char *s2, uptr size) { if (COMMON_INTERCEPTOR_NOTHING_IS_INITIALIZED) return internal_strncmp(s1, s2, size); void *ctx; COMMON_INTERCEPTOR_ENTER(ctx, strncmp, s1, s2, size); + CALL_WEAK_INTERCEPTOR_HOOK(__sanitizer_weak_hook_strncmp, GET_CALLER_PC(), s1, + s2, size); unsigned char c1 = 0, c2 = 0; uptr i; for (i = 0; i < size; i++) { @@ -362,8 +397,52 @@ INTERCEPTOR(char *, strpbrk, const char *s1, const char *s2) { #define INIT_STRPBRK #endif +#if SANITIZER_INTERCEPT_MEMCMP + +DECLARE_WEAK_INTERCEPTOR_HOOK(__sanitizer_weak_hook_memcmp, uptr called_pc, + const void *s1, const void *s2, uptr n) + +INTERCEPTOR(int, memcmp, const void *a1, const void *a2, uptr size) { + if (COMMON_INTERCEPTOR_NOTHING_IS_INITIALIZED) + return internal_memcmp(a1, a2, size); + void *ctx; + COMMON_INTERCEPTOR_ENTER(ctx, memcmp, a1, a2, size); + CALL_WEAK_INTERCEPTOR_HOOK(__sanitizer_weak_hook_memcmp, GET_CALLER_PC(), a1, + a2, size); + if (common_flags()->intercept_memcmp) { + if (common_flags()->strict_memcmp) { + // Check the entire regions even if the first bytes of the buffers are + // different. + COMMON_INTERCEPTOR_READ_RANGE(ctx, a1, size); + COMMON_INTERCEPTOR_READ_RANGE(ctx, a2, size); + // Fallthrough to REAL(memcmp) below. + } else { + unsigned char c1 = 0, c2 = 0; + const unsigned char *s1 = (const unsigned char*)a1; + const unsigned char *s2 = (const unsigned char*)a2; + uptr i; + for (i = 0; i < size; i++) { + c1 = s1[i]; + c2 = s2[i]; + if (c1 != c2) break; + } + COMMON_INTERCEPTOR_READ_RANGE(ctx, s1, Min(i + 1, size)); + COMMON_INTERCEPTOR_READ_RANGE(ctx, s2, Min(i + 1, size)); + return CharCmpX(c1, c2); + } + } + return REAL(memcmp(a1, a2, size)); +} + +#define INIT_MEMCMP COMMON_INTERCEPT_FUNCTION(memcmp) +#else +#define INIT_MEMCMP +#endif + #if SANITIZER_INTERCEPT_MEMCHR INTERCEPTOR(void*, memchr, const void *s, int c, SIZE_T n) { + if (COMMON_INTERCEPTOR_NOTHING_IS_INITIALIZED) + return internal_memchr(s, c, n); void *ctx; COMMON_INTERCEPTOR_ENTER(ctx, memchr, s, c, n); void *res = REAL(memchr)(s, c, n); @@ -411,7 +490,7 @@ INTERCEPTOR(float, frexpf, float x, int *exp) { COMMON_INTERCEPTOR_ENTER(ctx, frexpf, x, exp); // FIXME: under ASan the call below may write to freed memory and corrupt // its metadata. See - // https://code.google.com/p/address-sanitizer/issues/detail?id=321. + // https://github.com/google/sanitizers/issues/321. float res = REAL(frexpf)(x, exp); COMMON_INTERCEPTOR_WRITE_RANGE(ctx, exp, sizeof(*exp)); return res; @@ -422,7 +501,7 @@ INTERCEPTOR(long double, frexpl, long double x, int *exp) { COMMON_INTERCEPTOR_ENTER(ctx, frexpl, x, exp); // FIXME: under ASan the call below may write to freed memory and corrupt // its metadata. See - // https://code.google.com/p/address-sanitizer/issues/detail?id=321. + // https://github.com/google/sanitizers/issues/321. long double res = REAL(frexpl)(x, exp); COMMON_INTERCEPTOR_WRITE_RANGE(ctx, exp, sizeof(*exp)); return res; @@ -463,7 +542,7 @@ INTERCEPTOR(SSIZE_T, read, int fd, void *ptr, SIZE_T count) { COMMON_INTERCEPTOR_FD_ACCESS(ctx, fd); // FIXME: under ASan the call below may write to freed memory and corrupt // its metadata. See - // https://code.google.com/p/address-sanitizer/issues/detail?id=321. + // https://github.com/google/sanitizers/issues/321. SSIZE_T res = REAL(read)(fd, ptr, count); if (res > 0) COMMON_INTERCEPTOR_WRITE_RANGE(ctx, ptr, res); if (res >= 0 && fd >= 0) COMMON_INTERCEPTOR_FD_ACQUIRE(ctx, fd); @@ -481,7 +560,7 @@ INTERCEPTOR(SSIZE_T, pread, int fd, void *ptr, SIZE_T count, OFF_T offset) { COMMON_INTERCEPTOR_FD_ACCESS(ctx, fd); // FIXME: under ASan the call below may write to freed memory and corrupt // its metadata. See - // https://code.google.com/p/address-sanitizer/issues/detail?id=321. + // https://github.com/google/sanitizers/issues/321. SSIZE_T res = REAL(pread)(fd, ptr, count, offset); if (res > 0) COMMON_INTERCEPTOR_WRITE_RANGE(ctx, ptr, res); if (res >= 0 && fd >= 0) COMMON_INTERCEPTOR_FD_ACQUIRE(ctx, fd); @@ -499,7 +578,7 @@ INTERCEPTOR(SSIZE_T, pread64, int fd, void *ptr, SIZE_T count, OFF64_T offset) { COMMON_INTERCEPTOR_FD_ACCESS(ctx, fd); // FIXME: under ASan the call below may write to freed memory and corrupt // its metadata. See - // https://code.google.com/p/address-sanitizer/issues/detail?id=321. + // https://github.com/google/sanitizers/issues/321. SSIZE_T res = REAL(pread64)(fd, ptr, count, offset); if (res > 0) COMMON_INTERCEPTOR_WRITE_RANGE(ctx, ptr, res); if (res >= 0 && fd >= 0) COMMON_INTERCEPTOR_FD_ACQUIRE(ctx, fd); @@ -746,7 +825,7 @@ INTERCEPTOR(char *, ctime, unsigned long *timep) { COMMON_INTERCEPTOR_ENTER(ctx, ctime, timep); // FIXME: under ASan the call below may write to freed memory and corrupt // its metadata. See - // https://code.google.com/p/address-sanitizer/issues/detail?id=321. + // https://github.com/google/sanitizers/issues/321. char *res = REAL(ctime)(timep); if (res) { COMMON_INTERCEPTOR_READ_RANGE(ctx, timep, sizeof(*timep)); @@ -759,7 +838,7 @@ INTERCEPTOR(char *, ctime_r, unsigned long *timep, char *result) { COMMON_INTERCEPTOR_ENTER(ctx, ctime_r, timep, result); // FIXME: under ASan the call below may write to freed memory and corrupt // its metadata. See - // https://code.google.com/p/address-sanitizer/issues/detail?id=321. + // https://github.com/google/sanitizers/issues/321. char *res = REAL(ctime_r)(timep, result); if (res) { COMMON_INTERCEPTOR_READ_RANGE(ctx, timep, sizeof(*timep)); @@ -772,7 +851,7 @@ INTERCEPTOR(char *, asctime, __sanitizer_tm *tm) { COMMON_INTERCEPTOR_ENTER(ctx, asctime, tm); // FIXME: under ASan the call below may write to freed memory and corrupt // its metadata. See - // https://code.google.com/p/address-sanitizer/issues/detail?id=321. + // https://github.com/google/sanitizers/issues/321. char *res = REAL(asctime)(tm); if (res) { COMMON_INTERCEPTOR_READ_RANGE(ctx, tm, sizeof(*tm)); @@ -785,7 +864,7 @@ INTERCEPTOR(char *, asctime_r, __sanitizer_tm *tm, char *result) { COMMON_INTERCEPTOR_ENTER(ctx, asctime_r, tm, result); // FIXME: under ASan the call below may write to freed memory and corrupt // its metadata. See - // https://code.google.com/p/address-sanitizer/issues/detail?id=321. + // https://github.com/google/sanitizers/issues/321. char *res = REAL(asctime_r)(tm, result); if (res) { COMMON_INTERCEPTOR_READ_RANGE(ctx, tm, sizeof(*tm)); @@ -829,7 +908,7 @@ INTERCEPTOR(char *, strptime, char *s, char *format, __sanitizer_tm *tm) { COMMON_INTERCEPTOR_READ_RANGE(ctx, format, REAL(strlen)(format) + 1); // FIXME: under ASan the call below may write to freed memory and corrupt // its metadata. See - // https://code.google.com/p/address-sanitizer/issues/detail?id=321. + // https://github.com/google/sanitizers/issues/321. char *res = REAL(strptime)(s, format, tm); COMMON_INTERCEPTOR_READ_STRING(ctx, s, res ? res - s : 0); if (res && tm) { @@ -966,7 +1045,7 @@ FORMAT_INTERCEPTOR_IMPL(__isoc99_sscanf, __isoc99_vsscanf, str, format) // FIXME: under ASan the REAL() call below may write to freed memory and // corrupt its metadata. See -// https://code.google.com/p/address-sanitizer/issues/detail?id=321. +// https://github.com/google/sanitizers/issues/321. #define VSPRINTF_INTERCEPTOR_IMPL(vname, str, ...) \ { \ VPRINTF_INTERCEPTOR_ENTER(vname, str, __VA_ARGS__) \ @@ -983,7 +1062,7 @@ FORMAT_INTERCEPTOR_IMPL(__isoc99_sscanf, __isoc99_vsscanf, str, format) // FIXME: under ASan the REAL() call below may write to freed memory and // corrupt its metadata. See -// https://code.google.com/p/address-sanitizer/issues/detail?id=321. +// https://github.com/google/sanitizers/issues/321. #define VSNPRINTF_INTERCEPTOR_IMPL(vname, str, size, ...) \ { \ VPRINTF_INTERCEPTOR_ENTER(vname, str, size, __VA_ARGS__) \ @@ -1000,7 +1079,7 @@ FORMAT_INTERCEPTOR_IMPL(__isoc99_sscanf, __isoc99_vsscanf, str, format) // FIXME: under ASan the REAL() call below may write to freed memory and // corrupt its metadata. See -// https://code.google.com/p/address-sanitizer/issues/detail?id=321. +// https://github.com/google/sanitizers/issues/321. #define VASPRINTF_INTERCEPTOR_IMPL(vname, strp, ...) \ { \ VPRINTF_INTERCEPTOR_ENTER(vname, strp, __VA_ARGS__) \ @@ -1243,14 +1322,14 @@ INTERCEPTOR(__sanitizer_passwd *, getpwnam, const char *name) { COMMON_INTERCEPTOR_ENTER(ctx, getpwnam, name); COMMON_INTERCEPTOR_READ_RANGE(ctx, name, REAL(strlen)(name) + 1); __sanitizer_passwd *res = REAL(getpwnam)(name); - if (res != 0) unpoison_passwd(ctx, res); + if (res) unpoison_passwd(ctx, res); return res; } INTERCEPTOR(__sanitizer_passwd *, getpwuid, u32 uid) { void *ctx; COMMON_INTERCEPTOR_ENTER(ctx, getpwuid, uid); __sanitizer_passwd *res = REAL(getpwuid)(uid); - if (res != 0) unpoison_passwd(ctx, res); + if (res) unpoison_passwd(ctx, res); return res; } INTERCEPTOR(__sanitizer_group *, getgrnam, const char *name) { @@ -1258,14 +1337,14 @@ INTERCEPTOR(__sanitizer_group *, getgrnam, const char *name) { COMMON_INTERCEPTOR_ENTER(ctx, getgrnam, name); COMMON_INTERCEPTOR_READ_RANGE(ctx, name, REAL(strlen)(name) + 1); __sanitizer_group *res = REAL(getgrnam)(name); - if (res != 0) unpoison_group(ctx, res); + if (res) unpoison_group(ctx, res); return res; } INTERCEPTOR(__sanitizer_group *, getgrgid, u32 gid) { void *ctx; COMMON_INTERCEPTOR_ENTER(ctx, getgrgid, gid); __sanitizer_group *res = REAL(getgrgid)(gid); - if (res != 0) unpoison_group(ctx, res); + if (res) unpoison_group(ctx, res); return res; } #define INIT_GETPWNAM_AND_FRIENDS \ @@ -1285,7 +1364,7 @@ INTERCEPTOR(int, getpwnam_r, const char *name, __sanitizer_passwd *pwd, COMMON_INTERCEPTOR_READ_RANGE(ctx, name, REAL(strlen)(name) + 1); // FIXME: under ASan the call below may write to freed memory and corrupt // its metadata. See - // https://code.google.com/p/address-sanitizer/issues/detail?id=321. + // https://github.com/google/sanitizers/issues/321. int res = REAL(getpwnam_r)(name, pwd, buf, buflen, result); if (!res) { if (result && *result) unpoison_passwd(ctx, *result); @@ -1300,7 +1379,7 @@ INTERCEPTOR(int, getpwuid_r, u32 uid, __sanitizer_passwd *pwd, char *buf, COMMON_INTERCEPTOR_ENTER(ctx, getpwuid_r, uid, pwd, buf, buflen, result); // FIXME: under ASan the call below may write to freed memory and corrupt // its metadata. See - // https://code.google.com/p/address-sanitizer/issues/detail?id=321. + // https://github.com/google/sanitizers/issues/321. int res = REAL(getpwuid_r)(uid, pwd, buf, buflen, result); if (!res) { if (result && *result) unpoison_passwd(ctx, *result); @@ -1316,7 +1395,7 @@ INTERCEPTOR(int, getgrnam_r, const char *name, __sanitizer_group *grp, COMMON_INTERCEPTOR_READ_RANGE(ctx, name, REAL(strlen)(name) + 1); // FIXME: under ASan the call below may write to freed memory and corrupt // its metadata. See - // https://code.google.com/p/address-sanitizer/issues/detail?id=321. + // https://github.com/google/sanitizers/issues/321. int res = REAL(getgrnam_r)(name, grp, buf, buflen, result); if (!res) { if (result && *result) unpoison_group(ctx, *result); @@ -1331,7 +1410,7 @@ INTERCEPTOR(int, getgrgid_r, u32 gid, __sanitizer_group *grp, char *buf, COMMON_INTERCEPTOR_ENTER(ctx, getgrgid_r, gid, grp, buf, buflen, result); // FIXME: under ASan the call below may write to freed memory and corrupt // its metadata. See - // https://code.google.com/p/address-sanitizer/issues/detail?id=321. + // https://github.com/google/sanitizers/issues/321. int res = REAL(getgrgid_r)(gid, grp, buf, buflen, result); if (!res) { if (result && *result) unpoison_group(ctx, *result); @@ -1354,14 +1433,14 @@ INTERCEPTOR(__sanitizer_passwd *, getpwent, int dummy) { void *ctx; COMMON_INTERCEPTOR_ENTER(ctx, getpwent, dummy); __sanitizer_passwd *res = REAL(getpwent)(dummy); - if (res != 0) unpoison_passwd(ctx, res); + if (res) unpoison_passwd(ctx, res); return res; } INTERCEPTOR(__sanitizer_group *, getgrent, int dummy) { void *ctx; COMMON_INTERCEPTOR_ENTER(ctx, getgrent, dummy); __sanitizer_group *res = REAL(getgrent)(dummy); - if (res != 0) unpoison_group(ctx, res);; + if (res) unpoison_group(ctx, res);; return res; } #define INIT_GETPWENT \ @@ -1376,14 +1455,14 @@ INTERCEPTOR(__sanitizer_passwd *, fgetpwent, void *fp) { void *ctx; COMMON_INTERCEPTOR_ENTER(ctx, fgetpwent, fp); __sanitizer_passwd *res = REAL(fgetpwent)(fp); - if (res != 0) unpoison_passwd(ctx, res); + if (res) unpoison_passwd(ctx, res); return res; } INTERCEPTOR(__sanitizer_group *, fgetgrent, void *fp) { void *ctx; COMMON_INTERCEPTOR_ENTER(ctx, fgetgrent, fp); __sanitizer_group *res = REAL(fgetgrent)(fp); - if (res != 0) unpoison_group(ctx, res); + if (res) unpoison_group(ctx, res); return res; } #define INIT_FGETPWENT \ @@ -1400,7 +1479,7 @@ INTERCEPTOR(int, getpwent_r, __sanitizer_passwd *pwbuf, char *buf, COMMON_INTERCEPTOR_ENTER(ctx, getpwent_r, pwbuf, buf, buflen, pwbufp); // FIXME: under ASan the call below may write to freed memory and corrupt // its metadata. See - // https://code.google.com/p/address-sanitizer/issues/detail?id=321. + // https://github.com/google/sanitizers/issues/321. int res = REAL(getpwent_r)(pwbuf, buf, buflen, pwbufp); if (!res) { if (pwbufp && *pwbufp) unpoison_passwd(ctx, *pwbufp); @@ -1415,7 +1494,7 @@ INTERCEPTOR(int, fgetpwent_r, void *fp, __sanitizer_passwd *pwbuf, char *buf, COMMON_INTERCEPTOR_ENTER(ctx, fgetpwent_r, fp, pwbuf, buf, buflen, pwbufp); // FIXME: under ASan the call below may write to freed memory and corrupt // its metadata. See - // https://code.google.com/p/address-sanitizer/issues/detail?id=321. + // https://github.com/google/sanitizers/issues/321. int res = REAL(fgetpwent_r)(fp, pwbuf, buf, buflen, pwbufp); if (!res) { if (pwbufp && *pwbufp) unpoison_passwd(ctx, *pwbufp); @@ -1430,7 +1509,7 @@ INTERCEPTOR(int, getgrent_r, __sanitizer_group *pwbuf, char *buf, SIZE_T buflen, COMMON_INTERCEPTOR_ENTER(ctx, getgrent_r, pwbuf, buf, buflen, pwbufp); // FIXME: under ASan the call below may write to freed memory and corrupt // its metadata. See - // https://code.google.com/p/address-sanitizer/issues/detail?id=321. + // https://github.com/google/sanitizers/issues/321. int res = REAL(getgrent_r)(pwbuf, buf, buflen, pwbufp); if (!res) { if (pwbufp && *pwbufp) unpoison_group(ctx, *pwbufp); @@ -1445,7 +1524,7 @@ INTERCEPTOR(int, fgetgrent_r, void *fp, __sanitizer_group *pwbuf, char *buf, COMMON_INTERCEPTOR_ENTER(ctx, fgetgrent_r, fp, pwbuf, buf, buflen, pwbufp); // FIXME: under ASan the call below may write to freed memory and corrupt // its metadata. See - // https://code.google.com/p/address-sanitizer/issues/detail?id=321. + // https://github.com/google/sanitizers/issues/321. int res = REAL(fgetgrent_r)(fp, pwbuf, buf, buflen, pwbufp); if (!res) { if (pwbufp && *pwbufp) unpoison_group(ctx, *pwbufp); @@ -1502,7 +1581,7 @@ INTERCEPTOR(int, clock_getres, u32 clk_id, void *tp) { COMMON_INTERCEPTOR_ENTER(ctx, clock_getres, clk_id, tp); // FIXME: under ASan the call below may write to freed memory and corrupt // its metadata. See - // https://code.google.com/p/address-sanitizer/issues/detail?id=321. + // https://github.com/google/sanitizers/issues/321. int res = REAL(clock_getres)(clk_id, tp); if (!res && tp) { COMMON_INTERCEPTOR_WRITE_RANGE(ctx, tp, struct_timespec_sz); @@ -1514,7 +1593,7 @@ INTERCEPTOR(int, clock_gettime, u32 clk_id, void *tp) { COMMON_INTERCEPTOR_ENTER(ctx, clock_gettime, clk_id, tp); // FIXME: under ASan the call below may write to freed memory and corrupt // its metadata. See - // https://code.google.com/p/address-sanitizer/issues/detail?id=321. + // https://github.com/google/sanitizers/issues/321. int res = REAL(clock_gettime)(clk_id, tp); if (!res) { COMMON_INTERCEPTOR_WRITE_RANGE(ctx, tp, struct_timespec_sz); @@ -1541,7 +1620,7 @@ INTERCEPTOR(int, getitimer, int which, void *curr_value) { COMMON_INTERCEPTOR_ENTER(ctx, getitimer, which, curr_value); // FIXME: under ASan the call below may write to freed memory and corrupt // its metadata. See - // https://code.google.com/p/address-sanitizer/issues/detail?id=321. + // https://github.com/google/sanitizers/issues/321. int res = REAL(getitimer)(which, curr_value); if (!res && curr_value) { COMMON_INTERCEPTOR_WRITE_RANGE(ctx, curr_value, struct_itimerval_sz); @@ -1555,7 +1634,7 @@ INTERCEPTOR(int, setitimer, int which, const void *new_value, void *old_value) { COMMON_INTERCEPTOR_READ_RANGE(ctx, new_value, struct_itimerval_sz); // FIXME: under ASan the call below may write to freed memory and corrupt // its metadata. See - // https://code.google.com/p/address-sanitizer/issues/detail?id=321. + // https://github.com/google/sanitizers/issues/321. int res = REAL(setitimer)(which, new_value, old_value); if (!res && old_value) { COMMON_INTERCEPTOR_WRITE_RANGE(ctx, old_value, struct_itimerval_sz); @@ -1612,16 +1691,19 @@ static int wrapped_gl_stat(const char *s, void *st) { return pglob_copy->gl_stat(s, st); } +static const __sanitizer_glob_t kGlobCopy = { + 0, 0, 0, + 0, wrapped_gl_closedir, wrapped_gl_readdir, + wrapped_gl_opendir, wrapped_gl_lstat, wrapped_gl_stat}; + INTERCEPTOR(int, glob, const char *pattern, int flags, int (*errfunc)(const char *epath, int eerrno), __sanitizer_glob_t *pglob) { void *ctx; COMMON_INTERCEPTOR_ENTER(ctx, glob, pattern, flags, errfunc, pglob); COMMON_INTERCEPTOR_READ_STRING(ctx, pattern, 0); - __sanitizer_glob_t glob_copy = { - 0, 0, 0, - 0, wrapped_gl_closedir, wrapped_gl_readdir, - wrapped_gl_opendir, wrapped_gl_lstat, wrapped_gl_stat}; + __sanitizer_glob_t glob_copy; + internal_memcpy(&glob_copy, &kGlobCopy, sizeof(glob_copy)); if (flags & glob_altdirfunc) { Swap(pglob->gl_closedir, glob_copy.gl_closedir); Swap(pglob->gl_readdir, glob_copy.gl_readdir); @@ -1649,10 +1731,8 @@ INTERCEPTOR(int, glob64, const char *pattern, int flags, void *ctx; COMMON_INTERCEPTOR_ENTER(ctx, glob64, pattern, flags, errfunc, pglob); COMMON_INTERCEPTOR_READ_STRING(ctx, pattern, 0); - __sanitizer_glob_t glob_copy = { - 0, 0, 0, - 0, wrapped_gl_closedir, wrapped_gl_readdir, - wrapped_gl_opendir, wrapped_gl_lstat, wrapped_gl_stat}; + __sanitizer_glob_t glob_copy; + internal_memcpy(&glob_copy, &kGlobCopy, sizeof(glob_copy)); if (flags & glob_altdirfunc) { Swap(pglob->gl_closedir, glob_copy.gl_closedir); Swap(pglob->gl_readdir, glob_copy.gl_readdir); @@ -1689,7 +1769,7 @@ INTERCEPTOR_WITH_SUFFIX(int, wait, int *status) { COMMON_INTERCEPTOR_ENTER(ctx, wait, status); // FIXME: under ASan the call below may write to freed memory and corrupt // its metadata. See - // https://code.google.com/p/address-sanitizer/issues/detail?id=321. + // https://github.com/google/sanitizers/issues/321. int res = REAL(wait)(status); if (res != -1 && status) COMMON_INTERCEPTOR_WRITE_RANGE(ctx, status, sizeof(*status)); @@ -1707,7 +1787,7 @@ INTERCEPTOR_WITH_SUFFIX(int, waitid, int idtype, int id, void *infop, COMMON_INTERCEPTOR_ENTER(ctx, waitid, idtype, id, infop, options); // FIXME: under ASan the call below may write to freed memory and corrupt // its metadata. See - // https://code.google.com/p/address-sanitizer/issues/detail?id=321. + // https://github.com/google/sanitizers/issues/321. int res = REAL(waitid)(idtype, id, infop, options); if (res != -1 && infop) COMMON_INTERCEPTOR_WRITE_RANGE(ctx, infop, siginfo_t_sz); @@ -1718,7 +1798,7 @@ INTERCEPTOR_WITH_SUFFIX(int, waitpid, int pid, int *status, int options) { COMMON_INTERCEPTOR_ENTER(ctx, waitpid, pid, status, options); // FIXME: under ASan the call below may write to freed memory and corrupt // its metadata. See - // https://code.google.com/p/address-sanitizer/issues/detail?id=321. + // https://github.com/google/sanitizers/issues/321. int res = REAL(waitpid)(pid, status, options); if (res != -1 && status) COMMON_INTERCEPTOR_WRITE_RANGE(ctx, status, sizeof(*status)); @@ -1729,7 +1809,7 @@ INTERCEPTOR(int, wait3, int *status, int options, void *rusage) { COMMON_INTERCEPTOR_ENTER(ctx, wait3, status, options, rusage); // FIXME: under ASan the call below may write to freed memory and corrupt // its metadata. See - // https://code.google.com/p/address-sanitizer/issues/detail?id=321. + // https://github.com/google/sanitizers/issues/321. int res = REAL(wait3)(status, options, rusage); if (res != -1) { if (status) COMMON_INTERCEPTOR_WRITE_RANGE(ctx, status, sizeof(*status)); @@ -1743,7 +1823,7 @@ INTERCEPTOR(int, __wait4, int pid, int *status, int options, void *rusage) { COMMON_INTERCEPTOR_ENTER(ctx, __wait4, pid, status, options, rusage); // FIXME: under ASan the call below may write to freed memory and corrupt // its metadata. See - // https://code.google.com/p/address-sanitizer/issues/detail?id=321. + // https://github.com/google/sanitizers/issues/321. int res = REAL(__wait4)(pid, status, options, rusage); if (res != -1) { if (status) COMMON_INTERCEPTOR_WRITE_RANGE(ctx, status, sizeof(*status)); @@ -1758,7 +1838,7 @@ INTERCEPTOR(int, wait4, int pid, int *status, int options, void *rusage) { COMMON_INTERCEPTOR_ENTER(ctx, wait4, pid, status, options, rusage); // FIXME: under ASan the call below may write to freed memory and corrupt // its metadata. See - // https://code.google.com/p/address-sanitizer/issues/detail?id=321. + // https://github.com/google/sanitizers/issues/321. int res = REAL(wait4)(pid, status, options, rusage); if (res != -1) { if (status) COMMON_INTERCEPTOR_WRITE_RANGE(ctx, status, sizeof(*status)); @@ -1787,7 +1867,7 @@ INTERCEPTOR(char *, inet_ntop, int af, const void *src, char *dst, u32 size) { // FIXME: figure out read size based on the address family. // FIXME: under ASan the call below may write to freed memory and corrupt // its metadata. See - // https://code.google.com/p/address-sanitizer/issues/detail?id=321. + // https://github.com/google/sanitizers/issues/321. char *res = REAL(inet_ntop)(af, src, dst, size); if (res) COMMON_INTERCEPTOR_WRITE_RANGE(ctx, res, REAL(strlen)(res) + 1); return res; @@ -1799,7 +1879,7 @@ INTERCEPTOR(int, inet_pton, int af, const char *src, void *dst) { // FIXME: figure out read size based on the address family. // FIXME: under ASan the call below may write to freed memory and corrupt // its metadata. See - // https://code.google.com/p/address-sanitizer/issues/detail?id=321. + // https://github.com/google/sanitizers/issues/321. int res = REAL(inet_pton)(af, src, dst); if (res == 1) { uptr sz = __sanitizer_in_addr_sz(af); @@ -1821,7 +1901,7 @@ INTERCEPTOR(int, inet_aton, const char *cp, void *dst) { if (cp) COMMON_INTERCEPTOR_READ_RANGE(ctx, cp, REAL(strlen)(cp) + 1); // FIXME: under ASan the call below may write to freed memory and corrupt // its metadata. See - // https://code.google.com/p/address-sanitizer/issues/detail?id=321. + // https://github.com/google/sanitizers/issues/321. int res = REAL(inet_aton)(cp, dst); if (res != 0) { uptr sz = __sanitizer_in_addr_sz(af_inet); @@ -1840,7 +1920,7 @@ INTERCEPTOR(int, pthread_getschedparam, uptr thread, int *policy, int *param) { COMMON_INTERCEPTOR_ENTER(ctx, pthread_getschedparam, thread, policy, param); // FIXME: under ASan the call below may write to freed memory and corrupt // its metadata. See - // https://code.google.com/p/address-sanitizer/issues/detail?id=321. + // https://github.com/google/sanitizers/issues/321. int res = REAL(pthread_getschedparam)(thread, policy, param); if (res == 0) { if (policy) COMMON_INTERCEPTOR_WRITE_RANGE(ctx, policy, sizeof(*policy)); @@ -1867,7 +1947,7 @@ INTERCEPTOR(int, getaddrinfo, char *node, char *service, COMMON_INTERCEPTOR_READ_RANGE(ctx, hints, sizeof(__sanitizer_addrinfo)); // FIXME: under ASan the call below may write to freed memory and corrupt // its metadata. See - // https://code.google.com/p/address-sanitizer/issues/detail?id=321. + // https://github.com/google/sanitizers/issues/321. int res = REAL(getaddrinfo)(node, service, hints, out); if (res == 0 && out) { COMMON_INTERCEPTOR_WRITE_RANGE(ctx, out, sizeof(*out)); @@ -1899,7 +1979,7 @@ INTERCEPTOR(int, getnameinfo, void *sockaddr, unsigned salen, char *host, // There is padding in in_addr that may make this too noisy // FIXME: under ASan the call below may write to freed memory and corrupt // its metadata. See - // https://code.google.com/p/address-sanitizer/issues/detail?id=321. + // https://github.com/google/sanitizers/issues/321. int res = REAL(getnameinfo)(sockaddr, salen, host, hostlen, serv, servlen, flags); if (res == 0) { @@ -1923,7 +2003,7 @@ INTERCEPTOR(int, getsockname, int sock_fd, void *addr, int *addrlen) { int addrlen_in = *addrlen; // FIXME: under ASan the call below may write to freed memory and corrupt // its metadata. See - // https://code.google.com/p/address-sanitizer/issues/detail?id=321. + // https://github.com/google/sanitizers/issues/321. int res = REAL(getsockname)(sock_fd, addr, addrlen); if (res == 0) { COMMON_INTERCEPTOR_WRITE_RANGE(ctx, addr, Min(addrlen_in, *addrlen)); @@ -2009,7 +2089,7 @@ INTERCEPTOR(int, gethostbyname_r, char *name, struct __sanitizer_hostent *ret, h_errnop); // FIXME: under ASan the call below may write to freed memory and corrupt // its metadata. See - // https://code.google.com/p/address-sanitizer/issues/detail?id=321. + // https://github.com/google/sanitizers/issues/321. int res = REAL(gethostbyname_r)(name, ret, buf, buflen, result, h_errnop); if (result) { COMMON_INTERCEPTOR_WRITE_RANGE(ctx, result, sizeof(*result)); @@ -2032,7 +2112,7 @@ INTERCEPTOR(int, gethostent_r, struct __sanitizer_hostent *ret, char *buf, h_errnop); // FIXME: under ASan the call below may write to freed memory and corrupt // its metadata. See - // https://code.google.com/p/address-sanitizer/issues/detail?id=321. + // https://github.com/google/sanitizers/issues/321. int res = REAL(gethostent_r)(ret, buf, buflen, result, h_errnop); if (result) { COMMON_INTERCEPTOR_WRITE_RANGE(ctx, result, sizeof(*result)); @@ -2058,7 +2138,7 @@ INTERCEPTOR(int, gethostbyaddr_r, void *addr, int len, int type, COMMON_INTERCEPTOR_READ_RANGE(ctx, addr, len); // FIXME: under ASan the call below may write to freed memory and corrupt // its metadata. See - // https://code.google.com/p/address-sanitizer/issues/detail?id=321. + // https://github.com/google/sanitizers/issues/321. int res = REAL(gethostbyaddr_r)(addr, len, type, ret, buf, buflen, result, h_errnop); if (result) { @@ -2084,7 +2164,7 @@ INTERCEPTOR(int, gethostbyname2_r, char *name, int af, result, h_errnop); // FIXME: under ASan the call below may write to freed memory and corrupt // its metadata. See - // https://code.google.com/p/address-sanitizer/issues/detail?id=321. + // https://github.com/google/sanitizers/issues/321. int res = REAL(gethostbyname2_r)(name, af, ret, buf, buflen, result, h_errnop); if (result) { @@ -2110,7 +2190,7 @@ INTERCEPTOR(int, getsockopt, int sockfd, int level, int optname, void *optval, if (optlen) COMMON_INTERCEPTOR_READ_RANGE(ctx, optlen, sizeof(*optlen)); // FIXME: under ASan the call below may write to freed memory and corrupt // its metadata. See - // https://code.google.com/p/address-sanitizer/issues/detail?id=321. + // https://github.com/google/sanitizers/issues/321. int res = REAL(getsockopt)(sockfd, level, optname, optval, optlen); if (res == 0) if (optval && optlen) COMMON_INTERCEPTOR_WRITE_RANGE(ctx, optval, *optlen); @@ -2154,7 +2234,7 @@ INTERCEPTOR(int, accept4, int fd, void *addr, unsigned *addrlen, int f) { } // FIXME: under ASan the call below may write to freed memory and corrupt // its metadata. See - // https://code.google.com/p/address-sanitizer/issues/detail?id=321. + // https://github.com/google/sanitizers/issues/321. int fd2 = REAL(accept4)(fd, addr, addrlen, f); if (fd2 >= 0) { if (fd >= 0) COMMON_INTERCEPTOR_FD_SOCKET_ACCEPT(ctx, fd, fd2); @@ -2174,7 +2254,7 @@ INTERCEPTOR(double, modf, double x, double *iptr) { COMMON_INTERCEPTOR_ENTER(ctx, modf, x, iptr); // FIXME: under ASan the call below may write to freed memory and corrupt // its metadata. See - // https://code.google.com/p/address-sanitizer/issues/detail?id=321. + // https://github.com/google/sanitizers/issues/321. double res = REAL(modf)(x, iptr); if (iptr) { COMMON_INTERCEPTOR_WRITE_RANGE(ctx, iptr, sizeof(*iptr)); @@ -2186,7 +2266,7 @@ INTERCEPTOR(float, modff, float x, float *iptr) { COMMON_INTERCEPTOR_ENTER(ctx, modff, x, iptr); // FIXME: under ASan the call below may write to freed memory and corrupt // its metadata. See - // https://code.google.com/p/address-sanitizer/issues/detail?id=321. + // https://github.com/google/sanitizers/issues/321. float res = REAL(modff)(x, iptr); if (iptr) { COMMON_INTERCEPTOR_WRITE_RANGE(ctx, iptr, sizeof(*iptr)); @@ -2198,7 +2278,7 @@ INTERCEPTOR(long double, modfl, long double x, long double *iptr) { COMMON_INTERCEPTOR_ENTER(ctx, modfl, x, iptr); // FIXME: under ASan the call below may write to freed memory and corrupt // its metadata. See - // https://code.google.com/p/address-sanitizer/issues/detail?id=321. + // https://github.com/google/sanitizers/issues/321. long double res = REAL(modfl)(x, iptr); if (iptr) { COMMON_INTERCEPTOR_WRITE_RANGE(ctx, iptr, sizeof(*iptr)); @@ -2233,7 +2313,7 @@ INTERCEPTOR(SSIZE_T, recvmsg, int fd, struct __sanitizer_msghdr *msg, COMMON_INTERCEPTOR_ENTER(ctx, recvmsg, fd, msg, flags); // FIXME: under ASan the call below may write to freed memory and corrupt // its metadata. See - // https://code.google.com/p/address-sanitizer/issues/detail?id=321. + // https://github.com/google/sanitizers/issues/321. SSIZE_T res = REAL(recvmsg)(fd, msg, flags); if (res >= 0) { if (fd >= 0) COMMON_INTERCEPTOR_FD_ACQUIRE(ctx, fd); @@ -2257,7 +2337,7 @@ INTERCEPTOR(int, getpeername, int sockfd, void *addr, unsigned *addrlen) { if (addrlen) addr_sz = *addrlen; // FIXME: under ASan the call below may write to freed memory and corrupt // its metadata. See - // https://code.google.com/p/address-sanitizer/issues/detail?id=321. + // https://github.com/google/sanitizers/issues/321. int res = REAL(getpeername)(sockfd, addr, addrlen); if (!res && addr && addrlen) COMMON_INTERCEPTOR_WRITE_RANGE(ctx, addr, Min(addr_sz, *addrlen)); @@ -2273,7 +2353,7 @@ INTERCEPTOR(int, sysinfo, void *info) { void *ctx; // FIXME: under ASan the call below may write to freed memory and corrupt // its metadata. See - // https://code.google.com/p/address-sanitizer/issues/detail?id=321. + // https://github.com/google/sanitizers/issues/321. COMMON_INTERCEPTOR_ENTER(ctx, sysinfo, info); int res = REAL(sysinfo)(info); if (!res && info) @@ -2291,7 +2371,7 @@ INTERCEPTOR(__sanitizer_dirent *, opendir, const char *path) { COMMON_INTERCEPTOR_ENTER(ctx, opendir, path); COMMON_INTERCEPTOR_READ_RANGE(ctx, path, REAL(strlen)(path) + 1); __sanitizer_dirent *res = REAL(opendir)(path); - if (res != 0) + if (res) COMMON_INTERCEPTOR_DIR_ACQUIRE(ctx, path); return res; } @@ -2301,7 +2381,7 @@ INTERCEPTOR(__sanitizer_dirent *, readdir, void *dirp) { COMMON_INTERCEPTOR_ENTER(ctx, readdir, dirp); // FIXME: under ASan the call below may write to freed memory and corrupt // its metadata. See - // https://code.google.com/p/address-sanitizer/issues/detail?id=321. + // https://github.com/google/sanitizers/issues/321. __sanitizer_dirent *res = REAL(readdir)(dirp); if (res) COMMON_INTERCEPTOR_WRITE_RANGE(ctx, res, res->d_reclen); return res; @@ -2313,7 +2393,7 @@ INTERCEPTOR(int, readdir_r, void *dirp, __sanitizer_dirent *entry, COMMON_INTERCEPTOR_ENTER(ctx, readdir_r, dirp, entry, result); // FIXME: under ASan the call below may write to freed memory and corrupt // its metadata. See - // https://code.google.com/p/address-sanitizer/issues/detail?id=321. + // https://github.com/google/sanitizers/issues/321. int res = REAL(readdir_r)(dirp, entry, result); if (!res) { COMMON_INTERCEPTOR_WRITE_RANGE(ctx, result, sizeof(*result)); @@ -2337,7 +2417,7 @@ INTERCEPTOR(__sanitizer_dirent64 *, readdir64, void *dirp) { COMMON_INTERCEPTOR_ENTER(ctx, readdir64, dirp); // FIXME: under ASan the call below may write to freed memory and corrupt // its metadata. See - // https://code.google.com/p/address-sanitizer/issues/detail?id=321. + // https://github.com/google/sanitizers/issues/321. __sanitizer_dirent64 *res = REAL(readdir64)(dirp); if (res) COMMON_INTERCEPTOR_WRITE_RANGE(ctx, res, res->d_reclen); return res; @@ -2349,7 +2429,7 @@ INTERCEPTOR(int, readdir64_r, void *dirp, __sanitizer_dirent64 *entry, COMMON_INTERCEPTOR_ENTER(ctx, readdir64_r, dirp, entry, result); // FIXME: under ASan the call below may write to freed memory and corrupt // its metadata. See - // https://code.google.com/p/address-sanitizer/issues/detail?id=321. + // https://github.com/google/sanitizers/issues/321. int res = REAL(readdir64_r)(dirp, entry, result); if (!res) { COMMON_INTERCEPTOR_WRITE_RANGE(ctx, result, sizeof(*result)); @@ -2369,6 +2449,7 @@ INTERCEPTOR(int, readdir64_r, void *dirp, __sanitizer_dirent64 *entry, INTERCEPTOR(uptr, ptrace, int request, int pid, void *addr, void *data) { void *ctx; COMMON_INTERCEPTOR_ENTER(ctx, ptrace, request, pid, addr, data); + __sanitizer_iovec local_iovec; if (data) { if (request == ptrace_setregs) @@ -2377,17 +2458,25 @@ INTERCEPTOR(uptr, ptrace, int request, int pid, void *addr, void *data) { COMMON_INTERCEPTOR_READ_RANGE(ctx, data, struct_user_fpregs_struct_sz); else if (request == ptrace_setfpxregs) COMMON_INTERCEPTOR_READ_RANGE(ctx, data, struct_user_fpxregs_struct_sz); + else if (request == ptrace_setvfpregs) + COMMON_INTERCEPTOR_READ_RANGE(ctx, data, struct_user_vfpregs_struct_sz); else if (request == ptrace_setsiginfo) COMMON_INTERCEPTOR_READ_RANGE(ctx, data, siginfo_t_sz); - else if (request == ptrace_setregset) { - __sanitizer_iovec *iov = (__sanitizer_iovec *)data; - COMMON_INTERCEPTOR_READ_RANGE(ctx, iov->iov_base, iov->iov_len); + // Some kernel might zero the iovec::iov_base in case of invalid + // write access. In this case copy the invalid address for further + // inspection. + else if (request == ptrace_setregset || request == ptrace_getregset) { + __sanitizer_iovec *iovec = (__sanitizer_iovec*)data; + COMMON_INTERCEPTOR_READ_RANGE(ctx, iovec, sizeof(*iovec)); + local_iovec = *iovec; + if (request == ptrace_setregset) + COMMON_INTERCEPTOR_READ_RANGE(ctx, iovec->iov_base, iovec->iov_len); } } // FIXME: under ASan the call below may write to freed memory and corrupt // its metadata. See - // https://code.google.com/p/address-sanitizer/issues/detail?id=321. + // https://github.com/google/sanitizers/issues/321. uptr res = REAL(ptrace)(request, pid, addr, data); if (!res && data) { @@ -2399,13 +2488,17 @@ INTERCEPTOR(uptr, ptrace, int request, int pid, void *addr, void *data) { COMMON_INTERCEPTOR_WRITE_RANGE(ctx, data, struct_user_fpregs_struct_sz); else if (request == ptrace_getfpxregs) COMMON_INTERCEPTOR_WRITE_RANGE(ctx, data, struct_user_fpxregs_struct_sz); + else if (request == ptrace_getvfpregs) + COMMON_INTERCEPTOR_WRITE_RANGE(ctx, data, struct_user_vfpregs_struct_sz); else if (request == ptrace_getsiginfo) COMMON_INTERCEPTOR_WRITE_RANGE(ctx, data, siginfo_t_sz); else if (request == ptrace_geteventmsg) COMMON_INTERCEPTOR_WRITE_RANGE(ctx, data, sizeof(unsigned long)); else if (request == ptrace_getregset) { - __sanitizer_iovec *iov = (__sanitizer_iovec *)data; - COMMON_INTERCEPTOR_WRITE_RANGE(ctx, iov->iov_base, iov->iov_len); + __sanitizer_iovec *iovec = (__sanitizer_iovec*)data; + COMMON_INTERCEPTOR_WRITE_RANGE(ctx, iovec, sizeof(*iovec)); + COMMON_INTERCEPTOR_WRITE_RANGE(ctx, local_iovec.iov_base, + local_iovec.iov_len); } } return res; @@ -2438,7 +2531,7 @@ INTERCEPTOR(char *, getcwd, char *buf, SIZE_T size) { COMMON_INTERCEPTOR_ENTER(ctx, getcwd, buf, size); // FIXME: under ASan the call below may write to freed memory and corrupt // its metadata. See - // https://code.google.com/p/address-sanitizer/issues/detail?id=321. + // https://github.com/google/sanitizers/issues/321. char *res = REAL(getcwd)(buf, size); if (res) COMMON_INTERCEPTOR_WRITE_RANGE(ctx, res, REAL(strlen)(res) + 1); return res; @@ -2454,7 +2547,7 @@ INTERCEPTOR(char *, get_current_dir_name, int fake) { COMMON_INTERCEPTOR_ENTER(ctx, get_current_dir_name, fake); // FIXME: under ASan the call below may write to freed memory and corrupt // its metadata. See - // https://code.google.com/p/address-sanitizer/issues/detail?id=321. + // https://github.com/google/sanitizers/issues/321. char *res = REAL(get_current_dir_name)(fake); if (res) COMMON_INTERCEPTOR_WRITE_RANGE(ctx, res, REAL(strlen)(res) + 1); return res; @@ -2481,7 +2574,7 @@ UNUSED static inline void FixRealStrtolEndptr(const char *nptr, char **endptr) { UNUSED static inline void StrtolFixAndCheck(void *ctx, const char *nptr, char **endptr, char *real_endptr, int base) { - if (endptr != 0) { + if (endptr) { *endptr = real_endptr; COMMON_INTERCEPTOR_WRITE_RANGE(ctx, endptr, sizeof(*endptr)); } @@ -2503,7 +2596,7 @@ INTERCEPTOR(INTMAX_T, strtoimax, const char *nptr, char **endptr, int base) { COMMON_INTERCEPTOR_ENTER(ctx, strtoimax, nptr, endptr, base); // FIXME: under ASan the call below may write to freed memory and corrupt // its metadata. See - // https://code.google.com/p/address-sanitizer/issues/detail?id=321. + // https://github.com/google/sanitizers/issues/321. char *real_endptr; INTMAX_T res = REAL(strtoimax)(nptr, &real_endptr, base); StrtolFixAndCheck(ctx, nptr, endptr, real_endptr, base); @@ -2515,7 +2608,7 @@ INTERCEPTOR(INTMAX_T, strtoumax, const char *nptr, char **endptr, int base) { COMMON_INTERCEPTOR_ENTER(ctx, strtoumax, nptr, endptr, base); // FIXME: under ASan the call below may write to freed memory and corrupt // its metadata. See - // https://code.google.com/p/address-sanitizer/issues/detail?id=321. + // https://github.com/google/sanitizers/issues/321. char *real_endptr; INTMAX_T res = REAL(strtoumax)(nptr, &real_endptr, base); StrtolFixAndCheck(ctx, nptr, endptr, real_endptr, base); @@ -2535,7 +2628,7 @@ INTERCEPTOR(SIZE_T, mbstowcs, wchar_t *dest, const char *src, SIZE_T len) { COMMON_INTERCEPTOR_ENTER(ctx, mbstowcs, dest, src, len); // FIXME: under ASan the call below may write to freed memory and corrupt // its metadata. See - // https://code.google.com/p/address-sanitizer/issues/detail?id=321. + // https://github.com/google/sanitizers/issues/321. SIZE_T res = REAL(mbstowcs)(dest, src, len); if (res != (SIZE_T) - 1 && dest) { SIZE_T write_cnt = res + (res < len); @@ -2552,7 +2645,7 @@ INTERCEPTOR(SIZE_T, mbsrtowcs, wchar_t *dest, const char **src, SIZE_T len, if (ps) COMMON_INTERCEPTOR_READ_RANGE(ctx, ps, mbstate_t_sz); // FIXME: under ASan the call below may write to freed memory and corrupt // its metadata. See - // https://code.google.com/p/address-sanitizer/issues/detail?id=321. + // https://github.com/google/sanitizers/issues/321. SIZE_T res = REAL(mbsrtowcs)(dest, src, len, ps); if (res != (SIZE_T)(-1) && dest && src) { // This function, and several others, may or may not write the terminating @@ -2582,7 +2675,7 @@ INTERCEPTOR(SIZE_T, mbsnrtowcs, wchar_t *dest, const char **src, SIZE_T nms, if (ps) COMMON_INTERCEPTOR_READ_RANGE(ctx, ps, mbstate_t_sz); // FIXME: under ASan the call below may write to freed memory and corrupt // its metadata. See - // https://code.google.com/p/address-sanitizer/issues/detail?id=321. + // https://github.com/google/sanitizers/issues/321. SIZE_T res = REAL(mbsnrtowcs)(dest, src, nms, len, ps); if (res != (SIZE_T)(-1) && dest && src) { SIZE_T write_cnt = res + !*src; @@ -2602,7 +2695,7 @@ INTERCEPTOR(SIZE_T, wcstombs, char *dest, const wchar_t *src, SIZE_T len) { COMMON_INTERCEPTOR_ENTER(ctx, wcstombs, dest, src, len); // FIXME: under ASan the call below may write to freed memory and corrupt // its metadata. See - // https://code.google.com/p/address-sanitizer/issues/detail?id=321. + // https://github.com/google/sanitizers/issues/321. SIZE_T res = REAL(wcstombs)(dest, src, len); if (res != (SIZE_T) - 1 && dest) { SIZE_T write_cnt = res + (res < len); @@ -2619,7 +2712,7 @@ INTERCEPTOR(SIZE_T, wcsrtombs, char *dest, const wchar_t **src, SIZE_T len, if (ps) COMMON_INTERCEPTOR_READ_RANGE(ctx, ps, mbstate_t_sz); // FIXME: under ASan the call below may write to freed memory and corrupt // its metadata. See - // https://code.google.com/p/address-sanitizer/issues/detail?id=321. + // https://github.com/google/sanitizers/issues/321. SIZE_T res = REAL(wcsrtombs)(dest, src, len, ps); if (res != (SIZE_T) - 1 && dest && src) { SIZE_T write_cnt = res + !*src; @@ -2647,9 +2740,9 @@ INTERCEPTOR(SIZE_T, wcsnrtombs, char *dest, const wchar_t **src, SIZE_T nms, if (ps) COMMON_INTERCEPTOR_READ_RANGE(ctx, ps, mbstate_t_sz); // FIXME: under ASan the call below may write to freed memory and corrupt // its metadata. See - // https://code.google.com/p/address-sanitizer/issues/detail?id=321. + // https://github.com/google/sanitizers/issues/321. SIZE_T res = REAL(wcsnrtombs)(dest, src, nms, len, ps); - if (res != (SIZE_T) - 1 && dest && src) { + if (res != ((SIZE_T)-1) && dest && src) { SIZE_T write_cnt = res + !*src; COMMON_INTERCEPTOR_WRITE_RANGE(ctx, dest, write_cnt); } @@ -2661,13 +2754,35 @@ INTERCEPTOR(SIZE_T, wcsnrtombs, char *dest, const wchar_t **src, SIZE_T nms, #define INIT_WCSNRTOMBS #endif + +#if SANITIZER_INTERCEPT_WCRTOMB +INTERCEPTOR(SIZE_T, wcrtomb, char *dest, wchar_t src, void *ps) { + void *ctx; + COMMON_INTERCEPTOR_ENTER(ctx, wcrtomb, dest, src, ps); + if (ps) COMMON_INTERCEPTOR_READ_RANGE(ctx, ps, mbstate_t_sz); + // FIXME: under ASan the call below may write to freed memory and corrupt + // its metadata. See + // https://github.com/google/sanitizers/issues/321. + SIZE_T res = REAL(wcrtomb)(dest, src, ps); + if (res != ((SIZE_T)-1) && dest) { + SIZE_T write_cnt = res; + COMMON_INTERCEPTOR_WRITE_RANGE(ctx, dest, write_cnt); + } + return res; +} + +#define INIT_WCRTOMB COMMON_INTERCEPT_FUNCTION(wcrtomb); +#else +#define INIT_WCRTOMB +#endif + #if SANITIZER_INTERCEPT_TCGETATTR INTERCEPTOR(int, tcgetattr, int fd, void *termios_p) { void *ctx; COMMON_INTERCEPTOR_ENTER(ctx, tcgetattr, fd, termios_p); // FIXME: under ASan the call below may write to freed memory and corrupt // its metadata. See - // https://code.google.com/p/address-sanitizer/issues/detail?id=321. + // https://github.com/google/sanitizers/issues/321. int res = REAL(tcgetattr)(fd, termios_p); if (!res && termios_p) COMMON_INTERCEPTOR_WRITE_RANGE(ctx, termios_p, struct_termios_sz); @@ -2689,7 +2804,7 @@ INTERCEPTOR(char *, realpath, const char *path, char *resolved_path) { // version of a versioned symbol. For realpath(), this gives us something // (called __old_realpath) that does not handle NULL in the second argument. // Handle it as part of the interceptor. - char *allocated_path = 0; + char *allocated_path = nullptr; if (!resolved_path) allocated_path = resolved_path = (char *)WRAP(malloc)(path_max + 1); @@ -2724,7 +2839,7 @@ INTERCEPTOR(SIZE_T, confstr, int name, char *buf, SIZE_T len) { COMMON_INTERCEPTOR_ENTER(ctx, confstr, name, buf, len); // FIXME: under ASan the call below may write to freed memory and corrupt // its metadata. See - // https://code.google.com/p/address-sanitizer/issues/detail?id=321. + // https://github.com/google/sanitizers/issues/321. SIZE_T res = REAL(confstr)(name, buf, len); if (buf && res) COMMON_INTERCEPTOR_WRITE_RANGE(ctx, buf, res < len ? res : len); @@ -2741,7 +2856,7 @@ INTERCEPTOR(int, sched_getaffinity, int pid, SIZE_T cpusetsize, void *mask) { COMMON_INTERCEPTOR_ENTER(ctx, sched_getaffinity, pid, cpusetsize, mask); // FIXME: under ASan the call below may write to freed memory and corrupt // its metadata. See - // https://code.google.com/p/address-sanitizer/issues/detail?id=321. + // https://github.com/google/sanitizers/issues/321. int res = REAL(sched_getaffinity)(pid, cpusetsize, mask); if (mask && !res) COMMON_INTERCEPTOR_WRITE_RANGE(ctx, mask, cpusetsize); return res; @@ -2783,7 +2898,7 @@ INTERCEPTOR(char *, strerror_r, int errnum, char *buf, SIZE_T buflen) { COMMON_INTERCEPTOR_ENTER(ctx, strerror_r, errnum, buf, buflen); // FIXME: under ASan the call below may write to freed memory and corrupt // its metadata. See - // https://code.google.com/p/address-sanitizer/issues/detail?id=321. + // https://github.com/google/sanitizers/issues/321. char *res = REAL(strerror_r)(errnum, buf, buflen); // There are 2 versions of strerror_r: // * POSIX version returns 0 on success, negative error code on failure, @@ -2814,7 +2929,7 @@ INTERCEPTOR(int, __xpg_strerror_r, int errnum, char *buf, SIZE_T buflen) { COMMON_INTERCEPTOR_ENTER(ctx, __xpg_strerror_r, errnum, buf, buflen); // FIXME: under ASan the call below may write to freed memory and corrupt // its metadata. See - // https://code.google.com/p/address-sanitizer/issues/detail?id=321. + // https://github.com/google/sanitizers/issues/321. int res = REAL(__xpg_strerror_r)(errnum, buf, buflen); // This version always returns a null-terminated string. if (buf && buflen) @@ -2859,11 +2974,12 @@ INTERCEPTOR(int, scandir, char *dirp, __sanitizer_dirent ***namelist, scandir_compar = compar; // FIXME: under ASan the call below may write to freed memory and corrupt // its metadata. See - // https://code.google.com/p/address-sanitizer/issues/detail?id=321. - int res = REAL(scandir)(dirp, namelist, filter ? wrapped_scandir_filter : 0, - compar ? wrapped_scandir_compar : 0); - scandir_filter = 0; - scandir_compar = 0; + // https://github.com/google/sanitizers/issues/321. + int res = REAL(scandir)(dirp, namelist, + filter ? wrapped_scandir_filter : nullptr, + compar ? wrapped_scandir_compar : nullptr); + scandir_filter = nullptr; + scandir_compar = nullptr; if (namelist && res > 0) { COMMON_INTERCEPTOR_WRITE_RANGE(ctx, namelist, sizeof(*namelist)); COMMON_INTERCEPTOR_WRITE_RANGE(ctx, *namelist, sizeof(**namelist) * res); @@ -2911,12 +3027,13 @@ INTERCEPTOR(int, scandir64, char *dirp, __sanitizer_dirent64 ***namelist, scandir64_compar = compar; // FIXME: under ASan the call below may write to freed memory and corrupt // its metadata. See - // https://code.google.com/p/address-sanitizer/issues/detail?id=321. + // https://github.com/google/sanitizers/issues/321. int res = - REAL(scandir64)(dirp, namelist, filter ? wrapped_scandir64_filter : 0, - compar ? wrapped_scandir64_compar : 0); - scandir64_filter = 0; - scandir64_compar = 0; + REAL(scandir64)(dirp, namelist, + filter ? wrapped_scandir64_filter : nullptr, + compar ? wrapped_scandir64_compar : nullptr); + scandir64_filter = nullptr; + scandir64_compar = nullptr; if (namelist && res > 0) { COMMON_INTERCEPTOR_WRITE_RANGE(ctx, namelist, sizeof(*namelist)); COMMON_INTERCEPTOR_WRITE_RANGE(ctx, *namelist, sizeof(**namelist) * res); @@ -2937,7 +3054,7 @@ INTERCEPTOR(int, getgroups, int size, u32 *lst) { COMMON_INTERCEPTOR_ENTER(ctx, getgroups, size, lst); // FIXME: under ASan the call below may write to freed memory and corrupt // its metadata. See - // https://code.google.com/p/address-sanitizer/issues/detail?id=321. + // https://github.com/google/sanitizers/issues/321. int res = REAL(getgroups)(size, lst); if (res && lst) COMMON_INTERCEPTOR_WRITE_RANGE(ctx, lst, res * sizeof(*lst)); return res; @@ -3003,7 +3120,7 @@ INTERCEPTOR(int, wordexp, char *s, __sanitizer_wordexp_t *p, int flags) { if (s) COMMON_INTERCEPTOR_READ_RANGE(ctx, s, REAL(strlen)(s) + 1); // FIXME: under ASan the call below may write to freed memory and corrupt // its metadata. See - // https://code.google.com/p/address-sanitizer/issues/detail?id=321. + // https://github.com/google/sanitizers/issues/321. int res = REAL(wordexp)(s, p, flags); if (!res && p) { COMMON_INTERCEPTOR_WRITE_RANGE(ctx, p, sizeof(*p)); @@ -3029,7 +3146,7 @@ INTERCEPTOR(int, sigwait, __sanitizer_sigset_t *set, int *sig) { // FIXME: read sigset_t when all of sigemptyset, etc are intercepted // FIXME: under ASan the call below may write to freed memory and corrupt // its metadata. See - // https://code.google.com/p/address-sanitizer/issues/detail?id=321. + // https://github.com/google/sanitizers/issues/321. int res = REAL(sigwait)(set, sig); if (!res && sig) COMMON_INTERCEPTOR_WRITE_RANGE(ctx, sig, sizeof(*sig)); return res; @@ -3046,7 +3163,7 @@ INTERCEPTOR(int, sigwaitinfo, __sanitizer_sigset_t *set, void *info) { // FIXME: read sigset_t when all of sigemptyset, etc are intercepted // FIXME: under ASan the call below may write to freed memory and corrupt // its metadata. See - // https://code.google.com/p/address-sanitizer/issues/detail?id=321. + // https://github.com/google/sanitizers/issues/321. int res = REAL(sigwaitinfo)(set, info); if (res > 0 && info) COMMON_INTERCEPTOR_WRITE_RANGE(ctx, info, siginfo_t_sz); return res; @@ -3065,7 +3182,7 @@ INTERCEPTOR(int, sigtimedwait, __sanitizer_sigset_t *set, void *info, // FIXME: read sigset_t when all of sigemptyset, etc are intercepted // FIXME: under ASan the call below may write to freed memory and corrupt // its metadata. See - // https://code.google.com/p/address-sanitizer/issues/detail?id=321. + // https://github.com/google/sanitizers/issues/321. int res = REAL(sigtimedwait)(set, info, timeout); if (res > 0 && info) COMMON_INTERCEPTOR_WRITE_RANGE(ctx, info, siginfo_t_sz); return res; @@ -3081,7 +3198,7 @@ INTERCEPTOR(int, sigemptyset, __sanitizer_sigset_t *set) { COMMON_INTERCEPTOR_ENTER(ctx, sigemptyset, set); // FIXME: under ASan the call below may write to freed memory and corrupt // its metadata. See - // https://code.google.com/p/address-sanitizer/issues/detail?id=321. + // https://github.com/google/sanitizers/issues/321. int res = REAL(sigemptyset)(set); if (!res && set) COMMON_INTERCEPTOR_WRITE_RANGE(ctx, set, sizeof(*set)); return res; @@ -3092,7 +3209,7 @@ INTERCEPTOR(int, sigfillset, __sanitizer_sigset_t *set) { COMMON_INTERCEPTOR_ENTER(ctx, sigfillset, set); // FIXME: under ASan the call below may write to freed memory and corrupt // its metadata. See - // https://code.google.com/p/address-sanitizer/issues/detail?id=321. + // https://github.com/google/sanitizers/issues/321. int res = REAL(sigfillset)(set); if (!res && set) COMMON_INTERCEPTOR_WRITE_RANGE(ctx, set, sizeof(*set)); return res; @@ -3110,7 +3227,7 @@ INTERCEPTOR(int, sigpending, __sanitizer_sigset_t *set) { COMMON_INTERCEPTOR_ENTER(ctx, sigpending, set); // FIXME: under ASan the call below may write to freed memory and corrupt // its metadata. See - // https://code.google.com/p/address-sanitizer/issues/detail?id=321. + // https://github.com/google/sanitizers/issues/321. int res = REAL(sigpending)(set); if (!res && set) COMMON_INTERCEPTOR_WRITE_RANGE(ctx, set, sizeof(*set)); return res; @@ -3128,7 +3245,7 @@ INTERCEPTOR(int, sigprocmask, int how, __sanitizer_sigset_t *set, // FIXME: read sigset_t when all of sigemptyset, etc are intercepted // FIXME: under ASan the call below may write to freed memory and corrupt // its metadata. See - // https://code.google.com/p/address-sanitizer/issues/detail?id=321. + // https://github.com/google/sanitizers/issues/321. int res = REAL(sigprocmask)(how, set, oldset); if (!res && oldset) COMMON_INTERCEPTOR_WRITE_RANGE(ctx, oldset, sizeof(*oldset)); @@ -3145,7 +3262,7 @@ INTERCEPTOR(int, backtrace, void **buffer, int size) { COMMON_INTERCEPTOR_ENTER(ctx, backtrace, buffer, size); // FIXME: under ASan the call below may write to freed memory and corrupt // its metadata. See - // https://code.google.com/p/address-sanitizer/issues/detail?id=321. + // https://github.com/google/sanitizers/issues/321. int res = REAL(backtrace)(buffer, size); if (res && buffer) COMMON_INTERCEPTOR_WRITE_RANGE(ctx, buffer, res * sizeof(*buffer)); @@ -3159,7 +3276,7 @@ INTERCEPTOR(char **, backtrace_symbols, void **buffer, int size) { COMMON_INTERCEPTOR_READ_RANGE(ctx, buffer, size * sizeof(*buffer)); // FIXME: under ASan the call below may write to freed memory and corrupt // its metadata. See - // https://code.google.com/p/address-sanitizer/issues/detail?id=321. + // https://github.com/google/sanitizers/issues/321. char **res = REAL(backtrace_symbols)(buffer, size); if (res && size) { COMMON_INTERCEPTOR_WRITE_RANGE(ctx, res, size * sizeof(*res)); @@ -3267,7 +3384,7 @@ INTERCEPTOR(int, statfs, char *path, void *buf) { if (path) COMMON_INTERCEPTOR_READ_RANGE(ctx, path, REAL(strlen)(path) + 1); // FIXME: under ASan the call below may write to freed memory and corrupt // its metadata. See - // https://code.google.com/p/address-sanitizer/issues/detail?id=321. + // https://github.com/google/sanitizers/issues/321. int res = REAL(statfs)(path, buf); if (!res) COMMON_INTERCEPTOR_WRITE_RANGE(ctx, buf, struct_statfs_sz); return res; @@ -3277,7 +3394,7 @@ INTERCEPTOR(int, fstatfs, int fd, void *buf) { COMMON_INTERCEPTOR_ENTER(ctx, fstatfs, fd, buf); // FIXME: under ASan the call below may write to freed memory and corrupt // its metadata. See - // https://code.google.com/p/address-sanitizer/issues/detail?id=321. + // https://github.com/google/sanitizers/issues/321. int res = REAL(fstatfs)(fd, buf); if (!res) COMMON_INTERCEPTOR_WRITE_RANGE(ctx, buf, struct_statfs_sz); return res; @@ -3296,7 +3413,7 @@ INTERCEPTOR(int, statfs64, char *path, void *buf) { if (path) COMMON_INTERCEPTOR_READ_RANGE(ctx, path, REAL(strlen)(path) + 1); // FIXME: under ASan the call below may write to freed memory and corrupt // its metadata. See - // https://code.google.com/p/address-sanitizer/issues/detail?id=321. + // https://github.com/google/sanitizers/issues/321. int res = REAL(statfs64)(path, buf); if (!res) COMMON_INTERCEPTOR_WRITE_RANGE(ctx, buf, struct_statfs64_sz); return res; @@ -3306,7 +3423,7 @@ INTERCEPTOR(int, fstatfs64, int fd, void *buf) { COMMON_INTERCEPTOR_ENTER(ctx, fstatfs64, fd, buf); // FIXME: under ASan the call below may write to freed memory and corrupt // its metadata. See - // https://code.google.com/p/address-sanitizer/issues/detail?id=321. + // https://github.com/google/sanitizers/issues/321. int res = REAL(fstatfs64)(fd, buf); if (!res) COMMON_INTERCEPTOR_WRITE_RANGE(ctx, buf, struct_statfs64_sz); return res; @@ -3325,7 +3442,7 @@ INTERCEPTOR(int, statvfs, char *path, void *buf) { if (path) COMMON_INTERCEPTOR_READ_RANGE(ctx, path, REAL(strlen)(path) + 1); // FIXME: under ASan the call below may write to freed memory and corrupt // its metadata. See - // https://code.google.com/p/address-sanitizer/issues/detail?id=321. + // https://github.com/google/sanitizers/issues/321. int res = REAL(statvfs)(path, buf); if (!res) COMMON_INTERCEPTOR_WRITE_RANGE(ctx, buf, struct_statvfs_sz); return res; @@ -3335,7 +3452,7 @@ INTERCEPTOR(int, fstatvfs, int fd, void *buf) { COMMON_INTERCEPTOR_ENTER(ctx, fstatvfs, fd, buf); // FIXME: under ASan the call below may write to freed memory and corrupt // its metadata. See - // https://code.google.com/p/address-sanitizer/issues/detail?id=321. + // https://github.com/google/sanitizers/issues/321. int res = REAL(fstatvfs)(fd, buf); if (!res) COMMON_INTERCEPTOR_WRITE_RANGE(ctx, buf, struct_statvfs_sz); return res; @@ -3354,7 +3471,7 @@ INTERCEPTOR(int, statvfs64, char *path, void *buf) { if (path) COMMON_INTERCEPTOR_READ_RANGE(ctx, path, REAL(strlen)(path) + 1); // FIXME: under ASan the call below may write to freed memory and corrupt // its metadata. See - // https://code.google.com/p/address-sanitizer/issues/detail?id=321. + // https://github.com/google/sanitizers/issues/321. int res = REAL(statvfs64)(path, buf); if (!res) COMMON_INTERCEPTOR_WRITE_RANGE(ctx, buf, struct_statvfs64_sz); return res; @@ -3364,7 +3481,7 @@ INTERCEPTOR(int, fstatvfs64, int fd, void *buf) { COMMON_INTERCEPTOR_ENTER(ctx, fstatvfs64, fd, buf); // FIXME: under ASan the call below may write to freed memory and corrupt // its metadata. See - // https://code.google.com/p/address-sanitizer/issues/detail?id=321. + // https://github.com/google/sanitizers/issues/321. int res = REAL(fstatvfs64)(fd, buf); if (!res) COMMON_INTERCEPTOR_WRITE_RANGE(ctx, buf, struct_statvfs64_sz); return res; @@ -3420,7 +3537,7 @@ INTERCEPTOR(int, ether_ntohost, char *hostname, __sanitizer_ether_addr *addr) { if (addr) COMMON_INTERCEPTOR_READ_RANGE(ctx, addr, sizeof(*addr)); // FIXME: under ASan the call below may write to freed memory and corrupt // its metadata. See - // https://code.google.com/p/address-sanitizer/issues/detail?id=321. + // https://github.com/google/sanitizers/issues/321. int res = REAL(ether_ntohost)(hostname, addr); if (!res && hostname) COMMON_INTERCEPTOR_WRITE_RANGE(ctx, hostname, REAL(strlen)(hostname) + 1); @@ -3433,7 +3550,7 @@ INTERCEPTOR(int, ether_hostton, char *hostname, __sanitizer_ether_addr *addr) { COMMON_INTERCEPTOR_READ_RANGE(ctx, hostname, REAL(strlen)(hostname) + 1); // FIXME: under ASan the call below may write to freed memory and corrupt // its metadata. See - // https://code.google.com/p/address-sanitizer/issues/detail?id=321. + // https://github.com/google/sanitizers/issues/321. int res = REAL(ether_hostton)(hostname, addr); if (!res && addr) COMMON_INTERCEPTOR_WRITE_RANGE(ctx, addr, sizeof(*addr)); return res; @@ -3445,7 +3562,7 @@ INTERCEPTOR(int, ether_line, char *line, __sanitizer_ether_addr *addr, if (line) COMMON_INTERCEPTOR_READ_RANGE(ctx, line, REAL(strlen)(line) + 1); // FIXME: under ASan the call below may write to freed memory and corrupt // its metadata. See - // https://code.google.com/p/address-sanitizer/issues/detail?id=321. + // https://github.com/google/sanitizers/issues/321. int res = REAL(ether_line)(line, addr, hostname); if (!res) { if (addr) COMMON_INTERCEPTOR_WRITE_RANGE(ctx, addr, sizeof(*addr)); @@ -3469,7 +3586,7 @@ INTERCEPTOR(char *, ether_ntoa_r, __sanitizer_ether_addr *addr, char *buf) { if (addr) COMMON_INTERCEPTOR_READ_RANGE(ctx, addr, sizeof(*addr)); // FIXME: under ASan the call below may write to freed memory and corrupt // its metadata. See - // https://code.google.com/p/address-sanitizer/issues/detail?id=321. + // https://github.com/google/sanitizers/issues/321. char *res = REAL(ether_ntoa_r)(addr, buf); if (res) COMMON_INTERCEPTOR_WRITE_RANGE(ctx, res, REAL(strlen)(res) + 1); return res; @@ -3481,7 +3598,7 @@ INTERCEPTOR(__sanitizer_ether_addr *, ether_aton_r, char *buf, if (buf) COMMON_INTERCEPTOR_READ_RANGE(ctx, buf, REAL(strlen)(buf) + 1); // FIXME: under ASan the call below may write to freed memory and corrupt // its metadata. See - // https://code.google.com/p/address-sanitizer/issues/detail?id=321. + // https://github.com/google/sanitizers/issues/321. __sanitizer_ether_addr *res = REAL(ether_aton_r)(buf, addr); if (res) COMMON_INTERCEPTOR_WRITE_RANGE(ctx, res, sizeof(*res)); return res; @@ -3499,7 +3616,7 @@ INTERCEPTOR(int, shmctl, int shmid, int cmd, void *buf) { COMMON_INTERCEPTOR_ENTER(ctx, shmctl, shmid, cmd, buf); // FIXME: under ASan the call below may write to freed memory and corrupt // its metadata. See - // https://code.google.com/p/address-sanitizer/issues/detail?id=321. + // https://github.com/google/sanitizers/issues/321. int res = REAL(shmctl)(shmid, cmd, buf); if (res >= 0) { unsigned sz = 0; @@ -3524,7 +3641,7 @@ INTERCEPTOR(int, random_r, void *buf, u32 *result) { COMMON_INTERCEPTOR_ENTER(ctx, random_r, buf, result); // FIXME: under ASan the call below may write to freed memory and corrupt // its metadata. See - // https://code.google.com/p/address-sanitizer/issues/detail?id=321. + // https://github.com/google/sanitizers/issues/321. int res = REAL(random_r)(buf, result); if (!res && result) COMMON_INTERCEPTOR_WRITE_RANGE(ctx, result, sizeof(*result)); @@ -3537,7 +3654,7 @@ INTERCEPTOR(int, random_r, void *buf, u32 *result) { // FIXME: under ASan the REAL() call below may write to freed memory and corrupt // its metadata. See -// https://code.google.com/p/address-sanitizer/issues/detail?id=321. +// https://github.com/google/sanitizers/issues/321. #if SANITIZER_INTERCEPT_PTHREAD_ATTR_GET || \ SANITIZER_INTERCEPT_PTHREAD_ATTR_GETINHERITSSCHED || \ SANITIZER_INTERCEPT_PTHREAD_MUTEXATTR_GET || \ @@ -3576,7 +3693,7 @@ INTERCEPTOR(int, pthread_attr_getstack, void *attr, void **addr, SIZE_T *size) { COMMON_INTERCEPTOR_ENTER(ctx, pthread_attr_getstack, attr, addr, size); // FIXME: under ASan the call below may write to freed memory and corrupt // its metadata. See - // https://code.google.com/p/address-sanitizer/issues/detail?id=321. + // https://github.com/google/sanitizers/issues/321. int res = REAL(pthread_attr_getstack)(attr, addr, size); if (!res) { if (addr) COMMON_INTERCEPTOR_WRITE_RANGE(ctx, addr, sizeof(*addr)); @@ -3625,7 +3742,7 @@ INTERCEPTOR(int, pthread_attr_getaffinity_np, void *attr, SIZE_T cpusetsize, cpuset); // FIXME: under ASan the call below may write to freed memory and corrupt // its metadata. See - // https://code.google.com/p/address-sanitizer/issues/detail?id=321. + // https://github.com/google/sanitizers/issues/321. int res = REAL(pthread_attr_getaffinity_np)(attr, cpusetsize, cpuset); if (!res && cpusetsize && cpuset) COMMON_INTERCEPTOR_WRITE_RANGE(ctx, cpuset, cpusetsize); @@ -3735,7 +3852,7 @@ INTERCEPTOR(char *, tmpnam, char *s) { if (s) // FIXME: under ASan the call below may write to freed memory and corrupt // its metadata. See - // https://code.google.com/p/address-sanitizer/issues/detail?id=321. + // https://github.com/google/sanitizers/issues/321. COMMON_INTERCEPTOR_WRITE_RANGE(ctx, s, REAL(strlen)(s) + 1); else COMMON_INTERCEPTOR_INITIALIZE_RANGE(res, REAL(strlen)(res) + 1); @@ -3753,7 +3870,7 @@ INTERCEPTOR(char *, tmpnam_r, char *s) { COMMON_INTERCEPTOR_ENTER(ctx, tmpnam_r, s); // FIXME: under ASan the call below may write to freed memory and corrupt // its metadata. See - // https://code.google.com/p/address-sanitizer/issues/detail?id=321. + // https://github.com/google/sanitizers/issues/321. char *res = REAL(tmpnam_r)(s); if (res && s) COMMON_INTERCEPTOR_WRITE_RANGE(ctx, s, REAL(strlen)(s) + 1); return res; @@ -3797,7 +3914,7 @@ INTERCEPTOR(void, sincos, double x, double *sin, double *cos) { COMMON_INTERCEPTOR_ENTER(ctx, sincos, x, sin, cos); // FIXME: under ASan the call below may write to freed memory and corrupt // its metadata. See - // https://code.google.com/p/address-sanitizer/issues/detail?id=321. + // https://github.com/google/sanitizers/issues/321. REAL(sincos)(x, sin, cos); if (sin) COMMON_INTERCEPTOR_WRITE_RANGE(ctx, sin, sizeof(*sin)); if (cos) COMMON_INTERCEPTOR_WRITE_RANGE(ctx, cos, sizeof(*cos)); @@ -3807,7 +3924,7 @@ INTERCEPTOR(void, sincosf, float x, float *sin, float *cos) { COMMON_INTERCEPTOR_ENTER(ctx, sincosf, x, sin, cos); // FIXME: under ASan the call below may write to freed memory and corrupt // its metadata. See - // https://code.google.com/p/address-sanitizer/issues/detail?id=321. + // https://github.com/google/sanitizers/issues/321. REAL(sincosf)(x, sin, cos); if (sin) COMMON_INTERCEPTOR_WRITE_RANGE(ctx, sin, sizeof(*sin)); if (cos) COMMON_INTERCEPTOR_WRITE_RANGE(ctx, cos, sizeof(*cos)); @@ -3817,7 +3934,7 @@ INTERCEPTOR(void, sincosl, long double x, long double *sin, long double *cos) { COMMON_INTERCEPTOR_ENTER(ctx, sincosl, x, sin, cos); // FIXME: under ASan the call below may write to freed memory and corrupt // its metadata. See - // https://code.google.com/p/address-sanitizer/issues/detail?id=321. + // https://github.com/google/sanitizers/issues/321. REAL(sincosl)(x, sin, cos); if (sin) COMMON_INTERCEPTOR_WRITE_RANGE(ctx, sin, sizeof(*sin)); if (cos) COMMON_INTERCEPTOR_WRITE_RANGE(ctx, cos, sizeof(*cos)); @@ -3836,7 +3953,7 @@ INTERCEPTOR(double, remquo, double x, double y, int *quo) { COMMON_INTERCEPTOR_ENTER(ctx, remquo, x, y, quo); // FIXME: under ASan the call below may write to freed memory and corrupt // its metadata. See - // https://code.google.com/p/address-sanitizer/issues/detail?id=321. + // https://github.com/google/sanitizers/issues/321. double res = REAL(remquo)(x, y, quo); if (quo) COMMON_INTERCEPTOR_WRITE_RANGE(ctx, quo, sizeof(*quo)); return res; @@ -3846,7 +3963,7 @@ INTERCEPTOR(float, remquof, float x, float y, int *quo) { COMMON_INTERCEPTOR_ENTER(ctx, remquof, x, y, quo); // FIXME: under ASan the call below may write to freed memory and corrupt // its metadata. See - // https://code.google.com/p/address-sanitizer/issues/detail?id=321. + // https://github.com/google/sanitizers/issues/321. float res = REAL(remquof)(x, y, quo); if (quo) COMMON_INTERCEPTOR_WRITE_RANGE(ctx, quo, sizeof(*quo)); return res; @@ -3856,7 +3973,7 @@ INTERCEPTOR(long double, remquol, long double x, long double y, int *quo) { COMMON_INTERCEPTOR_ENTER(ctx, remquol, x, y, quo); // FIXME: under ASan the call below may write to freed memory and corrupt // its metadata. See - // https://code.google.com/p/address-sanitizer/issues/detail?id=321. + // https://github.com/google/sanitizers/issues/321. long double res = REAL(remquol)(x, y, quo); if (quo) COMMON_INTERCEPTOR_WRITE_RANGE(ctx, quo, sizeof(*quo)); return res; @@ -3906,7 +4023,7 @@ INTERCEPTOR(double, lgamma_r, double x, int *signp) { COMMON_INTERCEPTOR_ENTER(ctx, lgamma_r, x, signp); // FIXME: under ASan the call below may write to freed memory and corrupt // its metadata. See - // https://code.google.com/p/address-sanitizer/issues/detail?id=321. + // https://github.com/google/sanitizers/issues/321. double res = REAL(lgamma_r)(x, signp); if (signp) COMMON_INTERCEPTOR_WRITE_RANGE(ctx, signp, sizeof(*signp)); return res; @@ -3916,7 +4033,7 @@ INTERCEPTOR(float, lgammaf_r, float x, int *signp) { COMMON_INTERCEPTOR_ENTER(ctx, lgammaf_r, x, signp); // FIXME: under ASan the call below may write to freed memory and corrupt // its metadata. See - // https://code.google.com/p/address-sanitizer/issues/detail?id=321. + // https://github.com/google/sanitizers/issues/321. float res = REAL(lgammaf_r)(x, signp); if (signp) COMMON_INTERCEPTOR_WRITE_RANGE(ctx, signp, sizeof(*signp)); return res; @@ -3934,7 +4051,7 @@ INTERCEPTOR(long double, lgammal_r, long double x, int *signp) { COMMON_INTERCEPTOR_ENTER(ctx, lgammal_r, x, signp); // FIXME: under ASan the call below may write to freed memory and corrupt // its metadata. See - // https://code.google.com/p/address-sanitizer/issues/detail?id=321. + // https://github.com/google/sanitizers/issues/321. long double res = REAL(lgammal_r)(x, signp); if (signp) COMMON_INTERCEPTOR_WRITE_RANGE(ctx, signp, sizeof(*signp)); return res; @@ -3950,7 +4067,7 @@ INTERCEPTOR(int, drand48_r, void *buffer, double *result) { COMMON_INTERCEPTOR_ENTER(ctx, drand48_r, buffer, result); // FIXME: under ASan the call below may write to freed memory and corrupt // its metadata. See - // https://code.google.com/p/address-sanitizer/issues/detail?id=321. + // https://github.com/google/sanitizers/issues/321. int res = REAL(drand48_r)(buffer, result); if (result) COMMON_INTERCEPTOR_WRITE_RANGE(ctx, result, sizeof(*result)); return res; @@ -3960,7 +4077,7 @@ INTERCEPTOR(int, lrand48_r, void *buffer, long *result) { COMMON_INTERCEPTOR_ENTER(ctx, lrand48_r, buffer, result); // FIXME: under ASan the call below may write to freed memory and corrupt // its metadata. See - // https://code.google.com/p/address-sanitizer/issues/detail?id=321. + // https://github.com/google/sanitizers/issues/321. int res = REAL(lrand48_r)(buffer, result); if (result) COMMON_INTERCEPTOR_WRITE_RANGE(ctx, result, sizeof(*result)); return res; @@ -3990,7 +4107,7 @@ INTERCEPTOR(SSIZE_T, getline, char **lineptr, SIZE_T *n, void *stream) { COMMON_INTERCEPTOR_ENTER(ctx, getline, lineptr, n, stream); // FIXME: under ASan the call below may write to freed memory and corrupt // its metadata. See - // https://code.google.com/p/address-sanitizer/issues/detail?id=321. + // https://github.com/google/sanitizers/issues/321. SSIZE_T res = REAL(getline)(lineptr, n, stream); if (res > 0) { COMMON_INTERCEPTOR_WRITE_RANGE(ctx, lineptr, sizeof(*lineptr)); @@ -4002,7 +4119,7 @@ INTERCEPTOR(SSIZE_T, getline, char **lineptr, SIZE_T *n, void *stream) { // FIXME: under ASan the call below may write to freed memory and corrupt its // metadata. See -// https://code.google.com/p/address-sanitizer/issues/detail?id=321. +// https://github.com/google/sanitizers/issues/321. #define GETDELIM_INTERCEPTOR_IMPL(vname) \ { \ void *ctx; \ @@ -4046,10 +4163,10 @@ INTERCEPTOR(SIZE_T, iconv, void *cd, char **inbuf, SIZE_T *inbytesleft, COMMON_INTERCEPTOR_READ_RANGE(ctx, *inbuf, *inbytesleft); if (outbytesleft) COMMON_INTERCEPTOR_READ_RANGE(ctx, outbytesleft, sizeof(*outbytesleft)); - void *outbuf_orig = outbuf ? *outbuf : 0; + void *outbuf_orig = outbuf ? *outbuf : nullptr; // FIXME: under ASan the call below may write to freed memory and corrupt // its metadata. See - // https://code.google.com/p/address-sanitizer/issues/detail?id=321. + // https://github.com/google/sanitizers/issues/321. SIZE_T res = REAL(iconv)(cd, inbuf, inbytesleft, outbuf, outbytesleft); if (res != (SIZE_T) - 1 && outbuf && *outbuf > outbuf_orig) { SIZE_T sz = (char *)*outbuf - (char *)outbuf_orig; @@ -4068,7 +4185,7 @@ INTERCEPTOR(__sanitizer_clock_t, times, void *tms) { COMMON_INTERCEPTOR_ENTER(ctx, times, tms); // FIXME: under ASan the call below may write to freed memory and corrupt // its metadata. See - // https://code.google.com/p/address-sanitizer/issues/detail?id=321. + // https://github.com/google/sanitizers/issues/321. __sanitizer_clock_t res = REAL(times)(tms); if (res != (__sanitizer_clock_t)-1 && tms) COMMON_INTERCEPTOR_WRITE_RANGE(ctx, tms, struct_tms_sz); @@ -4111,7 +4228,7 @@ INTERCEPTOR(SSIZE_T, listxattr, const char *path, char *list, SIZE_T size) { if (path) COMMON_INTERCEPTOR_READ_RANGE(ctx, path, REAL(strlen)(path) + 1); // FIXME: under ASan the call below may write to freed memory and corrupt // its metadata. See - // https://code.google.com/p/address-sanitizer/issues/detail?id=321. + // https://github.com/google/sanitizers/issues/321. SSIZE_T res = REAL(listxattr)(path, list, size); // Here and below, size == 0 is a special case where nothing is written to the // buffer, and res contains the desired buffer size. @@ -4124,7 +4241,7 @@ INTERCEPTOR(SSIZE_T, llistxattr, const char *path, char *list, SIZE_T size) { if (path) COMMON_INTERCEPTOR_READ_RANGE(ctx, path, REAL(strlen)(path) + 1); // FIXME: under ASan the call below may write to freed memory and corrupt // its metadata. See - // https://code.google.com/p/address-sanitizer/issues/detail?id=321. + // https://github.com/google/sanitizers/issues/321. SSIZE_T res = REAL(llistxattr)(path, list, size); if (size && res > 0 && list) COMMON_INTERCEPTOR_WRITE_RANGE(ctx, list, res); return res; @@ -4134,7 +4251,7 @@ INTERCEPTOR(SSIZE_T, flistxattr, int fd, char *list, SIZE_T size) { COMMON_INTERCEPTOR_ENTER(ctx, flistxattr, fd, list, size); // FIXME: under ASan the call below may write to freed memory and corrupt // its metadata. See - // https://code.google.com/p/address-sanitizer/issues/detail?id=321. + // https://github.com/google/sanitizers/issues/321. SSIZE_T res = REAL(flistxattr)(fd, list, size); if (size && res > 0 && list) COMMON_INTERCEPTOR_WRITE_RANGE(ctx, list, res); return res; @@ -4156,7 +4273,7 @@ INTERCEPTOR(SSIZE_T, getxattr, const char *path, const char *name, char *value, if (name) COMMON_INTERCEPTOR_READ_RANGE(ctx, name, REAL(strlen)(name) + 1); // FIXME: under ASan the call below may write to freed memory and corrupt // its metadata. See - // https://code.google.com/p/address-sanitizer/issues/detail?id=321. + // https://github.com/google/sanitizers/issues/321. SSIZE_T res = REAL(getxattr)(path, name, value, size); if (size && res > 0 && value) COMMON_INTERCEPTOR_WRITE_RANGE(ctx, value, res); return res; @@ -4169,7 +4286,7 @@ INTERCEPTOR(SSIZE_T, lgetxattr, const char *path, const char *name, char *value, if (name) COMMON_INTERCEPTOR_READ_RANGE(ctx, name, REAL(strlen)(name) + 1); // FIXME: under ASan the call below may write to freed memory and corrupt // its metadata. See - // https://code.google.com/p/address-sanitizer/issues/detail?id=321. + // https://github.com/google/sanitizers/issues/321. SSIZE_T res = REAL(lgetxattr)(path, name, value, size); if (size && res > 0 && value) COMMON_INTERCEPTOR_WRITE_RANGE(ctx, value, res); return res; @@ -4181,7 +4298,7 @@ INTERCEPTOR(SSIZE_T, fgetxattr, int fd, const char *name, char *value, if (name) COMMON_INTERCEPTOR_READ_RANGE(ctx, name, REAL(strlen)(name) + 1); // FIXME: under ASan the call below may write to freed memory and corrupt // its metadata. See - // https://code.google.com/p/address-sanitizer/issues/detail?id=321. + // https://github.com/google/sanitizers/issues/321. SSIZE_T res = REAL(fgetxattr)(fd, name, value, size); if (size && res > 0 && value) COMMON_INTERCEPTOR_WRITE_RANGE(ctx, value, res); return res; @@ -4200,7 +4317,7 @@ INTERCEPTOR(int, getresuid, void *ruid, void *euid, void *suid) { COMMON_INTERCEPTOR_ENTER(ctx, getresuid, ruid, euid, suid); // FIXME: under ASan the call below may write to freed memory and corrupt // its metadata. See - // https://code.google.com/p/address-sanitizer/issues/detail?id=321. + // https://github.com/google/sanitizers/issues/321. int res = REAL(getresuid)(ruid, euid, suid); if (res >= 0) { if (ruid) COMMON_INTERCEPTOR_WRITE_RANGE(ctx, ruid, uid_t_sz); @@ -4214,7 +4331,7 @@ INTERCEPTOR(int, getresgid, void *rgid, void *egid, void *sgid) { COMMON_INTERCEPTOR_ENTER(ctx, getresgid, rgid, egid, sgid); // FIXME: under ASan the call below may write to freed memory and corrupt // its metadata. See - // https://code.google.com/p/address-sanitizer/issues/detail?id=321. + // https://github.com/google/sanitizers/issues/321. int res = REAL(getresgid)(rgid, egid, sgid); if (res >= 0) { if (rgid) COMMON_INTERCEPTOR_WRITE_RANGE(ctx, rgid, gid_t_sz); @@ -4239,7 +4356,7 @@ INTERCEPTOR(int, getifaddrs, __sanitizer_ifaddrs **ifap) { COMMON_INTERCEPTOR_ENTER(ctx, getifaddrs, ifap); // FIXME: under ASan the call below may write to freed memory and corrupt // its metadata. See - // https://code.google.com/p/address-sanitizer/issues/detail?id=321. + // https://github.com/google/sanitizers/issues/321. int res = REAL(getifaddrs)(ifap); if (res == 0 && ifap) { COMMON_INTERCEPTOR_WRITE_RANGE(ctx, ifap, sizeof(void *)); @@ -4275,7 +4392,7 @@ INTERCEPTOR(char *, if_indextoname, unsigned int ifindex, char* ifname) { COMMON_INTERCEPTOR_ENTER(ctx, if_indextoname, ifindex, ifname); // FIXME: under ASan the call below may write to freed memory and corrupt // its metadata. See - // https://code.google.com/p/address-sanitizer/issues/detail?id=321. + // https://github.com/google/sanitizers/issues/321. char *res = REAL(if_indextoname)(ifindex, ifname); if (res && ifname) COMMON_INTERCEPTOR_WRITE_RANGE(ctx, ifname, REAL(strlen)(ifname) + 1); @@ -4303,7 +4420,7 @@ INTERCEPTOR(int, capget, void *hdrp, void *datap) { COMMON_INTERCEPTOR_READ_RANGE(ctx, hdrp, __user_cap_header_struct_sz); // FIXME: under ASan the call below may write to freed memory and corrupt // its metadata. See - // https://code.google.com/p/address-sanitizer/issues/detail?id=321. + // https://github.com/google/sanitizers/issues/321. int res = REAL(capget)(hdrp, datap); if (res == 0 && datap) COMMON_INTERCEPTOR_WRITE_RANGE(ctx, datap, __user_cap_data_struct_sz); @@ -4329,9 +4446,9 @@ INTERCEPTOR(int, capset, void *hdrp, const void *datap) { #endif #if SANITIZER_INTERCEPT_AEABI_MEM -DECLARE_REAL_AND_INTERCEPTOR(void *, memmove, void *, const void *, uptr); -DECLARE_REAL_AND_INTERCEPTOR(void *, memcpy, void *, const void *, uptr); -DECLARE_REAL_AND_INTERCEPTOR(void *, memset, void *, int, uptr); +DECLARE_REAL_AND_INTERCEPTOR(void *, memmove, void *, const void *, uptr) +DECLARE_REAL_AND_INTERCEPTOR(void *, memcpy, void *, const void *, uptr) +DECLARE_REAL_AND_INTERCEPTOR(void *, memset, void *, int, uptr) INTERCEPTOR(void *, __aeabi_memmove, void *to, const void *from, uptr size) { return WRAP(memmove)(to, from, size); @@ -4404,7 +4521,7 @@ INTERCEPTOR(int, ftime, __sanitizer_timeb *tp) { COMMON_INTERCEPTOR_ENTER(ctx, ftime, tp); // FIXME: under ASan the call below may write to freed memory and corrupt // its metadata. See - // https://code.google.com/p/address-sanitizer/issues/detail?id=321. + // https://github.com/google/sanitizers/issues/321. int res = REAL(ftime)(tp); if (tp) COMMON_INTERCEPTOR_WRITE_RANGE(ctx, tp, sizeof(*tp)); @@ -4422,7 +4539,7 @@ INTERCEPTOR(void, xdrmem_create, __sanitizer_XDR *xdrs, uptr addr, COMMON_INTERCEPTOR_ENTER(ctx, xdrmem_create, xdrs, addr, size, op); // FIXME: under ASan the call below may write to freed memory and corrupt // its metadata. See - // https://code.google.com/p/address-sanitizer/issues/detail?id=321. + // https://github.com/google/sanitizers/issues/321. REAL(xdrmem_create)(xdrs, addr, size, op); COMMON_INTERCEPTOR_WRITE_RANGE(ctx, xdrs, sizeof(*xdrs)); if (op == __sanitizer_XDR_ENCODE) { @@ -4437,14 +4554,14 @@ INTERCEPTOR(void, xdrstdio_create, __sanitizer_XDR *xdrs, void *file, int op) { COMMON_INTERCEPTOR_ENTER(ctx, xdrstdio_create, xdrs, file, op); // FIXME: under ASan the call below may write to freed memory and corrupt // its metadata. See - // https://code.google.com/p/address-sanitizer/issues/detail?id=321. + // https://github.com/google/sanitizers/issues/321. REAL(xdrstdio_create)(xdrs, file, op); COMMON_INTERCEPTOR_WRITE_RANGE(ctx, xdrs, sizeof(*xdrs)); } // FIXME: under ASan the call below may write to freed memory and corrupt // its metadata. See -// https://code.google.com/p/address-sanitizer/issues/detail?id=321. +// https://github.com/google/sanitizers/issues/321. #define XDR_INTERCEPTOR(F, T) \ INTERCEPTOR(int, F, __sanitizer_XDR *xdrs, T *p) { \ void *ctx; \ @@ -4498,7 +4615,7 @@ INTERCEPTOR(int, xdr_bytes, __sanitizer_XDR *xdrs, char **p, unsigned *sizep, } // FIXME: under ASan the call below may write to freed memory and corrupt // its metadata. See - // https://code.google.com/p/address-sanitizer/issues/detail?id=321. + // https://github.com/google/sanitizers/issues/321. int res = REAL(xdr_bytes)(xdrs, p, sizep, maxsize); if (p && sizep && xdrs->x_op == __sanitizer_XDR_DECODE) { COMMON_INTERCEPTOR_WRITE_RANGE(ctx, p, sizeof(*p)); @@ -4518,7 +4635,7 @@ INTERCEPTOR(int, xdr_string, __sanitizer_XDR *xdrs, char **p, } // FIXME: under ASan the call below may write to freed memory and corrupt // its metadata. See - // https://code.google.com/p/address-sanitizer/issues/detail?id=321. + // https://github.com/google/sanitizers/issues/321. int res = REAL(xdr_string)(xdrs, p, maxsize); if (p && xdrs->x_op == __sanitizer_XDR_DECODE) { COMMON_INTERCEPTOR_WRITE_RANGE(ctx, p, sizeof(*p)); @@ -4570,7 +4687,7 @@ INTERCEPTOR(void *, tsearch, void *key, void **rootp, COMMON_INTERCEPTOR_ENTER(ctx, tsearch, key, rootp, compar); // FIXME: under ASan the call below may write to freed memory and corrupt // its metadata. See - // https://code.google.com/p/address-sanitizer/issues/detail?id=321. + // https://github.com/google/sanitizers/issues/321. void *res = REAL(tsearch)(key, rootp, compar); if (res && *(void **)res == key) COMMON_INTERCEPTOR_WRITE_RANGE(ctx, res, sizeof(void *)); @@ -4652,7 +4769,7 @@ INTERCEPTOR(int, __woverflow, __sanitizer_FILE *fp, int ch) { INTERCEPTOR(__sanitizer_FILE *, fopen, const char *path, const char *mode) { void *ctx; COMMON_INTERCEPTOR_ENTER(ctx, fopen, path, mode); - COMMON_INTERCEPTOR_READ_RANGE(ctx, path, REAL(strlen)(path) + 1); + if (path) COMMON_INTERCEPTOR_READ_RANGE(ctx, path, REAL(strlen)(path) + 1); COMMON_INTERCEPTOR_READ_RANGE(ctx, mode, REAL(strlen)(mode) + 1); __sanitizer_FILE *res = REAL(fopen)(path, mode); COMMON_INTERCEPTOR_FILE_OPEN(ctx, res, path); @@ -4671,7 +4788,7 @@ INTERCEPTOR(__sanitizer_FILE *, freopen, const char *path, const char *mode, __sanitizer_FILE *fp) { void *ctx; COMMON_INTERCEPTOR_ENTER(ctx, freopen, path, mode, fp); - COMMON_INTERCEPTOR_READ_RANGE(ctx, path, REAL(strlen)(path) + 1); + if (path) COMMON_INTERCEPTOR_READ_RANGE(ctx, path, REAL(strlen)(path) + 1); COMMON_INTERCEPTOR_READ_RANGE(ctx, mode, REAL(strlen)(mode) + 1); COMMON_INTERCEPTOR_FILE_CLOSE(ctx, fp); __sanitizer_FILE *res = REAL(freopen)(path, mode, fp); @@ -4702,7 +4819,7 @@ INTERCEPTOR(__sanitizer_FILE *, freopen64, const char *path, const char *mode, __sanitizer_FILE *fp) { void *ctx; COMMON_INTERCEPTOR_ENTER(ctx, freopen64, path, mode, fp); - COMMON_INTERCEPTOR_READ_RANGE(ctx, path, REAL(strlen)(path) + 1); + if (path) COMMON_INTERCEPTOR_READ_RANGE(ctx, path, REAL(strlen)(path) + 1); COMMON_INTERCEPTOR_READ_RANGE(ctx, mode, REAL(strlen)(mode) + 1); COMMON_INTERCEPTOR_FILE_CLOSE(ctx, fp); __sanitizer_FILE *res = REAL(freopen64)(path, mode, fp); @@ -4723,7 +4840,7 @@ INTERCEPTOR(__sanitizer_FILE *, open_memstream, char **ptr, SIZE_T *sizeloc) { COMMON_INTERCEPTOR_ENTER(ctx, open_memstream, ptr, sizeloc); // FIXME: under ASan the call below may write to freed memory and corrupt // its metadata. See - // https://code.google.com/p/address-sanitizer/issues/detail?id=321. + // https://github.com/google/sanitizers/issues/321. __sanitizer_FILE *res = REAL(open_memstream)(ptr, sizeloc); if (res) { COMMON_INTERCEPTOR_WRITE_RANGE(ctx, ptr, sizeof(*ptr)); @@ -4754,7 +4871,7 @@ INTERCEPTOR(__sanitizer_FILE *, fmemopen, void *buf, SIZE_T size, COMMON_INTERCEPTOR_ENTER(ctx, fmemopen, buf, size, mode); // FIXME: under ASan the call below may write to freed memory and corrupt // its metadata. See - // https://code.google.com/p/address-sanitizer/issues/detail?id=321. + // https://github.com/google/sanitizers/issues/321. __sanitizer_FILE *res = REAL(fmemopen)(buf, size, mode); if (res) unpoison_file(res); return res; @@ -4831,15 +4948,14 @@ INTERCEPTOR(int, fflush, __sanitizer_FILE *fp) { INTERCEPTOR(int, fclose, __sanitizer_FILE *fp) { void *ctx; COMMON_INTERCEPTOR_ENTER(ctx, fclose, fp); - if (fp) { - COMMON_INTERCEPTOR_FILE_CLOSE(ctx, fp); - const FileMetadata *m = GetInterceptorMetadata(fp); - if (m) { - COMMON_INTERCEPTOR_INITIALIZE_RANGE(*m->addr, *m->size); - DeleteInterceptorMetadata(fp); - } + COMMON_INTERCEPTOR_FILE_CLOSE(ctx, fp); + const FileMetadata *m = GetInterceptorMetadata(fp); + int res = REAL(fclose)(fp); + if (m) { + COMMON_INTERCEPTOR_INITIALIZE_RANGE(*m->addr, *m->size); + DeleteInterceptorMetadata(fp); } - return REAL(fclose)(fp); + return res; } #define INIT_FCLOSE COMMON_INTERCEPT_FUNCTION(fclose); #else @@ -4922,7 +5038,7 @@ static void MlockIsUnsupported() { static atomic_uint8_t printed; if (atomic_exchange(&printed, 1, memory_order_relaxed)) return; - VPrintf(1, "INFO: %s ignores mlock/mlockall/munlock/munlockall\n", + VPrintf(1, "%s ignores mlock/mlockall/munlock/munlockall\n", SanitizerToolName); } @@ -5013,6 +5129,192 @@ INTERCEPTOR(__sanitizer_FILE *, fopencookie, void *cookie, const char *mode, #define INIT_FOPENCOOKIE #endif // SANITIZER_INTERCEPT_FOPENCOOKIE +#if SANITIZER_INTERCEPT_SEM +INTERCEPTOR(int, sem_init, __sanitizer_sem_t *s, int pshared, unsigned value) { + void *ctx; + COMMON_INTERCEPTOR_ENTER(ctx, sem_init, s, pshared, value); + // Workaround a bug in glibc's "old" semaphore implementation by + // zero-initializing the sem_t contents. This has to be done here because + // interceptors bind to the lowest symbols version by default, hitting the + // buggy code path while the non-sanitized build of the same code works fine. + REAL(memset)(s, 0, sizeof(*s)); + int res = REAL(sem_init)(s, pshared, value); + return res; +} + +INTERCEPTOR(int, sem_destroy, __sanitizer_sem_t *s) { + void *ctx; + COMMON_INTERCEPTOR_ENTER(ctx, sem_destroy, s); + int res = REAL(sem_destroy)(s); + return res; +} + +INTERCEPTOR(int, sem_wait, __sanitizer_sem_t *s) { + void *ctx; + COMMON_INTERCEPTOR_ENTER(ctx, sem_wait, s); + int res = COMMON_INTERCEPTOR_BLOCK_REAL(sem_wait)(s); + if (res == 0) { + COMMON_INTERCEPTOR_ACQUIRE(ctx, (uptr)s); + } + return res; +} + +INTERCEPTOR(int, sem_trywait, __sanitizer_sem_t *s) { + void *ctx; + COMMON_INTERCEPTOR_ENTER(ctx, sem_trywait, s); + int res = COMMON_INTERCEPTOR_BLOCK_REAL(sem_trywait)(s); + if (res == 0) { + COMMON_INTERCEPTOR_ACQUIRE(ctx, (uptr)s); + } + return res; +} + +INTERCEPTOR(int, sem_timedwait, __sanitizer_sem_t *s, void *abstime) { + void *ctx; + COMMON_INTERCEPTOR_ENTER(ctx, sem_timedwait, s, abstime); + COMMON_INTERCEPTOR_READ_RANGE(ctx, abstime, struct_timespec_sz); + int res = COMMON_INTERCEPTOR_BLOCK_REAL(sem_timedwait)(s, abstime); + if (res == 0) { + COMMON_INTERCEPTOR_ACQUIRE(ctx, (uptr)s); + } + return res; +} + +INTERCEPTOR(int, sem_post, __sanitizer_sem_t *s) { + void *ctx; + COMMON_INTERCEPTOR_ENTER(ctx, sem_post, s); + COMMON_INTERCEPTOR_RELEASE(ctx, (uptr)s); + int res = REAL(sem_post)(s); + return res; +} + +INTERCEPTOR(int, sem_getvalue, __sanitizer_sem_t *s, int *sval) { + void *ctx; + COMMON_INTERCEPTOR_ENTER(ctx, sem_getvalue, s, sval); + int res = REAL(sem_getvalue)(s, sval); + if (res == 0) { + COMMON_INTERCEPTOR_ACQUIRE(ctx, (uptr)s); + COMMON_INTERCEPTOR_WRITE_RANGE(ctx, sval, sizeof(*sval)); + } + return res; +} +#define INIT_SEM \ + COMMON_INTERCEPT_FUNCTION(sem_init); \ + COMMON_INTERCEPT_FUNCTION(sem_destroy); \ + COMMON_INTERCEPT_FUNCTION(sem_wait); \ + COMMON_INTERCEPT_FUNCTION(sem_trywait); \ + COMMON_INTERCEPT_FUNCTION(sem_timedwait); \ + COMMON_INTERCEPT_FUNCTION(sem_post); \ + COMMON_INTERCEPT_FUNCTION(sem_getvalue); +#else +#define INIT_SEM +#endif // SANITIZER_INTERCEPT_SEM + +#if SANITIZER_INTERCEPT_PTHREAD_SETCANCEL +INTERCEPTOR(int, pthread_setcancelstate, int state, int *oldstate) { + void *ctx; + COMMON_INTERCEPTOR_ENTER(ctx, pthread_setcancelstate, state, oldstate); + int res = REAL(pthread_setcancelstate)(state, oldstate); + if (res == 0) + COMMON_INTERCEPTOR_WRITE_RANGE(ctx, oldstate, sizeof(*oldstate)); + return res; +} + +INTERCEPTOR(int, pthread_setcanceltype, int type, int *oldtype) { + void *ctx; + COMMON_INTERCEPTOR_ENTER(ctx, pthread_setcanceltype, type, oldtype); + int res = REAL(pthread_setcanceltype)(type, oldtype); + if (res == 0) + COMMON_INTERCEPTOR_WRITE_RANGE(ctx, oldtype, sizeof(*oldtype)); + return res; +} +#define INIT_PTHREAD_SETCANCEL \ + COMMON_INTERCEPT_FUNCTION(pthread_setcancelstate); \ + COMMON_INTERCEPT_FUNCTION(pthread_setcanceltype); +#else +#define INIT_PTHREAD_SETCANCEL +#endif + +#if SANITIZER_INTERCEPT_MINCORE +INTERCEPTOR(int, mincore, void *addr, uptr length, unsigned char *vec) { + void *ctx; + COMMON_INTERCEPTOR_ENTER(ctx, mincore, addr, length, vec); + int res = REAL(mincore)(addr, length, vec); + if (res == 0) { + uptr page_size = GetPageSizeCached(); + uptr vec_size = ((length + page_size - 1) & (~(page_size - 1))) / page_size; + COMMON_INTERCEPTOR_WRITE_RANGE(ctx, vec, vec_size); + } + return res; +} +#define INIT_MINCORE COMMON_INTERCEPT_FUNCTION(mincore); +#else +#define INIT_MINCORE +#endif + +#if SANITIZER_INTERCEPT_PROCESS_VM_READV +INTERCEPTOR(SSIZE_T, process_vm_readv, int pid, __sanitizer_iovec *local_iov, + uptr liovcnt, __sanitizer_iovec *remote_iov, uptr riovcnt, + uptr flags) { + void *ctx; + COMMON_INTERCEPTOR_ENTER(ctx, process_vm_readv, pid, local_iov, liovcnt, + remote_iov, riovcnt, flags); + SSIZE_T res = REAL(process_vm_readv)(pid, local_iov, liovcnt, remote_iov, + riovcnt, flags); + if (res > 0) + write_iovec(ctx, local_iov, liovcnt, res); + return res; +} + +INTERCEPTOR(SSIZE_T, process_vm_writev, int pid, __sanitizer_iovec *local_iov, + uptr liovcnt, __sanitizer_iovec *remote_iov, uptr riovcnt, + uptr flags) { + void *ctx; + COMMON_INTERCEPTOR_ENTER(ctx, process_vm_writev, pid, local_iov, liovcnt, + remote_iov, riovcnt, flags); + SSIZE_T res = REAL(process_vm_writev)(pid, local_iov, liovcnt, remote_iov, + riovcnt, flags); + if (res > 0) + read_iovec(ctx, local_iov, liovcnt, res); + return res; +} +#define INIT_PROCESS_VM_READV \ + COMMON_INTERCEPT_FUNCTION(process_vm_readv); \ + COMMON_INTERCEPT_FUNCTION(process_vm_writev); +#else +#define INIT_PROCESS_VM_READV +#endif + +#if SANITIZER_INTERCEPT_CTERMID +INTERCEPTOR(char *, ctermid, char *s) { + void *ctx; + COMMON_INTERCEPTOR_ENTER(ctx, ctermid, s); + char *res = REAL(ctermid)(s); + if (res) { + COMMON_INTERCEPTOR_INITIALIZE_RANGE(res, REAL(strlen)(res) + 1); + } + return res; +} +#define INIT_CTERMID COMMON_INTERCEPT_FUNCTION(ctermid); +#else +#define INIT_CTERMID +#endif + +#if SANITIZER_INTERCEPT_CTERMID_R +INTERCEPTOR(char *, ctermid_r, char *s) { + void *ctx; + COMMON_INTERCEPTOR_ENTER(ctx, ctermid_r, s); + char *res = REAL(ctermid_r)(s); + if (res) { + COMMON_INTERCEPTOR_INITIALIZE_RANGE(res, REAL(strlen)(res) + 1); + } + return res; +} +#define INIT_CTERMID_R COMMON_INTERCEPT_FUNCTION(ctermid_r); +#else +#define INIT_CTERMID_R +#endif + static void InitializeCommonInterceptors() { static u64 metadata_mem[sizeof(MetadataHashMap) / sizeof(u64) + 1]; interceptor_metadata_map = new((void *)&metadata_mem) MetadataHashMap(); @@ -5027,6 +5329,7 @@ static void InitializeCommonInterceptors() { INIT_STRSPN; INIT_STRPBRK; INIT_MEMCHR; + INIT_MEMCMP; INIT_MEMRCHR; INIT_READ; INIT_PREAD; @@ -5092,6 +5395,7 @@ static void InitializeCommonInterceptors() { INIT_MBSNRTOWCS; INIT_WCSTOMBS; INIT_WCSNRTOMBS; + INIT_WCRTOMB; INIT_TCGETATTR; INIT_REALPATH; INIT_CANONICALIZE_FILE_NAME; @@ -5181,4 +5485,10 @@ static void InitializeCommonInterceptors() { INIT_TIMERFD; INIT_MLOCKX; INIT_FOPENCOOKIE; + INIT_SEM; + INIT_PTHREAD_SETCANCEL; + INIT_MINCORE; + INIT_PROCESS_VM_READV; + INIT_CTERMID; + INIT_CTERMID_R; } diff --git a/lib/sanitizer_common/sanitizer_common_interceptors_format.inc b/lib/sanitizer_common/sanitizer_common_interceptors_format.inc index 8f94802aeae8..92318cda35fd 100644 --- a/lib/sanitizer_common/sanitizer_common_interceptors_format.inc +++ b/lib/sanitizer_common/sanitizer_common_interceptors_format.inc @@ -13,6 +13,7 @@ // with a few common GNU extensions. // //===----------------------------------------------------------------------===// + #include <stdarg.h> static const char *parse_number(const char *p, int *out) { @@ -191,7 +192,7 @@ static const char *scanf_parse_next(const char *p, bool allowGnuMalloc, continue; } if (*p == '\0') { - return 0; + return nullptr; } // %n$ p = maybe_parse_param_index(p, &dir->argIdx); @@ -206,7 +207,7 @@ static const char *scanf_parse_next(const char *p, bool allowGnuMalloc, p = parse_number(p, &dir->fieldWidth); CHECK(p); if (dir->fieldWidth <= 0) // Width if at all must be non-zero - return 0; + return nullptr; } // m if (*p == 'm') { @@ -226,8 +227,8 @@ static const char *scanf_parse_next(const char *p, bool allowGnuMalloc, while (*p && *p != ']') ++p; if (*p == 0) - return 0; // unexpected end of string - // Consume the closing ']'. + return nullptr; // unexpected end of string + // Consume the closing ']'. ++p; } // This is unfortunately ambiguous between old GNU extension @@ -251,7 +252,7 @@ static const char *scanf_parse_next(const char *p, bool allowGnuMalloc, while (*q && *q != ']' && *q != '%') ++q; if (*q == 0 || *q == '%') - return 0; + return nullptr; p = q + 1; // Consume the closing ']'. dir->maybeGnuMalloc = true; } @@ -395,7 +396,7 @@ static const char *printf_parse_next(const char *p, PrintfDirective *dir) { continue; } if (*p == '\0') { - return 0; + return nullptr; } // %n$ p = maybe_parse_param_index(p, &dir->precisionIdx); @@ -408,7 +409,7 @@ static const char *printf_parse_next(const char *p, PrintfDirective *dir) { p = maybe_parse_number_or_star(p, &dir->fieldWidth, &dir->starredWidth); if (!p) - return 0; + return nullptr; // Precision if (*p == '.') { ++p; @@ -416,7 +417,7 @@ static const char *printf_parse_next(const char *p, PrintfDirective *dir) { p = maybe_parse_number_or_star(p, &dir->fieldPrecision, &dir->starredPrecision); if (!p) - return 0; + return nullptr; // m$ if (dir->starredPrecision) { p = maybe_parse_param_index(p, &dir->precisionIdx); @@ -556,4 +557,4 @@ static void printf_common(void *ctx, const char *format, va_list aq) { } } -#endif // SANITIZER_INTERCEPT_PRINTF +#endif // SANITIZER_INTERCEPT_PRINTF diff --git a/lib/sanitizer_common/sanitizer_common_interceptors_ioctl.inc b/lib/sanitizer_common/sanitizer_common_interceptors_ioctl.inc index b94c21c96aa0..fcd0a3d70f52 100755 --- a/lib/sanitizer_common/sanitizer_common_interceptors_ioctl.inc +++ b/lib/sanitizer_common/sanitizer_common_interceptors_ioctl.inc @@ -520,7 +520,7 @@ static const ioctl_desc *ioctl_table_lookup(unsigned req) { if (left == right && ioctl_table[left].req == req) return ioctl_table + left; else - return 0; + return nullptr; } static bool ioctl_decode(unsigned req, ioctl_desc *desc) { @@ -567,7 +567,7 @@ static const ioctl_desc *ioctl_lookup(unsigned req) { (desc->type == ioctl_desc::READWRITE || desc->type == ioctl_desc::WRITE || desc->type == ioctl_desc::READ)) return desc; - return 0; + return nullptr; } static void ioctl_common_pre(void *ctx, const ioctl_desc *desc, int d, diff --git a/lib/sanitizer_common/sanitizer_common_libcdep.cc b/lib/sanitizer_common/sanitizer_common_libcdep.cc index 1b65bced75d5..b5d46f244f66 100644 --- a/lib/sanitizer_common/sanitizer_common_libcdep.cc +++ b/lib/sanitizer_common/sanitizer_common_libcdep.cc @@ -12,6 +12,7 @@ //===----------------------------------------------------------------------===// #include "sanitizer_common.h" + #include "sanitizer_flags.h" #include "sanitizer_stackdepot.h" #include "sanitizer_stacktrace.h" @@ -46,6 +47,7 @@ void SetSandboxingCallback(void (*f)()) { } void ReportErrorSummary(const char *error_type, StackTrace *stack) { +#if !SANITIZER_GO if (!common_flags()->print_summary) return; if (stack->size == 0) { @@ -58,6 +60,7 @@ void ReportErrorSummary(const char *error_type, StackTrace *stack) { SymbolizedStack *frame = Symbolizer::GetOrInit()->SymbolizePC(pc); ReportErrorSummary(error_type, frame->info); frame->ClearAll(); +#endif } static void (*SoftRssLimitExceededCallback)(bool exceeded); @@ -116,8 +119,30 @@ void BackgroundThread(void *arg) { } } +void WriteToSyslog(const char *msg) { + InternalScopedString msg_copy(kErrorMessageBufferSize); + msg_copy.append("%s", msg); + char *p = msg_copy.data(); + char *q; + + // Remove color sequences since syslogs cannot print them. + RemoveANSIEscapeSequencesFromString(p); + + // Print one line at a time. + // syslog, at least on Android, has an implicit message length limit. + do { + q = internal_strchr(p, '\n'); + if (q) + *q = '\0'; + WriteOneLineToSyslog(p); + if (q) + p = q + 1; + } while (q); +} + void MaybeStartBackgroudThread() { -#if SANITIZER_LINUX // Need to implement/test on other platforms. +#if SANITIZER_LINUX && \ + !SANITIZER_GO // Need to implement/test on other platforms. // Start the background thread if one of the rss limits is given. if (!common_flags()->hard_rss_limit_mb && !common_flags()->soft_rss_limit_mb) return; diff --git a/lib/sanitizer_common/sanitizer_common_nolibc.cc b/lib/sanitizer_common/sanitizer_common_nolibc.cc new file mode 100644 index 000000000000..89c17e0797ef --- /dev/null +++ b/lib/sanitizer_common/sanitizer_common_nolibc.cc @@ -0,0 +1,26 @@ +//===-- sanitizer_common_nolibc.cc ----------------------------------------===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// +// +// This file contains stubs for libc function to facilitate optional use of +// libc in no-libcdep sources. +//===----------------------------------------------------------------------===// + +#include "sanitizer_platform.h" +#include "sanitizer_common.h" +#include "sanitizer_libc.h" + +namespace __sanitizer { + +#if SANITIZER_LINUX +bool ShouldLogAfterPrintf() { return false; } +#endif +void WriteToSyslog(const char *buffer) {} +void Abort() { internal__exit(1); } + +} // namespace __sanitizer diff --git a/lib/sanitizer_common/sanitizer_common_syscalls.inc b/lib/sanitizer_common/sanitizer_common_syscalls.inc index 5d4840f961ef..008e57745cc5 100644 --- a/lib/sanitizer_common/sanitizer_common_syscalls.inc +++ b/lib/sanitizer_common/sanitizer_common_syscalls.inc @@ -2301,7 +2301,7 @@ POST_SYSCALL(ni_syscall)(long res) {} PRE_SYSCALL(ptrace)(long request, long pid, long addr, long data) { #if !SANITIZER_ANDROID && \ (defined(__i386) || defined(__x86_64) || defined(__mips64) || \ - defined(__powerpc64__)) + defined(__powerpc64__) || defined(__aarch64__)) if (data) { if (request == ptrace_setregs) { PRE_READ((void *)data, struct_user_regs_struct_sz); @@ -2322,7 +2322,7 @@ PRE_SYSCALL(ptrace)(long request, long pid, long addr, long data) { POST_SYSCALL(ptrace)(long res, long request, long pid, long addr, long data) { #if !SANITIZER_ANDROID && \ (defined(__i386) || defined(__x86_64) || defined(__mips64) || \ - defined(__powerpc64__)) + defined(__powerpc64__) || defined(__aarch64__)) if (res >= 0 && data) { // Note that this is different from the interceptor in // sanitizer_common_interceptors.inc. diff --git a/lib/sanitizer_common/sanitizer_coverage_libcdep.cc b/lib/sanitizer_common/sanitizer_coverage_libcdep.cc index f511c996d8c2..b9833c5a1d97 100644 --- a/lib/sanitizer_common/sanitizer_coverage_libcdep.cc +++ b/lib/sanitizer_common/sanitizer_coverage_libcdep.cc @@ -53,6 +53,12 @@ static const u64 kMagic32 = 0xC0BFFFFFFFFFFF32ULL; static atomic_uint32_t dump_once_guard; // Ensure that CovDump runs only once. static atomic_uintptr_t coverage_counter; +static atomic_uintptr_t caller_callee_counter; + +static void ResetGlobalCounters() { + return atomic_store(&coverage_counter, 0, memory_order_relaxed); + return atomic_store(&caller_callee_counter, 0, memory_order_relaxed); +} // pc_array is the array containing the covered PCs. // To make the pc_array thread- and async-signal-safe it has to be large enough. @@ -90,7 +96,7 @@ class CoverageData { void DumpAll(); ALWAYS_INLINE - void TraceBasicBlock(s32 *id); + void TraceBasicBlock(u32 *id); void InitializeGuardArray(s32 *guards); void InitializeGuards(s32 *guards, uptr n, const char *module_name, @@ -225,7 +231,8 @@ void CoverageData::InitializeGuardArray(s32 *guards) { Enable(); // Make sure coverage is enabled at this point. s32 n = guards[0]; for (s32 j = 1; j <= n; j++) { - uptr idx = atomic_fetch_add(&pc_array_index, 1, memory_order_relaxed); + uptr idx = atomic_load_relaxed(&pc_array_index); + atomic_store_relaxed(&pc_array_index, idx + 1); guards[j] = -static_cast<s32>(idx + 1); } } @@ -435,7 +442,7 @@ void CoverageData::IndirCall(uptr caller, uptr callee, uptr callee_cache[], uptr was = 0; if (atomic_compare_exchange_strong(&atomic_callee_cache[i], &was, callee, memory_order_seq_cst)) { - atomic_fetch_add(&coverage_counter, 1, memory_order_relaxed); + atomic_fetch_add(&caller_callee_counter, 1, memory_order_relaxed); return; } if (was == callee) // Already have this callee. @@ -675,11 +682,11 @@ void CoverageData::DumpCallerCalleePairs() { // it once and then cache in the provided 'cache' storage. // // This function will eventually be inlined by the compiler. -void CoverageData::TraceBasicBlock(s32 *id) { +void CoverageData::TraceBasicBlock(u32 *id) { // Will trap here if // 1. coverage is not enabled at run-time. // 2. The array tr_event_array is full. - *tr_event_pointer = static_cast<u32>(*id - 1); + *tr_event_pointer = *id - 1; tr_event_pointer++; } @@ -813,7 +820,7 @@ void CovPrepareForSandboxing(__sanitizer_sandbox_arguments *args) { } else { InternalScopedString path(kMaxPathLength); // Pre-open the file now. The sandbox won't allow us to do it later. - cov_fd = CovOpenFile(&path, true /* packed */, 0); + cov_fd = CovOpenFile(&path, true /* packed */, nullptr); } } @@ -832,6 +839,11 @@ void CovAfterFork(int child_pid) { coverage_data.AfterFork(child_pid); } +static void MaybeDumpCoverage() { + if (common_flags()->coverage) + __sanitizer_cov_dump(); +} + void InitializeCoverage(bool enabled, const char *dir) { if (coverage_enabled) return; // May happen if two sanitizer enable coverage in the same process. @@ -840,6 +852,7 @@ void InitializeCoverage(bool enabled, const char *dir) { coverage_data.Init(); if (enabled) coverage_data.Enable(); if (!common_flags()->coverage_direct) Atexit(__sanitizer_cov_dump); + AddDieCallback(MaybeDumpCoverage); } void ReInitializeCoverage(bool enabled, const char *dir) { @@ -853,7 +866,7 @@ void CoverageUpdateMapping() { CovUpdateMapping(coverage_dir); } -} // namespace __sanitizer +} // namespace __sanitizer extern "C" { SANITIZER_INTERFACE_ATTRIBUTE void __sanitizer_cov(u32 *guard) { @@ -902,15 +915,23 @@ uptr __sanitizer_get_total_unique_coverage() { } SANITIZER_INTERFACE_ATTRIBUTE -void __sanitizer_cov_trace_func_enter(s32 *id) { +uptr __sanitizer_get_total_unique_caller_callee_pairs() { + return atomic_load(&caller_callee_counter, memory_order_relaxed); +} + +SANITIZER_INTERFACE_ATTRIBUTE +void __sanitizer_cov_trace_func_enter(u32 *id) { + __sanitizer_cov_with_check(id); coverage_data.TraceBasicBlock(id); } SANITIZER_INTERFACE_ATTRIBUTE -void __sanitizer_cov_trace_basic_block(s32 *id) { +void __sanitizer_cov_trace_basic_block(u32 *id) { + __sanitizer_cov_with_check(id); coverage_data.TraceBasicBlock(id); } SANITIZER_INTERFACE_ATTRIBUTE void __sanitizer_reset_coverage() { + ResetGlobalCounters(); coverage_data.ReinitializeGuards(); internal_bzero_aligned16( coverage_data.data(), @@ -931,7 +952,9 @@ SANITIZER_INTERFACE_ATTRIBUTE uptr __sanitizer_update_counter_bitset_and_clear_counters(u8 *bitset) { return coverage_data.Update8bitCounterBitsetAndClearCounters(bitset); } -// Default empty implementation (weak). Users should redefine it. +// Default empty implementations (weak). Users should redefine them. SANITIZER_INTERFACE_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE void __sanitizer_cov_trace_cmp() {} -} // extern "C" +SANITIZER_INTERFACE_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE +void __sanitizer_cov_trace_switch() {} +} // extern "C" diff --git a/lib/sanitizer_common/sanitizer_coverage_mapping_libcdep.cc b/lib/sanitizer_common/sanitizer_coverage_mapping_libcdep.cc index a3d75abc0476..c8b5d9014ede 100644 --- a/lib/sanitizer_common/sanitizer_coverage_mapping_libcdep.cc +++ b/lib/sanitizer_common/sanitizer_coverage_mapping_libcdep.cc @@ -75,7 +75,7 @@ void CovUpdateMapping(const char *coverage_dir, uptr caller_pc) { InternalScopedBuffer<LoadedModule> modules(kMaxNumberOfModules); CHECK(modules.data()); int n_modules = GetListOfModules(modules.data(), kMaxNumberOfModules, - /* filter */ 0); + /* filter */ nullptr); text.append("%d\n", sizeof(uptr) * 8); for (int i = 0; i < n_modules; ++i) { @@ -124,4 +124,4 @@ void CovUpdateMapping(const char *coverage_dir, uptr caller_pc) { } } -} // namespace __sanitizer +} // namespace __sanitizer diff --git a/lib/sanitizer_common/sanitizer_deadlock_detector1.cc b/lib/sanitizer_common/sanitizer_deadlock_detector1.cc index 5890b54b933b..bd57a403bdcc 100644 --- a/lib/sanitizer_common/sanitizer_deadlock_detector1.cc +++ b/lib/sanitizer_common/sanitizer_deadlock_detector1.cc @@ -71,7 +71,7 @@ DD::DD(const DDFlags *flags) } DDPhysicalThread* DD::CreatePhysicalThread() { - return 0; + return nullptr; } void DD::DestroyPhysicalThread(DDPhysicalThread *pt) { @@ -181,10 +181,10 @@ void DD::MutexDestroy(DDCallback *cb, DDReport *DD::GetReport(DDCallback *cb) { if (!cb->lt->report_pending) - return 0; + return nullptr; cb->lt->report_pending = false; return &cb->lt->rep; } -} // namespace __sanitizer -#endif // #if SANITIZER_DEADLOCK_DETECTOR_VERSION == 1 +} // namespace __sanitizer +#endif // #if SANITIZER_DEADLOCK_DETECTOR_VERSION == 1 diff --git a/lib/sanitizer_common/sanitizer_deadlock_detector_interface.h b/lib/sanitizer_common/sanitizer_deadlock_detector_interface.h index 8cf26e0d5528..b6e91a16e257 100644 --- a/lib/sanitizer_common/sanitizer_deadlock_detector_interface.h +++ b/lib/sanitizer_common/sanitizer_deadlock_detector_interface.h @@ -72,10 +72,10 @@ struct DDCallback { struct DDetector { static DDetector *Create(const DDFlags *flags); - virtual DDPhysicalThread* CreatePhysicalThread() { return 0; } + virtual DDPhysicalThread* CreatePhysicalThread() { return nullptr; } virtual void DestroyPhysicalThread(DDPhysicalThread *pt) {} - virtual DDLogicalThread* CreateLogicalThread(u64 ctx) { return 0; } + virtual DDLogicalThread* CreateLogicalThread(u64 ctx) { return nullptr; } virtual void DestroyLogicalThread(DDLogicalThread *lt) {} virtual void MutexInit(DDCallback *cb, DDMutex *m) {} @@ -85,7 +85,7 @@ struct DDetector { virtual void MutexBeforeUnlock(DDCallback *cb, DDMutex *m, bool wlock) {} virtual void MutexDestroy(DDCallback *cb, DDMutex *m) {} - virtual DDReport *GetReport(DDCallback *cb) { return 0; } + virtual DDReport *GetReport(DDCallback *cb) { return nullptr; } }; } // namespace __sanitizer diff --git a/lib/sanitizer_common/sanitizer_flag_parser.cc b/lib/sanitizer_common/sanitizer_flag_parser.cc index d125002daf4c..67830b2940bb 100644 --- a/lib/sanitizer_common/sanitizer_flag_parser.cc +++ b/lib/sanitizer_common/sanitizer_flag_parser.cc @@ -127,6 +127,24 @@ void FlagParser::ParseString(const char *s) { pos_ = old_pos_; } +bool FlagParser::ParseFile(const char *path, bool ignore_missing) { + static const uptr kMaxIncludeSize = 1 << 15; + char *data; + uptr data_mapped_size; + error_t err; + uptr len; + if (!ReadFileToBuffer(path, &data, &data_mapped_size, &len, + Max(kMaxIncludeSize, GetPageSizeCached()), &err)) { + if (ignore_missing) + return true; + Printf("Failed to read options from '%s': error %d\n", path, err); + return false; + } + ParseString(data); + UnmapOrDie(data, data_mapped_size); + return true; +} + bool FlagParser::run_handler(const char *name, const char *value) { for (int i = 0; i < n_flags_; ++i) { if (internal_strcmp(name, flags_[i].name) == 0) diff --git a/lib/sanitizer_common/sanitizer_flag_parser.h b/lib/sanitizer_common/sanitizer_flag_parser.h index 0ac7634cb876..2477aeddba5f 100644 --- a/lib/sanitizer_common/sanitizer_flag_parser.h +++ b/lib/sanitizer_common/sanitizer_flag_parser.h @@ -93,6 +93,7 @@ class FlagParser { void RegisterHandler(const char *name, FlagHandlerBase *handler, const char *desc); void ParseString(const char *s); + bool ParseFile(const char *path, bool ignore_missing); void PrintFlagDescriptions(); static LowLevelAllocator Alloc; diff --git a/lib/sanitizer_common/sanitizer_flags.cc b/lib/sanitizer_common/sanitizer_flags.cc index 01098a3d6b87..18b9ea3df9e1 100644 --- a/lib/sanitizer_common/sanitizer_flags.cc +++ b/lib/sanitizer_common/sanitizer_flags.cc @@ -45,34 +45,49 @@ void CommonFlags::CopyFrom(const CommonFlags &other) { internal_memcpy(this, &other, sizeof(*this)); } +// Copy the string from "s" to "out", replacing "%b" with the binary basename. +static void SubstituteBinaryName(const char *s, char *out, uptr out_size) { + char *out_end = out + out_size; + while (*s && out < out_end - 1) { + if (s[0] != '%' || s[1] != 'b') { *out++ = *s++; continue; } + const char *base = GetProcessName(); + CHECK(base); + while (*base && out < out_end - 1) + *out++ = *base++; + s += 2; // skip "%b" + } + *out = '\0'; +} + class FlagHandlerInclude : public FlagHandlerBase { - static const uptr kMaxIncludeSize = 1 << 15; FlagParser *parser_; + bool ignore_missing_; public: - explicit FlagHandlerInclude(FlagParser *parser) : parser_(parser) {} + explicit FlagHandlerInclude(FlagParser *parser, bool ignore_missing) + : parser_(parser), ignore_missing_(ignore_missing) {} bool Parse(const char *value) final { - char *data; - uptr data_mapped_size; - error_t err; - uptr len = - ReadFileToBuffer(value, &data, &data_mapped_size, - Max(kMaxIncludeSize, GetPageSizeCached()), &err); - if (!len) { - Printf("Failed to read options from '%s': error %d\n", value, err); - return false; + if (internal_strchr(value, '%')) { + char *buf = (char *)MmapOrDie(kMaxPathLength, "FlagHandlerInclude"); + SubstituteBinaryName(value, buf, kMaxPathLength); + bool res = parser_->ParseFile(buf, ignore_missing_); + UnmapOrDie(buf, kMaxPathLength); + return res; } - parser_->ParseString(data); - UnmapOrDie(data, data_mapped_size); - return true; + return parser_->ParseFile(value, ignore_missing_); } }; -void RegisterIncludeFlag(FlagParser *parser, CommonFlags *cf) { - FlagHandlerInclude *fh_include = - new (FlagParser::Alloc) FlagHandlerInclude(parser); // NOLINT +void RegisterIncludeFlags(FlagParser *parser, CommonFlags *cf) { + FlagHandlerInclude *fh_include = new (FlagParser::Alloc) // NOLINT + FlagHandlerInclude(parser, /*ignore_missing*/ false); parser->RegisterHandler("include", fh_include, "read more options from the given file"); + FlagHandlerInclude *fh_include_if_exists = new (FlagParser::Alloc) // NOLINT + FlagHandlerInclude(parser, /*ignore_missing*/ true); + parser->RegisterHandler( + "include_if_exists", fh_include_if_exists, + "read more options from the given file (if it exists)"); } void RegisterCommonFlags(FlagParser *parser, CommonFlags *cf) { @@ -81,7 +96,7 @@ void RegisterCommonFlags(FlagParser *parser, CommonFlags *cf) { #include "sanitizer_flags.inc" #undef COMMON_FLAG - RegisterIncludeFlag(parser, cf); + RegisterIncludeFlags(parser, cf); } } // namespace __sanitizer diff --git a/lib/sanitizer_common/sanitizer_flags.h b/lib/sanitizer_common/sanitizer_flags.h index fda6d710757e..33c3c4512725 100644 --- a/lib/sanitizer_common/sanitizer_flags.h +++ b/lib/sanitizer_common/sanitizer_flags.h @@ -49,7 +49,7 @@ inline void OverrideCommonFlags(const CommonFlags &cf) { class FlagParser; void RegisterCommonFlags(FlagParser *parser, CommonFlags *cf = &common_flags_dont_use); -void RegisterIncludeFlag(FlagParser *parser, CommonFlags *cf); +void RegisterIncludeFlags(FlagParser *parser, CommonFlags *cf); } // namespace __sanitizer #endif // SANITIZER_FLAGS_H diff --git a/lib/sanitizer_common/sanitizer_flags.inc b/lib/sanitizer_common/sanitizer_flags.inc index bbb39c73e2d0..c89273117a5e 100644 --- a/lib/sanitizer_common/sanitizer_flags.inc +++ b/lib/sanitizer_common/sanitizer_flags.inc @@ -10,6 +10,7 @@ // This file describes common flags available in all sanitizers. // //===----------------------------------------------------------------------===// + #ifndef COMMON_FLAG #error "Define COMMON_FLAG prior to including this file!" #endif @@ -24,7 +25,7 @@ COMMON_FLAG( "If set, use the online symbolizer from common sanitizer runtime to turn " "virtual addresses to file/line locations.") COMMON_FLAG( - const char *, external_symbolizer_path, 0, + const char *, external_symbolizer_path, nullptr, "Path to external symbolizer. If empty, the tool will search $PATH for " "the symbolizer.") COMMON_FLAG( @@ -55,6 +56,10 @@ COMMON_FLAG( "Mention name of executable when reporting error and " "append executable name to logs (as in \"log_path.exe_name.pid\").") COMMON_FLAG( + bool, log_to_syslog, SANITIZER_ANDROID || SANITIZER_MAC, + "Write all sanitizer output to syslog in addition to other means of " + "logging.") +COMMON_FLAG( int, verbosity, 0, "Verbosity level (0 - silent, 1 - a bit of output, 2+ - more output).") COMMON_FLAG(bool, detect_leaks, true, "Enable memory leak detection.") @@ -74,6 +79,10 @@ COMMON_FLAG(bool, handle_segv, SANITIZER_NEEDS_SEGV, "If set, registers the tool's custom SIGSEGV/SIGBUS handler.") COMMON_FLAG(bool, handle_abort, false, "If set, registers the tool's custom SIGABRT handler.") +COMMON_FLAG(bool, handle_sigill, false, + "If set, registers the tool's custom SIGILL handler.") +COMMON_FLAG(bool, handle_sigfpe, true, + "If set, registers the tool's custom SIGFPE handler.") COMMON_FLAG(bool, allow_user_segv_handler, false, "If set, allows user to register a SEGV handler even if the tool " "registers one.") @@ -170,6 +179,21 @@ COMMON_FLAG(bool, intercept_strspn, true, COMMON_FLAG(bool, intercept_strpbrk, true, "If set, uses custom wrappers for strpbrk function " "to find more errors.") +COMMON_FLAG(bool, intercept_memcmp, true, + "If set, uses custom wrappers for memcmp function " + "to find more errors.") +COMMON_FLAG(bool, strict_memcmp, true, + "If true, assume that memcmp(p1, p2, n) always reads n bytes before " + "comparing p1 and p2.") COMMON_FLAG(bool, decorate_proc_maps, false, "If set, decorate sanitizer " "mappings in /proc/self/maps with " "user-readable names") +COMMON_FLAG(int, exitcode, 1, "Override the program exit status if the tool " + "found an error") +COMMON_FLAG( + bool, abort_on_error, SANITIZER_MAC, + "If set, the tool calls abort() instead of _exit() after printing the " + "error report.") +COMMON_FLAG(bool, suppress_equal_pcs, true, + "Deduplicate multiple reports for single source location in " + "halt_on_error=false mode (asan only).") diff --git a/lib/sanitizer_common/sanitizer_interface_internal.h b/lib/sanitizer_common/sanitizer_interface_internal.h index 94d9f4e9524a..b11ae30106f7 100644 --- a/lib/sanitizer_common/sanitizer_interface_internal.h +++ b/lib/sanitizer_common/sanitizer_interface_internal.h @@ -53,6 +53,9 @@ extern "C" { SANITIZER_INTERFACE_ATTRIBUTE int __sanitizer_verify_contiguous_container(const void *beg, const void *mid, const void *end); -} // extern "C" + SANITIZER_INTERFACE_ATTRIBUTE + const void *__sanitizer_contiguous_container_find_bad_address( + const void *beg, const void *mid, const void *end); + } // extern "C" #endif // SANITIZER_INTERFACE_INTERNAL_H diff --git a/lib/sanitizer_common/sanitizer_internal_defs.h b/lib/sanitizer_common/sanitizer_internal_defs.h index b76c6023aba2..e83eed0830d8 100644 --- a/lib/sanitizer_common/sanitizer_internal_defs.h +++ b/lib/sanitizer_common/sanitizer_internal_defs.h @@ -117,7 +117,10 @@ using namespace __sanitizer; // NOLINT // Common defs. #define INLINE inline #define INTERFACE_ATTRIBUTE SANITIZER_INTERFACE_ATTRIBUTE -#define WEAK SANITIZER_WEAK_ATTRIBUTE +#define SANITIZER_WEAK_DEFAULT_IMPL \ + extern "C" SANITIZER_INTERFACE_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE NOINLINE +#define SANITIZER_WEAK_CXX_DEFAULT_IMPL \ + extern "C++" SANITIZER_INTERFACE_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE NOINLINE // Platform-specific defs. #if defined(_MSC_VER) @@ -129,7 +132,6 @@ using namespace __sanitizer; // NOLINT # define NOINLINE __declspec(noinline) # define NORETURN __declspec(noreturn) # define THREADLOCAL __declspec(thread) -# define NOTHROW # define LIKELY(x) (x) # define UNLIKELY(x) (x) # define PREFETCH(x) /* _mm_prefetch(x, _MM_HINT_NTA) */ @@ -143,7 +145,6 @@ using namespace __sanitizer; // NOLINT # define NOINLINE __attribute__((noinline)) # define NORETURN __attribute__((noreturn)) # define THREADLOCAL __thread -# define NOTHROW throw() # define LIKELY(x) __builtin_expect(!!(x), 1) # define UNLIKELY(x) __builtin_expect(!!(x), 0) # if defined(__i386__) || defined(__x86_64__) @@ -162,6 +163,12 @@ using namespace __sanitizer; // NOLINT # define USED #endif +#if !defined(_MSC_VER) || defined(__clang__) || MSC_PREREQ(1900) +# define NOEXCEPT noexcept +#else +# define NOEXCEPT throw() +#endif + // Unaligned versions of basic types. typedef ALIGNED(1) u16 uu16; typedef ALIGNED(1) u32 uu32; diff --git a/lib/sanitizer_common/sanitizer_lfstack.h b/lib/sanitizer_common/sanitizer_lfstack.h index 088413908087..879cc80921c7 100644 --- a/lib/sanitizer_common/sanitizer_lfstack.h +++ b/lib/sanitizer_common/sanitizer_lfstack.h @@ -49,8 +49,8 @@ struct LFStack { u64 cmp = atomic_load(&head_, memory_order_acquire); for (;;) { T *cur = (T*)(uptr)(cmp & kPtrMask); - if (cur == 0) - return 0; + if (!cur) + return nullptr; T *nxt = cur->next; u64 cnt = (cmp & kCounterMask); u64 xch = (u64)(uptr)nxt | cnt; @@ -68,6 +68,6 @@ struct LFStack { atomic_uint64_t head_; }; -} // namespace __sanitizer +} // namespace __sanitizer -#endif // #ifndef SANITIZER_LFSTACK_H +#endif // SANITIZER_LFSTACK_H diff --git a/lib/sanitizer_common/sanitizer_libc.cc b/lib/sanitizer_common/sanitizer_libc.cc index cb162a4c4984..cf31e689653f 100644 --- a/lib/sanitizer_common/sanitizer_libc.cc +++ b/lib/sanitizer_common/sanitizer_libc.cc @@ -10,6 +10,7 @@ // This file is shared between AddressSanitizer and ThreadSanitizer // run-time libraries. See sanitizer_libc.h for details. //===----------------------------------------------------------------------===// + #include "sanitizer_allocator_internal.h" #include "sanitizer_common.h" #include "sanitizer_libc.h" @@ -17,7 +18,7 @@ namespace __sanitizer { s64 internal_atoll(const char *nptr) { - return internal_simple_strtoll(nptr, (char**)0, 10); + return internal_simple_strtoll(nptr, nullptr, 10); } void *internal_memchr(const void *s, int c, uptr n) { @@ -25,7 +26,7 @@ void *internal_memchr(const void *s, int c, uptr n) { for (uptr i = 0; i < n; ++i, ++t) if (*t == c) return reinterpret_cast<void *>(const_cast<char *>(t)); - return 0; + return nullptr; } void *internal_memrchr(const void *s, int c, uptr n) { @@ -77,7 +78,8 @@ void internal_bzero_aligned16(void *s, uptr n) { CHECK_EQ((reinterpret_cast<uptr>(s) | n) & 15, 0); for (S16 *p = reinterpret_cast<S16*>(s), *end = p + n / 16; p < end; p++) { p->a = p->b = 0; - SanitizerBreakOptimization(0); // Make sure this does not become memset. + // Make sure this does not become memset. + SanitizerBreakOptimization(nullptr); } } @@ -96,7 +98,7 @@ void *internal_memset(void* s, int c, uptr n) { uptr internal_strcspn(const char *s, const char *reject) { uptr i; for (i = 0; s[i]; i++) { - if (internal_strchr(reject, s[i]) != 0) + if (internal_strchr(reject, s[i])) return i; } return i; @@ -147,7 +149,7 @@ char* internal_strchr(const char *s, int c) { if (*s == (char)c) return const_cast<char *>(s); if (*s == 0) - return 0; + return nullptr; s++; } } @@ -160,7 +162,7 @@ char *internal_strchrnul(const char *s, int c) { } char *internal_strrchr(const char *s, int c) { - const char *res = 0; + const char *res = nullptr; for (uptr i = 0; s[i]; i++) { if (s[i] == c) res = s + i; } @@ -173,6 +175,19 @@ uptr internal_strlen(const char *s) { return i; } +uptr internal_strlcat(char *dst, const char *src, uptr maxlen) { + const uptr srclen = internal_strlen(src); + const uptr dstlen = internal_strnlen(dst, maxlen); + if (dstlen == maxlen) return maxlen + srclen; + if (srclen < maxlen - dstlen) { + internal_memmove(dst + dstlen, src, srclen + 1); + } else { + internal_memmove(dst + dstlen, src, maxlen - dstlen - 1); + dst[maxlen - 1] = '\0'; + } + return dstlen + srclen; +} + char *internal_strncat(char *dst, const char *src, uptr n) { uptr len = internal_strlen(dst); uptr i; @@ -182,6 +197,17 @@ char *internal_strncat(char *dst, const char *src, uptr n) { return dst; } +uptr internal_strlcpy(char *dst, const char *src, uptr maxlen) { + const uptr srclen = internal_strlen(src); + if (srclen < maxlen) { + internal_memmove(dst, src, srclen + 1); + } else if (maxlen != 0) { + internal_memmove(dst, src, maxlen - 1); + dst[maxlen - 1] = '\0'; + } + return srclen; +} + char *internal_strncpy(char *dst, const char *src, uptr n) { uptr i; for (i = 0; i < n && src[i]; i++) @@ -200,12 +226,12 @@ char *internal_strstr(const char *haystack, const char *needle) { // This is O(N^2), but we are not using it in hot places. uptr len1 = internal_strlen(haystack); uptr len2 = internal_strlen(needle); - if (len1 < len2) return 0; + if (len1 < len2) return nullptr; for (uptr pos = 0; pos <= len1 - len2; pos++) { if (internal_memcmp(haystack + pos, needle, len2) == 0) return const_cast<char *>(haystack) + pos; } - return 0; + return nullptr; } s64 internal_simple_strtoll(const char *nptr, char **endptr, int base) { @@ -229,7 +255,7 @@ s64 internal_simple_strtoll(const char *nptr, char **endptr, int base) { have_digits = true; nptr++; } - if (endptr != 0) { + if (endptr) { *endptr = (have_digits) ? const_cast<char *>(nptr) : old_nptr; } if (sgn > 0) { @@ -258,4 +284,4 @@ bool mem_is_zero(const char *beg, uptr size) { return all == 0; } -} // namespace __sanitizer +} // namespace __sanitizer diff --git a/lib/sanitizer_common/sanitizer_libc.h b/lib/sanitizer_common/sanitizer_libc.h index ae4e938f4af9..df28677c6700 100644 --- a/lib/sanitizer_common/sanitizer_libc.h +++ b/lib/sanitizer_common/sanitizer_libc.h @@ -13,6 +13,7 @@ // functions are intercepted. Instead, we implement a tiny subset of libc here. // FIXME: Some of functions declared in this file are in fact POSIX, not libc. //===----------------------------------------------------------------------===// + #ifndef SANITIZER_LIBC_H #define SANITIZER_LIBC_H @@ -42,8 +43,10 @@ uptr internal_strcspn(const char *s, const char *reject); char *internal_strdup(const char *s); char *internal_strndup(const char *s, uptr n); uptr internal_strlen(const char *s); +uptr internal_strlcat(char *dst, const char *src, uptr maxlen); char *internal_strncat(char *dst, const char *src, uptr n); int internal_strncmp(const char *s1, const char *s2, uptr n); +uptr internal_strlcpy(char *dst, const char *src, uptr maxlen); char *internal_strncpy(char *dst, const char *src, uptr n); uptr internal_strnlen(const char *s, uptr maxlen); char *internal_strrchr(const char *s, int c); @@ -75,8 +78,8 @@ uptr internal_getppid(); uptr internal_sched_yield(); // Error handling -bool internal_iserror(uptr retval, int *rverrno = 0); +bool internal_iserror(uptr retval, int *rverrno = nullptr); -} // namespace __sanitizer +} // namespace __sanitizer -#endif // SANITIZER_LIBC_H +#endif // SANITIZER_LIBC_H diff --git a/lib/sanitizer_common/sanitizer_libignore.cc b/lib/sanitizer_common/sanitizer_libignore.cc index 8c4aeffda45e..545393966b38 100644 --- a/lib/sanitizer_common/sanitizer_libignore.cc +++ b/lib/sanitizer_common/sanitizer_libignore.cc @@ -8,7 +8,8 @@ //===----------------------------------------------------------------------===// #include "sanitizer_platform.h" -#if SANITIZER_FREEBSD || SANITIZER_LINUX + +#if SANITIZER_FREEBSD || SANITIZER_LINUX || SANITIZER_MAC #include "sanitizer_libignore.h" #include "sanitizer_flags.h" @@ -38,11 +39,11 @@ void LibIgnore::OnLibraryLoaded(const char *name) { BlockingMutexLock lock(&mutex_); // Try to match suppressions with symlink target. InternalScopedString buf(kMaxPathLength); - if (name != 0 && internal_readlink(name, buf.data(), buf.size() - 1) > 0 && + if (name && internal_readlink(name, buf.data(), buf.size() - 1) > 0 && buf[0]) { for (uptr i = 0; i < count_; i++) { Lib *lib = &libs_[i]; - if (!lib->loaded && lib->real_name == 0 && + if (!lib->loaded && (!lib->real_name) && TemplateMatch(lib->templ, name)) lib->real_name = internal_strdup(buf.data()); } @@ -60,7 +61,7 @@ void LibIgnore::OnLibraryLoaded(const char *name) { if ((prot & MemoryMappingLayout::kProtectionExecute) == 0) continue; if (TemplateMatch(lib->templ, module.data()) || - (lib->real_name != 0 && + (lib->real_name && internal_strcmp(lib->real_name, module.data()) == 0)) { if (loaded) { Report("%s: called_from_lib suppression '%s' is matched against" @@ -93,9 +94,9 @@ void LibIgnore::OnLibraryLoaded(const char *name) { } void LibIgnore::OnLibraryUnloaded() { - OnLibraryLoaded(0); + OnLibraryLoaded(nullptr); } -} // namespace __sanitizer +} // namespace __sanitizer -#endif // #if SANITIZER_FREEBSD || SANITIZER_LINUX +#endif // #if SANITIZER_FREEBSD || SANITIZER_LINUX diff --git a/lib/sanitizer_common/sanitizer_linux.cc b/lib/sanitizer_common/sanitizer_linux.cc index 98e5d122a0f9..cba38c8c3057 100644 --- a/lib/sanitizer_common/sanitizer_linux.cc +++ b/lib/sanitizer_common/sanitizer_linux.cc @@ -13,9 +13,9 @@ //===----------------------------------------------------------------------===// #include "sanitizer_platform.h" + #if SANITIZER_FREEBSD || SANITIZER_LINUX -#include "sanitizer_allocator_internal.h" #include "sanitizer_common.h" #include "sanitizer_flags.h" #include "sanitizer_internal_defs.h" @@ -74,11 +74,6 @@ extern char **environ; // provided by crt1 #include <sys/signal.h> #endif -#if SANITIZER_ANDROID -#include <android/log.h> -#include <sys/system_properties.h> -#endif - #if SANITIZER_LINUX // <linux/time.h> struct kernel_timeval { @@ -94,7 +89,8 @@ const int FUTEX_WAKE = 1; // Are we using 32-bit or 64-bit Linux syscalls? // x32 (which defines __x86_64__) has SANITIZER_WORDSIZE == 32 // but it still needs to use 64-bit syscalls. -#if SANITIZER_LINUX && (defined(__x86_64__) || SANITIZER_WORDSIZE == 64) +#if SANITIZER_LINUX && (defined(__x86_64__) || defined(__powerpc64__) || \ + SANITIZER_WORDSIZE == 64) # define SANITIZER_LINUX_USES_64BIT_SYSCALLS 1 #else # define SANITIZER_LINUX_USES_64BIT_SYSCALLS 0 @@ -104,6 +100,8 @@ namespace __sanitizer { #if SANITIZER_LINUX && defined(__x86_64__) #include "sanitizer_syscall_linux_x86_64.inc" +#elif SANITIZER_LINUX && defined(__aarch64__) +#include "sanitizer_syscall_linux_aarch64.inc" #else #include "sanitizer_syscall_generic.inc" #endif @@ -375,23 +373,23 @@ const char *GetEnv(const char *name) { if (!inited) { inited = true; uptr environ_size; - len = ReadFileToBuffer("/proc/self/environ", - &environ, &environ_size, 1 << 26); + if (!ReadFileToBuffer("/proc/self/environ", &environ, &environ_size, &len)) + environ = nullptr; } - if (!environ || len == 0) return 0; + if (!environ || len == 0) return nullptr; uptr namelen = internal_strlen(name); const char *p = environ; while (*p != '\0') { // will happen at the \0\0 that terminates the buffer // proc file has the format NAME=value\0NAME=value\0NAME=value\0... const char* endp = (char*)internal_memchr(p, '\0', len - (p - environ)); - if (endp == 0) // this entry isn't NUL terminated - return 0; + if (!endp) // this entry isn't NUL terminated + return nullptr; else if (!internal_memcmp(p, name, namelen) && p[namelen] == '=') // Match. return p + namelen + 1; // point after = p = endp + 1; } - return 0; // Not found. + return nullptr; // Not found. #else #error "Unsupported platform" #endif @@ -405,9 +403,13 @@ extern "C" { static void ReadNullSepFileToArray(const char *path, char ***arr, int arr_size) { char *buff; - uptr buff_size = 0; + uptr buff_size; + uptr buff_len; *arr = (char **)MmapOrDie(arr_size * sizeof(char *), "NullSepFileArray"); - ReadFileToBuffer(path, &buff, &buff_size, 1024 * 1024); + if (!ReadFileToBuffer(path, &buff, &buff_size, &buff_len, 1024 * 1024)) { + (*arr)[0] = nullptr; + return; + } (*arr)[0] = buff; int count, i; for (count = 1, i = 1; ; i++) { @@ -418,7 +420,7 @@ static void ReadNullSepFileToArray(const char *path, char ***arr, count++; } } - (*arr)[count] = 0; + (*arr)[count] = nullptr; } #endif @@ -496,7 +498,7 @@ void BlockingMutex::CheckLocked() { // Note that getdents64 uses a different structure format. We only provide the // 32-bit syscall here. struct linux_dirent { -#if SANITIZER_X32 +#if SANITIZER_X32 || defined(__aarch64__) u64 d_ino; u64 d_off; #else @@ -504,6 +506,9 @@ struct linux_dirent { unsigned long d_off; #endif unsigned short d_reclen; +#ifdef __aarch64__ + unsigned char d_type; +#endif char d_name[256]; }; @@ -585,8 +590,8 @@ int internal_sigaction_norestorer(int signum, const void *act, void *oldact) { } uptr result = internal_syscall(SYSCALL(rt_sigaction), (uptr)signum, - (uptr)(u_act ? &k_act : NULL), - (uptr)(u_oldact ? &k_oldact : NULL), + (uptr)(u_act ? &k_act : nullptr), + (uptr)(u_oldact ? &k_oldact : nullptr), (uptr)sizeof(__sanitizer_kernel_sigset_t)); if ((result == 0) && u_oldact) { @@ -732,6 +737,21 @@ uptr ReadBinaryName(/*out*/char *buf, uptr buf_len) { return module_name_len; } +uptr ReadLongProcessName(/*out*/ char *buf, uptr buf_len) { +#if SANITIZER_LINUX + char *tmpbuf; + uptr tmpsize; + uptr tmplen; + if (ReadFileToBuffer("/proc/self/cmdline", &tmpbuf, &tmpsize, &tmplen, + 1024 * 1024)) { + internal_strncpy(buf, tmpbuf, buf_len); + UnmapOrDie(tmpbuf, tmpsize); + return internal_strlen(buf); + } +#endif + return ReadBinaryName(buf, buf_len); +} + // Match full names of the form /path/to/base_name{-,.}* bool LibraryNameIs(const char *full_name, const char *base_name) { const char *name = full_name; @@ -913,41 +933,142 @@ uptr internal_clone(int (*fn)(void *), void *child_stack, int flags, void *arg, : "memory", "$29" ); return res; } -#endif // defined(__x86_64__) && SANITIZER_LINUX +#elif defined(__aarch64__) +uptr internal_clone(int (*fn)(void *), void *child_stack, int flags, void *arg, + int *parent_tidptr, void *newtls, int *child_tidptr) { + long long res; + if (!fn || !child_stack) + return -EINVAL; + CHECK_EQ(0, (uptr)child_stack % 16); + child_stack = (char *)child_stack - 2 * sizeof(unsigned long long); + ((unsigned long long *)child_stack)[0] = (uptr)fn; + ((unsigned long long *)child_stack)[1] = (uptr)arg; -#if SANITIZER_ANDROID -static atomic_uint8_t android_log_initialized; + register int (*__fn)(void *) __asm__("x0") = fn; + register void *__stack __asm__("x1") = child_stack; + register int __flags __asm__("x2") = flags; + register void *__arg __asm__("x3") = arg; + register int *__ptid __asm__("x4") = parent_tidptr; + register void *__tls __asm__("x5") = newtls; + register int *__ctid __asm__("x6") = child_tidptr; -void AndroidLogInit() { - atomic_store(&android_log_initialized, 1, memory_order_release); -} -// This thing is not, strictly speaking, async signal safe, but it does not seem -// to cause any issues. Alternative is writing to log devices directly, but -// their location and message format might change in the future, so we'd really -// like to avoid that. -void AndroidLogWrite(const char *buffer) { - if (!atomic_load(&android_log_initialized, memory_order_acquire)) - return; + __asm__ __volatile__( + "mov x0,x2\n" /* flags */ + "mov x2,x4\n" /* ptid */ + "mov x3,x5\n" /* tls */ + "mov x4,x6\n" /* ctid */ + "mov x8,%9\n" /* clone */ - char *copy = internal_strdup(buffer); - char *p = copy; - char *q; - // __android_log_write has an implicit message length limit. - // Print one line at a time. - do { - q = internal_strchr(p, '\n'); - if (q) *q = '\0'; - __android_log_write(ANDROID_LOG_INFO, NULL, p); - if (q) p = q + 1; - } while (q); - InternalFree(copy); + "svc 0x0\n" + + /* if (%r0 != 0) + * return %r0; + */ + "cmp x0, #0\n" + "bne 1f\n" + + /* In the child, now. Call "fn(arg)". */ + "ldp x1, x0, [sp], #16\n" + "blr x1\n" + + /* Call _exit(%r0). */ + "mov x8, %10\n" + "svc 0x0\n" + "1:\n" + + : "=r" (res) + : "i"(-EINVAL), + "r"(__fn), "r"(__stack), "r"(__flags), "r"(__arg), + "r"(__ptid), "r"(__tls), "r"(__ctid), + "i"(__NR_clone), "i"(__NR_exit) + : "x30", "memory"); + return res; } +#elif defined(__powerpc64__) +uptr internal_clone(int (*fn)(void *), void *child_stack, int flags, void *arg, + int *parent_tidptr, void *newtls, int *child_tidptr) { + long long res; +/* Stack frame offsets. */ +#if _CALL_ELF != 2 +#define FRAME_MIN_SIZE 112 +#define FRAME_TOC_SAVE 40 +#else +#define FRAME_MIN_SIZE 32 +#define FRAME_TOC_SAVE 24 +#endif + if (!fn || !child_stack) + return -EINVAL; + CHECK_EQ(0, (uptr)child_stack % 16); + child_stack = (char *)child_stack - 2 * sizeof(unsigned long long); + ((unsigned long long *)child_stack)[0] = (uptr)fn; + ((unsigned long long *)child_stack)[1] = (uptr)arg; -void GetExtraActivationFlags(char *buf, uptr size) { - CHECK(size > PROP_VALUE_MAX); - __system_property_get("asan.options", buf); + register int (*__fn)(void *) __asm__("r3") = fn; + register void *__cstack __asm__("r4") = child_stack; + register int __flags __asm__("r5") = flags; + register void * __arg __asm__("r6") = arg; + register int * __ptidptr __asm__("r7") = parent_tidptr; + register void * __newtls __asm__("r8") = newtls; + register int * __ctidptr __asm__("r9") = child_tidptr; + + __asm__ __volatile__( + /* fn, arg, child_stack are saved acrVoss the syscall */ + "mr 28, %5\n\t" + "mr 29, %6\n\t" + "mr 27, %8\n\t" + + /* syscall + r3 == flags + r4 == child_stack + r5 == parent_tidptr + r6 == newtls + r7 == child_tidptr */ + "mr 3, %7\n\t" + "mr 5, %9\n\t" + "mr 6, %10\n\t" + "mr 7, %11\n\t" + "li 0, %3\n\t" + "sc\n\t" + + /* Test if syscall was successful */ + "cmpdi cr1, 3, 0\n\t" + "crandc cr1*4+eq, cr1*4+eq, cr0*4+so\n\t" + "bne- cr1, 1f\n\t" + + /* Do the function call */ + "std 2, %13(1)\n\t" +#if _CALL_ELF != 2 + "ld 0, 0(28)\n\t" + "ld 2, 8(28)\n\t" + "mtctr 0\n\t" +#else + "mr 12, 28\n\t" + "mtctr 12\n\t" +#endif + "mr 3, 27\n\t" + "bctrl\n\t" + "ld 2, %13(1)\n\t" + + /* Call _exit(r3) */ + "li 0, %4\n\t" + "sc\n\t" + + /* Return to parent */ + "1:\n\t" + "mr %0, 3\n\t" + : "=r" (res) + : "0" (-1), "i" (EINVAL), + "i" (__NR_clone), "i" (__NR_exit), + "r" (__fn), "r" (__cstack), "r" (__flags), + "r" (__arg), "r" (__ptidptr), "r" (__newtls), + "r" (__ctidptr), "i" (FRAME_MIN_SIZE), "i" (FRAME_TOC_SAVE) + : "cr0", "cr1", "memory", "ctr", + "r0", "r29", "r27", "r28"); + return res; } +#endif // defined(__x86_64__) && SANITIZER_LINUX +#if SANITIZER_ANDROID #if __ANDROID_API__ < 21 extern "C" __attribute__((weak)) int dl_iterate_phdr( int (*)(struct dl_phdr_info *, size_t, void *), void *); @@ -993,6 +1114,10 @@ AndroidApiLevel AndroidGetApiLevel() { bool IsDeadlySignal(int signum) { if (common_flags()->handle_abort && signum == SIGABRT) return true; + if (common_flags()->handle_sigill && signum == SIGILL) + return true; + if (common_flags()->handle_sigfpe && signum == SIGFPE) + return true; return (signum == SIGSEGV || signum == SIGBUS) && common_flags()->handle_segv; } @@ -1008,13 +1133,13 @@ void *internal_start_thread(void(*func)(void *arg), void *arg) { #endif internal_sigprocmask(SIG_SETMASK, &set, &old); void *th; - real_pthread_create(&th, 0, (void*(*)(void *arg))func, arg); - internal_sigprocmask(SIG_SETMASK, &old, 0); + real_pthread_create(&th, nullptr, (void*(*)(void *arg))func, arg); + internal_sigprocmask(SIG_SETMASK, &old, nullptr); return th; } void internal_join_thread(void *th) { - real_pthread_join(th, 0); + real_pthread_join(th, nullptr); } #else void *internal_start_thread(void (*func)(void *), void *arg) { return 0; } @@ -1094,6 +1219,14 @@ void GetPcSpBp(void *context, uptr *pc, uptr *sp, uptr *bp) { #endif } -} // namespace __sanitizer +void DisableReexec() { + // No need to re-exec on Linux. +} + +void MaybeReexec() { + // No need to re-exec on Linux. +} + +} // namespace __sanitizer -#endif // SANITIZER_FREEBSD || SANITIZER_LINUX +#endif // SANITIZER_FREEBSD || SANITIZER_LINUX diff --git a/lib/sanitizer_common/sanitizer_linux.h b/lib/sanitizer_common/sanitizer_linux.h index e9fc4ad448d8..77bfbd156815 100644 --- a/lib/sanitizer_common/sanitizer_linux.h +++ b/lib/sanitizer_common/sanitizer_linux.h @@ -44,7 +44,8 @@ uptr internal_prctl(int option, uptr arg2, uptr arg3, uptr arg4, uptr arg5); // internal_sigaction instead. int internal_sigaction_norestorer(int signum, const void *act, void *oldact); void internal_sigdelset(__sanitizer_sigset_t *set, int signum); -#if defined(__x86_64__) || defined(__mips__) +#if defined(__x86_64__) || defined(__mips__) || defined(__aarch64__) \ + || defined(__powerpc64__) uptr internal_clone(int (*fn)(void *), void *child_stack, int flags, void *arg, int *parent_tidptr, void *newtls, int *child_tidptr); #endif diff --git a/lib/sanitizer_common/sanitizer_linux_libcdep.cc b/lib/sanitizer_common/sanitizer_linux_libcdep.cc index 39eb1d216b2e..0bb66c9d634e 100644 --- a/lib/sanitizer_common/sanitizer_linux_libcdep.cc +++ b/lib/sanitizer_common/sanitizer_linux_libcdep.cc @@ -13,8 +13,10 @@ //===----------------------------------------------------------------------===// #include "sanitizer_platform.h" + #if SANITIZER_FREEBSD || SANITIZER_LINUX +#include "sanitizer_allocator_internal.h" #include "sanitizer_atomic.h" #include "sanitizer_common.h" #include "sanitizer_flags.h" @@ -47,6 +49,12 @@ #include <android/api-level.h> #endif +#if SANITIZER_ANDROID && __ANDROID_API__ < 21 +#include <android/log.h> +#else +#include <syslog.h> +#endif + #if !SANITIZER_ANDROID #include <elf.h> #include <unistd.h> @@ -54,20 +62,6 @@ namespace __sanitizer { -// This function is defined elsewhere if we intercepted pthread_attr_getstack. -extern "C" { -SANITIZER_WEAK_ATTRIBUTE int -real_pthread_attr_getstack(void *attr, void **addr, size_t *size); -} // extern "C" - -static int my_pthread_attr_getstack(void *attr, void **addr, size_t *size) { -#if !SANITIZER_GO - if (&real_pthread_attr_getstack) - return real_pthread_attr_getstack((pthread_attr_t *)attr, addr, size); -#endif - return pthread_attr_getstack((pthread_attr_t *)attr, addr, size); -} - SANITIZER_WEAK_ATTRIBUTE int real_sigaction(int signum, const void *act, void *oldact); @@ -93,7 +87,8 @@ void GetThreadStackTopAndBottom(bool at_initialization, uptr *stack_top, MemoryMappingLayout proc_maps(/*cache_enabled*/true); uptr start, end, offset; uptr prev_end = 0; - while (proc_maps.Next(&start, &end, &offset, 0, 0, /* protection */0)) { + while (proc_maps.Next(&start, &end, &offset, nullptr, 0, + /* protection */nullptr)) { if ((uptr)&rl < end) break; prev_end = end; @@ -118,8 +113,8 @@ void GetThreadStackTopAndBottom(bool at_initialization, uptr *stack_top, pthread_attr_init(&attr); CHECK_EQ(pthread_getattr_np(pthread_self(), &attr), 0); uptr stacksize = 0; - void *stackaddr = 0; - my_pthread_attr_getstack(&attr, &stackaddr, (size_t*)&stacksize); + void *stackaddr = nullptr; + my_pthread_attr_getstack(&attr, &stackaddr, &stacksize); pthread_attr_destroy(&attr); CHECK_LE(stacksize, kMaxThreadStackSize); // Sanity check. @@ -130,7 +125,7 @@ void GetThreadStackTopAndBottom(bool at_initialization, uptr *stack_top, #if !SANITIZER_GO bool SetEnv(const char *name, const char *value) { void *f = dlsym(RTLD_NEXT, "setenv"); - if (f == 0) + if (!f) return false; typedef int(*setenv_ft)(const char *name, const char *value, int overwrite); setenv_ft setenv_f; @@ -161,7 +156,7 @@ bool SanitizerGetThreadName(char *name, int max_len) { #endif } -#if !SANITIZER_FREEBSD +#if !SANITIZER_FREEBSD && !SANITIZER_ANDROID && !SANITIZER_GO static uptr g_tls_size; #endif @@ -171,11 +166,15 @@ static uptr g_tls_size; # define DL_INTERNAL_FUNCTION #endif -#if defined(__mips__) +#if defined(__mips__) || defined(__powerpc64__) // TlsPreTcbSize includes size of struct pthread_descr and size of tcb // head structure. It lies before the static tls blocks. static uptr TlsPreTcbSize() { - const uptr kTcbHead = 16; +# if defined(__mips__) + const uptr kTcbHead = 16; // sizeof (tcbhead_t) +# elif defined(__powerpc64__) + const uptr kTcbHead = 88; // sizeof (tcbhead_t) +# endif const uptr kTlsAlign = 16; const uptr kTlsPreTcbSize = (ThreadDescriptorSize() + kTcbHead + kTlsAlign - 1) & ~(kTlsAlign - 1); @@ -187,6 +186,8 @@ static uptr TlsPreTcbSize() { void InitTlsSize() { #if !SANITIZER_FREEBSD && !SANITIZER_ANDROID && !SANITIZER_GO +// all current supported platforms have 16 bytes stack alignment + const size_t kStackAlign = 16; typedef void (*get_tls_func)(size_t*, size_t*) DL_INTERNAL_FUNCTION; get_tls_func get_tls; void *get_tls_static_info_ptr = dlsym(RTLD_NEXT, "_dl_get_tls_static_info"); @@ -197,13 +198,16 @@ void InitTlsSize() { size_t tls_size = 0; size_t tls_align = 0; get_tls(&tls_size, &tls_align); - g_tls_size = tls_size; -#endif // !SANITIZER_FREEBSD && !SANITIZER_ANDROID + if (tls_align < kStackAlign) + tls_align = kStackAlign; + g_tls_size = RoundUpTo(tls_size, tls_align); +#endif // !SANITIZER_FREEBSD && !SANITIZER_ANDROID && !SANITIZER_GO } -#if (defined(__x86_64__) || defined(__i386__) || defined(__mips__)) \ - && SANITIZER_LINUX -// sizeof(struct thread) from glibc. +#if (defined(__x86_64__) || defined(__i386__) || defined(__mips__) \ + || defined(__aarch64__) || defined(__powerpc64__)) \ + && SANITIZER_LINUX && !SANITIZER_ANDROID +// sizeof(struct pthread) from glibc. static atomic_uintptr_t kThreadDescriptorSize; uptr ThreadDescriptorSize() { @@ -218,7 +222,7 @@ uptr ThreadDescriptorSize() { char *end; int minor = internal_simple_strtoll(buf + 8, &end, 10); if (end != buf + 8 && (*end == '\0' || *end == '.')) { - /* sizeof(struct thread) values from various glibc versions. */ + /* sizeof(struct pthread) values from various glibc versions. */ if (SANITIZER_X32) val = 1728; // Assume only one particular version for x32. else if (minor <= 3) @@ -249,6 +253,15 @@ uptr ThreadDescriptorSize() { if (val) atomic_store(&kThreadDescriptorSize, val, memory_order_relaxed); return val; +#elif defined(__aarch64__) + // The sizeof (struct pthread) is the same from GLIBC 2.17 to 2.22. + val = 1776; + atomic_store(&kThreadDescriptorSize, val, memory_order_relaxed); + return val; +#elif defined(__powerpc64__) + val = 1776; // from glibc.ppc64le 2.20-8.fc21 + atomic_store(&kThreadDescriptorSize, val, memory_order_relaxed); + return val; #endif return 0; } @@ -278,6 +291,17 @@ uptr ThreadSelf() { rdhwr %0,$29;\ .set pop" : "=r" (thread_pointer)); descr_addr = thread_pointer - kTlsTcbOffset - TlsPreTcbSize(); +# elif defined(__aarch64__) + descr_addr = reinterpret_cast<uptr>(__builtin_thread_pointer()); +# elif defined(__powerpc64__) + // PPC64LE uses TLS variant I. The thread pointer (in GPR 13) + // points to the end of the TCB + 0x7000. The pthread_descr structure is + // immediately in front of the TCB. TlsPreTcbSize() includes the size of the + // TCB and the size of pthread_descr. + const uptr kTlsTcbOffset = 0x7000; + uptr thread_pointer; + asm("addi %0,13,%1" : "=r"(thread_pointer) : "I"(-kTlsTcbOffset)); + descr_addr = thread_pointer - TlsPreTcbSize(); # else # error "unsupported CPU arch" # endif @@ -307,13 +331,13 @@ uptr ThreadSelf() { #if !SANITIZER_GO static void GetTls(uptr *addr, uptr *size) { -#if SANITIZER_LINUX +#if SANITIZER_LINUX && !SANITIZER_ANDROID # if defined(__x86_64__) || defined(__i386__) *addr = ThreadSelf(); *size = GetTlsSize(); *addr -= *size; *addr += ThreadDescriptorSize(); -# elif defined(__mips__) +# elif defined(__mips__) || defined(__aarch64__) || defined(__powerpc64__) *addr = ThreadSelf(); *size = GetTlsSize(); # else @@ -333,6 +357,9 @@ static void GetTls(uptr *addr, uptr *size) { *addr = (uptr) dtv[2]; *size = (*addr == 0) ? 0 : ((uptr) segbase[0] - (uptr) dtv[2]); } +#elif SANITIZER_ANDROID + *addr = 0; + *size = 0; #else # error "Unknown OS" #endif @@ -341,7 +368,7 @@ static void GetTls(uptr *addr, uptr *size) { #if !SANITIZER_GO uptr GetTlsSize() { -#if SANITIZER_FREEBSD +#if SANITIZER_FREEBSD || SANITIZER_ANDROID uptr addr, size; GetTls(&addr, &size); return size; @@ -376,31 +403,6 @@ void GetThreadStackAndTls(bool main, uptr *stk_addr, uptr *stk_size, #endif } -void AdjustStackSize(void *attr_) { - pthread_attr_t *attr = (pthread_attr_t *)attr_; - uptr stackaddr = 0; - size_t stacksize = 0; - my_pthread_attr_getstack(attr, (void**)&stackaddr, &stacksize); - // GLibC will return (0 - stacksize) as the stack address in the case when - // stacksize is set, but stackaddr is not. - bool stack_set = (stackaddr != 0) && (stackaddr + stacksize != 0); - // We place a lot of tool data into TLS, account for that. - const uptr minstacksize = GetTlsSize() + 128*1024; - if (stacksize < minstacksize) { - if (!stack_set) { - if (stacksize != 0) { - VPrintf(1, "Sanitizer: increasing stacksize %zu->%zu\n", stacksize, - minstacksize); - pthread_attr_setstacksize(attr, minstacksize); - } - } else { - Printf("Sanitizer: pre-allocated stack size is insufficient: " - "%zu < %zu\n", stacksize, minstacksize); - Printf("Sanitizer: pthread_create is likely to fail.\n"); - } - } -} - # if !SANITIZER_FREEBSD typedef ElfW(Phdr) Elf_Phdr; # elif SANITIZER_WORDSIZE == 32 && __FreeBSD_version <= 902001 // v9.2 @@ -455,7 +457,7 @@ extern "C" __attribute__((weak)) int dl_iterate_phdr( uptr GetListOfModules(LoadedModule *modules, uptr max_modules, string_predicate_t filter) { -#if SANITIZER_ANDROID && __ANDROID_API__ < 21 +#if SANITIZER_ANDROID && __ANDROID_API__ <= 22 u32 api_level = AndroidGetApiLevel(); // Fall back to /proc/maps if dl_iterate_phdr is unavailable or broken. // The runtime check allows the same library to work with @@ -510,6 +512,37 @@ uptr GetRSS() { return rss * GetPageSizeCached(); } -} // namespace __sanitizer +// 64-bit Android targets don't provide the deprecated __android_log_write. +// Starting with the L release, syslog() works and is preferable to +// __android_log_write. +#if SANITIZER_LINUX + +#if SANITIZER_ANDROID +static atomic_uint8_t android_log_initialized; + +void AndroidLogInit() { + atomic_store(&android_log_initialized, 1, memory_order_release); +} + +bool ShouldLogAfterPrintf() { + return atomic_load(&android_log_initialized, memory_order_acquire); +} +#else +void AndroidLogInit() {} + +bool ShouldLogAfterPrintf() { return true; } +#endif // SANITIZER_ANDROID + +void WriteOneLineToSyslog(const char *s) { +#if SANITIZER_ANDROID &&__ANDROID_API__ < 21 + __android_log_write(ANDROID_LOG_INFO, NULL, s); +#else + syslog(LOG_INFO, "%s", s); +#endif +} + +#endif // SANITIZER_LINUX + +} // namespace __sanitizer -#endif // SANITIZER_FREEBSD || SANITIZER_LINUX +#endif // SANITIZER_FREEBSD || SANITIZER_LINUX diff --git a/lib/sanitizer_common/sanitizer_list.h b/lib/sanitizer_common/sanitizer_list.h index 6dd9c8f7bca1..adbb97dc70dc 100644 --- a/lib/sanitizer_common/sanitizer_list.h +++ b/lib/sanitizer_common/sanitizer_list.h @@ -11,6 +11,7 @@ // ThreadSanitizer, etc run-times. // //===----------------------------------------------------------------------===// + #ifndef SANITIZER_LIST_H #define SANITIZER_LIST_H @@ -29,7 +30,7 @@ struct IntrusiveList { friend class Iterator; void clear() { - first_ = last_ = 0; + first_ = last_ = nullptr; size_ = 0; } @@ -38,11 +39,11 @@ struct IntrusiveList { void push_back(Item *x) { if (empty()) { - x->next = 0; + x->next = nullptr; first_ = last_ = x; size_ = 1; } else { - x->next = 0; + x->next = nullptr; last_->next = x; last_ = x; size_++; @@ -51,7 +52,7 @@ struct IntrusiveList { void push_front(Item *x) { if (empty()) { - x->next = 0; + x->next = nullptr; first_ = last_ = x; size_ = 1; } else { @@ -64,8 +65,8 @@ struct IntrusiveList { void pop_front() { CHECK(!empty()); first_ = first_->next; - if (first_ == 0) - last_ = 0; + if (!first_) + last_ = nullptr; size_--; } @@ -125,7 +126,7 @@ struct IntrusiveList { if (current_) current_ = current_->next; return ret; } - bool hasNext() const { return current_ != 0; } + bool hasNext() const { return current_ != nullptr; } private: ListTy *list_; ItemTy *current_; @@ -140,6 +141,6 @@ struct IntrusiveList { Item *last_; }; -} // namespace __sanitizer +} // namespace __sanitizer -#endif // SANITIZER_LIST_H +#endif // SANITIZER_LIST_H diff --git a/lib/sanitizer_common/sanitizer_mac.cc b/lib/sanitizer_common/sanitizer_mac.cc index dddce1c1583c..1c96a6b95621 100644 --- a/lib/sanitizer_common/sanitizer_mac.cc +++ b/lib/sanitizer_common/sanitizer_mac.cc @@ -13,6 +13,7 @@ #include "sanitizer_platform.h" #if SANITIZER_MAC +#include "sanitizer_mac.h" // Use 64-bit inodes in file operations. ASan does not support OS X 10.5, so // the clients will most certainly use 64-bit ones as well. @@ -25,7 +26,6 @@ #include "sanitizer_flags.h" #include "sanitizer_internal_defs.h" #include "sanitizer_libc.h" -#include "sanitizer_mac.h" #include "sanitizer_placement_new.h" #include "sanitizer_platform_limits_posix.h" #include "sanitizer_procmaps.h" @@ -36,11 +36,29 @@ extern char **environ; #endif +#if defined(__has_include) && __has_include(<os/trace.h>) +#define SANITIZER_OS_TRACE 1 +#include <os/trace.h> +#else +#define SANITIZER_OS_TRACE 0 +#endif + +#if !SANITIZER_IOS +#include <crt_externs.h> // for _NSGetArgv and _NSGetEnviron +#else +extern "C" { + extern char ***_NSGetArgv(void); +} +#endif + +#include <asl.h> +#include <dlfcn.h> // for dladdr() #include <errno.h> #include <fcntl.h> #include <libkern/OSAtomic.h> #include <mach-o/dyld.h> #include <mach/mach.h> +#include <mach/vm_statistics.h> #include <pthread.h> #include <sched.h> #include <signal.h> @@ -51,6 +69,7 @@ extern char **environ; #include <sys/sysctl.h> #include <sys/types.h> #include <unistd.h> +#include <util.h> namespace __sanitizer { @@ -59,6 +78,7 @@ namespace __sanitizer { // ---------------------- sanitizer_libc.h uptr internal_mmap(void *addr, size_t length, int prot, int flags, int fd, u64 offset) { + if (fd == -1) fd = VM_MAKE_TAG(VM_MEMORY_ANALYSIS_TOOL); return (uptr)mmap(addr, length, prot, flags, fd, offset); } @@ -145,9 +165,30 @@ uptr internal_sigprocmask(int how, __sanitizer_sigset_t *set, return sigprocmask(how, set, oldset); } +// Doesn't call pthread_atfork() handlers. +extern "C" pid_t __fork(void); + int internal_fork() { - // TODO(glider): this may call user's pthread_atfork() handlers which is bad. - return fork(); + return __fork(); +} + +int internal_forkpty(int *amaster) { + int master, slave; + if (openpty(&master, &slave, nullptr, nullptr, nullptr) == -1) return -1; + int pid = __fork(); + if (pid == -1) { + close(master); + close(slave); + return -1; + } + if (pid == 0) { + close(master); + CHECK_EQ(login_tty(slave), 0); + } else { + *amaster = master; + close(slave); + } + return pid; } uptr internal_rename(const char *oldpath, const char *newpath) { @@ -178,7 +219,7 @@ void GetThreadStackTopAndBottom(bool at_initialization, uptr *stack_top, uptr stacksize = pthread_get_stacksize_np(pthread_self()); // pthread_get_stacksize_np() returns an incorrect stack size for the main // thread on Mavericks. See - // https://code.google.com/p/address-sanitizer/issues/detail?id=261 + // https://github.com/google/sanitizers/issues/261 if ((GetMacosVersion() >= MACOS_VERSION_MAVERICKS) && at_initialization && stacksize == (1 << 19)) { struct rlimit rl; @@ -241,6 +282,10 @@ uptr ReadBinaryName(/*out*/char *buf, uptr buf_len) { return 0; } +uptr ReadLongProcessName(/*out*/char *buf, uptr buf_len) { + return ReadBinaryName(buf, buf_len); +} + void ReExec() { UNIMPLEMENTED(); } @@ -365,8 +410,63 @@ uptr GetRSS() { return info.resident_size; } -void *internal_start_thread(void (*func)(void *arg), void *arg) { return 0; } -void internal_join_thread(void *th) { } +void *internal_start_thread(void(*func)(void *arg), void *arg) { + // Start the thread with signals blocked, otherwise it can steal user signals. + __sanitizer_sigset_t set, old; + internal_sigfillset(&set); + internal_sigprocmask(SIG_SETMASK, &set, &old); + pthread_t th; + pthread_create(&th, 0, (void*(*)(void *arg))func, arg); + internal_sigprocmask(SIG_SETMASK, &old, 0); + return th; +} + +void internal_join_thread(void *th) { pthread_join((pthread_t)th, 0); } + +static BlockingMutex syslog_lock(LINKER_INITIALIZED); + +void WriteOneLineToSyslog(const char *s) { + syslog_lock.CheckLocked(); + asl_log(nullptr, nullptr, ASL_LEVEL_ERR, "%s", s); +} + +void LogFullErrorReport(const char *buffer) { + // Log with os_trace. This will make it into the crash log. +#if SANITIZER_OS_TRACE + if (GetMacosVersion() >= MACOS_VERSION_YOSEMITE) { + // os_trace requires the message (format parameter) to be a string literal. + if (internal_strncmp(SanitizerToolName, "AddressSanitizer", + sizeof("AddressSanitizer") - 1) == 0) + os_trace("Address Sanitizer reported a failure."); + else if (internal_strncmp(SanitizerToolName, "UndefinedBehaviorSanitizer", + sizeof("UndefinedBehaviorSanitizer") - 1) == 0) + os_trace("Undefined Behavior Sanitizer reported a failure."); + else if (internal_strncmp(SanitizerToolName, "ThreadSanitizer", + sizeof("ThreadSanitizer") - 1) == 0) + os_trace("Thread Sanitizer reported a failure."); + else + os_trace("Sanitizer tool reported a failure."); + + if (common_flags()->log_to_syslog) + os_trace("Consult syslog for more information."); + } +#endif + + // Log to syslog. + // The logging on OS X may call pthread_create so we need the threading + // environment to be fully initialized. Also, this should never be called when + // holding the thread registry lock since that may result in a deadlock. If + // the reporting thread holds the thread registry mutex, and asl_log waits + // for GCD to dispatch a new thread, the process will deadlock, because the + // pthread_create wrapper needs to acquire the lock as well. + BlockingMutexLock l(&syslog_lock); + if (common_flags()->log_to_syslog) + WriteToSyslog(buffer); + + // Log to CrashLog. + if (common_flags()->abort_on_error) + CRSetCrashLogMessage(buffer); +} void GetPcSpBp(void *context, uptr *pc, uptr *sp, uptr *bp) { ucontext_t *ucontext = (ucontext_t*)context; @@ -395,6 +495,173 @@ void GetPcSpBp(void *context, uptr *pc, uptr *sp, uptr *bp) { # endif } +static const char kDyldInsertLibraries[] = "DYLD_INSERT_LIBRARIES"; +LowLevelAllocator allocator_for_env; + +// Change the value of the env var |name|, leaking the original value. +// If |name_value| is NULL, the variable is deleted from the environment, +// otherwise the corresponding "NAME=value" string is replaced with +// |name_value|. +void LeakyResetEnv(const char *name, const char *name_value) { + char **env = GetEnviron(); + uptr name_len = internal_strlen(name); + while (*env != 0) { + uptr len = internal_strlen(*env); + if (len > name_len) { + const char *p = *env; + if (!internal_memcmp(p, name, name_len) && p[name_len] == '=') { + // Match. + if (name_value) { + // Replace the old value with the new one. + *env = const_cast<char*>(name_value); + } else { + // Shift the subsequent pointers back. + char **del = env; + do { + del[0] = del[1]; + } while (*del++); + } + } + } + env++; + } +} + +static bool reexec_disabled = false; + +void DisableReexec() { + reexec_disabled = true; +} + +extern "C" double dyldVersionNumber; +static const double kMinDyldVersionWithAutoInterposition = 360.0; + +bool DyldNeedsEnvVariable() { + // If running on OS X 10.11+ or iOS 9.0+, dyld will interpose even if + // DYLD_INSERT_LIBRARIES is not set. However, checking OS version via + // GetMacosVersion() doesn't work for the simulator. Let's instead check + // `dyldVersionNumber`, which is exported by dyld, against a known version + // number from the first OS release where this appeared. + return dyldVersionNumber < kMinDyldVersionWithAutoInterposition; +} + +void MaybeReexec() { + if (reexec_disabled) return; + + // Make sure the dynamic runtime library is preloaded so that the + // wrappers work. If it is not, set DYLD_INSERT_LIBRARIES and re-exec + // ourselves. + Dl_info info; + CHECK(dladdr((void*)((uptr)&__sanitizer_report_error_summary), &info)); + char *dyld_insert_libraries = + const_cast<char*>(GetEnv(kDyldInsertLibraries)); + uptr old_env_len = dyld_insert_libraries ? + internal_strlen(dyld_insert_libraries) : 0; + uptr fname_len = internal_strlen(info.dli_fname); + const char *dylib_name = StripModuleName(info.dli_fname); + uptr dylib_name_len = internal_strlen(dylib_name); + + bool lib_is_in_env = dyld_insert_libraries && + internal_strstr(dyld_insert_libraries, dylib_name); + if (DyldNeedsEnvVariable() && !lib_is_in_env) { + // DYLD_INSERT_LIBRARIES is not set or does not contain the runtime + // library. + InternalScopedString program_name(1024); + uint32_t buf_size = program_name.size(); + _NSGetExecutablePath(program_name.data(), &buf_size); + char *new_env = const_cast<char*>(info.dli_fname); + if (dyld_insert_libraries) { + // Append the runtime dylib name to the existing value of + // DYLD_INSERT_LIBRARIES. + new_env = (char*)allocator_for_env.Allocate(old_env_len + fname_len + 2); + internal_strncpy(new_env, dyld_insert_libraries, old_env_len); + new_env[old_env_len] = ':'; + // Copy fname_len and add a trailing zero. + internal_strncpy(new_env + old_env_len + 1, info.dli_fname, + fname_len + 1); + // Ok to use setenv() since the wrappers don't depend on the value of + // asan_inited. + setenv(kDyldInsertLibraries, new_env, /*overwrite*/1); + } else { + // Set DYLD_INSERT_LIBRARIES equal to the runtime dylib name. + setenv(kDyldInsertLibraries, info.dli_fname, /*overwrite*/0); + } + VReport(1, "exec()-ing the program with\n"); + VReport(1, "%s=%s\n", kDyldInsertLibraries, new_env); + VReport(1, "to enable wrappers.\n"); + execv(program_name.data(), *_NSGetArgv()); + + // We get here only if execv() failed. + Report("ERROR: The process is launched without DYLD_INSERT_LIBRARIES, " + "which is required for the sanitizer to work. We tried to set the " + "environment variable and re-execute itself, but execv() failed, " + "possibly because of sandbox restrictions. Make sure to launch the " + "executable with:\n%s=%s\n", kDyldInsertLibraries, new_env); + CHECK("execv failed" && 0); + } + + if (!lib_is_in_env) + return; + + // DYLD_INSERT_LIBRARIES is set and contains the runtime library. Let's remove + // the dylib from the environment variable, because interceptors are installed + // and we don't want our children to inherit the variable. + + uptr env_name_len = internal_strlen(kDyldInsertLibraries); + // Allocate memory to hold the previous env var name, its value, the '=' + // sign and the '\0' char. + char *new_env = (char*)allocator_for_env.Allocate( + old_env_len + 2 + env_name_len); + CHECK(new_env); + internal_memset(new_env, '\0', old_env_len + 2 + env_name_len); + internal_strncpy(new_env, kDyldInsertLibraries, env_name_len); + new_env[env_name_len] = '='; + char *new_env_pos = new_env + env_name_len + 1; + + // Iterate over colon-separated pieces of |dyld_insert_libraries|. + char *piece_start = dyld_insert_libraries; + char *piece_end = NULL; + char *old_env_end = dyld_insert_libraries + old_env_len; + do { + if (piece_start[0] == ':') piece_start++; + piece_end = internal_strchr(piece_start, ':'); + if (!piece_end) piece_end = dyld_insert_libraries + old_env_len; + if ((uptr)(piece_start - dyld_insert_libraries) > old_env_len) break; + uptr piece_len = piece_end - piece_start; + + char *filename_start = + (char *)internal_memrchr(piece_start, '/', piece_len); + uptr filename_len = piece_len; + if (filename_start) { + filename_start += 1; + filename_len = piece_len - (filename_start - piece_start); + } else { + filename_start = piece_start; + } + + // If the current piece isn't the runtime library name, + // append it to new_env. + if ((dylib_name_len != filename_len) || + (internal_memcmp(filename_start, dylib_name, dylib_name_len) != 0)) { + if (new_env_pos != new_env + env_name_len + 1) { + new_env_pos[0] = ':'; + new_env_pos++; + } + internal_strncpy(new_env_pos, piece_start, piece_len); + new_env_pos += piece_len; + } + // Move on to the next piece. + piece_start = piece_end; + } while (piece_start < old_env_end); + + // Can't use setenv() here, because it requires the allocator to be + // initialized. + // FIXME: instead of filtering DYLD_INSERT_LIBRARIES here, do it in + // a separate function called after InitializeAllocator(). + if (new_env_pos == new_env + env_name_len + 1) new_env = NULL; + LeakyResetEnv(kDyldInsertLibraries, new_env); +} + } // namespace __sanitizer #endif // SANITIZER_MAC diff --git a/lib/sanitizer_common/sanitizer_mac.h b/lib/sanitizer_common/sanitizer_mac.h index 50dbe93226c2..86a9956683d0 100644 --- a/lib/sanitizer_common/sanitizer_mac.h +++ b/lib/sanitizer_common/sanitizer_mac.h @@ -13,6 +13,7 @@ #ifndef SANITIZER_MAC_H #define SANITIZER_MAC_H +#include "sanitizer_common.h" #include "sanitizer_platform.h" #if SANITIZER_MAC #include "sanitizer_posix.h" @@ -37,5 +38,16 @@ char **GetEnviron(); } // namespace __sanitizer +extern "C" { +static char __crashreporter_info_buff__[kErrorMessageBufferSize] = {}; +static const char *__crashreporter_info__ __attribute__((__used__)) = + &__crashreporter_info_buff__[0]; +asm(".desc ___crashreporter_info__, 0x10"); +} // extern "C" + +INLINE void CRSetCrashLogMessage(const char *msg) { + internal_strlcpy(__crashreporter_info_buff__, msg, + sizeof(__crashreporter_info_buff__)); } + #endif // SANITIZER_MAC #endif // SANITIZER_MAC_H diff --git a/lib/sanitizer_common/sanitizer_malloc_mac.inc b/lib/sanitizer_common/sanitizer_malloc_mac.inc new file mode 100644 index 000000000000..149857c168c6 --- /dev/null +++ b/lib/sanitizer_common/sanitizer_malloc_mac.inc @@ -0,0 +1,329 @@ +//===-- sanitizer_malloc_mac.inc --------------------------------*- C++ -*-===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// +// +// This file contains Mac-specific malloc interceptors and a custom zone +// implementation, which together replace the system allocator. +// +//===----------------------------------------------------------------------===// + +#include "sanitizer_common/sanitizer_platform.h" +#if !SANITIZER_MAC +#error "This file should only be compiled on Darwin." +#endif + +#include <AvailabilityMacros.h> +#include <CoreFoundation/CFBase.h> +#include <dlfcn.h> +#include <malloc/malloc.h> +#include <sys/mman.h> + +#include "interception/interception.h" +#include "sanitizer_common/sanitizer_mac.h" + +// Similar code is used in Google Perftools, +// https://github.com/gperftools/gperftools. + +static malloc_zone_t sanitizer_zone; + +INTERCEPTOR(malloc_zone_t *, malloc_create_zone, + vm_size_t start_size, unsigned zone_flags) { + COMMON_MALLOC_ENTER(); + uptr page_size = GetPageSizeCached(); + uptr allocated_size = RoundUpTo(sizeof(sanitizer_zone), page_size); + COMMON_MALLOC_MEMALIGN(page_size, allocated_size); + malloc_zone_t *new_zone = (malloc_zone_t *)p; + internal_memcpy(new_zone, &sanitizer_zone, sizeof(sanitizer_zone)); + new_zone->zone_name = NULL; // The name will be changed anyway. + if (GetMacosVersion() >= MACOS_VERSION_LION) { + // Prevent the client app from overwriting the zone contents. + // Library functions that need to modify the zone will set PROT_WRITE on it. + // This matches the behavior of malloc_create_zone() on OSX 10.7 and higher. + mprotect(new_zone, allocated_size, PROT_READ); + } + return new_zone; +} + +INTERCEPTOR(malloc_zone_t *, malloc_default_zone, void) { + COMMON_MALLOC_ENTER(); + return &sanitizer_zone; +} + +INTERCEPTOR(malloc_zone_t *, malloc_default_purgeable_zone, void) { + // FIXME: ASan should support purgeable allocations. + // https://github.com/google/sanitizers/issues/139 + COMMON_MALLOC_ENTER(); + return &sanitizer_zone; +} + +INTERCEPTOR(void, malloc_make_purgeable, void *ptr) { + // FIXME: ASan should support purgeable allocations. Ignoring them is fine + // for now. + COMMON_MALLOC_ENTER(); +} + +INTERCEPTOR(int, malloc_make_nonpurgeable, void *ptr) { + // FIXME: ASan should support purgeable allocations. Ignoring them is fine + // for now. + COMMON_MALLOC_ENTER(); + // Must return 0 if the contents were not purged since the last call to + // malloc_make_purgeable(). + return 0; +} + +INTERCEPTOR(void, malloc_set_zone_name, malloc_zone_t *zone, const char *name) { + COMMON_MALLOC_ENTER(); + // Allocate |sizeof(COMMON_MALLOC_ZONE_NAME "-") + internal_strlen(name)| + // bytes. + size_t buflen = + sizeof(COMMON_MALLOC_ZONE_NAME "-") + (name ? internal_strlen(name) : 0); + InternalScopedString new_name(buflen); + if (name && zone->introspect == sanitizer_zone.introspect) { + new_name.append(COMMON_MALLOC_ZONE_NAME "-%s", name); + name = new_name.data(); + } + + // Call the system malloc's implementation for both external and our zones, + // since that appropriately changes VM region protections on the zone. + REAL(malloc_set_zone_name)(zone, name); +} + +INTERCEPTOR(void *, malloc, size_t size) { + COMMON_MALLOC_ENTER(); + COMMON_MALLOC_MALLOC(size); + return p; +} + +INTERCEPTOR(void, free, void *ptr) { + COMMON_MALLOC_ENTER(); + if (!ptr) return; + COMMON_MALLOC_FREE(ptr); +} + +INTERCEPTOR(void *, realloc, void *ptr, size_t size) { + COMMON_MALLOC_ENTER(); + COMMON_MALLOC_REALLOC(ptr, size); + return p; +} + +INTERCEPTOR(void *, calloc, size_t nmemb, size_t size) { + COMMON_MALLOC_ENTER(); + COMMON_MALLOC_CALLOC(nmemb, size); + return p; +} + +INTERCEPTOR(void *, valloc, size_t size) { + COMMON_MALLOC_ENTER(); + COMMON_MALLOC_VALLOC(size); + return p; +} + +INTERCEPTOR(size_t, malloc_good_size, size_t size) { + COMMON_MALLOC_ENTER(); + return sanitizer_zone.introspect->good_size(&sanitizer_zone, size); +} + +INTERCEPTOR(int, posix_memalign, void **memptr, size_t alignment, size_t size) { + COMMON_MALLOC_ENTER(); + CHECK(memptr); + COMMON_MALLOC_MEMALIGN(alignment, size); + if (p) { + *memptr = p; + return 0; + } + return -1; +} + +namespace { + +// TODO(glider): the __sanitizer_mz_* functions should be united with the Linux +// wrappers, as they are basically copied from there. +extern "C" +SANITIZER_INTERFACE_ATTRIBUTE +size_t __sanitizer_mz_size(malloc_zone_t* zone, const void* ptr) { + COMMON_MALLOC_SIZE(ptr); + return size; +} + +extern "C" +SANITIZER_INTERFACE_ATTRIBUTE +void *__sanitizer_mz_malloc(malloc_zone_t *zone, uptr size) { + COMMON_MALLOC_ENTER(); + COMMON_MALLOC_MALLOC(size); + return p; +} + +extern "C" +SANITIZER_INTERFACE_ATTRIBUTE +void *__sanitizer_mz_calloc(malloc_zone_t *zone, size_t nmemb, size_t size) { + if (UNLIKELY(!COMMON_MALLOC_SANITIZER_INITIALIZED)) { + // Hack: dlsym calls calloc before REAL(calloc) is retrieved from dlsym. + const size_t kCallocPoolSize = 1024; + static uptr calloc_memory_for_dlsym[kCallocPoolSize]; + static size_t allocated; + size_t size_in_words = ((nmemb * size) + kWordSize - 1) / kWordSize; + void *mem = (void*)&calloc_memory_for_dlsym[allocated]; + allocated += size_in_words; + CHECK(allocated < kCallocPoolSize); + return mem; + } + COMMON_MALLOC_CALLOC(nmemb, size); + return p; +} + +extern "C" +SANITIZER_INTERFACE_ATTRIBUTE +void *__sanitizer_mz_valloc(malloc_zone_t *zone, size_t size) { + COMMON_MALLOC_ENTER(); + COMMON_MALLOC_VALLOC(size); + return p; +} + +// TODO(glider): the allocation callbacks need to be refactored. +extern "C" +SANITIZER_INTERFACE_ATTRIBUTE +void __sanitizer_mz_free(malloc_zone_t *zone, void *ptr) { + if (!ptr) return; + COMMON_MALLOC_FREE(ptr); +} + +#define GET_ZONE_FOR_PTR(ptr) \ + malloc_zone_t *zone_ptr = malloc_zone_from_ptr(ptr); \ + const char *zone_name = (zone_ptr == 0) ? 0 : zone_ptr->zone_name + +extern "C" +SANITIZER_INTERFACE_ATTRIBUTE +void *__sanitizer_mz_realloc(malloc_zone_t *zone, void *ptr, size_t new_size) { + if (!ptr) { + COMMON_MALLOC_MALLOC(new_size); + return p; + } else { + COMMON_MALLOC_SIZE(ptr); + if (size) { + COMMON_MALLOC_REALLOC(ptr, new_size); + return p; + } else { + // We can't recover from reallocating an unknown address, because + // this would require reading at most |new_size| bytes from + // potentially unaccessible memory. + GET_ZONE_FOR_PTR(ptr); + COMMON_MALLOC_REPORT_UNKNOWN_REALLOC(ptr, zone_ptr, zone_name); + return nullptr; + } + } +} + +extern "C" +SANITIZER_INTERFACE_ATTRIBUTE +void __sanitizer_mz_destroy(malloc_zone_t* zone) { + // A no-op -- we will not be destroyed! + Report("__sanitizer_mz_destroy() called -- ignoring\n"); +} + +extern "C" +SANITIZER_INTERFACE_ATTRIBUTE +void *__sanitizer_mz_memalign(malloc_zone_t *zone, size_t align, size_t size) { + COMMON_MALLOC_ENTER(); + COMMON_MALLOC_MEMALIGN(align, size); + return p; +} + +// This function is currently unused, and we build with -Werror. +#if 0 +void __sanitizer_mz_free_definite_size( + malloc_zone_t* zone, void *ptr, size_t size) { + // TODO(glider): check that |size| is valid. + UNIMPLEMENTED(); +} +#endif + +kern_return_t mi_enumerator(task_t task, void *, + unsigned type_mask, vm_address_t zone_address, + memory_reader_t reader, + vm_range_recorder_t recorder) { + // Should enumerate all the pointers we have. Seems like a lot of work. + return KERN_FAILURE; +} + +size_t mi_good_size(malloc_zone_t *zone, size_t size) { + // I think it's always safe to return size, but we maybe could do better. + return size; +} + +boolean_t mi_check(malloc_zone_t *zone) { + UNIMPLEMENTED(); +} + +void mi_print(malloc_zone_t *zone, boolean_t verbose) { + UNIMPLEMENTED(); +} + +void mi_log(malloc_zone_t *zone, void *address) { + // I don't think we support anything like this +} + +void mi_force_lock(malloc_zone_t *zone) { + COMMON_MALLOC_FORCE_LOCK(); +} + +void mi_force_unlock(malloc_zone_t *zone) { + COMMON_MALLOC_FORCE_UNLOCK(); +} + +void mi_statistics(malloc_zone_t *zone, malloc_statistics_t *stats) { + COMMON_MALLOC_FILL_STATS(zone, stats); +} + +boolean_t mi_zone_locked(malloc_zone_t *zone) { + // UNIMPLEMENTED(); + return false; +} + +} // unnamed namespace + +namespace COMMON_MALLOC_NAMESPACE { + +void ReplaceSystemMalloc() { + static malloc_introspection_t sanitizer_zone_introspection; + // Ok to use internal_memset, these places are not performance-critical. + internal_memset(&sanitizer_zone_introspection, 0, + sizeof(sanitizer_zone_introspection)); + + sanitizer_zone_introspection.enumerator = &mi_enumerator; + sanitizer_zone_introspection.good_size = &mi_good_size; + sanitizer_zone_introspection.check = &mi_check; + sanitizer_zone_introspection.print = &mi_print; + sanitizer_zone_introspection.log = &mi_log; + sanitizer_zone_introspection.force_lock = &mi_force_lock; + sanitizer_zone_introspection.force_unlock = &mi_force_unlock; + sanitizer_zone_introspection.statistics = &mi_statistics; + sanitizer_zone_introspection.zone_locked = &mi_zone_locked; + + internal_memset(&sanitizer_zone, 0, sizeof(malloc_zone_t)); + + // Use version 6 for OSX >= 10.6. + sanitizer_zone.version = 6; + sanitizer_zone.zone_name = COMMON_MALLOC_ZONE_NAME; + sanitizer_zone.size = &__sanitizer_mz_size; + sanitizer_zone.malloc = &__sanitizer_mz_malloc; + sanitizer_zone.calloc = &__sanitizer_mz_calloc; + sanitizer_zone.valloc = &__sanitizer_mz_valloc; + sanitizer_zone.free = &__sanitizer_mz_free; + sanitizer_zone.realloc = &__sanitizer_mz_realloc; + sanitizer_zone.destroy = &__sanitizer_mz_destroy; + sanitizer_zone.batch_malloc = 0; + sanitizer_zone.batch_free = 0; + sanitizer_zone.free_definite_size = 0; + sanitizer_zone.memalign = &__sanitizer_mz_memalign; + sanitizer_zone.introspect = &sanitizer_zone_introspection; + + // Register the zone. + malloc_zone_register(&sanitizer_zone); +} + +} // namespace COMMON_MALLOC_NAMESPACE diff --git a/lib/sanitizer_common/sanitizer_persistent_allocator.h b/lib/sanitizer_common/sanitizer_persistent_allocator.h index 326406b12bfb..8e5ce06d4c3d 100644 --- a/lib/sanitizer_common/sanitizer_persistent_allocator.h +++ b/lib/sanitizer_common/sanitizer_persistent_allocator.h @@ -10,6 +10,7 @@ // A fast memory allocator that does not support free() nor realloc(). // All allocations are forever. //===----------------------------------------------------------------------===// + #ifndef SANITIZER_PERSISTENT_ALLOCATOR_H #define SANITIZER_PERSISTENT_ALLOCATOR_H @@ -36,7 +37,7 @@ inline void *PersistentAllocator::tryAlloc(uptr size) { for (;;) { uptr cmp = atomic_load(®ion_pos, memory_order_acquire); uptr end = atomic_load(®ion_end, memory_order_acquire); - if (cmp == 0 || cmp + size > end) return 0; + if (cmp == 0 || cmp + size > end) return nullptr; if (atomic_compare_exchange_weak(®ion_pos, &cmp, cmp + size, memory_order_acquire)) return (void *)cmp; @@ -68,4 +69,4 @@ inline void *PersistentAlloc(uptr sz) { } // namespace __sanitizer -#endif // SANITIZER_PERSISTENT_ALLOCATOR_H +#endif // SANITIZER_PERSISTENT_ALLOCATOR_H diff --git a/lib/sanitizer_common/sanitizer_platform.h b/lib/sanitizer_common/sanitizer_platform.h index b47281b589e9..841cceb510a8 100644 --- a/lib/sanitizer_common/sanitizer_platform.h +++ b/lib/sanitizer_common/sanitizer_platform.h @@ -87,7 +87,7 @@ // For such platforms build this code with -DSANITIZER_CAN_USE_ALLOCATOR64=0 or // change the definition of SANITIZER_CAN_USE_ALLOCATOR64 here. #ifndef SANITIZER_CAN_USE_ALLOCATOR64 -# if defined(__aarch64__) || defined(__mips64) +# if defined(__mips64) || defined(__aarch64__) # define SANITIZER_CAN_USE_ALLOCATOR64 0 # else # define SANITIZER_CAN_USE_ALLOCATOR64 (SANITIZER_WORDSIZE == 64) @@ -95,12 +95,9 @@ #endif // The range of addresses which can be returned my mmap. -// FIXME: this value should be different on different platforms, -// e.g. on AArch64 it is most likely (1ULL << 39). Larger values will still work -// but will consume more memory for TwoLevelByteMap. -#if defined(__aarch64__) -# define SANITIZER_MMAP_RANGE_SIZE FIRST_32_SECOND_64(1ULL << 32, 1ULL << 39) -#elif defined(__mips__) +// FIXME: this value should be different on different platforms. Larger values +// will still work but will consume more memory for TwoLevelByteMap. +#if defined(__mips__) # define SANITIZER_MMAP_RANGE_SIZE FIRST_32_SECOND_64(1ULL << 32, 1ULL << 40) #else # define SANITIZER_MMAP_RANGE_SIZE FIRST_32_SECOND_64(1ULL << 32, 1ULL << 47) @@ -130,7 +127,7 @@ #define SANITIZER_USES_UID16_SYSCALLS 0 #endif -#ifdef __mips__ +#if defined(__mips__) # define SANITIZER_POINTER_FORMAT_LENGTH FIRST_32_SECOND_64(8, 10) #else # define SANITIZER_POINTER_FORMAT_LENGTH FIRST_32_SECOND_64(8, 12) @@ -142,4 +139,15 @@ # define HAVE_TIRPC_RPC_XDR_H 0 #endif +/// \macro MSC_PREREQ +/// \brief Is the compiler MSVC of at least the specified version? +/// The common \param version values to check for are: +/// * 1800: Microsoft Visual Studio 2013 / 12.0 +/// * 1900: Microsoft Visual Studio 2015 / 14.0 +#ifdef _MSC_VER +# define MSC_PREREQ(version) (_MSC_VER >= (version)) +#else +# define MSC_PREREQ(version) 0 +#endif + #endif // SANITIZER_PLATFORM_H diff --git a/lib/sanitizer_common/sanitizer_platform_interceptors.h b/lib/sanitizer_common/sanitizer_platform_interceptors.h index 77cc84cd03af..430ad4839809 100644 --- a/lib/sanitizer_common/sanitizer_platform_interceptors.h +++ b/lib/sanitizer_common/sanitizer_platform_interceptors.h @@ -60,6 +60,7 @@ #define SANITIZER_INTERCEPT_STRPBRK 1 #define SANITIZER_INTERCEPT_TEXTDOMAIN SI_LINUX_NOT_ANDROID #define SANITIZER_INTERCEPT_STRCASECMP SI_NOT_WINDOWS +#define SANITIZER_INTERCEPT_MEMCMP 1 #define SANITIZER_INTERCEPT_MEMCHR 1 #define SANITIZER_INTERCEPT_MEMRCHR SI_FREEBSD || SI_LINUX @@ -132,7 +133,7 @@ #define SANITIZER_INTERCEPT_READDIR64 SI_LINUX_NOT_ANDROID #define SANITIZER_INTERCEPT_PTRACE SI_LINUX_NOT_ANDROID && \ (defined(__i386) || defined(__x86_64) || defined(__mips64) || \ - defined(__powerpc64__)) + defined(__powerpc64__) || defined(__aarch64__) || defined(__arm__)) #define SANITIZER_INTERCEPT_SETLOCALE SI_NOT_WINDOWS #define SANITIZER_INTERCEPT_GETCWD SI_NOT_WINDOWS #define SANITIZER_INTERCEPT_GET_CURRENT_DIR_NAME SI_LINUX_NOT_ANDROID @@ -142,6 +143,8 @@ #define SANITIZER_INTERCEPT_WCSTOMBS SI_NOT_WINDOWS #define SANITIZER_INTERCEPT_WCSNRTOMBS \ SI_FREEBSD || SI_MAC || SI_LINUX_NOT_ANDROID +#define SANITIZER_INTERCEPT_WCRTOMB \ + SI_FREEBSD || SI_MAC || SI_LINUX_NOT_ANDROID #define SANITIZER_INTERCEPT_TCGETATTR SI_LINUX_NOT_ANDROID #define SANITIZER_INTERCEPT_REALPATH SI_NOT_WINDOWS #define SANITIZER_INTERCEPT_CANONICALIZE_FILE_NAME SI_LINUX_NOT_ANDROID @@ -217,7 +220,7 @@ // FIXME: getline seems to be available on OSX 10.7 #define SANITIZER_INTERCEPT_GETLINE SI_FREEBSD || SI_LINUX_NOT_ANDROID -#define SANITIZER_INTERCEPT__EXIT SI_LINUX || SI_FREEBSD +#define SANITIZER_INTERCEPT__EXIT SI_LINUX || SI_FREEBSD || SI_MAC #define SANITIZER_INTERCEPT_PHTREAD_MUTEX SI_NOT_WINDOWS #define SANITIZER_INTERCEPT_PTHREAD_SETNAME_NP \ @@ -253,5 +256,13 @@ #define SANITIZER_INTERCEPT_MLOCKX SI_NOT_WINDOWS #define SANITIZER_INTERCEPT_FOPENCOOKIE SI_LINUX_NOT_ANDROID +#define SANITIZER_INTERCEPT_SEM SI_LINUX || SI_FREEBSD +#define SANITIZER_INTERCEPT_PTHREAD_SETCANCEL SI_NOT_WINDOWS +#define SANITIZER_INTERCEPT_MINCORE SI_LINUX +#define SANITIZER_INTERCEPT_PROCESS_VM_READV SI_LINUX +#define SANITIZER_INTERCEPT_CTERMID SI_LINUX || SI_MAC || SI_FREEBSD +#define SANITIZER_INTERCEPT_CTERMID_R SI_MAC || SI_FREEBSD + +#define SANITIZER_INTERCEPTOR_HOOKS SI_LINUX #endif // #ifndef SANITIZER_PLATFORM_INTERCEPTORS_H diff --git a/lib/sanitizer_common/sanitizer_platform_limits_posix.cc b/lib/sanitizer_common/sanitizer_platform_limits_posix.cc index aaa37ed02ebd..b642cba0fede 100644 --- a/lib/sanitizer_common/sanitizer_platform_limits_posix.cc +++ b/lib/sanitizer_common/sanitizer_platform_limits_posix.cc @@ -12,8 +12,8 @@ // Sizes and layouts of platform-specific POSIX data structures. //===----------------------------------------------------------------------===// - #include "sanitizer_platform.h" + #if SANITIZER_LINUX || SANITIZER_FREEBSD || SANITIZER_MAC // Tests in this file assume that off_t-dependent data structures match the // libc ABI. For example, struct dirent here is what readdir() function (as @@ -119,9 +119,17 @@ #if SANITIZER_LINUX || SANITIZER_FREEBSD # include <utime.h> # include <sys/ptrace.h> -# if defined(__mips64) +# if defined(__mips64) || defined(__aarch64__) || defined(__arm__) # include <asm/ptrace.h> +# ifdef __arm__ +typedef struct user_fpregs elf_fpregset_t; +# define ARM_VFPREGS_SIZE_ASAN (32 * 8 /*fpregs*/ + 4 /*fpscr*/) +# if !defined(ARM_VFPREGS_SIZE) +# define ARM_VFPREGS_SIZE ARM_VFPREGS_SIZE_ASAN +# endif +# endif # endif +# include <semaphore.h> #endif #if !SANITIZER_ANDROID @@ -195,7 +203,7 @@ namespace __sanitizer { unsigned struct_stat_sz = sizeof(struct stat); #if !SANITIZER_IOS && !SANITIZER_FREEBSD unsigned struct_stat64_sz = sizeof(struct stat64); -#endif // !SANITIZER_IOS && !SANITIZER_FREEBSD +#endif // !SANITIZER_IOS && !SANITIZER_FREEBSD unsigned struct_rusage_sz = sizeof(struct rusage); unsigned struct_tm_sz = sizeof(struct tm); unsigned struct_passwd_sz = sizeof(struct passwd); @@ -236,27 +244,27 @@ namespace __sanitizer { unsigned struct_new_utsname_sz = sizeof(struct new_utsname); unsigned struct_old_utsname_sz = sizeof(struct old_utsname); unsigned struct_oldold_utsname_sz = sizeof(struct oldold_utsname); -#endif // SANITIZER_LINUX +#endif // SANITIZER_LINUX #if SANITIZER_LINUX || SANITIZER_FREEBSD unsigned struct_rlimit_sz = sizeof(struct rlimit); unsigned struct_timespec_sz = sizeof(struct timespec); unsigned struct_utimbuf_sz = sizeof(struct utimbuf); unsigned struct_itimerspec_sz = sizeof(struct itimerspec); -#endif // SANITIZER_LINUX || SANITIZER_FREEBSD +#endif // SANITIZER_LINUX || SANITIZER_FREEBSD #if SANITIZER_LINUX && !SANITIZER_ANDROID unsigned struct_ustat_sz = sizeof(struct ustat); unsigned struct_rlimit64_sz = sizeof(struct rlimit64); unsigned struct_statvfs64_sz = sizeof(struct statvfs64); -#endif // SANITIZER_LINUX && !SANITIZER_ANDROID +#endif // SANITIZER_LINUX && !SANITIZER_ANDROID #if (SANITIZER_LINUX || SANITIZER_FREEBSD) && !SANITIZER_ANDROID unsigned struct_timex_sz = sizeof(struct timex); unsigned struct_msqid_ds_sz = sizeof(struct msqid_ds); unsigned struct_mq_attr_sz = sizeof(struct mq_attr); unsigned struct_statvfs_sz = sizeof(struct statvfs); -#endif // (SANITIZER_LINUX || SANITIZER_FREEBSD) && !SANITIZER_ANDROID +#endif // (SANITIZER_LINUX || SANITIZER_FREEBSD) && !SANITIZER_ANDROID uptr sig_ign = (uptr)SIG_IGN; uptr sig_dfl = (uptr)SIG_DFL; @@ -290,6 +298,12 @@ namespace __sanitizer { return 0; } +#if SANITIZER_LINUX +unsigned struct_ElfW_Phdr_sz = sizeof(ElfW(Phdr)); +#elif SANITIZER_FREEBSD +unsigned struct_ElfW_Phdr_sz = sizeof(Elf_Phdr); +#endif + #if (SANITIZER_LINUX || SANITIZER_FREEBSD) && !SANITIZER_ANDROID int glob_nomatch = GLOB_NOMATCH; int glob_altdirfunc = GLOB_ALTDIRFUNC; @@ -297,34 +311,63 @@ namespace __sanitizer { #if SANITIZER_LINUX && !SANITIZER_ANDROID && \ (defined(__i386) || defined(__x86_64) || defined(__mips64) || \ - defined(__powerpc64__)) -#if defined(__mips64) || defined(__powerpc64__) + defined(__powerpc64__) || defined(__aarch64__) || defined(__arm__)) +#if defined(__mips64) || defined(__powerpc64__) || defined(__arm__) unsigned struct_user_regs_struct_sz = sizeof(struct pt_regs); unsigned struct_user_fpregs_struct_sz = sizeof(elf_fpregset_t); +#elif defined(__aarch64__) + unsigned struct_user_regs_struct_sz = sizeof(struct user_pt_regs); + unsigned struct_user_fpregs_struct_sz = sizeof(struct user_fpsimd_state); #else unsigned struct_user_regs_struct_sz = sizeof(struct user_regs_struct); unsigned struct_user_fpregs_struct_sz = sizeof(struct user_fpregs_struct); -#endif // __mips64 || __powerpc64__ -#if defined(__x86_64) || defined(__mips64) || defined(__powerpc64__) +#endif // __mips64 || __powerpc64__ || __aarch64__ +#if defined(__x86_64) || defined(__mips64) || defined(__powerpc64__) || \ + defined(__aarch64__) || defined(__arm__) unsigned struct_user_fpxregs_struct_sz = 0; #else unsigned struct_user_fpxregs_struct_sz = sizeof(struct user_fpxregs_struct); -#endif // __x86_64 || __mips64 || __powerpc64__ +#endif // __x86_64 || __mips64 || __powerpc64__ || __aarch64__ || __arm__ +#ifdef __arm__ + unsigned struct_user_vfpregs_struct_sz = ARM_VFPREGS_SIZE; +#else + unsigned struct_user_vfpregs_struct_sz = 0; +#endif int ptrace_peektext = PTRACE_PEEKTEXT; int ptrace_peekdata = PTRACE_PEEKDATA; int ptrace_peekuser = PTRACE_PEEKUSER; +#if (defined(PTRACE_GETREGS) && defined(PTRACE_SETREGS)) || \ + (defined(PT_GETREGS) && defined(PT_SETREGS)) int ptrace_getregs = PTRACE_GETREGS; int ptrace_setregs = PTRACE_SETREGS; +#else + int ptrace_getregs = -1; + int ptrace_setregs = -1; +#endif +#if (defined(PTRACE_GETFPREGS) && defined(PTRACE_SETFPREGS)) || \ + (defined(PT_GETFPREGS) && defined(PT_SETFPREGS)) int ptrace_getfpregs = PTRACE_GETFPREGS; int ptrace_setfpregs = PTRACE_SETFPREGS; -#if defined(PTRACE_GETFPXREGS) && defined(PTRACE_SETFPXREGS) +#else + int ptrace_getfpregs = -1; + int ptrace_setfpregs = -1; +#endif +#if (defined(PTRACE_GETFPXREGS) && defined(PTRACE_SETFPXREGS)) || \ + (defined(PT_GETFPXREGS) && defined(PT_SETFPXREGS)) int ptrace_getfpxregs = PTRACE_GETFPXREGS; int ptrace_setfpxregs = PTRACE_SETFPXREGS; #else int ptrace_getfpxregs = -1; int ptrace_setfpxregs = -1; -#endif // PTRACE_GETFPXREGS/PTRACE_SETFPXREGS +#endif // PTRACE_GETFPXREGS/PTRACE_SETFPXREGS +#if defined(PTRACE_GETVFPREGS) && defined(PTRACE_SETVFPREGS) + int ptrace_getvfpregs = PTRACE_GETVFPREGS; + int ptrace_setvfpregs = PTRACE_SETVFPREGS; +#else + int ptrace_getvfpregs = -1; + int ptrace_setvfpregs = -1; +#endif int ptrace_geteventmsg = PTRACE_GETEVENTMSG; #if (defined(PTRACE_GETSIGINFO) && defined(PTRACE_SETSIGINFO)) || \ (defined(PT_GETSIGINFO) && defined(PT_SETSIGINFO)) @@ -333,14 +376,14 @@ namespace __sanitizer { #else int ptrace_getsiginfo = -1; int ptrace_setsiginfo = -1; -#endif // PTRACE_GETSIGINFO/PTRACE_SETSIGINFO +#endif // PTRACE_GETSIGINFO/PTRACE_SETSIGINFO #if defined(PTRACE_GETREGSET) && defined(PTRACE_SETREGSET) int ptrace_getregset = PTRACE_GETREGSET; int ptrace_setregset = PTRACE_SETREGSET; #else int ptrace_getregset = -1; int ptrace_setregset = -1; -#endif // PTRACE_GETREGSET/PTRACE_SETREGSET +#endif // PTRACE_GETREGSET/PTRACE_SETREGSET #endif unsigned path_max = PATH_MAX; @@ -378,7 +421,7 @@ namespace __sanitizer { unsigned struct_vt_consize_sz = sizeof(struct vt_consize); unsigned struct_vt_sizes_sz = sizeof(struct vt_sizes); unsigned struct_vt_stat_sz = sizeof(struct vt_stat); -#endif // SANITIZER_LINUX +#endif // SANITIZER_LINUX #if SANITIZER_LINUX || SANITIZER_FREEBSD #if SOUND_VERSION >= 0x040000 @@ -398,7 +441,7 @@ namespace __sanitizer { unsigned struct_seq_event_rec_sz = sizeof(struct seq_event_rec); unsigned struct_synth_info_sz = sizeof(struct synth_info); unsigned struct_vt_mode_sz = sizeof(struct vt_mode); -#endif // SANITIZER_LINUX || SANITIZER_FREEBSD +#endif // SANITIZER_LINUX || SANITIZER_FREEBSD #if SANITIZER_LINUX && !SANITIZER_ANDROID unsigned struct_ax25_parms_struct_sz = sizeof(struct ax25_parms_struct); @@ -423,12 +466,12 @@ namespace __sanitizer { unsigned struct_sockaddr_ax25_sz = sizeof(struct sockaddr_ax25); unsigned struct_unimapdesc_sz = sizeof(struct unimapdesc); unsigned struct_unimapinit_sz = sizeof(struct unimapinit); -#endif // SANITIZER_LINUX && !SANITIZER_ANDROID +#endif // SANITIZER_LINUX && !SANITIZER_ANDROID #if (SANITIZER_LINUX || SANITIZER_FREEBSD) && !SANITIZER_ANDROID unsigned struct_audio_buf_info_sz = sizeof(struct audio_buf_info); unsigned struct_ppp_stats_sz = sizeof(struct ppp_stats); -#endif // (SANITIZER_LINUX || SANITIZER_FREEBSD) && !SANITIZER_ANDROID +#endif // (SANITIZER_LINUX || SANITIZER_FREEBSD) && !SANITIZER_ANDROID #if !SANITIZER_ANDROID && !SANITIZER_MAC unsigned struct_sioc_sg_req_sz = sizeof(struct sioc_sg_req); @@ -643,7 +686,7 @@ namespace __sanitizer { unsigned IOCTL_SOUND_PCM_READ_RATE = SOUND_PCM_READ_RATE; unsigned IOCTL_SOUND_PCM_WRITE_CHANNELS = SOUND_PCM_WRITE_CHANNELS; unsigned IOCTL_SOUND_PCM_WRITE_FILTER = SOUND_PCM_WRITE_FILTER; -#endif // SOUND_VERSION +#endif // SOUND_VERSION unsigned IOCTL_TCFLSH = TCFLSH; unsigned IOCTL_TCGETA = TCGETA; unsigned IOCTL_TCGETS = TCGETS; @@ -766,7 +809,7 @@ namespace __sanitizer { unsigned IOCTL_VT_RELDISP = VT_RELDISP; unsigned IOCTL_VT_SETMODE = VT_SETMODE; unsigned IOCTL_VT_WAITACTIVE = VT_WAITACTIVE; -#endif // SANITIZER_LINUX || SANITIZER_FREEBSD +#endif // SANITIZER_LINUX || SANITIZER_FREEBSD #if SANITIZER_LINUX && !SANITIZER_ANDROID unsigned IOCTL_CYGETDEFTHRESH = CYGETDEFTHRESH; @@ -857,7 +900,7 @@ namespace __sanitizer { unsigned IOCTL_TIOCSERGETMULTI = TIOCSERGETMULTI; unsigned IOCTL_TIOCSERSETMULTI = TIOCSERSETMULTI; unsigned IOCTL_TIOCSSERIAL = TIOCSSERIAL; -#endif // SANITIZER_LINUX && !SANITIZER_ANDROID +#endif // SANITIZER_LINUX && !SANITIZER_ANDROID #if (SANITIZER_LINUX || SANITIZER_FREEBSD) && !SANITIZER_ANDROID unsigned IOCTL_GIO_SCRNMAP = GIO_SCRNMAP; @@ -875,7 +918,7 @@ namespace __sanitizer { unsigned IOCTL_PIO_SCRNMAP = PIO_SCRNMAP; unsigned IOCTL_SNDCTL_DSP_GETISPACE = SNDCTL_DSP_GETISPACE; unsigned IOCTL_SNDCTL_DSP_GETOSPACE = SNDCTL_DSP_GETOSPACE; -#endif // (SANITIZER_LINUX || SANITIZER_FREEBSD) && !SANITIZER_ANDROID +#endif // (SANITIZER_LINUX || SANITIZER_FREEBSD) && !SANITIZER_ANDROID const int errno_EINVAL = EINVAL; // EOWNERDEAD is not present in some older platforms. @@ -887,7 +930,7 @@ namespace __sanitizer { const int si_SEGV_MAPERR = SEGV_MAPERR; const int si_SEGV_ACCERR = SEGV_ACCERR; -} // namespace __sanitizer +} // namespace __sanitizer COMPILER_CHECK(sizeof(__sanitizer_pthread_attr_t) >= sizeof(pthread_attr_t)); @@ -917,7 +960,7 @@ COMPILER_CHECK(IOC_SIZE(0x12345678) == _IOC_SIZE(0x12345678)); COMPILER_CHECK(IOC_DIR(0x12345678) == _IOC_DIR(0x12345678)); COMPILER_CHECK(IOC_NR(0x12345678) == _IOC_NR(0x12345678)); COMPILER_CHECK(IOC_TYPE(0x12345678) == _IOC_TYPE(0x12345678)); -#endif // SANITIZER_LINUX +#endif // SANITIZER_LINUX #if SANITIZER_LINUX || SANITIZER_FREEBSD // There are more undocumented fields in dl_phdr_info that we are not interested @@ -927,7 +970,7 @@ CHECK_SIZE_AND_OFFSET(dl_phdr_info, dlpi_addr); CHECK_SIZE_AND_OFFSET(dl_phdr_info, dlpi_name); CHECK_SIZE_AND_OFFSET(dl_phdr_info, dlpi_phdr); CHECK_SIZE_AND_OFFSET(dl_phdr_info, dlpi_phnum); -#endif // SANITIZER_LINUX || SANITIZER_FREEBSD +#endif // SANITIZER_LINUX || SANITIZER_FREEBSD #if (SANITIZER_LINUX || SANITIZER_FREEBSD) && !SANITIZER_ANDROID CHECK_TYPE_SIZE(glob_t); @@ -1124,14 +1167,14 @@ CHECK_SIZE_AND_OFFSET(ifaddrs, ifa_netmask); # if SANITIZER_FREEBSD CHECK_SIZE_AND_OFFSET(ifaddrs, ifa_dstaddr); # else -COMPILER_CHECK(sizeof(((__sanitizer_ifaddrs *)NULL)->ifa_dstaddr) == - sizeof(((ifaddrs *)NULL)->ifa_ifu)); +COMPILER_CHECK(sizeof(((__sanitizer_ifaddrs *)nullptr)->ifa_dstaddr) == + sizeof(((ifaddrs *)nullptr)->ifa_ifu)); COMPILER_CHECK(offsetof(__sanitizer_ifaddrs, ifa_dstaddr) == offsetof(ifaddrs, ifa_ifu)); -# endif // SANITIZER_FREEBSD +# endif // SANITIZER_FREEBSD #else CHECK_SIZE_AND_OFFSET(ifaddrs, ifa_dstaddr); -#endif // SANITIZER_LINUX +#endif // SANITIZER_LINUX CHECK_SIZE_AND_OFFSET(ifaddrs, ifa_data); #endif @@ -1221,4 +1264,12 @@ CHECK_SIZE_AND_OFFSET(cookie_io_functions_t, seek); CHECK_SIZE_AND_OFFSET(cookie_io_functions_t, close); #endif -#endif // SANITIZER_LINUX || SANITIZER_FREEBSD || SANITIZER_MAC +#if SANITIZER_LINUX || SANITIZER_FREEBSD +CHECK_TYPE_SIZE(sem_t); +#endif + +#if SANITIZER_LINUX && defined(__arm__) +COMPILER_CHECK(ARM_VFPREGS_SIZE == ARM_VFPREGS_SIZE_ASAN); +#endif + +#endif // SANITIZER_LINUX || SANITIZER_FREEBSD || SANITIZER_MAC diff --git a/lib/sanitizer_common/sanitizer_platform_limits_posix.h b/lib/sanitizer_common/sanitizer_platform_limits_posix.h index 4da7c70da0a6..2978e7b9ce46 100644 --- a/lib/sanitizer_common/sanitizer_platform_limits_posix.h +++ b/lib/sanitizer_common/sanitizer_platform_limits_posix.h @@ -150,6 +150,18 @@ namespace __sanitizer { }; const unsigned old_sigset_t_sz = sizeof(unsigned long); + + struct __sanitizer_sem_t { +#if SANITIZER_ANDROID && defined(_LP64) + int data[4]; +#elif SANITIZER_ANDROID && !defined(_LP64) + int data; +#elif SANITIZER_LINUX + uptr data[4]; +#elif SANITIZER_FREEBSD + u32 data[4]; +#endif + }; #endif // SANITIZER_LINUX || SANITIZER_FREEBSD #if SANITIZER_ANDROID @@ -609,6 +621,8 @@ namespace __sanitizer { const void *dlpi_phdr; short dlpi_phnum; }; + + extern unsigned struct_ElfW_Phdr_sz; #endif struct __sanitizer_addrinfo { @@ -722,10 +736,11 @@ namespace __sanitizer { #if SANITIZER_LINUX && !SANITIZER_ANDROID && \ (defined(__i386) || defined(__x86_64) || defined(__mips64) || \ - defined(__powerpc64__)) + defined(__powerpc64__) || defined(__aarch64__) || defined(__arm__)) extern unsigned struct_user_regs_struct_sz; extern unsigned struct_user_fpregs_struct_sz; extern unsigned struct_user_fpxregs_struct_sz; + extern unsigned struct_user_vfpregs_struct_sz; extern int ptrace_peektext; extern int ptrace_peekdata; @@ -736,6 +751,8 @@ namespace __sanitizer { extern int ptrace_setfpregs; extern int ptrace_getfpxregs; extern int ptrace_setfpxregs; + extern int ptrace_getvfpregs; + extern int ptrace_setvfpregs; extern int ptrace_getsiginfo; extern int ptrace_setsiginfo; extern int ptrace_getregset; diff --git a/lib/sanitizer_common/sanitizer_posix.cc b/lib/sanitizer_common/sanitizer_posix.cc index abf6738a8daa..5ae68663df0e 100644 --- a/lib/sanitizer_common/sanitizer_posix.cc +++ b/lib/sanitizer_common/sanitizer_posix.cc @@ -13,6 +13,7 @@ //===----------------------------------------------------------------------===// #include "sanitizer_platform.h" + #if SANITIZER_POSIX #include "sanitizer_common.h" @@ -57,8 +58,8 @@ static uptr GetKernelAreaSize() { // mapped to top gigabyte (e.g. stack). MemoryMappingLayout proc_maps(/*cache_enabled*/true); uptr end, prot; - while (proc_maps.Next(/*start*/0, &end, - /*offset*/0, /*filename*/0, + while (proc_maps.Next(/*start*/nullptr, &end, + /*offset*/nullptr, /*filename*/nullptr, /*filename_size*/0, &prot)) { if ((end >= 3 * gbyte) && (prot & MemoryMappingLayout::kProtectionWrite) != 0) @@ -111,27 +112,14 @@ uptr GetMaxVirtualAddress() { #endif // SANITIZER_WORDSIZE } -void *MmapOrDie(uptr size, const char *mem_type) { +void *MmapOrDie(uptr size, const char *mem_type, bool raw_report) { size = RoundUpTo(size, GetPageSizeCached()); - uptr res = internal_mmap(0, size, - PROT_READ | PROT_WRITE, - MAP_PRIVATE | MAP_ANON, -1, 0); + uptr res = internal_mmap(nullptr, size, + PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANON, -1, 0); int reserrno; - if (internal_iserror(res, &reserrno)) { - static int recursion_count; - if (recursion_count) { - // The Report() and CHECK calls below may call mmap recursively and fail. - // If we went into recursion, just die. - RawWrite("ERROR: Failed to mmap\n"); - Die(); - } - recursion_count++; - Report("ERROR: %s failed to " - "allocate 0x%zx (%zd) bytes of %s (errno: %d)\n", - SanitizerToolName, size, size, mem_type, reserrno); - DumpProcessMap(); - CHECK("unable to mmap" && 0); - } + if (internal_iserror(res, &reserrno)) + ReportMmapFailureAndDie(size, mem_type, "allocate", reserrno, raw_report); IncreaseTotalMmap(size); return (void *)res; } @@ -149,18 +137,14 @@ void UnmapOrDie(void *addr, uptr size) { void *MmapNoReserveOrDie(uptr size, const char *mem_type) { uptr PageSize = GetPageSizeCached(); - uptr p = internal_mmap(0, - RoundUpTo(size, PageSize), - PROT_READ | PROT_WRITE, - MAP_PRIVATE | MAP_ANON | MAP_NORESERVE, - -1, 0); + uptr p = internal_mmap(nullptr, + RoundUpTo(size, PageSize), + PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANON | MAP_NORESERVE, + -1, 0); int reserrno; - if (internal_iserror(p, &reserrno)) { - Report("ERROR: %s failed to " - "allocate noreserve 0x%zx (%zd) bytes for '%s' (errno: %d)\n", - SanitizerToolName, size, size, mem_type, reserrno); - CHECK("unable to mmap" && 0); - } + if (internal_iserror(p, &reserrno)) + ReportMmapFailureAndDie(size, mem_type, "allocate noreserve", reserrno); IncreaseTotalMmap(size); return (void *)p; } @@ -174,10 +158,10 @@ void *MmapFixedOrDie(uptr fixed_addr, uptr size) { -1, 0); int reserrno; if (internal_iserror(p, &reserrno)) { - Report("ERROR: %s failed to " - "allocate 0x%zx (%zd) bytes at address %zx (errno: %d)\n", - SanitizerToolName, size, size, fixed_addr, reserrno); - CHECK("unable to mmap" && 0); + char mem_type[30]; + internal_snprintf(mem_type, sizeof(mem_type), "memory at address 0x%zx", + fixed_addr); + ReportMmapFailureAndDie(size, mem_type, "allocate", reserrno); } IncreaseTotalMmap(size); return (void *)p; @@ -236,8 +220,8 @@ void *MapFileToMemory(const char *file_name, uptr *buff_size) { CHECK_NE(fsize, (uptr)-1); CHECK_GT(fsize, 0); *buff_size = RoundUpTo(fsize, GetPageSizeCached()); - uptr map = internal_mmap(0, *buff_size, PROT_READ, MAP_PRIVATE, fd, 0); - return internal_iserror(map) ? 0 : (void *)map; + uptr map = internal_mmap(nullptr, *buff_size, PROT_READ, MAP_PRIVATE, fd, 0); + return internal_iserror(map) ? nullptr : (void *)map; } void *MapWritableFileToMemory(void *addr, uptr size, fd_t fd, OFF_T offset) { @@ -248,7 +232,7 @@ void *MapWritableFileToMemory(void *addr, uptr size, fd_t fd, OFF_T offset) { if (internal_iserror(p, &mmap_errno)) { Printf("could not map writable file (%d, %lld, %zu): %zd, errno: %d\n", fd, (long long)offset, size, p, mmap_errno); - return 0; + return nullptr; } return (void *)p; } @@ -268,8 +252,8 @@ bool MemoryRangeIsAvailable(uptr range_start, uptr range_end) { MemoryMappingLayout proc_maps(/*cache_enabled*/true); uptr start, end; while (proc_maps.Next(&start, &end, - /*offset*/0, /*filename*/0, /*filename_size*/0, - /*protection*/0)) { + /*offset*/nullptr, /*filename*/nullptr, + /*filename_size*/0, /*protection*/nullptr)) { if (start == end) continue; // Empty range. CHECK_NE(0, end); if (!IntervalsAreSeparate(start, end - 1, range_start, range_end)) @@ -284,8 +268,8 @@ void DumpProcessMap() { const sptr kBufSize = 4095; char *filename = (char*)MmapOrDie(kBufSize, __func__); Report("Process memory map follows:\n"); - while (proc_maps.Next(&start, &end, /* file_offset */0, - filename, kBufSize, /* protection */0)) { + while (proc_maps.Next(&start, &end, /* file_offset */nullptr, + filename, kBufSize, /* protection */nullptr)) { Printf("\t%p-%p\t%s\n", (void*)start, (void*)end, filename); } Report("End of process memory map.\n"); @@ -296,30 +280,6 @@ const char *GetPwd() { return GetEnv("PWD"); } -char *FindPathToBinary(const char *name) { - const char *path = GetEnv("PATH"); - if (!path) - return 0; - uptr name_len = internal_strlen(name); - InternalScopedBuffer<char> buffer(kMaxPathLength); - const char *beg = path; - while (true) { - const char *end = internal_strchrnul(beg, ':'); - uptr prefix_len = end - beg; - if (prefix_len + name_len + 2 <= kMaxPathLength) { - internal_memcpy(buffer.data(), beg, prefix_len); - buffer[prefix_len] = '/'; - internal_memcpy(&buffer[prefix_len + 1], name, name_len); - buffer[prefix_len + 1 + name_len] = '\0'; - if (FileExists(buffer.data())) - return internal_strdup(buffer.data()); - } - if (*end == '\0') break; - beg = end + 1; - } - return 0; -} - bool IsPathSeparator(const char c) { return c == '/'; } @@ -361,6 +321,6 @@ SignalContext SignalContext::Create(void *siginfo, void *context) { return SignalContext(context, addr, pc, sp, bp); } -} // namespace __sanitizer +} // namespace __sanitizer -#endif // SANITIZER_POSIX +#endif // SANITIZER_POSIX diff --git a/lib/sanitizer_common/sanitizer_posix.h b/lib/sanitizer_common/sanitizer_posix.h index 5a9e97d5b5a9..c0426a0b23fa 100644 --- a/lib/sanitizer_common/sanitizer_posix.h +++ b/lib/sanitizer_common/sanitizer_posix.h @@ -54,6 +54,7 @@ uptr internal_ptrace(int request, int pid, void *addr, void *data); uptr internal_waitpid(int pid, int *status, int options); int internal_fork(); +int internal_forkpty(int *amaster); // These functions call appropriate pthread_ functions directly, bypassing // the interceptor. They are weak and may not be present in some tools. @@ -74,6 +75,8 @@ int real_pthread_join(void *th, void **ret); } \ } // namespace __sanitizer +int my_pthread_attr_getstack(void *attr, void **addr, uptr *size); + int internal_sigaction(int signum, const void *act, void *oldact); } // namespace __sanitizer diff --git a/lib/sanitizer_common/sanitizer_posix_libcdep.cc b/lib/sanitizer_common/sanitizer_posix_libcdep.cc index 3f0a4f453cb4..c158eedae0e3 100644 --- a/lib/sanitizer_common/sanitizer_posix_libcdep.cc +++ b/lib/sanitizer_common/sanitizer_posix_libcdep.cc @@ -15,6 +15,7 @@ #include "sanitizer_platform.h" #if SANITIZER_POSIX + #include "sanitizer_common.h" #include "sanitizer_flags.h" #include "sanitizer_platform_limits_posix.h" @@ -30,9 +31,9 @@ #include <stdlib.h> #include <sys/mman.h> #include <sys/resource.h> +#include <sys/stat.h> #include <sys/time.h> #include <sys/types.h> -#include <sys/stat.h> #include <unistd.h> #if SANITIZER_FREEBSD @@ -141,7 +142,7 @@ static const uptr kAltStackSize = SIGSTKSZ * 4; // SIGSTKSZ is not enough. void SetAlternateSignalStack() { stack_t altstack, oldstack; - CHECK_EQ(0, sigaltstack(0, &oldstack)); + CHECK_EQ(0, sigaltstack(nullptr, &oldstack)); // If the alternate stack is already in place, do nothing. // Android always sets an alternate stack, but it's too small for us. if (!SANITIZER_ANDROID && !(oldstack.ss_flags & SS_DISABLE)) return; @@ -152,12 +153,12 @@ void SetAlternateSignalStack() { altstack.ss_sp = (char*) base; altstack.ss_flags = 0; altstack.ss_size = kAltStackSize; - CHECK_EQ(0, sigaltstack(&altstack, 0)); + CHECK_EQ(0, sigaltstack(&altstack, nullptr)); } void UnsetAlternateSignalStack() { stack_t altstack, oldstack; - altstack.ss_sp = 0; + altstack.ss_sp = nullptr; altstack.ss_flags = SS_DISABLE; altstack.ss_size = kAltStackSize; // Some sane value required on Darwin. CHECK_EQ(0, sigaltstack(&altstack, &oldstack)); @@ -176,7 +177,7 @@ static void MaybeInstallSigaction(int signum, // Clients are responsible for handling this correctly. sigact.sa_flags = SA_SIGINFO | SA_NODEFER; if (common_flags()->use_sigaltstack) sigact.sa_flags |= SA_ONSTACK; - CHECK_EQ(0, internal_sigaction(signum, &sigact, 0)); + CHECK_EQ(0, internal_sigaction(signum, &sigact, nullptr)); VReport(1, "Installed the sigaction for signal %d\n", signum); } @@ -188,6 +189,8 @@ void InstallDeadlySignalHandlers(SignalHandlerType handler) { MaybeInstallSigaction(SIGSEGV, handler); MaybeInstallSigaction(SIGBUS, handler); MaybeInstallSigaction(SIGABRT, handler); + MaybeInstallSigaction(SIGFPE, handler); + MaybeInstallSigaction(SIGILL, handler); } #endif // SANITIZER_GO @@ -226,7 +229,7 @@ void PrepareForSandboxing(__sanitizer_sandbox_arguments *args) { #endif } -#if SANITIZER_ANDROID +#if SANITIZER_ANDROID || SANITIZER_GO int GetNamedMappingFd(const char *name, uptr size) { return -1; } @@ -275,6 +278,48 @@ void *MmapNoAccess(uptr fixed_addr, uptr size, const char *name) { 0); } -} // namespace __sanitizer +// This function is defined elsewhere if we intercepted pthread_attr_getstack. +extern "C" { +SANITIZER_WEAK_ATTRIBUTE int +real_pthread_attr_getstack(void *attr, void **addr, size_t *size); +} // extern "C" + +int my_pthread_attr_getstack(void *attr, void **addr, uptr *size) { +#if !SANITIZER_GO && !SANITIZER_MAC + if (&real_pthread_attr_getstack) + return real_pthread_attr_getstack((pthread_attr_t *)attr, addr, + (size_t *)size); +#endif + return pthread_attr_getstack((pthread_attr_t *)attr, addr, (size_t *)size); +} + +#if !SANITIZER_GO +void AdjustStackSize(void *attr_) { + pthread_attr_t *attr = (pthread_attr_t *)attr_; + uptr stackaddr = 0; + uptr stacksize = 0; + my_pthread_attr_getstack(attr, (void**)&stackaddr, &stacksize); + // GLibC will return (0 - stacksize) as the stack address in the case when + // stacksize is set, but stackaddr is not. + bool stack_set = (stackaddr != 0) && (stackaddr + stacksize != 0); + // We place a lot of tool data into TLS, account for that. + const uptr minstacksize = GetTlsSize() + 128*1024; + if (stacksize < minstacksize) { + if (!stack_set) { + if (stacksize != 0) { + VPrintf(1, "Sanitizer: increasing stacksize %zu->%zu\n", stacksize, + minstacksize); + pthread_attr_setstacksize(attr, minstacksize); + } + } else { + Printf("Sanitizer: pre-allocated stack size is insufficient: " + "%zu < %zu\n", stacksize, minstacksize); + Printf("Sanitizer: pthread_create is likely to fail.\n"); + } + } +} +#endif // !SANITIZER_GO + +} // namespace __sanitizer -#endif // SANITIZER_POSIX +#endif // SANITIZER_POSIX diff --git a/lib/sanitizer_common/sanitizer_printf.cc b/lib/sanitizer_common/sanitizer_printf.cc index e4f67f5e0db5..2794e667e697 100644 --- a/lib/sanitizer_common/sanitizer_printf.cc +++ b/lib/sanitizer_common/sanitizer_printf.cc @@ -14,7 +14,6 @@ // inside it. //===----------------------------------------------------------------------===// - #include "sanitizer_common.h" #include "sanitizer_flags.h" #include "sanitizer_libc.h" @@ -98,7 +97,7 @@ static int AppendSignedDecimal(char **buff, const char *buff_end, s64 num, static int AppendString(char **buff, const char *buff_end, int precision, const char *s) { - if (s == 0) + if (!s) s = "<null>"; int result = 0; for (; *s; s++) { @@ -260,7 +259,7 @@ static void SharedPrintfCode(bool append_pid, const char *format, } if (append_pid) { int pid = internal_getpid(); - const char *exe_name = GetBinaryBasename(); + const char *exe_name = GetProcessName(); if (common_flags()->log_exe_name && exe_name) { needed_length += internal_snprintf(buffer, buffer_size, "==%s", exe_name); @@ -279,7 +278,8 @@ static void SharedPrintfCode(bool append_pid, const char *format, # undef CHECK_NEEDED_LENGTH } RawWrite(buffer); - AndroidLogWrite(buffer); + if (common_flags()->log_to_syslog && ShouldLogAfterPrintf()) + WriteToSyslog(buffer); CallPrintfAndReportCallback(buffer); // If we had mapped any memory, clean up. if (buffer != local_buffer) @@ -328,4 +328,4 @@ void InternalScopedString::append(const char *format, ...) { CHECK_LT(length_, size()); } -} // namespace __sanitizer +} // namespace __sanitizer diff --git a/lib/sanitizer_common/sanitizer_procmaps_common.cc b/lib/sanitizer_common/sanitizer_procmaps_common.cc index 2c6ce8e2b211..d43432cae909 100644 --- a/lib/sanitizer_common/sanitizer_procmaps_common.cc +++ b/lib/sanitizer_common/sanitizer_procmaps_common.cc @@ -11,7 +11,9 @@ //===----------------------------------------------------------------------===// #include "sanitizer_platform.h" + #if SANITIZER_FREEBSD || SANITIZER_LINUX + #include "sanitizer_common.h" #include "sanitizer_placement_new.h" #include "sanitizer_procmaps.h" @@ -151,10 +153,11 @@ uptr MemoryMappingLayout::DumpListOfModules(LoadedModule *modules, } void GetMemoryProfile(fill_profile_f cb, uptr *stats, uptr stats_size) { - char *smaps = 0; + char *smaps = nullptr; uptr smaps_cap = 0; - uptr smaps_len = ReadFileToBuffer("/proc/self/smaps", - &smaps, &smaps_cap, 64<<20); + uptr smaps_len = 0; + if (!ReadFileToBuffer("/proc/self/smaps", &smaps, &smaps_cap, &smaps_len)) + return; uptr start = 0; bool file = false; const char *pos = smaps; @@ -173,6 +176,6 @@ void GetMemoryProfile(fill_profile_f cb, uptr *stats, uptr stats_size) { UnmapOrDie(smaps, smaps_cap); } -} // namespace __sanitizer +} // namespace __sanitizer -#endif // SANITIZER_FREEBSD || SANITIZER_LINUX +#endif // SANITIZER_FREEBSD || SANITIZER_LINUX diff --git a/lib/sanitizer_common/sanitizer_procmaps_linux.cc b/lib/sanitizer_common/sanitizer_procmaps_linux.cc index 79ca4dfd8fca..b6fb7034ded4 100644 --- a/lib/sanitizer_common/sanitizer_procmaps_linux.cc +++ b/lib/sanitizer_common/sanitizer_procmaps_linux.cc @@ -18,8 +18,8 @@ namespace __sanitizer { void ReadProcMaps(ProcSelfMapsBuff *proc_maps) { - proc_maps->len = ReadFileToBuffer("/proc/self/maps", &proc_maps->data, - &proc_maps->mmaped_size, 1 << 26); + CHECK(ReadFileToBuffer("/proc/self/maps", &proc_maps->data, + &proc_maps->mmaped_size, &proc_maps->len)); } static bool IsOneOf(char c, char c1, char c2) { @@ -67,7 +67,7 @@ bool MemoryMappingLayout::Next(uptr *start, uptr *end, uptr *offset, while (IsDecimal(*current_)) current_++; // Qemu may lack the trailing space. - // http://code.google.com/p/address-sanitizer/issues/detail?id=160 + // https://github.com/google/sanitizers/issues/160 // CHECK_EQ(*current_++, ' '); // Skip spaces. while (current_ < next_line && *current_ == ' ') diff --git a/lib/sanitizer_common/sanitizer_procmaps_mac.cc b/lib/sanitizer_common/sanitizer_procmaps_mac.cc index 29d699609a6e..d10881e8a7f8 100644 --- a/lib/sanitizer_common/sanitizer_procmaps_mac.cc +++ b/lib/sanitizer_common/sanitizer_procmaps_mac.cc @@ -65,7 +65,7 @@ void MemoryMappingLayout::LoadFromCache() { } // Next and NextSegmentLoad were inspired by base/sysinfo.cc in -// Google Perftools, http://code.google.com/p/google-perftools. +// Google Perftools, https://github.com/gperftools/gperftools. // NextSegmentLoad scans the current image for the next segment load command // and returns the start and end addresses and file offset of the corresponding diff --git a/lib/sanitizer_common/sanitizer_quarantine.h b/lib/sanitizer_common/sanitizer_quarantine.h index 404d3753f7e9..9e0bf2d18fec 100644 --- a/lib/sanitizer_common/sanitizer_quarantine.h +++ b/lib/sanitizer_common/sanitizer_quarantine.h @@ -153,7 +153,7 @@ class QuarantineCache { QuarantineBatch *DequeueBatch() { if (list_.empty()) - return 0; + return nullptr; QuarantineBatch *b = list_.front(); list_.pop_front(); SizeSub(b->size); @@ -180,6 +180,6 @@ class QuarantineCache { return b; } }; -} // namespace __sanitizer +} // namespace __sanitizer -#endif // #ifndef SANITIZER_QUARANTINE_H +#endif // SANITIZER_QUARANTINE_H diff --git a/lib/sanitizer_common/sanitizer_stackdepot.cc b/lib/sanitizer_common/sanitizer_stackdepot.cc index 59b53f4dcd84..985193d1ed5e 100644 --- a/lib/sanitizer_common/sanitizer_stackdepot.cc +++ b/lib/sanitizer_common/sanitizer_stackdepot.cc @@ -152,7 +152,7 @@ StackDepotReverseMap::StackDepotReverseMap() StackTrace StackDepotReverseMap::Get(u32 id) { if (!map_.size()) return StackTrace(); - IdDescPair pair = {id, 0}; + IdDescPair pair = {id, nullptr}; uptr idx = InternalBinarySearch(map_, 0, map_.size(), pair, IdDescPair::IdComparator); if (idx > map_.size()) @@ -160,4 +160,4 @@ StackTrace StackDepotReverseMap::Get(u32 id) { return map_[idx].desc->load(); } -} // namespace __sanitizer +} // namespace __sanitizer diff --git a/lib/sanitizer_common/sanitizer_stackdepot.h b/lib/sanitizer_common/sanitizer_stackdepot.h index 5e3a8b783ac7..cb7345002a40 100644 --- a/lib/sanitizer_common/sanitizer_stackdepot.h +++ b/lib/sanitizer_common/sanitizer_stackdepot.h @@ -10,6 +10,7 @@ // This file is shared between AddressSanitizer and ThreadSanitizer // run-time libraries. //===----------------------------------------------------------------------===// + #ifndef SANITIZER_STACKDEPOT_H #define SANITIZER_STACKDEPOT_H @@ -23,7 +24,7 @@ namespace __sanitizer { struct StackDepotNode; struct StackDepotHandle { StackDepotNode *node_; - StackDepotHandle() : node_(0) {} + StackDepotHandle() : node_(nullptr) {} explicit StackDepotHandle(StackDepotNode *node) : node_(node) {} bool valid() { return node_; } u32 id(); @@ -66,6 +67,6 @@ class StackDepotReverseMap { void operator=(const StackDepotReverseMap&); }; -} // namespace __sanitizer +} // namespace __sanitizer -#endif // SANITIZER_STACKDEPOT_H +#endif // SANITIZER_STACKDEPOT_H diff --git a/lib/sanitizer_common/sanitizer_stackdepotbase.h b/lib/sanitizer_common/sanitizer_stackdepotbase.h index 5de2e711fe45..4ec77b467b1f 100644 --- a/lib/sanitizer_common/sanitizer_stackdepotbase.h +++ b/lib/sanitizer_common/sanitizer_stackdepotbase.h @@ -10,6 +10,7 @@ // Implementation of a mapping from arbitrary values to unique 32-bit // identifiers. //===----------------------------------------------------------------------===// + #ifndef SANITIZER_STACKDEPOTBASE_H #define SANITIZER_STACKDEPOTBASE_H @@ -26,7 +27,7 @@ class StackDepotBase { typedef typename Node::args_type args_type; typedef typename Node::handle_type handle_type; // Maps stack trace to an unique id. - handle_type Put(args_type args, bool *inserted = 0); + handle_type Put(args_type args, bool *inserted = nullptr); // Retrieves a stored stack trace by the id. args_type Get(u32 id); @@ -66,7 +67,7 @@ Node *StackDepotBase<Node, kReservedBits, kTabSizeLog>::find(Node *s, return s; } } - return 0; + return nullptr; } template <class Node, int kReservedBits, int kTabSizeLog> @@ -172,5 +173,6 @@ void StackDepotBase<Node, kReservedBits, kTabSizeLog>::UnlockAll() { } } -} // namespace __sanitizer -#endif // SANITIZER_STACKDEPOTBASE_H +} // namespace __sanitizer + +#endif // SANITIZER_STACKDEPOTBASE_H diff --git a/lib/sanitizer_common/sanitizer_stacktrace.cc b/lib/sanitizer_common/sanitizer_stacktrace.cc index 84ff9d9d9e3b..7862575b37bb 100644 --- a/lib/sanitizer_common/sanitizer_stacktrace.cc +++ b/lib/sanitizer_common/sanitizer_stacktrace.cc @@ -83,7 +83,18 @@ void BufferedStackTrace::FastUnwindStack(uptr pc, uptr bp, uptr stack_top, while (IsValidFrame((uptr)frame, stack_top, bottom) && IsAligned((uptr)frame, sizeof(*frame)) && size < max_depth) { +#ifdef __powerpc__ + // PowerPC ABIs specify that the return address is saved at offset + // 16 of the *caller's* stack frame. Thus we must dereference the + // back chain to find the caller frame before extracting it. + uhwptr *caller_frame = (uhwptr*)frame[0]; + if (!IsValidFrame((uptr)caller_frame, stack_top, bottom) || + !IsAligned((uptr)caller_frame, sizeof(uhwptr))) + break; + uhwptr pc1 = caller_frame[2]; +#else uhwptr pc1 = frame[1]; +#endif if (pc1 != pc) { trace_buffer[size++] = (uptr) pc1; } @@ -107,7 +118,7 @@ void BufferedStackTrace::PopStackFrames(uptr count) { uptr BufferedStackTrace::LocatePcInTrace(uptr pc) { // Use threshold to find PC in stack trace, as PC we want to unwind from may // slightly differ from return address in the actual unwinded stack trace. - const int kPcThreshold = 304; + const int kPcThreshold = 320; for (uptr i = 0; i < size; ++i) { if (MatchPc(pc, trace[i], kPcThreshold)) return i; diff --git a/lib/sanitizer_common/sanitizer_stacktrace.h b/lib/sanitizer_common/sanitizer_stacktrace.h index 6c3a1511f337..969cedb165c8 100644 --- a/lib/sanitizer_common/sanitizer_stacktrace.h +++ b/lib/sanitizer_common/sanitizer_stacktrace.h @@ -19,9 +19,7 @@ namespace __sanitizer { static const u32 kStackTraceMax = 256; -#if SANITIZER_LINUX && (defined(__aarch64__) || defined(__powerpc__) || \ - defined(__powerpc64__) || defined(__sparc__) || \ - defined(__mips__)) +#if SANITIZER_LINUX && (defined(__sparc__) || defined(__mips__)) # define SANITIZER_CAN_FAST_UNWIND 0 #elif SANITIZER_WINDOWS # define SANITIZER_CAN_FAST_UNWIND 0 @@ -31,7 +29,7 @@ static const u32 kStackTraceMax = 256; // Fast unwind is the only option on Mac for now; we will need to // revisit this macro when slow unwind works on Mac, see -// https://code.google.com/p/address-sanitizer/issues/detail?id=137 +// https://github.com/google/sanitizers/issues/137 #if SANITIZER_MAC # define SANITIZER_CAN_SLOW_UNWIND 0 #else diff --git a/lib/sanitizer_common/sanitizer_stacktrace_printer.cc b/lib/sanitizer_common/sanitizer_stacktrace_printer.cc index 3574fa3782c6..669b0ba28265 100644 --- a/lib/sanitizer_common/sanitizer_stacktrace_printer.cc +++ b/lib/sanitizer_common/sanitizer_stacktrace_printer.cc @@ -10,13 +10,14 @@ // This file is shared between sanitizers' run-time libraries. // //===----------------------------------------------------------------------===// + #include "sanitizer_stacktrace_printer.h" namespace __sanitizer { static const char *StripFunctionName(const char *function, const char *prefix) { - if (function == 0) return 0; - if (prefix == 0) return function; + if (!function) return nullptr; + if (!prefix) return function; uptr prefix_len = internal_strlen(prefix); if (0 == internal_strncmp(function, prefix, prefix_len)) return function + prefix_len; @@ -140,4 +141,4 @@ void RenderModuleLocation(InternalScopedString *buffer, const char *module, offset); } -} // namespace __sanitizer +} // namespace __sanitizer diff --git a/lib/sanitizer_common/sanitizer_stoptheworld_linux_libcdep.cc b/lib/sanitizer_common/sanitizer_stoptheworld_linux_libcdep.cc index 47b27e7e5c75..2376ee56e1d7 100644 --- a/lib/sanitizer_common/sanitizer_stoptheworld_linux_libcdep.cc +++ b/lib/sanitizer_common/sanitizer_stoptheworld_linux_libcdep.cc @@ -12,9 +12,10 @@ // //===----------------------------------------------------------------------===// - #include "sanitizer_platform.h" -#if SANITIZER_LINUX && (defined(__x86_64__) || defined(__mips__)) + +#if SANITIZER_LINUX && (defined(__x86_64__) || defined(__mips__) || \ + defined(__aarch64__) || defined(__powerpc64__)) #include "sanitizer_stoptheworld.h" @@ -27,9 +28,15 @@ #include <sys/prctl.h> // for PR_* definitions #include <sys/ptrace.h> // for PTRACE_* definitions #include <sys/types.h> // for pid_t +#include <sys/uio.h> // for iovec +#include <elf.h> // for NT_PRSTATUS #if SANITIZER_ANDROID && defined(__arm__) # include <linux/user.h> // for pt_regs #else +# ifdef __aarch64__ +// GLIBC 2.20+ sys/user does not include asm/ptrace.h +# include <asm/ptrace.h> +# endif # include <sys/user.h> // for user_regs_struct #endif #include <sys/wait.h> // for signal-related stuff @@ -112,7 +119,7 @@ bool ThreadSuspender::SuspendThread(SuspendedThreadID tid) { if (suspended_threads_list_.Contains(tid)) return false; int pterrno; - if (internal_iserror(internal_ptrace(PTRACE_ATTACH, tid, NULL, NULL), + if (internal_iserror(internal_ptrace(PTRACE_ATTACH, tid, nullptr, nullptr), &pterrno)) { // Either the thread is dead, or something prevented us from attaching. // Log this event and move on. @@ -139,11 +146,12 @@ bool ThreadSuspender::SuspendThread(SuspendedThreadID tid) { // doesn't hurt to report it. VReport(1, "Waiting on thread %d failed, detaching (errno %d).\n", tid, wperrno); - internal_ptrace(PTRACE_DETACH, tid, NULL, NULL); + internal_ptrace(PTRACE_DETACH, tid, nullptr, nullptr); return false; } if (WIFSTOPPED(status) && WSTOPSIG(status) != SIGSTOP) { - internal_ptrace(PTRACE_CONT, tid, 0, (void*)(uptr)WSTOPSIG(status)); + internal_ptrace(PTRACE_CONT, tid, nullptr, + (void*)(uptr)WSTOPSIG(status)); continue; } break; @@ -157,7 +165,7 @@ void ThreadSuspender::ResumeAllThreads() { for (uptr i = 0; i < suspended_threads_list_.thread_count(); i++) { pid_t tid = suspended_threads_list_.GetThreadID(i); int pterrno; - if (!internal_iserror(internal_ptrace(PTRACE_DETACH, tid, NULL, NULL), + if (!internal_iserror(internal_ptrace(PTRACE_DETACH, tid, nullptr, nullptr), &pterrno)) { VReport(2, "Detached from thread %d.\n", tid); } else { @@ -172,7 +180,7 @@ void ThreadSuspender::ResumeAllThreads() { void ThreadSuspender::KillAllThreads() { for (uptr i = 0; i < suspended_threads_list_.thread_count(); i++) internal_ptrace(PTRACE_KILL, suspended_threads_list_.GetThreadID(i), - NULL, NULL); + nullptr, nullptr); } bool ThreadSuspender::SuspendAllThreads() { @@ -198,13 +206,25 @@ bool ThreadSuspender::SuspendAllThreads() { } // Pointer to the ThreadSuspender instance for use in signal handler. -static ThreadSuspender *thread_suspender_instance = NULL; +static ThreadSuspender *thread_suspender_instance = nullptr; // Synchronous signals that should not be blocked. static const int kSyncSignals[] = { SIGABRT, SIGILL, SIGFPE, SIGSEGV, SIGBUS, SIGXCPU, SIGXFSZ }; -static DieCallbackType old_die_callback; +static void TracerThreadDieCallback() { + // Generally a call to Die() in the tracer thread should be fatal to the + // parent process as well, because they share the address space. + // This really only works correctly if all the threads are suspended at this + // point. So we correctly handle calls to Die() from within the callback, but + // not those that happen before or after the callback. Hopefully there aren't + // a lot of opportunities for that to happen... + ThreadSuspender *inst = thread_suspender_instance; + if (inst && stoptheworld_tracer_pid == internal_getpid()) { + inst->KillAllThreads(); + thread_suspender_instance = nullptr; + } +} // Signal handler to wake up suspended threads when the tracer thread dies. static void TracerThreadSignalHandler(int signum, void *siginfo, void *uctx) { @@ -212,37 +232,18 @@ static void TracerThreadSignalHandler(int signum, void *siginfo, void *uctx) { VPrintf(1, "Tracer caught signal %d: addr=0x%zx pc=0x%zx sp=0x%zx\n", signum, ctx.addr, ctx.pc, ctx.sp); ThreadSuspender *inst = thread_suspender_instance; - if (inst != NULL) { + if (inst) { if (signum == SIGABRT) inst->KillAllThreads(); else inst->ResumeAllThreads(); - SetDieCallback(old_die_callback); - old_die_callback = NULL; - thread_suspender_instance = NULL; + RAW_CHECK(RemoveDieCallback(TracerThreadDieCallback)); + thread_suspender_instance = nullptr; atomic_store(&inst->arg->done, 1, memory_order_relaxed); } internal__exit((signum == SIGABRT) ? 1 : 2); } -static void TracerThreadDieCallback() { - // Generally a call to Die() in the tracer thread should be fatal to the - // parent process as well, because they share the address space. - // This really only works correctly if all the threads are suspended at this - // point. So we correctly handle calls to Die() from within the callback, but - // not those that happen before or after the callback. Hopefully there aren't - // a lot of opportunities for that to happen... - ThreadSuspender *inst = thread_suspender_instance; - if (inst != NULL && stoptheworld_tracer_pid == internal_getpid()) { - inst->KillAllThreads(); - thread_suspender_instance = NULL; - } - if (old_die_callback) - old_die_callback(); - SetDieCallback(old_die_callback); - old_die_callback = NULL; -} - // Size of alternative stack for signal handlers in the tracer thread. static const int kHandlerStackSize = 4096; @@ -260,8 +261,7 @@ static int TracerThread(void* argument) { tracer_thread_argument->mutex.Lock(); tracer_thread_argument->mutex.Unlock(); - old_die_callback = GetDieCallback(); - SetDieCallback(TracerThreadDieCallback); + RAW_CHECK(AddDieCallback(TracerThreadDieCallback)); ThreadSuspender thread_suspender(internal_getppid(), tracer_thread_argument); // Global pointer for the signal handler. @@ -273,7 +273,7 @@ static int TracerThread(void* argument) { internal_memset(&handler_stack, 0, sizeof(handler_stack)); handler_stack.ss_sp = handler_stack_memory.data(); handler_stack.ss_size = kHandlerStackSize; - internal_sigaltstack(&handler_stack, NULL); + internal_sigaltstack(&handler_stack, nullptr); // Install our handler for synchronous signals. Other signals should be // blocked by the mask we inherited from the parent thread. @@ -295,8 +295,8 @@ static int TracerThread(void* argument) { thread_suspender.ResumeAllThreads(); exit_code = 0; } - SetDieCallback(old_die_callback); - thread_suspender_instance = NULL; + RAW_CHECK(RemoveDieCallback(TracerThreadDieCallback)); + thread_suspender_instance = nullptr; atomic_store(&tracer_thread_argument->done, 1, memory_order_relaxed); return exit_code; } @@ -404,8 +404,8 @@ void StopTheWorld(StopTheWorldCallback callback, void *argument) { uptr tracer_pid = internal_clone( TracerThread, tracer_stack.Bottom(), CLONE_VM | CLONE_FS | CLONE_FILES | CLONE_UNTRACED, - &tracer_thread_argument, 0 /* parent_tidptr */, 0 /* newtls */, 0 - /* child_tidptr */); + &tracer_thread_argument, nullptr /* parent_tidptr */, + nullptr /* newtls */, nullptr /* child_tidptr */); internal_sigprocmask(SIG_SETMASK, &old_sigset, 0); int local_errno = 0; if (internal_iserror(tracer_pid, &local_errno)) { @@ -432,7 +432,7 @@ void StopTheWorld(StopTheWorldCallback callback, void *argument) { // Now the tracer thread is about to exit and does not touch errno, // wait for it. for (;;) { - uptr waitpid_status = internal_waitpid(tracer_pid, NULL, __WALL); + uptr waitpid_status = internal_waitpid(tracer_pid, nullptr, __WALL); if (!internal_iserror(waitpid_status, &local_errno)) break; if (local_errno == EINTR) @@ -469,6 +469,11 @@ typedef pt_regs regs_struct; typedef struct user regs_struct; #define REG_SP regs[EF_REG29] +#elif defined(__aarch64__) +typedef struct user_pt_regs regs_struct; +#define REG_SP sp +#define ARCH_IOVEC_FOR_GETREGSET + #else #error "Unsupported architecture" #endif // SANITIZER_ANDROID && defined(__arm__) @@ -479,8 +484,18 @@ int SuspendedThreadsList::GetRegistersAndSP(uptr index, pid_t tid = GetThreadID(index); regs_struct regs; int pterrno; - if (internal_iserror(internal_ptrace(PTRACE_GETREGS, tid, NULL, ®s), - &pterrno)) { +#ifdef ARCH_IOVEC_FOR_GETREGSET + struct iovec regset_io; + regset_io.iov_base = ®s; + regset_io.iov_len = sizeof(regs_struct); + bool isErr = internal_iserror(internal_ptrace(PTRACE_GETREGSET, tid, + (void*)NT_PRSTATUS, (void*)®set_io), + &pterrno); +#else + bool isErr = internal_iserror(internal_ptrace(PTRACE_GETREGS, tid, nullptr, + ®s), &pterrno); +#endif + if (isErr) { VReport(1, "Could not get registers from thread %d (errno %d).\n", tid, pterrno); return -1; @@ -494,6 +509,7 @@ int SuspendedThreadsList::GetRegistersAndSP(uptr index, uptr SuspendedThreadsList::RegisterCount() { return sizeof(regs_struct) / sizeof(uptr); } -} // namespace __sanitizer +} // namespace __sanitizer -#endif // SANITIZER_LINUX && (defined(__x86_64__) || defined(__mips__)) +#endif // SANITIZER_LINUX && (defined(__x86_64__) || defined(__mips__) + // || defined(__aarch64__) || defined(__powerpc64__) diff --git a/lib/sanitizer_common/sanitizer_suppressions.cc b/lib/sanitizer_common/sanitizer_suppressions.cc index 08cb497269bf..f0f2c9c725a1 100644 --- a/lib/sanitizer_common/sanitizer_suppressions.cc +++ b/lib/sanitizer_common/sanitizer_suppressions.cc @@ -60,15 +60,13 @@ void SuppressionContext::ParseFromFile(const char *filename) { } // Read the file. - char *file_contents; - uptr buffer_size; - const uptr max_len = 1 << 26; - uptr contents_size = - ReadFileToBuffer(filename, &file_contents, &buffer_size, max_len); VPrintf(1, "%s: reading suppressions file at %s\n", SanitizerToolName, filename); - - if (contents_size == 0) { + char *file_contents; + uptr buffer_size; + uptr contents_size; + if (!ReadFileToBuffer(filename, &file_contents, &buffer_size, + &contents_size)) { Printf("%s: failed to read suppressions file '%s'\n", SanitizerToolName, filename); Die(); @@ -114,7 +112,8 @@ void SuppressionContext::Parse(const char *str) { end = line + internal_strlen(line); if (line != end && line[0] != '#') { const char *end2 = end; - while (line != end2 && (end2[-1] == ' ' || end2[-1] == '\t')) + while (line != end2 && + (end2[-1] == ' ' || end2[-1] == '\t' || end2[-1] == '\r')) end2--; int type; for (type = 0; type < suppression_types_num_; type++) { @@ -133,8 +132,6 @@ void SuppressionContext::Parse(const char *str) { s.templ = (char*)InternalAlloc(end2 - line + 1); internal_memcpy(s.templ, line, end2 - line); s.templ[end2 - line] = 0; - s.hit_count = 0; - s.weight = 0; suppressions_.push_back(s); has_suppression_type_[type] = true; } @@ -164,7 +161,7 @@ const Suppression *SuppressionContext::SuppressionAt(uptr i) const { void SuppressionContext::GetMatched( InternalMmapVector<Suppression *> *matched) { for (uptr i = 0; i < suppressions_.size(); i++) - if (suppressions_[i].hit_count) + if (atomic_load_relaxed(&suppressions_[i].hit_count)) matched->push_back(&suppressions_[i]); } diff --git a/lib/sanitizer_common/sanitizer_suppressions.h b/lib/sanitizer_common/sanitizer_suppressions.h index 02dbf6f9690b..0ca875a2dde6 100644 --- a/lib/sanitizer_common/sanitizer_suppressions.h +++ b/lib/sanitizer_common/sanitizer_suppressions.h @@ -14,14 +14,16 @@ #define SANITIZER_SUPPRESSIONS_H #include "sanitizer_common.h" +#include "sanitizer_atomic.h" #include "sanitizer_internal_defs.h" namespace __sanitizer { struct Suppression { + Suppression() { internal_memset(this, 0, sizeof(*this)); } const char *type; char *templ; - unsigned hit_count; + atomic_uint32_t hit_count; uptr weight; }; @@ -41,7 +43,7 @@ class SuppressionContext { void GetMatched(InternalMmapVector<Suppression *> *matched); private: - static const int kMaxSuppressionTypes = 16; + static const int kMaxSuppressionTypes = 32; const char **const suppression_types_; const int suppression_types_num_; diff --git a/lib/sanitizer_common/sanitizer_symbolizer_internal.h b/lib/sanitizer_common/sanitizer_symbolizer_internal.h index 66ae809ed53e..12c70b602e24 100644 --- a/lib/sanitizer_common/sanitizer_symbolizer_internal.h +++ b/lib/sanitizer_common/sanitizer_symbolizer_internal.h @@ -74,24 +74,31 @@ class SymbolizerProcess { explicit SymbolizerProcess(const char *path, bool use_forkpty = false); const char *SendCommand(const char *command); - private: - bool Restart(); - const char *SendCommandImpl(const char *command); - bool ReadFromSymbolizer(char *buffer, uptr max_length); - bool WriteToSymbolizer(const char *buffer, uptr length); - bool StartSymbolizerSubprocess(); - + protected: virtual bool ReachedEndOfOutput(const char *buffer, uptr length) const { UNIMPLEMENTED(); } - virtual void ExecuteWithDefaultArgs(const char *path_to_binary) const { + /// The maximum number of arguments required to invoke a tool process. + enum { kArgVMax = 6 }; + + /// Fill in an argv array to invoke the child process. + virtual void GetArgV(const char *path_to_binary, + const char *(&argv)[kArgVMax]) const { UNIMPLEMENTED(); } + virtual bool ReadFromSymbolizer(char *buffer, uptr max_length); + + private: + bool Restart(); + const char *SendCommandImpl(const char *command); + bool WriteToSymbolizer(const char *buffer, uptr length); + bool StartSymbolizerSubprocess(); + const char *path_; - int input_fd_; - int output_fd_; + fd_t input_fd_; + fd_t output_fd_; static const uptr kBufferSize = 16 * 1024; char buffer_[kBufferSize]; @@ -104,6 +111,41 @@ class SymbolizerProcess { bool use_forkpty_; }; +class LLVMSymbolizerProcess; + +// This tool invokes llvm-symbolizer in a subprocess. It should be as portable +// as the llvm-symbolizer tool is. +class LLVMSymbolizer : public SymbolizerTool { + public: + explicit LLVMSymbolizer(const char *path, LowLevelAllocator *allocator); + + bool SymbolizePC(uptr addr, SymbolizedStack *stack) override; + + bool SymbolizeData(uptr addr, DataInfo *info) override; + + private: + const char *SendCommand(bool is_data, const char *module_name, + uptr module_offset); + + LLVMSymbolizerProcess *symbolizer_process_; + static const uptr kBufferSize = 16 * 1024; + char buffer_[kBufferSize]; +}; + +// Parses one or more two-line strings in the following format: +// <function_name> +// <file_name>:<line_number>[:<column_number>] +// Used by LLVMSymbolizer, Addr2LinePool and InternalSymbolizer, since all of +// them use the same output format. Returns true if any useful debug +// information was found. +void ParseSymbolizePCOutput(const char *str, SymbolizedStack *res); + +// Parses a two-line string in the following format: +// <symbol_name> +// <start_address> <size> +// Used by LLVMSymbolizer and InternalSymbolizer. +void ParseSymbolizeDataOutput(const char *str, DataInfo *info); + } // namespace __sanitizer #endif // SANITIZER_SYMBOLIZER_INTERNAL_H diff --git a/lib/sanitizer_common/sanitizer_symbolizer_libbacktrace.h b/lib/sanitizer_common/sanitizer_symbolizer_libbacktrace.h index 00b465a72774..ddfd475592cb 100644 --- a/lib/sanitizer_common/sanitizer_symbolizer_libbacktrace.h +++ b/lib/sanitizer_common/sanitizer_symbolizer_libbacktrace.h @@ -16,6 +16,7 @@ #include "sanitizer_platform.h" #include "sanitizer_common.h" +#include "sanitizer_allocator_internal.h" #include "sanitizer_symbolizer_internal.h" #ifndef SANITIZER_LIBBACKTRACE diff --git a/lib/sanitizer_common/sanitizer_symbolizer_libcdep.cc b/lib/sanitizer_common/sanitizer_symbolizer_libcdep.cc index 160f55d422ca..8c3ad81f952a 100644 --- a/lib/sanitizer_common/sanitizer_symbolizer_libcdep.cc +++ b/lib/sanitizer_common/sanitizer_symbolizer_libcdep.cc @@ -184,4 +184,245 @@ Symbolizer *Symbolizer::GetOrInit() { return symbolizer_; } +// For now we assume the following protocol: +// For each request of the form +// <module_name> <module_offset> +// passed to STDIN, external symbolizer prints to STDOUT response: +// <function_name> +// <file_name>:<line_number>:<column_number> +// <function_name> +// <file_name>:<line_number>:<column_number> +// ... +// <empty line> +class LLVMSymbolizerProcess : public SymbolizerProcess { + public: + explicit LLVMSymbolizerProcess(const char *path) : SymbolizerProcess(path) {} + + private: + bool ReachedEndOfOutput(const char *buffer, uptr length) const override { + // Empty line marks the end of llvm-symbolizer output. + return length >= 2 && buffer[length - 1] == '\n' && + buffer[length - 2] == '\n'; + } + + void GetArgV(const char *path_to_binary, + const char *(&argv)[kArgVMax]) const override { +#if defined(__x86_64h__) + const char* const kSymbolizerArch = "--default-arch=x86_64h"; +#elif defined(__x86_64__) + const char* const kSymbolizerArch = "--default-arch=x86_64"; +#elif defined(__i386__) + const char* const kSymbolizerArch = "--default-arch=i386"; +#elif defined(__powerpc64__) && defined(__BIG_ENDIAN__) + const char* const kSymbolizerArch = "--default-arch=powerpc64"; +#elif defined(__powerpc64__) && defined(__LITTLE_ENDIAN__) + const char* const kSymbolizerArch = "--default-arch=powerpc64le"; +#else + const char* const kSymbolizerArch = "--default-arch=unknown"; +#endif + + const char *const inline_flag = common_flags()->symbolize_inline_frames + ? "--inlining=true" + : "--inlining=false"; + int i = 0; + argv[i++] = path_to_binary; + argv[i++] = inline_flag; + argv[i++] = kSymbolizerArch; + argv[i++] = nullptr; + } +}; + +LLVMSymbolizer::LLVMSymbolizer(const char *path, LowLevelAllocator *allocator) + : symbolizer_process_(new(*allocator) LLVMSymbolizerProcess(path)) {} + +// Parse a <file>:<line>[:<column>] buffer. The file path may contain colons on +// Windows, so extract tokens from the right hand side first. The column info is +// also optional. +static const char *ParseFileLineInfo(AddressInfo *info, const char *str) { + char *file_line_info = 0; + str = ExtractToken(str, "\n", &file_line_info); + CHECK(file_line_info); + // Parse the last :<int>, which must be there. + char *last_colon = internal_strrchr(file_line_info, ':'); + CHECK(last_colon); + int line_or_column = internal_atoll(last_colon + 1); + // Truncate the string at the last colon and find the next-to-last colon. + *last_colon = '\0'; + last_colon = internal_strrchr(file_line_info, ':'); + if (last_colon && IsDigit(last_colon[1])) { + // If the second-to-last colon is followed by a digit, it must be the line + // number, and the previous parsed number was a column. + info->line = internal_atoll(last_colon + 1); + info->column = line_or_column; + *last_colon = '\0'; + } else { + // Otherwise, we have line info but no column info. + info->line = line_or_column; + info->column = 0; + } + ExtractToken(file_line_info, "", &info->file); + InternalFree(file_line_info); + return str; +} + +// Parses one or more two-line strings in the following format: +// <function_name> +// <file_name>:<line_number>[:<column_number>] +// Used by LLVMSymbolizer, Addr2LinePool and InternalSymbolizer, since all of +// them use the same output format. +void ParseSymbolizePCOutput(const char *str, SymbolizedStack *res) { + bool top_frame = true; + SymbolizedStack *last = res; + while (true) { + char *function_name = 0; + str = ExtractToken(str, "\n", &function_name); + CHECK(function_name); + if (function_name[0] == '\0') { + // There are no more frames. + InternalFree(function_name); + break; + } + SymbolizedStack *cur; + if (top_frame) { + cur = res; + top_frame = false; + } else { + cur = SymbolizedStack::New(res->info.address); + cur->info.FillModuleInfo(res->info.module, res->info.module_offset); + last->next = cur; + last = cur; + } + + AddressInfo *info = &cur->info; + info->function = function_name; + str = ParseFileLineInfo(info, str); + + // Functions and filenames can be "??", in which case we write 0 + // to address info to mark that names are unknown. + if (0 == internal_strcmp(info->function, "??")) { + InternalFree(info->function); + info->function = 0; + } + if (0 == internal_strcmp(info->file, "??")) { + InternalFree(info->file); + info->file = 0; + } + } +} + +// Parses a two-line string in the following format: +// <symbol_name> +// <start_address> <size> +// Used by LLVMSymbolizer and InternalSymbolizer. +void ParseSymbolizeDataOutput(const char *str, DataInfo *info) { + str = ExtractToken(str, "\n", &info->name); + str = ExtractUptr(str, " ", &info->start); + str = ExtractUptr(str, "\n", &info->size); +} + +bool LLVMSymbolizer::SymbolizePC(uptr addr, SymbolizedStack *stack) { + if (const char *buf = SendCommand(/*is_data*/ false, stack->info.module, + stack->info.module_offset)) { + ParseSymbolizePCOutput(buf, stack); + return true; + } + return false; +} + +bool LLVMSymbolizer::SymbolizeData(uptr addr, DataInfo *info) { + if (const char *buf = + SendCommand(/*is_data*/ true, info->module, info->module_offset)) { + ParseSymbolizeDataOutput(buf, info); + info->start += (addr - info->module_offset); // Add the base address. + return true; + } + return false; +} + +const char *LLVMSymbolizer::SendCommand(bool is_data, const char *module_name, + uptr module_offset) { + CHECK(module_name); + internal_snprintf(buffer_, kBufferSize, "%s\"%s\" 0x%zx\n", + is_data ? "DATA " : "", module_name, module_offset); + return symbolizer_process_->SendCommand(buffer_); +} + +SymbolizerProcess::SymbolizerProcess(const char *path, bool use_forkpty) + : path_(path), + input_fd_(kInvalidFd), + output_fd_(kInvalidFd), + times_restarted_(0), + failed_to_start_(false), + reported_invalid_path_(false), + use_forkpty_(use_forkpty) { + CHECK(path_); + CHECK_NE(path_[0], '\0'); +} + +const char *SymbolizerProcess::SendCommand(const char *command) { + for (; times_restarted_ < kMaxTimesRestarted; times_restarted_++) { + // Start or restart symbolizer if we failed to send command to it. + if (const char *res = SendCommandImpl(command)) + return res; + Restart(); + } + if (!failed_to_start_) { + Report("WARNING: Failed to use and restart external symbolizer!\n"); + failed_to_start_ = true; + } + return 0; +} + +const char *SymbolizerProcess::SendCommandImpl(const char *command) { + if (input_fd_ == kInvalidFd || output_fd_ == kInvalidFd) + return 0; + if (!WriteToSymbolizer(command, internal_strlen(command))) + return 0; + if (!ReadFromSymbolizer(buffer_, kBufferSize)) + return 0; + return buffer_; +} + +bool SymbolizerProcess::Restart() { + if (input_fd_ != kInvalidFd) + CloseFile(input_fd_); + if (output_fd_ != kInvalidFd) + CloseFile(output_fd_); + return StartSymbolizerSubprocess(); +} + +bool SymbolizerProcess::ReadFromSymbolizer(char *buffer, uptr max_length) { + if (max_length == 0) + return true; + uptr read_len = 0; + while (true) { + uptr just_read = 0; + bool success = ReadFromFile(input_fd_, buffer + read_len, + max_length - read_len - 1, &just_read); + // We can't read 0 bytes, as we don't expect external symbolizer to close + // its stdout. + if (!success || just_read == 0) { + Report("WARNING: Can't read from symbolizer at fd %d\n", input_fd_); + return false; + } + read_len += just_read; + if (ReachedEndOfOutput(buffer, read_len)) + break; + } + buffer[read_len] = '\0'; + return true; +} + +bool SymbolizerProcess::WriteToSymbolizer(const char *buffer, uptr length) { + if (length == 0) + return true; + uptr write_len = 0; + bool success = WriteToFile(output_fd_, buffer, length, &write_len); + if (!success || write_len != length) { + Report("WARNING: Can't write to symbolizer at fd %d\n", output_fd_); + return false; + } + return true; +} + } // namespace __sanitizer diff --git a/lib/sanitizer_common/sanitizer_symbolizer_mac.cc b/lib/sanitizer_common/sanitizer_symbolizer_mac.cc index 9a64192b0353..64048fa7e58e 100644 --- a/lib/sanitizer_common/sanitizer_symbolizer_mac.cc +++ b/lib/sanitizer_common/sanitizer_symbolizer_mac.cc @@ -33,39 +33,50 @@ bool DlAddrSymbolizer::SymbolizePC(uptr addr, SymbolizedStack *stack) { int result = dladdr((const void *)addr, &info); if (!result) return false; const char *demangled = DemangleCXXABI(info.dli_sname); - stack->info.function = internal_strdup(demangled); + stack->info.function = demangled ? internal_strdup(demangled) : nullptr; return true; } -bool DlAddrSymbolizer::SymbolizeData(uptr addr, DataInfo *info) { - return false; +bool DlAddrSymbolizer::SymbolizeData(uptr addr, DataInfo *datainfo) { + Dl_info info; + int result = dladdr((const void *)addr, &info); + if (!result) return false; + const char *demangled = DemangleCXXABI(info.dli_sname); + datainfo->name = internal_strdup(demangled); + datainfo->start = (uptr)info.dli_saddr; + return true; } class AtosSymbolizerProcess : public SymbolizerProcess { public: explicit AtosSymbolizerProcess(const char *path, pid_t parent_pid) - : SymbolizerProcess(path, /*use_forkpty*/ true), - parent_pid_(parent_pid) {} + : SymbolizerProcess(path, /*use_forkpty*/ true) { + // Put the string command line argument in the object so that it outlives + // the call to GetArgV. + internal_snprintf(pid_str_, sizeof(pid_str_), "%d", parent_pid); + } private: bool ReachedEndOfOutput(const char *buffer, uptr length) const override { return (length >= 1 && buffer[length - 1] == '\n'); } - void ExecuteWithDefaultArgs(const char *path_to_binary) const override { - char pid_str[16]; - internal_snprintf(pid_str, sizeof(pid_str), "%d", parent_pid_); + void GetArgV(const char *path_to_binary, + const char *(&argv)[kArgVMax]) const override { + int i = 0; + argv[i++] = path_to_binary; + argv[i++] = "-p"; + argv[i++] = &pid_str_[0]; if (GetMacosVersion() == MACOS_VERSION_MAVERICKS) { // On Mavericks atos prints a deprecation warning which we suppress by // passing -d. The warning isn't present on other OSX versions, even the // newer ones. - execl(path_to_binary, path_to_binary, "-p", pid_str, "-d", (char *)0); - } else { - execl(path_to_binary, path_to_binary, "-p", pid_str, (char *)0); + argv[i++] = "-d"; } + argv[i++] = nullptr; } - pid_t parent_pid_; + char pid_str_[16]; }; static const char *kAtosErrorMessages[] = { @@ -85,7 +96,9 @@ static bool IsAtosErrorMessage(const char *str) { return false; } -static bool ParseCommandOutput(const char *str, SymbolizedStack *res) { +static bool ParseCommandOutput(const char *str, uptr addr, char **out_name, + char **out_module, char **out_file, uptr *line, + uptr *start_address) { // Trim ending newlines. char *trim; ExtractTokenUpToDelimiter(str, "\n", &trim); @@ -93,7 +106,9 @@ static bool ParseCommandOutput(const char *str, SymbolizedStack *res) { // The line from `atos` is in one of these formats: // myfunction (in library.dylib) (sourcefile.c:17) // myfunction (in library.dylib) + 0x1fe + // myfunction (in library.dylib) + 15 // 0xdeadbeef (in library.dylib) + 0x1fe + // 0xdeadbeef (in library.dylib) + 15 // 0xdeadbeef (in library.dylib) // 0xdeadbeef @@ -104,21 +119,33 @@ static bool ParseCommandOutput(const char *str, SymbolizedStack *res) { } const char *rest = trim; - char *function_name; - rest = ExtractTokenUpToDelimiter(rest, " (in ", &function_name); - if (internal_strncmp(function_name, "0x", 2) != 0) - res->info.function = function_name; + char *symbol_name; + rest = ExtractTokenUpToDelimiter(rest, " (in ", &symbol_name); + if (rest[0] == '\0') { + InternalFree(symbol_name); + InternalFree(trim); + return false; + } + + if (internal_strncmp(symbol_name, "0x", 2) != 0) + *out_name = symbol_name; else - InternalFree(function_name); - rest = ExtractTokenUpToDelimiter(rest, ") ", &res->info.module); + InternalFree(symbol_name); + rest = ExtractTokenUpToDelimiter(rest, ") ", out_module); if (rest[0] == '(') { - rest++; - rest = ExtractTokenUpToDelimiter(rest, ":", &res->info.file); - char *extracted_line_number; - rest = ExtractTokenUpToDelimiter(rest, ")", &extracted_line_number); - res->info.line = internal_atoll(extracted_line_number); - InternalFree(extracted_line_number); + if (out_file) { + rest++; + rest = ExtractTokenUpToDelimiter(rest, ":", out_file); + char *extracted_line_number; + rest = ExtractTokenUpToDelimiter(rest, ")", &extracted_line_number); + if (line) *line = (uptr)internal_atoll(extracted_line_number); + InternalFree(extracted_line_number); + } + } else if (rest[0] == '+') { + rest += 2; + uptr offset = internal_atoll(rest); + if (start_address) *start_address = addr - offset; } InternalFree(trim); @@ -134,14 +161,29 @@ bool AtosSymbolizer::SymbolizePC(uptr addr, SymbolizedStack *stack) { internal_snprintf(command, sizeof(command), "0x%zx\n", addr); const char *buf = process_->SendCommand(command); if (!buf) return false; - if (!ParseCommandOutput(buf, stack)) { + uptr line; + if (!ParseCommandOutput(buf, addr, &stack->info.function, &stack->info.module, + &stack->info.file, &line, nullptr)) { process_ = nullptr; return false; } + stack->info.line = (int)line; return true; } -bool AtosSymbolizer::SymbolizeData(uptr addr, DataInfo *info) { return false; } +bool AtosSymbolizer::SymbolizeData(uptr addr, DataInfo *info) { + if (!process_) return false; + char command[32]; + internal_snprintf(command, sizeof(command), "0x%zx\n", addr); + const char *buf = process_->SendCommand(command); + if (!buf) return false; + if (!ParseCommandOutput(buf, addr, &info->name, &info->module, nullptr, + nullptr, &info->start)) { + process_ = nullptr; + return false; + } + return true; +} } // namespace __sanitizer diff --git a/lib/sanitizer_common/sanitizer_symbolizer_posix_libcdep.cc b/lib/sanitizer_common/sanitizer_symbolizer_posix_libcdep.cc index cb8455a21ae2..fc8a7d91ac73 100644 --- a/lib/sanitizer_common/sanitizer_symbolizer_posix_libcdep.cc +++ b/lib/sanitizer_common/sanitizer_symbolizer_posix_libcdep.cc @@ -20,13 +20,21 @@ #include "sanitizer_internal_defs.h" #include "sanitizer_linux.h" #include "sanitizer_placement_new.h" +#include "sanitizer_posix.h" #include "sanitizer_procmaps.h" #include "sanitizer_symbolizer_internal.h" #include "sanitizer_symbolizer_libbacktrace.h" #include "sanitizer_symbolizer_mac.h" +#include <errno.h> +#include <stdlib.h> +#include <sys/wait.h> #include <unistd.h> +#if SANITIZER_MAC +#include <util.h> // for forkpty() +#endif // SANITIZER_MAC + // C++ demangling function, as required by Itanium C++ ABI. This is weak, // because we do not require a C++ ABI library to be linked to a program // using sanitizers; if it's not present, we'll just use the mangled name. @@ -53,149 +61,130 @@ const char *DemangleCXXABI(const char *name) { return name; } -// Parses one or more two-line strings in the following format: -// <function_name> -// <file_name>:<line_number>[:<column_number>] -// Used by LLVMSymbolizer, Addr2LinePool and InternalSymbolizer, since all of -// them use the same output format. -static void ParseSymbolizePCOutput(const char *str, SymbolizedStack *res) { - bool top_frame = true; - SymbolizedStack *last = res; - while (true) { - char *function_name = 0; - str = ExtractToken(str, "\n", &function_name); - CHECK(function_name); - if (function_name[0] == '\0') { - // There are no more frames. - InternalFree(function_name); - break; - } - SymbolizedStack *cur; - if (top_frame) { - cur = res; - top_frame = false; - } else { - cur = SymbolizedStack::New(res->info.address); - cur->info.FillModuleInfo(res->info.module, res->info.module_offset); - last->next = cur; - last = cur; - } - - AddressInfo *info = &cur->info; - info->function = function_name; - // Parse <file>:<line>:<column> buffer. - char *file_line_info = 0; - str = ExtractToken(str, "\n", &file_line_info); - CHECK(file_line_info); - const char *line_info = ExtractToken(file_line_info, ":", &info->file); - line_info = ExtractInt(line_info, ":", &info->line); - line_info = ExtractInt(line_info, "", &info->column); - InternalFree(file_line_info); - - // Functions and filenames can be "??", in which case we write 0 - // to address info to mark that names are unknown. - if (0 == internal_strcmp(info->function, "??")) { - InternalFree(info->function); - info->function = 0; - } - if (0 == internal_strcmp(info->file, "??")) { - InternalFree(info->file); - info->file = 0; +bool SymbolizerProcess::StartSymbolizerSubprocess() { + if (!FileExists(path_)) { + if (!reported_invalid_path_) { + Report("WARNING: invalid path to external symbolizer!\n"); + reported_invalid_path_ = true; } - } -} - -// Parses a two-line string in the following format: -// <symbol_name> -// <start_address> <size> -// Used by LLVMSymbolizer and InternalSymbolizer. -static void ParseSymbolizeDataOutput(const char *str, DataInfo *info) { - str = ExtractToken(str, "\n", &info->name); - str = ExtractUptr(str, " ", &info->start); - str = ExtractUptr(str, "\n", &info->size); -} - -// For now we assume the following protocol: -// For each request of the form -// <module_name> <module_offset> -// passed to STDIN, external symbolizer prints to STDOUT response: -// <function_name> -// <file_name>:<line_number>:<column_number> -// <function_name> -// <file_name>:<line_number>:<column_number> -// ... -// <empty line> -class LLVMSymbolizerProcess : public SymbolizerProcess { - public: - explicit LLVMSymbolizerProcess(const char *path) : SymbolizerProcess(path) {} - - private: - bool ReachedEndOfOutput(const char *buffer, uptr length) const override { - // Empty line marks the end of llvm-symbolizer output. - return length >= 2 && buffer[length - 1] == '\n' && - buffer[length - 2] == '\n'; + return false; } - void ExecuteWithDefaultArgs(const char *path_to_binary) const override { -#if defined(__x86_64h__) - const char* const kSymbolizerArch = "--default-arch=x86_64h"; -#elif defined(__x86_64__) - const char* const kSymbolizerArch = "--default-arch=x86_64"; -#elif defined(__i386__) - const char* const kSymbolizerArch = "--default-arch=i386"; -#elif defined(__powerpc64__) && defined(__BIG_ENDIAN__) - const char* const kSymbolizerArch = "--default-arch=powerpc64"; -#elif defined(__powerpc64__) && defined(__LITTLE_ENDIAN__) - const char* const kSymbolizerArch = "--default-arch=powerpc64le"; -#else - const char* const kSymbolizerArch = "--default-arch=unknown"; -#endif - - const char *const inline_flag = common_flags()->symbolize_inline_frames - ? "--inlining=true" - : "--inlining=false"; - execl(path_to_binary, path_to_binary, inline_flag, kSymbolizerArch, - (char *)0); - } -}; + int pid; + if (use_forkpty_) { +#if SANITIZER_MAC + fd_t fd = kInvalidFd; + // Use forkpty to disable buffering in the new terminal. + pid = internal_forkpty(&fd); + if (pid == -1) { + // forkpty() failed. + Report("WARNING: failed to fork external symbolizer (errno: %d)\n", + errno); + return false; + } else if (pid == 0) { + // Child subprocess. + const char *argv[kArgVMax]; + GetArgV(path_, argv); + execv(path_, const_cast<char **>(&argv[0])); + internal__exit(1); + } -class LLVMSymbolizer : public SymbolizerTool { - public: - explicit LLVMSymbolizer(const char *path, LowLevelAllocator *allocator) - : symbolizer_process_(new(*allocator) LLVMSymbolizerProcess(path)) {} + // Continue execution in parent process. + input_fd_ = output_fd_ = fd; - bool SymbolizePC(uptr addr, SymbolizedStack *stack) override { - if (const char *buf = SendCommand(/*is_data*/ false, stack->info.module, - stack->info.module_offset)) { - ParseSymbolizePCOutput(buf, stack); - return true; + // Disable echo in the new terminal, disable CR. + struct termios termflags; + tcgetattr(fd, &termflags); + termflags.c_oflag &= ~ONLCR; + termflags.c_lflag &= ~ECHO; + tcsetattr(fd, TCSANOW, &termflags); +#else // SANITIZER_MAC + UNIMPLEMENTED(); +#endif // SANITIZER_MAC + } else { + int *infd = NULL; + int *outfd = NULL; + // The client program may close its stdin and/or stdout and/or stderr + // thus allowing socketpair to reuse file descriptors 0, 1 or 2. + // In this case the communication between the forked processes may be + // broken if either the parent or the child tries to close or duplicate + // these descriptors. The loop below produces two pairs of file + // descriptors, each greater than 2 (stderr). + int sock_pair[5][2]; + for (int i = 0; i < 5; i++) { + if (pipe(sock_pair[i]) == -1) { + for (int j = 0; j < i; j++) { + internal_close(sock_pair[j][0]); + internal_close(sock_pair[j][1]); + } + Report("WARNING: Can't create a socket pair to start " + "external symbolizer (errno: %d)\n", errno); + return false; + } else if (sock_pair[i][0] > 2 && sock_pair[i][1] > 2) { + if (infd == NULL) { + infd = sock_pair[i]; + } else { + outfd = sock_pair[i]; + for (int j = 0; j < i; j++) { + if (sock_pair[j] == infd) continue; + internal_close(sock_pair[j][0]); + internal_close(sock_pair[j][1]); + } + break; + } + } } - return false; - } - - bool SymbolizeData(uptr addr, DataInfo *info) override { - if (const char *buf = - SendCommand(/*is_data*/ true, info->module, info->module_offset)) { - ParseSymbolizeDataOutput(buf, info); - info->start += (addr - info->module_offset); // Add the base address. - return true; + CHECK(infd); + CHECK(outfd); + + // Real fork() may call user callbacks registered with pthread_atfork(). + pid = internal_fork(); + if (pid == -1) { + // Fork() failed. + internal_close(infd[0]); + internal_close(infd[1]); + internal_close(outfd[0]); + internal_close(outfd[1]); + Report("WARNING: failed to fork external symbolizer " + " (errno: %d)\n", errno); + return false; + } else if (pid == 0) { + // Child subprocess. + internal_close(STDOUT_FILENO); + internal_close(STDIN_FILENO); + internal_dup2(outfd[0], STDIN_FILENO); + internal_dup2(infd[1], STDOUT_FILENO); + internal_close(outfd[0]); + internal_close(outfd[1]); + internal_close(infd[0]); + internal_close(infd[1]); + for (int fd = sysconf(_SC_OPEN_MAX); fd > 2; fd--) + internal_close(fd); + const char *argv[kArgVMax]; + GetArgV(path_, argv); + execv(path_, const_cast<char **>(&argv[0])); + internal__exit(1); } - return false; + + // Continue execution in parent process. + internal_close(outfd[0]); + internal_close(infd[1]); + input_fd_ = infd[0]; + output_fd_ = outfd[1]; } - private: - const char *SendCommand(bool is_data, const char *module_name, - uptr module_offset) { - CHECK(module_name); - internal_snprintf(buffer_, kBufferSize, "%s\"%s\" 0x%zx\n", - is_data ? "DATA " : "", module_name, module_offset); - return symbolizer_process_->SendCommand(buffer_); + // Check that symbolizer subprocess started successfully. + int pid_status; + SleepForMillis(kSymbolizerStartupTimeMillis); + int exited_pid = waitpid(pid, &pid_status, WNOHANG); + if (exited_pid != 0) { + // Either waitpid failed, or child has already exited. + Report("WARNING: external symbolizer didn't start up correctly!\n"); + return false; } - LLVMSymbolizerProcess *symbolizer_process_; - static const uptr kBufferSize = 16 * 1024; - char buffer_[kBufferSize]; -}; + return true; +} class Addr2LineProcess : public SymbolizerProcess { public: @@ -205,25 +194,54 @@ class Addr2LineProcess : public SymbolizerProcess { const char *module_name() const { return module_name_; } private: - bool ReachedEndOfOutput(const char *buffer, uptr length) const override { - // Output should consist of two lines. - int num_lines = 0; - for (uptr i = 0; i < length; ++i) { - if (buffer[i] == '\n') - num_lines++; - if (num_lines >= 2) - return true; - } - return false; + void GetArgV(const char *path_to_binary, + const char *(&argv)[kArgVMax]) const override { + int i = 0; + argv[i++] = path_to_binary; + argv[i++] = "-iCfe"; + argv[i++] = module_name_; + argv[i++] = nullptr; } - void ExecuteWithDefaultArgs(const char *path_to_binary) const override { - execl(path_to_binary, path_to_binary, "-Cfe", module_name_, (char *)0); + bool ReachedEndOfOutput(const char *buffer, uptr length) const override; + + bool ReadFromSymbolizer(char *buffer, uptr max_length) override { + if (!SymbolizerProcess::ReadFromSymbolizer(buffer, max_length)) + return false; + // We should cut out output_terminator_ at the end of given buffer, + // appended by addr2line to mark the end of its meaningful output. + // We cannot scan buffer from it's beginning, because it is legal for it + // to start with output_terminator_ in case given offset is invalid. So, + // scanning from second character. + char *garbage = internal_strstr(buffer + 1, output_terminator_); + // This should never be NULL since buffer must end up with + // output_terminator_. + CHECK(garbage); + // Trim the buffer. + garbage[0] = '\0'; + return true; } const char *module_name_; // Owned, leaked. + static const char output_terminator_[]; }; +const char Addr2LineProcess::output_terminator_[] = "??\n??:0\n"; + +bool Addr2LineProcess::ReachedEndOfOutput(const char *buffer, + uptr length) const { + const size_t kTerminatorLen = sizeof(output_terminator_) - 1; + // Skip, if we read just kTerminatorLen bytes, because Addr2Line output + // should consist at least of two pairs of lines: + // 1. First one, corresponding to given offset to be symbolized + // (may be equal to output_terminator_, if offset is not valid). + // 2. Second one for output_terminator_, itself to mark the end of output. + if (length <= kTerminatorLen) return false; + // Addr2Line output should end up with output_terminator_. + return !internal_memcmp(buffer + length - kTerminatorLen, + output_terminator_, kTerminatorLen); +} + class Addr2LinePool : public SymbolizerTool { public: explicit Addr2LinePool(const char *addr2line_path, @@ -260,15 +278,18 @@ class Addr2LinePool : public SymbolizerTool { addr2line_pool_.push_back(addr2line); } CHECK_EQ(0, internal_strcmp(module_name, addr2line->module_name())); - char buffer_[kBufferSize]; - internal_snprintf(buffer_, kBufferSize, "0x%zx\n", module_offset); - return addr2line->SendCommand(buffer_); + char buffer[kBufferSize]; + internal_snprintf(buffer, kBufferSize, "0x%zx\n0x%zx\n", + module_offset, dummy_address_); + return addr2line->SendCommand(buffer); } - static const uptr kBufferSize = 32; + static const uptr kBufferSize = 64; const char *addr2line_path_; LowLevelAllocator *allocator_; InternalMmapVector<Addr2LineProcess*> addr2line_pool_; + static const uptr dummy_address_ = + FIRST_32_SECOND_64(UINT32_MAX, UINT64_MAX); }; #if SANITIZER_SUPPORTS_WEAK_HOOKS @@ -425,8 +446,6 @@ static void ChooseSymbolizerTools(IntrusiveList<SymbolizerTool> *list, if (SymbolizerTool *tool = ChooseExternalSymbolizer(allocator)) { list->push_back(tool); - } else { - VReport(2, "No internal or external symbolizer found.\n"); } #if SANITIZER_MAC diff --git a/lib/sanitizer_common/sanitizer_symbolizer_process_libcdep.cc b/lib/sanitizer_common/sanitizer_symbolizer_process_libcdep.cc deleted file mode 100644 index f1c01a332499..000000000000 --- a/lib/sanitizer_common/sanitizer_symbolizer_process_libcdep.cc +++ /dev/null @@ -1,229 +0,0 @@ -//===-- sanitizer_symbolizer_process_libcdep.cc ---------------------------===// -// -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. -// -//===----------------------------------------------------------------------===// -// -// Implementation of SymbolizerProcess used by external symbolizers. -// -//===----------------------------------------------------------------------===// - -#include "sanitizer_platform.h" -#if SANITIZER_POSIX -#include "sanitizer_posix.h" -#include "sanitizer_symbolizer_internal.h" - -#include <errno.h> -#include <stdlib.h> -#include <sys/wait.h> -#include <unistd.h> - -#if SANITIZER_MAC -#include <util.h> // for forkpty() -#endif // SANITIZER_MAC - -namespace __sanitizer { - -SymbolizerProcess::SymbolizerProcess(const char *path, bool use_forkpty) - : path_(path), - input_fd_(kInvalidFd), - output_fd_(kInvalidFd), - times_restarted_(0), - failed_to_start_(false), - reported_invalid_path_(false), - use_forkpty_(use_forkpty) { - CHECK(path_); - CHECK_NE(path_[0], '\0'); -} - -const char *SymbolizerProcess::SendCommand(const char *command) { - for (; times_restarted_ < kMaxTimesRestarted; times_restarted_++) { - // Start or restart symbolizer if we failed to send command to it. - if (const char *res = SendCommandImpl(command)) - return res; - Restart(); - } - if (!failed_to_start_) { - Report("WARNING: Failed to use and restart external symbolizer!\n"); - failed_to_start_ = true; - } - return 0; -} - -bool SymbolizerProcess::Restart() { - if (input_fd_ != kInvalidFd) - internal_close(input_fd_); - if (output_fd_ != kInvalidFd) - internal_close(output_fd_); - return StartSymbolizerSubprocess(); -} - -const char *SymbolizerProcess::SendCommandImpl(const char *command) { - if (input_fd_ == kInvalidFd || output_fd_ == kInvalidFd) - return 0; - if (!WriteToSymbolizer(command, internal_strlen(command))) - return 0; - if (!ReadFromSymbolizer(buffer_, kBufferSize)) - return 0; - return buffer_; -} - -bool SymbolizerProcess::ReadFromSymbolizer(char *buffer, uptr max_length) { - if (max_length == 0) - return true; - uptr read_len = 0; - while (true) { - uptr just_read = internal_read(input_fd_, buffer + read_len, - max_length - read_len - 1); - // We can't read 0 bytes, as we don't expect external symbolizer to close - // its stdout. - if (just_read == 0 || just_read == (uptr)-1) { - Report("WARNING: Can't read from symbolizer at fd %d\n", input_fd_); - return false; - } - read_len += just_read; - if (ReachedEndOfOutput(buffer, read_len)) - break; - } - buffer[read_len] = '\0'; - return true; -} - -bool SymbolizerProcess::WriteToSymbolizer(const char *buffer, uptr length) { - if (length == 0) - return true; - uptr write_len = internal_write(output_fd_, buffer, length); - if (write_len == 0 || write_len == (uptr)-1) { - Report("WARNING: Can't write to symbolizer at fd %d\n", output_fd_); - return false; - } - return true; -} - -bool SymbolizerProcess::StartSymbolizerSubprocess() { - if (!FileExists(path_)) { - if (!reported_invalid_path_) { - Report("WARNING: invalid path to external symbolizer!\n"); - reported_invalid_path_ = true; - } - return false; - } - - int pid; - if (use_forkpty_) { -#if SANITIZER_MAC - fd_t fd = kInvalidFd; - // Use forkpty to disable buffering in the new terminal. - pid = forkpty(&fd, 0, 0, 0); - if (pid == -1) { - // forkpty() failed. - Report("WARNING: failed to fork external symbolizer (errno: %d)\n", - errno); - return false; - } else if (pid == 0) { - // Child subprocess. - ExecuteWithDefaultArgs(path_); - internal__exit(1); - } - - // Continue execution in parent process. - input_fd_ = output_fd_ = fd; - - // Disable echo in the new terminal, disable CR. - struct termios termflags; - tcgetattr(fd, &termflags); - termflags.c_oflag &= ~ONLCR; - termflags.c_lflag &= ~ECHO; - tcsetattr(fd, TCSANOW, &termflags); -#else // SANITIZER_MAC - UNIMPLEMENTED(); -#endif // SANITIZER_MAC - } else { - int *infd = NULL; - int *outfd = NULL; - // The client program may close its stdin and/or stdout and/or stderr - // thus allowing socketpair to reuse file descriptors 0, 1 or 2. - // In this case the communication between the forked processes may be - // broken if either the parent or the child tries to close or duplicate - // these descriptors. The loop below produces two pairs of file - // descriptors, each greater than 2 (stderr). - int sock_pair[5][2]; - for (int i = 0; i < 5; i++) { - if (pipe(sock_pair[i]) == -1) { - for (int j = 0; j < i; j++) { - internal_close(sock_pair[j][0]); - internal_close(sock_pair[j][1]); - } - Report("WARNING: Can't create a socket pair to start " - "external symbolizer (errno: %d)\n", errno); - return false; - } else if (sock_pair[i][0] > 2 && sock_pair[i][1] > 2) { - if (infd == NULL) { - infd = sock_pair[i]; - } else { - outfd = sock_pair[i]; - for (int j = 0; j < i; j++) { - if (sock_pair[j] == infd) continue; - internal_close(sock_pair[j][0]); - internal_close(sock_pair[j][1]); - } - break; - } - } - } - CHECK(infd); - CHECK(outfd); - - // Real fork() may call user callbacks registered with pthread_atfork(). - pid = internal_fork(); - if (pid == -1) { - // Fork() failed. - internal_close(infd[0]); - internal_close(infd[1]); - internal_close(outfd[0]); - internal_close(outfd[1]); - Report("WARNING: failed to fork external symbolizer " - " (errno: %d)\n", errno); - return false; - } else if (pid == 0) { - // Child subprocess. - internal_close(STDOUT_FILENO); - internal_close(STDIN_FILENO); - internal_dup2(outfd[0], STDIN_FILENO); - internal_dup2(infd[1], STDOUT_FILENO); - internal_close(outfd[0]); - internal_close(outfd[1]); - internal_close(infd[0]); - internal_close(infd[1]); - for (int fd = sysconf(_SC_OPEN_MAX); fd > 2; fd--) - internal_close(fd); - ExecuteWithDefaultArgs(path_); - internal__exit(1); - } - - // Continue execution in parent process. - internal_close(outfd[0]); - internal_close(infd[1]); - input_fd_ = infd[0]; - output_fd_ = outfd[1]; - } - - // Check that symbolizer subprocess started successfully. - int pid_status; - SleepForMillis(kSymbolizerStartupTimeMillis); - int exited_pid = waitpid(pid, &pid_status, WNOHANG); - if (exited_pid != 0) { - // Either waitpid failed, or child has already exited. - Report("WARNING: external symbolizer didn't start up correctly!\n"); - return false; - } - - return true; -} - -} // namespace __sanitizer - -#endif // SANITIZER_POSIX diff --git a/lib/sanitizer_common/sanitizer_symbolizer_win.cc b/lib/sanitizer_common/sanitizer_symbolizer_win.cc index 31f374687e96..b1dceebf45ce 100644 --- a/lib/sanitizer_common/sanitizer_symbolizer_win.cc +++ b/lib/sanitizer_common/sanitizer_symbolizer_win.cc @@ -14,17 +14,26 @@ #include "sanitizer_platform.h" #if SANITIZER_WINDOWS +#define WIN32_LEAN_AND_MEAN #include <windows.h> #include <dbghelp.h> #pragma comment(lib, "dbghelp.lib") -#include "sanitizer_symbolizer_win.h" #include "sanitizer_symbolizer_internal.h" namespace __sanitizer { namespace { +class WinSymbolizerTool : public SymbolizerTool { + public: + bool SymbolizePC(uptr addr, SymbolizedStack *stack) override; + bool SymbolizeData(uptr addr, DataInfo *info) override { + return false; + } + const char *Demangle(const char *name) override; +}; + bool is_dbghelp_initialized = false; bool TrySymInitialize() { @@ -115,7 +124,9 @@ bool WinSymbolizerTool::SymbolizePC(uptr addr, SymbolizedStack *frame) { frame->info.file = internal_strdup(line_info.FileName); frame->info.line = line_info.LineNumber; } - return true; + // Only consider this a successful symbolization attempt if we got file info. + // Otherwise, try llvm-symbolizer. + return got_fileline; } const char *WinSymbolizerTool::Demangle(const char *name) { @@ -137,10 +148,134 @@ void Symbolizer::PlatformPrepareForSandboxing() { // Do nothing. } +namespace { +struct ScopedHandle { + ScopedHandle() : h_(nullptr) {} + explicit ScopedHandle(HANDLE h) : h_(h) {} + ~ScopedHandle() { + if (h_) + ::CloseHandle(h_); + } + HANDLE get() { return h_; } + HANDLE *receive() { return &h_; } + HANDLE release() { + HANDLE h = h_; + h_ = nullptr; + return h; + } + HANDLE h_; +}; +} // namespace + +bool SymbolizerProcess::StartSymbolizerSubprocess() { + // Create inherited pipes for stdin and stdout. + ScopedHandle stdin_read, stdin_write; + ScopedHandle stdout_read, stdout_write; + SECURITY_ATTRIBUTES attrs; + attrs.nLength = sizeof(SECURITY_ATTRIBUTES); + attrs.bInheritHandle = TRUE; + attrs.lpSecurityDescriptor = nullptr; + if (!::CreatePipe(stdin_read.receive(), stdin_write.receive(), &attrs, 0) || + !::CreatePipe(stdout_read.receive(), stdout_write.receive(), &attrs, 0)) { + VReport(2, "WARNING: %s CreatePipe failed (error code: %d)\n", + SanitizerToolName, path_, GetLastError()); + return false; + } + + // Don't inherit the writing end of stdin or the reading end of stdout. + if (!SetHandleInformation(stdin_write.get(), HANDLE_FLAG_INHERIT, 0) || + !SetHandleInformation(stdout_read.get(), HANDLE_FLAG_INHERIT, 0)) { + VReport(2, "WARNING: %s SetHandleInformation failed (error code: %d)\n", + SanitizerToolName, path_, GetLastError()); + return false; + } + + // Compute the command line. Wrap double quotes around everything. + const char *argv[kArgVMax]; + GetArgV(path_, argv); + InternalScopedString command_line(kMaxPathLength * 3); + for (int i = 0; argv[i]; i++) { + const char *arg = argv[i]; + int arglen = internal_strlen(arg); + // Check that tool command lines are simple and that complete escaping is + // unnecessary. + CHECK(!internal_strchr(arg, '"') && "quotes in args unsupported"); + CHECK(!internal_strstr(arg, "\\\\") && + "double backslashes in args unsupported"); + CHECK(arglen > 0 && arg[arglen - 1] != '\\' && + "args ending in backslash and empty args unsupported"); + command_line.append("\"%s\" ", arg); + } + VReport(3, "Launching symbolizer command: %s\n", command_line.data()); + + // Launch llvm-symbolizer with stdin and stdout redirected. + STARTUPINFOA si; + memset(&si, 0, sizeof(si)); + si.cb = sizeof(si); + si.dwFlags |= STARTF_USESTDHANDLES; + si.hStdInput = stdin_read.get(); + si.hStdOutput = stdout_write.get(); + PROCESS_INFORMATION pi; + memset(&pi, 0, sizeof(pi)); + if (!CreateProcessA(path_, // Executable + command_line.data(), // Command line + nullptr, // Process handle not inheritable + nullptr, // Thread handle not inheritable + TRUE, // Set handle inheritance to TRUE + 0, // Creation flags + nullptr, // Use parent's environment block + nullptr, // Use parent's starting directory + &si, &pi)) { + VReport(2, "WARNING: %s failed to create process for %s (error code: %d)\n", + SanitizerToolName, path_, GetLastError()); + return false; + } + + // Process creation succeeded, so transfer handle ownership into the fields. + input_fd_ = stdout_read.release(); + output_fd_ = stdin_write.release(); + + // The llvm-symbolizer process is responsible for quitting itself when the + // stdin pipe is closed, so we don't need these handles. Close them to prevent + // leaks. If we ever want to try to kill the symbolizer process from the + // parent, we'll want to hang on to these handles. + CloseHandle(pi.hProcess); + CloseHandle(pi.hThread); + return true; +} + +static void ChooseSymbolizerTools(IntrusiveList<SymbolizerTool> *list, + LowLevelAllocator *allocator) { + if (!common_flags()->symbolize) { + VReport(2, "Symbolizer is disabled.\n"); + return; + } + + // Add llvm-symbolizer in case the binary has dwarf. + const char *user_path = common_flags()->external_symbolizer_path; + const char *path = + user_path ? user_path : FindPathToBinary("llvm-symbolizer.exe"); + if (path) { + VReport(2, "Using llvm-symbolizer at %spath: %s\n", + user_path ? "user-specified " : "", path); + list->push_back(new(*allocator) LLVMSymbolizer(path, allocator)); + } else { + if (user_path && user_path[0] == '\0') { + VReport(2, "External symbolizer is explicitly disabled.\n"); + } else { + VReport(2, "External symbolizer is not present.\n"); + } + } + + // Add the dbghelp based symbolizer. + list->push_back(new(*allocator) WinSymbolizerTool()); +} + Symbolizer *Symbolizer::PlatformInit() { IntrusiveList<SymbolizerTool> list; list.clear(); - list.push_back(new(symbolizer_allocator_) WinSymbolizerTool()); + ChooseSymbolizerTools(&list, &symbolizer_allocator_); + return new(symbolizer_allocator_) Symbolizer(list); } diff --git a/lib/sanitizer_common/sanitizer_symbolizer_win.h b/lib/sanitizer_common/sanitizer_symbolizer_win.h deleted file mode 100644 index 72ac5e5ee104..000000000000 --- a/lib/sanitizer_common/sanitizer_symbolizer_win.h +++ /dev/null @@ -1,31 +0,0 @@ -//===-- sanitizer_symbolizer_win.h ------------------------------*- C++ -*-===// -// -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. -// -//===----------------------------------------------------------------------===// -// -// Header file for the Windows symbolizer tool. -// -//===----------------------------------------------------------------------===// -#ifndef SANITIZER_SYMBOLIZER_WIN_H -#define SANITIZER_SYMBOLIZER_WIN_H - -#include "sanitizer_symbolizer_internal.h" - -namespace __sanitizer { - -class WinSymbolizerTool : public SymbolizerTool { - public: - bool SymbolizePC(uptr addr, SymbolizedStack *stack) override; - bool SymbolizeData(uptr addr, DataInfo *info) override { - return false; - } - const char *Demangle(const char *name) override; -}; - -} // namespace __sanitizer - -#endif // SANITIZER_SYMBOLIZER_WIN_H diff --git a/lib/sanitizer_common/sanitizer_syscall_linux_aarch64.inc b/lib/sanitizer_common/sanitizer_syscall_linux_aarch64.inc new file mode 100644 index 000000000000..7ab1d7641449 --- /dev/null +++ b/lib/sanitizer_common/sanitizer_syscall_linux_aarch64.inc @@ -0,0 +1,138 @@ +//===-- sanitizer_syscall_linux_aarch64.inc --------------------*- C++ -*-===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// +// +// Implementations of internal_syscall and internal_iserror for Linux/aarch64. +// +//===----------------------------------------------------------------------===// + +#define SYSCALL(name) __NR_ ## name + +static uptr __internal_syscall(u64 nr) { + register u64 x8 asm("x8") = nr; + register u64 x0 asm("x0"); + asm volatile("svc 0" + : "=r"(x0) + : "r"(x8) + : "memory", "cc"); + return x0; +} +#define __internal_syscall0(n) \ + (__internal_syscall)(n) + +static uptr __internal_syscall(u64 nr, u64 arg1) { + register u64 x8 asm("x8") = nr; + register u64 x0 asm("x0") = arg1; + asm volatile("svc 0" + : "=r"(x0) + : "r"(x8), "0"(x0) + : "memory", "cc"); + return x0; +} +#define __internal_syscall1(n, a1) \ + (__internal_syscall)(n, (u64)(a1)) + +static uptr __internal_syscall(u64 nr, u64 arg1, long arg2) { + register u64 x8 asm("x8") = nr; + register u64 x0 asm("x0") = arg1; + register u64 x1 asm("x1") = arg2; + asm volatile("svc 0" + : "=r"(x0) + : "r"(x8), "0"(x0), "r"(x1) + : "memory", "cc"); + return x0; +} +#define __internal_syscall2(n, a1, a2) \ + (__internal_syscall)(n, (u64)(a1), (long)(a2)) + +static uptr __internal_syscall(u64 nr, u64 arg1, long arg2, long arg3) { + register u64 x8 asm("x8") = nr; + register u64 x0 asm("x0") = arg1; + register u64 x1 asm("x1") = arg2; + register u64 x2 asm("x2") = arg3; + asm volatile("svc 0" + : "=r"(x0) + : "r"(x8), "0"(x0), "r"(x1), "r"(x2) + : "memory", "cc"); + return x0; +} +#define __internal_syscall3(n, a1, a2, a3) \ + (__internal_syscall)(n, (u64)(a1), (long)(a2), (long)(a3)) + +static uptr __internal_syscall(u64 nr, u64 arg1, long arg2, long arg3, + u64 arg4) { + register u64 x8 asm("x8") = nr; + register u64 x0 asm("x0") = arg1; + register u64 x1 asm("x1") = arg2; + register u64 x2 asm("x2") = arg3; + register u64 x3 asm("x3") = arg4; + asm volatile("svc 0" + : "=r"(x0) + : "r"(x8), "0"(x0), "r"(x1), "r"(x2), "r"(x3) + : "memory", "cc"); + return x0; +} +#define __internal_syscall4(n, a1, a2, a3, a4) \ + (__internal_syscall)(n, (u64)(a1), (long)(a2), (long)(a3), (long)(a4)) + +static uptr __internal_syscall(u64 nr, u64 arg1, long arg2, long arg3, + u64 arg4, long arg5) { + register u64 x8 asm("x8") = nr; + register u64 x0 asm("x0") = arg1; + register u64 x1 asm("x1") = arg2; + register u64 x2 asm("x2") = arg3; + register u64 x3 asm("x3") = arg4; + register u64 x4 asm("x4") = arg5; + asm volatile("svc 0" + : "=r"(x0) + : "r"(x8), "0"(x0), "r"(x1), "r"(x2), "r"(x3), "r"(x4) + : "memory", "cc"); + return x0; +} +#define __internal_syscall5(n, a1, a2, a3, a4, a5) \ + (__internal_syscall)(n, (u64)(a1), (long)(a2), (long)(a3), (long)(a4), \ + (u64)(a5)) + +static uptr __internal_syscall(u64 nr, u64 arg1, long arg2, long arg3, + u64 arg4, long arg5, long arg6) { + register u64 x8 asm("x8") = nr; + register u64 x0 asm("x0") = arg1; + register u64 x1 asm("x1") = arg2; + register u64 x2 asm("x2") = arg3; + register u64 x3 asm("x3") = arg4; + register u64 x4 asm("x4") = arg5; + register u64 x5 asm("x5") = arg6; + asm volatile("svc 0" + : "=r"(x0) + : "r"(x8), "0"(x0), "r"(x1), "r"(x2), "r"(x3), "r"(x4), "r"(x5) + : "memory", "cc"); + return x0; +} +#define __internal_syscall6(n, a1, a2, a3, a4, a5, a6) \ + (__internal_syscall)(n, (u64)(a1), (long)(a2), (long)(a3), (long)(a4), \ + (u64)(a5), (long)(a6)) + +#define __SYSCALL_NARGS_X(a1, a2, a3, a4, a5, a6, a7, a8, n, ...) n +#define __SYSCALL_NARGS(...) \ + __SYSCALL_NARGS_X(__VA_ARGS__, 7, 6, 5, 4, 3, 2, 1, 0, ) +#define __SYSCALL_CONCAT_X(a, b) a##b +#define __SYSCALL_CONCAT(a, b) __SYSCALL_CONCAT_X(a, b) +#define __SYSCALL_DISP(b, ...) \ + __SYSCALL_CONCAT(b, __SYSCALL_NARGS(__VA_ARGS__))(__VA_ARGS__) + +#define internal_syscall(...) __SYSCALL_DISP(__internal_syscall, __VA_ARGS__) + +// Helper function used to avoid cobbler errno. +bool internal_iserror(uptr retval, int *rverrno) { + if (retval >= (uptr)-4095) { + if (rverrno) + *rverrno = -retval; + return true; + } + return false; +} diff --git a/lib/sanitizer_common/sanitizer_thread_registry.h b/lib/sanitizer_common/sanitizer_thread_registry.h index 5d9c3b9694e8..a27bbb376e85 100644 --- a/lib/sanitizer_common/sanitizer_thread_registry.h +++ b/lib/sanitizer_common/sanitizer_thread_registry.h @@ -79,7 +79,8 @@ class ThreadRegistry { ThreadRegistry(ThreadContextFactory factory, u32 max_threads, u32 thread_quarantine_size, u32 max_reuse = 0); - void GetNumberOfThreads(uptr *total = 0, uptr *running = 0, uptr *alive = 0); + void GetNumberOfThreads(uptr *total = nullptr, uptr *running = nullptr, + uptr *alive = nullptr); uptr GetMaxAliveThreads(); void Lock() { mtx_.Lock(); } @@ -142,7 +143,6 @@ class ThreadRegistry { typedef GenericScopedLock<ThreadRegistry> ThreadRegistryLock; -} // namespace __sanitizer - -#endif // SANITIZER_THREAD_REGISTRY_H +} // namespace __sanitizer +#endif // SANITIZER_THREAD_REGISTRY_H diff --git a/lib/sanitizer_common/sanitizer_tls_get_addr.cc b/lib/sanitizer_common/sanitizer_tls_get_addr.cc index ea037159d00b..213aced89da7 100644 --- a/lib/sanitizer_common/sanitizer_tls_get_addr.cc +++ b/lib/sanitizer_common/sanitizer_tls_get_addr.cc @@ -78,6 +78,15 @@ void DTLS_Destroy() { DTLS_Deallocate(dtls.dtv, s); } +#if defined(__powerpc64__) +// This is glibc's TLS_DTV_OFFSET: +// "Dynamic thread vector pointers point 0x8000 past the start of each +// TLS block." +static const uptr kDtvOffset = 0x8000; +#else +static const uptr kDtvOffset = 0; +#endif + DTLS::DTV *DTLS_on_tls_get_addr(void *arg_void, void *res, uptr static_tls_begin, uptr static_tls_end) { if (!common_flags()->intercept_tls_get_addr) return 0; @@ -87,7 +96,7 @@ DTLS::DTV *DTLS_on_tls_get_addr(void *arg_void, void *res, DTLS_Resize(dso_id + 1); if (dtls.dtv[dso_id].beg) return 0; uptr tls_size = 0; - uptr tls_beg = reinterpret_cast<uptr>(res) - arg->offset; + uptr tls_beg = reinterpret_cast<uptr>(res) - arg->offset - kDtvOffset; VPrintf(2, "__tls_get_addr: %p {%p,%p} => %p; tls_beg: %p; sp: %p " "num_live_dtls %zd\n", arg, arg->dso_id, arg->offset, res, tls_beg, &tls_beg, diff --git a/lib/sanitizer_common/sanitizer_win.cc b/lib/sanitizer_common/sanitizer_win.cc index d5bbe4565068..861261d8402c 100644 --- a/lib/sanitizer_common/sanitizer_win.cc +++ b/lib/sanitizer_common/sanitizer_win.cc @@ -51,7 +51,7 @@ uptr GetMaxVirtualAddress() { } bool FileExists(const char *filename) { - UNIMPLEMENTED(); + return ::GetFileAttributesA(filename) != INVALID_FILE_ATTRIBUTES; } uptr internal_getpid() { @@ -83,14 +83,11 @@ void GetThreadStackTopAndBottom(bool at_initialization, uptr *stack_top, } #endif // #if !SANITIZER_GO -void *MmapOrDie(uptr size, const char *mem_type) { +void *MmapOrDie(uptr size, const char *mem_type, bool raw_report) { void *rv = VirtualAlloc(0, size, MEM_RESERVE | MEM_COMMIT, PAGE_READWRITE); - if (rv == 0) { - Report("ERROR: %s failed to " - "allocate 0x%zx (%zd) bytes of %s (error code: %d)\n", - SanitizerToolName, size, size, mem_type, GetLastError()); - CHECK("unable to mmap" && 0); - } + if (rv == 0) + ReportMmapFailureAndDie(size, mem_type, "allocate", + GetLastError(), raw_report); return rv; } @@ -224,12 +221,14 @@ struct ModuleInfo { uptr end_address; }; +#ifndef SANITIZER_GO int CompareModulesBase(const void *pl, const void *pr) { const ModuleInfo *l = (ModuleInfo *)pl, *r = (ModuleInfo *)pr; if (l->base_address < r->base_address) return -1; return l->base_address > r->base_address; } +#endif } // namespace #ifndef SANITIZER_GO @@ -292,11 +291,6 @@ void SetAddressSpaceUnlimited() { UNIMPLEMENTED(); } -char *FindPathToBinary(const char *name) { - // Nothing here for now. - return 0; -} - bool IsPathSeparator(const char c) { return c == '\\' || c == '/'; } @@ -323,6 +317,59 @@ void Abort() { internal__exit(3); } +// Read the file to extract the ImageBase field from the PE header. If ASLR is +// disabled and this virtual address is available, the loader will typically +// load the image at this address. Therefore, we call it the preferred base. Any +// addresses in the DWARF typically assume that the object has been loaded at +// this address. +static uptr GetPreferredBase(const char *modname) { + fd_t fd = OpenFile(modname, RdOnly, nullptr); + if (fd == kInvalidFd) + return 0; + FileCloser closer(fd); + + // Read just the DOS header. + IMAGE_DOS_HEADER dos_header; + uptr bytes_read; + if (!ReadFromFile(fd, &dos_header, sizeof(dos_header), &bytes_read) || + bytes_read != sizeof(dos_header)) + return 0; + + // The file should start with the right signature. + if (dos_header.e_magic != IMAGE_DOS_SIGNATURE) + return 0; + + // The layout at e_lfanew is: + // "PE\0\0" + // IMAGE_FILE_HEADER + // IMAGE_OPTIONAL_HEADER + // Seek to e_lfanew and read all that data. + char buf[4 + sizeof(IMAGE_FILE_HEADER) + sizeof(IMAGE_OPTIONAL_HEADER)]; + if (::SetFilePointer(fd, dos_header.e_lfanew, nullptr, FILE_BEGIN) == + INVALID_SET_FILE_POINTER) + return 0; + if (!ReadFromFile(fd, &buf[0], sizeof(buf), &bytes_read) || + bytes_read != sizeof(buf)) + return 0; + + // Check for "PE\0\0" before the PE header. + char *pe_sig = &buf[0]; + if (internal_memcmp(pe_sig, "PE\0\0", 4) != 0) + return 0; + + // Skip over IMAGE_FILE_HEADER. We could do more validation here if we wanted. + IMAGE_OPTIONAL_HEADER *pe_header = + (IMAGE_OPTIONAL_HEADER *)(pe_sig + 4 + sizeof(IMAGE_FILE_HEADER)); + + // Check for more magic in the PE header. + if (pe_header->Magic != IMAGE_NT_OPTIONAL_HDR_MAGIC) + return 0; + + // Finally, return the ImageBase. + return (uptr)pe_header->ImageBase; +} + +#ifndef SANITIZER_GO uptr GetListOfModules(LoadedModule *modules, uptr max_modules, string_predicate_t filter) { HANDLE cur_process = GetCurrentProcess(); @@ -355,19 +402,33 @@ uptr GetListOfModules(LoadedModule *modules, uptr max_modules, if (!GetModuleInformation(cur_process, handle, &mi, sizeof(mi))) continue; - char module_name[MAX_PATH]; - bool got_module_name = - GetModuleFileNameA(handle, module_name, sizeof(module_name)); - if (!got_module_name) - module_name[0] = '\0'; + // Get the UTF-16 path and convert to UTF-8. + wchar_t modname_utf16[kMaxPathLength]; + int modname_utf16_len = + GetModuleFileNameW(handle, modname_utf16, kMaxPathLength); + if (modname_utf16_len == 0) + modname_utf16[0] = '\0'; + char module_name[kMaxPathLength]; + int module_name_len = + ::WideCharToMultiByte(CP_UTF8, 0, modname_utf16, modname_utf16_len + 1, + &module_name[0], kMaxPathLength, NULL, NULL); + module_name[module_name_len] = '\0'; if (filter && !filter(module_name)) continue; uptr base_address = (uptr)mi.lpBaseOfDll; uptr end_address = (uptr)mi.lpBaseOfDll + mi.SizeOfImage; + + // Adjust the base address of the module so that we get a VA instead of an + // RVA when computing the module offset. This helps llvm-symbolizer find the + // right DWARF CU. In the common case that the image is loaded at it's + // preferred address, we will now print normal virtual addresses. + uptr preferred_base = GetPreferredBase(&module_name[0]); + uptr adjusted_base = base_address - preferred_base; + LoadedModule *cur_module = &modules[count]; - cur_module->set(module_name, base_address); + cur_module->set(module_name, adjusted_base); // We add the whole module as one single address range. cur_module->addAddressRange(base_address, end_address, /*executable*/ true); count++; @@ -377,7 +438,6 @@ uptr GetListOfModules(LoadedModule *modules, uptr max_modules, return count; }; -#ifndef SANITIZER_GO // We can't use atexit() directly at __asan_init time as the CRT is not fully // initialized at this point. Place the functions into a vector and use // atexit() as soon as it is ready for use (i.e. after .CRT$XIC initializers). @@ -397,15 +457,22 @@ static int RunAtexit() { } #pragma section(".CRT$XID", long, read) // NOLINT -static __declspec(allocate(".CRT$XID")) int (*__run_atexit)() = RunAtexit; +__declspec(allocate(".CRT$XID")) int (*__run_atexit)() = RunAtexit; #endif // ------------------ sanitizer_libc.h fd_t OpenFile(const char *filename, FileAccessMode mode, error_t *last_error) { - if (mode != WrOnly) + fd_t res; + if (mode == RdOnly) { + res = CreateFile(filename, GENERIC_READ, + FILE_SHARE_READ | FILE_SHARE_WRITE | FILE_SHARE_DELETE, + nullptr, OPEN_EXISTING, FILE_ATTRIBUTE_NORMAL, nullptr); + } else if (mode == WrOnly) { + res = CreateFile(filename, GENERIC_WRITE, 0, nullptr, CREATE_ALWAYS, + FILE_ATTRIBUTE_NORMAL, nullptr); + } else { UNIMPLEMENTED(); - fd_t res = CreateFile(filename, GENERIC_WRITE, 0, nullptr, CREATE_ALWAYS, - FILE_ATTRIBUTE_NORMAL, nullptr); + } CHECK(res != kStdoutFd || kStdoutFd == kInvalidFd); CHECK(res != kStderrFd || kStderrFd == kInvalidFd); if (res == kInvalidFd && last_error) @@ -419,7 +486,18 @@ void CloseFile(fd_t fd) { bool ReadFromFile(fd_t fd, void *buff, uptr buff_size, uptr *bytes_read, error_t *error_p) { - UNIMPLEMENTED(); + CHECK(fd != kInvalidFd); + + // bytes_read can't be passed directly to ReadFile: + // uptr is unsigned long long on 64-bit Windows. + unsigned long num_read_long; + + bool success = ::ReadFile(fd, buff, buff_size, &num_read_long, nullptr); + if (!success && error_p) + *error_p = GetLastError(); + if (bytes_read) + *bytes_read = num_read_long; + return success; } bool SupportsColoredOutput(fd_t fd) { @@ -431,21 +509,32 @@ bool WriteToFile(fd_t fd, const void *buff, uptr buff_size, uptr *bytes_written, error_t *error_p) { CHECK(fd != kInvalidFd); - if (fd == kStdoutFd) { - fd = GetStdHandle(STD_OUTPUT_HANDLE); - if (fd == 0) fd = kInvalidFd; - } else if (fd == kStderrFd) { - fd = GetStdHandle(STD_ERROR_HANDLE); - if (fd == 0) fd = kInvalidFd; + // Handle null optional parameters. + error_t dummy_error; + error_p = error_p ? error_p : &dummy_error; + uptr dummy_bytes_written; + bytes_written = bytes_written ? bytes_written : &dummy_bytes_written; + + // Initialize output parameters in case we fail. + *error_p = 0; + *bytes_written = 0; + + // Map the conventional Unix fds 1 and 2 to Windows handles. They might be + // closed, in which case this will fail. + if (fd == kStdoutFd || fd == kStderrFd) { + fd = GetStdHandle(fd == kStdoutFd ? STD_OUTPUT_HANDLE : STD_ERROR_HANDLE); + if (fd == 0) { + *error_p = ERROR_INVALID_HANDLE; + return false; + } } - DWORD internal_bytes_written; - if (fd == kInvalidFd || - WriteFile(fd, buff, buff_size, &internal_bytes_written, 0)) { - if (error_p) *error_p = GetLastError(); + DWORD bytes_written_32; + if (!WriteFile(fd, buff, buff_size, &bytes_written_32, 0)) { + *error_p = GetLastError(); return false; } else { - if (bytes_written) *bytes_written = internal_bytes_written; + *bytes_written = bytes_written_32; return true; } } @@ -665,6 +754,22 @@ uptr ReadBinaryName(/*out*/char *buf, uptr buf_len) { return 0; } +uptr ReadLongProcessName(/*out*/char *buf, uptr buf_len) { + return ReadBinaryName(buf, buf_len); +} + +void CheckVMASize() { + // Do nothing. +} + +void DisableReexec() { + // No need to re-exec on Windows. +} + +void MaybeReexec() { + // No need to re-exec on Windows. +} + } // namespace __sanitizer #endif // _WIN32 diff --git a/lib/sanitizer_common/scripts/gen_dynamic_list.py b/lib/sanitizer_common/scripts/gen_dynamic_list.py index f055bb44ba21..b8b79b5994b7 100755 --- a/lib/sanitizer_common/scripts/gen_dynamic_list.py +++ b/lib/sanitizer_common/scripts/gen_dynamic_list.py @@ -100,7 +100,7 @@ def main(argv): print('global:') result.sort() for f in result: - print(' ' + f.encode('utf-8') + ';') + print(u' %s;' % f) if args.version_list: print('local:') print(' *;') diff --git a/lib/sanitizer_common/tests/CMakeLists.txt b/lib/sanitizer_common/tests/CMakeLists.txt index b0165eac25f0..18b76369a675 100644 --- a/lib/sanitizer_common/tests/CMakeLists.txt +++ b/lib/sanitizer_common/tests/CMakeLists.txt @@ -4,6 +4,9 @@ clang_compiler_add_cxx_check() # FIXME: use SANITIZER_COMMON_SUPPORTED_ARCH here filter_available_targets(SANITIZER_UNITTEST_SUPPORTED_ARCH x86_64 i386 mips64 mips64el) +if(APPLE) + darwin_filter_host_archs(SANITIZER_UNITTEST_SUPPORTED_ARCH SANITIZER_COMMON_SUPPORTED_ARCH) +endif() set(SANITIZER_UNITTESTS sanitizer_allocator_test.cc @@ -72,7 +75,7 @@ if(ANDROID) endif() set(SANITIZER_TEST_LINK_LIBS) -append_list_if(ANDROID log SANITIZER_TEST_LINK_LIBS) +append_list_if(COMPILER_RT_HAS_LIBLOG log SANITIZER_TEST_LINK_LIBS) # NDK r10 requires -latomic almost always. append_list_if(ANDROID atomic SANITIZER_TEST_LINK_LIBS) @@ -168,11 +171,13 @@ if(COMPILER_RT_CAN_EXECUTE_TESTS AND NOT ANDROID) # be sure that produced binaries would work. if(APPLE) add_sanitizer_common_lib("RTSanitizerCommon.test.osx" - $<TARGET_OBJECTS:RTSanitizerCommon.osx>) + $<TARGET_OBJECTS:RTSanitizerCommon.osx> + $<TARGET_OBJECTS:RTSanitizerCommonLibc.osx>) else() if(CAN_TARGET_x86_64) add_sanitizer_common_lib("RTSanitizerCommon.test.nolibc.x86_64" - $<TARGET_OBJECTS:RTSanitizerCommon.x86_64>) + $<TARGET_OBJECTS:RTSanitizerCommon.x86_64> + $<TARGET_OBJECTS:RTSanitizerCommonNoLibc.x86_64>) endif() foreach(arch ${SANITIZER_UNITTEST_SUPPORTED_ARCH}) add_sanitizer_common_lib("RTSanitizerCommon.test.${arch}" diff --git a/lib/sanitizer_common/tests/sanitizer_allocator_test.cc b/lib/sanitizer_common/tests/sanitizer_allocator_test.cc index be8fc91aa861..7ba334568146 100644 --- a/lib/sanitizer_common/tests/sanitizer_allocator_test.cc +++ b/lib/sanitizer_common/tests/sanitizer_allocator_test.cc @@ -40,6 +40,8 @@ typedef SizeClassAllocator64< kAllocatorSpace, kAllocatorSize, 16, CompactSizeClassMap> Allocator64Compact; #elif defined(__mips64) static const u64 kAddressSpaceSize = 1ULL << 40; +#elif defined(__aarch64__) +static const u64 kAddressSpaceSize = 1ULL << 39; #else static const u64 kAddressSpaceSize = 1ULL << 32; #endif @@ -94,7 +96,7 @@ void TestSizeClassAllocator() { uptr size = sizes[s]; if (!a->CanAllocate(size, 1)) continue; // printf("s = %ld\n", size); - uptr n_iter = std::max((uptr)6, 8000000 / size); + uptr n_iter = std::max((uptr)6, 4000000 / size); // fprintf(stderr, "size: %ld iter: %ld\n", size, n_iter); for (uptr i = 0; i < n_iter; i++) { uptr class_id0 = Allocator::SizeClassMapT::ClassID(size); diff --git a/lib/sanitizer_common/tests/sanitizer_common_test.cc b/lib/sanitizer_common/tests/sanitizer_common_test.cc index e08a38c82450..6fc308ad14d4 100644 --- a/lib/sanitizer_common/tests/sanitizer_common_test.cc +++ b/lib/sanitizer_common/tests/sanitizer_common_test.cc @@ -188,6 +188,15 @@ TEST(SanitizerCommon, FindPathToBinary) { InternalFree(true_path); EXPECT_EQ(0, FindPathToBinary("unexisting_binary.ergjeorj")); } +#elif SANITIZER_WINDOWS +TEST(SanitizerCommon, FindPathToBinary) { + // ntdll.dll should be on PATH in all supported test environments on all + // supported Windows versions. + char *ntdll_path = FindPathToBinary("ntdll.dll"); + EXPECT_NE((char*)0, internal_strstr(ntdll_path, "ntdll.dll")); + InternalFree(ntdll_path); + EXPECT_EQ(0, FindPathToBinary("unexisting_binary.ergjeorj")); +} #endif TEST(SanitizerCommon, StripPathPrefix) { @@ -199,6 +208,30 @@ TEST(SanitizerCommon, StripPathPrefix) { EXPECT_STREQ("file.h", StripPathPrefix("/usr/lib/./file.h", "/usr/lib/")); } +TEST(SanitizerCommon, RemoveANSIEscapeSequencesFromString) { + RemoveANSIEscapeSequencesFromString(nullptr); + const char *buffs[22] = { + "Default", "Default", + "\033[95mLight magenta", "Light magenta", + "\033[30mBlack\033[32mGreen\033[90mGray", "BlackGreenGray", + "\033[106mLight cyan \033[107mWhite ", "Light cyan White ", + "\033[31mHello\033[0m World", "Hello World", + "\033[38;5;82mHello \033[38;5;198mWorld", "Hello World", + "123[653456789012", "123[653456789012", + "Normal \033[5mBlink \033[25mNormal", "Normal Blink Normal", + "\033[106m\033[107m", "", + "", "", + " ", " ", + }; + + for (size_t i = 0; i < ARRAY_SIZE(buffs); i+=2) { + char *buffer_copy = internal_strdup(buffs[i]); + RemoveANSIEscapeSequencesFromString(buffer_copy); + EXPECT_STREQ(buffer_copy, buffs[i+1]); + InternalFree(buffer_copy); + } +} + TEST(SanitizerCommon, InternalScopedString) { InternalScopedString str(10); EXPECT_EQ(0U, str.length()); diff --git a/lib/sanitizer_common/tests/sanitizer_libc_test.cc b/lib/sanitizer_common/tests/sanitizer_libc_test.cc index 3252db77653d..015e32a09e37 100644 --- a/lib/sanitizer_common/tests/sanitizer_libc_test.cc +++ b/lib/sanitizer_common/tests/sanitizer_libc_test.cc @@ -8,18 +8,21 @@ //===----------------------------------------------------------------------===// // Tests for sanitizer_libc.h. //===----------------------------------------------------------------------===// +#include <algorithm> #include "sanitizer_common/sanitizer_common.h" #include "sanitizer_common/sanitizer_libc.h" #include "sanitizer_common/sanitizer_platform.h" #include "gtest/gtest.h" -#if SANITIZER_LINUX || SANITIZER_MAC -# define SANITIZER_TEST_HAS_STAT_H 1 +#if SANITIZER_WINDOWS +#define NOMINMAX +#include <windows.h> +#undef NOMINMAX +#endif +#if SANITIZER_POSIX # include <sys/stat.h> # include "sanitizer_common/sanitizer_posix.h" -#else -# define SANITIZER_TEST_HAS_STAT_H 0 #endif // A regression test for internal_memmove() implementation. @@ -57,6 +60,17 @@ struct stat_and_more { }; static void temp_file_name(char *buf, size_t bufsize, const char *prefix) { +#if SANITIZER_WINDOWS + buf[0] = '\0'; + char tmp_dir[MAX_PATH]; + if (!::GetTempPathA(MAX_PATH, tmp_dir)) + return; + // GetTempFileNameA needs a MAX_PATH buffer. + char tmp_path[MAX_PATH]; + if (!::GetTempFileNameA(tmp_dir, prefix, 0, tmp_path)) + return; + internal_strncpy(buf, tmp_path, bufsize); +#else const char *tmpdir = "/tmp"; #if SANITIZER_ANDROID // I don't know a way to query temp directory location on Android without @@ -67,10 +81,9 @@ static void temp_file_name(char *buf, size_t bufsize, const char *prefix) { #endif u32 uid = GetUid(); internal_snprintf(buf, bufsize, "%s/%s%d", tmpdir, prefix, uid); +#endif } -// FIXME: File manipulations are not yet supported on Windows -#if !defined(_WIN32) TEST(SanitizerCommon, FileOps) { const char *str1 = "qwerty"; uptr len1 = internal_strlen(str1); @@ -81,16 +94,23 @@ TEST(SanitizerCommon, FileOps) { temp_file_name(tmpfile, sizeof(tmpfile), "sanitizer_common.fileops.tmp."); fd_t fd = OpenFile(tmpfile, WrOnly); ASSERT_NE(fd, kInvalidFd); - EXPECT_EQ(len1, internal_write(fd, str1, len1)); - EXPECT_EQ(len2, internal_write(fd, str2, len2)); + uptr bytes_written = 0; + EXPECT_TRUE(WriteToFile(fd, str1, len1, &bytes_written)); + EXPECT_EQ(len1, bytes_written); + EXPECT_TRUE(WriteToFile(fd, str2, len2, &bytes_written)); + EXPECT_EQ(len2, bytes_written); CloseFile(fd); + EXPECT_TRUE(FileExists(tmpfile)); + fd = OpenFile(tmpfile, RdOnly); ASSERT_NE(fd, kInvalidFd); + +#if SANITIZER_POSIX + // The stat wrappers are posix-only. uptr fsize = internal_filesize(fd); EXPECT_EQ(len1 + len2, fsize); -#if SANITIZER_TEST_HAS_STAT_H struct stat st1, st2, st3; EXPECT_EQ(0u, internal_stat(tmpfile, &st1)); EXPECT_EQ(0u, internal_lstat(tmpfile, &st2)); @@ -108,16 +128,43 @@ TEST(SanitizerCommon, FileOps) { #endif char buf[64] = {}; - EXPECT_EQ(len1, internal_read(fd, buf, len1)); + uptr bytes_read = 0; + EXPECT_TRUE(ReadFromFile(fd, buf, len1, &bytes_read)); + EXPECT_EQ(len1, bytes_read); EXPECT_EQ(0, internal_memcmp(buf, str1, len1)); EXPECT_EQ((char)0, buf[len1 + 1]); internal_memset(buf, 0, len1); - EXPECT_EQ(len2, internal_read(fd, buf, len2)); + EXPECT_TRUE(ReadFromFile(fd, buf, len2, &bytes_read)); + EXPECT_EQ(len2, bytes_read); EXPECT_EQ(0, internal_memcmp(buf, str2, len2)); CloseFile(fd); + +#if SANITIZER_WINDOWS + // No sanitizer needs to delete a file on Windows yet. If we ever do, we can + // add a portable wrapper and test it from here. + ::DeleteFileA(&tmpfile[0]); +#else internal_unlink(tmpfile); -} #endif +} + +static const size_t kStrlcpyBufSize = 8; +void test_internal_strlcpy(char *dbuf, const char *sbuf) { + uptr retval = 0; + retval = internal_strlcpy(dbuf, sbuf, kStrlcpyBufSize); + EXPECT_EQ(internal_strncmp(dbuf, sbuf, kStrlcpyBufSize - 1), 0); + EXPECT_EQ(internal_strlen(dbuf), + std::min(internal_strlen(sbuf), (uptr)(kStrlcpyBufSize - 1))); + EXPECT_EQ(retval, internal_strlen(sbuf)); + + // Test with shorter maxlen. + uptr maxlen = 2; + if (internal_strlen(sbuf) > maxlen) { + retval = internal_strlcpy(dbuf, sbuf, maxlen); + EXPECT_EQ(internal_strncmp(dbuf, sbuf, maxlen - 1), 0); + EXPECT_EQ(internal_strlen(dbuf), maxlen - 1); + } +} TEST(SanitizerCommon, InternalStrFunctions) { const char *haystack = "haystack"; @@ -125,10 +172,45 @@ TEST(SanitizerCommon, InternalStrFunctions) { EXPECT_EQ(haystack + 2, internal_strchrnul(haystack, 'y')); EXPECT_EQ(0, internal_strchr(haystack, 'z')); EXPECT_EQ(haystack + 8, internal_strchrnul(haystack, 'z')); + + char dbuf[kStrlcpyBufSize] = {}; + const char *samesizestr = "1234567"; + const char *shortstr = "123"; + const char *longerstr = "123456789"; + + // Test internal_strlcpy. + internal_strlcpy(dbuf, shortstr, 0); + EXPECT_EQ(dbuf[0], 0); + EXPECT_EQ(dbuf[0], 0); + test_internal_strlcpy(dbuf, samesizestr); + test_internal_strlcpy(dbuf, shortstr); + test_internal_strlcpy(dbuf, longerstr); + + // Test internal_strlcat. + char dcatbuf[kStrlcpyBufSize] = {}; + uptr retval = 0; + retval = internal_strlcat(dcatbuf, "aaa", 0); + EXPECT_EQ(internal_strlen(dcatbuf), (uptr)0); + EXPECT_EQ(retval, (uptr)3); + + retval = internal_strlcat(dcatbuf, "123", kStrlcpyBufSize); + EXPECT_EQ(internal_strcmp(dcatbuf, "123"), 0); + EXPECT_EQ(internal_strlen(dcatbuf), (uptr)3); + EXPECT_EQ(retval, (uptr)3); + + retval = internal_strlcat(dcatbuf, "123", kStrlcpyBufSize); + EXPECT_EQ(internal_strcmp(dcatbuf, "123123"), 0); + EXPECT_EQ(internal_strlen(dcatbuf), (uptr)6); + EXPECT_EQ(retval, (uptr)6); + + retval = internal_strlcat(dcatbuf, "123", kStrlcpyBufSize); + EXPECT_EQ(internal_strcmp(dcatbuf, "1231231"), 0); + EXPECT_EQ(internal_strlen(dcatbuf), (uptr)7); + EXPECT_EQ(retval, (uptr)9); } // FIXME: File manipulations are not yet supported on Windows -#if !defined(_WIN32) && !SANITIZER_MAC +#if SANITIZER_POSIX && !SANITIZER_MAC TEST(SanitizerCommon, InternalMmapWithOffset) { char tmpfile[128]; temp_file_name(tmpfile, sizeof(tmpfile), diff --git a/lib/sanitizer_common/tests/sanitizer_linux_test.cc b/lib/sanitizer_common/tests/sanitizer_linux_test.cc index 11342b775cc7..eef71010afe6 100644 --- a/lib/sanitizer_common/tests/sanitizer_linux_test.cc +++ b/lib/sanitizer_common/tests/sanitizer_linux_test.cc @@ -195,7 +195,7 @@ TEST(SanitizerCommon, SetEnvTest) { EXPECT_EQ(0, getenv(kEnvName)); } -#if defined(__x86_64__) || defined(__i386__) +#if (defined(__x86_64__) || defined(__i386__)) && !SANITIZER_ANDROID void *thread_self_offset_test_func(void *arg) { bool result = *(uptr *)((char *)ThreadSelf() + ThreadSelfOffset()) == ThreadSelf(); diff --git a/lib/sanitizer_common/tests/sanitizer_stacktrace_test.cc b/lib/sanitizer_common/tests/sanitizer_stacktrace_test.cc index 654ea1db82fb..3d57eded948f 100644 --- a/lib/sanitizer_common/tests/sanitizer_stacktrace_test.cc +++ b/lib/sanitizer_common/tests/sanitizer_stacktrace_test.cc @@ -82,7 +82,7 @@ TEST_F(FastUnwindTest, Basic) { } } -// From: http://code.google.com/p/address-sanitizer/issues/detail?id=162 +// From: https://github.com/google/sanitizers/issues/162 TEST_F(FastUnwindTest, FramePointerLoop) { // Make one fp point to itself. fake_stack[4] = (uhwptr)&fake_stack[4]; diff --git a/lib/sanitizer_common/tests/sanitizer_suppressions_test.cc b/lib/sanitizer_common/tests/sanitizer_suppressions_test.cc index e8c30d07e78c..224ab0538377 100644 --- a/lib/sanitizer_common/tests/sanitizer_suppressions_test.cc +++ b/lib/sanitizer_common/tests/sanitizer_suppressions_test.cc @@ -105,9 +105,10 @@ TEST_F(SuppressionContextTest, Parse3) { ctx_.Parse( "# last suppression w/o line-feed\n" "race:foo\n" - "race:bar" + "race:bar\r\n" + "race:baz" ); // NOLINT - CheckSuppressions(2, {"race", "race"}, {"foo", "bar"}); + CheckSuppressions(3, {"race", "race", "race"}, {"foo", "bar", "baz"}); } TEST_F(SuppressionContextTest, ParseType) { diff --git a/lib/sanitizer_common/tests/sanitizer_test_main.cc b/lib/sanitizer_common/tests/sanitizer_test_main.cc index b7fd3dafab26..20f8f53975d0 100644 --- a/lib/sanitizer_common/tests/sanitizer_test_main.cc +++ b/lib/sanitizer_common/tests/sanitizer_test_main.cc @@ -11,6 +11,7 @@ // //===----------------------------------------------------------------------===// #include "gtest/gtest.h" +#include "sanitizer_common/sanitizer_flags.h" const char *argv0; @@ -18,5 +19,6 @@ int main(int argc, char **argv) { argv0 = argv[0]; testing::GTEST_FLAG(death_test_style) = "threadsafe"; testing::InitGoogleTest(&argc, argv); + SetCommonFlagsDefaults(); return RUN_ALL_TESTS(); } diff --git a/lib/tsan/.clang-format b/lib/tsan/.clang-format new file mode 100644 index 000000000000..f6cb8ad931f5 --- /dev/null +++ b/lib/tsan/.clang-format @@ -0,0 +1 @@ +BasedOnStyle: Google diff --git a/lib/tsan/CMakeLists.txt b/lib/tsan/CMakeLists.txt index 90137800ffac..0e60cd3464d8 100644 --- a/lib/tsan/CMakeLists.txt +++ b/lib/tsan/CMakeLists.txt @@ -8,13 +8,19 @@ set(TSAN_CFLAGS ${SANITIZER_COMMON_CFLAGS}) append_list_if(COMPILER_RT_HAS_FPIE_FLAG -fPIE TSAN_CFLAGS) append_no_rtti_flag(TSAN_CFLAGS) +if(COMPILER_RT_TSAN_DEBUG_OUTPUT) + # Add extra debug information to TSan runtime. This configuration is rarely + # used, but we need to support it so that debug output will not bitrot. + list(APPEND TSAN_CFLAGS -DTSAN_COLLECT_STATS=1 + -DTSAN_DEBUG_OUTPUT=2) +endif() + set(TSAN_RTL_CFLAGS ${TSAN_CFLAGS}) append_list_if(COMPILER_RT_HAS_MSSE3_FLAG -msse3 TSAN_RTL_CFLAGS) append_list_if(SANITIZER_LIMIT_FRAME_SIZE -Wframe-larger-than=512 TSAN_RTL_CFLAGS) append_list_if(COMPILER_RT_HAS_WGLOBAL_CONSTRUCTORS_FLAG -Wglobal-constructors TSAN_RTL_CFLAGS) -# FIXME: Add support for --sysroot=. compile flag: set(TSAN_SOURCES rtl/tsan_clock.cc @@ -26,6 +32,7 @@ set(TSAN_SOURCES rtl/tsan_interface_atomic.cc rtl/tsan_interface.cc rtl/tsan_interface_java.cc + rtl/tsan_malloc_mac.cc rtl/tsan_md5.cc rtl/tsan_mman.cc rtl/tsan_mutex.cc @@ -45,11 +52,16 @@ set(TSAN_CXX_SOURCES rtl/tsan_new_delete.cc) if(APPLE) - list(APPEND TSAN_SOURCES rtl/tsan_platform_mac.cc) + list(APPEND TSAN_SOURCES + rtl/tsan_interceptors_mac.cc + rtl/tsan_libdispatch_mac.cc + rtl/tsan_platform_mac.cc + rtl/tsan_platform_posix.cc) elseif(UNIX) # Assume Linux list(APPEND TSAN_SOURCES - rtl/tsan_platform_linux.cc) + rtl/tsan_platform_linux.cc + rtl/tsan_platform_posix.cc) endif() set(TSAN_HEADERS @@ -83,47 +95,112 @@ set(TSAN_HEADERS set(TSAN_RUNTIME_LIBRARIES) add_custom_target(tsan) -foreach(arch ${TSAN_SUPPORTED_ARCH}) - if(arch STREQUAL "x86_64") - set(TSAN_ASM_SOURCES rtl/tsan_rtl_amd64.S) - # Pass ASM file directly to the C++ compiler. - set_source_files_properties(${TSAN_ASM_SOURCES} PROPERTIES - LANGUAGE C) - # Sanity check for Go runtime. - set(BUILDGO_SCRIPT ${CMAKE_CURRENT_SOURCE_DIR}/go/buildgo.sh) - add_custom_target(GotsanRuntimeCheck - COMMAND env "CC=${CMAKE_C_COMPILER} ${CMAKE_C_COMPILER_ARG1}" - IN_TMPDIR=1 SILENT=1 ${BUILDGO_SCRIPT} - DEPENDS clang_rt.tsan-${arch} ${BUILDGO_SCRIPT} - WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/go - COMMENT "Checking TSan Go runtime..." - VERBATIM) +if(APPLE) + set(TSAN_ASM_SOURCES rtl/tsan_rtl_amd64.S) + # Xcode will try to compile this file as C ('clang -x c'), and that will fail. + if (${CMAKE_GENERATOR} STREQUAL "Xcode") + enable_language(ASM) else() - set(TSAN_ASM_SOURCES) + # Pass ASM file directly to the C++ compiler. + set_source_files_properties(${TSAN_ASM_SOURCES} PROPERTIES LANGUAGE C) endif() - add_compiler_rt_runtime(clang_rt.tsan-${arch} ${arch} STATIC - SOURCES ${TSAN_SOURCES} ${TSAN_ASM_SOURCES} - $<TARGET_OBJECTS:RTInterception.${arch}> - $<TARGET_OBJECTS:RTSanitizerCommon.${arch}> - $<TARGET_OBJECTS:RTSanitizerCommonLibc.${arch}> - $<TARGET_OBJECTS:RTUbsan.${arch}> + add_compiler_rt_runtime(clang_rt.tsan + SHARED + OS ${TSAN_SUPPORTED_OS} + ARCHS ${TSAN_SUPPORTED_ARCH} + SOURCES ${TSAN_SOURCES} ${TSAN_CXX_SOURCES} ${TSAN_ASM_SOURCES} + OBJECT_LIBS RTInterception + RTSanitizerCommon + RTSanitizerCommonLibc + RTUbsan + CFLAGS ${TSAN_RTL_CFLAGS} + PARENT_TARGET tsan) + add_compiler_rt_object_libraries(RTTsan_dynamic + OS ${TSAN_SUPPORTED_OS} + ARCHS ${TSAN_SUPPORTED_ARCH} + SOURCES ${TSAN_SOURCES} ${TSAN_CXX_SOURCES} ${TSAN_ASM_SOURCES} CFLAGS ${TSAN_RTL_CFLAGS}) - add_compiler_rt_runtime(clang_rt.tsan_cxx-${arch} ${arch} STATIC - SOURCES ${TSAN_CXX_SOURCES} - $<TARGET_OBJECTS:RTUbsan_cxx.${arch}> - CFLAGS ${TSAN_RTL_CFLAGS}) - list(APPEND TSAN_RUNTIME_LIBRARIES clang_rt.tsan-${arch} - clang_rt.tsan_cxx-${arch}) - add_sanitizer_rt_symbols(clang_rt.tsan-${arch} rtl/tsan.syms.extra) - add_sanitizer_rt_symbols(clang_rt.tsan_cxx-${arch} rtl/tsan.syms.extra) - add_dependencies(tsan clang_rt.tsan-${arch} - clang_rt.tsan_cxx-${arch} - clang_rt.tsan-${arch}-symbols - clang_rt.tsan_cxx-${arch}-symbols) -endforeach() + + # Build and check Go runtime. + set(BUILDGO_SCRIPT ${CMAKE_CURRENT_SOURCE_DIR}/go/buildgo.sh) + add_custom_target(GotsanRuntimeCheck + COMMAND env "CC=${CMAKE_C_COMPILER} ${OSX_SYSROOT_FLAG}" + IN_TMPDIR=1 SILENT=1 ${BUILDGO_SCRIPT} + DEPENDS tsan ${BUILDGO_SCRIPT} + WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/go + COMMENT "Checking TSan Go runtime..." + VERBATIM) +else() + foreach(arch ${TSAN_SUPPORTED_ARCH}) + if(arch STREQUAL "x86_64") + set(TSAN_ASM_SOURCES rtl/tsan_rtl_amd64.S) + # Pass ASM file directly to the C++ compiler. + set_source_files_properties(${TSAN_ASM_SOURCES} PROPERTIES + LANGUAGE C) + # Sanity check for Go runtime. + set(BUILDGO_SCRIPT ${CMAKE_CURRENT_SOURCE_DIR}/go/buildgo.sh) + add_custom_target(GotsanRuntimeCheck + COMMAND env "CC=${CMAKE_C_COMPILER} ${CMAKE_C_COMPILER_ARG1}" + IN_TMPDIR=1 SILENT=1 ${BUILDGO_SCRIPT} + DEPENDS clang_rt.tsan-${arch} ${BUILDGO_SCRIPT} + WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/go + COMMENT "Checking TSan Go runtime..." + VERBATIM) + elseif(arch STREQUAL "aarch64") + set(TSAN_ASM_SOURCES rtl/tsan_rtl_aarch64.S) + # Pass ASM file directly to the C++ compiler. + set_source_files_properties(${TSAN_ASM_SOURCES} PROPERTIES + LANGUAGE C) + elseif(arch MATCHES "powerpc64|powerpc64le") + set(TSAN_ASM_SOURCES rtl/tsan_rtl_ppc64.S) + # Pass ASM file directly to the C++ compiler. + set_source_files_properties(${TSAN_ASM_SOURCES} PROPERTIES + LANGUAGE C) + else() + set(TSAN_ASM_SOURCES) + endif() + add_compiler_rt_runtime(clang_rt.tsan + STATIC + ARCHS ${arch} + SOURCES ${TSAN_SOURCES} ${TSAN_ASM_SOURCES} + $<TARGET_OBJECTS:RTInterception.${arch}> + $<TARGET_OBJECTS:RTSanitizerCommon.${arch}> + $<TARGET_OBJECTS:RTSanitizerCommonLibc.${arch}> + $<TARGET_OBJECTS:RTUbsan.${arch}> + CFLAGS ${TSAN_RTL_CFLAGS}) + add_compiler_rt_runtime(clang_rt.tsan_cxx + STATIC + ARCHS ${arch} + SOURCES ${TSAN_CXX_SOURCES} + $<TARGET_OBJECTS:RTUbsan_cxx.${arch}> + CFLAGS ${TSAN_RTL_CFLAGS}) + list(APPEND TSAN_RUNTIME_LIBRARIES clang_rt.tsan-${arch} + clang_rt.tsan_cxx-${arch}) + add_sanitizer_rt_symbols(clang_rt.tsan + ARCHS ${arch} + EXTRA rtl/tsan.syms.extra) + add_sanitizer_rt_symbols(clang_rt.tsan_cxx + ARCHS ${arch} + EXTRA rtl/tsan.syms.extra) + add_dependencies(tsan clang_rt.tsan-${arch} + clang_rt.tsan_cxx-${arch} + clang_rt.tsan-${arch}-symbols + clang_rt.tsan_cxx-${arch}-symbols) + endforeach() +endif() add_dependencies(compiler-rt tsan) +# Make sure that non-platform-specific files don't include any system headers. +if(COMPILER_RT_HAS_SYSROOT_FLAG) + file(GLOB _tsan_generic_sources rtl/tsan*) + file(GLOB _tsan_platform_sources rtl/tsan*posix* rtl/tsan*mac* + rtl/tsan*linux*) + list(REMOVE_ITEM _tsan_generic_sources ${_tsan_platform_sources}) + set_source_files_properties(${_tsan_generic_sources} + PROPERTIES COMPILE_FLAGS "--sysroot=.") +endif() + # Build libcxx instrumented with TSan. if(COMPILER_RT_HAS_LIBCXX_SOURCES AND COMPILER_RT_TEST_COMPILER_ID STREQUAL "Clang") diff --git a/lib/tsan/Makefile.old b/lib/tsan/Makefile.old deleted file mode 100644 index b2ac912d741d..000000000000 --- a/lib/tsan/Makefile.old +++ /dev/null @@ -1,109 +0,0 @@ -DEBUG=0 -LDFLAGS=-ldl -lrt -lpthread -pie -CXXFLAGS = -std=c++11 -fPIE -fno-rtti -g -Wall -Werror \ - -DGTEST_HAS_RTTI=0 -DSANITIZER_DEBUG=$(DEBUG) \ - -DTSAN_CONTAINS_UBSAN=0 -CLANG=clang -FILECHECK=FileCheck -# Silence warnings that Clang produces for gtest code. -# Use -Wno-attributes so that gcc doesn't complain about unknown warning types. -CXXFLAGS += -Wno-attributes -ifeq ($(DEBUG), 0) - CXXFLAGS += -O3 -endif -ifeq ($(CXX), $(CLANG)++) - CXXFLAGS += -Wno-unused-private-field -Wno-static-in-inline -Wgnu -else - CXXFLAGS += -Wno-maybe-uninitialized -endif - -LIBTSAN=rtl/libtsan.a -GTEST_ROOT=third_party/googletest -GTEST_INCLUDE=-I$(GTEST_ROOT)/include -GTEST_BUILD_DIR=$(GTEST_ROOT)/build -GTEST_LIB_NAME=gtest-all.o -GTEST_LIB=$(GTEST_BUILD_DIR)/$(GTEST_LIB_NAME) - -SANITIZER_TESTS_PATH=../sanitizer_common/tests -SANITIZER_COMMON_TESTS_SRC=$(wildcard $(SANITIZER_TESTS_PATH)/*_test.cc) -SANITIZER_COMMON_EXCLUDED_TESTS=$(SANITIZER_TESTS_PATH)/sanitizer_nolibc_test.cc -SANITIZER_COMMON_GOOD_TESTS=$(filter-out $(SANITIZER_COMMON_EXCLUDED_TESTS), $(SANITIZER_COMMON_TESTS_SRC)) -SANITIZER_COMMON_TESTS_OBJ=$(patsubst %.cc,%.o,$(SANITIZER_COMMON_GOOD_TESTS)) -RTL_TEST_SRC=$(wildcard tests/rtl/*.cc) -RTL_TEST_OBJ=$(patsubst %.cc,%.o,$(RTL_TEST_SRC)) -UNIT_TEST_SRC=$(wildcard tests/unit/*_test.cc) -UNIT_TEST_OBJ=$(patsubst %.cc,%.o,$(UNIT_TEST_SRC)) -UNIT_TEST_HDR=$(wildcard rtl/*.h) $(wildcard ../sanitizer_common/*.h) -LIT_TESTS_PATH=../../test/tsan - -INCLUDES=-Irtl -I.. -I../../include $(GTEST_INCLUDE) - -all: libtsan test - -help: - @ echo "A little help is always welcome!" - @ echo "The most useful targets are:" - @ echo " make install_deps # Install third-party dependencies required for building" - @ echo " make presubmit # Run it every time before committing" - @ echo - @ echo "For more info, see http://code.google.com/p/thread-sanitizer/wiki/Development" - -$(LIBTSAN): libtsan - -libtsan: - $(MAKE) -C rtl -f Makefile.old DEBUG=$(DEBUG) - -%.o: %.cc $(UNIT_TEST_HDR) $(LIBTSAN) - $(CXX) $(CXXFLAGS) $(CFLAGS) $(INCLUDES) -o $@ -c $< - -tsan_test: $(UNIT_TEST_OBJ) $(RTL_TEST_OBJ) \ - $(SANITIZER_COMMON_TESTS_OBJ) $(LIBTSAN) $(GTEST_LIB) - $(CXX) -Wl,--whole-archive $^ -Wl,--no-whole-archive -o $@ $(LDFLAGS) - -test: libtsan tsan_test - -run: all - (ulimit -s 8192; ./tsan_test) - CC=$(CLANG) CXX=$(CLANG)++ FILECHECK=$(FILECHECK) $(LIT_TESTS_PATH)/test_output.sh - -presubmit: - ../sanitizer_common/scripts/check_lint.sh - # Debug build with clang. - $(MAKE) -f Makefile.old clean - $(MAKE) -f Makefile.old run DEBUG=1 -j 16 CC=$(CLANG) CXX=$(CLANG)++ - # Release build with clang. - $(MAKE) -f Makefile.old clean - $(MAKE) -f Makefile.old run DEBUG=0 -j 16 CC=$(CLANG) CXX=$(CLANG)++ - ./check_memcpy.sh - # Debug build with gcc - $(MAKE) -f Makefile.old clean - $(MAKE) -f Makefile.old run DEBUG=1 -j 16 CC=gcc CXX=g++ - # Release build with gcc - $(MAKE) -f Makefile.old clean - $(MAKE) -f Makefile.old run DEBUG=0 -j 16 CC=gcc CXX=g++ - ./check_memcpy.sh - ./check_analyze.sh - # Sanity check for Go runtime - (cd go && ./buildgo.sh) - # Check cmake build - ./check_cmake.sh - @ echo PRESUBMIT PASSED - -install_deps: - rm -rf third_party - mkdir third_party - (cd third_party && \ - svn co -r613 http://googletest.googlecode.com/svn/trunk googletest \ - ) - -$(GTEST_LIB): - mkdir -p $(GTEST_BUILD_DIR) && \ - cd $(GTEST_BUILD_DIR) && \ - $(MAKE) -f ../make/Makefile CXXFLAGS="$(CXXFLAGS)" CFLAGS="$(CFLAGS)" CC=$(CC) CXX=$(CXX) $(GTEST_LIB_NAME) - -clean: - rm -f asm_*.s libtsan.nm libtsan.objdump */*.o tsan_test - rm -rf $(GTEST_BUILD_DIR) - $(MAKE) clean -C rtl -f Makefile.old - rm -f go/*.s - rm -rf build diff --git a/lib/tsan/analyze_libtsan.sh b/lib/tsan/analyze_libtsan.sh index 705e4c5460f2..ae29f1b5b05a 100755 --- a/lib/tsan/analyze_libtsan.sh +++ b/lib/tsan/analyze_libtsan.sh @@ -1,10 +1,17 @@ #!/bin/bash +# +# Script that prints information about generated code in TSan runtime. set -e set -u +if [[ "$#" != 1 ]]; then + echo "Usage: $0 /path/to/binary/built/with/tsan" + exit 1 +fi + get_asm() { - grep __tsan_$1.: -A 10000 libtsan.objdump | \ + grep __tsan_$1.: -A 10000 ${OBJDUMP_CONTENTS} | \ awk "/[^:]$/ {print;} />:/ {c++; if (c == 2) {exit}}" } @@ -19,15 +26,19 @@ list="write1 \ func_entry \ func_exit" -BIN=`dirname $0`/tsan_test -objdump -d $BIN > libtsan.objdump -nm -S $BIN | grep "__tsan_" > libtsan.nm +BIN=$1 +OUTPUT_DIR=$(mktemp -t -d analyze_libtsan_out.XXXXXXXX) +OBJDUMP_CONTENTS=${OUTPUT_DIR}/libtsan_objdump +NM_CONTENTS=${OUTPUT_DIR}/libtsan_nm + +objdump -d $BIN > ${OBJDUMP_CONTENTS} +nm -S $BIN | grep "__tsan_" > ${NM_CONTENTS} for f in $list; do - file=asm_$f.s + file=${OUTPUT_DIR}/asm_$f.s get_asm $f > $file tot=$(wc -l < $file) - size=$(grep __tsan_$f$ libtsan.nm | awk --non-decimal-data '{print ("0x"$2)+0}') + size=$(grep __tsan_$f$ ${NM_CONTENTS} | awk --non-decimal-data '{print ("0x"$2)+0}') rsp=$(grep '(%rsp)' $file | wc -l) push=$(grep 'push' $file | wc -l) pop=$(grep 'pop' $file | wc -l) diff --git a/lib/tsan/check_analyze.sh b/lib/tsan/check_analyze.sh index 4b33393ef648..0f6cc0698471 100755 --- a/lib/tsan/check_analyze.sh +++ b/lib/tsan/check_analyze.sh @@ -1,7 +1,17 @@ #!/bin/bash +# +# Script that checks that critical functions in TSan runtime have correct number +# of push/pop/rsp instructions to verify that runtime is efficient enough. + set -u -RES=$(./analyze_libtsan.sh) +if [[ "$#" != 1 ]]; then + echo "Usage: $0 /path/to/binary/built/with/tsan" + exit 1 +fi + +SCRIPTDIR=$(dirname $0) +RES=$(${SCRIPTDIR}/analyze_libtsan.sh $1) PrintRes() { printf "%s\n" "$RES" } @@ -22,7 +32,13 @@ for f in write1; do check $f pop 2 done -for f in write2 write4 write8; do +for f in write2 write4; do + check $f rsp 1 + check $f push 4 + check $f pop 4 +done + +for f in write8; do check $f rsp 1 check $f push 3 check $f pop 3 diff --git a/lib/tsan/check_memcpy.sh b/lib/tsan/check_memcpy.sh deleted file mode 100755 index 101df1166b7c..000000000000 --- a/lib/tsan/check_memcpy.sh +++ /dev/null @@ -1,31 +0,0 @@ -#!/bin/bash - -# Ensure that tsan runtime does not contain compiler-emitted memcpy and memset calls. - -set -eu - -ROOTDIR=$(dirname $0) -TEST_DIR=$ROOTDIR/../../test/tsan - -: ${CXX:=clang++} -CFLAGS="-fsanitize=thread -fPIE -O1 -g" -LDFLAGS="-pie -lpthread -ldl -lrt -lm -Wl,--whole-archive $ROOTDIR/rtl/libtsan.a -Wl,--no-whole-archive" - -SRC=$TEST_DIR/simple_race.cc -OBJ=$SRC.o -EXE=$SRC.exe -$CXX $SRC $CFLAGS -c -o $OBJ -$CXX $OBJ $LDFLAGS -o $EXE - -NCALL=$(objdump -d $EXE | egrep "callq .*<__interceptor_mem(cpy|set)>" | wc -l) -if [ "$NCALL" != "0" ]; then - echo FAIL: found $NCALL memcpy/memset calls - exit 1 -fi - -# tail calls -NCALL=$(objdump -d $EXE | egrep "jmpq .*<__interceptor_mem(cpy|set)>" | wc -l) -if [ "$NCALL" != "0" ]; then - echo FAIL: found $NCALL memcpy/memset calls - exit 1 -fi diff --git a/lib/tsan/dd/CMakeLists.txt b/lib/tsan/dd/CMakeLists.txt index 8130e09c27e8..6330bd9fbe7a 100644 --- a/lib/tsan/dd/CMakeLists.txt +++ b/lib/tsan/dd/CMakeLists.txt @@ -19,25 +19,29 @@ add_custom_target(dd) # Deadlock detector is currently supported on 64-bit Linux only. if(CAN_TARGET_x86_64 AND UNIX AND NOT APPLE AND NOT ANDROID) set(arch "x86_64") - add_compiler_rt_runtime(clang_rt.dd-${arch} ${arch} STATIC + add_compiler_rt_runtime(clang_rt.dd + STATIC + ARCHS ${arch} SOURCES ${DD_SOURCES} $<TARGET_OBJECTS:RTInterception.${arch}> $<TARGET_OBJECTS:RTSanitizerCommon.${arch}> $<TARGET_OBJECTS:RTSanitizerCommonLibc.${arch}> - CFLAGS ${DD_CFLAGS}) - add_dependencies(dd clang_rt.dd-${arch}) + CFLAGS ${DD_CFLAGS} + PARENT_TARGET dd) add_compiler_rt_object_libraries(RTDD ARCHS ${arch} SOURCES ${DD_SOURCES} CFLAGS ${DD_CFLAGS}) - add_compiler_rt_runtime(clang_rt.dyndd-${arch} ${arch} SHARED + add_compiler_rt_runtime(clang_rt.dyndd + SHARED + ARCHS ${arch} SOURCES $<TARGET_OBJECTS:RTDD.${arch}> $<TARGET_OBJECTS:RTInterception.${arch}> $<TARGET_OBJECTS:RTSanitizerCommon.${arch}> - $<TARGET_OBJECTS:RTSanitizerCommonLibc.${arch}>) - target_link_libraries(clang_rt.dyndd-${arch} ${DD_LINKLIBS}) - add_dependencies(dd clang_rt.dyndd-${arch}) + $<TARGET_OBJECTS:RTSanitizerCommonLibc.${arch}> + LINK_LIBS ${DD_LINKLIBS} + PARENT_TARGET dd) endif() add_dependencies(compiler-rt dd) diff --git a/lib/tsan/go/buildgo.sh b/lib/tsan/go/buildgo.sh index 7193b57684f2..fdbd4056959c 100755 --- a/lib/tsan/go/buildgo.sh +++ b/lib/tsan/go/buildgo.sh @@ -20,6 +20,7 @@ SRCS=" ../rtl/tsan_sync.cc ../../sanitizer_common/sanitizer_allocator.cc ../../sanitizer_common/sanitizer_common.cc + ../../sanitizer_common/sanitizer_common_libcdep.cc ../../sanitizer_common/sanitizer_deadlock_detector2.cc ../../sanitizer_common/sanitizer_flag_parser.cc ../../sanitizer_common/sanitizer_flags.cc @@ -36,7 +37,7 @@ SRCS=" if [ "`uname -a | grep Linux`" != "" ]; then SUFFIX="linux_amd64" OSCFLAGS="-fPIC -ffreestanding -Wno-maybe-uninitialized -Wno-unused-const-variable -Werror -Wno-unknown-warning-option" - OSLDFLAGS="-lpthread -lrt -fPIC -fpie" + OSLDFLAGS="-lpthread -fPIC -fpie" SRCS=" $SRCS ../rtl/tsan_platform_linux.cc @@ -45,6 +46,7 @@ if [ "`uname -a | grep Linux`" != "" ]; then ../../sanitizer_common/sanitizer_procmaps_common.cc ../../sanitizer_common/sanitizer_procmaps_linux.cc ../../sanitizer_common/sanitizer_linux.cc + ../../sanitizer_common/sanitizer_linux_libcdep.cc ../../sanitizer_common/sanitizer_stoptheworld_linux_libcdep.cc " elif [ "`uname -a | grep FreeBSD`" != "" ]; then diff --git a/lib/tsan/rtl/Makefile.old b/lib/tsan/rtl/Makefile.old deleted file mode 100644 index ee7095e88d96..000000000000 --- a/lib/tsan/rtl/Makefile.old +++ /dev/null @@ -1,63 +0,0 @@ -CXXFLAGS = -std=c++11 -fPIE -g -Wall -Werror -fno-builtin -msse3 -DSANITIZER_DEBUG=$(DEBUG) -DTSAN_CONTAINS_UBSAN=0 -CLANG=clang -ifeq ($(DEBUG), 0) - CXXFLAGS += -O3 -endif - -# For interception. FIXME: move interception one level higher. -INTERCEPTION=../../interception -COMMON=../../sanitizer_common -INCLUDES= -I../.. -I../../../include -EXTRA_CXXFLAGS=-fno-exceptions -fno-rtti -NO_SYSROOT=--sysroot=. -CXXFLAGS+=$(EXTRA_CXXFLAGS) -CXXFLAGS+=$(CFLAGS) -ifeq ($(DEBUG), 0) - CXXFLAGS+=-fomit-frame-pointer -ifeq ($(CXX), g++) - CXXFLAGS+=-Wno-maybe-uninitialized - CXXFLAGS+=-Wframe-larger-than=512 -endif # CXX=g++ -endif # DEBUG=0 - -ifeq ($(CXX), $(CLANG)++) - # Global constructors are banned. - CXXFLAGS+=-Wglobal-constructors -endif - - - -all: libtsan.a - -LIBTSAN_HEADERS=$(wildcard *.h) \ - $(wildcard $(INTERCEPTION)/*.h) \ - $(wildcard $(COMMON)/*.h) -LIBTSAN_SRC=$(wildcard *.cc) -LIBTSAN_ASM_SRC=$(wildcard *.S) -INTERCEPTION_SRC=$(wildcard $(INTERCEPTION)/*.cc) -COMMON_SRC=$(wildcard $(COMMON)/*.cc) - -LIBTSAN_OBJ=$(patsubst %.cc,%.o,$(LIBTSAN_SRC)) \ - $(patsubst %.S,%.o,$(LIBTSAN_ASM_SRC)) \ - $(patsubst $(INTERCEPTION)/%.cc,%.o,$(INTERCEPTION_SRC)) \ - $(patsubst $(COMMON)/%.cc,%.o,$(COMMON_SRC)) - -%_linux.o: %_linux.cc Makefile.old $(LIBTSAN_HEADERS) - $(CXX) $(CXXFLAGS) $(INCLUDES) -c $< -%.o: %.cc Makefile.old $(LIBTSAN_HEADERS) - $(CXX) $(CXXFLAGS) $(INCLUDES) $(NO_SYSROOT) -c $< -%.o: $(INTERCEPTION)/%.cc Makefile.old $(LIBTSAN_HEADERS) - $(CXX) $(CXXFLAGS) $(INCLUDES) -c $< -o $@ -%.o: $(COMMON)/%.cc Makefile.old $(LIBTSAN_HEADERS) - $(CXX) $(CXXFLAGS) $(INCLUDES) -c $< -o $@ -%.o: %.S - $(CXX) $(INCLUDES) -o $@ -c $< - -libtsan.a: $(LIBTSAN_OBJ) - ar ru $@ $(LIBTSAN_OBJ) - -libtsan_dummy.a: tsan_dummy_rtl.o - ar ru $@ $< - -clean: - rm -f *.o *.a diff --git a/lib/tsan/rtl/tsan_clock.cc b/lib/tsan/rtl/tsan_clock.cc index 59e3de435f1b..1e2050d1f203 100644 --- a/lib/tsan/rtl/tsan_clock.cc +++ b/lib/tsan/rtl/tsan_clock.cc @@ -90,8 +90,6 @@ namespace __tsan { -const unsigned kInvalidTid = (unsigned)-1; - ThreadClock::ThreadClock(unsigned tid, unsigned reused) : tid_(tid) , reused_(reused + 1) { // 0 has special meaning diff --git a/lib/tsan/rtl/tsan_defs.h b/lib/tsan/rtl/tsan_defs.h index d869d95e0878..9c7b329dcf00 100644 --- a/lib/tsan/rtl/tsan_defs.h +++ b/lib/tsan/rtl/tsan_defs.h @@ -38,13 +38,10 @@ namespace __tsan { const bool kGoMode = true; const bool kCppMode = false; const char *const kTsanOptionsEnv = "GORACE"; -// Go linker does not support weak symbols. -#define CPP_WEAK #else const bool kGoMode = false; const bool kCppMode = true; const char *const kTsanOptionsEnv = "TSAN_OPTIONS"; -#define CPP_WEAK WEAK #endif const int kTidBits = 13; @@ -83,6 +80,8 @@ const bool kCollectHistory = false; const bool kCollectHistory = true; #endif +const unsigned kInvalidTid = (unsigned)-1; + // The following "build consistency" machinery ensures that all source files // are built in the same configuration. Inconsistent builds lead to // hard to debug crashes. diff --git a/lib/tsan/rtl/tsan_dense_alloc.h b/lib/tsan/rtl/tsan_dense_alloc.h index a1cf84b8f166..e9815c90a953 100644 --- a/lib/tsan/rtl/tsan_dense_alloc.h +++ b/lib/tsan/rtl/tsan_dense_alloc.h @@ -108,7 +108,7 @@ class DenseSlabAlloc { // Reserve 0 as invalid index. IndexT start = fillpos_ == 0 ? 1 : 0; for (IndexT i = start; i < kL2Size; i++) { - new(batch + i) T(); + new(batch + i) T; *(IndexT*)(batch + i) = i + 1 + fillpos_ * kL2Size; } *(IndexT*)(batch + kL2Size - 1) = 0; diff --git a/lib/tsan/rtl/tsan_flags.cc b/lib/tsan/rtl/tsan_flags.cc index 5de227a42dee..761523171c77 100644 --- a/lib/tsan/rtl/tsan_flags.cc +++ b/lib/tsan/rtl/tsan_flags.cc @@ -29,8 +29,8 @@ Flags *flags() { #ifdef TSAN_EXTERNAL_HOOKS extern "C" const char* __tsan_default_options(); #else -extern "C" SANITIZER_INTERFACE_ATTRIBUTE -const char *WEAK __tsan_default_options() { +SANITIZER_WEAK_DEFAULT_IMPL +const char *__tsan_default_options() { return ""; } #endif @@ -61,11 +61,16 @@ void InitializeFlags(Flags *f, const char *env) { CommonFlags cf; cf.CopyFrom(*common_flags()); cf.allow_addr2line = true; -#ifndef SANITIZER_GO - cf.detect_deadlocks = true; -#endif + if (kGoMode) { + // Does not work as expected for Go: runtime handles SIGABRT and crashes. + cf.abort_on_error = false; + // Go does not have mutexes. + } else { + cf.detect_deadlocks = true; + } cf.print_suppressions = false; cf.stack_trace_format = " #%n %f %S %M"; + cf.exitcode = 66; OverrideCommonFlags(cf); } diff --git a/lib/tsan/rtl/tsan_flags.inc b/lib/tsan/rtl/tsan_flags.inc index e4994685fa0d..ab9ca9924936 100644 --- a/lib/tsan/rtl/tsan_flags.inc +++ b/lib/tsan/rtl/tsan_flags.inc @@ -45,7 +45,6 @@ TSAN_FLAG( "If set, all atomics are effectively sequentially consistent (seq_cst), " "regardless of what user actually specified.") TSAN_FLAG(bool, print_benign, false, "Print matched \"benign\" races at exit.") -TSAN_FLAG(int, exitcode, 66, "Override exit status if something was reported.") TSAN_FLAG(bool, halt_on_error, false, "Exit after first reported error.") TSAN_FLAG(int, atexit_sleep_ms, 1000, "Sleep in main thread before exiting for that many ms " diff --git a/lib/tsan/rtl/tsan_interceptors.cc b/lib/tsan/rtl/tsan_interceptors.cc index b1a7ae6de328..62c96cb42047 100644 --- a/lib/tsan/rtl/tsan_interceptors.cc +++ b/lib/tsan/rtl/tsan_interceptors.cc @@ -28,16 +28,42 @@ #include "tsan_mman.h" #include "tsan_fd.h" +#if SANITIZER_POSIX +#include "sanitizer_common/sanitizer_posix.h" +#endif + using namespace __tsan; // NOLINT -#if SANITIZER_FREEBSD +#if SANITIZER_FREEBSD || SANITIZER_MAC #define __errno_location __error -#define __libc_realloc __realloc -#define __libc_calloc __calloc #define stdout __stdoutp #define stderr __stderrp #endif +#if SANITIZER_FREEBSD +#define __libc_realloc __realloc +#define __libc_calloc __calloc +#elif SANITIZER_MAC +#define __libc_malloc REAL(malloc) +#define __libc_realloc REAL(realloc) +#define __libc_calloc REAL(calloc) +#define __libc_free REAL(free) +#elif SANITIZER_ANDROID +#define __errno_location __errno +#define __libc_malloc REAL(malloc) +#define __libc_realloc REAL(realloc) +#define __libc_calloc REAL(calloc) +#define __libc_free REAL(free) +#define mallopt(a, b) +#endif + +#if SANITIZER_LINUX || SANITIZER_FREEBSD +#define PTHREAD_CREATE_DETACHED 1 +#elif SANITIZER_MAC +#define PTHREAD_CREATE_DETACHED 2 +#endif + + #ifdef __mips__ const int kSigCount = 129; #else @@ -60,6 +86,14 @@ struct ucontext_t { }; #endif +#if defined(__x86_64__) || defined(__mips__) \ + || (defined(__powerpc64__) && defined(__BIG_ENDIAN__)) +#define PTHREAD_ABI_BASE "GLIBC_2.3.2" +#elif defined(__aarch64__) || (defined(__powerpc64__) \ + && defined(__LITTLE_ENDIAN__)) +#define PTHREAD_ABI_BASE "GLIBC_2.17" +#endif + extern "C" int pthread_attr_init(void *attr); extern "C" int pthread_attr_destroy(void *attr); DECLARE_REAL(int, pthread_attr_getdetachstate, void *, void *) @@ -67,20 +101,23 @@ extern "C" int pthread_attr_setstacksize(void *attr, uptr stacksize); extern "C" int pthread_key_create(unsigned *key, void (*destructor)(void* v)); extern "C" int pthread_setspecific(unsigned key, const void *v); DECLARE_REAL(int, pthread_mutexattr_gettype, void *, void *) -extern "C" int pthread_yield(); extern "C" int pthread_sigmask(int how, const __sanitizer_sigset_t *set, __sanitizer_sigset_t *oldset); // REAL(sigfillset) defined in common interceptors. DECLARE_REAL(int, sigfillset, __sanitizer_sigset_t *set) DECLARE_REAL(int, fflush, __sanitizer_FILE *fp) +DECLARE_REAL_AND_INTERCEPTOR(void *, malloc, uptr size) +DECLARE_REAL_AND_INTERCEPTOR(void, free, void *ptr) extern "C" void *pthread_self(); extern "C" void _exit(int status); extern "C" int *__errno_location(); extern "C" int fileno_unlocked(void *stream); +#if !SANITIZER_ANDROID extern "C" void *__libc_calloc(uptr size, uptr n); extern "C" void *__libc_realloc(void *ptr, uptr size); +#endif extern "C" int dirfd(void *dirp); -#if !SANITIZER_FREEBSD +#if !SANITIZER_FREEBSD && !SANITIZER_ANDROID extern "C" int mallopt(int param, int value); #endif extern __sanitizer_FILE *stdout, *stderr; @@ -89,14 +126,16 @@ const int PTHREAD_MUTEX_RECURSIVE_NP = 1; const int EINVAL = 22; const int EBUSY = 16; const int EOWNERDEAD = 130; +#if !SANITIZER_MAC const int EPOLL_CTL_ADD = 1; +#endif const int SIGILL = 4; const int SIGABRT = 6; const int SIGFPE = 8; const int SIGSEGV = 11; const int SIGPIPE = 13; const int SIGTERM = 15; -#ifdef __mips__ +#if defined(__mips__) || SANITIZER_MAC const int SIGBUS = 10; const int SIGSYS = 12; #else @@ -104,7 +143,9 @@ const int SIGBUS = 7; const int SIGSYS = 31; #endif void *const MAP_FAILED = (void*)-1; +#if !SANITIZER_MAC const int PTHREAD_BARRIER_SERIAL_THREAD = -1; +#endif const int MAP_FIXED = 0x10; typedef long long_t; // NOLINT @@ -119,6 +160,17 @@ typedef long long_t; // NOLINT typedef void (*sighandler_t)(int sig); typedef void (*sigactionhandler_t)(int sig, my_siginfo_t *siginfo, void *uctx); +#if SANITIZER_ANDROID +struct sigaction_t { + u32 sa_flags; + union { + sighandler_t sa_handler; + sigactionhandler_t sa_sgiaction; + }; + __sanitizer_sigset_t sa_mask; + void (*sa_restorer)(); +}; +#else struct sigaction_t { #ifdef __mips__ u32 sa_flags; @@ -130,6 +182,9 @@ struct sigaction_t { #if SANITIZER_FREEBSD int sa_flags; __sanitizer_sigset_t sa_mask; +#elif SANITIZER_MAC + __sanitizer_sigset_t sa_mask; + int sa_flags; #else __sanitizer_sigset_t sa_mask; #ifndef __mips__ @@ -138,11 +193,12 @@ struct sigaction_t { void (*sa_restorer)(); #endif }; +#endif const sighandler_t SIG_DFL = (sighandler_t)0; const sighandler_t SIG_IGN = (sighandler_t)1; const sighandler_t SIG_ERR = (sighandler_t)-1; -#if SANITIZER_FREEBSD +#if SANITIZER_FREEBSD || SANITIZER_MAC const int SA_SIGINFO = 0x40; const int SIG_SETMASK = 3; #elif defined(__mips__) @@ -171,6 +227,9 @@ struct ThreadSignalContext { atomic_uintptr_t in_blocking_func; atomic_uintptr_t have_pending_signals; SignalDesc pending_signals[kSigCount]; + // emptyset and oldset are too big for stack. + __sanitizer_sigset_t emptyset; + __sanitizer_sigset_t oldset; }; // The object is 64-byte aligned, because we want hot data to be located in @@ -203,7 +262,9 @@ static ThreadSignalContext *SigCtx(ThreadState *thr) { return ctx; } +#if !SANITIZER_MAC static unsigned g_thread_finalize_key; +#endif ScopedInterceptor::ScopedInterceptor(ThreadState *thr, const char *fname, uptr pc) @@ -234,17 +295,20 @@ ScopedInterceptor::~ScopedInterceptor() { } } -#define SCOPED_TSAN_INTERCEPTOR(func, ...) \ - SCOPED_INTERCEPTOR_RAW(func, __VA_ARGS__); \ - if (REAL(func) == 0) { \ - Report("FATAL: ThreadSanitizer: failed to intercept %s\n", #func); \ - Die(); \ - } \ - if (thr->ignore_interceptors || thr->in_ignored_lib) \ - return REAL(func)(__VA_ARGS__); \ -/**/ +void ScopedInterceptor::UserCallbackStart() { + if (in_ignored_lib_) { + thr_->in_ignored_lib = false; + ThreadIgnoreEnd(thr_, pc_); + } +} + +void ScopedInterceptor::UserCallbackEnd() { + if (in_ignored_lib_) { + thr_->in_ignored_lib = true; + ThreadIgnoreBegin(thr_, pc_); + } +} -#define TSAN_INTERCEPTOR(ret, func, ...) INTERCEPTOR(ret, func, __VA_ARGS__) #define TSAN_INTERCEPT(func) INTERCEPT_FUNCTION(func) #if SANITIZER_FREEBSD # define TSAN_INTERCEPT_VER(func, ver) INTERCEPT_FUNCTION(func) @@ -329,6 +393,7 @@ static void at_exit_wrapper(void *arg) { static int setup_at_exit_wrapper(ThreadState *thr, uptr pc, void(*f)(), void *arg, void *dso); +#if !SANITIZER_ANDROID TSAN_INTERCEPTOR(int, atexit, void (*f)()) { if (cur_thread()->in_symbolizer) return 0; @@ -337,6 +402,7 @@ TSAN_INTERCEPTOR(int, atexit, void (*f)()) { SCOPED_INTERCEPTOR_RAW(atexit, f); return setup_at_exit_wrapper(thr, pc, (void(*)())f, 0, 0); } +#endif TSAN_INTERCEPTOR(int, __cxa_atexit, void (*f)(void *a), void *arg, void *dso) { if (cur_thread()->in_symbolizer) @@ -359,6 +425,7 @@ static int setup_at_exit_wrapper(ThreadState *thr, uptr pc, void(*f)(), return res; } +#if !SANITIZER_MAC static void on_exit_wrapper(int status, void *arg) { ThreadState *thr = cur_thread(); uptr pc = 0; @@ -383,6 +450,7 @@ TSAN_INTERCEPTOR(int, on_exit, void(*f)(int, void*), void *arg) { ThreadIgnoreEnd(thr, pc); return res; } +#endif // Cleanup old bufs. static void JmpBufGarbageCollect(ThreadState *thr, uptr sp) { @@ -390,7 +458,7 @@ static void JmpBufGarbageCollect(ThreadState *thr, uptr sp) { JmpBuf *buf = &thr->jmp_bufs[i]; if (buf->sp <= sp) { uptr sz = thr->jmp_bufs.Size(); - thr->jmp_bufs[i] = thr->jmp_bufs[sz - 1]; + internal_memcpy(buf, &thr->jmp_bufs[sz - 1], sizeof(*buf)); thr->jmp_bufs.PopBack(); i--; } @@ -417,11 +485,17 @@ static void SetJmp(ThreadState *thr, uptr sp, uptr mangled_sp) { } static void LongJmp(ThreadState *thr, uptr *env) { -#if SANITIZER_FREEBSD +#ifdef __powerpc__ + uptr mangled_sp = env[0]; +#elif SANITIZER_FREEBSD || SANITIZER_MAC uptr mangled_sp = env[2]; -#else +#elif defined(SANITIZER_LINUX) +# ifdef __aarch64__ + uptr mangled_sp = env[13]; +# else uptr mangled_sp = env[6]; -#endif // SANITIZER_FREEBSD +# endif +#endif // Find the saved buf by mangled_sp. for (uptr i = 0; i < thr->jmp_bufs.Size(); i++) { JmpBuf *buf = &thr->jmp_bufs[i]; @@ -451,6 +525,11 @@ extern "C" void __tsan_setjmp(uptr sp, uptr mangled_sp) { SetJmp(cur_thread(), sp, mangled_sp); } +#if SANITIZER_MAC +TSAN_INTERCEPTOR(int, setjmp, void *env); +TSAN_INTERCEPTOR(int, _setjmp, void *env); +TSAN_INTERCEPTOR(int, sigsetjmp, void *env); +#else // SANITIZER_MAC // Not called. Merely to satisfy TSAN_INTERCEPT(). extern "C" SANITIZER_INTERFACE_ATTRIBUTE int __interceptor_setjmp(void *env); @@ -489,6 +568,7 @@ DEFINE_REAL(int, setjmp, void *env) DEFINE_REAL(int, _setjmp, void *env) DEFINE_REAL(int, sigsetjmp, void *env) DEFINE_REAL(int, __sigsetjmp, void *env) +#endif // SANITIZER_MAC TSAN_INTERCEPTOR(void, longjmp, uptr *env, int val) { { @@ -506,6 +586,7 @@ TSAN_INTERCEPTOR(void, siglongjmp, uptr *env, int val) { REAL(siglongjmp)(env, val); } +#if !SANITIZER_MAC TSAN_INTERCEPTOR(void*, malloc, uptr size) { if (cur_thread()->in_symbolizer) return __libc_malloc(size); @@ -572,6 +653,7 @@ TSAN_INTERCEPTOR(uptr, malloc_usable_size, void *p) { SCOPED_INTERCEPTOR_RAW(malloc_usable_size, p); return user_alloc_usable_size(p); } +#endif TSAN_INTERCEPTOR(uptr, strlen, const char *s) { SCOPED_TSAN_INTERCEPTOR(strlen, s); @@ -596,27 +678,18 @@ TSAN_INTERCEPTOR(void*, memcpy, void *dst, const void *src, uptr size) { MemoryAccessRange(thr, pc, (uptr)dst, size, true); MemoryAccessRange(thr, pc, (uptr)src, size, false); } - return internal_memcpy(dst, src, size); -} - -TSAN_INTERCEPTOR(int, memcmp, const void *s1, const void *s2, uptr n) { - SCOPED_TSAN_INTERCEPTOR(memcmp, s1, s2, n); - int res = 0; - uptr len = 0; - for (; len < n; len++) { - if ((res = ((const unsigned char *)s1)[len] - - ((const unsigned char *)s2)[len])) - break; - } - MemoryAccessRange(thr, pc, (uptr)s1, len < n ? len + 1 : n, false); - MemoryAccessRange(thr, pc, (uptr)s2, len < n ? len + 1 : n, false); - return res; + // On OS X, calling internal_memcpy here will cause memory corruptions, + // because memcpy and memmove are actually aliases of the same implementation. + // We need to use internal_memmove here. + return internal_memmove(dst, src, size); } TSAN_INTERCEPTOR(void*, memmove, void *dst, void *src, uptr n) { - SCOPED_TSAN_INTERCEPTOR(memmove, dst, src, n); - MemoryAccessRange(thr, pc, (uptr)dst, n, true); - MemoryAccessRange(thr, pc, (uptr)src, n, false); + if (!COMMON_INTERCEPTOR_NOTHING_IS_INITIALIZED) { + SCOPED_TSAN_INTERCEPTOR(memmove, dst, src, n); + MemoryAccessRange(thr, pc, (uptr)dst, n, true); + MemoryAccessRange(thr, pc, (uptr)src, n, false); + } return REAL(memmove)(dst, src, n); } @@ -629,6 +702,7 @@ TSAN_INTERCEPTOR(char*, strchr, char *s, int c) { return res; } +#if !SANITIZER_MAC TSAN_INTERCEPTOR(char*, strchrnul, char *s, int c) { SCOPED_TSAN_INTERCEPTOR(strchrnul, s, c); char *res = REAL(strchrnul)(s, c); @@ -636,6 +710,7 @@ TSAN_INTERCEPTOR(char*, strchrnul, char *s, int c) { READ_STRING(thr, pc, s, len); return res; } +#endif TSAN_INTERCEPTOR(char*, strrchr, char *s, int c) { SCOPED_TSAN_INTERCEPTOR(strrchr, s, c); @@ -679,8 +754,8 @@ static bool fix_mmap_addr(void **addr, long_t sz, int flags) { return true; } -TSAN_INTERCEPTOR(void*, mmap, void *addr, long_t sz, int prot, - int flags, int fd, unsigned off) { +TSAN_INTERCEPTOR(void *, mmap, void *addr, SIZE_T sz, int prot, int flags, + int fd, OFF_T off) { SCOPED_TSAN_INTERCEPTOR(mmap, addr, sz, prot, flags, fd, off); if (!fix_mmap_addr(&addr, sz, flags)) return MAP_FAILED; @@ -693,9 +768,9 @@ TSAN_INTERCEPTOR(void*, mmap, void *addr, long_t sz, int prot, return res; } -#if !SANITIZER_FREEBSD -TSAN_INTERCEPTOR(void*, mmap64, void *addr, long_t sz, int prot, - int flags, int fd, u64 off) { +#if SANITIZER_LINUX +TSAN_INTERCEPTOR(void *, mmap64, void *addr, SIZE_T sz, int prot, int flags, + int fd, OFF64_T off) { SCOPED_TSAN_INTERCEPTOR(mmap64, addr, sz, prot, flags, fd, off); if (!fix_mmap_addr(&addr, sz, flags)) return MAP_FAILED; @@ -723,7 +798,7 @@ TSAN_INTERCEPTOR(int, munmap, void *addr, long_t sz) { return res; } -#if !SANITIZER_FREEBSD +#if SANITIZER_LINUX TSAN_INTERCEPTOR(void*, memalign, uptr align, uptr sz) { SCOPED_INTERCEPTOR_RAW(memalign, align, sz); return user_alloc(thr, pc, sz, align); @@ -733,6 +808,7 @@ TSAN_INTERCEPTOR(void*, memalign, uptr align, uptr sz) { #define TSAN_MAYBE_INTERCEPT_MEMALIGN #endif +#if !SANITIZER_MAC TSAN_INTERCEPTOR(void*, aligned_alloc, uptr align, uptr sz) { SCOPED_INTERCEPTOR_RAW(memalign, align, sz); return user_alloc(thr, pc, sz, align); @@ -742,8 +818,9 @@ TSAN_INTERCEPTOR(void*, valloc, uptr sz) { SCOPED_INTERCEPTOR_RAW(valloc, sz); return user_alloc(thr, pc, sz, GetPageSizeCached()); } +#endif -#if !SANITIZER_FREEBSD +#if SANITIZER_LINUX TSAN_INTERCEPTOR(void*, pvalloc, uptr sz) { SCOPED_INTERCEPTOR_RAW(pvalloc, sz); sz = RoundUp(sz, GetPageSizeCached()); @@ -754,14 +831,33 @@ TSAN_INTERCEPTOR(void*, pvalloc, uptr sz) { #define TSAN_MAYBE_INTERCEPT_PVALLOC #endif +#if !SANITIZER_MAC TSAN_INTERCEPTOR(int, posix_memalign, void **memptr, uptr align, uptr sz) { SCOPED_INTERCEPTOR_RAW(posix_memalign, memptr, align, sz); *memptr = user_alloc(thr, pc, sz, align); return 0; } +#endif + +// __cxa_guard_acquire and friends need to be intercepted in a special way - +// regular interceptors will break statically-linked libstdc++. Linux +// interceptors are especially defined as weak functions (so that they don't +// cause link errors when user defines them as well). So they silently +// auto-disable themselves when such symbol is already present in the binary. If +// we link libstdc++ statically, it will bring own __cxa_guard_acquire which +// will silently replace our interceptor. That's why on Linux we simply export +// these interceptors with INTERFACE_ATTRIBUTE. +// On OS X, we don't support statically linking, so we just use a regular +// interceptor. +#if SANITIZER_MAC +#define STDCXX_INTERCEPTOR TSAN_INTERCEPTOR +#else +#define STDCXX_INTERCEPTOR(rettype, name, ...) \ + extern "C" rettype INTERFACE_ATTRIBUTE name(__VA_ARGS__) +#endif // Used in thread-safe function static initialization. -extern "C" int INTERFACE_ATTRIBUTE __cxa_guard_acquire(atomic_uint32_t *g) { +STDCXX_INTERCEPTOR(int, __cxa_guard_acquire, atomic_uint32_t *g) { SCOPED_INTERCEPTOR_RAW(__cxa_guard_acquire, g); for (;;) { u32 cmp = atomic_load(g, memory_order_acquire); @@ -777,17 +873,31 @@ extern "C" int INTERFACE_ATTRIBUTE __cxa_guard_acquire(atomic_uint32_t *g) { } } -extern "C" void INTERFACE_ATTRIBUTE __cxa_guard_release(atomic_uint32_t *g) { +STDCXX_INTERCEPTOR(void, __cxa_guard_release, atomic_uint32_t *g) { SCOPED_INTERCEPTOR_RAW(__cxa_guard_release, g); Release(thr, pc, (uptr)g); atomic_store(g, 1, memory_order_release); } -extern "C" void INTERFACE_ATTRIBUTE __cxa_guard_abort(atomic_uint32_t *g) { +STDCXX_INTERCEPTOR(void, __cxa_guard_abort, atomic_uint32_t *g) { SCOPED_INTERCEPTOR_RAW(__cxa_guard_abort, g); atomic_store(g, 0, memory_order_relaxed); } +namespace __tsan { +void DestroyThreadState() { + ThreadState *thr = cur_thread(); + ThreadFinish(thr); + ThreadSignalContext *sctx = thr->signal_ctx; + if (sctx) { + thr->signal_ctx = 0; + UnmapOrDie(sctx, sizeof(*sctx)); + } + cur_thread_finalize(); +} +} // namespace __tsan + +#if !SANITIZER_MAC static void thread_finalize(void *v) { uptr iter = (uptr)v; if (iter > 1) { @@ -797,16 +907,9 @@ static void thread_finalize(void *v) { } return; } - { - ThreadState *thr = cur_thread(); - ThreadFinish(thr); - ThreadSignalContext *sctx = thr->signal_ctx; - if (sctx) { - thr->signal_ctx = 0; - UnmapOrDie(sctx, sizeof(*sctx)); - } - } + DestroyThreadState(); } +#endif struct ThreadParam { @@ -824,6 +927,7 @@ extern "C" void *__tsan_thread_start_func(void *arg) { ThreadState *thr = cur_thread(); // Thread-local state is not initialized yet. ScopedIgnoreInterceptors ignore; +#if !SANITIZER_MAC ThreadIgnoreBegin(thr, 0); if (pthread_setspecific(g_thread_finalize_key, (void *)GetPthreadDestructorIterations())) { @@ -831,8 +935,9 @@ extern "C" void *__tsan_thread_start_func(void *arg) { Die(); } ThreadIgnoreEnd(thr, 0); +#endif while ((tid = atomic_load(&p->tid, memory_order_acquire)) == 0) - pthread_yield(); + internal_sched_yield(); ThreadStart(thr, tid, GetTid()); atomic_store(&p->tid, 0, memory_order_release); } @@ -880,7 +985,8 @@ TSAN_INTERCEPTOR(int, pthread_create, ThreadIgnoreEnd(thr, pc); } if (res == 0) { - int tid = ThreadCreate(thr, pc, *(uptr*)th, detached); + int tid = ThreadCreate(thr, pc, *(uptr*)th, + detached == PTHREAD_CREATE_DETACHED); CHECK_NE(tid, 0); // Synchronization on p.tid serves two purposes: // 1. ThreadCreate must finish before the new thread starts. @@ -891,7 +997,7 @@ TSAN_INTERCEPTOR(int, pthread_create, // before the new thread got a chance to acquire from it in ThreadStart. atomic_store(&p.tid, tid, memory_order_release); while (atomic_load(&p.tid, memory_order_acquire) != 0) - pthread_yield(); + internal_sched_yield(); } if (attr == &myattr) pthread_attr_destroy(&myattr); @@ -1094,6 +1200,7 @@ TSAN_INTERCEPTOR(int, pthread_mutex_trylock, void *m) { return res; } +#if !SANITIZER_MAC TSAN_INTERCEPTOR(int, pthread_mutex_timedlock, void *m, void *abstime) { SCOPED_TSAN_INTERCEPTOR(pthread_mutex_timedlock, m, abstime); int res = REAL(pthread_mutex_timedlock)(m, abstime); @@ -1102,7 +1209,9 @@ TSAN_INTERCEPTOR(int, pthread_mutex_timedlock, void *m, void *abstime) { } return res; } +#endif +#if !SANITIZER_MAC TSAN_INTERCEPTOR(int, pthread_spin_init, void *m, int pshared) { SCOPED_TSAN_INTERCEPTOR(pthread_spin_init, m, pshared); int res = REAL(pthread_spin_init)(m, pshared); @@ -1145,6 +1254,7 @@ TSAN_INTERCEPTOR(int, pthread_spin_unlock, void *m) { int res = REAL(pthread_spin_unlock)(m); return res; } +#endif TSAN_INTERCEPTOR(int, pthread_rwlock_init, void *m, void *a) { SCOPED_TSAN_INTERCEPTOR(pthread_rwlock_init, m, a); @@ -1182,6 +1292,7 @@ TSAN_INTERCEPTOR(int, pthread_rwlock_tryrdlock, void *m) { return res; } +#if !SANITIZER_MAC TSAN_INTERCEPTOR(int, pthread_rwlock_timedrdlock, void *m, void *abstime) { SCOPED_TSAN_INTERCEPTOR(pthread_rwlock_timedrdlock, m, abstime); int res = REAL(pthread_rwlock_timedrdlock)(m, abstime); @@ -1190,6 +1301,7 @@ TSAN_INTERCEPTOR(int, pthread_rwlock_timedrdlock, void *m, void *abstime) { } return res; } +#endif TSAN_INTERCEPTOR(int, pthread_rwlock_wrlock, void *m) { SCOPED_TSAN_INTERCEPTOR(pthread_rwlock_wrlock, m); @@ -1209,6 +1321,7 @@ TSAN_INTERCEPTOR(int, pthread_rwlock_trywrlock, void *m) { return res; } +#if !SANITIZER_MAC TSAN_INTERCEPTOR(int, pthread_rwlock_timedwrlock, void *m, void *abstime) { SCOPED_TSAN_INTERCEPTOR(pthread_rwlock_timedwrlock, m, abstime); int res = REAL(pthread_rwlock_timedwrlock)(m, abstime); @@ -1217,6 +1330,7 @@ TSAN_INTERCEPTOR(int, pthread_rwlock_timedwrlock, void *m, void *abstime) { } return res; } +#endif TSAN_INTERCEPTOR(int, pthread_rwlock_unlock, void *m) { SCOPED_TSAN_INTERCEPTOR(pthread_rwlock_unlock, m); @@ -1225,6 +1339,7 @@ TSAN_INTERCEPTOR(int, pthread_rwlock_unlock, void *m) { return res; } +#if !SANITIZER_MAC TSAN_INTERCEPTOR(int, pthread_barrier_init, void *b, void *a, unsigned count) { SCOPED_TSAN_INTERCEPTOR(pthread_barrier_init, b, a, count); MemoryWrite(thr, pc, (uptr)b, kSizeLog1); @@ -1250,12 +1365,17 @@ TSAN_INTERCEPTOR(int, pthread_barrier_wait, void *b) { } return res; } +#endif TSAN_INTERCEPTOR(int, pthread_once, void *o, void (*f)()) { SCOPED_INTERCEPTOR_RAW(pthread_once, o, f); if (o == 0 || f == 0) return EINVAL; - atomic_uint32_t *a = static_cast<atomic_uint32_t*>(o); + atomic_uint32_t *a; + if (!SANITIZER_MAC) + a = static_cast<atomic_uint32_t*>(o); + else // On OS X, pthread_once_t has a header with a long-sized signature. + a = static_cast<atomic_uint32_t*>((void *)((char *)o + sizeof(long_t))); u32 v = atomic_load(a, memory_order_acquire); if (v == 0 && atomic_compare_exchange_strong(a, &v, 1, memory_order_relaxed)) { @@ -1265,7 +1385,7 @@ TSAN_INTERCEPTOR(int, pthread_once, void *o, void (*f)()) { atomic_store(a, 2, memory_order_release); } else { while (v != 2) { - pthread_yield(); + internal_sched_yield(); v = atomic_load(a, memory_order_acquire); } if (!thr->in_ignored_lib) @@ -1274,62 +1394,7 @@ TSAN_INTERCEPTOR(int, pthread_once, void *o, void (*f)()) { return 0; } -TSAN_INTERCEPTOR(int, sem_init, void *s, int pshared, unsigned value) { - SCOPED_TSAN_INTERCEPTOR(sem_init, s, pshared, value); - int res = REAL(sem_init)(s, pshared, value); - return res; -} - -TSAN_INTERCEPTOR(int, sem_destroy, void *s) { - SCOPED_TSAN_INTERCEPTOR(sem_destroy, s); - int res = REAL(sem_destroy)(s); - return res; -} - -TSAN_INTERCEPTOR(int, sem_wait, void *s) { - SCOPED_TSAN_INTERCEPTOR(sem_wait, s); - int res = BLOCK_REAL(sem_wait)(s); - if (res == 0) { - Acquire(thr, pc, (uptr)s); - } - return res; -} - -TSAN_INTERCEPTOR(int, sem_trywait, void *s) { - SCOPED_TSAN_INTERCEPTOR(sem_trywait, s); - int res = BLOCK_REAL(sem_trywait)(s); - if (res == 0) { - Acquire(thr, pc, (uptr)s); - } - return res; -} - -TSAN_INTERCEPTOR(int, sem_timedwait, void *s, void *abstime) { - SCOPED_TSAN_INTERCEPTOR(sem_timedwait, s, abstime); - int res = BLOCK_REAL(sem_timedwait)(s, abstime); - if (res == 0) { - Acquire(thr, pc, (uptr)s); - } - return res; -} - -TSAN_INTERCEPTOR(int, sem_post, void *s) { - SCOPED_TSAN_INTERCEPTOR(sem_post, s); - Release(thr, pc, (uptr)s); - int res = REAL(sem_post)(s); - return res; -} - -TSAN_INTERCEPTOR(int, sem_getvalue, void *s, int *sval) { - SCOPED_TSAN_INTERCEPTOR(sem_getvalue, s, sval); - int res = REAL(sem_getvalue)(s, sval); - if (res == 0) { - Acquire(thr, pc, (uptr)s); - } - return res; -} - -#if !SANITIZER_FREEBSD +#if SANITIZER_LINUX && !SANITIZER_ANDROID TSAN_INTERCEPTOR(int, __xstat, int version, const char *path, void *buf) { SCOPED_TSAN_INTERCEPTOR(__xstat, version, path, buf); READ_STRING(thr, pc, path, 0); @@ -1341,7 +1406,7 @@ TSAN_INTERCEPTOR(int, __xstat, int version, const char *path, void *buf) { #endif TSAN_INTERCEPTOR(int, stat, const char *path, void *buf) { -#if SANITIZER_FREEBSD +#if SANITIZER_FREEBSD || SANITIZER_MAC || SANITIZER_ANDROID SCOPED_TSAN_INTERCEPTOR(stat, path, buf); READ_STRING(thr, pc, path, 0); return REAL(stat)(path, buf); @@ -1352,7 +1417,7 @@ TSAN_INTERCEPTOR(int, stat, const char *path, void *buf) { #endif } -#if !SANITIZER_FREEBSD +#if SANITIZER_LINUX && !SANITIZER_ANDROID TSAN_INTERCEPTOR(int, __xstat64, int version, const char *path, void *buf) { SCOPED_TSAN_INTERCEPTOR(__xstat64, version, path, buf); READ_STRING(thr, pc, path, 0); @@ -1363,7 +1428,7 @@ TSAN_INTERCEPTOR(int, __xstat64, int version, const char *path, void *buf) { #define TSAN_MAYBE_INTERCEPT___XSTAT64 #endif -#if !SANITIZER_FREEBSD +#if SANITIZER_LINUX && !SANITIZER_ANDROID TSAN_INTERCEPTOR(int, stat64, const char *path, void *buf) { SCOPED_TSAN_INTERCEPTOR(__xstat64, 0, path, buf); READ_STRING(thr, pc, path, 0); @@ -1374,7 +1439,7 @@ TSAN_INTERCEPTOR(int, stat64, const char *path, void *buf) { #define TSAN_MAYBE_INTERCEPT_STAT64 #endif -#if !SANITIZER_FREEBSD +#if SANITIZER_LINUX && !SANITIZER_ANDROID TSAN_INTERCEPTOR(int, __lxstat, int version, const char *path, void *buf) { SCOPED_TSAN_INTERCEPTOR(__lxstat, version, path, buf); READ_STRING(thr, pc, path, 0); @@ -1386,7 +1451,7 @@ TSAN_INTERCEPTOR(int, __lxstat, int version, const char *path, void *buf) { #endif TSAN_INTERCEPTOR(int, lstat, const char *path, void *buf) { -#if SANITIZER_FREEBSD +#if SANITIZER_FREEBSD || SANITIZER_MAC || SANITIZER_ANDROID SCOPED_TSAN_INTERCEPTOR(lstat, path, buf); READ_STRING(thr, pc, path, 0); return REAL(lstat)(path, buf); @@ -1397,7 +1462,7 @@ TSAN_INTERCEPTOR(int, lstat, const char *path, void *buf) { #endif } -#if !SANITIZER_FREEBSD +#if SANITIZER_LINUX && !SANITIZER_ANDROID TSAN_INTERCEPTOR(int, __lxstat64, int version, const char *path, void *buf) { SCOPED_TSAN_INTERCEPTOR(__lxstat64, version, path, buf); READ_STRING(thr, pc, path, 0); @@ -1408,7 +1473,7 @@ TSAN_INTERCEPTOR(int, __lxstat64, int version, const char *path, void *buf) { #define TSAN_MAYBE_INTERCEPT___LXSTAT64 #endif -#if !SANITIZER_FREEBSD +#if SANITIZER_LINUX && !SANITIZER_ANDROID TSAN_INTERCEPTOR(int, lstat64, const char *path, void *buf) { SCOPED_TSAN_INTERCEPTOR(__lxstat64, 0, path, buf); READ_STRING(thr, pc, path, 0); @@ -1419,7 +1484,7 @@ TSAN_INTERCEPTOR(int, lstat64, const char *path, void *buf) { #define TSAN_MAYBE_INTERCEPT_LSTAT64 #endif -#if !SANITIZER_FREEBSD +#if SANITIZER_LINUX && !SANITIZER_ANDROID TSAN_INTERCEPTOR(int, __fxstat, int version, int fd, void *buf) { SCOPED_TSAN_INTERCEPTOR(__fxstat, version, fd, buf); if (fd > 0) @@ -1432,7 +1497,7 @@ TSAN_INTERCEPTOR(int, __fxstat, int version, int fd, void *buf) { #endif TSAN_INTERCEPTOR(int, fstat, int fd, void *buf) { -#if SANITIZER_FREEBSD +#if SANITIZER_FREEBSD || SANITIZER_MAC || SANITIZER_ANDROID SCOPED_TSAN_INTERCEPTOR(fstat, fd, buf); if (fd > 0) FdAccess(thr, pc, fd); @@ -1445,7 +1510,7 @@ TSAN_INTERCEPTOR(int, fstat, int fd, void *buf) { #endif } -#if !SANITIZER_FREEBSD +#if SANITIZER_LINUX && !SANITIZER_ANDROID TSAN_INTERCEPTOR(int, __fxstat64, int version, int fd, void *buf) { SCOPED_TSAN_INTERCEPTOR(__fxstat64, version, fd, buf); if (fd > 0) @@ -1457,7 +1522,7 @@ TSAN_INTERCEPTOR(int, __fxstat64, int version, int fd, void *buf) { #define TSAN_MAYBE_INTERCEPT___FXSTAT64 #endif -#if !SANITIZER_FREEBSD +#if SANITIZER_LINUX && !SANITIZER_ANDROID TSAN_INTERCEPTOR(int, fstat64, int fd, void *buf) { SCOPED_TSAN_INTERCEPTOR(__fxstat64, 0, fd, buf); if (fd > 0) @@ -1478,7 +1543,7 @@ TSAN_INTERCEPTOR(int, open, const char *name, int flags, int mode) { return fd; } -#if !SANITIZER_FREEBSD +#if SANITIZER_LINUX TSAN_INTERCEPTOR(int, open64, const char *name, int flags, int mode) { SCOPED_TSAN_INTERCEPTOR(open64, name, flags, mode); READ_STRING(thr, pc, name, 0); @@ -1501,7 +1566,7 @@ TSAN_INTERCEPTOR(int, creat, const char *name, int mode) { return fd; } -#if !SANITIZER_FREEBSD +#if SANITIZER_LINUX TSAN_INTERCEPTOR(int, creat64, const char *name, int mode) { SCOPED_TSAN_INTERCEPTOR(creat64, name, mode); READ_STRING(thr, pc, name, 0); @@ -1531,6 +1596,7 @@ TSAN_INTERCEPTOR(int, dup2, int oldfd, int newfd) { return newfd2; } +#if !SANITIZER_MAC TSAN_INTERCEPTOR(int, dup3, int oldfd, int newfd, int flags) { SCOPED_TSAN_INTERCEPTOR(dup3, oldfd, newfd, flags); int newfd2 = REAL(dup3)(oldfd, newfd, flags); @@ -1538,8 +1604,9 @@ TSAN_INTERCEPTOR(int, dup3, int oldfd, int newfd, int flags) { FdDup(thr, pc, oldfd, newfd2, false); return newfd2; } +#endif -#if !SANITIZER_FREEBSD +#if SANITIZER_LINUX TSAN_INTERCEPTOR(int, eventfd, unsigned initval, int flags) { SCOPED_TSAN_INTERCEPTOR(eventfd, initval, flags); int fd = REAL(eventfd)(initval, flags); @@ -1552,7 +1619,7 @@ TSAN_INTERCEPTOR(int, eventfd, unsigned initval, int flags) { #define TSAN_MAYBE_INTERCEPT_EVENTFD #endif -#if !SANITIZER_FREEBSD +#if SANITIZER_LINUX TSAN_INTERCEPTOR(int, signalfd, int fd, void *mask, int flags) { SCOPED_TSAN_INTERCEPTOR(signalfd, fd, mask, flags); if (fd >= 0) @@ -1567,7 +1634,7 @@ TSAN_INTERCEPTOR(int, signalfd, int fd, void *mask, int flags) { #define TSAN_MAYBE_INTERCEPT_SIGNALFD #endif -#if !SANITIZER_FREEBSD +#if SANITIZER_LINUX TSAN_INTERCEPTOR(int, inotify_init, int fake) { SCOPED_TSAN_INTERCEPTOR(inotify_init, fake); int fd = REAL(inotify_init)(fake); @@ -1580,7 +1647,7 @@ TSAN_INTERCEPTOR(int, inotify_init, int fake) { #define TSAN_MAYBE_INTERCEPT_INOTIFY_INIT #endif -#if !SANITIZER_FREEBSD +#if SANITIZER_LINUX TSAN_INTERCEPTOR(int, inotify_init1, int flags) { SCOPED_TSAN_INTERCEPTOR(inotify_init1, flags); int fd = REAL(inotify_init1)(flags); @@ -1634,7 +1701,7 @@ TSAN_INTERCEPTOR(int, listen, int fd, int backlog) { return res; } -#if !SANITIZER_FREEBSD +#if SANITIZER_LINUX TSAN_INTERCEPTOR(int, epoll_create, int size) { SCOPED_TSAN_INTERCEPTOR(epoll_create, size); int fd = REAL(epoll_create)(size); @@ -1647,7 +1714,7 @@ TSAN_INTERCEPTOR(int, epoll_create, int size) { #define TSAN_MAYBE_INTERCEPT_EPOLL_CREATE #endif -#if !SANITIZER_FREEBSD +#if SANITIZER_LINUX TSAN_INTERCEPTOR(int, epoll_create1, int flags) { SCOPED_TSAN_INTERCEPTOR(epoll_create1, flags); int fd = REAL(epoll_create1)(flags); @@ -1667,7 +1734,7 @@ TSAN_INTERCEPTOR(int, close, int fd) { return REAL(close)(fd); } -#if !SANITIZER_FREEBSD +#if SANITIZER_LINUX TSAN_INTERCEPTOR(int, __close, int fd) { SCOPED_TSAN_INTERCEPTOR(__close, fd); if (fd >= 0) @@ -1680,7 +1747,7 @@ TSAN_INTERCEPTOR(int, __close, int fd) { #endif // glibc guts -#if !SANITIZER_FREEBSD +#if SANITIZER_LINUX && !SANITIZER_ANDROID TSAN_INTERCEPTOR(void, __res_iclose, void *state, bool free_addr) { SCOPED_TSAN_INTERCEPTOR(__res_iclose, state, free_addr); int fds[64]; @@ -1704,6 +1771,7 @@ TSAN_INTERCEPTOR(int, pipe, int *pipefd) { return res; } +#if !SANITIZER_MAC TSAN_INTERCEPTOR(int, pipe2, int *pipefd, int flags) { SCOPED_TSAN_INTERCEPTOR(pipe2, pipefd, flags); int res = REAL(pipe2)(pipefd, flags); @@ -1711,6 +1779,7 @@ TSAN_INTERCEPTOR(int, pipe2, int *pipefd, int flags) { FdPipeCreate(thr, pc, pipefd[0], pipefd[1]); return res; } +#endif TSAN_INTERCEPTOR(long_t, send, int fd, void *buf, long_t len, int flags) { SCOPED_TSAN_INTERCEPTOR(send, fd, buf, len, flags); @@ -1761,7 +1830,7 @@ TSAN_INTERCEPTOR(void*, tmpfile, int fake) { return res; } -#if !SANITIZER_FREEBSD +#if SANITIZER_LINUX TSAN_INTERCEPTOR(void*, tmpfile64, int fake) { SCOPED_TSAN_INTERCEPTOR(tmpfile64, fake); void *res = REAL(tmpfile64)(fake); @@ -1828,7 +1897,7 @@ TSAN_INTERCEPTOR(int, closedir, void *dirp) { return REAL(closedir)(dirp); } -#if !SANITIZER_FREEBSD +#if SANITIZER_LINUX TSAN_INTERCEPTOR(int, epoll_ctl, int epfd, int op, int fd, void *ev) { SCOPED_TSAN_INTERCEPTOR(epoll_ctl, epfd, op, fd, ev); if (epfd >= 0) @@ -1845,7 +1914,7 @@ TSAN_INTERCEPTOR(int, epoll_ctl, int epfd, int op, int fd, void *ev) { #define TSAN_MAYBE_INTERCEPT_EPOLL_CTL #endif -#if !SANITIZER_FREEBSD +#if SANITIZER_LINUX TSAN_INTERCEPTOR(int, epoll_wait, int epfd, void *ev, int cnt, int timeout) { SCOPED_TSAN_INTERCEPTOR(epoll_wait, epfd, ev, cnt, timeout); if (epfd >= 0) @@ -1895,7 +1964,7 @@ static void CallUserSignalHandler(ThreadState *thr, bool sync, bool acquire, ObtainCurrentStack(thr, StackTrace::GetNextInstructionPc(pc), &stack); ThreadRegistryLock l(ctx->thread_registry); ScopedReport rep(ReportTypeErrnoInSignal); - if (!IsFiredSuppression(ctx, rep, stack)) { + if (!IsFiredSuppression(ctx, ReportTypeErrnoInSignal, stack)) { rep.AddStack(stack, true); OutputReport(thr, rep); } @@ -1910,10 +1979,8 @@ void ProcessPendingSignals(ThreadState *thr) { return; atomic_store(&sctx->have_pending_signals, 0, memory_order_relaxed); atomic_fetch_add(&thr->in_signal_handler, 1, memory_order_relaxed); - // These are too big for stack. - static THREADLOCAL __sanitizer_sigset_t emptyset, oldset; - CHECK_EQ(0, REAL(sigfillset)(&emptyset)); - CHECK_EQ(0, pthread_sigmask(SIG_SETMASK, &emptyset, &oldset)); + CHECK_EQ(0, REAL(sigfillset)(&sctx->emptyset)); + CHECK_EQ(0, pthread_sigmask(SIG_SETMASK, &sctx->emptyset, &sctx->oldset)); for (int sig = 0; sig < kSigCount; sig++) { SignalDesc *signal = &sctx->pending_signals[sig]; if (signal->armed) { @@ -1922,7 +1989,7 @@ void ProcessPendingSignals(ThreadState *thr) { &signal->siginfo, &signal->ctx); } } - CHECK_EQ(0, pthread_sigmask(SIG_SETMASK, &oldset, 0)); + CHECK_EQ(0, pthread_sigmask(SIG_SETMASK, &sctx->oldset, 0)); atomic_fetch_add(&thr->in_signal_handler, -1, memory_order_relaxed); } @@ -2011,7 +2078,7 @@ TSAN_INTERCEPTOR(int, sigaction, int sig, sigaction_t *act, sigaction_t *old) { sigactions[sig].sa_flags = *(volatile int*)&act->sa_flags; internal_memcpy(&sigactions[sig].sa_mask, &act->sa_mask, sizeof(sigactions[sig].sa_mask)); -#if !SANITIZER_FREEBSD +#if !SANITIZER_FREEBSD && !SANITIZER_MAC sigactions[sig].sa_restorer = act->sa_restorer; #endif sigaction_t newact; @@ -2144,6 +2211,7 @@ TSAN_INTERCEPTOR(int, vfork, int fake) { return WRAP(fork)(fake); } +#if !SANITIZER_MAC && !SANITIZER_ANDROID typedef int (*dl_iterate_phdr_cb_t)(__sanitizer_dl_phdr_info *info, SIZE_T size, void *data); struct dl_iterate_phdr_data { @@ -2187,6 +2255,7 @@ TSAN_INTERCEPTOR(int, dl_iterate_phdr, dl_iterate_phdr_cb_t cb, void *data) { int res = REAL(dl_iterate_phdr)(dl_iterate_phdr_cb, &cbdata); return res; } +#endif static int OnExit(ThreadState *thr) { int status = Finalize(thr); @@ -2200,6 +2269,7 @@ struct TsanInterceptorContext { const uptr pc; }; +#if !SANITIZER_MAC static void HandleRecvmsg(ThreadState *thr, uptr pc, __sanitizer_msghdr *msg) { int fds[64]; @@ -2207,6 +2277,7 @@ static void HandleRecvmsg(ThreadState *thr, uptr pc, for (int i = 0; i < cnt; i++) FdEventCreate(thr, pc, fds[i]); } +#endif #include "sanitizer_common/sanitizer_platform_interceptors.h" // Causes interceptor recursion (getaddrinfo() and fopen()) @@ -2277,6 +2348,12 @@ static void HandleRecvmsg(ThreadState *thr, uptr pc, #define COMMON_INTERCEPTOR_LIBRARY_UNLOADED() \ libignore()->OnLibraryUnloaded() +#define COMMON_INTERCEPTOR_ACQUIRE(ctx, u) \ + Acquire(((TsanInterceptorContext *) ctx)->thr, pc, u) + +#define COMMON_INTERCEPTOR_RELEASE(ctx, u) \ + Release(((TsanInterceptorContext *) ctx)->thr, pc, u) + #define COMMON_INTERCEPTOR_DIR_ACQUIRE(ctx, path) \ Acquire(((TsanInterceptorContext *) ctx)->thr, pc, Dir2addr(path)) @@ -2315,9 +2392,11 @@ static void HandleRecvmsg(ThreadState *thr, uptr pc, MutexRepair(((TsanInterceptorContext *)ctx)->thr, \ ((TsanInterceptorContext *)ctx)->pc, (uptr)m) +#if !SANITIZER_MAC #define COMMON_INTERCEPTOR_HANDLE_RECVMSG(ctx, msg) \ HandleRecvmsg(((TsanInterceptorContext *)ctx)->thr, \ ((TsanInterceptorContext *)ctx)->pc, msg) +#endif #define COMMON_INTERCEPTOR_GET_TLS_RANGE(begin, end) \ if (TsanThread *t = GetCurrentThread()) { \ @@ -2349,6 +2428,7 @@ struct ScopedSyscall { } }; +#if !SANITIZER_MAC static void syscall_access_range(uptr pc, uptr p, uptr s, bool write) { TSAN_SYSCALL(); MemoryAccessRange(thr, pc, p, s, write); @@ -2402,6 +2482,7 @@ static void syscall_post_fork(uptr pc, int pid) { ForkParentAfter(thr, pc); } } +#endif #define COMMON_SYSCALL_PRE_READ_RANGE(p, s) \ syscall_access_range(GET_CALLER_PC(), (uptr)(p), (uptr)(s), false) @@ -2449,28 +2530,32 @@ static void finalize(void *arg) { // Make sure the output is not lost. FlushStreams(); if (status) - REAL(_exit)(status); + Die(); } +#if !SANITIZER_MAC && !SANITIZER_ANDROID static void unreachable() { Report("FATAL: ThreadSanitizer: unreachable called\n"); Die(); } +#endif void InitializeInterceptors() { +#if !SANITIZER_MAC // We need to setup it early, because functions like dlsym() can call it. REAL(memset) = internal_memset; REAL(memcpy) = internal_memcpy; - REAL(memcmp) = internal_memcmp; +#endif // Instruct libc malloc to consume less memory. -#if !SANITIZER_FREEBSD +#if SANITIZER_LINUX mallopt(1, 0); // M_MXFAST mallopt(-3, 32*1024); // M_MMAP_THRESHOLD #endif InitializeCommonInterceptors(); +#if !SANITIZER_MAC // We can not use TSAN_INTERCEPT to get setjmp addr, // because it does &setjmp and setjmp is not present in some versions of libc. using __interception::GetRealFunctionAddress; @@ -2478,6 +2563,7 @@ void InitializeInterceptors() { GetRealFunctionAddress("_setjmp", (uptr*)&REAL(_setjmp), 0, 0); GetRealFunctionAddress("sigsetjmp", (uptr*)&REAL(sigsetjmp), 0, 0); GetRealFunctionAddress("__sigsetjmp", (uptr*)&REAL(__sigsetjmp), 0, 0); +#endif TSAN_INTERCEPT(longjmp); TSAN_INTERCEPT(siglongjmp); @@ -2500,7 +2586,6 @@ void InitializeInterceptors() { TSAN_INTERCEPT(memset); TSAN_INTERCEPT(memcpy); TSAN_INTERCEPT(memmove); - TSAN_INTERCEPT(memcmp); TSAN_INTERCEPT(strchr); TSAN_INTERCEPT(strchrnul); TSAN_INTERCEPT(strrchr); @@ -2512,12 +2597,12 @@ void InitializeInterceptors() { TSAN_INTERCEPT(pthread_join); TSAN_INTERCEPT(pthread_detach); - TSAN_INTERCEPT_VER(pthread_cond_init, "GLIBC_2.3.2"); - TSAN_INTERCEPT_VER(pthread_cond_signal, "GLIBC_2.3.2"); - TSAN_INTERCEPT_VER(pthread_cond_broadcast, "GLIBC_2.3.2"); - TSAN_INTERCEPT_VER(pthread_cond_wait, "GLIBC_2.3.2"); - TSAN_INTERCEPT_VER(pthread_cond_timedwait, "GLIBC_2.3.2"); - TSAN_INTERCEPT_VER(pthread_cond_destroy, "GLIBC_2.3.2"); + TSAN_INTERCEPT_VER(pthread_cond_init, PTHREAD_ABI_BASE); + TSAN_INTERCEPT_VER(pthread_cond_signal, PTHREAD_ABI_BASE); + TSAN_INTERCEPT_VER(pthread_cond_broadcast, PTHREAD_ABI_BASE); + TSAN_INTERCEPT_VER(pthread_cond_wait, PTHREAD_ABI_BASE); + TSAN_INTERCEPT_VER(pthread_cond_timedwait, PTHREAD_ABI_BASE); + TSAN_INTERCEPT_VER(pthread_cond_destroy, PTHREAD_ABI_BASE); TSAN_INTERCEPT(pthread_mutex_init); TSAN_INTERCEPT(pthread_mutex_destroy); @@ -2546,14 +2631,6 @@ void InitializeInterceptors() { TSAN_INTERCEPT(pthread_once); - TSAN_INTERCEPT(sem_init); - TSAN_INTERCEPT(sem_destroy); - TSAN_INTERCEPT(sem_wait); - TSAN_INTERCEPT(sem_trywait); - TSAN_INTERCEPT(sem_timedwait); - TSAN_INTERCEPT(sem_post); - TSAN_INTERCEPT(sem_getvalue); - TSAN_INTERCEPT(stat); TSAN_MAYBE_INTERCEPT___XSTAT; TSAN_MAYBE_INTERCEPT_STAT64; @@ -2621,25 +2698,68 @@ void InitializeInterceptors() { TSAN_INTERCEPT(fork); TSAN_INTERCEPT(vfork); +#if !SANITIZER_ANDROID TSAN_INTERCEPT(dl_iterate_phdr); +#endif TSAN_INTERCEPT(on_exit); TSAN_INTERCEPT(__cxa_atexit); TSAN_INTERCEPT(_exit); +#if !SANITIZER_MAC && !SANITIZER_ANDROID // Need to setup it, because interceptors check that the function is resolved. // But atexit is emitted directly into the module, so can't be resolved. REAL(atexit) = (int(*)(void(*)()))unreachable; +#endif + if (REAL(__cxa_atexit)(&finalize, 0, 0)) { Printf("ThreadSanitizer: failed to setup atexit callback\n"); Die(); } +#if !SANITIZER_MAC if (pthread_key_create(&g_thread_finalize_key, &thread_finalize)) { Printf("ThreadSanitizer: failed to create thread key\n"); Die(); } +#endif FdInit(); } } // namespace __tsan + +// Invisible barrier for tests. +// There were several unsuccessful iterations for this functionality: +// 1. Initially it was implemented in user code using +// REAL(pthread_barrier_wait). But pthread_barrier_wait is not supported on +// MacOS. Futexes are linux-specific for this matter. +// 2. Then we switched to atomics+usleep(10). But usleep produced parasitic +// "as-if synchronized via sleep" messages in reports which failed some +// output tests. +// 3. Then we switched to atomics+sched_yield. But this produced tons of tsan- +// visible events, which lead to "failed to restore stack trace" failures. +// Note that no_sanitize_thread attribute does not turn off atomic interception +// so attaching it to the function defined in user code does not help. +// That's why we now have what we have. +extern "C" SANITIZER_INTERFACE_ATTRIBUTE +void __tsan_testonly_barrier_init(u64 *barrier, u32 count) { + if (count >= (1 << 8)) { + Printf("barrier_init: count is too large (%d)\n", count); + Die(); + } + // 8 lsb is thread count, the remaining are count of entered threads. + *barrier = count; +} + +extern "C" SANITIZER_INTERFACE_ATTRIBUTE +void __tsan_testonly_barrier_wait(u64 *barrier) { + unsigned old = __atomic_fetch_add(barrier, 1 << 8, __ATOMIC_RELAXED); + unsigned old_epoch = (old >> 8) / (old & 0xff); + for (;;) { + unsigned cur = __atomic_load_n(barrier, __ATOMIC_RELAXED); + unsigned cur_epoch = (cur >> 8) / (cur & 0xff); + if (cur_epoch != old_epoch) + return; + internal_sched_yield(); + } +} diff --git a/lib/tsan/rtl/tsan_interceptors.h b/lib/tsan/rtl/tsan_interceptors.h index 49b79a7c5f9e..d831620cfafe 100644 --- a/lib/tsan/rtl/tsan_interceptors.h +++ b/lib/tsan/rtl/tsan_interceptors.h @@ -10,6 +10,8 @@ class ScopedInterceptor { public: ScopedInterceptor(ThreadState *thr, const char *fname, uptr pc); ~ScopedInterceptor(); + void UserCallbackStart(); + void UserCallbackEnd(); private: ThreadState *const thr_; const uptr pc_; @@ -26,6 +28,24 @@ class ScopedInterceptor { (void)pc; \ /**/ +#define SCOPED_TSAN_INTERCEPTOR(func, ...) \ + SCOPED_INTERCEPTOR_RAW(func, __VA_ARGS__); \ + if (REAL(func) == 0) { \ + Report("FATAL: ThreadSanitizer: failed to intercept %s\n", #func); \ + Die(); \ + } \ + if (thr->ignore_interceptors || thr->in_ignored_lib) \ + return REAL(func)(__VA_ARGS__); \ +/**/ + +#define SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_START() \ + si.UserCallbackStart(); + +#define SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_END() \ + si.UserCallbackEnd(); + +#define TSAN_INTERCEPTOR(ret, func, ...) INTERCEPTOR(ret, func, __VA_ARGS__) + #if SANITIZER_FREEBSD #define __libc_free __free #define __libc_malloc __malloc diff --git a/lib/tsan/rtl/tsan_interceptors_mac.cc b/lib/tsan/rtl/tsan_interceptors_mac.cc new file mode 100644 index 000000000000..2bf7ad9861c4 --- /dev/null +++ b/lib/tsan/rtl/tsan_interceptors_mac.cc @@ -0,0 +1,91 @@ +//===-- tsan_interceptors_mac.cc ------------------------------------------===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// +// +// This file is a part of ThreadSanitizer (TSan), a race detector. +// +// Mac-specific interceptors. +//===----------------------------------------------------------------------===// + +#include "sanitizer_common/sanitizer_platform.h" +#if SANITIZER_MAC + +#include "interception/interception.h" +#include "tsan_interceptors.h" + +#include <libkern/OSAtomic.h> + +namespace __tsan { + +TSAN_INTERCEPTOR(void, OSSpinLockLock, volatile OSSpinLock *lock) { + CHECK(!cur_thread()->is_dead); + if (!cur_thread()->is_inited) { + return REAL(OSSpinLockLock)(lock); + } + SCOPED_TSAN_INTERCEPTOR(OSSpinLockLock, lock); + REAL(OSSpinLockLock)(lock); + Acquire(thr, pc, (uptr)lock); +} + +TSAN_INTERCEPTOR(bool, OSSpinLockTry, volatile OSSpinLock *lock) { + CHECK(!cur_thread()->is_dead); + if (!cur_thread()->is_inited) { + return REAL(OSSpinLockTry)(lock); + } + SCOPED_TSAN_INTERCEPTOR(OSSpinLockTry, lock); + bool result = REAL(OSSpinLockTry)(lock); + if (result) + Acquire(thr, pc, (uptr)lock); + return result; +} + +TSAN_INTERCEPTOR(void, OSSpinLockUnlock, volatile OSSpinLock *lock) { + CHECK(!cur_thread()->is_dead); + if (!cur_thread()->is_inited) { + return REAL(OSSpinLockUnlock)(lock); + } + SCOPED_TSAN_INTERCEPTOR(OSSpinLockUnlock, lock); + Release(thr, pc, (uptr)lock); + REAL(OSSpinLockUnlock)(lock); +} + +TSAN_INTERCEPTOR(void, os_lock_lock, void *lock) { + CHECK(!cur_thread()->is_dead); + if (!cur_thread()->is_inited) { + return REAL(os_lock_lock)(lock); + } + SCOPED_TSAN_INTERCEPTOR(os_lock_lock, lock); + REAL(os_lock_lock)(lock); + Acquire(thr, pc, (uptr)lock); +} + +TSAN_INTERCEPTOR(bool, os_lock_trylock, void *lock) { + CHECK(!cur_thread()->is_dead); + if (!cur_thread()->is_inited) { + return REAL(os_lock_trylock)(lock); + } + SCOPED_TSAN_INTERCEPTOR(os_lock_trylock, lock); + bool result = REAL(os_lock_trylock)(lock); + if (result) + Acquire(thr, pc, (uptr)lock); + return result; +} + +TSAN_INTERCEPTOR(void, os_lock_unlock, void *lock) { + CHECK(!cur_thread()->is_dead); + if (!cur_thread()->is_inited) { + return REAL(os_lock_unlock)(lock); + } + SCOPED_TSAN_INTERCEPTOR(os_lock_unlock, lock); + Release(thr, pc, (uptr)lock); + REAL(os_lock_unlock)(lock); +} + +} // namespace __tsan + +#endif // SANITIZER_MAC diff --git a/lib/tsan/rtl/tsan_interface_ann.cc b/lib/tsan/rtl/tsan_interface_ann.cc index fd3c846678f5..62db79661625 100644 --- a/lib/tsan/rtl/tsan_interface_ann.cc +++ b/lib/tsan/rtl/tsan_interface_ann.cc @@ -63,8 +63,8 @@ static const int kMaxDescLen = 128; struct ExpectRace { ExpectRace *next; ExpectRace *prev; - int hitcount; - int addcount; + atomic_uintptr_t hitcount; + atomic_uintptr_t addcount; uptr addr; uptr size; char *file; @@ -90,7 +90,8 @@ static void AddExpectRace(ExpectRace *list, ExpectRace *race = list->next; for (; race != list; race = race->next) { if (race->addr == addr && race->size == size) { - race->addcount++; + atomic_store_relaxed(&race->addcount, + atomic_load_relaxed(&race->addcount) + 1); return; } } @@ -100,8 +101,8 @@ static void AddExpectRace(ExpectRace *list, race->file = f; race->line = l; race->desc[0] = 0; - race->hitcount = 0; - race->addcount = 1; + atomic_store_relaxed(&race->hitcount, 0); + atomic_store_relaxed(&race->addcount, 1); if (desc) { int i = 0; for (; i < kMaxDescLen - 1 && desc[i]; i++) @@ -130,7 +131,7 @@ static bool CheckContains(ExpectRace *list, uptr addr, uptr size) { return false; DPrintf("Hit expected/benign race: %s addr=%zx:%d %s:%d\n", race->desc, race->addr, (int)race->size, race->file, race->line); - race->hitcount++; + atomic_fetch_add(&race->hitcount, 1, memory_order_relaxed); return true; } @@ -146,7 +147,7 @@ void InitializeDynamicAnnotations() { } bool IsExpectedReport(uptr addr, uptr size) { - Lock lock(&dyn_ann_ctx->mtx); + ReadLock lock(&dyn_ann_ctx->mtx); if (CheckContains(&dyn_ann_ctx->expect, addr, size)) return true; if (CheckContains(&dyn_ann_ctx->benign, addr, size)) @@ -155,20 +156,21 @@ bool IsExpectedReport(uptr addr, uptr size) { } static void CollectMatchedBenignRaces(Vector<ExpectRace> *matched, - int *unique_count, int *hit_count, int ExpectRace::*counter) { + int *unique_count, int *hit_count, atomic_uintptr_t ExpectRace::*counter) { ExpectRace *list = &dyn_ann_ctx->benign; for (ExpectRace *race = list->next; race != list; race = race->next) { (*unique_count)++; - if (race->*counter == 0) + const uptr cnt = atomic_load_relaxed(&(race->*counter)); + if (cnt == 0) continue; - (*hit_count) += race->*counter; + *hit_count += cnt; uptr i = 0; for (; i < matched->Size(); i++) { ExpectRace *race0 = &(*matched)[i]; if (race->line == race0->line && internal_strcmp(race->file, race0->file) == 0 && internal_strcmp(race->desc, race0->desc) == 0) { - race0->*counter += race->*counter; + atomic_fetch_add(&(race0->*counter), cnt, memory_order_relaxed); break; } } @@ -193,8 +195,8 @@ void PrintMatchedBenignRaces() { hit_count, (int)internal_getpid()); for (uptr i = 0; i < hit_matched.Size(); i++) { Printf("%d %s:%d %s\n", - hit_matched[i].hitcount, hit_matched[i].file, - hit_matched[i].line, hit_matched[i].desc); + atomic_load_relaxed(&hit_matched[i].hitcount), + hit_matched[i].file, hit_matched[i].line, hit_matched[i].desc); } } if (hit_matched.Size()) { @@ -203,8 +205,8 @@ void PrintMatchedBenignRaces() { add_count, unique_count, (int)internal_getpid()); for (uptr i = 0; i < add_matched.Size(); i++) { Printf("%d %s:%d %s\n", - add_matched[i].addcount, add_matched[i].file, - add_matched[i].line, add_matched[i].desc); + atomic_load_relaxed(&add_matched[i].addcount), + add_matched[i].file, add_matched[i].line, add_matched[i].desc); } } } @@ -303,7 +305,7 @@ void INTERFACE_ATTRIBUTE AnnotateFlushExpectedRaces(char *f, int l) { Lock lock(&dyn_ann_ctx->mtx); while (dyn_ann_ctx->expect.next != &dyn_ann_ctx->expect) { ExpectRace *race = dyn_ann_ctx->expect.next; - if (race->hitcount == 0) { + if (atomic_load_relaxed(&race->hitcount) == 0) { ctx->nmissed_expected++; ReportMissedExpectedRace(race); } diff --git a/lib/tsan/rtl/tsan_libdispatch_mac.cc b/lib/tsan/rtl/tsan_libdispatch_mac.cc new file mode 100644 index 000000000000..617dc91b33d0 --- /dev/null +++ b/lib/tsan/rtl/tsan_libdispatch_mac.cc @@ -0,0 +1,284 @@ +//===-- tsan_libdispatch_mac.cc -------------------------------------------===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// +// +// This file is a part of ThreadSanitizer (TSan), a race detector. +// +// Mac-specific libdispatch (GCD) support. +//===----------------------------------------------------------------------===// + +#include "sanitizer_common/sanitizer_platform.h" +#if SANITIZER_MAC + +#include "sanitizer_common/sanitizer_common.h" +#include "interception/interception.h" +#include "tsan_interceptors.h" +#include "tsan_platform.h" +#include "tsan_rtl.h" + +#include <Block.h> +#include <dispatch/dispatch.h> +#include <pthread.h> + +typedef long long_t; // NOLINT + +namespace __tsan { + +typedef struct { + dispatch_queue_t queue; + void *orig_context; + dispatch_function_t orig_work; + uptr object_to_acquire; + dispatch_object_t object_to_release; +} tsan_block_context_t; + +// The offsets of different fields of the dispatch_queue_t structure, exported +// by libdispatch.dylib. +extern "C" struct dispatch_queue_offsets_s { + const uint16_t dqo_version; + const uint16_t dqo_label; + const uint16_t dqo_label_size; + const uint16_t dqo_flags; + const uint16_t dqo_flags_size; + const uint16_t dqo_serialnum; + const uint16_t dqo_serialnum_size; + const uint16_t dqo_width; + const uint16_t dqo_width_size; + const uint16_t dqo_running; + const uint16_t dqo_running_size; + const uint16_t dqo_suspend_cnt; + const uint16_t dqo_suspend_cnt_size; + const uint16_t dqo_target_queue; + const uint16_t dqo_target_queue_size; + const uint16_t dqo_priority; + const uint16_t dqo_priority_size; +} dispatch_queue_offsets; + +static bool IsQueueSerial(dispatch_queue_t q) { + CHECK_EQ(dispatch_queue_offsets.dqo_width_size, 2); + uptr width = *(uint16_t *)(((uptr)q) + dispatch_queue_offsets.dqo_width); + CHECK_NE(width, 0); + return width == 1; +} + +static tsan_block_context_t *AllocContext(ThreadState *thr, uptr pc, + dispatch_queue_t queue, + void *orig_context, + dispatch_function_t orig_work) { + tsan_block_context_t *new_context = + (tsan_block_context_t *)user_alloc(thr, pc, sizeof(tsan_block_context_t)); + new_context->queue = queue; + new_context->orig_context = orig_context; + new_context->orig_work = orig_work; + new_context->object_to_acquire = (uptr)new_context; + new_context->object_to_release = nullptr; + return new_context; +} + +static void dispatch_callback_wrap_acquire(void *param) { + SCOPED_INTERCEPTOR_RAW(dispatch_async_f_callback_wrap); + tsan_block_context_t *context = (tsan_block_context_t *)param; + Acquire(thr, pc, context->object_to_acquire); + + // Extra retain/release is required for dispatch groups. We use the group + // itself to synchronize, but in a notification (dispatch_group_notify + // callback), it may be disposed already. To solve this, we retain the group + // and release it here. + if (context->object_to_release) dispatch_release(context->object_to_release); + + // In serial queues, work items can be executed on different threads, we need + // to explicitly synchronize on the queue itself. + if (IsQueueSerial(context->queue)) Acquire(thr, pc, (uptr)context->queue); + SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_START(); + context->orig_work(context->orig_context); + SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_END(); + if (IsQueueSerial(context->queue)) Release(thr, pc, (uptr)context->queue); + user_free(thr, pc, context); +} + +static void invoke_and_release_block(void *param) { + dispatch_block_t block = (dispatch_block_t)param; + block(); + Block_release(block); +} + +#define DISPATCH_INTERCEPT_B(name) \ + TSAN_INTERCEPTOR(void, name, dispatch_queue_t q, dispatch_block_t block) { \ + SCOPED_TSAN_INTERCEPTOR(name, q, block); \ + SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_START(); \ + dispatch_block_t heap_block = Block_copy(block); \ + SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_END(); \ + tsan_block_context_t *new_context = \ + AllocContext(thr, pc, q, heap_block, &invoke_and_release_block); \ + Release(thr, pc, (uptr)new_context); \ + SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_START(); \ + REAL(name##_f)(q, new_context, dispatch_callback_wrap_acquire); \ + SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_END(); \ + } + +#define DISPATCH_INTERCEPT_F(name) \ + TSAN_INTERCEPTOR(void, name, dispatch_queue_t q, void *context, \ + dispatch_function_t work) { \ + SCOPED_TSAN_INTERCEPTOR(name, q, context, work); \ + tsan_block_context_t *new_context = \ + AllocContext(thr, pc, q, context, work); \ + Release(thr, pc, (uptr)new_context); \ + SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_START(); \ + REAL(name)(q, new_context, dispatch_callback_wrap_acquire); \ + SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_END(); \ + } + +// We wrap dispatch_async, dispatch_sync and friends where we allocate a new +// context, which is used to synchronize (we release the context before +// submitting, and the callback acquires it before executing the original +// callback). +DISPATCH_INTERCEPT_B(dispatch_async) +DISPATCH_INTERCEPT_B(dispatch_barrier_async) +DISPATCH_INTERCEPT_F(dispatch_async_f) +DISPATCH_INTERCEPT_F(dispatch_barrier_async_f) +DISPATCH_INTERCEPT_B(dispatch_sync) +DISPATCH_INTERCEPT_B(dispatch_barrier_sync) +DISPATCH_INTERCEPT_F(dispatch_sync_f) +DISPATCH_INTERCEPT_F(dispatch_barrier_sync_f) + +// GCD's dispatch_once implementation has a fast path that contains a racy read +// and it's inlined into user's code. Furthermore, this fast path doesn't +// establish a proper happens-before relations between the initialization and +// code following the call to dispatch_once. We could deal with this in +// instrumented code, but there's not much we can do about it in system +// libraries. Let's disable the fast path (by never storing the value ~0 to +// predicate), so the interceptor is always called, and let's add proper release +// and acquire semantics. Since TSan does not see its own atomic stores, the +// race on predicate won't be reported - the only accesses to it that TSan sees +// are the loads on the fast path. Loads don't race. Secondly, dispatch_once is +// both a macro and a real function, we want to intercept the function, so we +// need to undefine the macro. +#undef dispatch_once +TSAN_INTERCEPTOR(void, dispatch_once, dispatch_once_t *predicate, + dispatch_block_t block) { + SCOPED_TSAN_INTERCEPTOR(dispatch_once, predicate, block); + atomic_uint32_t *a = reinterpret_cast<atomic_uint32_t *>(predicate); + u32 v = atomic_load(a, memory_order_acquire); + if (v == 0 && + atomic_compare_exchange_strong(a, &v, 1, memory_order_relaxed)) { + SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_START(); + block(); + SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_END(); + Release(thr, pc, (uptr)a); + atomic_store(a, 2, memory_order_release); + } else { + while (v != 2) { + internal_sched_yield(); + v = atomic_load(a, memory_order_acquire); + } + Acquire(thr, pc, (uptr)a); + } +} + +#undef dispatch_once_f +TSAN_INTERCEPTOR(void, dispatch_once_f, dispatch_once_t *predicate, + void *context, dispatch_function_t function) { + SCOPED_TSAN_INTERCEPTOR(dispatch_once_f, predicate, context, function); + SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_START(); + WRAP(dispatch_once)(predicate, ^(void) { + function(context); + }); + SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_END(); +} + +TSAN_INTERCEPTOR(long_t, dispatch_semaphore_signal, + dispatch_semaphore_t dsema) { + SCOPED_TSAN_INTERCEPTOR(dispatch_semaphore_signal, dsema); + Release(thr, pc, (uptr)dsema); + return REAL(dispatch_semaphore_signal)(dsema); +} + +TSAN_INTERCEPTOR(long_t, dispatch_semaphore_wait, dispatch_semaphore_t dsema, + dispatch_time_t timeout) { + SCOPED_TSAN_INTERCEPTOR(dispatch_semaphore_wait, dsema, timeout); + long_t result = REAL(dispatch_semaphore_wait)(dsema, timeout); + if (result == 0) Acquire(thr, pc, (uptr)dsema); + return result; +} + +TSAN_INTERCEPTOR(long_t, dispatch_group_wait, dispatch_group_t group, + dispatch_time_t timeout) { + SCOPED_TSAN_INTERCEPTOR(dispatch_group_wait, group, timeout); + long_t result = REAL(dispatch_group_wait)(group, timeout); + if (result == 0) Acquire(thr, pc, (uptr)group); + return result; +} + +TSAN_INTERCEPTOR(void, dispatch_group_leave, dispatch_group_t group) { + SCOPED_TSAN_INTERCEPTOR(dispatch_group_leave, group); + Release(thr, pc, (uptr)group); + REAL(dispatch_group_leave)(group); +} + +TSAN_INTERCEPTOR(void, dispatch_group_async, dispatch_group_t group, + dispatch_queue_t queue, dispatch_block_t block) { + SCOPED_TSAN_INTERCEPTOR(dispatch_group_async, group, queue, block); + dispatch_retain(group); + dispatch_group_enter(group); + WRAP(dispatch_async)(queue, ^(void) { + block(); + WRAP(dispatch_group_leave)(group); + dispatch_release(group); + }); +} + +TSAN_INTERCEPTOR(void, dispatch_group_async_f, dispatch_group_t group, + dispatch_queue_t queue, void *context, + dispatch_function_t work) { + SCOPED_TSAN_INTERCEPTOR(dispatch_group_async_f, group, queue, context, work); + dispatch_retain(group); + dispatch_group_enter(group); + WRAP(dispatch_async)(queue, ^(void) { + work(context); + WRAP(dispatch_group_leave)(group); + dispatch_release(group); + }); +} + +TSAN_INTERCEPTOR(void, dispatch_group_notify, dispatch_group_t group, + dispatch_queue_t q, dispatch_block_t block) { + SCOPED_TSAN_INTERCEPTOR(dispatch_group_notify, group, q, block); + SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_START(); + dispatch_block_t heap_block = Block_copy(block); + SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_END(); + tsan_block_context_t *new_context = + AllocContext(thr, pc, q, heap_block, &invoke_and_release_block); + new_context->object_to_acquire = (uptr)group; + + // Will be released in dispatch_callback_wrap_acquire. + new_context->object_to_release = group; + dispatch_retain(group); + + Release(thr, pc, (uptr)group); + REAL(dispatch_group_notify_f)(group, q, new_context, + dispatch_callback_wrap_acquire); +} + +TSAN_INTERCEPTOR(void, dispatch_group_notify_f, dispatch_group_t group, + dispatch_queue_t q, void *context, dispatch_function_t work) { + SCOPED_TSAN_INTERCEPTOR(dispatch_group_notify_f, group, q, context, work); + tsan_block_context_t *new_context = AllocContext(thr, pc, q, context, work); + new_context->object_to_acquire = (uptr)group; + + // Will be released in dispatch_callback_wrap_acquire. + new_context->object_to_release = group; + dispatch_retain(group); + + Release(thr, pc, (uptr)group); + REAL(dispatch_group_notify_f)(group, q, new_context, + dispatch_callback_wrap_acquire); +} + +} // namespace __tsan + +#endif // SANITIZER_MAC diff --git a/lib/tsan/rtl/tsan_malloc_mac.cc b/lib/tsan/rtl/tsan_malloc_mac.cc new file mode 100644 index 000000000000..7fd94273c314 --- /dev/null +++ b/lib/tsan/rtl/tsan_malloc_mac.cc @@ -0,0 +1,65 @@ +//===-- tsan_malloc_mac.cc ------------------------------------------------===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// +// +// This file is a part of ThreadSanitizer (TSan), a race detector. +// +// Mac-specific malloc interception. +//===----------------------------------------------------------------------===// + +#include "sanitizer_common/sanitizer_platform.h" +#if SANITIZER_MAC + +#include "tsan_interceptors.h" +#include "tsan_stack_trace.h" + +using namespace __tsan; +#define COMMON_MALLOC_ZONE_NAME "tsan" +#define COMMON_MALLOC_ENTER() +#define COMMON_MALLOC_SANITIZER_INITIALIZED (cur_thread()->is_inited) +#define COMMON_MALLOC_FORCE_LOCK() +#define COMMON_MALLOC_FORCE_UNLOCK() +#define COMMON_MALLOC_MEMALIGN(alignment, size) \ + void *p = \ + user_alloc(cur_thread(), StackTrace::GetCurrentPc(), size, alignment) +#define COMMON_MALLOC_MALLOC(size) \ + if (cur_thread()->in_symbolizer) \ + return REAL(malloc)(size); \ + SCOPED_INTERCEPTOR_RAW(malloc, size); \ + void *p = user_alloc(thr, pc, size) +#define COMMON_MALLOC_REALLOC(ptr, size) \ + if (cur_thread()->in_symbolizer) \ + return REAL(realloc)(ptr, size); \ + SCOPED_INTERCEPTOR_RAW(realloc, ptr, size); \ + void *p = user_realloc(thr, pc, ptr, size) +#define COMMON_MALLOC_CALLOC(count, size) \ + if (cur_thread()->in_symbolizer) \ + return REAL(calloc)(count, size); \ + SCOPED_INTERCEPTOR_RAW(calloc, size, count); \ + void *p = user_calloc(thr, pc, size, count) +#define COMMON_MALLOC_VALLOC(size) \ + if (cur_thread()->in_symbolizer) \ + return REAL(valloc)(size); \ + SCOPED_INTERCEPTOR_RAW(valloc, size); \ + void *p = user_alloc(thr, pc, size, GetPageSizeCached()) +#define COMMON_MALLOC_FREE(ptr) \ + if (cur_thread()->in_symbolizer) \ + return REAL(free)(ptr); \ + SCOPED_INTERCEPTOR_RAW(free, ptr); \ + user_free(thr, pc, ptr) +#define COMMON_MALLOC_SIZE(ptr) \ + uptr size = user_alloc_usable_size(ptr); +#define COMMON_MALLOC_FILL_STATS(zone, stats) +#define COMMON_MALLOC_REPORT_UNKNOWN_REALLOC(ptr, zone_ptr, zone_name) \ + (void)zone_name; \ + Report("mz_realloc(%p) -- attempting to realloc unallocated memory.\n", ptr); +#define COMMON_MALLOC_NAMESPACE __tsan + +#include "sanitizer_common/sanitizer_malloc_mac.inc" + +#endif diff --git a/lib/tsan/rtl/tsan_mman.cc b/lib/tsan/rtl/tsan_mman.cc index 12a616ff5ae8..7247c6e00035 100644 --- a/lib/tsan/rtl/tsan_mman.cc +++ b/lib/tsan/rtl/tsan_mman.cc @@ -19,12 +19,14 @@ #include "tsan_flags.h" // May be overriden by front-end. -extern "C" void WEAK __sanitizer_malloc_hook(void *ptr, uptr size) { +SANITIZER_WEAK_DEFAULT_IMPL +void __sanitizer_malloc_hook(void *ptr, uptr size) { (void)ptr; (void)size; } -extern "C" void WEAK __sanitizer_free_hook(void *ptr) { +SANITIZER_WEAK_DEFAULT_IMPL +void __sanitizer_free_hook(void *ptr) { (void)ptr; } @@ -80,17 +82,17 @@ void AllocatorPrintStats() { } static void SignalUnsafeCall(ThreadState *thr, uptr pc) { - if (atomic_load(&thr->in_signal_handler, memory_order_relaxed) == 0 || + if (atomic_load_relaxed(&thr->in_signal_handler) == 0 || !flags()->report_signal_unsafe) return; VarSizeStackTrace stack; ObtainCurrentStack(thr, pc, &stack); + if (IsFiredSuppression(ctx, ReportTypeSignalUnsafe, stack)) + return; ThreadRegistryLock l(ctx->thread_registry); ScopedReport rep(ReportTypeSignalUnsafe); - if (!IsFiredSuppression(ctx, rep, stack)) { - rep.AddStack(stack, true); - OutputReport(thr, rep); - } + rep.AddStack(stack, true); + OutputReport(thr, rep); } void *user_alloc(ThreadState *thr, uptr pc, uptr sz, uptr align, bool signal) { diff --git a/lib/tsan/rtl/tsan_mman.h b/lib/tsan/rtl/tsan_mman.h index 5ff956d827f6..b419b58ca457 100644 --- a/lib/tsan/rtl/tsan_mman.h +++ b/lib/tsan/rtl/tsan_mman.h @@ -20,6 +20,7 @@ namespace __tsan { const uptr kDefaultAlignment = 16; void InitializeAllocator(); +void ReplaceSystemMalloc(); void AllocatorThreadStart(ThreadState *thr); void AllocatorThreadFinish(ThreadState *thr); void AllocatorPrintStats(); diff --git a/lib/tsan/rtl/tsan_mutex.cc b/lib/tsan/rtl/tsan_mutex.cc index dc5a462a8081..9dd24803b183 100644 --- a/lib/tsan/rtl/tsan_mutex.cc +++ b/lib/tsan/rtl/tsan_mutex.cc @@ -41,6 +41,8 @@ static MutexType CanLockTab[MutexTypeCount][MutexTypeCount] = { /*9 MutexTypeMBlock*/ {MutexTypeSyncVar}, /*10 MutexTypeJavaMBlock*/ {MutexTypeSyncVar}, /*11 MutexTypeDDetector*/ {}, + /*12 MutexTypeFired*/ {MutexTypeLeaf}, + /*13 MutexTypeRacy*/ {MutexTypeLeaf}, }; static bool CanLockAdj[MutexTypeCount][MutexTypeCount]; diff --git a/lib/tsan/rtl/tsan_mutex.h b/lib/tsan/rtl/tsan_mutex.h index 88fad57c78a0..27f55385c959 100644 --- a/lib/tsan/rtl/tsan_mutex.h +++ b/lib/tsan/rtl/tsan_mutex.h @@ -32,6 +32,8 @@ enum MutexType { MutexTypeMBlock, MutexTypeJavaMBlock, MutexTypeDDetector, + MutexTypeFired, + MutexTypeRacy, // This must be the last. MutexTypeCount diff --git a/lib/tsan/rtl/tsan_new_delete.cc b/lib/tsan/rtl/tsan_new_delete.cc index 2d9d044e42fa..ebb422cf2023 100644 --- a/lib/tsan/rtl/tsan_new_delete.cc +++ b/lib/tsan/rtl/tsan_new_delete.cc @@ -11,6 +11,7 @@ // // Interceptors for operators new and delete. //===----------------------------------------------------------------------===// +#include "interception/interception.h" #include "sanitizer_common/sanitizer_internal_defs.h" #include "tsan_interceptors.h" @@ -20,6 +21,13 @@ namespace std { struct nothrow_t {}; } // namespace std +DECLARE_REAL(void *, malloc, uptr size) +DECLARE_REAL(void, free, void *ptr) +#if SANITIZER_MAC || SANITIZER_ANDROID +#define __libc_malloc REAL(malloc) +#define __libc_free REAL(free) +#endif + #define OPERATOR_NEW_BODY(mangled_name) \ if (cur_thread()->in_symbolizer) \ return __libc_malloc(size); \ @@ -64,14 +72,14 @@ void *operator new[](__sanitizer::uptr size, std::nothrow_t const&) { user_free(thr, pc, ptr); SANITIZER_INTERFACE_ATTRIBUTE -void operator delete(void *ptr) throw(); -void operator delete(void *ptr) throw() { +void operator delete(void *ptr) NOEXCEPT; +void operator delete(void *ptr) NOEXCEPT { OPERATOR_DELETE_BODY(_ZdlPv); } SANITIZER_INTERFACE_ATTRIBUTE -void operator delete[](void *ptr) throw(); -void operator delete[](void *ptr) throw() { +void operator delete[](void *ptr) NOEXCEPT; +void operator delete[](void *ptr) NOEXCEPT { OPERATOR_DELETE_BODY(_ZdaPv); } diff --git a/lib/tsan/rtl/tsan_platform.h b/lib/tsan/rtl/tsan_platform.h index 135e16027132..c2b487155300 100644 --- a/lib/tsan/rtl/tsan_platform.h +++ b/lib/tsan/rtl/tsan_platform.h @@ -41,21 +41,23 @@ C/C++ on linux/x86_64 and freebsd/x86_64 7e00 0000 0000 - 7e80 0000 0000: - 7e80 0000 0000 - 8000 0000 0000: modules and main thread stack */ -const uptr kMetaShadowBeg = 0x300000000000ull; -const uptr kMetaShadowEnd = 0x400000000000ull; -const uptr kTraceMemBeg = 0x600000000000ull; -const uptr kTraceMemEnd = 0x620000000000ull; -const uptr kShadowBeg = 0x020000000000ull; -const uptr kShadowEnd = 0x100000000000ull; -const uptr kHeapMemBeg = 0x7d0000000000ull; -const uptr kHeapMemEnd = 0x7e0000000000ull; -const uptr kLoAppMemBeg = 0x000000001000ull; -const uptr kLoAppMemEnd = 0x010000000000ull; -const uptr kHiAppMemBeg = 0x7e8000000000ull; -const uptr kHiAppMemEnd = 0x800000000000ull; -const uptr kAppMemMsk = 0x7c0000000000ull; -const uptr kAppMemXor = 0x020000000000ull; -const uptr kVdsoBeg = 0xf000000000000000ull; +struct Mapping { + static const uptr kMetaShadowBeg = 0x300000000000ull; + static const uptr kMetaShadowEnd = 0x400000000000ull; + static const uptr kTraceMemBeg = 0x600000000000ull; + static const uptr kTraceMemEnd = 0x620000000000ull; + static const uptr kShadowBeg = 0x020000000000ull; + static const uptr kShadowEnd = 0x100000000000ull; + static const uptr kHeapMemBeg = 0x7d0000000000ull; + static const uptr kHeapMemEnd = 0x7e0000000000ull; + static const uptr kLoAppMemBeg = 0x000000001000ull; + static const uptr kLoAppMemEnd = 0x010000000000ull; + static const uptr kHiAppMemBeg = 0x7e8000000000ull; + static const uptr kHiAppMemEnd = 0x800000000000ull; + static const uptr kAppMemMsk = 0x7c0000000000ull; + static const uptr kAppMemXor = 0x020000000000ull; + static const uptr kVdsoBeg = 0xf000000000000000ull; +}; #elif defined(__mips64) /* C/C++ on linux/mips64 @@ -71,69 +73,181 @@ fe00 0000 00 - ff00 0000 00: heap ff00 0000 00 - ff80 0000 00: - ff80 0000 00 - ffff ffff ff: modules and main thread stack */ -const uptr kMetaShadowBeg = 0x3000000000ull; -const uptr kMetaShadowEnd = 0x4000000000ull; -const uptr kTraceMemBeg = 0x6000000000ull; -const uptr kTraceMemEnd = 0x6200000000ull; -const uptr kShadowBeg = 0x1400000000ull; -const uptr kShadowEnd = 0x2400000000ull; -const uptr kHeapMemBeg = 0xfe00000000ull; -const uptr kHeapMemEnd = 0xff00000000ull; -const uptr kLoAppMemBeg = 0x0100000000ull; -const uptr kLoAppMemEnd = 0x0200000000ull; -const uptr kHiAppMemBeg = 0xff80000000ull; -const uptr kHiAppMemEnd = 0xffffffffffull; -const uptr kAppMemMsk = 0xfc00000000ull; -const uptr kAppMemXor = 0x0400000000ull; -const uptr kVdsoBeg = 0xfffff00000ull; -#endif - -ALWAYS_INLINE -bool IsAppMem(uptr mem) { - return (mem >= kHeapMemBeg && mem < kHeapMemEnd) || - (mem >= kLoAppMemBeg && mem < kLoAppMemEnd) || - (mem >= kHiAppMemBeg && mem < kHiAppMemEnd); -} - -ALWAYS_INLINE -bool IsShadowMem(uptr mem) { - return mem >= kShadowBeg && mem <= kShadowEnd; -} +struct Mapping { + static const uptr kMetaShadowBeg = 0x3000000000ull; + static const uptr kMetaShadowEnd = 0x4000000000ull; + static const uptr kTraceMemBeg = 0x6000000000ull; + static const uptr kTraceMemEnd = 0x6200000000ull; + static const uptr kShadowBeg = 0x1400000000ull; + static const uptr kShadowEnd = 0x2400000000ull; + static const uptr kHeapMemBeg = 0xfe00000000ull; + static const uptr kHeapMemEnd = 0xff00000000ull; + static const uptr kLoAppMemBeg = 0x0100000000ull; + static const uptr kLoAppMemEnd = 0x0200000000ull; + static const uptr kHiAppMemBeg = 0xff80000000ull; + static const uptr kHiAppMemEnd = 0xffffffffffull; + static const uptr kAppMemMsk = 0xfc00000000ull; + static const uptr kAppMemXor = 0x0400000000ull; + static const uptr kVdsoBeg = 0xfffff00000ull; +}; +#elif defined(__aarch64__) +// AArch64 supports multiple VMA which leads to multiple address transformation +// functions. To support these multiple VMAS transformations and mappings TSAN +// runtime for AArch64 uses an external memory read (vmaSize) to select which +// mapping to use. Although slower, it make a same instrumented binary run on +// multiple kernels. -ALWAYS_INLINE -bool IsMetaMem(uptr mem) { - return mem >= kMetaShadowBeg && mem <= kMetaShadowEnd; -} +/* +C/C++ on linux/aarch64 (39-bit VMA) +0000 0010 00 - 0100 0000 00: main binary +0100 0000 00 - 0800 0000 00: - +0800 0000 00 - 2000 0000 00: shadow memory +2000 0000 00 - 3100 0000 00: - +3100 0000 00 - 3400 0000 00: metainfo +3400 0000 00 - 5500 0000 00: - +5500 0000 00 - 5600 0000 00: main binary (PIE) +5600 0000 00 - 6000 0000 00: - +6000 0000 00 - 6200 0000 00: traces +6200 0000 00 - 7d00 0000 00: - +7c00 0000 00 - 7d00 0000 00: heap +7d00 0000 00 - 7fff ffff ff: modules and main thread stack +*/ +struct Mapping39 { + static const uptr kLoAppMemBeg = 0x0000001000ull; + static const uptr kLoAppMemEnd = 0x0100000000ull; + static const uptr kShadowBeg = 0x0800000000ull; + static const uptr kShadowEnd = 0x2000000000ull; + static const uptr kMetaShadowBeg = 0x3100000000ull; + static const uptr kMetaShadowEnd = 0x3400000000ull; + static const uptr kMidAppMemBeg = 0x5500000000ull; + static const uptr kMidAppMemEnd = 0x5600000000ull; + static const uptr kMidShadowOff = 0x5000000000ull; + static const uptr kTraceMemBeg = 0x6000000000ull; + static const uptr kTraceMemEnd = 0x6200000000ull; + static const uptr kHeapMemBeg = 0x7c00000000ull; + static const uptr kHeapMemEnd = 0x7d00000000ull; + static const uptr kHiAppMemBeg = 0x7e00000000ull; + static const uptr kHiAppMemEnd = 0x7fffffffffull; + static const uptr kAppMemMsk = 0x7800000000ull; + static const uptr kAppMemXor = 0x0200000000ull; + static const uptr kVdsoBeg = 0x7f00000000ull; +}; -ALWAYS_INLINE -uptr MemToShadow(uptr x) { - DCHECK(IsAppMem(x)); - return (((x) & ~(kAppMemMsk | (kShadowCell - 1))) - ^ kAppMemXor) * kShadowCnt; -} +/* +C/C++ on linux/aarch64 (42-bit VMA) +00000 0010 00 - 01000 0000 00: main binary +01000 0000 00 - 10000 0000 00: - +10000 0000 00 - 20000 0000 00: shadow memory +20000 0000 00 - 26000 0000 00: - +26000 0000 00 - 28000 0000 00: metainfo +28000 0000 00 - 2aa00 0000 00: - +2aa00 0000 00 - 2ab00 0000 00: main binary (PIE) +2ab00 0000 00 - 36200 0000 00: - +36200 0000 00 - 36240 0000 00: traces +36240 0000 00 - 3e000 0000 00: - +3e000 0000 00 - 3f000 0000 00: heap +3f000 0000 00 - 3ffff ffff ff: modules and main thread stack +*/ +struct Mapping42 { + static const uptr kLoAppMemBeg = 0x00000001000ull; + static const uptr kLoAppMemEnd = 0x01000000000ull; + static const uptr kShadowBeg = 0x10000000000ull; + static const uptr kShadowEnd = 0x20000000000ull; + static const uptr kMetaShadowBeg = 0x26000000000ull; + static const uptr kMetaShadowEnd = 0x28000000000ull; + static const uptr kMidAppMemBeg = 0x2aa00000000ull; + static const uptr kMidAppMemEnd = 0x2ab00000000ull; + static const uptr kMidShadowOff = 0x28000000000ull; + static const uptr kTraceMemBeg = 0x36200000000ull; + static const uptr kTraceMemEnd = 0x36400000000ull; + static const uptr kHeapMemBeg = 0x3e000000000ull; + static const uptr kHeapMemEnd = 0x3f000000000ull; + static const uptr kHiAppMemBeg = 0x3f000000000ull; + static const uptr kHiAppMemEnd = 0x3ffffffffffull; + static const uptr kAppMemMsk = 0x3c000000000ull; + static const uptr kAppMemXor = 0x04000000000ull; + static const uptr kVdsoBeg = 0x37f00000000ull; +}; -ALWAYS_INLINE -u32 *MemToMeta(uptr x) { - DCHECK(IsAppMem(x)); - return (u32*)(((((x) & ~(kAppMemMsk | (kMetaShadowCell - 1))) - ^ kAppMemXor) / kMetaShadowCell * kMetaShadowSize) | kMetaShadowBeg); -} +// Indicates the runtime will define the memory regions at runtime. +#define TSAN_RUNTIME_VMA 1 +// Indicates that mapping defines a mid range memory segment. +#define TSAN_MID_APP_RANGE 1 +#elif defined(__powerpc64__) +// PPC64 supports multiple VMA which leads to multiple address transformation +// functions. To support these multiple VMAS transformations and mappings TSAN +// runtime for PPC64 uses an external memory read (vmaSize) to select which +// mapping to use. Although slower, it make a same instrumented binary run on +// multiple kernels. -ALWAYS_INLINE -uptr ShadowToMem(uptr s) { - CHECK(IsShadowMem(s)); - if (s >= MemToShadow(kLoAppMemBeg) && s <= MemToShadow(kLoAppMemEnd - 1)) - return (s / kShadowCnt) ^ kAppMemXor; - else - return ((s / kShadowCnt) ^ kAppMemXor) | kAppMemMsk; -} +/* +C/C++ on linux/powerpc64 (44-bit VMA) +0000 0000 0100 - 0001 0000 0000: main binary +0001 0000 0000 - 0001 0000 0000: - +0001 0000 0000 - 0b00 0000 0000: shadow +0b00 0000 0000 - 0b00 0000 0000: - +0b00 0000 0000 - 0d00 0000 0000: metainfo (memory blocks and sync objects) +0d00 0000 0000 - 0d00 0000 0000: - +0d00 0000 0000 - 0f00 0000 0000: traces +0f00 0000 0000 - 0f00 0000 0000: - +0f00 0000 0000 - 0f50 0000 0000: heap +0f50 0000 0000 - 0f60 0000 0000: - +0f60 0000 0000 - 1000 0000 0000: modules and main thread stack +*/ +struct Mapping44 { + static const uptr kMetaShadowBeg = 0x0b0000000000ull; + static const uptr kMetaShadowEnd = 0x0d0000000000ull; + static const uptr kTraceMemBeg = 0x0d0000000000ull; + static const uptr kTraceMemEnd = 0x0f0000000000ull; + static const uptr kShadowBeg = 0x000100000000ull; + static const uptr kShadowEnd = 0x0b0000000000ull; + static const uptr kLoAppMemBeg = 0x000000000100ull; + static const uptr kLoAppMemEnd = 0x000100000000ull; + static const uptr kHeapMemBeg = 0x0f0000000000ull; + static const uptr kHeapMemEnd = 0x0f5000000000ull; + static const uptr kHiAppMemBeg = 0x0f6000000000ull; + static const uptr kHiAppMemEnd = 0x100000000000ull; // 44 bits + static const uptr kAppMemMsk = 0x0f0000000000ull; + static const uptr kAppMemXor = 0x002100000000ull; + static const uptr kVdsoBeg = 0x3c0000000000000ull; +}; -static USED uptr UserRegions[] = { - kLoAppMemBeg, kLoAppMemEnd, - kHiAppMemBeg, kHiAppMemEnd, - kHeapMemBeg, kHeapMemEnd, +/* +C/C++ on linux/powerpc64 (46-bit VMA) +0000 0000 1000 - 0100 0000 0000: main binary +0100 0000 0000 - 0200 0000 0000: - +0100 0000 0000 - 1000 0000 0000: shadow +1000 0000 0000 - 1000 0000 0000: - +1000 0000 0000 - 2000 0000 0000: metainfo (memory blocks and sync objects) +2000 0000 0000 - 2000 0000 0000: - +2000 0000 0000 - 2200 0000 0000: traces +2200 0000 0000 - 3d00 0000 0000: - +3d00 0000 0000 - 3e00 0000 0000: heap +3e00 0000 0000 - 3e80 0000 0000: - +3e80 0000 0000 - 4000 0000 0000: modules and main thread stack +*/ +struct Mapping46 { + static const uptr kMetaShadowBeg = 0x100000000000ull; + static const uptr kMetaShadowEnd = 0x200000000000ull; + static const uptr kTraceMemBeg = 0x200000000000ull; + static const uptr kTraceMemEnd = 0x220000000000ull; + static const uptr kShadowBeg = 0x010000000000ull; + static const uptr kShadowEnd = 0x100000000000ull; + static const uptr kHeapMemBeg = 0x3d0000000000ull; + static const uptr kHeapMemEnd = 0x3e0000000000ull; + static const uptr kLoAppMemBeg = 0x000000001000ull; + static const uptr kLoAppMemEnd = 0x010000000000ull; + static const uptr kHiAppMemBeg = 0x3e8000000000ull; + static const uptr kHiAppMemEnd = 0x400000000000ull; // 46 bits + static const uptr kAppMemMsk = 0x3c0000000000ull; + static const uptr kAppMemXor = 0x020000000000ull; + static const uptr kVdsoBeg = 0x7800000000000000ull; }; +// Indicates the runtime will define the memory regions at runtime. +#define TSAN_RUNTIME_VMA 1 +#endif + #elif defined(SANITIZER_GO) && !SANITIZER_WINDOWS /* Go on linux, darwin and freebsd @@ -149,138 +263,495 @@ static USED uptr UserRegions[] = { 6200 0000 0000 - 8000 0000 0000: - */ -const uptr kMetaShadowBeg = 0x300000000000ull; -const uptr kMetaShadowEnd = 0x400000000000ull; -const uptr kTraceMemBeg = 0x600000000000ull; -const uptr kTraceMemEnd = 0x620000000000ull; -const uptr kShadowBeg = 0x200000000000ull; -const uptr kShadowEnd = 0x238000000000ull; -const uptr kAppMemBeg = 0x000000001000ull; -const uptr kAppMemEnd = 0x00e000000000ull; +struct Mapping { + static const uptr kMetaShadowBeg = 0x300000000000ull; + static const uptr kMetaShadowEnd = 0x400000000000ull; + static const uptr kTraceMemBeg = 0x600000000000ull; + static const uptr kTraceMemEnd = 0x620000000000ull; + static const uptr kShadowBeg = 0x200000000000ull; + static const uptr kShadowEnd = 0x238000000000ull; + static const uptr kAppMemBeg = 0x000000001000ull; + static const uptr kAppMemEnd = 0x00e000000000ull; +}; + +#elif defined(SANITIZER_GO) && SANITIZER_WINDOWS + +/* Go on windows +0000 0000 1000 - 0000 1000 0000: executable +0000 1000 0000 - 00f8 0000 0000: - +00c0 0000 0000 - 00e0 0000 0000: heap +00e0 0000 0000 - 0100 0000 0000: - +0100 0000 0000 - 0500 0000 0000: shadow +0500 0000 0000 - 0560 0000 0000: - +0560 0000 0000 - 0760 0000 0000: traces +0760 0000 0000 - 07d0 0000 0000: metainfo (memory blocks and sync objects) +07d0 0000 0000 - 8000 0000 0000: - +*/ + +struct Mapping { + static const uptr kMetaShadowBeg = 0x076000000000ull; + static const uptr kMetaShadowEnd = 0x07d000000000ull; + static const uptr kTraceMemBeg = 0x056000000000ull; + static const uptr kTraceMemEnd = 0x076000000000ull; + static const uptr kShadowBeg = 0x010000000000ull; + static const uptr kShadowEnd = 0x050000000000ull; + static const uptr kAppMemBeg = 0x000000001000ull; + static const uptr kAppMemEnd = 0x00e000000000ull; +} + +#else +# error "Unknown platform" +#endif + + +#ifdef TSAN_RUNTIME_VMA +extern uptr vmaSize; +#endif + + +enum MappingType { + MAPPING_LO_APP_BEG, + MAPPING_LO_APP_END, + MAPPING_HI_APP_BEG, + MAPPING_HI_APP_END, +#ifdef TSAN_MID_APP_RANGE + MAPPING_MID_APP_BEG, + MAPPING_MID_APP_END, +#endif + MAPPING_HEAP_BEG, + MAPPING_HEAP_END, + MAPPING_APP_BEG, + MAPPING_APP_END, + MAPPING_SHADOW_BEG, + MAPPING_SHADOW_END, + MAPPING_META_SHADOW_BEG, + MAPPING_META_SHADOW_END, + MAPPING_TRACE_BEG, + MAPPING_TRACE_END, + MAPPING_VDSO_BEG, +}; + +template<typename Mapping, int Type> +uptr MappingImpl(void) { + switch (Type) { +#ifndef SANITIZER_GO + case MAPPING_LO_APP_BEG: return Mapping::kLoAppMemBeg; + case MAPPING_LO_APP_END: return Mapping::kLoAppMemEnd; +# ifdef TSAN_MID_APP_RANGE + case MAPPING_MID_APP_BEG: return Mapping::kMidAppMemBeg; + case MAPPING_MID_APP_END: return Mapping::kMidAppMemEnd; +# endif + case MAPPING_HI_APP_BEG: return Mapping::kHiAppMemBeg; + case MAPPING_HI_APP_END: return Mapping::kHiAppMemEnd; + case MAPPING_HEAP_BEG: return Mapping::kHeapMemBeg; + case MAPPING_HEAP_END: return Mapping::kHeapMemEnd; + case MAPPING_VDSO_BEG: return Mapping::kVdsoBeg; +#else + case MAPPING_APP_BEG: return Mapping::kAppMemBeg; + case MAPPING_APP_END: return Mapping::kAppMemEnd; +#endif + case MAPPING_SHADOW_BEG: return Mapping::kShadowBeg; + case MAPPING_SHADOW_END: return Mapping::kShadowEnd; + case MAPPING_META_SHADOW_BEG: return Mapping::kMetaShadowBeg; + case MAPPING_META_SHADOW_END: return Mapping::kMetaShadowEnd; + case MAPPING_TRACE_BEG: return Mapping::kTraceMemBeg; + case MAPPING_TRACE_END: return Mapping::kTraceMemEnd; + } +} + +template<int Type> +uptr MappingArchImpl(void) { +#ifdef __aarch64__ + if (vmaSize == 39) + return MappingImpl<Mapping39, Type>(); + else + return MappingImpl<Mapping42, Type>(); + DCHECK(0); +#elif defined(__powerpc64__) + if (vmaSize == 44) + return MappingImpl<Mapping44, Type>(); + else + return MappingImpl<Mapping46, Type>(); + DCHECK(0); +#else + return MappingImpl<Mapping, Type>(); +#endif +} +#ifndef SANITIZER_GO ALWAYS_INLINE -bool IsAppMem(uptr mem) { - return mem >= kAppMemBeg && mem < kAppMemEnd; +uptr LoAppMemBeg(void) { + return MappingArchImpl<MAPPING_LO_APP_BEG>(); +} +ALWAYS_INLINE +uptr LoAppMemEnd(void) { + return MappingArchImpl<MAPPING_LO_APP_END>(); } +#ifdef TSAN_MID_APP_RANGE ALWAYS_INLINE -bool IsShadowMem(uptr mem) { - return mem >= kShadowBeg && mem <= kShadowEnd; +uptr MidAppMemBeg(void) { + return MappingArchImpl<MAPPING_MID_APP_BEG>(); +} +ALWAYS_INLINE +uptr MidAppMemEnd(void) { + return MappingArchImpl<MAPPING_MID_APP_END>(); } +#endif ALWAYS_INLINE -bool IsMetaMem(uptr mem) { - return mem >= kMetaShadowBeg && mem <= kMetaShadowEnd; +uptr HeapMemBeg(void) { + return MappingArchImpl<MAPPING_HEAP_BEG>(); +} +ALWAYS_INLINE +uptr HeapMemEnd(void) { + return MappingArchImpl<MAPPING_HEAP_END>(); } ALWAYS_INLINE -uptr MemToShadow(uptr x) { - DCHECK(IsAppMem(x)); - return ((x & ~(kShadowCell - 1)) * kShadowCnt) | kShadowBeg; +uptr HiAppMemBeg(void) { + return MappingArchImpl<MAPPING_HI_APP_BEG>(); +} +ALWAYS_INLINE +uptr HiAppMemEnd(void) { + return MappingArchImpl<MAPPING_HI_APP_END>(); } ALWAYS_INLINE -u32 *MemToMeta(uptr x) { - DCHECK(IsAppMem(x)); - return (u32*)(((x & ~(kMetaShadowCell - 1)) / \ - kMetaShadowCell * kMetaShadowSize) | kMetaShadowBeg); +uptr VdsoBeg(void) { + return MappingArchImpl<MAPPING_VDSO_BEG>(); } +#else + ALWAYS_INLINE -uptr ShadowToMem(uptr s) { - CHECK(IsShadowMem(s)); - return (s & ~kShadowBeg) / kShadowCnt; +uptr AppMemBeg(void) { + return MappingArchImpl<MAPPING_APP_BEG>(); +} +ALWAYS_INLINE +uptr AppMemEnd(void) { + return MappingArchImpl<MAPPING_APP_END>(); } -static USED uptr UserRegions[] = { - kAppMemBeg, kAppMemEnd, -}; +#endif -#elif defined(SANITIZER_GO) && SANITIZER_WINDOWS +static inline +bool GetUserRegion(int i, uptr *start, uptr *end) { + switch (i) { + default: + return false; +#ifndef SANITIZER_GO + case 0: + *start = LoAppMemBeg(); + *end = LoAppMemEnd(); + return true; + case 1: + *start = HiAppMemBeg(); + *end = HiAppMemEnd(); + return true; + case 2: + *start = HeapMemBeg(); + *end = HeapMemEnd(); + return true; +# ifdef TSAN_MID_APP_RANGE + case 3: + *start = MidAppMemBeg(); + *end = MidAppMemEnd(); + return true; +# endif +#else + case 0: + *start = AppMemBeg(); + *end = AppMemEnd(); + return true; +#endif + } +} -/* Go on windows -0000 0000 1000 - 0000 1000 0000: executable -0000 1000 0000 - 00f8 0000 0000: - -00c0 0000 0000 - 00e0 0000 0000: heap -00e0 0000 0000 - 0100 0000 0000: - -0100 0000 0000 - 0500 0000 0000: shadow -0500 0000 0000 - 0560 0000 0000: - -0560 0000 0000 - 0760 0000 0000: traces -0760 0000 0000 - 07d0 0000 0000: metainfo (memory blocks and sync objects) -07d0 0000 0000 - 8000 0000 0000: - -*/ +ALWAYS_INLINE +uptr ShadowBeg(void) { + return MappingArchImpl<MAPPING_SHADOW_BEG>(); +} +ALWAYS_INLINE +uptr ShadowEnd(void) { + return MappingArchImpl<MAPPING_SHADOW_END>(); +} + +ALWAYS_INLINE +uptr MetaShadowBeg(void) { + return MappingArchImpl<MAPPING_META_SHADOW_BEG>(); +} +ALWAYS_INLINE +uptr MetaShadowEnd(void) { + return MappingArchImpl<MAPPING_META_SHADOW_END>(); +} + +ALWAYS_INLINE +uptr TraceMemBeg(void) { + return MappingArchImpl<MAPPING_TRACE_BEG>(); +} +ALWAYS_INLINE +uptr TraceMemEnd(void) { + return MappingArchImpl<MAPPING_TRACE_END>(); +} -const uptr kMetaShadowBeg = 0x076000000000ull; -const uptr kMetaShadowEnd = 0x07d000000000ull; -const uptr kTraceMemBeg = 0x056000000000ull; -const uptr kTraceMemEnd = 0x076000000000ull; -const uptr kShadowBeg = 0x010000000000ull; -const uptr kShadowEnd = 0x050000000000ull; -const uptr kAppMemBeg = 0x000000001000ull; -const uptr kAppMemEnd = 0x00e000000000ull; + +template<typename Mapping> +bool IsAppMemImpl(uptr mem) { +#ifndef SANITIZER_GO + return (mem >= Mapping::kHeapMemBeg && mem < Mapping::kHeapMemEnd) || +# ifdef TSAN_MID_APP_RANGE + (mem >= Mapping::kMidAppMemBeg && mem < Mapping::kMidAppMemEnd) || +# endif + (mem >= Mapping::kLoAppMemBeg && mem < Mapping::kLoAppMemEnd) || + (mem >= Mapping::kHiAppMemBeg && mem < Mapping::kHiAppMemEnd); +#else + return mem >= Mapping::kAppMemBeg && mem < Mapping::kAppMemEnd; +#endif +} ALWAYS_INLINE bool IsAppMem(uptr mem) { - return mem >= kAppMemBeg && mem < kAppMemEnd; +#ifdef __aarch64__ + if (vmaSize == 39) + return IsAppMemImpl<Mapping39>(mem); + else + return IsAppMemImpl<Mapping42>(mem); + DCHECK(0); +#elif defined(__powerpc64__) + if (vmaSize == 44) + return IsAppMemImpl<Mapping44>(mem); + else + return IsAppMemImpl<Mapping46>(mem); + DCHECK(0); +#else + return IsAppMemImpl<Mapping>(mem); +#endif +} + + +template<typename Mapping> +bool IsShadowMemImpl(uptr mem) { + return mem >= Mapping::kShadowBeg && mem <= Mapping::kShadowEnd; } ALWAYS_INLINE bool IsShadowMem(uptr mem) { - return mem >= kShadowBeg && mem <= kShadowEnd; +#ifdef __aarch64__ + if (vmaSize == 39) + return IsShadowMemImpl<Mapping39>(mem); + else + return IsShadowMemImpl<Mapping42>(mem); + DCHECK(0); +#elif defined(__powerpc64__) + if (vmaSize == 44) + return IsShadowMemImpl<Mapping44>(mem); + else + return IsShadowMemImpl<Mapping46>(mem); + DCHECK(0); +#else + return IsShadowMemImpl<Mapping>(mem); +#endif +} + + +template<typename Mapping> +bool IsMetaMemImpl(uptr mem) { + return mem >= Mapping::kMetaShadowBeg && mem <= Mapping::kMetaShadowEnd; } ALWAYS_INLINE bool IsMetaMem(uptr mem) { - return mem >= kMetaShadowBeg && mem <= kMetaShadowEnd; +#ifdef __aarch64__ + if (vmaSize == 39) + return IsMetaMemImpl<Mapping39>(mem); + else + return IsMetaMemImpl<Mapping42>(mem); + DCHECK(0); +#elif defined(__powerpc64__) + if (vmaSize == 44) + return IsMetaMemImpl<Mapping44>(mem); + else + return IsMetaMemImpl<Mapping46>(mem); + DCHECK(0); +#else + return IsMetaMemImpl<Mapping>(mem); +#endif } -ALWAYS_INLINE -uptr MemToShadow(uptr x) { + +template<typename Mapping> +uptr MemToShadowImpl(uptr x) { DCHECK(IsAppMem(x)); - return ((x & ~(kShadowCell - 1)) * kShadowCnt) + kShadowBeg; +#ifndef SANITIZER_GO + return (((x) & ~(Mapping::kAppMemMsk | (kShadowCell - 1))) + ^ Mapping::kAppMemXor) * kShadowCnt; +#else + return ((x & ~(kShadowCell - 1)) * kShadowCnt) | Mapping::kShadowBeg; +#endif } ALWAYS_INLINE -u32 *MemToMeta(uptr x) { +uptr MemToShadow(uptr x) { +#ifdef __aarch64__ + if (vmaSize == 39) + return MemToShadowImpl<Mapping39>(x); + else + return MemToShadowImpl<Mapping42>(x); + DCHECK(0); +#elif defined(__powerpc64__) + if (vmaSize == 44) + return MemToShadowImpl<Mapping44>(x); + else + return MemToShadowImpl<Mapping46>(x); + DCHECK(0); +#else + return MemToShadowImpl<Mapping>(x); +#endif +} + + +template<typename Mapping> +u32 *MemToMetaImpl(uptr x) { DCHECK(IsAppMem(x)); +#ifndef SANITIZER_GO + return (u32*)(((((x) & ~(Mapping::kAppMemMsk | (kMetaShadowCell - 1))) + ^ Mapping::kAppMemXor) / kMetaShadowCell * kMetaShadowSize) + | Mapping::kMetaShadowBeg); +#else return (u32*)(((x & ~(kMetaShadowCell - 1)) / \ - kMetaShadowCell * kMetaShadowSize) | kMetaShadowBeg); + kMetaShadowCell * kMetaShadowSize) | Mapping::kMetaShadowBeg); +#endif } ALWAYS_INLINE -uptr ShadowToMem(uptr s) { - CHECK(IsShadowMem(s)); - // FIXME(dvyukov): this is most likely wrong as the mapping is not bijection. - return (s - kShadowBeg) / kShadowCnt; +u32 *MemToMeta(uptr x) { +#ifdef __aarch64__ + if (vmaSize == 39) + return MemToMetaImpl<Mapping39>(x); + else + return MemToMetaImpl<Mapping42>(x); + DCHECK(0); +#elif defined(__powerpc64__) + if (vmaSize == 44) + return MemToMetaImpl<Mapping44>(x); + else + return MemToMetaImpl<Mapping46>(x); + DCHECK(0); +#else + return MemToMetaImpl<Mapping>(x); +#endif } -static USED uptr UserRegions[] = { - kAppMemBeg, kAppMemEnd, -}; +template<typename Mapping> +uptr ShadowToMemImpl(uptr s) { + DCHECK(IsShadowMem(s)); +#ifndef SANITIZER_GO + if (s >= MemToShadow(Mapping::kLoAppMemBeg) + && s <= MemToShadow(Mapping::kLoAppMemEnd - 1)) + return (s / kShadowCnt) ^ Mapping::kAppMemXor; +# ifdef TSAN_MID_APP_RANGE + if (s >= MemToShadow(Mapping::kMidAppMemBeg) + && s <= MemToShadow(Mapping::kMidAppMemEnd - 1)) + return ((s / kShadowCnt) ^ Mapping::kAppMemXor) + Mapping::kMidShadowOff; +# endif + else + return ((s / kShadowCnt) ^ Mapping::kAppMemXor) | Mapping::kAppMemMsk; #else -# error "Unknown platform" +# ifndef SANITIZER_WINDOWS + return (s & ~Mapping::kShadowBeg) / kShadowCnt; +# else + // FIXME(dvyukov): this is most likely wrong as the mapping is not bijection. + return (s - Mapping::kShadowBeg) / kShadowCnt; +# endif // SANITIZER_WINDOWS +#endif +} + +ALWAYS_INLINE +uptr ShadowToMem(uptr s) { +#ifdef __aarch64__ + if (vmaSize == 39) + return ShadowToMemImpl<Mapping39>(s); + else + return ShadowToMemImpl<Mapping42>(s); + DCHECK(0); +#elif defined(__powerpc64__) + if (vmaSize == 44) + return ShadowToMemImpl<Mapping44>(s); + else + return ShadowToMemImpl<Mapping46>(s); + DCHECK(0); +#else + return ShadowToMemImpl<Mapping>(s); #endif +} + + // The additional page is to catch shadow stack overflow as paging fault. // Windows wants 64K alignment for mmaps. const uptr kTotalTraceSize = (kTraceSize * sizeof(Event) + sizeof(Trace) + (64 << 10) + (64 << 10) - 1) & ~((64 << 10) - 1); -uptr ALWAYS_INLINE GetThreadTrace(int tid) { - uptr p = kTraceMemBeg + (uptr)tid * kTotalTraceSize; - DCHECK_LT(p, kTraceMemEnd); +template<typename Mapping> +uptr GetThreadTraceImpl(int tid) { + uptr p = Mapping::kTraceMemBeg + (uptr)tid * kTotalTraceSize; + DCHECK_LT(p, Mapping::kTraceMemEnd); return p; } -uptr ALWAYS_INLINE GetThreadTraceHeader(int tid) { - uptr p = kTraceMemBeg + (uptr)tid * kTotalTraceSize +ALWAYS_INLINE +uptr GetThreadTrace(int tid) { +#ifdef __aarch64__ + if (vmaSize == 39) + return GetThreadTraceImpl<Mapping39>(tid); + else + return GetThreadTraceImpl<Mapping42>(tid); + DCHECK(0); +#elif defined(__powerpc64__) + if (vmaSize == 44) + return GetThreadTraceImpl<Mapping44>(tid); + else + return GetThreadTraceImpl<Mapping46>(tid); + DCHECK(0); +#else + return GetThreadTraceImpl<Mapping>(tid); +#endif +} + + +template<typename Mapping> +uptr GetThreadTraceHeaderImpl(int tid) { + uptr p = Mapping::kTraceMemBeg + (uptr)tid * kTotalTraceSize + kTraceSize * sizeof(Event); - DCHECK_LT(p, kTraceMemEnd); + DCHECK_LT(p, Mapping::kTraceMemEnd); return p; } +ALWAYS_INLINE +uptr GetThreadTraceHeader(int tid) { +#ifdef __aarch64__ + if (vmaSize == 39) + return GetThreadTraceHeaderImpl<Mapping39>(tid); + else + return GetThreadTraceHeaderImpl<Mapping42>(tid); + DCHECK(0); +#elif defined(__powerpc64__) + if (vmaSize == 44) + return GetThreadTraceHeaderImpl<Mapping44>(tid); + else + return GetThreadTraceHeaderImpl<Mapping46>(tid); + DCHECK(0); +#else + return GetThreadTraceHeaderImpl<Mapping>(tid); +#endif +} + void InitializePlatform(); +void InitializePlatformEarly(); +void CheckAndProtect(); +void InitializeShadowMemoryPlatform(); void FlushShadowMemory(); void WriteMemoryProfile(char *buf, uptr buf_size, uptr nthread, uptr nlive); @@ -294,6 +765,8 @@ int call_pthread_cancel_with_cleanup(int(*fn)(void *c, void *m, void *abstime), void *c, void *m, void *abstime, void(*cleanup)(void *arg), void *arg); +void DestroyThreadState(); + } // namespace __tsan #endif // TSAN_PLATFORM_H diff --git a/lib/tsan/rtl/tsan_platform_linux.cc b/lib/tsan/rtl/tsan_platform_linux.cc index 1309058210ce..6602561186ce 100644 --- a/lib/tsan/rtl/tsan_platform_linux.cc +++ b/lib/tsan/rtl/tsan_platform_linux.cc @@ -67,6 +67,11 @@ namespace __tsan { static uptr g_data_start; static uptr g_data_end; +#ifdef TSAN_RUNTIME_VMA +// Runtime detected VMA size. +uptr vmaSize; +#endif + enum { MemTotal = 0, MemShadow = 1, @@ -82,29 +87,30 @@ enum { void FillProfileCallback(uptr p, uptr rss, bool file, uptr *mem, uptr stats_size) { mem[MemTotal] += rss; - if (p >= kShadowBeg && p < kShadowEnd) + if (p >= ShadowBeg() && p < ShadowEnd()) mem[MemShadow] += rss; - else if (p >= kMetaShadowBeg && p < kMetaShadowEnd) + else if (p >= MetaShadowBeg() && p < MetaShadowEnd()) mem[MemMeta] += rss; #ifndef SANITIZER_GO - else if (p >= kHeapMemBeg && p < kHeapMemEnd) + else if (p >= HeapMemBeg() && p < HeapMemEnd()) mem[MemHeap] += rss; - else if (p >= kLoAppMemBeg && p < kLoAppMemEnd) + else if (p >= LoAppMemBeg() && p < LoAppMemEnd()) mem[file ? MemFile : MemMmap] += rss; - else if (p >= kHiAppMemBeg && p < kHiAppMemEnd) + else if (p >= HiAppMemBeg() && p < HiAppMemEnd()) mem[file ? MemFile : MemMmap] += rss; #else - else if (p >= kAppMemBeg && p < kAppMemEnd) + else if (p >= AppMemBeg() && p < AppMemEnd()) mem[file ? MemFile : MemMmap] += rss; #endif - else if (p >= kTraceMemBeg && p < kTraceMemEnd) + else if (p >= TraceMemBeg() && p < TraceMemEnd()) mem[MemTrace] += rss; else mem[MemOther] += rss; } void WriteMemoryProfile(char *buf, uptr buf_size, uptr nthread, uptr nlive) { - uptr mem[MemCount] = {}; + uptr mem[MemCount]; + internal_memset(mem, 0, sizeof(mem[0]) * MemCount); __sanitizer::GetMemoryProfile(FillProfileCallback, mem, 7); StackDepotStats *stacks = StackDepotGetStats(); internal_snprintf(buf, buf_size, @@ -121,7 +127,7 @@ void WriteMemoryProfile(char *buf, uptr buf_size, uptr nthread, uptr nlive) { void FlushShadowMemoryCallback( const SuspendedThreadsList &suspended_threads_list, void *argument) { - FlushUnneededShadowMemory(kShadowBeg, kShadowEnd - kShadowBeg); + FlushUnneededShadowMemory(ShadowBeg(), ShadowEnd() - ShadowBeg()); } #endif @@ -132,17 +138,6 @@ void FlushShadowMemory() { } #ifndef SANITIZER_GO -static void ProtectRange(uptr beg, uptr end) { - CHECK_LE(beg, end); - if (beg == end) - return; - if (beg != (uptr)MmapNoAccess(beg, end - beg)) { - Printf("FATAL: ThreadSanitizer can not protect [%zx,%zx]\n", beg, end); - Printf("FATAL: Make sure you are not using unlimited stack\n"); - Die(); - } -} - // Mark shadow for .rodata sections with the special kShadowRodata marker. // Accesses to .rodata can't race, so this saves time, memory and trace space. static void MapRodata() { @@ -200,55 +195,7 @@ static void MapRodata() { internal_close(fd); } -void InitializeShadowMemory() { - // Map memory shadow. - uptr shadow = - (uptr)MmapFixedNoReserve(kShadowBeg, kShadowEnd - kShadowBeg, "shadow"); - if (shadow != kShadowBeg) { - Printf("FATAL: ThreadSanitizer can not mmap the shadow memory\n"); - Printf("FATAL: Make sure to compile with -fPIE and " - "to link with -pie (%p, %p).\n", shadow, kShadowBeg); - Die(); - } - // This memory range is used for thread stacks and large user mmaps. - // Frequently a thread uses only a small part of stack and similarly - // a program uses a small part of large mmap. On some programs - // we see 20% memory usage reduction without huge pages for this range. - // FIXME: don't use constants here. -#if defined(__x86_64__) - const uptr kMadviseRangeBeg = 0x7f0000000000ull; - const uptr kMadviseRangeSize = 0x010000000000ull; -#elif defined(__mips64) - const uptr kMadviseRangeBeg = 0xff00000000ull; - const uptr kMadviseRangeSize = 0x0100000000ull; -#endif - NoHugePagesInRegion(MemToShadow(kMadviseRangeBeg), - kMadviseRangeSize * kShadowMultiplier); - // Meta shadow is compressing and we don't flush it, - // so it makes sense to mark it as NOHUGEPAGE to not over-allocate memory. - // On one program it reduces memory consumption from 5GB to 2.5GB. - NoHugePagesInRegion(kMetaShadowBeg, kMetaShadowEnd - kMetaShadowBeg); - if (common_flags()->use_madv_dontdump) - DontDumpShadowMemory(kShadowBeg, kShadowEnd - kShadowBeg); - DPrintf("memory shadow: %zx-%zx (%zuGB)\n", - kShadowBeg, kShadowEnd, - (kShadowEnd - kShadowBeg) >> 30); - - // Map meta shadow. - uptr meta_size = kMetaShadowEnd - kMetaShadowBeg; - uptr meta = - (uptr)MmapFixedNoReserve(kMetaShadowBeg, meta_size, "meta shadow"); - if (meta != kMetaShadowBeg) { - Printf("FATAL: ThreadSanitizer can not mmap the shadow memory\n"); - Printf("FATAL: Make sure to compile with -fPIE and " - "to link with -pie (%p, %p).\n", meta, kMetaShadowBeg); - Die(); - } - if (common_flags()->use_madv_dontdump) - DontDumpShadowMemory(meta, meta_size); - DPrintf("meta shadow: %zx-%zx (%zuGB)\n", - meta, meta + meta_size, meta_size >> 30); - +void InitializeShadowMemoryPlatform() { MapRodata(); } @@ -292,32 +239,27 @@ static void InitDataSeg() { CHECK_LT((uptr)&g_data_start, g_data_end); } -static void CheckAndProtect() { - // Ensure that the binary is indeed compiled with -pie. - MemoryMappingLayout proc_maps(true); - uptr p, end; - while (proc_maps.Next(&p, &end, 0, 0, 0, 0)) { - if (IsAppMem(p)) - continue; - if (p >= kHeapMemEnd && - p < HeapEnd()) - continue; - if (p >= kVdsoBeg) // vdso - break; - Printf("FATAL: ThreadSanitizer: unexpected memory mapping %p-%p\n", p, end); +#endif // #ifndef SANITIZER_GO + +void InitializePlatformEarly() { +#ifdef TSAN_RUNTIME_VMA + vmaSize = + (MostSignificantSetBitIndex(GET_CURRENT_FRAME()) + 1); +#if defined(__aarch64__) + if (vmaSize != 39 && vmaSize != 42) { + Printf("FATAL: ThreadSanitizer: unsupported VMA range\n"); + Printf("FATAL: Found %d - Supported 39 and 42\n", vmaSize); Die(); } - - ProtectRange(kLoAppMemEnd, kShadowBeg); - ProtectRange(kShadowEnd, kMetaShadowBeg); - ProtectRange(kMetaShadowEnd, kTraceMemBeg); - // Memory for traces is mapped lazily in MapThreadTrace. - // Protect the whole range for now, so that user does not map something here. - ProtectRange(kTraceMemBeg, kTraceMemEnd); - ProtectRange(kTraceMemEnd, kHeapMemBeg); - ProtectRange(HeapEnd(), kHiAppMemBeg); +#elif defined(__powerpc64__) + if (vmaSize != 44 && vmaSize != 46) { + Printf("FATAL: ThreadSanitizer: unsupported VMA range\n"); + Printf("FATAL: Found %d - Supported 44 and 46\n", vmaSize); + Die(); + } +#endif +#endif } -#endif // #ifndef SANITIZER_GO void InitializePlatform() { DisableCoreDumperIfNecessary(); @@ -367,7 +309,7 @@ bool IsGlobalVar(uptr addr) { // This is required to properly "close" the fds, because we do not see internal // closes within glibc. The code is a pure hack. int ExtractResolvFDs(void *state, int *fds, int nfd) { -#if SANITIZER_LINUX +#if SANITIZER_LINUX && !SANITIZER_ANDROID int cnt = 0; __res_state *statp = (__res_state*)state; for (int i = 0; i < MAXNS && cnt < nfd; i++) { @@ -415,6 +357,10 @@ int call_pthread_cancel_with_cleanup(int(*fn)(void *c, void *m, } #endif +#ifndef SANITIZER_GO +void ReplaceSystemMalloc() { } +#endif + } // namespace __tsan #endif // SANITIZER_LINUX || SANITIZER_FREEBSD diff --git a/lib/tsan/rtl/tsan_platform_mac.cc b/lib/tsan/rtl/tsan_platform_mac.cc index b72d9b07ef35..31caf37dee5a 100644 --- a/lib/tsan/rtl/tsan_platform_mac.cc +++ b/lib/tsan/rtl/tsan_platform_mac.cc @@ -15,8 +15,10 @@ #include "sanitizer_common/sanitizer_platform.h" #if SANITIZER_MAC +#include "sanitizer_common/sanitizer_atomic.h" #include "sanitizer_common/sanitizer_common.h" #include "sanitizer_common/sanitizer_libc.h" +#include "sanitizer_common/sanitizer_posix.h" #include "sanitizer_common/sanitizer_procmaps.h" #include "tsan_platform.h" #include "tsan_rtl.h" @@ -40,6 +42,62 @@ namespace __tsan { +#ifndef SANITIZER_GO +static void *SignalSafeGetOrAllocate(uptr *dst, uptr size) { + atomic_uintptr_t *a = (atomic_uintptr_t *)dst; + void *val = (void *)atomic_load_relaxed(a); + atomic_signal_fence(memory_order_acquire); // Turns the previous load into + // acquire wrt signals. + if (UNLIKELY(val == nullptr)) { + val = (void *)internal_mmap(nullptr, size, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANON, -1, 0); + CHECK(val); + void *cmp = nullptr; + if (!atomic_compare_exchange_strong(a, (uintptr_t *)&cmp, (uintptr_t)val, + memory_order_acq_rel)) { + internal_munmap(val, size); + val = cmp; + } + } + return val; +} + +// On OS X, accessing TLVs via __thread or manually by using pthread_key_* is +// problematic, because there are several places where interceptors are called +// when TLVs are not accessible (early process startup, thread cleanup, ...). +// The following provides a "poor man's TLV" implementation, where we use the +// shadow memory of the pointer returned by pthread_self() to store a pointer to +// the ThreadState object. The main thread's ThreadState pointer is stored +// separately in a static variable, because we need to access it even before the +// shadow memory is set up. +static uptr main_thread_identity = 0; +static ThreadState *main_thread_state = nullptr; + +ThreadState *cur_thread() { + ThreadState **fake_tls; + uptr thread_identity = (uptr)pthread_self(); + if (thread_identity == main_thread_identity || main_thread_identity == 0) { + fake_tls = &main_thread_state; + } else { + fake_tls = (ThreadState **)MemToShadow(thread_identity); + } + ThreadState *thr = (ThreadState *)SignalSafeGetOrAllocate( + (uptr *)fake_tls, sizeof(ThreadState)); + return thr; +} + +// TODO(kuba.brecka): This is not async-signal-safe. In particular, we call +// munmap first and then clear `fake_tls`; if we receive a signal in between, +// handler will try to access the unmapped ThreadState. +void cur_thread_finalize() { + uptr thread_identity = (uptr)pthread_self(); + CHECK_NE(thread_identity, main_thread_identity); + ThreadState **fake_tls = (ThreadState **)MemToShadow(thread_identity); + internal_munmap(*fake_tls, sizeof(ThreadState)); + *fake_tls = nullptr; +} +#endif + uptr GetShadowMemoryConsumption() { return 0; } @@ -51,28 +109,62 @@ void WriteMemoryProfile(char *buf, uptr buf_size, uptr nthread, uptr nlive) { } #ifndef SANITIZER_GO -void InitializeShadowMemory() { - uptr shadow = (uptr)MmapFixedNoReserve(kShadowBeg, - kShadowEnd - kShadowBeg); - if (shadow != kShadowBeg) { - Printf("FATAL: ThreadSanitizer can not mmap the shadow memory\n"); - Printf("FATAL: Make sure to compile with -fPIE and " - "to link with -pie.\n"); - Die(); +void InitializeShadowMemoryPlatform() { } + +// On OS X, GCD worker threads are created without a call to pthread_create. We +// need to properly register these threads with ThreadCreate and ThreadStart. +// These threads don't have a parent thread, as they are created "spuriously". +// We're using a libpthread API that notifies us about a newly created thread. +// The `thread == pthread_self()` check indicates this is actually a worker +// thread. If it's just a regular thread, this hook is called on the parent +// thread. +typedef void (*pthread_introspection_hook_t)(unsigned int event, + pthread_t thread, void *addr, + size_t size); +extern "C" pthread_introspection_hook_t pthread_introspection_hook_install( + pthread_introspection_hook_t hook); +static const uptr PTHREAD_INTROSPECTION_THREAD_CREATE = 1; +static const uptr PTHREAD_INTROSPECTION_THREAD_TERMINATE = 3; +static pthread_introspection_hook_t prev_pthread_introspection_hook; +static void my_pthread_introspection_hook(unsigned int event, pthread_t thread, + void *addr, size_t size) { + if (event == PTHREAD_INTROSPECTION_THREAD_CREATE) { + if (thread == pthread_self()) { + // The current thread is a newly created GCD worker thread. + ThreadState *parent_thread_state = nullptr; // No parent. + int tid = ThreadCreate(parent_thread_state, 0, (uptr)thread, true); + CHECK_NE(tid, 0); + ThreadState *thr = cur_thread(); + ThreadStart(thr, tid, GetTid()); + } + } else if (event == PTHREAD_INTROSPECTION_THREAD_TERMINATE) { + if (thread == pthread_self()) { + ThreadState *thr = cur_thread(); + if (thr->tctx) { + DestroyThreadState(); + } + } } - if (common_flags()->use_madv_dontdump) - DontDumpShadowMemory(kShadowBeg, kShadowEnd - kShadowBeg); - DPrintf("kShadow %zx-%zx (%zuGB)\n", - kShadowBeg, kShadowEnd, - (kShadowEnd - kShadowBeg) >> 30); - DPrintf("kAppMem %zx-%zx (%zuGB)\n", - kAppMemBeg, kAppMemEnd, - (kAppMemEnd - kAppMemBeg) >> 30); + + if (prev_pthread_introspection_hook != nullptr) + prev_pthread_introspection_hook(event, thread, addr, size); } #endif +void InitializePlatformEarly() { +} + void InitializePlatform() { DisableCoreDumperIfNecessary(); +#ifndef SANITIZER_GO + CheckAndProtect(); + + CHECK_EQ(main_thread_identity, 0); + main_thread_identity = (uptr)pthread_self(); + + prev_pthread_introspection_hook = + pthread_introspection_hook_install(&my_pthread_introspection_hook); +#endif } #ifndef SANITIZER_GO @@ -91,6 +183,10 @@ int call_pthread_cancel_with_cleanup(int(*fn)(void *c, void *m, } #endif +bool IsGlobalVar(uptr addr) { + return false; +} + } // namespace __tsan #endif // SANITIZER_MAC diff --git a/lib/tsan/rtl/tsan_platform_posix.cc b/lib/tsan/rtl/tsan_platform_posix.cc new file mode 100644 index 000000000000..90476cbc5fd5 --- /dev/null +++ b/lib/tsan/rtl/tsan_platform_posix.cc @@ -0,0 +1,151 @@ +//===-- tsan_platform_posix.cc --------------------------------------------===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// +// +// This file is a part of ThreadSanitizer (TSan), a race detector. +// +// POSIX-specific code. +//===----------------------------------------------------------------------===// + +#include "sanitizer_common/sanitizer_platform.h" +#if SANITIZER_POSIX + +#include "sanitizer_common/sanitizer_common.h" +#include "sanitizer_common/sanitizer_libc.h" +#include "sanitizer_common/sanitizer_procmaps.h" +#include "tsan_platform.h" +#include "tsan_rtl.h" + +namespace __tsan { + +#ifndef SANITIZER_GO +void InitializeShadowMemory() { + // Map memory shadow. + uptr shadow = + (uptr)MmapFixedNoReserve(ShadowBeg(), ShadowEnd() - ShadowBeg(), + "shadow"); + if (shadow != ShadowBeg()) { + Printf("FATAL: ThreadSanitizer can not mmap the shadow memory\n"); + Printf("FATAL: Make sure to compile with -fPIE and " + "to link with -pie (%p, %p).\n", shadow, ShadowBeg()); + Die(); + } + // This memory range is used for thread stacks and large user mmaps. + // Frequently a thread uses only a small part of stack and similarly + // a program uses a small part of large mmap. On some programs + // we see 20% memory usage reduction without huge pages for this range. + // FIXME: don't use constants here. +#if defined(__x86_64__) + const uptr kMadviseRangeBeg = 0x7f0000000000ull; + const uptr kMadviseRangeSize = 0x010000000000ull; +#elif defined(__mips64) + const uptr kMadviseRangeBeg = 0xff00000000ull; + const uptr kMadviseRangeSize = 0x0100000000ull; +#elif defined(__aarch64__) + uptr kMadviseRangeBeg = 0; + uptr kMadviseRangeSize = 0; + if (vmaSize == 39) { + kMadviseRangeBeg = 0x7d00000000ull; + kMadviseRangeSize = 0x0300000000ull; + } else if (vmaSize == 42) { + kMadviseRangeBeg = 0x3f000000000ull; + kMadviseRangeSize = 0x01000000000ull; + } else { + DCHECK(0); + } +#elif defined(__powerpc64__) + uptr kMadviseRangeBeg = 0; + uptr kMadviseRangeSize = 0; + if (vmaSize == 44) { + kMadviseRangeBeg = 0x0f60000000ull; + kMadviseRangeSize = 0x0010000000ull; + } else if (vmaSize == 46) { + kMadviseRangeBeg = 0x3f0000000000ull; + kMadviseRangeSize = 0x010000000000ull; + } else { + DCHECK(0); + } +#endif + NoHugePagesInRegion(MemToShadow(kMadviseRangeBeg), + kMadviseRangeSize * kShadowMultiplier); + // Meta shadow is compressing and we don't flush it, + // so it makes sense to mark it as NOHUGEPAGE to not over-allocate memory. + // On one program it reduces memory consumption from 5GB to 2.5GB. + NoHugePagesInRegion(MetaShadowBeg(), MetaShadowEnd() - MetaShadowBeg()); + if (common_flags()->use_madv_dontdump) + DontDumpShadowMemory(ShadowBeg(), ShadowEnd() - ShadowBeg()); + DPrintf("memory shadow: %zx-%zx (%zuGB)\n", + ShadowBeg(), ShadowEnd(), + (ShadowEnd() - ShadowBeg()) >> 30); + + // Map meta shadow. + uptr meta_size = MetaShadowEnd() - MetaShadowBeg(); + uptr meta = + (uptr)MmapFixedNoReserve(MetaShadowBeg(), meta_size, "meta shadow"); + if (meta != MetaShadowBeg()) { + Printf("FATAL: ThreadSanitizer can not mmap the shadow memory\n"); + Printf("FATAL: Make sure to compile with -fPIE and " + "to link with -pie (%p, %p).\n", meta, MetaShadowBeg()); + Die(); + } + if (common_flags()->use_madv_dontdump) + DontDumpShadowMemory(meta, meta_size); + DPrintf("meta shadow: %zx-%zx (%zuGB)\n", + meta, meta + meta_size, meta_size >> 30); + + InitializeShadowMemoryPlatform(); +} + +static void ProtectRange(uptr beg, uptr end) { + CHECK_LE(beg, end); + if (beg == end) + return; + if (beg != (uptr)MmapNoAccess(beg, end - beg)) { + Printf("FATAL: ThreadSanitizer can not protect [%zx,%zx]\n", beg, end); + Printf("FATAL: Make sure you are not using unlimited stack\n"); + Die(); + } +} + +void CheckAndProtect() { + // Ensure that the binary is indeed compiled with -pie. + MemoryMappingLayout proc_maps(true); + uptr p, end, prot; + while (proc_maps.Next(&p, &end, 0, 0, 0, &prot)) { + if (IsAppMem(p)) + continue; + if (p >= HeapMemEnd() && + p < HeapEnd()) + continue; + if (prot == 0) // Zero page or mprotected. + continue; + if (p >= VdsoBeg()) // vdso + break; + Printf("FATAL: ThreadSanitizer: unexpected memory mapping %p-%p\n", p, end); + Die(); + } + + ProtectRange(LoAppMemEnd(), ShadowBeg()); + ProtectRange(ShadowEnd(), MetaShadowBeg()); +#ifdef TSAN_MID_APP_RANGE + ProtectRange(MetaShadowEnd(), MidAppMemBeg()); + ProtectRange(MidAppMemEnd(), TraceMemBeg()); +#else + ProtectRange(MetaShadowEnd(), TraceMemBeg()); +#endif + // Memory for traces is mapped lazily in MapThreadTrace. + // Protect the whole range for now, so that user does not map something here. + ProtectRange(TraceMemBeg(), TraceMemEnd()); + ProtectRange(TraceMemEnd(), HeapMemBeg()); + ProtectRange(HeapEnd(), HiAppMemBeg()); +} +#endif + +} // namespace __tsan + +#endif // SANITIZER_POSIX diff --git a/lib/tsan/rtl/tsan_platform_windows.cc b/lib/tsan/rtl/tsan_platform_windows.cc index cfbe77da2c07..c6d5058d96fc 100644 --- a/lib/tsan/rtl/tsan_platform_windows.cc +++ b/lib/tsan/rtl/tsan_platform_windows.cc @@ -31,6 +31,9 @@ void FlushShadowMemory() { void WriteMemoryProfile(char *buf, uptr buf_size, uptr nthread, uptr nlive) { } +void InitializePlatformEarly() { +} + void InitializePlatform() { } diff --git a/lib/tsan/rtl/tsan_ppc_regs.h b/lib/tsan/rtl/tsan_ppc_regs.h new file mode 100644 index 000000000000..5b43f3ddada3 --- /dev/null +++ b/lib/tsan/rtl/tsan_ppc_regs.h @@ -0,0 +1,96 @@ +#define r0 0 +#define r1 1 +#define r2 2 +#define r3 3 +#define r4 4 +#define r5 5 +#define r6 6 +#define r7 7 +#define r8 8 +#define r9 9 +#define r10 10 +#define r11 11 +#define r12 12 +#define r13 13 +#define r14 14 +#define r15 15 +#define r16 16 +#define r17 17 +#define r18 18 +#define r19 19 +#define r20 20 +#define r21 21 +#define r22 22 +#define r23 23 +#define r24 24 +#define r25 25 +#define r26 26 +#define r27 27 +#define r28 28 +#define r29 29 +#define r30 30 +#define r31 31 +#define f0 0 +#define f1 1 +#define f2 2 +#define f3 3 +#define f4 4 +#define f5 5 +#define f6 6 +#define f7 7 +#define f8 8 +#define f9 9 +#define f10 10 +#define f11 11 +#define f12 12 +#define f13 13 +#define f14 14 +#define f15 15 +#define f16 16 +#define f17 17 +#define f18 18 +#define f19 19 +#define f20 20 +#define f21 21 +#define f22 22 +#define f23 23 +#define f24 24 +#define f25 25 +#define f26 26 +#define f27 27 +#define f28 28 +#define f29 29 +#define f30 30 +#define f31 31 +#define v0 0 +#define v1 1 +#define v2 2 +#define v3 3 +#define v4 4 +#define v5 5 +#define v6 6 +#define v7 7 +#define v8 8 +#define v9 9 +#define v10 10 +#define v11 11 +#define v12 12 +#define v13 13 +#define v14 14 +#define v15 15 +#define v16 16 +#define v17 17 +#define v18 18 +#define v19 19 +#define v20 20 +#define v21 21 +#define v22 22 +#define v23 23 +#define v24 24 +#define v25 25 +#define v26 26 +#define v27 27 +#define v28 28 +#define v29 29 +#define v30 30 +#define v31 31 diff --git a/lib/tsan/rtl/tsan_report.cc b/lib/tsan/rtl/tsan_report.cc index f4b06878a58e..c1d2fd07c0d9 100644 --- a/lib/tsan/rtl/tsan_report.cc +++ b/lib/tsan/rtl/tsan_report.cc @@ -111,6 +111,12 @@ static const char *ReportTypeString(ReportType typ) { return ""; } +#if SANITIZER_MAC +static const char *const kInterposedFunctionPrefix = "wrap_"; +#else +static const char *const kInterposedFunctionPrefix = "__interceptor_"; +#endif + void PrintStack(const ReportStack *ent) { if (ent == 0 || ent->frames == 0) { Printf(" [failed to restore the stack]\n\n"); @@ -121,7 +127,7 @@ void PrintStack(const ReportStack *ent) { InternalScopedString res(2 * GetPageSizeCached()); RenderFrame(&res, common_flags()->stack_trace_format, i, frame->info, common_flags()->symbolize_vs_style, - common_flags()->strip_path_prefix, "__interceptor_"); + common_flags()->strip_path_prefix, kInterposedFunctionPrefix); Printf("%s\n", res.data()); } Printf("\n"); @@ -165,9 +171,14 @@ static void PrintLocation(const ReportLocation *loc) { Printf("%s", d.Location()); if (loc->type == ReportLocationGlobal) { const DataInfo &global = loc->global; - Printf(" Location is global '%s' of size %zu at %p (%s+%p)\n\n", - global.name, global.size, global.start, - StripModuleName(global.module), global.module_offset); + if (global.size != 0) + Printf(" Location is global '%s' of size %zu at %p (%s+%p)\n\n", + global.name, global.size, global.start, + StripModuleName(global.module), global.module_offset); + else + Printf(" Location is global '%s' at %p (%s+%p)\n\n", global.name, + global.start, StripModuleName(global.module), + global.module_offset); } else if (loc->type == ReportLocationHeap) { char thrbuf[kThreadBufSize]; Printf(" Location is heap block of size %zu at %p allocated by %s:\n", @@ -256,10 +267,15 @@ static bool FrameIsInternal(const SymbolizedStack *frame) { if (frame == 0) return false; const char *file = frame->info.file; - return file != 0 && - (internal_strstr(file, "tsan_interceptors.cc") || - internal_strstr(file, "sanitizer_common_interceptors.inc") || - internal_strstr(file, "tsan_interface_")); + const char *module = frame->info.module; + if (file != 0 && + (internal_strstr(file, "tsan_interceptors.cc") || + internal_strstr(file, "sanitizer_common_interceptors.inc") || + internal_strstr(file, "tsan_interface_"))) + return true; + if (module != 0 && (internal_strstr(module, "libclang_rt.tsan_"))) + return true; + return false; } static SymbolizedStack *SkipTsanInternalFrames(SymbolizedStack *frames) { diff --git a/lib/tsan/rtl/tsan_rtl.cc b/lib/tsan/rtl/tsan_rtl.cc index 63c356b228a4..4df4db557a24 100644 --- a/lib/tsan/rtl/tsan_rtl.cc +++ b/lib/tsan/rtl/tsan_rtl.cc @@ -44,7 +44,7 @@ extern "C" void __tsan_resume() { namespace __tsan { -#ifndef SANITIZER_GO +#if !defined(SANITIZER_GO) && !SANITIZER_MAC THREADLOCAL char cur_thread_placeholder[sizeof(ThreadState)] ALIGNED(64); #endif static char ctx_placeholder[sizeof(Context)] ALIGNED(64); @@ -55,12 +55,12 @@ Context *ctx; bool OnFinalize(bool failed); void OnInitialize(); #else -SANITIZER_INTERFACE_ATTRIBUTE -bool WEAK OnFinalize(bool failed) { +SANITIZER_WEAK_CXX_DEFAULT_IMPL +bool OnFinalize(bool failed) { return failed; } -SANITIZER_INTERFACE_ATTRIBUTE -void WEAK OnInitialize() {} +SANITIZER_WEAK_CXX_DEFAULT_IMPL +void OnInitialize() {} #endif static char thread_registry_placeholder[sizeof(ThreadRegistry)]; @@ -99,8 +99,10 @@ Context::Context() , nmissed_expected() , thread_registry(new(thread_registry_placeholder) ThreadRegistry( CreateThreadContext, kMaxTid, kThreadQuarantineSize, kMaxTidReuse)) + , racy_mtx(MutexTypeRacy, StatMtxRacy) , racy_stacks(MBlockRacyStacks) , racy_addresses(MBlockRacyAddresses) + , fired_suppressions_mtx(MutexTypeFired, StatMtxFired) , fired_suppressions(8) { } @@ -271,8 +273,8 @@ void MapShadow(uptr addr, uptr size) { void MapThreadTrace(uptr addr, uptr size, const char *name) { DPrintf("#0: Mapping trace at %p-%p(0x%zx)\n", addr, addr + size, size); - CHECK_GE(addr, kTraceMemBeg); - CHECK_LE(addr + size, kTraceMemEnd); + CHECK_GE(addr, TraceMemBeg()); + CHECK_LE(addr + size, TraceMemEnd()); CHECK_EQ(addr, addr & ~((64 << 10) - 1)); // windows wants 64K alignment uptr addr1 = (uptr)MmapFixedNoReserve(addr, size, name); if (addr1 != addr) { @@ -283,9 +285,8 @@ void MapThreadTrace(uptr addr, uptr size, const char *name) { } static void CheckShadowMapping() { - for (uptr i = 0; i < ARRAY_SIZE(UserRegions); i += 2) { - const uptr beg = UserRegions[i]; - const uptr end = UserRegions[i + 1]; + uptr beg, end; + for (int i = 0; GetUserRegion(i, &beg, &end); i++) { VPrintf(3, "checking shadow region %p-%p\n", beg, end); for (uptr p0 = beg; p0 <= end; p0 += (end - beg) / 4) { for (int x = -1; x <= 1; x++) { @@ -318,10 +319,15 @@ void Initialize(ThreadState *thr) { ctx = new(ctx_placeholder) Context; const char *options = GetEnv(kTsanOptionsEnv); - InitializeFlags(&ctx->flags, options); CacheBinaryName(); + InitializeFlags(&ctx->flags, options); + InitializePlatformEarly(); #ifndef SANITIZER_GO + // Re-exec ourselves if we need to set additional env or command line args. + MaybeReexec(); + InitializeAllocator(); + ReplaceSystemMalloc(); #endif InitializeInterceptors(); CheckShadowMapping(); @@ -417,7 +423,7 @@ int Finalize(ThreadState *thr) { StatOutput(ctx->stat); #endif - return failed ? flags()->exitcode : 0; + return failed ? common_flags()->exitcode : 0; } #ifndef SANITIZER_GO diff --git a/lib/tsan/rtl/tsan_rtl.h b/lib/tsan/rtl/tsan_rtl.h index a13e4b6379f0..04104b162f98 100644 --- a/lib/tsan/rtl/tsan_rtl.h +++ b/lib/tsan/rtl/tsan_rtl.h @@ -54,7 +54,7 @@ namespace __tsan { #ifndef SANITIZER_GO struct MapUnmapCallback; -#ifdef __mips64 +#if defined(__mips64) || defined(__aarch64__) || defined(__powerpc__) static const uptr kAllocatorSpace = 0; static const uptr kAllocatorSize = SANITIZER_MMAP_RANGE_SIZE; static const uptr kAllocatorRegionSizeLog = 20; @@ -66,7 +66,8 @@ typedef SizeClassAllocator32<kAllocatorSpace, kAllocatorSize, 0, CompactSizeClassMap, kAllocatorRegionSizeLog, ByteMap, MapUnmapCallback> PrimaryAllocator; #else -typedef SizeClassAllocator64<kHeapMemBeg, kHeapMemEnd - kHeapMemBeg, 0, +typedef SizeClassAllocator64<Mapping::kHeapMemBeg, + Mapping::kHeapMemEnd - Mapping::kHeapMemBeg, 0, DefaultSizeClassMap, MapUnmapCallback> PrimaryAllocator; #endif typedef SizeClassAllocatorLocalCache<PrimaryAllocator> AllocatorCache; @@ -410,12 +411,18 @@ struct ThreadState { }; #ifndef SANITIZER_GO +#if SANITIZER_MAC +ThreadState *cur_thread(); +void cur_thread_finalize(); +#else __attribute__((tls_model("initial-exec"))) extern THREADLOCAL char cur_thread_placeholder[]; INLINE ThreadState *cur_thread() { return reinterpret_cast<ThreadState *>(&cur_thread_placeholder); } -#endif +INLINE void cur_thread_finalize() { } +#endif // SANITIZER_MAC +#endif // SANITIZER_GO class ThreadContext : public ThreadContextBase { public: @@ -458,7 +465,7 @@ struct RacyAddress { struct FiredSuppression { ReportType type; - uptr pc; + uptr pc_or_addr; Suppression *supp; }; @@ -480,9 +487,11 @@ struct Context { ThreadRegistry *thread_registry; + Mutex racy_mtx; Vector<RacyStacks> racy_stacks; Vector<RacyAddress> racy_addresses; // Number of fired suppressions may be large enough. + Mutex fired_suppressions_mtx; InternalMmapVector<FiredSuppression> fired_suppressions; DDetector *dd; @@ -587,8 +596,7 @@ void ForkChildAfter(ThreadState *thr, uptr pc); void ReportRace(ThreadState *thr); bool OutputReport(ThreadState *thr, const ScopedReport &srep); -bool IsFiredSuppression(Context *ctx, const ScopedReport &srep, - StackTrace trace); +bool IsFiredSuppression(Context *ctx, ReportType type, StackTrace trace); bool IsExpectedReport(uptr addr, uptr size); void PrintMatchedBenignRaces(); @@ -708,7 +716,7 @@ void AcquireReleaseImpl(ThreadState *thr, uptr pc, SyncClock *c); // The trick is that the call preserves all registers and the compiler // does not treat it as a call. // If it does not work for you, use normal call. -#if !SANITIZER_DEBUG && defined(__x86_64__) +#if !SANITIZER_DEBUG && defined(__x86_64__) && !SANITIZER_MAC // The caller may not create the stack frame for itself at all, // so we create a reserve stack frame for it (1024b must be enough). #define HACKY_CALL(f) \ @@ -754,11 +762,7 @@ void ALWAYS_INLINE TraceAddEvent(ThreadState *thr, FastState fs, #ifndef SANITIZER_GO uptr ALWAYS_INLINE HeapEnd() { -#if SANITIZER_CAN_USE_ALLOCATOR64 - return kHeapMemEnd + PrimaryAllocator::AdditionalSize(); -#else - return kHeapMemEnd; -#endif + return HeapMemEnd() + PrimaryAllocator::AdditionalSize(); } #endif diff --git a/lib/tsan/rtl/tsan_rtl_aarch64.S b/lib/tsan/rtl/tsan_rtl_aarch64.S new file mode 100644 index 000000000000..9cea3cf02800 --- /dev/null +++ b/lib/tsan/rtl/tsan_rtl_aarch64.S @@ -0,0 +1,206 @@ +#include "sanitizer_common/sanitizer_asm.h" +.section .text + +.hidden __tsan_setjmp +.comm _ZN14__interception11real_setjmpE,8,8 +.type setjmp, @function +setjmp: + CFI_STARTPROC + + // save env parameters for function call + stp x29, x30, [sp, -32]! + CFI_DEF_CFA_OFFSET (32) + CFI_OFFSET (29, -32) + CFI_OFFSET (30, -24) + + // Adjust the SP for previous frame + add x29, sp, 0 + CFI_DEF_CFA_REGISTER (29) + + // Save jmp_buf + str x19, [sp, 16] + CFI_OFFSET (19, -16) + mov x19, x0 + + // SP pointer mangling (see glibc setjmp) + adrp x2, :got:__pointer_chk_guard + ldr x2, [x2, #:got_lo12:__pointer_chk_guard] + add x0, x29, 32 + ldr x2, [x2] + eor x1, x2, x0 + + // call tsan interceptor + bl __tsan_setjmp + + // restore env parameter + mov x0, x19 + ldr x19, [sp, 16] + ldp x29, x30, [sp], 32 + CFI_RESTORE (30) + CFI_RESTORE (19) + CFI_DEF_CFA (31, 0) + + // tail jump to libc setjmp + adrp x1, :got:_ZN14__interception11real_setjmpE + ldr x1, [x1, #:got_lo12:_ZN14__interception11real_setjmpE] + ldr x1, [x1] + br x1 + + CFI_ENDPROC +.size setjmp, .-setjmp + +.comm _ZN14__interception12real__setjmpE,8,8 +.globl _setjmp +.type _setjmp, @function +_setjmp: + CFI_STARTPROC + + // save env parameters for function call + stp x29, x30, [sp, -32]! + CFI_DEF_CFA_OFFSET (32) + CFI_OFFSET (29, -32) + CFI_OFFSET (30, -24) + + // Adjust the SP for previous frame + add x29, sp, 0 + CFI_DEF_CFA_REGISTER (29) + + // Save jmp_buf + str x19, [sp, 16] + CFI_OFFSET (19, -16) + mov x19, x0 + + // SP pointer mangling (see glibc setjmp) + adrp x2, :got:__pointer_chk_guard + ldr x2, [x2, #:got_lo12:__pointer_chk_guard] + add x0, x29, 32 + ldr x2, [x2] + eor x1, x2, x0 + + // call tsan interceptor + bl __tsan_setjmp + + // Restore jmp_buf parameter + mov x0, x19 + ldr x19, [sp, 16] + ldp x29, x30, [sp], 32 + CFI_RESTORE (30) + CFI_RESTORE (19) + CFI_DEF_CFA (31, 0) + + // tail jump to libc setjmp + adrp x1, :got:_ZN14__interception12real__setjmpE + ldr x1, [x1, #:got_lo12:_ZN14__interception12real__setjmpE] + ldr x1, [x1] + br x1 + + CFI_ENDPROC +.size _setjmp, .-_setjmp + +.comm _ZN14__interception14real_sigsetjmpE,8,8 +.globl sigsetjmp +.type sigsetjmp, @function +sigsetjmp: + CFI_STARTPROC + + // save env parameters for function call + stp x29, x30, [sp, -32]! + CFI_DEF_CFA_OFFSET (32) + CFI_OFFSET (29, -32) + CFI_OFFSET (30, -24) + + // Adjust the SP for previous frame + add x29, sp, 0 + CFI_DEF_CFA_REGISTER (29) + + // Save jmp_buf and savesigs + stp x19, x20, [sp, 16] + CFI_OFFSET (19, -16) + CFI_OFFSET (20, -8) + mov w20, w1 + mov x19, x0 + + // SP pointer mangling (see glibc setjmp) + adrp x2, :got:__pointer_chk_guard + ldr x2, [x2, #:got_lo12:__pointer_chk_guard] + add x0, x29, 32 + ldr x2, [x2] + eor x1, x2, x0 + + // call tsan interceptor + bl __tsan_setjmp + + // restore env parameter + mov w1, w20 + mov x0, x19 + ldp x19, x20, [sp, 16] + ldp x29, x30, [sp], 32 + CFI_RESTORE (30) + CFI_RESTORE (29) + CFI_RESTORE (19) + CFI_RESTORE (20) + CFI_DEF_CFA (31, 0) + + // tail jump to libc sigsetjmp + adrp x2, :got:_ZN14__interception14real_sigsetjmpE + ldr x2, [x2, #:got_lo12:_ZN14__interception14real_sigsetjmpE] + ldr x2, [x2] + br x2 + CFI_ENDPROC +.size sigsetjmp, .-sigsetjmp + +.comm _ZN14__interception16real___sigsetjmpE,8,8 +.globl __sigsetjmp +.type __sigsetjmp, @function +__sigsetjmp: + CFI_STARTPROC + + // save env parameters for function call + stp x29, x30, [sp, -32]! + CFI_DEF_CFA_OFFSET (32) + CFI_OFFSET (29, -32) + CFI_OFFSET (30, -24) + + // Adjust the SP for previous frame + add x29, sp, 0 + CFI_DEF_CFA_REGISTER (29) + + // Save jmp_buf and savesigs + stp x19, x20, [sp, 16] + CFI_OFFSET (19, -16) + CFI_OFFSET (20, -8) + mov w20, w1 + mov x19, x0 + + // SP pointer mangling (see glibc setjmp) + adrp x2, :got:__pointer_chk_guard + ldr x2, [x2, #:got_lo12:__pointer_chk_guard] + add x0, x29, 32 + ldr x2, [x2] + eor x1, x2, x0 + + // call tsan interceptor + bl __tsan_setjmp + + mov w1, w20 + mov x0, x19 + ldp x19, x20, [sp, 16] + ldp x29, x30, [sp], 32 + CFI_RESTORE (30) + CFI_RESTORE (29) + CFI_RESTORE (19) + CFI_RESTORE (20) + CFI_DEF_CFA (31, 0) + + // tail jump to libc __sigsetjmp + adrp x2, :got:_ZN14__interception16real___sigsetjmpE + ldr x2, [x2, #:got_lo12:_ZN14__interception16real___sigsetjmpE] + ldr x2, [x2] + br x2 + CFI_ENDPROC +.size __sigsetjmp, .-__sigsetjmp + +#if defined(__linux__) +/* We do not need executable stack. */ +.section .note.GNU-stack,"",@progbits +#endif diff --git a/lib/tsan/rtl/tsan_rtl_amd64.S b/lib/tsan/rtl/tsan_rtl_amd64.S index 8db62f9013a3..caa832375e52 100644 --- a/lib/tsan/rtl/tsan_rtl_amd64.S +++ b/lib/tsan/rtl/tsan_rtl_amd64.S @@ -1,9 +1,13 @@ #include "sanitizer_common/sanitizer_asm.h" +#if !defined(__APPLE__) .section .text +#else +.section __TEXT,__text +#endif -.hidden __tsan_trace_switch -.globl __tsan_trace_switch_thunk -__tsan_trace_switch_thunk: +ASM_HIDDEN(__tsan_trace_switch) +.globl ASM_TSAN_SYMBOL(__tsan_trace_switch_thunk) +ASM_TSAN_SYMBOL(__tsan_trace_switch_thunk): CFI_STARTPROC # Save scratch registers. push %rax @@ -42,7 +46,7 @@ __tsan_trace_switch_thunk: shr $4, %rsp # clear 4 lsb, align to 16 shl $4, %rsp - call __tsan_trace_switch + call ASM_TSAN_SYMBOL(__tsan_trace_switch) # Unalign stack frame back. mov %rbx, %rsp # restore the original rsp @@ -81,9 +85,9 @@ __tsan_trace_switch_thunk: ret CFI_ENDPROC -.hidden __tsan_report_race -.globl __tsan_report_race_thunk -__tsan_report_race_thunk: +ASM_HIDDEN(__tsan_report_race) +.globl ASM_TSAN_SYMBOL(__tsan_report_race_thunk) +ASM_TSAN_SYMBOL(__tsan_report_race_thunk): CFI_STARTPROC # Save scratch registers. push %rax @@ -122,7 +126,7 @@ __tsan_report_race_thunk: shr $4, %rsp # clear 4 lsb, align to 16 shl $4, %rsp - call __tsan_report_race + call ASM_TSAN_SYMBOL(__tsan_report_race) # Unalign stack frame back. mov %rbx, %rsp # restore the original rsp @@ -161,11 +165,13 @@ __tsan_report_race_thunk: ret CFI_ENDPROC -.hidden __tsan_setjmp +ASM_HIDDEN(__tsan_setjmp) +#if !defined(__APPLE__) .comm _ZN14__interception11real_setjmpE,8,8 -.globl setjmp -.type setjmp, @function -setjmp: +#endif +.globl ASM_TSAN_SYMBOL_INTERCEPTOR(setjmp) +ASM_TYPE_FUNCTION(ASM_TSAN_SYMBOL_INTERCEPTOR(setjmp)) +ASM_TSAN_SYMBOL_INTERCEPTOR(setjmp): CFI_STARTPROC // save env parameter push %rdi @@ -175,29 +181,38 @@ setjmp: #if defined(__FreeBSD__) lea 8(%rsp), %rdi mov %rdi, %rsi -#else +#elif defined(__APPLE__) + lea 16(%rsp), %rdi + mov %rdi, %rsi +#elif defined(__linux__) lea 16(%rsp), %rdi mov %rdi, %rsi xor %fs:0x30, %rsi // magic mangling of rsp (see libc setjmp) rol $0x11, %rsi +#else +# error "Unknown platform" #endif // call tsan interceptor - call __tsan_setjmp + call ASM_TSAN_SYMBOL(__tsan_setjmp) // restore env parameter pop %rdi CFI_ADJUST_CFA_OFFSET(-8) CFI_RESTORE(%rdi) // tail jump to libc setjmp movl $0, %eax +#if !defined(__APPLE__) movq _ZN14__interception11real_setjmpE@GOTPCREL(%rip), %rdx jmp *(%rdx) +#else + jmp ASM_TSAN_SYMBOL(setjmp) +#endif CFI_ENDPROC -.size setjmp, .-setjmp +ASM_SIZE(ASM_TSAN_SYMBOL_INTERCEPTOR(setjmp)) .comm _ZN14__interception12real__setjmpE,8,8 -.globl _setjmp -.type _setjmp, @function -_setjmp: +.globl ASM_TSAN_SYMBOL_INTERCEPTOR(_setjmp) +ASM_TYPE_FUNCTION(ASM_TSAN_SYMBOL_INTERCEPTOR(_setjmp)) +ASM_TSAN_SYMBOL_INTERCEPTOR(_setjmp): CFI_STARTPROC // save env parameter push %rdi @@ -207,29 +222,38 @@ _setjmp: #if defined(__FreeBSD__) lea 8(%rsp), %rdi mov %rdi, %rsi -#else +#elif defined(__APPLE__) + lea 16(%rsp), %rdi + mov %rdi, %rsi +#elif defined(__linux__) lea 16(%rsp), %rdi mov %rdi, %rsi xor %fs:0x30, %rsi // magic mangling of rsp (see libc setjmp) rol $0x11, %rsi +#else +# error "Unknown platform" #endif // call tsan interceptor - call __tsan_setjmp + call ASM_TSAN_SYMBOL(__tsan_setjmp) // restore env parameter pop %rdi CFI_ADJUST_CFA_OFFSET(-8) CFI_RESTORE(%rdi) // tail jump to libc setjmp movl $0, %eax +#if !defined(__APPLE__) movq _ZN14__interception12real__setjmpE@GOTPCREL(%rip), %rdx jmp *(%rdx) +#else + jmp ASM_TSAN_SYMBOL(_setjmp) +#endif CFI_ENDPROC -.size _setjmp, .-_setjmp +ASM_SIZE(ASM_TSAN_SYMBOL_INTERCEPTOR(_setjmp)) .comm _ZN14__interception14real_sigsetjmpE,8,8 -.globl sigsetjmp -.type sigsetjmp, @function -sigsetjmp: +.globl ASM_TSAN_SYMBOL_INTERCEPTOR(sigsetjmp) +ASM_TYPE_FUNCTION(ASM_TSAN_SYMBOL_INTERCEPTOR(sigsetjmp)) +ASM_TSAN_SYMBOL_INTERCEPTOR(sigsetjmp): CFI_STARTPROC // save env parameter push %rdi @@ -246,14 +270,19 @@ sigsetjmp: #if defined(__FreeBSD__) lea 24(%rsp), %rdi mov %rdi, %rsi -#else +#elif defined(__APPLE__) + lea 32(%rsp), %rdi + mov %rdi, %rsi +#elif defined(__linux__) lea 32(%rsp), %rdi mov %rdi, %rsi xor %fs:0x30, %rsi // magic mangling of rsp (see libc setjmp) rol $0x11, %rsi +#else +# error "Unknown platform" #endif // call tsan interceptor - call __tsan_setjmp + call ASM_TSAN_SYMBOL(__tsan_setjmp) // unalign stack frame add $8, %rsp CFI_ADJUST_CFA_OFFSET(-8) @@ -267,15 +296,20 @@ sigsetjmp: CFI_RESTORE(%rdi) // tail jump to libc sigsetjmp movl $0, %eax +#if !defined(__APPLE__) movq _ZN14__interception14real_sigsetjmpE@GOTPCREL(%rip), %rdx jmp *(%rdx) +#else + jmp ASM_TSAN_SYMBOL(sigsetjmp) +#endif CFI_ENDPROC -.size sigsetjmp, .-sigsetjmp +ASM_SIZE(ASM_TSAN_SYMBOL_INTERCEPTOR(sigsetjmp)) +#if !defined(__APPLE__) .comm _ZN14__interception16real___sigsetjmpE,8,8 -.globl __sigsetjmp -.type __sigsetjmp, @function -__sigsetjmp: +.globl ASM_TSAN_SYMBOL_INTERCEPTOR(__sigsetjmp) +ASM_TYPE_FUNCTION(ASM_TSAN_SYMBOL_INTERCEPTOR(__sigsetjmp)) +ASM_TSAN_SYMBOL_INTERCEPTOR(__sigsetjmp): CFI_STARTPROC // save env parameter push %rdi @@ -299,7 +333,7 @@ __sigsetjmp: rol $0x11, %rsi #endif // call tsan interceptor - call __tsan_setjmp + call ASM_TSAN_SYMBOL(__tsan_setjmp) // unalign stack frame add $8, %rsp CFI_ADJUST_CFA_OFFSET(-8) @@ -316,7 +350,8 @@ __sigsetjmp: movq _ZN14__interception16real___sigsetjmpE@GOTPCREL(%rip), %rdx jmp *(%rdx) CFI_ENDPROC -.size __sigsetjmp, .-__sigsetjmp +ASM_SIZE(ASM_TSAN_SYMBOL_INTERCEPTOR(__sigsetjmp)) +#endif // !defined(__APPLE__) #if defined(__FreeBSD__) || defined(__linux__) /* We do not need executable stack. */ diff --git a/lib/tsan/rtl/tsan_rtl_mutex.cc b/lib/tsan/rtl/tsan_rtl_mutex.cc index 09180d88a6fb..62ab7aa6b2b4 100644 --- a/lib/tsan/rtl/tsan_rtl_mutex.cc +++ b/lib/tsan/rtl/tsan_rtl_mutex.cc @@ -472,7 +472,7 @@ void ReportDeadlock(ThreadState *thr, uptr pc, DDReport *r) { for (int i = 0; i < r->n; i++) { for (int j = 0; j < (flags()->second_deadlock_stack ? 2 : 1); j++) { u32 stk = r->loop[i].stk[j]; - if (stk) { + if (stk && stk != 0xffffffff) { rep.AddStack(StackDepotGet(stk), true); } else { // Sometimes we fail to extract the stack trace (FIXME: investigate), diff --git a/lib/tsan/rtl/tsan_rtl_ppc64.S b/lib/tsan/rtl/tsan_rtl_ppc64.S new file mode 100644 index 000000000000..8285e21aa1ec --- /dev/null +++ b/lib/tsan/rtl/tsan_rtl_ppc64.S @@ -0,0 +1,288 @@ +#include "tsan_ppc_regs.h" + + .section .text + .hidden __tsan_setjmp + .globl _setjmp + .type _setjmp, @function + .align 4 +#if _CALL_ELF == 2 +_setjmp: +#else + .section ".opd","aw" + .align 3 +_setjmp: + .quad .L._setjmp,.TOC.@tocbase,0 + .previous +#endif +.L._setjmp: + mflr r0 + stdu r1,-48(r1) + std r2,24(r1) + std r3,32(r1) + std r0,40(r1) + // r3 is the original stack pointer. + addi r3,r1,48 + // r4 is the mangled stack pointer (see glibc) + ld r4,-28696(r13) + xor r4,r3,r4 + // Materialize a TOC in case we were called from libc. + // For big-endian, we load the TOC from the OPD. For little- + // endian, we use the .TOC. symbol to find it. + nop + bcl 20,31,0f +0: + mflr r2 +#if _CALL_ELF == 2 + addis r2,r2,.TOC.-0b@ha + addi r2,r2,.TOC.-0b@l +#else + addis r2,r2,_setjmp-0b@ha + addi r2,r2,_setjmp-0b@l + ld r2,8(r2) +#endif + // Call the interceptor. + bl __tsan_setjmp + nop + // Restore regs needed for setjmp. + ld r3,32(r1) + ld r0,40(r1) + // Emulate the real setjmp function. We do this because we can't + // perform a sibcall: The real setjmp function trashes the TOC + // pointer, and with a sibcall we have no way to restore it. + // This way we can make sure our caller's stack pointer and + // link register are saved correctly in the jmpbuf. + ld r6,-28696(r13) + addi r5,r1,48 // original stack ptr of caller + xor r5,r6,r5 + std r5,0(r3) // mangled stack ptr of caller + ld r5,24(r1) + std r5,8(r3) // caller's saved TOC pointer + xor r0,r6,r0 + std r0,16(r3) // caller's mangled return address + mfcr r0 + // Nonvolatiles. + std r14,24(r3) + stfd f14,176(r3) + stw r0,172(r3) // CR + std r15,32(r3) + stfd f15,184(r3) + std r16,40(r3) + stfd f16,192(r3) + std r17,48(r3) + stfd f17,200(r3) + std r18,56(r3) + stfd f18,208(r3) + std r19,64(r3) + stfd f19,216(r3) + std r20,72(r3) + stfd f20,224(r3) + std r21,80(r3) + stfd f21,232(r3) + std r22,88(r3) + stfd f22,240(r3) + std r23,96(r3) + stfd f23,248(r3) + std r24,104(r3) + stfd f24,256(r3) + std r25,112(r3) + stfd f25,264(r3) + std r26,120(r3) + stfd f26,272(r3) + std r27,128(r3) + stfd f27,280(r3) + std r28,136(r3) + stfd f28,288(r3) + std r29,144(r3) + stfd f29,296(r3) + std r30,152(r3) + stfd f30,304(r3) + std r31,160(r3) + stfd f31,312(r3) + addi r5,r3,320 + mfspr r0,256 + stw r0,168(r3) // VRSAVE + addi r6,r5,16 + stvx v20,0,r5 + addi r5,r5,32 + stvx v21,0,r6 + addi r6,r6,32 + stvx v22,0,r5 + addi r5,r5,32 + stvx v23,0,r6 + addi r6,r6,32 + stvx v24,0,r5 + addi r5,r5,32 + stvx v25,0,r6 + addi r6,r6,32 + stvx v26,0,r5 + addi r5,r5,32 + stvx v27,0,r6 + addi r6,r6,32 + stvx v28,0,r5 + addi r5,r5,32 + stvx v29,0,r6 + addi r6,r6,32 + stvx v30,0,r5 + stvx v31,0,r6 + // Clear the "mask-saved" slot. + li r4,0 + stw r4,512(r3) + // Restore TOC, LR, and stack and return to caller. + ld r2,24(r1) + ld r0,40(r1) + addi r1,r1,48 + li r3,0 // This is the setjmp return path + mtlr r0 + blr + .size _setjmp, .-.L._setjmp + + .globl setjmp + .type setjmp, @function + .align 4 +setjmp: + b _setjmp + .size setjmp, .-setjmp + + // sigsetjmp is like setjmp, except that the mask in r4 needs + // to be saved at offset 512 of the jump buffer. + .globl __sigsetjmp + .type __sigsetjmp, @function + .align 4 +#if _CALL_ELF == 2 +__sigsetjmp: +#else + .section ".opd","aw" + .align 3 +__sigsetjmp: + .quad .L.__sigsetjmp,.TOC.@tocbase,0 + .previous +#endif +.L.__sigsetjmp: + mflr r0 + stdu r1,-64(r1) + std r2,24(r1) + std r3,32(r1) + std r4,40(r1) + std r0,48(r1) + // r3 is the original stack pointer. + addi r3,r1,64 + // r4 is the mangled stack pointer (see glibc) + ld r4,-28696(r13) + xor r4,r3,r4 + // Materialize a TOC in case we were called from libc. + // For big-endian, we load the TOC from the OPD. For little- + // endian, we use the .TOC. symbol to find it. + nop + bcl 20,31,1f +1: + mflr r2 +#if _CALL_ELF == 2 + addis r2,r2,.TOC.-1b@ha + addi r2,r2,.TOC.-1b@l +#else + addis r2,r2,_setjmp-1b@ha + addi r2,r2,_setjmp-1b@l + ld r2,8(r2) +#endif + // Call the interceptor. + bl __tsan_setjmp + nop + // Restore regs needed for __sigsetjmp. + ld r3,32(r1) + ld r4,40(r1) + ld r0,48(r1) + // Emulate the real sigsetjmp function. We do this because we can't + // perform a sibcall: The real sigsetjmp function trashes the TOC + // pointer, and with a sibcall we have no way to restore it. + // This way we can make sure our caller's stack pointer and + // link register are saved correctly in the jmpbuf. + ld r6,-28696(r13) + addi r5,r1,64 // original stack ptr of caller + xor r5,r6,r5 + std r5,0(r3) // mangled stack ptr of caller + ld r5,24(r1) + std r5,8(r3) // caller's saved TOC pointer + xor r0,r6,r0 + std r0,16(r3) // caller's mangled return address + mfcr r0 + // Nonvolatiles. + std r14,24(r3) + stfd f14,176(r3) + stw r0,172(r3) // CR + std r15,32(r3) + stfd f15,184(r3) + std r16,40(r3) + stfd f16,192(r3) + std r17,48(r3) + stfd f17,200(r3) + std r18,56(r3) + stfd f18,208(r3) + std r19,64(r3) + stfd f19,216(r3) + std r20,72(r3) + stfd f20,224(r3) + std r21,80(r3) + stfd f21,232(r3) + std r22,88(r3) + stfd f22,240(r3) + std r23,96(r3) + stfd f23,248(r3) + std r24,104(r3) + stfd f24,256(r3) + std r25,112(r3) + stfd f25,264(r3) + std r26,120(r3) + stfd f26,272(r3) + std r27,128(r3) + stfd f27,280(r3) + std r28,136(r3) + stfd f28,288(r3) + std r29,144(r3) + stfd f29,296(r3) + std r30,152(r3) + stfd f30,304(r3) + std r31,160(r3) + stfd f31,312(r3) + addi r5,r3,320 + mfspr r0,256 + stw r0,168(r3) // VRSAVE + addi r6,r5,16 + stvx v20,0,r5 + addi r5,r5,32 + stvx v21,0,r6 + addi r6,r6,32 + stvx v22,0,r5 + addi r5,r5,32 + stvx v23,0,r6 + addi r6,r6,32 + stvx v24,0,r5 + addi r5,r5,32 + stvx v25,0,r6 + addi r6,r6,32 + stvx v26,0,r5 + addi r5,r5,32 + stvx v27,0,r6 + addi r6,r6,32 + stvx v28,0,r5 + addi r5,r5,32 + stvx v29,0,r6 + addi r6,r6,32 + stvx v30,0,r5 + stvx v31,0,r6 + // Save into the "mask-saved" slot. + stw r4,512(r3) + // Restore TOC, LR, and stack and return to caller. + ld r2,24(r1) + ld r0,48(r1) + addi r1,r1,64 + li r3,0 // This is the sigsetjmp return path + mtlr r0 + blr + .size __sigsetjmp, .-.L.__sigsetjmp + + .globl sigsetjmp + .type sigsetjmp, @function + .align 4 +sigsetjmp: + b __sigsetjmp + .size sigsetjmp, .-sigsetjmp diff --git a/lib/tsan/rtl/tsan_rtl_report.cc b/lib/tsan/rtl/tsan_rtl_report.cc index dc9438e6371b..5aff6ca56adf 100644 --- a/lib/tsan/rtl/tsan_rtl_report.cc +++ b/lib/tsan/rtl/tsan_rtl_report.cc @@ -49,8 +49,8 @@ void TsanCheckFailed(const char *file, int line, const char *cond, #ifdef TSAN_EXTERNAL_HOOKS bool OnReport(const ReportDesc *rep, bool suppressed); #else -SANITIZER_INTERFACE_ATTRIBUTE -bool WEAK OnReport(const ReportDesc *rep, bool suppressed) { +SANITIZER_WEAK_CXX_DEFAULT_IMPL +bool OnReport(const ReportDesc *rep, bool suppressed) { (void)rep; return suppressed; } @@ -186,7 +186,7 @@ void ScopedReport::AddThread(const ThreadContext *tctx, bool suppressable) { return; } void *mem = internal_alloc(MBlockReportThread, sizeof(ReportThread)); - ReportThread *rt = new(mem) ReportThread(); + ReportThread *rt = new(mem) ReportThread; rep_->threads.PushBack(rt); rt->id = tctx->tid; rt->pid = tctx->os_id; @@ -200,16 +200,16 @@ void ScopedReport::AddThread(const ThreadContext *tctx, bool suppressable) { } #ifndef SANITIZER_GO +static bool FindThreadByUidLockedCallback(ThreadContextBase *tctx, void *arg) { + int unique_id = *(int *)arg; + return tctx->unique_id == (u32)unique_id; +} + static ThreadContext *FindThreadByUidLocked(int unique_id) { ctx->thread_registry->CheckLocked(); - for (unsigned i = 0; i < kMaxTid; i++) { - ThreadContext *tctx = static_cast<ThreadContext*>( - ctx->thread_registry->GetThreadLocked(i)); - if (tctx && tctx->unique_id == (u32)unique_id) { - return tctx; - } - } - return 0; + return static_cast<ThreadContext *>( + ctx->thread_registry->FindThreadContextLocked( + FindThreadByUidLockedCallback, &unique_id)); } static ThreadContext *FindThreadByTidLocked(int tid) { @@ -256,7 +256,7 @@ void ScopedReport::AddMutex(const SyncVar *s) { return; } void *mem = internal_alloc(MBlockReportMutex, sizeof(ReportMutex)); - ReportMutex *rm = new(mem) ReportMutex(); + ReportMutex *rm = new(mem) ReportMutex; rep_->mutexes.PushBack(rm); rm->id = s->uid; rm->addr = s->addr; @@ -289,7 +289,7 @@ void ScopedReport::AddDeadMutex(u64 id) { return; } void *mem = internal_alloc(MBlockReportMutex, sizeof(ReportMutex)); - ReportMutex *rm = new(mem) ReportMutex(); + ReportMutex *rm = new(mem) ReportMutex; rep_->mutexes.PushBack(rm); rm->id = id; rm->addr = 0; @@ -369,27 +369,20 @@ void RestoreStack(int tid, const u64 epoch, VarSizeStackTrace *stk, // This function restores stack trace and mutex set for the thread/epoch. // It does so by getting stack trace and mutex set at the beginning of // trace part, and then replaying the trace till the given epoch. - ctx->thread_registry->CheckLocked(); - ThreadContext *tctx = static_cast<ThreadContext*>( - ctx->thread_registry->GetThreadLocked(tid)); - if (tctx == 0) - return; - if (tctx->status != ThreadStatusRunning - && tctx->status != ThreadStatusFinished - && tctx->status != ThreadStatusDead) - return; - Trace* trace = ThreadTrace(tctx->tid); - Lock l(&trace->mtx); + Trace* trace = ThreadTrace(tid); + ReadLock l(&trace->mtx); const int partidx = (epoch / kTracePartSize) % TraceParts(); TraceHeader* hdr = &trace->headers[partidx]; - if (epoch < hdr->epoch0) + if (epoch < hdr->epoch0 || epoch >= hdr->epoch0 + kTracePartSize) return; + CHECK_EQ(RoundDown(epoch, kTracePartSize), hdr->epoch0); const u64 epoch0 = RoundDown(epoch, TraceSize()); const u64 eend = epoch % TraceSize(); const u64 ebegin = RoundDown(eend, kTracePartSize); DPrintf("#%d: RestoreStack epoch=%zu ebegin=%zu eend=%zu partidx=%d\n", tid, (uptr)epoch, (uptr)ebegin, (uptr)eend, partidx); - InternalScopedBuffer<uptr> stack(kShadowStackSize); + Vector<uptr> stack(MBlockReportStack); + stack.Resize(hdr->stack0.size + 64); for (uptr i = 0; i < hdr->stack0.size; i++) { stack[i] = hdr->stack0.trace[i]; DPrintf2(" #%02zu: pc=%zx\n", i, stack[i]); @@ -406,6 +399,8 @@ void RestoreStack(int tid, const u64 epoch, VarSizeStackTrace *stk, if (typ == EventTypeMop) { stack[pos] = pc; } else if (typ == EventTypeFuncEnter) { + if (stack.Size() < pos + 2) + stack.Resize(pos + 2); stack[pos++] = pc; } else if (typ == EventTypeFuncExit) { if (pos > 0) @@ -428,50 +423,58 @@ void RestoreStack(int tid, const u64 epoch, VarSizeStackTrace *stk, if (pos == 0 && stack[0] == 0) return; pos++; - stk->Init(stack.data(), pos); + stk->Init(&stack[0], pos); } static bool HandleRacyStacks(ThreadState *thr, VarSizeStackTrace traces[2], uptr addr_min, uptr addr_max) { bool equal_stack = false; RacyStacks hash; - if (flags()->suppress_equal_stacks) { - hash.hash[0] = md5_hash(traces[0].trace, traces[0].size * sizeof(uptr)); - hash.hash[1] = md5_hash(traces[1].trace, traces[1].size * sizeof(uptr)); - for (uptr i = 0; i < ctx->racy_stacks.Size(); i++) { - if (hash == ctx->racy_stacks[i]) { - DPrintf("ThreadSanitizer: suppressing report as doubled (stack)\n"); - equal_stack = true; - break; - } - } - } bool equal_address = false; RacyAddress ra0 = {addr_min, addr_max}; - if (flags()->suppress_equal_addresses) { - for (uptr i = 0; i < ctx->racy_addresses.Size(); i++) { - RacyAddress ra2 = ctx->racy_addresses[i]; - uptr maxbeg = max(ra0.addr_min, ra2.addr_min); - uptr minend = min(ra0.addr_max, ra2.addr_max); - if (maxbeg < minend) { - DPrintf("ThreadSanitizer: suppressing report as doubled (addr)\n"); - equal_address = true; - break; + { + ReadLock lock(&ctx->racy_mtx); + if (flags()->suppress_equal_stacks) { + hash.hash[0] = md5_hash(traces[0].trace, traces[0].size * sizeof(uptr)); + hash.hash[1] = md5_hash(traces[1].trace, traces[1].size * sizeof(uptr)); + for (uptr i = 0; i < ctx->racy_stacks.Size(); i++) { + if (hash == ctx->racy_stacks[i]) { + VPrintf(2, + "ThreadSanitizer: suppressing report as doubled (stack)\n"); + equal_stack = true; + break; + } + } + } + if (flags()->suppress_equal_addresses) { + for (uptr i = 0; i < ctx->racy_addresses.Size(); i++) { + RacyAddress ra2 = ctx->racy_addresses[i]; + uptr maxbeg = max(ra0.addr_min, ra2.addr_min); + uptr minend = min(ra0.addr_max, ra2.addr_max); + if (maxbeg < minend) { + VPrintf(2, "ThreadSanitizer: suppressing report as doubled (addr)\n"); + equal_address = true; + break; + } } } } - if (equal_stack || equal_address) { - if (!equal_stack) - ctx->racy_stacks.PushBack(hash); - if (!equal_address) - ctx->racy_addresses.PushBack(ra0); - return true; + if (!equal_stack && !equal_address) + return false; + if (!equal_stack) { + Lock lock(&ctx->racy_mtx); + ctx->racy_stacks.PushBack(hash); } - return false; + if (!equal_address) { + Lock lock(&ctx->racy_mtx); + ctx->racy_addresses.PushBack(ra0); + } + return true; } static void AddRacyStacks(ThreadState *thr, VarSizeStackTrace traces[2], uptr addr_min, uptr addr_max) { + Lock lock(&ctx->racy_mtx); if (flags()->suppress_equal_stacks) { RacyStacks hash; hash.hash[0] = md5_hash(traces[0].trace, traces[0].size * sizeof(uptr)); @@ -485,26 +488,29 @@ static void AddRacyStacks(ThreadState *thr, VarSizeStackTrace traces[2], } bool OutputReport(ThreadState *thr, const ScopedReport &srep) { - atomic_store(&ctx->last_symbolize_time_ns, NanoTime(), memory_order_relaxed); + if (!flags()->report_bugs) + return false; + atomic_store_relaxed(&ctx->last_symbolize_time_ns, NanoTime()); const ReportDesc *rep = srep.GetReport(); Suppression *supp = 0; - uptr suppress_pc = 0; - for (uptr i = 0; suppress_pc == 0 && i < rep->mops.Size(); i++) - suppress_pc = IsSuppressed(rep->typ, rep->mops[i]->stack, &supp); - for (uptr i = 0; suppress_pc == 0 && i < rep->stacks.Size(); i++) - suppress_pc = IsSuppressed(rep->typ, rep->stacks[i], &supp); - for (uptr i = 0; suppress_pc == 0 && i < rep->threads.Size(); i++) - suppress_pc = IsSuppressed(rep->typ, rep->threads[i]->stack, &supp); - for (uptr i = 0; suppress_pc == 0 && i < rep->locs.Size(); i++) - suppress_pc = IsSuppressed(rep->typ, rep->locs[i], &supp); - if (suppress_pc != 0) { - FiredSuppression s = {srep.GetReport()->typ, suppress_pc, supp}; + uptr pc_or_addr = 0; + for (uptr i = 0; pc_or_addr == 0 && i < rep->mops.Size(); i++) + pc_or_addr = IsSuppressed(rep->typ, rep->mops[i]->stack, &supp); + for (uptr i = 0; pc_or_addr == 0 && i < rep->stacks.Size(); i++) + pc_or_addr = IsSuppressed(rep->typ, rep->stacks[i], &supp); + for (uptr i = 0; pc_or_addr == 0 && i < rep->threads.Size(); i++) + pc_or_addr = IsSuppressed(rep->typ, rep->threads[i]->stack, &supp); + for (uptr i = 0; pc_or_addr == 0 && i < rep->locs.Size(); i++) + pc_or_addr = IsSuppressed(rep->typ, rep->locs[i], &supp); + if (pc_or_addr != 0) { + Lock lock(&ctx->fired_suppressions_mtx); + FiredSuppression s = {srep.GetReport()->typ, pc_or_addr, supp}; ctx->fired_suppressions.push_back(s); } { bool old_is_freeing = thr->is_freeing; thr->is_freeing = false; - bool suppressed = OnReport(rep, suppress_pc != 0); + bool suppressed = OnReport(rep, pc_or_addr != 0); thr->is_freeing = old_is_freeing; if (suppressed) return false; @@ -512,20 +518,20 @@ bool OutputReport(ThreadState *thr, const ScopedReport &srep) { PrintReport(rep); ctx->nreported++; if (flags()->halt_on_error) - internal__exit(flags()->exitcode); + Die(); return true; } -bool IsFiredSuppression(Context *ctx, const ScopedReport &srep, - StackTrace trace) { +bool IsFiredSuppression(Context *ctx, ReportType type, StackTrace trace) { + ReadLock lock(&ctx->fired_suppressions_mtx); for (uptr k = 0; k < ctx->fired_suppressions.size(); k++) { - if (ctx->fired_suppressions[k].type != srep.GetReport()->typ) + if (ctx->fired_suppressions[k].type != type) continue; for (uptr j = 0; j < trace.size; j++) { FiredSuppression *s = &ctx->fired_suppressions[k]; - if (trace.trace[j] == s->pc) { + if (trace.trace[j] == s->pc_or_addr) { if (s->supp) - s->supp->hit_count++; + atomic_fetch_add(&s->supp->hit_count, 1, memory_order_relaxed); return true; } } @@ -533,16 +539,15 @@ bool IsFiredSuppression(Context *ctx, const ScopedReport &srep, return false; } -static bool IsFiredSuppression(Context *ctx, - const ScopedReport &srep, - uptr addr) { +static bool IsFiredSuppression(Context *ctx, ReportType type, uptr addr) { + ReadLock lock(&ctx->fired_suppressions_mtx); for (uptr k = 0; k < ctx->fired_suppressions.size(); k++) { - if (ctx->fired_suppressions[k].type != srep.GetReport()->typ) + if (ctx->fired_suppressions[k].type != type) continue; FiredSuppression *s = &ctx->fired_suppressions[k]; - if (addr == s->pc) { + if (addr == s->pc_or_addr) { if (s->supp) - s->supp->hit_count++; + atomic_fetch_add(&s->supp->hit_count, 1, memory_order_relaxed); return true; } } @@ -595,8 +600,6 @@ void ReportRace(ThreadState *thr) { return; } - ThreadRegistryLock l0(ctx->thread_registry); - ReportType typ = ReportTypeRace; if (thr->is_vptr_access && freed) typ = ReportTypeVptrUseAfterFree; @@ -604,29 +607,35 @@ void ReportRace(ThreadState *thr) { typ = ReportTypeVptrRace; else if (freed) typ = ReportTypeUseAfterFree; - ScopedReport rep(typ); - if (IsFiredSuppression(ctx, rep, addr)) + + if (IsFiredSuppression(ctx, typ, addr)) return; + const uptr kMop = 2; VarSizeStackTrace traces[kMop]; const uptr toppc = TraceTopPC(thr); ObtainCurrentStack(thr, toppc, &traces[0]); - if (IsFiredSuppression(ctx, rep, traces[0])) + if (IsFiredSuppression(ctx, typ, traces[0])) return; - InternalScopedBuffer<MutexSet> mset2(1); - new(mset2.data()) MutexSet(); + + // MutexSet is too large to live on stack. + Vector<u64> mset_buffer(MBlockScopedBuf); + mset_buffer.Resize(sizeof(MutexSet) / sizeof(u64) + 1); + MutexSet *mset2 = new(&mset_buffer[0]) MutexSet(); + Shadow s2(thr->racy_state[1]); - RestoreStack(s2.tid(), s2.epoch(), &traces[1], mset2.data()); - if (IsFiredSuppression(ctx, rep, traces[1])) + RestoreStack(s2.tid(), s2.epoch(), &traces[1], mset2); + if (IsFiredSuppression(ctx, typ, traces[1])) return; if (HandleRacyStacks(thr, traces, addr_min, addr_max)) return; + ThreadRegistryLock l0(ctx->thread_registry); + ScopedReport rep(typ); for (uptr i = 0; i < kMop; i++) { Shadow s(thr->racy_state[i]); - rep.AddMemoryAccess(addr, s, traces[i], - i == 0 ? &thr->mset : mset2.data()); + rep.AddMemoryAccess(addr, s, traces[i], i == 0 ? &thr->mset : mset2); } for (uptr i = 0; i < kMop; i++) { diff --git a/lib/tsan/rtl/tsan_rtl_thread.cc b/lib/tsan/rtl/tsan_rtl_thread.cc index 66c78cfdd7c0..dcae255f7643 100644 --- a/lib/tsan/rtl/tsan_rtl_thread.cc +++ b/lib/tsan/rtl/tsan_rtl_thread.cc @@ -55,6 +55,8 @@ void ThreadContext::OnCreated(void *arg) { if (tid == 0) return; OnCreatedArgs *args = static_cast<OnCreatedArgs *>(arg); + if (!args->thr) // GCD workers don't have a parent thread. + return; args->thr->fast_state.IncrementEpoch(); // Can't increment epoch w/o writing to the trace as well. TraceAddEvent(args->thr, args->thr->fast_state, EventTypeMop, 0); @@ -231,8 +233,10 @@ int ThreadCount(ThreadState *thr) { int ThreadCreate(ThreadState *thr, uptr pc, uptr uid, bool detached) { StatInc(thr, StatThreadCreate); OnCreatedArgs args = { thr, pc }; - int tid = ctx->thread_registry->CreateThread(uid, detached, thr->tid, &args); - DPrintf("#%d: ThreadCreate tid=%d uid=%zu\n", thr->tid, tid, uid); + u32 parent_tid = thr ? thr->tid : kInvalidTid; // No parent for GCD workers. + int tid = + ctx->thread_registry->CreateThread(uid, detached, parent_tid, &args); + DPrintf("#%d: ThreadCreate tid=%d uid=%zu\n", parent_tid, tid, uid); StatSet(thr, StatThreadMaxAlive, ctx->thread_registry->GetMaxAliveThreads()); return tid; } diff --git a/lib/tsan/rtl/tsan_stat.cc b/lib/tsan/rtl/tsan_stat.cc index 15fa43d6f8a1..a5cca9679582 100644 --- a/lib/tsan/rtl/tsan_stat.cc +++ b/lib/tsan/rtl/tsan_stat.cc @@ -164,8 +164,9 @@ void StatOutput(u64 *stat) { name[StatMtxAtExit] = " Atexit "; name[StatMtxAnnotations] = " Annotations "; name[StatMtxMBlock] = " MBlock "; - name[StatMtxJavaMBlock] = " JavaMBlock "; name[StatMtxDeadlockDetector] = " DeadlockDetector "; + name[StatMtxFired] = " FiredSuppressions "; + name[StatMtxRacy] = " RacyStacks "; name[StatMtxFD] = " FD "; Printf("Statistics:\n"); diff --git a/lib/tsan/rtl/tsan_stat.h b/lib/tsan/rtl/tsan_stat.h index 0bd949ed1563..8ea32048e147 100644 --- a/lib/tsan/rtl/tsan_stat.h +++ b/lib/tsan/rtl/tsan_stat.h @@ -169,8 +169,9 @@ enum StatType { StatMtxAnnotations, StatMtxAtExit, StatMtxMBlock, - StatMtxJavaMBlock, StatMtxDeadlockDetector, + StatMtxFired, + StatMtxRacy, StatMtxFD, // This must be the last. diff --git a/lib/tsan/rtl/tsan_suppressions.cc b/lib/tsan/rtl/tsan_suppressions.cc index e382f21f0dff..8754b61c60cd 100644 --- a/lib/tsan/rtl/tsan_suppressions.cc +++ b/lib/tsan/rtl/tsan_suppressions.cc @@ -34,7 +34,8 @@ static const char *const std_suppressions = "race:std::_Sp_counted_ptr_inplace<std::thread::_Impl\n"; // Can be overriden in frontend. -extern "C" const char *WEAK __tsan_default_suppressions() { +SANITIZER_WEAK_DEFAULT_IMPL +const char *__tsan_default_suppressions() { return 0; } #endif @@ -100,8 +101,8 @@ static uptr IsSuppressed(const char *stype, const AddressInfo &info, if (suppression_ctx->Match(info.function, stype, sp) || suppression_ctx->Match(info.file, stype, sp) || suppression_ctx->Match(info.module, stype, sp)) { - DPrintf("ThreadSanitizer: matched suppression '%s'\n", (*sp)->templ); - (*sp)->hit_count++; + VPrintf(2, "ThreadSanitizer: matched suppression '%s'\n", (*sp)->templ); + atomic_fetch_add(&(*sp)->hit_count, 1, memory_order_relaxed); return info.address; } return 0; @@ -138,8 +139,8 @@ uptr IsSuppressed(ReportType typ, const ReportLocation *loc, Suppression **sp) { const DataInfo &global = loc->global; if (suppression_ctx->Match(global.name, stype, &s) || suppression_ctx->Match(global.module, stype, &s)) { - DPrintf("ThreadSanitizer: matched suppression '%s'\n", s->templ); - s->hit_count++; + VPrintf(2, "ThreadSanitizer: matched suppression '%s'\n", s->templ); + atomic_fetch_add(&s->hit_count, 1, memory_order_relaxed); *sp = s; return global.start; } @@ -154,7 +155,7 @@ void PrintMatchedSuppressions() { return; int hit_count = 0; for (uptr i = 0; i < matched.size(); i++) - hit_count += matched[i]->hit_count; + hit_count += atomic_load_relaxed(&matched[i]->hit_count); Printf("ThreadSanitizer: Matched %d suppressions (pid=%d):\n", hit_count, (int)internal_getpid()); for (uptr i = 0; i < matched.size(); i++) { diff --git a/lib/tsan/rtl/tsan_symbolize.cc b/lib/tsan/rtl/tsan_symbolize.cc index a6b9bca0501d..b2423951795f 100644 --- a/lib/tsan/rtl/tsan_symbolize.cc +++ b/lib/tsan/rtl/tsan_symbolize.cc @@ -38,10 +38,10 @@ void ExitSymbolizer() { // May be overriden by JIT/JAVA/etc, // whatever produces PCs marked with kExternalPCBit. -extern "C" bool WEAK __tsan_symbolize_external(uptr pc, - char *func_buf, uptr func_siz, - char *file_buf, uptr file_siz, - int *line, int *col) { +SANITIZER_WEAK_DEFAULT_IMPL +bool __tsan_symbolize_external(uptr pc, char *func_buf, uptr func_siz, + char *file_buf, uptr file_siz, int *line, + int *col) { return false; } @@ -71,7 +71,7 @@ ReportLocation *SymbolizeData(uptr addr) { if (!Symbolizer::GetOrInit()->SymbolizeData(addr, &info)) return 0; ReportLocation *ent = ReportLocation::New(ReportLocationGlobal); - ent->global = info; + internal_memcpy(&ent->global, &info, sizeof(info)); return ent; } diff --git a/lib/tsan/rtl/tsan_sync.h b/lib/tsan/rtl/tsan_sync.h index 2d12cdff8b2f..f07ea3b9776b 100644 --- a/lib/tsan/rtl/tsan_sync.h +++ b/lib/tsan/rtl/tsan_sync.h @@ -86,9 +86,9 @@ class MetaMap { void OnThreadIdle(ThreadState *thr); private: - static const u32 kFlagMask = 3 << 30; - static const u32 kFlagBlock = 1 << 30; - static const u32 kFlagSync = 2 << 30; + static const u32 kFlagMask = 3u << 30; + static const u32 kFlagBlock = 1u << 30; + static const u32 kFlagSync = 2u << 30; typedef DenseSlabAlloc<MBlock, 1<<16, 1<<12> BlockAlloc; typedef DenseSlabAlloc<SyncVar, 1<<16, 1<<10> SyncAlloc; BlockAlloc block_alloc_; diff --git a/lib/tsan/tests/CMakeLists.txt b/lib/tsan/tests/CMakeLists.txt index 1c3f98f3f0bc..51181bab3a79 100644 --- a/lib/tsan/tests/CMakeLists.txt +++ b/lib/tsan/tests/CMakeLists.txt @@ -33,9 +33,12 @@ macro(tsan_compile obj_list source arch) endmacro() macro(add_tsan_unittest testname) - # Build unit tests only for 64-bit Linux. - if(UNIX AND NOT APPLE) - foreach(arch ${TSAN_SUPPORTED_ARCH}) + set(TSAN_TEST_ARCH ${TSAN_SUPPORTED_ARCH}) + if(APPLE) + darwin_filter_host_archs(TSAN_SUPPORTED_ARCH TSAN_TEST_ARCH) + endif() + if(UNIX) + foreach(arch ${TSAN_TEST_ARCH}) cmake_parse_arguments(TEST "" "" "SOURCES;HEADERS" ${ARGN}) set(TEST_OBJECTS) foreach(SOURCE ${TEST_SOURCES} ${COMPILER_RT_GTEST_SOURCE}) @@ -46,15 +49,38 @@ macro(add_tsan_unittest testname) if(NOT COMPILER_RT_STANDALONE_BUILD) list(APPEND TEST_DEPS tsan) endif() - # FIXME: Looks like we should link TSan with just-built runtime, - # and not rely on -fsanitize=thread, as these tests are essentially - # unit tests. - add_compiler_rt_test(TsanUnitTests ${testname} - OBJECTS ${TEST_OBJECTS} - DEPS ${TEST_DEPS} - LINK_FLAGS ${TARGET_LINK_FLAGS} - -fsanitize=thread - -lstdc++ -lm) + if(NOT APPLE) + # FIXME: Looks like we should link TSan with just-built runtime, + # and not rely on -fsanitize=thread, as these tests are essentially + # unit tests. + add_compiler_rt_test(TsanUnitTests ${testname} + OBJECTS ${TEST_OBJECTS} + DEPS ${TEST_DEPS} + LINK_FLAGS ${TARGET_LINK_FLAGS} + -fsanitize=thread + -lstdc++ -lm) + else() + set(TSAN_TEST_RUNTIME_OBJECTS + $<TARGET_OBJECTS:RTTsan_dynamic.osx> + $<TARGET_OBJECTS:RTInterception.osx> + $<TARGET_OBJECTS:RTSanitizerCommon.osx> + $<TARGET_OBJECTS:RTSanitizerCommonLibc.osx> + $<TARGET_OBJECTS:RTUbsan.osx>) + set(TSAN_TEST_RUNTIME RTTsanTest.${testname}.${arch}) + add_library(${TSAN_TEST_RUNTIME} STATIC ${TSAN_TEST_RUNTIME_OBJECTS}) + set_target_properties(${TSAN_TEST_RUNTIME} PROPERTIES + ARCHIVE_OUTPUT_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}) + list(APPEND TEST_OBJECTS lib${TSAN_TEST_RUNTIME}.a) + list(APPEND TEST_DEPS ${TSAN_TEST_RUNTIME}) + # Intentionally do *not* link with `-fsanitize=thread`. We already link + # against a static version of the runtime, and we don't want the dynamic + # one. + add_compiler_rt_test(TsanUnitTests "${testname}-${arch}-Test" + OBJECTS ${TEST_OBJECTS} + DEPS ${TEST_DEPS} + LINK_FLAGS ${TARGET_LINK_FLAGS} + -lc++) + endif() endforeach() endif() endmacro() diff --git a/lib/tsan/tests/rtl/CMakeLists.txt b/lib/tsan/tests/rtl/CMakeLists.txt index 989566d9e041..a34f08ea965b 100644 --- a/lib/tsan/tests/rtl/CMakeLists.txt +++ b/lib/tsan/tests/rtl/CMakeLists.txt @@ -7,8 +7,8 @@ set(TSAN_RTL_TEST_SOURCES tsan_test.cc tsan_thread.cc) -if(UNIX AND NOT APPLE) - list(APPEND TSAN_RTL_TEST_SOURCES tsan_test_util_linux.cc) +if(UNIX) + list(APPEND TSAN_RTL_TEST_SOURCES tsan_test_util_posix.cc) endif() set(TSAN_RTL_TEST_HEADERS diff --git a/lib/tsan/tests/rtl/tsan_posix.cc b/lib/tsan/tests/rtl/tsan_posix.cc index 0caedd7207e6..e1a61b5e43f1 100644 --- a/lib/tsan/tests/rtl/tsan_posix.cc +++ b/lib/tsan/tests/rtl/tsan_posix.cc @@ -35,7 +35,7 @@ static void thread_secific_dtor(void *v) { __tsan_write4(&k->cnt); EXPECT_EQ(pthread_mutex_unlock(k->mtx), 0); if (k->val == 42) { - delete k; + // Okay. } else if (k->val == 43 || k->val == 44) { k->val--; EXPECT_EQ(pthread_setspecific(k->key, k), 0); @@ -57,20 +57,20 @@ TEST(Posix, ThreadSpecificDtors) { pthread_mutex_t mtx; EXPECT_EQ(pthread_mutex_init(&mtx, 0), 0); pthread_t th[3]; - thread_key *k[3]; - k[0] = new thread_key(key, &mtx, 42, &cnt); - k[1] = new thread_key(key, &mtx, 43, &cnt); - k[2] = new thread_key(key, &mtx, 44, &cnt); - EXPECT_EQ(pthread_create(&th[0], 0, dtors_thread, k[0]), 0); - EXPECT_EQ(pthread_create(&th[1], 0, dtors_thread, k[1]), 0); + thread_key k1 = thread_key(key, &mtx, 42, &cnt); + thread_key k2 = thread_key(key, &mtx, 43, &cnt); + thread_key k3 = thread_key(key, &mtx, 44, &cnt); + EXPECT_EQ(pthread_create(&th[0], 0, dtors_thread, &k1), 0); + EXPECT_EQ(pthread_create(&th[1], 0, dtors_thread, &k2), 0); EXPECT_EQ(pthread_join(th[0], 0), 0); - EXPECT_EQ(pthread_create(&th[2], 0, dtors_thread, k[2]), 0); + EXPECT_EQ(pthread_create(&th[2], 0, dtors_thread, &k3), 0); EXPECT_EQ(pthread_join(th[1], 0), 0); EXPECT_EQ(pthread_join(th[2], 0), 0); EXPECT_EQ(pthread_key_delete(key), 0); EXPECT_EQ(6, cnt); } +#ifndef __aarch64__ static __thread int local_var; static void *local_thread(void *p) { @@ -87,9 +87,14 @@ static void *local_thread(void *p) { EXPECT_EQ(pthread_join(th[i], 0), 0); return 0; } +#endif TEST(Posix, ThreadLocalAccesses) { +// The test is failing with high thread count for aarch64. +// FIXME: track down the issue and re-enable the test. +#ifndef __aarch64__ local_thread((void*)2); +#endif } struct CondContext { diff --git a/lib/tsan/tests/rtl/tsan_test.cc b/lib/tsan/tests/rtl/tsan_test.cc index b8b9555c2bff..edfede078b68 100644 --- a/lib/tsan/tests/rtl/tsan_test.cc +++ b/lib/tsan/tests/rtl/tsan_test.cc @@ -47,6 +47,13 @@ int run_tests(int argc, char **argv) { const char *argv0; +#ifdef __APPLE__ +// On Darwin, turns off symbolication and crash logs to make tests faster. +extern "C" const char* __tsan_default_options() { + return "symbolize=false:abort_on_error=0"; +} +#endif + int main(int argc, char **argv) { argv0 = argv[0]; return run_tests(argc, argv); diff --git a/lib/tsan/tests/rtl/tsan_test_util.h b/lib/tsan/tests/rtl/tsan_test_util.h index 84d277b137f0..31b1b188f624 100644 --- a/lib/tsan/tests/rtl/tsan_test_util.h +++ b/lib/tsan/tests/rtl/tsan_test_util.h @@ -31,7 +31,15 @@ class MemLoc { class Mutex { public: - enum Type { Normal, Spin, RW }; + enum Type { + Normal, + RW, +#ifndef __APPLE__ + Spin +#else + Spin = Normal +#endif + }; explicit Mutex(Type type = Normal); ~Mutex(); diff --git a/lib/tsan/tests/rtl/tsan_test_util_linux.cc b/lib/tsan/tests/rtl/tsan_test_util_posix.cc index 9298bf051af2..c8be088d266f 100644 --- a/lib/tsan/tests/rtl/tsan_test_util_linux.cc +++ b/lib/tsan/tests/rtl/tsan_test_util_posix.cc @@ -1,5 +1,4 @@ - -//===-- tsan_test_util_linux.cc -------------------------------------------===// +//===-- tsan_test_util_posix.cc -------------------------------------------===// // // The LLVM Compiler Infrastructure // @@ -10,7 +9,7 @@ // // This file is a part of ThreadSanitizer (TSan), a race detector. // -// Test utils, Linux and FreeBSD implementation. +// Test utils, Linux, FreeBSD and Darwin implementation. //===----------------------------------------------------------------------===// #include "sanitizer_common/sanitizer_atomic.h" @@ -34,8 +33,51 @@ static __thread bool expect_report; static __thread bool expect_report_reported; static __thread ReportType expect_report_type; -extern "C" void *__interceptor_memcpy(void*, const void*, uptr); -extern "C" void *__interceptor_memset(void*, int, uptr); +#ifdef __APPLE__ +#define __interceptor_memcpy wrap_memcpy +#define __interceptor_memset wrap_memset +#define __interceptor_pthread_create wrap_pthread_create +#define __interceptor_pthread_join wrap_pthread_join +#define __interceptor_pthread_detach wrap_pthread_detach +#define __interceptor_pthread_mutex_init wrap_pthread_mutex_init +#define __interceptor_pthread_mutex_lock wrap_pthread_mutex_lock +#define __interceptor_pthread_mutex_unlock wrap_pthread_mutex_unlock +#define __interceptor_pthread_mutex_destroy wrap_pthread_mutex_destroy +#define __interceptor_pthread_mutex_trylock wrap_pthread_mutex_trylock +#define __interceptor_pthread_rwlock_init wrap_pthread_rwlock_init +#define __interceptor_pthread_rwlock_destroy wrap_pthread_rwlock_destroy +#define __interceptor_pthread_rwlock_trywrlock wrap_pthread_rwlock_trywrlock +#define __interceptor_pthread_rwlock_wrlock wrap_pthread_rwlock_wrlock +#define __interceptor_pthread_rwlock_unlock wrap_pthread_rwlock_unlock +#define __interceptor_pthread_rwlock_rdlock wrap_pthread_rwlock_rdlock +#define __interceptor_pthread_rwlock_tryrdlock wrap_pthread_rwlock_tryrdlock +#endif + +extern "C" void *__interceptor_memcpy(void *, const void *, uptr); +extern "C" void *__interceptor_memset(void *, int, uptr); +extern "C" int __interceptor_pthread_create(pthread_t *thread, + const pthread_attr_t *attr, + void *(*start_routine)(void *), + void *arg); +extern "C" int __interceptor_pthread_join(pthread_t thread, void **value_ptr); +extern "C" int __interceptor_pthread_detach(pthread_t thread); + +extern "C" int __interceptor_pthread_mutex_init( + pthread_mutex_t *mutex, const pthread_mutexattr_t *attr); +extern "C" int __interceptor_pthread_mutex_lock(pthread_mutex_t *mutex); +extern "C" int __interceptor_pthread_mutex_unlock(pthread_mutex_t *mutex); +extern "C" int __interceptor_pthread_mutex_destroy(pthread_mutex_t *mutex); +extern "C" int __interceptor_pthread_mutex_trylock(pthread_mutex_t *mutex); + +extern "C" int __interceptor_pthread_rwlock_init( + pthread_rwlock_t *rwlock, const pthread_rwlockattr_t *attr); +extern "C" int __interceptor_pthread_rwlock_destroy(pthread_rwlock_t *rwlock); +extern "C" int __interceptor_pthread_rwlock_trywrlock(pthread_rwlock_t *rwlock); +extern "C" int __interceptor_pthread_rwlock_wrlock(pthread_rwlock_t *rwlock); +extern "C" int __interceptor_pthread_rwlock_unlock(pthread_rwlock_t *rwlock); +extern "C" int __interceptor_pthread_rwlock_rdlock(pthread_rwlock_t *rwlock); +extern "C" int __interceptor_pthread_rwlock_tryrdlock(pthread_rwlock_t *rwlock); + static void *BeforeInitThread(void *param) { (void)param; @@ -48,12 +90,12 @@ static void AtExit() { void TestMutexBeforeInit() { // Mutexes must be usable before __tsan_init(); pthread_mutex_t mtx = PTHREAD_MUTEX_INITIALIZER; - pthread_mutex_lock(&mtx); - pthread_mutex_unlock(&mtx); - pthread_mutex_destroy(&mtx); + __interceptor_pthread_mutex_lock(&mtx); + __interceptor_pthread_mutex_unlock(&mtx); + __interceptor_pthread_mutex_destroy(&mtx); pthread_t thr; - pthread_create(&thr, 0, BeforeInitThread, 0); - pthread_join(thr, 0); + __interceptor_pthread_create(&thr, 0, BeforeInitThread, 0); + __interceptor_pthread_join(thr, 0); atexit(AtExit); } @@ -105,11 +147,13 @@ void Mutex::Init() { CHECK(!alive_); alive_ = true; if (type_ == Normal) - CHECK_EQ(pthread_mutex_init((pthread_mutex_t*)mtx_, 0), 0); + CHECK_EQ(__interceptor_pthread_mutex_init((pthread_mutex_t*)mtx_, 0), 0); +#ifndef __APPLE__ else if (type_ == Spin) CHECK_EQ(pthread_spin_init((pthread_spinlock_t*)mtx_, 0), 0); +#endif else if (type_ == RW) - CHECK_EQ(pthread_rwlock_init((pthread_rwlock_t*)mtx_, 0), 0); + CHECK_EQ(__interceptor_pthread_rwlock_init((pthread_rwlock_t*)mtx_, 0), 0); else CHECK(0); } @@ -126,60 +170,68 @@ void Mutex::Destroy() { CHECK(alive_); alive_ = false; if (type_ == Normal) - CHECK_EQ(pthread_mutex_destroy((pthread_mutex_t*)mtx_), 0); + CHECK_EQ(__interceptor_pthread_mutex_destroy((pthread_mutex_t*)mtx_), 0); +#ifndef __APPLE__ else if (type_ == Spin) CHECK_EQ(pthread_spin_destroy((pthread_spinlock_t*)mtx_), 0); +#endif else if (type_ == RW) - CHECK_EQ(pthread_rwlock_destroy((pthread_rwlock_t*)mtx_), 0); + CHECK_EQ(__interceptor_pthread_rwlock_destroy((pthread_rwlock_t*)mtx_), 0); } void Mutex::Lock() { CHECK(alive_); if (type_ == Normal) - CHECK_EQ(pthread_mutex_lock((pthread_mutex_t*)mtx_), 0); + CHECK_EQ(__interceptor_pthread_mutex_lock((pthread_mutex_t*)mtx_), 0); +#ifndef __APPLE__ else if (type_ == Spin) CHECK_EQ(pthread_spin_lock((pthread_spinlock_t*)mtx_), 0); +#endif else if (type_ == RW) - CHECK_EQ(pthread_rwlock_wrlock((pthread_rwlock_t*)mtx_), 0); + CHECK_EQ(__interceptor_pthread_rwlock_wrlock((pthread_rwlock_t*)mtx_), 0); } bool Mutex::TryLock() { CHECK(alive_); if (type_ == Normal) - return pthread_mutex_trylock((pthread_mutex_t*)mtx_) == 0; + return __interceptor_pthread_mutex_trylock((pthread_mutex_t*)mtx_) == 0; +#ifndef __APPLE__ else if (type_ == Spin) return pthread_spin_trylock((pthread_spinlock_t*)mtx_) == 0; +#endif else if (type_ == RW) - return pthread_rwlock_trywrlock((pthread_rwlock_t*)mtx_) == 0; + return __interceptor_pthread_rwlock_trywrlock((pthread_rwlock_t*)mtx_) == 0; return false; } void Mutex::Unlock() { CHECK(alive_); if (type_ == Normal) - CHECK_EQ(pthread_mutex_unlock((pthread_mutex_t*)mtx_), 0); + CHECK_EQ(__interceptor_pthread_mutex_unlock((pthread_mutex_t*)mtx_), 0); +#ifndef __APPLE__ else if (type_ == Spin) CHECK_EQ(pthread_spin_unlock((pthread_spinlock_t*)mtx_), 0); +#endif else if (type_ == RW) - CHECK_EQ(pthread_rwlock_unlock((pthread_rwlock_t*)mtx_), 0); + CHECK_EQ(__interceptor_pthread_rwlock_unlock((pthread_rwlock_t*)mtx_), 0); } void Mutex::ReadLock() { CHECK(alive_); CHECK(type_ == RW); - CHECK_EQ(pthread_rwlock_rdlock((pthread_rwlock_t*)mtx_), 0); + CHECK_EQ(__interceptor_pthread_rwlock_rdlock((pthread_rwlock_t*)mtx_), 0); } bool Mutex::TryReadLock() { CHECK(alive_); CHECK(type_ == RW); - return pthread_rwlock_tryrdlock((pthread_rwlock_t*)mtx_) == 0; + return __interceptor_pthread_rwlock_tryrdlock((pthread_rwlock_t*)mtx_) == 0; } void Mutex::ReadUnlock() { CHECK(alive_); CHECK(type_ == RW); - CHECK_EQ(pthread_rwlock_unlock((pthread_rwlock_t*)mtx_), 0); + CHECK_EQ(__interceptor_pthread_rwlock_unlock((pthread_rwlock_t*)mtx_), 0); } struct Event { @@ -263,7 +315,7 @@ void ScopedThread::Impl::HandleEvent(Event *ev) { } } CHECK_NE(tsan_mop, 0); -#if defined(__FreeBSD__) +#if defined(__FreeBSD__) || defined(__APPLE__) const int ErrCode = ESOCKTNOSUPPORT; #else const int ErrCode = ECHRNG; @@ -327,7 +379,7 @@ void *ScopedThread::Impl::ScopedThreadCallback(void *arg) { for (;;) { Event* ev = (Event*)atomic_load(&impl->event, memory_order_acquire); if (ev == 0) { - pthread_yield(); + sched_yield(); continue; } if (ev->type == Event::SHUTDOWN) { @@ -348,7 +400,7 @@ void ScopedThread::Impl::send(Event *e) { CHECK_EQ(atomic_load(&event, memory_order_relaxed), 0); atomic_store(&event, (uintptr_t)e, memory_order_release); while (atomic_load(&event, memory_order_acquire) != 0) - pthread_yield(); + sched_yield(); } } @@ -360,9 +412,10 @@ ScopedThread::ScopedThread(bool detached, bool main) { if (!main) { pthread_attr_t attr; pthread_attr_init(&attr); - pthread_attr_setdetachstate(&attr, detached); + pthread_attr_setdetachstate( + &attr, detached ? PTHREAD_CREATE_DETACHED : PTHREAD_CREATE_JOINABLE); pthread_attr_setstacksize(&attr, 64*1024); - pthread_create(&impl_->thread, &attr, + __interceptor_pthread_create(&impl_->thread, &attr, ScopedThread::Impl::ScopedThreadCallback, impl_); } } @@ -372,7 +425,7 @@ ScopedThread::~ScopedThread() { Event event(Event::SHUTDOWN); impl_->send(&event); if (!impl_->detached) - pthread_join(impl_->thread, 0); + __interceptor_pthread_join(impl_->thread, 0); } delete impl_; } @@ -381,7 +434,7 @@ void ScopedThread::Detach() { CHECK(!impl_->main); CHECK(!impl_->detached); impl_->detached = true; - pthread_detach(impl_->thread); + __interceptor_pthread_detach(impl_->thread); } void ScopedThread::Access(void *addr, bool is_write, diff --git a/lib/tsan/tests/unit/tsan_clock_test.cc b/lib/tsan/tests/unit/tsan_clock_test.cc index 92071827d3d8..83e25fb5a933 100644 --- a/lib/tsan/tests/unit/tsan_clock_test.cc +++ b/lib/tsan/tests/unit/tsan_clock_test.cc @@ -13,6 +13,7 @@ #include "tsan_clock.h" #include "tsan_rtl.h" #include "gtest/gtest.h" +#include <sys/time.h> #include <time.h> namespace __tsan { @@ -416,9 +417,9 @@ static bool ClockFuzzer(bool printing) { } TEST(Clock, Fuzzer) { - timespec ts; - clock_gettime(CLOCK_MONOTONIC, &ts); - int seed = ts.tv_sec + ts.tv_nsec; + struct timeval tv; + gettimeofday(&tv, NULL); + int seed = tv.tv_sec + tv.tv_usec; printf("seed=%d\n", seed); srand(seed); if (!ClockFuzzer(false)) { diff --git a/lib/tsan/tests/unit/tsan_flags_test.cc b/lib/tsan/tests/unit/tsan_flags_test.cc index 22610c0dc42f..aa8a024f9dc5 100644 --- a/lib/tsan/tests/unit/tsan_flags_test.cc +++ b/lib/tsan/tests/unit/tsan_flags_test.cc @@ -28,9 +28,7 @@ TEST(Flags, DefaultValues) { Flags f; f.enable_annotations = false; - f.exitcode = -11; InitializeFlags(&f, ""); - EXPECT_EQ(66, f.exitcode); EXPECT_EQ(true, f.enable_annotations); } @@ -46,7 +44,6 @@ static const char *options1 = " report_atomic_races=0" " force_seq_cst_atomics=0" " print_benign=0" - " exitcode=111" " halt_on_error=0" " atexit_sleep_ms=222" " profile_memory=qqq" @@ -72,7 +69,6 @@ static const char *options2 = " report_atomic_races=true" " force_seq_cst_atomics=true" " print_benign=true" - " exitcode=222" " halt_on_error=true" " atexit_sleep_ms=123" " profile_memory=bbbbb" @@ -98,7 +94,6 @@ void VerifyOptions1(Flags *f) { EXPECT_EQ(f->report_atomic_races, 0); EXPECT_EQ(f->force_seq_cst_atomics, 0); EXPECT_EQ(f->print_benign, 0); - EXPECT_EQ(f->exitcode, 111); EXPECT_EQ(f->halt_on_error, 0); EXPECT_EQ(f->atexit_sleep_ms, 222); EXPECT_EQ(f->profile_memory, std::string("qqq")); @@ -124,7 +119,6 @@ void VerifyOptions2(Flags *f) { EXPECT_EQ(f->report_atomic_races, true); EXPECT_EQ(f->force_seq_cst_atomics, true); EXPECT_EQ(f->print_benign, true); - EXPECT_EQ(f->exitcode, 222); EXPECT_EQ(f->halt_on_error, true); EXPECT_EQ(f->atexit_sleep_ms, 123); EXPECT_EQ(f->profile_memory, std::string("bbbbb")); diff --git a/lib/tsan/tests/unit/tsan_mman_test.cc b/lib/tsan/tests/unit/tsan_mman_test.cc index bfaefe648705..609141c59294 100644 --- a/lib/tsan/tests/unit/tsan_mman_test.cc +++ b/lib/tsan/tests/unit/tsan_mman_test.cc @@ -141,11 +141,13 @@ TEST(Mman, CallocOverflow) { // which is overflown by tsan memory accesses functions in debug mode. return; #endif + ThreadState *thr = cur_thread(); + uptr pc = 0; size_t kArraySize = 4096; volatile size_t kMaxSizeT = std::numeric_limits<size_t>::max(); volatile size_t kArraySize2 = kMaxSizeT / kArraySize + 10; volatile void *p = NULL; - EXPECT_DEATH(p = calloc(kArraySize, kArraySize2), + EXPECT_DEATH(p = user_calloc(thr, pc, kArraySize, kArraySize2), "allocator is terminating the process instead of returning 0"); EXPECT_EQ(0L, p); } diff --git a/lib/ubsan/CMakeLists.txt b/lib/ubsan/CMakeLists.txt index 246236b78ab5..5ece9a62cfeb 100644 --- a/lib/ubsan/CMakeLists.txt +++ b/lib/ubsan/CMakeLists.txt @@ -49,15 +49,16 @@ if(APPLE) ARCHS ${UBSAN_SUPPORTED_ARCH} SOURCES ${UBSAN_STANDALONE_SOURCES} CFLAGS ${UBSAN_STANDALONE_CFLAGS}) - foreach(os ${SANITIZER_COMMON_SUPPORTED_OS}) - add_compiler_rt_darwin_dynamic_runtime(clang_rt.ubsan_${os}_dynamic ${os} - ARCHS ${UBSAN_SUPPORTED_ARCH} - SOURCES $<TARGET_OBJECTS:RTUbsan.${os}> - $<TARGET_OBJECTS:RTUbsan_standalone.${os}> - $<TARGET_OBJECTS:RTSanitizerCommon.${os}>) - add_dependencies(ubsan clang_rt.ubsan_${os}_dynamic) - endforeach() + add_compiler_rt_runtime(clang_rt.ubsan + SHARED + OS ${SANITIZER_COMMON_SUPPORTED_OS} + ARCHS ${UBSAN_SUPPORTED_ARCH} + OBJECT_LIBS RTUbsan + RTUbsan_standalone + RTSanitizerCommon + RTSanitizerCommonLibc + PARENT_TARGET ubsan) endif() else() @@ -76,29 +77,36 @@ else() ARCHS ${UBSAN_SUPPORTED_ARCH} SOURCES ${UBSAN_STANDALONE_SOURCES} CFLAGS ${UBSAN_STANDALONE_CFLAGS}) - foreach(arch ${UBSAN_SUPPORTED_ARCH}) - # Standalone UBSan runtimes. - add_compiler_rt_runtime(clang_rt.ubsan_standalone-${arch} ${arch} STATIC - SOURCES $<TARGET_OBJECTS:RTSanitizerCommon.${arch}> - $<TARGET_OBJECTS:RTSanitizerCommonLibc.${arch}> - $<TARGET_OBJECTS:RTUbsan.${arch}> - $<TARGET_OBJECTS:RTUbsan_standalone.${arch}> - CFLAGS ${UBSAN_CFLAGS}) - add_compiler_rt_runtime(clang_rt.ubsan_standalone_cxx-${arch} ${arch} STATIC - SOURCES $<TARGET_OBJECTS:RTUbsan_cxx.${arch}> - CFLAGS ${UBSAN_CXXFLAGS}) + # Standalone UBSan runtimes. + add_compiler_rt_runtime(clang_rt.ubsan_standalone + STATIC + ARCHS ${UBSAN_SUPPORTED_ARCH} + OBJECT_LIBS RTSanitizerCommon + RTSanitizerCommonLibc + RTUbsan + RTUbsan_standalone + CFLAGS ${UBSAN_CFLAGS} + PARENT_TARGET ubsan) + + add_compiler_rt_runtime(clang_rt.ubsan_standalone_cxx + STATIC + ARCHS ${UBSAN_SUPPORTED_ARCH} + OBJECT_LIBS RTUbsan_cxx + CFLAGS ${UBSAN_CXXFLAGS} + PARENT_TARGET ubsan) - add_dependencies(ubsan - clang_rt.ubsan_standalone-${arch} - clang_rt.ubsan_standalone_cxx-${arch}) - if (UNIX AND NOT ${arch} MATCHES "i386|i686") - add_sanitizer_rt_symbols(clang_rt.ubsan_standalone-${arch} ubsan.syms.extra) - add_sanitizer_rt_symbols(clang_rt.ubsan_standalone_cxx-${arch} ubsan.syms.extra) - add_dependencies(ubsan - clang_rt.ubsan_standalone-${arch}-symbols - clang_rt.ubsan_standalone_cxx-${arch}-symbols) - endif() - endforeach() + if (UNIX) + set(ARCHS_FOR_SYMBOLS ${UBSAN_SUPPORTED_ARCH}) + list(REMOVE_ITEM ARCHS_FOR_SYMBOLS i386 i686) + add_sanitizer_rt_symbols(clang_rt.ubsan_standalone + ARCHS ${ARCHS_FOR_SYMBOLS} + PARENT_TARGET ubsan + EXTRA ubsan.syms.extra) + add_sanitizer_rt_symbols(clang_rt.ubsan_standalone_cxx + ARCHS ${ARCHS_FOR_SYMBOLS} + PARENT_TARGET ubsan + EXTRA ubsan.syms.extra) + endif() endif() endif() diff --git a/lib/ubsan/ubsan_checks.inc b/lib/ubsan/ubsan_checks.inc new file mode 100644 index 000000000000..6e086414051e --- /dev/null +++ b/lib/ubsan/ubsan_checks.inc @@ -0,0 +1,45 @@ +//===-- ubsan_checks.inc ----------------------------------------*- C++ -*-===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// +// +// List of checks handled by UBSan runtime. +// +//===----------------------------------------------------------------------===// +#ifndef UBSAN_CHECK +# error "Define UBSAN_CHECK prior to including this file!" +#endif + +// UBSAN_CHECK(Name, SummaryKind, FSanitizeFlagName) +// SummaryKind and FSanitizeFlagName should be string literals. + +UBSAN_CHECK(GenericUB, "undefined-behavior", "undefined") +UBSAN_CHECK(NullPointerUse, "null-pointer-use", "null") +UBSAN_CHECK(MisalignedPointerUse, "misaligned-pointer-use", "alignment") +UBSAN_CHECK(InsufficientObjectSize, "insufficient-object-size", "object-size") +UBSAN_CHECK(SignedIntegerOverflow, "signed-integer-overflow", + "signed-integer-overflow") +UBSAN_CHECK(UnsignedIntegerOverflow, "unsigned-integer-overflow", + "unsigned-integer-overflow") +UBSAN_CHECK(IntegerDivideByZero, "integer-divide-by-zero", + "integer-divide-by-zero") +UBSAN_CHECK(FloatDivideByZero, "float-divide-by-zero", "float-divide-by-zero") +UBSAN_CHECK(InvalidShiftBase, "invalid-shift-base", "shift-base") +UBSAN_CHECK(InvalidShiftExponent, "invalid-shift-exponent", "shift-exponent") +UBSAN_CHECK(OutOfBoundsIndex, "out-of-bounds-index", "bounds") +UBSAN_CHECK(UnreachableCall, "unreachable-call", "unreachable") +UBSAN_CHECK(MissingReturn, "missing-return", "return") +UBSAN_CHECK(NonPositiveVLAIndex, "non-positive-vla-index", "vla-bound") +UBSAN_CHECK(FloatCastOverflow, "float-cast-overflow", "float-cast-overflow") +UBSAN_CHECK(InvalidBoolLoad, "invalid-bool-load", "bool") +UBSAN_CHECK(InvalidEnumLoad, "invalid-enum-load", "enum") +UBSAN_CHECK(FunctionTypeMismatch, "function-type-mismatch", "function") +UBSAN_CHECK(InvalidNullReturn, "invalid-null-return", + "returns-nonnull-attribute") +UBSAN_CHECK(InvalidNullArgument, "invalid-null-argument", "nonnull-attribute") +UBSAN_CHECK(DynamicTypeMismatch, "dynamic-type-mismatch", "vptr") +UBSAN_CHECK(CFIBadType, "cfi-bad-type", "cfi") diff --git a/lib/ubsan/ubsan_diag.cc b/lib/ubsan/ubsan_diag.cc index 3d5042b51d2e..2476947dc914 100644 --- a/lib/ubsan/ubsan_diag.cc +++ b/lib/ubsan/ubsan_diag.cc @@ -43,10 +43,34 @@ static void MaybePrintStackTrace(uptr pc, uptr bp) { stack.Print(); } -static void MaybeReportErrorSummary(Location Loc) { +static const char *ConvertTypeToString(ErrorType Type) { + switch (Type) { +#define UBSAN_CHECK(Name, SummaryKind, FSanitizeFlagName) \ + case ErrorType::Name: \ + return SummaryKind; +#include "ubsan_checks.inc" +#undef UBSAN_CHECK + } + UNREACHABLE("unknown ErrorType!"); +} + +static const char *ConvertTypeToFlagName(ErrorType Type) { + switch (Type) { +#define UBSAN_CHECK(Name, SummaryKind, FSanitizeFlagName) \ + case ErrorType::Name: \ + return FSanitizeFlagName; +#include "ubsan_checks.inc" +#undef UBSAN_CHECK + } + UNREACHABLE("unknown ErrorType!"); +} + +static void MaybeReportErrorSummary(Location Loc, ErrorType Type) { if (!common_flags()->print_summary) return; - const char *ErrorType = "undefined-behavior"; + if (!flags()->report_error_type) + Type = ErrorType::GenericUB; + const char *ErrorKind = ConvertTypeToString(Type); if (Loc.isSourceLocation()) { SourceLocation SLoc = Loc.getSourceLocation(); if (!SLoc.isInvalid()) { @@ -55,16 +79,16 @@ static void MaybeReportErrorSummary(Location Loc) { AI.line = SLoc.getLine(); AI.column = SLoc.getColumn(); AI.function = internal_strdup(""); // Avoid printing ?? as function name. - ReportErrorSummary(ErrorType, AI); + ReportErrorSummary(ErrorKind, AI); AI.Clear(); return; } } else if (Loc.isSymbolizedStack()) { const AddressInfo &AI = Loc.getSymbolizedStack()->info; - ReportErrorSummary(ErrorType, AI); + ReportErrorSummary(ErrorKind, AI); return; } - ReportErrorSummary(ErrorType); + ReportErrorSummary(ErrorKind); } namespace { @@ -341,24 +365,30 @@ Diag::~Diag() { NumRanges, Args); } -ScopedReport::ScopedReport(ReportOptions Opts, Location SummaryLoc) - : Opts(Opts), SummaryLoc(SummaryLoc) { +ScopedReport::ScopedReport(ReportOptions Opts, Location SummaryLoc, + ErrorType Type) + : Opts(Opts), SummaryLoc(SummaryLoc), Type(Type) { InitAsStandaloneIfNecessary(); CommonSanitizerReportMutex.Lock(); } ScopedReport::~ScopedReport() { MaybePrintStackTrace(Opts.pc, Opts.bp); - MaybeReportErrorSummary(SummaryLoc); + MaybeReportErrorSummary(SummaryLoc, Type); CommonSanitizerReportMutex.Unlock(); - if (Opts.DieAfterReport || flags()->halt_on_error) + if (flags()->halt_on_error) Die(); } ALIGNED(64) static char suppression_placeholder[sizeof(SuppressionContext)]; static SuppressionContext *suppression_ctx = nullptr; static const char kVptrCheck[] = "vptr_check"; -static const char *kSuppressionTypes[] = { kVptrCheck }; +static const char *kSuppressionTypes[] = { +#define UBSAN_CHECK(Name, SummaryKind, FSanitizeFlagName) FSanitizeFlagName, +#include "ubsan_checks.inc" +#undef UBSAN_CHECK + kVptrCheck, +}; void __ubsan::InitializeSuppressions() { CHECK_EQ(nullptr, suppression_ctx); @@ -374,4 +404,28 @@ bool __ubsan::IsVptrCheckSuppressed(const char *TypeName) { return suppression_ctx->Match(TypeName, kVptrCheck, &s); } +bool __ubsan::IsPCSuppressed(ErrorType ET, uptr PC, const char *Filename) { + InitAsStandaloneIfNecessary(); + CHECK(suppression_ctx); + const char *SuppType = ConvertTypeToFlagName(ET); + // Fast path: don't symbolize PC if there is no suppressions for given UB + // type. + if (!suppression_ctx->HasSuppressionType(SuppType)) + return false; + Suppression *s = nullptr; + // Suppress by file name known to runtime. + if (Filename != nullptr && suppression_ctx->Match(Filename, SuppType, &s)) + return true; + // Suppress by module name. + if (const char *Module = Symbolizer::GetOrInit()->GetModuleNameForPc(PC)) { + if (suppression_ctx->Match(Module, SuppType, &s)) + return true; + } + // Suppress by function or source file name from debug info. + SymbolizedStackHolder Stack(Symbolizer::GetOrInit()->SymbolizePC(PC)); + const AddressInfo &AI = Stack.get()->info; + return suppression_ctx->Match(AI.function, SuppType, &s) || + suppression_ctx->Match(AI.file, SuppType, &s); +} + #endif // CAN_SANITIZE_UB diff --git a/lib/ubsan/ubsan_diag.h b/lib/ubsan/ubsan_diag.h index 615a646d9419..3edb67a03c1f 100644 --- a/lib/ubsan/ubsan_diag.h +++ b/lib/ubsan/ubsan_diag.h @@ -211,17 +211,25 @@ public: }; struct ReportOptions { - /// If DieAfterReport is specified, UBSan will terminate the program after the - /// report is printed. - bool DieAfterReport; + // If FromUnrecoverableHandler is specified, UBSan runtime handler is not + // expected to return. + bool FromUnrecoverableHandler; /// pc/bp are used to unwind the stack trace. uptr pc; uptr bp; }; -#define GET_REPORT_OPTIONS(die_after_report) \ +enum class ErrorType { +#define UBSAN_CHECK(Name, SummaryKind, FSanitizeFlagName) Name, +#include "ubsan_checks.inc" +#undef UBSAN_CHECK +}; + +bool ignoreReport(SourceLocation SLoc, ReportOptions Opts, ErrorType ET); + +#define GET_REPORT_OPTIONS(unrecoverable_handler) \ GET_CALLER_PC_BP; \ - ReportOptions Opts = {die_after_report, pc, bp} + ReportOptions Opts = {unrecoverable_handler, pc, bp} /// \brief Instantiate this class before printing diagnostics in the error /// report. This class ensures that reports from different threads and from @@ -229,14 +237,18 @@ struct ReportOptions { class ScopedReport { ReportOptions Opts; Location SummaryLoc; + ErrorType Type; public: - ScopedReport(ReportOptions Opts, Location SummaryLoc); + ScopedReport(ReportOptions Opts, Location SummaryLoc, ErrorType Type); ~ScopedReport(); }; void InitializeSuppressions(); bool IsVptrCheckSuppressed(const char *TypeName); +// Sometimes UBSan runtime can know filename from handlers arguments, even if +// debug info is missing. +bool IsPCSuppressed(ErrorType ET, uptr PC, const char *Filename); } // namespace __ubsan diff --git a/lib/ubsan/ubsan_flags.cc b/lib/ubsan/ubsan_flags.cc index 6e5ad4c7967c..20087b968782 100644 --- a/lib/ubsan/ubsan_flags.cc +++ b/lib/ubsan/ubsan_flags.cc @@ -60,6 +60,9 @@ void InitializeFlags() { // Override from environment variable. parser.ParseString(GetEnv("UBSAN_OPTIONS")); SetVerbosity(common_flags()->verbosity); + if (Verbosity()) ReportUnrecognizedFlags(); + + if (common_flags()->help) parser.PrintFlagDescriptions(); } } // namespace __ubsan diff --git a/lib/ubsan/ubsan_flags.inc b/lib/ubsan/ubsan_flags.inc index 9ca31d13a9b6..d171a98e1730 100644 --- a/lib/ubsan/ubsan_flags.inc +++ b/lib/ubsan/ubsan_flags.inc @@ -22,4 +22,5 @@ UBSAN_FLAG(bool, halt_on_error, false, UBSAN_FLAG(bool, print_stacktrace, false, "Include full stacktrace into an error report") UBSAN_FLAG(const char *, suppressions, "", "Suppressions file name.") - +UBSAN_FLAG(bool, report_error_type, false, + "Print specific error type instead of 'undefined-behavior' in summary.") diff --git a/lib/ubsan/ubsan_handlers.cc b/lib/ubsan/ubsan_handlers.cc index a65b2f5e3fc7..5d82e9afcd09 100644 --- a/lib/ubsan/ubsan_handlers.cc +++ b/lib/ubsan/ubsan_handlers.cc @@ -21,17 +21,20 @@ using namespace __sanitizer; using namespace __ubsan; -static bool ignoreReport(SourceLocation SLoc, ReportOptions Opts) { - // If source location is already acquired, we don't need to print an error - // report for the second time. However, if we're in an unrecoverable handler, - // it's possible that location was required by concurrently running thread. - // In this case, we should continue the execution to ensure that any of - // threads will grab the report mutex and print the report before - // crashing the program. - return SLoc.isDisabled() && !Opts.DieAfterReport; +namespace __ubsan { +bool ignoreReport(SourceLocation SLoc, ReportOptions Opts, ErrorType ET) { + // We are not allowed to skip error report: if we are in unrecoverable + // handler, we have to terminate the program right now, and therefore + // have to print some diagnostic. + // + // Even if source location is disabled, it doesn't mean that we have + // already report an error to the user: some concurrently running + // thread could have acquired it, but not yet printed the report. + if (Opts.FromUnrecoverableHandler) + return false; + return SLoc.isDisabled() || IsPCSuppressed(ET, Opts.pc, SLoc.getFilename()); } -namespace __ubsan { const char *TypeCheckKinds[] = { "load of", "store to", "reference binding to", "member access within", "member call on", "constructor call on", "downcast of", "downcast of", @@ -41,8 +44,18 @@ const char *TypeCheckKinds[] = { static void handleTypeMismatchImpl(TypeMismatchData *Data, ValueHandle Pointer, ReportOptions Opts) { Location Loc = Data->Loc.acquire(); - // Use the SourceLocation from Data to track deduplication, even if 'invalid' - if (ignoreReport(Loc.getSourceLocation(), Opts)) + + ErrorType ET; + if (!Pointer) + ET = ErrorType::NullPointerUse; + else if (Data->Alignment && (Pointer & (Data->Alignment - 1))) + ET = ErrorType::MisalignedPointerUse; + else + ET = ErrorType::InsufficientObjectSize; + + // Use the SourceLocation from Data to track deduplication, even if it's + // invalid. + if (ignoreReport(Loc.getSourceLocation(), Opts, ET)) return; SymbolizedStackHolder FallbackLoc; @@ -51,20 +64,28 @@ static void handleTypeMismatchImpl(TypeMismatchData *Data, ValueHandle Pointer, Loc = FallbackLoc; } - ScopedReport R(Opts, Loc); + ScopedReport R(Opts, Loc, ET); - if (!Pointer) + switch (ET) { + case ErrorType::NullPointerUse: Diag(Loc, DL_Error, "%0 null pointer of type %1") - << TypeCheckKinds[Data->TypeCheckKind] << Data->Type; - else if (Data->Alignment && (Pointer & (Data->Alignment - 1))) + << TypeCheckKinds[Data->TypeCheckKind] << Data->Type; + break; + case ErrorType::MisalignedPointerUse: Diag(Loc, DL_Error, "%0 misaligned address %1 for type %3, " "which requires %2 byte alignment") - << TypeCheckKinds[Data->TypeCheckKind] << (void*)Pointer - << Data->Alignment << Data->Type; - else + << TypeCheckKinds[Data->TypeCheckKind] << (void *)Pointer + << Data->Alignment << Data->Type; + break; + case ErrorType::InsufficientObjectSize: Diag(Loc, DL_Error, "%0 address %1 with insufficient space " "for an object of type %2") - << TypeCheckKinds[Data->TypeCheckKind] << (void*)Pointer << Data->Type; + << TypeCheckKinds[Data->TypeCheckKind] << (void *)Pointer << Data->Type; + break; + default: + UNREACHABLE("unexpected error type!"); + } + if (Pointer) Diag(Pointer, DL_Note, "pointer points here"); } @@ -87,23 +108,28 @@ static void handleIntegerOverflowImpl(OverflowData *Data, ValueHandle LHS, const char *Operator, T RHS, ReportOptions Opts) { SourceLocation Loc = Data->Loc.acquire(); - if (ignoreReport(Loc, Opts)) + bool IsSigned = Data->Type.isSignedIntegerTy(); + ErrorType ET = IsSigned ? ErrorType::SignedIntegerOverflow + : ErrorType::UnsignedIntegerOverflow; + + if (ignoreReport(Loc, Opts, ET)) return; - ScopedReport R(Opts, Loc); + ScopedReport R(Opts, Loc, ET); Diag(Loc, DL_Error, "%0 integer overflow: " "%1 %2 %3 cannot be represented in type %4") - << (Data->Type.isSignedIntegerTy() ? "signed" : "unsigned") + << (IsSigned ? "signed" : "unsigned") << Value(Data->Type, LHS) << Operator << RHS << Data->Type; } -#define UBSAN_OVERFLOW_HANDLER(handler_name, op, abort) \ +#define UBSAN_OVERFLOW_HANDLER(handler_name, op, unrecoverable) \ void __ubsan::handler_name(OverflowData *Data, ValueHandle LHS, \ ValueHandle RHS) { \ - GET_REPORT_OPTIONS(abort); \ + GET_REPORT_OPTIONS(unrecoverable); \ handleIntegerOverflowImpl(Data, LHS, op, Value(Data->Type, RHS), Opts); \ - if (abort) Die(); \ + if (unrecoverable) \ + Die(); \ } UBSAN_OVERFLOW_HANDLER(__ubsan_handle_add_overflow, "+", false) @@ -116,20 +142,23 @@ UBSAN_OVERFLOW_HANDLER(__ubsan_handle_mul_overflow_abort, "*", true) static void handleNegateOverflowImpl(OverflowData *Data, ValueHandle OldVal, ReportOptions Opts) { SourceLocation Loc = Data->Loc.acquire(); - if (ignoreReport(Loc, Opts)) + bool IsSigned = Data->Type.isSignedIntegerTy(); + ErrorType ET = IsSigned ? ErrorType::SignedIntegerOverflow + : ErrorType::UnsignedIntegerOverflow; + + if (ignoreReport(Loc, Opts, ET)) return; - ScopedReport R(Opts, Loc); + ScopedReport R(Opts, Loc, ET); - if (Data->Type.isSignedIntegerTy()) + if (IsSigned) Diag(Loc, DL_Error, "negation of %0 cannot be represented in type %1; " "cast to an unsigned type to negate this value to itself") - << Value(Data->Type, OldVal) << Data->Type; + << Value(Data->Type, OldVal) << Data->Type; else - Diag(Loc, DL_Error, - "negation of %0 cannot be represented in type %1") - << Value(Data->Type, OldVal) << Data->Type; + Diag(Loc, DL_Error, "negation of %0 cannot be represented in type %1") + << Value(Data->Type, OldVal) << Data->Type; } void __ubsan::__ubsan_handle_negate_overflow(OverflowData *Data, @@ -147,19 +176,31 @@ void __ubsan::__ubsan_handle_negate_overflow_abort(OverflowData *Data, static void handleDivremOverflowImpl(OverflowData *Data, ValueHandle LHS, ValueHandle RHS, ReportOptions Opts) { SourceLocation Loc = Data->Loc.acquire(); - if (ignoreReport(Loc, Opts)) - return; - - ScopedReport R(Opts, Loc); - Value LHSVal(Data->Type, LHS); Value RHSVal(Data->Type, RHS); + + ErrorType ET; if (RHSVal.isMinusOne()) - Diag(Loc, DL_Error, - "division of %0 by -1 cannot be represented in type %1") - << LHSVal << Data->Type; + ET = ErrorType::SignedIntegerOverflow; + else if (Data->Type.isIntegerTy()) + ET = ErrorType::IntegerDivideByZero; else + ET = ErrorType::FloatDivideByZero; + + if (ignoreReport(Loc, Opts, ET)) + return; + + ScopedReport R(Opts, Loc, ET); + + switch (ET) { + case ErrorType::SignedIntegerOverflow: + Diag(Loc, DL_Error, "division of %0 by -1 cannot be represented in type %1") + << LHSVal << Data->Type; + break; + default: Diag(Loc, DL_Error, "division by zero"); + break; + } } void __ubsan::__ubsan_handle_divrem_overflow(OverflowData *Data, @@ -179,25 +220,35 @@ static void handleShiftOutOfBoundsImpl(ShiftOutOfBoundsData *Data, ValueHandle LHS, ValueHandle RHS, ReportOptions Opts) { SourceLocation Loc = Data->Loc.acquire(); - if (ignoreReport(Loc, Opts)) - return; - - ScopedReport R(Opts, Loc); - Value LHSVal(Data->LHSType, LHS); Value RHSVal(Data->RHSType, RHS); - if (RHSVal.isNegative()) - Diag(Loc, DL_Error, "shift exponent %0 is negative") << RHSVal; - else if (RHSVal.getPositiveIntValue() >= Data->LHSType.getIntegerBitWidth()) - Diag(Loc, DL_Error, - "shift exponent %0 is too large for %1-bit type %2") - << RHSVal << Data->LHSType.getIntegerBitWidth() << Data->LHSType; - else if (LHSVal.isNegative()) - Diag(Loc, DL_Error, "left shift of negative value %0") << LHSVal; + + ErrorType ET; + if (RHSVal.isNegative() || + RHSVal.getPositiveIntValue() >= Data->LHSType.getIntegerBitWidth()) + ET = ErrorType::InvalidShiftExponent; else - Diag(Loc, DL_Error, - "left shift of %0 by %1 places cannot be represented in type %2") - << LHSVal << RHSVal << Data->LHSType; + ET = ErrorType::InvalidShiftBase; + + if (ignoreReport(Loc, Opts, ET)) + return; + + ScopedReport R(Opts, Loc, ET); + + if (ET == ErrorType::InvalidShiftExponent) { + if (RHSVal.isNegative()) + Diag(Loc, DL_Error, "shift exponent %0 is negative") << RHSVal; + else + Diag(Loc, DL_Error, "shift exponent %0 is too large for %1-bit type %2") + << RHSVal << Data->LHSType.getIntegerBitWidth() << Data->LHSType; + } else { + if (LHSVal.isNegative()) + Diag(Loc, DL_Error, "left shift of negative value %0") << LHSVal; + else + Diag(Loc, DL_Error, + "left shift of %0 by %1 places cannot be represented in type %2") + << LHSVal << RHSVal << Data->LHSType; + } } void __ubsan::__ubsan_handle_shift_out_of_bounds(ShiftOutOfBoundsData *Data, @@ -218,10 +269,12 @@ void __ubsan::__ubsan_handle_shift_out_of_bounds_abort( static void handleOutOfBoundsImpl(OutOfBoundsData *Data, ValueHandle Index, ReportOptions Opts) { SourceLocation Loc = Data->Loc.acquire(); - if (ignoreReport(Loc, Opts)) + ErrorType ET = ErrorType::OutOfBoundsIndex; + + if (ignoreReport(Loc, Opts, ET)) return; - ScopedReport R(Opts, Loc); + ScopedReport R(Opts, Loc, ET); Value IndexVal(Data->IndexType, Index); Diag(Loc, DL_Error, "index %0 out of bounds for type %1") @@ -242,7 +295,7 @@ void __ubsan::__ubsan_handle_out_of_bounds_abort(OutOfBoundsData *Data, static void handleBuiltinUnreachableImpl(UnreachableData *Data, ReportOptions Opts) { - ScopedReport R(Opts, Data->Loc); + ScopedReport R(Opts, Data->Loc, ErrorType::UnreachableCall); Diag(Data->Loc, DL_Error, "execution reached a __builtin_unreachable() call"); } @@ -253,7 +306,7 @@ void __ubsan::__ubsan_handle_builtin_unreachable(UnreachableData *Data) { } static void handleMissingReturnImpl(UnreachableData *Data, ReportOptions Opts) { - ScopedReport R(Opts, Data->Loc); + ScopedReport R(Opts, Data->Loc, ErrorType::MissingReturn); Diag(Data->Loc, DL_Error, "execution reached the end of a value-returning function " "without returning a value"); @@ -268,10 +321,12 @@ void __ubsan::__ubsan_handle_missing_return(UnreachableData *Data) { static void handleVLABoundNotPositive(VLABoundData *Data, ValueHandle Bound, ReportOptions Opts) { SourceLocation Loc = Data->Loc.acquire(); - if (ignoreReport(Loc, Opts)) + ErrorType ET = ErrorType::NonPositiveVLAIndex; + + if (ignoreReport(Loc, Opts, ET)) return; - ScopedReport R(Opts, Loc); + ScopedReport R(Opts, Loc, ET); Diag(Loc, DL_Error, "variable length array bound evaluates to " "non-positive value %0") @@ -290,26 +345,60 @@ void __ubsan::__ubsan_handle_vla_bound_not_positive_abort(VLABoundData *Data, Die(); } -static void handleFloatCastOverflow(FloatCastOverflowData *Data, - ValueHandle From, ReportOptions Opts) { - // TODO: Add deduplication once a SourceLocation is generated for this check. - SymbolizedStackHolder CallerLoc(getCallerLocation(Opts.pc)); - Location Loc = CallerLoc; - ScopedReport R(Opts, Loc); +static bool looksLikeFloatCastOverflowDataV1(void *Data) { + // First field is either a pointer to filename or a pointer to a + // TypeDescriptor. + u8 *FilenameOrTypeDescriptor; + internal_memcpy(&FilenameOrTypeDescriptor, Data, + sizeof(FilenameOrTypeDescriptor)); + + // Heuristic: For float_cast_overflow, the TypeKind will be either TK_Integer + // (0x0), TK_Float (0x1) or TK_Unknown (0xff). If both types are known, + // adding both bytes will be 0 or 1 (for BE or LE). If it were a filename, + // adding two printable characters will not yield such a value. Otherwise, + // if one of them is 0xff, this is most likely TK_Unknown type descriptor. + u16 MaybeFromTypeKind = + FilenameOrTypeDescriptor[0] + FilenameOrTypeDescriptor[1]; + return MaybeFromTypeKind < 2 || FilenameOrTypeDescriptor[0] == 0xff || + FilenameOrTypeDescriptor[1] == 0xff; +} + +static void handleFloatCastOverflow(void *DataPtr, ValueHandle From, + ReportOptions Opts) { + SymbolizedStackHolder CallerLoc; + Location Loc; + const TypeDescriptor *FromType, *ToType; + ErrorType ET = ErrorType::FloatCastOverflow; + + if (looksLikeFloatCastOverflowDataV1(DataPtr)) { + auto Data = reinterpret_cast<FloatCastOverflowData *>(DataPtr); + CallerLoc.reset(getCallerLocation(Opts.pc)); + Loc = CallerLoc; + FromType = &Data->FromType; + ToType = &Data->ToType; + } else { + auto Data = reinterpret_cast<FloatCastOverflowDataV2 *>(DataPtr); + SourceLocation SLoc = Data->Loc.acquire(); + if (ignoreReport(SLoc, Opts, ET)) + return; + Loc = SLoc; + FromType = &Data->FromType; + ToType = &Data->ToType; + } + + ScopedReport R(Opts, Loc, ET); Diag(Loc, DL_Error, "value %0 is outside the range of representable values of type %2") - << Value(Data->FromType, From) << Data->FromType << Data->ToType; + << Value(*FromType, From) << *FromType << *ToType; } -void __ubsan::__ubsan_handle_float_cast_overflow(FloatCastOverflowData *Data, - ValueHandle From) { +void __ubsan::__ubsan_handle_float_cast_overflow(void *Data, ValueHandle From) { GET_REPORT_OPTIONS(false); handleFloatCastOverflow(Data, From, Opts); } -void -__ubsan::__ubsan_handle_float_cast_overflow_abort(FloatCastOverflowData *Data, - ValueHandle From) { +void __ubsan::__ubsan_handle_float_cast_overflow_abort(void *Data, + ValueHandle From) { GET_REPORT_OPTIONS(true); handleFloatCastOverflow(Data, From, Opts); Die(); @@ -318,10 +407,16 @@ __ubsan::__ubsan_handle_float_cast_overflow_abort(FloatCastOverflowData *Data, static void handleLoadInvalidValue(InvalidValueData *Data, ValueHandle Val, ReportOptions Opts) { SourceLocation Loc = Data->Loc.acquire(); - if (ignoreReport(Loc, Opts)) + // This check could be more precise if we used different handlers for + // -fsanitize=bool and -fsanitize=enum. + bool IsBool = (0 == internal_strcmp(Data->Type.getTypeName(), "'bool'")); + ErrorType ET = + IsBool ? ErrorType::InvalidBoolLoad : ErrorType::InvalidEnumLoad; + + if (ignoreReport(Loc, Opts, ET)) return; - ScopedReport R(Opts, Loc); + ScopedReport R(Opts, Loc, ET); Diag(Loc, DL_Error, "load of value %0, which is not a valid value for type %1") @@ -344,10 +439,12 @@ static void handleFunctionTypeMismatch(FunctionTypeMismatchData *Data, ValueHandle Function, ReportOptions Opts) { SourceLocation CallLoc = Data->Loc.acquire(); - if (ignoreReport(CallLoc, Opts)) + ErrorType ET = ErrorType::FunctionTypeMismatch; + + if (ignoreReport(CallLoc, Opts, ET)) return; - ScopedReport R(Opts, CallLoc); + ScopedReport R(Opts, CallLoc, ET); SymbolizedStackHolder FLoc(getSymbolizedLocation(Function)); const char *FName = FLoc.get()->info.function; @@ -376,10 +473,12 @@ void __ubsan::__ubsan_handle_function_type_mismatch_abort( static void handleNonNullReturn(NonNullReturnData *Data, ReportOptions Opts) { SourceLocation Loc = Data->Loc.acquire(); - if (ignoreReport(Loc, Opts)) + ErrorType ET = ErrorType::InvalidNullReturn; + + if (ignoreReport(Loc, Opts, ET)) return; - ScopedReport R(Opts, Loc); + ScopedReport R(Opts, Loc, ET); Diag(Loc, DL_Error, "null pointer returned from function declared to never " "return null"); @@ -400,10 +499,12 @@ void __ubsan::__ubsan_handle_nonnull_return_abort(NonNullReturnData *Data) { static void handleNonNullArg(NonNullArgData *Data, ReportOptions Opts) { SourceLocation Loc = Data->Loc.acquire(); - if (ignoreReport(Loc, Opts)) + ErrorType ET = ErrorType::InvalidNullArgument; + + if (ignoreReport(Loc, Opts, ET)) return; - ScopedReport R(Opts, Loc); + ScopedReport R(Opts, Loc, ET); Diag(Loc, DL_Error, "null pointer passed as argument %0, which is declared to " "never be null") << Data->ArgIndex; @@ -422,4 +523,38 @@ void __ubsan::__ubsan_handle_nonnull_arg_abort(NonNullArgData *Data) { Die(); } +static void handleCFIBadIcall(CFIBadIcallData *Data, ValueHandle Function, + ReportOptions Opts) { + SourceLocation Loc = Data->Loc.acquire(); + ErrorType ET = ErrorType::CFIBadType; + + if (ignoreReport(Loc, Opts, ET)) + return; + + ScopedReport R(Opts, Loc, ET); + + Diag(Loc, DL_Error, "control flow integrity check for type %0 failed during " + "indirect function call") + << Data->Type; + + SymbolizedStackHolder FLoc(getSymbolizedLocation(Function)); + const char *FName = FLoc.get()->info.function; + if (!FName) + FName = "(unknown)"; + Diag(FLoc, DL_Note, "%0 defined here") << FName; +} + +void __ubsan::__ubsan_handle_cfi_bad_icall(CFIBadIcallData *Data, + ValueHandle Function) { + GET_REPORT_OPTIONS(false); + handleCFIBadIcall(Data, Function, Opts); +} + +void __ubsan::__ubsan_handle_cfi_bad_icall_abort(CFIBadIcallData *Data, + ValueHandle Function) { + GET_REPORT_OPTIONS(true); + handleCFIBadIcall(Data, Function, Opts); + Die(); +} + #endif // CAN_SANITIZE_UB diff --git a/lib/ubsan/ubsan_handlers.h b/lib/ubsan/ubsan_handlers.h index 87149f259607..6f309cf9aaa9 100644 --- a/lib/ubsan/ubsan_handlers.h +++ b/lib/ubsan/ubsan_handlers.h @@ -97,14 +97,22 @@ struct VLABoundData { /// \brief Handle a VLA with a non-positive bound. RECOVERABLE(vla_bound_not_positive, VLABoundData *Data, ValueHandle Bound) +// Keeping this around for binary compatibility with (sanitized) programs +// compiled with older compilers. struct FloatCastOverflowData { - // FIXME: SourceLocation Loc; const TypeDescriptor &FromType; const TypeDescriptor &ToType; }; -/// \brief Handle overflow in a conversion to or from a floating-point type. -RECOVERABLE(float_cast_overflow, FloatCastOverflowData *Data, ValueHandle From) +struct FloatCastOverflowDataV2 { + SourceLocation Loc; + const TypeDescriptor &FromType; + const TypeDescriptor &ToType; +}; + +/// Handle overflow in a conversion to or from a floating-point type. +/// void *Data is one of FloatCastOverflowData* or FloatCastOverflowDataV2* +RECOVERABLE(float_cast_overflow, void *Data, ValueHandle From) struct InvalidValueData { SourceLocation Loc; @@ -140,6 +148,14 @@ struct NonNullArgData { /// \brief Handle passing null pointer to function with nonnull attribute. RECOVERABLE(nonnull_arg, NonNullArgData *Data) +struct CFIBadIcallData { + SourceLocation Loc; + const TypeDescriptor &Type; +}; + +/// \brief Handle control flow integrity failure for indirect function calls. +RECOVERABLE(cfi_bad_icall, CFIBadIcallData *Data, ValueHandle Function) + } #endif // UBSAN_HANDLERS_H diff --git a/lib/ubsan/ubsan_handlers_cxx.cc b/lib/ubsan/ubsan_handlers_cxx.cc index 6984a963deb6..3e81be67163b 100644 --- a/lib/ubsan/ubsan_handlers_cxx.cc +++ b/lib/ubsan/ubsan_handlers_cxx.cc @@ -29,23 +29,25 @@ namespace __ubsan { extern const char *TypeCheckKinds[]; } -static void HandleDynamicTypeCacheMiss( +// Returns true if UBSan has printed an error report. +static bool HandleDynamicTypeCacheMiss( DynamicTypeCacheMissData *Data, ValueHandle Pointer, ValueHandle Hash, ReportOptions Opts) { if (checkDynamicType((void*)Pointer, Data->TypeInfo, Hash)) // Just a cache miss. The type matches after all. - return; + return false; // Check if error report should be suppressed. DynamicTypeInfo DTI = getDynamicTypeInfoFromObject((void*)Pointer); if (DTI.isValid() && IsVptrCheckSuppressed(DTI.getMostDerivedTypeName())) - return; + return false; SourceLocation Loc = Data->Loc.acquire(); - if (Loc.isDisabled()) - return; + ErrorType ET = ErrorType::DynamicTypeMismatch; + if (ignoreReport(Loc, Opts, ET)) + return false; - ScopedReport R(Opts, Loc); + ScopedReport R(Opts, Loc, ET); Diag(Loc, DL_Error, "%0 address %1 which does not point to an object of type %2") @@ -69,6 +71,7 @@ static void HandleDynamicTypeCacheMiss( << TypeName(DTI.getSubobjectTypeName()) << Range(Pointer, Pointer + sizeof(uptr), "vptr for %2 base class of %1"); + return true; } void __ubsan::__ubsan_handle_dynamic_type_cache_miss( @@ -78,14 +81,21 @@ void __ubsan::__ubsan_handle_dynamic_type_cache_miss( } void __ubsan::__ubsan_handle_dynamic_type_cache_miss_abort( DynamicTypeCacheMissData *Data, ValueHandle Pointer, ValueHandle Hash) { - GET_REPORT_OPTIONS(true); - HandleDynamicTypeCacheMiss(Data, Pointer, Hash, Opts); + // Note: -fsanitize=vptr is always recoverable. + GET_REPORT_OPTIONS(false); + if (HandleDynamicTypeCacheMiss(Data, Pointer, Hash, Opts)) + Die(); } static void HandleCFIBadType(CFIBadTypeData *Data, ValueHandle Vtable, ReportOptions Opts) { SourceLocation Loc = Data->Loc.acquire(); - ScopedReport R(Opts, Loc); + ErrorType ET = ErrorType::CFIBadType; + + if (ignoreReport(Loc, Opts, ET)) + return; + + ScopedReport R(Opts, Loc, ET); DynamicTypeInfo DTI = getDynamicTypeInfoFromVtable((void*)Vtable); static const char *TypeCheckKinds[] = { @@ -117,6 +127,7 @@ void __ubsan::__ubsan_handle_cfi_bad_type_abort(CFIBadTypeData *Data, ValueHandle Vtable) { GET_REPORT_OPTIONS(true); HandleCFIBadType(Data, Vtable, Opts); + Die(); } #endif // CAN_SANITIZE_UB |