diff options
Diffstat (limited to 'lib')
63 files changed, 494 insertions, 169 deletions
diff --git a/lib/asan/CMakeLists.txt b/lib/asan/CMakeLists.txt index 4bebb0a45a64..5ac5708a63d7 100644 --- a/lib/asan/CMakeLists.txt +++ b/lib/asan/CMakeLists.txt @@ -106,9 +106,9 @@ endif() add_compiler_rt_component(asan) if(APPLE) - add_weak_symbols("asan" WEAK_SYMBOL_LINKFLAGS) - add_weak_symbols("ubsan" WEAK_SYMBOL_LINKFLAGS) - add_weak_symbols("sanitizer_common" WEAK_SYMBOL_LINKFLAGS) + add_weak_symbols("asan" WEAK_SYMBOL_LINK_FLAGS) + add_weak_symbols("ubsan" WEAK_SYMBOL_LINK_FLAGS) + add_weak_symbols("sanitizer_common" WEAK_SYMBOL_LINK_FLAGS) add_compiler_rt_runtime(clang_rt.asan SHARED @@ -121,7 +121,7 @@ if(APPLE) RTLSanCommon RTUbsan CFLAGS ${ASAN_DYNAMIC_CFLAGS} - LINKFLAGS ${WEAK_SYMBOL_LINKFLAGS} + LINK_FLAGS ${WEAK_SYMBOL_LINK_FLAGS} DEFS ${ASAN_DYNAMIC_DEFINITIONS} PARENT_TARGET asan) else() @@ -188,7 +188,7 @@ else() RTAsan_dynamic_version_script_dummy RTUbsan_cxx CFLAGS ${ASAN_DYNAMIC_CFLAGS} - LINKFLAGS ${ASAN_DYNAMIC_LINK_FLAGS} + LINK_FLAGS ${ASAN_DYNAMIC_LINK_FLAGS} ${VERSION_SCRIPT_FLAG} LINK_LIBS ${ASAN_DYNAMIC_LIBS} DEFS ${ASAN_DYNAMIC_DEFINITIONS} diff --git a/lib/asan/asan_activation.cc b/lib/asan/asan_activation.cc index bb41a0eb559e..7e4e604dc218 100644 --- a/lib/asan/asan_activation.cc +++ b/lib/asan/asan_activation.cc @@ -77,12 +77,13 @@ static struct AsanDeactivatedFlags { void Print() { Report( - "quarantine_size_mb %d, max_redzone %d, poison_heap %d, " - "malloc_context_size %d, alloc_dealloc_mismatch %d, " - "allocator_may_return_null %d, coverage %d, coverage_dir %s, " - "allocator_release_to_os_interval_ms %d\n", - allocator_options.quarantine_size_mb, allocator_options.max_redzone, - poison_heap, malloc_context_size, + "quarantine_size_mb %d, thread_local_quarantine_size_kb %d, " + "max_redzone %d, poison_heap %d, malloc_context_size %d, " + "alloc_dealloc_mismatch %d, allocator_may_return_null %d, coverage %d, " + "coverage_dir %s, allocator_release_to_os_interval_ms %d\n", + allocator_options.quarantine_size_mb, + allocator_options.thread_local_quarantine_size_kb, + allocator_options.max_redzone, poison_heap, malloc_context_size, allocator_options.alloc_dealloc_mismatch, allocator_options.may_return_null, coverage, coverage_dir, allocator_options.release_to_os_interval_ms); @@ -109,6 +110,7 @@ void AsanDeactivate() { AllocatorOptions disabled = asan_deactivated_flags.allocator_options; disabled.quarantine_size_mb = 0; + disabled.thread_local_quarantine_size_kb = 0; disabled.min_redzone = 16; // Redzone must be at least 16 bytes long. disabled.max_redzone = 16; disabled.alloc_dealloc_mismatch = false; diff --git a/lib/asan/asan_activation_flags.inc b/lib/asan/asan_activation_flags.inc index 67440e6fde03..1c66e5bb53a5 100644 --- a/lib/asan/asan_activation_flags.inc +++ b/lib/asan/asan_activation_flags.inc @@ -24,6 +24,7 @@ ASAN_ACTIVATION_FLAG(int, redzone) ASAN_ACTIVATION_FLAG(int, max_redzone) ASAN_ACTIVATION_FLAG(int, quarantine_size_mb) +ASAN_ACTIVATION_FLAG(int, thread_local_quarantine_size_kb) ASAN_ACTIVATION_FLAG(bool, alloc_dealloc_mismatch) ASAN_ACTIVATION_FLAG(bool, poison_heap) diff --git a/lib/asan/asan_allocator.cc b/lib/asan/asan_allocator.cc index 2cf9d08d441e..ee9b1a6a04cf 100644 --- a/lib/asan/asan_allocator.cc +++ b/lib/asan/asan_allocator.cc @@ -269,24 +269,24 @@ struct Allocator { } void RePoisonChunk(uptr chunk) { - // This could a user-facing chunk (with redzones), or some internal + // This could be a user-facing chunk (with redzones), or some internal // housekeeping chunk, like TransferBatch. Start by assuming the former. AsanChunk *ac = GetAsanChunk((void *)chunk); uptr allocated_size = allocator.GetActuallyAllocatedSize((void *)ac); uptr beg = ac->Beg(); uptr end = ac->Beg() + ac->UsedSize(true); uptr chunk_end = chunk + allocated_size; - if (chunk < beg && beg < end && end <= chunk_end) { - // Looks like a valid AsanChunk. Or maybe not. Be conservative and only - // poison the redzones. + if (chunk < beg && beg < end && end <= chunk_end && + ac->chunk_state == CHUNK_ALLOCATED) { + // Looks like a valid AsanChunk in use, poison redzones only. PoisonShadow(chunk, beg - chunk, kAsanHeapLeftRedzoneMagic); uptr end_aligned_down = RoundDownTo(end, SHADOW_GRANULARITY); FastPoisonShadowPartialRightRedzone( end_aligned_down, end - end_aligned_down, chunk_end - end_aligned_down, kAsanHeapLeftRedzoneMagic); } else { - // This can not be an AsanChunk. Poison everything. It may be reused as - // AsanChunk later. + // This is either not an AsanChunk or freed or quarantined AsanChunk. + // In either case, poison everything. PoisonShadow(chunk, allocated_size, kAsanHeapLeftRedzoneMagic); } } diff --git a/lib/asan/asan_flags.cc b/lib/asan/asan_flags.cc index 4db407d4512b..ad5bbff28c37 100644 --- a/lib/asan/asan_flags.cc +++ b/lib/asan/asan_flags.cc @@ -169,6 +169,11 @@ void InitializeFlags() { (ASAN_LOW_MEMORY) ? 1 << 6 : FIRST_32_SECOND_64(1 << 8, 1 << 10); f->thread_local_quarantine_size_kb = kDefaultThreadLocalQuarantineSizeKb; } + if (f->thread_local_quarantine_size_kb == 0 && f->quarantine_size_mb > 0) { + Report("%s: thread_local_quarantine_size_kb can be set to 0 only when " + "quarantine_size_mb is set to 0\n", SanitizerToolName); + Die(); + } if (!f->replace_str && common_flags()->intercept_strlen) { Report("WARNING: strlen interceptor is enabled even though replace_str=0. " "Use intercept_strlen=0 to disable it."); diff --git a/lib/asan/tests/CMakeLists.txt b/lib/asan/tests/CMakeLists.txt index 22fa61dd1cf2..8089d51efe68 100644 --- a/lib/asan/tests/CMakeLists.txt +++ b/lib/asan/tests/CMakeLists.txt @@ -36,8 +36,8 @@ append_list_if(COMPILER_RT_HAS_WVARIADIC_MACROS_FLAG -Wno-variadic-macros ASAN_U # This will ensure the target linker is used # during cross compilation -set(ASAN_UNITTEST_COMMON_LINKFLAGS - ${COMPILER_RT_UNITTEST_LINKFLAGS}) +set(ASAN_UNITTEST_COMMON_LINK_FLAGS + ${COMPILER_RT_UNITTEST_LINK_FLAGS}) # -gline-tables-only must be enough for ASan, so use it if possible. if(COMPILER_RT_TEST_COMPILER_ID MATCHES "Clang") @@ -48,7 +48,7 @@ endif() if(MSVC) list(APPEND ASAN_UNITTEST_COMMON_CFLAGS -gcodeview) endif() -list(APPEND ASAN_UNITTEST_COMMON_LINKFLAGS -g) +list(APPEND ASAN_UNITTEST_COMMON_LINK_FLAGS -g) # Use -D instead of definitions to please custom compile command. list(APPEND ASAN_UNITTEST_COMMON_CFLAGS @@ -58,12 +58,12 @@ list(APPEND ASAN_UNITTEST_COMMON_CFLAGS if(APPLE) list(APPEND ASAN_UNITTEST_COMMON_CFLAGS ${DARWIN_osx_CFLAGS}) - list(APPEND ASAN_UNITTEST_COMMON_LINKFLAGS ${DARWIN_osx_LINKFLAGS}) + list(APPEND ASAN_UNITTEST_COMMON_LINK_FLAGS ${DARWIN_osx_LINK_FLAGS}) - add_weak_symbols("asan" WEAK_SYMBOL_LINKFLAGS) - add_weak_symbols("ubsan" WEAK_SYMBOL_LINKFLAGS) - add_weak_symbols("sanitizer_common" WEAK_SYMBOL_LINKFLAGS) - list(APPEND ASAN_UNITTEST_COMMON_LINKFLAGS ${WEAK_SYMBOL_LINKFLAGS}) + add_weak_symbols("asan" WEAK_SYMBOL_LINK_FLAGS) + add_weak_symbols("ubsan" WEAK_SYMBOL_LINK_FLAGS) + add_weak_symbols("sanitizer_common" WEAK_SYMBOL_LINK_FLAGS) + list(APPEND ASAN_UNITTEST_COMMON_LINK_FLAGS ${WEAK_SYMBOL_LINK_FLAGS}) endif() if(MSVC) @@ -82,41 +82,41 @@ if(CAN_TARGET_x86_64 OR CAN_TARGET_i386) endif() if(NOT MSVC) - list(APPEND ASAN_UNITTEST_COMMON_LINKFLAGS --driver-mode=g++) + list(APPEND ASAN_UNITTEST_COMMON_LINK_FLAGS --driver-mode=g++) endif() # x86_64 FreeBSD 9.2 additionally requires libc++ to build the tests. if(CMAKE_SYSTEM MATCHES "FreeBSD-9.2-RELEASE") - list(APPEND ASAN_UNITTEST_COMMON_LINKFLAGS "-lc++") + list(APPEND ASAN_UNITTEST_COMMON_LINK_FLAGS "-lc++") endif() # Unit tests on Mac depend on Foundation. if(APPLE) - list(APPEND ASAN_UNITTEST_COMMON_LINKFLAGS -framework Foundation) + list(APPEND ASAN_UNITTEST_COMMON_LINK_FLAGS -framework Foundation) endif() if(ANDROID) - list(APPEND ASAN_UNITTEST_COMMON_LINKFLAGS -pie) + list(APPEND ASAN_UNITTEST_COMMON_LINK_FLAGS -pie) endif() -set(ASAN_UNITTEST_INSTRUMENTED_LINKFLAGS - ${ASAN_UNITTEST_COMMON_LINKFLAGS}) -list(APPEND ASAN_UNITTEST_INSTRUMENTED_LINKFLAGS -fsanitize=address) +set(ASAN_UNITTEST_INSTRUMENTED_LINK_FLAGS + ${ASAN_UNITTEST_COMMON_LINK_FLAGS}) +list(APPEND ASAN_UNITTEST_INSTRUMENTED_LINK_FLAGS -fsanitize=address) -set(ASAN_DYNAMIC_UNITTEST_INSTRUMENTED_LINKFLAGS - ${ASAN_UNITTEST_INSTRUMENTED_LINKFLAGS} +set(ASAN_DYNAMIC_UNITTEST_INSTRUMENTED_LINK_FLAGS + ${ASAN_UNITTEST_INSTRUMENTED_LINK_FLAGS} -shared-libasan) set(ASAN_UNITTEST_INSTRUMENTED_LIBS) # NDK r10 requires -latomic almost always. append_list_if(ANDROID atomic ASAN_UNITTEST_INSTRUMENTED_LIBS) -set(ASAN_UNITTEST_NOINST_LINKFLAGS ${ASAN_UNITTEST_COMMON_LINKFLAGS}) +set(ASAN_UNITTEST_NOINST_LINK_FLAGS ${ASAN_UNITTEST_COMMON_LINK_FLAGS}) if(NOT APPLE) - append_list_if(COMPILER_RT_HAS_LIBM -lm ASAN_UNITTEST_NOINST_LINKFLAGS) - append_list_if(COMPILER_RT_HAS_LIBDL -ldl ASAN_UNITTEST_NOINST_LINKFLAGS) - append_list_if(COMPILER_RT_HAS_LIBRT -lrt ASAN_UNITTEST_NOINST_LINKFLAGS) - append_list_if(COMPILER_RT_HAS_LIBPTHREAD -pthread ASAN_UNITTEST_NOINST_LINKFLAGS) - append_list_if(COMPILER_RT_HAS_LIBPTHREAD -pthread ASAN_DYNAMIC_UNITTEST_INSTRUMENTED_LINKFLAGS) + append_list_if(COMPILER_RT_HAS_LIBM -lm ASAN_UNITTEST_NOINST_LINK_FLAGS) + append_list_if(COMPILER_RT_HAS_LIBDL -ldl ASAN_UNITTEST_NOINST_LINK_FLAGS) + append_list_if(COMPILER_RT_HAS_LIBRT -lrt ASAN_UNITTEST_NOINST_LINK_FLAGS) + append_list_if(COMPILER_RT_HAS_LIBPTHREAD -pthread ASAN_UNITTEST_NOINST_LINK_FLAGS) + append_list_if(COMPILER_RT_HAS_LIBPTHREAD -pthread ASAN_DYNAMIC_UNITTEST_INSTRUMENTED_LINK_FLAGS) endif() # TODO(eugenis): move all -l flags above to _LIBS? @@ -148,7 +148,7 @@ endmacro() # Link ASan unit test for a given architecture from a set # of objects in with given linker flags. macro(add_asan_test test_suite test_name arch kind) - cmake_parse_arguments(TEST "WITH_TEST_RUNTIME" "" "OBJECTS;LINKFLAGS;SUBDIR" ${ARGN}) + cmake_parse_arguments(TEST "WITH_TEST_RUNTIME" "" "OBJECTS;LINK_FLAGS;SUBDIR" ${ARGN}) get_target_flags_for_arch(${arch} TARGET_LINK_FLAGS) set(TEST_DEPS ${TEST_OBJECTS}) if(NOT COMPILER_RT_STANDALONE_BUILD) @@ -172,7 +172,7 @@ macro(add_asan_test test_suite test_name arch kind) SUBDIR ${TEST_SUBDIR} OBJECTS ${TEST_OBJECTS} DEPS ${TEST_DEPS} - LINK_FLAGS ${TEST_LINKFLAGS} + LINK_FLAGS ${TEST_LINK_FLAGS} ${TARGET_LINK_FLAGS}) endmacro() @@ -237,8 +237,8 @@ macro(add_asan_tests_for_arch_and_kind arch kind) endforeach() # Clang links the static CRT by default. Override that to use the dynamic # CRT. - set(ASAN_DYNAMIC_UNITTEST_INSTRUMENTED_LINKFLAGS - ${ASAN_DYNAMIC_UNITTEST_INSTRUMENTED_LINKFLAGS} + set(ASAN_DYNAMIC_UNITTEST_INSTRUMENTED_LINK_FLAGS + ${ASAN_DYNAMIC_UNITTEST_INSTRUMENTED_LINK_FLAGS} -Wl,-nodefaultlib:libcmt,-defaultlib:msvcrt,-defaultlib:oldnames) else() set(ASAN_INST_DYNAMIC_TEST_OBJECTS ${ASAN_INST_TEST_OBJECTS}) @@ -256,7 +256,7 @@ macro(add_asan_tests_for_arch_and_kind arch kind) add_asan_test(AsanUnitTests "Asan-${arch}${kind}-Test" ${arch} ${kind} SUBDIR "default" OBJECTS ${ASAN_INST_TEST_OBJECTS} - LINKFLAGS ${ASAN_UNITTEST_INSTRUMENTED_LINKFLAGS}) + LINK_FLAGS ${ASAN_UNITTEST_INSTRUMENTED_LINK_FLAGS}) if(COMPILER_RT_ASAN_HAS_STATIC_RUNTIME) # Create the 'dynamic' folder where ASAN tests are produced. if(CMAKE_CONFIGURATION_TYPES) @@ -270,7 +270,7 @@ macro(add_asan_tests_for_arch_and_kind arch kind) add_asan_test(AsanDynamicUnitTests "Asan-${arch}${kind}-Dynamic-Test" ${arch} ${kind} SUBDIR "dynamic" OBJECTS ${ASAN_INST_DYNAMIC_TEST_OBJECTS} - LINKFLAGS ${ASAN_DYNAMIC_UNITTEST_INSTRUMENTED_LINKFLAGS}) + LINK_FLAGS ${ASAN_DYNAMIC_UNITTEST_INSTRUMENTED_LINK_FLAGS}) endif() # Add static ASan runtime that will be linked with uninstrumented tests. @@ -307,7 +307,7 @@ macro(add_asan_tests_for_arch_and_kind arch kind) add_asan_test(AsanUnitTests "Asan-${arch}${kind}-Noinst-Test" ${arch} ${kind} SUBDIR "default" OBJECTS ${ASAN_NOINST_TEST_OBJECTS} - LINKFLAGS ${ASAN_UNITTEST_NOINST_LINKFLAGS} + LINK_FLAGS ${ASAN_UNITTEST_NOINST_LINK_FLAGS} WITH_TEST_RUNTIME) # Benchmarks. @@ -319,7 +319,7 @@ macro(add_asan_tests_for_arch_and_kind arch kind) add_asan_test(AsanBenchmarks "Asan-${arch}${kind}-Benchmark" ${arch} ${kind} SUBDIR "default" OBJECTS ${ASAN_BENCHMARKS_OBJECTS} - LINKFLAGS ${ASAN_UNITTEST_INSTRUMENTED_LINKFLAGS}) + LINK_FLAGS ${ASAN_UNITTEST_INSTRUMENTED_LINK_FLAGS}) endmacro() if(COMPILER_RT_CAN_EXECUTE_TESTS AND NOT ANDROID) @@ -347,7 +347,7 @@ if(ANDROID) ${COMPILER_RT_GTEST_SOURCE} ${ASAN_NOINST_TEST_SOURCES}) set_target_compile_flags(AsanNoinstTest ${ASAN_UNITTEST_COMMON_CFLAGS}) - set_target_link_flags(AsanNoinstTest ${ASAN_UNITTEST_NOINST_LINKFLAGS}) + set_target_link_flags(AsanNoinstTest ${ASAN_UNITTEST_NOINST_LINK_FLAGS}) target_link_libraries(AsanNoinstTest ${ASAN_UNITTEST_NOINST_LIBS}) # Test with ASan instrumentation. Link with ASan dynamic runtime. @@ -355,7 +355,7 @@ if(ANDROID) ${COMPILER_RT_GTEST_SOURCE} ${ASAN_INST_TEST_SOURCES}) set_target_compile_flags(AsanTest ${ASAN_UNITTEST_INSTRUMENTED_CFLAGS}) - set_target_link_flags(AsanTest ${ASAN_UNITTEST_INSTRUMENTED_LINKFLAGS}) + set_target_link_flags(AsanTest ${ASAN_UNITTEST_INSTRUMENTED_LINK_FLAGS}) target_link_libraries(AsanTest ${ASAN_UNITTEST_INSTRUMENTED_LIBS}) # Setup correct output directory and link flags. diff --git a/lib/builtins/arm/adddf3vfp.S b/lib/builtins/arm/adddf3vfp.S index f4c00a03e05f..8e476cad1624 100644 --- a/lib/builtins/arm/adddf3vfp.S +++ b/lib/builtins/arm/adddf3vfp.S @@ -18,10 +18,14 @@ .syntax unified .p2align 2 DEFINE_COMPILERRT_FUNCTION(__adddf3vfp) +#if defined(COMPILER_RT_ARMHF_TARGET) + vadd.f64 d0, d0, d1 +#else vmov d6, r0, r1 // move first param from r0/r1 pair into d6 vmov d7, r2, r3 // move second param from r2/r3 pair into d7 vadd.f64 d6, d6, d7 vmov r0, r1, d6 // move result back to r0/r1 pair +#endif bx lr END_COMPILERRT_FUNCTION(__adddf3vfp) diff --git a/lib/builtins/arm/addsf3vfp.S b/lib/builtins/arm/addsf3vfp.S index af40c1cc92af..8871efdcc5d1 100644 --- a/lib/builtins/arm/addsf3vfp.S +++ b/lib/builtins/arm/addsf3vfp.S @@ -18,10 +18,14 @@ .syntax unified .p2align 2 DEFINE_COMPILERRT_FUNCTION(__addsf3vfp) +#if defined(COMPILER_RT_ARMHF_TARGET) + vadd.f32 s0, s0, s1 +#else vmov s14, r0 // move first param from r0 into float register vmov s15, r1 // move second param from r1 into float register vadd.f32 s14, s14, s15 vmov r0, s14 // move result back to r0 +#endif bx lr END_COMPILERRT_FUNCTION(__addsf3vfp) diff --git a/lib/builtins/arm/comparesf2.S b/lib/builtins/arm/comparesf2.S index b8ac81ffac09..e8095650e4fb 100644 --- a/lib/builtins/arm/comparesf2.S +++ b/lib/builtins/arm/comparesf2.S @@ -43,8 +43,14 @@ .thumb #endif -.p2align 2 +@ int __eqsf2(float a, float b) + + .p2align 2 DEFINE_COMPILERRT_FUNCTION(__eqsf2) +#if defined(COMPILER_RT_ARMHF_TARGET) + vmov r0, s0 + vmov r1, s1 +#endif // Make copies of a and b with the sign bit shifted off the top. These will // be used to detect zeros and NaNs. #if __ARM_ARCH_ISA_THUMB == 1 @@ -166,16 +172,23 @@ LOCAL_LABEL(CHECK_NAN): JMP(lr) #endif END_COMPILERRT_FUNCTION(__eqsf2) + DEFINE_COMPILERRT_FUNCTION_ALIAS(__lesf2, __eqsf2) DEFINE_COMPILERRT_FUNCTION_ALIAS(__ltsf2, __eqsf2) DEFINE_COMPILERRT_FUNCTION_ALIAS(__nesf2, __eqsf2) -.p2align 2 +@ int __gtsf2(float a, float b) + + .p2align 2 DEFINE_COMPILERRT_FUNCTION(__gtsf2) // Identical to the preceding except in that we return -1 for NaN values. // Given that the two paths share so much code, one might be tempted to // unify them; however, the extra code needed to do so makes the code size // to performance tradeoff very hard to justify for such small functions. +#if defined(COMPILER_RT_ARMHF_TARGET) + vmov r0, s0 + vmov r1, s1 +#endif #if __ARM_ARCH_ISA_THUMB == 1 push {r6, lr} lsls r2, r0, #1 @@ -215,6 +228,8 @@ LOCAL_LABEL(CHECK_NAN_2): 6: pop {r6, pc} #else + mov r2, r0, lsl #1 + mov r3, r1, lsl #1 orrs r12, r2, r3, lsr #1 it ne eorsne r12, r0, r1 @@ -233,10 +248,17 @@ LOCAL_LABEL(CHECK_NAN_2): JMP(lr) #endif END_COMPILERRT_FUNCTION(__gtsf2) + DEFINE_COMPILERRT_FUNCTION_ALIAS(__gesf2, __gtsf2) -.p2align 2 +@ int __unordsf2(float a, float b) + + .p2align 2 DEFINE_COMPILERRT_FUNCTION(__unordsf2) +#if defined(COMPILER_RT_ARMHF_TARGET) + vmov r0, s0 + vmov r1, s1 +#endif // Return 1 for NaN values, 0 otherwise. lsls r2, r0, #1 lsls r3, r1, #1 @@ -260,7 +282,15 @@ DEFINE_COMPILERRT_FUNCTION(__unordsf2) JMP(lr) END_COMPILERRT_FUNCTION(__unordsf2) +#if defined(COMPILER_RT_ARMHF_TARGET) +DEFINE_COMPILERRT_FUNCTION(__aeabi_fcmpum): + vmov s0, r0 + vmov s1, r1 + b SYMBOL_NAME(__unordsf2) +END_COMPILERRT_FUNCTION(__aeabi_fcmpum) +#else DEFINE_AEABI_FUNCTION_ALIAS(__aeabi_fcmpun, __unordsf2) +#endif NO_EXEC_STACK_DIRECTIVE diff --git a/lib/builtins/arm/divdf3vfp.S b/lib/builtins/arm/divdf3vfp.S index 928f53809f12..776ba4f24b47 100644 --- a/lib/builtins/arm/divdf3vfp.S +++ b/lib/builtins/arm/divdf3vfp.S @@ -18,10 +18,14 @@ .syntax unified .p2align 2 DEFINE_COMPILERRT_FUNCTION(__divdf3vfp) +#if defined(COMPILER_RT_ARMHF_TARGET) + vdiv.f64 d0, d0, d1 +#else vmov d6, r0, r1 // move first param from r0/r1 pair into d6 vmov d7, r2, r3 // move second param from r2/r3 pair into d7 - vdiv.f64 d5, d6, d7 + vdiv.f64 d5, d6, d7 vmov r0, r1, d5 // move result back to r0/r1 pair +#endif bx lr END_COMPILERRT_FUNCTION(__divdf3vfp) diff --git a/lib/builtins/arm/divsf3vfp.S b/lib/builtins/arm/divsf3vfp.S index a2e297f70157..130318f0c37b 100644 --- a/lib/builtins/arm/divsf3vfp.S +++ b/lib/builtins/arm/divsf3vfp.S @@ -18,10 +18,14 @@ .syntax unified .p2align 2 DEFINE_COMPILERRT_FUNCTION(__divsf3vfp) +#if defined(COMPILER_RT_ARMHF_TARGET) + vdiv.f32 s0, s0, s1 +#else vmov s14, r0 // move first param from r0 into float register vmov s15, r1 // move second param from r1 into float register vdiv.f32 s13, s14, s15 vmov r0, s13 // move result back to r0 +#endif bx lr END_COMPILERRT_FUNCTION(__divsf3vfp) diff --git a/lib/builtins/arm/eqdf2vfp.S b/lib/builtins/arm/eqdf2vfp.S index 95e6bb36334b..8fa0b2debc77 100644 --- a/lib/builtins/arm/eqdf2vfp.S +++ b/lib/builtins/arm/eqdf2vfp.S @@ -19,9 +19,13 @@ .syntax unified .p2align 2 DEFINE_COMPILERRT_FUNCTION(__eqdf2vfp) +#if defined(COMPILER_RT_ARMHF_TARGET) + vcmp.f64 d0, d1 +#else vmov d6, r0, r1 // load r0/r1 pair in double register vmov d7, r2, r3 // load r2/r3 pair in double register vcmp.f64 d6, d7 +#endif vmrs apsr_nzcv, fpscr moveq r0, #1 // set result register to 1 if equal movne r0, #0 diff --git a/lib/builtins/arm/eqsf2vfp.S b/lib/builtins/arm/eqsf2vfp.S index fbac139c193a..3776bf4874c2 100644 --- a/lib/builtins/arm/eqsf2vfp.S +++ b/lib/builtins/arm/eqsf2vfp.S @@ -19,9 +19,13 @@ .syntax unified .p2align 2 DEFINE_COMPILERRT_FUNCTION(__eqsf2vfp) +#if defined(COMPILER_RT_ARMHF_TARGET) + vcmp.f32 s0, s1 +#else vmov s14, r0 // move from GPR 0 to float register vmov s15, r1 // move from GPR 1 to float register vcmp.f32 s14, s15 +#endif vmrs apsr_nzcv, fpscr moveq r0, #1 // set result register to 1 if equal movne r0, #0 diff --git a/lib/builtins/arm/extendsfdf2vfp.S b/lib/builtins/arm/extendsfdf2vfp.S index 563bf92afc36..1079f977bae6 100644 --- a/lib/builtins/arm/extendsfdf2vfp.S +++ b/lib/builtins/arm/extendsfdf2vfp.S @@ -19,9 +19,13 @@ .syntax unified .p2align 2 DEFINE_COMPILERRT_FUNCTION(__extendsfdf2vfp) +#if defined(COMPILER_RT_ARMHF_TARGET) + vcvt.f64.f32 d0, s0 +#else vmov s15, r0 // load float register from R0 vcvt.f64.f32 d7, s15 // convert single to double vmov r0, r1, d7 // return result in r0/r1 pair +#endif bx lr END_COMPILERRT_FUNCTION(__extendsfdf2vfp) diff --git a/lib/builtins/arm/fixdfsivfp.S b/lib/builtins/arm/fixdfsivfp.S index 8263ff942f8c..5d7b0f856549 100644 --- a/lib/builtins/arm/fixdfsivfp.S +++ b/lib/builtins/arm/fixdfsivfp.S @@ -19,9 +19,14 @@ .syntax unified .p2align 2 DEFINE_COMPILERRT_FUNCTION(__fixdfsivfp) +#if defined(COMPILER_RT_ARMHF_TARGET) + vcvt.s32.f64 s0, d0 + vmov r0, s0 +#else vmov d7, r0, r1 // load double register from R0/R1 vcvt.s32.f64 s15, d7 // convert double to 32-bit int into s15 vmov r0, s15 // move s15 to result register +#endif bx lr END_COMPILERRT_FUNCTION(__fixdfsivfp) diff --git a/lib/builtins/arm/fixsfsivfp.S b/lib/builtins/arm/fixsfsivfp.S index c7c3b8117876..805a277afa34 100644 --- a/lib/builtins/arm/fixsfsivfp.S +++ b/lib/builtins/arm/fixsfsivfp.S @@ -19,9 +19,14 @@ .syntax unified .p2align 2 DEFINE_COMPILERRT_FUNCTION(__fixsfsivfp) +#if defined(COMPILER_RT_ARMHF_TARGET) + vcvt.s32.f32 s0, s0 + vmov r0, s0 +#else vmov s15, r0 // load float register from R0 vcvt.s32.f32 s15, s15 // convert single to 32-bit int into s15 vmov r0, s15 // move s15 to result register +#endif bx lr END_COMPILERRT_FUNCTION(__fixsfsivfp) diff --git a/lib/builtins/arm/fixunsdfsivfp.S b/lib/builtins/arm/fixunsdfsivfp.S index 9cc1e628699e..4f1b2c8cefdc 100644 --- a/lib/builtins/arm/fixunsdfsivfp.S +++ b/lib/builtins/arm/fixunsdfsivfp.S @@ -20,9 +20,14 @@ .syntax unified .p2align 2 DEFINE_COMPILERRT_FUNCTION(__fixunsdfsivfp) +#if defined(COMPILER_RT_ARMHF_TARGET) + vcvt.u32.f64 s0, d0 + vmov r0, s0 +#else vmov d7, r0, r1 // load double register from R0/R1 vcvt.u32.f64 s15, d7 // convert double to 32-bit int into s15 vmov r0, s15 // move s15 to result register +#endif bx lr END_COMPILERRT_FUNCTION(__fixunsdfsivfp) diff --git a/lib/builtins/arm/fixunssfsivfp.S b/lib/builtins/arm/fixunssfsivfp.S index 79d708229112..e5d778236879 100644 --- a/lib/builtins/arm/fixunssfsivfp.S +++ b/lib/builtins/arm/fixunssfsivfp.S @@ -20,9 +20,14 @@ .syntax unified .p2align 2 DEFINE_COMPILERRT_FUNCTION(__fixunssfsivfp) +#if defined(COMPILER_RT_ARMHF_TARGET) + vcvt.u32.f32 s0, s0 + vmov r0, s0 +#else vmov s15, r0 // load float register from R0 vcvt.u32.f32 s15, s15 // convert single to 32-bit unsigned into s15 vmov r0, s15 // move s15 to result register +#endif bx lr END_COMPILERRT_FUNCTION(__fixunssfsivfp) diff --git a/lib/builtins/arm/floatsidfvfp.S b/lib/builtins/arm/floatsidfvfp.S index 7623f26c6e6d..3297ad44d8cd 100644 --- a/lib/builtins/arm/floatsidfvfp.S +++ b/lib/builtins/arm/floatsidfvfp.S @@ -19,9 +19,14 @@ .syntax unified .p2align 2 DEFINE_COMPILERRT_FUNCTION(__floatsidfvfp) +#if defined(COMPILER_RT_ARMHF_TARGET) + vmov s0, r0 + vcvt.f64.s32 d0, s0 +#else vmov s15, r0 // move int to float register s15 vcvt.f64.s32 d7, s15 // convert 32-bit int in s15 to double in d7 vmov r0, r1, d7 // move d7 to result register pair r0/r1 +#endif bx lr END_COMPILERRT_FUNCTION(__floatsidfvfp) diff --git a/lib/builtins/arm/floatsisfvfp.S b/lib/builtins/arm/floatsisfvfp.S index c73dfac13eb2..65408b54b8d4 100644 --- a/lib/builtins/arm/floatsisfvfp.S +++ b/lib/builtins/arm/floatsisfvfp.S @@ -19,9 +19,14 @@ .syntax unified .p2align 2 DEFINE_COMPILERRT_FUNCTION(__floatsisfvfp) +#if defined(COMPILER_RT_ARMHF_TARGET) + vmov s0, r0 + vcvt.f32.s32 s0, s0 +#else vmov s15, r0 // move int to float register s15 vcvt.f32.s32 s15, s15 // convert 32-bit int in s15 to float in s15 vmov r0, s15 // move s15 to result register +#endif bx lr END_COMPILERRT_FUNCTION(__floatsisfvfp) diff --git a/lib/builtins/arm/floatunssidfvfp.S b/lib/builtins/arm/floatunssidfvfp.S index 2a59fdb830b2..d7a7024a25b8 100644 --- a/lib/builtins/arm/floatunssidfvfp.S +++ b/lib/builtins/arm/floatunssidfvfp.S @@ -19,9 +19,14 @@ .syntax unified .p2align 2 DEFINE_COMPILERRT_FUNCTION(__floatunssidfvfp) +#if defined(COMPILER_RT_ARMHF_TARGET) + vmov s0, r0 + vcvt.f64.u32 d0, s0 +#else vmov s15, r0 // move int to float register s15 vcvt.f64.u32 d7, s15 // convert 32-bit int in s15 to double in d7 vmov r0, r1, d7 // move d7 to result register pair r0/r1 +#endif bx lr END_COMPILERRT_FUNCTION(__floatunssidfvfp) diff --git a/lib/builtins/arm/floatunssisfvfp.S b/lib/builtins/arm/floatunssisfvfp.S index c096263c1bca..1ca856519a92 100644 --- a/lib/builtins/arm/floatunssisfvfp.S +++ b/lib/builtins/arm/floatunssisfvfp.S @@ -19,9 +19,14 @@ .syntax unified .p2align 2 DEFINE_COMPILERRT_FUNCTION(__floatunssisfvfp) +#if defined(COMPILER_RT_ARMHF_TARGET) + vmov s0, r0 + vcvt.f32.u32 s0, s0 +#else vmov s15, r0 // move int to float register s15 vcvt.f32.u32 s15, s15 // convert 32-bit int in s15 to float in s15 vmov r0, s15 // move s15 to result register +#endif bx lr END_COMPILERRT_FUNCTION(__floatunssisfvfp) diff --git a/lib/builtins/arm/gedf2vfp.S b/lib/builtins/arm/gedf2vfp.S index 72f13ef4e718..14899f00aab6 100644 --- a/lib/builtins/arm/gedf2vfp.S +++ b/lib/builtins/arm/gedf2vfp.S @@ -19,9 +19,13 @@ .syntax unified .p2align 2 DEFINE_COMPILERRT_FUNCTION(__gedf2vfp) +#if defined(COMPILER_RT_ARMHF_TARGET) + vcmp.f64 d0, d1 +#else vmov d6, r0, r1 // load r0/r1 pair in double register vmov d7, r2, r3 // load r2/r3 pair in double register vcmp.f64 d6, d7 +#endif vmrs apsr_nzcv, fpscr movge r0, #1 // set result register to 1 if greater than or equal movlt r0, #0 diff --git a/lib/builtins/arm/gesf2vfp.S b/lib/builtins/arm/gesf2vfp.S index c9ee52c9c449..b49d04d1c239 100644 --- a/lib/builtins/arm/gesf2vfp.S +++ b/lib/builtins/arm/gesf2vfp.S @@ -19,9 +19,13 @@ .syntax unified .p2align 2 DEFINE_COMPILERRT_FUNCTION(__gesf2vfp) +#if defined(COMPILER_RT_ARMHF_TARGET) + vcmp.f32 s0, s1 +#else vmov s14, r0 // move from GPR 0 to float register vmov s15, r1 // move from GPR 1 to float register vcmp.f32 s14, s15 +#endif vmrs apsr_nzcv, fpscr movge r0, #1 // set result register to 1 if greater than or equal movlt r0, #0 diff --git a/lib/builtins/arm/gtdf2vfp.S b/lib/builtins/arm/gtdf2vfp.S index c7f277552fa6..8166305e3af1 100644 --- a/lib/builtins/arm/gtdf2vfp.S +++ b/lib/builtins/arm/gtdf2vfp.S @@ -19,9 +19,13 @@ .syntax unified .p2align 2 DEFINE_COMPILERRT_FUNCTION(__gtdf2vfp) +#if defined(COMPILER_RT_ARMHF_TARGET) + vcmp.f64 d0, d1 +#else vmov d6, r0, r1 // load r0/r1 pair in double register vmov d7, r2, r3 // load r2/r3 pair in double register vcmp.f64 d6, d7 +#endif vmrs apsr_nzcv, fpscr movgt r0, #1 // set result register to 1 if equal movle r0, #0 diff --git a/lib/builtins/arm/gtsf2vfp.S b/lib/builtins/arm/gtsf2vfp.S index 7d49e4564a8b..d2d8a2380fc8 100644 --- a/lib/builtins/arm/gtsf2vfp.S +++ b/lib/builtins/arm/gtsf2vfp.S @@ -19,9 +19,13 @@ .syntax unified .p2align 2 DEFINE_COMPILERRT_FUNCTION(__gtsf2vfp) +#if defined(COMPILER_RT_ARMHF_TARGET) + vcmp.f32 s0, s1 +#else vmov s14, r0 // move from GPR 0 to float register vmov s15, r1 // move from GPR 1 to float register vcmp.f32 s14, s15 +#endif vmrs apsr_nzcv, fpscr movgt r0, #1 // set result register to 1 if equal movle r0, #0 diff --git a/lib/builtins/arm/ledf2vfp.S b/lib/builtins/arm/ledf2vfp.S index ca5b553f1153..a9dab77c1469 100644 --- a/lib/builtins/arm/ledf2vfp.S +++ b/lib/builtins/arm/ledf2vfp.S @@ -19,9 +19,13 @@ .syntax unified .p2align 2 DEFINE_COMPILERRT_FUNCTION(__ledf2vfp) +#if defined(COMPILER_RT_ARMHF_TARGET) + vcmp.f64 d0, d1 +#else vmov d6, r0, r1 // load r0/r1 pair in double register vmov d7, r2, r3 // load r2/r3 pair in double register vcmp.f64 d6, d7 +#endif vmrs apsr_nzcv, fpscr movls r0, #1 // set result register to 1 if equal movhi r0, #0 diff --git a/lib/builtins/arm/lesf2vfp.S b/lib/builtins/arm/lesf2vfp.S index f25422ece8f5..7e127f465cfd 100644 --- a/lib/builtins/arm/lesf2vfp.S +++ b/lib/builtins/arm/lesf2vfp.S @@ -19,9 +19,13 @@ .syntax unified .p2align 2 DEFINE_COMPILERRT_FUNCTION(__lesf2vfp) +#if defined(COMPILER_RT_ARMHF_TARGET) + vcmp.f32 s0, s1 +#else vmov s14, r0 // move from GPR 0 to float register vmov s15, r1 // move from GPR 1 to float register vcmp.f32 s14, s15 +#endif vmrs apsr_nzcv, fpscr movls r0, #1 // set result register to 1 if equal movhi r0, #0 diff --git a/lib/builtins/arm/ltdf2vfp.S b/lib/builtins/arm/ltdf2vfp.S index 6e2c0997c01b..8b6f8e4cc8a4 100644 --- a/lib/builtins/arm/ltdf2vfp.S +++ b/lib/builtins/arm/ltdf2vfp.S @@ -19,9 +19,13 @@ .syntax unified .p2align 2 DEFINE_COMPILERRT_FUNCTION(__ltdf2vfp) +#if defined(COMPILER_RT_ARMHF_TARGET) + vcmp.f64 d0, d1 +#else vmov d6, r0, r1 // load r0/r1 pair in double register vmov d7, r2, r3 // load r2/r3 pair in double register vcmp.f64 d6, d7 +#endif vmrs apsr_nzcv, fpscr movmi r0, #1 // set result register to 1 if equal movpl r0, #0 diff --git a/lib/builtins/arm/ltsf2vfp.S b/lib/builtins/arm/ltsf2vfp.S index 95febb60672a..c4ff812b49a3 100644 --- a/lib/builtins/arm/ltsf2vfp.S +++ b/lib/builtins/arm/ltsf2vfp.S @@ -19,9 +19,13 @@ .syntax unified .p2align 2 DEFINE_COMPILERRT_FUNCTION(__ltsf2vfp) +#if defined(COMPILER_RT_ARMHF_TARGET) + vcmp.f32 s0, s1 +#else vmov s14, r0 // move from GPR 0 to float register vmov s15, r1 // move from GPR 1 to float register vcmp.f32 s14, s15 +#endif vmrs apsr_nzcv, fpscr movmi r0, #1 // set result register to 1 if equal movpl r0, #0 diff --git a/lib/builtins/arm/muldf3vfp.S b/lib/builtins/arm/muldf3vfp.S index f638de1ad28a..aa7b23495034 100644 --- a/lib/builtins/arm/muldf3vfp.S +++ b/lib/builtins/arm/muldf3vfp.S @@ -18,10 +18,14 @@ .syntax unified .p2align 2 DEFINE_COMPILERRT_FUNCTION(__muldf3vfp) +#if defined(COMPILER_RT_ARMHF_TARGET) + vmul.f64 d0, d0, d1 +#else vmov d6, r0, r1 // move first param from r0/r1 pair into d6 vmov d7, r2, r3 // move second param from r2/r3 pair into d7 - vmul.f64 d6, d6, d7 + vmul.f64 d6, d6, d7 vmov r0, r1, d6 // move result back to r0/r1 pair +#endif bx lr END_COMPILERRT_FUNCTION(__muldf3vfp) diff --git a/lib/builtins/arm/mulsf3vfp.S b/lib/builtins/arm/mulsf3vfp.S index bef58d3a0c89..a1da789dcade 100644 --- a/lib/builtins/arm/mulsf3vfp.S +++ b/lib/builtins/arm/mulsf3vfp.S @@ -18,9 +18,13 @@ .syntax unified .p2align 2 DEFINE_COMPILERRT_FUNCTION(__mulsf3vfp) +#if defined(COMPILER_RT_ARMHF_TARGET) + vmul.f32 s0, s0, s1 +#else vmov s14, r0 // move first param from r0 into float register vmov s15, r1 // move second param from r1 into float register vmul.f32 s13, s14, s15 +#endif vmov r0, s13 // move result back to r0 bx lr END_COMPILERRT_FUNCTION(__mulsf3vfp) diff --git a/lib/builtins/arm/nedf2vfp.S b/lib/builtins/arm/nedf2vfp.S index 78cf529d665b..7d884e07204c 100644 --- a/lib/builtins/arm/nedf2vfp.S +++ b/lib/builtins/arm/nedf2vfp.S @@ -19,9 +19,13 @@ .syntax unified .p2align 2 DEFINE_COMPILERRT_FUNCTION(__nedf2vfp) +#if defined(COMPILER_RT_ARMHF_TARGET) + vcmp.f64 d0, d1 +#else vmov d6, r0, r1 // load r0/r1 pair in double register vmov d7, r2, r3 // load r2/r3 pair in double register vcmp.f64 d6, d7 +#endif vmrs apsr_nzcv, fpscr movne r0, #1 // set result register to 0 if unequal moveq r0, #0 diff --git a/lib/builtins/arm/negdf2vfp.S b/lib/builtins/arm/negdf2vfp.S index 01c8ba6a120f..81f0ab8eec1d 100644 --- a/lib/builtins/arm/negdf2vfp.S +++ b/lib/builtins/arm/negdf2vfp.S @@ -18,7 +18,11 @@ .syntax unified .p2align 2 DEFINE_COMPILERRT_FUNCTION(__negdf2vfp) +#if defined(COMPILER_RT_ARMHF_TARGET) + vneg.f64 d0, d0 +#else eor r1, r1, #-2147483648 // flip sign bit on double in r0/r1 pair +#endif bx lr END_COMPILERRT_FUNCTION(__negdf2vfp) diff --git a/lib/builtins/arm/negsf2vfp.S b/lib/builtins/arm/negsf2vfp.S index 797abb32ead3..46ab4a9cf164 100644 --- a/lib/builtins/arm/negsf2vfp.S +++ b/lib/builtins/arm/negsf2vfp.S @@ -18,7 +18,11 @@ .syntax unified .p2align 2 DEFINE_COMPILERRT_FUNCTION(__negsf2vfp) +#if defined(COMPILER_RT_ARMHF_TARGET) + vneg.f32 s0, s0 +#else eor r0, r0, #-2147483648 // flip sign bit on float in r0 +#endif bx lr END_COMPILERRT_FUNCTION(__negsf2vfp) diff --git a/lib/builtins/arm/nesf2vfp.S b/lib/builtins/arm/nesf2vfp.S index 554d3e467512..97c764f63697 100644 --- a/lib/builtins/arm/nesf2vfp.S +++ b/lib/builtins/arm/nesf2vfp.S @@ -19,9 +19,13 @@ .syntax unified .p2align 2 DEFINE_COMPILERRT_FUNCTION(__nesf2vfp) +#if defined(COMPILER_RT_ARMHF_TARGET) + vcmp.f32 s0, s1 +#else vmov s14, r0 // move from GPR 0 to float register vmov s15, r1 // move from GPR 1 to float register vcmp.f32 s14, s15 +#endif vmrs apsr_nzcv, fpscr movne r0, #1 // set result register to 1 if unequal moveq r0, #0 diff --git a/lib/builtins/arm/subdf3vfp.S b/lib/builtins/arm/subdf3vfp.S index 1fc7d18c3d3c..2b6f2bdbfdd5 100644 --- a/lib/builtins/arm/subdf3vfp.S +++ b/lib/builtins/arm/subdf3vfp.S @@ -18,10 +18,14 @@ .syntax unified .p2align 2 DEFINE_COMPILERRT_FUNCTION(__subdf3vfp) +#if defined(COMPILER_RT_ARMHF_TARGET) + vsub.f64 d0, d0, d1 +#else vmov d6, r0, r1 // move first param from r0/r1 pair into d6 vmov d7, r2, r3 // move second param from r2/r3 pair into d7 vsub.f64 d6, d6, d7 vmov r0, r1, d6 // move result back to r0/r1 pair +#endif bx lr END_COMPILERRT_FUNCTION(__subdf3vfp) diff --git a/lib/builtins/arm/subsf3vfp.S b/lib/builtins/arm/subsf3vfp.S index 11fe386cd0d1..a9f3ba9422cf 100644 --- a/lib/builtins/arm/subsf3vfp.S +++ b/lib/builtins/arm/subsf3vfp.S @@ -12,17 +12,21 @@ // // extern float __subsf3vfp(float a, float b); // -// Returns the difference between two single precision floating point numbers +// Returns the difference between two single precision floating point numbers // using the Darwin calling convention where single arguments are passsed // like 32-bit ints. // .syntax unified .p2align 2 DEFINE_COMPILERRT_FUNCTION(__subsf3vfp) +#if defined(COMPILER_RT_ARMHF_TARGET) + vsub.f32 s0, s0, s1 +#elsee vmov s14, r0 // move first param from r0 into float register vmov s15, r1 // move second param from r1 into float register vsub.f32 s14, s14, s15 vmov r0, s14 // move result back to r0 +#endif bx lr END_COMPILERRT_FUNCTION(__subsf3vfp) diff --git a/lib/builtins/arm/truncdfsf2vfp.S b/lib/builtins/arm/truncdfsf2vfp.S index 04287ad27ce6..682e54d3d294 100644 --- a/lib/builtins/arm/truncdfsf2vfp.S +++ b/lib/builtins/arm/truncdfsf2vfp.S @@ -19,9 +19,13 @@ .syntax unified .p2align 2 DEFINE_COMPILERRT_FUNCTION(__truncdfsf2vfp) +#if defined(COMPILER_RT_ARMHF_TARGET) + vcvt.f32.f64 s0, d0 +#else vmov d7, r0, r1 // load double from r0/r1 pair vcvt.f32.f64 s15, d7 // convert double to single (trucate precision) vmov r0, s15 // return result in r0 +#endif bx lr END_COMPILERRT_FUNCTION(__truncdfsf2vfp) diff --git a/lib/builtins/arm/unorddf2vfp.S b/lib/builtins/arm/unorddf2vfp.S index 022dd7a978af..855637547003 100644 --- a/lib/builtins/arm/unorddf2vfp.S +++ b/lib/builtins/arm/unorddf2vfp.S @@ -19,9 +19,13 @@ .syntax unified .p2align 2 DEFINE_COMPILERRT_FUNCTION(__unorddf2vfp) +#if defined(COMPILER_RT_ARMHF_TARGET) + vcmp.f64 d0, d1 +#else vmov d6, r0, r1 // load r0/r1 pair in double register vmov d7, r2, r3 // load r2/r3 pair in double register - vcmp.f64 d6, d7 + vcmp.f64 d6, d7 +#endif vmrs apsr_nzcv, fpscr movvs r0, #1 // set result register to 1 if "overflow" (any NaNs) movvc r0, #0 diff --git a/lib/builtins/arm/unordsf2vfp.S b/lib/builtins/arm/unordsf2vfp.S index 5ebdd3df5505..2b16b4905c48 100644 --- a/lib/builtins/arm/unordsf2vfp.S +++ b/lib/builtins/arm/unordsf2vfp.S @@ -19,9 +19,13 @@ .syntax unified .p2align 2 DEFINE_COMPILERRT_FUNCTION(__unordsf2vfp) +#if defined(COMPILER_RT_ARMHF_TARGET) + vcmp.f32 s0, s1 +#else vmov s14, r0 // move from GPR 0 to float register vmov s15, r1 // move from GPR 1 to float register vcmp.f32 s14, s15 +#endif vmrs apsr_nzcv, fpscr movvs r0, #1 // set result register to 1 if "overflow" (any NaNs) movvc r0, #0 diff --git a/lib/msan/tests/CMakeLists.txt b/lib/msan/tests/CMakeLists.txt index 130a872a1ff1..8e911dc10bed 100644 --- a/lib/msan/tests/CMakeLists.txt +++ b/lib/msan/tests/CMakeLists.txt @@ -69,15 +69,15 @@ macro(msan_compile obj_list source arch kind) endmacro() macro(msan_link_shared so_list so_name arch kind) - cmake_parse_arguments(SOURCE "" "" "OBJECTS;LINKFLAGS;DEPS" ${ARGN}) + cmake_parse_arguments(SOURCE "" "" "OBJECTS;LINK_FLAGS;DEPS" ${ARGN}) set(output_so "${CMAKE_CURRENT_BINARY_DIR}/${so_name}.${arch}${kind}.so") - get_target_flags_for_arch(${arch} TARGET_LINKFLAGS) + get_target_flags_for_arch(${arch} TARGET_LINK_FLAGS) if(NOT COMPILER_RT_STANDALONE_BUILD) list(APPEND SOURCE_DEPS msan) endif() clang_link_shared(${output_so} OBJECTS ${SOURCE_OBJECTS} - LINKFLAGS ${TARGET_LINKFLAGS} ${SOURCE_LINKFLAGS} + LINK_FLAGS ${TARGET_LINK_FLAGS} ${SOURCE_LINK_FLAGS} DEPS ${SOURCE_DEPS}) list(APPEND ${so_list} ${output_so}) endmacro() diff --git a/lib/sanitizer_common/sanitizer_common.h b/lib/sanitizer_common/sanitizer_common.h index fc44ecd5b871..2dabb5066ecf 100644 --- a/lib/sanitizer_common/sanitizer_common.h +++ b/lib/sanitizer_common/sanitizer_common.h @@ -690,6 +690,7 @@ inline const char *ModuleArchToString(ModuleArch arch) { return "arm64"; } CHECK(0 && "Invalid module arch"); + return ""; } const uptr kModuleUUIDSize = 16; diff --git a/lib/sanitizer_common/sanitizer_coverage_libcdep.cc b/lib/sanitizer_common/sanitizer_coverage_libcdep.cc index ebdee33d7d5b..5945ebbe90b2 100644 --- a/lib/sanitizer_common/sanitizer_coverage_libcdep.cc +++ b/lib/sanitizer_common/sanitizer_coverage_libcdep.cc @@ -954,7 +954,9 @@ SANITIZER_INTERFACE_ATTRIBUTE void __sanitizer_cov_init() { } SANITIZER_INTERFACE_ATTRIBUTE void __sanitizer_cov_dump() { coverage_data.DumpAll(); +#if SANITIZER_LINUX __sanitizer_dump_trace_pc_guard_coverage(); +#endif } SANITIZER_INTERFACE_ATTRIBUTE void __sanitizer_cov_module_init(s32 *guards, uptr npcs, u8 *counters, diff --git a/lib/sanitizer_common/sanitizer_libignore.cc b/lib/sanitizer_common/sanitizer_libignore.cc index 33a1763d28ea..aa4fa88efffa 100644 --- a/lib/sanitizer_common/sanitizer_libignore.cc +++ b/lib/sanitizer_common/sanitizer_libignore.cc @@ -78,10 +78,12 @@ void LibIgnore::OnLibraryLoaded(const char *name) { lib->templ, mod.full_name()); lib->loaded = true; lib->name = internal_strdup(mod.full_name()); - const uptr idx = atomic_load(&loaded_count_, memory_order_relaxed); - code_ranges_[idx].begin = range.beg; - code_ranges_[idx].end = range.end; - atomic_store(&loaded_count_, idx + 1, memory_order_release); + const uptr idx = + atomic_load(&ignored_ranges_count_, memory_order_relaxed); + CHECK_LT(idx, kMaxLibs); + ignored_code_ranges_[idx].begin = range.beg; + ignored_code_ranges_[idx].end = range.end; + atomic_store(&ignored_ranges_count_, idx + 1, memory_order_release); break; } } @@ -92,6 +94,29 @@ void LibIgnore::OnLibraryLoaded(const char *name) { Die(); } } + + // Track instrumented ranges. + if (track_instrumented_libs_) { + for (const auto &mod : modules) { + if (!mod.instrumented()) + continue; + for (const auto &range : mod.ranges()) { + if (!range.executable) + continue; + if (IsPcInstrumented(range.beg) && IsPcInstrumented(range.end - 1)) + continue; + VReport(1, "Adding instrumented range %p-%p from library '%s'\n", + range.beg, range.end, mod.full_name()); + const uptr idx = + atomic_load(&instrumented_ranges_count_, memory_order_relaxed); + CHECK_LT(idx, kMaxLibs); + instrumented_code_ranges_[idx].begin = range.beg; + instrumented_code_ranges_[idx].end = range.end; + atomic_store(&instrumented_ranges_count_, idx + 1, + memory_order_release); + } + } + } } void LibIgnore::OnLibraryUnloaded() { diff --git a/lib/sanitizer_common/sanitizer_libignore.h b/lib/sanitizer_common/sanitizer_libignore.h index cd56c36c1c0e..17b0f563d47d 100644 --- a/lib/sanitizer_common/sanitizer_libignore.h +++ b/lib/sanitizer_common/sanitizer_libignore.h @@ -30,6 +30,9 @@ class LibIgnore { // Must be called during initialization. void AddIgnoredLibrary(const char *name_templ); + void IgnoreNoninstrumentedModules(bool enable) { + track_instrumented_libs_ = enable; + } // Must be called after a new dynamic library is loaded. void OnLibraryLoaded(const char *name); @@ -37,8 +40,14 @@ class LibIgnore { // Must be called after a dynamic library is unloaded. void OnLibraryUnloaded(); - // Checks whether the provided PC belongs to one of the ignored libraries. - bool IsIgnored(uptr pc) const; + // Checks whether the provided PC belongs to one of the ignored libraries or + // the PC should be ignored because it belongs to an non-instrumented module + // (when ignore_noninstrumented_modules=1). Also returns true via + // "pc_in_ignored_lib" if the PC is in an ignored library, false otherwise. + bool IsIgnored(uptr pc, bool *pc_in_ignored_lib) const; + + // Checks whether the provided PC belongs to an instrumented module. + bool IsPcInstrumented(uptr pc) const; private: struct Lib { @@ -53,26 +62,48 @@ class LibIgnore { uptr end; }; + inline bool IsInRange(uptr pc, const LibCodeRange &range) const { + return (pc >= range.begin && pc < range.end); + } + static const uptr kMaxLibs = 128; // Hot part: - atomic_uintptr_t loaded_count_; - LibCodeRange code_ranges_[kMaxLibs]; + atomic_uintptr_t ignored_ranges_count_; + LibCodeRange ignored_code_ranges_[kMaxLibs]; + + atomic_uintptr_t instrumented_ranges_count_; + LibCodeRange instrumented_code_ranges_[kMaxLibs]; // Cold part: BlockingMutex mutex_; uptr count_; Lib libs_[kMaxLibs]; + bool track_instrumented_libs_; // Disallow copying of LibIgnore objects. LibIgnore(const LibIgnore&); // not implemented void operator = (const LibIgnore&); // not implemented }; -inline bool LibIgnore::IsIgnored(uptr pc) const { - const uptr n = atomic_load(&loaded_count_, memory_order_acquire); +inline bool LibIgnore::IsIgnored(uptr pc, bool *pc_in_ignored_lib) const { + const uptr n = atomic_load(&ignored_ranges_count_, memory_order_acquire); + for (uptr i = 0; i < n; i++) { + if (IsInRange(pc, ignored_code_ranges_[i])) { + *pc_in_ignored_lib = true; + return true; + } + } + *pc_in_ignored_lib = false; + if (track_instrumented_libs_ && !IsPcInstrumented(pc)) + return true; + return false; +} + +inline bool LibIgnore::IsPcInstrumented(uptr pc) const { + const uptr n = atomic_load(&instrumented_ranges_count_, memory_order_acquire); for (uptr i = 0; i < n; i++) { - if (pc >= code_ranges_[i].begin && pc < code_ranges_[i].end) + if (IsInRange(pc, instrumented_code_ranges_[i])) return true; } return false; diff --git a/lib/sanitizer_common/sanitizer_platform_limits_posix.h b/lib/sanitizer_common/sanitizer_platform_limits_posix.h index 477a0ecbe0e4..c2d9f2cd3762 100644 --- a/lib/sanitizer_common/sanitizer_platform_limits_posix.h +++ b/lib/sanitizer_common/sanitizer_platform_limits_posix.h @@ -635,9 +635,12 @@ namespace __sanitizer { #ifndef __mips__ #if defined(__sparc__) #if __GLIBC_PREREQ (2, 20) - // On sparc glibc 2.19 and earlier sa_flags was unsigned long, and - // __glibc_reserved0 didn't exist. + // On sparc glibc 2.19 and earlier sa_flags was unsigned long. +#if defined(__arch64__) + // To maintain ABI compatibility on sparc64 when switching to an int, + // __glibc_reserved0 was added. int __glibc_reserved0; +#endif int sa_flags; #else unsigned long sa_flags; diff --git a/lib/sanitizer_common/sanitizer_quarantine.h b/lib/sanitizer_common/sanitizer_quarantine.h index 3d74ef2b6abd..1a0d9545b7e1 100644 --- a/lib/sanitizer_common/sanitizer_quarantine.h +++ b/lib/sanitizer_common/sanitizer_quarantine.h @@ -49,18 +49,31 @@ class Quarantine { } void Init(uptr size, uptr cache_size) { - atomic_store(&max_size_, size, memory_order_release); + // Thread local quarantine size can be zero only when global quarantine size + // is zero (it allows us to perform just one atomic read per Put() call). + CHECK((size == 0 && cache_size == 0) || cache_size != 0); + + atomic_store(&max_size_, size, memory_order_relaxed); atomic_store(&min_size_, size / 10 * 9, - memory_order_release); // 90% of max size. - max_cache_size_ = cache_size; + memory_order_relaxed); // 90% of max size. + atomic_store(&max_cache_size_, cache_size, memory_order_relaxed); } - uptr GetSize() const { return atomic_load(&max_size_, memory_order_acquire); } - uptr GetCacheSize() const { return max_cache_size_; } + uptr GetSize() const { return atomic_load(&max_size_, memory_order_relaxed); } + uptr GetCacheSize() const { + return atomic_load(&max_cache_size_, memory_order_relaxed); + } void Put(Cache *c, Callback cb, Node *ptr, uptr size) { - c->Enqueue(cb, ptr, size); - if (c->Size() > max_cache_size_) + uptr cache_size = GetCacheSize(); + if (cache_size) { + c->Enqueue(cb, ptr, size); + } else { + // cache_size == 0 only when size == 0 (see Init). + cb.Recycle(ptr); + } + // Check cache size anyway to accommodate for runtime cache_size change. + if (c->Size() > cache_size) Drain(c, cb); } @@ -83,7 +96,7 @@ class Quarantine { char pad0_[kCacheLineSize]; atomic_uintptr_t max_size_; atomic_uintptr_t min_size_; - uptr max_cache_size_; + atomic_uintptr_t max_cache_size_; char pad1_[kCacheLineSize]; SpinMutex cache_mutex_; SpinMutex recycle_mutex_; @@ -92,7 +105,7 @@ class Quarantine { void NOINLINE Recycle(Callback cb) { Cache tmp; - uptr min_size = atomic_load(&min_size_, memory_order_acquire); + uptr min_size = atomic_load(&min_size_, memory_order_relaxed); { SpinMutexLock l(&cache_mutex_); while (cache_.Size() > min_size) { @@ -205,6 +218,7 @@ class QuarantineCache { return b; } }; + } // namespace __sanitizer #endif // SANITIZER_QUARANTINE_H diff --git a/lib/sanitizer_common/tests/CMakeLists.txt b/lib/sanitizer_common/tests/CMakeLists.txt index 841acb315435..20698b9a8a96 100644 --- a/lib/sanitizer_common/tests/CMakeLists.txt +++ b/lib/sanitizer_common/tests/CMakeLists.txt @@ -80,10 +80,10 @@ endif() if(APPLE) list(APPEND SANITIZER_TEST_CFLAGS_COMMON ${DARWIN_osx_CFLAGS}) - list(APPEND SANITIZER_TEST_LINK_FLAGS_COMMON ${DARWIN_osx_LINKFLAGS}) + list(APPEND SANITIZER_TEST_LINK_FLAGS_COMMON ${DARWIN_osx_LINK_FLAGS}) - add_weak_symbols("sanitizer_common" WEAK_SYMBOL_LINKFLAGS) - list(APPEND SANITIZER_TEST_LINK_FLAGS_COMMON ${WEAK_SYMBOL_LINKFLAGS}) + add_weak_symbols("sanitizer_common" WEAK_SYMBOL_LINK_FLAGS) + list(APPEND SANITIZER_TEST_LINK_FLAGS_COMMON ${WEAK_SYMBOL_LINK_FLAGS}) endif() # MSVC linker is allocating 1M for the stack by default, which is not diff --git a/lib/scudo/CMakeLists.txt b/lib/scudo/CMakeLists.txt index e683379b2b30..4f1acec78c8f 100644 --- a/lib/scudo/CMakeLists.txt +++ b/lib/scudo/CMakeLists.txt @@ -3,16 +3,23 @@ add_compiler_rt_component(scudo) include_directories(..) set(SCUDO_CFLAGS ${SANITIZER_COMMON_CFLAGS}) +# SANITIZER_COMMON_CFLAGS include -fno-builtin, but we actually want builtins! +list(APPEND SCUDO_CFLAGS -fbuiltin) append_rtti_flag(OFF SCUDO_CFLAGS) set(SCUDO_SOURCES scudo_allocator.cpp scudo_flags.cpp + scudo_crc32.cpp scudo_interceptors.cpp scudo_new_delete.cpp scudo_termination.cpp scudo_utils.cpp) +if (COMPILER_RT_HAS_MSSE4_2_FLAG) + set_source_files_properties(scudo_crc32.cpp PROPERTIES COMPILE_FLAGS -msse4.2) +endif() + if(COMPILER_RT_HAS_SCUDO) foreach(arch ${SCUDO_SUPPORTED_ARCH}) add_compiler_rt_runtime(clang_rt.scudo diff --git a/lib/scudo/scudo_allocator.cpp b/lib/scudo/scudo_allocator.cpp index 96cfbdbc1af7..d1121b0e7a74 100644 --- a/lib/scudo/scudo_allocator.cpp +++ b/lib/scudo/scudo_allocator.cpp @@ -15,6 +15,7 @@ //===----------------------------------------------------------------------===// #include "scudo_allocator.h" +#include "scudo_crc32.h" #include "scudo_utils.h" #include "sanitizer_common/sanitizer_allocator_interface.h" @@ -25,22 +26,6 @@ #include <cstring> -// Hardware CRC32 is supported at compilation via the following: -// - for i386 & x86_64: -msse4.2 -// - for ARM & AArch64: -march=armv8-a+crc -// An additional check must be performed at runtime as well to make sure the -// emitted instructions are valid on the target host. -#if defined(__SSE4_2__) || defined(__ARM_FEATURE_CRC32) -# ifdef __SSE4_2__ -# include <smmintrin.h> -# define HW_CRC32 FIRST_32_SECOND_64(_mm_crc32_u32, _mm_crc32_u64) -# endif -# ifdef __ARM_FEATURE_CRC32 -# include <arm_acle.h> -# define HW_CRC32 FIRST_32_SECOND_64(__crc32cw, __crc32cd) -# endif -#endif - namespace __scudo { #if SANITIZER_CAN_USE_ALLOCATOR64 @@ -84,10 +69,6 @@ static thread_local Xorshift128Plus Prng; // Global static cookie, initialized at start-up. static uptr Cookie; -enum : u8 { - CRC32Software = 0, - CRC32Hardware = 1, -}; // We default to software CRC32 if the alternatives are not supported, either // at compilation or at runtime. static atomic_uint8_t HashAlgorithm = { CRC32Software }; @@ -97,17 +78,9 @@ static atomic_uint8_t HashAlgorithm = { CRC32Software }; // the checksumming function if available. INLINE u32 hashUptrs(uptr Pointer, uptr *Array, uptr ArraySize, u8 HashType) { u32 Crc; -#if defined(__SSE4_2__) || defined(__ARM_FEATURE_CRC32) - if (HashType == CRC32Hardware) { - Crc = HW_CRC32(Cookie, Pointer); - for (uptr i = 0; i < ArraySize; i++) - Crc = HW_CRC32(Crc, Array[i]); - return Crc; - } -#endif - Crc = computeCRC32(Cookie, Pointer); + Crc = computeCRC32(Cookie, Pointer, HashType); for (uptr i = 0; i < ArraySize; i++) - Crc = computeCRC32(Crc, Array[i]); + Crc = computeCRC32(Crc, Array[i], HashType); return Crc; } diff --git a/lib/scudo/scudo_crc32.cpp b/lib/scudo/scudo_crc32.cpp new file mode 100644 index 000000000000..94c8c2424929 --- /dev/null +++ b/lib/scudo/scudo_crc32.cpp @@ -0,0 +1,53 @@ +//===-- scudo_crc32.cpp -----------------------------------------*- C++ -*-===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// +/// +/// CRC32 function leveraging hardware specific instructions. This has to be +/// kept separated to restrict the use of compiler specific flags to this file. +/// +//===----------------------------------------------------------------------===// + +// Hardware CRC32 is supported at compilation via the following: +// - for i386 & x86_64: -msse4.2 +// - for ARM & AArch64: -march=armv8-a+crc or -mcrc +// An additional check must be performed at runtime as well to make sure the +// emitted instructions are valid on the target host. +#include "scudo_crc32.h" +#include "scudo_utils.h" + +#if defined(__SSE4_2__) || defined(__ARM_FEATURE_CRC32) +# ifdef __SSE4_2__ +# include <smmintrin.h> +# define CRC32_INTRINSIC FIRST_32_SECOND_64(_mm_crc32_u32, _mm_crc32_u64) +# endif +# ifdef __ARM_FEATURE_CRC32 +# include <arm_acle.h> +# define CRC32_INTRINSIC FIRST_32_SECOND_64(__crc32cw, __crc32cd) +# endif +#endif // defined(__SSE4_2__) || defined(__ARM_FEATURE_CRC32) + +namespace __scudo { + +#if defined(__SSE4_2__) || defined(__ARM_FEATURE_CRC32) +INLINE u32 computeHardwareCRC32(u32 Crc, uptr Data) { + return CRC32_INTRINSIC(Crc, Data); +} + +u32 computeCRC32(u32 Crc, uptr Data, u8 HashType) { + if (HashType == CRC32Hardware) { + return computeHardwareCRC32(Crc, Data); + } + return computeSoftwareCRC32(Crc, Data); +} +#else +u32 computeCRC32(u32 Crc, uptr Data, u8 HashType) { + return computeSoftwareCRC32(Crc, Data); +} +#endif // defined(__SSE4_2__) || defined(__ARM_FEATURE_CRC32) + +} // namespace __scudo diff --git a/lib/scudo/scudo_crc32.h b/lib/scudo/scudo_crc32.h new file mode 100644 index 000000000000..6635cc78bbab --- /dev/null +++ b/lib/scudo/scudo_crc32.h @@ -0,0 +1,30 @@ +//===-- scudo_crc32.h -------------------------------------------*- C++ -*-===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// +/// +/// Header for scudo_crc32.cpp. +/// +//===----------------------------------------------------------------------===// + +#ifndef SCUDO_CRC32_H_ +#define SCUDO_CRC32_H_ + +#include "sanitizer_common/sanitizer_internal_defs.h" + +namespace __scudo { + +enum : u8 { + CRC32Software = 0, + CRC32Hardware = 1, +}; + +u32 computeCRC32(u32 Crc, uptr Data, u8 HashType); + +} // namespace __scudo + +#endif // SCUDO_CRC32_H_ diff --git a/lib/scudo/scudo_utils.cpp b/lib/scudo/scudo_utils.cpp index c0269ec11efb..ffa65b219fd1 100644 --- a/lib/scudo/scudo_utils.cpp +++ b/lib/scudo/scudo_utils.cpp @@ -185,8 +185,7 @@ const static u32 CRC32Table[] = { 0xb40bbe37, 0xc30c8ea1, 0x5a05df1b, 0x2d02ef8d }; -u32 computeCRC32(u32 Crc, uptr Data) -{ +u32 computeSoftwareCRC32(u32 Crc, uptr Data) { for (uptr i = 0; i < sizeof(Data); i++) { Crc = CRC32Table[(Crc ^ Data) & 0xff] ^ (Crc >> 8); Data >>= 8; diff --git a/lib/scudo/scudo_utils.h b/lib/scudo/scudo_utils.h index f93f26ef122a..ef2a609671ac 100644 --- a/lib/scudo/scudo_utils.h +++ b/lib/scudo/scudo_utils.h @@ -53,8 +53,8 @@ struct Xorshift128Plus { u64 State[2]; }; -// Software CRC32 functions, to be used when SSE 4.2 support is not detected. -u32 computeCRC32(u32 Crc, uptr Data); +// Software CRC32 functions, to be used when hardware support is not detected. +u32 computeSoftwareCRC32(u32 Crc, uptr Data); } // namespace __scudo diff --git a/lib/stats/CMakeLists.txt b/lib/stats/CMakeLists.txt index ec75262d46fc..2b3d6474bb76 100644 --- a/lib/stats/CMakeLists.txt +++ b/lib/stats/CMakeLists.txt @@ -6,13 +6,13 @@ set_target_properties(stats PROPERTIES FOLDER "Compiler-RT Misc") if(APPLE) set(STATS_LIB_FLAVOR SHARED) - add_weak_symbols("asan" WEAK_SYMBOL_LINKFLAGS) - add_weak_symbols("ubsan" WEAK_SYMBOL_LINKFLAGS) - add_weak_symbols("sanitizer_common" WEAK_SYMBOL_LINKFLAGS) + add_weak_symbols("asan" WEAK_SYMBOL_LINK_FLAGS) + add_weak_symbols("ubsan" WEAK_SYMBOL_LINK_FLAGS) + add_weak_symbols("sanitizer_common" WEAK_SYMBOL_LINK_FLAGS) else() set(STATS_LIB_FLAVOR STATIC) - set(WEAK_SYMBOL_LINKFLAGS) + set(WEAK_SYMBOL_LINK_FLAGS) endif() add_compiler_rt_runtime(clang_rt.stats @@ -23,7 +23,7 @@ add_compiler_rt_runtime(clang_rt.stats OBJECT_LIBS RTSanitizerCommon RTSanitizerCommonLibc CFLAGS ${SANITIZER_COMMON_CFLAGS} - LINKFLAGS ${WEAK_SYMBOL_LINKFLAGS} + LINK_FLAGS ${WEAK_SYMBOL_LINK_FLAGS} PARENT_TARGET stats) add_compiler_rt_runtime(clang_rt.stats_client @@ -32,5 +32,5 @@ add_compiler_rt_runtime(clang_rt.stats_client OS ${SANITIZER_COMMON_SUPPORTED_OS} SOURCES stats_client.cc CFLAGS ${SANITIZER_COMMON_CFLAGS} - LINKFLAGS ${WEAK_SYMBOL_LINKFLAGS} + LINK_FLAGS ${WEAK_SYMBOL_LINK_FLAGS} PARENT_TARGET stats) diff --git a/lib/tsan/CMakeLists.txt b/lib/tsan/CMakeLists.txt index 75d26349ae30..d5195459656c 100644 --- a/lib/tsan/CMakeLists.txt +++ b/lib/tsan/CMakeLists.txt @@ -108,8 +108,8 @@ if(APPLE) set_source_files_properties(${TSAN_ASM_SOURCES} PROPERTIES LANGUAGE C) endif() - add_weak_symbols("ubsan" WEAK_SYMBOL_LINKFLAGS) - add_weak_symbols("sanitizer_common" WEAK_SYMBOL_LINKFLAGS) + add_weak_symbols("ubsan" WEAK_SYMBOL_LINK_FLAGS) + add_weak_symbols("sanitizer_common" WEAK_SYMBOL_LINK_FLAGS) add_compiler_rt_runtime(clang_rt.tsan SHARED @@ -121,7 +121,7 @@ if(APPLE) RTSanitizerCommonLibc RTUbsan CFLAGS ${TSAN_RTL_CFLAGS} - LINKFLAGS ${WEAK_SYMBOL_LINKFLAGS} + LINK_FLAGS ${WEAK_SYMBOL_LINK_FLAGS} PARENT_TARGET tsan) add_compiler_rt_object_libraries(RTTsan_dynamic OS ${TSAN_SUPPORTED_OS} diff --git a/lib/tsan/rtl/tsan_flags.inc b/lib/tsan/rtl/tsan_flags.inc index 071cf427d23b..a48545c433ba 100644 --- a/lib/tsan/rtl/tsan_flags.inc +++ b/lib/tsan/rtl/tsan_flags.inc @@ -79,5 +79,8 @@ TSAN_FLAG(bool, die_after_fork, true, TSAN_FLAG(const char *, suppressions, "", "Suppressions file name.") TSAN_FLAG(bool, ignore_interceptors_accesses, false, "Ignore reads and writes from all interceptors.") +TSAN_FLAG(bool, ignore_noninstrumented_modules, false, + "Interceptors should only detect races when called from instrumented " + "modules.") TSAN_FLAG(bool, shared_ptr_interceptor, true, "Track atomic reference counting in libc++ shared_ptr and weak_ptr.") diff --git a/lib/tsan/rtl/tsan_interceptors.cc b/lib/tsan/rtl/tsan_interceptors.cc index a3a50e13f9bf..898f32df182b 100644 --- a/lib/tsan/rtl/tsan_interceptors.cc +++ b/lib/tsan/rtl/tsan_interceptors.cc @@ -231,6 +231,8 @@ void InitializeLibIgnore() { if (0 == internal_strcmp(s->type, kSuppressionLib)) libignore()->AddIgnoredLibrary(s->templ); } + if (flags()->ignore_noninstrumented_modules) + libignore()->IgnoreNoninstrumentedModules(true); libignore()->OnLibraryLoaded(0); } @@ -252,31 +254,20 @@ static unsigned g_thread_finalize_key; ScopedInterceptor::ScopedInterceptor(ThreadState *thr, const char *fname, uptr pc) - : thr_(thr) - , pc_(pc) - , in_ignored_lib_(false) { + : thr_(thr), pc_(pc), in_ignored_lib_(false), ignoring_(false) { Initialize(thr); - if (!thr_->is_inited) - return; - if (!thr_->ignore_interceptors) - FuncEntry(thr, pc); + if (!thr_->is_inited) return; + if (!thr_->ignore_interceptors) FuncEntry(thr, pc); DPrintf("#%d: intercept %s()\n", thr_->tid, fname); - if (!thr_->in_ignored_lib && libignore()->IsIgnored(pc)) { - in_ignored_lib_ = true; - thr_->in_ignored_lib = true; - ThreadIgnoreBegin(thr_, pc_); - } - if (flags()->ignore_interceptors_accesses) ThreadIgnoreBegin(thr_, pc_); + ignoring_ = + !thr_->in_ignored_lib && (flags()->ignore_interceptors_accesses || + libignore()->IsIgnored(pc, &in_ignored_lib_)); + EnableIgnores(); } ScopedInterceptor::~ScopedInterceptor() { - if (!thr_->is_inited) - return; - if (flags()->ignore_interceptors_accesses) ThreadIgnoreEnd(thr_, pc_); - if (in_ignored_lib_) { - thr_->in_ignored_lib = false; - ThreadIgnoreEnd(thr_, pc_); - } + if (!thr_->is_inited) return; + DisableIgnores(); if (!thr_->ignore_interceptors) { ProcessPendingSignals(thr_); FuncExit(thr_); @@ -284,20 +275,24 @@ ScopedInterceptor::~ScopedInterceptor() { } } -void ScopedInterceptor::UserCallbackStart() { - if (flags()->ignore_interceptors_accesses) ThreadIgnoreEnd(thr_, pc_); - if (in_ignored_lib_) { - thr_->in_ignored_lib = false; - ThreadIgnoreEnd(thr_, pc_); +void ScopedInterceptor::EnableIgnores() { + if (ignoring_) { + ThreadIgnoreBegin(thr_, pc_); + if (in_ignored_lib_) { + DCHECK(!thr_->in_ignored_lib); + thr_->in_ignored_lib = true; + } } } -void ScopedInterceptor::UserCallbackEnd() { - if (in_ignored_lib_) { - thr_->in_ignored_lib = true; - ThreadIgnoreBegin(thr_, pc_); +void ScopedInterceptor::DisableIgnores() { + if (ignoring_) { + ThreadIgnoreEnd(thr_, pc_); + if (in_ignored_lib_) { + DCHECK(thr_->in_ignored_lib); + thr_->in_ignored_lib = false; + } } - if (flags()->ignore_interceptors_accesses) ThreadIgnoreBegin(thr_, pc_); } #define TSAN_INTERCEPT(func) INTERCEPT_FUNCTION(func) diff --git a/lib/tsan/rtl/tsan_interceptors.h b/lib/tsan/rtl/tsan_interceptors.h index a0f9a0753a63..72534f4a24a6 100644 --- a/lib/tsan/rtl/tsan_interceptors.h +++ b/lib/tsan/rtl/tsan_interceptors.h @@ -10,12 +10,13 @@ class ScopedInterceptor { public: ScopedInterceptor(ThreadState *thr, const char *fname, uptr pc); ~ScopedInterceptor(); - void UserCallbackStart(); - void UserCallbackEnd(); + void DisableIgnores(); + void EnableIgnores(); private: ThreadState *const thr_; const uptr pc_; bool in_ignored_lib_; + bool ignoring_; }; } // namespace __tsan @@ -39,10 +40,10 @@ class ScopedInterceptor { /**/ #define SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_START() \ - si.UserCallbackStart(); + si.DisableIgnores(); #define SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_END() \ - si.UserCallbackEnd(); + si.EnableIgnores(); #define TSAN_INTERCEPTOR(ret, func, ...) INTERCEPTOR(ret, func, __VA_ARGS__) diff --git a/lib/tsan/tests/CMakeLists.txt b/lib/tsan/tests/CMakeLists.txt index ac7412940cf0..87e14174ad1f 100644 --- a/lib/tsan/tests/CMakeLists.txt +++ b/lib/tsan/tests/CMakeLists.txt @@ -77,8 +77,8 @@ macro(add_tsan_unittest testname) list(APPEND TEST_OBJECTS lib${TSAN_TEST_RUNTIME}.a) list(APPEND TEST_DEPS ${TSAN_TEST_RUNTIME}) - add_weak_symbols("ubsan" WEAK_SYMBOL_LINKFLAGS) - add_weak_symbols("sanitizer_common" WEAK_SYMBOL_LINKFLAGS) + add_weak_symbols("ubsan" WEAK_SYMBOL_LINK_FLAGS) + add_weak_symbols("sanitizer_common" WEAK_SYMBOL_LINK_FLAGS) # Intentionally do *not* link with `-fsanitize=thread`. We already link # against a static version of the runtime, and we don't want the dynamic @@ -87,7 +87,7 @@ macro(add_tsan_unittest testname) OBJECTS ${TEST_OBJECTS} DEPS ${TEST_DEPS} LINK_FLAGS ${TARGET_LINK_FLAGS} ${DARWIN_osx_LINK_FLAGS} - ${WEAK_SYMBOL_LINKFLAGS} -lc++) + ${WEAK_SYMBOL_LINK_FLAGS} -lc++) endif() endforeach() endif() diff --git a/lib/ubsan/CMakeLists.txt b/lib/ubsan/CMakeLists.txt index ebff36019d45..9bb36edc72cf 100644 --- a/lib/ubsan/CMakeLists.txt +++ b/lib/ubsan/CMakeLists.txt @@ -56,8 +56,8 @@ if(APPLE) SOURCES ${UBSAN_STANDALONE_SOURCES} CFLAGS ${UBSAN_STANDALONE_CFLAGS}) - add_weak_symbols("ubsan" WEAK_SYMBOL_LINKFLAGS) - add_weak_symbols("sanitizer_common" WEAK_SYMBOL_LINKFLAGS) + add_weak_symbols("ubsan" WEAK_SYMBOL_LINK_FLAGS) + add_weak_symbols("sanitizer_common" WEAK_SYMBOL_LINK_FLAGS) add_compiler_rt_runtime(clang_rt.ubsan SHARED @@ -67,7 +67,7 @@ if(APPLE) RTUbsan_standalone RTSanitizerCommon RTSanitizerCommonLibc - LINKFLAGS ${WEAK_SYMBOL_LINKFLAGS} + LINK_FLAGS ${WEAK_SYMBOL_LINK_FLAGS} PARENT_TARGET ubsan) endif() diff --git a/lib/xray/xray_AArch64.cc b/lib/xray/xray_AArch64.cc index 8f24610dab27..0c1df22ec2ef 100644 --- a/lib/xray/xray_AArch64.cc +++ b/lib/xray/xray_AArch64.cc @@ -19,6 +19,9 @@ #include <atomic> #include <cassert> + +extern "C" void __clear_cache(void* start, void* end); + namespace __xray { uint64_t cycleFrequency() XRAY_NEVER_INSTRUMENT { @@ -75,8 +78,8 @@ inline static bool patchSled(const bool Enable, const uint32_t FuncId, // B #32 uint32_t *FirstAddress = reinterpret_cast<uint32_t *>(Sled.Address); + uint32_t *CurAddress = FirstAddress + 1; if (Enable) { - uint32_t *CurAddress = FirstAddress + 1; *CurAddress = uint32_t(PatchOpcodes::PO_LdrW0_12); CurAddress++; *CurAddress = uint32_t(PatchOpcodes::PO_LdrX16_12); @@ -88,6 +91,7 @@ inline static bool patchSled(const bool Enable, const uint32_t FuncId, *reinterpret_cast<void (**)()>(CurAddress) = TracingHook; CurAddress += 2; *CurAddress = uint32_t(PatchOpcodes::PO_LdpX0X30SP_16); + CurAddress++; std::atomic_store_explicit( reinterpret_cast<std::atomic<uint32_t> *>(FirstAddress), uint32_t(PatchOpcodes::PO_StpX0X30SP_m16e), std::memory_order_release); @@ -96,6 +100,8 @@ inline static bool patchSled(const bool Enable, const uint32_t FuncId, reinterpret_cast<std::atomic<uint32_t> *>(FirstAddress), uint32_t(PatchOpcodes::PO_B32), std::memory_order_release); } + __clear_cache(reinterpret_cast<char*>(FirstAddress), + reinterpret_cast<char*>(CurAddress)); return true; } |