diff options
Diffstat (limited to 'compiler-rt')
33 files changed, 545 insertions, 178 deletions
diff --git a/compiler-rt/include/profile/instr_prof_interface.h b/compiler-rt/include/profile/instr_prof_interface.h new file mode 100644 index 000000000000..be40f2685934 --- /dev/null +++ b/compiler-rt/include/profile/instr_prof_interface.h @@ -0,0 +1,92 @@ +/*===---- instr_prof_interface.h - Instrumentation PGO User Program API ----=== + * + * Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. + * See https://llvm.org/LICENSE.txt for license information. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + * + *===-----------------------------------------------------------------------=== + * + * This header provides a public interface for fine-grained control of counter + * reset and profile dumping. These interface functions can be directly called + * in user programs. + * +\*===---------------------------------------------------------------------===*/ + +#ifndef COMPILER_RT_INSTR_PROFILING +#define COMPILER_RT_INSTR_PROFILING + +#ifdef __cplusplus +extern "C" { +#endif + +#ifdef __LLVM_INSTR_PROFILE_GENERATE +// Profile file reset and dump interfaces. +// When `-fprofile[-instr]-generate`/`-fcs-profile-generate` is in effect, +// clang defines __LLVM_INSTR_PROFILE_GENERATE to pick up the API calls. + +/*! + * \brief Set the filename for writing instrumentation data. + * + * Sets the filename to be used for subsequent calls to + * \a __llvm_profile_write_file(). + * + * \c Name is not copied, so it must remain valid. Passing NULL resets the + * filename logic to the default behaviour. + * + * Note: There may be multiple copies of the profile runtime (one for each + * instrumented image/DSO). This API only modifies the filename within the + * copy of the runtime available to the calling image. + * + * Warning: This is a no-op if continuous mode (\ref + * __llvm_profile_is_continuous_mode_enabled) is on. The reason for this is + * that in continuous mode, profile counters are mmap()'d to the profile at + * program initialization time. Support for transferring the mmap'd profile + * counts to a new file has not been implemented. + */ +void __llvm_profile_set_filename(const char *Name); + +/*! + * \brief Interface to set all PGO counters to zero for the current process. + * + */ +void __llvm_profile_reset_counters(void); + +/*! + * \brief this is a wrapper interface to \c __llvm_profile_write_file. + * After this interface is invoked, an already dumped flag will be set + * so that profile won't be dumped again during program exit. + * Invocation of interface __llvm_profile_reset_counters will clear + * the flag. This interface is designed to be used to collect profile + * data from user selected hot regions. The use model is + * __llvm_profile_reset_counters(); + * ... hot region 1 + * __llvm_profile_dump(); + * .. some other code + * __llvm_profile_reset_counters(); + * ... hot region 2 + * __llvm_profile_dump(); + * + * It is expected that on-line profile merging is on with \c %m specifier + * used in profile filename . If merging is not turned on, user is expected + * to invoke __llvm_profile_set_filename to specify different profile names + * for different regions before dumping to avoid profile write clobbering. + */ +int __llvm_profile_dump(void); + +// Interface to dump the current process' order file to disk. +int __llvm_orderfile_dump(void); + +#else + +#define __llvm_profile_set_filename(Name) +#define __llvm_profile_reset_counters() +#define __llvm_profile_dump() (0) +#define __llvm_orderfile_dump() (0) + +#endif + +#ifdef __cplusplus +} // extern "C" +#endif + +#endif diff --git a/compiler-rt/lib/asan/asan_win.cpp b/compiler-rt/lib/asan/asan_win.cpp index f16ce677618e..8507e675684e 100644 --- a/compiler-rt/lib/asan/asan_win.cpp +++ b/compiler-rt/lib/asan/asan_win.cpp @@ -266,12 +266,7 @@ void PlatformTSDDtor(void *tsd) { AsanThread::TSDDtor(tsd); } // }}} // ---------------------- Various stuff ---------------- {{{ -void *AsanDoesNotSupportStaticLinkage() { -#if defined(_DEBUG) -#error Please build the runtime with a non-debug CRT: /MD or /MT -#endif - return 0; -} +void *AsanDoesNotSupportStaticLinkage() { return 0; } uptr FindDynamicShadowStart() { return MapDynamicShadow(MemToShadowSize(kHighMemEnd), ASAN_SHADOW_SCALE, diff --git a/compiler-rt/lib/builtins/aarch64/sme-libc-routines.c b/compiler-rt/lib/builtins/aarch64/sme-libc-routines.c new file mode 100644 index 000000000000..cd73025a19cc --- /dev/null +++ b/compiler-rt/lib/builtins/aarch64/sme-libc-routines.c @@ -0,0 +1,87 @@ +#include <stdlib.h> + +// WARNING: When building the scalar versions of these functions you need to +// use the compiler flag "-mllvm -disable-loop-idiom-all" to prevent clang +// from recognising a loop idiom and planting calls to memcpy! + +static void *__arm_sc_memcpy_fwd(void *dest, const void *src, + size_t n) __arm_streaming_compatible { + unsigned char *destp = (unsigned char *)dest; + const unsigned char *srcp = (const unsigned char *)src; + for (size_t i = 0; i < n; ++i) + destp[i] = srcp[i]; + + return dest; +} + +// If dest and src overlap then behaviour is undefined, hence we can add the +// restrict keywords here. This also matches the definition of the libc memcpy +// according to the man page. +void *__arm_sc_memcpy(void *__restrict__ dest, const void *__restrict__ src, + size_t n) __arm_streaming_compatible { + return __arm_sc_memcpy_fwd(dest, src, n); +} + +void *__arm_sc_memset(void *dest, int c, size_t n) __arm_streaming_compatible { + unsigned char *destp = (unsigned char *)dest; + unsigned char c8 = (unsigned char)c; + for (size_t i = 0; i < n; ++i) + destp[i] = c8; + + return dest; +} + +static void *__arm_sc_memcpy_rev(void *dest, const void *src, + size_t n) __arm_streaming_compatible { + unsigned char *destp = (unsigned char *)dest; + const unsigned char *srcp = (const unsigned char *)src; + // TODO: Improve performance by copying larger chunks in reverse, or by + // using SVE. + while (n > 0) { + --n; + destp[n] = srcp[n]; + } + return dest; +} + +// Semantically a memmove is equivalent to the following: +// 1. Copy the entire contents of src to a temporary array that does not +// overlap with src or dest. +// 2. Copy the contents of the temporary array into dest. +void *__arm_sc_memmove(void *dest, const void *src, + size_t n) __arm_streaming_compatible { + unsigned char *destp = (unsigned char *)dest; + const unsigned char *srcp = (const unsigned char *)src; + + // If src and dest don't overlap then just invoke memcpy + if ((srcp > (destp + n)) || (destp > (srcp + n))) + return __arm_sc_memcpy_fwd(dest, src, n); + + // Overlap case 1: + // src: Low | -> | High + // dest: Low | -> | High + // Here src is always ahead of dest at a higher addres. If we first read a + // chunk of data from src we can safely write the same chunk to dest without + // corrupting future reads of src. + if (srcp > destp) + return __arm_sc_memcpy_fwd(dest, src, n); + + // Overlap case 2: + // src: Low | -> | High + // dest: Low | -> | High + // While we're in the overlap region we're always corrupting future reads of + // src when writing to dest. An efficient way to do this is to copy the data + // in reverse by starting at the highest address. + return __arm_sc_memcpy_rev(dest, src, n); +} + +const void *__arm_sc_memchr(const void *src, int c, + size_t n) __arm_streaming_compatible { + const unsigned char *srcp = (const unsigned char *)src; + unsigned char c8 = (unsigned char)c; + for (size_t i = 0; i < n; ++i) + if (srcp[i] == c8) + return &srcp[i]; + + return NULL; +} diff --git a/compiler-rt/lib/builtins/cpu_model/aarch64.c b/compiler-rt/lib/builtins/cpu_model/aarch64.c index 44e1cf49d1e9..17bddfca46f0 100644 --- a/compiler-rt/lib/builtins/cpu_model/aarch64.c +++ b/compiler-rt/lib/builtins/cpu_model/aarch64.c @@ -115,6 +115,7 @@ enum CPUFeatures { FEAT_SME_I64, FEAT_SME2, FEAT_RCPC3, + FEAT_MOPS, FEAT_MAX, FEAT_EXT = 62, // Reserved to indicate presence of additional features field // in __aarch64_cpu_features diff --git a/compiler-rt/lib/builtins/cpu_model/aarch64/fmv/apple.inc b/compiler-rt/lib/builtins/cpu_model/aarch64/fmv/apple.inc index 79d4ac70bd1b..0bb755f4b305 100644 --- a/compiler-rt/lib/builtins/cpu_model/aarch64/fmv/apple.inc +++ b/compiler-rt/lib/builtins/cpu_model/aarch64/fmv/apple.inc @@ -14,19 +14,17 @@ static bool isKnownAndSupported(const char *name) { void __init_cpu_features_resolver(void) { // On Darwin platforms, this may be called concurrently by multiple threads // because the resolvers that use it are called lazily at runtime (unlike on - // ELF platforms, where IFuncs are resolved serially at load time). This - // function's effect on __aarch64_cpu_features should be idempotent, but even - // so we need dispatch_once to resolve the race condition. Dispatch is - // available through libSystem, which we need anyway for the sysctl, so this - // does not add a new dependency. + // ELF platforms, where IFuncs are resolved serially at load time). This + // function's effect on __aarch64_cpu_features must be idempotent. + + if (!__atomic_load_n(&__aarch64_cpu_features.features, __ATOMIC_RELAXED)) { + uint64_t features = 0; - static dispatch_once_t onceToken = 0; - dispatch_once(&onceToken, ^{ // https://developer.apple.com/documentation/kernel/1387446-sysctlbyname/determining_instruction_set_characteristics - static struct { + static const struct { const char *sysctl_name; enum CPUFeatures feature; - } features[] = { + } feature_checks[] = { {"hw.optional.arm.FEAT_FlagM", FEAT_FLAGM}, {"hw.optional.arm.FEAT_FlagM2", FEAT_FLAGM2}, {"hw.optional.arm.FEAT_FHM", FEAT_FP16FML}, @@ -58,12 +56,16 @@ void __init_cpu_features_resolver(void) { {"hw.optional.arm.FEAT_BTI", FEAT_BTI}, }; - for (size_t I = 0, E = sizeof(features) / sizeof(features[0]); I != E; ++I) - if (isKnownAndSupported(features[I].sysctl_name)) - __aarch64_cpu_features.features |= (1ULL << features[I].feature); + for (size_t I = 0, E = sizeof(feature_checks) / sizeof(feature_checks[0]); + I != E; ++I) + if (isKnownAndSupported(feature_checks[I].sysctl_name)) + features |= (1ULL << feature_checks[I].feature); + + features |= (1ULL << FEAT_INIT); - __aarch64_cpu_features.features |= (1ULL << FEAT_INIT); - }); + __atomic_store(&__aarch64_cpu_features.features, &features, + __ATOMIC_RELAXED); + } } #endif // TARGET_OS_OSX || TARGET_OS_IPHONE diff --git a/compiler-rt/lib/builtins/cpu_model/aarch64/fmv/mrs.inc b/compiler-rt/lib/builtins/cpu_model/aarch64/fmv/mrs.inc index 2f712f41f497..32a21a2fba9a 100644 --- a/compiler-rt/lib/builtins/cpu_model/aarch64/fmv/mrs.inc +++ b/compiler-rt/lib/builtins/cpu_model/aarch64/fmv/mrs.inc @@ -109,6 +109,8 @@ static void __init_cpu_features_constructor(unsigned long hwcap, setCPUFeature(FEAT_SME_I64); if (hwcap2 & HWCAP2_SME_F64F64) setCPUFeature(FEAT_SME_F64); + if (hwcap2 & HWCAP2_MOPS) + setCPUFeature(FEAT_MOPS); if (hwcap & HWCAP_CPUID) { unsigned long ftr; getCPUFeature(ID_AA64PFR1_EL1, ftr); diff --git a/compiler-rt/lib/builtins/cpu_model/aarch64/hwcap.inc b/compiler-rt/lib/builtins/cpu_model/aarch64/hwcap.inc index 328d9c4140b9..7ddc125b26da 100644 --- a/compiler-rt/lib/builtins/cpu_model/aarch64/hwcap.inc +++ b/compiler-rt/lib/builtins/cpu_model/aarch64/hwcap.inc @@ -178,3 +178,6 @@ #ifndef HWCAP2_SVE_EBF16 #define HWCAP2_SVE_EBF16 (1ULL << 33) #endif +#ifndef HWCAP2_MOPS +#define HWCAP2_MOPS (1ULL << 43) +#endif diff --git a/compiler-rt/lib/builtins/cpu_model/x86.c b/compiler-rt/lib/builtins/cpu_model/x86.c index 9d9a5d3f1542..0750e29f989a 100644 --- a/compiler-rt/lib/builtins/cpu_model/x86.c +++ b/compiler-rt/lib/builtins/cpu_model/x86.c @@ -148,7 +148,8 @@ enum ProcessorFeatures { FEATURE_LZCNT, FEATURE_MOVBE, - FEATURE_X86_64_BASELINE = 95, + FEATURE_AVX512FP16 = 94, + FEATURE_X86_64_BASELINE, FEATURE_X86_64_V2, FEATURE_X86_64_V3, FEATURE_X86_64_V4, @@ -812,6 +813,8 @@ static void getAvailableFeatures(unsigned ECX, unsigned EDX, unsigned MaxLeaf, setFeature(FEATURE_AVX5124FMAPS); if (HasLeaf7 && ((EDX >> 8) & 1) && HasAVX512Save) setFeature(FEATURE_AVX512VP2INTERSECT); + if (HasLeaf7 && ((EDX >> 23) & 1) && HasAVX512Save) + setFeature(FEATURE_AVX512FP16); // EAX from subleaf 0 is the maximum subleaf supported. Some CPUs don't // return all 0s for invalid subleaves so check the limit. diff --git a/compiler-rt/lib/builtins/fp_extend_impl.inc b/compiler-rt/lib/builtins/fp_extend_impl.inc index e16b55d150d2..f4f663066431 100644 --- a/compiler-rt/lib/builtins/fp_extend_impl.inc +++ b/compiler-rt/lib/builtins/fp_extend_impl.inc @@ -60,7 +60,7 @@ static __inline dst_t __extendXfYf2__(src_t a) { dst_rep_t dstExp; dst_rep_t dstSigFrac; - if (srcExp >= 1 && srcExp < srcInfExp) { + if (srcExp >= 1 && srcExp < (src_rep_t)srcInfExp) { // a is a normal number. dstExp = (dst_rep_t)srcExp + (dst_rep_t)(dstExpBias - srcExpBias); dstSigFrac = (dst_rep_t)srcSigFrac << (dstSigFracBits - srcSigFracBits); diff --git a/compiler-rt/lib/builtins/int_lib.h b/compiler-rt/lib/builtins/int_lib.h index 04ea2d910574..f6c1b7cff4b9 100644 --- a/compiler-rt/lib/builtins/int_lib.h +++ b/compiler-rt/lib/builtins/int_lib.h @@ -119,14 +119,14 @@ COMPILER_RT_ABI tu_int __udivmodti4(tu_int a, tu_int b, tu_int *rem); #if defined(_MSC_VER) && !defined(__clang__) #include <intrin.h> -int __inline __builtin_ctz(uint32_t value) { +static int __inline __builtin_ctz(uint32_t value) { unsigned long trailing_zero = 0; if (_BitScanForward(&trailing_zero, value)) return trailing_zero; return 32; } -int __inline __builtin_clz(uint32_t value) { +static int __inline __builtin_clz(uint32_t value) { unsigned long leading_zero = 0; if (_BitScanReverse(&leading_zero, value)) return 31 - leading_zero; @@ -134,14 +134,14 @@ int __inline __builtin_clz(uint32_t value) { } #if defined(_M_ARM) || defined(_M_X64) -int __inline __builtin_clzll(uint64_t value) { +static int __inline __builtin_clzll(uint64_t value) { unsigned long leading_zero = 0; if (_BitScanReverse64(&leading_zero, value)) return 63 - leading_zero; return 64; } #else -int __inline __builtin_clzll(uint64_t value) { +static int __inline __builtin_clzll(uint64_t value) { if (value == 0) return 64; uint32_t msh = (uint32_t)(value >> 32); @@ -154,7 +154,7 @@ int __inline __builtin_clzll(uint64_t value) { #define __builtin_clzl __builtin_clzll -bool __inline __builtin_sadd_overflow(int x, int y, int *result) { +static bool __inline __builtin_sadd_overflow(int x, int y, int *result) { if ((x < 0) != (y < 0)) { *result = x + y; return false; diff --git a/compiler-rt/lib/dfsan/dfsan_custom.cpp b/compiler-rt/lib/dfsan/dfsan_custom.cpp index 05b48fd0525e..85b796bd6349 100644 --- a/compiler-rt/lib/dfsan/dfsan_custom.cpp +++ b/compiler-rt/lib/dfsan/dfsan_custom.cpp @@ -842,7 +842,7 @@ __dfsw_dlopen(const char *filename, int flag, dfsan_label filename_label, dfsan_label flag_label, dfsan_label *ret_label) { void *handle = dlopen(filename, flag); link_map *map = GET_LINK_MAP_BY_DLOPEN_HANDLE(handle); - if (map) + if (filename && map) ForEachMappedRegion(map, dfsan_set_zero_label); *ret_label = 0; return handle; @@ -2792,7 +2792,7 @@ int __dfsw_sprintf(char *str, const char *format, dfsan_label str_label, va_list ap; va_start(ap, ret_label); - int ret = format_buffer(str, ~0ul, format, va_labels, ret_label, nullptr, + int ret = format_buffer(str, INT32_MAX, format, va_labels, ret_label, nullptr, nullptr, ap); va_end(ap); return ret; @@ -2806,8 +2806,8 @@ int __dfso_sprintf(char *str, const char *format, dfsan_label str_label, dfsan_origin *ret_origin, ...) { va_list ap; va_start(ap, ret_origin); - int ret = format_buffer(str, ~0ul, format, va_labels, ret_label, va_origins, - ret_origin, ap); + int ret = format_buffer(str, INT32_MAX, format, va_labels, ret_label, + va_origins, ret_origin, ap); va_end(ap); return ret; } diff --git a/compiler-rt/lib/hwasan/hwasan_report.cpp b/compiler-rt/lib/hwasan/hwasan_report.cpp index 784cfb904aa2..12a4fa47f215 100644 --- a/compiler-rt/lib/hwasan/hwasan_report.cpp +++ b/compiler-rt/lib/hwasan/hwasan_report.cpp @@ -222,7 +222,7 @@ static void PrintStackAllocations(const StackAllocationsRingBuffer *sa, if (!local.has_frame_offset || !local.has_size || !local.has_tag_offset) continue; if (!(local.name && internal_strlen(local.name)) && - !(local.function_name && internal_strlen(local.name)) && + !(local.function_name && internal_strlen(local.function_name)) && !(local.decl_file && internal_strlen(local.decl_file))) continue; tag_t obj_tag = base_tag ^ local.tag_offset; diff --git a/compiler-rt/lib/interception/interception.h b/compiler-rt/lib/interception/interception.h index 9d8b60b2eef5..58e969378a90 100644 --- a/compiler-rt/lib/interception/interception.h +++ b/compiler-rt/lib/interception/interception.h @@ -205,8 +205,9 @@ const interpose_substitution substitution_##func_name[] \ ASM_TYPE_FUNCTION_STR "\n" \ SANITIZER_STRINGIFY(TRAMPOLINE(func)) ":\n" \ SANITIZER_STRINGIFY(CFI_STARTPROC) "\n" \ - SANITIZER_STRINGIFY(ASM_TAIL_CALL) " __interceptor_" \ - SANITIZER_STRINGIFY(ASM_PREEMPTIBLE_SYM(func)) "\n" \ + C_ASM_TAIL_CALL(SANITIZER_STRINGIFY(TRAMPOLINE(func)), \ + "__interceptor_" \ + SANITIZER_STRINGIFY(ASM_PREEMPTIBLE_SYM(func))) "\n" \ SANITIZER_STRINGIFY(CFI_ENDPROC) "\n" \ ".size " SANITIZER_STRINGIFY(TRAMPOLINE(func)) ", " \ ".-" SANITIZER_STRINGIFY(TRAMPOLINE(func)) "\n" \ diff --git a/compiler-rt/lib/interception/interception_win.cpp b/compiler-rt/lib/interception/interception_win.cpp index f1a549b938e2..1829358705fe 100644 --- a/compiler-rt/lib/interception/interception_win.cpp +++ b/compiler-rt/lib/interception/interception_win.cpp @@ -945,19 +945,26 @@ bool OverrideFunction( static void **InterestingDLLsAvailable() { static const char *InterestingDLLs[] = { - "kernel32.dll", - "msvcr100.dll", // VS2010 - "msvcr110.dll", // VS2012 - "msvcr120.dll", // VS2013 - "vcruntime140.dll", // VS2015 - "ucrtbase.dll", // Universal CRT -#if (defined(__MINGW32__) && defined(__i386__)) - "libc++.dll", // libc++ - "libunwind.dll", // libunwind -#endif - // NTDLL should go last as it exports some functions that we should - // override in the CRT [presumably only used internally]. - "ntdll.dll", NULL}; + "kernel32.dll", + "msvcr100d.dll", // VS2010 + "msvcr110d.dll", // VS2012 + "msvcr120d.dll", // VS2013 + "vcruntime140d.dll", // VS2015 + "ucrtbased.dll", // Universal CRT + "msvcr100.dll", // VS2010 + "msvcr110.dll", // VS2012 + "msvcr120.dll", // VS2013 + "vcruntime140.dll", // VS2015 + "ucrtbase.dll", // Universal CRT +# if (defined(__MINGW32__) && defined(__i386__)) + "libc++.dll", // libc++ + "libunwind.dll", // libunwind +# endif + // NTDLL should go last as it exports some functions that we should + // override in the CRT [presumably only used internally]. + "ntdll.dll", + NULL + }; static void *result[ARRAY_SIZE(InterestingDLLs)] = { 0 }; if (!result[0]) { for (size_t i = 0, j = 0; InterestingDLLs[i]; ++i) { diff --git a/compiler-rt/lib/orc/coff_platform.cpp b/compiler-rt/lib/orc/coff_platform.cpp index 78c86c855dcb..9fe5c0b06289 100644 --- a/compiler-rt/lib/orc/coff_platform.cpp +++ b/compiler-rt/lib/orc/coff_platform.cpp @@ -509,7 +509,6 @@ Error COFFPlatformRuntimeState::deregisterObjectSections( << HeaderAddr.getValue(); return make_error<StringError>(ErrStream.str()); } - auto &JDState = I->second; for (auto &KV : Secs) { if (auto Err = deregisterBlockRange(HeaderAddr, KV.second)) return Err; @@ -687,7 +686,14 @@ struct ThrowInfo { ORC_RT_INTERFACE void __stdcall __orc_rt_coff_cxx_throw_exception( void *pExceptionObject, ThrowInfo *pThrowInfo) { +#ifdef __clang__ +#pragma clang diagnostic push +#pragma clang diagnostic ignored "-Wmultichar" +#endif constexpr uint32_t EH_EXCEPTION_NUMBER = 'msc' | 0xE0000000; +#ifdef __clang__ +#pragma clang diagnostic pop +#endif constexpr uint32_t EH_MAGIC_NUMBER1 = 0x19930520; auto BaseAddr = COFFPlatformRuntimeState::get().findJITDylibBaseByPC( reinterpret_cast<uint64_t>(pThrowInfo)); diff --git a/compiler-rt/lib/orc/wrapper_function_utils.h b/compiler-rt/lib/orc/wrapper_function_utils.h index dcb6d0e6addb..8009438547a3 100644 --- a/compiler-rt/lib/orc/wrapper_function_utils.h +++ b/compiler-rt/lib/orc/wrapper_function_utils.h @@ -296,11 +296,15 @@ public: // operation fails. detail::ResultDeserializer<SPSRetTagT, RetT>::makeSafe(Result); + // Since the functions cannot be zero/unresolved on Windows, the following + // reference taking would always be non-zero, thus generating a compiler + // warning otherwise. +#if !defined(_WIN32) if (ORC_RT_UNLIKELY(!&__orc_rt_jit_dispatch_ctx)) return make_error<StringError>("__orc_rt_jit_dispatch_ctx not set"); if (ORC_RT_UNLIKELY(!&__orc_rt_jit_dispatch)) return make_error<StringError>("__orc_rt_jit_dispatch not set"); - +#endif auto ArgBuffer = WrapperFunctionResult::fromSPSArgs<SPSArgList<SPSTagTs...>>(Args...); if (const char *ErrMsg = ArgBuffer.getOutOfBandError()) diff --git a/compiler-rt/lib/profile/InstrProfiling.h b/compiler-rt/lib/profile/InstrProfiling.h index 137115996748..012390833691 100644 --- a/compiler-rt/lib/profile/InstrProfiling.h +++ b/compiler-rt/lib/profile/InstrProfiling.h @@ -12,6 +12,17 @@ #include "InstrProfilingPort.h" #include <stdio.h> +// Make sure __LLVM_INSTR_PROFILE_GENERATE is always defined before +// including instr_prof_interface.h so the interface functions are +// declared correctly for the runtime. +// __LLVM_INSTR_PROFILE_GENERATE is always `#undef`ed after the header, +// because compiler-rt does not support profiling the profiling runtime itself. +#ifndef __LLVM_INSTR_PROFILE_GENERATE +#define __LLVM_INSTR_PROFILE_GENERATE +#endif +#include "profile/instr_prof_interface.h" +#undef __LLVM_INSTR_PROFILE_GENERATE + #define INSTR_PROF_VISIBILITY COMPILER_RT_VISIBILITY #include "profile/InstrProfData.inc" @@ -101,12 +112,6 @@ ValueProfNode *__llvm_profile_end_vnodes(); uint32_t *__llvm_profile_begin_orderfile(); /*! - * \brief Clear profile counters to zero. - * - */ -void __llvm_profile_reset_counters(void); - -/*! * \brief Merge profile data from buffer. * * Read profile data from buffer \p Profile and merge with in-process profile @@ -156,50 +161,6 @@ void __llvm_profile_instrument_target_value(uint64_t TargetValue, void *Data, int __llvm_profile_write_file(void); int __llvm_orderfile_write_file(void); -/*! - * \brief this is a wrapper interface to \c __llvm_profile_write_file. - * After this interface is invoked, an already dumped flag will be set - * so that profile won't be dumped again during program exit. - * Invocation of interface __llvm_profile_reset_counters will clear - * the flag. This interface is designed to be used to collect profile - * data from user selected hot regions. The use model is - * __llvm_profile_reset_counters(); - * ... hot region 1 - * __llvm_profile_dump(); - * .. some other code - * __llvm_profile_reset_counters(); - * ... hot region 2 - * __llvm_profile_dump(); - * - * It is expected that on-line profile merging is on with \c %m specifier - * used in profile filename . If merging is not turned on, user is expected - * to invoke __llvm_profile_set_filename to specify different profile names - * for different regions before dumping to avoid profile write clobbering. - */ -int __llvm_profile_dump(void); - -int __llvm_orderfile_dump(void); - -/*! - * \brief Set the filename for writing instrumentation data. - * - * Sets the filename to be used for subsequent calls to - * \a __llvm_profile_write_file(). - * - * \c Name is not copied, so it must remain valid. Passing NULL resets the - * filename logic to the default behaviour. - * - * Note: There may be multiple copies of the profile runtime (one for each - * instrumented image/DSO). This API only modifies the filename within the - * copy of the runtime available to the calling image. - * - * Warning: This is a no-op if continuous mode (\ref - * __llvm_profile_is_continuous_mode_enabled) is on. The reason for this is - * that in continuous mode, profile counters are mmap()'d to the profile at - * program initialization time. Support for transferring the mmap'd profile - * counts to a new file has not been implemented. - */ -void __llvm_profile_set_filename(const char *Name); /*! * \brief Set the FILE object for writing instrumentation data. Return 0 if set diff --git a/compiler-rt/lib/profile/InstrProfilingFile.c b/compiler-rt/lib/profile/InstrProfilingFile.c index 745c567f2167..867ae73f0d3b 100644 --- a/compiler-rt/lib/profile/InstrProfilingFile.c +++ b/compiler-rt/lib/profile/InstrProfilingFile.c @@ -335,10 +335,10 @@ static void initFileWriter(ProfDataWriter *This, FILE *File) { COMPILER_RT_VISIBILITY ProfBufferIO * lprofCreateBufferIOInternal(void *File, uint32_t BufferSz) { FreeHook = &free; - DynamicBufferIOBuffer = (uint8_t *)calloc(BufferSz, 1); + DynamicBufferIOBuffer = (uint8_t *)calloc(1, BufferSz); VPBufferSize = BufferSz; ProfDataWriter *fileWriter = - (ProfDataWriter *)calloc(sizeof(ProfDataWriter), 1); + (ProfDataWriter *)calloc(1, sizeof(ProfDataWriter)); initFileWriter(fileWriter, File); ProfBufferIO *IO = lprofCreateBufferIO(fileWriter); IO->OwnFileWriter = 1; @@ -702,10 +702,15 @@ static void resetFilenameToDefault(void) { #ifdef __GNUC__ #pragma GCC diagnostic push #pragma GCC diagnostic ignored "-Wcast-qual" +#elif defined(__clang__) +#pragma clang diagnostic push +#pragma clang diagnostic ignored "-Wcast-qual" #endif free((void *)lprofCurFilename.FilenamePat); #ifdef __GNUC__ #pragma GCC diagnostic pop +#elif defined(__clang__) +#pragma clang diagnostic pop #endif } memset(&lprofCurFilename, 0, sizeof(lprofCurFilename)); @@ -751,6 +756,9 @@ static int parseFilenamePattern(const char *FilenamePat, #ifdef __GNUC__ #pragma GCC diagnostic push #pragma GCC diagnostic ignored "-Wcast-qual" +#elif defined(__clang__) +#pragma clang diagnostic push +#pragma clang diagnostic ignored "-Wcast-qual" #endif /* Clean up cached prefix and filename. */ if (lprofCurFilename.ProfilePathPrefix) @@ -761,6 +769,8 @@ static int parseFilenamePattern(const char *FilenamePat, } #ifdef __GNUC__ #pragma GCC diagnostic pop +#elif defined(__clang__) +#pragma clang diagnostic pop #endif memset(&lprofCurFilename, 0, sizeof(lprofCurFilename)); diff --git a/compiler-rt/lib/profile/InstrProfilingMerge.c b/compiler-rt/lib/profile/InstrProfilingMerge.c index e8ae2189cccd..b5850e99ee37 100644 --- a/compiler-rt/lib/profile/InstrProfilingMerge.c +++ b/compiler-rt/lib/profile/InstrProfilingMerge.c @@ -41,6 +41,9 @@ uint64_t lprofGetLoadModuleSignature(void) { #ifdef __GNUC__ #pragma GCC diagnostic push #pragma GCC diagnostic ignored "-Wcast-qual" +#elif defined(__clang__) +#pragma clang diagnostic push +#pragma clang diagnostic ignored "-Wcast-qual" #endif /* Returns 1 if profile is not structurally compatible. */ @@ -234,4 +237,6 @@ int __llvm_profile_merge_from_buffer(const char *ProfileData, #ifdef __GNUC__ #pragma GCC diagnostic pop +#elif defined(__clang__) +#pragma clang diagnostic pop #endif diff --git a/compiler-rt/lib/profile/InstrProfilingValue.c b/compiler-rt/lib/profile/InstrProfilingValue.c index 3d7c245f795f..a608d41d39e7 100644 --- a/compiler-rt/lib/profile/InstrProfilingValue.c +++ b/compiler-rt/lib/profile/InstrProfilingValue.c @@ -62,10 +62,15 @@ __llvm_profile_set_num_value_sites(__llvm_profile_data *Data, #ifdef __GNUC__ #pragma GCC diagnostic push #pragma GCC diagnostic ignored "-Wcast-qual" +#elif defined(__clang__) +#pragma clang diagnostic push +#pragma clang diagnostic ignored "-Wcast-qual" #endif *((uint16_t *)&Data->NumValueSites[ValueKind]) = NumValueSites; #ifdef __GNUC__ #pragma GCC diagnostic pop +#elif defined(__clang__) +#pragma clang diagnostic pop #endif } diff --git a/compiler-rt/lib/profile/WindowsMMap.c b/compiler-rt/lib/profile/WindowsMMap.c index 07c0a689feae..9d7da835b1ed 100644 --- a/compiler-rt/lib/profile/WindowsMMap.c +++ b/compiler-rt/lib/profile/WindowsMMap.c @@ -124,8 +124,7 @@ int madvise(void *addr, size_t length, int advice) return 0; } -COMPILER_RT_VISIBILITY -int lock(HANDLE handle, DWORD lockType, BOOL blocking) { +static int lock(HANDLE handle, DWORD lockType, BOOL blocking) { DWORD flags = lockType; if (!blocking) flags |= LOCKFILE_FAIL_IMMEDIATELY; diff --git a/compiler-rt/lib/profile/WindowsMMap.h b/compiler-rt/lib/profile/WindowsMMap.h index 68b8de2398d6..1df1a0be0b02 100644 --- a/compiler-rt/lib/profile/WindowsMMap.h +++ b/compiler-rt/lib/profile/WindowsMMap.h @@ -60,6 +60,12 @@ # define DWORD_LO(x) (x) #endif +#define mmap __llvm_profile_mmap +#define munmap __llvm_profile_munmap +#define msync __llvm_profile_msync +#define madvise __llvm_profile_madvise +#define flock __llvm_profile_flock + void *mmap(void *start, size_t length, int prot, int flags, int fd, off_t offset); diff --git a/compiler-rt/lib/sanitizer_common/sanitizer_asm.h b/compiler-rt/lib/sanitizer_common/sanitizer_asm.h index bbb18cfbdf15..3af66a4e4499 100644 --- a/compiler-rt/lib/sanitizer_common/sanitizer_asm.h +++ b/compiler-rt/lib/sanitizer_common/sanitizer_asm.h @@ -53,6 +53,29 @@ # define ASM_TAIL_CALL tail #endif +// Currently, almost all of the shared libraries rely on the value of +// $t9 to get the address of current function, instead of PCREL, even +// on MIPSr6. To be compatiable with them, we have to set $t9 properly. +// MIPS uses GOT to get the address of preemptible functions. +#if defined(__mips64) +# define C_ASM_TAIL_CALL(t_func, i_func) \ + "lui $t8, %hi(%neg(%gp_rel(" t_func ")))\n" \ + "daddu $t8, $t8, $t9\n" \ + "daddiu $t8, $t8, %lo(%neg(%gp_rel(" t_func ")))\n" \ + "ld $t9, %got_disp(" i_func ")($t8)\n" \ + "jr $t9\n" +#elif defined(__mips__) +# define C_ASM_TAIL_CALL(t_func, i_func) \ + ".set noreorder\n" \ + ".cpload $t9\n" \ + ".set reorder\n" \ + "lw $t9, %got(" i_func ")($gp)\n" \ + "jr $t9\n" +#elif defined(ASM_TAIL_CALL) +# define C_ASM_TAIL_CALL(t_func, i_func) \ + SANITIZER_STRINGIFY(ASM_TAIL_CALL) " " i_func +#endif + #if defined(__ELF__) && defined(__x86_64__) || defined(__i386__) || \ defined(__riscv) # define ASM_PREEMPTIBLE_SYM(sym) sym@plt diff --git a/compiler-rt/lib/sanitizer_common/sanitizer_common_interceptors.inc b/compiler-rt/lib/sanitizer_common/sanitizer_common_interceptors.inc index 77fa1b4965a7..1b56bebac64e 100644 --- a/compiler-rt/lib/sanitizer_common/sanitizer_common_interceptors.inc +++ b/compiler-rt/lib/sanitizer_common/sanitizer_common_interceptors.inc @@ -9965,7 +9965,13 @@ INTERCEPTOR(void, sl_free, void *sl, int freeall) { INTERCEPTOR(SSIZE_T, getrandom, void *buf, SIZE_T buflen, unsigned int flags) { void *ctx; COMMON_INTERCEPTOR_ENTER(ctx, getrandom, buf, buflen, flags); - SSIZE_T n = REAL(getrandom)(buf, buflen, flags); + // If GRND_NONBLOCK is set in the flags, it is non blocking. + static const int grnd_nonblock = 1; + SSIZE_T n; + if ((flags & grnd_nonblock)) + n = REAL(getrandom)(buf, buflen, flags); + else + n = COMMON_INTERCEPTOR_BLOCK_REAL(getrandom)(buf, buflen, flags); if (n > 0) { COMMON_INTERCEPTOR_WRITE_RANGE(ctx, buf, n); } diff --git a/compiler-rt/lib/sanitizer_common/sanitizer_fuchsia.cpp b/compiler-rt/lib/sanitizer_common/sanitizer_fuchsia.cpp index 0245164403c5..5f4f8c8c0078 100644 --- a/compiler-rt/lib/sanitizer_common/sanitizer_fuchsia.cpp +++ b/compiler-rt/lib/sanitizer_common/sanitizer_fuchsia.cpp @@ -129,6 +129,60 @@ uptr GetMaxVirtualAddress() { return GetMaxUserVirtualAddress(); } bool ErrorIsOOM(error_t err) { return err == ZX_ERR_NO_MEMORY; } +// For any sanitizer internal that needs to map something which can be unmapped +// later, first attempt to map to a pre-allocated VMAR. This helps reduce +// fragmentation from many small anonymous mmap calls. A good value for this +// VMAR size would be the total size of your typical sanitizer internal objects +// allocated in an "average" process lifetime. Examples of this include: +// FakeStack, LowLevelAllocator mappings, TwoLevelMap, InternalMmapVector, +// StackStore, CreateAsanThread, etc. +// +// This is roughly equal to the total sum of sanitizer internal mappings for a +// large test case. +constexpr size_t kSanitizerHeapVmarSize = 13ULL << 20; +static zx_handle_t gSanitizerHeapVmar = ZX_HANDLE_INVALID; + +static zx_status_t GetSanitizerHeapVmar(zx_handle_t *vmar) { + zx_status_t status = ZX_OK; + if (gSanitizerHeapVmar == ZX_HANDLE_INVALID) { + CHECK_EQ(kSanitizerHeapVmarSize % GetPageSizeCached(), 0); + uintptr_t base; + status = _zx_vmar_allocate( + _zx_vmar_root_self(), + ZX_VM_CAN_MAP_READ | ZX_VM_CAN_MAP_WRITE | ZX_VM_CAN_MAP_SPECIFIC, 0, + kSanitizerHeapVmarSize, &gSanitizerHeapVmar, &base); + } + *vmar = gSanitizerHeapVmar; + if (status == ZX_OK) + CHECK_NE(gSanitizerHeapVmar, ZX_HANDLE_INVALID); + return status; +} + +static zx_status_t TryVmoMapSanitizerVmar(zx_vm_option_t options, + size_t vmar_offset, zx_handle_t vmo, + size_t size, uintptr_t *addr, + zx_handle_t *vmar_used = nullptr) { + zx_handle_t vmar; + zx_status_t status = GetSanitizerHeapVmar(&vmar); + if (status != ZX_OK) + return status; + + status = _zx_vmar_map(gSanitizerHeapVmar, options, vmar_offset, vmo, + /*vmo_offset=*/0, size, addr); + if (vmar_used) + *vmar_used = gSanitizerHeapVmar; + if (status == ZX_ERR_NO_RESOURCES || status == ZX_ERR_INVALID_ARGS) { + // This means there's no space in the heap VMAR, so fallback to the root + // VMAR. + status = _zx_vmar_map(_zx_vmar_root_self(), options, vmar_offset, vmo, + /*vmo_offset=*/0, size, addr); + if (vmar_used) + *vmar_used = _zx_vmar_root_self(); + } + + return status; +} + static void *DoAnonymousMmapOrDie(uptr size, const char *mem_type, bool raw_report, bool die_for_nomem) { size = RoundUpTo(size, GetPageSize()); @@ -144,11 +198,9 @@ static void *DoAnonymousMmapOrDie(uptr size, const char *mem_type, _zx_object_set_property(vmo, ZX_PROP_NAME, mem_type, internal_strlen(mem_type)); - // TODO(mcgrathr): Maybe allocate a VMAR for all sanitizer heap and use that? uintptr_t addr; - status = - _zx_vmar_map(_zx_vmar_root_self(), ZX_VM_PERM_READ | ZX_VM_PERM_WRITE, 0, - vmo, 0, size, &addr); + status = TryVmoMapSanitizerVmar(ZX_VM_PERM_READ | ZX_VM_PERM_WRITE, + /*vmar_offset=*/0, vmo, size, &addr); _zx_handle_close(vmo); if (status != ZX_OK) { @@ -243,6 +295,12 @@ void UnmapOrDieVmar(void *addr, uptr size, zx_handle_t target_vmar) { zx_status_t status = _zx_vmar_unmap(target_vmar, reinterpret_cast<uintptr_t>(addr), size); + if (status == ZX_ERR_INVALID_ARGS && target_vmar == gSanitizerHeapVmar) { + // If there wasn't any space in the heap vmar, the fallback was the root + // vmar. + status = _zx_vmar_unmap(_zx_vmar_root_self(), + reinterpret_cast<uintptr_t>(addr), size); + } if (status != ZX_OK) { Report("ERROR: %s failed to deallocate 0x%zx (%zd) bytes at address %p\n", SanitizerToolName, size, size, addr); @@ -308,17 +366,16 @@ void *MmapAlignedOrDieOnFatalError(uptr size, uptr alignment, _zx_object_set_property(vmo, ZX_PROP_NAME, mem_type, internal_strlen(mem_type)); - // TODO(mcgrathr): Maybe allocate a VMAR for all sanitizer heap and use that? - // Map a larger size to get a chunk of address space big enough that // it surely contains an aligned region of the requested size. Then // overwrite the aligned middle portion with a mapping from the // beginning of the VMO, and unmap the excess before and after. size_t map_size = size + alignment; uintptr_t addr; - status = - _zx_vmar_map(_zx_vmar_root_self(), ZX_VM_PERM_READ | ZX_VM_PERM_WRITE, 0, - vmo, 0, map_size, &addr); + zx_handle_t vmar_used; + status = TryVmoMapSanitizerVmar(ZX_VM_PERM_READ | ZX_VM_PERM_WRITE, + /*vmar_offset=*/0, vmo, map_size, &addr, + &vmar_used); if (status == ZX_OK) { uintptr_t map_addr = addr; uintptr_t map_end = map_addr + map_size; @@ -326,12 +383,12 @@ void *MmapAlignedOrDieOnFatalError(uptr size, uptr alignment, uintptr_t end = addr + size; if (addr != map_addr) { zx_info_vmar_t info; - status = _zx_object_get_info(_zx_vmar_root_self(), ZX_INFO_VMAR, &info, - sizeof(info), NULL, NULL); + status = _zx_object_get_info(vmar_used, ZX_INFO_VMAR, &info, sizeof(info), + NULL, NULL); if (status == ZX_OK) { uintptr_t new_addr; status = _zx_vmar_map( - _zx_vmar_root_self(), + vmar_used, ZX_VM_PERM_READ | ZX_VM_PERM_WRITE | ZX_VM_SPECIFIC_OVERWRITE, addr - info.base, vmo, 0, size, &new_addr); if (status == ZX_OK) @@ -339,9 +396,9 @@ void *MmapAlignedOrDieOnFatalError(uptr size, uptr alignment, } } if (status == ZX_OK && addr != map_addr) - status = _zx_vmar_unmap(_zx_vmar_root_self(), map_addr, addr - map_addr); + status = _zx_vmar_unmap(vmar_used, map_addr, addr - map_addr); if (status == ZX_OK && end != map_end) - status = _zx_vmar_unmap(_zx_vmar_root_self(), end, map_end - end); + status = _zx_vmar_unmap(vmar_used, end, map_end - end); } _zx_handle_close(vmo); @@ -357,7 +414,7 @@ void *MmapAlignedOrDieOnFatalError(uptr size, uptr alignment, } void UnmapOrDie(void *addr, uptr size) { - UnmapOrDieVmar(addr, size, _zx_vmar_root_self()); + UnmapOrDieVmar(addr, size, gSanitizerHeapVmar); } void ReleaseMemoryPagesToOS(uptr beg, uptr end) { diff --git a/compiler-rt/lib/sanitizer_common/sanitizer_internal_defs.h b/compiler-rt/lib/sanitizer_common/sanitizer_internal_defs.h index 3809669dd48b..992721757e88 100644 --- a/compiler-rt/lib/sanitizer_common/sanitizer_internal_defs.h +++ b/compiler-rt/lib/sanitizer_common/sanitizer_internal_defs.h @@ -35,13 +35,20 @@ # define SANITIZER_INTERFACE_ATTRIBUTE __declspec(dllexport) #endif # define SANITIZER_WEAK_ATTRIBUTE +# define SANITIZER_WEAK_IMPORT #elif SANITIZER_GO # define SANITIZER_INTERFACE_ATTRIBUTE # define SANITIZER_WEAK_ATTRIBUTE +# define SANITIZER_WEAK_IMPORT #else # define SANITIZER_INTERFACE_ATTRIBUTE __attribute__((visibility("default"))) # define SANITIZER_WEAK_ATTRIBUTE __attribute__((weak)) -#endif +# if SANITIZER_APPLE +# define SANITIZER_WEAK_IMPORT extern "C" __attribute((weak_import)) +# else +# define SANITIZER_WEAK_IMPORT extern "C" SANITIZER_WEAK_ATTRIBUTE +# endif // SANITIZER_APPLE +#endif // SANITIZER_WINDOWS //--------------------------- WEAK FUNCTIONS ---------------------------------// // When working with weak functions, to simplify the code and make it more diff --git a/compiler-rt/lib/sanitizer_common/sanitizer_symbolizer_report.cpp b/compiler-rt/lib/sanitizer_common/sanitizer_symbolizer_report.cpp index 253dc10607a6..8438e019591b 100644 --- a/compiler-rt/lib/sanitizer_common/sanitizer_symbolizer_report.cpp +++ b/compiler-rt/lib/sanitizer_common/sanitizer_symbolizer_report.cpp @@ -34,7 +34,8 @@ static bool FrameIsInternal(const SymbolizedStack *frame) { return true; const char *file = frame->info.file; const char *module = frame->info.module; - if (file && (internal_strstr(file, "/compiler-rt/lib/"))) + if (file && (internal_strstr(file, "/compiler-rt/lib/") || + internal_strstr(file, "/include/c++/"))) return true; if (module && (internal_strstr(module, "libclang_rt."))) return true; diff --git a/compiler-rt/lib/sanitizer_common/symbolizer/scripts/build_symbolizer.sh b/compiler-rt/lib/sanitizer_common/symbolizer/scripts/build_symbolizer.sh index c9e31be61cb7..f24d42cc84e4 100755 --- a/compiler-rt/lib/sanitizer_common/symbolizer/scripts/build_symbolizer.sh +++ b/compiler-rt/lib/sanitizer_common/symbolizer/scripts/build_symbolizer.sh @@ -104,6 +104,7 @@ if [[ ! -f ${LLVM_BUILD}/build.ninja ]]; then -DCMAKE_CXX_FLAGS_RELEASE="${LIBCXX_FLAGS}" \ -DLIBCXXABI_ENABLE_ASSERTIONS=OFF \ -DLIBCXXABI_ENABLE_EXCEPTIONS=OFF \ + -DLIBCXXABI_USE_LLVM_UNWINDER=OFF \ -DLIBCXX_ENABLE_ASSERTIONS=OFF \ -DLIBCXX_ENABLE_EXCEPTIONS=OFF \ -DLIBCXX_ENABLE_RTTI=OFF \ diff --git a/compiler-rt/lib/tsan/rtl/tsan_dispatch_defs.h b/compiler-rt/lib/tsan/rtl/tsan_dispatch_defs.h index 54c0b0ba4b40..8d38beb0b0a2 100644 --- a/compiler-rt/lib/tsan/rtl/tsan_dispatch_defs.h +++ b/compiler-rt/lib/tsan/rtl/tsan_dispatch_defs.h @@ -56,13 +56,6 @@ extern const dispatch_block_t _dispatch_data_destructor_munmap; # define DISPATCH_NOESCAPE #endif -#if SANITIZER_APPLE -# define SANITIZER_WEAK_IMPORT extern "C" __attribute((weak_import)) -#else -# define SANITIZER_WEAK_IMPORT extern "C" __attribute((weak)) -#endif - - // Data types used in dispatch APIs typedef unsigned long size_t; typedef unsigned long uintptr_t; diff --git a/compiler-rt/lib/tsan/rtl/tsan_platform.h b/compiler-rt/lib/tsan/rtl/tsan_platform.h index 84ff4bfade09..377f8aeb8d66 100644 --- a/compiler-rt/lib/tsan/rtl/tsan_platform.h +++ b/compiler-rt/lib/tsan/rtl/tsan_platform.h @@ -1024,7 +1024,7 @@ inline uptr RestoreAddr(uptr addr) { void InitializePlatform(); void InitializePlatformEarly(); -void CheckAndProtect(); +bool CheckAndProtect(bool protect, bool ignore_heap, bool print_warnings); void InitializeShadowMemoryPlatform(); void WriteMemoryProfile(char *buf, uptr buf_size, u64 uptime_ns); int ExtractResolvFDs(void *state, int *fds, int nfd); diff --git a/compiler-rt/lib/tsan/rtl/tsan_platform_linux.cpp b/compiler-rt/lib/tsan/rtl/tsan_platform_linux.cpp index b45adea45b27..0d0b1aba1f85 100644 --- a/compiler-rt/lib/tsan/rtl/tsan_platform_linux.cpp +++ b/compiler-rt/lib/tsan/rtl/tsan_platform_linux.cpp @@ -214,6 +214,88 @@ void InitializeShadowMemoryPlatform() { #endif // #if !SANITIZER_GO +# if !SANITIZER_GO +static void ReExecIfNeeded() { + // Go maps shadow memory lazily and works fine with limited address space. + // Unlimited stack is not a problem as well, because the executable + // is not compiled with -pie. + bool reexec = false; + // TSan doesn't play well with unlimited stack size (as stack + // overlaps with shadow memory). If we detect unlimited stack size, + // we re-exec the program with limited stack size as a best effort. + if (StackSizeIsUnlimited()) { + const uptr kMaxStackSize = 32 * 1024 * 1024; + VReport(1, + "Program is run with unlimited stack size, which wouldn't " + "work with ThreadSanitizer.\n" + "Re-execing with stack size limited to %zd bytes.\n", + kMaxStackSize); + SetStackSizeLimitInBytes(kMaxStackSize); + reexec = true; + } + + if (!AddressSpaceIsUnlimited()) { + Report( + "WARNING: Program is run with limited virtual address space," + " which wouldn't work with ThreadSanitizer.\n"); + Report("Re-execing with unlimited virtual address space.\n"); + SetAddressSpaceUnlimited(); + reexec = true; + } + +# if SANITIZER_LINUX + // ASLR personality check. + int old_personality = personality(0xffffffff); + bool aslr_on = + (old_personality != -1) && ((old_personality & ADDR_NO_RANDOMIZE) == 0); + +# if SANITIZER_ANDROID && (defined(__aarch64__) || defined(__x86_64__)) + // After patch "arm64: mm: support ARCH_MMAP_RND_BITS." is introduced in + // linux kernel, the random gap between stack and mapped area is increased + // from 128M to 36G on 39-bit aarch64. As it is almost impossible to cover + // this big range, we should disable randomized virtual space on aarch64. + if (aslr_on) { + VReport(1, + "WARNING: Program is run with randomized virtual address " + "space, which wouldn't work with ThreadSanitizer on Android.\n" + "Re-execing with fixed virtual address space.\n"); + CHECK_NE(personality(old_personality | ADDR_NO_RANDOMIZE), -1); + reexec = true; + } +# endif + + if (reexec) { + // Don't check the address space since we're going to re-exec anyway. + } else if (!CheckAndProtect(false, false, false)) { + if (aslr_on) { + // Disable ASLR if the memory layout was incompatible. + // Alternatively, we could just keep re-execing until we get lucky + // with a compatible randomized layout, but the risk is that if it's + // not an ASLR-related issue, we will be stuck in an infinite loop of + // re-execing (unless we change ReExec to pass a parameter of the + // number of retries allowed.) + VReport(1, + "WARNING: ThreadSanitizer: memory layout is incompatible, " + "possibly due to high-entropy ASLR.\n" + "Re-execing with fixed virtual address space.\n" + "N.B. reducing ASLR entropy is preferable.\n"); + CHECK_NE(personality(old_personality | ADDR_NO_RANDOMIZE), -1); + reexec = true; + } else { + VReport(1, + "FATAL: ThreadSanitizer: memory layout is incompatible, " + "even though ASLR is disabled.\n" + "Please file a bug.\n"); + Die(); + } + } +# endif // SANITIZER_LINUX + + if (reexec) + ReExec(); +} +# endif + void InitializePlatformEarly() { vmaSize = (MostSignificantSetBitIndex(GET_CURRENT_FRAME()) + 1); @@ -284,6 +366,10 @@ void InitializePlatformEarly() { } # endif # endif + +# if !SANITIZER_GO + ReExecIfNeeded(); +# endif } void InitializePlatform() { @@ -294,52 +380,22 @@ void InitializePlatform() { // is not compiled with -pie. #if !SANITIZER_GO { - bool reexec = false; - // TSan doesn't play well with unlimited stack size (as stack - // overlaps with shadow memory). If we detect unlimited stack size, - // we re-exec the program with limited stack size as a best effort. - if (StackSizeIsUnlimited()) { - const uptr kMaxStackSize = 32 * 1024 * 1024; - VReport(1, "Program is run with unlimited stack size, which wouldn't " - "work with ThreadSanitizer.\n" - "Re-execing with stack size limited to %zd bytes.\n", - kMaxStackSize); - SetStackSizeLimitInBytes(kMaxStackSize); - reexec = true; - } - - if (!AddressSpaceIsUnlimited()) { - Report("WARNING: Program is run with limited virtual address space," - " which wouldn't work with ThreadSanitizer.\n"); - Report("Re-execing with unlimited virtual address space.\n"); - SetAddressSpaceUnlimited(); - reexec = true; - } -#if SANITIZER_ANDROID && (defined(__aarch64__) || defined(__x86_64__)) - // After patch "arm64: mm: support ARCH_MMAP_RND_BITS." is introduced in - // linux kernel, the random gap between stack and mapped area is increased - // from 128M to 36G on 39-bit aarch64. As it is almost impossible to cover - // this big range, we should disable randomized virtual space on aarch64. - // ASLR personality check. - int old_personality = personality(0xffffffff); - if (old_personality != -1 && (old_personality & ADDR_NO_RANDOMIZE) == 0) { - VReport(1, "WARNING: Program is run with randomized virtual address " - "space, which wouldn't work with ThreadSanitizer.\n" - "Re-execing with fixed virtual address space.\n"); - CHECK_NE(personality(old_personality | ADDR_NO_RANDOMIZE), -1); - reexec = true; - } - -#endif -#if SANITIZER_LINUX && (defined(__aarch64__) || defined(__loongarch_lp64)) +# if SANITIZER_LINUX && (defined(__aarch64__) || defined(__loongarch_lp64)) // Initialize the xor key used in {sig}{set,long}jump. InitializeLongjmpXorKey(); -#endif - if (reexec) - ReExec(); +# endif + } + + // Earlier initialization steps already re-exec'ed until we got a compatible + // memory layout, so we don't expect any more issues here. + if (!CheckAndProtect(true, true, true)) { + Printf( + "FATAL: ThreadSanitizer: unexpectedly found incompatible memory " + "layout.\n"); + Printf("FATAL: Please file a bug.\n"); + Die(); } - CheckAndProtect(); InitTlsSize(); #endif // !SANITIZER_GO } diff --git a/compiler-rt/lib/tsan/rtl/tsan_platform_mac.cpp b/compiler-rt/lib/tsan/rtl/tsan_platform_mac.cpp index 1aac0fb27520..07d83e1a9a9f 100644 --- a/compiler-rt/lib/tsan/rtl/tsan_platform_mac.cpp +++ b/compiler-rt/lib/tsan/rtl/tsan_platform_mac.cpp @@ -239,7 +239,10 @@ static uptr longjmp_xor_key = 0; void InitializePlatform() { DisableCoreDumperIfNecessary(); #if !SANITIZER_GO - CheckAndProtect(); + if (!CheckAndProtect(true, true, true)) { + Printf("FATAL: ThreadSanitizer: found incompatible memory layout.\n"); + Die(); + } InitializeThreadStateStorage(); diff --git a/compiler-rt/lib/tsan/rtl/tsan_platform_posix.cpp b/compiler-rt/lib/tsan/rtl/tsan_platform_posix.cpp index e7dcd664dc0d..7d5a992dc665 100644 --- a/compiler-rt/lib/tsan/rtl/tsan_platform_posix.cpp +++ b/compiler-rt/lib/tsan/rtl/tsan_platform_posix.cpp @@ -94,22 +94,51 @@ static void ProtectRange(uptr beg, uptr end) { } } -void CheckAndProtect() { +// CheckAndProtect will check if the memory layout is compatible with TSan. +// Optionally (if 'protect' is true), it will set the memory regions between +// app memory to be inaccessible. +// 'ignore_heap' means it will not consider heap memory allocations to be a +// conflict. Set this based on whether we are calling CheckAndProtect before +// or after the allocator has initialized the heap. +bool CheckAndProtect(bool protect, bool ignore_heap, bool print_warnings) { // Ensure that the binary is indeed compiled with -pie. MemoryMappingLayout proc_maps(true); MemoryMappedSegment segment; while (proc_maps.Next(&segment)) { - if (IsAppMem(segment.start)) continue; + if (segment.start >= HeapMemBeg() && segment.end <= HeapEnd()) { + if (ignore_heap) { + continue; + } else { + return false; + } + } + + // Note: IsAppMem includes if it is heap memory, hence we must + // put this check after the heap bounds check. + if (IsAppMem(segment.start) && IsAppMem(segment.end - 1)) + continue; + + // Guard page after the heap end if (segment.start >= HeapMemEnd() && segment.start < HeapEnd()) continue; + if (segment.protection == 0) // Zero page or mprotected. continue; + if (segment.start >= VdsoBeg()) // vdso break; - Printf("FATAL: ThreadSanitizer: unexpected memory mapping 0x%zx-0x%zx\n", - segment.start, segment.end); - Die(); + + // Debug output can break tests. Suppress this message in most cases. + if (print_warnings) + Printf( + "WARNING: ThreadSanitizer: unexpected memory mapping 0x%zx-0x%zx\n", + segment.start, segment.end); + + return false; } + if (!protect) + return true; + # if SANITIZER_IOS && !SANITIZER_IOSSIM ProtectRange(HeapMemEnd(), ShadowBeg()); ProtectRange(ShadowEnd(), MetaShadowBeg()); @@ -135,8 +164,10 @@ void CheckAndProtect() { // Older s390x kernels may not support 5-level page tables. TryProtectRange(user_addr_max_l4, user_addr_max_l5); #endif + + return true; } -#endif +# endif } // namespace __tsan |