diff options
Diffstat (limited to 'compiler-rt/lib')
34 files changed, 765 insertions, 191 deletions
diff --git a/compiler-rt/lib/asan/asan_interceptors.cpp b/compiler-rt/lib/asan/asan_interceptors.cpp index 13311b7e409b..817008253fc0 100644 --- a/compiler-rt/lib/asan/asan_interceptors.cpp +++ b/compiler-rt/lib/asan/asan_interceptors.cpp @@ -243,13 +243,26 @@ DEFINE_REAL_PTHREAD_FUNCTIONS #if ASAN_INTERCEPT_SWAPCONTEXT static void ClearShadowMemoryForContextStack(uptr stack, uptr ssize) { + // Only clear if we know the stack. This should be true only for contexts + // created with makecontext(). + if (!ssize) + return; // Align to page size. uptr PageSize = GetPageSizeCached(); - uptr bottom = stack & ~(PageSize - 1); + uptr bottom = RoundDownTo(stack, PageSize); + if (!AddrIsInMem(bottom)) + return; ssize += stack - bottom; ssize = RoundUpTo(ssize, PageSize); - if (AddrIsInMem(bottom) && ssize) - PoisonShadow(bottom, ssize, 0); + PoisonShadow(bottom, ssize, 0); +} + +INTERCEPTOR(int, getcontext, struct ucontext_t *ucp) { + // API does not requires to have ucp clean, and sets only part of fields. We + // use ucp->uc_stack to unpoison new stack. We prefer to have zeroes then + // uninitialized bytes. + ResetContextStack(ucp); + return REAL(getcontext)(ucp); } INTERCEPTOR(int, swapcontext, struct ucontext_t *oucp, @@ -265,15 +278,18 @@ INTERCEPTOR(int, swapcontext, struct ucontext_t *oucp, uptr stack, ssize; ReadContextStack(ucp, &stack, &ssize); ClearShadowMemoryForContextStack(stack, ssize); -#if __has_attribute(__indirect_return__) && \ - (defined(__x86_64__) || defined(__i386__)) + + // See getcontext interceptor. + ResetContextStack(oucp); + +# if __has_attribute(__indirect_return__) && \ + (defined(__x86_64__) || defined(__i386__)) int (*real_swapcontext)(struct ucontext_t *, struct ucontext_t *) - __attribute__((__indirect_return__)) - = REAL(swapcontext); + __attribute__((__indirect_return__)) = REAL(swapcontext); int res = real_swapcontext(oucp, ucp); -#else +# else int res = REAL(swapcontext)(oucp, ucp); -#endif +# endif // swapcontext technically does not return, but program may swap context to // "oucp" later, that would look as if swapcontext() returned 0. // We need to clear shadow for ucp once again, as it may be in arbitrary @@ -643,6 +659,7 @@ void InitializeAsanInterceptors() { ASAN_INTERCEPT_FUNC(longjmp); #if ASAN_INTERCEPT_SWAPCONTEXT + ASAN_INTERCEPT_FUNC(getcontext); ASAN_INTERCEPT_FUNC(swapcontext); #endif #if ASAN_INTERCEPT__LONGJMP diff --git a/compiler-rt/lib/asan/asan_interceptors_vfork.S b/compiler-rt/lib/asan/asan_interceptors_vfork.S index 3ae5503e83cd..ec29adc7b132 100644 --- a/compiler-rt/lib/asan/asan_interceptors_vfork.S +++ b/compiler-rt/lib/asan/asan_interceptors_vfork.S @@ -6,6 +6,7 @@ #include "sanitizer_common/sanitizer_common_interceptors_vfork_aarch64.inc.S" #include "sanitizer_common/sanitizer_common_interceptors_vfork_arm.inc.S" #include "sanitizer_common/sanitizer_common_interceptors_vfork_i386.inc.S" +#include "sanitizer_common/sanitizer_common_interceptors_vfork_loongarch64.inc.S" #include "sanitizer_common/sanitizer_common_interceptors_vfork_riscv64.inc.S" #include "sanitizer_common/sanitizer_common_interceptors_vfork_x86_64.inc.S" #endif diff --git a/compiler-rt/lib/asan/asan_internal.h b/compiler-rt/lib/asan/asan_internal.h index 7468f126d37b..9c46f225116e 100644 --- a/compiler-rt/lib/asan/asan_internal.h +++ b/compiler-rt/lib/asan/asan_internal.h @@ -106,6 +106,7 @@ void AsanApplyToGlobals(globals_op_fptr op, const void *needle); void AsanOnDeadlySignal(int, void *siginfo, void *context); void ReadContextStack(void *context, uptr *stack, uptr *ssize); +void ResetContextStack(void *context); void StopInitOrderChecking(); // Wrapper for TLS/TSD. diff --git a/compiler-rt/lib/asan/asan_linux.cpp b/compiler-rt/lib/asan/asan_linux.cpp index defd81bc19e2..89450fc120a0 100644 --- a/compiler-rt/lib/asan/asan_linux.cpp +++ b/compiler-rt/lib/asan/asan_linux.cpp @@ -214,11 +214,19 @@ void ReadContextStack(void *context, uptr *stack, uptr *ssize) { *stack = (uptr)ucp->uc_stack.ss_sp; *ssize = ucp->uc_stack.ss_size; } -#else + +void ResetContextStack(void *context) { + ucontext_t *ucp = (ucontext_t *)context; + ucp->uc_stack.ss_sp = nullptr; + ucp->uc_stack.ss_size = 0; +} +# else void ReadContextStack(void *context, uptr *stack, uptr *ssize) { UNIMPLEMENTED(); } -#endif + +void ResetContextStack(void *context) { UNIMPLEMENTED(); } +# endif void *AsanDlSymNext(const char *sym) { return dlsym(RTLD_NEXT, sym); diff --git a/compiler-rt/lib/asan/asan_mac.cpp b/compiler-rt/lib/asan/asan_mac.cpp index 4f4ce92cc6a1..a2d5c31a3f77 100644 --- a/compiler-rt/lib/asan/asan_mac.cpp +++ b/compiler-rt/lib/asan/asan_mac.cpp @@ -99,6 +99,8 @@ void ReadContextStack(void *context, uptr *stack, uptr *ssize) { UNIMPLEMENTED(); } +void ResetContextStack(void *context) { UNIMPLEMENTED(); } + // Support for the following functions from libdispatch on Mac OS: // dispatch_async_f() // dispatch_async() diff --git a/compiler-rt/lib/asan/asan_mapping.h b/compiler-rt/lib/asan/asan_mapping.h index cc5f5836e742..aeadb9d94ebd 100644 --- a/compiler-rt/lib/asan/asan_mapping.h +++ b/compiler-rt/lib/asan/asan_mapping.h @@ -114,6 +114,13 @@ // || `[0x0080000000000, 0x008ffffffffff]` || LowShadow || // || `[0x0000000000000, 0x007ffffffffff]` || LowMem || // +// Default Linux/LoongArch64 (47-bit VMA) mapping: +// || `[0x500000000000, 0x7fffffffffff]` || HighMem || +// || `[0x4a0000000000, 0x4fffffffffff]` || HighShadow || +// || `[0x480000000000, 0x49ffffffffff]` || ShadowGap || +// || `[0x400000000000, 0x47ffffffffff]` || LowShadow || +// || `[0x000000000000, 0x3fffffffffff]` || LowMem || +// // Shadow mapping on FreeBSD/x86-64 with SHADOW_OFFSET == 0x400000000000: // || `[0x500000000000, 0x7fffffffffff]` || HighMem || // || `[0x4a0000000000, 0x4fffffffffff]` || HighShadow || @@ -196,6 +203,8 @@ # define ASAN_SHADOW_OFFSET_CONST 0x0000002000000000 # elif defined(__sparc__) # define ASAN_SHADOW_OFFSET_CONST 0x0000080000000000 +# elif SANITIZER_LOONGARCH64 +# define ASAN_SHADOW_OFFSET_CONST 0x0000400000000000 # elif SANITIZER_WINDOWS64 # define ASAN_SHADOW_OFFSET_DYNAMIC # else diff --git a/compiler-rt/lib/asan/asan_win.cpp b/compiler-rt/lib/asan/asan_win.cpp index 81958038fb1c..f11df0613d1f 100644 --- a/compiler-rt/lib/asan/asan_win.cpp +++ b/compiler-rt/lib/asan/asan_win.cpp @@ -267,6 +267,8 @@ void ReadContextStack(void *context, uptr *stack, uptr *ssize) { UNIMPLEMENTED(); } +void ResetContextStack(void *context) { UNIMPLEMENTED(); } + void AsanOnDeadlySignal(int, void *siginfo, void *context) { UNIMPLEMENTED(); } bool PlatformUnpoisonStacks() { return false; } diff --git a/compiler-rt/lib/hwasan/hwasan_linux.cpp b/compiler-rt/lib/hwasan/hwasan_linux.cpp index dcab473d8ad1..2d7a44525c5f 100644 --- a/compiler-rt/lib/hwasan/hwasan_linux.cpp +++ b/compiler-rt/lib/hwasan/hwasan_linux.cpp @@ -114,19 +114,20 @@ void InitializeOsSupport() { # define PR_SET_TAGGED_ADDR_CTRL 55 # define PR_GET_TAGGED_ADDR_CTRL 56 # define PR_TAGGED_ADDR_ENABLE (1UL << 0) -# define ARCH_GET_UNTAG_MASK 0x4001 +# define ARCH_GET_UNTAG_MASK 0x4001 # define ARCH_ENABLE_TAGGED_ADDR 0x4002 // Check we're running on a kernel that can use the tagged address ABI. int local_errno = 0; bool has_abi; # if defined(__x86_64__) has_abi = (internal_iserror(internal_arch_prctl(ARCH_GET_UNTAG_MASK, 0), - &local_errno) && - local_errno == EINVAL); + &local_errno) && + local_errno == EINVAL); # else - has_abi = (internal_iserror(internal_prctl(PR_GET_TAGGED_ADDR_CTRL, 0, 0, 0, 0), - &local_errno) && - local_errno == EINVAL); + has_abi = + (internal_iserror(internal_prctl(PR_GET_TAGGED_ADDR_CTRL, 0, 0, 0, 0), + &local_errno) && + local_errno == EINVAL); # endif if (has_abi) { # if SANITIZER_ANDROID || defined(HWASAN_ALIASING_MODE) diff --git a/compiler-rt/lib/lsan/lsan_common.cpp b/compiler-rt/lib/lsan/lsan_common.cpp index d6510d34fd68..94bb3cca0083 100644 --- a/compiler-rt/lib/lsan/lsan_common.cpp +++ b/compiler-rt/lib/lsan/lsan_common.cpp @@ -949,7 +949,7 @@ void __lsan_ignore_object(const void *p) { Lock l(&global_mutex); IgnoreObjectResult res = IgnoreObjectLocked(p); if (res == kIgnoreObjectInvalid) - VReport(1, "__lsan_ignore_object(): no heap object found at %p", p); + VReport(1, "__lsan_ignore_object(): no heap object found at %p\n", p); if (res == kIgnoreObjectAlreadyIgnored) VReport(1, "__lsan_ignore_object(): " diff --git a/compiler-rt/lib/memprof/memprof_internal.h b/compiler-rt/lib/memprof/memprof_internal.h index 1adb368e3e41..bba465e60d82 100644 --- a/compiler-rt/lib/memprof/memprof_internal.h +++ b/compiler-rt/lib/memprof/memprof_internal.h @@ -66,8 +66,6 @@ void *MemprofDoesNotSupportStaticLinkage(); // memprof_thread.cpp MemprofThread *CreateMainThread(); -void ReadContextStack(void *context, uptr *stack, uptr *ssize); - // Wrapper for TLS/TSD. void TSDInit(void (*destructor)(void *tsd)); void *TSDGet(); diff --git a/compiler-rt/lib/memprof/memprof_linux.cpp b/compiler-rt/lib/memprof/memprof_linux.cpp index 61c833bfdf64..fcd927023f5c 100644 --- a/compiler-rt/lib/memprof/memprof_linux.cpp +++ b/compiler-rt/lib/memprof/memprof_linux.cpp @@ -69,12 +69,6 @@ uptr FindDynamicShadowStart() { /*min_shadow_base_alignment*/ 0, kHighMemEnd); } -void ReadContextStack(void *context, uptr *stack, uptr *ssize) { - ucontext_t *ucp = (ucontext_t *)context; - *stack = (uptr)ucp->uc_stack.ss_sp; - *ssize = ucp->uc_stack.ss_size; -} - void *MemprofDlSymNext(const char *sym) { return dlsym(RTLD_NEXT, sym); } } // namespace __memprof diff --git a/compiler-rt/lib/msan/msan.h b/compiler-rt/lib/msan/msan.h index 30efce94af54..1a4cc8c0c975 100644 --- a/compiler-rt/lib/msan/msan.h +++ b/compiler-rt/lib/msan/msan.h @@ -195,6 +195,27 @@ const MappingDesc kMemoryLayout[] = { ((((uptr)(mem)) & ~0xC00000000000ULL) + 0x080000000000ULL) #define SHADOW_TO_ORIGIN(shadow) (((uptr)(shadow)) + 0x140000000000ULL) +#elif SANITIZER_FREEBSD && defined(__aarch64__) + +// Low memory: main binary, MAP_32BIT mappings and modules +// High memory: heap, modules and main thread stack +const MappingDesc kMemoryLayout[] = { + {0x000000000000ULL, 0x020000000000ULL, MappingDesc::APP, "low memory"}, + {0x020000000000ULL, 0x200000000000ULL, MappingDesc::INVALID, "invalid"}, + {0x200000000000ULL, 0x620000000000ULL, MappingDesc::SHADOW, "shadow"}, + {0x620000000000ULL, 0x700000000000ULL, MappingDesc::INVALID, "invalid"}, + {0x700000000000ULL, 0xb20000000000ULL, MappingDesc::ORIGIN, "origin"}, + {0xb20000000000ULL, 0xc00000000000ULL, MappingDesc::INVALID, "invalid"}, + {0xc00000000000ULL, 0x1000000000000ULL, MappingDesc::APP, "high memory"}}; + +// Maps low and high app ranges to contiguous space with zero base: +// Low: 0000 0000 0000 - 01ff ffff ffff -> 4000 0000 0000 - 41ff ffff ffff +// High: c000 0000 0000 - ffff ffff ffff -> 0000 0000 0000 - 3fff ffff ffff +#define LINEARIZE_MEM(mem) \ + (((uptr)(mem) & ~0x1800000000000ULL) ^ 0x400000000000ULL) +#define MEM_TO_SHADOW(mem) (LINEARIZE_MEM((mem)) + 0x200000000000ULL) +#define SHADOW_TO_ORIGIN(shadow) (((uptr)(shadow)) + 0x500000000000) + #elif SANITIZER_FREEBSD && SANITIZER_WORDSIZE == 64 // Low memory: main binary, MAP_32BIT mappings and modules diff --git a/compiler-rt/lib/sanitizer_common/sanitizer_common_interceptors_vfork_loongarch64.inc.S b/compiler-rt/lib/sanitizer_common/sanitizer_common_interceptors_vfork_loongarch64.inc.S new file mode 100644 index 000000000000..05192485d597 --- /dev/null +++ b/compiler-rt/lib/sanitizer_common/sanitizer_common_interceptors_vfork_loongarch64.inc.S @@ -0,0 +1,63 @@ +#if defined(__loongarch_lp64) && defined(__linux__) + +#include "sanitizer_common/sanitizer_asm.h" + +ASM_HIDDEN(COMMON_INTERCEPTOR_SPILL_AREA) +ASM_HIDDEN(_ZN14__interception10real_vforkE) + +.bss +.type _ZN14__interception10real_vforkE, @object +.size _ZN14__interception10real_vforkE, 8 +_ZN14__interception10real_vforkE: + .zero 8 + +.text +.globl ASM_WRAPPER_NAME(vfork) +ASM_TYPE_FUNCTION(ASM_WRAPPER_NAME(vfork)) +ASM_WRAPPER_NAME(vfork): + // Save ra in the off-stack spill area. + // allocate space on stack + addi.d $sp, $sp, -16 + // store $ra value + st.d $ra, $sp, 8 + bl COMMON_INTERCEPTOR_SPILL_AREA + // restore previous values from stack + ld.d $ra, $sp, 8 + // adjust stack + addi.d $sp, $sp, 16 + // store $ra by $a0 + st.d $ra, $a0, 0 + + // Call real vfork. This may return twice. User code that runs between the first and the second return + // may clobber the stack frame of the interceptor; that's why it does not have a frame. + la.local $a0, _ZN14__interception10real_vforkE + ld.d $a0, $a0, 0 + jirl $ra, $a0, 0 + + // adjust stack + addi.d $sp, $sp, -16 + // store $a0 by adjusted stack + st.d $a0, $sp, 8 + // jump to exit label if $a0 is 0 + beqz $a0, .L_exit + + // $a0 != 0 => parent process. Clear stack shadow. + // put old $sp to $a0 + addi.d $a0, $sp, 16 + bl %plt(COMMON_INTERCEPTOR_HANDLE_VFORK) + +.L_exit: + // Restore $ra + bl COMMON_INTERCEPTOR_SPILL_AREA + ld.d $ra, $a0, 0 + // load value by stack + ld.d $a0, $sp, 8 + // adjust stack + addi.d $sp, $sp, 16 + jr $ra +ASM_SIZE(vfork) + +.weak vfork +.set vfork, ASM_WRAPPER_NAME(vfork) + +#endif diff --git a/compiler-rt/lib/sanitizer_common/sanitizer_coverage_interface.inc b/compiler-rt/lib/sanitizer_common/sanitizer_coverage_interface.inc index d7ab0c3d98c1..9d36a40270d5 100644 --- a/compiler-rt/lib/sanitizer_common/sanitizer_coverage_interface.inc +++ b/compiler-rt/lib/sanitizer_common/sanitizer_coverage_interface.inc @@ -27,6 +27,16 @@ INTERFACE_WEAK_FUNCTION(__sanitizer_cov_trace_gep) INTERFACE_WEAK_FUNCTION(__sanitizer_cov_trace_pc_guard) INTERFACE_WEAK_FUNCTION(__sanitizer_cov_trace_pc_guard_init) INTERFACE_WEAK_FUNCTION(__sanitizer_cov_trace_pc_indir) +INTERFACE_WEAK_FUNCTION(__sanitizer_cov_load1) +INTERFACE_WEAK_FUNCTION(__sanitizer_cov_load2) +INTERFACE_WEAK_FUNCTION(__sanitizer_cov_load4) +INTERFACE_WEAK_FUNCTION(__sanitizer_cov_load8) +INTERFACE_WEAK_FUNCTION(__sanitizer_cov_load16) +INTERFACE_WEAK_FUNCTION(__sanitizer_cov_store1) +INTERFACE_WEAK_FUNCTION(__sanitizer_cov_store2) +INTERFACE_WEAK_FUNCTION(__sanitizer_cov_store4) +INTERFACE_WEAK_FUNCTION(__sanitizer_cov_store8) +INTERFACE_WEAK_FUNCTION(__sanitizer_cov_store16) INTERFACE_WEAK_FUNCTION(__sanitizer_cov_trace_switch) INTERFACE_WEAK_FUNCTION(__sanitizer_cov_8bit_counters_init) INTERFACE_WEAK_FUNCTION(__sanitizer_cov_bool_flag_init) diff --git a/compiler-rt/lib/sanitizer_common/sanitizer_coverage_libcdep_new.cpp b/compiler-rt/lib/sanitizer_common/sanitizer_coverage_libcdep_new.cpp index 3dcb39f32f6c..956b48e0b434 100644 --- a/compiler-rt/lib/sanitizer_common/sanitizer_coverage_libcdep_new.cpp +++ b/compiler-rt/lib/sanitizer_common/sanitizer_coverage_libcdep_new.cpp @@ -259,6 +259,16 @@ SANITIZER_INTERFACE_WEAK_DEF(void, __sanitizer_cov_trace_div4, void) {} SANITIZER_INTERFACE_WEAK_DEF(void, __sanitizer_cov_trace_div8, void) {} SANITIZER_INTERFACE_WEAK_DEF(void, __sanitizer_cov_trace_gep, void) {} SANITIZER_INTERFACE_WEAK_DEF(void, __sanitizer_cov_trace_pc_indir, void) {} +SANITIZER_INTERFACE_WEAK_DEF(void, __sanitizer_cov_load1, void){} +SANITIZER_INTERFACE_WEAK_DEF(void, __sanitizer_cov_load2, void){} +SANITIZER_INTERFACE_WEAK_DEF(void, __sanitizer_cov_load4, void){} +SANITIZER_INTERFACE_WEAK_DEF(void, __sanitizer_cov_load8, void){} +SANITIZER_INTERFACE_WEAK_DEF(void, __sanitizer_cov_load16, void){} +SANITIZER_INTERFACE_WEAK_DEF(void, __sanitizer_cov_store1, void){} +SANITIZER_INTERFACE_WEAK_DEF(void, __sanitizer_cov_store2, void){} +SANITIZER_INTERFACE_WEAK_DEF(void, __sanitizer_cov_store4, void){} +SANITIZER_INTERFACE_WEAK_DEF(void, __sanitizer_cov_store8, void){} +SANITIZER_INTERFACE_WEAK_DEF(void, __sanitizer_cov_store16, void){} SANITIZER_INTERFACE_WEAK_DEF(void, __sanitizer_cov_8bit_counters_init, char* start, char* end) { __sancov::SingletonCounterCoverage::Cov8bitCountersInit(start, end); diff --git a/compiler-rt/lib/sanitizer_common/sanitizer_interface_internal.h b/compiler-rt/lib/sanitizer_common/sanitizer_interface_internal.h index e9dc78c6354e..ad34e5e5ba54 100644 --- a/compiler-rt/lib/sanitizer_common/sanitizer_interface_internal.h +++ b/compiler-rt/lib/sanitizer_common/sanitizer_interface_internal.h @@ -107,6 +107,26 @@ __sanitizer_cov_trace_gep(); SANITIZER_INTERFACE_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE void __sanitizer_cov_trace_pc_indir(); SANITIZER_INTERFACE_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE void +__sanitizer_cov_load1(); +SANITIZER_INTERFACE_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE void +__sanitizer_cov_load2(); +SANITIZER_INTERFACE_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE void +__sanitizer_cov_load4(); +SANITIZER_INTERFACE_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE void +__sanitizer_cov_load8(); +SANITIZER_INTERFACE_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE void +__sanitizer_cov_load16(); +SANITIZER_INTERFACE_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE void +__sanitizer_cov_store1(); +SANITIZER_INTERFACE_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE void +__sanitizer_cov_store2(); +SANITIZER_INTERFACE_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE void +__sanitizer_cov_store4(); +SANITIZER_INTERFACE_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE void +__sanitizer_cov_store8(); +SANITIZER_INTERFACE_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE void +__sanitizer_cov_store16(); +SANITIZER_INTERFACE_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE void __sanitizer_cov_trace_pc_guard(__sanitizer::u32 *); SANITIZER_INTERFACE_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE void __sanitizer_cov_trace_pc_guard_init(__sanitizer::u32 *, __sanitizer::u32 *); diff --git a/compiler-rt/lib/sanitizer_common/sanitizer_linux.cpp b/compiler-rt/lib/sanitizer_common/sanitizer_linux.cpp index be37fd7f68b3..dc2ea933fadc 100644 --- a/compiler-rt/lib/sanitizer_common/sanitizer_linux.cpp +++ b/compiler-rt/lib/sanitizer_common/sanitizer_linux.cpp @@ -78,6 +78,10 @@ #include <sys/personality.h> #endif +#if SANITIZER_LINUX && defined(__loongarch__) +# include <sys/sysmacros.h> +#endif + #if SANITIZER_FREEBSD #include <sys/exec.h> #include <sys/procctl.h> @@ -188,6 +192,8 @@ ScopedBlockSignals::~ScopedBlockSignals() { SetSigProcMask(&saved_, nullptr); } # include "sanitizer_syscall_linux_arm.inc" # elif SANITIZER_LINUX && defined(__hexagon__) # include "sanitizer_syscall_linux_hexagon.inc" +# elif SANITIZER_LINUX && SANITIZER_LOONGARCH64 +# include "sanitizer_syscall_linux_loongarch64.inc" # else # include "sanitizer_syscall_generic.inc" # endif @@ -290,6 +296,28 @@ static void stat64_to_stat(struct stat64 *in, struct stat *out) { } #endif +#if SANITIZER_LINUX && defined(__loongarch__) +static void statx_to_stat(struct statx *in, struct stat *out) { + internal_memset(out, 0, sizeof(*out)); + out->st_dev = makedev(in->stx_dev_major, in->stx_dev_minor); + out->st_ino = in->stx_ino; + out->st_mode = in->stx_mode; + out->st_nlink = in->stx_nlink; + out->st_uid = in->stx_uid; + out->st_gid = in->stx_gid; + out->st_rdev = makedev(in->stx_rdev_major, in->stx_rdev_minor); + out->st_size = in->stx_size; + out->st_blksize = in->stx_blksize; + out->st_blocks = in->stx_blocks; + out->st_atime = in->stx_atime.tv_sec; + out->st_atim.tv_nsec = in->stx_atime.tv_nsec; + out->st_mtime = in->stx_mtime.tv_sec; + out->st_mtim.tv_nsec = in->stx_mtime.tv_nsec; + out->st_ctime = in->stx_ctime.tv_sec; + out->st_ctim.tv_nsec = in->stx_ctime.tv_nsec; +} +#endif + #if SANITIZER_MIPS64 // Undefine compatibility macros from <sys/stat.h> // so that they would not clash with the kernel_stat @@ -341,52 +369,65 @@ static void kernel_stat_to_stat(struct kernel_stat *in, struct stat *out) { #endif uptr internal_stat(const char *path, void *buf) { -#if SANITIZER_FREEBSD +# if SANITIZER_FREEBSD return internal_syscall(SYSCALL(fstatat), AT_FDCWD, (uptr)path, (uptr)buf, 0); -# elif SANITIZER_LINUX -# if (SANITIZER_WORDSIZE == 64 || SANITIZER_X32 || \ +# elif SANITIZER_LINUX +# if defined(__loongarch__) + struct statx bufx; + int res = internal_syscall(SYSCALL(statx), AT_FDCWD, (uptr)path, + AT_NO_AUTOMOUNT, STATX_BASIC_STATS, (uptr)&bufx); + statx_to_stat(&bufx, (struct stat *)buf); + return res; +# elif (SANITIZER_WORDSIZE == 64 || SANITIZER_X32 || \ (defined(__mips__) && _MIPS_SIM == _ABIN32)) && \ - !SANITIZER_SPARC + !SANITIZER_SPARC return internal_syscall(SYSCALL(newfstatat), AT_FDCWD, (uptr)path, (uptr)buf, 0); -# else +# else struct stat64 buf64; int res = internal_syscall(SYSCALL(fstatat64), AT_FDCWD, (uptr)path, (uptr)&buf64, 0); stat64_to_stat(&buf64, (struct stat *)buf); return res; -# endif -# else +# endif +# else struct stat64 buf64; int res = internal_syscall(SYSCALL(stat64), path, &buf64); stat64_to_stat(&buf64, (struct stat *)buf); return res; -# endif +# endif } uptr internal_lstat(const char *path, void *buf) { -#if SANITIZER_FREEBSD +# if SANITIZER_FREEBSD return internal_syscall(SYSCALL(fstatat), AT_FDCWD, (uptr)path, (uptr)buf, AT_SYMLINK_NOFOLLOW); -# elif SANITIZER_LINUX -# if (defined(_LP64) || SANITIZER_X32 || \ +# elif SANITIZER_LINUX +# if defined(__loongarch__) + struct statx bufx; + int res = internal_syscall(SYSCALL(statx), AT_FDCWD, (uptr)path, + AT_SYMLINK_NOFOLLOW | AT_NO_AUTOMOUNT, + STATX_BASIC_STATS, (uptr)&bufx); + statx_to_stat(&bufx, (struct stat *)buf); + return res; +# elif (defined(_LP64) || SANITIZER_X32 || \ (defined(__mips__) && _MIPS_SIM == _ABIN32)) && \ - !SANITIZER_SPARC + !SANITIZER_SPARC return internal_syscall(SYSCALL(newfstatat), AT_FDCWD, (uptr)path, (uptr)buf, AT_SYMLINK_NOFOLLOW); -# else +# else struct stat64 buf64; int res = internal_syscall(SYSCALL(fstatat64), AT_FDCWD, (uptr)path, (uptr)&buf64, AT_SYMLINK_NOFOLLOW); stat64_to_stat(&buf64, (struct stat *)buf); return res; -# endif -# else +# endif +# else struct stat64 buf64; int res = internal_syscall(SYSCALL(lstat64), path, &buf64); stat64_to_stat(&buf64, (struct stat *)buf); return res; -# endif +# endif } uptr internal_fstat(fd_t fd, void *buf) { @@ -397,9 +438,15 @@ uptr internal_fstat(fd_t fd, void *buf) { int res = internal_syscall(SYSCALL(fstat), fd, &kbuf); kernel_stat_to_stat(&kbuf, (struct stat *)buf); return res; -# else +# elif SANITIZER_LINUX && defined(__loongarch__) + struct statx bufx; + int res = internal_syscall(SYSCALL(statx), fd, 0, AT_EMPTY_PATH, + STATX_BASIC_STATS, (uptr)&bufx); + statx_to_stat(&bufx, (struct stat *)buf); + return res; +# else return internal_syscall(SYSCALL(fstat), fd, (uptr)buf); -# endif +# endif #else struct stat64 buf64; int res = internal_syscall(SYSCALL(fstat64), fd, &buf64); @@ -445,15 +492,15 @@ uptr internal_unlink(const char *path) { } uptr internal_rename(const char *oldpath, const char *newpath) { -#if defined(__riscv) && defined(__linux__) +# if (defined(__riscv) || defined(__loongarch__)) && defined(__linux__) return internal_syscall(SYSCALL(renameat2), AT_FDCWD, (uptr)oldpath, AT_FDCWD, (uptr)newpath, 0); -# elif SANITIZER_LINUX +# elif SANITIZER_LINUX return internal_syscall(SYSCALL(renameat), AT_FDCWD, (uptr)oldpath, AT_FDCWD, (uptr)newpath); -# else +# else return internal_syscall(SYSCALL(rename), (uptr)oldpath, (uptr)newpath); -# endif +# endif } uptr internal_sched_yield() { @@ -763,14 +810,14 @@ uptr internal_lseek(fd_t fd, OFF_T offset, int whence) { uptr internal_prctl(int option, uptr arg2, uptr arg3, uptr arg4, uptr arg5) { return internal_syscall(SYSCALL(prctl), option, arg2, arg3, arg4, arg5); } -# if defined(__x86_64__) -#include <asm/unistd_64.h> +# if defined(__x86_64__) +# include <asm/unistd_64.h> // Currently internal_arch_prctl() is only needed on x86_64. uptr internal_arch_prctl(int option, uptr arg2) { return internal_syscall(__NR_arch_prctl, option, arg2); } -# endif -#endif +# endif +# endif uptr internal_sigaltstack(const void *ss, void *oss) { return internal_syscall(SYSCALL(sigaltstack), (uptr)ss, (uptr)oss); @@ -2176,6 +2223,11 @@ static void GetPcSpBp(void *context, uptr *pc, uptr *sp, uptr *bp) { *pc = ucontext->uc_mcontext.pc; *bp = ucontext->uc_mcontext.r30; *sp = ucontext->uc_mcontext.r29; +# elif defined(__loongarch__) + ucontext_t *ucontext = (ucontext_t *)context; + *pc = ucontext->uc_mcontext.__pc; + *bp = ucontext->uc_mcontext.__gregs[22]; + *sp = ucontext->uc_mcontext.__gregs[3]; # else # error "Unsupported arch" # endif diff --git a/compiler-rt/lib/sanitizer_common/sanitizer_linux_libcdep.cpp b/compiler-rt/lib/sanitizer_common/sanitizer_linux_libcdep.cpp index 9bf14ef64731..d74851c43e14 100644 --- a/compiler-rt/lib/sanitizer_common/sanitizer_linux_libcdep.cpp +++ b/compiler-rt/lib/sanitizer_common/sanitizer_linux_libcdep.cpp @@ -27,6 +27,7 @@ #include "sanitizer_linux.h" #include "sanitizer_placement_new.h" #include "sanitizer_procmaps.h" +#include "sanitizer_solaris.h" #if SANITIZER_NETBSD #define _RTLD_SOURCE // for __lwp_gettcb_fast() / __lwp_getprivate_fast() @@ -62,6 +63,7 @@ #endif #if SANITIZER_SOLARIS +#include <stddef.h> #include <stdlib.h> #include <thread.h> #endif @@ -350,19 +352,43 @@ static uptr TlsGetOffset(uptr ti_module, uptr ti_offset) { extern "C" void *__tls_get_addr(size_t *); #endif +static size_t main_tls_modid; + static int CollectStaticTlsBlocks(struct dl_phdr_info *info, size_t size, void *data) { - if (!info->dlpi_tls_modid) + size_t tls_modid; +#if SANITIZER_SOLARIS + // dlpi_tls_modid is only available since Solaris 11.4 SRU 10. Use + // dlinfo(RTLD_DI_LINKMAP) instead which works on all of Solaris 11.3, + // 11.4, and Illumos. The tlsmodid of the executable was changed to 1 in + // 11.4 to match other implementations. + if (size >= offsetof(dl_phdr_info_test, dlpi_tls_modid)) + main_tls_modid = 1; + else + main_tls_modid = 0; + g_use_dlpi_tls_data = 0; + Rt_map *map; + dlinfo(RTLD_SELF, RTLD_DI_LINKMAP, &map); + tls_modid = map->rt_tlsmodid; +#else + main_tls_modid = 1; + tls_modid = info->dlpi_tls_modid; +#endif + + if (tls_modid < main_tls_modid) return 0; - uptr begin = (uptr)info->dlpi_tls_data; + uptr begin; +#if !SANITIZER_SOLARIS + begin = (uptr)info->dlpi_tls_data; +#endif if (!g_use_dlpi_tls_data) { // Call __tls_get_addr as a fallback. This forces TLS allocation on glibc // and FreeBSD. #ifdef __s390__ begin = (uptr)__builtin_thread_pointer() + - TlsGetOffset(info->dlpi_tls_modid, 0); + TlsGetOffset(tls_modid, 0); #else - size_t mod_and_off[2] = {info->dlpi_tls_modid, 0}; + size_t mod_and_off[2] = {tls_modid, 0}; begin = (uptr)__tls_get_addr(mod_and_off); #endif } @@ -370,7 +396,7 @@ static int CollectStaticTlsBlocks(struct dl_phdr_info *info, size_t size, if (info->dlpi_phdr[i].p_type == PT_TLS) { static_cast<InternalMmapVector<TlsBlock> *>(data)->push_back( TlsBlock{begin, begin + info->dlpi_phdr[i].p_memsz, - info->dlpi_phdr[i].p_align, info->dlpi_tls_modid}); + info->dlpi_phdr[i].p_align, tls_modid}); break; } return 0; @@ -382,11 +408,11 @@ __attribute__((unused)) static void GetStaticTlsBoundary(uptr *addr, uptr *size, dl_iterate_phdr(CollectStaticTlsBlocks, &ranges); uptr len = ranges.size(); Sort(ranges.begin(), len); - // Find the range with tls_modid=1. For glibc, because libc.so uses PT_TLS, - // this module is guaranteed to exist and is one of the initially loaded - // modules. + // Find the range with tls_modid == main_tls_modid. For glibc, because + // libc.so uses PT_TLS, this module is guaranteed to exist and is one of + // the initially loaded modules. uptr one = 0; - while (one != len && ranges[one].tls_modid != 1) ++one; + while (one != len && ranges[one].tls_modid != main_tls_modid) ++one; if (one == len) { // This may happen with musl if no module uses PT_TLS. *addr = 0; @@ -395,14 +421,14 @@ __attribute__((unused)) static void GetStaticTlsBoundary(uptr *addr, uptr *size, return; } // Find the maximum consecutive ranges. We consider two modules consecutive if - // the gap is smaller than the alignment. The dynamic loader places static TLS - // blocks this way not to waste space. + // the gap is smaller than the alignment of the latter range. The dynamic + // loader places static TLS blocks this way not to waste space. uptr l = one; *align = ranges[l].align; - while (l != 0 && ranges[l].begin < ranges[l - 1].end + ranges[l - 1].align) + while (l != 0 && ranges[l].begin < ranges[l - 1].end + ranges[l].align) *align = Max(*align, ranges[--l].align); uptr r = one + 1; - while (r != len && ranges[r].begin < ranges[r - 1].end + ranges[r - 1].align) + while (r != len && ranges[r].begin < ranges[r - 1].end + ranges[r].align) *align = Max(*align, ranges[r++].align); *addr = ranges[l].begin; *size = ranges[r - 1].end - ranges[l].begin; diff --git a/compiler-rt/lib/sanitizer_common/sanitizer_mac.cpp b/compiler-rt/lib/sanitizer_common/sanitizer_mac.cpp index 7ce6eff832e5..1ae69e14b237 100644 --- a/compiler-rt/lib/sanitizer_common/sanitizer_mac.cpp +++ b/compiler-rt/lib/sanitizer_common/sanitizer_mac.cpp @@ -73,6 +73,7 @@ extern "C" { #include <malloc/malloc.h> #include <os/log.h> #include <pthread.h> +#include <pthread/introspection.h> #include <sched.h> #include <signal.h> #include <spawn.h> @@ -1395,6 +1396,61 @@ u32 GetNumberOfCPUs() { void InitializePlatformCommonFlags(CommonFlags *cf) {} +// Pthread introspection hook +// +// * GCD worker threads are created without a call to pthread_create(), but we +// still need to register these threads (with ThreadCreate/Start()). +// * We use the "pthread introspection hook" below to observe the creation of +// such threads. +// * GCD worker threads don't have parent threads and the CREATE event is +// delivered in the context of the thread itself. CREATE events for regular +// threads, are delivered on the parent. We use this to tell apart which +// threads are GCD workers with `thread == pthread_self()`. +// +static pthread_introspection_hook_t prev_pthread_introspection_hook; +static ThreadEventCallbacks thread_event_callbacks; + +static void sanitizer_pthread_introspection_hook(unsigned int event, + pthread_t thread, void *addr, + size_t size) { + // create -> start -> terminate -> destroy + // * create/destroy are usually (not guaranteed) delivered on the parent and + // track resource allocation/reclamation + // * start/terminate are guaranteed to be delivered in the context of the + // thread and give hooks into "just after (before) thread starts (stops) + // executing" + DCHECK(event >= PTHREAD_INTROSPECTION_THREAD_CREATE && + event <= PTHREAD_INTROSPECTION_THREAD_DESTROY); + + if (event == PTHREAD_INTROSPECTION_THREAD_CREATE) { + bool gcd_worker = (thread == pthread_self()); + if (thread_event_callbacks.create) + thread_event_callbacks.create((uptr)thread, gcd_worker); + } else if (event == PTHREAD_INTROSPECTION_THREAD_START) { + CHECK_EQ(thread, pthread_self()); + if (thread_event_callbacks.start) + thread_event_callbacks.start((uptr)thread); + } + + if (prev_pthread_introspection_hook) + prev_pthread_introspection_hook(event, thread, addr, size); + + if (event == PTHREAD_INTROSPECTION_THREAD_TERMINATE) { + CHECK_EQ(thread, pthread_self()); + if (thread_event_callbacks.terminate) + thread_event_callbacks.terminate((uptr)thread); + } else if (event == PTHREAD_INTROSPECTION_THREAD_DESTROY) { + if (thread_event_callbacks.destroy) + thread_event_callbacks.destroy((uptr)thread); + } +} + +void InstallPthreadIntrospectionHook(const ThreadEventCallbacks &callbacks) { + thread_event_callbacks = callbacks; + prev_pthread_introspection_hook = + pthread_introspection_hook_install(&sanitizer_pthread_introspection_hook); +} + } // namespace __sanitizer #endif // SANITIZER_APPLE diff --git a/compiler-rt/lib/sanitizer_common/sanitizer_mac.h b/compiler-rt/lib/sanitizer_common/sanitizer_mac.h index a8b274e8c82c..f0a97d098eea 100644 --- a/compiler-rt/lib/sanitizer_common/sanitizer_mac.h +++ b/compiler-rt/lib/sanitizer_common/sanitizer_mac.h @@ -62,6 +62,17 @@ char **GetEnviron(); void RestrictMemoryToMaxAddress(uptr max_address); +using ThreadEventCallback = void (*)(uptr thread); +using ThreadCreateEventCallback = void (*)(uptr thread, bool gcd_worker); +struct ThreadEventCallbacks { + ThreadCreateEventCallback create; + ThreadEventCallback start; + ThreadEventCallback terminate; + ThreadEventCallback destroy; +}; + +void InstallPthreadIntrospectionHook(const ThreadEventCallbacks &callbacks); + } // namespace __sanitizer #endif // SANITIZER_APPLE diff --git a/compiler-rt/lib/sanitizer_common/sanitizer_platform.h b/compiler-rt/lib/sanitizer_common/sanitizer_platform.h index 4d89ecaf1071..ea4e5b015d11 100644 --- a/compiler-rt/lib/sanitizer_common/sanitizer_platform.h +++ b/compiler-rt/lib/sanitizer_common/sanitizer_platform.h @@ -272,6 +272,12 @@ # define SANITIZER_RISCV64 0 #endif +#if defined(__loongarch_lp64) +# define SANITIZER_LOONGARCH64 1 +#else +# define SANITIZER_LOONGARCH64 0 +#endif + // By default we allow to use SizeClassAllocator64 on 64-bit platform. // But in some cases (e.g. AArch64's 39-bit address space) SizeClassAllocator64 // does not work well and we need to fallback to SizeClassAllocator32. diff --git a/compiler-rt/lib/sanitizer_common/sanitizer_platform_limits_linux.cpp b/compiler-rt/lib/sanitizer_common/sanitizer_platform_limits_linux.cpp index 06bafd68dc74..bf0f355847cb 100644 --- a/compiler-rt/lib/sanitizer_common/sanitizer_platform_limits_linux.cpp +++ b/compiler-rt/lib/sanitizer_common/sanitizer_platform_limits_linux.cpp @@ -59,7 +59,8 @@ using namespace __sanitizer; # if !defined(__powerpc64__) && !defined(__x86_64__) && \ !defined(__aarch64__) && !defined(__mips__) && !defined(__s390__) && \ - !defined(__sparc__) && !defined(__riscv) && !defined(__hexagon__) + !defined(__sparc__) && !defined(__riscv) && !defined(__hexagon__) && \ + !defined(__loongarch__) COMPILER_CHECK(struct___old_kernel_stat_sz == sizeof(struct __old_kernel_stat)); #endif diff --git a/compiler-rt/lib/sanitizer_common/sanitizer_platform_limits_posix.cpp b/compiler-rt/lib/sanitizer_common/sanitizer_platform_limits_posix.cpp index 3a94b260686f..c85cf1626a75 100644 --- a/compiler-rt/lib/sanitizer_common/sanitizer_platform_limits_posix.cpp +++ b/compiler-rt/lib/sanitizer_common/sanitizer_platform_limits_posix.cpp @@ -271,6 +271,10 @@ namespace __sanitizer { defined(__powerpc__) || defined(__s390__) || defined(__sparc__) || \ defined(__hexagon__) # define SIZEOF_STRUCT_USTAT 20 +# elif defined(__loongarch__) + // Not used. The minimum Glibc version available for LoongArch is 2.36 + // so ustat() wrapper is already gone. +# define SIZEOF_STRUCT_USTAT 0 # else # error Unknown size of struct ustat # endif @@ -1271,7 +1275,7 @@ CHECK_SIZE_AND_OFFSET(group, gr_passwd); CHECK_SIZE_AND_OFFSET(group, gr_gid); CHECK_SIZE_AND_OFFSET(group, gr_mem); -#if HAVE_RPC_XDR_H +#if HAVE_RPC_XDR_H && !SANITIZER_APPLE CHECK_TYPE_SIZE(XDR); CHECK_SIZE_AND_OFFSET(XDR, x_op); CHECK_SIZE_AND_OFFSET(XDR, x_ops); diff --git a/compiler-rt/lib/sanitizer_common/sanitizer_platform_limits_posix.h b/compiler-rt/lib/sanitizer_common/sanitizer_platform_limits_posix.h index 8f7df12c4986..bd5692ed511b 100644 --- a/compiler-rt/lib/sanitizer_common/sanitizer_platform_limits_posix.h +++ b/compiler-rt/lib/sanitizer_common/sanitizer_platform_limits_posix.h @@ -98,9 +98,10 @@ const unsigned struct_kernel_stat64_sz = 104; const unsigned struct_kernel_stat_sz = 144; const unsigned struct_kernel_stat64_sz = 104; #elif defined(__mips__) -const unsigned struct_kernel_stat_sz = SANITIZER_ANDROID - ? FIRST_32_SECOND_64(104, 128) - : FIRST_32_SECOND_64(160, 216); +const unsigned struct_kernel_stat_sz = + SANITIZER_ANDROID + ? FIRST_32_SECOND_64(104, 128) + : FIRST_32_SECOND_64((_MIPS_SIM == _ABIN32) ? 160 : 144, 216); const unsigned struct_kernel_stat64_sz = 104; #elif defined(__s390__) && !defined(__s390x__) const unsigned struct_kernel_stat_sz = 64; @@ -122,6 +123,9 @@ const unsigned struct_kernel_stat64_sz = 0; // RISCV64 does not use stat64 # elif defined(__hexagon__) const unsigned struct_kernel_stat_sz = 128; const unsigned struct_kernel_stat64_sz = 0; +# elif defined(__loongarch__) +const unsigned struct_kernel_stat_sz = 128; +const unsigned struct_kernel_stat64_sz = 0; # endif struct __sanitizer_perf_event_attr { unsigned type; @@ -142,7 +146,7 @@ const unsigned struct_kexec_segment_sz = 4 * sizeof(unsigned long); #if SANITIZER_LINUX -#if defined(__powerpc64__) || defined(__s390__) +#if defined(__powerpc64__) || defined(__s390__) || defined(__loongarch__) const unsigned struct___old_kernel_stat_sz = 0; #elif !defined(__sparc__) const unsigned struct___old_kernel_stat_sz = 32; diff --git a/compiler-rt/lib/sanitizer_common/sanitizer_procmaps_solaris.cpp b/compiler-rt/lib/sanitizer_common/sanitizer_procmaps_solaris.cpp index e16c4e938cb2..6f43817aedb1 100644 --- a/compiler-rt/lib/sanitizer_common/sanitizer_procmaps_solaris.cpp +++ b/compiler-rt/lib/sanitizer_common/sanitizer_procmaps_solaris.cpp @@ -9,25 +9,32 @@ // Information about the process mappings (Solaris-specific parts). //===----------------------------------------------------------------------===// -// Before Solaris 11.4, <procfs.h> doesn't work in a largefile environment. -#undef _FILE_OFFSET_BITS #include "sanitizer_platform.h" #if SANITIZER_SOLARIS -#include "sanitizer_common.h" -#include "sanitizer_procmaps.h" +# include <fcntl.h> +# include <limits.h> +# include <procfs.h> -#include <procfs.h> -#include <limits.h> +# include "sanitizer_common.h" +# include "sanitizer_procmaps.h" namespace __sanitizer { void ReadProcMaps(ProcSelfMapsBuff *proc_maps) { - if (!ReadFileToBuffer("/proc/self/xmap", &proc_maps->data, - &proc_maps->mmaped_size, &proc_maps->len)) { - proc_maps->data = nullptr; - proc_maps->mmaped_size = 0; - proc_maps->len = 0; - } + uptr fd = internal_open("/proc/self/xmap", O_RDONLY); + CHECK_NE(fd, -1); + uptr Size = internal_filesize(fd); + CHECK_GT(Size, 0); + + // Allow for additional entries by following mmap. + size_t MmapedSize = Size * 4 / 3; + void *VmMap = MmapOrDie(MmapedSize, "ReadProcMaps()"); + Size = internal_read(fd, VmMap, MmapedSize); + CHECK_NE(Size, -1); + internal_close(fd); + proc_maps->data = (char *)VmMap; + proc_maps->mmaped_size = MmapedSize; + proc_maps->len = Size; } bool MemoryMappingLayout::Next(MemoryMappedSegment *segment) { @@ -49,21 +56,28 @@ bool MemoryMappingLayout::Next(MemoryMappedSegment *segment) { segment->protection |= kProtectionWrite; if ((xmapentry->pr_mflags & MA_EXEC) != 0) segment->protection |= kProtectionExecute; + if ((xmapentry->pr_mflags & MA_SHARED) != 0) + segment->protection |= kProtectionShared; if (segment->filename != NULL && segment->filename_size > 0) { char proc_path[PATH_MAX + 1]; - internal_snprintf(proc_path, sizeof(proc_path), "/proc/self/path/%s", - xmapentry->pr_mapname); - ssize_t sz = internal_readlink(proc_path, segment->filename, - segment->filename_size - 1); - - // If readlink failed, the map is anonymous. - if (sz == -1) { + // Avoid unnecessary readlink on unnamed entires. + if (xmapentry->pr_mapname[0] == '\0') segment->filename[0] = '\0'; - } else if ((size_t)sz < segment->filename_size) - // readlink doesn't NUL-terminate. - segment->filename[sz] = '\0'; + else { + internal_snprintf(proc_path, sizeof(proc_path), "/proc/self/path/%s", + xmapentry->pr_mapname); + ssize_t sz = internal_readlink(proc_path, segment->filename, + segment->filename_size - 1); + + // If readlink failed, the map is anonymous. + if (sz == -1) + segment->filename[0] = '\0'; + else if ((size_t)sz < segment->filename_size) + // readlink doesn't NUL-terminate. + segment->filename[sz] = '\0'; + } } data_.current += sizeof(prxmap_t); diff --git a/compiler-rt/lib/sanitizer_common/sanitizer_solaris.h b/compiler-rt/lib/sanitizer_common/sanitizer_solaris.h new file mode 100644 index 000000000000..2a21693efbf1 --- /dev/null +++ b/compiler-rt/lib/sanitizer_common/sanitizer_solaris.h @@ -0,0 +1,56 @@ +//===-- sanitizer_solaris.h -------------------------------------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file is a part of Sanitizer runtime. It contains Solaris-specific +// definitions. +// +//===----------------------------------------------------------------------===// + +#ifndef SANITIZER_SOLARIS_H +#define SANITIZER_SOLARIS_H + +#include "sanitizer_internal_defs.h" + +#if SANITIZER_SOLARIS + +#include <link.h> + +namespace __sanitizer { + +// Beginning of declaration from OpenSolaris/Illumos +// $SRC/cmd/sgs/include/rtld.h. +struct Rt_map { + Link_map rt_public; + const char *rt_pathname; + ulong_t rt_padstart; + ulong_t rt_padimlen; + ulong_t rt_msize; + uint_t rt_flags; + uint_t rt_flags1; + ulong_t rt_tlsmodid; +}; + +// Structure matching the Solaris 11.4 struct dl_phdr_info used to determine +// presence of dlpi_tls_modid field at runtime. Cf. Solaris 11.4 +// dl_iterate_phdr(3C), Example 2. +struct dl_phdr_info_test { + ElfW(Addr) dlpi_addr; + const char *dlpi_name; + const ElfW(Phdr) * dlpi_phdr; + ElfW(Half) dlpi_phnum; + u_longlong_t dlpi_adds; + u_longlong_t dlpi_subs; + size_t dlpi_tls_modid; + void *dlpi_tls_data; +}; + +} // namespace __sanitizer + +#endif // SANITIZER_SOLARIS + +#endif // SANITIZER_SOLARIS_H diff --git a/compiler-rt/lib/sanitizer_common/sanitizer_stacktrace.cpp b/compiler-rt/lib/sanitizer_common/sanitizer_stacktrace.cpp index 3013a0c4abdf..d24fae98213a 100644 --- a/compiler-rt/lib/sanitizer_common/sanitizer_stacktrace.cpp +++ b/compiler-rt/lib/sanitizer_common/sanitizer_stacktrace.cpp @@ -121,7 +121,7 @@ void BufferedStackTrace::UnwindFast(uptr pc, uptr bp, uptr stack_top, uhwptr pc1 = caller_frame[2]; #elif defined(__s390__) uhwptr pc1 = frame[14]; -#elif defined(__riscv) +#elif defined(__loongarch__) || defined(__riscv) // frame[-1] contains the return address uhwptr pc1 = frame[-1]; #else @@ -136,7 +136,7 @@ void BufferedStackTrace::UnwindFast(uptr pc, uptr bp, uptr stack_top, trace_buffer[size++] = (uptr) pc1; } bottom = (uptr)frame; -#if defined(__riscv) +#if defined(__loongarch__) || defined(__riscv) // frame[-2] contain fp of the previous frame uptr new_bp = (uptr)frame[-2]; #else diff --git a/compiler-rt/lib/sanitizer_common/sanitizer_stoptheworld_mac.cpp b/compiler-rt/lib/sanitizer_common/sanitizer_stoptheworld_mac.cpp index c87674ff7b76..87f5250db648 100644 --- a/compiler-rt/lib/sanitizer_common/sanitizer_stoptheworld_mac.cpp +++ b/compiler-rt/lib/sanitizer_common/sanitizer_stoptheworld_mac.cpp @@ -29,7 +29,7 @@ typedef struct { class SuspendedThreadsListMac final : public SuspendedThreadsList { public: - SuspendedThreadsListMac() : threads_(1024) {} + SuspendedThreadsListMac() = default; tid_t GetThreadID(uptr index) const override; thread_t GetThread(uptr index) const; diff --git a/compiler-rt/lib/sanitizer_common/sanitizer_symbolizer_posix_libcdep.cpp b/compiler-rt/lib/sanitizer_common/sanitizer_symbolizer_posix_libcdep.cpp index 8be7709b6038..b223f6cd01e3 100644 --- a/compiler-rt/lib/sanitizer_common/sanitizer_symbolizer_posix_libcdep.cpp +++ b/compiler-rt/lib/sanitizer_common/sanitizer_symbolizer_posix_libcdep.cpp @@ -72,7 +72,6 @@ static swift_demangle_ft swift_demangle_f; // symbolication. static void InitializeSwiftDemangler() { swift_demangle_f = (swift_demangle_ft)dlsym(RTLD_DEFAULT, "swift_demangle"); - (void)dlerror(); // Cleanup error message in case of failure } // Attempts to demangle a Swift name. The demangler will return nullptr if a diff --git a/compiler-rt/lib/sanitizer_common/sanitizer_syscall_linux_loongarch64.inc b/compiler-rt/lib/sanitizer_common/sanitizer_syscall_linux_loongarch64.inc new file mode 100644 index 000000000000..97ca7f2f3f92 --- /dev/null +++ b/compiler-rt/lib/sanitizer_common/sanitizer_syscall_linux_loongarch64.inc @@ -0,0 +1,167 @@ +//===-- sanitizer_syscall_linux_loongarch64.inc -----------------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// Implementations of internal_syscall and internal_iserror for +// Linux/loongarch64. +// +//===----------------------------------------------------------------------===// + +// About local register variables: +// https://gcc.gnu.org/onlinedocs/gcc/Local-Register-Variables.html#Local-Register-Variables +// +// Kernel ABI... +// syscall number is passed in a7 +// (http://man7.org/linux/man-pages/man2/syscall.2.html) results are return in +// a0 and a1 (http://man7.org/linux/man-pages/man2/syscall.2.html) arguments +// are passed in: a0-a7 (confirmed by inspecting glibc sources). +#define SYSCALL(name) __NR_##name + +#define INTERNAL_SYSCALL_CLOBBERS "memory" + +static uptr __internal_syscall(u64 nr) { + register u64 a7 asm("a7") = nr; + register u64 a0 asm("a0"); + __asm__ volatile("syscall 0\n\t" + : "=r"(a0) + : "r"(a7) + : INTERNAL_SYSCALL_CLOBBERS); + return a0; +} +#define __internal_syscall0(n) (__internal_syscall)(n) + +static uptr __internal_syscall(u64 nr, u64 arg1) { + register u64 a7 asm("a7") = nr; + register u64 a0 asm("a0") = arg1; + __asm__ volatile("syscall 0\n\t" + : "+r"(a0) + : "r"(a7) + : INTERNAL_SYSCALL_CLOBBERS); + return a0; +} +#define __internal_syscall1(n, a1) (__internal_syscall)(n, (u64)(a1)) + +static uptr __internal_syscall(u64 nr, u64 arg1, long arg2) { + register u64 a7 asm("a7") = nr; + register u64 a0 asm("a0") = arg1; + register u64 a1 asm("a1") = arg2; + __asm__ volatile("syscall 0\n\t" + : "+r"(a0) + : "r"(a7), "r"(a1) + : INTERNAL_SYSCALL_CLOBBERS); + return a0; +} +#define __internal_syscall2(n, a1, a2) \ + (__internal_syscall)(n, (u64)(a1), (long)(a2)) + +static uptr __internal_syscall(u64 nr, u64 arg1, long arg2, long arg3) { + register u64 a7 asm("a7") = nr; + register u64 a0 asm("a0") = arg1; + register u64 a1 asm("a1") = arg2; + register u64 a2 asm("a2") = arg3; + __asm__ volatile("syscall 0\n\t" + : "+r"(a0) + : "r"(a7), "r"(a1), "r"(a2) + : INTERNAL_SYSCALL_CLOBBERS); + return a0; +} +#define __internal_syscall3(n, a1, a2, a3) \ + (__internal_syscall)(n, (u64)(a1), (long)(a2), (long)(a3)) + +static uptr __internal_syscall(u64 nr, u64 arg1, long arg2, long arg3, + u64 arg4) { + register u64 a7 asm("a7") = nr; + register u64 a0 asm("a0") = arg1; + register u64 a1 asm("a1") = arg2; + register u64 a2 asm("a2") = arg3; + register u64 a3 asm("a3") = arg4; + __asm__ volatile("syscall 0\n\t" + : "+r"(a0) + : "r"(a7), "r"(a1), "r"(a2), "r"(a3) + : INTERNAL_SYSCALL_CLOBBERS); + return a0; +} +#define __internal_syscall4(n, a1, a2, a3, a4) \ + (__internal_syscall)(n, (u64)(a1), (long)(a2), (long)(a3), (long)(a4)) + +static uptr __internal_syscall(u64 nr, u64 arg1, long arg2, long arg3, u64 arg4, + long arg5) { + register u64 a7 asm("a7") = nr; + register u64 a0 asm("a0") = arg1; + register u64 a1 asm("a1") = arg2; + register u64 a2 asm("a2") = arg3; + register u64 a3 asm("a3") = arg4; + register u64 a4 asm("a4") = arg5; + __asm__ volatile("syscall 0\n\t" + : "+r"(a0) + : "r"(a7), "r"(a1), "r"(a2), "r"(a3), "r"(a4) + : INTERNAL_SYSCALL_CLOBBERS); + return a0; +} +#define __internal_syscall5(n, a1, a2, a3, a4, a5) \ + (__internal_syscall)(n, (u64)(a1), (long)(a2), (long)(a3), (long)(a4), \ + (u64)(a5)) + +static uptr __internal_syscall(u64 nr, u64 arg1, long arg2, long arg3, u64 arg4, + long arg5, long arg6) { + register u64 a7 asm("a7") = nr; + register u64 a0 asm("a0") = arg1; + register u64 a1 asm("a1") = arg2; + register u64 a2 asm("a2") = arg3; + register u64 a3 asm("a3") = arg4; + register u64 a4 asm("a4") = arg5; + register u64 a5 asm("a5") = arg6; + __asm__ volatile("syscall 0\n\t" + : "+r"(a0) + : "r"(a7), "r"(a1), "r"(a2), "r"(a3), "r"(a4), "r"(a5) + : INTERNAL_SYSCALL_CLOBBERS); + return a0; +} +#define __internal_syscall6(n, a1, a2, a3, a4, a5, a6) \ + (__internal_syscall)(n, (u64)(a1), (long)(a2), (long)(a3), (long)(a4), \ + (u64)(a5), (long)(a6)) + +static uptr __internal_syscall(u64 nr, u64 arg1, long arg2, long arg3, u64 arg4, + long arg5, long arg6, long arg7) { + register u64 a7 asm("a7") = nr; + register u64 a0 asm("a0") = arg1; + register u64 a1 asm("a1") = arg2; + register u64 a2 asm("a2") = arg3; + register u64 a3 asm("a3") = arg4; + register u64 a4 asm("a4") = arg5; + register u64 a5 asm("a5") = arg6; + register u64 a6 asm("a6") = arg7; + __asm__ volatile("syscall 0\n\t" + : "+r"(a0) + : "r"(a7), "r"(a1), "r"(a2), "r"(a3), "r"(a4), "r"(a5), + "r"(a6) + : INTERNAL_SYSCALL_CLOBBERS); + return a0; +} +#define __internal_syscall7(n, a1, a2, a3, a4, a5, a6, a7) \ + (__internal_syscall)(n, (u64)(a1), (long)(a2), (long)(a3), (long)(a4), \ + (u64)(a5), (long)(a6), (long)(a7)) + +#define __SYSCALL_NARGS_X(a1, a2, a3, a4, a5, a6, a7, a8, n, ...) n +#define __SYSCALL_NARGS(...) \ + __SYSCALL_NARGS_X(__VA_ARGS__, 7, 6, 5, 4, 3, 2, 1, 0, ) +#define __SYSCALL_CONCAT_X(a, b) a##b +#define __SYSCALL_CONCAT(a, b) __SYSCALL_CONCAT_X(a, b) +#define __SYSCALL_DISP(b, ...) \ + __SYSCALL_CONCAT(b, __SYSCALL_NARGS(__VA_ARGS__))(__VA_ARGS__) + +#define internal_syscall(...) __SYSCALL_DISP(__internal_syscall, __VA_ARGS__) + +// Helper function used to avoid clobbering of errno. +bool internal_iserror(uptr retval, int *internal_errno) { + if (retval >= (uptr)-4095) { + if (internal_errno) + *internal_errno = -retval; + return true; + } + return false; +} diff --git a/compiler-rt/lib/tsan/rtl/tsan_dense_alloc.h b/compiler-rt/lib/tsan/rtl/tsan_dense_alloc.h index 7a39a39d51de..2eaff39057bc 100644 --- a/compiler-rt/lib/tsan/rtl/tsan_dense_alloc.h +++ b/compiler-rt/lib/tsan/rtl/tsan_dense_alloc.h @@ -85,14 +85,7 @@ class DenseSlabAlloc { } void FlushCache(Cache *c) { - if (!c->pos) - return; - SpinMutexLock lock(&mtx_); - while (c->pos) { - IndexT idx = c->cache[--c->pos]; - *(IndexT*)Map(idx) = freelist_; - freelist_ = idx; - } + while (c->pos) Drain(c); } void InitCache(Cache *c) { @@ -106,7 +99,7 @@ class DenseSlabAlloc { template <typename Func> void ForEach(Func func) { - SpinMutexLock lock(&mtx_); + Lock lock(&mtx_); uptr fillpos = atomic_load_relaxed(&fillpos_); for (uptr l1 = 0; l1 < fillpos; l1++) { for (IndexT l2 = l1 == 0 ? 1 : 0; l2 < kL2Size; l2++) func(&map_[l1][l2]); @@ -115,48 +108,86 @@ class DenseSlabAlloc { private: T *map_[kL1Size]; - SpinMutex mtx_; - IndexT freelist_ = {0}; + Mutex mtx_; + // The freelist is organized as a lock-free stack of batches of nodes. + // The stack itself uses Block::next links, while the batch within each + // stack node uses Block::batch links. + // Low 32-bits of freelist_ is the node index, top 32-bits is ABA-counter. + atomic_uint64_t freelist_ = {0}; atomic_uintptr_t fillpos_ = {0}; const char *const name_; - void Refill(Cache *c) { - SpinMutexLock lock(&mtx_); - if (freelist_ == 0) { - uptr fillpos = atomic_load_relaxed(&fillpos_); - if (fillpos == kL1Size) { - Printf("ThreadSanitizer: %s overflow (%zu*%zu). Dying.\n", - name_, kL1Size, kL2Size); - Die(); - } - VPrintf(2, "ThreadSanitizer: growing %s: %zu out of %zu*%zu\n", name_, - fillpos, kL1Size, kL2Size); - T *batch = (T*)MmapOrDie(kL2Size * sizeof(T), name_); - // Reserve 0 as invalid index. - IndexT start = fillpos == 0 ? 1 : 0; - for (IndexT i = start; i < kL2Size; i++) { - new(batch + i) T; - *(IndexT *)(batch + i) = i + 1 + fillpos * kL2Size; - } - *(IndexT*)(batch + kL2Size - 1) = 0; - freelist_ = fillpos * kL2Size + start; - map_[fillpos] = batch; - atomic_store_relaxed(&fillpos_, fillpos + 1); - } - for (uptr i = 0; i < Cache::kSize / 2 && freelist_ != 0; i++) { - IndexT idx = freelist_; + struct Block { + IndexT next; + IndexT batch; + }; + + Block *MapBlock(IndexT idx) { return reinterpret_cast<Block *>(Map(idx)); } + + static constexpr u64 kCounterInc = 1ull << 32; + static constexpr u64 kCounterMask = ~(kCounterInc - 1); + + NOINLINE void Refill(Cache *c) { + // Pop 1 batch of nodes from the freelist. + IndexT idx; + u64 xchg; + u64 cmp = atomic_load(&freelist_, memory_order_acquire); + do { + idx = static_cast<IndexT>(cmp); + if (!idx) + return AllocSuperBlock(c); + Block *ptr = MapBlock(idx); + xchg = ptr->next | (cmp & kCounterMask); + } while (!atomic_compare_exchange_weak(&freelist_, &cmp, xchg, + memory_order_acq_rel)); + // Unpack it into c->cache. + while (idx) { c->cache[c->pos++] = idx; - freelist_ = *(IndexT*)Map(idx); + idx = MapBlock(idx)->batch; } } - void Drain(Cache *c) { - SpinMutexLock lock(&mtx_); - for (uptr i = 0; i < Cache::kSize / 2; i++) { + NOINLINE void Drain(Cache *c) { + // Build a batch of at most Cache::kSize / 2 nodes linked by Block::batch. + IndexT head_idx = 0; + for (uptr i = 0; i < Cache::kSize / 2 && c->pos; i++) { IndexT idx = c->cache[--c->pos]; - *(IndexT*)Map(idx) = freelist_; - freelist_ = idx; + Block *ptr = MapBlock(idx); + ptr->batch = head_idx; + head_idx = idx; + } + // Push it onto the freelist stack. + Block *head = MapBlock(head_idx); + u64 xchg; + u64 cmp = atomic_load(&freelist_, memory_order_acquire); + do { + head->next = static_cast<IndexT>(cmp); + xchg = head_idx | (cmp & kCounterMask) + kCounterInc; + } while (!atomic_compare_exchange_weak(&freelist_, &cmp, xchg, + memory_order_acq_rel)); + } + + NOINLINE void AllocSuperBlock(Cache *c) { + Lock lock(&mtx_); + uptr fillpos = atomic_load_relaxed(&fillpos_); + if (fillpos == kL1Size) { + Printf("ThreadSanitizer: %s overflow (%zu*%zu). Dying.\n", name_, kL1Size, + kL2Size); + Die(); + } + VPrintf(2, "ThreadSanitizer: growing %s: %zu out of %zu*%zu\n", name_, + fillpos, kL1Size, kL2Size); + T *batch = (T *)MmapOrDie(kL2Size * sizeof(T), name_); + map_[fillpos] = batch; + // Reserve 0 as invalid index. + for (IndexT i = fillpos ? 0 : 1; i < kL2Size; i++) { + new (batch + i) T; + c->cache[c->pos++] = i + fillpos * kL2Size; + if (c->pos == Cache::kSize) + Drain(c); } + atomic_store_relaxed(&fillpos_, fillpos + 1); + CHECK(c->pos); } }; diff --git a/compiler-rt/lib/tsan/rtl/tsan_platform_mac.cpp b/compiler-rt/lib/tsan/rtl/tsan_platform_mac.cpp index 68ce5f83bdbd..1aac0fb27520 100644 --- a/compiler-rt/lib/tsan/rtl/tsan_platform_mac.cpp +++ b/compiler-rt/lib/tsan/rtl/tsan_platform_mac.cpp @@ -200,44 +200,26 @@ void WriteMemoryProfile(char *buf, uptr buf_size, u64 uptime_ns) { # if !SANITIZER_GO void InitializeShadowMemoryPlatform() { } -// On OS X, GCD worker threads are created without a call to pthread_create. We -// need to properly register these threads with ThreadCreate and ThreadStart. -// These threads don't have a parent thread, as they are created "spuriously". -// We're using a libpthread API that notifies us about a newly created thread. -// The `thread == pthread_self()` check indicates this is actually a worker -// thread. If it's just a regular thread, this hook is called on the parent -// thread. -typedef void (*pthread_introspection_hook_t)(unsigned int event, - pthread_t thread, void *addr, - size_t size); -extern "C" pthread_introspection_hook_t pthread_introspection_hook_install( - pthread_introspection_hook_t hook); -static const uptr PTHREAD_INTROSPECTION_THREAD_CREATE = 1; -static const uptr PTHREAD_INTROSPECTION_THREAD_TERMINATE = 3; -static pthread_introspection_hook_t prev_pthread_introspection_hook; -static void my_pthread_introspection_hook(unsigned int event, pthread_t thread, - void *addr, size_t size) { - if (event == PTHREAD_INTROSPECTION_THREAD_CREATE) { - if (thread == pthread_self()) { - // The current thread is a newly created GCD worker thread. - ThreadState *thr = cur_thread(); - Processor *proc = ProcCreate(); - ProcWire(proc, thr); - ThreadState *parent_thread_state = nullptr; // No parent. - Tid tid = ThreadCreate(parent_thread_state, 0, (uptr)thread, true); - CHECK_NE(tid, kMainTid); - ThreadStart(thr, tid, GetTid(), ThreadType::Worker); - } - } else if (event == PTHREAD_INTROSPECTION_THREAD_TERMINATE) { - CHECK_EQ(thread, pthread_self()); +// Register GCD worker threads, which are created without an observable call to +// pthread_create(). +static void ThreadCreateCallback(uptr thread, bool gcd_worker) { + if (gcd_worker) { ThreadState *thr = cur_thread(); - if (thr->tctx) { - DestroyThreadState(); - } + Processor *proc = ProcCreate(); + ProcWire(proc, thr); + ThreadState *parent_thread_state = nullptr; // No parent. + Tid tid = ThreadCreate(parent_thread_state, 0, (uptr)thread, true); + CHECK_NE(tid, kMainTid); + ThreadStart(thr, tid, GetTid(), ThreadType::Worker); } +} - if (prev_pthread_introspection_hook != nullptr) - prev_pthread_introspection_hook(event, thread, addr, size); +// Destroy thread state for *all* threads. +static void ThreadTerminateCallback(uptr thread) { + ThreadState *thr = cur_thread(); + if (thr->tctx) { + DestroyThreadState(); + } } #endif @@ -261,8 +243,11 @@ void InitializePlatform() { InitializeThreadStateStorage(); - prev_pthread_introspection_hook = - pthread_introspection_hook_install(&my_pthread_introspection_hook); + ThreadEventCallbacks callbacks = { + .create = ThreadCreateCallback, + .terminate = ThreadTerminateCallback, + }; + InstallPthreadIntrospectionHook(callbacks); #endif if (GetMacosAlignedVersion() >= MacosVersion(10, 14)) { diff --git a/compiler-rt/lib/tsan/rtl/tsan_rtl.cpp b/compiler-rt/lib/tsan/rtl/tsan_rtl.cpp index 607f373871b4..825a9d791ecc 100644 --- a/compiler-rt/lib/tsan/rtl/tsan_rtl.cpp +++ b/compiler-rt/lib/tsan/rtl/tsan_rtl.cpp @@ -210,17 +210,15 @@ static void DoResetImpl(uptr epoch) { // Clang does not understand locking all slots in the loop: // error: expecting mutex 'slot.mtx' to be held at start of each loop void DoReset(ThreadState* thr, uptr epoch) SANITIZER_NO_THREAD_SAFETY_ANALYSIS { - { - for (auto& slot : ctx->slots) { - slot.mtx.Lock(); - if (UNLIKELY(epoch == 0)) - epoch = ctx->global_epoch; - if (UNLIKELY(epoch != ctx->global_epoch)) { - // Epoch can't change once we've locked the first slot. - CHECK_EQ(slot.sid, 0); - slot.mtx.Unlock(); - return; - } + for (auto& slot : ctx->slots) { + slot.mtx.Lock(); + if (UNLIKELY(epoch == 0)) + epoch = ctx->global_epoch; + if (UNLIKELY(epoch != ctx->global_epoch)) { + // Epoch can't change once we've locked the first slot. + CHECK_EQ(slot.sid, 0); + slot.mtx.Unlock(); + return; } } DPrintf("#%d: DoReset epoch=%lu\n", thr ? thr->tid : -1, epoch); @@ -951,6 +949,15 @@ void TraceSwitchPartImpl(ThreadState* thr) { TraceMutexLock(thr, d.write ? EventType::kLock : EventType::kRLock, 0, d.addr, d.stack_id); } + // Callers of TraceSwitchPart expect that TraceAcquire will always succeed + // after the call. It's possible that TryTraceFunc/TraceMutexLock above + // filled the trace part exactly up to the TracePart::kAlignment gap + // and the next TraceAcquire won't succeed. Skip the gap to avoid that. + EventFunc *ev; + if (!TraceAcquire(thr, &ev)) { + CHECK(TraceSkipGap(thr)); + CHECK(TraceAcquire(thr, &ev)); + } { Lock lock(&ctx->slot_mtx); // There is a small chance that the slot may be not queued at this point. diff --git a/compiler-rt/lib/ubsan/ubsan_value.cpp b/compiler-rt/lib/ubsan/ubsan_value.cpp index 5a93a0d7fc2d..dc61e5b939d9 100644 --- a/compiler-rt/lib/ubsan/ubsan_value.cpp +++ b/compiler-rt/lib/ubsan/ubsan_value.cpp @@ -18,9 +18,7 @@ #include "sanitizer_common/sanitizer_libc.h" #include "sanitizer_common/sanitizer_mutex.h" -// TODO(dliew): Prefer '__APPLE__' here over 'SANITIZER_APPLE', as the latter is -// unclear. rdar://58124919 tracks using a more obviously portable guard. -#if defined(__APPLE__) +#if SANITIZER_APPLE #include <dlfcn.h> #endif @@ -29,7 +27,7 @@ using namespace __ubsan; typedef const char *(*ObjCGetClassNameTy)(void *); const char *__ubsan::getObjCClassName(ValueHandle Pointer) { -#if defined(__APPLE__) +#if SANITIZER_APPLE // We need to query the ObjC runtime for some information, but do not want // to introduce a static dependency from the ubsan runtime onto ObjC. Try to // grab a handle to the ObjC runtime used by the process. |