aboutsummaryrefslogtreecommitdiff
path: root/lib/scudo
diff options
context:
space:
mode:
Diffstat (limited to 'lib/scudo')
-rw-r--r--lib/scudo/CMakeLists.txt8
-rw-r--r--lib/scudo/scudo_allocator.cpp405
-rw-r--r--lib/scudo/scudo_allocator.h49
-rw-r--r--lib/scudo/scudo_allocator_secondary.h188
-rw-r--r--lib/scudo/scudo_flags.cpp26
-rw-r--r--lib/scudo/scudo_flags.h2
-rw-r--r--lib/scudo/scudo_interceptors.cpp2
-rw-r--r--lib/scudo/scudo_new_delete.cpp2
-rw-r--r--lib/scudo/scudo_termination.cpp19
-rw-r--r--lib/scudo/scudo_utils.cpp156
-rw-r--r--lib/scudo/scudo_utils.h26
11 files changed, 635 insertions, 248 deletions
diff --git a/lib/scudo/CMakeLists.txt b/lib/scudo/CMakeLists.txt
index 6cbb85f83255..332c3a972e6f 100644
--- a/lib/scudo/CMakeLists.txt
+++ b/lib/scudo/CMakeLists.txt
@@ -1,11 +1,10 @@
-add_custom_target(scudo)
-set_target_properties(scudo PROPERTIES FOLDER "Compiler-RT Misc")
+add_compiler_rt_component(scudo)
include_directories(..)
set(SCUDO_CFLAGS ${SANITIZER_COMMON_CFLAGS})
append_rtti_flag(OFF SCUDO_CFLAGS)
-list(APPEND SCUDO_CFLAGS -msse4.2 -mcx16)
+append_list_if(COMPILER_RT_HAS_MSSE4_2_FLAG -msse4.2 SCUDO_CFLAGS)
set(SCUDO_SOURCES
scudo_allocator.cpp
@@ -28,6 +27,3 @@ if(COMPILER_RT_HAS_SCUDO)
PARENT_TARGET scudo)
endforeach()
endif()
-
-add_dependencies(compiler-rt scudo)
-
diff --git a/lib/scudo/scudo_allocator.cpp b/lib/scudo/scudo_allocator.cpp
index 3ad499aed102..96cfbdbc1af7 100644
--- a/lib/scudo/scudo_allocator.cpp
+++ b/lib/scudo/scudo_allocator.cpp
@@ -22,23 +22,59 @@
#include <limits.h>
#include <pthread.h>
-#include <smmintrin.h>
-#include <atomic>
#include <cstring>
+// Hardware CRC32 is supported at compilation via the following:
+// - for i386 & x86_64: -msse4.2
+// - for ARM & AArch64: -march=armv8-a+crc
+// An additional check must be performed at runtime as well to make sure the
+// emitted instructions are valid on the target host.
+#if defined(__SSE4_2__) || defined(__ARM_FEATURE_CRC32)
+# ifdef __SSE4_2__
+# include <smmintrin.h>
+# define HW_CRC32 FIRST_32_SECOND_64(_mm_crc32_u32, _mm_crc32_u64)
+# endif
+# ifdef __ARM_FEATURE_CRC32
+# include <arm_acle.h>
+# define HW_CRC32 FIRST_32_SECOND_64(__crc32cw, __crc32cd)
+# endif
+#endif
+
namespace __scudo {
+#if SANITIZER_CAN_USE_ALLOCATOR64
const uptr AllocatorSpace = ~0ULL;
-const uptr AllocatorSize = 0x10000000000ULL;
-const uptr MinAlignmentLog = 4; // 16 bytes for x64
-const uptr MaxAlignmentLog = 24;
-
+const uptr AllocatorSize = 0x40000000000ULL;
typedef DefaultSizeClassMap SizeClassMap;
-typedef SizeClassAllocator64<AllocatorSpace, AllocatorSize, 0, SizeClassMap>
- PrimaryAllocator;
+struct AP {
+ static const uptr kSpaceBeg = AllocatorSpace;
+ static const uptr kSpaceSize = AllocatorSize;
+ static const uptr kMetadataSize = 0;
+ typedef __scudo::SizeClassMap SizeClassMap;
+ typedef NoOpMapUnmapCallback MapUnmapCallback;
+ static const uptr kFlags =
+ SizeClassAllocator64FlagMasks::kRandomShuffleChunks;
+};
+typedef SizeClassAllocator64<AP> PrimaryAllocator;
+#else
+// Currently, the 32-bit Sanitizer allocator has not yet benefited from all the
+// security improvements brought to the 64-bit one. This makes the 32-bit
+// version of Scudo slightly less toughened.
+static const uptr RegionSizeLog = 20;
+static const uptr NumRegions = SANITIZER_MMAP_RANGE_SIZE >> RegionSizeLog;
+# if SANITIZER_WORDSIZE == 32
+typedef FlatByteMap<NumRegions> ByteMap;
+# elif SANITIZER_WORDSIZE == 64
+typedef TwoLevelByteMap<(NumRegions >> 12), 1 << 12> ByteMap;
+# endif // SANITIZER_WORDSIZE
+typedef DefaultSizeClassMap SizeClassMap;
+typedef SizeClassAllocator32<0, SANITIZER_MMAP_RANGE_SIZE, 0, SizeClassMap,
+ RegionSizeLog, ByteMap> PrimaryAllocator;
+#endif // SANITIZER_CAN_USE_ALLOCATOR64
+
typedef SizeClassAllocatorLocalCache<PrimaryAllocator> AllocatorCache;
-typedef LargeMmapAllocator<> SecondaryAllocator;
+typedef ScudoLargeMmapAllocator SecondaryAllocator;
typedef CombinedAllocator<PrimaryAllocator, AllocatorCache, SecondaryAllocator>
ScudoAllocator;
@@ -46,67 +82,75 @@ static ScudoAllocator &getAllocator();
static thread_local Xorshift128Plus Prng;
// Global static cookie, initialized at start-up.
-static u64 Cookie;
+static uptr Cookie;
-enum ChunkState : u8 {
- ChunkAvailable = 0,
- ChunkAllocated = 1,
- ChunkQuarantine = 2
+enum : u8 {
+ CRC32Software = 0,
+ CRC32Hardware = 1,
};
+// We default to software CRC32 if the alternatives are not supported, either
+// at compilation or at runtime.
+static atomic_uint8_t HashAlgorithm = { CRC32Software };
-typedef unsigned __int128 PackedHeader;
-typedef std::atomic<PackedHeader> AtomicPackedHeader;
-
-// Our header requires 128-bit of storage on x64 (the only platform supported
-// as of now), which fits nicely with the alignment requirements.
-// Having the offset saves us from using functions such as GetBlockBegin, that
-// is fairly costly. Our first implementation used the MetaData as well, which
-// offers the advantage of being stored away from the chunk itself, but
-// accessing it was costly as well.
-// The header will be atomically loaded and stored using the 16-byte primitives
-// offered by the platform (likely requires cmpxchg16b support).
-struct UnpackedHeader {
- // 1st 8 bytes
- u16 Checksum : 16;
- u64 RequestedSize : 40; // Needed for reallocation purposes.
- u8 State : 2; // available, allocated, or quarantined
- u8 AllocType : 2; // malloc, new, new[], or memalign
- u8 Unused_0_ : 4;
- // 2nd 8 bytes
- u64 Offset : 20; // Offset from the beginning of the backend
- // allocation to the beginning chunk itself, in
- // multiples of MinAlignment. See comment about its
- // maximum value and test in Initialize.
- u64 Unused_1_ : 28;
- u16 Salt : 16;
-};
-
-COMPILER_CHECK(sizeof(UnpackedHeader) == sizeof(PackedHeader));
-
-const uptr ChunkHeaderSize = sizeof(PackedHeader);
+// Helper function that will compute the chunk checksum, being passed all the
+// the needed information as uptrs. It will opt for the hardware version of
+// the checksumming function if available.
+INLINE u32 hashUptrs(uptr Pointer, uptr *Array, uptr ArraySize, u8 HashType) {
+ u32 Crc;
+#if defined(__SSE4_2__) || defined(__ARM_FEATURE_CRC32)
+ if (HashType == CRC32Hardware) {
+ Crc = HW_CRC32(Cookie, Pointer);
+ for (uptr i = 0; i < ArraySize; i++)
+ Crc = HW_CRC32(Crc, Array[i]);
+ return Crc;
+ }
+#endif
+ Crc = computeCRC32(Cookie, Pointer);
+ for (uptr i = 0; i < ArraySize; i++)
+ Crc = computeCRC32(Crc, Array[i]);
+ return Crc;
+}
struct ScudoChunk : UnpackedHeader {
// We can't use the offset member of the chunk itself, as we would double
// fetch it without any warranty that it wouldn't have been tampered. To
// prevent this, we work with a local copy of the header.
- void *AllocBeg(UnpackedHeader *Header) {
+ void *getAllocBeg(UnpackedHeader *Header) {
return reinterpret_cast<void *>(
reinterpret_cast<uptr>(this) - (Header->Offset << MinAlignmentLog));
}
- // CRC32 checksum of the Chunk pointer and its ChunkHeader.
- // It currently uses the Intel Nehalem SSE4.2 crc32 64-bit instruction.
- u16 Checksum(UnpackedHeader *Header) const {
- u64 HeaderHolder[2];
- memcpy(HeaderHolder, Header, sizeof(HeaderHolder));
- u64 Crc = _mm_crc32_u64(Cookie, reinterpret_cast<uptr>(this));
- // This is somewhat of a shortcut. The checksum is stored in the 16 least
- // significant bits of the first 8 bytes of the header, hence zero-ing
- // those bits out. It would be more valid to zero the checksum field of the
- // UnpackedHeader, but would require holding an additional copy of it.
- Crc = _mm_crc32_u64(Crc, HeaderHolder[0] & 0xffffffffffff0000ULL);
- Crc = _mm_crc32_u64(Crc, HeaderHolder[1]);
- return static_cast<u16>(Crc);
+ // Returns the usable size for a chunk, meaning the amount of bytes from the
+ // beginning of the user data to the end of the backend allocated chunk.
+ uptr getUsableSize(UnpackedHeader *Header) {
+ uptr Size = getAllocator().GetActuallyAllocatedSize(getAllocBeg(Header));
+ if (Size == 0)
+ return Size;
+ return Size - AlignedChunkHeaderSize - (Header->Offset << MinAlignmentLog);
+ }
+
+ // Compute the checksum of the Chunk pointer and its ChunkHeader.
+ u16 computeChecksum(UnpackedHeader *Header) const {
+ UnpackedHeader ZeroChecksumHeader = *Header;
+ ZeroChecksumHeader.Checksum = 0;
+ uptr HeaderHolder[sizeof(UnpackedHeader) / sizeof(uptr)];
+ memcpy(&HeaderHolder, &ZeroChecksumHeader, sizeof(HeaderHolder));
+ u32 Hash = hashUptrs(reinterpret_cast<uptr>(this),
+ HeaderHolder,
+ ARRAY_SIZE(HeaderHolder),
+ atomic_load_relaxed(&HashAlgorithm));
+ return static_cast<u16>(Hash);
+ }
+
+ // Checks the validity of a chunk by verifying its checksum.
+ bool isValid() {
+ UnpackedHeader NewUnpackedHeader;
+ const AtomicPackedHeader *AtomicHeader =
+ reinterpret_cast<const AtomicPackedHeader *>(this);
+ PackedHeader NewPackedHeader =
+ AtomicHeader->load(std::memory_order_relaxed);
+ NewUnpackedHeader = bit_cast<UnpackedHeader>(NewPackedHeader);
+ return (NewUnpackedHeader.Checksum == computeChecksum(&NewUnpackedHeader));
}
// Loads and unpacks the header, verifying the checksum in the process.
@@ -116,16 +160,14 @@ struct ScudoChunk : UnpackedHeader {
PackedHeader NewPackedHeader =
AtomicHeader->load(std::memory_order_relaxed);
*NewUnpackedHeader = bit_cast<UnpackedHeader>(NewPackedHeader);
- if ((NewUnpackedHeader->Unused_0_ != 0) ||
- (NewUnpackedHeader->Unused_1_ != 0) ||
- (NewUnpackedHeader->Checksum != Checksum(NewUnpackedHeader))) {
+ if (NewUnpackedHeader->Checksum != computeChecksum(NewUnpackedHeader)) {
dieWithMessage("ERROR: corrupted chunk header at address %p\n", this);
}
}
// Packs and stores the header, computing the checksum in the process.
void storeHeader(UnpackedHeader *NewUnpackedHeader) {
- NewUnpackedHeader->Checksum = Checksum(NewUnpackedHeader);
+ NewUnpackedHeader->Checksum = computeChecksum(NewUnpackedHeader);
PackedHeader NewPackedHeader = bit_cast<PackedHeader>(*NewUnpackedHeader);
AtomicPackedHeader *AtomicHeader =
reinterpret_cast<AtomicPackedHeader *>(this);
@@ -137,7 +179,7 @@ struct ScudoChunk : UnpackedHeader {
// we are not being raced by a corruption occurring in another thread.
void compareExchangeHeader(UnpackedHeader *NewUnpackedHeader,
UnpackedHeader *OldUnpackedHeader) {
- NewUnpackedHeader->Checksum = Checksum(NewUnpackedHeader);
+ NewUnpackedHeader->Checksum = computeChecksum(NewUnpackedHeader);
PackedHeader NewPackedHeader = bit_cast<PackedHeader>(*NewUnpackedHeader);
PackedHeader OldPackedHeader = bit_cast<PackedHeader>(*OldUnpackedHeader);
AtomicPackedHeader *AtomicHeader =
@@ -154,7 +196,7 @@ struct ScudoChunk : UnpackedHeader {
static bool ScudoInitIsRunning = false;
static pthread_once_t GlobalInited = PTHREAD_ONCE_INIT;
-static pthread_key_t pkey;
+static pthread_key_t PThreadKey;
static thread_local bool ThreadInited = false;
static thread_local bool ThreadTornDown = false;
@@ -168,7 +210,7 @@ static void teardownThread(void *p) {
// like, so we wait until PTHREAD_DESTRUCTOR_ITERATIONS before draining the
// quarantine and swallowing the cache.
if (v < PTHREAD_DESTRUCTOR_ITERATIONS) {
- pthread_setspecific(pkey, reinterpret_cast<void *>(v + 1));
+ pthread_setspecific(PThreadKey, reinterpret_cast<void *>(v + 1));
return;
}
drainQuarantine();
@@ -181,23 +223,30 @@ static void initInternal() {
CHECK(!ScudoInitIsRunning && "Scudo init calls itself!");
ScudoInitIsRunning = true;
+ // Check is SSE4.2 is supported, if so, opt for the CRC32 hardware version.
+ if (testCPUFeature(CRC32CPUFeature)) {
+ atomic_store_relaxed(&HashAlgorithm, CRC32Hardware);
+ }
+
initFlags();
AllocatorOptions Options;
Options.setFrom(getFlags(), common_flags());
initAllocator(Options);
+ MaybeStartBackgroudThread();
+
ScudoInitIsRunning = false;
}
static void initGlobal() {
- pthread_key_create(&pkey, teardownThread);
+ pthread_key_create(&PThreadKey, teardownThread);
initInternal();
}
static void NOINLINE initThread() {
pthread_once(&GlobalInited, initGlobal);
- pthread_setspecific(pkey, reinterpret_cast<void *>(1));
+ pthread_setspecific(PThreadKey, reinterpret_cast<void *>(1));
getAllocator().InitCache(&Cache);
ThreadInited = true;
}
@@ -214,7 +263,7 @@ struct QuarantineCallback {
dieWithMessage("ERROR: invalid chunk state when recycling address %p\n",
Chunk);
}
- void *Ptr = Chunk->AllocBeg(&Header);
+ void *Ptr = Chunk->getAllocBeg(&Header);
getAllocator().Deallocate(Cache_, Ptr);
}
@@ -245,6 +294,7 @@ static thread_local QuarantineCache ThreadQuarantineCache;
void AllocatorOptions::setFrom(const Flags *f, const CommonFlags *cf) {
MayReturnNull = cf->allocator_may_return_null;
+ ReleaseToOSIntervalMs = cf->allocator_release_to_os_interval_ms;
QuarantineSizeMb = f->QuarantineSizeMb;
ThreadLocalQuarantineSizeKb = f->ThreadLocalQuarantineSizeKb;
DeallocationTypeMismatch = f->DeallocationTypeMismatch;
@@ -254,6 +304,7 @@ void AllocatorOptions::setFrom(const Flags *f, const CommonFlags *cf) {
void AllocatorOptions::copyTo(Flags *f, CommonFlags *cf) const {
cf->allocator_may_return_null = MayReturnNull;
+ cf->allocator_release_to_os_interval_ms = ReleaseToOSIntervalMs;
f->QuarantineSizeMb = QuarantineSizeMb;
f->ThreadLocalQuarantineSizeKb = ThreadLocalQuarantineSizeKb;
f->DeallocationTypeMismatch = DeallocationTypeMismatch;
@@ -262,9 +313,8 @@ void AllocatorOptions::copyTo(Flags *f, CommonFlags *cf) const {
}
struct Allocator {
- static const uptr MaxAllowedMallocSize = 1ULL << 40;
- static const uptr MinAlignment = 1 << MinAlignmentLog;
- static const uptr MaxAlignment = 1 << MaxAlignmentLog; // 16 MB
+ static const uptr MaxAllowedMallocSize =
+ FIRST_32_SECOND_64(2UL << 30, 1ULL << 40);
ScudoAllocator BackendAllocator;
ScudoQuarantine AllocatorQuarantine;
@@ -285,85 +335,129 @@ struct Allocator {
FallbackQuarantineCache(LINKER_INITIALIZED) {}
void init(const AllocatorOptions &Options) {
- // Currently SSE 4.2 support is required. This might change later.
- CHECK(testCPUFeature(SSE4_2)); // for crc32
-
// Verify that the header offset field can hold the maximum offset. In the
- // worst case scenario, the backend allocation is already aligned on
- // MaxAlignment, so in order to store the header and still be aligned, we
- // add an extra MaxAlignment. As a result, the offset from the beginning of
- // the backend allocation to the chunk will be MaxAlignment -
- // ChunkHeaderSize.
+ // case of the Secondary allocator, it takes care of alignment and the
+ // offset will always be 0. In the case of the Primary, the worst case
+ // scenario happens in the last size class, when the backend allocation
+ // would already be aligned on the requested alignment, which would happen
+ // to be the maximum alignment that would fit in that size class. As a
+ // result, the maximum offset will be at most the maximum alignment for the
+ // last size class minus the header size, in multiples of MinAlignment.
UnpackedHeader Header = {};
- uptr MaximumOffset = (MaxAlignment - ChunkHeaderSize) >> MinAlignmentLog;
- Header.Offset = MaximumOffset;
- if (Header.Offset != MaximumOffset) {
+ uptr MaxPrimaryAlignment = 1 << MostSignificantSetBitIndex(
+ SizeClassMap::kMaxSize - MinAlignment);
+ uptr MaxOffset = (MaxPrimaryAlignment - AlignedChunkHeaderSize) >>
+ MinAlignmentLog;
+ Header.Offset = MaxOffset;
+ if (Header.Offset != MaxOffset) {
dieWithMessage("ERROR: the maximum possible offset doesn't fit in the "
"header\n");
}
+ // Verify that we can fit the maximum amount of unused bytes in the header.
+ // Given that the Secondary fits the allocation to a page, the worst case
+ // scenario happens in the Primary. It will depend on the second to last
+ // and last class sizes, as well as the dynamic base for the Primary. The
+ // following is an over-approximation that works for our needs.
+ uptr MaxUnusedBytes = SizeClassMap::kMaxSize - 1 - AlignedChunkHeaderSize;
+ Header.UnusedBytes = MaxUnusedBytes;
+ if (Header.UnusedBytes != MaxUnusedBytes) {
+ dieWithMessage("ERROR: the maximum possible unused bytes doesn't fit in "
+ "the header\n");
+ }
DeallocationTypeMismatch = Options.DeallocationTypeMismatch;
DeleteSizeMismatch = Options.DeleteSizeMismatch;
ZeroContents = Options.ZeroContents;
- BackendAllocator.Init(Options.MayReturnNull);
- AllocatorQuarantine.Init(static_cast<uptr>(Options.QuarantineSizeMb) << 20,
- static_cast<uptr>(
- Options.ThreadLocalQuarantineSizeKb) << 10);
+ BackendAllocator.Init(Options.MayReturnNull, Options.ReleaseToOSIntervalMs);
+ AllocatorQuarantine.Init(
+ static_cast<uptr>(Options.QuarantineSizeMb) << 20,
+ static_cast<uptr>(Options.ThreadLocalQuarantineSizeKb) << 10);
BackendAllocator.InitCache(&FallbackAllocatorCache);
Cookie = Prng.Next();
}
+ // Helper function that checks for a valid Scudo chunk.
+ bool isValidPointer(const void *UserPtr) {
+ uptr ChunkBeg = reinterpret_cast<uptr>(UserPtr);
+ if (!IsAligned(ChunkBeg, MinAlignment)) {
+ return false;
+ }
+ ScudoChunk *Chunk =
+ reinterpret_cast<ScudoChunk *>(ChunkBeg - AlignedChunkHeaderSize);
+ return Chunk->isValid();
+ }
+
// Allocates a chunk.
void *allocate(uptr Size, uptr Alignment, AllocType Type) {
if (UNLIKELY(!ThreadInited))
initThread();
if (!IsPowerOfTwo(Alignment)) {
- dieWithMessage("ERROR: malloc alignment is not a power of 2\n");
+ dieWithMessage("ERROR: alignment is not a power of 2\n");
}
if (Alignment > MaxAlignment)
- return BackendAllocator.ReturnNullOrDie();
+ return BackendAllocator.ReturnNullOrDieOnBadRequest();
if (Alignment < MinAlignment)
Alignment = MinAlignment;
if (Size == 0)
Size = 1;
if (Size >= MaxAllowedMallocSize)
- return BackendAllocator.ReturnNullOrDie();
- uptr RoundedSize = RoundUpTo(Size, MinAlignment);
- uptr ExtraBytes = ChunkHeaderSize;
+ return BackendAllocator.ReturnNullOrDieOnBadRequest();
+
+ uptr NeededSize = RoundUpTo(Size, MinAlignment) + AlignedChunkHeaderSize;
if (Alignment > MinAlignment)
- ExtraBytes += Alignment;
- uptr NeededSize = RoundedSize + ExtraBytes;
+ NeededSize += Alignment;
if (NeededSize >= MaxAllowedMallocSize)
- return BackendAllocator.ReturnNullOrDie();
+ return BackendAllocator.ReturnNullOrDieOnBadRequest();
+
+ // Primary backed and Secondary backed allocations have a different
+ // treatment. We deal with alignment requirements of Primary serviced
+ // allocations here, but the Secondary will take care of its own alignment
+ // needs, which means we also have to work around some limitations of the
+ // combined allocator to accommodate the situation.
+ bool FromPrimary = PrimaryAllocator::CanAllocate(NeededSize, MinAlignment);
void *Ptr;
if (LIKELY(!ThreadTornDown)) {
- Ptr = BackendAllocator.Allocate(&Cache, NeededSize, MinAlignment);
+ Ptr = BackendAllocator.Allocate(&Cache, NeededSize,
+ FromPrimary ? MinAlignment : Alignment);
} else {
SpinMutexLock l(&FallbackMutex);
Ptr = BackendAllocator.Allocate(&FallbackAllocatorCache, NeededSize,
- MinAlignment);
+ FromPrimary ? MinAlignment : Alignment);
}
if (!Ptr)
- return BackendAllocator.ReturnNullOrDie();
+ return BackendAllocator.ReturnNullOrDieOnOOM();
+
+ uptr AllocBeg = reinterpret_cast<uptr>(Ptr);
+ // If the allocation was serviced by the secondary, the returned pointer
+ // accounts for ChunkHeaderSize to pass the alignment check of the combined
+ // allocator. Adjust it here.
+ if (!FromPrimary) {
+ AllocBeg -= AlignedChunkHeaderSize;
+ if (Alignment > MinAlignment)
+ NeededSize -= Alignment;
+ }
+ uptr ActuallyAllocatedSize = BackendAllocator.GetActuallyAllocatedSize(
+ reinterpret_cast<void *>(AllocBeg));
// If requested, we will zero out the entire contents of the returned chunk.
- if (ZeroContents && BackendAllocator.FromPrimary(Ptr))
- memset(Ptr, 0, BackendAllocator.GetActuallyAllocatedSize(Ptr));
+ if (ZeroContents && FromPrimary)
+ memset(Ptr, 0, ActuallyAllocatedSize);
- uptr AllocBeg = reinterpret_cast<uptr>(Ptr);
- uptr ChunkBeg = AllocBeg + ChunkHeaderSize;
+ uptr ChunkBeg = AllocBeg + AlignedChunkHeaderSize;
if (!IsAligned(ChunkBeg, Alignment))
ChunkBeg = RoundUpTo(ChunkBeg, Alignment);
CHECK_LE(ChunkBeg + Size, AllocBeg + NeededSize);
ScudoChunk *Chunk =
- reinterpret_cast<ScudoChunk *>(ChunkBeg - ChunkHeaderSize);
+ reinterpret_cast<ScudoChunk *>(ChunkBeg - AlignedChunkHeaderSize);
UnpackedHeader Header = {};
Header.State = ChunkAllocated;
- Header.Offset = (ChunkBeg - ChunkHeaderSize - AllocBeg) >> MinAlignmentLog;
+ uptr Offset = ChunkBeg - AlignedChunkHeaderSize - AllocBeg;
+ Header.Offset = Offset >> MinAlignmentLog;
Header.AllocType = Type;
- Header.RequestedSize = Size;
- Header.Salt = static_cast<u16>(Prng.Next());
+ Header.UnusedBytes = ActuallyAllocatedSize - Offset -
+ AlignedChunkHeaderSize - Size;
+ Header.Salt = static_cast<u8>(Prng.Next());
Chunk->storeHeader(&Header);
void *UserPtr = reinterpret_cast<void *>(ChunkBeg);
// TODO(kostyak): hooks sound like a terrible idea security wise but might
@@ -387,13 +481,14 @@ struct Allocator {
"aligned at address %p\n", UserPtr);
}
ScudoChunk *Chunk =
- reinterpret_cast<ScudoChunk *>(ChunkBeg - ChunkHeaderSize);
+ reinterpret_cast<ScudoChunk *>(ChunkBeg - AlignedChunkHeaderSize);
UnpackedHeader OldHeader;
Chunk->loadHeader(&OldHeader);
if (OldHeader.State != ChunkAllocated) {
dieWithMessage("ERROR: invalid chunk state when deallocating address "
- "%p\n", Chunk);
+ "%p\n", UserPtr);
}
+ uptr UsableSize = Chunk->getUsableSize(&OldHeader);
UnpackedHeader NewHeader = OldHeader;
NewHeader.State = ChunkQuarantine;
Chunk->compareExchangeHeader(&NewHeader, &OldHeader);
@@ -407,69 +502,40 @@ struct Allocator {
}
}
}
- uptr Size = NewHeader.RequestedSize;
+ uptr Size = UsableSize - OldHeader.UnusedBytes;
if (DeleteSizeMismatch) {
if (DeleteSize && DeleteSize != Size) {
dieWithMessage("ERROR: invalid sized delete on chunk at address %p\n",
Chunk);
}
}
+
if (LIKELY(!ThreadTornDown)) {
AllocatorQuarantine.Put(&ThreadQuarantineCache,
- QuarantineCallback(&Cache), Chunk, Size);
+ QuarantineCallback(&Cache), Chunk, UsableSize);
} else {
SpinMutexLock l(&FallbackMutex);
AllocatorQuarantine.Put(&FallbackQuarantineCache,
QuarantineCallback(&FallbackAllocatorCache),
- Chunk, Size);
+ Chunk, UsableSize);
}
}
- // Returns the actual usable size of a chunk. Since this requires loading the
- // header, we will return it in the second parameter, as it can be required
- // by the caller to perform additional processing.
- uptr getUsableSize(const void *Ptr, UnpackedHeader *Header) {
- if (UNLIKELY(!ThreadInited))
- initThread();
- if (!Ptr)
- return 0;
- uptr ChunkBeg = reinterpret_cast<uptr>(Ptr);
- ScudoChunk *Chunk =
- reinterpret_cast<ScudoChunk *>(ChunkBeg - ChunkHeaderSize);
- Chunk->loadHeader(Header);
- // Getting the usable size of a chunk only makes sense if it's allocated.
- if (Header->State != ChunkAllocated) {
- dieWithMessage("ERROR: attempted to size a non-allocated chunk at "
- "address %p\n", Chunk);
- }
- uptr Size =
- BackendAllocator.GetActuallyAllocatedSize(Chunk->AllocBeg(Header));
- // UsableSize works as malloc_usable_size, which is also what (AFAIU)
- // tcmalloc's MallocExtension::GetAllocatedSize aims at providing. This
- // means we will return the size of the chunk from the user beginning to
- // the end of the 'user' allocation, hence us subtracting the header size
- // and the offset from the size.
- if (Size == 0)
- return Size;
- return Size - ChunkHeaderSize - (Header->Offset << MinAlignmentLog);
- }
-
- // Helper function that doesn't care about the header.
- uptr getUsableSize(const void *Ptr) {
- UnpackedHeader Header;
- return getUsableSize(Ptr, &Header);
- }
-
// Reallocates a chunk. We can save on a new allocation if the new requested
// size still fits in the chunk.
void *reallocate(void *OldPtr, uptr NewSize) {
if (UNLIKELY(!ThreadInited))
initThread();
- UnpackedHeader OldHeader;
- uptr Size = getUsableSize(OldPtr, &OldHeader);
uptr ChunkBeg = reinterpret_cast<uptr>(OldPtr);
ScudoChunk *Chunk =
- reinterpret_cast<ScudoChunk *>(ChunkBeg - ChunkHeaderSize);
+ reinterpret_cast<ScudoChunk *>(ChunkBeg - AlignedChunkHeaderSize);
+ UnpackedHeader OldHeader;
+ Chunk->loadHeader(&OldHeader);
+ if (OldHeader.State != ChunkAllocated) {
+ dieWithMessage("ERROR: invalid chunk state when reallocating address "
+ "%p\n", OldPtr);
+ }
+ uptr Size = Chunk->getUsableSize(&OldHeader);
if (OldHeader.AllocType != FromMalloc) {
dieWithMessage("ERROR: invalid chunk type when reallocating address %p\n",
Chunk);
@@ -477,7 +543,7 @@ struct Allocator {
UnpackedHeader NewHeader = OldHeader;
// The new size still fits in the current chunk.
if (NewSize <= Size) {
- NewHeader.RequestedSize = NewSize;
+ NewHeader.UnusedBytes = Size - NewSize;
Chunk->compareExchangeHeader(&NewHeader, &OldHeader);
return OldPtr;
}
@@ -485,29 +551,48 @@ struct Allocator {
// old one.
void *NewPtr = allocate(NewSize, MinAlignment, FromMalloc);
if (NewPtr) {
- uptr OldSize = OldHeader.RequestedSize;
+ uptr OldSize = Size - OldHeader.UnusedBytes;
memcpy(NewPtr, OldPtr, Min(NewSize, OldSize));
NewHeader.State = ChunkQuarantine;
Chunk->compareExchangeHeader(&NewHeader, &OldHeader);
if (LIKELY(!ThreadTornDown)) {
AllocatorQuarantine.Put(&ThreadQuarantineCache,
- QuarantineCallback(&Cache), Chunk, OldSize);
+ QuarantineCallback(&Cache), Chunk, Size);
} else {
SpinMutexLock l(&FallbackMutex);
AllocatorQuarantine.Put(&FallbackQuarantineCache,
QuarantineCallback(&FallbackAllocatorCache),
- Chunk, OldSize);
+ Chunk, Size);
}
}
return NewPtr;
}
+ // Helper function that returns the actual usable size of a chunk.
+ uptr getUsableSize(const void *Ptr) {
+ if (UNLIKELY(!ThreadInited))
+ initThread();
+ if (!Ptr)
+ return 0;
+ uptr ChunkBeg = reinterpret_cast<uptr>(Ptr);
+ ScudoChunk *Chunk =
+ reinterpret_cast<ScudoChunk *>(ChunkBeg - AlignedChunkHeaderSize);
+ UnpackedHeader Header;
+ Chunk->loadHeader(&Header);
+ // Getting the usable size of a chunk only makes sense if it's allocated.
+ if (Header.State != ChunkAllocated) {
+ dieWithMessage("ERROR: invalid chunk state when sizing address %p\n",
+ Ptr);
+ }
+ return Chunk->getUsableSize(&Header);
+ }
+
void *calloc(uptr NMemB, uptr Size) {
if (UNLIKELY(!ThreadInited))
initThread();
uptr Total = NMemB * Size;
if (Size != 0 && Total / Size != NMemB) // Overflow check
- return BackendAllocator.ReturnNullOrDie();
+ return BackendAllocator.ReturnNullOrDieOnBadRequest();
void *Ptr = allocate(Total, MinAlignment, FromMalloc);
// If ZeroContents, the content of the chunk has already been zero'd out.
if (!ZeroContents && Ptr && BackendAllocator.FromPrimary(Ptr))
@@ -536,7 +621,7 @@ void drainQuarantine() {
}
void *scudoMalloc(uptr Size, AllocType Type) {
- return Instance.allocate(Size, Allocator::MinAlignment, Type);
+ return Instance.allocate(Size, MinAlignment, Type);
}
void scudoFree(void *Ptr, AllocType Type) {
@@ -549,7 +634,7 @@ void scudoSizedFree(void *Ptr, uptr Size, AllocType Type) {
void *scudoRealloc(void *Ptr, uptr Size) {
if (!Ptr)
- return Instance.allocate(Size, Allocator::MinAlignment, FromMalloc);
+ return Instance.allocate(Size, MinAlignment, FromMalloc);
if (Size == 0) {
Instance.deallocate(Ptr, 0, FromMalloc);
return nullptr;
@@ -596,7 +681,7 @@ uptr scudoMallocUsableSize(void *Ptr) {
return Instance.getUsableSize(Ptr);
}
-} // namespace __scudo
+} // namespace __scudo
using namespace __scudo;
@@ -626,10 +711,10 @@ uptr __sanitizer_get_estimated_allocated_size(uptr size) {
return size;
}
-int __sanitizer_get_ownership(const void *p) {
- return Instance.getUsableSize(p) != 0;
+int __sanitizer_get_ownership(const void *Ptr) {
+ return Instance.isValidPointer(Ptr);
}
-uptr __sanitizer_get_allocated_size(const void *p) {
- return Instance.getUsableSize(p);
+uptr __sanitizer_get_allocated_size(const void *Ptr) {
+ return Instance.getUsableSize(Ptr);
}
diff --git a/lib/scudo/scudo_allocator.h b/lib/scudo/scudo_allocator.h
index 7e9c7886055e..6431a2aa07d7 100644
--- a/lib/scudo/scudo_allocator.h
+++ b/lib/scudo/scudo_allocator.h
@@ -14,14 +14,12 @@
#ifndef SCUDO_ALLOCATOR_H_
#define SCUDO_ALLOCATOR_H_
-#ifndef __x86_64__
-# error "The Scudo hardened allocator currently only supports x86_64."
-#endif
-
#include "scudo_flags.h"
#include "sanitizer_common/sanitizer_allocator.h"
+#include <atomic>
+
namespace __scudo {
enum AllocType : u8 {
@@ -31,10 +29,49 @@ enum AllocType : u8 {
FromMemalign = 3, // Memory block came from memalign, posix_memalign, etc.
};
+enum ChunkState : u8 {
+ ChunkAvailable = 0,
+ ChunkAllocated = 1,
+ ChunkQuarantine = 2
+};
+
+// Our header requires 64 bits of storage. Having the offset saves us from
+// using functions such as GetBlockBegin, that is fairly costly. Our first
+// implementation used the MetaData as well, which offers the advantage of
+// being stored away from the chunk itself, but accessing it was costly as
+// well. The header will be atomically loaded and stored using the 16-byte
+// primitives offered by the platform (likely requires cmpxchg16b support).
+typedef u64 PackedHeader;
+struct UnpackedHeader {
+ u64 Checksum : 16;
+ u64 UnusedBytes : 20; // Needed for reallocation purposes.
+ u64 State : 2; // available, allocated, or quarantined
+ u64 AllocType : 2; // malloc, new, new[], or memalign
+ u64 Offset : 16; // Offset from the beginning of the backend
+ // allocation to the beginning of the chunk itself,
+ // in multiples of MinAlignment. See comment about
+ // its maximum value and test in init().
+ u64 Salt : 8;
+};
+
+typedef std::atomic<PackedHeader> AtomicPackedHeader;
+COMPILER_CHECK(sizeof(UnpackedHeader) == sizeof(PackedHeader));
+
+// Minimum alignment of 8 bytes for 32-bit, 16 for 64-bit
+const uptr MinAlignmentLog = FIRST_32_SECOND_64(3, 4);
+const uptr MaxAlignmentLog = 24; // 16 MB
+const uptr MinAlignment = 1 << MinAlignmentLog;
+const uptr MaxAlignment = 1 << MaxAlignmentLog;
+
+const uptr ChunkHeaderSize = sizeof(PackedHeader);
+const uptr AlignedChunkHeaderSize =
+ (ChunkHeaderSize + MinAlignment - 1) & ~(MinAlignment - 1);
+
struct AllocatorOptions {
u32 QuarantineSizeMb;
u32 ThreadLocalQuarantineSizeKb;
bool MayReturnNull;
+ s32 ReleaseToOSIntervalMs;
bool DeallocationTypeMismatch;
bool DeleteSizeMismatch;
bool ZeroContents;
@@ -58,6 +95,8 @@ int scudoPosixMemalign(void **MemPtr, uptr Alignment, uptr Size);
void *scudoAlignedAlloc(uptr Alignment, uptr Size);
uptr scudoMallocUsableSize(void *Ptr);
-} // namespace __scudo
+#include "scudo_allocator_secondary.h"
+
+} // namespace __scudo
#endif // SCUDO_ALLOCATOR_H_
diff --git a/lib/scudo/scudo_allocator_secondary.h b/lib/scudo/scudo_allocator_secondary.h
new file mode 100644
index 000000000000..b984f0db4dbd
--- /dev/null
+++ b/lib/scudo/scudo_allocator_secondary.h
@@ -0,0 +1,188 @@
+//===-- scudo_allocator_secondary.h -----------------------------*- C++ -*-===//
+//
+// The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+///
+/// Scudo Secondary Allocator.
+/// This services allocation that are too large to be serviced by the Primary
+/// Allocator. It is directly backed by the memory mapping functions of the
+/// operating system.
+///
+//===----------------------------------------------------------------------===//
+
+#ifndef SCUDO_ALLOCATOR_SECONDARY_H_
+#define SCUDO_ALLOCATOR_SECONDARY_H_
+
+#ifndef SCUDO_ALLOCATOR_H_
+# error "This file must be included inside scudo_allocator.h."
+#endif
+
+class ScudoLargeMmapAllocator {
+ public:
+
+ void Init(bool AllocatorMayReturnNull) {
+ PageSize = GetPageSizeCached();
+ atomic_store(&MayReturnNull, AllocatorMayReturnNull, memory_order_relaxed);
+ }
+
+ void *Allocate(AllocatorStats *Stats, uptr Size, uptr Alignment) {
+ // The Scudo frontend prevents us from allocating more than
+ // MaxAllowedMallocSize, so integer overflow checks would be superfluous.
+ uptr MapSize = Size + SecondaryHeaderSize;
+ MapSize = RoundUpTo(MapSize, PageSize);
+ // Account for 2 guard pages, one before and one after the chunk.
+ MapSize += 2 * PageSize;
+ // The size passed to the Secondary comprises the alignment, if large
+ // enough. Subtract it here to get the requested size, including header.
+ if (Alignment > MinAlignment)
+ Size -= Alignment;
+
+ uptr MapBeg = reinterpret_cast<uptr>(MmapNoAccess(MapSize));
+ if (MapBeg == ~static_cast<uptr>(0))
+ return ReturnNullOrDieOnOOM();
+ // A page-aligned pointer is assumed after that, so check it now.
+ CHECK(IsAligned(MapBeg, PageSize));
+ uptr MapEnd = MapBeg + MapSize;
+ // The beginning of the user area for that allocation comes after the
+ // initial guard page, and both headers. This is the pointer that has to
+ // abide by alignment requirements.
+ uptr UserBeg = MapBeg + PageSize + HeadersSize;
+
+ // In the rare event of larger alignments, we will attempt to fit the mmap
+ // area better and unmap extraneous memory. This will also ensure that the
+ // offset and unused bytes field of the header stay small.
+ if (Alignment > MinAlignment) {
+ if (UserBeg & (Alignment - 1))
+ UserBeg += Alignment - (UserBeg & (Alignment - 1));
+ CHECK_GE(UserBeg, MapBeg);
+ uptr NewMapBeg = RoundDownTo(UserBeg - HeadersSize, PageSize) - PageSize;
+ CHECK_GE(NewMapBeg, MapBeg);
+ uptr NewMapEnd = RoundUpTo(UserBeg + (Size - AlignedChunkHeaderSize),
+ PageSize) + PageSize;
+ CHECK_LE(NewMapEnd, MapEnd);
+ // Unmap the extra memory if it's large enough, on both sides.
+ uptr Diff = NewMapBeg - MapBeg;
+ if (Diff > PageSize)
+ UnmapOrDie(reinterpret_cast<void *>(MapBeg), Diff);
+ Diff = MapEnd - NewMapEnd;
+ if (Diff > PageSize)
+ UnmapOrDie(reinterpret_cast<void *>(NewMapEnd), Diff);
+ MapBeg = NewMapBeg;
+ MapEnd = NewMapEnd;
+ MapSize = NewMapEnd - NewMapBeg;
+ }
+
+ uptr UserEnd = UserBeg + (Size - AlignedChunkHeaderSize);
+ CHECK_LE(UserEnd, MapEnd - PageSize);
+ // Actually mmap the memory, preserving the guard pages on either side.
+ CHECK_EQ(MapBeg + PageSize, reinterpret_cast<uptr>(
+ MmapFixedOrDie(MapBeg + PageSize, MapSize - 2 * PageSize)));
+ uptr Ptr = UserBeg - AlignedChunkHeaderSize;
+ SecondaryHeader *Header = getHeader(Ptr);
+ Header->MapBeg = MapBeg;
+ Header->MapSize = MapSize;
+ // The primary adds the whole class size to the stats when allocating a
+ // chunk, so we will do something similar here. But we will not account for
+ // the guard pages.
+ Stats->Add(AllocatorStatAllocated, MapSize - 2 * PageSize);
+ Stats->Add(AllocatorStatMapped, MapSize - 2 * PageSize);
+
+ return reinterpret_cast<void *>(UserBeg);
+ }
+
+ void *ReturnNullOrDieOnBadRequest() {
+ if (atomic_load(&MayReturnNull, memory_order_acquire))
+ return nullptr;
+ ReportAllocatorCannotReturnNull(false);
+ }
+
+ void *ReturnNullOrDieOnOOM() {
+ if (atomic_load(&MayReturnNull, memory_order_acquire))
+ return nullptr;
+ ReportAllocatorCannotReturnNull(true);
+ }
+
+ void SetMayReturnNull(bool AllocatorMayReturnNull) {
+ atomic_store(&MayReturnNull, AllocatorMayReturnNull, memory_order_release);
+ }
+
+ void Deallocate(AllocatorStats *Stats, void *Ptr) {
+ SecondaryHeader *Header = getHeader(Ptr);
+ Stats->Sub(AllocatorStatAllocated, Header->MapSize - 2 * PageSize);
+ Stats->Sub(AllocatorStatMapped, Header->MapSize - 2 * PageSize);
+ UnmapOrDie(reinterpret_cast<void *>(Header->MapBeg), Header->MapSize);
+ }
+
+ uptr TotalMemoryUsed() {
+ UNIMPLEMENTED();
+ }
+
+ bool PointerIsMine(const void *Ptr) {
+ UNIMPLEMENTED();
+ }
+
+ uptr GetActuallyAllocatedSize(void *Ptr) {
+ SecondaryHeader *Header = getHeader(Ptr);
+ // Deduct PageSize as MapEnd includes the trailing guard page.
+ uptr MapEnd = Header->MapBeg + Header->MapSize - PageSize;
+ return MapEnd - reinterpret_cast<uptr>(Ptr);
+ }
+
+ void *GetMetaData(const void *Ptr) {
+ UNIMPLEMENTED();
+ }
+
+ void *GetBlockBegin(const void *Ptr) {
+ UNIMPLEMENTED();
+ }
+
+ void *GetBlockBeginFastLocked(void *Ptr) {
+ UNIMPLEMENTED();
+ }
+
+ void PrintStats() {
+ UNIMPLEMENTED();
+ }
+
+ void ForceLock() {
+ UNIMPLEMENTED();
+ }
+
+ void ForceUnlock() {
+ UNIMPLEMENTED();
+ }
+
+ void ForEachChunk(ForEachChunkCallback Callback, void *Arg) {
+ UNIMPLEMENTED();
+ }
+
+ private:
+ // A Secondary allocated chunk header contains the base of the mapping and
+ // its size. Currently, the base is always a page before the header, but
+ // we might want to extend that number in the future based on the size of
+ // the allocation.
+ struct SecondaryHeader {
+ uptr MapBeg;
+ uptr MapSize;
+ };
+ // Check that sizeof(SecondaryHeader) is a multiple of MinAlignment.
+ COMPILER_CHECK((sizeof(SecondaryHeader) & (MinAlignment - 1)) == 0);
+
+ SecondaryHeader *getHeader(uptr Ptr) {
+ return reinterpret_cast<SecondaryHeader*>(Ptr - sizeof(SecondaryHeader));
+ }
+ SecondaryHeader *getHeader(const void *Ptr) {
+ return getHeader(reinterpret_cast<uptr>(Ptr));
+ }
+
+ const uptr SecondaryHeaderSize = sizeof(SecondaryHeader);
+ const uptr HeadersSize = SecondaryHeaderSize + AlignedChunkHeaderSize;
+ uptr PageSize;
+ atomic_uint8_t MayReturnNull;
+};
+
+#endif // SCUDO_ALLOCATOR_SECONDARY_H_
diff --git a/lib/scudo/scudo_flags.cpp b/lib/scudo/scudo_flags.cpp
index 430dcd2995dd..b9c838107305 100644
--- a/lib/scudo/scudo_flags.cpp
+++ b/lib/scudo/scudo_flags.cpp
@@ -17,9 +17,12 @@
#include "sanitizer_common/sanitizer_flags.h"
#include "sanitizer_common/sanitizer_flag_parser.h"
+extern "C" SANITIZER_INTERFACE_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE
+const char* __scudo_default_options();
+
namespace __scudo {
-Flags scudo_flags_dont_use_directly; // use via flags().
+Flags ScudoFlags; // Use via getFlags().
void Flags::setDefaults() {
#define SCUDO_FLAG(Type, Name, DefaultValue, Description) Name = DefaultValue;
@@ -34,6 +37,10 @@ static void RegisterScudoFlags(FlagParser *parser, Flags *f) {
#undef SCUDO_FLAG
}
+static const char *callGetScudoDefaultOptions() {
+ return (&__scudo_default_options) ? __scudo_default_options() : "";
+}
+
void initFlags() {
SetCommonFlagsDefaults();
{
@@ -45,11 +52,16 @@ void initFlags() {
Flags *f = getFlags();
f->setDefaults();
- FlagParser scudo_parser;
- RegisterScudoFlags(&scudo_parser, f);
- RegisterCommonFlags(&scudo_parser);
+ FlagParser ScudoParser;
+ RegisterScudoFlags(&ScudoParser, f);
+ RegisterCommonFlags(&ScudoParser);
- scudo_parser.ParseString(GetEnv("SCUDO_OPTIONS"));
+ // Override from user-specified string.
+ const char *ScudoDefaultOptions = callGetScudoDefaultOptions();
+ ScudoParser.ParseString(ScudoDefaultOptions);
+
+ // Override from environment.
+ ScudoParser.ParseString(GetEnv("SCUDO_OPTIONS"));
InitializeCommonFlags();
@@ -75,7 +87,7 @@ void initFlags() {
}
Flags *getFlags() {
- return &scudo_flags_dont_use_directly;
+ return &ScudoFlags;
}
-}
+} // namespace __scudo
diff --git a/lib/scudo/scudo_flags.h b/lib/scudo/scudo_flags.h
index c16f635d366f..d4ae31031fbb 100644
--- a/lib/scudo/scudo_flags.h
+++ b/lib/scudo/scudo_flags.h
@@ -28,6 +28,6 @@ Flags *getFlags();
void initFlags();
-} // namespace __scudo
+} // namespace __scudo
#endif // SCUDO_FLAGS_H_
diff --git a/lib/scudo/scudo_interceptors.cpp b/lib/scudo/scudo_interceptors.cpp
index 9204652d8b7d..735a13196757 100644
--- a/lib/scudo/scudo_interceptors.cpp
+++ b/lib/scudo/scudo_interceptors.cpp
@@ -72,4 +72,4 @@ INTERCEPTOR(int, mallopt, int cmd, int value) {
return -1;
}
-#endif // SANITIZER_LINUX
+#endif // SANITIZER_LINUX
diff --git a/lib/scudo/scudo_new_delete.cpp b/lib/scudo/scudo_new_delete.cpp
index 172f5659c933..c022bd0acbe7 100644
--- a/lib/scudo/scudo_new_delete.cpp
+++ b/lib/scudo/scudo_new_delete.cpp
@@ -24,7 +24,7 @@ using namespace __scudo;
// Fake std::nothrow_t to avoid including <new>.
namespace std {
struct nothrow_t {};
-} // namespace std
+} // namespace std
CXX_OPERATOR_ATTRIBUTE
void *operator new(size_t size) {
diff --git a/lib/scudo/scudo_termination.cpp b/lib/scudo/scudo_termination.cpp
index 32421d3d810f..c441ff3c126a 100644
--- a/lib/scudo/scudo_termination.cpp
+++ b/lib/scudo/scudo_termination.cpp
@@ -13,15 +13,17 @@
///
//===----------------------------------------------------------------------===//
+#include "scudo_utils.h"
+
#include "sanitizer_common/sanitizer_common.h"
namespace __sanitizer {
-bool AddDieCallback(DieCallbackType callback) { return true; }
+bool AddDieCallback(DieCallbackType Callback) { return true; }
-bool RemoveDieCallback(DieCallbackType callback) { return true; }
+bool RemoveDieCallback(DieCallbackType Callback) { return true; }
-void SetUserDieCallback(DieCallbackType callback) {}
+void SetUserDieCallback(DieCallbackType Callback) {}
void NORETURN Die() {
if (common_flags()->abort_on_error)
@@ -31,11 +33,10 @@ void NORETURN Die() {
void SetCheckFailedCallback(CheckFailedCallbackType callback) {}
-void NORETURN CheckFailed(const char *file, int line, const char *cond,
- u64 v1, u64 v2) {
- Report("Sanitizer CHECK failed: %s:%d %s (%lld, %lld)\n", file, line, cond,
- v1, v2);
- Die();
+void NORETURN CheckFailed(const char *File, int Line, const char *Condition,
+ u64 Value1, u64 Value2) {
+ __scudo::dieWithMessage("Scudo CHECK failed: %s:%d %s (%lld, %lld)\n",
+ File, Line, Condition, Value1, Value2);
}
-} // namespace __sanitizer
+} // namespace __sanitizer
diff --git a/lib/scudo/scudo_utils.cpp b/lib/scudo/scudo_utils.cpp
index 6b96e84ee9d9..c0269ec11efb 100644
--- a/lib/scudo/scudo_utils.cpp
+++ b/lib/scudo/scudo_utils.cpp
@@ -17,6 +17,9 @@
#include <fcntl.h>
#include <stdarg.h>
#include <unistd.h>
+#if defined(__x86_64__) || defined(__i386__)
+# include <cpuid.h>
+#endif
#include <cstring>
@@ -28,14 +31,14 @@ namespace __sanitizer {
extern int VSNPrintf(char *buff, int buff_length, const char *format,
va_list args);
-} // namespace __sanitizer
+} // namespace __sanitizer
namespace __scudo {
FORMAT(1, 2)
-void dieWithMessage(const char *Format, ...) {
- // Our messages are tiny, 128 characters is more than enough.
- char Message[128];
+void NORETURN dieWithMessage(const char *Format, ...) {
+ // Our messages are tiny, 256 characters is more than enough.
+ char Message[256];
va_list Args;
va_start(Args, Format);
__sanitizer::VSNPrintf(Message, sizeof(Message), Format, Args);
@@ -44,60 +47,61 @@ void dieWithMessage(const char *Format, ...) {
Die();
}
+#if defined(__x86_64__) || defined(__i386__)
+// i386 and x86_64 specific code to detect CRC32 hardware support via CPUID.
+// CRC32 requires the SSE 4.2 instruction set.
typedef struct {
u32 Eax;
u32 Ebx;
u32 Ecx;
u32 Edx;
-} CPUIDInfo;
+} CPUIDRegs;
-static void getCPUID(CPUIDInfo *info, u32 leaf, u32 subleaf)
+static void getCPUID(CPUIDRegs *Regs, u32 Level)
{
- asm volatile("cpuid"
- : "=a" (info->Eax), "=b" (info->Ebx), "=c" (info->Ecx), "=d" (info->Edx)
- : "a" (leaf), "c" (subleaf)
- );
+ __get_cpuid(Level, &Regs->Eax, &Regs->Ebx, &Regs->Ecx, &Regs->Edx);
}
-// Returns true is the CPU is a "GenuineIntel" or "AuthenticAMD"
-static bool isSupportedCPU()
-{
- CPUIDInfo Info;
-
- getCPUID(&Info, 0, 0);
- if (memcmp(reinterpret_cast<char *>(&Info.Ebx), "Genu", 4) == 0 &&
- memcmp(reinterpret_cast<char *>(&Info.Edx), "ineI", 4) == 0 &&
- memcmp(reinterpret_cast<char *>(&Info.Ecx), "ntel", 4) == 0) {
- return true;
- }
- if (memcmp(reinterpret_cast<char *>(&Info.Ebx), "Auth", 4) == 0 &&
- memcmp(reinterpret_cast<char *>(&Info.Edx), "enti", 4) == 0 &&
- memcmp(reinterpret_cast<char *>(&Info.Ecx), "cAMD", 4) == 0) {
- return true;
+CPUIDRegs getCPUFeatures() {
+ CPUIDRegs VendorRegs = {};
+ getCPUID(&VendorRegs, 0);
+ bool IsIntel =
+ (VendorRegs.Ebx == signature_INTEL_ebx) &&
+ (VendorRegs.Edx == signature_INTEL_edx) &&
+ (VendorRegs.Ecx == signature_INTEL_ecx);
+ bool IsAMD =
+ (VendorRegs.Ebx == signature_AMD_ebx) &&
+ (VendorRegs.Edx == signature_AMD_edx) &&
+ (VendorRegs.Ecx == signature_AMD_ecx);
+ // Default to an empty feature set if not on a supported CPU.
+ CPUIDRegs FeaturesRegs = {};
+ if (IsIntel || IsAMD) {
+ getCPUID(&FeaturesRegs, 1);
}
- return false;
+ return FeaturesRegs;
}
-bool testCPUFeature(CPUFeature feature)
+#ifndef bit_SSE4_2
+#define bit_SSE4_2 bit_SSE42 // clang and gcc have different defines.
+#endif
+
+bool testCPUFeature(CPUFeature Feature)
{
- static bool InfoInitialized = false;
- static CPUIDInfo CPUInfo = {};
+ static CPUIDRegs FeaturesRegs = getCPUFeatures();
- if (InfoInitialized == false) {
- if (isSupportedCPU() == true)
- getCPUID(&CPUInfo, 1, 0);
- else
- UNIMPLEMENTED();
- InfoInitialized = true;
- }
- switch (feature) {
- case SSE4_2:
- return ((CPUInfo.Ecx >> 20) & 0x1) != 0;
+ switch (Feature) {
+ case CRC32CPUFeature: // CRC32 is provided by SSE 4.2.
+ return !!(FeaturesRegs.Ecx & bit_SSE4_2);
default:
break;
}
return false;
}
+#else
+bool testCPUFeature(CPUFeature Feature) {
+ return false;
+}
+#endif // defined(__x86_64__) || defined(__i386__)
// readRetry will attempt to read Count bytes from the Fd specified, and if
// interrupted will retry to read additional bytes to reach Count.
@@ -117,17 +121,77 @@ static ssize_t readRetry(int Fd, u8 *Buffer, size_t Count) {
return AmountRead;
}
-// Default constructor for Xorshift128Plus seeds the state with /dev/urandom
-Xorshift128Plus::Xorshift128Plus() {
+static void fillRandom(u8 *Data, ssize_t Size) {
int Fd = open("/dev/urandom", O_RDONLY);
- bool Success = readRetry(Fd, reinterpret_cast<u8 *>(&State_0_),
- sizeof(State_0_)) == sizeof(State_0_);
- Success &= readRetry(Fd, reinterpret_cast<u8 *>(&State_1_),
- sizeof(State_1_)) == sizeof(State_1_);
+ if (Fd < 0) {
+ dieWithMessage("ERROR: failed to open /dev/urandom.\n");
+ }
+ bool Success = readRetry(Fd, Data, Size) == Size;
close(Fd);
if (!Success) {
dieWithMessage("ERROR: failed to read enough data from /dev/urandom.\n");
}
}
-} // namespace __scudo
+// Default constructor for Xorshift128Plus seeds the state with /dev/urandom.
+// TODO(kostyak): investigate using getrandom() if available.
+Xorshift128Plus::Xorshift128Plus() {
+ fillRandom(reinterpret_cast<u8 *>(State), sizeof(State));
+}
+
+const static u32 CRC32Table[] = {
+ 0x00000000, 0x77073096, 0xee0e612c, 0x990951ba, 0x076dc419, 0x706af48f,
+ 0xe963a535, 0x9e6495a3, 0x0edb8832, 0x79dcb8a4, 0xe0d5e91e, 0x97d2d988,
+ 0x09b64c2b, 0x7eb17cbd, 0xe7b82d07, 0x90bf1d91, 0x1db71064, 0x6ab020f2,
+ 0xf3b97148, 0x84be41de, 0x1adad47d, 0x6ddde4eb, 0xf4d4b551, 0x83d385c7,
+ 0x136c9856, 0x646ba8c0, 0xfd62f97a, 0x8a65c9ec, 0x14015c4f, 0x63066cd9,
+ 0xfa0f3d63, 0x8d080df5, 0x3b6e20c8, 0x4c69105e, 0xd56041e4, 0xa2677172,
+ 0x3c03e4d1, 0x4b04d447, 0xd20d85fd, 0xa50ab56b, 0x35b5a8fa, 0x42b2986c,
+ 0xdbbbc9d6, 0xacbcf940, 0x32d86ce3, 0x45df5c75, 0xdcd60dcf, 0xabd13d59,
+ 0x26d930ac, 0x51de003a, 0xc8d75180, 0xbfd06116, 0x21b4f4b5, 0x56b3c423,
+ 0xcfba9599, 0xb8bda50f, 0x2802b89e, 0x5f058808, 0xc60cd9b2, 0xb10be924,
+ 0x2f6f7c87, 0x58684c11, 0xc1611dab, 0xb6662d3d, 0x76dc4190, 0x01db7106,
+ 0x98d220bc, 0xefd5102a, 0x71b18589, 0x06b6b51f, 0x9fbfe4a5, 0xe8b8d433,
+ 0x7807c9a2, 0x0f00f934, 0x9609a88e, 0xe10e9818, 0x7f6a0dbb, 0x086d3d2d,
+ 0x91646c97, 0xe6635c01, 0x6b6b51f4, 0x1c6c6162, 0x856530d8, 0xf262004e,
+ 0x6c0695ed, 0x1b01a57b, 0x8208f4c1, 0xf50fc457, 0x65b0d9c6, 0x12b7e950,
+ 0x8bbeb8ea, 0xfcb9887c, 0x62dd1ddf, 0x15da2d49, 0x8cd37cf3, 0xfbd44c65,
+ 0x4db26158, 0x3ab551ce, 0xa3bc0074, 0xd4bb30e2, 0x4adfa541, 0x3dd895d7,
+ 0xa4d1c46d, 0xd3d6f4fb, 0x4369e96a, 0x346ed9fc, 0xad678846, 0xda60b8d0,
+ 0x44042d73, 0x33031de5, 0xaa0a4c5f, 0xdd0d7cc9, 0x5005713c, 0x270241aa,
+ 0xbe0b1010, 0xc90c2086, 0x5768b525, 0x206f85b3, 0xb966d409, 0xce61e49f,
+ 0x5edef90e, 0x29d9c998, 0xb0d09822, 0xc7d7a8b4, 0x59b33d17, 0x2eb40d81,
+ 0xb7bd5c3b, 0xc0ba6cad, 0xedb88320, 0x9abfb3b6, 0x03b6e20c, 0x74b1d29a,
+ 0xead54739, 0x9dd277af, 0x04db2615, 0x73dc1683, 0xe3630b12, 0x94643b84,
+ 0x0d6d6a3e, 0x7a6a5aa8, 0xe40ecf0b, 0x9309ff9d, 0x0a00ae27, 0x7d079eb1,
+ 0xf00f9344, 0x8708a3d2, 0x1e01f268, 0x6906c2fe, 0xf762575d, 0x806567cb,
+ 0x196c3671, 0x6e6b06e7, 0xfed41b76, 0x89d32be0, 0x10da7a5a, 0x67dd4acc,
+ 0xf9b9df6f, 0x8ebeeff9, 0x17b7be43, 0x60b08ed5, 0xd6d6a3e8, 0xa1d1937e,
+ 0x38d8c2c4, 0x4fdff252, 0xd1bb67f1, 0xa6bc5767, 0x3fb506dd, 0x48b2364b,
+ 0xd80d2bda, 0xaf0a1b4c, 0x36034af6, 0x41047a60, 0xdf60efc3, 0xa867df55,
+ 0x316e8eef, 0x4669be79, 0xcb61b38c, 0xbc66831a, 0x256fd2a0, 0x5268e236,
+ 0xcc0c7795, 0xbb0b4703, 0x220216b9, 0x5505262f, 0xc5ba3bbe, 0xb2bd0b28,
+ 0x2bb45a92, 0x5cb36a04, 0xc2d7ffa7, 0xb5d0cf31, 0x2cd99e8b, 0x5bdeae1d,
+ 0x9b64c2b0, 0xec63f226, 0x756aa39c, 0x026d930a, 0x9c0906a9, 0xeb0e363f,
+ 0x72076785, 0x05005713, 0x95bf4a82, 0xe2b87a14, 0x7bb12bae, 0x0cb61b38,
+ 0x92d28e9b, 0xe5d5be0d, 0x7cdcefb7, 0x0bdbdf21, 0x86d3d2d4, 0xf1d4e242,
+ 0x68ddb3f8, 0x1fda836e, 0x81be16cd, 0xf6b9265b, 0x6fb077e1, 0x18b74777,
+ 0x88085ae6, 0xff0f6a70, 0x66063bca, 0x11010b5c, 0x8f659eff, 0xf862ae69,
+ 0x616bffd3, 0x166ccf45, 0xa00ae278, 0xd70dd2ee, 0x4e048354, 0x3903b3c2,
+ 0xa7672661, 0xd06016f7, 0x4969474d, 0x3e6e77db, 0xaed16a4a, 0xd9d65adc,
+ 0x40df0b66, 0x37d83bf0, 0xa9bcae53, 0xdebb9ec5, 0x47b2cf7f, 0x30b5ffe9,
+ 0xbdbdf21c, 0xcabac28a, 0x53b39330, 0x24b4a3a6, 0xbad03605, 0xcdd70693,
+ 0x54de5729, 0x23d967bf, 0xb3667a2e, 0xc4614ab8, 0x5d681b02, 0x2a6f2b94,
+ 0xb40bbe37, 0xc30c8ea1, 0x5a05df1b, 0x2d02ef8d
+};
+
+u32 computeCRC32(u32 Crc, uptr Data)
+{
+ for (uptr i = 0; i < sizeof(Data); i++) {
+ Crc = CRC32Table[(Crc ^ Data) & 0xff] ^ (Crc >> 8);
+ Data >>= 8;
+ }
+ return Crc;
+}
+
+} // namespace __scudo
diff --git a/lib/scudo/scudo_utils.h b/lib/scudo/scudo_utils.h
index 07394ff4b91d..f93f26ef122a 100644
--- a/lib/scudo/scudo_utils.h
+++ b/lib/scudo/scudo_utils.h
@@ -28,11 +28,11 @@ inline Dest bit_cast(const Source& source) {
return dest;
}
-void dieWithMessage(const char *Format, ...);
+void NORETURN dieWithMessage(const char *Format, ...);
-enum CPUFeature {
- SSE4_2 = 0,
- ENUM_CPUFEATURE_MAX
+enum CPUFeature {
+ CRC32CPUFeature = 0,
+ MaxCPUFeature,
};
bool testCPUFeature(CPUFeature feature);
@@ -42,18 +42,20 @@ struct Xorshift128Plus {
public:
Xorshift128Plus();
u64 Next() {
- u64 x = State_0_;
- const u64 y = State_1_;
- State_0_ = y;
+ u64 x = State[0];
+ const u64 y = State[1];
+ State[0] = y;
x ^= x << 23;
- State_1_ = x ^ y ^ (x >> 17) ^ (y >> 26);
- return State_1_ + y;
+ State[1] = x ^ y ^ (x >> 17) ^ (y >> 26);
+ return State[1] + y;
}
private:
- u64 State_0_;
- u64 State_1_;
+ u64 State[2];
};
-} // namespace __scudo
+// Software CRC32 functions, to be used when SSE 4.2 support is not detected.
+u32 computeCRC32(u32 Crc, uptr Data);
+
+} // namespace __scudo
#endif // SCUDO_UTILS_H_
generated by cgit v1.3 (git 2.53.0) at 2026-08-10 06:38:03 +0000