diff options
Diffstat (limited to 'contrib/llvm-project/compiler-rt/lib/scudo/standalone/secondary.h')
-rw-r--r-- | contrib/llvm-project/compiler-rt/lib/scudo/standalone/secondary.h | 260 |
1 files changed, 260 insertions, 0 deletions
diff --git a/contrib/llvm-project/compiler-rt/lib/scudo/standalone/secondary.h b/contrib/llvm-project/compiler-rt/lib/scudo/standalone/secondary.h new file mode 100644 index 000000000000..ab68e5a1d38d --- /dev/null +++ b/contrib/llvm-project/compiler-rt/lib/scudo/standalone/secondary.h @@ -0,0 +1,260 @@ +//===-- secondary.h ---------------------------------------------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#ifndef SCUDO_SECONDARY_H_ +#define SCUDO_SECONDARY_H_ + +#include "common.h" +#include "list.h" +#include "mutex.h" +#include "stats.h" +#include "string_utils.h" + +namespace scudo { + +// This allocator wraps the platform allocation primitives, and as such is on +// the slower side and should preferably be used for larger sized allocations. +// Blocks allocated will be preceded and followed by a guard page, and hold +// their own header that is not checksummed: the guard pages and the Combined +// header should be enough for our purpose. + +namespace LargeBlock { + +struct Header { + LargeBlock::Header *Prev; + LargeBlock::Header *Next; + uptr BlockEnd; + uptr MapBase; + uptr MapSize; + MapPlatformData Data; +}; + +constexpr uptr getHeaderSize() { + return roundUpTo(sizeof(Header), 1U << SCUDO_MIN_ALIGNMENT_LOG); +} + +static Header *getHeader(uptr Ptr) { + return reinterpret_cast<Header *>(Ptr - getHeaderSize()); +} + +static Header *getHeader(const void *Ptr) { + return getHeader(reinterpret_cast<uptr>(Ptr)); +} + +} // namespace LargeBlock + +template <uptr MaxFreeListSize = 32U> class MapAllocator { +public: + // Ensure the freelist is disabled on Fuchsia, since it doesn't support + // releasing Secondary blocks yet. + static_assert(!SCUDO_FUCHSIA || MaxFreeListSize == 0U, ""); + + void initLinkerInitialized(GlobalStats *S) { + Stats.initLinkerInitialized(); + if (LIKELY(S)) + S->link(&Stats); + } + void init(GlobalStats *S) { + memset(this, 0, sizeof(*this)); + initLinkerInitialized(S); + } + + void *allocate(uptr Size, uptr AlignmentHint = 0, uptr *BlockEnd = nullptr, + bool ZeroContents = false); + + void deallocate(void *Ptr); + + static uptr getBlockEnd(void *Ptr) { + return LargeBlock::getHeader(Ptr)->BlockEnd; + } + + static uptr getBlockSize(void *Ptr) { + return getBlockEnd(Ptr) - reinterpret_cast<uptr>(Ptr); + } + + void getStats(ScopedString *Str) const; + + void disable() { Mutex.lock(); } + + void enable() { Mutex.unlock(); } + + template <typename F> void iterateOverBlocks(F Callback) const { + for (const auto &H : InUseBlocks) + Callback(reinterpret_cast<uptr>(&H) + LargeBlock::getHeaderSize()); + } + + static uptr getMaxFreeListSize(void) { return MaxFreeListSize; } + +private: + HybridMutex Mutex; + DoublyLinkedList<LargeBlock::Header> InUseBlocks; + // The free list is sorted based on the committed size of blocks. + DoublyLinkedList<LargeBlock::Header> FreeBlocks; + uptr AllocatedBytes; + uptr FreedBytes; + uptr LargestSize; + u32 NumberOfAllocs; + u32 NumberOfFrees; + LocalStats Stats; +}; + +// As with the Primary, the size passed to this function includes any desired +// alignment, so that the frontend can align the user allocation. The hint +// parameter allows us to unmap spurious memory when dealing with larger +// (greater than a page) alignments on 32-bit platforms. +// Due to the sparsity of address space available on those platforms, requesting +// an allocation from the Secondary with a large alignment would end up wasting +// VA space (even though we are not committing the whole thing), hence the need +// to trim off some of the reserved space. +// For allocations requested with an alignment greater than or equal to a page, +// the committed memory will amount to something close to Size - AlignmentHint +// (pending rounding and headers). +template <uptr MaxFreeListSize> +void *MapAllocator<MaxFreeListSize>::allocate(uptr Size, uptr AlignmentHint, + uptr *BlockEnd, + bool ZeroContents) { + DCHECK_GE(Size, AlignmentHint); + const uptr PageSize = getPageSizeCached(); + const uptr RoundedSize = + roundUpTo(Size + LargeBlock::getHeaderSize(), PageSize); + + if (MaxFreeListSize && AlignmentHint < PageSize) { + ScopedLock L(Mutex); + for (auto &H : FreeBlocks) { + const uptr FreeBlockSize = H.BlockEnd - reinterpret_cast<uptr>(&H); + if (FreeBlockSize < RoundedSize) + continue; + // Candidate free block should only be at most 4 pages larger. + if (FreeBlockSize > RoundedSize + 4 * PageSize) + break; + FreeBlocks.remove(&H); + InUseBlocks.push_back(&H); + AllocatedBytes += FreeBlockSize; + NumberOfAllocs++; + Stats.add(StatAllocated, FreeBlockSize); + if (BlockEnd) + *BlockEnd = H.BlockEnd; + void *Ptr = reinterpret_cast<void *>(reinterpret_cast<uptr>(&H) + + LargeBlock::getHeaderSize()); + if (ZeroContents) + memset(Ptr, 0, H.BlockEnd - reinterpret_cast<uptr>(Ptr)); + return Ptr; + } + } + + MapPlatformData Data = {}; + const uptr MapSize = RoundedSize + 2 * PageSize; + uptr MapBase = + reinterpret_cast<uptr>(map(nullptr, MapSize, "scudo:secondary", + MAP_NOACCESS | MAP_ALLOWNOMEM, &Data)); + if (UNLIKELY(!MapBase)) + return nullptr; + uptr CommitBase = MapBase + PageSize; + uptr MapEnd = MapBase + MapSize; + + // In the unlikely event of alignments larger than a page, adjust the amount + // of memory we want to commit, and trim the extra memory. + if (UNLIKELY(AlignmentHint >= PageSize)) { + // For alignments greater than or equal to a page, the user pointer (eg: the + // pointer that is returned by the C or C++ allocation APIs) ends up on a + // page boundary , and our headers will live in the preceding page. + CommitBase = roundUpTo(MapBase + PageSize + 1, AlignmentHint) - PageSize; + const uptr NewMapBase = CommitBase - PageSize; + DCHECK_GE(NewMapBase, MapBase); + // We only trim the extra memory on 32-bit platforms: 64-bit platforms + // are less constrained memory wise, and that saves us two syscalls. + if (SCUDO_WORDSIZE == 32U && NewMapBase != MapBase) { + unmap(reinterpret_cast<void *>(MapBase), NewMapBase - MapBase, 0, &Data); + MapBase = NewMapBase; + } + const uptr NewMapEnd = CommitBase + PageSize + + roundUpTo((Size - AlignmentHint), PageSize) + + PageSize; + DCHECK_LE(NewMapEnd, MapEnd); + if (SCUDO_WORDSIZE == 32U && NewMapEnd != MapEnd) { + unmap(reinterpret_cast<void *>(NewMapEnd), MapEnd - NewMapEnd, 0, &Data); + MapEnd = NewMapEnd; + } + } + + const uptr CommitSize = MapEnd - PageSize - CommitBase; + const uptr Ptr = + reinterpret_cast<uptr>(map(reinterpret_cast<void *>(CommitBase), + CommitSize, "scudo:secondary", 0, &Data)); + LargeBlock::Header *H = reinterpret_cast<LargeBlock::Header *>(Ptr); + H->MapBase = MapBase; + H->MapSize = MapEnd - MapBase; + H->BlockEnd = CommitBase + CommitSize; + H->Data = Data; + { + ScopedLock L(Mutex); + InUseBlocks.push_back(H); + AllocatedBytes += CommitSize; + if (LargestSize < CommitSize) + LargestSize = CommitSize; + NumberOfAllocs++; + Stats.add(StatAllocated, CommitSize); + Stats.add(StatMapped, H->MapSize); + } + if (BlockEnd) + *BlockEnd = CommitBase + CommitSize; + return reinterpret_cast<void *>(Ptr + LargeBlock::getHeaderSize()); +} + +template <uptr MaxFreeListSize> +void MapAllocator<MaxFreeListSize>::deallocate(void *Ptr) { + LargeBlock::Header *H = LargeBlock::getHeader(Ptr); + const uptr Block = reinterpret_cast<uptr>(H); + { + ScopedLock L(Mutex); + InUseBlocks.remove(H); + const uptr CommitSize = H->BlockEnd - Block; + FreedBytes += CommitSize; + NumberOfFrees++; + Stats.sub(StatAllocated, CommitSize); + if (MaxFreeListSize && FreeBlocks.size() < MaxFreeListSize) { + bool Inserted = false; + for (auto &F : FreeBlocks) { + const uptr FreeBlockSize = F.BlockEnd - reinterpret_cast<uptr>(&F); + if (FreeBlockSize >= CommitSize) { + FreeBlocks.insert(H, &F); + Inserted = true; + break; + } + } + if (!Inserted) + FreeBlocks.push_back(H); + const uptr RoundedAllocationStart = + roundUpTo(Block + LargeBlock::getHeaderSize(), getPageSizeCached()); + MapPlatformData Data = H->Data; + // TODO(kostyak): use release_to_os_interval_ms + releasePagesToOS(Block, RoundedAllocationStart - Block, + H->BlockEnd - RoundedAllocationStart, &Data); + return; + } + Stats.sub(StatMapped, H->MapSize); + } + void *Addr = reinterpret_cast<void *>(H->MapBase); + const uptr Size = H->MapSize; + MapPlatformData Data = H->Data; + unmap(Addr, Size, UNMAP_ALL, &Data); +} + +template <uptr MaxFreeListSize> +void MapAllocator<MaxFreeListSize>::getStats(ScopedString *Str) const { + Str->append( + "Stats: MapAllocator: allocated %zu times (%zuK), freed %zu times " + "(%zuK), remains %zu (%zuK) max %zuM\n", + NumberOfAllocs, AllocatedBytes >> 10, NumberOfFrees, FreedBytes >> 10, + NumberOfAllocs - NumberOfFrees, (AllocatedBytes - FreedBytes) >> 10, + LargestSize >> 20); +} + +} // namespace scudo + +#endif // SCUDO_SECONDARY_H_ |