aboutsummaryrefslogtreecommitdiff
path: root/lib/Transforms/Instrumentation
diff options
context:
space:
mode:
Diffstat (limited to 'lib/Transforms/Instrumentation')
-rw-r--r--lib/Transforms/Instrumentation/AddressSanitizer.cpp858
-rw-r--r--lib/Transforms/Instrumentation/CMakeLists.txt3
-rw-r--r--lib/Transforms/Instrumentation/DataFlowSanitizer.cpp204
-rw-r--r--lib/Transforms/Instrumentation/DebugIR.cpp617
-rw-r--r--lib/Transforms/Instrumentation/DebugIR.h98
-rw-r--r--lib/Transforms/Instrumentation/GCOVProfiling.cpp57
-rw-r--r--lib/Transforms/Instrumentation/InstrProfiling.cpp309
-rw-r--r--lib/Transforms/Instrumentation/Instrumentation.cpp2
-rw-r--r--lib/Transforms/Instrumentation/LLVMBuild.txt2
-rw-r--r--lib/Transforms/Instrumentation/MemorySanitizer.cpp419
-rw-r--r--lib/Transforms/Instrumentation/SanitizerCoverage.cpp314
-rw-r--r--lib/Transforms/Instrumentation/ThreadSanitizer.cpp44
12 files changed, 1606 insertions, 1321 deletions
diff --git a/lib/Transforms/Instrumentation/AddressSanitizer.cpp b/lib/Transforms/Instrumentation/AddressSanitizer.cpp
index 124ffe2f8f87..745c85a98e2f 100644
--- a/lib/Transforms/Instrumentation/AddressSanitizer.cpp
+++ b/lib/Transforms/Instrumentation/AddressSanitizer.cpp
@@ -27,6 +27,7 @@
#include "llvm/IR/CallSite.h"
#include "llvm/IR/DIBuilder.h"
#include "llvm/IR/DataLayout.h"
+#include "llvm/IR/Dominators.h"
#include "llvm/IR/Function.h"
#include "llvm/IR/IRBuilder.h"
#include "llvm/IR/InlineAsm.h"
@@ -36,10 +37,13 @@
#include "llvm/IR/MDBuilder.h"
#include "llvm/IR/Module.h"
#include "llvm/IR/Type.h"
+#include "llvm/MC/MCSectionMachO.h"
#include "llvm/Support/CommandLine.h"
#include "llvm/Support/DataTypes.h"
#include "llvm/Support/Debug.h"
#include "llvm/Support/Endian.h"
+#include "llvm/Support/SwapByteOrder.h"
+#include "llvm/Transforms/Scalar.h"
#include "llvm/Transforms/Utils/ASanStackFrameLayout.h"
#include "llvm/Transforms/Utils/BasicBlockUtils.h"
#include "llvm/Transforms/Utils/Cloning.h"
@@ -59,9 +63,11 @@ static const uint64_t kIOSShadowOffset32 = 1ULL << 30;
static const uint64_t kDefaultShadowOffset64 = 1ULL << 44;
static const uint64_t kSmallX86_64ShadowOffset = 0x7FFF8000; // < 2G.
static const uint64_t kPPC64_ShadowOffset64 = 1ULL << 41;
-static const uint64_t kMIPS32_ShadowOffset32 = 0x0aaa8000;
+static const uint64_t kMIPS32_ShadowOffset32 = 0x0aaa0000;
+static const uint64_t kMIPS64_ShadowOffset64 = 1ULL << 36;
static const uint64_t kFreeBSD_ShadowOffset32 = 1ULL << 30;
static const uint64_t kFreeBSD_ShadowOffset64 = 1ULL << 46;
+static const uint64_t kWindowsShadowOffset32 = 1ULL << 30;
static const size_t kMinStackMallocSize = 1 << 6; // 64B
static const size_t kMaxStackMallocSize = 1 << 16; // 64K
@@ -70,7 +76,7 @@ static const uintptr_t kRetiredStackFrameMagic = 0x45E0360E;
static const char *const kAsanModuleCtorName = "asan.module_ctor";
static const char *const kAsanModuleDtorName = "asan.module_dtor";
-static const int kAsanCtorAndDtorPriority = 1;
+static const uint64_t kAsanCtorAndDtorPriority = 1;
static const char *const kAsanReportErrorTemplate = "__asan_report_";
static const char *const kAsanReportLoadN = "__asan_report_load_n";
static const char *const kAsanReportStoreN = "__asan_report_store_n";
@@ -79,9 +85,7 @@ static const char *const kAsanUnregisterGlobalsName =
"__asan_unregister_globals";
static const char *const kAsanPoisonGlobalsName = "__asan_before_dynamic_init";
static const char *const kAsanUnpoisonGlobalsName = "__asan_after_dynamic_init";
-static const char *const kAsanInitName = "__asan_init_v4";
-static const char *const kAsanCovModuleInitName = "__sanitizer_cov_module_init";
-static const char *const kAsanCovName = "__sanitizer_cov";
+static const char *const kAsanInitName = "__asan_init_v5";
static const char *const kAsanPtrCmp = "__sanitizer_ptr_cmp";
static const char *const kAsanPtrSub = "__sanitizer_ptr_sub";
static const char *const kAsanHandleNoReturnName = "__asan_handle_no_return";
@@ -89,6 +93,7 @@ static const int kMaxAsanStackMallocSizeClass = 10;
static const char *const kAsanStackMallocNameTemplate = "__asan_stack_malloc_";
static const char *const kAsanStackFreeNameTemplate = "__asan_stack_free_";
static const char *const kAsanGenPrefix = "__asan_gen_";
+static const char *const kSanCovGenPrefix = "__sancov_gen_";
static const char *const kAsanPoisonStackMemoryName =
"__asan_poison_stack_memory";
static const char *const kAsanUnpoisonStackMemoryName =
@@ -104,6 +109,12 @@ static const int kAsanStackAfterReturnMagic = 0xf5;
// Accesses sizes are powers of two: 1, 2, 4, 8, 16.
static const size_t kNumberOfAccessSizes = 5;
+static const unsigned kAllocaRzSize = 32;
+static const unsigned kAsanAllocaLeftMagic = 0xcacacacaU;
+static const unsigned kAsanAllocaRightMagic = 0xcbcbcbcbU;
+static const unsigned kAsanAllocaPartialVal1 = 0xcbcbcb00U;
+static const unsigned kAsanAllocaPartialVal2 = 0x000000cbU;
+
// Command-line flags.
// This flag may need to be replaced with -f[no-]asan-reads.
@@ -133,13 +144,6 @@ static cl::opt<bool> ClUseAfterReturn("asan-use-after-return",
// This flag may need to be replaced with -f[no]asan-globals.
static cl::opt<bool> ClGlobals("asan-globals",
cl::desc("Handle global objects"), cl::Hidden, cl::init(true));
-static cl::opt<int> ClCoverage("asan-coverage",
- cl::desc("ASan coverage. 0: none, 1: entry block, 2: all blocks"),
- cl::Hidden, cl::init(false));
-static cl::opt<int> ClCoverageBlockThreshold("asan-coverage-block-threshold",
- cl::desc("Add coverage instrumentation only to the entry block if there "
- "are more than this number of blocks."),
- cl::Hidden, cl::init(1500));
static cl::opt<bool> ClInitializers("asan-initialization-order",
cl::desc("Handle C++ initializer order"), cl::Hidden, cl::init(true));
static cl::opt<bool> ClInvalidPointerPairs("asan-detect-invalid-pointer-pair",
@@ -158,19 +162,8 @@ static cl::opt<std::string> ClMemoryAccessCallbackPrefix(
"asan-memory-access-callback-prefix",
cl::desc("Prefix for memory access callbacks"), cl::Hidden,
cl::init("__asan_"));
-
-// This is an experimental feature that will allow to choose between
-// instrumented and non-instrumented code at link-time.
-// If this option is on, just before instrumenting a function we create its
-// clone; if the function is not changed by asan the clone is deleted.
-// If we end up with a clone, we put the instrumented function into a section
-// called "ASAN" and the uninstrumented function into a section called "NOASAN".
-//
-// This is still a prototype, we need to figure out a way to keep two copies of
-// a function so that the linker can easily choose one of them.
-static cl::opt<bool> ClKeepUninstrumented("asan-keep-uninstrumented-functions",
- cl::desc("Keep uninstrumented copies of functions"),
- cl::Hidden, cl::init(false));
+static cl::opt<bool> ClInstrumentAllocas("asan-instrument-allocas",
+ cl::desc("instrument dynamic allocas"), cl::Hidden, cl::init(false));
// These flags allow to change the shadow mapping.
// The shadow mapping looks like
@@ -192,6 +185,11 @@ static cl::opt<bool> ClCheckLifetime("asan-check-lifetime",
cl::desc("Use llvm.lifetime intrinsics to insert extra checks"),
cl::Hidden, cl::init(false));
+static cl::opt<bool> ClDynamicAllocaStack(
+ "asan-stack-dynamic-alloca",
+ cl::desc("Use dynamic alloca to represent stack variables"), cl::Hidden,
+ cl::init(false));
+
// Debug flags.
static cl::opt<int> ClDebug("asan-debug", cl::desc("debug"), cl::Hidden,
cl::init(0));
@@ -206,21 +204,44 @@ static cl::opt<int> ClDebugMax("asan-debug-max", cl::desc("Debug man inst"),
STATISTIC(NumInstrumentedReads, "Number of instrumented reads");
STATISTIC(NumInstrumentedWrites, "Number of instrumented writes");
+STATISTIC(NumInstrumentedDynamicAllocas,
+ "Number of instrumented dynamic allocas");
STATISTIC(NumOptimizedAccessesToGlobalArray,
"Number of optimized accesses to global arrays");
STATISTIC(NumOptimizedAccessesToGlobalVar,
"Number of optimized accesses to global vars");
namespace {
+/// Frontend-provided metadata for source location.
+struct LocationMetadata {
+ StringRef Filename;
+ int LineNo;
+ int ColumnNo;
+
+ LocationMetadata() : Filename(), LineNo(0), ColumnNo(0) {}
+
+ bool empty() const { return Filename.empty(); }
+
+ void parse(MDNode *MDN) {
+ assert(MDN->getNumOperands() == 3);
+ MDString *MDFilename = cast<MDString>(MDN->getOperand(0));
+ Filename = MDFilename->getString();
+ LineNo =
+ mdconst::extract<ConstantInt>(MDN->getOperand(1))->getLimitedValue();
+ ColumnNo =
+ mdconst::extract<ConstantInt>(MDN->getOperand(2))->getLimitedValue();
+ }
+};
+
/// Frontend-provided metadata for global variables.
class GlobalsMetadata {
public:
struct Entry {
Entry()
- : SourceLoc(nullptr), Name(nullptr), IsDynInit(false),
+ : SourceLoc(), Name(), IsDynInit(false),
IsBlacklisted(false) {}
- GlobalVariable *SourceLoc;
- GlobalVariable *Name;
+ LocationMetadata SourceLoc;
+ StringRef Name;
bool IsDynInit;
bool IsBlacklisted;
};
@@ -236,27 +257,22 @@ class GlobalsMetadata {
for (auto MDN : Globals->operands()) {
// Metadata node contains the global and the fields of "Entry".
assert(MDN->getNumOperands() == 5);
- Value *V = MDN->getOperand(0);
+ auto *GV = mdconst::extract_or_null<GlobalVariable>(MDN->getOperand(0));
// The optimizer may optimize away a global entirely.
- if (!V)
+ if (!GV)
continue;
- GlobalVariable *GV = cast<GlobalVariable>(V);
// We can already have an entry for GV if it was merged with another
// global.
Entry &E = Entries[GV];
- if (Value *Loc = MDN->getOperand(1)) {
- GlobalVariable *GVLoc = cast<GlobalVariable>(Loc);
- E.SourceLoc = GVLoc;
- addSourceLocationGlobal(GVLoc);
- }
- if (Value *Name = MDN->getOperand(2)) {
- GlobalVariable *GVName = cast<GlobalVariable>(Name);
- E.Name = GVName;
- InstrumentationGlobals.insert(GVName);
- }
- ConstantInt *IsDynInit = cast<ConstantInt>(MDN->getOperand(3));
+ if (auto *Loc = cast_or_null<MDNode>(MDN->getOperand(1)))
+ E.SourceLoc.parse(Loc);
+ if (auto *Name = cast_or_null<MDString>(MDN->getOperand(2)))
+ E.Name = Name->getString();
+ ConstantInt *IsDynInit =
+ mdconst::extract<ConstantInt>(MDN->getOperand(3));
E.IsDynInit |= IsDynInit->isOne();
- ConstantInt *IsBlacklisted = cast<ConstantInt>(MDN->getOperand(4));
+ ConstantInt *IsBlacklisted =
+ mdconst::extract<ConstantInt>(MDN->getOperand(4));
E.IsBlacklisted |= IsBlacklisted->isOne();
}
}
@@ -267,31 +283,9 @@ class GlobalsMetadata {
return (Pos != Entries.end()) ? Pos->second : Entry();
}
- /// Check if the global was generated by the instrumentation
- /// (we don't want to instrument it again in this case).
- bool isInstrumentationGlobal(GlobalVariable *G) const {
- return InstrumentationGlobals.count(G);
- }
-
private:
bool inited_;
DenseMap<GlobalVariable*, Entry> Entries;
- // Globals generated by the frontend instrumentation.
- DenseSet<GlobalVariable*> InstrumentationGlobals;
-
- void addSourceLocationGlobal(GlobalVariable *SourceLocGV) {
- // Source location global is a struct with layout:
- // {
- // filename,
- // i32 line_number,
- // i32 column_number,
- // }
- InstrumentationGlobals.insert(SourceLocGV);
- ConstantStruct *Contents =
- cast<ConstantStruct>(SourceLocGV->getInitializer());
- GlobalVariable *FilenameGV = cast<GlobalVariable>(Contents->getOperand(0));
- InstrumentationGlobals.insert(FilenameGV);
- }
};
/// This struct defines the shadow mapping using the rule:
@@ -302,17 +296,19 @@ struct ShadowMapping {
bool OrShadowOffset;
};
-static ShadowMapping getShadowMapping(const Module &M, int LongSize) {
- llvm::Triple TargetTriple(M.getTargetTriple());
+static ShadowMapping getShadowMapping(Triple &TargetTriple, int LongSize) {
bool IsAndroid = TargetTriple.getEnvironment() == llvm::Triple::Android;
- bool IsIOS = TargetTriple.getOS() == llvm::Triple::IOS;
- bool IsFreeBSD = TargetTriple.getOS() == llvm::Triple::FreeBSD;
- bool IsLinux = TargetTriple.getOS() == llvm::Triple::Linux;
+ bool IsIOS = TargetTriple.isiOS();
+ bool IsFreeBSD = TargetTriple.isOSFreeBSD();
+ bool IsLinux = TargetTriple.isOSLinux();
bool IsPPC64 = TargetTriple.getArch() == llvm::Triple::ppc64 ||
TargetTriple.getArch() == llvm::Triple::ppc64le;
bool IsX86_64 = TargetTriple.getArch() == llvm::Triple::x86_64;
bool IsMIPS32 = TargetTriple.getArch() == llvm::Triple::mips ||
TargetTriple.getArch() == llvm::Triple::mipsel;
+ bool IsMIPS64 = TargetTriple.getArch() == llvm::Triple::mips64 ||
+ TargetTriple.getArch() == llvm::Triple::mips64el;
+ bool IsWindows = TargetTriple.isOSWindows();
ShadowMapping Mapping;
@@ -325,6 +321,8 @@ static ShadowMapping getShadowMapping(const Module &M, int LongSize) {
Mapping.Offset = kFreeBSD_ShadowOffset32;
else if (IsIOS)
Mapping.Offset = kIOSShadowOffset32;
+ else if (IsWindows)
+ Mapping.Offset = kWindowsShadowOffset32;
else
Mapping.Offset = kDefaultShadowOffset32;
} else { // LongSize == 64
@@ -334,6 +332,8 @@ static ShadowMapping getShadowMapping(const Module &M, int LongSize) {
Mapping.Offset = kFreeBSD_ShadowOffset64;
else if (IsLinux && IsX86_64)
Mapping.Offset = kSmallX86_64ShadowOffset;
+ else if (IsMIPS64)
+ Mapping.Offset = kMIPS64_ShadowOffset64;
else
Mapping.Offset = kDefaultShadowOffset64;
}
@@ -359,10 +359,15 @@ static size_t RedzoneSizeForScale(int MappingScale) {
/// AddressSanitizer: instrument the code in module to find memory bugs.
struct AddressSanitizer : public FunctionPass {
- AddressSanitizer() : FunctionPass(ID) {}
+ AddressSanitizer() : FunctionPass(ID) {
+ initializeAddressSanitizerPass(*PassRegistry::getPassRegistry());
+ }
const char *getPassName() const override {
return "AddressSanitizerFunctionPass";
}
+ void getAnalysisUsage(AnalysisUsage &AU) const override {
+ AU.addRequired<DominatorTreeWrapperPass>();
+ }
void instrumentMop(Instruction *I, bool UseCalls);
void instrumentPointerComparisonOrSubtraction(Instruction *I);
void instrumentAddress(Instruction *OrigIns, Instruction *InsertBefore,
@@ -380,23 +385,24 @@ struct AddressSanitizer : public FunctionPass {
bool doInitialization(Module &M) override;
static char ID; // Pass identification, replacement for typeid
+ DominatorTree &getDominatorTree() const { return *DT; }
+
private:
void initializeCallbacks(Module &M);
bool LooksLikeCodeInBug11395(Instruction *I);
bool GlobalIsLinkerInitialized(GlobalVariable *G);
- bool InjectCoverage(Function &F, const ArrayRef<BasicBlock*> AllBlocks);
- void InjectCoverageAtBlock(Function &F, BasicBlock &BB);
LLVMContext *C;
const DataLayout *DL;
+ Triple TargetTriple;
int LongSize;
Type *IntptrTy;
ShadowMapping Mapping;
+ DominatorTree *DT;
Function *AsanCtorFunction;
Function *AsanInitFunction;
Function *AsanHandleNoReturnFunc;
- Function *AsanCovFunction;
Function *AsanPtrCmpFunction, *AsanPtrSubFunction;
// This array is indexed by AccessIsWrite and log2(AccessSize).
Function *AsanErrorCallback[2][kNumberOfAccessSizes];
@@ -435,12 +441,12 @@ class AddressSanitizerModule : public ModulePass {
Type *IntptrTy;
LLVMContext *C;
const DataLayout *DL;
+ Triple TargetTriple;
ShadowMapping Mapping;
Function *AsanPoisonGlobals;
Function *AsanUnpoisonGlobals;
Function *AsanRegisterGlobals;
Function *AsanUnregisterGlobals;
- Function *AsanCovModuleInit;
};
// Stack poisoning does not play well with exception handling.
@@ -478,15 +484,36 @@ struct FunctionStackPoisoner : public InstVisitor<FunctionStackPoisoner> {
};
SmallVector<AllocaPoisonCall, 8> AllocaPoisonCallVec;
+ // Stores left and right redzone shadow addresses for dynamic alloca
+ // and pointer to alloca instruction itself.
+ // LeftRzAddr is a shadow address for alloca left redzone.
+ // RightRzAddr is a shadow address for alloca right redzone.
+ struct DynamicAllocaCall {
+ AllocaInst *AI;
+ Value *LeftRzAddr;
+ Value *RightRzAddr;
+ bool Poison;
+ explicit DynamicAllocaCall(AllocaInst *AI,
+ Value *LeftRzAddr = nullptr,
+ Value *RightRzAddr = nullptr)
+ : AI(AI), LeftRzAddr(LeftRzAddr), RightRzAddr(RightRzAddr), Poison(true)
+ {}
+ };
+ SmallVector<DynamicAllocaCall, 1> DynamicAllocaVec;
+
// Maps Value to an AllocaInst from which the Value is originated.
typedef DenseMap<Value*, AllocaInst*> AllocaForValueMapTy;
AllocaForValueMapTy AllocaForValue;
+ bool HasNonEmptyInlineAsm;
+ std::unique_ptr<CallInst> EmptyInlineAsm;
+
FunctionStackPoisoner(Function &F, AddressSanitizer &ASan)
- : F(F), ASan(ASan), DIB(*F.getParent()), C(ASan.C),
- IntptrTy(ASan.IntptrTy), IntptrPtrTy(PointerType::get(IntptrTy, 0)),
- Mapping(ASan.Mapping),
- StackAlignment(1 << Mapping.Scale) {}
+ : F(F), ASan(ASan), DIB(*F.getParent(), /*AllowUnresolved*/ false),
+ C(ASan.C), IntptrTy(ASan.IntptrTy),
+ IntptrPtrTy(PointerType::get(IntptrTy, 0)), Mapping(ASan.Mapping),
+ StackAlignment(1 << Mapping.Scale), HasNonEmptyInlineAsm(false),
+ EmptyInlineAsm(CallInst::Create(ASan.EmptyAsm)) {}
bool runOnFunction() {
if (!ClStack) return false;
@@ -494,7 +521,7 @@ struct FunctionStackPoisoner : public InstVisitor<FunctionStackPoisoner> {
for (BasicBlock *BB : depth_first(&F.getEntryBlock()))
visit(*BB);
- if (AllocaVec.empty()) return false;
+ if (AllocaVec.empty() && DynamicAllocaVec.empty()) return false;
initializeCallbacks(*F.getParent());
@@ -506,7 +533,7 @@ struct FunctionStackPoisoner : public InstVisitor<FunctionStackPoisoner> {
return true;
}
- // Finds all static Alloca instructions and puts
+ // Finds all Alloca instructions and puts
// poisoned red zones around all of them.
// Then unpoison everything back before the function returns.
void poisonStack();
@@ -517,12 +544,64 @@ struct FunctionStackPoisoner : public InstVisitor<FunctionStackPoisoner> {
RetVec.push_back(&RI);
}
+ // Unpoison dynamic allocas redzones.
+ void unpoisonDynamicAlloca(DynamicAllocaCall &AllocaCall) {
+ if (!AllocaCall.Poison)
+ return;
+ for (auto Ret : RetVec) {
+ IRBuilder<> IRBRet(Ret);
+ PointerType *Int32PtrTy = PointerType::getUnqual(IRBRet.getInt32Ty());
+ Value *Zero = Constant::getNullValue(IRBRet.getInt32Ty());
+ Value *PartialRzAddr = IRBRet.CreateSub(AllocaCall.RightRzAddr,
+ ConstantInt::get(IntptrTy, 4));
+ IRBRet.CreateStore(Zero, IRBRet.CreateIntToPtr(AllocaCall.LeftRzAddr,
+ Int32PtrTy));
+ IRBRet.CreateStore(Zero, IRBRet.CreateIntToPtr(PartialRzAddr,
+ Int32PtrTy));
+ IRBRet.CreateStore(Zero, IRBRet.CreateIntToPtr(AllocaCall.RightRzAddr,
+ Int32PtrTy));
+ }
+ }
+
+ // Right shift for BigEndian and left shift for LittleEndian.
+ Value *shiftAllocaMagic(Value *Val, IRBuilder<> &IRB, Value *Shift) {
+ return ASan.DL->isLittleEndian() ? IRB.CreateShl(Val, Shift)
+ : IRB.CreateLShr(Val, Shift);
+ }
+
+ // Compute PartialRzMagic for dynamic alloca call. Since we don't know the
+ // size of requested memory until runtime, we should compute it dynamically.
+ // If PartialSize is 0, PartialRzMagic would contain kAsanAllocaRightMagic,
+ // otherwise it would contain the value that we will use to poison the
+ // partial redzone for alloca call.
+ Value *computePartialRzMagic(Value *PartialSize, IRBuilder<> &IRB);
+
+ // Deploy and poison redzones around dynamic alloca call. To do this, we
+ // should replace this call with another one with changed parameters and
+ // replace all its uses with new address, so
+ // addr = alloca type, old_size, align
+ // is replaced by
+ // new_size = (old_size + additional_size) * sizeof(type)
+ // tmp = alloca i8, new_size, max(align, 32)
+ // addr = tmp + 32 (first 32 bytes are for the left redzone).
+ // Additional_size is added to make new memory allocation contain not only
+ // requested memory, but also left, partial and right redzones.
+ // After that, we should poison redzones:
+ // (1) Left redzone with kAsanAllocaLeftMagic.
+ // (2) Partial redzone with the value, computed in runtime by
+ // computePartialRzMagic function.
+ // (3) Right redzone with kAsanAllocaRightMagic.
+ void handleDynamicAllocaCall(DynamicAllocaCall &AllocaCall);
+
/// \brief Collect Alloca instructions we want (and can) handle.
void visitAllocaInst(AllocaInst &AI) {
if (!isInterestingAlloca(AI)) return;
StackAlignment = std::max(StackAlignment, AI.getAlignment());
- AllocaVec.push_back(&AI);
+ if (isDynamicAlloca(AI))
+ DynamicAllocaVec.push_back(DynamicAllocaCall(&AI));
+ else
+ AllocaVec.push_back(&AI);
}
/// \brief Collect lifetime intrinsic calls to check for use-after-scope
@@ -551,13 +630,29 @@ struct FunctionStackPoisoner : public InstVisitor<FunctionStackPoisoner> {
AllocaPoisonCallVec.push_back(APC);
}
+ void visitCallInst(CallInst &CI) {
+ HasNonEmptyInlineAsm |=
+ CI.isInlineAsm() && !CI.isIdenticalTo(EmptyInlineAsm.get());
+ }
+
// ---------------------- Helpers.
void initializeCallbacks(Module &M);
+ bool doesDominateAllExits(const Instruction *I) const {
+ for (auto Ret : RetVec) {
+ if (!ASan.getDominatorTree().dominates(I, Ret))
+ return false;
+ }
+ return true;
+ }
+
+ bool isDynamicAlloca(AllocaInst &AI) const {
+ return AI.isArrayAllocation() || !AI.isStaticAlloca();
+ }
+
// Check if we want (and can) handle this alloca.
bool isInterestingAlloca(AllocaInst &AI) const {
- return (!AI.isArrayAllocation() && AI.isStaticAlloca() &&
- AI.getAllocatedType()->isSized() &&
+ return (AI.getAllocatedType()->isSized() &&
// alloca() may be called with 0 size, ignore it.
getAllocaSizeInBytes(&AI) > 0);
}
@@ -569,18 +664,26 @@ struct FunctionStackPoisoner : public InstVisitor<FunctionStackPoisoner> {
}
/// Finds alloca where the value comes from.
AllocaInst *findAllocaForValue(Value *V);
- void poisonRedZones(const ArrayRef<uint8_t> ShadowBytes, IRBuilder<> &IRB,
+ void poisonRedZones(ArrayRef<uint8_t> ShadowBytes, IRBuilder<> &IRB,
Value *ShadowBase, bool DoPoison);
void poisonAlloca(Value *V, uint64_t Size, IRBuilder<> &IRB, bool DoPoison);
void SetShadowToStackAfterReturnInlined(IRBuilder<> &IRB, Value *ShadowBase,
int Size);
+ Value *createAllocaForLayout(IRBuilder<> &IRB, const ASanStackFrameLayout &L,
+ bool Dynamic);
+ PHINode *createPHI(IRBuilder<> &IRB, Value *Cond, Value *ValueIfTrue,
+ Instruction *ThenTerm, Value *ValueIfFalse);
};
} // namespace
char AddressSanitizer::ID = 0;
-INITIALIZE_PASS(AddressSanitizer, "asan",
+INITIALIZE_PASS_BEGIN(AddressSanitizer, "asan",
+ "AddressSanitizer: detects use-after-free and out-of-bounds bugs.",
+ false, false)
+INITIALIZE_PASS_DEPENDENCY(DominatorTreeWrapperPass)
+INITIALIZE_PASS_END(AddressSanitizer, "asan",
"AddressSanitizer: detects use-after-free and out-of-bounds bugs.",
false, false)
FunctionPass *llvm::createAddressSanitizerFunctionPass() {
@@ -616,8 +719,25 @@ static GlobalVariable *createPrivateGlobalForString(
return GV;
}
+/// \brief Create a global describing a source location.
+static GlobalVariable *createPrivateGlobalForSourceLoc(Module &M,
+ LocationMetadata MD) {
+ Constant *LocData[] = {
+ createPrivateGlobalForString(M, MD.Filename, true),
+ ConstantInt::get(Type::getInt32Ty(M.getContext()), MD.LineNo),
+ ConstantInt::get(Type::getInt32Ty(M.getContext()), MD.ColumnNo),
+ };
+ auto LocStruct = ConstantStruct::getAnon(LocData);
+ auto GV = new GlobalVariable(M, LocStruct->getType(), true,
+ GlobalValue::PrivateLinkage, LocStruct,
+ kAsanGenPrefix);
+ GV->setUnnamedAddr(true);
+ return GV;
+}
+
static bool GlobalWasGeneratedByAsan(GlobalVariable *G) {
- return G->getName().find(kAsanGenPrefix) == 0;
+ return G->getName().find(kAsanGenPrefix) == 0 ||
+ G->getName().find(kSanCovGenPrefix) == 0;
}
Value *AddressSanitizer::memToShadow(Value *Shadow, IRBuilder<> &IRB) {
@@ -652,7 +772,7 @@ void AddressSanitizer::instrumentMemIntrinsic(MemIntrinsic *MI) {
}
// If I is an interesting memory access, return the PointerOperand
-// and set IsWrite/Alignment. Otherwise return NULL.
+// and set IsWrite/Alignment. Otherwise return nullptr.
static Value *isInterestingMemoryAccess(Instruction *I, bool *IsWrite,
unsigned *Alignment) {
// Skip memory accesses inserted by another instrumentation.
@@ -861,8 +981,11 @@ void AddressSanitizer::instrumentAddress(Instruction *OrigIns,
TerminatorInst *CrashTerm = nullptr;
if (ClAlwaysSlowPath || (TypeSize < 8 * Granularity)) {
+ // We use branch weights for the slow path check, to indicate that the slow
+ // path is rarely taken. This seems to be the case for SPEC benchmarks.
TerminatorInst *CheckTerm =
- SplitBlockAndInsertIfThen(Cmp, InsertBefore, false);
+ SplitBlockAndInsertIfThen(Cmp, InsertBefore, false,
+ MDBuilder(*C).createBranchWeights(1, 100000));
assert(dyn_cast<BranchInst>(CheckTerm)->isUnconditional());
BasicBlock *NextBB = CheckTerm->getSuccessor(0);
IRB.SetInsertPoint(CheckTerm);
@@ -907,10 +1030,12 @@ void AddressSanitizerModule::createInitializerPoisonCalls(
ConstantStruct *CS = cast<ConstantStruct>(OP);
// Must have a function or null ptr.
- // (CS->getOperand(0) is the init priority.)
if (Function* F = dyn_cast<Function>(CS->getOperand(1))) {
- if (F->getName() != kAsanModuleCtorName)
- poisonOneInitializer(*F, ModuleName);
+ if (F->getName() == kAsanModuleCtorName) continue;
+ ConstantInt *Priority = dyn_cast<ConstantInt>(CS->getOperand(0));
+ // Don't instrument CTORs that will run before asan.module_ctor.
+ if (Priority->getLimitedValue() <= kAsanCtorAndDtorPriority) continue;
+ poisonOneInitializer(*F, ModuleName);
}
}
}
@@ -920,7 +1045,6 @@ bool AddressSanitizerModule::ShouldInstrumentGlobal(GlobalVariable *G) {
DEBUG(dbgs() << "GLOBAL: " << *G << "\n");
if (GlobalsMD.get(G).IsBlacklisted) return false;
- if (GlobalsMD.isInstrumentationGlobal(G)) return false;
if (!Ty->isSized()) return false;
if (!G->hasInitializer()) return false;
if (GlobalWasGeneratedByAsan(G)) return false; // Our own global.
@@ -941,43 +1065,48 @@ bool AddressSanitizerModule::ShouldInstrumentGlobal(GlobalVariable *G) {
// For now, just ignore this Global if the alignment is large.
if (G->getAlignment() > MinRedzoneSizeForGlobal()) return false;
- // Ignore all the globals with the names starting with "\01L_OBJC_".
- // Many of those are put into the .cstring section. The linker compresses
- // that section by removing the spare \0s after the string terminator, so
- // our redzones get broken.
- if ((G->getName().find("\01L_OBJC_") == 0) ||
- (G->getName().find("\01l_OBJC_") == 0)) {
- DEBUG(dbgs() << "Ignoring \\01L_OBJC_* global: " << *G << "\n");
- return false;
- }
-
if (G->hasSection()) {
StringRef Section(G->getSection());
- // Ignore the globals from the __OBJC section. The ObjC runtime assumes
- // those conform to /usr/lib/objc/runtime.h, so we can't add redzones to
- // them.
- if (Section.startswith("__OBJC,") ||
- Section.startswith("__DATA, __objc_")) {
- DEBUG(dbgs() << "Ignoring ObjC runtime global: " << *G << "\n");
- return false;
- }
- // See http://code.google.com/p/address-sanitizer/issues/detail?id=32
- // Constant CFString instances are compiled in the following way:
- // -- the string buffer is emitted into
- // __TEXT,__cstring,cstring_literals
- // -- the constant NSConstantString structure referencing that buffer
- // is placed into __DATA,__cfstring
- // Therefore there's no point in placing redzones into __DATA,__cfstring.
- // Moreover, it causes the linker to crash on OS X 10.7
- if (Section.startswith("__DATA,__cfstring")) {
- DEBUG(dbgs() << "Ignoring CFString: " << *G << "\n");
- return false;
- }
- // The linker merges the contents of cstring_literals and removes the
- // trailing zeroes.
- if (Section.startswith("__TEXT,__cstring,cstring_literals")) {
- DEBUG(dbgs() << "Ignoring a cstring literal: " << *G << "\n");
- return false;
+
+ if (TargetTriple.isOSBinFormatMachO()) {
+ StringRef ParsedSegment, ParsedSection;
+ unsigned TAA = 0, StubSize = 0;
+ bool TAAParsed;
+ std::string ErrorCode =
+ MCSectionMachO::ParseSectionSpecifier(Section, ParsedSegment,
+ ParsedSection, TAA, TAAParsed,
+ StubSize);
+ if (!ErrorCode.empty()) {
+ report_fatal_error("Invalid section specifier '" + ParsedSection +
+ "': " + ErrorCode + ".");
+ }
+
+ // Ignore the globals from the __OBJC section. The ObjC runtime assumes
+ // those conform to /usr/lib/objc/runtime.h, so we can't add redzones to
+ // them.
+ if (ParsedSegment == "__OBJC" ||
+ (ParsedSegment == "__DATA" && ParsedSection.startswith("__objc_"))) {
+ DEBUG(dbgs() << "Ignoring ObjC runtime global: " << *G << "\n");
+ return false;
+ }
+ // See http://code.google.com/p/address-sanitizer/issues/detail?id=32
+ // Constant CFString instances are compiled in the following way:
+ // -- the string buffer is emitted into
+ // __TEXT,__cstring,cstring_literals
+ // -- the constant NSConstantString structure referencing that buffer
+ // is placed into __DATA,__cfstring
+ // Therefore there's no point in placing redzones into __DATA,__cfstring.
+ // Moreover, it causes the linker to crash on OS X 10.7
+ if (ParsedSegment == "__DATA" && ParsedSection == "__cfstring") {
+ DEBUG(dbgs() << "Ignoring CFString: " << *G << "\n");
+ return false;
+ }
+ // The linker merges the contents of cstring_literals and removes the
+ // trailing zeroes.
+ if (ParsedSegment == "__TEXT" && (TAA & MachO::S_CSTRING_LITERALS)) {
+ DEBUG(dbgs() << "Ignoring a cstring literal: " << *G << "\n");
+ return false;
+ }
}
// Callbacks put into the CRT initializer/terminator sections
@@ -1000,24 +1129,20 @@ void AddressSanitizerModule::initializeCallbacks(Module &M) {
IRBuilder<> IRB(*C);
// Declare our poisoning and unpoisoning functions.
AsanPoisonGlobals = checkInterfaceFunction(M.getOrInsertFunction(
- kAsanPoisonGlobalsName, IRB.getVoidTy(), IntptrTy, NULL));
+ kAsanPoisonGlobalsName, IRB.getVoidTy(), IntptrTy, nullptr));
AsanPoisonGlobals->setLinkage(Function::ExternalLinkage);
AsanUnpoisonGlobals = checkInterfaceFunction(M.getOrInsertFunction(
- kAsanUnpoisonGlobalsName, IRB.getVoidTy(), NULL));
+ kAsanUnpoisonGlobalsName, IRB.getVoidTy(), nullptr));
AsanUnpoisonGlobals->setLinkage(Function::ExternalLinkage);
// Declare functions that register/unregister globals.
AsanRegisterGlobals = checkInterfaceFunction(M.getOrInsertFunction(
kAsanRegisterGlobalsName, IRB.getVoidTy(),
- IntptrTy, IntptrTy, NULL));
+ IntptrTy, IntptrTy, nullptr));
AsanRegisterGlobals->setLinkage(Function::ExternalLinkage);
AsanUnregisterGlobals = checkInterfaceFunction(M.getOrInsertFunction(
kAsanUnregisterGlobalsName,
- IRB.getVoidTy(), IntptrTy, IntptrTy, NULL));
+ IRB.getVoidTy(), IntptrTy, IntptrTy, nullptr));
AsanUnregisterGlobals->setLinkage(Function::ExternalLinkage);
- AsanCovModuleInit = checkInterfaceFunction(M.getOrInsertFunction(
- kAsanCovModuleInitName,
- IRB.getVoidTy(), IntptrTy, NULL));
- AsanCovModuleInit->setLinkage(Function::ExternalLinkage);
}
// This function replaces all global variables with new variables that have
@@ -1047,7 +1172,7 @@ bool AddressSanitizerModule::InstrumentGlobals(IRBuilder<> &IRB, Module &M) {
// We initialize an array of such structures and pass it to a run-time call.
StructType *GlobalStructTy =
StructType::get(IntptrTy, IntptrTy, IntptrTy, IntptrTy, IntptrTy,
- IntptrTy, IntptrTy, NULL);
+ IntptrTy, IntptrTy, nullptr);
SmallVector<Constant *, 16> Initializers(n);
bool HasDynamicallyInitializedGlobals = false;
@@ -1062,11 +1187,11 @@ bool AddressSanitizerModule::InstrumentGlobals(IRBuilder<> &IRB, Module &M) {
GlobalVariable *G = GlobalsToChange[i];
auto MD = GlobalsMD.get(G);
- // Create string holding the global name unless it was provided by
- // the metadata.
- GlobalVariable *Name =
- MD.Name ? MD.Name : createPrivateGlobalForString(M, G->getName(),
- /*AllowMerging*/ true);
+ // Create string holding the global name (use global name from metadata
+ // if it's available, otherwise just write the name of global variable).
+ GlobalVariable *Name = createPrivateGlobalForString(
+ M, MD.Name.empty() ? G->getName() : MD.Name,
+ /*AllowMerging*/ true);
PointerType *PtrTy = cast<PointerType>(G->getType());
Type *Ty = PtrTy->getElementType();
@@ -1084,10 +1209,10 @@ bool AddressSanitizerModule::InstrumentGlobals(IRBuilder<> &IRB, Module &M) {
assert(((RightRedzoneSize + SizeInBytes) % MinRZ) == 0);
Type *RightRedZoneTy = ArrayType::get(IRB.getInt8Ty(), RightRedzoneSize);
- StructType *NewTy = StructType::get(Ty, RightRedZoneTy, NULL);
+ StructType *NewTy = StructType::get(Ty, RightRedZoneTy, nullptr);
Constant *NewInitializer = ConstantStruct::get(
NewTy, G->getInitializer(),
- Constant::getNullValue(RightRedZoneTy), NULL);
+ Constant::getNullValue(RightRedZoneTy), nullptr);
// Create a new global variable with enough space for a redzone.
GlobalValue::LinkageTypes Linkage = G->getLinkage();
@@ -1108,16 +1233,21 @@ bool AddressSanitizerModule::InstrumentGlobals(IRBuilder<> &IRB, Module &M) {
NewGlobal->takeName(G);
G->eraseFromParent();
+ Constant *SourceLoc;
+ if (!MD.SourceLoc.empty()) {
+ auto SourceLocGlobal = createPrivateGlobalForSourceLoc(M, MD.SourceLoc);
+ SourceLoc = ConstantExpr::getPointerCast(SourceLocGlobal, IntptrTy);
+ } else {
+ SourceLoc = ConstantInt::get(IntptrTy, 0);
+ }
+
Initializers[i] = ConstantStruct::get(
GlobalStructTy, ConstantExpr::getPointerCast(NewGlobal, IntptrTy),
ConstantInt::get(IntptrTy, SizeInBytes),
ConstantInt::get(IntptrTy, SizeInBytes + RightRedzoneSize),
ConstantExpr::getPointerCast(Name, IntptrTy),
ConstantExpr::getPointerCast(ModuleName, IntptrTy),
- ConstantInt::get(IntptrTy, MD.IsDynInit),
- MD.SourceLoc ? ConstantExpr::getPointerCast(MD.SourceLoc, IntptrTy)
- : ConstantInt::get(IntptrTy, 0),
- NULL);
+ ConstantInt::get(IntptrTy, MD.IsDynInit), SourceLoc, nullptr);
if (ClInitializers && MD.IsDynInit)
HasDynamicallyInitializedGlobals = true;
@@ -1161,7 +1291,8 @@ bool AddressSanitizerModule::runOnModule(Module &M) {
C = &(M.getContext());
int LongSize = DL->getPointerSizeInBits();
IntptrTy = Type::getIntNTy(*C, LongSize);
- Mapping = getShadowMapping(M, LongSize);
+ TargetTriple = Triple(M.getTargetTriple());
+ Mapping = getShadowMapping(TargetTriple, LongSize);
initializeCallbacks(M);
bool Changed = false;
@@ -1170,13 +1301,6 @@ bool AddressSanitizerModule::runOnModule(Module &M) {
assert(CtorFunc);
IRBuilder<> IRB(CtorFunc->getEntryBlock().getTerminator());
- if (ClCoverage > 0) {
- Function *CovFunc = M.getFunction(kAsanCovName);
- int nCov = CovFunc ? CovFunc->getNumUses() : 0;
- IRB.CreateCall(AsanCovModuleInit, ConstantInt::get(IntptrTy, nCov));
- Changed = true;
- }
-
if (ClGlobals)
Changed |= InstrumentGlobals(IRB, M);
@@ -1195,43 +1319,42 @@ void AddressSanitizer::initializeCallbacks(Module &M) {
AsanErrorCallback[AccessIsWrite][AccessSizeIndex] =
checkInterfaceFunction(
M.getOrInsertFunction(kAsanReportErrorTemplate + Suffix,
- IRB.getVoidTy(), IntptrTy, NULL));
+ IRB.getVoidTy(), IntptrTy, nullptr));
AsanMemoryAccessCallback[AccessIsWrite][AccessSizeIndex] =
checkInterfaceFunction(
M.getOrInsertFunction(ClMemoryAccessCallbackPrefix + Suffix,
- IRB.getVoidTy(), IntptrTy, NULL));
+ IRB.getVoidTy(), IntptrTy, nullptr));
}
}
AsanErrorCallbackSized[0] = checkInterfaceFunction(M.getOrInsertFunction(
- kAsanReportLoadN, IRB.getVoidTy(), IntptrTy, IntptrTy, NULL));
+ kAsanReportLoadN, IRB.getVoidTy(), IntptrTy, IntptrTy, nullptr));
AsanErrorCallbackSized[1] = checkInterfaceFunction(M.getOrInsertFunction(
- kAsanReportStoreN, IRB.getVoidTy(), IntptrTy, IntptrTy, NULL));
+ kAsanReportStoreN, IRB.getVoidTy(), IntptrTy, IntptrTy, nullptr));
AsanMemoryAccessCallbackSized[0] = checkInterfaceFunction(
M.getOrInsertFunction(ClMemoryAccessCallbackPrefix + "loadN",
- IRB.getVoidTy(), IntptrTy, IntptrTy, NULL));
+ IRB.getVoidTy(), IntptrTy, IntptrTy, nullptr));
AsanMemoryAccessCallbackSized[1] = checkInterfaceFunction(
M.getOrInsertFunction(ClMemoryAccessCallbackPrefix + "storeN",
- IRB.getVoidTy(), IntptrTy, IntptrTy, NULL));
+ IRB.getVoidTy(), IntptrTy, IntptrTy, nullptr));
AsanMemmove = checkInterfaceFunction(M.getOrInsertFunction(
ClMemoryAccessCallbackPrefix + "memmove", IRB.getInt8PtrTy(),
- IRB.getInt8PtrTy(), IRB.getInt8PtrTy(), IntptrTy, NULL));
+ IRB.getInt8PtrTy(), IRB.getInt8PtrTy(), IntptrTy, nullptr));
AsanMemcpy = checkInterfaceFunction(M.getOrInsertFunction(
ClMemoryAccessCallbackPrefix + "memcpy", IRB.getInt8PtrTy(),
- IRB.getInt8PtrTy(), IRB.getInt8PtrTy(), IntptrTy, NULL));
+ IRB.getInt8PtrTy(), IRB.getInt8PtrTy(), IntptrTy, nullptr));
AsanMemset = checkInterfaceFunction(M.getOrInsertFunction(
ClMemoryAccessCallbackPrefix + "memset", IRB.getInt8PtrTy(),
- IRB.getInt8PtrTy(), IRB.getInt32Ty(), IntptrTy, NULL));
+ IRB.getInt8PtrTy(), IRB.getInt32Ty(), IntptrTy, nullptr));
AsanHandleNoReturnFunc = checkInterfaceFunction(
- M.getOrInsertFunction(kAsanHandleNoReturnName, IRB.getVoidTy(), NULL));
- AsanCovFunction = checkInterfaceFunction(M.getOrInsertFunction(
- kAsanCovName, IRB.getVoidTy(), NULL));
+ M.getOrInsertFunction(kAsanHandleNoReturnName, IRB.getVoidTy(), nullptr));
+
AsanPtrCmpFunction = checkInterfaceFunction(M.getOrInsertFunction(
- kAsanPtrCmp, IRB.getVoidTy(), IntptrTy, IntptrTy, NULL));
+ kAsanPtrCmp, IRB.getVoidTy(), IntptrTy, IntptrTy, nullptr));
AsanPtrSubFunction = checkInterfaceFunction(M.getOrInsertFunction(
- kAsanPtrSub, IRB.getVoidTy(), IntptrTy, IntptrTy, NULL));
+ kAsanPtrSub, IRB.getVoidTy(), IntptrTy, IntptrTy, nullptr));
// We insert an empty inline asm after __asan_report* to avoid callback merge.
EmptyAsm = InlineAsm::get(FunctionType::get(IRB.getVoidTy(), false),
StringRef(""), StringRef(""),
@@ -1251,6 +1374,7 @@ bool AddressSanitizer::doInitialization(Module &M) {
C = &(M.getContext());
LongSize = DL->getPointerSizeInBits();
IntptrTy = Type::getIntNTy(*C, LongSize);
+ TargetTriple = Triple(M.getTargetTriple());
AsanCtorFunction = Function::Create(
FunctionType::get(Type::getVoidTy(*C), false),
@@ -1259,11 +1383,11 @@ bool AddressSanitizer::doInitialization(Module &M) {
// call __asan_init in the module ctor.
IRBuilder<> IRB(ReturnInst::Create(*C, AsanCtorBB));
AsanInitFunction = checkInterfaceFunction(
- M.getOrInsertFunction(kAsanInitName, IRB.getVoidTy(), NULL));
+ M.getOrInsertFunction(kAsanInitName, IRB.getVoidTy(), nullptr));
AsanInitFunction->setLinkage(Function::ExternalLinkage);
IRB.CreateCall(AsanInitFunction);
- Mapping = getShadowMapping(M, LongSize);
+ Mapping = getShadowMapping(TargetTriple, LongSize);
appendToGlobalCtors(M, AsanCtorFunction, kAsanCtorAndDtorPriority);
return true;
@@ -1285,80 +1409,14 @@ bool AddressSanitizer::maybeInsertAsanInitAtFunctionEntry(Function &F) {
return false;
}
-void AddressSanitizer::InjectCoverageAtBlock(Function &F, BasicBlock &BB) {
- BasicBlock::iterator IP = BB.getFirstInsertionPt(), BE = BB.end();
- // Skip static allocas at the top of the entry block so they don't become
- // dynamic when we split the block. If we used our optimized stack layout,
- // then there will only be one alloca and it will come first.
- for (; IP != BE; ++IP) {
- AllocaInst *AI = dyn_cast<AllocaInst>(IP);
- if (!AI || !AI->isStaticAlloca())
- break;
- }
-
- DebugLoc EntryLoc = IP->getDebugLoc().getFnDebugLoc(*C);
- IRBuilder<> IRB(IP);
- IRB.SetCurrentDebugLocation(EntryLoc);
- Type *Int8Ty = IRB.getInt8Ty();
- GlobalVariable *Guard = new GlobalVariable(
- *F.getParent(), Int8Ty, false, GlobalValue::PrivateLinkage,
- Constant::getNullValue(Int8Ty), "__asan_gen_cov_" + F.getName());
- LoadInst *Load = IRB.CreateLoad(Guard);
- Load->setAtomic(Monotonic);
- Load->setAlignment(1);
- Value *Cmp = IRB.CreateICmpEQ(Constant::getNullValue(Int8Ty), Load);
- Instruction *Ins = SplitBlockAndInsertIfThen(
- Cmp, IP, false, MDBuilder(*C).createBranchWeights(1, 100000));
- IRB.SetInsertPoint(Ins);
- IRB.SetCurrentDebugLocation(EntryLoc);
- // We pass &F to __sanitizer_cov. We could avoid this and rely on
- // GET_CALLER_PC, but having the PC of the first instruction is just nice.
- IRB.CreateCall(AsanCovFunction);
- StoreInst *Store = IRB.CreateStore(ConstantInt::get(Int8Ty, 1), Guard);
- Store->setAtomic(Monotonic);
- Store->setAlignment(1);
-}
-
-// Poor man's coverage that works with ASan.
-// We create a Guard boolean variable with the same linkage
-// as the function and inject this code into the entry block (-asan-coverage=1)
-// or all blocks (-asan-coverage=2):
-// if (*Guard) {
-// __sanitizer_cov(&F);
-// *Guard = 1;
-// }
-// The accesses to Guard are atomic. The rest of the logic is
-// in __sanitizer_cov (it's fine to call it more than once).
-//
-// This coverage implementation provides very limited data:
-// it only tells if a given function (block) was ever executed.
-// No counters, no per-edge data.
-// But for many use cases this is what we need and the added slowdown
-// is negligible. This simple implementation will probably be obsoleted
-// by the upcoming Clang-based coverage implementation.
-// By having it here and now we hope to
-// a) get the functionality to users earlier and
-// b) collect usage statistics to help improve Clang coverage design.
-bool AddressSanitizer::InjectCoverage(Function &F,
- const ArrayRef<BasicBlock *> AllBlocks) {
- if (!ClCoverage) return false;
-
- if (ClCoverage == 1 ||
- (unsigned)ClCoverageBlockThreshold < AllBlocks.size()) {
- InjectCoverageAtBlock(F, F.getEntryBlock());
- } else {
- for (auto BB : AllBlocks)
- InjectCoverageAtBlock(F, *BB);
- }
- return true;
-}
-
bool AddressSanitizer::runOnFunction(Function &F) {
if (&F == AsanCtorFunction) return false;
if (F.getLinkage() == GlobalValue::AvailableExternallyLinkage) return false;
DEBUG(dbgs() << "ASAN instrumenting:\n" << F << "\n");
initializeCallbacks(*F.getParent());
+ DT = &getAnalysis<DominatorTreeWrapperPass>().getDomTree();
+
// If needed, insert __asan_init before checking for SanitizeAddress attr.
maybeInsertAsanInitAtFunctionEntry(F);
@@ -1389,7 +1447,7 @@ bool AddressSanitizer::runOnFunction(Function &F) {
if (Value *Addr =
isInterestingMemoryAccess(&Inst, &IsWrite, &Alignment)) {
if (ClOpt && ClOptSameTemp) {
- if (!TempsToInstrument.insert(Addr))
+ if (!TempsToInstrument.insert(Addr).second)
continue; // We've seen this temp in the current BB.
}
} else if (ClInvalidPointerPairs &&
@@ -1417,17 +1475,6 @@ bool AddressSanitizer::runOnFunction(Function &F) {
}
}
- Function *UninstrumentedDuplicate = nullptr;
- bool LikelyToInstrument =
- !NoReturnCalls.empty() || !ToInstrument.empty() || (NumAllocas > 0);
- if (ClKeepUninstrumented && LikelyToInstrument) {
- ValueToValueMapTy VMap;
- UninstrumentedDuplicate = CloneFunction(&F, VMap, false);
- UninstrumentedDuplicate->removeFnAttr(Attribute::SanitizeAddress);
- UninstrumentedDuplicate->setName("NOASAN_" + F.getName());
- F.getParent()->getFunctionList().push_back(UninstrumentedDuplicate);
- }
-
bool UseCalls = false;
if (ClInstrumentationWithCallsThreshold >= 0 &&
ToInstrument.size() > (unsigned)ClInstrumentationWithCallsThreshold)
@@ -1463,25 +1510,8 @@ bool AddressSanitizer::runOnFunction(Function &F) {
bool res = NumInstrumented > 0 || ChangedStack || !NoReturnCalls.empty();
- if (InjectCoverage(F, AllBlocks))
- res = true;
-
DEBUG(dbgs() << "ASAN done instrumenting: " << res << " " << F << "\n");
- if (ClKeepUninstrumented) {
- if (!res) {
- // No instrumentation is done, no need for the duplicate.
- if (UninstrumentedDuplicate)
- UninstrumentedDuplicate->eraseFromParent();
- } else {
- // The function was instrumented. We must have the duplicate.
- assert(UninstrumentedDuplicate);
- UninstrumentedDuplicate->setSection("NOASAN");
- assert(!F.hasSection());
- F.setSection("ASAN");
- }
- }
-
return res;
}
@@ -1501,21 +1531,22 @@ void FunctionStackPoisoner::initializeCallbacks(Module &M) {
IRBuilder<> IRB(*C);
for (int i = 0; i <= kMaxAsanStackMallocSizeClass; i++) {
std::string Suffix = itostr(i);
- AsanStackMallocFunc[i] = checkInterfaceFunction(
- M.getOrInsertFunction(kAsanStackMallocNameTemplate + Suffix, IntptrTy,
- IntptrTy, IntptrTy, NULL));
- AsanStackFreeFunc[i] = checkInterfaceFunction(M.getOrInsertFunction(
- kAsanStackFreeNameTemplate + Suffix, IRB.getVoidTy(), IntptrTy,
- IntptrTy, IntptrTy, NULL));
+ AsanStackMallocFunc[i] = checkInterfaceFunction(M.getOrInsertFunction(
+ kAsanStackMallocNameTemplate + Suffix, IntptrTy, IntptrTy, nullptr));
+ AsanStackFreeFunc[i] = checkInterfaceFunction(
+ M.getOrInsertFunction(kAsanStackFreeNameTemplate + Suffix,
+ IRB.getVoidTy(), IntptrTy, IntptrTy, nullptr));
}
- AsanPoisonStackMemoryFunc = checkInterfaceFunction(M.getOrInsertFunction(
- kAsanPoisonStackMemoryName, IRB.getVoidTy(), IntptrTy, IntptrTy, NULL));
- AsanUnpoisonStackMemoryFunc = checkInterfaceFunction(M.getOrInsertFunction(
- kAsanUnpoisonStackMemoryName, IRB.getVoidTy(), IntptrTy, IntptrTy, NULL));
+ AsanPoisonStackMemoryFunc = checkInterfaceFunction(
+ M.getOrInsertFunction(kAsanPoisonStackMemoryName, IRB.getVoidTy(),
+ IntptrTy, IntptrTy, nullptr));
+ AsanUnpoisonStackMemoryFunc = checkInterfaceFunction(
+ M.getOrInsertFunction(kAsanUnpoisonStackMemoryName, IRB.getVoidTy(),
+ IntptrTy, IntptrTy, nullptr));
}
void
-FunctionStackPoisoner::poisonRedZones(const ArrayRef<uint8_t> ShadowBytes,
+FunctionStackPoisoner::poisonRedZones(ArrayRef<uint8_t> ShadowBytes,
IRBuilder<> &IRB, Value *ShadowBase,
bool DoPoison) {
size_t n = ShadowBytes.size();
@@ -1576,11 +1607,49 @@ static DebugLoc getFunctionEntryDebugLocation(Function &F) {
return DebugLoc();
}
+PHINode *FunctionStackPoisoner::createPHI(IRBuilder<> &IRB, Value *Cond,
+ Value *ValueIfTrue,
+ Instruction *ThenTerm,
+ Value *ValueIfFalse) {
+ PHINode *PHI = IRB.CreatePHI(IntptrTy, 2);
+ BasicBlock *CondBlock = cast<Instruction>(Cond)->getParent();
+ PHI->addIncoming(ValueIfFalse, CondBlock);
+ BasicBlock *ThenBlock = ThenTerm->getParent();
+ PHI->addIncoming(ValueIfTrue, ThenBlock);
+ return PHI;
+}
+
+Value *FunctionStackPoisoner::createAllocaForLayout(
+ IRBuilder<> &IRB, const ASanStackFrameLayout &L, bool Dynamic) {
+ AllocaInst *Alloca;
+ if (Dynamic) {
+ Alloca = IRB.CreateAlloca(IRB.getInt8Ty(),
+ ConstantInt::get(IRB.getInt64Ty(), L.FrameSize),
+ "MyAlloca");
+ } else {
+ Alloca = IRB.CreateAlloca(ArrayType::get(IRB.getInt8Ty(), L.FrameSize),
+ nullptr, "MyAlloca");
+ assert(Alloca->isStaticAlloca());
+ }
+ assert((ClRealignStack & (ClRealignStack - 1)) == 0);
+ size_t FrameAlignment = std::max(L.FrameAlignment, (size_t)ClRealignStack);
+ Alloca->setAlignment(FrameAlignment);
+ return IRB.CreatePointerCast(Alloca, IntptrTy);
+}
+
void FunctionStackPoisoner::poisonStack() {
+ assert(AllocaVec.size() > 0 || DynamicAllocaVec.size() > 0);
+
+ if (ClInstrumentAllocas)
+ // Handle dynamic allocas.
+ for (auto &AllocaCall : DynamicAllocaVec)
+ handleDynamicAllocaCall(AllocaCall);
+
+ if (AllocaVec.size() == 0) return;
+
int StackMallocIdx = -1;
DebugLoc EntryDebugLocation = getFunctionEntryDebugLocation(F);
- assert(AllocaVec.size() > 0);
Instruction *InsBefore = AllocaVec[0];
IRBuilder<> IRB(InsBefore);
IRB.SetCurrentDebugLocation(EntryDebugLocation);
@@ -1602,42 +1671,56 @@ void FunctionStackPoisoner::poisonStack() {
uint64_t LocalStackSize = L.FrameSize;
bool DoStackMalloc =
ClUseAfterReturn && LocalStackSize <= kMaxStackMallocSize;
+ // Don't do dynamic alloca in presence of inline asm: too often it
+ // makes assumptions on which registers are available.
+ bool DoDynamicAlloca = ClDynamicAllocaStack && !HasNonEmptyInlineAsm;
- Type *ByteArrayTy = ArrayType::get(IRB.getInt8Ty(), LocalStackSize);
- AllocaInst *MyAlloca =
- new AllocaInst(ByteArrayTy, "MyAlloca", InsBefore);
- MyAlloca->setDebugLoc(EntryDebugLocation);
- assert((ClRealignStack & (ClRealignStack - 1)) == 0);
- size_t FrameAlignment = std::max(L.FrameAlignment, (size_t)ClRealignStack);
- MyAlloca->setAlignment(FrameAlignment);
- assert(MyAlloca->isStaticAlloca());
- Value *OrigStackBase = IRB.CreatePointerCast(MyAlloca, IntptrTy);
- Value *LocalStackBase = OrigStackBase;
+ Value *StaticAlloca =
+ DoDynamicAlloca ? nullptr : createAllocaForLayout(IRB, L, false);
+
+ Value *FakeStack;
+ Value *LocalStackBase;
if (DoStackMalloc) {
- // LocalStackBase = OrigStackBase
- // if (__asan_option_detect_stack_use_after_return)
- // LocalStackBase = __asan_stack_malloc_N(LocalStackBase, OrigStackBase);
- StackMallocIdx = StackMallocSizeClass(LocalStackSize);
- assert(StackMallocIdx <= kMaxAsanStackMallocSizeClass);
+ // void *FakeStack = __asan_option_detect_stack_use_after_return
+ // ? __asan_stack_malloc_N(LocalStackSize)
+ // : nullptr;
+ // void *LocalStackBase = (FakeStack) ? FakeStack : alloca(LocalStackSize);
Constant *OptionDetectUAR = F.getParent()->getOrInsertGlobal(
kAsanOptionDetectUAR, IRB.getInt32Ty());
- Value *Cmp = IRB.CreateICmpNE(IRB.CreateLoad(OptionDetectUAR),
- Constant::getNullValue(IRB.getInt32Ty()));
- Instruction *Term = SplitBlockAndInsertIfThen(Cmp, InsBefore, false);
- BasicBlock *CmpBlock = cast<Instruction>(Cmp)->getParent();
+ Value *UARIsEnabled =
+ IRB.CreateICmpNE(IRB.CreateLoad(OptionDetectUAR),
+ Constant::getNullValue(IRB.getInt32Ty()));
+ Instruction *Term =
+ SplitBlockAndInsertIfThen(UARIsEnabled, InsBefore, false);
IRBuilder<> IRBIf(Term);
IRBIf.SetCurrentDebugLocation(EntryDebugLocation);
- LocalStackBase = IRBIf.CreateCall2(
- AsanStackMallocFunc[StackMallocIdx],
- ConstantInt::get(IntptrTy, LocalStackSize), OrigStackBase);
- BasicBlock *SetBlock = cast<Instruction>(LocalStackBase)->getParent();
+ StackMallocIdx = StackMallocSizeClass(LocalStackSize);
+ assert(StackMallocIdx <= kMaxAsanStackMallocSizeClass);
+ Value *FakeStackValue =
+ IRBIf.CreateCall(AsanStackMallocFunc[StackMallocIdx],
+ ConstantInt::get(IntptrTy, LocalStackSize));
+ IRB.SetInsertPoint(InsBefore);
+ IRB.SetCurrentDebugLocation(EntryDebugLocation);
+ FakeStack = createPHI(IRB, UARIsEnabled, FakeStackValue, Term,
+ ConstantInt::get(IntptrTy, 0));
+
+ Value *NoFakeStack =
+ IRB.CreateICmpEQ(FakeStack, Constant::getNullValue(IntptrTy));
+ Term = SplitBlockAndInsertIfThen(NoFakeStack, InsBefore, false);
+ IRBIf.SetInsertPoint(Term);
+ IRBIf.SetCurrentDebugLocation(EntryDebugLocation);
+ Value *AllocaValue =
+ DoDynamicAlloca ? createAllocaForLayout(IRBIf, L, true) : StaticAlloca;
IRB.SetInsertPoint(InsBefore);
IRB.SetCurrentDebugLocation(EntryDebugLocation);
- PHINode *Phi = IRB.CreatePHI(IntptrTy, 2);
- Phi->addIncoming(OrigStackBase, CmpBlock);
- Phi->addIncoming(LocalStackBase, SetBlock);
- LocalStackBase = Phi;
+ LocalStackBase = createPHI(IRB, NoFakeStack, AllocaValue, Term, FakeStack);
+ } else {
+ // void *FakeStack = nullptr;
+ // void *LocalStackBase = alloca(LocalStackSize);
+ FakeStack = ConstantInt::get(IntptrTy, 0);
+ LocalStackBase =
+ DoDynamicAlloca ? createAllocaForLayout(IRB, L, true) : StaticAlloca;
}
// Insert poison calls for lifetime intrinsics for alloca.
@@ -1694,17 +1777,18 @@ void FunctionStackPoisoner::poisonStack() {
BasePlus0);
if (DoStackMalloc) {
assert(StackMallocIdx >= 0);
- // if LocalStackBase != OrigStackBase:
+ // if FakeStack != 0 // LocalStackBase == FakeStack
// // In use-after-return mode, poison the whole stack frame.
// if StackMallocIdx <= 4
// // For small sizes inline the whole thing:
// memset(ShadowBase, kAsanStackAfterReturnMagic, ShadowSize);
- // **SavedFlagPtr(LocalStackBase) = 0
+ // **SavedFlagPtr(FakeStack) = 0
// else
- // __asan_stack_free_N(LocalStackBase, OrigStackBase)
+ // __asan_stack_free_N(FakeStack, LocalStackSize)
// else
// <This is not a fake stack; unpoison the redzones>
- Value *Cmp = IRBRet.CreateICmpNE(LocalStackBase, OrigStackBase);
+ Value *Cmp =
+ IRBRet.CreateICmpNE(FakeStack, Constant::getNullValue(IntptrTy));
TerminatorInst *ThenTerm, *ElseTerm;
SplitBlockAndInsertIfThenElse(Cmp, Ret, &ThenTerm, &ElseTerm);
@@ -1714,7 +1798,7 @@ void FunctionStackPoisoner::poisonStack() {
SetShadowToStackAfterReturnInlined(IRBPoison, ShadowBase,
ClassSize >> Mapping.Scale);
Value *SavedFlagPtrPtr = IRBPoison.CreateAdd(
- LocalStackBase,
+ FakeStack,
ConstantInt::get(IntptrTy, ClassSize - ASan.LongSize / 8));
Value *SavedFlagPtr = IRBPoison.CreateLoad(
IRBPoison.CreateIntToPtr(SavedFlagPtrPtr, IntptrPtrTy));
@@ -1723,9 +1807,8 @@ void FunctionStackPoisoner::poisonStack() {
IRBPoison.CreateIntToPtr(SavedFlagPtr, IRBPoison.getInt8PtrTy()));
} else {
// For larger frames call __asan_stack_free_*.
- IRBPoison.CreateCall3(AsanStackFreeFunc[StackMallocIdx], LocalStackBase,
- ConstantInt::get(IntptrTy, LocalStackSize),
- OrigStackBase);
+ IRBPoison.CreateCall2(AsanStackFreeFunc[StackMallocIdx], FakeStack,
+ ConstantInt::get(IntptrTy, LocalStackSize));
}
IRBuilder<> IRBElse(ElseTerm);
@@ -1733,13 +1816,17 @@ void FunctionStackPoisoner::poisonStack() {
} else if (HavePoisonedAllocas) {
// If we poisoned some allocas in llvm.lifetime analysis,
// unpoison whole stack frame now.
- assert(LocalStackBase == OrigStackBase);
poisonAlloca(LocalStackBase, LocalStackSize, IRBRet, false);
} else {
poisonRedZones(L.ShadowBytes, IRBRet, ShadowBase, false);
}
}
+ if (ClInstrumentAllocas)
+ // Unpoison dynamic allocas.
+ for (auto &AllocaCall : DynamicAllocaVec)
+ unpoisonDynamicAlloca(AllocaCall);
+
// We are done. Remove the old unused alloca instructions.
for (auto AI : AllocaVec)
AI->eraseFromParent();
@@ -1795,3 +1882,140 @@ AllocaInst *FunctionStackPoisoner::findAllocaForValue(Value *V) {
AllocaForValue[V] = Res;
return Res;
}
+
+// Compute PartialRzMagic for dynamic alloca call. PartialRzMagic is
+// constructed from two separate 32-bit numbers: PartialRzMagic = Val1 | Val2.
+// (1) Val1 is resposible for forming base value for PartialRzMagic, containing
+// only 00 for fully addressable and 0xcb for fully poisoned bytes for each
+// 8-byte chunk of user memory respectively.
+// (2) Val2 forms the value for marking first poisoned byte in shadow memory
+// with appropriate value (0x01 - 0x07 or 0xcb if Padding % 8 == 0).
+
+// Shift = Padding & ~7; // the number of bits we need to shift to access first
+// chunk in shadow memory, containing nonzero bytes.
+// Example:
+// Padding = 21 Padding = 16
+// Shadow: |00|00|05|cb| Shadow: |00|00|cb|cb|
+// ^ ^
+// | |
+// Shift = 21 & ~7 = 16 Shift = 16 & ~7 = 16
+//
+// Val1 = 0xcbcbcbcb << Shift;
+// PartialBits = Padding ? Padding & 7 : 0xcb;
+// Val2 = PartialBits << Shift;
+// Result = Val1 | Val2;
+Value *FunctionStackPoisoner::computePartialRzMagic(Value *PartialSize,
+ IRBuilder<> &IRB) {
+ PartialSize = IRB.CreateIntCast(PartialSize, IRB.getInt32Ty(), false);
+ Value *Shift = IRB.CreateAnd(PartialSize, IRB.getInt32(~7));
+ unsigned Val1Int = kAsanAllocaPartialVal1;
+ unsigned Val2Int = kAsanAllocaPartialVal2;
+ if (!ASan.DL->isLittleEndian()) {
+ Val1Int = sys::getSwappedBytes(Val1Int);
+ Val2Int = sys::getSwappedBytes(Val2Int);
+ }
+ Value *Val1 = shiftAllocaMagic(IRB.getInt32(Val1Int), IRB, Shift);
+ Value *PartialBits = IRB.CreateAnd(PartialSize, IRB.getInt32(7));
+ // For BigEndian get 0x000000YZ -> 0xYZ000000.
+ if (ASan.DL->isBigEndian())
+ PartialBits = IRB.CreateShl(PartialBits, IRB.getInt32(24));
+ Value *Val2 = IRB.getInt32(Val2Int);
+ Value *Cond =
+ IRB.CreateICmpNE(PartialBits, Constant::getNullValue(IRB.getInt32Ty()));
+ Val2 = IRB.CreateSelect(Cond, shiftAllocaMagic(PartialBits, IRB, Shift),
+ shiftAllocaMagic(Val2, IRB, Shift));
+ return IRB.CreateOr(Val1, Val2);
+}
+
+void FunctionStackPoisoner::handleDynamicAllocaCall(
+ DynamicAllocaCall &AllocaCall) {
+ AllocaInst *AI = AllocaCall.AI;
+ if (!doesDominateAllExits(AI)) {
+ // We do not yet handle complex allocas
+ AllocaCall.Poison = false;
+ return;
+ }
+
+ IRBuilder<> IRB(AI);
+
+ PointerType *Int32PtrTy = PointerType::getUnqual(IRB.getInt32Ty());
+ const unsigned Align = std::max(kAllocaRzSize, AI->getAlignment());
+ const uint64_t AllocaRedzoneMask = kAllocaRzSize - 1;
+
+ Value *Zero = Constant::getNullValue(IntptrTy);
+ Value *AllocaRzSize = ConstantInt::get(IntptrTy, kAllocaRzSize);
+ Value *AllocaRzMask = ConstantInt::get(IntptrTy, AllocaRedzoneMask);
+ Value *NotAllocaRzMask = ConstantInt::get(IntptrTy, ~AllocaRedzoneMask);
+
+ // Since we need to extend alloca with additional memory to locate
+ // redzones, and OldSize is number of allocated blocks with
+ // ElementSize size, get allocated memory size in bytes by
+ // OldSize * ElementSize.
+ unsigned ElementSize = ASan.DL->getTypeAllocSize(AI->getAllocatedType());
+ Value *OldSize = IRB.CreateMul(AI->getArraySize(),
+ ConstantInt::get(IntptrTy, ElementSize));
+
+ // PartialSize = OldSize % 32
+ Value *PartialSize = IRB.CreateAnd(OldSize, AllocaRzMask);
+
+ // Misalign = kAllocaRzSize - PartialSize;
+ Value *Misalign = IRB.CreateSub(AllocaRzSize, PartialSize);
+
+ // PartialPadding = Misalign != kAllocaRzSize ? Misalign : 0;
+ Value *Cond = IRB.CreateICmpNE(Misalign, AllocaRzSize);
+ Value *PartialPadding = IRB.CreateSelect(Cond, Misalign, Zero);
+
+ // AdditionalChunkSize = Align + PartialPadding + kAllocaRzSize
+ // Align is added to locate left redzone, PartialPadding for possible
+ // partial redzone and kAllocaRzSize for right redzone respectively.
+ Value *AdditionalChunkSize = IRB.CreateAdd(
+ ConstantInt::get(IntptrTy, Align + kAllocaRzSize), PartialPadding);
+
+ Value *NewSize = IRB.CreateAdd(OldSize, AdditionalChunkSize);
+
+ // Insert new alloca with new NewSize and Align params.
+ AllocaInst *NewAlloca = IRB.CreateAlloca(IRB.getInt8Ty(), NewSize);
+ NewAlloca->setAlignment(Align);
+
+ // NewAddress = Address + Align
+ Value *NewAddress = IRB.CreateAdd(IRB.CreatePtrToInt(NewAlloca, IntptrTy),
+ ConstantInt::get(IntptrTy, Align));
+
+ Value *NewAddressPtr = IRB.CreateIntToPtr(NewAddress, AI->getType());
+
+ // LeftRzAddress = NewAddress - kAllocaRzSize
+ Value *LeftRzAddress = IRB.CreateSub(NewAddress, AllocaRzSize);
+
+ // Poisoning left redzone.
+ AllocaCall.LeftRzAddr = ASan.memToShadow(LeftRzAddress, IRB);
+ IRB.CreateStore(ConstantInt::get(IRB.getInt32Ty(), kAsanAllocaLeftMagic),
+ IRB.CreateIntToPtr(AllocaCall.LeftRzAddr, Int32PtrTy));
+
+ // PartialRzAligned = PartialRzAddr & ~AllocaRzMask
+ Value *PartialRzAddr = IRB.CreateAdd(NewAddress, OldSize);
+ Value *PartialRzAligned = IRB.CreateAnd(PartialRzAddr, NotAllocaRzMask);
+
+ // Poisoning partial redzone.
+ Value *PartialRzMagic = computePartialRzMagic(PartialSize, IRB);
+ Value *PartialRzShadowAddr = ASan.memToShadow(PartialRzAligned, IRB);
+ IRB.CreateStore(PartialRzMagic,
+ IRB.CreateIntToPtr(PartialRzShadowAddr, Int32PtrTy));
+
+ // RightRzAddress
+ // = (PartialRzAddr + AllocaRzMask) & ~AllocaRzMask
+ Value *RightRzAddress = IRB.CreateAnd(
+ IRB.CreateAdd(PartialRzAddr, AllocaRzMask), NotAllocaRzMask);
+
+ // Poisoning right redzone.
+ AllocaCall.RightRzAddr = ASan.memToShadow(RightRzAddress, IRB);
+ IRB.CreateStore(ConstantInt::get(IRB.getInt32Ty(), kAsanAllocaRightMagic),
+ IRB.CreateIntToPtr(AllocaCall.RightRzAddr, Int32PtrTy));
+
+ // Replace all uses of AddessReturnedByAlloca with NewAddress.
+ AI->replaceAllUsesWith(NewAddressPtr);
+
+ // We are done. Erase old alloca and store left, partial and right redzones
+ // shadow addresses for future unpoisoning.
+ AI->eraseFromParent();
+ NumInstrumentedDynamicAllocas++;
+}
diff --git a/lib/Transforms/Instrumentation/CMakeLists.txt b/lib/Transforms/Instrumentation/CMakeLists.txt
index 35635934b81c..92e1091aa3b1 100644
--- a/lib/Transforms/Instrumentation/CMakeLists.txt
+++ b/lib/Transforms/Instrumentation/CMakeLists.txt
@@ -2,10 +2,11 @@ add_llvm_library(LLVMInstrumentation
AddressSanitizer.cpp
BoundsChecking.cpp
DataFlowSanitizer.cpp
- DebugIR.cpp
GCOVProfiling.cpp
MemorySanitizer.cpp
Instrumentation.cpp
+ InstrProfiling.cpp
+ SanitizerCoverage.cpp
ThreadSanitizer.cpp
)
diff --git a/lib/Transforms/Instrumentation/DataFlowSanitizer.cpp b/lib/Transforms/Instrumentation/DataFlowSanitizer.cpp
index 35057cdd47e9..8f24476f03c1 100644
--- a/lib/Transforms/Instrumentation/DataFlowSanitizer.cpp
+++ b/lib/Transforms/Instrumentation/DataFlowSanitizer.cpp
@@ -49,8 +49,10 @@
#include "llvm/ADT/DenseSet.h"
#include "llvm/ADT/DepthFirstIterator.h"
#include "llvm/ADT/StringExtras.h"
+#include "llvm/ADT/Triple.h"
#include "llvm/Analysis/ValueTracking.h"
#include "llvm/IR/Dominators.h"
+#include "llvm/IR/DebugInfo.h"
#include "llvm/IR/IRBuilder.h"
#include "llvm/IR/InlineAsm.h"
#include "llvm/IR/InstVisitor.h"
@@ -139,11 +141,11 @@ class DFSanABIList {
std::unique_ptr<SpecialCaseList> SCL;
public:
- DFSanABIList(SpecialCaseList *SCL) : SCL(SCL) {}
+ DFSanABIList(std::unique_ptr<SpecialCaseList> SCL) : SCL(std::move(SCL)) {}
/// Returns whether either this function or its source file are listed in the
/// given category.
- bool isIn(const Function &F, const StringRef Category) const {
+ bool isIn(const Function &F, StringRef Category) const {
return isIn(*F.getParent(), Category) ||
SCL->inSection("fun", F.getName(), Category);
}
@@ -152,7 +154,7 @@ class DFSanABIList {
///
/// If GA aliases a function, the alias's name is matched as a function name
/// would be. Similarly, aliases of globals are matched like globals.
- bool isIn(const GlobalAlias &GA, const StringRef Category) const {
+ bool isIn(const GlobalAlias &GA, StringRef Category) const {
if (isIn(*GA.getParent(), Category))
return true;
@@ -164,7 +166,7 @@ class DFSanABIList {
}
/// Returns whether this module is listed in the given category.
- bool isIn(const Module &M, const StringRef Category) const {
+ bool isIn(const Module &M, StringRef Category) const {
return SCL->inSection("src", M.getModuleIdentifier(), Category);
}
};
@@ -233,15 +235,19 @@ class DataFlowSanitizer : public ModulePass {
FunctionType *DFSanUnimplementedFnTy;
FunctionType *DFSanSetLabelFnTy;
FunctionType *DFSanNonzeroLabelFnTy;
+ FunctionType *DFSanVarargWrapperFnTy;
Constant *DFSanUnionFn;
+ Constant *DFSanCheckedUnionFn;
Constant *DFSanUnionLoadFn;
Constant *DFSanUnimplementedFn;
Constant *DFSanSetLabelFn;
Constant *DFSanNonzeroLabelFn;
+ Constant *DFSanVarargWrapperFn;
MDNode *ColdCallWeights;
DFSanABIList ABIList;
DenseMap<Value *, Function *> UnwrappedFnMap;
AttributeSet ReadOnlyNoneAttrs;
+ DenseMap<const Function *, DISubprogram> FunctionDIs;
Value *getShadowAddress(Value *Addr, Instruction *Pos);
bool isInstrumented(const Function *F);
@@ -279,7 +285,8 @@ struct DFSanFunction {
DenseMap<AllocaInst *, AllocaInst *> AllocaShadowMap;
std::vector<std::pair<PHINode *, PHINode *> > PHIFixups;
DenseSet<Instruction *> SkipInsts;
- DenseSet<Value *> NonZeroChecks;
+ std::vector<Value *> NonZeroChecks;
+ bool AvoidNewBlocks;
struct CachedCombinedShadow {
BasicBlock *Block;
@@ -294,6 +301,9 @@ struct DFSanFunction {
IsNativeABI(IsNativeABI), ArgTLSPtr(nullptr), RetvalTLSPtr(nullptr),
LabelReturnAlloca(nullptr) {
DT.recalculate(*F);
+ // FIXME: Need to track down the register allocator issue which causes poor
+ // performance in pathological cases with large numbers of basic blocks.
+ AvoidNewBlocks = F->size() > 1000;
}
Value *getArgTLSPtr();
Value *getArgTLS(unsigned Index, Instruction *Pos);
@@ -382,7 +392,6 @@ FunctionType *DataFlowSanitizer::getTrampolineFunctionType(FunctionType *T) {
}
FunctionType *DataFlowSanitizer::getCustomFunctionType(FunctionType *T) {
- assert(!T->isVarArg());
llvm::SmallVector<Type *, 4> ArgTypes;
for (FunctionType::param_iterator i = T->param_begin(), e = T->param_end();
i != e; ++i) {
@@ -397,13 +406,20 @@ FunctionType *DataFlowSanitizer::getCustomFunctionType(FunctionType *T) {
}
for (unsigned i = 0, e = T->getNumParams(); i != e; ++i)
ArgTypes.push_back(ShadowTy);
+ if (T->isVarArg())
+ ArgTypes.push_back(ShadowPtrTy);
Type *RetType = T->getReturnType();
if (!RetType->isVoidTy())
ArgTypes.push_back(ShadowPtrTy);
- return FunctionType::get(T->getReturnType(), ArgTypes, false);
+ return FunctionType::get(T->getReturnType(), ArgTypes, T->isVarArg());
}
bool DataFlowSanitizer::doInitialization(Module &M) {
+ llvm::Triple TargetTriple(M.getTargetTriple());
+ bool IsX86_64 = TargetTriple.getArch() == llvm::Triple::x86_64;
+ bool IsMIPS64 = TargetTriple.getArch() == llvm::Triple::mips64 ||
+ TargetTriple.getArch() == llvm::Triple::mips64el;
+
DataLayoutPass *DLP = getAnalysisIfAvailable<DataLayoutPass>();
if (!DLP)
report_fatal_error("data layout missing");
@@ -415,8 +431,13 @@ bool DataFlowSanitizer::doInitialization(Module &M) {
ShadowPtrTy = PointerType::getUnqual(ShadowTy);
IntptrTy = DL->getIntPtrType(*Ctx);
ZeroShadow = ConstantInt::getSigned(ShadowTy, 0);
- ShadowPtrMask = ConstantInt::getSigned(IntptrTy, ~0x700000000000LL);
ShadowPtrMul = ConstantInt::getSigned(IntptrTy, ShadowWidth / 8);
+ if (IsX86_64)
+ ShadowPtrMask = ConstantInt::getSigned(IntptrTy, ~0x700000000000LL);
+ else if (IsMIPS64)
+ ShadowPtrMask = ConstantInt::getSigned(IntptrTy, ~0xF000000000LL);
+ else
+ report_fatal_error("unsupported triple");
Type *DFSanUnionArgs[2] = { ShadowTy, ShadowTy };
DFSanUnionFnTy =
@@ -430,7 +451,9 @@ bool DataFlowSanitizer::doInitialization(Module &M) {
DFSanSetLabelFnTy = FunctionType::get(Type::getVoidTy(*Ctx),
DFSanSetLabelArgs, /*isVarArg=*/false);
DFSanNonzeroLabelFnTy = FunctionType::get(
- Type::getVoidTy(*Ctx), ArrayRef<Type *>(), /*isVarArg=*/false);
+ Type::getVoidTy(*Ctx), None, /*isVarArg=*/false);
+ DFSanVarargWrapperFnTy = FunctionType::get(
+ Type::getVoidTy(*Ctx), Type::getInt8PtrTy(*Ctx), /*isVarArg=*/false);
if (GetArgTLSPtr) {
Type *ArgTLSTy = ArrayType::get(ShadowTy, 64);
@@ -510,15 +533,26 @@ DataFlowSanitizer::buildWrapperFunction(Function *F, StringRef NewFName,
AttributeSet::ReturnIndex));
BasicBlock *BB = BasicBlock::Create(*Ctx, "entry", NewF);
- std::vector<Value *> Args;
- unsigned n = FT->getNumParams();
- for (Function::arg_iterator ai = NewF->arg_begin(); n != 0; ++ai, --n)
- Args.push_back(&*ai);
- CallInst *CI = CallInst::Create(F, Args, "", BB);
- if (FT->getReturnType()->isVoidTy())
- ReturnInst::Create(*Ctx, BB);
- else
- ReturnInst::Create(*Ctx, CI, BB);
+ if (F->isVarArg()) {
+ NewF->removeAttributes(
+ AttributeSet::FunctionIndex,
+ AttributeSet().addAttribute(*Ctx, AttributeSet::FunctionIndex,
+ "split-stack"));
+ CallInst::Create(DFSanVarargWrapperFn,
+ IRBuilder<>(BB).CreateGlobalStringPtr(F->getName()), "",
+ BB);
+ new UnreachableInst(*Ctx, BB);
+ } else {
+ std::vector<Value *> Args;
+ unsigned n = FT->getNumParams();
+ for (Function::arg_iterator ai = NewF->arg_begin(); n != 0; ++ai, --n)
+ Args.push_back(&*ai);
+ CallInst *CI = CallInst::Create(F, Args, "", BB);
+ if (FT->getReturnType()->isVoidTy())
+ ReturnInst::Create(*Ctx, BB);
+ else
+ ReturnInst::Create(*Ctx, CI, BB);
+ }
return NewF;
}
@@ -563,6 +597,8 @@ bool DataFlowSanitizer::runOnModule(Module &M) {
if (ABIList.isIn(M, "skip"))
return false;
+ FunctionDIs = makeSubprogramMap(M);
+
if (!GetArgTLSPtr) {
Type *ArgTLSTy = ArrayType::get(ShadowTy, 64);
ArgTLS = Mod->getOrInsertGlobal("__dfsan_arg_tls", ArgTLSTy);
@@ -577,6 +613,15 @@ bool DataFlowSanitizer::runOnModule(Module &M) {
DFSanUnionFn = Mod->getOrInsertFunction("__dfsan_union", DFSanUnionFnTy);
if (Function *F = dyn_cast<Function>(DFSanUnionFn)) {
+ F->addAttribute(AttributeSet::FunctionIndex, Attribute::NoUnwind);
+ F->addAttribute(AttributeSet::FunctionIndex, Attribute::ReadNone);
+ F->addAttribute(AttributeSet::ReturnIndex, Attribute::ZExt);
+ F->addAttribute(1, Attribute::ZExt);
+ F->addAttribute(2, Attribute::ZExt);
+ }
+ DFSanCheckedUnionFn = Mod->getOrInsertFunction("dfsan_union", DFSanUnionFnTy);
+ if (Function *F = dyn_cast<Function>(DFSanCheckedUnionFn)) {
+ F->addAttribute(AttributeSet::FunctionIndex, Attribute::NoUnwind);
F->addAttribute(AttributeSet::FunctionIndex, Attribute::ReadNone);
F->addAttribute(AttributeSet::ReturnIndex, Attribute::ZExt);
F->addAttribute(1, Attribute::ZExt);
@@ -585,6 +630,7 @@ bool DataFlowSanitizer::runOnModule(Module &M) {
DFSanUnionLoadFn =
Mod->getOrInsertFunction("__dfsan_union_load", DFSanUnionLoadFnTy);
if (Function *F = dyn_cast<Function>(DFSanUnionLoadFn)) {
+ F->addAttribute(AttributeSet::FunctionIndex, Attribute::NoUnwind);
F->addAttribute(AttributeSet::FunctionIndex, Attribute::ReadOnly);
F->addAttribute(AttributeSet::ReturnIndex, Attribute::ZExt);
}
@@ -597,16 +643,20 @@ bool DataFlowSanitizer::runOnModule(Module &M) {
}
DFSanNonzeroLabelFn =
Mod->getOrInsertFunction("__dfsan_nonzero_label", DFSanNonzeroLabelFnTy);
+ DFSanVarargWrapperFn = Mod->getOrInsertFunction("__dfsan_vararg_wrapper",
+ DFSanVarargWrapperFnTy);
std::vector<Function *> FnsToInstrument;
llvm::SmallPtrSet<Function *, 2> FnsWithNativeABI;
for (Module::iterator i = M.begin(), e = M.end(); i != e; ++i) {
if (!i->isIntrinsic() &&
i != DFSanUnionFn &&
+ i != DFSanCheckedUnionFn &&
i != DFSanUnionLoadFn &&
i != DFSanUnimplementedFn &&
i != DFSanSetLabelFn &&
- i != DFSanNonzeroLabelFn)
+ i != DFSanNonzeroLabelFn &&
+ i != DFSanVarargWrapperFn)
FnsToInstrument.push_back(&*i);
}
@@ -688,11 +738,6 @@ bool DataFlowSanitizer::runOnModule(Module &M) {
} else {
addGlobalNamePrefix(&F);
}
- // Hopefully, nobody will try to indirectly call a vararg
- // function... yet.
- } else if (FT->isVarArg()) {
- UnwrappedFnMap[&F] = &F;
- *i = nullptr;
} else if (!IsZeroArgsVoidRet || getWrapperKind(&F) == WK_Custom) {
// Build a wrapper function for F. The wrapper simply calls F, and is
// added to FnsToInstrument so that any instrumentation according to its
@@ -709,6 +754,12 @@ bool DataFlowSanitizer::runOnModule(Module &M) {
Value *WrappedFnCst =
ConstantExpr::getBitCast(NewF, PointerType::getUnqual(FT));
F.replaceAllUsesWith(WrappedFnCst);
+
+ // Patch the pointer to LLVM function in debug info descriptor.
+ auto DI = FunctionDIs.find(&F);
+ if (DI != FunctionDIs.end())
+ DI->second.replaceFunction(&F);
+
UnwrappedFnMap[WrappedFnCst] = &F;
*i = NewF;
@@ -728,6 +779,11 @@ bool DataFlowSanitizer::runOnModule(Module &M) {
i = FnsToInstrument.begin() + N;
e = FnsToInstrument.begin() + Count;
}
+ // Hopefully, nobody will try to indirectly call a vararg
+ // function... yet.
+ } else if (FT->isVarArg()) {
+ UnwrappedFnMap[&F] = &F;
+ *i = nullptr;
}
}
@@ -786,18 +842,16 @@ bool DataFlowSanitizer::runOnModule(Module &M) {
// yet). To make our life easier, do this work in a pass after the main
// instrumentation.
if (ClDebugNonzeroLabels) {
- for (DenseSet<Value *>::iterator i = DFSF.NonZeroChecks.begin(),
- e = DFSF.NonZeroChecks.end();
- i != e; ++i) {
+ for (Value *V : DFSF.NonZeroChecks) {
Instruction *Pos;
- if (Instruction *I = dyn_cast<Instruction>(*i))
+ if (Instruction *I = dyn_cast<Instruction>(V))
Pos = I->getNextNode();
else
Pos = DFSF.F->getEntryBlock().begin();
while (isa<PHINode>(Pos) || isa<AllocaInst>(Pos))
Pos = Pos->getNextNode();
IRBuilder<> IRB(Pos);
- Value *Ne = IRB.CreateICmpNE(*i, DFSF.DFS.ZeroShadow);
+ Value *Ne = IRB.CreateICmpNE(V, DFSF.DFS.ZeroShadow);
BranchInst *BI = cast<BranchInst>(SplitBlockAndInsertIfThen(
Ne, Pos, /*Unreachable=*/false, ColdCallWeights));
IRBuilder<> ThenIRB(BI);
@@ -862,7 +916,7 @@ Value *DFSanFunction::getShadow(Value *V) {
break;
}
}
- NonZeroChecks.insert(Shadow);
+ NonZeroChecks.push_back(Shadow);
} else {
Shadow = DFS.ZeroShadow;
}
@@ -922,23 +976,33 @@ Value *DFSanFunction::combineShadows(Value *V1, Value *V2, Instruction *Pos) {
return CCS.Shadow;
IRBuilder<> IRB(Pos);
- BasicBlock *Head = Pos->getParent();
- Value *Ne = IRB.CreateICmpNE(V1, V2);
- BranchInst *BI = cast<BranchInst>(SplitBlockAndInsertIfThen(
- Ne, Pos, /*Unreachable=*/false, DFS.ColdCallWeights, &DT));
- IRBuilder<> ThenIRB(BI);
- CallInst *Call = ThenIRB.CreateCall2(DFS.DFSanUnionFn, V1, V2);
- Call->addAttribute(AttributeSet::ReturnIndex, Attribute::ZExt);
- Call->addAttribute(1, Attribute::ZExt);
- Call->addAttribute(2, Attribute::ZExt);
-
- BasicBlock *Tail = BI->getSuccessor(0);
- PHINode *Phi = PHINode::Create(DFS.ShadowTy, 2, "", Tail->begin());
- Phi->addIncoming(Call, Call->getParent());
- Phi->addIncoming(V1, Head);
-
- CCS.Block = Tail;
- CCS.Shadow = Phi;
+ if (AvoidNewBlocks) {
+ CallInst *Call = IRB.CreateCall2(DFS.DFSanCheckedUnionFn, V1, V2);
+ Call->addAttribute(AttributeSet::ReturnIndex, Attribute::ZExt);
+ Call->addAttribute(1, Attribute::ZExt);
+ Call->addAttribute(2, Attribute::ZExt);
+
+ CCS.Block = Pos->getParent();
+ CCS.Shadow = Call;
+ } else {
+ BasicBlock *Head = Pos->getParent();
+ Value *Ne = IRB.CreateICmpNE(V1, V2);
+ BranchInst *BI = cast<BranchInst>(SplitBlockAndInsertIfThen(
+ Ne, Pos, /*Unreachable=*/false, DFS.ColdCallWeights, &DT));
+ IRBuilder<> ThenIRB(BI);
+ CallInst *Call = ThenIRB.CreateCall2(DFS.DFSanUnionFn, V1, V2);
+ Call->addAttribute(AttributeSet::ReturnIndex, Attribute::ZExt);
+ Call->addAttribute(1, Attribute::ZExt);
+ Call->addAttribute(2, Attribute::ZExt);
+
+ BasicBlock *Tail = BI->getSuccessor(0);
+ PHINode *Phi = PHINode::Create(DFS.ShadowTy, 2, "", Tail->begin());
+ Phi->addIncoming(Call, Call->getParent());
+ Phi->addIncoming(V1, Head);
+
+ CCS.Block = Tail;
+ CCS.Shadow = Phi;
+ }
std::set<Value *> UnionElems;
if (V1Elems != ShadowElements.end()) {
@@ -951,9 +1015,9 @@ Value *DFSanFunction::combineShadows(Value *V1, Value *V2, Instruction *Pos) {
} else {
UnionElems.insert(V2);
}
- ShadowElements[Phi] = std::move(UnionElems);
+ ShadowElements[CCS.Shadow] = std::move(UnionElems);
- return Phi;
+ return CCS.Shadow;
}
// A convenience function which folds the shadows of each of the operands
@@ -1022,7 +1086,7 @@ Value *DFSanFunction::loadShadow(Value *Addr, uint64_t Size, uint64_t Align,
IRB.CreateAlignedLoad(ShadowAddr1, ShadowAlign), Pos);
}
}
- if (Size % (64 / DFS.ShadowWidth) == 0) {
+ if (!AvoidNewBlocks && Size % (64 / DFS.ShadowWidth) == 0) {
// Fast path for the common case where each byte has identical shadow: load
// shadow 64 bits at a time, fall out to a __dfsan_union_load call if any
// shadow is non-equal.
@@ -1092,6 +1156,11 @@ Value *DFSanFunction::loadShadow(Value *Addr, uint64_t Size, uint64_t Align,
void DFSanVisitor::visitLoadInst(LoadInst &LI) {
uint64_t Size = DFSF.DFS.DL->getTypeStoreSize(LI.getType());
+ if (Size == 0) {
+ DFSF.setShadow(&LI, DFSF.DFS.ZeroShadow);
+ return;
+ }
+
uint64_t Align;
if (ClPreserveAlignment) {
Align = LI.getAlignment();
@@ -1107,7 +1176,7 @@ void DFSanVisitor::visitLoadInst(LoadInst &LI) {
Shadow = DFSF.combineShadows(Shadow, PtrShadow, &LI);
}
if (Shadow != DFSF.DFS.ZeroShadow)
- DFSF.NonZeroChecks.insert(Shadow);
+ DFSF.NonZeroChecks.push_back(Shadow);
DFSF.setShadow(&LI, Shadow);
}
@@ -1166,6 +1235,9 @@ void DFSanFunction::storeShadow(Value *Addr, uint64_t Size, uint64_t Align,
void DFSanVisitor::visitStoreInst(StoreInst &SI) {
uint64_t Size =
DFSF.DFS.DL->getTypeStoreSize(SI.getValueOperand()->getType());
+ if (Size == 0)
+ return;
+
uint64_t Align;
if (ClPreserveAlignment) {
Align = SI.getAlignment();
@@ -1320,6 +1392,15 @@ void DFSanVisitor::visitCallSite(CallSite CS) {
return;
}
+ // Calls to this function are synthesized in wrappers, and we shouldn't
+ // instrument them.
+ if (F == DFSF.DFS.DFSanVarargWrapperFn)
+ return;
+
+ assert(!(cast<FunctionType>(
+ CS.getCalledValue()->getType()->getPointerElementType())->isVarArg() &&
+ dyn_cast<InvokeInst>(CS.getInstruction())));
+
IRBuilder<> IRB(CS.getInstruction());
DenseMap<Value *, Function *>::iterator i =
@@ -1391,6 +1472,20 @@ void DFSanVisitor::visitCallSite(CallSite CS) {
for (unsigned n = FT->getNumParams(); n != 0; ++i, --n)
Args.push_back(DFSF.getShadow(*i));
+ if (FT->isVarArg()) {
+ auto LabelVAAlloca =
+ new AllocaInst(ArrayType::get(DFSF.DFS.ShadowTy,
+ CS.arg_size() - FT->getNumParams()),
+ "labelva", DFSF.F->getEntryBlock().begin());
+
+ for (unsigned n = 0; i != CS.arg_end(); ++i, ++n) {
+ auto LabelVAPtr = IRB.CreateStructGEP(LabelVAAlloca, n);
+ IRB.CreateStore(DFSF.getShadow(*i), LabelVAPtr);
+ }
+
+ Args.push_back(IRB.CreateStructGEP(LabelVAAlloca, 0));
+ }
+
if (!FT->getReturnType()->isVoidTy()) {
if (!DFSF.LabelReturnAlloca) {
DFSF.LabelReturnAlloca =
@@ -1400,6 +1495,9 @@ void DFSanVisitor::visitCallSite(CallSite CS) {
Args.push_back(DFSF.LabelReturnAlloca);
}
+ for (i = CS.arg_begin() + FT->getNumParams(); i != CS.arg_end(); ++i)
+ Args.push_back(*i);
+
CallInst *CustomCI = IRB.CreateCall(CustomF, Args);
CustomCI->setCallingConv(CI->getCallingConv());
CustomCI->setAttributes(CI->getAttributes());
@@ -1446,7 +1544,7 @@ void DFSanVisitor::visitCallSite(CallSite CS) {
LoadInst *LI = NextIRB.CreateLoad(DFSF.getRetvalTLS());
DFSF.SkipInsts.insert(LI);
DFSF.setShadow(CS.getInstruction(), LI);
- DFSF.NonZeroChecks.insert(LI);
+ DFSF.NonZeroChecks.push_back(LI);
}
}
@@ -1500,7 +1598,7 @@ void DFSanVisitor::visitCallSite(CallSite CS) {
ExtractValueInst::Create(NewCS.getInstruction(), 1, "", Next);
DFSF.SkipInsts.insert(ExShadow);
DFSF.setShadow(ExVal, ExShadow);
- DFSF.NonZeroChecks.insert(ExShadow);
+ DFSF.NonZeroChecks.push_back(ExShadow);
CS.getInstruction()->replaceAllUsesWith(ExVal);
}
diff --git a/lib/Transforms/Instrumentation/DebugIR.cpp b/lib/Transforms/Instrumentation/DebugIR.cpp
deleted file mode 100644
index f2f1738808be..000000000000
--- a/lib/Transforms/Instrumentation/DebugIR.cpp
+++ /dev/null
@@ -1,617 +0,0 @@
-//===--- DebugIR.cpp - Transform debug metadata to allow debugging IR -----===//
-//
-// The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
-//
-//===----------------------------------------------------------------------===//
-//
-// A Module transform pass that emits a succinct version of the IR and replaces
-// the source file metadata to allow debuggers to step through the IR.
-//
-// FIXME: instead of replacing debug metadata, this pass should allow for
-// additional metadata to be used to point capable debuggers to the IR file
-// without destroying the mapping to the original source file.
-//
-//===----------------------------------------------------------------------===//
-
-#include "llvm/IR/ValueMap.h"
-#include "DebugIR.h"
-#include "llvm/IR/AssemblyAnnotationWriter.h"
-#include "llvm/IR/DIBuilder.h"
-#include "llvm/IR/DataLayout.h"
-#include "llvm/IR/DebugInfo.h"
-#include "llvm/IR/InstVisitor.h"
-#include "llvm/IR/Instruction.h"
-#include "llvm/IR/LLVMContext.h"
-#include "llvm/IR/Module.h"
-#include "llvm/Support/Debug.h"
-#include "llvm/Support/FileSystem.h"
-#include "llvm/Support/FormattedStream.h"
-#include "llvm/Support/Path.h"
-#include "llvm/Support/ToolOutputFile.h"
-#include "llvm/Transforms/Instrumentation.h"
-#include "llvm/Transforms/Utils/Cloning.h"
-#include <string>
-
-#define STR_HELPER(x) #x
-#define STR(x) STR_HELPER(x)
-
-using namespace llvm;
-
-#define DEBUG_TYPE "debug-ir"
-
-namespace {
-
-/// Builds a map of Value* to line numbers on which the Value appears in a
-/// textual representation of the IR by plugging into the AssemblyWriter by
-/// masquerading as an AssemblyAnnotationWriter.
-class ValueToLineMap : public AssemblyAnnotationWriter {
- ValueMap<const Value *, unsigned int> Lines;
- typedef ValueMap<const Value *, unsigned int>::const_iterator LineIter;
-
- void addEntry(const Value *V, formatted_raw_ostream &Out) {
- Out.flush();
- Lines.insert(std::make_pair(V, Out.getLine() + 1));
- }
-
-public:
-
- /// Prints Module to a null buffer in order to build the map of Value pointers
- /// to line numbers.
- ValueToLineMap(const Module *M) {
- raw_null_ostream ThrowAway;
- M->print(ThrowAway, this);
- }
-
- // This function is called after an Instruction, GlobalValue, or GlobalAlias
- // is printed.
- void printInfoComment(const Value &V, formatted_raw_ostream &Out) override {
- addEntry(&V, Out);
- }
-
- void emitFunctionAnnot(const Function *F,
- formatted_raw_ostream &Out) override {
- addEntry(F, Out);
- }
-
- /// If V appears on a line in the textual IR representation, sets Line to the
- /// line number and returns true, otherwise returns false.
- bool getLine(const Value *V, unsigned int &Line) const {
- LineIter i = Lines.find(V);
- if (i != Lines.end()) {
- Line = i->second;
- return true;
- }
- return false;
- }
-};
-
-/// Removes debug intrisncs like llvm.dbg.declare and llvm.dbg.value.
-class DebugIntrinsicsRemover : public InstVisitor<DebugIntrinsicsRemover> {
- void remove(Instruction &I) { I.eraseFromParent(); }
-
-public:
- static void process(Module &M) {
- DebugIntrinsicsRemover Remover;
- Remover.visit(&M);
- }
- void visitDbgDeclareInst(DbgDeclareInst &I) { remove(I); }
- void visitDbgValueInst(DbgValueInst &I) { remove(I); }
- void visitDbgInfoIntrinsic(DbgInfoIntrinsic &I) { remove(I); }
-};
-
-/// Removes debug metadata (!dbg) nodes from all instructions, and optionally
-/// metadata named "llvm.dbg.cu" if RemoveNamedInfo is true.
-class DebugMetadataRemover : public InstVisitor<DebugMetadataRemover> {
- bool RemoveNamedInfo;
-
-public:
- static void process(Module &M, bool RemoveNamedInfo = true) {
- DebugMetadataRemover Remover(RemoveNamedInfo);
- Remover.run(&M);
- }
-
- DebugMetadataRemover(bool RemoveNamedInfo)
- : RemoveNamedInfo(RemoveNamedInfo) {}
-
- void visitInstruction(Instruction &I) {
- if (I.getMetadata(LLVMContext::MD_dbg))
- I.setMetadata(LLVMContext::MD_dbg, nullptr);
- }
-
- void run(Module *M) {
- // Remove debug metadata attached to instructions
- visit(M);
-
- if (RemoveNamedInfo) {
- // Remove CU named metadata (and all children nodes)
- NamedMDNode *Node = M->getNamedMetadata("llvm.dbg.cu");
- if (Node)
- M->eraseNamedMetadata(Node);
- }
- }
-};
-
-/// Updates debug metadata in a Module:
-/// - changes Filename/Directory to values provided on construction
-/// - adds/updates line number (DebugLoc) entries associated with each
-/// instruction to reflect the instruction's location in an LLVM IR file
-class DIUpdater : public InstVisitor<DIUpdater> {
- /// Builder of debug information
- DIBuilder Builder;
-
- /// Helper for type attributes/sizes/etc
- DataLayout Layout;
-
- /// Map of Value* to line numbers
- const ValueToLineMap LineTable;
-
- /// Map of Value* (in original Module) to Value* (in optional cloned Module)
- const ValueToValueMapTy *VMap;
-
- /// Directory of debug metadata
- DebugInfoFinder Finder;
-
- /// Source filename and directory
- StringRef Filename;
- StringRef Directory;
-
- // CU nodes needed when creating DI subprograms
- MDNode *FileNode;
- MDNode *LexicalBlockFileNode;
- const MDNode *CUNode;
-
- ValueMap<const Function *, MDNode *> SubprogramDescriptors;
- DenseMap<const Type *, MDNode *> TypeDescriptors;
-
-public:
- DIUpdater(Module &M, StringRef Filename = StringRef(),
- StringRef Directory = StringRef(), const Module *DisplayM = nullptr,
- const ValueToValueMapTy *VMap = nullptr)
- : Builder(M), Layout(&M), LineTable(DisplayM ? DisplayM : &M), VMap(VMap),
- Finder(), Filename(Filename), Directory(Directory), FileNode(nullptr),
- LexicalBlockFileNode(nullptr), CUNode(nullptr) {
- Finder.processModule(M);
- visit(&M);
- }
-
- ~DIUpdater() { Builder.finalize(); }
-
- void visitModule(Module &M) {
- if (Finder.compile_unit_count() > 1)
- report_fatal_error("DebugIR pass supports only a signle compile unit per "
- "Module.");
- createCompileUnit(Finder.compile_unit_count() == 1 ?
- (MDNode*)*Finder.compile_units().begin() : nullptr);
- }
-
- void visitFunction(Function &F) {
- if (F.isDeclaration() || findDISubprogram(&F))
- return;
-
- StringRef MangledName = F.getName();
- DICompositeType Sig = createFunctionSignature(&F);
-
- // find line of function declaration
- unsigned Line = 0;
- if (!findLine(&F, Line)) {
- DEBUG(dbgs() << "WARNING: No line for Function " << F.getName().str()
- << "\n");
- return;
- }
-
- Instruction *FirstInst = F.begin()->begin();
- unsigned ScopeLine = 0;
- if (!findLine(FirstInst, ScopeLine)) {
- DEBUG(dbgs() << "WARNING: No line for 1st Instruction in Function "
- << F.getName().str() << "\n");
- return;
- }
-
- bool Local = F.hasInternalLinkage();
- bool IsDefinition = !F.isDeclaration();
- bool IsOptimized = false;
-
- int FuncFlags = llvm::DIDescriptor::FlagPrototyped;
- assert(CUNode && FileNode);
- DISubprogram Sub = Builder.createFunction(
- DICompileUnit(CUNode), F.getName(), MangledName, DIFile(FileNode), Line,
- Sig, Local, IsDefinition, ScopeLine, FuncFlags, IsOptimized, &F);
- assert(Sub.isSubprogram());
- DEBUG(dbgs() << "create subprogram mdnode " << *Sub << ": "
- << "\n");
-
- SubprogramDescriptors.insert(std::make_pair(&F, Sub));
- }
-
- void visitInstruction(Instruction &I) {
- DebugLoc Loc(I.getDebugLoc());
-
- /// If a ValueToValueMap is provided, use it to get the real instruction as
- /// the line table was generated on a clone of the module on which we are
- /// operating.
- Value *RealInst = nullptr;
- if (VMap)
- RealInst = VMap->lookup(&I);
-
- if (!RealInst)
- RealInst = &I;
-
- unsigned Col = 0; // FIXME: support columns
- unsigned Line;
- if (!LineTable.getLine(RealInst, Line)) {
- // Instruction has no line, it may have been removed (in the module that
- // will be passed to the debugger) so there is nothing to do here.
- DEBUG(dbgs() << "WARNING: no LineTable entry for instruction " << RealInst
- << "\n");
- DEBUG(RealInst->dump());
- return;
- }
-
- DebugLoc NewLoc;
- if (!Loc.isUnknown())
- // I had a previous debug location: re-use the DebugLoc
- NewLoc = DebugLoc::get(Line, Col, Loc.getScope(RealInst->getContext()),
- Loc.getInlinedAt(RealInst->getContext()));
- else if (MDNode *scope = findScope(&I))
- NewLoc = DebugLoc::get(Line, Col, scope, nullptr);
- else {
- DEBUG(dbgs() << "WARNING: no valid scope for instruction " << &I
- << ". no DebugLoc will be present."
- << "\n");
- return;
- }
-
- addDebugLocation(I, NewLoc);
- }
-
-private:
-
- void createCompileUnit(MDNode *CUToReplace) {
- std::string Flags;
- bool IsOptimized = false;
- StringRef Producer;
- unsigned RuntimeVersion(0);
- StringRef SplitName;
-
- if (CUToReplace) {
- // save fields from existing CU to re-use in the new CU
- DICompileUnit ExistingCU(CUToReplace);
- Producer = ExistingCU.getProducer();
- IsOptimized = ExistingCU.isOptimized();
- Flags = ExistingCU.getFlags();
- RuntimeVersion = ExistingCU.getRunTimeVersion();
- SplitName = ExistingCU.getSplitDebugFilename();
- } else {
- Producer =
- "LLVM Version " STR(LLVM_VERSION_MAJOR) "." STR(LLVM_VERSION_MINOR);
- }
-
- CUNode =
- Builder.createCompileUnit(dwarf::DW_LANG_C99, Filename, Directory,
- Producer, IsOptimized, Flags, RuntimeVersion);
-
- if (CUToReplace)
- CUToReplace->replaceAllUsesWith(const_cast<MDNode *>(CUNode));
-
- DICompileUnit CU(CUNode);
- FileNode = Builder.createFile(Filename, Directory);
- LexicalBlockFileNode = Builder.createLexicalBlockFile(CU, DIFile(FileNode));
- }
-
- /// Returns the MDNode* that represents the DI scope to associate with I
- MDNode *findScope(const Instruction *I) {
- const Function *F = I->getParent()->getParent();
- if (MDNode *ret = findDISubprogram(F))
- return ret;
-
- DEBUG(dbgs() << "WARNING: Using fallback lexical block file scope "
- << LexicalBlockFileNode << " as scope for instruction " << I
- << "\n");
- return LexicalBlockFileNode;
- }
-
- /// Returns the MDNode* that is the descriptor for F
- MDNode *findDISubprogram(const Function *F) {
- typedef ValueMap<const Function *, MDNode *>::const_iterator FuncNodeIter;
- FuncNodeIter i = SubprogramDescriptors.find(F);
- if (i != SubprogramDescriptors.end())
- return i->second;
-
- DEBUG(dbgs() << "searching for DI scope node for Function " << F
- << " in a list of " << Finder.subprogram_count()
- << " subprogram nodes"
- << "\n");
-
- for (DISubprogram S : Finder.subprograms()) {
- if (S.getFunction() == F) {
- DEBUG(dbgs() << "Found DISubprogram " << S << " for function "
- << S.getFunction() << "\n");
- return S;
- }
- }
- DEBUG(dbgs() << "unable to find DISubprogram node for function "
- << F->getName().str() << "\n");
- return nullptr;
- }
-
- /// Sets Line to the line number on which V appears and returns true. If a
- /// line location for V is not found, returns false.
- bool findLine(const Value *V, unsigned &Line) {
- if (LineTable.getLine(V, Line))
- return true;
-
- if (VMap) {
- Value *mapped = VMap->lookup(V);
- if (mapped && LineTable.getLine(mapped, Line))
- return true;
- }
- return false;
- }
-
- std::string getTypeName(Type *T) {
- std::string TypeName;
- raw_string_ostream TypeStream(TypeName);
- if (T)
- T->print(TypeStream);
- else
- TypeStream << "Printing <null> Type";
- TypeStream.flush();
- return TypeName;
- }
-
- /// Returns the MDNode that represents type T if it is already created, or 0
- /// if it is not.
- MDNode *getType(const Type *T) {
- typedef DenseMap<const Type *, MDNode *>::const_iterator TypeNodeIter;
- TypeNodeIter i = TypeDescriptors.find(T);
- if (i != TypeDescriptors.end())
- return i->second;
- return nullptr;
- }
-
- /// Returns a DebugInfo type from an LLVM type T.
- DIDerivedType getOrCreateType(Type *T) {
- MDNode *N = getType(T);
- if (N)
- return DIDerivedType(N);
- else if (T->isVoidTy())
- return DIDerivedType(nullptr);
- else if (T->isStructTy()) {
- N = Builder.createStructType(
- DIScope(LexicalBlockFileNode), T->getStructName(), DIFile(FileNode),
- 0, Layout.getTypeSizeInBits(T), Layout.getABITypeAlignment(T), 0,
- DIType(nullptr), DIArray(nullptr)); // filled in later
-
- // N is added to the map (early) so that element search below can find it,
- // so as to avoid infinite recursion for structs that contain pointers to
- // their own type.
- TypeDescriptors[T] = N;
- DICompositeType StructDescriptor(N);
-
- SmallVector<Value *, 4> Elements;
- for (unsigned i = 0; i < T->getStructNumElements(); ++i)
- Elements.push_back(getOrCreateType(T->getStructElementType(i)));
-
- // set struct elements
- StructDescriptor.setTypeArray(Builder.getOrCreateArray(Elements));
- } else if (T->isPointerTy()) {
- Type *PointeeTy = T->getPointerElementType();
- if (!(N = getType(PointeeTy)))
- N = Builder.createPointerType(
- getOrCreateType(PointeeTy), Layout.getPointerTypeSizeInBits(T),
- Layout.getPrefTypeAlignment(T), getTypeName(T));
- } else if (T->isArrayTy()) {
- SmallVector<Value *, 1> Subrange;
- Subrange.push_back(
- Builder.getOrCreateSubrange(0, T->getArrayNumElements() - 1));
-
- N = Builder.createArrayType(Layout.getTypeSizeInBits(T),
- Layout.getPrefTypeAlignment(T),
- getOrCreateType(T->getArrayElementType()),
- Builder.getOrCreateArray(Subrange));
- } else {
- int encoding = llvm::dwarf::DW_ATE_signed;
- if (T->isIntegerTy())
- encoding = llvm::dwarf::DW_ATE_unsigned;
- else if (T->isFloatingPointTy())
- encoding = llvm::dwarf::DW_ATE_float;
-
- N = Builder.createBasicType(getTypeName(T), T->getPrimitiveSizeInBits(),
- 0, encoding);
- }
- TypeDescriptors[T] = N;
- return DIDerivedType(N);
- }
-
- /// Returns a DebugInfo type that represents a function signature for Func.
- DICompositeType createFunctionSignature(const Function *Func) {
- SmallVector<Value *, 4> Params;
- DIDerivedType ReturnType(getOrCreateType(Func->getReturnType()));
- Params.push_back(ReturnType);
-
- const Function::ArgumentListType &Args(Func->getArgumentList());
- for (Function::ArgumentListType::const_iterator i = Args.begin(),
- e = Args.end();
- i != e; ++i) {
- Type *T(i->getType());
- Params.push_back(getOrCreateType(T));
- }
-
- DIArray ParamArray = Builder.getOrCreateArray(Params);
- return Builder.createSubroutineType(DIFile(FileNode), ParamArray);
- }
-
- /// Associates Instruction I with debug location Loc.
- void addDebugLocation(Instruction &I, DebugLoc Loc) {
- MDNode *MD = Loc.getAsMDNode(I.getContext());
- I.setMetadata(LLVMContext::MD_dbg, MD);
- }
-};
-
-/// Sets Filename/Directory from the Module identifier and returns true, or
-/// false if source information is not present.
-bool getSourceInfoFromModule(const Module &M, std::string &Directory,
- std::string &Filename) {
- std::string PathStr(M.getModuleIdentifier());
- if (PathStr.length() == 0 || PathStr == "<stdin>")
- return false;
-
- Filename = sys::path::filename(PathStr);
- SmallVector<char, 16> Path(PathStr.begin(), PathStr.end());
- sys::path::remove_filename(Path);
- Directory = StringRef(Path.data(), Path.size());
- return true;
-}
-
-// Sets Filename/Directory from debug information in M and returns true, or
-// false if no debug information available, or cannot be parsed.
-bool getSourceInfoFromDI(const Module &M, std::string &Directory,
- std::string &Filename) {
- NamedMDNode *CUNode = M.getNamedMetadata("llvm.dbg.cu");
- if (!CUNode || CUNode->getNumOperands() == 0)
- return false;
-
- DICompileUnit CU(CUNode->getOperand(0));
- if (!CU.Verify())
- return false;
-
- Filename = CU.getFilename();
- Directory = CU.getDirectory();
- return true;
-}
-
-} // anonymous namespace
-
-namespace llvm {
-
-bool DebugIR::getSourceInfo(const Module &M) {
- ParsedPath = getSourceInfoFromDI(M, Directory, Filename) ||
- getSourceInfoFromModule(M, Directory, Filename);
- return ParsedPath;
-}
-
-bool DebugIR::updateExtension(StringRef NewExtension) {
- size_t dot = Filename.find_last_of(".");
- if (dot == std::string::npos)
- return false;
-
- Filename.erase(dot);
- Filename += NewExtension.str();
- return true;
-}
-
-void DebugIR::generateFilename(std::unique_ptr<int> &fd) {
- SmallVector<char, 16> PathVec;
- fd.reset(new int);
- sys::fs::createTemporaryFile("debug-ir", "ll", *fd, PathVec);
- StringRef Path(PathVec.data(), PathVec.size());
- Filename = sys::path::filename(Path);
- sys::path::remove_filename(PathVec);
- Directory = StringRef(PathVec.data(), PathVec.size());
-
- GeneratedPath = true;
-}
-
-std::string DebugIR::getPath() {
- SmallVector<char, 16> Path;
- sys::path::append(Path, Directory, Filename);
- Path.resize(Filename.size() + Directory.size() + 2);
- Path[Filename.size() + Directory.size() + 1] = '\0';
- return std::string(Path.data());
-}
-
-void DebugIR::writeDebugBitcode(const Module *M, int *fd) {
- std::unique_ptr<raw_fd_ostream> Out;
- std::string error;
-
- if (!fd) {
- std::string Path = getPath();
- Out.reset(new raw_fd_ostream(Path.c_str(), error, sys::fs::F_Text));
- DEBUG(dbgs() << "WRITING debug bitcode from Module " << M << " to file "
- << Path << "\n");
- } else {
- DEBUG(dbgs() << "WRITING debug bitcode from Module " << M << " to fd "
- << *fd << "\n");
- Out.reset(new raw_fd_ostream(*fd, true));
- }
-
- M->print(*Out, nullptr);
- Out->close();
-}
-
-void DebugIR::createDebugInfo(Module &M, std::unique_ptr<Module> &DisplayM) {
- if (M.getFunctionList().size() == 0)
- // no functions -- no debug info needed
- return;
-
- std::unique_ptr<ValueToValueMapTy> VMap;
-
- if (WriteSourceToDisk && (HideDebugIntrinsics || HideDebugMetadata)) {
- VMap.reset(new ValueToValueMapTy);
- DisplayM.reset(CloneModule(&M, *VMap));
-
- if (HideDebugIntrinsics)
- DebugIntrinsicsRemover::process(*DisplayM);
-
- if (HideDebugMetadata)
- DebugMetadataRemover::process(*DisplayM);
- }
-
- DIUpdater R(M, Filename, Directory, DisplayM.get(), VMap.get());
-}
-
-bool DebugIR::isMissingPath() { return Filename.empty() || Directory.empty(); }
-
-bool DebugIR::runOnModule(Module &M) {
- std::unique_ptr<int> fd;
-
- if (isMissingPath() && !getSourceInfo(M)) {
- if (!WriteSourceToDisk)
- report_fatal_error("DebugIR unable to determine file name in input. "
- "Ensure Module contains an identifier, a valid "
- "DICompileUnit, or construct DebugIR with "
- "non-empty Filename/Directory parameters.");
- else
- generateFilename(fd);
- }
-
- if (!GeneratedPath && WriteSourceToDisk)
- updateExtension(".debug-ll");
-
- // Clear line numbers. Keep debug info (if any) if we were able to read the
- // file name from the DICompileUnit descriptor.
- DebugMetadataRemover::process(M, !ParsedPath);
-
- std::unique_ptr<Module> DisplayM;
- createDebugInfo(M, DisplayM);
- if (WriteSourceToDisk) {
- Module *OutputM = DisplayM.get() ? DisplayM.get() : &M;
- writeDebugBitcode(OutputM, fd.get());
- }
-
- DEBUG(M.dump());
- return true;
-}
-
-bool DebugIR::runOnModule(Module &M, std::string &Path) {
- bool result = runOnModule(M);
- Path = getPath();
- return result;
-}
-
-} // llvm namespace
-
-char DebugIR::ID = 0;
-INITIALIZE_PASS(DebugIR, "debug-ir", "Enable debugging IR", false, false)
-
-ModulePass *llvm::createDebugIRPass(bool HideDebugIntrinsics,
- bool HideDebugMetadata, StringRef Directory,
- StringRef Filename) {
- return new DebugIR(HideDebugIntrinsics, HideDebugMetadata, Directory,
- Filename);
-}
-
-ModulePass *llvm::createDebugIRPass() { return new DebugIR(); }
diff --git a/lib/Transforms/Instrumentation/DebugIR.h b/lib/Transforms/Instrumentation/DebugIR.h
deleted file mode 100644
index 02831eda2d9f..000000000000
--- a/lib/Transforms/Instrumentation/DebugIR.h
+++ /dev/null
@@ -1,98 +0,0 @@
-//===- llvm/Transforms/Instrumentation/DebugIR.h - Interface ----*- C++ -*-===//
-//
-// The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
-//
-//===----------------------------------------------------------------------===//
-//
-// This file defines the interface of the DebugIR pass. For most users,
-// including Instrumentation.h and calling createDebugIRPass() is sufficient and
-// there is no need to include this file.
-//
-//===----------------------------------------------------------------------===//
-
-#ifndef LLVM_TRANSFORMS_INSTRUMENTATION_DEBUGIR_H
-#define LLVM_TRANSFORMS_INSTRUMENTATION_DEBUGIR_H
-
-#include "llvm/Pass.h"
-
-namespace llvm {
-
-class DebugIR : public llvm::ModulePass {
- /// If true, write a source file to disk.
- bool WriteSourceToDisk;
-
- /// Hide certain (non-essential) debug information (only relevant if
- /// createSource is true.
- bool HideDebugIntrinsics;
- bool HideDebugMetadata;
-
- /// The location of the source file.
- std::string Directory;
- std::string Filename;
-
- /// True if a temporary file name was generated.
- bool GeneratedPath;
-
- /// True if the file name was read from the Module.
- bool ParsedPath;
-
-public:
- static char ID;
-
- const char *getPassName() const override { return "DebugIR"; }
-
- /// Generate a file on disk to be displayed in a debugger. If Filename and
- /// Directory are empty, a temporary path will be generated.
- DebugIR(bool HideDebugIntrinsics, bool HideDebugMetadata,
- llvm::StringRef Directory, llvm::StringRef Filename)
- : ModulePass(ID), WriteSourceToDisk(true),
- HideDebugIntrinsics(HideDebugIntrinsics),
- HideDebugMetadata(HideDebugMetadata), Directory(Directory),
- Filename(Filename), GeneratedPath(false), ParsedPath(false) {}
-
- /// Modify input in-place; do not generate additional files, and do not hide
- /// any debug intrinsics/metadata that might be present.
- DebugIR()
- : ModulePass(ID), WriteSourceToDisk(false), HideDebugIntrinsics(false),
- HideDebugMetadata(false), GeneratedPath(false), ParsedPath(false) {}
-
- /// Run pass on M and set Path to the source file path in the output module.
- bool runOnModule(llvm::Module &M, std::string &Path);
- bool runOnModule(llvm::Module &M) override;
-
-private:
-
- /// Returns the concatenated Directory + Filename, without error checking
- std::string getPath();
-
- /// Attempts to read source information from debug information in M, and if
- /// that fails, from M's identifier. Returns true on success, false otherwise.
- bool getSourceInfo(const llvm::Module &M);
-
- /// Replace the extension of Filename with NewExtension, and return true if
- /// successful. Return false if extension could not be found or Filename is
- /// empty.
- bool updateExtension(llvm::StringRef NewExtension);
-
- /// Generate a temporary filename and open an fd
- void generateFilename(std::unique_ptr<int> &fd);
-
- /// Creates DWARF CU/Subroutine metadata
- void createDebugInfo(llvm::Module &M,
- std::unique_ptr<llvm::Module> &DisplayM);
-
- /// Returns true if either Directory or Filename is missing, false otherwise.
- bool isMissingPath();
-
- /// Write M to disk, optionally passing in an fd to an open file which is
- /// closed by this function after writing. If no fd is specified, a new file
- /// is opened, written, and closed.
- void writeDebugBitcode(const llvm::Module *M, int *fd = nullptr);
-};
-
-} // llvm namespace
-
-#endif // LLVM_TRANSFORMS_INSTRUMENTATION_DEBUGIR_H
diff --git a/lib/Transforms/Instrumentation/GCOVProfiling.cpp b/lib/Transforms/Instrumentation/GCOVProfiling.cpp
index cfeb62eb1f9f..cb965fb9a225 100644
--- a/lib/Transforms/Instrumentation/GCOVProfiling.cpp
+++ b/lib/Transforms/Instrumentation/GCOVProfiling.cpp
@@ -285,6 +285,14 @@ namespace {
DeleteContainerSeconds(LinesByFile);
}
+ GCOVBlock(const GCOVBlock &RHS) : GCOVRecord(RHS), Number(RHS.Number) {
+ // Only allow copy before edges and lines have been added. After that,
+ // there are inter-block pointers (eg: edges) that won't take kindly to
+ // blocks being copied or moved around.
+ assert(LinesByFile.empty());
+ assert(OutEdges.empty());
+ }
+
private:
friend class GCOVFunction;
@@ -303,18 +311,22 @@ namespace {
// object users can construct, the blocks and lines will be rooted here.
class GCOVFunction : public GCOVRecord {
public:
- GCOVFunction(DISubprogram SP, raw_ostream *os, uint32_t Ident,
- bool UseCfgChecksum) :
- SP(SP), Ident(Ident), UseCfgChecksum(UseCfgChecksum), CfgChecksum(0) {
+ GCOVFunction(DISubprogram SP, raw_ostream *os, uint32_t Ident,
+ bool UseCfgChecksum)
+ : SP(SP), Ident(Ident), UseCfgChecksum(UseCfgChecksum), CfgChecksum(0),
+ ReturnBlock(1, os) {
this->os = os;
Function *F = SP.getFunction();
DEBUG(dbgs() << "Function: " << getFunctionName(SP) << "\n");
+
uint32_t i = 0;
- for (Function::iterator BB = F->begin(), E = F->end(); BB != E; ++BB) {
- Blocks[BB] = new GCOVBlock(i++, os);
+ for (auto &BB : *F) {
+ // Skip index 1 (0, 2, 3, 4, ...) because that's assigned to the
+ // ReturnBlock.
+ bool first = i == 0;
+ Blocks.insert(std::make_pair(&BB, GCOVBlock(i++ + !first, os)));
}
- ReturnBlock = new GCOVBlock(i++, os);
std::string FunctionNameAndLine;
raw_string_ostream FNLOS(FunctionNameAndLine);
@@ -323,17 +335,12 @@ namespace {
FuncChecksum = hash_value(FunctionNameAndLine);
}
- ~GCOVFunction() {
- DeleteContainerSeconds(Blocks);
- delete ReturnBlock;
- }
-
GCOVBlock &getBlock(BasicBlock *BB) {
- return *Blocks[BB];
+ return Blocks.find(BB)->second;
}
GCOVBlock &getReturnBlock() {
- return *ReturnBlock;
+ return ReturnBlock;
}
std::string getEdgeDestinations() {
@@ -341,7 +348,7 @@ namespace {
raw_string_ostream EDOS(EdgeDestinations);
Function *F = Blocks.begin()->first->getParent();
for (Function::iterator I = F->begin(), E = F->end(); I != E; ++I) {
- GCOVBlock &Block = *Blocks[I];
+ GCOVBlock &Block = getBlock(I);
for (int i = 0, e = Block.OutEdges.size(); i != e; ++i)
EDOS << Block.OutEdges[i]->Number;
}
@@ -383,7 +390,7 @@ namespace {
if (Blocks.empty()) return;
Function *F = Blocks.begin()->first->getParent();
for (Function::iterator I = F->begin(), E = F->end(); I != E; ++I) {
- GCOVBlock &Block = *Blocks[I];
+ GCOVBlock &Block = getBlock(I);
if (Block.OutEdges.empty()) continue;
writeBytes(EdgeTag, 4);
@@ -399,7 +406,7 @@ namespace {
// Emit lines for each block.
for (Function::iterator I = F->begin(), E = F->end(); I != E; ++I) {
- Blocks[I]->writeOut();
+ getBlock(I).writeOut();
}
}
@@ -409,8 +416,8 @@ namespace {
uint32_t FuncChecksum;
bool UseCfgChecksum;
uint32_t CfgChecksum;
- DenseMap<BasicBlock *, GCOVBlock *> Blocks;
- GCOVBlock *ReturnBlock;
+ DenseMap<BasicBlock *, GCOVBlock> Blocks;
+ GCOVBlock ReturnBlock;
};
}
@@ -480,12 +487,12 @@ void GCOVProfiler::emitProfileNotes() {
// LTO, we'll generate the same .gcno files.
DICompileUnit CU(CU_Nodes->getOperand(i));
- std::string ErrorInfo;
- raw_fd_ostream out(mangleName(CU, "gcno").c_str(), ErrorInfo,
- sys::fs::F_None);
+ std::error_code EC;
+ raw_fd_ostream out(mangleName(CU, "gcno"), EC, sys::fs::F_None);
std::string EdgeDestinations;
DIArray SPs = CU.getSubprograms();
+ unsigned FunctionIdent = 0;
for (unsigned i = 0, e = SPs.getNumElements(); i != e; ++i) {
DISubprogram SP(SPs.getElement(i));
assert((!SP || SP.isSubprogram()) &&
@@ -505,8 +512,8 @@ void GCOVProfiler::emitProfileNotes() {
++It;
EntryBlock.splitBasicBlock(It);
- Funcs.push_back(
- make_unique<GCOVFunction>(SP, &out, i, Options.UseCfgChecksum));
+ Funcs.push_back(make_unique<GCOVFunction>(SP, &out, FunctionIdent++,
+ Options.UseCfgChecksum));
GCOVFunction &Func = *Funcs.back();
for (Function::iterator BB = F->begin(), E = F->end(); BB != E; ++BB) {
@@ -738,11 +745,11 @@ GlobalVariable *GCOVProfiler::buildEdgeLookupTable(
Edge += Successors;
}
- ArrayRef<Constant*> V(&EdgeTable[0], TableSize);
GlobalVariable *EdgeTableGV =
new GlobalVariable(
*M, EdgeTableTy, true, GlobalValue::InternalLinkage,
- ConstantArray::get(EdgeTableTy, V),
+ ConstantArray::get(EdgeTableTy,
+ makeArrayRef(&EdgeTable[0],TableSize)),
"__llvm_gcda_edge_table");
EdgeTableGV->setUnnamedAddr(true);
return EdgeTableGV;
diff --git a/lib/Transforms/Instrumentation/InstrProfiling.cpp b/lib/Transforms/Instrumentation/InstrProfiling.cpp
new file mode 100644
index 000000000000..5f73b89e8551
--- /dev/null
+++ b/lib/Transforms/Instrumentation/InstrProfiling.cpp
@@ -0,0 +1,309 @@
+//===-- InstrProfiling.cpp - Frontend instrumentation based profiling -----===//
+//
+// The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+//
+// This pass lowers instrprof_increment intrinsics emitted by a frontend for
+// profiling. It also builds the data structures and initialization code needed
+// for updating execution counts and emitting the profile at runtime.
+//
+//===----------------------------------------------------------------------===//
+
+#include "llvm/Transforms/Instrumentation.h"
+
+#include "llvm/ADT/Triple.h"
+#include "llvm/IR/IRBuilder.h"
+#include "llvm/IR/IntrinsicInst.h"
+#include "llvm/IR/Module.h"
+#include "llvm/Transforms/Utils/ModuleUtils.h"
+
+using namespace llvm;
+
+#define DEBUG_TYPE "instrprof"
+
+namespace {
+
+class InstrProfiling : public ModulePass {
+public:
+ static char ID;
+
+ InstrProfiling() : ModulePass(ID) {}
+
+ InstrProfiling(const InstrProfOptions &Options)
+ : ModulePass(ID), Options(Options) {}
+
+ const char *getPassName() const override {
+ return "Frontend instrumentation-based coverage lowering";
+ }
+
+ bool runOnModule(Module &M) override;
+
+ void getAnalysisUsage(AnalysisUsage &AU) const override {
+ AU.setPreservesCFG();
+ }
+
+private:
+ InstrProfOptions Options;
+ Module *M;
+ DenseMap<GlobalVariable *, GlobalVariable *> RegionCounters;
+ std::vector<Value *> UsedVars;
+
+ bool isMachO() const {
+ return Triple(M->getTargetTriple()).isOSBinFormatMachO();
+ }
+
+ /// Get the section name for the counter variables.
+ StringRef getCountersSection() const {
+ return isMachO() ? "__DATA,__llvm_prf_cnts" : "__llvm_prf_cnts";
+ }
+
+ /// Get the section name for the name variables.
+ StringRef getNameSection() const {
+ return isMachO() ? "__DATA,__llvm_prf_names" : "__llvm_prf_names";
+ }
+
+ /// Get the section name for the profile data variables.
+ StringRef getDataSection() const {
+ return isMachO() ? "__DATA,__llvm_prf_data" : "__llvm_prf_data";
+ }
+
+ /// Replace instrprof_increment with an increment of the appropriate value.
+ void lowerIncrement(InstrProfIncrementInst *Inc);
+
+ /// Get the region counters for an increment, creating them if necessary.
+ ///
+ /// If the counter array doesn't yet exist, the profile data variables
+ /// referring to them will also be created.
+ GlobalVariable *getOrCreateRegionCounters(InstrProfIncrementInst *Inc);
+
+ /// Emit runtime registration functions for each profile data variable.
+ void emitRegistration();
+
+ /// Emit the necessary plumbing to pull in the runtime initialization.
+ void emitRuntimeHook();
+
+ /// Add uses of our data variables and runtime hook.
+ void emitUses();
+
+ /// Create a static initializer for our data, on platforms that need it.
+ void emitInitialization();
+};
+
+} // anonymous namespace
+
+char InstrProfiling::ID = 0;
+INITIALIZE_PASS(InstrProfiling, "instrprof",
+ "Frontend instrumentation-based coverage lowering.", false,
+ false)
+
+ModulePass *llvm::createInstrProfilingPass(const InstrProfOptions &Options) {
+ return new InstrProfiling(Options);
+}
+
+bool InstrProfiling::runOnModule(Module &M) {
+ bool MadeChange = false;
+
+ this->M = &M;
+ RegionCounters.clear();
+ UsedVars.clear();
+
+ for (Function &F : M)
+ for (BasicBlock &BB : F)
+ for (auto I = BB.begin(), E = BB.end(); I != E;)
+ if (auto *Inc = dyn_cast<InstrProfIncrementInst>(I++)) {
+ lowerIncrement(Inc);
+ MadeChange = true;
+ }
+ if (!MadeChange)
+ return false;
+
+ emitRegistration();
+ emitRuntimeHook();
+ emitUses();
+ emitInitialization();
+ return true;
+}
+
+void InstrProfiling::lowerIncrement(InstrProfIncrementInst *Inc) {
+ GlobalVariable *Counters = getOrCreateRegionCounters(Inc);
+
+ IRBuilder<> Builder(Inc->getParent(), *Inc);
+ uint64_t Index = Inc->getIndex()->getZExtValue();
+ llvm::Value *Addr = Builder.CreateConstInBoundsGEP2_64(Counters, 0, Index);
+ llvm::Value *Count = Builder.CreateLoad(Addr, "pgocount");
+ Count = Builder.CreateAdd(Count, Builder.getInt64(1));
+ Inc->replaceAllUsesWith(Builder.CreateStore(Count, Addr));
+ Inc->eraseFromParent();
+}
+
+/// Get the name of a profiling variable for a particular function.
+static std::string getVarName(InstrProfIncrementInst *Inc, StringRef VarName) {
+ auto *Arr = cast<ConstantDataArray>(Inc->getName()->getInitializer());
+ StringRef Name = Arr->isCString() ? Arr->getAsCString() : Arr->getAsString();
+ return ("__llvm_profile_" + VarName + "_" + Name).str();
+}
+
+GlobalVariable *
+InstrProfiling::getOrCreateRegionCounters(InstrProfIncrementInst *Inc) {
+ GlobalVariable *Name = Inc->getName();
+ auto It = RegionCounters.find(Name);
+ if (It != RegionCounters.end())
+ return It->second;
+
+ // Move the name variable to the right section.
+ Name->setSection(getNameSection());
+ Name->setAlignment(1);
+
+ uint64_t NumCounters = Inc->getNumCounters()->getZExtValue();
+ LLVMContext &Ctx = M->getContext();
+ ArrayType *CounterTy = ArrayType::get(Type::getInt64Ty(Ctx), NumCounters);
+
+ // Create the counters variable.
+ auto *Counters = new GlobalVariable(*M, CounterTy, false, Name->getLinkage(),
+ Constant::getNullValue(CounterTy),
+ getVarName(Inc, "counters"));
+ Counters->setVisibility(Name->getVisibility());
+ Counters->setSection(getCountersSection());
+ Counters->setAlignment(8);
+
+ RegionCounters[Inc->getName()] = Counters;
+
+ // Create data variable.
+ auto *NameArrayTy = Name->getType()->getPointerElementType();
+ auto *Int32Ty = Type::getInt32Ty(Ctx);
+ auto *Int64Ty = Type::getInt64Ty(Ctx);
+ auto *Int8PtrTy = Type::getInt8PtrTy(Ctx);
+ auto *Int64PtrTy = Type::getInt64PtrTy(Ctx);
+
+ Type *DataTypes[] = {Int32Ty, Int32Ty, Int64Ty, Int8PtrTy, Int64PtrTy};
+ auto *DataTy = StructType::get(Ctx, makeArrayRef(DataTypes));
+ Constant *DataVals[] = {
+ ConstantInt::get(Int32Ty, NameArrayTy->getArrayNumElements()),
+ ConstantInt::get(Int32Ty, NumCounters),
+ ConstantInt::get(Int64Ty, Inc->getHash()->getZExtValue()),
+ ConstantExpr::getBitCast(Name, Int8PtrTy),
+ ConstantExpr::getBitCast(Counters, Int64PtrTy)};
+ auto *Data = new GlobalVariable(*M, DataTy, true, Name->getLinkage(),
+ ConstantStruct::get(DataTy, DataVals),
+ getVarName(Inc, "data"));
+ Data->setVisibility(Name->getVisibility());
+ Data->setSection(getDataSection());
+ Data->setAlignment(8);
+
+ // Mark the data variable as used so that it isn't stripped out.
+ UsedVars.push_back(Data);
+
+ return Counters;
+}
+
+void InstrProfiling::emitRegistration() {
+ // Don't do this for Darwin. compiler-rt uses linker magic.
+ if (Triple(M->getTargetTriple()).isOSDarwin())
+ return;
+
+ // Construct the function.
+ auto *VoidTy = Type::getVoidTy(M->getContext());
+ auto *VoidPtrTy = Type::getInt8PtrTy(M->getContext());
+ auto *RegisterFTy = FunctionType::get(VoidTy, false);
+ auto *RegisterF = Function::Create(RegisterFTy, GlobalValue::InternalLinkage,
+ "__llvm_profile_register_functions", M);
+ RegisterF->setUnnamedAddr(true);
+ if (Options.NoRedZone)
+ RegisterF->addFnAttr(Attribute::NoRedZone);
+
+ auto *RuntimeRegisterTy = llvm::FunctionType::get(VoidTy, VoidPtrTy, false);
+ auto *RuntimeRegisterF =
+ Function::Create(RuntimeRegisterTy, GlobalVariable::ExternalLinkage,
+ "__llvm_profile_register_function", M);
+
+ IRBuilder<> IRB(BasicBlock::Create(M->getContext(), "", RegisterF));
+ for (Value *Data : UsedVars)
+ IRB.CreateCall(RuntimeRegisterF, IRB.CreateBitCast(Data, VoidPtrTy));
+ IRB.CreateRetVoid();
+}
+
+void InstrProfiling::emitRuntimeHook() {
+ const char *const RuntimeVarName = "__llvm_profile_runtime";
+ const char *const RuntimeUserName = "__llvm_profile_runtime_user";
+
+ // If the module's provided its own runtime, we don't need to do anything.
+ if (M->getGlobalVariable(RuntimeVarName))
+ return;
+
+ // Declare an external variable that will pull in the runtime initialization.
+ auto *Int32Ty = Type::getInt32Ty(M->getContext());
+ auto *Var =
+ new GlobalVariable(*M, Int32Ty, false, GlobalValue::ExternalLinkage,
+ nullptr, RuntimeVarName);
+
+ // Make a function that uses it.
+ auto *User =
+ Function::Create(FunctionType::get(Int32Ty, false),
+ GlobalValue::LinkOnceODRLinkage, RuntimeUserName, M);
+ User->addFnAttr(Attribute::NoInline);
+ if (Options.NoRedZone)
+ User->addFnAttr(Attribute::NoRedZone);
+
+ IRBuilder<> IRB(BasicBlock::Create(M->getContext(), "", User));
+ auto *Load = IRB.CreateLoad(Var);
+ IRB.CreateRet(Load);
+
+ // Mark the user variable as used so that it isn't stripped out.
+ UsedVars.push_back(User);
+}
+
+void InstrProfiling::emitUses() {
+ if (UsedVars.empty())
+ return;
+
+ GlobalVariable *LLVMUsed = M->getGlobalVariable("llvm.used");
+ std::vector<Constant*> MergedVars;
+ if (LLVMUsed) {
+ // Collect the existing members of llvm.used.
+ ConstantArray *Inits = cast<ConstantArray>(LLVMUsed->getInitializer());
+ for (unsigned I = 0, E = Inits->getNumOperands(); I != E; ++I)
+ MergedVars.push_back(Inits->getOperand(I));
+ LLVMUsed->eraseFromParent();
+ }
+
+ Type *i8PTy = Type::getInt8PtrTy(M->getContext());
+ // Add uses for our data.
+ for (auto *Value : UsedVars)
+ MergedVars.push_back(
+ ConstantExpr::getBitCast(cast<llvm::Constant>(Value), i8PTy));
+
+ // Recreate llvm.used.
+ ArrayType *ATy = ArrayType::get(i8PTy, MergedVars.size());
+ LLVMUsed = new llvm::GlobalVariable(
+ *M, ATy, false, llvm::GlobalValue::AppendingLinkage,
+ llvm::ConstantArray::get(ATy, MergedVars), "llvm.used");
+
+ LLVMUsed->setSection("llvm.metadata");
+}
+
+void InstrProfiling::emitInitialization() {
+ Constant *RegisterF = M->getFunction("__llvm_profile_register_functions");
+ if (!RegisterF)
+ return;
+
+ // Create the initialization function.
+ auto *VoidTy = Type::getVoidTy(M->getContext());
+ auto *F =
+ Function::Create(FunctionType::get(VoidTy, false),
+ GlobalValue::InternalLinkage, "__llvm_profile_init", M);
+ F->setUnnamedAddr(true);
+ F->addFnAttr(Attribute::NoInline);
+ if (Options.NoRedZone)
+ F->addFnAttr(Attribute::NoRedZone);
+
+ // Add the basic block and the necessary calls.
+ IRBuilder<> IRB(BasicBlock::Create(M->getContext(), "", F));
+ IRB.CreateCall(RegisterF);
+ IRB.CreateRetVoid();
+
+ appendToGlobalCtors(*M, F, 0);
+}
diff --git a/lib/Transforms/Instrumentation/Instrumentation.cpp b/lib/Transforms/Instrumentation/Instrumentation.cpp
index ac1dd43c3ae4..a91fc0ec2a48 100644
--- a/lib/Transforms/Instrumentation/Instrumentation.cpp
+++ b/lib/Transforms/Instrumentation/Instrumentation.cpp
@@ -25,8 +25,10 @@ void llvm::initializeInstrumentation(PassRegistry &Registry) {
initializeAddressSanitizerModulePass(Registry);
initializeBoundsCheckingPass(Registry);
initializeGCOVProfilerPass(Registry);
+ initializeInstrProfilingPass(Registry);
initializeMemorySanitizerPass(Registry);
initializeThreadSanitizerPass(Registry);
+ initializeSanitizerCoverageModulePass(Registry);
initializeDataFlowSanitizerPass(Registry);
}
diff --git a/lib/Transforms/Instrumentation/LLVMBuild.txt b/lib/Transforms/Instrumentation/LLVMBuild.txt
index 99e95dfa375a..59249e6ab443 100644
--- a/lib/Transforms/Instrumentation/LLVMBuild.txt
+++ b/lib/Transforms/Instrumentation/LLVMBuild.txt
@@ -19,4 +19,4 @@
type = Library
name = Instrumentation
parent = Transforms
-required_libraries = Analysis Core Support Target TransformUtils
+required_libraries = Analysis Core MC Support Target TransformUtils
diff --git a/lib/Transforms/Instrumentation/MemorySanitizer.cpp b/lib/Transforms/Instrumentation/MemorySanitizer.cpp
index 57e308c20dba..9f00d3d6c824 100644
--- a/lib/Transforms/Instrumentation/MemorySanitizer.cpp
+++ b/lib/Transforms/Instrumentation/MemorySanitizer.cpp
@@ -120,13 +120,13 @@ using namespace llvm;
#define DEBUG_TYPE "msan"
-static const uint64_t kShadowMask32 = 1ULL << 31;
-static const uint64_t kShadowMask64 = 1ULL << 46;
-static const uint64_t kOriginOffset32 = 1ULL << 30;
-static const uint64_t kOriginOffset64 = 1ULL << 45;
static const unsigned kMinOriginAlignment = 4;
static const unsigned kShadowTLSAlignment = 8;
+// These constants must be kept in sync with the ones in msan.h.
+static const unsigned kParamTLSSize = 800;
+static const unsigned kRetvalTLSSize = 800;
+
// Accesses sizes are powers of two: 1, 2, 4, 8.
static const size_t kNumberOfAccessSizes = 4;
@@ -183,20 +183,73 @@ static cl::opt<int> ClInstrumentationWithCallThreshold(
"inline checks (-1 means never use callbacks)."),
cl::Hidden, cl::init(3500));
-// Experimental. Wraps all indirect calls in the instrumented code with
-// a call to the given function. This is needed to assist the dynamic
-// helper tool (MSanDR) to regain control on transition between instrumented and
-// non-instrumented code.
-static cl::opt<std::string> ClWrapIndirectCalls("msan-wrap-indirect-calls",
- cl::desc("Wrap indirect calls with a given function"),
- cl::Hidden);
-
-static cl::opt<bool> ClWrapIndirectCallsFast("msan-wrap-indirect-calls-fast",
- cl::desc("Do not wrap indirect calls with target in the same module"),
- cl::Hidden, cl::init(true));
+// This is an experiment to enable handling of cases where shadow is a non-zero
+// compile-time constant. For some unexplainable reason they were silently
+// ignored in the instrumentation.
+static cl::opt<bool> ClCheckConstantShadow("msan-check-constant-shadow",
+ cl::desc("Insert checks for constant shadow values"),
+ cl::Hidden, cl::init(false));
namespace {
+// Memory map parameters used in application-to-shadow address calculation.
+// Offset = (Addr & ~AndMask) ^ XorMask
+// Shadow = ShadowBase + Offset
+// Origin = OriginBase + Offset
+struct MemoryMapParams {
+ uint64_t AndMask;
+ uint64_t XorMask;
+ uint64_t ShadowBase;
+ uint64_t OriginBase;
+};
+
+struct PlatformMemoryMapParams {
+ const MemoryMapParams *bits32;
+ const MemoryMapParams *bits64;
+};
+
+// i386 Linux
+static const MemoryMapParams LinuxMemoryMapParams32 = {
+ 0x000080000000, // AndMask
+ 0, // XorMask (not used)
+ 0, // ShadowBase (not used)
+ 0x000040000000, // OriginBase
+};
+
+// x86_64 Linux
+static const MemoryMapParams LinuxMemoryMapParams64 = {
+ 0x400000000000, // AndMask
+ 0, // XorMask (not used)
+ 0, // ShadowBase (not used)
+ 0x200000000000, // OriginBase
+};
+
+// i386 FreeBSD
+static const MemoryMapParams FreeBSDMemoryMapParams32 = {
+ 0x000180000000, // AndMask
+ 0x000040000000, // XorMask
+ 0x000020000000, // ShadowBase
+ 0x000700000000, // OriginBase
+};
+
+// x86_64 FreeBSD
+static const MemoryMapParams FreeBSDMemoryMapParams64 = {
+ 0xc00000000000, // AndMask
+ 0x200000000000, // XorMask
+ 0x100000000000, // ShadowBase
+ 0x380000000000, // OriginBase
+};
+
+static const PlatformMemoryMapParams LinuxMemoryMapParams = {
+ &LinuxMemoryMapParams32,
+ &LinuxMemoryMapParams64,
+};
+
+static const PlatformMemoryMapParams FreeBSDMemoryMapParams = {
+ &FreeBSDMemoryMapParams32,
+ &FreeBSDMemoryMapParams64,
+};
+
/// \brief An instrumentation pass implementing detection of uninitialized
/// reads.
///
@@ -208,8 +261,7 @@ class MemorySanitizer : public FunctionPass {
: FunctionPass(ID),
TrackOrigins(std::max(TrackOrigins, (int)ClTrackOrigins)),
DL(nullptr),
- WarningFn(nullptr),
- WrapIndirectCalls(!ClWrapIndirectCalls.empty()) {}
+ WarningFn(nullptr) {}
const char *getPassName() const override { return "MemorySanitizer"; }
bool runOnFunction(Function &F) override;
bool doInitialization(Module &M) override;
@@ -243,9 +295,6 @@ class MemorySanitizer : public FunctionPass {
/// function.
GlobalVariable *OriginTLS;
- GlobalVariable *MsandrModuleStart;
- GlobalVariable *MsandrModuleEnd;
-
/// \brief The run-time callback to print a warning.
Value *WarningFn;
// These arrays are indexed by log2(AccessSize).
@@ -263,25 +312,15 @@ class MemorySanitizer : public FunctionPass {
/// \brief MSan runtime replacements for memmove, memcpy and memset.
Value *MemmoveFn, *MemcpyFn, *MemsetFn;
- /// \brief Address mask used in application-to-shadow address calculation.
- /// ShadowAddr is computed as ApplicationAddr & ~ShadowMask.
- uint64_t ShadowMask;
- /// \brief Offset of the origin shadow from the "normal" shadow.
- /// OriginAddr is computed as (ShadowAddr + OriginOffset) & ~3ULL
- uint64_t OriginOffset;
- /// \brief Branch weights for error reporting.
+ /// \brief Memory map parameters used in application-to-shadow calculation.
+ const MemoryMapParams *MapParams;
+
MDNode *ColdCallWeights;
/// \brief Branch weights for origin store.
MDNode *OriginStoreWeights;
/// \brief An empty volatile inline asm that prevents callback merge.
InlineAsm *EmptyAsm;
- bool WrapIndirectCalls;
- /// \brief Run-time wrapper for indirect calls.
- Value *IndirectCallWrapperFn;
- // Argument and return type of IndirectCallWrapperFn: void (*f)(void).
- Type *AnyFunctionPtrTy;
-
friend struct MemorySanitizerVisitor;
friend struct VarArgAMD64Helper;
};
@@ -321,7 +360,7 @@ void MemorySanitizer::initializeCallbacks(Module &M) {
// which is not yet implemented.
StringRef WarningFnName = ClKeepGoing ? "__msan_warning"
: "__msan_warning_noreturn";
- WarningFn = M.getOrInsertFunction(WarningFnName, IRB.getVoidTy(), NULL);
+ WarningFn = M.getOrInsertFunction(WarningFnName, IRB.getVoidTy(), nullptr);
for (size_t AccessSizeIndex = 0; AccessSizeIndex < kNumberOfAccessSizes;
AccessSizeIndex++) {
@@ -329,34 +368,35 @@ void MemorySanitizer::initializeCallbacks(Module &M) {
std::string FunctionName = "__msan_maybe_warning_" + itostr(AccessSize);
MaybeWarningFn[AccessSizeIndex] = M.getOrInsertFunction(
FunctionName, IRB.getVoidTy(), IRB.getIntNTy(AccessSize * 8),
- IRB.getInt32Ty(), NULL);
+ IRB.getInt32Ty(), nullptr);
FunctionName = "__msan_maybe_store_origin_" + itostr(AccessSize);
MaybeStoreOriginFn[AccessSizeIndex] = M.getOrInsertFunction(
FunctionName, IRB.getVoidTy(), IRB.getIntNTy(AccessSize * 8),
- IRB.getInt8PtrTy(), IRB.getInt32Ty(), NULL);
+ IRB.getInt8PtrTy(), IRB.getInt32Ty(), nullptr);
}
MsanSetAllocaOrigin4Fn = M.getOrInsertFunction(
"__msan_set_alloca_origin4", IRB.getVoidTy(), IRB.getInt8PtrTy(), IntptrTy,
- IRB.getInt8PtrTy(), IntptrTy, NULL);
- MsanPoisonStackFn = M.getOrInsertFunction(
- "__msan_poison_stack", IRB.getVoidTy(), IRB.getInt8PtrTy(), IntptrTy, NULL);
+ IRB.getInt8PtrTy(), IntptrTy, nullptr);
+ MsanPoisonStackFn =
+ M.getOrInsertFunction("__msan_poison_stack", IRB.getVoidTy(),
+ IRB.getInt8PtrTy(), IntptrTy, nullptr);
MsanChainOriginFn = M.getOrInsertFunction(
- "__msan_chain_origin", IRB.getInt32Ty(), IRB.getInt32Ty(), NULL);
+ "__msan_chain_origin", IRB.getInt32Ty(), IRB.getInt32Ty(), nullptr);
MemmoveFn = M.getOrInsertFunction(
"__msan_memmove", IRB.getInt8PtrTy(), IRB.getInt8PtrTy(),
- IRB.getInt8PtrTy(), IntptrTy, NULL);
+ IRB.getInt8PtrTy(), IntptrTy, nullptr);
MemcpyFn = M.getOrInsertFunction(
"__msan_memcpy", IRB.getInt8PtrTy(), IRB.getInt8PtrTy(), IRB.getInt8PtrTy(),
- IntptrTy, NULL);
+ IntptrTy, nullptr);
MemsetFn = M.getOrInsertFunction(
"__msan_memset", IRB.getInt8PtrTy(), IRB.getInt8PtrTy(), IRB.getInt32Ty(),
- IntptrTy, NULL);
+ IntptrTy, nullptr);
// Create globals.
RetvalTLS = new GlobalVariable(
- M, ArrayType::get(IRB.getInt64Ty(), 8), false,
+ M, ArrayType::get(IRB.getInt64Ty(), kRetvalTLSSize / 8), false,
GlobalVariable::ExternalLinkage, nullptr, "__msan_retval_tls", nullptr,
GlobalVariable::InitialExecTLSModel);
RetvalOriginTLS = new GlobalVariable(
@@ -364,16 +404,16 @@ void MemorySanitizer::initializeCallbacks(Module &M) {
"__msan_retval_origin_tls", nullptr, GlobalVariable::InitialExecTLSModel);
ParamTLS = new GlobalVariable(
- M, ArrayType::get(IRB.getInt64Ty(), 1000), false,
+ M, ArrayType::get(IRB.getInt64Ty(), kParamTLSSize / 8), false,
GlobalVariable::ExternalLinkage, nullptr, "__msan_param_tls", nullptr,
GlobalVariable::InitialExecTLSModel);
ParamOriginTLS = new GlobalVariable(
- M, ArrayType::get(OriginTy, 1000), false, GlobalVariable::ExternalLinkage,
- nullptr, "__msan_param_origin_tls", nullptr,
- GlobalVariable::InitialExecTLSModel);
+ M, ArrayType::get(OriginTy, kParamTLSSize / 4), false,
+ GlobalVariable::ExternalLinkage, nullptr, "__msan_param_origin_tls",
+ nullptr, GlobalVariable::InitialExecTLSModel);
VAArgTLS = new GlobalVariable(
- M, ArrayType::get(IRB.getInt64Ty(), 1000), false,
+ M, ArrayType::get(IRB.getInt64Ty(), kParamTLSSize / 8), false,
GlobalVariable::ExternalLinkage, nullptr, "__msan_va_arg_tls", nullptr,
GlobalVariable::InitialExecTLSModel);
VAArgOverflowSizeTLS = new GlobalVariable(
@@ -388,24 +428,6 @@ void MemorySanitizer::initializeCallbacks(Module &M) {
EmptyAsm = InlineAsm::get(FunctionType::get(IRB.getVoidTy(), false),
StringRef(""), StringRef(""),
/*hasSideEffects=*/true);
-
- if (WrapIndirectCalls) {
- AnyFunctionPtrTy =
- PointerType::getUnqual(FunctionType::get(IRB.getVoidTy(), false));
- IndirectCallWrapperFn = M.getOrInsertFunction(
- ClWrapIndirectCalls, AnyFunctionPtrTy, AnyFunctionPtrTy, NULL);
- }
-
- if (WrapIndirectCalls && ClWrapIndirectCallsFast) {
- MsandrModuleStart = new GlobalVariable(
- M, IRB.getInt32Ty(), false, GlobalValue::ExternalLinkage,
- nullptr, "__executable_start");
- MsandrModuleStart->setVisibility(GlobalVariable::HiddenVisibility);
- MsandrModuleEnd = new GlobalVariable(
- M, IRB.getInt32Ty(), false, GlobalValue::ExternalLinkage,
- nullptr, "_end");
- MsandrModuleEnd->setVisibility(GlobalVariable::HiddenVisibility);
- }
}
/// \brief Module-level initialization.
@@ -417,16 +439,21 @@ bool MemorySanitizer::doInitialization(Module &M) {
report_fatal_error("data layout missing");
DL = &DLP->getDataLayout();
+ Triple TargetTriple(M.getTargetTriple());
+ const PlatformMemoryMapParams *PlatformMapParams;
+ if (TargetTriple.getOS() == Triple::FreeBSD)
+ PlatformMapParams = &FreeBSDMemoryMapParams;
+ else
+ PlatformMapParams = &LinuxMemoryMapParams;
+
C = &(M.getContext());
unsigned PtrSize = DL->getPointerSizeInBits(/* AddressSpace */0);
switch (PtrSize) {
case 64:
- ShadowMask = kShadowMask64;
- OriginOffset = kOriginOffset64;
+ MapParams = PlatformMapParams->bits64;
break;
case 32:
- ShadowMask = kShadowMask32;
- OriginOffset = kOriginOffset32;
+ MapParams = PlatformMapParams->bits32;
break;
default:
report_fatal_error("unsupported pointer size");
@@ -442,7 +469,7 @@ bool MemorySanitizer::doInitialization(Module &M) {
// Insert a call to __msan_init/__msan_track_origins into the module's CTORs.
appendToGlobalCtors(M, cast<Function>(M.getOrInsertFunction(
- "__msan_init", IRB.getVoidTy(), NULL)), 0);
+ "__msan_init", IRB.getVoidTy(), nullptr)), 0);
if (TrackOrigins)
new GlobalVariable(M, IRB.getInt32Ty(), true, GlobalValue::WeakODRLinkage,
@@ -525,7 +552,6 @@ struct MemorySanitizerVisitor : public InstVisitor<MemorySanitizerVisitor> {
};
SmallVector<ShadowOriginAndInsertPoint, 16> InstrumentationList;
SmallVector<Instruction*, 16> StoreList;
- SmallVector<CallSite, 16> IndirectCallList;
MemorySanitizerVisitor(Function &F, MemorySanitizer &MS)
: F(F), MS(MS), VAHelper(CreateVarArgHelper(F, MS, *this)) {
@@ -551,15 +577,18 @@ struct MemorySanitizerVisitor : public InstVisitor<MemorySanitizerVisitor> {
void storeOrigin(IRBuilder<> &IRB, Value *Addr, Value *Shadow, Value *Origin,
unsigned Alignment, bool AsCall) {
+ unsigned OriginAlignment = std::max(kMinOriginAlignment, Alignment);
if (isa<StructType>(Shadow->getType())) {
- IRB.CreateAlignedStore(updateOrigin(Origin, IRB), getOriginPtr(Addr, IRB),
- Alignment);
+ IRB.CreateAlignedStore(updateOrigin(Origin, IRB),
+ getOriginPtr(Addr, IRB, Alignment),
+ OriginAlignment);
} else {
Value *ConvertedShadow = convertToShadowTyNoVec(Shadow, IRB);
// TODO(eugenis): handle non-zero constant shadow by inserting an
// unconditional check (can not simply fail compilation as this could
// be in the dead code).
- if (isa<Constant>(ConvertedShadow)) return;
+ if (!ClCheckConstantShadow)
+ if (isa<Constant>(ConvertedShadow)) return;
unsigned TypeSizeInBits =
MS.DL->getTypeSizeInBits(ConvertedShadow->getType());
unsigned SizeIndex = TypeSizeToSizeIndex(TypeSizeInBits);
@@ -577,7 +606,8 @@ struct MemorySanitizerVisitor : public InstVisitor<MemorySanitizerVisitor> {
Cmp, IRB.GetInsertPoint(), false, MS.OriginStoreWeights);
IRBuilder<> IRBNew(CheckTerm);
IRBNew.CreateAlignedStore(updateOrigin(Origin, IRBNew),
- getOriginPtr(Addr, IRBNew), Alignment);
+ getOriginPtr(Addr, IRBNew, Alignment),
+ OriginAlignment);
}
}
}
@@ -601,11 +631,9 @@ struct MemorySanitizerVisitor : public InstVisitor<MemorySanitizerVisitor> {
if (SI.isAtomic()) SI.setOrdering(addReleaseOrdering(SI.getOrdering()));
- if (MS.TrackOrigins) {
- unsigned Alignment = std::max(kMinOriginAlignment, SI.getAlignment());
- storeOrigin(IRB, Addr, Shadow, getOrigin(Val), Alignment,
+ if (MS.TrackOrigins)
+ storeOrigin(IRB, Addr, Shadow, getOrigin(Val), SI.getAlignment(),
InstrumentWithCalls);
- }
}
}
@@ -615,8 +643,9 @@ struct MemorySanitizerVisitor : public InstVisitor<MemorySanitizerVisitor> {
DEBUG(dbgs() << " SHAD0 : " << *Shadow << "\n");
Value *ConvertedShadow = convertToShadowTyNoVec(Shadow, IRB);
DEBUG(dbgs() << " SHAD1 : " << *ConvertedShadow << "\n");
- // See the comment in materializeStores().
- if (isa<Constant>(ConvertedShadow)) return;
+ // See the comment in storeOrigin().
+ if (!ClCheckConstantShadow)
+ if (isa<Constant>(ConvertedShadow)) return;
unsigned TypeSizeInBits =
MS.DL->getTypeSizeInBits(ConvertedShadow->getType());
unsigned SizeIndex = TypeSizeToSizeIndex(TypeSizeInBits);
@@ -655,47 +684,6 @@ struct MemorySanitizerVisitor : public InstVisitor<MemorySanitizerVisitor> {
DEBUG(dbgs() << "DONE:\n" << F);
}
- void materializeIndirectCalls() {
- for (auto &CS : IndirectCallList) {
- Instruction *I = CS.getInstruction();
- BasicBlock *B = I->getParent();
- IRBuilder<> IRB(I);
- Value *Fn0 = CS.getCalledValue();
- Value *Fn = IRB.CreateBitCast(Fn0, MS.AnyFunctionPtrTy);
-
- if (ClWrapIndirectCallsFast) {
- // Check that call target is inside this module limits.
- Value *Start =
- IRB.CreateBitCast(MS.MsandrModuleStart, MS.AnyFunctionPtrTy);
- Value *End = IRB.CreateBitCast(MS.MsandrModuleEnd, MS.AnyFunctionPtrTy);
-
- Value *NotInThisModule = IRB.CreateOr(IRB.CreateICmpULT(Fn, Start),
- IRB.CreateICmpUGE(Fn, End));
-
- PHINode *NewFnPhi =
- IRB.CreatePHI(Fn0->getType(), 2, "msandr.indirect_target");
-
- Instruction *CheckTerm = SplitBlockAndInsertIfThen(
- NotInThisModule, NewFnPhi,
- /* Unreachable */ false, MS.ColdCallWeights);
-
- IRB.SetInsertPoint(CheckTerm);
- // Slow path: call wrapper function to possibly transform the call
- // target.
- Value *NewFn = IRB.CreateBitCast(
- IRB.CreateCall(MS.IndirectCallWrapperFn, Fn), Fn0->getType());
-
- NewFnPhi->addIncoming(Fn0, B);
- NewFnPhi->addIncoming(NewFn, dyn_cast<Instruction>(NewFn)->getParent());
- CS.setCalledFunction(NewFnPhi);
- } else {
- Value *NewFn = IRB.CreateBitCast(
- IRB.CreateCall(MS.IndirectCallWrapperFn, Fn), Fn0->getType());
- CS.setCalledFunction(NewFn);
- }
- }
- }
-
/// \brief Add MemorySanitizer instrumentation to a function.
bool runOnFunction() {
MS.initializeCallbacks(*F.getParent());
@@ -738,9 +726,6 @@ struct MemorySanitizerVisitor : public InstVisitor<MemorySanitizerVisitor> {
// Insert shadow value checks.
materializeChecks(InstrumentWithCalls);
- // Wrap indirect calls.
- materializeIndirectCalls();
-
return true;
}
@@ -763,6 +748,10 @@ struct MemorySanitizerVisitor : public InstVisitor<MemorySanitizerVisitor> {
return VectorType::get(IntegerType::get(*MS.C, EltSize),
VT->getNumElements());
}
+ if (ArrayType *AT = dyn_cast<ArrayType>(OrigTy)) {
+ return ArrayType::get(getShadowTy(AT->getElementType()),
+ AT->getNumElements());
+ }
if (StructType *ST = dyn_cast<StructType>(OrigTy)) {
SmallVector<Type*, 4> Elements;
for (unsigned i = 0, n = ST->getNumElements(); i < n; i++)
@@ -790,32 +779,57 @@ struct MemorySanitizerVisitor : public InstVisitor<MemorySanitizerVisitor> {
return IRB.CreateBitCast(V, NoVecTy);
}
+ /// \brief Compute the integer shadow offset that corresponds to a given
+ /// application address.
+ ///
+ /// Offset = (Addr & ~AndMask) ^ XorMask
+ Value *getShadowPtrOffset(Value *Addr, IRBuilder<> &IRB) {
+ uint64_t AndMask = MS.MapParams->AndMask;
+ assert(AndMask != 0 && "AndMask shall be specified");
+ Value *OffsetLong =
+ IRB.CreateAnd(IRB.CreatePointerCast(Addr, MS.IntptrTy),
+ ConstantInt::get(MS.IntptrTy, ~AndMask));
+
+ uint64_t XorMask = MS.MapParams->XorMask;
+ if (XorMask != 0)
+ OffsetLong = IRB.CreateXor(OffsetLong,
+ ConstantInt::get(MS.IntptrTy, XorMask));
+ return OffsetLong;
+ }
+
/// \brief Compute the shadow address that corresponds to a given application
/// address.
///
- /// Shadow = Addr & ~ShadowMask.
+ /// Shadow = ShadowBase + Offset
Value *getShadowPtr(Value *Addr, Type *ShadowTy,
IRBuilder<> &IRB) {
- Value *ShadowLong =
- IRB.CreateAnd(IRB.CreatePointerCast(Addr, MS.IntptrTy),
- ConstantInt::get(MS.IntptrTy, ~MS.ShadowMask));
+ Value *ShadowLong = getShadowPtrOffset(Addr, IRB);
+ uint64_t ShadowBase = MS.MapParams->ShadowBase;
+ if (ShadowBase != 0)
+ ShadowLong =
+ IRB.CreateAdd(ShadowLong,
+ ConstantInt::get(MS.IntptrTy, ShadowBase));
return IRB.CreateIntToPtr(ShadowLong, PointerType::get(ShadowTy, 0));
}
/// \brief Compute the origin address that corresponds to a given application
/// address.
///
- /// OriginAddr = (ShadowAddr + OriginOffset) & ~3ULL
- Value *getOriginPtr(Value *Addr, IRBuilder<> &IRB) {
- Value *ShadowLong =
- IRB.CreateAnd(IRB.CreatePointerCast(Addr, MS.IntptrTy),
- ConstantInt::get(MS.IntptrTy, ~MS.ShadowMask));
- Value *Add =
- IRB.CreateAdd(ShadowLong,
- ConstantInt::get(MS.IntptrTy, MS.OriginOffset));
- Value *SecondAnd =
- IRB.CreateAnd(Add, ConstantInt::get(MS.IntptrTy, ~3ULL));
- return IRB.CreateIntToPtr(SecondAnd, PointerType::get(IRB.getInt32Ty(), 0));
+ /// OriginAddr = (OriginBase + Offset) & ~3ULL
+ Value *getOriginPtr(Value *Addr, IRBuilder<> &IRB, unsigned Alignment) {
+ Value *OriginLong = getShadowPtrOffset(Addr, IRB);
+ uint64_t OriginBase = MS.MapParams->OriginBase;
+ if (OriginBase != 0)
+ OriginLong =
+ IRB.CreateAdd(OriginLong,
+ ConstantInt::get(MS.IntptrTy, OriginBase));
+ if (Alignment < kMinOriginAlignment) {
+ uint64_t Mask = kMinOriginAlignment - 1;
+ OriginLong = IRB.CreateAnd(OriginLong,
+ ConstantInt::get(MS.IntptrTy, ~Mask));
+ }
+ return IRB.CreateIntToPtr(OriginLong,
+ PointerType::get(IRB.getInt32Ty(), 0));
}
/// \brief Compute the shadow address for a given function argument.
@@ -882,11 +896,18 @@ struct MemorySanitizerVisitor : public InstVisitor<MemorySanitizerVisitor> {
assert(ShadowTy);
if (isa<IntegerType>(ShadowTy) || isa<VectorType>(ShadowTy))
return Constant::getAllOnesValue(ShadowTy);
- StructType *ST = cast<StructType>(ShadowTy);
- SmallVector<Constant *, 4> Vals;
- for (unsigned i = 0, n = ST->getNumElements(); i < n; i++)
- Vals.push_back(getPoisonedShadow(ST->getElementType(i)));
- return ConstantStruct::get(ST, Vals);
+ if (ArrayType *AT = dyn_cast<ArrayType>(ShadowTy)) {
+ SmallVector<Constant *, 4> Vals(AT->getNumElements(),
+ getPoisonedShadow(AT->getElementType()));
+ return ConstantArray::get(AT, Vals);
+ }
+ if (StructType *ST = dyn_cast<StructType>(ShadowTy)) {
+ SmallVector<Constant *, 4> Vals;
+ for (unsigned i = 0, n = ST->getNumElements(); i < n; i++)
+ Vals.push_back(getPoisonedShadow(ST->getElementType(i)));
+ return ConstantStruct::get(ST, Vals);
+ }
+ llvm_unreachable("Unexpected shadow type");
}
/// \brief Create a dirty shadow for a given value.
@@ -941,6 +962,7 @@ struct MemorySanitizerVisitor : public InstVisitor<MemorySanitizerVisitor> {
? MS.DL->getTypeAllocSize(FArg.getType()->getPointerElementType())
: MS.DL->getTypeAllocSize(FArg.getType());
if (A == &FArg) {
+ bool Overflow = ArgOffset + Size > kParamTLSSize;
Value *Base = getShadowPtrForArgument(&FArg, EntryIRB, ArgOffset);
if (FArg.hasByValAttr()) {
// ByVal pointer itself has clean shadow. We copy the actual
@@ -951,25 +973,40 @@ struct MemorySanitizerVisitor : public InstVisitor<MemorySanitizerVisitor> {
Type *EltType = A->getType()->getPointerElementType();
ArgAlign = MS.DL->getABITypeAlignment(EltType);
}
- unsigned CopyAlign = std::min(ArgAlign, kShadowTLSAlignment);
- Value *Cpy = EntryIRB.CreateMemCpy(
- getShadowPtr(V, EntryIRB.getInt8Ty(), EntryIRB), Base, Size,
- CopyAlign);
- DEBUG(dbgs() << " ByValCpy: " << *Cpy << "\n");
- (void)Cpy;
+ if (Overflow) {
+ // ParamTLS overflow.
+ EntryIRB.CreateMemSet(
+ getShadowPtr(V, EntryIRB.getInt8Ty(), EntryIRB),
+ Constant::getNullValue(EntryIRB.getInt8Ty()), Size, ArgAlign);
+ } else {
+ unsigned CopyAlign = std::min(ArgAlign, kShadowTLSAlignment);
+ Value *Cpy = EntryIRB.CreateMemCpy(
+ getShadowPtr(V, EntryIRB.getInt8Ty(), EntryIRB), Base, Size,
+ CopyAlign);
+ DEBUG(dbgs() << " ByValCpy: " << *Cpy << "\n");
+ (void)Cpy;
+ }
*ShadowPtr = getCleanShadow(V);
} else {
- *ShadowPtr = EntryIRB.CreateAlignedLoad(Base, kShadowTLSAlignment);
+ if (Overflow) {
+ // ParamTLS overflow.
+ *ShadowPtr = getCleanShadow(V);
+ } else {
+ *ShadowPtr =
+ EntryIRB.CreateAlignedLoad(Base, kShadowTLSAlignment);
+ }
}
DEBUG(dbgs() << " ARG: " << FArg << " ==> " <<
**ShadowPtr << "\n");
- if (MS.TrackOrigins) {
+ if (MS.TrackOrigins && !Overflow) {
Value *OriginPtr =
getOriginPtrForArgument(&FArg, EntryIRB, ArgOffset);
setOrigin(A, EntryIRB.CreateLoad(OriginPtr));
+ } else {
+ setOrigin(A, getCleanOrigin());
}
}
- ArgOffset += DataLayout::RoundUpAlignment(Size, kShadowTLSAlignment);
+ ArgOffset += RoundUpToAlignment(Size, kShadowTLSAlignment);
}
assert(*ShadowPtr && "Could not find shadow for an argument");
return *ShadowPtr;
@@ -986,15 +1023,13 @@ struct MemorySanitizerVisitor : public InstVisitor<MemorySanitizerVisitor> {
/// \brief Get the origin for a value.
Value *getOrigin(Value *V) {
if (!MS.TrackOrigins) return nullptr;
- if (isa<Instruction>(V) || isa<Argument>(V)) {
- Value *Origin = OriginMap[V];
- if (!Origin) {
- DEBUG(dbgs() << "NO ORIGIN: " << *V << "\n");
- Origin = getCleanOrigin();
- }
- return Origin;
- }
- return getCleanOrigin();
+ if (!PropagateShadow) return getCleanOrigin();
+ if (isa<Constant>(V)) return getCleanOrigin();
+ assert((isa<Instruction>(V) || isa<Argument>(V)) &&
+ "Unexpected value type in getOrigin()");
+ Value *Origin = OriginMap[V];
+ assert(Origin && "Missing origin");
+ return Origin;
}
/// \brief Get the origin for i-th argument of the instruction I.
@@ -1024,9 +1059,16 @@ struct MemorySanitizerVisitor : public InstVisitor<MemorySanitizerVisitor> {
/// UMR warning in runtime if the value is not fully defined.
void insertShadowCheck(Value *Val, Instruction *OrigIns) {
assert(Val);
- Instruction *Shadow = dyn_cast_or_null<Instruction>(getShadow(Val));
- if (!Shadow) return;
- Instruction *Origin = dyn_cast_or_null<Instruction>(getOrigin(Val));
+ Value *Shadow, *Origin;
+ if (ClCheckConstantShadow) {
+ Shadow = getShadow(Val);
+ if (!Shadow) return;
+ Origin = getOrigin(Val);
+ } else {
+ Shadow = dyn_cast_or_null<Instruction>(getShadow(Val));
+ if (!Shadow) return;
+ Origin = dyn_cast_or_null<Instruction>(getOrigin(Val));
+ }
insertShadowCheck(Shadow, Origin, OrigIns);
}
@@ -1075,7 +1117,7 @@ struct MemorySanitizerVisitor : public InstVisitor<MemorySanitizerVisitor> {
IRBuilder<> IRB(I.getNextNode());
Type *ShadowTy = getShadowTy(&I);
Value *Addr = I.getPointerOperand();
- if (PropagateShadow) {
+ if (PropagateShadow && !I.getMetadata("nosanitize")) {
Value *ShadowPtr = getShadowPtr(Addr, ShadowTy, IRB);
setShadow(&I,
IRB.CreateAlignedLoad(ShadowPtr, I.getAlignment(), "_msld"));
@@ -1091,9 +1133,10 @@ struct MemorySanitizerVisitor : public InstVisitor<MemorySanitizerVisitor> {
if (MS.TrackOrigins) {
if (PropagateShadow) {
- unsigned Alignment = std::max(kMinOriginAlignment, I.getAlignment());
- setOrigin(&I,
- IRB.CreateAlignedLoad(getOriginPtr(Addr, IRB), Alignment));
+ unsigned Alignment = I.getAlignment();
+ unsigned OriginAlignment = std::max(kMinOriginAlignment, Alignment);
+ setOrigin(&I, IRB.CreateAlignedLoad(getOriginPtr(Addr, IRB, Alignment),
+ OriginAlignment));
} else {
setOrigin(&I, getCleanOrigin());
}
@@ -1127,6 +1170,7 @@ struct MemorySanitizerVisitor : public InstVisitor<MemorySanitizerVisitor> {
IRB.CreateStore(getCleanShadow(&I), ShadowPtr);
setShadow(&I, getCleanShadow(&I));
+ setOrigin(&I, getCleanOrigin());
}
void visitAtomicRMWInst(AtomicRMWInst &I) {
@@ -1744,7 +1788,7 @@ struct MemorySanitizerVisitor : public InstVisitor<MemorySanitizerVisitor> {
// FIXME: use ClStoreCleanOrigin
// FIXME: factor out common code from materializeStores
if (MS.TrackOrigins)
- IRB.CreateStore(getOrigin(&I, 1), getOriginPtr(Addr, IRB));
+ IRB.CreateStore(getOrigin(&I, 1), getOriginPtr(Addr, IRB, 1));
return true;
}
@@ -1771,7 +1815,7 @@ struct MemorySanitizerVisitor : public InstVisitor<MemorySanitizerVisitor> {
if (MS.TrackOrigins) {
if (PropagateShadow)
- setOrigin(&I, IRB.CreateLoad(getOriginPtr(Addr, IRB)));
+ setOrigin(&I, IRB.CreateLoad(getOriginPtr(Addr, IRB, 1)));
else
setOrigin(&I, getCleanOrigin());
}
@@ -1859,7 +1903,7 @@ struct MemorySanitizerVisitor : public InstVisitor<MemorySanitizerVisitor> {
Value *Op = I.getArgOperand(0);
Type *OpType = Op->getType();
Function *BswapFunc = Intrinsic::getDeclaration(
- F.getParent(), Intrinsic::bswap, ArrayRef<Type*>(&OpType, 1));
+ F.getParent(), Intrinsic::bswap, makeArrayRef(&OpType, 1));
setShadow(&I, IRB.CreateCall(BswapFunc, getShadow(Op)));
setOrigin(&I, getOrigin(Op));
}
@@ -1935,6 +1979,7 @@ struct MemorySanitizerVisitor : public InstVisitor<MemorySanitizerVisitor> {
setOrigin(&I, getOrigin(CopyOp));
} else {
setShadow(&I, getCleanShadow(&I));
+ setOrigin(&I, getCleanOrigin());
}
}
@@ -2291,9 +2336,6 @@ struct MemorySanitizerVisitor : public InstVisitor<MemorySanitizerVisitor> {
}
IRBuilder<> IRB(&I);
- if (MS.WrapIndirectCalls && !CS.getCalledFunction())
- IndirectCallList.push_back(CS);
-
unsigned ArgOffset = 0;
DEBUG(dbgs() << " CallSite: " << I << "\n");
for (CallSite::arg_iterator ArgIt = CS.arg_begin(), End = CS.arg_end();
@@ -2318,12 +2360,15 @@ struct MemorySanitizerVisitor : public InstVisitor<MemorySanitizerVisitor> {
assert(A->getType()->isPointerTy() &&
"ByVal argument is not a pointer!");
Size = MS.DL->getTypeAllocSize(A->getType()->getPointerElementType());
- unsigned Alignment = CS.getParamAlignment(i + 1);
+ if (ArgOffset + Size > kParamTLSSize) break;
+ unsigned ParamAlignment = CS.getParamAlignment(i + 1);
+ unsigned Alignment = std::min(ParamAlignment, kShadowTLSAlignment);
Store = IRB.CreateMemCpy(ArgShadowBase,
getShadowPtr(A, Type::getInt8Ty(*MS.C), IRB),
Size, Alignment);
} else {
Size = MS.DL->getTypeAllocSize(A->getType());
+ if (ArgOffset + Size > kParamTLSSize) break;
Store = IRB.CreateAlignedStore(ArgShadow, ArgShadowBase,
kShadowTLSAlignment);
Constant *Cst = dyn_cast<Constant>(ArgShadow);
@@ -2335,7 +2380,7 @@ struct MemorySanitizerVisitor : public InstVisitor<MemorySanitizerVisitor> {
(void)Store;
assert(Size != 0 && Store != nullptr);
DEBUG(dbgs() << " Param:" << *Store << "\n");
- ArgOffset += DataLayout::RoundUpAlignment(Size, 8);
+ ArgOffset += RoundUpToAlignment(Size, 8);
}
DEBUG(dbgs() << " done with call args\n");
@@ -2399,6 +2444,7 @@ struct MemorySanitizerVisitor : public InstVisitor<MemorySanitizerVisitor> {
IRBuilder<> IRB(&I);
if (!PropagateShadow) {
setShadow(&I, getCleanShadow(&I));
+ setOrigin(&I, getCleanOrigin());
return;
}
@@ -2412,6 +2458,7 @@ struct MemorySanitizerVisitor : public InstVisitor<MemorySanitizerVisitor> {
void visitAllocaInst(AllocaInst &I) {
setShadow(&I, getCleanShadow(&I));
+ setOrigin(&I, getCleanOrigin());
IRBuilder<> IRB(I.getNextNode());
uint64_t Size = MS.DL->getTypeAllocSize(I.getAllocatedType());
if (PoisonStack && ClPoisonStackWithCall) {
@@ -2425,7 +2472,6 @@ struct MemorySanitizerVisitor : public InstVisitor<MemorySanitizerVisitor> {
}
if (PoisonStack && MS.TrackOrigins) {
- setOrigin(&I, getCleanOrigin());
SmallString<2048> StackDescriptionStorage;
raw_svector_ostream StackDescription(StackDescriptionStorage);
// We create a string with a description of the stack allocation and
@@ -2491,9 +2537,10 @@ struct MemorySanitizerVisitor : public InstVisitor<MemorySanitizerVisitor> {
}
// a = select b, c, d
// Oa = Sb ? Ob : (b ? Oc : Od)
- setOrigin(&I, IRB.CreateSelect(
- Sb, getOrigin(I.getCondition()),
- IRB.CreateSelect(B, getOrigin(C), getOrigin(D))));
+ setOrigin(
+ &I, IRB.CreateSelect(Sb, getOrigin(I.getCondition()),
+ IRB.CreateSelect(B, getOrigin(I.getTrueValue()),
+ getOrigin(I.getFalseValue()))));
}
}
@@ -2616,7 +2663,7 @@ struct VarArgAMD64Helper : public VarArgHelper {
Type *RealTy = A->getType()->getPointerElementType();
uint64_t ArgSize = MS.DL->getTypeAllocSize(RealTy);
Value *Base = getShadowPtrForVAArgument(RealTy, IRB, OverflowOffset);
- OverflowOffset += DataLayout::RoundUpAlignment(ArgSize, 8);
+ OverflowOffset += RoundUpToAlignment(ArgSize, 8);
IRB.CreateMemCpy(Base, MSV.getShadowPtr(A, IRB.getInt8Ty(), IRB),
ArgSize, kShadowTLSAlignment);
} else {
@@ -2638,7 +2685,7 @@ struct VarArgAMD64Helper : public VarArgHelper {
case AK_Memory:
uint64_t ArgSize = MS.DL->getTypeAllocSize(A->getType());
Base = getShadowPtrForVAArgument(A->getType(), IRB, OverflowOffset);
- OverflowOffset += DataLayout::RoundUpAlignment(ArgSize, 8);
+ OverflowOffset += RoundUpToAlignment(ArgSize, 8);
}
IRB.CreateAlignedStore(MSV.getShadow(A), Base, kShadowTLSAlignment);
}
diff --git a/lib/Transforms/Instrumentation/SanitizerCoverage.cpp b/lib/Transforms/Instrumentation/SanitizerCoverage.cpp
new file mode 100644
index 000000000000..c048a99f8880
--- /dev/null
+++ b/lib/Transforms/Instrumentation/SanitizerCoverage.cpp
@@ -0,0 +1,314 @@
+//===-- SanitizerCoverage.cpp - coverage instrumentation for sanitizers ---===//
+//
+// The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+//
+// Coverage instrumentation that works with AddressSanitizer
+// and potentially with other Sanitizers.
+//
+// We create a Guard variable with the same linkage
+// as the function and inject this code into the entry block (CoverageLevel=1)
+// or all blocks (CoverageLevel>=2):
+// if (Guard < 0) {
+// __sanitizer_cov(&Guard);
+// }
+// The accesses to Guard are atomic. The rest of the logic is
+// in __sanitizer_cov (it's fine to call it more than once).
+//
+// With CoverageLevel>=3 we also split critical edges this effectively
+// instrumenting all edges.
+//
+// CoverageLevel>=4 add indirect call profiling implented as a function call.
+//
+// This coverage implementation provides very limited data:
+// it only tells if a given function (block) was ever executed. No counters.
+// But for many use cases this is what we need and the added slowdown small.
+//
+//===----------------------------------------------------------------------===//
+
+#include "llvm/Transforms/Instrumentation.h"
+#include "llvm/ADT/ArrayRef.h"
+#include "llvm/ADT/SmallVector.h"
+#include "llvm/IR/CallSite.h"
+#include "llvm/IR/DataLayout.h"
+#include "llvm/IR/Function.h"
+#include "llvm/IR/IRBuilder.h"
+#include "llvm/IR/InlineAsm.h"
+#include "llvm/IR/LLVMContext.h"
+#include "llvm/IR/MDBuilder.h"
+#include "llvm/IR/Module.h"
+#include "llvm/IR/Type.h"
+#include "llvm/Support/CommandLine.h"
+#include "llvm/Support/Debug.h"
+#include "llvm/Support/raw_ostream.h"
+#include "llvm/Transforms/Scalar.h"
+#include "llvm/Transforms/Utils/BasicBlockUtils.h"
+#include "llvm/Transforms/Utils/ModuleUtils.h"
+
+using namespace llvm;
+
+#define DEBUG_TYPE "sancov"
+
+static const char *const kSanCovModuleInitName = "__sanitizer_cov_module_init";
+static const char *const kSanCovName = "__sanitizer_cov";
+static const char *const kSanCovIndirCallName = "__sanitizer_cov_indir_call16";
+static const char *const kSanCovTraceEnter = "__sanitizer_cov_trace_func_enter";
+static const char *const kSanCovTraceBB = "__sanitizer_cov_trace_basic_block";
+static const char *const kSanCovModuleCtorName = "sancov.module_ctor";
+static const uint64_t kSanCtorAndDtorPriority = 1;
+
+static cl::opt<int> ClCoverageLevel("sanitizer-coverage-level",
+ cl::desc("Sanitizer Coverage. 0: none, 1: entry block, 2: all blocks, "
+ "3: all blocks and critical edges, "
+ "4: above plus indirect calls"),
+ cl::Hidden, cl::init(0));
+
+static cl::opt<int> ClCoverageBlockThreshold(
+ "sanitizer-coverage-block-threshold",
+ cl::desc("Add coverage instrumentation only to the entry block if there "
+ "are more than this number of blocks."),
+ cl::Hidden, cl::init(1500));
+
+static cl::opt<bool>
+ ClExperimentalTracing("sanitizer-coverage-experimental-tracing",
+ cl::desc("Experimental basic-block tracing: insert "
+ "callbacks at every basic block"),
+ cl::Hidden, cl::init(false));
+
+namespace {
+
+class SanitizerCoverageModule : public ModulePass {
+ public:
+ SanitizerCoverageModule(int CoverageLevel = 0)
+ : ModulePass(ID),
+ CoverageLevel(std::max(CoverageLevel, (int)ClCoverageLevel)) {}
+ bool runOnModule(Module &M) override;
+ bool runOnFunction(Function &F);
+ static char ID; // Pass identification, replacement for typeid
+ const char *getPassName() const override {
+ return "SanitizerCoverageModule";
+ }
+
+ void getAnalysisUsage(AnalysisUsage &AU) const override {
+ AU.addRequired<DataLayoutPass>();
+ }
+
+ private:
+ void InjectCoverageForIndirectCalls(Function &F,
+ ArrayRef<Instruction *> IndirCalls);
+ bool InjectCoverage(Function &F, ArrayRef<BasicBlock *> AllBlocks,
+ ArrayRef<Instruction *> IndirCalls);
+ void InjectCoverageAtBlock(Function &F, BasicBlock &BB);
+ Function *SanCovFunction;
+ Function *SanCovIndirCallFunction;
+ Function *SanCovModuleInit;
+ Function *SanCovTraceEnter, *SanCovTraceBB;
+ InlineAsm *EmptyAsm;
+ Type *IntptrTy;
+ LLVMContext *C;
+
+ GlobalVariable *GuardArray;
+
+ int CoverageLevel;
+};
+
+} // namespace
+
+static Function *checkInterfaceFunction(Constant *FuncOrBitcast) {
+ if (Function *F = dyn_cast<Function>(FuncOrBitcast))
+ return F;
+ std::string Err;
+ raw_string_ostream Stream(Err);
+ Stream << "SanitizerCoverage interface function redefined: "
+ << *FuncOrBitcast;
+ report_fatal_error(Err);
+}
+
+bool SanitizerCoverageModule::runOnModule(Module &M) {
+ if (!CoverageLevel) return false;
+ C = &(M.getContext());
+ DataLayoutPass *DLP = &getAnalysis<DataLayoutPass>();
+ IntptrTy = Type::getIntNTy(*C, DLP->getDataLayout().getPointerSizeInBits());
+ Type *VoidTy = Type::getVoidTy(*C);
+ IRBuilder<> IRB(*C);
+ Type *Int32PtrTy = PointerType::getUnqual(IRB.getInt32Ty());
+
+ Function *CtorFunc =
+ Function::Create(FunctionType::get(VoidTy, false),
+ GlobalValue::InternalLinkage, kSanCovModuleCtorName, &M);
+ ReturnInst::Create(*C, BasicBlock::Create(*C, "", CtorFunc));
+ appendToGlobalCtors(M, CtorFunc, kSanCtorAndDtorPriority);
+
+ SanCovFunction = checkInterfaceFunction(
+ M.getOrInsertFunction(kSanCovName, VoidTy, Int32PtrTy, nullptr));
+ SanCovIndirCallFunction = checkInterfaceFunction(M.getOrInsertFunction(
+ kSanCovIndirCallName, VoidTy, IntptrTy, IntptrTy, nullptr));
+ SanCovModuleInit = checkInterfaceFunction(
+ M.getOrInsertFunction(kSanCovModuleInitName, Type::getVoidTy(*C),
+ Int32PtrTy, IntptrTy, nullptr));
+ SanCovModuleInit->setLinkage(Function::ExternalLinkage);
+ // We insert an empty inline asm after cov callbacks to avoid callback merge.
+ EmptyAsm = InlineAsm::get(FunctionType::get(IRB.getVoidTy(), false),
+ StringRef(""), StringRef(""),
+ /*hasSideEffects=*/true);
+
+ if (ClExperimentalTracing) {
+ SanCovTraceEnter = checkInterfaceFunction(
+ M.getOrInsertFunction(kSanCovTraceEnter, VoidTy, Int32PtrTy, nullptr));
+ SanCovTraceBB = checkInterfaceFunction(
+ M.getOrInsertFunction(kSanCovTraceBB, VoidTy, Int32PtrTy, nullptr));
+ }
+
+ // At this point we create a dummy array of guards because we don't
+ // know how many elements we will need.
+ Type *Int32Ty = IRB.getInt32Ty();
+ GuardArray =
+ new GlobalVariable(M, Int32Ty, false, GlobalValue::ExternalLinkage,
+ nullptr, "__sancov_gen_cov_tmp");
+
+ for (auto &F : M)
+ runOnFunction(F);
+
+ // Now we know how many elements we need. Create an array of guards
+ // with one extra element at the beginning for the size.
+ Type *Int32ArrayNTy =
+ ArrayType::get(Int32Ty, SanCovFunction->getNumUses() + 1);
+ GlobalVariable *RealGuardArray = new GlobalVariable(
+ M, Int32ArrayNTy, false, GlobalValue::PrivateLinkage,
+ Constant::getNullValue(Int32ArrayNTy), "__sancov_gen_cov");
+
+ // Replace the dummy array with the real one.
+ GuardArray->replaceAllUsesWith(
+ IRB.CreatePointerCast(RealGuardArray, Int32PtrTy));
+ GuardArray->eraseFromParent();
+
+ // Call __sanitizer_cov_module_init
+ IRB.SetInsertPoint(CtorFunc->getEntryBlock().getTerminator());
+ IRB.CreateCall2(SanCovModuleInit,
+ IRB.CreatePointerCast(RealGuardArray, Int32PtrTy),
+ ConstantInt::get(IntptrTy, SanCovFunction->getNumUses()));
+ return true;
+}
+
+bool SanitizerCoverageModule::runOnFunction(Function &F) {
+ if (F.empty()) return false;
+ if (F.getName().find(".module_ctor") != std::string::npos)
+ return false; // Should not instrument sanitizer init functions.
+ if (CoverageLevel >= 3)
+ SplitAllCriticalEdges(F, this);
+ SmallVector<Instruction*, 8> IndirCalls;
+ SmallVector<BasicBlock*, 16> AllBlocks;
+ for (auto &BB : F) {
+ AllBlocks.push_back(&BB);
+ if (CoverageLevel >= 4)
+ for (auto &Inst : BB) {
+ CallSite CS(&Inst);
+ if (CS && !CS.getCalledFunction())
+ IndirCalls.push_back(&Inst);
+ }
+ }
+ InjectCoverage(F, AllBlocks, IndirCalls);
+ return true;
+}
+
+bool
+SanitizerCoverageModule::InjectCoverage(Function &F,
+ ArrayRef<BasicBlock *> AllBlocks,
+ ArrayRef<Instruction *> IndirCalls) {
+ if (!CoverageLevel) return false;
+
+ if (CoverageLevel == 1 ||
+ (unsigned)ClCoverageBlockThreshold < AllBlocks.size()) {
+ InjectCoverageAtBlock(F, F.getEntryBlock());
+ } else {
+ for (auto BB : AllBlocks)
+ InjectCoverageAtBlock(F, *BB);
+ }
+ InjectCoverageForIndirectCalls(F, IndirCalls);
+ return true;
+}
+
+// On every indirect call we call a run-time function
+// __sanitizer_cov_indir_call* with two parameters:
+// - callee address,
+// - global cache array that contains kCacheSize pointers (zero-initialized).
+// The cache is used to speed up recording the caller-callee pairs.
+// The address of the caller is passed implicitly via caller PC.
+// kCacheSize is encoded in the name of the run-time function.
+void SanitizerCoverageModule::InjectCoverageForIndirectCalls(
+ Function &F, ArrayRef<Instruction *> IndirCalls) {
+ if (IndirCalls.empty()) return;
+ const int kCacheSize = 16;
+ const int kCacheAlignment = 64; // Align for better performance.
+ Type *Ty = ArrayType::get(IntptrTy, kCacheSize);
+ for (auto I : IndirCalls) {
+ IRBuilder<> IRB(I);
+ CallSite CS(I);
+ Value *Callee = CS.getCalledValue();
+ if (dyn_cast<InlineAsm>(Callee)) continue;
+ GlobalVariable *CalleeCache = new GlobalVariable(
+ *F.getParent(), Ty, false, GlobalValue::PrivateLinkage,
+ Constant::getNullValue(Ty), "__sancov_gen_callee_cache");
+ CalleeCache->setAlignment(kCacheAlignment);
+ IRB.CreateCall2(SanCovIndirCallFunction,
+ IRB.CreatePointerCast(Callee, IntptrTy),
+ IRB.CreatePointerCast(CalleeCache, IntptrTy));
+ }
+}
+
+void SanitizerCoverageModule::InjectCoverageAtBlock(Function &F,
+ BasicBlock &BB) {
+ BasicBlock::iterator IP = BB.getFirstInsertionPt(), BE = BB.end();
+ // Skip static allocas at the top of the entry block so they don't become
+ // dynamic when we split the block. If we used our optimized stack layout,
+ // then there will only be one alloca and it will come first.
+ for (; IP != BE; ++IP) {
+ AllocaInst *AI = dyn_cast<AllocaInst>(IP);
+ if (!AI || !AI->isStaticAlloca())
+ break;
+ }
+
+ bool IsEntryBB = &BB == &F.getEntryBlock();
+ DebugLoc EntryLoc =
+ IsEntryBB ? IP->getDebugLoc().getFnDebugLoc(*C) : IP->getDebugLoc();
+ IRBuilder<> IRB(IP);
+ IRB.SetCurrentDebugLocation(EntryLoc);
+ SmallVector<Value *, 1> Indices;
+ Value *GuardP = IRB.CreateAdd(
+ IRB.CreatePointerCast(GuardArray, IntptrTy),
+ ConstantInt::get(IntptrTy, (1 + SanCovFunction->getNumUses()) * 4));
+ Type *Int32PtrTy = PointerType::getUnqual(IRB.getInt32Ty());
+ GuardP = IRB.CreateIntToPtr(GuardP, Int32PtrTy);
+ LoadInst *Load = IRB.CreateLoad(GuardP);
+ Load->setAtomic(Monotonic);
+ Load->setAlignment(4);
+ Load->setMetadata(F.getParent()->getMDKindID("nosanitize"),
+ MDNode::get(*C, None));
+ Value *Cmp = IRB.CreateICmpSGE(Constant::getNullValue(Load->getType()), Load);
+ Instruction *Ins = SplitBlockAndInsertIfThen(
+ Cmp, IP, false, MDBuilder(*C).createBranchWeights(1, 100000));
+ IRB.SetInsertPoint(Ins);
+ IRB.SetCurrentDebugLocation(EntryLoc);
+ // __sanitizer_cov gets the PC of the instruction using GET_CALLER_PC.
+ IRB.CreateCall(SanCovFunction, GuardP);
+ IRB.CreateCall(EmptyAsm); // Avoids callback merge.
+
+ if (ClExperimentalTracing) {
+ // Experimental support for tracing.
+ // Insert a callback with the same guard variable as used for coverage.
+ IRB.SetInsertPoint(IP);
+ IRB.CreateCall(IsEntryBB ? SanCovTraceEnter : SanCovTraceBB, GuardP);
+ }
+}
+
+char SanitizerCoverageModule::ID = 0;
+INITIALIZE_PASS(SanitizerCoverageModule, "sancov",
+ "SanitizerCoverage: TODO."
+ "ModulePass", false, false)
+ModulePass *llvm::createSanitizerCoverageModulePass(int CoverageLevel) {
+ return new SanitizerCoverageModule(CoverageLevel);
+}
diff --git a/lib/Transforms/Instrumentation/ThreadSanitizer.cpp b/lib/Transforms/Instrumentation/ThreadSanitizer.cpp
index 89386a6a86de..1b86ae5acf7d 100644
--- a/lib/Transforms/Instrumentation/ThreadSanitizer.cpp
+++ b/lib/Transforms/Instrumentation/ThreadSanitizer.cpp
@@ -135,33 +135,33 @@ void ThreadSanitizer::initializeCallbacks(Module &M) {
IRBuilder<> IRB(M.getContext());
// Initialize the callbacks.
TsanFuncEntry = checkInterfaceFunction(M.getOrInsertFunction(
- "__tsan_func_entry", IRB.getVoidTy(), IRB.getInt8PtrTy(), NULL));
+ "__tsan_func_entry", IRB.getVoidTy(), IRB.getInt8PtrTy(), nullptr));
TsanFuncExit = checkInterfaceFunction(M.getOrInsertFunction(
- "__tsan_func_exit", IRB.getVoidTy(), NULL));
+ "__tsan_func_exit", IRB.getVoidTy(), nullptr));
OrdTy = IRB.getInt32Ty();
for (size_t i = 0; i < kNumberOfAccessSizes; ++i) {
const size_t ByteSize = 1 << i;
const size_t BitSize = ByteSize * 8;
SmallString<32> ReadName("__tsan_read" + itostr(ByteSize));
TsanRead[i] = checkInterfaceFunction(M.getOrInsertFunction(
- ReadName, IRB.getVoidTy(), IRB.getInt8PtrTy(), NULL));
+ ReadName, IRB.getVoidTy(), IRB.getInt8PtrTy(), nullptr));
SmallString<32> WriteName("__tsan_write" + itostr(ByteSize));
TsanWrite[i] = checkInterfaceFunction(M.getOrInsertFunction(
- WriteName, IRB.getVoidTy(), IRB.getInt8PtrTy(), NULL));
+ WriteName, IRB.getVoidTy(), IRB.getInt8PtrTy(), nullptr));
Type *Ty = Type::getIntNTy(M.getContext(), BitSize);
Type *PtrTy = Ty->getPointerTo();
SmallString<32> AtomicLoadName("__tsan_atomic" + itostr(BitSize) +
"_load");
TsanAtomicLoad[i] = checkInterfaceFunction(M.getOrInsertFunction(
- AtomicLoadName, Ty, PtrTy, OrdTy, NULL));
+ AtomicLoadName, Ty, PtrTy, OrdTy, nullptr));
SmallString<32> AtomicStoreName("__tsan_atomic" + itostr(BitSize) +
"_store");
TsanAtomicStore[i] = checkInterfaceFunction(M.getOrInsertFunction(
AtomicStoreName, IRB.getVoidTy(), PtrTy, Ty, OrdTy,
- NULL));
+ nullptr));
for (int op = AtomicRMWInst::FIRST_BINOP;
op <= AtomicRMWInst::LAST_BINOP; ++op) {
@@ -185,33 +185,33 @@ void ThreadSanitizer::initializeCallbacks(Module &M) {
continue;
SmallString<32> RMWName("__tsan_atomic" + itostr(BitSize) + NamePart);
TsanAtomicRMW[op][i] = checkInterfaceFunction(M.getOrInsertFunction(
- RMWName, Ty, PtrTy, Ty, OrdTy, NULL));
+ RMWName, Ty, PtrTy, Ty, OrdTy, nullptr));
}
SmallString<32> AtomicCASName("__tsan_atomic" + itostr(BitSize) +
"_compare_exchange_val");
TsanAtomicCAS[i] = checkInterfaceFunction(M.getOrInsertFunction(
- AtomicCASName, Ty, PtrTy, Ty, Ty, OrdTy, OrdTy, NULL));
+ AtomicCASName, Ty, PtrTy, Ty, Ty, OrdTy, OrdTy, nullptr));
}
TsanVptrUpdate = checkInterfaceFunction(M.getOrInsertFunction(
"__tsan_vptr_update", IRB.getVoidTy(), IRB.getInt8PtrTy(),
- IRB.getInt8PtrTy(), NULL));
+ IRB.getInt8PtrTy(), nullptr));
TsanVptrLoad = checkInterfaceFunction(M.getOrInsertFunction(
- "__tsan_vptr_read", IRB.getVoidTy(), IRB.getInt8PtrTy(), NULL));
+ "__tsan_vptr_read", IRB.getVoidTy(), IRB.getInt8PtrTy(), nullptr));
TsanAtomicThreadFence = checkInterfaceFunction(M.getOrInsertFunction(
- "__tsan_atomic_thread_fence", IRB.getVoidTy(), OrdTy, NULL));
+ "__tsan_atomic_thread_fence", IRB.getVoidTy(), OrdTy, nullptr));
TsanAtomicSignalFence = checkInterfaceFunction(M.getOrInsertFunction(
- "__tsan_atomic_signal_fence", IRB.getVoidTy(), OrdTy, NULL));
+ "__tsan_atomic_signal_fence", IRB.getVoidTy(), OrdTy, nullptr));
MemmoveFn = checkInterfaceFunction(M.getOrInsertFunction(
"memmove", IRB.getInt8PtrTy(), IRB.getInt8PtrTy(),
- IRB.getInt8PtrTy(), IntptrTy, NULL));
+ IRB.getInt8PtrTy(), IntptrTy, nullptr));
MemcpyFn = checkInterfaceFunction(M.getOrInsertFunction(
"memcpy", IRB.getInt8PtrTy(), IRB.getInt8PtrTy(), IRB.getInt8PtrTy(),
- IntptrTy, NULL));
+ IntptrTy, nullptr));
MemsetFn = checkInterfaceFunction(M.getOrInsertFunction(
"memset", IRB.getInt8PtrTy(), IRB.getInt8PtrTy(), IRB.getInt32Ty(),
- IntptrTy, NULL));
+ IntptrTy, nullptr));
}
bool ThreadSanitizer::doInitialization(Module &M) {
@@ -224,7 +224,7 @@ bool ThreadSanitizer::doInitialization(Module &M) {
IRBuilder<> IRB(M.getContext());
IntptrTy = IRB.getIntPtrTy(DL);
Value *TsanInit = M.getOrInsertFunction("__tsan_init",
- IRB.getVoidTy(), NULL);
+ IRB.getVoidTy(), nullptr);
appendToGlobalCtors(M, cast<Function>(TsanInit), 0);
return true;
@@ -422,7 +422,7 @@ bool ThreadSanitizer::instrumentLoadOrStore(Instruction *I) {
static ConstantInt *createOrdering(IRBuilder<> *IRB, AtomicOrdering ord) {
uint32_t v = 0;
switch (ord) {
- case NotAtomic: assert(false);
+ case NotAtomic: llvm_unreachable("unexpected atomic ordering!");
case Unordered: // Fall-through.
case Monotonic: v = 0; break;
// case Consume: v = 1; break; // Not specified yet.
@@ -481,8 +481,7 @@ bool ThreadSanitizer::instrumentAtomic(Instruction *I) {
Type *PtrTy = Ty->getPointerTo();
Value *Args[] = {IRB.CreatePointerCast(Addr, PtrTy),
createOrdering(&IRB, LI->getOrdering())};
- CallInst *C = CallInst::Create(TsanAtomicLoad[Idx],
- ArrayRef<Value*>(Args));
+ CallInst *C = CallInst::Create(TsanAtomicLoad[Idx], Args);
ReplaceInstWithInst(I, C);
} else if (StoreInst *SI = dyn_cast<StoreInst>(I)) {
@@ -497,8 +496,7 @@ bool ThreadSanitizer::instrumentAtomic(Instruction *I) {
Value *Args[] = {IRB.CreatePointerCast(Addr, PtrTy),
IRB.CreateIntCast(SI->getValueOperand(), Ty, false),
createOrdering(&IRB, SI->getOrdering())};
- CallInst *C = CallInst::Create(TsanAtomicStore[Idx],
- ArrayRef<Value*>(Args));
+ CallInst *C = CallInst::Create(TsanAtomicStore[Idx], Args);
ReplaceInstWithInst(I, C);
} else if (AtomicRMWInst *RMWI = dyn_cast<AtomicRMWInst>(I)) {
Value *Addr = RMWI->getPointerOperand();
@@ -515,7 +513,7 @@ bool ThreadSanitizer::instrumentAtomic(Instruction *I) {
Value *Args[] = {IRB.CreatePointerCast(Addr, PtrTy),
IRB.CreateIntCast(RMWI->getValOperand(), Ty, false),
createOrdering(&IRB, RMWI->getOrdering())};
- CallInst *C = CallInst::Create(F, ArrayRef<Value*>(Args));
+ CallInst *C = CallInst::Create(F, Args);
ReplaceInstWithInst(I, C);
} else if (AtomicCmpXchgInst *CASI = dyn_cast<AtomicCmpXchgInst>(I)) {
Value *Addr = CASI->getPointerOperand();
@@ -543,7 +541,7 @@ bool ThreadSanitizer::instrumentAtomic(Instruction *I) {
Value *Args[] = {createOrdering(&IRB, FI->getOrdering())};
Function *F = FI->getSynchScope() == SingleThread ?
TsanAtomicSignalFence : TsanAtomicThreadFence;
- CallInst *C = CallInst::Create(F, ArrayRef<Value*>(Args));
+ CallInst *C = CallInst::Create(F, Args);
ReplaceInstWithInst(I, C);
}
return true;
generated by cgit v1.2.3 (git 2.25.1) at 2025-10-29 02:24:23 +0000