diff options
Diffstat (limited to 'contrib/llvm-project/llvm/lib/Transforms/Instrumentation/DataFlowSanitizer.cpp')
| -rw-r--r-- | contrib/llvm-project/llvm/lib/Transforms/Instrumentation/DataFlowSanitizer.cpp | 859 |
1 files changed, 631 insertions, 228 deletions
diff --git a/contrib/llvm-project/llvm/lib/Transforms/Instrumentation/DataFlowSanitizer.cpp b/contrib/llvm-project/llvm/lib/Transforms/Instrumentation/DataFlowSanitizer.cpp index 284631900731..1b14b8d56994 100644 --- a/contrib/llvm-project/llvm/lib/Transforms/Instrumentation/DataFlowSanitizer.cpp +++ b/contrib/llvm-project/llvm/lib/Transforms/Instrumentation/DataFlowSanitizer.cpp @@ -46,6 +46,7 @@ // //===----------------------------------------------------------------------===// +#include "llvm/Transforms/Instrumentation/DataFlowSanitizer.h" #include "llvm/ADT/DenseMap.h" #include "llvm/ADT/DenseSet.h" #include "llvm/ADT/DepthFirstIterator.h" @@ -78,6 +79,7 @@ #include "llvm/IR/LLVMContext.h" #include "llvm/IR/MDBuilder.h" #include "llvm/IR/Module.h" +#include "llvm/IR/PassManager.h" #include "llvm/IR/Type.h" #include "llvm/IR/User.h" #include "llvm/IR/Value.h" @@ -104,10 +106,18 @@ using namespace llvm; +// This must be consistent with ShadowWidthBits. +static const Align kShadowTLSAlignment = Align(2); + +// The size of TLS variables. These constants must be kept in sync with the ones +// in dfsan.cpp. +static const unsigned kArgTLSSize = 800; +static const unsigned kRetvalTLSSize = 800; + // External symbol to be used when generating the shadow address for // architectures with multiple VMAs. Instead of using a constant integer // the runtime will set the external mask based on the VMA range. -static const char *const kDFSanExternShadowPtrMask = "__dfsan_shadow_ptr_mask"; +const char kDFSanExternShadowPtrMask[] = "__dfsan_shadow_ptr_mask"; // The -dfsan-preserve-alignment flag controls whether this pass assumes that // alignment requirements provided by the input IR are correct. For example, @@ -167,8 +177,8 @@ static cl::opt<bool> ClDebugNonzeroLabels( // // If this flag is set to true, the user must provide definitions for the // following callback functions: -// void __dfsan_load_callback(dfsan_label Label); -// void __dfsan_store_callback(dfsan_label Label); +// void __dfsan_load_callback(dfsan_label Label, void* addr); +// void __dfsan_store_callback(dfsan_label Label, void* addr); // void __dfsan_mem_transfer_callback(dfsan_label *Start, size_t Len); // void __dfsan_cmp_callback(dfsan_label CombinedLabel); static cl::opt<bool> ClEventCallbacks( @@ -176,6 +186,21 @@ static cl::opt<bool> ClEventCallbacks( cl::desc("Insert calls to __dfsan_*_callback functions on data events."), cl::Hidden, cl::init(false)); +// Use a distinct bit for each base label, enabling faster unions with less +// instrumentation. Limits the max number of base labels to 16. +static cl::opt<bool> ClFast16Labels( + "dfsan-fast-16-labels", + cl::desc("Use more efficient instrumentation, limiting the number of " + "labels to 16."), + cl::Hidden, cl::init(false)); + +// Controls whether the pass tracks the control flow of select instructions. +static cl::opt<bool> ClTrackSelectControlFlow( + "dfsan-track-select-control-flow", + cl::desc("Propagate labels from condition values of select instructions " + "to results."), + cl::Hidden, cl::init(true)); + static StringRef GetGlobalTypeString(const GlobalValue &G) { // Types of GlobalVariables are always pointer types. Type *GType = G.getValueType(); @@ -292,7 +317,7 @@ AttributeList TransformFunctionAttributes( llvm::makeArrayRef(ArgumentAttributes)); } -class DataFlowSanitizer : public ModulePass { +class DataFlowSanitizer { friend struct DFSanFunction; friend class DFSanVisitor; @@ -336,20 +361,16 @@ class DataFlowSanitizer : public ModulePass { Module *Mod; LLVMContext *Ctx; - IntegerType *ShadowTy; - PointerType *ShadowPtrTy; + Type *Int8Ptr; + /// The shadow type for all primitive types and vector types. + IntegerType *PrimitiveShadowTy; + PointerType *PrimitiveShadowPtrTy; IntegerType *IntptrTy; - ConstantInt *ZeroShadow; + ConstantInt *ZeroPrimitiveShadow; ConstantInt *ShadowPtrMask; ConstantInt *ShadowPtrMul; Constant *ArgTLS; Constant *RetvalTLS; - void *(*GetArgTLSPtr)(); - void *(*GetRetvalTLSPtr)(); - FunctionType *GetArgTLSTy; - FunctionType *GetRetvalTLSTy; - Constant *GetArgTLS; - Constant *GetRetvalTLS; Constant *ExternalShadowMask; FunctionType *DFSanUnionFnTy; FunctionType *DFSanUnionLoadFnTy; @@ -357,11 +378,13 @@ class DataFlowSanitizer : public ModulePass { FunctionType *DFSanSetLabelFnTy; FunctionType *DFSanNonzeroLabelFnTy; FunctionType *DFSanVarargWrapperFnTy; - FunctionType *DFSanLoadStoreCmpCallbackFnTy; + FunctionType *DFSanCmpCallbackFnTy; + FunctionType *DFSanLoadStoreCallbackFnTy; FunctionType *DFSanMemTransferCallbackFnTy; FunctionCallee DFSanUnionFn; FunctionCallee DFSanCheckedUnionFn; FunctionCallee DFSanUnionLoadFn; + FunctionCallee DFSanUnionLoadFast16LabelsFn; FunctionCallee DFSanUnimplementedFn; FunctionCallee DFSanSetLabelFn; FunctionCallee DFSanNonzeroLabelFn; @@ -392,15 +415,43 @@ class DataFlowSanitizer : public ModulePass { void initializeCallbackFunctions(Module &M); void initializeRuntimeFunctions(Module &M); -public: - static char ID; + bool init(Module &M); + + /// Returns whether the pass tracks labels for struct fields and array + /// indices. Support only fast16 mode in TLS ABI mode. + bool shouldTrackFieldsAndIndices(); + + /// Returns a zero constant with the shadow type of OrigTy. + /// + /// getZeroShadow({T1,T2,...}) = {getZeroShadow(T1),getZeroShadow(T2,...} + /// getZeroShadow([n x T]) = [n x getZeroShadow(T)] + /// getZeroShadow(other type) = i16(0) + /// + /// Note that a zero shadow is always i16(0) when shouldTrackFieldsAndIndices + /// returns false. + Constant *getZeroShadow(Type *OrigTy); + /// Returns a zero constant with the shadow type of V's type. + Constant *getZeroShadow(Value *V); + + /// Checks if V is a zero shadow. + bool isZeroShadow(Value *V); + + /// Returns the shadow type of OrigTy. + /// + /// getShadowTy({T1,T2,...}) = {getShadowTy(T1),getShadowTy(T2),...} + /// getShadowTy([n x T]) = [n x getShadowTy(T)] + /// getShadowTy(other type) = i16 + /// + /// Note that a shadow type is always i16 when shouldTrackFieldsAndIndices + /// returns false. + Type *getShadowTy(Type *OrigTy); + /// Returns the shadow type of of V's type. + Type *getShadowTy(Value *V); - DataFlowSanitizer( - const std::vector<std::string> &ABIListFiles = std::vector<std::string>(), - void *(*getArgTLS)() = nullptr, void *(*getRetValTLS)() = nullptr); +public: + DataFlowSanitizer(const std::vector<std::string> &ABIListFiles); - bool doInitialization(Module &M) override; - bool runOnModule(Module &M) override; + bool runImpl(Module &M); }; struct DFSanFunction { @@ -409,8 +460,6 @@ struct DFSanFunction { DominatorTree DT; DataFlowSanitizer::InstrumentedABI IA; bool IsNativeABI; - Value *ArgTLSPtr = nullptr; - Value *RetvalTLSPtr = nullptr; AllocaInst *LabelReturnAlloca = nullptr; DenseMap<Value *, Value *> ValShadowMap; DenseMap<AllocaInst *, AllocaInst *> AllocaShadowMap; @@ -419,12 +468,17 @@ struct DFSanFunction { std::vector<Value *> NonZeroChecks; bool AvoidNewBlocks; - struct CachedCombinedShadow { - BasicBlock *Block; + struct CachedShadow { + BasicBlock *Block; // The block where Shadow is defined. Value *Shadow; }; - DenseMap<std::pair<Value *, Value *>, CachedCombinedShadow> - CachedCombinedShadows; + /// Maps a value to its latest shadow value in terms of domination tree. + DenseMap<std::pair<Value *, Value *>, CachedShadow> CachedShadows; + /// Maps a value to its latest collapsed shadow value it was converted to in + /// terms of domination tree. When ClDebugNonzeroLabels is on, this cache is + /// used at a post process where CFG blocks are split. So it does not cache + /// BasicBlock like CachedShadows, but uses domination between values. + DenseMap<Value *, Value *> CachedCollapsedShadows; DenseMap<Value *, std::set<Value *>> ShadowElements; DFSanFunction(DataFlowSanitizer &DFS, Function *F, bool IsNativeABI) @@ -435,17 +489,56 @@ struct DFSanFunction { AvoidNewBlocks = F->size() > 1000; } - Value *getArgTLSPtr(); - Value *getArgTLS(unsigned Index, Instruction *Pos); - Value *getRetvalTLS(); + /// Computes the shadow address for a given function argument. + /// + /// Shadow = ArgTLS+ArgOffset. + Value *getArgTLS(Type *T, unsigned ArgOffset, IRBuilder<> &IRB); + + /// Computes the shadow address for a retval. + Value *getRetvalTLS(Type *T, IRBuilder<> &IRB); + Value *getShadow(Value *V); void setShadow(Instruction *I, Value *Shadow); + /// Generates IR to compute the union of the two given shadows, inserting it + /// before Pos. The combined value is with primitive type. Value *combineShadows(Value *V1, Value *V2, Instruction *Pos); + /// Combines the shadow values of V1 and V2, then converts the combined value + /// with primitive type into a shadow value with the original type T. + Value *combineShadowsThenConvert(Type *T, Value *V1, Value *V2, + Instruction *Pos); Value *combineOperandShadows(Instruction *Inst); Value *loadShadow(Value *ShadowAddr, uint64_t Size, uint64_t Align, Instruction *Pos); - void storeShadow(Value *Addr, uint64_t Size, Align Alignment, Value *Shadow, - Instruction *Pos); + void storePrimitiveShadow(Value *Addr, uint64_t Size, Align Alignment, + Value *PrimitiveShadow, Instruction *Pos); + /// Applies PrimitiveShadow to all primitive subtypes of T, returning + /// the expanded shadow value. + /// + /// EFP({T1,T2, ...}, PS) = {EFP(T1,PS),EFP(T2,PS),...} + /// EFP([n x T], PS) = [n x EFP(T,PS)] + /// EFP(other types, PS) = PS + Value *expandFromPrimitiveShadow(Type *T, Value *PrimitiveShadow, + Instruction *Pos); + /// Collapses Shadow into a single primitive shadow value, unioning all + /// primitive shadow values in the process. Returns the final primitive + /// shadow value. + /// + /// CTP({V1,V2, ...}) = UNION(CFP(V1,PS),CFP(V2,PS),...) + /// CTP([V1,V2,...]) = UNION(CFP(V1,PS),CFP(V2,PS),...) + /// CTP(other types, PS) = PS + Value *collapseToPrimitiveShadow(Value *Shadow, Instruction *Pos); + +private: + /// Collapses the shadow with aggregate type into a single primitive shadow + /// value. + template <class AggregateType> + Value *collapseAggregateShadow(AggregateType *AT, Value *Shadow, + IRBuilder<> &IRB); + + Value *collapseToPrimitiveShadow(Value *Shadow, IRBuilder<> &IRB); + + /// Returns the shadow value of an argument A. + Value *getShadowForTLSArgument(Argument *A); }; class DFSanVisitor : public InstVisitor<DFSanVisitor> { @@ -485,25 +578,10 @@ public: } // end anonymous namespace -char DataFlowSanitizer::ID; - -INITIALIZE_PASS(DataFlowSanitizer, "dfsan", - "DataFlowSanitizer: dynamic data flow analysis.", false, false) - -ModulePass * -llvm::createDataFlowSanitizerPass(const std::vector<std::string> &ABIListFiles, - void *(*getArgTLS)(), - void *(*getRetValTLS)()) { - return new DataFlowSanitizer(ABIListFiles, getArgTLS, getRetValTLS); -} - DataFlowSanitizer::DataFlowSanitizer( - const std::vector<std::string> &ABIListFiles, void *(*getArgTLS)(), - void *(*getRetValTLS)()) - : ModulePass(ID), GetArgTLSPtr(getArgTLS), GetRetvalTLSPtr(getRetValTLS) { + const std::vector<std::string> &ABIListFiles) { std::vector<std::string> AllABIListFiles(std::move(ABIListFiles)); - AllABIListFiles.insert(AllABIListFiles.end(), ClABIListFiles.begin(), - ClABIListFiles.end()); + llvm::append_range(AllABIListFiles, ClABIListFiles); // FIXME: should we propagate vfs::FileSystem to this constructor? ABIList.set( SpecialCaseList::createOrDie(AllABIListFiles, *vfs::getRealFileSystem())); @@ -511,12 +589,12 @@ DataFlowSanitizer::DataFlowSanitizer( FunctionType *DataFlowSanitizer::getArgsFunctionType(FunctionType *T) { SmallVector<Type *, 4> ArgTypes(T->param_begin(), T->param_end()); - ArgTypes.append(T->getNumParams(), ShadowTy); + ArgTypes.append(T->getNumParams(), PrimitiveShadowTy); if (T->isVarArg()) - ArgTypes.push_back(ShadowPtrTy); + ArgTypes.push_back(PrimitiveShadowPtrTy); Type *RetType = T->getReturnType(); if (!RetType->isVoidTy()) - RetType = StructType::get(RetType, ShadowTy); + RetType = StructType::get(RetType, PrimitiveShadowTy); return FunctionType::get(RetType, ArgTypes, T->isVarArg()); } @@ -525,10 +603,10 @@ FunctionType *DataFlowSanitizer::getTrampolineFunctionType(FunctionType *T) { SmallVector<Type *, 4> ArgTypes; ArgTypes.push_back(T->getPointerTo()); ArgTypes.append(T->param_begin(), T->param_end()); - ArgTypes.append(T->getNumParams(), ShadowTy); + ArgTypes.append(T->getNumParams(), PrimitiveShadowTy); Type *RetType = T->getReturnType(); if (!RetType->isVoidTy()) - ArgTypes.push_back(ShadowPtrTy); + ArgTypes.push_back(PrimitiveShadowPtrTy); return FunctionType::get(T->getReturnType(), ArgTypes, false); } @@ -554,18 +632,174 @@ TransformedFunction DataFlowSanitizer::getCustomFunctionType(FunctionType *T) { } } for (unsigned i = 0, e = T->getNumParams(); i != e; ++i) - ArgTypes.push_back(ShadowTy); + ArgTypes.push_back(PrimitiveShadowTy); if (T->isVarArg()) - ArgTypes.push_back(ShadowPtrTy); + ArgTypes.push_back(PrimitiveShadowPtrTy); Type *RetType = T->getReturnType(); if (!RetType->isVoidTy()) - ArgTypes.push_back(ShadowPtrTy); + ArgTypes.push_back(PrimitiveShadowPtrTy); return TransformedFunction( T, FunctionType::get(T->getReturnType(), ArgTypes, T->isVarArg()), ArgumentIndexMapping); } -bool DataFlowSanitizer::doInitialization(Module &M) { +bool DataFlowSanitizer::isZeroShadow(Value *V) { + if (!shouldTrackFieldsAndIndices()) + return ZeroPrimitiveShadow == V; + + Type *T = V->getType(); + if (!isa<ArrayType>(T) && !isa<StructType>(T)) { + if (const ConstantInt *CI = dyn_cast<ConstantInt>(V)) + return CI->isZero(); + return false; + } + + return isa<ConstantAggregateZero>(V); +} + +bool DataFlowSanitizer::shouldTrackFieldsAndIndices() { + return getInstrumentedABI() == DataFlowSanitizer::IA_TLS && ClFast16Labels; +} + +Constant *DataFlowSanitizer::getZeroShadow(Type *OrigTy) { + if (!shouldTrackFieldsAndIndices()) + return ZeroPrimitiveShadow; + + if (!isa<ArrayType>(OrigTy) && !isa<StructType>(OrigTy)) + return ZeroPrimitiveShadow; + Type *ShadowTy = getShadowTy(OrigTy); + return ConstantAggregateZero::get(ShadowTy); +} + +Constant *DataFlowSanitizer::getZeroShadow(Value *V) { + return getZeroShadow(V->getType()); +} + +static Value *expandFromPrimitiveShadowRecursive( + Value *Shadow, SmallVector<unsigned, 4> &Indices, Type *SubShadowTy, + Value *PrimitiveShadow, IRBuilder<> &IRB) { + if (!isa<ArrayType>(SubShadowTy) && !isa<StructType>(SubShadowTy)) + return IRB.CreateInsertValue(Shadow, PrimitiveShadow, Indices); + + if (ArrayType *AT = dyn_cast<ArrayType>(SubShadowTy)) { + for (unsigned Idx = 0; Idx < AT->getNumElements(); Idx++) { + Indices.push_back(Idx); + Shadow = expandFromPrimitiveShadowRecursive( + Shadow, Indices, AT->getElementType(), PrimitiveShadow, IRB); + Indices.pop_back(); + } + return Shadow; + } + + if (StructType *ST = dyn_cast<StructType>(SubShadowTy)) { + for (unsigned Idx = 0; Idx < ST->getNumElements(); Idx++) { + Indices.push_back(Idx); + Shadow = expandFromPrimitiveShadowRecursive( + Shadow, Indices, ST->getElementType(Idx), PrimitiveShadow, IRB); + Indices.pop_back(); + } + return Shadow; + } + llvm_unreachable("Unexpected shadow type"); +} + +Value *DFSanFunction::expandFromPrimitiveShadow(Type *T, Value *PrimitiveShadow, + Instruction *Pos) { + Type *ShadowTy = DFS.getShadowTy(T); + + if (!isa<ArrayType>(ShadowTy) && !isa<StructType>(ShadowTy)) + return PrimitiveShadow; + + if (DFS.isZeroShadow(PrimitiveShadow)) + return DFS.getZeroShadow(ShadowTy); + + IRBuilder<> IRB(Pos); + SmallVector<unsigned, 4> Indices; + Value *Shadow = UndefValue::get(ShadowTy); + Shadow = expandFromPrimitiveShadowRecursive(Shadow, Indices, ShadowTy, + PrimitiveShadow, IRB); + + // Caches the primitive shadow value that built the shadow value. + CachedCollapsedShadows[Shadow] = PrimitiveShadow; + return Shadow; +} + +template <class AggregateType> +Value *DFSanFunction::collapseAggregateShadow(AggregateType *AT, Value *Shadow, + IRBuilder<> &IRB) { + if (!AT->getNumElements()) + return DFS.ZeroPrimitiveShadow; + + Value *FirstItem = IRB.CreateExtractValue(Shadow, 0); + Value *Aggregator = collapseToPrimitiveShadow(FirstItem, IRB); + + for (unsigned Idx = 1; Idx < AT->getNumElements(); Idx++) { + Value *ShadowItem = IRB.CreateExtractValue(Shadow, Idx); + Value *ShadowInner = collapseToPrimitiveShadow(ShadowItem, IRB); + Aggregator = IRB.CreateOr(Aggregator, ShadowInner); + } + return Aggregator; +} + +Value *DFSanFunction::collapseToPrimitiveShadow(Value *Shadow, + IRBuilder<> &IRB) { + Type *ShadowTy = Shadow->getType(); + if (!isa<ArrayType>(ShadowTy) && !isa<StructType>(ShadowTy)) + return Shadow; + if (ArrayType *AT = dyn_cast<ArrayType>(ShadowTy)) + return collapseAggregateShadow<>(AT, Shadow, IRB); + if (StructType *ST = dyn_cast<StructType>(ShadowTy)) + return collapseAggregateShadow<>(ST, Shadow, IRB); + llvm_unreachable("Unexpected shadow type"); +} + +Value *DFSanFunction::collapseToPrimitiveShadow(Value *Shadow, + Instruction *Pos) { + Type *ShadowTy = Shadow->getType(); + if (!isa<ArrayType>(ShadowTy) && !isa<StructType>(ShadowTy)) + return Shadow; + + assert(DFS.shouldTrackFieldsAndIndices()); + + // Checks if the cached collapsed shadow value dominates Pos. + Value *&CS = CachedCollapsedShadows[Shadow]; + if (CS && DT.dominates(CS, Pos)) + return CS; + + IRBuilder<> IRB(Pos); + Value *PrimitiveShadow = collapseToPrimitiveShadow(Shadow, IRB); + // Caches the converted primitive shadow value. + CS = PrimitiveShadow; + return PrimitiveShadow; +} + +Type *DataFlowSanitizer::getShadowTy(Type *OrigTy) { + if (!shouldTrackFieldsAndIndices()) + return PrimitiveShadowTy; + + if (!OrigTy->isSized()) + return PrimitiveShadowTy; + if (isa<IntegerType>(OrigTy)) + return PrimitiveShadowTy; + if (isa<VectorType>(OrigTy)) + return PrimitiveShadowTy; + if (ArrayType *AT = dyn_cast<ArrayType>(OrigTy)) + return ArrayType::get(getShadowTy(AT->getElementType()), + AT->getNumElements()); + if (StructType *ST = dyn_cast<StructType>(OrigTy)) { + SmallVector<Type *, 4> Elements; + for (unsigned I = 0, N = ST->getNumElements(); I < N; ++I) + Elements.push_back(getShadowTy(ST->getElementType(I))); + return StructType::get(*Ctx, Elements); + } + return PrimitiveShadowTy; +} + +Type *DataFlowSanitizer::getShadowTy(Value *V) { + return getShadowTy(V->getType()); +} + +bool DataFlowSanitizer::init(Module &M) { Triple TargetTriple(M.getTargetTriple()); bool IsX86_64 = TargetTriple.getArch() == Triple::x86_64; bool IsMIPS64 = TargetTriple.isMIPS64(); @@ -576,10 +810,11 @@ bool DataFlowSanitizer::doInitialization(Module &M) { Mod = &M; Ctx = &M.getContext(); - ShadowTy = IntegerType::get(*Ctx, ShadowWidthBits); - ShadowPtrTy = PointerType::getUnqual(ShadowTy); + Int8Ptr = Type::getInt8PtrTy(*Ctx); + PrimitiveShadowTy = IntegerType::get(*Ctx, ShadowWidthBits); + PrimitiveShadowPtrTy = PointerType::getUnqual(PrimitiveShadowTy); IntptrTy = DL.getIntPtrType(*Ctx); - ZeroShadow = ConstantInt::getSigned(ShadowTy, 0); + ZeroPrimitiveShadow = ConstantInt::getSigned(PrimitiveShadowTy, 0); ShadowPtrMul = ConstantInt::getSigned(IntptrTy, ShadowWidthBytes); if (IsX86_64) ShadowPtrMask = ConstantInt::getSigned(IntptrTy, ~0x700000000000LL); @@ -591,44 +826,34 @@ bool DataFlowSanitizer::doInitialization(Module &M) { else report_fatal_error("unsupported triple"); - Type *DFSanUnionArgs[2] = { ShadowTy, ShadowTy }; + Type *DFSanUnionArgs[2] = {PrimitiveShadowTy, PrimitiveShadowTy}; DFSanUnionFnTy = - FunctionType::get(ShadowTy, DFSanUnionArgs, /*isVarArg=*/ false); - Type *DFSanUnionLoadArgs[2] = { ShadowPtrTy, IntptrTy }; - DFSanUnionLoadFnTy = - FunctionType::get(ShadowTy, DFSanUnionLoadArgs, /*isVarArg=*/ false); + FunctionType::get(PrimitiveShadowTy, DFSanUnionArgs, /*isVarArg=*/false); + Type *DFSanUnionLoadArgs[2] = {PrimitiveShadowPtrTy, IntptrTy}; + DFSanUnionLoadFnTy = FunctionType::get(PrimitiveShadowTy, DFSanUnionLoadArgs, + /*isVarArg=*/false); DFSanUnimplementedFnTy = FunctionType::get( Type::getVoidTy(*Ctx), Type::getInt8PtrTy(*Ctx), /*isVarArg=*/false); - Type *DFSanSetLabelArgs[3] = { ShadowTy, Type::getInt8PtrTy(*Ctx), IntptrTy }; + Type *DFSanSetLabelArgs[3] = {PrimitiveShadowTy, Type::getInt8PtrTy(*Ctx), + IntptrTy}; DFSanSetLabelFnTy = FunctionType::get(Type::getVoidTy(*Ctx), DFSanSetLabelArgs, /*isVarArg=*/false); - DFSanNonzeroLabelFnTy = FunctionType::get( - Type::getVoidTy(*Ctx), None, /*isVarArg=*/false); + DFSanNonzeroLabelFnTy = + FunctionType::get(Type::getVoidTy(*Ctx), None, /*isVarArg=*/false); DFSanVarargWrapperFnTy = FunctionType::get( Type::getVoidTy(*Ctx), Type::getInt8PtrTy(*Ctx), /*isVarArg=*/false); - DFSanLoadStoreCmpCallbackFnTy = - FunctionType::get(Type::getVoidTy(*Ctx), ShadowTy, /*isVarArg=*/false); - Type *DFSanMemTransferCallbackArgs[2] = {ShadowPtrTy, IntptrTy}; + DFSanCmpCallbackFnTy = + FunctionType::get(Type::getVoidTy(*Ctx), PrimitiveShadowTy, + /*isVarArg=*/false); + Type *DFSanLoadStoreCallbackArgs[2] = {PrimitiveShadowTy, Int8Ptr}; + DFSanLoadStoreCallbackFnTy = + FunctionType::get(Type::getVoidTy(*Ctx), DFSanLoadStoreCallbackArgs, + /*isVarArg=*/false); + Type *DFSanMemTransferCallbackArgs[2] = {PrimitiveShadowPtrTy, IntptrTy}; DFSanMemTransferCallbackFnTy = FunctionType::get(Type::getVoidTy(*Ctx), DFSanMemTransferCallbackArgs, /*isVarArg=*/false); - if (GetArgTLSPtr) { - Type *ArgTLSTy = ArrayType::get(ShadowTy, 64); - ArgTLS = nullptr; - GetArgTLSTy = FunctionType::get(PointerType::getUnqual(ArgTLSTy), false); - GetArgTLS = ConstantExpr::getIntToPtr( - ConstantInt::get(IntptrTy, uintptr_t(GetArgTLSPtr)), - PointerType::getUnqual(GetArgTLSTy)); - } - if (GetRetvalTLSPtr) { - RetvalTLS = nullptr; - GetRetvalTLSTy = FunctionType::get(PointerType::getUnqual(ShadowTy), false); - GetRetvalTLS = ConstantExpr::getIntToPtr( - ConstantInt::get(IntptrTy, uintptr_t(GetRetvalTLSPtr)), - PointerType::getUnqual(GetRetvalTLSTy)); - } - ColdCallWeights = MDBuilder(*Ctx).createBranchWeights(1, 1000); return true; } @@ -729,14 +954,21 @@ Constant *DataFlowSanitizer::getOrBuildTrampolineFunction(FunctionType *FT, else RI = ReturnInst::Create(*Ctx, CI, BB); + // F is called by a wrapped custom function with primitive shadows. So + // its arguments and return value need conversion. DFSanFunction DFSF(*this, F, /*IsNativeABI=*/true); Function::arg_iterator ValAI = F->arg_begin(), ShadowAI = AI; ++ValAI; - for (unsigned N = FT->getNumParams(); N != 0; ++ValAI, ++ShadowAI, --N) - DFSF.ValShadowMap[&*ValAI] = &*ShadowAI; + for (unsigned N = FT->getNumParams(); N != 0; ++ValAI, ++ShadowAI, --N) { + Value *Shadow = + DFSF.expandFromPrimitiveShadow(ValAI->getType(), &*ShadowAI, CI); + DFSF.ValShadowMap[&*ValAI] = Shadow; + } DFSanVisitor(DFSF).visitCallInst(*CI); - if (!FT->getReturnType()->isVoidTy()) - new StoreInst(DFSF.getShadow(RI->getReturnValue()), - &*std::prev(F->arg_end()), RI); + if (!FT->getReturnType()->isVoidTy()) { + Value *PrimitiveShadow = DFSF.collapseToPrimitiveShadow( + DFSF.getShadow(RI->getReturnValue()), RI); + new StoreInst(PrimitiveShadow, &*std::prev(F->arg_end()), RI); + } } return cast<Constant>(C.getCallee()); @@ -781,6 +1013,17 @@ void DataFlowSanitizer::initializeRuntimeFunctions(Module &M) { DFSanUnionLoadFn = Mod->getOrInsertFunction("__dfsan_union_load", DFSanUnionLoadFnTy, AL); } + { + AttributeList AL; + AL = AL.addAttribute(M.getContext(), AttributeList::FunctionIndex, + Attribute::NoUnwind); + AL = AL.addAttribute(M.getContext(), AttributeList::FunctionIndex, + Attribute::ReadOnly); + AL = AL.addAttribute(M.getContext(), AttributeList::ReturnIndex, + Attribute::ZExt); + DFSanUnionLoadFast16LabelsFn = Mod->getOrInsertFunction( + "__dfsan_union_load_fast16labels", DFSanUnionLoadFnTy, AL); + } DFSanUnimplementedFn = Mod->getOrInsertFunction("__dfsan_unimplemented", DFSanUnimplementedFnTy); { @@ -798,16 +1041,18 @@ void DataFlowSanitizer::initializeRuntimeFunctions(Module &M) { // Initializes event callback functions and declare them in the module void DataFlowSanitizer::initializeCallbackFunctions(Module &M) { DFSanLoadCallbackFn = Mod->getOrInsertFunction("__dfsan_load_callback", - DFSanLoadStoreCmpCallbackFnTy); - DFSanStoreCallbackFn = Mod->getOrInsertFunction( - "__dfsan_store_callback", DFSanLoadStoreCmpCallbackFnTy); + DFSanLoadStoreCallbackFnTy); + DFSanStoreCallbackFn = Mod->getOrInsertFunction("__dfsan_store_callback", + DFSanLoadStoreCallbackFnTy); DFSanMemTransferCallbackFn = Mod->getOrInsertFunction( "__dfsan_mem_transfer_callback", DFSanMemTransferCallbackFnTy); - DFSanCmpCallbackFn = Mod->getOrInsertFunction("__dfsan_cmp_callback", - DFSanLoadStoreCmpCallbackFnTy); + DFSanCmpCallbackFn = + Mod->getOrInsertFunction("__dfsan_cmp_callback", DFSanCmpCallbackFnTy); } -bool DataFlowSanitizer::runOnModule(Module &M) { +bool DataFlowSanitizer::runImpl(Module &M) { + init(M); + if (ABIList.isIn(M, "skip")) return false; @@ -816,20 +1061,18 @@ bool DataFlowSanitizer::runOnModule(Module &M) { bool Changed = false; - if (!GetArgTLSPtr) { - Type *ArgTLSTy = ArrayType::get(ShadowTy, 64); - ArgTLS = Mod->getOrInsertGlobal("__dfsan_arg_tls", ArgTLSTy); - if (GlobalVariable *G = dyn_cast<GlobalVariable>(ArgTLS)) { - Changed |= G->getThreadLocalMode() != GlobalVariable::InitialExecTLSModel; - G->setThreadLocalMode(GlobalVariable::InitialExecTLSModel); - } + Type *ArgTLSTy = ArrayType::get(Type::getInt64Ty(*Ctx), kArgTLSSize / 8); + ArgTLS = Mod->getOrInsertGlobal("__dfsan_arg_tls", ArgTLSTy); + if (GlobalVariable *G = dyn_cast<GlobalVariable>(ArgTLS)) { + Changed |= G->getThreadLocalMode() != GlobalVariable::InitialExecTLSModel; + G->setThreadLocalMode(GlobalVariable::InitialExecTLSModel); } - if (!GetRetvalTLSPtr) { - RetvalTLS = Mod->getOrInsertGlobal("__dfsan_retval_tls", ShadowTy); - if (GlobalVariable *G = dyn_cast<GlobalVariable>(RetvalTLS)) { - Changed |= G->getThreadLocalMode() != GlobalVariable::InitialExecTLSModel; - G->setThreadLocalMode(GlobalVariable::InitialExecTLSModel); - } + Type *RetvalTLSTy = + ArrayType::get(Type::getInt64Ty(*Ctx), kRetvalTLSSize / 8); + RetvalTLS = Mod->getOrInsertGlobal("__dfsan_retval_tls", RetvalTLSTy); + if (GlobalVariable *G = dyn_cast<GlobalVariable>(RetvalTLS)) { + Changed |= G->getThreadLocalMode() != GlobalVariable::InitialExecTLSModel; + G->setThreadLocalMode(GlobalVariable::InitialExecTLSModel); } ExternalShadowMask = @@ -845,6 +1088,7 @@ bool DataFlowSanitizer::runOnModule(Module &M) { &i != DFSanUnionFn.getCallee()->stripPointerCasts() && &i != DFSanCheckedUnionFn.getCallee()->stripPointerCasts() && &i != DFSanUnionLoadFn.getCallee()->stripPointerCasts() && + &i != DFSanUnionLoadFast16LabelsFn.getCallee()->stripPointerCasts() && &i != DFSanUnimplementedFn.getCallee()->stripPointerCasts() && &i != DFSanSetLabelFn.getCallee()->stripPointerCasts() && &i != DFSanNonzeroLabelFn.getCallee()->stripPointerCasts() && @@ -1044,7 +1288,9 @@ bool DataFlowSanitizer::runOnModule(Module &M) { while (isa<PHINode>(Pos) || isa<AllocaInst>(Pos)) Pos = Pos->getNextNode(); IRBuilder<> IRB(Pos); - Value *Ne = IRB.CreateICmpNE(V, DFSF.DFS.ZeroShadow); + Value *PrimitiveShadow = DFSF.collapseToPrimitiveShadow(V, Pos); + Value *Ne = + IRB.CreateICmpNE(PrimitiveShadow, DFSF.DFS.ZeroPrimitiveShadow); BranchInst *BI = cast<BranchInst>(SplitBlockAndInsertIfThen( Ne, Pos, /*Unreachable=*/false, ColdCallWeights)); IRBuilder<> ThenIRB(BI); @@ -1057,50 +1303,61 @@ bool DataFlowSanitizer::runOnModule(Module &M) { M.global_size() != InitialGlobalSize || M.size() != InitialModuleSize; } -Value *DFSanFunction::getArgTLSPtr() { - if (ArgTLSPtr) - return ArgTLSPtr; - if (DFS.ArgTLS) - return ArgTLSPtr = DFS.ArgTLS; +Value *DFSanFunction::getArgTLS(Type *T, unsigned ArgOffset, IRBuilder<> &IRB) { + Value *Base = IRB.CreatePointerCast(DFS.ArgTLS, DFS.IntptrTy); + if (ArgOffset) + Base = IRB.CreateAdd(Base, ConstantInt::get(DFS.IntptrTy, ArgOffset)); + return IRB.CreateIntToPtr(Base, PointerType::get(DFS.getShadowTy(T), 0), + "_dfsarg"); +} - IRBuilder<> IRB(&F->getEntryBlock().front()); - return ArgTLSPtr = IRB.CreateCall(DFS.GetArgTLSTy, DFS.GetArgTLS, {}); +Value *DFSanFunction::getRetvalTLS(Type *T, IRBuilder<> &IRB) { + return IRB.CreatePointerCast( + DFS.RetvalTLS, PointerType::get(DFS.getShadowTy(T), 0), "_dfsret"); } -Value *DFSanFunction::getRetvalTLS() { - if (RetvalTLSPtr) - return RetvalTLSPtr; - if (DFS.RetvalTLS) - return RetvalTLSPtr = DFS.RetvalTLS; +Value *DFSanFunction::getShadowForTLSArgument(Argument *A) { + unsigned ArgOffset = 0; + const DataLayout &DL = F->getParent()->getDataLayout(); + for (auto &FArg : F->args()) { + if (!FArg.getType()->isSized()) { + if (A == &FArg) + break; + continue; + } - IRBuilder<> IRB(&F->getEntryBlock().front()); - return RetvalTLSPtr = - IRB.CreateCall(DFS.GetRetvalTLSTy, DFS.GetRetvalTLS, {}); -} + unsigned Size = DL.getTypeAllocSize(DFS.getShadowTy(&FArg)); + if (A != &FArg) { + ArgOffset += alignTo(Size, kShadowTLSAlignment); + if (ArgOffset > kArgTLSSize) + break; // ArgTLS overflows, uses a zero shadow. + continue; + } -Value *DFSanFunction::getArgTLS(unsigned Idx, Instruction *Pos) { - IRBuilder<> IRB(Pos); - return IRB.CreateConstGEP2_64(ArrayType::get(DFS.ShadowTy, 64), - getArgTLSPtr(), 0, Idx); + if (ArgOffset + Size > kArgTLSSize) + break; // ArgTLS overflows, uses a zero shadow. + + Instruction *ArgTLSPos = &*F->getEntryBlock().begin(); + IRBuilder<> IRB(ArgTLSPos); + Value *ArgShadowPtr = getArgTLS(FArg.getType(), ArgOffset, IRB); + return IRB.CreateAlignedLoad(DFS.getShadowTy(&FArg), ArgShadowPtr, + kShadowTLSAlignment); + } + + return DFS.getZeroShadow(A); } Value *DFSanFunction::getShadow(Value *V) { if (!isa<Argument>(V) && !isa<Instruction>(V)) - return DFS.ZeroShadow; + return DFS.getZeroShadow(V); Value *&Shadow = ValShadowMap[V]; if (!Shadow) { if (Argument *A = dyn_cast<Argument>(V)) { if (IsNativeABI) - return DFS.ZeroShadow; + return DFS.getZeroShadow(V); switch (IA) { case DataFlowSanitizer::IA_TLS: { - Value *ArgTLSPtr = getArgTLSPtr(); - Instruction *ArgTLSPos = - DFS.ArgTLS ? &*F->getEntryBlock().begin() - : cast<Instruction>(ArgTLSPtr)->getNextNode(); - IRBuilder<> IRB(ArgTLSPos); - Shadow = - IRB.CreateLoad(DFS.ShadowTy, getArgTLS(A->getArgNo(), ArgTLSPos)); + Shadow = getShadowForTLSArgument(A); break; } case DataFlowSanitizer::IA_Args: { @@ -1109,13 +1366,13 @@ Value *DFSanFunction::getShadow(Value *V) { while (ArgIdx--) ++i; Shadow = &*i; - assert(Shadow->getType() == DFS.ShadowTy); + assert(Shadow->getType() == DFS.PrimitiveShadowTy); break; } } NonZeroChecks.push_back(Shadow); } else { - Shadow = DFS.ZeroShadow; + Shadow = DFS.getZeroShadow(V); } } return Shadow; @@ -1123,7 +1380,8 @@ Value *DFSanFunction::getShadow(Value *V) { void DFSanFunction::setShadow(Instruction *I, Value *Shadow) { assert(!ValShadowMap.count(I)); - assert(Shadow->getType() == DFS.ShadowTy); + assert(DFS.shouldTrackFieldsAndIndices() || + Shadow->getType() == DFS.PrimitiveShadowTy); ValShadowMap[I] = Shadow; } @@ -1140,47 +1398,60 @@ Value *DataFlowSanitizer::getShadowAddress(Value *Addr, Instruction *Pos) { IRB.CreateAnd(IRB.CreatePtrToInt(Addr, IntptrTy), IRB.CreatePtrToInt(ShadowPtrMaskValue, IntptrTy)), ShadowPtrMul), - ShadowPtrTy); + PrimitiveShadowPtrTy); +} + +Value *DFSanFunction::combineShadowsThenConvert(Type *T, Value *V1, Value *V2, + Instruction *Pos) { + Value *PrimitiveValue = combineShadows(V1, V2, Pos); + return expandFromPrimitiveShadow(T, PrimitiveValue, Pos); } // Generates IR to compute the union of the two given shadows, inserting it -// before Pos. Returns the computed union Value. +// before Pos. The combined value is with primitive type. Value *DFSanFunction::combineShadows(Value *V1, Value *V2, Instruction *Pos) { - if (V1 == DFS.ZeroShadow) - return V2; - if (V2 == DFS.ZeroShadow) - return V1; + if (DFS.isZeroShadow(V1)) + return collapseToPrimitiveShadow(V2, Pos); + if (DFS.isZeroShadow(V2)) + return collapseToPrimitiveShadow(V1, Pos); if (V1 == V2) - return V1; + return collapseToPrimitiveShadow(V1, Pos); auto V1Elems = ShadowElements.find(V1); auto V2Elems = ShadowElements.find(V2); if (V1Elems != ShadowElements.end() && V2Elems != ShadowElements.end()) { if (std::includes(V1Elems->second.begin(), V1Elems->second.end(), V2Elems->second.begin(), V2Elems->second.end())) { - return V1; + return collapseToPrimitiveShadow(V1, Pos); } else if (std::includes(V2Elems->second.begin(), V2Elems->second.end(), V1Elems->second.begin(), V1Elems->second.end())) { - return V2; + return collapseToPrimitiveShadow(V2, Pos); } } else if (V1Elems != ShadowElements.end()) { if (V1Elems->second.count(V2)) - return V1; + return collapseToPrimitiveShadow(V1, Pos); } else if (V2Elems != ShadowElements.end()) { if (V2Elems->second.count(V1)) - return V2; + return collapseToPrimitiveShadow(V2, Pos); } auto Key = std::make_pair(V1, V2); if (V1 > V2) std::swap(Key.first, Key.second); - CachedCombinedShadow &CCS = CachedCombinedShadows[Key]; + CachedShadow &CCS = CachedShadows[Key]; if (CCS.Block && DT.dominates(CCS.Block, Pos->getParent())) return CCS.Shadow; + // Converts inputs shadows to shadows with primitive types. + Value *PV1 = collapseToPrimitiveShadow(V1, Pos); + Value *PV2 = collapseToPrimitiveShadow(V2, Pos); + IRBuilder<> IRB(Pos); - if (AvoidNewBlocks) { - CallInst *Call = IRB.CreateCall(DFS.DFSanCheckedUnionFn, {V1, V2}); + if (ClFast16Labels) { + CCS.Block = Pos->getParent(); + CCS.Shadow = IRB.CreateOr(PV1, PV2); + } else if (AvoidNewBlocks) { + CallInst *Call = IRB.CreateCall(DFS.DFSanCheckedUnionFn, {PV1, PV2}); Call->addAttribute(AttributeList::ReturnIndex, Attribute::ZExt); Call->addParamAttr(0, Attribute::ZExt); Call->addParamAttr(1, Attribute::ZExt); @@ -1189,19 +1460,20 @@ Value *DFSanFunction::combineShadows(Value *V1, Value *V2, Instruction *Pos) { CCS.Shadow = Call; } else { BasicBlock *Head = Pos->getParent(); - Value *Ne = IRB.CreateICmpNE(V1, V2); + Value *Ne = IRB.CreateICmpNE(PV1, PV2); BranchInst *BI = cast<BranchInst>(SplitBlockAndInsertIfThen( Ne, Pos, /*Unreachable=*/false, DFS.ColdCallWeights, &DT)); IRBuilder<> ThenIRB(BI); - CallInst *Call = ThenIRB.CreateCall(DFS.DFSanUnionFn, {V1, V2}); + CallInst *Call = ThenIRB.CreateCall(DFS.DFSanUnionFn, {PV1, PV2}); Call->addAttribute(AttributeList::ReturnIndex, Attribute::ZExt); Call->addParamAttr(0, Attribute::ZExt); Call->addParamAttr(1, Attribute::ZExt); BasicBlock *Tail = BI->getSuccessor(0); - PHINode *Phi = PHINode::Create(DFS.ShadowTy, 2, "", &Tail->front()); + PHINode *Phi = + PHINode::Create(DFS.PrimitiveShadowTy, 2, "", &Tail->front()); Phi->addIncoming(Call, Call->getParent()); - Phi->addIncoming(V1, Head); + Phi->addIncoming(PV1, Head); CCS.Block = Tail; CCS.Shadow = Phi; @@ -1228,13 +1500,13 @@ Value *DFSanFunction::combineShadows(Value *V1, Value *V2, Instruction *Pos) { // the computed union Value. Value *DFSanFunction::combineOperandShadows(Instruction *Inst) { if (Inst->getNumOperands() == 0) - return DFS.ZeroShadow; + return DFS.getZeroShadow(Inst); Value *Shadow = getShadow(Inst->getOperand(0)); for (unsigned i = 1, n = Inst->getNumOperands(); i != n; ++i) { Shadow = combineShadows(Shadow, getShadow(Inst->getOperand(i)), Inst); } - return Shadow; + return expandFromPrimitiveShadow(Inst->getType(), Shadow, Inst); } Value *DFSanVisitor::visitOperandShadowInst(Instruction &I) { @@ -1244,20 +1516,21 @@ Value *DFSanVisitor::visitOperandShadowInst(Instruction &I) { } // Generates IR to load shadow corresponding to bytes [Addr, Addr+Size), where -// Addr has alignment Align, and take the union of each of those shadows. +// Addr has alignment Align, and take the union of each of those shadows. The +// returned shadow always has primitive type. Value *DFSanFunction::loadShadow(Value *Addr, uint64_t Size, uint64_t Align, Instruction *Pos) { if (AllocaInst *AI = dyn_cast<AllocaInst>(Addr)) { const auto i = AllocaShadowMap.find(AI); if (i != AllocaShadowMap.end()) { IRBuilder<> IRB(Pos); - return IRB.CreateLoad(DFS.ShadowTy, i->second); + return IRB.CreateLoad(DFS.PrimitiveShadowTy, i->second); } } const llvm::Align ShadowAlign(Align * DFS.ShadowWidthBytes); SmallVector<const Value *, 2> Objs; - GetUnderlyingObjects(Addr, Objs, Pos->getModule()->getDataLayout()); + getUnderlyingObjects(Addr, Objs); bool AllConstants = true; for (const Value *Obj : Objs) { if (isa<Function>(Obj) || isa<BlockAddress>(Obj)) @@ -1269,26 +1542,51 @@ Value *DFSanFunction::loadShadow(Value *Addr, uint64_t Size, uint64_t Align, break; } if (AllConstants) - return DFS.ZeroShadow; + return DFS.ZeroPrimitiveShadow; Value *ShadowAddr = DFS.getShadowAddress(Addr, Pos); switch (Size) { case 0: - return DFS.ZeroShadow; + return DFS.ZeroPrimitiveShadow; case 1: { - LoadInst *LI = new LoadInst(DFS.ShadowTy, ShadowAddr, "", Pos); + LoadInst *LI = new LoadInst(DFS.PrimitiveShadowTy, ShadowAddr, "", Pos); LI->setAlignment(ShadowAlign); return LI; } case 2: { IRBuilder<> IRB(Pos); - Value *ShadowAddr1 = IRB.CreateGEP(DFS.ShadowTy, ShadowAddr, + Value *ShadowAddr1 = IRB.CreateGEP(DFS.PrimitiveShadowTy, ShadowAddr, ConstantInt::get(DFS.IntptrTy, 1)); return combineShadows( - IRB.CreateAlignedLoad(DFS.ShadowTy, ShadowAddr, ShadowAlign), - IRB.CreateAlignedLoad(DFS.ShadowTy, ShadowAddr1, ShadowAlign), Pos); + IRB.CreateAlignedLoad(DFS.PrimitiveShadowTy, ShadowAddr, ShadowAlign), + IRB.CreateAlignedLoad(DFS.PrimitiveShadowTy, ShadowAddr1, ShadowAlign), + Pos); } } + + if (ClFast16Labels && Size % (64 / DFS.ShadowWidthBits) == 0) { + // First OR all the WideShadows, then OR individual shadows within the + // combined WideShadow. This is fewer instructions than ORing shadows + // individually. + IRBuilder<> IRB(Pos); + Value *WideAddr = + IRB.CreateBitCast(ShadowAddr, Type::getInt64PtrTy(*DFS.Ctx)); + Value *CombinedWideShadow = + IRB.CreateAlignedLoad(IRB.getInt64Ty(), WideAddr, ShadowAlign); + for (uint64_t Ofs = 64 / DFS.ShadowWidthBits; Ofs != Size; + Ofs += 64 / DFS.ShadowWidthBits) { + WideAddr = IRB.CreateGEP(Type::getInt64Ty(*DFS.Ctx), WideAddr, + ConstantInt::get(DFS.IntptrTy, 1)); + Value *NextWideShadow = + IRB.CreateAlignedLoad(IRB.getInt64Ty(), WideAddr, ShadowAlign); + CombinedWideShadow = IRB.CreateOr(CombinedWideShadow, NextWideShadow); + } + for (unsigned Width = 32; Width >= DFS.ShadowWidthBits; Width >>= 1) { + Value *ShrShadow = IRB.CreateLShr(CombinedWideShadow, Width); + CombinedWideShadow = IRB.CreateOr(CombinedWideShadow, ShrShadow); + } + return IRB.CreateTrunc(CombinedWideShadow, DFS.PrimitiveShadowTy); + } if (!AvoidNewBlocks && Size % (64 / DFS.ShadowWidthBits) == 0) { // Fast path for the common case where each byte has identical shadow: load // shadow 64 bits at a time, fall out to a __dfsan_union_load call if any @@ -1307,7 +1605,7 @@ Value *DFSanFunction::loadShadow(Value *Addr, uint64_t Size, uint64_t Align, IRB.CreateBitCast(ShadowAddr, Type::getInt64PtrTy(*DFS.Ctx)); Value *WideShadow = IRB.CreateAlignedLoad(IRB.getInt64Ty(), WideAddr, ShadowAlign); - Value *TruncShadow = IRB.CreateTrunc(WideShadow, DFS.ShadowTy); + Value *TruncShadow = IRB.CreateTrunc(WideShadow, DFS.PrimitiveShadowTy); Value *ShlShadow = IRB.CreateShl(WideShadow, DFS.ShadowWidthBits); Value *ShrShadow = IRB.CreateLShr(WideShadow, 64 - DFS.ShadowWidthBits); Value *RotShadow = IRB.CreateOr(ShlShadow, ShrShadow); @@ -1348,15 +1646,18 @@ Value *DFSanFunction::loadShadow(Value *Addr, uint64_t Size, uint64_t Align, LastBr->setSuccessor(0, Tail); FallbackIRB.CreateBr(Tail); - PHINode *Shadow = PHINode::Create(DFS.ShadowTy, 2, "", &Tail->front()); + PHINode *Shadow = + PHINode::Create(DFS.PrimitiveShadowTy, 2, "", &Tail->front()); Shadow->addIncoming(FallbackCall, FallbackBB); Shadow->addIncoming(TruncShadow, LastBr->getParent()); return Shadow; } IRBuilder<> IRB(Pos); + FunctionCallee &UnionLoadFn = + ClFast16Labels ? DFS.DFSanUnionLoadFast16LabelsFn : DFS.DFSanUnionLoadFn; CallInst *FallbackCall = IRB.CreateCall( - DFS.DFSanUnionLoadFn, {ShadowAddr, ConstantInt::get(DFS.IntptrTy, Size)}); + UnionLoadFn, {ShadowAddr, ConstantInt::get(DFS.IntptrTy, Size)}); FallbackCall->addAttribute(AttributeList::ReturnIndex, Attribute::ZExt); return FallbackCall; } @@ -1365,34 +1666,39 @@ void DFSanVisitor::visitLoadInst(LoadInst &LI) { auto &DL = LI.getModule()->getDataLayout(); uint64_t Size = DL.getTypeStoreSize(LI.getType()); if (Size == 0) { - DFSF.setShadow(&LI, DFSF.DFS.ZeroShadow); + DFSF.setShadow(&LI, DFSF.DFS.getZeroShadow(&LI)); return; } Align Alignment = ClPreserveAlignment ? LI.getAlign() : Align(1); - Value *Shadow = + Value *PrimitiveShadow = DFSF.loadShadow(LI.getPointerOperand(), Size, Alignment.value(), &LI); if (ClCombinePointerLabelsOnLoad) { Value *PtrShadow = DFSF.getShadow(LI.getPointerOperand()); - Shadow = DFSF.combineShadows(Shadow, PtrShadow, &LI); + PrimitiveShadow = DFSF.combineShadows(PrimitiveShadow, PtrShadow, &LI); } - if (Shadow != DFSF.DFS.ZeroShadow) - DFSF.NonZeroChecks.push_back(Shadow); + if (!DFSF.DFS.isZeroShadow(PrimitiveShadow)) + DFSF.NonZeroChecks.push_back(PrimitiveShadow); + Value *Shadow = + DFSF.expandFromPrimitiveShadow(LI.getType(), PrimitiveShadow, &LI); DFSF.setShadow(&LI, Shadow); if (ClEventCallbacks) { IRBuilder<> IRB(&LI); - IRB.CreateCall(DFSF.DFS.DFSanLoadCallbackFn, Shadow); + Value *Addr8 = IRB.CreateBitCast(LI.getPointerOperand(), DFSF.DFS.Int8Ptr); + IRB.CreateCall(DFSF.DFS.DFSanLoadCallbackFn, {PrimitiveShadow, Addr8}); } } -void DFSanFunction::storeShadow(Value *Addr, uint64_t Size, Align Alignment, - Value *Shadow, Instruction *Pos) { +void DFSanFunction::storePrimitiveShadow(Value *Addr, uint64_t Size, + Align Alignment, + Value *PrimitiveShadow, + Instruction *Pos) { if (AllocaInst *AI = dyn_cast<AllocaInst>(Addr)) { const auto i = AllocaShadowMap.find(AI); if (i != AllocaShadowMap.end()) { IRBuilder<> IRB(Pos); - IRB.CreateStore(Shadow, i->second); + IRB.CreateStore(PrimitiveShadow, i->second); return; } } @@ -1400,7 +1706,7 @@ void DFSanFunction::storeShadow(Value *Addr, uint64_t Size, Align Alignment, const Align ShadowAlign(Alignment.value() * DFS.ShadowWidthBytes); IRBuilder<> IRB(Pos); Value *ShadowAddr = DFS.getShadowAddress(Addr, Pos); - if (Shadow == DFS.ZeroShadow) { + if (DFS.isZeroShadow(PrimitiveShadow)) { IntegerType *ShadowTy = IntegerType::get(*DFS.Ctx, Size * DFS.ShadowWidthBits); Value *ExtZeroShadow = ConstantInt::get(ShadowTy, 0); @@ -1413,11 +1719,13 @@ void DFSanFunction::storeShadow(Value *Addr, uint64_t Size, Align Alignment, const unsigned ShadowVecSize = 128 / DFS.ShadowWidthBits; uint64_t Offset = 0; if (Size >= ShadowVecSize) { - auto *ShadowVecTy = FixedVectorType::get(DFS.ShadowTy, ShadowVecSize); + auto *ShadowVecTy = + FixedVectorType::get(DFS.PrimitiveShadowTy, ShadowVecSize); Value *ShadowVec = UndefValue::get(ShadowVecTy); for (unsigned i = 0; i != ShadowVecSize; ++i) { ShadowVec = IRB.CreateInsertElement( - ShadowVec, Shadow, ConstantInt::get(Type::getInt32Ty(*DFS.Ctx), i)); + ShadowVec, PrimitiveShadow, + ConstantInt::get(Type::getInt32Ty(*DFS.Ctx), i)); } Value *ShadowVecAddr = IRB.CreateBitCast(ShadowAddr, PointerType::getUnqual(ShadowVecTy)); @@ -1432,8 +1740,8 @@ void DFSanFunction::storeShadow(Value *Addr, uint64_t Size, Align Alignment, } while (Size > 0) { Value *CurShadowAddr = - IRB.CreateConstGEP1_32(DFS.ShadowTy, ShadowAddr, Offset); - IRB.CreateAlignedStore(Shadow, CurShadowAddr, ShadowAlign); + IRB.CreateConstGEP1_32(DFS.PrimitiveShadowTy, ShadowAddr, Offset); + IRB.CreateAlignedStore(PrimitiveShadow, CurShadowAddr, ShadowAlign); --Size; ++Offset; } @@ -1448,14 +1756,19 @@ void DFSanVisitor::visitStoreInst(StoreInst &SI) { const Align Alignment = ClPreserveAlignment ? SI.getAlign() : Align(1); Value* Shadow = DFSF.getShadow(SI.getValueOperand()); + Value *PrimitiveShadow; if (ClCombinePointerLabelsOnStore) { Value *PtrShadow = DFSF.getShadow(SI.getPointerOperand()); - Shadow = DFSF.combineShadows(Shadow, PtrShadow, &SI); + PrimitiveShadow = DFSF.combineShadows(Shadow, PtrShadow, &SI); + } else { + PrimitiveShadow = DFSF.collapseToPrimitiveShadow(Shadow, &SI); } - DFSF.storeShadow(SI.getPointerOperand(), Size, Alignment, Shadow, &SI); + DFSF.storePrimitiveShadow(SI.getPointerOperand(), Size, Alignment, + PrimitiveShadow, &SI); if (ClEventCallbacks) { IRBuilder<> IRB(&SI); - IRB.CreateCall(DFSF.DFS.DFSanStoreCallbackFn, Shadow); + Value *Addr8 = IRB.CreateBitCast(SI.getPointerOperand(), DFSF.DFS.Int8Ptr); + IRB.CreateCall(DFSF.DFS.DFSanStoreCallbackFn, {PrimitiveShadow, Addr8}); } } @@ -1494,11 +1807,29 @@ void DFSanVisitor::visitShuffleVectorInst(ShuffleVectorInst &I) { } void DFSanVisitor::visitExtractValueInst(ExtractValueInst &I) { - visitOperandShadowInst(I); + if (!DFSF.DFS.shouldTrackFieldsAndIndices()) { + visitOperandShadowInst(I); + return; + } + + IRBuilder<> IRB(&I); + Value *Agg = I.getAggregateOperand(); + Value *AggShadow = DFSF.getShadow(Agg); + Value *ResShadow = IRB.CreateExtractValue(AggShadow, I.getIndices()); + DFSF.setShadow(&I, ResShadow); } void DFSanVisitor::visitInsertValueInst(InsertValueInst &I) { - visitOperandShadowInst(I); + if (!DFSF.DFS.shouldTrackFieldsAndIndices()) { + visitOperandShadowInst(I); + return; + } + + IRBuilder<> IRB(&I); + Value *AggShadow = DFSF.getShadow(I.getAggregateOperand()); + Value *InsShadow = DFSF.getShadow(I.getInsertedValueOperand()); + Value *Res = IRB.CreateInsertValue(AggShadow, InsShadow, I.getIndices()); + DFSF.setShadow(&I, Res); } void DFSanVisitor::visitAllocaInst(AllocaInst &I) { @@ -1517,31 +1848,32 @@ void DFSanVisitor::visitAllocaInst(AllocaInst &I) { } if (AllLoadsStores) { IRBuilder<> IRB(&I); - DFSF.AllocaShadowMap[&I] = IRB.CreateAlloca(DFSF.DFS.ShadowTy); + DFSF.AllocaShadowMap[&I] = IRB.CreateAlloca(DFSF.DFS.PrimitiveShadowTy); } - DFSF.setShadow(&I, DFSF.DFS.ZeroShadow); + DFSF.setShadow(&I, DFSF.DFS.ZeroPrimitiveShadow); } void DFSanVisitor::visitSelectInst(SelectInst &I) { Value *CondShadow = DFSF.getShadow(I.getCondition()); Value *TrueShadow = DFSF.getShadow(I.getTrueValue()); Value *FalseShadow = DFSF.getShadow(I.getFalseValue()); + Value *ShadowSel = nullptr; if (isa<VectorType>(I.getCondition()->getType())) { - DFSF.setShadow( - &I, - DFSF.combineShadows( - CondShadow, DFSF.combineShadows(TrueShadow, FalseShadow, &I), &I)); + ShadowSel = DFSF.combineShadowsThenConvert(I.getType(), TrueShadow, + FalseShadow, &I); } else { - Value *ShadowSel; if (TrueShadow == FalseShadow) { ShadowSel = TrueShadow; } else { ShadowSel = SelectInst::Create(I.getCondition(), TrueShadow, FalseShadow, "", &I); } - DFSF.setShadow(&I, DFSF.combineShadows(CondShadow, ShadowSel, &I)); } + DFSF.setShadow(&I, ClTrackSelectControlFlow + ? DFSF.combineShadowsThenConvert( + I.getType(), CondShadow, ShadowSel, &I) + : ShadowSel); } void DFSanVisitor::visitMemSetInst(MemSetInst &I) { @@ -1585,7 +1917,15 @@ void DFSanVisitor::visitReturnInst(ReturnInst &RI) { case DataFlowSanitizer::IA_TLS: { Value *S = DFSF.getShadow(RI.getReturnValue()); IRBuilder<> IRB(&RI); - IRB.CreateStore(S, DFSF.getRetvalTLS()); + Type *RT = DFSF.F->getFunctionType()->getReturnType(); + unsigned Size = + getDataLayout().getTypeAllocSize(DFSF.DFS.getShadowTy(RT)); + if (Size <= kRetvalTLSSize) { + // If the size overflows, stores nothing. At callsite, oversized return + // shadows are set to zero. + IRB.CreateAlignedStore(S, DFSF.getRetvalTLS(RT, IRB), + kShadowTLSAlignment); + } break; } case DataFlowSanitizer::IA_Args: { @@ -1625,11 +1965,11 @@ void DFSanVisitor::visitCallBase(CallBase &CB) { CB.setCalledFunction(F); IRB.CreateCall(DFSF.DFS.DFSanUnimplementedFn, IRB.CreateGlobalStringPtr(F->getName())); - DFSF.setShadow(&CB, DFSF.DFS.ZeroShadow); + DFSF.setShadow(&CB, DFSF.DFS.getZeroShadow(&CB)); return; case DataFlowSanitizer::WK_Discard: CB.setCalledFunction(F); - DFSF.setShadow(&CB, DFSF.DFS.ZeroShadow); + DFSF.setShadow(&CB, DFSF.DFS.getZeroShadow(&CB)); return; case DataFlowSanitizer::WK_Functional: CB.setCalledFunction(F); @@ -1681,10 +2021,11 @@ void DFSanVisitor::visitCallBase(CallBase &CB) { i = CB.arg_begin(); const unsigned ShadowArgStart = Args.size(); for (unsigned n = FT->getNumParams(); n != 0; ++i, --n) - Args.push_back(DFSF.getShadow(*i)); + Args.push_back( + DFSF.collapseToPrimitiveShadow(DFSF.getShadow(*i), &CB)); if (FT->isVarArg()) { - auto *LabelVATy = ArrayType::get(DFSF.DFS.ShadowTy, + auto *LabelVATy = ArrayType::get(DFSF.DFS.PrimitiveShadowTy, CB.arg_size() - FT->getNumParams()); auto *LabelVAAlloca = new AllocaInst( LabelVATy, getDataLayout().getAllocaAddrSpace(), @@ -1692,7 +2033,9 @@ void DFSanVisitor::visitCallBase(CallBase &CB) { for (unsigned n = 0; i != CB.arg_end(); ++i, ++n) { auto LabelVAPtr = IRB.CreateStructGEP(LabelVATy, LabelVAAlloca, n); - IRB.CreateStore(DFSF.getShadow(*i), LabelVAPtr); + IRB.CreateStore( + DFSF.collapseToPrimitiveShadow(DFSF.getShadow(*i), &CB), + LabelVAPtr); } Args.push_back(IRB.CreateStructGEP(LabelVATy, LabelVAAlloca, 0)); @@ -1701,9 +2044,9 @@ void DFSanVisitor::visitCallBase(CallBase &CB) { if (!FT->getReturnType()->isVoidTy()) { if (!DFSF.LabelReturnAlloca) { DFSF.LabelReturnAlloca = - new AllocaInst(DFSF.DFS.ShadowTy, - getDataLayout().getAllocaAddrSpace(), - "labelreturn", &DFSF.F->getEntryBlock().front()); + new AllocaInst(DFSF.DFS.PrimitiveShadowTy, + getDataLayout().getAllocaAddrSpace(), + "labelreturn", &DFSF.F->getEntryBlock().front()); } Args.push_back(DFSF.LabelReturnAlloca); } @@ -1718,17 +2061,19 @@ void DFSanVisitor::visitCallBase(CallBase &CB) { // Update the parameter attributes of the custom call instruction to // zero extend the shadow parameters. This is required for targets - // which consider ShadowTy an illegal type. + // which consider PrimitiveShadowTy an illegal type. for (unsigned n = 0; n < FT->getNumParams(); n++) { const unsigned ArgNo = ShadowArgStart + n; - if (CustomCI->getArgOperand(ArgNo)->getType() == DFSF.DFS.ShadowTy) + if (CustomCI->getArgOperand(ArgNo)->getType() == + DFSF.DFS.PrimitiveShadowTy) CustomCI->addParamAttr(ArgNo, Attribute::ZExt); } if (!FT->getReturnType()->isVoidTy()) { - LoadInst *LabelLoad = - IRB.CreateLoad(DFSF.DFS.ShadowTy, DFSF.LabelReturnAlloca); - DFSF.setShadow(CustomCI, LabelLoad); + LoadInst *LabelLoad = IRB.CreateLoad(DFSF.DFS.PrimitiveShadowTy, + DFSF.LabelReturnAlloca); + DFSF.setShadow(CustomCI, DFSF.expandFromPrimitiveShadow( + FT->getReturnType(), LabelLoad, &CB)); } CI->replaceAllUsesWith(CustomCI); @@ -1741,9 +2086,20 @@ void DFSanVisitor::visitCallBase(CallBase &CB) { FunctionType *FT = CB.getFunctionType(); if (DFSF.DFS.getInstrumentedABI() == DataFlowSanitizer::IA_TLS) { - for (unsigned i = 0, n = FT->getNumParams(); i != n; ++i) { - IRB.CreateStore(DFSF.getShadow(CB.getArgOperand(i)), - DFSF.getArgTLS(i, &CB)); + unsigned ArgOffset = 0; + const DataLayout &DL = getDataLayout(); + for (unsigned I = 0, N = FT->getNumParams(); I != N; ++I) { + unsigned Size = + DL.getTypeAllocSize(DFSF.DFS.getShadowTy(FT->getParamType(I))); + // Stop storing if arguments' size overflows. Inside a function, arguments + // after overflow have zero shadow values. + if (ArgOffset + Size > kArgTLSSize) + break; + IRB.CreateAlignedStore( + DFSF.getShadow(CB.getArgOperand(I)), + DFSF.getArgTLS(FT->getParamType(I), ArgOffset, IRB), + kShadowTLSAlignment); + ArgOffset += alignTo(Size, kShadowTLSAlignment); } } @@ -1764,10 +2120,19 @@ void DFSanVisitor::visitCallBase(CallBase &CB) { if (DFSF.DFS.getInstrumentedABI() == DataFlowSanitizer::IA_TLS) { IRBuilder<> NextIRB(Next); - LoadInst *LI = NextIRB.CreateLoad(DFSF.DFS.ShadowTy, DFSF.getRetvalTLS()); - DFSF.SkipInsts.insert(LI); - DFSF.setShadow(&CB, LI); - DFSF.NonZeroChecks.push_back(LI); + const DataLayout &DL = getDataLayout(); + unsigned Size = DL.getTypeAllocSize(DFSF.DFS.getShadowTy(&CB)); + if (Size > kRetvalTLSSize) { + // Set overflowed return shadow to be zero. + DFSF.setShadow(&CB, DFSF.DFS.getZeroShadow(&CB)); + } else { + LoadInst *LI = NextIRB.CreateAlignedLoad( + DFSF.DFS.getShadowTy(&CB), DFSF.getRetvalTLS(CB.getType(), NextIRB), + kShadowTLSAlignment, "_dfsret"); + DFSF.SkipInsts.insert(LI); + DFSF.setShadow(&CB, LI); + DFSF.NonZeroChecks.push_back(LI); + } } } @@ -1789,7 +2154,8 @@ void DFSanVisitor::visitCallBase(CallBase &CB) { if (FT->isVarArg()) { unsigned VarArgSize = CB.arg_size() - FT->getNumParams(); - ArrayType *VarArgArrayTy = ArrayType::get(DFSF.DFS.ShadowTy, VarArgSize); + ArrayType *VarArgArrayTy = + ArrayType::get(DFSF.DFS.PrimitiveShadowTy, VarArgSize); AllocaInst *VarArgShadow = new AllocaInst(VarArgArrayTy, getDataLayout().getAllocaAddrSpace(), "", &DFSF.F->getEntryBlock().front()); @@ -1830,11 +2196,12 @@ void DFSanVisitor::visitCallBase(CallBase &CB) { } void DFSanVisitor::visitPHINode(PHINode &PN) { + Type *ShadowTy = DFSF.DFS.getShadowTy(&PN); PHINode *ShadowPN = - PHINode::Create(DFSF.DFS.ShadowTy, PN.getNumIncomingValues(), "", &PN); + PHINode::Create(ShadowTy, PN.getNumIncomingValues(), "", &PN); // Give the shadow phi node valid predecessors to fool SplitEdge into working. - Value *UndefShadow = UndefValue::get(DFSF.DFS.ShadowTy); + Value *UndefShadow = UndefValue::get(ShadowTy); for (PHINode::block_iterator i = PN.block_begin(), e = PN.block_end(); i != e; ++i) { ShadowPN->addIncoming(UndefShadow, *i); @@ -1843,3 +2210,39 @@ void DFSanVisitor::visitPHINode(PHINode &PN) { DFSF.PHIFixups.push_back(std::make_pair(&PN, ShadowPN)); DFSF.setShadow(&PN, ShadowPN); } + +namespace { +class DataFlowSanitizerLegacyPass : public ModulePass { +private: + std::vector<std::string> ABIListFiles; + +public: + static char ID; + + DataFlowSanitizerLegacyPass( + const std::vector<std::string> &ABIListFiles = std::vector<std::string>()) + : ModulePass(ID), ABIListFiles(ABIListFiles) {} + + bool runOnModule(Module &M) override { + return DataFlowSanitizer(ABIListFiles).runImpl(M); + } +}; +} // namespace + +char DataFlowSanitizerLegacyPass::ID; + +INITIALIZE_PASS(DataFlowSanitizerLegacyPass, "dfsan", + "DataFlowSanitizer: dynamic data flow analysis.", false, false) + +ModulePass *llvm::createDataFlowSanitizerLegacyPassPass( + const std::vector<std::string> &ABIListFiles) { + return new DataFlowSanitizerLegacyPass(ABIListFiles); +} + +PreservedAnalyses DataFlowSanitizerPass::run(Module &M, + ModuleAnalysisManager &AM) { + if (DataFlowSanitizer(ABIListFiles).runImpl(M)) { + return PreservedAnalyses::none(); + } + return PreservedAnalyses::all(); +} |