diff options
Diffstat (limited to 'lib/CodeGen/CGLoopInfo.cpp')
-rw-r--r-- | lib/CodeGen/CGLoopInfo.cpp | 60 |
1 files changed, 52 insertions, 8 deletions
diff --git a/lib/CodeGen/CGLoopInfo.cpp b/lib/CodeGen/CGLoopInfo.cpp index b2bc42bfa013..c21d4feee7a8 100644 --- a/lib/CodeGen/CGLoopInfo.cpp +++ b/lib/CodeGen/CGLoopInfo.cpp @@ -218,6 +218,7 @@ LoopInfo::createLoopVectorizeMetadata(const LoopAttributes &Attrs, if (Attrs.VectorizeEnable == LoopAttributes::Disable) Enabled = false; else if (Attrs.VectorizeEnable != LoopAttributes::Unspecified || + Attrs.VectorizePredicateEnable != LoopAttributes::Unspecified || Attrs.InterleaveCount != 0 || Attrs.VectorizeWidth != 0) Enabled = true; @@ -251,8 +252,32 @@ LoopInfo::createLoopVectorizeMetadata(const LoopAttributes &Attrs, Args.push_back(TempNode.get()); Args.append(LoopProperties.begin(), LoopProperties.end()); + // Setting vectorize.predicate + bool IsVectorPredicateEnabled = false; + if (Attrs.VectorizePredicateEnable != LoopAttributes::Unspecified && + Attrs.VectorizeEnable != LoopAttributes::Disable && + Attrs.VectorizeWidth < 1) { + + IsVectorPredicateEnabled = + (Attrs.VectorizePredicateEnable == LoopAttributes::Enable); + + Metadata *Vals[] = { + MDString::get(Ctx, "llvm.loop.vectorize.predicate.enable"), + ConstantAsMetadata::get(ConstantInt::get(llvm::Type::getInt1Ty(Ctx), + IsVectorPredicateEnabled))}; + Args.push_back(MDNode::get(Ctx, Vals)); + } + // Setting vectorize.width if (Attrs.VectorizeWidth > 0) { + // This implies vectorize.enable = true, but only add it when it is not + // already enabled. + if (Attrs.VectorizeEnable != LoopAttributes::Enable) + Args.push_back( + MDNode::get(Ctx, {MDString::get(Ctx, "llvm.loop.vectorize.enable"), + ConstantAsMetadata::get(ConstantInt::get( + llvm::Type::getInt1Ty(Ctx), 1))})); + Metadata *Vals[] = { MDString::get(Ctx, "llvm.loop.vectorize.width"), ConstantAsMetadata::get(ConstantInt::get(llvm::Type::getInt32Ty(Ctx), @@ -270,12 +295,15 @@ LoopInfo::createLoopVectorizeMetadata(const LoopAttributes &Attrs, } // Setting vectorize.enable - if (Attrs.VectorizeEnable != LoopAttributes::Unspecified) { + if (Attrs.VectorizeEnable != LoopAttributes::Unspecified || + IsVectorPredicateEnabled) { Metadata *Vals[] = { MDString::get(Ctx, "llvm.loop.vectorize.enable"), ConstantAsMetadata::get(ConstantInt::get( llvm::Type::getInt1Ty(Ctx), - (Attrs.VectorizeEnable == LoopAttributes::Enable)))}; + IsVectorPredicateEnabled + ? true + : (Attrs.VectorizeEnable == LoopAttributes::Enable)))}; Args.push_back(MDNode::get(Ctx, Vals)); } @@ -411,7 +439,8 @@ MDNode *LoopInfo::createMetadata( LoopAttributes::LoopAttributes(bool IsParallel) : IsParallel(IsParallel), VectorizeEnable(LoopAttributes::Unspecified), UnrollEnable(LoopAttributes::Unspecified), - UnrollAndJamEnable(LoopAttributes::Unspecified), VectorizeWidth(0), + UnrollAndJamEnable(LoopAttributes::Unspecified), + VectorizePredicateEnable(LoopAttributes::Unspecified), VectorizeWidth(0), InterleaveCount(0), UnrollCount(0), UnrollAndJamCount(0), DistributeEnable(LoopAttributes::Unspecified), PipelineDisabled(false), PipelineInitiationInterval(0) {} @@ -425,6 +454,7 @@ void LoopAttributes::clear() { VectorizeEnable = LoopAttributes::Unspecified; UnrollEnable = LoopAttributes::Unspecified; UnrollAndJamEnable = LoopAttributes::Unspecified; + VectorizePredicateEnable = LoopAttributes::Unspecified; DistributeEnable = LoopAttributes::Unspecified; PipelineDisabled = false; PipelineInitiationInterval = 0; @@ -446,6 +476,7 @@ LoopInfo::LoopInfo(BasicBlock *Header, const LoopAttributes &Attrs, Attrs.InterleaveCount == 0 && Attrs.UnrollCount == 0 && Attrs.UnrollAndJamCount == 0 && !Attrs.PipelineDisabled && Attrs.PipelineInitiationInterval == 0 && + Attrs.VectorizePredicateEnable == LoopAttributes::Unspecified && Attrs.VectorizeEnable == LoopAttributes::Unspecified && Attrs.UnrollEnable == LoopAttributes::Unspecified && Attrs.UnrollAndJamEnable == LoopAttributes::Unspecified && @@ -480,6 +511,7 @@ void LoopInfo::finish() { BeforeJam.InterleaveCount = Attrs.InterleaveCount; BeforeJam.VectorizeEnable = Attrs.VectorizeEnable; BeforeJam.DistributeEnable = Attrs.DistributeEnable; + BeforeJam.VectorizePredicateEnable = Attrs.VectorizePredicateEnable; switch (Attrs.UnrollEnable) { case LoopAttributes::Unspecified: @@ -495,6 +527,7 @@ void LoopInfo::finish() { break; } + AfterJam.VectorizePredicateEnable = Attrs.VectorizePredicateEnable; AfterJam.UnrollCount = Attrs.UnrollCount; AfterJam.PipelineDisabled = Attrs.PipelineDisabled; AfterJam.PipelineInitiationInterval = Attrs.PipelineInitiationInterval; @@ -516,6 +549,7 @@ void LoopInfo::finish() { // add it manually. SmallVector<Metadata *, 1> BeforeLoopProperties; if (BeforeJam.VectorizeEnable != LoopAttributes::Unspecified || + BeforeJam.VectorizePredicateEnable != LoopAttributes::Unspecified || BeforeJam.InterleaveCount != 0 || BeforeJam.VectorizeWidth != 0) BeforeLoopProperties.push_back( MDNode::get(Ctx, MDString::get(Ctx, "llvm.loop.isvectorized"))); @@ -537,8 +571,9 @@ void LoopInfo::finish() { void LoopInfoStack::push(BasicBlock *Header, const llvm::DebugLoc &StartLoc, const llvm::DebugLoc &EndLoc) { - Active.push_back(LoopInfo(Header, StagedAttrs, StartLoc, EndLoc, - Active.empty() ? nullptr : &Active.back())); + Active.emplace_back( + new LoopInfo(Header, StagedAttrs, StartLoc, EndLoc, + Active.empty() ? nullptr : Active.back().get())); // Clear the attributes so nested loops do not inherit them. StagedAttrs.clear(); } @@ -603,6 +638,9 @@ void LoopInfoStack::push(BasicBlock *Header, clang::ASTContext &Ctx, case LoopHintAttr::UnrollAndJam: setUnrollAndJamState(LoopAttributes::Disable); break; + case LoopHintAttr::VectorizePredicate: + setVectorizePredicateState(LoopAttributes::Disable); + break; case LoopHintAttr::Distribute: setDistributeState(false); break; @@ -630,6 +668,9 @@ void LoopInfoStack::push(BasicBlock *Header, clang::ASTContext &Ctx, case LoopHintAttr::UnrollAndJam: setUnrollAndJamState(LoopAttributes::Enable); break; + case LoopHintAttr::VectorizePredicate: + setVectorizePredicateState(LoopAttributes::Enable); + break; case LoopHintAttr::Distribute: setDistributeState(true); break; @@ -653,6 +694,7 @@ void LoopInfoStack::push(BasicBlock *Header, clang::ASTContext &Ctx, break; case LoopHintAttr::Unroll: case LoopHintAttr::UnrollAndJam: + case LoopHintAttr::VectorizePredicate: case LoopHintAttr::UnrollCount: case LoopHintAttr::UnrollAndJamCount: case LoopHintAttr::VectorizeWidth: @@ -681,6 +723,7 @@ void LoopInfoStack::push(BasicBlock *Header, clang::ASTContext &Ctx, case LoopHintAttr::Distribute: case LoopHintAttr::PipelineDisabled: case LoopHintAttr::PipelineInitiationInterval: + case LoopHintAttr::VectorizePredicate: llvm_unreachable("Options cannot be used with 'full' hint."); break; } @@ -704,6 +747,7 @@ void LoopInfoStack::push(BasicBlock *Header, clang::ASTContext &Ctx, break; case LoopHintAttr::Unroll: case LoopHintAttr::UnrollAndJam: + case LoopHintAttr::VectorizePredicate: case LoopHintAttr::Vectorize: case LoopHintAttr::Interleave: case LoopHintAttr::Distribute: @@ -721,16 +765,16 @@ void LoopInfoStack::push(BasicBlock *Header, clang::ASTContext &Ctx, void LoopInfoStack::pop() { assert(!Active.empty() && "No active loops to pop"); - Active.back().finish(); + Active.back()->finish(); Active.pop_back(); } void LoopInfoStack::InsertHelper(Instruction *I) const { if (I->mayReadOrWriteMemory()) { SmallVector<Metadata *, 4> AccessGroups; - for (const LoopInfo &AL : Active) { + for (const auto &AL : Active) { // Here we assume that every loop that has an access group is parallel. - if (MDNode *Group = AL.getAccessGroup()) + if (MDNode *Group = AL->getAccessGroup()) AccessGroups.push_back(Group); } MDNode *UnionMD = nullptr; |