12 #include "llvm/IR/BasicBlock.h" 13 #include "llvm/IR/CFG.h" 14 #include "llvm/IR/Constants.h" 15 #include "llvm/IR/InstrTypes.h" 16 #include "llvm/IR/Instructions.h" 17 #include "llvm/IR/Metadata.h" 23 LLVMContext &Ctx = Header->getContext();
25 TempMDTuple TempNode = MDNode::getTemporary(Ctx, None);
26 NewLoopProperties.push_back(TempNode.get());
27 NewLoopProperties.append(LoopProperties.begin(), LoopProperties.end());
29 MDNode *LoopID = MDNode::getDistinct(Ctx, NewLoopProperties);
30 LoopID->replaceOperandWith(0, LoopID);
34 MDNode *LoopInfo::createPipeliningMetadata(
const LoopAttributes &Attrs,
36 bool &HasUserTransforms) {
37 LLVMContext &Ctx = Header->getContext();
45 if (Enabled !=
true) {
47 if (Enabled ==
false) {
48 NewLoopProperties.append(LoopProperties.begin(), LoopProperties.end());
49 NewLoopProperties.push_back(
50 MDNode::get(Ctx, {MDString::get(Ctx,
"llvm.loop.pipeline.disable"),
51 ConstantAsMetadata::get(ConstantInt::get(
52 llvm::Type::getInt1Ty(Ctx), 1))}));
53 LoopProperties = NewLoopProperties;
55 return createLoopPropertiesMetadata(LoopProperties);
59 TempMDTuple TempNode = MDNode::getTemporary(Ctx, None);
60 Args.push_back(TempNode.get());
61 Args.append(LoopProperties.begin(), LoopProperties.end());
65 MDString::get(Ctx,
"llvm.loop.pipeline.initiationinterval"),
66 ConstantAsMetadata::get(ConstantInt::get(
68 Args.push_back(MDNode::get(Ctx, Vals));
73 MDNode *LoopID = MDNode::getDistinct(Ctx, Args);
74 LoopID->replaceOperandWith(0, LoopID);
75 HasUserTransforms =
true;
82 bool &HasUserTransforms) {
83 LLVMContext &Ctx = Header->getContext();
94 if (Enabled !=
true) {
97 return createPipeliningMetadata(Attrs, LoopProperties, HasUserTransforms);
103 FollowupLoopProperties.append(LoopProperties.begin(), LoopProperties.end());
106 FollowupLoopProperties.push_back(
107 MDNode::get(Ctx, MDString::get(Ctx,
"llvm.loop.unroll.disable")));
109 bool FollowupHasTransforms =
false;
110 MDNode *Followup = createPipeliningMetadata(Attrs, FollowupLoopProperties,
111 FollowupHasTransforms);
114 TempMDTuple TempNode = MDNode::getTemporary(Ctx, None);
115 Args.push_back(TempNode.get());
116 Args.append(LoopProperties.begin(), LoopProperties.end());
120 Metadata *Vals[] = {MDString::get(Ctx,
"llvm.loop.unroll.count"),
121 ConstantAsMetadata::get(ConstantInt::get(
123 Args.push_back(MDNode::get(Ctx, Vals));
128 Metadata *Vals[] = {MDString::get(Ctx,
"llvm.loop.unroll.enable")};
129 Args.push_back(MDNode::get(Ctx, Vals));
132 if (FollowupHasTransforms)
133 Args.push_back(MDNode::get(
134 Ctx, {MDString::get(Ctx,
"llvm.loop.unroll.followup_all"), Followup}));
136 MDNode *LoopID = MDNode::getDistinct(Ctx, Args);
137 LoopID->replaceOperandWith(0, LoopID);
138 HasUserTransforms =
true;
145 bool &HasUserTransforms) {
146 LLVMContext &Ctx = Header->getContext();
155 if (Enabled !=
true) {
157 if (Enabled ==
false) {
158 NewLoopProperties.append(LoopProperties.begin(), LoopProperties.end());
159 NewLoopProperties.push_back(MDNode::get(
160 Ctx, MDString::get(Ctx,
"llvm.loop.unroll_and_jam.disable")));
161 LoopProperties = NewLoopProperties;
163 return createPartialUnrollMetadata(Attrs, LoopProperties,
168 FollowupLoopProperties.append(LoopProperties.begin(), LoopProperties.end());
169 FollowupLoopProperties.push_back(
170 MDNode::get(Ctx, MDString::get(Ctx,
"llvm.loop.unroll_and_jam.disable")));
172 bool FollowupHasTransforms =
false;
173 MDNode *Followup = createPartialUnrollMetadata(Attrs, FollowupLoopProperties,
174 FollowupHasTransforms);
177 TempMDTuple TempNode = MDNode::getTemporary(Ctx, None);
178 Args.push_back(TempNode.get());
179 Args.append(LoopProperties.begin(), LoopProperties.end());
184 MDString::get(Ctx,
"llvm.loop.unroll_and_jam.count"),
185 ConstantAsMetadata::get(ConstantInt::get(llvm::Type::getInt32Ty(Ctx),
187 Args.push_back(MDNode::get(Ctx, Vals));
191 Metadata *Vals[] = {MDString::get(Ctx,
"llvm.loop.unroll_and_jam.enable")};
192 Args.push_back(MDNode::get(Ctx, Vals));
195 if (FollowupHasTransforms)
196 Args.push_back(MDNode::get(
197 Ctx, {MDString::get(Ctx,
"llvm.loop.unroll_and_jam.followup_outer"),
200 if (UnrollAndJamInnerFollowup)
201 Args.push_back(MDNode::get(
202 Ctx, {MDString::get(Ctx,
"llvm.loop.unroll_and_jam.followup_inner"),
203 UnrollAndJamInnerFollowup}));
205 MDNode *LoopID = MDNode::getDistinct(Ctx, Args);
206 LoopID->replaceOperandWith(0, LoopID);
207 HasUserTransforms =
true;
212 LoopInfo::createLoopVectorizeMetadata(
const LoopAttributes &Attrs,
214 bool &HasUserTransforms) {
215 LLVMContext &Ctx = Header->getContext();
225 if (Enabled !=
true) {
227 if (Enabled ==
false) {
228 NewLoopProperties.append(LoopProperties.begin(), LoopProperties.end());
229 NewLoopProperties.push_back(
230 MDNode::get(Ctx, {MDString::get(Ctx,
"llvm.loop.vectorize.enable"),
231 ConstantAsMetadata::get(ConstantInt::get(
232 llvm::Type::getInt1Ty(Ctx), 0))}));
233 LoopProperties = NewLoopProperties;
235 return createUnrollAndJamMetadata(Attrs, LoopProperties, HasUserTransforms);
240 FollowupLoopProperties.append(LoopProperties.begin(), LoopProperties.end());
243 FollowupLoopProperties.push_back(
244 MDNode::get(Ctx, MDString::get(Ctx,
"llvm.loop.isvectorized")));
246 bool FollowupHasTransforms =
false;
247 MDNode *Followup = createUnrollAndJamMetadata(Attrs, FollowupLoopProperties,
248 FollowupHasTransforms);
251 TempMDTuple TempNode = MDNode::getTemporary(Ctx, None);
252 Args.push_back(TempNode.get());
253 Args.append(LoopProperties.begin(), LoopProperties.end());
256 bool IsVectorPredicateEnabled =
false;
261 IsVectorPredicateEnabled =
265 MDString::get(Ctx,
"llvm.loop.vectorize.predicate.enable"),
266 ConstantAsMetadata::get(ConstantInt::get(llvm::Type::getInt1Ty(Ctx),
267 IsVectorPredicateEnabled))};
268 Args.push_back(MDNode::get(Ctx, Vals));
274 MDString::get(Ctx,
"llvm.loop.vectorize.width"),
275 ConstantAsMetadata::get(ConstantInt::get(llvm::Type::getInt32Ty(Ctx),
277 Args.push_back(MDNode::get(Ctx, Vals));
283 MDString::get(Ctx,
"llvm.loop.interleave.count"),
284 ConstantAsMetadata::get(ConstantInt::get(llvm::Type::getInt32Ty(Ctx),
286 Args.push_back(MDNode::get(Ctx, Vals));
294 IsVectorPredicateEnabled ||
298 MDNode::get(Ctx, {MDString::get(Ctx,
"llvm.loop.vectorize.enable"),
299 ConstantAsMetadata::get(ConstantInt::get(
300 llvm::Type::getInt1Ty(Ctx), AttrVal))}));
303 if (FollowupHasTransforms)
304 Args.push_back(MDNode::get(
306 {MDString::get(Ctx,
"llvm.loop.vectorize.followup_all"), Followup}));
308 MDNode *LoopID = MDNode::get(Ctx, Args);
309 LoopID->replaceOperandWith(0, LoopID);
310 HasUserTransforms =
true;
315 LoopInfo::createLoopDistributeMetadata(
const LoopAttributes &Attrs,
317 bool &HasUserTransforms) {
318 LLVMContext &Ctx = Header->getContext();
326 if (Enabled !=
true) {
328 if (Enabled ==
false) {
329 NewLoopProperties.append(LoopProperties.begin(), LoopProperties.end());
330 NewLoopProperties.push_back(
331 MDNode::get(Ctx, {MDString::get(Ctx,
"llvm.loop.distribute.enable"),
332 ConstantAsMetadata::get(ConstantInt::get(
333 llvm::Type::getInt1Ty(Ctx), 0))}));
334 LoopProperties = NewLoopProperties;
336 return createLoopVectorizeMetadata(Attrs, LoopProperties,
340 bool FollowupHasTransforms =
false;
342 createLoopVectorizeMetadata(Attrs, LoopProperties, FollowupHasTransforms);
345 TempMDTuple TempNode = MDNode::getTemporary(Ctx, None);
346 Args.push_back(TempNode.get());
347 Args.append(LoopProperties.begin(), LoopProperties.end());
349 Metadata *Vals[] = {MDString::get(Ctx,
"llvm.loop.distribute.enable"),
350 ConstantAsMetadata::get(ConstantInt::get(
351 llvm::Type::getInt1Ty(Ctx),
353 Args.push_back(MDNode::get(Ctx, Vals));
355 if (FollowupHasTransforms)
356 Args.push_back(MDNode::get(
358 {MDString::get(Ctx,
"llvm.loop.distribute.followup_all"), Followup}));
360 MDNode *LoopID = MDNode::get(Ctx, Args);
361 LoopID->replaceOperandWith(0, LoopID);
362 HasUserTransforms =
true;
366 MDNode *LoopInfo::createFullUnrollMetadata(
const LoopAttributes &Attrs,
368 bool &HasUserTransforms) {
369 LLVMContext &Ctx = Header->getContext();
377 if (Enabled !=
true) {
379 if (Enabled ==
false) {
380 NewLoopProperties.append(LoopProperties.begin(), LoopProperties.end());
381 NewLoopProperties.push_back(
382 MDNode::get(Ctx, MDString::get(Ctx,
"llvm.loop.unroll.disable")));
383 LoopProperties = NewLoopProperties;
385 return createLoopDistributeMetadata(Attrs, LoopProperties,
390 TempMDTuple TempNode = MDNode::getTemporary(Ctx, None);
391 Args.push_back(TempNode.get());
392 Args.append(LoopProperties.begin(), LoopProperties.end());
393 Args.push_back(MDNode::get(Ctx, MDString::get(Ctx,
"llvm.loop.unroll.full")));
398 MDNode *LoopID = MDNode::getDistinct(Ctx, Args);
399 LoopID->replaceOperandWith(0, LoopID);
400 HasUserTransforms =
true;
404 MDNode *LoopInfo::createMetadata(
407 bool &HasUserTransforms) {
412 LoopProperties.push_back(StartLoc.getAsMDNode());
416 LoopProperties.push_back(EndLoc.getAsMDNode());
420 "There must be an access group iff the loop is parallel");
422 LLVMContext &Ctx = Header->getContext();
423 LoopProperties.push_back(MDNode::get(
424 Ctx, {MDString::get(Ctx,
"llvm.loop.parallel_accesses"), AccGroup}));
427 LoopProperties.insert(LoopProperties.end(), AdditionalLoopProperties.begin(),
428 AdditionalLoopProperties.end());
429 return createFullUnrollMetadata(Attrs, LoopProperties, HasUserTransforms);
437 InterleaveCount(0), UnrollCount(0), UnrollAndJamCount(0),
439 PipelineInitiationInterval(0) {}
457 const llvm::DebugLoc &StartLoc,
const llvm::DebugLoc &EndLoc,
459 : Header(Header), Attrs(Attrs), StartLoc(StartLoc), EndLoc(EndLoc),
464 LLVMContext &Ctx = Header->getContext();
465 AccGroup = MDNode::getDistinct(Ctx, {});
480 TempLoopID = MDNode::getTemporary(Header->getContext(), None);
491 LLVMContext &Ctx = Header->getContext();
538 if (!Parent->UnrollAndJamInnerFollowup) {
547 BeforeLoopProperties.push_back(
548 MDNode::get(Ctx, MDString::get(Ctx,
"llvm.loop.isvectorized")));
550 bool InnerFollowupHasTransform =
false;
551 MDNode *InnerFollowup = createMetadata(AfterJam, BeforeLoopProperties,
552 InnerFollowupHasTransform);
553 if (InnerFollowupHasTransform)
554 Parent->UnrollAndJamInnerFollowup = InnerFollowup;
557 CurLoopAttr = BeforeJam;
560 bool HasUserTransforms =
false;
561 LoopID = createMetadata(CurLoopAttr, {}, HasUserTransforms);
562 TempLoopID->replaceAllUsesWith(LoopID);
566 const llvm::DebugLoc &EndLoc) {
568 new LoopInfo(Header, StagedAttrs, StartLoc, EndLoc,
569 Active.empty() ? nullptr : Active.back().get()));
576 const llvm::DebugLoc &StartLoc,
577 const llvm::DebugLoc &EndLoc) {
580 for (
const auto *
Attr : Attrs) {
581 const LoopHintAttr *LH = dyn_cast<LoopHintAttr>(
Attr);
582 const OpenCLUnrollHintAttr *OpenCLHint =
583 dyn_cast<OpenCLUnrollHintAttr>(
Attr);
586 if (!LH && !OpenCLHint) {
590 LoopHintAttr::OptionType Option = LoopHintAttr::Unroll;
591 LoopHintAttr::LoopHintState
State = LoopHintAttr::Disable;
592 unsigned ValueInt = 1;
600 ValueInt = OpenCLHint->getUnrollHint();
602 State = LoopHintAttr::Enable;
603 }
else if (ValueInt != 1) {
604 Option = LoopHintAttr::UnrollCount;
605 State = LoopHintAttr::Numeric;
608 auto *ValueExpr = LH->getValue();
610 llvm::APSInt ValueAPS = ValueExpr->EvaluateKnownConstInt(Ctx);
611 ValueInt = ValueAPS.getSExtValue();
614 Option = LH->getOption();
615 State = LH->getState();
618 case LoopHintAttr::Disable:
620 case LoopHintAttr::Vectorize:
622 setVectorizeWidth(1);
624 case LoopHintAttr::Interleave:
626 setInterleaveCount(1);
628 case LoopHintAttr::Unroll:
631 case LoopHintAttr::UnrollAndJam:
634 case LoopHintAttr::VectorizePredicate:
637 case LoopHintAttr::Distribute:
638 setDistributeState(
false);
640 case LoopHintAttr::PipelineDisabled:
641 setPipelineDisabled(
true);
643 case LoopHintAttr::UnrollCount:
644 case LoopHintAttr::UnrollAndJamCount:
645 case LoopHintAttr::VectorizeWidth:
646 case LoopHintAttr::InterleaveCount:
647 case LoopHintAttr::PipelineInitiationInterval:
648 llvm_unreachable(
"Options cannot be disabled.");
652 case LoopHintAttr::Enable:
654 case LoopHintAttr::Vectorize:
655 case LoopHintAttr::Interleave:
656 setVectorizeEnable(
true);
658 case LoopHintAttr::Unroll:
661 case LoopHintAttr::UnrollAndJam:
664 case LoopHintAttr::VectorizePredicate:
667 case LoopHintAttr::Distribute:
668 setDistributeState(
true);
670 case LoopHintAttr::UnrollCount:
671 case LoopHintAttr::UnrollAndJamCount:
672 case LoopHintAttr::VectorizeWidth:
673 case LoopHintAttr::InterleaveCount:
674 case LoopHintAttr::PipelineDisabled:
675 case LoopHintAttr::PipelineInitiationInterval:
676 llvm_unreachable(
"Options cannot enabled.");
680 case LoopHintAttr::AssumeSafety:
682 case LoopHintAttr::Vectorize:
683 case LoopHintAttr::Interleave:
686 setVectorizeEnable(
true);
688 case LoopHintAttr::Unroll:
689 case LoopHintAttr::UnrollAndJam:
690 case LoopHintAttr::VectorizePredicate:
691 case LoopHintAttr::UnrollCount:
692 case LoopHintAttr::UnrollAndJamCount:
693 case LoopHintAttr::VectorizeWidth:
694 case LoopHintAttr::InterleaveCount:
695 case LoopHintAttr::Distribute:
696 case LoopHintAttr::PipelineDisabled:
697 case LoopHintAttr::PipelineInitiationInterval:
698 llvm_unreachable(
"Options cannot be used to assume mem safety.");
702 case LoopHintAttr::Full:
704 case LoopHintAttr::Unroll:
707 case LoopHintAttr::UnrollAndJam:
710 case LoopHintAttr::Vectorize:
711 case LoopHintAttr::Interleave:
712 case LoopHintAttr::UnrollCount:
713 case LoopHintAttr::UnrollAndJamCount:
714 case LoopHintAttr::VectorizeWidth:
715 case LoopHintAttr::InterleaveCount:
716 case LoopHintAttr::Distribute:
717 case LoopHintAttr::PipelineDisabled:
718 case LoopHintAttr::PipelineInitiationInterval:
719 case LoopHintAttr::VectorizePredicate:
720 llvm_unreachable(
"Options cannot be used with 'full' hint.");
724 case LoopHintAttr::Numeric:
726 case LoopHintAttr::VectorizeWidth:
727 setVectorizeWidth(ValueInt);
729 case LoopHintAttr::InterleaveCount:
730 setInterleaveCount(ValueInt);
732 case LoopHintAttr::UnrollCount:
733 setUnrollCount(ValueInt);
735 case LoopHintAttr::UnrollAndJamCount:
736 setUnrollAndJamCount(ValueInt);
738 case LoopHintAttr::PipelineInitiationInterval:
739 setPipelineInitiationInterval(ValueInt);
741 case LoopHintAttr::Unroll:
742 case LoopHintAttr::UnrollAndJam:
743 case LoopHintAttr::VectorizePredicate:
744 case LoopHintAttr::Vectorize:
745 case LoopHintAttr::Interleave:
746 case LoopHintAttr::Distribute:
747 case LoopHintAttr::PipelineDisabled:
748 llvm_unreachable(
"Options cannot be assigned a value.");
756 push(Header, StartLoc, EndLoc);
760 assert(!Active.empty() &&
"No active loops to pop");
761 Active.back()->finish();
766 if (I->mayReadOrWriteMemory()) {
768 for (
const auto &AL : Active) {
770 if (MDNode *Group = AL->getAccessGroup())
771 AccessGroups.push_back(Group);
773 MDNode *UnionMD =
nullptr;
774 if (AccessGroups.size() == 1)
775 UnionMD = cast<MDNode>(AccessGroups[0]);
776 else if (AccessGroups.size() >= 2)
777 UnionMD = MDNode::get(I->getContext(), AccessGroups);
778 I->setMetadata(
"llvm.access.group", UnionMD);
788 if (I->isTerminator()) {
789 for (BasicBlock *Succ : successors(I))
791 I->setMetadata(llvm::LLVMContext::MD_loop, L.
getLoopID());
Defines the clang::ASTContext interface.
Specialize PointerLikeTypeTraits to allow LazyGenerationalUpdatePtr to be placed into a PointerUnion...
Attributes that may be specified on loops.
unsigned UnrollAndJamCount
llvm.unroll.
Information used when generating a structured loop.
LVEnableState VectorizePredicateEnable
Value for llvm.loop.vectorize.predicate metadata.
LoopAttributes(bool IsParallel=false)
LVEnableState UnrollEnable
Value for llvm.loop.unroll.* metadata (enable, disable, or full).
unsigned PipelineInitiationInterval
Value for llvm.loop.pipeline.iicount metadata.
Holds long-lived AST nodes (such as types and decls) that can be referred to throughout the semantic ...
unsigned InterleaveCount
Value for llvm.loop.interleave.count metadata.
LVEnableState VectorizeEnable
Value for llvm.loop.vectorize.enable metadata.
void pop()
End the current loop.
Whether values of this type can be null is (explicitly) unspecified.
LVEnableState UnrollAndJamEnable
Value for llvm.loop.unroll_and_jam.* metadata (enable, disable, or full).
void finish()
Create the loop's metadata.
void InsertHelper(llvm::Instruction *I) const
Function called by the CodeGenFunction when an instruction is created.
bool IsParallel
Generate llvm.loop.parallel metadata for loads and stores.
unsigned UnrollCount
llvm.unroll.
LVEnableState DistributeEnable
Value for llvm.loop.distribute.enable metadata.
llvm::MDNode * getLoopID() const
Get the loop id metadata for this loop.
constexpr XRayInstrMask None
void push(llvm::BasicBlock *Header, const llvm::DebugLoc &StartLoc, const llvm::DebugLoc &EndLoc)
Begin a new structured loop.
bool PipelineDisabled
Value for llvm.loop.pipeline.disable metadata.
static const TypeInfo & getInfo(unsigned id)
LoopInfo(llvm::BasicBlock *Header, const LoopAttributes &Attrs, const llvm::DebugLoc &StartLoc, const llvm::DebugLoc &EndLoc, LoopInfo *Parent)
Construct a new LoopInfo for the loop with entry Header.
unsigned VectorizeWidth
Value for llvm.loop.vectorize.width metadata.
llvm::BasicBlock * getHeader() const
Get the header block of this loop.
Attr - This represents one attribute.