clang  10.0.0git
CGLoopInfo.cpp
Go to the documentation of this file.
1 //===---- CGLoopInfo.cpp - LLVM CodeGen for loop metadata -*- C++ -*-------===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 
9 #include "CGLoopInfo.h"
10 #include "clang/AST/ASTContext.h"
11 #include "clang/AST/Attr.h"
12 #include "llvm/IR/BasicBlock.h"
13 #include "llvm/IR/CFG.h"
14 #include "llvm/IR/Constants.h"
15 #include "llvm/IR/InstrTypes.h"
16 #include "llvm/IR/Instructions.h"
17 #include "llvm/IR/Metadata.h"
18 using namespace clang::CodeGen;
19 using namespace llvm;
20 
21 MDNode *
22 LoopInfo::createLoopPropertiesMetadata(ArrayRef<Metadata *> LoopProperties) {
23  LLVMContext &Ctx = Header->getContext();
24  SmallVector<Metadata *, 4> NewLoopProperties;
25  TempMDTuple TempNode = MDNode::getTemporary(Ctx, None);
26  NewLoopProperties.push_back(TempNode.get());
27  NewLoopProperties.append(LoopProperties.begin(), LoopProperties.end());
28 
29  MDNode *LoopID = MDNode::getDistinct(Ctx, NewLoopProperties);
30  LoopID->replaceOperandWith(0, LoopID);
31  return LoopID;
32 }
33 
34 MDNode *LoopInfo::createPipeliningMetadata(const LoopAttributes &Attrs,
35  ArrayRef<Metadata *> LoopProperties,
36  bool &HasUserTransforms) {
37  LLVMContext &Ctx = Header->getContext();
38 
39  Optional<bool> Enabled;
40  if (Attrs.PipelineDisabled)
41  Enabled = false;
42  else if (Attrs.PipelineInitiationInterval != 0)
43  Enabled = true;
44 
45  if (Enabled != true) {
46  SmallVector<Metadata *, 4> NewLoopProperties;
47  if (Enabled == false) {
48  NewLoopProperties.append(LoopProperties.begin(), LoopProperties.end());
49  NewLoopProperties.push_back(
50  MDNode::get(Ctx, {MDString::get(Ctx, "llvm.loop.pipeline.disable"),
51  ConstantAsMetadata::get(ConstantInt::get(
52  llvm::Type::getInt1Ty(Ctx), 1))}));
53  LoopProperties = NewLoopProperties;
54  }
55  return createLoopPropertiesMetadata(LoopProperties);
56  }
57 
59  TempMDTuple TempNode = MDNode::getTemporary(Ctx, None);
60  Args.push_back(TempNode.get());
61  Args.append(LoopProperties.begin(), LoopProperties.end());
62 
63  if (Attrs.PipelineInitiationInterval > 0) {
64  Metadata *Vals[] = {
65  MDString::get(Ctx, "llvm.loop.pipeline.initiationinterval"),
66  ConstantAsMetadata::get(ConstantInt::get(
67  llvm::Type::getInt32Ty(Ctx), Attrs.PipelineInitiationInterval))};
68  Args.push_back(MDNode::get(Ctx, Vals));
69  }
70 
71  // No follow-up: This is the last transformation.
72 
73  MDNode *LoopID = MDNode::getDistinct(Ctx, Args);
74  LoopID->replaceOperandWith(0, LoopID);
75  HasUserTransforms = true;
76  return LoopID;
77 }
78 
79 MDNode *
80 LoopInfo::createPartialUnrollMetadata(const LoopAttributes &Attrs,
81  ArrayRef<Metadata *> LoopProperties,
82  bool &HasUserTransforms) {
83  LLVMContext &Ctx = Header->getContext();
84 
85  Optional<bool> Enabled;
87  Enabled = false;
88  else if (Attrs.UnrollEnable == LoopAttributes::Full)
89  Enabled = None;
90  else if (Attrs.UnrollEnable != LoopAttributes::Unspecified ||
91  Attrs.UnrollCount != 0)
92  Enabled = true;
93 
94  if (Enabled != true) {
95  // createFullUnrollMetadata will already have added llvm.loop.unroll.disable
96  // if unrolling is disabled.
97  return createPipeliningMetadata(Attrs, LoopProperties, HasUserTransforms);
98  }
99 
100  SmallVector<Metadata *, 4> FollowupLoopProperties;
101 
102  // Apply all loop properties to the unrolled loop.
103  FollowupLoopProperties.append(LoopProperties.begin(), LoopProperties.end());
104 
105  // Don't unroll an already unrolled loop.
106  FollowupLoopProperties.push_back(
107  MDNode::get(Ctx, MDString::get(Ctx, "llvm.loop.unroll.disable")));
108 
109  bool FollowupHasTransforms = false;
110  MDNode *Followup = createPipeliningMetadata(Attrs, FollowupLoopProperties,
111  FollowupHasTransforms);
112 
114  TempMDTuple TempNode = MDNode::getTemporary(Ctx, None);
115  Args.push_back(TempNode.get());
116  Args.append(LoopProperties.begin(), LoopProperties.end());
117 
118  // Setting unroll.count
119  if (Attrs.UnrollCount > 0) {
120  Metadata *Vals[] = {MDString::get(Ctx, "llvm.loop.unroll.count"),
121  ConstantAsMetadata::get(ConstantInt::get(
122  llvm::Type::getInt32Ty(Ctx), Attrs.UnrollCount))};
123  Args.push_back(MDNode::get(Ctx, Vals));
124  }
125 
126  // Setting unroll.full or unroll.disable
127  if (Attrs.UnrollEnable == LoopAttributes::Enable) {
128  Metadata *Vals[] = {MDString::get(Ctx, "llvm.loop.unroll.enable")};
129  Args.push_back(MDNode::get(Ctx, Vals));
130  }
131 
132  if (FollowupHasTransforms)
133  Args.push_back(MDNode::get(
134  Ctx, {MDString::get(Ctx, "llvm.loop.unroll.followup_all"), Followup}));
135 
136  MDNode *LoopID = MDNode::getDistinct(Ctx, Args);
137  LoopID->replaceOperandWith(0, LoopID);
138  HasUserTransforms = true;
139  return LoopID;
140 }
141 
142 MDNode *
143 LoopInfo::createUnrollAndJamMetadata(const LoopAttributes &Attrs,
144  ArrayRef<Metadata *> LoopProperties,
145  bool &HasUserTransforms) {
146  LLVMContext &Ctx = Header->getContext();
147 
148  Optional<bool> Enabled;
150  Enabled = false;
151  else if (Attrs.UnrollAndJamEnable == LoopAttributes::Enable ||
152  Attrs.UnrollAndJamCount != 0)
153  Enabled = true;
154 
155  if (Enabled != true) {
156  SmallVector<Metadata *, 4> NewLoopProperties;
157  if (Enabled == false) {
158  NewLoopProperties.append(LoopProperties.begin(), LoopProperties.end());
159  NewLoopProperties.push_back(MDNode::get(
160  Ctx, MDString::get(Ctx, "llvm.loop.unroll_and_jam.disable")));
161  LoopProperties = NewLoopProperties;
162  }
163  return createPartialUnrollMetadata(Attrs, LoopProperties,
164  HasUserTransforms);
165  }
166 
167  SmallVector<Metadata *, 4> FollowupLoopProperties;
168  FollowupLoopProperties.append(LoopProperties.begin(), LoopProperties.end());
169  FollowupLoopProperties.push_back(
170  MDNode::get(Ctx, MDString::get(Ctx, "llvm.loop.unroll_and_jam.disable")));
171 
172  bool FollowupHasTransforms = false;
173  MDNode *Followup = createPartialUnrollMetadata(Attrs, FollowupLoopProperties,
174  FollowupHasTransforms);
175 
177  TempMDTuple TempNode = MDNode::getTemporary(Ctx, None);
178  Args.push_back(TempNode.get());
179  Args.append(LoopProperties.begin(), LoopProperties.end());
180 
181  // Setting unroll_and_jam.count
182  if (Attrs.UnrollAndJamCount > 0) {
183  Metadata *Vals[] = {
184  MDString::get(Ctx, "llvm.loop.unroll_and_jam.count"),
185  ConstantAsMetadata::get(ConstantInt::get(llvm::Type::getInt32Ty(Ctx),
186  Attrs.UnrollAndJamCount))};
187  Args.push_back(MDNode::get(Ctx, Vals));
188  }
189 
191  Metadata *Vals[] = {MDString::get(Ctx, "llvm.loop.unroll_and_jam.enable")};
192  Args.push_back(MDNode::get(Ctx, Vals));
193  }
194 
195  if (FollowupHasTransforms)
196  Args.push_back(MDNode::get(
197  Ctx, {MDString::get(Ctx, "llvm.loop.unroll_and_jam.followup_outer"),
198  Followup}));
199 
200  if (UnrollAndJamInnerFollowup)
201  Args.push_back(MDNode::get(
202  Ctx, {MDString::get(Ctx, "llvm.loop.unroll_and_jam.followup_inner"),
203  UnrollAndJamInnerFollowup}));
204 
205  MDNode *LoopID = MDNode::getDistinct(Ctx, Args);
206  LoopID->replaceOperandWith(0, LoopID);
207  HasUserTransforms = true;
208  return LoopID;
209 }
210 
211 MDNode *
212 LoopInfo::createLoopVectorizeMetadata(const LoopAttributes &Attrs,
213  ArrayRef<Metadata *> LoopProperties,
214  bool &HasUserTransforms) {
215  LLVMContext &Ctx = Header->getContext();
216 
217  Optional<bool> Enabled;
219  Enabled = false;
220  else if (Attrs.VectorizeEnable != LoopAttributes::Unspecified ||
222  Attrs.InterleaveCount != 0 || Attrs.VectorizeWidth != 0)
223  Enabled = true;
224 
225  if (Enabled != true) {
226  SmallVector<Metadata *, 4> NewLoopProperties;
227  if (Enabled == false) {
228  NewLoopProperties.append(LoopProperties.begin(), LoopProperties.end());
229  NewLoopProperties.push_back(
230  MDNode::get(Ctx, {MDString::get(Ctx, "llvm.loop.vectorize.enable"),
231  ConstantAsMetadata::get(ConstantInt::get(
232  llvm::Type::getInt1Ty(Ctx), 0))}));
233  LoopProperties = NewLoopProperties;
234  }
235  return createUnrollAndJamMetadata(Attrs, LoopProperties, HasUserTransforms);
236  }
237 
238  // Apply all loop properties to the vectorized loop.
239  SmallVector<Metadata *, 4> FollowupLoopProperties;
240  FollowupLoopProperties.append(LoopProperties.begin(), LoopProperties.end());
241 
242  // Don't vectorize an already vectorized loop.
243  FollowupLoopProperties.push_back(
244  MDNode::get(Ctx, MDString::get(Ctx, "llvm.loop.isvectorized")));
245 
246  bool FollowupHasTransforms = false;
247  MDNode *Followup = createUnrollAndJamMetadata(Attrs, FollowupLoopProperties,
248  FollowupHasTransforms);
249 
251  TempMDTuple TempNode = MDNode::getTemporary(Ctx, None);
252  Args.push_back(TempNode.get());
253  Args.append(LoopProperties.begin(), LoopProperties.end());
254 
255  // Setting vectorize.predicate
256  bool IsVectorPredicateEnabled = false;
259  Attrs.VectorizeWidth < 1) {
260 
261  IsVectorPredicateEnabled =
263 
264  Metadata *Vals[] = {
265  MDString::get(Ctx, "llvm.loop.vectorize.predicate.enable"),
266  ConstantAsMetadata::get(ConstantInt::get(llvm::Type::getInt1Ty(Ctx),
267  IsVectorPredicateEnabled))};
268  Args.push_back(MDNode::get(Ctx, Vals));
269  }
270 
271  // Setting vectorize.width
272  if (Attrs.VectorizeWidth > 0) {
273  Metadata *Vals[] = {
274  MDString::get(Ctx, "llvm.loop.vectorize.width"),
275  ConstantAsMetadata::get(ConstantInt::get(llvm::Type::getInt32Ty(Ctx),
276  Attrs.VectorizeWidth))};
277  Args.push_back(MDNode::get(Ctx, Vals));
278  }
279 
280  // Setting interleave.count
281  if (Attrs.InterleaveCount > 0) {
282  Metadata *Vals[] = {
283  MDString::get(Ctx, "llvm.loop.interleave.count"),
284  ConstantAsMetadata::get(ConstantInt::get(llvm::Type::getInt32Ty(Ctx),
285  Attrs.InterleaveCount))};
286  Args.push_back(MDNode::get(Ctx, Vals));
287  }
288 
289  // vectorize.enable is set if:
290  // 1) loop hint vectorize.enable is set, or
291  // 2) it is implied when vectorize.predicate is set, or
292  // 3) it is implied when vectorize.width is set.
294  IsVectorPredicateEnabled ||
295  Attrs.VectorizeWidth > 1 ) {
296  bool AttrVal = Attrs.VectorizeEnable != LoopAttributes::Disable;
297  Args.push_back(
298  MDNode::get(Ctx, {MDString::get(Ctx, "llvm.loop.vectorize.enable"),
299  ConstantAsMetadata::get(ConstantInt::get(
300  llvm::Type::getInt1Ty(Ctx), AttrVal))}));
301  }
302 
303  if (FollowupHasTransforms)
304  Args.push_back(MDNode::get(
305  Ctx,
306  {MDString::get(Ctx, "llvm.loop.vectorize.followup_all"), Followup}));
307 
308  MDNode *LoopID = MDNode::get(Ctx, Args);
309  LoopID->replaceOperandWith(0, LoopID);
310  HasUserTransforms = true;
311  return LoopID;
312 }
313 
314 MDNode *
315 LoopInfo::createLoopDistributeMetadata(const LoopAttributes &Attrs,
316  ArrayRef<Metadata *> LoopProperties,
317  bool &HasUserTransforms) {
318  LLVMContext &Ctx = Header->getContext();
319 
320  Optional<bool> Enabled;
322  Enabled = false;
324  Enabled = true;
325 
326  if (Enabled != true) {
327  SmallVector<Metadata *, 4> NewLoopProperties;
328  if (Enabled == false) {
329  NewLoopProperties.append(LoopProperties.begin(), LoopProperties.end());
330  NewLoopProperties.push_back(
331  MDNode::get(Ctx, {MDString::get(Ctx, "llvm.loop.distribute.enable"),
332  ConstantAsMetadata::get(ConstantInt::get(
333  llvm::Type::getInt1Ty(Ctx), 0))}));
334  LoopProperties = NewLoopProperties;
335  }
336  return createLoopVectorizeMetadata(Attrs, LoopProperties,
337  HasUserTransforms);
338  }
339 
340  bool FollowupHasTransforms = false;
341  MDNode *Followup =
342  createLoopVectorizeMetadata(Attrs, LoopProperties, FollowupHasTransforms);
343 
345  TempMDTuple TempNode = MDNode::getTemporary(Ctx, None);
346  Args.push_back(TempNode.get());
347  Args.append(LoopProperties.begin(), LoopProperties.end());
348 
349  Metadata *Vals[] = {MDString::get(Ctx, "llvm.loop.distribute.enable"),
350  ConstantAsMetadata::get(ConstantInt::get(
351  llvm::Type::getInt1Ty(Ctx),
353  Args.push_back(MDNode::get(Ctx, Vals));
354 
355  if (FollowupHasTransforms)
356  Args.push_back(MDNode::get(
357  Ctx,
358  {MDString::get(Ctx, "llvm.loop.distribute.followup_all"), Followup}));
359 
360  MDNode *LoopID = MDNode::get(Ctx, Args);
361  LoopID->replaceOperandWith(0, LoopID);
362  HasUserTransforms = true;
363  return LoopID;
364 }
365 
366 MDNode *LoopInfo::createFullUnrollMetadata(const LoopAttributes &Attrs,
367  ArrayRef<Metadata *> LoopProperties,
368  bool &HasUserTransforms) {
369  LLVMContext &Ctx = Header->getContext();
370 
371  Optional<bool> Enabled;
373  Enabled = false;
374  else if (Attrs.UnrollEnable == LoopAttributes::Full)
375  Enabled = true;
376 
377  if (Enabled != true) {
378  SmallVector<Metadata *, 4> NewLoopProperties;
379  if (Enabled == false) {
380  NewLoopProperties.append(LoopProperties.begin(), LoopProperties.end());
381  NewLoopProperties.push_back(
382  MDNode::get(Ctx, MDString::get(Ctx, "llvm.loop.unroll.disable")));
383  LoopProperties = NewLoopProperties;
384  }
385  return createLoopDistributeMetadata(Attrs, LoopProperties,
386  HasUserTransforms);
387  }
388 
390  TempMDTuple TempNode = MDNode::getTemporary(Ctx, None);
391  Args.push_back(TempNode.get());
392  Args.append(LoopProperties.begin(), LoopProperties.end());
393  Args.push_back(MDNode::get(Ctx, MDString::get(Ctx, "llvm.loop.unroll.full")));
394 
395  // No follow-up: there is no loop after full unrolling.
396  // TODO: Warn if there are transformations after full unrolling.
397 
398  MDNode *LoopID = MDNode::getDistinct(Ctx, Args);
399  LoopID->replaceOperandWith(0, LoopID);
400  HasUserTransforms = true;
401  return LoopID;
402 }
403 
404 MDNode *LoopInfo::createMetadata(
405  const LoopAttributes &Attrs,
406  llvm::ArrayRef<llvm::Metadata *> AdditionalLoopProperties,
407  bool &HasUserTransforms) {
408  SmallVector<Metadata *, 3> LoopProperties;
409 
410  // If we have a valid start debug location for the loop, add it.
411  if (StartLoc) {
412  LoopProperties.push_back(StartLoc.getAsMDNode());
413 
414  // If we also have a valid end debug location for the loop, add it.
415  if (EndLoc)
416  LoopProperties.push_back(EndLoc.getAsMDNode());
417  }
418 
419  assert(!!AccGroup == Attrs.IsParallel &&
420  "There must be an access group iff the loop is parallel");
421  if (Attrs.IsParallel) {
422  LLVMContext &Ctx = Header->getContext();
423  LoopProperties.push_back(MDNode::get(
424  Ctx, {MDString::get(Ctx, "llvm.loop.parallel_accesses"), AccGroup}));
425  }
426 
427  LoopProperties.insert(LoopProperties.end(), AdditionalLoopProperties.begin(),
428  AdditionalLoopProperties.end());
429  return createFullUnrollMetadata(Attrs, LoopProperties, HasUserTransforms);
430 }
431 
433  : IsParallel(IsParallel), VectorizeEnable(LoopAttributes::Unspecified),
434  UnrollEnable(LoopAttributes::Unspecified),
435  UnrollAndJamEnable(LoopAttributes::Unspecified),
436  VectorizePredicateEnable(LoopAttributes::Unspecified), VectorizeWidth(0),
437  InterleaveCount(0), UnrollCount(0), UnrollAndJamCount(0),
438  DistributeEnable(LoopAttributes::Unspecified), PipelineDisabled(false),
439  PipelineInitiationInterval(0) {}
440 
442  IsParallel = false;
443  VectorizeWidth = 0;
444  InterleaveCount = 0;
445  UnrollCount = 0;
446  UnrollAndJamCount = 0;
452  PipelineDisabled = false;
454 }
455 
456 LoopInfo::LoopInfo(BasicBlock *Header, const LoopAttributes &Attrs,
457  const llvm::DebugLoc &StartLoc, const llvm::DebugLoc &EndLoc,
458  LoopInfo *Parent)
459  : Header(Header), Attrs(Attrs), StartLoc(StartLoc), EndLoc(EndLoc),
460  Parent(Parent) {
461 
462  if (Attrs.IsParallel) {
463  // Create an access group for this loop.
464  LLVMContext &Ctx = Header->getContext();
465  AccGroup = MDNode::getDistinct(Ctx, {});
466  }
467 
468  if (!Attrs.IsParallel && Attrs.VectorizeWidth == 0 &&
469  Attrs.InterleaveCount == 0 && Attrs.UnrollCount == 0 &&
470  Attrs.UnrollAndJamCount == 0 && !Attrs.PipelineDisabled &&
471  Attrs.PipelineInitiationInterval == 0 &&
476  Attrs.DistributeEnable == LoopAttributes::Unspecified && !StartLoc &&
477  !EndLoc)
478  return;
479 
480  TempLoopID = MDNode::getTemporary(Header->getContext(), None);
481 }
482 
484  // We did not annotate the loop body instructions because there are no
485  // attributes for this loop.
486  if (!TempLoopID)
487  return;
488 
489  MDNode *LoopID;
490  LoopAttributes CurLoopAttr = Attrs;
491  LLVMContext &Ctx = Header->getContext();
492 
493  if (Parent && (Parent->Attrs.UnrollAndJamEnable ||
494  Parent->Attrs.UnrollAndJamCount != 0)) {
495  // Parent unroll-and-jams this loop.
496  // Split the transformations in those that happens before the unroll-and-jam
497  // and those after.
498 
499  LoopAttributes BeforeJam, AfterJam;
500 
501  BeforeJam.IsParallel = AfterJam.IsParallel = Attrs.IsParallel;
502 
503  BeforeJam.VectorizeWidth = Attrs.VectorizeWidth;
504  BeforeJam.InterleaveCount = Attrs.InterleaveCount;
505  BeforeJam.VectorizeEnable = Attrs.VectorizeEnable;
506  BeforeJam.DistributeEnable = Attrs.DistributeEnable;
508 
509  switch (Attrs.UnrollEnable) {
512  BeforeJam.UnrollEnable = Attrs.UnrollEnable;
513  AfterJam.UnrollEnable = Attrs.UnrollEnable;
514  break;
517  break;
520  break;
521  }
522 
524  AfterJam.UnrollCount = Attrs.UnrollCount;
525  AfterJam.PipelineDisabled = Attrs.PipelineDisabled;
527 
528  // If this loop is subject of an unroll-and-jam by the parent loop, and has
529  // an unroll-and-jam annotation itself, we have to decide whether to first
530  // apply the parent's unroll-and-jam or this loop's unroll-and-jam. The
531  // UnrollAndJam pass processes loops from inner to outer, so we apply the
532  // inner first.
533  BeforeJam.UnrollAndJamCount = Attrs.UnrollAndJamCount;
534  BeforeJam.UnrollAndJamEnable = Attrs.UnrollAndJamEnable;
535 
536  // Set the inner followup metadata to process by the outer loop. Only
537  // consider the first inner loop.
538  if (!Parent->UnrollAndJamInnerFollowup) {
539  // Splitting the attributes into a BeforeJam and an AfterJam part will
540  // stop 'llvm.loop.isvectorized' (generated by vectorization in BeforeJam)
541  // to be forwarded to the AfterJam part. We detect the situation here and
542  // add it manually.
543  SmallVector<Metadata *, 1> BeforeLoopProperties;
544  if (BeforeJam.VectorizeEnable != LoopAttributes::Unspecified ||
546  BeforeJam.InterleaveCount != 0 || BeforeJam.VectorizeWidth != 0)
547  BeforeLoopProperties.push_back(
548  MDNode::get(Ctx, MDString::get(Ctx, "llvm.loop.isvectorized")));
549 
550  bool InnerFollowupHasTransform = false;
551  MDNode *InnerFollowup = createMetadata(AfterJam, BeforeLoopProperties,
552  InnerFollowupHasTransform);
553  if (InnerFollowupHasTransform)
554  Parent->UnrollAndJamInnerFollowup = InnerFollowup;
555  }
556 
557  CurLoopAttr = BeforeJam;
558  }
559 
560  bool HasUserTransforms = false;
561  LoopID = createMetadata(CurLoopAttr, {}, HasUserTransforms);
562  TempLoopID->replaceAllUsesWith(LoopID);
563 }
564 
565 void LoopInfoStack::push(BasicBlock *Header, const llvm::DebugLoc &StartLoc,
566  const llvm::DebugLoc &EndLoc) {
567  Active.emplace_back(
568  new LoopInfo(Header, StagedAttrs, StartLoc, EndLoc,
569  Active.empty() ? nullptr : Active.back().get()));
570  // Clear the attributes so nested loops do not inherit them.
571  StagedAttrs.clear();
572 }
573 
574 void LoopInfoStack::push(BasicBlock *Header, clang::ASTContext &Ctx,
576  const llvm::DebugLoc &StartLoc,
577  const llvm::DebugLoc &EndLoc) {
578 
579  // Identify loop hint attributes from Attrs.
580  for (const auto *Attr : Attrs) {
581  const LoopHintAttr *LH = dyn_cast<LoopHintAttr>(Attr);
582  const OpenCLUnrollHintAttr *OpenCLHint =
583  dyn_cast<OpenCLUnrollHintAttr>(Attr);
584 
585  // Skip non loop hint attributes
586  if (!LH && !OpenCLHint) {
587  continue;
588  }
589 
590  LoopHintAttr::OptionType Option = LoopHintAttr::Unroll;
591  LoopHintAttr::LoopHintState State = LoopHintAttr::Disable;
592  unsigned ValueInt = 1;
593  // Translate opencl_unroll_hint attribute argument to
594  // equivalent LoopHintAttr enums.
595  // OpenCL v2.0 s6.11.5:
596  // 0 - enable unroll (no argument).
597  // 1 - disable unroll.
598  // other positive integer n - unroll by n.
599  if (OpenCLHint) {
600  ValueInt = OpenCLHint->getUnrollHint();
601  if (ValueInt == 0) {
602  State = LoopHintAttr::Enable;
603  } else if (ValueInt != 1) {
604  Option = LoopHintAttr::UnrollCount;
605  State = LoopHintAttr::Numeric;
606  }
607  } else if (LH) {
608  auto *ValueExpr = LH->getValue();
609  if (ValueExpr) {
610  llvm::APSInt ValueAPS = ValueExpr->EvaluateKnownConstInt(Ctx);
611  ValueInt = ValueAPS.getSExtValue();
612  }
613 
614  Option = LH->getOption();
615  State = LH->getState();
616  }
617  switch (State) {
618  case LoopHintAttr::Disable:
619  switch (Option) {
620  case LoopHintAttr::Vectorize:
621  // Disable vectorization by specifying a width of 1.
622  setVectorizeWidth(1);
623  break;
624  case LoopHintAttr::Interleave:
625  // Disable interleaving by speciyfing a count of 1.
626  setInterleaveCount(1);
627  break;
628  case LoopHintAttr::Unroll:
629  setUnrollState(LoopAttributes::Disable);
630  break;
631  case LoopHintAttr::UnrollAndJam:
632  setUnrollAndJamState(LoopAttributes::Disable);
633  break;
634  case LoopHintAttr::VectorizePredicate:
635  setVectorizePredicateState(LoopAttributes::Disable);
636  break;
637  case LoopHintAttr::Distribute:
638  setDistributeState(false);
639  break;
640  case LoopHintAttr::PipelineDisabled:
641  setPipelineDisabled(true);
642  break;
643  case LoopHintAttr::UnrollCount:
644  case LoopHintAttr::UnrollAndJamCount:
645  case LoopHintAttr::VectorizeWidth:
646  case LoopHintAttr::InterleaveCount:
647  case LoopHintAttr::PipelineInitiationInterval:
648  llvm_unreachable("Options cannot be disabled.");
649  break;
650  }
651  break;
652  case LoopHintAttr::Enable:
653  switch (Option) {
654  case LoopHintAttr::Vectorize:
655  case LoopHintAttr::Interleave:
656  setVectorizeEnable(true);
657  break;
658  case LoopHintAttr::Unroll:
659  setUnrollState(LoopAttributes::Enable);
660  break;
661  case LoopHintAttr::UnrollAndJam:
662  setUnrollAndJamState(LoopAttributes::Enable);
663  break;
664  case LoopHintAttr::VectorizePredicate:
665  setVectorizePredicateState(LoopAttributes::Enable);
666  break;
667  case LoopHintAttr::Distribute:
668  setDistributeState(true);
669  break;
670  case LoopHintAttr::UnrollCount:
671  case LoopHintAttr::UnrollAndJamCount:
672  case LoopHintAttr::VectorizeWidth:
673  case LoopHintAttr::InterleaveCount:
674  case LoopHintAttr::PipelineDisabled:
675  case LoopHintAttr::PipelineInitiationInterval:
676  llvm_unreachable("Options cannot enabled.");
677  break;
678  }
679  break;
680  case LoopHintAttr::AssumeSafety:
681  switch (Option) {
682  case LoopHintAttr::Vectorize:
683  case LoopHintAttr::Interleave:
684  // Apply "llvm.mem.parallel_loop_access" metadata to load/stores.
685  setParallel(true);
686  setVectorizeEnable(true);
687  break;
688  case LoopHintAttr::Unroll:
689  case LoopHintAttr::UnrollAndJam:
690  case LoopHintAttr::VectorizePredicate:
691  case LoopHintAttr::UnrollCount:
692  case LoopHintAttr::UnrollAndJamCount:
693  case LoopHintAttr::VectorizeWidth:
694  case LoopHintAttr::InterleaveCount:
695  case LoopHintAttr::Distribute:
696  case LoopHintAttr::PipelineDisabled:
697  case LoopHintAttr::PipelineInitiationInterval:
698  llvm_unreachable("Options cannot be used to assume mem safety.");
699  break;
700  }
701  break;
702  case LoopHintAttr::Full:
703  switch (Option) {
704  case LoopHintAttr::Unroll:
705  setUnrollState(LoopAttributes::Full);
706  break;
707  case LoopHintAttr::UnrollAndJam:
708  setUnrollAndJamState(LoopAttributes::Full);
709  break;
710  case LoopHintAttr::Vectorize:
711  case LoopHintAttr::Interleave:
712  case LoopHintAttr::UnrollCount:
713  case LoopHintAttr::UnrollAndJamCount:
714  case LoopHintAttr::VectorizeWidth:
715  case LoopHintAttr::InterleaveCount:
716  case LoopHintAttr::Distribute:
717  case LoopHintAttr::PipelineDisabled:
718  case LoopHintAttr::PipelineInitiationInterval:
719  case LoopHintAttr::VectorizePredicate:
720  llvm_unreachable("Options cannot be used with 'full' hint.");
721  break;
722  }
723  break;
724  case LoopHintAttr::Numeric:
725  switch (Option) {
726  case LoopHintAttr::VectorizeWidth:
727  setVectorizeWidth(ValueInt);
728  break;
729  case LoopHintAttr::InterleaveCount:
730  setInterleaveCount(ValueInt);
731  break;
732  case LoopHintAttr::UnrollCount:
733  setUnrollCount(ValueInt);
734  break;
735  case LoopHintAttr::UnrollAndJamCount:
736  setUnrollAndJamCount(ValueInt);
737  break;
738  case LoopHintAttr::PipelineInitiationInterval:
739  setPipelineInitiationInterval(ValueInt);
740  break;
741  case LoopHintAttr::Unroll:
742  case LoopHintAttr::UnrollAndJam:
743  case LoopHintAttr::VectorizePredicate:
744  case LoopHintAttr::Vectorize:
745  case LoopHintAttr::Interleave:
746  case LoopHintAttr::Distribute:
747  case LoopHintAttr::PipelineDisabled:
748  llvm_unreachable("Options cannot be assigned a value.");
749  break;
750  }
751  break;
752  }
753  }
754 
755  /// Stage the attributes.
756  push(Header, StartLoc, EndLoc);
757 }
758 
760  assert(!Active.empty() && "No active loops to pop");
761  Active.back()->finish();
762  Active.pop_back();
763 }
764 
765 void LoopInfoStack::InsertHelper(Instruction *I) const {
766  if (I->mayReadOrWriteMemory()) {
767  SmallVector<Metadata *, 4> AccessGroups;
768  for (const auto &AL : Active) {
769  // Here we assume that every loop that has an access group is parallel.
770  if (MDNode *Group = AL->getAccessGroup())
771  AccessGroups.push_back(Group);
772  }
773  MDNode *UnionMD = nullptr;
774  if (AccessGroups.size() == 1)
775  UnionMD = cast<MDNode>(AccessGroups[0]);
776  else if (AccessGroups.size() >= 2)
777  UnionMD = MDNode::get(I->getContext(), AccessGroups);
778  I->setMetadata("llvm.access.group", UnionMD);
779  }
780 
781  if (!hasInfo())
782  return;
783 
784  const LoopInfo &L = getInfo();
785  if (!L.getLoopID())
786  return;
787 
788  if (I->isTerminator()) {
789  for (BasicBlock *Succ : successors(I))
790  if (Succ == L.getHeader()) {
791  I->setMetadata(llvm::LLVMContext::MD_loop, L.getLoopID());
792  break;
793  }
794  return;
795  }
796 }
Defines the clang::ASTContext interface.
Specialize PointerLikeTypeTraits to allow LazyGenerationalUpdatePtr to be placed into a PointerUnion...
Definition: Dominators.h:30
Attributes that may be specified on loops.
Definition: CGLoopInfo.h:35
unsigned UnrollAndJamCount
llvm.unroll.
Definition: CGLoopInfo.h:67
Information used when generating a structured loop.
Definition: CGLoopInfo.h:80
LVEnableState VectorizePredicateEnable
Value for llvm.loop.vectorize.predicate metadata.
Definition: CGLoopInfo.h:55
LoopAttributes(bool IsParallel=false)
Definition: CGLoopInfo.cpp:432
LVEnableState UnrollEnable
Value for llvm.loop.unroll.* metadata (enable, disable, or full).
Definition: CGLoopInfo.h:49
unsigned PipelineInitiationInterval
Value for llvm.loop.pipeline.iicount metadata.
Definition: CGLoopInfo.h:76
Holds long-lived AST nodes (such as types and decls) that can be referred to throughout the semantic ...
Definition: ASTContext.h:168
LineState State
unsigned InterleaveCount
Value for llvm.loop.interleave.count metadata.
Definition: CGLoopInfo.h:61
LVEnableState VectorizeEnable
Value for llvm.loop.vectorize.enable metadata.
Definition: CGLoopInfo.h:46
void pop()
End the current loop.
Definition: CGLoopInfo.cpp:759
Whether values of this type can be null is (explicitly) unspecified.
LVEnableState UnrollAndJamEnable
Value for llvm.loop.unroll_and_jam.* metadata (enable, disable, or full).
Definition: CGLoopInfo.h:52
void finish()
Create the loop&#39;s metadata.
Definition: CGLoopInfo.cpp:483
NodeId Parent
Definition: ASTDiff.cpp:191
void InsertHelper(llvm::Instruction *I) const
Function called by the CodeGenFunction when an instruction is created.
Definition: CGLoopInfo.cpp:765
bool IsParallel
Generate llvm.loop.parallel metadata for loads and stores.
Definition: CGLoopInfo.h:40
unsigned UnrollCount
llvm.unroll.
Definition: CGLoopInfo.h:64
LVEnableState DistributeEnable
Value for llvm.loop.distribute.enable metadata.
Definition: CGLoopInfo.h:70
#define false
Definition: stdbool.h:17
llvm::MDNode * getLoopID() const
Get the loop id metadata for this loop.
Definition: CGLoopInfo.h:88
llvm::APSInt APSInt
constexpr XRayInstrMask None
Definition: XRayInstr.h:37
void push(llvm::BasicBlock *Header, const llvm::DebugLoc &StartLoc, const llvm::DebugLoc &EndLoc)
Begin a new structured loop.
bool PipelineDisabled
Value for llvm.loop.pipeline.disable metadata.
Definition: CGLoopInfo.h:73
static const TypeInfo & getInfo(unsigned id)
Definition: Types.cpp:38
LoopInfo(llvm::BasicBlock *Header, const LoopAttributes &Attrs, const llvm::DebugLoc &StartLoc, const llvm::DebugLoc &EndLoc, LoopInfo *Parent)
Construct a new LoopInfo for the loop with entry Header.
Definition: CGLoopInfo.cpp:456
unsigned VectorizeWidth
Value for llvm.loop.vectorize.width metadata.
Definition: CGLoopInfo.h:58
llvm::BasicBlock * getHeader() const
Get the header block of this loop.
Definition: CGLoopInfo.h:91
Attr - This represents one attribute.
Definition: Attr.h:45