28 #include "llvm/ADT/SmallPtrSet.h" 29 #include "llvm/ADT/StringExtras.h" 30 #include "llvm/IR/CallSite.h" 31 #include "llvm/IR/DataLayout.h" 32 #include "llvm/IR/InlineAsm.h" 33 #include "llvm/IR/Intrinsics.h" 34 #include "llvm/IR/MDBuilder.h" 35 #include "llvm/Support/ConvertUTF.h" 36 #include "llvm/Support/ScopedPrinter.h" 37 #include "llvm/Support/TargetParser.h" 40 using namespace clang;
41 using namespace CodeGen;
45 int64_t
clamp(int64_t
Value, int64_t Low, int64_t High) {
62 if (FD->
hasAttr<AsmLabelAttr>())
63 Name = getMangledName(D);
67 llvm::FunctionType *Ty =
68 cast<llvm::FunctionType>(getTypes().ConvertType(FD->
getType()));
70 return GetOrCreateLLVMFunction(Name, Ty, D,
false);
76 QualType T, llvm::IntegerType *IntType) {
79 if (V->getType()->isPointerTy())
80 return CGF.
Builder.CreatePtrToInt(V, IntType);
82 assert(V->getType() == IntType);
90 if (ResultType->isPointerTy())
91 return CGF.
Builder.CreateIntToPtr(V, ResultType);
93 assert(V->getType() == ResultType);
101 AtomicOrdering Ordering = AtomicOrdering::SequentiallyConsistent) {
109 unsigned AddrSpace = DestPtr->getType()->getPointerAddressSpace();
111 llvm::IntegerType *IntType =
114 llvm::Type *IntPtrType = IntType->getPointerTo(AddrSpace);
120 Args[1] =
EmitToInt(CGF, Args[1], T, IntType);
123 Kind, Args[0], Args[1], Ordering);
134 Address, llvm::PointerType::getUnqual(Val->getType()),
"cast");
150 llvm::AtomicRMWInst::BinOp
Kind,
159 llvm::AtomicRMWInst::BinOp
Kind,
161 Instruction::BinaryOps Op,
162 bool Invert =
false) {
170 unsigned AddrSpace = DestPtr->getType()->getPointerAddressSpace();
172 llvm::IntegerType *IntType =
175 llvm::Type *IntPtrType = IntType->getPointerTo(AddrSpace);
180 Args[1] =
EmitToInt(CGF, Args[1], T, IntType);
184 Kind, Args[0], Args[1], llvm::AtomicOrdering::SequentiallyConsistent);
185 Result = CGF.
Builder.CreateBinOp(Op, Result, Args[1]);
187 Result = CGF.
Builder.CreateBinOp(llvm::Instruction::Xor, Result,
188 llvm::ConstantInt::get(IntType, -1));
211 unsigned AddrSpace = DestPtr->getType()->getPointerAddressSpace();
213 llvm::IntegerType *IntType = llvm::IntegerType::get(
215 llvm::Type *IntPtrType = IntType->getPointerTo(AddrSpace);
221 Args[1] =
EmitToInt(CGF, Args[1], T, IntType);
225 Args[0], Args[1], Args[2], llvm::AtomicOrdering::SequentiallyConsistent,
226 llvm::AtomicOrdering::SequentiallyConsistent);
229 return CGF.
Builder.CreateZExt(CGF.
Builder.CreateExtractValue(Pair, 1),
252 AtomicOrdering SuccessOrdering = AtomicOrdering::SequentiallyConsistent) {
266 auto FailureOrdering = SuccessOrdering == AtomicOrdering::Release ?
267 AtomicOrdering::Monotonic :
270 auto *Result = CGF.
Builder.CreateAtomicCmpXchg(
271 Destination, Comparand, Exchange,
272 SuccessOrdering, FailureOrdering);
273 Result->setVolatile(
true);
274 return CGF.
Builder.CreateExtractValue(Result, 0);
278 AtomicOrdering Ordering = AtomicOrdering::SequentiallyConsistent) {
282 auto *Result = CGF.
Builder.CreateAtomicRMW(
285 ConstantInt::get(IntTy, 1),
287 return CGF.
Builder.CreateAdd(Result, ConstantInt::get(IntTy, 1));
291 AtomicOrdering Ordering = AtomicOrdering::SequentiallyConsistent) {
295 auto *Result = CGF.
Builder.CreateAtomicRMW(
298 ConstantInt::get(IntTy, 1),
300 return CGF.
Builder.CreateSub(Result, ConstantInt::get(IntTy, 1));
307 unsigned IntrinsicID) {
311 return CGF.
Builder.CreateCall(F, Src0);
317 unsigned IntrinsicID) {
322 return CGF.
Builder.CreateCall(F, { Src0, Src1 });
328 unsigned IntrinsicID) {
334 return CGF.
Builder.CreateCall(F, { Src0, Src1, Src2 });
340 unsigned IntrinsicID) {
345 return CGF.
Builder.CreateCall(F, {Src0, Src1});
351 llvm::CallInst *Call = CGF.
Builder.CreateCall(F, V);
352 Call->setDoesNotAccessMemory();
362 int Width = Ty->getPrimitiveSizeInBits();
363 llvm::Type *IntTy = llvm::IntegerType::get(C, Width);
365 if (Ty->isPPC_FP128Ty()) {
375 Value *ShiftCst = llvm::ConstantInt::get(IntTy, Width);
376 V = CGF.
Builder.CreateLShr(V, ShiftCst);
380 IntTy = llvm::IntegerType::get(C, Width);
381 V = CGF.
Builder.CreateTrunc(V, IntTy);
383 Value *Zero = llvm::Constant::getNullValue(IntTy);
384 return CGF.
Builder.CreateICmpSLT(V, Zero);
388 const CallExpr *E, llvm::Constant *calleeValue) {
407 assert(X->getType() == Y->getType() &&
408 "Arguments must be the same type. (Did you forget to make sure both " 409 "arguments have the same integer width?)");
413 Carry = CGF.
Builder.CreateExtractValue(Tmp, 1);
414 return CGF.
Builder.CreateExtractValue(Tmp, 0);
418 unsigned IntrinsicID,
421 llvm::MDNode *RNode = MDHelper.createRange(APInt(32, low), APInt(32, high));
423 llvm::Instruction *Call = CGF.
Builder.CreateCall(F);
424 Call->setMetadata(llvm::LLVMContext::MD_range, RNode);
429 struct WidthAndSignedness {
435 static WidthAndSignedness
438 assert(Type->
isIntegerType() &&
"Given type is not an integer.");
441 return {Width, Signed};
447 static struct WidthAndSignedness
449 assert(Types.size() > 0 &&
"Empty list of types.");
453 for (
const auto &
Type : Types) {
454 Signed |=
Type.Signed;
462 for (
const auto &
Type : Types) {
463 unsigned MinWidth =
Type.Width + (Signed && !
Type.Signed);
464 if (Width < MinWidth) {
469 return {Width, Signed};
474 if (ArgValue->getType() != DestType)
476 Builder.CreateBitCast(ArgValue, DestType, ArgValue->getName().data());
478 Intrinsic::ID inst = IsStart ? Intrinsic::vastart : Intrinsic::vaend;
479 return Builder.CreateCall(CGM.getIntrinsic(inst), ArgValue);
488 return From == To || (From == 0 && To == 1) || (From == 3 && To == 2);
493 return ConstantInt::get(ResType, (Type & 2) ? 0 : -1,
true);
497 CodeGenFunction::evaluateOrEmitBuiltinObjectSize(
const Expr *E,
unsigned Type,
498 llvm::IntegerType *ResType,
502 return emitBuiltinObjectSize(E,
Type, ResType, EmittedE);
503 return ConstantInt::get(ResType, ObjectSize,
true);
516 CodeGenFunction::emitBuiltinObjectSize(
const Expr *E,
unsigned Type,
517 llvm::IntegerType *ResType,
523 auto *PS = D->getDecl()->
getAttr<PassObjectSizeAttr>();
524 if (Param !=
nullptr && PS !=
nullptr &&
526 auto Iter = SizeArguments.find(Param);
527 assert(Iter != SizeArguments.end());
530 auto DIter = LocalDeclMap.find(D);
531 assert(DIter != LocalDeclMap.end());
533 return EmitLoadOfScalar(DIter->second,
false,
544 Value *Ptr = EmittedE ? EmittedE : EmitScalarExpr(E);
545 assert(Ptr->getType()->isPointerTy() &&
546 "Non-pointer passed to __builtin_object_size?");
548 Value *F = CGM.getIntrinsic(Intrinsic::objectsize, {ResType, Ptr->getType()});
551 Value *Min = Builder.getInt1((
Type & 2) != 0);
553 Value *NullIsUnknown = Builder.getTrue();
554 return Builder.CreateCall(F, {Ptr, Min, NullIsUnknown});
560 enum ActionKind : uint8_t { TestOnly, Complement, Reset, Set };
561 enum InterlockingKind : uint8_t {
570 InterlockingKind Interlocking;
573 static BitTest decodeBitTestBuiltin(
unsigned BuiltinID);
577 BitTest BitTest::decodeBitTestBuiltin(
unsigned BuiltinID) {
580 case Builtin::BI_bittest:
581 return {TestOnly, Unlocked,
false};
582 case Builtin::BI_bittestandcomplement:
583 return {Complement, Unlocked,
false};
584 case Builtin::BI_bittestandreset:
585 return {Reset, Unlocked,
false};
586 case Builtin::BI_bittestandset:
587 return {Set, Unlocked,
false};
588 case Builtin::BI_interlockedbittestandreset:
589 return {Reset, Sequential,
false};
590 case Builtin::BI_interlockedbittestandset:
591 return {Set, Sequential,
false};
594 case Builtin::BI_bittest64:
595 return {TestOnly, Unlocked,
true};
596 case Builtin::BI_bittestandcomplement64:
597 return {Complement, Unlocked,
true};
598 case Builtin::BI_bittestandreset64:
599 return {Reset, Unlocked,
true};
600 case Builtin::BI_bittestandset64:
601 return {Set, Unlocked,
true};
602 case Builtin::BI_interlockedbittestandreset64:
603 return {Reset, Sequential,
true};
604 case Builtin::BI_interlockedbittestandset64:
605 return {Set, Sequential,
true};
608 case Builtin::BI_interlockedbittestandset_acq:
609 return {Set, Acquire,
false};
610 case Builtin::BI_interlockedbittestandset_rel:
611 return {Set, Release,
false};
612 case Builtin::BI_interlockedbittestandset_nf:
613 return {Set, NoFence,
false};
614 case Builtin::BI_interlockedbittestandreset_acq:
615 return {Reset, Acquire,
false};
616 case Builtin::BI_interlockedbittestandreset_rel:
617 return {Reset, Release,
false};
618 case Builtin::BI_interlockedbittestandreset_nf:
619 return {Reset, NoFence,
false};
621 llvm_unreachable(
"expected only bittest intrinsics");
626 case BitTest::TestOnly:
return '\0';
627 case BitTest::Complement:
return 'c';
628 case BitTest::Reset:
return 'r';
629 case BitTest::Set:
return 's';
631 llvm_unreachable(
"invalid action");
639 char SizeSuffix = BT.Is64Bit ?
'q' :
'l';
643 raw_svector_ostream AsmOS(Asm);
644 if (BT.Interlocking != BitTest::Unlocked)
649 AsmOS << SizeSuffix <<
" $2, ($1)\n\tsetc ${0:b}";
652 std::string Constraints =
"=r,r,r,~{cc},~{flags},~{fpsr}";
653 llvm::IntegerType *IntType = llvm::IntegerType::get(
656 llvm::Type *IntPtrType = IntType->getPointerTo();
657 llvm::FunctionType *FTy =
658 llvm::FunctionType::get(CGF.
Int8Ty, {IntPtrType, IntType},
false);
660 llvm::InlineAsm *IA =
661 llvm::InlineAsm::get(FTy, Asm, Constraints,
true);
662 return CGF.
Builder.CreateCall(IA, {BitBase, BitPos});
665 static llvm::AtomicOrdering
668 case BitTest::Unlocked:
return llvm::AtomicOrdering::NotAtomic;
669 case BitTest::Sequential:
return llvm::AtomicOrdering::SequentiallyConsistent;
670 case BitTest::Acquire:
return llvm::AtomicOrdering::Acquire;
671 case BitTest::Release:
return llvm::AtomicOrdering::Release;
672 case BitTest::NoFence:
return llvm::AtomicOrdering::Monotonic;
674 llvm_unreachable(
"invalid interlocking");
687 BitTest BT = BitTest::decodeBitTestBuiltin(BuiltinID);
692 if (Arch == llvm::Triple::x86 || Arch == llvm::Triple::x86_64)
700 BitPos, llvm::ConstantInt::get(BitPos->getType(), 3),
"bittest.byteidx");
703 ByteIndex,
"bittest.byteaddr"),
707 llvm::ConstantInt::get(CGF.
Int8Ty, 0x7));
710 Value *Mask =
nullptr;
711 if (BT.Action != BitTest::TestOnly) {
712 Mask = CGF.
Builder.CreateShl(llvm::ConstantInt::get(CGF.
Int8Ty, 1), PosLow,
719 Value *OldByte =
nullptr;
720 if (Ordering != llvm::AtomicOrdering::NotAtomic) {
723 llvm::AtomicRMWInst::BinOp RMWOp = llvm::AtomicRMWInst::Or;
724 if (BT.Action == BitTest::Reset) {
725 Mask = CGF.
Builder.CreateNot(Mask);
728 OldByte = CGF.
Builder.CreateAtomicRMW(RMWOp, ByteAddr.getPointer(), Mask,
733 Value *NewByte =
nullptr;
735 case BitTest::TestOnly:
738 case BitTest::Complement:
739 NewByte = CGF.
Builder.CreateXor(OldByte, Mask);
742 NewByte = CGF.
Builder.CreateAnd(OldByte, CGF.
Builder.CreateNot(Mask));
745 NewByte = CGF.
Builder.CreateOr(OldByte, Mask);
754 Value *ShiftedByte = CGF.
Builder.CreateLShr(OldByte, PosLow,
"bittest.shr");
756 ShiftedByte, llvm::ConstantInt::get(CGF.
Int8Ty, 1),
"bittest.res");
775 bool IsVarArg =
false;
776 if (SJKind == MSVCSetJmpKind::_setjmp3) {
779 Arg1 = llvm::ConstantInt::get(CGF.
IntTy, 0);
782 Name = SJKind == MSVCSetJmpKind::_setjmp ?
"_setjmp" :
"_setjmpex";
788 llvm::ConstantInt::get(CGF.
Int32Ty, 0));
793 llvm::AttributeList ReturnsTwiceAttr = llvm::AttributeList::get(
795 llvm::Attribute::ReturnsTwice);
797 llvm::FunctionType::get(CGF.
IntTy, ArgTypes, IsVarArg), Name,
798 ReturnsTwiceAttr,
true);
804 CS.setAttributes(ReturnsTwiceAttr);
814 _InterlockedDecrement,
815 _InterlockedExchange,
816 _InterlockedExchangeAdd,
817 _InterlockedExchangeSub,
818 _InterlockedIncrement,
821 _InterlockedExchangeAdd_acq,
822 _InterlockedExchangeAdd_rel,
823 _InterlockedExchangeAdd_nf,
824 _InterlockedExchange_acq,
825 _InterlockedExchange_rel,
826 _InterlockedExchange_nf,
827 _InterlockedCompareExchange_acq,
828 _InterlockedCompareExchange_rel,
829 _InterlockedCompareExchange_nf,
839 _InterlockedIncrement_acq,
840 _InterlockedIncrement_rel,
841 _InterlockedIncrement_nf,
842 _InterlockedDecrement_acq,
843 _InterlockedDecrement_rel,
844 _InterlockedDecrement_nf,
851 case MSVCIntrin::_BitScanForward:
852 case MSVCIntrin::_BitScanReverse: {
857 EmitScalarExpr(E->
getArg(0))->getType()->getPointerElementType();
860 Value *ArgZero = llvm::Constant::getNullValue(ArgType);
861 Value *ResZero = llvm::Constant::getNullValue(ResultType);
862 Value *ResOne = llvm::ConstantInt::get(ResultType, 1);
864 BasicBlock *
Begin = Builder.GetInsertBlock();
865 BasicBlock *
End = createBasicBlock(
"bitscan_end", this->CurFn);
866 Builder.SetInsertPoint(End);
867 PHINode *Result = Builder.CreatePHI(ResultType, 2,
"bitscan_result");
869 Builder.SetInsertPoint(Begin);
870 Value *IsZero = Builder.CreateICmpEQ(ArgValue, ArgZero);
871 BasicBlock *NotZero = createBasicBlock(
"bitscan_not_zero", this->CurFn);
872 Builder.CreateCondBr(IsZero, End, NotZero);
873 Result->addIncoming(ResZero, Begin);
875 Builder.SetInsertPoint(NotZero);
876 Address IndexAddress = EmitPointerWithAlignment(E->
getArg(0));
878 if (BuiltinID == MSVCIntrin::_BitScanForward) {
879 Value *F = CGM.getIntrinsic(Intrinsic::cttz, ArgType);
880 Value *ZeroCount = Builder.CreateCall(F, {ArgValue, Builder.getTrue()});
881 ZeroCount = Builder.CreateIntCast(ZeroCount, IndexType,
false);
882 Builder.CreateStore(ZeroCount, IndexAddress,
false);
884 unsigned ArgWidth = cast<llvm::IntegerType>(ArgType)->getBitWidth();
885 Value *ArgTypeLastIndex = llvm::ConstantInt::get(IndexType, ArgWidth - 1);
887 Value *F = CGM.getIntrinsic(Intrinsic::ctlz, ArgType);
888 Value *ZeroCount = Builder.CreateCall(F, {ArgValue, Builder.getTrue()});
889 ZeroCount = Builder.CreateIntCast(ZeroCount, IndexType,
false);
890 Value *Index = Builder.CreateNSWSub(ArgTypeLastIndex, ZeroCount);
891 Builder.CreateStore(Index, IndexAddress,
false);
893 Builder.CreateBr(End);
894 Result->addIncoming(ResOne, NotZero);
896 Builder.SetInsertPoint(End);
899 case MSVCIntrin::_InterlockedAnd:
901 case MSVCIntrin::_InterlockedExchange:
903 case MSVCIntrin::_InterlockedExchangeAdd:
905 case MSVCIntrin::_InterlockedExchangeSub:
907 case MSVCIntrin::_InterlockedOr:
909 case MSVCIntrin::_InterlockedXor:
911 case MSVCIntrin::_InterlockedExchangeAdd_acq:
913 AtomicOrdering::Acquire);
914 case MSVCIntrin::_InterlockedExchangeAdd_rel:
916 AtomicOrdering::Release);
917 case MSVCIntrin::_InterlockedExchangeAdd_nf:
919 AtomicOrdering::Monotonic);
920 case MSVCIntrin::_InterlockedExchange_acq:
922 AtomicOrdering::Acquire);
923 case MSVCIntrin::_InterlockedExchange_rel:
925 AtomicOrdering::Release);
926 case MSVCIntrin::_InterlockedExchange_nf:
928 AtomicOrdering::Monotonic);
929 case MSVCIntrin::_InterlockedCompareExchange_acq:
931 case MSVCIntrin::_InterlockedCompareExchange_rel:
933 case MSVCIntrin::_InterlockedCompareExchange_nf:
935 case MSVCIntrin::_InterlockedOr_acq:
937 AtomicOrdering::Acquire);
938 case MSVCIntrin::_InterlockedOr_rel:
940 AtomicOrdering::Release);
941 case MSVCIntrin::_InterlockedOr_nf:
943 AtomicOrdering::Monotonic);
944 case MSVCIntrin::_InterlockedXor_acq:
946 AtomicOrdering::Acquire);
947 case MSVCIntrin::_InterlockedXor_rel:
949 AtomicOrdering::Release);
950 case MSVCIntrin::_InterlockedXor_nf:
952 AtomicOrdering::Monotonic);
953 case MSVCIntrin::_InterlockedAnd_acq:
955 AtomicOrdering::Acquire);
956 case MSVCIntrin::_InterlockedAnd_rel:
958 AtomicOrdering::Release);
959 case MSVCIntrin::_InterlockedAnd_nf:
961 AtomicOrdering::Monotonic);
962 case MSVCIntrin::_InterlockedIncrement_acq:
964 case MSVCIntrin::_InterlockedIncrement_rel:
966 case MSVCIntrin::_InterlockedIncrement_nf:
968 case MSVCIntrin::_InterlockedDecrement_acq:
970 case MSVCIntrin::_InterlockedDecrement_rel:
972 case MSVCIntrin::_InterlockedDecrement_nf:
975 case MSVCIntrin::_InterlockedDecrement:
977 case MSVCIntrin::_InterlockedIncrement:
980 case MSVCIntrin::__fastfail: {
984 llvm::Triple::ArchType ISA = getTarget().getTriple().getArch();
985 StringRef Asm, Constraints;
988 ErrorUnsupported(E,
"__fastfail call for this architecture");
990 case llvm::Triple::x86:
991 case llvm::Triple::x86_64:
993 Constraints =
"{cx}";
995 case llvm::Triple::thumb:
997 Constraints =
"{r0}";
1000 llvm::FunctionType *FTy = llvm::FunctionType::get(VoidTy, {Int32Ty},
false);
1001 llvm::InlineAsm *IA =
1002 llvm::InlineAsm::get(FTy, Asm, Constraints,
true);
1003 llvm::AttributeList NoReturnAttr = llvm::AttributeList::get(
1004 getLLVMContext(), llvm::AttributeList::FunctionIndex,
1005 llvm::Attribute::NoReturn);
1006 CallSite CS = Builder.CreateCall(IA, EmitScalarExpr(E->
getArg(0)));
1007 CS.setAttributes(NoReturnAttr);
1008 return CS.getInstruction();
1011 llvm_unreachable(
"Incorrect MSVC intrinsic!");
1017 CallObjCArcUse(
llvm::Value *
object) : object(
object) {}
1028 assert((Kind == BCK_CLZPassedZero || Kind == BCK_CTZPassedZero)
1029 &&
"Unsupported builtin check kind");
1031 Value *ArgValue = EmitScalarExpr(E);
1032 if (!SanOpts.has(SanitizerKind::Builtin) || !getTarget().isCLZForZeroUndef())
1036 Value *Cond = Builder.CreateICmpNE(
1037 ArgValue, llvm::Constant::getNullValue(ArgValue->getType()));
1038 EmitCheck(std::make_pair(Cond, SanitizerKind::Builtin),
1039 SanitizerHandler::InvalidBuiltin,
1041 llvm::ConstantInt::get(Builder.getInt8Ty(),
Kind)},
1059 raw_svector_ostream OS(Name);
1060 OS <<
"__os_log_helper";
1064 for (
const auto &Item : Layout.
Items)
1065 OS <<
"_" << int(Item.getSizeByte()) <<
"_" 1066 <<
int(Item.getDescriptorByte());
1069 if (llvm::Function *F = CGM.getModule().getFunction(Name))
1078 for (
unsigned int I = 0, E = Layout.
Items.size(); I < E; ++I) {
1079 char Size = Layout.
Items[I].getSizeByte();
1084 Params.emplace_back(
1086 &Ctx.
Idents.
get(std::string(
"arg") + llvm::to_string(I)), ArgTy,
1088 ArgTys.emplace_back(ArgTy);
1092 for (
auto &
P : Params)
1102 CGM.getTypes().arrangeBuiltinFunctionDeclaration(ReturnTy, Args);
1103 llvm::FunctionType *FuncTy = CGM.getTypes().GetFunctionType(FI);
1105 FuncTy, llvm::GlobalValue::LinkOnceODRLinkage, Name, &CGM.getModule());
1107 CGM.SetLLVMFunctionAttributes(
GlobalDecl(), FI, Fn);
1108 CGM.SetLLVMFunctionAttributesForDefinition(
nullptr, Fn);
1111 if (CGM.getCodeGenOpts().OptimizeSize == 2)
1112 Fn->addFnAttr(llvm::Attribute::NoInline);
1120 StartFunction(FD, ReturnTy, Fn, FI, Args);
1126 Address BufAddr(Builder.CreateLoad(GetAddrOfLocalVar(&Params[0]),
"buf"),
1129 Builder.CreateConstByteGEP(BufAddr, Offset++,
"summary"));
1131 Builder.CreateConstByteGEP(BufAddr, Offset++,
"numArgs"));
1134 for (
const auto &Item : Layout.
Items) {
1135 Builder.CreateStore(
1136 Builder.getInt8(Item.getDescriptorByte()),
1137 Builder.CreateConstByteGEP(BufAddr, Offset++,
"argDescriptor"));
1138 Builder.CreateStore(
1139 Builder.getInt8(Item.getSizeByte()),
1140 Builder.CreateConstByteGEP(BufAddr, Offset++,
"argSize"));
1146 Address Arg = GetAddrOfLocalVar(&Params[I]);
1147 Address Addr = Builder.CreateConstByteGEP(BufAddr, Offset,
"argData");
1148 Addr = Builder.CreateBitCast(Addr, Arg.
getPointer()->getType(),
1150 Builder.CreateStore(Builder.CreateLoad(Arg), Addr);
1162 "__builtin_os_log_format takes at least 2 arguments");
1173 for (
const auto &Item : Layout.
Items) {
1174 int Size = Item.getSizeByte();
1182 for (
unsigned I = 0, E = Item.getMaskType().size(); I < E; ++I)
1183 Val |= ((uint64_t)Item.getMaskType()[I]) << I * 8;
1184 ArgVal = llvm::Constant::getIntegerValue(Int64Ty, llvm::APInt(64, Val));
1185 }
else if (
const Expr *TheExpr = Item.getExpr()) {
1186 ArgVal = EmitScalarExpr(TheExpr,
false);
1189 if (TheExpr->getType()->isObjCRetainableType()) {
1190 assert(getEvaluationKind(TheExpr->getType()) ==
TEK_Scalar &&
1191 "Only scalar can be a ObjC retainable type");
1194 if (!isa<Constant>(ArgVal))
1195 RetainableOperands.push_back(ArgVal);
1198 ArgVal = Builder.getInt32(Item.getConstValue().getQuantity());
1201 unsigned ArgValSize =
1202 CGM.getDataLayout().getTypeSizeInBits(ArgVal->getType());
1203 llvm::IntegerType *IntTy = llvm::Type::getIntNTy(getLLVMContext(),
1205 ArgVal = Builder.CreateBitOrPointerCast(ArgVal, IntTy);
1208 ArgVal = Builder.CreateZExtOrBitCast(ArgVal, ConvertType(ArgTy));
1213 CGM.getTypes().arrangeBuiltinFunctionCall(Ctx.VoidTy, Args);
1224 if (!RetainableOperands.empty() && getLangOpts().ObjCAutoRefCount &&
1225 CGM.getCodeGenOpts().OptimizationLevel != 0)
1227 pushFullExprCleanup<CallObjCArcUse>(getARCCleanupKind(), Object);
1234 WidthAndSignedness Op1Info,
1235 WidthAndSignedness Op2Info,
1236 WidthAndSignedness ResultInfo) {
1237 return BuiltinID == Builtin::BI__builtin_mul_overflow &&
1238 std::max(Op1Info.Width, Op2Info.Width) >= ResultInfo.Width &&
1239 Op1Info.Signed != Op2Info.Signed;
1246 WidthAndSignedness Op1Info,
const clang::Expr *Op2,
1247 WidthAndSignedness Op2Info,
1249 WidthAndSignedness ResultInfo) {
1251 Op2Info, ResultInfo) &&
1252 "Not a mixed-sign multipliction we can specialize");
1255 const clang::Expr *SignedOp = Op1Info.Signed ? Op1 : Op2;
1256 const clang::Expr *UnsignedOp = Op1Info.Signed ? Op2 : Op1;
1259 unsigned SignedOpWidth = Op1Info.Signed ? Op1Info.Width : Op2Info.Width;
1260 unsigned UnsignedOpWidth = Op1Info.Signed ? Op2Info.Width : Op1Info.Width;
1263 if (SignedOpWidth < UnsignedOpWidth)
1264 Signed = CGF.
Builder.CreateSExt(Signed, Unsigned->getType(),
"op.sext");
1265 if (UnsignedOpWidth < SignedOpWidth)
1266 Unsigned = CGF.
Builder.CreateZExt(Unsigned, Signed->getType(),
"op.zext");
1269 llvm::Value *Zero = llvm::Constant::getNullValue(OpTy);
1272 unsigned OpWidth =
std::max(Op1Info.Width, Op2Info.Width);
1278 CGF.
Builder.CreateSelect(IsNegative, AbsOfNegative, Signed);
1284 Unsigned, UnsignedOverflow);
1287 if (ResultInfo.Signed) {
1291 llvm::APInt::getSignedMaxValue(ResultInfo.Width).zextOrSelf(OpWidth);
1293 CGF.
Builder.CreateAdd(llvm::ConstantInt::get(OpTy, IntMax),
1294 CGF.
Builder.CreateZExt(IsNegative, OpTy));
1296 CGF.
Builder.CreateICmpUGT(UnsignedResult, MaxResult);
1297 Overflow = CGF.
Builder.CreateOr(UnsignedOverflow, SignedOverflow);
1302 CGF.
Builder.CreateSelect(IsNegative, NegativeResult, UnsignedResult);
1303 Result = CGF.
Builder.CreateTrunc(SignedResult, ResTy);
1307 IsNegative, CGF.
Builder.CreateIsNotNull(UnsignedResult));
1308 Overflow = CGF.
Builder.CreateOr(UnsignedOverflow, Underflow);
1309 if (ResultInfo.Width < OpWidth) {
1311 llvm::APInt::getMaxValue(ResultInfo.Width).zext(OpWidth);
1313 UnsignedResult, llvm::ConstantInt::get(OpTy, IntMax));
1314 Overflow = CGF.
Builder.CreateOr(Overflow, TruncOverflow);
1318 Result = CGF.
Builder.CreateSelect(
1319 IsNegative, CGF.
Builder.CreateNeg(UnsignedResult), UnsignedResult);
1321 Result = CGF.
Builder.CreateTrunc(Result, ResTy);
1323 assert(Overflow && Result &&
"Missing overflow or result");
1340 std::string Pad = std::string(Lvl * 4,
' ');
1346 static llvm::DenseMap<QualType, const char *> Types;
1347 if (Types.empty()) {
1348 Types[Context.CharTy] =
"%c";
1349 Types[Context.BoolTy] =
"%d";
1350 Types[Context.SignedCharTy] =
"%hhd";
1351 Types[Context.UnsignedCharTy] =
"%hhu";
1352 Types[Context.IntTy] =
"%d";
1353 Types[Context.UnsignedIntTy] =
"%u";
1354 Types[Context.LongTy] =
"%ld";
1355 Types[Context.UnsignedLongTy] =
"%lu";
1356 Types[Context.LongLongTy] =
"%lld";
1357 Types[Context.UnsignedLongLongTy] =
"%llu";
1358 Types[Context.ShortTy] =
"%hd";
1359 Types[Context.UnsignedShortTy] =
"%hu";
1360 Types[Context.VoidPtrTy] =
"%p";
1361 Types[Context.FloatTy] =
"%f";
1362 Types[Context.DoubleTy] =
"%f";
1363 Types[Context.LongDoubleTy] =
"%Lf";
1364 Types[Context.getPointerType(Context.CharTy)] =
"%s";
1365 Types[Context.getPointerType(Context.getConstType(Context.CharTy))] =
"%s";
1368 for (
const auto *FD : RD->
fields()) {
1372 Value *FieldPtr = RecordPtr;
1374 FieldPtr = CGF.
Builder.CreatePointerCast(
1375 FieldPtr, CGF.
ConvertType(Context.getPointerType(FD->getType())));
1378 FD->getFieldIndex());
1380 GString = CGF.
Builder.CreateGlobalStringPtr(
1382 .concat(FD->getType().getAsString())
1383 .concat(llvm::Twine(
' '))
1384 .concat(FD->getNameAsString())
1387 Value *TmpRes = CGF.
Builder.CreateCall(Func, {GString});
1388 Res = CGF.
Builder.CreateAdd(Res, TmpRes);
1391 FD->getType().getUnqualifiedType().getCanonicalType();
1396 dumpRecord(CGF, CanonicalType, FieldPtr, Align, Func, Lvl + 1);
1397 Res = CGF.
Builder.CreateAdd(TmpRes, Res);
1402 llvm::Twine Format = Types.find(CanonicalType) == Types.end()
1403 ? Types[Context.VoidPtrTy]
1404 : Types[CanonicalType];
1410 GString = CGF.
Builder.CreateGlobalStringPtr(
1411 Format.concat(llvm::Twine(
'\n')).str());
1412 TmpRes = CGF.
Builder.CreateCall(Func, {GString, FieldPtr});
1413 Res = CGF.
Builder.CreateAdd(Res, TmpRes);
1416 GString = CGF.
Builder.CreateGlobalStringPtr(Pad +
"}\n");
1417 Value *TmpRes = CGF.
Builder.CreateCall(Func, {GString});
1418 Res = CGF.
Builder.CreateAdd(Res, TmpRes);
1424 llvm::SmallPtrSetImpl<const Decl *> &Seen) {
1433 if (!Seen.insert(Record).second)
1436 assert(Record->hasDefinition() &&
1437 "Incomplete types should already be diagnosed");
1439 if (Record->isDynamicClass())
1454 llvm::SmallPtrSet<const Decl *, 16> Seen;
1465 ShiftAmt = Builder.CreateIntCast(ShiftAmt, Ty,
false);
1468 unsigned IID = IsRotateRight ? Intrinsic::fshr : Intrinsic::fshl;
1469 Value *F = CGM.getIntrinsic(IID, Ty);
1470 return RValue::get(Builder.CreateCall(F, { Src, Src, ShiftAmt }));
1482 return RValue::get(llvm::ConstantInt::get(getLLVMContext(),
1485 return RValue::get(llvm::ConstantFP::get(getLLVMContext(),
1494 if (FD->
hasAttr<ConstAttr>()) {
1495 switch (BuiltinID) {
1496 case Builtin::BIceil:
1497 case Builtin::BIceilf:
1498 case Builtin::BIceill:
1499 case Builtin::BI__builtin_ceil:
1500 case Builtin::BI__builtin_ceilf:
1501 case Builtin::BI__builtin_ceill:
1504 case Builtin::BIcopysign:
1505 case Builtin::BIcopysignf:
1506 case Builtin::BIcopysignl:
1507 case Builtin::BI__builtin_copysign:
1508 case Builtin::BI__builtin_copysignf:
1509 case Builtin::BI__builtin_copysignl:
1510 case Builtin::BI__builtin_copysignf128:
1513 case Builtin::BIcos:
1514 case Builtin::BIcosf:
1515 case Builtin::BIcosl:
1516 case Builtin::BI__builtin_cos:
1517 case Builtin::BI__builtin_cosf:
1518 case Builtin::BI__builtin_cosl:
1521 case Builtin::BIexp:
1522 case Builtin::BIexpf:
1523 case Builtin::BIexpl:
1524 case Builtin::BI__builtin_exp:
1525 case Builtin::BI__builtin_expf:
1526 case Builtin::BI__builtin_expl:
1529 case Builtin::BIexp2:
1530 case Builtin::BIexp2f:
1531 case Builtin::BIexp2l:
1532 case Builtin::BI__builtin_exp2:
1533 case Builtin::BI__builtin_exp2f:
1534 case Builtin::BI__builtin_exp2l:
1537 case Builtin::BIfabs:
1538 case Builtin::BIfabsf:
1539 case Builtin::BIfabsl:
1540 case Builtin::BI__builtin_fabs:
1541 case Builtin::BI__builtin_fabsf:
1542 case Builtin::BI__builtin_fabsl:
1543 case Builtin::BI__builtin_fabsf128:
1546 case Builtin::BIfloor:
1547 case Builtin::BIfloorf:
1548 case Builtin::BIfloorl:
1549 case Builtin::BI__builtin_floor:
1550 case Builtin::BI__builtin_floorf:
1551 case Builtin::BI__builtin_floorl:
1554 case Builtin::BIfma:
1555 case Builtin::BIfmaf:
1556 case Builtin::BIfmal:
1557 case Builtin::BI__builtin_fma:
1558 case Builtin::BI__builtin_fmaf:
1559 case Builtin::BI__builtin_fmal:
1562 case Builtin::BIfmax:
1563 case Builtin::BIfmaxf:
1564 case Builtin::BIfmaxl:
1565 case Builtin::BI__builtin_fmax:
1566 case Builtin::BI__builtin_fmaxf:
1567 case Builtin::BI__builtin_fmaxl:
1570 case Builtin::BIfmin:
1571 case Builtin::BIfminf:
1572 case Builtin::BIfminl:
1573 case Builtin::BI__builtin_fmin:
1574 case Builtin::BI__builtin_fminf:
1575 case Builtin::BI__builtin_fminl:
1580 case Builtin::BIfmod:
1581 case Builtin::BIfmodf:
1582 case Builtin::BIfmodl:
1583 case Builtin::BI__builtin_fmod:
1584 case Builtin::BI__builtin_fmodf:
1585 case Builtin::BI__builtin_fmodl: {
1588 return RValue::get(Builder.CreateFRem(Arg1, Arg2,
"fmod"));
1591 case Builtin::BIlog:
1592 case Builtin::BIlogf:
1593 case Builtin::BIlogl:
1594 case Builtin::BI__builtin_log:
1595 case Builtin::BI__builtin_logf:
1596 case Builtin::BI__builtin_logl:
1599 case Builtin::BIlog10:
1600 case Builtin::BIlog10f:
1601 case Builtin::BIlog10l:
1602 case Builtin::BI__builtin_log10:
1603 case Builtin::BI__builtin_log10f:
1604 case Builtin::BI__builtin_log10l:
1607 case Builtin::BIlog2:
1608 case Builtin::BIlog2f:
1609 case Builtin::BIlog2l:
1610 case Builtin::BI__builtin_log2:
1611 case Builtin::BI__builtin_log2f:
1612 case Builtin::BI__builtin_log2l:
1615 case Builtin::BInearbyint:
1616 case Builtin::BInearbyintf:
1617 case Builtin::BInearbyintl:
1618 case Builtin::BI__builtin_nearbyint:
1619 case Builtin::BI__builtin_nearbyintf:
1620 case Builtin::BI__builtin_nearbyintl:
1623 case Builtin::BIpow:
1624 case Builtin::BIpowf:
1625 case Builtin::BIpowl:
1626 case Builtin::BI__builtin_pow:
1627 case Builtin::BI__builtin_powf:
1628 case Builtin::BI__builtin_powl:
1631 case Builtin::BIrint:
1632 case Builtin::BIrintf:
1633 case Builtin::BIrintl:
1634 case Builtin::BI__builtin_rint:
1635 case Builtin::BI__builtin_rintf:
1636 case Builtin::BI__builtin_rintl:
1639 case Builtin::BIround:
1640 case Builtin::BIroundf:
1641 case Builtin::BIroundl:
1642 case Builtin::BI__builtin_round:
1643 case Builtin::BI__builtin_roundf:
1644 case Builtin::BI__builtin_roundl:
1647 case Builtin::BIsin:
1648 case Builtin::BIsinf:
1649 case Builtin::BIsinl:
1650 case Builtin::BI__builtin_sin:
1651 case Builtin::BI__builtin_sinf:
1652 case Builtin::BI__builtin_sinl:
1655 case Builtin::BIsqrt:
1656 case Builtin::BIsqrtf:
1657 case Builtin::BIsqrtl:
1658 case Builtin::BI__builtin_sqrt:
1659 case Builtin::BI__builtin_sqrtf:
1660 case Builtin::BI__builtin_sqrtl:
1663 case Builtin::BItrunc:
1664 case Builtin::BItruncf:
1665 case Builtin::BItruncl:
1666 case Builtin::BI__builtin_trunc:
1667 case Builtin::BI__builtin_truncf:
1668 case Builtin::BI__builtin_truncl:
1676 switch (BuiltinID) {
1678 case Builtin::BI__builtin___CFStringMakeConstantString:
1679 case Builtin::BI__builtin___NSStringMakeConstantString:
1681 case Builtin::BI__builtin_stdarg_start:
1682 case Builtin::BI__builtin_va_start:
1683 case Builtin::BI__va_start:
1684 case Builtin::BI__builtin_va_end:
1686 EmitVAStartEnd(BuiltinID == Builtin::BI__va_start
1687 ? EmitScalarExpr(E->
getArg(0))
1688 : EmitVAListRef(E->
getArg(0)).getPointer(),
1689 BuiltinID != Builtin::BI__builtin_va_end));
1690 case Builtin::BI__builtin_va_copy: {
1691 Value *DstPtr = EmitVAListRef(E->
getArg(0)).getPointer();
1692 Value *SrcPtr = EmitVAListRef(E->
getArg(1)).getPointer();
1696 DstPtr = Builder.CreateBitCast(DstPtr, Type);
1697 SrcPtr = Builder.CreateBitCast(SrcPtr, Type);
1698 return RValue::get(Builder.CreateCall(CGM.getIntrinsic(Intrinsic::vacopy),
1701 case Builtin::BI__builtin_abs:
1702 case Builtin::BI__builtin_labs:
1703 case Builtin::BI__builtin_llabs: {
1707 Value *NegOp = Builder.CreateNSWNeg(ArgValue,
"neg");
1708 Constant *Zero = llvm::Constant::getNullValue(ArgValue->getType());
1709 Value *CmpResult = Builder.CreateICmpSLT(ArgValue, Zero,
"abscond");
1710 Value *Result = Builder.CreateSelect(CmpResult, NegOp, ArgValue,
"abs");
1713 case Builtin::BI__builtin_conj:
1714 case Builtin::BI__builtin_conjf:
1715 case Builtin::BI__builtin_conjl: {
1717 Value *Real = ComplexVal.first;
1718 Value *Imag = ComplexVal.second;
1720 Imag->getType()->isFPOrFPVectorTy()
1721 ? llvm::ConstantFP::getZeroValueForNegation(Imag->getType())
1722 : llvm::Constant::getNullValue(Imag->getType());
1724 Imag = Builder.CreateFSub(Zero, Imag,
"sub");
1727 case Builtin::BI__builtin_creal:
1728 case Builtin::BI__builtin_crealf:
1729 case Builtin::BI__builtin_creall:
1730 case Builtin::BIcreal:
1731 case Builtin::BIcrealf:
1732 case Builtin::BIcreall: {
1737 case Builtin::BI__builtin_dump_struct: {
1739 CharUnits Arg0Align = EmitPointerWithAlignment(E->
getArg(0)).getAlignment();
1744 Value *RecordPtr = EmitScalarExpr(Arg0);
1745 Value *Res =
dumpRecord(*
this, Arg0Type, RecordPtr, Arg0Align, Func, 0);
1749 case Builtin::BI__builtin_cimag:
1750 case Builtin::BI__builtin_cimagf:
1751 case Builtin::BI__builtin_cimagl:
1752 case Builtin::BIcimag:
1753 case Builtin::BIcimagf:
1754 case Builtin::BIcimagl: {
1759 case Builtin::BI__builtin_clrsb:
1760 case Builtin::BI__builtin_clrsbl:
1761 case Builtin::BI__builtin_clrsbll: {
1766 Value *F = CGM.getIntrinsic(Intrinsic::ctlz, ArgType);
1769 Value *Zero = llvm::Constant::getNullValue(ArgType);
1770 Value *IsNeg = Builder.CreateICmpSLT(ArgValue, Zero,
"isneg");
1771 Value *Inverse = Builder.CreateNot(ArgValue,
"not");
1772 Value *Tmp = Builder.CreateSelect(IsNeg, Inverse, ArgValue);
1773 Value *Ctlz = Builder.CreateCall(F, {Tmp, Builder.getFalse()});
1774 Value *Result = Builder.CreateSub(Ctlz, llvm::ConstantInt::get(ArgType, 1));
1775 Result = Builder.CreateIntCast(Result, ResultType,
true,
1779 case Builtin::BI__builtin_ctzs:
1780 case Builtin::BI__builtin_ctz:
1781 case Builtin::BI__builtin_ctzl:
1782 case Builtin::BI__builtin_ctzll: {
1783 Value *ArgValue = EmitCheckedArgForBuiltin(E->
getArg(0), BCK_CTZPassedZero);
1786 Value *F = CGM.getIntrinsic(Intrinsic::cttz, ArgType);
1789 Value *ZeroUndef = Builder.getInt1(getTarget().isCLZForZeroUndef());
1790 Value *Result = Builder.CreateCall(F, {ArgValue, ZeroUndef});
1791 if (Result->getType() != ResultType)
1792 Result = Builder.CreateIntCast(Result, ResultType,
true,
1796 case Builtin::BI__builtin_clzs:
1797 case Builtin::BI__builtin_clz:
1798 case Builtin::BI__builtin_clzl:
1799 case Builtin::BI__builtin_clzll: {
1800 Value *ArgValue = EmitCheckedArgForBuiltin(E->
getArg(0), BCK_CLZPassedZero);
1803 Value *F = CGM.getIntrinsic(Intrinsic::ctlz, ArgType);
1806 Value *ZeroUndef = Builder.getInt1(getTarget().isCLZForZeroUndef());
1807 Value *Result = Builder.CreateCall(F, {ArgValue, ZeroUndef});
1808 if (Result->getType() != ResultType)
1809 Result = Builder.CreateIntCast(Result, ResultType,
true,
1813 case Builtin::BI__builtin_ffs:
1814 case Builtin::BI__builtin_ffsl:
1815 case Builtin::BI__builtin_ffsll: {
1820 Value *F = CGM.getIntrinsic(Intrinsic::cttz, ArgType);
1824 Builder.CreateAdd(Builder.CreateCall(F, {ArgValue, Builder.getTrue()}),
1825 llvm::ConstantInt::get(ArgType, 1));
1826 Value *Zero = llvm::Constant::getNullValue(ArgType);
1827 Value *IsZero = Builder.CreateICmpEQ(ArgValue, Zero,
"iszero");
1828 Value *Result = Builder.CreateSelect(IsZero, Zero, Tmp,
"ffs");
1829 if (Result->getType() != ResultType)
1830 Result = Builder.CreateIntCast(Result, ResultType,
true,
1834 case Builtin::BI__builtin_parity:
1835 case Builtin::BI__builtin_parityl:
1836 case Builtin::BI__builtin_parityll: {
1841 Value *F = CGM.getIntrinsic(Intrinsic::ctpop, ArgType);
1844 Value *Tmp = Builder.CreateCall(F, ArgValue);
1845 Value *Result = Builder.CreateAnd(Tmp, llvm::ConstantInt::get(ArgType, 1));
1846 if (Result->getType() != ResultType)
1847 Result = Builder.CreateIntCast(Result, ResultType,
true,
1851 case Builtin::BI__lzcnt16:
1852 case Builtin::BI__lzcnt:
1853 case Builtin::BI__lzcnt64: {
1857 Value *F = CGM.getIntrinsic(Intrinsic::ctlz, ArgType);
1860 Value *Result = Builder.CreateCall(F, {ArgValue, Builder.getFalse()});
1861 if (Result->getType() != ResultType)
1862 Result = Builder.CreateIntCast(Result, ResultType,
true,
1866 case Builtin::BI__popcnt16:
1867 case Builtin::BI__popcnt:
1868 case Builtin::BI__popcnt64:
1869 case Builtin::BI__builtin_popcount:
1870 case Builtin::BI__builtin_popcountl:
1871 case Builtin::BI__builtin_popcountll: {
1875 Value *F = CGM.getIntrinsic(Intrinsic::ctpop, ArgType);
1878 Value *Result = Builder.CreateCall(F, ArgValue);
1879 if (Result->getType() != ResultType)
1880 Result = Builder.CreateIntCast(Result, ResultType,
true,
1884 case Builtin::BI__builtin_unpredictable: {
1890 case Builtin::BI__builtin_expect: {
1894 Value *ExpectedValue = EmitScalarExpr(E->
getArg(1));
1898 if (CGM.getCodeGenOpts().OptimizationLevel == 0)
1901 Value *FnExpect = CGM.getIntrinsic(Intrinsic::expect, ArgType);
1903 Builder.CreateCall(FnExpect, {ArgValue, ExpectedValue},
"expval");
1906 case Builtin::BI__builtin_assume_aligned: {
1908 Value *PtrValue = EmitScalarExpr(Ptr);
1909 Value *OffsetValue =
1912 Value *AlignmentValue = EmitScalarExpr(E->
getArg(1));
1913 ConstantInt *AlignmentCI = cast<ConstantInt>(AlignmentValue);
1914 unsigned Alignment = (unsigned)AlignmentCI->getZExtValue();
1917 Alignment, OffsetValue);
1920 case Builtin::BI__assume:
1921 case Builtin::BI__builtin_assume: {
1926 Value *FnAssume = CGM.getIntrinsic(Intrinsic::assume);
1927 return RValue::get(Builder.CreateCall(FnAssume, ArgValue));
1929 case Builtin::BI__builtin_bswap16:
1930 case Builtin::BI__builtin_bswap32:
1931 case Builtin::BI__builtin_bswap64: {
1934 case Builtin::BI__builtin_bitreverse8:
1935 case Builtin::BI__builtin_bitreverse16:
1936 case Builtin::BI__builtin_bitreverse32:
1937 case Builtin::BI__builtin_bitreverse64: {
1940 case Builtin::BI__builtin_rotateleft8:
1941 case Builtin::BI__builtin_rotateleft16:
1942 case Builtin::BI__builtin_rotateleft32:
1943 case Builtin::BI__builtin_rotateleft64:
1944 case Builtin::BI_rotl8:
1945 case Builtin::BI_rotl16:
1946 case Builtin::BI_rotl:
1947 case Builtin::BI_lrotl:
1948 case Builtin::BI_rotl64:
1949 return emitRotate(E,
false);
1951 case Builtin::BI__builtin_rotateright8:
1952 case Builtin::BI__builtin_rotateright16:
1953 case Builtin::BI__builtin_rotateright32:
1954 case Builtin::BI__builtin_rotateright64:
1955 case Builtin::BI_rotr8:
1956 case Builtin::BI_rotr16:
1957 case Builtin::BI_rotr:
1958 case Builtin::BI_lrotr:
1959 case Builtin::BI_rotr64:
1960 return emitRotate(E,
true);
1962 case Builtin::BI__builtin_constant_p: {
1964 if (CGM.getCodeGenOpts().OptimizationLevel == 0)
1967 return RValue::get(ConstantInt::get(ResultType, 0));
1971 if (!hasScalarEvaluationKind(ArgType) || ArgType->
isFunctionType())
1973 return RValue::get(ConstantInt::get(ResultType, 0));
1975 Value *ArgValue = EmitScalarExpr(Arg);
1976 Value *F = CGM.getIntrinsic(Intrinsic::is_constant, ConvertType(ArgType));
1977 Value *Result = Builder.CreateCall(F, ArgValue);
1978 if (Result->getType() != ResultType)
1979 Result = Builder.CreateIntCast(Result, ResultType,
false);
1982 case Builtin::BI__builtin_object_size: {
1985 auto *ResType = cast<llvm::IntegerType>(ConvertType(E->
getType()));
1992 case Builtin::BI__builtin_prefetch: {
1996 llvm::ConstantInt::get(Int32Ty, 0);
1998 llvm::ConstantInt::get(Int32Ty, 3);
1999 Value *Data = llvm::ConstantInt::get(Int32Ty, 1);
2001 return RValue::get(Builder.CreateCall(F, {Address, RW, Locality, Data}));
2003 case Builtin::BI__builtin_readcyclecounter: {
2004 Value *F = CGM.getIntrinsic(Intrinsic::readcyclecounter);
2007 case Builtin::BI__builtin___clear_cache: {
2010 Value *F = CGM.getIntrinsic(Intrinsic::clear_cache);
2011 return RValue::get(Builder.CreateCall(F, {Begin, End}));
2013 case Builtin::BI__builtin_trap:
2014 return RValue::get(EmitTrapCall(Intrinsic::trap));
2015 case Builtin::BI__debugbreak:
2016 return RValue::get(EmitTrapCall(Intrinsic::debugtrap));
2017 case Builtin::BI__builtin_unreachable: {
2021 EmitBlock(createBasicBlock(
"unreachable.cont"));
2026 case Builtin::BI__builtin_powi:
2027 case Builtin::BI__builtin_powif:
2028 case Builtin::BI__builtin_powil: {
2033 return RValue::get(Builder.CreateCall(F, {Base, Exponent}));
2036 case Builtin::BI__builtin_isgreater:
2037 case Builtin::BI__builtin_isgreaterequal:
2038 case Builtin::BI__builtin_isless:
2039 case Builtin::BI__builtin_islessequal:
2040 case Builtin::BI__builtin_islessgreater:
2041 case Builtin::BI__builtin_isunordered: {
2047 switch (BuiltinID) {
2048 default: llvm_unreachable(
"Unknown ordered comparison");
2049 case Builtin::BI__builtin_isgreater:
2050 LHS = Builder.CreateFCmpOGT(LHS, RHS,
"cmp");
2052 case Builtin::BI__builtin_isgreaterequal:
2053 LHS = Builder.CreateFCmpOGE(LHS, RHS,
"cmp");
2055 case Builtin::BI__builtin_isless:
2056 LHS = Builder.CreateFCmpOLT(LHS, RHS,
"cmp");
2058 case Builtin::BI__builtin_islessequal:
2059 LHS = Builder.CreateFCmpOLE(LHS, RHS,
"cmp");
2061 case Builtin::BI__builtin_islessgreater:
2062 LHS = Builder.CreateFCmpONE(LHS, RHS,
"cmp");
2064 case Builtin::BI__builtin_isunordered:
2065 LHS = Builder.CreateFCmpUNO(LHS, RHS,
"cmp");
2071 case Builtin::BI__builtin_isnan: {
2073 V = Builder.CreateFCmpUNO(V, V,
"cmp");
2077 case Builtin::BIfinite:
2078 case Builtin::BI__finite:
2079 case Builtin::BIfinitef:
2080 case Builtin::BI__finitef:
2081 case Builtin::BIfinitel:
2082 case Builtin::BI__finitel:
2083 case Builtin::BI__builtin_isinf:
2084 case Builtin::BI__builtin_isfinite: {
2090 Constant *Infinity = ConstantFP::getInfinity(V->getType());
2091 CmpInst::Predicate Pred = (BuiltinID == Builtin::BI__builtin_isinf)
2093 : CmpInst::FCMP_ONE;
2094 Value *FCmp = Builder.CreateFCmp(Pred, Fabs, Infinity,
"cmpinf");
2098 case Builtin::BI__builtin_isinf_sign: {
2102 Value *IsInf = Builder.CreateFCmpOEQ(
2103 AbsArg, ConstantFP::getInfinity(Arg->getType()),
"isinf");
2107 Value *Zero = Constant::getNullValue(IntTy);
2108 Value *One = ConstantInt::get(IntTy, 1);
2109 Value *NegativeOne = ConstantInt::get(IntTy, -1);
2110 Value *SignResult = Builder.CreateSelect(IsNeg, NegativeOne, One);
2111 Value *Result = Builder.CreateSelect(IsInf, SignResult, Zero);
2115 case Builtin::BI__builtin_isnormal: {
2118 Value *Eq = Builder.CreateFCmpOEQ(V, V,
"iseq");
2121 Value *IsLessThanInf =
2122 Builder.CreateFCmpULT(Abs, ConstantFP::getInfinity(V->getType()),
"isinf");
2123 APFloat Smallest = APFloat::getSmallestNormalized(
2124 getContext().getFloatTypeSemantics(E->
getArg(0)->
getType()));
2126 Builder.CreateFCmpUGE(Abs, ConstantFP::get(V->getContext(), Smallest),
2128 V = Builder.CreateAnd(Eq, IsLessThanInf,
"and");
2129 V = Builder.CreateAnd(V, IsNormal,
"and");
2133 case Builtin::BI__builtin_fpclassify: {
2138 BasicBlock *
Begin = Builder.GetInsertBlock();
2139 BasicBlock *
End = createBasicBlock(
"fpclassify_end", this->CurFn);
2140 Builder.SetInsertPoint(End);
2143 "fpclassify_result");
2146 Builder.SetInsertPoint(Begin);
2147 Value *IsZero = Builder.CreateFCmpOEQ(V, Constant::getNullValue(Ty),
2149 Value *ZeroLiteral = EmitScalarExpr(E->
getArg(4));
2150 BasicBlock *NotZero = createBasicBlock(
"fpclassify_not_zero", this->CurFn);
2151 Builder.CreateCondBr(IsZero, End, NotZero);
2152 Result->addIncoming(ZeroLiteral, Begin);
2155 Builder.SetInsertPoint(NotZero);
2156 Value *IsNan = Builder.CreateFCmpUNO(V, V,
"cmp");
2158 BasicBlock *NotNan = createBasicBlock(
"fpclassify_not_nan", this->CurFn);
2159 Builder.CreateCondBr(IsNan, End, NotNan);
2160 Result->addIncoming(NanLiteral, NotZero);
2163 Builder.SetInsertPoint(NotNan);
2166 Builder.CreateFCmpOEQ(VAbs, ConstantFP::getInfinity(V->getType()),
2169 BasicBlock *NotInf = createBasicBlock(
"fpclassify_not_inf", this->CurFn);
2170 Builder.CreateCondBr(IsInf, End, NotInf);
2171 Result->addIncoming(InfLiteral, NotNan);
2174 Builder.SetInsertPoint(NotInf);
2175 APFloat Smallest = APFloat::getSmallestNormalized(
2176 getContext().getFloatTypeSemantics(E->
getArg(5)->
getType()));
2178 Builder.CreateFCmpUGE(VAbs, ConstantFP::get(V->getContext(), Smallest),
2180 Value *NormalResult =
2181 Builder.CreateSelect(IsNormal, EmitScalarExpr(E->
getArg(2)),
2182 EmitScalarExpr(E->
getArg(3)));
2183 Builder.CreateBr(End);
2184 Result->addIncoming(NormalResult, NotInf);
2187 Builder.SetInsertPoint(End);
2191 case Builtin::BIalloca:
2192 case Builtin::BI_alloca:
2193 case Builtin::BI__builtin_alloca: {
2195 const TargetInfo &TI = getContext().getTargetInfo();
2197 unsigned SuitableAlignmentInBytes =
2199 .toCharUnitsFromBits(TI.getSuitableAlign())
2201 AllocaInst *AI = Builder.CreateAlloca(Builder.getInt8Ty(), Size);
2202 AI->setAlignment(SuitableAlignmentInBytes);
2206 case Builtin::BI__builtin_alloca_with_align: {
2208 Value *AlignmentInBitsValue = EmitScalarExpr(E->
getArg(1));
2209 auto *AlignmentInBitsCI = cast<ConstantInt>(AlignmentInBitsValue);
2210 unsigned AlignmentInBits = AlignmentInBitsCI->getZExtValue();
2211 unsigned AlignmentInBytes =
2212 CGM.getContext().toCharUnitsFromBits(AlignmentInBits).getQuantity();
2213 AllocaInst *AI = Builder.CreateAlloca(Builder.getInt8Ty(), Size);
2214 AI->setAlignment(AlignmentInBytes);
2218 case Builtin::BIbzero:
2219 case Builtin::BI__builtin_bzero: {
2224 Builder.CreateMemSet(Dest, Builder.getInt8(0), SizeVal,
false);
2227 case Builtin::BImemcpy:
2228 case Builtin::BI__builtin_memcpy: {
2236 Builder.CreateMemCpy(Dest, Src, SizeVal,
false);
2240 case Builtin::BI__builtin_char_memchr:
2241 BuiltinID = Builtin::BI__builtin_memchr;
2244 case Builtin::BI__builtin___memcpy_chk: {
2250 llvm::APSInt Size = SizeResult.
Val.
getInt();
2251 llvm::APSInt DstSize = DstSizeResult.
Val.
getInt();
2252 if (Size.ugt(DstSize))
2256 Value *SizeVal = llvm::ConstantInt::get(Builder.getContext(), Size);
2257 Builder.CreateMemCpy(Dest, Src, SizeVal,
false);
2261 case Builtin::BI__builtin_objc_memmove_collectable: {
2265 CGM.getObjCRuntime().EmitGCMemmoveCollectable(*
this,
2266 DestAddr, SrcAddr, SizeVal);
2270 case Builtin::BI__builtin___memmove_chk: {
2276 llvm::APSInt Size = SizeResult.
Val.
getInt();
2277 llvm::APSInt DstSize = DstSizeResult.
Val.
getInt();
2278 if (Size.ugt(DstSize))
2282 Value *SizeVal = llvm::ConstantInt::get(Builder.getContext(), Size);
2283 Builder.CreateMemMove(Dest, Src, SizeVal,
false);
2287 case Builtin::BImemmove:
2288 case Builtin::BI__builtin_memmove: {
2296 Builder.CreateMemMove(Dest, Src, SizeVal,
false);
2299 case Builtin::BImemset:
2300 case Builtin::BI__builtin_memset: {
2302 Value *ByteVal = Builder.CreateTrunc(EmitScalarExpr(E->
getArg(1)),
2303 Builder.getInt8Ty());
2307 Builder.CreateMemSet(Dest, ByteVal, SizeVal,
false);
2310 case Builtin::BI__builtin___memset_chk: {
2316 llvm::APSInt Size = SizeResult.
Val.
getInt();
2317 llvm::APSInt DstSize = DstSizeResult.
Val.
getInt();
2318 if (Size.ugt(DstSize))
2321 Value *ByteVal = Builder.CreateTrunc(EmitScalarExpr(E->
getArg(1)),
2322 Builder.getInt8Ty());
2323 Value *SizeVal = llvm::ConstantInt::get(Builder.getContext(), Size);
2324 Builder.CreateMemSet(Dest, ByteVal, SizeVal,
false);
2327 case Builtin::BI__builtin_wmemcmp: {
2330 if (!getTarget().getTriple().isOSMSVCRT())
2333 llvm::Type *WCharTy = ConvertType(getContext().WCharTy);
2339 BasicBlock *Entry = Builder.GetInsertBlock();
2340 BasicBlock *CmpGT = createBasicBlock(
"wmemcmp.gt");
2341 BasicBlock *CmpLT = createBasicBlock(
"wmemcmp.lt");
2342 BasicBlock *Next = createBasicBlock(
"wmemcmp.next");
2343 BasicBlock *Exit = createBasicBlock(
"wmemcmp.exit");
2344 Value *SizeEq0 = Builder.CreateICmpEQ(Size, ConstantInt::get(SizeTy, 0));
2345 Builder.CreateCondBr(SizeEq0, Exit, CmpGT);
2348 PHINode *DstPhi = Builder.CreatePHI(Dst->getType(), 2);
2349 DstPhi->addIncoming(Dst, Entry);
2350 PHINode *SrcPhi = Builder.CreatePHI(Src->getType(), 2);
2351 SrcPhi->addIncoming(Src, Entry);
2352 PHINode *SizePhi = Builder.CreatePHI(SizeTy, 2);
2353 SizePhi->addIncoming(Size, Entry);
2355 getContext().getTypeAlignInChars(getContext().WCharTy);
2356 Value *DstCh = Builder.CreateAlignedLoad(WCharTy, DstPhi, WCharAlign);
2357 Value *SrcCh = Builder.CreateAlignedLoad(WCharTy, SrcPhi, WCharAlign);
2358 Value *DstGtSrc = Builder.CreateICmpUGT(DstCh, SrcCh);
2359 Builder.CreateCondBr(DstGtSrc, Exit, CmpLT);
2362 Value *DstLtSrc = Builder.CreateICmpULT(DstCh, SrcCh);
2363 Builder.CreateCondBr(DstLtSrc, Exit, Next);
2366 Value *NextDst = Builder.CreateConstInBoundsGEP1_32(WCharTy, DstPhi, 1);
2367 Value *NextSrc = Builder.CreateConstInBoundsGEP1_32(WCharTy, SrcPhi, 1);
2368 Value *NextSize = Builder.CreateSub(SizePhi, ConstantInt::get(SizeTy, 1));
2369 Value *NextSizeEq0 =
2370 Builder.CreateICmpEQ(NextSize, ConstantInt::get(SizeTy, 0));
2371 Builder.CreateCondBr(NextSizeEq0, Exit, CmpGT);
2372 DstPhi->addIncoming(NextDst, Next);
2373 SrcPhi->addIncoming(NextSrc, Next);
2374 SizePhi->addIncoming(NextSize, Next);
2377 PHINode *Ret = Builder.CreatePHI(IntTy, 4);
2378 Ret->addIncoming(ConstantInt::get(IntTy, 0), Entry);
2379 Ret->addIncoming(ConstantInt::get(IntTy, 1), CmpGT);
2380 Ret->addIncoming(ConstantInt::get(IntTy, -1), CmpLT);
2381 Ret->addIncoming(ConstantInt::get(IntTy, 0), Next);
2384 case Builtin::BI__builtin_dwarf_cfa: {
2395 Value *F = CGM.getIntrinsic(Intrinsic::eh_dwarf_cfa);
2397 llvm::ConstantInt::get(Int32Ty, Offset)));
2399 case Builtin::BI__builtin_return_address: {
2401 getContext().UnsignedIntTy);
2402 Value *F = CGM.getIntrinsic(Intrinsic::returnaddress);
2405 case Builtin::BI_ReturnAddress: {
2406 Value *F = CGM.getIntrinsic(Intrinsic::returnaddress);
2407 return RValue::get(Builder.CreateCall(F, Builder.getInt32(0)));
2409 case Builtin::BI__builtin_frame_address: {
2411 getContext().UnsignedIntTy);
2412 Value *F = CGM.getIntrinsic(Intrinsic::frameaddress);
2415 case Builtin::BI__builtin_extract_return_addr: {
2417 Value *Result = getTargetHooks().decodeReturnAddress(*
this, Address);
2420 case Builtin::BI__builtin_frob_return_addr: {
2422 Value *Result = getTargetHooks().encodeReturnAddress(*
this, Address);
2425 case Builtin::BI__builtin_dwarf_sp_column: {
2426 llvm::IntegerType *Ty
2427 = cast<llvm::IntegerType>(ConvertType(E->
getType()));
2428 int Column = getTargetHooks().getDwarfEHStackPointer(CGM);
2430 CGM.ErrorUnsupported(E,
"__builtin_dwarf_sp_column");
2433 return RValue::get(llvm::ConstantInt::get(Ty, Column,
true));
2435 case Builtin::BI__builtin_init_dwarf_reg_size_table: {
2437 if (getTargetHooks().initDwarfEHRegSizeTable(*
this, Address))
2438 CGM.ErrorUnsupported(E,
"__builtin_init_dwarf_reg_size_table");
2441 case Builtin::BI__builtin_eh_return: {
2445 llvm::IntegerType *IntTy = cast<llvm::IntegerType>(Int->getType());
2446 assert((IntTy->getBitWidth() == 32 || IntTy->getBitWidth() == 64) &&
2447 "LLVM's __builtin_eh_return only supports 32- and 64-bit variants");
2448 Value *F = CGM.getIntrinsic(IntTy->getBitWidth() == 32
2449 ? Intrinsic::eh_return_i32
2450 : Intrinsic::eh_return_i64);
2451 Builder.CreateCall(F, {Int, Ptr});
2452 Builder.CreateUnreachable();
2455 EmitBlock(createBasicBlock(
"builtin_eh_return.cont"));
2459 case Builtin::BI__builtin_unwind_init: {
2460 Value *F = CGM.getIntrinsic(Intrinsic::eh_unwind_init);
2463 case Builtin::BI__builtin_extend_pointer: {
2476 Value *Result = Builder.CreatePtrToInt(Ptr, IntPtrTy,
"extend.cast");
2479 if (IntPtrTy->getBitWidth() == 64)
2483 if (getTargetHooks().extendPointerWithSExt())
2484 return RValue::get(Builder.CreateSExt(Result, Int64Ty,
"extend.sext"));
2486 return RValue::get(Builder.CreateZExt(Result, Int64Ty,
"extend.zext"));
2488 case Builtin::BI__builtin_setjmp: {
2494 Builder.CreateCall(CGM.getIntrinsic(Intrinsic::frameaddress),
2495 ConstantInt::get(Int32Ty, 0));
2496 Builder.CreateStore(FrameAddr, Buf);
2500 Builder.CreateCall(CGM.getIntrinsic(Intrinsic::stacksave));
2502 Builder.CreateConstInBoundsGEP(Buf, 2, getPointerSize());
2503 Builder.CreateStore(StackAddr, StackSaveSlot);
2506 Value *F = CGM.getIntrinsic(Intrinsic::eh_sjlj_setjmp);
2507 Buf = Builder.CreateBitCast(Buf, Int8PtrTy);
2510 case Builtin::BI__builtin_longjmp: {
2512 Buf = Builder.CreateBitCast(Buf, Int8PtrTy);
2515 Builder.CreateCall(CGM.getIntrinsic(Intrinsic::eh_sjlj_longjmp), Buf);
2518 Builder.CreateUnreachable();
2521 EmitBlock(createBasicBlock(
"longjmp.cont"));
2525 case Builtin::BI__builtin_launder: {
2528 Value *Ptr = EmitScalarExpr(Arg);
2530 Ptr = Builder.CreateLaunderInvariantGroup(Ptr);
2534 case Builtin::BI__sync_fetch_and_add:
2535 case Builtin::BI__sync_fetch_and_sub:
2536 case Builtin::BI__sync_fetch_and_or:
2537 case Builtin::BI__sync_fetch_and_and:
2538 case Builtin::BI__sync_fetch_and_xor:
2539 case Builtin::BI__sync_fetch_and_nand:
2540 case Builtin::BI__sync_add_and_fetch:
2541 case Builtin::BI__sync_sub_and_fetch:
2542 case Builtin::BI__sync_and_and_fetch:
2543 case Builtin::BI__sync_or_and_fetch:
2544 case Builtin::BI__sync_xor_and_fetch:
2545 case Builtin::BI__sync_nand_and_fetch:
2546 case Builtin::BI__sync_val_compare_and_swap:
2547 case Builtin::BI__sync_bool_compare_and_swap:
2548 case Builtin::BI__sync_lock_test_and_set:
2549 case Builtin::BI__sync_lock_release:
2550 case Builtin::BI__sync_swap:
2551 llvm_unreachable(
"Shouldn't make it through sema");
2552 case Builtin::BI__sync_fetch_and_add_1:
2553 case Builtin::BI__sync_fetch_and_add_2:
2554 case Builtin::BI__sync_fetch_and_add_4:
2555 case Builtin::BI__sync_fetch_and_add_8:
2556 case Builtin::BI__sync_fetch_and_add_16:
2558 case Builtin::BI__sync_fetch_and_sub_1:
2559 case Builtin::BI__sync_fetch_and_sub_2:
2560 case Builtin::BI__sync_fetch_and_sub_4:
2561 case Builtin::BI__sync_fetch_and_sub_8:
2562 case Builtin::BI__sync_fetch_and_sub_16:
2564 case Builtin::BI__sync_fetch_and_or_1:
2565 case Builtin::BI__sync_fetch_and_or_2:
2566 case Builtin::BI__sync_fetch_and_or_4:
2567 case Builtin::BI__sync_fetch_and_or_8:
2568 case Builtin::BI__sync_fetch_and_or_16:
2570 case Builtin::BI__sync_fetch_and_and_1:
2571 case Builtin::BI__sync_fetch_and_and_2:
2572 case Builtin::BI__sync_fetch_and_and_4:
2573 case Builtin::BI__sync_fetch_and_and_8:
2574 case Builtin::BI__sync_fetch_and_and_16:
2576 case Builtin::BI__sync_fetch_and_xor_1:
2577 case Builtin::BI__sync_fetch_and_xor_2:
2578 case Builtin::BI__sync_fetch_and_xor_4:
2579 case Builtin::BI__sync_fetch_and_xor_8:
2580 case Builtin::BI__sync_fetch_and_xor_16:
2582 case Builtin::BI__sync_fetch_and_nand_1:
2583 case Builtin::BI__sync_fetch_and_nand_2:
2584 case Builtin::BI__sync_fetch_and_nand_4:
2585 case Builtin::BI__sync_fetch_and_nand_8:
2586 case Builtin::BI__sync_fetch_and_nand_16:
2590 case Builtin::BI__sync_fetch_and_min:
2592 case Builtin::BI__sync_fetch_and_max:
2594 case Builtin::BI__sync_fetch_and_umin:
2596 case Builtin::BI__sync_fetch_and_umax:
2599 case Builtin::BI__sync_add_and_fetch_1:
2600 case Builtin::BI__sync_add_and_fetch_2:
2601 case Builtin::BI__sync_add_and_fetch_4:
2602 case Builtin::BI__sync_add_and_fetch_8:
2603 case Builtin::BI__sync_add_and_fetch_16:
2605 llvm::Instruction::Add);
2606 case Builtin::BI__sync_sub_and_fetch_1:
2607 case Builtin::BI__sync_sub_and_fetch_2:
2608 case Builtin::BI__sync_sub_and_fetch_4:
2609 case Builtin::BI__sync_sub_and_fetch_8:
2610 case Builtin::BI__sync_sub_and_fetch_16:
2612 llvm::Instruction::Sub);
2613 case Builtin::BI__sync_and_and_fetch_1:
2614 case Builtin::BI__sync_and_and_fetch_2:
2615 case Builtin::BI__sync_and_and_fetch_4:
2616 case Builtin::BI__sync_and_and_fetch_8:
2617 case Builtin::BI__sync_and_and_fetch_16:
2620 case Builtin::BI__sync_or_and_fetch_1:
2621 case Builtin::BI__sync_or_and_fetch_2:
2622 case Builtin::BI__sync_or_and_fetch_4:
2623 case Builtin::BI__sync_or_and_fetch_8:
2624 case Builtin::BI__sync_or_and_fetch_16:
2626 llvm::Instruction::Or);
2627 case Builtin::BI__sync_xor_and_fetch_1:
2628 case Builtin::BI__sync_xor_and_fetch_2:
2629 case Builtin::BI__sync_xor_and_fetch_4:
2630 case Builtin::BI__sync_xor_and_fetch_8:
2631 case Builtin::BI__sync_xor_and_fetch_16:
2633 llvm::Instruction::Xor);
2634 case Builtin::BI__sync_nand_and_fetch_1:
2635 case Builtin::BI__sync_nand_and_fetch_2:
2636 case Builtin::BI__sync_nand_and_fetch_4:
2637 case Builtin::BI__sync_nand_and_fetch_8:
2638 case Builtin::BI__sync_nand_and_fetch_16:
2642 case Builtin::BI__sync_val_compare_and_swap_1:
2643 case Builtin::BI__sync_val_compare_and_swap_2:
2644 case Builtin::BI__sync_val_compare_and_swap_4:
2645 case Builtin::BI__sync_val_compare_and_swap_8:
2646 case Builtin::BI__sync_val_compare_and_swap_16:
2649 case Builtin::BI__sync_bool_compare_and_swap_1:
2650 case Builtin::BI__sync_bool_compare_and_swap_2:
2651 case Builtin::BI__sync_bool_compare_and_swap_4:
2652 case Builtin::BI__sync_bool_compare_and_swap_8:
2653 case Builtin::BI__sync_bool_compare_and_swap_16:
2656 case Builtin::BI__sync_swap_1:
2657 case Builtin::BI__sync_swap_2:
2658 case Builtin::BI__sync_swap_4:
2659 case Builtin::BI__sync_swap_8:
2660 case Builtin::BI__sync_swap_16:
2663 case Builtin::BI__sync_lock_test_and_set_1:
2664 case Builtin::BI__sync_lock_test_and_set_2:
2665 case Builtin::BI__sync_lock_test_and_set_4:
2666 case Builtin::BI__sync_lock_test_and_set_8:
2667 case Builtin::BI__sync_lock_test_and_set_16:
2670 case Builtin::BI__sync_lock_release_1:
2671 case Builtin::BI__sync_lock_release_2:
2672 case Builtin::BI__sync_lock_release_4:
2673 case Builtin::BI__sync_lock_release_8:
2674 case Builtin::BI__sync_lock_release_16: {
2677 CharUnits StoreSize = getContext().getTypeSizeInChars(ElTy);
2678 llvm::Type *ITy = llvm::IntegerType::get(getLLVMContext(),
2680 Ptr = Builder.CreateBitCast(Ptr, ITy->getPointerTo());
2681 llvm::StoreInst *
Store =
2682 Builder.CreateAlignedStore(llvm::Constant::getNullValue(ITy), Ptr,
2684 Store->setAtomic(llvm::AtomicOrdering::Release);
2688 case Builtin::BI__sync_synchronize: {
2696 Builder.CreateFence(llvm::AtomicOrdering::SequentiallyConsistent);
2700 case Builtin::BI__builtin_nontemporal_load:
2702 case Builtin::BI__builtin_nontemporal_store:
2704 case Builtin::BI__c11_atomic_is_lock_free:
2705 case Builtin::BI__atomic_is_lock_free: {
2709 const char *LibCallName =
"__atomic_is_lock_free";
2712 getContext().getSizeType());
2713 if (BuiltinID == Builtin::BI__atomic_is_lock_free)
2715 getContext().VoidPtrTy);
2718 getContext().VoidPtrTy);
2720 CGM.getTypes().arrangeBuiltinFunctionCall(E->
getType(), Args);
2721 llvm::FunctionType *FTy = CGM.getTypes().GetFunctionType(FuncInfo);
2722 llvm::Constant *Func = CGM.CreateRuntimeFunction(FTy, LibCallName);
2727 case Builtin::BI__atomic_test_and_set: {
2735 unsigned AddrSpace = Ptr->getType()->getPointerAddressSpace();
2736 Ptr = Builder.CreateBitCast(Ptr, Int8Ty->getPointerTo(AddrSpace));
2737 Value *NewVal = Builder.getInt8(1);
2739 if (isa<llvm::ConstantInt>(Order)) {
2740 int ord = cast<llvm::ConstantInt>(Order)->getZExtValue();
2741 AtomicRMWInst *Result =
nullptr;
2745 Result = Builder.CreateAtomicRMW(llvm::AtomicRMWInst::Xchg, Ptr, NewVal,
2746 llvm::AtomicOrdering::Monotonic);
2750 Result = Builder.CreateAtomicRMW(llvm::AtomicRMWInst::Xchg, Ptr, NewVal,
2751 llvm::AtomicOrdering::Acquire);
2754 Result = Builder.CreateAtomicRMW(llvm::AtomicRMWInst::Xchg, Ptr, NewVal,
2755 llvm::AtomicOrdering::Release);
2759 Result = Builder.CreateAtomicRMW(llvm::AtomicRMWInst::Xchg, Ptr, NewVal,
2760 llvm::AtomicOrdering::AcquireRelease);
2763 Result = Builder.CreateAtomicRMW(
2764 llvm::AtomicRMWInst::Xchg, Ptr, NewVal,
2765 llvm::AtomicOrdering::SequentiallyConsistent);
2768 Result->setVolatile(Volatile);
2769 return RValue::get(Builder.CreateIsNotNull(Result,
"tobool"));
2772 llvm::BasicBlock *ContBB = createBasicBlock(
"atomic.continue", CurFn);
2774 llvm::BasicBlock *BBs[5] = {
2775 createBasicBlock(
"monotonic", CurFn),
2776 createBasicBlock(
"acquire", CurFn),
2777 createBasicBlock(
"release", CurFn),
2778 createBasicBlock(
"acqrel", CurFn),
2779 createBasicBlock(
"seqcst", CurFn)
2781 llvm::AtomicOrdering Orders[5] = {
2782 llvm::AtomicOrdering::Monotonic, llvm::AtomicOrdering::Acquire,
2783 llvm::AtomicOrdering::Release, llvm::AtomicOrdering::AcquireRelease,
2784 llvm::AtomicOrdering::SequentiallyConsistent};
2786 Order = Builder.CreateIntCast(Order, Builder.getInt32Ty(),
false);
2787 llvm::SwitchInst *SI = Builder.CreateSwitch(Order, BBs[0]);
2789 Builder.SetInsertPoint(ContBB);
2790 PHINode *Result = Builder.CreatePHI(Int8Ty, 5,
"was_set");
2792 for (
unsigned i = 0; i < 5; ++i) {
2793 Builder.SetInsertPoint(BBs[i]);
2794 AtomicRMWInst *RMW = Builder.CreateAtomicRMW(llvm::AtomicRMWInst::Xchg,
2795 Ptr, NewVal, Orders[i]);
2796 RMW->setVolatile(Volatile);
2797 Result->addIncoming(RMW, BBs[i]);
2798 Builder.CreateBr(ContBB);
2801 SI->addCase(Builder.getInt32(0), BBs[0]);
2802 SI->addCase(Builder.getInt32(1), BBs[1]);
2803 SI->addCase(Builder.getInt32(2), BBs[1]);
2804 SI->addCase(Builder.getInt32(3), BBs[2]);
2805 SI->addCase(Builder.getInt32(4), BBs[3]);
2806 SI->addCase(Builder.getInt32(5), BBs[4]);
2808 Builder.SetInsertPoint(ContBB);
2809 return RValue::get(Builder.CreateIsNotNull(Result,
"tobool"));
2812 case Builtin::BI__atomic_clear: {
2818 unsigned AddrSpace = Ptr.
getPointer()->getType()->getPointerAddressSpace();
2819 Ptr = Builder.CreateBitCast(Ptr, Int8Ty->getPointerTo(AddrSpace));
2820 Value *NewVal = Builder.getInt8(0);
2822 if (isa<llvm::ConstantInt>(Order)) {
2823 int ord = cast<llvm::ConstantInt>(Order)->getZExtValue();
2824 StoreInst *
Store = Builder.CreateStore(NewVal, Ptr, Volatile);
2828 Store->setOrdering(llvm::AtomicOrdering::Monotonic);
2831 Store->setOrdering(llvm::AtomicOrdering::Release);
2834 Store->setOrdering(llvm::AtomicOrdering::SequentiallyConsistent);
2840 llvm::BasicBlock *ContBB = createBasicBlock(
"atomic.continue", CurFn);
2842 llvm::BasicBlock *BBs[3] = {
2843 createBasicBlock(
"monotonic", CurFn),
2844 createBasicBlock(
"release", CurFn),
2845 createBasicBlock(
"seqcst", CurFn)
2847 llvm::AtomicOrdering Orders[3] = {
2848 llvm::AtomicOrdering::Monotonic, llvm::AtomicOrdering::Release,
2849 llvm::AtomicOrdering::SequentiallyConsistent};
2851 Order = Builder.CreateIntCast(Order, Builder.getInt32Ty(),
false);
2852 llvm::SwitchInst *SI = Builder.CreateSwitch(Order, BBs[0]);
2854 for (
unsigned i = 0; i < 3; ++i) {
2855 Builder.SetInsertPoint(BBs[i]);
2856 StoreInst *
Store = Builder.CreateStore(NewVal, Ptr, Volatile);
2857 Store->setOrdering(Orders[i]);
2858 Builder.CreateBr(ContBB);
2861 SI->addCase(Builder.getInt32(0), BBs[0]);
2862 SI->addCase(Builder.getInt32(3), BBs[1]);
2863 SI->addCase(Builder.getInt32(5), BBs[2]);
2865 Builder.SetInsertPoint(ContBB);
2869 case Builtin::BI__atomic_thread_fence:
2870 case Builtin::BI__atomic_signal_fence:
2871 case Builtin::BI__c11_atomic_thread_fence:
2872 case Builtin::BI__c11_atomic_signal_fence: {
2874 if (BuiltinID == Builtin::BI__atomic_signal_fence ||
2875 BuiltinID == Builtin::BI__c11_atomic_signal_fence)
2876 SSID = llvm::SyncScope::SingleThread;
2880 if (isa<llvm::ConstantInt>(Order)) {
2881 int ord = cast<llvm::ConstantInt>(Order)->getZExtValue();
2888 Builder.CreateFence(llvm::AtomicOrdering::Acquire, SSID);
2891 Builder.CreateFence(llvm::AtomicOrdering::Release, SSID);
2894 Builder.CreateFence(llvm::AtomicOrdering::AcquireRelease, SSID);
2897 Builder.CreateFence(llvm::AtomicOrdering::SequentiallyConsistent, SSID);
2903 llvm::BasicBlock *AcquireBB, *ReleaseBB, *AcqRelBB, *SeqCstBB;
2904 AcquireBB = createBasicBlock(
"acquire", CurFn);
2905 ReleaseBB = createBasicBlock(
"release", CurFn);
2906 AcqRelBB = createBasicBlock(
"acqrel", CurFn);
2907 SeqCstBB = createBasicBlock(
"seqcst", CurFn);
2908 llvm::BasicBlock *ContBB = createBasicBlock(
"atomic.continue", CurFn);
2910 Order = Builder.CreateIntCast(Order, Builder.getInt32Ty(),
false);
2911 llvm::SwitchInst *SI = Builder.CreateSwitch(Order, ContBB);
2913 Builder.SetInsertPoint(AcquireBB);
2914 Builder.CreateFence(llvm::AtomicOrdering::Acquire, SSID);
2915 Builder.CreateBr(ContBB);
2916 SI->addCase(Builder.getInt32(1), AcquireBB);
2917 SI->addCase(Builder.getInt32(2), AcquireBB);
2919 Builder.SetInsertPoint(ReleaseBB);
2920 Builder.CreateFence(llvm::AtomicOrdering::Release, SSID);
2921 Builder.CreateBr(ContBB);
2922 SI->addCase(Builder.getInt32(3), ReleaseBB);
2924 Builder.SetInsertPoint(AcqRelBB);
2925 Builder.CreateFence(llvm::AtomicOrdering::AcquireRelease, SSID);
2926 Builder.CreateBr(ContBB);
2927 SI->addCase(Builder.getInt32(4), AcqRelBB);
2929 Builder.SetInsertPoint(SeqCstBB);
2930 Builder.CreateFence(llvm::AtomicOrdering::SequentiallyConsistent, SSID);
2931 Builder.CreateBr(ContBB);
2932 SI->addCase(Builder.getInt32(5), SeqCstBB);
2934 Builder.SetInsertPoint(ContBB);
2938 case Builtin::BI__builtin_signbit:
2939 case Builtin::BI__builtin_signbitf:
2940 case Builtin::BI__builtin_signbitl: {
2945 case Builtin::BI__annotation: {
2949 const auto *Str = cast<StringLiteral>(Arg->IgnoreParenCasts());
2950 assert(Str->getCharByteWidth() == 2);
2951 StringRef WideBytes = Str->getBytes();
2952 std::string StrUtf8;
2953 if (!convertUTF16ToUTF8String(
2954 makeArrayRef(WideBytes.data(), WideBytes.size()), StrUtf8)) {
2955 CGM.ErrorUnsupported(E,
"non-UTF16 __annotation argument");
2958 Strings.push_back(llvm::MDString::get(getLLVMContext(), StrUtf8));
2962 llvm::Value *F = CGM.getIntrinsic(llvm::Intrinsic::codeview_annotation, {});
2963 MDTuple *StrTuple = MDTuple::get(getLLVMContext(), Strings);
2964 Builder.CreateCall(F, MetadataAsValue::get(getLLVMContext(), StrTuple));
2967 case Builtin::BI__builtin_annotation: {
2969 llvm::Value *F = CGM.getIntrinsic(llvm::Intrinsic::annotation,
2975 StringRef Str = cast<StringLiteral>(AnnotationStrExpr)->getString();
2978 case Builtin::BI__builtin_addcb:
2979 case Builtin::BI__builtin_addcs:
2980 case Builtin::BI__builtin_addc:
2981 case Builtin::BI__builtin_addcl:
2982 case Builtin::BI__builtin_addcll:
2983 case Builtin::BI__builtin_subcb:
2984 case Builtin::BI__builtin_subcs:
2985 case Builtin::BI__builtin_subc:
2986 case Builtin::BI__builtin_subcl:
2987 case Builtin::BI__builtin_subcll: {
3010 Address CarryOutPtr = EmitPointerWithAlignment(E->
getArg(3));
3014 switch (BuiltinID) {
3015 default: llvm_unreachable(
"Unknown multiprecision builtin id.");
3016 case Builtin::BI__builtin_addcb:
3017 case Builtin::BI__builtin_addcs:
3018 case Builtin::BI__builtin_addc:
3019 case Builtin::BI__builtin_addcl:
3020 case Builtin::BI__builtin_addcll:
3021 IntrinsicId = llvm::Intrinsic::uadd_with_overflow;
3023 case Builtin::BI__builtin_subcb:
3024 case Builtin::BI__builtin_subcs:
3025 case Builtin::BI__builtin_subc:
3026 case Builtin::BI__builtin_subcl:
3027 case Builtin::BI__builtin_subcll:
3028 IntrinsicId = llvm::Intrinsic::usub_with_overflow;
3038 Sum1, Carryin, Carry2);
3039 llvm::Value *CarryOut = Builder.CreateZExt(Builder.CreateOr(Carry1, Carry2),
3041 Builder.CreateStore(CarryOut, CarryOutPtr);
3045 case Builtin::BI__builtin_add_overflow:
3046 case Builtin::BI__builtin_sub_overflow:
3047 case Builtin::BI__builtin_mul_overflow: {
3055 WidthAndSignedness LeftInfo =
3057 WidthAndSignedness RightInfo =
3059 WidthAndSignedness ResultInfo =
3066 RightInfo, ResultArg, ResultQTy,
3069 WidthAndSignedness EncompassingInfo =
3073 llvm::IntegerType::get(CGM.getLLVMContext(), EncompassingInfo.Width);
3075 llvm::Type *ResultLLVMTy = CGM.getTypes().ConvertType(ResultQTy);
3078 switch (BuiltinID) {
3080 llvm_unreachable(
"Unknown overflow builtin id.");
3081 case Builtin::BI__builtin_add_overflow:
3082 IntrinsicId = EncompassingInfo.Signed
3083 ? llvm::Intrinsic::sadd_with_overflow
3084 : llvm::Intrinsic::uadd_with_overflow;
3086 case Builtin::BI__builtin_sub_overflow:
3087 IntrinsicId = EncompassingInfo.Signed
3088 ? llvm::Intrinsic::ssub_with_overflow
3089 : llvm::Intrinsic::usub_with_overflow;
3091 case Builtin::BI__builtin_mul_overflow:
3092 IntrinsicId = EncompassingInfo.Signed
3093 ? llvm::Intrinsic::smul_with_overflow
3094 : llvm::Intrinsic::umul_with_overflow;
3100 Address ResultPtr = EmitPointerWithAlignment(ResultArg);
3103 Left = Builder.CreateIntCast(Left, EncompassingLLVMTy, LeftInfo.Signed);
3104 Right = Builder.CreateIntCast(Right, EncompassingLLVMTy, RightInfo.Signed);
3110 if (EncompassingInfo.Width > ResultInfo.Width) {
3113 llvm::Value *ResultTrunc = Builder.CreateTrunc(Result, ResultLLVMTy);
3117 llvm::Value *ResultTruncExt = Builder.CreateIntCast(
3118 ResultTrunc, EncompassingLLVMTy, ResultInfo.Signed);
3120 Builder.CreateICmpNE(Result, ResultTruncExt);
3122 Overflow = Builder.CreateOr(Overflow, TruncationOverflow);
3123 Result = ResultTrunc;
3129 Builder.CreateStore(EmitToMemory(Result, ResultQTy), ResultPtr, isVolatile);
3134 case Builtin::BI__builtin_uadd_overflow:
3135 case Builtin::BI__builtin_uaddl_overflow:
3136 case Builtin::BI__builtin_uaddll_overflow:
3137 case Builtin::BI__builtin_usub_overflow:
3138 case Builtin::BI__builtin_usubl_overflow:
3139 case Builtin::BI__builtin_usubll_overflow:
3140 case Builtin::BI__builtin_umul_overflow:
3141 case Builtin::BI__builtin_umull_overflow:
3142 case Builtin::BI__builtin_umulll_overflow:
3143 case Builtin::BI__builtin_sadd_overflow:
3144 case Builtin::BI__builtin_saddl_overflow:
3145 case Builtin::BI__builtin_saddll_overflow:
3146 case Builtin::BI__builtin_ssub_overflow:
3147 case Builtin::BI__builtin_ssubl_overflow:
3148 case Builtin::BI__builtin_ssubll_overflow:
3149 case Builtin::BI__builtin_smul_overflow:
3150 case Builtin::BI__builtin_smull_overflow:
3151 case Builtin::BI__builtin_smulll_overflow: {
3162 switch (BuiltinID) {
3163 default: llvm_unreachable(
"Unknown overflow builtin id.");
3164 case Builtin::BI__builtin_uadd_overflow:
3165 case Builtin::BI__builtin_uaddl_overflow:
3166 case Builtin::BI__builtin_uaddll_overflow:
3167 IntrinsicId = llvm::Intrinsic::uadd_with_overflow;
3169 case Builtin::BI__builtin_usub_overflow:
3170 case Builtin::BI__builtin_usubl_overflow:
3171 case Builtin::BI__builtin_usubll_overflow:
3172 IntrinsicId = llvm::Intrinsic::usub_with_overflow;
3174 case Builtin::BI__builtin_umul_overflow:
3175 case Builtin::BI__builtin_umull_overflow:
3176 case Builtin::BI__builtin_umulll_overflow:
3177 IntrinsicId = llvm::Intrinsic::umul_with_overflow;
3179 case Builtin::BI__builtin_sadd_overflow:
3180 case Builtin::BI__builtin_saddl_overflow:
3181 case Builtin::BI__builtin_saddll_overflow:
3182 IntrinsicId = llvm::Intrinsic::sadd_with_overflow;
3184 case Builtin::BI__builtin_ssub_overflow:
3185 case Builtin::BI__builtin_ssubl_overflow:
3186 case Builtin::BI__builtin_ssubll_overflow:
3187 IntrinsicId = llvm::Intrinsic::ssub_with_overflow;
3189 case Builtin::BI__builtin_smul_overflow:
3190 case Builtin::BI__builtin_smull_overflow:
3191 case Builtin::BI__builtin_smulll_overflow:
3192 IntrinsicId = llvm::Intrinsic::smul_with_overflow;
3199 Builder.CreateStore(Sum, SumOutPtr);
3203 case Builtin::BI__builtin_addressof:
3205 case Builtin::BI__builtin_operator_new:
3206 return EmitBuiltinNewDeleteCall(
3208 case Builtin::BI__builtin_operator_delete:
3209 return EmitBuiltinNewDeleteCall(
3212 case Builtin::BI__noop:
3215 case Builtin::BI__builtin_call_with_static_chain: {
3219 EmitCallee(Call->
getCallee()), Call, ReturnValue,
3220 EmitScalarExpr(Chain));
3222 case Builtin::BI_InterlockedExchange8:
3223 case Builtin::BI_InterlockedExchange16:
3224 case Builtin::BI_InterlockedExchange:
3225 case Builtin::BI_InterlockedExchangePointer:
3227 EmitMSVCBuiltinExpr(MSVCIntrin::_InterlockedExchange, E));
3228 case Builtin::BI_InterlockedCompareExchangePointer:
3229 case Builtin::BI_InterlockedCompareExchangePointer_nf: {
3231 llvm::IntegerType *IntType =
3232 IntegerType::get(getLLVMContext(),
3233 getContext().getTypeSize(E->
getType()));
3234 llvm::Type *IntPtrType = IntType->getPointerTo();
3237 Builder.CreateBitCast(EmitScalarExpr(E->
getArg(0)), IntPtrType);
3240 RTy = Exchange->getType();
3241 Exchange = Builder.CreatePtrToInt(Exchange, IntType);
3244 Builder.CreatePtrToInt(EmitScalarExpr(E->
getArg(2)), IntType);
3247 BuiltinID == Builtin::BI_InterlockedCompareExchangePointer_nf ?
3248 AtomicOrdering::Monotonic : AtomicOrdering::SequentiallyConsistent;
3250 auto Result = Builder.CreateAtomicCmpXchg(Destination, Comparand, Exchange,
3251 Ordering, Ordering);
3252 Result->setVolatile(
true);
3254 return RValue::get(Builder.CreateIntToPtr(Builder.CreateExtractValue(Result,
3258 case Builtin::BI_InterlockedCompareExchange8:
3259 case Builtin::BI_InterlockedCompareExchange16:
3260 case Builtin::BI_InterlockedCompareExchange:
3261 case Builtin::BI_InterlockedCompareExchange64:
3263 case Builtin::BI_InterlockedIncrement16:
3264 case Builtin::BI_InterlockedIncrement:
3266 EmitMSVCBuiltinExpr(MSVCIntrin::_InterlockedIncrement, E));
3267 case Builtin::BI_InterlockedDecrement16:
3268 case Builtin::BI_InterlockedDecrement:
3270 EmitMSVCBuiltinExpr(MSVCIntrin::_InterlockedDecrement, E));
3271 case Builtin::BI_InterlockedAnd8:
3272 case Builtin::BI_InterlockedAnd16:
3273 case Builtin::BI_InterlockedAnd:
3274 return RValue::get(EmitMSVCBuiltinExpr(MSVCIntrin::_InterlockedAnd, E));
3275 case Builtin::BI_InterlockedExchangeAdd8:
3276 case Builtin::BI_InterlockedExchangeAdd16:
3277 case Builtin::BI_InterlockedExchangeAdd:
3279 EmitMSVCBuiltinExpr(MSVCIntrin::_InterlockedExchangeAdd, E));
3280 case Builtin::BI_InterlockedExchangeSub8:
3281 case Builtin::BI_InterlockedExchangeSub16:
3282 case Builtin::BI_InterlockedExchangeSub:
3284 EmitMSVCBuiltinExpr(MSVCIntrin::_InterlockedExchangeSub, E));
3285 case Builtin::BI_InterlockedOr8:
3286 case Builtin::BI_InterlockedOr16:
3287 case Builtin::BI_InterlockedOr:
3288 return RValue::get(EmitMSVCBuiltinExpr(MSVCIntrin::_InterlockedOr, E));
3289 case Builtin::BI_InterlockedXor8:
3290 case Builtin::BI_InterlockedXor16:
3291 case Builtin::BI_InterlockedXor:
3292 return RValue::get(EmitMSVCBuiltinExpr(MSVCIntrin::_InterlockedXor, E));
3294 case Builtin::BI_bittest64:
3295 case Builtin::BI_bittest:
3296 case Builtin::BI_bittestandcomplement64:
3297 case Builtin::BI_bittestandcomplement:
3298 case Builtin::BI_bittestandreset64:
3299 case Builtin::BI_bittestandreset:
3300 case Builtin::BI_bittestandset64:
3301 case Builtin::BI_bittestandset:
3302 case Builtin::BI_interlockedbittestandreset:
3303 case Builtin::BI_interlockedbittestandreset64:
3304 case Builtin::BI_interlockedbittestandset64:
3305 case Builtin::BI_interlockedbittestandset:
3306 case Builtin::BI_interlockedbittestandset_acq:
3307 case Builtin::BI_interlockedbittestandset_rel:
3308 case Builtin::BI_interlockedbittestandset_nf:
3309 case Builtin::BI_interlockedbittestandreset_acq:
3310 case Builtin::BI_interlockedbittestandreset_rel:
3311 case Builtin::BI_interlockedbittestandreset_nf:
3314 case Builtin::BI__exception_code:
3315 case Builtin::BI_exception_code:
3317 case Builtin::BI__exception_info:
3318 case Builtin::BI_exception_info:
3320 case Builtin::BI__abnormal_termination:
3321 case Builtin::BI_abnormal_termination:
3323 case Builtin::BI_setjmpex:
3324 if (getTarget().getTriple().isOSMSVCRT())
3327 case Builtin::BI_setjmp:
3328 if (getTarget().getTriple().isOSMSVCRT()) {
3329 if (getTarget().getTriple().getArch() == llvm::Triple::x86)
3331 else if (getTarget().getTriple().getArch() == llvm::Triple::aarch64)
3337 case Builtin::BI__GetExceptionInfo: {
3338 if (llvm::GlobalVariable *GV =
3340 return RValue::get(llvm::ConstantExpr::getBitCast(GV, CGM.Int8PtrTy));
3344 case Builtin::BI__fastfail:
3345 return RValue::get(EmitMSVCBuiltinExpr(MSVCIntrin::__fastfail, E));
3347 case Builtin::BI__builtin_coro_size: {
3348 auto & Context = getContext();
3350 auto T = Builder.getIntNTy(Context.
getTypeSize(SizeTy));
3351 Value *F = CGM.getIntrinsic(Intrinsic::coro_size, T);
3355 case Builtin::BI__builtin_coro_id:
3356 return EmitCoroutineIntrinsic(E, Intrinsic::coro_id);
3357 case Builtin::BI__builtin_coro_promise:
3358 return EmitCoroutineIntrinsic(E, Intrinsic::coro_promise);
3359 case Builtin::BI__builtin_coro_resume:
3360 return EmitCoroutineIntrinsic(E, Intrinsic::coro_resume);
3361 case Builtin::BI__builtin_coro_frame:
3362 return EmitCoroutineIntrinsic(E, Intrinsic::coro_frame);
3363 case Builtin::BI__builtin_coro_noop:
3364 return EmitCoroutineIntrinsic(E, Intrinsic::coro_noop);
3365 case Builtin::BI__builtin_coro_free:
3366 return EmitCoroutineIntrinsic(E, Intrinsic::coro_free);
3367 case Builtin::BI__builtin_coro_destroy:
3368 return EmitCoroutineIntrinsic(E, Intrinsic::coro_destroy);
3369 case Builtin::BI__builtin_coro_done:
3370 return EmitCoroutineIntrinsic(E, Intrinsic::coro_done);
3371 case Builtin::BI__builtin_coro_alloc:
3372 return EmitCoroutineIntrinsic(E, Intrinsic::coro_alloc);
3373 case Builtin::BI__builtin_coro_begin:
3374 return EmitCoroutineIntrinsic(E, Intrinsic::coro_begin);
3375 case Builtin::BI__builtin_coro_end:
3376 return EmitCoroutineIntrinsic(E, Intrinsic::coro_end);
3377 case Builtin::BI__builtin_coro_suspend:
3378 return EmitCoroutineIntrinsic(E, Intrinsic::coro_suspend);
3379 case Builtin::BI__builtin_coro_param:
3380 return EmitCoroutineIntrinsic(E, Intrinsic::coro_param);
3383 case Builtin::BIread_pipe:
3384 case Builtin::BIwrite_pipe: {
3386 *Arg1 = EmitScalarExpr(E->
getArg(1));
3388 Value *PacketSize = OpenCLRT.getPipeElemSize(E->
getArg(0));
3389 Value *PacketAlign = OpenCLRT.getPipeElemAlign(E->
getArg(0));
3392 unsigned GenericAS =
3395 llvm::Type::getInt8Ty(getLLVMContext()), GenericAS);
3399 const char *Name = (BuiltinID == Builtin::BIread_pipe) ?
"__read_pipe_2" 3403 llvm::Type *ArgTys[] = {Arg0->getType(), I8PTy, Int32Ty, Int32Ty};
3404 llvm::FunctionType *FTy = llvm::FunctionType::get(
3406 Value *BCast = Builder.CreatePointerCast(Arg1, I8PTy);
3408 Builder.CreateCall(CGM.CreateRuntimeFunction(FTy, Name),
3409 {Arg0, BCast, PacketSize, PacketAlign}));
3412 "Illegal number of parameters to pipe function");
3413 const char *Name = (BuiltinID == Builtin::BIread_pipe) ?
"__read_pipe_4" 3416 llvm::Type *ArgTys[] = {Arg0->getType(), Arg1->getType(), Int32Ty, I8PTy,
3419 *Arg3 = EmitScalarExpr(E->
getArg(3));
3420 llvm::FunctionType *FTy = llvm::FunctionType::get(
3422 Value *BCast = Builder.CreatePointerCast(Arg3, I8PTy);
3425 if (Arg2->getType() != Int32Ty)
3426 Arg2 = Builder.CreateZExtOrTrunc(Arg2, Int32Ty);
3428 CGM.CreateRuntimeFunction(FTy, Name),
3429 {Arg0, Arg1, Arg2, BCast, PacketSize, PacketAlign}));
3434 case Builtin::BIreserve_read_pipe:
3435 case Builtin::BIreserve_write_pipe:
3436 case Builtin::BIwork_group_reserve_read_pipe:
3437 case Builtin::BIwork_group_reserve_write_pipe:
3438 case Builtin::BIsub_group_reserve_read_pipe:
3439 case Builtin::BIsub_group_reserve_write_pipe: {
3442 if (BuiltinID == Builtin::BIreserve_read_pipe)
3443 Name =
"__reserve_read_pipe";
3444 else if (BuiltinID == Builtin::BIreserve_write_pipe)
3445 Name =
"__reserve_write_pipe";
3446 else if (BuiltinID == Builtin::BIwork_group_reserve_read_pipe)
3447 Name =
"__work_group_reserve_read_pipe";
3448 else if (BuiltinID == Builtin::BIwork_group_reserve_write_pipe)
3449 Name =
"__work_group_reserve_write_pipe";
3450 else if (BuiltinID == Builtin::BIsub_group_reserve_read_pipe)
3451 Name =
"__sub_group_reserve_read_pipe";
3453 Name =
"__sub_group_reserve_write_pipe";
3456 *Arg1 = EmitScalarExpr(E->
getArg(1));
3457 llvm::Type *ReservedIDTy = ConvertType(getContext().OCLReserveIDTy);
3463 llvm::Type *ArgTys[] = {Arg0->getType(), Int32Ty, Int32Ty, Int32Ty};
3464 llvm::FunctionType *FTy = llvm::FunctionType::get(
3468 if (Arg1->getType() != Int32Ty)
3469 Arg1 = Builder.CreateZExtOrTrunc(Arg1, Int32Ty);
3471 Builder.CreateCall(CGM.CreateRuntimeFunction(FTy, Name),
3472 {Arg0, Arg1, PacketSize, PacketAlign}));
3476 case Builtin::BIcommit_read_pipe:
3477 case Builtin::BIcommit_write_pipe:
3478 case Builtin::BIwork_group_commit_read_pipe:
3479 case Builtin::BIwork_group_commit_write_pipe:
3480 case Builtin::BIsub_group_commit_read_pipe:
3481 case Builtin::BIsub_group_commit_write_pipe: {
3483 if (BuiltinID == Builtin::BIcommit_read_pipe)
3484 Name =
"__commit_read_pipe";
3485 else if (BuiltinID == Builtin::BIcommit_write_pipe)
3486 Name =
"__commit_write_pipe";
3487 else if (BuiltinID == Builtin::BIwork_group_commit_read_pipe)
3488 Name =
"__work_group_commit_read_pipe";
3489 else if (BuiltinID == Builtin::BIwork_group_commit_write_pipe)
3490 Name =
"__work_group_commit_write_pipe";
3491 else if (BuiltinID == Builtin::BIsub_group_commit_read_pipe)
3492 Name =
"__sub_group_commit_read_pipe";
3494 Name =
"__sub_group_commit_write_pipe";
3497 *Arg1 = EmitScalarExpr(E->
getArg(1));
3499 Value *PacketSize = OpenCLRT.getPipeElemSize(E->
getArg(0));
3500 Value *PacketAlign = OpenCLRT.getPipeElemAlign(E->
getArg(0));
3503 llvm::Type *ArgTys[] = {Arg0->getType(), Arg1->getType(), Int32Ty, Int32Ty};
3504 llvm::FunctionType *FTy =
3505 llvm::FunctionType::get(llvm::Type::getVoidTy(getLLVMContext()),
3509 Builder.CreateCall(CGM.CreateRuntimeFunction(FTy, Name),
3510 {Arg0, Arg1, PacketSize, PacketAlign}));
3513 case Builtin::BIget_pipe_num_packets:
3514 case Builtin::BIget_pipe_max_packets: {
3515 const char *BaseName;
3517 if (BuiltinID == Builtin::BIget_pipe_num_packets)
3518 BaseName =
"__get_pipe_num_packets";
3520 BaseName =
"__get_pipe_max_packets";
3521 auto Name = std::string(BaseName) +
3522 std::string(PipeTy->
isReadOnly() ?
"_ro" :
"_wo");
3527 Value *PacketSize = OpenCLRT.getPipeElemSize(E->
getArg(0));
3528 Value *PacketAlign = OpenCLRT.getPipeElemAlign(E->
getArg(0));
3529 llvm::Type *ArgTys[] = {Arg0->getType(), Int32Ty, Int32Ty};
3530 llvm::FunctionType *FTy = llvm::FunctionType::get(
3533 return RValue::get(Builder.CreateCall(CGM.CreateRuntimeFunction(FTy, Name),
3534 {Arg0, PacketSize, PacketAlign}));
3538 case Builtin::BIto_global:
3539 case Builtin::BIto_local:
3540 case Builtin::BIto_private: {
3541 auto Arg0 = EmitScalarExpr(E->
getArg(0));
3542 auto NewArgT = llvm::PointerType::get(Int8Ty,
3544 auto NewRetT = llvm::PointerType::get(Int8Ty,
3545 CGM.getContext().getTargetAddressSpace(
3547 auto FTy = llvm::FunctionType::get(NewRetT, {NewArgT},
false);
3549 if (Arg0->getType()->getPointerAddressSpace() !=
3550 NewArgT->getPointerAddressSpace())
3551 NewArg = Builder.CreateAddrSpaceCast(Arg0, NewArgT);
3553 NewArg = Builder.CreateBitOrPointerCast(Arg0, NewArgT);
3556 Builder.CreateCall(CGM.CreateRuntimeFunction(FTy, NewName), {NewArg});
3557 return RValue::get(Builder.CreateBitOrPointerCast(NewCall,
3563 case Builtin::BIenqueue_kernel: {
3567 llvm::Type *QueueTy = ConvertType(getContext().OCLQueueTy);
3568 llvm::Type *GenericVoidPtrTy = Builder.getInt8PtrTy(
3574 llvm::Value *Range = NDRangeL.getAddress().getPointer();
3575 llvm::Type *RangeTy = NDRangeL.getAddress().getType();
3580 Name =
"__enqueue_kernel_basic";
3581 llvm::Type *ArgTys[] = {QueueTy, Int32Ty, RangeTy, GenericVoidPtrTy,
3583 llvm::FunctionType *FTy = llvm::FunctionType::get(
3587 CGM.getOpenCLRuntime().emitOpenCLEnqueuedBlock(*
this, E->
getArg(3));
3589 Builder.CreatePointerCast(Info.Kernel, GenericVoidPtrTy);
3591 Builder.CreatePointerCast(Info.BlockArg, GenericVoidPtrTy);
3594 B.addAttribute(Attribute::ByVal);
3595 llvm::AttributeList ByValAttrSet =
3596 llvm::AttributeList::get(CGM.getModule().getContext(), 3U, B);
3599 Builder.CreateCall(CGM.CreateRuntimeFunction(FTy, Name, ByValAttrSet),
3600 {Queue, Flags, Range, Kernel, Block});
3601 RTCall->setAttributes(ByValAttrSet);
3604 assert(NumArgs >= 5 &&
"Invalid enqueue_kernel signature");
3608 auto CreateArrayForSizeVar = [=](
unsigned First)
3609 -> std::tuple<llvm::Value *, llvm::Value *, llvm::Value *> {
3610 llvm::APInt ArraySize(32, NumArgs -
First);
3611 QualType SizeArrayTy = getContext().getConstantArrayType(
3614 auto Tmp = CreateMemTemp(SizeArrayTy,
"block_sizes");
3617 CGM.getDataLayout().getTypeAllocSize(Tmp.getElementType()), TmpPtr);
3621 auto *Zero = llvm::ConstantInt::get(IntTy, 0);
3622 for (
unsigned I =
First; I < NumArgs; ++I) {
3623 auto *Index = llvm::ConstantInt::get(IntTy, I -
First);
3624 auto *GEP = Builder.CreateGEP(TmpPtr, {Zero, Index});
3628 Builder.CreateZExtOrTrunc(EmitScalarExpr(E->
getArg(I)), SizeTy);
3629 Builder.CreateAlignedStore(
3630 V, GEP, CGM.getDataLayout().getPrefTypeAlignment(SizeTy));
3632 return std::tie(ElemPtr, TmpSize, TmpPtr);
3638 Name =
"__enqueue_kernel_varargs";
3640 CGM.getOpenCLRuntime().emitOpenCLEnqueuedBlock(*
this, E->
getArg(3));
3642 Builder.CreatePointerCast(Info.Kernel, GenericVoidPtrTy);
3643 auto *Block = Builder.CreatePointerCast(Info.BlockArg, GenericVoidPtrTy);
3645 std::tie(ElemPtr, TmpSize, TmpPtr) = CreateArrayForSizeVar(4);
3649 std::vector<llvm::Value *> Args = {
3650 Queue, Flags, Range,
3651 Kernel, Block, ConstantInt::get(IntTy, NumArgs - 4),
3653 std::vector<llvm::Type *> ArgTys = {
3654 QueueTy, IntTy, RangeTy, GenericVoidPtrTy,
3655 GenericVoidPtrTy, IntTy, ElemPtr->getType()};
3657 llvm::FunctionType *FTy = llvm::FunctionType::get(
3660 RValue::get(Builder.CreateCall(CGM.CreateRuntimeFunction(FTy, Name),
3663 EmitLifetimeEnd(TmpSize, TmpPtr);
3668 llvm::Type *EventTy = ConvertType(getContext().OCLClkEventTy);
3669 llvm::Type *EventPtrTy = EventTy->getPointerTo(
3673 Builder.CreateZExtOrTrunc(EmitScalarExpr(E->
getArg(3)), Int32Ty);
3676 ? EmitArrayToPointerDecay(E->
getArg(4)).getPointer()
3677 : EmitScalarExpr(E->
getArg(4));
3680 EventList = Builder.CreatePointerCast(EventList, EventPtrTy);
3681 ClkEvent = ClkEvent->getType()->isIntegerTy()
3682 ? Builder.CreateBitOrPointerCast(ClkEvent, EventPtrTy)
3683 : Builder.CreatePointerCast(ClkEvent, EventPtrTy);
3685 CGM.getOpenCLRuntime().emitOpenCLEnqueuedBlock(*
this, E->
getArg(6));
3687 Builder.CreatePointerCast(Info.Kernel, GenericVoidPtrTy);
3689 Builder.CreatePointerCast(Info.BlockArg, GenericVoidPtrTy);
3691 std::vector<llvm::Type *> ArgTys = {
3692 QueueTy, Int32Ty, RangeTy, Int32Ty,
3693 EventPtrTy, EventPtrTy, GenericVoidPtrTy, GenericVoidPtrTy};
3695 std::vector<llvm::Value *> Args = {Queue, Flags, Range, NumEvents,
3696 EventList, ClkEvent, Kernel, Block};
3700 Name =
"__enqueue_kernel_basic_events";
3701 llvm::FunctionType *FTy = llvm::FunctionType::get(
3704 Builder.CreateCall(CGM.CreateRuntimeFunction(FTy, Name),
3709 Args.push_back(ConstantInt::get(Int32Ty, NumArgs - 7));
3710 ArgTys.push_back(Int32Ty);
3711 Name =
"__enqueue_kernel_events_varargs";
3714 std::tie(ElemPtr, TmpSize, TmpPtr) = CreateArrayForSizeVar(7);
3715 Args.push_back(ElemPtr);
3716 ArgTys.push_back(ElemPtr->getType());
3718 llvm::FunctionType *FTy = llvm::FunctionType::get(
3721 RValue::get(Builder.CreateCall(CGM.CreateRuntimeFunction(FTy, Name),
3724 EmitLifetimeEnd(TmpSize, TmpPtr);
3731 case Builtin::BIget_kernel_work_group_size: {
3732 llvm::Type *GenericVoidPtrTy = Builder.getInt8PtrTy(
3735 CGM.getOpenCLRuntime().emitOpenCLEnqueuedBlock(*
this, E->
getArg(0));
3736 Value *Kernel = Builder.CreatePointerCast(Info.Kernel, GenericVoidPtrTy);
3737 Value *Arg = Builder.CreatePointerCast(Info.BlockArg, GenericVoidPtrTy);
3739 CGM.CreateRuntimeFunction(
3740 llvm::FunctionType::get(IntTy, {GenericVoidPtrTy, GenericVoidPtrTy},
3742 "__get_kernel_work_group_size_impl"),
3745 case Builtin::BIget_kernel_preferred_work_group_size_multiple: {
3746 llvm::Type *GenericVoidPtrTy = Builder.getInt8PtrTy(
3749 CGM.getOpenCLRuntime().emitOpenCLEnqueuedBlock(*
this, E->
getArg(0));
3750 Value *Kernel = Builder.CreatePointerCast(Info.Kernel, GenericVoidPtrTy);
3751 Value *Arg = Builder.CreatePointerCast(Info.BlockArg, GenericVoidPtrTy);
3753 CGM.CreateRuntimeFunction(
3754 llvm::FunctionType::get(IntTy, {GenericVoidPtrTy, GenericVoidPtrTy},
3756 "__get_kernel_preferred_work_group_size_multiple_impl"),
3759 case Builtin::BIget_kernel_max_sub_group_size_for_ndrange:
3760 case Builtin::BIget_kernel_sub_group_count_for_ndrange: {
3761 llvm::Type *GenericVoidPtrTy = Builder.getInt8PtrTy(
3766 CGM.getOpenCLRuntime().emitOpenCLEnqueuedBlock(*
this, E->
getArg(1));
3767 Value *Kernel = Builder.CreatePointerCast(Info.Kernel, GenericVoidPtrTy);
3768 Value *Block = Builder.CreatePointerCast(Info.BlockArg, GenericVoidPtrTy);
3770 BuiltinID == Builtin::BIget_kernel_max_sub_group_size_for_ndrange
3771 ?
"__get_kernel_max_sub_group_size_for_ndrange_impl" 3772 :
"__get_kernel_sub_group_count_for_ndrange_impl";
3774 CGM.CreateRuntimeFunction(
3775 llvm::FunctionType::get(
3776 IntTy, {NDRange->getType(), GenericVoidPtrTy, GenericVoidPtrTy},
3779 {NDRange, Kernel, Block}));
3782 case Builtin::BI__builtin_store_half:
3783 case Builtin::BI__builtin_store_halff: {
3786 Value *HalfVal = Builder.CreateFPTrunc(Val, Builder.getHalfTy());
3787 return RValue::get(Builder.CreateStore(HalfVal, Address));
3789 case Builtin::BI__builtin_load_half: {
3791 Value *HalfVal = Builder.CreateLoad(Address);
3792 return RValue::get(Builder.CreateFPExt(HalfVal, Builder.getDoubleTy()));
3794 case Builtin::BI__builtin_load_halff: {
3796 Value *HalfVal = Builder.CreateLoad(Address);
3797 return RValue::get(Builder.CreateFPExt(HalfVal, Builder.getFloatTy()));
3799 case Builtin::BIprintf:
3800 if (getTarget().getTriple().isNVPTX())
3801 return EmitNVPTXDevicePrintfCallExpr(E, ReturnValue);
3803 case Builtin::BI__builtin_canonicalize:
3804 case Builtin::BI__builtin_canonicalizef:
3805 case Builtin::BI__builtin_canonicalizel:
3808 case Builtin::BI__builtin_thread_pointer: {
3809 if (!getContext().getTargetInfo().isTLSSupported())
3810 CGM.ErrorUnsupported(E,
"__builtin_thread_pointer");
3814 case Builtin::BI__builtin_os_log_format:
3815 return emitBuiltinOSLogFormat(*E);
3817 case Builtin::BI__xray_customevent: {
3818 if (!ShouldXRayInstrumentFunction())
3821 if (!CGM.getCodeGenOpts().XRayInstrumentationBundle.has(
3825 if (
const auto *XRayAttr = CurFuncDecl->getAttr<XRayInstrumentAttr>())
3826 if (XRayAttr->neverXRayInstrument() && !AlwaysEmitXRayCustomEvents())
3829 Function *F = CGM.getIntrinsic(Intrinsic::xray_customevent);
3830 auto FTy = F->getFunctionType();
3831 auto Arg0 = E->
getArg(0);
3832 auto Arg0Val = EmitScalarExpr(Arg0);
3833 auto Arg0Ty = Arg0->getType();
3834 auto PTy0 = FTy->getParamType(0);
3835 if (PTy0 != Arg0Val->getType()) {
3836 if (Arg0Ty->isArrayType())
3837 Arg0Val = EmitArrayToPointerDecay(Arg0).getPointer();
3839 Arg0Val = Builder.CreatePointerCast(Arg0Val, PTy0);
3841 auto Arg1 = EmitScalarExpr(E->
getArg(1));
3842 auto PTy1 = FTy->getParamType(1);
3843 if (PTy1 != Arg1->getType())
3844 Arg1 = Builder.CreateTruncOrBitCast(Arg1, PTy1);
3845 return RValue::get(Builder.CreateCall(F, {Arg0Val, Arg1}));
3848 case Builtin::BI__xray_typedevent: {
3852 if (!ShouldXRayInstrumentFunction())
3855 if (!CGM.getCodeGenOpts().XRayInstrumentationBundle.has(
3859 if (
const auto *XRayAttr = CurFuncDecl->getAttr<XRayInstrumentAttr>())
3860 if (XRayAttr->neverXRayInstrument() && !AlwaysEmitXRayTypedEvents())
3863 Function *F = CGM.getIntrinsic(Intrinsic::xray_typedevent);
3864 auto FTy = F->getFunctionType();
3865 auto Arg0 = EmitScalarExpr(E->
getArg(0));
3866 auto PTy0 = FTy->getParamType(0);
3867 if (PTy0 != Arg0->getType())
3868 Arg0 = Builder.CreateTruncOrBitCast(Arg0, PTy0);
3869 auto Arg1 = E->
getArg(1);
3870 auto Arg1Val = EmitScalarExpr(Arg1);
3871 auto Arg1Ty = Arg1->getType();
3872 auto PTy1 = FTy->getParamType(1);
3873 if (PTy1 != Arg1Val->getType()) {
3874 if (Arg1Ty->isArrayType())
3875 Arg1Val = EmitArrayToPointerDecay(Arg1).getPointer();
3877 Arg1Val = Builder.CreatePointerCast(Arg1Val, PTy1);
3879 auto Arg2 = EmitScalarExpr(E->
getArg(2));
3880 auto PTy2 = FTy->getParamType(2);
3881 if (PTy2 != Arg2->getType())
3882 Arg2 = Builder.CreateTruncOrBitCast(Arg2, PTy2);
3883 return RValue::get(Builder.CreateCall(F, {Arg0, Arg1Val, Arg2}));
3886 case Builtin::BI__builtin_ms_va_start:
3887 case Builtin::BI__builtin_ms_va_end:
3889 EmitVAStartEnd(EmitMSVAListRef(E->
getArg(0)).getPointer(),
3890 BuiltinID == Builtin::BI__builtin_ms_va_start));
3892 case Builtin::BI__builtin_ms_va_copy: {
3909 Value *ArgPtr = Builder.CreateLoad(SrcAddr,
"ap.val");
3910 return RValue::get(Builder.CreateStore(ArgPtr, DestAddr));
3917 if (getContext().BuiltinInfo.isLibFunction(BuiltinID))
3919 CGM.getBuiltinLibFunction(FD, BuiltinID));
3923 if (getContext().BuiltinInfo.isPredefinedLibFunction(BuiltinID))
3925 cast<llvm::Constant>(EmitScalarExpr(E->
getCallee())));
3932 checkTargetFeatures(E, FD);
3934 if (
unsigned VectorWidth = getContext().
BuiltinInfo.getRequiredVectorWidth(BuiltinID))
3935 LargestVectorWidth =
std::max(LargestVectorWidth, VectorWidth);
3938 const char *Name = getContext().BuiltinInfo.getName(BuiltinID);
3941 llvm::Triple::getArchTypePrefix(getTarget().getTriple().getArch());
3942 if (!Prefix.empty()) {
3943 IntrinsicID = Intrinsic::getIntrinsicForGCCBuiltin(Prefix.data(), Name);
3947 if (IntrinsicID == Intrinsic::not_intrinsic)
3948 IntrinsicID = Intrinsic::getIntrinsicForMSBuiltin(Prefix.data(), Name);
3951 if (IntrinsicID != Intrinsic::not_intrinsic) {
3956 unsigned ICEArguments = 0;
3958 getContext().GetBuiltinType(BuiltinID, Error, &ICEArguments);
3961 Function *F = CGM.getIntrinsic(IntrinsicID);
3962 llvm::FunctionType *FTy = F->getFunctionType();
3964 for (
unsigned i = 0, e = E->
getNumArgs(); i != e; ++i) {
3967 if ((ICEArguments & (1 << i)) == 0) {
3968 ArgValue = EmitScalarExpr(E->
getArg(i));
3972 llvm::APSInt Result;
3974 assert(IsConst &&
"Constant arg isn't actually constant?");
3976 ArgValue = llvm::ConstantInt::get(getLLVMContext(), Result);
3982 if (PTy != ArgValue->getType()) {
3984 if (
auto *PtrTy = dyn_cast<llvm::PointerType>(PTy)) {
3985 if (PtrTy->getAddressSpace() !=
3986 ArgValue->getType()->getPointerAddressSpace()) {
3987 ArgValue = Builder.CreateAddrSpaceCast(
3989 ArgValue->getType()->getPointerTo(PtrTy->getAddressSpace()));
3993 assert(PTy->canLosslesslyBitCastTo(FTy->getParamType(i)) &&
3994 "Must be able to losslessly bit cast to param");
3995 ArgValue = Builder.CreateBitCast(ArgValue, PTy);
3998 Args.push_back(ArgValue);
4001 Value *V = Builder.CreateCall(F, Args);
4006 RetTy = ConvertType(BuiltinRetType);
4008 if (RetTy != V->getType()) {
4010 if (
auto *PtrTy = dyn_cast<llvm::PointerType>(RetTy)) {
4011 if (PtrTy->getAddressSpace() != V->getType()->getPointerAddressSpace()) {
4012 V = Builder.CreateAddrSpaceCast(
4013 V, V->getType()->getPointerTo(PtrTy->getAddressSpace()));
4017 assert(V->getType()->canLosslesslyBitCastTo(RetTy) &&
4018 "Must be able to losslessly bit cast result type");
4019 V = Builder.CreateBitCast(V, RetTy);
4026 if (
Value *V = EmitTargetBuiltinExpr(BuiltinID, E))
4029 ErrorUnsupported(E,
"builtin function");
4032 return GetUndefRValue(E->
getType());
4036 unsigned BuiltinID,
const CallExpr *E,
4037 llvm::Triple::ArchType Arch) {
4039 case llvm::Triple::arm:
4040 case llvm::Triple::armeb:
4041 case llvm::Triple::thumb:
4042 case llvm::Triple::thumbeb:
4044 case llvm::Triple::aarch64:
4045 case llvm::Triple::aarch64_be:
4047 case llvm::Triple::x86:
4048 case llvm::Triple::x86_64:
4050 case llvm::Triple::ppc:
4051 case llvm::Triple::ppc64:
4052 case llvm::Triple::ppc64le:
4054 case llvm::Triple::r600:
4055 case llvm::Triple::amdgcn:
4057 case llvm::Triple::systemz:
4059 case llvm::Triple::nvptx:
4060 case llvm::Triple::nvptx64:
4062 case llvm::Triple::wasm32:
4063 case llvm::Triple::wasm64:
4065 case llvm::Triple::hexagon:
4074 if (getContext().
BuiltinInfo.isAuxBuiltinID(BuiltinID)) {
4075 assert(getContext().getAuxTargetInfo() &&
"Missing aux target info");
4077 this, getContext().
BuiltinInfo.getAuxBuiltinID(BuiltinID), E,
4078 getContext().getAuxTargetInfo()->getTriple().getArch());
4082 getTarget().getTriple().getArch());
4087 bool HasLegalHalfType=
true,
4089 int IsQuad = TypeFlags.
isQuad();
4093 return llvm::VectorType::get(CGF->
Int8Ty, V1Ty ? 1 : (8 << IsQuad));
4096 return llvm::VectorType::get(CGF->
Int16Ty, V1Ty ? 1 : (4 << IsQuad));
4098 if (HasLegalHalfType)
4099 return llvm::VectorType::get(CGF->
HalfTy, V1Ty ? 1 : (4 << IsQuad));
4101 return llvm::VectorType::get(CGF->
Int16Ty, V1Ty ? 1 : (4 << IsQuad));
4103 return llvm::VectorType::get(CGF->
Int32Ty, V1Ty ? 1 : (2 << IsQuad));
4106 return llvm::VectorType::get(CGF->
Int64Ty, V1Ty ? 1 : (1 << IsQuad));
4111 return llvm::VectorType::get(CGF->
Int8Ty, 16);
4113 return llvm::VectorType::get(CGF->
FloatTy, V1Ty ? 1 : (2 << IsQuad));
4115 return llvm::VectorType::get(CGF->
DoubleTy, V1Ty ? 1 : (1 << IsQuad));
4117 llvm_unreachable(
"Unknown vector element type!");
4122 int IsQuad = IntTypeFlags.
isQuad();
4125 return llvm::VectorType::get(CGF->
HalfTy, (4 << IsQuad));
4127 return llvm::VectorType::get(CGF->
FloatTy, (2 << IsQuad));
4129 return llvm::VectorType::get(CGF->
DoubleTy, (1 << IsQuad));
4131 llvm_unreachable(
"Type can't be converted to floating-point!");
4136 unsigned nElts = V->getType()->getVectorNumElements();
4137 Value* SV = llvm::ConstantVector::getSplat(nElts, C);
4138 return Builder.CreateShuffleVector(V, V, SV,
"lane");
4143 unsigned shift,
bool rightshift) {
4145 for (Function::const_arg_iterator ai = F->arg_begin(), ae = F->arg_end();
4146 ai != ae; ++ai, ++j)
4147 if (shift > 0 && shift == j)
4148 Ops[j] = EmitNeonShiftVector(Ops[j], ai->getType(), rightshift);
4150 Ops[j] = Builder.CreateBitCast(Ops[j], ai->getType(), name);
4152 return Builder.CreateCall(F, Ops, name);
4157 int SV = cast<ConstantInt>(V)->getSExtValue();
4158 return ConstantInt::get(Ty, neg ? -SV : SV);
4165 llvm::VectorType *VTy = cast<llvm::VectorType>(Ty);
4167 int ShiftAmt = cast<ConstantInt>(
Shift)->getSExtValue();
4168 int EltSize = VTy->getScalarSizeInBits();
4170 Vec = Builder.CreateBitCast(Vec, Ty);
4174 if (ShiftAmt == EltSize) {
4177 return llvm::ConstantAggregateZero::get(VTy);
4182 Shift = ConstantInt::get(VTy->getElementType(), ShiftAmt);
4186 Shift = EmitNeonShiftVector(Shift, Ty,
false);
4188 return Builder.CreateLShr(Vec, Shift, name);
4190 return Builder.CreateAShr(Vec, Shift, name);
4216 struct NeonIntrinsicInfo {
4217 const char *NameHint;
4219 unsigned LLVMIntrinsic;
4220 unsigned AltLLVMIntrinsic;
4221 unsigned TypeModifier;
4223 bool operator<(
unsigned RHSBuiltinID)
const {
4224 return BuiltinID < RHSBuiltinID;
4226 bool operator<(
const NeonIntrinsicInfo &TE)
const {
4227 return BuiltinID < TE.BuiltinID;
4232 #define NEONMAP0(NameBase) \ 4233 { #NameBase, NEON::BI__builtin_neon_ ## NameBase, 0, 0, 0 } 4235 #define NEONMAP1(NameBase, LLVMIntrinsic, TypeModifier) \ 4236 { #NameBase, NEON:: BI__builtin_neon_ ## NameBase, \ 4237 Intrinsic::LLVMIntrinsic, 0, TypeModifier } 4239 #define NEONMAP2(NameBase, LLVMIntrinsic, AltLLVMIntrinsic, TypeModifier) \ 4240 { #NameBase, NEON:: BI__builtin_neon_ ## NameBase, \ 4241 Intrinsic::LLVMIntrinsic, Intrinsic::AltLLVMIntrinsic, \ 4247 NEONMAP1(vabs_v, arm_neon_vabs, 0),
4248 NEONMAP1(vabsq_v, arm_neon_vabs, 0),
4250 NEONMAP1(vaesdq_v, arm_neon_aesd, 0),
4251 NEONMAP1(vaeseq_v, arm_neon_aese, 0),
4252 NEONMAP1(vaesimcq_v, arm_neon_aesimc, 0),
4253 NEONMAP1(vaesmcq_v, arm_neon_aesmc, 0),
4256 NEONMAP1(vcage_v, arm_neon_vacge, 0),
4257 NEONMAP1(vcageq_v, arm_neon_vacge, 0),
4258 NEONMAP1(vcagt_v, arm_neon_vacgt, 0),
4259 NEONMAP1(vcagtq_v, arm_neon_vacgt, 0),
4260 NEONMAP1(vcale_v, arm_neon_vacge, 0),
4261 NEONMAP1(vcaleq_v, arm_neon_vacge, 0),
4262 NEONMAP1(vcalt_v, arm_neon_vacgt, 0),
4263 NEONMAP1(vcaltq_v, arm_neon_vacgt, 0),
4280 NEONMAP1(vcvt_f16_f32, arm_neon_vcvtfp2hf, 0),
4282 NEONMAP1(vcvt_f32_f16, arm_neon_vcvthf2fp, 0),
4284 NEONMAP2(vcvt_n_f16_v, arm_neon_vcvtfxu2fp, arm_neon_vcvtfxs2fp, 0),
4285 NEONMAP2(vcvt_n_f32_v, arm_neon_vcvtfxu2fp, arm_neon_vcvtfxs2fp, 0),
4286 NEONMAP1(vcvt_n_s16_v, arm_neon_vcvtfp2fxs, 0),
4287 NEONMAP1(vcvt_n_s32_v, arm_neon_vcvtfp2fxs, 0),
4288 NEONMAP1(vcvt_n_s64_v, arm_neon_vcvtfp2fxs, 0),
4289 NEONMAP1(vcvt_n_u16_v, arm_neon_vcvtfp2fxu, 0),
4290 NEONMAP1(vcvt_n_u32_v, arm_neon_vcvtfp2fxu, 0),
4291 NEONMAP1(vcvt_n_u64_v, arm_neon_vcvtfp2fxu, 0),
4298 NEONMAP1(vcvta_s16_v, arm_neon_vcvtas, 0),
4299 NEONMAP1(vcvta_s32_v, arm_neon_vcvtas, 0),
4300 NEONMAP1(vcvta_s64_v, arm_neon_vcvtas, 0),
4301 NEONMAP1(vcvta_u16_v, arm_neon_vcvtau, 0),
4302 NEONMAP1(vcvta_u32_v, arm_neon_vcvtau, 0),
4303 NEONMAP1(vcvta_u64_v, arm_neon_vcvtau, 0),
4304 NEONMAP1(vcvtaq_s16_v, arm_neon_vcvtas, 0),
4305 NEONMAP1(vcvtaq_s32_v, arm_neon_vcvtas, 0),
4306 NEONMAP1(vcvtaq_s64_v, arm_neon_vcvtas, 0),
4307 NEONMAP1(vcvtaq_u16_v, arm_neon_vcvtau, 0),
4308 NEONMAP1(vcvtaq_u32_v, arm_neon_vcvtau, 0),
4309 NEONMAP1(vcvtaq_u64_v, arm_neon_vcvtau, 0),
4310 NEONMAP1(vcvtm_s16_v, arm_neon_vcvtms, 0),
4311 NEONMAP1(vcvtm_s32_v, arm_neon_vcvtms, 0),
4312 NEONMAP1(vcvtm_s64_v, arm_neon_vcvtms, 0),
4313 NEONMAP1(vcvtm_u16_v, arm_neon_vcvtmu, 0),
4314 NEONMAP1(vcvtm_u32_v, arm_neon_vcvtmu, 0),
4315 NEONMAP1(vcvtm_u64_v, arm_neon_vcvtmu, 0),
4316 NEONMAP1(vcvtmq_s16_v, arm_neon_vcvtms, 0),
4317 NEONMAP1(vcvtmq_s32_v, arm_neon_vcvtms, 0),
4318 NEONMAP1(vcvtmq_s64_v, arm_neon_vcvtms, 0),
4319 NEONMAP1(vcvtmq_u16_v, arm_neon_vcvtmu, 0),
4320 NEONMAP1(vcvtmq_u32_v, arm_neon_vcvtmu, 0),
4321 NEONMAP1(vcvtmq_u64_v, arm_neon_vcvtmu, 0),
4322 NEONMAP1(vcvtn_s16_v, arm_neon_vcvtns, 0),
4323 NEONMAP1(vcvtn_s32_v, arm_neon_vcvtns, 0),
4324 NEONMAP1(vcvtn_s64_v, arm_neon_vcvtns, 0),
4325 NEONMAP1(vcvtn_u16_v, arm_neon_vcvtnu, 0),
4326 NEONMAP1(vcvtn_u32_v, arm_neon_vcvtnu, 0),
4327 NEONMAP1(vcvtn_u64_v, arm_neon_vcvtnu, 0),
4328 NEONMAP1(vcvtnq_s16_v, arm_neon_vcvtns, 0),
4329 NEONMAP1(vcvtnq_s32_v, arm_neon_vcvtns, 0),
4330 NEONMAP1(vcvtnq_s64_v, arm_neon_vcvtns, 0),
4331 NEONMAP1(vcvtnq_u16_v, arm_neon_vcvtnu, 0),
4332 NEONMAP1(vcvtnq_u32_v, arm_neon_vcvtnu, 0),
4333 NEONMAP1(vcvtnq_u64_v, arm_neon_vcvtnu, 0),
4334 NEONMAP1(vcvtp_s16_v, arm_neon_vcvtps, 0),
4335 NEONMAP1(vcvtp_s32_v, arm_neon_vcvtps, 0),
4336 NEONMAP1(vcvtp_s64_v, arm_neon_vcvtps, 0),
4337 NEONMAP1(vcvtp_u16_v, arm_neon_vcvtpu, 0),
4338 NEONMAP1(vcvtp_u32_v, arm_neon_vcvtpu, 0),
4339 NEONMAP1(vcvtp_u64_v, arm_neon_vcvtpu, 0),
4340 NEONMAP1(vcvtpq_s16_v, arm_neon_vcvtps, 0),
4341 NEONMAP1(vcvtpq_s32_v, arm_neon_vcvtps, 0),
4342 NEONMAP1(vcvtpq_s64_v, arm_neon_vcvtps, 0),
4343 NEONMAP1(vcvtpq_u16_v, arm_neon_vcvtpu, 0),
4344 NEONMAP1(vcvtpq_u32_v, arm_neon_vcvtpu, 0),
4345 NEONMAP1(vcvtpq_u64_v, arm_neon_vcvtpu, 0),
4348 NEONMAP2(vcvtq_n_f16_v, arm_neon_vcvtfxu2fp, arm_neon_vcvtfxs2fp, 0),
4349 NEONMAP2(vcvtq_n_f32_v, arm_neon_vcvtfxu2fp, arm_neon_vcvtfxs2fp, 0),
4350 NEONMAP1(vcvtq_n_s16_v, arm_neon_vcvtfp2fxs, 0),
4351 NEONMAP1(vcvtq_n_s32_v, arm_neon_vcvtfp2fxs, 0),
4352 NEONMAP1(vcvtq_n_s64_v, arm_neon_vcvtfp2fxs, 0),
4353 NEONMAP1(vcvtq_n_u16_v, arm_neon_vcvtfp2fxu, 0),
4354 NEONMAP1(vcvtq_n_u32_v, arm_neon_vcvtfp2fxu, 0),
4355 NEONMAP1(vcvtq_n_u64_v, arm_neon_vcvtfp2fxu, 0),
4362 NEONMAP2(vdot_v, arm_neon_udot, arm_neon_sdot, 0),
4363 NEONMAP2(vdotq_v, arm_neon_udot, arm_neon_sdot, 0),
4373 NEONMAP1(vld1_v, arm_neon_vld1, 0),
4374 NEONMAP1(vld1_x2_v, arm_neon_vld1x2, 0),
4375 NEONMAP1(vld1_x3_v, arm_neon_vld1x3, 0),
4376 NEONMAP1(vld1_x4_v, arm_neon_vld1x4, 0),
4378 NEONMAP1(vld1q_v, arm_neon_vld1, 0),
4379 NEONMAP1(vld1q_x2_v, arm_neon_vld1x2, 0),
4380 NEONMAP1(vld1q_x3_v, arm_neon_vld1x3, 0),
4381 NEONMAP1(vld1q_x4_v, arm_neon_vld1x4, 0),
4382 NEONMAP1(vld2_dup_v, arm_neon_vld2dup, 0),
4383 NEONMAP1(vld2_lane_v, arm_neon_vld2lane, 0),
4384 NEONMAP1(vld2_v, arm_neon_vld2, 0),
4385 NEONMAP1(vld2q_dup_v, arm_neon_vld2dup, 0),
4386 NEONMAP1(vld2q_lane_v, arm_neon_vld2lane, 0),
4387 NEONMAP1(vld2q_v, arm_neon_vld2, 0),
4388 NEONMAP1(vld3_dup_v, arm_neon_vld3dup, 0),
4389 NEONMAP1(vld3_lane_v, arm_neon_vld3lane, 0),
4390 NEONMAP1(vld3_v, arm_neon_vld3, 0),
4391 NEONMAP1(vld3q_dup_v, arm_neon_vld3dup, 0),
4392 NEONMAP1(vld3q_lane_v, arm_neon_vld3lane, 0),
4393 NEONMAP1(vld3q_v, arm_neon_vld3, 0),
4394 NEONMAP1(vld4_dup_v, arm_neon_vld4dup, 0),
4395 NEONMAP1(vld4_lane_v, arm_neon_vld4lane, 0),
4396 NEONMAP1(vld4_v, arm_neon_vld4, 0),
4397 NEONMAP1(vld4q_dup_v, arm_neon_vld4dup, 0),
4398 NEONMAP1(vld4q_lane_v, arm_neon_vld4lane, 0),
4399 NEONMAP1(vld4q_v, arm_neon_vld4, 0),
4425 NEONMAP2(vqdmlal_v, arm_neon_vqdmull, arm_neon_vqadds, 0),
4426 NEONMAP2(vqdmlsl_v, arm_neon_vqdmull, arm_neon_vqsubs, 0),
4442 NEONMAP1(vqshlu_n_v, arm_neon_vqshiftsu, 0),
4443 NEONMAP1(vqshluq_n_v, arm_neon_vqshiftsu, 0),
4447 NEONMAP2(vrecpe_v, arm_neon_vrecpe, arm_neon_vrecpe, 0),
4448 NEONMAP2(vrecpeq_v, arm_neon_vrecpe, arm_neon_vrecpe, 0),
4471 NEONMAP2(vrsqrte_v, arm_neon_vrsqrte, arm_neon_vrsqrte, 0),
4472 NEONMAP2(vrsqrteq_v, arm_neon_vrsqrte, arm_neon_vrsqrte, 0),
4476 NEONMAP1(vsha1su0q_v, arm_neon_sha1su0, 0),
4477 NEONMAP1(vsha1su1q_v, arm_neon_sha1su1, 0),
4478 NEONMAP1(vsha256h2q_v, arm_neon_sha256h2, 0),
4479 NEONMAP1(vsha256hq_v, arm_neon_sha256h, 0),
4480 NEONMAP1(vsha256su0q_v, arm_neon_sha256su0, 0),
4481 NEONMAP1(vsha256su1q_v, arm_neon_sha256su1, 0),
4490 NEONMAP1(vst1_v, arm_neon_vst1, 0),
4491 NEONMAP1(vst1_x2_v, arm_neon_vst1x2, 0),
4492 NEONMAP1(vst1_x3_v, arm_neon_vst1x3, 0),
4493 NEONMAP1(vst1_x4_v, arm_neon_vst1x4, 0),
4494 NEONMAP1(vst1q_v, arm_neon_vst1, 0),
4495 NEONMAP1(vst1q_x2_v, arm_neon_vst1x2, 0),
4496 NEONMAP1(vst1q_x3_v, arm_neon_vst1x3, 0),
4497 NEONMAP1(vst1q_x4_v, arm_neon_vst1x4, 0),
4498 NEONMAP1(vst2_lane_v, arm_neon_vst2lane, 0),
4499 NEONMAP1(vst2_v, arm_neon_vst2, 0),
4500 NEONMAP1(vst2q_lane_v, arm_neon_vst2lane, 0),
4501 NEONMAP1(vst2q_v, arm_neon_vst2, 0),
4502 NEONMAP1(vst3_lane_v, arm_neon_vst3lane, 0),
4503 NEONMAP1(vst3_v, arm_neon_vst3, 0),
4504 NEONMAP1(vst3q_lane_v, arm_neon_vst3lane, 0),
4505 NEONMAP1(vst3q_v, arm_neon_vst3, 0),
4506 NEONMAP1(vst4_lane_v, arm_neon_vst4lane, 0),
4507 NEONMAP1(vst4_v, arm_neon_vst4, 0),
4508 NEONMAP1(vst4q_lane_v, arm_neon_vst4lane, 0),
4509 NEONMAP1(vst4q_v, arm_neon_vst4, 0),
4522 NEONMAP1(vabs_v, aarch64_neon_abs, 0),
4523 NEONMAP1(vabsq_v, aarch64_neon_abs, 0),
4525 NEONMAP1(vaesdq_v, aarch64_crypto_aesd, 0),
4526 NEONMAP1(vaeseq_v, aarch64_crypto_aese, 0),
4527 NEONMAP1(vaesimcq_v, aarch64_crypto_aesimc, 0),
4528 NEONMAP1(vaesmcq_v, aarch64_crypto_aesmc, 0),
4529 NEONMAP1(vcage_v, aarch64_neon_facge, 0),
4530 NEONMAP1(vcageq_v, aarch64_neon_facge, 0),
4531 NEONMAP1(vcagt_v, aarch64_neon_facgt, 0),
4532 NEONMAP1(vcagtq_v, aarch64_neon_facgt, 0),
4533 NEONMAP1(vcale_v, aarch64_neon_facge, 0),
4534 NEONMAP1(vcaleq_v, aarch64_neon_facge, 0),
4535 NEONMAP1(vcalt_v, aarch64_neon_facgt, 0),
4536 NEONMAP1(vcaltq_v, aarch64_neon_facgt, 0),
4553 NEONMAP1(vcvt_f16_f32, aarch64_neon_vcvtfp2hf, 0),
4555 NEONMAP1(vcvt_f32_f16, aarch64_neon_vcvthf2fp, 0),
4557 NEONMAP2(vcvt_n_f16_v, aarch64_neon_vcvtfxu2fp, aarch64_neon_vcvtfxs2fp, 0),
4558 NEONMAP2(vcvt_n_f32_v, aarch64_neon_vcvtfxu2fp, aarch64_neon_vcvtfxs2fp, 0),
4559 NEONMAP2(vcvt_n_f64_v, aarch64_neon_vcvtfxu2fp, aarch64_neon_vcvtfxs2fp, 0),
4560 NEONMAP1(vcvt_n_s16_v, aarch64_neon_vcvtfp2fxs, 0),
4561 NEONMAP1(vcvt_n_s32_v, aarch64_neon_vcvtfp2fxs, 0),
4562 NEONMAP1(vcvt_n_s64_v, aarch64_neon_vcvtfp2fxs, 0),
4563 NEONMAP1(vcvt_n_u16_v, aarch64_neon_vcvtfp2fxu, 0),
4564 NEONMAP1(vcvt_n_u32_v, aarch64_neon_vcvtfp2fxu, 0),
4565 NEONMAP1(vcvt_n_u64_v, aarch64_neon_vcvtfp2fxu, 0),
4568 NEONMAP2(vcvtq_n_f16_v, aarch64_neon_vcvtfxu2fp, aarch64_neon_vcvtfxs2fp, 0),
4569 NEONMAP2(vcvtq_n_f32_v, aarch64_neon_vcvtfxu2fp, aarch64_neon_vcvtfxs2fp, 0),
4570 NEONMAP2(vcvtq_n_f64_v, aarch64_neon_vcvtfxu2fp, aarch64_neon_vcvtfxs2fp, 0),
4571 NEONMAP1(vcvtq_n_s16_v, aarch64_neon_vcvtfp2fxs, 0),
4572 NEONMAP1(vcvtq_n_s32_v, aarch64_neon_vcvtfp2fxs, 0),
4573 NEONMAP1(vcvtq_n_s64_v, aarch64_neon_vcvtfp2fxs, 0),
4574 NEONMAP1(vcvtq_n_u16_v, aarch64_neon_vcvtfp2fxu, 0),
4575 NEONMAP1(vcvtq_n_u32_v, aarch64_neon_vcvtfp2fxu, 0),
4576 NEONMAP1(vcvtq_n_u64_v, aarch64_neon_vcvtfp2fxu, 0),
4578 NEONMAP2(vdot_v, aarch64_neon_udot, aarch64_neon_sdot, 0),
4579 NEONMAP2(vdotq_v, aarch64_neon_udot, aarch64_neon_sdot, 0),
4584 NEONMAP1(vfmlal_high_v, aarch64_neon_fmlal2, 0),
4585 NEONMAP1(vfmlal_low_v, aarch64_neon_fmlal, 0),
4586 NEONMAP1(vfmlalq_high_v, aarch64_neon_fmlal2, 0),
4587 NEONMAP1(vfmlalq_low_v, aarch64_neon_fmlal, 0),
4588 NEONMAP1(vfmlsl_high_v, aarch64_neon_fmlsl2, 0),
4589 NEONMAP1(vfmlsl_low_v, aarch64_neon_fmlsl, 0),
4590 NEONMAP1(vfmlslq_high_v, aarch64_neon_fmlsl2, 0),
4591 NEONMAP1(vfmlslq_low_v, aarch64_neon_fmlsl, 0),
4596 NEONMAP1(vld1_x2_v, aarch64_neon_ld1x2, 0),
4597 NEONMAP1(vld1_x3_v, aarch64_neon_ld1x3, 0),
4598 NEONMAP1(vld1_x4_v, aarch64_neon_ld1x4, 0),
4599 NEONMAP1(vld1q_x2_v, aarch64_neon_ld1x2, 0),
4600 NEONMAP1(vld1q_x3_v, aarch64_neon_ld1x3, 0),
4601 NEONMAP1(vld1q_x4_v, aarch64_neon_ld1x4, 0),
4614 NEONMAP2(vqdmlal_v, aarch64_neon_sqdmull, aarch64_neon_sqadd, 0),
4615 NEONMAP2(vqdmlsl_v, aarch64_neon_sqdmull, aarch64_neon_sqsub, 0),
4631 NEONMAP1(vqshlu_n_v, aarch64_neon_sqshlu, 0),
4632 NEONMAP1(vqshluq_n_v, aarch64_neon_sqshlu, 0),
4636 NEONMAP2(vrecpe_v, aarch64_neon_frecpe, aarch64_neon_urecpe, 0),
4637 NEONMAP2(vrecpeq_v, aarch64_neon_frecpe, aarch64_neon_urecpe, 0),
4648 NEONMAP2(vrsqrte_v, aarch64_neon_frsqrte, aarch64_neon_ursqrte, 0),
4649 NEONMAP2(vrsqrteq_v, aarch64_neon_frsqrte, aarch64_neon_ursqrte, 0),
4653 NEONMAP1(vsha1su0q_v, aarch64_crypto_sha1su0, 0),
4654 NEONMAP1(vsha1su1q_v, aarch64_crypto_sha1su1, 0),
4655 NEONMAP1(vsha256h2q_v, aarch64_crypto_sha256h2, 0),
4656 NEONMAP1(vsha256hq_v, aarch64_crypto_sha256h, 0),
4657 NEONMAP1(vsha256su0q_v, aarch64_crypto_sha256su0, 0),
4658 NEONMAP1(vsha256su1q_v, aarch64_crypto_sha256su1, 0),
4667 NEONMAP1(vst1_x2_v, aarch64_neon_st1x2, 0),
4668 NEONMAP1(vst1_x3_v, aarch64_neon_st1x3, 0),
4669 NEONMAP1(vst1_x4_v, aarch64_neon_st1x4, 0),
4670 NEONMAP1(vst1q_x2_v, aarch64_neon_st1x2, 0),
4671 NEONMAP1(vst1q_x3_v, aarch64_neon_st1x3, 0),
4672 NEONMAP1(vst1q_x4_v, aarch64_neon_st1x4, 0),
4727 NEONMAP1(vcvtxd_f32_f64, aarch64_sisd_fcvtxn, 0),
4748 NEONMAP1(vmull_p64, aarch64_neon_pmull64, 0),
4776 NEONMAP1(vqdmulls_s32, aarch64_neon_sqdmulls_scalar, 0),
4853 NEONMAP1(vsha1cq_u32, aarch64_crypto_sha1c, 0),
4854 NEONMAP1(vsha1h_u32, aarch64_crypto_sha1h, 0),
4855 NEONMAP1(vsha1mq_u32, aarch64_crypto_sha1m, 0),
4856 NEONMAP1(vsha1pq_u32, aarch64_crypto_sha1p, 0),
4914 static const NeonIntrinsicInfo *
4916 unsigned BuiltinID,
bool &MapProvenSorted) {
4919 if (!MapProvenSorted) {
4920 assert(std::is_sorted(std::begin(IntrinsicMap), std::end(IntrinsicMap)));
4921 MapProvenSorted =
true;
4925 const NeonIntrinsicInfo *Builtin =
4926 std::lower_bound(IntrinsicMap.begin(), IntrinsicMap.end(), BuiltinID);
4928 if (Builtin != IntrinsicMap.end() && Builtin->BuiltinID == BuiltinID)
4949 Ty = llvm::VectorType::get(
4950 Ty, VectorSize ? VectorSize / Ty->getPrimitiveSizeInBits() : 1);
4957 int Elts = VectorSize ? VectorSize / ArgType->getPrimitiveSizeInBits() : 1;
4958 ArgType = llvm::VectorType::get(ArgType, Elts);
4962 Tys.push_back(ArgType);
4965 Tys.push_back(ArgType);
4968 Tys.push_back(FloatTy);
4970 return CGM.getIntrinsic(IntrinsicID, Tys);
4974 const NeonIntrinsicInfo &SISDInfo,
4977 unsigned BuiltinID = SISDInfo.BuiltinID;
4978 unsigned int Int = SISDInfo.LLVMIntrinsic;
4979 unsigned Modifier = SISDInfo.TypeModifier;
4980 const char *s = SISDInfo.NameHint;
4982 switch (BuiltinID) {
4983 case NEON::BI__builtin_neon_vcled_s64:
4984 case NEON::BI__builtin_neon_vcled_u64:
4985 case NEON::BI__builtin_neon_vcles_f32:
4986 case NEON::BI__builtin_neon_vcled_f64:
4987 case NEON::BI__builtin_neon_vcltd_s64:
4988 case NEON::BI__builtin_neon_vcltd_u64:
4989 case NEON::BI__builtin_neon_vclts_f32:
4990 case NEON::BI__builtin_neon_vcltd_f64:
4991 case NEON::BI__builtin_neon_vcales_f32:
4992 case NEON::BI__builtin_neon_vcaled_f64:
4993 case NEON::BI__builtin_neon_vcalts_f32:
4994 case NEON::BI__builtin_neon_vcaltd_f64:
4998 std::swap(Ops[0], Ops[1]);
5002 assert(Int &&
"Generic code assumes a valid intrinsic");
5010 ConstantInt *C0 = ConstantInt::get(CGF.
SizeTy, 0);
5011 for (Function::const_arg_iterator ai = F->arg_begin(), ae = F->arg_end();
5012 ai != ae; ++ai, ++j) {
5014 if (Ops[j]->getType()->getPrimitiveSizeInBits() ==
5015 ArgTy->getPrimitiveSizeInBits())
5018 assert(ArgTy->isVectorTy() && !Ops[j]->getType()->isVectorTy());
5022 CGF.
Builder.CreateTruncOrBitCast(Ops[j], ArgTy->getVectorElementType());
5024 CGF.
Builder.CreateInsertElement(UndefValue::get(ArgTy), Ops[j], C0);
5029 if (ResultType->getPrimitiveSizeInBits() <
5030 Result->getType()->getPrimitiveSizeInBits())
5031 return CGF.
Builder.CreateExtractElement(Result, C0);
5037 unsigned BuiltinID,
unsigned LLVMIntrinsic,
unsigned AltLLVMIntrinsic,
5040 llvm::Triple::ArchType Arch) {
5042 llvm::APSInt NeonTypeConst;
5049 bool Usgn =
Type.isUnsigned();
5050 bool Quad =
Type.isQuad();
5051 const bool HasLegalHalfType = getTarget().hasLegalHalfType();
5058 auto getAlignmentValue32 = [&](
Address addr) ->
Value* {
5059 return Builder.getInt32(addr.getAlignment().getQuantity());
5062 unsigned Int = LLVMIntrinsic;
5064 Int = AltLLVMIntrinsic;
5066 switch (BuiltinID) {
5068 case NEON::BI__builtin_neon_vabs_v:
5069 case NEON::BI__builtin_neon_vabsq_v:
5070 if (VTy->getElementType()->isFloatingPointTy())
5071 return EmitNeonCall(CGM.getIntrinsic(
Intrinsic::fabs, Ty), Ops,
"vabs");
5072 return EmitNeonCall(CGM.getIntrinsic(LLVMIntrinsic, Ty), Ops,
"vabs");
5073 case NEON::BI__builtin_neon_vaddhn_v: {
5074 llvm::VectorType *SrcTy =
5075 llvm::VectorType::getExtendedElementVectorType(VTy);
5078 Ops[0] = Builder.CreateBitCast(Ops[0], SrcTy);
5079 Ops[1] = Builder.CreateBitCast(Ops[1], SrcTy);
5080 Ops[0] = Builder.CreateAdd(Ops[0], Ops[1],
"vaddhn");
5083 Constant *ShiftAmt =
5084 ConstantInt::get(SrcTy, SrcTy->getScalarSizeInBits() / 2);
5085 Ops[0] = Builder.CreateLShr(Ops[0], ShiftAmt,
"vaddhn");
5088 return Builder.CreateTrunc(Ops[0], VTy,
"vaddhn");
5090 case NEON::BI__builtin_neon_vcale_v:
5091 case NEON::BI__builtin_neon_vcaleq_v:
5092 case NEON::BI__builtin_neon_vcalt_v:
5093 case NEON::BI__builtin_neon_vcaltq_v:
5094 std::swap(Ops[0], Ops[1]);
5096 case NEON::BI__builtin_neon_vcage_v:
5097 case NEON::BI__builtin_neon_vcageq_v:
5098 case NEON::BI__builtin_neon_vcagt_v:
5099 case NEON::BI__builtin_neon_vcagtq_v: {
5101 switch (VTy->getScalarSizeInBits()) {
5102 default: llvm_unreachable(
"unexpected type");
5113 llvm::Type *VecFlt = llvm::VectorType::get(Ty, VTy->getNumElements());
5115 Function *F = CGM.getIntrinsic(LLVMIntrinsic, Tys);
5116 return EmitNeonCall(F, Ops, NameHint);
5118 case NEON::BI__builtin_neon_vceqz_v:
5119 case NEON::BI__builtin_neon_vceqzq_v:
5120 return EmitAArch64CompareBuiltinExpr(Ops[0], Ty, ICmpInst::FCMP_OEQ,
5121 ICmpInst::ICMP_EQ,
"vceqz");
5122 case NEON::BI__builtin_neon_vcgez_v:
5123 case NEON::BI__builtin_neon_vcgezq_v:
5124 return EmitAArch64CompareBuiltinExpr(Ops[0], Ty, ICmpInst::FCMP_OGE,
5125 ICmpInst::ICMP_SGE,
"vcgez");
5126 case NEON::BI__builtin_neon_vclez_v:
5127 case NEON::BI__builtin_neon_vclezq_v:
5128 return EmitAArch64CompareBuiltinExpr(Ops[0], Ty, ICmpInst::FCMP_OLE,
5129 ICmpInst::ICMP_SLE,
"vclez");
5130 case NEON::BI__builtin_neon_vcgtz_v:
5131 case NEON::BI__builtin_neon_vcgtzq_v:
5132 return EmitAArch64CompareBuiltinExpr(Ops[0], Ty, ICmpInst::FCMP_OGT,
5133 ICmpInst::ICMP_SGT,
"vcgtz");
5134 case NEON::BI__builtin_neon_vcltz_v:
5135 case NEON::BI__builtin_neon_vcltzq_v:
5136 return EmitAArch64CompareBuiltinExpr(Ops[0], Ty, ICmpInst::FCMP_OLT,
5137 ICmpInst::ICMP_SLT,
"vcltz");
5138 case NEON::BI__builtin_neon_vclz_v:
5139 case NEON::BI__builtin_neon_vclzq_v:
5142 Ops.push_back(Builder.getInt1(getTarget().isCLZForZeroUndef()));
5144 case NEON::BI__builtin_neon_vcvt_f32_v:
5145 case NEON::BI__builtin_neon_vcvtq_f32_v:
5146 Ops[0] = Builder.CreateBitCast(Ops[0], Ty);
5149 return Usgn ? Builder.CreateUIToFP(Ops[0], Ty,
"vcvt")
5150 : Builder.CreateSIToFP(Ops[0], Ty,
"vcvt");
5151 case NEON::BI__builtin_neon_vcvt_f16_v:
5152 case NEON::BI__builtin_neon_vcvtq_f16_v:
5153 Ops[0] = Builder.CreateBitCast(Ops[0], Ty);
5156 return Usgn ? Builder.CreateUIToFP(Ops[0], Ty,
"vcvt")
5157 : Builder.CreateSIToFP(Ops[0], Ty,
"vcvt");
5158 case NEON::BI__builtin_neon_vcvt_n_f16_v:
5159 case NEON::BI__builtin_neon_vcvt_n_f32_v:
5160 case NEON::BI__builtin_neon_vcvt_n_f64_v:
5161 case NEON::BI__builtin_neon_vcvtq_n_f16_v:
5162 case NEON::BI__builtin_neon_vcvtq_n_f32_v:
5163 case NEON::BI__builtin_neon_vcvtq_n_f64_v: {
5165 Int = Usgn ? LLVMIntrinsic : AltLLVMIntrinsic;
5166 Function *F = CGM.getIntrinsic(Int, Tys);
5167 return EmitNeonCall(F, Ops,
"vcvt_n");
5169 case NEON::BI__builtin_neon_vcvt_n_s16_v:
5170 case NEON::BI__builtin_neon_vcvt_n_s32_v:
5171 case NEON::BI__builtin_neon_vcvt_n_u16_v:
5172 case NEON::BI__builtin_neon_vcvt_n_u32_v:
5173 case NEON::BI__builtin_neon_vcvt_n_s64_v:
5174 case NEON::BI__builtin_neon_vcvt_n_u64_v:
5175 case NEON::BI__builtin_neon_vcvtq_n_s16_v:
5176 case NEON::BI__builtin_neon_vcvtq_n_s32_v:
5177 case NEON::BI__builtin_neon_vcvtq_n_u16_v:
5178 case NEON::BI__builtin_neon_vcvtq_n_u32_v:
5179 case NEON::BI__builtin_neon_vcvtq_n_s64_v:
5180 case NEON::BI__builtin_neon_vcvtq_n_u64_v: {
5182 Function *F = CGM.getIntrinsic(LLVMIntrinsic, Tys);
5183 return EmitNeonCall(F, Ops,
"vcvt_n");
5185 case NEON::BI__builtin_neon_vcvt_s32_v:
5186 case NEON::BI__builtin_neon_vcvt_u32_v:
5187 case NEON::BI__builtin_neon_vcvt_s64_v:
5188 case NEON::BI__builtin_neon_vcvt_u64_v:
5189 case NEON::BI__builtin_neon_vcvt_s16_v:
5190 case NEON::BI__builtin_neon_vcvt_u16_v:
5191 case NEON::BI__builtin_neon_vcvtq_s32_v:
5192 case NEON::BI__builtin_neon_vcvtq_u32_v:
5193 case NEON::BI__builtin_neon_vcvtq_s64_v:
5194 case NEON::BI__builtin_neon_vcvtq_u64_v:
5195 case NEON::BI__builtin_neon_vcvtq_s16_v:
5196 case NEON::BI__builtin_neon_vcvtq_u16_v: {
5198 return Usgn ? Builder.CreateFPToUI(Ops[0], Ty,
"vcvt")
5199 : Builder.CreateFPToSI(Ops[0], Ty,
"vcvt");
5201 case NEON::BI__builtin_neon_vcvta_s16_v:
5202 case NEON::BI__builtin_neon_vcvta_s32_v:
5203 case NEON::BI__builtin_neon_vcvta_s64_v:
5204 case NEON::BI__builtin_neon_vcvta_u16_v:
5205 case NEON::BI__builtin_neon_vcvta_u32_v:
5206 case NEON::BI__builtin_neon_vcvta_u64_v:
5207 case NEON::BI__builtin_neon_vcvtaq_s16_v:
5208 case NEON::BI__builtin_neon_vcvtaq_s32_v:
5209 case NEON::BI__builtin_neon_vcvtaq_s64_v:
5210 case NEON::BI__builtin_neon_vcvtaq_u16_v:
5211 case NEON::BI__builtin_neon_vcvtaq_u32_v:
5212 case NEON::BI__builtin_neon_vcvtaq_u64_v:
5213 case NEON::BI__builtin_neon_vcvtn_s16_v:
5214 case NEON::BI__builtin_neon_vcvtn_s32_v:
5215 case NEON::BI__builtin_neon_vcvtn_s64_v:
5216 case NEON::BI__builtin_neon_vcvtn_u16_v:
5217 case NEON::BI__builtin_neon_vcvtn_u32_v:
5218 case NEON::BI__builtin_neon_vcvtn_u64_v:
5219 case NEON::BI__builtin_neon_vcvtnq_s16_v:
5220 case NEON::BI__builtin_neon_vcvtnq_s32_v:
5221 case NEON::BI__builtin_neon_vcvtnq_s64_v:
5222 case NEON::BI__builtin_neon_vcvtnq_u16_v:
5223 case NEON::BI__builtin_neon_vcvtnq_u32_v:
5224 case NEON::BI__builtin_neon_vcvtnq_u64_v:
5225 case NEON::BI__builtin_neon_vcvtp_s16_v:
5226 case NEON::BI__builtin_neon_vcvtp_s32_v:
5227 case NEON::BI__builtin_neon_vcvtp_s64_v:
5228 case NEON::BI__builtin_neon_vcvtp_u16_v:
5229 case NEON::BI__builtin_neon_vcvtp_u32_v:
5230 case NEON::BI__builtin_neon_vcvtp_u64_v:
5231 case NEON::BI__builtin_neon_vcvtpq_s16_v:
5232 case NEON::BI__builtin_neon_vcvtpq_s32_v:
5233 case NEON::BI__builtin_neon_vcvtpq_s64_v:
5234 case NEON::BI__builtin_neon_vcvtpq_u16_v:
5235 case NEON::BI__builtin_neon_vcvtpq_u32_v:
5236 case NEON::BI__builtin_neon_vcvtpq_u64_v:
5237 case NEON::BI__builtin_neon_vcvtm_s16_v:
5238 case NEON::BI__builtin_neon_vcvtm_s32_v:
5239 case NEON::BI__builtin_neon_vcvtm_s64_v:
5240 case NEON::BI__builtin_neon_vcvtm_u16_v:
5241 case NEON::BI__builtin_neon_vcvtm_u32_v:
5242 case NEON::BI__builtin_neon_vcvtm_u64_v:
5243 case NEON::BI__builtin_neon_vcvtmq_s16_v:
5244 case NEON::BI__builtin_neon_vcvtmq_s32_v:
5245 case NEON::BI__builtin_neon_vcvtmq_s64_v:
5246 case NEON::BI__builtin_neon_vcvtmq_u16_v:
5247 case NEON::BI__builtin_neon_vcvtmq_u32_v:
5248 case NEON::BI__builtin_neon_vcvtmq_u64_v: {
5250 return EmitNeonCall(CGM.getIntrinsic(LLVMIntrinsic, Tys), Ops, NameHint);
5252 case NEON::BI__builtin_neon_vext_v:
5253 case NEON::BI__builtin_neon_vextq_v: {
5254 int CV = cast<ConstantInt>(Ops[2])->getSExtValue();
5256 for (
unsigned i = 0, e = VTy->getNumElements(); i != e; ++i)
5257 Indices.push_back(i+CV);
5259 Ops[0] = Builder.CreateBitCast(Ops[0], Ty);
5260 Ops[1] = Builder.CreateBitCast(Ops[1], Ty);
5261 return Builder.CreateShuffleVector(Ops[0], Ops[1], Indices,
"vext");
5263 case NEON::BI__builtin_neon_vfma_v:
5264 case NEON::BI__builtin_neon_vfmaq_v: {
5266 Ops[0] = Builder.CreateBitCast(Ops[0], Ty);
5267 Ops[1] = Builder.CreateBitCast(Ops[1], Ty);
5268 Ops[2] = Builder.CreateBitCast(Ops[2], Ty);
5271 return Builder.CreateCall(F, {Ops[1], Ops[2], Ops[0]});
5273 case NEON::BI__builtin_neon_vld1_v:
5274 case NEON::BI__builtin_neon_vld1q_v: {
5276 Ops.push_back(getAlignmentValue32(PtrOp0));
5277 return EmitNeonCall(CGM.getIntrinsic(LLVMIntrinsic, Tys), Ops,
"vld1");
5279 case NEON::BI__builtin_neon_vld1_x2_v:
5280 case NEON::BI__builtin_neon_vld1q_x2_v:
5281 case NEON::BI__builtin_neon_vld1_x3_v:
5282 case NEON::BI__builtin_neon_vld1q_x3_v:
5283 case NEON::BI__builtin_neon_vld1_x4_v:
5284 case NEON::BI__builtin_neon_vld1q_x4_v: {
5285 llvm::Type *PTy = llvm::PointerType::getUnqual(VTy->getVectorElementType());
5286 Ops[1] = Builder.CreateBitCast(Ops[1], PTy);
5288 Function *F = CGM.getIntrinsic(LLVMIntrinsic, Tys);
5289 Ops[1] = Builder.CreateCall(F, Ops[1],
"vld1xN");
5290 Ty = llvm::PointerType::getUnqual(Ops[1]->getType());
5291 Ops[0] = Builder.CreateBitCast(Ops[0], Ty);
5292 return Builder.CreateDefaultAlignedStore(Ops[1], Ops[0]);
5294 case NEON::BI__builtin_neon_vld2_v:
5295 case NEON::BI__builtin_neon_vld2q_v:
5296 case NEON::BI__builtin_neon_vld3_v:
5297 case NEON::BI__builtin_neon_vld3q_v:
5298 case NEON::BI__builtin_neon_vld4_v:
5299 case NEON::BI__builtin_neon_vld4q_v:
5300 case NEON::BI__builtin_neon_vld2_dup_v:
5301 case NEON::BI__builtin_neon_vld2q_dup_v:
5302 case NEON::BI__builtin_neon_vld3_dup_v:
5303 case NEON::BI__builtin_neon_vld3q_dup_v:
5304 case NEON::BI__builtin_neon_vld4_dup_v:
5305 case NEON::BI__builtin_neon_vld4q_dup_v: {
5307 Function *F = CGM.getIntrinsic(LLVMIntrinsic, Tys);
5308 Value *Align = getAlignmentValue32(PtrOp1);
5309 Ops[1] = Builder.CreateCall(F, {Ops[1], Align}, NameHint);
5310 Ty = llvm::PointerType::getUnqual(Ops[1]->getType());
5311 Ops[0] = Builder.CreateBitCast(Ops[0], Ty);
5312 return Builder.CreateDefaultAlignedStore(Ops[1], Ops[0]);
5314 case NEON::BI__builtin_neon_vld1_dup_v:
5315 case NEON::BI__builtin_neon_vld1q_dup_v: {
5316 Value *V = UndefValue::get(Ty);
5317 Ty = llvm::PointerType::getUnqual(VTy->getElementType());
5318 PtrOp0 = Builder.CreateBitCast(PtrOp0, Ty);
5319 LoadInst *Ld = Builder.CreateLoad(PtrOp0);
5320 llvm::Constant *CI = ConstantInt::get(SizeTy, 0);
5321 Ops[0] = Builder.CreateInsertElement(V, Ld, CI);
5322 return EmitNeonSplat(Ops[0], CI);
5324 case NEON::BI__builtin_neon_vld2_lane_v:
5325 case NEON::BI__builtin_neon_vld2q_lane_v:
5326 case NEON::BI__builtin_neon_vld3_lane_v:
5327 case NEON::BI__builtin_neon_vld3q_lane_v:
5328 case NEON::BI__builtin_neon_vld4_lane_v:
5329 case NEON::BI__builtin_neon_vld4q_lane_v: {
5331 Function *F = CGM.getIntrinsic(LLVMIntrinsic, Tys);
5332 for (
unsigned I = 2; I < Ops.size() - 1; ++I)
5333 Ops[I] = Builder.CreateBitCast(Ops[I], Ty);
5334 Ops.push_back(getAlignmentValue32(PtrOp1));
5335 Ops[1] = Builder.CreateCall(F, makeArrayRef(Ops).slice(1), NameHint);
5336 Ty = llvm::PointerType::getUnqual(Ops[1]->getType());
5337 Ops[0] = Builder.CreateBitCast(Ops[0], Ty);
5338 return Builder.CreateDefaultAlignedStore(Ops[1], Ops[0]);
5340 case NEON::BI__builtin_neon_vmovl_v: {
5341 llvm::Type *DTy =llvm::VectorType::getTruncatedElementVectorType(VTy);
5342 Ops[0] = Builder.CreateBitCast(Ops[0], DTy);
5344 return Builder.CreateZExt(Ops[0], Ty,
"vmovl");
5345 return Builder.CreateSExt(Ops[0], Ty,
"vmovl");
5347 case NEON::BI__builtin_neon_vmovn_v: {
5348 llvm::Type *QTy = llvm::VectorType::getExtendedElementVectorType(VTy);
5349 Ops[0] = Builder.CreateBitCast(Ops[0], QTy);
5350 return Builder.CreateTrunc(Ops[0], Ty,
"vmovn");
5352 case NEON::BI__builtin_neon_vmull_v:
5358 Int = Usgn ? Intrinsic::arm_neon_vmullu : Intrinsic::arm_neon_vmulls;
5359 Int =
Type.isPoly() ? (unsigned)Intrinsic::arm_neon_vmullp : Int;
5360 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops,
"vmull");
5361 case NEON::BI__builtin_neon_vpadal_v:
5362 case NEON::BI__builtin_neon_vpadalq_v: {
5364 unsigned EltBits = VTy->getElementType()->getPrimitiveSizeInBits();
5366 llvm::IntegerType::get(getLLVMContext(), EltBits / 2);
5368 llvm::VectorType::get(EltTy, VTy->getNumElements() * 2);
5370 return EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, NameHint);
5372 case NEON::BI__builtin_neon_vpaddl_v:
5373 case NEON::BI__builtin_neon_vpaddlq_v: {
5375 unsigned EltBits = VTy->getElementType()->getPrimitiveSizeInBits();
5376 llvm::Type *EltTy = llvm::IntegerType::get(getLLVMContext(), EltBits / 2);
5378 llvm::VectorType::get(EltTy, VTy->getNumElements() * 2);
5380 return EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops,
"vpaddl");
5382 case NEON::BI__builtin_neon_vqdmlal_v:
5383 case NEON::BI__builtin_neon_vqdmlsl_v: {
5386 EmitNeonCall(CGM.getIntrinsic(LLVMIntrinsic, Ty), MulOps,
"vqdmlal");
5388 return EmitNeonCall(CGM.getIntrinsic(AltLLVMIntrinsic, Ty), Ops, NameHint);
5390 case NEON::BI__builtin_neon_vqshl_n_v:
5391 case NEON::BI__builtin_neon_vqshlq_n_v:
5392 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops,
"vqshl_n",
5394 case NEON::BI__builtin_neon_vqshlu_n_v:
5395 case NEON::BI__builtin_neon_vqshluq_n_v:
5396 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops,
"vqshlu_n",
5398 case NEON::BI__builtin_neon_vrecpe_v:
5399 case NEON::BI__builtin_neon_vrecpeq_v:
5400 case NEON::BI__builtin_neon_vrsqrte_v:
5401 case NEON::BI__builtin_neon_vrsqrteq_v:
5402 Int = Ty->isFPOrFPVectorTy() ? LLVMIntrinsic : AltLLVMIntrinsic;
5403 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, NameHint);
5404 case NEON::BI__builtin_neon_vrndi_v:
5405 case NEON::BI__builtin_neon_vrndiq_v:
5407 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, NameHint);
5408 case NEON::BI__builtin_neon_vrshr_n_v:
5409 case NEON::BI__builtin_neon_vrshrq_n_v:
5410 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops,
"vrshr_n",
5412 case NEON::BI__builtin_neon_vshl_n_v:
5413 case NEON::BI__builtin_neon_vshlq_n_v:
5414 Ops[1] = EmitNeonShiftVector(Ops[1], Ty,
false);
5415 return Builder.CreateShl(Builder.CreateBitCast(Ops[0],Ty), Ops[1],
5417 case NEON::BI__builtin_neon_vshll_n_v: {
5418 llvm::Type *SrcTy = llvm::VectorType::getTruncatedElementVectorType(VTy);
5419 Ops[0] = Builder.CreateBitCast(Ops[0], SrcTy);
5421 Ops[0] = Builder.CreateZExt(Ops[0], VTy);
5423 Ops[0] = Builder.CreateSExt(Ops[0], VTy);
5424 Ops[1] = EmitNeonShiftVector(Ops[1], VTy,
false);
5425 return Builder.CreateShl(Ops[0], Ops[1],
"vshll_n");
5427 case NEON::BI__builtin_neon_vshrn_n_v: {
5428 llvm::Type *SrcTy = llvm::VectorType::getExtendedElementVectorType(VTy);
5429 Ops[0] = Builder.CreateBitCast(Ops[0], SrcTy);
5430 Ops[1] = EmitNeonShiftVector(Ops[1], SrcTy,
false);
5432 Ops[0] = Builder.CreateLShr(Ops[0], Ops[1]);
5434 Ops[0] = Builder.CreateAShr(Ops[0], Ops[1]);
5435 return Builder.CreateTrunc(Ops[0], Ty,
"vshrn_n");
5437 case NEON::BI__builtin_neon_vshr_n_v:
5438 case NEON::BI__builtin_neon_vshrq_n_v:
5439 return EmitNeonRShiftImm(Ops[0], Ops[1], Ty, Usgn,
"vshr_n");
5440 case NEON::BI__builtin_neon_vst1_v:
5441 case NEON::BI__builtin_neon_vst1q_v:
5442 case NEON::BI__builtin_neon_vst2_v:
5443 case NEON::BI__builtin_neon_vst2q_v:
5444 case NEON::BI__builtin_neon_vst3_v:
5445 case NEON::BI__builtin_neon_vst3q_v:
5446 case NEON::BI__builtin_neon_vst4_v:
5447 case NEON::BI__builtin_neon_vst4q_v:
5448 case NEON::BI__builtin_neon_vst2_lane_v:
5449 case NEON::BI__builtin_neon_vst2q_lane_v:
5450 case NEON::BI__builtin_neon_vst3_lane_v:
5451 case NEON::BI__builtin_neon_vst3q_lane_v:
5452 case NEON::BI__builtin_neon_vst4_lane_v:
5453 case NEON::BI__builtin_neon_vst4q_lane_v: {
5455 Ops.push_back(getAlignmentValue32(PtrOp0));
5456 return EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops,
"");
5458 case NEON::BI__builtin_neon_vst1_x2_v:
5459 case NEON::BI__builtin_neon_vst1q_x2_v:
5460 case NEON::BI__builtin_neon_vst1_x3_v:
5461 case NEON::BI__builtin_neon_vst1q_x3_v:
5462 case NEON::BI__builtin_neon_vst1_x4_v:
5463 case NEON::BI__builtin_neon_vst1q_x4_v: {
5464 llvm::Type *PTy = llvm::PointerType::getUnqual(VTy->getVectorElementType());
5467 if (Arch == llvm::Triple::aarch64 || Arch == llvm::Triple::aarch64_be) {
5469 std::rotate(Ops.begin(), Ops.begin() + 1, Ops.end());
5470 return EmitNeonCall(CGM.getIntrinsic(LLVMIntrinsic, Tys), Ops,
"");
5473 return EmitNeonCall(CGM.getIntrinsic(LLVMIntrinsic, Tys), Ops,
"");
5475 case NEON::BI__builtin_neon_vsubhn_v: {
5476 llvm::VectorType *SrcTy =
5477 llvm::VectorType::getExtendedElementVectorType(VTy);
5480 Ops[0] = Builder.CreateBitCast(Ops[0], SrcTy);
5481 Ops[1] = Builder.CreateBitCast(Ops[1], SrcTy);
5482 Ops[0] = Builder.CreateSub(Ops[0], Ops[1],
"vsubhn");
5485 Constant *ShiftAmt =
5486 ConstantInt::get(SrcTy, SrcTy->getScalarSizeInBits() / 2);
5487 Ops[0] = Builder.CreateLShr(Ops[0], ShiftAmt,
"vsubhn");
5490 return Builder.CreateTrunc(Ops[0], VTy,
"vsubhn");
5492 case NEON::BI__builtin_neon_vtrn_v:
5493 case NEON::BI__builtin_neon_vtrnq_v: {
5494 Ops[0] = Builder.CreateBitCast(Ops[0], llvm::PointerType::getUnqual(Ty));
5495 Ops[1] = Builder.CreateBitCast(Ops[1], Ty);
5496 Ops[2] = Builder.CreateBitCast(Ops[2], Ty);
5497 Value *SV =
nullptr;
5499 for (
unsigned vi = 0; vi != 2; ++vi) {
5501 for (
unsigned i = 0, e = VTy->getNumElements(); i != e; i += 2) {
5502 Indices.push_back(i+vi);
5503 Indices.push_back(i+e+vi);
5505 Value *Addr = Builder.CreateConstInBoundsGEP1_32(Ty, Ops[0], vi);
5506 SV = Builder.CreateShuffleVector(Ops[1], Ops[2], Indices,
"vtrn");
5507 SV = Builder.CreateDefaultAlignedStore(SV, Addr);
5511 case NEON::BI__builtin_neon_vtst_v:
5512 case NEON::BI__builtin_neon_vtstq_v: {
5513 Ops[0] = Builder.CreateBitCast(Ops[0], Ty);
5514 Ops[1] = Builder.CreateBitCast(Ops[1], Ty);
5515 Ops[0] = Builder.CreateAnd(Ops[0], Ops[1]);
5516 Ops[0] = Builder.CreateICmp(ICmpInst::ICMP_NE, Ops[0],
5517 ConstantAggregateZero::get(Ty));
5518 return Builder.CreateSExt(Ops[0], Ty,
"vtst");
5520 case NEON::BI__builtin_neon_vuzp_v:
5521 case NEON::BI__builtin_neon_vuzpq_v: {
5522 Ops[0] = Builder.CreateBitCast(Ops[0], llvm::PointerType::getUnqual(Ty));
5523 Ops[1] = Builder.CreateBitCast(Ops[1], Ty);
5524 Ops[2] = Builder.CreateBitCast(Ops[2], Ty);
5525 Value *SV =
nullptr;
5527 for (
unsigned vi = 0; vi != 2; ++vi) {
5529 for (
unsigned i = 0, e = VTy->getNumElements(); i != e; ++i)
5530 Indices.push_back(2*i+vi);
5532 Value *Addr = Builder.CreateConstInBoundsGEP1_32(Ty, Ops[0], vi);
5533 SV = Builder.CreateShuffleVector(Ops[1], Ops[2], Indices,
"vuzp");
5534 SV = Builder.CreateDefaultAlignedStore(SV, Addr);
5538 case NEON::BI__builtin_neon_vzip_v:
5539 case NEON::BI__builtin_neon_vzipq_v: {
5540 Ops[0] = Builder.CreateBitCast(Ops[0], llvm::PointerType::getUnqual(Ty));
5541 Ops[1] = Builder.CreateBitCast(Ops[1], Ty);
5542 Ops[2] = Builder.CreateBitCast(Ops[2], Ty);
5543 Value *SV =
nullptr;
5545 for (
unsigned vi = 0; vi != 2; ++vi) {
5547 for (
unsigned i = 0, e = VTy->getNumElements(); i != e; i += 2) {
5548 Indices.push_back((i + vi*e) >> 1);
5549 Indices.push_back(((i + vi*e) >> 1)+e);
5551 Value *Addr = Builder.CreateConstInBoundsGEP1_32(Ty, Ops[0], vi);
5552 SV = Builder.CreateShuffleVector(Ops[1], Ops[2], Indices,
"vzip");
5553 SV = Builder.CreateDefaultAlignedStore(SV, Addr);
5557 case NEON::BI__builtin_neon_vdot_v:
5558 case NEON::BI__builtin_neon_vdotq_v: {
5560 llvm::VectorType::get(Int8Ty, Ty->getPrimitiveSizeInBits() / 8);
5562 Int = Usgn ? LLVMIntrinsic : AltLLVMIntrinsic;
5563 return EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops,
"vdot");
5565 case NEON::BI__builtin_neon_vfmlal_low_v:
5566 case NEON::BI__builtin_neon_vfmlalq_low_v: {
5568 llvm::VectorType::get(HalfTy, Ty->getPrimitiveSizeInBits() / 16);
5570 return EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops,
"vfmlal_low");
5572 case NEON::BI__builtin_neon_vfmlsl_low_v:
5573 case NEON::BI__builtin_neon_vfmlslq_low_v: {
5575 llvm::VectorType::get(HalfTy, Ty->getPrimitiveSizeInBits() / 16);
5577 return EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops,
"vfmlsl_low");
5579 case NEON::BI__builtin_neon_vfmlal_high_v:
5580 case NEON::BI__builtin_neon_vfmlalq_high_v: {
5582 llvm::VectorType::get(HalfTy, Ty->getPrimitiveSizeInBits() / 16);
5584 return EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops,
"vfmlal_high");
5586 case NEON::BI__builtin_neon_vfmlsl_high_v:
5587 case NEON::BI__builtin_neon_vfmlslq_high_v: {
5589 llvm::VectorType::get(HalfTy, Ty->getPrimitiveSizeInBits() / 16);
5591 return EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops,
"vfmlsl_high");
5595 assert(Int &&
"Expected valid intrinsic number");
5598 Function *F = LookupNeonLLVMIntrinsic(Int, Modifier, Ty, E);
5600 Value *Result = EmitNeonCall(F, Ops, NameHint);
5604 return Builder.CreateBitCast(Result, ResultType, NameHint);
5609 const CmpInst::Predicate Ip,
const Twine &Name) {
5616 if (BitCastInst *BI = dyn_cast<BitCastInst>(Op))
5617 OTy = BI->getOperand(0)->getType();
5619 Op = Builder.CreateBitCast(Op, OTy);
5620 if (OTy->getScalarType()->isFloatingPointTy()) {
5621 Op = Builder.CreateFCmp(Fp, Op, Constant::getNullValue(OTy));
5623 Op = Builder.CreateICmp(Ip, Op, Constant::getNullValue(OTy));
5625 return Builder.CreateSExt(Op, Ty, Name);
5634 TblOps.push_back(ExtOp);
5638 llvm::VectorType *TblTy = cast<llvm::VectorType>(Ops[0]->getType());
5639 for (
unsigned i = 0, e = TblTy->getNumElements(); i != e; ++i) {
5640 Indices.push_back(2*i);
5641 Indices.push_back(2*i+1);
5644 int PairPos = 0,
End = Ops.size() - 1;
5645 while (PairPos <
End) {
5646 TblOps.push_back(CGF.
Builder.CreateShuffleVector(Ops[PairPos],
5647 Ops[PairPos+1], Indices,
5654 if (PairPos ==
End) {
5655 Value *ZeroTbl = ConstantAggregateZero::get(TblTy);
5656 TblOps.push_back(CGF.
Builder.CreateShuffleVector(Ops[PairPos],
5657 ZeroTbl, Indices, Name));
5661 TblOps.push_back(IndexOp);
5667 Value *CodeGenFunction::GetValueForARMHint(
unsigned BuiltinID) {
5669 switch (BuiltinID) {
5672 case ARM::BI__builtin_arm_nop:
5675 case ARM::BI__builtin_arm_yield:
5676 case ARM::BI__yield:
5679 case ARM::BI__builtin_arm_wfe:
5683 case ARM::BI__builtin_arm_wfi:
5687 case ARM::BI__builtin_arm_sev:
5691 case ARM::BI__builtin_arm_sevl:
5697 return Builder.CreateCall(CGM.getIntrinsic(Intrinsic::arm_hint),
5698 llvm::ConstantInt::get(Int32Ty, Value));
5709 StringRef SysReg =
"") {
5711 assert((RegisterType->isIntegerTy(32) || RegisterType->isIntegerTy(64))
5712 &&
"Unsupported size for register.");
5718 if (SysReg.empty()) {
5720 SysReg = cast<clang::StringLiteral>(SysRegStrExpr)->getString();
5723 llvm::Metadata *Ops[] = { llvm::MDString::get(Context, SysReg) };
5724 llvm::MDNode *RegName = llvm::MDNode::get(Context, Ops);
5725 llvm::Value *Metadata = llvm::MetadataAsValue::get(Context, RegName);
5729 bool MixedTypes = RegisterType->isIntegerTy(64) && ValueType->isIntegerTy(32);
5730 assert(!(RegisterType->isIntegerTy(32) && ValueType->isIntegerTy(64))
5731 &&
"Can't fit 64-bit value in 32-bit register");
5735 llvm::Value *Call = Builder.CreateCall(F, Metadata);
5739 return Builder.CreateTrunc(Call, ValueType);
5741 if (ValueType->isPointerTy())
5743 return Builder.CreateIntToPtr(Call, ValueType);
5752 ArgValue = Builder.CreateZExt(ArgValue, RegisterType);
5753 return Builder.CreateCall(F, { Metadata, ArgValue });
5756 if (ValueType->isPointerTy()) {
5758 ArgValue = Builder.CreatePtrToInt(ArgValue, RegisterType);
5759 return Builder.CreateCall(F, { Metadata, ArgValue });
5762 return Builder.CreateCall(F, { Metadata, ArgValue });
5768 switch (BuiltinID) {
5770 case NEON::BI__builtin_neon_vget_lane_i8:
5771 case NEON::BI__builtin_neon_vget_lane_i16:
5772 case NEON::BI__builtin_neon_vget_lane_i32:
5773 case NEON::BI__builtin_neon_vget_lane_i64:
5774 case NEON::BI__builtin_neon_vget_lane_f32:
5775 case NEON::BI__builtin_neon_vgetq_lane_i8:
5776 case NEON::BI__builtin_neon_vgetq_lane_i16:
5777 case NEON::BI__builtin_neon_vgetq_lane_i32:
5778 case NEON::BI__builtin_neon_vgetq_lane_i64:
5779 case NEON::BI__builtin_neon_vgetq_lane_f32:
5780 case NEON::BI__builtin_neon_vset_lane_i8:
5781 case NEON::BI__builtin_neon_vset_lane_i16:
5782 case NEON::BI__builtin_neon_vset_lane_i32:
5783 case NEON::BI__builtin_neon_vset_lane_i64:
5784 case NEON::BI__builtin_neon_vset_lane_f32:
5785 case NEON::BI__builtin_neon_vsetq_lane_i8:
5786 case NEON::BI__builtin_neon_vsetq_lane_i16:
5787 case NEON::BI__builtin_neon_vsetq_lane_i32:
5788 case NEON::BI__builtin_neon_vsetq_lane_i64:
5789 case NEON::BI__builtin_neon_vsetq_lane_f32:
5790 case NEON::BI__builtin_neon_vsha1h_u32:
5791 case NEON::BI__builtin_neon_vsha1cq_u32:
5792 case NEON::BI__builtin_neon_vsha1pq_u32:
5793 case NEON::BI__builtin_neon_vsha1mq_u32:
5794 case clang::ARM::BI_MoveToCoprocessor:
5795 case clang::ARM::BI_MoveToCoprocessor2:
5804 CharUnits LoadSize = getContext().getTypeSizeInChars(ElTy);
5805 llvm::Type *ITy = llvm::IntegerType::get(getLLVMContext(),
5807 Ptr = Builder.CreateBitCast(Ptr, ITy->getPointerTo());
5808 llvm::LoadInst *Load =
5809 Builder.CreateAlignedLoad(Ptr, LoadSize);
5810 Load->setVolatile(
true);
5818 CharUnits StoreSize = getContext().getTypeSizeInChars(ElTy);
5819 llvm::Type *ITy = llvm::IntegerType::get(getLLVMContext(),
5821 Ptr = Builder.CreateBitCast(Ptr, ITy->getPointerTo());
5822 llvm::StoreInst *
Store =
5823 Builder.CreateAlignedStore(Value, Ptr,
5825 Store->setVolatile(
true);
5831 llvm::Triple::ArchType Arch) {
5832 if (
auto Hint = GetValueForARMHint(BuiltinID))
5835 if (BuiltinID == ARM::BI__emit) {
5836 bool IsThumb = getTarget().getTriple().getArch() == llvm::Triple::thumb;
5837 llvm::FunctionType *FTy =
5838 llvm::FunctionType::get(VoidTy,
false);
5842 llvm_unreachable(
"Sema will ensure that the parameter is constant");
5845 uint64_t ZExtValue = Value.zextOrTrunc(IsThumb ? 16 : 32).getZExtValue();
5847 llvm::InlineAsm *Emit =
5848 IsThumb ? InlineAsm::get(FTy,
".inst.n 0x" + utohexstr(ZExtValue),
"",
5850 : InlineAsm::get(FTy,
".inst 0x" + utohexstr(ZExtValue),
"",
5853 return Builder.CreateCall(Emit);
5856 if (BuiltinID == ARM::BI__builtin_arm_dbg) {
5858 return Builder.CreateCall(CGM.getIntrinsic(Intrinsic::arm_dbg), Option);
5861 if (BuiltinID == ARM::BI__builtin_arm_prefetch) {
5867 Value *Locality = llvm::ConstantInt::get(Int32Ty, 3);
5870 return Builder.CreateCall(F, {Address, RW, Locality, IsData});
5873 if (BuiltinID == ARM::BI__builtin_arm_rbit) {
5875 return Builder.CreateCall(
5876 CGM.getIntrinsic(Intrinsic::bitreverse, Arg->getType()), Arg,
"rbit");
5879 if (BuiltinID == ARM::BI__clear_cache) {
5880 assert(E->
getNumArgs() == 2 &&
"__clear_cache takes 2 arguments");
5883 for (
unsigned i = 0; i < 2; i++)
5884 Ops[i] = EmitScalarExpr(E->
getArg(i));
5885 llvm::Type *Ty = CGM.getTypes().ConvertType(FD->getType());
5886 llvm::FunctionType *FTy = cast<llvm::FunctionType>(Ty);
5887 StringRef Name = FD->getName();
5888 return EmitNounwindRuntimeCall(CGM.CreateRuntimeFunction(FTy, Name), Ops);
5891 if (BuiltinID == ARM::BI__builtin_arm_mcrr ||
5892 BuiltinID == ARM::BI__builtin_arm_mcrr2) {
5895 switch (BuiltinID) {
5896 default: llvm_unreachable(
"unexpected builtin");
5897 case ARM::BI__builtin_arm_mcrr:
5898 F = CGM.getIntrinsic(Intrinsic::arm_mcrr);
5900 case ARM::BI__builtin_arm_mcrr2:
5901 F = CGM.getIntrinsic(Intrinsic::arm_mcrr2);
5917 Value *C1 = llvm::ConstantInt::get(Int64Ty, 32);
5918 Value *Rt = Builder.CreateTruncOrBitCast(RtAndRt2, Int32Ty);
5919 Value *Rt2 = Builder.CreateLShr(RtAndRt2, C1);
5920 Rt2 = Builder.CreateTruncOrBitCast(Rt2, Int32Ty);
5922 return Builder.CreateCall(F, {Coproc, Opc1, Rt, Rt2, CRm});
5925 if (BuiltinID == ARM::BI__builtin_arm_mrrc ||
5926 BuiltinID == ARM::BI__builtin_arm_mrrc2) {
5929 switch (BuiltinID) {
5930 default: llvm_unreachable(
"unexpected builtin");
5931 case ARM::BI__builtin_arm_mrrc:
5932 F = CGM.getIntrinsic(Intrinsic::arm_mrrc);
5934 case ARM::BI__builtin_arm_mrrc2:
5935 F = CGM.getIntrinsic(Intrinsic::arm_mrrc2);
5942 Value *RtAndRt2 = Builder.CreateCall(F, {Coproc, Opc1, CRm});
5947 Value *Rt = Builder.CreateExtractValue(RtAndRt2, 1);
5948 Value *Rt1 = Builder.CreateExtractValue(RtAndRt2, 0);
5949 Rt = Builder.CreateZExt(Rt, Int64Ty);
5950 Rt1 = Builder.CreateZExt(Rt1, Int64Ty);
5952 Value *ShiftCast = llvm::ConstantInt::get(Int64Ty, 32);
5953 RtAndRt2 = Builder.CreateShl(Rt, ShiftCast,
"shl",
true);
5954 RtAndRt2 = Builder.CreateOr(RtAndRt2, Rt1);
5956 return Builder.CreateBitCast(RtAndRt2, ConvertType(E->
getType()));
5959 if (BuiltinID == ARM::BI__builtin_arm_ldrexd ||
5960 ((BuiltinID == ARM::BI__builtin_arm_ldrex ||
5961 BuiltinID == ARM::BI__builtin_arm_ldaex) &&
5962 getContext().getTypeSize(E->
getType()) == 64) ||
5963 BuiltinID == ARM::BI__ldrexd) {
5966 switch (BuiltinID) {
5967 default: llvm_unreachable(
"unexpected builtin");
5968 case ARM::BI__builtin_arm_ldaex:
5969 F = CGM.getIntrinsic(Intrinsic::arm_ldaexd);
5971 case ARM::BI__builtin_arm_ldrexd:
5972 case ARM::BI__builtin_arm_ldrex:
5973 case ARM::BI__ldrexd:
5974 F = CGM.getIntrinsic(Intrinsic::arm_ldrexd);
5979 Value *Val = Builder.CreateCall(F, Builder.CreateBitCast(LdPtr, Int8PtrTy),
5982 Value *Val0 = Builder.CreateExtractValue(Val, 1);
5983 Value *Val1 = Builder.CreateExtractValue(Val, 0);
5984 Val0 = Builder.CreateZExt(Val0, Int64Ty);
5985 Val1 = Builder.CreateZExt(Val1, Int64Ty);
5987 Value *ShiftCst = llvm::ConstantInt::get(Int64Ty, 32);
5988 Val = Builder.CreateShl(Val0, ShiftCst,
"shl",
true );
5989 Val = Builder.CreateOr(Val, Val1);
5990 return Builder.CreateBitCast(Val, ConvertType(E->
getType()));
5993 if (BuiltinID == ARM::BI__builtin_arm_ldrex ||
5994 BuiltinID == ARM::BI__builtin_arm_ldaex) {
6000 getLLVMContext(), getContext().getTypeSize(Ty))->getPointerTo();
6001 LoadAddr = Builder.CreateBitCast(LoadAddr, PtrTy);
6003 Function *F = CGM.getIntrinsic(BuiltinID == ARM::BI__builtin_arm_ldaex
6004 ? Intrinsic::arm_ldaex
6005 : Intrinsic::arm_ldrex,
6007 Value *Val = Builder.CreateCall(F, LoadAddr,
"ldrex");
6009 if (RealResTy->isPointerTy())
6010 return Builder.CreateIntToPtr(Val, RealResTy);
6012 llvm::Type *IntResTy = llvm::IntegerType::get(
6013 getLLVMContext(), CGM.getDataLayout().getTypeSizeInBits(RealResTy));
6014 Val = Builder.CreateTruncOrBitCast(Val, IntResTy);
6015 return Builder.CreateBitCast(Val, RealResTy);
6019 if (BuiltinID == ARM::BI__builtin_arm_strexd ||
6020 ((BuiltinID == ARM::BI__builtin_arm_stlex ||
6021 BuiltinID == ARM::BI__builtin_arm_strex) &&
6023 Function *F = CGM.getIntrinsic(BuiltinID == ARM::BI__builtin_arm_stlex
6024 ? Intrinsic::arm_stlexd
6025 : Intrinsic::arm_strexd);
6026 llvm::Type *STy = llvm::StructType::get(Int32Ty, Int32Ty);
6030 Builder.CreateStore(Val, Tmp);
6032 Address LdPtr = Builder.CreateBitCast(Tmp,llvm::PointerType::getUnqual(STy));
6033 Val = Builder.CreateLoad(LdPtr);
6035 Value *Arg0 = Builder.CreateExtractValue(Val, 0);
6036 Value *Arg1 = Builder.CreateExtractValue(Val, 1);
6037 Value *StPtr = Builder.CreateBitCast(EmitScalarExpr(E->
getArg(1)), Int8PtrTy);
6038 return Builder.CreateCall(F, {Arg0, Arg1, StPtr},
"strexd");
6041 if (BuiltinID == ARM::BI__builtin_arm_strex ||
6042 BuiltinID == ARM::BI__builtin_arm_stlex) {
6047 llvm::Type *StoreTy = llvm::IntegerType::get(getLLVMContext(),
6048 getContext().getTypeSize(Ty));
6049 StoreAddr = Builder.CreateBitCast(StoreAddr, StoreTy->getPointerTo());
6051 if (StoreVal->getType()->isPointerTy())
6052 StoreVal = Builder.CreatePtrToInt(StoreVal, Int32Ty);
6056 CGM.getDataLayout().getTypeSizeInBits(StoreVal->getType()));
6057 StoreVal = Builder.CreateBitCast(StoreVal, IntTy);
6058 StoreVal = Builder.CreateZExtOrBitCast(StoreVal, Int32Ty);
6061 Function *F = CGM.getIntrinsic(BuiltinID == ARM::BI__builtin_arm_stlex
6062 ? Intrinsic::arm_stlex
6063 : Intrinsic::arm_strex,
6064 StoreAddr->getType());
6065 return Builder.CreateCall(F, {StoreVal, StoreAddr},
"strex");
6068 switch (BuiltinID) {
6069 case ARM::BI__iso_volatile_load8:
6070 case ARM::BI__iso_volatile_load16:
6071 case ARM::BI__iso_volatile_load32:
6072 case ARM::BI__iso_volatile_load64:
6073 return EmitISOVolatileLoad(E);
6074 case ARM::BI__iso_volatile_store8:
6075 case ARM::BI__iso_volatile_store16:
6076 case ARM::BI__iso_volatile_store32:
6077 case ARM::BI__iso_volatile_store64:
6078 return EmitISOVolatileStore(E);
6081 if (BuiltinID == ARM::BI__builtin_arm_clrex) {
6082 Function *F = CGM.getIntrinsic(Intrinsic::arm_clrex);
6083 return Builder.CreateCall(F);
6088 switch (BuiltinID) {
6089 case ARM::BI__builtin_arm_crc32b:
6090 CRCIntrinsicID = Intrinsic::arm_crc32b;
break;
6091 case ARM::BI__builtin_arm_crc32cb:
6092 CRCIntrinsicID = Intrinsic::arm_crc32cb;
break;
6093 case ARM::BI__builtin_arm_crc32h:
6094 CRCIntrinsicID = Intrinsic::arm_crc32h;
break;
6095 case ARM::BI__builtin_arm_crc32ch:
6096 CRCIntrinsicID = Intrinsic::arm_crc32ch;
break;
6097 case ARM::BI__builtin_arm_crc32w:
6098 case ARM::BI__builtin_arm_crc32d:
6099 CRCIntrinsicID = Intrinsic::arm_crc32w;
break;
6100 case ARM::BI__builtin_arm_crc32cw:
6101 case ARM::BI__builtin_arm_crc32cd:
6102 CRCIntrinsicID = Intrinsic::arm_crc32cw;
break;
6105 if (CRCIntrinsicID != Intrinsic::not_intrinsic) {
6111 if (BuiltinID == ARM::BI__builtin_arm_crc32d ||
6112 BuiltinID == ARM::BI__builtin_arm_crc32cd) {
6113 Value *C1 = llvm::ConstantInt::get(Int64Ty, 32);
6114 Value *Arg1a = Builder.CreateTruncOrBitCast(Arg1, Int32Ty);
6115 Value *Arg1b = Builder.CreateLShr(Arg1, C1);
6116 Arg1b = Builder.CreateTruncOrBitCast(Arg1b, Int32Ty);
6118 Function *F = CGM.getIntrinsic(CRCIntrinsicID);
6119 Value *Res = Builder.CreateCall(F, {Arg0, Arg1a});
6120 return Builder.CreateCall(F, {Res, Arg1b});
6122 Arg1 = Builder.CreateZExtOrBitCast(Arg1, Int32Ty);
6124 Function *F = CGM.getIntrinsic(CRCIntrinsicID);
6125 return Builder.CreateCall(F, {Arg0, Arg1});
6129 if (BuiltinID == ARM::BI__builtin_arm_rsr ||
6130 BuiltinID == ARM::BI__builtin_arm_rsr64 ||
6131 BuiltinID == ARM::BI__builtin_arm_rsrp ||
6132 BuiltinID == ARM::BI__builtin_arm_wsr ||
6133 BuiltinID == ARM::BI__builtin_arm_wsr64 ||
6134 BuiltinID == ARM::BI__builtin_arm_wsrp) {
6136 bool IsRead = BuiltinID == ARM::BI__builtin_arm_rsr ||
6137 BuiltinID == ARM::BI__builtin_arm_rsr64 ||
6138 BuiltinID == ARM::BI__builtin_arm_rsrp;
6140 bool IsPointerBuiltin = BuiltinID == ARM::BI__builtin_arm_rsrp ||
6141 BuiltinID == ARM::BI__builtin_arm_wsrp;
6143 bool Is64Bit = BuiltinID == ARM::BI__builtin_arm_rsr64 ||
6144 BuiltinID == ARM::BI__builtin_arm_wsr64;
6148 if (IsPointerBuiltin) {
6149 ValueType = VoidPtrTy;
6150 RegisterType = Int32Ty;
6151 }
else if (Is64Bit) {
6152 ValueType = RegisterType = Int64Ty;
6154 ValueType = RegisterType = Int32Ty;
6162 unsigned ICEArguments = 0;
6164 getContext().GetBuiltinType(BuiltinID, Error, &ICEArguments);
6167 auto getAlignmentValue32 = [&](
Address addr) ->
Value* {
6168 return Builder.getInt32(addr.getAlignment().getQuantity());
6175 unsigned NumArgs = E->
getNumArgs() - (HasExtraArg ? 1 : 0);
6176 for (
unsigned i = 0, e = NumArgs; i != e; i++) {
6178 switch (BuiltinID) {
6179 case NEON::BI__builtin_neon_vld1_v:
6180 case NEON::BI__builtin_neon_vld1q_v:
6181 case NEON::BI__builtin_neon_vld1q_lane_v:
6182 case NEON::BI__builtin_neon_vld1_lane_v:
6183 case NEON::BI__builtin_neon_vld1_dup_v:
6184 case NEON::BI__builtin_neon_vld1q_dup_v:
6185 case NEON::BI__builtin_neon_vst1_v:
6186 case NEON::BI__builtin_neon_vst1q_v:
6187 case NEON::BI__builtin_neon_vst1q_lane_v:
6188 case NEON::BI__builtin_neon_vst1_lane_v:
6189 case NEON::BI__builtin_neon_vst2_v:
6190 case NEON::BI__builtin_neon_vst2q_v:
6191 case NEON::BI__builtin_neon_vst2_lane_v:
6192 case NEON::BI__builtin_neon_vst2q_lane_v:
6193 case NEON::BI__builtin_neon_vst3_v:
6194 case NEON::BI__builtin_neon_vst3q_v:
6195 case NEON::BI__builtin_neon_vst3_lane_v:
6196 case NEON::BI__builtin_neon_vst3q_lane_v:
6197 case NEON::BI__builtin_neon_vst4_v:
6198 case NEON::BI__builtin_neon_vst4q_v:
6199 case NEON::BI__builtin_neon_vst4_lane_v:
6200 case NEON::BI__builtin_neon_vst4q_lane_v:
6203 PtrOp0 = EmitPointerWithAlignment(E->
getArg(0));
6209 switch (BuiltinID) {
6210 case NEON::BI__builtin_neon_vld2_v:
6211 case NEON::BI__builtin_neon_vld2q_v:
6212 case NEON::BI__builtin_neon_vld3_v:
6213 case NEON::BI__builtin_neon_vld3q_v:
6214 case NEON::BI__builtin_neon_vld4_v:
6215 case NEON::BI__builtin_neon_vld4q_v:
6216 case NEON::BI__builtin_neon_vld2_lane_v:
6217 case NEON::BI__builtin_neon_vld2q_lane_v:
6218 case NEON::BI__builtin_neon_vld3_lane_v:
6219 case NEON::BI__builtin_neon_vld3q_lane_v:
6220 case NEON::BI__builtin_neon_vld4_lane_v:
6221 case NEON::BI__builtin_neon_vld4q_lane_v:
6222 case NEON::BI__builtin_neon_vld2_dup_v:
6223 case NEON::BI__builtin_neon_vld2q_dup_v:
6224 case NEON::BI__builtin_neon_vld3_dup_v:
6225 case NEON::BI__builtin_neon_vld3q_dup_v:
6226 case NEON::BI__builtin_neon_vld4_dup_v:
6227 case NEON::BI__builtin_neon_vld4q_dup_v:
6230 PtrOp1 = EmitPointerWithAlignment(E->
getArg(1));
6236 if ((ICEArguments & (1 << i)) == 0) {
6237 Ops.push_back(EmitScalarExpr(E->
getArg(i)));
6241 llvm::APSInt Result;
6243 assert(IsConst &&
"Constant arg isn't actually constant?"); (void)IsConst;
6244 Ops.push_back(llvm::ConstantInt::get(getLLVMContext(), Result));
6248 switch (BuiltinID) {
6251 case NEON::BI__builtin_neon_vget_lane_i8:
6252 case NEON::BI__builtin_neon_vget_lane_i16:
6253 case NEON::BI__builtin_neon_vget_lane_i32:
6254 case NEON::BI__builtin_neon_vget_lane_i64:
6255 case NEON::BI__builtin_neon_vget_lane_f32:
6256 case NEON::BI__builtin_neon_vgetq_lane_i8:
6257 case NEON::BI__builtin_neon_vgetq_lane_i16:
6258 case NEON::BI__builtin_neon_vgetq_lane_i32:
6259 case NEON::BI__builtin_neon_vgetq_lane_i64:
6260 case NEON::BI__builtin_neon_vgetq_lane_f32:
6261 return Builder.CreateExtractElement(Ops[0], Ops[1],
"vget_lane");
6263 case NEON::BI__builtin_neon_vrndns_f32: {
6266 Function *F = CGM.getIntrinsic(Intrinsic::arm_neon_vrintn, Tys);
6267 return Builder.CreateCall(F, {Arg},
"vrndn"); }
6269 case NEON::BI__builtin_neon_vset_lane_i8:
6270 case NEON::BI__builtin_neon_vset_lane_i16:
6271 case NEON::BI__builtin_neon_vset_lane_i32:
6272 case NEON::BI__builtin_neon_vset_lane_i64:
6273 case NEON::BI__builtin_neon_vset_lane_f32:
6274 case NEON::BI__builtin_neon_vsetq_lane_i8:
6275 case NEON::BI__builtin_neon_vsetq_lane_i16:
6276 case NEON::BI__builtin_neon_vsetq_lane_i32:
6277 case NEON::BI__builtin_neon_vsetq_lane_i64:
6278 case NEON::BI__builtin_neon_vsetq_lane_f32:
6279 return Builder.CreateInsertElement(Ops[1], Ops[0], Ops[2],
"vset_lane");
6281 case NEON::BI__builtin_neon_vsha1h_u32:
6282 return EmitNeonCall(CGM.getIntrinsic(Intrinsic::arm_neon_sha1h), Ops,
6284 case NEON::BI__builtin_neon_vsha1cq_u32:
6285 return EmitNeonCall(CGM.getIntrinsic(Intrinsic::arm_neon_sha1c), Ops,
6287 case NEON::BI__builtin_neon_vsha1pq_u32:
6288 return EmitNeonCall(CGM.getIntrinsic(Intrinsic::arm_neon_sha1p), Ops,
6290 case NEON::BI__builtin_neon_vsha1mq_u32:
6291 return EmitNeonCall(CGM.getIntrinsic(Intrinsic::arm_neon_sha1m), Ops,
6296 case ARM::BI_MoveToCoprocessor:
6297 case ARM::BI_MoveToCoprocessor2: {
6298 Function *F = CGM.getIntrinsic(BuiltinID == ARM::BI_MoveToCoprocessor ?
6299 Intrinsic::arm_mcr : Intrinsic::arm_mcr2);
6300 return Builder.CreateCall(F, {Ops[1], Ops[2], Ops[0],
6301 Ops[3], Ops[4], Ops[5]});
6303 case ARM::BI_BitScanForward:
6304 case ARM::BI_BitScanForward64:
6305 return EmitMSVCBuiltinExpr(MSVCIntrin::_BitScanForward, E);
6306 case ARM::BI_BitScanReverse:
6307 case ARM::BI_BitScanReverse64:
6308 return EmitMSVCBuiltinExpr(MSVCIntrin::_BitScanReverse, E);
6310 case ARM::BI_InterlockedAnd64:
6311 return EmitMSVCBuiltinExpr(MSVCIntrin::_InterlockedAnd, E);
6312 case ARM::BI_InterlockedExchange64:
6313 return EmitMSVCBuiltinExpr(MSVCIntrin::_InterlockedExchange, E);
6314 case ARM::BI_InterlockedExchangeAdd64:
6315 return EmitMSVCBuiltinExpr(MSVCIntrin::_InterlockedExchangeAdd, E);
6316 case ARM::BI_InterlockedExchangeSub64:
6317 return EmitMSVCBuiltinExpr(MSVCIntrin::_InterlockedExchangeSub, E);
6318 case ARM::BI_InterlockedOr64:
6319 return EmitMSVCBuiltinExpr(MSVCIntrin::_InterlockedOr, E);
6320 case ARM::BI_InterlockedXor64:
6321 return EmitMSVCBuiltinExpr(MSVCIntrin::_InterlockedXor, E);
6322 case ARM::BI_InterlockedDecrement64:
6323 return EmitMSVCBuiltinExpr(MSVCIntrin::_InterlockedDecrement, E);
6324 case ARM::BI_InterlockedIncrement64:
6325 return EmitMSVCBuiltinExpr(MSVCIntrin::_InterlockedIncrement, E);
6326 case ARM::BI_InterlockedExchangeAdd8_acq:
6327 case ARM::BI_InterlockedExchangeAdd16_acq:
6328 case ARM::BI_InterlockedExchangeAdd_acq:
6329 case ARM::BI_InterlockedExchangeAdd64_acq:
6330 return EmitMSVCBuiltinExpr(MSVCIntrin::_InterlockedExchangeAdd_acq, E);
6331 case ARM::BI_InterlockedExchangeAdd8_rel:
6332 case ARM::BI_InterlockedExchangeAdd16_rel:
6333 case ARM::BI_InterlockedExchangeAdd_rel:
6334 case ARM::BI_InterlockedExchangeAdd64_rel:
6335 return EmitMSVCBuiltinExpr(MSVCIntrin::_InterlockedExchangeAdd_rel, E);
6336 case ARM::BI_InterlockedExchangeAdd8_nf:
6337 case ARM::BI_InterlockedExchangeAdd16_nf:
6338 case ARM::BI_InterlockedExchangeAdd_nf:
6339 case ARM::BI_InterlockedExchangeAdd64_nf:
6340 return EmitMSVCBuiltinExpr(MSVCIntrin::_InterlockedExchangeAdd_nf, E);
6341 case ARM::BI_InterlockedExchange8_acq:
6342 case ARM::BI_InterlockedExchange16_acq:
6343 case ARM::BI_InterlockedExchange_acq:
6344 case ARM::BI_InterlockedExchange64_acq:
6345 return EmitMSVCBuiltinExpr(MSVCIntrin::_InterlockedExchange_acq, E);
6346 case ARM::BI_InterlockedExchange8_rel:
6347 case ARM::BI_InterlockedExchange16_rel:
6348 case ARM::BI_InterlockedExchange_rel:
6349 case ARM::BI_InterlockedExchange64_rel:
6350 return EmitMSVCBuiltinExpr(MSVCIntrin::_InterlockedExchange_rel, E);
6351 case ARM::BI_InterlockedExchange8_nf:
6352 case ARM::BI_InterlockedExchange16_nf:
6353 case ARM::BI_InterlockedExchange_nf:
6354 case ARM::BI_InterlockedExchange64_nf:
6355 return EmitMSVCBuiltinExpr(MSVCIntrin::_InterlockedExchange_nf, E);
6356 case ARM::BI_InterlockedCompareExchange8_acq:
6357 case ARM::BI_InterlockedCompareExchange16_acq:
6358 case ARM::BI_InterlockedCompareExchange_acq:
6359 case ARM::BI_InterlockedCompareExchange64_acq:
6360 return EmitMSVCBuiltinExpr(MSVCIntrin::_InterlockedCompareExchange_acq, E);
6361 case ARM::BI_InterlockedCompareExchange8_rel:
6362 case ARM::BI_InterlockedCompareExchange16_rel:
6363 case ARM::BI_InterlockedCompareExchange_rel:
6364 case ARM::BI_InterlockedCompareExchange64_rel:
6365 return EmitMSVCBuiltinExpr(MSVCIntrin::_InterlockedCompareExchange_rel, E);
6366 case ARM::BI_InterlockedCompareExchange8_nf:
6367 case ARM::BI_InterlockedCompareExchange16_nf:
6368 case ARM::BI_InterlockedCompareExchange_nf:
6369 case ARM::BI_InterlockedCompareExchange64_nf:
6370 return EmitMSVCBuiltinExpr(MSVCIntrin::_InterlockedCompareExchange_nf, E);
6371 case ARM::BI_InterlockedOr8_acq:
6372 case ARM::BI_InterlockedOr16_acq:
6373 case ARM::BI_InterlockedOr_acq:
6374 case ARM::BI_InterlockedOr64_acq:
6375 return EmitMSVCBuiltinExpr(MSVCIntrin::_InterlockedOr_acq, E);
6376 case ARM::BI_InterlockedOr8_rel:
6377 case ARM::BI_InterlockedOr16_rel:
6378 case ARM::BI_InterlockedOr_rel:
6379 case ARM::BI_InterlockedOr64_rel:
6380 return EmitMSVCBuiltinExpr(MSVCIntrin::_InterlockedOr_rel, E);
6381 case ARM::BI_InterlockedOr8_nf:
6382 case ARM::BI_InterlockedOr16_nf:
6383 case ARM::BI_InterlockedOr_nf:
6384 case ARM::BI_InterlockedOr64_nf:
6385 return EmitMSVCBuiltinExpr(MSVCIntrin::_InterlockedOr_nf, E);
6386 case ARM::BI_InterlockedXor8_acq:
6387 case ARM::BI_InterlockedXor16_acq:
6388 case ARM::BI_InterlockedXor_acq:
6389 case ARM::BI_InterlockedXor64_acq:
6390 return EmitMSVCBuiltinExpr(MSVCIntrin::_InterlockedXor_acq, E);
6391 case ARM::BI_InterlockedXor8_rel:
6392 case ARM::BI_InterlockedXor16_rel:
6393 case ARM::BI_InterlockedXor_rel:
6394 case ARM::BI_InterlockedXor64_rel:
6395 return EmitMSVCBuiltinExpr(MSVCIntrin::_InterlockedXor_rel, E);
6396 case ARM::BI_InterlockedXor8_nf:
6397 case ARM::BI_InterlockedXor16_nf:
6398 case ARM::BI_InterlockedXor_nf:
6399 case ARM::BI_InterlockedXor64_nf:
6400 return EmitMSVCBuiltinExpr(MSVCIntrin::_InterlockedXor_nf, E);
6401 case ARM::BI_InterlockedAnd8_acq:
6402 case ARM::BI_InterlockedAnd16_acq:
6403 case ARM::BI_InterlockedAnd_acq:
6404 case ARM::BI_InterlockedAnd64_acq:
6405 return EmitMSVCBuiltinExpr(MSVCIntrin::_InterlockedAnd_acq, E);
6406 case ARM::BI_InterlockedAnd8_rel:
6407 case ARM::BI_InterlockedAnd16_rel:
6408 case ARM::BI_InterlockedAnd_rel:
6409 case ARM::BI_InterlockedAnd64_rel:
6410 return EmitMSVCBuiltinExpr(MSVCIntrin::_InterlockedAnd_rel, E);
6411 case ARM::BI_InterlockedAnd8_nf:
6412 case ARM::BI_InterlockedAnd16_nf:
6413 case ARM::BI_InterlockedAnd_nf:
6414 case ARM::BI_InterlockedAnd64_nf:
6415 return EmitMSVCBuiltinExpr(MSVCIntrin::_InterlockedAnd_nf, E);
6416 case ARM::BI_InterlockedIncrement16_acq:
6417 case ARM::BI_InterlockedIncrement_acq:
6418 case ARM::BI_InterlockedIncrement64_acq:
6419 return EmitMSVCBuiltinExpr(MSVCIntrin::_InterlockedIncrement_acq, E);
6420 case ARM::BI_InterlockedIncrement16_rel:
6421 case ARM::BI_InterlockedIncrement_rel:
6422 case ARM::BI_InterlockedIncrement64_rel:
6423 return EmitMSVCBuiltinExpr(MSVCIntrin::_InterlockedIncrement_rel, E);
6424 case ARM::BI_InterlockedIncrement16_nf:
6425 case ARM::BI_InterlockedIncrement_nf:
6426 case ARM::BI_InterlockedIncrement64_nf:
6427 return EmitMSVCBuiltinExpr(MSVCIntrin::_InterlockedIncrement_nf, E);
6428 case ARM::BI_InterlockedDecrement16_acq:
6429 case ARM::BI_InterlockedDecrement_acq:
6430 case ARM::BI_InterlockedDecrement64_acq:
6431 return EmitMSVCBuiltinExpr(MSVCIntrin::_InterlockedDecrement_acq, E);
6432 case ARM::BI_InterlockedDecrement16_rel:
6433 case ARM::BI_InterlockedDecrement_rel:
6434 case ARM::BI_InterlockedDecrement64_rel:
6435 return EmitMSVCBuiltinExpr(MSVCIntrin::_InterlockedDecrement_rel, E);
6436 case ARM::BI_InterlockedDecrement16_nf:
6437 case ARM::BI_InterlockedDecrement_nf:
6438 case ARM::BI_InterlockedDecrement64_nf:
6439 return EmitMSVCBuiltinExpr(MSVCIntrin::_InterlockedDecrement_nf, E);
6443 assert(HasExtraArg);
6444 llvm::APSInt Result;
6449 if (BuiltinID == ARM::BI__builtin_arm_vcvtr_f ||
6450 BuiltinID == ARM::BI__builtin_arm_vcvtr_d) {
6453 if (BuiltinID == ARM::BI__builtin_arm_vcvtr_f)
6459 bool usgn = Result.getZExtValue() == 1;
6460 unsigned Int = usgn ? Intrinsic::arm_vcvtru : Intrinsic::arm_vcvtr;
6463 Function *F = CGM.getIntrinsic(Int, Ty);
6464 return Builder.CreateCall(F, Ops,
"vcvtr");
6469 bool usgn =
Type.isUnsigned();
6470 bool rightShift =
false;
6473 getTarget().hasLegalHalfType());
6480 auto IntrinsicMap = makeArrayRef(ARMSIMDIntrinsicMap);
6482 IntrinsicMap, BuiltinID, NEONSIMDIntrinsicsProvenSorted);
6484 return EmitCommonNeonBuiltinExpr(
6485 Builtin->BuiltinID, Builtin->LLVMIntrinsic, Builtin->AltLLVMIntrinsic,
6486 Builtin->NameHint, Builtin->TypeModifier, E, Ops, PtrOp0, PtrOp1, Arch);
6489 switch (BuiltinID) {
6490 default:
return nullptr;
6491 case NEON::BI__builtin_neon_vld1q_lane_v:
6494 if (VTy->getElementType()->isIntegerTy(64)) {
6496 Ops[1] = Builder.CreateBitCast(Ops[1], Ty);
6497 uint32_t Lane = cast<ConstantInt>(Ops[2])->getZExtValue();
6498 Value *SV = llvm::ConstantVector::get(ConstantInt::get(Int32Ty, 1-Lane));
6499 Ops[1] = Builder.CreateShuffleVector(Ops[1], Ops[1], SV);
6501 Ty = llvm::VectorType::get(VTy->getElementType(), 1);
6503 Function *F = CGM.getIntrinsic(Intrinsic::arm_neon_vld1, Tys);
6504 Value *Align = getAlignmentValue32(PtrOp0);
6505 Value *Ld = Builder.CreateCall(F, {Ops[0], Align});
6507 uint32_t Indices[] = {1 - Lane, Lane};
6508 SV = llvm::ConstantDataVector::get(getLLVMContext(), Indices);
6509 return Builder.CreateShuffleVector(Ops[1], Ld, SV,
"vld1q_lane");
6512 case NEON::BI__builtin_neon_vld1_lane_v: {
6513 Ops[1] = Builder.CreateBitCast(Ops[1], Ty);
6514 PtrOp0 = Builder.CreateElementBitCast(PtrOp0, VTy->
getElementType());
6515 Value *Ld = Builder.CreateLoad(PtrOp0);
6516 return Builder.CreateInsertElement(Ops[1], Ld, Ops[2],
"vld1_lane");
6518 case NEON::BI__builtin_neon_vqrshrn_n_v:
6520 usgn ? Intrinsic::arm_neon_vqrshiftnu : Intrinsic::arm_neon_vqrshiftns;
6521 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops,
"vqrshrn_n",
6523 case NEON::BI__builtin_neon_vqrshrun_n_v:
6524 return EmitNeonCall(CGM.getIntrinsic(Intrinsic::arm_neon_vqrshiftnsu, Ty),
6525 Ops,
"vqrshrun_n", 1,
true);
6526 case NEON::BI__builtin_neon_vqshrn_n_v:
6527 Int = usgn ? Intrinsic::arm_neon_vqshiftnu : Intrinsic::arm_neon_vqshiftns;
6528 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops,
"vqshrn_n",
6530 case NEON::BI__builtin_neon_vqshrun_n_v:
6531 return EmitNeonCall(CGM.getIntrinsic(Intrinsic::arm_neon_vqshiftnsu, Ty),
6532 Ops,
"vqshrun_n", 1,
true);
6533 case NEON::BI__builtin_neon_vrecpe_v:
6534 case NEON::BI__builtin_neon_vrecpeq_v:
6535 return EmitNeonCall(CGM.getIntrinsic(Intrinsic::arm_neon_vrecpe, Ty),
6537 case NEON::BI__builtin_neon_vrshrn_n_v:
6538 return EmitNeonCall(CGM.getIntrinsic(Intrinsic::arm_neon_vrshiftn, Ty),
6539 Ops,
"vrshrn_n", 1,
true);
6540 case NEON::BI__builtin_neon_vrsra_n_v:
6541 case NEON::BI__builtin_neon_vrsraq_n_v:
6542 Ops[0] = Builder.CreateBitCast(Ops[0], Ty);
6543 Ops[1] = Builder.CreateBitCast(Ops[1], Ty);
6544 Ops[2] = EmitNeonShiftVector(Ops[2], Ty,
true);
6545 Int = usgn ? Intrinsic::arm_neon_vrshiftu : Intrinsic::arm_neon_vrshifts;
6546 Ops[1] = Builder.CreateCall(CGM.getIntrinsic(Int, Ty), {Ops[1], Ops[2]});
6547 return Builder.CreateAdd(Ops[0], Ops[1],
"vrsra_n");
6548 case NEON::BI__builtin_neon_vsri_n_v:
6549 case NEON::BI__builtin_neon_vsriq_n_v:
6552 case NEON::BI__builtin_neon_vsli_n_v:
6553 case NEON::BI__builtin_neon_vsliq_n_v:
6554 Ops[2] = EmitNeonShiftVector(Ops[2], Ty, rightShift);
6555 return EmitNeonCall(CGM.getIntrinsic(Intrinsic::arm_neon_vshiftins, Ty),
6557 case NEON::BI__builtin_neon_vsra_n_v:
6558 case NEON::BI__builtin_neon_vsraq_n_v:
6559 Ops[0] = Builder.CreateBitCast(Ops[0], Ty);
6560 Ops[1] = EmitNeonRShiftImm(Ops[1], Ops[2], Ty, usgn,
"vsra_n");
6561 return Builder.CreateAdd(Ops[0], Ops[1]);
6562 case NEON::BI__builtin_neon_vst1q_lane_v:
6565 if (VTy->getElementType()->isIntegerTy(64)) {
6566 Ops[1] = Builder.CreateBitCast(Ops[1], Ty);
6567 Value *SV = llvm::ConstantVector::get(cast<llvm::Constant>(Ops[2]));
6568 Ops[1] = Builder.CreateShuffleVector(Ops[1], Ops[1], SV);
6569 Ops[2] = getAlignmentValue32(PtrOp0);
6570 llvm::Type *Tys[] = {Int8PtrTy, Ops[1]->getType()};
6571 return Builder.CreateCall(CGM.getIntrinsic(Intrinsic::arm_neon_vst1,
6575 case NEON::BI__builtin_neon_vst1_lane_v: {
6576 Ops[1] = Builder.CreateBitCast(Ops[1], Ty);
6577 Ops[1] = Builder.CreateExtractElement(Ops[1], Ops[2]);
6578 Ty = llvm::PointerType::getUnqual(Ops[1]->getType());
6579 auto St = Builder.CreateStore(Ops[1], Builder.CreateBitCast(PtrOp0, Ty));
6582 case NEON::BI__builtin_neon_vtbl1_v:
6583 return EmitNeonCall(CGM.getIntrinsic(Intrinsic::arm_neon_vtbl1),
6585 case NEON::BI__builtin_neon_vtbl2_v:
6586 return EmitNeonCall(CGM.getIntrinsic(Intrinsic::arm_neon_vtbl2),
6588 case NEON::BI__builtin_neon_vtbl3_v:
6589 return EmitNeonCall(CGM.getIntrinsic(Intrinsic::arm_neon_vtbl3),
6591 case NEON::BI__builtin_neon_vtbl4_v:
6592 return EmitNeonCall(CGM.getIntrinsic(Intrinsic::arm_neon_vtbl4),
6594 case NEON::BI__builtin_neon_vtbx1_v:
6595 return EmitNeonCall(CGM.getIntrinsic(Intrinsic::arm_neon_vtbx1),
6597 case NEON::BI__builtin_neon_vtbx2_v:
6598 return EmitNeonCall(CGM.getIntrinsic(Intrinsic::arm_neon_vtbx2),
6600 case NEON::BI__builtin_neon_vtbx3_v:
6601 return EmitNeonCall(CGM.getIntrinsic(Intrinsic::arm_neon_vtbx3),
6603 case NEON::BI__builtin_neon_vtbx4_v:
6604 return EmitNeonCall(CGM.getIntrinsic(Intrinsic::arm_neon_vtbx4),
6612 llvm::Triple::ArchType Arch) {
6613 unsigned int Int = 0;
6614 const char *s =
nullptr;
6616 switch (BuiltinID) {
6619 case NEON::BI__builtin_neon_vtbl1_v:
6620 case NEON::BI__builtin_neon_vqtbl1_v:
6621 case NEON::BI__builtin_neon_vqtbl1q_v:
6622 case NEON::BI__builtin_neon_vtbl2_v:
6623 case NEON::BI__builtin_neon_vqtbl2_v:
6624 case NEON::BI__builtin_neon_vqtbl2q_v:
6625 case NEON::BI__builtin_neon_vtbl3_v:
6626 case NEON::BI__builtin_neon_vqtbl3_v:
6627 case NEON::BI__builtin_neon_vqtbl3q_v:
6628 case NEON::BI__builtin_neon_vtbl4_v:
6629 case NEON::BI__builtin_neon_vqtbl4_v:
6630 case NEON::BI__builtin_neon_vqtbl4q_v:
6632 case NEON::BI__builtin_neon_vtbx1_v:
6633 case NEON::BI__builtin_neon_vqtbx1_v:
6634 case NEON::BI__builtin_neon_vqtbx1q_v:
6635 case NEON::BI__builtin_neon_vtbx2_v:
6636 case NEON::BI__builtin_neon_vqtbx2_v:
6637 case NEON::BI__builtin_neon_vqtbx2q_v:
6638 case NEON::BI__builtin_neon_vtbx3_v:
6639 case NEON::BI__builtin_neon_vqtbx3_v:
6640 case NEON::BI__builtin_neon_vqtbx3q_v:
6641 case NEON::BI__builtin_neon_vtbx4_v:
6642 case NEON::BI__builtin_neon_vqtbx4_v:
6643 case NEON::BI__builtin_neon_vqtbx4q_v:
6650 llvm::APSInt Result;
6665 switch (BuiltinID) {
6666 case NEON::BI__builtin_neon_vtbl1_v: {
6668 Ops[1], Ty, Intrinsic::aarch64_neon_tbl1,
6671 case NEON::BI__builtin_neon_vtbl2_v: {
6673 Ops[2], Ty, Intrinsic::aarch64_neon_tbl1,
6676 case NEON::BI__builtin_neon_vtbl3_v: {
6678 Ops[3], Ty, Intrinsic::aarch64_neon_tbl2,
6681 case NEON::BI__builtin_neon_vtbl4_v: {
6683 Ops[4], Ty, Intrinsic::aarch64_neon_tbl2,
6686 case NEON::BI__builtin_neon_vtbx1_v: {
6689 Ty, Intrinsic::aarch64_neon_tbl1,
"vtbl1");
6691 llvm::Constant *EightV = ConstantInt::get(Ty, 8);
6692 Value *CmpRes = Builder.CreateICmp(ICmpInst::ICMP_UGE, Ops[2], EightV);
6693 CmpRes = Builder.CreateSExt(CmpRes, Ty);
6695 Value *EltsFromInput = Builder.CreateAnd(CmpRes, Ops[0]);
6696 Value *EltsFromTbl = Builder.CreateAnd(Builder.CreateNot(CmpRes), TblRes);
6697 return Builder.CreateOr(EltsFromInput, EltsFromTbl,
"vtbx");
6699 case NEON::BI__builtin_neon_vtbx2_v: {
6701 Ops[3], Ty, Intrinsic::aarch64_neon_tbx1,
6704 case NEON::BI__builtin_neon_vtbx3_v: {
6707 Ty, Intrinsic::aarch64_neon_tbl2,
"vtbl2");
6709 llvm::Constant *TwentyFourV = ConstantInt::get(Ty, 24);
6710 Value *CmpRes = Builder.CreateICmp(ICmpInst::ICMP_UGE, Ops[4],
6712 CmpRes = Builder.CreateSExt(CmpRes, Ty);
6714 Value *EltsFromInput = Builder.CreateAnd(CmpRes, Ops[0]);
6715 Value *EltsFromTbl = Builder.CreateAnd(Builder.CreateNot(CmpRes), TblRes);
6716 return Builder.CreateOr(EltsFromInput, EltsFromTbl,
"vtbx");
6718 case NEON::BI__builtin_neon_vtbx4_v: {
6720 Ops[5], Ty, Intrinsic::aarch64_neon_tbx2,
6723 case NEON::BI__builtin_neon_vqtbl1_v:
6724 case NEON::BI__builtin_neon_vqtbl1q_v:
6725 Int = Intrinsic::aarch64_neon_tbl1; s =
"vtbl1";
break;
6726 case NEON::BI__builtin_neon_vqtbl2_v:
6727 case NEON::BI__builtin_neon_vqtbl2q_v: {
6728 Int = Intrinsic::aarch64_neon_tbl2; s =
"vtbl2";
break;
6729 case NEON::BI__builtin_neon_vqtbl3_v:
6730 case NEON::BI__builtin_neon_vqtbl3q_v:
6731 Int = Intrinsic::aarch64_neon_tbl3; s =
"vtbl3";
break;
6732 case NEON::BI__builtin_neon_vqtbl4_v:
6733 case NEON::BI__builtin_neon_vqtbl4q_v:
6734 Int = Intrinsic::aarch64_neon_tbl4; s =
"vtbl4";
break;
6735 case NEON::BI__builtin_neon_vqtbx1_v:
6736 case NEON::BI__builtin_neon_vqtbx1q_v:
6737 Int = Intrinsic::aarch64_neon_tbx1; s =
"vtbx1";
break;
6738 case NEON::BI__builtin_neon_vqtbx2_v:
6739 case NEON::BI__builtin_neon_vqtbx2q_v:
6740 Int = Intrinsic::aarch64_neon_tbx2; s =
"vtbx2";
break;
6741 case NEON::BI__builtin_neon_vqtbx3_v:
6742 case NEON::BI__builtin_neon_vqtbx3q_v:
6743 Int = Intrinsic::aarch64_neon_tbx3; s =
"vtbx3";
break;
6744 case NEON::BI__builtin_neon_vqtbx4_v:
6745 case NEON::BI__builtin_neon_vqtbx4q_v:
6746 Int = Intrinsic::aarch64_neon_tbx4; s =
"vtbx4";
break;
6758 llvm::Type *VTy = llvm::VectorType::get(Int16Ty, 4);
6759 Op = Builder.CreateBitCast(Op, Int16Ty);
6760 Value *V = UndefValue::get(VTy);
6761 llvm::Constant *CI = ConstantInt::get(SizeTy, 0);
6762 Op = Builder.CreateInsertElement(V, Op, CI);
6768 llvm::Triple::ArchType Arch) {
6769 unsigned HintID =
static_cast<unsigned>(-1);
6770 switch (BuiltinID) {
6772 case AArch64::BI__builtin_arm_nop:
6775 case AArch64::BI__builtin_arm_yield:
6776 case AArch64::BI__yield:
6779 case AArch64::BI__builtin_arm_wfe:
6780 case AArch64::BI__wfe:
6783 case AArch64::BI__builtin_arm_wfi:
6784 case AArch64::BI__wfi:
6787 case AArch64::BI__builtin_arm_sev:
6788 case AArch64::BI__sev:
6791 case AArch64::BI__builtin_arm_sevl:
6792 case AArch64::BI__sevl:
6797 if (HintID != static_cast<unsigned>(-1)) {
6798 Function *F = CGM.getIntrinsic(Intrinsic::aarch64_hint);
6799 return Builder.CreateCall(F, llvm::ConstantInt::get(Int32Ty, HintID));
6802 if (BuiltinID == AArch64::BI__builtin_arm_prefetch) {
6806 Value *RetentionPolicy = EmitScalarExpr(E->
getArg(3));
6809 Value *Locality =
nullptr;
6810 if (cast<llvm::ConstantInt>(RetentionPolicy)->isZero()) {
6812 Locality = llvm::ConstantInt::get(Int32Ty,
6813 -cast<llvm::ConstantInt>(CacheLevel)->
getValue() + 3);
6816 Locality = llvm::ConstantInt::get(Int32Ty, 0);
6822 return Builder.CreateCall(F, {Address, RW, Locality, IsData});
6825 if (BuiltinID == AArch64::BI__builtin_arm_rbit) {
6826 assert((getContext().getTypeSize(E->
getType()) == 32) &&
6827 "rbit of unusual size!");
6829 return Builder.CreateCall(
6830 CGM.getIntrinsic(Intrinsic::bitreverse, Arg->getType()), Arg,
"rbit");
6832 if (BuiltinID == AArch64::BI__builtin_arm_rbit64) {
6833 assert((getContext().getTypeSize(E->
getType()) == 64) &&
6834 "rbit of unusual size!");
6836 return Builder.CreateCall(
6837 CGM.getIntrinsic(Intrinsic::bitreverse, Arg->getType()), Arg,
"rbit");
6840 if (BuiltinID == AArch64::BI__clear_cache) {
6841 assert(E->
getNumArgs() == 2 &&
"__clear_cache takes 2 arguments");
6844 for (
unsigned i = 0; i < 2; i++)
6845 Ops[i] = EmitScalarExpr(E->
getArg(i));
6846 llvm::Type *Ty = CGM.getTypes().ConvertType(FD->getType());
6847 llvm::FunctionType *FTy = cast<llvm::FunctionType>(Ty);
6848 StringRef Name = FD->getName();
6849 return EmitNounwindRuntimeCall(CGM.CreateRuntimeFunction(FTy, Name), Ops);
6852 if ((BuiltinID == AArch64::BI__builtin_arm_ldrex ||
6853 BuiltinID == AArch64::BI__builtin_arm_ldaex) &&
6854 getContext().getTypeSize(E->
getType()) == 128) {
6855 Function *F = CGM.getIntrinsic(BuiltinID == AArch64::BI__builtin_arm_ldaex
6856 ? Intrinsic::aarch64_ldaxp
6857 : Intrinsic::aarch64_ldxp);
6860 Value *Val = Builder.CreateCall(F, Builder.CreateBitCast(LdPtr, Int8PtrTy),
6863 Value *Val0 = Builder.CreateExtractValue(Val, 1);
6864 Value *Val1 = Builder.CreateExtractValue(Val, 0);
6865 llvm::Type *Int128Ty = llvm::IntegerType::get(getLLVMContext(), 128);
6866 Val0 = Builder.CreateZExt(Val0, Int128Ty);
6867 Val1 = Builder.CreateZExt(Val1, Int128Ty);
6869 Value *ShiftCst = llvm::ConstantInt::get(Int128Ty, 64);
6870 Val = Builder.CreateShl(Val0, ShiftCst,
"shl",
true );
6871 Val = Builder.CreateOr(Val, Val1);
6872 return Builder.CreateBitCast(Val, ConvertType(E->
getType()));
6873 }
else if (BuiltinID == AArch64::BI__builtin_arm_ldrex ||
6874 BuiltinID == AArch64::BI__builtin_arm_ldaex) {
6880 getLLVMContext(), getContext().getTypeSize(Ty))->getPointerTo();
6881 LoadAddr = Builder.CreateBitCast(LoadAddr, PtrTy);
6883 Function *F = CGM.getIntrinsic(BuiltinID == AArch64::BI__builtin_arm_ldaex
6884 ? Intrinsic::aarch64_ldaxr
6885 : Intrinsic::aarch64_ldxr,
6887 Value *Val = Builder.CreateCall(F, LoadAddr,
"ldxr");
6889 if (RealResTy->isPointerTy())
6890 return Builder.CreateIntToPtr(Val, RealResTy);
6892 llvm::Type *IntResTy = llvm::IntegerType::get(
6893 getLLVMContext(), CGM.getDataLayout().getTypeSizeInBits(RealResTy));
6894 Val = Builder.CreateTruncOrBitCast(Val, IntResTy);
6895 return Builder.CreateBitCast(Val, RealResTy);
6898 if ((BuiltinID == AArch64::BI__builtin_arm_strex ||
6899 BuiltinID == AArch64::BI__builtin_arm_stlex) &&
6901 Function *F = CGM.getIntrinsic(BuiltinID == AArch64::BI__builtin_arm_stlex
6902 ? Intrinsic::aarch64_stlxp
6903 : Intrinsic::aarch64_stxp);
6904 llvm::Type *STy = llvm::StructType::get(Int64Ty, Int64Ty);
6909 Tmp = Builder.CreateBitCast(Tmp, llvm::PointerType::getUnqual(STy));
6912 Value *Arg0 = Builder.CreateExtractValue(Val, 0);
6913 Value *Arg1 = Builder.CreateExtractValue(Val, 1);
6914 Value *StPtr = Builder.CreateBitCast(EmitScalarExpr(E->
getArg(1)),
6916 return Builder.CreateCall(F, {Arg0, Arg1, StPtr},
"stxp");
6919 if (BuiltinID == AArch64::BI__builtin_arm_strex ||
6920 BuiltinID == AArch64::BI__builtin_arm_stlex) {
6925 llvm::Type *StoreTy = llvm::IntegerType::get(getLLVMContext(),
6926 getContext().getTypeSize(Ty));
6927 StoreAddr = Builder.CreateBitCast(StoreAddr, StoreTy->getPointerTo());
6929 if (StoreVal->getType()->isPointerTy())
6930 StoreVal = Builder.CreatePtrToInt(StoreVal, Int64Ty);
6934 CGM.getDataLayout().getTypeSizeInBits(StoreVal->getType()));
6935 StoreVal = Builder.CreateBitCast(StoreVal, IntTy);
6936 StoreVal = Builder.CreateZExtOrBitCast(StoreVal, Int64Ty);
6939 Function *F = CGM.getIntrinsic(BuiltinID == AArch64::BI__builtin_arm_stlex
6940 ? Intrinsic::aarch64_stlxr
6941 : Intrinsic::aarch64_stxr,
6942 StoreAddr->getType());
6943 return Builder.CreateCall(F, {StoreVal, StoreAddr},
"stxr");
6946 if (BuiltinID == AArch64::BI__getReg) {
6949 llvm_unreachable(
"Sema will ensure that the parameter is constant");
6952 LLVMContext &Context = CGM.getLLVMContext();
6953 std::string Reg = Value == 31 ?
"sp" :
"x" + Value.toString(10);
6955 llvm::Metadata *Ops[] = {llvm::MDString::get(Context, Reg)};
6956 llvm::MDNode *RegName = llvm::MDNode::get(Context, Ops);
6957 llvm::Value *Metadata = llvm::MetadataAsValue::get(Context, RegName);
6960 CGM.getIntrinsic(llvm::Intrinsic::read_register, {Int64Ty});
6961 return Builder.CreateCall(F, Metadata);
6964 if (BuiltinID == AArch64::BI__builtin_arm_clrex) {
6965 Function *F = CGM.getIntrinsic(Intrinsic::aarch64_clrex);
6966 return Builder.CreateCall(F);
6969 if (BuiltinID == AArch64::BI_ReadWriteBarrier)
6970 return Builder.CreateFence(llvm::AtomicOrdering::SequentiallyConsistent,
6971 llvm::SyncScope::SingleThread);
6975 switch (BuiltinID) {
6976 case AArch64::BI__builtin_arm_crc32b:
6977 CRCIntrinsicID = Intrinsic::aarch64_crc32b;
break;
6978 case AArch64::BI__builtin_arm_crc32cb:
6979 CRCIntrinsicID = Intrinsic::aarch64_crc32cb;
break;
6980 case AArch64::BI__builtin_arm_crc32h:
6981 CRCIntrinsicID = Intrinsic::aarch64_crc32h;
break;
6982 case AArch64::BI__builtin_arm_crc32ch:
6983 CRCIntrinsicID = Intrinsic::aarch64_crc32ch;
break;
6984 case AArch64::BI__builtin_arm_crc32w:
6985 CRCIntrinsicID = Intrinsic::aarch64_crc32w;
break;
6986 case AArch64::BI__builtin_arm_crc32cw:
6987 CRCIntrinsicID = Intrinsic::aarch64_crc32cw;
break;
6988 case AArch64::BI__builtin_arm_crc32d:
6989 CRCIntrinsicID = Intrinsic::aarch64_crc32x;
break;
6990 case AArch64::BI__builtin_arm_crc32cd:
6991 CRCIntrinsicID = Intrinsic::aarch64_crc32cx;
break;
6994 if (CRCIntrinsicID != Intrinsic::not_intrinsic) {
6997 Function *F = CGM.getIntrinsic(CRCIntrinsicID);
6999 llvm::Type *DataTy = F->getFunctionType()->getParamType(1);
7000 Arg1 = Builder.CreateZExtOrBitCast(Arg1, DataTy);
7002 return Builder.CreateCall(F, {Arg0, Arg1});
7005 if (BuiltinID == AArch64::BI__builtin_arm_rsr ||
7006 BuiltinID == AArch64::BI__builtin_arm_rsr64 ||
7007 BuiltinID == AArch64::BI__builtin_arm_rsrp ||
7008 BuiltinID == AArch64::BI__builtin_arm_wsr ||
7009 BuiltinID == AArch64::BI__builtin_arm_wsr64 ||
7010 BuiltinID == AArch64::BI__builtin_arm_wsrp) {
7012 bool IsRead = BuiltinID == AArch64::BI__builtin_arm_rsr ||
7013 BuiltinID == AArch64::BI__builtin_arm_rsr64 ||
7014 BuiltinID == AArch64::BI__builtin_arm_rsrp;
7016 bool IsPointerBuiltin = BuiltinID == AArch64::BI__builtin_arm_rsrp ||
7017 BuiltinID == AArch64::BI__builtin_arm_wsrp;
7019 bool Is64Bit = BuiltinID != AArch64::BI__builtin_arm_rsr &&
7020 BuiltinID != AArch64::BI__builtin_arm_wsr;
7024 if (IsPointerBuiltin) {
7025 ValueType = VoidPtrTy;
7026 }
else if (Is64Bit) {
7027 ValueType = Int64Ty;
7029 ValueType = Int32Ty;
7035 if (BuiltinID == AArch64::BI_ReadStatusReg ||
7036 BuiltinID == AArch64::BI_WriteStatusReg) {
7037 LLVMContext &Context = CGM.getLLVMContext();
7042 std::string SysRegStr;
7043 llvm::raw_string_ostream(SysRegStr) <<
7044 ((1 << 1) | ((SysReg >> 14) & 1)) <<
":" <<
7045 ((SysReg >> 11) & 7) <<
":" <<
7046 ((SysReg >> 7) & 15) <<
":" <<
7047 ((SysReg >> 3) & 15) <<
":" <<
7050 llvm::Metadata *Ops[] = { llvm::MDString::get(Context, SysRegStr) };
7051 llvm::MDNode *RegName = llvm::MDNode::get(Context, Ops);
7052 llvm::Value *Metadata = llvm::MetadataAsValue::get(Context, RegName);
7058 if (BuiltinID == AArch64::BI_ReadStatusReg) {
7059 llvm::Value *F = CGM.getIntrinsic(llvm::Intrinsic::read_register, Types);
7060 llvm::Value *Call = Builder.CreateCall(F, Metadata);
7062 return Builder.CreateTrunc(Call, ValueType);
7065 llvm::Value *F = CGM.getIntrinsic(llvm::Intrinsic::write_register, Types);
7067 ArgValue = Builder.CreateZExt(ArgValue, RegisterType);
7069 return Builder.CreateCall(F, { Metadata, ArgValue });
7072 if (BuiltinID == AArch64::BI_AddressOfReturnAddress) {
7073 llvm::Value *F = CGM.getIntrinsic(Intrinsic::addressofreturnaddress);
7074 return Builder.CreateCall(F);
7079 unsigned ICEArguments = 0;
7081 getContext().GetBuiltinType(BuiltinID, Error, &ICEArguments);
7085 for (
unsigned i = 0, e = E->
getNumArgs() - 1; i != e; i++) {
7086 if ((ICEArguments & (1 << i)) == 0) {
7087 Ops.push_back(EmitScalarExpr(E->
getArg(i)));
7091 llvm::APSInt Result;
7093 assert(IsConst &&
"Constant arg isn't actually constant?");
7095 Ops.push_back(llvm::ConstantInt::get(getLLVMContext(), Result));
7099 auto SISDMap = makeArrayRef(AArch64SISDIntrinsicMap);
7101 SISDMap, BuiltinID, AArch64SISDIntrinsicsProvenSorted);
7106 assert(Result &&
"SISD intrinsic should have been handled");
7110 llvm::APSInt Result;
7118 bool quad = Type.
isQuad();
7121 switch (BuiltinID) {
7123 case NEON::BI__builtin_neon_vabsh_f16:
7124 Ops.push_back(EmitScalarExpr(E->
getArg(0)));
7125 return EmitNeonCall(CGM.getIntrinsic(
Intrinsic::fabs, HalfTy), Ops,
"vabs");
7126 case NEON::BI__builtin_neon_vldrq_p128: {
7127 llvm::Type *Int128Ty = llvm::Type::getIntNTy(getLLVMContext(), 128);
7128 llvm::Type *Int128PTy = llvm::PointerType::get(Int128Ty, 0);
7129 Value *Ptr = Builder.CreateBitCast(EmitScalarExpr(E->
getArg(0)), Int128PTy);
7130 return Builder.CreateAlignedLoad(Int128Ty, Ptr,
7133 case NEON::BI__builtin_neon_vstrq_p128: {
7134 llvm::Type *Int128PTy = llvm::Type::getIntNPtrTy(getLLVMContext(), 128);
7135 Value *Ptr = Builder.CreateBitCast(Ops[0], Int128PTy);
7136 return Builder.CreateDefaultAlignedStore(EmitScalarExpr(E->
getArg(1)), Ptr);
7138 case NEON::BI__builtin_neon_vcvts_u32_f32:
7139 case NEON::BI__builtin_neon_vcvtd_u64_f64:
7142 case NEON::BI__builtin_neon_vcvts_s32_f32:
7143 case NEON::BI__builtin_neon_vcvtd_s64_f64: {
7144 Ops.push_back(EmitScalarExpr(E->
getArg(0)));
7145 bool Is64 = Ops[0]->getType()->getPrimitiveSizeInBits() == 64;
7148 Ops[0] = Builder.CreateBitCast(Ops[0], FTy);
7150 return Builder.CreateFPToUI(Ops[0], InTy);
7151 return Builder.CreateFPToSI(Ops[0], InTy);
7153 case NEON::BI__builtin_neon_vcvts_f32_u32:
7154 case NEON::BI__builtin_neon_vcvtd_f64_u64:
7157 case NEON::BI__builtin_neon_vcvts_f32_s32:
7158 case NEON::BI__builtin_neon_vcvtd_f64_s64: {
7159 Ops.push_back(EmitScalarExpr(E->
getArg(0)));
7160 bool Is64 = Ops[0]->getType()->getPrimitiveSizeInBits() == 64;
7163 Ops[0] = Builder.CreateBitCast(Ops[0], InTy);
7165 return Builder.CreateUIToFP(Ops[0], FTy);
7166 return Builder.CreateSIToFP(Ops[0], FTy);
7168 case NEON::BI__builtin_neon_vcvth_f16_u16:
7169 case NEON::BI__builtin_neon_vcvth_f16_u32:
7170 case NEON::BI__builtin_neon_vcvth_f16_u64:
7173 case NEON::BI__builtin_neon_vcvth_f16_s16:
7174 case NEON::BI__builtin_neon_vcvth_f16_s32:
7175 case NEON::BI__builtin_neon_vcvth_f16_s64: {
7176 Ops.push_back(EmitScalarExpr(E->
getArg(0)));
7179 if (Ops[0]->getType()->getPrimitiveSizeInBits() == 64)
7181 else if (Ops[0]->getType()->getPrimitiveSizeInBits() == 32)
7185 Ops[0] = Builder.CreateBitCast(Ops[0], InTy);
7187 return Builder.CreateUIToFP(Ops[0], FTy);
7188 return Builder.CreateSIToFP(Ops[0], FTy);
7190 case NEON::BI__builtin_neon_vcvth_u16_f16:
7193 case NEON::BI__builtin_neon_vcvth_s16_f16: {
7194 Ops.push_back(EmitScalarExpr(E->
getArg(0)));
7195 Ops[0] = Builder.CreateBitCast(Ops[0], HalfTy);
7197 return Builder.CreateFPToUI(Ops[0], Int16Ty);
7198 return Builder.CreateFPToSI(Ops[0], Int16Ty);
7200 case NEON::BI__builtin_neon_vcvth_u32_f16:
7203 case NEON::BI__builtin_neon_vcvth_s32_f16: {
7204 Ops.push_back(EmitScalarExpr(E->
getArg(0)));
7205 Ops[0] = Builder.CreateBitCast(Ops[0], HalfTy);
7207 return Builder.CreateFPToUI(Ops[0], Int32Ty);
7208 return Builder.CreateFPToSI(Ops[0], Int32Ty);
7210 case NEON::BI__builtin_neon_vcvth_u64_f16:
7213 case NEON::BI__builtin_neon_vcvth_s64_f16: {
7214 Ops.push_back(EmitScalarExpr(E->
getArg(0)));
7215 Ops[0] = Builder.CreateBitCast(Ops[0], HalfTy);
7217 return Builder.CreateFPToUI(Ops[0], Int64Ty);
7218 return Builder.CreateFPToSI(Ops[0], Int64Ty);
7220 case NEON::BI__builtin_neon_vcvtah_u16_f16:
7221 case NEON::BI__builtin_neon_vcvtmh_u16_f16:
7222 case NEON::BI__builtin_neon_vcvtnh_u16_f16:
7223 case NEON::BI__builtin_neon_vcvtph_u16_f16:
7224 case NEON::BI__builtin_neon_vcvtah_s16_f16:
7225 case NEON::BI__builtin_neon_vcvtmh_s16_f16:
7226 case NEON::BI__builtin_neon_vcvtnh_s16_f16:
7227 case NEON::BI__builtin_neon_vcvtph_s16_f16: {
7232 Ops.push_back(EmitScalarExpr(E->
getArg(0)));
7233 switch (BuiltinID) {
7234 default: llvm_unreachable(
"missing builtin ID in switch!");
7235 case NEON::BI__builtin_neon_vcvtah_u16_f16:
7236 Int = Intrinsic::aarch64_neon_fcvtau;
break;
7237 case NEON::BI__builtin_neon_vcvtmh_u16_f16:
7238 Int = Intrinsic::aarch64_neon_fcvtmu;
break;
7239 case NEON::BI__builtin_neon_vcvtnh_u16_f16:
7240 Int = Intrinsic::aarch64_neon_fcvtnu;
break;
7241 case NEON::BI__builtin_neon_vcvtph_u16_f16:
7242 Int = Intrinsic::aarch64_neon_fcvtpu;
break;
7243 case NEON::BI__builtin_neon_vcvtah_s16_f16:
7244 Int = Intrinsic::aarch64_neon_fcvtas;
break;
7245 case NEON::BI__builtin_neon_vcvtmh_s16_f16:
7246 Int = Intrinsic::aarch64_neon_fcvtms;
break;
7247 case NEON::BI__builtin_neon_vcvtnh_s16_f16:
7248 Int = Intrinsic::aarch64_neon_fcvtns;
break;
7249 case NEON::BI__builtin_neon_vcvtph_s16_f16:
7250 Int = Intrinsic::aarch64_neon_fcvtps;
break;
7252 Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops,
"fcvt");
7253 return Builder.CreateTrunc(Ops[0], Int16Ty);
7255 case NEON::BI__builtin_neon_vcaleh_f16:
7256 case NEON::BI__builtin_neon_vcalth_f16:
7257 case NEON::BI__builtin_neon_vcageh_f16:
7258 case NEON::BI__builtin_neon_vcagth_f16: {
7263 Ops.push_back(EmitScalarExpr(E->
getArg(1)));
7264 switch (BuiltinID) {
7265 default: llvm_unreachable(
"missing builtin ID in switch!");
7266 case NEON::BI__builtin_neon_vcageh_f16:
7267 Int = Intrinsic::aarch64_neon_facge;
break;
7268 case NEON::BI__builtin_neon_vcagth_f16:
7269 Int = Intrinsic::aarch64_neon_facgt;
break;
7270 case NEON::BI__builtin_neon_vcaleh_f16:
7271 Int = Intrinsic::aarch64_neon_facge; std::swap(Ops[0], Ops[1]);
break;
7272 case NEON::BI__builtin_neon_vcalth_f16:
7273 Int = Intrinsic::aarch64_neon_facgt; std::swap(Ops[0], Ops[1]);
break;
7275 Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops,
"facg");
7276 return Builder.CreateTrunc(Ops[0], Int16Ty);
7278 case NEON::BI__builtin_neon_vcvth_n_s16_f16:
7279 case NEON::BI__builtin_neon_vcvth_n_u16_f16: {
7284 Ops.push_back(EmitScalarExpr(E->
getArg(1)));
7285 switch (BuiltinID) {
7286 default: llvm_unreachable(
"missing builtin ID in switch!");
7287 case NEON::BI__builtin_neon_vcvth_n_s16_f16:
7288 Int = Intrinsic::aarch64_neon_vcvtfp2fxs;
break;
7289 case NEON::BI__builtin_neon_vcvth_n_u16_f16:
7290 Int = Intrinsic::aarch64_neon_vcvtfp2fxu;
break;
7292 Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops,
"fcvth_n");
7293 return Builder.CreateTrunc(Ops[0], Int16Ty);
7295 case NEON::BI__builtin_neon_vcvth_n_f16_s16:
7296 case NEON::BI__builtin_neon_vcvth_n_f16_u16: {
7301 Ops.push_back(EmitScalarExpr(E->
getArg(1)));
7302 switch (BuiltinID) {
7303 default: llvm_unreachable(
"missing builtin ID in switch!");
7304 case NEON::BI__builtin_neon_vcvth_n_f16_s16:
7305 Int = Intrinsic::aarch64_neon_vcvtfxs2fp;
7306 Ops[0] = Builder.CreateSExt(Ops[0], InTy,
"sext");
7308 case NEON::BI__builtin_neon_vcvth_n_f16_u16:
7309 Int = Intrinsic::aarch64_neon_vcvtfxu2fp;
7310 Ops[0] = Builder.CreateZExt(Ops[0], InTy);
7313 return EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops,
"fcvth_n");
7315 case NEON::BI__builtin_neon_vpaddd_s64: {
7316 llvm::Type *Ty = llvm::VectorType::get(Int64Ty, 2);
7319 Vec = Builder.CreateBitCast(Vec, Ty,
"v2i64");
7320 llvm::Value *Idx0 = llvm::ConstantInt::get(SizeTy, 0);
7321 llvm::Value *Idx1 = llvm::ConstantInt::get(SizeTy, 1);
7322 Value *Op0 = Builder.CreateExtractElement(Vec, Idx0,
"lane0");
7323 Value *Op1 = Builder.CreateExtractElement(Vec, Idx1,
"lane1");
7325 return Builder.CreateAdd(Op0, Op1,
"vpaddd");
7327 case NEON::BI__builtin_neon_vpaddd_f64: {
7329 llvm::VectorType::get(DoubleTy, 2);
7332 Vec = Builder.CreateBitCast(Vec, Ty,
"v2f64");
7333 llvm::Value *Idx0 = llvm::ConstantInt::get(SizeTy, 0);
7334 llvm::Value *Idx1 = llvm::ConstantInt::get(SizeTy, 1);
7335 Value *Op0 = Builder.CreateExtractElement(Vec, Idx0,
"lane0");
7336 Value *Op1 = Builder.CreateExtractElement(Vec, Idx1,
"lane1");
7338 return Builder.CreateFAdd(Op0, Op1,
"vpaddd");
7340 case NEON::BI__builtin_neon_vpadds_f32: {
7342 llvm::VectorType::get(FloatTy, 2);
7345 Vec = Builder.CreateBitCast(Vec, Ty,
"v2f32");
7346 llvm::Value *Idx0 = llvm::ConstantInt::get(SizeTy, 0);
7347 llvm::Value *Idx1 = llvm::ConstantInt::get(SizeTy, 1);
7348 Value *Op0 = Builder.CreateExtractElement(Vec, Idx0,
"lane0");
7349 Value *Op1 = Builder.CreateExtractElement(Vec, Idx1,
"lane1");
7351 return Builder.CreateFAdd(Op0, Op1,
"vpaddd");
7353 case NEON::BI__builtin_neon_vceqzd_s64:
7354 case NEON::BI__builtin_neon_vceqzd_f64:
7355 case NEON::BI__builtin_neon_vceqzs_f32:
7356 case NEON::BI__builtin_neon_vceqzh_f16:
7357 Ops.push_back(EmitScalarExpr(E->
getArg(0)));
7358 return EmitAArch64CompareBuiltinExpr(
7360 ICmpInst::FCMP_OEQ, ICmpInst::ICMP_EQ,
"vceqz");
7361 case NEON::BI__builtin_neon_vcgezd_s64:
7362 case NEON::BI__builtin_neon_vcgezd_f64:
7363 case NEON::BI__builtin_neon_vcgezs_f32:
7364 case NEON::BI__builtin_neon_vcgezh_f16:
7365 Ops.push_back(EmitScalarExpr(E->
getArg(0)));
7366 return EmitAArch64CompareBuiltinExpr(
7368 ICmpInst::FCMP_OGE, ICmpInst::ICMP_SGE,
"vcgez");
7369 case NEON::BI__builtin_neon_vclezd_s64:
7370 case NEON::BI__builtin_neon_vclezd_f64:
7371 case NEON::BI__builtin_neon_vclezs_f32:
7372 case NEON::BI__builtin_neon_vclezh_f16:
7373 Ops.push_back(EmitScalarExpr(E->
getArg(0)));
7374 return EmitAArch64CompareBuiltinExpr(
7376 ICmpInst::FCMP_OLE, ICmpInst::ICMP_SLE,
"vclez");
7377 case NEON::BI__builtin_neon_vcgtzd_s64:
7378 case NEON::BI__builtin_neon_vcgtzd_f64:
7379 case NEON::BI__builtin_neon_vcgtzs_f32:
7380 case NEON::BI__builtin_neon_vcgtzh_f16:
7381 Ops.push_back(EmitScalarExpr(E->
getArg(0)));
7382 return EmitAArch64CompareBuiltinExpr(
7384 ICmpInst::FCMP_OGT, ICmpInst::ICMP_SGT,
"vcgtz");
7385 case NEON::BI__builtin_neon_vcltzd_s64:
7386 case NEON::BI__builtin_neon_vcltzd_f64:
7387 case NEON::BI__builtin_neon_vcltzs_f32:
7388 case NEON::BI__builtin_neon_vcltzh_f16:
7389 Ops.push_back(EmitScalarExpr(E->
getArg(0)));
7390 return EmitAArch64CompareBuiltinExpr(
7392 ICmpInst::FCMP_OLT, ICmpInst::ICMP_SLT,
"vcltz");
7394 case NEON::BI__builtin_neon_vceqzd_u64: {
7395 Ops.push_back(EmitScalarExpr(E->
getArg(0)));
7396 Ops[0] = Builder.CreateBitCast(Ops[0], Int64Ty);
7398 Builder.CreateICmpEQ(Ops[0], llvm::Constant::getNullValue(Int64Ty));
7399 return Builder.CreateSExt(Ops[0], Int64Ty,
"vceqzd");
7401 case NEON::BI__builtin_neon_vceqd_f64:
7402 case NEON::BI__builtin_neon_vcled_f64:
7403 case NEON::BI__builtin_neon_vcltd_f64:
7404 case NEON::BI__builtin_neon_vcged_f64:
7405 case NEON::BI__builtin_neon_vcgtd_f64: {
7406 llvm::CmpInst::Predicate
P;
7407 switch (BuiltinID) {
7408 default: llvm_unreachable(
"missing builtin ID in switch!");
7409 case NEON::BI__builtin_neon_vceqd_f64: P = llvm::FCmpInst::FCMP_OEQ;
break;
7410 case NEON::BI__builtin_neon_vcled_f64: P = llvm::FCmpInst::FCMP_OLE;
break;
7411 case NEON::BI__builtin_neon_vcltd_f64: P = llvm::FCmpInst::FCMP_OLT;
break;
7412 case NEON::BI__builtin_neon_vcged_f64: P = llvm::FCmpInst::FCMP_OGE;
break;
7413 case NEON::BI__builtin_neon_vcgtd_f64: P = llvm::FCmpInst::FCMP_OGT;
break;
7415 Ops.push_back(EmitScalarExpr(E->
getArg(1)));
7416 Ops[0] = Builder.CreateBitCast(Ops[0], DoubleTy);
7417 Ops[1] = Builder.CreateBitCast(Ops[1], DoubleTy);
7418 Ops[0] = Builder.CreateFCmp(P, Ops[0], Ops[1]);
7419 return Builder.CreateSExt(Ops[0], Int64Ty,
"vcmpd");
7421 case NEON::BI__builtin_neon_vceqs_f32:
7422 case NEON::BI__builtin_neon_vcles_f32:
7423 case NEON::BI__builtin_neon_vclts_f32:
7424 case NEON::BI__builtin_neon_vcges_f32:
7425 case NEON::BI__builtin_neon_vcgts_f32: {
7426 llvm::CmpInst::Predicate
P;
7427 switch (BuiltinID) {
7428 default: llvm_unreachable(
"missing builtin ID in switch!");
7429 case NEON::BI__builtin_neon_vceqs_f32: P = llvm::FCmpInst::FCMP_OEQ;
break;
7430 case NEON::BI__builtin_neon_vcles_f32: P = llvm::FCmpInst::FCMP_OLE;
break;
7431 case NEON::BI__builtin_neon_vclts_f32: P = llvm::FCmpInst::FCMP_OLT;
break;
7432 case NEON::BI__builtin_neon_vcges_f32: P = llvm::FCmpInst::FCMP_OGE;
break;
7433 case NEON::BI__builtin_neon_vcgts_f32: P = llvm::FCmpInst::FCMP_OGT;
break;
7435 Ops.push_back(EmitScalarExpr(E->
getArg(1)));
7436 Ops[0] = Builder.CreateBitCast(Ops[0], FloatTy);
7437 Ops[1] = Builder.CreateBitCast(Ops[1], FloatTy);
7438 Ops[0] = Builder.CreateFCmp(P, Ops[0], Ops[1]);
7439 return Builder.CreateSExt(Ops[0], Int32Ty,
"vcmpd");
7441 case NEON::BI__builtin_neon_vceqh_f16:
7442 case NEON::BI__builtin_neon_vcleh_f16:
7443 case NEON::BI__builtin_neon_vclth_f16:
7444 case NEON::BI__builtin_neon_vcgeh_f16:
7445 case NEON::BI__builtin_neon_vcgth_f16: {
7446 llvm::CmpInst::Predicate
P;
7447 switch (BuiltinID) {
7448 default: llvm_unreachable(
"missing builtin ID in switch!");
7449 case NEON::BI__builtin_neon_vceqh_f16: P = llvm::FCmpInst::FCMP_OEQ;
break;
7450 case NEON::BI__builtin_neon_vcleh_f16: P = llvm::FCmpInst::FCMP_OLE;
break;
7451 case NEON::BI__builtin_neon_vclth_f16: P = llvm::FCmpInst::FCMP_OLT;
break;
7452 case NEON::BI__builtin_neon_vcgeh_f16: P = llvm::FCmpInst::FCMP_OGE;
break;
7453 case NEON::BI__builtin_neon_vcgth_f16: P = llvm::FCmpInst::FCMP_OGT;
break;
7455 Ops.push_back(EmitScalarExpr(E->
getArg(1)));
7456 Ops[0] = Builder.CreateBitCast(Ops[0], HalfTy);
7457 Ops[1] = Builder.CreateBitCast(Ops[1], HalfTy);
7458 Ops[0] = Builder.CreateFCmp(P, Ops[0], Ops[1]);
7459 return Builder.CreateSExt(Ops[0], Int16Ty,
"vcmpd");
7461 case NEON::BI__builtin_neon_vceqd_s64:
7462 case NEON::BI__builtin_neon_vceqd_u64:
7463 case NEON::BI__builtin_neon_vcgtd_s64:
7464 case NEON::BI__builtin_neon_vcgtd_u64:
7465 case NEON::BI__builtin_neon_vcltd_s64:
7466 case NEON::BI__builtin_neon_vcltd_u64:
7467 case NEON::BI__builtin_neon_vcged_u64:
7468 case NEON::BI__builtin_neon_vcged_s64:
7469 case NEON::BI__builtin_neon_vcled_u64:
7470 case NEON::BI__builtin_neon_vcled_s64: {
7471 llvm::CmpInst::Predicate
P;
7472 switch (BuiltinID) {
7473 default: llvm_unreachable(
"missing builtin ID in switch!");
7474 case NEON::BI__builtin_neon_vceqd_s64:
7475 case NEON::BI__builtin_neon_vceqd_u64:P = llvm::ICmpInst::ICMP_EQ;
break;
7476 case NEON::BI__builtin_neon_vcgtd_s64:P = llvm::ICmpInst::ICMP_SGT;
break;
7477 case NEON::BI__builtin_neon_vcgtd_u64:P = llvm::ICmpInst::ICMP_UGT;
break;
7478 case NEON::BI__builtin_neon_vcltd_s64:P = llvm::ICmpInst::ICMP_SLT;
break;
7479 case NEON::BI__builtin_neon_vcltd_u64:P = llvm::ICmpInst::ICMP_ULT;
break;
7480 case NEON::BI__builtin_neon_vcged_u64:P = llvm::ICmpInst::ICMP_UGE;
break;
7481 case NEON::BI__builtin_neon_vcged_s64:P = llvm::ICmpInst::ICMP_SGE;
break;
7482 case NEON::BI__builtin_neon_vcled_u64:P = llvm::ICmpInst::ICMP_ULE;
break;
7483 case NEON::BI__builtin_neon_vcled_s64:P = llvm::ICmpInst::ICMP_SLE;
break;
7485 Ops.push_back(EmitScalarExpr(E->
getArg(1)));
7486 Ops[0] = Builder.CreateBitCast(Ops[0], Int64Ty);
7487 Ops[1] = Builder.CreateBitCast(Ops[1], Int64Ty);
7488 Ops[0] = Builder.CreateICmp(P, Ops[0], Ops[1]);
7489 return Builder.CreateSExt(Ops[0], Int64Ty,
"vceqd");
7491 case NEON::BI__builtin_neon_vtstd_s64:
7492 case NEON::BI__builtin_neon_vtstd_u64: {
7493 Ops.push_back(EmitScalarExpr(E->
getArg(1)));
7494 Ops[0] = Builder.CreateBitCast(Ops[0], Int64Ty);
7495 Ops[1] = Builder.CreateBitCast(Ops[1], Int64Ty);
7496 Ops[0] = Builder.CreateAnd(Ops[0], Ops[1]);
7497 Ops[0] = Builder.CreateICmp(ICmpInst::ICMP_NE, Ops[0],
7498 llvm::Constant::getNullValue(Int64Ty));
7499 return Builder.CreateSExt(Ops[0], Int64Ty,
"vtstd");
7501 case NEON::BI__builtin_neon_vset_lane_i8:
7502 case NEON::BI__builtin_neon_vset_lane_i16:
7503 case NEON::BI__builtin_neon_vset_lane_i32:
7504 case NEON::BI__builtin_neon_vset_lane_i64:
7505 case NEON::BI__builtin_neon_vset_lane_f32:
7506 case NEON::BI__builtin_neon_vsetq_lane_i8:
7507 case NEON::BI__builtin_neon_vsetq_lane_i16:
7508 case NEON::BI__builtin_neon_vsetq_lane_i32:
7509 case NEON::BI__builtin_neon_vsetq_lane_i64:
7510 case NEON::BI__builtin_neon_vsetq_lane_f32:
7511 Ops.push_back(EmitScalarExpr(E->
getArg(2)));
7512 return Builder.CreateInsertElement(Ops[1], Ops[0], Ops[2],
"vset_lane");
7513 case NEON::BI__builtin_neon_vset_lane_f64:
7515 Ops[1] = Builder.CreateBitCast(Ops[1],
7516 llvm::VectorType::get(DoubleTy, 1));
7517 Ops.push_back(EmitScalarExpr(E->
getArg(2)));
7518 return Builder.CreateInsertElement(Ops[1], Ops[0], Ops[2],
"vset_lane");
7519 case NEON::BI__builtin_neon_vsetq_lane_f64:
7521 Ops[1] = Builder.CreateBitCast(Ops[1],
7522 llvm::VectorType::get(DoubleTy, 2));
7523 Ops.push_back(EmitScalarExpr(E->
getArg(2)));
7524 return Builder.CreateInsertElement(Ops[1], Ops[0], Ops[2],
"vset_lane");
7526 case NEON::BI__builtin_neon_vget_lane_i8:
7527 case NEON::BI__builtin_neon_vdupb_lane_i8:
7528 Ops[0] = Builder.CreateBitCast(Ops[0], llvm::VectorType::get(Int8Ty, 8));
7529 return Builder.CreateExtractElement(Ops[0], EmitScalarExpr(E->
getArg(1)),
7531 case NEON::BI__builtin_neon_vgetq_lane_i8:
7532 case NEON::BI__builtin_neon_vdupb_laneq_i8:
7533 Ops[0] = Builder.CreateBitCast(Ops[0], llvm::VectorType::get(Int8Ty, 16));
7534 return Builder.CreateExtractElement(Ops[0], EmitScalarExpr(E->
getArg(1)),
7536 case NEON::BI__builtin_neon_vget_lane_i16:
7537 case NEON::BI__builtin_neon_vduph_lane_i16:
7538 Ops[0] = Builder.CreateBitCast(Ops[0], llvm::VectorType::get(Int16Ty, 4));
7539 return Builder.CreateExtractElement(Ops[0], EmitScalarExpr(E->
getArg(1)),
7541 case NEON::BI__builtin_neon_vgetq_lane_i16:
7542 case NEON::BI__builtin_neon_vduph_laneq_i16:
7543 Ops[0] = Builder.CreateBitCast(Ops[0], llvm::VectorType::get(Int16Ty, 8));
7544 return Builder.CreateExtractElement(Ops[0], EmitScalarExpr(E->
getArg(1)),
7546 case NEON::BI__builtin_neon_vget_lane_i32:
7547 case NEON::BI__builtin_neon_vdups_lane_i32:
7548 Ops[0] = Builder.CreateBitCast(Ops[0], llvm::VectorType::get(Int32Ty, 2));
7549 return Builder.CreateExtractElement(Ops[0], EmitScalarExpr(E->
getArg(1)),
7551 case NEON::BI__builtin_neon_vdups_lane_f32:
7552 Ops[0] = Builder.CreateBitCast(Ops[0],
7553 llvm::VectorType::get(FloatTy, 2));
7554 return Builder.CreateExtractElement(Ops[0], EmitScalarExpr(E->
getArg(1)),
7556 case NEON::BI__builtin_neon_vgetq_lane_i32:
7557 case NEON::BI__builtin_neon_vdups_laneq_i32:
7558 Ops[0] = Builder.CreateBitCast(Ops[0], llvm::VectorType::get(Int32Ty, 4));
7559 return Builder.CreateExtractElement(Ops[0], EmitScalarExpr(E->
getArg(1)),
7561 case NEON::BI__builtin_neon_vget_lane_i64:
7562 case NEON::BI__builtin_neon_vdupd_lane_i64:
7563 Ops[0] = Builder.CreateBitCast(Ops[0], llvm::VectorType::get(Int64Ty, 1));
7564 return Builder.CreateExtractElement(Ops[0], EmitScalarExpr(E->
getArg(1)),
7566 case NEON::BI__builtin_neon_vdupd_lane_f64:
7567 Ops[0] = Builder.CreateBitCast(Ops[0],
7568 llvm::VectorType::get(DoubleTy, 1));
7569 return Builder.CreateExtractElement(Ops[0], EmitScalarExpr(E->
getArg(1)),
7571 case NEON::BI__builtin_neon_vgetq_lane_i64:
7572 case NEON::BI__builtin_neon_vdupd_laneq_i64:
7573 Ops[0] = Builder.CreateBitCast(Ops[0], llvm::VectorType::get(Int64Ty, 2));
7574 return Builder.CreateExtractElement(Ops[0], EmitScalarExpr(E->
getArg(1)),
7576 case NEON::BI__builtin_neon_vget_lane_f32:
7577 Ops[0] = Builder.CreateBitCast(Ops[0],
7578 llvm::VectorType::get(FloatTy, 2));
7579 return Builder.CreateExtractElement(Ops[0], EmitScalarExpr(E->
getArg(1)),
7581 case NEON::BI__builtin_neon_vget_lane_f64:
7582 Ops[0] = Builder.CreateBitCast(Ops[0],
7583 llvm::VectorType::get(DoubleTy, 1));
7584 return Builder.CreateExtractElement(Ops[0], EmitScalarExpr(E->
getArg(1)),
7586 case NEON::BI__builtin_neon_vgetq_lane_f32:
7587 case NEON::BI__builtin_neon_vdups_laneq_f32:
7588 Ops[0] = Builder.CreateBitCast(Ops[0],
7589 llvm::VectorType::get(FloatTy, 4));
7590 return Builder.CreateExtractElement(Ops[0], EmitScalarExpr(E->
getArg(1)),
7592 case NEON::BI__builtin_neon_vgetq_lane_f64:
7593 case NEON::BI__builtin_neon_vdupd_laneq_f64:
7594 Ops[0] = Builder.CreateBitCast(Ops[0],
7595 llvm::VectorType::get(DoubleTy, 2));
7596 return Builder.CreateExtractElement(Ops[0], EmitScalarExpr(E->
getArg(1)),
7598 case NEON::BI__builtin_neon_vaddh_f16:
7599 Ops.push_back(EmitScalarExpr(E->
getArg(1)));
7600 return Builder.CreateFAdd(Ops[0], Ops[1],
"vaddh");
7601 case NEON::BI__builtin_neon_vsubh_f16:
7602 Ops.push_back(EmitScalarExpr(E->
getArg(1)));
7603 return Builder.CreateFSub(Ops[0], Ops[1],
"vsubh");
7604 case NEON::BI__builtin_neon_vmulh_f16:
7605 Ops.push_back(EmitScalarExpr(E->
getArg(1)));
7606 return Builder.CreateFMul(Ops[0], Ops[1],
"vmulh");
7607 case NEON::BI__builtin_neon_vdivh_f16:
7608 Ops.push_back(EmitScalarExpr(E->
getArg(1)));
7609 return Builder.CreateFDiv(Ops[0], Ops[1],
"vdivh");
7610 case NEON::BI__builtin_neon_vfmah_f16: {
7613 return Builder.CreateCall(F,
7614 {EmitScalarExpr(E->
getArg(1)), EmitScalarExpr(E->
getArg(2)), Ops[0]});
7616 case NEON::BI__builtin_neon_vfmsh_f16: {
7618 Value *Zero = llvm::ConstantFP::getZeroValueForNegation(HalfTy);
7619 Value* Sub = Builder.CreateFSub(Zero, EmitScalarExpr(E->
getArg(1)),
"vsubh");
7621 return Builder.CreateCall(F, {Sub, EmitScalarExpr(E->
getArg(2)), Ops[0]});
7623 case NEON::BI__builtin_neon_vaddd_s64:
7624 case NEON::BI__builtin_neon_vaddd_u64:
7625 return Builder.CreateAdd(Ops[0], EmitScalarExpr(E->
getArg(1)),
"vaddd");
7626 case NEON::BI__builtin_neon_vsubd_s64:
7627 case NEON::BI__builtin_neon_vsubd_u64:
7628 return Builder.CreateSub(Ops[0], EmitScalarExpr(E->
getArg(1)),
"vsubd");
7629 case NEON::BI__builtin_neon_vqdmlalh_s16:
7630 case NEON::BI__builtin_neon_vqdmlslh_s16: {
7632 ProductOps.push_back(vectorWrapScalar16(Ops[1]));
7633 ProductOps.push_back(vectorWrapScalar16(EmitScalarExpr(E->
getArg(2))));
7634 llvm::Type *VTy = llvm::VectorType::get(Int32Ty, 4);
7635 Ops[1] = EmitNeonCall(CGM.getIntrinsic(Intrinsic::aarch64_neon_sqdmull, VTy),
7636 ProductOps,
"vqdmlXl");
7637 Constant *CI = ConstantInt::get(SizeTy, 0);
7638 Ops[1] = Builder.CreateExtractElement(Ops[1], CI,
"lane0");
7640 unsigned AccumInt = BuiltinID == NEON::BI__builtin_neon_vqdmlalh_s16
7641 ? Intrinsic::aarch64_neon_sqadd
7642 : Intrinsic::aarch64_neon_sqsub;
7643 return EmitNeonCall(CGM.getIntrinsic(AccumInt, Int32Ty), Ops,
"vqdmlXl");
7645 case NEON::BI__builtin_neon_vqshlud_n_s64: {
7646 Ops.push_back(EmitScalarExpr(E->
getArg(1)));
7647 Ops[1] = Builder.CreateZExt(Ops[1], Int64Ty);
7648 return EmitNeonCall(CGM.getIntrinsic(Intrinsic::aarch64_neon_sqshlu, Int64Ty),
7651 case NEON::BI__builtin_neon_vqshld_n_u64:
7652 case NEON::BI__builtin_neon_vqshld_n_s64: {
7653 unsigned Int = BuiltinID == NEON::BI__builtin_neon_vqshld_n_u64
7654 ? Intrinsic::aarch64_neon_uqshl
7655 : Intrinsic::aarch64_neon_sqshl;
7656 Ops.push_back(EmitScalarExpr(E->
getArg(1)));
7657 Ops[1] = Builder.CreateZExt(Ops[1], Int64Ty);
7658 return EmitNeonCall(CGM.getIntrinsic(Int, Int64Ty), Ops,
"vqshl_n");
7660 case NEON::BI__builtin_neon_vrshrd_n_u64:
7661 case NEON::BI__builtin_neon_vrshrd_n_s64: {
7662 unsigned Int = BuiltinID == NEON::BI__builtin_neon_vrshrd_n_u64
7663 ? Intrinsic::aarch64_neon_urshl
7664 : Intrinsic::aarch64_neon_srshl;
7665 Ops.push_back(EmitScalarExpr(E->
getArg(1)));
7666 int SV = cast<ConstantInt>(Ops[1])->getSExtValue();
7667 Ops[1] = ConstantInt::get(Int64Ty, -SV);
7668 return EmitNeonCall(CGM.getIntrinsic(Int, Int64Ty), Ops,
"vrshr_n");
7670 case NEON::BI__builtin_neon_vrsrad_n_u64:
7671 case NEON::BI__builtin_neon_vrsrad_n_s64: {
7672 unsigned Int = BuiltinID == NEON::BI__builtin_neon_vrsrad_n_u64
7673 ? Intrinsic::aarch64_neon_urshl
7674 : Intrinsic::aarch64_neon_srshl;
7675 Ops[1] = Builder.CreateBitCast(Ops[1], Int64Ty);
7676 Ops.push_back(Builder.CreateNeg(EmitScalarExpr(E->
getArg(2))));
7677 Ops[1] = Builder.CreateCall(CGM.getIntrinsic(Int, Int64Ty),
7678 {Ops[1], Builder.CreateSExt(Ops[2], Int64Ty)});
7679 return Builder.CreateAdd(Ops[0], Builder.CreateBitCast(Ops[1], Int64Ty));
7681 case NEON::BI__builtin_neon_vshld_n_s64:
7682 case NEON::BI__builtin_neon_vshld_n_u64: {
7683 llvm::ConstantInt *Amt = cast<ConstantInt>(EmitScalarExpr(E->
getArg(1)));
7684 return Builder.CreateShl(
7685 Ops[0], ConstantInt::get(Int64Ty, Amt->getZExtValue()),
"shld_n");
7687 case NEON::BI__builtin_neon_vshrd_n_s64: {
7688 llvm::ConstantInt *Amt = cast<ConstantInt>(EmitScalarExpr(E->
getArg(1)));
7689 return Builder.CreateAShr(
7690 Ops[0], ConstantInt::get(Int64Ty,
std::min(static_cast<uint64_t>(63),
7691 Amt->getZExtValue())),
7694 case NEON::BI__builtin_neon_vshrd_n_u64: {
7695 llvm::ConstantInt *Amt = cast<ConstantInt>(EmitScalarExpr(E->
getArg(1)));
7696 uint64_t ShiftAmt = Amt->getZExtValue();
7699 return ConstantInt::get(Int64Ty, 0);
7700 return Builder.CreateLShr(Ops[0], ConstantInt::get(Int64Ty, ShiftAmt),
7703 case NEON::BI__builtin_neon_vsrad_n_s64: {
7704 llvm::ConstantInt *Amt = cast<ConstantInt>(EmitScalarExpr(E->
getArg(2)));
7705 Ops[1] = Builder.CreateAShr(
7706 Ops[1], ConstantInt::get(Int64Ty,
std::min(static_cast<uint64_t>(63),
7707 Amt->getZExtValue())),
7709 return Builder.CreateAdd(Ops[0], Ops[1]);
7711 case NEON::BI__builtin_neon_vsrad_n_u64: {
7712 llvm::ConstantInt *Amt = cast<ConstantInt>(EmitScalarExpr(E->
getArg(2)));
7713 uint64_t ShiftAmt = Amt->getZExtValue();
7718 Ops[1] = Builder.CreateLShr(Ops[1], ConstantInt::get(Int64Ty, ShiftAmt),
7720 return Builder.CreateAdd(Ops[0], Ops[1]);
7722 case NEON::BI__builtin_neon_vqdmlalh_lane_s16:
7723 case NEON::BI__builtin_neon_vqdmlalh_laneq_s16:
7724 case NEON::BI__builtin_neon_vqdmlslh_lane_s16:
7725 case NEON::BI__builtin_neon_vqdmlslh_laneq_s16: {
7726 Ops[2] = Builder.CreateExtractElement(Ops[2], EmitScalarExpr(E->
getArg(3)),
7729 ProductOps.push_back(vectorWrapScalar16(Ops[1]));
7730 ProductOps.push_back(vectorWrapScalar16(Ops[2]));
7731 llvm::Type *VTy = llvm::VectorType::get(Int32Ty, 4);
7732 Ops[1] = EmitNeonCall(CGM.getIntrinsic(Intrinsic::aarch64_neon_sqdmull, VTy),
7733 ProductOps,
"vqdmlXl");
7734 Constant *CI = ConstantInt::get(SizeTy, 0);
7735 Ops[1] = Builder.CreateExtractElement(Ops[1], CI,
"lane0");
7738 unsigned AccInt = (BuiltinID == NEON::BI__builtin_neon_vqdmlalh_lane_s16 ||
7739 BuiltinID == NEON::BI__builtin_neon_vqdmlalh_laneq_s16)
7740 ? Intrinsic::aarch64_neon_sqadd
7741 : Intrinsic::aarch64_neon_sqsub;
7742 return EmitNeonCall(CGM.getIntrinsic(AccInt, Int32Ty), Ops,
"vqdmlXl");
7744 case NEON::BI__builtin_neon_vqdmlals_s32:
7745 case NEON::BI__builtin_neon_vqdmlsls_s32: {
7747 ProductOps.push_back(Ops[1]);
7748 ProductOps.push_back(EmitScalarExpr(E->
getArg(2)));
7750 EmitNeonCall(CGM.getIntrinsic(Intrinsic::aarch64_neon_sqdmulls_scalar),
7751 ProductOps,
"vqdmlXl");
7753 unsigned AccumInt = BuiltinID == NEON::BI__builtin_neon_vqdmlals_s32
7754 ? Intrinsic::aarch64_neon_sqadd
7755 : Intrinsic::aarch64_neon_sqsub;
7756 return EmitNeonCall(CGM.getIntrinsic(AccumInt, Int64Ty), Ops,
"vqdmlXl");
7758 case NEON::BI__builtin_neon_vqdmlals_lane_s32:
7759 case NEON::BI__builtin_neon_vqdmlals_laneq_s32:
7760 case NEON::BI__builtin_neon_vqdmlsls_lane_s32:
7761 case NEON::BI__builtin_neon_vqdmlsls_laneq_s32: {
7762 Ops[2] = Builder.CreateExtractElement(Ops[2], EmitScalarExpr(E->
getArg(3)),
7765 ProductOps.push_back(Ops[1]);
7766 ProductOps.push_back(Ops[2]);
7768 EmitNeonCall(CGM.getIntrinsic(Intrinsic::aarch64_neon_sqdmulls_scalar),
7769 ProductOps,
"vqdmlXl");
7772 unsigned AccInt = (BuiltinID == NEON::BI__builtin_neon_vqdmlals_lane_s32 ||
7773 BuiltinID == NEON::BI__builtin_neon_vqdmlals_laneq_s32)
7774 ? Intrinsic::aarch64_neon_sqadd
7775 : Intrinsic::aarch64_neon_sqsub;
7776 return EmitNeonCall(CGM.getIntrinsic(AccInt, Int64Ty), Ops,
"vqdmlXl");
7788 AArch64SIMDIntrinsicsProvenSorted);
7791 return EmitCommonNeonBuiltinExpr(
7792 Builtin->BuiltinID, Builtin->LLVMIntrinsic, Builtin->AltLLVMIntrinsic,
7793 Builtin->NameHint, Builtin->TypeModifier, E, Ops,
7800 switch (BuiltinID) {
7801 default:
return nullptr;
7802 case NEON::BI__builtin_neon_vbsl_v:
7803 case NEON::BI__builtin_neon_vbslq_v: {
7804 llvm::Type *BitTy = llvm::VectorType::getInteger(VTy);
7805 Ops[0] = Builder.CreateBitCast(Ops[0], BitTy,
"vbsl");
7806 Ops[1] = Builder.CreateBitCast(Ops[1], BitTy,
"vbsl");
7807 Ops[2] = Builder.CreateBitCast(Ops[2], BitTy,
"vbsl");
7809 Ops[1] = Builder.CreateAnd(Ops[0], Ops[1],
"vbsl");
7810 Ops[2] = Builder.CreateAnd(Builder.CreateNot(Ops[0]), Ops[2],
"vbsl");
7811 Ops[0] = Builder.CreateOr(Ops[1], Ops[2],
"vbsl");
7812 return Builder.CreateBitCast(Ops[0], Ty);
7814 case NEON::BI__builtin_neon_vfma_lane_v:
7815 case NEON::BI__builtin_neon_vfmaq_lane_v: {
7818 Value *Addend = Ops[0];
7819 Value *Multiplicand = Ops[1];
7820 Value *LaneSource = Ops[2];
7821 Ops[0] = Multiplicand;
7822 Ops[1] = LaneSource;
7826 llvm::Type *SourceTy = BuiltinID == NEON::BI__builtin_neon_vfmaq_lane_v ?
7827 llvm::VectorType::get(VTy->getElementType(), VTy->getNumElements() / 2) :
7829 llvm::Constant *cst = cast<Constant>(Ops[3]);
7830 Value *SV = llvm::ConstantVector::getSplat(VTy->getNumElements(), cst);
7831 Ops[1] = Builder.CreateBitCast(Ops[1], SourceTy);
7832 Ops[1] = Builder.CreateShuffleVector(Ops[1], Ops[1], SV,
"lane");
7836 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops,
"fmla");
7838 case NEON::BI__builtin_neon_vfma_laneq_v: {
7839 llvm::VectorType *VTy = cast<llvm::VectorType>(Ty);
7841 if (VTy && VTy->getElementType() == DoubleTy) {
7842 Ops[0] = Builder.CreateBitCast(Ops[0], DoubleTy);
7843 Ops[1] = Builder.CreateBitCast(Ops[1], DoubleTy);
7846 Ops[2] = Builder.CreateBitCast(Ops[2], VTy);
7847 Ops[2] = Builder.CreateExtractElement(Ops[2], Ops[3],
"extract");
7849 Value *Result = Builder.CreateCall(F, {Ops[1], Ops[2], Ops[0]});
7850 return Builder.CreateBitCast(Result, Ty);
7853 Ops[0] = Builder.CreateBitCast(Ops[0], Ty);
7854 Ops[1] = Builder.CreateBitCast(Ops[1], Ty);
7856 llvm::Type *STy = llvm::VectorType::get(VTy->getElementType(),
7857 VTy->getNumElements() * 2);
7858 Ops[2] = Builder.CreateBitCast(Ops[2], STy);
7859 Value* SV = llvm::ConstantVector::getSplat(VTy->getNumElements(),
7860 cast<ConstantInt>(Ops[3]));
7861 Ops[2] = Builder.CreateShuffleVector(Ops[2], Ops[2], SV,
"lane");
7863 return Builder.CreateCall(F, {Ops[2], Ops[1], Ops[0]});
7865 case NEON::BI__builtin_neon_vfmaq_laneq_v: {
7867 Ops[0] = Builder.CreateBitCast(Ops[0], Ty);
7868 Ops[1] = Builder.CreateBitCast(Ops[1], Ty);
7870 Ops[2] = Builder.CreateBitCast(Ops[2], Ty);
7871 Ops[2] = EmitNeonSplat(Ops[2], cast<ConstantInt>(Ops[3]));
7872 return Builder.CreateCall(F, {Ops[2], Ops[1], Ops[0]});
7874 case NEON::BI__builtin_neon_vfmah_lane_f16:
7875 case NEON::BI__builtin_neon_vfmas_lane_f32:
7876 case NEON::BI__builtin_neon_vfmah_laneq_f16:
7877 case NEON::BI__builtin_neon_vfmas_laneq_f32:
7878 case NEON::BI__builtin_neon_vfmad_lane_f64:
7879 case NEON::BI__builtin_neon_vfmad_laneq_f64: {
7880 Ops.push_back(EmitScalarExpr(E->
getArg(3)));
7883 Ops[2] = Builder.CreateExtractElement(Ops[2], Ops[3],
"extract");
7884 return Builder.CreateCall(F, {Ops[1], Ops[2], Ops[0]});
7886 case NEON::BI__builtin_neon_vmull_v:
7888 Int = usgn ? Intrinsic::aarch64_neon_umull : Intrinsic::aarch64_neon_smull;
7889 if (Type.
isPoly()) Int = Intrinsic::aarch64_neon_pmull;
7890 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops,
"vmull");
7891 case NEON::BI__builtin_neon_vmax_v:
7892 case NEON::BI__builtin_neon_vmaxq_v:
7894 Int = usgn ? Intrinsic::aarch64_neon_umax : Intrinsic::aarch64_neon_smax;
7895 if (Ty->isFPOrFPVectorTy()) Int = Intrinsic::aarch64_neon_fmax;
7896 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops,
"vmax");
7897 case NEON::BI__builtin_neon_vmaxh_f16: {
7898 Ops.push_back(EmitScalarExpr(E->
getArg(1)));
7899 Int = Intrinsic::aarch64_neon_fmax;
7900 return EmitNeonCall(CGM.getIntrinsic(Int, HalfTy), Ops,
"vmax");
7902 case NEON::BI__builtin_neon_vmin_v:
7903 case NEON::BI__builtin_neon_vminq_v:
7905 Int = usgn ? Intrinsic::aarch64_neon_umin : Intrinsic::aarch64_neon_smin;
7906 if (Ty->isFPOrFPVectorTy()) Int = Intrinsic::aarch64_neon_fmin;
7907 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops,
"vmin");
7908 case NEON::BI__builtin_neon_vminh_f16: {
7909 Ops.push_back(EmitScalarExpr(E->
getArg(1)));
7910 Int = Intrinsic::aarch64_neon_fmin;
7911 return EmitNeonCall(CGM.getIntrinsic(Int, HalfTy), Ops,
"vmin");
7913 case NEON::BI__builtin_neon_vabd_v:
7914 case NEON::BI__builtin_neon_vabdq_v:
7916 Int = usgn ? Intrinsic::aarch64_neon_uabd : Intrinsic::aarch64_neon_sabd;
7917 if (Ty->isFPOrFPVectorTy()) Int = Intrinsic::aarch64_neon_fabd;
7918 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops,
"vabd");
7919 case NEON::BI__builtin_neon_vpadal_v:
7920 case NEON::BI__builtin_neon_vpadalq_v: {
7921 unsigned ArgElts = VTy->getNumElements();
7922 llvm::IntegerType *EltTy = cast<IntegerType>(VTy->getElementType());
7923 unsigned BitWidth = EltTy->getBitWidth();
7925 llvm::IntegerType::get(getLLVMContext(), BitWidth/2), 2*ArgElts);
7927 Int = usgn ? Intrinsic::aarch64_neon_uaddlp : Intrinsic::aarch64_neon_saddlp;
7929 TmpOps.push_back(Ops[1]);
7930 Function *F = CGM.getIntrinsic(Int, Tys);
7931 llvm::Value *tmp = EmitNeonCall(F, TmpOps,
"vpadal");
7932 llvm::Value *addend = Builder.CreateBitCast(Ops[0], tmp->getType());
7933 return Builder.CreateAdd(tmp, addend);
7935 case NEON::BI__builtin_neon_vpmin_v:
7936 case NEON::BI__builtin_neon_vpminq_v:
7938 Int = usgn ? Intrinsic::aarch64_neon_uminp : Intrinsic::aarch64_neon_sminp;
7939 if (Ty->isFPOrFPVectorTy()) Int = Intrinsic::aarch64_neon_fminp;
7940 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops,
"vpmin");
7941 case NEON::BI__builtin_neon_vpmax_v:
7942 case NEON::BI__builtin_neon_vpmaxq_v:
7944 Int = usgn ? Intrinsic::aarch64_neon_umaxp : Intrinsic::aarch64_neon_smaxp;
7945 if (Ty->isFPOrFPVectorTy()) Int = Intrinsic::aarch64_neon_fmaxp;
7946 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops,
"vpmax");
7947 case NEON::BI__builtin_neon_vminnm_v:
7948 case NEON::BI__builtin_neon_vminnmq_v:
7949 Int = Intrinsic::aarch64_neon_fminnm;
7950 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops,
"vminnm");
7951 case NEON::BI__builtin_neon_vminnmh_f16:
7952 Ops.push_back(EmitScalarExpr(E->
getArg(1)));
7953 Int = Intrinsic::aarch64_neon_fminnm;
7954 return EmitNeonCall(CGM.getIntrinsic(Int, HalfTy), Ops,
"vminnm");
7955 case NEON::BI__builtin_neon_vmaxnm_v:
7956 case NEON::BI__builtin_neon_vmaxnmq_v:
7957 Int = Intrinsic::aarch64_neon_fmaxnm;
7958 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops,
"vmaxnm");
7959 case NEON::BI__builtin_neon_vmaxnmh_f16:
7960 Ops.push_back(EmitScalarExpr(E->
getArg(1)));
7961 Int = Intrinsic::aarch64_neon_fmaxnm;
7962 return EmitNeonCall(CGM.getIntrinsic(Int, HalfTy), Ops,
"vmaxnm");
7963 case NEON::BI__builtin_neon_vrecpss_f32: {
7964 Ops.push_back(EmitScalarExpr(E->
getArg(1)));
7965 return EmitNeonCall(CGM.getIntrinsic(Intrinsic::aarch64_neon_frecps, FloatTy),
7968 case NEON::BI__builtin_neon_vrecpsd_f64:
7969 Ops.push_back(EmitScalarExpr(E->
getArg(1)));
7970 return EmitNeonCall(CGM.getIntrinsic(Intrinsic::aarch64_neon_frecps, DoubleTy),
7972 case NEON::BI__builtin_neon_vrecpsh_f16:
7973 Ops.push_back(EmitScalarExpr(E->
getArg(1)));
7974 return EmitNeonCall(CGM.getIntrinsic(Intrinsic::aarch64_neon_frecps, HalfTy),
7976 case NEON::BI__builtin_neon_vqshrun_n_v:
7977 Int = Intrinsic::aarch64_neon_sqshrun;
7978 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops,
"vqshrun_n");
7979 case NEON::BI__builtin_neon_vqrshrun_n_v:
7980 Int = Intrinsic::aarch64_neon_sqrshrun;
7981 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops,
"vqrshrun_n");
7982 case NEON::BI__builtin_neon_vqshrn_n_v:
7983 Int = usgn ? Intrinsic::aarch64_neon_uqshrn : Intrinsic::aarch64_neon_sqshrn;
7984 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops,
"vqshrn_n");
7985 case NEON::BI__builtin_neon_vrshrn_n_v:
7986 Int = Intrinsic::aarch64_neon_rshrn;
7987 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops,
"vrshrn_n");
7988 case NEON::BI__builtin_neon_vqrshrn_n_v:
7989 Int = usgn ? Intrinsic::aarch64_neon_uqrshrn : Intrinsic::aarch64_neon_sqrshrn;
7990 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops,
"vqrshrn_n");
7991 case NEON::BI__builtin_neon_vrndah_f16: {
7992 Ops.push_back(EmitScalarExpr(E->
getArg(0)));
7994 return EmitNeonCall(CGM.getIntrinsic(Int, HalfTy), Ops,
"vrnda");
7996 case NEON::BI__builtin_neon_vrnda_v:
7997 case NEON::BI__builtin_neon_vrndaq_v: {
7999 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops,
"vrnda");
8001 case NEON::BI__builtin_neon_vrndih_f16: {
8002 Ops.push_back(EmitScalarExpr(E->
getArg(0)));
8004 return EmitNeonCall(CGM.getIntrinsic(Int, HalfTy), Ops,
"vrndi");
8006 case NEON::BI__builtin_neon_vrndmh_f16: {
8007 Ops.push_back(EmitScalarExpr(E->
getArg(0)));
8009 return EmitNeonCall(CGM.getIntrinsic(Int, HalfTy), Ops,
"vrndm");
8011 case NEON::BI__builtin_neon_vrndm_v:
8012 case NEON::BI__builtin_neon_vrndmq_v: {
8014 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops,
"vrndm");
8016 case NEON::BI__builtin_neon_vrndnh_f16: {
8017 Ops.push_back(EmitScalarExpr(E->
getArg(0)));
8018 Int = Intrinsic::aarch64_neon_frintn;
8019 return EmitNeonCall(CGM.getIntrinsic(Int, HalfTy), Ops,
"vrndn");
8021 case NEON::BI__builtin_neon_vrndn_v:
8022 case NEON::BI__builtin_neon_vrndnq_v: {
8023 Int = Intrinsic::aarch64_neon_frintn;
8024 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops,
"vrndn");
8026 case NEON::BI__builtin_neon_vrndns_f32: {
8027 Ops.push_back(EmitScalarExpr(E->
getArg(0)));
8028 Int = Intrinsic::aarch64_neon_frintn;
8029 return EmitNeonCall(CGM.getIntrinsic(Int, FloatTy), Ops,
"vrndn");
8031 case NEON::BI__builtin_neon_vrndph_f16: {
8032 Ops.push_back(EmitScalarExpr(E->
getArg(0)));
8034 return EmitNeonCall(CGM.getIntrinsic(Int, HalfTy), Ops,
"vrndp");
8036 case NEON::BI__builtin_neon_vrndp_v:
8037 case NEON::BI__builtin_neon_vrndpq_v: {
8039 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops,
"vrndp");
8041 case NEON::BI__builtin_neon_vrndxh_f16: {
8042 Ops.push_back(EmitScalarExpr(E->
getArg(0)));
8044 return EmitNeonCall(CGM.getIntrinsic(Int, HalfTy), Ops,
"vrndx");
8046 case NEON::BI__builtin_neon_vrndx_v:
8047 case NEON::BI__builtin_neon_vrndxq_v: {
8049 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops,
"vrndx");
8051 case NEON::BI__builtin_neon_vrndh_f16: {
8052 Ops.push_back(EmitScalarExpr(E->
getArg(0)));
8054 return EmitNeonCall(CGM.getIntrinsic(Int, HalfTy), Ops,
"vrndz");
8056 case NEON::BI__builtin_neon_vrnd_v:
8057 case NEON::BI__builtin_neon_vrndq_v: {
8059 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops,
"vrndz");
8061 case NEON::BI__builtin_neon_vcvt_f64_v:
8062 case NEON::BI__builtin_neon_vcvtq_f64_v:
8063 Ops[0] = Builder.CreateBitCast(Ops[0], Ty);
8065 return usgn ? Builder.CreateUIToFP(Ops[0], Ty,
"vcvt")
8066 : Builder.CreateSIToFP(Ops[0], Ty,
"vcvt");
8067 case NEON::BI__builtin_neon_vcvt_f64_f32: {
8069 "unexpected vcvt_f64_f32 builtin");
8071 Ops[0] = Builder.CreateBitCast(Ops[0],
GetNeonType(
this, SrcFlag));
8073 return Builder.CreateFPExt(Ops[0], Ty,
"vcvt");
8075 case NEON::BI__builtin_neon_vcvt_f32_f64: {
8077 "unexpected vcvt_f32_f64 builtin");
8079 Ops[0] = Builder.CreateBitCast(Ops[0],
GetNeonType(
this, SrcFlag));
8081 return Builder.CreateFPTrunc(Ops[0], Ty,
"vcvt");
8083 case NEON::BI__builtin_neon_vcvt_s32_v:
8084 case NEON::BI__builtin_neon_vcvt_u32_v:
8085 case NEON::BI__builtin_neon_vcvt_s64_v:
8086 case NEON::BI__builtin_neon_vcvt_u64_v:
8087 case NEON::BI__builtin_neon_vcvt_s16_v:
8088 case NEON::BI__builtin_neon_vcvt_u16_v:
8089 case NEON::BI__builtin_neon_vcvtq_s32_v:
8090 case NEON::BI__builtin_neon_vcvtq_u32_v:
8091 case NEON::BI__builtin_neon_vcvtq_s64_v:
8092 case NEON::BI__builtin_neon_vcvtq_u64_v:
8093 case NEON::BI__builtin_neon_vcvtq_s16_v:
8094 case NEON::BI__builtin_neon_vcvtq_u16_v: {
8097 return Builder.CreateFPToUI(Ops[0], Ty);
8098 return Builder.CreateFPToSI(Ops[0], Ty);
8100 case NEON::BI__builtin_neon_vcvta_s16_v:
8101 case NEON::BI__builtin_neon_vcvta_u16_v:
8102 case NEON::BI__builtin_neon_vcvta_s32_v:
8103 case NEON::BI__builtin_neon_vcvtaq_s16_v:
8104 case NEON::BI__builtin_neon_vcvtaq_s32_v:
8105 case NEON::BI__builtin_neon_vcvta_u32_v:
8106 case NEON::BI__builtin_neon_vcvtaq_u16_v:
8107 case NEON::BI__builtin_neon_vcvtaq_u32_v:
8108 case NEON::BI__builtin_neon_vcvta_s64_v:
8109 case NEON::BI__builtin_neon_vcvtaq_s64_v:
8110 case NEON::BI__builtin_neon_vcvta_u64_v:
8111 case NEON::BI__builtin_neon_vcvtaq_u64_v: {
8112 Int = usgn ? Intrinsic::aarch64_neon_fcvtau : Intrinsic::aarch64_neon_fcvtas;
8114 return EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops,
"vcvta");
8116 case NEON::BI__builtin_neon_vcvtm_s16_v:
8117 case NEON::BI__builtin_neon_vcvtm_s32_v:
8118 case NEON::BI__builtin_neon_vcvtmq_s16_v:
8119 case NEON::BI__builtin_neon_vcvtmq_s32_v:
8120 case NEON::BI__builtin_neon_vcvtm_u16_v:
8121 case NEON::BI__builtin_neon_vcvtm_u32_v:
8122 case NEON::BI__builtin_neon_vcvtmq_u16_v:
8123 case NEON::BI__builtin_neon_vcvtmq_u32_v:
8124 case NEON::BI__builtin_neon_vcvtm_s64_v:
8125 case NEON::BI__builtin_neon_vcvtmq_s64_v:
8126 case NEON::BI__builtin_neon_vcvtm_u64_v:
8127 case NEON::BI__builtin_neon_vcvtmq_u64_v: {
8128 Int = usgn ? Intrinsic::aarch64_neon_fcvtmu : Intrinsic::aarch64_neon_fcvtms;
8130 return EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops,
"vcvtm");
8132 case NEON::BI__builtin_neon_vcvtn_s16_v:
8133 case NEON::BI__builtin_neon_vcvtn_s32_v:
8134 case NEON::BI__builtin_neon_vcvtnq_s16_v:
8135 case NEON::BI__builtin_neon_vcvtnq_s32_v:
8136 case NEON::BI__builtin_neon_vcvtn_u16_v:
8137 case NEON::BI__builtin_neon_vcvtn_u32_v:
8138 case NEON::BI__builtin_neon_vcvtnq_u16_v:
8139 case NEON::BI__builtin_neon_vcvtnq_u32_v:
8140 case NEON::BI__builtin_neon_vcvtn_s64_v:
8141 case NEON::BI__builtin_neon_vcvtnq_s64_v:
8142 case NEON::BI__builtin_neon_vcvtn_u64_v:
8143 case NEON::BI__builtin_neon_vcvtnq_u64_v: {
8144 Int = usgn ? Intrinsic::aarch64_neon_fcvtnu : Intrinsic::aarch64_neon_fcvtns;
8146 return EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops,
"vcvtn");
8148 case NEON::BI__builtin_neon_vcvtp_s16_v:
8149 case NEON::BI__builtin_neon_vcvtp_s32_v:
8150 case NEON::BI__builtin_neon_vcvtpq_s16_v:
8151 case NEON::BI__builtin_neon_vcvtpq_s32_v:
8152 case NEON::BI__builtin_neon_vcvtp_u16_v:
8153 case NEON::BI__builtin_neon_vcvtp_u32_v:
8154 case NEON::BI__builtin_neon_vcvtpq_u16_v:
8155 case NEON::BI__builtin_neon_vcvtpq_u32_v:
8156 case NEON::BI__builtin_neon_vcvtp_s64_v:
8157 case NEON::BI__builtin_neon_vcvtpq_s64_v:
8158 case NEON::BI__builtin_neon_vcvtp_u64_v:
8159 case NEON::BI__builtin_neon_vcvtpq_u64_v: {
8160 Int = usgn ? Intrinsic::aarch64_neon_fcvtpu : Intrinsic::aarch64_neon_fcvtps;
8162 return EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops,
"vcvtp");
8164 case NEON::BI__builtin_neon_vmulx_v:
8165 case NEON::BI__builtin_neon_vmulxq_v: {
8166 Int = Intrinsic::aarch64_neon_fmulx;
8167 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops,
"vmulx");
8169 case NEON::BI__builtin_neon_vmulxh_lane_f16:
8170 case NEON::BI__builtin_neon_vmulxh_laneq_f16: {
8173 Ops.push_back(EmitScalarExpr(E->
getArg(2)));
8174 Ops[1] = Builder.CreateExtractElement(Ops[1], Ops[2],
"extract");
8176 Int = Intrinsic::aarch64_neon_fmulx;
8177 return EmitNeonCall(CGM.getIntrinsic(Int, HalfTy), Ops,
"vmulx");
8179 case NEON::BI__builtin_neon_vmul_lane_v:
8180 case NEON::BI__builtin_neon_vmul_laneq_v: {
8183 if (BuiltinID == NEON::BI__builtin_neon_vmul_laneq_v)
8185 Ops[0] = Builder.CreateBitCast(Ops[0], DoubleTy);
8188 Ops[1] = Builder.CreateBitCast(Ops[1], VTy);
8189 Ops[1] = Builder.CreateExtractElement(Ops[1], Ops[2],
"extract");
8190 Value *Result = Builder.CreateFMul(Ops[0], Ops[1]);
8191 return Builder.CreateBitCast(Result, Ty);
8193 case NEON::BI__builtin_neon_vnegd_s64:
8194 return Builder.CreateNeg(EmitScalarExpr(E->
getArg(0)),
"vnegd");
8195 case NEON::BI__builtin_neon_vnegh_f16:
8196 return Builder.CreateFNeg(EmitScalarExpr(E->
getArg(0)),
"vnegh");
8197 case NEON::BI__builtin_neon_vpmaxnm_v:
8198 case NEON::BI__builtin_neon_vpmaxnmq_v: {
8199 Int = Intrinsic::aarch64_neon_fmaxnmp;
8200 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops,
"vpmaxnm");
8202 case NEON::BI__builtin_neon_vpminnm_v:
8203 case NEON::BI__builtin_neon_vpminnmq_v: {
8204 Int = Intrinsic::aarch64_neon_fminnmp;
8205 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops,
"vpminnm");
8207 case NEON::BI__builtin_neon_vsqrth_f16: {
8208 Ops.push_back(EmitScalarExpr(E->
getArg(0)));
8210 return EmitNeonCall(CGM.getIntrinsic(Int, HalfTy), Ops,
"vsqrt");
8212 case NEON::BI__builtin_neon_vsqrt_v:
8213 case NEON::BI__builtin_neon_vsqrtq_v: {
8215 Ops[0] = Builder.CreateBitCast(Ops[0], Ty);
8216 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops,
"vsqrt");
8218 case NEON::BI__builtin_neon_vrbit_v:
8219 case NEON::BI__builtin_neon_vrbitq_v: {
8220 Int = Intrinsic::aarch64_neon_rbit;
8221 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops,
"vrbit");
8223 case NEON::BI__builtin_neon_vaddv_u8:
8227 case NEON::BI__builtin_neon_vaddv_s8: {
8228 Int = usgn ? Intrinsic::aarch64_neon_uaddv : Intrinsic::aarch64_neon_saddv;
8230 VTy = llvm::VectorType::get(Int8Ty, 8);
8232 Ops.push_back(EmitScalarExpr(E->
getArg(0)));
8233 Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops,
"vaddv");
8234 return Builder.CreateTrunc(Ops[0], Int8Ty);
8236 case NEON::BI__builtin_neon_vaddv_u16:
8239 case NEON::BI__builtin_neon_vaddv_s16: {
8240 Int = usgn ? Intrinsic::aarch64_neon_uaddv : Intrinsic::aarch64_neon_saddv;
8242 VTy = llvm::VectorType::get(Int16Ty, 4);
8244 Ops.push_back(EmitScalarExpr(E->
getArg(0)));
8245 Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops,
"vaddv");
8246 return Builder.CreateTrunc(Ops[0], Int16Ty);
8248 case NEON::BI__builtin_neon_vaddvq_u8:
8251 case NEON::BI__builtin_neon_vaddvq_s8: {
8252 Int = usgn ? Intrinsic::aarch64_neon_uaddv : Intrinsic::aarch64_neon_saddv;
8254 VTy = llvm::VectorType::get(Int8Ty, 16);
8256 Ops.push_back(EmitScalarExpr(E->
getArg(0)));
8257 Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops,
"vaddv");
8258 return Builder.CreateTrunc(Ops[0], Int8Ty);
8260 case NEON::BI__builtin_neon_vaddvq_u16:
8263 case NEON::BI__builtin_neon_vaddvq_s16: {
8264 Int = usgn ? Intrinsic::aarch64_neon_uaddv : Intrinsic::aarch64_neon_saddv;
8266 VTy = llvm::VectorType::get(Int16Ty, 8);
8268 Ops.push_back(EmitScalarExpr(E->
getArg(0)));
8269 Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops,
"vaddv");
8270 return Builder.CreateTrunc(Ops[0], Int16Ty);
8272 case NEON::BI__builtin_neon_vmaxv_u8: {
8273 Int = Intrinsic::aarch64_neon_umaxv;
8275 VTy = llvm::VectorType::get(Int8Ty, 8);
8277 Ops.push_back(EmitScalarExpr(E->
getArg(0)));
8278 Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops,
"vmaxv");
8279 return Builder.CreateTrunc(Ops[0], Int8Ty);
8281 case NEON::BI__builtin_neon_vmaxv_u16: {
8282 Int = Intrinsic::aarch64_neon_umaxv;
8284 VTy = llvm::VectorType::get(Int16Ty, 4);
8286 Ops.push_back(EmitScalarExpr(E->
getArg(0)));
8287 Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops,
"vmaxv");
8288 return Builder.CreateTrunc(Ops[0], Int16Ty);
8290 case NEON::BI__builtin_neon_vmaxvq_u8: {
8291 Int = Intrinsic::aarch64_neon_umaxv;
8293 VTy = llvm::VectorType::get(Int8Ty, 16);
8295 Ops.push_back(EmitScalarExpr(E->
getArg(0)));
8296 Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops,
"vmaxv");
8297 return Builder.CreateTrunc(Ops[0], Int8Ty);
8299 case NEON::BI__builtin_neon_vmaxvq_u16: {
8300 Int = Intrinsic::aarch64_neon_umaxv;
8302 VTy = llvm::VectorType::get(Int16Ty, 8);
8304 Ops.push_back(EmitScalarExpr(E->
getArg(0)));
8305 Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops,
"vmaxv");
8306 return Builder.CreateTrunc(Ops[0], Int16Ty);
8308 case NEON::BI__builtin_neon_vmaxv_s8: {
8309 Int = Intrinsic::aarch64_neon_smaxv;
8311 VTy = llvm::VectorType::get(Int8Ty, 8);
8313 Ops.push_back(EmitScalarExpr(E->
getArg(0)));
8314 Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops,
"vmaxv");
8315 return Builder.CreateTrunc(Ops[0], Int8Ty);
8317 case NEON::BI__builtin_neon_vmaxv_s16: {
8318 Int = Intrinsic::aarch64_neon_smaxv;
8320 VTy = llvm::VectorType::get(Int16Ty, 4);
8322 Ops.push_back(EmitScalarExpr(E->
getArg(0)));
8323 Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops,
"vmaxv");
8324 return Builder.CreateTrunc(Ops[0], Int16Ty);
8326 case NEON::BI__builtin_neon_vmaxvq_s8: {
8327 Int = Intrinsic::aarch64_neon_smaxv;
8329 VTy = llvm::VectorType::get(Int8Ty, 16);
8331 Ops.push_back(EmitScalarExpr(E->
getArg(0)));
8332 Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops,
"vmaxv");
8333 return Builder.CreateTrunc(Ops[0], Int8Ty);
8335 case NEON::BI__builtin_neon_vmaxvq_s16: {
8336 Int = Intrinsic::aarch64_neon_smaxv;
8338 VTy = llvm::VectorType::get(Int16Ty, 8);
8340 Ops.push_back(EmitScalarExpr(E->
getArg(0)));
8341 Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops,
"vmaxv");
8342 return Builder.CreateTrunc(Ops[0], Int16Ty);
8344 case NEON::BI__builtin_neon_vmaxv_f16: {
8345 Int = Intrinsic::aarch64_neon_fmaxv;
8347 VTy = llvm::VectorType::get(HalfTy, 4);
8349 Ops.push_back(EmitScalarExpr(E->
getArg(0)));
8350 Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops,
"vmaxv");
8351 return Builder.CreateTrunc(Ops[0], HalfTy);
8353 case NEON::BI__builtin_neon_vmaxvq_f16: {
8354 Int = Intrinsic::aarch64_neon_fmaxv;
8356 VTy = llvm::VectorType::get(HalfTy, 8);
8358 Ops.push_back(EmitScalarExpr(E->
getArg(0)));
8359 Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops,
"vmaxv");
8360 return Builder.CreateTrunc(Ops[0], HalfTy);
8362 case NEON::BI__builtin_neon_vminv_u8: {
8363 Int = Intrinsic::aarch64_neon_uminv;
8365 VTy = llvm::VectorType::get(Int8Ty, 8);
8367 Ops.push_back(EmitScalarExpr(E->
getArg(0)));
8368 Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops,
"vminv");
8369 return Builder.CreateTrunc(Ops[0], Int8Ty);
8371 case NEON::BI__builtin_neon_vminv_u16: {
8372 Int = Intrinsic::aarch64_neon_uminv;
8374 VTy = llvm::VectorType::get(Int16Ty, 4);
8376 Ops.push_back(EmitScalarExpr(E->
getArg(0)));
8377 Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops,
"vminv");
8378 return Builder.CreateTrunc(Ops[0], Int16Ty);
8380 case NEON::BI__builtin_neon_vminvq_u8: {
8381 Int = Intrinsic::aarch64_neon_uminv;
8383 VTy = llvm::VectorType::get(Int8Ty, 16);
8385 Ops.push_back(EmitScalarExpr(E->
getArg(0)));
8386 Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops,
"vminv");
8387 return Builder.CreateTrunc(Ops[0], Int8Ty);
8389 case NEON::BI__builtin_neon_vminvq_u16: {
8390 Int = Intrinsic::aarch64_neon_uminv;
8392 VTy = llvm::VectorType::get(Int16Ty, 8);
8394 Ops.push_back(EmitScalarExpr(E->
getArg(0)));
8395 Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops,
"vminv");
8396 return Builder.CreateTrunc(Ops[0], Int16Ty);
8398 case NEON::BI__builtin_neon_vminv_s8: {
8399 Int = Intrinsic::aarch64_neon_sminv;
8401 VTy = llvm::VectorType::get(Int8Ty, 8);
8403 Ops.push_back(EmitScalarExpr(E->
getArg(0)));
8404 Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops,
"vminv");
8405 return Builder.CreateTrunc(Ops[0], Int8Ty);
8407 case NEON::BI__builtin_neon_vminv_s16: {
8408 Int = Intrinsic::aarch64_neon_sminv;
8410 VTy = llvm::VectorType::get(Int16Ty, 4);
8412 Ops.push_back(EmitScalarExpr(E->
getArg(0)));
8413 Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops,
"vminv");
8414 return Builder.CreateTrunc(Ops[0], Int16Ty);
8416 case NEON::BI__builtin_neon_vminvq_s8: {
8417 Int = Intrinsic::aarch64_neon_sminv;
8419 VTy = llvm::VectorType::get(Int8Ty, 16);
8421 Ops.push_back(EmitScalarExpr(E->
getArg(0)));
8422 Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops,
"vminv");
8423 return Builder.CreateTrunc(Ops[0], Int8Ty);
8425 case NEON::BI__builtin_neon_vminvq_s16: {
8426 Int = Intrinsic::aarch64_neon_sminv;
8428 VTy = llvm::VectorType::get(Int16Ty, 8);
8430 Ops.push_back(EmitScalarExpr(E->
getArg(0)));
8431 Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops,
"vminv");
8432 return Builder.CreateTrunc(Ops[0], Int16Ty);
8434 case NEON::BI__builtin_neon_vminv_f16: {
8435 Int = Intrinsic::aarch64_neon_fminv;
8437 VTy = llvm::VectorType::get(HalfTy, 4);
8439 Ops.push_back(EmitScalarExpr(E->
getArg(0)));
8440 Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops,
"vminv");
8441 return Builder.CreateTrunc(Ops[0], HalfTy);
8443 case NEON::BI__builtin_neon_vminvq_f16: {
8444 Int = Intrinsic::aarch64_neon_fminv;
8446 VTy = llvm::VectorType::get(HalfTy, 8);
8448 Ops.push_back(EmitScalarExpr(E->
getArg(0)));
8449 Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops,
"vminv");
8450 return Builder.CreateTrunc(Ops[0], HalfTy);
8452 case NEON::BI__builtin_neon_vmaxnmv_f16: {
8453 Int = Intrinsic::aarch64_neon_fmaxnmv;
8455 VTy = llvm::VectorType::get(HalfTy, 4);
8457 Ops.push_back(EmitScalarExpr(E->
getArg(0)));
8458 Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops,
"vmaxnmv");
8459 return Builder.CreateTrunc(Ops[0], HalfTy);
8461 case NEON::BI__builtin_neon_vmaxnmvq_f16: {
8462 Int = Intrinsic::aarch64_neon_fmaxnmv;
8464 VTy = llvm::VectorType::get(HalfTy, 8);
8466 Ops.push_back(EmitScalarExpr(E->
getArg(0)));
8467 Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops,
"vmaxnmv");
8468 return Builder.CreateTrunc(Ops[0], HalfTy);
8470 case NEON::BI__builtin_neon_vminnmv_f16: {
8471 Int = Intrinsic::aarch64_neon_fminnmv;
8473 VTy = llvm::VectorType::get(HalfTy, 4);
8475 Ops.push_back(EmitScalarExpr(E->
getArg(0)));
8476 Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops,
"vminnmv");
8477 return Builder.CreateTrunc(Ops[0], HalfTy);
8479 case NEON::BI__builtin_neon_vminnmvq_f16: {
8480 Int = Intrinsic::aarch64_neon_fminnmv;
8482 VTy = llvm::VectorType::get(HalfTy, 8);
8484 Ops.push_back(EmitScalarExpr(E->
getArg(0)));
8485 Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops,
"vminnmv");
8486 return Builder.CreateTrunc(Ops[0], HalfTy);
8488 case NEON::BI__builtin_neon_vmul_n_f64: {
8489 Ops[0] = Builder.CreateBitCast(Ops[0], DoubleTy);
8490 Value *RHS = Builder.CreateBitCast(EmitScalarExpr(E->
getArg(1)), DoubleTy);
8491 return Builder.CreateFMul(Ops[0], RHS);
8493 case NEON::BI__builtin_neon_vaddlv_u8: {
8494 Int = Intrinsic::aarch64_neon_uaddlv;
8496 VTy = llvm::VectorType::get(Int8Ty, 8);
8498 Ops.push_back(EmitScalarExpr(E->
getArg(0)));
8499 Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops,
"vaddlv");
8500 return Builder.CreateTrunc(Ops[0], Int16Ty);
8502 case NEON::BI__builtin_neon_vaddlv_u16: {
8503 Int = Intrinsic::aarch64_neon_uaddlv;
8505 VTy = llvm::VectorType::get(Int16Ty, 4);
8507 Ops.push_back(EmitScalarExpr(E->
getArg(0)));
8508 return EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops,
"vaddlv");
8510 case NEON::BI__builtin_neon_vaddlvq_u8: {
8511 Int = Intrinsic::aarch64_neon_uaddlv;
8513 VTy = llvm::VectorType::get(Int8Ty, 16);
8515 Ops.push_back(EmitScalarExpr(E->
getArg(0)));
8516 Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops,
"vaddlv");
8517 return Builder.CreateTrunc(Ops[0], Int16Ty);
8519 case NEON::BI__builtin_neon_vaddlvq_u16: {
8520 Int = Intrinsic::aarch64_neon_uaddlv;
8522 VTy = llvm::VectorType::get(Int16Ty, 8);
8524 Ops.push_back(EmitScalarExpr(E->
getArg(0)));
8525 return EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops,
"vaddlv");
8527 case NEON::BI__builtin_neon_vaddlv_s8: {
8528 Int = Intrinsic::aarch64_neon_saddlv;
8530 VTy = llvm::VectorType::get(Int8Ty, 8);
8532 Ops.push_back(EmitScalarExpr(E->
getArg(0)));
8533 Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops,
"vaddlv");
8534 return Builder.CreateTrunc(Ops[0], Int16Ty);
8536 case NEON::BI__builtin_neon_vaddlv_s16: {
8537 Int = Intrinsic::aarch64_neon_saddlv;
8539 VTy = llvm::VectorType::get(Int16Ty, 4);
8541 Ops.push_back(EmitScalarExpr(E->
getArg(0)));
8542 return EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops,
"vaddlv");
8544 case NEON::BI__builtin_neon_vaddlvq_s8: {
8545 Int = Intrinsic::aarch64_neon_saddlv;
8547 VTy = llvm::VectorType::get(Int8Ty, 16);
8549 Ops.push_back(EmitScalarExpr(E->
getArg(0)));
8550 Ops[0] = EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops,
"vaddlv");
8551 return Builder.CreateTrunc(Ops[0], Int16Ty);
8553 case NEON::BI__builtin_neon_vaddlvq_s16: {
8554 Int = Intrinsic::aarch64_neon_saddlv;
8556 VTy = llvm::VectorType::get(Int16Ty, 8);
8558 Ops.push_back(EmitScalarExpr(E->
getArg(0)));
8559 return EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops,
"vaddlv");
8561 case NEON::BI__builtin_neon_vsri_n_v:
8562 case NEON::BI__builtin_neon_vsriq_n_v: {
8563 Int = Intrinsic::aarch64_neon_vsri;
8564 llvm::Function *Intrin = CGM.getIntrinsic(Int, Ty);
8565 return EmitNeonCall(Intrin, Ops,
"vsri_n");
8567 case NEON::BI__builtin_neon_vsli_n_v:
8568 case NEON::BI__builtin_neon_vsliq_n_v: {
8569 Int = Intrinsic::aarch64_neon_vsli;
8570 llvm::Function *Intrin = CGM.getIntrinsic(Int, Ty);
8571 return EmitNeonCall(Intrin, Ops,
"vsli_n");
8573 case NEON::BI__builtin_neon_vsra_n_v:
8574 case NEON::BI__builtin_neon_vsraq_n_v:
8575 Ops[0] = Builder.CreateBitCast(Ops[0], Ty);
8576 Ops[1] = EmitNeonRShiftImm(Ops[1], Ops[2], Ty, usgn,
"vsra_n");
8577 return Builder.CreateAdd(Ops[0], Ops[1]);
8578 case NEON::BI__builtin_neon_vrsra_n_v:
8579 case NEON::BI__builtin_neon_vrsraq_n_v: {
8580 Int = usgn ? Intrinsic::aarch64_neon_urshl : Intrinsic::aarch64_neon_srshl;
8582 TmpOps.push_back(Ops[1]);
8583 TmpOps.push_back(Ops[2]);
8584 Function* F = CGM.getIntrinsic(Int, Ty);
8585 llvm::Value *tmp = EmitNeonCall(F, TmpOps,
"vrshr_n", 1,
true);
8586 Ops[0] = Builder.CreateBitCast(Ops[0], VTy);
8587 return Builder.CreateAdd(Ops[0], tmp);
8589 case NEON::BI__builtin_neon_vld1_v:
8590 case NEON::BI__builtin_neon_vld1q_v: {
8591 Ops[0] = Builder.CreateBitCast(Ops[0], llvm::PointerType::getUnqual(VTy));
8593 BuiltinID == NEON::BI__builtin_neon_vld1_v ? 8 : 16);
8594 return Builder.CreateAlignedLoad(VTy, Ops[0], Alignment);
8596 case NEON::BI__builtin_neon_vst1_v:
8597 case NEON::BI__builtin_neon_vst1q_v:
8598 Ops[0] = Builder.CreateBitCast(Ops[0], llvm::PointerType::getUnqual(VTy));
8599 Ops[1] = Builder.CreateBitCast(Ops[1], VTy);
8600 return Builder.CreateDefaultAlignedStore(Ops[1], Ops[0]);
8601 case NEON::BI__builtin_neon_vld1_lane_v:
8602 case NEON::BI__builtin_neon_vld1q_lane_v: {
8603 Ops[1] = Builder.CreateBitCast(Ops[1], Ty);
8604 Ty = llvm::PointerType::getUnqual(VTy->getElementType());
8605 Ops[0] = Builder.CreateBitCast(Ops[0], Ty);
8607 BuiltinID == NEON::BI__builtin_neon_vld1_lane_v ? 8 : 16);
8609 Builder.CreateAlignedLoad(VTy->getElementType(), Ops[0], Alignment);
8610 return Builder.CreateInsertElement(Ops[1], Ops[0], Ops[2],
"vld1_lane");
8612 case NEON::BI__builtin_neon_vld1_dup_v:
8613 case NEON::BI__builtin_neon_vld1q_dup_v: {
8614 Value *V = UndefValue::get(Ty);
8615 Ty = llvm::PointerType::getUnqual(VTy->getElementType());
8616 Ops[0] = Builder.CreateBitCast(Ops[0], Ty);
8618 BuiltinID == NEON::BI__builtin_neon_vld1_dup_v ? 8 : 16);
8620 Builder.CreateAlignedLoad(VTy->getElementType(), Ops[0], Alignment);
8621 llvm::Constant *CI = ConstantInt::get(Int32Ty, 0);
8622 Ops[0] = Builder.CreateInsertElement(V, Ops[0], CI);
8623 return EmitNeonSplat(Ops[0], CI);
8625 case NEON::BI__builtin_neon_vst1_lane_v:
8626 case NEON::BI__builtin_neon_vst1q_lane_v:
8627 Ops[1] = Builder.CreateBitCast(Ops[1], Ty);
8628 Ops[1] = Builder.CreateExtractElement(Ops[1], Ops[2]);
8629 Ty = llvm::PointerType::getUnqual(Ops[1]->getType());
8630 return Builder.CreateDefaultAlignedStore(Ops[1],
8631 Builder.CreateBitCast(Ops[0], Ty));
8632 case NEON::BI__builtin_neon_vld2_v:
8633 case NEON::BI__builtin_neon_vld2q_v: {
8634 llvm::Type *PTy = llvm::PointerType::getUnqual(VTy);
8635 Ops[1] = Builder.CreateBitCast(Ops[1], PTy);
8637 Function *F = CGM.getIntrinsic(Intrinsic::aarch64_neon_ld2, Tys);
8638 Ops[1] = Builder.CreateCall(F, Ops[1],
"vld2");
8639 Ops[0] = Builder.CreateBitCast(Ops[0],
8640 llvm::PointerType::getUnqual(Ops[1]->getType()));
8641 return Builder.CreateDefaultAlignedStore(Ops[1], Ops[0]);
8643 case NEON::BI__builtin_neon_vld3_v:
8644 case NEON::BI__builtin_neon_vld3q_v: {
8645 llvm::Type *PTy = llvm::PointerType::getUnqual(VTy);
8646 Ops[1] = Builder.CreateBitCast(Ops[1], PTy);
8648 Function *F = CGM.getIntrinsic(Intrinsic::aarch64_neon_ld3, Tys);
8649 Ops[1] = Builder.CreateCall(F, Ops[1],
"vld3");
8650 Ops[0] = Builder.CreateBitCast(Ops[0],
8651 llvm::PointerType::getUnqual(Ops[1]->getType()));
8652 return Builder.CreateDefaultAlignedStore(Ops[1], Ops[0]);
8654 case NEON::BI__builtin_neon_vld4_v:
8655 case NEON::BI__builtin_neon_vld4q_v: {
8656 llvm::Type *PTy = llvm::PointerType::getUnqual(VTy);
8657 Ops[1] = Builder.CreateBitCast(Ops[1], PTy);
8659 Function *F = CGM.getIntrinsic(Intrinsic::aarch64_neon_ld4, Tys);
8660 Ops[1] = Builder.CreateCall(F, Ops[1],
"vld4");
8661 Ops[0] = Builder.CreateBitCast(Ops[0],
8662 llvm::PointerType::getUnqual(Ops[1]->getType()));
8663 return Builder.CreateDefaultAlignedStore(Ops[1], Ops[0]);
8665 case NEON::BI__builtin_neon_vld2_dup_v:
8666 case NEON::BI__builtin_neon_vld2q_dup_v: {
8668 llvm::PointerType::getUnqual(VTy->getElementType());
8669 Ops[1] = Builder.CreateBitCast(Ops[1], PTy);
8671 Function *F = CGM.getIntrinsic(Intrinsic::aarch64_neon_ld2r, Tys);
8672 Ops[1] = Builder.CreateCall(F, Ops[1],
"vld2");
8673 Ops[0] = Builder.CreateBitCast(Ops[0],
8674 llvm::PointerType::getUnqual(Ops[1]->getType()));
8675 return Builder.CreateDefaultAlignedStore(Ops[1], Ops[0]);
8677 case NEON::BI__builtin_neon_vld3_dup_v:
8678 case NEON::BI__builtin_neon_vld3q_dup_v: {
8680 llvm::PointerType::getUnqual(VTy->getElementType());
8681 Ops[1] = Builder.CreateBitCast(Ops[1], PTy);
8683 Function *F = CGM.getIntrinsic(Intrinsic::aarch64_neon_ld3r, Tys);
8684 Ops[1] = Builder.CreateCall(F, Ops[1],
"vld3");
8685 Ops[0] = Builder.CreateBitCast(Ops[0],
8686 llvm::PointerType::getUnqual(Ops[1]->getType()));
8687 return Builder.CreateDefaultAlignedStore(Ops[1], Ops[0]);
8689 case NEON::BI__builtin_neon_vld4_dup_v:
8690 case NEON::BI__builtin_neon_vld4q_dup_v: {
8692 llvm::PointerType::getUnqual(VTy->getElementType());
8693 Ops[1] = Builder.CreateBitCast(Ops[1], PTy);
8695 Function *F = CGM.getIntrinsic(Intrinsic::aarch64_neon_ld4r, Tys);
8696 Ops[1] = Builder.CreateCall(F, Ops[1],
"vld4");
8697 Ops[0] = Builder.CreateBitCast(Ops[0],
8698 llvm::PointerType::getUnqual(Ops[1]->getType()));
8699 return Builder.CreateDefaultAlignedStore(Ops[1], Ops[0]);
8701 case NEON::BI__builtin_neon_vld2_lane_v:
8702 case NEON::BI__builtin_neon_vld2q_lane_v: {
8703 llvm::Type *Tys[2] = { VTy, Ops[1]->getType() };
8704 Function *F = CGM.getIntrinsic(Intrinsic::aarch64_neon_ld2lane, Tys);
8705 Ops.push_back(Ops[1]);
8706 Ops.erase(Ops.begin()+1);
8707 Ops[1] = Builder.CreateBitCast(Ops[1], Ty);
8708 Ops[2] = Builder.CreateBitCast(Ops[2], Ty);
8709 Ops[3] = Builder.CreateZExt(Ops[3], Int64Ty);
8710 Ops[1] = Builder.CreateCall(F, makeArrayRef(Ops).slice(1),
"vld2_lane");
8711 Ty = llvm::PointerType::getUnqual(Ops[1]->getType());
8712 Ops[0] = Builder.CreateBitCast(Ops[0], Ty);
8713 return Builder.CreateDefaultAlignedStore(Ops[1], Ops[0]);
8715 case NEON::BI__builtin_neon_vld3_lane_v:
8716 case NEON::BI__builtin_neon_vld3q_lane_v: {
8717 llvm::Type *Tys[2] = { VTy, Ops[1]->getType() };
8718 Function *F = CGM.getIntrinsic(Intrinsic::aarch64_neon_ld3lane, Tys);
8719 Ops.push_back(Ops[1]);
8720 Ops.erase(Ops.begin()+1);
8721 Ops[1] = Builder.CreateBitCast(Ops[1], Ty);
8722 Ops[2] = Builder.CreateBitCast(Ops[2], Ty);
8723 Ops[3] = Builder.CreateBitCast(Ops[3], Ty);
8724 Ops[4] = Builder.CreateZExt(Ops[4], Int64Ty);
8725 Ops[1] = Builder.CreateCall(F, makeArrayRef(Ops).slice(1),
"vld3_lane");
8726 Ty = llvm::PointerType::getUnqual(Ops[1]->getType());
8727 Ops[0] = Builder.CreateBitCast(Ops[0], Ty);
8728 return Builder.CreateDefaultAlignedStore(Ops[1], Ops[0]);
8730 case NEON::BI__builtin_neon_vld4_lane_v:
8731 case NEON::BI__builtin_neon_vld4q_lane_v: {
8732 llvm::Type *Tys[2] = { VTy, Ops[1]->getType() };
8733 Function *F = CGM.getIntrinsic(Intrinsic::aarch64_neon_ld4lane, Tys);
8734 Ops.push_back(Ops[1]);
8735 Ops.erase(Ops.begin()+1);
8736 Ops[1] = Builder.CreateBitCast(Ops[1], Ty);
8737 Ops[2] = Builder.CreateBitCast(Ops[2], Ty);
8738 Ops[3] = Builder.CreateBitCast(Ops[3], Ty);
8739 Ops[4] = Builder.CreateBitCast(Ops[4], Ty);
8740 Ops[5] = Builder.CreateZExt(Ops[5], Int64Ty);
8741 Ops[1] = Builder.CreateCall(F, makeArrayRef(Ops).slice(1),
"vld4_lane");
8742 Ty = llvm::PointerType::getUnqual(Ops[1]->getType());
8743 Ops[0] = Builder.CreateBitCast(Ops[0], Ty);
8744 return Builder.CreateDefaultAlignedStore(Ops[1], Ops[0]);
8746 case NEON::BI__builtin_neon_vst2_v:
8747 case NEON::BI__builtin_neon_vst2q_v: {
8748 Ops.push_back(Ops[0]);
8749 Ops.erase(Ops.begin());
8750 llvm::Type *Tys[2] = { VTy, Ops[2]->getType() };
8751 return EmitNeonCall(CGM.getIntrinsic(Intrinsic::aarch64_neon_st2, Tys),
8754 case NEON::BI__builtin_neon_vst2_lane_v:
8755 case NEON::BI__builtin_neon_vst2q_lane_v: {
8756 Ops.push_back(Ops[0]);
8757 Ops.erase(Ops.begin());
8758 Ops[2] = Builder.CreateZExt(Ops[2], Int64Ty);
8759 llvm::Type *Tys[2] = { VTy, Ops[3]->getType() };
8760 return EmitNeonCall(CGM.getIntrinsic(Intrinsic::aarch64_neon_st2lane, Tys),
8763 case NEON::BI__builtin_neon_vst3_v:
8764 case NEON::BI__builtin_neon_vst3q_v: {
8765 Ops.push_back(Ops[0]);
8766 Ops.erase(Ops.begin());
8767 llvm::Type *Tys[2] = { VTy, Ops[3]->getType() };
8768 return EmitNeonCall(CGM.getIntrinsic(Intrinsic::aarch64_neon_st3, Tys),
8771 case NEON::BI__builtin_neon_vst3_lane_v:
8772 case NEON::BI__builtin_neon_vst3q_lane_v: {
8773 Ops.push_back(Ops[0]);
8774 Ops.erase(Ops.begin());
8775 Ops[3] = Builder.CreateZExt(Ops[3], Int64Ty);
8776 llvm::Type *Tys[2] = { VTy, Ops[4]->getType() };
8777 return EmitNeonCall(CGM.getIntrinsic(Intrinsic::aarch64_neon_st3lane, Tys),
8780 case NEON::BI__builtin_neon_vst4_v:
8781 case NEON::BI__builtin_neon_vst4q_v: {
8782 Ops.push_back(Ops[0]);
8783 Ops.erase(Ops.begin());
8784 llvm::Type *Tys[2] = { VTy, Ops[4]->getType() };
8785 return EmitNeonCall(CGM.getIntrinsic(Intrinsic::aarch64_neon_st4, Tys),
8788 case NEON::BI__builtin_neon_vst4_lane_v:
8789 case NEON::BI__builtin_neon_vst4q_lane_v: {
8790 Ops.push_back(Ops[0]);
8791 Ops.erase(Ops.begin());
8792 Ops[4] = Builder.CreateZExt(Ops[4], Int64Ty);
8793 llvm::Type *Tys[2] = { VTy, Ops[5]->getType() };
8794 return EmitNeonCall(CGM.getIntrinsic(Intrinsic::aarch64_neon_st4lane, Tys),
8797 case NEON::BI__builtin_neon_vtrn_v:
8798 case NEON::BI__builtin_neon_vtrnq_v: {
8799 Ops[0] = Builder.CreateBitCast(Ops[0], llvm::PointerType::getUnqual(Ty));
8800 Ops[1] = Builder.CreateBitCast(Ops[1], Ty);
8801 Ops[2] = Builder.CreateBitCast(Ops[2], Ty);
8802 Value *SV =
nullptr;
8804 for (
unsigned vi = 0; vi != 2; ++vi) {
8806 for (
unsigned i = 0, e = VTy->getNumElements(); i != e; i += 2) {
8807 Indices.push_back(i+vi);
8808 Indices.push_back(i+e+vi);
8810 Value *Addr = Builder.CreateConstInBoundsGEP1_32(Ty, Ops[0], vi);
8811 SV = Builder.CreateShuffleVector(Ops[1], Ops[2], Indices,
"vtrn");
8812 SV = Builder.CreateDefaultAlignedStore(SV, Addr);
8816 case NEON::BI__builtin_neon_vuzp_v:
8817 case NEON::BI__builtin_neon_vuzpq_v: {
8818 Ops[0] = Builder.CreateBitCast(Ops[0], llvm::PointerType::getUnqual(Ty));
8819 Ops[1] = Builder.CreateBitCast(Ops[1], Ty);
8820 Ops[2] = Builder.CreateBitCast(Ops[2], Ty);
8821 Value *SV =
nullptr;
8823 for (
unsigned vi = 0; vi != 2; ++vi) {
8825 for (
unsigned i = 0, e = VTy->getNumElements(); i != e; ++i)
8826 Indices.push_back(2*i+vi);
8828 Value *Addr = Builder.CreateConstInBoundsGEP1_32(Ty, Ops[0], vi);
8829 SV = Builder.CreateShuffleVector(Ops[1], Ops[2], Indices,
"vuzp");
8830 SV = Builder.CreateDefaultAlignedStore(SV, Addr);
8834 case NEON::BI__builtin_neon_vzip_v:
8835 case NEON::BI__builtin_neon_vzipq_v: {
8836 Ops[0] = Builder.CreateBitCast(Ops[0], llvm::PointerType::getUnqual(Ty));
8837 Ops[1] = Builder.CreateBitCast(Ops[1], Ty);
8838 Ops[2] = Builder.CreateBitCast(Ops[2], Ty);
8839 Value *SV =
nullptr;
8841 for (
unsigned vi = 0; vi != 2; ++vi) {
8843 for (
unsigned i = 0, e = VTy->getNumElements(); i != e; i += 2) {
8844 Indices.push_back((i + vi*e) >> 1);
8845 Indices.push_back(((i + vi*e) >> 1)+e);
8847 Value *Addr = Builder.CreateConstInBoundsGEP1_32(Ty, Ops[0], vi);
8848 SV = Builder.CreateShuffleVector(Ops[1], Ops[2], Indices,
"vzip");
8849 SV = Builder.CreateDefaultAlignedStore(SV, Addr);
8853 case NEON::BI__builtin_neon_vqtbl1q_v: {
8854 return EmitNeonCall(CGM.getIntrinsic(Intrinsic::aarch64_neon_tbl1, Ty),
8857 case NEON::BI__builtin_neon_vqtbl2q_v: {
8858 return EmitNeonCall(CGM.getIntrinsic(Intrinsic::aarch64_neon_tbl2, Ty),
8861 case NEON::BI__builtin_neon_vqtbl3q_v: {
8862 return EmitNeonCall(CGM.getIntrinsic(Intrinsic::aarch64_neon_tbl3, Ty),
8865 case NEON::BI__builtin_neon_vqtbl4q_v: {
8866 return EmitNeonCall(CGM.getIntrinsic(Intrinsic::aarch64_neon_tbl4, Ty),
8869 case NEON::BI__builtin_neon_vqtbx1q_v: {
8870 return EmitNeonCall(CGM.getIntrinsic(Intrinsic::aarch64_neon_tbx1, Ty),
8873 case NEON::BI__builtin_neon_vqtbx2q_v: {
8874 return EmitNeonCall(CGM.getIntrinsic(Intrinsic::aarch64_neon_tbx2, Ty),
8877 case NEON::BI__builtin_neon_vqtbx3q_v: {
8878 return EmitNeonCall(CGM.getIntrinsic(Intrinsic::aarch64_neon_tbx3, Ty),
8881 case NEON::BI__builtin_neon_vqtbx4q_v: {
8882 return EmitNeonCall(CGM.getIntrinsic(Intrinsic::aarch64_neon_tbx4, Ty),
8885 case NEON::BI__builtin_neon_vsqadd_v:
8886 case NEON::BI__builtin_neon_vsqaddq_v: {
8887 Int = Intrinsic::aarch64_neon_usqadd;
8888 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops,
"vsqadd");
8890 case NEON::BI__builtin_neon_vuqadd_v:
8891 case NEON::BI__builtin_neon_vuqaddq_v: {
8892 Int = Intrinsic::aarch64_neon_suqadd;
8893 return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops,
"vuqadd");
8895 case AArch64::BI__iso_volatile_load8:
8896 case AArch64::BI__iso_volatile_load16:
8897 case AArch64::BI__iso_volatile_load32:
8898 case AArch64::BI__iso_volatile_load64:
8899 return EmitISOVolatileLoad(E);
8900 case AArch64::BI__iso_volatile_store8:
8901 case AArch64::BI__iso_volatile_store16:
8902 case AArch64::BI__iso_volatile_store32:
8903 case AArch64::BI__iso_volatile_store64:
8904 return EmitISOVolatileStore(E);
8905 case AArch64::BI_BitScanForward:
8906 case AArch64::BI_BitScanForward64:
8907 return EmitMSVCBuiltinExpr(MSVCIntrin::_BitScanForward, E);
8908 case AArch64::BI_BitScanReverse:
8909 case AArch64::BI_BitScanReverse64:
8910 return EmitMSVCBuiltinExpr(MSVCIntrin::_BitScanReverse, E);
8911 case AArch64::BI_InterlockedAnd64:
8912 return EmitMSVCBuiltinExpr(MSVCIntrin::_InterlockedAnd, E);
8913 case AArch64::BI_InterlockedExchange64:
8914 return EmitMSVCBuiltinExpr(MSVCIntrin::_InterlockedExchange, E);
8915 case AArch64::BI_InterlockedExchangeAdd64:
8916 return EmitMSVCBuiltinExpr(MSVCIntrin::_InterlockedExchangeAdd, E);
8917 case AArch64::BI_InterlockedExchangeSub64:
8918 return EmitMSVCBuiltinExpr(MSVCIntrin::_InterlockedExchangeSub, E);
8919 case AArch64::BI_InterlockedOr64:
8920 return EmitMSVCBuiltinExpr(MSVCIntrin::_InterlockedOr, E);
8921 case AArch64::BI_InterlockedXor64:
8922 return EmitMSVCBuiltinExpr(MSVCIntrin::_InterlockedXor, E);
8923 case AArch64::BI_InterlockedDecrement64:
8924 return EmitMSVCBuiltinExpr(MSVCIntrin::_InterlockedDecrement, E);
8925 case AArch64::BI_InterlockedIncrement64:
8926 return EmitMSVCBuiltinExpr(MSVCIntrin::_InterlockedIncrement, E);
8927 case AArch64::BI_InterlockedExchangeAdd8_acq:
8928 case AArch64::BI_InterlockedExchangeAdd16_acq:
8929 case AArch64::BI_InterlockedExchangeAdd_acq:
8930 case AArch64::BI_InterlockedExchangeAdd64_acq:
8931 return EmitMSVCBuiltinExpr(MSVCIntrin::_InterlockedExchangeAdd_acq, E);
8932 case AArch64::BI_InterlockedExchangeAdd8_rel:
8933 case AArch64::BI_InterlockedExchangeAdd16_rel:
8934 case AArch64::BI_InterlockedExchangeAdd_rel:
8935 case AArch64::BI_InterlockedExchangeAdd64_rel:
8936 return EmitMSVCBuiltinExpr(MSVCIntrin::_InterlockedExchangeAdd_rel, E);
8937 case AArch64::BI_InterlockedExchangeAdd8_nf:
8938 case AArch64::BI_InterlockedExchangeAdd16_nf:
8939 case AArch64::BI_InterlockedExchangeAdd_nf:
8940 case AArch64::BI_InterlockedExchangeAdd64_nf:
8941 return EmitMSVCBuiltinExpr(MSVCIntrin::_InterlockedExchangeAdd_nf, E);
8942 case AArch64::BI_InterlockedExchange8_acq:
8943 case AArch64::BI_InterlockedExchange16_acq:
8944 case AArch64::BI_InterlockedExchange_acq:
8945 case AArch64::BI_InterlockedExchange64_acq:
8946 return EmitMSVCBuiltinExpr(MSVCIntrin::_InterlockedExchange_acq, E);
8947 case AArch64::BI_InterlockedExchange8_rel:
8948 case AArch64::BI_InterlockedExchange16_rel:
8949 case AArch64::BI_InterlockedExchange_rel:
8950 case AArch64::BI_InterlockedExchange64_rel:
8951 return EmitMSVCBuiltinExpr(MSVCIntrin::_InterlockedExchange_rel, E);
8952 case AArch64::BI_InterlockedExchange8_nf:
8953 case AArch64::BI_InterlockedExchange16_nf:
8954 case AArch64::BI_InterlockedExchange_nf:
8955 case AArch64::BI_InterlockedExchange64_nf:
8956 return EmitMSVCBuiltinExpr(MSVCIntrin::_InterlockedExchange_nf, E);
8957 case AArch64::BI_InterlockedCompareExchange8_acq:
8958 case AArch64::BI_InterlockedCompareExchange16_acq:
8959 case AArch64::BI_InterlockedCompareExchange_acq:
8960 case AArch64::BI_InterlockedCompareExchange64_acq:
8961 return EmitMSVCBuiltinExpr(MSVCIntrin::_InterlockedCompareExchange_acq, E);
8962 case AArch64::BI_InterlockedCompareExchange8_rel:
8963 case AArch64::BI_InterlockedCompareExchange16_rel:
8964 case AArch64::BI_InterlockedCompareExchange_rel:
8965 case AArch64::BI_InterlockedCompareExchange64_rel:
8966 return EmitMSVCBuiltinExpr(MSVCIntrin::_InterlockedCompareExchange_rel, E);
8967 case AArch64::BI_InterlockedCompareExchange8_nf:
8968 case AArch64::BI_InterlockedCompareExchange16_nf:
8969 case AArch64::BI_InterlockedCompareExchange_nf:
8970 case AArch64::BI_InterlockedCompareExchange64_nf:
8971 return EmitMSVCBuiltinExpr(MSVCIntrin::_InterlockedCompareExchange_nf, E);
8972 case AArch64::BI_InterlockedOr8_acq:
8973 case AArch64::BI_InterlockedOr16_acq:
8974 case AArch64::BI_InterlockedOr_acq:
8975 case AArch64::BI_InterlockedOr64_acq:
8976 return EmitMSVCBuiltinExpr(MSVCIntrin::_InterlockedOr_acq, E);
8977 case AArch64::BI_InterlockedOr8_rel:
8978 case AArch64::BI_InterlockedOr16_rel:
8979 case AArch64::BI_InterlockedOr_rel:
8980 case AArch64::BI_InterlockedOr64_rel:
8981 return EmitMSVCBuiltinExpr(MSVCIntrin::_InterlockedOr_rel, E);
8982 case AArch64::BI_InterlockedOr8_nf:
8983 case AArch64::BI_InterlockedOr16_nf:
8984 case AArch64::BI_InterlockedOr_nf:
8985 case AArch64::BI_InterlockedOr64_nf:
8986 return EmitMSVCBuiltinExpr(MSVCIntrin::_InterlockedOr_nf, E);
8987 case AArch64::BI_InterlockedXor8_acq:
8988 case AArch64::BI_InterlockedXor16_acq:
8989 case AArch64::BI_InterlockedXor_acq:
8990 case AArch64::BI_InterlockedXor64_acq:
8991 return EmitMSVCBuiltinExpr(MSVCIntrin::_InterlockedXor_acq, E);
8992 case AArch64::BI_InterlockedXor8_rel:
8993 case AArch64::BI_InterlockedXor16_rel:
8994 case AArch64::BI_InterlockedXor_rel:
8995 case AArch64::BI_InterlockedXor64_rel:
8996 return EmitMSVCBuiltinExpr(MSVCIntrin::_InterlockedXor_rel, E);
8997 case AArch64::BI_InterlockedXor8_nf:
8998 case AArch64::BI_InterlockedXor16_nf:
8999 case AArch64::BI_InterlockedXor_nf:
9000 case AArch64::BI_InterlockedXor64_nf:
9001 return EmitMSVCBuiltinExpr(MSVCIntrin::_InterlockedXor_nf, E);
9002 case AArch64::BI_InterlockedAnd8_acq:
9003 case AArch64::BI_InterlockedAnd16_acq:
9004 case AArch64::BI_InterlockedAnd_acq:
9005 case AArch64::BI_InterlockedAnd64_acq:
9006 return EmitMSVCBuiltinExpr(MSVCIntrin::_InterlockedAnd_acq, E);
9007 case AArch64::BI_InterlockedAnd8_rel:
9008 case AArch64::BI_InterlockedAnd16_rel:
9009 case AArch64::BI_InterlockedAnd_rel:
9010 case AArch64::BI_InterlockedAnd64_rel:
9011 return EmitMSVCBuiltinExpr(MSVCIntrin::_InterlockedAnd_rel, E);
9012 case AArch64::BI_InterlockedAnd8_nf:
9013 case AArch64::BI_InterlockedAnd16_nf:
9014 case AArch64::BI_InterlockedAnd_nf:
9015 case AArch64::BI_InterlockedAnd64_nf:
9016 return EmitMSVCBuiltinExpr(MSVCIntrin::_InterlockedAnd_nf, E);
9017 case AArch64::BI_InterlockedIncrement16_acq:
9018 case AArch64::BI_InterlockedIncrement_acq:
9019 case AArch64::BI_InterlockedIncrement64_acq:
9020 return EmitMSVCBuiltinExpr(MSVCIntrin::_InterlockedIncrement_acq, E);
9021 case AArch64::BI_InterlockedIncrement16_rel:
9022 case AArch64::BI_InterlockedIncrement_rel:
9023 case AArch64::BI_InterlockedIncrement64_rel:
9024 return EmitMSVCBuiltinExpr(MSVCIntrin::_InterlockedIncrement_rel, E);
9025 case AArch64::BI_InterlockedIncrement16_nf:
9026 case AArch64::BI_InterlockedIncrement_nf:
9027 case AArch64::BI_InterlockedIncrement64_nf:
9028 return EmitMSVCBuiltinExpr(MSVCIntrin::_InterlockedIncrement_nf, E);
9029 case AArch64::BI_InterlockedDecrement16_acq:
9030 case AArch64::BI_InterlockedDecrement_acq:
9031 case AArch64::BI_InterlockedDecrement64_acq:
9032 return EmitMSVCBuiltinExpr(MSVCIntrin::_InterlockedDecrement_acq, E);
9033 case AArch64::BI_InterlockedDecrement16_rel:
9034 case AArch64::BI_InterlockedDecrement_rel:
9035 case AArch64::BI_InterlockedDecrement64_rel:
9036 return EmitMSVCBuiltinExpr(MSVCIntrin::_InterlockedDecrement_rel, E);
9037 case AArch64::BI_InterlockedDecrement16_nf:
9038 case AArch64::BI_InterlockedDecrement_nf:
9039 case AArch64::BI_InterlockedDecrement64_nf:
9040 return EmitMSVCBuiltinExpr(MSVCIntrin::_InterlockedDecrement_nf, E);
9042 case AArch64::BI_InterlockedAdd: {
9045 AtomicRMWInst *RMWI = Builder.CreateAtomicRMW(
9046 AtomicRMWInst::Add, Arg0, Arg1,
9047 llvm::AtomicOrdering::SequentiallyConsistent);
9048 return Builder.CreateAdd(RMWI, Arg1);
9055 assert((Ops.size() & (Ops.size() - 1)) == 0 &&
9056 "Not a power-of-two sized vector!");
9057 bool AllConstants =
true;
9058 for (
unsigned i = 0, e = Ops.size(); i != e && AllConstants; ++i)
9059 AllConstants &= isa<Constant>(Ops[i]);
9064 for (
unsigned i = 0, e = Ops.size(); i != e; ++i)
9065 CstOps.push_back(cast<Constant>(Ops[i]));
9066 return llvm::ConstantVector::get(CstOps);
9071 llvm::UndefValue::get(llvm::VectorType::get(Ops[0]->getType(), Ops.size()));
9073 for (
unsigned i = 0, e = Ops.size(); i != e; ++i)
9074 Result = Builder.CreateInsertElement(Result, Ops[i], Builder.getInt32(i));
9083 llvm::VectorType *MaskTy = llvm::VectorType::get(CGF.
Builder.getInt1Ty(),
9084 cast<IntegerType>(Mask->getType())->getBitWidth());
9090 uint32_t Indices[4];
9091 for (
unsigned i = 0; i != NumElts; ++i)
9093 MaskVec = CGF.
Builder.CreateShuffleVector(MaskVec, MaskVec,
9094 makeArrayRef(Indices, NumElts),
9105 llvm::PointerType::getUnqual(Ops[1]->getType()));
9108 Ops[1]->getType()->getVectorNumElements());
9110 return CGF.
Builder.CreateMaskedStore(Ops[1], Ptr, Align, MaskVec);
9117 llvm::PointerType::getUnqual(Ops[1]->getType()));
9120 Ops[1]->getType()->getVectorNumElements());
9122 return CGF.
Builder.CreateMaskedLoad(Ptr, Align, MaskVec, Ops[1]);
9128 llvm::Type *PtrTy = ResultTy->getVectorElementType();
9132 llvm::PointerType::getUnqual(PtrTy));
9135 ResultTy->getVectorNumElements());
9137 llvm::Function *F = CGF.
CGM.
getIntrinsic(Intrinsic::masked_expandload,
9139 return CGF.
Builder.CreateCall(F, { Ptr, MaskVec, Ops[1] });
9145 llvm::Type *PtrTy = ResultTy->getVectorElementType();
9149 llvm::PointerType::getUnqual(PtrTy));
9152 ResultTy->getVectorNumElements());
9154 llvm::Function *F = CGF.
CGM.
getIntrinsic(Intrinsic::masked_compressstore,
9156 return CGF.
Builder.CreateCall(F, { Ops[1], Ptr, MaskVec });
9161 bool InvertLHS =
false) {
9162 unsigned NumElts = Ops[0]->getType()->getIntegerBitWidth();
9167 LHS = CGF.
Builder.CreateNot(LHS);
9174 Value *Amt,
bool IsRight) {
9180 if (Amt->getType() != Ty) {
9181 unsigned NumElts = Ty->getVectorNumElements();
9182 Amt = CGF.
Builder.CreateIntCast(Amt, Ty->getScalarType(),
false);
9183 Amt = CGF.
Builder.CreateVectorSplat(NumElts, Amt);
9186 unsigned IID = IsRight ? Intrinsic::fshr : Intrinsic::fshl;
9188 return CGF.
Builder.CreateCall(F, {Op0, Op1, Amt});
9195 if (
const auto *C = dyn_cast<Constant>(Mask))
9196 if (C->isAllOnesValue())
9199 Mask =
getMaskVecValue(CGF, Mask, Op0->getType()->getVectorNumElements());
9201 return CGF.
Builder.CreateSelect(Mask, Op0, Op1);
9207 if (
const auto *C = dyn_cast<Constant>(Mask))
9208 if (C->isAllOnesValue())
9211 llvm::VectorType *MaskTy =
9212 llvm::VectorType::get(CGF.
Builder.getInt1Ty(),
9213 Mask->getType()->getIntegerBitWidth());
9215 Mask = CGF.
Builder.CreateExtractElement(Mask, (uint64_t)0);
9216 return CGF.
Builder.CreateSelect(Mask, Op0, Op1);
9220 unsigned NumElts,
Value *MaskIn) {
9222 const auto *C = dyn_cast<Constant>(MaskIn);
9223 if (!C || !C->isAllOnesValue())
9228 uint32_t Indices[8];
9229 for (
unsigned i = 0; i != NumElts; ++i)
9231 for (
unsigned i = NumElts; i != 8; ++i)
9232 Indices[i] = i % NumElts + NumElts;
9233 Cmp = CGF.
Builder.CreateShuffleVector(
9234 Cmp, llvm::Constant::getNullValue(Cmp->getType()), Indices);
9244 assert((Ops.size() == 2 || Ops.size() == 4) &&
9245 "Unexpected number of arguments");
9246 unsigned NumElts = Ops[0]->getType()->getVectorNumElements();
9250 Cmp = Constant::getNullValue(
9251 llvm::VectorType::get(CGF.
Builder.getInt1Ty(), NumElts));
9252 }
else if (CC == 7) {
9253 Cmp = Constant::getAllOnesValue(
9254 llvm::VectorType::get(CGF.
Builder.getInt1Ty(), NumElts));
9256 ICmpInst::Predicate Pred;
9258 default: llvm_unreachable(
"Unknown condition code");
9259 case 0: Pred = ICmpInst::ICMP_EQ;
break;
9260 case 1: Pred = Signed ? ICmpInst::ICMP_SLT : ICmpInst::ICMP_ULT;
break;
9261 case 2: Pred = Signed ? ICmpInst::ICMP_SLE : ICmpInst::ICMP_ULE;
break;
9262 case 4: Pred = ICmpInst::ICMP_NE;
break;
9263 case 5: Pred = Signed ? ICmpInst::ICMP_SGE : ICmpInst::ICMP_UGE;
break;
9264 case 6: Pred = Signed ? ICmpInst::ICMP_SGT : ICmpInst::ICMP_UGT;
break;
9266 Cmp = CGF.
Builder.CreateICmp(Pred, Ops[0], Ops[1]);
9269 Value *MaskIn =
nullptr;
9270 if (Ops.size() == 4)
9277 Value *Zero = Constant::getNullValue(In->getType());
9284 Value *Zero = llvm::Constant::getNullValue(Ty);
9286 Value *Cmp = CGF.
Builder.CreateICmp(ICmpInst::ICMP_SGT, Ops[0], Zero);
9287 Value *Res = CGF.
Builder.CreateSelect(Cmp, Ops[0], Sub);
9293 Value *Cmp = CGF.
Builder.CreateICmp(Pred, Ops[0], Ops[1]);
9294 Value *Res = CGF.
Builder.CreateSelect(Cmp, Ops[0], Ops[1]);
9296 assert(Ops.size() == 2);
9302 unsigned BuiltinID,
bool IsAddSub) {
9304 bool Subtract =
false;
9306 switch (BuiltinID) {
9308 case clang::X86::BI__builtin_ia32_vfmsubps512_mask3:
9311 case clang::X86::BI__builtin_ia32_vfmaddps512_mask:
9312 case clang::X86::BI__builtin_ia32_vfmaddps512_maskz:
9313 case clang::X86::BI__builtin_ia32_vfmaddps512_mask3:
9314 IID = llvm::Intrinsic::x86_avx512_vfmadd_ps_512;
break;
9315 case clang::X86::BI__builtin_ia32_vfmsubpd512_mask3:
9318 case clang::X86::BI__builtin_ia32_vfmaddpd512_mask:
9319 case clang::X86::BI__builtin_ia32_vfmaddpd512_maskz:
9320 case clang::X86::BI__builtin_ia32_vfmaddpd512_mask3:
9321 IID = llvm::Intrinsic::x86_avx512_vfmadd_pd_512;
break;
9322 case clang::X86::BI__builtin_ia32_vfmsubaddps512_mask3:
9325 case clang::X86::BI__builtin_ia32_vfmaddsubps512_mask:
9326 case clang::X86::BI__builtin_ia32_vfmaddsubps512_maskz:
9327 case clang::X86::BI__builtin_ia32_vfmaddsubps512_mask3:
9328 IID = llvm::Intrinsic::x86_avx512_vfmaddsub_ps_512;
9330 case clang::X86::BI__builtin_ia32_vfmsubaddpd512_mask3:
9333 case clang::X86::BI__builtin_ia32_vfmaddsubpd512_mask:
9334 case clang::X86::BI__builtin_ia32_vfmaddsubpd512_maskz:
9335 case clang::X86::BI__builtin_ia32_vfmaddsubpd512_mask3:
9336 IID = llvm::Intrinsic::x86_avx512_vfmaddsub_pd_512;
9345 C = CGF.
Builder.CreateFNeg(C);
9350 if (IID != Intrinsic::not_intrinsic &&
9351 cast<llvm::ConstantInt>(Ops.back())->getZExtValue() != (uint64_t)4) {
9353 Res = CGF.
Builder.CreateCall(Intr, {A, B, C, Ops.back() });
9357 Res = CGF.
Builder.CreateCall(FMA, {A, B, C} );
9361 unsigned NumElts = Ty->getVectorNumElements();
9363 for (
unsigned i = 0; i != NumElts; ++i)
9364 Indices[i] = i + (i % 2) * NumElts;
9367 Value *FMSub = CGF.
Builder.CreateCall(FMA, {A, B, NegC} );
9368 Res = CGF.
Builder.CreateShuffleVector(FMSub, Res, Indices);
9373 Value *MaskFalseVal =
nullptr;
9374 switch (BuiltinID) {
9375 case clang::X86::BI__builtin_ia32_vfmaddps512_mask:
9376 case clang::X86::BI__builtin_ia32_vfmaddpd512_mask:
9377 case clang::X86::BI__builtin_ia32_vfmaddsubps512_mask:
9378 case clang::X86::BI__builtin_ia32_vfmaddsubpd512_mask:
9379 MaskFalseVal = Ops[0];
9381 case clang::X86::BI__builtin_ia32_vfmaddps512_maskz:
9382 case clang::X86::BI__builtin_ia32_vfmaddpd512_maskz:
9383 case clang::X86::BI__builtin_ia32_vfmaddsubps512_maskz:
9384 case clang::X86::BI__builtin_ia32_vfmaddsubpd512_maskz:
9385 MaskFalseVal = Constant::getNullValue(Ops[0]->getType());
9387 case clang::X86::BI__builtin_ia32_vfmsubps512_mask3:
9388 case clang::X86::BI__builtin_ia32_vfmaddps512_mask3:
9389 case clang::X86::BI__builtin_ia32_vfmsubpd512_mask3:
9390 case clang::X86::BI__builtin_ia32_vfmaddpd512_mask3:
9391 case clang::X86::BI__builtin_ia32_vfmsubaddps512_mask3:
9392 case clang::X86::BI__builtin_ia32_vfmaddsubps512_mask3:
9393 case clang::X86::BI__builtin_ia32_vfmsubaddpd512_mask3:
9394 case clang::X86::BI__builtin_ia32_vfmaddsubpd512_mask3:
9395 MaskFalseVal = Ops[2];
9407 Value *Upper,
bool ZeroMask =
false,
unsigned PTIdx = 0,
9408 bool NegAcc =
false) {
9411 Rnd = cast<llvm::ConstantInt>(Ops[4])->getZExtValue();
9414 Ops[2] = CGF.
Builder.CreateFNeg(Ops[2]);
9416 Ops[0] = CGF.
Builder.CreateExtractElement(Ops[0], (uint64_t)0);
9417 Ops[1] = CGF.
Builder.CreateExtractElement(Ops[1], (uint64_t)0);
9418 Ops[2] = CGF.
Builder.CreateExtractElement(Ops[2], (uint64_t)0);
9421 Intrinsic::ID IID = Ops[0]->getType()->getPrimitiveSizeInBits() == 32 ?
9422 Intrinsic::x86_avx512_vfmadd_f32 :
9423 Intrinsic::x86_avx512_vfmadd_f64;
9425 {Ops[0], Ops[1], Ops[2], Ops[4]});
9428 Res = CGF.
Builder.CreateCall(FMA, Ops.slice(0, 3));
9431 if (Ops.size() > 3) {
9432 Value *PassThru = ZeroMask ? Constant::getNullValue(Res->getType())
9438 if (NegAcc && PTIdx == 2)
9439 PassThru = CGF.
Builder.CreateExtractElement(Upper, (uint64_t)0);
9443 return CGF.
Builder.CreateInsertElement(Upper, Res, (uint64_t)0);
9450 Ty = llvm::VectorType::get(CGF.
Int64Ty,
9451 Ty->getPrimitiveSizeInBits() / 64);
9457 Constant *ShiftAmt = ConstantInt::get(Ty, 32);
9458 LHS = CGF.
Builder.CreateShl(LHS, ShiftAmt);
9459 LHS = CGF.
Builder.CreateAShr(LHS, ShiftAmt);
9460 RHS = CGF.
Builder.CreateShl(RHS, ShiftAmt);
9461 RHS = CGF.
Builder.CreateAShr(RHS, ShiftAmt);
9464 Constant *Mask = ConstantInt::get(Ty, 0xffffffff);
9465 LHS = CGF.
Builder.CreateAnd(LHS, Mask);
9466 RHS = CGF.
Builder.CreateAnd(RHS, Mask);
9469 return CGF.
Builder.CreateMul(LHS, RHS);
9479 unsigned VecWidth = Ty->getPrimitiveSizeInBits();
9480 unsigned EltWidth = Ty->getScalarSizeInBits();
9482 if (VecWidth == 128 && EltWidth == 32)
9483 IID = Intrinsic::x86_avx512_pternlog_d_128;
9484 else if (VecWidth == 256 && EltWidth == 32)
9485 IID = Intrinsic::x86_avx512_pternlog_d_256;
9486 else if (VecWidth == 512 && EltWidth == 32)
9487 IID = Intrinsic::x86_avx512_pternlog_d_512;
9488 else if (VecWidth == 128 && EltWidth == 64)
9489 IID = Intrinsic::x86_avx512_pternlog_q_128;
9490 else if (VecWidth == 256 && EltWidth == 64)
9491 IID = Intrinsic::x86_avx512_pternlog_q_256;
9492 else if (VecWidth == 512 && EltWidth == 64)
9493 IID = Intrinsic::x86_avx512_pternlog_q_512;
9495 llvm_unreachable(
"Unexpected intrinsic");
9499 Value *PassThru = ZeroMask ? ConstantAggregateZero::get(Ty) : Ops[0];
9505 unsigned NumberOfElements = DstTy->getVectorNumElements();
9507 return CGF.
Builder.CreateSExt(Mask, DstTy,
"vpmovm2");
9515 IsSigned ? (IsAddition ? Intrinsic::sadd_sat : Intrinsic::ssub_sat)
9516 : (IsAddition ? Intrinsic::uadd_sat : Intrinsic::usub_sat);
9518 return CGF.
Builder.CreateCall(F, {Ops[0], Ops[1]});
9523 StringRef CPUStr = cast<clang::StringLiteral>(CPUExpr)->getString();
9524 return EmitX86CpuIs(CPUStr);
9527 Value *CodeGenFunction::EmitX86CpuIs(StringRef CPUStr) {
9537 llvm::Type *STy = llvm::StructType::get(Int32Ty, Int32Ty, Int32Ty,
9538 llvm::ArrayType::get(Int32Ty, 1));
9541 llvm::Constant *CpuModel = CGM.CreateRuntimeVariable(STy,
"__cpu_model");
9542 cast<llvm::GlobalValue>(CpuModel)->setDSOLocal(
true);
9548 std::tie(Index, Value) = StringSwitch<std::pair<unsigned, unsigned>>(CPUStr)
9550 .Case(STRING, {0u,
static_cast<unsigned>(llvm::X86::ENUM)})
9551 #define X86_CPU_TYPE_COMPAT_WITH_ALIAS(ARCHNAME, ENUM, STR, ALIAS) \ 9552 .Cases(STR, ALIAS, {1u, static_cast<unsigned>(llvm::X86::ENUM)}) 9553 #define X86_CPU_TYPE_COMPAT(ARCHNAME, ENUM, STR) \ 9554 .Case(STR, {1u, static_cast<unsigned>(llvm::X86::ENUM)}) 9555 #define X86_CPU_SUBTYPE_COMPAT(ARCHNAME, ENUM, STR) \ 9556 .Case(STR, {2u, static_cast<unsigned>(llvm::X86::ENUM)}) 9557 #include "llvm/Support/X86TargetParser.def" 9559 assert(Value != 0 &&
"Invalid CPUStr passed to CpuIs");
9562 llvm::Value *Idxs[] = {ConstantInt::get(Int32Ty, 0),
9563 ConstantInt::get(Int32Ty, Index)};
9564 llvm::Value *CpuValue = Builder.CreateGEP(STy, CpuModel, Idxs);
9568 return Builder.CreateICmpEQ(CpuValue,
9569 llvm::ConstantInt::get(Int32Ty, Value));
9572 Value *CodeGenFunction::EmitX86CpuSupports(
const CallExpr *E) {
9574 StringRef FeatureStr = cast<StringLiteral>(FeatureExpr)->getString();
9575 return EmitX86CpuSupports(FeatureStr);
9581 uint64_t FeaturesMask = 0;
9582 for (
const StringRef &FeatureStr : FeatureStrs) {
9584 StringSwitch<unsigned>(FeatureStr)
9586 #include "llvm/Support/X86TargetParser.def" 9588 FeaturesMask |= (1ULL << Feature);
9590 return FeaturesMask;
9594 return EmitX86CpuSupports(GetX86CpuSupportsMask(FeatureStrs));
9597 llvm::Value *CodeGenFunction::EmitX86CpuSupports(uint64_t FeaturesMask) {
9598 uint32_t Features1 = Lo_32(FeaturesMask);
9599 uint32_t Features2 = Hi_32(FeaturesMask);
9601 Value *Result = Builder.getTrue();
9603 if (Features1 != 0) {
9610 llvm::Type *STy = llvm::StructType::get(Int32Ty, Int32Ty, Int32Ty,
9611 llvm::ArrayType::get(Int32Ty, 1));
9614 llvm::Constant *CpuModel = CGM.CreateRuntimeVariable(STy,
"__cpu_model");
9615 cast<llvm::GlobalValue>(CpuModel)->setDSOLocal(
true);
9619 Value *Idxs[] = {Builder.getInt32(0), Builder.getInt32(3),
9620 Builder.getInt32(0)};
9621 Value *CpuFeatures = Builder.CreateGEP(STy, CpuModel, Idxs);
9626 Value *Mask = Builder.getInt32(Features1);
9627 Value *Bitset = Builder.CreateAnd(Features, Mask);
9628 Value *Cmp = Builder.CreateICmpEQ(Bitset, Mask);
9629 Result = Builder.CreateAnd(Result, Cmp);
9632 if (Features2 != 0) {
9633 llvm::Constant *CpuFeatures2 = CGM.CreateRuntimeVariable(Int32Ty,
9635 cast<llvm::GlobalValue>(CpuFeatures2)->setDSOLocal(
true);
9641 Value *Mask = Builder.getInt32(Features2);
9642 Value *Bitset = Builder.CreateAnd(Features, Mask);
9643 Value *Cmp = Builder.CreateICmpEQ(Bitset, Mask);
9644 Result = Builder.CreateAnd(Result, Cmp);
9650 Value *CodeGenFunction::EmitX86CpuInit() {
9651 llvm::FunctionType *FTy = llvm::FunctionType::get(VoidTy,
9653 llvm::Constant *Func = CGM.CreateRuntimeFunction(FTy,
"__cpu_indicator_init");
9654 cast<llvm::GlobalValue>(Func)->setDSOLocal(
true);
9655 cast<llvm::GlobalValue>(Func)->setDLLStorageClass(
9656 llvm::GlobalValue::DefaultStorageClass);
9657 return Builder.CreateCall(Func);
9662 if (BuiltinID == X86::BI__builtin_cpu_is)
9663 return EmitX86CpuIs(E);
9664 if (BuiltinID == X86::BI__builtin_cpu_supports)
9665 return EmitX86CpuSupports(E);
9666 if (BuiltinID == X86::BI__builtin_cpu_init)
9667 return EmitX86CpuInit();
9672 unsigned ICEArguments = 0;
9674 getContext().GetBuiltinType(BuiltinID, Error, &ICEArguments);
9677 for (
unsigned i = 0, e = E->
getNumArgs(); i != e; i++) {
9679 if ((ICEArguments & (1 << i)) == 0) {
9680 Ops.push_back(EmitScalarExpr(E->
getArg(i)));
9686 llvm::APSInt Result;
9688 assert(IsConst &&
"Constant arg isn't actually constant?"); (void)IsConst;
9689 Ops.push_back(llvm::ConstantInt::get(getLLVMContext(), Result));
9698 auto getCmpIntrinsicCall = [
this, &Ops](
Intrinsic::ID ID,
unsigned Imm) {
9699 Ops.push_back(llvm::ConstantInt::get(Int8Ty, Imm));
9700 llvm::Function *F = CGM.getIntrinsic(ID);
9701 return Builder.CreateCall(F, Ops);
9709 auto getVectorFCmpIR = [
this, &Ops](CmpInst::Predicate Pred) {
9710 Value *Cmp = Builder.CreateFCmp(Pred, Ops[0], Ops[1]);
9711 llvm::VectorType *FPVecTy = cast<llvm::VectorType>(Ops[0]->getType());
9712 llvm::VectorType *IntVecTy = llvm::VectorType::getInteger(FPVecTy);
9713 Value *Sext = Builder.CreateSExt(Cmp, IntVecTy);
9714 return Builder.CreateBitCast(Sext, FPVecTy);
9717 switch (BuiltinID) {
9718 default:
return nullptr;
9719 case X86::BI_mm_prefetch: {
9721 ConstantInt *C = cast<ConstantInt>(Ops[1]);
9722 Value *RW = ConstantInt::get(Int32Ty, (C->getZExtValue() >> 2) & 0x1);
9723 Value *Locality = ConstantInt::get(Int32Ty, C->getZExtValue() & 0x3);
9724 Value *Data = ConstantInt::get(Int32Ty, 1);
9726 return Builder.CreateCall(F, {Address, RW, Locality, Data});
9728 case X86::BI_mm_clflush: {
9729 return Builder.CreateCall(CGM.getIntrinsic(Intrinsic::x86_sse2_clflush),
9732 case X86::BI_mm_lfence: {
9733 return Builder.CreateCall(CGM.getIntrinsic(Intrinsic::x86_sse2_lfence));
9735 case X86::BI_mm_mfence: {
9736 return Builder.CreateCall(CGM.getIntrinsic(Intrinsic::x86_sse2_mfence));
9738 case X86::BI_mm_sfence: {
9739 return Builder.CreateCall(CGM.getIntrinsic(Intrinsic::x86_sse_sfence));
9741 case X86::BI_mm_pause: {
9742 return Builder.CreateCall(CGM.getIntrinsic(Intrinsic::x86_sse2_pause));
9744 case X86::BI__rdtsc: {
9745 return Builder.CreateCall(CGM.getIntrinsic(Intrinsic::x86_rdtsc));
9747 case X86::BI__builtin_ia32_rdtscp: {
9748 Value *Call = Builder.CreateCall(CGM.getIntrinsic(Intrinsic::x86_rdtscp));
9749 Builder.CreateDefaultAlignedStore(Builder.CreateExtractValue(Call, 1),
9751 return Builder.CreateExtractValue(Call, 0);
9753 case X86::BI__builtin_ia32_lzcnt_u16:
9754 case X86::BI__builtin_ia32_lzcnt_u32:
9755 case X86::BI__builtin_ia32_lzcnt_u64: {
9756 Value *F = CGM.getIntrinsic(Intrinsic::ctlz, Ops[0]->getType());
9757 return Builder.CreateCall(F, {Ops[0], Builder.getInt1(
false)});
9759 case X86::BI__builtin_ia32_tzcnt_u16:
9760 case X86::BI__builtin_ia32_tzcnt_u32:
9761 case X86::BI__builtin_ia32_tzcnt_u64: {
9762 Value *F = CGM.getIntrinsic(Intrinsic::cttz, Ops[0]->getType());
9763 return Builder.CreateCall(F, {Ops[0], Builder.getInt1(
false)});
9765 case X86::BI__builtin_ia32_undef128:
9766 case X86::BI__builtin_ia32_undef256:
9767 case X86::BI__builtin_ia32_undef512:
9773 return llvm::Constant::getNullValue(ConvertType(E->
getType()));
9774 case X86::BI__builtin_ia32_vec_init_v8qi:
9775 case X86::BI__builtin_ia32_vec_init_v4hi:
9776 case X86::BI__builtin_ia32_vec_init_v2si:
9777 return Builder.CreateBitCast(BuildVector(Ops),
9778 llvm::Type::getX86_MMXTy(getLLVMContext()));
9779 case X86::BI__builtin_ia32_vec_ext_v2si:
9780 case X86::BI__builtin_ia32_vec_ext_v16qi:
9781 case X86::BI__builtin_ia32_vec_ext_v8hi:
9782 case X86::BI__builtin_ia32_vec_ext_v4si:
9783 case X86::BI__builtin_ia32_vec_ext_v4sf:
9784 case X86::BI__builtin_ia32_vec_ext_v2di:
9785 case X86::BI__builtin_ia32_vec_ext_v32qi:
9786 case X86::BI__builtin_ia32_vec_ext_v16hi:
9787 case X86::BI__builtin_ia32_vec_ext_v8si:
9788 case X86::BI__builtin_ia32_vec_ext_v4di: {
9789 unsigned NumElts = Ops[0]->getType()->getVectorNumElements();
9790 uint64_t Index = cast<ConstantInt>(Ops[1])->getZExtValue();
9791 Index &= NumElts - 1;
9794 return Builder.CreateExtractElement(Ops[0], Index);
9796 case X86::BI__builtin_ia32_vec_set_v16qi:
9797 case X86::BI__builtin_ia32_vec_set_v8hi:
9798 case X86::BI__builtin_ia32_vec_set_v4si:
9799 case X86::BI__builtin_ia32_vec_set_v2di:
9800 case X86::BI__builtin_ia32_vec_set_v32qi:
9801 case X86::BI__builtin_ia32_vec_set_v16hi:
9802 case X86::BI__builtin_ia32_vec_set_v8si:
9803 case X86::BI__builtin_ia32_vec_set_v4di: {
9804 unsigned NumElts = Ops[0]->getType()->getVectorNumElements();
9805 unsigned Index = cast<ConstantInt>(Ops[2])->getZExtValue();
9806 Index &= NumElts - 1;
9809 return Builder.CreateInsertElement(Ops[0], Ops[1], Index);
9811 case X86::BI_mm_setcsr:
9812 case X86::BI__builtin_ia32_ldmxcsr: {
9814 Builder.CreateStore(Ops[0], Tmp);
9815 return Builder.CreateCall(CGM.getIntrinsic(Intrinsic::x86_sse_ldmxcsr),
9816 Builder.CreateBitCast(Tmp.
getPointer(), Int8PtrTy));
9818 case X86::BI_mm_getcsr:
9819 case X86::BI__builtin_ia32_stmxcsr: {
9821 Builder.CreateCall(CGM.getIntrinsic(Intrinsic::x86_sse_stmxcsr),
9822 Builder.CreateBitCast(Tmp.
getPointer(), Int8PtrTy));
9823 return Builder.CreateLoad(Tmp,
"stmxcsr");
9825 case X86::BI__builtin_ia32_xsave:
9826 case X86::BI__builtin_ia32_xsave64:
9827 case X86::BI__builtin_ia32_xrstor:
9828 case X86::BI__builtin_ia32_xrstor64:
9829 case X86::BI__builtin_ia32_xsaveopt:
9830 case X86::BI__builtin_ia32_xsaveopt64:
9831 case X86::BI__builtin_ia32_xrstors:
9832 case X86::BI__builtin_ia32_xrstors64:
9833 case X86::BI__builtin_ia32_xsavec:
9834 case X86::BI__builtin_ia32_xsavec64:
9835 case X86::BI__builtin_ia32_xsaves:
9836 case X86::BI__builtin_ia32_xsaves64: {
9838 #define INTRINSIC_X86_XSAVE_ID(NAME) \ 9839 case X86::BI__builtin_ia32_##NAME: \ 9840 ID = Intrinsic::x86_##NAME; \ 9842 switch (BuiltinID) {
9843 default: llvm_unreachable(
"Unsupported intrinsic!");
9857 #undef INTRINSIC_X86_XSAVE_ID 9858 Value *Mhi = Builder.CreateTrunc(
9859 Builder.CreateLShr(Ops[1], ConstantInt::get(Int64Ty, 32)), Int32Ty);
9860 Value *Mlo = Builder.CreateTrunc(Ops[1], Int32Ty);
9863 return Builder.CreateCall(CGM.getIntrinsic(ID), Ops);
9865 case X86::BI__builtin_ia32_storedqudi128_mask:
9866 case X86::BI__builtin_ia32_storedqusi128_mask:
9867 case X86::BI__builtin_ia32_storedquhi128_mask:
9868 case X86::BI__builtin_ia32_storedquqi128_mask:
9869 case X86::BI__builtin_ia32_storeupd128_mask:
9870 case X86::BI__builtin_ia32_storeups128_mask:
9871 case X86::BI__builtin_ia32_storedqudi256_mask:
9872 case X86::BI__builtin_ia32_storedqusi256_mask:
9873 case X86::BI__builtin_ia32_storedquhi256_mask:
9874 case X86::BI__builtin_ia32_storedquqi256_mask:
9875 case X86::BI__builtin_ia32_storeupd256_mask:
9876 case X86::BI__builtin_ia32_storeups256_mask:
9877 case X86::BI__builtin_ia32_storedqudi512_mask:
9878 case X86::BI__builtin_ia32_storedqusi512_mask:
9879 case X86::BI__builtin_ia32_storedquhi512_mask:
9880 case X86::BI__builtin_ia32_storedquqi512_mask:
9881 case X86::BI__builtin_ia32_storeupd512_mask:
9882 case X86::BI__builtin_ia32_storeups512_mask:
9885 case X86::BI__builtin_ia32_storess128_mask:
9886 case X86::BI__builtin_ia32_storesd128_mask: {
9889 case X86::BI__builtin_ia32_vpopcntb_128:
9890 case X86::BI__builtin_ia32_vpopcntd_128:
9891 case X86::BI__builtin_ia32_vpopcntq_128:
9892 case X86::BI__builtin_ia32_vpopcntw_128:
9893 case X86::BI__builtin_ia32_vpopcntb_256:
9894 case X86::BI__builtin_ia32_vpopcntd_256:
9895 case X86::BI__builtin_ia32_vpopcntq_256:
9896 case X86::BI__builtin_ia32_vpopcntw_256:
9897 case X86::BI__builtin_ia32_vpopcntb_512:
9898 case X86::BI__builtin_ia32_vpopcntd_512:
9899 case X86::BI__builtin_ia32_vpopcntq_512:
9900 case X86::BI__builtin_ia32_vpopcntw_512: {
9902 llvm::Function *F = CGM.getIntrinsic(Intrinsic::ctpop, ResultType);
9903 return Builder.CreateCall(F, Ops);
9905 case X86::BI__builtin_ia32_cvtmask2b128:
9906 case X86::BI__builtin_ia32_cvtmask2b256:
9907 case X86::BI__builtin_ia32_cvtmask2b512:
9908 case X86::BI__builtin_ia32_cvtmask2w128:
9909 case X86::BI__builtin_ia32_cvtmask2w256:
9910 case X86::BI__builtin_ia32_cvtmask2w512:
9911 case X86::BI__builtin_ia32_cvtmask2d128:
9912 case X86::BI__builtin_ia32_cvtmask2d256:
9913 case X86::BI__builtin_ia32_cvtmask2d512:
9914 case X86::BI__builtin_ia32_cvtmask2q128:
9915 case X86::BI__builtin_ia32_cvtmask2q256:
9916 case X86::BI__builtin_ia32_cvtmask2q512:
9919 case X86::BI__builtin_ia32_cvtb2mask128:
9920 case X86::BI__builtin_ia32_cvtb2mask256:
9921 case X86::BI__builtin_ia32_cvtb2mask512:
9922 case X86::BI__builtin_ia32_cvtw2mask128:
9923 case X86::BI__builtin_ia32_cvtw2mask256:
9924 case X86::BI__builtin_ia32_cvtw2mask512:
9925 case X86::BI__builtin_ia32_cvtd2mask128:
9926 case X86::BI__builtin_ia32_cvtd2mask256:
9927 case X86::BI__builtin_ia32_cvtd2mask512:
9928 case X86::BI__builtin_ia32_cvtq2mask128:
9929 case X86::BI__builtin_ia32_cvtq2mask256:
9930 case X86::BI__builtin_ia32_cvtq2mask512:
9933 case X86::BI__builtin_ia32_vfmaddss3:
9934 case X86::BI__builtin_ia32_vfmaddsd3:
9935 case X86::BI__builtin_ia32_vfmaddss3_mask:
9936 case X86::BI__builtin_ia32_vfmaddsd3_mask:
9938 case X86::BI__builtin_ia32_vfmaddss:
9939 case X86::BI__builtin_ia32_vfmaddsd:
9941 Constant::getNullValue(Ops[0]->getType()));
9942 case X86::BI__builtin_ia32_vfmaddss3_maskz:
9943 case X86::BI__builtin_ia32_vfmaddsd3_maskz:
9945 case X86::BI__builtin_ia32_vfmaddss3_mask3:
9946 case X86::BI__builtin_ia32_vfmaddsd3_mask3:
9948 case X86::BI__builtin_ia32_vfmsubss3_mask3:
9949 case X86::BI__builtin_ia32_vfmsubsd3_mask3:
9952 case X86::BI__builtin_ia32_vfmaddps:
9953 case X86::BI__builtin_ia32_vfmaddpd:
9954 case X86::BI__builtin_ia32_vfmaddps256:
9955 case X86::BI__builtin_ia32_vfmaddpd256:
9956 case X86::BI__builtin_ia32_vfmaddps512_mask:
9957 case X86::BI__builtin_ia32_vfmaddps512_maskz:
9958 case X86::BI__builtin_ia32_vfmaddps512_mask3:
9959 case X86::BI__builtin_ia32_vfmsubps512_mask3:
9960 case X86::BI__builtin_ia32_vfmaddpd512_mask:
9961 case X86::BI__builtin_ia32_vfmaddpd512_maskz:
9962 case X86::BI__builtin_ia32_vfmaddpd512_mask3:
9963 case X86::BI__builtin_ia32_vfmsubpd512_mask3:
9965 case X86::BI__builtin_ia32_vfmaddsubps:
9966 case X86::BI__builtin_ia32_vfmaddsubpd:
9967 case X86::BI__builtin_ia32_vfmaddsubps256:
9968 case X86::BI__builtin_ia32_vfmaddsubpd256:
9969 case X86::BI__builtin_ia32_vfmaddsubps512_mask:
9970 case X86::BI__builtin_ia32_vfmaddsubps512_maskz:
9971 case X86::BI__builtin_ia32_vfmaddsubps512_mask3:
9972 case X86::BI__builtin_ia32_vfmsubaddps512_mask3:
9973 case X86::BI__builtin_ia32_vfmaddsubpd512_mask:
9974 case X86::BI__builtin_ia32_vfmaddsubpd512_maskz:
9975 case X86::BI__builtin_ia32_vfmaddsubpd512_mask3:
9976 case X86::BI__builtin_ia32_vfmsubaddpd512_mask3:
9979 case X86::BI__builtin_ia32_movdqa32store128_mask:
9980 case X86::BI__builtin_ia32_movdqa64store128_mask:
9981 case X86::BI__builtin_ia32_storeaps128_mask:
9982 case X86::BI__builtin_ia32_storeapd128_mask:
9983 case X86::BI__builtin_ia32_movdqa32store256_mask:
9984 case X86::BI__builtin_ia32_movdqa64store256_mask:
9985 case X86::BI__builtin_ia32_storeaps256_mask:
9986 case X86::BI__builtin_ia32_storeapd256_mask:
9987 case X86::BI__builtin_ia32_movdqa32store512_mask:
9988 case X86::BI__builtin_ia32_movdqa64store512_mask:
9989 case X86::BI__builtin_ia32_storeaps512_mask:
9990 case X86::BI__builtin_ia32_storeapd512_mask: {
9992 getContext().getTypeAlignInChars(E->
getArg(1)->
getType()).getQuantity();
9995 case X86::BI__builtin_ia32_loadups128_mask:
9996 case X86::BI__builtin_ia32_loadups256_mask:
9997 case X86::BI__builtin_ia32_loadups512_mask:
9998 case X86::BI__builtin_ia32_loadupd128_mask:
9999 case X86::BI__builtin_ia32_loadupd256_mask:
10000 case X86::BI__builtin_ia32_loadupd512_mask:
10001 case X86::BI__builtin_ia32_loaddquqi128_mask:
10002 case X86::BI__builtin_ia32_loaddquqi256_mask:
10003 case X86::BI__builtin_ia32_loaddquqi512_mask:
10004 case X86::BI__builtin_ia32_loaddquhi128_mask:
10005 case X86::BI__builtin_ia32_loaddquhi256_mask:
10006 case X86::BI__builtin_ia32_loaddquhi512_mask:
10007 case X86::BI__builtin_ia32_loaddqusi128_mask:
10008 case X86::BI__builtin_ia32_loaddqusi256_mask:
10009 case X86::BI__builtin_ia32_loaddqusi512_mask:
10010 case X86::BI__builtin_ia32_loaddqudi128_mask:
10011 case X86::BI__builtin_ia32_loaddqudi256_mask:
10012 case X86::BI__builtin_ia32_loaddqudi512_mask:
10015 case X86::BI__builtin_ia32_loadss128_mask:
10016 case X86::BI__builtin_ia32_loadsd128_mask:
10019 case X86::BI__builtin_ia32_loadaps128_mask:
10020 case X86::BI__builtin_ia32_loadaps256_mask:
10021 case X86::BI__builtin_ia32_loadaps512_mask:
10022 case X86::BI__builtin_ia32_loadapd128_mask:
10023 case X86::BI__builtin_ia32_loadapd256_mask:
10024 case X86::BI__builtin_ia32_loadapd512_mask:
10025 case X86::BI__builtin_ia32_movdqa32load128_mask:
10026 case X86::BI__builtin_ia32_movdqa32load256_mask:
10027 case X86::BI__builtin_ia32_movdqa32load512_mask:
10028 case X86::BI__builtin_ia32_movdqa64load128_mask:
10029 case X86::BI__builtin_ia32_movdqa64load256_mask:
10030 case X86::BI__builtin_ia32_movdqa64load512_mask: {
10032 getContext().getTypeAlignInChars(E->
getArg(1)->
getType()).getQuantity();
10036 case X86::BI__builtin_ia32_expandloaddf128_mask:
10037 case X86::BI__builtin_ia32_expandloaddf256_mask:
10038 case X86::BI__builtin_ia32_expandloaddf512_mask:
10039 case X86::BI__builtin_ia32_expandloadsf128_mask:
10040 case X86::BI__builtin_ia32_expandloadsf256_mask:
10041 case X86::BI__builtin_ia32_expandloadsf512_mask:
10042 case X86::BI__builtin_ia32_expandloaddi128_mask:
10043 case X86::BI__builtin_ia32_expandloaddi256_mask:
10044 case X86::BI__builtin_ia32_expandloaddi512_mask:
10045 case X86::BI__builtin_ia32_expandloadsi128_mask:
10046 case X86::BI__builtin_ia32_expandloadsi256_mask:
10047 case X86::BI__builtin_ia32_expandloadsi512_mask:
10048 case X86::BI__builtin_ia32_expandloadhi128_mask:
10049 case X86::BI__builtin_ia32_expandloadhi256_mask:
10050 case X86::BI__builtin_ia32_expandloadhi512_mask:
10051 case X86::BI__builtin_ia32_expandloadqi128_mask:
10052 case X86::BI__builtin_ia32_expandloadqi256_mask:
10053 case X86::BI__builtin_ia32_expandloadqi512_mask:
10056 case X86::BI__builtin_ia32_compressstoredf128_mask:
10057 case X86::BI__builtin_ia32_compressstoredf256_mask:
10058 case X86::BI__builtin_ia32_compressstoredf512_mask:
10059 case X86::BI__builtin_ia32_compressstoresf128_mask:
10060 case X86::BI__builtin_ia32_compressstoresf256_mask:
10061 case X86::BI__builtin_ia32_compressstoresf512_mask:
10062 case X86::BI__builtin_ia32_compressstoredi128_mask:
10063 case X86::BI__builtin_ia32_compressstoredi256_mask:
10064 case X86::BI__builtin_ia32_compressstoredi512_mask:
10065 case X86::BI__builtin_ia32_compressstoresi128_mask:
10066 case X86::BI__builtin_ia32_compressstoresi256_mask:
10067 case X86::BI__builtin_ia32_compressstoresi512_mask:
10068 case X86::BI__builtin_ia32_compressstorehi128_mask:
10069 case X86::BI__builtin_ia32_compressstorehi256_mask:
10070 case X86::BI__builtin_ia32_compressstorehi512_mask:
10071 case X86::BI__builtin_ia32_compressstoreqi128_mask:
10072 case X86::BI__builtin_ia32_compressstoreqi256_mask:
10073 case X86::BI__builtin_ia32_compressstoreqi512_mask:
10076 case X86::BI__builtin_ia32_storehps:
10077 case X86::BI__builtin_ia32_storelps: {
10078 llvm::Type *PtrTy = llvm::PointerType::getUnqual(Int64Ty);
10079 llvm::Type *VecTy = llvm::VectorType::get(Int64Ty, 2);
10082 Ops[1] = Builder.CreateBitCast(Ops[1], VecTy,
"cast");
10085 unsigned Index = BuiltinID == X86::BI__builtin_ia32_storelps ? 0 : 1;
10086 Ops[1] = Builder.CreateExtractElement(Ops[1], Index,
"extract");
10089 Ops[0] = Builder.CreateBitCast(Ops[0], PtrTy);
10090 return Builder.CreateDefaultAlignedStore(Ops[1], Ops[0]);
10092 case X86::BI__builtin_ia32_vextractf128_pd256:
10093 case X86::BI__builtin_ia32_vextractf128_ps256:
10094 case X86::BI__builtin_ia32_vextractf128_si256:
10095 case X86::BI__builtin_ia32_extract128i256:
10096 case X86::BI__builtin_ia32_extractf64x4_mask:
10097 case X86::BI__builtin_ia32_extractf32x4_mask:
10098 case X86::BI__builtin_ia32_extracti64x4_mask:
10099 case X86::BI__builtin_ia32_extracti32x4_mask:
10100 case X86::BI__builtin_ia32_extractf32x8_mask:
10101 case X86::BI__builtin_ia32_extracti32x8_mask:
10102 case X86::BI__builtin_ia32_extractf32x4_256_mask:
10103 case X86::BI__builtin_ia32_extracti32x4_256_mask:
10104 case X86::BI__builtin_ia32_extractf64x2_256_mask:
10105 case X86::BI__builtin_ia32_extracti64x2_256_mask:
10106 case X86::BI__builtin_ia32_extractf64x2_512_mask:
10107 case X86::BI__builtin_ia32_extracti64x2_512_mask: {
10109 unsigned NumElts = DstTy->getVectorNumElements();
10110 unsigned SrcNumElts = Ops[0]->getType()->getVectorNumElements();
10111 unsigned SubVectors = SrcNumElts / NumElts;
10112 unsigned Index = cast<ConstantInt>(Ops[1])->getZExtValue();
10113 assert(llvm::isPowerOf2_32(SubVectors) &&
"Expected power of 2 subvectors");
10114 Index &= SubVectors - 1;
10117 uint32_t Indices[16];
10118 for (
unsigned i = 0; i != NumElts; ++i)
10119 Indices[i] = i + Index;
10121 Value *Res = Builder.CreateShuffleVector(Ops[0],
10122 UndefValue::get(Ops[0]->getType()),
10123 makeArrayRef(Indices, NumElts),
10126 if (Ops.size() == 4)
10131 case X86::BI__builtin_ia32_vinsertf128_pd256:
10132 case X86::BI__builtin_ia32_vinsertf128_ps256:
10133 case X86::BI__builtin_ia32_vinsertf128_si256:
10134 case X86::BI__builtin_ia32_insert128i256:
10135 case X86::BI__builtin_ia32_insertf64x4:
10136 case X86::BI__builtin_ia32_insertf32x4:
10137 case X86::BI__builtin_ia32_inserti64x4:
10138 case X86::BI__builtin_ia32_inserti32x4:
10139 case X86::BI__builtin_ia32_insertf32x8:
10140 case X86::BI__builtin_ia32_inserti32x8:
10141 case X86::BI__builtin_ia32_insertf32x4_256:
10142 case X86::BI__builtin_ia32_inserti32x4_256:
10143 case X86::BI__builtin_ia32_insertf64x2_256:
10144 case X86::BI__builtin_ia32_inserti64x2_256:
10145 case X86::BI__builtin_ia32_insertf64x2_512:
10146 case X86::BI__builtin_ia32_inserti64x2_512: {
10147 unsigned DstNumElts = Ops[0]->getType()->getVectorNumElements();
10148 unsigned SrcNumElts = Ops[1]->getType()->getVectorNumElements();
10149 unsigned SubVectors = DstNumElts / SrcNumElts;
10150 unsigned Index = cast<ConstantInt>(Ops[2])->getZExtValue();
10151 assert(llvm::isPowerOf2_32(SubVectors) &&
"Expected power of 2 subvectors");
10152 Index &= SubVectors - 1;
10153 Index *= SrcNumElts;
10155 uint32_t Indices[16];
10156 for (
unsigned i = 0; i != DstNumElts; ++i)
10157 Indices[i] = (i >= SrcNumElts) ? SrcNumElts + (i % SrcNumElts) : i;
10159 Value *Op1 = Builder.CreateShuffleVector(Ops[1],
10160 UndefValue::get(Ops[1]->getType()),
10161 makeArrayRef(Indices, DstNumElts),
10164 for (
unsigned i = 0; i != DstNumElts; ++i) {
10165 if (i >= Index && i < (Index + SrcNumElts))
10166 Indices[i] = (i - Index) + DstNumElts;
10171 return Builder.CreateShuffleVector(Ops[0], Op1,
10172 makeArrayRef(Indices, DstNumElts),
10175 case X86::BI__builtin_ia32_pmovqd512_mask:
10176 case X86::BI__builtin_ia32_pmovwb512_mask: {
10177 Value *Res = Builder.CreateTrunc(Ops[0], Ops[1]->getType());
10180 case X86::BI__builtin_ia32_pmovdb512_mask:
10181 case X86::BI__builtin_ia32_pmovdw512_mask:
10182 case X86::BI__builtin_ia32_pmovqw512_mask: {
10183 if (
const auto *C = dyn_cast<Constant>(Ops[2]))
10184 if (C->isAllOnesValue())
10185 return Builder.CreateTrunc(Ops[0], Ops[1]->getType());
10188 switch (BuiltinID) {
10189 default: llvm_unreachable(
"Unsupported intrinsic!");
10190 case X86::BI__builtin_ia32_pmovdb512_mask:
10191 IID = Intrinsic::x86_avx512_mask_pmov_db_512;
10193 case X86::BI__builtin_ia32_pmovdw512_mask:
10194 IID = Intrinsic::x86_avx512_mask_pmov_dw_512;
10196 case X86::BI__builtin_ia32_pmovqw512_mask:
10197 IID = Intrinsic::x86_avx512_mask_pmov_qw_512;
10201 Function *Intr = CGM.getIntrinsic(IID);
10202 return Builder.CreateCall(Intr, Ops);
10204 case X86::BI__builtin_ia32_pblendw128:
10205 case X86::BI__builtin_ia32_blendpd:
10206 case X86::BI__builtin_ia32_blendps:
10207 case X86::BI__builtin_ia32_blendpd256:
10208 case X86::BI__builtin_ia32_blendps256:
10209 case X86::BI__builtin_ia32_pblendw256:
10210 case X86::BI__builtin_ia32_pblendd128:
10211 case X86::BI__builtin_ia32_pblendd256: {
10212 unsigned NumElts = Ops[0]->getType()->getVectorNumElements();
10213 unsigned Imm = cast<llvm::ConstantInt>(Ops[2])->getZExtValue();
10215 uint32_t Indices[16];
10218 for (
unsigned i = 0; i != NumElts; ++i)
10219 Indices[i] = ((Imm >> (i % 8)) & 0x1) ? NumElts + i : i;
10221 return Builder.CreateShuffleVector(Ops[0], Ops[1],
10222 makeArrayRef(Indices, NumElts),
10225 case X86::BI__builtin_ia32_pshuflw:
10226 case X86::BI__builtin_ia32_pshuflw256:
10227 case X86::BI__builtin_ia32_pshuflw512: {
10228 uint32_t Imm = cast<llvm::ConstantInt>(Ops[1])->getZExtValue();
10230 unsigned NumElts = Ty->getVectorNumElements();
10233 Imm = (Imm & 0xff) * 0x01010101;
10235 uint32_t Indices[32];
10236 for (
unsigned l = 0; l != NumElts; l += 8) {
10237 for (
unsigned i = 0; i != 4; ++i) {
10238 Indices[l + i] = l + (Imm & 3);
10241 for (
unsigned i = 4; i != 8; ++i)
10242 Indices[l + i] = l + i;
10245 return Builder.CreateShuffleVector(Ops[0], UndefValue::get(Ty),
10246 makeArrayRef(Indices, NumElts),
10249 case X86::BI__builtin_ia32_pshufhw:
10250 case X86::BI__builtin_ia32_pshufhw256:
10251 case X86::BI__builtin_ia32_pshufhw512: {
10252 uint32_t Imm = cast<llvm::ConstantInt>(Ops[1])->getZExtValue();
10254 unsigned NumElts = Ty->getVectorNumElements();
10257 Imm = (Imm & 0xff) * 0x01010101;
10259 uint32_t Indices[32];
10260 for (
unsigned l = 0; l != NumElts; l += 8) {
10261 for (
unsigned i = 0; i != 4; ++i)
10262 Indices[l + i] = l + i;
10263 for (
unsigned i = 4; i != 8; ++i) {
10264 Indices[l + i] = l + 4 + (Imm & 3);
10269 return Builder.CreateShuffleVector(Ops[0], UndefValue::get(Ty),
10270 makeArrayRef(Indices, NumElts),
10273 case X86::BI__builtin_ia32_pshufd:
10274 case X86::BI__builtin_ia32_pshufd256:
10275 case X86::BI__builtin_ia32_pshufd512:
10276 case X86::BI__builtin_ia32_vpermilpd:
10277 case X86::BI__builtin_ia32_vpermilps:
10278 case X86::BI__builtin_ia32_vpermilpd256:
10279 case X86::BI__builtin_ia32_vpermilps256:
10280 case X86::BI__builtin_ia32_vpermilpd512:
10281 case X86::BI__builtin_ia32_vpermilps512: {
10282 uint32_t Imm = cast<llvm::ConstantInt>(Ops[1])->getZExtValue();
10284 unsigned NumElts = Ty->getVectorNumElements();
10285 unsigned NumLanes = Ty->getPrimitiveSizeInBits() / 128;
10286 unsigned NumLaneElts = NumElts / NumLanes;
10289 Imm = (Imm & 0xff) * 0x01010101;
10291 uint32_t Indices[16];
10292 for (
unsigned l = 0; l != NumElts; l += NumLaneElts) {
10293 for (
unsigned i = 0; i != NumLaneElts; ++i) {
10294 Indices[i + l] = (Imm % NumLaneElts) + l;
10295 Imm /= NumLaneElts;
10299 return Builder.CreateShuffleVector(Ops[0], UndefValue::get(Ty),
10300 makeArrayRef(Indices, NumElts),
10303 case X86::BI__builtin_ia32_shufpd:
10304 case X86::BI__builtin_ia32_shufpd256:
10305 case X86::BI__builtin_ia32_shufpd512:
10306 case X86::BI__builtin_ia32_shufps:
10307 case X86::BI__builtin_ia32_shufps256:
10308 case X86::BI__builtin_ia32_shufps512: {
10309 uint32_t Imm = cast<llvm::ConstantInt>(Ops[2])->getZExtValue();
10311 unsigned NumElts = Ty->getVectorNumElements();
10312 unsigned NumLanes = Ty->getPrimitiveSizeInBits() / 128;
10313 unsigned NumLaneElts = NumElts / NumLanes;
10316 Imm = (Imm & 0xff) * 0x01010101;
10318 uint32_t Indices[16];
10319 for (
unsigned l = 0; l != NumElts; l += NumLaneElts) {
10320 for (
unsigned i = 0; i != NumLaneElts; ++i) {
10321 unsigned Index = Imm % NumLaneElts;
10322 Imm /= NumLaneElts;
10323 if (i >= (NumLaneElts / 2))
10325 Indices[l + i] = l + Index;
10329 return Builder.CreateShuffleVector(Ops[0], Ops[1],
10330 makeArrayRef(Indices, NumElts),
10333 case X86::BI__builtin_ia32_permdi256:
10334 case X86::BI__builtin_ia32_permdf256:
10335 case X86::BI__builtin_ia32_permdi512:
10336 case X86::BI__builtin_ia32_permdf512: {
10337 unsigned Imm = cast<llvm::ConstantInt>(Ops[1])->getZExtValue();
10339 unsigned NumElts = Ty->getVectorNumElements();
10342 uint32_t Indices[8];
10343 for (
unsigned l = 0; l != NumElts; l += 4)
10344 for (
unsigned i = 0; i != 4; ++i)
10345 Indices[l + i] = l + ((Imm >> (2 * i)) & 0x3);
10347 return Builder.CreateShuffleVector(Ops[0], UndefValue::get(Ty),
10348 makeArrayRef(Indices, NumElts),
10351 case X86::BI__builtin_ia32_palignr128:
10352 case X86::BI__builtin_ia32_palignr256:
10353 case X86::BI__builtin_ia32_palignr512: {
10354 unsigned ShiftVal = cast<llvm::ConstantInt>(Ops[2])->getZExtValue() & 0xff;
10356 unsigned NumElts = Ops[0]->getType()->getVectorNumElements();
10357 assert(NumElts % 16 == 0);
10361 if (ShiftVal >= 32)
10362 return llvm::Constant::getNullValue(ConvertType(E->
getType()));
10366 if (ShiftVal > 16) {
10369 Ops[0] = llvm::Constant::getNullValue(Ops[0]->getType());
10372 uint32_t Indices[64];
10374 for (
unsigned l = 0; l != NumElts; l += 16) {
10375 for (
unsigned i = 0; i != 16; ++i) {
10376 unsigned Idx = ShiftVal + i;
10378 Idx += NumElts - 16;
10379 Indices[l + i] = Idx + l;
10383 return Builder.CreateShuffleVector(Ops[1], Ops[0],
10384 makeArrayRef(Indices, NumElts),
10387 case X86::BI__builtin_ia32_alignd128:
10388 case X86::BI__builtin_ia32_alignd256:
10389 case X86::BI__builtin_ia32_alignd512:
10390 case X86::BI__builtin_ia32_alignq128:
10391 case X86::BI__builtin_ia32_alignq256:
10392 case X86::BI__builtin_ia32_alignq512: {
10393 unsigned NumElts = Ops[0]->getType()->getVectorNumElements();
10394 unsigned ShiftVal = cast<llvm::ConstantInt>(Ops[2])->getZExtValue() & 0xff;
10397 ShiftVal &= (2 * NumElts) - 1;
10399 uint32_t Indices[16];
10400 for (
unsigned i = 0; i != NumElts; ++i)
10401 Indices[i] = i + ShiftVal;
10403 return Builder.CreateShuffleVector(Ops[1], Ops[0],
10404 makeArrayRef(Indices, NumElts),
10407 case X86::BI__builtin_ia32_shuf_f32x4_256:
10408 case X86::BI__builtin_ia32_shuf_f64x2_256:
10409 case X86::BI__builtin_ia32_shuf_i32x4_256:
10410 case X86::BI__builtin_ia32_shuf_i64x2_256:
10411 case X86::BI__builtin_ia32_shuf_f32x4:
10412 case X86::BI__builtin_ia32_shuf_f64x2:
10413 case X86::BI__builtin_ia32_shuf_i32x4:
10414 case X86::BI__builtin_ia32_shuf_i64x2: {
10415 unsigned Imm = cast<llvm::ConstantInt>(Ops[2])->getZExtValue();
10417 unsigned NumElts = Ty->getVectorNumElements();
10418 unsigned NumLanes = Ty->getPrimitiveSizeInBits() == 512 ? 4 : 2;
10419 unsigned NumLaneElts = NumElts / NumLanes;
10421 uint32_t Indices[16];
10422 for (
unsigned l = 0; l != NumElts; l += NumLaneElts) {
10423 unsigned Index = (Imm % NumLanes) * NumLaneElts;
10425 if (l >= (NumElts / 2))
10427 for (
unsigned i = 0; i != NumLaneElts; ++i) {
10428 Indices[l + i] = Index + i;
10432 return Builder.CreateShuffleVector(Ops[0], Ops[1],
10433 makeArrayRef(Indices, NumElts),
10437 case X86::BI__builtin_ia32_vperm2f128_pd256:
10438 case X86::BI__builtin_ia32_vperm2f128_ps256:
10439 case X86::BI__builtin_ia32_vperm2f128_si256:
10440 case X86::BI__builtin_ia32_permti256: {
10441 unsigned Imm = cast<llvm::ConstantInt>(Ops[2])->getZExtValue();
10442 unsigned NumElts = Ops[0]->getType()->getVectorNumElements();
10450 uint32_t Indices[8];
10451 for (
unsigned l = 0; l != 2; ++l) {
10453 if (Imm & (1 << ((l * 4) + 3)))
10454 OutOps[l] = llvm::ConstantAggregateZero::get(Ops[0]->getType());
10455 else if (Imm & (1 << ((l * 4) + 1)))
10456 OutOps[l] = Ops[1];
10458 OutOps[l] = Ops[0];
10460 for (
unsigned i = 0; i != NumElts/2; ++i) {
10462 unsigned Idx = (l * NumElts) + i;
10465 if (Imm & (1 << (l * 4)))
10467 Indices[(l * (NumElts/2)) + i] = Idx;
10471 return Builder.CreateShuffleVector(OutOps[0], OutOps[1],
10472 makeArrayRef(Indices, NumElts),
10476 case X86::BI__builtin_ia32_pslldqi128_byteshift:
10477 case X86::BI__builtin_ia32_pslldqi256_byteshift:
10478 case X86::BI__builtin_ia32_pslldqi512_byteshift: {
10479 unsigned ShiftVal = cast<llvm::ConstantInt>(Ops[1])->getZExtValue() & 0xff;
10482 unsigned NumElts = ResultType->getVectorNumElements() * 8;
10485 if (ShiftVal >= 16)
10486 return llvm::Constant::getNullValue(ResultType);
10488 uint32_t Indices[64];
10490 for (
unsigned l = 0; l != NumElts; l += 16) {
10491 for (
unsigned i = 0; i != 16; ++i) {
10492 unsigned Idx = NumElts + i - ShiftVal;
10493 if (Idx < NumElts) Idx -= NumElts - 16;
10494 Indices[l + i] = Idx + l;
10498 llvm::Type *VecTy = llvm::VectorType::get(Int8Ty, NumElts);
10499 Value *Cast = Builder.CreateBitCast(Ops[0], VecTy,
"cast");
10500 Value *Zero = llvm::Constant::getNullValue(VecTy);
10501 Value *SV = Builder.CreateShuffleVector(Zero, Cast,
10502 makeArrayRef(Indices, NumElts),
10504 return Builder.CreateBitCast(SV, Ops[0]->getType(),
"cast");
10506 case X86::BI__builtin_ia32_psrldqi128_byteshift:
10507 case X86::BI__builtin_ia32_psrldqi256_byteshift:
10508 case X86::BI__builtin_ia32_psrldqi512_byteshift: {
10509 unsigned ShiftVal = cast<llvm::ConstantInt>(Ops[1])->getZExtValue() & 0xff;
10512 unsigned NumElts = ResultType->getVectorNumElements() * 8;
10515 if (ShiftVal >= 16)
10516 return llvm::Constant::getNullValue(ResultType);
10518 uint32_t Indices[64];
10520 for (
unsigned l = 0; l != NumElts; l += 16) {
10521 for (
unsigned i = 0; i != 16; ++i) {
10522 unsigned Idx = i + ShiftVal;
10523 if (Idx >= 16) Idx += NumElts - 16;
10524 Indices[l + i] = Idx + l;
10528 llvm::Type *VecTy = llvm::VectorType::get(Int8Ty, NumElts);
10529 Value *Cast = Builder.CreateBitCast(Ops[0], VecTy,
"cast");
10530 Value *Zero = llvm::Constant::getNullValue(VecTy);
10531 Value *SV = Builder.CreateShuffleVector(Cast, Zero,
10532 makeArrayRef(Indices, NumElts),
10534 return Builder.CreateBitCast(SV, ResultType,
"cast");
10536 case X86::BI__builtin_ia32_kshiftliqi:
10537 case X86::BI__builtin_ia32_kshiftlihi:
10538 case X86::BI__builtin_ia32_kshiftlisi:
10539 case X86::BI__builtin_ia32_kshiftlidi: {
10540 unsigned ShiftVal = cast<llvm::ConstantInt>(Ops[1])->getZExtValue() & 0xff;
10541 unsigned NumElts = Ops[0]->getType()->getIntegerBitWidth();
10543 if (ShiftVal >= NumElts)
10544 return llvm::Constant::getNullValue(Ops[0]->getType());
10548 uint32_t Indices[64];
10549 for (
unsigned i = 0; i != NumElts; ++i)
10550 Indices[i] = NumElts + i - ShiftVal;
10552 Value *Zero = llvm::Constant::getNullValue(In->getType());
10553 Value *SV = Builder.CreateShuffleVector(Zero, In,
10554 makeArrayRef(Indices, NumElts),
10556 return Builder.CreateBitCast(SV, Ops[0]->getType());
10558 case X86::BI__builtin_ia32_kshiftriqi:
10559 case X86::BI__builtin_ia32_kshiftrihi:
10560 case X86::BI__builtin_ia32_kshiftrisi:
10561 case X86::BI__builtin_ia32_kshiftridi: {
10562 unsigned ShiftVal = cast<llvm::ConstantInt>(Ops[1])->getZExtValue() & 0xff;
10563 unsigned NumElts = Ops[0]->getType()->getIntegerBitWidth();
10565 if (ShiftVal >= NumElts)
10566 return llvm::Constant::getNullValue(Ops[0]->getType());
10570 uint32_t Indices[64];
10571 for (
unsigned i = 0; i != NumElts; ++i)
10572 Indices[i] = i + ShiftVal;
10574 Value *Zero = llvm::Constant::getNullValue(In->getType());
10575 Value *SV = Builder.CreateShuffleVector(In, Zero,
10576 makeArrayRef(Indices, NumElts),
10578 return Builder.CreateBitCast(SV, Ops[0]->getType());
10580 case X86::BI__builtin_ia32_movnti:
10581 case X86::BI__builtin_ia32_movnti64:
10582 case X86::BI__builtin_ia32_movntsd:
10583 case X86::BI__builtin_ia32_movntss: {
10584 llvm::MDNode *
Node = llvm::MDNode::get(
10585 getLLVMContext(), llvm::ConstantAsMetadata::get(Builder.getInt32(1)));
10587 Value *Ptr = Ops[0];
10588 Value *Src = Ops[1];
10591 if (BuiltinID == X86::BI__builtin_ia32_movntsd ||
10592 BuiltinID == X86::BI__builtin_ia32_movntss)
10593 Src = Builder.CreateExtractElement(Src, (uint64_t)0,
"extract");
10596 Value *BC = Builder.CreateBitCast(
10597 Ptr, llvm::PointerType::getUnqual(Src->getType()),
"cast");
10600 StoreInst *SI = Builder.CreateDefaultAlignedStore(Src, BC);
10601 SI->setMetadata(CGM.getModule().getMDKindID(
"nontemporal"),
Node);
10602 SI->setAlignment(1);
10606 case X86::BI__builtin_ia32_vprotb:
10607 case X86::BI__builtin_ia32_vprotw:
10608 case X86::BI__builtin_ia32_vprotd:
10609 case X86::BI__builtin_ia32_vprotq:
10610 case X86::BI__builtin_ia32_vprotbi:
10611 case X86::BI__builtin_ia32_vprotwi:
10612 case X86::BI__builtin_ia32_vprotdi:
10613 case X86::BI__builtin_ia32_vprotqi:
10614 case X86::BI__builtin_ia32_prold128:
10615 case X86::BI__builtin_ia32_prold256:
10616 case X86::BI__builtin_ia32_prold512:
10617 case X86::BI__builtin_ia32_prolq128:
10618 case X86::BI__builtin_ia32_prolq256:
10619 case X86::BI__builtin_ia32_prolq512:
10620 case X86::BI__builtin_ia32_prolvd128:
10621 case X86::BI__builtin_ia32_prolvd256:
10622 case X86::BI__builtin_ia32_prolvd512:
10623 case X86::BI__builtin_ia32_prolvq128:
10624 case X86::BI__builtin_ia32_prolvq256:
10625 case X86::BI__builtin_ia32_prolvq512:
10627 case X86::BI__builtin_ia32_prord128:
10628 case X86::BI__builtin_ia32_prord256:
10629 case X86::BI__builtin_ia32_prord512:
10630 case X86::BI__builtin_ia32_prorq128:
10631 case X86::BI__builtin_ia32_prorq256:
10632 case X86::BI__builtin_ia32_prorq512:
10633 case X86::BI__builtin_ia32_prorvd128:
10634 case X86::BI__builtin_ia32_prorvd256:
10635 case X86::BI__builtin_ia32_prorvd512:
10636 case X86::BI__builtin_ia32_prorvq128:
10637 case X86::BI__builtin_ia32_prorvq256:
10638 case X86::BI__builtin_ia32_prorvq512:
10640 case X86::BI__builtin_ia32_selectb_128:
10641 case X86::BI__builtin_ia32_selectb_256:
10642 case X86::BI__builtin_ia32_selectb_512:
10643 case X86::BI__builtin_ia32_selectw_128:
10644 case X86::BI__builtin_ia32_selectw_256:
10645 case X86::BI__builtin_ia32_selectw_512:
10646 case X86::BI__builtin_ia32_selectd_128:
10647 case X86::BI__builtin_ia32_selectd_256:
10648 case X86::BI__builtin_ia32_selectd_512:
10649 case X86::BI__builtin_ia32_selectq_128:
10650 case X86::BI__builtin_ia32_selectq_256:
10651 case X86::BI__builtin_ia32_selectq_512:
10652 case X86::BI__builtin_ia32_selectps_128:
10653 case X86::BI__builtin_ia32_selectps_256:
10654 case X86::BI__builtin_ia32_selectps_512:
10655 case X86::BI__builtin_ia32_selectpd_128:
10656 case X86::BI__builtin_ia32_selectpd_256:
10657 case X86::BI__builtin_ia32_selectpd_512:
10659 case X86::BI__builtin_ia32_selectss_128:
10660 case X86::BI__builtin_ia32_selectsd_128: {
10661 Value *A = Builder.CreateExtractElement(Ops[1], (uint64_t)0);
10662 Value *B = Builder.CreateExtractElement(Ops[2], (uint64_t)0);
10664 return Builder.CreateInsertElement(Ops[1], A, (uint64_t)0);
10666 case X86::BI__builtin_ia32_cmpb128_mask:
10667 case X86::BI__builtin_ia32_cmpb256_mask:
10668 case X86::BI__builtin_ia32_cmpb512_mask:
10669 case X86::BI__builtin_ia32_cmpw128_mask:
10670 case X86::BI__builtin_ia32_cmpw256_mask:
10671 case X86::BI__builtin_ia32_cmpw512_mask:
10672 case X86::BI__builtin_ia32_cmpd128_mask:
10673 case X86::BI__builtin_ia32_cmpd256_mask:
10674 case X86::BI__builtin_ia32_cmpd512_mask:
10675 case X86::BI__builtin_ia32_cmpq128_mask:
10676 case X86::BI__builtin_ia32_cmpq256_mask:
10677 case X86::BI__builtin_ia32_cmpq512_mask: {
10678 unsigned CC = cast<llvm::ConstantInt>(Ops[2])->getZExtValue() & 0x7;
10681 case X86::BI__builtin_ia32_ucmpb128_mask:
10682 case X86::BI__builtin_ia32_ucmpb256_mask:
10683 case X86::BI__builtin_ia32_ucmpb512_mask:
10684 case X86::BI__builtin_ia32_ucmpw128_mask:
10685 case X86::BI__builtin_ia32_ucmpw256_mask:
10686 case X86::BI__builtin_ia32_ucmpw512_mask:
10687 case X86::BI__builtin_ia32_ucmpd128_mask:
10688 case X86::BI__builtin_ia32_ucmpd256_mask:
10689 case X86::BI__builtin_ia32_ucmpd512_mask:
10690 case X86::BI__builtin_ia32_ucmpq128_mask:
10691 case X86::BI__builtin_ia32_ucmpq256_mask:
10692 case X86::BI__builtin_ia32_ucmpq512_mask: {
10693 unsigned CC = cast<llvm::ConstantInt>(Ops[2])->getZExtValue() & 0x7;
10697 case X86::BI__builtin_ia32_kortestcqi:
10698 case X86::BI__builtin_ia32_kortestchi:
10699 case X86::BI__builtin_ia32_kortestcsi:
10700 case X86::BI__builtin_ia32_kortestcdi: {
10702 Value *C = llvm::Constant::getAllOnesValue(Ops[0]->getType());
10703 Value *Cmp = Builder.CreateICmpEQ(Or, C);
10704 return Builder.CreateZExt(Cmp, ConvertType(E->
getType()));
10706 case X86::BI__builtin_ia32_kortestzqi:
10707 case X86::BI__builtin_ia32_kortestzhi:
10708 case X86::BI__builtin_ia32_kortestzsi:
10709 case X86::BI__builtin_ia32_kortestzdi: {
10711 Value *C = llvm::Constant::getNullValue(Ops[0]->getType());
10712 Value *Cmp = Builder.CreateICmpEQ(Or, C);
10713 return Builder.CreateZExt(Cmp, ConvertType(E->
getType()));
10716 case X86::BI__builtin_ia32_ktestcqi:
10717 case X86::BI__builtin_ia32_ktestzqi:
10718 case X86::BI__builtin_ia32_ktestchi:
10719 case X86::BI__builtin_ia32_ktestzhi:
10720 case X86::BI__builtin_ia32_ktestcsi:
10721 case X86::BI__builtin_ia32_ktestzsi:
10722 case X86::BI__builtin_ia32_ktestcdi:
10723 case X86::BI__builtin_ia32_ktestzdi: {
10725 switch (BuiltinID) {
10726 default: llvm_unreachable(
"Unsupported intrinsic!");
10727 case X86::BI__builtin_ia32_ktestcqi:
10728 IID = Intrinsic::x86_avx512_ktestc_b;
10730 case X86::BI__builtin_ia32_ktestzqi:
10731 IID = Intrinsic::x86_avx512_ktestz_b;
10733 case X86::BI__builtin_ia32_ktestchi:
10734 IID = Intrinsic::x86_avx512_ktestc_w;
10736 case X86::BI__builtin_ia32_ktestzhi:
10737 IID = Intrinsic::x86_avx512_ktestz_w;
10739 case X86::BI__builtin_ia32_ktestcsi:
10740 IID = Intrinsic::x86_avx512_ktestc_d;
10742 case X86::BI__builtin_ia32_ktestzsi:
10743 IID = Intrinsic::x86_avx512_ktestz_d;
10745 case X86::BI__builtin_ia32_ktestcdi:
10746 IID = Intrinsic::x86_avx512_ktestc_q;
10748 case X86::BI__builtin_ia32_ktestzdi:
10749 IID = Intrinsic::x86_avx512_ktestz_q;
10753 unsigned NumElts = Ops[0]->getType()->getIntegerBitWidth();
10756 Function *Intr = CGM.getIntrinsic(IID);
10757 return Builder.CreateCall(Intr, {LHS, RHS});
10760 case X86::BI__builtin_ia32_kaddqi:
10761 case X86::BI__builtin_ia32_kaddhi:
10762 case X86::BI__builtin_ia32_kaddsi:
10763 case X86::BI__builtin_ia32_kadddi: {
10765 switch (BuiltinID) {
10766 default: llvm_unreachable(
"Unsupported intrinsic!");
10767 case X86::BI__builtin_ia32_kaddqi:
10768 IID = Intrinsic::x86_avx512_kadd_b;
10770 case X86::BI__builtin_ia32_kaddhi:
10771 IID = Intrinsic::x86_avx512_kadd_w;
10773 case X86::BI__builtin_ia32_kaddsi:
10774 IID = Intrinsic::x86_avx512_kadd_d;
10776 case X86::BI__builtin_ia32_kadddi:
10777 IID = Intrinsic::x86_avx512_kadd_q;
10781 unsigned NumElts = Ops[0]->getType()->getIntegerBitWidth();
10784 Function *Intr = CGM.getIntrinsic(IID);
10785 Value *Res = Builder.CreateCall(Intr, {LHS, RHS});
10786 return Builder.CreateBitCast(Res, Ops[0]->getType());
10788 case X86::BI__builtin_ia32_kandqi:
10789 case X86::BI__builtin_ia32_kandhi:
10790 case X86::BI__builtin_ia32_kandsi:
10791 case X86::BI__builtin_ia32_kanddi:
10793 case X86::BI__builtin_ia32_kandnqi:
10794 case X86::BI__builtin_ia32_kandnhi:
10795 case X86::BI__builtin_ia32_kandnsi:
10796 case X86::BI__builtin_ia32_kandndi:
10798 case X86::BI__builtin_ia32_korqi:
10799 case X86::BI__builtin_ia32_korhi:
10800 case X86::BI__builtin_ia32_korsi:
10801 case X86::BI__builtin_ia32_kordi:
10803 case X86::BI__builtin_ia32_kxnorqi:
10804 case X86::BI__builtin_ia32_kxnorhi:
10805 case X86::BI__builtin_ia32_kxnorsi:
10806 case X86::BI__builtin_ia32_kxnordi:
10808 case X86::BI__builtin_ia32_kxorqi:
10809 case X86::BI__builtin_ia32_kxorhi:
10810 case X86::BI__builtin_ia32_kxorsi:
10811 case X86::BI__builtin_ia32_kxordi:
10813 case X86::BI__builtin_ia32_knotqi:
10814 case X86::BI__builtin_ia32_knothi:
10815 case X86::BI__builtin_ia32_knotsi:
10816 case X86::BI__builtin_ia32_knotdi: {
10817 unsigned NumElts = Ops[0]->getType()->getIntegerBitWidth();
10819 return Builder.CreateBitCast(Builder.CreateNot(Res),
10820 Ops[0]->getType());
10822 case X86::BI__builtin_ia32_kmovb:
10823 case X86::BI__builtin_ia32_kmovw:
10824 case X86::BI__builtin_ia32_kmovd:
10825 case X86::BI__builtin_ia32_kmovq: {
10829 unsigned NumElts = Ops[0]->getType()->getIntegerBitWidth();
10831 return Builder.CreateBitCast(Res, Ops[0]->getType());
10834 case X86::BI__builtin_ia32_kunpckdi:
10835 case X86::BI__builtin_ia32_kunpcksi:
10836 case X86::BI__builtin_ia32_kunpckhi: {
10837 unsigned NumElts = Ops[0]->getType()->getIntegerBitWidth();
10840 uint32_t Indices[64];
10841 for (
unsigned i = 0; i != NumElts; ++i)
10846 LHS = Builder.CreateShuffleVector(LHS, LHS,
10847 makeArrayRef(Indices, NumElts / 2));
10848 RHS = Builder.CreateShuffleVector(RHS, RHS,
10849 makeArrayRef(Indices, NumElts / 2));
10852 Value *Res = Builder.CreateShuffleVector(RHS, LHS,
10853 makeArrayRef(Indices, NumElts));
10854 return Builder.CreateBitCast(Res, Ops[0]->getType());
10857 case X86::BI__builtin_ia32_vplzcntd_128:
10858 case X86::BI__builtin_ia32_vplzcntd_256:
10859 case X86::BI__builtin_ia32_vplzcntd_512:
10860 case X86::BI__builtin_ia32_vplzcntq_128:
10861 case X86::BI__builtin_ia32_vplzcntq_256:
10862 case X86::BI__builtin_ia32_vplzcntq_512: {
10863 Function *F = CGM.getIntrinsic(Intrinsic::ctlz, Ops[0]->getType());
10864 return Builder.CreateCall(F, {Ops[0],Builder.getInt1(
false)});
10866 case X86::BI__builtin_ia32_sqrtss:
10867 case X86::BI__builtin_ia32_sqrtsd: {
10868 Value *A = Builder.CreateExtractElement(Ops[0], (uint64_t)0);
10870 A = Builder.CreateCall(F, {A});
10871 return Builder.CreateInsertElement(Ops[0], A, (uint64_t)0);
10873 case X86::BI__builtin_ia32_sqrtsd_round_mask:
10874 case X86::BI__builtin_ia32_sqrtss_round_mask: {
10875 unsigned CC = cast<llvm::ConstantInt>(Ops[4])->getZExtValue();
10879 Intrinsic::ID IID = BuiltinID == X86::BI__builtin_ia32_sqrtsd_round_mask ?
10880 Intrinsic::x86_avx512_mask_sqrt_sd :
10881 Intrinsic::x86_avx512_mask_sqrt_ss;
10882 return Builder.CreateCall(CGM.getIntrinsic(IID), Ops);
10884 Value *A = Builder.CreateExtractElement(Ops[1], (uint64_t)0);
10886 A = Builder.CreateCall(F, A);
10887 Value *Src = Builder.CreateExtractElement(Ops[2], (uint64_t)0);
10889 return Builder.CreateInsertElement(Ops[0], A, (uint64_t)0);
10891 case X86::BI__builtin_ia32_sqrtpd256:
10892 case X86::BI__builtin_ia32_sqrtpd:
10893 case X86::BI__builtin_ia32_sqrtps256:
10894 case X86::BI__builtin_ia32_sqrtps:
10895 case X86::BI__builtin_ia32_sqrtps512:
10896 case X86::BI__builtin_ia32_sqrtpd512: {
10897 if (Ops.size() == 2) {
10898 unsigned CC = cast<llvm::ConstantInt>(Ops[1])->getZExtValue();
10902 Intrinsic::ID IID = BuiltinID == X86::BI__builtin_ia32_sqrtps512 ?
10903 Intrinsic::x86_avx512_sqrt_ps_512 :
10904 Intrinsic::x86_avx512_sqrt_pd_512;
10905 return Builder.CreateCall(CGM.getIntrinsic(IID), Ops);
10909 return Builder.CreateCall(F, Ops[0]);
10911 case X86::BI__builtin_ia32_pabsb128:
10912 case X86::BI__builtin_ia32_pabsw128:
10913 case X86::BI__builtin_ia32_pabsd128:
10914 case X86::BI__builtin_ia32_pabsb256:
10915 case X86::BI__builtin_ia32_pabsw256:
10916 case X86::BI__builtin_ia32_pabsd256:
10917 case X86::BI__builtin_ia32_pabsq128:
10918 case X86::BI__builtin_ia32_pabsq256:
10919 case X86::BI__builtin_ia32_pabsb512:
10920 case X86::BI__builtin_ia32_pabsw512:
10921 case X86::BI__builtin_ia32_pabsd512:
10922 case X86::BI__builtin_ia32_pabsq512:
10925 case X86::BI__builtin_ia32_pmaxsb128:
10926 case X86::BI__builtin_ia32_pmaxsw128:
10927 case X86::BI__builtin_ia32_pmaxsd128:
10928 case X86::BI__builtin_ia32_pmaxsq128:
10929 case X86::BI__builtin_ia32_pmaxsb256:
10930 case X86::BI__builtin_ia32_pmaxsw256:
10931 case X86::BI__builtin_ia32_pmaxsd256:
10932 case X86::BI__builtin_ia32_pmaxsq256:
10933 case X86::BI__builtin_ia32_pmaxsb512:
10934 case X86::BI__builtin_ia32_pmaxsw512:
10935 case X86::BI__builtin_ia32_pmaxsd512:
10936 case X86::BI__builtin_ia32_pmaxsq512:
10938 case X86::BI__builtin_ia32_pmaxub128:
10939 case X86::BI__builtin_ia32_pmaxuw128:
10940 case X86::BI__builtin_ia32_pmaxud128:
10941 case X86::BI__builtin_ia32_pmaxuq128:
10942 case X86::BI__builtin_ia32_pmaxub256:
10943 case X86::BI__builtin_ia32_pmaxuw256:
10944 case X86::BI__builtin_ia32_pmaxud256:
10945 case X86::BI__builtin_ia32_pmaxuq256:
10946 case X86::BI__builtin_ia32_pmaxub512:
10947 case X86::BI__builtin_ia32_pmaxuw512:
10948 case X86::BI__builtin_ia32_pmaxud512:
10949 case X86::BI__builtin_ia32_pmaxuq512:
10951 case X86::BI__builtin_ia32_pminsb128:
10952 case X86::BI__builtin_ia32_pminsw128:
10953 case X86::BI__builtin_ia32_pminsd128:
10954 case X86::BI__builtin_ia32_pminsq128:
10955 case X86::BI__builtin_ia32_pminsb256:
10956 case X86::BI__builtin_ia32_pminsw256:
10957 case X86::BI__builtin_ia32_pminsd256:
10958 case X86::BI__builtin_ia32_pminsq256:
10959 case X86::BI__builtin_ia32_pminsb512:
10960 case X86::BI__builtin_ia32_pminsw512:
10961 case X86::BI__builtin_ia32_pminsd512:
10962 case X86::BI__builtin_ia32_pminsq512:
10964 case X86::BI__builtin_ia32_pminub128:
10965 case X86::BI__builtin_ia32_pminuw128:
10966 case X86::BI__builtin_ia32_pminud128:
10967 case X86::BI__builtin_ia32_pminuq128:
10968 case X86::BI__builtin_ia32_pminub256:
10969 case X86::BI__builtin_ia32_pminuw256:
10970 case X86::BI__builtin_ia32_pminud256:
10971 case X86::BI__builtin_ia32_pminuq256:
10972 case X86::BI__builtin_ia32_pminub512:
10973 case X86::BI__builtin_ia32_pminuw512:
10974 case X86::BI__builtin_ia32_pminud512:
10975 case X86::BI__builtin_ia32_pminuq512:
10978 case X86::BI__builtin_ia32_pmuludq128:
10979 case X86::BI__builtin_ia32_pmuludq256:
10980 case X86::BI__builtin_ia32_pmuludq512:
10983 case X86::BI__builtin_ia32_pmuldq128:
10984 case X86::BI__builtin_ia32_pmuldq256:
10985 case X86::BI__builtin_ia32_pmuldq512:
10988 case X86::BI__builtin_ia32_pternlogd512_mask:
10989 case X86::BI__builtin_ia32_pternlogq512_mask:
10990 case X86::BI__builtin_ia32_pternlogd128_mask:
10991 case X86::BI__builtin_ia32_pternlogd256_mask:
10992 case X86::BI__builtin_ia32_pternlogq128_mask:
10993 case X86::BI__builtin_ia32_pternlogq256_mask:
10996 case X86::BI__builtin_ia32_pternlogd512_maskz:
10997 case X86::BI__builtin_ia32_pternlogq512_maskz:
10998 case X86::BI__builtin_ia32_pternlogd128_maskz:
10999 case X86::BI__builtin_ia32_pternlogd256_maskz:
11000 case X86::BI__builtin_ia32_pternlogq128_maskz:
11001 case X86::BI__builtin_ia32_pternlogq256_maskz:
11004 case X86::BI__builtin_ia32_vpshldd128:
11005 case X86::BI__builtin_ia32_vpshldd256:
11006 case X86::BI__builtin_ia32_vpshldd512:
11007 case X86::BI__builtin_ia32_vpshldq128:
11008 case X86::BI__builtin_ia32_vpshldq256:
11009 case X86::BI__builtin_ia32_vpshldq512:
11010 case X86::BI__builtin_ia32_vpshldw128:
11011 case X86::BI__builtin_ia32_vpshldw256:
11012 case X86::BI__builtin_ia32_vpshldw512:
11015 case X86::BI__builtin_ia32_vpshrdd128:
11016 case X86::BI__builtin_ia32_vpshrdd256:
11017 case X86::BI__builtin_ia32_vpshrdd512:
11018 case X86::BI__builtin_ia32_vpshrdq128:
11019 case X86::BI__builtin_ia32_vpshrdq256:
11020 case X86::BI__builtin_ia32_vpshrdq512:
11021 case X86::BI__builtin_ia32_vpshrdw128:
11022 case X86::BI__builtin_ia32_vpshrdw256:
11023 case X86::BI__builtin_ia32_vpshrdw512:
11027 case X86::BI__builtin_ia32_vpshldvd128:
11028 case X86::BI__builtin_ia32_vpshldvd256:
11029 case X86::BI__builtin_ia32_vpshldvd512:
11030 case X86::BI__builtin_ia32_vpshldvq128:
11031 case X86::BI__builtin_ia32_vpshldvq256:
11032 case X86::BI__builtin_ia32_vpshldvq512:
11033 case X86::BI__builtin_ia32_vpshldvw128:
11034 case X86::BI__builtin_ia32_vpshldvw256:
11035 case X86::BI__builtin_ia32_vpshldvw512:
11038 case X86::BI__builtin_ia32_vpshrdvd128:
11039 case X86::BI__builtin_ia32_vpshrdvd256:
11040 case X86::BI__builtin_ia32_vpshrdvd512:
11041 case X86::BI__builtin_ia32_vpshrdvq128:
11042 case X86::BI__builtin_ia32_vpshrdvq256:
11043 case X86::BI__builtin_ia32_vpshrdvq512:
11044 case X86::BI__builtin_ia32_vpshrdvw128:
11045 case X86::BI__builtin_ia32_vpshrdvw256:
11046 case X86::BI__builtin_ia32_vpshrdvw512:
11051 case X86::BI__builtin_ia32_pswapdsf:
11052 case X86::BI__builtin_ia32_pswapdsi: {
11053 llvm::Type *MMXTy = llvm::Type::getX86_MMXTy(getLLVMContext());
11054 Ops[0] = Builder.CreateBitCast(Ops[0], MMXTy,
"cast");
11055 llvm::Function *F = CGM.getIntrinsic(Intrinsic::x86_3dnowa_pswapd);
11056 return Builder.CreateCall(F, Ops,
"pswapd");
11058 case X86::BI__builtin_ia32_rdrand16_step:
11059 case X86::BI__builtin_ia32_rdrand32_step:
11060 case X86::BI__builtin_ia32_rdrand64_step:
11061 case X86::BI__builtin_ia32_rdseed16_step:
11062 case X86::BI__builtin_ia32_rdseed32_step:
11063 case X86::BI__builtin_ia32_rdseed64_step: {
11065 switch (BuiltinID) {
11066 default: llvm_unreachable(
"Unsupported intrinsic!");
11067 case X86::BI__builtin_ia32_rdrand16_step:
11068 ID = Intrinsic::x86_rdrand_16;
11070 case X86::BI__builtin_ia32_rdrand32_step:
11071 ID = Intrinsic::x86_rdrand_32;
11073 case X86::BI__builtin_ia32_rdrand64_step:
11074 ID = Intrinsic::x86_rdrand_64;
11076 case X86::BI__builtin_ia32_rdseed16_step:
11077 ID = Intrinsic::x86_rdseed_16;
11079 case X86::BI__builtin_ia32_rdseed32_step:
11080 ID = Intrinsic::x86_rdseed_32;
11082 case X86::BI__builtin_ia32_rdseed64_step:
11083 ID = Intrinsic::x86_rdseed_64;
11087 Value *Call = Builder.CreateCall(CGM.getIntrinsic(ID));
11088 Builder.CreateDefaultAlignedStore(Builder.CreateExtractValue(Call, 0),
11090 return Builder.CreateExtractValue(Call, 1);
11092 case X86::BI__builtin_ia32_addcarryx_u32:
11093 case X86::BI__builtin_ia32_addcarryx_u64:
11094 case X86::BI__builtin_ia32_subborrow_u32:
11095 case X86::BI__builtin_ia32_subborrow_u64: {
11097 switch (BuiltinID) {
11098 default: llvm_unreachable(
"Unsupported intrinsic!");
11099 case X86::BI__builtin_ia32_addcarryx_u32:
11100 IID = Intrinsic::x86_addcarry_32;
11102 case X86::BI__builtin_ia32_addcarryx_u64:
11103 IID = Intrinsic::x86_addcarry_64;
11105 case X86::BI__builtin_ia32_subborrow_u32:
11106 IID = Intrinsic::x86_subborrow_32;
11108 case X86::BI__builtin_ia32_subborrow_u64:
11109 IID = Intrinsic::x86_subborrow_64;
11113 Value *Call = Builder.CreateCall(CGM.getIntrinsic(IID),
11114 { Ops[0], Ops[1], Ops[2] });
11115 Builder.CreateDefaultAlignedStore(Builder.CreateExtractValue(Call, 1),
11117 return Builder.CreateExtractValue(Call, 0);
11120 case X86::BI__builtin_ia32_fpclassps128_mask:
11121 case X86::BI__builtin_ia32_fpclassps256_mask:
11122 case X86::BI__builtin_ia32_fpclassps512_mask:
11123 case X86::BI__builtin_ia32_fpclasspd128_mask:
11124 case X86::BI__builtin_ia32_fpclasspd256_mask:
11125 case X86::BI__builtin_ia32_fpclasspd512_mask: {
11126 unsigned NumElts = Ops[0]->getType()->getVectorNumElements();
11127 Value *MaskIn = Ops[2];
11128 Ops.erase(&Ops[2]);
11131 switch (BuiltinID) {
11132 default: llvm_unreachable(
"Unsupported intrinsic!");
11133 case X86::BI__builtin_ia32_fpclassps128_mask:
11134 ID = Intrinsic::x86_avx512_fpclass_ps_128;
11136 case X86::BI__builtin_ia32_fpclassps256_mask:
11137 ID = Intrinsic::x86_avx512_fpclass_ps_256;
11139 case X86::BI__builtin_ia32_fpclassps512_mask:
11140 ID = Intrinsic::x86_avx512_fpclass_ps_512;
11142 case X86::BI__builtin_ia32_fpclasspd128_mask:
11143 ID = Intrinsic::x86_avx512_fpclass_pd_128;
11145 case X86::BI__builtin_ia32_fpclasspd256_mask:
11146 ID = Intrinsic::x86_avx512_fpclass_pd_256;
11148 case X86::BI__builtin_ia32_fpclasspd512_mask:
11149 ID = Intrinsic::x86_avx512_fpclass_pd_512;
11153 Value *Fpclass = Builder.CreateCall(CGM.getIntrinsic(ID), Ops);
11157 case X86::BI__builtin_ia32_vpmultishiftqb128:
11158 case X86::BI__builtin_ia32_vpmultishiftqb256:
11159 case X86::BI__builtin_ia32_vpmultishiftqb512: {
11161 switch (BuiltinID) {
11162 default: llvm_unreachable(
"Unsupported intrinsic!");
11163 case X86::BI__builtin_ia32_vpmultishiftqb128:
11164 ID = Intrinsic::x86_avx512_pmultishift_qb_128;
11166 case X86::BI__builtin_ia32_vpmultishiftqb256:
11167 ID = Intrinsic::x86_avx512_pmultishift_qb_256;
11169 case X86::BI__builtin_ia32_vpmultishiftqb512:
11170 ID = Intrinsic::x86_avx512_pmultishift_qb_512;
11174 return Builder.CreateCall(CGM.getIntrinsic(ID), Ops);
11177 case X86::BI__builtin_ia32_vpshufbitqmb128_mask:
11178 case X86::BI__builtin_ia32_vpshufbitqmb256_mask:
11179 case X86::BI__builtin_ia32_vpshufbitqmb512_mask: {
11180 unsigned NumElts = Ops[0]->getType()->getVectorNumElements();
11181 Value *MaskIn = Ops[2];
11182 Ops.erase(&Ops[2]);
11185 switch (BuiltinID) {
11186 default: llvm_unreachable(
"Unsupported intrinsic!");
11187 case X86::BI__builtin_ia32_vpshufbitqmb128_mask:
11188 ID = Intrinsic::x86_avx512_vpshufbitqmb_128;
11190 case X86::BI__builtin_ia32_vpshufbitqmb256_mask:
11191 ID = Intrinsic::x86_avx512_vpshufbitqmb_256;
11193 case X86::BI__builtin_ia32_vpshufbitqmb512_mask:
11194 ID = Intrinsic::x86_avx512_vpshufbitqmb_512;
11198 Value *Shufbit = Builder.CreateCall(CGM.getIntrinsic(ID), Ops);
11203 case X86::BI__builtin_ia32_cmpeqps:
11204 case X86::BI__builtin_ia32_cmpeqpd:
11205 return getVectorFCmpIR(CmpInst::FCMP_OEQ);
11206 case X86::BI__builtin_ia32_cmpltps:
11207 case X86::BI__builtin_ia32_cmpltpd:
11208 return getVectorFCmpIR(CmpInst::FCMP_OLT);
11209 case X86::BI__builtin_ia32_cmpleps:
11210 case X86::BI__builtin_ia32_cmplepd:
11211 return getVectorFCmpIR(CmpInst::FCMP_OLE);
11212 case X86::BI__builtin_ia32_cmpunordps:
11213 case X86::BI__builtin_ia32_cmpunordpd:
11214 return getVectorFCmpIR(CmpInst::FCMP_UNO);
11215 case X86::BI__builtin_ia32_cmpneqps:
11216 case X86::BI__builtin_ia32_cmpneqpd:
11217 return getVectorFCmpIR(CmpInst::FCMP_UNE);
11218 case X86::BI__builtin_ia32_cmpnltps:
11219 case X86::BI__builtin_ia32_cmpnltpd:
11220 return getVectorFCmpIR(CmpInst::FCMP_UGE);
11221 case X86::BI__builtin_ia32_cmpnleps:
11222 case X86::BI__builtin_ia32_cmpnlepd:
11223 return getVectorFCmpIR(CmpInst::FCMP_UGT);
11224 case X86::BI__builtin_ia32_cmpordps:
11225 case X86::BI__builtin_ia32_cmpordpd:
11226 return getVectorFCmpIR(CmpInst::FCMP_ORD);
11227 case X86::BI__builtin_ia32_cmpps:
11228 case X86::BI__builtin_ia32_cmpps256:
11229 case X86::BI__builtin_ia32_cmppd:
11230 case X86::BI__builtin_ia32_cmppd256:
11231 case X86::BI__builtin_ia32_cmpps128_mask:
11232 case X86::BI__builtin_ia32_cmpps256_mask:
11233 case X86::BI__builtin_ia32_cmpps512_mask:
11234 case X86::BI__builtin_ia32_cmppd128_mask:
11235 case X86::BI__builtin_ia32_cmppd256_mask:
11236 case X86::BI__builtin_ia32_cmppd512_mask: {
11245 unsigned CC = cast<llvm::ConstantInt>(Ops[2])->getZExtValue() & 0x1f;
11250 FCmpInst::Predicate Pred;
11252 case 0x00: Pred = FCmpInst::FCMP_OEQ;
break;
11253 case 0x01: Pred = FCmpInst::FCMP_OLT;
break;
11254 case 0x02: Pred = FCmpInst::FCMP_OLE;
break;
11255 case 0x03: Pred = FCmpInst::FCMP_UNO;
break;
11256 case 0x04: Pred = FCmpInst::FCMP_UNE;
break;
11257 case 0x05: Pred = FCmpInst::FCMP_UGE;
break;
11258 case 0x06: Pred = FCmpInst::FCMP_UGT;
break;
11259 case 0x07: Pred = FCmpInst::FCMP_ORD;
break;
11260 case 0x08: Pred = FCmpInst::FCMP_UEQ;
break;
11261 case 0x09: Pred = FCmpInst::FCMP_ULT;
break;
11262 case 0x0a: Pred = FCmpInst::FCMP_ULE;
break;
11263 case 0x0b: Pred = FCmpInst::FCMP_FALSE;
break;
11264 case 0x0c: Pred = FCmpInst::FCMP_ONE;
break;
11265 case 0x0d: Pred = FCmpInst::FCMP_OGE;
break;
11266 case 0x0e: Pred = FCmpInst::FCMP_OGT;
break;
11267 case 0x0f: Pred = FCmpInst::FCMP_TRUE;
break;
11268 case 0x10: Pred = FCmpInst::FCMP_OEQ;
break;
11269 case 0x11: Pred = FCmpInst::FCMP_OLT;
break;
11270 case 0x12: Pred = FCmpInst::FCMP_OLE;
break;
11271 case 0x13: Pred = FCmpInst::FCMP_UNO;
break;
11272 case 0x14: Pred = FCmpInst::FCMP_UNE;
break;
11273 case 0x15: Pred = FCmpInst::FCMP_UGE;
break;
11274 case 0x16: Pred = FCmpInst::FCMP_UGT;
break;
11275 case 0x17: Pred = FCmpInst::FCMP_ORD;
break;
11276 case 0x18: Pred = FCmpInst::FCMP_UEQ;
break;
11277 case 0x19: Pred = FCmpInst::FCMP_ULT;
break;
11278 case 0x1a: Pred = FCmpInst::FCMP_ULE;
break;
11279 case 0x1b: Pred = FCmpInst::FCMP_FALSE;
break;
11280 case 0x1c: Pred = FCmpInst::FCMP_ONE;
break;
11281 case 0x1d: Pred = FCmpInst::FCMP_OGE;
break;
11282 case 0x1e: Pred = FCmpInst::FCMP_OGT;
break;
11283 case 0x1f: Pred = FCmpInst::FCMP_TRUE;
break;
11284 default: llvm_unreachable(
"Unhandled CC");
11289 switch (BuiltinID) {
11290 case X86::BI__builtin_ia32_cmpps512_mask:
11291 case X86::BI__builtin_ia32_cmppd512_mask:
11292 case X86::BI__builtin_ia32_cmpps128_mask:
11293 case X86::BI__builtin_ia32_cmpps256_mask:
11294 case X86::BI__builtin_ia32_cmppd128_mask:
11295 case X86::BI__builtin_ia32_cmppd256_mask: {
11296 unsigned NumElts = Ops[0]->getType()->getVectorNumElements();
11297 Value *Cmp = Builder.CreateFCmp(Pred, Ops[0], Ops[1]);
11301 return getVectorFCmpIR(Pred);
11306 case X86::BI__builtin_ia32_cmpeqss:
11307 return getCmpIntrinsicCall(Intrinsic::x86_sse_cmp_ss, 0);
11308 case X86::BI__builtin_ia32_cmpltss:
11309 return getCmpIntrinsicCall(Intrinsic::x86_sse_cmp_ss, 1);
11310 case X86::BI__builtin_ia32_cmpless:
11311 return getCmpIntrinsicCall(Intrinsic::x86_sse_cmp_ss, 2);
11312 case X86::BI__builtin_ia32_cmpunordss:
11313 return getCmpIntrinsicCall(Intrinsic::x86_sse_cmp_ss, 3);
11314 case X86::BI__builtin_ia32_cmpneqss:
11315 return getCmpIntrinsicCall(Intrinsic::x86_sse_cmp_ss, 4);
11316 case X86::BI__builtin_ia32_cmpnltss:
11317 return getCmpIntrinsicCall(Intrinsic::x86_sse_cmp_ss, 5);
11318 case X86::BI__builtin_ia32_cmpnless:
11319 return getCmpIntrinsicCall(Intrinsic::x86_sse_cmp_ss, 6);
11320 case X86::BI__builtin_ia32_cmpordss:
11321 return getCmpIntrinsicCall(Intrinsic::x86_sse_cmp_ss, 7);
11322 case X86::BI__builtin_ia32_cmpeqsd:
11323 return getCmpIntrinsicCall(Intrinsic::x86_sse2_cmp_sd, 0);
11324 case X86::BI__builtin_ia32_cmpltsd:
11325 return getCmpIntrinsicCall(Intrinsic::x86_sse2_cmp_sd, 1);
11326 case X86::BI__builtin_ia32_cmplesd:
11327 return getCmpIntrinsicCall(Intrinsic::x86_sse2_cmp_sd, 2);
11328 case X86::BI__builtin_ia32_cmpunordsd:
11329 return getCmpIntrinsicCall(Intrinsic::x86_sse2_cmp_sd, 3);
11330 case X86::BI__builtin_ia32_cmpneqsd:
11331 return getCmpIntrinsicCall(Intrinsic::x86_sse2_cmp_sd, 4);
11332 case X86::BI__builtin_ia32_cmpnltsd:
11333 return getCmpIntrinsicCall(Intrinsic::x86_sse2_cmp_sd, 5);
11334 case X86::BI__builtin_ia32_cmpnlesd:
11335 return getCmpIntrinsicCall(Intrinsic::x86_sse2_cmp_sd, 6);
11336 case X86::BI__builtin_ia32_cmpordsd:
11337 return getCmpIntrinsicCall(Intrinsic::x86_sse2_cmp_sd, 7);
11339 case X86::BI__emul:
11340 case X86::BI__emulu: {
11341 llvm::Type *Int64Ty = llvm::IntegerType::get(getLLVMContext(), 64);
11342 bool isSigned = (BuiltinID == X86::BI__emul);
11343 Value *LHS = Builder.CreateIntCast(Ops[0], Int64Ty, isSigned);
11344 Value *RHS = Builder.CreateIntCast(Ops[1], Int64Ty, isSigned);
11345 return Builder.CreateMul(LHS, RHS,
"", !isSigned, isSigned);
11347 case X86::BI__mulh:
11348 case X86::BI__umulh:
11349 case X86::BI_mul128:
11350 case X86::BI_umul128: {
11352 llvm::Type *Int128Ty = llvm::IntegerType::get(getLLVMContext(), 128);
11354 bool IsSigned = (BuiltinID == X86::BI__mulh || BuiltinID == X86::BI_mul128);
11355 Value *LHS = Builder.CreateIntCast(Ops[0], Int128Ty, IsSigned);
11356 Value *RHS = Builder.CreateIntCast(Ops[1], Int128Ty, IsSigned);
11358 Value *MulResult, *HigherBits;
11360 MulResult = Builder.CreateNSWMul(LHS, RHS);
11361 HigherBits = Builder.CreateAShr(MulResult, 64);
11363 MulResult = Builder.CreateNUWMul(LHS, RHS);
11364 HigherBits = Builder.CreateLShr(MulResult, 64);
11366 HigherBits = Builder.CreateIntCast(HigherBits, ResType, IsSigned);
11368 if (BuiltinID == X86::BI__mulh || BuiltinID == X86::BI__umulh)
11371 Address HighBitsAddress = EmitPointerWithAlignment(E->
getArg(2));
11372 Builder.CreateStore(HigherBits, HighBitsAddress);
11373 return Builder.CreateIntCast(MulResult, ResType, IsSigned);
11376 case X86::BI__faststorefence: {
11377 return Builder.CreateFence(llvm::AtomicOrdering::SequentiallyConsistent,
11380 case X86::BI__shiftleft128:
11381 case X86::BI__shiftright128: {
11388 llvm::Type *Int128Ty = Builder.getInt128Ty();
11389 Value *Val = Builder.CreateOr(
11390 Builder.CreateShl(Builder.CreateZExt(Ops[1], Int128Ty), 64),
11391 Builder.CreateZExt(Ops[0], Int128Ty));
11392 Value *Amt = Builder.CreateAnd(Builder.CreateZExt(Ops[2], Int128Ty),
11393 llvm::ConstantInt::get(Int128Ty, 0x3f));
11395 if (BuiltinID == X86::BI__shiftleft128)
11396 Res = Builder.CreateLShr(Builder.CreateShl(Val, Amt), 64);
11398 Res = Builder.CreateLShr(Val, Amt);
11399 return Builder.CreateTrunc(Res, Int64Ty);
11401 case X86::BI_ReadWriteBarrier:
11402 case X86::BI_ReadBarrier:
11403 case X86::BI_WriteBarrier: {
11404 return Builder.CreateFence(llvm::AtomicOrdering::SequentiallyConsistent,
11405 llvm::SyncScope::SingleThread);
11407 case X86::BI_BitScanForward:
11408 case X86::BI_BitScanForward64:
11409 return EmitMSVCBuiltinExpr(MSVCIntrin::_BitScanForward, E);
11410 case X86::BI_BitScanReverse:
11411 case X86::BI_BitScanReverse64:
11412 return EmitMSVCBuiltinExpr(MSVCIntrin::_BitScanReverse, E);
11414 case X86::BI_InterlockedAnd64:
11415 return EmitMSVCBuiltinExpr(MSVCIntrin::_InterlockedAnd, E);
11416 case X86::BI_InterlockedExchange64:
11417 return EmitMSVCBuiltinExpr(MSVCIntrin::_InterlockedExchange, E);
11418 case X86::BI_InterlockedExchangeAdd64:
11419 return EmitMSVCBuiltinExpr(MSVCIntrin::_InterlockedExchangeAdd, E);
11420 case X86::BI_InterlockedExchangeSub64:
11421 return EmitMSVCBuiltinExpr(MSVCIntrin::_InterlockedExchangeSub, E);
11422 case X86::BI_InterlockedOr64:
11423 return EmitMSVCBuiltinExpr(MSVCIntrin::_InterlockedOr, E);
11424 case X86::BI_InterlockedXor64:
11425 return EmitMSVCBuiltinExpr(MSVCIntrin::_InterlockedXor, E);
11426 case X86::BI_InterlockedDecrement64:
11427 return EmitMSVCBuiltinExpr(MSVCIntrin::_InterlockedDecrement, E);
11428 case X86::BI_InterlockedIncrement64:
11429 return EmitMSVCBuiltinExpr(MSVCIntrin::_InterlockedIncrement, E);
11430 case X86::BI_InterlockedCompareExchange128: {
11437 llvm::Type *Int128Ty = Builder.getInt128Ty();
11438 llvm::Type *Int128PtrTy = Int128Ty->getPointerTo();
11440 Value *Destination =
11441 Builder.CreateBitCast(Ops[0], Int128PtrTy);
11442 Value *ExchangeHigh128 = Builder.CreateZExt(Ops[1], Int128Ty);
11443 Value *ExchangeLow128 = Builder.CreateZExt(Ops[2], Int128Ty);
11444 Address ComparandResult(Builder.CreateBitCast(Ops[3], Int128PtrTy),
11445 getContext().toCharUnitsFromBits(128));
11447 Value *Exchange = Builder.CreateOr(
11448 Builder.CreateShl(ExchangeHigh128, 64,
"",
false,
false),
11451 Value *Comparand = Builder.CreateLoad(ComparandResult);
11453 AtomicCmpXchgInst *CXI =
11454 Builder.CreateAtomicCmpXchg(Destination, Comparand, Exchange,
11455 AtomicOrdering::SequentiallyConsistent,
11456 AtomicOrdering::SequentiallyConsistent);
11457 CXI->setVolatile(
true);
11460 Builder.CreateStore(Builder.CreateExtractValue(CXI, 0), ComparandResult);
11463 Value *Success = Builder.CreateExtractValue(CXI, 1);
11464 return Builder.CreateZExt(Success, ConvertType(E->
getType()));
11467 case X86::BI_AddressOfReturnAddress: {
11468 Value *F = CGM.getIntrinsic(Intrinsic::addressofreturnaddress);
11469 return Builder.CreateCall(F);
11471 case X86::BI__stosb: {
11474 return Builder.CreateMemSet(Ops[0], Ops[1], Ops[2], 1,
true);
11478 return EmitTrapCall(Intrinsic::trap);
11479 case X86::BI__int2c: {
11481 llvm::FunctionType *FTy = llvm::FunctionType::get(VoidTy,
false);
11482 llvm::InlineAsm *IA =
11483 llvm::InlineAsm::get(FTy,
"int $$0x2c",
"",
true);
11484 llvm::AttributeList NoReturnAttr = llvm::AttributeList::get(
11485 getLLVMContext(), llvm::AttributeList::FunctionIndex,
11486 llvm::Attribute::NoReturn);
11487 CallSite CS = Builder.CreateCall(IA);
11488 CS.setAttributes(NoReturnAttr);
11489 return CS.getInstruction();
11491 case X86::BI__readfsbyte:
11492 case X86::BI__readfsword:
11493 case X86::BI__readfsdword:
11494 case X86::BI__readfsqword: {
11497 Builder.CreateIntToPtr(Ops[0], llvm::PointerType::get(IntTy, 257));
11498 LoadInst *Load = Builder.CreateAlignedLoad(
11499 IntTy, Ptr, getContext().getTypeAlignInChars(E->
getType()));
11500 Load->setVolatile(
true);
11503 case X86::BI__readgsbyte:
11504 case X86::BI__readgsword:
11505 case X86::BI__readgsdword:
11506 case X86::BI__readgsqword: {
11509 Builder.CreateIntToPtr(Ops[0], llvm::PointerType::get(IntTy, 256));
11510 LoadInst *Load = Builder.CreateAlignedLoad(
11511 IntTy, Ptr, getContext().getTypeAlignInChars(E->
getType()));
11512 Load->setVolatile(
true);
11515 case X86::BI__builtin_ia32_paddsb512:
11516 case X86::BI__builtin_ia32_paddsw512:
11517 case X86::BI__builtin_ia32_paddsb256:
11518 case X86::BI__builtin_ia32_paddsw256:
11519 case X86::BI__builtin_ia32_paddsb128:
11520 case X86::BI__builtin_ia32_paddsw128:
11522 case X86::BI__builtin_ia32_paddusb512:
11523 case X86::BI__builtin_ia32_paddusw512:
11524 case X86::BI__builtin_ia32_paddusb256:
11525 case X86::BI__builtin_ia32_paddusw256:
11526 case X86::BI__builtin_ia32_paddusb128:
11527 case X86::BI__builtin_ia32_paddusw128:
11529 case X86::BI__builtin_ia32_psubsb512:
11530 case X86::BI__builtin_ia32_psubsw512:
11531 case X86::BI__builtin_ia32_psubsb256:
11532 case X86::BI__builtin_ia32_psubsw256:
11533 case X86::BI__builtin_ia32_psubsb128:
11534 case X86::BI__builtin_ia32_psubsw128:
11536 case X86::BI__builtin_ia32_psubusb512:
11537 case X86::BI__builtin_ia32_psubusw512:
11538 case X86::BI__builtin_ia32_psubusb256:
11539 case X86::BI__builtin_ia32_psubusw256:
11540 case X86::BI__builtin_ia32_psubusb128:
11541 case X86::BI__builtin_ia32_psubusw128:
11550 for (
unsigned i = 0, e = E->
getNumArgs(); i != e; i++)
11551 Ops.push_back(EmitScalarExpr(E->
getArg(i)));
11555 switch (BuiltinID) {
11556 default:
return nullptr;
11560 case PPC::BI__builtin_ppc_get_timebase:
11561 return Builder.CreateCall(CGM.getIntrinsic(Intrinsic::readcyclecounter));
11564 case PPC::BI__builtin_altivec_lvx:
11565 case PPC::BI__builtin_altivec_lvxl:
11566 case PPC::BI__builtin_altivec_lvebx:
11567 case PPC::BI__builtin_altivec_lvehx:
11568 case PPC::BI__builtin_altivec_lvewx:
11569 case PPC::BI__builtin_altivec_lvsl:
11570 case PPC::BI__builtin_altivec_lvsr:
11571 case PPC::BI__builtin_vsx_lxvd2x:
11572 case PPC::BI__builtin_vsx_lxvw4x:
11573 case PPC::BI__builtin_vsx_lxvd2x_be:
11574 case PPC::BI__builtin_vsx_lxvw4x_be:
11575 case PPC::BI__builtin_vsx_lxvl:
11576 case PPC::BI__builtin_vsx_lxvll:
11578 if(BuiltinID == PPC::BI__builtin_vsx_lxvl ||
11579 BuiltinID == PPC::BI__builtin_vsx_lxvll){
11580 Ops[0] = Builder.CreateBitCast(Ops[0], Int8PtrTy);
11582 Ops[1] = Builder.CreateBitCast(Ops[1], Int8PtrTy);
11583 Ops[0] = Builder.CreateGEP(Ops[1], Ops[0]);
11587 switch (BuiltinID) {
11588 default: llvm_unreachable(
"Unsupported ld/lvsl/lvsr intrinsic!");
11589 case PPC::BI__builtin_altivec_lvx:
11590 ID = Intrinsic::ppc_altivec_lvx;
11592 case PPC::BI__builtin_altivec_lvxl:
11593 ID = Intrinsic::ppc_altivec_lvxl;
11595 case PPC::BI__builtin_altivec_lvebx:
11596 ID = Intrinsic::ppc_altivec_lvebx;
11598 case PPC::BI__builtin_altivec_lvehx:
11599 ID = Intrinsic::ppc_altivec_lvehx;
11601 case PPC::BI__builtin_altivec_lvewx:
11602 ID = Intrinsic::ppc_altivec_lvewx;
11604 case PPC::BI__builtin_altivec_lvsl:
11605 ID = Intrinsic::ppc_altivec_lvsl;
11607 case PPC::BI__builtin_altivec_lvsr:
11608 ID = Intrinsic::ppc_altivec_lvsr;
11610 case PPC::BI__builtin_vsx_lxvd2x:
11611 ID = Intrinsic::ppc_vsx_lxvd2x;
11613 case PPC::BI__builtin_vsx_lxvw4x:
11614 ID = Intrinsic::ppc_vsx_lxvw4x;
11616 case PPC::BI__builtin_vsx_lxvd2x_be:
11617 ID = Intrinsic::ppc_vsx_lxvd2x_be;
11619 case PPC::BI__builtin_vsx_lxvw4x_be:
11620 ID = Intrinsic::ppc_vsx_lxvw4x_be;
11622 case PPC::BI__builtin_vsx_lxvl:
11623 ID = Intrinsic::ppc_vsx_lxvl;
11625 case PPC::BI__builtin_vsx_lxvll:
11626 ID = Intrinsic::ppc_vsx_lxvll;
11629 llvm::Function *F = CGM.getIntrinsic(ID);
11630 return Builder.CreateCall(F, Ops,
"");
11634 case PPC::BI__builtin_altivec_stvx:
11635 case PPC::BI__builtin_altivec_stvxl:
11636 case PPC::BI__builtin_altivec_stvebx:
11637 case PPC::BI__builtin_altivec_stvehx:
11638 case PPC::BI__builtin_altivec_stvewx:
11639 case PPC::BI__builtin_vsx_stxvd2x:
11640 case PPC::BI__builtin_vsx_stxvw4x:
11641 case PPC::BI__builtin_vsx_stxvd2x_be:
11642 case PPC::BI__builtin_vsx_stxvw4x_be:
11643 case PPC::BI__builtin_vsx_stxvl:
11644 case PPC::BI__builtin_vsx_stxvll:
11646 if(BuiltinID == PPC::BI__builtin_vsx_stxvl ||
11647 BuiltinID == PPC::BI__builtin_vsx_stxvll ){
11648 Ops[1] = Builder.CreateBitCast(Ops[1], Int8PtrTy);
11650 Ops[2] = Builder.CreateBitCast(Ops[2], Int8PtrTy);
11651 Ops[1] = Builder.CreateGEP(Ops[2], Ops[1]);
11655 switch (BuiltinID) {
11656 default: llvm_unreachable(
"Unsupported st intrinsic!");
11657 case PPC::BI__builtin_altivec_stvx:
11658 ID = Intrinsic::ppc_altivec_stvx;
11660 case PPC::BI__builtin_altivec_stvxl:
11661 ID = Intrinsic::ppc_altivec_stvxl;
11663 case PPC::BI__builtin_altivec_stvebx:
11664 ID = Intrinsic::ppc_altivec_stvebx;
11666 case PPC::BI__builtin_altivec_stvehx:
11667 ID = Intrinsic::ppc_altivec_stvehx;
11669 case PPC::BI__builtin_altivec_stvewx:
11670 ID = Intrinsic::ppc_altivec_stvewx;
11672 case PPC::BI__builtin_vsx_stxvd2x:
11673 ID = Intrinsic::ppc_vsx_stxvd2x;
11675 case PPC::BI__builtin_vsx_stxvw4x:
11676 ID = Intrinsic::ppc_vsx_stxvw4x;
11678 case PPC::BI__builtin_vsx_stxvd2x_be:
11679 ID = Intrinsic::ppc_vsx_stxvd2x_be;
11681 case PPC::BI__builtin_vsx_stxvw4x_be:
11682 ID = Intrinsic::ppc_vsx_stxvw4x_be;
11684 case PPC::BI__builtin_vsx_stxvl:
11685 ID = Intrinsic::ppc_vsx_stxvl;
11687 case PPC::BI__builtin_vsx_stxvll:
11688 ID = Intrinsic::ppc_vsx_stxvll;
11691 llvm::Function *F = CGM.getIntrinsic(ID);
11692 return Builder.CreateCall(F, Ops,
"");
11695 case PPC::BI__builtin_vsx_xvsqrtsp:
11696 case PPC::BI__builtin_vsx_xvsqrtdp: {
11700 llvm::Function *F = CGM.getIntrinsic(ID, ResultType);
11701 return Builder.CreateCall(F, X);
11704 case PPC::BI__builtin_altivec_vclzb:
11705 case PPC::BI__builtin_altivec_vclzh:
11706 case PPC::BI__builtin_altivec_vclzw:
11707 case PPC::BI__builtin_altivec_vclzd: {
11710 Value *Undef = ConstantInt::get(Builder.getInt1Ty(),
false);
11711 Function *F = CGM.getIntrinsic(Intrinsic::ctlz, ResultType);
11712 return Builder.CreateCall(F, {
X, Undef});
11714 case PPC::BI__builtin_altivec_vctzb:
11715 case PPC::BI__builtin_altivec_vctzh:
11716 case PPC::BI__builtin_altivec_vctzw:
11717 case PPC::BI__builtin_altivec_vctzd: {
11720 Value *Undef = ConstantInt::get(Builder.getInt1Ty(),
false);
11721 Function *F = CGM.getIntrinsic(Intrinsic::cttz, ResultType);
11722 return Builder.CreateCall(F, {
X, Undef});
11724 case PPC::BI__builtin_altivec_vpopcntb:
11725 case PPC::BI__builtin_altivec_vpopcnth:
11726 case PPC::BI__builtin_altivec_vpopcntw:
11727 case PPC::BI__builtin_altivec_vpopcntd: {
11730 llvm::Function *F = CGM.getIntrinsic(Intrinsic::ctpop, ResultType);
11731 return Builder.CreateCall(F, X);
11734 case PPC::BI__builtin_vsx_xvcpsgnsp:
11735 case PPC::BI__builtin_vsx_xvcpsgndp: {
11740 llvm::Function *F = CGM.getIntrinsic(ID, ResultType);
11741 return Builder.CreateCall(F, {
X, Y});
11744 case PPC::BI__builtin_vsx_xvrspip:
11745 case PPC::BI__builtin_vsx_xvrdpip:
11746 case PPC::BI__builtin_vsx_xvrdpim:
11747 case PPC::BI__builtin_vsx_xvrspim:
11748 case PPC::BI__builtin_vsx_xvrdpi:
11749 case PPC::BI__builtin_vsx_xvrspi:
11750 case PPC::BI__builtin_vsx_xvrdpic:
11751 case PPC::BI__builtin_vsx_xvrspic:
11752 case PPC::BI__builtin_vsx_xvrdpiz:
11753 case PPC::BI__builtin_vsx_xvrspiz: {
11756 if (BuiltinID == PPC::BI__builtin_vsx_xvrdpim ||
11757 BuiltinID == PPC::BI__builtin_vsx_xvrspim)
11759 else if (BuiltinID == PPC::BI__builtin_vsx_xvrdpi ||
11760 BuiltinID == PPC::BI__builtin_vsx_xvrspi)
11762 else if (BuiltinID == PPC::BI__builtin_vsx_xvrdpic ||
11763 BuiltinID == PPC::BI__builtin_vsx_xvrspic)
11765 else if (BuiltinID == PPC::BI__builtin_vsx_xvrdpip ||
11766 BuiltinID == PPC::BI__builtin_vsx_xvrspip)
11768 else if (BuiltinID == PPC::BI__builtin_vsx_xvrdpiz ||
11769 BuiltinID == PPC::BI__builtin_vsx_xvrspiz)
11771 llvm::Function *F = CGM.getIntrinsic(ID, ResultType);
11772 return Builder.CreateCall(F, X);
11776 case PPC::BI__builtin_vsx_xvabsdp:
11777 case PPC::BI__builtin_vsx_xvabssp: {
11781 return Builder.CreateCall(F, X);
11785 case PPC::BI__builtin_vsx_xvmaddadp:
11786 case PPC::BI__builtin_vsx_xvmaddasp:
11787 case PPC::BI__builtin_vsx_xvnmaddadp:
11788 case PPC::BI__builtin_vsx_xvnmaddasp:
11789 case PPC::BI__builtin_vsx_xvmsubadp:
11790 case PPC::BI__builtin_vsx_xvmsubasp:
11791 case PPC::BI__builtin_vsx_xvnmsubadp:
11792 case PPC::BI__builtin_vsx_xvnmsubasp: {
11797 Value *Zero = llvm::ConstantFP::getZeroValueForNegation(ResultType);
11798 llvm::Function *F = CGM.getIntrinsic(
Intrinsic::fma, ResultType);
11799 switch (BuiltinID) {
11800 case PPC::BI__builtin_vsx_xvmaddadp:
11801 case PPC::BI__builtin_vsx_xvmaddasp:
11802 return Builder.CreateCall(F, {
X, Y, Z});
11803 case PPC::BI__builtin_vsx_xvnmaddadp:
11804 case PPC::BI__builtin_vsx_xvnmaddasp:
11805 return Builder.CreateFSub(Zero,
11806 Builder.CreateCall(F, {X, Y, Z}),
"sub");
11807 case PPC::BI__builtin_vsx_xvmsubadp:
11808 case PPC::BI__builtin_vsx_xvmsubasp:
11809 return Builder.CreateCall(F,
11810 {
X, Y, Builder.CreateFSub(Zero, Z,
"sub")});
11811 case PPC::BI__builtin_vsx_xvnmsubadp:
11812 case PPC::BI__builtin_vsx_xvnmsubasp:
11814 Builder.CreateCall(F, {
X, Y, Builder.CreateFSub(Zero, Z,
"sub")});
11815 return Builder.CreateFSub(Zero, FsubRes,
"sub");
11817 llvm_unreachable(
"Unknown FMA operation");
11821 case PPC::BI__builtin_vsx_insertword: {
11822 llvm::Function *F = CGM.getIntrinsic(Intrinsic::ppc_vsx_xxinsertw);
11826 ConstantInt *ArgCI = dyn_cast<ConstantInt>(Ops[2]);
11828 "Third arg to xxinsertw intrinsic must be constant integer");
11829 const int64_t MaxIndex = 12;
11830 int64_t Index =
clamp(ArgCI->getSExtValue(), 0, MaxIndex);
11837 std::swap(Ops[0], Ops[1]);
11841 Ops[1] = Builder.CreateBitCast(Ops[1], llvm::VectorType::get(Int64Ty, 2));
11843 if (getTarget().isLittleEndian()) {
11845 Constant *ShuffleElts[2] = { ConstantInt::get(Int32Ty, 1),
11846 ConstantInt::get(Int32Ty, 0)
11848 Constant *ShuffleMask = llvm::ConstantVector::get(ShuffleElts);
11851 Ops[0] = Builder.CreateBitCast(Ops[0], llvm::VectorType::get(Int64Ty, 2));
11852 Ops[0] = Builder.CreateShuffleVector(Ops[0], Ops[0], ShuffleMask);
11855 Index = MaxIndex - Index;
11859 Ops[0] = Builder.CreateBitCast(Ops[0], llvm::VectorType::get(Int32Ty, 4));
11860 Ops[2] = ConstantInt::getSigned(Int32Ty, Index);
11861 return Builder.CreateCall(F, Ops);
11864 case PPC::BI__builtin_vsx_extractuword: {
11865 llvm::Function *F = CGM.getIntrinsic(Intrinsic::ppc_vsx_xxextractuw);
11868 Ops[0] = Builder.CreateBitCast(Ops[0], llvm::VectorType::get(Int64Ty, 2));
11872 ConstantInt *ArgCI = dyn_cast<ConstantInt>(Ops[1]);
11874 "Second Arg to xxextractuw intrinsic must be a constant integer!");
11875 const int64_t MaxIndex = 12;
11876 int64_t Index =
clamp(ArgCI->getSExtValue(), 0, MaxIndex);
11878 if (getTarget().isLittleEndian()) {
11880 Index = MaxIndex - Index;
11881 Ops[1] = ConstantInt::getSigned(Int32Ty, Index);
11884 Value *Call = Builder.CreateCall(F, Ops);
11887 Constant *ShuffleElts[2] = { ConstantInt::get(Int32Ty, 1),
11888 ConstantInt::get(Int32Ty, 0)
11890 Constant *ShuffleMask = llvm::ConstantVector::get(ShuffleElts);
11892 Value *ShuffleCall = Builder.CreateShuffleVector(Call, Call, ShuffleMask);
11893 return ShuffleCall;
11895 Ops[1] = ConstantInt::getSigned(Int32Ty, Index);
11896 return Builder.CreateCall(F, Ops);
11900 case PPC::BI__builtin_vsx_xxpermdi: {
11901 ConstantInt *ArgCI = dyn_cast<ConstantInt>(Ops[2]);
11902 assert(ArgCI &&
"Third arg must be constant integer!");
11904 unsigned Index = ArgCI->getZExtValue();
11905 Ops[0] = Builder.CreateBitCast(Ops[0], llvm::VectorType::get(Int64Ty, 2));
11906 Ops[1] = Builder.CreateBitCast(Ops[1], llvm::VectorType::get(Int64Ty, 2));
11911 unsigned ElemIdx0 = (Index & 2) >> 1;
11912 unsigned ElemIdx1 = 2 + (Index & 1);
11914 Constant *ShuffleElts[2] = {ConstantInt::get(Int32Ty, ElemIdx0),
11915 ConstantInt::get(Int32Ty, ElemIdx1)};
11916 Constant *ShuffleMask = llvm::ConstantVector::get(ShuffleElts);
11918 Value *ShuffleCall =
11919 Builder.CreateShuffleVector(Ops[0], Ops[1], ShuffleMask);
11921 auto RetTy = ConvertType(BIRetType);
11922 return Builder.CreateBitCast(ShuffleCall, RetTy);
11925 case PPC::BI__builtin_vsx_xxsldwi: {
11926 ConstantInt *ArgCI = dyn_cast<ConstantInt>(Ops[2]);
11927 assert(ArgCI &&
"Third argument must be a compile time constant");
11928 unsigned Index = ArgCI->getZExtValue() & 0x3;
11929 Ops[0] = Builder.CreateBitCast(Ops[0], llvm::VectorType::get(Int32Ty, 4));
11930 Ops[1] = Builder.CreateBitCast(Ops[1], llvm::VectorType::get(Int32Ty, 4));
11937 if (getTarget().isLittleEndian()) {
11941 ElemIdx0 = (8 - Index) % 8;
11942 ElemIdx1 = (9 - Index) % 8;
11943 ElemIdx2 = (10 - Index) % 8;
11944 ElemIdx3 = (11 - Index) % 8;
11948 ElemIdx1 = Index + 1;
11949 ElemIdx2 = Index + 2;
11950 ElemIdx3 = Index + 3;
11953 Constant *ShuffleElts[4] = {ConstantInt::get(Int32Ty, ElemIdx0),
11954 ConstantInt::get(Int32Ty, ElemIdx1),
11955 ConstantInt::get(Int32Ty, ElemIdx2),
11956 ConstantInt::get(Int32Ty, ElemIdx3)};
11958 Constant *ShuffleMask = llvm::ConstantVector::get(ShuffleElts);
11959 Value *ShuffleCall =
11960 Builder.CreateShuffleVector(Ops[0], Ops[1], ShuffleMask);
11962 auto RetTy = ConvertType(BIRetType);
11963 return Builder.CreateBitCast(ShuffleCall, RetTy);
11966 case PPC::BI__builtin_pack_vector_int128: {
11967 bool isLittleEndian = getTarget().isLittleEndian();
11968 Value *UndefValue =
11969 llvm::UndefValue::get(llvm::VectorType::get(Ops[0]->getType(), 2));
11970 Value *Res = Builder.CreateInsertElement(
11971 UndefValue, Ops[0], (uint64_t)(isLittleEndian ? 1 : 0));
11972 Res = Builder.CreateInsertElement(Res, Ops[1],
11973 (uint64_t)(isLittleEndian ? 0 : 1));
11974 return Builder.CreateBitCast(Res, ConvertType(E->
getType()));
11977 case PPC::BI__builtin_unpack_vector_int128: {
11978 ConstantInt *Index = cast<ConstantInt>(Ops[1]);
11979 Value *Unpacked = Builder.CreateBitCast(
11980 Ops[0], llvm::VectorType::get(ConvertType(E->
getType()), 2));
11982 if (getTarget().isLittleEndian())
11983 Index = ConstantInt::get(Index->getType(), 1 - Index->getZExtValue());
11985 return Builder.CreateExtractElement(Unpacked, Index);
11992 switch (BuiltinID) {
11993 case AMDGPU::BI__builtin_amdgcn_div_scale:
11994 case AMDGPU::BI__builtin_amdgcn_div_scalef: {
11998 Address FlagOutPtr = EmitPointerWithAlignment(E->
getArg(3));
12004 llvm::Value *Callee = CGM.getIntrinsic(Intrinsic::amdgcn_div_scale,
12007 llvm::Value *Tmp = Builder.CreateCall(Callee, {
X, Y, Z});
12009 llvm::Value *Result = Builder.CreateExtractValue(Tmp, 0);
12010 llvm::Value *Flag = Builder.CreateExtractValue(Tmp, 1);
12013 = FlagOutPtr.
getPointer()->getType()->getPointerElementType();
12015 llvm::Value *FlagExt = Builder.CreateZExt(Flag, RealFlagType);
12016 Builder.CreateStore(FlagExt, FlagOutPtr);
12019 case AMDGPU::BI__builtin_amdgcn_div_fmas:
12020 case AMDGPU::BI__builtin_amdgcn_div_fmasf: {
12026 llvm::Value *F = CGM.getIntrinsic(Intrinsic::amdgcn_div_fmas,
12028 llvm::Value *Src3ToBool = Builder.CreateIsNotNull(Src3);
12029 return Builder.CreateCall(F, {Src0, Src1, Src2, Src3ToBool});
12032 case AMDGPU::BI__builtin_amdgcn_ds_swizzle:
12034 case AMDGPU::BI__builtin_amdgcn_mov_dpp:
12035 case AMDGPU::BI__builtin_amdgcn_update_dpp: {
12037 for (
unsigned I = 0; I != E->
getNumArgs(); ++I)
12038 Args.push_back(EmitScalarExpr(E->
getArg(I)));
12039 assert(Args.size() == 5 || Args.size() == 6);
12040 if (Args.size() == 5)
12041 Args.insert(Args.begin(), llvm::UndefValue::get(Args[0]->getType()));
12043 CGM.getIntrinsic(Intrinsic::amdgcn_update_dpp, Args[0]->getType());
12044 return Builder.CreateCall(F, Args);
12046 case AMDGPU::BI__builtin_amdgcn_div_fixup:
12047 case AMDGPU::BI__builtin_amdgcn_div_fixupf:
12048 case AMDGPU::BI__builtin_amdgcn_div_fixuph:
12050 case AMDGPU::BI__builtin_amdgcn_trig_preop:
12051 case AMDGPU::BI__builtin_amdgcn_trig_preopf:
12053 case AMDGPU::BI__builtin_amdgcn_rcp:
12054 case AMDGPU::BI__builtin_amdgcn_rcpf:
12055 case AMDGPU::BI__builtin_amdgcn_rcph:
12057 case AMDGPU::BI__builtin_amdgcn_rsq:
12058 case AMDGPU::BI__builtin_amdgcn_rsqf:
12059 case AMDGPU::BI__builtin_amdgcn_rsqh:
12061 case AMDGPU::BI__builtin_amdgcn_rsq_clamp:
12062 case AMDGPU::BI__builtin_amdgcn_rsq_clampf:
12064 case AMDGPU::BI__builtin_amdgcn_sinf:
12065 case AMDGPU::BI__builtin_amdgcn_sinh:
12067 case AMDGPU::BI__builtin_amdgcn_cosf:
12068 case AMDGPU::BI__builtin_amdgcn_cosh:
12070 case AMDGPU::BI__builtin_amdgcn_log_clampf:
12072 case AMDGPU::BI__builtin_amdgcn_ldexp:
12073 case AMDGPU::BI__builtin_amdgcn_ldexpf:
12074 case AMDGPU::BI__builtin_amdgcn_ldexph:
12076 case AMDGPU::BI__builtin_amdgcn_frexp_mant:
12077 case AMDGPU::BI__builtin_amdgcn_frexp_mantf:
12078 case AMDGPU::BI__builtin_amdgcn_frexp_manth:
12080 case AMDGPU::BI__builtin_amdgcn_frexp_exp:
12081 case AMDGPU::BI__builtin_amdgcn_frexp_expf: {
12083 Value *F = CGM.getIntrinsic(Intrinsic::amdgcn_frexp_exp,
12084 { Builder.getInt32Ty(), Src0->getType() });
12085 return Builder.CreateCall(F, Src0);
12087 case AMDGPU::BI__builtin_amdgcn_frexp_exph: {
12089 Value *F = CGM.getIntrinsic(Intrinsic::amdgcn_frexp_exp,
12090 { Builder.getInt16Ty(), Src0->getType() });
12091 return Builder.CreateCall(F, Src0);
12093 case AMDGPU::BI__builtin_amdgcn_fract:
12094 case AMDGPU::BI__builtin_amdgcn_fractf:
12095 case AMDGPU::BI__builtin_amdgcn_fracth:
12097 case AMDGPU::BI__builtin_amdgcn_lerp:
12099 case AMDGPU::BI__builtin_amdgcn_uicmp:
12100 case AMDGPU::BI__builtin_amdgcn_uicmpl:
12101 case AMDGPU::BI__builtin_amdgcn_sicmp:
12102 case AMDGPU::BI__builtin_amdgcn_sicmpl:
12104 case AMDGPU::BI__builtin_amdgcn_fcmp:
12105 case AMDGPU::BI__builtin_amdgcn_fcmpf:
12107 case AMDGPU::BI__builtin_amdgcn_class:
12108 case AMDGPU::BI__builtin_amdgcn_classf:
12109 case AMDGPU::BI__builtin_amdgcn_classh:
12111 case AMDGPU::BI__builtin_amdgcn_fmed3f:
12112 case AMDGPU::BI__builtin_amdgcn_fmed3h:
12114 case AMDGPU::BI__builtin_amdgcn_read_exec: {
12115 CallInst *CI = cast<CallInst>(
12117 CI->setConvergent();
12120 case AMDGPU::BI__builtin_amdgcn_read_exec_lo:
12121 case AMDGPU::BI__builtin_amdgcn_read_exec_hi: {
12122 StringRef RegName = BuiltinID == AMDGPU::BI__builtin_amdgcn_read_exec_lo ?
12123 "exec_lo" :
"exec_hi";
12124 CallInst *CI = cast<CallInst>(
12126 CI->setConvergent();
12130 case AMDGPU::BI__builtin_amdgcn_workitem_id_x:
12132 case AMDGPU::BI__builtin_amdgcn_workitem_id_y:
12134 case AMDGPU::BI__builtin_amdgcn_workitem_id_z:
12138 case AMDGPU::BI__builtin_r600_recipsqrt_ieee:
12139 case AMDGPU::BI__builtin_r600_recipsqrt_ieeef:
12141 case AMDGPU::BI__builtin_r600_read_tidig_x:
12143 case AMDGPU::BI__builtin_r600_read_tidig_y:
12145 case AMDGPU::BI__builtin_r600_read_tidig_z:
12156 unsigned IntrinsicID,
12160 for (
unsigned I = 0; I < NumArgs; ++I)
12167 return CGF.
Builder.CreateExtractValue(Call, 0);
12172 switch (BuiltinID) {
12173 case SystemZ::BI__builtin_tbegin: {
12175 Value *Control = llvm::ConstantInt::get(Int32Ty, 0xff0c);
12176 Value *F = CGM.getIntrinsic(Intrinsic::s390_tbegin);
12177 return Builder.CreateCall(F, {TDB, Control});
12179 case SystemZ::BI__builtin_tbegin_nofloat: {
12181 Value *Control = llvm::ConstantInt::get(Int32Ty, 0xff0c);
12182 Value *F = CGM.getIntrinsic(Intrinsic::s390_tbegin_nofloat);
12183 return Builder.CreateCall(F, {TDB, Control});
12185 case SystemZ::BI__builtin_tbeginc: {
12186 Value *TDB = llvm::ConstantPointerNull::get(Int8PtrTy);
12187 Value *Control = llvm::ConstantInt::get(Int32Ty, 0xff08);
12188 Value *F = CGM.getIntrinsic(Intrinsic::s390_tbeginc);
12189 return Builder.CreateCall(F, {TDB, Control});
12191 case SystemZ::BI__builtin_tabort: {
12193 Value *F = CGM.getIntrinsic(Intrinsic::s390_tabort);
12194 return Builder.CreateCall(F, Builder.CreateSExt(Data, Int64Ty,
"tabort"));
12196 case SystemZ::BI__builtin_non_tx_store: {
12199 Value *F = CGM.getIntrinsic(Intrinsic::s390_ntstg);
12200 return Builder.CreateCall(F, {Data, Address});
12208 case SystemZ::BI__builtin_s390_vpopctb:
12209 case SystemZ::BI__builtin_s390_vpopcth:
12210 case SystemZ::BI__builtin_s390_vpopctf:
12211 case SystemZ::BI__builtin_s390_vpopctg: {
12214 Function *F = CGM.getIntrinsic(Intrinsic::ctpop, ResultType);
12215 return Builder.CreateCall(F, X);
12218 case SystemZ::BI__builtin_s390_vclzb:
12219 case SystemZ::BI__builtin_s390_vclzh:
12220 case SystemZ::BI__builtin_s390_vclzf:
12221 case SystemZ::BI__builtin_s390_vclzg: {
12224 Value *Undef = ConstantInt::get(Builder.getInt1Ty(),
false);
12225 Function *F = CGM.getIntrinsic(Intrinsic::ctlz, ResultType);
12226 return Builder.CreateCall(F, {
X, Undef});
12229 case SystemZ::BI__builtin_s390_vctzb:
12230 case SystemZ::BI__builtin_s390_vctzh:
12231 case SystemZ::BI__builtin_s390_vctzf:
12232 case SystemZ::BI__builtin_s390_vctzg: {
12235 Value *Undef = ConstantInt::get(Builder.getInt1Ty(),
false);
12236 Function *F = CGM.getIntrinsic(Intrinsic::cttz, ResultType);
12237 return Builder.CreateCall(F, {
X, Undef});
12240 case SystemZ::BI__builtin_s390_vfsqsb:
12241 case SystemZ::BI__builtin_s390_vfsqdb: {
12245 return Builder.CreateCall(F, X);
12247 case SystemZ::BI__builtin_s390_vfmasb:
12248 case SystemZ::BI__builtin_s390_vfmadb: {
12254 return Builder.CreateCall(F, {
X, Y, Z});
12256 case SystemZ::BI__builtin_s390_vfmssb:
12257 case SystemZ::BI__builtin_s390_vfmsdb: {
12262 Value *Zero = llvm::ConstantFP::getZeroValueForNegation(ResultType);
12264 return Builder.CreateCall(F, {
X, Y, Builder.CreateFSub(Zero, Z,
"sub")});
12266 case SystemZ::BI__builtin_s390_vfnmasb:
12267 case SystemZ::BI__builtin_s390_vfnmadb: {
12272 Value *Zero = llvm::ConstantFP::getZeroValueForNegation(ResultType);
12274 return Builder.CreateFSub(Zero, Builder.CreateCall(F, {X, Y, Z}),
"sub");
12276 case SystemZ::BI__builtin_s390_vfnmssb:
12277 case SystemZ::BI__builtin_s390_vfnmsdb: {
12282 Value *Zero = llvm::ConstantFP::getZeroValueForNegation(ResultType);
12284 Value *NegZ = Builder.CreateFSub(Zero, Z,
"sub");
12285 return Builder.CreateFSub(Zero, Builder.CreateCall(F, {X, Y, NegZ}));
12287 case SystemZ::BI__builtin_s390_vflpsb:
12288 case SystemZ::BI__builtin_s390_vflpdb: {
12292 return Builder.CreateCall(F, X);
12294 case SystemZ::BI__builtin_s390_vflnsb:
12295 case SystemZ::BI__builtin_s390_vflndb: {
12298 Value *Zero = llvm::ConstantFP::getZeroValueForNegation(ResultType);
12300 return Builder.CreateFSub(Zero, Builder.CreateCall(F, X),
"sub");
12302 case SystemZ::BI__builtin_s390_vfisb:
12303 case SystemZ::BI__builtin_s390_vfidb: {
12307 llvm::APSInt M4, M5;
12310 assert(IsConstM4 && IsConstM5 &&
"Constant arg isn't actually constant?");
12311 (void)IsConstM4; (void)IsConstM5;
12315 switch (M4.getZExtValue()) {
12318 switch (M5.getZExtValue()) {
12324 switch (M5.getZExtValue()) {
12334 if (ID != Intrinsic::not_intrinsic) {
12335 Function *F = CGM.getIntrinsic(ID, ResultType);
12336 return Builder.CreateCall(F, X);
12338 switch (BuiltinID) {
12339 case SystemZ::BI__builtin_s390_vfisb: ID = Intrinsic::s390_vfisb;
break;
12340 case SystemZ::BI__builtin_s390_vfidb: ID = Intrinsic::s390_vfidb;
break;
12341 default: llvm_unreachable(
"Unknown BuiltinID");
12343 Function *F = CGM.getIntrinsic(ID);
12344 Value *M4Value = llvm::ConstantInt::get(getLLVMContext(), M4);
12345 Value *M5Value = llvm::ConstantInt::get(getLLVMContext(), M5);
12346 return Builder.CreateCall(F, {
X, M4Value, M5Value});
12348 case SystemZ::BI__builtin_s390_vfmaxsb:
12349 case SystemZ::BI__builtin_s390_vfmaxdb: {
12356 assert(IsConstM4 &&
"Constant arg isn't actually constant?");
12361 switch (M4.getZExtValue()) {
12363 case 4: ID = Intrinsic::maxnum;
break;
12365 if (ID != Intrinsic::not_intrinsic) {
12366 Function *F = CGM.getIntrinsic(ID, ResultType);
12367 return Builder.CreateCall(F, {
X, Y});
12369 switch (BuiltinID) {
12370 case SystemZ::BI__builtin_s390_vfmaxsb: ID = Intrinsic::s390_vfmaxsb;
break;
12371 case SystemZ::BI__builtin_s390_vfmaxdb: ID = Intrinsic::s390_vfmaxdb;
break;
12372 default: llvm_unreachable(
"Unknown BuiltinID");
12374 Function *F = CGM.getIntrinsic(ID);
12375 Value *M4Value = llvm::ConstantInt::get(getLLVMContext(), M4);
12376 return Builder.CreateCall(F, {
X, Y, M4Value});
12378 case SystemZ::BI__builtin_s390_vfminsb:
12379 case SystemZ::BI__builtin_s390_vfmindb: {
12386 assert(IsConstM4 &&
"Constant arg isn't actually constant?");
12391 switch (M4.getZExtValue()) {
12393 case 4: ID = Intrinsic::minnum;
break;
12395 if (ID != Intrinsic::not_intrinsic) {
12396 Function *F = CGM.getIntrinsic(ID, ResultType);
12397 return Builder.CreateCall(F, {
X, Y});
12399 switch (BuiltinID) {
12400 case SystemZ::BI__builtin_s390_vfminsb: ID = Intrinsic::s390_vfminsb;
break;
12401 case SystemZ::BI__builtin_s390_vfmindb: ID = Intrinsic::s390_vfmindb;
break;
12402 default: llvm_unreachable(
"Unknown BuiltinID");
12404 Function *F = CGM.getIntrinsic(ID);
12405 Value *M4Value = llvm::ConstantInt::get(getLLVMContext(), M4);
12406 return Builder.CreateCall(F, {
X, Y, M4Value});
12411 #define INTRINSIC_WITH_CC(NAME) \ 12412 case SystemZ::BI__builtin_##NAME: \ 12413 return EmitSystemZIntrinsicWithCC(*this, Intrinsic::NAME, E) 12484 #undef INTRINSIC_WITH_CC 12493 auto MakeLdg = [&](
unsigned IntrinsicID) {
12497 return Builder.CreateCall(
12498 CGM.getIntrinsic(IntrinsicID, {Ptr->getType()->getPointerElementType(),
12500 {Ptr, ConstantInt::get(Builder.getInt32Ty(), Align.getQuantity())});
12502 auto MakeScopedAtomic = [&](
unsigned IntrinsicID) {
12504 return Builder.CreateCall(
12505 CGM.getIntrinsic(IntrinsicID, {Ptr->getType()->getPointerElementType(),
12507 {Ptr, EmitScalarExpr(E->
getArg(1))});
12509 switch (BuiltinID) {
12510 case NVPTX::BI__nvvm_atom_add_gen_i:
12511 case NVPTX::BI__nvvm_atom_add_gen_l:
12512 case NVPTX::BI__nvvm_atom_add_gen_ll:
12515 case NVPTX::BI__nvvm_atom_sub_gen_i:
12516 case NVPTX::BI__nvvm_atom_sub_gen_l:
12517 case NVPTX::BI__nvvm_atom_sub_gen_ll:
12520 case NVPTX::BI__nvvm_atom_and_gen_i:
12521 case NVPTX::BI__nvvm_atom_and_gen_l:
12522 case NVPTX::BI__nvvm_atom_and_gen_ll:
12525 case NVPTX::BI__nvvm_atom_or_gen_i:
12526 case NVPTX::BI__nvvm_atom_or_gen_l:
12527 case NVPTX::BI__nvvm_atom_or_gen_ll:
12530 case NVPTX::BI__nvvm_atom_xor_gen_i:
12531 case NVPTX::BI__nvvm_atom_xor_gen_l:
12532 case NVPTX::BI__nvvm_atom_xor_gen_ll:
12535 case NVPTX::BI__nvvm_atom_xchg_gen_i:
12536 case NVPTX::BI__nvvm_atom_xchg_gen_l:
12537 case NVPTX::BI__nvvm_atom_xchg_gen_ll:
12540 case NVPTX::BI__nvvm_atom_max_gen_i:
12541 case NVPTX::BI__nvvm_atom_max_gen_l:
12542 case NVPTX::BI__nvvm_atom_max_gen_ll:
12545 case NVPTX::BI__nvvm_atom_max_gen_ui:
12546 case NVPTX::BI__nvvm_atom_max_gen_ul:
12547 case NVPTX::BI__nvvm_atom_max_gen_ull:
12550 case NVPTX::BI__nvvm_atom_min_gen_i:
12551 case NVPTX::BI__nvvm_atom_min_gen_l:
12552 case NVPTX::BI__nvvm_atom_min_gen_ll:
12555 case NVPTX::BI__nvvm_atom_min_gen_ui:
12556 case NVPTX::BI__nvvm_atom_min_gen_ul:
12557 case NVPTX::BI__nvvm_atom_min_gen_ull:
12560 case NVPTX::BI__nvvm_atom_cas_gen_i:
12561 case NVPTX::BI__nvvm_atom_cas_gen_l:
12562 case NVPTX::BI__nvvm_atom_cas_gen_ll:
12567 case NVPTX::BI__nvvm_atom_add_gen_f: {
12573 CGM.getIntrinsic(Intrinsic::nvvm_atomic_load_add_f32, Ptr->getType());
12574 return Builder.CreateCall(FnALAF32, {Ptr, Val});
12577 case NVPTX::BI__nvvm_atom_add_gen_d: {
12583 CGM.getIntrinsic(Intrinsic::nvvm_atomic_load_add_f64, Ptr->getType());
12584 return Builder.CreateCall(FnALAF64, {Ptr, Val});
12587 case NVPTX::BI__nvvm_atom_inc_gen_ui: {
12591 CGM.getIntrinsic(Intrinsic::nvvm_atomic_load_inc_32, Ptr->getType());
12592 return Builder.CreateCall(FnALI32, {Ptr, Val});
12595 case NVPTX::BI__nvvm_atom_dec_gen_ui: {
12599 CGM.getIntrinsic(Intrinsic::nvvm_atomic_load_dec_32, Ptr->getType());
12600 return Builder.CreateCall(FnALD32, {Ptr, Val});
12603 case NVPTX::BI__nvvm_ldg_c:
12604 case NVPTX::BI__nvvm_ldg_c2:
12605 case NVPTX::BI__nvvm_ldg_c4:
12606 case NVPTX::BI__nvvm_ldg_s:
12607 case NVPTX::BI__nvvm_ldg_s2:
12608 case NVPTX::BI__nvvm_ldg_s4:
12609 case NVPTX::BI__nvvm_ldg_i:
12610 case NVPTX::BI__nvvm_ldg_i2:
12611 case NVPTX::BI__nvvm_ldg_i4:
12612 case NVPTX::BI__nvvm_ldg_l:
12613 case NVPTX::BI__nvvm_ldg_ll:
12614 case NVPTX::BI__nvvm_ldg_ll2:
12615 case NVPTX::BI__nvvm_ldg_uc:
12616 case NVPTX::BI__nvvm_ldg_uc2:
12617 case NVPTX::BI__nvvm_ldg_uc4:
12618 case NVPTX::BI__nvvm_ldg_us:
12619 case NVPTX::BI__nvvm_ldg_us2:
12620 case NVPTX::BI__nvvm_ldg_us4:
12621 case NVPTX::BI__nvvm_ldg_ui:
12622 case NVPTX::BI__nvvm_ldg_ui2:
12623 case NVPTX::BI__nvvm_ldg_ui4:
12624 case NVPTX::BI__nvvm_ldg_ul:
12625 case NVPTX::BI__nvvm_ldg_ull:
12626 case NVPTX::BI__nvvm_ldg_ull2:
12630 return MakeLdg(Intrinsic::nvvm_ldg_global_i);
12631 case NVPTX::BI__nvvm_ldg_f:
12632 case NVPTX::BI__nvvm_ldg_f2:
12633 case NVPTX::BI__nvvm_ldg_f4:
12634 case NVPTX::BI__nvvm_ldg_d:
12635 case NVPTX::BI__nvvm_ldg_d2:
12636 return MakeLdg(Intrinsic::nvvm_ldg_global_f);
12638 case NVPTX::BI__nvvm_atom_cta_add_gen_i:
12639 case NVPTX::BI__nvvm_atom_cta_add_gen_l:
12640 case NVPTX::BI__nvvm_atom_cta_add_gen_ll:
12641 return MakeScopedAtomic(Intrinsic::nvvm_atomic_add_gen_i_cta);
12642 case NVPTX::BI__nvvm_atom_sys_add_gen_i:
12643 case NVPTX::BI__nvvm_atom_sys_add_gen_l:
12644 case NVPTX::BI__nvvm_atom_sys_add_gen_ll:
12645 return MakeScopedAtomic(Intrinsic::nvvm_atomic_add_gen_i_sys);
12646 case NVPTX::BI__nvvm_atom_cta_add_gen_f:
12647 case NVPTX::BI__nvvm_atom_cta_add_gen_d:
12648 return MakeScopedAtomic(Intrinsic::nvvm_atomic_add_gen_f_cta);
12649 case NVPTX::BI__nvvm_atom_sys_add_gen_f:
12650 case NVPTX::BI__nvvm_atom_sys_add_gen_d:
12651 return MakeScopedAtomic(Intrinsic::nvvm_atomic_add_gen_f_sys);
12652 case NVPTX::BI__nvvm_atom_cta_xchg_gen_i:
12653 case NVPTX::BI__nvvm_atom_cta_xchg_gen_l:
12654 case NVPTX::BI__nvvm_atom_cta_xchg_gen_ll:
12655 return MakeScopedAtomic(Intrinsic::nvvm_atomic_exch_gen_i_cta);
12656 case NVPTX::BI__nvvm_atom_sys_xchg_gen_i:
12657 case NVPTX::BI__nvvm_atom_sys_xchg_gen_l:
12658 case NVPTX::BI__nvvm_atom_sys_xchg_gen_ll:
12659 return MakeScopedAtomic(Intrinsic::nvvm_atomic_exch_gen_i_sys);
12660 case NVPTX::BI__nvvm_atom_cta_max_gen_i:
12661 case NVPTX::BI__nvvm_atom_cta_max_gen_ui:
12662 case NVPTX::BI__nvvm_atom_cta_max_gen_l:
12663 case NVPTX::BI__nvvm_atom_cta_max_gen_ul:
12664 case NVPTX::BI__nvvm_atom_cta_max_gen_ll:
12665 case NVPTX::BI__nvvm_atom_cta_max_gen_ull:
12666 return MakeScopedAtomic(Intrinsic::nvvm_atomic_max_gen_i_cta);
12667 case NVPTX::BI__nvvm_atom_sys_max_gen_i:
12668 case NVPTX::BI__nvvm_atom_sys_max_gen_ui:
12669 case NVPTX::BI__nvvm_atom_sys_max_gen_l:
12670 case NVPTX::BI__nvvm_atom_sys_max_gen_ul:
12671 case NVPTX::BI__nvvm_atom_sys_max_gen_ll:
12672 case NVPTX::BI__nvvm_atom_sys_max_gen_ull:
12673 return MakeScopedAtomic(Intrinsic::nvvm_atomic_max_gen_i_sys);
12674 case NVPTX::BI__nvvm_atom_cta_min_gen_i:
12675 case NVPTX::BI__nvvm_atom_cta_min_gen_ui:
12676 case NVPTX::BI__nvvm_atom_cta_min_gen_l:
12677 case NVPTX::BI__nvvm_atom_cta_min_gen_ul:
12678 case NVPTX::BI__nvvm_atom_cta_min_gen_ll:
12679 case NVPTX::BI__nvvm_atom_cta_min_gen_ull:
12680 return MakeScopedAtomic(Intrinsic::nvvm_atomic_min_gen_i_cta);
12681 case NVPTX::BI__nvvm_atom_sys_min_gen_i:
12682 case NVPTX::BI__nvvm_atom_sys_min_gen_ui:
12683 case NVPTX::BI__nvvm_atom_sys_min_gen_l:
12684 case NVPTX::BI__nvvm_atom_sys_min_gen_ul:
12685 case NVPTX::BI__nvvm_atom_sys_min_gen_ll:
12686 case NVPTX::BI__nvvm_atom_sys_min_gen_ull:
12687 return MakeScopedAtomic(Intrinsic::nvvm_atomic_min_gen_i_sys);
12688 case NVPTX::BI__nvvm_atom_cta_inc_gen_ui:
12689 return MakeScopedAtomic(Intrinsic::nvvm_atomic_inc_gen_i_cta);
12690 case NVPTX::BI__nvvm_atom_cta_dec_gen_ui:
12691 return MakeScopedAtomic(Intrinsic::nvvm_atomic_dec_gen_i_cta);
12692 case NVPTX::BI__nvvm_atom_sys_inc_gen_ui:
12693 return MakeScopedAtomic(Intrinsic::nvvm_atomic_inc_gen_i_sys);
12694 case NVPTX::BI__nvvm_atom_sys_dec_gen_ui:
12695 return MakeScopedAtomic(Intrinsic::nvvm_atomic_dec_gen_i_sys);
12696 case NVPTX::BI__nvvm_atom_cta_and_gen_i:
12697 case NVPTX::BI__nvvm_atom_cta_and_gen_l:
12698 case NVPTX::BI__nvvm_atom_cta_and_gen_ll:
12699 return MakeScopedAtomic(Intrinsic::nvvm_atomic_and_gen_i_cta);
12700 case NVPTX::BI__nvvm_atom_sys_and_gen_i:
12701 case NVPTX::BI__nvvm_atom_sys_and_gen_l:
12702 case NVPTX::BI__nvvm_atom_sys_and_gen_ll:
12703 return MakeScopedAtomic(Intrinsic::nvvm_atomic_and_gen_i_sys);
12704 case NVPTX::BI__nvvm_atom_cta_or_gen_i:
12705 case NVPTX::BI__nvvm_atom_cta_or_gen_l:
12706 case NVPTX::BI__nvvm_atom_cta_or_gen_ll:
12707 return MakeScopedAtomic(Intrinsic::nvvm_atomic_or_gen_i_cta);
12708 case NVPTX::BI__nvvm_atom_sys_or_gen_i:
12709 case NVPTX::BI__nvvm_atom_sys_or_gen_l:
12710 case NVPTX::BI__nvvm_atom_sys_or_gen_ll:
12711 return MakeScopedAtomic(Intrinsic::nvvm_atomic_or_gen_i_sys);
12712 case NVPTX::BI__nvvm_atom_cta_xor_gen_i:
12713 case NVPTX::BI__nvvm_atom_cta_xor_gen_l:
12714 case NVPTX::BI__nvvm_atom_cta_xor_gen_ll:
12715 return MakeScopedAtomic(Intrinsic::nvvm_atomic_xor_gen_i_cta);
12716 case NVPTX::BI__nvvm_atom_sys_xor_gen_i:
12717 case NVPTX::BI__nvvm_atom_sys_xor_gen_l:
12718 case NVPTX::BI__nvvm_atom_sys_xor_gen_ll:
12719 return MakeScopedAtomic(Intrinsic::nvvm_atomic_xor_gen_i_sys);
12720 case NVPTX::BI__nvvm_atom_cta_cas_gen_i:
12721 case NVPTX::BI__nvvm_atom_cta_cas_gen_l:
12722 case NVPTX::BI__nvvm_atom_cta_cas_gen_ll: {
12724 return Builder.CreateCall(
12726 Intrinsic::nvvm_atomic_cas_gen_i_cta,
12727 {Ptr->getType()->getPointerElementType(), Ptr->getType()}),
12728 {Ptr, EmitScalarExpr(E->
getArg(1)), EmitScalarExpr(E->
getArg(2))});
12730 case NVPTX::BI__nvvm_atom_sys_cas_gen_i:
12731 case NVPTX::BI__nvvm_atom_sys_cas_gen_l:
12732 case NVPTX::BI__nvvm_atom_sys_cas_gen_ll: {
12734 return Builder.CreateCall(
12736 Intrinsic::nvvm_atomic_cas_gen_i_sys,
12737 {Ptr->getType()->getPointerElementType(), Ptr->getType()}),
12738 {Ptr, EmitScalarExpr(E->
getArg(1)), EmitScalarExpr(E->
getArg(2))});
12740 case NVPTX::BI__nvvm_match_all_sync_i32p:
12741 case NVPTX::BI__nvvm_match_all_sync_i64p: {
12744 Address PredOutPtr = EmitPointerWithAlignment(E->
getArg(2));
12745 Value *ResultPair = Builder.CreateCall(
12746 CGM.getIntrinsic(BuiltinID == NVPTX::BI__nvvm_match_all_sync_i32p
12747 ? Intrinsic::nvvm_match_all_sync_i32p
12748 : Intrinsic::nvvm_match_all_sync_i64p),
12750 Value *Pred = Builder.CreateZExt(Builder.CreateExtractValue(ResultPair, 1),
12751 PredOutPtr.getElementType());
12752 Builder.CreateStore(Pred, PredOutPtr);
12753 return Builder.CreateExtractValue(ResultPair, 0);
12755 case NVPTX::BI__hmma_m16n16k16_ld_a:
12756 case NVPTX::BI__hmma_m16n16k16_ld_b:
12757 case NVPTX::BI__hmma_m16n16k16_ld_c_f16:
12758 case NVPTX::BI__hmma_m16n16k16_ld_c_f32:
12759 case NVPTX::BI__hmma_m32n8k16_ld_a:
12760 case NVPTX::BI__hmma_m32n8k16_ld_b:
12761 case NVPTX::BI__hmma_m32n8k16_ld_c_f16:
12762 case NVPTX::BI__hmma_m32n8k16_ld_c_f32:
12763 case NVPTX::BI__hmma_m8n32k16_ld_a:
12764 case NVPTX::BI__hmma_m8n32k16_ld_b:
12765 case NVPTX::BI__hmma_m8n32k16_ld_c_f16:
12766 case NVPTX::BI__hmma_m8n32k16_ld_c_f32: {
12770 llvm::APSInt isColMajorArg;
12773 bool isColMajor = isColMajorArg.getSExtValue();
12775 unsigned NumResults;
12776 switch (BuiltinID) {
12777 case NVPTX::BI__hmma_m16n16k16_ld_a:
12778 IID = isColMajor ? Intrinsic::nvvm_wmma_m16n16k16_load_a_f16_col_stride
12779 : Intrinsic::nvvm_wmma_m16n16k16_load_a_f16_row_stride;
12782 case NVPTX::BI__hmma_m16n16k16_ld_b:
12783 IID = isColMajor ? Intrinsic::nvvm_wmma_m16n16k16_load_b_f16_col_stride
12784 : Intrinsic::nvvm_wmma_m16n16k16_load_b_f16_row_stride;
12787 case NVPTX::BI__hmma_m16n16k16_ld_c_f16:
12788 IID = isColMajor ? Intrinsic::nvvm_wmma_m16n16k16_load_c_f16_col_stride
12789 : Intrinsic::nvvm_wmma_m16n16k16_load_c_f16_row_stride;
12792 case NVPTX::BI__hmma_m16n16k16_ld_c_f32:
12793 IID = isColMajor ? Intrinsic::nvvm_wmma_m16n16k16_load_c_f32_col_stride
12794 : Intrinsic::nvvm_wmma_m16n16k16_load_c_f32_row_stride;
12797 case NVPTX::BI__hmma_m32n8k16_ld_a:
12798 IID = isColMajor ? Intrinsic::nvvm_wmma_m32n8k16_load_a_f16_col_stride
12799 : Intrinsic::nvvm_wmma_m32n8k16_load_a_f16_row_stride;
12802 case NVPTX::BI__hmma_m32n8k16_ld_b:
12803 IID = isColMajor ? Intrinsic::nvvm_wmma_m32n8k16_load_b_f16_col_stride
12804 : Intrinsic::nvvm_wmma_m32n8k16_load_b_f16_row_stride;
12807 case NVPTX::BI__hmma_m32n8k16_ld_c_f16:
12808 IID = isColMajor ? Intrinsic::nvvm_wmma_m32n8k16_load_c_f16_col_stride
12809 : Intrinsic::nvvm_wmma_m32n8k16_load_c_f16_row_stride;
12812 case NVPTX::BI__hmma_m32n8k16_ld_c_f32:
12813 IID = isColMajor ? Intrinsic::nvvm_wmma_m32n8k16_load_c_f32_col_stride
12814 : Intrinsic::nvvm_wmma_m32n8k16_load_c_f32_row_stride;
12817 case NVPTX::BI__hmma_m8n32k16_ld_a:
12818 IID = isColMajor ? Intrinsic::nvvm_wmma_m8n32k16_load_a_f16_col_stride
12819 : Intrinsic::nvvm_wmma_m8n32k16_load_a_f16_row_stride;
12822 case NVPTX::BI__hmma_m8n32k16_ld_b:
12823 IID = isColMajor ? Intrinsic::nvvm_wmma_m8n32k16_load_b_f16_col_stride
12824 : Intrinsic::nvvm_wmma_m8n32k16_load_b_f16_row_stride;
12827 case NVPTX::BI__hmma_m8n32k16_ld_c_f16:
12828 IID = isColMajor ? Intrinsic::nvvm_wmma_m8n32k16_load_c_f16_col_stride
12829 : Intrinsic::nvvm_wmma_m8n32k16_load_c_f16_row_stride;
12832 case NVPTX::BI__hmma_m8n32k16_ld_c_f32:
12833 IID = isColMajor ? Intrinsic::nvvm_wmma_m8n32k16_load_c_f32_col_stride
12834 : Intrinsic::nvvm_wmma_m8n32k16_load_c_f32_row_stride;
12838 llvm_unreachable(
"Unexpected builtin ID.");
12841 Builder.CreateCall(CGM.getIntrinsic(IID, Src->getType()), {Src, Ldm});
12844 for (
unsigned i = 0; i < NumResults; ++i) {
12845 Builder.CreateAlignedStore(
12846 Builder.CreateBitCast(Builder.CreateExtractValue(Result, i),
12848 Builder.CreateGEP(Dst.
getPointer(), llvm::ConstantInt::get(IntTy, i)),
12854 case NVPTX::BI__hmma_m16n16k16_st_c_f16:
12855 case NVPTX::BI__hmma_m16n16k16_st_c_f32:
12856 case NVPTX::BI__hmma_m32n8k16_st_c_f16:
12857 case NVPTX::BI__hmma_m32n8k16_st_c_f32:
12858 case NVPTX::BI__hmma_m8n32k16_st_c_f16:
12859 case NVPTX::BI__hmma_m8n32k16_st_c_f32: {
12863 llvm::APSInt isColMajorArg;
12866 bool isColMajor = isColMajorArg.getSExtValue();
12868 unsigned NumResults = 8;
12871 switch (BuiltinID) {
12872 case NVPTX::BI__hmma_m16n16k16_st_c_f16:
12873 IID = isColMajor ? Intrinsic::nvvm_wmma_m16n16k16_store_d_f16_col_stride
12874 : Intrinsic::nvvm_wmma_m16n16k16_store_d_f16_row_stride;
12877 case NVPTX::BI__hmma_m16n16k16_st_c_f32:
12878 IID = isColMajor ? Intrinsic::nvvm_wmma_m16n16k16_store_d_f32_col_stride
12879 : Intrinsic::nvvm_wmma_m16n16k16_store_d_f32_row_stride;
12881 case NVPTX::BI__hmma_m32n8k16_st_c_f16:
12882 IID = isColMajor ? Intrinsic::nvvm_wmma_m32n8k16_store_d_f16_col_stride
12883 : Intrinsic::nvvm_wmma_m32n8k16_store_d_f16_row_stride;
12886 case NVPTX::BI__hmma_m32n8k16_st_c_f32:
12887 IID = isColMajor ? Intrinsic::nvvm_wmma_m32n8k16_store_d_f32_col_stride
12888 : Intrinsic::nvvm_wmma_m32n8k16_store_d_f32_row_stride;
12890 case NVPTX::BI__hmma_m8n32k16_st_c_f16:
12891 IID = isColMajor ? Intrinsic::nvvm_wmma_m8n32k16_store_d_f16_col_stride
12892 : Intrinsic::nvvm_wmma_m8n32k16_store_d_f16_row_stride;
12895 case NVPTX::BI__hmma_m8n32k16_st_c_f32:
12896 IID = isColMajor ? Intrinsic::nvvm_wmma_m8n32k16_store_d_f32_col_stride
12897 : Intrinsic::nvvm_wmma_m8n32k16_store_d_f32_row_stride;
12900 llvm_unreachable(
"Unexpected builtin ID.");
12902 Function *Intrinsic = CGM.getIntrinsic(IID, Dst->getType());
12903 llvm::Type *ParamType = Intrinsic->getFunctionType()->getParamType(1);
12905 for (
unsigned i = 0; i < NumResults; ++i) {
12906 Value *V = Builder.CreateAlignedLoad(
12907 Builder.CreateGEP(Src.getPointer(), llvm::ConstantInt::get(IntTy, i)),
12909 Values.push_back(Builder.CreateBitCast(V, ParamType));
12911 Values.push_back(Ldm);
12912 Value *Result = Builder.CreateCall(Intrinsic, Values);
12918 case NVPTX::BI__hmma_m16n16k16_mma_f16f16:
12919 case NVPTX::BI__hmma_m16n16k16_mma_f32f16:
12920 case NVPTX::BI__hmma_m16n16k16_mma_f32f32:
12921 case NVPTX::BI__hmma_m16n16k16_mma_f16f32:
12922 case NVPTX::BI__hmma_m32n8k16_mma_f16f16:
12923 case NVPTX::BI__hmma_m32n8k16_mma_f32f16:
12924 case NVPTX::BI__hmma_m32n8k16_mma_f32f32:
12925 case NVPTX::BI__hmma_m32n8k16_mma_f16f32:
12926 case NVPTX::BI__hmma_m8n32k16_mma_f16f16:
12927 case NVPTX::BI__hmma_m8n32k16_mma_f32f16:
12928 case NVPTX::BI__hmma_m8n32k16_mma_f32f32:
12929 case NVPTX::BI__hmma_m8n32k16_mma_f16f32: {
12934 llvm::APSInt LayoutArg;
12937 int Layout = LayoutArg.getSExtValue();
12938 if (Layout < 0 || Layout > 3)
12940 llvm::APSInt SatfArg;
12943 bool Satf = SatfArg.getSExtValue();
12946 #define MMA_VARIANTS(geom, type) {{ \ 12947 Intrinsic::nvvm_wmma_##geom##_mma_row_row_##type, \ 12948 Intrinsic::nvvm_wmma_##geom##_mma_row_row_##type##_satfinite, \ 12949 Intrinsic::nvvm_wmma_##geom##_mma_row_col_##type, \ 12950 Intrinsic::nvvm_wmma_##geom##_mma_row_col_##type##_satfinite, \ 12951 Intrinsic::nvvm_wmma_##geom##_mma_col_row_##type, \ 12952 Intrinsic::nvvm_wmma_##geom##_mma_col_row_##type##_satfinite, \ 12953 Intrinsic::nvvm_wmma_##geom##_mma_col_col_##type, \ 12954 Intrinsic::nvvm_wmma_##geom##_mma_col_col_##type##_satfinite \ 12958 auto getMMAIntrinsic = [Layout, Satf](std::array<unsigned, 8> Variants) {
12959 unsigned Index = Layout * 2 + Satf;
12961 return Variants[Index];
12966 switch (BuiltinID) {
12967 case NVPTX::BI__hmma_m16n16k16_mma_f16f16:
12968 IID = getMMAIntrinsic(
MMA_VARIANTS(m16n16k16, f16_f16));
12972 case NVPTX::BI__hmma_m16n16k16_mma_f32f16:
12973 IID = getMMAIntrinsic(
MMA_VARIANTS(m16n16k16, f32_f16));
12977 case NVPTX::BI__hmma_m16n16k16_mma_f16f32:
12978 IID = getMMAIntrinsic(
MMA_VARIANTS(m16n16k16, f16_f32));
12982 case NVPTX::BI__hmma_m16n16k16_mma_f32f32:
12983 IID = getMMAIntrinsic(
MMA_VARIANTS(m16n16k16, f32_f32));
12987 case NVPTX::BI__hmma_m32n8k16_mma_f16f16:
12988 IID = getMMAIntrinsic(
MMA_VARIANTS(m32n8k16, f16_f16));
12992 case NVPTX::BI__hmma_m32n8k16_mma_f32f16:
12993 IID = getMMAIntrinsic(
MMA_VARIANTS(m32n8k16, f32_f16));
12997 case NVPTX::BI__hmma_m32n8k16_mma_f16f32:
12998 IID = getMMAIntrinsic(
MMA_VARIANTS(m32n8k16, f16_f32));
13002 case NVPTX::BI__hmma_m32n8k16_mma_f32f32:
13003 IID = getMMAIntrinsic(
MMA_VARIANTS(m32n8k16, f32_f32));
13007 case NVPTX::BI__hmma_m8n32k16_mma_f16f16:
13008 IID = getMMAIntrinsic(
MMA_VARIANTS(m8n32k16, f16_f16));
13012 case NVPTX::BI__hmma_m8n32k16_mma_f32f16:
13013 IID = getMMAIntrinsic(
MMA_VARIANTS(m8n32k16, f32_f16));
13017 case NVPTX::BI__hmma_m8n32k16_mma_f16f32:
13018 IID = getMMAIntrinsic(
MMA_VARIANTS(m8n32k16, f16_f32));
13022 case NVPTX::BI__hmma_m8n32k16_mma_f32f32:
13023 IID = getMMAIntrinsic(
MMA_VARIANTS(m8n32k16, f32_f32));
13028 llvm_unreachable(
"Unexpected builtin ID.");
13030 #undef MMA_VARIANTS 13033 Function *Intrinsic = CGM.getIntrinsic(IID);
13034 llvm::Type *ABType = Intrinsic->getFunctionType()->getParamType(0);
13036 for (
unsigned i = 0; i < 8; ++i) {
13037 Value *V = Builder.CreateAlignedLoad(
13038 Builder.CreateGEP(SrcA.getPointer(),
13039 llvm::ConstantInt::get(IntTy, i)),
13041 Values.push_back(Builder.CreateBitCast(V, ABType));
13044 for (
unsigned i = 0; i < 8; ++i) {
13045 Value *V = Builder.CreateAlignedLoad(
13046 Builder.CreateGEP(SrcB.getPointer(),
13047 llvm::ConstantInt::get(IntTy, i)),
13049 Values.push_back(Builder.CreateBitCast(V, ABType));
13052 llvm::Type *CType = Intrinsic->getFunctionType()->getParamType(16);
13053 for (
unsigned i = 0; i < NumEltsC; ++i) {
13054 Value *V = Builder.CreateAlignedLoad(
13055 Builder.CreateGEP(SrcC.getPointer(),
13056 llvm::ConstantInt::get(IntTy, i)),
13058 Values.push_back(Builder.CreateBitCast(V, CType));
13060 Value *Result = Builder.CreateCall(Intrinsic, Values);
13062 for (
unsigned i = 0; i < NumEltsD; ++i)
13063 Builder.CreateAlignedStore(
13064 Builder.CreateBitCast(Builder.CreateExtractValue(Result, i), DType),
13065 Builder.CreateGEP(Dst.
getPointer(), llvm::ConstantInt::get(IntTy, i)),
13076 switch (BuiltinID) {
13077 case WebAssembly::BI__builtin_wasm_memory_size: {
13080 Value *Callee = CGM.getIntrinsic(Intrinsic::wasm_memory_size, ResultType);
13081 return Builder.CreateCall(Callee, I);
13083 case WebAssembly::BI__builtin_wasm_memory_grow: {
13086 EmitScalarExpr(E->
getArg(0)),
13087 EmitScalarExpr(E->
getArg(1))
13089 Value *Callee = CGM.getIntrinsic(Intrinsic::wasm_memory_grow, ResultType);
13090 return Builder.CreateCall(Callee, Args);
13092 case WebAssembly::BI__builtin_wasm_throw: {
13095 Value *Callee = CGM.getIntrinsic(Intrinsic::wasm_throw);
13096 return Builder.CreateCall(Callee, {Tag, Obj});
13098 case WebAssembly::BI__builtin_wasm_rethrow: {
13099 Value *Callee = CGM.getIntrinsic(Intrinsic::wasm_rethrow);
13100 return Builder.CreateCall(Callee);
13102 case WebAssembly::BI__builtin_wasm_atomic_wait_i32: {
13106 Value *Callee = CGM.getIntrinsic(Intrinsic::wasm_atomic_wait_i32);
13107 return Builder.CreateCall(Callee, {Addr, Expected, Timeout});
13109 case WebAssembly::BI__builtin_wasm_atomic_wait_i64: {
13113 Value *Callee = CGM.getIntrinsic(Intrinsic::wasm_atomic_wait_i64);
13114 return Builder.CreateCall(Callee, {Addr, Expected, Timeout});
13116 case WebAssembly::BI__builtin_wasm_atomic_notify: {
13119 Value *Callee = CGM.getIntrinsic(Intrinsic::wasm_atomic_notify);
13120 return Builder.CreateCall(Callee, {Addr, Count});
13122 case WebAssembly::BI__builtin_wasm_trunc_saturate_s_i32_f32:
13123 case WebAssembly::BI__builtin_wasm_trunc_saturate_s_i32_f64:
13124 case WebAssembly::BI__builtin_wasm_trunc_saturate_s_i64_f32:
13125 case WebAssembly::BI__builtin_wasm_trunc_saturate_s_i64_f64:
13126 case WebAssembly::BI__builtin_wasm_trunc_saturate_s_i32x4_f32x4:
13127 case WebAssembly::BI__builtin_wasm_trunc_saturate_s_i64x2_f64x2: {
13130 Value *Callee = CGM.getIntrinsic(Intrinsic::wasm_trunc_saturate_signed,
13131 {ResT, Src->getType()});
13132 return Builder.CreateCall(Callee, {Src});
13134 case WebAssembly::BI__builtin_wasm_trunc_saturate_u_i32_f32:
13135 case WebAssembly::BI__builtin_wasm_trunc_saturate_u_i32_f64:
13136 case WebAssembly::BI__builtin_wasm_trunc_saturate_u_i64_f32:
13137 case WebAssembly::BI__builtin_wasm_trunc_saturate_u_i64_f64:
13138 case WebAssembly::BI__builtin_wasm_trunc_saturate_u_i32x4_f32x4:
13139 case WebAssembly::BI__builtin_wasm_trunc_saturate_u_i64x2_f64x2: {
13142 Value *Callee = CGM.getIntrinsic(Intrinsic::wasm_trunc_saturate_unsigned,
13143 {ResT, Src->getType()});
13144 return Builder.CreateCall(Callee, {Src});
13146 case WebAssembly::BI__builtin_wasm_min_f32:
13147 case WebAssembly::BI__builtin_wasm_min_f64:
13148 case WebAssembly::BI__builtin_wasm_min_f32x4:
13149 case WebAssembly::BI__builtin_wasm_min_f64x2: {
13152 Value *Callee = CGM.getIntrinsic(Intrinsic::minimum,
13154 return Builder.CreateCall(Callee, {LHS, RHS});
13156 case WebAssembly::BI__builtin_wasm_max_f32:
13157 case WebAssembly::BI__builtin_wasm_max_f64:
13158 case WebAssembly::BI__builtin_wasm_max_f32x4:
13159 case WebAssembly::BI__builtin_wasm_max_f64x2: {
13162 Value *Callee = CGM.getIntrinsic(Intrinsic::maximum,
13164 return Builder.CreateCall(Callee, {LHS, RHS});
13166 case WebAssembly::BI__builtin_wasm_extract_lane_s_i8x16:
13167 case WebAssembly::BI__builtin_wasm_extract_lane_u_i8x16:
13168 case WebAssembly::BI__builtin_wasm_extract_lane_s_i16x8:
13169 case WebAssembly::BI__builtin_wasm_extract_lane_u_i16x8:
13170 case WebAssembly::BI__builtin_wasm_extract_lane_i32x4:
13171 case WebAssembly::BI__builtin_wasm_extract_lane_i64x2:
13172 case WebAssembly::BI__builtin_wasm_extract_lane_f32x4:
13173 case WebAssembly::BI__builtin_wasm_extract_lane_f64x2: {
13174 llvm::APSInt LaneConst;
13176 llvm_unreachable(
"Constant arg isn't actually constant?");
13178 Value *Lane = llvm::ConstantInt::get(getLLVMContext(), LaneConst);
13179 Value *Extract = Builder.CreateExtractElement(Vec, Lane);
13180 switch (BuiltinID) {
13181 case WebAssembly::BI__builtin_wasm_extract_lane_s_i8x16:
13182 case WebAssembly::BI__builtin_wasm_extract_lane_s_i16x8:
13183 return Builder.CreateSExt(Extract, ConvertType(E->
getType()));
13184 case WebAssembly::BI__builtin_wasm_extract_lane_u_i8x16:
13185 case WebAssembly::BI__builtin_wasm_extract_lane_u_i16x8:
13186 return Builder.CreateZExt(Extract, ConvertType(E->
getType()));
13187 case WebAssembly::BI__builtin_wasm_extract_lane_i32x4:
13188 case WebAssembly::BI__builtin_wasm_extract_lane_i64x2:
13189 case WebAssembly::BI__builtin_wasm_extract_lane_f32x4:
13190 case WebAssembly::BI__builtin_wasm_extract_lane_f64x2:
13193 llvm_unreachable(
"unexpected builtin ID");
13196 case WebAssembly::BI__builtin_wasm_replace_lane_i8x16:
13197 case WebAssembly::BI__builtin_wasm_replace_lane_i16x8:
13198 case WebAssembly::BI__builtin_wasm_replace_lane_i32x4:
13199 case WebAssembly::BI__builtin_wasm_replace_lane_i64x2:
13200 case WebAssembly::BI__builtin_wasm_replace_lane_f32x4:
13201 case WebAssembly::BI__builtin_wasm_replace_lane_f64x2: {
13202 llvm::APSInt LaneConst;
13204 llvm_unreachable(
"Constant arg isn't actually constant?");
13206 Value *Lane = llvm::ConstantInt::get(getLLVMContext(), LaneConst);
13208 switch (BuiltinID) {
13209 case WebAssembly::BI__builtin_wasm_replace_lane_i8x16:
13210 case WebAssembly::BI__builtin_wasm_replace_lane_i16x8: {
13212 Value *Trunc = Builder.CreateTrunc(Val, ElemType);
13213 return Builder.CreateInsertElement(Vec, Trunc, Lane);
13215 case WebAssembly::BI__builtin_wasm_replace_lane_i32x4:
13216 case WebAssembly::BI__builtin_wasm_replace_lane_i64x2:
13217 case WebAssembly::BI__builtin_wasm_replace_lane_f32x4:
13218 case WebAssembly::BI__builtin_wasm_replace_lane_f64x2:
13219 return Builder.CreateInsertElement(Vec, Val, Lane);
13221 llvm_unreachable(
"unexpected builtin ID");
13224 case WebAssembly::BI__builtin_wasm_add_saturate_s_i8x16:
13225 case WebAssembly::BI__builtin_wasm_add_saturate_u_i8x16:
13226 case WebAssembly::BI__builtin_wasm_add_saturate_s_i16x8:
13227 case WebAssembly::BI__builtin_wasm_add_saturate_u_i16x8:
13228 case WebAssembly::BI__builtin_wasm_sub_saturate_s_i8x16:
13229 case WebAssembly::BI__builtin_wasm_sub_saturate_u_i8x16:
13230 case WebAssembly::BI__builtin_wasm_sub_saturate_s_i16x8:
13231 case WebAssembly::BI__builtin_wasm_sub_saturate_u_i16x8: {
13233 switch (BuiltinID) {
13234 case WebAssembly::BI__builtin_wasm_add_saturate_s_i8x16:
13235 case WebAssembly::BI__builtin_wasm_add_saturate_s_i16x8:
13236 IntNo = Intrinsic::sadd_sat;
13238 case WebAssembly::BI__builtin_wasm_add_saturate_u_i8x16:
13239 case WebAssembly::BI__builtin_wasm_add_saturate_u_i16x8:
13240 IntNo = Intrinsic::uadd_sat;
13242 case WebAssembly::BI__builtin_wasm_sub_saturate_s_i8x16:
13243 case WebAssembly::BI__builtin_wasm_sub_saturate_s_i16x8:
13244 IntNo = Intrinsic::wasm_sub_saturate_signed;
13246 case WebAssembly::BI__builtin_wasm_sub_saturate_u_i8x16:
13247 case WebAssembly::BI__builtin_wasm_sub_saturate_u_i16x8:
13248 IntNo = Intrinsic::wasm_sub_saturate_unsigned;
13251 llvm_unreachable(
"unexpected builtin ID");
13255 Value *Callee = CGM.getIntrinsic(IntNo, ConvertType(E->
getType()));
13256 return Builder.CreateCall(Callee, {LHS, RHS});
13258 case WebAssembly::BI__builtin_wasm_bitselect: {
13262 Value *Callee = CGM.getIntrinsic(Intrinsic::wasm_bitselect,
13264 return Builder.CreateCall(Callee, {V1, V2, C});
13266 case WebAssembly::BI__builtin_wasm_any_true_i8x16:
13267 case WebAssembly::BI__builtin_wasm_any_true_i16x8:
13268 case WebAssembly::BI__builtin_wasm_any_true_i32x4:
13269 case WebAssembly::BI__builtin_wasm_any_true_i64x2:
13270 case WebAssembly::BI__builtin_wasm_all_true_i8x16:
13271 case WebAssembly::BI__builtin_wasm_all_true_i16x8:
13272 case WebAssembly::BI__builtin_wasm_all_true_i32x4:
13273 case WebAssembly::BI__builtin_wasm_all_true_i64x2: {
13275 switch (BuiltinID) {
13276 case WebAssembly::BI__builtin_wasm_any_true_i8x16:
13277 case WebAssembly::BI__builtin_wasm_any_true_i16x8:
13278 case WebAssembly::BI__builtin_wasm_any_true_i32x4:
13279 case WebAssembly::BI__builtin_wasm_any_true_i64x2:
13280 IntNo = Intrinsic::wasm_anytrue;
13282 case WebAssembly::BI__builtin_wasm_all_true_i8x16:
13283 case WebAssembly::BI__builtin_wasm_all_true_i16x8:
13284 case WebAssembly::BI__builtin_wasm_all_true_i32x4:
13285 case WebAssembly::BI__builtin_wasm_all_true_i64x2:
13286 IntNo = Intrinsic::wasm_alltrue;
13289 llvm_unreachable(
"unexpected builtin ID");
13292 Value *Callee = CGM.getIntrinsic(IntNo, Vec->getType());
13293 return Builder.CreateCall(Callee, {Vec});
13295 case WebAssembly::BI__builtin_wasm_abs_f32x4:
13296 case WebAssembly::BI__builtin_wasm_abs_f64x2: {
13299 return Builder.CreateCall(Callee, {Vec});
13301 case WebAssembly::BI__builtin_wasm_sqrt_f32x4:
13302 case WebAssembly::BI__builtin_wasm_sqrt_f64x2: {
13305 return Builder.CreateCall(Callee, {Vec});
13318 auto MakeCircLd = [&](
unsigned IntID,
bool HasImm) {
13321 BP =
Address(Builder.CreateBitCast(BP.getPointer(), Int8PtrPtrTy),
13322 BP.getAlignment());
13327 EmitScalarExpr(E->
getArg(3)) };
13330 EmitScalarExpr(E->
getArg(2)) };
13332 llvm::Value *Result = Builder.CreateCall(CGM.getIntrinsic(IntID), Ops);
13333 llvm::Value *NewBase = Builder.CreateExtractValue(Result, 1);
13335 NewBase->getType()->getPointerTo());
13339 Builder.CreateAlignedStore(NewBase, LV, Dest.
getAlignment());
13340 return Builder.CreateExtractValue(Result, 0);
13343 auto MakeCircSt = [&](
unsigned IntID,
bool HasImm) {
13346 BP =
Address(Builder.CreateBitCast(BP.getPointer(), Int8PtrPtrTy),
13347 BP.getAlignment());
13352 EmitScalarExpr(E->
getArg(3)), EmitScalarExpr(E->
getArg(4)) };
13355 EmitScalarExpr(E->
getArg(2)), EmitScalarExpr(E->
getArg(3)) };
13357 llvm::Value *NewBase = Builder.CreateCall(CGM.getIntrinsic(IntID), Ops);
13359 NewBase->getType()->getPointerTo());
13363 return Builder.CreateAlignedStore(NewBase, LV, Dest.
getAlignment());
13369 auto MakeBrevLd = [&](
unsigned IntID,
llvm::Type *DestTy) {
13374 Builder.CreateBitCast(EmitScalarExpr(E->
getArg(0)), Int8PtrTy);
13380 DestAddr =
Address(Builder.CreateBitCast(DestAddr.getPointer(), Int8PtrTy),
13381 DestAddr.getAlignment());
13382 llvm::Value *DestAddress = DestAddr.getPointer();
13387 Ops = {BaseAddress, EmitScalarExpr(E->
getArg(2))};
13389 llvm::Value *Result = Builder.CreateCall(CGM.getIntrinsic(IntID), Ops);
13391 llvm::Value *DestVal = Builder.CreateExtractValue(Result, 0);
13396 DestVal = Builder.CreateTrunc(DestVal, DestTy);
13399 Builder.CreateBitCast(DestAddress, DestVal->getType()->getPointerTo());
13400 Builder.CreateAlignedStore(DestVal, DestForStore, DestAddr.getAlignment());
13402 return Builder.CreateExtractValue(Result, 1);
13405 switch (BuiltinID) {
13406 case Hexagon::BI__builtin_HEXAGON_V6_vaddcarry:
13407 case Hexagon::BI__builtin_HEXAGON_V6_vaddcarry_128B: {
13410 if (BuiltinID == Hexagon::BI__builtin_HEXAGON_V6_vaddcarry) {
13412 ID = Intrinsic::hexagon_V6_vaddcarry;
13415 ID = Intrinsic::hexagon_V6_vaddcarry_128B;
13417 Dest = Builder.CreateBitCast(Dest,
13418 llvm::VectorType::get(Builder.getInt1Ty(), Size)->getPointerTo(0));
13419 LoadInst *QLd = Builder.CreateLoad(Dest);
13420 Ops = { EmitScalarExpr(E->
getArg(0)), EmitScalarExpr(E->
getArg(1)), QLd };
13421 llvm::Value *Result = Builder.CreateCall(CGM.getIntrinsic(ID), Ops);
13422 llvm::Value *Vprd = Builder.CreateExtractValue(Result, 1);
13424 Vprd->getType()->getPointerTo(0));
13425 Builder.CreateAlignedStore(Vprd, Base, Dest.
getAlignment());
13426 return Builder.CreateExtractValue(Result, 0);
13428 case Hexagon::BI__builtin_HEXAGON_V6_vsubcarry:
13429 case Hexagon::BI__builtin_HEXAGON_V6_vsubcarry_128B: {
13432 if (BuiltinID == Hexagon::BI__builtin_HEXAGON_V6_vsubcarry) {
13434 ID = Intrinsic::hexagon_V6_vsubcarry;
13437 ID = Intrinsic::hexagon_V6_vsubcarry_128B;
13439 Dest = Builder.CreateBitCast(Dest,
13440 llvm::VectorType::get(Builder.getInt1Ty(), Size)->getPointerTo(0));
13441 LoadInst *QLd = Builder.CreateLoad(Dest);
13442 Ops = { EmitScalarExpr(E->
getArg(0)), EmitScalarExpr(E->
getArg(1)), QLd };
13443 llvm::Value *Result = Builder.CreateCall(CGM.getIntrinsic(ID), Ops);
13444 llvm::Value *Vprd = Builder.CreateExtractValue(Result, 1);
13446 Vprd->getType()->getPointerTo(0));
13447 Builder.CreateAlignedStore(Vprd, Base, Dest.
getAlignment());
13448 return Builder.CreateExtractValue(Result, 0);
13450 case Hexagon::BI__builtin_HEXAGON_L2_loadrub_pci:
13451 return MakeCircLd(Intrinsic::hexagon_L2_loadrub_pci,
true);
13452 case Hexagon::BI__builtin_HEXAGON_L2_loadrb_pci:
13453 return MakeCircLd(Intrinsic::hexagon_L2_loadrb_pci,
true);
13454 case Hexagon::BI__builtin_HEXAGON_L2_loadruh_pci:
13455 return MakeCircLd(Intrinsic::hexagon_L2_loadruh_pci,
true);
13456 case Hexagon::BI__builtin_HEXAGON_L2_loadrh_pci:
13457 return MakeCircLd(Intrinsic::hexagon_L2_loadrh_pci,
true);
13458 case Hexagon::BI__builtin_HEXAGON_L2_loadri_pci:
13459 return MakeCircLd(Intrinsic::hexagon_L2_loadri_pci,
true);
13460 case Hexagon::BI__builtin_HEXAGON_L2_loadrd_pci:
13461 return MakeCircLd(Intrinsic::hexagon_L2_loadrd_pci,
true);
13462 case Hexagon::BI__builtin_HEXAGON_L2_loadrub_pcr:
13463 return MakeCircLd(Intrinsic::hexagon_L2_loadrub_pcr,
false);
13464 case Hexagon::BI__builtin_HEXAGON_L2_loadrb_pcr:
13465 return MakeCircLd(Intrinsic::hexagon_L2_loadrb_pcr,
false);
13466 case Hexagon::BI__builtin_HEXAGON_L2_loadruh_pcr:
13467 return MakeCircLd(Intrinsic::hexagon_L2_loadruh_pcr,
false);
13468 case Hexagon::BI__builtin_HEXAGON_L2_loadrh_pcr:
13469 return MakeCircLd(Intrinsic::hexagon_L2_loadrh_pcr,
false);
13470 case Hexagon::BI__builtin_HEXAGON_L2_loadri_pcr:
13471 return MakeCircLd(Intrinsic::hexagon_L2_loadri_pcr,
false);
13472 case Hexagon::BI__builtin_HEXAGON_L2_loadrd_pcr:
13473 return MakeCircLd(Intrinsic::hexagon_L2_loadrd_pcr,
false);
13474 case Hexagon::BI__builtin_HEXAGON_S2_storerb_pci:
13475 return MakeCircSt(Intrinsic::hexagon_S2_storerb_pci,
true);
13476 case Hexagon::BI__builtin_HEXAGON_S2_storerh_pci:
13477 return MakeCircSt(Intrinsic::hexagon_S2_storerh_pci,
true);
13478 case Hexagon::BI__builtin_HEXAGON_S2_storerf_pci:
13479 return MakeCircSt(Intrinsic::hexagon_S2_storerf_pci,
true);
13480 case Hexagon::BI__builtin_HEXAGON_S2_storeri_pci:
13481 return MakeCircSt(Intrinsic::hexagon_S2_storeri_pci,
true);
13482 case Hexagon::BI__builtin_HEXAGON_S2_storerd_pci:
13483 return MakeCircSt(Intrinsic::hexagon_S2_storerd_pci,
true);
13484 case Hexagon::BI__builtin_HEXAGON_S2_storerb_pcr:
13485 return MakeCircSt(Intrinsic::hexagon_S2_storerb_pcr,
false);
13486 case Hexagon::BI__builtin_HEXAGON_S2_storerh_pcr:
13487 return MakeCircSt(Intrinsic::hexagon_S2_storerh_pcr,
false);
13488 case Hexagon::BI__builtin_HEXAGON_S2_storerf_pcr:
13489 return MakeCircSt(Intrinsic::hexagon_S2_storerf_pcr,
false);
13490 case Hexagon::BI__builtin_HEXAGON_S2_storeri_pcr:
13491 return MakeCircSt(Intrinsic::hexagon_S2_storeri_pcr,
false);
13492 case Hexagon::BI__builtin_HEXAGON_S2_storerd_pcr:
13493 return MakeCircSt(Intrinsic::hexagon_S2_storerd_pcr,
false);
13494 case Hexagon::BI__builtin_brev_ldub:
13495 return MakeBrevLd(Intrinsic::hexagon_L2_loadrub_pbr, Int8Ty);
13496 case Hexagon::BI__builtin_brev_ldb:
13497 return MakeBrevLd(Intrinsic::hexagon_L2_loadrb_pbr, Int8Ty);
13498 case Hexagon::BI__builtin_brev_lduh:
13499 return MakeBrevLd(Intrinsic::hexagon_L2_loadruh_pbr, Int16Ty);
13500 case Hexagon::BI__builtin_brev_ldh:
13501 return MakeBrevLd(Intrinsic::hexagon_L2_loadrh_pbr, Int16Ty);
13502 case Hexagon::BI__builtin_brev_ldw:
13503 return MakeBrevLd(Intrinsic::hexagon_L2_loadri_pbr, Int32Ty);
13504 case Hexagon::BI__builtin_brev_ldd:
13505 return MakeBrevLd(Intrinsic::hexagon_L2_loadrd_pbr, Int64Ty);
ReturnValueSlot - Contains the address where the return value of a function can be stored...
Defines the clang::ASTContext interface.
static Value * emitFPIntBuiltin(CodeGenFunction &CGF, const CallExpr *E, unsigned IntrinsicID)
Represents a function declaration or definition.
llvm::IntegerType * IntTy
int
Other implicit parameter.
llvm::Value * EmitAArch64CompareBuiltinExpr(llvm::Value *Op, llvm::Type *Ty, const llvm::CmpInst::Predicate Fp, const llvm::CmpInst::Predicate Ip, const llvm::Twine &Name="")
static Value * EmitX86FMAExpr(CodeGenFunction &CGF, ArrayRef< Value *> Ops, unsigned BuiltinID, bool IsAddSub)
PointerType - C99 6.7.5.1 - Pointer Declarators.
static Value * EmitX86ExpandLoad(CodeGenFunction &CGF, ArrayRef< Value *> Ops)
A (possibly-)qualified type.
bool isBlockPointerType() const
#define fma(__x, __y, __z)
static WidthAndSignedness getIntegerWidthAndSignedness(const clang::ASTContext &context, const clang::QualType Type)
unsigned char getSummaryByte() const
const CodeGenOptions & getCodeGenOpts() const
Expr * getArg(unsigned Arg)
getArg - Return the specified argument.
static Value * emitBinaryBuiltin(CodeGenFunction &CGF, const CallExpr *E, unsigned IntrinsicID)
static struct WidthAndSignedness EncompassingIntegerType(ArrayRef< struct WidthAndSignedness > Types)
llvm::Value * EmitNVPTXBuiltinExpr(unsigned BuiltinID, const CallExpr *E)
llvm::LLVMContext & getLLVMContext()
DominatorTree GraphTraits specialization so the DominatorTree can be iterable by generic graph iterat...
bool EvaluateAsInt(EvalResult &Result, const ASTContext &Ctx, SideEffectsKind AllowSideEffects=SE_NoSideEffects) const
EvaluateAsInt - Return true if this is a constant which we can fold and convert to an integer...
const ASTRecordLayout & getASTRecordLayout(const RecordDecl *D) const
Get or compute information about the layout of the specified record (struct/union/class) D...
unsigned getNumArgs() const
getNumArgs - Return the number of actual arguments to this call.
QualType getPointeeType() const
If this is a pointer, ObjC object pointer, or block pointer, this returns the respective pointee...
const llvm::Triple & getTriple() const
Returns the target triple of the primary target.
static const Builtin::Info BuiltinInfo[]
static Value * EmitX86MinMax(CodeGenFunction &CGF, ICmpInst::Predicate Pred, ArrayRef< Value *> Ops)
Address EmitPointerWithAlignment(const Expr *Addr, LValueBaseInfo *BaseInfo=nullptr, TBAAAccessInfo *TBAAInfo=nullptr)
EmitPointerWithAlignment - Given an expression with a pointer type, emit the value and compute our be...
constexpr XRayInstrMask Typed
bool isRecordType() const
static Value * getMaskVecValue(CodeGenFunction &CGF, Value *Mask, unsigned NumElts)
static Value * EmitX86Ternlog(CodeGenFunction &CGF, bool ZeroMask, ArrayRef< Value *> Ops)
llvm::Value * EmitMSVCBuiltinExpr(MSVCIntrin BuiltinID, const CallExpr *E)
static Value * EmitToInt(CodeGenFunction &CGF, llvm::Value *V, QualType T, llvm::IntegerType *IntType)
Emit the conversions required to turn the given value into an integer of the given size...
static Value * EmitSpecialRegisterBuiltin(CodeGenFunction &CGF, const CallExpr *E, llvm::Type *RegisterType, llvm::Type *ValueType, bool IsRead, StringRef SysReg="")
The base class of the type hierarchy.
static Value * EmitX86Muldq(CodeGenFunction &CGF, bool IsSigned, ArrayRef< Value *> Ops)
#define NEONMAP1(NameBase, LLVMIntrinsic, TypeModifier)
llvm::IntegerType * Int8Ty
i8, i16, i32, and i64
static CanQualType getOSLogArgType(ASTContext &C, int Size)
Get the argument type for arguments to os_log_helper.
RValue emitRotate(const CallExpr *E, bool IsRotateRight)
virtual llvm::Value * getPipeElemAlign(const Expr *PipeArg)
RValue EmitCall(const CGFunctionInfo &CallInfo, const CGCallee &Callee, ReturnValueSlot ReturnValue, const CallArgList &Args, llvm::Instruction **callOrInvoke, SourceLocation Loc)
EmitCall - Generate a call of the given function, expecting the given result type, and using the given argument list which specifies both the LLVM arguments and the types they were derived from.
llvm::Value * EmitCheckedArgForBuiltin(const Expr *E, BuiltinCheckKind Kind)
Emits an argument for a call to a builtin.
Objects with "hidden" visibility are not seen by the dynamic linker.
static bool isSpecialMixedSignMultiply(unsigned BuiltinID, WidthAndSignedness Op1Info, WidthAndSignedness Op2Info, WidthAndSignedness ResultInfo)
Determine if a binop is a checked mixed-sign multiply we can specialize.
RAII object to set/unset CodeGenFunction::IsSanitizerScope.
static Value * EmitX86MaskedLoad(CodeGenFunction &CGF, ArrayRef< Value *> Ops, unsigned Align)
const T * getAs() const
Member-template getAs<specific type>'.
const void * Store
Store - This opaque type encapsulates an immutable mapping from locations to values.
static Value * EmitSystemZIntrinsicWithCC(CodeGenFunction &CGF, unsigned IntrinsicID, const CallExpr *E)
Handle a SystemZ function in which the final argument is a pointer to an int that receives the post-i...
static int64_t clamp(int64_t Value, int64_t Low, int64_t High)
llvm::Value * getPointer() const
Represents a parameter to a function.
static bool HasExtraNeonArgument(unsigned BuiltinID)
Return true if BuiltinID is an overloaded Neon intrinsic with an extra argument that specifies the ve...
The collection of all-type qualifiers we support.
void add(RValue rvalue, QualType type)
Expr * IgnoreImpCasts() LLVM_READONLY
IgnoreImpCasts - Skip past any implicit casts which might surround this expression.
Represents a struct/union/class.
void __ovld prefetch(const __global char *p, size_t num_elements)
Prefetch num_elements * sizeof(gentype) bytes into the global cache.
const TargetInfo & getTarget() const
static bool TypeRequiresBuiltinLaunder(CodeGenModule &CGM, QualType Ty)
Determine if the specified type requires laundering by checking if it is a dynamic class type or cont...
One of these records is kept for each identifier that is lexed.
Address getAddress() const
CodeGenFunction - This class organizes the per-function state that is used while generating LLVM code...
Holds long-lived AST nodes (such as types and decls) that can be referred to throughout the semantic ...
llvm::IntegerType * Int64Ty
RecordDecl * getDefinition() const
Returns the RecordDecl that actually defines this struct/union/class.
field_range fields() const
llvm::Value * BuildVector(ArrayRef< llvm::Value *> Ops)
SourceLocation getBeginLoc() const LLVM_READONLY
Represents a member of a struct/union/class.
llvm::IntegerType * SizeTy
static Value * EmitX86CompressStore(CodeGenFunction &CGF, ArrayRef< Value *> Ops)
static Value * EmitX86FunnelShift(CodeGenFunction &CGF, Value *Op0, Value *Op1, Value *Amt, bool IsRight)
llvm::Value * EmitAArch64BuiltinExpr(unsigned BuiltinID, const CallExpr *E, llvm::Triple::ArchType Arch)
static Value * EmitScalarFMAExpr(CodeGenFunction &CGF, MutableArrayRef< Value *> Ops, Value *Upper, bool ZeroMask=false, unsigned PTIdx=0, bool NegAcc=false)
static bool NEONSIMDIntrinsicsProvenSorted
llvm::Value * EmitHexagonBuiltinExpr(unsigned BuiltinID, const CallExpr *E)
static Value * EmitAArch64TblBuiltinExpr(CodeGenFunction &CGF, unsigned BuiltinID, const CallExpr *E, SmallVectorImpl< Value *> &Ops, llvm::Triple::ArchType Arch)
static RValue EmitBinaryAtomicPost(CodeGenFunction &CGF, llvm::AtomicRMWInst::BinOp Kind, const CallExpr *E, Instruction::BinaryOps Op, bool Invert=false)
Utility to insert an atomic instruction based Intrinsic::ID and the expression node, where the return value is the result of the operation.
static llvm::Value * EmitX86BitTestIntrinsic(CodeGenFunction &CGF, BitTest BT, const CallExpr *E, Value *BitBase, Value *BitPos)
static const NeonIntrinsicInfo AArch64SIMDIntrinsicMap[]
void EmitStoreOfScalar(llvm::Value *Value, Address Addr, bool Volatile, QualType Ty, AlignmentSource Source=AlignmentSource::Type, bool isInit=false, bool isNontemporal=false)
EmitStoreOfScalar - Store a scalar value to an address, taking care to appropriately convert from the...
static Value * EmitX86AddSubSatExpr(CodeGenFunction &CGF, ArrayRef< Value *> Ops, bool IsSigned, bool IsAddition)
static Value * EmitX86MaskLogic(CodeGenFunction &CGF, Instruction::BinaryOps Opc, ArrayRef< Value *> Ops, bool InvertLHS=false)
static ApplyDebugLocation CreateArtificial(CodeGenFunction &CGF)
Apply TemporaryLocation if it is valid.
llvm::Value * EmitCommonNeonBuiltinExpr(unsigned BuiltinID, unsigned LLVMIntrinsic, unsigned AltLLVMIntrinsic, const char *NameHint, unsigned Modifier, const CallExpr *E, SmallVectorImpl< llvm::Value *> &Ops, Address PtrOp0, Address PtrOp1, llvm::Triple::ArchType Arch)
static Value * EmitX86MaskedStore(CodeGenFunction &CGF, ArrayRef< Value *> Ops, unsigned Align)
unsigned char getNumArgsByte() const
bool isLibFunction(unsigned ID) const
Return true if this is a builtin for a libc/libm function, with a "__builtin_" prefix (e...
CharUnits - This is an opaque type for sizes expressed in character units.
static Value * EmitAtomicDecrementValue(CodeGenFunction &CGF, const CallExpr *E, AtomicOrdering Ordering=AtomicOrdering::SequentiallyConsistent)
static const NeonIntrinsicInfo AArch64SISDIntrinsicMap[]
APValue Val
Val - This is the value the expression can be folded to.
#define INTRINSIC_WITH_CC(NAME)
CharUnits getAlignment() const
Return the alignment of this pointer.
bool isVolatileQualified() const
Determine whether this type is volatile-qualified.
static bool AArch64SISDIntrinsicsProvenSorted
__DEVICE__ double powi(double __a, int __b)
static Value * EmitX86ScalarSelect(CodeGenFunction &CGF, Value *Mask, Value *Op0, Value *Op1)
Expr * IgnoreParenCasts() LLVM_READONLY
IgnoreParenCasts - Ignore parentheses and casts.
bool computeOSLogBufferLayout(clang::ASTContext &Ctx, const clang::CallExpr *E, OSLogBufferLayout &layout)
static FunctionDecl * Create(ASTContext &C, DeclContext *DC, SourceLocation StartLoc, SourceLocation NLoc, DeclarationName N, QualType T, TypeSourceInfo *TInfo, StorageClass SC, bool isInlineSpecified=false, bool hasWrittenPrototype=true, bool isConstexprSpecified=false)
static llvm::Value * dumpRecord(CodeGenFunction &CGF, QualType RType, Value *&RecordPtr, CharUnits Align, Value *Func, int Lvl)
llvm::Constant * CreateRuntimeFunction(llvm::FunctionType *Ty, StringRef Name, llvm::AttributeList ExtraAttrs=llvm::AttributeList(), bool Local=false)
Create a new runtime function with the specified type and name.
bool tryEvaluateObjectSize(uint64_t &Result, ASTContext &Ctx, unsigned Type) const
If the current Expr is a pointer, this will try to statically determine the number of bytes available...
llvm::Value * EmitLoadOfScalar(Address Addr, bool Volatile, QualType Ty, SourceLocation Loc, AlignmentSource Source=AlignmentSource::Type, bool isNontemporal=false)
EmitLoadOfScalar - Load a scalar value from an address, taking care to appropriately convert from the...
llvm::Type * HalfTy
float, double
static CharUnits One()
One - Construct a CharUnits quantity of one.
std::pair< llvm::Value *, llvm::Value * > ComplexPairTy
CXXRecordDecl * getAsCXXRecordDecl() const
Retrieves the CXXRecordDecl that this type refers to, either because the type is a RecordType or beca...
ASTContext & getContext() const
Represents a prototype with parameter type info, e.g.
llvm::Value * EmitISOVolatileStore(const CallExpr *E)
#define NEONMAP0(NameBase)
RValue - This trivial value class is used to represent the result of an expression that is evaluated...
RValue EmitBuiltinExpr(const GlobalDecl GD, unsigned BuiltinID, const CallExpr *E, ReturnValueSlot ReturnValue)
const char * getName(unsigned ID) const
Return the identifier name for the specified builtin, e.g.
QuantityType getQuantity() const
getQuantity - Get the raw integer representation of this quantity.
llvm::APSInt EvaluateKnownConstInt(const ASTContext &Ctx, SmallVectorImpl< PartialDiagnosticAt > *Diag=nullptr) const
EvaluateKnownConstInt - Call EvaluateAsRValue and return the folded integer.
static Value * EmitX86MaskedCompare(CodeGenFunction &CGF, unsigned CC, bool Signed, ArrayRef< Value *> Ops)
ASTRecordLayout - This class contains layout information for one RecordDecl, which is a struct/union/...
bool HasSideEffects(const ASTContext &Ctx, bool IncludePossibleEffects=true) const
HasSideEffects - This routine returns true for all those expressions which have any effect other than...
llvm::Value * EmitVAStartEnd(llvm::Value *ArgValue, bool IsStart)
Emits a call to an LLVM variable-argument intrinsic, either llvm.va_start or llvm.va_end.
Exposes information about the current target.
#define copysign(__x, __y)
This represents one expression.
static RValue emitLibraryCall(CodeGenFunction &CGF, const FunctionDecl *FD, const CallExpr *E, llvm::Constant *calleeValue)
const T * castAs() const
Member-template castAs<specific type>.
static CGCallee forDirect(llvm::Constant *functionPtr, const CGCalleeInfo &abstractInfo=CGCalleeInfo())
static uint64_t GetX86CpuSupportsMask(ArrayRef< StringRef > FeatureStrs)
#define INTRINSIC_X86_XSAVE_ID(NAME)
static Value * EmitX86Select(CodeGenFunction &CGF, Value *Mask, Value *Op0, Value *Op1)
llvm::Value * EmitToMemory(llvm::Value *Value, QualType Ty)
EmitToMemory - Change a scalar value from its value representation to its in-memory representation...
static Value * emitUnaryBuiltin(CodeGenFunction &CGF, const CallExpr *E, unsigned IntrinsicID)
static CharUnits fromQuantity(QuantityType Quantity)
fromQuantity - Construct a CharUnits quantity from a raw integer type.
FunctionDecl * getDirectCallee()
If the callee is a FunctionDecl, return it. Otherwise return null.
static Value * EmitFromInt(CodeGenFunction &CGF, llvm::Value *V, QualType T, llvm::Type *ResultType)
static SVal getValue(SVal val, SValBuilder &svalBuilder)
llvm::LLVMContext & getLLVMContext()
bool isSignedIntegerType() const
Return true if this is an integer type that is signed, according to C99 6.2.5p4 [char, signed char, short, int, long..], or an enum decl which has a signed representation.
llvm::IntegerType * Int32Ty
static Value * emitRangedBuiltin(CodeGenFunction &CGF, unsigned IntrinsicID, int low, int high)
static const NeonIntrinsicInfo ARMSIMDIntrinsicMap[]
LValue MakeNaturalAlignAddrLValue(llvm::Value *V, QualType T)
static bool areBOSTypesCompatible(int From, int To)
Checks if using the result of __builtin_object_size(p, From) in place of __builtin_object_size(p, To) is correct.
QualType getFunctionType(QualType ResultTy, ArrayRef< QualType > Args, const FunctionProtoType::ExtProtoInfo &EPI) const
Return a normal function type with a typed argument list.
RValue emitBuiltinOSLogFormat(const CallExpr &E)
Emit IR for __builtin_os_log_format.
static const NeonIntrinsicInfo * findNeonIntrinsicInMap(ArrayRef< NeonIntrinsicInfo > IntrinsicMap, unsigned BuiltinID, bool &MapProvenSorted)
ASTContext & getContext() const
static Value * EmitNontemporalLoad(CodeGenFunction &CGF, const CallExpr *E)
virtual llvm::Value * getPipeElemSize(const Expr *PipeArg)
llvm::Value * EmitX86BuiltinExpr(unsigned BuiltinID, const CallExpr *E)
GlobalDecl - represents a global declaration.
static Value * EmitAtomicIncrementValue(CodeGenFunction &CGF, const CallExpr *E, AtomicOrdering Ordering=AtomicOrdering::SequentiallyConsistent)
llvm::Value * EmitAMDGPUBuiltinExpr(unsigned BuiltinID, const CallExpr *E)
The l-value was considered opaque, so the alignment was determined from a type.
llvm::Value * EmitPPCBuiltinExpr(unsigned BuiltinID, const CallExpr *E)
static llvm::Value * EmitOverflowIntrinsic(CodeGenFunction &CGF, const llvm::Intrinsic::ID IntrinsicID, llvm::Value *X, llvm::Value *Y, llvm::Value *&Carry)
Emit a call to llvm.
uint64_t getFieldOffset(unsigned FieldNo) const
getFieldOffset - Get the offset of the given field index, in bits.
EltType getEltType() const
bool EvaluateAsRValue(EvalResult &Result, const ASTContext &Ctx, bool InConstantContext=false) const
EvaluateAsRValue - Return true if this is a constant which we can fold to an rvalue using any crazy t...
Enumerates target-specific builtins in their own namespaces within namespace clang.
Address CreateBitCast(Address Addr, llvm::Type *Ty, const llvm::Twine &Name="")
FunctionDecl * getAsFunction() LLVM_READONLY
Returns the function itself, or the templated function if this is a function template.
Encodes a location in the source.
static RValue getIgnored()
IdentifierInfo & get(StringRef Name)
Return the identifier token info for the specified named identifier.
LangAS getAddressSpace() const
Return the address space of this type.
llvm::Value * EmitFromMemory(llvm::Value *Value, QualType Ty)
EmitFromMemory - Change a scalar value from its memory representation to its value representation...
static Value * EmitX86MaskedCompareResult(CodeGenFunction &CGF, Value *Cmp, unsigned NumElts, Value *MaskIn)
static llvm::VectorType * GetFloatNeonType(CodeGenFunction *CGF, NeonTypeFlags IntTypeFlags)
llvm::CallSite EmitRuntimeCallOrInvoke(llvm::Value *callee, ArrayRef< llvm::Value *> args, const Twine &name="")
Emits a call or invoke instruction to the given runtime function.
ASTContext & getASTContext() const LLVM_READONLY
static Value * EmitNontemporalStore(CodeGenFunction &CGF, const CallExpr *E)
llvm::IntegerType * Int16Ty
const Decl * getDecl() const
QualType getBaseElementType(const ArrayType *VAT) const
Return the innermost element type of an array type.
static Value * MakeAtomicCmpXchgValue(CodeGenFunction &CGF, const CallExpr *E, bool ReturnBool)
Utility to insert an atomic cmpxchg instruction.
static llvm::AtomicOrdering getBitTestAtomicOrdering(BitTest::InterlockingKind I)
OpenMPLinearClauseKind Modifier
Modifier of 'linear' clause.
#define MMA_VARIANTS(geom, type)
const ParmVarDecl * getParamDecl(unsigned i) const
constexpr XRayInstrMask Custom
const ArrayType * getAsArrayType(QualType T) const
Type Query functions.
Represents a canonical, potentially-qualified type.
llvm::Constant * emitAbstract(const Expr *E, QualType T)
Emit the result of the given expression as an abstract constant, asserting that it succeeded...
llvm::Function * generateBuiltinOSLogHelperFunction(const analyze_os_log::OSLogBufferLayout &Layout, CharUnits BufferAlignment)
static RValue EmitMSVCRTSetJmp(CodeGenFunction &CGF, MSVCSetJmpKind SJKind, const CallExpr *E)
MSVC handles setjmp a bit differently on different platforms.
All available information about a concrete callee.
bool operator<(DeclarationName LHS, DeclarationName RHS)
Ordering on two declaration names.
static Value * EmitX86SExtMask(CodeGenFunction &CGF, Value *Op, llvm::Type *DstTy)
static Value * EmitAtomicCmpXchgForMSIntrin(CodeGenFunction &CGF, const CallExpr *E, AtomicOrdering SuccessOrdering=AtomicOrdering::SequentiallyConsistent)
This function should be invoked to emit atomic cmpxchg for Microsoft's _InterlockedCompareExchange* i...
char __ovld __cnfn rotate(char v, char i)
For each element in v, the bits are shifted left by the number of bits given by the corresponding ele...
bool hasSameUnqualifiedType(QualType T1, QualType T2) const
Determine whether the given types are equivalent after cvr-qualifiers have been removed.
SourceLocation getExprLoc() const LLVM_READONLY
getExprLoc - Return the preferred location for the arrow when diagnosing a problem with a generic exp...
static Value * EmitFAbs(CodeGenFunction &CGF, Value *V)
EmitFAbs - Emit a call to .fabs().
Like Angled, but marks system directories.
llvm::Value * EmitScalarExpr(const Expr *E, bool IgnoreResultAssign=false)
EmitScalarExpr - Emit the computation of the specified expression of LLVM scalar type, returning the result.
FunctionArgList - Type for representing both the decl and type of parameters to a function...
ast_type_traits::DynTypedNode Node
CGFunctionInfo - Class to encapsulate the information about a function definition.
llvm::Value * EmitNeonRShiftImm(llvm::Value *Vec, llvm::Value *Amt, llvm::Type *Ty, bool usgn, const char *name)
This class organizes the cross-function state that is used while generating LLVM code.
static bool TypeRequiresBuiltinLaunderImp(const ASTContext &Ctx, QualType Ty, llvm::SmallPtrSetImpl< const Decl *> &Seen)
Dataflow Directional Tag Classes.
#define NEONMAP2(NameBase, LLVMIntrinsic, AltLLVMIntrinsic, TypeModifier)
bool hasSideEffects() const
static RValue getComplex(llvm::Value *V1, llvm::Value *V2)
EvalResult is a struct with detailed info about an evaluated expression.
Address CreateStructGEP(Address Addr, unsigned Index, CharUnits Offset, const llvm::Twine &Name="")
llvm::Value * EmitWebAssemblyBuiltinExpr(unsigned BuiltinID, const CallExpr *E)
llvm::Value * EmitNeonCall(llvm::Function *F, SmallVectorImpl< llvm::Value *> &O, const char *name, unsigned shift=0, bool rightshift=false)
void EmitARCIntrinsicUse(ArrayRef< llvm::Value *> values)
Given a number of pointers, inform the optimizer that they're being intrinsically used up until this ...
llvm::LoadInst * CreateLoad(Address Addr, const llvm::Twine &Name="")
static RValue EmitCheckedMixedSignMultiply(CodeGenFunction &CGF, const clang::Expr *Op1, WidthAndSignedness Op1Info, const clang::Expr *Op2, WidthAndSignedness Op2Info, const clang::Expr *ResultArg, QualType ResultQTy, WidthAndSignedness ResultInfo)
Emit a checked mixed-sign multiply.
static std::string getAsString(SplitQualType split, const PrintingPolicy &Policy)
bool isBooleanType() const
static Value * EmitTargetArchBuiltinExpr(CodeGenFunction *CGF, unsigned BuiltinID, const CallExpr *E, llvm::Triple::ArchType Arch)
llvm::Function * getIntrinsic(unsigned IID, ArrayRef< llvm::Type *> Tys=None)
bool isIntegerConstantExpr(llvm::APSInt &Result, const ASTContext &Ctx, SourceLocation *Loc=nullptr, bool isEvaluated=true) const
isIntegerConstantExpr - Return true if this expression is a valid integer constant expression...
llvm::StoreInst * CreateStore(llvm::Value *Val, Address Addr, bool IsVolatile=false)
Flags to identify the types for overloaded Neon builtins.
QualType getCallReturnType(const ASTContext &Ctx) const
getCallReturnType - Get the return type of the call expr.
Expr * IgnoreParenImpCasts() LLVM_READONLY
IgnoreParenImpCasts - Ignore parentheses and implicit casts.
SmallVector< OSLogBufferItem, 4 > Items
llvm::Value * EmitNeonSplat(llvm::Value *V, llvm::Constant *Idx)
#define X86_VENDOR(ENUM, STRING)
A helper class that allows the use of isa/cast/dyncast to detect TagType objects of structs/unions/cl...
llvm::Value * vectorWrapScalar16(llvm::Value *Op)
static llvm::Value * getDefaultBuiltinObjectSizeResult(unsigned Type, llvm::IntegerType *ResType)
bool isIntegerType() const
isIntegerType() does not include complex integers (a GCC extension).
llvm::Type * getElementType() const
Return the type of the values stored in this address.
llvm::Value * EmitNeonShiftVector(llvm::Value *V, llvm::Type *Ty, bool negateForRightShift)
bool isFunctionType() const
llvm::PointerType * Int8PtrTy
static Value * packTBLDVectorList(CodeGenFunction &CGF, ArrayRef< Value *> Ops, Value *ExtOp, Value *IndexOp, llvm::Type *ResTy, unsigned IntID, const char *Name)
static llvm::Value * EmitBitTestIntrinsic(CodeGenFunction &CGF, unsigned BuiltinID, const CallExpr *E)
Emit a _bittest* intrinsic.
void setNontemporal(bool Value)
llvm::Value * EmitARMBuiltinExpr(unsigned BuiltinID, const CallExpr *E, llvm::Triple::ArchType Arch)
uint64_t getTypeSize(QualType T) const
Return the size of the specified (complete) type T, in bits.
CanQualType getCanonicalType(QualType T) const
Return the canonical (structural) type corresponding to the specified potentially non-canonical type ...
X
Add a minimal nested name specifier fixit hint to allow lookup of a tag name from an outer enclosing ...
CharUnits toCharUnitsFromBits(int64_t BitSize) const
Convert a size in bits to a size in characters.
static Value * EmitX86Abs(CodeGenFunction &CGF, ArrayRef< Value *> Ops)
TranslationUnitDecl * getTranslationUnitDecl() const
TypeInfo getTypeInfo(const Type *T) const
Get the size and alignment of the specified complete type in bits.
llvm::Function * LookupNeonLLVMIntrinsic(unsigned IntrinsicID, unsigned Modifier, llvm::Type *ArgTy, const CallExpr *E)
llvm::Type * ConvertType(QualType T)
BuiltinCheckKind
Specifies which type of sanitizer check to apply when handling a particular builtin.
static Value * EmitCommonNeonSISDBuiltinExpr(CodeGenFunction &CGF, const NeonIntrinsicInfo &SISDInfo, SmallVectorImpl< Value *> &Ops, const CallExpr *E)
Builtin::Context & BuiltinInfo
static char bitActionToX86BTCode(BitTest::ActionKind A)
llvm::Constant * getBuiltinLibFunction(const FunctionDecl *FD, unsigned BuiltinID)
Given a builtin id for a function like "__builtin_fabsf", return a Function* for "fabsf".
Defines the clang::TargetInfo interface.
CallExpr - Represents a function call (C99 6.5.2.2, C++ [expr.call]).
static Value * emitTernaryBuiltin(CodeGenFunction &CGF, const CallExpr *E, unsigned IntrinsicID)
StringRef getName() const
Get the name of identifier for this declaration as a StringRef.
__DEVICE__ int max(int __a, int __b)
static Value * EmitSignBit(CodeGenFunction &CGF, Value *V)
Emit the computation of the sign bit for a floating point value.
static RValue get(llvm::Value *V)
bool isPointerType() const
__DEVICE__ int min(int __a, int __b)
static ApplyDebugLocation CreateEmpty(CodeGenFunction &CGF)
Set the IRBuilder to not attach debug locations.
LValue - This represents an lvalue references.
Information for lazily generating a cleanup.
static Value * EmitX86ConvertToMask(CodeGenFunction &CGF, Value *In)
static Value * MakeBinaryAtomicValue(CodeGenFunction &CGF, llvm::AtomicRMWInst::BinOp Kind, const CallExpr *E, AtomicOrdering Ordering=AtomicOrdering::SequentiallyConsistent)
Utility to insert an atomic instruction based on Intrinsic::ID and the expression node...
CallArgList - Type for representing both the value and type of arguments in a call.
static RValue EmitBinaryAtomic(CodeGenFunction &CGF, llvm::AtomicRMWInst::BinOp Kind, const CallExpr *E)
#define X86_FEATURE_COMPAT(VAL, ENUM, STR)
llvm::Value * EmitTargetBuiltinExpr(unsigned BuiltinID, const CallExpr *E)
EmitTargetBuiltinExpr - Emit the given builtin call.
static llvm::VectorType * GetNeonType(CodeGenFunction *CGF, NeonTypeFlags TypeFlags, bool HasLegalHalfType=true, bool V1Ty=false)
static bool AArch64SIMDIntrinsicsProvenSorted
QualType getIntTypeForBitwidth(unsigned DestWidth, unsigned Signed) const
getIntTypeForBitwidth - sets integer QualTy according to specified details: bitwidth, signed/unsigned.
llvm::Value * EmitISOVolatileLoad(const CallExpr *E)
llvm::Value * EmitSystemZBuiltinExpr(unsigned BuiltinID, const CallExpr *E)
static OMPLinearClause * Create(const ASTContext &C, SourceLocation StartLoc, SourceLocation LParenLoc, OpenMPLinearClauseKind Modifier, SourceLocation ModifierLoc, SourceLocation ColonLoc, SourceLocation EndLoc, ArrayRef< Expr *> VL, ArrayRef< Expr *> PL, ArrayRef< Expr *> IL, Expr *Step, Expr *CalcStep, Stmt *PreInit, Expr *PostUpdate)
Creates clause with a list of variables VL and a linear step Step.
CanQualType getSizeType() const
Return the unique type for "size_t" (C99 7.17), defined in <stddef.h>.