54 #include "llvm/ADT/APInt.h" 55 #include "llvm/ADT/ArrayRef.h" 56 #include "llvm/ADT/DenseMap.h" 57 #include "llvm/ADT/STLExtras.h" 58 #include "llvm/ADT/SmallString.h" 59 #include "llvm/ADT/SmallVector.h" 60 #include "llvm/ADT/StringRef.h" 61 #include "llvm/ADT/StringSwitch.h" 62 #include "llvm/Support/Capacity.h" 63 #include "llvm/Support/ErrorHandling.h" 64 #include "llvm/Support/MemoryBuffer.h" 65 #include "llvm/Support/raw_ostream.h" 73 using namespace clang;
85 : PPOpts(
std::move(PPOpts)), Diags(&diags), LangOpts(opts),
86 FileMgr(Headers.getFileMgr()), SourceMgr(SM),
87 ScratchBuf(new
ScratchBuffer(SourceMgr)), HeaderInfo(Headers),
88 TheModuleLoader(TheModuleLoader), ExternalSource(nullptr),
92 Identifiers(IILookup), PragmaHandlers(new
PragmaNamespace(StringRef())),
93 TUKind(TUKind), SkipMainFilePreamble(0,
true),
94 CurSubmoduleState(&NullSubmoduleState) {
95 OwnsHeaderSearch = OwnsHeaders;
99 KeepMacroComments =
false;
100 SuppressIncludeNotFoundError =
false;
103 DisableMacroExpansion =
false;
104 MacroExpansionInDirectivesOverride =
false;
107 InMacroArgPreExpansion =
false;
108 NumCachedTokenLexers = 0;
109 PragmasEnabled =
true;
110 ParsingIfOrElifDirective =
false;
111 PreprocessedOutput =
false;
114 ReadMacrosFromExternalSource =
false;
116 BuiltinInfo = std::make_unique<Builtin::Context>();
126 Ident__VA_OPT__ =
nullptr;
130 RegisterBuiltinPragmas();
133 RegisterBuiltinMacros();
135 if(LangOpts.Borland) {
146 Ident__exception_info = Ident__exception_code =
nullptr;
147 Ident__abnormal_termination = Ident___exception_info =
nullptr;
148 Ident___exception_code = Ident___abnormal_termination =
nullptr;
149 Ident_GetExceptionInfo = Ident_GetExceptionCode =
nullptr;
150 Ident_AbnormalTermination =
nullptr;
155 SkippingUntilPragmaHdrStop =
true;
158 if (!this->PPOpts->PCHThroughHeader.empty() &&
159 !this->PPOpts->ImplicitPCHInclude.empty())
160 SkippingUntilPCHThroughHeader =
true;
162 if (this->PPOpts->GeneratePreamble)
163 PreambleConditionalStack.startRecording();
165 ExcludedConditionalDirectiveSkipMappings =
166 this->PPOpts->ExcludedConditionalDirectiveSkipMappings;
167 if (ExcludedConditionalDirectiveSkipMappings)
168 ExcludedConditionalDirectiveSkipMappings->clear();
172 assert(BacktrackPositions.empty() &&
"EnableBacktrack/Backtrack imbalance!");
174 IncludeMacroStack.clear();
177 while (MacroInfoChain *I = MIChainHead) {
178 MIChainHead = I->Next;
179 I->~MacroInfoChain();
185 std::fill(TokenLexerCache, TokenLexerCache + NumCachedTokenLexers,
nullptr);
186 CurTokenLexer.reset();
189 for (
MacroArgs *ArgList = MacroArgCache; ArgList;)
190 ArgList = ArgList->deallocate();
193 if (OwnsHeaderSearch)
199 assert((!this->Target || this->Target == &Target) &&
200 "Invalid override of target information");
201 this->Target = &Target;
203 assert((!this->AuxTarget || this->AuxTarget == AuxTarget) &&
204 "Invalid override of aux target information.");
205 this->AuxTarget = AuxTarget;
208 BuiltinInfo->InitializeTarget(Target, AuxTarget);
216 NumEnteredSourceFiles = 0;
219 PragmaHandlersBackup = std::move(PragmaHandlers);
220 PragmaHandlers = std::make_unique<PragmaNamespace>(StringRef());
221 RegisterBuiltinPragmas();
224 PredefinesFileID =
FileID();
228 NumEnteredSourceFiles = 1;
230 PragmaHandlers = std::move(PragmaHandlersBackup);
237 if (!DumpFlags)
return;
239 llvm::errs() <<
"\t";
241 llvm::errs() <<
" [StartOfLine]";
243 llvm::errs() <<
" [LeadingSpace]";
245 llvm::errs() <<
" [ExpandDisabled]";
248 llvm::errs() <<
" [UnClean='" << StringRef(Start, Tok.
getLength())
252 llvm::errs() <<
"\tLoc=<";
258 Loc.
print(llvm::errs(), SourceMgr);
262 llvm::errs() <<
"MACRO: ";
263 for (
unsigned i = 0, e = MI.
getNumTokens(); i != e; ++i) {
267 llvm::errs() <<
"\n";
271 llvm::errs() <<
"\n*** Preprocessor Stats:\n";
272 llvm::errs() << NumDirectives <<
" directives found:\n";
273 llvm::errs() <<
" " << NumDefined <<
" #define.\n";
274 llvm::errs() <<
" " << NumUndefined <<
" #undef.\n";
275 llvm::errs() <<
" #include/#include_next/#import:\n";
276 llvm::errs() <<
" " << NumEnteredSourceFiles <<
" source files entered.\n";
277 llvm::errs() <<
" " << MaxIncludeStackDepth <<
" max include stack depth\n";
278 llvm::errs() <<
" " << NumIf <<
" #if/#ifndef/#ifdef.\n";
279 llvm::errs() <<
" " << NumElse <<
" #else/#elif.\n";
280 llvm::errs() <<
" " << NumEndif <<
" #endif.\n";
281 llvm::errs() <<
" " << NumPragma <<
" #pragma.\n";
282 llvm::errs() << NumSkipped <<
" #if/#ifndef#ifdef regions skipped\n";
284 llvm::errs() << NumMacroExpanded <<
"/" << NumFnMacroExpanded <<
"/" 285 << NumBuiltinMacroExpanded <<
" obj/fn/builtin macros expanded, " 286 << NumFastMacroExpanded <<
" on the fast path.\n";
287 llvm::errs() << (NumFastTokenPaste+NumTokenPaste)
288 <<
" token paste (##) operations performed, " 289 << NumFastTokenPaste <<
" on the fast path.\n";
291 llvm::errs() <<
"\nPreprocessor Memory: " <<
getTotalMemory() <<
"B total";
293 llvm::errs() <<
"\n BumpPtr: " << BP.getTotalMemory();
294 llvm::errs() <<
"\n Macro Expanded Tokens: " 295 << llvm::capacity_in_bytes(MacroExpandedTokens);
296 llvm::errs() <<
"\n Predefines Buffer: " << Predefines.capacity();
298 llvm::errs() <<
"\n Macros: " 299 << llvm::capacity_in_bytes(CurSubmoduleState->Macros);
300 llvm::errs() <<
"\n #pragma push_macro Info: " 301 << llvm::capacity_in_bytes(PragmaPushMacroInfo);
302 llvm::errs() <<
"\n Poison Reasons: " 303 << llvm::capacity_in_bytes(PoisonReasons);
304 llvm::errs() <<
"\n Comment Handlers: " 305 << llvm::capacity_in_bytes(CommentHandlers) <<
"\n";
310 if (IncludeExternalMacros && ExternalSource &&
311 !ReadMacrosFromExternalSource) {
312 ReadMacrosFromExternalSource =
true;
318 CurSubmoduleState->Macros.insert(std::make_pair(Macro.II, MacroState()));
320 return CurSubmoduleState->Macros.begin();
324 return BP.getTotalMemory()
325 + llvm::capacity_in_bytes(MacroExpandedTokens)
326 + Predefines.capacity()
329 + llvm::capacity_in_bytes(CurSubmoduleState->Macros)
330 + llvm::capacity_in_bytes(PragmaPushMacroInfo)
331 + llvm::capacity_in_bytes(PoisonReasons)
332 + llvm::capacity_in_bytes(CommentHandlers);
337 if (IncludeExternalMacros && ExternalSource &&
338 !ReadMacrosFromExternalSource) {
339 ReadMacrosFromExternalSource =
true;
343 return CurSubmoduleState->Macros.end();
350 std::equal(Tokens.begin(), Tokens.end(), MI->
tokens_begin());
357 StringRef BestSpelling;
361 Def = I->second.findDirectiveAtLoc(Loc, SourceMgr);
373 BestLocation = Location;
374 BestSpelling = I->first->getName();
382 CurLexerKind = CLK_Lexer;
383 else if (CurTokenLexer)
384 CurLexerKind = CLK_TokenLexer;
386 CurLexerKind = CLK_CachingLexer;
390 unsigned CompleteLine,
391 unsigned CompleteColumn) {
393 assert(CompleteLine && CompleteColumn &&
"Starts from 1:1");
394 assert(!CodeCompletionFile &&
"Already set");
396 using llvm::MemoryBuffer;
399 bool Invalid =
false;
405 const char *Position = Buffer->getBufferStart();
407 for (; *Position; ++Position) {
408 if (*Position !=
'\r' && *Position !=
'\n')
412 if ((Position[1] ==
'\r' || Position[1] ==
'\n') &&
413 Position[0] != Position[1])
420 Position += CompleteColumn - 1;
424 if (SkipMainFilePreamble.first &&
426 if (Position - Buffer->getBufferStart() < SkipMainFilePreamble.first)
427 Position = Buffer->getBufferStart() + SkipMainFilePreamble.first;
430 if (Position > Buffer->getBufferEnd())
431 Position = Buffer->getBufferEnd();
433 CodeCompletionFile = File;
434 CodeCompletionOffset = Position - Buffer->getBufferStart();
436 auto NewBuffer = llvm::WritableMemoryBuffer::getNewUninitMemBuffer(
437 Buffer->getBufferSize() + 1, Buffer->getBufferIdentifier());
438 char *NewBuf = NewBuffer->getBufferStart();
439 char *NewPos = std::copy(Buffer->getBufferStart(), Position, NewBuf);
441 std::copy(Position, Buffer->getBufferEnd(), NewPos+1);
465 bool *Invalid)
const {
467 if (Tok.
isNot(tok::raw_identifier) && !Tok.
hasUCN()) {
470 return II->getName();
477 const char *Ptr = Buffer.data();
479 return StringRef(Ptr, Len);
491 SourceLocation Loc = ScratchBuf->getToken(Str.data(), Str.size(), DestPtr);
493 if (ExpansionLocStart.
isValid())
495 ExpansionLocEnd, Str.size());
499 if (Tok.
is(tok::raw_identifier))
509 bool Invalid =
false;
517 ScratchBuf->getToken(Buffer.data() + LocInfo.second, Length, DestPtr);
538 assert(NumEnteredSourceFiles == 0 &&
"Cannot reenter the main file!");
549 if (SkipMainFilePreamble.first > 0)
550 CurLexer->SetByteOffset(SkipMainFilePreamble.first,
551 SkipMainFilePreamble.second);
560 std::unique_ptr<llvm::MemoryBuffer> SB =
561 llvm::MemoryBuffer::getMemBufferCopy(Predefines,
"<built-in>");
562 assert(SB &&
"Cannot create predefined source buffer");
564 assert(FID.
isValid() &&
"Could not create FileID for predefines?");
565 setPredefinesFileID(FID);
570 if (!PPOpts->PCHThroughHeader.empty()) {
576 false,
nullptr,
nullptr, CurDir,
582 << PPOpts->PCHThroughHeader;
585 setPCHThroughHeaderFileID(
595 void Preprocessor::setPCHThroughHeaderFileID(
FileID FID) {
596 assert(PCHThroughHeaderFileID.
isInvalid() &&
597 "PCHThroughHeaderFileID already set!");
598 PCHThroughHeaderFileID = FID;
602 assert(PCHThroughHeaderFileID.
isValid() &&
603 "Invalid PCH through header FileID");
608 return TUKind ==
TU_Prefix && !PPOpts->PCHThroughHeader.empty() &&
609 PCHThroughHeaderFileID.
isValid();
613 return TUKind !=
TU_Prefix && !PPOpts->PCHThroughHeader.empty() &&
614 PCHThroughHeaderFileID.
isValid();
618 return TUKind ==
TU_Prefix && PPOpts->PCHWithHdrStop;
622 return TUKind !=
TU_Prefix && PPOpts->PCHWithHdrStop;
631 bool ReachedMainFileEOF =
false;
632 bool UsingPCHThroughHeader = SkippingUntilPCHThroughHeader;
633 bool UsingPragmaHdrStop = SkippingUntilPragmaHdrStop;
638 switch (CurLexerKind) {
643 CurTokenLexer->Lex(Tok);
645 case CLK_CachingLexer:
648 case CLK_LexAfterModuleImport:
653 ReachedMainFileEOF =
true;
656 if (UsingPCHThroughHeader && !SkippingUntilPCHThroughHeader)
658 if (UsingPragmaHdrStop && !SkippingUntilPragmaHdrStop)
661 if (ReachedMainFileEOF) {
662 if (UsingPCHThroughHeader)
664 << PPOpts->PCHThroughHeader << 1;
665 else if (!PPOpts->PCHWithHdrStopCreate)
670 void Preprocessor::replayPreambleConditionalStack() {
672 if (PreambleConditionalStack.isReplaying()) {
674 "CurPPLexer is null when calling replayPreambleConditionalStack.");
676 PreambleConditionalStack.doneReplaying();
677 if (PreambleConditionalStack.reachedEOFWhileSkipping())
678 SkipExcludedConditionalBlock(
679 PreambleConditionalStack.SkipInfo->HashTokenLoc,
680 PreambleConditionalStack.SkipInfo->IfTokenLoc,
681 PreambleConditionalStack.SkipInfo->FoundNonSkipPortion,
682 PreambleConditionalStack.SkipInfo->FoundElse,
683 PreambleConditionalStack.SkipInfo->ElseLoc);
690 Callbacks->EndOfMainFile();
701 assert(!Identifier.
getRawIdentifier().empty() &&
"No raw identifier data!");
711 StringRef CleanedStr =
getSpelling(Identifier, IdentifierBuffer);
713 if (Identifier.
hasUCN()) {
724 if (
getLangOpts().MSVCCompat && II->isCPlusPlusOperatorKeyword() &&
726 Identifier.
setKind(tok::identifier);
728 Identifier.
setKind(II->getTokenID());
734 PoisonReasons[II] = DiagID;
738 assert(Ident__exception_code && Ident__exception_info);
739 assert(Ident___exception_code && Ident___exception_info);
753 "Can't handle identifiers without identifier info!");
754 llvm::DenseMap<IdentifierInfo*,unsigned>::const_iterator it =
756 if(it == PoisonReasons.end())
757 Diag(Identifier, diag::err_pp_used_poisoned_id);
768 if (LangOpts.CPlusPlus)
769 return llvm::StringSwitch<diag::kind>(II.
getName())
771 .Case(#NAME, diag::warn_cxx11_keyword)
772 #define CXX2A_KEYWORD(NAME, FLAGS) \ 773 .Case(#NAME, diag::warn_cxx2a_keyword) 774 #include "clang/Basic/TokenKinds.def" 778 "Keyword not known to come from a newer Standard or proposed Standard");
781 void Preprocessor::updateOutOfDateIdentifier(
IdentifierInfo &II)
const {
796 "Can't handle identifiers without identifier info!");
805 if (II.isOutOfDate()) {
806 bool CurrentIsPoisoned =
false;
807 const bool IsSpecialVariadicMacro =
808 &II == Ident__VA_ARGS__ || &II == Ident__VA_OPT__;
809 if (IsSpecialVariadicMacro)
812 updateOutOfDateIdentifier(II);
813 Identifier.
setKind(II.getTokenID());
815 if (IsSpecialVariadicMacro)
816 II.setIsPoisoned(CurrentIsPoisoned);
821 if (II.isPoisoned() && CurPPLexer) {
827 auto *MI = MD.getMacroInfo();
828 assert(MI &&
"macro definition with no macro info?");
829 if (!DisableMacroExpansion) {
833 if (!MI->isFunctionLike() || isNextPPTokenLParen())
834 return HandleMacroExpandedIdentifier(Identifier, MD);
840 if (MI->isObjectLike() || isNextPPTokenLParen())
841 Diag(Identifier, diag::pp_disabled_macro_expansion);
851 if (II.isFutureCompatKeyword() && !DisableMacroExpansion) {
855 II.setIsFutureCompatKeyword(
false);
862 if (II.isExtensionToken() && !DisableMacroExpansion)
863 Diag(Identifier, diag::ext_token_used);
873 if (((LastTokenWasAt && II.isModulesImport()) ||
874 Identifier.
is(tok::kw_import)) &&
875 !InMacroArgs && !DisableMacroExpansion &&
877 CurLexerKind != CLK_CachingLexer) {
879 ModuleImportPath.clear();
880 ModuleImportExpectsIdentifier =
true;
881 CurLexerKind = CLK_LexAfterModuleImport;
892 switch (CurLexerKind) {
894 ReturnedToken = CurLexer->Lex(Result);
897 ReturnedToken = CurTokenLexer->Lex(Result);
899 case CLK_CachingLexer:
901 ReturnedToken =
true;
903 case CLK_LexAfterModuleImport:
907 }
while (!ReturnedToken);
920 if (
getLangOpts().CPlusPlusModules && LexLevel == 1 &&
923 case tok::l_paren:
case tok::l_square:
case tok::l_brace:
924 ImportSeqState.handleOpenBracket();
926 case tok::r_paren:
case tok::r_square:
927 ImportSeqState.handleCloseBracket();
930 ImportSeqState.handleCloseBrace();
933 ImportSeqState.handleSemi();
935 case tok::header_name:
936 case tok::annot_header_unit:
937 ImportSeqState.handleHeaderName();
940 ImportSeqState.handleExport();
942 case tok::identifier:
944 ImportSeqState.handleImport();
945 if (ImportSeqState.afterImportSeq()) {
947 ModuleImportPath.clear();
948 ModuleImportExpectsIdentifier =
true;
949 CurLexerKind = CLK_LexAfterModuleImport;
955 ImportSeqState.handleMisc();
960 LastTokenWasAt = Result.
is(tok::at);
988 if (FilenameTok.
is(tok::less) && AllowMacroExpansion) {
995 FilenameBuffer.push_back(
'<');
1001 while (FilenameTok.
isNot(tok::greater)) {
1005 Diag(Start, diag::note_matching) << tok::less;
1012 if (FilenameTok.
is(tok::code_completion)) {
1021 FilenameBuffer.push_back(
' ');
1025 size_t PreAppendSize = FilenameBuffer.size();
1026 FilenameBuffer.resize(PreAppendSize + FilenameTok.
getLength());
1028 const char *BufPtr = &FilenameBuffer[PreAppendSize];
1029 unsigned ActualLen =
getSpelling(FilenameTok, BufPtr);
1032 if (BufPtr != &FilenameBuffer[PreAppendSize])
1033 memcpy(&FilenameBuffer[PreAppendSize], BufPtr, ActualLen);
1036 if (FilenameTok.
getLength() != ActualLen)
1037 FilenameBuffer.resize(PreAppendSize + ActualLen);
1041 FilenameTok.
setKind(tok::header_name);
1046 }
else if (FilenameTok.
is(tok::string_literal) && AllowMacroExpansion) {
1057 StringRef Str =
getSpelling(FilenameTok, FilenameBuffer);
1058 if (Str.size() >= 2 && Str.front() ==
'"' && Str.back() ==
'"')
1059 FilenameTok.
setKind(tok::header_name);
1070 unsigned BracketDepth = 0;
1072 Toks.emplace_back();
1075 switch (Toks.back().getKind()) {
1076 case tok::l_paren:
case tok::l_square:
case tok::l_brace:
1080 case tok::r_paren:
case tok::r_square:
case tok::r_brace:
1081 if (BracketDepth == 0)
1087 if (BracketDepth == 0)
1130 if (ModuleImportPath.empty() &&
getLangOpts().CPlusPlusModules) {
1140 auto ToksCopy = std::make_unique<Token[]>(Toks.size());
1141 std::copy(Toks.begin(), Toks.end(), ToksCopy.get());
1142 EnterTokenStream(std::move(ToksCopy), Toks.size(),
1148 if (Result.
is(tok::header_name)) {
1152 Suffix.push_back(Result);
1157 if (Suffix.back().isNot(tok::semi)) {
1159 EnterTokens(Suffix);
1168 Diag(SemiLoc, diag::err_header_import_semi_in_macro);
1173 ImportTok.
setKind(tok::kw_import);
1178 auto Action = HandleHeaderIncludeOrImport(
1180 switch (Action.Kind) {
1184 case ImportAction::ModuleBegin:
1186 Suffix.emplace_back();
1187 Suffix.back().startToken();
1188 Suffix.back().setKind(tok::annot_module_begin);
1189 Suffix.back().setLocation(SemiLoc);
1190 Suffix.back().setAnnotationEndLoc(SemiLoc);
1191 Suffix.back().setAnnotationValue(Action.ModuleForHeader);
1194 case ImportAction::ModuleImport:
1195 case ImportAction::SkippedModuleImport:
1198 Suffix[0].setKind(tok::annot_header_unit);
1199 Suffix[0].setAnnotationEndLoc(Suffix[0].getLocation());
1200 Suffix[0].setAnnotationValue(Action.ModuleForHeader);
1205 EnterTokens(Suffix);
1215 if (ModuleImportExpectsIdentifier && Result.
getKind() == tok::identifier) {
1220 ModuleImportExpectsIdentifier =
false;
1221 CurLexerKind = CLK_LexAfterModuleImport;
1228 if (!ModuleImportExpectsIdentifier && Result.
getKind() == tok::period) {
1229 ModuleImportExpectsIdentifier =
true;
1230 CurLexerKind = CLK_LexAfterModuleImport;
1235 if (ModuleImportPath.empty() || Result.
is(
tok::eof))
1241 if (Result.
isNot(tok::semi)) {
1242 Suffix.push_back(Result);
1244 if (Suffix.back().isNot(tok::semi)) {
1246 EnterTokens(Suffix);
1249 SemiLoc = Suffix.back().getLocation();
1256 std::string FlatModuleName;
1258 for (
auto &Piece : ModuleImportPath) {
1259 if (!FlatModuleName.empty())
1260 FlatModuleName +=
".";
1261 FlatModuleName += Piece.first->getName();
1264 ModuleImportPath.clear();
1265 ModuleImportPath.push_back(
1269 Module *Imported =
nullptr;
1271 Imported = TheModuleLoader.
loadModule(ModuleImportLoc,
1279 Callbacks->moduleImport(ModuleImportLoc, ModuleImportPath, Imported);
1281 if (!Suffix.empty()) {
1282 EnterTokens(Suffix);
1289 CurSubmoduleState->VisibleModules.setVisible(
1294 Diag(ModuleImportLoc, diag::warn_module_conflict)
1295 << Path[0]->getFullModuleName()
1296 << Conflict->getFullModuleName()
1301 if (!BuildingSubmoduleStack.empty() && M != BuildingSubmoduleStack.back().M)
1302 BuildingSubmoduleStack.back().M->Imports.insert(M);
1306 const char *DiagnosticTag,
1307 bool AllowMacroExpansion) {
1309 if (Result.
isNot(tok::string_literal)) {
1310 Diag(Result, diag::err_expected_string_literal)
1311 << 0 << DiagnosticTag;
1318 StrToks.push_back(Result);
1321 Diag(Result, diag::err_invalid_string_udl);
1323 if (AllowMacroExpansion)
1327 }
while (Result.
is(tok::string_literal));
1331 assert(Literal.
isAscii() &&
"Didn't allow wide strings in");
1337 Diag(StrToks[0].getLocation(), diag::err_expected_string_literal)
1338 << 0 << DiagnosticTag;
1347 assert(Tok.
is(tok::numeric_constant));
1349 bool NumberInvalid =
false;
1350 StringRef Spelling =
getSpelling(Tok, IntegerBuffer, &NumberInvalid);
1354 if (Literal.hadError || !Literal.isIntegerLiteral() || Literal.hasUDSuffix())
1357 if (Literal.GetIntegerValue(APVal))
1360 Value = APVal.getLimitedValue();
1365 assert(Handler &&
"NULL comment handler");
1366 assert(llvm::find(CommentHandlers, Handler) == CommentHandlers.end() &&
1367 "Comment handler already registered");
1368 CommentHandlers.push_back(Handler);
1372 std::vector<CommentHandler *>::iterator Pos =
1373 llvm::find(CommentHandlers, Handler);
1374 assert(Pos != CommentHandlers.end() &&
"Comment handler not registered");
1375 CommentHandlers.erase(Pos);
1379 bool AnyPendingTokens =
false;
1380 for (std::vector<CommentHandler *>::iterator H = CommentHandlers.begin(),
1381 HEnd = CommentHandlers.end();
1383 if ((*H)->HandleComment(*
this, Comment))
1384 AnyPendingTokens =
true;
void setCodeCompletionTokenRange(const SourceLocation Start, const SourceLocation End)
Set the code completion token range for detecting replacement range later on.
void AddKeywords(const LangOptions &LangOpts)
Populate the identifier table with info about the language keywords for the language specified by Lan...
SourceLocation getLocWithOffset(int Offset) const
Return a source location with the specified offset from this SourceLocation.
void FinalizeForModelFile()
Cleanup after model file parsing.
void SkipTokensWhileUsingPCH()
Skip tokens until after the #include of the through header or until after a #pragma hdrstop...
IdentifierInfo * getIdentifierInfo(StringRef Name) const
Return information about the specified preprocessor identifier token.
void setFlagValue(TokenFlags Flag, bool Val)
Set a flag to either true or false.
Defines the clang::FileManager interface and associated types.
bool is(tok::TokenKind K) const
is/isNot - Predicates to check if this token is a specific kind, as in "if (Tok.is(tok::l_brace)) {...
Defines the SourceManager interface.
bool creatingPCHWithPragmaHdrStop()
True if creating a PCH with a #pragma hdrstop.
Module * getCurrentModule()
Retrieves the module that we're currently building, if any.
Defines the FileSystemStatCache interface.
bool isLoadedFileID(FileID FID) const
Returns true if FID came from a PCH/Module.
void EndSourceFile()
Inform the preprocessor callbacks that processing is complete.
The translation unit is a prefix to a translation unit, and is not complete.
virtual void CodeCompleteNaturalLanguage()
Callback invoked when performing code completion in a part of the file where we expect natural langua...
bool isBeforeInTranslationUnit(SourceLocation LHS, SourceLocation RHS) const
Determines the order of 2 source locations in the translation unit.
const char * getCharacterData(SourceLocation SL, bool *Invalid=nullptr) const
Return a pointer to the start of the specified location in the appropriate spelling MemoryBuffer...
Defines the clang::MacroInfo and clang::MacroDirective classes.
A description of the current definition of a macro.
bool isOutOfDate() const
Determine whether the information for this identifier is out of date with respect to the external sou...
void setFlag(TokenFlags Flag)
Set the specified flag.
StringRef getLastMacroWithSpelling(SourceLocation Loc, ArrayRef< TokenValue > Tokens) const
Return the name of the macro defined before Loc that has spelling Tokens.
void setCodeCompletionIdentifierInfo(IdentifierInfo *Filter)
Set the code completion token for filtering purposes.
void setCodeCompletionReached()
Note that we hit the code-completion point.
Preprocessor(std::shared_ptr< PreprocessorOptions > PPOpts, DiagnosticsEngine &diags, LangOptions &opts, SourceManager &SM, HeaderSearch &Headers, ModuleLoader &TheModuleLoader, IdentifierInfoLookup *IILookup=nullptr, bool OwnsHeaderSearch=false, TranslationUnitKind TUKind=TU_Complete)
void createPreprocessingRecord()
Create a new preprocessing record, which will keep track of all macro expansions, macro definitions...
StringRef getBufferData(FileID FID, bool *Invalid=nullptr) const
Return a StringRef to the source buffer data for the specified FileID.
bool parseSimpleIntegerLiteral(Token &Tok, uint64_t &Value)
Parses a simple integer literal to get its numeric value.
tokens_iterator tokens_begin() const
tok::TokenKind getKind() const
bool isLiteral() const
Return true if this is a "literal", like a numeric constant, string, etc.
void SetPoisonReason(IdentifierInfo *II, unsigned DiagID)
Specifies the reason for poisoning an identifier.
One of these records is kept for each identifier that is lexed.
Represents a macro directive exported by a module.
bool LexAfterModuleImport(Token &Result)
Lex a token following the 'import' contextual keyword.
void print(raw_ostream &OS, const SourceManager &SM) const
void setRawIdentifierData(const char *Ptr)
virtual ~ExternalPreprocessorSource()
Token - This structure provides full information about a lexed token.
void setKind(tok::TokenKind K)
virtual void CodeCompleteIncludedFile(llvm::StringRef Dir, bool IsAngled)
Callback invoked when performing code completion inside the filename part of an #include directive...
void removeCommentHandler(CommentHandler *Handler)
Remove the specified comment handler.
Keeps track of the various options that can be enabled, which controls the dialect of C or C++ that i...
const LangOptions & getLangOpts() const
Describes a module or submodule.
bool SetCodeCompletionPoint(const FileEntry *File, unsigned Line, unsigned Column)
Specify the point at which code-completion will be performed.
llvm::Registry< PragmaHandler > PragmaHandlerRegistry
Registry of pragma handlers added by plugins.
StringRef getSpelling(SourceLocation loc, SmallVectorImpl< char > &buffer, bool *invalid=nullptr) const
Return the 'spelling' of the token at the given location; does not go up to the spelling location or ...
A record of the steps taken while preprocessing a source file, including the various preprocessing di...
HeaderSearch & getHeaderSearchInfo() const
const Token & getReplacementToken(unsigned Tok) const
bool hasUDSuffix() const
Return true if this token is a string or character literal which has a ud-suffix. ...
macro_iterator macro_begin(bool IncludeExternalMacros=true) const
Forward-declares and imports various common LLVM datatypes that clang wants to use unqualified...
void Initialize(const TargetInfo &Target, const TargetInfo *AuxTarget=nullptr)
Initialize the preprocessor using information about the target.
Concrete class used by the front-end to report problems and issues.
SourceLocation getSpellingLoc(SourceLocation Loc) const
Given a SourceLocation object, return the spelling location referenced by the ID. ...
macro_iterator macro_end(bool IncludeExternalMacros=true) const
bool HandleComment(Token &result, SourceRange Comment)
void LexUnexpandedToken(Token &Result)
Just like Lex, but disables macro expansion of identifier tokens.
void recomputeCurLexerKind()
Recompute the current lexer kind based on the CurLexer/ CurTokenLexer pointers.
Provides lookups to, and iteration over, IdentiferInfo objects.
Exposes information about the current target.
IdentifierInfo * LookUpIdentifierInfo(Token &Identifier) const
Given a tok::raw_identifier token, look up the identifier information for the token and install it in...
Defines the clang::LangOptions interface.
void makeModuleVisible(Module *M, SourceLocation Loc)
SourceLocation getLocation() const
bool isInSystemHeader(SourceLocation Loc) const
Returns if a SourceLocation is in a system header.
const FileEntry * getFileEntryForID(FileID FID) const
Returns the FileEntry record for the provided FileID.
const AnnotatedLine * Line
FileID createFileID(const FileEntry *SourceFile, SourceLocation IncludePos, SrcMgr::CharacteristicKind FileCharacter, int LoadedID=0, unsigned LoadedOffset=0)
Create a new FileID that represents the specified file being #included from the specified IncludePosi...
bool usingPCHWithThroughHeader()
True if using a PCH with a through header.
bool hasLeadingEmptyMacro() const
Return true if this token has an empty macro before it.
virtual void ReadDefinedMacros()=0
Read the set of macros defined by this external macro source.
MacroArgs - An instance of this class captures information about the formal arguments specified to a ...
bool isPCHThroughHeader(const FileEntry *FE)
Returns true if the FileEntry is the PCH through header.
void EnterMainSourceFile()
Enter the specified FileID as the main source file, which implicitly adds the builtin defines etc...
SourceLocation getLocation() const
Return a source location identifier for the specified offset in the current file. ...
Defines the clang::Preprocessor interface.
void setIsPoisoned(bool Value=true)
setIsPoisoned - Mark this identifier as poisoned.
static bool MacroDefinitionEquals(const MacroInfo *MI, ArrayRef< TokenValue > Tokens)
Compares macro tokens with a specified token value sequence.
size_t getTotalMemory() const
bool getCommentRetentionState() const
Defines the clang::IdentifierInfo, clang::IdentifierTable, and clang::Selector interfaces.
bool isObjectLike() const
The result type of a method or function.
StringRef GetString() const
DirectoryLookup - This class represents one entry in the search list that specifies the search order ...
void HandlePoisonedIdentifier(Token &Identifier)
Display reason for poisoned identifier.
SourceManager & getSourceManager() const
void DumpMacro(const MacroInfo &MI) const
void overrideFileContents(const FileEntry *SourceFile, llvm::MemoryBuffer *Buffer, bool DoNotFree)
Override the contents of the given source file by providing an already-allocated buffer.
ExternalPreprocessorSource * getExternalSource() const
bool isExpandDisabled() const
Return true if this identifier token should never be expanded in the future, due to C99 6...
Encodes a location in the source.
void setLength(unsigned Len)
SourceLocation createExpansionLoc(SourceLocation Loc, SourceLocation ExpansionLocStart, SourceLocation ExpansionLocEnd, unsigned TokLength, bool ExpansionIsTokenRange=true, int LoadedID=0, unsigned LoadedOffset=0)
Return a new SourceLocation that encodes the fact that a token from SpellingLoc should actually be re...
MacroDefinition getMacroDefinition(const IdentifierInfo *II)
All of the names in this module are hidden.
IdentifierInfo * getIdentifierInfo() const
Cached information about one file (either on disk or in the virtual file system). ...
void setIdentifierInfo(IdentifierInfo *II)
void Lex(Token &Result)
Lex the next token for this preprocessor.
bool isPoisoned() const
Return true if this token has been poisoned.
bool isAtStartOfLine() const
isAtStartOfLine - Return true if this token is at the start of a line.
Defines the clang::Module class, which describes a module in the source code.
void expandUCNs(SmallVectorImpl< char > &Buf, StringRef Input)
Copy characters from Input to Buf, expanding any UCNs.
__DEVICE__ void * memcpy(void *__a, const void *__b, size_t __c)
void DumpToken(const Token &Tok, bool DumpFlags=false) const
Print the token to stderr, used for debugging.
StringRef getName() const
Return the actual identifier string.
An opaque identifier used by SourceManager which refers to a source file (MemoryBuffer) along with it...
bool isNot(tok::TokenKind K) const
bool EnterSourceFile(FileID FID, const DirectoryLookup *Dir, SourceLocation Loc)
Add a source file to the top of the include stack and start lexing tokens from it instead of the curr...
SourceLocation SplitToken(SourceLocation TokLoc, unsigned Length)
Split the first Length characters out of the token starting at TokLoc and return a location pointing ...
Dataflow Directional Tag Classes.
bool isValid() const
Return true if this is a valid SourceLocation object.
bool getFlag(TokenFlags Flag) const
Get the specified flag.
unsigned getNumTokens() const
Return the number of tokens that this macro expands to.
static diag::kind getFutureCompatDiagKind(const IdentifierInfo &II, const LangOptions &LangOpts)
Returns a diagnostic message kind for reporting a future keyword as appropriate for the identifier an...
void addCommentHandler(CommentHandler *Handler)
Add the specified comment handler to the preprocessor.
Optional< FileEntryRef > LookupFile(SourceLocation FilenameLoc, StringRef Filename, bool isAngled, const DirectoryLookup *FromDir, const FileEntry *FromFile, const DirectoryLookup *&CurDir, SmallVectorImpl< char > *SearchPath, SmallVectorImpl< char > *RelativePath, ModuleMap::KnownHeader *SuggestedModule, bool *IsMapped, bool *IsFrameworkFound, bool SkipCache=false)
Given a "foo" or <foo> reference, look up the indicated file.
bool usingPCHWithPragmaHdrStop()
True if using a PCH with a #pragma hdrstop.
const llvm::MemoryBuffer * getMemoryBufferForFile(const FileEntry *File, bool *Invalid=nullptr)
Retrieve the memory buffer associated with the given file.
FileID getMainFileID() const
Returns the FileID of the main source file.
bool isOneOf(tok::TokenKind K1, tok::TokenKind K2) const
MacroInfo * getMacroInfo()
virtual ModuleLoadResult loadModule(SourceLocation ImportLoc, ModuleIdPath Path, Module::NameVisibilityKind Visibility, bool IsInclusionDirective)=0
Attempt to load the given module.
void CodeCompleteNaturalLanguage()
Hook used by the lexer to invoke the "natural language" code completion point.
Abstract interface for a module loader.
unsigned getLength() const
void PoisonSEHIdentifiers(bool Poison=true)
void setLiteralData(const char *Ptr)
Encapsulates the data about a macro definition (e.g.
void CollectPpImportSuffix(SmallVectorImpl< Token > &Toks)
Collect the tokens of a C++20 pp-import-suffix.
PragmaNamespace - This PragmaHandler subdivides the namespace of pragmas, allowing hierarchical pragm...
bool HandleIdentifier(Token &Identifier)
Callback invoked when the lexer reads an identifier and has filled in the tokens IdentifierInfo membe...
bool FinishLexStringLiteral(Token &Result, std::string &String, const char *DiagnosticTag, bool AllowMacroExpansion)
Complete the lexing of a string literal where the first token has already been lexed (see LexStringLi...
void setConditionalLevels(ArrayRef< PPConditionalInfo > CL)
virtual void updateOutOfDateIdentifier(IdentifierInfo &II)=0
Update an out-of-date identifier.
void CreateString(StringRef Str, Token &Tok, SourceLocation ExpansionLocStart=SourceLocation(), SourceLocation ExpansionLocEnd=SourceLocation())
Plop the specified string into a scratch buffer and set the specified token's location and length to ...
void InitializeForModelFile()
Initialize the preprocessor to parse a model file.
#define CXX11_KEYWORD(NAME, FLAGS)
Defines the clang::SourceLocation class and associated facilities.
FileID getPredefinesFileID() const
Returns the FileID for the preprocessor predefines.
StringRef getRawIdentifier() const
getRawIdentifier - For a raw identifier token (i.e., an identifier lexed in raw mode), returns a reference to the text substring in the buffer if known.
SourceLocation createTokenSplitLoc(SourceLocation SpellingLoc, SourceLocation TokenStart, SourceLocation TokenEnd)
Return a new SourceLocation that encodes that the token starting at TokenStart ends prematurely at To...
TranslationUnitKind
Describes the kind of translation unit being processed.
void DumpLocation(SourceLocation Loc) const
StringLiteralParser - This decodes string escape characters and performs wide string analysis and Tra...
unsigned kind
All of the diagnostics that can be emitted by the frontend.
bool creatingPCHWithThroughHeader()
True if creating a PCH with a through header.
const char * getTokenName(TokenKind Kind) LLVM_READNONE
Determines the name of a token as used within the front end.
Defines the clang::TargetInfo interface.
bool needsCleaning() const
Return true if this token has trigraphs or escaped newlines in it.
NumericLiteralParser - This performs strict semantic analysis of the content of a ppnumber...
MacroMap::const_iterator macro_iterator
bool isModulesImport() const
Determine whether this is the contextual keyword import.
void CodeCompleteIncludedFile(llvm::StringRef Dir, bool IsAngled)
Hook used by the lexer to invoke the "included file" code completion point.
void setLocation(SourceLocation L)
A trivial tuple used to represent a source range.
bool LexHeaderName(Token &Result, bool AllowMacroExpansion=true)
Lex a token, forming a header-name token if possible.
bool hasUCN() const
Returns true if this token contains a universal character name.
bool isFutureCompatKeyword() const
is/setIsFutureCompatKeyword - Initialize information about whether or not this language token is a ke...
Defines the PreprocessorLexer interface.
void LexIncludeFilename(Token &FilenameTok)
Lex a token, producing a header-name token if possible.
ScratchBuffer - This class exposes a simple interface for the dynamic construction of tokens...
bool hasLeadingSpace() const
Return true if this token has whitespace before it.
DiagnosticBuilder Diag(SourceLocation Loc, unsigned DiagID) const
Forwarding function for diagnostics.
void addPPCallbacks(std::unique_ptr< PPCallbacks > C)
This class handles loading and caching of source files into memory.
virtual ~CodeCompletionHandler()
Defines enum values for all the target-independent builtin functions.
void startToken()
Reset all flags to cleared.
std::pair< FileID, unsigned > getDecomposedLoc(SourceLocation Loc) const
Decompose the specified location into a raw FileID + Offset pair.
SourceLocation getEndLoc() const