14 #ifndef LLVM_CLANG_LEX_PREPROCESSOR_H 15 #define LLVM_CLANG_LEX_PREPROCESSOR_H 33 #include "llvm/ADT/ArrayRef.h" 34 #include "llvm/ADT/DenseMap.h" 35 #include "llvm/ADT/FoldingSet.h" 36 #include "llvm/ADT/FunctionExtras.h" 37 #include "llvm/ADT/None.h" 38 #include "llvm/ADT/Optional.h" 39 #include "llvm/ADT/PointerUnion.h" 40 #include "llvm/ADT/STLExtras.h" 41 #include "llvm/ADT/SmallPtrSet.h" 42 #include "llvm/ADT/SmallVector.h" 43 #include "llvm/ADT/StringRef.h" 44 #include "llvm/ADT/TinyPtrVector.h" 45 #include "llvm/ADT/iterator_range.h" 46 #include "llvm/Support/Allocator.h" 47 #include "llvm/Support/Casting.h" 48 #include "llvm/Support/Registry.h" 60 template<
unsigned InternalLen>
class SmallString;
66 class CodeCompletionHandler;
69 class DirectoryLookup;
70 class ExternalPreprocessorSource;
76 class PragmaNamespace;
77 class PreprocessingRecord;
78 class PreprocessorLexer;
79 class PreprocessorOptions;
95 assert(Kind != tok::raw_identifier &&
"Raw identifiers are not supported.");
96 assert(Kind != tok::identifier &&
97 "Identifiers should be created by TokenValue(IdentifierInfo *)");
105 return Tok.
getKind() == Kind &&
132 llvm::unique_function<void(const clang::Token &)> OnToken;
133 std::shared_ptr<PreprocessorOptions> PPOpts;
140 std::unique_ptr<ScratchBuffer> ScratchBuf;
149 llvm::BumpPtrAllocator BP;
187 unsigned CounterValue = 0;
191 MaxAllowedIncludeStackDepth = 200
195 bool KeepComments : 1;
196 bool KeepMacroComments : 1;
197 bool SuppressIncludeNotFoundError : 1;
200 bool InMacroArgs : 1;
203 bool OwnsHeaderSearch : 1;
206 bool DisableMacroExpansion : 1;
210 bool MacroExpansionInDirectivesOverride : 1;
215 mutable bool ReadMacrosFromExternalSource : 1;
218 bool PragmasEnabled : 1;
221 bool PreprocessedOutput : 1;
224 bool ParsingIfOrElifDirective;
227 bool InMacroArgPreExpansion;
249 std::unique_ptr<PragmaNamespace> PragmaHandlers;
253 std::unique_ptr<PragmaNamespace> PragmaHandlersBackup;
257 std::vector<CommentHandler *> CommentHandlers;
261 bool IncrementalProcessing =
false;
270 const FileEntry *CodeCompletionFile =
nullptr;
273 unsigned CodeCompletionOffset = 0;
293 bool LastTokenWasAt =
false;
301 AfterTopLevelTokenSeq = -1,
306 ImportSeq(
State S) : S(S) {}
309 void handleOpenBracket() {
310 S =
static_cast<State>(std::max<int>(S, 0) + 1);
313 void handleCloseBracket() {
314 S =
static_cast<State>(std::max<int>(S, 1) - 1);
317 void handleCloseBrace() {
318 handleCloseBracket();
319 if (S == AtTopLevel && !AfterHeaderName)
320 S = AfterTopLevelTokenSeq;
325 S = AfterTopLevelTokenSeq;
326 AfterHeaderName =
false;
331 void handleExport() {
332 if (S == AfterTopLevelTokenSeq)
338 void handleImport() {
339 if (S == AfterTopLevelTokenSeq || S == AfterExport)
347 void handleHeaderName() {
348 if (S == AfterImportSeq)
349 AfterHeaderName =
true;
359 bool atTopLevel() {
return S <= 0; }
360 bool afterImportSeq() {
return S == AfterImportSeq; }
367 bool AfterHeaderName =
false;
371 ImportSeq ImportSeqState = ImportSeq::AfterTopLevelTokenSeq;
375 bool ModuleImportExpectsIdentifier =
false;
379 std::pair<IdentifierInfo *, SourceLocation> PragmaARCCFCodeAuditedInfo;
386 bool CodeCompletionReached =
false;
405 std::pair<int, bool> SkipMainFilePreamble;
409 bool HasReachedMaxIncludeDepth =
false;
417 unsigned LexLevel = 0;
428 bool FoundNonSkipPortion,
bool FoundElse,
430 : HashTokenLoc(HashTokenLoc), IfTokenLoc(IfTokenLoc),
431 FoundNonSkipPortion(FoundNonSkipPortion), FoundElse(FoundElse),
439 class PreambleConditionalStackStore {
447 PreambleConditionalStackStore() =
default;
449 void startRecording() { ConditionalStackState = Recording; }
450 void startReplaying() { ConditionalStackState = Replaying; }
451 bool isRecording()
const {
return ConditionalStackState == Recording; }
452 bool isReplaying()
const {
return ConditionalStackState == Replaying; }
455 return ConditionalStack;
458 void doneReplaying() {
459 ConditionalStack.clear();
460 ConditionalStackState = Off;
464 if (!isRecording() && !isReplaying())
466 ConditionalStack.clear();
467 ConditionalStack.append(s.begin(), s.end());
470 bool hasRecordedPreamble()
const {
return !ConditionalStack.empty(); }
472 bool reachedEOFWhileSkipping()
const {
return SkipInfo.hasValue(); }
474 void clearSkipInfo() { SkipInfo.reset(); }
480 State ConditionalStackState = Off;
481 } PreambleConditionalStack;
487 std::unique_ptr<Lexer> CurLexer;
505 std::unique_ptr<TokenLexer> CurTokenLexer;
512 CLK_LexAfterModuleImport
513 } CurLexerKind = CLK_Lexer;
517 Module *CurLexerSubmodule =
nullptr;
522 struct IncludeStackInfo {
523 enum CurLexerKind CurLexerKind;
525 std::unique_ptr<Lexer> TheLexer;
527 std::unique_ptr<TokenLexer> TheTokenLexer;
532 IncludeStackInfo(
enum CurLexerKind CurLexerKind,
Module *TheSubmodule,
533 std::unique_ptr<Lexer> &&TheLexer,
535 std::unique_ptr<TokenLexer> &&TheTokenLexer,
537 : CurLexerKind(
std::move(CurLexerKind)),
538 TheSubmodule(
std::move(TheSubmodule)), TheLexer(
std::move(TheLexer)),
539 ThePPLexer(
std::move(ThePPLexer)),
540 TheTokenLexer(
std::move(TheTokenLexer)),
541 TheDirLookup(
std::move(TheDirLookup)) {}
543 std::vector<IncludeStackInfo> IncludeMacroStack;
547 std::unique_ptr<PPCallbacks> Callbacks;
549 struct MacroExpandsInfo {
555 : Tok(Tok), MD(MD), Range(Range) {}
560 struct ModuleMacroInfo {
565 llvm::TinyPtrVector<ModuleMacro *> ActiveModuleMacros;
569 unsigned ActiveModuleMacrosGeneration = 0;
572 bool IsAmbiguous =
false;
575 llvm::TinyPtrVector<ModuleMacro *> OverriddenMacros;
582 mutable llvm::PointerUnion<MacroDirective *, ModuleMacroInfo *>
State;
587 PP.updateOutOfDateIdentifier(const_cast<IdentifierInfo&>(*II));
593 !PP.CurSubmoduleState->VisibleModules.getGeneration())
596 auto *Info = State.dyn_cast<ModuleMacroInfo*>();
603 if (PP.CurSubmoduleState->VisibleModules.getGeneration() !=
604 Info->ActiveModuleMacrosGeneration)
605 PP.updateModuleMacroInfo(II, *Info);
610 MacroState() : MacroState(nullptr) {}
613 MacroState(MacroState &&O) noexcept :
State(O.State) {
617 MacroState &operator=(MacroState &&O) noexcept {
625 if (
auto *Info = State.dyn_cast<ModuleMacroInfo*>())
626 Info->~ModuleMacroInfo();
630 if (
auto *Info = State.dyn_cast<ModuleMacroInfo*>())
636 if (
auto *Info = State.dyn_cast<ModuleMacroInfo*>())
643 auto *Info = getModuleInfo(PP, II);
644 return Info ? Info->IsAmbiguous :
false;
649 if (
auto *Info = getModuleInfo(PP, II))
650 return Info->ActiveModuleMacros;
657 if (
auto *Latest = getLatest())
658 return Latest->findDirectiveAtLoc(Loc, SourceMgr);
663 if (
auto *Info = getModuleInfo(PP, II)) {
664 Info->OverriddenMacros.insert(Info->OverriddenMacros.end(),
665 Info->ActiveModuleMacros.begin(),
666 Info->ActiveModuleMacros.end());
667 Info->ActiveModuleMacros.clear();
668 Info->IsAmbiguous =
false;
673 if (
auto *Info = State.dyn_cast<ModuleMacroInfo*>())
674 return Info->OverriddenMacros;
680 auto *Info = State.dyn_cast<ModuleMacroInfo*>();
682 if (Overrides.empty())
688 Info->OverriddenMacros.clear();
689 Info->OverriddenMacros.insert(Info->OverriddenMacros.end(),
690 Overrides.begin(), Overrides.end());
691 Info->ActiveModuleMacrosGeneration = 0;
700 using MacroMap = llvm::DenseMap<const IdentifierInfo *, MacroState>;
702 struct SubmoduleState;
705 struct BuildingSubmoduleInfo {
716 SubmoduleState *OuterSubmoduleState;
719 unsigned OuterPendingModuleMacroNames;
722 SubmoduleState *OuterSubmoduleState,
723 unsigned OuterPendingModuleMacroNames)
724 : M(M), ImportLoc(ImportLoc), IsPragma(IsPragma),
725 OuterSubmoduleState(OuterSubmoduleState),
726 OuterPendingModuleMacroNames(OuterPendingModuleMacroNames) {}
731 struct SubmoduleState {
741 std::map<Module *, SubmoduleState> Submodules;
744 SubmoduleState NullSubmoduleState;
748 SubmoduleState *CurSubmoduleState;
751 llvm::FoldingSet<ModuleMacro> ModuleMacros;
758 llvm::DenseMap<const IdentifierInfo *, llvm::TinyPtrVector<ModuleMacro *>>
770 using WarnUnusedMacroLocsTy = llvm::SmallPtrSet<SourceLocation, 32>;
771 WarnUnusedMacroLocsTy WarnUnusedMacroLocs;
779 llvm::DenseMap<IdentifierInfo *, std::vector<MacroInfo *>>
783 unsigned NumDirectives = 0;
784 unsigned NumDefined = 0;
785 unsigned NumUndefined = 0;
786 unsigned NumPragma = 0;
788 unsigned NumElse = 0;
789 unsigned NumEndif = 0;
790 unsigned NumEnteredSourceFiles = 0;
791 unsigned MaxIncludeStackDepth = 0;
792 unsigned NumMacroExpanded = 0;
793 unsigned NumFnMacroExpanded = 0;
794 unsigned NumBuiltinMacroExpanded = 0;
795 unsigned NumFastMacroExpanded = 0;
796 unsigned NumTokenPaste = 0;
797 unsigned NumFastTokenPaste = 0;
798 unsigned NumSkipped = 0;
802 std::string Predefines;
808 FileID PCHThroughHeaderFileID;
811 bool SkippingUntilPragmaHdrStop =
false;
814 bool SkippingUntilPCHThroughHeader =
false;
818 enum { TokenLexerCacheSize = 8 };
819 unsigned NumCachedTokenLexers;
820 std::unique_ptr<TokenLexer> TokenLexerCache[TokenLexerCacheSize];
829 std::vector<std::pair<TokenLexer *, size_t>> MacroExpandingLexersStack;
850 CachedTokensTy::size_type CachedLexPos = 0;
857 std::vector<CachedTokensTy::size_type> BacktrackPositions;
859 struct MacroInfoChain {
861 MacroInfoChain *Next;
866 MacroInfoChain *MIChainHead =
nullptr;
871 Preprocessor(std::shared_ptr<PreprocessorOptions> PPOpts,
875 bool OwnsHeaderSearch =
false,
895 void InitializeForModelFile();
898 void FinalizeForModelFile();
921 ExternalSource = Source;
925 return ExternalSource;
938 return NumDirectives;
943 return ParsingIfOrElifDirective;
948 this->KeepComments = KeepComments | KeepMacroComments;
949 this->KeepMacroComments = KeepMacroComments;
958 SuppressIncludeNotFoundError = Suppress;
962 return SuppressIncludeNotFoundError;
968 PreprocessedOutput = IsPreprocessedOutput;
977 return CurPPLexer == L;
1007 C = std::make_unique<PPChainedCallbacks>(std::move(C),
1008 std::move(Callbacks));
1009 Callbacks = std::move(C);
1017 OnToken = std::move(F);
1025 (!getLangOpts().Modules || (
bool)getMacroDefinition(II));
1034 auto I = Submodules.find(M);
1035 if (I == Submodules.end())
1037 auto J = I->second.Macros.find(II);
1038 if (J == I->second.Macros.end())
1040 auto *MD = J->second.getLatest();
1048 MacroState &S = CurSubmoduleState->Macros[II];
1049 auto *MD = S.getLatest();
1050 while (MD && isa<VisibilityMacroDirective>(MD))
1053 S.getActiveModuleMacros(*
this, II),
1054 S.isAmbiguous(*
this, II));
1062 MacroState &S = CurSubmoduleState->Macros[II];
1064 if (
auto *MD = S.getLatest())
1068 S.getActiveModuleMacros(*
this, II),
1069 S.isAmbiguous(*
this, II));
1078 auto *MD = getLocalMacroDirectiveHistory(II);
1086 return const_cast<Preprocessor*
>(
this)->getMacroInfo(II);
1092 if (
auto MD = getMacroDefinition(II))
1109 appendMacroDirective(II, MD);
1129 updateOutOfDateIdentifier(const_cast<IdentifierInfo&>(*II));
1130 auto I = LeafModuleMacros.find(II);
1131 if (I != LeafModuleMacros.end())
1142 macro_iterator macro_begin(
bool IncludeExternalMacros =
true)
const;
1143 macro_iterator macro_end(
bool IncludeExternalMacros =
true)
const;
1145 llvm::iterator_range<macro_iterator>
1146 macros(
bool IncludeExternalMacros =
true)
const {
1149 return llvm::make_range(begin, end);
1171 return &Identifiers.
get(Name);
1178 void AddPragmaHandler(StringRef Namespace,
PragmaHandler *Handler);
1180 AddPragmaHandler(StringRef(), Handler);
1188 void RemovePragmaHandler(StringRef Namespace,
PragmaHandler *Handler);
1190 RemovePragmaHandler(StringRef(), Handler);
1194 void IgnorePragmas();
1206 CodeComplete = &Handler;
1211 return CodeComplete;
1216 CodeComplete =
nullptr;
1221 void CodeCompleteIncludedFile(llvm::StringRef Dir,
bool IsAngled);
1225 void CodeCompleteNaturalLanguage();
1229 CodeCompletionII =
Filter;
1236 CodeCompletionTokenRange = {Start, End};
1239 return CodeCompletionTokenRange;
1244 if (CodeCompletionII)
1245 return CodeCompletionII->
getName();
1255 void createPreprocessingRecord();
1258 bool isPCHThroughHeader(
const FileEntry *FE);
1261 bool creatingPCHWithThroughHeader();
1264 bool usingPCHWithThroughHeader();
1267 bool creatingPCHWithPragmaHdrStop();
1270 bool usingPCHWithPragmaHdrStop();
1274 void SkipTokensWhileUsingPCH();
1278 void HandleSkippedDirectiveWhileUsingPCH(
Token &Result,
1283 void EnterMainSourceFile();
1286 void EndSourceFile();
1319 void EnterTokenStream(
const Token *Toks,
unsigned NumToks,
1320 bool DisableMacroExpansion,
bool OwnsTokens,
1325 bool DisableMacroExpansion,
bool IsReinject) {
1326 EnterTokenStream(Toks.release(), NumToks, DisableMacroExpansion,
true,
1332 EnterTokenStream(Toks.data(), Toks.size(), DisableMacroExpansion,
false,
1340 void RemoveTopOfLexerStack();
1355 void EnableBacktrackAtThisPos();
1358 void CommitBacktrackedTokens();
1369 void Lex(
Token &Result);
1372 bool LexHeaderName(
Token &Result,
bool AllowMacroExpansion =
true);
1374 bool LexAfterModuleImport(
Token &Result);
1380 return CurSubmoduleState->VisibleModules.getImportLoc(M);
1387 const char *DiagnosticTag,
bool AllowMacroExpansion) {
1388 if (AllowMacroExpansion)
1391 LexUnexpandedToken(Result);
1392 return FinishLexStringLiteral(Result, String, DiagnosticTag,
1393 AllowMacroExpansion);
1398 bool FinishLexStringLiteral(
Token &Result, std::string &String,
1399 const char *DiagnosticTag,
1400 bool AllowMacroExpansion);
1410 while (Result.
getKind() == tok::comment);
1416 bool OldVal = DisableMacroExpansion;
1417 DisableMacroExpansion =
true;
1422 DisableMacroExpansion = OldVal;
1429 LexUnexpandedToken(Result);
1430 while (Result.
getKind() == tok::comment);
1436 bool parseSimpleIntegerLiteral(
Token &Tok, uint64_t &
Value);
1440 DisableMacroExpansion =
true;
1441 MacroExpansionInDirectivesOverride =
true;
1452 assert(LexLevel == 0 &&
"cannot use lookahead while lexing");
1453 if (CachedLexPos + N < CachedTokens.size())
1454 return CachedTokens[CachedLexPos+N];
1456 return PeekAhead(N+1);
1465 assert(isBacktrackEnabled() &&
1466 "Should only be called when tokens are cached for backtracking");
1467 assert(
signed(CachedLexPos) -
signed(N) >=
signed(BacktrackPositions.back())
1468 &&
"Should revert tokens up to the last backtrack position, not more");
1469 assert(
signed(CachedLexPos) -
signed(N) >= 0 &&
1470 "Corrupted backtrack positions ?");
1484 auto TokCopy = std::make_unique<Token[]>(1);
1486 EnterTokenStream(std::move(TokCopy), 1,
true, IsReinject);
1488 EnterCachingLexMode();
1489 assert(IsReinject &&
"new tokens in the middle of cached stream");
1490 CachedTokens.insert(CachedTokens.begin()+CachedLexPos,
Tok);
1503 assert(Tok.
isAnnotation() &&
"Expected annotation token");
1504 if (CachedLexPos != 0 && isBacktrackEnabled())
1505 AnnotatePreviousCachedTokens(Tok);
1511 assert(CachedLexPos != 0);
1512 return CachedTokens[CachedLexPos-1].getLastLoc();
1517 bool IsPreviousCachedToken(
const Token &Tok)
const;
1535 assert(Tok.
isAnnotation() &&
"Expected annotation token");
1536 if (CachedLexPos != 0 && isBacktrackEnabled())
1537 CachedTokens[CachedLexPos-1] = Tok;
1542 void *AnnotationVal);
1548 if (CachedLexPos != 0 && isBacktrackEnabled())
1549 CachedTokens[CachedLexPos-1] = Tok;
1554 void recomputeCurLexerKind();
1561 IncrementalProcessing = value;
1578 bool SetCodeCompletionPoint(
const FileEntry *File,
1579 unsigned Line,
unsigned Column);
1595 return CodeCompletionFileLoc;
1604 assert(isCodeCompletionEnabled() &&
"Code-completion not enabled!");
1605 CodeCompletionReached =
true;
1607 getDiagnostics().setSuppressAllDiagnostics(
true);
1614 std::pair<IdentifierInfo *, SourceLocation>
1616 return PragmaARCCFCodeAuditedInfo;
1623 PragmaARCCFCodeAuditedInfo = {Ident, Loc};
1631 return PragmaAssumeNonNullLoc;
1637 PragmaAssumeNonNullLoc = Loc;
1653 SkipMainFilePreamble.first = Bytes;
1654 SkipMainFilePreamble.second = StartOfLine;
1661 return Diags->
Report(Loc, DiagID);
1677 bool *invalid =
nullptr)
const {
1678 return Lexer::getSpelling(loc, buffer, SourceMgr, LangOpts, invalid);
1690 return Lexer::getSpelling(Tok, SourceMgr, LangOpts, Invalid);
1706 bool *Invalid =
nullptr)
const {
1707 return Lexer::getSpelling(Tok, Buffer, SourceMgr, LangOpts, Invalid);
1714 StringRef getSpelling(
const Token &Tok,
1716 bool *Invalid =
nullptr)
const;
1721 bool IgnoreWhiteSpace =
false) {
1722 return Lexer::getRawToken(Loc, Result, SourceMgr, LangOpts, IgnoreWhiteSpace);
1729 bool *Invalid =
nullptr)
const {
1730 assert(Tok.
is(tok::numeric_constant) &&
1731 Tok.
getLength() == 1 &&
"Called on unsupported token");
1732 assert(!Tok.
needsCleaning() &&
"Token can't need cleaning with length 1");
1752 return Lexer::getImmediateMacroName(Loc, SourceMgr, getLangOpts());
1760 void CreateString(StringRef Str,
Token &Tok,
1785 return Lexer::getLocForEndOfToken(Loc,
Offset, SourceMgr, LangOpts);
1795 return Lexer::isAtStartOfMacroExpansion(loc, SourceMgr, LangOpts,
1806 return Lexer::isAtEndOfMacroExpansion(loc, SourceMgr, LangOpts, MacroEnd);
1810 void DumpToken(
const Token &Tok,
bool DumpFlags =
false)
const;
1812 void DumpMacro(
const MacroInfo &MI)
const;
1818 unsigned Char)
const {
1819 return Lexer::AdvanceToTokenCharacter(TokStart, Char, SourceMgr, LangOpts);
1828 ++NumFastTokenPaste;
1835 size_t getTotalMemory()
const;
1840 void HandleMicrosoftCommentPaste(
Token &Tok);
1852 llvm::DenseMap<IdentifierInfo*,unsigned> PoisonReasons;
1862 void HandlePoisonedIdentifier(
Token & Identifier);
1867 HandlePoisonedIdentifier(Identifier);
1877 *Ident___exception_code,
1878 *Ident_GetExceptionCode;
1881 *Ident___exception_info,
1882 *Ident_GetExceptionInfo;
1885 *Ident___abnormal_termination,
1886 *Ident_AbnormalTermination;
1888 const char *getCurLexerEndPos();
1889 void diagnoseMissingHeaderInUmbrellaDir(
const Module &Mod);
1892 void PoisonSEHIdentifiers(
bool Poison =
true);
1902 bool HandleIdentifier(
Token &Identifier);
1909 bool HandleEndOfFile(
Token &Result,
bool isEndOfMacro =
false);
1913 bool HandleEndOfTokenLexer(
Token &Result);
1920 void HandleDirective(
Token &Result);
1931 bool EnableMacros =
false);
1946 Module *getCurrentModule();
1960 bool GetIncludeFilenameSpelling(
SourceLocation Loc,StringRef &Buffer);
1972 bool *IsFrameworkFound,
bool SkipCache =
false);
1982 bool isInPrimaryFile()
const;
1988 bool CheckMacroName(
Token &MacroNameTok,
MacroUse isDefineUndef,
1989 bool *ShadowFlag =
nullptr);
1992 Module *LeaveSubmodule(
bool ForPragma);
1995 friend void TokenLexer::ExpandFunctionArguments();
1997 void PushIncludeMacroStack() {
1998 assert(CurLexerKind != CLK_CachingLexer &&
"cannot push a caching lexer");
1999 IncludeMacroStack.emplace_back(CurLexerKind, CurLexerSubmodule,
2000 std::move(CurLexer), CurPPLexer,
2001 std::move(CurTokenLexer), CurDirLookup);
2002 CurPPLexer =
nullptr;
2005 void PopIncludeMacroStack() {
2006 CurLexer = std::move(IncludeMacroStack.back().TheLexer);
2007 CurPPLexer = IncludeMacroStack.back().ThePPLexer;
2008 CurTokenLexer = std::move(IncludeMacroStack.back().TheTokenLexer);
2009 CurDirLookup = IncludeMacroStack.back().TheDirLookup;
2010 CurLexerSubmodule = IncludeMacroStack.back().TheSubmodule;
2011 CurLexerKind = IncludeMacroStack.back().CurLexerKind;
2012 IncludeMacroStack.pop_back();
2015 void PropagateLineStartLeadingSpaceInfo(
Token &Result);
2019 bool needModuleMacros()
const;
2023 void updateModuleMacroInfo(
const IdentifierInfo *II, ModuleMacroInfo &Info);
2042 bool *ShadowFlag =
nullptr);
2053 MacroInfo *ReadOptionalMacroParameterListAndBody(
2054 const Token &MacroNameTok,
bool ImmediatelyAfterHeaderGuard);
2072 bool FoundNonSkipPortion,
bool FoundElse,
2077 struct DirectiveEvalResult {
2082 bool IncludedUndefinedIds;
2092 DirectiveEvalResult EvaluateDirectiveExpression(
IdentifierInfo *&IfNDefMacro);
2096 void RegisterBuiltinPragmas();
2099 void RegisterBuiltinMacros();
2114 void removeCachedMacroExpandedTokensOfLastLexer();
2119 bool isNextPPTokenLParen();
2128 void ExpandBuiltinMacro(
Token &Tok);
2133 void Handle_Pragma(
Token &Tok);
2137 void HandleMicrosoft__pragma(
Token &Tok);
2144 void setPredefinesFileID(
FileID FID) {
2145 assert(PredefinesFileID.
isInvalid() &&
"PredefinesFileID already set!");
2146 PredefinesFileID = FID;
2150 void setPCHThroughHeaderFileID(
FileID FID);
2158 static bool IsFileLexer(
const IncludeStackInfo& I) {
2159 return IsFileLexer(I.TheLexer.get(), I.ThePPLexer);
2162 bool IsFileLexer()
const {
2163 return IsFileLexer(CurLexer.get(), CurPPLexer);
2168 void CachingLex(
Token &Result);
2170 bool InCachingLexMode()
const {
2173 return !CurPPLexer && !CurTokenLexer && !IncludeMacroStack.empty();
2176 void EnterCachingLexMode();
2177 void EnterCachingLexModeUnchecked();
2179 void ExitCachingLexMode() {
2180 if (InCachingLexMode())
2181 RemoveTopOfLexerStack();
2184 const Token &PeekAhead(
unsigned N);
2185 void AnnotatePreviousCachedTokens(
const Token &Tok);
2191 void HandleLineDirective();
2192 void HandleDigitDirective(
Token &Tok);
2193 void HandleUserDiagnosticDirective(
Token &Tok,
bool isWarning);
2194 void HandleIdentSCCSDirective(
Token &Tok);
2195 void HandleMacroPublicDirective(
Token &Tok);
2196 void HandleMacroPrivateDirective();
2200 struct ImportAction {
2205 SkippedModuleImport,
2207 Module *ModuleForHeader =
nullptr;
2210 :
Kind(AK), ModuleForHeader(Mod) {
2211 assert((AK == None || Mod) &&
"no module for module action");
2218 const Token &FilenameTok,
bool &IsFrameworkFound,
bool IsImportDecl,
2220 const FileEntry *LookupFromFile, StringRef LookupFilename,
2227 const FileEntry *LookupFromFile =
nullptr);
2232 const FileEntry *LookupFromFile =
nullptr);
2236 void HandleMicrosoftImportDirective(
Token &Tok);
2242 static bool checkModuleIsAvailable(
const LangOptions &LangOpts,
2268 return PreambleConditionalStack.isRecording();
2272 return PreambleConditionalStack.hasRecordedPreamble();
2276 return PreambleConditionalStack.getStack();
2280 PreambleConditionalStack.setStack(s);
2285 PreambleConditionalStack.startReplaying();
2286 PreambleConditionalStack.setStack(s);
2287 PreambleConditionalStack.SkipInfo = SkipInfo;
2291 return PreambleConditionalStack.SkipInfo;
2297 void replayPreambleConditionalStack();
2300 void HandleDefineDirective(
Token &Tok,
bool ImmediatelyAfterHeaderGuard);
2301 void HandleUndefDirective();
2304 void HandleIfdefDirective(
Token &Result,
const Token &HashToken,
2305 bool isIfndef,
bool ReadAnyTokensBeforeDirective);
2306 void HandleIfDirective(
Token &IfToken,
const Token &HashToken,
2307 bool ReadAnyTokensBeforeDirective);
2308 void HandleEndifDirective(
Token &EndifToken);
2309 void HandleElseDirective(
Token &Result,
const Token &HashToken);
2310 void HandleElifDirective(
Token &ElifToken,
const Token &HashToken);
2316 void HandlePragmaOnce(
Token &OnceTok);
2317 void HandlePragmaMark();
2318 void HandlePragmaPoison();
2319 void HandlePragmaSystemHeader(
Token &SysHeaderTok);
2320 void HandlePragmaDependency(
Token &DependencyTok);
2321 void HandlePragmaPushMacro(
Token &Tok);
2322 void HandlePragmaPopMacro(
Token &Tok);
2323 void HandlePragmaIncludeAlias(
Token &Tok);
2324 void HandlePragmaModuleBuild(
Token &Tok);
2325 void HandlePragmaHdrstop(
Token &Tok);
2338 getSkippedRangeForExcludedConditionalBlock(
SourceLocation HashLoc);
2343 *ExcludedConditionalDirectiveSkipMappings;
2362 #endif // LLVM_CLANG_LEX_PREPROCESSOR_H PreambleSkipInfo(SourceLocation HashTokenLoc, SourceLocation IfTokenLoc, bool FoundNonSkipPortion, bool FoundElse, SourceLocation ElseLoc)
void EnterTokenStream(std::unique_ptr< Token[]> Toks, unsigned NumToks, bool DisableMacroExpansion, bool IsReinject)
A set of visible modules.
llvm::BumpPtrAllocator & getPreprocessorAllocator()
void setCodeCompletionTokenRange(const SourceLocation Start, const SourceLocation End)
Set the code completion token range for detecting replacement range later on.
Lexer - This provides a simple interface that turns a text buffer into a stream of tokens...
SelectorTable & getSelectorTable()
Holds information about both target-independent and target-specific builtins, allowing easy queries b...
Implements support for file system lookup, file system caching, and directory search management...
IdentifierInfo * getIdentifierInfo(StringRef Name) const
Return information about the specified preprocessor identifier token.
std::string getSpelling(const Token &Tok, bool *Invalid=nullptr) const
Return the 'spelling' of the Tok token.
Specialize PointerLikeTypeTraits to allow LazyGenerationalUpdatePtr to be placed into a PointerUnion...
const Token & LookAhead(unsigned N)
Peeks ahead N tokens and returns that token without consuming any tokens.
void MaybeHandlePoisonedIdentifier(Token &Identifier)
bool is(tok::TokenKind K) const
is/isNot - Predicates to check if this token is a specific kind, as in "if (Tok.is(tok::l_brace)) {...
Defines the SourceManager interface.
static const Builtin::Info BuiltinInfo[]
bool LexStringLiteral(Token &Result, std::string &String, const char *DiagnosticTag, bool AllowMacroExpansion)
Lex a string literal, which may be the concatenation of multiple string literals and may even come fr...
FileManager & getFileManager() const
bool isCodeCompletionReached() const
Returns true if code-completion is enabled and we have hit the code-completion point.
const char * getCharacterData(SourceLocation SL, bool *Invalid=nullptr) const
Return a pointer to the start of the specified location in the appropriate spelling MemoryBuffer...
Defines the clang::MacroInfo and clang::MacroDirective classes.
A description of the current definition of a macro.
bool isOutOfDate() const
Determine whether the information for this identifier is out of date with respect to the external sou...
DiagnosticBuilder Report(SourceLocation Loc, unsigned DiagID)
Issue the message to the client.
A directive for an undefined macro.
PreprocessorOptions - This class is used for passing the various options used in preprocessor initial...
void setCodeCompletionIdentifierInfo(IdentifierInfo *Filter)
Set the code completion token for filtering purposes.
void setCodeCompletionReached()
Note that we hit the code-completion point.
bool hadModuleLoaderFatalFailure() const
bool getRawToken(SourceLocation Loc, Token &Result, bool IgnoreWhiteSpace=false)
Relex the token at the specified location.
void EnterTokenStream(ArrayRef< Token > Toks, bool DisableMacroExpansion, bool IsReinject)
bool operator==(const Token &Tok) const
Builtin::Context & getBuiltinInfo()
void setPredefines(const char *P)
Set the predefines for this Preprocessor.
SourceLocation getCodeCompletionFileLoc() const
Returns the start location of the file of code-completion point.
char getSpellingOfSingleCharacterNumericConstant(const Token &Tok, bool *Invalid=nullptr) const
Given a Token Tok that is a numeric constant with length 1, return the character. ...
This interface provides a way to observe the actions of the preprocessor as it does its thing...
bool isAnnotation() const
Return true if this is any of tok::annot_* kind tokens.
const MacroDirective * getPrevious() const
Get previous definition of the macro with the same name.
tok::TokenKind getKind() const
TokenValue(tok::TokenKind Kind)
SourceLocation getLocForEndOfToken(SourceLocation Loc, unsigned Offset=0)
Computes the source location just past the end of the token at this source location.
DefMacroDirective * appendDefMacroDirective(IdentifierInfo *II, MacroInfo *MI, SourceLocation Loc)
One of these records is kept for each identifier that is lexed.
ArrayRef< PPConditionalInfo > getPreambleConditionalStack() const
Represents a macro directive exported by a module.
A directive for a defined macro or a macro imported from a module.
This table allows us to fully hide how we implement multi-keyword caching.
const TargetInfo & getTargetInfo() const
Token - This structure provides full information about a lexed token.
void SetCommentRetentionState(bool KeepComments, bool KeepMacroComments)
Control whether the preprocessor retains comments in output.
Keeps track of the various options that can be enabled, which controls the dialect of C or C++ that i...
const LangOptions & getLangOpts() const
Describes a module or submodule.
bool isPragmaLexer() const
isPragmaLexer - Returns true if this Lexer is being used to lex a pragma.
A directive for setting the module visibility of a macro.
void SetSuppressIncludeNotFoundError(bool Suppress)
SourceLocation AdvanceToTokenCharacter(SourceLocation TokStart, unsigned Char) const
Given a location that specifies the start of a token, return a new location that specifies a characte...
MacroUse
Context in which macro name is used.
llvm::Registry< PragmaHandler > PragmaHandlerRegistry
Registry of pragma handlers added by plugins.
Module * getCurrentLexerSubmodule() const
Return the submodule owning the file being lexed.
StringRef getSpelling(SourceLocation loc, SmallVectorImpl< char > &buffer, bool *invalid=nullptr) const
Return the 'spelling' of the token at the given location; does not go up to the spelling location or ...
A record of the steps taken while preprocessing a source file, including the various preprocessing di...
HeaderSearch & getHeaderSearchInfo() const
Forward-declares and imports various common LLVM datatypes that clang wants to use unqualified...
void LexNonComment(Token &Result)
Lex a token.
Concrete class used by the front-end to report problems and issues.
bool isMacroDefined(const IdentifierInfo *II)
bool isMacroDefinedInLocalModule(const IdentifierInfo *II, Module *M)
Determine whether II is defined as a macro within the module M, if that is a module that we've alread...
Defines the Diagnostic-related interfaces.
const TargetInfo * getAuxTargetInfo() const
TokenLexer - This implements a lexer that returns tokens from a macro body or token stream instead of...
void setCodeCompletionHandler(CodeCompletionHandler &Handler)
Set the code completion handler to the given object.
void setTokenWatcher(llvm::unique_function< void(const clang::Token &)> F)
Register a function that would be called on each token in the final expanded token stream...
void setPreprocessedOutput(bool IsPreprocessedOutput)
Sets whether the preprocessor is responsible for producing output or if it is producing tokens to be ...
void LexUnexpandedToken(Token &Result)
Just like Lex, but disables macro expansion of identifier tokens.
A little helper class used to produce diagnostics.
void clearCodeCompletionHandler()
Clear out the code completion handler.
Provides lookups to, and iteration over, IdentiferInfo objects.
Exposes information about the current target.
const IdentifierTable & getIdentifierTable() const
bool isCodeCompletionEnabled() const
Determine if we are performing code completion.
Abstract interface for external sources of preprocessor information.
Defines the clang::LangOptions interface.
unsigned getCounterValue() const
Represents a character-granular source range.
const DefMacroDirective * getDirective() const
void AnnotateCachedTokens(const Token &Tok)
We notify the Preprocessor that if it is caching tokens (because backtrack is enabled) it should repl...
const AnnotatedLine * Line
SourceLocation HashTokenLoc
void SetMacroExpansionOnlyInDirectives()
Disables macro expansion everywhere except for preprocessor directives.
bool isCurrentLexer(const PreprocessorLexer *L) const
Return true if we are lexing directly from the specified lexer.
Implements an efficient mapping from strings to IdentifierInfo nodes.
MacroArgs - An instance of this class captures information about the formal arguments specified to a ...
void RevertCachedTokens(unsigned N)
When backtracking is enabled and tokens are cached, this allows to revert a specific number of tokens...
A class for tracking whether we're inside a VA_OPT during a traversal of the tokens of a variadic mac...
SourceLocation getLocation() const
Return a source location identifier for the specified offset in the current file. ...
DefInfo getDefinition()
Traverses the macro directives history and returns the next macro definition directive along with inf...
void IncrementPasteCounter(bool isFast)
Increment the counters for the number of token paste operations performed.
Stores token information for comparing actual tokens with predefined values.
llvm::Optional< PreambleSkipInfo > getPreambleSkipInfo() const
DiagnosticBuilder Diag(const Token &Tok, unsigned DiagID) const
void setPragmaAssumeNonNullLoc(SourceLocation Loc)
Set the location of the currently-active #pragma clang assume_nonnull begin.
const MacroInfo * getMacroInfo(const IdentifierInfo *II) const
bool getCommentRetentionState() const
Defines the clang::IdentifierInfo, clang::IdentifierTable, and clang::Selector interfaces.
TokenValue(IdentifierInfo *II)
PPCallbacks * getPPCallbacks() const
DirectoryLookup - This class represents one entry in the search list that specifies the search order ...
bool isRecordingPreamble() const
const DirectoryLookup * GetCurDirLookup()
Get the DirectoryLookup structure used to find the current FileEntry, if CurLexer is non-null and if ...
SourceManager & getSourceManager() const
MacroDirective * getLocalMacroDirective(const IdentifierInfo *II) const
Given an identifier, return its latest non-imported MacroDirective if it is #define'd and not #undef'...
const DefInfo findDirectiveAtLoc(SourceLocation L, const SourceManager &SM) const
Find macro definition active in the specified source location.
ExternalPreprocessorSource * getExternalSource() const
Encapsulates changes to the "macros namespace" (the location where the macro name became active...
void TypoCorrectToken(const Token &Tok)
Update the current token to represent the provided identifier, in order to cache an action performed ...
void setExternalSource(ExternalPreprocessorSource *Source)
Encodes a location in the source.
IdentifierInfo & get(StringRef Name)
Return the identifier token info for the specified named identifier.
bool isAnnotation(TokenKind K)
Return true if this is any of tok::annot_* kinds.
MacroDefinition getMacroDefinition(const IdentifierInfo *II)
IdentifierInfo * getIdentifierInfo() const
IdentifierTable & getIdentifierTable()
void setPragmasEnabled(bool Enabled)
Cached information about one file (either on disk or in the virtual file system). ...
CodeCompletionHandler * getCodeCompletionHandler() const
Retrieve the current code-completion handler.
bool isAtStartOfMacroExpansion(SourceLocation loc, SourceLocation *MacroBegin=nullptr) const
Returns true if the given MacroID location points at the first token of the macro expansion...
bool isLiteral(TokenKind K)
Return true if this is a "literal" kind, like a numeric constant, string, etc.
bool isPoisoned() const
Return true if this token has been poisoned.
bool SawDateOrTime() const
Returns true if the preprocessor has seen a use of DATE or TIME in the file so far.
TokenKind
Provides a simple uniform namespace for tokens from all C languages.
void enableIncrementalProcessing(bool value=true)
Enables the incremental processing.
MacroDefinition getMacroDefinitionAtLoc(const IdentifierInfo *II, SourceLocation Loc)
Defines the clang::Module class, which describes a module in the source code.
PreprocessorLexer * getCurrentLexer() const
Return the current lexer being lexed from.
bool isAtEndOfMacroExpansion(SourceLocation loc, SourceLocation *MacroEnd=nullptr) const
Returns true if the given MacroID location points at the last token of the macro expansion.
void EnterToken(const Token &Tok, bool IsReinject)
Enters a token in the token stream to be lexed next.
void ReplaceLastTokenWithAnnotation(const Token &Tok)
Replace the last token with an annotation token.
SourceLocation getPragmaAssumeNonNullLoc() const
The location of the currently-active #pragma clang assume_nonnull begin.
SourceLocation getCodeCompletionLoc() const
Returns the location of the code-completion point.
const MacroInfo * getMacroInfo() const
StringRef getName() const
Return the actual identifier string.
bool isMacroDefined(StringRef Id)
An opaque identifier used by SourceManager which refers to a source file (MemoryBuffer) along with it...
bool isBacktrackEnabled() const
True if EnableBacktrackAtThisPos() was called and caching of tokens is on.
bool hadMacroDefinition() const
Returns true if this identifier was #defined to some value at any moment.
Dataflow Directional Tag Classes.
SourceLocation IfTokenLoc
SmallVector< Token, 4 > CachedTokens
A set of tokens that has been cached for later parsing.
SourceRange getCodeCompletionTokenRange() const
std::pair< IdentifierInfo *, SourceLocation > getPragmaARCCFCodeAuditedInfo() const
The location of the currently-active #pragma clang arc_cf_code_audited begin.
PreprocessingRecord * getPreprocessingRecord() const
Retrieve the preprocessing record, or NULL if there is no preprocessing record.
Reads an AST files chain containing the contents of a translation unit.
PragmaHandler - Instances of this interface defined to handle the various pragmas that the language f...
llvm::iterator_range< macro_iterator > macros(bool IncludeExternalMacros=true) const
SourceLocation getLastCachedTokenLocation() const
Get the location of the last cached token, suitable for setting the end location of an annotation tok...
StringRef getCodeCompletionFilter()
Get the code completion token for filtering purposes.
Abstract interface for a module loader.
unsigned getLength() const
Encapsulates the data about a macro definition (e.g.
OnOffSwitch
Defines the possible values of an on-off-switch (C99 6.10.6p2).
const char * getLiteralData() const
getLiteralData - For a literal token (numeric constant, string, etc), this returns a pointer to the s...
void setPragmaARCCFCodeAuditedInfo(IdentifierInfo *Ident, SourceLocation Loc)
Set the location of the currently-active #pragma clang arc_cf_code_audited begin. ...
bool getPragmasEnabled() const
unsigned getSpelling(const Token &Tok, const char *&Buffer, bool *Invalid=nullptr) const
Get the spelling of a token into a preallocated buffer, instead of as an std::string.
SourceLocation getDefinitionLoc() const
Return the location that the macro was defined at.
ModuleLoader & getModuleLoader() const
Retrieve the module loader associated with this preprocessor.
void LexUnexpandedNonComment(Token &Result)
Like LexNonComment, but this disables macro expansion of identifier tokens.
PreprocessorOptions & getPreprocessorOpts() const
Retrieve the preprocessor options used to initialize this preprocessor.
Cached information about one directory (either on disk or in the virtual file system).
Defines the PPCallbacks interface.
Defines the clang::TokenKind enum and support functions.
DefMacroDirective * appendDefMacroDirective(IdentifierInfo *II, MacroInfo *MI)
SourceLocation getModuleImportLoc(Module *M) const
Defines the clang::SourceLocation class and associated facilities.
void RemovePragmaHandler(PragmaHandler *Handler)
FileID getPredefinesFileID() const
Returns the FileID for the preprocessor predefines.
DiagnosticsEngine & getDiagnostics() const
void setMainFileDir(const DirectoryEntry *Dir)
Set the directory in which the main file should be considered to have been found, if it is not a real...
static bool isMacroDefined(const Sema &S, SourceLocation Loc, StringRef Name)
TranslationUnitKind
Describes the kind of translation unit being processed.
bool hasRecordedPreamble() const
const std::string & getPredefines() const
bool needsCleaning() const
Return true if this token has trigraphs or escaped newlines in it.
void setPredefines(StringRef P)
ArrayRef< ModuleMacro * > getLeafModuleMacros(const IdentifierInfo *II) const
Get the list of leaf (non-overridden) module macros for a name.
bool isIncrementalProcessingEnabled() const
Returns true if incremental processing is enabled.
The translation unit is a complete translation unit.
StringRef getImmediateMacroName(SourceLocation Loc)
Retrieve the name of the immediate macro expansion.
llvm::DenseMap< const llvm::MemoryBuffer *, const PreprocessorSkippedRangeMapping * > ExcludedPreprocessorDirectiveSkipMapping
The datastructure that holds the mapping between the active memory buffers and the individual skip ma...
MacroMap::const_iterator macro_iterator
Describes how and where the pragma was introduced.
void setCounterValue(unsigned V)
bool isParsingIfOrElifDirective() const
True if we are currently preprocessing a if or #elif directive.
void setSkipMainFilePreamble(unsigned Bytes, bool StartOfLine)
Instruct the preprocessor to skip part of the main source file.
A trivial tuple used to represent a source range.
bool GetSuppressIncludeNotFoundError()
void setReplayablePreambleConditionalStack(ArrayRef< PPConditionalInfo > s, llvm::Optional< PreambleSkipInfo > SkipInfo)
MacroInfo * getMacroInfo(const IdentifierInfo *II)
Callback handler that receives notifications when performing code completion within the preprocessor...
bool isPreprocessedOutput() const
Returns true if the preprocessor is responsible for generating output, false if it is producing token...
bool hasMacroDefinition() const
Return true if this identifier is #defined to some other value.
DiagnosticBuilder Diag(SourceLocation Loc, unsigned DiagID) const
Forwarding function for diagnostics.
void addPPCallbacks(std::unique_ptr< PPCallbacks > C)
void setRecordedPreambleConditionalStack(ArrayRef< PPConditionalInfo > s)
This class handles loading and caching of source files into memory.
An RAII class that tracks when the Preprocessor starts and stops lexing the definition of a (ISO C/C+...
void AddPragmaHandler(PragmaHandler *Handler)
Engages in a tight little dance with the lexer to efficiently preprocess tokens.
void setDiagnostics(DiagnosticsEngine &D)
unsigned getNumDirectives() const
Retrieve the number of Directives that have been processed by the Preprocessor.