22 #include "llvm/ADT/StringSwitch.h" 23 #include "llvm/Support/FileSystem.h" 24 #include "llvm/Support/MemoryBuffer.h" 25 #include "llvm/Support/Path.h" 26 using namespace clang;
38 return IncludeMacroStack.empty();
41 assert(IsFileLexer(IncludeMacroStack[0]) &&
42 "Top level include stack isn't our primary lexer?");
44 IncludeMacroStack.begin() + 1, IncludeMacroStack.end(),
45 [&](
const IncludeStackInfo &ISI) ->
bool {
return IsFileLexer(ISI); });
56 for (
const IncludeStackInfo &ISI : llvm::reverse(IncludeMacroStack)) {
58 return ISI.ThePPLexer;
72 assert(!CurTokenLexer &&
"Cannot #include a file inside a macro!");
73 ++NumEnteredSourceFiles;
75 if (MaxIncludeStackDepth < IncludeMacroStack.size())
76 MaxIncludeStackDepth = IncludeMacroStack.size();
80 const llvm::MemoryBuffer *InputFile =
81 getSourceManager().getBuffer(FID, Loc, &Invalid);
84 Diag(Loc, diag::err_pp_error_opening_file)
85 << std::string(SourceMgr.getBufferName(FileStart)) <<
"";
89 if (isCodeCompletionEnabled() &&
90 SourceMgr.getFileEntryForID(FID) == CodeCompletionFile) {
91 CodeCompletionFileLoc = SourceMgr.getLocForStartOfFile(FID);
93 CodeCompletionFileLoc.getLocWithOffset(CodeCompletionOffset);
96 EnterSourceFileWithLexer(
new Lexer(FID, InputFile, *
this), CurDir);
102 void Preprocessor::EnterSourceFileWithLexer(
Lexer *TheLexer,
106 if (CurPPLexer || CurTokenLexer)
107 PushIncludeMacroStack();
109 CurLexer.reset(TheLexer);
110 CurPPLexer = TheLexer;
111 CurDirLookup = CurDir;
112 CurLexerSubmodule =
nullptr;
113 if (CurLexerKind != CLK_LexAfterModuleImport)
114 CurLexerKind = CLK_Lexer;
117 if (Callbacks && !CurLexer->Is_PragmaLexer) {
119 SourceMgr.getFileCharacteristic(CurLexer->getFileLoc());
121 Callbacks->FileChanged(CurLexer->getFileLoc(),
130 std::unique_ptr<TokenLexer> TokLexer;
131 if (NumCachedTokenLexers == 0) {
132 TokLexer = llvm::make_unique<TokenLexer>(
Tok, ILEnd, Macro, Args, *
this);
134 TokLexer = std::move(TokenLexerCache[--NumCachedTokenLexers]);
135 TokLexer->Init(Tok, ILEnd, Macro, Args);
138 PushIncludeMacroStack();
139 CurDirLookup =
nullptr;
140 CurTokenLexer = std::move(TokLexer);
141 if (CurLexerKind != CLK_LexAfterModuleImport)
142 CurLexerKind = CLK_TokenLexer;
157 void Preprocessor::EnterTokenStream(
const Token *Toks,
unsigned NumToks,
158 bool DisableMacroExpansion,
160 if (CurLexerKind == CLK_CachingLexer) {
165 Toks, Toks + NumToks);
173 ExitCachingLexMode();
174 EnterTokenStream(Toks, NumToks, DisableMacroExpansion, OwnsTokens);
175 EnterCachingLexMode();
180 std::unique_ptr<TokenLexer> TokLexer;
181 if (NumCachedTokenLexers == 0) {
182 TokLexer = llvm::make_unique<TokenLexer>(
183 Toks, NumToks, DisableMacroExpansion, OwnsTokens, *
this);
185 TokLexer = std::move(TokenLexerCache[--NumCachedTokenLexers]);
186 TokLexer->Init(Toks, NumToks, DisableMacroExpansion, OwnsTokens);
190 PushIncludeMacroStack();
191 CurDirLookup =
nullptr;
192 CurTokenLexer = std::move(TokLexer);
193 if (CurLexerKind != CLK_LexAfterModuleImport)
194 CurLexerKind = CLK_TokenLexer;
205 StringRef Path = FilePath;
206 while (!Path.empty()) {
209 Result = FilePath.substr(Path.size());
210 llvm::sys::path::append(Result,
211 llvm::sys::path::filename(File->
getName()));
216 Path = llvm::sys::path::parent_path(Path);
222 void Preprocessor::PropagateLineStartLeadingSpaceInfo(
Token &
Result) {
224 CurTokenLexer->PropagateLineStartLeadingSpaceInfo(Result);
228 CurLexer->PropagateLineStartLeadingSpaceInfo(Result);
241 const char *Preprocessor::getCurLexerEndPos() {
242 const char *EndPos = CurLexer->BufferEnd;
243 if (EndPos != CurLexer->BufferStart &&
244 (EndPos[-1] ==
'\n' || EndPos[-1] ==
'\r')) {
248 if (EndPos != CurLexer->BufferStart &&
249 (EndPos[-1] ==
'\n' || EndPos[-1] ==
'\r') &&
250 EndPos[-1] != EndPos[0])
260 SubMods.push_back(&Mod);
265 void Preprocessor::diagnoseMissingHeaderInUmbrellaDir(
const Module &Mod) {
268 SourceMgr.getLocForStartOfFile(SourceMgr.getMainFileID());
269 if (getDiagnostics().isIgnored(diag::warn_uncovered_module_header, StartLoc))
272 ModuleMap &ModMap = getHeaderSearchInfo().getModuleMap();
274 llvm::vfs::FileSystem &FS = *FileMgr.getVirtualFileSystem();
276 for (llvm::vfs::recursive_directory_iterator Entry(FS, Dir->
getName(), EC),
278 Entry !=
End && !EC; Entry.increment(EC)) {
279 using llvm::StringSwitch;
283 if (!StringSwitch<bool>(llvm::sys::path::extension(Entry->path()))
284 .Cases(
".h",
".H",
".hh",
".hpp",
true)
288 if (
const FileEntry *Header = getFileManager().getFile(Entry->path()))
289 if (!getSourceManager().hasFileInfo(Header)) {
294 Diag(StartLoc, diag::warn_uncovered_module_header)
305 assert(!CurTokenLexer &&
306 "Ending a file when currently in a macro!");
310 const bool LeavingSubmodule = CurLexer && CurLexerSubmodule;
311 if ((LeavingSubmodule || IncludeMacroStack.empty()) &&
312 !BuildingSubmoduleStack.empty() &&
313 BuildingSubmoduleStack.back().IsPragma) {
314 Diag(BuildingSubmoduleStack.back().ImportLoc,
315 diag::err_pp_module_begin_without_module_end);
316 Module *M = LeaveSubmodule(
true);
319 const char *EndPos = getCurLexerEndPos();
320 CurLexer->BufferPtr = EndPos;
321 CurLexer->FormTokenWithChars(Result, EndPos, tok::annot_module_end);
330 CurPPLexer->MIOpt.GetControllingMacroAtEndOfFile()) {
332 if (
const FileEntry *FE = CurPPLexer->getFileEntry()) {
333 HeaderInfo.SetFileControllingMacro(FE, ControllingMacro);
335 getMacroInfo(const_cast<IdentifierInfo*>(ControllingMacro)))
336 MI->setUsedForHeaderGuard(
true);
338 CurPPLexer->MIOpt.GetDefinedMacro()) {
340 DefinedMacro != ControllingMacro &&
341 HeaderInfo.FirstTimeLexingFile(FE)) {
349 const StringRef ControllingMacroName = ControllingMacro->getName();
350 const StringRef DefinedMacroName = DefinedMacro->getName();
351 const size_t MaxHalfLength =
std::max(ControllingMacroName.size(),
352 DefinedMacroName.size()) / 2;
353 const unsigned ED = ControllingMacroName.edit_distance(
354 DefinedMacroName,
true, MaxHalfLength);
355 if (ED <= MaxHalfLength) {
357 Diag(CurPPLexer->MIOpt.GetMacroLocation(),
358 diag::warn_header_guard)
359 << CurPPLexer->MIOpt.GetMacroLocation() << ControllingMacro;
360 Diag(CurPPLexer->MIOpt.GetDefinedLocation(),
361 diag::note_header_guard)
362 << CurPPLexer->MIOpt.GetDefinedLocation() << DefinedMacro
365 CurPPLexer->MIOpt.GetDefinedLocation(),
366 ControllingMacro->getName());
377 if (PragmaARCCFCodeAuditedLoc.isValid() &&
378 !isEndOfMacro && !(CurLexer && CurLexer->Is_PragmaLexer)) {
379 Diag(PragmaARCCFCodeAuditedLoc, diag::err_pp_eof_in_arc_cf_code_audited);
388 if (PragmaAssumeNonNullLoc.isValid() &&
389 !isEndOfMacro && !(CurLexer && CurLexer->Is_PragmaLexer)) {
390 Diag(PragmaAssumeNonNullLoc, diag::err_pp_eof_in_assume_nonnull);
396 bool LeavingPCHThroughHeader =
false;
400 if (!IncludeMacroStack.empty()) {
403 if (isCodeCompletionEnabled() && CurPPLexer &&
404 SourceMgr.getLocForStartOfFile(CurPPLexer->getFileID()) ==
405 CodeCompletionFileLoc) {
406 assert(CurLexer &&
"Got EOF but no current lexer set!");
408 CurLexer->FormTokenWithChars(Result, CurLexer->BufferEnd,
tok::eof);
411 CurPPLexer =
nullptr;
412 recomputeCurLexerKind();
416 if (!isEndOfMacro && CurPPLexer &&
417 SourceMgr.getIncludeLoc(CurPPLexer->getFileID()).isValid()) {
421 SourceMgr.local_sloc_entry_size() -
422 CurPPLexer->getInitialNumSLocEntries() + 1;
423 SourceMgr.setNumCreatedFIDsForFileID(CurPPLexer->getFileID(), NumFIDs);
426 bool ExitedFromPredefinesFile =
false;
428 if (!isEndOfMacro && CurPPLexer) {
429 ExitedFID = CurPPLexer->getFileID();
431 assert(PredefinesFileID.isValid() &&
432 "HandleEndOfFile is called before PredefinesFileId is set");
433 ExitedFromPredefinesFile = (PredefinesFileID == ExitedFID);
436 if (LeavingSubmodule) {
438 Module *M = LeaveSubmodule(
false);
441 const char *EndPos = getCurLexerEndPos();
443 CurLexer->BufferPtr = EndPos;
444 CurLexer->FormTokenWithChars(Result, EndPos, tok::annot_module_end);
449 bool FoundPCHThroughHeader =
false;
450 if (CurPPLexer && creatingPCHWithThroughHeader() &&
452 SourceMgr.getFileEntryForID(CurPPLexer->getFileID())))
453 FoundPCHThroughHeader =
true;
456 RemoveTopOfLexerStack();
459 PropagateLineStartLeadingSpaceInfo(Result);
462 if (Callbacks && !isEndOfMacro && CurPPLexer) {
464 SourceMgr.getFileCharacteristic(CurPPLexer->getSourceLocation());
465 Callbacks->FileChanged(CurPPLexer->getSourceLocation(),
471 if (ExitedFromPredefinesFile)
472 replayPreambleConditionalStack();
474 if (!isEndOfMacro && CurPPLexer && FoundPCHThroughHeader &&
475 (isInPrimaryFile() ||
476 CurPPLexer->getFileID() == getPredefinesFileID())) {
479 LeavingPCHThroughHeader =
true;
482 return LeavingSubmodule;
487 assert(CurLexer &&
"Got EOF but no current lexer set!");
488 const char *EndPos = getCurLexerEndPos();
490 CurLexer->BufferPtr = EndPos;
491 CurLexer->FormTokenWithChars(Result, EndPos,
tok::eof);
493 if (isCodeCompletionEnabled()) {
500 if (CurLexer->getFileLoc() == CodeCompletionFileLoc)
504 if (creatingPCHWithThroughHeader() && !LeavingPCHThroughHeader) {
506 Diag(CurLexer->getFileLoc(), diag::err_pp_through_header_not_seen)
507 << PPOpts->PCHThroughHeader << 0;
510 if (!isIncrementalProcessingEnabled())
514 if (!isIncrementalProcessingEnabled())
515 CurPPLexer =
nullptr;
521 for (WarnUnusedMacroLocsTy::iterator
522 I=WarnUnusedMacroLocs.begin(), E=WarnUnusedMacroLocs.end();
524 Diag(*I, diag::pp_macro_not_used);
531 if (
Module *Mod = getCurrentModule()) {
534 for (
auto *M : AllMods)
535 diagnoseMissingHeaderInUmbrellaDir(*M);
544 assert(CurTokenLexer && !CurPPLexer &&
545 "Ending a macro when currently in a #include file!");
547 if (!MacroExpandingLexersStack.empty() &&
548 MacroExpandingLexersStack.back().first == CurTokenLexer.get())
549 removeCachedMacroExpandedTokensOfLastLexer();
552 if (NumCachedTokenLexers == TokenLexerCacheSize)
553 CurTokenLexer.reset();
555 TokenLexerCache[NumCachedTokenLexers++] = std::move(CurTokenLexer);
558 return HandleEndOfFile(Result,
true);
565 assert(!IncludeMacroStack.empty() &&
"Ran out of stack entries to load");
569 if (NumCachedTokenLexers == TokenLexerCacheSize)
570 CurTokenLexer.reset();
572 TokenLexerCache[NumCachedTokenLexers++] = std::move(CurTokenLexer);
575 PopIncludeMacroStack();
582 assert(CurTokenLexer && !CurPPLexer &&
583 "Pasted comment can only be formed from macro");
588 bool LexerWasInPPMode =
false;
589 for (
const IncludeStackInfo &ISI : llvm::reverse(IncludeMacroStack)) {
590 if (ISI.ThePPLexer ==
nullptr)
continue;
598 FoundLexer = ISI.ThePPLexer;
608 if (!HandleEndOfTokenLexer(Tok)) Lex(Tok);
620 if (Tok.
is(tok::eod)) {
621 assert(FoundLexer &&
"Can't get end of line without an active lexer");
627 if (LexerWasInPPMode)
return;
638 assert(!FoundLexer &&
"Lexer should return EOD before EOF in PP mode");
643 if (!getLangOpts().ModulesLocalVisibility) {
645 BuildingSubmoduleStack.push_back(
646 BuildingSubmoduleInfo(M, ImportLoc, ForPragma, CurSubmoduleState,
647 PendingModuleMacroNames.size()));
655 ModuleMap &ModMap = getHeaderSearchInfo().getModuleMap();
661 auto R = Submodules.insert(std::make_pair(M, SubmoduleState()));
662 auto &
State = R.first->second;
663 bool FirstTime = R.second;
671 auto &StartingMacros = NullSubmoduleState.Macros;
675 for (
auto &Macro : StartingMacros) {
677 if (!Macro.second.getLatest() &&
678 Macro.second.getOverriddenMacros().empty())
681 MacroState MS(Macro.second.getLatest());
682 MS.setOverriddenMacros(*
this, Macro.second.getOverriddenMacros());
683 State.Macros.insert(std::make_pair(Macro.first, std::move(MS)));
688 BuildingSubmoduleStack.push_back(
689 BuildingSubmoduleInfo(M, ImportLoc, ForPragma, CurSubmoduleState,
690 PendingModuleMacroNames.size()));
693 CurSubmoduleState = &
State;
697 makeModuleVisible(M, ImportLoc);
700 bool Preprocessor::needModuleMacros()
const {
702 if (BuildingSubmoduleStack.empty())
706 if (getLangOpts().ModulesLocalVisibility)
710 return getLangOpts().isCompilingModule();
714 if (BuildingSubmoduleStack.empty() ||
715 BuildingSubmoduleStack.back().IsPragma != ForPragma) {
716 assert(ForPragma &&
"non-pragma module enter/leave mismatch");
720 auto &Info = BuildingSubmoduleStack.back();
722 Module *LeavingMod = Info.M;
725 if (!needModuleMacros() ||
726 (!getLangOpts().ModulesLocalVisibility &&
731 BuildingSubmoduleStack.pop_back();
732 makeModuleVisible(LeavingMod, ImportLoc);
737 llvm::SmallPtrSet<const IdentifierInfo*, 8> VisitedMacros;
738 for (
unsigned I = Info.OuterPendingModuleMacroNames;
739 I != PendingModuleMacroNames.size(); ++I) {
740 auto *II =
const_cast<IdentifierInfo*
>(PendingModuleMacroNames[I]);
741 if (!VisitedMacros.insert(II).second)
744 auto MacroIt = CurSubmoduleState->Macros.find(II);
745 if (MacroIt == CurSubmoduleState->Macros.end())
747 auto &Macro = MacroIt->second;
751 auto *OldState = Info.OuterSubmoduleState;
752 if (getLangOpts().ModulesLocalVisibility)
753 OldState = &NullSubmoduleState;
754 if (OldState && OldState != CurSubmoduleState) {
757 auto &OldMacros = OldState->Macros;
758 auto OldMacroIt = OldMacros.find(II);
759 if (OldMacroIt == OldMacros.end())
762 OldMD = OldMacroIt->second.getLatest();
767 bool ExplicitlyPublic =
false;
768 for (
auto *MD = Macro.getLatest(); MD != OldMD; MD = MD->
getPrevious()) {
769 assert(MD &&
"broken macro directive chain");
771 if (
auto *VisMD = dyn_cast<VisibilityMacroDirective>(MD)) {
774 if (VisMD->isPublic())
775 ExplicitlyPublic =
true;
776 else if (!ExplicitlyPublic)
782 Def = DefMD->getInfo();
789 if (Def || !Macro.getOverriddenMacros().empty())
790 addModuleMacro(LeavingMod, II, Def,
791 Macro.getOverriddenMacros(), IsNew);
793 if (!getLangOpts().ModulesLocalVisibility) {
796 Macro.setLatest(
nullptr);
797 Macro.setOverriddenMacros(*
this, {});
803 PendingModuleMacroNames.resize(Info.OuterPendingModuleMacroNames);
811 if (getLangOpts().ModulesLocalVisibility)
812 CurSubmoduleState = Info.OuterSubmoduleState;
814 BuildingSubmoduleStack.pop_back();
817 makeModuleVisible(LeavingMod, ImportLoc);
static DiagnosticBuilder Diag(DiagnosticsEngine *Diags, const LangOptions &Features, FullSourceLoc TokLoc, const char *TokBegin, const char *TokRangeBegin, const char *TokRangeEnd, unsigned DiagID)
Produce a diagnostic highlighting some portion of a literal.
Lexer - This provides a simple interface that turns a text buffer into a stream of tokens...
SourceLocation getLocWithOffset(int Offset) const
Return a source location with the specified offset from this SourceLocation.
Implements support for file system lookup, file system caching, and directory search management...
Defines the clang::FileManager interface and associated types.
bool is(tok::TokenKind K) const
is/isNot - Predicates to check if this token is a specific kind, as in "if (Tok.is(tok::l_brace)) {...
Defines the SourceManager interface.
bool isInPrimaryFile() const
Return true if we're in the top-level file, not in a #include.
Defines the clang::MacroInfo and clang::MacroDirective classes.
bool resolveUses(Module *Mod, bool Complain)
Resolve all of the unresolved uses in the given module.
CharacteristicKind
Indicates whether a file or directory holds normal user code, system code, or system code which is im...
const MacroDirective * getPrevious() const
Get previous definition of the macro with the same name.
bool HandleEndOfTokenLexer(Token &Result)
Callback invoked when the current TokenLexer hits the end of its token stream.
One of these records is kept for each identifier that is lexed.
A directive for a defined macro or a macro imported from a module.
bool ParsingPreprocessorDirective
True when parsing #XXX; turns '\n' into a tok::eod token.
bool resolveConflicts(Module *Mod, bool Complain)
Resolve all of the unresolved conflicts in the given module.
void EnterSubmodule(Module *M, SourceLocation ImportLoc, bool ForPragma)
Module * LeaveSubmodule(bool ForPragma)
Header getUmbrellaHeader() const
Retrieve the header that serves as the umbrella header for this module.
Token - This structure provides full information about a lexed token.
Describes a module or submodule.
StringRef getTopLevelModuleName() const
Retrieve the name of the top-level module.
std::string getFullModuleName(bool AllowStringLiterals=false) const
Retrieve the full name of this module, including the path from its top-level module.
const DirectoryEntry * getDirectory(StringRef DirName, bool CacheFailure=true)
Lookup, cache, and verify the specified directory (real or virtual).
const DirectoryEntry * Entry
void setAnnotationValue(void *val)
bool LexingRawMode
True if in raw mode.
MacroArgs - An instance of this class captures information about the formal arguments specified to a ...
SourceLocation getLocation() const
Return a source location identifier for the specified offset in the current file. ...
Defines the clang::Preprocessor interface.
void HandleMicrosoftCommentPaste(Token &Tok)
When the macro expander pastes together a comment (/##/) in Microsoft mode, this method handles updat...
bool resolveExports(Module *Mod, bool Complain)
Resolve all of the unresolved exports in the given module.
static void computeRelativePath(FileManager &FM, const DirectoryEntry *Dir, const FileEntry *File, SmallString< 128 > &Result)
Compute the relative path that names the given file relative to the given directory.
bool HandleEndOfFile(Token &Result, bool isEndOfMacro=false)
Callback invoked when the lexer hits the end of the current file.
The result type of a method or function.
DirectoryLookup - This class represents one entry in the search list that specifies the search order ...
const DirectoryEntry * getDir() const
Return the directory the file lives in.
Encapsulates changes to the "macros namespace" (the location where the macro name became active...
Encodes a location in the source.
StringRef getName() const
void setAnnotationEndLoc(SourceLocation L)
Cached information about one file (either on disk or in the virtual file system). ...
An opaque identifier used by SourceManager which refers to a source file (MemoryBuffer) along with it...
bool isNot(tok::TokenKind K) const
bool EnterSourceFile(FileID FID, const DirectoryLookup *Dir, SourceLocation Loc)
Add a source file to the top of the include stack and start lexing tokens from it instead of the curr...
Dataflow Directional Tag Classes.
llvm::iterator_range< submodule_iterator > submodules()
Encapsulates the data about a macro definition (e.g.
DirectoryName getUmbrellaDir() const
Retrieve the directory for which this module serves as the umbrella.
Cached information about one directory (either on disk or in the virtual file system).
static void collectAllSubModulesWithUmbrellaHeader(const Module &Mod, SmallVectorImpl< const Module *> &SubMods)
static bool isMacroDefined(const Sema &S, SourceLocation Loc, StringRef Name)
void RemoveTopOfLexerStack()
Pop the current lexer/macro exp off the top of the lexer stack.
static FixItHint CreateReplacement(CharSourceRange RemoveRange, StringRef Code)
Create a code modification hint that replaces the given source range with the given code string...
The translation unit is a complete translation unit.
bool isHeaderInUnavailableModule(const FileEntry *Header) const
Determine whether the given header is part of a module marked 'unavailable'.
__DEVICE__ int max(int __a, int __b)
void EnterMacro(Token &Tok, SourceLocation ILEnd, MacroInfo *Macro, MacroArgs *Args)
Add a Macro to the top of the include stack and start lexing tokens from it instead of the current bu...
void setLocation(SourceLocation L)
PreprocessorLexer * getCurrentFileLexer() const
Return the current file lexer being lexed from.
void startToken()
Reset all flags to cleared.
StringRef getName() const