clang  10.0.0git
Preprocessor.h
Go to the documentation of this file.
1 //===- Preprocessor.h - C Language Family Preprocessor ----------*- C++ -*-===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 //
9 /// \file
10 /// Defines the clang::Preprocessor interface.
11 //
12 //===----------------------------------------------------------------------===//
13 
14 #ifndef LLVM_CLANG_LEX_PREPROCESSOR_H
15 #define LLVM_CLANG_LEX_PREPROCESSOR_H
16 
17 #include "clang/Basic/Diagnostic.h"
19 #include "clang/Basic/LLVM.h"
21 #include "clang/Basic/Module.h"
24 #include "clang/Basic/TokenKinds.h"
25 #include "clang/Lex/Lexer.h"
26 #include "clang/Lex/MacroInfo.h"
27 #include "clang/Lex/ModuleLoader.h"
28 #include "clang/Lex/ModuleMap.h"
29 #include "clang/Lex/PPCallbacks.h"
31 #include "clang/Lex/Token.h"
32 #include "clang/Lex/TokenLexer.h"
33 #include "llvm/ADT/ArrayRef.h"
34 #include "llvm/ADT/DenseMap.h"
35 #include "llvm/ADT/FoldingSet.h"
36 #include "llvm/ADT/FunctionExtras.h"
37 #include "llvm/ADT/None.h"
38 #include "llvm/ADT/Optional.h"
39 #include "llvm/ADT/PointerUnion.h"
40 #include "llvm/ADT/STLExtras.h"
41 #include "llvm/ADT/SmallPtrSet.h"
42 #include "llvm/ADT/SmallVector.h"
43 #include "llvm/ADT/StringRef.h"
44 #include "llvm/ADT/TinyPtrVector.h"
45 #include "llvm/ADT/iterator_range.h"
46 #include "llvm/Support/Allocator.h"
47 #include "llvm/Support/Casting.h"
48 #include "llvm/Support/Registry.h"
49 #include <cassert>
50 #include <cstddef>
51 #include <cstdint>
52 #include <map>
53 #include <memory>
54 #include <string>
55 #include <utility>
56 #include <vector>
57 
58 namespace llvm {
59 
60 template<unsigned InternalLen> class SmallString;
61 
62 } // namespace llvm
63 
64 namespace clang {
65 
66 class CodeCompletionHandler;
67 class CommentHandler;
68 class DirectoryEntry;
69 class DirectoryLookup;
70 class ExternalPreprocessorSource;
71 class FileEntry;
72 class FileManager;
73 class HeaderSearch;
74 class MacroArgs;
75 class PragmaHandler;
76 class PragmaNamespace;
77 class PreprocessingRecord;
78 class PreprocessorLexer;
79 class PreprocessorOptions;
80 class ScratchBuffer;
81 class TargetInfo;
82 
83 namespace Builtin {
84 class Context;
85 }
86 
87 /// Stores token information for comparing actual tokens with
88 /// predefined values. Only handles simple tokens and identifiers.
89 class TokenValue {
91  IdentifierInfo *II;
92 
93 public:
94  TokenValue(tok::TokenKind Kind) : Kind(Kind), II(nullptr) {
95  assert(Kind != tok::raw_identifier && "Raw identifiers are not supported.");
96  assert(Kind != tok::identifier &&
97  "Identifiers should be created by TokenValue(IdentifierInfo *)");
98  assert(!tok::isLiteral(Kind) && "Literals are not supported.");
99  assert(!tok::isAnnotation(Kind) && "Annotations are not supported.");
100  }
101 
102  TokenValue(IdentifierInfo *II) : Kind(tok::identifier), II(II) {}
103 
104  bool operator==(const Token &Tok) const {
105  return Tok.getKind() == Kind &&
106  (!II || II == Tok.getIdentifierInfo());
107  }
108 };
109 
110 /// Context in which macro name is used.
111 enum MacroUse {
112  // other than #define or #undef
113  MU_Other = 0,
114 
115  // macro name specified in #define
117 
118  // macro name specified in #undef
120 };
121 
122 /// Engages in a tight little dance with the lexer to efficiently
123 /// preprocess tokens.
124 ///
125 /// Lexers know only about tokens within a single source file, and don't
126 /// know anything about preprocessor-level issues like the \#include stack,
127 /// token expansion, etc.
131 
132  llvm::unique_function<void(const clang::Token &)> OnToken;
133  std::shared_ptr<PreprocessorOptions> PPOpts;
134  DiagnosticsEngine *Diags;
135  LangOptions &LangOpts;
136  const TargetInfo *Target = nullptr;
137  const TargetInfo *AuxTarget = nullptr;
138  FileManager &FileMgr;
139  SourceManager &SourceMgr;
140  std::unique_ptr<ScratchBuffer> ScratchBuf;
141  HeaderSearch &HeaderInfo;
142  ModuleLoader &TheModuleLoader;
143 
144  /// External source of macros.
145  ExternalPreprocessorSource *ExternalSource;
146 
147  /// A BumpPtrAllocator object used to quickly allocate and release
148  /// objects internal to the Preprocessor.
149  llvm::BumpPtrAllocator BP;
150 
151  /// Identifiers for builtin macros and other builtins.
152  IdentifierInfo *Ident__LINE__, *Ident__FILE__; // __LINE__, __FILE__
153  IdentifierInfo *Ident__DATE__, *Ident__TIME__; // __DATE__, __TIME__
154  IdentifierInfo *Ident__INCLUDE_LEVEL__; // __INCLUDE_LEVEL__
155  IdentifierInfo *Ident__BASE_FILE__; // __BASE_FILE__
156  IdentifierInfo *Ident__FILE_NAME__; // __FILE_NAME__
157  IdentifierInfo *Ident__TIMESTAMP__; // __TIMESTAMP__
158  IdentifierInfo *Ident__COUNTER__; // __COUNTER__
159  IdentifierInfo *Ident_Pragma, *Ident__pragma; // _Pragma, __pragma
160  IdentifierInfo *Ident__identifier; // __identifier
161  IdentifierInfo *Ident__VA_ARGS__; // __VA_ARGS__
162  IdentifierInfo *Ident__VA_OPT__; // __VA_OPT__
163  IdentifierInfo *Ident__has_feature; // __has_feature
164  IdentifierInfo *Ident__has_extension; // __has_extension
165  IdentifierInfo *Ident__has_builtin; // __has_builtin
166  IdentifierInfo *Ident__has_attribute; // __has_attribute
167  IdentifierInfo *Ident__has_include; // __has_include
168  IdentifierInfo *Ident__has_include_next; // __has_include_next
169  IdentifierInfo *Ident__has_warning; // __has_warning
170  IdentifierInfo *Ident__is_identifier; // __is_identifier
171  IdentifierInfo *Ident__building_module; // __building_module
172  IdentifierInfo *Ident__MODULE__; // __MODULE__
173  IdentifierInfo *Ident__has_cpp_attribute; // __has_cpp_attribute
174  IdentifierInfo *Ident__has_c_attribute; // __has_c_attribute
175  IdentifierInfo *Ident__has_declspec; // __has_declspec_attribute
176  IdentifierInfo *Ident__is_target_arch; // __is_target_arch
177  IdentifierInfo *Ident__is_target_vendor; // __is_target_vendor
178  IdentifierInfo *Ident__is_target_os; // __is_target_os
179  IdentifierInfo *Ident__is_target_environment; // __is_target_environment
180 
181  // Weak, only valid (and set) while InMacroArgs is true.
182  Token* ArgMacro;
183 
184  SourceLocation DATELoc, TIMELoc;
185 
186  // Next __COUNTER__ value, starts at 0.
187  unsigned CounterValue = 0;
188 
189  enum {
190  /// Maximum depth of \#includes.
191  MaxAllowedIncludeStackDepth = 200
192  };
193 
194  // State that is set before the preprocessor begins.
195  bool KeepComments : 1;
196  bool KeepMacroComments : 1;
197  bool SuppressIncludeNotFoundError : 1;
198 
199  // State that changes while the preprocessor runs:
200  bool InMacroArgs : 1; // True if parsing fn macro invocation args.
201 
202  /// Whether the preprocessor owns the header search object.
203  bool OwnsHeaderSearch : 1;
204 
205  /// True if macro expansion is disabled.
206  bool DisableMacroExpansion : 1;
207 
208  /// Temporarily disables DisableMacroExpansion (i.e. enables expansion)
209  /// when parsing preprocessor directives.
210  bool MacroExpansionInDirectivesOverride : 1;
211 
213 
214  /// Whether we have already loaded macros from the external source.
215  mutable bool ReadMacrosFromExternalSource : 1;
216 
217  /// True if pragmas are enabled.
218  bool PragmasEnabled : 1;
219 
220  /// True if the current build action is a preprocessing action.
221  bool PreprocessedOutput : 1;
222 
223  /// True if we are currently preprocessing a #if or #elif directive
224  bool ParsingIfOrElifDirective;
225 
226  /// True if we are pre-expanding macro arguments.
227  bool InMacroArgPreExpansion;
228 
229  /// Mapping/lookup information for all identifiers in
230  /// the program, including program keywords.
231  mutable IdentifierTable Identifiers;
232 
233  /// This table contains all the selectors in the program.
234  ///
235  /// Unlike IdentifierTable above, this table *isn't* populated by the
236  /// preprocessor. It is declared/expanded here because its role/lifetime is
237  /// conceptually similar to the IdentifierTable. In addition, the current
238  /// control flow (in clang::ParseAST()), make it convenient to put here.
239  ///
240  /// FIXME: Make sure the lifetime of Identifiers/Selectors *isn't* tied to
241  /// the lifetime of the preprocessor.
242  SelectorTable Selectors;
243 
244  /// Information about builtins.
245  std::unique_ptr<Builtin::Context> BuiltinInfo;
246 
247  /// Tracks all of the pragmas that the client registered
248  /// with this preprocessor.
249  std::unique_ptr<PragmaNamespace> PragmaHandlers;
250 
251  /// Pragma handlers of the original source is stored here during the
252  /// parsing of a model file.
253  std::unique_ptr<PragmaNamespace> PragmaHandlersBackup;
254 
255  /// Tracks all of the comment handlers that the client registered
256  /// with this preprocessor.
257  std::vector<CommentHandler *> CommentHandlers;
258 
259  /// True if we want to ignore EOF token and continue later on (thus
260  /// avoid tearing the Lexer and etc. down).
261  bool IncrementalProcessing = false;
262 
263  /// The kind of translation unit we are processing.
264  TranslationUnitKind TUKind;
265 
266  /// The code-completion handler.
267  CodeCompletionHandler *CodeComplete = nullptr;
268 
269  /// The file that we're performing code-completion for, if any.
270  const FileEntry *CodeCompletionFile = nullptr;
271 
272  /// The offset in file for the code-completion point.
273  unsigned CodeCompletionOffset = 0;
274 
275  /// The location for the code-completion point. This gets instantiated
276  /// when the CodeCompletionFile gets \#include'ed for preprocessing.
277  SourceLocation CodeCompletionLoc;
278 
279  /// The start location for the file of the code-completion point.
280  ///
281  /// This gets instantiated when the CodeCompletionFile gets \#include'ed
282  /// for preprocessing.
283  SourceLocation CodeCompletionFileLoc;
284 
285  /// The source location of the \c import contextual keyword we just
286  /// lexed, if any.
287  SourceLocation ModuleImportLoc;
288 
289  /// The module import path that we're currently processing.
291 
292  /// Whether the last token we lexed was an '@'.
293  bool LastTokenWasAt = false;
294 
295  /// A position within a C++20 import-seq.
296  class ImportSeq {
297  public:
298  enum State : int {
299  // Positive values represent a number of unclosed brackets.
300  AtTopLevel = 0,
301  AfterTopLevelTokenSeq = -1,
302  AfterExport = -2,
303  AfterImportSeq = -3,
304  };
305 
306  ImportSeq(State S) : S(S) {}
307 
308  /// Saw any kind of open bracket.
309  void handleOpenBracket() {
310  S = static_cast<State>(std::max<int>(S, 0) + 1);
311  }
312  /// Saw any kind of close bracket other than '}'.
313  void handleCloseBracket() {
314  S = static_cast<State>(std::max<int>(S, 1) - 1);
315  }
316  /// Saw a close brace.
317  void handleCloseBrace() {
318  handleCloseBracket();
319  if (S == AtTopLevel && !AfterHeaderName)
320  S = AfterTopLevelTokenSeq;
321  }
322  /// Saw a semicolon.
323  void handleSemi() {
324  if (atTopLevel()) {
325  S = AfterTopLevelTokenSeq;
326  AfterHeaderName = false;
327  }
328  }
329 
330  /// Saw an 'export' identifier.
331  void handleExport() {
332  if (S == AfterTopLevelTokenSeq)
333  S = AfterExport;
334  else if (S <= 0)
335  S = AtTopLevel;
336  }
337  /// Saw an 'import' identifier.
338  void handleImport() {
339  if (S == AfterTopLevelTokenSeq || S == AfterExport)
340  S = AfterImportSeq;
341  else if (S <= 0)
342  S = AtTopLevel;
343  }
344 
345  /// Saw a 'header-name' token; do not recognize any more 'import' tokens
346  /// until we reach a top-level semicolon.
347  void handleHeaderName() {
348  if (S == AfterImportSeq)
349  AfterHeaderName = true;
350  handleMisc();
351  }
352 
353  /// Saw any other token.
354  void handleMisc() {
355  if (S <= 0)
356  S = AtTopLevel;
357  }
358 
359  bool atTopLevel() { return S <= 0; }
360  bool afterImportSeq() { return S == AfterImportSeq; }
361 
362  private:
363  State S;
364  /// Whether we're in the pp-import-suffix following the header-name in a
365  /// pp-import. If so, a close-brace is not sufficient to end the
366  /// top-level-token-seq of an import-seq.
367  bool AfterHeaderName = false;
368  };
369 
370  /// Our current position within a C++20 import-seq.
371  ImportSeq ImportSeqState = ImportSeq::AfterTopLevelTokenSeq;
372 
373  /// Whether the module import expects an identifier next. Otherwise,
374  /// it expects a '.' or ';'.
375  bool ModuleImportExpectsIdentifier = false;
376 
377  /// The identifier and source location of the currently-active
378  /// \#pragma clang arc_cf_code_audited begin.
379  std::pair<IdentifierInfo *, SourceLocation> PragmaARCCFCodeAuditedInfo;
380 
381  /// The source location of the currently-active
382  /// \#pragma clang assume_nonnull begin.
383  SourceLocation PragmaAssumeNonNullLoc;
384 
385  /// True if we hit the code-completion point.
386  bool CodeCompletionReached = false;
387 
388  /// The code completion token containing the information
389  /// on the stem that is to be code completed.
390  IdentifierInfo *CodeCompletionII = nullptr;
391 
392  /// Range for the code completion token.
393  SourceRange CodeCompletionTokenRange;
394 
395  /// The directory that the main file should be considered to occupy,
396  /// if it does not correspond to a real file (as happens when building a
397  /// module).
398  const DirectoryEntry *MainFileDir = nullptr;
399 
400  /// The number of bytes that we will initially skip when entering the
401  /// main file, along with a flag that indicates whether skipping this number
402  /// of bytes will place the lexer at the start of a line.
403  ///
404  /// This is used when loading a precompiled preamble.
405  std::pair<int, bool> SkipMainFilePreamble;
406 
407  /// Whether we hit an error due to reaching max allowed include depth. Allows
408  /// to avoid hitting the same error over and over again.
409  bool HasReachedMaxIncludeDepth = false;
410 
411  /// The number of currently-active calls to Lex.
412  ///
413  /// Lex is reentrant, and asking for an (end-of-phase-4) token can often
414  /// require asking for multiple additional tokens. This counter makes it
415  /// possible for Lex to detect whether it's producing a token for the end
416  /// of phase 4 of translation or for some other situation.
417  unsigned LexLevel = 0;
418 
419 public:
424  bool FoundElse;
426 
428  bool FoundNonSkipPortion, bool FoundElse,
429  SourceLocation ElseLoc)
430  : HashTokenLoc(HashTokenLoc), IfTokenLoc(IfTokenLoc),
431  FoundNonSkipPortion(FoundNonSkipPortion), FoundElse(FoundElse),
432  ElseLoc(ElseLoc) {}
433  };
434 
435 private:
436  friend class ASTReader;
437  friend class MacroArgs;
438 
439  class PreambleConditionalStackStore {
440  enum State {
441  Off = 0,
442  Recording = 1,
443  Replaying = 2,
444  };
445 
446  public:
447  PreambleConditionalStackStore() = default;
448 
449  void startRecording() { ConditionalStackState = Recording; }
450  void startReplaying() { ConditionalStackState = Replaying; }
451  bool isRecording() const { return ConditionalStackState == Recording; }
452  bool isReplaying() const { return ConditionalStackState == Replaying; }
453 
454  ArrayRef<PPConditionalInfo> getStack() const {
455  return ConditionalStack;
456  }
457 
458  void doneReplaying() {
459  ConditionalStack.clear();
460  ConditionalStackState = Off;
461  }
462 
463  void setStack(ArrayRef<PPConditionalInfo> s) {
464  if (!isRecording() && !isReplaying())
465  return;
466  ConditionalStack.clear();
467  ConditionalStack.append(s.begin(), s.end());
468  }
469 
470  bool hasRecordedPreamble() const { return !ConditionalStack.empty(); }
471 
472  bool reachedEOFWhileSkipping() const { return SkipInfo.hasValue(); }
473 
474  void clearSkipInfo() { SkipInfo.reset(); }
475 
477 
478  private:
479  SmallVector<PPConditionalInfo, 4> ConditionalStack;
480  State ConditionalStackState = Off;
481  } PreambleConditionalStack;
482 
483  /// The current top of the stack that we're lexing from if
484  /// not expanding a macro and we are lexing directly from source code.
485  ///
486  /// Only one of CurLexer, or CurTokenLexer will be non-null.
487  std::unique_ptr<Lexer> CurLexer;
488 
489  /// The current top of the stack what we're lexing from
490  /// if not expanding a macro.
491  ///
492  /// This is an alias for CurLexer.
493  PreprocessorLexer *CurPPLexer = nullptr;
494 
495  /// Used to find the current FileEntry, if CurLexer is non-null
496  /// and if applicable.
497  ///
498  /// This allows us to implement \#include_next and find directory-specific
499  /// properties.
500  const DirectoryLookup *CurDirLookup = nullptr;
501 
502  /// The current macro we are expanding, if we are expanding a macro.
503  ///
504  /// One of CurLexer and CurTokenLexer must be null.
505  std::unique_ptr<TokenLexer> CurTokenLexer;
506 
507  /// The kind of lexer we're currently working with.
508  enum CurLexerKind {
509  CLK_Lexer,
510  CLK_TokenLexer,
511  CLK_CachingLexer,
512  CLK_LexAfterModuleImport
513  } CurLexerKind = CLK_Lexer;
514 
515  /// If the current lexer is for a submodule that is being built, this
516  /// is that submodule.
517  Module *CurLexerSubmodule = nullptr;
518 
519  /// Keeps track of the stack of files currently
520  /// \#included, and macros currently being expanded from, not counting
521  /// CurLexer/CurTokenLexer.
522  struct IncludeStackInfo {
523  enum CurLexerKind CurLexerKind;
524  Module *TheSubmodule;
525  std::unique_ptr<Lexer> TheLexer;
526  PreprocessorLexer *ThePPLexer;
527  std::unique_ptr<TokenLexer> TheTokenLexer;
528  const DirectoryLookup *TheDirLookup;
529 
530  // The following constructors are completely useless copies of the default
531  // versions, only needed to pacify MSVC.
532  IncludeStackInfo(enum CurLexerKind CurLexerKind, Module *TheSubmodule,
533  std::unique_ptr<Lexer> &&TheLexer,
534  PreprocessorLexer *ThePPLexer,
535  std::unique_ptr<TokenLexer> &&TheTokenLexer,
536  const DirectoryLookup *TheDirLookup)
537  : CurLexerKind(std::move(CurLexerKind)),
538  TheSubmodule(std::move(TheSubmodule)), TheLexer(std::move(TheLexer)),
539  ThePPLexer(std::move(ThePPLexer)),
540  TheTokenLexer(std::move(TheTokenLexer)),
541  TheDirLookup(std::move(TheDirLookup)) {}
542  };
543  std::vector<IncludeStackInfo> IncludeMacroStack;
544 
545  /// Actions invoked when some preprocessor activity is
546  /// encountered (e.g. a file is \#included, etc).
547  std::unique_ptr<PPCallbacks> Callbacks;
548 
549  struct MacroExpandsInfo {
550  Token Tok;
551  MacroDefinition MD;
552  SourceRange Range;
553 
554  MacroExpandsInfo(Token Tok, MacroDefinition MD, SourceRange Range)
555  : Tok(Tok), MD(MD), Range(Range) {}
556  };
557  SmallVector<MacroExpandsInfo, 2> DelayedMacroExpandsCallbacks;
558 
559  /// Information about a name that has been used to define a module macro.
560  struct ModuleMacroInfo {
561  /// The most recent macro directive for this identifier.
562  MacroDirective *MD;
563 
564  /// The active module macros for this identifier.
565  llvm::TinyPtrVector<ModuleMacro *> ActiveModuleMacros;
566 
567  /// The generation number at which we last updated ActiveModuleMacros.
568  /// \see Preprocessor::VisibleModules.
569  unsigned ActiveModuleMacrosGeneration = 0;
570 
571  /// Whether this macro name is ambiguous.
572  bool IsAmbiguous = false;
573 
574  /// The module macros that are overridden by this macro.
575  llvm::TinyPtrVector<ModuleMacro *> OverriddenMacros;
576 
577  ModuleMacroInfo(MacroDirective *MD) : MD(MD) {}
578  };
579 
580  /// The state of a macro for an identifier.
581  class MacroState {
582  mutable llvm::PointerUnion<MacroDirective *, ModuleMacroInfo *> State;
583 
584  ModuleMacroInfo *getModuleInfo(Preprocessor &PP,
585  const IdentifierInfo *II) const {
586  if (II->isOutOfDate())
587  PP.updateOutOfDateIdentifier(const_cast<IdentifierInfo&>(*II));
588  // FIXME: Find a spare bit on IdentifierInfo and store a
589  // HasModuleMacros flag.
590  if (!II->hasMacroDefinition() ||
591  (!PP.getLangOpts().Modules &&
592  !PP.getLangOpts().ModulesLocalVisibility) ||
593  !PP.CurSubmoduleState->VisibleModules.getGeneration())
594  return nullptr;
595 
596  auto *Info = State.dyn_cast<ModuleMacroInfo*>();
597  if (!Info) {
598  Info = new (PP.getPreprocessorAllocator())
599  ModuleMacroInfo(State.get<MacroDirective *>());
600  State = Info;
601  }
602 
603  if (PP.CurSubmoduleState->VisibleModules.getGeneration() !=
604  Info->ActiveModuleMacrosGeneration)
605  PP.updateModuleMacroInfo(II, *Info);
606  return Info;
607  }
608 
609  public:
610  MacroState() : MacroState(nullptr) {}
611  MacroState(MacroDirective *MD) : State(MD) {}
612 
613  MacroState(MacroState &&O) noexcept : State(O.State) {
614  O.State = (MacroDirective *)nullptr;
615  }
616 
617  MacroState &operator=(MacroState &&O) noexcept {
618  auto S = O.State;
619  O.State = (MacroDirective *)nullptr;
620  State = S;
621  return *this;
622  }
623 
624  ~MacroState() {
625  if (auto *Info = State.dyn_cast<ModuleMacroInfo*>())
626  Info->~ModuleMacroInfo();
627  }
628 
629  MacroDirective *getLatest() const {
630  if (auto *Info = State.dyn_cast<ModuleMacroInfo*>())
631  return Info->MD;
632  return State.get<MacroDirective*>();
633  }
634 
635  void setLatest(MacroDirective *MD) {
636  if (auto *Info = State.dyn_cast<ModuleMacroInfo*>())
637  Info->MD = MD;
638  else
639  State = MD;
640  }
641 
642  bool isAmbiguous(Preprocessor &PP, const IdentifierInfo *II) const {
643  auto *Info = getModuleInfo(PP, II);
644  return Info ? Info->IsAmbiguous : false;
645  }
646 
648  getActiveModuleMacros(Preprocessor &PP, const IdentifierInfo *II) const {
649  if (auto *Info = getModuleInfo(PP, II))
650  return Info->ActiveModuleMacros;
651  return None;
652  }
653 
654  MacroDirective::DefInfo findDirectiveAtLoc(SourceLocation Loc,
655  SourceManager &SourceMgr) const {
656  // FIXME: Incorporate module macros into the result of this.
657  if (auto *Latest = getLatest())
658  return Latest->findDirectiveAtLoc(Loc, SourceMgr);
659  return {};
660  }
661 
662  void overrideActiveModuleMacros(Preprocessor &PP, IdentifierInfo *II) {
663  if (auto *Info = getModuleInfo(PP, II)) {
664  Info->OverriddenMacros.insert(Info->OverriddenMacros.end(),
665  Info->ActiveModuleMacros.begin(),
666  Info->ActiveModuleMacros.end());
667  Info->ActiveModuleMacros.clear();
668  Info->IsAmbiguous = false;
669  }
670  }
671 
672  ArrayRef<ModuleMacro*> getOverriddenMacros() const {
673  if (auto *Info = State.dyn_cast<ModuleMacroInfo*>())
674  return Info->OverriddenMacros;
675  return None;
676  }
677 
678  void setOverriddenMacros(Preprocessor &PP,
679  ArrayRef<ModuleMacro *> Overrides) {
680  auto *Info = State.dyn_cast<ModuleMacroInfo*>();
681  if (!Info) {
682  if (Overrides.empty())
683  return;
684  Info = new (PP.getPreprocessorAllocator())
685  ModuleMacroInfo(State.get<MacroDirective *>());
686  State = Info;
687  }
688  Info->OverriddenMacros.clear();
689  Info->OverriddenMacros.insert(Info->OverriddenMacros.end(),
690  Overrides.begin(), Overrides.end());
691  Info->ActiveModuleMacrosGeneration = 0;
692  }
693  };
694 
695  /// For each IdentifierInfo that was associated with a macro, we
696  /// keep a mapping to the history of all macro definitions and #undefs in
697  /// the reverse order (the latest one is in the head of the list).
698  ///
699  /// This mapping lives within the \p CurSubmoduleState.
700  using MacroMap = llvm::DenseMap<const IdentifierInfo *, MacroState>;
701 
702  struct SubmoduleState;
703 
704  /// Information about a submodule that we're currently building.
705  struct BuildingSubmoduleInfo {
706  /// The module that we are building.
707  Module *M;
708 
709  /// The location at which the module was included.
710  SourceLocation ImportLoc;
711 
712  /// Whether we entered this submodule via a pragma.
713  bool IsPragma;
714 
715  /// The previous SubmoduleState.
716  SubmoduleState *OuterSubmoduleState;
717 
718  /// The number of pending module macro names when we started building this.
719  unsigned OuterPendingModuleMacroNames;
720 
721  BuildingSubmoduleInfo(Module *M, SourceLocation ImportLoc, bool IsPragma,
722  SubmoduleState *OuterSubmoduleState,
723  unsigned OuterPendingModuleMacroNames)
724  : M(M), ImportLoc(ImportLoc), IsPragma(IsPragma),
725  OuterSubmoduleState(OuterSubmoduleState),
726  OuterPendingModuleMacroNames(OuterPendingModuleMacroNames) {}
727  };
728  SmallVector<BuildingSubmoduleInfo, 8> BuildingSubmoduleStack;
729 
730  /// Information about a submodule's preprocessor state.
731  struct SubmoduleState {
732  /// The macros for the submodule.
733  MacroMap Macros;
734 
735  /// The set of modules that are visible within the submodule.
736  VisibleModuleSet VisibleModules;
737 
738  // FIXME: CounterValue?
739  // FIXME: PragmaPushMacroInfo?
740  };
741  std::map<Module *, SubmoduleState> Submodules;
742 
743  /// The preprocessor state for preprocessing outside of any submodule.
744  SubmoduleState NullSubmoduleState;
745 
746  /// The current submodule state. Will be \p NullSubmoduleState if we're not
747  /// in a submodule.
748  SubmoduleState *CurSubmoduleState;
749 
750  /// The set of known macros exported from modules.
751  llvm::FoldingSet<ModuleMacro> ModuleMacros;
752 
753  /// The names of potential module macros that we've not yet processed.
754  llvm::SmallVector<const IdentifierInfo *, 32> PendingModuleMacroNames;
755 
756  /// The list of module macros, for each identifier, that are not overridden by
757  /// any other module macro.
758  llvm::DenseMap<const IdentifierInfo *, llvm::TinyPtrVector<ModuleMacro *>>
759  LeafModuleMacros;
760 
761  /// Macros that we want to warn because they are not used at the end
762  /// of the translation unit.
763  ///
764  /// We store just their SourceLocations instead of
765  /// something like MacroInfo*. The benefit of this is that when we are
766  /// deserializing from PCH, we don't need to deserialize identifier & macros
767  /// just so that we can report that they are unused, we just warn using
768  /// the SourceLocations of this set (that will be filled by the ASTReader).
769  /// We are using SmallPtrSet instead of a vector for faster removal.
770  using WarnUnusedMacroLocsTy = llvm::SmallPtrSet<SourceLocation, 32>;
771  WarnUnusedMacroLocsTy WarnUnusedMacroLocs;
772 
773  /// A "freelist" of MacroArg objects that can be
774  /// reused for quick allocation.
775  MacroArgs *MacroArgCache = nullptr;
776 
777  /// For each IdentifierInfo used in a \#pragma push_macro directive,
778  /// we keep a MacroInfo stack used to restore the previous macro value.
779  llvm::DenseMap<IdentifierInfo *, std::vector<MacroInfo *>>
780  PragmaPushMacroInfo;
781 
782  // Various statistics we track for performance analysis.
783  unsigned NumDirectives = 0;
784  unsigned NumDefined = 0;
785  unsigned NumUndefined = 0;
786  unsigned NumPragma = 0;
787  unsigned NumIf = 0;
788  unsigned NumElse = 0;
789  unsigned NumEndif = 0;
790  unsigned NumEnteredSourceFiles = 0;
791  unsigned MaxIncludeStackDepth = 0;
792  unsigned NumMacroExpanded = 0;
793  unsigned NumFnMacroExpanded = 0;
794  unsigned NumBuiltinMacroExpanded = 0;
795  unsigned NumFastMacroExpanded = 0;
796  unsigned NumTokenPaste = 0;
797  unsigned NumFastTokenPaste = 0;
798  unsigned NumSkipped = 0;
799 
800  /// The predefined macros that preprocessor should use from the
801  /// command line etc.
802  std::string Predefines;
803 
804  /// The file ID for the preprocessor predefines.
805  FileID PredefinesFileID;
806 
807  /// The file ID for the PCH through header.
808  FileID PCHThroughHeaderFileID;
809 
810  /// Whether tokens are being skipped until a #pragma hdrstop is seen.
811  bool SkippingUntilPragmaHdrStop = false;
812 
813  /// Whether tokens are being skipped until the through header is seen.
814  bool SkippingUntilPCHThroughHeader = false;
815 
816  /// \{
817  /// Cache of macro expanders to reduce malloc traffic.
818  enum { TokenLexerCacheSize = 8 };
819  unsigned NumCachedTokenLexers;
820  std::unique_ptr<TokenLexer> TokenLexerCache[TokenLexerCacheSize];
821  /// \}
822 
823  /// Keeps macro expanded tokens for TokenLexers.
824  //
825  /// Works like a stack; a TokenLexer adds the macro expanded tokens that is
826  /// going to lex in the cache and when it finishes the tokens are removed
827  /// from the end of the cache.
828  SmallVector<Token, 16> MacroExpandedTokens;
829  std::vector<std::pair<TokenLexer *, size_t>> MacroExpandingLexersStack;
830 
831  /// A record of the macro definitions and expansions that
832  /// occurred during preprocessing.
833  ///
834  /// This is an optional side structure that can be enabled with
835  /// \c createPreprocessingRecord() prior to preprocessing.
836  PreprocessingRecord *Record = nullptr;
837 
838  /// Cached tokens state.
840 
841  /// Cached tokens are stored here when we do backtracking or
842  /// lookahead. They are "lexed" by the CachingLex() method.
844 
845  /// The position of the cached token that CachingLex() should
846  /// "lex" next.
847  ///
848  /// If it points beyond the CachedTokens vector, it means that a normal
849  /// Lex() should be invoked.
850  CachedTokensTy::size_type CachedLexPos = 0;
851 
852  /// Stack of backtrack positions, allowing nested backtracks.
853  ///
854  /// The EnableBacktrackAtThisPos() method pushes a position to
855  /// indicate where CachedLexPos should be set when the BackTrack() method is
856  /// invoked (at which point the last position is popped).
857  std::vector<CachedTokensTy::size_type> BacktrackPositions;
858 
859  struct MacroInfoChain {
860  MacroInfo MI;
861  MacroInfoChain *Next;
862  };
863 
864  /// MacroInfos are managed as a chain for easy disposal. This is the head
865  /// of that list.
866  MacroInfoChain *MIChainHead = nullptr;
867 
868  void updateOutOfDateIdentifier(IdentifierInfo &II) const;
869 
870 public:
871  Preprocessor(std::shared_ptr<PreprocessorOptions> PPOpts,
873  HeaderSearch &Headers, ModuleLoader &TheModuleLoader,
874  IdentifierInfoLookup *IILookup = nullptr,
875  bool OwnsHeaderSearch = false,
877 
878  ~Preprocessor();
879 
880  /// Initialize the preprocessor using information about the target.
881  ///
882  /// \param Target is owned by the caller and must remain valid for the
883  /// lifetime of the preprocessor.
884  /// \param AuxTarget is owned by the caller and must remain valid for
885  /// the lifetime of the preprocessor.
886  void Initialize(const TargetInfo &Target,
887  const TargetInfo *AuxTarget = nullptr);
888 
889  /// Initialize the preprocessor to parse a model file
890  ///
891  /// To parse model files the preprocessor of the original source is reused to
892  /// preserver the identifier table. However to avoid some duplicate
893  /// information in the preprocessor some cleanup is needed before it is used
894  /// to parse model files. This method does that cleanup.
895  void InitializeForModelFile();
896 
897  /// Cleanup after model file parsing
898  void FinalizeForModelFile();
899 
900  /// Retrieve the preprocessor options used to initialize this
901  /// preprocessor.
902  PreprocessorOptions &getPreprocessorOpts() const { return *PPOpts; }
903 
904  DiagnosticsEngine &getDiagnostics() const { return *Diags; }
905  void setDiagnostics(DiagnosticsEngine &D) { Diags = &D; }
906 
907  const LangOptions &getLangOpts() const { return LangOpts; }
908  const TargetInfo &getTargetInfo() const { return *Target; }
909  const TargetInfo *getAuxTargetInfo() const { return AuxTarget; }
910  FileManager &getFileManager() const { return FileMgr; }
911  SourceManager &getSourceManager() const { return SourceMgr; }
912  HeaderSearch &getHeaderSearchInfo() const { return HeaderInfo; }
913 
914  IdentifierTable &getIdentifierTable() { return Identifiers; }
915  const IdentifierTable &getIdentifierTable() const { return Identifiers; }
916  SelectorTable &getSelectorTable() { return Selectors; }
918  llvm::BumpPtrAllocator &getPreprocessorAllocator() { return BP; }
919 
921  ExternalSource = Source;
922  }
923 
925  return ExternalSource;
926  }
927 
928  /// Retrieve the module loader associated with this preprocessor.
929  ModuleLoader &getModuleLoader() const { return TheModuleLoader; }
930 
932  return TheModuleLoader.HadFatalFailure;
933  }
934 
935  /// Retrieve the number of Directives that have been processed by the
936  /// Preprocessor.
937  unsigned getNumDirectives() const {
938  return NumDirectives;
939  }
940 
941  /// True if we are currently preprocessing a #if or #elif directive
943  return ParsingIfOrElifDirective;
944  }
945 
946  /// Control whether the preprocessor retains comments in output.
947  void SetCommentRetentionState(bool KeepComments, bool KeepMacroComments) {
948  this->KeepComments = KeepComments | KeepMacroComments;
949  this->KeepMacroComments = KeepMacroComments;
950  }
951 
952  bool getCommentRetentionState() const { return KeepComments; }
953 
954  void setPragmasEnabled(bool Enabled) { PragmasEnabled = Enabled; }
955  bool getPragmasEnabled() const { return PragmasEnabled; }
956 
957  void SetSuppressIncludeNotFoundError(bool Suppress) {
958  SuppressIncludeNotFoundError = Suppress;
959  }
960 
962  return SuppressIncludeNotFoundError;
963  }
964 
965  /// Sets whether the preprocessor is responsible for producing output or if
966  /// it is producing tokens to be consumed by Parse and Sema.
967  void setPreprocessedOutput(bool IsPreprocessedOutput) {
968  PreprocessedOutput = IsPreprocessedOutput;
969  }
970 
971  /// Returns true if the preprocessor is responsible for generating output,
972  /// false if it is producing tokens to be consumed by Parse and Sema.
973  bool isPreprocessedOutput() const { return PreprocessedOutput; }
974 
975  /// Return true if we are lexing directly from the specified lexer.
976  bool isCurrentLexer(const PreprocessorLexer *L) const {
977  return CurPPLexer == L;
978  }
979 
980  /// Return the current lexer being lexed from.
981  ///
982  /// Note that this ignores any potentially active macro expansions and _Pragma
983  /// expansions going on at the time.
984  PreprocessorLexer *getCurrentLexer() const { return CurPPLexer; }
985 
986  /// Return the current file lexer being lexed from.
987  ///
988  /// Note that this ignores any potentially active macro expansions and _Pragma
989  /// expansions going on at the time.
990  PreprocessorLexer *getCurrentFileLexer() const;
991 
992  /// Return the submodule owning the file being lexed. This may not be
993  /// the current module if we have changed modules since entering the file.
994  Module *getCurrentLexerSubmodule() const { return CurLexerSubmodule; }
995 
996  /// Returns the FileID for the preprocessor predefines.
997  FileID getPredefinesFileID() const { return PredefinesFileID; }
998 
999  /// \{
1000  /// Accessors for preprocessor callbacks.
1001  ///
1002  /// Note that this class takes ownership of any PPCallbacks object given to
1003  /// it.
1004  PPCallbacks *getPPCallbacks() const { return Callbacks.get(); }
1005  void addPPCallbacks(std::unique_ptr<PPCallbacks> C) {
1006  if (Callbacks)
1007  C = std::make_unique<PPChainedCallbacks>(std::move(C),
1008  std::move(Callbacks));
1009  Callbacks = std::move(C);
1010  }
1011  /// \}
1012 
1013  /// Register a function that would be called on each token in the final
1014  /// expanded token stream.
1015  /// This also reports annotation tokens produced by the parser.
1016  void setTokenWatcher(llvm::unique_function<void(const clang::Token &)> F) {
1017  OnToken = std::move(F);
1018  }
1019 
1020  bool isMacroDefined(StringRef Id) {
1021  return isMacroDefined(&Identifiers.get(Id));
1022  }
1023  bool isMacroDefined(const IdentifierInfo *II) {
1024  return II->hasMacroDefinition() &&
1025  (!getLangOpts().Modules || (bool)getMacroDefinition(II));
1026  }
1027 
1028  /// Determine whether II is defined as a macro within the module M,
1029  /// if that is a module that we've already preprocessed. Does not check for
1030  /// macros imported into M.
1032  if (!II->hasMacroDefinition())
1033  return false;
1034  auto I = Submodules.find(M);
1035  if (I == Submodules.end())
1036  return false;
1037  auto J = I->second.Macros.find(II);
1038  if (J == I->second.Macros.end())
1039  return false;
1040  auto *MD = J->second.getLatest();
1041  return MD && MD->isDefined();
1042  }
1043 
1045  if (!II->hasMacroDefinition())
1046  return {};
1047 
1048  MacroState &S = CurSubmoduleState->Macros[II];
1049  auto *MD = S.getLatest();
1050  while (MD && isa<VisibilityMacroDirective>(MD))
1051  MD = MD->getPrevious();
1052  return MacroDefinition(dyn_cast_or_null<DefMacroDirective>(MD),
1053  S.getActiveModuleMacros(*this, II),
1054  S.isAmbiguous(*this, II));
1055  }
1056 
1058  SourceLocation Loc) {
1059  if (!II->hadMacroDefinition())
1060  return {};
1061 
1062  MacroState &S = CurSubmoduleState->Macros[II];
1064  if (auto *MD = S.getLatest())
1065  DI = MD->findDirectiveAtLoc(Loc, getSourceManager());
1066  // FIXME: Compute the set of active module macros at the specified location.
1067  return MacroDefinition(DI.getDirective(),
1068  S.getActiveModuleMacros(*this, II),
1069  S.isAmbiguous(*this, II));
1070  }
1071 
1072  /// Given an identifier, return its latest non-imported MacroDirective
1073  /// if it is \#define'd and not \#undef'd, or null if it isn't \#define'd.
1075  if (!II->hasMacroDefinition())
1076  return nullptr;
1077 
1078  auto *MD = getLocalMacroDirectiveHistory(II);
1079  if (!MD || MD->getDefinition().isUndefined())
1080  return nullptr;
1081 
1082  return MD;
1083  }
1084 
1085  const MacroInfo *getMacroInfo(const IdentifierInfo *II) const {
1086  return const_cast<Preprocessor*>(this)->getMacroInfo(II);
1087  }
1088 
1090  if (!II->hasMacroDefinition())
1091  return nullptr;
1092  if (auto MD = getMacroDefinition(II))
1093  return MD.getMacroInfo();
1094  return nullptr;
1095  }
1096 
1097  /// Given an identifier, return the latest non-imported macro
1098  /// directive for that identifier.
1099  ///
1100  /// One can iterate over all previous macro directives from the most recent
1101  /// one.
1102  MacroDirective *getLocalMacroDirectiveHistory(const IdentifierInfo *II) const;
1103 
1104  /// Add a directive to the macro directive history for this identifier.
1105  void appendMacroDirective(IdentifierInfo *II, MacroDirective *MD);
1107  SourceLocation Loc) {
1108  DefMacroDirective *MD = AllocateDefMacroDirective(MI, Loc);
1109  appendMacroDirective(II, MD);
1110  return MD;
1111  }
1113  MacroInfo *MI) {
1114  return appendDefMacroDirective(II, MI, MI->getDefinitionLoc());
1115  }
1116 
1117  /// Set a MacroDirective that was loaded from a PCH file.
1118  void setLoadedMacroDirective(IdentifierInfo *II, MacroDirective *ED,
1119  MacroDirective *MD);
1120 
1121  /// Register an exported macro for a module and identifier.
1122  ModuleMacro *addModuleMacro(Module *Mod, IdentifierInfo *II, MacroInfo *Macro,
1123  ArrayRef<ModuleMacro *> Overrides, bool &IsNew);
1124  ModuleMacro *getModuleMacro(Module *Mod, IdentifierInfo *II);
1125 
1126  /// Get the list of leaf (non-overridden) module macros for a name.
1128  if (II->isOutOfDate())
1129  updateOutOfDateIdentifier(const_cast<IdentifierInfo&>(*II));
1130  auto I = LeafModuleMacros.find(II);
1131  if (I != LeafModuleMacros.end())
1132  return I->second;
1133  return None;
1134  }
1135 
1136  /// \{
1137  /// Iterators for the macro history table. Currently defined macros have
1138  /// IdentifierInfo::hasMacroDefinition() set and an empty
1139  /// MacroInfo::getUndefLoc() at the head of the list.
1140  using macro_iterator = MacroMap::const_iterator;
1141 
1142  macro_iterator macro_begin(bool IncludeExternalMacros = true) const;
1143  macro_iterator macro_end(bool IncludeExternalMacros = true) const;
1144 
1145  llvm::iterator_range<macro_iterator>
1146  macros(bool IncludeExternalMacros = true) const {
1147  macro_iterator begin = macro_begin(IncludeExternalMacros);
1148  macro_iterator end = macro_end(IncludeExternalMacros);
1149  return llvm::make_range(begin, end);
1150  }
1151 
1152  /// \}
1153 
1154  /// Return the name of the macro defined before \p Loc that has
1155  /// spelling \p Tokens. If there are multiple macros with same spelling,
1156  /// return the last one defined.
1157  StringRef getLastMacroWithSpelling(SourceLocation Loc,
1158  ArrayRef<TokenValue> Tokens) const;
1159 
1160  const std::string &getPredefines() const { return Predefines; }
1161 
1162  /// Set the predefines for this Preprocessor.
1163  ///
1164  /// These predefines are automatically injected when parsing the main file.
1165  void setPredefines(const char *P) { Predefines = P; }
1166  void setPredefines(StringRef P) { Predefines = P; }
1167 
1168  /// Return information about the specified preprocessor
1169  /// identifier token.
1170  IdentifierInfo *getIdentifierInfo(StringRef Name) const {
1171  return &Identifiers.get(Name);
1172  }
1173 
1174  /// Add the specified pragma handler to this preprocessor.
1175  ///
1176  /// If \p Namespace is non-null, then it is a token required to exist on the
1177  /// pragma line before the pragma string starts, e.g. "STDC" or "GCC".
1178  void AddPragmaHandler(StringRef Namespace, PragmaHandler *Handler);
1180  AddPragmaHandler(StringRef(), Handler);
1181  }
1182 
1183  /// Remove the specific pragma handler from this preprocessor.
1184  ///
1185  /// If \p Namespace is non-null, then it should be the namespace that
1186  /// \p Handler was added to. It is an error to remove a handler that
1187  /// has not been registered.
1188  void RemovePragmaHandler(StringRef Namespace, PragmaHandler *Handler);
1190  RemovePragmaHandler(StringRef(), Handler);
1191  }
1192 
1193  /// Install empty handlers for all pragmas (making them ignored).
1194  void IgnorePragmas();
1195 
1196  /// Add the specified comment handler to the preprocessor.
1197  void addCommentHandler(CommentHandler *Handler);
1198 
1199  /// Remove the specified comment handler.
1200  ///
1201  /// It is an error to remove a handler that has not been registered.
1202  void removeCommentHandler(CommentHandler *Handler);
1203 
1204  /// Set the code completion handler to the given object.
1206  CodeComplete = &Handler;
1207  }
1208 
1209  /// Retrieve the current code-completion handler.
1211  return CodeComplete;
1212  }
1213 
1214  /// Clear out the code completion handler.
1216  CodeComplete = nullptr;
1217  }
1218 
1219  /// Hook used by the lexer to invoke the "included file" code
1220  /// completion point.
1221  void CodeCompleteIncludedFile(llvm::StringRef Dir, bool IsAngled);
1222 
1223  /// Hook used by the lexer to invoke the "natural language" code
1224  /// completion point.
1225  void CodeCompleteNaturalLanguage();
1226 
1227  /// Set the code completion token for filtering purposes.
1229  CodeCompletionII = Filter;
1230  }
1231 
1232  /// Set the code completion token range for detecting replacement range later
1233  /// on.
1235  const SourceLocation End) {
1236  CodeCompletionTokenRange = {Start, End};
1237  }
1239  return CodeCompletionTokenRange;
1240  }
1241 
1242  /// Get the code completion token for filtering purposes.
1244  if (CodeCompletionII)
1245  return CodeCompletionII->getName();
1246  return {};
1247  }
1248 
1249  /// Retrieve the preprocessing record, or NULL if there is no
1250  /// preprocessing record.
1251  PreprocessingRecord *getPreprocessingRecord() const { return Record; }
1252 
1253  /// Create a new preprocessing record, which will keep track of
1254  /// all macro expansions, macro definitions, etc.
1255  void createPreprocessingRecord();
1256 
1257  /// Returns true if the FileEntry is the PCH through header.
1258  bool isPCHThroughHeader(const FileEntry *FE);
1259 
1260  /// True if creating a PCH with a through header.
1261  bool creatingPCHWithThroughHeader();
1262 
1263  /// True if using a PCH with a through header.
1264  bool usingPCHWithThroughHeader();
1265 
1266  /// True if creating a PCH with a #pragma hdrstop.
1267  bool creatingPCHWithPragmaHdrStop();
1268 
1269  /// True if using a PCH with a #pragma hdrstop.
1270  bool usingPCHWithPragmaHdrStop();
1271 
1272  /// Skip tokens until after the #include of the through header or
1273  /// until after a #pragma hdrstop.
1274  void SkipTokensWhileUsingPCH();
1275 
1276  /// Process directives while skipping until the through header or
1277  /// #pragma hdrstop is found.
1278  void HandleSkippedDirectiveWhileUsingPCH(Token &Result,
1279  SourceLocation HashLoc);
1280 
1281  /// Enter the specified FileID as the main source file,
1282  /// which implicitly adds the builtin defines etc.
1283  void EnterMainSourceFile();
1284 
1285  /// Inform the preprocessor callbacks that processing is complete.
1286  void EndSourceFile();
1287 
1288  /// Add a source file to the top of the include stack and
1289  /// start lexing tokens from it instead of the current buffer.
1290  ///
1291  /// Emits a diagnostic, doesn't enter the file, and returns true on error.
1292  bool EnterSourceFile(FileID FID, const DirectoryLookup *Dir,
1293  SourceLocation Loc);
1294 
1295  /// Add a Macro to the top of the include stack and start lexing
1296  /// tokens from it instead of the current buffer.
1297  ///
1298  /// \param Args specifies the tokens input to a function-like macro.
1299  /// \param ILEnd specifies the location of the ')' for a function-like macro
1300  /// or the identifier for an object-like macro.
1301  void EnterMacro(Token &Tok, SourceLocation ILEnd, MacroInfo *Macro,
1302  MacroArgs *Args);
1303 
1304 private:
1305  /// Add a "macro" context to the top of the include stack,
1306  /// which will cause the lexer to start returning the specified tokens.
1307  ///
1308  /// If \p DisableMacroExpansion is true, tokens lexed from the token stream
1309  /// will not be subject to further macro expansion. Otherwise, these tokens
1310  /// will be re-macro-expanded when/if expansion is enabled.
1311  ///
1312  /// If \p OwnsTokens is false, this method assumes that the specified stream
1313  /// of tokens has a permanent owner somewhere, so they do not need to be
1314  /// copied. If it is true, it assumes the array of tokens is allocated with
1315  /// \c new[] and the Preprocessor will delete[] it.
1316  ///
1317  /// If \p IsReinject the resulting tokens will have Token::IsReinjected flag
1318  /// set, see the flag documentation for details.
1319  void EnterTokenStream(const Token *Toks, unsigned NumToks,
1320  bool DisableMacroExpansion, bool OwnsTokens,
1321  bool IsReinject);
1322 
1323 public:
1324  void EnterTokenStream(std::unique_ptr<Token[]> Toks, unsigned NumToks,
1325  bool DisableMacroExpansion, bool IsReinject) {
1326  EnterTokenStream(Toks.release(), NumToks, DisableMacroExpansion, true,
1327  IsReinject);
1328  }
1329 
1330  void EnterTokenStream(ArrayRef<Token> Toks, bool DisableMacroExpansion,
1331  bool IsReinject) {
1332  EnterTokenStream(Toks.data(), Toks.size(), DisableMacroExpansion, false,
1333  IsReinject);
1334  }
1335 
1336  /// Pop the current lexer/macro exp off the top of the lexer stack.
1337  ///
1338  /// This should only be used in situations where the current state of the
1339  /// top-of-stack lexer is known.
1340  void RemoveTopOfLexerStack();
1341 
1342  /// From the point that this method is called, and until
1343  /// CommitBacktrackedTokens() or Backtrack() is called, the Preprocessor
1344  /// keeps track of the lexed tokens so that a subsequent Backtrack() call will
1345  /// make the Preprocessor re-lex the same tokens.
1346  ///
1347  /// Nested backtracks are allowed, meaning that EnableBacktrackAtThisPos can
1348  /// be called multiple times and CommitBacktrackedTokens/Backtrack calls will
1349  /// be combined with the EnableBacktrackAtThisPos calls in reverse order.
1350  ///
1351  /// NOTE: *DO NOT* forget to call either CommitBacktrackedTokens or Backtrack
1352  /// at some point after EnableBacktrackAtThisPos. If you don't, caching of
1353  /// tokens will continue indefinitely.
1354  ///
1355  void EnableBacktrackAtThisPos();
1356 
1357  /// Disable the last EnableBacktrackAtThisPos call.
1358  void CommitBacktrackedTokens();
1359 
1360  /// Make Preprocessor re-lex the tokens that were lexed since
1361  /// EnableBacktrackAtThisPos() was previously called.
1362  void Backtrack();
1363 
1364  /// True if EnableBacktrackAtThisPos() was called and
1365  /// caching of tokens is on.
1366  bool isBacktrackEnabled() const { return !BacktrackPositions.empty(); }
1367 
1368  /// Lex the next token for this preprocessor.
1369  void Lex(Token &Result);
1370 
1371  /// Lex a token, forming a header-name token if possible.
1372  bool LexHeaderName(Token &Result, bool AllowMacroExpansion = true);
1373 
1374  bool LexAfterModuleImport(Token &Result);
1375  void CollectPpImportSuffix(SmallVectorImpl<Token> &Toks);
1376 
1377  void makeModuleVisible(Module *M, SourceLocation Loc);
1378 
1380  return CurSubmoduleState->VisibleModules.getImportLoc(M);
1381  }
1382 
1383  /// Lex a string literal, which may be the concatenation of multiple
1384  /// string literals and may even come from macro expansion.
1385  /// \returns true on success, false if a error diagnostic has been generated.
1386  bool LexStringLiteral(Token &Result, std::string &String,
1387  const char *DiagnosticTag, bool AllowMacroExpansion) {
1388  if (AllowMacroExpansion)
1389  Lex(Result);
1390  else
1391  LexUnexpandedToken(Result);
1392  return FinishLexStringLiteral(Result, String, DiagnosticTag,
1393  AllowMacroExpansion);
1394  }
1395 
1396  /// Complete the lexing of a string literal where the first token has
1397  /// already been lexed (see LexStringLiteral).
1398  bool FinishLexStringLiteral(Token &Result, std::string &String,
1399  const char *DiagnosticTag,
1400  bool AllowMacroExpansion);
1401 
1402  /// Lex a token. If it's a comment, keep lexing until we get
1403  /// something not a comment.
1404  ///
1405  /// This is useful in -E -C mode where comments would foul up preprocessor
1406  /// directive handling.
1407  void LexNonComment(Token &Result) {
1408  do
1409  Lex(Result);
1410  while (Result.getKind() == tok::comment);
1411  }
1412 
1413  /// Just like Lex, but disables macro expansion of identifier tokens.
1414  void LexUnexpandedToken(Token &Result) {
1415  // Disable macro expansion.
1416  bool OldVal = DisableMacroExpansion;
1417  DisableMacroExpansion = true;
1418  // Lex the token.
1419  Lex(Result);
1420 
1421  // Reenable it.
1422  DisableMacroExpansion = OldVal;
1423  }
1424 
1425  /// Like LexNonComment, but this disables macro expansion of
1426  /// identifier tokens.
1428  do
1429  LexUnexpandedToken(Result);
1430  while (Result.getKind() == tok::comment);
1431  }
1432 
1433  /// Parses a simple integer literal to get its numeric value. Floating
1434  /// point literals and user defined literals are rejected. Used primarily to
1435  /// handle pragmas that accept integer arguments.
1436  bool parseSimpleIntegerLiteral(Token &Tok, uint64_t &Value);
1437 
1438  /// Disables macro expansion everywhere except for preprocessor directives.
1440  DisableMacroExpansion = true;
1441  MacroExpansionInDirectivesOverride = true;
1442  }
1443 
1444  /// Peeks ahead N tokens and returns that token without consuming any
1445  /// tokens.
1446  ///
1447  /// LookAhead(0) returns the next token that would be returned by Lex(),
1448  /// LookAhead(1) returns the token after it, etc. This returns normal
1449  /// tokens after phase 5. As such, it is equivalent to using
1450  /// 'Lex', not 'LexUnexpandedToken'.
1451  const Token &LookAhead(unsigned N) {
1452  assert(LexLevel == 0 && "cannot use lookahead while lexing");
1453  if (CachedLexPos + N < CachedTokens.size())
1454  return CachedTokens[CachedLexPos+N];
1455  else
1456  return PeekAhead(N+1);
1457  }
1458 
1459  /// When backtracking is enabled and tokens are cached,
1460  /// this allows to revert a specific number of tokens.
1461  ///
1462  /// Note that the number of tokens being reverted should be up to the last
1463  /// backtrack position, not more.
1464  void RevertCachedTokens(unsigned N) {
1465  assert(isBacktrackEnabled() &&
1466  "Should only be called when tokens are cached for backtracking");
1467  assert(signed(CachedLexPos) - signed(N) >= signed(BacktrackPositions.back())
1468  && "Should revert tokens up to the last backtrack position, not more");
1469  assert(signed(CachedLexPos) - signed(N) >= 0 &&
1470  "Corrupted backtrack positions ?");
1471  CachedLexPos -= N;
1472  }
1473 
1474  /// Enters a token in the token stream to be lexed next.
1475  ///
1476  /// If BackTrack() is called afterwards, the token will remain at the
1477  /// insertion point.
1478  /// If \p IsReinject is true, resulting token will have Token::IsReinjected
1479  /// flag set. See the flag documentation for details.
1480  void EnterToken(const Token &Tok, bool IsReinject) {
1481  if (LexLevel) {
1482  // It's not correct in general to enter caching lex mode while in the
1483  // middle of a nested lexing action.
1484  auto TokCopy = std::make_unique<Token[]>(1);
1485  TokCopy[0] = Tok;
1486  EnterTokenStream(std::move(TokCopy), 1, true, IsReinject);
1487  } else {
1488  EnterCachingLexMode();
1489  assert(IsReinject && "new tokens in the middle of cached stream");
1490  CachedTokens.insert(CachedTokens.begin()+CachedLexPos, Tok);
1491  }
1492  }
1493 
1494  /// We notify the Preprocessor that if it is caching tokens (because
1495  /// backtrack is enabled) it should replace the most recent cached tokens
1496  /// with the given annotation token. This function has no effect if
1497  /// backtracking is not enabled.
1498  ///
1499  /// Note that the use of this function is just for optimization, so that the
1500  /// cached tokens doesn't get re-parsed and re-resolved after a backtrack is
1501  /// invoked.
1502  void AnnotateCachedTokens(const Token &Tok) {
1503  assert(Tok.isAnnotation() && "Expected annotation token");
1504  if (CachedLexPos != 0 && isBacktrackEnabled())
1505  AnnotatePreviousCachedTokens(Tok);
1506  }
1507 
1508  /// Get the location of the last cached token, suitable for setting the end
1509  /// location of an annotation token.
1511  assert(CachedLexPos != 0);
1512  return CachedTokens[CachedLexPos-1].getLastLoc();
1513  }
1514 
1515  /// Whether \p Tok is the most recent token (`CachedLexPos - 1`) in
1516  /// CachedTokens.
1517  bool IsPreviousCachedToken(const Token &Tok) const;
1518 
1519  /// Replace token in `CachedLexPos - 1` in CachedTokens by the tokens
1520  /// in \p NewToks.
1521  ///
1522  /// Useful when a token needs to be split in smaller ones and CachedTokens
1523  /// most recent token must to be updated to reflect that.
1524  void ReplacePreviousCachedToken(ArrayRef<Token> NewToks);
1525 
1526  /// Replace the last token with an annotation token.
1527  ///
1528  /// Like AnnotateCachedTokens(), this routine replaces an
1529  /// already-parsed (and resolved) token with an annotation
1530  /// token. However, this routine only replaces the last token with
1531  /// the annotation token; it does not affect any other cached
1532  /// tokens. This function has no effect if backtracking is not
1533  /// enabled.
1535  assert(Tok.isAnnotation() && "Expected annotation token");
1536  if (CachedLexPos != 0 && isBacktrackEnabled())
1537  CachedTokens[CachedLexPos-1] = Tok;
1538  }
1539 
1540  /// Enter an annotation token into the token stream.
1541  void EnterAnnotationToken(SourceRange Range, tok::TokenKind Kind,
1542  void *AnnotationVal);
1543 
1544  /// Update the current token to represent the provided
1545  /// identifier, in order to cache an action performed by typo correction.
1546  void TypoCorrectToken(const Token &Tok) {
1547  assert(Tok.getIdentifierInfo() && "Expected identifier token");
1548  if (CachedLexPos != 0 && isBacktrackEnabled())
1549  CachedTokens[CachedLexPos-1] = Tok;
1550  }
1551 
1552  /// Recompute the current lexer kind based on the CurLexer/
1553  /// CurTokenLexer pointers.
1554  void recomputeCurLexerKind();
1555 
1556  /// Returns true if incremental processing is enabled
1557  bool isIncrementalProcessingEnabled() const { return IncrementalProcessing; }
1558 
1559  /// Enables the incremental processing
1560  void enableIncrementalProcessing(bool value = true) {
1561  IncrementalProcessing = value;
1562  }
1563 
1564  /// Specify the point at which code-completion will be performed.
1565  ///
1566  /// \param File the file in which code completion should occur. If
1567  /// this file is included multiple times, code-completion will
1568  /// perform completion the first time it is included. If NULL, this
1569  /// function clears out the code-completion point.
1570  ///
1571  /// \param Line the line at which code completion should occur
1572  /// (1-based).
1573  ///
1574  /// \param Column the column at which code completion should occur
1575  /// (1-based).
1576  ///
1577  /// \returns true if an error occurred, false otherwise.
1578  bool SetCodeCompletionPoint(const FileEntry *File,
1579  unsigned Line, unsigned Column);
1580 
1581  /// Determine if we are performing code completion.
1582  bool isCodeCompletionEnabled() const { return CodeCompletionFile != nullptr; }
1583 
1584  /// Returns the location of the code-completion point.
1585  ///
1586  /// Returns an invalid location if code-completion is not enabled or the file
1587  /// containing the code-completion point has not been lexed yet.
1588  SourceLocation getCodeCompletionLoc() const { return CodeCompletionLoc; }
1589 
1590  /// Returns the start location of the file of code-completion point.
1591  ///
1592  /// Returns an invalid location if code-completion is not enabled or the file
1593  /// containing the code-completion point has not been lexed yet.
1595  return CodeCompletionFileLoc;
1596  }
1597 
1598  /// Returns true if code-completion is enabled and we have hit the
1599  /// code-completion point.
1600  bool isCodeCompletionReached() const { return CodeCompletionReached; }
1601 
1602  /// Note that we hit the code-completion point.
1604  assert(isCodeCompletionEnabled() && "Code-completion not enabled!");
1605  CodeCompletionReached = true;
1606  // Silence any diagnostics that occur after we hit the code-completion.
1607  getDiagnostics().setSuppressAllDiagnostics(true);
1608  }
1609 
1610  /// The location of the currently-active \#pragma clang
1611  /// arc_cf_code_audited begin.
1612  ///
1613  /// Returns an invalid location if there is no such pragma active.
1614  std::pair<IdentifierInfo *, SourceLocation>
1616  return PragmaARCCFCodeAuditedInfo;
1617  }
1618 
1619  /// Set the location of the currently-active \#pragma clang
1620  /// arc_cf_code_audited begin. An invalid location ends the pragma.
1622  SourceLocation Loc) {
1623  PragmaARCCFCodeAuditedInfo = {Ident, Loc};
1624  }
1625 
1626  /// The location of the currently-active \#pragma clang
1627  /// assume_nonnull begin.
1628  ///
1629  /// Returns an invalid location if there is no such pragma active.
1631  return PragmaAssumeNonNullLoc;
1632  }
1633 
1634  /// Set the location of the currently-active \#pragma clang
1635  /// assume_nonnull begin. An invalid location ends the pragma.
1637  PragmaAssumeNonNullLoc = Loc;
1638  }
1639 
1640  /// Set the directory in which the main file should be considered
1641  /// to have been found, if it is not a real file.
1642  void setMainFileDir(const DirectoryEntry *Dir) {
1643  MainFileDir = Dir;
1644  }
1645 
1646  /// Instruct the preprocessor to skip part of the main source file.
1647  ///
1648  /// \param Bytes The number of bytes in the preamble to skip.
1649  ///
1650  /// \param StartOfLine Whether skipping these bytes puts the lexer at the
1651  /// start of a line.
1652  void setSkipMainFilePreamble(unsigned Bytes, bool StartOfLine) {
1653  SkipMainFilePreamble.first = Bytes;
1654  SkipMainFilePreamble.second = StartOfLine;
1655  }
1656 
1657  /// Forwarding function for diagnostics. This emits a diagnostic at
1658  /// the specified Token's location, translating the token's start
1659  /// position in the current buffer into a SourcePosition object for rendering.
1660  DiagnosticBuilder Diag(SourceLocation Loc, unsigned DiagID) const {
1661  return Diags->Report(Loc, DiagID);
1662  }
1663 
1664  DiagnosticBuilder Diag(const Token &Tok, unsigned DiagID) const {
1665  return Diags->Report(Tok.getLocation(), DiagID);
1666  }
1667 
1668  /// Return the 'spelling' of the token at the given
1669  /// location; does not go up to the spelling location or down to the
1670  /// expansion location.
1671  ///
1672  /// \param buffer A buffer which will be used only if the token requires
1673  /// "cleaning", e.g. if it contains trigraphs or escaped newlines
1674  /// \param invalid If non-null, will be set \c true if an error occurs.
1676  SmallVectorImpl<char> &buffer,
1677  bool *invalid = nullptr) const {
1678  return Lexer::getSpelling(loc, buffer, SourceMgr, LangOpts, invalid);
1679  }
1680 
1681  /// Return the 'spelling' of the Tok token.
1682  ///
1683  /// The spelling of a token is the characters used to represent the token in
1684  /// the source file after trigraph expansion and escaped-newline folding. In
1685  /// particular, this wants to get the true, uncanonicalized, spelling of
1686  /// things like digraphs, UCNs, etc.
1687  ///
1688  /// \param Invalid If non-null, will be set \c true if an error occurs.
1689  std::string getSpelling(const Token &Tok, bool *Invalid = nullptr) const {
1690  return Lexer::getSpelling(Tok, SourceMgr, LangOpts, Invalid);
1691  }
1692 
1693  /// Get the spelling of a token into a preallocated buffer, instead
1694  /// of as an std::string.
1695  ///
1696  /// The caller is required to allocate enough space for the token, which is
1697  /// guaranteed to be at least Tok.getLength() bytes long. The length of the
1698  /// actual result is returned.
1699  ///
1700  /// Note that this method may do two possible things: it may either fill in
1701  /// the buffer specified with characters, or it may *change the input pointer*
1702  /// to point to a constant buffer with the data already in it (avoiding a
1703  /// copy). The caller is not allowed to modify the returned buffer pointer
1704  /// if an internal buffer is returned.
1705  unsigned getSpelling(const Token &Tok, const char *&Buffer,
1706  bool *Invalid = nullptr) const {
1707  return Lexer::getSpelling(Tok, Buffer, SourceMgr, LangOpts, Invalid);
1708  }
1709 
1710  /// Get the spelling of a token into a SmallVector.
1711  ///
1712  /// Note that the returned StringRef may not point to the
1713  /// supplied buffer if a copy can be avoided.
1714  StringRef getSpelling(const Token &Tok,
1715  SmallVectorImpl<char> &Buffer,
1716  bool *Invalid = nullptr) const;
1717 
1718  /// Relex the token at the specified location.
1719  /// \returns true if there was a failure, false on success.
1720  bool getRawToken(SourceLocation Loc, Token &Result,
1721  bool IgnoreWhiteSpace = false) {
1722  return Lexer::getRawToken(Loc, Result, SourceMgr, LangOpts, IgnoreWhiteSpace);
1723  }
1724 
1725  /// Given a Token \p Tok that is a numeric constant with length 1,
1726  /// return the character.
1727  char
1729  bool *Invalid = nullptr) const {
1730  assert(Tok.is(tok::numeric_constant) &&
1731  Tok.getLength() == 1 && "Called on unsupported token");
1732  assert(!Tok.needsCleaning() && "Token can't need cleaning with length 1");
1733 
1734  // If the token is carrying a literal data pointer, just use it.
1735  if (const char *D = Tok.getLiteralData())
1736  return *D;
1737 
1738  // Otherwise, fall back on getCharacterData, which is slower, but always
1739  // works.
1740  return *SourceMgr.getCharacterData(Tok.getLocation(), Invalid);
1741  }
1742 
1743  /// Retrieve the name of the immediate macro expansion.
1744  ///
1745  /// This routine starts from a source location, and finds the name of the
1746  /// macro responsible for its immediate expansion. It looks through any
1747  /// intervening macro argument expansions to compute this. It returns a
1748  /// StringRef that refers to the SourceManager-owned buffer of the source
1749  /// where that macro name is spelled. Thus, the result shouldn't out-live
1750  /// the SourceManager.
1752  return Lexer::getImmediateMacroName(Loc, SourceMgr, getLangOpts());
1753  }
1754 
1755  /// Plop the specified string into a scratch buffer and set the
1756  /// specified token's location and length to it.
1757  ///
1758  /// If specified, the source location provides a location of the expansion
1759  /// point of the token.
1760  void CreateString(StringRef Str, Token &Tok,
1761  SourceLocation ExpansionLocStart = SourceLocation(),
1762  SourceLocation ExpansionLocEnd = SourceLocation());
1763 
1764  /// Split the first Length characters out of the token starting at TokLoc
1765  /// and return a location pointing to the split token. Re-lexing from the
1766  /// split token will return the split token rather than the original.
1767  SourceLocation SplitToken(SourceLocation TokLoc, unsigned Length);
1768 
1769  /// Computes the source location just past the end of the
1770  /// token at this source location.
1771  ///
1772  /// This routine can be used to produce a source location that
1773  /// points just past the end of the token referenced by \p Loc, and
1774  /// is generally used when a diagnostic needs to point just after a
1775  /// token where it expected something different that it received. If
1776  /// the returned source location would not be meaningful (e.g., if
1777  /// it points into a macro), this routine returns an invalid
1778  /// source location.
1779  ///
1780  /// \param Offset an offset from the end of the token, where the source
1781  /// location should refer to. The default offset (0) produces a source
1782  /// location pointing just past the end of the token; an offset of 1 produces
1783  /// a source location pointing to the last character in the token, etc.
1785  return Lexer::getLocForEndOfToken(Loc, Offset, SourceMgr, LangOpts);
1786  }
1787 
1788  /// Returns true if the given MacroID location points at the first
1789  /// token of the macro expansion.
1790  ///
1791  /// \param MacroBegin If non-null and function returns true, it is set to
1792  /// begin location of the macro.
1794  SourceLocation *MacroBegin = nullptr) const {
1795  return Lexer::isAtStartOfMacroExpansion(loc, SourceMgr, LangOpts,
1796  MacroBegin);
1797  }
1798 
1799  /// Returns true if the given MacroID location points at the last
1800  /// token of the macro expansion.
1801  ///
1802  /// \param MacroEnd If non-null and function returns true, it is set to
1803  /// end location of the macro.
1805  SourceLocation *MacroEnd = nullptr) const {
1806  return Lexer::isAtEndOfMacroExpansion(loc, SourceMgr, LangOpts, MacroEnd);
1807  }
1808 
1809  /// Print the token to stderr, used for debugging.
1810  void DumpToken(const Token &Tok, bool DumpFlags = false) const;
1811  void DumpLocation(SourceLocation Loc) const;
1812  void DumpMacro(const MacroInfo &MI) const;
1813  void dumpMacroInfo(const IdentifierInfo *II);
1814 
1815  /// Given a location that specifies the start of a
1816  /// token, return a new location that specifies a character within the token.
1818  unsigned Char) const {
1819  return Lexer::AdvanceToTokenCharacter(TokStart, Char, SourceMgr, LangOpts);
1820  }
1821 
1822  /// Increment the counters for the number of token paste operations
1823  /// performed.
1824  ///
1825  /// If fast was specified, this is a 'fast paste' case we handled.
1826  void IncrementPasteCounter(bool isFast) {
1827  if (isFast)
1828  ++NumFastTokenPaste;
1829  else
1830  ++NumTokenPaste;
1831  }
1832 
1833  void PrintStats();
1834 
1835  size_t getTotalMemory() const;
1836 
1837  /// When the macro expander pastes together a comment (/##/) in Microsoft
1838  /// mode, this method handles updating the current state, returning the
1839  /// token on the next source line.
1840  void HandleMicrosoftCommentPaste(Token &Tok);
1841 
1842  //===--------------------------------------------------------------------===//
1843  // Preprocessor callback methods. These are invoked by a lexer as various
1844  // directives and events are found.
1845 
1846  /// Given a tok::raw_identifier token, look up the
1847  /// identifier information for the token and install it into the token,
1848  /// updating the token kind accordingly.
1849  IdentifierInfo *LookUpIdentifierInfo(Token &Identifier) const;
1850 
1851 private:
1852  llvm::DenseMap<IdentifierInfo*,unsigned> PoisonReasons;
1853 
1854 public:
1855  /// Specifies the reason for poisoning an identifier.
1856  ///
1857  /// If that identifier is accessed while poisoned, then this reason will be
1858  /// used instead of the default "poisoned" diagnostic.
1859  void SetPoisonReason(IdentifierInfo *II, unsigned DiagID);
1860 
1861  /// Display reason for poisoned identifier.
1862  void HandlePoisonedIdentifier(Token & Identifier);
1863 
1865  if(IdentifierInfo * II = Identifier.getIdentifierInfo()) {
1866  if(II->isPoisoned()) {
1867  HandlePoisonedIdentifier(Identifier);
1868  }
1869  }
1870  }
1871 
1872 private:
1873  /// Identifiers used for SEH handling in Borland. These are only
1874  /// allowed in particular circumstances
1875  // __except block
1876  IdentifierInfo *Ident__exception_code,
1877  *Ident___exception_code,
1878  *Ident_GetExceptionCode;
1879  // __except filter expression
1880  IdentifierInfo *Ident__exception_info,
1881  *Ident___exception_info,
1882  *Ident_GetExceptionInfo;
1883  // __finally
1884  IdentifierInfo *Ident__abnormal_termination,
1885  *Ident___abnormal_termination,
1886  *Ident_AbnormalTermination;
1887 
1888  const char *getCurLexerEndPos();
1889  void diagnoseMissingHeaderInUmbrellaDir(const Module &Mod);
1890 
1891 public:
1892  void PoisonSEHIdentifiers(bool Poison = true); // Borland
1893 
1894  /// Callback invoked when the lexer reads an identifier and has
1895  /// filled in the tokens IdentifierInfo member.
1896  ///
1897  /// This callback potentially macro expands it or turns it into a named
1898  /// token (like 'for').
1899  ///
1900  /// \returns true if we actually computed a token, false if we need to
1901  /// lex again.
1902  bool HandleIdentifier(Token &Identifier);
1903 
1904  /// Callback invoked when the lexer hits the end of the current file.
1905  ///
1906  /// This either returns the EOF token and returns true, or
1907  /// pops a level off the include stack and returns false, at which point the
1908  /// client should call lex again.
1909  bool HandleEndOfFile(Token &Result, bool isEndOfMacro = false);
1910 
1911  /// Callback invoked when the current TokenLexer hits the end of its
1912  /// token stream.
1913  bool HandleEndOfTokenLexer(Token &Result);
1914 
1915  /// Callback invoked when the lexer sees a # token at the start of a
1916  /// line.
1917  ///
1918  /// This consumes the directive, modifies the lexer/preprocessor state, and
1919  /// advances the lexer(s) so that the next token read is the correct one.
1920  void HandleDirective(Token &Result);
1921 
1922  /// Ensure that the next token is a tok::eod token.
1923  ///
1924  /// If not, emit a diagnostic and consume up until the eod.
1925  /// If \p EnableMacros is true, then we consider macros that expand to zero
1926  /// tokens as being ok.
1927  ///
1928  /// \return The location of the end of the directive (the terminating
1929  /// newline).
1930  SourceLocation CheckEndOfDirective(const char *DirType,
1931  bool EnableMacros = false);
1932 
1933  /// Read and discard all tokens remaining on the current line until
1934  /// the tok::eod token is found. Returns the range of the skipped tokens.
1935  SourceRange DiscardUntilEndOfDirective();
1936 
1937  /// Returns true if the preprocessor has seen a use of
1938  /// __DATE__ or __TIME__ in the file so far.
1939  bool SawDateOrTime() const {
1940  return DATELoc != SourceLocation() || TIMELoc != SourceLocation();
1941  }
1942  unsigned getCounterValue() const { return CounterValue; }
1943  void setCounterValue(unsigned V) { CounterValue = V; }
1944 
1945  /// Retrieves the module that we're currently building, if any.
1946  Module *getCurrentModule();
1947 
1948  /// Allocate a new MacroInfo object with the provided SourceLocation.
1949  MacroInfo *AllocateMacroInfo(SourceLocation L);
1950 
1951  /// Turn the specified lexer token into a fully checked and spelled
1952  /// filename, e.g. as an operand of \#include.
1953  ///
1954  /// The caller is expected to provide a buffer that is large enough to hold
1955  /// the spelling of the filename, but is also expected to handle the case
1956  /// when this method decides to use a different buffer.
1957  ///
1958  /// \returns true if the input filename was in <>'s or false if it was
1959  /// in ""'s.
1960  bool GetIncludeFilenameSpelling(SourceLocation Loc,StringRef &Buffer);
1961 
1962  /// Given a "foo" or <foo> reference, look up the indicated file.
1963  ///
1964  /// Returns None on failure. \p isAngled indicates whether the file
1965  /// reference is for system \#include's or not (i.e. using <> instead of "").
1967  LookupFile(SourceLocation FilenameLoc, StringRef Filename, bool isAngled,
1968  const DirectoryLookup *FromDir, const FileEntry *FromFile,
1969  const DirectoryLookup *&CurDir, SmallVectorImpl<char> *SearchPath,
1970  SmallVectorImpl<char> *RelativePath,
1971  ModuleMap::KnownHeader *SuggestedModule, bool *IsMapped,
1972  bool *IsFrameworkFound, bool SkipCache = false);
1973 
1974  /// Get the DirectoryLookup structure used to find the current
1975  /// FileEntry, if CurLexer is non-null and if applicable.
1976  ///
1977  /// This allows us to implement \#include_next and find directory-specific
1978  /// properties.
1979  const DirectoryLookup *GetCurDirLookup() { return CurDirLookup; }
1980 
1981  /// Return true if we're in the top-level file, not in a \#include.
1982  bool isInPrimaryFile() const;
1983 
1984  /// Lex an on-off-switch (C99 6.10.6p2) and verify that it is
1985  /// followed by EOD. Return true if the token is not a valid on-off-switch.
1986  bool LexOnOffSwitch(tok::OnOffSwitch &Result);
1987 
1988  bool CheckMacroName(Token &MacroNameTok, MacroUse isDefineUndef,
1989  bool *ShadowFlag = nullptr);
1990 
1991  void EnterSubmodule(Module *M, SourceLocation ImportLoc, bool ForPragma);
1992  Module *LeaveSubmodule(bool ForPragma);
1993 
1994 private:
1995  friend void TokenLexer::ExpandFunctionArguments();
1996 
1997  void PushIncludeMacroStack() {
1998  assert(CurLexerKind != CLK_CachingLexer && "cannot push a caching lexer");
1999  IncludeMacroStack.emplace_back(CurLexerKind, CurLexerSubmodule,
2000  std::move(CurLexer), CurPPLexer,
2001  std::move(CurTokenLexer), CurDirLookup);
2002  CurPPLexer = nullptr;
2003  }
2004 
2005  void PopIncludeMacroStack() {
2006  CurLexer = std::move(IncludeMacroStack.back().TheLexer);
2007  CurPPLexer = IncludeMacroStack.back().ThePPLexer;
2008  CurTokenLexer = std::move(IncludeMacroStack.back().TheTokenLexer);
2009  CurDirLookup = IncludeMacroStack.back().TheDirLookup;
2010  CurLexerSubmodule = IncludeMacroStack.back().TheSubmodule;
2011  CurLexerKind = IncludeMacroStack.back().CurLexerKind;
2012  IncludeMacroStack.pop_back();
2013  }
2014 
2015  void PropagateLineStartLeadingSpaceInfo(Token &Result);
2016 
2017  /// Determine whether we need to create module macros for #defines in the
2018  /// current context.
2019  bool needModuleMacros() const;
2020 
2021  /// Update the set of active module macros and ambiguity flag for a module
2022  /// macro name.
2023  void updateModuleMacroInfo(const IdentifierInfo *II, ModuleMacroInfo &Info);
2024 
2025  DefMacroDirective *AllocateDefMacroDirective(MacroInfo *MI,
2026  SourceLocation Loc);
2027  UndefMacroDirective *AllocateUndefMacroDirective(SourceLocation UndefLoc);
2028  VisibilityMacroDirective *AllocateVisibilityMacroDirective(SourceLocation Loc,
2029  bool isPublic);
2030 
2031  /// Lex and validate a macro name, which occurs after a
2032  /// \#define or \#undef.
2033  ///
2034  /// \param MacroNameTok Token that represents the name defined or undefined.
2035  /// \param IsDefineUndef Kind if preprocessor directive.
2036  /// \param ShadowFlag Points to flag that is set if macro name shadows
2037  /// a keyword.
2038  ///
2039  /// This emits a diagnostic, sets the token kind to eod,
2040  /// and discards the rest of the macro line if the macro name is invalid.
2041  void ReadMacroName(Token &MacroNameTok, MacroUse IsDefineUndef = MU_Other,
2042  bool *ShadowFlag = nullptr);
2043 
2044  /// ReadOptionalMacroParameterListAndBody - This consumes all (i.e. the
2045  /// entire line) of the macro's tokens and adds them to MacroInfo, and while
2046  /// doing so performs certain validity checks including (but not limited to):
2047  /// - # (stringization) is followed by a macro parameter
2048  /// \param MacroNameTok - Token that represents the macro name
2049  /// \param ImmediatelyAfterHeaderGuard - Macro follows an #ifdef header guard
2050  ///
2051  /// Either returns a pointer to a MacroInfo object OR emits a diagnostic and
2052  /// returns a nullptr if an invalid sequence of tokens is encountered.
2053  MacroInfo *ReadOptionalMacroParameterListAndBody(
2054  const Token &MacroNameTok, bool ImmediatelyAfterHeaderGuard);
2055 
2056  /// The ( starting an argument list of a macro definition has just been read.
2057  /// Lex the rest of the parameters and the closing ), updating \p MI with
2058  /// what we learn and saving in \p LastTok the last token read.
2059  /// Return true if an error occurs parsing the arg list.
2060  bool ReadMacroParameterList(MacroInfo *MI, Token& LastTok);
2061 
2062  /// We just read a \#if or related directive and decided that the
2063  /// subsequent tokens are in the \#if'd out portion of the
2064  /// file. Lex the rest of the file, until we see an \#endif. If \p
2065  /// FoundNonSkipPortion is true, then we have already emitted code for part of
2066  /// this \#if directive, so \#else/\#elif blocks should never be entered. If
2067  /// \p FoundElse is false, then \#else directives are ok, if not, then we have
2068  /// already seen one so a \#else directive is a duplicate. When this returns,
2069  /// the caller can lex the first valid token.
2070  void SkipExcludedConditionalBlock(SourceLocation HashTokenLoc,
2071  SourceLocation IfTokenLoc,
2072  bool FoundNonSkipPortion, bool FoundElse,
2073  SourceLocation ElseLoc = SourceLocation());
2074 
2075  /// Information about the result for evaluating an expression for a
2076  /// preprocessor directive.
2077  struct DirectiveEvalResult {
2078  /// Whether the expression was evaluated as true or not.
2079  bool Conditional;
2080 
2081  /// True if the expression contained identifiers that were undefined.
2082  bool IncludedUndefinedIds;
2083 
2084  /// The source range for the expression.
2085  SourceRange ExprRange;
2086  };
2087 
2088  /// Evaluate an integer constant expression that may occur after a
2089  /// \#if or \#elif directive and return a \p DirectiveEvalResult object.
2090  ///
2091  /// If the expression is equivalent to "!defined(X)" return X in IfNDefMacro.
2092  DirectiveEvalResult EvaluateDirectiveExpression(IdentifierInfo *&IfNDefMacro);
2093 
2094  /// Install the standard preprocessor pragmas:
2095  /// \#pragma GCC poison/system_header/dependency and \#pragma once.
2096  void RegisterBuiltinPragmas();
2097 
2098  /// Register builtin macros such as __LINE__ with the identifier table.
2099  void RegisterBuiltinMacros();
2100 
2101  /// If an identifier token is read that is to be expanded as a macro, handle
2102  /// it and return the next token as 'Tok'. If we lexed a token, return true;
2103  /// otherwise the caller should lex again.
2104  bool HandleMacroExpandedIdentifier(Token &Identifier, const MacroDefinition &MD);
2105 
2106  /// Cache macro expanded tokens for TokenLexers.
2107  //
2108  /// Works like a stack; a TokenLexer adds the macro expanded tokens that is
2109  /// going to lex in the cache and when it finishes the tokens are removed
2110  /// from the end of the cache.
2111  Token *cacheMacroExpandedTokens(TokenLexer *tokLexer,
2112  ArrayRef<Token> tokens);
2113 
2114  void removeCachedMacroExpandedTokensOfLastLexer();
2115 
2116  /// Determine whether the next preprocessor token to be
2117  /// lexed is a '('. If so, consume the token and return true, if not, this
2118  /// method should have no observable side-effect on the lexed tokens.
2119  bool isNextPPTokenLParen();
2120 
2121  /// After reading "MACRO(", this method is invoked to read all of the formal
2122  /// arguments specified for the macro invocation. Returns null on error.
2123  MacroArgs *ReadMacroCallArgumentList(Token &MacroName, MacroInfo *MI,
2124  SourceLocation &MacroEnd);
2125 
2126  /// If an identifier token is read that is to be expanded
2127  /// as a builtin macro, handle it and return the next token as 'Tok'.
2128  void ExpandBuiltinMacro(Token &Tok);
2129 
2130  /// Read a \c _Pragma directive, slice it up, process it, then
2131  /// return the first token after the directive.
2132  /// This assumes that the \c _Pragma token has just been read into \p Tok.
2133  void Handle_Pragma(Token &Tok);
2134 
2135  /// Like Handle_Pragma except the pragma text is not enclosed within
2136  /// a string literal.
2137  void HandleMicrosoft__pragma(Token &Tok);
2138 
2139  /// Add a lexer to the top of the include stack and
2140  /// start lexing tokens from it instead of the current buffer.
2141  void EnterSourceFileWithLexer(Lexer *TheLexer, const DirectoryLookup *Dir);
2142 
2143  /// Set the FileID for the preprocessor predefines.
2144  void setPredefinesFileID(FileID FID) {
2145  assert(PredefinesFileID.isInvalid() && "PredefinesFileID already set!");
2146  PredefinesFileID = FID;
2147  }
2148 
2149  /// Set the FileID for the PCH through header.
2150  void setPCHThroughHeaderFileID(FileID FID);
2151 
2152  /// Returns true if we are lexing from a file and not a
2153  /// pragma or a macro.
2154  static bool IsFileLexer(const Lexer* L, const PreprocessorLexer* P) {
2155  return L ? !L->isPragmaLexer() : P != nullptr;
2156  }
2157 
2158  static bool IsFileLexer(const IncludeStackInfo& I) {
2159  return IsFileLexer(I.TheLexer.get(), I.ThePPLexer);
2160  }
2161 
2162  bool IsFileLexer() const {
2163  return IsFileLexer(CurLexer.get(), CurPPLexer);
2164  }
2165 
2166  //===--------------------------------------------------------------------===//
2167  // Caching stuff.
2168  void CachingLex(Token &Result);
2169 
2170  bool InCachingLexMode() const {
2171  // If the Lexer pointers are 0 and IncludeMacroStack is empty, it means
2172  // that we are past EOF, not that we are in CachingLex mode.
2173  return !CurPPLexer && !CurTokenLexer && !IncludeMacroStack.empty();
2174  }
2175 
2176  void EnterCachingLexMode();
2177  void EnterCachingLexModeUnchecked();
2178 
2179  void ExitCachingLexMode() {
2180  if (InCachingLexMode())
2181  RemoveTopOfLexerStack();
2182  }
2183 
2184  const Token &PeekAhead(unsigned N);
2185  void AnnotatePreviousCachedTokens(const Token &Tok);
2186 
2187  //===--------------------------------------------------------------------===//
2188  /// Handle*Directive - implement the various preprocessor directives. These
2189  /// should side-effect the current preprocessor object so that the next call
2190  /// to Lex() will return the appropriate token next.
2191  void HandleLineDirective();
2192  void HandleDigitDirective(Token &Tok);
2193  void HandleUserDiagnosticDirective(Token &Tok, bool isWarning);
2194  void HandleIdentSCCSDirective(Token &Tok);
2195  void HandleMacroPublicDirective(Token &Tok);
2196  void HandleMacroPrivateDirective();
2197 
2198  /// An additional notification that can be produced by a header inclusion or
2199  /// import to tell the parser what happened.
2200  struct ImportAction {
2201  enum ActionKind {
2202  None,
2203  ModuleBegin,
2204  ModuleImport,
2205  SkippedModuleImport,
2206  } Kind;
2207  Module *ModuleForHeader = nullptr;
2208 
2209  ImportAction(ActionKind AK, Module *Mod = nullptr)
2210  : Kind(AK), ModuleForHeader(Mod) {
2211  assert((AK == None || Mod) && "no module for module action");
2212  }
2213  };
2214 
2215  Optional<FileEntryRef> LookupHeaderIncludeOrImport(
2216  const DirectoryLookup *&CurDir, StringRef Filename,
2217  SourceLocation FilenameLoc, CharSourceRange FilenameRange,
2218  const Token &FilenameTok, bool &IsFrameworkFound, bool IsImportDecl,
2219  bool &IsMapped, const DirectoryLookup *LookupFrom,
2220  const FileEntry *LookupFromFile, StringRef LookupFilename,
2221  SmallVectorImpl<char> &RelativePath, SmallVectorImpl<char> &SearchPath,
2222  ModuleMap::KnownHeader &SuggestedModule, bool isAngled);
2223 
2224  // File inclusion.
2225  void HandleIncludeDirective(SourceLocation HashLoc, Token &Tok,
2226  const DirectoryLookup *LookupFrom = nullptr,
2227  const FileEntry *LookupFromFile = nullptr);
2228  ImportAction
2229  HandleHeaderIncludeOrImport(SourceLocation HashLoc, Token &IncludeTok,
2230  Token &FilenameTok, SourceLocation EndLoc,
2231  const DirectoryLookup *LookupFrom = nullptr,
2232  const FileEntry *LookupFromFile = nullptr);
2233  void HandleIncludeNextDirective(SourceLocation HashLoc, Token &Tok);
2234  void HandleIncludeMacrosDirective(SourceLocation HashLoc, Token &Tok);
2235  void HandleImportDirective(SourceLocation HashLoc, Token &Tok);
2236  void HandleMicrosoftImportDirective(Token &Tok);
2237 
2238 public:
2239  /// Check that the given module is available, producing a diagnostic if not.
2240  /// \return \c true if the check failed (because the module is not available).
2241  /// \c false if the module appears to be usable.
2242  static bool checkModuleIsAvailable(const LangOptions &LangOpts,
2243  const TargetInfo &TargetInfo,
2244  DiagnosticsEngine &Diags, Module *M);
2245 
2246  // Module inclusion testing.
2247  /// Find the module that owns the source or header file that
2248  /// \p Loc points to. If the location is in a file that was included
2249  /// into a module, or is outside any module, returns nullptr.
2250  Module *getModuleForLocation(SourceLocation Loc);
2251 
2252  /// We want to produce a diagnostic at location IncLoc concerning a
2253  /// missing module import.
2254  ///
2255  /// \param IncLoc The location at which the missing import was detected.
2256  /// \param M The desired module.
2257  /// \param MLoc A location within the desired module at which some desired
2258  /// effect occurred (eg, where a desired entity was declared).
2259  ///
2260  /// \return A file that can be #included to import a module containing MLoc.
2261  /// Null if no such file could be determined or if a #include is not
2262  /// appropriate.
2263  const FileEntry *getModuleHeaderToIncludeForDiagnostics(SourceLocation IncLoc,
2264  Module *M,
2265  SourceLocation MLoc);
2266 
2267  bool isRecordingPreamble() const {
2268  return PreambleConditionalStack.isRecording();
2269  }
2270 
2271  bool hasRecordedPreamble() const {
2272  return PreambleConditionalStack.hasRecordedPreamble();
2273  }
2274 
2276  return PreambleConditionalStack.getStack();
2277  }
2278 
2280  PreambleConditionalStack.setStack(s);
2281  }
2282 
2285  PreambleConditionalStack.startReplaying();
2286  PreambleConditionalStack.setStack(s);
2287  PreambleConditionalStack.SkipInfo = SkipInfo;
2288  }
2289 
2291  return PreambleConditionalStack.SkipInfo;
2292  }
2293 
2294 private:
2295  /// After processing predefined file, initialize the conditional stack from
2296  /// the preamble.
2297  void replayPreambleConditionalStack();
2298 
2299  // Macro handling.
2300  void HandleDefineDirective(Token &Tok, bool ImmediatelyAfterHeaderGuard);
2301  void HandleUndefDirective();
2302 
2303  // Conditional Inclusion.
2304  void HandleIfdefDirective(Token &Result, const Token &HashToken,
2305  bool isIfndef, bool ReadAnyTokensBeforeDirective);
2306  void HandleIfDirective(Token &IfToken, const Token &HashToken,
2307  bool ReadAnyTokensBeforeDirective);
2308  void HandleEndifDirective(Token &EndifToken);
2309  void HandleElseDirective(Token &Result, const Token &HashToken);
2310  void HandleElifDirective(Token &ElifToken, const Token &HashToken);
2311 
2312  // Pragmas.
2313  void HandlePragmaDirective(PragmaIntroducer Introducer);
2314 
2315 public:
2316  void HandlePragmaOnce(Token &OnceTok);
2317  void HandlePragmaMark();
2318  void HandlePragmaPoison();
2319  void HandlePragmaSystemHeader(Token &SysHeaderTok);
2320  void HandlePragmaDependency(Token &DependencyTok);
2321  void HandlePragmaPushMacro(Token &Tok);
2322  void HandlePragmaPopMacro(Token &Tok);
2323  void HandlePragmaIncludeAlias(Token &Tok);
2324  void HandlePragmaModuleBuild(Token &Tok);
2325  void HandlePragmaHdrstop(Token &Tok);
2326  IdentifierInfo *ParsePragmaPushOrPopMacro(Token &Tok);
2327 
2328  // Return true and store the first token only if any CommentHandler
2329  // has inserted some tokens and getCommentRetentionState() is false.
2330  bool HandleComment(Token &result, SourceRange Comment);
2331 
2332  /// A macro is used, update information about macros that need unused
2333  /// warnings.
2334  void markMacroAsUsed(MacroInfo *MI);
2335 
2336 private:
2338  getSkippedRangeForExcludedConditionalBlock(SourceLocation HashLoc);
2339 
2340  /// Contains the currently active skipped range mappings for skipping excluded
2341  /// conditional directives.
2343  *ExcludedConditionalDirectiveSkipMappings;
2344 };
2345 
2346 /// Abstract base class that describes a handler that will receive
2347 /// source ranges for each of the comments encountered in the source file.
2349 public:
2350  virtual ~CommentHandler();
2351 
2352  // The handler shall return true if it has pushed any tokens
2353  // to be read using e.g. EnterToken or EnterTokenStream.
2354  virtual bool HandleComment(Preprocessor &PP, SourceRange Comment) = 0;
2355 };
2356 
2357 /// Registry of pragma handlers added by plugins
2358 using PragmaHandlerRegistry = llvm::Registry<PragmaHandler>;
2359 
2360 } // namespace clang
2361 
2362 #endif // LLVM_CLANG_LEX_PREPROCESSOR_H
PreambleSkipInfo(SourceLocation HashTokenLoc, SourceLocation IfTokenLoc, bool FoundNonSkipPortion, bool FoundElse, SourceLocation ElseLoc)
Definition: Preprocessor.h:427
void EnterTokenStream(std::unique_ptr< Token[]> Toks, unsigned NumToks, bool DisableMacroExpansion, bool IsReinject)
A set of visible modules.
Definition: Module.h:596
StringRef Identifier
Definition: Format.cpp:1833
llvm::BumpPtrAllocator & getPreprocessorAllocator()
Definition: Preprocessor.h:918
void setCodeCompletionTokenRange(const SourceLocation Start, const SourceLocation End)
Set the code completion token range for detecting replacement range later on.
Lexer - This provides a simple interface that turns a text buffer into a stream of tokens...
Definition: Lexer.h:76
SelectorTable & getSelectorTable()
Definition: Preprocessor.h:916
Holds information about both target-independent and target-specific builtins, allowing easy queries b...
Definition: Builtins.h:65
Implements support for file system lookup, file system caching, and directory search management...
Definition: FileManager.h:171
IdentifierInfo * getIdentifierInfo(StringRef Name) const
Return information about the specified preprocessor identifier token.
std::string getSpelling(const Token &Tok, bool *Invalid=nullptr) const
Return the &#39;spelling&#39; of the Tok token.
Specialize PointerLikeTypeTraits to allow LazyGenerationalUpdatePtr to be placed into a PointerUnion...
Definition: Dominators.h:30
const Token & LookAhead(unsigned N)
Peeks ahead N tokens and returns that token without consuming any tokens.
void MaybeHandlePoisonedIdentifier(Token &Identifier)
bool is(tok::TokenKind K) const
is/isNot - Predicates to check if this token is a specific kind, as in "if (Tok.is(tok::l_brace)) {...
Definition: Token.h:97
Defines the SourceManager interface.
static const Builtin::Info BuiltinInfo[]
Definition: Builtins.cpp:20
bool LexStringLiteral(Token &Result, std::string &String, const char *DiagnosticTag, bool AllowMacroExpansion)
Lex a string literal, which may be the concatenation of multiple string literals and may even come fr...
FileManager & getFileManager() const
Definition: Preprocessor.h:910
bool isCodeCompletionReached() const
Returns true if code-completion is enabled and we have hit the code-completion point.
StringRef P
const char * getCharacterData(SourceLocation SL, bool *Invalid=nullptr) const
Return a pointer to the start of the specified location in the appropriate spelling MemoryBuffer...
Defines the clang::MacroInfo and clang::MacroDirective classes.
A description of the current definition of a macro.
Definition: MacroInfo.h:564
bool isOutOfDate() const
Determine whether the information for this identifier is out of date with respect to the external sou...
DiagnosticBuilder Report(SourceLocation Loc, unsigned DiagID)
Issue the message to the client.
Definition: Diagnostic.h:1300
A directive for an undefined macro.
Definition: MacroInfo.h:429
PreprocessorOptions - This class is used for passing the various options used in preprocessor initial...
void setCodeCompletionIdentifierInfo(IdentifierInfo *Filter)
Set the code completion token for filtering purposes.
void setCodeCompletionReached()
Note that we hit the code-completion point.
bool hadModuleLoaderFatalFailure() const
Definition: Preprocessor.h:931
bool getRawToken(SourceLocation Loc, Token &Result, bool IgnoreWhiteSpace=false)
Relex the token at the specified location.
void EnterTokenStream(ArrayRef< Token > Toks, bool DisableMacroExpansion, bool IsReinject)
bool operator==(const Token &Tok) const
Definition: Preprocessor.h:104
Builtin::Context & getBuiltinInfo()
Definition: Preprocessor.h:917
void setPredefines(const char *P)
Set the predefines for this Preprocessor.
SourceLocation getCodeCompletionFileLoc() const
Returns the start location of the file of code-completion point.
char getSpellingOfSingleCharacterNumericConstant(const Token &Tok, bool *Invalid=nullptr) const
Given a Token Tok that is a numeric constant with length 1, return the character. ...
This interface provides a way to observe the actions of the preprocessor as it does its thing...
Definition: PPCallbacks.h:35
bool isAnnotation() const
Return true if this is any of tok::annot_* kind tokens.
Definition: Token.h:120
const MacroDirective * getPrevious() const
Get previous definition of the macro with the same name.
Definition: MacroInfo.h:328
tok::TokenKind getKind() const
Definition: Token.h:92
TokenValue(tok::TokenKind Kind)
Definition: Preprocessor.h:94
SourceLocation getLocForEndOfToken(SourceLocation Loc, unsigned Offset=0)
Computes the source location just past the end of the token at this source location.
DefMacroDirective * appendDefMacroDirective(IdentifierInfo *II, MacroInfo *MI, SourceLocation Loc)
One of these records is kept for each identifier that is lexed.
ArrayRef< PPConditionalInfo > getPreambleConditionalStack() const
Represents a macro directive exported by a module.
Definition: MacroInfo.h:488
A directive for a defined macro or a macro imported from a module.
Definition: MacroInfo.h:406
This table allows us to fully hide how we implement multi-keyword caching.
LineState State
Definition: Format.h:2445
const TargetInfo & getTargetInfo() const
Definition: Preprocessor.h:908
Token - This structure provides full information about a lexed token.
Definition: Token.h:34
void SetCommentRetentionState(bool KeepComments, bool KeepMacroComments)
Control whether the preprocessor retains comments in output.
Definition: Preprocessor.h:947
Keeps track of the various options that can be enabled, which controls the dialect of C or C++ that i...
Definition: LangOptions.h:53
const LangOptions & getLangOpts() const
Definition: Preprocessor.h:907
Describes a module or submodule.
Definition: Module.h:64
bool isPragmaLexer() const
isPragmaLexer - Returns true if this Lexer is being used to lex a pragma.
Definition: Lexer.h:183
A directive for setting the module visibility of a macro.
Definition: MacroInfo.h:444
bool isInvalid() const
void SetSuppressIncludeNotFoundError(bool Suppress)
Definition: Preprocessor.h:957
SourceLocation AdvanceToTokenCharacter(SourceLocation TokStart, unsigned Char) const
Given a location that specifies the start of a token, return a new location that specifies a characte...
MacroUse
Context in which macro name is used.
Definition: Preprocessor.h:111
llvm::Registry< PragmaHandler > PragmaHandlerRegistry
Registry of pragma handlers added by plugins.
Module * getCurrentLexerSubmodule() const
Return the submodule owning the file being lexed.
Definition: Preprocessor.h:994
StringRef getSpelling(SourceLocation loc, SmallVectorImpl< char > &buffer, bool *invalid=nullptr) const
Return the &#39;spelling&#39; of the token at the given location; does not go up to the spelling location or ...
A record of the steps taken while preprocessing a source file, including the various preprocessing di...
HeaderSearch & getHeaderSearchInfo() const
Definition: Preprocessor.h:912
const FormatToken & Tok
Forward-declares and imports various common LLVM datatypes that clang wants to use unqualified...
void LexNonComment(Token &Result)
Lex a token.
Concrete class used by the front-end to report problems and issues.
Definition: Diagnostic.h:149
bool isMacroDefined(const IdentifierInfo *II)
bool isMacroDefinedInLocalModule(const IdentifierInfo *II, Module *M)
Determine whether II is defined as a macro within the module M, if that is a module that we&#39;ve alread...
Defines the Diagnostic-related interfaces.
const TargetInfo * getAuxTargetInfo() const
Definition: Preprocessor.h:909
TokenLexer - This implements a lexer that returns tokens from a macro body or token stream instead of...
Definition: TokenLexer.h:30
void setCodeCompletionHandler(CodeCompletionHandler &Handler)
Set the code completion handler to the given object.
void setTokenWatcher(llvm::unique_function< void(const clang::Token &)> F)
Register a function that would be called on each token in the final expanded token stream...
void setPreprocessedOutput(bool IsPreprocessedOutput)
Sets whether the preprocessor is responsible for producing output or if it is producing tokens to be ...
Definition: Preprocessor.h:967
void LexUnexpandedToken(Token &Result)
Just like Lex, but disables macro expansion of identifier tokens.
Encapsulates the information needed to find the file referenced by a #include or #include_next, (sub-)framework lookup, etc.
Definition: HeaderSearch.h:158
A little helper class used to produce diagnostics.
Definition: Diagnostic.h:1053
void clearCodeCompletionHandler()
Clear out the code completion handler.
StringRef Filename
Definition: Format.cpp:1825
Provides lookups to, and iteration over, IdentiferInfo objects.
unsigned Offset
Definition: Format.cpp:1827
Exposes information about the current target.
Definition: TargetInfo.h:164
const IdentifierTable & getIdentifierTable() const
Definition: Preprocessor.h:915
bool isCodeCompletionEnabled() const
Determine if we are performing code completion.
Abstract interface for external sources of preprocessor information.
Defines the clang::LangOptions interface.
unsigned getCounterValue() const
SourceLocation End
Represents a character-granular source range.
const DefMacroDirective * getDirective() const
Definition: MacroInfo.h:349
int Id
Definition: ASTDiff.cpp:190
void AnnotateCachedTokens(const Token &Tok)
We notify the Preprocessor that if it is caching tokens (because backtrack is enabled) it should repl...
const AnnotatedLine * Line
bool isDefined() const
Definition: MacroInfo.h:384
void SetMacroExpansionOnlyInDirectives()
Disables macro expansion everywhere except for preprocessor directives.
bool isCurrentLexer(const PreprocessorLexer *L) const
Return true if we are lexing directly from the specified lexer.
Definition: Preprocessor.h:976
Implements an efficient mapping from strings to IdentifierInfo nodes.
MacroArgs - An instance of this class captures information about the formal arguments specified to a ...
Definition: MacroArgs.h:29
#define V(N, I)
Definition: ASTContext.h:2941
void RevertCachedTokens(unsigned N)
When backtracking is enabled and tokens are cached, this allows to revert a specific number of tokens...
A class for tracking whether we&#39;re inside a VA_OPT during a traversal of the tokens of a variadic mac...
SourceLocation getLocation() const
Return a source location identifier for the specified offset in the current file. ...
Definition: Token.h:126
#define bool
Definition: stdbool.h:15
DefInfo getDefinition()
Traverses the macro directives history and returns the next macro definition directive along with inf...
Definition: MacroInfo.cpp:178
void IncrementPasteCounter(bool isFast)
Increment the counters for the number of token paste operations performed.
Stores token information for comparing actual tokens with predefined values.
Definition: Preprocessor.h:89
llvm::Optional< PreambleSkipInfo > getPreambleSkipInfo() const
DiagnosticBuilder Diag(const Token &Tok, unsigned DiagID) const
void setPragmaAssumeNonNullLoc(SourceLocation Loc)
Set the location of the currently-active #pragma clang assume_nonnull begin.
const MacroInfo * getMacroInfo(const IdentifierInfo *II) const
bool getCommentRetentionState() const
Definition: Preprocessor.h:952
Defines the clang::IdentifierInfo, clang::IdentifierTable, and clang::Selector interfaces.
TokenValue(IdentifierInfo *II)
Definition: Preprocessor.h:102
PPCallbacks * getPPCallbacks() const
const SourceManager & SM
Definition: Format.cpp:1685
DirectoryLookup - This class represents one entry in the search list that specifies the search order ...
bool isRecordingPreamble() const
const DirectoryLookup * GetCurDirLookup()
Get the DirectoryLookup structure used to find the current FileEntry, if CurLexer is non-null and if ...
SourceManager & getSourceManager() const
Definition: Preprocessor.h:911
MacroDirective * getLocalMacroDirective(const IdentifierInfo *II) const
Given an identifier, return its latest non-imported MacroDirective if it is #define&#39;d and not #undef&#39;...
const DefInfo findDirectiveAtLoc(SourceLocation L, const SourceManager &SM) const
Find macro definition active in the specified source location.
Definition: MacroInfo.cpp:202
ExternalPreprocessorSource * getExternalSource() const
Definition: Preprocessor.h:924
llvm::cl::opt< std::string > Filter
Encapsulates changes to the "macros namespace" (the location where the macro name became active...
Definition: MacroInfo.h:290
Kind
void TypoCorrectToken(const Token &Tok)
Update the current token to represent the provided identifier, in order to cache an action performed ...
void setExternalSource(ExternalPreprocessorSource *Source)
Definition: Preprocessor.h:920
Encodes a location in the source.
IdentifierInfo & get(StringRef Name)
Return the identifier token info for the specified named identifier.
bool isAnnotation(TokenKind K)
Return true if this is any of tok::annot_* kinds.
Definition: TokenKinds.cpp:49
MacroDefinition getMacroDefinition(const IdentifierInfo *II)
IdentifierInfo * getIdentifierInfo() const
Definition: Token.h:179
IdentifierTable & getIdentifierTable()
Definition: Preprocessor.h:914
void setPragmasEnabled(bool Enabled)
Definition: Preprocessor.h:954
Cached information about one file (either on disk or in the virtual file system). ...
Definition: FileManager.h:78
CodeCompletionHandler * getCodeCompletionHandler() const
Retrieve the current code-completion handler.
bool isAtStartOfMacroExpansion(SourceLocation loc, SourceLocation *MacroBegin=nullptr) const
Returns true if the given MacroID location points at the first token of the macro expansion...
bool isLiteral(TokenKind K)
Return true if this is a "literal" kind, like a numeric constant, string, etc.
Definition: TokenKinds.h:85
bool isPoisoned() const
Return true if this token has been poisoned.
bool SawDateOrTime() const
Returns true if the preprocessor has seen a use of DATE or TIME in the file so far.
TokenKind
Provides a simple uniform namespace for tokens from all C languages.
Definition: TokenKinds.h:24
void enableIncrementalProcessing(bool value=true)
Enables the incremental processing.
MacroDefinition getMacroDefinitionAtLoc(const IdentifierInfo *II, SourceLocation Loc)
Defines the clang::Module class, which describes a module in the source code.
PreprocessorLexer * getCurrentLexer() const
Return the current lexer being lexed from.
Definition: Preprocessor.h:984
bool isAtEndOfMacroExpansion(SourceLocation loc, SourceLocation *MacroEnd=nullptr) const
Returns true if the given MacroID location points at the last token of the macro expansion.
void EnterToken(const Token &Tok, bool IsReinject)
Enters a token in the token stream to be lexed next.
void ReplaceLastTokenWithAnnotation(const Token &Tok)
Replace the last token with an annotation token.
SourceLocation getPragmaAssumeNonNullLoc() const
The location of the currently-active #pragma clang assume_nonnull begin.
SourceLocation getCodeCompletionLoc() const
Returns the location of the code-completion point.
const MacroInfo * getMacroInfo() const
Definition: MacroInfo.h:390
StringRef getName() const
Return the actual identifier string.
bool isMacroDefined(StringRef Id)
An opaque identifier used by SourceManager which refers to a source file (MemoryBuffer) along with it...
bool isBacktrackEnabled() const
True if EnableBacktrackAtThisPos() was called and caching of tokens is on.
bool hadMacroDefinition() const
Returns true if this identifier was #defined to some value at any moment.
Dataflow Directional Tag Classes.
SmallVector< Token, 4 > CachedTokens
A set of tokens that has been cached for later parsing.
Definition: DeclSpec.h:1168
SourceRange getCodeCompletionTokenRange() const
std::pair< IdentifierInfo *, SourceLocation > getPragmaARCCFCodeAuditedInfo() const
The location of the currently-active #pragma clang arc_cf_code_audited begin.
PreprocessingRecord * getPreprocessingRecord() const
Retrieve the preprocessing record, or NULL if there is no preprocessing record.
Reads an AST files chain containing the contents of a translation unit.
Definition: ASTReader.h:340
PragmaHandler - Instances of this interface defined to handle the various pragmas that the language f...
Definition: Pragma.h:65
llvm::iterator_range< macro_iterator > macros(bool IncludeExternalMacros=true) const
SourceLocation getLastCachedTokenLocation() const
Get the location of the last cached token, suitable for setting the end location of an annotation tok...
StringRef getCodeCompletionFilter()
Get the code completion token for filtering purposes.
Abstract interface for a module loader.
Definition: ModuleLoader.h:80
unsigned getLength() const
Definition: Token.h:129
Encapsulates the data about a macro definition (e.g.
Definition: MacroInfo.h:39
OnOffSwitch
Defines the possible values of an on-off-switch (C99 6.10.6p2).
Definition: TokenKinds.h:47
const char * getLiteralData() const
getLiteralData - For a literal token (numeric constant, string, etc), this returns a pointer to the s...
Definition: Token.h:217
void setPragmaARCCFCodeAuditedInfo(IdentifierInfo *Ident, SourceLocation Loc)
Set the location of the currently-active #pragma clang arc_cf_code_audited begin. ...
bool getPragmasEnabled() const
Definition: Preprocessor.h:955
unsigned getSpelling(const Token &Tok, const char *&Buffer, bool *Invalid=nullptr) const
Get the spelling of a token into a preallocated buffer, instead of as an std::string.
SourceLocation getDefinitionLoc() const
Return the location that the macro was defined at.
Definition: MacroInfo.h:123
ModuleLoader & getModuleLoader() const
Retrieve the module loader associated with this preprocessor.
Definition: Preprocessor.h:929
void LexUnexpandedNonComment(Token &Result)
Like LexNonComment, but this disables macro expansion of identifier tokens.
PreprocessorOptions & getPreprocessorOpts() const
Retrieve the preprocessor options used to initialize this preprocessor.
Definition: Preprocessor.h:902
Cached information about one directory (either on disk or in the virtual file system).
Definition: FileManager.h:45
Defines the PPCallbacks interface.
Defines the clang::TokenKind enum and support functions.
DefMacroDirective * appendDefMacroDirective(IdentifierInfo *II, MacroInfo *MI)
SourceLocation getModuleImportLoc(Module *M) const
Defines the clang::SourceLocation class and associated facilities.
void RemovePragmaHandler(PragmaHandler *Handler)
FileID getPredefinesFileID() const
Returns the FileID for the preprocessor predefines.
Definition: Preprocessor.h:997
DiagnosticsEngine & getDiagnostics() const
Definition: Preprocessor.h:904
void setMainFileDir(const DirectoryEntry *Dir)
Set the directory in which the main file should be considered to have been found, if it is not a real...
static bool isMacroDefined(const Sema &S, SourceLocation Loc, StringRef Name)
TranslationUnitKind
Describes the kind of translation unit being processed.
Definition: LangOptions.h:412
bool hasRecordedPreamble() const
const std::string & getPredefines() const
bool needsCleaning() const
Return true if this token has trigraphs or escaped newlines in it.
Definition: Token.h:285
void setPredefines(StringRef P)
ArrayRef< ModuleMacro * > getLeafModuleMacros(const IdentifierInfo *II) const
Get the list of leaf (non-overridden) module macros for a name.
bool isIncrementalProcessingEnabled() const
Returns true if incremental processing is enabled.
The translation unit is a complete translation unit.
Definition: LangOptions.h:414
StringRef getImmediateMacroName(SourceLocation Loc)
Retrieve the name of the immediate macro expansion.
llvm::DenseMap< const llvm::MemoryBuffer *, const PreprocessorSkippedRangeMapping * > ExcludedPreprocessorDirectiveSkipMapping
The datastructure that holds the mapping between the active memory buffers and the individual skip ma...
MacroMap::const_iterator macro_iterator
Abstract base class that describes a handler that will receive source ranges for each of the comments...
Describes how and where the pragma was introduced.
Definition: Pragma.h:51
void setCounterValue(unsigned V)
bool isParsingIfOrElifDirective() const
True if we are currently preprocessing a if or #elif directive.
Definition: Preprocessor.h:942
void setSkipMainFilePreamble(unsigned Bytes, bool StartOfLine)
Instruct the preprocessor to skip part of the main source file.
A trivial tuple used to represent a source range.
bool GetSuppressIncludeNotFoundError()
Definition: Preprocessor.h:961
void setReplayablePreambleConditionalStack(ArrayRef< PPConditionalInfo > s, llvm::Optional< PreambleSkipInfo > SkipInfo)
MacroInfo * getMacroInfo(const IdentifierInfo *II)
Callback handler that receives notifications when performing code completion within the preprocessor...
A header that is known to reside within a given module, whether it was included or excluded...
Definition: ModuleMap.h:155
bool isPreprocessedOutput() const
Returns true if the preprocessor is responsible for generating output, false if it is producing token...
Definition: Preprocessor.h:973
bool hasMacroDefinition() const
Return true if this identifier is #defined to some other value.
DiagnosticBuilder Diag(SourceLocation Loc, unsigned DiagID) const
Forwarding function for diagnostics.
void addPPCallbacks(std::unique_ptr< PPCallbacks > C)
void setRecordedPreambleConditionalStack(ArrayRef< PPConditionalInfo > s)
This class handles loading and caching of source files into memory.
An RAII class that tracks when the Preprocessor starts and stops lexing the definition of a (ISO C/C+...
void AddPragmaHandler(PragmaHandler *Handler)
Engages in a tight little dance with the lexer to efficiently preprocess tokens.
Definition: Preprocessor.h:128
void setDiagnostics(DiagnosticsEngine &D)
Definition: Preprocessor.h:905
unsigned getNumDirectives() const
Retrieve the number of Directives that have been processed by the Preprocessor.
Definition: Preprocessor.h:937