28 #include "llvm/ADT/ArrayRef.h" 29 #include "llvm/ADT/SmallString.h" 30 #include "llvm/ADT/SmallVector.h" 31 #include "llvm/ADT/iterator_range.h" 35 using namespace clang;
53 NextTokGetsSpace =
false;
56 DisableMacroExpansion =
false;
64 assert(Tokens[0].getLocation().isValid());
65 assert((Tokens[0].getLocation().isFileID() || Tokens[0].is(tok::comment)) &&
66 "Macro defined in macro?");
67 assert(ExpandLocStart.
isValid());
84 ExpandFunctionArguments();
95 bool disableMacroExpansion,
bool ownsTokens) {
101 ActualArgs =
nullptr;
103 OwnsTokens = ownsTokens;
104 DisableMacroExpansion = disableMacroExpansion;
108 AtStartOfLine =
false;
109 HasLeadingSpace =
false;
110 NextTokGetsSpace =
false;
121 void TokenLexer::destroy() {
131 if (ActualArgs) ActualArgs->
destroy(PP);
134 bool TokenLexer::MaybeRemoveCommaBeforeVaArgs(
144 if (!HasPasteOperator && !PP.
getLangOpts().MSVCCompat)
157 if (ResultToks.empty() || !ResultToks.back().is(tok::comma))
161 if (HasPasteOperator)
162 PP.
Diag(ResultToks.back().getLocation(), diag::ext_paste_comma);
165 ResultToks.pop_back();
167 if (!ResultToks.empty()) {
172 if (ResultToks.back().is(tok::hashhash))
173 ResultToks.pop_back();
180 NextTokGetsSpace =
false;
184 void TokenLexer::stringifyVAOPTContents(
188 const unsigned int NumVAOptTokens = ResultToks.size() - NumToksPriorToVAOpt;
189 Token *
const VAOPTTokens =
190 NumVAOptTokens ? &ResultToks[NumToksPriorToVAOpt] :
nullptr;
197 for (
unsigned int CurTokenIdx = 0; CurTokenIdx != NumVAOptTokens;
199 if (VAOPTTokens[CurTokenIdx].is(tok::hashhash)) {
200 assert(CurTokenIdx != 0 &&
201 "Can not have __VAOPT__ contents begin with a ##");
202 Token &LHS = VAOPTTokens[CurTokenIdx - 1];
203 pasteTokens(LHS, llvm::makeArrayRef(VAOPTTokens, NumVAOptTokens),
206 ConcatenatedVAOPTResultToks.back() = LHS;
207 if (CurTokenIdx == NumVAOptTokens)
210 ConcatenatedVAOPTResultToks.push_back(VAOPTTokens[CurTokenIdx]);
213 ConcatenatedVAOPTResultToks.push_back(VCtx.
getEOFTok());
222 getExpansionLocForMacroDefLoc(VAOPTClosingParenLoc);
226 ExpansionLocStartWithinMacro, ExpansionLocEndWithinMacro);
233 ResultToks.resize(NumToksPriorToVAOpt + 1);
234 ResultToks.back() = StringifiedVAOPT;
239 void TokenLexer::ExpandFunctionArguments() {
245 bool MadeChange =
false;
247 const bool CalledWithVariadicArguments =
252 for (
unsigned I = 0, E = NumTokens; I != E; ++I) {
253 const Token &CurTok = Tokens[I];
260 if (I != 0 && !Tokens[I-1].is(tok::hashhash) && CurTok.
hasLeadingSpace())
261 NextTokGetsSpace =
true;
265 assert(Tokens[I + 1].is(tok::l_paren) &&
266 "__VA_OPT__ must be followed by '('");
288 if (Tokens[I].is(tok::l_paren))
295 if (!CalledWithVariadicArguments) {
315 stringifyVAOPTContents(ResultToks, VCtx,
316 Tokens[I].getLocation());
325 if (ResultToks.size() && ResultToks.back().is(tok::hashhash)) {
326 ResultToks.pop_back();
327 }
else if ((I + 1 != E) && Tokens[I + 1].is(tok::hashhash)) {
342 if (CurTok.
isOneOf(tok::hash, tok::hashat)) {
344 assert((ArgNo != -1 || VCtx.
isVAOptToken(Tokens[I + 1])) &&
345 "Token following # is not an argument or __VA_OPT__!");
350 CurTok.
is(tok::hashat));
355 getExpansionLocForMacroDefLoc(CurTok.
getLocation());
357 getExpansionLocForMacroDefLoc(Tokens[I+1].getLocation());
360 if (CurTok.
is(tok::hash))
375 if (NextTokGetsSpace)
378 ResultToks.push_back(Res);
381 NextTokGetsSpace =
false;
386 bool NonEmptyPasteBefore =
387 !ResultToks.empty() && ResultToks.back().is(tok::hashhash);
388 bool PasteBefore = I != 0 && Tokens[I-1].
is(tok::hashhash);
389 bool PasteAfter = I+1 != E && Tokens[I+1].
is(tok::hashhash);
391 assert((!NonEmptyPasteBefore || PasteBefore || VCtx.
isInVAOpt()) &&
392 "unexpected ## in ResultToks");
400 ResultToks.push_back(CurTok);
402 if (NextTokGetsSpace) {
404 NextTokGetsSpace =
false;
405 }
else if (PasteBefore && !NonEmptyPasteBefore)
420 MaybeRemoveCommaBeforeVaArgs(ResultToks,
428 if (!PasteBefore && !PasteAfter) {
429 const Token *ResultArgToks;
437 ResultArgToks = ArgTok;
441 size_t FirstResult = ResultToks.size();
443 ResultToks.append(ResultArgToks, ResultArgToks+NumToks);
450 ResultToks.back().is(tok::comma))
455 for (
Token &
Tok : llvm::make_range(ResultToks.begin() + FirstResult,
457 if (
Tok.
is(tok::hashhash))
458 Tok.setKind(tok::unknown);
463 ResultToks.begin()+FirstResult,
473 NextTokGetsSpace =
false;
483 bool VaArgsPseudoPaste =
false;
488 if (NonEmptyPasteBefore && ResultToks.size() >= 2 &&
489 ResultToks[ResultToks.size()-2].is(tok::comma) &&
492 VaArgsPseudoPaste =
true;
494 PP.
Diag(ResultToks.pop_back_val().getLocation(), diag::ext_paste_comma);
497 ResultToks.append(ArgToks, ArgToks+NumToks);
501 for (
Token &
Tok : llvm::make_range(ResultToks.end() - NumToks,
503 if (
Tok.
is(tok::hashhash))
504 Tok.setKind(tok::unknown);
507 if (ExpandLocStart.
isValid()) {
509 ResultToks.end()-NumToks, ResultToks.end());
516 if (!VaArgsPseudoPaste) {
519 ResultToks[ResultToks.size() - NumToks].setFlagValue(
523 NextTokGetsSpace =
false;
542 if (NonEmptyPasteBefore) {
543 assert(ResultToks.back().is(tok::hashhash));
550 ResultToks.pop_back();
558 MaybeRemoveCommaBeforeVaArgs(ResultToks,
565 assert(!OwnsTokens &&
"This would leak if we already own the token list");
567 NumTokens = ResultToks.size();
570 Tokens = PP.cacheMacroExpandedTokens(
this, ResultToks);
579 const Token &SecondTok) {
580 return FirstTok.
is(tok::identifier) &&
596 if (CurTokenIdx == 0)
605 bool isFirstToken = CurTokenIdx == 0;
608 Tok = Tokens[CurTokenIdx++];
610 bool TokenIsFromPaste =
false;
614 if (!isAtEnd() && Macro &&
615 (Tokens[CurTokenIdx].is(tok::hashhash) ||
623 if (pasteTokens(Tok))
626 TokenIsFromPaste =
true;
634 if (ExpandLocStart.
isValid() &&
638 if (Tok.
is(tok::comment)) {
644 instLoc = getExpansionLocForMacroDefLoc(Tok.
getLocation());
661 AtStartOfLine =
false;
662 HasLeadingSpace =
false;
686 bool TokenLexer::pasteTokens(
Token &
Tok) {
687 return pasteTokens(Tok, llvm::makeArrayRef(Tokens, NumTokens), CurTokenIdx);
695 unsigned int &CurIdx) {
696 assert(CurIdx > 0 &&
"## can not be the first token within tokens");
697 assert((TokenStream[CurIdx].is(tok::hashhash) ||
700 "Token at this Index must be ## or part of the MSVC 'L " 701 "#macro-arg' pasting pair");
706 if (PP.
getLangOpts().MicrosoftExt && (CurIdx >= 2) &&
707 TokenStream[CurIdx - 2].is(tok::hashhash))
711 const char *ResultTokStrPtr =
nullptr;
715 auto IsAtEnd = [&TokenStream, &CurIdx] {
716 return TokenStream.size() == CurIdx;
721 PasteOpLoc = TokenStream[CurIdx].getLocation();
722 if (TokenStream[CurIdx].is(tok::hashhash))
724 assert(!IsAtEnd() &&
"No token on the RHS of a paste operator!");
727 const Token &RHS = TokenStream[CurIdx];
734 const char *BufPtr = &Buffer[0];
735 bool Invalid =
false;
736 unsigned LHSLen = PP.
getSpelling(LHSTok, BufPtr, &Invalid);
737 if (BufPtr != &Buffer[0])
738 memcpy(&Buffer[0], BufPtr, LHSLen);
742 BufPtr = Buffer.data() + LHSLen;
743 unsigned RHSLen = PP.
getSpelling(RHS, BufPtr, &Invalid);
746 if (RHSLen && BufPtr != &Buffer[LHSLen])
748 memcpy(&Buffer[LHSLen], BufPtr, RHSLen);
751 Buffer.resize(LHSLen+RHSLen);
760 ResultTokTmp.
setKind(tok::string_literal);
773 Result.
setKind(tok::raw_identifier);
781 "Should be a raw location into scratch buffer");
783 FileID LocFileID = SourceMgr.getFileID(ResultTokLoc);
785 bool Invalid =
false;
786 const char *ScratchBufStart
787 = SourceMgr.getBufferData(LocFileID, &Invalid).data();
793 Lexer TL(SourceMgr.getLocForStartOfFile(LocFileID),
795 ResultTokStrPtr, ResultTokStrPtr+LHSLen+RHSLen);
801 bool isInvalid = !TL.LexFromRawLexer(Result);
819 if (PP.
getLangOpts().MicrosoftExt && LHSTok.
is(tok::slash) &&
820 RHS.
is(tok::slash)) {
821 HandleMicrosoftCommentPaste(LHSTok, Loc);
831 : diag::err_pp_bad_paste)
841 if (Result.
is(tok::hashhash))
852 }
while (!IsAtEnd() && TokenStream[CurIdx].is(tok::hashhash));
863 StartLoc = getExpansionLocForMacroDefLoc(StartLoc);
865 EndLoc = getExpansionLocForMacroDefLoc(EndLoc);
867 while (SM.
getFileID(StartLoc) != MacroFID)
878 if (LHSTok.
is(tok::raw_identifier)) {
893 return Tokens[CurTokenIdx].
is(tok::l_paren);
899 return Tokens[NumTokens-1].
is(tok::eod) && !isAtEnd();
908 PP.
Diag(OpLoc, diag::ext_comment_paste_microsoft);
915 assert(Macro &&
"Token streams can't paste comments");
926 TokenLexer::getExpansionLocForMacroDefLoc(
SourceLocation loc)
const {
927 assert(ExpandLocStart.
isValid() && MacroExpansionStart.
isValid() &&
928 "Not appropriate for token streams");
932 assert(SM.isInSLocAddrSpace(loc, MacroDefStart, MacroDefLength) &&
933 "Expected loc to come from the macro definition");
935 unsigned relativeOffset = 0;
951 Token *&begin_tokens,
952 Token * end_tokens) {
953 assert(begin_tokens < end_tokens);
968 Token *NextTok = begin_tokens + 1;
969 for (; NextTok < end_tokens; ++NextTok) {
979 if (RelOffs < 0 || RelOffs > 50)
990 Token &LastConsecutiveTok = *(NextTok-1);
994 unsigned FullLength = LastRelOffs + LastConsecutiveTok.
getLength();
1002 for (; begin_tokens < NextTok; ++begin_tokens) {
1003 Token &Tok = *begin_tokens;
1015 void TokenLexer::updateLocForMacroArgTokens(
SourceLocation ArgIdSpellLoc,
1016 Token *begin_tokens,
1017 Token *end_tokens) {
1021 getExpansionLocForMacroDefLoc(ArgIdSpellLoc);
1023 while (begin_tokens < end_tokens) {
1025 if (end_tokens - begin_tokens == 1) {
1026 Token &Tok = *begin_tokens;
1037 void TokenLexer::PropagateLineStartLeadingSpaceInfo(
Token &
Result) {
bool invokedWithVariadicArgument(const MacroInfo *const MI) const
Returns true if the macro was defined with a variadic (ellipsis) parameter AND was invoked with at le...
bool isWrittenInSameFile(SourceLocation Loc1, SourceLocation Loc2) const
Returns true if the spelling locations for both SourceLocations are part of the same file buffer...
Lexer - This provides a simple interface that turns a text buffer into a stream of tokens...
SourceLocation getLocWithOffset(int Offset) const
Return a source location with the specified offset from this SourceLocation.
void Init(Token &Tok, SourceLocation ILEnd, MacroInfo *MI, MacroArgs *ActualArgs)
Init - Initialize this TokenLexer to expand from the specified macro with the specified argument info...
void sawOpeningParen(SourceLocation LParenLoc)
Call this function each time an lparen is seen.
void setFlagValue(TokenFlags Flag, bool Val)
Set a flag to either true or false.
bool ArgNeedsPreexpansion(const Token *ArgTok, Preprocessor &PP) const
ArgNeedsPreexpansion - If we can prove that the argument won't be affected by pre-expansion, return false.
bool is(tok::TokenKind K) const
is/isNot - Predicates to check if this token is a specific kind, as in "if (Tok.is(tok::l_brace)) {...
bool sawClosingParen()
Call this function each time an rparen is seen.
Defines the SourceManager interface.
static void updateConsecutiveMacroArgTokens(SourceManager &SM, SourceLocation InstLoc, Token *&begin_tokens, Token *end_tokens)
Finds the tokens that are consecutive (from the same FileID) creates a single SLocEntry, and assigns SourceLocations to each token that point to that SLocEntry.
A class for tracking whether we're inside a VA_OPT during a traversal of the tokens of a macro during...
bool hasCharifyBefore() const
Defines the clang::MacroInfo and clang::MacroDirective classes.
bool isBeforeInSLocAddrSpace(SourceLocation LHS, SourceLocation RHS) const
Determines the order of 2 source locations in the "source location address space".
void setFlag(TokenFlags Flag)
Set the specified flag.
bool isVarargsElidedUse() const
isVarargsElidedUse - Return true if this is a C99 style varargs macro invocation and there was no arg...
bool isInSameSLocAddrSpace(SourceLocation LHS, SourceLocation RHS, int *RelativeOffset) const
Return true if both LHS and RHS are in the local source location address space or the loaded one...
bool isAnyIdentifier() const
Return true if this is a raw identifier (when lexing in raw mode) or a non-keyword identifier (when l...
unsigned getNextLocalOffset() const
bool isAnnotation() const
Return true if this is any of tok::annot_* kind tokens.
bool getLeadingSpaceForStringifiedToken() const
tokens_iterator tokens_begin() const
bool isLiteral() const
Return true if this is a "literal", like a numeric constant, string, etc.
bool HandleEndOfTokenLexer(Token &Result)
Callback invoked when the current TokenLexer hits the end of its token stream.
bool hasStringifyOrCharifyBefore() const
One of these records is kept for each identifier that is lexed.
const Token & getEOFTok() const
bool isStr(const char(&Str)[StrLen]) const
Return true if this is the identifier for the specified string.
void setRawIdentifierData(const char *Ptr)
void sawVAOptFollowedByOpeningParens(const SourceLocation VAOptLoc, const unsigned int NumPriorTokens)
Token - This structure provides full information about a lexed token.
void setKind(tok::TokenKind K)
const LangOptions & getLangOpts() const
bool Lex(Token &Tok)
Lex - Lex and return a token from this macro stream.
StringRef getSpelling(SourceLocation loc, SmallVectorImpl< char > &buffer, bool *invalid=nullptr) const
Return the 'spelling' of the token at the given location; does not go up to the spelling location or ...
tok::TokenKind getTokenID() const
If this is a source-language token (e.g.
SourceLocation getExpansionLoc(SourceLocation Loc) const
Given a SourceLocation object Loc, return the expansion location referenced by the ID...
Defines the Diagnostic-related interfaces.
void destroy(Preprocessor &PP)
destroy - Destroy and deallocate the memory for this object.
const Token & getStringifiedArgument(unsigned ArgNo, Preprocessor &PP, SourceLocation ExpansionLocStart, SourceLocation ExpansionLocEnd)
getStringifiedArgument - Compute, cache, and return the specified argument that has been 'stringified...
static bool isWideStringLiteralFromMacro(const Token &FirstTok, const Token &SecondTok)
Checks if two tokens form wide string literal.
static Token StringifyArgument(const Token *ArgToks, Preprocessor &PP, bool Charify, SourceLocation ExpansionLocStart, SourceLocation ExpansionLocEnd)
StringifyArgument - Implement C99 6.10.3.2p2, converting a sequence of tokens into the literal string...
unsigned getNumParams() const
IdentifierInfo * LookUpIdentifierInfo(Token &Identifier) const
Given a tok::raw_identifier token, look up the identifier information for the token and install it in...
Defines the clang::LangOptions interface.
bool isInSLocAddrSpace(SourceLocation Loc, SourceLocation Start, unsigned Length, unsigned *RelativeOffset=nullptr) const
Returns true if Loc is inside the [Start, +Length) chunk of the source location address space...
SourceLocation getVAOptLoc() const
MacroArgs - An instance of this class captures information about the formal arguments specified to a ...
SourceLocation createMacroArgExpansionLoc(SourceLocation Loc, SourceLocation ExpansionLoc, unsigned TokLength)
Return a new SourceLocation that encodes the fact that a token from SpellingLoc should actually be re...
SourceLocation getLocation() const
Return a source location identifier for the specified offset in the current file. ...
Defines the clang::Preprocessor interface.
void HandleMicrosoftCommentPaste(Token &Tok)
When the macro expander pastes together a comment (/##/) in Microsoft mode, this method handles updat...
void IncrementPasteCounter(bool isFast)
Increment the counters for the number of token paste operations performed.
Defines the clang::IdentifierInfo, clang::IdentifierTable, and clang::Selector interfaces.
SourceLocation createExpansionLoc(SourceLocation Loc, SourceLocation ExpansionLocStart, SourceLocation ExpansionLocEnd, unsigned TokLength, int LoadedID=0, unsigned LoadedOffset=0)
Return a new SourceLocation that encodes the fact that a token from SpellingLoc should actually be re...
The result type of a method or function.
SourceManager & getSourceManager() const
bool isVAOptToken(const Token &T) const
Encodes a location in the source.
std::pair< SourceLocation, SourceLocation > getImmediateExpansionRange(SourceLocation Loc) const
Return the start/end of the expansion information for an expansion location.
void setLength(unsigned Len)
IdentifierInfo * getIdentifierInfo() const
bool isInVAOpt() const
Returns true if we have seen the VA_OPT and '(' but before having seen the matching ')'...
bool isPoisoned() const
Return true if this token has been poisoned.
bool isAtStartOfLine() const
isAtStartOfLine - Return true if this token is at the start of a line.
An opaque identifier used by SourceManager which refers to a source file (MemoryBuffer) along with it...
bool isNot(tok::TokenKind K) const
static bool isInvalid(LocType Loc, bool *Invalid)
Dataflow Directional Tag Classes.
bool isValid() const
Return true if this is a valid SourceLocation object.
bool isFunctionLike() const
const std::vector< Token > & getPreExpArgument(unsigned Arg, Preprocessor &PP)
getPreExpArgument - Return the pre-expanded form of the specified argument.
bool isHandleIdentifierCase() const
Return true if the Preprocessor::HandleIdentifier must be called on a token of this identifier...
bool isOneOf(tok::TokenKind K1, tok::TokenKind K2) const
tokens_iterator tokens_end() const
static unsigned getArgLength(const Token *ArgPtr)
getArgLength - Given a pointer to an expanded or unexpanded argument, return the number of tokens...
void sawHashOrHashAtBefore(const bool HasLeadingSpace, const bool IsHashAt)
unsigned getLength() const
Encapsulates the data about a macro definition (e.g.
const char * getLiteralData() const
getLiteralData - For a literal token (numeric constant, string, etc), this returns a pointer to the s...
FileID getFileID(SourceLocation SpellingLoc) const
Return the FileID for a SourceLocation.
int getParameterNum(const IdentifierInfo *Arg) const
Return the parameter number of the specified identifier, or -1 if the identifier is not a formal para...
bool HandleIdentifier(Token &Identifier)
Callback invoked when the lexer reads an identifier and has filled in the tokens IdentifierInfo membe...
unsigned int getNumberOfTokensPriorToVAOpt() const
void CreateString(StringRef Str, Token &Tok, SourceLocation ExpansionLocStart=SourceLocation(), SourceLocation ExpansionLocEnd=SourceLocation())
Plop the specified string into a scratch buffer and set the specified token's location and length to ...
unsigned isNextTokenLParen() const
isNextTokenLParen - If the next token lexed will pop this macro off the expansion stack...
Defines the clang::TokenKind enum and support functions.
void HandlePoisonedIdentifier(Token &Tok)
Display reason for poisoned identifier.
unsigned getDefinitionLength(const SourceManager &SM) const
Get length in characters of the macro definition.
Defines the clang::SourceLocation class and associated facilities.
bool stringifiedInMacro() const
Returns true if this token is formed by macro by stringizing or charizing operator.
void setLocation(SourceLocation L)
void clearFlag(TokenFlags Flag)
Unset the specified flag.
bool isParsingPreprocessorDirective() const
isParsingPreprocessorDirective - Return true if we are in the middle of a preprocessor directive...
bool hasLeadingSpace() const
Return true if this token has whitespace before it.
DiagnosticBuilder Diag(SourceLocation Loc, unsigned DiagID) const
Forwarding function for diagnostics.
This class handles loading and caching of source files into memory.
const Token * getUnexpArgument(unsigned Arg) const
getUnexpArgument - Return a pointer to the first token of the unexpanded token list for the specified...
void startToken()
Reset all flags to cleared.
Engages in a tight little dance with the lexer to efficiently preprocess tokens.