27 #include "llvm/ADT/ArrayRef.h" 28 #include "llvm/ADT/SmallString.h" 29 #include "llvm/ADT/SmallVector.h" 30 #include "llvm/ADT/iterator_range.h" 34 using namespace clang;
52 NextTokGetsSpace =
false;
55 DisableMacroExpansion =
false;
64 assert(Tokens[0].getLocation().isValid());
65 assert((Tokens[0].getLocation().isFileID() || Tokens[0].is(tok::comment)) &&
66 "Macro defined in macro?");
67 assert(ExpandLocStart.
isValid());
84 ExpandFunctionArguments();
95 bool disableMacroExpansion,
bool ownsTokens,
97 assert(!isReinject || disableMacroExpansion);
103 ActualArgs =
nullptr;
105 OwnsTokens = ownsTokens;
106 DisableMacroExpansion = disableMacroExpansion;
107 IsReinject = isReinject;
111 AtStartOfLine =
false;
112 HasLeadingSpace =
false;
113 NextTokGetsSpace =
false;
124 void TokenLexer::destroy() {
134 if (ActualArgs) ActualArgs->
destroy(PP);
137 bool TokenLexer::MaybeRemoveCommaBeforeVaArgs(
147 if (!HasPasteOperator && !PP.
getLangOpts().MSVCCompat)
160 if (ResultToks.empty() || !ResultToks.back().is(tok::comma))
164 if (HasPasteOperator)
165 PP.
Diag(ResultToks.back().getLocation(), diag::ext_paste_comma);
168 ResultToks.pop_back();
170 if (!ResultToks.empty()) {
175 if (ResultToks.back().is(tok::hashhash))
176 ResultToks.pop_back();
183 NextTokGetsSpace =
false;
187 void TokenLexer::stringifyVAOPTContents(
191 const unsigned int NumVAOptTokens = ResultToks.size() - NumToksPriorToVAOpt;
192 Token *
const VAOPTTokens =
193 NumVAOptTokens ? &ResultToks[NumToksPriorToVAOpt] :
nullptr;
200 for (
unsigned int CurTokenIdx = 0; CurTokenIdx != NumVAOptTokens;
202 if (VAOPTTokens[CurTokenIdx].is(tok::hashhash)) {
203 assert(CurTokenIdx != 0 &&
204 "Can not have __VAOPT__ contents begin with a ##");
205 Token &LHS = VAOPTTokens[CurTokenIdx - 1];
206 pasteTokens(LHS, llvm::makeArrayRef(VAOPTTokens, NumVAOptTokens),
209 ConcatenatedVAOPTResultToks.back() = LHS;
210 if (CurTokenIdx == NumVAOptTokens)
213 ConcatenatedVAOPTResultToks.push_back(VAOPTTokens[CurTokenIdx]);
216 ConcatenatedVAOPTResultToks.push_back(VCtx.
getEOFTok());
225 getExpansionLocForMacroDefLoc(VAOPTClosingParenLoc);
229 ExpansionLocStartWithinMacro, ExpansionLocEndWithinMacro);
236 ResultToks.resize(NumToksPriorToVAOpt + 1);
237 ResultToks.back() = StringifiedVAOPT;
242 void TokenLexer::ExpandFunctionArguments() {
248 bool MadeChange =
false;
254 for (
unsigned I = 0, E = NumTokens; I != E; ++I) {
255 const Token &CurTok = Tokens[I];
262 if (I != 0 && !Tokens[I-1].is(tok::hashhash) && CurTok.
hasLeadingSpace())
263 NextTokGetsSpace =
true;
267 assert(Tokens[I + 1].is(tok::l_paren) &&
268 "__VA_OPT__ must be followed by '('");
290 if (Tokens[I].is(tok::l_paren))
298 if (!CalledWithVariadicArguments.hasValue()) {
299 CalledWithVariadicArguments =
302 if (!*CalledWithVariadicArguments) {
322 stringifyVAOPTContents(ResultToks, VCtx,
323 Tokens[I].getLocation());
332 if (ResultToks.size() && ResultToks.back().is(tok::hashhash)) {
333 ResultToks.pop_back();
334 }
else if ((I + 1 != E) && Tokens[I + 1].is(tok::hashhash)) {
347 "no token paste before __VA_OPT__");
348 ResultToks.erase(ResultToks.begin() +
354 Tokens[I + 1].
is(tok::hashhash)) {
369 if (CurTok.
isOneOf(tok::hash, tok::hashat)) {
371 assert((ArgNo != -1 || VCtx.
isVAOptToken(Tokens[I + 1])) &&
372 "Token following # is not an argument or __VA_OPT__!");
377 CurTok.
is(tok::hashat));
382 getExpansionLocForMacroDefLoc(CurTok.
getLocation());
384 getExpansionLocForMacroDefLoc(Tokens[I+1].getLocation());
386 bool Charify = CurTok.
is(tok::hashat);
389 UnexpArg, PP, Charify, ExpansionLocStart, ExpansionLocEnd);
394 if (NextTokGetsSpace)
397 ResultToks.push_back(Res);
400 NextTokGetsSpace =
false;
405 bool NonEmptyPasteBefore =
406 !ResultToks.empty() && ResultToks.back().is(tok::hashhash);
407 bool PasteBefore = I != 0 && Tokens[I-1].
is(tok::hashhash);
408 bool PasteAfter = I+1 != E && Tokens[I+1].
is(tok::hashhash);
409 bool RParenAfter = I+1 != E && Tokens[I+1].
is(tok::r_paren);
411 assert((!NonEmptyPasteBefore || PasteBefore || VCtx.
isInVAOpt()) &&
412 "unexpected ## in ResultToks");
420 ResultToks.push_back(CurTok);
422 if (NextTokGetsSpace) {
424 NextTokGetsSpace =
false;
425 }
else if (PasteBefore && !NonEmptyPasteBefore)
440 MaybeRemoveCommaBeforeVaArgs(ResultToks,
448 if (!PasteBefore && !PasteAfter) {
449 const Token *ResultArgToks;
457 ResultArgToks = ArgTok;
461 size_t FirstResult = ResultToks.size();
463 ResultToks.append(ResultArgToks, ResultArgToks+NumToks);
470 ResultToks.back().is(tok::comma))
475 for (
Token &
Tok : llvm::make_range(ResultToks.begin() + FirstResult,
477 if (
Tok.
is(tok::hashhash))
478 Tok.setKind(tok::unknown);
483 ResultToks.begin()+FirstResult,
493 NextTokGetsSpace =
false;
498 if (NonEmptyPasteBefore) {
501 assert(VCtx.
isInVAOpt() &&
"should only happen inside a __VA_OPT__");
515 bool VaArgsPseudoPaste =
false;
520 if (NonEmptyPasteBefore && ResultToks.size() >= 2 &&
521 ResultToks[ResultToks.size()-2].is(tok::comma) &&
524 VaArgsPseudoPaste =
true;
526 PP.
Diag(ResultToks.pop_back_val().getLocation(), diag::ext_paste_comma);
529 ResultToks.append(ArgToks, ArgToks+NumToks);
533 for (
Token &
Tok : llvm::make_range(ResultToks.end() - NumToks,
535 if (
Tok.
is(tok::hashhash))
536 Tok.setKind(tok::unknown);
539 if (ExpandLocStart.
isValid()) {
541 ResultToks.end()-NumToks, ResultToks.end());
548 if (!VaArgsPseudoPaste) {
551 ResultToks[ResultToks.size() - NumToks].setFlagValue(
555 NextTokGetsSpace =
false;
577 if (NonEmptyPasteBefore) {
578 assert(ResultToks.back().is(tok::hashhash));
585 ResultToks.pop_back();
595 MaybeRemoveCommaBeforeVaArgs(ResultToks,
602 assert(!OwnsTokens &&
"This would leak if we already own the token list");
604 NumTokens = ResultToks.size();
607 Tokens = PP.cacheMacroExpandedTokens(
this, ResultToks);
616 const Token &SecondTok) {
617 return FirstTok.
is(tok::identifier) &&
633 if (CurTokenIdx == 0)
642 bool isFirstToken = CurTokenIdx == 0;
645 Tok = Tokens[CurTokenIdx++];
649 bool TokenIsFromPaste =
false;
653 if (!isAtEnd() && Macro &&
654 (Tokens[CurTokenIdx].is(tok::hashhash) ||
662 if (pasteTokens(Tok))
665 TokenIsFromPaste =
true;
673 if (ExpandLocStart.
isValid() &&
677 if (Tok.
is(tok::comment)) {
683 instLoc = getExpansionLocForMacroDefLoc(Tok.
getLocation());
700 AtStartOfLine =
false;
701 HasLeadingSpace =
false;
725 bool TokenLexer::pasteTokens(
Token &
Tok) {
726 return pasteTokens(Tok, llvm::makeArrayRef(Tokens, NumTokens), CurTokenIdx);
734 unsigned int &CurIdx) {
735 assert(CurIdx > 0 &&
"## can not be the first token within tokens");
736 assert((TokenStream[CurIdx].is(tok::hashhash) ||
739 "Token at this Index must be ## or part of the MSVC 'L " 740 "#macro-arg' pasting pair");
745 if (PP.
getLangOpts().MicrosoftExt && (CurIdx >= 2) &&
746 TokenStream[CurIdx - 2].is(tok::hashhash))
750 const char *ResultTokStrPtr =
nullptr;
754 auto IsAtEnd = [&TokenStream, &CurIdx] {
755 return TokenStream.size() == CurIdx;
760 PasteOpLoc = TokenStream[CurIdx].getLocation();
761 if (TokenStream[CurIdx].is(tok::hashhash))
763 assert(!IsAtEnd() &&
"No token on the RHS of a paste operator!");
766 const Token &RHS = TokenStream[CurIdx];
773 const char *BufPtr = &Buffer[0];
774 bool Invalid =
false;
775 unsigned LHSLen = PP.
getSpelling(LHSTok, BufPtr, &Invalid);
776 if (BufPtr != &Buffer[0])
777 memcpy(&Buffer[0], BufPtr, LHSLen);
781 BufPtr = Buffer.data() + LHSLen;
782 unsigned RHSLen = PP.
getSpelling(RHS, BufPtr, &Invalid);
785 if (RHSLen && BufPtr != &Buffer[LHSLen])
787 memcpy(&Buffer[LHSLen], BufPtr, RHSLen);
790 Buffer.resize(LHSLen+RHSLen);
799 ResultTokTmp.
setKind(tok::string_literal);
812 Result.
setKind(tok::raw_identifier);
820 "Should be a raw location into scratch buffer");
822 FileID LocFileID = SourceMgr.getFileID(ResultTokLoc);
824 bool Invalid =
false;
825 const char *ScratchBufStart
826 = SourceMgr.getBufferData(LocFileID, &Invalid).data();
832 Lexer TL(SourceMgr.getLocForStartOfFile(LocFileID),
834 ResultTokStrPtr, ResultTokStrPtr+LHSLen+RHSLen);
840 bool isInvalid = !TL.LexFromRawLexer(Result);
858 if (PP.
getLangOpts().MicrosoftExt && LHSTok.
is(tok::slash) &&
859 RHS.
is(tok::slash)) {
860 HandleMicrosoftCommentPaste(LHSTok, Loc);
870 : diag::err_pp_bad_paste)
880 if (Result.
is(tok::hashhash))
891 }
while (!IsAtEnd() && TokenStream[CurIdx].is(tok::hashhash));
902 StartLoc = getExpansionLocForMacroDefLoc(StartLoc);
904 EndLoc = getExpansionLocForMacroDefLoc(EndLoc);
906 while (SM.
getFileID(StartLoc) != MacroFID)
917 if (LHSTok.
is(tok::raw_identifier)) {
932 return Tokens[CurTokenIdx].
is(tok::l_paren);
938 return Tokens[NumTokens-1].
is(tok::eod) && !isAtEnd();
947 PP.
Diag(OpLoc, diag::ext_comment_paste_microsoft);
954 assert(Macro &&
"Token streams can't paste comments");
965 TokenLexer::getExpansionLocForMacroDefLoc(
SourceLocation loc)
const {
966 assert(ExpandLocStart.
isValid() && MacroExpansionStart.
isValid() &&
967 "Not appropriate for token streams");
971 assert(SM.isInSLocAddrSpace(loc, MacroDefStart, MacroDefLength) &&
972 "Expected loc to come from the macro definition");
974 unsigned relativeOffset = 0;
990 Token *&begin_tokens,
991 Token * end_tokens) {
992 assert(begin_tokens < end_tokens);
1007 Token *NextTok = begin_tokens + 1;
1008 for (; NextTok < end_tokens; ++NextTok) {
1018 if (RelOffs < 0 || RelOffs > 50)
1029 Token &LastConsecutiveTok = *(NextTok-1);
1030 int LastRelOffs = 0;
1033 unsigned FullLength = LastRelOffs + LastConsecutiveTok.
getLength();
1041 for (; begin_tokens < NextTok; ++begin_tokens) {
1042 Token &Tok = *begin_tokens;
1054 void TokenLexer::updateLocForMacroArgTokens(
SourceLocation ArgIdSpellLoc,
1055 Token *begin_tokens,
1056 Token *end_tokens) {
1060 getExpansionLocForMacroDefLoc(ArgIdSpellLoc);
1062 while (begin_tokens < end_tokens) {
1064 if (end_tokens - begin_tokens == 1) {
1065 Token &Tok = *begin_tokens;
1076 void TokenLexer::PropagateLineStartLeadingSpaceInfo(
Token &
Result) {
bool isWrittenInSameFile(SourceLocation Loc1, SourceLocation Loc2) const
Returns true if the spelling locations for both SourceLocations are part of the same file buffer...
Lexer - This provides a simple interface that turns a text buffer into a stream of tokens...
SourceLocation getLocWithOffset(int Offset) const
Return a source location with the specified offset from this SourceLocation.
void sawOpeningParen(SourceLocation LParenLoc)
Call this function each time an lparen is seen.
void setFlagValue(TokenFlags Flag, bool Val)
Set a flag to either true or false.
bool ArgNeedsPreexpansion(const Token *ArgTok, Preprocessor &PP) const
ArgNeedsPreexpansion - If we can prove that the argument won't be affected by pre-expansion, return false.
bool is(tok::TokenKind K) const
is/isNot - Predicates to check if this token is a specific kind, as in "if (Tok.is(tok::l_brace)) {...
bool sawClosingParen()
Call this function each time an rparen is seen.
Defines the SourceManager interface.
static void updateConsecutiveMacroArgTokens(SourceManager &SM, SourceLocation InstLoc, Token *&begin_tokens, Token *end_tokens)
Finds the tokens that are consecutive (from the same FileID) creates a single SLocEntry, and assigns SourceLocations to each token that point to that SLocEntry.
A class for tracking whether we're inside a VA_OPT during a traversal of the tokens of a macro during...
bool hasCharifyBefore() const
Defines the clang::MacroInfo and clang::MacroDirective classes.
bool isBeforeInSLocAddrSpace(SourceLocation LHS, SourceLocation RHS) const
Determines the order of 2 source locations in the "source location address space".
void setFlag(TokenFlags Flag)
Set the specified flag.
bool isVarargsElidedUse() const
isVarargsElidedUse - Return true if this is a C99 style varargs macro invocation and there was no arg...
bool isInSameSLocAddrSpace(SourceLocation LHS, SourceLocation RHS, int *RelativeOffset) const
Return true if both LHS and RHS are in the local source location address space or the loaded one...
bool isAnyIdentifier() const
Return true if this is a raw identifier (when lexing in raw mode) or a non-keyword identifier (when l...
unsigned getNextLocalOffset() const
bool isAnnotation() const
Return true if this is any of tok::annot_* kind tokens.
bool getLeadingSpaceForStringifiedToken() const
tokens_iterator tokens_begin() const
bool isLiteral() const
Return true if this is a "literal", like a numeric constant, string, etc.
bool HandleEndOfTokenLexer(Token &Result)
Callback invoked when the current TokenLexer hits the end of its token stream.
bool hasStringifyOrCharifyBefore() const
One of these records is kept for each identifier that is lexed.
const Token & getEOFTok() const
SourceLocation getBegin() const
bool isStr(const char(&Str)[StrLen]) const
Return true if this is the identifier for the specified string.
void setRawIdentifierData(const char *Ptr)
void sawVAOptFollowedByOpeningParens(const SourceLocation VAOptLoc, const unsigned int NumPriorTokens)
Token - This structure provides full information about a lexed token.
void setKind(tok::TokenKind K)
const LangOptions & getLangOpts() const
bool Lex(Token &Tok)
Lex and return a token from this macro stream.
StringRef getSpelling(SourceLocation loc, SmallVectorImpl< char > &buffer, bool *invalid=nullptr) const
Return the 'spelling' of the token at the given location; does not go up to the spelling location or ...
tok::TokenKind getTokenID() const
If this is a source-language token (e.g.
SourceLocation getExpansionLoc(SourceLocation Loc) const
Given a SourceLocation object Loc, return the expansion location referenced by the ID...
Defines the Diagnostic-related interfaces.
void destroy(Preprocessor &PP)
destroy - Destroy and deallocate the memory for this object.
void Init(Token &Tok, SourceLocation ELEnd, MacroInfo *MI, MacroArgs *Actuals)
Initialize this TokenLexer to expand from the specified macro with the specified argument information...
static bool isWideStringLiteralFromMacro(const Token &FirstTok, const Token &SecondTok)
Checks if two tokens form wide string literal.
static Token StringifyArgument(const Token *ArgToks, Preprocessor &PP, bool Charify, SourceLocation ExpansionLocStart, SourceLocation ExpansionLocEnd)
StringifyArgument - Implement C99 6.10.3.2p2, converting a sequence of tokens into the literal string...
unsigned getNumParams() const
IdentifierInfo * LookUpIdentifierInfo(Token &Identifier) const
Given a tok::raw_identifier token, look up the identifier information for the token and install it in...
bool invokedWithVariadicArgument(const MacroInfo *const MI, Preprocessor &PP)
Returns true if the macro was defined with a variadic (ellipsis) parameter AND was invoked with at le...
Defines the clang::LangOptions interface.
bool isInSLocAddrSpace(SourceLocation Loc, SourceLocation Start, unsigned Length, unsigned *RelativeOffset=nullptr) const
Returns true if Loc is inside the [Start, +Length) chunk of the source location address space...
SourceLocation getVAOptLoc() const
MacroArgs - An instance of this class captures information about the formal arguments specified to a ...
SourceLocation createMacroArgExpansionLoc(SourceLocation Loc, SourceLocation ExpansionLoc, unsigned TokLength)
Return a new SourceLocation that encodes the fact that a token from SpellingLoc should actually be re...
SourceLocation getLocation() const
Return a source location identifier for the specified offset in the current file. ...
Defines the clang::Preprocessor interface.
void HandleMicrosoftCommentPaste(Token &Tok)
When the macro expander pastes together a comment (/##/) in Microsoft mode, this method handles updat...
void IncrementPasteCounter(bool isFast)
Increment the counters for the number of token paste operations performed.
Defines the clang::IdentifierInfo, clang::IdentifierTable, and clang::Selector interfaces.
CharSourceRange getImmediateExpansionRange(SourceLocation Loc) const
Return the start/end of the expansion information for an expansion location.
The result type of a method or function.
void HandlePoisonedIdentifier(Token &Identifier)
Display reason for poisoned identifier.
SourceManager & getSourceManager() const
bool isVAOptToken(const Token &T) const
Encodes a location in the source.
void setLength(unsigned Len)
SourceLocation createExpansionLoc(SourceLocation Loc, SourceLocation ExpansionLocStart, SourceLocation ExpansionLocEnd, unsigned TokLength, bool ExpansionIsTokenRange=true, int LoadedID=0, unsigned LoadedOffset=0)
Return a new SourceLocation that encodes the fact that a token from SpellingLoc should actually be re...
bool beginsWithPlaceholder() const
IdentifierInfo * getIdentifierInfo() const
bool isInVAOpt() const
Returns true if we have seen the VA_OPT and '(' but before having seen the matching ')'...
void hasPlaceholderBeforeRParen()
bool isPoisoned() const
Return true if this token has been poisoned.
bool isAtStartOfLine() const
isAtStartOfLine - Return true if this token is at the start of a line.
__DEVICE__ void * memcpy(void *__a, const void *__b, size_t __c)
An opaque identifier used by SourceManager which refers to a source file (MemoryBuffer) along with it...
bool isNot(tok::TokenKind K) const
static bool isInvalid(LocType Loc, bool *Invalid)
Dataflow Directional Tag Classes.
bool isValid() const
Return true if this is a valid SourceLocation object.
bool isFunctionLike() const
const std::vector< Token > & getPreExpArgument(unsigned Arg, Preprocessor &PP)
getPreExpArgument - Return the pre-expanded form of the specified argument.
bool isHandleIdentifierCase() const
Return true if the Preprocessor::HandleIdentifier must be called on a token of this identifier...
bool isOneOf(tok::TokenKind K1, tok::TokenKind K2) const
tokens_iterator tokens_end() const
static unsigned getArgLength(const Token *ArgPtr)
getArgLength - Given a pointer to an expanded or unexpanded argument, return the number of tokens...
void hasPlaceholderAfterHashhashAtStart()
bool endsWithPlaceholder() const
void sawHashOrHashAtBefore(const bool HasLeadingSpace, const bool IsHashAt)
unsigned getLength() const
Encapsulates the data about a macro definition (e.g.
const char * getLiteralData() const
getLiteralData - For a literal token (numeric constant, string, etc), this returns a pointer to the s...
FileID getFileID(SourceLocation SpellingLoc) const
Return the FileID for a SourceLocation.
int getParameterNum(const IdentifierInfo *Arg) const
Return the parameter number of the specified identifier, or -1 if the identifier is not a formal para...
bool HandleIdentifier(Token &Identifier)
Callback invoked when the lexer reads an identifier and has filled in the tokens IdentifierInfo membe...
unsigned int getNumberOfTokensPriorToVAOpt() const
void CreateString(StringRef Str, Token &Tok, SourceLocation ExpansionLocStart=SourceLocation(), SourceLocation ExpansionLocEnd=SourceLocation())
Plop the specified string into a scratch buffer and set the specified token's location and length to ...
SourceLocation getEnd() const
unsigned isNextTokenLParen() const
If the next token lexed will pop this macro off the expansion stack, return 2.
Defines the clang::TokenKind enum and support functions.
unsigned getDefinitionLength(const SourceManager &SM) const
Get length in characters of the macro definition.
Defines the clang::SourceLocation class and associated facilities.
bool stringifiedInMacro() const
Returns true if this token is formed by macro by stringizing or charizing operator.
void setLocation(SourceLocation L)
void clearFlag(TokenFlags Flag)
Unset the specified flag.
bool isParsingPreprocessorDirective() const
isParsingPreprocessorDirective - Return true if we are in the middle of a preprocessor directive...
bool hasLeadingSpace() const
Return true if this token has whitespace before it.
DiagnosticBuilder Diag(SourceLocation Loc, unsigned DiagID) const
Forwarding function for diagnostics.
This class handles loading and caching of source files into memory.
const Token * getUnexpArgument(unsigned Arg) const
getUnexpArgument - Return a pointer to the first token of the unexpanded token list for the specified...
void startToken()
Reset all flags to cleared.
Engages in a tight little dance with the lexer to efficiently preprocess tokens.