24 using namespace clang;
31 const llvm::MemoryBuffer *FromFile = SM.
getBuffer(FID);
32 Lexer RawLex(FID, FromFile, SM, LangOpts);
42 if (
Tok.
is(tok::raw_identifier)) {
45 PP.LookUpIdentifierInfo(
Tok);
49 AddToken(RawTok, TokenList.end());
58 TokenRewriter::TokenRefTy TokenRewriter::RemapIterator(
token_iterator I) {
59 if (I ==
token_end())
return TokenList.end();
63 std::map<SourceLocation, TokenRefTy>::iterator MapIt =
64 TokenAtLoc.find(I->getLocation());
65 assert(MapIt != TokenAtLoc.end() &&
"iterator not in rewriter?");
71 TokenRewriter::TokenRefTy
72 TokenRewriter::AddToken(
const Token &T, TokenRefTy Where) {
73 Where = TokenList.insert(Where, T);
75 bool InsertSuccess = TokenAtLoc.insert(std::make_pair(T.
getLocation(),
77 assert(InsertSuccess &&
"Token location already in rewriter!");
84 unsigned Len = strlen(Val);
91 Tok.
setLocation(ScratchBuf->getToken(Val, Len, Spelling));
98 return AddToken(Tok, RemapIterator(I));
Lexer - This provides a simple interface that turns a text buffer into a stream of tokens...
bool LexFromRawLexer(Token &Result)
LexFromRawLexer - Lex a token from a designated raw lexer (one with no associated preprocessor object...
Defines the SourceManager interface.
token_iterator token_end() const
Token - This structure provides full information about a lexed token.
void setKind(tok::TokenKind K)
Keeps track of the various options that can be enabled, which controls the dialect of C or C++ that i...
SourceLocation getLocation() const
Return a source location identifier for the specified offset in the current file. ...
token_iterator AddTokenBefore(token_iterator I, const char *Val)
std::list< Token >::const_iterator token_iterator
void setLength(unsigned Len)
TokenRewriter(FileID FID, SourceManager &SM, const LangOptions &LO)
TokenRewriter - This creates a TokenRewriter for the file with the specified FileID.
An opaque identifier used by SourceManager which refers to a source file (MemoryBuffer) along with it...
bool isNot(tok::TokenKind K) const
const llvm::MemoryBuffer * getBuffer(FileID FID, SourceLocation Loc, bool *Invalid=nullptr) const
Return the buffer for the specified FileID.
Dataflow Directional Tag Classes.
void setLocation(SourceLocation L)
void SetKeepWhitespaceMode(bool Val)
SetKeepWhitespaceMode - This method lets clients enable or disable whitespace retention mode...
ScratchBuffer - This class exposes a simple interface for the dynamic construction of tokens...
This class handles loading and caching of source files into memory.
void startToken()
Reset all flags to cleared.