clang 8.0.0

TokenRewriter.cpp
Go to the documentation of this file.
1 //===- TokenRewriter.cpp - Token-based code rewriting interface -----------===//
2 //
3 // The LLVM Compiler Infrastructure
4 //
5 // This file is distributed under the University of Illinois Open Source
6 // License. See LICENSE.TXT for details.
7 //
8 //===----------------------------------------------------------------------===//
9 //
10 // This file implements the TokenRewriter class, which is used for code
11 // transformations.
12 //
13 //===----------------------------------------------------------------------===//
14
15 #include "clang/Rewrite/Core/TokenRewriter.h"
16 #include "clang/Basic/SourceManager.h"
17 #include "clang/Lex/Lexer.h"
18 #include "clang/Lex/ScratchBuffer.h"
19 #include "clang/Lex/Token.h"
20 #include <cassert>
21 #include <cstring>
22 #include <map>
23 #include <utility>
24
25 using namespace clang;
26
27 TokenRewriter::TokenRewriter(FileID FID, SourceManager &SM,
28 const LangOptions &LangOpts) {
29 ScratchBuf.reset(new ScratchBuffer(SM));
30
31 // Create a lexer to lex all the tokens of the main file in raw mode.
32 const llvm::MemoryBuffer *FromFile = SM.getBuffer(FID);
33 Lexer RawLex(FID, FromFile, SM, LangOpts);
34
35 // Return all comments and whitespace as tokens.
36 RawLex.SetKeepWhitespaceMode(true);
37
38 // Lex the file, populating our datastructures.
39 Token RawTok;
40 RawLex.LexFromRawLexer(RawTok);
41 while (RawTok.isNot(tok::eof)) {
42 #if 0
43 if (Tok.is(tok::raw_identifier)) {
44 // Look up the identifier info for the token. This should use
45 // IdentifierTable directly instead of PP.
46 PP.LookUpIdentifierInfo(Tok);
47 }
48 #endif
49
50 AddToken(RawTok, TokenList.end());
51 RawLex.LexFromRawLexer(RawTok);
52 }
53 }
54
55 TokenRewriter::~TokenRewriter() = default;
56
57 /// RemapIterator - Convert from token_iterator (a const iterator) to
58 /// TokenRefTy (a non-const iterator).
59 TokenRewriter::TokenRefTy TokenRewriter::RemapIterator(token_iterator I) {
60 if (I == token_end()) return TokenList.end();
61
62 // FIXME: This is horrible, we should use our own list or something to avoid
63 // this.
64 std::map<SourceLocation, TokenRefTy>::iterator MapIt =
65 TokenAtLoc.find(I->getLocation());
66 assert(MapIt != TokenAtLoc.end() && "iterator not in rewriter?");
67 return MapIt->second;
68 }
69
70 /// AddToken - Add the specified token into the Rewriter before the other
71 /// position.
72 TokenRewriter::TokenRefTy
73 TokenRewriter::AddToken(const Token &T, TokenRefTy Where) {
74 Where = TokenList.insert(Where, T);
75
76 bool InsertSuccess = TokenAtLoc.insert(std::make_pair(T.getLocation(),
77 Where)).second;
78 assert(InsertSuccess && "Token location already in rewriter!");
79 (void)InsertSuccess;
80 return Where;
81 }
82
83 TokenRewriter::token_iterator
84 TokenRewriter::AddTokenBefore(token_iterator I, const char *Val) {
85 unsigned Len = strlen(Val);
86
87 // Plop the string into the scratch buffer, then create a token for this
88 // string.
89 Token Tok;
90 Tok.startToken();
91 const char *Spelling;
92 Tok.setLocation(ScratchBuf->getToken(Val, Len, Spelling));
93 Tok.setLength(Len);
94
95 // TODO: Form a whole lexer around this and relex the token! For now, just
96 // set kind to tok::unknown.
97 Tok.setKind(tok::unknown);
98
99 return AddToken(Tok, RemapIterator(I));
100 }
ScratchBuffer.h
clang::Lexer
Lexer - This provides a simple interface that turns a text buffer into a stream of tokens...
Definition: Lexer.h:77
clang::Lexer::LexFromRawLexer
bool LexFromRawLexer(Token &Result)
LexFromRawLexer - Lex a token from a designated raw lexer (one with no associated preprocessor object...
Definition: Lexer.h:195
SourceManager.h
Defines the SourceManager interface.
clang::TokenRewriter::token_end
token_iterator token_end() const
Definition: TokenRewriter.h:60
clang::Token
Token - This structure provides full information about a lexed token.
Definition: Token.h:35
clang::Token::setKind
void setKind(tok::TokenKind K)
Definition: Token.h:91
clang::LangOptions
Keeps track of the various options that can be enabled, which controls the dialect of C or C++ that i...
Definition: LangOptions.h:50
clang::TokenRewriter::~TokenRewriter
~TokenRewriter()
clang::format::Tok
const FormatToken & Tok
Definition: BreakableToken.h:244
TokenRewriter.h
clang::Token::getLocation
SourceLocation getLocation() const
Return a source location identifier for the specified offset in the current file. ...
Definition: Token.h:124
Lexer.h
SM
const SourceManager & SM
Definition: Format.cpp:1490
clang::TokenRewriter::AddTokenBefore
token_iterator AddTokenBefore(token_iterator I, const char *Val)
Definition: TokenRewriter.cpp:84
clang::TokenRewriter::token_iterator
std::list< Token >::const_iterator token_iterator
Definition: TokenRewriter.h:57
clang::SourceManager::getBuffer
llvm::MemoryBuffer * getBuffer(FileID FID, SourceLocation Loc, bool *Invalid=nullptr) const
Return the buffer for the specified FileID.
Definition: SourceManager.h:955
clang::Token::setLength
void setLength(unsigned Len)
Definition: Token.h:133
clang::format::FormatToken::is
bool is(tok::TokenKind Kind) const
Definition: FormatToken.h:307
clang::TokenRewriter::TokenRewriter
TokenRewriter(FileID FID, SourceManager &SM, const LangOptions &LO)
TokenRewriter - This creates a TokenRewriter for the file with the specified FileID.
Definition: TokenRewriter.cpp:27
clang::FileID
An opaque identifier used by SourceManager which refers to a source file (MemoryBuffer) along with it...
Definition: SourceLocation.h:40
clang::Token::isNot
bool isNot(tok::TokenKind K) const
Definition: Token.h:96
clang
Dataflow Directional Tag Classes.
Definition: CFGReachabilityAnalysis.h:22
clang::comments::tok::eof
Definition: CommentLexer.h:34
clang::Token::setLocation
void setLocation(SourceLocation L)
Definition: Token.h:132
Token.h
clang::Lexer::SetKeepWhitespaceMode
void SetKeepWhitespaceMode(bool Val)
SetKeepWhitespaceMode - This method lets clients enable or disable whitespace retention mode...
Definition: Lexer.h:213
clang::ScratchBuffer
ScratchBuffer - This class exposes a simple interface for the dynamic construction of tokens...
Definition: ScratchBuffer.h:25
clang::SourceManager
This class handles loading and caching of source files into memory.
Definition: SourceManager.h:582
clang::Token::startToken
void startToken()
Reset all flags to cleared.
Definition: Token.h:169