24 #include "llvm/ADT/StringExtras.h" 25 #include "llvm/ADT/StringMap.h" 26 #include "llvm/Support/EndianStream.h" 27 #include "llvm/Support/FileSystem.h" 28 #include "llvm/Support/MemoryBuffer.h" 29 #include "llvm/Support/OnDiskHashTable.h" 30 #include "llvm/Support/Path.h" 34 #define S_ISDIR(x) (((x)&_S_IFDIR)!=0) 37 using namespace clang;
47 Offset TokenData, PPCondData;
53 : TokenData(td), PPCondData(ppcd) {}
55 Offset getTokenOffset()
const {
return TokenData; }
56 Offset getPPCondTableOffset()
const {
return PPCondData; }
60 class PTHEntryKeyVariant {
67 enum { IsFE = 0x1, IsDE = 0x2, IsNoExist = 0x0 }
Kind;
71 PTHEntryKeyVariant(
const FileEntry *fe) : FE(fe),
Kind(IsFE), Data(
nullptr) {}
73 PTHEntryKeyVariant(
FileData *Data, StringRef Path)
74 : PathPtr(Path.data()), PathSize(Path.size()),
Kind(IsDE),
77 explicit PTHEntryKeyVariant(StringRef Path)
78 : PathPtr(Path.data()), PathSize(Path.size()),
Kind(IsNoExist),
81 bool isFile()
const {
return Kind == IsFE; }
83 StringRef getString()
const {
84 return Kind == IsFE ? FE->getName() : StringRef(PathPtr, PathSize);
87 unsigned getKind()
const {
return (
unsigned)
Kind; }
89 void EmitData(raw_ostream& Out) {
90 using namespace llvm::support;
91 endian::Writer<little> LE(Out);
95 llvm::sys::fs::UniqueID UID = FE->getUniqueID();
96 LE.write<uint64_t>(UID.getFile());
97 LE.write<uint64_t>(UID.getDevice());
98 LE.write<uint64_t>(FE->getModificationTime());
99 LE.write<uint64_t>(FE->getSize());
103 LE.write<uint64_t>(Data->
UniqueID.getFile());
104 LE.write<uint64_t>(Data->
UniqueID.getDevice());
105 LE.write<uint64_t>(Data->
ModTime);
106 LE.write<uint64_t>(Data->
Size);
114 unsigned getRepresentationLength()
const {
115 return Kind == IsNoExist ? 0 : 4 * 8;
119 class FileEntryPTHEntryInfo {
121 typedef PTHEntryKeyVariant key_type;
122 typedef key_type key_type_ref;
124 typedef PTHEntry data_type;
125 typedef const PTHEntry& data_type_ref;
127 typedef unsigned hash_value_type;
128 typedef unsigned offset_type;
130 static hash_value_type
ComputeHash(PTHEntryKeyVariant V) {
131 return llvm::HashString(V.getString());
134 static std::pair<unsigned,unsigned>
135 EmitKeyDataLength(raw_ostream& Out, PTHEntryKeyVariant V,
137 using namespace llvm::support;
138 endian::Writer<little> LE(Out);
140 unsigned n = V.getString().size() + 1 + 1;
141 LE.write<uint16_t>(n);
143 unsigned m = V.getRepresentationLength() + (V.isFile() ? 4 + 4 : 0);
144 LE.write<uint8_t>(m);
146 return std::make_pair(n, m);
149 static void EmitKey(raw_ostream& Out, PTHEntryKeyVariant V,
unsigned n){
150 using namespace llvm::support;
152 endian::Writer<little>(Out).write<uint8_t>((
unsigned)V.getKind());
154 Out.write(V.getString().data(), n - 1);
157 static void EmitData(raw_ostream& Out, PTHEntryKeyVariant V,
158 const PTHEntry& E,
unsigned) {
159 using namespace llvm::support;
160 endian::Writer<little> LE(Out);
165 LE.write<uint32_t>(E.getTokenOffset());
166 LE.write<uint32_t>(E.getPPCondTableOffset());
178 OffsetOpt() : valid(
false) {}
179 bool hasOffset()
const {
return valid; }
180 Offset getOffset()
const { assert(valid);
return off; }
181 void setOffset(
Offset o) { off = o; valid =
true; }
185 typedef llvm::OnDiskChainedHashTableGenerator<FileEntryPTHEntryInfo>
PTHMap;
189 typedef llvm::DenseMap<const IdentifierInfo*,uint32_t> IDMap;
190 typedef llvm::StringMap<OffsetOpt, llvm::BumpPtrAllocator> CachedStrsTy;
192 raw_pwrite_stream &Out;
195 std::vector<llvm::StringMapEntry<OffsetOpt>*> StrEntries;
197 CachedStrsTy CachedStrs;
205 void EmitToken(
const Token&
T);
207 void Emit8(uint32_t V) {
208 using namespace llvm::support;
209 endian::Writer<little>(Out).write<uint8_t>(V);
212 void Emit16(uint32_t V) {
213 using namespace llvm::support;
214 endian::Writer<little>(Out).write<uint16_t>(V);
217 void Emit32(uint32_t V) {
218 using namespace llvm::support;
219 endian::Writer<little>(Out).write<uint32_t>(V);
222 void EmitBuf(
const char *Ptr,
unsigned NumBytes) {
223 Out.write(Ptr, NumBytes);
227 using namespace llvm::support;
228 endian::Writer<little>(Out).write<uint16_t>(V.size());
229 EmitBuf(V.data(), V.size());
236 std::pair<Offset, Offset> EmitIdentifierTable();
240 Offset EmitFileTable() {
return PM.Emit(Out); }
242 PTHEntry LexTokens(
Lexer& L);
243 Offset EmitCachedSpellings();
247 : Out(out), PP(pp), idcount(0), CurStrOffset(0) {}
249 PTHMap &getPM() {
return PM; }
259 IDMap::iterator I = IM.find(II);
267 void PTHWriter::EmitToken(
const Token&
T) {
280 auto &E = *CachedStrs.insert(std::make_pair(s, OffsetOpt())).first;
283 if (!E.second.hasOffset()) {
284 E.second.setOffset(CurStrOffset);
285 StrEntries.push_back(&E);
286 CurStrOffset += s.size() + 1;
290 Emit32(E.second.getOffset());
295 Emit32(PP.getSourceManager().getFileOffset(T.
getLocation()));
298 PTHEntry PTHWriter::LexTokens(
Lexer& L) {
301 using namespace llvm::support;
302 endian::Writer<little> LE(Out);
303 uint32_t TokenOff = Out.tell();
304 for (uint64_t N = llvm::OffsetToAlignment(TokenOff, 4); N; --N, ++TokenOff)
305 LE.write<uint8_t>(0);
308 typedef std::vector<std::pair<Offset, unsigned> > PPCondTable;
310 std::vector<unsigned> PPStartCond;
311 bool ParsingPreprocessorDirective =
false;
319 ParsingPreprocessorDirective) {
329 ParsingPreprocessorDirective =
false;
332 if (Tok.
is(tok::raw_identifier)) {
333 PP.LookUpIdentifierInfo(Tok);
341 assert(!ParsingPreprocessorDirective);
358 if (Tok.
isNot(tok::raw_identifier)) {
366 ParsingPreprocessorDirective =
true;
369 case tok::pp_not_keyword:
375 case tok::pp_include:
377 case tok::pp_include_next: {
385 if (Tok.
is(tok::raw_identifier))
386 PP.LookUpIdentifierInfo(Tok);
392 case tok::pp_ifndef: {
395 PPStartCond.push_back(PPCond.size());
396 PPCond.push_back(std::make_pair(HashOff, 0U));
399 case tok::pp_endif: {
403 unsigned index = PPCond.size();
405 assert(!PPStartCond.empty());
406 assert(PPCond.size() > PPStartCond.back());
407 assert(PPCond[PPStartCond.back()].second == 0);
408 PPCond[PPStartCond.back()].second = index;
409 PPStartCond.pop_back();
411 PPCond.push_back(std::make_pair(HashOff, index));
428 unsigned index = PPCond.size();
430 assert(!PPStartCond.empty());
431 assert(PPCond.size() > PPStartCond.back());
432 assert(PPCond[PPStartCond.back()].second == 0);
433 PPCond[PPStartCond.back()].second = index;
434 PPStartCond.pop_back();
436 PPCond.push_back(std::make_pair(HashOff, 0U));
437 PPStartCond.push_back(index);
447 assert(PPStartCond.empty() &&
"Error: imblanced preprocessor conditionals.");
453 Emit32(PPCond.size());
455 for (
unsigned i = 0, e = PPCond.size(); i!=e; ++i) {
456 Emit32(PPCond[i].first - TokenOff);
457 uint32_t x = PPCond[i].second;
458 assert(x != 0 &&
"PPCond entry not backpatched.");
461 Emit32(x == i ? 0 : x);
464 return PTHEntry(TokenOff, PPCondOff);
467 Offset PTHWriter::EmitCachedSpellings() {
469 Offset SpellingsOff = Out.tell();
471 for (std::vector<llvm::StringMapEntry<OffsetOpt>*>::iterator
472 I = StrEntries.begin(), E = StrEntries.end(); I!=E; ++I)
473 EmitBuf((*I)->getKeyData(), (*I)->getKeyLength()+1 );
479 return llvm::support::endian::byte_swap<uint32_t, llvm::support::little>(
X);
482 static void pwrite32le(raw_pwrite_stream &OS, uint32_t Val, uint64_t &Off) {
484 OS.pwrite(reinterpret_cast<const char *>(&LEVal), 4, Off);
490 Out <<
"cfe-pth" <<
'\0';
494 Offset PrologueOffset = Out.tell();
495 for (
unsigned i = 0; i < 4; ++i)
499 if (!MainFile.empty()) {
518 if (llvm::sys::path::is_relative(FE->
getName()))
521 const llvm::MemoryBuffer *B = C.
getBuffer(PP.getDiagnostics(),
SM);
525 const llvm::MemoryBuffer *FromFile = SM.
getBuffer(FID);
526 Lexer L(FID, FromFile, SM, LOpts);
527 PM.insert(FE, LexTokens(L));
531 const std::pair<Offset,Offset> &IdTableOff = EmitIdentifierTable();
534 Offset SpellingOff = EmitCachedSpellings();
537 Offset FileTableOff = EmitFileTable();
540 uint64_t Off = PrologueOffset;
556 StatListener(
PTHMap &pm) : PM(pm) {}
557 ~StatListener()
override {}
560 std::unique_ptr<vfs::File> *F,
564 if (Result == CacheMissing)
565 PM.insert(PTHEntryKeyVariant(Path), PTHEntry());
568 if (llvm::sys::path::is_relative(Path))
571 PM.insert(PTHEntryKeyVariant(&Data, Path), PTHEntry());
585 llvm::sys::fs::make_absolute(MainFilePath);
588 PTHWriter PW(*OS, PP);
591 auto StatCacheOwner = llvm::make_unique<StatListener>(PW.getPM());
592 StatListener *StatCache = StatCacheOwner.get();
604 PW.GeneratePTH(MainFilePath.str());
616 class PTHIdentifierTableTrait {
618 typedef PTHIdKey* key_type;
619 typedef key_type key_type_ref;
621 typedef uint32_t data_type;
622 typedef data_type data_type_ref;
624 typedef unsigned hash_value_type;
625 typedef unsigned offset_type;
627 static hash_value_type
ComputeHash(PTHIdKey* key) {
628 return llvm::HashString(key->II->getName());
631 static std::pair<unsigned,unsigned>
632 EmitKeyDataLength(raw_ostream& Out,
const PTHIdKey* key, uint32_t) {
633 using namespace llvm::support;
634 unsigned n = key->II->getLength() + 1;
635 endian::Writer<little>(Out).write<uint16_t>(n);
636 return std::make_pair(n,
sizeof(uint32_t));
639 static void EmitKey(raw_ostream& Out, PTHIdKey* key,
unsigned n) {
642 key->FileOffset = Out.tell();
643 Out.write(key->II->getNameStart(), n);
646 static void EmitData(raw_ostream& Out, PTHIdKey*, uint32_t pID,
648 using namespace llvm::support;
649 endian::Writer<little>(Out).write<uint32_t>(pID);
659 std::pair<Offset,Offset> PTHWriter::EmitIdentifierTable() {
665 PTHIdKey *IIDMap = (PTHIdKey*)calloc(idcount,
sizeof(PTHIdKey));
668 llvm::OnDiskChainedHashTableGenerator<PTHIdentifierTableTrait> IIOffMap;
671 for (IDMap::iterator I = IM.begin(), E = IM.end(); I != E; ++I) {
674 assert(I->second > 0);
675 assert(I->second-1 < idcount);
676 unsigned idx = I->second-1;
679 IIDMap[idx].II = I->first;
682 IIOffMap.insert(&IIDMap[idx], I->second);
688 Offset StringTableOffset = IIOffMap.Emit(Out);
691 Offset IDOff = Out.tell();
693 for (
unsigned i = 0 ; i < idcount; ++i)
694 Emit32(IIDMap[i].FileOffset);
699 return std::make_pair(IDOff, StringTableOffset);
const FileEntry * OrigEntry
Reference to the file entry representing this ContentCache.
Lexer - This provides a simple interface that turns a text buffer into a stream of tokens...
Defines the clang::FileManager interface and associated types.
bool LexFromRawLexer(Token &Result)
LexFromRawLexer - Lex a token from a designated raw lexer (one with no associated preprocessor object...
bool is(tok::TokenKind K) const
is/isNot - Predicates to check if this token is a specific kind, as in "if (Tok.is(tok::l_brace)) {...
Defines the SourceManager interface.
FileManager & getFileManager() const
fileinfo_iterator fileinfo_end() const
Defines the FileSystemStatCache interface.
tok::TokenKind getKind() const
bool isLiteral() const
Return true if this is a "literal", like a numeric constant, string, etc.
The virtual file system interface.
One of these records is kept for each identifier that is lexed.
One instance of this struct is kept for every file loaded or used.
Token - This structure provides full information about a lexed token.
static void pwrite32le(raw_pwrite_stream &OS, uint32_t Val, uint64_t &Off)
llvm::OnDiskChainedHashTableGenerator< FileEntryPTHEntryInfo > PTHMap
void setKind(tok::TokenKind K)
Keeps track of the various options that can be enabled, which controls the dialect of C or C++ that i...
Abstract interface for introducing a FileManager cache for 'stat' system calls, which is used by prec...
Represents the results of name lookup.
Defines the Diagnostic-related interfaces.
void setParsingPreprocessorDirective(bool f)
Inform the lexer whether or not we are currently lexing a preprocessor directive. ...
void CacheTokens(Preprocessor &PP, raw_pwrite_stream *OS)
Cache tokens for use with PCH. Note that this requires a seekable stream.
llvm::MemoryBuffer * getBuffer(DiagnosticsEngine &Diag, const SourceManager &SM, SourceLocation Loc=SourceLocation(), bool *Invalid=nullptr) const
Returns the memory buffer for the associated content.
const FileEntry * getFileEntryForID(FileID FID) const
Returns the FileEntry record for the provided FileID.
const FunctionProtoType * T
FileID createFileID(const FileEntry *SourceFile, SourceLocation IncludePos, SrcMgr::CharacteristicKind FileCharacter, int LoadedID=0, unsigned LoadedOffset=0)
Create a new FileID that represents the specified file being #included from the specified IncludePosi...
void EnterMainSourceFile()
Enter the specified FileID as the main source file, which implicitly adds the builtin defines etc...
SourceLocation getLocation() const
Return a source location identifier for the specified offset in the current file. ...
Defines the clang::Preprocessor interface.
PPKeywordKind
Provides a namespace for preprocessor keywords which start with a '#' at the beginning of the line...
Defines the clang::IdentifierInfo, clang::IdentifierTable, and clang::Selector interfaces.
fileinfo_iterator fileinfo_begin() const
The result type of a method or function.
SourceManager & getSourceManager() const
static uint32_t swap32le(uint32_t X)
llvm::MemoryBuffer * getBuffer(FileID FID, SourceLocation Loc, bool *Invalid=nullptr) const
Return the buffer for the specified FileID.
void addStatCache(std::unique_ptr< FileSystemStatCache > statCache, bool AtBeginning=false)
Installs the provided FileSystemStatCache object within the FileManager.
Encodes a location in the source.
llvm::sys::fs::UniqueID UniqueID
StringRef getName() const
IdentifierInfo * getIdentifierInfo() const
Cached information about one file (either on disk or in the virtual file system). ...
void setIdentifierInfo(IdentifierInfo *II)
void Lex(Token &Result)
Lex the next token for this preprocessor.
bool isAtStartOfLine() const
isAtStartOfLine - Return true if this token is at the start of a line.
llvm::DenseMap< const FileEntry *, SrcMgr::ContentCache * >::const_iterator fileinfo_iterator
unsigned ComputeHash(Selector Sel)
An opaque identifier used by SourceManager which refers to a source file (MemoryBuffer) along with it...
bool isNot(tok::TokenKind K) const
Dataflow Directional Tag Classes.
void removeStatCache(FileSystemStatCache *statCache)
Removes the specified FileSystemStatCache object from the manager.
FileID getMainFileID() const
Returns the FileID of the main source file.
tok::PPKeywordKind getPPKeywordID() const
Return the preprocessor keyword ID for this identifier.
unsigned getLength() const
const char * getLiteralData() const
getLiteralData - For a literal token (numeric constant, string, etc), this returns a pointer to the s...
void LexIncludeFilename(Token &Result)
After the preprocessor has parsed a #include, lex and (potentially) macro expand the filename...
raw_ostream & EmitString(raw_ostream &o, StringRef s)
X
Add a minimal nested name specifier fixit hint to allow lookup of a tag name from an outer enclosing ...
static Decl::Kind getKind(const Decl *D)
Generate pre-tokenized header.
unsigned getFlags() const
Return the internal represtation of the flags.
void clearFlag(TokenFlags Flag)
Unset the specified flag.
This class handles loading and caching of source files into memory.
Engages in a tight little dance with the lexer to efficiently preprocess tokens.