25 #include "llvm/ADT/STLExtras.h" 26 #include "llvm/ADT/StringExtras.h" 27 #include "llvm/ADT/StringRef.h" 28 #include "llvm/Support/Endian.h" 29 #include "llvm/Support/ErrorOr.h" 30 #include "llvm/Support/FileSystem.h" 31 #include "llvm/Support/MemoryBuffer.h" 32 #include "llvm/Support/OnDiskHashTable.h" 41 using namespace clang;
52 CurPPCondPtr(ppcond), PTHMgr(PM) {
60 using namespace llvm::support;
63 const unsigned char *CurPtrShadow = CurPtr;
66 unsigned Word0 = endian::readNext<uint32_t, little, aligned>(CurPtrShadow);
68 endian::readNext<uint32_t, little, aligned>(CurPtrShadow);
70 endian::readNext<uint32_t, little, aligned>(CurPtrShadow);
74 uint32_t Len = Word0 >> 16;
76 CurPtr = CurPtrShadow;
86 Tok.
setLocation(FileStartLoc.getLocWithOffset(FileOffset));
91 Tok.
setLiteralData((
const char*) (PTHMgr.SpellingBase + IdentifierID));
93 else if (IdentifierID) {
119 return LexEndOfFile(Tok);
130 if (TKind == tok::eod) {
155 diag::err_pp_unterminated_conditional);
172 "Must be in a preprocessing directive!");
181 const unsigned char* p = CurPtr;
200 using namespace llvm::support;
202 assert(CurPPCondPtr &&
"No cached PP conditional information.");
203 assert(LastHashTokPtr &&
"No known '#' token.");
205 const unsigned char *HashEntryI =
nullptr;
210 uint32_t
Offset = endian::readNext<uint32_t, little, aligned>(CurPPCondPtr);
213 TableIdx = endian::readNext<uint32_t, little, aligned>(CurPPCondPtr);
216 HashEntryI = TokBuf +
Offset;
222 if (HashEntryI < LastHashTokPtr && TableIdx) {
226 const unsigned char* NextPPCondPtr =
227 PPCond + TableIdx*(
sizeof(uint32_t)*2);
228 assert(NextPPCondPtr >= CurPPCondPtr);
230 const unsigned char *HashEntryJ =
231 TokBuf + endian::readNext<uint32_t, little, aligned>(NextPPCondPtr);
233 if (HashEntryJ <= LastHashTokPtr) {
235 HashEntryI = HashEntryJ;
236 TableIdx = endian::readNext<uint32_t, little, aligned>(NextPPCondPtr);
237 CurPPCondPtr = NextPPCondPtr;
241 while (HashEntryI < LastHashTokPtr);
242 assert(HashEntryI == LastHashTokPtr &&
"No PP-cond entry found for '#'");
243 assert(TableIdx &&
"No jumping from #endifs.");
246 const unsigned char* NextPPCondPtr = PPCond + TableIdx*(
sizeof(uint32_t)*2);
247 assert(NextPPCondPtr >= CurPPCondPtr);
248 CurPPCondPtr = NextPPCondPtr;
252 TokBuf + endian::readNext<uint32_t, little, aligned>(NextPPCondPtr);
253 uint32_t NextIdx = endian::readNext<uint32_t, little, aligned>(NextPPCondPtr);
257 bool isEndif = NextIdx == 0;
268 if (CurPtr > HashEntryI) {
274 LastHashTokPtr = HashEntryI;
284 LastHashTokPtr = CurPtr;
304 using namespace llvm::support;
307 uint32_t
Offset = endian::readNext<uint32_t, little, aligned>(OffsetPtr);
321 const uint32_t TokenOff;
322 const uint32_t PPCondOff;
325 PTHFileData(uint32_t tokenOff, uint32_t ppCondOff)
326 : TokenOff(tokenOff), PPCondOff(ppCondOff) {}
328 uint32_t getTokenOffset()
const {
return TokenOff; }
329 uint32_t getPPCondOffset()
const {
return PPCondOff; }
332 class PTHFileLookupCommonTrait {
334 using internal_key_type = std::pair<unsigned char, StringRef>;
335 using hash_value_type = unsigned;
336 using offset_type = unsigned;
338 static hash_value_type
ComputeHash(internal_key_type x) {
339 return llvm::HashString(x.second);
342 static std::pair<unsigned, unsigned>
343 ReadKeyDataLength(
const unsigned char*& d) {
344 using namespace llvm::support;
347 (unsigned)endian::readNext<uint16_t, little, unaligned>(d);
348 unsigned dataLen = (unsigned) *(d++);
349 return std::make_pair(keyLen, dataLen);
352 static internal_key_type ReadKey(
const unsigned char* d,
unsigned) {
353 unsigned char k = *(d++);
354 return std::make_pair(k, (
const char*) d);
366 return std::make_pair((
unsigned char) 0x1, FE->
getName());
369 static bool EqualKey(internal_key_type a, internal_key_type b) {
370 return a.first == b.first && a.second == b.second;
373 static PTHFileData
ReadData(
const internal_key_type& k,
374 const unsigned char* d,
unsigned) {
375 using namespace llvm::support;
377 assert(k.first == 0x1 &&
"Only file lookups can match!");
378 uint32_t x = endian::readNext<uint32_t, little, unaligned>(d);
379 uint32_t y = endian::readNext<uint32_t, little, unaligned>(d);
380 return PTHFileData(x, y);
394 return (a.second == b.second) ? memcmp(a.first, b.first, a.second) == 0
399 return llvm::HashString(StringRef(a.first, a.second));
406 static std::pair<unsigned, unsigned>
408 using namespace llvm::support;
410 return std::make_pair(
411 (
unsigned)endian::readNext<uint16_t, little, unaligned>(d),
415 static std::pair<const char*, unsigned>
417 assert(n >= 2 && d[n-1] ==
'\0');
418 return std::make_pair((
const char*) d, n-1);
423 using namespace llvm::support;
425 return endian::readNext<uint32_t, little, unaligned>(d);
433 PTHManager::PTHManager(
434 std::unique_ptr<const llvm::MemoryBuffer> buf,
435 std::unique_ptr<PTHFileLookup> fileLookup,
const unsigned char *idDataTable,
436 std::unique_ptr<
IdentifierInfo *[], llvm::FreeDeleter> perIDCache,
437 std::unique_ptr<PTHStringIdLookup> stringIdLookup,
unsigned numIds,
438 const unsigned char *spellingBase,
const char *originalSourceFile)
439 : Buf(std::move(buf)), PerIDCache(std::move(perIDCache)),
440 FileLookup(std::move(fileLookup)), IdDataTable(idDataTable),
441 StringIdLookup(std::move(stringIdLookup)), NumIds(numIds),
442 SpellingBase(spellingBase), OriginalSourceFile(originalSourceFile) {}
452 llvm::ErrorOr<std::unique_ptr<llvm::MemoryBuffer>> FileOrErr =
453 llvm::MemoryBuffer::getFile(file);
457 Diags.
Report(diag::err_invalid_pth_file) << file;
460 std::unique_ptr<llvm::MemoryBuffer> File = std::move(FileOrErr.get());
462 using namespace llvm::support;
466 const unsigned char *BufBeg = (
const unsigned char*)File->getBufferStart();
467 const unsigned char *BufEnd = (
const unsigned char*)File->getBufferEnd();
470 if ((BufEnd - BufBeg) < (
signed)(
sizeof(
"cfe-pth") + 4 + 4) ||
471 memcmp(BufBeg,
"cfe-pth",
sizeof(
"cfe-pth")) != 0) {
472 Diags.
Report(diag::err_invalid_pth_file) << file;
477 const unsigned char *p = BufBeg + (
sizeof(
"cfe-pth"));
478 unsigned Version = endian::readNext<uint32_t, little, aligned>(p);
483 ?
"PTH file uses an older PTH format that is no longer supported" 484 :
"PTH file uses a newer PTH format that cannot be read");
489 const unsigned char *PrologueOffset = p;
491 if (PrologueOffset >= BufEnd) {
492 Diags.
Report(diag::err_invalid_pth_file) << file;
498 const unsigned char* FileTableOffset = PrologueOffset +
sizeof(uint32_t)*2;
499 const unsigned char *FileTable =
500 BufBeg + endian::readNext<uint32_t, little, aligned>(FileTableOffset);
502 if (!(FileTable > BufBeg && FileTable < BufEnd)) {
503 Diags.
Report(diag::err_invalid_pth_file) << file;
512 InvalidPTH(Diags,
"PTH file contains no cached source data");
516 const unsigned char* IDTableOffset = PrologueOffset +
sizeof(uint32_t)*0;
517 const unsigned char *IData =
518 BufBeg + endian::readNext<uint32_t, little, aligned>(IDTableOffset);
520 if (!(IData >= BufBeg && IData < BufEnd)) {
521 Diags.
Report(diag::err_invalid_pth_file) << file;
527 const unsigned char* StringIdTableOffset = PrologueOffset +
sizeof(uint32_t)*1;
528 const unsigned char *StringIdTable =
529 BufBeg + endian::readNext<uint32_t, little, aligned>(StringIdTableOffset);
530 if (!(StringIdTable >= BufBeg && StringIdTable < BufEnd)) {
531 Diags.
Report(diag::err_invalid_pth_file) << file;
535 std::unique_ptr<PTHStringIdLookup> SL(
539 const unsigned char* spellingBaseOffset = PrologueOffset +
sizeof(uint32_t)*3;
540 const unsigned char *spellingBase =
541 BufBeg + endian::readNext<uint32_t, little, aligned>(spellingBaseOffset);
542 if (!(spellingBase >= BufBeg && spellingBase < BufEnd)) {
543 Diags.
Report(diag::err_invalid_pth_file) << file;
548 uint32_t NumIds = endian::readNext<uint32_t, little, aligned>(IData);
553 std::unique_ptr<IdentifierInfo *[], llvm::FreeDeleter> PerIDCache;
556 PerIDCache.reset((
IdentifierInfo **)calloc(NumIds,
sizeof(PerIDCache[0])));
558 InvalidPTH(Diags,
"Could not allocate memory for processing PTH file");
564 const unsigned char* originalSourceBase = PrologueOffset +
sizeof(uint32_t)*4;
566 endian::readNext<uint16_t, little, unaligned>(originalSourceBase);
567 if (!len) originalSourceBase =
nullptr;
570 return new PTHManager(std::move(File), std::move(FL), IData,
571 std::move(PerIDCache), std::move(SL), NumIds,
572 spellingBase, (
const char *)originalSourceBase);
575 IdentifierInfo* PTHManager::LazilyCreateIdentifierInfo(
unsigned PersistentID) {
576 using namespace llvm::support;
579 const unsigned char* TableEntry = IdDataTable +
sizeof(uint32_t)*PersistentID;
580 const unsigned char *IDData =
581 (
const unsigned char *)Buf->getBufferStart() +
582 endian::readNext<uint32_t, little, aligned>(TableEntry);
583 assert(IDData < (
const unsigned char*)Buf->getBufferEnd());
586 std::pair<IdentifierInfo,const unsigned char*> *Mem =
587 Alloc.Allocate<std::pair<IdentifierInfo, const unsigned char *>>();
589 Mem->second = IDData;
590 assert(IDData[0] !=
'\0');
594 PerIDCache[PersistentID] = II;
601 assert(Name.empty() || Name.back() !=
'\0');
602 PTHStringIdLookup::iterator I =
603 StringIdLookup->find(std::make_pair(Name.data(), Name.size()));
604 if (I == StringIdLookup->end())
609 return GetIdentifierInfo(*I-1);
617 using namespace llvm::support;
622 PTHFileLookup::iterator I = FileLookup->find(FE);
624 if (I == FileLookup->end())
629 const unsigned char *BufStart = (
const unsigned char *)Buf->getBufferStart();
631 const unsigned char* data = BufStart + FileData.getTokenOffset();
634 const unsigned char* ppcond = BufStart + FileData.getPPCondOffset();
635 uint32_t Len = endian::readNext<uint32_t, little, aligned>(ppcond);
636 if (Len == 0) ppcond =
nullptr;
638 assert(
PP &&
"No preprocessor set yet!");
639 return new PTHLexer(*
PP, FID, data, ppcond, *
this);
652 llvm::sys::fs::UniqueID UniqueID;
653 const bool HasData =
false;
656 PTHStatData() =
default;
657 PTHStatData(uint64_t Size, time_t ModTime, llvm::sys::fs::UniqueID UniqueID,
659 : Size(Size), ModTime(ModTime), UniqueID(UniqueID), HasData(
true),
660 IsDirectory(IsDirectory) {}
663 class PTHStatLookupTrait :
public PTHFileLookupCommonTrait {
665 using external_key_type = StringRef;
666 using data_type = PTHStatData;
668 static internal_key_type GetInternalKey(StringRef path) {
670 return std::make_pair((
unsigned char) 0x0, path);
673 static bool EqualKey(internal_key_type a, internal_key_type b) {
676 return a.second == b.second;
679 static data_type ReadData(
const internal_key_type& k,
const unsigned char* d,
682 bool IsDirectory =
true;
683 if (k.first == 0x1 ) {
688 using namespace llvm::support;
690 uint64_t File = endian::readNext<uint64_t, little, unaligned>(d);
691 uint64_t Device = endian::readNext<uint64_t, little, unaligned>(d);
692 llvm::sys::fs::UniqueID UniqueID(Device, File);
693 time_t ModTime = endian::readNext<uint64_t, little, unaligned>(d);
694 uint64_t Size = endian::readNext<uint64_t, little, unaligned>(d);
695 return data_type(Size, ModTime, UniqueID, IsDirectory);
714 : Cache(FL.getNumBuckets(), FL.getNumEntries(), FL.getBuckets(),
718 std::unique_ptr<vfs::File> *F,
721 CacheTy::iterator I = Cache.find(Path);
724 if (I == Cache.end())
725 return statChained(Path, Data, isFile, F, FS);
727 const PTHStatData &D = *I;
747 return llvm::make_unique<PTHStatCache>(*FileLookup);
IdentifierInfo * get(StringRef Name) override
get - Return the identifier token info for the specified named identifier.
SourceLocation getLocForStartOfFile(FileID FID) const
Return the source location corresponding to the first byte of the specified file. ...
LookupResult getStat(StringRef Path, FileData &Data, bool isFile, std::unique_ptr< vfs::File > *F, vfs::FileSystem &FS) override
SourceLocation getLocWithOffset(int Offset) const
Return a source location with the specified offset from this SourceLocation.
static const unsigned StoredTokenSize
Defines the clang::FileManager interface and associated types.
TypePropertyCache< Private > Cache
bool Lex(Token &Tok)
Lex - Return the next token.
Defines the SourceManager interface.
static PTHFileData ReadData(const internal_key_type &k, const unsigned char *d, unsigned)
static std::pair< unsigned, unsigned > ReadKeyDataLength(const unsigned char *&d)
Defines the FileSystemStatCache interface.
DiagnosticBuilder Report(SourceLocation Loc, unsigned DiagID)
Issue the message to the client.
void setFlag(TokenFlags Flag)
Set the specified flag.
PTHLexer * CreateLexer(FileID FID)
CreateLexer - Return a PTHLexer that "lexes" the cached tokens for the specified file.
PTHLexer(Preprocessor &pp, FileID FID, const unsigned char *D, const unsigned char *ppcond, PTHManager &PM)
Create a PTHLexer for the specified token stream.
static PTHManager * Create(StringRef file, DiagnosticsEngine &Diags)
Create - This method creates PTHManager objects.
SourceLocation getCodeCompletionFileLoc() const
Returns the start location of the file of code-completion point.
bool isLiteral() const
Return true if this is a "literal", like a numeric constant, string, etc.
static hash_value_type ComputeHash(const internal_key_type &a)
The virtual file system interface.
One of these records is kept for each identifier that is lexed.
bool ParsingPreprocessorDirective
True when parsing #XXX; turns '\n' into a tok::eod token.
SmallVector< PPConditionalInfo, 4 > ConditionalStack
Information about the set of #if/#ifdef/#ifndef blocks we are currently in.
Token - This structure provides full information about a lexed token.
static uint32_t ReadData(const internal_key_type &k, const unsigned char *d, unsigned)
void setKind(tok::TokenKind K)
Abstract interface for introducing a FileManager cache for 'stat' system calls, which is used by prec...
tok::TokenKind getTokenID() const
If this is a source-language token (e.g.
const std::pair< const char *, unsigned > external_key_type
void HandleDirective(Token &Result)
Callback invoked when the lexer sees a # token at the start of a line.
Concrete class used by the front-end to report problems and issues.
std::unique_ptr< FileSystemStatCache > createStatCache()
createStatCache - Returns a FileSystemStatCache object for use with FileManager objects.
Defines the Diagnostic-related interfaces.
static internal_key_type GetInternalKey(const FileEntry *FE)
const FileID FID
The SourceManager FileID corresponding to the file being lexed.
bool ParsingFilename
True after #include; turns <xx> into a tok::angle_string_literal token.
bool LexingRawMode
True if in raw mode.
const FileEntry * getFileEntryForID(FileID FID) const
Returns the FileEntry record for the provided FileID.
Defines the clang::Preprocessor interface.
MultipleIncludeOpt MIOpt
A state machine that detects the #ifndef-wrapping a file idiom for the multiple-include optimization...
void DiscardToEndOfLine()
DiscardToEndOfLine - Read the rest of the current preprocessor line as an uninterpreted string...
uint32_t IdentifierID
An ID number that refers to an identifier in an AST file.
Defines the clang::IdentifierInfo, clang::IdentifierTable, and clang::Selector interfaces.
bool HandleEndOfFile(Token &Result, bool isEndOfMacro=false)
Callback invoked when the lexer hits the end of the current file.
The result type of a method or function.
SourceManager & getSourceManager() const
SourceLocation getSourceLocation() override
getSourceLocation - Return a source location for the token in the current file.
Encodes a location in the source.
llvm::sys::fs::UniqueID UniqueID
StringRef getName() const
void setLength(unsigned Len)
Cached information about one file (either on disk or in the virtual file system). ...
void setIdentifierInfo(IdentifierInfo *II)
const char * getNameStart() const
Return the beginning of the actual null-terminated string for this identifier.
static bool EqualKey(const internal_key_type &a, const internal_key_type &b)
bool isAtStartOfLine() const
isAtStartOfLine - Return true if this token is at the start of a line.
TokenKind
Provides a simple uniform namespace for tokens from all C languages.
unsigned getCustomDiagID(Level L, const char(&FormatString)[N])
Return an ID for a diagnostic with the specified format string and level.
unsigned ComputeHash(Selector Sel)
An opaque identifier used by SourceManager which refers to a source file (MemoryBuffer) along with it...
bool SkipBlock()
SkipBlock - Used by Preprocessor to skip the current conditional block.
Dataflow Directional Tag Classes.
external_key_type internal_key_type
bool isHandleIdentifierCase() const
Return true if the Preprocessor::HandleIdentifier must be called on a token of this identifier...
void setLiteralData(const char *Ptr)
PTHStatCache(PTHManager::PTHFileLookup &FL)
bool HandleIdentifier(Token &Identifier)
Callback invoked when the lexer reads an identifier and has filled in the tokens IdentifierInfo membe...
Defines the clang::TokenKind enum and support functions.
void setLocation(SourceLocation L)
static void InvalidPTH(DiagnosticsEngine &Diags, const char *Msg)
static const internal_key_type & GetInternalKey(const external_key_type &x)
static std::pair< const char *, unsigned > ReadKey(const unsigned char *d, unsigned n)
static bool EqualKey(internal_key_type a, internal_key_type b)
DiagnosticBuilder Diag(SourceLocation Loc, unsigned DiagID) const
Forwarding function for diagnostics.
void startToken()
Reset all flags to cleared.
Engages in a tight little dance with the lexer to efficiently preprocess tokens.
static OMPLinearClause * Create(const ASTContext &C, SourceLocation StartLoc, SourceLocation LParenLoc, OpenMPLinearClauseKind Modifier, SourceLocation ModifierLoc, SourceLocation ColonLoc, SourceLocation EndLoc, ArrayRef< Expr *> VL, ArrayRef< Expr *> PL, ArrayRef< Expr *> IL, Expr *Step, Expr *CalcStep, Stmt *PreInit, Expr *PostUpdate)
Creates clause with a list of variables VL and a linear step Step.