20 #include "llvm/ADT/DenseMap.h" 21 #include "llvm/ADT/MapVector.h" 22 #include "llvm/ADT/SmallString.h" 23 #include "llvm/ADT/StringRef.h" 24 #include "llvm/Bitstream/BitstreamReader.h" 25 #include "llvm/Bitstream/BitstreamWriter.h" 26 #include "llvm/Support/DJB.h" 27 #include "llvm/Support/FileSystem.h" 28 #include "llvm/Support/FileUtilities.h" 29 #include "llvm/Support/LockFileManager.h" 30 #include "llvm/Support/MemoryBuffer.h" 31 #include "llvm/Support/OnDiskHashTable.h" 32 #include "llvm/Support/Path.h" 33 #include "llvm/Support/TimeProfiler.h" 35 using namespace clang;
36 using namespace serialization;
44 GLOBAL_INDEX_BLOCK_ID = llvm::bitc::FIRST_APPLICATION_BLOCKID
73 class IdentifierIndexReaderTrait {
75 typedef StringRef external_key_type;
76 typedef StringRef internal_key_type;
78 typedef unsigned hash_value_type;
79 typedef unsigned offset_type;
81 static bool EqualKey(
const internal_key_type& a,
const internal_key_type& b) {
85 static hash_value_type
ComputeHash(
const internal_key_type& a) {
86 return llvm::djbHash(a);
89 static std::pair<unsigned, unsigned>
90 ReadKeyDataLength(
const unsigned char*& d) {
91 using namespace llvm::support;
92 unsigned KeyLen = endian::readNext<uint16_t, little, unaligned>(d);
93 unsigned DataLen = endian::readNext<uint16_t, little, unaligned>(d);
94 return std::make_pair(KeyLen, DataLen);
97 static const internal_key_type&
98 GetInternalKey(
const external_key_type& x) {
return x; }
100 static const external_key_type&
101 GetExternalKey(
const internal_key_type& x) {
return x; }
103 static internal_key_type ReadKey(
const unsigned char* d,
unsigned n) {
104 return StringRef((
const char *)d, n);
107 static data_type ReadData(
const internal_key_type& k,
108 const unsigned char* d,
110 using namespace llvm::support;
113 while (DataLen > 0) {
114 unsigned ID = endian::readNext<uint32_t, little, unaligned>(d);
115 Result.push_back(ID);
123 typedef llvm::OnDiskIterableChainedHashTable<IdentifierIndexReaderTrait>
124 IdentifierIndexTable;
128 GlobalModuleIndex::GlobalModuleIndex(
129 std::unique_ptr<llvm::MemoryBuffer> IndexBuffer,
130 llvm::BitstreamCursor
Cursor)
131 : Buffer(std::move(IndexBuffer)), IdentifierIndex(), NumIdentifierLookups(),
132 NumIdentifierLookupHits() {
134 report_fatal_error(
"Module index '" + Buffer->getBufferIdentifier() +
135 "' failed: " +
toString(std::move(Err)));
138 llvm::TimeTraceScope TimeScope(
"Module LoadIndex");
140 bool InGlobalIndexBlock =
false;
143 llvm::BitstreamEntry Entry;
147 Fail(Res.takeError());
149 switch (Entry.Kind) {
153 case llvm::BitstreamEntry::EndBlock:
154 if (InGlobalIndexBlock) {
155 InGlobalIndexBlock =
false;
162 case llvm::BitstreamEntry::Record:
164 if (InGlobalIndexBlock)
169 case llvm::BitstreamEntry::SubBlock:
170 if (!InGlobalIndexBlock && Entry.ID == GLOBAL_INDEX_BLOCK_ID) {
172 Fail(std::move(Err));
173 InGlobalIndexBlock =
true;
175 Fail(std::move(Err));
182 Cursor.readRecord(Entry.ID, Record, &Blob);
183 if (!MaybeIndexRecord)
184 Fail(MaybeIndexRecord.takeError());
187 switch (IndexRecord) {
196 unsigned ID = Record[Idx++];
199 if (ID == Modules.size())
200 Modules.push_back(ModuleInfo());
202 Modules.resize(ID + 1);
206 Modules[
ID].Size = Record[Idx++];
207 Modules[
ID].ModTime = Record[Idx++];
210 unsigned NameLen = Record[Idx++];
211 Modules[
ID].FileName.assign(Record.begin() + Idx,
212 Record.begin() + Idx + NameLen);
216 unsigned NumDeps = Record[Idx++];
217 Modules[
ID].Dependencies.insert(Modules[ID].Dependencies.end(),
218 Record.begin() + Idx,
219 Record.begin() + Idx + NumDeps);
223 assert(Idx == Record.size() &&
"More module info?");
228 StringRef ModuleName = llvm::sys::path::stem(Modules[ID].FileName);
230 ModuleName = ModuleName.rsplit(
'-').first;
231 UnresolvedModules[ModuleName] =
ID;
235 case IDENTIFIER_INDEX:
239 (
const unsigned char *)Blob.data() + Record[0],
240 (
const unsigned char *)Blob.data() +
sizeof(uint32_t),
241 (
const unsigned char *)Blob.data(), IdentifierIndexReaderTrait());
249 delete static_cast<IdentifierIndexTable *
>(IdentifierIndex);
252 std::pair<GlobalModuleIndex *, llvm::Error>
259 llvm::ErrorOr<std::unique_ptr<llvm::MemoryBuffer>> BufferOrErr =
260 llvm::MemoryBuffer::getFile(IndexPath.c_str());
262 return std::make_pair(
nullptr,
263 llvm::errorCodeToError(BufferOrErr.getError()));
264 std::unique_ptr<llvm::MemoryBuffer> Buffer = std::move(BufferOrErr.get());
267 llvm::BitstreamCursor
Cursor(*Buffer);
270 for (
unsigned char C : {
'B',
'C',
'G',
'I'}) {
273 return std::make_pair(
274 nullptr, llvm::createStringError(std::errc::illegal_byte_sequence,
275 "expected signature BCGI"));
277 return std::make_pair(
nullptr, Res.takeError());
281 llvm::Error::success());
287 for (
unsigned I = 0, N = Modules.size(); I != N; ++I) {
289 ModuleFiles.push_back(MF);
297 llvm::DenseMap<ModuleFile *, unsigned>::iterator Known
298 = ModulesByFile.find(File);
299 if (Known == ModulesByFile.end())
303 Dependencies.clear();
305 for (
unsigned I = 0, N = StoredDependencies.size(); I != N; ++I) {
307 Dependencies.push_back(MF);
315 if (!IdentifierIndex)
319 ++NumIdentifierLookups;
320 IdentifierIndexTable &Table
321 = *
static_cast<IdentifierIndexTable *
>(IdentifierIndex);
322 IdentifierIndexTable::iterator Known = Table.find(Name);
323 if (Known == Table.end()) {
328 for (
unsigned I = 0, N = ModuleIDs.size(); I != N; ++I) {
329 if (
ModuleFile *MF = Modules[ModuleIDs[I]].File)
333 ++NumIdentifierLookupHits;
340 llvm::StringMap<unsigned>::iterator Known = UnresolvedModules.find(Name);
341 if (Known == UnresolvedModules.end()) {
346 ModuleInfo &Info = Modules[Known->second];
354 ModulesByFile[File] = Known->second;
360 UnresolvedModules.erase(Known);
365 std::fprintf(stderr,
"*** Global Module Index Statistics:\n");
366 if (NumIdentifierLookups) {
367 fprintf(stderr,
" %u / %u identifier lookups succeeded (%f%%)\n",
368 NumIdentifierLookupHits, NumIdentifierLookups,
369 (
double)NumIdentifierLookupHits*100.0/NumIdentifierLookups);
371 std::fprintf(stderr,
"\n");
375 llvm::errs() <<
"*** Global Module Index Dump:\n";
376 llvm::errs() <<
"Module files:\n";
377 for (
auto &MI : Modules) {
378 llvm::errs() <<
"** " << MI.FileName <<
"\n";
382 llvm::errs() <<
"\n";
384 llvm::errs() <<
"\n";
403 struct ImportedModuleFileInfo {
405 time_t StoredModTime;
408 : StoredSize(Size), StoredModTime(ModTime), StoredSignature(Sig) {}
412 class GlobalModuleIndexBuilder {
417 typedef llvm::MapVector<const FileEntry *, ModuleFileInfo> ModuleFilesMap;
420 ModuleFilesMap ModuleFiles;
424 typedef std::multimap<const FileEntry *, ImportedModuleFileInfo>
425 ImportedModuleFilesMap;
428 ImportedModuleFilesMap ImportedModuleFiles;
432 typedef llvm::StringMap<SmallVector<unsigned, 2> > InterestingIdentifierMap;
436 InterestingIdentifierMap InterestingIdentifiers;
439 void emitBlockInfoBlock(llvm::BitstreamWriter &Stream);
443 llvm::MapVector<const FileEntry *, ModuleFileInfo>::iterator Known
444 = ModuleFiles.find(File);
445 if (Known != ModuleFiles.end())
446 return Known->second;
448 unsigned NewID = ModuleFiles.size();
455 explicit GlobalModuleIndexBuilder(
457 : FileMgr(FileMgr), PCHContainerRdr(PCHContainerRdr) {}
464 bool writeIndex(llvm::BitstreamWriter &Stream);
469 llvm::BitstreamWriter &Stream,
472 Record.push_back(ID);
473 Stream.EmitRecord(llvm::bitc::BLOCKINFO_CODE_SETBID, Record);
476 if (!Name || Name[0] == 0)
return;
479 Record.push_back(*Name++);
480 Stream.EmitRecord(llvm::bitc::BLOCKINFO_CODE_BLOCKNAME, Record);
484 llvm::BitstreamWriter &Stream,
487 Record.push_back(ID);
489 Record.push_back(*Name++);
490 Stream.EmitRecord(llvm::bitc::BLOCKINFO_CODE_SETRECORDNAME, Record);
494 GlobalModuleIndexBuilder::emitBlockInfoBlock(llvm::BitstreamWriter &Stream) {
496 Stream.EnterBlockInfoBlock();
498 #define BLOCK(X) emitBlockID(X ## _ID, #X, Stream, Record) 499 #define RECORD(X) emitRecordID(X, #X, Stream, Record) 500 BLOCK(GLOBAL_INDEX_BLOCK);
511 class InterestingASTIdentifierLookupTrait
516 typedef std::pair<StringRef, bool> data_type;
518 data_type ReadData(
const internal_key_type& k,
519 const unsigned char* d,
523 using namespace llvm::support;
524 unsigned RawID = endian::readNext<uint32_t, little, unaligned>(d);
525 bool IsInteresting = RawID & 0x01;
526 return std::make_pair(k, IsInteresting);
534 auto Buffer = FileMgr.getBufferForFile(File,
true);
536 return llvm::createStringError(Buffer.getError(),
537 "failed getting buffer for module file");
540 llvm::BitstreamCursor InStream(PCHContainerRdr.ExtractPCH(**Buffer));
543 for (
unsigned char C : {
'C',
'P',
'C',
'H'})
546 return llvm::createStringError(std::errc::illegal_byte_sequence,
547 "expected signature CPCH");
549 return Res.takeError();
553 unsigned ID = getModuleFileInfo(File).ID;
556 enum { Other, ControlBlock, ASTBlock, DiagnosticOptionsBlock }
State = Other;
561 return MaybeEntry.takeError();
562 llvm::BitstreamEntry Entry = MaybeEntry.get();
564 switch (Entry.Kind) {
569 case llvm::BitstreamEntry::Record:
571 if (State == Other) {
575 return Skipped.takeError();
581 case llvm::BitstreamEntry::SubBlock:
587 State = ControlBlock;
605 State = DiagnosticOptionsBlock;
614 case llvm::BitstreamEntry::EndBlock:
624 return MaybeCode.takeError();
625 unsigned Code = MaybeCode.get();
628 if (State == ControlBlock && Code ==
IMPORTS) {
630 unsigned Idx = 0, N = Record.size();
641 off_t StoredSize = (off_t)Record[Idx++];
642 time_t StoredModTime = (time_t)Record[Idx++];
647 {{(uint32_t)Record[Idx++], (uint32_t)Record[Idx++],
648 (uint32_t)Record[Idx++], (uint32_t)Record[Idx++],
649 (uint32_t)Record[Idx++]}}};
653 Idx += Record[Idx] + 1;
656 unsigned Length = Record[Idx++];
658 Record.begin() + Idx + Length);
663 = FileMgr.getFile(ImportedFile,
false,
667 return llvm::createStringError(std::errc::bad_file_descriptor,
668 "imported file \"%s\" not found",
669 ImportedFile.c_str());
673 ImportedModuleFiles.insert(std::make_pair(
674 *DependsOnFile, ImportedModuleFileInfo(StoredSize, StoredModTime,
678 unsigned DependsOnID = getModuleFileInfo(*DependsOnFile).ID;
679 getModuleFileInfo(File).Dependencies.push_back(DependsOnID);
687 typedef llvm::OnDiskIterableChainedHashTable<
688 InterestingASTIdentifierLookupTrait> InterestingIdentifierTable;
689 std::unique_ptr<InterestingIdentifierTable> Table(
691 (
const unsigned char *)Blob.data() + Record[0],
692 (
const unsigned char *)Blob.data() +
sizeof(uint32_t),
693 (
const unsigned char *)Blob.data()));
694 for (InterestingIdentifierTable::data_iterator D = Table->data_begin(),
695 DEnd = Table->data_end();
697 std::pair<StringRef, bool> Ident = *D;
699 InterestingIdentifiers[Ident.first].push_back(ID);
701 (
void)InterestingIdentifiers[Ident.first];
706 if (State == DiagnosticOptionsBlock && Code ==
SIGNATURE)
707 getModuleFileInfo(File).Signature = {
708 {{(uint32_t)Record[0], (uint32_t)Record[1], (uint32_t)Record[2],
709 (uint32_t)Record[3], (uint32_t)Record[4]}}};
714 return llvm::Error::success();
721 class IdentifierIndexWriterTrait {
723 typedef StringRef key_type;
724 typedef StringRef key_type_ref;
727 typedef unsigned hash_value_type;
728 typedef unsigned offset_type;
730 static hash_value_type
ComputeHash(key_type_ref Key) {
731 return llvm::djbHash(Key);
734 std::pair<unsigned,unsigned>
735 EmitKeyDataLength(raw_ostream& Out, key_type_ref Key, data_type_ref Data) {
736 using namespace llvm::support;
737 endian::Writer
LE(Out, little);
738 unsigned KeyLen = Key.size();
739 unsigned DataLen = Data.size() * 4;
740 LE.write<uint16_t>(KeyLen);
741 LE.write<uint16_t>(DataLen);
742 return std::make_pair(KeyLen, DataLen);
745 void EmitKey(raw_ostream& Out, key_type_ref Key,
unsigned KeyLen) {
746 Out.write(Key.data(), KeyLen);
749 void EmitData(raw_ostream& Out, key_type_ref Key, data_type_ref Data,
751 using namespace llvm::support;
752 for (
unsigned I = 0, N = Data.size(); I != N; ++I)
753 endian::write<uint32_t>(Out, Data[I], little);
759 bool GlobalModuleIndexBuilder::writeIndex(llvm::BitstreamWriter &Stream) {
760 for (
auto MapEntry : ImportedModuleFiles) {
761 auto *File = MapEntry.first;
762 ImportedModuleFileInfo &Info = MapEntry.second;
763 if (getModuleFileInfo(File).Signature) {
764 if (getModuleFileInfo(File).Signature != Info.StoredSignature)
767 }
else if (Info.StoredSize != File->
getSize() ||
773 using namespace llvm;
774 llvm::TimeTraceScope TimeScope(
"Module WriteIndex");
777 Stream.Emit((
unsigned)
'B', 8);
778 Stream.Emit((
unsigned)
'C', 8);
779 Stream.Emit((
unsigned)
'G', 8);
780 Stream.Emit((
unsigned)
'I', 8);
784 emitBlockInfoBlock(Stream);
786 Stream.EnterSubblock(GLOBAL_INDEX_BLOCK_ID, 3);
791 Stream.EmitRecord(INDEX_METADATA, Record);
794 for (ModuleFilesMap::iterator M = ModuleFiles.begin(),
795 MEnd = ModuleFiles.end();
798 Record.push_back(M->second.ID);
799 Record.push_back(M->first->getSize());
800 Record.push_back(M->first->getModificationTime());
803 StringRef Name(M->first->getName());
804 Record.push_back(Name.size());
805 Record.append(Name.begin(), Name.end());
808 Record.push_back(M->second.Dependencies.size());
809 Record.append(M->second.Dependencies.begin(), M->second.Dependencies.end());
810 Stream.EmitRecord(MODULE, Record);
815 llvm::OnDiskChainedHashTableGenerator<IdentifierIndexWriterTrait> Generator;
816 IdentifierIndexWriterTrait Trait;
819 for (InterestingIdentifierMap::iterator I = InterestingIdentifiers.begin(),
820 IEnd = InterestingIdentifiers.end();
822 Generator.insert(I->first(), I->second, Trait);
827 uint32_t BucketOffset;
829 using namespace llvm::support;
830 llvm::raw_svector_ostream Out(IdentifierTable);
832 endian::write<uint32_t>(Out, 0, little);
833 BucketOffset = Generator.Emit(Out, Trait);
837 auto Abbrev = std::make_shared<BitCodeAbbrev>();
838 Abbrev->Add(BitCodeAbbrevOp(IDENTIFIER_INDEX));
839 Abbrev->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 32));
840 Abbrev->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Blob));
841 unsigned IDTableAbbrev = Stream.EmitAbbrev(std::move(Abbrev));
844 uint64_t Record[] = {IDENTIFIER_INDEX, BucketOffset};
845 Stream.EmitRecordWithBlob(IDTableAbbrev, Record, IdentifierTable);
862 llvm::LockFileManager Locked(IndexPath);
864 case llvm::LockFileManager::LFS_Error:
865 return llvm::createStringError(std::errc::io_error,
"LFS error");
867 case llvm::LockFileManager::LFS_Owned:
871 case llvm::LockFileManager::LFS_Shared:
874 return llvm::createStringError(std::errc::device_or_resource_busy,
875 "someone else is building the index");
879 GlobalModuleIndexBuilder Builder(FileMgr, PCHContainerRdr);
883 for (llvm::sys::fs::directory_iterator D(Path, EC), DEnd;
887 if (llvm::sys::path::extension(D->path()) !=
".pcm") {
891 if (llvm::sys::path::extension(D->path()) ==
".pcm.lock")
892 return llvm::createStringError(std::errc::device_or_resource_busy,
893 "someone else is building the index");
911 llvm::BitstreamWriter OutputStream(OutputBuffer);
912 if (Builder.writeIndex(OutputStream))
913 return llvm::createStringError(std::errc::io_error,
914 "failed writing index");
917 return llvm::writeFileAtomically(
918 (IndexPath +
"-%%%%%%%%").str(), IndexPath,
919 llvm::StringRef(OutputBuffer.data(), OutputBuffer.size()));
925 IdentifierIndexTable::key_iterator Current;
928 IdentifierIndexTable::key_iterator
End;
931 explicit GlobalIndexIdentifierIterator(IdentifierIndexTable &Idx) {
932 Current = Idx.key_begin();
936 StringRef Next()
override {
940 StringRef Result = *Current;
948 IdentifierIndexTable &Table =
949 *
static_cast<IdentifierIndexTable *
>(IdentifierIndex);
950 return new GlobalIndexIdentifierIterator(Table);
Implements support for file system lookup, file system caching, and directory search management...
Defines the clang::FileManager interface and associated types.
time_t getModificationTime() const
Specialize PointerLikeTypeTraits to allow LazyGenerationalUpdatePtr to be placed into a PointerUnion...
std::string ModuleName
The name of the module.
static const unsigned CurrentVersion
The global index file version.
A block with unhashed content.
void printStats()
Print statistics to standard error.
static void emitRecordID(unsigned ID, const char *Name, llvm::BitstreamWriter &Stream, SmallVectorImpl< uint64_t > &Record)
static llvm::Error writeIndex(FileManager &FileMgr, const PCHContainerReader &PCHContainerRdr, llvm::StringRef Path)
Write a global index into the given.
static std::string toString(const clang::SanitizerSet &Sanitizers)
Produce a string containing comma-separated names of sanitizers in Sanitizers set.
llvm::SmallPtrSet< ModuleFile *, 4 > HitSet
A set of module files in which we found a result.
The signature of a module, which is a hash of the AST content.
This abstract interface provides operations for unwrapping containers for serialized ASTs (precompile...
bool lookupIdentifier(llvm::StringRef Name, HitSet &Hits)
Look for all of the module files with information about the given identifier, e.g., a global function, variable, or type with that name.
void dump()
Print debugging view to standard error.
static void emitBlockID(unsigned ID, const char *Name, llvm::BitstreamWriter &Stream, SmallVectorImpl< uint64_t > &Record)
IdentifierIterator * createIdentifierIterator() const
Returns an iterator for identifiers stored in the index table.
Implements an efficient mapping from strings to IdentifierInfo nodes.
Base class for the trait describing the on-disk hash table for the identifiers in an AST file...
Information about a module that has been loaded by the ASTReader.
An iterator that walks over all of the known identifiers in the lookup table.
static const char *const IndexFileName
The name of the global index file.
Record code for the identifier table.
IndexRecordTypes
Describes the record types in the index.
Cached information about one file (either on disk or in the virtual file system). ...
static std::pair< GlobalModuleIndex *, llvm::Error > readIndex(llvm::StringRef Path)
Read a global index file for the given directory.
void getKnownModules(llvm::SmallVectorImpl< ModuleFile *> &ModuleFiles)
Retrieve the set of modules that have up-to-date indexes.
The AST block, which acts as a container around the full AST block.
A global index for a set of module files, providing information about the identifiers within those mo...
unsigned ComputeHash(Selector Sel)
Record code for the signature that identifiers this AST file.
Record code for the list of other AST files imported by this AST file.
Dataflow Directional Tag Classes.
llvm::ErrorOr< const FileEntry * > getFile(StringRef Filename, bool OpenFile=false, bool CacheFailure=true)
Lookup, cache, and verify the specified file (real or virtual).
void getModuleDependencies(ModuleFile *File, llvm::SmallVectorImpl< ModuleFile *> &Dependencies)
Retrieve the set of module files on which the given module file directly depends. ...
bool LE(InterpState &S, CodePtr OpPC)
Dump information about a module file.
The control block, which contains all of the information that needs to be validated prior to committi...
const FileEntry * File
The file entry for the module file.
static OMPLinearClause * Create(const ASTContext &C, SourceLocation StartLoc, SourceLocation LParenLoc, OpenMPLinearClauseKind Modifier, SourceLocation ModifierLoc, SourceLocation ColonLoc, SourceLocation EndLoc, ArrayRef< Expr *> VL, ArrayRef< Expr *> PL, ArrayRef< Expr *> IL, Expr *Step, Expr *CalcStep, Stmt *PreInit, Expr *PostUpdate)
Creates clause with a list of variables VL and a linear step Step.
bool loadedModuleFile(ModuleFile *File)
Note that the given module file has been loaded.