clang 10.0.0git

GlobalModuleIndex.cpp
Go to the documentation of this file.
1 //===--- GlobalModuleIndex.cpp - Global Module Index ------------*- C++ -*-===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 //
9 // This file implements the GlobalModuleIndex class.
10 //
11 //===----------------------------------------------------------------------===//
12
13 #include "clang/Serialization/GlobalModuleIndex.h"
14 #include "ASTReaderInternals.h"
15 #include "clang/Basic/FileManager.h"
16 #include "clang/Lex/HeaderSearch.h"
17 #include "clang/Serialization/ASTBitCodes.h"
18 #include "clang/Serialization/ModuleFile.h"
19 #include "clang/Serialization/PCHContainerOperations.h"
20 #include "llvm/ADT/DenseMap.h"
21 #include "llvm/ADT/MapVector.h"
22 #include "llvm/ADT/SmallString.h"
23 #include "llvm/ADT/StringRef.h"
24 #include "llvm/Bitstream/BitstreamReader.h"
25 #include "llvm/Bitstream/BitstreamWriter.h"
26 #include "llvm/Support/DJB.h"
27 #include "llvm/Support/FileSystem.h"
28 #include "llvm/Support/FileUtilities.h"
29 #include "llvm/Support/LockFileManager.h"
30 #include "llvm/Support/MemoryBuffer.h"
31 #include "llvm/Support/OnDiskHashTable.h"
32 #include "llvm/Support/Path.h"
33 #include "llvm/Support/TimeProfiler.h"
34 #include <cstdio>
35 using namespace clang;
36 using namespace serialization;
37
38 //----------------------------------------------------------------------------//
39 // Shared constants
40 //----------------------------------------------------------------------------//
41 namespace {
42 enum {
43 /// The block containing the index.
44 GLOBAL_INDEX_BLOCK_ID = llvm::bitc::FIRST_APPLICATION_BLOCKID
45 };
46
47 /// Describes the record types in the index.
48 enum IndexRecordTypes {
49 /// Contains version information and potentially other metadata,
50 /// used to determine if we can read this global index file.
51 INDEX_METADATA,
52 /// Describes a module, including its file name and dependencies.
53 MODULE,
54 /// The index for identifiers.
55 IDENTIFIER_INDEX
56 };
57 }
58
59 /// The name of the global index file.
60 static const char * const IndexFileName = "modules.idx";
61
62 /// The global index file version.
63 static const unsigned CurrentVersion = 1;
64
65 //----------------------------------------------------------------------------//
66 // Global module index reader.
67 //----------------------------------------------------------------------------//
68
69 namespace {
70
71 /// Trait used to read the identifier index from the on-disk hash
72 /// table.
73 class IdentifierIndexReaderTrait {
74 public:
75 typedef StringRef external_key_type;
76 typedef StringRef internal_key_type;
77 typedef SmallVector<unsigned, 2> data_type;
78 typedef unsigned hash_value_type;
79 typedef unsigned offset_type;
80
81 static bool EqualKey(const internal_key_type& a, const internal_key_type& b) {
82 return a == b;
83 }
84
85 static hash_value_type ComputeHash(const internal_key_type& a) {
86 return llvm::djbHash(a);
87 }
88
89 static std::pair<unsigned, unsigned>
90 ReadKeyDataLength(const unsigned char*& d) {
91 using namespace llvm::support;
92 unsigned KeyLen = endian::readNext<uint16_t, little, unaligned>(d);
93 unsigned DataLen = endian::readNext<uint16_t, little, unaligned>(d);
94 return std::make_pair(KeyLen, DataLen);
95 }
96
97 static const internal_key_type&
98 GetInternalKey(const external_key_type& x) { return x; }
99
100 static const external_key_type&
101 GetExternalKey(const internal_key_type& x) { return x; }
102
103 static internal_key_type ReadKey(const unsigned char* d, unsigned n) {
104 return StringRef((const char *)d, n);
105 }
106
107 static data_type ReadData(const internal_key_type& k,
108 const unsigned char* d,
109 unsigned DataLen) {
110 using namespace llvm::support;
111
112 data_type Result;
113 while (DataLen > 0) {
114 unsigned ID = endian::readNext<uint32_t, little, unaligned>(d);
115 Result.push_back(ID);
116 DataLen -= 4;
117 }
118
119 return Result;
120 }
121 };
122
123 typedef llvm::OnDiskIterableChainedHashTable<IdentifierIndexReaderTrait>
124 IdentifierIndexTable;
125
126 }
127
128 GlobalModuleIndex::GlobalModuleIndex(
129 std::unique_ptr<llvm::MemoryBuffer> IndexBuffer,
130 llvm::BitstreamCursor Cursor)
131 : Buffer(std::move(IndexBuffer)), IdentifierIndex(), NumIdentifierLookups(),
132 NumIdentifierLookupHits() {
133 auto Fail = [&](llvm::Error &&Err) {
134 report_fatal_error("Module index '" + Buffer->getBufferIdentifier() +
135 "' failed: " + toString(std::move(Err)));
136 };
137
138 llvm::TimeTraceScope TimeScope("Module LoadIndex");
139 // Read the global index.
140 bool InGlobalIndexBlock = false;
141 bool Done = false;
142 while (!Done) {
143 llvm::BitstreamEntry Entry;
144 if (Expected<llvm::BitstreamEntry> Res = Cursor.advance())
145 Entry = Res.get();
146 else
147 Fail(Res.takeError());
148
149 switch (Entry.Kind) {
150 case llvm::BitstreamEntry::Error:
151 return;
152
153 case llvm::BitstreamEntry::EndBlock:
154 if (InGlobalIndexBlock) {
155 InGlobalIndexBlock = false;
156 Done = true;
157 continue;
158 }
159 return;
160
161
162 case llvm::BitstreamEntry::Record:
163 // Entries in the global index block are handled below.
164 if (InGlobalIndexBlock)
165 break;
166
167 return;
168
169 case llvm::BitstreamEntry::SubBlock:
170 if (!InGlobalIndexBlock && Entry.ID == GLOBAL_INDEX_BLOCK_ID) {
171 if (llvm::Error Err = Cursor.EnterSubBlock(GLOBAL_INDEX_BLOCK_ID))
172 Fail(std::move(Err));
173 InGlobalIndexBlock = true;
174 } else if (llvm::Error Err = Cursor.SkipBlock())
175 Fail(std::move(Err));
176 continue;
177 }
178
179 SmallVector<uint64_t, 64> Record;
180 StringRef Blob;
181 Expected<unsigned> MaybeIndexRecord =
182 Cursor.readRecord(Entry.ID, Record, &Blob);
183 if (!MaybeIndexRecord)
184 Fail(MaybeIndexRecord.takeError());
185 IndexRecordTypes IndexRecord =
186 static_cast<IndexRecordTypes>(MaybeIndexRecord.get());
187 switch (IndexRecord) {
188 case INDEX_METADATA:
189 // Make sure that the version matches.
190 if (Record.size() < 1 || Record[0] != CurrentVersion)
191 return;
192 break;
193
194 case MODULE: {
195 unsigned Idx = 0;
196 unsigned ID = Record[Idx++];
197
198 // Make room for this module's information.
199 if (ID == Modules.size())
200 Modules.push_back(ModuleInfo());
201 else
202 Modules.resize(ID + 1);
203
204 // Size/modification time for this module file at the time the
205 // global index was built.
206 Modules[ID].Size = Record[Idx++];
207 Modules[ID].ModTime = Record[Idx++];
208
209 // File name.
210 unsigned NameLen = Record[Idx++];
211 Modules[ID].FileName.assign(Record.begin() + Idx,
212 Record.begin() + Idx + NameLen);
213 Idx += NameLen;
214
215 // Dependencies
216 unsigned NumDeps = Record[Idx++];
217 Modules[ID].Dependencies.insert(Modules[ID].Dependencies.end(),
218 Record.begin() + Idx,
219 Record.begin() + Idx + NumDeps);
220 Idx += NumDeps;
221
222 // Make sure we're at the end of the record.
223 assert(Idx == Record.size() && "More module info?");
224
225 // Record this module as an unresolved module.
226 // FIXME: this doesn't work correctly for module names containing path
227 // separators.
228 StringRef ModuleName = llvm::sys::path::stem(Modules[ID].FileName);
229 // Remove the -<hash of ModuleMapPath>
230 ModuleName = ModuleName.rsplit('-').first;
231 UnresolvedModules[ModuleName] = ID;
232 break;
233 }
234
235 case IDENTIFIER_INDEX:
236 // Wire up the identifier index.
237 if (Record[0]) {
238 IdentifierIndex = IdentifierIndexTable::Create(
239 (const unsigned char *)Blob.data() + Record[0],
240 (const unsigned char *)Blob.data() + sizeof(uint32_t),
241 (const unsigned char *)Blob.data(), IdentifierIndexReaderTrait());
242 }
243 break;
244 }
245 }
246 }
247
248 GlobalModuleIndex::~GlobalModuleIndex() {
249 delete static_cast<IdentifierIndexTable *>(IdentifierIndex);
250 }
251
252 std::pair<GlobalModuleIndex *, llvm::Error>
253 GlobalModuleIndex::readIndex(StringRef Path) {
254 // Load the index file, if it's there.
255 llvm::SmallString<128> IndexPath;
256 IndexPath += Path;
257 llvm::sys::path::append(IndexPath, IndexFileName);
258
259 llvm::ErrorOr<std::unique_ptr<llvm::MemoryBuffer>> BufferOrErr =
260 llvm::MemoryBuffer::getFile(IndexPath.c_str());
261 if (!BufferOrErr)
262 return std::make_pair(nullptr,
263 llvm::errorCodeToError(BufferOrErr.getError()));
264 std::unique_ptr<llvm::MemoryBuffer> Buffer = std::move(BufferOrErr.get());
265
266 /// The main bitstream cursor for the main block.
267 llvm::BitstreamCursor Cursor(*Buffer);
268
269 // Sniff for the signature.
270 for (unsigned char C : {'B', 'C', 'G', 'I'}) {
271 if (Expected<llvm::SimpleBitstreamCursor::word_t> Res = Cursor.Read(8)) {
272 if (Res.get() != C)
273 return std::make_pair(
274 nullptr, llvm::createStringError(std::errc::illegal_byte_sequence,
275 "expected signature BCGI"));
276 } else
277 return std::make_pair(nullptr, Res.takeError());
278 }
279
280 return std::make_pair(new GlobalModuleIndex(std::move(Buffer), Cursor),
281 llvm::Error::success());
282 }
283
284 void
285 GlobalModuleIndex::getKnownModules(SmallVectorImpl<ModuleFile *> &ModuleFiles) {
286 ModuleFiles.clear();
287 for (unsigned I = 0, N = Modules.size(); I != N; ++I) {
288 if (ModuleFile *MF = Modules[I].File)
289 ModuleFiles.push_back(MF);
290 }
291 }
292
293 void GlobalModuleIndex::getModuleDependencies(
294 ModuleFile *File,
295 SmallVectorImpl<ModuleFile *> &Dependencies) {
296 // Look for information about this module file.
297 llvm::DenseMap<ModuleFile *, unsigned>::iterator Known
298 = ModulesByFile.find(File);
299 if (Known == ModulesByFile.end())
300 return;
301
302 // Record dependencies.
303 Dependencies.clear();
304 ArrayRef<unsigned> StoredDependencies = Modules[Known->second].Dependencies;
305 for (unsigned I = 0, N = StoredDependencies.size(); I != N; ++I) {
306 if (ModuleFile *MF = Modules[I].File)
307 Dependencies.push_back(MF);
308 }
309 }
310
311 bool GlobalModuleIndex::lookupIdentifier(StringRef Name, HitSet &Hits) {
312 Hits.clear();
313
314 // If there's no identifier index, there is nothing we can do.
315 if (!IdentifierIndex)
316 return false;
317
318 // Look into the identifier index.
319 ++NumIdentifierLookups;
320 IdentifierIndexTable &Table
321 = *static_cast<IdentifierIndexTable *>(IdentifierIndex);
322 IdentifierIndexTable::iterator Known = Table.find(Name);
323 if (Known == Table.end()) {
324 return true;
325 }
326
327 SmallVector<unsigned, 2> ModuleIDs = *Known;
328 for (unsigned I = 0, N = ModuleIDs.size(); I != N; ++I) {
329 if (ModuleFile *MF = Modules[ModuleIDs[I]].File)
330 Hits.insert(MF);
331 }
332
333 ++NumIdentifierLookupHits;
334 return true;
335 }
336
337 bool GlobalModuleIndex::loadedModuleFile(ModuleFile *File) {
338 // Look for the module in the global module index based on the module name.
339 StringRef Name = File->ModuleName;
340 llvm::StringMap<unsigned>::iterator Known = UnresolvedModules.find(Name);
341 if (Known == UnresolvedModules.end()) {
342 return true;
343 }
344
345 // Rectify this module with the global module index.
346 ModuleInfo &Info = Modules[Known->second];
347
348 // If the size and modification time match what we expected, record this
349 // module file.
350 bool Failed = true;
351 if (File->File->getSize() == Info.Size &&
352 File->File->getModificationTime() == Info.ModTime) {
353 Info.File = File;
354 ModulesByFile[File] = Known->second;
355
356 Failed = false;
357 }
358
359 // One way or another, we have resolved this module file.
360 UnresolvedModules.erase(Known);
361 return Failed;
362 }
363
364 void GlobalModuleIndex::printStats() {
365 std::fprintf(stderr, "*** Global Module Index Statistics:\n");
366 if (NumIdentifierLookups) {
367 fprintf(stderr, " %u / %u identifier lookups succeeded (%f%%)\n",
368 NumIdentifierLookupHits, NumIdentifierLookups,
369 (double)NumIdentifierLookupHits*100.0/NumIdentifierLookups);
370 }
371 std::fprintf(stderr, "\n");
372 }
373
374 LLVM_DUMP_METHOD void GlobalModuleIndex::dump() {
375 llvm::errs() << "*** Global Module Index Dump:\n";
376 llvm::errs() << "Module files:\n";
377 for (auto &MI : Modules) {
378 llvm::errs() << "** " << MI.FileName << "\n";
379 if (MI.File)
380 MI.File->dump();
381 else
382 llvm::errs() << "\n";
383 }
384 llvm::errs() << "\n";
385 }
386
387 //----------------------------------------------------------------------------//
388 // Global module index writer.
389 //----------------------------------------------------------------------------//
390
391 namespace {
392 /// Provides information about a specific module file.
393 struct ModuleFileInfo {
394 /// The numberic ID for this module file.
395 unsigned ID;
396
397 /// The set of modules on which this module depends. Each entry is
398 /// a module ID.
399 SmallVector<unsigned, 4> Dependencies;
400 ASTFileSignature Signature;
401 };
402
403 struct ImportedModuleFileInfo {
404 off_t StoredSize;
405 time_t StoredModTime;
406 ASTFileSignature StoredSignature;
407 ImportedModuleFileInfo(off_t Size, time_t ModTime, ASTFileSignature Sig)
408 : StoredSize(Size), StoredModTime(ModTime), StoredSignature(Sig) {}
409 };
410
411 /// Builder that generates the global module index file.
412 class GlobalModuleIndexBuilder {
413 FileManager &FileMgr;
414 const PCHContainerReader &PCHContainerRdr;
415
416 /// Mapping from files to module file information.
417 typedef llvm::MapVector<const FileEntry *, ModuleFileInfo> ModuleFilesMap;
418
419 /// Information about each of the known module files.
420 ModuleFilesMap ModuleFiles;
421
422 /// Mapping from the imported module file to the imported
423 /// information.
424 typedef std::multimap<const FileEntry *, ImportedModuleFileInfo>
425 ImportedModuleFilesMap;
426
427 /// Information about each importing of a module file.
428 ImportedModuleFilesMap ImportedModuleFiles;
429
430 /// Mapping from identifiers to the list of module file IDs that
431 /// consider this identifier to be interesting.
432 typedef llvm::StringMap<SmallVector<unsigned, 2> > InterestingIdentifierMap;
433
434 /// A mapping from all interesting identifiers to the set of module
435 /// files in which those identifiers are considered interesting.
436 InterestingIdentifierMap InterestingIdentifiers;
437
438 /// Write the block-info block for the global module index file.
439 void emitBlockInfoBlock(llvm::BitstreamWriter &Stream);
440
441 /// Retrieve the module file information for the given file.
442 ModuleFileInfo &getModuleFileInfo(const FileEntry *File) {
443 llvm::MapVector<const FileEntry *, ModuleFileInfo>::iterator Known
444 = ModuleFiles.find(File);
445 if (Known != ModuleFiles.end())
446 return Known->second;
447
448 unsigned NewID = ModuleFiles.size();
449 ModuleFileInfo &Info = ModuleFiles[File];
450 Info.ID = NewID;
451 return Info;
452 }
453
454 public:
455 explicit GlobalModuleIndexBuilder(
456 FileManager &FileMgr, const PCHContainerReader &PCHContainerRdr)
457 : FileMgr(FileMgr), PCHContainerRdr(PCHContainerRdr) {}
458
459 /// Load the contents of the given module file into the builder.
460 llvm::Error loadModuleFile(const FileEntry *File);
461
462 /// Write the index to the given bitstream.
463 /// \returns true if an error occurred, false otherwise.
464 bool writeIndex(llvm::BitstreamWriter &Stream);
465 };
466 }
467
468 static void emitBlockID(unsigned ID, const char *Name,
469 llvm::BitstreamWriter &Stream,
470 SmallVectorImpl<uint64_t> &Record) {
471 Record.clear();
472 Record.push_back(ID);
473 Stream.EmitRecord(llvm::bitc::BLOCKINFO_CODE_SETBID, Record);
474
475 // Emit the block name if present.
476 if (!Name || Name[0] == 0) return;
477 Record.clear();
478 while (*Name)
479 Record.push_back(*Name++);
480 Stream.EmitRecord(llvm::bitc::BLOCKINFO_CODE_BLOCKNAME, Record);
481 }
482
483 static void emitRecordID(unsigned ID, const char *Name,
484 llvm::BitstreamWriter &Stream,
485 SmallVectorImpl<uint64_t> &Record) {
486 Record.clear();
487 Record.push_back(ID);
488 while (*Name)
489 Record.push_back(*Name++);
490 Stream.EmitRecord(llvm::bitc::BLOCKINFO_CODE_SETRECORDNAME, Record);
491 }
492
493 void
494 GlobalModuleIndexBuilder::emitBlockInfoBlock(llvm::BitstreamWriter &Stream) {
495 SmallVector<uint64_t, 64> Record;
496 Stream.EnterBlockInfoBlock();
497
498 #define BLOCK(X) emitBlockID(X ## _ID, #X, Stream, Record)
499 #define RECORD(X) emitRecordID(X, #X, Stream, Record)
500 BLOCK(GLOBAL_INDEX_BLOCK);
501 RECORD(INDEX_METADATA);
502 RECORD(MODULE);
503 RECORD(IDENTIFIER_INDEX);
504 #undef RECORD
505 #undef BLOCK
506
507 Stream.ExitBlock();
508 }
509
510 namespace {
511 class InterestingASTIdentifierLookupTrait
512 : public serialization::reader::ASTIdentifierLookupTraitBase {
513
514 public:
515 /// The identifier and whether it is "interesting".
516 typedef std::pair<StringRef, bool> data_type;
517
518 data_type ReadData(const internal_key_type& k,
519 const unsigned char* d,
520 unsigned DataLen) {
521 // The first bit indicates whether this identifier is interesting.
522 // That's all we care about.
523 using namespace llvm::support;
524 unsigned RawID = endian::readNext<uint32_t, little, unaligned>(d);
525 bool IsInteresting = RawID & 0x01;
526 return std::make_pair(k, IsInteresting);
527 }
528 };
529 }
530
531 llvm::Error GlobalModuleIndexBuilder::loadModuleFile(const FileEntry *File) {
532 // Open the module file.
533
534 auto Buffer = FileMgr.getBufferForFile(File, /*isVolatile=*/true);
535 if (!Buffer)
536 return llvm::createStringError(Buffer.getError(),
537 "failed getting buffer for module file");
538
539 // Initialize the input stream
540 llvm::BitstreamCursor InStream(PCHContainerRdr.ExtractPCH(**Buffer));
541
542 // Sniff for the signature.
543 for (unsigned char C : {'C', 'P', 'C', 'H'})
544 if (Expected<llvm::SimpleBitstreamCursor::word_t> Res = InStream.Read(8)) {
545 if (Res.get() != C)
546 return llvm::createStringError(std::errc::illegal_byte_sequence,
547 "expected signature CPCH");
548 } else
549 return Res.takeError();
550
551 // Record this module file and assign it a unique ID (if it doesn't have
552 // one already).
553 unsigned ID = getModuleFileInfo(File).ID;
554
555 // Search for the blocks and records we care about.
556 enum { Other, ControlBlock, ASTBlock, DiagnosticOptionsBlock } State = Other;
557 bool Done = false;
558 while (!Done) {
559 Expected<llvm::BitstreamEntry> MaybeEntry = InStream.advance();
560 if (!MaybeEntry)
561 return MaybeEntry.takeError();
562 llvm::BitstreamEntry Entry = MaybeEntry.get();
563
564 switch (Entry.Kind) {
565 case llvm::BitstreamEntry::Error:
566 Done = true;
567 continue;
568
569 case llvm::BitstreamEntry::Record:
570 // In the 'other' state, just skip the record. We don't care.
571 if (State == Other) {
572 if (llvm::Expected<unsigned> Skipped = InStream.skipRecord(Entry.ID))
573 continue;
574 else
575 return Skipped.takeError();
576 }
577
578 // Handle potentially-interesting records below.
579 break;
580
581 case llvm::BitstreamEntry::SubBlock:
582 if (Entry.ID == CONTROL_BLOCK_ID) {
583 if (llvm::Error Err = InStream.EnterSubBlock(CONTROL_BLOCK_ID))
584 return Err;
585
586 // Found the control block.
587 State = ControlBlock;
588 continue;
589 }
590
591 if (Entry.ID == AST_BLOCK_ID) {
592 if (llvm::Error Err = InStream.EnterSubBlock(AST_BLOCK_ID))
593 return Err;
594
595 // Found the AST block.
596 State = ASTBlock;
597 continue;
598 }
599
600 if (Entry.ID == UNHASHED_CONTROL_BLOCK_ID) {
601 if (llvm::Error Err = InStream.EnterSubBlock(UNHASHED_CONTROL_BLOCK_ID))
602 return Err;
603
604 // Found the Diagnostic Options block.
605 State = DiagnosticOptionsBlock;
606 continue;
607 }
608
609 if (llvm::Error Err = InStream.SkipBlock())
610 return Err;
611
612 continue;
613
614 case llvm::BitstreamEntry::EndBlock:
615 State = Other;
616 continue;
617 }
618
619 // Read the given record.
620 SmallVector<uint64_t, 64> Record;
621 StringRef Blob;
622 Expected<unsigned> MaybeCode = InStream.readRecord(Entry.ID, Record, &Blob);
623 if (!MaybeCode)
624 return MaybeCode.takeError();
625 unsigned Code = MaybeCode.get();
626
627 // Handle module dependencies.
628 if (State == ControlBlock && Code == IMPORTS) {
629 // Load each of the imported PCH files.
630 unsigned Idx = 0, N = Record.size();
631 while (Idx < N) {
632 // Read information about the AST file.
633
634 // Skip the imported kind
635 ++Idx;
636
637 // Skip the import location
638 ++Idx;
639
640 // Load stored size/modification time.
641 off_t StoredSize = (off_t)Record[Idx++];
642 time_t StoredModTime = (time_t)Record[Idx++];
643
644 // Skip the stored signature.
645 // FIXME: we could read the signature out of the import and validate it.
646 ASTFileSignature StoredSignature = {
647 {{(uint32_t)Record[Idx++], (uint32_t)Record[Idx++],
648 (uint32_t)Record[Idx++], (uint32_t)Record[Idx++],
649 (uint32_t)Record[Idx++]}}};
650
651 // Skip the module name (currently this is only used for prebuilt
652 // modules while here we are only dealing with cached).
653 Idx += Record[Idx] + 1;
654
655 // Retrieve the imported file name.
656 unsigned Length = Record[Idx++];
657 SmallString<128> ImportedFile(Record.begin() + Idx,
658 Record.begin() + Idx + Length);
659 Idx += Length;
660
661 // Find the imported module file.
662 auto DependsOnFile
663 = FileMgr.getFile(ImportedFile, /*OpenFile=*/false,
664 /*CacheFailure=*/false);
665
666 if (!DependsOnFile)
667 return llvm::createStringError(std::errc::bad_file_descriptor,
668 "imported file \"%s\" not found",
669 ImportedFile.c_str());
670
671 // Save the information in ImportedModuleFileInfo so we can verify after
672 // loading all pcms.
673 ImportedModuleFiles.insert(std::make_pair(
674 *DependsOnFile, ImportedModuleFileInfo(StoredSize, StoredModTime,
675 StoredSignature)));
676
677 // Record the dependency.
678 unsigned DependsOnID = getModuleFileInfo(*DependsOnFile).ID;
679 getModuleFileInfo(File).Dependencies.push_back(DependsOnID);
680 }
681
682 continue;
683 }
684
685 // Handle the identifier table
686 if (State == ASTBlock && Code == IDENTIFIER_TABLE && Record[0] > 0) {
687 typedef llvm::OnDiskIterableChainedHashTable<
688 InterestingASTIdentifierLookupTrait> InterestingIdentifierTable;
689 std::unique_ptr<InterestingIdentifierTable> Table(
690 InterestingIdentifierTable::Create(
691 (const unsigned char *)Blob.data() + Record[0],
692 (const unsigned char *)Blob.data() + sizeof(uint32_t),
693 (const unsigned char *)Blob.data()));
694 for (InterestingIdentifierTable::data_iterator D = Table->data_begin(),
695 DEnd = Table->data_end();
696 D != DEnd; ++D) {
697 std::pair<StringRef, bool> Ident = *D;
698 if (Ident.second)
699 InterestingIdentifiers[Ident.first].push_back(ID);
700 else
701 (void)InterestingIdentifiers[Ident.first];
702 }
703 }
704
705 // Get Signature.
706 if (State == DiagnosticOptionsBlock && Code == SIGNATURE)
707 getModuleFileInfo(File).Signature = {
708 {{(uint32_t)Record[0], (uint32_t)Record[1], (uint32_t)Record[2],
709 (uint32_t)Record[3], (uint32_t)Record[4]}}};
710
711 // We don't care about this record.
712 }
713
714 return llvm::Error::success();
715 }
716
717 namespace {
718
719 /// Trait used to generate the identifier index as an on-disk hash
720 /// table.
721 class IdentifierIndexWriterTrait {
722 public:
723 typedef StringRef key_type;
724 typedef StringRef key_type_ref;
725 typedef SmallVector<unsigned, 2> data_type;
726 typedef const SmallVector<unsigned, 2> &data_type_ref;
727 typedef unsigned hash_value_type;
728 typedef unsigned offset_type;
729
730 static hash_value_type ComputeHash(key_type_ref Key) {
731 return llvm::djbHash(Key);
732 }
733
734 std::pair<unsigned,unsigned>
735 EmitKeyDataLength(raw_ostream& Out, key_type_ref Key, data_type_ref Data) {
736 using namespace llvm::support;
737 endian::Writer LE(Out, little);
738 unsigned KeyLen = Key.size();
739 unsigned DataLen = Data.size() * 4;
740 LE.write<uint16_t>(KeyLen);
741 LE.write<uint16_t>(DataLen);
742 return std::make_pair(KeyLen, DataLen);
743 }
744
745 void EmitKey(raw_ostream& Out, key_type_ref Key, unsigned KeyLen) {
746 Out.write(Key.data(), KeyLen);
747 }
748
749 void EmitData(raw_ostream& Out, key_type_ref Key, data_type_ref Data,
750 unsigned DataLen) {
751 using namespace llvm::support;
752 for (unsigned I = 0, N = Data.size(); I != N; ++I)
753 endian::write<uint32_t>(Out, Data[I], little);
754 }
755 };
756
757 }
758
759 bool GlobalModuleIndexBuilder::writeIndex(llvm::BitstreamWriter &Stream) {
760 for (auto MapEntry : ImportedModuleFiles) {
761 auto *File = MapEntry.first;
762 ImportedModuleFileInfo &Info = MapEntry.second;
763 if (getModuleFileInfo(File).Signature) {
764 if (getModuleFileInfo(File).Signature != Info.StoredSignature)
765 // Verify Signature.
766 return true;
767 } else if (Info.StoredSize != File->getSize() ||
768 Info.StoredModTime != File->getModificationTime())
769 // Verify Size and ModTime.
770 return true;
771 }
772
773 using namespace llvm;
774 llvm::TimeTraceScope TimeScope("Module WriteIndex");
775
776 // Emit the file header.
777 Stream.Emit((unsigned)'B', 8);
778 Stream.Emit((unsigned)'C', 8);
779 Stream.Emit((unsigned)'G', 8);
780 Stream.Emit((unsigned)'I', 8);
781
782 // Write the block-info block, which describes the records in this bitcode
783 // file.
784 emitBlockInfoBlock(Stream);
785
786 Stream.EnterSubblock(GLOBAL_INDEX_BLOCK_ID, 3);
787
788 // Write the metadata.
789 SmallVector<uint64_t, 2> Record;
790 Record.push_back(CurrentVersion);
791 Stream.EmitRecord(INDEX_METADATA, Record);
792
793 // Write the set of known module files.
794 for (ModuleFilesMap::iterator M = ModuleFiles.begin(),
795 MEnd = ModuleFiles.end();
796 M != MEnd; ++M) {
797 Record.clear();
798 Record.push_back(M->second.ID);
799 Record.push_back(M->first->getSize());
800 Record.push_back(M->first->getModificationTime());
801
802 // File name
803 StringRef Name(M->first->getName());
804 Record.push_back(Name.size());
805 Record.append(Name.begin(), Name.end());
806
807 // Dependencies
808 Record.push_back(M->second.Dependencies.size());
809 Record.append(M->second.Dependencies.begin(), M->second.Dependencies.end());
810 Stream.EmitRecord(MODULE, Record);
811 }
812
813 // Write the identifier -> module file mapping.
814 {
815 llvm::OnDiskChainedHashTableGenerator<IdentifierIndexWriterTrait> Generator;
816 IdentifierIndexWriterTrait Trait;
817
818 // Populate the hash table.
819 for (InterestingIdentifierMap::iterator I = InterestingIdentifiers.begin(),
820 IEnd = InterestingIdentifiers.end();
821 I != IEnd; ++I) {
822 Generator.insert(I->first(), I->second, Trait);
823 }
824
825 // Create the on-disk hash table in a buffer.
826 SmallString<4096> IdentifierTable;
827 uint32_t BucketOffset;
828 {
829 using namespace llvm::support;
830 llvm::raw_svector_ostream Out(IdentifierTable);
831 // Make sure that no bucket is at offset 0
832 endian::write<uint32_t>(Out, 0, little);
833 BucketOffset = Generator.Emit(Out, Trait);
834 }
835
836 // Create a blob abbreviation
837 auto Abbrev = std::make_shared<BitCodeAbbrev>();
838 Abbrev->Add(BitCodeAbbrevOp(IDENTIFIER_INDEX));
839 Abbrev->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 32));
840 Abbrev->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Blob));
841 unsigned IDTableAbbrev = Stream.EmitAbbrev(std::move(Abbrev));
842
843 // Write the identifier table
844 uint64_t Record[] = {IDENTIFIER_INDEX, BucketOffset};
845 Stream.EmitRecordWithBlob(IDTableAbbrev, Record, IdentifierTable);
846 }
847
848 Stream.ExitBlock();
849 return false;
850 }
851
852 llvm::Error
853 GlobalModuleIndex::writeIndex(FileManager &FileMgr,
854 const PCHContainerReader &PCHContainerRdr,
855 StringRef Path) {
856 llvm::SmallString<128> IndexPath;
857 IndexPath += Path;
858 llvm::sys::path::append(IndexPath, IndexFileName);
859
860 // Coordinate building the global index file with other processes that might
861 // try to do the same.
862 llvm::LockFileManager Locked(IndexPath);
863 switch (Locked) {
864 case llvm::LockFileManager::LFS_Error:
865 return llvm::createStringError(std::errc::io_error, "LFS error");
866
867 case llvm::LockFileManager::LFS_Owned:
868 // We're responsible for building the index ourselves. Do so below.
869 break;
870
871 case llvm::LockFileManager::LFS_Shared:
872 // Someone else is responsible for building the index. We don't care
873 // when they finish, so we're done.
874 return llvm::createStringError(std::errc::device_or_resource_busy,
875 "someone else is building the index");
876 }
877
878 // The module index builder.
879 GlobalModuleIndexBuilder Builder(FileMgr, PCHContainerRdr);
880
881 // Load each of the module files.
882 std::error_code EC;
883 for (llvm::sys::fs::directory_iterator D(Path, EC), DEnd;
884 D != DEnd && !EC;
885 D.increment(EC)) {
886 // If this isn't a module file, we don't care.
887 if (llvm::sys::path::extension(D->path()) != ".pcm") {
888 // ... unless it's a .pcm.lock file, which indicates that someone is
889 // in the process of rebuilding a module. They'll rebuild the index
890 // at the end of that translation unit, so we don't have to.
891 if (llvm::sys::path::extension(D->path()) == ".pcm.lock")
892 return llvm::createStringError(std::errc::device_or_resource_busy,
893 "someone else is building the index");
894
895 continue;
896 }
897
898 // If we can't find the module file, skip it.
899 auto ModuleFile = FileMgr.getFile(D->path());
900 if (!ModuleFile)
901 continue;
902
903 // Load this module file.
904 if (llvm::Error Err = Builder.loadModuleFile(*ModuleFile))
905 return Err;
906 }
907
908 // The output buffer, into which the global index will be written.
909 SmallVector<char, 16> OutputBuffer;
910 {
911 llvm::BitstreamWriter OutputStream(OutputBuffer);
912 if (Builder.writeIndex(OutputStream))
913 return llvm::createStringError(std::errc::io_error,
914 "failed writing index");
915 }
916
917 return llvm::writeFileAtomically(
918 (IndexPath + "-%%%%%%%%").str(), IndexPath,
919 llvm::StringRef(OutputBuffer.data(), OutputBuffer.size()));
920 }
921
922 namespace {
923 class GlobalIndexIdentifierIterator : public IdentifierIterator {
924 /// The current position within the identifier lookup table.
925 IdentifierIndexTable::key_iterator Current;
926
927 /// The end position within the identifier lookup table.
928 IdentifierIndexTable::key_iterator End;
929
930 public:
931 explicit GlobalIndexIdentifierIterator(IdentifierIndexTable &Idx) {
932 Current = Idx.key_begin();
933 End = Idx.key_end();
934 }
935
936 StringRef Next() override {
937 if (Current == End)
938 return StringRef();
939
940 StringRef Result = *Current;
941 ++Current;
942 return Result;
943 }
944 };
945 }
946
947 IdentifierIterator *GlobalModuleIndex::createIdentifierIterator() const {
948 IdentifierIndexTable &Table =
949 *static_cast<IdentifierIndexTable *>(IdentifierIndex);
950 return new GlobalIndexIdentifierIterator(Table);
951 }
clang::Builtin::ID
ID
Definition: Builtins.h:46
ASTReaderInternals.h
clang::FileManager
Implements support for file system lookup, file system caching, and directory search management...
Definition: FileManager.h:171
FileManager.h
Defines the clang::FileManager interface and associated types.
clang::FileEntry::getModificationTime
time_t getModificationTime() const
Definition: FileManager.h:108
llvm
Specialize PointerLikeTypeTraits to allow LazyGenerationalUpdatePtr to be placed into a PointerUnion...
Definition: Dominators.h:30
Cursor
Cursor
Definition: SerializedDiagnosticReader.cpp:109
clang::serialization::ModuleFile::ModuleName
std::string ModuleName
The name of the module.
Definition: ModuleFile.h:125
HeaderSearch.h
CurrentVersion
static const unsigned CurrentVersion
The global index file version.
Definition: GlobalModuleIndex.cpp:63
clang::serialization::UNHASHED_CONTROL_BLOCK_ID
A block with unhashed content.
Definition: ASTBitCodes.h:296
clang::GlobalModuleIndex::printStats
void printStats()
Print statistics to standard error.
Definition: GlobalModuleIndex.cpp:364
emitRecordID
static void emitRecordID(unsigned ID, const char *Name, llvm::BitstreamWriter &Stream, SmallVectorImpl< uint64_t > &Record)
Definition: GlobalModuleIndex.cpp:483
State
LineState State
Definition: UnwrappedLineFormatter.cpp:932
clang::GlobalModuleIndex::writeIndex
static llvm::Error writeIndex(FileManager &FileMgr, const PCHContainerReader &PCHContainerRdr, llvm::StringRef Path)
Write a global index into the given.
Definition: GlobalModuleIndex.cpp:853
toString
static std::string toString(const clang::SanitizerSet &Sanitizers)
Produce a string containing comma-separated names of sanitizers in Sanitizers set.
Definition: SanitizerArgs.cpp:856
clang::GlobalModuleIndex::HitSet
llvm::SmallPtrSet< ModuleFile *, 4 > HitSet
A set of module files in which we found a result.
Definition: GlobalModuleIndex.h:153
clang::ASTFileSignature
The signature of a module, which is a hash of the AST content.
Definition: Module.h:54
llvm::SmallVectorImpl
Definition: LLVM.h:36
llvm::Expected
Definition: LLVM.h:38
ModuleFile.h
Error
llvm::Error Error
Definition: ByteCodeEmitter.cpp:19
clang::PCHContainerReader
This abstract interface provides operations for unwrapping containers for serialized ASTs (precompile...
Definition: PCHContainerOperations.h:57
llvm::ArrayRef< unsigned >
llvm::SmallString< 128 >
clang::GlobalModuleIndex::lookupIdentifier
bool lookupIdentifier(llvm::StringRef Name, HitSet &Hits)
Look for all of the module files with information about the given identifier, e.g., a global function, variable, or type with that name.
Definition: GlobalModuleIndex.cpp:311
clang::GlobalModuleIndex::dump
void dump()
Print debugging view to standard error.
Definition: GlobalModuleIndex.cpp:374
End
SourceLocation End
Definition: USRLocFinder.cpp:166
emitBlockID
static void emitBlockID(unsigned ID, const char *Name, llvm::BitstreamWriter &Stream, SmallVectorImpl< uint64_t > &Record)
Definition: GlobalModuleIndex.cpp:468
clang::GlobalModuleIndex::createIdentifierIterator
IdentifierIterator * createIdentifierIterator() const
Returns an iterator for identifiers stored in the index table.
Definition: GlobalModuleIndex.cpp:947
clang::IdentifierTable
Implements an efficient mapping from strings to IdentifierInfo nodes.
Definition: IdentifierTable.h:496
clang::serialization::reader::ASTIdentifierLookupTraitBase
Base class for the trait describing the on-disk hash table for the identifiers in an AST file...
Definition: ASTReaderInternals.h:131
clang::serialization::ModuleFile
Information about a module that has been loaded by the ASTReader.
Definition: ModuleFile.h:107
clang::IdentifierIterator
An iterator that walks over all of the known identifiers in the lookup table.
Definition: IdentifierTable.h:448
ASTBitCodes.h
clang::DiagnosticLevelMask::Error
IndexFileName
static const char *const IndexFileName
The name of the global index file.
Definition: GlobalModuleIndex.cpp:60
clang::serialization::IDENTIFIER_TABLE
Record code for the identifier table.
Definition: ASTBitCodes.h:444
IndexRecordTypes
IndexRecordTypes
Describes the record types in the index.
Definition: GlobalModuleIndex.cpp:48
clang::FileEntry
Cached information about one file (either on disk or in the virtual file system). ...
Definition: FileManager.h:78
GlobalModuleIndex.h
clang::GlobalModuleIndex::readIndex
static std::pair< GlobalModuleIndex *, llvm::Error > readIndex(llvm::StringRef Path)
Read a global index file for the given directory.
Definition: GlobalModuleIndex.cpp:253
clang::GlobalModuleIndex::getKnownModules
void getKnownModules(llvm::SmallVectorImpl< ModuleFile *> &ModuleFiles)
Retrieve the set of modules that have up-to-date indexes.
Definition: GlobalModuleIndex.cpp:285
llvm::SmallVector
Definition: LLVM.h:35
clang::GlobalModuleIndex::~GlobalModuleIndex
~GlobalModuleIndex()
Definition: GlobalModuleIndex.cpp:248
clang::serialization::AST_BLOCK_ID
The AST block, which acts as a container around the full AST block.
Definition: ASTBitCodes.h:248
clang::GlobalModuleIndex
A global index for a set of module files, providing information about the identifiers within those mo...
Definition: GlobalModuleIndex.h:56
clang::serialization::ComputeHash
unsigned ComputeHash(Selector Sel)
Definition: ASTCommon.cpp:251
clang::serialization::SIGNATURE
Record code for the signature that identifiers this AST file.
Definition: ASTBitCodes.h:363
clang::serialization::IMPORTS
Record code for the list of other AST files imported by this AST file.
Definition: ASTBitCodes.h:307
clang
Dataflow Directional Tag Classes.
Definition: CFGReachabilityAnalysis.h:21
clang::FileEntry::getSize
off_t getSize() const
Definition: FileManager.h:105
BLOCK
#define BLOCK(X)
clang::FileManager::getFile
llvm::ErrorOr< const FileEntry * > getFile(StringRef Filename, bool OpenFile=false, bool CacheFailure=true)
Lookup, cache, and verify the specified file (real or virtual).
Definition: FileManager.cpp:195
clang::GlobalModuleIndex::getModuleDependencies
void getModuleDependencies(ModuleFile *File, llvm::SmallVectorImpl< ModuleFile *> &Dependencies)
Retrieve the set of module files on which the given module file directly depends. ...
Definition: GlobalModuleIndex.cpp:293
PCHContainerOperations.h
clang::interp::LE
bool LE(InterpState &S, CodePtr OpPC)
Definition: Interp.h:237
clang::frontend::ModuleFileInfo
Dump information about a module file.
Definition: FrontendOptions.h:99
RECORD
#define RECORD(X)
clang::serialization::CONTROL_BLOCK_ID
The control block, which contains all of the information that needs to be validated prior to committi...
Definition: ASTBitCodes.h:274
clang::serialization::ModuleFile::File
const FileEntry * File
The file entry for the module file.
Definition: ModuleFile.h:166
clang::Create
static OMPLinearClause * Create(const ASTContext &C, SourceLocation StartLoc, SourceLocation LParenLoc, OpenMPLinearClauseKind Modifier, SourceLocation ModifierLoc, SourceLocation ColonLoc, SourceLocation EndLoc, ArrayRef< Expr *> VL, ArrayRef< Expr *> PL, ArrayRef< Expr *> IL, Expr *Step, Expr *CalcStep, Stmt *PreInit, Expr *PostUpdate)
Creates clause with a list of variables VL and a linear step Step.
clang::GlobalModuleIndex::loadedModuleFile
bool loadedModuleFile(ModuleFile *File)
Note that the given module file has been loaded.
Definition: GlobalModuleIndex.cpp:337