22 #include "clang/Basic/SourceLocation.h" 23 #include "clang/Basic/SourceManager.h" 24 #include "llvm/ADT/STLExtras.h" 25 #include "llvm/ADT/ScopeExit.h" 26 #include "llvm/ADT/StringMap.h" 27 #include "llvm/ADT/StringRef.h" 28 #include "llvm/Support/SHA1.h" 42 class URIToFileCache {
44 URIToFileCache(llvm::StringRef HintPath) : HintPath(HintPath) {}
46 llvm::StringRef resolve(llvm::StringRef FileURI) {
47 auto I = URIToPathCache.try_emplace(FileURI);
51 elog(
"Failed to parse URI {0}: {1}", FileURI, U.takeError());
52 assert(
false &&
"Failed to parse URI");
57 elog(
"Failed to resolve URI {0}: {1}", FileURI,
Path.takeError());
58 assert(
false &&
"Failed to resolve URI");
61 I.first->second = *
Path;
63 return I.first->second;
68 llvm::StringMap<std::string> URIToPathCache;
76 std::string FileURI = U.toString();
77 auto Entry = IG.try_emplace(FileURI).first;
78 auto &Node =
Entry->getValue();
79 Node = FullGraph.lookup(
Entry->getKey());
80 Node.URI =
Entry->getKey();
83 for (
auto &Include : Node.DirectIncludes) {
84 auto I = IG.try_emplace(Include).first;
85 I->getValue().URI = I->getKey();
86 Include = I->getKey();
96 createFileFilter(
const llvm::StringMap<FileDigest> &FileDigests) {
97 return [&FileDigests](
const SourceManager &SM, FileID FID) {
98 const auto *F = SM.getFileEntryForID(FID);
107 auto D = FileDigests.find(*AbsPath);
108 if (
D != FileDigests.end() &&
D->second == Digest)
117 llvm::SmallString<128> getAbsolutePath(
const tooling::CompileCommand &Cmd) {
118 llvm::SmallString<128> AbsolutePath;
119 if (llvm::sys::path::is_absolute(Cmd.Filename)) {
120 AbsolutePath = Cmd.Filename;
122 AbsolutePath = Cmd.Directory;
123 llvm::sys::path::append(AbsolutePath, Cmd.Filename);
133 size_t BuildIndexPeriodMs,
size_t ThreadPoolSize)
135 CDB(CDB), BackgroundContext(std::move(BackgroundContext)),
136 BuildIndexPeriodMs(BuildIndexPeriodMs),
137 SymbolsUpdatedSinceLastIndex(false),
138 IndexStorageFactory(std::move(IndexStorageFactory)),
140 CDB.watch([&](const std::vector<std::string> &ChangedFiles) {
143 assert(ThreadPoolSize > 0 &&
"Thread pool size can't be zero.");
144 assert(this->IndexStorageFactory &&
"Storage factory can not be null!");
145 while (ThreadPoolSize--)
146 ThreadPool.emplace_back([
this] { run(); });
147 if (BuildIndexPeriodMs > 0) {
148 log(
"BackgroundIndex: build symbol index periodically every {0} ms.",
150 ThreadPool.emplace_back([
this] { buildIndex(); });
156 for (
auto &Thread : ThreadPool)
162 std::lock_guard<std::mutex> QueueLock(QueueMu);
163 std::lock_guard<std::mutex> IndexLock(IndexMu);
166 QueueCV.notify_all();
167 IndexCV.notify_all();
170 void BackgroundIndex::run() {
173 llvm::Optional<Task> Task;
176 std::unique_lock<std::mutex> Lock(QueueMu);
177 QueueCV.wait(Lock, [&] {
return ShouldStop || !Queue.empty(); });
180 QueueCV.notify_all();
184 std::tie(Task, Priority) = std::move(Queue.front());
195 std::unique_lock<std::mutex> Lock(QueueMu);
196 assert(NumActiveTasks > 0 &&
"before decrementing");
199 QueueCV.notify_all();
204 llvm::Optional<double> TimeoutSeconds) {
205 std::unique_lock<std::mutex> Lock(QueueMu);
207 [&] {
return Queue.empty() && NumActiveTasks == 0; });
212 [
this, ChangedFiles] {
215 log(
"Enqueueing {0} commands for indexing", ChangedFiles.size());
216 SPAN_ATTACH(Tracer,
"files", int64_t(ChangedFiles.size()));
218 auto NeedsReIndexing = loadShards(std::move(ChangedFiles));
220 std::shuffle(NeedsReIndexing.begin(), NeedsReIndexing.end(),
221 std::mt19937(std::random_device{}()));
222 for (
auto &Elem : NeedsReIndexing)
223 enqueue(std::move(Elem.first), Elem.second);
231 [
this, Storage](tooling::CompileCommand Cmd) {
234 const std::string
FileName = Cmd.Filename;
235 if (
auto Error = index(std::move(Cmd), Storage))
236 elog(
"Indexing {0} failed: {1}", FileName,
243 void BackgroundIndex::enqueueTask(Task T,
ThreadPriority Priority) {
245 std::lock_guard<std::mutex> Lock(QueueMu);
246 auto I = Queue.end();
252 I = llvm::find_if(Queue, [](
const std::pair<Task, ThreadPriority> &Elem) {
256 Queue.insert(I, {std::move(T), Priority});
258 QueueCV.notify_all();
264 void BackgroundIndex::update(llvm::StringRef MainFile,
IndexFileIn Index,
265 const llvm::StringMap<FileDigest> &DigestsSnapshot,
269 llvm::DenseSet<const Symbol *> Symbols;
270 llvm::DenseSet<const Ref *> Refs;
273 llvm::StringMap<File> Files;
274 URIToFileCache URICache(MainFile);
275 for (
const auto &IndexIt : *Index.
Sources) {
276 const auto &IGN = IndexIt.getValue();
277 const auto AbsPath = URICache.resolve(IGN.URI);
278 const auto DigestIt = DigestsSnapshot.find(AbsPath);
280 if (DigestIt == DigestsSnapshot.end() || DigestIt->getValue() != IGN.Digest)
281 Files.try_emplace(AbsPath).first->getValue().Digest = IGN.Digest;
283 for (
const auto &Sym : *Index.
Symbols) {
284 if (Sym.CanonicalDeclaration) {
285 auto DeclPath = URICache.resolve(Sym.CanonicalDeclaration.FileURI);
286 const auto FileIt = Files.find(DeclPath);
287 if (FileIt != Files.end())
288 FileIt->second.Symbols.insert(&Sym);
294 if (Sym.Definition &&
295 Sym.Definition.FileURI != Sym.CanonicalDeclaration.FileURI) {
296 auto DefPath = URICache.resolve(Sym.Definition.FileURI);
297 const auto FileIt = Files.find(DefPath);
298 if (FileIt != Files.end())
299 FileIt->second.Symbols.insert(&Sym);
302 llvm::DenseMap<const Ref *, SymbolID> RefToIDs;
303 for (
const auto &SymRefs : *Index.
Refs) {
304 for (
const auto &R : SymRefs.second) {
305 auto Path = URICache.resolve(R.Location.FileURI);
306 const auto FileIt = Files.find(
Path);
307 if (FileIt != Files.end()) {
308 auto &F = FileIt->getValue();
309 RefToIDs[&R] = SymRefs.first;
316 for (
const auto &FileIt : Files) {
317 llvm::StringRef
Path = FileIt.getKey();
320 for (
const auto *S : FileIt.second.Symbols)
322 for (
const auto *R : FileIt.second.Refs)
323 Refs.
insert(RefToIDs[R], *R);
324 auto SS = llvm::make_unique<SymbolSlab>(std::move(Syms).build());
325 auto RS = llvm::make_unique<RefSlab>(std::move(Refs).build());
326 auto IG = llvm::make_unique<IncludeGraph>(
333 Shard.
Refs = RS.get();
336 if (
auto Error = IndexStorage->
storeShard(Path, Shard))
337 elog(
"Failed to write background-index shard for file {0}: {1}", Path,
341 std::lock_guard<std::mutex> Lock(DigestsMu);
342 auto Hash = FileIt.second.Digest;
344 auto DigestIt = IndexedFileDigests.try_emplace(Path);
345 if (!DigestIt.second && DigestIt.first->second == Hash)
347 DigestIt.first->second = Hash;
351 IndexedSymbols.update(Path, std::move(SS), std::move(RS));
356 void BackgroundIndex::buildIndex() {
357 assert(BuildIndexPeriodMs > 0);
360 std::unique_lock<std::mutex> Lock(IndexMu);
364 IndexCV.wait_for(Lock, std::chrono::milliseconds(BuildIndexPeriodMs));
368 if (!SymbolsUpdatedSinceLastIndex.exchange(
false))
376 log(
"BackgroundIndex: rebuilt symbol index.");
380 llvm::Error BackgroundIndex::index(tooling::CompileCommand Cmd,
384 auto AbsolutePath = getAbsolutePath(Cmd);
386 auto FS = FSProvider.getFileSystem();
387 auto Buf = FS->getBufferForFile(AbsolutePath);
389 return llvm::errorCodeToError(Buf.getError());
390 auto Hash =
digest(Buf->get()->getBuffer());
393 llvm::StringMap<FileDigest> DigestsSnapshot;
395 std::lock_guard<std::mutex> Lock(DigestsMu);
396 DigestsSnapshot = IndexedFileDigests;
399 log(
"Indexing {0} (digest:={1})", Cmd.Filename, llvm::toHex(Hash));
401 Inputs.
FS = std::move(FS);
402 Inputs.
FS->setCurrentWorkingDirectory(Cmd.Directory);
406 return llvm::createStringError(llvm::inconvertibleErrorCode(),
407 "Couldn't build compiler invocation");
410 std::move(CI),
nullptr, std::move(*Buf),
411 std::make_shared<PCHContainerOperations>(), Inputs.
FS, IgnoreDiags);
413 return llvm::createStringError(llvm::inconvertibleErrorCode(),
414 "Couldn't build compiler instance");
417 IndexOpts.
FileFilter = createFileFilter(DigestsSnapshot);
429 const FrontendInputFile &Input = Clang->getFrontendOpts().Inputs.front();
430 if (!
Action->BeginSourceFile(*Clang, Input))
431 return llvm::createStringError(llvm::inconvertibleErrorCode(),
432 "BeginSourceFile() failed");
434 return llvm::createStringError(llvm::inconvertibleErrorCode(),
437 if (Clang->hasDiagnostics() &&
438 Clang->getDiagnostics().hasUncompilableErrorOccurred()) {
439 return llvm::createStringError(
440 llvm::inconvertibleErrorCode(),
441 "IndexingAction failed: has uncompilable errors");
445 "Symbols, Refs and Sources must be set.");
447 log(
"Indexed {0} ({1} symbols, {2} refs, {3} files)",
454 update(AbsolutePath, std::move(Index), DigestsSnapshot, IndexStorage);
456 if (BuildIndexPeriodMs > 0)
457 SymbolsUpdatedSinceLastIndex =
true;
462 return llvm::Error::success();
465 std::vector<BackgroundIndex::Source>
466 BackgroundIndex::loadShard(
const tooling::CompileCommand &Cmd,
468 llvm::StringSet<> &LoadedShards) {
470 std::string AbsolutePath;
471 std::unique_ptr<IndexFileIn> Shard;
474 std::vector<ShardInfo> IntermediateSymbols;
477 llvm::StringSet<> InQueue;
478 auto FS = FSProvider.getFileSystem();
481 std::vector<Source> Dependencies;
482 std::queue<Source> ToVisit;
483 std::string AbsolutePath = getAbsolutePath(Cmd).str();
486 ToVisit.emplace(AbsolutePath,
true);
487 InQueue.insert(AbsolutePath);
489 while (!ToVisit.empty()) {
490 Dependencies.push_back(std::move(ToVisit.front()));
493 auto &CurDependency = Dependencies.back();
498 if (!LoadedShards.try_emplace(CurDependency.Path).second) {
501 CurDependency.NeedsReIndexing =
false;
505 auto Shard = IndexStorage->
loadShard(CurDependency.Path);
506 if (!Shard || !Shard->Sources) {
508 vlog(
"Failed to load shard: {0}", CurDependency.Path);
512 for (
const auto &I : *Shard->Sources) {
516 auto AbsolutePath =
URI::resolve(*U, CurDependency.Path);
520 if (InQueue.try_emplace(*AbsolutePath).second)
521 ToVisit.emplace(*AbsolutePath,
true);
524 if (*AbsolutePath != CurDependency.Path)
528 assert(I.getValue().Digest !=
FileDigest{{0}} &&
"Digest is empty?");
530 SI.AbsolutePath = CurDependency.Path;
531 SI.Shard = std::move(Shard);
532 SI.Digest = I.getValue().Digest;
533 IntermediateSymbols.push_back(std::move(SI));
536 auto Buf = FS->getBufferForFile(CurDependency.Path);
538 elog(
"Couldn't get buffer for file: {0}: {1}", CurDependency.Path,
539 Buf.getError().message());
543 CurDependency.NeedsReIndexing =
544 digest(Buf->get()->getBuffer()) != I.getValue().Digest;
549 std::lock_guard<std::mutex> Lock(DigestsMu);
553 for (
const ShardInfo &SI : IntermediateSymbols) {
556 ? llvm::make_unique<SymbolSlab>(std::move(*SI.Shard->Symbols))
558 auto RS = SI.Shard->Refs
559 ? llvm::make_unique<RefSlab>(std::move(*SI.Shard->Refs))
561 IndexedFileDigests[SI.AbsolutePath] = SI.Digest;
562 IndexedSymbols.update(SI.AbsolutePath, std::move(SS), std::move(RS));
571 std::vector<std::pair<tooling::CompileCommand, BackgroundIndexStorage *>>
572 BackgroundIndex::loadShards(std::vector<std::string> ChangedFiles) {
573 std::vector<std::pair<tooling::CompileCommand, BackgroundIndexStorage *>>
577 llvm::StringSet<> FilesToIndex;
580 llvm::StringSet<> LoadedShards;
581 for (
const auto &
File : ChangedFiles) {
583 auto Cmd = CDB.getCompileCommand(
File, &PI);
587 auto Dependencies = loadShard(*Cmd, IndexStorage, LoadedShards);
588 for (
const auto &Dependency : Dependencies) {
589 if (!Dependency.NeedsReIndexing || FilesToIndex.count(Dependency.Path))
594 vlog(
"Enqueueing TU {0} because its dependency {1} needs re-indexing.",
595 Cmd->Filename, Dependency.Path);
596 NeedsReIndexing.push_back({std::move(*Cmd), IndexStorage});
599 for (
const auto &Dependency : Dependencies)
600 FilesToIndex.insert(Dependency.Path);
604 vlog(
"Loaded all shards");
607 return NeedsReIndexing;
llvm::Optional< SymbolSlab > Symbols
Some operations such as code completion produce a set of candidates.
decltype(llvm::SHA1::hash({})) FileDigest
void insert(const Symbol &S)
void vlog(const char *Fmt, Ts &&... Vals)
void elog(const char *Fmt, Ts &&... Vals)
llvm::Optional< IncludeGraph > Sources
const SymbolSlab * Symbols
llvm::unique_function< BackgroundIndexStorage *(llvm::StringRef)> Factory
Provides compilation arguments used for parsing C and C++ files.
llvm::StringMap< IncludeGraphNode > IncludeGraph
Context clone() const
Clone this context object.
ForwardBinder< Func, Args... > Bind(Func F, Args &&... As)
Creates an object that stores a callable (F) and first arguments to the callable (As) and allows to c...
llvm::unique_function< void()> Action
MemIndex is a naive in-memory index suitable for a small set of symbols.
void log(const char *Fmt, Ts &&... Vals)
std::string Path
A typedef to represent a file path.
std::unique_ptr< CompilerInvocation > buildCompilerInvocation(const ParseInputs &Inputs)
Builds compiler invocation that could be used to build AST or preamble.
void enqueue(const std::vector< std::string > &ChangedFiles)
const IncludeGraph * Sources
void wait(std::unique_lock< std::mutex > &Lock, std::condition_variable &CV, Deadline D)
Wait once on CV for the specified duration.
llvm::Optional< FileDigest > digestFile(const SourceManager &SM, FileID FID)
A context is an immutable container for per-request data that must be propagated through layers that ...
virtual std::unique_ptr< IndexFileIn > loadShard(llvm::StringRef ShardIdentifier) const =0
std::function< bool(const SourceManager &, FileID)> FileFilter
If this is set, only collect symbols/references from a file if FileFilter(SM, FID) is true...
WithContext replaces Context::current() with a provided scope.
FileDigest digest(llvm::StringRef Content)
virtual llvm::Error storeShard(llvm::StringRef ShardIdentifier, IndexFileOut Shard) const =0
llvm::Optional< RefSlab > Refs
static llvm::Expected< URI > create(llvm::StringRef AbsolutePath, llvm::StringRef Scheme)
Creates a URI for a file in the given scheme.
void setCurrentThreadPriority(ThreadPriority Priority)
===– Representation.cpp - ClangDoc Representation --------—*- C++ -*-===//
Deadline timeoutSeconds(llvm::Optional< double > Seconds)
Makes a deadline from a timeout in seconds. None means wait forever.
LLVM_NODISCARD bool blockUntilIdleForTest(llvm::Optional< double > TimeoutSeconds=10)
llvm::Optional< std::string > getCanonicalPath(const FileEntry *F, const SourceManager &SourceMgr)
Get the canonical path of F.
std::unique_ptr< FrontendAction > createStaticIndexingAction(SymbolCollector::Options Opts, std::function< void(SymbolSlab)> SymbolsCallback, std::function< void(RefSlab)> RefsCallback, std::function< void(IncludeGraph)> IncludeGraphCallback)
void insert(const SymbolID &ID, const Ref &S)
static llvm::Expected< std::string > resolve(const URI &U, llvm::StringRef HintPath="")
Resolves the absolute path of U.
BackgroundIndex(Context BackgroundContext, const FileSystemProvider &, const GlobalCompilationDatabase &CDB, BackgroundIndexStorage::Factory IndexStorageFactory, size_t BuildIndexPeriodMs=0, size_t ThreadPoolSize=llvm::heavyweight_hardware_concurrency())
If BuildIndexPeriodMs is greater than 0, the symbol index will only be rebuilt periodically (one per ...
static llvm::Expected< URI > parse(llvm::StringRef Uri)
Parse a URI string "<scheme>:[//<authority>/]<path>".
std::unique_ptr< CompilerInstance > prepareCompilerInstance(std::unique_ptr< clang::CompilerInvocation > CI, const PrecompiledPreamble *Preamble, std::unique_ptr< llvm::MemoryBuffer > Buffer, std::shared_ptr< PCHContainerOperations > PCHs, llvm::IntrusiveRefCntPtr< llvm::vfs::FileSystem > VFS, DiagnosticConsumer &DiagsClient)
Records an event whose duration is the lifetime of the Span object.
#define SPAN_ATTACH(S, Name, Expr)
Attach a key-value pair to a Span event.
void reset(std::unique_ptr< SymbolIndex >)