clang-tools  8.0.0
Background.cpp
Go to the documentation of this file.
1 //===-- Background.cpp - Build an index in a background thread ------------===//
2 //
3 // The LLVM Compiler Infrastructure
4 //
5 // This file is distributed under the University of Illinois Open Source
6 // License. See LICENSE.TXT for details.
7 //
8 //===----------------------------------------------------------------------===//
9 
10 #include "index/Background.h"
11 #include "ClangdUnit.h"
12 #include "Compiler.h"
13 #include "Logger.h"
14 #include "SourceCode.h"
15 #include "Threading.h"
16 #include "Trace.h"
17 #include "URI.h"
18 #include "index/IndexAction.h"
19 #include "index/MemIndex.h"
20 #include "index/Serialization.h"
21 #include "index/SymbolCollector.h"
22 #include "clang/Basic/SourceLocation.h"
23 #include "clang/Basic/SourceManager.h"
24 #include "llvm/ADT/STLExtras.h"
25 #include "llvm/ADT/ScopeExit.h"
26 #include "llvm/ADT/StringMap.h"
27 #include "llvm/ADT/StringRef.h"
28 #include "llvm/Support/SHA1.h"
29 
30 #include <chrono>
31 #include <memory>
32 #include <numeric>
33 #include <queue>
34 #include <random>
35 #include <string>
36 #include <thread>
37 
38 namespace clang {
39 namespace clangd {
40 namespace {
41 // Resolves URI to file paths with cache.
42 class URIToFileCache {
43 public:
44  URIToFileCache(llvm::StringRef HintPath) : HintPath(HintPath) {}
45 
46  llvm::StringRef resolve(llvm::StringRef FileURI) {
47  auto I = URIToPathCache.try_emplace(FileURI);
48  if (I.second) {
49  auto U = URI::parse(FileURI);
50  if (!U) {
51  elog("Failed to parse URI {0}: {1}", FileURI, U.takeError());
52  assert(false && "Failed to parse URI");
53  return "";
54  }
55  auto Path = URI::resolve(*U, HintPath);
56  if (!Path) {
57  elog("Failed to resolve URI {0}: {1}", FileURI, Path.takeError());
58  assert(false && "Failed to resolve URI");
59  return "";
60  }
61  I.first->second = *Path;
62  }
63  return I.first->second;
64  }
65 
66 private:
67  std::string HintPath;
68  llvm::StringMap<std::string> URIToPathCache;
69 };
70 
71 // We keep only the node "U" and its edges. Any node other than "U" will be
72 // empty in the resultant graph.
73 IncludeGraph getSubGraph(const URI &U, const IncludeGraph &FullGraph) {
74  IncludeGraph IG;
75 
76  std::string FileURI = U.toString();
77  auto Entry = IG.try_emplace(FileURI).first;
78  auto &Node = Entry->getValue();
79  Node = FullGraph.lookup(Entry->getKey());
80  Node.URI = Entry->getKey();
81 
82  // URIs inside nodes must point into the keys of the same IncludeGraph.
83  for (auto &Include : Node.DirectIncludes) {
84  auto I = IG.try_emplace(Include).first;
85  I->getValue().URI = I->getKey();
86  Include = I->getKey();
87  }
88 
89  return IG;
90 }
91 
92 // Creates a filter to not collect index results from files with unchanged
93 // digests.
94 // \p FileDigests contains file digests for the current indexed files.
96 createFileFilter(const llvm::StringMap<FileDigest> &FileDigests) {
97  return [&FileDigests](const SourceManager &SM, FileID FID) {
98  const auto *F = SM.getFileEntryForID(FID);
99  if (!F)
100  return false; // Skip invalid files.
101  auto AbsPath = getCanonicalPath(F, SM);
102  if (!AbsPath)
103  return false; // Skip files without absolute path.
104  auto Digest = digestFile(SM, FID);
105  if (!Digest)
106  return false;
107  auto D = FileDigests.find(*AbsPath);
108  if (D != FileDigests.end() && D->second == Digest)
109  return false; // Skip files that haven't changed.
110  return true;
111  };
112 }
113 
114 // We cannot use vfs->makeAbsolute because Cmd.FileName is either absolute or
115 // relative to Cmd.Directory, which might not be the same as current working
116 // directory.
117 llvm::SmallString<128> getAbsolutePath(const tooling::CompileCommand &Cmd) {
118  llvm::SmallString<128> AbsolutePath;
119  if (llvm::sys::path::is_absolute(Cmd.Filename)) {
120  AbsolutePath = Cmd.Filename;
121  } else {
122  AbsolutePath = Cmd.Directory;
123  llvm::sys::path::append(AbsolutePath, Cmd.Filename);
124  }
125  return AbsolutePath;
126 }
127 } // namespace
128 
130  Context BackgroundContext, const FileSystemProvider &FSProvider,
131  const GlobalCompilationDatabase &CDB,
132  BackgroundIndexStorage::Factory IndexStorageFactory,
133  size_t BuildIndexPeriodMs, size_t ThreadPoolSize)
134  : SwapIndex(llvm::make_unique<MemIndex>()), FSProvider(FSProvider),
135  CDB(CDB), BackgroundContext(std::move(BackgroundContext)),
136  BuildIndexPeriodMs(BuildIndexPeriodMs),
137  SymbolsUpdatedSinceLastIndex(false),
138  IndexStorageFactory(std::move(IndexStorageFactory)),
139  CommandsChanged(
140  CDB.watch([&](const std::vector<std::string> &ChangedFiles) {
141  enqueue(ChangedFiles);
142  })) {
143  assert(ThreadPoolSize > 0 && "Thread pool size can't be zero.");
144  assert(this->IndexStorageFactory && "Storage factory can not be null!");
145  while (ThreadPoolSize--)
146  ThreadPool.emplace_back([this] { run(); });
147  if (BuildIndexPeriodMs > 0) {
148  log("BackgroundIndex: build symbol index periodically every {0} ms.",
149  BuildIndexPeriodMs);
150  ThreadPool.emplace_back([this] { buildIndex(); });
151  }
152 }
153 
155  stop();
156  for (auto &Thread : ThreadPool)
157  Thread.join();
158 }
159 
161  {
162  std::lock_guard<std::mutex> QueueLock(QueueMu);
163  std::lock_guard<std::mutex> IndexLock(IndexMu);
164  ShouldStop = true;
165  }
166  QueueCV.notify_all();
167  IndexCV.notify_all();
168 }
169 
170 void BackgroundIndex::run() {
171  WithContext Background(BackgroundContext.clone());
172  while (true) {
173  llvm::Optional<Task> Task;
174  ThreadPriority Priority;
175  {
176  std::unique_lock<std::mutex> Lock(QueueMu);
177  QueueCV.wait(Lock, [&] { return ShouldStop || !Queue.empty(); });
178  if (ShouldStop) {
179  Queue.clear();
180  QueueCV.notify_all();
181  return;
182  }
183  ++NumActiveTasks;
184  std::tie(Task, Priority) = std::move(Queue.front());
185  Queue.pop_front();
186  }
187 
188  if (Priority != ThreadPriority::Normal)
189  setCurrentThreadPriority(Priority);
190  (*Task)();
191  if (Priority != ThreadPriority::Normal)
193 
194  {
195  std::unique_lock<std::mutex> Lock(QueueMu);
196  assert(NumActiveTasks > 0 && "before decrementing");
197  --NumActiveTasks;
198  }
199  QueueCV.notify_all();
200  }
201 }
202 
204  llvm::Optional<double> TimeoutSeconds) {
205  std::unique_lock<std::mutex> Lock(QueueMu);
206  return wait(Lock, QueueCV, timeoutSeconds(TimeoutSeconds),
207  [&] { return Queue.empty() && NumActiveTasks == 0; });
208 }
209 
210 void BackgroundIndex::enqueue(const std::vector<std::string> &ChangedFiles) {
211  enqueueTask(
212  [this, ChangedFiles] {
213  trace::Span Tracer("BackgroundIndexEnqueue");
214  // We're doing this asynchronously, because we'll read shards here too.
215  log("Enqueueing {0} commands for indexing", ChangedFiles.size());
216  SPAN_ATTACH(Tracer, "files", int64_t(ChangedFiles.size()));
217 
218  auto NeedsReIndexing = loadShards(std::move(ChangedFiles));
219  // Run indexing for files that need to be updated.
220  std::shuffle(NeedsReIndexing.begin(), NeedsReIndexing.end(),
221  std::mt19937(std::random_device{}()));
222  for (auto &Elem : NeedsReIndexing)
223  enqueue(std::move(Elem.first), Elem.second);
224  },
226 }
227 
228 void BackgroundIndex::enqueue(tooling::CompileCommand Cmd,
229  BackgroundIndexStorage *Storage) {
230  enqueueTask(Bind(
231  [this, Storage](tooling::CompileCommand Cmd) {
232  // We can't use llvm::StringRef here since we are going to
233  // move from Cmd during the call below.
234  const std::string FileName = Cmd.Filename;
235  if (auto Error = index(std::move(Cmd), Storage))
236  elog("Indexing {0} failed: {1}", FileName,
237  std::move(Error));
238  },
239  std::move(Cmd)),
241 }
242 
243 void BackgroundIndex::enqueueTask(Task T, ThreadPriority Priority) {
244  {
245  std::lock_guard<std::mutex> Lock(QueueMu);
246  auto I = Queue.end();
247  // We first store the tasks with Normal priority in the front of the queue.
248  // Then we store low priority tasks. Normal priority tasks are pretty rare,
249  // they should not grow beyond single-digit numbers, so it is OK to do
250  // linear search and insert after that.
251  if (Priority == ThreadPriority::Normal) {
252  I = llvm::find_if(Queue, [](const std::pair<Task, ThreadPriority> &Elem) {
253  return Elem.second == ThreadPriority::Low;
254  });
255  }
256  Queue.insert(I, {std::move(T), Priority});
257  }
258  QueueCV.notify_all();
259 }
260 
261 /// Given index results from a TU, only update symbols coming from files that
262 /// are different or missing from than \p DigestsSnapshot. Also stores new index
263 /// information on IndexStorage.
264 void BackgroundIndex::update(llvm::StringRef MainFile, IndexFileIn Index,
265  const llvm::StringMap<FileDigest> &DigestsSnapshot,
266  BackgroundIndexStorage *IndexStorage) {
267  // Partition symbols/references into files.
268  struct File {
269  llvm::DenseSet<const Symbol *> Symbols;
270  llvm::DenseSet<const Ref *> Refs;
271  FileDigest Digest;
272  };
273  llvm::StringMap<File> Files;
274  URIToFileCache URICache(MainFile);
275  for (const auto &IndexIt : *Index.Sources) {
276  const auto &IGN = IndexIt.getValue();
277  const auto AbsPath = URICache.resolve(IGN.URI);
278  const auto DigestIt = DigestsSnapshot.find(AbsPath);
279  // File has different contents.
280  if (DigestIt == DigestsSnapshot.end() || DigestIt->getValue() != IGN.Digest)
281  Files.try_emplace(AbsPath).first->getValue().Digest = IGN.Digest;
282  }
283  for (const auto &Sym : *Index.Symbols) {
284  if (Sym.CanonicalDeclaration) {
285  auto DeclPath = URICache.resolve(Sym.CanonicalDeclaration.FileURI);
286  const auto FileIt = Files.find(DeclPath);
287  if (FileIt != Files.end())
288  FileIt->second.Symbols.insert(&Sym);
289  }
290  // For symbols with different declaration and definition locations, we store
291  // the full symbol in both the header file and the implementation file, so
292  // that merging can tell the preferred symbols (from canonical headers) from
293  // other symbols (e.g. forward declarations).
294  if (Sym.Definition &&
295  Sym.Definition.FileURI != Sym.CanonicalDeclaration.FileURI) {
296  auto DefPath = URICache.resolve(Sym.Definition.FileURI);
297  const auto FileIt = Files.find(DefPath);
298  if (FileIt != Files.end())
299  FileIt->second.Symbols.insert(&Sym);
300  }
301  }
302  llvm::DenseMap<const Ref *, SymbolID> RefToIDs;
303  for (const auto &SymRefs : *Index.Refs) {
304  for (const auto &R : SymRefs.second) {
305  auto Path = URICache.resolve(R.Location.FileURI);
306  const auto FileIt = Files.find(Path);
307  if (FileIt != Files.end()) {
308  auto &F = FileIt->getValue();
309  RefToIDs[&R] = SymRefs.first;
310  F.Refs.insert(&R);
311  }
312  }
313  }
314 
315  // Build and store new slabs for each updated file.
316  for (const auto &FileIt : Files) {
317  llvm::StringRef Path = FileIt.getKey();
318  SymbolSlab::Builder Syms;
319  RefSlab::Builder Refs;
320  for (const auto *S : FileIt.second.Symbols)
321  Syms.insert(*S);
322  for (const auto *R : FileIt.second.Refs)
323  Refs.insert(RefToIDs[R], *R);
324  auto SS = llvm::make_unique<SymbolSlab>(std::move(Syms).build());
325  auto RS = llvm::make_unique<RefSlab>(std::move(Refs).build());
326  auto IG = llvm::make_unique<IncludeGraph>(
327  getSubGraph(URI::create(Path), Index.Sources.getValue()));
328  // We need to store shards before updating the index, since the latter
329  // consumes slabs.
330  if (IndexStorage) {
331  IndexFileOut Shard;
332  Shard.Symbols = SS.get();
333  Shard.Refs = RS.get();
334  Shard.Sources = IG.get();
335 
336  if (auto Error = IndexStorage->storeShard(Path, Shard))
337  elog("Failed to write background-index shard for file {0}: {1}", Path,
338  std::move(Error));
339  }
340  {
341  std::lock_guard<std::mutex> Lock(DigestsMu);
342  auto Hash = FileIt.second.Digest;
343  // Skip if file is already up to date.
344  auto DigestIt = IndexedFileDigests.try_emplace(Path);
345  if (!DigestIt.second && DigestIt.first->second == Hash)
346  continue;
347  DigestIt.first->second = Hash;
348  // This can override a newer version that is added in another thread, if
349  // this thread sees the older version but finishes later. This should be
350  // rare in practice.
351  IndexedSymbols.update(Path, std::move(SS), std::move(RS));
352  }
353  }
354 }
355 
356 void BackgroundIndex::buildIndex() {
357  assert(BuildIndexPeriodMs > 0);
358  while (true) {
359  {
360  std::unique_lock<std::mutex> Lock(IndexMu);
361  if (ShouldStop) // Avoid waiting if stopped.
362  break;
363  // Wait until this is notified to stop or `BuildIndexPeriodMs` has past.
364  IndexCV.wait_for(Lock, std::chrono::milliseconds(BuildIndexPeriodMs));
365  if (ShouldStop) // Avoid rebuilding index if stopped.
366  break;
367  }
368  if (!SymbolsUpdatedSinceLastIndex.exchange(false))
369  continue;
370  // There can be symbol update right after the flag is reset above and before
371  // index is rebuilt below. The new index would contain the updated symbols
372  // but the flag would still be true. This is fine as we would simply run an
373  // extra index build.
374  reset(
375  IndexedSymbols.buildIndex(IndexType::Heavy, DuplicateHandling::Merge));
376  log("BackgroundIndex: rebuilt symbol index.");
377  }
378 }
379 
380 llvm::Error BackgroundIndex::index(tooling::CompileCommand Cmd,
381  BackgroundIndexStorage *IndexStorage) {
382  trace::Span Tracer("BackgroundIndex");
383  SPAN_ATTACH(Tracer, "file", Cmd.Filename);
384  auto AbsolutePath = getAbsolutePath(Cmd);
385 
386  auto FS = FSProvider.getFileSystem();
387  auto Buf = FS->getBufferForFile(AbsolutePath);
388  if (!Buf)
389  return llvm::errorCodeToError(Buf.getError());
390  auto Hash = digest(Buf->get()->getBuffer());
391 
392  // Take a snapshot of the digests to avoid locking for each file in the TU.
393  llvm::StringMap<FileDigest> DigestsSnapshot;
394  {
395  std::lock_guard<std::mutex> Lock(DigestsMu);
396  DigestsSnapshot = IndexedFileDigests;
397  }
398 
399  log("Indexing {0} (digest:={1})", Cmd.Filename, llvm::toHex(Hash));
400  ParseInputs Inputs;
401  Inputs.FS = std::move(FS);
402  Inputs.FS->setCurrentWorkingDirectory(Cmd.Directory);
403  Inputs.CompileCommand = std::move(Cmd);
404  auto CI = buildCompilerInvocation(Inputs);
405  if (!CI)
406  return llvm::createStringError(llvm::inconvertibleErrorCode(),
407  "Couldn't build compiler invocation");
408  IgnoreDiagnostics IgnoreDiags;
409  auto Clang = prepareCompilerInstance(
410  std::move(CI), /*Preamble=*/nullptr, std::move(*Buf),
411  std::make_shared<PCHContainerOperations>(), Inputs.FS, IgnoreDiags);
412  if (!Clang)
413  return llvm::createStringError(llvm::inconvertibleErrorCode(),
414  "Couldn't build compiler instance");
415 
416  SymbolCollector::Options IndexOpts;
417  IndexOpts.FileFilter = createFileFilter(DigestsSnapshot);
418  IndexFileIn Index;
420  IndexOpts, [&](SymbolSlab S) { Index.Symbols = std::move(S); },
421  [&](RefSlab R) { Index.Refs = std::move(R); },
422  [&](IncludeGraph IG) { Index.Sources = std::move(IG); });
423 
424  // We're going to run clang here, and it could potentially crash.
425  // We could use CrashRecoveryContext to try to make indexing crashes nonfatal,
426  // but the leaky "recovery" is pretty scary too in a long-running process.
427  // If crashes are a real problem, maybe we should fork a child process.
428 
429  const FrontendInputFile &Input = Clang->getFrontendOpts().Inputs.front();
430  if (!Action->BeginSourceFile(*Clang, Input))
431  return llvm::createStringError(llvm::inconvertibleErrorCode(),
432  "BeginSourceFile() failed");
433  if (!Action->Execute())
434  return llvm::createStringError(llvm::inconvertibleErrorCode(),
435  "Execute() failed");
436  Action->EndSourceFile();
437  if (Clang->hasDiagnostics() &&
438  Clang->getDiagnostics().hasUncompilableErrorOccurred()) {
439  return llvm::createStringError(
440  llvm::inconvertibleErrorCode(),
441  "IndexingAction failed: has uncompilable errors");
442  }
443 
444  assert(Index.Symbols && Index.Refs && Index.Sources &&
445  "Symbols, Refs and Sources must be set.");
446 
447  log("Indexed {0} ({1} symbols, {2} refs, {3} files)",
448  Inputs.CompileCommand.Filename, Index.Symbols->size(),
449  Index.Refs->numRefs(), Index.Sources->size());
450  SPAN_ATTACH(Tracer, "symbols", int(Index.Symbols->size()));
451  SPAN_ATTACH(Tracer, "refs", int(Index.Refs->numRefs()));
452  SPAN_ATTACH(Tracer, "sources", int(Index.Sources->size()));
453 
454  update(AbsolutePath, std::move(Index), DigestsSnapshot, IndexStorage);
455 
456  if (BuildIndexPeriodMs > 0)
457  SymbolsUpdatedSinceLastIndex = true;
458  else
459  reset(
460  IndexedSymbols.buildIndex(IndexType::Light, DuplicateHandling::Merge));
461 
462  return llvm::Error::success();
463 }
464 
465 std::vector<BackgroundIndex::Source>
466 BackgroundIndex::loadShard(const tooling::CompileCommand &Cmd,
467  BackgroundIndexStorage *IndexStorage,
468  llvm::StringSet<> &LoadedShards) {
469  struct ShardInfo {
470  std::string AbsolutePath;
471  std::unique_ptr<IndexFileIn> Shard;
472  FileDigest Digest;
473  };
474  std::vector<ShardInfo> IntermediateSymbols;
475  // Make sure we don't have duplicate elements in the queue. Keys are absolute
476  // paths.
477  llvm::StringSet<> InQueue;
478  auto FS = FSProvider.getFileSystem();
479  // Dependencies of this TU, paired with the information about whether they
480  // need to be re-indexed or not.
481  std::vector<Source> Dependencies;
482  std::queue<Source> ToVisit;
483  std::string AbsolutePath = getAbsolutePath(Cmd).str();
484  // Up until we load the shard related to a dependency it needs to be
485  // re-indexed.
486  ToVisit.emplace(AbsolutePath, true);
487  InQueue.insert(AbsolutePath);
488  // Goes over each dependency.
489  while (!ToVisit.empty()) {
490  Dependencies.push_back(std::move(ToVisit.front()));
491  // Dependencies is not modified during the rest of the loop, so it is safe
492  // to keep the reference.
493  auto &CurDependency = Dependencies.back();
494  ToVisit.pop();
495  // If we have already seen this shard before(either loaded or failed) don't
496  // re-try again. Since the information in the shard won't change from one TU
497  // to another.
498  if (!LoadedShards.try_emplace(CurDependency.Path).second) {
499  // If the dependency needs to be re-indexed, first occurence would already
500  // have detected that, so we don't need to issue it again.
501  CurDependency.NeedsReIndexing = false;
502  continue;
503  }
504 
505  auto Shard = IndexStorage->loadShard(CurDependency.Path);
506  if (!Shard || !Shard->Sources) {
507  // File will be returned as requiring re-indexing to caller.
508  vlog("Failed to load shard: {0}", CurDependency.Path);
509  continue;
510  }
511  // These are the edges in the include graph for current dependency.
512  for (const auto &I : *Shard->Sources) {
513  auto U = URI::parse(I.getKey());
514  if (!U)
515  continue;
516  auto AbsolutePath = URI::resolve(*U, CurDependency.Path);
517  if (!AbsolutePath)
518  continue;
519  // Add file as dependency if haven't seen before.
520  if (InQueue.try_emplace(*AbsolutePath).second)
521  ToVisit.emplace(*AbsolutePath, true);
522  // The node contains symbol information only for current file, the rest is
523  // just edges.
524  if (*AbsolutePath != CurDependency.Path)
525  continue;
526 
527  // We found source file info for current dependency.
528  assert(I.getValue().Digest != FileDigest{{0}} && "Digest is empty?");
529  ShardInfo SI;
530  SI.AbsolutePath = CurDependency.Path;
531  SI.Shard = std::move(Shard);
532  SI.Digest = I.getValue().Digest;
533  IntermediateSymbols.push_back(std::move(SI));
534  // Check if the source needs re-indexing.
535  // Get the digest, skip it if file doesn't exist.
536  auto Buf = FS->getBufferForFile(CurDependency.Path);
537  if (!Buf) {
538  elog("Couldn't get buffer for file: {0}: {1}", CurDependency.Path,
539  Buf.getError().message());
540  continue;
541  }
542  // If digests match then dependency doesn't need re-indexing.
543  CurDependency.NeedsReIndexing =
544  digest(Buf->get()->getBuffer()) != I.getValue().Digest;
545  }
546  }
547  // Load shard information into background-index.
548  {
549  std::lock_guard<std::mutex> Lock(DigestsMu);
550  // This can override a newer version that is added in another thread,
551  // if this thread sees the older version but finishes later. This
552  // should be rare in practice.
553  for (const ShardInfo &SI : IntermediateSymbols) {
554  auto SS =
555  SI.Shard->Symbols
556  ? llvm::make_unique<SymbolSlab>(std::move(*SI.Shard->Symbols))
557  : nullptr;
558  auto RS = SI.Shard->Refs
559  ? llvm::make_unique<RefSlab>(std::move(*SI.Shard->Refs))
560  : nullptr;
561  IndexedFileDigests[SI.AbsolutePath] = SI.Digest;
562  IndexedSymbols.update(SI.AbsolutePath, std::move(SS), std::move(RS));
563  }
564  }
565 
566  return Dependencies;
567 }
568 
569 // Goes over each changed file and loads them from index. Returns the list of
570 // TUs that had out-of-date/no shards.
571 std::vector<std::pair<tooling::CompileCommand, BackgroundIndexStorage *>>
572 BackgroundIndex::loadShards(std::vector<std::string> ChangedFiles) {
573  std::vector<std::pair<tooling::CompileCommand, BackgroundIndexStorage *>>
574  NeedsReIndexing;
575  // Keeps track of the files that will be reindexed, to make sure we won't
576  // re-index same dependencies more than once. Keys are AbsolutePaths.
577  llvm::StringSet<> FilesToIndex;
578  // Keeps track of the loaded shards to make sure we don't perform redundant
579  // disk IO. Keys are absolute paths.
580  llvm::StringSet<> LoadedShards;
581  for (const auto &File : ChangedFiles) {
582  ProjectInfo PI;
583  auto Cmd = CDB.getCompileCommand(File, &PI);
584  if (!Cmd)
585  continue;
586  BackgroundIndexStorage *IndexStorage = IndexStorageFactory(PI.SourceRoot);
587  auto Dependencies = loadShard(*Cmd, IndexStorage, LoadedShards);
588  for (const auto &Dependency : Dependencies) {
589  if (!Dependency.NeedsReIndexing || FilesToIndex.count(Dependency.Path))
590  continue;
591  // FIXME: Currently, we simply schedule indexing on a TU whenever any of
592  // its dependencies needs re-indexing. We might do it smarter by figuring
593  // out a minimal set of TUs that will cover all the stale dependencies.
594  vlog("Enqueueing TU {0} because its dependency {1} needs re-indexing.",
595  Cmd->Filename, Dependency.Path);
596  NeedsReIndexing.push_back({std::move(*Cmd), IndexStorage});
597  // Mark all of this TU's dependencies as to-be-indexed so that we won't
598  // try to re-index those.
599  for (const auto &Dependency : Dependencies)
600  FilesToIndex.insert(Dependency.Path);
601  break;
602  }
603  }
604  vlog("Loaded all shards");
605  reset(IndexedSymbols.buildIndex(IndexType::Light, DuplicateHandling::Merge));
606 
607  return NeedsReIndexing;
608 }
609 
610 } // namespace clangd
611 } // namespace clang
llvm::Optional< SymbolSlab > Symbols
Definition: Serialization.h:41
Some operations such as code completion produce a set of candidates.
decltype(llvm::SHA1::hash({})) FileDigest
Definition: SourceCode.h:31
void insert(const Symbol &S)
Definition: Index.cpp:91
void vlog(const char *Fmt, Ts &&... Vals)
Definition: Logger.h:68
void elog(const char *Fmt, Ts &&... Vals)
Definition: Logger.h:57
llvm::Optional< IncludeGraph > Sources
Definition: Serialization.h:44
const SymbolSlab * Symbols
Definition: Serialization.h:51
llvm::unique_function< BackgroundIndexStorage *(llvm::StringRef)> Factory
Definition: Background.h:55
Provides compilation arguments used for parsing C and C++ files.
llvm::StringMap< IncludeGraphNode > IncludeGraph
Definition: Headers.h:65
Context clone() const
Clone this context object.
Definition: Context.cpp:21
ForwardBinder< Func, Args... > Bind(Func F, Args &&... As)
Creates an object that stores a callable (F) and first arguments to the callable (As) and allows to c...
Definition: Function.h:82
llvm::unique_function< void()> Action
MemIndex is a naive in-memory index suitable for a small set of symbols.
Definition: MemIndex.h:20
void log(const char *Fmt, Ts &&... Vals)
Definition: Logger.h:63
std::string Path
A typedef to represent a file path.
Definition: Path.h:21
std::unique_ptr< CompilerInvocation > buildCompilerInvocation(const ParseInputs &Inputs)
Builds compiler invocation that could be used to build AST or preamble.
Definition: ClangdUnit.cpp:426
tooling::CompileCommand CompileCommand
Definition: ClangdUnit.h:65
void enqueue(const std::vector< std::string > &ChangedFiles)
Definition: Background.cpp:210
const Decl * D
Definition: XRefs.cpp:79
PathRef FileName
const IncludeGraph * Sources
Definition: Serialization.h:54
void wait(std::unique_lock< std::mutex > &Lock, std::condition_variable &CV, Deadline D)
Wait once on CV for the specified duration.
Definition: Threading.cpp:107
llvm::Optional< FileDigest > digestFile(const SourceManager &SM, FileID FID)
Definition: SourceCode.cpp:244
A context is an immutable container for per-request data that must be propagated through layers that ...
Definition: Context.h:70
virtual std::unique_ptr< IndexFileIn > loadShard(llvm::StringRef ShardIdentifier) const =0
std::function< bool(const SourceManager &, FileID)> FileFilter
If this is set, only collect symbols/references from a file if FileFilter(SM, FID) is true...
WithContext replaces Context::current() with a provided scope.
Definition: Context.h:190
FileDigest digest(llvm::StringRef Content)
Definition: SourceCode.cpp:240
virtual llvm::Error storeShard(llvm::StringRef ShardIdentifier, IndexFileOut Shard) const =0
llvm::Optional< RefSlab > Refs
Definition: Serialization.h:42
Information required to run clang, e.g. to parse AST or do code completion.
Definition: ClangdUnit.h:64
static llvm::Expected< URI > create(llvm::StringRef AbsolutePath, llvm::StringRef Scheme)
Creates a URI for a file in the given scheme.
Definition: URI.cpp:188
void setCurrentThreadPriority(ThreadPriority Priority)
Definition: Threading.cpp:118
===– Representation.cpp - ClangDoc Representation --------—*- C++ -*-===//
Deadline timeoutSeconds(llvm::Optional< double > Seconds)
Makes a deadline from a timeout in seconds. None means wait forever.
Definition: Threading.cpp:99
LLVM_NODISCARD bool blockUntilIdleForTest(llvm::Optional< double > TimeoutSeconds=10)
Definition: Background.cpp:203
llvm::Optional< std::string > getCanonicalPath(const FileEntry *F, const SourceManager &SourceMgr)
Get the canonical path of F.
Definition: SourceCode.cpp:187
std::unique_ptr< FrontendAction > createStaticIndexingAction(SymbolCollector::Options Opts, std::function< void(SymbolSlab)> SymbolsCallback, std::function< void(RefSlab)> RefsCallback, std::function< void(IncludeGraph)> IncludeGraphCallback)
void insert(const SymbolID &ID, const Ref &S)
Definition: Index.cpp:135
static llvm::Expected< std::string > resolve(const URI &U, llvm::StringRef HintPath="")
Resolves the absolute path of U.
Definition: URI.cpp:224
BackgroundIndex(Context BackgroundContext, const FileSystemProvider &, const GlobalCompilationDatabase &CDB, BackgroundIndexStorage::Factory IndexStorageFactory, size_t BuildIndexPeriodMs=0, size_t ThreadPoolSize=llvm::heavyweight_hardware_concurrency())
If BuildIndexPeriodMs is greater than 0, the symbol index will only be rebuilt periodically (one per ...
Definition: Background.cpp:129
IntrusiveRefCntPtr< llvm::vfs::FileSystem > FS
Definition: ClangdUnit.h:66
static llvm::Expected< URI > parse(llvm::StringRef Uri)
Parse a URI string "<scheme>:[//<authority>/]<path>".
Definition: URI.cpp:166
std::unique_ptr< CompilerInstance > prepareCompilerInstance(std::unique_ptr< clang::CompilerInvocation > CI, const PrecompiledPreamble *Preamble, std::unique_ptr< llvm::MemoryBuffer > Buffer, std::shared_ptr< PCHContainerOperations > PCHs, llvm::IntrusiveRefCntPtr< llvm::vfs::FileSystem > VFS, DiagnosticConsumer &DiagsClient)
Definition: Compiler.cpp:44
Records an event whose duration is the lifetime of the Span object.
Definition: Trace.h:83
#define SPAN_ATTACH(S, Name, Expr)
Attach a key-value pair to a Span event.
Definition: Trace.h:99
void reset(std::unique_ptr< SymbolIndex >)
Definition: Index.cpp:162