clang-tools  8.0.0
Background.h
Go to the documentation of this file.
1 //===--- Background.h - Build an index in a background thread ----*- C++-*-===//
2 //
3 // The LLVM Compiler Infrastructure
4 //
5 // This file is distributed under the University of Illinois Open Source
6 // License. See LICENSE.TXT for details.
7 //
8 //===----------------------------------------------------------------------===//
9 
10 #ifndef LLVM_CLANG_TOOLS_EXTRA_CLANGD_INDEX_BACKGROUND_H
11 #define LLVM_CLANG_TOOLS_EXTRA_CLANGD_INDEX_BACKGROUND_H
12 
13 #include "Context.h"
14 #include "FSProvider.h"
16 #include "Threading.h"
17 #include "index/FileIndex.h"
18 #include "index/Index.h"
19 #include "index/Serialization.h"
20 #include "clang/Tooling/CompilationDatabase.h"
21 #include "llvm/ADT/StringMap.h"
22 #include "llvm/Support/SHA1.h"
23 #include "llvm/Support/Threading.h"
24 #include <atomic>
25 #include <condition_variable>
26 #include <deque>
27 #include <mutex>
28 #include <string>
29 #include <thread>
30 #include <vector>
31 
32 namespace clang {
33 namespace clangd {
34 
35 // Handles storage and retrieval of index shards. Both store and load
36 // operations can be called from multiple-threads concurrently.
38 public:
39  virtual ~BackgroundIndexStorage() = default;
40 
41  // Shards of the index are stored and retrieved independently, keyed by shard
42  // identifier - in practice this is a source file name
43  virtual llvm::Error storeShard(llvm::StringRef ShardIdentifier,
44  IndexFileOut Shard) const = 0;
45 
46  // Tries to load shard with given identifier, returns nullptr if shard
47  // couldn't be loaded.
48  virtual std::unique_ptr<IndexFileIn>
49  loadShard(llvm::StringRef ShardIdentifier) const = 0;
50 
51  // The factory provides storage for each CDB.
52  // It keeps ownership of the storage instances, and should manage caching
53  // itself. Factory must be threadsafe and never returns nullptr.
54  using Factory =
55  llvm::unique_function<BackgroundIndexStorage *(llvm::StringRef)>;
56 
57  // Creates an Index Storage that saves shards into disk. Index storage uses
58  // CDBDirectory + ".clangd/index/" as the folder to save shards.
60 };
61 
62 // Builds an in-memory index by by running the static indexer action over
63 // all commands in a compilation database. Indexing happens in the background.
64 // FIXME: it should also persist its state on disk for fast start.
65 // FIXME: it should watch for changes to files on disk.
66 class BackgroundIndex : public SwapIndex {
67 public:
68  /// If BuildIndexPeriodMs is greater than 0, the symbol index will only be
69  /// rebuilt periodically (one per \p BuildIndexPeriodMs); otherwise, index is
70  /// rebuilt for each indexed file.
72  Context BackgroundContext, const FileSystemProvider &,
73  const GlobalCompilationDatabase &CDB,
74  BackgroundIndexStorage::Factory IndexStorageFactory,
75  size_t BuildIndexPeriodMs = 0,
76  size_t ThreadPoolSize = llvm::heavyweight_hardware_concurrency());
77  ~BackgroundIndex(); // Blocks while the current task finishes.
78 
79  // Enqueue translation units for indexing.
80  // The indexing happens in a background thread, so the symbols will be
81  // available sometime later.
82  void enqueue(const std::vector<std::string> &ChangedFiles);
83 
84  // Cause background threads to stop after ther current task, any remaining
85  // tasks will be discarded.
86  void stop();
87 
88  // Wait until the queue is empty, to allow deterministic testing.
89  LLVM_NODISCARD bool
90  blockUntilIdleForTest(llvm::Optional<double> TimeoutSeconds = 10);
91 
92 private:
93  /// Given index results from a TU, only update symbols coming from files with
94  /// different digests than \p DigestsSnapshot. Also stores new index
95  /// information on IndexStorage.
96  void update(llvm::StringRef MainFile, IndexFileIn Index,
97  const llvm::StringMap<FileDigest> &DigestsSnapshot,
98  BackgroundIndexStorage *IndexStorage);
99 
100  // configuration
101  const FileSystemProvider &FSProvider;
102  const GlobalCompilationDatabase &CDB;
103  Context BackgroundContext;
104 
105  // index state
106  llvm::Error index(tooling::CompileCommand,
107  BackgroundIndexStorage *IndexStorage);
108  void buildIndex(); // Rebuild index periodically every BuildIndexPeriodMs.
109  const size_t BuildIndexPeriodMs;
110  std::atomic<bool> SymbolsUpdatedSinceLastIndex;
111  std::mutex IndexMu;
112  std::condition_variable IndexCV;
113 
114  FileSymbols IndexedSymbols;
115  llvm::StringMap<FileDigest> IndexedFileDigests; // Key is absolute file path.
116  std::mutex DigestsMu;
117 
118  BackgroundIndexStorage::Factory IndexStorageFactory;
119  struct Source {
120  std::string Path;
121  bool NeedsReIndexing;
122  Source(llvm::StringRef Path, bool NeedsReIndexing)
123  : Path(Path), NeedsReIndexing(NeedsReIndexing) {}
124  };
125  // Loads the shards for a single TU and all of its dependencies. Returns the
126  // list of sources and whether they need to be re-indexed.
127  std::vector<Source> loadShard(const tooling::CompileCommand &Cmd,
128  BackgroundIndexStorage *IndexStorage,
129  llvm::StringSet<> &LoadedShards);
130  // Tries to load shards for the ChangedFiles.
131  std::vector<std::pair<tooling::CompileCommand, BackgroundIndexStorage *>>
132  loadShards(std::vector<std::string> ChangedFiles);
133  void enqueue(tooling::CompileCommand Cmd, BackgroundIndexStorage *Storage);
134 
135  // queue management
136  using Task = std::function<void()>;
137  void run(); // Main loop executed by Thread. Runs tasks from Queue.
138  void enqueueTask(Task T, ThreadPriority Prioirty);
139  void enqueueLocked(tooling::CompileCommand Cmd,
140  BackgroundIndexStorage *IndexStorage);
141  std::mutex QueueMu;
142  unsigned NumActiveTasks = 0; // Only idle when queue is empty *and* no tasks.
143  std::condition_variable QueueCV;
144  bool ShouldStop = false;
145  std::deque<std::pair<Task, ThreadPriority>> Queue;
146  std::vector<std::thread> ThreadPool; // FIXME: Abstract this away.
147  GlobalCompilationDatabase::CommandChanged::Subscription CommandsChanged;
148 };
149 
150 } // namespace clangd
151 } // namespace clang
152 
153 #endif
A container of Symbols from several source files.
Definition: FileIndex.h:58
llvm::unique_function< BackgroundIndexStorage *(llvm::StringRef)> Factory
Definition: Background.h:55
Provides compilation arguments used for parsing C and C++ files.
std::string Path
A typedef to represent a file path.
Definition: Path.h:21
A context is an immutable container for per-request data that must be propagated through layers that ...
Definition: Context.h:70
virtual std::unique_ptr< IndexFileIn > loadShard(llvm::StringRef ShardIdentifier) const =0
virtual llvm::Error storeShard(llvm::StringRef ShardIdentifier, IndexFileOut Shard) const =0
===– Representation.cpp - ClangDoc Representation --------—*- C++ -*-===//
const SymbolIndex * Index
Definition: Dexp.cpp:85