clang-tools  8.0.0
Background.h
Go to the documentation of this file.
1 //===--- Background.h - Build an index in a background thread ----*- C++-*-===//
2 //
3 // The LLVM Compiler Infrastructure
4 //
5 // This file is distributed under the University of Illinois Open Source
6 // License. See LICENSE.TXT for details.
7 //
8 //===----------------------------------------------------------------------===//
9 
10 #ifndef LLVM_CLANG_TOOLS_EXTRA_CLANGD_INDEX_BACKGROUND_H
11 #define LLVM_CLANG_TOOLS_EXTRA_CLANGD_INDEX_BACKGROUND_H
12 
13 #include "Context.h"
14 #include "FSProvider.h"
16 #include "Threading.h"
17 #include "index/FileIndex.h"
18 #include "index/Index.h"
19 #include "index/Serialization.h"
20 #include "clang/Tooling/CompilationDatabase.h"
21 #include "llvm/ADT/StringMap.h"
22 #include "llvm/Support/SHA1.h"
23 #include "llvm/Support/Threading.h"
24 #include <atomic>
25 #include <condition_variable>
26 #include <deque>
27 #include <mutex>
28 #include <string>
29 #include <thread>
30 #include <vector>
31 
32 namespace clang {
33 namespace clangd {
34 
35 // Handles storage and retrieval of index shards. Both store and load
36 // operations can be called from multiple-threads concurrently.
38 public:
39  virtual ~BackgroundIndexStorage() = default;
40 
41  // Shards of the index are stored and retrieved independently, keyed by shard
42  // identifier - in practice this is a source file name
43  virtual llvm::Error storeShard(llvm::StringRef ShardIdentifier,
44  IndexFileOut Shard) const = 0;
45 
46  // Tries to load shard with given identifier, returns nullptr if shard
47  // couldn't be loaded.
48  virtual std::unique_ptr<IndexFileIn>
49  loadShard(llvm::StringRef ShardIdentifier) const = 0;
50 
51  // The factory provides storage for each CDB.
52  // It keeps ownership of the storage instances, and should manage caching
53  // itself. Factory must be threadsafe and never returns nullptr.
54  using Factory =
55  llvm::unique_function<BackgroundIndexStorage *(llvm::StringRef)>;
56 
57  // Creates an Index Storage that saves shards into disk. Index storage uses
58  // CDBDirectory + ".clangd-index/" as the folder to save shards.
60 };
61 
62 // Builds an in-memory index by by running the static indexer action over
63 // all commands in a compilation database. Indexing happens in the background.
64 // FIXME: it should also persist its state on disk for fast start.
65 // FIXME: it should watch for changes to files on disk.
66 class BackgroundIndex : public SwapIndex {
67 public:
68  /// If BuildIndexPeriodMs is greater than 0, the symbol index will only be
69  /// rebuilt periodically (one per \p BuildIndexPeriodMs); otherwise, index is
70  /// rebuilt for each indexed file.
71  BackgroundIndex(Context BackgroundContext, const FileSystemProvider &,
72  const GlobalCompilationDatabase &CDB,
73  BackgroundIndexStorage::Factory IndexStorageFactory,
74  size_t BuildIndexPeriodMs = 0,
75  size_t ThreadPoolSize = llvm::hardware_concurrency());
76  ~BackgroundIndex(); // Blocks while the current task finishes.
77 
78  // Enqueue translation units for indexing.
79  // The indexing happens in a background thread, so the symbols will be
80  // available sometime later.
81  void enqueue(const std::vector<std::string> &ChangedFiles);
82 
83  // Cause background threads to stop after ther current task, any remaining
84  // tasks will be discarded.
85  void stop();
86 
87  // Wait until the queue is empty, to allow deterministic testing.
88  LLVM_NODISCARD bool
89  blockUntilIdleForTest(llvm::Optional<double> TimeoutSeconds = 10);
90 
91 private:
92  /// Given index results from a TU, only update symbols coming from files with
93  /// different digests than \p DigestsSnapshot. Also stores new index
94  /// information on IndexStorage.
95  void update(llvm::StringRef MainFile, IndexFileIn Index,
96  const llvm::StringMap<FileDigest> &DigestsSnapshot,
97  BackgroundIndexStorage *IndexStorage);
98 
99  // configuration
100  const FileSystemProvider &FSProvider;
101  const GlobalCompilationDatabase &CDB;
102  Context BackgroundContext;
103 
104  // index state
105  llvm::Error index(tooling::CompileCommand,
106  BackgroundIndexStorage *IndexStorage);
107  void buildIndex(); // Rebuild index periodically every BuildIndexPeriodMs.
108  const size_t BuildIndexPeriodMs;
109  std::atomic<bool> SymbolsUpdatedSinceLastIndex;
110  std::mutex IndexMu;
111  std::condition_variable IndexCV;
112 
113  FileSymbols IndexedSymbols;
114  llvm::StringMap<FileDigest> IndexedFileDigests; // Key is absolute file path.
115  std::mutex DigestsMu;
116 
117  BackgroundIndexStorage::Factory IndexStorageFactory;
118  struct Source {
119  std::string Path;
120  bool NeedsReIndexing;
121  Source(llvm::StringRef Path, bool NeedsReIndexing)
122  : Path(Path), NeedsReIndexing(NeedsReIndexing) {}
123  };
124  // Loads the shards for a single TU and all of its dependencies. Returns the
125  // list of sources and whether they need to be re-indexed.
126  std::vector<Source> loadShard(const tooling::CompileCommand &Cmd,
127  BackgroundIndexStorage *IndexStorage,
128  llvm::StringSet<> &LoadedShards);
129  // Tries to load shards for the ChangedFiles.
130  std::vector<std::pair<tooling::CompileCommand, BackgroundIndexStorage *>>
131  loadShards(std::vector<std::string> ChangedFiles);
132  void enqueue(tooling::CompileCommand Cmd, BackgroundIndexStorage *Storage);
133 
134  // queue management
135  using Task = std::function<void()>;
136  void run(); // Main loop executed by Thread. Runs tasks from Queue.
137  void enqueueTask(Task T, ThreadPriority Prioirty);
138  void enqueueLocked(tooling::CompileCommand Cmd,
139  BackgroundIndexStorage *IndexStorage);
140  std::mutex QueueMu;
141  unsigned NumActiveTasks = 0; // Only idle when queue is empty *and* no tasks.
142  std::condition_variable QueueCV;
143  bool ShouldStop = false;
144  std::deque<std::pair<Task, ThreadPriority>> Queue;
145  std::vector<std::thread> ThreadPool; // FIXME: Abstract this away.
146  GlobalCompilationDatabase::CommandChanged::Subscription CommandsChanged;
147 };
148 
149 } // namespace clangd
150 } // namespace clang
151 
152 #endif
A container of Symbols from several source files.
Definition: FileIndex.h:58
llvm::unique_function< BackgroundIndexStorage *(llvm::StringRef)> Factory
Definition: Background.h:55
Provides compilation arguments used for parsing C and C++ files.
std::string Path
A typedef to represent a file path.
Definition: Path.h:21
A context is an immutable container for per-request data that must be propagated through layers that ...
Definition: Context.h:70
virtual std::unique_ptr< IndexFileIn > loadShard(llvm::StringRef ShardIdentifier) const =0
virtual llvm::Error storeShard(llvm::StringRef ShardIdentifier, IndexFileOut Shard) const =0
===– Representation.cpp - ClangDoc Representation --------—*- C++ -*-===//
const SymbolIndex * Index
Definition: Dexp.cpp:85