clang-tools  8.0.0
TUScheduler.h
Go to the documentation of this file.
1 //===--- TUScheduler.h -------------------------------------------*-C++-*-===//
2 //
3 // The LLVM Compiler Infrastructure
4 //
5 // This file is distributed under the University of Illinois Open Source
6 // License. See LICENSE.TXT for details.
7 //
8 //===----------------------------------------------------------------------===//
9 
10 #ifndef LLVM_CLANG_TOOLS_EXTRA_CLANGD_TUSCHEDULER_H
11 #define LLVM_CLANG_TOOLS_EXTRA_CLANGD_TUSCHEDULER_H
12 
13 #include "ClangdUnit.h"
14 #include "Function.h"
15 #include "Threading.h"
16 #include "llvm/ADT/StringMap.h"
17 #include <future>
18 
19 namespace clang {
20 namespace clangd {
21 
22 /// Returns a number of a default async threads to use for TUScheduler.
23 /// Returned value is always >= 1 (i.e. will not cause requests to be processed
24 /// synchronously).
26 
27 struct InputsAndAST {
30 };
31 
33  llvm::StringRef Contents;
34  const tooling::CompileCommand &Command;
36 };
37 
38 /// Determines whether diagnostics should be generated for a file snapshot.
39 enum class WantDiagnostics {
40  Yes, /// Diagnostics must be generated for this snapshot.
41  No, /// Diagnostics must not be generated for this snapshot.
42  Auto, /// Diagnostics must be generated for this snapshot or a subsequent one,
43  /// within a bounded amount of time.
44 };
45 
46 /// Configuration of the AST retention policy. This only covers retention of
47 /// *idle* ASTs. If queue has operations requiring the AST, they might be
48 /// kept in memory.
50  /// Maximum number of ASTs to be retained in memory when there are no pending
51  /// requests for them.
52  unsigned MaxRetainedASTs = 3;
53 };
54 
55 struct TUAction {
56  enum State {
57  Queued, // The TU is pending in the thread task queue to be built.
58  RunningAction, // Starting running actions on the TU.
59  BuildingPreamble, // The preamble of the TU is being built.
60  BuildingFile, // The TU is being built. It is only emitted when building
61  // the AST for diagnostics in write action (update).
62  Idle, // Indicates the worker thread is idle, and ready to run any upcoming
63  // actions.
64  };
65  TUAction(State S, llvm::StringRef Name) : S(S), Name(Name) {}
67  /// The name of the action currently running, e.g. Update, GoToDef, Hover.
68  /// Empty if we are in the idle state.
69  std::string Name;
70 };
71 
72 // Internal status of the TU in TUScheduler.
73 struct TUStatus {
74  struct BuildDetails {
75  /// Indicates whether clang failed to build the TU.
76  bool BuildFailed = false;
77  /// Indicates whether we reused the prebuilt AST.
78  bool ReuseAST = false;
79  };
80  /// Serialize this to an LSP file status item.
81  FileStatus render(PathRef File) const;
82 
85 };
86 
88 public:
89  virtual ~ParsingCallbacks() = default;
90 
91  /// Called on the AST that was built for emitting the preamble. The built AST
92  /// contains only AST nodes from the #include directives at the start of the
93  /// file. AST node in the current file should be observed on onMainAST call.
94  virtual void onPreambleAST(PathRef Path, ASTContext &Ctx,
95  std::shared_ptr<clang::Preprocessor> PP) {}
96  /// Called on the AST built for the file itself. Note that preamble AST nodes
97  /// are not deserialized and should be processed in the onPreambleAST call
98  /// instead.
99  /// The \p AST always contains all AST nodes for the main file itself, and
100  /// only a portion of the AST nodes deserialized from the preamble. Note that
101  /// some nodes from the preamble may have been deserialized and may also be
102  /// accessed from the main file AST, e.g. redecls of functions from preamble,
103  /// etc. Clients are expected to process only the AST nodes from the main file
104  /// in this callback (obtained via ParsedAST::getLocalTopLevelDecls) to obtain
105  /// optimal performance.
106  virtual void onMainAST(PathRef Path, ParsedAST &AST) {}
107 
108  /// Called whenever the diagnostics for \p File are produced.
109  virtual void onDiagnostics(PathRef File, std::vector<Diag> Diags) {}
110 
111  /// Called whenever the TU status is updated.
112  virtual void onFileUpdated(PathRef File, const TUStatus &Status) {}
113 };
114 
115 /// Handles running tasks for ClangdServer and managing the resources (e.g.,
116 /// preambles and ASTs) for opened files.
117 /// TUScheduler is not thread-safe, only one thread should be providing updates
118 /// and scheduling tasks.
119 /// Callbacks are run on a threadpool and it's appropriate to do slow work in
120 /// them. Each task has a name, used for tracing (should be UpperCamelCase).
121 /// FIXME(sammccall): pull out a scheduler options struct.
122 class TUScheduler {
123 public:
124  TUScheduler(unsigned AsyncThreadsCount, bool StorePreamblesInMemory,
125  std::unique_ptr<ParsingCallbacks> ASTCallbacks,
126  std::chrono::steady_clock::duration UpdateDebounce,
127  ASTRetentionPolicy RetentionPolicy);
128  ~TUScheduler();
129 
130  /// Returns estimated memory usage for each of the currently open files.
131  /// The order of results is unspecified.
132  std::vector<std::pair<Path, std::size_t>> getUsedBytesPerFile() const;
133 
134  /// Returns a list of files with ASTs currently stored in memory. This method
135  /// is not very reliable and is only used for test. E.g., the results will not
136  /// contain files that currently run something over their AST.
137  std::vector<Path> getFilesWithCachedAST() const;
138 
139  /// Schedule an update for \p File. Adds \p File to a list of tracked files if
140  /// \p File was not part of it before.
141  /// If diagnostics are requested (Yes), and the context is cancelled before
142  /// they are prepared, they may be skipped if eventual-consistency permits it
143  /// (i.e. WantDiagnostics is downgraded to Auto).
144  void update(PathRef File, ParseInputs Inputs, WantDiagnostics WD);
145 
146  /// Remove \p File from the list of tracked files and schedule removal of its
147  /// resources. Pending diagnostics for closed files may not be delivered, even
148  /// if requested with WantDiags::Auto or WantDiags::Yes.
149  void remove(PathRef File);
150 
151  /// Schedule an async task with no dependencies.
152  void run(llvm::StringRef Name, llvm::unique_function<void()> Action);
153 
154  /// Schedule an async read of the AST. \p Action will be called when AST is
155  /// ready. The AST passed to \p Action refers to the version of \p File
156  /// tracked at the time of the call, even if new updates are received before
157  /// \p Action is executed.
158  /// If an error occurs during processing, it is forwarded to the \p Action
159  /// callback.
160  /// If the context is cancelled before the AST is ready, the callback will
161  /// receive a CancelledError.
162  void runWithAST(llvm::StringRef Name, PathRef File,
164 
165  /// Controls whether preamble reads wait for the preamble to be up-to-date.
167  /// The preamble is generated from the current version of the file.
168  /// If the content was recently updated, we will wait until we have a
169  /// preamble that reflects that update.
170  /// This is the slowest option, and may be delayed by other tasks.
172  /// The preamble may be generated from an older version of the file.
173  /// Reading from locations in the preamble may cause files to be re-read.
174  /// This gives callers two options:
175  /// - validate that the preamble is still valid, and only use it if so
176  /// - accept that the preamble contents may be outdated, and try to avoid
177  /// reading source code from headers.
178  /// This is the fastest option, usually a preamble is available immediately.
180  };
181  /// Schedule an async read of the preamble.
182  /// If there's no preamble yet (because the file was just opened), we'll wait
183  /// for it to build. The result may be null if it fails to build or is empty.
184  /// If an error occurs, it is forwarded to the \p Action callback.
185  /// Context cancellation is ignored and should be handled by the Action.
186  /// (In practice, the Action is almost always executed immediately).
187  void runWithPreamble(llvm::StringRef Name, PathRef File,
188  PreambleConsistency Consistency,
190 
191  /// Wait until there are no scheduled or running tasks.
192  /// Mostly useful for synchronizing tests.
193  bool blockUntilIdle(Deadline D) const;
194 
195 private:
196  /// This class stores per-file data in the Files map.
197  struct FileData;
198 
199 public:
200  /// Responsible for retaining and rebuilding idle ASTs. An implementation is
201  /// an LRU cache.
202  class ASTCache;
203 
204  // The file being built/processed in the current thread. This is a hack in
205  // order to get the file name into the index implementations. Do not depend on
206  // this inside clangd.
207  // FIXME: remove this when there is proper index support via build system
208  // integration.
209  static llvm::Optional<llvm::StringRef> getFileBeingProcessedInContext();
210 
211 private:
212  const bool StorePreamblesInMemory;
213  const std::shared_ptr<PCHContainerOperations> PCHOps;
214  std::unique_ptr<ParsingCallbacks> Callbacks; // not nullptr
215  Semaphore Barrier;
216  llvm::StringMap<std::unique_ptr<FileData>> Files;
217  std::unique_ptr<ASTCache> IdleASTs;
218  // None when running tasks synchronously and non-None when running tasks
219  // asynchronously.
220  llvm::Optional<AsyncTaskRunner> PreambleTasks;
221  llvm::Optional<AsyncTaskRunner> WorkerThreads;
222  std::chrono::steady_clock::duration UpdateDebounce;
223 };
224 
225 /// Runs \p Action asynchronously with a new std::thread. The context will be
226 /// propagated.
227 template <typename T>
228 std::future<T> runAsync(llvm::unique_function<T()> Action) {
229  return std::async(
230  std::launch::async,
231  [](llvm::unique_function<T()> &&Action, Context Ctx) {
232  WithContext WithCtx(std::move(Ctx));
233  return Action();
234  },
235  std::move(Action), Context::current().clone());
236 }
237 
238 } // namespace clangd
239 } // namespace clang
240 
241 #endif
PreambleConsistency
Controls whether preamble reads wait for the preamble to be up-to-date.
Definition: TUScheduler.h:166
WantDiagnostics
Determines whether diagnostics should be generated for a file snapshot.
Definition: TUScheduler.h:39
The preamble may be generated from an older version of the file.
Definition: TUScheduler.h:179
Diagnostics must be generated for this snapshot.
const tooling::CompileCommand & Command
Definition: TUScheduler.h:34
llvm::StringRef PathRef
A typedef to represent a ref to file path.
Definition: Path.h:24
llvm::unique_function< void(llvm::Expected< T >)> Callback
A Callback<T> is a void function that accepts Expected<T>.
Definition: Function.h:29
virtual void onMainAST(PathRef Path, ParsedAST &AST)
Called on the AST built for the file itself.
Definition: TUScheduler.h:106
Limits the number of threads that can acquire the lock at the same time.
Definition: Threading.h:41
virtual void onFileUpdated(PathRef File, const TUStatus &Status)
Called whenever the TU status is updated.
Definition: TUScheduler.h:112
BuildDetails Details
Definition: TUScheduler.h:84
Configuration of the AST retention policy.
Definition: TUScheduler.h:49
std::future< T > runAsync(llvm::unique_function< T()> Action)
Runs Action asynchronously with a new std::thread.
Definition: TUScheduler.h:228
Context Ctx
Context clone() const
Clone this context object.
Definition: Context.cpp:21
llvm::unique_function< void()> Action
virtual void onPreambleAST(PathRef Path, ASTContext &Ctx, std::shared_ptr< clang::Preprocessor > PP)
Called on the AST that was built for emitting the preamble.
Definition: TUScheduler.h:94
std::string Path
A typedef to represent a file path.
Definition: Path.h:21
static const Context & current()
Returns the context for the current thread, creating it if needed.
Definition: Context.cpp:28
const ParseInputs & Inputs
Definition: TUScheduler.h:28
static constexpr llvm::StringLiteral Name
const Decl * D
Definition: XRefs.cpp:79
virtual void onDiagnostics(PathRef File, std::vector< Diag > Diags)
Called whenever the diagnostics for File are produced.
Definition: TUScheduler.h:109
A context is an immutable container for per-request data that must be propagated through layers that ...
Definition: Context.h:70
Stores and provides access to parsed AST.
Definition: ClangdUnit.h:71
unsigned getDefaultAsyncThreadsCount()
Returns a number of a default async threads to use for TUScheduler.
WithContext replaces Context::current() with a provided scope.
Definition: Context.h:190
TUAction(State S, llvm::StringRef Name)
Definition: TUScheduler.h:65
Information required to run clang, e.g. to parse AST or do code completion.
Definition: ClangdUnit.h:64
===– Representation.cpp - ClangDoc Representation --------—*- C++ -*-===//
std::string Name
The name of the action currently running, e.g.
Definition: TUScheduler.h:69
A point in time we can wait for.
Definition: Threading.h:58
Clangd extension: indicates the current state of the file in clangd, sent from server via the textDoc...
Definition: Protocol.h:1006
Handles running tasks for ClangdServer and managing the resources (e.g., preambles and ASTs) for open...
Definition: TUScheduler.h:122
const PreambleData * Preamble
Definition: TUScheduler.h:35
An LRU cache of idle ASTs.
Definition: TUScheduler.cpp:79
Diagnostics must not be generated for this snapshot.
The preamble is generated from the current version of the file.
Definition: TUScheduler.h:171