clang-tools  8.0.0
FileIndex.cpp
Go to the documentation of this file.
1 //===--- FileIndex.cpp - Indexes for files. ------------------------ C++-*-===//
2 //
3 // The LLVM Compiler Infrastructure
4 //
5 // This file is distributed under the University of Illinois Open Source
6 // License. See LICENSE.TXT for details.
7 //
8 //===----------------------------------------------------------------------===//
9 
10 #include "FileIndex.h"
11 #include "ClangdUnit.h"
12 #include "Logger.h"
13 #include "SymbolCollector.h"
14 #include "index/Index.h"
15 #include "index/MemIndex.h"
16 #include "index/Merge.h"
17 #include "index/dex/Dex.h"
18 #include "clang/Index/IndexingAction.h"
19 #include "clang/Lex/MacroInfo.h"
20 #include "clang/Lex/Preprocessor.h"
21 #include "llvm/ADT/DenseMap.h"
22 #include "llvm/ADT/DenseSet.h"
23 #include "llvm/ADT/STLExtras.h"
24 #include "llvm/ADT/StringRef.h"
25 #include <memory>
26 
27 namespace clang {
28 namespace clangd {
29 
30 static std::pair<SymbolSlab, RefSlab>
31 indexSymbols(ASTContext &AST, std::shared_ptr<Preprocessor> PP,
32  llvm::ArrayRef<Decl *> DeclsToIndex, bool IsIndexMainAST) {
33  SymbolCollector::Options CollectorOpts;
34  // FIXME(ioeric): we might also want to collect include headers. We would need
35  // to make sure all includes are canonicalized (with CanonicalIncludes), which
36  // is not trivial given the current way of collecting symbols: we only have
37  // AST at this point, but we also need preprocessor callbacks (e.g.
38  // CommentHandler for IWYU pragma) to canonicalize includes.
39  CollectorOpts.CollectIncludePath = false;
40  CollectorOpts.CountReferences = false;
41  CollectorOpts.Origin = SymbolOrigin::Dynamic;
42 
43  index::IndexingOptions IndexOpts;
44  // We only need declarations, because we don't count references.
45  IndexOpts.SystemSymbolFilter =
46  index::IndexingOptions::SystemSymbolFilterKind::DeclarationsOnly;
47  IndexOpts.IndexFunctionLocals = false;
48  if (IsIndexMainAST) {
49  // We only collect refs when indexing main AST.
50  CollectorOpts.RefFilter = RefKind::All;
51  }else {
52  IndexOpts.IndexMacrosInPreprocessor = true;
53  CollectorOpts.CollectMacro = true;
54  }
55 
56  SymbolCollector Collector(std::move(CollectorOpts));
57  Collector.setPreprocessor(PP);
58  index::indexTopLevelDecls(AST, *PP, DeclsToIndex, Collector, IndexOpts);
59 
60  const auto &SM = AST.getSourceManager();
61  const auto *MainFileEntry = SM.getFileEntryForID(SM.getMainFileID());
62  std::string FileName = MainFileEntry ? MainFileEntry->getName() : "";
63 
64  auto Syms = Collector.takeSymbols();
65  auto Refs = Collector.takeRefs();
66  vlog("index AST for {0} (main={1}): \n"
67  " symbol slab: {2} symbols, {3} bytes\n"
68  " ref slab: {4} symbols, {5} refs, {6} bytes",
69  FileName, IsIndexMainAST, Syms.size(), Syms.bytes(), Refs.size(),
70  Refs.numRefs(), Refs.bytes());
71  return {std::move(Syms), std::move(Refs)};
72 }
73 
74 std::pair<SymbolSlab, RefSlab> indexMainDecls(ParsedAST &AST) {
75  return indexSymbols(AST.getASTContext(), AST.getPreprocessorPtr(),
77  /*IsIndexMainAST=*/true);
78 }
79 
81  std::shared_ptr<Preprocessor> PP) {
82  std::vector<Decl *> DeclsToIndex(
83  AST.getTranslationUnitDecl()->decls().begin(),
84  AST.getTranslationUnitDecl()->decls().end());
85  return indexSymbols(AST, std::move(PP), DeclsToIndex,
86  /*IsIndexMainAST=*/false)
87  .first;
88 }
89 
90 void FileSymbols::update(PathRef Path, std::unique_ptr<SymbolSlab> Symbols,
91  std::unique_ptr<RefSlab> Refs) {
92  std::lock_guard<std::mutex> Lock(Mutex);
93  if (!Symbols)
94  FileToSymbols.erase(Path);
95  else
96  FileToSymbols[Path] = std::move(Symbols);
97  if (!Refs)
98  FileToRefs.erase(Path);
99  else
100  FileToRefs[Path] = std::move(Refs);
101 }
102 
103 std::unique_ptr<SymbolIndex>
105  std::vector<std::shared_ptr<SymbolSlab>> SymbolSlabs;
106  std::vector<std::shared_ptr<RefSlab>> RefSlabs;
107  {
108  std::lock_guard<std::mutex> Lock(Mutex);
109  for (const auto &FileAndSymbols : FileToSymbols)
110  SymbolSlabs.push_back(FileAndSymbols.second);
111  for (const auto &FileAndRefs : FileToRefs)
112  RefSlabs.push_back(FileAndRefs.second);
113  }
114  std::vector<const Symbol *> AllSymbols;
115  std::vector<Symbol> SymsStorage;
116  switch (DuplicateHandle) {
118  llvm::DenseMap<SymbolID, Symbol> Merged;
119  for (const auto &Slab : SymbolSlabs) {
120  for (const auto &Sym : *Slab) {
121  auto I = Merged.try_emplace(Sym.ID, Sym);
122  if (!I.second)
123  I.first->second = mergeSymbol(I.first->second, Sym);
124  }
125  }
126  SymsStorage.reserve(Merged.size());
127  for (auto &Sym : Merged) {
128  SymsStorage.push_back(std::move(Sym.second));
129  AllSymbols.push_back(&SymsStorage.back());
130  }
131  // FIXME: aggregate symbol reference count based on references.
132  break;
133  }
135  llvm::DenseSet<SymbolID> AddedSymbols;
136  for (const auto &Slab : SymbolSlabs)
137  for (const auto &Sym : *Slab)
138  if (AddedSymbols.insert(Sym.ID).second)
139  AllSymbols.push_back(&Sym);
140  break;
141  }
142  }
143 
144  std::vector<Ref> RefsStorage; // Contiguous ranges for each SymbolID.
145  llvm::DenseMap<SymbolID, llvm::ArrayRef<Ref>> AllRefs;
146  {
147  llvm::DenseMap<SymbolID, llvm::SmallVector<Ref, 4>> MergedRefs;
148  size_t Count = 0;
149  for (const auto &RefSlab : RefSlabs)
150  for (const auto &Sym : *RefSlab) {
151  MergedRefs[Sym.first].append(Sym.second.begin(), Sym.second.end());
152  Count += Sym.second.size();
153  }
154  RefsStorage.reserve(Count);
155  AllRefs.reserve(MergedRefs.size());
156  for (auto &Sym : MergedRefs) {
157  auto &SymRefs = Sym.second;
158  // Sorting isn't required, but yields more stable results over rebuilds.
159  llvm::sort(SymRefs);
160  llvm::copy(SymRefs, back_inserter(RefsStorage));
161  AllRefs.try_emplace(
162  Sym.first,
163  llvm::ArrayRef<Ref>(&RefsStorage[RefsStorage.size() - SymRefs.size()],
164  SymRefs.size()));
165  }
166  }
167 
168  size_t StorageSize =
169  RefsStorage.size() * sizeof(Ref) + SymsStorage.size() * sizeof(Symbol);
170  for (const auto &Slab : SymbolSlabs)
171  StorageSize += Slab->bytes();
172  for (const auto &RefSlab : RefSlabs)
173  StorageSize += RefSlab->bytes();
174 
175  // Index must keep the slabs and contiguous ranges alive.
176  switch (Type) {
177  case IndexType::Light:
178  return llvm::make_unique<MemIndex>(
179  llvm::make_pointee_range(AllSymbols), std::move(AllRefs),
180  std::make_tuple(std::move(SymbolSlabs), std::move(RefSlabs),
181  std::move(RefsStorage), std::move(SymsStorage)),
182  StorageSize);
183  case IndexType::Heavy:
184  return llvm::make_unique<dex::Dex>(
185  llvm::make_pointee_range(AllSymbols), std::move(AllRefs),
186  std::make_tuple(std::move(SymbolSlabs), std::move(RefSlabs),
187  std::move(RefsStorage), std::move(SymsStorage)),
188  StorageSize);
189  }
190  llvm_unreachable("Unknown clangd::IndexType");
191 }
192 
194  : MergedIndex(&MainFileIndex, &PreambleIndex), UseDex(UseDex),
195  PreambleIndex(llvm::make_unique<MemIndex>()),
196  MainFileIndex(llvm::make_unique<MemIndex>()) {}
197 
199  std::shared_ptr<Preprocessor> PP) {
200  auto Symbols = indexHeaderSymbols(AST, std::move(PP));
201  PreambleSymbols.update(Path,
202  llvm::make_unique<SymbolSlab>(std::move(Symbols)),
203  llvm::make_unique<RefSlab>());
204  PreambleIndex.reset(
205  PreambleSymbols.buildIndex(UseDex ? IndexType::Heavy : IndexType::Light,
207 }
208 
210  auto Contents = indexMainDecls(AST);
211  MainFileSymbols.update(
212  Path, llvm::make_unique<SymbolSlab>(std::move(Contents.first)),
213  llvm::make_unique<RefSlab>(std::move(Contents.second)));
214  MainFileIndex.reset(
216 }
217 
218 } // namespace clangd
219 } // namespace clang
SymbolSlab indexHeaderSymbols(ASTContext &AST, std::shared_ptr< Preprocessor > PP)
Idex declarations from AST and macros from PP that are declared in included headers.
Definition: FileIndex.cpp:80
llvm::StringRef Contents
Some operations such as code completion produce a set of candidates.
This defines Dex - a symbol index implementation based on query iterators over symbol tokens...
IndexType
Select between in-memory index implementations, which have tradeoffs.
Definition: FileIndex.h:30
Collect declarations (symbols) from an AST.
llvm::StringRef PathRef
A typedef to represent a ref to file path.
Definition: Path.h:24
ArrayRef< Decl * > getLocalTopLevelDecls()
This function returns top-level decls present in the main file of the AST.
Definition: ClangdUnit.cpp:364
void vlog(const char *Fmt, Ts &&... Vals)
Definition: Logger.h:68
void updateMain(PathRef Path, ParsedAST &AST)
Update symbols and references from main file Path with indexMainDecls.
Definition: FileIndex.cpp:209
ASTContext & getASTContext()
Note that the returned ast will not contain decls from the preamble that were not deserialized during...
Definition: ClangdUnit.cpp:348
static llvm::cl::opt< bool > UseDex("use-dex-index", llvm::cl::desc("Use experimental Dex dynamic index."), llvm::cl::init(false), llvm::cl::Hidden)
DuplicateHandling
How to handle duplicated symbols across multiple files.
Definition: FileIndex.h:38
void updatePreamble(PathRef Path, ASTContext &AST, std::shared_ptr< Preprocessor > PP)
Update preamble symbols of file Path with all declarations in AST and macros in PP.
Definition: FileIndex.cpp:198
MemIndex is a naive in-memory index suitable for a small set of symbols.
Definition: MemIndex.h:20
std::shared_ptr< Preprocessor > getPreprocessorPtr()
Definition: ClangdUnit.cpp:356
std::pair< SymbolSlab, RefSlab > indexMainDecls(ParsedAST &AST)
Retrieves symbols and refs of local top level decls in AST (i.e.
Definition: FileIndex.cpp:74
std::string Path
A typedef to represent a file path.
Definition: Path.h:21
std::unique_ptr< SymbolIndex > buildIndex(IndexType, DuplicateHandling DuplicateHandle=DuplicateHandling::PickOne)
Definition: FileIndex.cpp:104
Symbol mergeSymbol(const Symbol &L, const Symbol &R)
Definition: Merge.cpp:119
PathRef FileName
Stores and provides access to parsed AST.
Definition: ClangdUnit.h:71
FileIndex(bool UseDex=true)
Definition: FileIndex.cpp:193
static std::pair< SymbolSlab, RefSlab > indexSymbols(ASTContext &AST, std::shared_ptr< Preprocessor > PP, llvm::ArrayRef< Decl *> DeclsToIndex, bool IsIndexMainAST)
Definition: FileIndex.cpp:31
===– Representation.cpp - ClangDoc Representation --------—*- C++ -*-===//
void update(PathRef Path, std::unique_ptr< SymbolSlab > Slab, std::unique_ptr< RefSlab > Refs)
Updates all symbols and refs in a file.
Definition: FileIndex.cpp:90
void setPreprocessor(std::shared_ptr< Preprocessor > PP) override
size_t bytes() const
Definition: Index.h:405
void reset(std::unique_ptr< SymbolIndex >)
Definition: Index.cpp:162
RefKind RefFilter
The symbol ref kinds that will be collected.