clang-tools  8.0.0
Dex.cpp
Go to the documentation of this file.
1 //===--- Dex.cpp - Dex Symbol Index Implementation --------------*- C++ -*-===//
2 //
3 // The LLVM Compiler Infrastructure
4 //
5 // This file is distributed under the University of Illinois Open Source
6 // License. See LICENSE.TXT for details.
7 //
8 //===----------------------------------------------------------------------===//
9 
10 #include "Dex.h"
11 #include "FileDistance.h"
12 #include "FuzzyMatch.h"
13 #include "Logger.h"
14 #include "Quality.h"
15 #include "Trace.h"
16 #include "index/Index.h"
17 #include "index/dex/Iterator.h"
18 #include "llvm/ADT/StringSet.h"
19 #include "llvm/Support/ScopedPrinter.h"
20 #include <algorithm>
21 #include <queue>
22 
23 namespace clang {
24 namespace clangd {
25 namespace dex {
26 
27 std::unique_ptr<SymbolIndex> Dex::build(SymbolSlab Symbols, RefSlab Refs) {
28  auto Size = Symbols.bytes() + Refs.bytes();
29  auto Data = std::make_pair(std::move(Symbols), std::move(Refs));
30  return llvm::make_unique<Dex>(Data.first, Data.second, std::move(Data), Size);
31 }
32 
33 namespace {
34 
35 // Mark symbols which are can be used for code completion.
36 const Token RestrictedForCodeCompletion =
37  Token(Token::Kind::Sentinel, "Restricted For Code Completion");
38 
39 // Returns the tokens which are given symbol's characteristics. Currently, the
40 // generated tokens only contain fuzzy matching trigrams and symbol's scope,
41 // but in the future this will also return path proximity tokens and other
42 // types of tokens such as symbol type (if applicable).
43 // Returns the tokens which are given symbols's characteristics. For example,
44 // trigrams and scopes.
45 // FIXME(kbobyrev): Support more token types:
46 // * Types
47 // * Namespace proximity
48 std::vector<Token> generateSearchTokens(const Symbol &Sym) {
49  std::vector<Token> Result = generateIdentifierTrigrams(Sym.Name);
50  Result.emplace_back(Token::Kind::Scope, Sym.Scope);
51  // Skip token generation for symbols with unknown declaration location.
52  if (!llvm::StringRef(Sym.CanonicalDeclaration.FileURI).empty())
53  for (const auto &ProximityURI :
55  Result.emplace_back(Token::Kind::ProximityURI, ProximityURI);
57  Result.emplace_back(RestrictedForCodeCompletion);
58  return Result;
59 }
60 
61 // Constructs BOOST iterators for Path Proximities.
62 std::unique_ptr<Iterator> createFileProximityIterator(
63  llvm::ArrayRef<std::string> ProximityPaths,
64  const llvm::DenseMap<Token, PostingList> &InvertedIndex,
65  const Corpus &Corpus) {
66  std::vector<std::unique_ptr<Iterator>> BoostingIterators;
67  // Deduplicate parent URIs extracted from the ProximityPaths.
68  llvm::StringSet<> ParentURIs;
69  llvm::StringMap<SourceParams> Sources;
70  for (const auto &Path : ProximityPaths) {
71  Sources[Path] = SourceParams();
72  auto PathURI = URI::create(Path);
73  const auto PathProximityURIs = generateProximityURIs(PathURI.toString());
74  for (const auto &ProximityURI : PathProximityURIs)
75  ParentURIs.insert(ProximityURI);
76  }
77  // Use SymbolRelevanceSignals for symbol relevance evaluation: use defaults
78  // for all parameters except for Proximity Path distance signal.
79  SymbolRelevanceSignals PathProximitySignals;
80  // DistanceCalculator will find the shortest distance from ProximityPaths to
81  // any URI extracted from the ProximityPaths.
82  URIDistance DistanceCalculator(Sources);
83  PathProximitySignals.FileProximityMatch = &DistanceCalculator;
84  // Try to build BOOST iterator for each Proximity Path provided by
85  // ProximityPaths. Boosting factor should depend on the distance to the
86  // Proximity Path: the closer processed path is, the higher boosting factor.
87  for (const auto &ParentURI : ParentURIs.keys()) {
88  Token Tok(Token::Kind::ProximityURI, ParentURI);
89  const auto It = InvertedIndex.find(Tok);
90  if (It != InvertedIndex.end()) {
91  // FIXME(kbobyrev): Append LIMIT on top of every BOOST iterator.
92  PathProximitySignals.SymbolURI = ParentURI;
93  BoostingIterators.push_back(Corpus.boost(
94  It->second.iterator(&It->first), PathProximitySignals.evaluate()));
95  }
96  }
97  BoostingIterators.push_back(Corpus.all());
98  return Corpus.unionOf(std::move(BoostingIterators));
99 }
100 
101 } // namespace
102 
103 void Dex::buildIndex() {
104  this->Corpus = dex::Corpus(Symbols.size());
105  std::vector<std::pair<float, const Symbol *>> ScoredSymbols(Symbols.size());
106 
107  for (size_t I = 0; I < Symbols.size(); ++I) {
108  const Symbol *Sym = Symbols[I];
109  LookupTable[Sym->ID] = Sym;
110  ScoredSymbols[I] = {quality(*Sym), Sym};
111  }
112 
113  // Symbols are sorted by symbol qualities so that items in the posting lists
114  // are stored in the descending order of symbol quality.
115  llvm::sort(ScoredSymbols, std::greater<std::pair<float, const Symbol *>>());
116 
117  // SymbolQuality was empty up until now.
118  SymbolQuality.resize(Symbols.size());
119  // Populate internal storage using Symbol + Score pairs.
120  for (size_t I = 0; I < ScoredSymbols.size(); ++I) {
121  SymbolQuality[I] = ScoredSymbols[I].first;
122  Symbols[I] = ScoredSymbols[I].second;
123  }
124 
125  // Populate TempInvertedIndex with lists for index symbols.
126  llvm::DenseMap<Token, std::vector<DocID>> TempInvertedIndex;
127  for (DocID SymbolRank = 0; SymbolRank < Symbols.size(); ++SymbolRank) {
128  const auto *Sym = Symbols[SymbolRank];
129  for (const auto &Token : generateSearchTokens(*Sym))
130  TempInvertedIndex[Token].push_back(SymbolRank);
131  }
132 
133  // Convert lists of items to posting lists.
134  for (const auto &TokenToPostingList : TempInvertedIndex)
135  InvertedIndex.insert(
136  {TokenToPostingList.first, PostingList(TokenToPostingList.second)});
137 }
138 
139 std::unique_ptr<Iterator> Dex::iterator(const Token &Tok) const {
140  auto It = InvertedIndex.find(Tok);
141  return It == InvertedIndex.end() ? Corpus.none()
142  : It->second.iterator(&It->first);
143 }
144 
145 /// Constructs iterators over tokens extracted from the query and exhausts it
146 /// while applying Callback to each symbol in the order of decreasing quality
147 /// of the matched symbols.
149  llvm::function_ref<void(const Symbol &)> Callback) const {
150  assert(!StringRef(Req.Query).contains("::") &&
151  "There must be no :: in query.");
152  trace::Span Tracer("Dex fuzzyFind");
153  FuzzyMatcher Filter(Req.Query);
154  // For short queries we use specialized trigrams that don't yield all results.
155  // Prevent clients from postfiltering them for longer queries.
156  bool More = !Req.Query.empty() && Req.Query.size() < 3;
157 
158  std::vector<std::unique_ptr<Iterator>> Criteria;
159  const auto TrigramTokens = generateQueryTrigrams(Req.Query);
160 
161  // Generate query trigrams and construct AND iterator over all query
162  // trigrams.
163  std::vector<std::unique_ptr<Iterator>> TrigramIterators;
164  for (const auto &Trigram : TrigramTokens)
165  TrigramIterators.push_back(iterator(Trigram));
166  Criteria.push_back(Corpus.intersect(move(TrigramIterators)));
167 
168  // Generate scope tokens for search query.
169  std::vector<std::unique_ptr<Iterator>> ScopeIterators;
170  for (const auto &Scope : Req.Scopes)
171  ScopeIterators.push_back(iterator(Token(Token::Kind::Scope, Scope)));
172  if (Req.AnyScope)
173  ScopeIterators.push_back(
174  Corpus.boost(Corpus.all(), ScopeIterators.empty() ? 1.0 : 0.2));
175  Criteria.push_back(Corpus.unionOf(move(ScopeIterators)));
176 
177  // Add proximity paths boosting (all symbols, some boosted).
178  Criteria.push_back(
179  createFileProximityIterator(Req.ProximityPaths, InvertedIndex, Corpus));
180 
182  Criteria.push_back(iterator(RestrictedForCodeCompletion));
183 
184  // Use TRUE iterator if both trigrams and scopes from the query are not
185  // present in the symbol index.
186  auto Root = Corpus.intersect(move(Criteria));
187  // Retrieve more items than it was requested: some of the items with high
188  // final score might not be retrieved otherwise.
189  // FIXME(kbobyrev): Tune this ratio.
190  if (Req.Limit)
191  Root = Corpus.limit(move(Root), *Req.Limit * 100);
192  SPAN_ATTACH(Tracer, "query", llvm::to_string(*Root));
193  vlog("Dex query tree: {0}", *Root);
194 
195  using IDAndScore = std::pair<DocID, float>;
196  std::vector<IDAndScore> IDAndScores = consume(*Root);
197 
198  auto Compare = [](const IDAndScore &LHS, const IDAndScore &RHS) {
199  return LHS.second > RHS.second;
200  };
202  Req.Limit ? *Req.Limit : std::numeric_limits<size_t>::max(), Compare);
203  for (const auto &IDAndScore : IDAndScores) {
204  const DocID SymbolDocID = IDAndScore.first;
205  const auto *Sym = Symbols[SymbolDocID];
206  const llvm::Optional<float> Score = Filter.match(Sym->Name);
207  if (!Score)
208  continue;
209  // Combine Fuzzy Matching score, precomputed symbol quality and boosting
210  // score for a cumulative final symbol score.
211  const float FinalScore =
212  (*Score) * SymbolQuality[SymbolDocID] * IDAndScore.second;
213  // If Top.push(...) returns true, it means that it had to pop an item. In
214  // this case, it is possible to retrieve more symbols.
215  if (Top.push({SymbolDocID, FinalScore}))
216  More = true;
217  }
218 
219  // Apply callback to the top Req.Limit items in the descending
220  // order of cumulative score.
221  for (const auto &Item : std::move(Top).items())
222  Callback(*Symbols[Item.first]);
223  return More;
224 }
225 
226 void Dex::lookup(const LookupRequest &Req,
227  llvm::function_ref<void(const Symbol &)> Callback) const {
228  trace::Span Tracer("Dex lookup");
229  for (const auto &ID : Req.IDs) {
230  auto I = LookupTable.find(ID);
231  if (I != LookupTable.end())
232  Callback(*I->second);
233  }
234 }
235 
236 void Dex::refs(const RefsRequest &Req,
237  llvm::function_ref<void(const Ref &)> Callback) const {
238  trace::Span Tracer("Dex refs");
239  uint32_t Remaining =
240  Req.Limit.getValueOr(std::numeric_limits<uint32_t>::max());
241  for (const auto &ID : Req.IDs)
242  for (const auto &Ref : Refs.lookup(ID)) {
243  if (Remaining > 0 && static_cast<int>(Req.Filter & Ref.Kind)) {
244  --Remaining;
245  Callback(Ref);
246  }
247  }
248 }
249 
250 size_t Dex::estimateMemoryUsage() const {
251  size_t Bytes = Symbols.size() * sizeof(const Symbol *);
252  Bytes += SymbolQuality.size() * sizeof(float);
253  Bytes += LookupTable.getMemorySize();
254  Bytes += InvertedIndex.getMemorySize();
255  for (const auto &TokenToPostingList : InvertedIndex)
256  Bytes += TokenToPostingList.second.bytes();
257  Bytes += Refs.getMemorySize();
258  return Bytes + BackingDataSize;
259 }
260 
261 std::vector<std::string> generateProximityURIs(llvm::StringRef URIPath) {
262  std::vector<std::string> Result;
263  auto ParsedURI = URI::parse(URIPath);
264  assert(ParsedURI &&
265  "Non-empty argument of generateProximityURIs() should be a valid "
266  "URI.");
267  llvm::StringRef Body = ParsedURI->body();
268  // FIXME(kbobyrev): Currently, this is a heuristic which defines the maximum
269  // size of resulting vector. Some projects might want to have higher limit if
270  // the file hierarchy is deeper. For the generic case, it would be useful to
271  // calculate Limit in the index build stage by calculating the maximum depth
272  // of the project source tree at runtime.
273  size_t Limit = 5;
274  // Insert original URI before the loop: this would save a redundant iteration
275  // with a URI parse.
276  Result.emplace_back(ParsedURI->toString());
277  while (!Body.empty() && --Limit > 0) {
278  // FIXME(kbobyrev): Parsing and encoding path to URIs is not necessary and
279  // could be optimized.
280  Body = llvm::sys::path::parent_path(Body, llvm::sys::path::Style::posix);
281  URI TokenURI(ParsedURI->scheme(), ParsedURI->authority(), Body);
282  if (!Body.empty())
283  Result.emplace_back(TokenURI.toString());
284  }
285  return Result;
286 }
287 
288 } // namespace dex
289 } // namespace clangd
290 } // namespace clang
size_t bytes() const
Definition: Index.h:307
llvm::DenseSet< SymbolID > IDs
Definition: Index.h:477
bool AnyScope
If set to true, allow symbols from any scope.
Definition: Index.h:449
std::unique_ptr< Iterator > intersect(std::vector< std::unique_ptr< Iterator >> Children) const
Returns AND Iterator which performs the intersection of the PostingLists of its children.
Definition: Iterator.cpp:359
void refs(const RefsRequest &Req, llvm::function_ref< void(const Ref &)> Callback) const override
Finds all occurrences (e.g.
Definition: Dex.cpp:236
bool RestrictForCodeCompletion
If set to true, only symbols for completion support will be considered.
Definition: Index.h:454
This defines Dex - a symbol index implementation based on query iterators over symbol tokens...
PostingList is the storage of DocIDs which can be inserted to the Query Tree as a leaf by constructin...
Definition: PostingList.h:60
llvm::DenseSet< SymbolID > IDs
Definition: Index.h:473
llvm::unique_function< void(llvm::Expected< T >)> Callback
A Callback<T> is a void function that accepts Expected<T>.
Definition: Function.h:29
llvm::StringRef Scope
Definition: Index.h:168
std::vector< Token > generateIdentifierTrigrams(llvm::StringRef Identifier)
Returns list of unique fuzzy-search trigrams from unqualified symbol.
Definition: Trigram.cpp:24
bool fuzzyFind(const FuzzyFindRequest &Req, llvm::function_ref< void(const Symbol &)> Callback) const override
Constructs iterators over tokens extracted from the query and exhausts it while applying Callback to ...
Definition: Dex.cpp:148
void vlog(const char *Fmt, Ts &&... Vals)
Definition: Logger.h:68
std::vector< std::string > Scopes
If this is non-empty, symbols must be in at least one of the scopes (e.g.
Definition: Index.h:446
A Token represents an attribute of a symbol, such as a particular trigram present in the name (used f...
Definition: Token.h:41
std::vector< std::pair< DocID, float > > consume(Iterator &It)
Advances the iterator until it is exhausted.
Definition: Iterator.cpp:351
std::unique_ptr< Iterator > unionOf(std::vector< std::unique_ptr< Iterator >> Children) const
Returns OR Iterator which performs the union of the PostingLists of its children. ...
Definition: Iterator.cpp:389
Whether or not this symbol is meant to be used for the code completion.
Definition: Index.h:239
std::vector< std::string > generateProximityURIs(llvm::StringRef URIPath)
Returns Search Token for a number of parent directories of given Path.
Definition: Dex.cpp:261
std::vector< Token > generateQueryTrigrams(llvm::StringRef Query)
Returns list of unique fuzzy-search trigrams given a query.
Definition: Trigram.cpp:87
SymbolFlag Flags
Definition: Index.h:248
std::unique_ptr< Iterator > limit(std::unique_ptr< Iterator > Child, size_t Limit) const
Returns LIMIT iterator, which yields up to N elements of its child iterator.
Definition: Iterator.cpp:435
static std::unique_ptr< SymbolIndex > build(SymbolSlab, RefSlab)
Builds an index from slabs. The index takes ownership of the slab.
Definition: Dex.cpp:27
std::unique_ptr< Iterator > none() const
Returns FALSE Iterator which iterates over no documents.
Definition: Iterator.cpp:422
std::string Path
A typedef to represent a file path.
Definition: Path.h:21
std::string Query
A query string for the fuzzy find.
Definition: Index.h:439
llvm::Optional< float > match(llvm::StringRef Word)
Definition: FuzzyMatch.cpp:93
SymbolLocation CanonicalDeclaration
Definition: Index.h:180
bool push(value_type &&V)
Definition: Quality.h:156
uint32_t DocID
Symbol position in the list of all index symbols sorted by a pre-computed symbol quality.
Definition: Iterator.h:47
llvm::Optional< llvm::Expected< tooling::AtomicChanges > > Result
void lookup(const LookupRequest &Req, llvm::function_ref< void(const Symbol &)> Callback) const override
Looks up symbols with any of the given symbol IDs and applies Callback on each matched symbol...
Definition: Dex.cpp:226
llvm::StringRef SymbolURI
These are used to calculate proximity between the index symbol and the query.
Definition: Quality.h:98
static llvm::Expected< URI > create(llvm::StringRef AbsolutePath, llvm::StringRef Scheme)
Creates a URI for a file in the given scheme.
Definition: URI.cpp:188
===– Representation.cpp - ClangDoc Representation --------—*- C++ -*-===//
std::vector< std::string > ProximityPaths
Contextually relevant files (e.g.
Definition: Index.h:457
Path Proximity URI to symbol declaration.
llvm::StringRef Name
Definition: Index.h:166
llvm::Optional< uint32_t > Limit
If set, limit the number of refers returned from the index.
Definition: Index.h:482
Symbol index queries consist of specific requirements for the requested symbol, such as high fuzzy ma...
A URI describes the location of a source file.
Definition: URI.h:29
llvm::Optional< uint32_t > Limit
The number of top candidates to return.
Definition: Index.h:452
size_t bytes() const
Definition: Index.h:405
Internal Token type for invalid/special tokens, e.g.
static llvm::Expected< URI > parse(llvm::StringRef Uri)
Parse a URI string "<scheme>:[//<authority>/]<path>".
Definition: URI.cpp:166
std::unique_ptr< Iterator > boost(std::unique_ptr< Iterator > Child, float Factor) const
Returns BOOST iterator which multiplies the score of each item by given factor.
Definition: Iterator.cpp:426
std::unique_ptr< Iterator > all() const
Returns TRUE Iterator which iterates over "virtual" PostingList containing all items in range [0...
Definition: Iterator.cpp:418
size_t estimateMemoryUsage() const override
Returns estimated size of index (in bytes).
Definition: Dex.cpp:250
Records an event whose duration is the lifetime of the Span object.
Definition: Trace.h:83
const char * FileURI
Definition: Index.h:72
Attributes of a symbol-query pair that affect how much we like it.
Definition: Quality.h:87
#define SPAN_ATTACH(S, Name, Expr)
Attach a key-value pair to a Span event.
Definition: Trace.h:99
RefKind Kind
Definition: Index.h:376
float quality(const Symbol &S)
Definition: Index.cpp:69
TopN<T> is a lossy container that preserves only the "best" N elements.
Definition: Quality.h:148