16 #include "llvm/Support/Compression.h" 17 #include "llvm/Support/Endian.h" 18 #include "llvm/Support/Error.h" 23 llvm::Error
makeError(
const llvm::Twine &Msg) {
24 return llvm::make_error<llvm::StringError>(Msg,
25 llvm::inconvertibleErrorCode());
39 const char *Begin, *End;
43 Reader(llvm::StringRef Data) : Begin(Data.begin()), End(Data.end()) {}
46 bool err()
const {
return Err; }
48 bool eof()
const {
return Begin == End || Err; }
50 llvm::StringRef rest()
const {
return llvm::StringRef(Begin, End - Begin); }
53 if (LLVM_UNLIKELY(Begin == End)) {
60 uint32_t consume32() {
61 if (LLVM_UNLIKELY(Begin + 4 > End)) {
65 auto Ret = llvm::support::endian::read32le(Begin);
70 llvm::StringRef
consume(
int N) {
71 if (LLVM_UNLIKELY(Begin + N > End)) {
73 return llvm::StringRef();
75 llvm::StringRef Ret(Begin, N);
80 uint32_t consumeVar() {
81 constexpr
static uint8_t More = 1 << 7;
82 uint8_t B = consume8();
83 if (LLVM_LIKELY(!(B & More)))
85 uint32_t Val = B & ~More;
86 for (
int Shift = 7; B & More && Shift < 32; Shift += 7) {
88 Val |= (B & ~More) << Shift;
93 llvm::StringRef consumeString(llvm::ArrayRef<llvm::StringRef>
Strings) {
94 auto StringIndex = consumeVar();
95 if (LLVM_UNLIKELY(StringIndex >= Strings.size())) {
97 return llvm::StringRef();
99 return Strings[StringIndex];
108 void write32(uint32_t I, llvm::raw_ostream &OS) {
110 llvm::support::endian::write32le(buf, I);
111 OS.write(buf,
sizeof(buf));
114 void writeVar(uint32_t I, llvm::raw_ostream &OS) {
115 constexpr
static uint8_t More = 1 << 7;
116 if (LLVM_LIKELY(I < 1 << 7)) {
144 class StringTableOut {
145 llvm::DenseSet<llvm::StringRef> Unique;
146 std::vector<llvm::StringRef> Sorted;
148 llvm::DenseMap<std::pair<const char *, size_t>,
unsigned>
Index;
157 void intern(llvm::StringRef &S) { S = *Unique.insert(S).first; };
159 void finalize(llvm::raw_ostream &OS) {
160 Sorted = {Unique.begin(), Unique.end()};
162 for (
unsigned I = 0; I < Sorted.size(); ++I)
163 Index.try_emplace({Sorted[I].data(), Sorted[I].size()}, I);
165 std::string RawTable;
166 for (llvm::StringRef S : Sorted) {
168 RawTable.push_back(0);
170 if (llvm::zlib::isAvailable()) {
171 llvm::SmallString<1> Compressed;
172 llvm::cantFail(llvm::zlib::compress(RawTable, Compressed));
173 write32(RawTable.size(), OS);
181 unsigned index(llvm::StringRef S)
const {
182 assert(!Sorted.empty() &&
"table not finalized");
183 assert(Index.count({S.data(), S.size()}) &&
"string not interned");
184 return Index.find({S.data(), S.size()})->second;
188 struct StringTableIn {
193 llvm::Expected<StringTableIn> readStringTable(llvm::StringRef Data) {
195 size_t UncompressedSize = R.consume32();
197 return makeError(
"Truncated string table");
199 llvm::StringRef Uncompressed;
200 llvm::SmallString<1> UncompressedStorage;
201 if (UncompressedSize == 0)
202 Uncompressed = R.rest();
204 if (llvm::Error E = llvm::zlib::uncompress(R.rest(), UncompressedStorage,
207 Uncompressed = UncompressedStorage;
211 llvm::StringSaver Saver(Table.Arena);
212 R = Reader(Uncompressed);
213 for (Reader R(Uncompressed); !R.eof();) {
214 auto Len = R.rest().find(0);
215 if (Len == llvm::StringRef::npos)
216 return makeError(
"Bad string table: not null terminated");
217 Table.Strings.push_back(Saver.save(R.consume(Len)));
221 return makeError(
"Truncated string table");
222 return std::move(Table);
231 void writeLocation(
const SymbolLocation &
Loc,
const StringTableOut &Strings,
232 llvm::raw_ostream &OS) {
233 writeVar(Strings.index(Loc.FileURI), OS);
234 for (
const auto &Endpoint : {Loc.Start, Loc.End}) {
235 writeVar(Endpoint.line(), OS);
236 writeVar(Endpoint.column(), OS);
240 SymbolLocation readLocation(Reader &Data,
241 llvm::ArrayRef<llvm::StringRef> Strings) {
243 Loc.FileURI = Data.consumeString(Strings).data();
244 for (
auto *Endpoint : {&Loc.Start, &Loc.End}) {
245 Endpoint->setLine(Data.consumeVar());
246 Endpoint->setColumn(Data.consumeVar());
251 IncludeGraphNode readIncludeGraphNode(Reader &Data,
252 llvm::ArrayRef<llvm::StringRef> Strings) {
253 IncludeGraphNode IGN;
254 IGN.IsTU = Data.consume8();
255 IGN.URI = Data.consumeString(Strings);
256 llvm::StringRef Digest = Data.consume(IGN.Digest.size());
257 std::copy(Digest.bytes_begin(), Digest.bytes_end(), IGN.Digest.begin());
258 IGN.DirectIncludes.resize(Data.consumeVar());
259 for (llvm::StringRef &Include : IGN.DirectIncludes)
260 Include = Data.consumeString(Strings);
264 void writeIncludeGraphNode(
const IncludeGraphNode &IGN,
265 const StringTableOut &Strings,
266 llvm::raw_ostream &OS) {
268 writeVar(Strings.index(IGN.URI), OS);
269 llvm::StringRef Hash(reinterpret_cast<const char *>(IGN.Digest.data()),
272 writeVar(IGN.DirectIncludes.size(), OS);
273 for (llvm::StringRef Include : IGN.DirectIncludes)
274 writeVar(Strings.index(Include), OS);
277 void writeSymbol(
const Symbol &Sym,
const StringTableOut &Strings,
278 llvm::raw_ostream &OS) {
281 OS.write(static_cast<uint8_t>(Sym.SymInfo.Kind));
282 OS.write(static_cast<uint8_t>(Sym.SymInfo.Lang));
283 writeVar(Strings.index(Sym.Name), OS);
284 writeVar(Strings.index(Sym.Scope), OS);
285 writeLocation(Sym.Definition, Strings, OS);
286 writeLocation(Sym.CanonicalDeclaration, Strings, OS);
287 writeVar(Sym.References, OS);
288 OS.write(static_cast<uint8_t>(Sym.Flags));
289 OS.write(static_cast<uint8_t>(Sym.Origin));
290 writeVar(Strings.index(Sym.Signature), OS);
291 writeVar(Strings.index(Sym.CompletionSnippetSuffix), OS);
292 writeVar(Strings.index(Sym.Documentation), OS);
293 writeVar(Strings.index(Sym.ReturnType), OS);
294 writeVar(Strings.index(Sym.Type), OS);
296 auto WriteInclude = [&](
const Symbol::IncludeHeaderWithReferences &Include) {
297 writeVar(Strings.index(Include.IncludeHeader), OS);
298 writeVar(Include.References, OS);
300 writeVar(Sym.IncludeHeaders.size(), OS);
301 for (
const auto &Include : Sym.IncludeHeaders)
302 WriteInclude(Include);
305 Symbol readSymbol(Reader &Data, llvm::ArrayRef<llvm::StringRef> Strings) {
307 Sym.ID = Data.consumeID();
309 Sym.SymInfo.Lang =
static_cast<index::SymbolLanguage
>(Data.consume8());
310 Sym.Name = Data.consumeString(Strings);
311 Sym.Scope = Data.consumeString(Strings);
312 Sym.Definition = readLocation(Data, Strings);
313 Sym.CanonicalDeclaration = readLocation(Data, Strings);
314 Sym.References = Data.consumeVar();
316 Sym.Origin =
static_cast<SymbolOrigin>(Data.consumeVar());
317 Sym.Signature = Data.consumeString(Strings);
318 Sym.CompletionSnippetSuffix = Data.consumeString(Strings);
319 Sym.Documentation = Data.consumeString(Strings);
320 Sym.ReturnType = Data.consumeString(Strings);
321 Sym.Type = Data.consumeString(Strings);
322 Sym.IncludeHeaders.resize(Data.consumeVar());
323 for (
auto &I : Sym.IncludeHeaders) {
324 I.IncludeHeader = Data.consumeString(Strings);
325 I.References = Data.consumeVar();
337 void writeRefs(
const SymbolID &ID, llvm::ArrayRef<Ref> Refs,
338 const StringTableOut &Strings, llvm::raw_ostream &OS) {
340 writeVar(Refs.size(), OS);
341 for (
const auto &Ref : Refs) {
342 OS.write(static_cast<unsigned char>(Ref.Kind));
343 writeLocation(Ref.Location, Strings, OS);
347 std::pair<SymbolID, std::vector<Ref>>
348 readRefs(Reader &Data, llvm::ArrayRef<llvm::StringRef> Strings) {
349 std::pair<SymbolID, std::vector<Ref>>
Result;
350 Result.first = Data.consumeID();
351 Result.second.resize(Data.consumeVar());
352 for (
auto &Ref : Result.second) {
353 Ref.Kind =
static_cast<RefKind>(Data.consume8());
354 Ref.Location = readLocation(Data, Strings);
371 constexpr
static uint32_t Version = 8;
373 llvm::Expected<IndexFileIn> readRIFF(llvm::StringRef Data) {
376 return RIFF.takeError();
379 llvm::StringMap<llvm::StringRef> Chunks;
380 for (
const auto &Chunk :
RIFF->Chunks)
381 Chunks.try_emplace(llvm::StringRef(Chunk.ID.data(), Chunk.ID.size()),
384 for (llvm::StringRef RequiredChunk : {
"meta",
"stri"})
385 if (!Chunks.count(RequiredChunk))
386 return makeError(
"missing required chunk " + RequiredChunk);
388 Reader Meta(Chunks.lookup(
"meta"));
389 if (Meta.consume32() != Version)
392 auto Strings = readStringTable(Chunks.lookup(
"stri"));
394 return Strings.takeError();
397 if (Chunks.count(
"srcs")) {
398 Reader SrcsReader(Chunks.lookup(
"srcs"));
399 Result.Sources.emplace();
400 while (!SrcsReader.eof()) {
401 auto IGN = readIncludeGraphNode(SrcsReader, Strings->Strings);
402 auto Entry = Result.Sources->try_emplace(IGN.URI).first;
403 Entry->getValue() = std::move(IGN);
407 for (
auto &Include :
Entry->getValue().DirectIncludes)
408 Include = Result.Sources->try_emplace(Include).first->getKey();
410 if (SrcsReader.err())
411 return makeError(
"malformed or truncated include uri");
414 if (Chunks.count(
"symb")) {
415 Reader SymbolReader(Chunks.lookup(
"symb"));
416 SymbolSlab::Builder Symbols;
417 while (!SymbolReader.eof())
418 Symbols.insert(readSymbol(SymbolReader, Strings->Strings));
419 if (SymbolReader.err())
420 return makeError(
"malformed or truncated symbol");
421 Result.Symbols = std::move(Symbols).build();
423 if (Chunks.count(
"refs")) {
424 Reader RefsReader(Chunks.lookup(
"refs"));
425 RefSlab::Builder Refs;
426 while (!RefsReader.eof()) {
427 auto RefsBundle = readRefs(RefsReader, Strings->Strings);
428 for (
const auto &Ref : RefsBundle.second)
429 Refs.insert(RefsBundle.first, Ref);
431 if (RefsReader.err())
432 return makeError(
"malformed or truncated refs");
433 Result.Refs = std::move(Refs).build();
435 return std::move(Result);
438 template <
class Callback>
441 for (llvm::StringRef &Include : IGN.DirectIncludes)
445 void writeRIFF(
const IndexFileOut &Data, llvm::raw_ostream &OS) {
446 assert(Data.Symbols &&
"An index file without symbols makes no sense!");
450 llvm::SmallString<4> Meta;
452 llvm::raw_svector_ostream MetaOS(Meta);
453 write32(Version, MetaOS);
458 std::vector<Symbol> Symbols;
459 for (
const auto &Sym : *Data.Symbols) {
460 Symbols.emplace_back(Sym);
462 [&](llvm::StringRef &S) { Strings.intern(S); });
464 std::vector<IncludeGraphNode> Sources;
466 for (
const auto &Source : *Data.Sources) {
467 Sources.push_back(Source.getValue());
469 [&](llvm::StringRef &S) { Strings.intern(S); });
472 std::vector<std::pair<SymbolID, std::vector<Ref>>> Refs;
474 for (
const auto &Sym : *Data.Refs) {
475 Refs.emplace_back(Sym);
476 for (
auto &Ref : Refs.back().second) {
477 llvm::StringRef
File = Ref.Location.FileURI;
478 Strings.intern(File);
479 Ref.Location.FileURI = File.data();
484 std::string StringSection;
486 llvm::raw_string_ostream StringOS(StringSection);
487 Strings.finalize(StringOS);
489 RIFF.Chunks.push_back({
riff::fourCC(
"stri"), StringSection});
491 std::string SymbolSection;
493 llvm::raw_string_ostream SymbolOS(SymbolSection);
494 for (
const auto &Sym : Symbols)
495 writeSymbol(Sym, Strings, SymbolOS);
497 RIFF.Chunks.push_back({
riff::fourCC(
"symb"), SymbolSection});
499 std::string RefsSection;
502 llvm::raw_string_ostream RefsOS(RefsSection);
503 for (
const auto &Sym : Refs)
504 writeRefs(Sym.first, Sym.second, Strings, RefsOS);
506 RIFF.Chunks.push_back({
riff::fourCC(
"refs"), RefsSection});
509 std::string SrcsSection;
512 llvm::raw_string_ostream SrcsOS(SrcsSection);
513 for (
const auto &SF : Sources)
514 writeIncludeGraphNode(SF, Strings, SrcsOS);
516 RIFF.Chunks.push_back({
riff::fourCC(
"srcs"), SrcsSection});
525 void writeYAML(
const IndexFileOut &, llvm::raw_ostream &);
526 llvm::Expected<IndexFileIn>
readYAML(llvm::StringRef);
541 if (Data.startswith(
"RIFF")) {
542 return readRIFF(Data);
543 }
else if (
auto YAMLContents =
readYAML(Data)) {
544 return std::move(*YAMLContents);
546 return makeError(
"Not a RIFF file and failed to parse as YAML: " +
551 std::unique_ptr<SymbolIndex>
loadIndex(llvm::StringRef SymbolFilename,
554 auto Buffer = llvm::MemoryBuffer::getFile(SymbolFilename);
556 llvm::errs() <<
"Can't open " << SymbolFilename <<
"\n";
566 Symbols = std::move(*I->Symbols);
568 Refs = std::move(*I->Refs);
570 llvm::errs() <<
"Bad Index: " <<
llvm::toString(I.takeError()) <<
"\n";
575 size_t NumSym = Symbols.
size();
576 size_t NumRefs = Refs.
numRefs();
579 auto Index = UseDex ?
dex::Dex::build(std::move(Symbols), std::move(Refs))
581 vlog(
"Loaded {0} from {1} with estimated memory usage {2} bytes\n" 582 " - number of symbols: {3}\n" 583 " - number of refs: {4}\n",
584 UseDex ?
"Dex" :
"MemIndex", SymbolFilename,
585 Index->estimateMemoryUsage(), NumSym, NumRefs);
SourceLocation Loc
'#' location in the include directive
llvm::Expected< IndexFileIn > readYAML(llvm::StringRef)
std::unique_ptr< SymbolIndex > loadIndex(llvm::StringRef SymbolFilename, bool UseDex)
static llvm::Error makeError(const char *Msg)
This defines Dex - a symbol index implementation based on query iterators over symbol tokens...
void writeYAML(const IndexFileOut &, llvm::raw_ostream &)
clang::find_all_symbols::SymbolInfo::SymbolKind SymbolKind
constexpr FourCC fourCC(const char(&Literal)[5])
static std::unique_ptr< SymbolIndex > build(SymbolSlab Symbols, RefSlab Refs)
Builds an index from slabs. The index takes ownership of the data.
static llvm::StringRef toString(SpecialMemberFunctionsCheck::SpecialMemberFunctionKind K)
llvm::unique_function< void(llvm::Expected< T >)> Callback
A Callback<T> is a void function that accepts Expected<T>.
llvm::Expected< File > readFile(llvm::StringRef Stream)
llvm::Expected< IndexFileIn > readIndexFile(llvm::StringRef Data)
void vlog(const char *Fmt, Ts &&... Vals)
static constexpr size_t RawSize
static llvm::cl::opt< bool > UseDex("use-dex-index", llvm::cl::desc("Use experimental Dex dynamic index."), llvm::cl::init(false), llvm::cl::Hidden)
llvm::BumpPtrAllocator Arena
std::vector< std::pair< DocID, float > > consume(Iterator &It)
Advances the iterator until it is exhausted.
static std::unique_ptr< SymbolIndex > build(SymbolSlab, RefSlab)
Builds an index from slabs. The index takes ownership of the slab.
llvm::Optional< llvm::Expected< tooling::AtomicChanges > > Result
std::vector< llvm::StringRef > Strings
===– Representation.cpp - ClangDoc Representation --------—*- C++ -*-===//
llvm::raw_ostream & operator<<(llvm::raw_ostream &OS, const CodeCompletion &C)
Records an event whose duration is the lifetime of the Span object.
std::array< uint8_t, 20 > SymbolID
void visitStrings(Symbol &S, const Callback &CB)
const SymbolIndex * Index