clang-tools  8.0.0
BackgroundIndexStorage.cpp
Go to the documentation of this file.
1 //== BackgroundIndexStorage.cpp - Provide caching support to BackgroundIndex ==/
2 //
3 // The LLVM Compiler Infrastructure
4 //
5 // This file is distributed under the University of Illinois Open Source
6 // License. See LICENSE.TXT for details.
7 //
8 //===----------------------------------------------------------------------===//
9 
10 #include "Logger.h"
11 #include "index/Background.h"
12 #include "llvm/ADT/ScopeExit.h"
13 #include "llvm/Support/Error.h"
14 #include "llvm/Support/FileSystem.h"
15 #include "llvm/Support/MemoryBuffer.h"
16 #include "llvm/Support/Path.h"
17 #include "llvm/Support/SHA1.h"
18 
19 namespace clang {
20 namespace clangd {
21 namespace {
22 
23 using FileDigest = decltype(llvm::SHA1::hash({}));
24 
25 static FileDigest digest(StringRef Content) {
26  return llvm::SHA1::hash({(const uint8_t *)Content.data(), Content.size()});
27 }
28 
29 std::string getShardPathFromFilePath(llvm::StringRef ShardRoot,
30  llvm::StringRef FilePath) {
31  llvm::SmallString<128> ShardRootSS(ShardRoot);
32  llvm::sys::path::append(ShardRootSS, llvm::sys::path::filename(FilePath) +
33  "." + llvm::toHex(digest(FilePath)) +
34  ".idx");
35  return ShardRootSS.str();
36 }
37 
38 llvm::Error
39 writeAtomically(llvm::StringRef OutPath,
40  llvm::function_ref<void(llvm::raw_ostream &)> Writer) {
41  // Write to a temporary file first.
42  llvm::SmallString<128> TempPath;
43  int FD;
44  auto EC =
45  llvm::sys::fs::createUniqueFile(OutPath + ".tmp.%%%%%%%%", FD, TempPath);
46  if (EC)
47  return llvm::errorCodeToError(EC);
48  // Make sure temp file is destroyed on failure.
49  auto RemoveOnFail =
50  llvm::make_scope_exit([TempPath] { llvm::sys::fs::remove(TempPath); });
51  llvm::raw_fd_ostream OS(FD, /*shouldClose=*/true);
52  Writer(OS);
53  OS.close();
54  if (OS.has_error())
55  return llvm::errorCodeToError(OS.error());
56  // Then move to real location.
57  EC = llvm::sys::fs::rename(TempPath, OutPath);
58  if (EC)
59  return llvm::errorCodeToError(EC);
60  // If everything went well, we already moved the file to another name. So
61  // don't delete the file, as the name might be taken by another file.
62  RemoveOnFail.release();
63  return llvm::ErrorSuccess();
64 }
65 
66 // Uses disk as a storage for index shards. Creates a directory called
67 // ".clangd/index/" under the path provided during construction.
68 class DiskBackedIndexStorage : public BackgroundIndexStorage {
69  std::string DiskShardRoot;
70 
71 public:
72  // Sets DiskShardRoot to (Directory + ".clangd/index/") which is the base
73  // directory for all shard files.
74  DiskBackedIndexStorage(llvm::StringRef Directory) {
75  llvm::SmallString<128> CDBDirectory(Directory);
76  llvm::sys::path::append(CDBDirectory, ".clangd", "index");
77  DiskShardRoot = CDBDirectory.str();
78  std::error_code OK;
79  std::error_code EC = llvm::sys::fs::create_directories(DiskShardRoot);
80  if (EC != OK) {
81  elog("Failed to create directory {0} for index storage: {1}",
82  DiskShardRoot, EC.message());
83  }
84  }
85 
86  std::unique_ptr<IndexFileIn>
87  loadShard(llvm::StringRef ShardIdentifier) const override {
88  const std::string ShardPath =
89  getShardPathFromFilePath(DiskShardRoot, ShardIdentifier);
90  auto Buffer = llvm::MemoryBuffer::getFile(ShardPath);
91  if (!Buffer)
92  return nullptr;
93  if (auto I = readIndexFile(Buffer->get()->getBuffer()))
94  return llvm::make_unique<IndexFileIn>(std::move(*I));
95  else
96  elog("Error while reading shard {0}: {1}", ShardIdentifier,
97  I.takeError());
98  return nullptr;
99  }
100 
101  llvm::Error storeShard(llvm::StringRef ShardIdentifier,
102  IndexFileOut Shard) const override {
103  return writeAtomically(
104  getShardPathFromFilePath(DiskShardRoot, ShardIdentifier),
105  [&Shard](llvm::raw_ostream &OS) { OS << Shard; });
106  }
107 };
108 
109 // Doesn't persist index shards anywhere (used when the CDB dir is unknown).
110 // We could consider indexing into ~/.clangd/ or so instead.
111 class NullStorage : public BackgroundIndexStorage {
112 public:
113  std::unique_ptr<IndexFileIn>
114  loadShard(llvm::StringRef ShardIdentifier) const override {
115  return nullptr;
116  }
117 
118  llvm::Error storeShard(llvm::StringRef ShardIdentifier,
119  IndexFileOut Shard) const override {
120  vlog("Couldn't find project for {0}, indexing in-memory only",
121  ShardIdentifier);
122  return llvm::Error::success();
123  }
124 };
125 
126 // Creates and owns IndexStorages for multiple CDBs.
127 class DiskBackedIndexStorageManager {
128 public:
129  DiskBackedIndexStorageManager()
130  : IndexStorageMapMu(llvm::make_unique<std::mutex>()) {}
131 
132  // Creates or fetches to storage from cache for the specified CDB.
133  BackgroundIndexStorage *operator()(llvm::StringRef CDBDirectory) {
134  std::lock_guard<std::mutex> Lock(*IndexStorageMapMu);
135  auto &IndexStorage = IndexStorageMap[CDBDirectory];
136  if (!IndexStorage)
137  IndexStorage = create(CDBDirectory);
138  return IndexStorage.get();
139  }
140 
141  // Creates or fetches to storage from cache for the specified CDB.
142  BackgroundIndexStorage *createStorage(llvm::StringRef CDBDirectory);
143 
144 private:
145  std::unique_ptr<BackgroundIndexStorage> create(llvm::StringRef CDBDirectory) {
146  if (CDBDirectory.empty())
147  return llvm::make_unique<NullStorage>();
148  return llvm::make_unique<DiskBackedIndexStorage>(CDBDirectory);
149  }
150 
151  llvm::StringMap<std::unique_ptr<BackgroundIndexStorage>> IndexStorageMap;
152  std::unique_ptr<std::mutex> IndexStorageMapMu;
153 };
154 
155 } // namespace
156 
159  return DiskBackedIndexStorageManager();
160 }
161 
162 } // namespace clangd
163 } // namespace clang
Some operations such as code completion produce a set of candidates.
decltype(llvm::SHA1::hash({})) FileDigest
Definition: SourceCode.h:31
llvm::Expected< IndexFileIn > readIndexFile(llvm::StringRef Data)
void vlog(const char *Fmt, Ts &&... Vals)
Definition: Logger.h:68
void elog(const char *Fmt, Ts &&... Vals)
Definition: Logger.h:57
llvm::unique_function< BackgroundIndexStorage *(llvm::StringRef)> Factory
Definition: Background.h:55
static cl::opt< std::string > Directory(cl::Positional, cl::Required, cl::desc("<Search Root Directory>"))
FileDigest digest(llvm::StringRef Content)
Definition: SourceCode.cpp:240
===– Representation.cpp - ClangDoc Representation --------—*- C++ -*-===//