18 #ifndef LLVM_CLANG_LIB_SERIALIZATION_MULTIONDISKHASHTABLE_H 19 #define LLVM_CLANG_LIB_SERIALIZATION_MULTIONDISKHASHTABLE_H 21 #include "llvm/ADT/DenseMap.h" 22 #include "llvm/ADT/DenseSet.h" 23 #include "llvm/ADT/PointerUnion.h" 24 #include "llvm/ADT/STLExtras.h" 25 #include "llvm/ADT/SmallVector.h" 26 #include "llvm/ADT/TinyPtrVector.h" 27 #include "llvm/ADT/iterator_range.h" 28 #include "llvm/Support/Endian.h" 29 #include "llvm/Support/EndianStream.h" 30 #include "llvm/Support/OnDiskHashTable.h" 31 #include "llvm/Support/raw_ostream.h" 37 namespace serialization {
56 template<
typename ReaderInfo,
typename WriterInfo>
61 using HashTable = llvm::OnDiskIterableChainedHashTable<Info>;
66 OnDiskTable(
file_type File,
unsigned NumBuckets,
unsigned NumEntries,
70 Table(NumBuckets, NumEntries, Buckets, Payload, Base, InfoObj) {}
74 std::vector<file_type> Files;
75 llvm::DenseMap<internal_key_type, data_type> Data;
78 using Table = llvm::PointerUnion<OnDiskTable *, MergedTable *>;
79 using TableVector = llvm::TinyPtrVector<void *>;
90 llvm::TinyPtrVector<file_type> PendingOverrides;
92 struct AsOnDiskTable {
93 using result_type = OnDiskTable *;
95 result_type operator()(
void *
P)
const {
96 return Table::getFromOpaqueValue(P).template get<OnDiskTable *>();
100 using table_iterator =
101 llvm::mapped_iterator<TableVector::iterator, AsOnDiskTable>;
102 using table_range = llvm::iterator_range<table_iterator>;
105 table_range tables() {
106 auto Begin = Tables.begin(),
End = Tables.end();
107 if (getMergedTable())
109 return llvm::make_range(llvm::map_iterator(
Begin, AsOnDiskTable()),
110 llvm::map_iterator(
End, AsOnDiskTable()));
113 MergedTable *getMergedTable()
const {
115 return Tables.empty() ? nullptr : Table::getFromOpaqueValue(*Tables.begin())
116 .
template dyn_cast<MergedTable*>();
121 for (
auto *T : tables())
123 if (
auto *M = getMergedTable())
128 void removeOverriddenTables() {
130 Files.insert(PendingOverrides.begin(), PendingOverrides.end());
132 auto ShouldRemove = [&Files](
void *T) ->
bool {
133 auto *ODT = Table::getFromOpaqueValue(T).template get<OnDiskTable *>();
134 bool Remove = Files.count(ODT->File);
139 Tables.erase(std::remove_if(tables().begin().getCurrent(), Tables.end(),
142 PendingOverrides.clear();
146 MergedTable *Merged = getMergedTable();
148 Merged =
new MergedTable;
152 for (
auto *ODT : tables()) {
153 auto &HT = ODT->Table;
154 Info &InfoObj = HT.getInfoObj();
156 for (
auto I = HT.data_begin(), E = HT.data_end(); I != E; ++I) {
157 auto *LocalPtr = I.getItem();
160 auto L = InfoObj.ReadKeyDataLength(LocalPtr);
163 InfoObj.ReadDataInto(Key, LocalPtr + L.first, L.second,
167 Merged->Files.push_back(ODT->File);
172 Tables.push_back(Table(Merged).getOpaqueValue());
179 : Tables(
std::move(O.Tables)),
180 PendingOverrides(
std::move(O.PendingOverrides)) {
188 Tables = std::move(O.Tables);
190 PendingOverrides = std::move(O.PendingOverrides);
198 using namespace llvm::support;
202 uint32_t BucketOffset = endian::readNext<uint32_t, little, unaligned>(Ptr);
205 uint32_t NumFiles = endian::readNext<uint32_t, little, unaligned>(Ptr);
209 OverriddenFiles.reserve(NumFiles);
210 for (; NumFiles != 0; --NumFiles)
211 OverriddenFiles.push_back(InfoObj.ReadFileRef(Ptr));
212 PendingOverrides.insert(PendingOverrides.end(), OverriddenFiles.begin(),
213 OverriddenFiles.end());
217 auto NumBucketsAndEntries =
218 OnDiskTable::HashTable::readNumBucketsAndEntries(Buckets);
221 Table NewTable =
new OnDiskTable(File, NumBucketsAndEntries.first,
222 NumBucketsAndEntries.second,
223 Buckets, Ptr, Data, std::move(InfoObj));
224 Tables.push_back(NewTable.getOpaqueValue());
231 if (!PendingOverrides.empty())
232 removeOverriddenTables();
234 if (Tables.size() >
static_cast<unsigned>(Info::MaxTables))
240 if (MergedTable *M = getMergedTable()) {
241 auto It = M->Data.find(Key);
242 if (It != M->Data.end())
248 for (
auto *ODT : tables()) {
249 auto &HT = ODT->Table;
250 auto It = HT.find_hashed(Key, KeyHash);
252 HT.getInfoObj().ReadDataInto(Key, It.getDataPtr(), It.getDataLen(),
265 if (!PendingOverrides.empty())
266 removeOverriddenTables();
268 if (MergedTable *M = getMergedTable()) {
269 for (
auto &KV : M->Data)
270 Info::MergeDataInto(KV.second, ResultBuilder);
273 for (
auto *ODT : tables()) {
274 auto &HT = ODT->Table;
275 Info &InfoObj = HT.getInfoObj();
276 for (
auto I = HT.data_begin(), E = HT.data_end(); I != E; ++I) {
277 auto *LocalPtr = I.getItem();
280 auto L = InfoObj.ReadKeyDataLength(LocalPtr);
282 InfoObj.ReadDataInto(Key, LocalPtr + L.first, L.second, ResultBuilder);
291 template<
typename ReaderInfo,
typename WriterInfo>
294 using Generator = llvm::OnDiskChainedHashTableGenerator<WriterInfo>;
301 void insert(
typename WriterInfo::key_type_ref Key,
302 typename WriterInfo::data_type_ref Data, WriterInfo &Info) {
303 Gen.insert(Key, Data, Info);
308 using namespace llvm::support;
310 llvm::raw_svector_ostream OutStream(Out);
314 endian::Writer Writer(OutStream, little);
317 Writer.write<uint32_t>(0);
319 if (
auto *Merged = Base ? Base->getMergedTable() :
nullptr) {
321 Writer.write<uint32_t>(Merged->Files.size());
322 for (
const auto &F : Merged->Files)
323 Info.EmitFileRef(OutStream, F);
326 for (
auto &KV : Merged->Data) {
327 if (!Gen.contains(KV.first, Info))
328 Gen.insert(KV.first, Info.ImportData(KV.second), Info);
331 Writer.write<uint32_t>(0);
336 uint32_t BucketOffset = Gen.Emit(OutStream, Info);
339 endian::write32le(Out.data(), BucketOffset);
346 #endif // LLVM_CLANG_LIB_SERIALIZATION_MULTIONDISKHASHTABLE_H typename clang::serialization::reader::ASTDeclContextNameLookupTrait ::data_type data_type
Writer for the on-disk hash table.
void insert(typename WriterInfo::key_type_ref Key, typename WriterInfo::data_type_ref Data, WriterInfo &Info)
data_type find(const external_key_type &EKey)
Find and read the lookup results for EKey.
A collection of on-disk hash tables, merged when relevant for performance.
MultiOnDiskHashTable()=default
const unsigned char * storage_type
A pointer to an on-disk representation of the hash table.
typename clang::serialization::reader::ASTDeclContextNameLookupTrait ::file_type file_type
A handle to a file, used when overriding tables.
unsigned ComputeHash(Selector Sel)
MultiOnDiskHashTable(MultiOnDiskHashTable &&O)
Dataflow Directional Tag Classes.
void emit(llvm::SmallVectorImpl< char > &Out, WriterInfo &Info, const BaseTable *Base)
MultiOnDiskHashTable & operator=(MultiOnDiskHashTable &&O)
typename clang::serialization::reader::ASTDeclContextNameLookupTrait ::internal_key_type internal_key_type
data_type findAll()
Read all the lookup results into a single value.
typename clang::serialization::reader::ASTDeclContextNameLookupTrait ::external_key_type external_key_type
void add(file_type File, storage_type Data, Info InfoObj=Info())
Add the table Data loaded from file File.
MultiOnDiskHashTableGenerator()
typename clang::serialization::reader::ASTDeclContextNameLookupTrait ::data_type_builder data_type_builder