g2px1/SharedStorage
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

moved to 128-bit keys

Browse Source
This commit is contained in:
g2px1 2025-04-20 18:35:14 +03:00
parent b6cefc8536
commit ff335ec251
6 changed files with 48 additions and 65 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
.gitignore vendored Normal file
View File

@ -0,0 +1,4 @@
cmake-build-debug
.idea
.DS_Store
build

4
utils/io/Compactor.cpp
View File

@ -27,7 +27,7 @@ namespace usub::utils
std::string merged_filename = "L1_" + std::to_string(std::time(nullptr)) + ".dat"; std::string merged_filename = "L1_" + std::to_string(std::time(nullptr)) + ".dat";
LFSkipList<std::string, std::string> merged(this->version_manager); LFSkipList<Hash128, std::string> merged(this->version_manager);
for (const auto& file : this->level0_files) for (const auto& file : this->level0_files)
{ {
read_sstable_with_mmap(merged, file); read_sstable_with_mmap(merged, file);
@ -49,7 +49,7 @@ namespace usub::utils
std::string merged_filename = "L2_" + std::to_string(std::time(nullptr)) + ".dat"; std::string merged_filename = "L2_" + std::to_string(std::time(nullptr)) + ".dat";
LFSkipList<std::string, std::string> merged(this->version_manager); LFSkipList<Hash128, std::string> merged(this->version_manager);
for (const auto& file : this->level1_files) for (const auto& file : this->level1_files)
{ {
read_sstable_with_mmap(merged, file); read_sstable_with_mmap(merged, file);

1
utils/io/Compactor.h
View File

@ -10,6 +10,7 @@
#include <string> #include <string>
#include "utils/datastructures/LFSkipList.h" #include "utils/datastructures/LFSkipList.h"
#include "utils/io/SSTableIO.h" #include "utils/io/SSTableIO.h"
#include "utils/hash/Hash128.h"
namespace usub::utils namespace usub::utils
{ {

80
utils/io/SSTableIO.h
View File

@ -15,6 +15,7 @@
#include <utility> #include <utility>
#include <string> #include <string>
#include "utils/hash/Hash128.h"
namespace usub::utils namespace usub::utils
{ {
@ -113,57 +114,40 @@ namespace usub::utils
template <typename SkipList> template <typename SkipList>
void write_sstable_with_index(const SkipList& memtable, const std::string& filename) void write_sstable_with_index(const SkipList& memtable, const std::string& filename)
{ {
int fd = ::open(filename.c_str(), O_CREAT | O_WRONLY | O_TRUNC, 0644); std::ofstream out(filename, std::ios::binary | std::ios::trunc);
if (fd < 0) if (!out.is_open())
{ {
throw std::runtime_error("Failed to open file: " + filename); throw std::runtime_error("Failed to open SSTable file");
} }
FILE* file = ::fdopen(fd, "wb"); std::vector<std::pair<Hash128, uint64_t>> index_entries;
if (!file)
{
::close(fd);
throw std::runtime_error("Failed to fdopen file: " + filename);
}
std::vector<std::pair<typename SkipList::key_type, uint64_t>> index_entries;
uint64_t current_offset = 0; uint64_t current_offset = 0;
memtable.for_each_raw([&](const auto& key, const auto& value, bool is_tombstone, uint64_t version) memtable.for_each_raw([&](const auto& key, const auto& value, bool is_tombstone, uint64_t version)
{ {
uint32_t key_len = key.size(); out.write(reinterpret_cast<const char*>(&key), sizeof(key));
uint32_t value_len = value.size();
uint8_t tombstone_flag = is_tombstone ? 1 : 0;
::fwrite(&key_len, sizeof(key_len), 1, file); uint32_t value_len = value.size();
::fwrite(key.data(), key_len, 1, file); out.write(reinterpret_cast<const char*>(&value_len), sizeof(value_len));
::fwrite(&value_len, sizeof(value_len), 1, file); out.write(value.data(), value_len);
::fwrite(value.data(), value_len, 1, file);
::fwrite(&tombstone_flag, sizeof(tombstone_flag), 1, file); uint8_t tombstone_flag = is_tombstone ? 1 : 0;
::fwrite(&version, sizeof(version), 1, file); out.write(reinterpret_cast<const char*>(&tombstone_flag), sizeof(tombstone_flag));
out.write(reinterpret_cast<const char*>(&version), sizeof(version));
index_entries.emplace_back(key, current_offset); index_entries.emplace_back(key, current_offset);
current_offset += sizeof(key) + sizeof(value_len) + value_len + sizeof(tombstone_flag) + sizeof(version);
current_offset += sizeof(key_len) + key_len + sizeof(value_len) + value_len + sizeof(tombstone_flag) +
sizeof(version);
}); });
uint64_t index_start_offset = current_offset; uint64_t index_offset = current_offset;
// Записываем индекс
for (const auto& [key, offset] : index_entries) for (const auto& [key, offset] : index_entries)
{ {
uint32_t key_len = key.size(); out.write(reinterpret_cast<const char*>(&key), sizeof(key));
::fwrite(&key_len, sizeof(key_len), 1, file); out.write(reinterpret_cast<const char*>(&offset), sizeof(offset));
::fwrite(key.data(), key_len, 1, file);
::fwrite(&offset, sizeof(offset), 1, file);
} }
::fwrite(&index_start_offset, sizeof(index_start_offset), 1, file); out.write(reinterpret_cast<const char*>(&index_offset), sizeof(index_offset));
::fflush(file);
::fsync(fd);
::fclose(file);
} }
template <typename SkipList> template <typename SkipList>
@ -342,18 +326,16 @@ namespace usub::utils
uint64_t index_offset = *reinterpret_cast<const uint64_t*>(end - sizeof(uint64_t)); uint64_t index_offset = *reinterpret_cast<const uint64_t*>(end - sizeof(uint64_t));
const char* index_ptr = ptr + index_offset; const char* index_ptr = ptr + index_offset;
std::vector<std::pair<std::string, uint64_t>> index_entries; std::vector<std::pair<usub::utils::Hash128, uint64_t>> index_entries;
while (index_ptr < end - sizeof(uint64_t)) while (index_ptr < end - sizeof(uint64_t))
{ {
uint32_t key_len = *reinterpret_cast<const uint32_t*>(index_ptr); usub::utils::Hash128 key;
index_ptr += sizeof(uint32_t); std::memcpy(&key, index_ptr, sizeof(key));
index_ptr += sizeof(key);
std::string key(key_len, '\0'); uint64_t offset;
std::memcpy(key.data(), index_ptr, key_len); std::memcpy(&offset, index_ptr, sizeof(offset));
index_ptr += key_len; index_ptr += sizeof(offset);
uint64_t offset = *reinterpret_cast<const uint64_t*>(index_ptr);
index_ptr += sizeof(uint64_t);
index_entries.emplace_back(key, offset); index_entries.emplace_back(key, offset);
} }
@ -362,12 +344,9 @@ namespace usub::utils
{ {
const char* record = ptr + offset; const char* record = ptr + offset;
uint32_t key_len = *reinterpret_cast<const uint32_t*>(record); usub::utils::Hash128 file_key;
record += sizeof(uint32_t); std::memcpy(&file_key, record, sizeof(file_key));
record += sizeof(file_key);
std::string file_key(key_len, '\0');
std::memcpy(file_key.data(), record, key_len);
record += key_len;
uint32_t value_len = *reinterpret_cast<const uint32_t*>(record); uint32_t value_len = *reinterpret_cast<const uint32_t*>(record);
record += sizeof(uint32_t); record += sizeof(uint32_t);
@ -385,10 +364,11 @@ namespace usub::utils
memtable.insert_raw(file_key, value, tombstone_flag == 1, version); memtable.insert_raw(file_key, value, tombstone_flag == 1, version);
} }
munmap(data, st.st_size); munmap((void*)ptr, st.st_size);
::close(fd); ::close(fd);
} }
template <typename SkipList, typename Callback> template <typename SkipList, typename Callback>
void optimized_range_query_sstable(const std::string& filename, void optimized_range_query_sstable(const std::string& filename,
const typename SkipList::key_type& from_key, const typename SkipList::key_type& from_key,

18
utils/io/Wal.cpp
View File

@ -12,26 +12,22 @@ namespace usub::utils
if (!this->out.is_open()) throw std::runtime_error("Failed to open WAL"); if (!this->out.is_open()) throw std::runtime_error("Failed to open WAL");
} }
void WAL::write_put(const std::string& key, const std::string& value) void WAL::write_put(const Hash128& key, const std::string& value)
{ {
uint8_t op = 0; uint8_t op = 0;
uint32_t key_len = key.size();
uint32_t value_len = value.size(); uint32_t value_len = value.size();
this->out.write(reinterpret_cast<char*>(&op), sizeof(op)); this->out.write(reinterpret_cast<const char*>(&op), sizeof(op));
this->out.write(reinterpret_cast<char*>(&key_len), sizeof(key_len)); this->out.write(reinterpret_cast<const char*>(&key), sizeof(key));
this->out.write(key.data(), key_len); this->out.write(reinterpret_cast<const char*>(&value_len), sizeof(value_len));
this->out.write(reinterpret_cast<char*>(&value_len), sizeof(value_len));
this->out.write(value.data(), value_len); this->out.write(value.data(), value_len);
this->out.flush(); this->out.flush();
} }
void WAL::write_delete(const std::string& key) void WAL::write_delete(const Hash128& key)
{ {
uint8_t op = 1; uint8_t op = 1;
uint32_t key_len = key.size(); this->out.write(reinterpret_cast<const char*>(&op), sizeof(op));
this->out.write(reinterpret_cast<char*>(&op), sizeof(op)); this->out.write(reinterpret_cast<const char*>(&key), sizeof(key));
this->out.write(reinterpret_cast<char*>(&key_len), sizeof(key_len));
this->out.write(key.data(), key_len);
this->out.flush(); this->out.flush();
} }

6
utils/io/Wal.h
View File

@ -7,6 +7,8 @@
#include <fstream> #include <fstream>
#include "utils/hash/Hash128.h"
namespace usub::utils namespace usub::utils
{ {
class WAL class WAL
@ -16,9 +18,9 @@ namespace usub::utils
public: public:
explicit WAL(const std::string& filename); explicit WAL(const std::string& filename);
void write_put(const std::string& key, const std::string& value); void write_put(const Hash128& key, const std::string& value);
void write_delete(const std::string& key); void write_delete(const Hash128& key);
void close(); void close();
}; };