g2px1/SharedStorage
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

moved to 128-bit keys

Browse Source
This commit is contained in:
g2px1 2025-04-20 18:35:14 +03:00
parent b6cefc8536
commit ff335ec251
6 changed files with 48 additions and 65 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
.gitignore vendored Normal file
View File

@ -0,0 +1,4 @@
cmake-build-debug
.idea
.DS_Store
build

4
utils/io/Compactor.cpp
View File

@ -27,7 +27,7 @@ namespace usub::utils
std::string merged_filename = "L1_" + std::to_string(std::time(nullptr)) + ".dat";
LFSkipList<std::string, std::string> merged(this->version_manager);
LFSkipList<Hash128, std::string> merged(this->version_manager);
for (const auto& file : this->level0_files)
{
read_sstable_with_mmap(merged, file);
@ -49,7 +49,7 @@ namespace usub::utils
std::string merged_filename = "L2_" + std::to_string(std::time(nullptr)) + ".dat";
LFSkipList<std::string, std::string> merged(this->version_manager);
LFSkipList<Hash128, std::string> merged(this->version_manager);
for (const auto& file : this->level1_files)
{
read_sstable_with_mmap(merged, file);

1
utils/io/Compactor.h
View File

@ -10,6 +10,7 @@
#include <string>
#include "utils/datastructures/LFSkipList.h"
#include "utils/io/SSTableIO.h"
#include "utils/hash/Hash128.h"
namespace usub::utils
{

80
utils/io/SSTableIO.h
View File

@ -15,6 +15,7 @@
#include <utility>
#include <string>
#include "utils/hash/Hash128.h"
namespace usub::utils
{
@ -113,57 +114,40 @@ namespace usub::utils
template <typename SkipList>
void write_sstable_with_index(const SkipList& memtable, const std::string& filename)
{
int fd = ::open(filename.c_str(), O_CREAT | O_WRONLY | O_TRUNC, 0644);
if (fd < 0)
std::ofstream out(filename, std::ios::binary | std::ios::trunc);
if (!out.is_open())
{
throw std::runtime_error("Failed to open file: " + filename);
throw std::runtime_error("Failed to open SSTable file");
}
FILE* file = ::fdopen(fd, "wb");
if (!file)
{
::close(fd);
throw std::runtime_error("Failed to fdopen file: " + filename);
}
std::vector<std::pair<typename SkipList::key_type, uint64_t>> index_entries;
std::vector<std::pair<Hash128, uint64_t>> index_entries;
uint64_t current_offset = 0;
memtable.for_each_raw([&](const auto& key, const auto& value, bool is_tombstone, uint64_t version)
{
uint32_t key_len = key.size();
uint32_t value_len = value.size();
uint8_t tombstone_flag = is_tombstone ? 1 : 0;
out.write(reinterpret_cast<const char*>(&key), sizeof(key));
::fwrite(&key_len, sizeof(key_len), 1, file);
::fwrite(key.data(), key_len, 1, file);
::fwrite(&value_len, sizeof(value_len), 1, file);
::fwrite(value.data(), value_len, 1, file);
::fwrite(&tombstone_flag, sizeof(tombstone_flag), 1, file);
::fwrite(&version, sizeof(version), 1, file);
uint32_t value_len = value.size();
out.write(reinterpret_cast<const char*>(&value_len), sizeof(value_len));
out.write(value.data(), value_len);
uint8_t tombstone_flag = is_tombstone ? 1 : 0;
out.write(reinterpret_cast<const char*>(&tombstone_flag), sizeof(tombstone_flag));
out.write(reinterpret_cast<const char*>(&version), sizeof(version));
index_entries.emplace_back(key, current_offset);
current_offset += sizeof(key_len) + key_len + sizeof(value_len) + value_len + sizeof(tombstone_flag) +
sizeof(version);
current_offset += sizeof(key) + sizeof(value_len) + value_len + sizeof(tombstone_flag) + sizeof(version);
});
uint64_t index_start_offset = current_offset;
uint64_t index_offset = current_offset;
// Записываем индекс
for (const auto& [key, offset] : index_entries)
{
uint32_t key_len = key.size();
::fwrite(&key_len, sizeof(key_len), 1, file);
::fwrite(key.data(), key_len, 1, file);
::fwrite(&offset, sizeof(offset), 1, file);
out.write(reinterpret_cast<const char*>(&key), sizeof(key));
out.write(reinterpret_cast<const char*>(&offset), sizeof(offset));
}
::fwrite(&index_start_offset, sizeof(index_start_offset), 1, file);
::fflush(file);
::fsync(fd);
::fclose(file);
out.write(reinterpret_cast<const char*>(&index_offset), sizeof(index_offset));
}
template <typename SkipList>
@ -342,18 +326,16 @@ namespace usub::utils
uint64_t index_offset = *reinterpret_cast<const uint64_t*>(end - sizeof(uint64_t));
const char* index_ptr = ptr + index_offset;
std::vector<std::pair<std::string, uint64_t>> index_entries;
std::vector<std::pair<usub::utils::Hash128, uint64_t>> index_entries;
while (index_ptr < end - sizeof(uint64_t))
{
uint32_t key_len = *reinterpret_cast<const uint32_t*>(index_ptr);
index_ptr += sizeof(uint32_t);
usub::utils::Hash128 key;
std::memcpy(&key, index_ptr, sizeof(key));
index_ptr += sizeof(key);
std::string key(key_len, '\0');
std::memcpy(key.data(), index_ptr, key_len);
index_ptr += key_len;
uint64_t offset = *reinterpret_cast<const uint64_t*>(index_ptr);
index_ptr += sizeof(uint64_t);
uint64_t offset;
std::memcpy(&offset, index_ptr, sizeof(offset));
index_ptr += sizeof(offset);
index_entries.emplace_back(key, offset);
}
@ -362,12 +344,9 @@ namespace usub::utils
{
const char* record = ptr + offset;
uint32_t key_len = *reinterpret_cast<const uint32_t*>(record);
record += sizeof(uint32_t);
std::string file_key(key_len, '\0');
std::memcpy(file_key.data(), record, key_len);
record += key_len;
usub::utils::Hash128 file_key;
std::memcpy(&file_key, record, sizeof(file_key));
record += sizeof(file_key);
uint32_t value_len = *reinterpret_cast<const uint32_t*>(record);
record += sizeof(uint32_t);
@ -385,10 +364,11 @@ namespace usub::utils
memtable.insert_raw(file_key, value, tombstone_flag == 1, version);
}
munmap(data, st.st_size);
munmap((void*)ptr, st.st_size);
::close(fd);
}
template <typename SkipList, typename Callback>
void optimized_range_query_sstable(const std::string& filename,
const typename SkipList::key_type& from_key,

18
utils/io/Wal.cpp
View File

@ -12,26 +12,22 @@ namespace usub::utils
if (!this->out.is_open()) throw std::runtime_error("Failed to open WAL");
}
void WAL::write_put(const std::string& key, const std::string& value)
void WAL::write_put(const Hash128& key, const std::string& value)
{
uint8_t op = 0;
uint32_t key_len = key.size();
uint32_t value_len = value.size();
this->out.write(reinterpret_cast<char*>(&op), sizeof(op));
this->out.write(reinterpret_cast<char*>(&key_len), sizeof(key_len));
this->out.write(key.data(), key_len);
this->out.write(reinterpret_cast<char*>(&value_len), sizeof(value_len));
this->out.write(reinterpret_cast<const char*>(&op), sizeof(op));
this->out.write(reinterpret_cast<const char*>(&key), sizeof(key));
this->out.write(reinterpret_cast<const char*>(&value_len), sizeof(value_len));
this->out.write(value.data(), value_len);
this->out.flush();
}
void WAL::write_delete(const std::string& key)
void WAL::write_delete(const Hash128& key)
{
uint8_t op = 1;
uint32_t key_len = key.size();
this->out.write(reinterpret_cast<char*>(&op), sizeof(op));
this->out.write(reinterpret_cast<char*>(&key_len), sizeof(key_len));
this->out.write(key.data(), key_len);
this->out.write(reinterpret_cast<const char*>(&op), sizeof(op));
this->out.write(reinterpret_cast<const char*>(&key), sizeof(key));
this->out.flush();
}

6
utils/io/Wal.h
View File

@ -7,6 +7,8 @@
#include <fstream>
#include "utils/hash/Hash128.h"
namespace usub::utils
{
class WAL
@ -16,9 +18,9 @@ namespace usub::utils
public:
explicit WAL(const std::string& filename);
void write_put(const std::string& key, const std::string& value);
void write_put(const Hash128& key, const std::string& value);
void write_delete(const std::string& key);
void write_delete(const Hash128& key);
void close();
};