Skip to content

Commit 8d9b526

Browse files
committed
feat: persist hash tree state to disk
1 parent 3a6f209 commit 8d9b526

15 files changed

Lines changed: 448 additions & 116 deletions

src/cartesi-machine.lua

Lines changed: 19 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -106,6 +106,7 @@ where options are:
106106
length:<number>
107107
data_filename:<filename>
108108
dht_filename:<filename>
109+
dpt_filename:<filename>
109110
shared
110111
create
111112
truncate
@@ -138,6 +139,10 @@ where options are:
138139
range for the drive, down to one hash per page.)
139140
when omitted or set to the empty, the hash tree will be built from scratch.
140141
142+
dpt_filename (optional)
143+
gives the name of the file containing the dirty page tree for the flash drive.
144+
when omitted or set to the empty, the dirty page tree will be built from scratch.
145+
141146
shared (optional)
142147
target modifications to flash drive modify the memory and hash tree files.
143148
by default, image files are not modified and changes are lost.
@@ -185,6 +190,7 @@ where options are:
185190
length:<number>
186191
data_filename:<filename>
187192
dht_filename:<filename>
193+
dpt_filename:<filename>
188194
shared
189195
190196
semantics are the same as for the --flash-drive option with the following
@@ -204,6 +210,7 @@ where options are:
204210
<key>:<value> is one of
205211
data_filename:<filename>
206212
dht_filename:<filename>
213+
dpt_filename:<filename>
207214
shared
208215
create
209216
truncate
@@ -681,6 +688,7 @@ local default_config = cartesi.machine:get_default_config()
681688
local images_path = adjust_images_path(os.getenv("CARTESI_IMAGES_PATH"))
682689
local flash_data_filename = { root = images_path .. "rootfs.ext2" }
683690
local flash_dht_filename = {}
691+
local flash_dpt_filename = {}
684692
local flash_label_order = { "root" }
685693
local flash_shared = {}
686694
local flash_create = {}
@@ -703,6 +711,7 @@ local ram = {
703711
backing_store = {
704712
data_filename = images_path .. "linux.bin",
705713
dht_filename = "",
714+
dpt_filename = "",
706715
},
707716
}
708717
local init_splash = true
@@ -736,12 +745,14 @@ local uarch = {
736745
backing_store = {
737746
data_filename = "",
738747
dht_filename = "",
748+
dpt_filename = "",
739749
},
740750
},
741751
ram = {
742752
backing_store = {
743753
data_filename = "",
744754
dht_filename = "",
755+
dpt_filename = "",
745756
},
746757
},
747758
}
@@ -790,6 +801,7 @@ local function parse_memory_range(opts, all)
790801
local f = util.parse_options(opts, all, {
791802
data_filename = "string",
792803
dht_filename = "string",
804+
dpt_filename = "string",
793805
shared = "boolean",
794806
create = "boolean",
795807
truncate = "boolean",
@@ -799,12 +811,14 @@ local function parse_memory_range(opts, all)
799811
f.backing_store = {
800812
data_filename = f.data_filename or "",
801813
dht_filename = f.dht_filename or "",
814+
dpt_filename = f.dpt_filename or "",
802815
shared = f.shared,
803816
create = f.create,
804817
truncate = f.truncate,
805818
}
806819
f.data_filename = nil
807820
f.dht_filename = nil
821+
f.dpt_filename = nil
808822
f.shared = nil
809823
f.create = nil
810824
f.truncate = nil
@@ -815,6 +829,7 @@ local function parse_backing_store(opts, all, def)
815829
local f = util.parse_options(opts, all, {
816830
data_filename = "string",
817831
dht_filename = "string",
832+
dpt_filename = "string",
818833
shared = "boolean",
819834
create = "boolean",
820835
truncate = "boolean",
@@ -1241,6 +1256,7 @@ local options = {
12411256
label = "string",
12421257
data_filename = "string",
12431258
dht_filename = "string",
1259+
dpt_filename = "string",
12441260
shared = "boolean",
12451261
create = "boolean",
12461262
truncate = "boolean",
@@ -1254,6 +1270,7 @@ local options = {
12541270
if f.label == nil then f.label = "drive" .. #flash_label_order end
12551271
f.data_filename = f.data_filename or ""
12561272
f.dht_filename = f.dht_filename or ""
1273+
f.dpt_filename = f.dpt_filename or ""
12571274
if f.mke2fs == nil then f.mke2fs = f.data_filename == "" end
12581275
if f.mount == nil then
12591276
-- mount only if there is a file backing
@@ -1274,6 +1291,7 @@ local options = {
12741291
end
12751292
flash_data_filename[d] = f.data_filename or flash_data_filename[d]
12761293
flash_dht_filename[d] = f.dht_filename or flash_dht_filename[d]
1294+
flash_dpt_filename[d] = f.dpt_filename or flash_dpt_filename[d]
12771295
flash_start[d] = f.start or flash_start[d]
12781296
flash_length[d] = f.length or flash_length[d]
12791297
flash_shared[d] = f.shared or flash_shared[d]
@@ -1999,6 +2017,7 @@ echo "
19992017
backing_store = {
20002018
data_filename = flash_data_filename[label],
20012019
dht_filename = flash_dht_filename[label],
2020+
dpt_filename = flash_dpt_filename[label],
20022021
shared = flash_shared[label],
20032022
create = flash_create[label],
20042023
truncate = flash_truncate[label],

src/dense-hash-tree.h

Lines changed: 24 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -19,19 +19,20 @@
1919

2020
#include <cstdint>
2121
#include <limits>
22+
#include <span>
2223
#include <stdexcept>
23-
#include <vector>
2424

2525
#include "address-range-constants.h"
2626
#include "hash-tree-constants.h"
2727
#include "i-dense-hash-tree.h"
2828
#include "machine-hash.h"
29+
#include "os-mapped-memory.h"
2930

3031
namespace cartesi {
3132

3233
class dense_hash_tree final : public i_dense_hash_tree {
3334

34-
using container_type = std::vector<machine_hash>;
35+
using container_type = std::span<machine_hash>;
3536
using size_type = container_type::size_type;
3637

3738
static constexpr auto invalid_index = std::numeric_limits<size_type>::max();
@@ -47,7 +48,7 @@ class dense_hash_tree final : public i_dense_hash_tree {
4748
throw std::invalid_argument{"too many leaves for level count"};
4849
}
4950
// Make sure we can allocate a vector of size 1 << level_count
50-
if ((container_type{}.max_size() >> level_count) == 0) {
51+
if ((std::numeric_limits<container_type::size_type>::max() >> level_count) == 0) {
5152
throw std::invalid_argument{"too many levels"};
5253
}
5354
return level_count;
@@ -57,9 +58,12 @@ class dense_hash_tree final : public i_dense_hash_tree {
5758
static constexpr int m_log2_word_size = HASH_TREE_LOG2_WORD_SIZE;
5859

5960
public:
60-
dense_hash_tree(int level_count, size_type leaf_count) :
61+
dense_hash_tree(int level_count, size_type leaf_count, const std::string &backing_filename, bool shared) :
6162
m_level_count{check_level_count(level_count, leaf_count)},
62-
m_storage{size_type{1} << level_count} {}
63+
m_mapped_memory{get_storage_length(level_count, leaf_count), os::mapped_memory_flags{.shared = shared},
64+
backing_filename},
65+
// NOLINTNEXTLINE(cppcoreguidelines-pro-type-reinterpret-cast)
66+
m_tree{reinterpret_cast<machine_hash *>(m_mapped_memory.get_ptr()), size_type{1} << level_count} {}
6367

6468
dense_hash_tree(const dense_hash_tree &) = delete;
6569
dense_hash_tree &operator=(const dense_hash_tree &) = delete;
@@ -68,28 +72,36 @@ class dense_hash_tree final : public i_dense_hash_tree {
6872

6973
~dense_hash_tree() override = default;
7074

75+
static uint64_t get_storage_length(int level_count, size_type leaf_count) {
76+
return (UINT64_C(1) << check_level_count(level_count, leaf_count)) * sizeof(machine_hash);
77+
}
78+
79+
private:
7180
machine_hash_view do_node_hash_view(uint64_t offset, int log2_size) noexcept override {
7281
if (auto index = to_index(offset, log2_size); index != invalid_index) {
73-
return machine_hash_view{m_storage[index]};
82+
return machine_hash_view{m_tree[index]};
7483
}
7584
return no_hash_view();
7685
}
7786

7887
const_machine_hash_view do_node_hash_view(uint64_t offset, int log2_size) const noexcept override {
7988
if (auto index = to_index(offset, log2_size); index != invalid_index) {
80-
return const_machine_hash_view{m_storage[index]};
89+
return const_machine_hash_view{m_tree[index]};
8190
}
8291
return no_hash_view();
8392
}
8493

8594
const_machine_hash_view do_root_hash_view() const noexcept override {
8695
if (m_level_count != 0) {
87-
return m_storage[1];
96+
return m_tree[1];
8897
}
8998
return no_hash_view();
9099
}
91100

92-
private:
101+
std::span<const unsigned char> do_get_storage_data() const noexcept override {
102+
return m_mapped_memory.get_storage_data();
103+
}
104+
93105
/// \brief Converts a node size to its level in the tree
94106
/// \param log2_size Log<sub>2</sub> of node size
95107
/// \returns Corresponding level (which may be out of range)
@@ -110,9 +122,9 @@ class dense_hash_tree final : public i_dense_hash_tree {
110122
return start + index;
111123
}
112124

113-
int m_level_count;
114-
//??(edubart): convert to std::span over mapped memory
115-
std::vector<machine_hash> m_storage;
125+
int m_level_count; ///< Number of levels in tree
126+
os::mapped_memory m_mapped_memory; ///< Mapped memory for tree storage
127+
container_type m_tree; ///< Complete tree of hashes
116128
};
117129

118130
} // namespace cartesi

src/dirty-page-tree.h

Lines changed: 52 additions & 32 deletions
Original file line numberDiff line numberDiff line change
@@ -28,22 +28,20 @@
2828
#include <ranges>
2929
#include <span>
3030
#include <stdexcept>
31-
#include <vector>
3231

3332
#include "assert-printf.h"
3433
#include "i-dirty-page-tree.h"
34+
#include "os-mapped-memory.h"
3535
#include "ranges.h"
3636

3737
namespace cartesi {
3838

3939
/// \brief Dirty page tree
4040
class dirty_page_tree final : public i_dirty_page_tree {
41-
public:
4241
/// \brief Each node in tree is either clean or dirty
43-
enum class status_type : uint8_t { clean, dirty };
42+
enum class status_type : uint8_t { clean = 0, dirty = 1 };
4443

45-
private:
46-
using container_type = std::vector<status_type>;
44+
using container_type = std::span<status_type>;
4745

4846
/// \brief Checks if we can represent a tree with this many levels
4947
static int check_level_count(int level_count, size_type leaf_count) {
@@ -57,7 +55,7 @@ class dirty_page_tree final : public i_dirty_page_tree {
5755
throw std::invalid_argument{"too many leaves for level count"};
5856
}
5957
// Make sure we can allocate a vector of size 1 << level_count
60-
if ((container_type{}.max_size() >> level_count) == 0) {
58+
if ((std::numeric_limits<container_type::size_type>::max() >> level_count) == 0) {
6159
throw std::invalid_argument{"too many levels"};
6260
}
6361
// Make sure we can address position 1 << level_count
@@ -78,35 +76,43 @@ class dirty_page_tree final : public i_dirty_page_tree {
7876
}
7977
}
8078

79+
/// \brief Tells if the tree has been initialized
80+
/// \returns True if initialized, false otherwise
81+
bool is_initialized() const {
82+
// Use tree position 0 as initialization flag, if it's not clean, then the tree is initialized
83+
return m_tree[*position_iterator{0}] != status_type::clean;
84+
}
85+
86+
/// \brief Marks the tree as initialized
87+
void set_initialized() const {
88+
// Use tree position 0 as initialization flag, marking it as dirty makes the tree initialized
89+
m_tree[*position_iterator{0}] = status_type::dirty;
90+
}
91+
8192
public:
8293
/// \brief Constructor for leaves marked the same way
8394
/// \param leaf_count Number of leaves in tree. This is rounded up to the next power of 2.
8495
/// \param init Status of the first \p leaf_count leaves (the ones past leaf_count start clean)
85-
explicit dirty_page_tree(int level_count, size_type leaf_count, status_type init = status_type::dirty) :
96+
explicit dirty_page_tree(int level_count, size_type leaf_count, const std::string &backing_filename, bool shared,
97+
status_type init = status_type::dirty) :
8698
i_dirty_page_tree{check_level_count(level_count, leaf_count)},
8799
m_leaf_positions{level_positions_view(0)},
88100
m_valid_positions{position_iterator(1), position_iterator(position_iterator::value_type{1} << level_count)},
89-
m_tree{typename container_type::size_type{1} << level_count, status_type::clean} {
90-
const auto lp = m_leaf_positions;
91-
const auto first_leaf = *lp.begin();
92-
const auto first_pad = first_leaf + leaf_count;
93-
const auto pad_count = *lp.end() - first_pad;
94-
std::ranges::fill(std::span(m_tree).subspan(first_leaf, leaf_count), init);
95-
std::ranges::fill(std::span(m_tree).subspan(first_pad, pad_count), status_type::dirty);
96-
build_from_leaves();
97-
}
98-
99-
/// \brief Constructor from initializer list
100-
/// \param leaves Status of first few leaves in tree.
101-
/// \details This is a constructor mostly used in simple tests
102-
explicit dirty_page_tree(int level_count, std::initializer_list<status_type> leaves) :
103-
i_dirty_page_tree{check_level_count(level_count, leaves.size())},
104-
m_leaf_positions{level_positions_view(0)},
105-
m_valid_positions{position_iterator(1), position_iterator(position_iterator::value_type{1} << level_count)},
106-
m_tree{typename container_type::size_type{1} << level_count, status_type::clean} {
107-
const auto lp = m_leaf_positions;
108-
std::ranges::copy(leaves, &m_tree[*lp.begin()]);
109-
build_from_leaves();
101+
m_mapped_memory{get_storage_length(level_count, leaf_count), os::mapped_memory_flags{.shared = shared},
102+
backing_filename},
103+
// NOLINTNEXTLINE(cppcoreguidelines-pro-type-reinterpret-cast)
104+
m_tree{reinterpret_cast<status_type *>(m_mapped_memory.get_ptr()), size_type{1} << level_count} {
105+
if (!is_initialized()) {
106+
const auto lp = m_leaf_positions;
107+
const auto first_leaf = *lp.begin();
108+
const auto first_pad = first_leaf + leaf_count;
109+
const auto pad_count = *lp.end() - first_pad;
110+
std::ranges::fill(m_tree, status_type::clean);
111+
std::ranges::fill(m_tree.subspan(first_leaf, leaf_count), init);
112+
std::ranges::fill(m_tree.subspan(first_pad, pad_count), status_type::dirty);
113+
build_from_leaves();
114+
set_initialized();
115+
}
110116
}
111117

112118
// No copy or move constructors or assignments
@@ -118,6 +124,10 @@ class dirty_page_tree final : public i_dirty_page_tree {
118124
/// \brief Destructor
119125
~dirty_page_tree() override = default;
120126

127+
static uint64_t get_storage_length(int level_count, size_type leaf_count) {
128+
return (UINT64_C(1) << check_level_count(level_count, leaf_count)) * sizeof(status_type);
129+
}
130+
121131
// Dump tree in DOT format
122132
void dump() const {
123133
std::cout << "digraph HashTree {\n";
@@ -328,6 +338,11 @@ class dirty_page_tree final : public i_dirty_page_tree {
328338
return m_tree[*pos] == status_type::dirty;
329339
}
330340

341+
/// \brief Tells if the dirty page tree is clean
342+
bool do_is_clean() const noexcept override {
343+
return m_tree[*get_root_position()] == status_type::clean;
344+
}
345+
331346
/// \brief Clean entire tree
332347
void do_clean() noexcept override {
333348
for (auto pos : dirty_positions_view(leaf_positions_view())) {
@@ -336,14 +351,19 @@ class dirty_page_tree final : public i_dirty_page_tree {
336351
assert(m_tree[*get_root_position()] == status_type::clean);
337352
}
338353

354+
/// \brief Gets storage data
355+
std::span<const unsigned char> do_get_storage_data() const noexcept override {
356+
return m_mapped_memory.get_storage_data();
357+
}
358+
339359
// -----
340360
// Fields
341361
// -----
342362

343-
positions_range m_leaf_positions; // Bounds on leaf positions
344-
positions_range m_valid_positions; // Bounds on all positions
345-
//??(edubart): convert to std::span over mapped memory
346-
container_type m_tree; // Complete tree of flags
363+
positions_range m_leaf_positions; ///< Bounds on leaf positions
364+
positions_range m_valid_positions; ///< Bounds on all positions
365+
os::mapped_memory m_mapped_memory; ///< Mapped memory for tree storage
366+
container_type m_tree; ///< Complete tree of flags
347367
};
348368

349369
} // namespace cartesi

0 commit comments

Comments
 (0)