test: add backing store tests

Author: edubart

src/cartesi-machine.lua

@@ -2068,7 +2068,7 @@ echo "
 end
 
 for _, r in ipairs(memory_range_replace) do
-    main_machine:replace_memory_range(r.start, r.length, r.shared, r.image_filename)
+    main_machine:replace_memory_range(r)
 end
 
 local function dump_config(what, whatdef, out, indent)

src/machine-address-ranges.cpp

@@ -218,8 +218,9 @@ static void prepare_ar_backing_store_for_share(const backing_store_config &from_
             os::truncate_file(to_c.dpt_filename, dpt_length, true);
         }
         remover.add_file(to_c.dpt_filename);
+
+        os::change_writable(to_c.data_filename, !read_only);
     }
-    os::change_writable(to_c.data_filename, !read_only);
 }
 
 static void prepare_ar_backing_stores(const machine_config &c, scope_remove &remover) {

src/machine-config.cpp

@@ -55,11 +55,11 @@ std::string machine_config::get_dpt_filename(const std::string &dir, uint64_t st
 }
 
 std::string machine_config::get_sht_filename(const std::string &dir) {
-    return dir + "/global.sht";
+    return dir + "/hash_tree.sht";
 }
 
 std::string machine_config::get_phtc_filename(const std::string &dir) {
-    return dir + "/global.phtc";
+    return dir + "/hash_tree.phtc";
 }
 
 std::string machine_config::get_config_filename(const std::string &dir) {

src/machine-config.h

@@ -210,6 +210,8 @@ struct machine_config final {
     machine_config &adjust_defaults();
 
     /// \brief Adjust backing stores to point to the directory
+    /// \param dir Directory where "config" is stored
+    /// \param sharing Sharing mode conversion policy to use for backing stores
     machine_config &adjust_backing_stores(const std::string &dir, sharing_mode sharing = sharing_mode::config);
 
     /// \brief Loads a machine config from a directory

src/machine.cpp

@@ -395,7 +395,7 @@ void machine::store(const std::string &dir, sharing_mode sharing) const {
     remover.add_directory(dir);
 
     // Store config
-    remover.add_file(m_c.store(dir, sharing));
+    remover.add_file(m_c.store(dir));
 
     // Store all address ranges
     auto store_address_range = [&](const backing_store_config &c, const address_range &ar, bool read_only) {
@@ -418,10 +418,10 @@ void machine::store(const std::string &dir, sharing_mode sharing) const {
             remover.add_file(dpt_filename);
         } else { // Copy unshared backing store
             if (c.data_filename.empty()) {
-                throw std::runtime_error{"attempt to restore unbacked address range "s.append(ar.get_description())};
+                throw std::runtime_error{"attempt to rollback unbacked address range "s.append(ar.get_description())};
             }
             if (c.shared) {
-                throw std::runtime_error{"attempt to restore shared address range "s.append(ar.get_description())};
+                throw std::runtime_error{"attempt to rollback shared address range "s.append(ar.get_description())};
             }
             os::copy_file(c.data_filename, data_filename, ar.get_length());
             remover.add_file(data_filename);
@@ -462,8 +462,8 @@ void machine::store(const std::string &dir, sharing_mode sharing) const {
     } else {
         os::truncate_file(sht_filename, m_ht.get_sht_storage_data().size(), true);
         remover.add_file(sht_filename);
-        os::truncate_file(sht_filename, m_ht.get_phtc_storage_data().size(), true);
-        remover.add_file(sht_filename);
+        os::truncate_file(phtc_filename, m_ht.get_phtc_storage_data().size(), true);
+        remover.add_file(phtc_filename);
     }
 
     // Retain all stored files

src/os-mapped-memory.cpp

@@ -144,6 +144,11 @@ mapped_memory::mapped_memory(uint64_t length, const mapped_memory_flags &flags,
 
         // Check backing file length mismatch
         if (backing_file_length != desired_backing_length) {
+            if (backing_file_length > desired_backing_length) {
+                throw std::runtime_error{"backing file '"s + backing_filename + "' length ("s +
+                    std::to_string(backing_file_length) + ") cannot be less than desired backing length ("s +
+                    std::to_string(desired_backing_length) + ")"s};
+            }
             if (flags.shared || !flags.backing_gap) {
                 throw std::runtime_error{"backing file '"s + backing_filename + "' length ("s +
                     std::to_string(backing_file_length) + ") does not match desired backing length ("s +
@@ -214,7 +219,7 @@ mapped_memory::mapped_memory(uint64_t length, const mapped_memory_flags &flags,
             if (flags.read_only) {
                 // We can't write on read-only mappings.
                 // This kind of mapping is unlikely to happen, so there is no need to support it.
-                throw std::system_error{errno, std::generic_category(),
+                throw std::runtime_error{
                     "possible non zero partial page when mapping backing file '"s + backing_filename + "' to memory"s};
             }
             // NOLINTNEXTLINE(cppcoreguidelines-pro-type-reinterpret-cast)
@@ -304,6 +309,11 @@ mapped_memory::mapped_memory(uint64_t length, const mapped_memory_flags &flags,
 
         // Check backing file length mismatch
         if (backing_file_length != desired_backing_length) {
+            if (backing_file_length > desired_backing_length) {
+                throw std::runtime_error{"backing file '"s + backing_filename + "' length ("s +
+                    std::to_string(backing_file_length) + ") cannot be less than desired backing length ("s +
+                    std::to_string(desired_backing_length) + ")"s};
+            }
             if (flags.shared || !flags.backing_gap) {
                 throw std::runtime_error{"backing file '"s + backing_filename + "' length ("s +
                     std::to_string(backing_file_length) + ") does not match desired backing length ("s +
@@ -455,6 +465,11 @@ mapped_memory::mapped_memory(uint64_t length, const mapped_memory_flags &flags,
 
         // Check backing file length mismatch
         if (backing_file_length != desired_backing_length) {
+            if (backing_file_length > desired_backing_length) {
+                throw std::runtime_error{"backing file '"s + backing_filename + "' length ("s +
+                    std::to_string(backing_file_length) + ") cannot be less than desired backing length ("s +
+                    std::to_string(desired_backing_length) + ")"s};
+            }
             if (flags.shared || !flags.backing_gap) {
                 throw std::runtime_error{"backing file '"s + backing_filename + "' length ("s +
                     std::to_string(backing_file_length) + ") does not match desired backing length ("s +

tests/lua/cartesi/filesystem.lua

@@ -0,0 +1,78 @@
+-- Copyright Cartesi and individual authors (see AUTHORS)
+-- SPDX-License-Identifier: LGPL-3.0-or-later
+--
+-- This program is free software: you can redistribute it and/or modify it under
+-- the terms of the GNU Lesser General Public License as published by the Free
+-- Software Foundation, either version 3 of the License, or (at your option) any
+-- later version.
+--
+-- This program is distributed in the hope that it will be useful, but WITHOUT ANY
+-- WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A
+-- PARTICULAR PURPOSE. See the GNU Lesser General Public License for more details.
+--
+-- You should have received a copy of the GNU Lesser General Public License along
+-- with this program (see COPYING). If not, see <https://www.gnu.org/licenses/>.
+--
+
+local utils = require("cartesi.utils")
+
+-- Module providing helpers for filesystem operations.
+local filesystem = {}
+
+-- Creates a temporary filename (an empty file is also created).
+function filesystem.temp_filename()
+    return (assert(os.tmpname()))
+end
+
+-- Returns a temporary path name (without creating a file or directory).
+function filesystem.temp_pathname()
+    local pathname = assert(os.tmpname())
+    assert(os.remove(pathname))
+    return pathname
+end
+
+-- Reads binary data from a file.
+function filesystem.read_file(filename)
+    local f <close> = assert(io.open(filename, "rb"))
+    return (assert(f:read("a")))
+end
+
+-- Writes a binary data to a new file (the file is overwritten if it exists).
+function filesystem.write_file(filename, data)
+    local f <close> = assert(io.open(filename, "wb"))
+    assert(f:write(data))
+end
+
+-- Returns the size of a file in bytes.
+function filesystem.get_file_size(filename)
+    local f <close> = assert(io.open(filename, "rb"))
+    return (assert(f:seek("end")))
+end
+
+-- Writes a binary data to a temporary file.
+function filesystem.write_temp_file(data)
+    local filename = filesystem.temp_filename()
+    local ok, err = pcall(function()
+        filesystem.write_file(filename, data)
+    end)
+    if not ok then
+        os.remove(filename)
+    end
+    assert(ok, err)
+    return filename
+end
+
+-- Writes a binary data to a temporary file that is auto removed when scoped ends.
+function filesystem.write_scope_temp_file(data)
+    local filename = filesystem.write_temp_file(data)
+    return utils.scope_exit(function()
+        filesystem.remove_file(filename)
+    end), filename
+end
+
+-- Removes a file.
+function filesystem.remove_file(filename)
+    assert(os.remove(filename))
+end
+
+return filesystem

tests/lua/cartesi/tabular.lua

@@ -14,40 +14,79 @@
 -- with this program (see COPYING). If not, see <https://www.gnu.org/licenses/>.
 --
 
--- module to convert array tables into key-value tables
-local M = {}
+-- Module with helpers for table manipulations.
+local tabular = {}
 
--- convert a table with indexed rows into a table with named rows. e.g.
+-- Convert a table with indexed rows into a table with named rows. e.g.
 -- {"foo", 999} -> {name = "foo", cycles = 999} with keys described by md
-local function expand_row(metadata, row)
+function tabular.expand_row(metadata, row)
     local expanded_row = {}
     for key, val in ipairs(metadata) do
         expanded_row[val] = row[key]
     end
     return expanded_row
 end
 
--- apply `expand_row` for each row of `t`. e.g.
+-- Apply `expand_row` for each row of `t`. e.g.
 -- {{ "foo", 999 }} -> {{name = "foo", cycles = 999}} with keys described by md
-M.expand = function(metadata, t)
+function tabular.expand(metadata, t)
     local expanded_t = {}
     for _, row in ipairs(t) do
-        expanded_t[#expanded_t + 1] = expand_row(metadata, row)
+        expanded_t[#expanded_t + 1] = tabular.expand_row(metadata, row)
     end
     return expanded_t
 end
 
-function M.clear(t)
+-- Clear all key-value pairs from table `t`.
+function tabular.clear(t)
     for k in pairs(t) do
         t[k] = nil
     end
 end
 
-function M.append(t, elems)
+-- Append all elements from `elems` to the end of table `t`.
+function tabular.append(t, elems)
     local n = #t
     for i = 1, #elems do
         t[n + i] = elems[i]
     end
 end
 
-return M
+local function deep_traverse_iter(t, path)
+    path = path or {}
+    for k, v in pairs(t) do
+        local current_path = {}
+        for i = 1, #path do
+            current_path[i] = path[i]
+        end
+        current_path[#current_path + 1] = k
+        if type(v) == "table" then
+            deep_traverse_iter(v, current_path)
+        else
+            coroutine.yield(current_path, k, v)
+        end
+    end
+end
+
+-- Iterator that recursively traverses all key-value pairs in table `t`,
+-- yielding the path to each value, its key, and the value itself.
+function tabular.deep_traverse(t)
+    return coroutine.wrap(function()
+        deep_traverse_iter(t)
+    end)
+end
+
+-- Recursively copy a table `t` and all its subtables.
+function tabular.deep_copy(t)
+    local copy = {}
+    for k, v in pairs(t) do
+        if type(v) == "table" then
+            copy[k] = tabular.deep_copy(v)
+        else
+            copy[k] = v
+        end
+    end
+    return copy
+end
+
+return tabular

tests/lua/cartesi/utils.lua

@@ -0,0 +1,26 @@
+-- Copyright Cartesi and individual authors (see AUTHORS)
+-- SPDX-License-Identifier: LGPL-3.0-or-later
+--
+-- This program is free software: you can redistribute it and/or modify it under
+-- the terms of the GNU Lesser General Public License as published by the Free
+-- Software Foundation, either version 3 of the License, or (at your option) any
+-- later version.
+--
+-- This program is distributed in the hope that it will be useful, but WITHOUT ANY
+-- WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A
+-- PARTICULAR PURPOSE. See the GNU Lesser General Public License for more details.
+--
+-- You should have received a copy of the GNU Lesser General Public License along
+-- with this program (see COPYING). If not, see <https://www.gnu.org/licenses/>.
+--
+
+-- Module providing utilities.
+local utils = {}
+
+-- Executes a callback when the scope exits.
+-- REMARKS: Use <close> on its returned value to make it behave deterministically.
+function utils.scope_exit(callback)
+    return setmetatable({}, { __gc = callback, __close = callback })
+end
+
+return utils

tests/lua/mcycle-overflow.lua

@@ -25,8 +25,10 @@ local MAX_UARCH_CYCLE = cartesi.UARCH_CYCLE_MAX
 local function build_machine()
     local config = {
         ram = {
-            image_filename = test_util.tests_path .. "mcycle_overflow.bin",
             length = 32 << 20,
+            backing_store = {
+                data_filename = test_util.tests_path .. "mcycle_overflow.bin",
+            },
         },
     }
     local machine = cartesi.machine(config)