Add pre-optimization for multi-versioning

This commit introduces a new pre-optimization step aimed at multi-versioning support. It inserts an internal filter into the `QueryGraph`.

The newly added filter checks whether the `start_time` of the `Transaction` within the `QueryGraph` falls within the timestamp range defined by the internal attributes `$ts_begin` and `$ts_end`.

With this inclusion, the implementation of append-only multi-versioning in mutable is complete.

Author: JonasLauermann

include/mutable/IR/Optimizer.hpp

@@ -34,23 +34,23 @@ struct M_EXPORT Optimizer
     auto & cost_function() const { return cf_; }
 
     /** Apply this optimizer to the given query graph to compute an operator tree. */
-    std::unique_ptr<Producer> operator()(const QueryGraph &G) const { return optimize(G).first; }
+    std::unique_ptr<Producer> operator()(QueryGraph &G) const { return optimize(G).first; }
 
     /** Computes and constructs an optimal plan for the given query graph.  Delegates to `optimize_recursive()`, then
      * assigns IDs to the optimal plan in post-order. */
     std::pair<std::unique_ptr<Producer>, PlanTableEntry>
-    optimize(const QueryGraph &G) const;
+    optimize(QueryGraph &G) const;
 
     /** Recursively computes and constructs an optimal plan for the given query graph.  Selects a `PlanTable*` type to
      * represent the internal state of planning progress, then delegates to `optimize_with_plantable<>()`. */
     std::pair<std::unique_ptr<Producer>, PlanTableEntry>
-    optimize_recursive(const QueryGraph &G) const;
+    optimize_recursive(QueryGraph &G) const;
 
     /** Recursively computes and constructs an optimal plan for the given query graph, using the given `PlanTable` type
      * to represent the state of planning progress. */
     template<typename PlanTable>
     std::pair<std::unique_ptr<Producer>, PlanTable>
-    optimize_with_plantable(const QueryGraph &G) const;
+    optimize_with_plantable(QueryGraph &G) const;
 
     private:
     /** Optimizes a plan table after initialization of the data source entries. */

include/mutable/IR/QueryGraph.hpp

@@ -190,6 +190,8 @@ struct M_EXPORT QueryGraph
     mutable std::unique_ptr<AdjacencyMatrix> adjacency_matrix_;
 
     Scheduler::Transaction *t_ = nullptr; ///< the transaction this query graph belongs to
+    ///> Stores the expressions of custom filters that have been added to the `DataSource`s after semantic analysis.
+    std::vector<std::unique_ptr<ast::Expr>> custom_filter_exprs_;
 
     public:
     friend void swap(QueryGraph &first, QueryGraph &second) {
@@ -296,6 +298,18 @@ struct M_EXPORT QueryGraph
     /** Returns the transaction ID. */
     Scheduler::Transaction * transaction() const { return t_; }
 
+    /** Creates a `cnf::CNF` from `filter_expr` and adds it to the current filter of the given `DataSource` `ds` by logical conjunction. */
+    void add_custom_filter(std::unique_ptr<ast::Expr> filter_expr, DataSource &ds) {
+        M_insist(std::find_if(sources_.begin(), sources_.end(), [&](std::unique_ptr<DataSource> &source){
+            return *source == ds;
+        }) != sources_.end());
+
+        auto filter = cnf::to_CNF(*filter_expr);
+        ds.update_filter(filter);
+        custom_filter_exprs_.push_back(std::move(filter_expr));
+    }
+
+
     void dump(std::ostream &out) const;
     void dump() const;
 

include/mutable/catalog/Catalog.hpp

@@ -327,7 +327,7 @@ struct M_EXPORT Catalog
     using TableFactoryDecoratorCallback = std::function<std::unique_ptr<TableFactory>(std::unique_ptr<TableFactory>)>;
     ComponentSet<TableFactoryDecoratorCallback> table_properties_; // stores callback functions that decorate a table with the given decorator
 
-    using PreOptimizationCallback = std::function<void(const QueryGraph &)>;
+    using PreOptimizationCallback = std::function<void(QueryGraph &)>;
     ComponentSet<PreOptimizationCallback> pre_optimizations_;
 
     using PostOptimizationCallback = std::function<std::unique_ptr<Producer>(std::unique_ptr<Producer>)>;

include/mutable/parse/AST.hpp

@@ -56,6 +56,9 @@ struct M_EXPORT Expr
 
     /** Returns the `Type` of this `Expr`.  Assumes that the `Expr` has been assigned a `Type` by the `Sema`. */
     const Type * type() const { return M_notnull(type_); }
+    /** Sets the `Type` of this `Expr`.  Should be used when constructing an `Expr` outside of the `Sema`.
+     * Should not be called to replace an already assigned type. */
+    void type(const Type *type) { M_insist(not type_); type_ = type; }
     /** Returns true iff this `Expr` has been assigned a `Type`, most likely by `Sema`. */
     bool has_type() const { return type_ != nullptr; }
 

src/IR/Optimizer.cpp

@@ -114,13 +114,13 @@ std::vector<cnf::CNF> optimize_filter(cnf::CNF filter)
  *====================================================================================================================*/
 
 std::pair<std::unique_ptr<Producer>, PlanTableEntry>
-Optimizer::optimize(const QueryGraph &G) const
+Optimizer::optimize(QueryGraph &G) const
 {
     return optimize_recursive(G);
 }
 
 std::pair<std::unique_ptr<Producer>, PlanTableEntry>
-Optimizer::optimize_recursive(const QueryGraph &G) const
+Optimizer::optimize_recursive(QueryGraph &G) const
 {
     switch (Options::Get().plan_table_type)
     {
@@ -151,7 +151,7 @@ Optimizer::optimize_recursive(const QueryGraph &G) const
 
 template<typename PlanTable>
 std::pair<std::unique_ptr<Producer>, PlanTable>
-Optimizer::optimize_with_plantable(const QueryGraph &G) const
+Optimizer::optimize_with_plantable(QueryGraph &G) const
 {
     PlanTable plan_table(G);
     const auto num_sources = G.sources().size();
@@ -498,7 +498,7 @@ Optimizer::compute_projections_required_for_order_by(const std::vector<projectio
 #define DEFINE(PLANTABLE) \
 template \
 std::pair<std::unique_ptr<Producer>, PLANTABLE> \
-Optimizer::optimize_with_plantable(const QueryGraph &G) const; \
+Optimizer::optimize_with_plantable(QueryGraph &G) const; \
 template \
 void \
 Optimizer::optimize_locally(const QueryGraph &G, PLANTABLE &plan_table) const; \

src/catalog/Schema.cpp

@@ -10,7 +10,9 @@
 #include <mutable/catalog/CostFunctionCout.hpp>
 #include <mutable/IR/Operator.hpp>
 #include <mutable/IR/PlanTable.hpp>
+#include <mutable/lex/Token.hpp>
 #include <mutable/Options.hpp>
+#include <mutable/parse/AST.hpp>
 #include <mutable/util/enum_ops.hpp>
 #include <mutable/util/fn.hpp>
 #include <stdexcept>
@@ -126,6 +128,7 @@ MultiVersioningTable::MultiVersioningTable(std::unique_ptr<Table> table)
     (*table_)[ts_begin].is_hidden = true;
     (*table_)[ts_begin].not_nullable = true;
     (*table_)[ts_end].is_hidden = true;
+    (*table_)[ts_end].not_nullable = true;
 }
 
 M_LCOV_EXCL_START
@@ -139,6 +142,137 @@ void MultiVersioningTable::dump() const { dump(std::cerr); }
 M_LCOV_EXCL_STOP
 
 
+namespace {
+
+
+void apply_timestamp_filter(QueryGraph &G)
+{
+    Catalog &C = Catalog::Get();
+
+    auto pos = Position(nullptr);
+    ast::Token ts_begin(pos, C.pool("$ts_begin"), TK_IDENTIFIER);
+    ast::Token ts_end(pos, C.pool("$ts_end"), TK_IDENTIFIER);
+
+    for (auto &ds : G.sources()) {
+        if (auto bt = cast<const BaseTable>(ds.get())) {
+            /* Set timestamp filter */
+            auto it = std::find_if(bt->table().cbegin_hidden(),
+                                   bt->table().end_hidden(),
+                                   [&](const Attribute & attr) {
+                                       return attr.name == C.pool("$ts_begin");
+                                   });
+
+            if (it != bt->table().end_hidden()) {
+                ast::Token table_name(pos, bt->table().name(), TK_EOF);
+                /*----- Build AST -----*/
+                // $ts_begin
+                std::unique_ptr<ast::Expr> ts_begin_designator = std::make_unique<ast::Designator>(
+                        ts_begin,
+                        table_name,
+                        ts_begin,
+                        Type::Get_Integer(Type::TY_Vector, 8),
+                        &*it
+                );
+
+                // TST := transaction start time constant
+                std::unique_ptr<ast::Expr> ts_begin_transaction_constant = std::make_unique<ast::Constant>(ast::Token(
+                        pos,
+                        C.pool(std::to_string(G.transaction()->start_time()).c_str()),
+                        m::TK_DEC_INT
+                ));
+                ts_begin_transaction_constant->type(Type::Get_Integer(Type::TY_Vector, 8));
+
+                // $ts_begin <= TST
+                std::unique_ptr<ast::Expr> ts_begin_filter_clause = std::make_unique<ast::BinaryExpr>(
+                        ast::Token(pos, C.pool("<="), TK_LESS_EQUAL),
+                        std::move(ts_begin_designator),
+                        std::move(ts_begin_transaction_constant)
+                );
+                ts_begin_filter_clause->type(Type::Get_Boolean(Type::TY_Vector));
+
+                // $ts_end
+                std::unique_ptr<ast::Expr> ts_end_designator = std::make_unique<ast::Designator>(
+                        ts_end,
+                        table_name,
+                        ts_end,
+                        Type::Get_Integer(Type::TY_Vector, 8),
+                        &bt->table()[C.pool("$ts_end")]
+                );
+
+                // TST := transaction start time constant
+                std::unique_ptr<ast::Expr> ts_end_transaction_constant = std::make_unique<ast::Constant>(ast::Token(
+                        pos,
+                        C.pool(std::to_string(G.transaction()->start_time()).c_str()),
+                        m::TK_DEC_INT
+                ));
+                ts_end_transaction_constant->type(Type::Get_Integer(Type::TY_Vector, 8));
+
+                // $ts_end > TST
+                std::unique_ptr<ast::Expr> ts_end_greater_transaction_expr = std::make_unique<ast::BinaryExpr>(
+                        ast::Token(pos, C.pool(">"), TK_GREATER),
+                        std::move(ts_end_designator),
+                        std::move(ts_end_transaction_constant)
+                );
+                ts_end_greater_transaction_expr->type(Type::Get_Boolean(Type::TY_Vector));
+
+                // $ts_end
+                std::unique_ptr<ast::Expr> ts_end_designator_2 = std::make_unique<ast::Designator>(
+                        ts_end,
+                        table_name,
+                        ts_end,
+                        Type::Get_Integer(Type::TY_Vector, 8),
+                        &bt->table()[C.pool("$ts_end")]
+                );
+
+                // 1
+                std::unique_ptr<ast::Expr> zero_constant = std::make_unique<ast::Constant>(ast::Token(
+                        pos,
+                        C.pool("0"),
+                        m::TK_DEC_INT
+                ));
+                zero_constant->type(Type::Get_Integer(Type::TY_Vector, 8));
+
+                // $ts_end = 0
+                std::unique_ptr<ast::Expr> ts_end_eq_zero = std::make_unique<ast::BinaryExpr>(
+                        ast::Token(pos, C.pool("="), TK_EQUAL),
+                        std::move(ts_end_designator_2),
+                        std::move(zero_constant)
+                );
+                ts_end_eq_zero->type(Type::Get_Boolean(Type::TY_Vector));
+
+                // $ts_end > TST OR $ts_end = 0
+                std::unique_ptr<ast::Expr> ts_end_filter_clause = std::make_unique<ast::BinaryExpr>(
+                        ast::Token(pos, C.pool("OR"), TK_Or),
+                        std::move(ts_end_greater_transaction_expr),
+                        std::move(ts_end_eq_zero)
+                );
+                ts_end_filter_clause->type(Type::Get_Boolean(Type::TY_Vector));
+
+                // $ts_begin <= TST AND ($ts_end > TST OR $ts_end = 0)
+                std::unique_ptr<ast::Expr> filter_expr = std::make_unique<ast::BinaryExpr>(
+                        ast::Token(pos, C.pool("AND"), TK_And),
+                        std::move(ts_begin_filter_clause),
+                        std::move(ts_end_filter_clause)
+                );
+                filter_expr->type(Type::Get_Boolean(Type::TY_Vector));
+
+                G.add_custom_filter(std::move(filter_expr), *ds);
+            }
+        }
+    }
+}
+
+__attribute__((constructor(202)))
+void register_pre_optimization()
+{
+    Catalog &C = Catalog::Get();
+    C.register_pre_optimization("multi-versioning", apply_timestamp_filter, "adds timestamp filters to the QueryGraph");
+}
+
+
+}
+
+
 /*======================================================================================================================
  * Function
  *====================================================================================================================*/