Add option for diff expire

The "expire" configuration for a geometry column in the table definition
now has an option "diff_expire", which is set to "false" by default but
can be set to "true" to get "diff expire", which means expire will be
based on the symmetrical difference between old and new geometry.

Note that "diff expire" will only be used for ways and relations that
change based on their members. If a way or relation is changed itself,
we always do a complete expire. This is needed, because we don't know
what changes there are (the tags or attributes could have changed, or
even the geometry type).

Internally this is implemented by not calculating expire tiles first for
the "delete" of the old geometry and then for the "add" of the new
geometry, but by writing old and new geometries for each object in a
cache and then, after we have collected all geometries, calculating the
symmetrical differences between those geometries.

Author: joto

src/expire-config.hpp

@@ -43,6 +43,9 @@ struct expire_config_t
     /// Expire mode.
     expire_mode mode = expire_mode::full_area;
 
+    /// Do expire based on symmetric difference of old and new geometry
+    bool diff_expire = false;
+
 }; // struct expire_config_t
 
 #endif // OSM2PGSQL_EXPIRE_CONFIG_HPP

src/flex-lua-table.cpp

@@ -310,6 +310,15 @@ void parse_and_set_expire_options(lua_State *lua_state,
         }
         lua_pop(lua_state, 1); // "buffer"
 
+        lua_getfield(lua_state, -1, "diff_expire");
+        if (lua_isboolean(lua_state, -1)) {
+            config.diff_expire = lua_toboolean(lua_state, -1);
+        } else if (!lua_isnil(lua_state, -1)) {
+            throw std::runtime_error{
+                "Optional expire field 'diff_expire' must contain a boolean."};
+        }
+        lua_pop(lua_state, 1); // "diff_expire"
+
         // Actually add the expire only if we are in append mode.
         if (append_mode) {
             column->add_expire(config);

src/flex-table-column.cpp

@@ -10,6 +10,8 @@
 #include "flex-table-column.hpp"
 
 #include "format.hpp"
+#include "geom-boost-adaptor.hpp"
+#include "overloaded.hpp"
 #include "pgsql-capabilities.hpp"
 #include "projection.hpp"
 #include "util.hpp"
@@ -208,18 +210,146 @@ void flex_table_column_t::add_expire(expire_config_t const &config)
     m_expires.push_back(config);
 }
 
+namespace {
+
+/**
+ * This expires all geometries in "geoms" by themselves. Used when we don't
+ * need diff expire.
+ */
+void separate_expire(std::vector<geom::geometry_t> const &geoms,
+                     expire_config_t const &expire_config,
+                     expire_tiles_t &expire_tiles,
+                     std::vector<expire_output_t> *expire_outputs)
+{
+    assert(expire_outputs);
+
+    for (auto const &geom : geoms) {
+        expire_tiles.from_geometry(geom, expire_config);
+    }
+    expire_tiles.commit_tiles(&expire_outputs->at(expire_config.expire_output));
+}
+
+/**
+ * When doing diff expire, we need to calculate the symmetric difference
+ * between old and new geometries. The difference is done by type, so points
+ * are compared with points, linestrings with linestrings, etc. This function
+ * separates out the input geometries by the three fundamental types.
+ */
+// NOLINTBEGIN(cppcoreguidelines-rvalue-reference-param-not-moved)
+template <typename T>
+void classify_geometries(T input_geoms, geom::multipoint_t *points,
+                         geom::multilinestring_t *linestrings,
+                         geom::multipolygon_t *polygons)
+{
+    assert(points);
+    assert(linestrings);
+    assert(polygons);
+
+    for (auto &&geom : *input_geoms) {
+        visit(overloaded{
+                  [&](geom::nullgeom_t && /*input*/) {},
+                  [&](geom::point_t &&input) { points->add_geometry(input); },
+                  [&](geom::linestring_t &&input) {
+                      linestrings->add_geometry(std::move(input));
+                  },
+                  [&](geom::polygon_t &&input) {
+                      polygons->add_geometry(std::move(input));
+                  },
+                  [&](geom::multipoint_t &&input) {
+                      for (auto &&point : input) {
+                          points->add_geometry(point);
+                      }
+                  },
+                  [&](geom::multilinestring_t &&input) {
+                      for (auto &&linestring : input) {
+                          linestrings->add_geometry(std::move(linestring));
+                      }
+                  },
+                  [&](geom::multipolygon_t &&input) {
+                      for (auto &&polygon : input) {
+                          polygons->add_geometry(std::move(polygon));
+                      }
+                  },
+                  [&](geom::collection_t &&input) {
+                      classify_geometries(&input, points, linestrings,
+                                          polygons);
+                  }},
+              std::move(geom));
+    }
+}
+// NOLINTEND(cppcoreguidelines-rvalue-reference-param-not-moved)
+
+template <typename T>
+void diff_and_expire(geom::multigeometry_t<T> const &old_geoms,
+                     geom::multigeometry_t<T> const &new_geoms,
+                     expire_config_t const &expire_config,
+                     expire_tiles_t &expire_tiles)
+{
+    std::vector<T> diffs;
+    boost::geometry::sym_difference(old_geoms, new_geoms, diffs);
+    for (auto const &geom : diffs) {
+        expire_tiles.from_geometry(geom, expire_config);
+    }
+}
+
+void diff_expire(std::vector<geom::geometry_t> *geoms_old,
+                 std::vector<geom::geometry_t> *geoms_new,
+                 expire_config_t const &expire_config,
+                 expire_tiles_t &expire_tiles,
+                 std::vector<expire_output_t> *expire_outputs)
+{
+    assert(geoms_old);
+    assert(geoms_new);
+    assert(expire_outputs);
+
+    geom::multipoint_t old_points;
+    geom::multilinestring_t old_linestrings;
+    geom::multipolygon_t old_polygons;
+
+    classify_geometries(geoms_old, &old_points, &old_linestrings,
+                        &old_polygons);
+
+    geom::multipoint_t new_points;
+    geom::multilinestring_t new_linestrings;
+    geom::multipolygon_t new_polygons;
+
+    classify_geometries(geoms_new, &new_points, &new_linestrings,
+                        &new_polygons);
+
+    diff_and_expire(old_points, new_points, expire_config, expire_tiles);
+    diff_and_expire(old_linestrings, new_linestrings, expire_config,
+                    expire_tiles);
+    diff_and_expire(old_polygons, new_polygons, expire_config, expire_tiles);
+
+    expire_tiles.commit_tiles(&expire_outputs->at(expire_config.expire_output));
+}
+
+} // anonymous namespace
+
 void flex_table_column_t::do_expire(
-    geom::geometry_t const &geom, std::vector<expire_tiles_t> *expire,
-    std::vector<expire_output_t> *expire_outputs) const
+    std::vector<geom::geometry_t> *geoms_old,
+    std::vector<geom::geometry_t> *geoms_new,
+    std::vector<expire_tiles_t> *expire,
+    std::vector<expire_output_t> *expire_outputs, bool enable_diff_expire) const
 {
+    assert(geoms_old);
+    assert(geoms_new);
     assert(expire);
     assert(expire_outputs);
 
     for (auto const &expire_config : m_expires) {
         assert(expire_config.expire_output < expire->size());
         auto &expire_tiles = expire->at(expire_config.expire_output);
-        expire_tiles.from_geometry(geom, expire_config);
-        expire_tiles.commit_tiles(
-            &expire_outputs->at(expire_config.expire_output));
+
+        if (!expire_config.diff_expire || !enable_diff_expire ||
+            geoms_old->empty() || geoms_new->empty()) {
+            separate_expire(*geoms_old, expire_config, expire_tiles,
+                            expire_outputs);
+            separate_expire(*geoms_new, expire_config, expire_tiles,
+                            expire_outputs);
+        } else {
+            diff_expire(geoms_old, geoms_new, expire_config, expire_tiles,
+                        expire_outputs);
+        }
     }
 }

src/flex-table-column.hpp

@@ -18,6 +18,7 @@
 #include <cassert>
 #include <cstdint>
 #include <string>
+#include <unordered_map>
 #include <vector>
 
 enum class table_column_type : uint8_t
@@ -54,6 +55,8 @@ enum class table_column_type : uint8_t
     id_num
 };
 
+class geometry_cache_t;
+
 /**
  * A column in a flex_table_t.
  */
@@ -134,9 +137,11 @@ class flex_table_column_t
         return m_expires;
     }
 
-    void do_expire(geom::geometry_t const &geom,
+    void do_expire(std::vector<geom::geometry_t> *geoms_old,
+                   std::vector<geom::geometry_t> *geoms_new,
                    std::vector<expire_tiles_t> *expire,
-                   std::vector<expire_output_t> *expire_outputs) const;
+                   std::vector<expire_output_t> *expire_outputs,
+                   bool enable_diff_expire) const;
 
 private:
     std::vector<expire_config_t> m_expires;
@@ -170,6 +175,40 @@ class flex_table_column_t
 
     /// Column will be created but not filled by osm2pgsql.
     bool m_create_only = false;
-};
+}; // class flex_table_column_t
+
+/**
+ * While processing an OSM object, this cache is used to hold all old and all
+ * new geometries stored in all geometry columns with expire config in a table.
+ * Later those geometries are used to calculate the expire.
+ */
+class geometry_cache_t
+{
+public:
+    template <typename GEOM>
+    void add_old(flex_table_column_t const *column, GEOM &&geom)
+    {
+        m_geometries[column].first.push_back(std::forward<GEOM>(geom));
+    }
+
+    template <typename GEOM>
+    void add_new(flex_table_column_t const *column, GEOM &&geom)
+    {
+        m_geometries[column].second.push_back(std::forward<GEOM>(geom));
+    }
+
+    auto begin() noexcept { return m_geometries.begin(); }
+
+    auto end() noexcept { return m_geometries.end(); }
+
+    void clear() { m_geometries.clear(); }
+
+private:
+    std::unordered_map<
+        flex_table_column_t const *,
+        std::pair<std::vector<geom::geometry_t>, std::vector<geom::geometry_t>>>
+        m_geometries;
+
+}; // class geometry_cache_t
 
 #endif // OSM2PGSQL_FLEX_TABLE_COLUMN_HPP

src/flex-write.cpp

@@ -8,6 +8,7 @@
  */
 
 #include "flex-lua-geom.hpp"
+#include "flex-table-column.hpp"
 #include "flex-write.hpp"
 #include "geom-functions.hpp"
 #include "json-writer.hpp"
@@ -257,11 +258,9 @@ bool is_compatible(geom::geometry_t const &geom,
 
 } // anonymous namespace
 
-void flex_write_column(lua_State *lua_state,
+void flex_write_column(lua_State *lua_state, geometry_cache_t *geom_cache,
                        db_copy_mgr_t<db_deleter_by_type_and_id_t> *copy_mgr,
-                       flex_table_column_t const &column,
-                       std::vector<expire_tiles_t> *expire,
-                       std::vector<expire_output_t> *expire_outputs)
+                       flex_table_column_t const &column)
 {
     lua_getfield(lua_state, -1, column.name().c_str());
     int const ltype = lua_type(lua_state, -1);
@@ -446,13 +445,13 @@ void flex_write_column(lua_State *lua_state,
                      type == table_column_type::multilinestring ||
                      type == table_column_type::multipolygon);
                 if (geom->srid() == column.srid()) {
-                    column.do_expire(*geom, expire, expire_outputs);
                     copy_mgr->add_hex_geom(geom_to_ewkb(*geom, wrap_multi));
+                    geom_cache->add_new(&column, *geom);
                 } else {
                     auto const &proj = get_projection(column.srid());
-                    auto const tgeom = geom::transform(*geom, proj);
-                    column.do_expire(tgeom, expire, expire_outputs);
+                    auto tgeom = geom::transform(*geom, proj);
                     copy_mgr->add_hex_geom(geom_to_ewkb(tgeom, wrap_multi));
+                    geom_cache->add_new(&column, std::move(tgeom));
                 }
             } else {
                 write_null(copy_mgr, column);

src/flex-write.hpp

@@ -19,6 +19,7 @@
 #include <vector>
 
 class expire_tiles_t;
+class geometry_cache_t;
 
 class not_null_exception_t : public std::runtime_error
 {
@@ -34,10 +35,8 @@ class not_null_exception_t : public std::runtime_error
     flex_table_column_t const *m_column;
 }; // class not_null_exception_t
 
-void flex_write_column(lua_State *lua_state,
+void flex_write_column(lua_State *lua_state, geometry_cache_t *geom_cache,
                        db_copy_mgr_t<db_deleter_by_type_and_id_t> *copy_mgr,
-                       flex_table_column_t const &column,
-                       std::vector<expire_tiles_t> *expire,
-                       std::vector<expire_output_t> *expire_outputs);
+                       flex_table_column_t const &column);
 
 #endif // OSM2PGSQL_FLEX_WRITE_HPP