Merge pull request #2476 from joto/diff-expire

Add option for diff expire

Author: lonvia

src/expire-config.hpp

@@ -43,6 +43,9 @@ struct expire_config_t
     /// Expire mode.
     expire_mode mode = expire_mode::full_area;
 
+    /// Do expire based on symmetric difference of old and new geometry
+    bool diff_expire = false;
+
 }; // struct expire_config_t
 
 #endif // OSM2PGSQL_EXPIRE_CONFIG_HPP

src/flex-lua-table.cpp

@@ -310,6 +310,15 @@ void parse_and_set_expire_options(lua_State *lua_state,
         }
         lua_pop(lua_state, 1); // "buffer"
 
+        lua_getfield(lua_state, -1, "diff_expire");
+        if (lua_isboolean(lua_state, -1)) {
+            config.diff_expire = lua_toboolean(lua_state, -1);
+        } else if (!lua_isnil(lua_state, -1)) {
+            throw std::runtime_error{
+                "Optional expire field 'diff_expire' must contain a boolean."};
+        }
+        lua_pop(lua_state, 1); // "diff_expire"
+
         // Actually add the expire only if we are in append mode.
         if (append_mode) {
             column->add_expire(config);

src/flex-table-column.cpp

@@ -10,6 +10,8 @@
 #include "flex-table-column.hpp"
 
 #include "format.hpp"
+#include "geom-boost-adaptor.hpp"
+#include "overloaded.hpp"
 #include "pgsql-capabilities.hpp"
 #include "projection.hpp"
 #include "util.hpp"
@@ -208,18 +210,146 @@ void flex_table_column_t::add_expire(expire_config_t const &config)
     m_expires.push_back(config);
 }
 
+namespace {
+
+/**
+ * This expires all geometries in "geoms" by themselves. Used when we don't
+ * need diff expire.
+ */
+void separate_expire(std::vector<geom::geometry_t> const &geoms,
+                     expire_config_t const &expire_config,
+                     expire_tiles_t &expire_tiles,
+                     std::vector<expire_output_t> *expire_outputs)
+{
+    assert(expire_outputs);
+
+    for (auto const &geom : geoms) {
+        expire_tiles.from_geometry(geom, expire_config);
+    }
+    expire_tiles.commit_tiles(&expire_outputs->at(expire_config.expire_output));
+}
+
+/**
+ * When doing diff expire, we need to calculate the symmetric difference
+ * between old and new geometries. The difference is done by type, so points
+ * are compared with points, linestrings with linestrings, etc. This function
+ * separates out the input geometries by the three fundamental types.
+ */
+// NOLINTBEGIN(cppcoreguidelines-rvalue-reference-param-not-moved)
+template <typename T>
+void classify_geometries(T input_geoms, geom::multipoint_t *points,
+                         geom::multilinestring_t *linestrings,
+                         geom::multipolygon_t *polygons)
+{
+    assert(points);
+    assert(linestrings);
+    assert(polygons);
+
+    for (auto &&geom : *input_geoms) {
+        visit(overloaded{
+                  [&](geom::nullgeom_t && /*input*/) {},
+                  [&](geom::point_t &&input) { points->add_geometry(input); },
+                  [&](geom::linestring_t &&input) {
+                      linestrings->add_geometry(std::move(input));
+                  },
+                  [&](geom::polygon_t &&input) {
+                      polygons->add_geometry(std::move(input));
+                  },
+                  [&](geom::multipoint_t &&input) {
+                      for (auto &&point : input) {
+                          points->add_geometry(point);
+                      }
+                  },
+                  [&](geom::multilinestring_t &&input) {
+                      for (auto &&linestring : input) {
+                          linestrings->add_geometry(std::move(linestring));
+                      }
+                  },
+                  [&](geom::multipolygon_t &&input) {
+                      for (auto &&polygon : input) {
+                          polygons->add_geometry(std::move(polygon));
+                      }
+                  },
+                  [&](geom::collection_t &&input) {
+                      classify_geometries(&input, points, linestrings,
+                                          polygons);
+                  }},
+              std::move(geom));
+    }
+}
+// NOLINTEND(cppcoreguidelines-rvalue-reference-param-not-moved)
+
+template <typename T>
+void diff_and_expire(geom::multigeometry_t<T> const &old_geoms,
+                     geom::multigeometry_t<T> const &new_geoms,
+                     expire_config_t const &expire_config,
+                     expire_tiles_t &expire_tiles)
+{
+    std::vector<T> diffs;
+    boost::geometry::sym_difference(old_geoms, new_geoms, diffs);
+    for (auto const &geom : diffs) {
+        expire_tiles.from_geometry(geom, expire_config);
+    }
+}
+
+void diff_expire(std::vector<geom::geometry_t> *geoms_old,
+                 std::vector<geom::geometry_t> *geoms_new,
+                 expire_config_t const &expire_config,
+                 expire_tiles_t &expire_tiles,
+                 std::vector<expire_output_t> *expire_outputs)
+{
+    assert(geoms_old);
+    assert(geoms_new);
+    assert(expire_outputs);
+
+    geom::multipoint_t old_points;
+    geom::multilinestring_t old_linestrings;
+    geom::multipolygon_t old_polygons;
+
+    classify_geometries(geoms_old, &old_points, &old_linestrings,
+                        &old_polygons);
+
+    geom::multipoint_t new_points;
+    geom::multilinestring_t new_linestrings;
+    geom::multipolygon_t new_polygons;
+
+    classify_geometries(geoms_new, &new_points, &new_linestrings,
+                        &new_polygons);
+
+    diff_and_expire(old_points, new_points, expire_config, expire_tiles);
+    diff_and_expire(old_linestrings, new_linestrings, expire_config,
+                    expire_tiles);
+    diff_and_expire(old_polygons, new_polygons, expire_config, expire_tiles);
+
+    expire_tiles.commit_tiles(&expire_outputs->at(expire_config.expire_output));
+}
+
+} // anonymous namespace
+
 void flex_table_column_t::do_expire(
-    geom::geometry_t const &geom, std::vector<expire_tiles_t> *expire,
-    std::vector<expire_output_t> *expire_outputs) const
+    std::vector<geom::geometry_t> *geoms_old,
+    std::vector<geom::geometry_t> *geoms_new,
+    std::vector<expire_tiles_t> *expire,
+    std::vector<expire_output_t> *expire_outputs, bool enable_diff_expire) const
 {
+    assert(geoms_old);
+    assert(geoms_new);
     assert(expire);
     assert(expire_outputs);
 
     for (auto const &expire_config : m_expires) {
         assert(expire_config.expire_output < expire->size());
         auto &expire_tiles = expire->at(expire_config.expire_output);
-        expire_tiles.from_geometry(geom, expire_config);
-        expire_tiles.commit_tiles(
-            &expire_outputs->at(expire_config.expire_output));
+
+        if (!expire_config.diff_expire || !enable_diff_expire ||
+            geoms_old->empty() || geoms_new->empty()) {
+            separate_expire(*geoms_old, expire_config, expire_tiles,
+                            expire_outputs);
+            separate_expire(*geoms_new, expire_config, expire_tiles,
+                            expire_outputs);
+        } else {
+            diff_expire(geoms_old, geoms_new, expire_config, expire_tiles,
+                        expire_outputs);
+        }
     }
 }

src/flex-table-column.hpp

@@ -18,6 +18,7 @@
 #include <cassert>
 #include <cstdint>
 #include <string>
+#include <unordered_map>
 #include <vector>
 
 enum class table_column_type : uint8_t
@@ -54,6 +55,8 @@ enum class table_column_type : uint8_t
     id_num
 };
 
+class geometry_cache_t;
+
 /**
  * A column in a flex_table_t.
  */
@@ -134,9 +137,11 @@ class flex_table_column_t
         return m_expires;
     }
 
-    void do_expire(geom::geometry_t const &geom,
+    void do_expire(std::vector<geom::geometry_t> *geoms_old,
+                   std::vector<geom::geometry_t> *geoms_new,
                    std::vector<expire_tiles_t> *expire,
-                   std::vector<expire_output_t> *expire_outputs) const;
+                   std::vector<expire_output_t> *expire_outputs,
+                   bool enable_diff_expire) const;
 
 private:
     std::vector<expire_config_t> m_expires;
@@ -170,6 +175,40 @@ class flex_table_column_t
 
     /// Column will be created but not filled by osm2pgsql.
     bool m_create_only = false;
-};
+}; // class flex_table_column_t
+
+/**
+ * While processing an OSM object, this cache is used to hold all old and all
+ * new geometries stored in all geometry columns with expire config in a table.
+ * Later those geometries are used to calculate the expire.
+ */
+class geometry_cache_t
+{
+public:
+    template <typename GEOM>
+    void add_old(flex_table_column_t const *column, GEOM &&geom)
+    {
+        m_geometries[column].first.push_back(std::forward<GEOM>(geom));
+    }
+
+    template <typename GEOM>
+    void add_new(flex_table_column_t const *column, GEOM &&geom)
+    {
+        m_geometries[column].second.push_back(std::forward<GEOM>(geom));
+    }
+
+    auto begin() noexcept { return m_geometries.begin(); }
+
+    auto end() noexcept { return m_geometries.end(); }
+
+    void clear() { m_geometries.clear(); }
+
+private:
+    std::unordered_map<
+        flex_table_column_t const *,
+        std::pair<std::vector<geom::geometry_t>, std::vector<geom::geometry_t>>>
+        m_geometries;
+
+}; // class geometry_cache_t
 
 #endif // OSM2PGSQL_FLEX_TABLE_COLUMN_HPP

src/flex-write.cpp

@@ -8,6 +8,7 @@
  */
 
 #include "flex-lua-geom.hpp"
+#include "flex-table-column.hpp"
 #include "flex-write.hpp"
 #include "geom-functions.hpp"
 #include "json-writer.hpp"
@@ -257,11 +258,9 @@ bool is_compatible(geom::geometry_t const &geom,
 
 } // anonymous namespace
 
-void flex_write_column(lua_State *lua_state,
+void flex_write_column(lua_State *lua_state, geometry_cache_t *geom_cache,
                        db_copy_mgr_t<db_deleter_by_type_and_id_t> *copy_mgr,
-                       flex_table_column_t const &column,
-                       std::vector<expire_tiles_t> *expire,
-                       std::vector<expire_output_t> *expire_outputs)
+                       flex_table_column_t const &column)
 {
     lua_getfield(lua_state, -1, column.name().c_str());
     int const ltype = lua_type(lua_state, -1);
@@ -446,13 +445,13 @@ void flex_write_column(lua_State *lua_state,
                      type == table_column_type::multilinestring ||
                      type == table_column_type::multipolygon);
                 if (geom->srid() == column.srid()) {
-                    column.do_expire(*geom, expire, expire_outputs);
                     copy_mgr->add_hex_geom(geom_to_ewkb(*geom, wrap_multi));
+                    geom_cache->add_new(&column, *geom);
                 } else {
                     auto const &proj = get_projection(column.srid());
-                    auto const tgeom = geom::transform(*geom, proj);
-                    column.do_expire(tgeom, expire, expire_outputs);
+                    auto tgeom = geom::transform(*geom, proj);
                     copy_mgr->add_hex_geom(geom_to_ewkb(tgeom, wrap_multi));
+                    geom_cache->add_new(&column, std::move(tgeom));
                 }
             } else {
                 write_null(copy_mgr, column);

src/flex-write.hpp

@@ -19,6 +19,7 @@
 #include <vector>
 
 class expire_tiles_t;
+class geometry_cache_t;
 
 class not_null_exception_t : public std::runtime_error
 {
@@ -34,10 +35,8 @@ class not_null_exception_t : public std::runtime_error
     flex_table_column_t const *m_column;
 }; // class not_null_exception_t
 
-void flex_write_column(lua_State *lua_state,
+void flex_write_column(lua_State *lua_state, geometry_cache_t *geom_cache,
                        db_copy_mgr_t<db_deleter_by_type_and_id_t> *copy_mgr,
-                       flex_table_column_t const &column,
-                       std::vector<expire_tiles_t> *expire,
-                       std::vector<expire_output_t> *expire_outputs);
+                       flex_table_column_t const &column);
 
 #endif // OSM2PGSQL_FLEX_WRITE_HPP