Merge pull request #2449 from joto/limit-expire-tiles

Introduce per geometry and overall limits on number of expire tiles

Author: lonvia

src/expire-output.cpp

@@ -17,16 +17,67 @@
 #include <cerrno>
 #include <system_error>
 
+void expire_output_t::add_tiles(
+    std::unordered_set<quadkey_t> const &dirty_tiles)
+{
+    std::lock_guard<std::mutex> const guard{*m_tiles_mutex};
+
+    if (m_overall_tile_limit_reached) {
+        return;
+    }
+
+    if (dirty_tiles.size() > m_max_tiles_geometry) {
+        log_warn("Tile limit {} reached for single geometry!",
+                 m_max_tiles_geometry);
+        return;
+    }
+
+    /**
+     * This check is not quite correct, because some tiles could be in both,
+     * the dirty_list and in m_tiles, which means we might not reach
+     * m_max_tiles_overall if we join those in. But this check is much
+     * easier and cheaper than trying to add all the tiles into the dirty_list,
+     * checking each time whether we reached the limit. And with the number
+     * of tiles involved in doesn't matter that much anyway.
+     */
+    if (dirty_tiles.size() + m_tiles.size() > m_max_tiles_overall) {
+        m_overall_tile_limit_reached = true;
+        log_warn("Overall tile limit {} reached for this run!",
+                 m_max_tiles_overall);
+        return;
+    }
+
+    m_tiles.insert(dirty_tiles.cbegin(), dirty_tiles.cend());
+}
+
+bool expire_output_t::empty() noexcept
+{
+    std::lock_guard<std::mutex> const guard{*m_tiles_mutex};
+    return m_tiles.empty();
+}
+
+quadkey_list_t expire_output_t::get_tiles()
+{
+    std::lock_guard<std::mutex> const guard{*m_tiles_mutex};
+    quadkey_list_t tile_list;
+
+    tile_list.reserve(m_tiles.size());
+    tile_list.assign(m_tiles.cbegin(), m_tiles.cend());
+    std::sort(tile_list.begin(), tile_list.end());
+    m_tiles.clear();
+
+    return tile_list;
+}
+
 std::size_t
-expire_output_t::output(quadkey_list_t const &tile_list,
-                        connection_params_t const &connection_params) const
+expire_output_t::output(connection_params_t const &connection_params)
 {
     std::size_t num = 0;
     if (!m_filename.empty()) {
-        num = output_tiles_to_file(tile_list);
+        num = output_tiles_to_file(get_tiles());
     }
     if (!m_table.empty()) {
-        num = output_tiles_to_table(tile_list, connection_params);
+        num = output_tiles_to_table(get_tiles(), connection_params);
     }
     return num;
 }

src/expire-output.hpp

@@ -15,9 +15,15 @@
 #include <cassert>
 #include <cstddef>
 #include <cstdint>
+#include <memory>
+#include <mutex>
 #include <string>
+#include <unordered_set>
 #include <utility>
 
+constexpr std::size_t DEFAULT_MAX_TILES_GEOMETRY = 10'000'000;
+constexpr std::size_t DEFAULT_MAX_TILES_OVERALL = 50'000'000;
+
 class pg_conn_t;
 class connection_params_t;
 
@@ -53,9 +59,45 @@ class expire_output_t
     uint32_t maxzoom() const noexcept { return m_maxzoom; }
     void set_maxzoom(uint32_t maxzoom) noexcept { m_maxzoom = maxzoom; }
 
-    std::size_t output(quadkey_list_t const &tile_list,
-                       connection_params_t const &connection_params) const;
+    std::size_t max_tiles_geometry() const noexcept
+    {
+        return m_max_tiles_geometry;
+    }
+
+    void set_max_tiles_geometry(std::size_t max_tiles_geometry) noexcept
+    {
+        m_max_tiles_geometry = max_tiles_geometry;
+    }
+
+    std::size_t max_tiles_overall() const noexcept
+    {
+        return m_max_tiles_overall;
+    }
+
+    void set_max_tiles_overall(std::size_t max_tiles_overall) noexcept
+    {
+        m_max_tiles_overall = max_tiles_overall;
+    }
+
+    bool empty() noexcept;
+
+    void add_tiles(std::unordered_set<quadkey_t> const &dirty_tiles);
+
+    quadkey_list_t get_tiles();
+
+    /**
+     * Write the list of tiles to a database table or file.
+     *
+     * \param connection_params Database connection parameters
+     */
+    std::size_t output(connection_params_t const &connection_params);
+
+    /**
+     * Create table for tiles.
+     */
+    void create_output_table(pg_conn_t const &db_connection) const;
 
+private:
     /**
      * Write the list of tiles to a file.
      *
@@ -75,11 +117,16 @@ class expire_output_t
                           connection_params_t const &connection_params) const;
 
     /**
-     * Create table for tiles.
+     * Access to the m_tiles collection of expired tiles must go through
+     * this mutex, because it can happend from several threads at the same
+     * time. Mutex is wrapped in a shared_ptr to make this class movable so
+     * we can store instances in std::vector.
      */
-    void create_output_table(pg_conn_t const &db_connection) const;
+    std::shared_ptr<std::mutex> m_tiles_mutex = std::make_shared<std::mutex>();
+
+    /// This is where we collect all the expired tiles.
+    std::unordered_set<quadkey_t> m_tiles;
 
-private:
     /// The filename (if any) for output
     std::string m_filename;
 
@@ -95,6 +142,22 @@ class expire_output_t
     /// Zoom level we capture tiles on
     uint32_t m_maxzoom = 0;
 
+    /**
+     * The following two settings are for protecting osm2pgsql from overload as
+     * well as downstream tile expiry mechanisms in case of large changes to
+     * OSM data (possibly from vandalism). They should be large enough to not
+     * trigger in normal use.
+     */
+
+    /// Maximum number of tiles that can be affected by a single geometry.
+    std::size_t m_max_tiles_geometry = DEFAULT_MAX_TILES_GEOMETRY;
+
+    /// Maximum number of tiles that can be affected per run.
+    std::size_t m_max_tiles_overall = DEFAULT_MAX_TILES_OVERALL;
+
+    /// Has the overall tile limit been reached already.
+    bool m_overall_tile_limit_reached = false;
+
 }; // class expire_output_t
 
 #endif // OSM2PGSQL_EXPIRE_OUTPUT_HPP

src/expire-tiles.cpp

@@ -27,20 +27,30 @@
 #include "wkb.hpp"
 
 expire_tiles_t::expire_tiles_t(uint32_t max_zoom,
-                               std::shared_ptr<reprojection_t> projection)
-: m_projection(std::move(projection)), m_maxzoom(max_zoom),
-  m_map_width(static_cast<int>(1U << m_maxzoom))
-{}
+                               std::shared_ptr<reprojection_t> projection,
+                               std::size_t max_tiles_geometry)
+: m_projection(std::move(projection)), m_max_tiles_geometry(max_tiles_geometry),
+  m_maxzoom(max_zoom), m_map_width(static_cast<int>(1U << m_maxzoom))
+{
+}
 
 void expire_tiles_t::expire_tile(uint32_t x, uint32_t y)
 {
-    // Only try to insert to tile into the set if the last inserted tile
-    // is different from this tile.
+    if (m_dirty_tiles.size() > m_max_tiles_geometry) {
+        return;
+    }
+
     tile_t const new_tile{m_maxzoom, x, y};
-    if (!m_prev_tile.valid() || m_prev_tile != new_tile) {
-        m_dirty_tiles.insert(new_tile.quadkey());
-        m_prev_tile = new_tile;
+    m_dirty_tiles.insert(new_tile.quadkey());
+}
+
+void expire_tiles_t::commit_tiles(expire_output_t *expire_output)
+{
+    if (!expire_output || m_dirty_tiles.empty()) {
+        return;
     }
+    expire_output->add_tiles(m_dirty_tiles);
+    m_dirty_tiles.clear();
 }
 
 uint32_t expire_tiles_t::normalise_tile_x_coord(int x) const
@@ -281,24 +291,6 @@ quadkey_list_t expire_tiles_t::get_tiles()
     return tiles;
 }
 
-void expire_tiles_t::merge_and_destroy(expire_tiles_t *other)
-{
-    if (m_map_width != other->m_map_width) {
-        throw fmt_error("Unable to merge tile expiry sets when "
-                        "map_width does not match: {} != {}.",
-                        m_map_width, other->m_map_width);
-    }
-
-    if (m_dirty_tiles.empty()) {
-        using std::swap;
-        swap(m_dirty_tiles, other->m_dirty_tiles);
-    } else {
-        m_dirty_tiles.insert(other->m_dirty_tiles.cbegin(),
-                             other->m_dirty_tiles.cend());
-        other->m_dirty_tiles.clear();
-    }
-}
-
 int expire_from_result(expire_tiles_t *expire, pg_result_t const &result,
                        expire_config_t const &expire_config)
 {

src/expire-tiles.hpp

@@ -18,6 +18,7 @@
 #include <vector>
 
 #include "expire-config.hpp"
+#include "expire-output.hpp"
 #include "geom.hpp"
 #include "geom-box.hpp"
 #include "logging.hpp"
@@ -31,9 +32,8 @@ class expire_tiles_t
 {
 public:
     expire_tiles_t(uint32_t max_zoom,
-                   std::shared_ptr<reprojection_t> projection);
-
-    bool empty() const noexcept { return m_dirty_tiles.empty(); }
+                   std::shared_ptr<reprojection_t> projection,
+                   std::size_t max_tiles_geometry = DEFAULT_MAX_TILES_GEOMETRY);
 
     bool enabled() const noexcept { return m_maxzoom != 0; }
 
@@ -83,10 +83,13 @@ class expire_tiles_t
     quadkey_list_t get_tiles();
 
     /**
-     * Merge the list of expired tiles in the other object into this
-     * object, destroying the list in the other object.
+     * Must be called after calling expire_tile() one or more times for a
+     * single geometry to "commit" all tiles to be expired for that geometry.
+     *
+     * \param expire_output The expire output to write tiles to. If this is
+     *                      the nullptr, nothing is done.
      */
-    void merge_and_destroy(expire_tiles_t *other);
+    void commit_tiles(expire_output_t* expire_output);
 
 private:
     /**
@@ -113,11 +116,9 @@ class expire_tiles_t
     /// This is where we collect all the expired tiles.
     std::unordered_set<quadkey_t> m_dirty_tiles;
 
-    /// The tile which has been added last to the unordered set.
-    tile_t m_prev_tile;
-
     std::shared_ptr<reprojection_t> m_projection;
 
+    std::size_t m_max_tiles_geometry;
     uint32_t m_maxzoom;
     int m_map_width;
 

src/flex-lua-expire-output.cpp

@@ -62,6 +62,26 @@ create_expire_output(lua_State *lua_state, std::string const &default_schema,
     }
     lua_pop(lua_state, 1); // "minzoom"
 
+    // optional "max_tiles_geometry" field
+    auto const max_tiles_geometry = luaX_get_table_optional_uint64(
+        lua_state, "max_tiles_geometry", -1,
+        "The 'max_tiles_geometry' field in a expire output", 1, (4ULL << 20ULL),
+        "1 and 4 << 20");
+    if (max_tiles_geometry > 0) {
+        new_expire_output.set_max_tiles_geometry(max_tiles_geometry);
+    }
+    lua_pop(lua_state, 1); // "max_tiles_geometry"
+
+    // optional "max_tiles_overall" field
+    auto const max_tiles_overall = luaX_get_table_optional_uint64(
+        lua_state, "max_tiles_overall", -1,
+        "The 'max_tiles_overall' field in a expire output", 1, (4ULL << 20ULL),
+        "1 and 4 << 20");
+    if (max_tiles_overall > 0) {
+        new_expire_output.set_max_tiles_overall(max_tiles_overall);
+    }
+    lua_pop(lua_state, 1); // "max_tiles_overall"
+
     return new_expire_output;
 }
 
@@ -71,6 +91,8 @@ TRAMPOLINE_WRAPPED_OBJECT(expire_output, maxzoom)
 TRAMPOLINE_WRAPPED_OBJECT(expire_output, minzoom)
 TRAMPOLINE_WRAPPED_OBJECT(expire_output, schema)
 TRAMPOLINE_WRAPPED_OBJECT(expire_output, table)
+TRAMPOLINE_WRAPPED_OBJECT(expire_output, max_tiles_geometry)
+TRAMPOLINE_WRAPPED_OBJECT(expire_output, max_tiles_overall)
 
 } // anonymous namespace
 
@@ -106,7 +128,11 @@ void lua_wrapper_expire_output_t::init(lua_State *lua_state)
          {"maxzoom", lua_trampoline_expire_output_maxzoom},
          {"minzoom", lua_trampoline_expire_output_minzoom},
          {"schema", lua_trampoline_expire_output_schema},
-         {"table", lua_trampoline_expire_output_table}});
+         {"table", lua_trampoline_expire_output_table},
+         {"max_tiles_geometry",
+          lua_trampoline_expire_output_max_tiles_geometry},
+         {"max_tiles_overall",
+          lua_trampoline_expire_output_max_tiles_overall}});
 }
 
 int lua_wrapper_expire_output_t::tostring() const
@@ -150,3 +176,17 @@ int lua_wrapper_expire_output_t::table() const noexcept
     luaX_pushstring(lua_state(), self().table());
     return 1;
 }
+
+int lua_wrapper_expire_output_t::max_tiles_geometry() const noexcept
+{
+    lua_pushinteger(lua_state(),
+                    static_cast<lua_Integer>(self().max_tiles_geometry()));
+    return 1;
+}
+
+int lua_wrapper_expire_output_t::max_tiles_overall() const noexcept
+{
+    lua_pushinteger(lua_state(),
+                    static_cast<lua_Integer>(self().max_tiles_overall()));
+    return 1;
+}

src/flex-lua-expire-output.hpp

@@ -42,6 +42,8 @@ class lua_wrapper_expire_output_t : public lua_wrapper_base_t<expire_output_t>
     int minzoom() const noexcept;
     int schema() const noexcept;
     int table() const noexcept;
+    int max_tiles_geometry() const noexcept;
+    int max_tiles_overall() const noexcept;
 
 }; // class lua_wrapper_expire_output_t
 

src/flex-table-column.cpp

@@ -202,13 +202,18 @@ void flex_table_column_t::add_expire(expire_config_t const &config)
     m_expires.push_back(config);
 }
 
-void flex_table_column_t::do_expire(geom::geometry_t const &geom,
-                                    std::vector<expire_tiles_t> *expire) const
+void flex_table_column_t::do_expire(
+    geom::geometry_t const &geom, std::vector<expire_tiles_t> *expire,
+    std::vector<expire_output_t> *expire_outputs) const
 {
     assert(expire);
+    assert(expire_outputs);
+
     for (auto const &expire_config : m_expires) {
         assert(expire_config.expire_output < expire->size());
-        (*expire)[expire_config.expire_output].from_geometry(geom,
-                                                             expire_config);
+        auto &expire_tiles = expire->at(expire_config.expire_output);
+        expire_tiles.from_geometry(geom, expire_config);
+        expire_tiles.commit_tiles(
+            &expire_outputs->at(expire_config.expire_output));
     }
 }