Add live HostDB cache clearing

doc/appendices/command-line/traffic_ctl.en.rst

@@ -1290,6 +1290,13 @@ traffic_ctl hostdb
 
    If ``HOSTNAME`` is specified, the output is filtered to show only records whose names contain the given string.
 
+.. option:: clear
+
+   :ref:`admin_hostdb_clear`
+
+   Remove all DNS resolution records from HostDB without restarting |TS|. In-flight transactions retain their current resolution,
+   while subsequent transactions resolve their upstream hosts again.
+
 traffic_ctl rpc
 ---------------
 .. program:: traffic_ctl rpc

doc/developer-guide/jsonrpc/jsonrpc-api.en.rst

@@ -453,6 +453,8 @@ JSONRPC API
 
 * `admin_host_set_status`_
 
+* `admin_hostdb_clear`_
+
 * `admin_server_stop_drain`_
 
 * `admin_server_start_drain`_
@@ -1396,6 +1398,58 @@ Response:
    }
 
 
+HostDB
+======
+
+.. _admin_hostdb_clear:
+
+admin_hostdb_clear
+------------------
+
+|method|
+
+Description
+~~~~~~~~~~~
+
+Remove all cached DNS resolution records from HostDB. In-flight transactions retain their current resolution, while subsequent
+transactions resolve their upstream hosts again.
+
+Parameters
+~~~~~~~~~~
+
+* ``params``: Omitted
+
+Result
+~~~~~~
+
+The response will contain the default `success_response` or an error. :ref:`jsonrpc-node-errors`.
+
+Examples
+~~~~~~~~
+
+Request:
+
+.. code-block:: json
+   :linenos:
+
+   {
+      "id": "dc8b7f1e-2521-4bc0-b0ed-5e3ec751599d",
+      "jsonrpc": "2.0",
+      "method": "admin_hostdb_clear"
+   }
+
+Response:
+
+.. code-block:: json
+   :linenos:
+
+   {
+      "jsonrpc": "2.0",
+      "result": "success",
+      "id": "dc8b7f1e-2521-4bc0-b0ed-5e3ec751599d"
+   }
+
+
 .. _admin_server_stop_drain:
 
 admin_server_stop_drain

include/iocore/hostdb/HostDBProcessor.h

@@ -652,6 +652,9 @@ struct HostDBProcessor : public Processor {
    */
   int start(int no_of_additional_event_threads = 0, size_t stacksize = DEFAULT_STACKSIZE) override;
 
+  /// Remove all cached DNS resolution records.
+  void clear();
+
   // Private
   HostDBCache *cache();
 

include/mgmt/rpc/handlers/hostdb/HostDB.h

@@ -25,4 +25,5 @@
 namespace rpc::handlers::hostdb
 {
 swoc::Rv<YAML::Node> get_hostdb_status(std::string_view const &id, YAML::Node const &);
+swoc::Rv<YAML::Node> clear_hostdb(std::string_view const &id, YAML::Node const &);
 } // namespace rpc::handlers::hostdb

src/iocore/hostdb/HostDB.cc

@@ -254,6 +254,12 @@ HostDBProcessor::cache()
   return &hostDB;
 }
 
+void
+HostDBProcessor::clear()
+{
+  hostDB.refcountcache->clear();
+}
+
 int
 HostDBCache::start(int flags)
 {

src/iocore/hostdb/P_RefCountCache.h

@@ -35,6 +35,7 @@
 
 #include "swoc/IntrusiveHashMap.h"
 #include <cstdint>
+#include <mutex>
 #include <unistd.h>
 
 using ts::Metrics;
@@ -507,6 +508,9 @@ void
 RefCountCache<C>::clear()
 {
   for (unsigned int i = 0; i < this->num_partitions; i++) {
-    this->partitions[i]->clear();
+    auto &partition = *this->partitions[i];
+
+    std::unique_lock<ts::shared_mutex> lock{partition.lock};
+    partition.clear();
   }
 }

src/mgmt/rpc/handlers/hostdb/HostDB.cc

@@ -84,24 +84,28 @@ template <> struct convert<HostDBCache> {
   {
     Node partitions;
     for (size_t i = 0; i < hostDB->refcountcache->partition_count(); i++) {
-      auto                                 &partition = hostDB->refcountcache->get_partition(i);
-      std::vector<RefCountCacheHashEntry *> partition_entries;
+      auto                             &partition = hostDB->refcountcache->get_partition(i);
+      std::vector<HostDBRecord::Handle> partition_records;
 
       {
         std::shared_lock<ts::shared_mutex> shared_lock{partition.lock};
-        partition_entries.reserve(partition.count());
-        partition.copy(partition_entries);
+
+        partition_records.reserve(partition.count());
+        for (auto &&entry : partition.get_map()) {
+          partition_records.emplace_back(make_ptr(static_cast<HostDBRecord *>(entry.item.get())));
+        }
       }
 
-      if (partition_entries.empty()) {
+      if (partition_records.empty()) {
         continue;
       }
 
       Node partition_node;
       partition_node["id"] = i;
 
-      for (RefCountCacheHashEntry *entry : partition_entries) {
-        HostDBRecord *record = static_cast<HostDBRecord *>(entry->item.get());
+      for (auto const &record_handle : partition_records) {
+        HostDBRecord *record = record_handle.get();
+
         if (!hostname.empty() && record->name_view().find(hostname) == std::string_view::npos) {
           continue;
         }
@@ -203,4 +207,11 @@ get_hostdb_status(std::string_view const & /* id ATS_UNUSED */, YAML::Node const
   }
   return resp;
 }
+
+swoc::Rv<YAML::Node>
+clear_hostdb(std::string_view const & /* id ATS_UNUSED */, YAML::Node const & /* params ATS_UNUSED */)
+{
+  hostDBProcessor.clear();
+  return {};
+}
 } // namespace rpc::handlers::hostdb

src/traffic_ctl/CtrlCommands.cc

@@ -927,6 +927,9 @@ HostDBCommand::HostDBCommand(ts::Arguments *args) : CtrlCommand(args)
   if (get_parsed_arguments()->get(STATUS_STR)) {
     _printer      = std::make_unique<HostDBStatusPrinter>(printOpts);
     _invoked_func = [&]() { status_get(); };
+  } else if (get_parsed_arguments()->get(CLEAR_STR)) {
+    _printer      = std::make_unique<GenericPrinter>(printOpts);
+    _invoked_func = [&]() { clear(); };
   }
 }
 
@@ -948,6 +951,16 @@ HostDBCommand::status_get()
 
   _printer->write_output(response);
 }
+
+void
+HostDBCommand::clear()
+{
+  HostDBClearRequest request;
+
+  auto response = invoke_rpc(request);
+
+  _printer->write_output(response);
+}
 //------------------------------------------------------------------------------------------------------------------------------------
 PluginCommand::PluginCommand(ts::Arguments *args) : CtrlCommand(args)
 {

src/traffic_ctl/CtrlCommands.h

@@ -196,8 +196,10 @@ class HostDBCommand : public CtrlCommand
 
 private:
   static inline const std::string STATUS_STR{"status"};
+  static inline const std::string CLEAR_STR{"clear"};
 
   void status_get();
+  void clear();
 };
 // -----------------------------------------------------------------------------------------------------------------------------------
 class PluginCommand : public CtrlCommand

src/traffic_ctl/jsonrpc/CtrlRPCRequests.h

@@ -198,6 +198,14 @@ struct HostDBGetStatusRequest : shared::rpc::ClientRequest {
     return "get_hostdb_status";
   }
 };
+
+struct HostDBClearRequest : shared::rpc::ClientRequest {
+  std::string
+  get_method() const override
+  {
+    return "admin_hostdb_clear";
+  }
+};
 //------------------------------------------------------------------------------------------------------------------------------------
 struct GetPluginListRequest : shared::rpc::ClientRequest {
   using super = shared::rpc::ClientRequest;

src/traffic_ctl/traffic_ctl.cc

@@ -259,6 +259,8 @@ main([[maybe_unused]] int argc, const char **argv)
   // hostdb commands
   hostdb_command.add_command("status", "Get HostDB info", "", MORE_THAN_ZERO_ARG_N, Command_Execute)
     .add_example_usage("traffic_ctl hostdb status");
+  hostdb_command.add_command("clear", "Clear all HostDB entries", "", 0, Command_Execute)
+    .add_example_usage("traffic_ctl hostdb clear");
 
   // plugin command
   plugin_command

src/traffic_server/RpcAdminPubHandlers.cc

@@ -46,6 +46,7 @@ register_admin_jsonrpc_handlers()
   // HostDB
   using namespace rpc::handlers::hostdb;
   rpc::add_method_handler("get_hostdb_status", &get_hostdb_status, &core_ats_rpc_service_provider_handle, {{rpc::RESTRICTED_API}});
+  rpc::add_method_handler("admin_hostdb_clear", &clear_hostdb, &core_ats_rpc_service_provider_handle, {{rpc::RESTRICTED_API}});
 
   // Records
   using namespace rpc::handlers::records;

tests/gold_tests/dns/hostdb_clear.test.py

@@ -0,0 +1,85 @@
+'''
+Verify HostDB can be cleared without restarting Traffic Server.
+'''
+#  Licensed to the Apache Software Foundation (ASF) under one
+#  or more contributor license agreements.  See the NOTICE file
+#  distributed with this work for additional information
+#  regarding copyright ownership.  The ASF licenses this file
+#  to you under the Apache License, Version 2.0 (the
+#  "License"); you may not use this file except in compliance
+#  with the License.  You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+
+from ports import get_port
+
+Test.Summary = 'Verify HostDB can be cleared without restarting Traffic Server.'
+
+replay_file = 'replay/single_transaction.replay.yaml'
+hostname = 'resolve.this.com'
+
+server = Test.MakeVerifierServerProcess('server', replay_file)
+ts = Test.MakeATSProcess('ts', enable_cache=False)
+dns_port = get_port(ts, 'dns_port')
+dns = Test.MakeDNServer('dns', port=dns_port)
+dns.addRecords(records={hostname: ['127.0.0.1']})
+
+ts.Disk.records_config.update({
+    'proxy.config.dns.nameservers': f'127.0.0.1:{dns_port}',
+    'proxy.config.dns.resolv_conf': 'NULL',
+})
+ts.Disk.remap_config.AddLine(f'map / http://{hostname}:{server.Variables.http_port}/')
+
+tr = Test.AddTestRun('Populate HostDB')
+tr.AddVerifierClientProcess('client-before-clear', replay_file, http_ports=[ts.Variables.port])
+tr.Processes.Default.StartBefore(dns)
+tr.Processes.Default.StartBefore(server)
+tr.Processes.Default.StartBefore(ts)
+tr.StillRunningAfter = dns
+tr.StillRunningAfter = server
+tr.StillRunningAfter = ts
+
+tr = Test.AddTestRun('Verify HostDB is populated')
+tr.Processes.Default.Command = f'traffic_ctl hostdb status {hostname}'
+tr.Processes.Default.Env = ts.Env
+tr.Processes.Default.ReturnCode = 0
+tr.Processes.Default.Streams.All = Testers.ContainsExpression(hostname, 'HostDB should contain the resolved host')
+tr.StillRunningAfter = dns
+tr.StillRunningAfter = server
+tr.StillRunningAfter = ts
+
+tr = Test.AddTestRun('Clear HostDB')
+tr.Processes.Default.Command = 'traffic_ctl hostdb clear'
+tr.Processes.Default.Env = ts.Env
+tr.Processes.Default.ReturnCode = 0
+tr.StillRunningAfter = dns
+tr.StillRunningAfter = server
+tr.StillRunningAfter = ts
+
+tr = Test.AddTestRun('Verify HostDB is empty')
+tr.Processes.Default.Command = f'traffic_ctl hostdb status {hostname}'
+tr.Processes.Default.Env = ts.Env
+tr.Processes.Default.ReturnCode = 0
+tr.Processes.Default.Streams.All = Testers.ExcludesExpression(hostname, 'HostDB should no longer contain the resolved host')
+tr.StillRunningAfter = dns
+tr.StillRunningAfter = server
+tr.StillRunningAfter = ts
+
+tr = Test.AddTestRun('Resolve the host again without restarting Traffic Server')
+tr.AddVerifierClientProcess('client-after-clear', replay_file, http_ports=[ts.Variables.port])
+tr.StillRunningAfter = dns
+tr.StillRunningAfter = server
+tr.StillRunningAfter = ts
+
+tr = Test.AddTestRun('Verify HostDB is repopulated')
+tr.Processes.Default.Command = f'traffic_ctl hostdb status {hostname}'
+tr.Processes.Default.Env = ts.Env
+tr.Processes.Default.ReturnCode = 0
+tr.Processes.Default.Streams.All = Testers.ContainsExpression(hostname, 'HostDB should contain the resolved host again')
+tr.StillRunningAfter = ts