kvm: support Host HA on Ceph RBD primary storage

engine/components-api/src/main/java/com/cloud/ha/HighAvailabilityManager.java

@@ -33,7 +33,8 @@
  */
 public interface HighAvailabilityManager extends Manager {
 
-    List<StoragePoolType> LIBVIRT_STORAGE_POOL_TYPES_WITH_HA_SUPPORT = List.of(StoragePoolType.NetworkFilesystem, StoragePoolType.SharedMountPoint);
+    List<StoragePoolType> LIBVIRT_STORAGE_POOL_TYPES_WITH_HA_SUPPORT = List.of(StoragePoolType.NetworkFilesystem, StoragePoolType.SharedMountPoint,
+            StoragePoolType.RBD);
 
     ConfigKey<Boolean> ForceHA = new ConfigKey<>("Advanced", Boolean.class, "force.ha", "false",
         "Force High-Availability to happen even if the VM says no.", true, Cluster);

plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/KVMHAMonitor.java

@@ -130,7 +130,7 @@ private void checkForNotExistingLibvirtStoragePools(Set<String> removedPools, St
                 removedPools.add(uuid);
             }
 
-            logger.debug("Found NFS storage pool [{}] in libvirt, continuing.", uuid);
+            logger.debug("Found storage pool [{}] in libvirt, continuing.", uuid);
 
         } catch (LibvirtException e) {
             logger.debug("Failed to lookup libvirt storage pool [{}].", uuid, e);

plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/LibvirtStoragePool.java

@@ -330,29 +330,55 @@ public boolean isPoolSupportHA() {
 
     public String getHearthBeatPath() {
         if (StoragePoolType.NetworkFilesystem.equals(type)) {
-            String kvmScriptsDir = AgentPropertiesFileHandler.getPropertyValue(AgentProperties.KVM_SCRIPTS_DIR);
-            String scriptPath = Script.findScript(kvmScriptsDir, "kvmheartbeat.sh");
-            if (scriptPath == null) {
-                throw new CloudRuntimeException("Unable to find heartbeat script 'kvmheartbeat.sh' in directory: " + kvmScriptsDir);
-            }
-            return scriptPath;
+            return findKvmHaScript("kvmheartbeat.sh");
         } else if (StoragePoolType.SharedMountPoint.equals(type)) {
-            String kvmScriptsDir = AgentPropertiesFileHandler.getPropertyValue(AgentProperties.KVM_SCRIPTS_DIR);
-            String scriptPath = Script.findScript(kvmScriptsDir, "kvmsmpheartbeat.sh");
-            if (scriptPath == null) {
-                throw new CloudRuntimeException("Unable to find heartbeat script 'kvmsmpheartbeat.sh' in directory: " + kvmScriptsDir);
-            }
-            return scriptPath;
+            return findKvmHaScript("kvmsmpheartbeat.sh");
+        } else if (StoragePoolType.RBD.equals(type)) {
+            return findKvmHaScript("kvmheartbeat_rbd.sh");
         }
         return null;
     }
 
+    private String findKvmHaScript(String scriptName) {
+        String kvmScriptsDir = AgentPropertiesFileHandler.getPropertyValue(AgentProperties.KVM_SCRIPTS_DIR);
+        String scriptPath = Script.findScript(kvmScriptsDir, scriptName);
+        if (scriptPath == null) {
+            throw new CloudRuntimeException(String.format("Unable to find heartbeat script '%s' in directory: %s", scriptName, kvmScriptsDir));
+        }
+        return scriptPath;
+    }
+
+    /**
+     * Adds the Ceph cluster connection details (monitors, pool and, if cephx is enabled, credentials)
+     * to a heartbeat/VM-activity check {@link Script} for a RBD storage pool. Mirrors the "mon_host"/"id"/"key"
+     * options that qemu itself uses to talk to RBD (see {@link KVMPhysicalDisk#RBDStringBuilder}).
+     */
+    private void addRbdConnectionArgs(Script cmd) {
+        cmd.add("-s", sourceHost);
+        cmd.add("-o", sourceDir);
+        if (authUsername != null) {
+            cmd.add("-n", authUsername);
+            cmd.add("-k", authSecret);
+        }
+    }
+
+    /**
+     * Adds the arguments identifying the storage to a heartbeat/VM-activity check {@link Script}:
+     * the Ceph connection details for a RBD pool, or the NFS server, path and mount point otherwise.
+     */
+    private void addPoolConnectionArgs(Script cmd, HAStoragePool pool) {
+        if (StoragePoolType.RBD.equals(type)) {
+            addRbdConnectionArgs(cmd);
+        } else {
+            cmd.add("-i", pool.getPoolIp());
+            cmd.add("-p", pool.getPoolMountSourcePath());
+            cmd.add("-m", pool.getMountDestPath());
+        }
+    }
 
     public String createHeartBeatCommand(HAStoragePool primaryStoragePool, String hostPrivateIp, boolean hostValidation) {
         Script cmd = new Script(primaryStoragePool.getPool().getHearthBeatPath(), HeartBeatUpdateTimeoutInMs, logger);
-        cmd.add("-i", primaryStoragePool.getPoolIp());
-        cmd.add("-p", primaryStoragePool.getPoolMountSourcePath());
-        cmd.add("-m", primaryStoragePool.getMountDestPath());
+        addPoolConnectionArgs(cmd, primaryStoragePool);
 
         if (hostValidation) {
             cmd.add("-h", hostPrivateIp);
@@ -377,9 +403,7 @@ public String getStorageNodeId() {
     public Boolean hasHeartBeat(HAStoragePool pool, HostTO host) {
         String hostIp = host.getPrivateNetwork().getIp();
         Script cmd = new Script(getHearthBeatPath(), HeartBeatCheckerTimeoutInMs, logger);
-        cmd.add("-i", pool.getPoolIp());
-        cmd.add("-p", pool.getPoolMountSourcePath());
-        cmd.add("-m", pool.getMountDestPath());
+        addPoolConnectionArgs(cmd, pool);
         cmd.add("-h", hostIp);
         cmd.add("-r");
         cmd.add("-t", String.valueOf(HeartBeatUpdateFreqInMs / 1000));
@@ -401,10 +425,12 @@ public Boolean hasHeartBeat(HAStoragePool pool, HostTO host) {
     @Override
     public Boolean hasVmActivity(HAStoragePool pool, HostTO host, Duration activityScriptTimeout, String volumeUUIDListString, String vmActivityCheckPath, long duration) {
         String hostIp = host.getPrivateNetwork().getIp();
-        Script cmd = new Script(vmActivityCheckPath, activityScriptTimeout.getStandardSeconds(), logger);
-        cmd.add("-i", pool.getPoolIp());
-        cmd.add("-p", pool.getPoolMountSourcePath());
-        cmd.add("-m", pool.getMountDestPath());
+        // RBD volumes have no shared mount point to stat(), so the RBD-specific script
+        // (using RBD watchers to detect activity) is used instead of the generic,
+        // NFS/SharedMountPoint-oriented script path passed in by the caller.
+        String scriptPath = StoragePoolType.RBD.equals(type) ? findKvmHaScript("kvmvmactivity_rbd.sh") : vmActivityCheckPath;
+        Script cmd = new Script(scriptPath, activityScriptTimeout.getStandardSeconds(), logger);
+        addPoolConnectionArgs(cmd, pool);
         cmd.add("-h", hostIp);
         cmd.add("-u", volumeUUIDListString);
         cmd.add("-t", String.valueOf(System.currentTimeMillis() / 1000));

plugins/hypervisors/kvm/src/test/java/com/cloud/hypervisor/kvm/storage/LibvirtStoragePoolTest.java

@@ -98,4 +98,23 @@ public void testExternalSnapshot() {
         LibvirtStoragePool clvmPool = new LibvirtStoragePool(uuid, name, StoragePoolType.CLVM, adapter, storage);
         assertTrue(clvmPool.isExternalSnapshot());
     }
+
+    @Test
+    public void testIsPoolSupportHA() {
+        String uuid = "0f7a58bd-1a85-4b1f-9f91-12f3d1ecf5a5";
+        String name = "myfirstpool";
+
+        StorageAdaptor adapter = Mockito.mock(LibvirtStorageAdaptor.class);
+        StoragePool storage = Mockito.mock(StoragePool.class);
+
+        // NetworkFilesystem, SharedMountPoint and RBD all support the KVM Host-HA
+        // heartbeat/VM-activity check mechanism.
+        assertTrue(new LibvirtStoragePool(uuid, name, StoragePoolType.NetworkFilesystem, adapter, storage).isPoolSupportHA());
+        assertTrue(new LibvirtStoragePool(uuid, name, StoragePoolType.SharedMountPoint, adapter, storage).isPoolSupportHA());
+        assertTrue(new LibvirtStoragePool(uuid, name, StoragePoolType.RBD, adapter, storage).isPoolSupportHA());
+
+        // Other pool types have no HA support.
+        assertFalse(new LibvirtStoragePool(uuid, name, StoragePoolType.CLVM, adapter, storage).isPoolSupportHA());
+        assertFalse(new LibvirtStoragePool(uuid, name, StoragePoolType.Filesystem, adapter, storage).isPoolSupportHA());
+    }
 }

scripts/vm/hypervisor/kvm/kvmheartbeat_rbd.sh

@@ -0,0 +1,143 @@
+#!/bin/bash
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+# Ceph RBD flavor of kvmheartbeat.sh/kvmsmpheartbeat.sh.
+#
+# There is no shared POSIX mount point to write a heartbeat file to when the
+# primary storage pool is Ceph RBD, so the heartbeat timestamp is instead
+# stored as a small RADOS object (one object per host) in the same RBD pool.
+# Any host with a working path to the Ceph cluster can write/read this object,
+# which gives the same semantics as the NFS/SharedMountPoint heartbeat file.
+
+help() {
+  printf "Usage: $0
+                    -s ceph monitor host(s), comma separated
+                    -o ceph/rbd pool name
+                    -n cephx auth user (optional)
+                    -k cephx auth key, base64 (optional, required if -n is set)
+                    -h host
+                    -r write/read hb log
+                    -c cleanup
+                    -t interval between read hb log\n"
+  exit 1
+}
+#set -x
+MonHosts=
+PoolName=
+CephUser=
+CephKey=
+HostIP=
+interval=
+rflag=0
+cflag=0
+
+while getopts 's:o:n:k:h:t:rc' OPTION
+do
+  case $OPTION in
+  s)
+     MonHosts="$OPTARG"
+     ;;
+  o)
+     PoolName="$OPTARG"
+     ;;
+  n)
+     CephUser="$OPTARG"
+     ;;
+  k)
+     CephKey="$OPTARG"
+     ;;
+  h)
+     HostIP="$OPTARG"
+     ;;
+  r)
+     rflag=1
+     ;;
+  t)
+     interval="$OPTARG"
+     ;;
+  c)
+    cflag=1
+     ;;
+  *)
+     help
+     ;;
+  esac
+done
+
+if [ -z "$MonHosts" ] || [ -z "$PoolName" ]
+then
+   exit 1
+fi
+
+RadosOpts=(--mon-host "$MonHosts")
+if [ -n "$CephUser" ]
+then
+   RadosOpts+=(--id "$CephUser" --key "$CephKey")
+fi
+
+hbObject="KVMHA-hb-$HostIP"
+
+write_hbLog() {
+  tmpFile=$(mktemp)
+  date +%s > "$tmpFile"
+  rados -p "$PoolName" "${RadosOpts[@]}" put "$hbObject" "$tmpFile" &> /dev/null
+  rc=$?
+  rm -f "$tmpFile"
+  return $rc
+}
+
+check_hbLog() {
+  now=$(date +%s)
+  hb=$(rados -p "$PoolName" "${RadosOpts[@]}" get "$hbObject" - 2> /dev/null)
+  if ! [[ "$hb" =~ ^[0-9]+$ ]]
+  then
+    # Either the RADOS object doesn't exist yet (host never wrote a heartbeat)
+    # or the Ceph cluster can't be reached right now. Either way we can't
+    # confirm the host is alive, so fail safe and report it as DEAD.
+    return 255
+  fi
+  diff=$(expr $now - $hb)
+  if [ $diff -gt $interval ]
+  then
+    return $diff
+  fi
+  return 0
+}
+
+if [ "$rflag" == "1" ]
+then
+  check_hbLog
+  diff=$?
+  if [ $diff == 0 ]
+  then
+    echo "=====> ALIVE <====="
+  else
+    echo "=====> Considering host as DEAD because last write to RADOS object [$hbObject] in pool [$PoolName] was [$diff] seconds ago, but the max interval is [$interval] <======"
+  fi
+  exit 0
+elif [ "$cflag" == "1" ]
+then
+  /usr/bin/logger -t heartbeat "kvmheartbeat_rbd.sh will reboot system because it was unable to write the heartbeat to the Ceph RBD storage."
+  sync &
+  sleep 5
+  echo b > /proc/sysrq-trigger
+  exit $?
+else
+  write_hbLog
+  exit $?
+fi