feat(telemetry,test): status reporting redesign, external_victoria, SFM removal, credential utility, Vast Automation

src/telemetry/galaxy.yml

@@ -21,7 +21,7 @@ authors:
 description: >-
   Dell Omnia Telemetry — deploys and manages a telemetry stack for HPC and AI
   clusters. Supports VictoriaMetrics, VictoriaLogs, Kafka (Strimzi), LDMS,
-  iDRAC, UFM, OME, PowerScale, VAST, and SFM
+  iDRAC, UFM, OME, PowerScale, and VAST
   telemetry sources with tag-based deployment and cleanup.
 license:
   - Apache-2.0

src/telemetry/playbooks/cleanup/cleanup.yml

@@ -44,7 +44,7 @@
 # Available tags:
 #   Sinks:   cleanup_kafka, cleanup_victoria_metrics, cleanup_victoria_logs
 #   Sources: cleanup_idrac, cleanup_ldms, cleanup_ome, cleanup_powerscale,
-#            cleanup_ufm, cleanup_vast, cleanup_sfm
+#            cleanup_ufm, cleanup_vast
 # Design: Sinks (Kafka, VictoriaMetrics, VictoriaLogs) are shared
 # infrastructure that multiple sources depend on. They are ONLY cleaned
 # when --tags cleanup (all) is used, never individually.
@@ -63,7 +63,7 @@
   connection: local
   gather_facts: false
   any_errors_fatal: true
-  tags: [cleanup, cleanup_idrac, cleanup_ldms, cleanup_ome, cleanup_powerscale, cleanup_ufm, cleanup_vast, cleanup_sfm, cleanup_kafka, cleanup_victoria_metrics, cleanup_victoria_logs]  # noqa: yaml[line-length]
+  tags: [cleanup, cleanup_idrac, cleanup_ldms, cleanup_ome, cleanup_powerscale, cleanup_ufm, cleanup_vast, cleanup_kafka, cleanup_victoria_metrics, cleanup_victoria_logs]  # noqa: yaml[line-length]
   tasks:
     - name: Load telemetry configuration
       ansible.builtin.include_role:
@@ -93,7 +93,6 @@
         _cleanup_powerscale: "{{ 'cleanup_powerscale' in ansible_run_tags }}"
         _cleanup_ufm: "{{ 'cleanup_ufm' in ansible_run_tags }}"
         _cleanup_vast: "{{ 'cleanup_vast' in ansible_run_tags }}"
-        _cleanup_sfm: "{{ 'cleanup_sfm' in ansible_run_tags }}"
         cacheable: true
 
 # ---------------------------------------------------------------------------
@@ -140,20 +139,14 @@
         tasks_from: vast
       when: hostvars['localhost']['_cleanup_all'] | bool or hostvars['localhost']['_cleanup_vast'] | bool
 
-    - name: "Cleanup SFM telemetry"
-      ansible.builtin.include_role:
-        name: cleanup
-        tasks_from: sfm
-      when: hostvars['localhost']['_cleanup_all'] | bool or hostvars['localhost']['_cleanup_sfm'] | bool
-
     # -- Pre-sink guard: verify source pods have terminated --
     - name: "Check for remaining source pods before sink cleanup"
       vars:
         telemetry_namespace: "telemetry"
         _source_apps: >-
           idrac-telemetry,nersc-ldms,vector-ome,vector-ldms,
           karavi-metrics-powerscale,otel-collector,
-          csi-volume-exporter,sfm-telemetry,
+          csi-volume-exporter,
           ufm-external,vast-external,skyway-telemetry,powervault-telemetry
       ansible.builtin.command: >
         kubectl get pods -n {{ telemetry_namespace }}
@@ -179,7 +172,7 @@
         _source_apps: >-
           idrac-telemetry,nersc-ldms,vector-ome,vector-ldms,
           karavi-metrics-powerscale,otel-collector,
-          csi-volume-exporter,sfm-telemetry,
+          csi-volume-exporter,
           ufm-external,vast-external,skyway-telemetry,powervault-telemetry
       ansible.builtin.command: >
         kubectl wait --for=delete pods
@@ -216,7 +209,7 @@
 - name: "Telemetry Cleanup | Verification"
   hosts: kube_vip_group
   gather_facts: false
-  tags: [cleanup, cleanup_idrac, cleanup_ldms, cleanup_ome, cleanup_powerscale, cleanup_ufm, cleanup_vast, cleanup_sfm, cleanup_kafka, cleanup_victoria_metrics, cleanup_victoria_logs]  # noqa: yaml[line-length]
+  tags: [cleanup, cleanup_idrac, cleanup_ldms, cleanup_ome, cleanup_powerscale, cleanup_ufm, cleanup_vast, cleanup_kafka, cleanup_victoria_metrics, cleanup_victoria_logs]  # noqa: yaml[line-length]
   vars:
     telemetry_namespace: "telemetry"
   tasks:
@@ -245,7 +238,7 @@
   hosts: localhost
   connection: local
   gather_facts: false
-  tags: [cleanup, cleanup_idrac, cleanup_ldms, cleanup_ome, cleanup_powerscale, cleanup_ufm, cleanup_vast, cleanup_sfm, cleanup_kafka, cleanup_victoria_metrics, cleanup_victoria_logs]  # noqa: yaml[line-length]
+  tags: [cleanup, cleanup_idrac, cleanup_ldms, cleanup_ome, cleanup_powerscale, cleanup_ufm, cleanup_vast, cleanup_kafka, cleanup_victoria_metrics, cleanup_victoria_logs]  # noqa: yaml[line-length]
   tasks:
     - name: Display cleanup summary
       vars:

src/telemetry/playbooks/deploy/deploy.yml

@@ -69,11 +69,6 @@
     hostvars['localhost']['telemetry_config'].telemetry_bridges.vector_ome.metrics_enabled | default(false) | bool or
     hostvars['localhost']['telemetry_config'].telemetry_bridges.vector_ome.logs_enabled | default(false) | bool
 
-- name: "Phase 2 | Deploy SFM telemetry"
-  ansible.builtin.import_playbook: sources/deploy_sfm.yml
-  tags: deploy
-  when: hostvars['localhost']['telemetry_config'].telemetry_sources.sfm.metrics_enabled | default(false) | bool
-
 # ── Phase 3: Reconciliation — root kustomization + full-stack apply ───────────
 - name: "Reconcile full-stack deployment via kustomization"
   hosts: kube_vip_group
@@ -148,6 +143,239 @@
           Check pod logs with: kubectl logs -n {{ telemetry_namespace }} <pod-name>
       when: failed_pods | length > 0
 
+# ── Phase 4: Collect deployment results ───────────────────────────────────────
+# Inspect cluster state to determine passed/failed/skipped for each component
+- name: "Collect deployment results"
+  hosts: kube_vip_group
+  gather_facts: false
+  vars_files:
+    - ../../roles/common/vars/main.yml
+  tags: deploy
+  tasks:
+    # Load config for checking enabled flags
+    - name: Load telemetry config from localhost
+      ansible.builtin.set_fact:
+        _cfg: "{{ hostvars['localhost']['telemetry_config'] }}"
+
+    # -- Sink checks ----------------------------------------------------------
+    - name: Check Kafka pods
+      ansible.builtin.command: >
+        kubectl get pods -n {{ telemetry_namespace }} -l strimzi.io/cluster=kafka --no-headers
+      register: _kafka_pods
+      changed_when: false
+      failed_when: false
+
+    - name: Check VictoriaMetrics pods
+      ansible.builtin.command: >
+        kubectl get pods -n {{ telemetry_namespace }} -l app.kubernetes.io/name=vmstorage --no-headers
+      register: _vm_pods
+      changed_when: false
+      failed_when: false
+
+    - name: Check VictoriaLogs pods
+      ansible.builtin.command: >
+        kubectl get pods -n {{ telemetry_namespace }} -l app.kubernetes.io/name=vlstorage --no-headers
+      register: _vl_pods
+      changed_when: false
+      failed_when: false
+
+    # -- Source checks --------------------------------------------------------
+    - name: Check iDRAC pods
+      ansible.builtin.command: >
+        kubectl get pods -n {{ telemetry_namespace }} -l app=idrac-telemetry --no-headers
+      register: _idrac_pods
+      changed_when: false
+      failed_when: false
+
+    - name: Check LDMS pods (nersc-ldms)
+      ansible.builtin.command: >
+        kubectl get pods -n {{ telemetry_namespace }} -l app=nersc-ldms --no-headers
+      register: _ldms_pods
+      changed_when: false
+      failed_when: false
+
+    - name: Check PowerScale pods (karavi-metrics-powerscale)
+      ansible.builtin.command: >
+        kubectl get pods -n {{ telemetry_namespace }} -l app.kubernetes.io/name=karavi-metrics-powerscale --no-headers
+      register: _powerscale_pods
+      changed_when: false
+      failed_when: false
+
+    - name: Check UFM service (ufm-external)
+      ansible.builtin.command: >
+        kubectl get svc -n {{ telemetry_namespace }} ufm-external --no-headers
+      register: _ufm_check
+      changed_when: false
+      failed_when: false
+
+    - name: Check VAST service (vast-external)
+      ansible.builtin.command: >
+        kubectl get svc -n {{ telemetry_namespace }} vast-external --no-headers
+      register: _vast_check
+      changed_when: false
+      failed_when: false
+
+    - name: Check Vector-OME pods
+      ansible.builtin.command: >
+        kubectl get pods -n {{ telemetry_namespace }} -l app=vector-ome --no-headers
+      register: _vector_ome_pods
+      changed_when: false
+      failed_when: false
+
+    - name: Check Vector-LDMS pods
+      ansible.builtin.command: >
+        kubectl get pods -n {{ telemetry_namespace }} -l app=vector-ldms --no-headers
+      register: _vector_ldms_pods
+      changed_when: false
+      failed_when: false
+
+    - name: Check VLAgent pods (for logs sinks)
+      ansible.builtin.command: >
+        kubectl get pods -n {{ telemetry_namespace }} -l app.kubernetes.io/name=vlagent --no-headers
+      register: _vlagent_pods
+      changed_when: false
+      failed_when: false
+
+    # -- Store results in hostvars for localhost to read ----------------------
+    - name: Store kubectl results as facts
+      ansible.builtin.set_fact:
+        _kv:
+          _kafka_pods: "{{ _kafka_pods }}"
+          _vm_pods: "{{ _vm_pods }}"
+          _vl_pods: "{{ _vl_pods }}"
+          _idrac_pods: "{{ _idrac_pods }}"
+          _ldms_pods: "{{ _ldms_pods }}"
+          _powerscale_pods: "{{ _powerscale_pods }}"
+          _ufm_check: "{{ _ufm_check }}"
+          _vast_check: "{{ _vast_check }}"
+          _vector_ome_pods: "{{ _vector_ome_pods }}"
+          _vector_ldms_pods: "{{ _vector_ldms_pods }}"
+          _vlagent_pods: "{{ _vlagent_pods }}"
+        cacheable: true
+
+# ── Phase 4b: Set deployment result facts on localhost ────────────────────────
+- name: "Set deployment result facts"
+  hosts: localhost
+  connection: local
+  gather_facts: false
+  vars_files:
+    - ../../roles/common/vars/main.yml
+  tags: deploy
+  tasks:
+    - name: Get kubectl results from kube_vip
+      ansible.builtin.set_fact:
+        _kv: "{{ hostvars[kube_vip]['_kv'] }}"
+        _cfg: "{{ telemetry_config }}"
+
+    # -- Sinks ----------------------------------------------------------------
+    - name: Set sink results
+      vars:
+        _kafka_running: >-
+          {{ (_kv['_kafka_pods'].stdout_lines | default([])
+             | select('search', 'Running') | list | length) > 0 }}
+        _vm_running: >-
+          {{ (_kv['_vm_pods'].stdout_lines | default([])
+             | select('search', 'Running') | list | length) > 0 }}
+        _vl_running: >-
+          {{ (_kv['_vl_pods'].stdout_lines | default([])
+             | select('search', 'Running') | list | length) > 0 }}
+      ansible.builtin.set_fact:
+        _deploy_result_kafka: "{{ 'passed' if (_kafka_running | bool) else 'failed' }}"
+        _deploy_result_victoria_metrics: "{{ 'passed' if (_vm_running | bool) else 'failed' }}"
+        _deploy_result_victoria_logs: "{{ 'passed' if (_vl_running | bool) else 'failed' }}"
+
+    # -- Sources (metrics only) -----------------------------------------------
+    - name: Set source metrics-only results
+      vars:
+        _idrac_on: "{{ _cfg.telemetry_sources.idrac.metrics_enabled | default(false) | bool }}"
+        _idrac_running: >-
+          {{ (_kv['_idrac_pods'].stdout_lines | default([])
+             | select('search', 'Running') | list | length) > 0 }}
+        _ldms_on: "{{ _cfg.telemetry_sources.ldms.metrics_enabled | default(false) | bool }}"
+        _ldms_running: >-
+          {{ (_kv['_ldms_pods'].stdout_lines | default([])
+             | select('search', 'Running') | list | length) > 0
+             or (_kv['_vector_ldms_pods'].stdout_lines | default([])
+             | select('search', 'Running') | list | length) > 0 }}
+      ansible.builtin.set_fact:
+        _deploy_result_idrac_metrics: >-
+          {{ 'skipped' if not (_idrac_on | bool)
+             else ('passed' if (_idrac_running | bool) else 'failed') }}
+        _deploy_result_ldms_metrics: >-
+          {{ 'skipped' if not (_ldms_on | bool)
+             else ('passed' if (_ldms_running | bool) else 'failed') }}
+
+    # -- Sources (metrics + logs) ---------------------------------------------
+    - name: Set source metrics+logs results
+      vars:
+        _ps_m_on: "{{ _cfg.telemetry_sources.powerscale.metrics_enabled | default(false) | bool }}"
+        _ps_l_on: "{{ _cfg.telemetry_sources.powerscale.logs_enabled | default(false) | bool }}"
+        _ps_running: >-
+          {{ (_kv['_powerscale_pods'].stdout_lines | default([])
+             | select('search', 'Running') | list | length) > 0 }}
+        _vlagent_running: >-
+          {{ (_kv['_vlagent_pods'].stdout_lines | default([])
+             | select('search', 'Running') | list | length) > 0 }}
+        _ufm_m_on: "{{ _cfg.telemetry_sources.ufm.metrics_enabled | default(false) | bool }}"
+        _ufm_l_on: "{{ _cfg.telemetry_sources.ufm.logs_enabled | default(false) | bool }}"
+        _ufm_ok: "{{ _kv['_ufm_check'].rc | default(1) == 0 }}"
+        _vast_m_on: "{{ _cfg.telemetry_sources.vast.metrics_enabled | default(false) | bool }}"
+        _vast_l_on: "{{ _cfg.telemetry_sources.vast.logs_enabled | default(false) | bool }}"
+        _ome_m_on: "{{ _cfg.telemetry_sources.ome.metrics_enabled | default(false) | bool }}"
+        _ome_l_on: "{{ _cfg.telemetry_sources.ome.logs_enabled | default(false) | bool }}"
+        _vome_running: >-
+          {{ (_kv['_vector_ome_pods'].stdout_lines | default([])
+             | select('search', 'Running') | list | length) > 0 }}
+      ansible.builtin.set_fact:
+        _deploy_result_powerscale_metrics: >-
+          {{ 'skipped' if not (_ps_m_on | bool)
+             else ('passed' if (_ps_running | bool) else 'failed') }}
+        _deploy_result_powerscale_logs: >-
+          {{ 'skipped' if not (_ps_l_on | bool)
+             else ('passed' if (_vlagent_running | bool) else 'failed') }}
+        _deploy_result_ufm_metrics: >-
+          {{ 'skipped' if not (_ufm_m_on | bool)
+             else ('passed' if (_ufm_ok | bool) else 'failed') }}
+        _deploy_result_ufm_logs: >-
+          {{ 'skipped' if not (_ufm_l_on | bool)
+             else ('passed' if (_vlagent_running | bool) else 'failed') }}
+        _deploy_result_vast_metrics: >-
+          {{ 'skipped' if not (_vast_m_on | bool) else 'passed' }}
+        _deploy_result_vast_logs: >-
+          {{ 'skipped' if not (_vast_l_on | bool) else 'passed' }}
+        _deploy_result_ome_metrics: >-
+          {{ 'skipped' if not (_ome_m_on | bool)
+             else ('passed' if (_vome_running | bool) else 'failed') }}
+        _deploy_result_ome_logs: >-
+          {{ 'skipped' if not (_ome_l_on | bool)
+             else ('passed' if (_vome_running | bool) else 'failed') }}
+
+    # -- Bridges --------------------------------------------------------------
+    # Bridge status depends on both bridge config AND source config
+    - name: Set bridge results
+      vars:
+        _vldms_bridge_on: >-
+          {{ _cfg.telemetry_bridges.vector_ldms.metrics_enabled | default(false) | bool }}
+        _ldms_src_on: >-
+          {{ _cfg.telemetry_sources.ldms.metrics_enabled | default(false) | bool }}
+        _vldms_running: >-
+          {{ (_kv['_vector_ldms_pods'].stdout_lines | default([])
+             | select('search', 'Running') | list | length) > 0 }}
+        _vome_m: >-
+          {{ _cfg.telemetry_bridges.vector_ome.metrics_enabled | default(false) | bool }}
+        _vome_l: >-
+          {{ _cfg.telemetry_bridges.vector_ome.logs_enabled | default(false) | bool }}
+        _vome_running: >-
+          {{ (_kv['_vector_ome_pods'].stdout_lines | default([])
+             | select('search', 'Running') | list | length) > 0 }}
+      ansible.builtin.set_fact:
+        _deploy_result_vector_ldms: >-
+          {{ 'skipped' if not ((_vldms_bridge_on | bool) and (_ldms_src_on | bool))
+             else ('passed' if (_vldms_running | bool) else 'failed') }}
+        _deploy_result_vector_ome: >-
+          {{ 'skipped' if not ((_vome_m | bool) or (_vome_l | bool))
+             else ('passed' if (_vome_running | bool) else 'failed') }}
+
 # ── Phase 5: Write telemetry_status.yml ───────────────────────────────────────
 - name: "Write telemetry deployment status"
   hosts: localhost