fix(orchestrator): fix runtime crashes and test automation API compatibility

.config/ansible-lint.yml

@@ -2,35 +2,19 @@
 exclude_paths:
   - .git/
   - .github/
-  - accelerator/tests/
-  - network/tests/
-  - provision/tests/
-  - scheduler/tests/
-  - security/tests/
-  - storage/tests/
-  - test/
-  - utils/obsolete/
   - docs/
-  - platforms/
-  - examples/
-  - input/
   - .ansible-lint.yml
   - .readthedocs.yaml
-  - prepare_oim/roles/configure_proxy/tasks/configure_proxy_rocky.yml
-  - upgrade/roles/upgrade_idrac_telemetry/tasks/filter_idrac.yml
-  - utils/server_spec_update/roles/os_update/tasks/kcmdline_update_rocky.yml
-  - utils/roles/oim_cleanup/vars/rocky.yml
-  - scheduler/roles/k8s_start_services/files/k8s_dashboard_admin.yaml
-  - scheduler/playbooks/k8s_add_node.yml
-  - "*ubuntu*"
-  - "*rocky*"
+  - .github/workflows/
+  - .github/workflows/ansible-lint.yml
 
 skip_list:
   - var-naming
   - unresolved-module
   - fqcn[canonical]
   - internal-error
   - role-name[path]
+  - galaxy[no-changelog]
 
 verbosity: 1
 profile: production

.github/workflows/.gitleaks.toml

@@ -0,0 +1,170 @@
+title = "Gitleaks config for Omnia"
+
+# ---------------------------------------------------------------------------
+# Extend the default gitleaks ruleset so AWS, GCP, private-key, JWT, and
+# all other built-in detections remain active without manual maintenance.
+# ---------------------------------------------------------------------------
+[extend]
+useDefault = true
+
+# ===========================  GLOBAL ALLOWLIST  ============================
+[allowlist]
+  description = "Ignore known false positives for Omnia infra repos"
+
+  paths = [
+    '''(^|/)\.git/''',
+    '''(^|/)vendor/''',
+    '''go\.sum$''',
+    '''package-lock\.json$''',
+    '''yarn\.lock$''',
+    '''poetry\.lock$''',
+    '''\.css\.map$''',
+  ]
+
+  regexTarget = "match"
+
+  regexes = [
+    # ------------------------------------------------------------------
+    # 1. Test / example / demo / dummy credentials
+    # ------------------------------------------------------------------
+    '''(?i)(example|dummy|test|demo)[_-]?(password|secret|token|key)''',
+    '''(?i)(CHANGEME|changeme)''',
+    '''(?i)password:\s*dell\d+''',
+    '''(?i)PASSWORD\s*=\s*"?dell\d+''',
+    '''(?i)password:\s*"?slurmPassword"?''',
+    '''(?i)VAULT_PASSWORD.*omnia_test''',
+    '''(?i)PASSWORD\s*=\s*"omnia_test''',
+    '''(?i)password:\s*"?correct_password"''',
+
+    # ------------------------------------------------------------------
+    # 2. Ansible / Jinja2 variable references (never real secrets)
+    # ------------------------------------------------------------------
+    '''(?i)password:\s*"?\{\{.*\}\}"?''',
+    '''(?i)ansible_.*password.*\{\{''',
+    '''(?i)password:\s*"?hostvars''',
+    '''(?i)password:\s*\$\{''',
+    '''(?i)password:\s*\$[A-Z_]+''',
+
+    # ------------------------------------------------------------------
+    # 3. Variable names used as values (not actual secrets)
+    # ------------------------------------------------------------------
+    '''(?i)(token|secret|password)\s*=\s*[a-z_]+$''',
+    '''(?i)password:\s*[a-z_]+(password|secret|key|token)''',
+    '''(?i)secret:\s*[a-z_]+(password|secret|key|token)''',
+    '''(?i)token:\s*[a-z_]+(password|secret|key|token)''',
+    '''(?i)password:\s*\{\s*password:''',
+    '''(?i)docker_password_cipher''',
+    '''(?i)load_docker_credentials''',
+    '''(?i)password:\s*s3_secret_key''',
+    '''(?i)password:\s*minio_s3_password''',
+    '''(?i)password:\s*switch_snmp3_password''',
+    '''(?i)password:\s*"(mysql|grafana|switch|kerberos_admin|directory_manager)[_a-z]*_password"?''',
+    '''(?i)secret:\s*"grafana_''',
+    '''(?i)password:\s*aarch64_ssh_password''',
+    '''(?i)password:\s*"[a-z_]+_password"''',
+    '''(?i)[a-z_]+_password\s*=.*(OMNIA_SH_VARS|get\()''',
+    '''(?i)password\s*=\s*(OMNIA_SH_VARS|[a-z_]+\.get\()''',
+
+    # ------------------------------------------------------------------
+    # 4. Instructional / documentation / user-facing messages
+    #    (the pattern that caused the original false positive)
+    # ------------------------------------------------------------------
+    '''(?i)user/password''',
+    '''(?i)set-password''',
+    '''(?i)setup_env\.sh''',
+    '''(?i)Verify.*password''',
+
+    # ------------------------------------------------------------------
+    # 5. Validation / success / failure messages
+    # ------------------------------------------------------------------
+    '''(?i)success_msg.*password.*validated''',
+    '''(?i)fail_msg.*password''',
+    '''(?i)_password.*validated''',
+    '''(?i)msg.*password.*valid''',
+
+    # ------------------------------------------------------------------
+    # 6. Database connection strings (localhost / templates)
+    # ------------------------------------------------------------------
+    '''(?i)postgresql://.*@(localhost|127\.0\.0\.1)''',
+    '''(?i)postgresql://user:pass@host''',
+    '''(?i)postgresql://.*%\([^)]+\)s''',
+    '''(?i)%\([^)]+\)s''',
+    '''(?i)(timescaledb_)?password:\s*postgres''',
+    '''(?i)PASSWORD:\s*postgres''',
+
+    # ------------------------------------------------------------------
+    # 7. Shell / hashing commands (not actual secrets)
+    # ------------------------------------------------------------------
+    '''(?i)passwd:\s*\$''',
+    '''(?i)passwd=?\$\(openssl''',
+    '''(?i)hashed_passwd=\$\(openssl''',
+    '''(?i)openssl passwd''',
+    '''(?i)passwd:key=''',
+
+    # ------------------------------------------------------------------
+    # 8. Config field descriptions / schema definitions (no real values)
+    # ------------------------------------------------------------------
+    '''(?i)password:\s*(Optional|Password|"?Password"?|"?Openldap|"?Registration|None)''',
+    '''(?i)password:\s*"?password"?''',
+    '''(?i)password\s*=\s*IntegrationTestConfig''',
+    '''(?i)password:.*description''',
+    '''(?i)password:.*request_args''',
+    '''(?i)password:.*database''',
+    '''(?i)password\s*=\s*AUTH_PASSWORD''',
+    '''(?i)password\s*=\s*\$MINIO_PASSWORD''',
+    '''(?i)password:\s*$''',
+    '''(?i)password:\s+(description|required|type:)''',
+    '''(?i)password=None''',
+
+    # ------------------------------------------------------------------
+    # 9. Documentation placeholder tokens
+    # ------------------------------------------------------------------
+    '''(?i)TOKEN=hf_x+''',
+    '''(?i)vault_password="x+''',
+    '''(?i)Password:\s*\d{8}''',
+    '''(?i)secret\s*=\s*"bld_s_[A-Za-z0-9_-]+"''',
+
+    # ------------------------------------------------------------------
+    # 10. Known test / example tokens
+    # ------------------------------------------------------------------
+    '''1c8572f630701e8792bede122ec9c417''',
+    '''(?i)(secretToken|cookieSecret).*1c8572f6''',
+
+    # ------------------------------------------------------------------
+    # 11. Certificate references (not secrets)
+    # ------------------------------------------------------------------
+    '''(?i)secret:.*-(cert|ca-cert)''',
+  ]
+
+# ==========================  CUSTOM RULES  =================================
+# These supplement the default ruleset pulled in via [extend].
+# Each rule uses secretGroup so only the VALUE is flagged, not the key name.
+# ---------------------------------------------------------------------------
+
+[[rules]]
+  id          = "generic-password"
+  description = "Generic Password Detection"
+  regex       = '''(?i)(password|passwd|pwd)\s*[:=]\s*["']?([A-Za-z0-9!@#$%^&*()_+=\-]{8,})["']?'''
+  secretGroup = 2
+  tags        = ["password"]
+
+[[rules]]
+  id          = "credentials-in-url"
+  description = "Credentials in URL"
+  regex       = '''(?i)\b\w+:\/\/[^:\s]+:([^@\s]+)@[^:\s]+'''
+  secretGroup = 1
+  tags        = ["credentials", "url"]
+
+[[rules]]
+  id          = "generic-token"
+  description = "Generic Token/Secret"
+  regex       = '''(?i)(secret|token|api[_-]?key)\s*[:=]\s*["']?([A-Za-z0-9_\-]{16,})["']?'''
+  secretGroup = 2
+  tags        = ["token", "secret"]
+
+[[rules]]
+  id          = "ansible-secret"
+  description = "Ansible hardcoded secret"
+  regex       = '''(?i)(ansible_.*password|vault_password)\s*[:=]\s*["']([^"']{6,})["']'''
+  secretGroup = 2
+  tags        = ["ansible", "secret"]

.github/workflows/README.md

@@ -0,0 +1,114 @@
+# CI/CD Workflows -- omnia-bsm
+
+All workflows run on pull requests targeting `main`, `staging`, `release_*`, `issue-*`, and `pub/**` branches.
+
+## Workflow Summary
+
+| # | Workflow | File | Jobs | Gate | Description |
+|---|---------|------|------|------|-------------|
+| 1 | **Ansible Lint** | `ansible-lint.yml` | 1 | Blocking | Runs `ansible-lint` with production profile (FQCN, named tasks, module-vs-shell) |
+| 2 | **Bandit Security Scan** | `bandit.yml` | 1 | Blocking | Python SAST -- `bandit -r` to detect security issues in Python code |
+| 3 | **Commit Hygiene** | `commit-hygiene.yml` | 3 | Blocking (Job 1) | Validates commit authors, messages, copyright headers, and test co-changes |
+| 4 | **HPC Compliance Scanner** | `ansible-module-lint.yml` | 1 | Mixed | HPC anti-patterns + Checkmarx pre-scan (see below) |
+| 5 | **Secret Leak Scan** | `gitleaks.yml` | 1 | Blocking | Scans for secrets and credentials using `gitleaks` with custom `.gitleaks.toml` |
+| 6 | **Dependency Vulnerability Scan** | `pip-audit.yml` | 1 | Blocking | `pip-audit` scans Python dependencies for known CVEs |
+| 7 | **Pylint** | `pylint.yml` | 1 | Blocking | Lint Python code -- minimum score >= 8.0 per file |
+| 8 | **Unit Tests & Coverage** | `pytest.yml` | 1 | Blocking | Runs `pytest` with coverage reporting |
+| 9 | **ShellCheck** | `shellcheck.yml` | 1 | Blocking | Static analysis of shell scripts |
+
+**Total: 9 workflows, 11 jobs**
+
+> **Note:** YAML linting is handled by `ansible-lint` (production profile). A separate `yamllint` workflow is not required.
+
+---
+
+## HPC Compliance Scanner Details
+
+The `ansible-module-lint.yml` workflow enforces Omnia-specific HPC rules that `ansible-lint` does not cover.
+
+### Ansible Checks (Advisory)
+
+| Check | What It Detects | Style Guide Reference |
+|-------|----------------|----------------------|
+| `loop:` + `delegate_to:` | Potential serial fan-out across 1000 nodes -- manual review required | `ansible.md` §13.1 |
+| `with_items:` + `delegate_to:` | Potential legacy serial fan-out pattern -- manual review required | `ansible.md` §13.1 |
+
+### Python Checks -- Blocking (Errors)
+
+| Check | What It Detects | Style Guide Reference |
+|-------|----------------|----------------------|
+| `shell=True` | OS Command Injection risk in subprocess | `python.md` §8.3 |
+| `os.system()` | OS Command Injection | `python.md` §8.3 |
+| `eval()` | Code Injection | `python.md` §8.3 |
+| `exec()` | Dynamic code execution | `python.md` §8.3 |
+| `yaml.load()` | Insecure YAML deserialization | `python.md` §8.3 |
+| `yaml.full_load()` | Unsafe YAML loader | `python.md` §8.3 |
+| `yaml.UnsafeLoader` / `yaml.FullLoader` | Unsafe YAML loader classes | `python.md` §8.3 |
+
+### Python Checks -- Advisory (Warnings)
+
+| Check | What It Detects | Note |
+|-------|----------------|------|
+| `pickle.loads()` | Potentially unsafe deserialization | May have valid internal uses |
+| Hardcoded credentials | `password`, `secret`, `api_key`, `token`, `access_token`, `auth_token` patterns | Excludes `test/`, `examples/`, `docs/`, `build/` |
+
+**Design principle:** This workflow only checks rules that `ansible-lint` cannot detect. All FQCN, module-vs-shell, named-task, and bare-variable checks are handled by `ansible-lint` with the production profile.
+
+---
+
+## Quality Gate Summary
+
+### Code Quality
+
+| Gate | Tool | Threshold | Reference |
+|------|------|-----------|-----------|
+| Ansible Lint | `ansible-lint` | Zero errors (production profile) | `ansible.md` §14.1 |
+| Pylint | `pylint` | Score >= 8.0 per file | `python.md` §7.1 |
+| ShellCheck | `shellcheck` | Zero errors | `ansible.md` §14 |
+
+### Security
+
+| Gate | Tool | Threshold | Reference |
+|------|------|-----------|-----------|
+| Python SAST | `bandit` | Zero High/Critical | `python.md` §7 |
+| Secret Leak | `gitleaks` | Zero findings | `ansible.md` §14.4 |
+| Dependency CVE | `pip-audit` | Zero known vulnerabilities | `python.md` §7 |
+| Checkmarx Pre-scan | HPC Compliance Scanner | No `shell=True`, `os.system()`, `eval()`, `exec()`, unsafe `yaml.load()` | `python.md` §8.3 |
+
+---
+
+## Commit Hygiene Details
+
+The `commit-hygiene.yml` workflow enforces the AI Agent Usage Policy from `docs/code-style/general.md`:
+
+| Job | Check | Severity |
+|-----|-------|----------|
+| **Commit Validation** | Block commits authored by AI bots (Devin, Codex, Copilot, etc.) | ERROR |
+| | Block commits from root user | ERROR |
+| | Validate `<type>(<scope>): <description>` format | WARN |
+| | Block trivially short commit messages (<10 chars) | ERROR |
+| | Detect LLM-style language in commit messages | WARN |
+| **Copyright Header** | Check Dell Apache 2.0 copyright header in new/changed source files | WARN (advisory) |
+| **Test Co-Change** | Warn when `src/` changes without `test/` updates | WARN (advisory) |
+
+**Note:** `Co-Authored-By` trailers are acceptable. The check validates the primary Author and Committer fields, not trailers.
+
+---
+
+## Security Scanning
+
+| Scanner | Tool | What It Checks |
+|---------|------|----------------|
+| SAST | `bandit` | Python security anti-patterns (hardcoded passwords, SQL injection, etc.) |
+| Checkmarx Pre-scan | HPC Compliance Scanner | `shell=True`, `os.system()`, `eval()`, `exec()`, `yaml.load()`, hardcoded credentials |
+| Secrets | `gitleaks` | Leaked credentials, API keys, tokens in code and history |
+| Dependencies | `pip-audit` | Known CVEs in Python package dependencies |
+
+---
+
+## Adding a New Workflow
+
+1. Create the workflow file in `.github/workflows/`
+2. Use the standard branch triggers: `main`, `staging`, `release_*`, `issue-*`, `pub/**`
+3. Update this README with the new workflow details
+4. Ensure the workflow follows the commit format: `ci(workflows): <description>`