From 66b72e1ab9cc11ab12cecdd775b832a9077db4f7 Mon Sep 17 00:00:00 2001 From: alemacci Date: Sun, 16 Aug 2026 14:55:40 +0200 Subject: [PATCH 1/9] feat(openbao): add openbao role --- openbao.yml | 21 ++ roles/openbao/README.md | 414 +++++++++++++++++++++++++ roles/openbao/defaults/main.yml | 23 ++ roles/openbao/meta/main.yml | 18 ++ roles/openbao/tasks/config.yml | 129 ++++++++ roles/openbao/tasks/init.yml | 53 ++++ roles/openbao/tasks/main.yml | 118 +++++++ roles/openbao/tasks/provision.yml | 91 ++++++ roles/openbao/templates/openbao.hcl.j2 | 53 ++++ roles/openbao/templates/openbao.svc | 33 ++ 10 files changed, 953 insertions(+) create mode 100755 openbao.yml create mode 100644 roles/openbao/README.md create mode 100644 roles/openbao/defaults/main.yml create mode 100644 roles/openbao/meta/main.yml create mode 100644 roles/openbao/tasks/config.yml create mode 100644 roles/openbao/tasks/init.yml create mode 100644 roles/openbao/tasks/main.yml create mode 100644 roles/openbao/tasks/provision.yml create mode 100644 roles/openbao/templates/openbao.hcl.j2 create mode 100644 roles/openbao/templates/openbao.svc diff --git a/openbao.yml b/openbao.yml new file mode 100755 index 000000000..d1618ca6a --- /dev/null +++ b/openbao.yml @@ -0,0 +1,21 @@ +#!/usr/bin/env ansible-playbook +--- +#==============================================================# +# File : openbao.yml +# Desc : init openbao kms service +# Ctime : 2022-11-10 +# Mtime : 2025-12-31 +# Path : openbao.yml +# Docs : https://pigsty.io/docs/minio/playbook +# License : Apache-2.0 @ https://pigsty.io/docs/about/license/ +# Copyright : 2018-2026 Ruohang Feng / Vonng (rh@vonng.com) +#==============================================================# +- name: openbao INIT + become: true + hosts: openbao + gather_facts: no + roles: + - { role: node_id ,tags: node-id ,when: openbao_seq is defined } # get node identity (always) + - { role: openbao ,tags: openbao ,when: openbao_seq is defined } + +... diff --git a/roles/openbao/README.md b/roles/openbao/README.md new file mode 100644 index 000000000..6ad516200 --- /dev/null +++ b/roles/openbao/README.md @@ -0,0 +1,414 @@ +# Role: openbao + +> Deploy OpenBao KMS Cluster + +| **Module** | [OPENBAO](https://openbao.org/docs/) | +|-------------------|----------------------------------------------------------------------------------------------------| +| **Docs** | https://pigsty.io/docs/conf/pgtde | +| **Related Roles** | [`openbao_remove`](../openbao_remove), [`pgsql`](../pgsql), [`ca`](../ca), [`haproxy`](../haproxy) | + + +## Overview + +The `openbao` role deploys an **OpenBao cluster** - an open-source, Apache-2.0 licensed fork of HashiCorp Vault - used by Pigsty as the **external Key Management Service (KMS)** backing PostgreSQL Transparent Data Encryption: + +- Install openbao package +- Create configuration, data, log, TLS, and seal directories +- Issue and distribute TLS server certificates +- Generate and distribute a cluster-wide static unseal key +- Render the server config +- Launch the openbao systemd service and open firewall ports +- Initialize the cluster and persist the root token / recovery keys to the admin node +- Provision the `pg_tde` KV v2 mount, policy, and a long-lived token, persisted to the admin node + +Its role in the installation is to hold the **principal keys** for the [`pg_tde`](https://docs.percona.com/pg-tde/) extension. With `pg_tde`, PostgreSQL encrypts heap data and WAL at rest, but the master key never lives on the database host - PostgreSQL fetches it from OpenBao over TLS at startup and on key rotation. This separates the encrypted data (on the PGSQL nodes) from the key material (in the KMS), which is the property most at-rest encryption compliance regimes actually require. + +> `pg_tde` is available in the **Percona** PostgreSQL kernel. See the [`pgtde`](../../conf/pgtde.yml) configuration template. + + +## Playbooks + +| Playbook | Description | +|------------------------------------------|------------------------| +| [`openbao.yml`](../../openbao.yml) | Deploy OpenBao cluster | +| [`openbao-rm.yml`](../../openbao-rm.yml) | Remove OpenBao cluster | + + +## File Structure + +``` +roles/openbao/ +├── defaults/ +│ └── main.yml # Default variables +├── meta/ +│ └── main.yml # Role metadata +├── tasks/ +│ ├── main.yml # Entry point +│ ├── config.yml # [openbao_config] Certs, seal key, configuration +│ ├── init.yml # [openbao_init] Cluster initialization +│ └── provision.yml # [openbao_provision] pg_tde KV mount, policy & token +└── templates/ + ├── openbao.hcl.j2 # OpenBao server configuration + └── openbao.svc # Systemd service unit +``` + + +## Tags + +### Tag Hierarchy + +``` +openbao (full role) +│ +├── openbao_assert # Validate identity parameters (openbao_seq) +│ +├── openbao_install # Install openbao package +│ +├── openbao_dir # Create data / log / conf / tls / seal directories +│ +├── openbao_config # Configure openbao +│ ├── openbao_cert # TLS certificates +│ │ ├── openbao_cert_issue # Issue server certificate on localhost +│ │ └── openbao_cert_copy # Copy ca.crt / server.crt / server.key to node +│ ├── openbao_seal # Static unseal key +│ │ ├── openbao_seal_issue # Generate cluster-wide unseal key on localhost +│ │ └── openbao_seal_copy # Copy unseal key to node (aborts if divergent) +│ └── openbao_conf # Render openbao.hcl and systemd unit +│ +├── openbao_launch # Start openbao service, open firewall ports +│ +├── openbao_init # Initialize cluster, persist root token & recovery keys +│ +├── openbao_provision # Provision pg_tde KV mount, policy & token +│ +└── openbao_register # Register /v1/sys/metrics as a victoria-metrics target +``` + +> `openbao_init` is also applied to the initialization task itself when you run with `--tags openbao_provision`, so provisioning alone still ensures the cluster gets initialized first - you don't need to pass both tags. + + +## Key Variables + +### Identity (Required) + +| Variable | Level | Description | +|-------------------|----------|--------------------------| +| `openbao_cluster` | CLUSTER | OpenBao cluster name | +| `openbao_seq` | INSTANCE | Instance sequence number | + +### Configuration + +| Variable | Default | Description | +|---------------------------|----------------------|--------------------------------------------------------| +| `openbao_data` | `/data/openbao` | Raft data directory | +| `openbao_log_dir` | `/data/openbao/log` | Log directory (holds the file audit device) | +| `openbao_domain` | `bao.pigsty` | Domain name, added as a cert SAN | +| `openbao_api_port` | `8200` | API port - client requests, advertised for redirection | +| `openbao_cluster_port` | `8201` | Cluster port - inter-node request forwarding | + + +## Configuration Example + +OpenBao supports single-node and multi-node distributed modes: + +### Single Node + +```yaml +openbao: + hosts: + 10.10.10.10: { openbao_seq: 1 } + vars: + openbao_cluster: openbao +``` + +### Multi-Node Distributed + +```yaml +all: + children: + openbao: + hosts: + 10.10.10.11: { openbao_seq: 1 } + 10.10.10.12: { openbao_seq: 2 } + 10.10.10.13: { openbao_seq: 3 } + vars: + openbao_cluster: openbao + vars: + openbao_data: /data/openbao + openbao_domain: bao.pigsty +``` + +To expose the UI/API through the infra portal and load balancer: + +```yaml + infra_portal: + openbao: { domain: bao.pigsty ,endpoint: "${admin_ip}:8200" ,scheme: https } + + haproxy_services: + - name: openbao + port: 8200 + options: + - option httpchk + - http-check send meth GET uri /v1/sys/health ver HTTP/1.1 hdr host bao.pigsty + - http-check expect status 200 + servers: + - { name: openbao-1 ,ip: 10.10.10.11 ,port: 8200 ,options: 'check-ssl ca-file /etc/pki/ca.crt check' } + - { name: openbao-2 ,ip: 10.10.10.12 ,port: 8200 ,options: 'check-ssl ca-file /etc/pki/ca.crt check' } + - { name: openbao-3 ,ip: 10.10.10.13 ,port: 8200 ,options: 'check-ssl ca-file /etc/pki/ca.crt check' } +``` + +See [`conf/pgtde.yml`](../../conf/pgtde.yml) for a complete production example. + + +## Cluster Topology + +OpenBao uses **integrated Raft storage**, so the same quorum arithmetic as ETCD applies: + +| Nodes | Fault Tolerance | Recommended | +|-------|-----------------|-------------| +| 1 | 0 | Demo / Dev | +| 3 | 1 | Minimum HA | +| 5 | 2 | Production | + +All members are listed as `retry_join` targets in `openbao.hcl`, so nodes discover the leader and join automatically on start. Adding a member is simply a matter of declaring it in the inventory and running `./openbao.yml -l `. + +> The KMS is on the critical path for starting an encrypted PostgreSQL cluster: if no OpenBao node is reachable, `pg_tde` cannot fetch the principal key. Deploy at least 3 nodes in production and treat KMS availability as seriously as etcd availability. + + +## TLS Configuration + +All OpenBao traffic is TLS-only (`tls_min_version = "tls13"`), using certificates signed by the [Pigsty CA](../ca): + +- **CA**: `files/pki/ca/ca.crt` → `/etc/openbao.d/tls/ca.crt` +- **Server Cert**: `files/pki/bao/.crt` → `/etc/openbao.d/tls/server.crt` +- **Server Key**: `files/pki/bao/.key` → `/etc/openbao.d/tls/server.key` + +The server certificate carries SANs for `localhost`, the cluster name, the instance name, `openbao_domain`, `127.0.0.1`, and the node IP - so clients may connect through any of those. + +Clients (including PostgreSQL/`pg_tde`) must trust `/etc/pki/ca.crt`, which the NODE module already distributes to every managed host. + + +## Seal & Unseal + +Pigsty configures OpenBao with the **static seal**, so the cluster auto-unseals on restart without manual key-share entry: + +```hcl +seal "static" { + current_key_id = "-" + current_key = "file:///etc/openbao.d/seal/-unseal.key" +} +``` + +- The 32-byte key is generated **once** on the admin node at `files/pki/bao/unseal.key`, then copied to `/etc/openbao.d/seal/` on every member (mode `0640`, group `openbao`). +- The role **aborts the play** if a node already carries a seal key whose checksum differs from the admin node's - this prevents silently splitting a cluster across two key generations. +- `current_key_id` is derived from the key checksum, so the identifier changes if and only if the key material changes. + +> **Critical**: `files/pki/bao/unseal.key` decrypts the entire OpenBao store, which in turn holds your TDE principal keys. Back it up alongside `files/pki/ca/ca.key` and guard it just as carefully. Losing it means losing access to every key OpenBao holds - and therefore to your encrypted data. + + +## Cluster Initialization + +The `openbao_init` task runs once per cluster and is idempotent - it checks `bao status` first and skips an already-initialized cluster: + +```bash +bao operator init -recovery-shares=5 -recovery-threshold=3 -format=json +``` + +The output is written at: + +``` +files/openbao/-init.json # mode 0600 +``` + +This file contains the **initial root token** and the **recovery key shares**. Move it to your secret store as soon as the deployment completes: + +1. Use the root token to create the policies and tokens your workloads need +2. Distribute the recovery shares to separate custodians (3 of 5 are required to recover) +3. Revoke the initial root token (`bao token revoke `) +4. Delete the file from the admin node + +An audit device writes every request to `{{ openbao_log_dir }}/audit.log` in JSON, and telemetry is exposed in Prometheus format. + +Immediately after init, `openbao_provision` (tag `openbao_provision`, `tasks/provision.yml`) sets up what `pg_tde` needs on the KMS side, using the root token from the file above: + +```bash +bao secrets enable -path=pg_tde -version=2 kv # KV v2 mount for principal keys +bao policy write pg-tde-policy # least-privilege policy on that mount +bao token create -policy=pg-tde-policy -orphan -period=0 -format=json +``` + +The token is orphan (survives revocation of the root token) and periodic (renews indefinitely instead of expiring at a max TTL and taking an encrypted PostgreSQL cluster down with it). It is written to: + +``` +files/openbao/-token.json # mode 0600, raw `bao token create` output +``` + +This step is idempotent, but not the same way as `openbao_init`: it does **not** check server-side state, only whether `files/openbao/-token.json` already exists on the controller. Delete that file to force a re-issue (the old token keeps working until you revoke it). + + +## Rebuilding a Cluster + +```bash +# 1. destroy the existing cluster (safeguard is on by default) +./openbao-rm.yml -l openbao -e openbao_safeguard=false + +# 2. redeploy on the same nodes +./openbao.yml +``` + +The cluster-wide key material on the admin node survives step 1 on purpose: + +``` +files/pki/bao/unseal.key # reused, so the rebuilt cluster keeps the same seal key +files/openbao/-init.json # overwritten by the new init in step 2 +files/openbao/-token.json # NOT overwritten - see warning below +``` + +Because the seal key is reused, `current_key_id` does not change and the rebuilt cluster unseals normally. The **contents** are gone, though: every secret, policy, token and `pg_tde` principal key must be recreated, and any PostgreSQL cluster encrypted against the old store stays unreadable. + +> **Trap**: `openbao_provision` only checks whether `files/openbao/-token.json` exists on the controller, not whether the `pg_tde` mount actually exists in the (now empty) rebuilt store. Since that file survives the rebuild, step 2 above will silently skip re-provisioning, leaving `pg_tde` pointed at a token for a mount that no longer exists. Delete `files/openbao/-token.json` before redeploying, or re-provision explicitly afterwards with `./openbao.yml -l openbao -t openbao_provision`. + +Rebuilding the KMS is not a recovery procedure - restore from a raft snapshot instead: + +```bash +bao operator raft snapshot save bao.snap # take one before you need it +bao operator raft snapshot restore bao.snap +``` + +To rebuild a **single member** rather than the whole cluster, remove and re-add it so the raft peer list stays consistent: + +```bash +./openbao-rm.yml -l 10.10.10.13 -e openbao_safeguard=false +./openbao.yml -l 10.10.10.13 +``` + + +## Monitoring + +Metrics come straight from OpenBao's own telemetry - there is no separate exporter. The listener carries a `telemetry` sub-stanza so VictoriaMetrics can read `/v1/sys/metrics` without a token: + +```hcl +listener "tcp" { + telemetry { + unauthenticated_metrics_access = true + } +} +``` + +> A **sealed** node answers `503` on `/v1/sys/metrics`, so it is scraped as `up=0` and is indistinguishable from a dead node by metrics alone. `OpenbaoServerDown` therefore covers both cases, and `OpenbaoSealed` only fires for a node that still serves metrics while reporting itself sealed. Confirm with `bao status` on the node. + + +## Integration with pg_tde + +Wiring a PostgreSQL cluster to this OpenBao cluster is **mostly automatic** - the manual `bao`/`psql` ceremony only matters for verification, rotation, and troubleshooting. + +### 1. Deploy OpenBao, then a Percona kernel cluster with `pg_tde` + +Deploy `openbao.yml` first so `files/openbao/-token.json` exists before you bring up the database - the [`pgsql`](../pgsql) role reads it, and an empty/missing token means an encrypted cluster that cannot fetch its principal key at startup. + +Use the [`pgtde`](../../conf/pgtde.yml) template as a starting point, or configure it yourself. + +> The shipped [`conf/pgtde.yml`](../../conf/pgtde.yml) template does **not** currently declare an `openbao` group - add one (see [Configuration Example](#configuration-example) above) or point `openbao_cluster`/`openbao_domain` at an existing OpenBao deployment, otherwise `pg_tde` has no KMS to talk to. + +### 2. OpenBao side - handled by `openbao_provision` + +No manual `bao secrets enable` / `bao policy write` / `bao token create` needed: `openbao_provision` does this once per cluster (see [Cluster Initialization](#cluster-initialization) above) and leaves the token at `files/openbao/-token.json` on the admin node. + +### 3. PostgreSQL side - handled by `pgsql` + +When `pg_mode` is `pgtde`, the [`pgsql`](../pgsql) role does the rest during cluster bootstrap: + +- `tde.yml` (tag `pg_tde`) installs the OpenBao token to `/pg/tde-token` (mode `0600`) before Patroni starts, reading it from this role's `files/openbao/-token.json` +- `pg-init-kms.sql`, run once against the primary, registers OpenBao as a global key provider and creates/sets the default principal key: + ```sql + SELECT pg_tde_add_global_key_provider_vault_v2( + '{{ openbao_cluster }}', -- provider name + 'https://{{ openbao_domain }}:{{ openbao_api_port }}', -- KMS endpoint + 'pg_tde', -- mount path + '/pg/tde-token', -- token file installed above + '/pg/cert/ca.crt' -- CA to verify the KMS certificate + ); + SELECT pg_tde_create_key_using_global_key_provider('{{ pg_cluster }}-principal-key', '{{ openbao_cluster }}'); + SELECT pg_tde_set_default_key_using_global_key_provider('{{ pg_cluster }}-principal-key', '{{ openbao_cluster }}'); + ``` +- Patroni is configured with `pg_tde.wal_encrypt=on`, `pg_tde.enforce_encryption=on`, and `default_table_access_method=tde_heap` + +### 4. Verify and rotate + +```bash +export BAO_ADDR=https://bao.pigsty:8200 +export BAO_CACERT=/etc/pki/ca.crt +bao login # from files/openbao/-init.json +bao secrets list # confirm the pg_tde/ mount +bao token lookup -accessor # inspect the pg_tde token without exposing it +``` + +```sql +SELECT * FROM pg_tde_global_key_provider_info(); +SELECT pg_tde_key_info(); -- current database principal key +``` + +Rotate principal keys on your compliance schedule with `pg_tde_create_key_using_global_key_provider` + `pg_tde_set_default_key_using_global_key_provider`; rotate the OpenBao token itself by deleting `files/openbao/-token.json` and re-running the playbook with `--tags openbao_provision`, then re-running the `pg_tde` tag on the PGSQL cluster to install the new token. + +> Function names and signatures vary across `pg_tde` releases. Confirm against the [Percona pg_tde documentation](https://docs.percona.com/pg-tde/) for the version you deployed, and always test the whole flow - including a full restart of the PostgreSQL cluster with the KMS reachable - before putting real data behind it. + +### 5. Operational consequences + +- **Startup dependency**: an encrypted cluster cannot start without reaching OpenBao. Verify KMS health before any PGSQL restart, switchover, or PITR. +- **Backups**: pgBackRest backs up the *encrypted* files. A restore is useless without the corresponding principal key - the KMS and its unseal key are now part of your recovery plan, and should be exercised in every DR drill. +- **Token lifetime**: the periodic token in `files/openbao/-token.json` renews indefinitely as long as it's used before it expires - don't let a PGSQL cluster stay disconnected from OpenBao longer than the token's TTL, or the provider will start failing at the worst possible moment. + + +## Administration + +```bash +export BAO_ADDR=https://bao.pigsty:8200 +export BAO_CACERT=/etc/pki/ca.crt + +bao status # seal / init / HA status +bao operator raft list-peers # raft cluster members +bao operator raft snapshot save bao.snap # take a snapshot +bao secrets list # mounted secret engines +bao token lookup # inspect the current token + +systemctl status openbao # service status +journalctl -u openbao -f # service logs +tail -f /data/openbao/log/audit.log # audit log +``` + +Health endpoints (used by the HAProxy check): + +```bash +curl --cacert /etc/pki/ca.crt https://bao.pigsty:8200/v1/sys/health +curl --cacert /etc/pki/ca.crt https://bao.pigsty:8200/v1/sys/metrics?format=prometheus +``` + + +## Security Notes + +The systemd unit is hardened: dedicated `openbao` user, `ProtectSystem=full`, `ProtectHome=read-only`, `PrivateTmp`, `PrivateDevices`, `NoNewPrivileges`, and `CAP_IPC_LOCK` so secrets are never swapped to disk. + +Files to protect, in order of severity: + +| Path | Location | Why it matters | +|--------------------------------------------|-------------|--------------------------------------------------| +| `files/pki/bao/unseal.key` | Admin node | Decrypts the whole OpenBao store | +| `files/openbao/-init.json` | Admin node | Root token + recovery shares | +| `files/openbao/-token.json` | Admin node | Orphan, non-expiring `pg_tde` token de/` mount | +| `/etc/openbao.d/seal/-unseal.key` | Members | Same key, distributed for auto-unseal | +| `files/pki/bao/*.key` | Admin node | TLS server private keys | +| `{{ openbao_data }}` | Members | Encrypted raft store (mode `0700`) | +| `/pg/tde-token` | PGSQL nodes | Copy of the `pg_tde` token, installed by `pgsql` | + +> The static seal trades key-ceremony security for operational simplicity: anyone with root on a member node can read the seal key and unseal the store offline. That is an acceptable trade when the threat model is *stolen disks and stolen backups* - which is exactly what `pg_tde` defends against. If your threat model includes a compromised database host, use an external seal (KMS/HSM/transit) instead and keep the seal key off the nodes entirely. + + +## See Also + +- [`openbao_remove`](../openbao_remove): Remove OpenBao clusters and members +- [`ca`](../ca): Self-signed CA that issues OpenBao server certificates +- [`pgsql`](../pgsql): PostgreSQL deployment (`pg_tde` consumer) +- [`haproxy`](../haproxy): Load balancing the KMS endpoint +- [`conf/pgtde.yml`](../../conf/pgtde.yml): Percona + `pg_tde` configuration template +- [OpenBao Documentation](https://openbao.org/docs/) +- [Percona pg_tde Documentation](https://docs.percona.com/pg-tde/) diff --git a/roles/openbao/defaults/main.yml b/roles/openbao/defaults/main.yml new file mode 100644 index 000000000..47c0e7d70 --- /dev/null +++ b/roles/openbao/defaults/main.yml @@ -0,0 +1,23 @@ +--- +#----------------------------------------------------------------- +# OPENBAO +#----------------------------------------------------------------- +#openbao_seq: 1 # bao instance identifier, explicitly required +openbao_cluster: openbao # bao cluster & group name, openbao by default +openbao_data: /data/openbao # bao data dir, `/data/openbao` by default +openbao_log_dir: /data/openbao/log # bao log dir, `/data/openbao/log` by default +openbao_domain: bao.pigsty # bao domain name, `bao.pigsty` by default +openbao_api_port: 8200 # used to advertise other servers in the cluster for client redirection, `8200` by default +openbao_cluster_port: 8201 # used to advertise other servers in the cluster for request forwarding, `8201` by default + +#----------------------------------------------------------------- +# CA (Reference) +#----------------------------------------------------------------- +cert_validity: 7300d # cert validity, 20 years by default +#----------------------------------------------------------------- +# NODE (Reference) +#----------------------------------------------------------------- +node_data: /data # node main data directory, /data by default +node_admin_enabled: true # create a admin user on target node? +node_admin_username: dba # name of node admin user, `dba` by default +... diff --git a/roles/openbao/meta/main.yml b/roles/openbao/meta/main.yml new file mode 100644 index 000000000..992382d1b --- /dev/null +++ b/roles/openbao/meta/main.yml @@ -0,0 +1,18 @@ +--- +galaxy_info: + role_name: openbao # openbao role + description: init openbao module + min_ansible_version: "2.9" + author: 'Ruohang Feng / @Vonng / rh@vonng.com' + license: Apache-2.0 + platforms: + - name: EL + versions: [all] + - name: Ubuntu + versions: [all] + - name: Debian + versions: [all] + galaxy_tags: + - pigsty + - openbao +... diff --git a/roles/openbao/tasks/config.yml b/roles/openbao/tasks/config.yml new file mode 100644 index 000000000..351a7f731 --- /dev/null +++ b/roles/openbao/tasks/config.yml @@ -0,0 +1,129 @@ +--- +#--------------------------------------------------------------# +# Issue openbao certs [openbao_cert_issue] +#--------------------------------------------------------------# +- name: generate openbao server certs + tags: [ openbao_cert, openbao_cert_issue ] + become: false + connection: local + vars: + ansible_python_interpreter: "{{ ansible_playbook_python }}" + block: + - name: generate openbao private key + connection: local + openssl_privatekey: + path: files/pki/bao/{{ openbao_instance }}.key + mode: 0600 + + - name: generate openbao signing request + connection: local + openssl_csr: + path: "files/pki/csr/{{ openbao_instance }}.csr" + privatekey_path: "files/pki/bao/{{ openbao_instance }}.key" + common_name: "{{ openbao_instance }}" + organization_name: pigsty + organizational_unit_name: openbao + #force: true + subject_alt_name: + - "DNS:localhost" + - "DNS:{{ openbao_cluster }}" + - "DNS:{{ openbao_cluster }}-{{ openbao_seq }}" + - "DNS:{{ openbao_domain }}" + - IP:127.0.0.1 + - "IP:{{ inventory_hostname }}" + + - name: issue openbao server certificate + connection: local + openssl_certificate: + path: "files/pki/bao/{{ openbao_instance }}.crt" + csr_path: "files/pki/csr/{{ openbao_instance }}.csr" + provider: ownca + ownca_path: files/pki/ca/ca.crt + ownca_privatekey_path: files/pki/ca/ca.key + ownca_not_after: "+{{ cert_validity }}" + mode: 0644 + + +#--------------------------------------------------------------# +# Issue openbao unseal static key [openbao_seal_issue] +#--------------------------------------------------------------# +- name: generate openbao unseal static key + tags: [ openbao_seal, openbao_seal_issue ] + become: false + connection: local + run_once: true + vars: + ansible_python_interpreter: "{{ ansible_playbook_python }}" + block: + - name: check if unseal key exists + stat: path="files/pki/bao/unseal.key" + register: unseal_key_exists + + - name: generate cluster-wide unseal key + when: not unseal_key_exists.stat.exists + command: openssl rand -out "files/pki/bao/unseal.key" 32 + + - name: re-stat unseal key + stat: + path: "files/pki/bao/unseal.key" + get_checksum: true + checksum_algorithm: sha256 + register: seal_key_stat + + - name: derive current_key_id from key material + set_fact: + openbao_seal_key_id: "{{ openbao_cluster }}-{{ seal_key_stat.stat.checksum[:12] }}" + + +#--------------------------------------------------------------# +# Copy openbao unseal static key [openbao_seal_copy] +#--------------------------------------------------------------# +- name: assert openbao unseal static key on nodes + tags: [ openbao_seal, openbao_seal_copy ] + become: true + block: + - name: check existing remote seal key + stat: + path: "/etc/openbao.d/seal/{{ openbao_cluster }}-unseal.key" + get_checksum: true + checksum_algorithm: sha256 + register: remote_seal_key + + - name: ABORT if divergent seal key + when: + - remote_seal_key.stat.exists + - remote_seal_key.stat.checksum != seal_key_stat.stat.checksum + any_errors_fatal: true + fail: + msg: 'Abort because divergent seal key on node.' + + - name: copy openbao unseal static key + copy: src="files/pki/bao/unseal.key" dest="/etc/openbao.d/seal/{{ openbao_cluster }}-unseal.key" owner=root group=openbao mode=0640 + + +#--------------------------------------------------------------# +# Copy openbao certs [openbao_cert_copy] +#--------------------------------------------------------------# +- name: copy openbao certs + tags: [ openbao_cert, openbao_cert_copy ] + become: true + copy: src={{ item.src }} dest={{ item.dest }} owner={{ item.owner }} group={{ item.group|default(item.owner) }} mode={{ item.mode }} + with_items: + - { src: "files/pki/ca/ca.crt" ,dest: "/etc/openbao.d/tls/ca.crt" ,owner: "root", group: "openbao" ,mode: "0640" } + - { src: "files/pki/bao/{{ openbao_instance }}.crt" ,dest: "/etc/openbao.d/tls/server.crt" ,owner: "root", group: "openbao" ,mode: "0640" } + - { src: "files/pki/bao/{{ openbao_instance }}.key" ,dest: "/etc/openbao.d/tls/server.key" ,owner: "root", group: "openbao" ,mode: "0640" } + + +#--------------------------------------------------------------# +# Config openbao [openbao_conf] +#--------------------------------------------------------------# +- name: setup openbao server + tags: openbao_conf + block: + - name: generate openbao config + become: true + template: src={{ item.src }} dest={{ item.dest }} owner={{ item.owner }} group={{ item.group|default(item.owner) }} mode={{ item.mode }} + with_items: + - { src: "openbao.hcl.j2" ,dest: "/etc/openbao.d/openbao.hcl" ,owner: "root", group: "openbao" ,mode: "0640" } + - { src: "openbao.svc" ,dest: "/usr/lib/systemd/system/openbao.service" ,owner: "root", group: "root" ,mode: "0644" } +... diff --git a/roles/openbao/tasks/init.yml b/roles/openbao/tasks/init.yml new file mode 100644 index 000000000..c5b4d7f21 --- /dev/null +++ b/roles/openbao/tasks/init.yml @@ -0,0 +1,53 @@ +--- +#--------------------------------------------------------------# +# Initialize openbao cluster [openbao_init] +#--------------------------------------------------------------# +- name: initialize openbao cluster + tags: openbao_init + run_once: true + become: true + block: + - name: check initialization status + command: bao status -format=json + register: bao_status + changed_when: false + failed_when: bao_status.rc not in [0, 1, 2] + + - name: parse init state + set_fact: + bao_initialized: "{{ (bao_status.stdout | from_json).initialized }}" + + - name: initialize cluster + when: not bao_initialized + command: > + bao operator init + -recovery-shares=5 -recovery-threshold=3 + -format=json + register: bao_init + + - name: persist init material to controller + when: not bao_initialized + delegate_to: localhost + become: false + copy: + content: "{{ bao_init.stdout }}" + dest: "files/openbao/{{ openbao_cluster }}-init.json" + mode: '0600' + + - name: wait for active node to be ready + command: bao status -format=json + register: bao_ready + changed_when: false + failed_when: false + retries: 5 + delay: 5 + until: + - bao_ready.rc == 0 + - (bao_ready.stdout | from_json).initialized | default(false) + - not (bao_ready.stdout | from_json).sealed | default(true) + + - name: re-parse init state + set_fact: + bao_initialized: "{{ (bao_ready.stdout | from_json).initialized }}" + +... diff --git a/roles/openbao/tasks/main.yml b/roles/openbao/tasks/main.yml new file mode 100644 index 000000000..ea6221e08 --- /dev/null +++ b/roles/openbao/tasks/main.yml @@ -0,0 +1,118 @@ +--- +#--------------------------------------------------------------# +# Assert openbao identity [openbao_assert] +#--------------------------------------------------------------# +- name: assert openbao identity + tags: openbao_assert + assert: + that: + - openbao_seq is defined and openbao_seq|int >= 0 + fail_msg: variable 'openbao_seq' is required for openbao instance + + +#--------------------------------------------------------------# +# Install openbao [openbao_install] +#--------------------------------------------------------------# +- name: install openbao + tags: openbao_install + environment: "{{ proxy_env | default({}) }}" + package: name=openbao state=present + + +#--------------------------------------------------------------# +# Create openbao directory [openbao_dir] +#--------------------------------------------------------------# +- name: create openbao directories + tags: openbao_dir + block: + - name: create openbao directory on local files/openbao + delegate_to: localhost + become: false + file: path=files/openbao state=directory mode='0755' + - name: remove unused openbao data dir + when: openbao_data != '/var/lib/openbao' + file: path=/var/lib/openbao state=absent + - name: ensure node_data directory exists + when: openbao_data.startswith(node_data) + file: path={{ node_data }} state=directory owner=root group=root mode=0755 + - name: ensure openbao data directory exists + file: path={{ item.path }} state={{ item.state }} owner={{ item.owner }} group={{ item.group }} mode={{ item.mode }} + loop: + - { path: "/etc/openbao.d" ,state: "directory" ,mode: "0755" ,owner: "root" ,group: "root" } + - { path: "/etc/openbao.d/tls" ,state: "directory" ,mode: "0750" ,owner: "root" ,group: "openbao" } + - { path: "/etc/openbao.d/seal" ,state: "directory" ,mode: "0750" ,owner: "root" ,group: "openbao" } + - { path: "{{ openbao_data }}" ,state: "directory" ,mode: "0700" ,owner: "openbao" ,group: "openbao" } + - { path: "{{ openbao_log_dir }}" ,state: "directory" ,mode: "0700" ,owner: "openbao" ,group: "openbao" } + + +#--------------------------------------------------------------# +# Config openbao [openbao_config] +#--------------------------------------------------------------# +- import_tasks: config.yml + tags: openbao_config + vars: { openbao_instance: "{{ openbao_cluster }}-{{ openbao_seq }}" } + + +#--------------------------------------------------------------# +# Launch openbao [openbao_launch] +#--------------------------------------------------------------# +- name: launch openbao server + tags: openbao_launch + block: + - name: openbao pre-launch adjustment + ignore_errors: true + shell: | + if systemctl is-active firewalld &>/dev/null; then + firewall-cmd --permanent --add-port={{ openbao_api_port }}/tcp --add-port={{ openbao_cluster_port }}/tcp + firewall-cmd --reload + fi + if [[ "{{ os_package }}" == "deb" ]]; then + if command -v ufw &>/dev/null && ufw status | grep -q "Status: active"; then + ufw allow {{ openbao_api_port }}/tcp {{ openbao_cluster_port }}/tcp + fi + fi + /bin/true + args: { executable: /bin/bash } + + - name: launch openbao server service + systemd: name=openbao state=restarted enabled=yes daemon_reload=yes + + - name: wait for openbao server online + wait_for: host={{ inventory_hostname }} port={{ openbao_api_port }} state=started timeout=60 + + +#--------------------------------------------------------------# +# Init openbao [openbao_init] +#--------------------------------------------------------------# +- import_tasks: init.yml + tags: [ openbao_init, openbao_provision ] + vars: { openbao_instance: "{{ openbao_cluster }}-{{ openbao_seq }}" } + + +#--------------------------------------------------------------# +# Provision pg_tde key provider [openbao_provision] +#--------------------------------------------------------------# +- import_tasks: provision.yml + tags: openbao_provision + vars: { openbao_instance: "{{ openbao_cluster }}-{{ openbao_seq }}" } + + +#--------------------------------------------------------------# +# Register Openbao Metrics to VictoriaMetrics [openbao_register] +#--------------------------------------------------------------# +- name: register openbao as victoria target + tags: [ openbao_register, register, add_metrics ] + ignore_errors: true + delegate_to: '{{ item }}' + loop: '{{ groups["infra"]|default([]) }}' + copy: + dest: "/infra/targets/openbao/{{ openbao_cluster }}-{{ openbao_seq }}.yml" + owner: victoria + group: infra + mode: '0640' + content: | + # {{ openbao_cluster }}-{{ openbao_seq }} @ {{ inventory_hostname }} + - labels: { ip: {{ inventory_hostname }} , ins: {{ openbao_cluster }}-{{ openbao_seq }} , cls: {{ openbao_cluster }} } + targets: [ {{ inventory_hostname }}:{{ openbao_api_port }} ] + +... diff --git a/roles/openbao/tasks/provision.yml b/roles/openbao/tasks/provision.yml new file mode 100644 index 000000000..0741067d2 --- /dev/null +++ b/roles/openbao/tasks/provision.yml @@ -0,0 +1,91 @@ +--- +#-----------------------------------------------------------# +# Provision pg_tde key provider [openbao_provision] +#-----------------------------------------------------------# +# Prepares what pg_tde needs on the KMS side: a KV v2 mount +# for the principal keys, a least-privilege policy on it, +# and a long-lived token bound to that policy. +# +# The token is persisted on the controller as the raw `bao +# token create` output: +# +# files/openbao/-token.json +# +# and installed on the PGSQL nodes by the pgsql role. That +# file is also the guard: provisioning runs only when it is +# absent, so the playbook does not mint a new token on every +# pass. Delete it to force a re-issue. +#-----------------------------------------------------------# + +- name: provision pg_tde key provider + tags: openbao_provision + when: bao_initialized + run_once: true + become: true + environment: + BAO_ADDR: "https://127.0.0.1:{{ openbao_api_port }}/" + BAO_CACERT: "/etc/openbao.d/tls/ca.crt" + BAO_CLIENT_CERT: "/etc/openbao.d/tls/server.crt" + BAO_CLIENT_KEY: "/etc/openbao.d/tls/server.key" + BAO_TOKEN: >- + {{ (lookup('file', 'files/openbao/' ~ openbao_cluster ~ '-init.json', errors='ignore') + | default('{}', true) | from_json).root_token | default('') }} + block: + + - name: check for an already provisioned pg_tde token + delegate_to: localhost + become: false + stat: path="files/openbao/{{ openbao_cluster }}-token.json" + register: openbao_token_stat + + - name: provision pg_tde mount, policy and token + when: not openbao_token_stat.stat.exists + block: + + # tolerate a mount left behind by an earlier run + - name: check existing secret engines + command: bao secrets list -format=json + register: openbao_mounts + changed_when: false + + - name: enable kv v2 secret engine for pg_tde + command: bao secrets enable -path=pg_tde -version=2 -description="principal keys for pg_tde" kv + when: "'pg_tde/' not in (openbao_mounts.stdout | from_json)" + + # written on stdin: a policy file under /tmp is world-traversable + # and outlives the play if it aborts between write and cleanup + - name: write pg_tde policy + command: + cmd: bao policy write pg-tde-policy - + stdin: | + path "pg_tde/data/*" { + capabilities = ["create", "read", "update", "delete", "list"] + } + path "pg_tde/metadata/*" { + capabilities = ["read", "list", "delete"] + } + + # -orphan: the token must survive revocation of the root token, + # which the README tells operators to do after deployment + # -period: makes it periodic, so it renews indefinitely instead of + # dying at max TTL and taking the encrypted cluster with it + - name: create pg_tde token + command: > + bao token create + -policy=pg-tde-policy + -display-name=pg-tde + -no-default-policy + -orphan + -period=768h + -format=json + register: openbao_token_create + + - name: persist pg_tde token to controller + delegate_to: localhost + become: false + copy: + content: "{{ openbao_token_create.stdout }}" + dest: "files/openbao/{{ openbao_cluster }}-token.json" + mode: '0600' + +... diff --git a/roles/openbao/templates/openbao.hcl.j2 b/roles/openbao/templates/openbao.hcl.j2 new file mode 100644 index 000000000..ea59baecb --- /dev/null +++ b/roles/openbao/templates/openbao.hcl.j2 @@ -0,0 +1,53 @@ +cluster_name = "pigsty" +ui = true +log_level = "info" +log_format = "json" + +storage "raft" { + path = "{{ openbao_data }}" + node_id = "{{ openbao_instance }}" + performance_multiplier = 3 + +{% if groups[openbao_cluster]|length > 1 %} +{% for ip in groups[openbao_cluster]|sort %} + retry_join { + leader_api_addr = "https://{{ ip }}:{{ openbao_api_port }}" + leader_tls_servername = "{{ openbao_cluster }}-{{ hostvars[ip].openbao_seq }}" + leader_ca_cert_file = "/etc/openbao.d/tls/ca.crt" + } +{% endfor %} +{% endif %} +} + +listener "tcp" { + address = "0.0.0.0:{{ openbao_api_port }}" + cluster_address = "0.0.0.0:{{ openbao_cluster_port }}" + tls_cert_file = "/etc/openbao.d/tls/server.crt" + tls_key_file = "/etc/openbao.d/tls/server.key" + tls_client_ca_file = "/etc/openbao.d/tls/ca.crt" + tls_min_version = "tls13" + telemetry { + unauthenticated_metrics_access = true + } +} + +api_addr = "https://{{ openbao_domain }}:{{ openbao_api_port }}" +cluster_addr = "https://{{ inventory_hostname }}:{{ openbao_cluster_port }}" + +seal "static" { + current_key_id = "{{ openbao_seal_key_id }}" + current_key = "file:///etc/openbao.d/seal/{{ openbao_cluster }}-unseal.key" +} + +audit "file" "audit-log" { + options { + file_path = "{{ openbao_log_dir }}/audit.log" + } +} + +telemetry { + prometheus_retention_time = "24h" + disable_hostname = true + usage_gauge_period = "10m" + maximum_gauge_cardinality = 500 +} diff --git a/roles/openbao/templates/openbao.svc b/roles/openbao/templates/openbao.svc new file mode 100644 index 000000000..a46d23896 --- /dev/null +++ b/roles/openbao/templates/openbao.svc @@ -0,0 +1,33 @@ +[Unit] +Description="Openbao - A tool for managing secrets" +Documentation=https://openbao.org/docs/ +Requires=network-online.target +After=network-online.target +ConditionFileNotEmpty=/etc/openbao.d/openbao.hcl + +[Service] +User=openbao +Group=openbao +ExecStart=/usr/bin/bao server -config=/etc/openbao.d/openbao.hcl +ExecReload=/bin/kill --signal HUP $MAINPID +Restart=on-failure +RestartSec=5 +TimeoutStopSec=30 +LimitNOFILE=65536 +LimitMEMLOCK=infinity +MemorySwapMax=0 +KillMode=process +KillSignal=SIGINT + +# Hardening and protections +ProtectSystem=full +ProtectHome=read-only +PrivateTmp=yes +PrivateDevices=yes +SecureBits=keep-caps +AmbientCapabilities=CAP_IPC_LOCK +CapabilityBoundingSet=CAP_SYSLOG CAP_IPC_LOCK +NoNewPrivileges=yes + +[Install] +WantedBy=multi-user.target From 2092fe44dfa537f575fdbf324ac4afb268a23e4a Mon Sep 17 00:00:00 2001 From: alemacci Date: Sun, 16 Aug 2026 14:56:07 +0200 Subject: [PATCH 2/9] feat(openbao): add openbao remove role --- openbao-rm.yml | 68 ++++++++++ roles/openbao_remove/README.md | 167 +++++++++++++++++++++++++ roles/openbao_remove/defaults/main.yml | 21 ++++ roles/openbao_remove/meta/main.yml | 18 +++ roles/openbao_remove/tasks/leave.yml | 49 ++++++++ roles/openbao_remove/tasks/main.yml | 110 ++++++++++++++++ 6 files changed, 433 insertions(+) create mode 100755 openbao-rm.yml create mode 100644 roles/openbao_remove/README.md create mode 100644 roles/openbao_remove/defaults/main.yml create mode 100644 roles/openbao_remove/meta/main.yml create mode 100644 roles/openbao_remove/tasks/leave.yml create mode 100644 roles/openbao_remove/tasks/main.yml diff --git a/openbao-rm.yml b/openbao-rm.yml new file mode 100755 index 000000000..ad1924449 --- /dev/null +++ b/openbao-rm.yml @@ -0,0 +1,68 @@ +#!/usr/bin/env ansible-playbook +--- +#==============================================================# +# File : openbao-rm.yml +# Desc : remove openbao from hosts +# Ctime : 2026-07-30 +# Mtime : 2026-07-30 +# Path : openbao-rm.yml +# Docs : https://pigsty.io/docs/openbao/playbook +# License : Apache-2.0 @ https://pigsty.io/docs/about/license/ +# Copyright : 2018-2026 Ruohang Feng / Vonng (rh@vonng.com) +#==============================================================# +- name: OPENBAO REMOVE + become: true + hosts: openbao + gather_facts: no + ignore_errors: true + vars: + #openbao_safeguard: true # safeguard for openbao remove, TRUE by default, halts execution + #openbao_rm_data: true # remove openbao data during remove? true by default + #openbao_rm_log: true # remove openbao log dir (audit trail)? true by default + #openbao_rm_pkg: false # uninstall openbao packages during remove? false by default + roles: [ { role: node_id }, { role: openbao_remove } ] + + +#--------------------------------------------------------------# +# Usage +#--------------------------------------------------------------# +# Remove openbao cluster `openbao` +# openbao-rm.yml -l openbao # remove cluster `openbao` +# -e openbao_safeguard=false # REQUIRED: safeguard is enabled by default +# -e openbao_rm_data=true # remove openbao data by default +# -e openbao_rm_log=true # remove the audit trail by default +# -e openbao_rm_pkg=false # do not uninstall openbao packages by default +# It aborts unless `openbao_safeguard` is explicitly set to `false` +# +# Remove an openbao member: +# ./openbao-rm.yml -l 10.10.10.12 -e openbao_safeguard=false +# vi pigsty.yml # remove `10.10.10.12` from the `openbao` group +# ./openbao.yml # refresh openbao cluster after member removal +# +#--------------------------------------------------------------# +# Utils +#--------------------------------------------------------------# +# bin/openbao-rm # remove the entire openbao cluster +# bin/openbao-rm 10.10.10.13 # remove member `10.10.10.13` from openbao cluster +# +#--------------------------------------------------------------# +# Tasks +#--------------------------------------------------------------# +# openbao_safeguard : check safeguard and abort unless disabled (default: enabled) +# openbao_pause : pause for confirmation (5s) +# openbao_deregister : remove openbao targets from victoria-metrics +# openbao_leave : try graceful raft leave before purging (partial removal only) +# openbao_svc : stop and disable openbao service with systemd +# openbao_log : remove the audit trail (disable with `openbao_rm_log=false`) +# openbao_data : remove data, TLS keys and node seal key (disable with `openbao_rm_data=false`) +# openbao_pkg : uninstall openbao packages (enable with `openbao_rm_pkg=true`) +# +#--------------------------------------------------------------# +# DANGER +#--------------------------------------------------------------# +# Destroying the openbao cluster destroys the pg_tde principal keys. +# Any PostgreSQL cluster encrypted against this KMS - and every +# pgBackRest backup of it - becomes permanently unreadable. +# Verify no pg_tde cluster depends on this KMS before proceeding. +#--------------------------------------------------------------# +... diff --git a/roles/openbao_remove/README.md b/roles/openbao_remove/README.md new file mode 100644 index 000000000..4b578638e --- /dev/null +++ b/roles/openbao_remove/README.md @@ -0,0 +1,167 @@ +# Role: openbao_remove + +> Remove OpenBao KMS Cluster / Instance + +| **Module** | [OPENBAO](https://openbao.org/docs/) | +|-------------------|--------------------------------------| +| **Docs** | https://pigsty.io/docs/conf/pgtde | +| **Related Roles** | [`openbao`](../openbao) | + + +## Overview + +The `openbao_remove` role removes OpenBao instances from a cluster: + +- Deregister from Victoria Metrics +- Gracefully leave the raft cluster (partial removal only) +- Stop the openbao service +- Remove the audit trail +- Remove data, TLS keys and the node copy of the seal key +- Uninstall packages (optional) + +> **DANGER**: this is the most destructive removal in Pigsty. The OpenBao store holds the `pg_tde` **principal keys**. Destroying it makes every PostgreSQL cluster encrypted against this KMS permanently unreadable. Losing etcd costs you HA; losing the KMS costs you the data. + + +## Playbooks + +| Playbook | Description | +|------------------------------------------|-------------------------| +| [`openbao-rm.yml`](../../openbao-rm.yml) | Remove OpenBao instance | + + +## File Structure + +``` +roles/openbao_remove/ +├── defaults/ +│ └── main.yml # Default variables +├── meta/ +│ └── main.yml # Role metadata +└── tasks/ + ├── main.yml # Entry point + └── leave.yml # Graceful raft peer removal +``` + + +## Tags + +### Tag Hierarchy + +``` +openbao_remove (full role) +│ +├── openbao_safeguard # Safeguard check (always) - ENABLED by default +│ +├── openbao_pause # Pause for confirmation (5s) +│ +├── openbao_deregister # Deregister from monitoring +│ └── rm_metrics # Remove Victoria targets +│ +├── openbao_leave # Leave raft cluster gracefully +│ +├── openbao_svc # Stop openbao service +│ +├── openbao_log # Remove audit trail (if openbao_rm_log) +│ +├── openbao_data # Remove data + keys (if openbao_rm_data) +│ +└── openbao_pkg # Uninstall (if openbao_rm_pkg) +``` + + +## Key Variables + +| Variable | Default | Description | +|----------------------|---------|----------------------------| +| `openbao_safeguard` | `false` | Prevent accidental removal | +| `openbao_rm_data` | `true` | Remove data directories | +| `openbao_rm_log` | `true` | Remove log directories | +| `openbao_rm_pkg` | `false` | Uninstall openbao package | + + +## Safeguard Protection + +Enable safeguard to prevent accidental removal: + +```yaml +openbao: + vars: + openbao_safeguard: true +``` + +Override with: + +```bash +./openbao-rm.yml -l -e openbao_safeguard=false +``` + + +## Removal Scope + +| Component | What's Removed | +|--------------|---------------------------------------------| +| Monitoring | `/infra/targets/openbao/.yml` | +| Config | `/etc/openbao.d/` | +| TLS | `/etc/openbao.d/tls/{ca,server}.{crt,key}` | +| Seal key | `/etc/openbao.d/seal/-unseal.key` | +| Data | `{{ openbao_data }}`, `/var/lib/openbao` | +| Audit trail | `{{ openbao_log_dir }}` | +| Service | `{{ systemd_dir }}/openbao.service` | + +### Never touched + +The cluster-wide key material on the **admin node** survives the role, so a cluster can be rebuilt in place: + +``` +files/pki/bao/unseal.key # static seal key, decrypts the whole store +files/openbao/-init.json # root token + recovery shares +files/openbao/-token.json # pg_tde token issued by openbao_provision +files/pki/bao/-*.crt / *.key # per-member TLS material +``` + +Shred them by hand once you are certain the KMS will not be rebuilt. + +> **Trap on rebuild**: `-token.json` surviving is what lets a rebuilt cluster skip re-provisioning by mistake - `openbao_provision` only checks that this file exists on the controller, not that the `pg_tde` mount still exists in the (now empty) store. See [Rebuilding a Cluster](../openbao/README.md#rebuilding-a-cluster) in the `openbao` role. + +> With the default layout `openbao_log_dir` is nested inside `openbao_data`, so `openbao_rm_data=true` takes the audit trail with it regardless of `openbao_rm_log`. To keep the audit trail, either archive it first or configure `openbao_log_dir` outside the data directory. + + +## Cluster Leave Process + +Graceful raft removal is attempted **only when part of the cluster survives the play** - removing every member at once leaves nobody to evict peers from. + +1. Pick a surviving member from `groups[openbao_cluster]` minus the play hosts +2. Read the root token from `files/openbao/-init.json` on the controller +3. Run `bao operator raft remove-peer ` against the surviving member +4. Stop the local service and purge + +`bao operator raft remove-peer` needs a token with `sudo` capability on `sys/storage/raft/remove-peer`. The role opportunistically reuses the root token captured at init time; if that token was revoked - as the [`openbao`](../openbao) role's README recommends - the step is skipped and the play prints the manual command instead: + +```bash +bao operator raft remove-peer - +``` + +Always verify afterwards on a surviving member: + +```bash +bao operator raft list-peers +``` + +A stale peer left in the raft configuration counts against quorum. + + +## Wipe and Rebuild + +```bash +./openbao-rm.yml -l openbao -e openbao_safeguard=false # destroy +./openbao.yml # redeploy +``` + +The seal key on the admin node is reused, so the rebuilt cluster unseals normally - but it comes up **empty**. See [Rebuilding a Cluster](../openbao/README.md#rebuilding-a-cluster) in the `openbao` role for why this is not a recovery procedure. + + +## See Also + +- [`openbao`](../openbao): Deploy the OpenBao KMS cluster +- [`ca`](../ca): Issues the OpenBao server certificates +- [OpenBao Documentation](https://openbao.org/docs/) diff --git a/roles/openbao_remove/defaults/main.yml b/roles/openbao_remove/defaults/main.yml new file mode 100644 index 000000000..15297998b --- /dev/null +++ b/roles/openbao_remove/defaults/main.yml @@ -0,0 +1,21 @@ +--- +#----------------------------------------------------------------- +# OPENBAO_REMOVE +#----------------------------------------------------------------- +# NOTE: unlike the other *_remove roles, the safeguard defaults to TRUE. +# Destroying an openbao cluster destroys the principal keys held for +# pg_tde: the encrypted PostgreSQL data and its backups become +# permanently unreadable. Removal must be an explicit, deliberate act. +openbao_safeguard: true # prevents purging openbao instance? true by default +openbao_rm_data: true # purging openbao data and config? true by default +openbao_rm_log: true # purging openbao log dir (holds the audit trail)? true by default +openbao_rm_pkg: false # uninstall openbao packages? false by default + +#----------------------------------------------------------------- +# OPENBAO (REFERENCE) +#----------------------------------------------------------------- +openbao_cluster: openbao # bao cluster & group name, openbao by default +openbao_data: /data/openbao # bao data dir, `/data/openbao` by default +openbao_log_dir: /data/openbao/log # bao log dir, `/data/openbao/log` by default +openbao_api_port: 8200 # bao api port, 8200 by default +... diff --git a/roles/openbao_remove/meta/main.yml b/roles/openbao_remove/meta/main.yml new file mode 100644 index 000000000..88bb33133 --- /dev/null +++ b/roles/openbao_remove/meta/main.yml @@ -0,0 +1,18 @@ +--- +galaxy_info: + role_name: openbao_remove + description: remove openbao module from pigsty managed nodes + min_ansible_version: "2.9" + author: 'Ruohang Feng / @Vonng / rh@vonng.com' + license: Apache-2.0 + platforms: + - name: EL + versions: [all] + - name: Ubuntu + versions: [all] + - name: Debian + versions: [all] + galaxy_tags: + - pigsty + - openbao +... diff --git a/roles/openbao_remove/tasks/leave.yml b/roles/openbao_remove/tasks/leave.yml new file mode 100644 index 000000000..fb954f822 --- /dev/null +++ b/roles/openbao_remove/tasks/leave.yml @@ -0,0 +1,49 @@ +--- +#--------------------------------------------------------------# +# Leave openbao raft cluster gracefully +#--------------------------------------------------------------# +# `bao operator raft remove-peer` must run against a REMAINING member, +# not the one being evicted, and it needs a token with sudo capability +# on sys/storage/raft/remove-peer. The role has no token of its own, so +# it opportunistically reuses the root token captured at init time. If +# that token was revoked (as the openbao role's README recommends), the +# graceful leave is skipped and the operator has to do it by hand. +#--------------------------------------------------------------# + +- name: pick a surviving openbao peer to run remove-peer against + set_fact: + openbao_surviving_peer: "{{ (groups[openbao_cluster] | default([]) | difference(ansible_play_hosts | default([]))) | first | default('') }}" + +- name: read root token from init material on the controller + no_log: true + ignore_errors: true + set_fact: + openbao_root_token: >- + {{ (lookup('file', playbook_dir ~ '/files/openbao/' ~ openbao_cluster ~ '-init.json', errors='ignore') + | default('{}', true) | from_json).root_token | default('') }} + +- name: remove this member from the raft configuration + when: + - openbao_surviving_peer != '' + - openbao_root_token | default('') != '' + delegate_to: "{{ openbao_surviving_peer }}" + ignore_errors: true + no_log: true + environment: + BAO_ADDR: "https://127.0.0.1:{{ openbao_api_port }}" + BAO_CACERT: "/etc/openbao.d/tls/ca.crt" + BAO_TOKEN: "{{ openbao_root_token }}" + command: bao operator raft remove-peer {{ openbao_instance }} + register: openbao_leave_result + +- name: report graceful leave outcome + debug: + msg: >- + {% if openbao_leave_result is defined and not (openbao_leave_result.skipped | default(false)) and openbao_leave_result.rc | default(1) == 0 -%} + removed raft peer {{ openbao_instance }} via {{ openbao_surviving_peer }} + {%- else -%} + could not remove raft peer {{ openbao_instance }} automatically - + run this on a surviving member: bao operator raft remove-peer {{ openbao_instance }} + {%- endif %} + +... diff --git a/roles/openbao_remove/tasks/main.yml b/roles/openbao_remove/tasks/main.yml new file mode 100644 index 000000000..4169cba4d --- /dev/null +++ b/roles/openbao_remove/tasks/main.yml @@ -0,0 +1,110 @@ +--- +#--------------------------------------------------------------# +# HONOR OPENBAO_SAFEGUARD [openbao_safeguard] +#--------------------------------------------------------------# +- name: ABORT due to openbao_safeguard enabled + connection: local + when: openbao_safeguard|bool + tags: [ always , openbao_safeguard ] + any_errors_fatal: true + ignore_errors: false + fail: msg="Abort due to openbao_safeguard for {{ openbao_cluster }}-{{ openbao_seq }} @ {{ inventory_hostname }} is enabled, override with -e openbao_safeguard=false" + +#--------------------------------------------------------------# +# PAUSE FOR CONFIRMATION [openbao_pause] +#--------------------------------------------------------------# +# pause for 5 seconds to allow user to abort in case of fat finger +# longer than the other modules on purpose: this destroys key material +- name: "pause for confirmation, Use Ctrl-C to ABORT" + tags: openbao_pause + pause: { seconds: 5 } + +#--------------------------------------------------------------# +# remove openbao targets from victoria [openbao_deregister] +#--------------------------------------------------------------# +- name: remove openbao from victoria targets + tags: [ openbao_deregister, deregister , rm_metrics ] + ignore_errors: true + delegate_to: '{{ item }}' + loop: '{{ groups["infra"]|default([]) }}' + file: path="/infra/targets/openbao/{{ openbao_cluster }}-{{ openbao_seq }}.yml" state=absent + +#--------------------------------------------------------------# +# leave openbao raft cluster gracefully [openbao_leave] +#--------------------------------------------------------------# +# only attempted when part of the cluster survives this play, +# removing the whole cluster has nobody left to remove peers from +- import_tasks: 'leave.yml' + ignore_errors: true + tags: openbao_leave + when: (groups[openbao_cluster]|default([])|length) > (ansible_play_hosts|default([])|length) + vars: { openbao_instance: "{{ openbao_cluster }}-{{ openbao_seq }}" } + +#--------------------------------------------------------------# +# Remove openbao service [openbao_svc] +#--------------------------------------------------------------# +- name: stop openbao server service + ignore_errors: true + tags: [openbao, openbao_svc, openbao_stop] + shell: | + systemctl disable openbao + systemctl stop openbao + + # try graceful shutdown first again + if ps -u openbao -o command | grep -q 'bao' ; then + systemctl stop openbao + fi + + # kill if still exists + if ps -u openbao -o command | grep -q 'bao' ; then + sleep 2 + ps -u openbao -o pid:1,command | grep 'bao' | awk '{print $1}' | xargs kill + fi + + # kill -9 if not killed + if ps -u openbao -o command | grep -q 'bao' ; then + sleep 5 + ps -u openbao -o pid:1,command | grep 'bao' | awk '{print $1}' | xargs kill -9 + fi + # guaranteed success + exit 0 + args: { executable: /bin/bash } + +#--------------------------------------------------------------# +# Remove openbao log [openbao_log] +#--------------------------------------------------------------# +# the audit trail lives here, so it is removed as its own step. +# NOTE: with the default layout openbao_log_dir is nested inside +# openbao_data, so openbao_rm_data=true takes it regardless. +- name: remove openbao log dir + tags: openbao_log + when: openbao_rm_log|bool + file: path={{ openbao_log_dir }} state=absent + +#--------------------------------------------------------------# +# Remove openbao data [openbao_data] +#--------------------------------------------------------------# +# /etc/openbao.d holds the TLS keys AND the node copy of the static +# seal key, so it always goes with the data. The cluster-wide copies +# on the admin node (files/pki/bao/unseal.key, files/openbao/*.json) +# are deliberately never touched by this role. +- name: remove openbao data + tags: openbao_data + when: openbao_rm_data|bool + file: path={{ item }} state=absent + loop: + - "/etc/openbao.d" + - "/var/lib/openbao" + - "{{ openbao_data }}" + - "{{ systemd_dir }}/openbao.service" + +#--------------------------------------------------------------# +# Uninstall openbao packages [openbao_pkg] +#--------------------------------------------------------------# +- name: uninstall openbao package + tags: openbao_pkg + when: openbao_rm_pkg|bool + ignore_errors: true + package: name=openbao state=absent + +... From 01d47daa2a0b93133b6ab190872169933fb0b97f Mon Sep 17 00:00:00 2001 From: alemacci Date: Sun, 16 Aug 2026 14:56:33 +0200 Subject: [PATCH 3/9] feat(ca): add dedicated directory for OpenBao TLS material --- roles/ca/tasks/main.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/roles/ca/tasks/main.yml b/roles/ca/tasks/main.yml index b80e5b0ab..8d4826509 100644 --- a/roles/ca/tasks/main.yml +++ b/roles/ca/tasks/main.yml @@ -14,6 +14,7 @@ - { path: files/pki, mode: '0755' } - { path: files/pki/ca, mode: '0700' } # CA private key directory, restricted access - { path: files/pki/csr, mode: '0755' } + - { path: files/pki/bao, mode: '0755' } - { path: files/pki/misc, mode: '0755' } - { path: files/pki/etcd, mode: '0755' } - { path: files/pki/kafka, mode: '0755' } From 3b288d30bfda65e67d024054a27c7f33751dbf20 Mon Sep 17 00:00:00 2001 From: alemacci Date: Sun, 16 Aug 2026 14:59:04 +0200 Subject: [PATCH 4/9] feat(pgsql): pg_tde logic, install token and adjust templates --- roles/pgsql/defaults/main.yml | 6 ++++ roles/pgsql/tasks/main.yml | 8 +++++ roles/pgsql/tasks/patroni.yml | 20 +++++++++++- roles/pgsql/tasks/tde.yml | 26 ++++++++++++++++ roles/pgsql/templates/crit.yml | 15 +++++++++ roles/pgsql/templates/olap.yml | 15 +++++++++ roles/pgsql/templates/oltp.yml | 15 +++++++++ roles/pgsql/templates/pg-init | 6 ++++ roles/pgsql/templates/pg-init-kms.sql | 44 +++++++++++++++++++++++++++ roles/pgsql/templates/pgbackrest.conf | 16 ++++++++-- roles/pgsql/templates/tiny.yml | 15 +++++++++ 11 files changed, 182 insertions(+), 4 deletions(-) create mode 100644 roles/pgsql/tasks/tde.yml create mode 100644 roles/pgsql/templates/pg-init-kms.sql diff --git a/roles/pgsql/defaults/main.yml b/roles/pgsql/defaults/main.yml index 9871eda22..209e7b4f0 100644 --- a/roles/pgsql/defaults/main.yml +++ b/roles/pgsql/defaults/main.yml @@ -242,6 +242,12 @@ node_firewall_intranet: # which intranet cidr considered as internal n etcd_port: 2379 # etcd client port, 2379 by default etcd_root_password: Etcd.Root # etcd root password for RBAC #----------------------------------------------------------------- +# OPENBAO (Reference) +#----------------------------------------------------------------- +openbao_cluster: openbao # bao cluster & group name, openbao by default +openbao_domain: bao.pigsty # bao domain name, `bao.pigsty` by default +openbao_api_port: 8200 # used to advertise other servers in the cluster for client redirection, `8200` by default +#----------------------------------------------------------------- # PROMETHEUS (Reference) #----------------------------------------------------------------- exporter_metrics_path: /metrics # exporter metric path, `/metrics` by default diff --git a/roles/pgsql/tasks/main.yml b/roles/pgsql/tasks/main.yml index 2a794221f..d394c6505 100644 --- a/roles/pgsql/tasks/main.yml +++ b/roles/pgsql/tasks/main.yml @@ -28,6 +28,14 @@ - import_tasks: cert.yml tags: [ pg_bootstrap, postgres, patroni, pg_cert ] +#--------------------------------------------------------------# +# pg_tde Token [pg_bootstrap][pg_tde] +#--------------------------------------------------------------# +# install the bao token pg_tde uses to fetch its principal keys +- import_tasks: tde.yml + tags: [ pg_bootstrap, postgres, patroni, pg_tde ] + when: pg_mode == 'pgtde' + #--------------------------------------------------------------# # Launch [pg_bootstrap][pg_launch] #--------------------------------------------------------------# diff --git a/roles/pgsql/tasks/patroni.yml b/roles/pgsql/tasks/patroni.yml index fa888e175..211f3baef 100644 --- a/roles/pgsql/tasks/patroni.yml +++ b/roles/pgsql/tasks/patroni.yml @@ -77,7 +77,7 @@ tags: pg_restart become_user: "{{ pg_dbsu }}" shell: | - {% if pg_mode|default('pgsql') == 'citus' %} + {% if pg_mode|default('pgsql') == 'citus' %} /usr/bin/patronictl -c /etc/patroni/patroni.yml restart {{ pg_shard }} --group {{ pg_group }} --force {% else %} /usr/bin/patronictl -c /etc/patroni/patroni.yml restart {{ pg_cluster }} --force @@ -112,6 +112,7 @@ with_items: - { src: "pg-init-roles.sql" ,dest: "/pg/tmp/pg-init-roles.sql" ,mode: "0640" } - { src: "pg-init-template.sql" ,dest: "/pg/tmp/pg-init-template.sql" ,mode: "0640" } + - { src: "pg-init-kms.sql" ,dest: "/pg/tmp/pg-init-kms.sql" ,mode: "0640" } - { src: "{{ pg_init }}" ,dest: "/pg/bin/pg-init" ,owner: root ,mode: "0740" } - name: init postgres cluster @@ -119,6 +120,23 @@ become_user: "{{ pg_dbsu }}" command: bash /pg/bin/pg-init + - name: apply pgtde GUCs and restart + tags: pg_init_tde + when: + - pg_mode == 'pgtde' + - groups['openbao'] is defined + - groups['openbao'] | length > 0 + become_user: "{{ pg_dbsu }}" + shell: | + /usr/bin/patronictl -c /etc/patroni/patroni.yml edit-config {{ pg_cluster }} --pg "pg_tde.wal_encrypt='on'" --force + /usr/bin/patronictl -c /etc/patroni/patroni.yml edit-config {{ pg_cluster }} --pg "pg_tde.enforce_encryption='on'" --force + /usr/bin/patronictl -c /etc/patroni/patroni.yml edit-config {{ pg_cluster }} --pg "default_table_access_method=tde_heap" --force + /usr/bin/patronictl -c /etc/patroni/patroni.yml restart {{ pg_cluster }} --force + args: { executable: /bin/bash } + + - name: wait for postgres primary + wait_for: host={{ inventory_hostname }} port={{ pg_port }} state=started timeout=60 + - name: check postgres primary inited tags: pg_init_exec become_user: "{{ pg_dbsu }}" diff --git a/roles/pgsql/tasks/tde.yml b/roles/pgsql/tasks/tde.yml new file mode 100644 index 000000000..1be81f70a --- /dev/null +++ b/roles/pgsql/tasks/tde.yml @@ -0,0 +1,26 @@ +--- +#----------------------------------------------------------------# +# Install pg_tde key provider token [pg_tde] +#----------------------------------------------------------------# +# Takes the token issued by the openbao role and installs it where +# pg_tde reads it. Token must be in place before patroni starts +# postgres, since an encrypted cluster needs it to fetch its +# principal keys at startup. +#--------------------------------------------------------------------# +- name: install pg_tde key provider token + tags: pg_tde + when: + - groups['openbao'] is defined + - groups['openbao'] | length > 0 + copy: + content: "{{ pg_tde_token }}" + dest: "/pg/tde-token" + owner: "{{ pg_dbsu }}" + group: postgres + mode: '0600' + vars: + pg_tde_token: >- + {{ (lookup('file', 'files/openbao/' ~ (openbao_cluster | default('openbao')) ~ '-token.json', + errors='ignore') | default('{}', true) | from_json).auth.client_token | default('') }} + +... diff --git a/roles/pgsql/templates/crit.yml b/roles/pgsql/templates/crit.yml index 7183b162f..4afeba9ae 100644 --- a/roles/pgsql/templates/crit.yml +++ b/roles/pgsql/templates/crit.yml @@ -352,7 +352,12 @@ bootstrap: {% if pgbackrest_enabled|bool %} archive_mode: 'on' archive_timeout: 300 # 5min archive timeout +{% if pg_mode == 'pgtde' %} + archive_command: '{{ pg_bin_dir }}/pg_tde_archive_decrypt %f %p "pgbackrest --stanza={{ pg_cluster }} archive-push %%p"' + restore_command: '{{ pg_bin_dir }}/pg_tde_restore_encrypt %f %p "pgbackrest --stanza={{ pg_cluster }} archive-get %%f %%p"' +{% else %} archive_command: 'pgbackrest --stanza={{ pg_cluster }} archive-push %p' +{% endif %} {% endif %} #------------------------------------------------------# @@ -491,6 +496,11 @@ bootstrap: polar_resource_manager.enable_resource_manager: 'off' {% endif %} +{% if pg_mode|default('pgsql') == 'pgtde' %} + # pg_tde + pg_tde.inherit_global_providers: 'on' +{% endif %} + # pg_cron {% if pg_parameters is defined and pg_parameters is mapping and 'cron.database_name' in pg_parameters %} cron.database_name: {{ pg_parameters['cron.database_name'] }} @@ -511,6 +521,11 @@ postgresql: bin_dir: {{ pg_bin_dir }} data_dir: {{ pg_data }} config_dir: {{ pg_data }} +{% if pg_mode == 'pgtde' %} + bin_name: + pg_basebackup: pg_tde_basebackup + pg_rewind: pg_tde_rewind +{% endif %} pgpass: {{ pg_dbsu_home }}/.pgpass listen: {{ pg_listen_address }}:{{ pg_port }} connect_address: {{ inventory_hostname }}:{{ pg_port }} diff --git a/roles/pgsql/templates/olap.yml b/roles/pgsql/templates/olap.yml index 1b594f3d9..27840c65c 100644 --- a/roles/pgsql/templates/olap.yml +++ b/roles/pgsql/templates/olap.yml @@ -336,7 +336,12 @@ bootstrap: {% if pgbackrest_enabled|bool %} archive_mode: 'on' archive_timeout: 300 # 5min archive timeout +{% if pg_mode == 'pgtde' %} + archive_command: '{{ pg_bin_dir }}/pg_tde_archive_decrypt %f %p "pgbackrest --stanza={{ pg_cluster }} archive-push %%p"' + restore_command: '{{ pg_bin_dir }}/pg_tde_restore_encrypt %f %p "pgbackrest --stanza={{ pg_cluster }} archive-get %%f %%p"' +{% else %} archive_command: 'pgbackrest --stanza={{ pg_cluster }} archive-push %p' +{% endif %} {% endif %} min_wal_size: {{ ([pg_size_twentieth, 200])|min }}GB # 1/20 disk size, max 200GB max_wal_size: {{ ([pg_size_twentieth * 4, 2000])|min }}GB # 2/10 disk size, max 2000GB @@ -485,6 +490,11 @@ bootstrap: polar_resource_manager.enable_resource_manager: 'off' {% endif %} +{% if pg_mode|default('pgsql') == 'pgtde' %} + # pg_tde + pg_tde.inherit_global_providers: 'on' +{% endif %} + # pg_cron {% if pg_parameters is defined and pg_parameters is mapping and 'cron.database_name' in pg_parameters %} cron.database_name: {{ pg_parameters['cron.database_name'] }} @@ -505,6 +515,11 @@ postgresql: bin_dir: {{ pg_bin_dir }} data_dir: {{ pg_data }} config_dir: {{ pg_data }} +{% if pg_mode == 'pgtde' %} + bin_name: + pg_basebackup: pg_tde_basebackup + pg_rewind: pg_tde_rewind +{% endif %} pgpass: {{ pg_dbsu_home }}/.pgpass listen: {{ pg_listen_address }}:{{ pg_port }} connect_address: {{ inventory_hostname }}:{{ pg_port }} diff --git a/roles/pgsql/templates/oltp.yml b/roles/pgsql/templates/oltp.yml index c20b91577..b296f0481 100644 --- a/roles/pgsql/templates/oltp.yml +++ b/roles/pgsql/templates/oltp.yml @@ -356,7 +356,12 @@ bootstrap: {% if pgbackrest_enabled|bool %} archive_mode: 'on' archive_timeout: 300 # 5min archive timeout +{% if pg_mode == 'pgtde' %} + archive_command: '{{ pg_bin_dir }}/pg_tde_archive_decrypt %f %p "pgbackrest --stanza={{ pg_cluster }} archive-push %%p"' + restore_command: '{{ pg_bin_dir }}/pg_tde_restore_encrypt %f %p "pgbackrest --stanza={{ pg_cluster }} archive-get %%f %%p"' +{% else %} archive_command: 'pgbackrest --stanza={{ pg_cluster }} archive-push %p' +{% endif %} {% endif %} #------------------------------------------------------# @@ -491,6 +496,11 @@ bootstrap: polar_resource_manager.enable_resource_manager: 'off' {% endif %} +{% if pg_mode|default('pgsql') == 'pgtde' %} + # pg_tde + pg_tde.inherit_global_providers: 'on' +{% endif %} + # pg_cron {% if pg_parameters is defined and pg_parameters is mapping and 'cron.database_name' in pg_parameters %} cron.database_name: {{ pg_parameters['cron.database_name'] }} @@ -511,6 +521,11 @@ postgresql: bin_dir: {{ pg_bin_dir }} data_dir: {{ pg_data }} config_dir: {{ pg_data }} +{% if pg_mode == 'pgtde' %} + bin_name: + pg_basebackup: pg_tde_basebackup + pg_rewind: pg_tde_rewind +{% endif %} pgpass: {{ pg_dbsu_home }}/.pgpass listen: {{ pg_listen_address }}:{{ pg_port }} connect_address: {{ inventory_hostname }}:{{ pg_port }} diff --git a/roles/pgsql/templates/pg-init b/roles/pgsql/templates/pg-init index 8694212d8..88c56eab9 100755 --- a/roles/pgsql/templates/pg-init +++ b/roles/pgsql/templates/pg-init @@ -39,6 +39,12 @@ PROG_DIR="$(cd $(dirname $0) && pwd)" {{ pg_bin_dir }}/psql -h {{ pg_localhost|default('/var/run/postgresql') }} -p {{ pg_port|default(5432) }} ivorysql -qAXwtf /pg/tmp/pg-init-template.sql {% endif %} +{% if pg_mode == 'pgtde' %} +# configure key management +{{ pg_bin_dir }}/psql -h {{ pg_localhost|default('/var/run/postgresql') }} -p {{ pg_port|default(5432) }} postgres -qAXwtc "CREATE EXTENSION IF NOT EXISTS pg_tde;" +{{ pg_bin_dir }}/psql -h {{ pg_localhost|default('/var/run/postgresql') }} -p {{ pg_port|default(5432) }} postgres -qAXwtf /pg/tmp/pg-init-kms.sql +{% endif %} + #--------------------------------------------------------------# # Customize Logic # #--------------------------------------------------------------# diff --git a/roles/pgsql/templates/pg-init-kms.sql b/roles/pgsql/templates/pg-init-kms.sql new file mode 100644 index 000000000..f51a80972 --- /dev/null +++ b/roles/pgsql/templates/pg-init-kms.sql @@ -0,0 +1,44 @@ +---------------------------------------------------------------------- +-- File : pg-init-kms.sql +-- Desc : init kms for postgres cluster {{ pg_cluster }} +-- Time : {{ '%Y-%m-%d %H:%M' | strftime }} +-- Host : {{ pg_instance }} @ {{ inventory_hostname }}:{{ pg_port }} +-- Path : /pg/tmp/pg-init-kms.sql +-- Note : ANSIBLE MANAGED, DO NOT CHANGE! +-- License : Apache-2.0 @ https://pigsty.io/docs/about/license/ +-- Copyright : 2018-2026 Ruohang Feng / Vonng (rh@vonng.com) +---------------------------------------------------------------------- + +--==================================================================-- +-- Executions -- +--==================================================================-- +-- {{ pg_bin_dir}}/psql template1 -AXtwqf /pg/tmp/pg-init-kms.sql +-- this sql scripts is responsible for kms init procedure +-- it will +-- * connect to your PostgreSQL database and add OpenBao as a global key provider +-- * create and set the default principal key using global key provider + + + +--==================================================================-- +-- Logic -- +--==================================================================-- +{% if groups['openbao'] is defined and groups['openbao'] | length > 0 %} +SELECT pg_tde_add_global_key_provider_vault_v2( + '{{ openbao_cluster }}', + 'https://{{ openbao_domain }}:{{ openbao_api_port }}', + 'pg_tde', + '/pg/tde-token', + '/pg/cert/ca.crt' +); + +SELECT pg_tde_create_key_using_global_key_provider( + '{{ pg_cluster }}-principal-key', + '{{ openbao_cluster }}' +); + +SELECT pg_tde_set_default_key_using_global_key_provider( + '{{ pg_cluster }}-principal-key', + '{{ openbao_cluster }}' +); +{% endif %} diff --git a/roles/pgsql/templates/pgbackrest.conf b/roles/pgsql/templates/pgbackrest.conf index fb0fd5073..8fb1fbbbb 100644 --- a/roles/pgsql/templates/pgbackrest.conf +++ b/roles/pgsql/templates/pgbackrest.conf @@ -16,8 +16,7 @@ [{{ pg_cluster }}] pg1-path={{ pg_data }} pg1-port={{ pg_port }} -{%if pg_mode|default('pgsql') == 'ivory' %} -archive-header-check=n +{% if pg_mode == 'ivory' or pg_mode == 'pgtde' %} pg-version-force={{ pg_version|default(18) }} {% endif %} @@ -25,8 +24,15 @@ pg-version-force={{ pg_version|default(18) }} # 2. archive #--------------------------------------------------------------# [global] +{% if pg_mode == 'pgtde' %} +archive-header-check=n +checksum-page=n +archive-async=n +{% else %} archive-async=y +archive-get-queue-max=512MiB archive-push-queue-max=4GiB +{% endif %} #--------------------------------------------------------------# # 3. backup @@ -39,7 +45,11 @@ start-fast=y #--------------------------------------------------------------# # 4. general #--------------------------------------------------------------# +{% if pg_mode == 'pgtde' %} +compress-type=none +{% else %} compress-type=zst +{% endif %} spool-path=/pg/spool #--------------------------------------------------------------# @@ -48,6 +58,7 @@ spool-path=/pg/spool log-path={{ pgbackrest_log_dir }} log-level-file=info log-level-console=info + #--------------------------------------------------------------# # 6. maintainer #--------------------------------------------------------------# @@ -59,7 +70,6 @@ log-level-console=info repo1-{{ k|replace('_', '-') }}={{ v|replace('${pg_cluster}', pg_cluster) }} {% endfor %} - #--------------------------------------------------------------# # 8. restore #--------------------------------------------------------------# diff --git a/roles/pgsql/templates/tiny.yml b/roles/pgsql/templates/tiny.yml index 8bd8489b4..a95054a4f 100644 --- a/roles/pgsql/templates/tiny.yml +++ b/roles/pgsql/templates/tiny.yml @@ -349,7 +349,12 @@ bootstrap: {% if pgbackrest_enabled|bool %} archive_mode: 'on' archive_timeout: 300 # 5min archive timeout +{% if pg_mode == 'pgtde' %} + archive_command: '{{ pg_bin_dir }}/pg_tde_archive_decrypt %f %p "pgbackrest --stanza={{ pg_cluster }} archive-push %%p"' + restore_command: '{{ pg_bin_dir }}/pg_tde_restore_encrypt %f %p "pgbackrest --stanza={{ pg_cluster }} archive-get %%f %%p"' +{% else %} archive_command: 'pgbackrest --stanza={{ pg_cluster }} archive-push %p' +{% endif %} {% endif %} #------------------------------------------------------# @@ -478,6 +483,11 @@ bootstrap: polar_resource_manager.enable_resource_manager: 'off' {% endif %} +{% if pg_mode|default('pgsql') == 'pgtde' %} + # pg_tde + pg_tde.inherit_global_providers: 'on' +{% endif %} + # pg_cron {% if pg_parameters is defined and pg_parameters is mapping and 'cron.database_name' in pg_parameters %} cron.database_name: {{ pg_parameters['cron.database_name'] }} @@ -498,6 +508,11 @@ postgresql: bin_dir: {{ pg_bin_dir }} data_dir: {{ pg_data }} config_dir: {{ pg_data }} +{% if pg_mode == 'pgtde' %} + bin_name: + pg_basebackup: pg_tde_basebackup + pg_rewind: pg_tde_rewind +{% endif %} pgpass: {{ pg_dbsu_home }}/.pgpass listen: {{ pg_listen_address }}:{{ pg_port }} connect_address: {{ inventory_hostname }}:{{ pg_port }} From e61ac368a33c39fa7fa23e377de92fbb888bf50d Mon Sep 17 00:00:00 2001 From: alemacci Date: Sun, 16 Aug 2026 15:00:28 +0200 Subject: [PATCH 5/9] feat(openbao): add dashboard, monitoring integration and update docs index --- files/grafana/infra/openbao-overview.json | 4318 +++++++++++++++++ files/grafana/pigsty.json | 20 +- files/victoria/rules/openbao.yml | 219 + roles/infra/tasks/dir.yml | 1 + roles/infra/templates/nginx/index.html.j2 | 9 + roles/infra/templates/nginx/index.zh.html.j2 | 9 + .../templates/prometheus/alertmanager.yml | 2 +- roles/infra/templates/victoria/agent.yml | 4 + roles/infra/templates/victoria/prometheus.yml | 16 + 9 files changed, 4595 insertions(+), 3 deletions(-) create mode 100644 files/grafana/infra/openbao-overview.json create mode 100644 files/victoria/rules/openbao.yml diff --git a/files/grafana/infra/openbao-overview.json b/files/grafana/infra/openbao-overview.json new file mode 100644 index 000000000..be5650f78 --- /dev/null +++ b/files/grafana/infra/openbao-overview.json @@ -0,0 +1,4318 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "grafana", + "uid": "grafana" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "target": { + "limit": 100, + "matchAny": false, + "tags": [], + "type": "dashboard" + }, + "type": "dashboard" + } + ] + }, + "author": "Ruohang Feng (rh@vonng.com)", + "description": "OpenBao KMS Service Overview", + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 0, + "id": null, + "license": "https://pigsty.io/docs/about/license/", + "liveNow": false, + "panels": [ + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 100, + "title": "Overview", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "ds-prometheus" + }, + "description": "Raft autopilot verdict on the whole cluster: 1 = all voters healthy", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "decimals": 0, + "mappings": [ + { + "options": { + "0": { + "color": "#cc4637d9", + "index": 0, + "text": "Degraded" + }, + "1": { + "color": "#346f36cc", + "index": 1, + "text": "Healthy" + } + }, + "type": "value" + } + ], + "noValue": "N/A", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "#cc4637d9", + "value": 0 + }, + { + "color": "#346f36cc", + "value": 1 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 4, + "x": 0, + "y": 1 + }, + "id": 101, + "options": { + "colorMode": "background", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "horizontal", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "last" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "value", + "wideLayout": true + }, + "pluginVersion": "13.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "ds-prometheus" + }, + "editorMode": "code", + "expr": "min(vault_autopilot_healthy{job=\"openbao\", cls=\"$cls\"})", + "legendFormat": "Autopilot", + "refId": "A", + "exemplar": false, + "instant": true, + "range": false + } + ], + "title": "Autopilot", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "ds-prometheus" + }, + "description": "Number of members in the raft configuration", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "decimals": 0, + "mappings": [], + "noValue": "N/A", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "#3e668f", + "value": 0 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 4, + "x": 4, + "y": 1 + }, + "id": 102, + "options": { + "colorMode": "background", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "horizontal", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "last" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "value", + "wideLayout": true + }, + "pluginVersion": "13.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "ds-prometheus" + }, + "editorMode": "code", + "expr": "max(vault_raft_peers{job=\"openbao\", cls=\"$cls\"})", + "legendFormat": "Peers", + "refId": "A", + "exemplar": false, + "instant": true, + "range": false + } + ], + "title": "Raft Peers", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "ds-prometheus" + }, + "description": "How many more members can fail before quorum is lost", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "decimals": 0, + "mappings": [], + "noValue": "N/A", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "#cc4637d9", + "value": 0 + }, + { + "color": "#f79f64", + "value": 1 + }, + { + "color": "#346f36cc", + "value": 2 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 4, + "x": 8, + "y": 1 + }, + "id": 103, + "options": { + "colorMode": "background", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "horizontal", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "last" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "value", + "wideLayout": true + }, + "pluginVersion": "13.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "ds-prometheus" + }, + "editorMode": "code", + "expr": "min(vault_autopilot_failure_tolerance{job=\"openbao\", cls=\"$cls\"})", + "legendFormat": "Tolerance", + "refId": "A", + "exemplar": false, + "instant": true, + "range": false + } + ], + "title": "Failure Tolerance", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "ds-prometheus" + }, + "description": "Members currently unsealed and able to serve secrets", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "decimals": 0, + "mappings": [], + "noValue": "N/A", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "#cc4637d9", + "value": 0 + }, + { + "color": "#f79f64", + "value": 1 + }, + { + "color": "#346f36cc", + "value": 2 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 4, + "x": 12, + "y": 1 + }, + "id": 104, + "options": { + "colorMode": "background", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "horizontal", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "last" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "value", + "wideLayout": true + }, + "pluginVersion": "13.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "ds-prometheus" + }, + "editorMode": "code", + "expr": "openbao:cls:unsealed{job=\"openbao\", cls=\"$cls\"}", + "legendFormat": "Unsealed", + "refId": "A", + "exemplar": false, + "instant": true, + "range": false + } + ], + "title": "Unsealed Members", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "ds-prometheus" + }, + "description": "Members claiming the active (leader) role, must be exactly 1", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "decimals": 0, + "mappings": [], + "noValue": "N/A", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "#cc4637d9", + "value": 0 + }, + { + "color": "#346f36cc", + "value": 1 + }, + { + "color": "#f79f64", + "value": 2 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 4, + "x": 16, + "y": 1 + }, + "id": 105, + "options": { + "colorMode": "background", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "horizontal", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "last" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "value", + "wideLayout": true + }, + "pluginVersion": "13.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "ds-prometheus" + }, + "editorMode": "code", + "expr": "openbao:cls:active{job=\"openbao\", cls=\"$cls\"}", + "legendFormat": "Active", + "refId": "A", + "exemplar": false, + "instant": true, + "range": false + } + ], + "title": "Active Node", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "ds-prometheus" + }, + "description": "Total number of leases eligible for eventual expiry", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "decimals": 0, + "mappings": [], + "noValue": "N/A", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "#3e668f", + "value": 0 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 4, + "x": 20, + "y": 1 + }, + "id": 106, + "options": { + "colorMode": "background", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "horizontal", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "last" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "value", + "wideLayout": true + }, + "pluginVersion": "13.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "ds-prometheus" + }, + "editorMode": "code", + "expr": "max(vault_expire_num_leases{job=\"openbao\", cls=\"$cls\"})", + "legendFormat": "Leases", + "refId": "A", + "exemplar": false, + "instant": true, + "range": false + } + ], + "title": "Leases", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "ds-prometheus" + }, + "description": "One row per openbao member of cluster $cls", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "custom": { + "align": "auto", + "cellOptions": { + "type": "auto" + }, + "filterable": false, + "inspect": false + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "text", + "value": 0 + } + ] + } + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Alive" + }, + "properties": [ + { + "id": "mappings", + "value": [ + { + "options": { + "0": { + "color": "#cc4637d9", + "index": 0, + "text": "Down" + }, + "1": { + "color": "#346f36cc", + "index": 1, + "text": "Alive" + } + }, + "type": "value" + } + ] + }, + { + "id": "custom.cellOptions", + "value": { + "type": "color-text" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Seal" + }, + "properties": [ + { + "id": "mappings", + "value": [ + { + "options": { + "0": { + "color": "#cc4637d9", + "index": 0, + "text": "🔒 Sealed" + }, + "1": { + "color": "#346f36cc", + "index": 1, + "text": "🔓 Unsealed" + } + }, + "type": "value" + } + ] + }, + { + "id": "custom.cellOptions", + "value": { + "type": "color-text" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Role" + }, + "properties": [ + { + "id": "mappings", + "value": [ + { + "options": { + "0": { + "color": "#3e668f", + "index": 0, + "text": "Standby" + }, + "1": { + "color": "#346f36cc", + "index": 1, + "text": "Active" + } + }, + "type": "value" + } + ] + }, + { + "id": "custom.cellOptions", + "value": { + "type": "color-text" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Healthy" + }, + "properties": [ + { + "id": "mappings", + "value": [ + { + "options": { + "0": { + "color": "#cc4637d9", + "index": 0, + "text": "Degraded" + }, + "1": { + "color": "#346f36cc", + "index": 1, + "text": "Healthy" + } + }, + "type": "value" + } + ] + }, + { + "id": "custom.cellOptions", + "value": { + "type": "color-text" + } + } + ] + } + ] + }, + "gridPos": { + "h": 7, + "w": 24, + "x": 0, + "y": 5 + }, + "id": 107, + "options": { + "cellHeight": "sm", + "footer": { + "countRows": false, + "fields": "", + "reducer": [ + "sum" + ], + "show": false + }, + "showHeader": true, + "sortBy": [ + { + "desc": false, + "displayName": "Instance" + } + ] + }, + "pluginVersion": "13.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "ds-prometheus" + }, + "editorMode": "code", + "expr": "max by (ins,instance) (openbao_up{job=\"openbao\", cls=\"$cls\"})", + "legendFormat": "", + "refId": "A", + "exemplar": false, + "instant": true, + "range": false, + "format": "table" + }, + { + "datasource": { + "type": "prometheus", + "uid": "ds-prometheus" + }, + "editorMode": "code", + "expr": "max by (ins) (vault_core_unsealed{job=\"openbao\", cls=\"$cls\"})", + "legendFormat": "", + "refId": "B", + "exemplar": false, + "instant": true, + "range": false, + "format": "table" + }, + { + "datasource": { + "type": "prometheus", + "uid": "ds-prometheus" + }, + "editorMode": "code", + "expr": "max by (ins) (vault_core_active{job=\"openbao\", cls=\"$cls\"})", + "legendFormat": "", + "refId": "C", + "exemplar": false, + "instant": true, + "range": false, + "format": "table" + }, + { + "datasource": { + "type": "prometheus", + "uid": "ds-prometheus" + }, + "editorMode": "code", + "expr": "max by (ins) (vault_autopilot_node_healthy{job=\"openbao\", cls=\"$cls\"})", + "legendFormat": "", + "refId": "D", + "exemplar": false, + "instant": true, + "range": false, + "format": "table" + }, + { + "datasource": { + "type": "prometheus", + "uid": "ds-prometheus" + }, + "editorMode": "code", + "expr": "max by (ins) (vault_raft_storage_stats_applied_index{job=\"openbao\", cls=\"$cls\"})", + "legendFormat": "", + "refId": "E", + "exemplar": false, + "instant": true, + "range": false, + "format": "table" + }, + { + "datasource": { + "type": "prometheus", + "uid": "ds-prometheus" + }, + "editorMode": "code", + "expr": "max by (ins) (vault_raft_storage_stats_commit_index{job=\"openbao\", cls=\"$cls\"})", + "legendFormat": "", + "refId": "F", + "exemplar": false, + "instant": true, + "range": false, + "format": "table" + }, + { + "datasource": { + "type": "prometheus", + "uid": "ds-prometheus" + }, + "editorMode": "code", + "expr": "max by (ins) (vault_raft_storage_stats_fsm_pending{job=\"openbao\", cls=\"$cls\"})", + "legendFormat": "", + "refId": "G", + "exemplar": false, + "instant": true, + "range": false, + "format": "table" + }, + { + "datasource": { + "type": "prometheus", + "uid": "ds-prometheus" + }, + "editorMode": "code", + "expr": "max by (ins) (vault_expire_num_leases{job=\"openbao\", cls=\"$cls\"})", + "legendFormat": "", + "refId": "H", + "exemplar": false, + "instant": true, + "range": false, + "format": "table" + }, + { + "datasource": { + "type": "prometheus", + "uid": "ds-prometheus" + }, + "editorMode": "code", + "expr": "max by (ins) (vault_token_count{job=\"openbao\", cls=\"$cls\"})", + "legendFormat": "", + "refId": "I", + "exemplar": false, + "instant": true, + "range": false, + "format": "table" + }, + { + "datasource": { + "type": "prometheus", + "uid": "ds-prometheus" + }, + "editorMode": "code", + "expr": "max by (ins) (vault_runtime_num_goroutines{job=\"openbao\", cls=\"$cls\"})", + "legendFormat": "", + "refId": "J", + "exemplar": false, + "instant": true, + "range": false, + "format": "table" + } + ], + "title": "Openbao Member List", + "transformations": [ + { + "id": "seriesToColumns", + "options": { + "byField": "ins" + } + }, + { + "id": "organize", + "options": { + "excludeByName": { + "Time 1": true, + "Time 2": true, + "Time 3": true, + "Time 4": true, + "Time 5": true, + "Time 6": true, + "Time 7": true, + "Time 8": true, + "Time 9": true, + "Time 10": true + }, + "includeByName": {}, + "indexByName": { + "ins": 0, + "instance": 1, + "Value #A": 2, + "Value #B": 3, + "Value #C": 4, + "Value #D": 5, + "Value #E": 6, + "Value #F": 7, + "Value #G": 8, + "Value #H": 9, + "Value #I": 10, + "Value #J": 11 + }, + "renameByName": { + "ins": "Instance", + "instance": "Endpoint", + "Value #A": "Alive", + "Value #B": "Seal", + "Value #C": "Role", + "Value #D": "Healthy", + "Value #E": "Applied Index", + "Value #F": "Commit Index", + "Value #G": "FSM Pending", + "Value #H": "Leases", + "Value #I": "Tokens", + "Value #J": "Goroutines" + } + } + } + ], + "type": "table" + }, + { + "datasource": { + "type": "prometheus", + "uid": "ds-prometheus" + }, + "description": "Scrape result per member. A sealed node answers 503 and shows up as Down here", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "custom": { + "axisPlacement": "auto", + "fillOpacity": 70, + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineWidth": 0, + "spanNulls": false + }, + "decimals": 0, + "min": -1, + "mappings": [ + { + "options": { + "0": { + "color": "#cc4637d9", + "index": 0, + "text": "Down" + }, + "1": { + "color": "#346f36cc", + "index": 1, + "text": "Alive" + } + }, + "type": "value" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "#cc4637d9", + "value": 0 + }, + { + "color": "#346f36cc", + "value": 1 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 12, + "x": 0, + "y": 12 + }, + "id": 108, + "options": { + "alignValue": "left", + "annotations": { + "clustering": -1, + "multiLane": false + }, + "legend": { + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "mergeValues": true, + "rowHeight": 0.96, + "showValue": "never", + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "13.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "ds-prometheus" + }, + "editorMode": "code", + "expr": "openbao_up{job=\"openbao\", cls=\"$cls\"}", + "legendFormat": "{{ ins }}", + "refId": "A", + "range": true + } + ], + "title": "Openbao Aliveness", + "type": "state-timeline" + }, + { + "datasource": { + "type": "prometheus", + "uid": "ds-prometheus" + }, + "description": "Unsealed state (1 = unsealed) and active role (1 = active/leader)", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "custom": { + "axisPlacement": "auto", + "fillOpacity": 70, + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineWidth": 0, + "spanNulls": false + }, + "decimals": 0, + "min": -1, + "mappings": [ + { + "options": { + "0": { + "color": "#cc4637d9", + "index": 0, + "text": "🔒 Sealed" + }, + "1": { + "color": "#346f36cc", + "index": 1, + "text": "🔓 Unsealed" + } + }, + "type": "value" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "#cc4637d9", + "value": 0 + }, + { + "color": "#346f36cc", + "value": 1 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 12, + "x": 12, + "y": 12 + }, + "id": 109, + "options": { + "alignValue": "left", + "annotations": { + "clustering": -1, + "multiLane": false + }, + "legend": { + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "mergeValues": true, + "rowHeight": 0.96, + "showValue": "never", + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "13.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "ds-prometheus" + }, + "editorMode": "code", + "expr": "vault_core_unsealed{job=\"openbao\", cls=\"$cls\"}", + "legendFormat": "{{ ins }} seal", + "refId": "A", + "range": true + }, + { + "datasource": { + "type": "prometheus", + "uid": "ds-prometheus" + }, + "editorMode": "code", + "expr": "vault_core_active{job=\"openbao\", cls=\"$cls\"}", + "legendFormat": "{{ ins }} role", + "refId": "B", + "range": true + } + ], + "title": "Seal Status & Role", + "type": "state-timeline" + }, + { + "datasource": { + "type": "prometheus", + "uid": "ds-prometheus" + }, + "description": "Raft elections won and candidacies started in the last 5m", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "showValues": false, + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "#346f36cc", + "value": 0 + } + ] + }, + "unit": "short", + "decimals": 0, + "min": 0 + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 12, + "x": 0, + "y": 18 + }, + "id": 110, + "options": { + "annotations": { + "clustering": -1, + "multiLane": false + }, + "legend": { + "calcs": [ + "lastNotNull", + "max" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "13.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "ds-prometheus" + }, + "editorMode": "code", + "expr": "increase(vault_raft_state_leader{job=\"openbao\", cls=\"$cls\"}[5m])", + "legendFormat": "{{ ins }} leader", + "refId": "A", + "range": true + }, + { + "datasource": { + "type": "prometheus", + "uid": "ds-prometheus" + }, + "editorMode": "code", + "expr": "increase(vault_raft_state_candidate{job=\"openbao\", cls=\"$cls\"}[5m])", + "legendFormat": "{{ ins }} candidate", + "refId": "B", + "range": true + } + ], + "title": "Leadership Changes", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "ds-prometheus" + }, + "description": "Healthy nodes in excess of quorum, 0 means the next failure breaks the cluster", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "showValues": false, + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "#346f36cc", + "value": 0 + } + ] + }, + "unit": "short", + "decimals": 0, + "min": 0 + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 12, + "x": 12, + "y": 18 + }, + "id": 111, + "options": { + "annotations": { + "clustering": -1, + "multiLane": false + }, + "legend": { + "calcs": [ + "lastNotNull", + "max" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "13.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "ds-prometheus" + }, + "editorMode": "code", + "expr": "vault_autopilot_failure_tolerance{job=\"openbao\", cls=\"$cls\"}", + "legendFormat": "{{ ins }}", + "refId": "A", + "range": true + } + ], + "title": "Failure Tolerance", + "type": "timeseries" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 24 + }, + "id": 112, + "title": "Requests", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "ds-prometheus" + }, + "description": "Non-login API requests handled per second", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "showValues": false, + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "#346f36cc", + "value": 0 + } + ] + }, + "unit": "reqps", + "min": 0 + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 12, + "x": 0, + "y": 25 + }, + "id": 113, + "options": { + "annotations": { + "clustering": -1, + "multiLane": false + }, + "legend": { + "calcs": [ + "lastNotNull", + "max" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "13.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "ds-prometheus" + }, + "editorMode": "code", + "expr": "openbao:ins:req_rps{job=\"openbao\", cls=\"$cls\"}", + "legendFormat": "{{ ins }}", + "refId": "A", + "range": true + } + ], + "title": "Request Rate", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "ds-prometheus" + }, + "description": "Login requests handled per second", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "showValues": false, + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "#346f36cc", + "value": 0 + } + ] + }, + "unit": "reqps", + "min": 0 + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 12, + "x": 12, + "y": 25 + }, + "id": 114, + "options": { + "annotations": { + "clustering": -1, + "multiLane": false + }, + "legend": { + "calcs": [ + "lastNotNull", + "max" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "13.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "ds-prometheus" + }, + "editorMode": "code", + "expr": "openbao:ins:login_rps{job=\"openbao\", cls=\"$cls\"}", + "legendFormat": "{{ ins }}", + "refId": "A", + "range": true + } + ], + "title": "Login Request Rate", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "ds-prometheus" + }, + "description": "Time to complete a non-login request, quantiles reported by openbao itself", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "showValues": false, + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "#346f36cc", + "value": 0 + } + ] + }, + "unit": "ms", + "min": 0 + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 12, + "x": 0, + "y": 31 + }, + "id": 115, + "options": { + "annotations": { + "clustering": -1, + "multiLane": false + }, + "legend": { + "calcs": [ + "lastNotNull", + "max" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "13.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "ds-prometheus" + }, + "editorMode": "code", + "expr": "vault_core_handle_request{job=\"openbao\", cls=\"$cls\", quantile=\"0.5\"}", + "legendFormat": "{{ ins }} p50", + "refId": "A", + "range": true + }, + { + "datasource": { + "type": "prometheus", + "uid": "ds-prometheus" + }, + "editorMode": "code", + "expr": "vault_core_handle_request{job=\"openbao\", cls=\"$cls\", quantile=\"0.9\"}", + "legendFormat": "{{ ins }} p90", + "refId": "B", + "range": true + }, + { + "datasource": { + "type": "prometheus", + "uid": "ds-prometheus" + }, + "editorMode": "code", + "expr": "vault_core_handle_request{job=\"openbao\", cls=\"$cls\", quantile=\"0.99\"}", + "legendFormat": "{{ ins }} p99", + "refId": "C", + "range": true + } + ], + "title": "Request Latency", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "ds-prometheus" + }, + "description": "Average request time over the last 5m", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "showValues": false, + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "#346f36cc", + "value": 0 + } + ] + }, + "unit": "ms", + "min": 0 + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 12, + "x": 12, + "y": 31 + }, + "id": 116, + "options": { + "annotations": { + "clustering": -1, + "multiLane": false + }, + "legend": { + "calcs": [ + "lastNotNull", + "max" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "13.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "ds-prometheus" + }, + "editorMode": "code", + "expr": "openbao:ins:req_rt_avg_5m{job=\"openbao\", cls=\"$cls\"} * 1000", + "legendFormat": "{{ ins }}", + "refId": "A", + "range": true + } + ], + "title": "Mean Request Latency (5m)", + "type": "timeseries" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 37 + }, + "id": 117, + "title": "Raft Storage", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "ds-prometheus" + }, + "description": "Highest raft log index applied to the FSM and committed to disk", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "showValues": false, + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "#346f36cc", + "value": 0 + } + ] + }, + "unit": "short", + "decimals": 0 + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 12, + "x": 0, + "y": 38 + }, + "id": 118, + "options": { + "annotations": { + "clustering": -1, + "multiLane": false + }, + "legend": { + "calcs": [ + "lastNotNull", + "max" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "13.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "ds-prometheus" + }, + "editorMode": "code", + "expr": "vault_raft_storage_stats_applied_index{job=\"openbao\", cls=\"$cls\"}", + "legendFormat": "{{ ins }} applied", + "refId": "A", + "range": true + }, + { + "datasource": { + "type": "prometheus", + "uid": "ds-prometheus" + }, + "editorMode": "code", + "expr": "vault_raft_storage_stats_commit_index{job=\"openbao\", cls=\"$cls\"}", + "legendFormat": "{{ ins }} commit", + "refId": "B", + "range": true + } + ], + "title": "Raft Index", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "ds-prometheus" + }, + "description": "Raft logs queued waiting for the finite state machine to apply them", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "showValues": false, + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "#346f36cc", + "value": 0 + } + ] + }, + "unit": "short", + "decimals": 0, + "min": 0 + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 12, + "x": 12, + "y": 38 + }, + "id": 119, + "options": { + "annotations": { + "clustering": -1, + "multiLane": false + }, + "legend": { + "calcs": [ + "lastNotNull", + "max" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "13.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "ds-prometheus" + }, + "editorMode": "code", + "expr": "vault_raft_storage_stats_fsm_pending{job=\"openbao\", cls=\"$cls\"}", + "legendFormat": "{{ ins }}", + "refId": "A", + "range": true + } + ], + "title": "FSM Pending", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "ds-prometheus" + }, + "description": "How far each follower trails the leader, in log entries", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "showValues": false, + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "#346f36cc", + "value": 0 + } + ] + }, + "unit": "short", + "decimals": 0, + "min": 0 + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 12, + "x": 0, + "y": 44 + }, + "id": 120, + "options": { + "annotations": { + "clustering": -1, + "multiLane": false + }, + "legend": { + "calcs": [ + "lastNotNull", + "max" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "13.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "ds-prometheus" + }, + "editorMode": "code", + "expr": "openbao:ins:raft_lag{job=\"openbao\", cls=\"$cls\"}", + "legendFormat": "{{ ins }}", + "refId": "A", + "range": true + } + ], + "title": "Follower Applied Index Delta", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "ds-prometheus" + }, + "description": "Time since the follower last received a heartbeat from the leader", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "showValues": false, + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "#346f36cc", + "value": 0 + } + ] + }, + "unit": "ms", + "min": 0 + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 12, + "x": 12, + "y": 44 + }, + "id": 121, + "options": { + "annotations": { + "clustering": -1, + "multiLane": false + }, + "legend": { + "calcs": [ + "lastNotNull", + "max" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "13.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "ds-prometheus" + }, + "editorMode": "code", + "expr": "vault_raft_storage_follower_last_heartbeat_ms{job=\"openbao\", cls=\"$cls\"}", + "legendFormat": "{{ ins }}", + "refId": "A", + "range": true + } + ], + "title": "Follower Last Heartbeat", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "ds-prometheus" + }, + "description": "Time for the leader to commit a new entry to the raft log", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "showValues": false, + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "#346f36cc", + "value": 0 + } + ] + }, + "unit": "ms", + "min": 0 + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 12, + "x": 0, + "y": 50 + }, + "id": 122, + "options": { + "annotations": { + "clustering": -1, + "multiLane": false + }, + "legend": { + "calcs": [ + "lastNotNull", + "max" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "13.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "ds-prometheus" + }, + "editorMode": "code", + "expr": "vault_raft_commitTime{job=\"openbao\", cls=\"$cls\", quantile=\"0.5\"}", + "legendFormat": "{{ ins }} p50", + "refId": "A", + "range": true + }, + { + "datasource": { + "type": "prometheus", + "uid": "ds-prometheus" + }, + "editorMode": "code", + "expr": "vault_raft_commitTime{job=\"openbao\", cls=\"$cls\", quantile=\"0.99\"}", + "legendFormat": "{{ ins }} p99", + "refId": "B", + "range": true + } + ], + "title": "Raft Commit Time", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "ds-prometheus" + }, + "description": "Time since the leader could last reach the followers when checking its lease", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "showValues": false, + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "#346f36cc", + "value": 0 + } + ] + }, + "unit": "ms", + "min": 0 + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 12, + "x": 12, + "y": 50 + }, + "id": 123, + "options": { + "annotations": { + "clustering": -1, + "multiLane": false + }, + "legend": { + "calcs": [ + "lastNotNull", + "max" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "13.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "ds-prometheus" + }, + "editorMode": "code", + "expr": "vault_raft_leader_lastContact{job=\"openbao\", cls=\"$cls\", quantile=\"0.5\"}", + "legendFormat": "{{ ins }} p50", + "refId": "A", + "range": true + }, + { + "datasource": { + "type": "prometheus", + "uid": "ds-prometheus" + }, + "editorMode": "code", + "expr": "vault_raft_leader_lastContact{job=\"openbao\", cls=\"$cls\", quantile=\"0.99\"}", + "legendFormat": "{{ ins }} p99", + "refId": "B", + "range": true + } + ], + "title": "Leader Last Contact", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "ds-prometheus" + }, + "description": "Raft transactions applied per second", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "showValues": false, + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "#346f36cc", + "value": 0 + } + ] + }, + "unit": "ops", + "min": 0 + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 12, + "x": 0, + "y": 56 + }, + "id": 124, + "options": { + "annotations": { + "clustering": -1, + "multiLane": false + }, + "legend": { + "calcs": [ + "lastNotNull", + "max" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "13.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "ds-prometheus" + }, + "editorMode": "code", + "expr": "rate(vault_raft_apply{job=\"openbao\", cls=\"$cls\"}[1m])", + "legendFormat": "{{ ins }}", + "refId": "A", + "range": true + } + ], + "title": "Raft Apply Rate", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "ds-prometheus" + }, + "description": "Space allocated to the underlying bolt database and its freelist", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "showValues": false, + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "#346f36cc", + "value": 0 + } + ] + }, + "unit": "bytes", + "min": 0 + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 12, + "x": 12, + "y": 56 + }, + "id": 125, + "options": { + "annotations": { + "clustering": -1, + "multiLane": false + }, + "legend": { + "calcs": [ + "lastNotNull", + "max" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "13.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "ds-prometheus" + }, + "editorMode": "code", + "expr": "vault_raft_storage_bolt_page_bytes_allocated{job=\"openbao\", cls=\"$cls\"}", + "legendFormat": "{{ ins }} allocated", + "refId": "A", + "range": true + }, + { + "datasource": { + "type": "prometheus", + "uid": "ds-prometheus" + }, + "editorMode": "code", + "expr": "vault_raft_storage_bolt_freelist_used_bytes{job=\"openbao\", cls=\"$cls\"}", + "legendFormat": "{{ ins }} freelist", + "refId": "B", + "range": true + } + ], + "title": "BoltDB Size", + "type": "timeseries" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 62 + }, + "id": 126, + "title": "Secrets & Leases", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "ds-prometheus" + }, + "description": "Leases eligible for eventual expiry, a permanently growing line means leases are never revoked", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "showValues": false, + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "#346f36cc", + "value": 0 + } + ] + }, + "unit": "short", + "decimals": 0, + "min": 0 + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 12, + "x": 0, + "y": 63 + }, + "id": 127, + "options": { + "annotations": { + "clustering": -1, + "multiLane": false + }, + "legend": { + "calcs": [ + "lastNotNull", + "max" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "13.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "ds-prometheus" + }, + "editorMode": "code", + "expr": "vault_expire_num_leases{job=\"openbao\", cls=\"$cls\"}", + "legendFormat": "{{ ins }}", + "refId": "A", + "range": true + } + ], + "title": "Leases", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "ds-prometheus" + }, + "description": "Un-expired and un-revoked tokens in the token store", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "showValues": false, + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "#346f36cc", + "value": 0 + } + ] + }, + "unit": "short", + "decimals": 0, + "min": 0 + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 12, + "x": 12, + "y": 63 + }, + "id": 128, + "options": { + "annotations": { + "clustering": -1, + "multiLane": false + }, + "legend": { + "calcs": [ + "lastNotNull", + "max" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "13.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "ds-prometheus" + }, + "editorMode": "code", + "expr": "vault_token_count{job=\"openbao\", cls=\"$cls\"}", + "legendFormat": "{{ ins }}", + "refId": "A", + "range": true + } + ], + "title": "Tokens", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "ds-prometheus" + }, + "description": "Token creation, lease revoke and lease renew rate", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "showValues": false, + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "#346f36cc", + "value": 0 + } + ] + }, + "unit": "ops", + "min": 0 + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 12, + "x": 0, + "y": 69 + }, + "id": 129, + "options": { + "annotations": { + "clustering": -1, + "multiLane": false + }, + "legend": { + "calcs": [ + "lastNotNull", + "max" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "13.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "ds-prometheus" + }, + "editorMode": "code", + "expr": "rate(vault_token_creation{job=\"openbao\", cls=\"$cls\"}[1m])", + "legendFormat": "{{ ins }} create", + "refId": "A", + "range": true + }, + { + "datasource": { + "type": "prometheus", + "uid": "ds-prometheus" + }, + "editorMode": "code", + "expr": "rate(vault_expire_revoke_count{job=\"openbao\", cls=\"$cls\"}[1m])", + "legendFormat": "{{ ins }} revoke", + "refId": "B", + "range": true + }, + { + "datasource": { + "type": "prometheus", + "uid": "ds-prometheus" + }, + "editorMode": "code", + "expr": "rate(vault_expire_renew_count{job=\"openbao\", cls=\"$cls\"}[1m])", + "legendFormat": "{{ ins }} renew", + "refId": "C", + "range": true + } + ], + "title": "Token & Lease Ops", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "ds-prometheus" + }, + "description": "Entries per kv secrets engine and total identity entities", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "showValues": false, + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "#346f36cc", + "value": 0 + } + ] + }, + "unit": "short", + "decimals": 0, + "min": 0 + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 12, + "x": 12, + "y": 69 + }, + "id": 130, + "options": { + "annotations": { + "clustering": -1, + "multiLane": false + }, + "legend": { + "calcs": [ + "lastNotNull", + "max" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "13.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "ds-prometheus" + }, + "editorMode": "code", + "expr": "vault_secret_kv_count{job=\"openbao\", cls=\"$cls\"}", + "legendFormat": "{{ ins }} kv {{ mount_point }}", + "refId": "A", + "range": true + }, + { + "datasource": { + "type": "prometheus", + "uid": "ds-prometheus" + }, + "editorMode": "code", + "expr": "vault_identity_num_entities{job=\"openbao\", cls=\"$cls\"}", + "legendFormat": "{{ ins }} entities", + "refId": "B", + "range": true + } + ], + "title": "KV Secrets & Identities", + "type": "timeseries" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 75 + }, + "id": 131, + "title": "Audit", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "ds-prometheus" + }, + "description": "Audit log requests and responses written per second", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "showValues": false, + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "#346f36cc", + "value": 0 + } + ] + }, + "unit": "ops", + "min": 0 + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 12, + "x": 0, + "y": 76 + }, + "id": 132, + "options": { + "annotations": { + "clustering": -1, + "multiLane": false + }, + "legend": { + "calcs": [ + "lastNotNull", + "max" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "13.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "ds-prometheus" + }, + "editorMode": "code", + "expr": "rate(vault_audit_log_request_count{job=\"openbao\", cls=\"$cls\"}[1m])", + "legendFormat": "{{ ins }} request", + "refId": "A", + "range": true + }, + { + "datasource": { + "type": "prometheus", + "uid": "ds-prometheus" + }, + "editorMode": "code", + "expr": "rate(vault_audit_log_response_count{job=\"openbao\", cls=\"$cls\"}[1m])", + "legendFormat": "{{ ins }} response", + "refId": "B", + "range": true + } + ], + "title": "Audit Log Rate", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "ds-prometheus" + }, + "description": "Audit write failures. Openbao blocks requests while the audit device is failing", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "showValues": false, + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "#346f36cc", + "value": 0 + } + ] + }, + "unit": "ops", + "min": 0 + }, + "overrides": [ + { + "matcher": { + "id": "byRegexp", + "options": ".*" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#cc4637d9", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 6, + "w": 12, + "x": 12, + "y": 76 + }, + "id": 133, + "options": { + "annotations": { + "clustering": -1, + "multiLane": false + }, + "legend": { + "calcs": [ + "lastNotNull", + "max" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "13.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "ds-prometheus" + }, + "editorMode": "code", + "expr": "rate(vault_audit_log_request_failure{job=\"openbao\", cls=\"$cls\"}[1m])", + "legendFormat": "{{ ins }} request", + "refId": "A", + "range": true + }, + { + "datasource": { + "type": "prometheus", + "uid": "ds-prometheus" + }, + "editorMode": "code", + "expr": "rate(vault_audit_log_response_failure{job=\"openbao\", cls=\"$cls\"}[1m])", + "legendFormat": "{{ ins }} response", + "refId": "B", + "range": true + } + ], + "title": "Audit Failure Rate", + "type": "timeseries" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 82 + }, + "id": 134, + "title": "Runtime", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "ds-prometheus" + }, + "description": "Go routines running in the openbao process", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "showValues": false, + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "#346f36cc", + "value": 0 + } + ] + }, + "unit": "short", + "decimals": 0, + "min": 0 + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 12, + "x": 0, + "y": 83 + }, + "id": 135, + "options": { + "annotations": { + "clustering": -1, + "multiLane": false + }, + "legend": { + "calcs": [ + "lastNotNull", + "max" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "13.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "ds-prometheus" + }, + "editorMode": "code", + "expr": "vault_runtime_num_goroutines{job=\"openbao\", cls=\"$cls\"}", + "legendFormat": "{{ ins }}", + "refId": "A", + "range": true + } + ], + "title": "Goroutines", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "ds-prometheus" + }, + "description": "Heap currently allocated and total bytes obtained from the OS", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "showValues": false, + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "#346f36cc", + "value": 0 + } + ] + }, + "unit": "bytes", + "min": 0 + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 12, + "x": 12, + "y": 83 + }, + "id": 136, + "options": { + "annotations": { + "clustering": -1, + "multiLane": false + }, + "legend": { + "calcs": [ + "lastNotNull", + "max" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "13.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "ds-prometheus" + }, + "editorMode": "code", + "expr": "vault_runtime_alloc_bytes{job=\"openbao\", cls=\"$cls\"}", + "legendFormat": "{{ ins }} alloc", + "refId": "A", + "range": true + }, + { + "datasource": { + "type": "prometheus", + "uid": "ds-prometheus" + }, + "editorMode": "code", + "expr": "vault_runtime_sys_bytes{job=\"openbao\", cls=\"$cls\"}", + "legendFormat": "{{ ins }} sys", + "refId": "B", + "range": true + } + ], + "title": "Memory", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "ds-prometheus" + }, + "description": "Garbage collector pause time per second and GC runs per second", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "showValues": false, + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "#346f36cc", + "value": 0 + } + ] + }, + "unit": "short", + "min": 0 + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 12, + "x": 0, + "y": 89 + }, + "id": 137, + "options": { + "annotations": { + "clustering": -1, + "multiLane": false + }, + "legend": { + "calcs": [ + "lastNotNull", + "max" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "13.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "ds-prometheus" + }, + "editorMode": "code", + "expr": "rate(vault_runtime_total_gc_pause_ns{job=\"openbao\", cls=\"$cls\"}[1m]) / 1000000000", + "legendFormat": "{{ ins }} pause ratio", + "refId": "A", + "range": true + }, + { + "datasource": { + "type": "prometheus", + "uid": "ds-prometheus" + }, + "editorMode": "code", + "expr": "rate(vault_runtime_total_gc_runs{job=\"openbao\", cls=\"$cls\"}[1m])", + "legendFormat": "{{ ins }} runs", + "refId": "B", + "range": true + } + ], + "title": "GC Pause", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "ds-prometheus" + }, + "description": "Objects on the heap", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "showValues": false, + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "#346f36cc", + "value": 0 + } + ] + }, + "unit": "short", + "decimals": 0, + "min": 0 + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 12, + "x": 12, + "y": 89 + }, + "id": 138, + "options": { + "annotations": { + "clustering": -1, + "multiLane": false + }, + "legend": { + "calcs": [ + "lastNotNull", + "max" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "13.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "ds-prometheus" + }, + "editorMode": "code", + "expr": "vault_runtime_heap_objects{job=\"openbao\", cls=\"$cls\"}", + "legendFormat": "{{ ins }}", + "refId": "A", + "range": true + } + ], + "title": "Heap Objects", + "type": "timeseries" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 95 + }, + "id": 139, + "title": "HA Forwarding", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "ds-prometheus" + }, + "description": "Requests forwarded from a standby to the active node, and echo keepalives", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "showValues": false, + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "#346f36cc", + "value": 0 + } + ] + }, + "unit": "ops", + "min": 0 + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 12, + "x": 0, + "y": 96 + }, + "id": 140, + "options": { + "annotations": { + "clustering": -1, + "multiLane": false + }, + "legend": { + "calcs": [ + "lastNotNull", + "max" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "13.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "ds-prometheus" + }, + "editorMode": "code", + "expr": "rate(vault_ha_rpc_client_forward_count{job=\"openbao\", cls=\"$cls\"}[1m])", + "legendFormat": "{{ ins }} forward", + "refId": "A", + "range": true + }, + { + "datasource": { + "type": "prometheus", + "uid": "ds-prometheus" + }, + "editorMode": "code", + "expr": "rate(vault_ha_rpc_client_echo_count{job=\"openbao\", cls=\"$cls\"}[1m])", + "legendFormat": "{{ ins }} echo", + "refId": "B", + "range": true + } + ], + "title": "Standby Forward Rate", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "ds-prometheus" + }, + "description": "Failures while forwarding or echoing to the active node", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "showValues": false, + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "#346f36cc", + "value": 0 + } + ] + }, + "unit": "ops", + "min": 0 + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 12, + "x": 12, + "y": 96 + }, + "id": 141, + "options": { + "annotations": { + "clustering": -1, + "multiLane": false + }, + "legend": { + "calcs": [ + "lastNotNull", + "max" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "13.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "ds-prometheus" + }, + "editorMode": "code", + "expr": "rate(vault_ha_rpc_client_forward_errors{job=\"openbao\", cls=\"$cls\"}[1m])", + "legendFormat": "{{ ins }} forward", + "refId": "A", + "range": true + }, + { + "datasource": { + "type": "prometheus", + "uid": "ds-prometheus" + }, + "editorMode": "code", + "expr": "rate(vault_ha_rpc_client_echo_errors{job=\"openbao\", cls=\"$cls\"}[1m])", + "legendFormat": "{{ ins }} echo", + "refId": "B", + "range": true + } + ], + "title": "Standby RPC Errors", + "type": "timeseries" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 102 + }, + "id": 142, + "title": "Logs", + "type": "row" + }, + { + "datasource": { + "type": "victoriametrics-logs-datasource", + "uid": "ds-vlogs" + }, + "description": "Journald log entries produced by the openbao unit on cluster $cls", + "fieldConfig": { + "defaults": { + "color": { + "fixedColor": "#346f36cc", + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "bars", + "fillOpacity": 100, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "showValues": false, + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "#346f36cc", + "value": 0 + } + ] + }, + "unit": "short", + "decimals": 0 + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 24, + "x": 0, + "y": 103 + }, + "id": 143, + "options": { + "annotations": { + "clustering": -1, + "multiLane": false + }, + "legend": { + "calcs": [ + "sum" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "13.0.0", + "targets": [ + { + "datasource": { + "type": "victoriametrics-logs-datasource", + "uid": "ds-vlogs" + }, + "editorMode": "code", + "expr": "job:syslog ip:in(${members:doublequote}) unit:openbao | stats by (ins) count() as count", + "legendFormat": "{{ ins }}", + "refId": "A", + "queryType": "statsRange" + } + ], + "title": "Openbao Log Rate", + "type": "timeseries" + }, + { + "datasource": { + "type": "victoriametrics-logs-datasource", + "uid": "ds-vlogs" + }, + "description": "Openbao logs of cluster $cls", + "gridPos": { + "h": 18, + "w": 24, + "x": 0, + "y": 109 + }, + "id": 144, + "options": { + "dedupStrategy": "none", + "enableInfiniteScrolling": false, + "enableLogDetails": true, + "fontSize": "small", + "prettifyLogMessage": false, + "showCommonLabels": false, + "showControls": true, + "showLabels": false, + "showTime": true, + "sortOrder": "Descending", + "syntaxHighlighting": true, + "wrapLogMessage": false + }, + "targets": [ + { + "datasource": { + "type": "victoriametrics-logs-datasource", + "uid": "ds-vlogs" + }, + "editorMode": "code", + "expr": "job:syslog ip:in(${members:doublequote}) unit:openbao", + "instant": false, + "legendFormat": "{{ ins }}", + "queryType": "instant", + "range": true, + "refId": "A" + } + ], + "title": "", + "type": "logs" + } + ], + "preload": false, + "refresh": "", + "schemaVersion": 42, + "tags": [ + "Pigsty", + "OPENBAO", + "Overview", + "HOME" + ], + "templating": { + "list": [ + { + "allowCustomValue": true, + "current": {}, + "datasource": { + "type": "prometheus", + "uid": "ds-prometheus" + }, + "definition": "label_values(vault_core_unsealed, cls)", + "description": "openbao cluster name", + "hide": 0, + "includeAll": false, + "label": "Cluster", + "multi": false, + "name": "cls", + "options": [], + "query": { + "query": "label_values(vault_core_unsealed, cls)", + "refId": "StandardVariableQuery" + }, + "refresh": 1, + "regex": "", + "regexApplyTo": "value", + "skipUrlSync": false, + "sort": 0, + "type": "query" + }, + { + "allowCustomValue": true, + "current": {}, + "datasource": { + "type": "prometheus" + }, + "definition": "label_values(vault_core_unsealed{cls=\"$cls\"},ip)", + "description": "IP address of this cluster members", + "hide": 2, + "includeAll": true, + "label": "Members", + "multi": true, + "name": "members", + "options": [], + "query": { + "qryType": 1, + "query": "label_values(vault_core_unsealed{cls=\"$cls\"},ip)", + "refId": "PrometheusVariableQueryEditor-VariableQuery" + }, + "refresh": 1, + "regex": "", + "regexApplyTo": "value", + "skipUrlSync": false, + "sort": 0, + "type": "query" + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "2s", + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ] + }, + "timezone": "browser", + "title": "OpenBao Overview", + "uid": "openbao-overview", + "version": 1 +} diff --git a/files/grafana/pigsty.json b/files/grafana/pigsty.json index 162f24b89..2bdb79f29 100644 --- a/files/grafana/pigsty.json +++ b/files/grafana/pigsty.json @@ -56,6 +56,7 @@ {"matcher":{"id":"byName","options":"INFRA"},"properties":[{"id":"links","value":[{"title":"Infra Overview","url":"/d/infra-overview?${__url_time_range}"}]},{"id":"color","value":{"fixedColor":"#346f36cc","mode":"fixed"}}]}, {"matcher":{"id":"byName","options":"REDIS"},"properties":[{"id":"links","value":[{"title":"Redis Overview","url":"/d/redis-overview?${__url_time_range}"}]},{"id":"color","value":{"fixedColor":"#cc4637d9","mode":"fixed"}}]}, {"matcher":{"id":"byName","options":"ETCD"},"properties":[{"id":"color","value":{"fixedColor":"#5b9cd5","mode":"fixed"}},{"id":"links","value":[{"title":"ETCD Overview","url":"/d/etcd-overview?${__url_time_range}"}]}]}, + {"matcher":{"id":"byName","options":"BAO"},"properties":[{"id":"color","value":{"fixedColor":"#8f6fae","mode":"fixed"}},{"id":"links","value":[{"title":"OpenBao Overview","url":"/d/openbao-overview?${__url_time_range}"}]}]}, {"matcher":{"id":"byName","options":"MINIO"},"properties":[{"id":"color","value":{"fixedColor":"#d0475799","mode":"fixed"}},{"id":"links","value":[{"title":"MinIO Overview","url":"/d/minio-overview?${__url_time_range}"}]}]}, {"matcher":{"id":"byName","options":"MYSQL"},"properties":[{"id":"color","value":{"fixedColor":"#00758fe0","mode":"fixed"}},{"id":"links","value":[{"title":"MySQL Overview","url":"/d/mysql-overview?${__url_time_range}"}]}]}, {"matcher":{"id":"byName","options":"PGRDS"},"properties":[{"id":"links","value":[{"title":"PGRDS Clusters","url":"/d/pgrds-cluster?${__url_time_range}"}]},{"id":"color","value":{"fixedColor":"#6986a3","mode":"fixed"}}]}, @@ -70,6 +71,7 @@ {"targetBlank":true,"title":"Module: INFRA","url":"https://pigsty.io/docs/infra"}, {"targetBlank":true,"title":"Module: NODE","url":"https://pigsty.io/docs/node"}, {"targetBlank":true,"title":"Module: ETCD","url":"https://pigsty.io/docs/etcd"}, + {"targetBlank":true,"title":"Module: OPENBAO","url":"https://pigsty.io/docs/openbao"}, {"targetBlank":true,"title":"Module: MinIO","url":"https://pigsty.io/docs/minio"}, {"targetBlank":true,"title":"Module: REDIS","url":"https://pigsty.io/docs/redis"} ], @@ -82,6 +84,7 @@ {"datasource":{"type":"prometheus","uid":"ds-prometheus"},"editorMode":"code","exemplar":false,"expr":"count(up{job=\"redis\"})","interval":"","legendFormat":"REDIS","range":true,"refId":"B"}, {"datasource":{"type":"prometheus","uid":"ds-prometheus"},"editorMode":"code","exemplar":false,"expr":"count(up{job=\"node\"})","interval":"","legendFormat":"NODE","range":true,"refId":"C"}, {"datasource":{"type":"prometheus","uid":"ds-prometheus"},"editorMode":"code","exemplar":false,"expr":"count(up{job=\"minio\"})","interval":"","legendFormat":"MINIO","range":true,"refId":"E"}, + {"datasource":{"type":"prometheus","uid":"ds-prometheus"},"editorMode":"code","exemplar":false,"expr":"count(up{job=\"openbao\"})","interval":"","legendFormat":"BAO","range":true,"refId":"O"}, {"datasource":{"type":"prometheus","uid":"ds-prometheus"},"editorMode":"code","exemplar":false,"expr":"count(up{job=\"etcd\"})","interval":"","legendFormat":"ETCD","range":true,"refId":"D"}, {"datasource":{"type":"prometheus","uid":"ds-prometheus"},"editorMode":"code","exemplar":false,"expr":"count(up{job=\"kafka\"})","interval":"","legendFormat":"KAFKA","range":true,"refId":"G"}, {"datasource":{"type":"prometheus","uid":"ds-prometheus"},"editorMode":"code","exemplar":false,"expr":"count(up{job=\"kube\"})","interval":"","legendFormat":"KUBE","range":true,"refId":"KUBE"}, @@ -101,6 +104,7 @@ {"matcher":{"id":"byFrameRefID","options":"C"},"properties":[{"id":"thresholds","value":{"mode":"absolute","steps":[{"color":"#3e3012","value":0},{"color":"#e6b03ee6","value":1}]}},{"id":"links","value":[{"title":"Node Instance : ${__field.labels.ip}","url":"/d/node-instance?var-id=${__field.labels.ip}\u0026${__url_time_range}"}]}]}, {"matcher":{"id":"byFrameRefID","options":"D"},"properties":[{"id":"thresholds","value":{"mode":"absolute","steps":[{"color":"#324e64","value":0},{"color":"#5b9cd5","value":1}]}},{"id":"links","value":[{"title":"ETCD Overview","url":"/d/etcd-overview?${__url_time_range}"}]}]}, {"matcher":{"id":"byFrameRefID","options":"E"},"properties":[{"id":"thresholds","value":{"mode":"absolute","steps":[{"color":"#561c24","value":0},{"color":"#d0475799","value":1}]}},{"id":"links","value":[{"title":"MinIO Instance: ${__field.labels.ins}","url":"/d/minio-instance?var-ins=${__field.labels.ins}\u0026${__url_time_range}"}]}]}, + {"matcher":{"id":"byFrameRefID","options":"O"},"properties":[{"id":"thresholds","value":{"mode":"absolute","steps":[{"color":"#2a2136","value":0},{"color":"#8f6fae","value":1}]}},{"id":"links","value":[{"title":"OpenBao Overview","url":"/d/openbao-overview?${__url_time_range}"}]}]}, {"matcher":{"id":"byFrameRefID","options":"M"},"properties":[{"id":"thresholds","value":{"mode":"absolute","steps":[{"color":"#00272fe0","value":0},{"color":"#00758fe0","value":1}]}},{"id":"links","value":[{"title":"MySQL Instance : ${__field.labels.ins}","url":"/d/mysql-instance?var-ins=${__field.labels.ins}\u0026${__url_time_range}"}]}]}, {"matcher":{"id":"byFrameRefID","options":"R"},"properties":[{"id":"links","value":[{"title":"PGRDS Instance: ${__field.labels.ins}","url":"/d/pgrds-instance?var-ins=${__field.labels.ins}\u0026${__url_time_range}"}]},{"id":"thresholds","value":{"mode":"absolute","steps":[{"color":"#1c232a","value":0},{"color":"#6986a3","value":1}]}}]}, {"matcher":{"id":"byFrameRefID","options":"K"},"properties":[{"id":"thresholds","value":{"mode":"absolute","steps":[{"color":"#193e36e0","value":0},{"color":"#4bb39ce0","value":1}]}},{"id":"links","value":[{"title":"Kafka Instance : ${__field.labels.ins}","url":"/d/kafka-instance?var-ins=${__field.labels.ins}\u0026${__url_time_range}"}]}]}, @@ -133,8 +137,9 @@ {"datasource":{"type":"prometheus","uid":"ds-prometheus"},"editorMode":"code","exemplar":false,"expr":"max by (ins) (up{cls=\"vtraces\"})","format":"time_series","instant":true,"interval":"","legendFormat":"{{ ins }}","refId":"VTRACES"}, {"datasource":{"type":"prometheus","uid":"ds-prometheus"},"editorMode":"code","exemplar":false,"expr":"max by (ins) (up{cls=\"vmalert\"})","format":"time_series","instant":true,"interval":"","legendFormat":"{{ ins }}","refId":"VMALERT"}, {"datasource":{"type":"prometheus","uid":"ds-prometheus"},"editorMode":"code","exemplar":false,"expr":"max by (ins) (up{cls=\"alertmanager\"})","format":"time_series","instant":true,"interval":"","legendFormat":"{{ ins }}","refId":"ALERTMGR"}, - {"datasource":{"type":"prometheus","uid":"ds-prometheus"},"editorMode":"code","exemplar":false,"expr":"min by (instance) (up{job!~\"(node|infra|etcd|pgsql|redis|minio|kafka|kafka_exporter|pgrds)\"})","hide":true,"instant":true,"interval":"","legendFormat":"{{ instance }}","refId":"Z"}, + {"datasource":{"type":"prometheus","uid":"ds-prometheus"},"editorMode":"code","exemplar":false,"expr":"min by (instance) (up{job!~\"(node|infra|etcd|openbao|pgsql|redis|minio|kafka|kafka_exporter|pgrds)\"})","hide":true,"instant":true,"interval":"","legendFormat":"{{ instance }}","refId":"Z"}, {"datasource":{"type":"prometheus","uid":"ds-prometheus"},"editorMode":"code","exemplar":false,"expr":"min by (ins,instance) (up{job=\"minio\"})","instant":true,"interval":"","legendFormat":"{{ ins }}","refId":"E"}, + {"datasource":{"type":"prometheus","uid":"ds-prometheus"},"editorMode":"code","exemplar":false,"expr":"min by (ins,instance) (up{job=\"openbao\"})","instant":true,"interval":"","legendFormat":"{{ ins }}","refId":"O"}, {"datasource":{"type":"prometheus","uid":"ds-prometheus"},"editorMode":"code","exemplar":false,"expr":"min by (ins,instance) (up{job=\"etcd\"})","instant":true,"interval":"","legendFormat":"{{ ins }}","refId":"D"} ], "title":"Instances", @@ -160,6 +165,16 @@ {"id":"mappings","value":[{"options":{"0":{"color":"text","index":1,"text":"Dead"},"1":{"color":"#5b9cd5","index":0,"text":"Healthy"}},"type":"value"}]} ] }, + { + "matcher":{"id":"byFrameRefID","options":"O"}, + "properties":[ + {"id":"color","value":{"fixedColor":"#8f6fae","mode":"thresholds"}}, + {"id":"links","value":[{"title":"OpenBao Overview","url":"/d/openbao-overview?${__url_time_range}"}]}, + {"id":"thresholds","value":{"mode":"absolute","steps":[{"color":"text","value":0},{"color":"#b783af","value":0.5},{"color":"red","value":0.66666},{"color":"#8f6fae","value":1}]}}, + {"id":"decimals","value":0}, + {"id":"mappings","value":[{"options":{"0":{"color":"text","index":1,"text":"Dead"},"1":{"color":"#8f6fae","index":0,"text":"Healthy"}},"type":"value"}]} + ] + }, {"matcher":{"id":"byFrameRefID","options":"E"},"properties":[{"id":"color","value":{"fixedColor":"#d0475799","mode":"fixed"}},{"id":"links","value":[{"title":"MinIO Overview","url":"/d/minio-overview?${__url_time_range}"}]}]}, {"matcher":{"id":"byFrameRefID","options":"R"},"properties":[{"id":"color","value":{"fixedColor":"#6986a3","mode":"fixed"}},{"id":"unit","value":"tps"},{"id":"decimals","value":0},{"id":"links","value":[{"title":"PGRDS Cluster : ${__field.labels.cls}","url":"/d/pgrds-cluster?var-cls=${__field.labels.cls}\u0026${__url_time_range}"}]}]}, { @@ -179,7 +194,8 @@ {"datasource":{"type":"prometheus","uid":"ds-prometheus"},"editorMode":"code","exemplar":false,"expr":"node:cls:cpu_usage{job=\"node\"}","interval":"","legendFormat":"{{ cls }}","range":true,"refId":"C"}, {"datasource":{"type":"prometheus","uid":"ds-prometheus"},"editorMode":"code","exemplar":false,"expr":"avg by (cls) (rate(minio_system_process_cpu_total_seconds{job=\"minio\"}[1m]))","interval":"","legendFormat":"{{ cls }}","range":true,"refId":"E"}, {"datasource":{"type":"prometheus","uid":"ds-prometheus"},"editorMode":"code","exemplar":false,"expr":"kafka:cls:load{job=\"kafka\"}","interval":"","legendFormat":"{{ cls }}","range":true,"refId":"K"}, - {"datasource":{"type":"prometheus","uid":"ds-prometheus"},"editorMode":"code","exemplar":false,"expr":"sum(etcd_up{job=\"etcd\"}) by (cls) / count(etcd_up{job=\"etcd\"}) by (cls)","interval":"","legendFormat":"{{ cls }}","range":true,"refId":"D"} + {"datasource":{"type":"prometheus","uid":"ds-prometheus"},"editorMode":"code","exemplar":false,"expr":"sum(etcd_up{job=\"etcd\"}) by (cls) / count(etcd_up{job=\"etcd\"}) by (cls)","interval":"","legendFormat":"{{ cls }}","range":true,"refId":"D"}, + {"datasource":{"type":"prometheus","uid":"ds-prometheus"},"editorMode":"code","exemplar":false,"expr":"sum(openbao_up{job=\"openbao\"}) by (cls) / count(openbao_up{job=\"openbao\"}) by (cls)","interval":"","legendFormat":"{{ cls }}","range":true,"refId":"O"} ], "title":"", "type":"stat" diff --git a/files/victoria/rules/openbao.yml b/files/victoria/rules/openbao.yml new file mode 100644 index 000000000..a8e93ff9c --- /dev/null +++ b/files/victoria/rules/openbao.yml @@ -0,0 +1,219 @@ +--- +#==============================================================# +# File : openbao.yml +# Ctime : 2026-07-30 +# Mtime : 2026-07-30 +# Desc : Record & alert rules for openbao +# Path : /infra/rules/openbao.yml +# License : Apache-2.0 @ https://pigsty.io/docs/about/license/ +# Copyright : 2018-2026 Ruohang Feng / Vonng (rh@vonng.com) +#==============================================================# +# OpenBao keeps the upstream `vault_` metric prefix on /v1/sys/metrics +# https://openbao.org/docs/internals/telemetry/metrics/ +# +# go-metrics summaries are emitted in ms, as {quantile=...}, +# _sum and _count. Counters carry no _total suffix. +#==============================================================# + + +groups: + ################################################################ + # Openbao Rules # + ################################################################ + - name: openbao-rules + rules: + + # request rate per second (non-login + login) + - record: openbao:ins:req_rps + expr: rate(vault_core_handle_request_count{job="openbao"}[1m]) + - record: openbao:ins:login_rps + expr: rate(vault_core_handle_login_request_count{job="openbao"}[1m]) + + # request latency p99 in seconds (source metric is in ms) + - record: openbao:ins:req_rt_p99 + expr: vault_core_handle_request{job="openbao", quantile="0.99"} / 1000 + + # mean request latency in seconds over 5m + - record: openbao:ins:req_rt_avg_5m + expr: >- + rate(vault_core_handle_request_sum{job="openbao"}[5m]) + / clamp_min(rate(vault_core_handle_request_count{job="openbao"}[5m]), 1) / 1000 + + # how many members of the cluster are unsealed / sealed + - record: openbao:cls:unsealed + expr: sum by (job,cls) (vault_core_unsealed{job="openbao"}) + - record: openbao:cls:sealed + expr: count by (job,cls) (vault_core_unsealed{job="openbao"}) - sum by (job,cls) (vault_core_unsealed{job="openbao"}) + + # number of active (leader) nodes, should be exactly 1 + - record: openbao:cls:active + expr: sum by (job,cls) (vault_core_active{job="openbao"}) + + # raft replication lag of followers, in log entries + - record: openbao:ins:raft_lag + expr: vault_raft_storage_follower_applied_index_delta{job="openbao"} + - record: openbao:cls:raft_lag + expr: max by (job,cls) (vault_raft_storage_follower_applied_index_delta{job="openbao"}) + + # raft commit latency p99 in seconds + - record: openbao:ins:raft_commit_rt_p99 + expr: vault_raft_commitTime{job="openbao", quantile="0.99"} / 1000 + + # utilisation of the main raft goroutine, 0..1 + - record: openbao:ins:raft_saturation_p99 + expr: vault_raft_thread_main_saturation{job="openbao", quantile="0.99"} + + # audit device failure rate per second + - record: openbao:ins:audit_failure_rps + expr: >- + rate(vault_audit_log_request_failure{job="openbao"}[1m]) + + rate(vault_audit_log_response_failure{job="openbao"}[1m]) + + + ################################################################ + # Openbao Alert # + ################################################################ + - name: openbao-alert + rules: + + #==============================================================# + # Aliveness # + #==============================================================# + # openbao instance unreachable: process down, or sealed (503 on /sys/metrics) + - alert: OpenbaoServerDown + expr: openbao_up < 1 + for: 1m + labels: { level: 0, severity: CRIT, category: openbao } + annotations: + summary: "CRIT OpenbaoServerDown {{ $labels.ins }}@{{ $labels.instance }}" + description: | + openbao_up[ins={{ $labels.ins }}, instance={{ $labels.instance }}] = {{ $value }} < 1 + the node is down, or sealed: check `bao status` on {{ $labels.ip }} + /ui/d/openbao-overview + + # a member reports itself sealed while still serving metrics + - alert: OpenbaoSealed + expr: vault_core_unsealed < 1 + for: 1m + labels: { level: 0, severity: CRIT, category: openbao } + annotations: + summary: "CRIT OpenbaoSealed {{ $labels.ins }}@{{ $labels.instance }}" + description: | + vault_core_unsealed[ins={{ $labels.ins }}] = {{ $value }} < 1 + secrets are unavailable on this node, pg_tde keys can not be fetched + /ui/d/openbao-overview?var-cls={{ $labels.cls }} + + #==============================================================# + # Error # + #==============================================================# + # no active node in the cluster: total KMS outage + - alert: OpenbaoNoLeader + expr: openbao:cls:active < 1 + for: 30s + labels: { level: 0, severity: CRIT, category: openbao } + annotations: + summary: "CRIT OpenbaoNoLeader: {{ $labels.cls }} {{ $value }}" + description: | + openbao:cls:active[cls={{ $labels.cls }}] = {{ $value }} < 1 + /ui/d/openbao-overview?from=now-5m&to=now&var-cls={{ $labels.cls }} + + # split brain: more than one node claims to be active + - alert: OpenbaoMultipleLeader + expr: openbao:cls:active > 1 + for: 30s + labels: { level: 0, severity: CRIT, category: openbao } + annotations: + summary: "CRIT OpenbaoMultipleLeader: {{ $labels.cls }} {{ $value }}" + description: | + openbao:cls:active[cls={{ $labels.cls }}] = {{ $value }} > 1 + /ui/d/openbao-overview?from=now-5m&to=now&var-cls={{ $labels.cls }} + + # audit device is rejecting writes: bao blocks requests when audit fails + - alert: OpenbaoAuditFailure + expr: openbao:ins:audit_failure_rps > 0 + for: 1m + labels: { level: 1, severity: WARN, category: openbao } + annotations: + summary: "WARN OpenbaoAuditFailure: {{ $labels.cls }} {{ $labels.ins }}" + description: | + openbao:ins:audit_failure_rps[ins={{ $labels.ins }}] = {{ $value | printf "%.3f" }} > 0 + requests are blocked while the audit device can not be written, check disk on {{ $labels.ip }} + /ui/d/openbao-overview?var-cls={{ $labels.cls }} + + #==============================================================# + # Saturation # + #==============================================================# + # raft autopilot reports a degraded cluster + - alert: OpenbaoAutopilotUnhealthy + expr: min by (job,cls) (vault_autopilot_healthy) < 1 + for: 2m + labels: { level: 1, severity: WARN, category: openbao } + annotations: + summary: "WARN OpenbaoAutopilotUnhealthy: {{ $labels.cls }}" + description: | + vault_autopilot_healthy[cls={{ $labels.cls }}] = {{ $value }} < 1 + /ui/d/openbao-overview?var-cls={{ $labels.cls }} + + # no spare node left: losing one more member loses quorum + - alert: OpenbaoNoFailureTolerance + expr: min by (job,cls) (vault_autopilot_failure_tolerance) < 1 + for: 2m + labels: { level: 1, severity: WARN, category: openbao } + annotations: + summary: "WARN OpenbaoNoFailureTolerance: {{ $labels.cls }} {{ $value }}" + description: | + vault_autopilot_failure_tolerance[cls={{ $labels.cls }}] = {{ $value }} < 1 + the cluster can not survive another member failure + /ui/d/openbao-overview?var-cls={{ $labels.cls }} + + # follower falling behind the leader on the raft log + # evaluated per-peer so the alert names the lagging follower + - alert: OpenbaoRaftFollowerLag + expr: openbao:ins:raft_lag > 1000 + for: 2m + labels: { level: 2, severity: INFO, category: openbao } + annotations: + summary: "INFO OpenbaoRaftFollowerLag: {{ $labels.cls }} peer {{ $labels.peer_id }} {{ $value }}" + description: | + openbao:ins:raft_lag[cls={{ $labels.cls }}, peer_id={{ $labels.peer_id }}] = {{ $value }} > 1000 entries + follower {{ $labels.peer_id }} trails the leader {{ $labels.ins }} + /ui/d/openbao-overview?var-cls={{ $labels.cls }} + + #==============================================================# + # Latency # + #==============================================================# + # request rt p99 > 500ms for 3m + - alert: OpenbaoRequestRTSlow + expr: openbao:ins:req_rt_p99 > 0.500 + for: 3m + labels: { level: 2, severity: INFO, category: openbao } + annotations: + summary: "INFO OpenbaoRequestRTSlow: {{ $labels.cls }} {{ $labels.ins }}" + description: | + openbao:ins:req_rt_p99[ins={{ $labels.ins }}] = {{ $value }} > 500ms + /ui/d/openbao-overview?from=now-10m&to=now&var-cls={{ $labels.cls }} + + # main raft goroutine busy more than half the time for 5m + - alert: OpenbaoRaftThreadSaturated + expr: openbao:ins:raft_saturation_p99 > 0.5 + for: 5m + labels: { level: 2, severity: INFO, category: openbao } + annotations: + summary: "INFO OpenbaoRaftThreadSaturated: {{ $labels.cls }} {{ $labels.ins }}" + description: | + openbao:ins:raft_saturation_p99[ins={{ $labels.ins }}] = {{ $value | printf "%.3f" }} > 0.5 + the raft goroutine is the bottleneck, check disk latency on {{ $labels.ip }} + /ui/d/openbao-overview?from=now-30m&to=now&var-cls={{ $labels.cls }} + + # raft commit rt p99 > 100ms for 3m, usually a disk problem + - alert: OpenbaoRaftCommitSlow + expr: openbao:ins:raft_commit_rt_p99 > 0.100 + for: 3m + labels: { level: 2, severity: INFO, category: openbao } + annotations: + summary: "INFO OpenbaoRaftCommitSlow: {{ $labels.cls }} {{ $labels.ins }}" + description: | + openbao:ins:raft_commit_rt_p99[ins={{ $labels.ins }}] = {{ $value }} > 100ms + /ui/d/openbao-overview?from=now-10m&to=now&var-cls={{ $labels.cls }} + +... diff --git a/roles/infra/tasks/dir.yml b/roles/infra/tasks/dir.yml index 2d7894323..555233dc1 100644 --- a/roles/infra/tasks/dir.yml +++ b/roles/infra/tasks/dir.yml @@ -72,6 +72,7 @@ - { path: /infra/targets/mysql ,owner: victoria } - { path: /infra/targets/kafka ,owner: victoria } - { path: /infra/targets/docker ,owner: victoria } + - { path: /infra/targets/openbao ,owner: victoria } - { path: /infra/targets/patroni ,owner: victoria } ... diff --git a/roles/infra/templates/nginx/index.html.j2 b/roles/infra/templates/nginx/index.html.j2 index dd57400b0..a97c53f2e 100644 --- a/roles/infra/templates/nginx/index.html.j2 +++ b/roles/infra/templates/nginx/index.html.j2 @@ -785,6 +785,7 @@ PGSQL INFRA ETCD + OPENBAO MINIO @@ -808,6 +809,7 @@ 'pg_seq': pg_seq, 'infra_seq': node_info.infra_seq | default(''), 'etcd_seq': node_info.etcd_seq | default(''), + 'openbao_seq': node_info.openbao_seq | default(''), 'minio_cluster': node_info.minio_cluster | default(''), 'minio_seq': node_info.minio_seq | default('') }) %} @@ -841,6 +843,13 @@ etcd-{{ node.etcd_seq }} {% else %} - +{% endif %} + + +{% if node.openbao_seq %} + openbao-{{ node.openbao_seq }} +{% else %} + - {% endif %} diff --git a/roles/infra/templates/nginx/index.zh.html.j2 b/roles/infra/templates/nginx/index.zh.html.j2 index f6a80dd78..b8d37ba7f 100644 --- a/roles/infra/templates/nginx/index.zh.html.j2 +++ b/roles/infra/templates/nginx/index.zh.html.j2 @@ -788,6 +788,7 @@ PGSQL INFRA ETCD + OPENBAO MINIO @@ -811,6 +812,7 @@ 'pg_seq': pg_seq, 'infra_seq': node_info.infra_seq | default(''), 'etcd_seq': node_info.etcd_seq | default(''), + 'openbao_seq': node_info.openbao_seq | default(''), 'minio_cluster': node_info.minio_cluster | default(''), 'minio_seq': node_info.minio_seq | default('') }) %} @@ -844,6 +846,13 @@ etcd-{{ node.etcd_seq }} {% else %} - +{% endif %} + + +{% if node.openbao_seq %} + openbao-{{ node.openbao_seq }} +{% else %} + - {% endif %} diff --git a/roles/infra/templates/prometheus/alertmanager.yml b/roles/infra/templates/prometheus/alertmanager.yml index 29c0dd297..e66e37781 100644 --- a/roles/infra/templates/prometheus/alertmanager.yml +++ b/roles/infra/templates/prometheus/alertmanager.yml @@ -28,7 +28,7 @@ # --------------------------------------------------------------- # level: 0 = CRIT (fatal) 1 = WARN (error) 2 = INFO (event) # severity: CRIT | WARN | INFO -# category: pgsql | node | redis | etcd | minio | infra +# category: pgsql | node | redis | etcd | openbao | minio | infra # cls: cluster name ins: instance name ip: instance ip # # RECEIVER TYPES (see examples below): diff --git a/roles/infra/templates/victoria/agent.yml b/roles/infra/templates/victoria/agent.yml index c0e229280..0b2fc968f 100644 --- a/roles/infra/templates/victoria/agent.yml +++ b/roles/infra/templates/victoria/agent.yml @@ -34,6 +34,10 @@ groups: - record: etcd_up expr: up{job="etcd"} + # OPENBAO + - record: openbao_up + expr: up{job="openbao"} + # MINIO - record: minio_up expr: up{job="minio"} diff --git a/roles/infra/templates/victoria/prometheus.yml b/roles/infra/templates/victoria/prometheus.yml index ec7f9c0eb..5b4d081de 100644 --- a/roles/infra/templates/victoria/prometheus.yml +++ b/roles/infra/templates/victoria/prometheus.yml @@ -94,6 +94,22 @@ scrape_configs: cert_file: /etc/pki/infra.crt key_file: /etc/pki/infra.key + #--------------------------------------------------------------# + # job: openbao + # labels: [cls, ins, ip, instance] + # path: targets/openbao/.yml + #--------------------------------------------------------------# + - job_name: openbao + metrics_path: /v1/sys/metrics + params: { format: [ prometheus ] } + file_sd_configs: + - files: [ /infra/targets/openbao/*.yml ] + scheme: https + tls_config: + ca_file: /etc/pki/ca.crt + cert_file: /etc/pki/infra.crt + key_file: /etc/pki/infra.key + #--------------------------------------------------------------# # job: minio # labels: [cls, ins, ip, instance] From e08982b78cf5d648f8d4b92def4d1d81421e9e6c Mon Sep 17 00:00:00 2001 From: alemacci Date: Sun, 16 Aug 2026 15:01:06 +0200 Subject: [PATCH 6/9] feat(openbao): add openbao package to distro/arch package list --- roles/node_id/tasks/main.yml | 9 +++++---- roles/node_id/vars/el10.aarch64.yml | 2 +- roles/node_id/vars/el10.x86_64.yml | 2 +- roles/node_id/vars/el7.x86_64.yml | 2 +- roles/node_id/vars/el8.aarch64.yml | 2 +- roles/node_id/vars/el8.x86_64.yml | 2 +- roles/node_id/vars/el9.aarch64.yml | 2 +- roles/node_id/vars/el9.x86_64.yml | 2 +- 8 files changed, 12 insertions(+), 11 deletions(-) diff --git a/roles/node_id/tasks/main.yml b/roles/node_id/tasks/main.yml index 7a9717772..737f9edca 100644 --- a/roles/node_id/tasks/main.yml +++ b/roles/node_id/tasks/main.yml @@ -98,10 +98,11 @@ {% if inventory_hostname == admin_ip %}[ADMIN]{% endif %} {% if inventory_hostname in groups["infra"]|default([]) %}[INFRA]{% endif %} {% if pg_cluster is defined %}[PGSQL: {{ pg_cluster }}]{% endif %} - {% if redis_cluster is defined %}[REDIS: {{ redis_cluster }}]{% endif %} - {% if etcd_cluster is defined %}[ETCD: {{ etcd_cluster }}]{% endif %} - {% if minio_cluster is defined %}[MINIO: {{ minio_cluster }}]{% endif %} - {% if kafka_cluster is defined %}[KAFKA: {{ kafka_cluster }}]{% endif %} + {% if redis_cluster is defined %}[REDIS: {{ redis_cluster }}]{% endif %} + {% if etcd_cluster is defined %}[ETCD: {{ etcd_cluster }}]{% endif %} + {% if openbao_cluster is defined %}[OPENBAO: {{ openbao_cluster }}]{% endif %} + {% if minio_cluster is defined %}[MINIO: {{ minio_cluster }}]{% endif %} + {% if kafka_cluster is defined %}[KAFKA: {{ kafka_cluster }}]{% endif %} {% if vip_enabled is defined and vip_enabled|bool %}[VIP: {{ vip_address }}]{% endif %} ... diff --git a/roles/node_id/vars/el10.aarch64.yml b/roles/node_id/vars/el10.aarch64.yml index 15ac0056d..bb993f4c5 100644 --- a/roles/node_id/vars/el10.aarch64.yml +++ b/roles/node_id/vars/el10.aarch64.yml @@ -83,7 +83,7 @@ package_map: # REPO: download packages #--------------------------------# node-bootstrap: "ansible python3 python3-requests python3-jmespath python3-cryptography dnf-utils sow sshpass crypto-policies-scripts" - infra-package: "nginx dnsmasq etcd haproxy vip-manager node-exporter keepalived-exporter pg-exporter pgbackrest-exporter redis-exporter redis valkey silo mcli sow pig" + infra-package: "nginx dnsmasq etcd openbao haproxy vip-manager node-exporter keepalived-exporter pg-exporter pgbackrest-exporter redis-exporter redis valkey silo mcli sow pig" infra-addons: "grafana grafana-plugins grafana-victoriametrics-ds grafana-victorialogs-ds victoria-metrics victoria-logs victoria-traces vlogscli vmutils vector alertmanager" extra-modules: "blackbox-exporter nginx-exporter pev2 certbot python3-certbot-nginx docker-ce docker-compose-plugin duckdb restic juicefs vray grafana-infinity-ds" node-package1: "bash python3 sudo acl ca-certificates openssl curl wget lz4 zstd unzip bzip2 gzip tar tzdata chrony openssh-server util-linux rsync psmisc logrotate" diff --git a/roles/node_id/vars/el10.x86_64.yml b/roles/node_id/vars/el10.x86_64.yml index a3c6c2dbc..6ee1650c5 100644 --- a/roles/node_id/vars/el10.x86_64.yml +++ b/roles/node_id/vars/el10.x86_64.yml @@ -83,7 +83,7 @@ package_map: # REPO: download packages #--------------------------------# node-bootstrap: "ansible python3 python3-requests python3-jmespath python3-cryptography dnf-utils sow sshpass crypto-policies-scripts" - infra-package: "nginx dnsmasq etcd haproxy vip-manager node-exporter keepalived-exporter pg-exporter pgbackrest-exporter redis-exporter redis valkey silo mcli sow pig" + infra-package: "nginx dnsmasq etcd openbao haproxy vip-manager node-exporter keepalived-exporter pg-exporter pgbackrest-exporter redis-exporter redis valkey silo mcli sow pig" infra-addons: "grafana grafana-plugins grafana-victoriametrics-ds grafana-victorialogs-ds victoria-metrics victoria-logs victoria-traces vlogscli vmutils vector alertmanager" extra-modules: "blackbox-exporter nginx-exporter pev2 certbot python3-certbot-nginx docker-ce docker-compose-plugin duckdb restic juicefs vray grafana-infinity-ds" node-package1: "bash python3 sudo acl ca-certificates openssl curl wget lz4 zstd unzip bzip2 gzip tar tzdata chrony openssh-server util-linux rsync psmisc logrotate" diff --git a/roles/node_id/vars/el7.x86_64.yml b/roles/node_id/vars/el7.x86_64.yml index 4c977a8bf..287fb8681 100644 --- a/roles/node_id/vars/el7.x86_64.yml +++ b/roles/node_id/vars/el7.x86_64.yml @@ -67,7 +67,7 @@ package_map: # REPO: download packages #--------------------------------# node-bootstrap: "ansible python3 python36-requests python36-idna yum-utils sow sshpass" - infra-package: "nginx dnsmasq etcd haproxy vip-manager node-exporter keepalived-exporter pg-exporter pgbackrest-exporter redis-exporter redis valkey silo mcli sow pig" + infra-package: "nginx dnsmasq etcd openbao haproxy vip-manager node-exporter keepalived-exporter pg-exporter pgbackrest-exporter redis-exporter redis valkey silo mcli sow pig" infra-addons: "grafana grafana-plugins grafana-victoriametrics-ds grafana-victorialogs-ds victoria-metrics victoria-logs victoria-traces vlogscli vmutils vector alertmanager uv" extra-modules: "blackbox-exporter nginx-exporter pev2 docker-ce docker-compose-plugin duckdb restic juicefs vray grafana-infinity-ds" node-package1: "lz4 unzip bzip2 zlib yum pv jq git ncdu make patch bash lsof wget uuid tuned nvme-cli numactl grubby sysstat iotop htop rsync tcpdump perf flamegraph chkconfig uv" diff --git a/roles/node_id/vars/el8.aarch64.yml b/roles/node_id/vars/el8.aarch64.yml index 33dc116cd..016700b53 100644 --- a/roles/node_id/vars/el8.aarch64.yml +++ b/roles/node_id/vars/el8.aarch64.yml @@ -83,7 +83,7 @@ package_map: # REPO: download packages #--------------------------------# node-bootstrap: "ansible python3 python3-requests python3.12-jmespath python3-cryptography dnf-utils sow sshpass" - infra-package: "nginx dnsmasq etcd haproxy vip-manager node-exporter keepalived-exporter pg-exporter pgbackrest-exporter redis-exporter redis valkey silo mcli sow pig" + infra-package: "nginx dnsmasq etcd openbao haproxy vip-manager node-exporter keepalived-exporter pg-exporter pgbackrest-exporter redis-exporter redis valkey silo mcli sow pig" infra-addons: "grafana grafana-plugins grafana-victoriametrics-ds grafana-victorialogs-ds victoria-metrics victoria-logs victoria-traces vlogscli vmutils vector alertmanager" extra-modules: "blackbox-exporter nginx-exporter pev2 certbot python3-certbot-nginx docker-ce docker-compose-plugin duckdb restic juicefs vray grafana-infinity-ds" node-package1: "bash python3 sudo acl ca-certificates openssl curl wget lz4 zstd unzip bzip2 gzip tar tzdata chrony openssh-server util-linux rsync psmisc logrotate" diff --git a/roles/node_id/vars/el8.x86_64.yml b/roles/node_id/vars/el8.x86_64.yml index 8e0e2d80a..c5a50b19f 100644 --- a/roles/node_id/vars/el8.x86_64.yml +++ b/roles/node_id/vars/el8.x86_64.yml @@ -83,7 +83,7 @@ package_map: # REPO: download packages #--------------------------------# node-bootstrap: "ansible python3 python3-requests python3.12-jmespath python3-cryptography dnf-utils sow sshpass" - infra-package: "nginx dnsmasq etcd haproxy vip-manager node-exporter keepalived-exporter pg-exporter pgbackrest-exporter redis-exporter redis valkey silo mcli sow pig" + infra-package: "nginx dnsmasq etcd openbao haproxy vip-manager node-exporter keepalived-exporter pg-exporter pgbackrest-exporter redis-exporter redis valkey silo mcli sow pig" infra-addons: "grafana grafana-plugins grafana-victoriametrics-ds grafana-victorialogs-ds victoria-metrics victoria-logs victoria-traces vlogscli vmutils vector alertmanager" extra-modules: "blackbox-exporter nginx-exporter pev2 certbot python3-certbot-nginx docker-ce docker-compose-plugin duckdb restic juicefs vray grafana-infinity-ds" node-package1: "bash python3 sudo acl ca-certificates openssl curl wget lz4 zstd unzip bzip2 gzip tar tzdata chrony openssh-server util-linux rsync psmisc logrotate" diff --git a/roles/node_id/vars/el9.aarch64.yml b/roles/node_id/vars/el9.aarch64.yml index a273cf780..f22416cbf 100644 --- a/roles/node_id/vars/el9.aarch64.yml +++ b/roles/node_id/vars/el9.aarch64.yml @@ -83,7 +83,7 @@ package_map: # REPO: download packages #--------------------------------# node-bootstrap: "ansible python3 python3-requests python3-jmespath python3-cryptography dnf-utils sow sshpass" - infra-package: "nginx dnsmasq etcd haproxy vip-manager node-exporter keepalived-exporter pg-exporter pgbackrest-exporter redis-exporter redis valkey silo mcli sow pig" + infra-package: "nginx dnsmasq etcd openbao haproxy vip-manager node-exporter keepalived-exporter pg-exporter pgbackrest-exporter redis-exporter redis valkey silo mcli sow pig" infra-addons: "grafana grafana-plugins grafana-victoriametrics-ds grafana-victorialogs-ds victoria-metrics victoria-logs victoria-traces vlogscli vmutils vector alertmanager" extra-modules: "blackbox-exporter nginx-exporter pev2 certbot python3-certbot-nginx docker-ce docker-compose-plugin duckdb restic juicefs vray grafana-infinity-ds" node-package1: "bash python3 sudo acl ca-certificates openssl curl wget lz4 zstd unzip bzip2 gzip tar tzdata chrony openssh-server util-linux rsync psmisc logrotate" diff --git a/roles/node_id/vars/el9.x86_64.yml b/roles/node_id/vars/el9.x86_64.yml index bf94b55f5..4282e1462 100644 --- a/roles/node_id/vars/el9.x86_64.yml +++ b/roles/node_id/vars/el9.x86_64.yml @@ -83,7 +83,7 @@ package_map: # REPO: download packages #--------------------------------# node-bootstrap: "ansible python3 python3-requests python3-jmespath python3-cryptography dnf-utils sow sshpass" - infra-package: "nginx dnsmasq etcd haproxy vip-manager node-exporter keepalived-exporter pg-exporter pgbackrest-exporter redis-exporter redis valkey silo mcli sow pig" + infra-package: "nginx dnsmasq etcd openbao haproxy vip-manager node-exporter keepalived-exporter pg-exporter pgbackrest-exporter redis-exporter redis valkey silo mcli sow pig" infra-addons: "grafana grafana-plugins grafana-victoriametrics-ds grafana-victorialogs-ds victoria-metrics victoria-logs victoria-traces vlogscli vmutils vector alertmanager" extra-modules: "blackbox-exporter nginx-exporter pev2 certbot python3-certbot-nginx docker-ce docker-compose-plugin duckdb restic juicefs vray grafana-infinity-ds" node-package1: "bash python3 sudo acl ca-certificates openssl curl wget lz4 zstd unzip bzip2 gzip tar tzdata chrony openssh-server util-linux rsync psmisc logrotate" From 700e68e0f25ad3fecbba23a46cdfe24b7baaadb9 Mon Sep 17 00:00:00 2001 From: alemacci Date: Sun, 16 Aug 2026 15:01:35 +0200 Subject: [PATCH 7/9] feat(openbao): wire openbao role into deploy.yml --- deploy.yml | 23 ++++++++++++++++++----- 1 file changed, 18 insertions(+), 5 deletions(-) diff --git a/deploy.yml b/deploy.yml index 03bca05cd..3276967dd 100755 --- a/deploy.yml +++ b/deploy.yml @@ -22,16 +22,18 @@ # - infra : init nginx, dns, victoria, grafana # - node-monitor : init node-exporter vector # - etcd : init etcd (required for pgsql HA) +# - openbao : init openbao (optional for pg_tde) # - minio : init minio (optional) # - pgsql : init pgsql # - pgsql-monitor : init pgsql # # Which is equivalent to the following 4 playbooks altogether: -# - infra.yml -l infra deploy infrastructure on group 'infra' -# - node.yml -l all init all nodes -# - etcd.yml -l etcd init etcd on group 'etcd' for pg ha -# - minio.yml -l init minio hosts marked with minio_cluster -# - pgsql.yml -l all init pgsql database clusters on all nodes +# - infra.yml -l infra deploy infrastructure on group 'infra' +# - node.yml -l all init all nodes +# - etcd.yml -l etcd init etcd on group 'etcd' for pg ha +# - openbao.yml -l openbao init openbao on group 'openbao' for pg_tde +# - minio.yml -l init minio hosts marked with minio_cluster +# - pgsql.yml -l all init pgsql database clusters on all nodes #==============================================================# @@ -115,6 +117,17 @@ roles: [ { role: etcd } ] # init etcd on fixed group 'etcd' +#--------------------------------------------------------------- +# OPENBAO INIT +#--------------------------------------------------------------- +- name: OPENBAO INIT + become: true + hosts: openbao + gather_facts: no + tags: openbao + roles: [ { role: openbao ,when: openbao_cluster is defined } ] + + #--------------------------------------------------------------- # MINIO INIT #--------------------------------------------------------------- From ffd49a2a125cef705040d30aa76c86e156c617c6 Mon Sep 17 00:00:00 2001 From: alemacci Date: Sun, 16 Aug 2026 15:01:51 +0200 Subject: [PATCH 8/9] chore(openbao): ignore generated credential files under files/openbao --- .gitignore | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.gitignore b/.gitignore index f8b22d5eb..65656d018 100644 --- a/.gitignore +++ b/.gitignore @@ -8,6 +8,11 @@ files/*.key files/*.crt files/pki/* +######################################################### +# openbao init material: root token + recovery shares +######################################################### +files/openbao/* + ######################################################### # vagrant & terraform & docker ######################################################### From 8966bedd0d627fdb65a778f91b4dddc22439c21a Mon Sep 17 00:00:00 2001 From: alemacci Date: Sun, 16 Aug 2026 15:03:42 +0200 Subject: [PATCH 9/9] feat(conf): refactor pgtde.yml and add HA template --- conf/ha/pgtde.yml | 135 ++++++++++++++++++++++++++++++++++++++++++++++ conf/pgtde.yml | 19 +++++-- 2 files changed, 149 insertions(+), 5 deletions(-) create mode 100644 conf/ha/pgtde.yml diff --git a/conf/ha/pgtde.yml b/conf/ha/pgtde.yml new file mode 100644 index 000000000..66adbd51e --- /dev/null +++ b/conf/ha/pgtde.yml @@ -0,0 +1,135 @@ +--- +#==============================================================# +# File : pgtde.yml +# Desc : PG TDE with Percona PostgreSQL 3-node HA template +# Ctime : 2025-07-04 +# Mtime : 2026-07-23 +# Docs : https://pigsty.io/docs/conf/pgtde +# License : Apache-2.0 @ https://pigsty.io/docs/about/license/ +# Copyright : 2018-2026 Ruohang Feng / Vonng (rh@vonng.com) +#==============================================================# + +# HA variant of the single-node template with pg_tde +# currently based on PostgreSQL 18 +# tutorial: https://pigsty.io/docs/pgsql/kernel/percona +# +# Usage: +# curl https://repo.pigsty.io/get | bash +# ./configure -c pgtde +# ./deploy.yml + +all: + children: + + infra: + hosts: + 10.10.10.10: { infra_seq: 1 } + vars: + repo_enabled: false + haproxy_services: + - name: openbao # [REQUIRED] service name, unique + port: 8200 # [REQUIRED] service port, unique + options: # [OPTIONAL] openbao health check + - option httpchk + - http-check send meth GET uri /v1/sys/health + - http-check expect status 200 + servers: + - { name: openbao-1 ,ip: 10.10.10.11 ,port: 8200 ,options: 'check-ssl ca-file /etc/pki/ca.crt check' } + - { name: openbao-2 ,ip: 10.10.10.12 ,port: 8200 ,options: 'check-ssl ca-file /etc/pki/ca.crt check' } + - { name: openbao-3 ,ip: 10.10.10.13 ,port: 8200 ,options: 'check-ssl ca-file /etc/pki/ca.crt check' } + + # etcd cluster for ha postgres + etcd: + hosts: + 10.10.10.11: { etcd_seq: 1 } + 10.10.10.12: { etcd_seq: 2 } + 10.10.10.13: { etcd_seq: 3 } + vars: + etcd_cluster: etcd + + # openbao kms cluster for pg_tde + openbao: + hosts: + 10.10.10.11: { openbao_seq: 1 } + 10.10.10.12: { openbao_seq: 2 } + 10.10.10.13: { openbao_seq: 3 } + vars: + openbao_cluster: openbao + + #minio: { hosts: { 10.10.10.10: { minio_seq: 1 }} ,vars: { minio_cluster: minio }} + + #----------------------------------------------# + # Percona Postgres Database Cluster + #----------------------------------------------# + pg-meta: + hosts: + 10.10.10.21: { pg_seq: 1 , pg_role: primary } + 10.10.10.22: { pg_seq: 2 , pg_role: replica } + 10.10.10.23: { pg_seq: 3 , pg_role: replica } + vars: + pg_mode: pgtde + pg_cluster: pg-meta + pg_vip_enabled: true + pg_vip_address: 10.10.10.20/24 + pg_users: + - { name: dbuser_meta ,password: DBUser.Meta ,pgbouncer: true ,roles: [dbrole_admin ] ,comment: pigsty admin user } + - { name: dbuser_view ,password: DBUser.Viewer ,pgbouncer: true ,roles: [dbrole_readonly] ,comment: read-only viewer } + pg_databases: + - name: meta + baseline: cmdb.sql + comment: pigsty tde database + schemas: [pigsty] + extensions: [ pg_tde, vector, postgis, pgaudit, { name: pg_stat_monitor, schema: monitor } ] + pg_hba_rules: # https://pigsty.io/docs/pgsql/config/hba + - { user: all ,db: all ,addr: intra ,auth: pwd ,title: 'everyone intranet access with password' ,order: 800 } + pg_crontab: # https://pigsty.io/docs/pgsql/admin/crontab + - '00 01 * * * /pg/bin/pg-backup full' + + # Percona PostgreSQL TDE Kernel Settings + pg_packages: [ pgtde, pgsql-common ] # install Pigsty private-prefix Percona packages + pg_libs: 'pg_tde, pgaudit, pg_stat_statements, pg_stat_monitor, auto_explain' + + vars: + #----------------------------------------------# + # INFRA : https://pigsty.io/docs/infra/param + #----------------------------------------------# + version: v4.5.0 # pigsty version string + admin_ip: 10.10.10.10 # admin node ip address + region: default # upstream mirror region: default,china,europe + infra_portal: # infra services exposed via portal + home : { domain: i.pigsty } # default domain name + openbao : { domain: bao.pigsty ,endpoint: "10.10.10.10:8200" ,scheme: https } + + #----------------------------------------------# + # NODE : https://pigsty.io/docs/node/param + #----------------------------------------------# + nodename_overwrite: false # do not overwrite node hostname on single node mode + node_repo_modules: node,infra,pgsql + node_tune: oltp + node_etc_hosts: ["10.10.10.10 bao.pigsty"] # domain name to access openbao from all nodes (required) + + #----------------------------------------------# + # PGSQL : https://pigsty.io/docs/pgsql/param + #----------------------------------------------# + pg_version: 18 # Default Percona TDE PG Major Version is 18 + pg_conf: oltp.yml # pgsql tuning specs: {oltp,olap,tiny,crit}.yml + + #----------------------------------------------# + # OPENBAO + #----------------------------------------------# + openbao_cluster: openbao # bao cluster & group name, openbao by default + openbao_domain: bao.pigsty # bao domain name, `bao.pigsty` by default + + #----------------------------------------------# + # PASSWORD : https://pigsty.io/docs/setup/security/ + #----------------------------------------------# + grafana_admin_password: pigsty + grafana_view_password: DBUser.Viewer + pg_admin_password: DBUser.DBA + pg_monitor_password: DBUser.Monitor + pg_replication_password: DBUser.Replicator + patroni_password: Patroni.API + haproxy_admin_password: pigsty + minio_secret_key: S3User.MinIO + etcd_root_password: Etcd.Root +... diff --git a/conf/pgtde.yml b/conf/pgtde.yml index 90e4f0bfb..9557f2cc6 100644 --- a/conf/pgtde.yml +++ b/conf/pgtde.yml @@ -20,9 +20,10 @@ all: children: - infra: { hosts: { 10.10.10.10: { infra_seq: 1 }} ,vars: { repo_enabled: false }} - etcd: { hosts: { 10.10.10.10: { etcd_seq: 1 }} ,vars: { etcd_cluster: etcd }} - #minio: { hosts: { 10.10.10.10: { minio_seq: 1 }} ,vars: { minio_cluster: minio }} + infra: { hosts: { 10.10.10.10: { infra_seq: 1 }} ,vars: { repo_enabled: false }} + etcd: { hosts: { 10.10.10.10: { etcd_seq: 1 }} ,vars: { etcd_cluster: etcd }} + openbao: { hosts: { 10.10.10.10: { openbao_seq: 1 }} ,vars: { openbao_cluster: openbao }} + #minio: { hosts: { 10.10.10.10: { minio_seq: 1 }} ,vars: { minio_cluster: minio }} #----------------------------------------------# # Percona Postgres Database Cluster @@ -41,7 +42,7 @@ all: baseline: cmdb.sql comment: pigsty tde database schemas: [pigsty] - extensions: [ vector, postgis, pg_tde ,pgaudit, { name: pg_stat_monitor, schema: monitor } ] + extensions: [ pg_tde, vector, postgis ,pgaudit, { name: pg_stat_monitor, schema: monitor } ] pg_hba_rules: # https://pigsty.io/docs/pgsql/config/hba - { user: all ,db: all ,addr: intra ,auth: pwd ,title: 'everyone intranet access with password' ,order: 800 } pg_crontab: # https://pigsty.io/docs/pgsql/admin/crontab @@ -59,7 +60,8 @@ all: admin_ip: 10.10.10.10 # admin node ip address region: default # upstream mirror region: default,china,europe infra_portal: # infra services exposed via portal - home : { domain: i.pigsty } # default domain name + home : { domain: i.pigsty } # default domain name + openbao : { domain: bao.pigsty ,endpoint: "10.10.10.10:8200" ,scheme: https } #----------------------------------------------# # NODE : https://pigsty.io/docs/node/param @@ -67,6 +69,7 @@ all: nodename_overwrite: false # do not overwrite node hostname on single node mode node_repo_modules: node,infra,pgsql node_tune: oltp + node_etc_hosts: ["10.10.10.10 bao.pigsty"] # domain name to access openbao from all nodes (required) #----------------------------------------------# # PGSQL : https://pigsty.io/docs/pgsql/param @@ -74,6 +77,12 @@ all: pg_version: 18 # Default Percona TDE PG Major Version is 18 pg_conf: oltp.yml # pgsql tuning specs: {oltp,olap,tiny,crit}.yml + #----------------------------------------------# + # OPENBAO + #----------------------------------------------# + openbao_cluster: openbao # bao cluster & group name, openbao by default + openbao_domain: bao.pigsty # bao domain name, `bao.pigsty` by default + #----------------------------------------------# # PASSWORD : https://pigsty.io/docs/setup/security/ #----------------------------------------------#