add alerts to grafana

Author: JuArce

grafana/provisioning/dashboards/aligned/aggregation_mode_gateway.json

@@ -19,14 +19,14 @@ datasources:
     access: proxy
     orgId: 1
     url: "${POSTGRES_HOST}:${POSTGRES_PORT}"
-    database: ${POSTGRES_DB}
-    user: ${POSTGRES_USER}
+    user: "${POSTGRES_USER}"
     secureJsonData:
-      password: ${POSTGRES_PASSWORD}
+      password: "${POSTGRES_PASSWORD}"
     basicAuth: false
     isDefault: false
     editable: true
     jsonData:
+      database: "${POSTGRES_DB}"
       sslmode: disable
       postgresVersion: 1600
 
@@ -72,13 +72,13 @@ datasources:
     access: proxy
     orgId: 1
     url: "${MONITOR_DB_HOST}:${MONITOR_DB_PORT}"
-    database: "${MONITOR_DB_DB}"
     user: "${MONITOR_DB_USER}"
     secureJsonData:
-      password: ${MONITOR_DB_PASSWORD}
+      password: "${MONITOR_DB_PASSWORD}"
     basicAuth: false
     isDefault: false
     editable: true
     jsonData:
+      database: "${MONITOR_DB_DB}"
       sslmode: require
-      postgresVersion: 1700
+      postgresVersion: 1600

grafana/provisioning/datasources/datasource.yaml

@@ -31,6 +31,8 @@
         grafana_var_payments_contract: "{{ lookup('ini', 'grafana_var_payments_contract', file=config_file) }}"
         grafana_var_proof_aggregator_contract: "{{ lookup('ini', 'grafana_var_proof_aggregator_contract', file=config_file) }}"
         grafana_var_proof_aggregator_wallet: "{{ lookup('ini', 'grafana_var_proof_aggregator_wallet', file=config_file) }}"
+        grafana_slack_webhook_url: "{{ lookup('ini', 'grafana_slack_webhook_url', file=config_file) }}"
+        grafana_pagerduty_integration_key: "{{ lookup('ini', 'grafana_pagerduty_integration_key', file=config_file) }}"
       no_log: true
 
     - name: Install required packages
@@ -137,6 +139,50 @@
       vars:
         ansible_ssh_user: "{{ admin_user }}"
 
+    - name: Create alerting provisioning directory
+      become: true
+      file:
+        path: /etc/grafana/provisioning/alerting
+        state: directory
+        owner: grafana
+        group: grafana
+        mode: '0755'
+      vars:
+        ansible_ssh_user: "{{ admin_user }}"
+
+    - name: Template alerting contact points
+      become: true
+      template:
+        src: grafana/alerting/contact-points.yaml.j2
+        dest: /etc/grafana/provisioning/alerting/contact-points.yaml
+        owner: grafana
+        group: grafana
+        mode: '0644'
+      vars:
+        ansible_ssh_user: "{{ admin_user }}"
+
+    - name: Template alerting notification policies
+      become: true
+      template:
+        src: grafana/alerting/notification-policies.yaml.j2
+        dest: /etc/grafana/provisioning/alerting/notification-policies.yaml
+        owner: grafana
+        group: grafana
+        mode: '0644'
+      vars:
+        ansible_ssh_user: "{{ admin_user }}"
+
+    - name: Template alerting rules
+      become: true
+      template:
+        src: grafana/alerting/alert-rules.yaml.j2
+        dest: /etc/grafana/provisioning/alerting/alert-rules.yaml
+        owner: grafana
+        group: grafana
+        mode: '0644'
+      vars:
+        ansible_ssh_user: "{{ admin_user }}"
+
     - name: Enable and start Grafana service
       become: true
       systemd_service:

infra/aggregation_mode/ansible/playbooks/grafana_agg_mode.yaml

@@ -18,6 +18,9 @@ tls_key_source_path=
 grafana_admin_password=
 # Task sender private key (for sending proofs)
 task_sender_private_key=
+# Alerting configuration (Slack and PagerDuty)
+grafana_slack_webhook_url=
+grafana_pagerduty_integration_key=
 
 # ============================================
 # Environment

infra/aggregation_mode/ansible/playbooks/ini/config-hoodi.ini

@@ -18,6 +18,9 @@ tls_key_source_path=
 grafana_admin_password=
 # Task sender private key (for sending proofs)
 task_sender_private_key=
+# Alerting configuration (Slack and PagerDuty)
+grafana_slack_webhook_url=
+grafana_pagerduty_integration_key=
 
 # ============================================
 # Environment