☀ projected k8s-manifests-github from 11c55da

Source-holobranch: k8s-manifests-github
Source-commit: 11c55da
Source: 11c55da

Author: themightychris

grafana/ConfigMap/grafana.yaml

@@ -1,5 +1,16 @@
 apiVersion: v1
 data:
+  contactpoints.yaml: |
+    apiVersion: 1
+    contactPoints:
+    - name: cluster-alerts-email
+      orgId: 1
+      receivers:
+      - settings:
+          addresses: ${ALERT_EMAIL_TO}
+          singleEmail: false
+        type: email
+        uid: cluster-alerts-email
   dashboardproviders.yaml: |
     apiVersion: 1
     providers:
@@ -17,10 +28,12 @@ data:
     - isDefault: true
       name: Prometheus
       type: prometheus
+      uid: prometheus
       url: http://prometheus-server.prometheus.svc.cluster.local
     - access: proxy
       name: Loki
       type: loki
+      uid: loki
       url: http://loki.loki.svc.cluster.local:3100
       version: 1
   download_dashboards.sh: |
@@ -86,6 +99,201 @@ data:
     provisioning = /etc/grafana/provisioning
     [server]
     domain = metrics.sandbox.k8s.phl.io
+    [smtp]
+    enabled = true
+    from_name = Cluster Monitoring
+    host = smtp.postmarkapp.com:587
+  policies.yaml: |
+    apiVersion: 1
+    policies:
+    - group_by:
+      - alertname
+      - namespace
+      - name
+      group_interval: 5m
+      group_wait: 30s
+      orgId: 1
+      receiver: cluster-alerts-email
+      repeat_interval: 12h
+  rules.yaml: |
+    apiVersion: 1
+    groups:
+    - folder: Certificates
+      interval: 5m
+      name: cert-manager
+      orgId: 1
+      rules:
+      - annotations:
+          description: renewalTime is over a day in the past and renewal hasn't happened.
+            The cert is probably still valid, so nothing looks broken yet — which is exactly
+            how this failure mode hides. Check `kubectl get challenges -A` for a stuck
+            challenge.
+          summary: 'Certificate {{ $labels.namespace }}/{{ $labels.name }} is overdue
+            for renewal'
+        condition: C
+        data:
+        - datasourceUid: prometheus
+          model:
+            expr: time() - certmanager_certificate_renewal_timestamp_seconds
+            instant: true
+            refId: A
+          refId: A
+          relativeTimeRange:
+            from: 600
+            to: 0
+        - datasourceUid: __expr__
+          model:
+            conditions:
+            - evaluator:
+                params:
+                - 86400
+                type: gt
+            expression: A
+            refId: C
+            type: threshold
+          refId: C
+        execErrState: OK
+        for: 1h
+        labels:
+          severity: warning
+        noDataState: OK
+        title: CertRenewalOverdue
+        uid: cert-renewal-overdue
+      - annotations:
+          description: Challenge in flight for over an hour. Healthy ones resolve in seconds.
+            If its Order is gone it must be deleted by name, or it starves every future
+            renewal for this hostname.
+          summary: 'ACME challenge stuck for {{ $labels.namespace }}/{{ $labels.name
+            }}'
+        condition: C
+        data:
+        - datasourceUid: prometheus
+          model:
+            expr: certmanager_certificate_challenge_status
+            instant: true
+            refId: A
+          refId: A
+          relativeTimeRange:
+            from: 600
+            to: 0
+        - datasourceUid: __expr__
+          model:
+            conditions:
+            - evaluator:
+                params:
+                - 0
+                type: gt
+            expression: A
+            refId: C
+            type: threshold
+          refId: C
+        execErrState: OK
+        for: 1h
+        labels:
+          severity: warning
+        noDataState: OK
+        title: CertChallengeStuck
+        uid: cert-challenge-stuck
+      - annotations:
+          description: Renewal starts at 30 days, so this has been failing for a week
+            or more.
+          summary: 'Certificate {{ $labels.namespace }}/{{ $labels.name }} expires
+            within 21 days'
+        condition: C
+        data:
+        - datasourceUid: prometheus
+          model:
+            expr: certmanager_certificate_expiration_timestamp_seconds - time()
+            instant: true
+            refId: A
+          refId: A
+          relativeTimeRange:
+            from: 600
+            to: 0
+        - datasourceUid: __expr__
+          model:
+            conditions:
+            - evaluator:
+                params:
+                - 1814400
+                type: lt
+            expression: A
+            refId: C
+            type: threshold
+          refId: C
+        execErrState: OK
+        for: 1h
+        labels:
+          severity: warning
+        noDataState: OK
+        title: CertExpiringSoon
+        uid: cert-expiring-soon
+      - annotations:
+          description: Under a week to expiry. Users will see browser warnings when this
+            lapses.
+          summary: 'Certificate {{ $labels.namespace }}/{{ $labels.name }} expires
+            within 7 days'
+        condition: C
+        data:
+        - datasourceUid: prometheus
+          model:
+            expr: certmanager_certificate_expiration_timestamp_seconds - time()
+            instant: true
+            refId: A
+          refId: A
+          relativeTimeRange:
+            from: 600
+            to: 0
+        - datasourceUid: __expr__
+          model:
+            conditions:
+            - evaluator:
+                params:
+                - 604800
+                type: lt
+            expression: A
+            refId: C
+            type: threshold
+          refId: C
+        execErrState: OK
+        for: 10m
+        labels:
+          severity: critical
+        noDataState: OK
+        title: CertExpiringCritical
+        uid: cert-expiring-critical
+      - annotations:
+          description: No certificate can be issued or renewed through this issuer.
+          summary: 'ClusterIssuer {{ $labels.name }} is not Ready'
+        condition: C
+        data:
+        - datasourceUid: prometheus
+          model:
+            expr: certmanager_clusterissuer_ready_status{condition="True"}
+            instant: true
+            refId: A
+          refId: A
+          relativeTimeRange:
+            from: 600
+            to: 0
+        - datasourceUid: __expr__
+          model:
+            conditions:
+            - evaluator:
+                params:
+                - 1
+                type: lt
+            expression: A
+            refId: C
+            type: threshold
+          refId: C
+        execErrState: OK
+        for: 15m
+        labels:
+          severity: critical
+        noDataState: OK
+        title: ClusterIssuerNotReady
+        uid: clusterissuer-not-ready
 kind: ConfigMap
 metadata:
   labels:

grafana/Deployment/grafana.yaml

@@ -21,7 +21,7 @@ spec:
   template:
     metadata:
       annotations:
-        checksum/config: 72c936e8e1ff007758598b923885b11f3001441e27b8b2f9b2c466de0def2f1e
+        checksum/config: 25260123a4e6ba00201f2dd3fd0a5afafb2e7db07bd3e3cc764cde04ea9dc9e0
         checksum/dashboards-json-config: 4f1a6849aba728c6f66b9b27a78f020f7f74b2cb327597cec42eedc8ab62986a
         checksum/sc-dashboard-provider-config: 01ba4719c80b6fe911b091a7c05124b64eeece964e09c058ef8f9805daca546b
       labels:
@@ -49,6 +49,30 @@ spec:
               value: /var/lib/grafana/plugins
             - name: GF_PATHS_PROVISIONING
               value: /etc/grafana/provisioning
+            - name: ALERT_EMAIL_TO
+              valueFrom:
+                secretKeyRef:
+                  key: recipients
+                  name: grafana-smtp
+                  optional: true
+            - name: GF_SMTP_FROM_ADDRESS
+              valueFrom:
+                secretKeyRef:
+                  key: from-address
+                  name: grafana-smtp
+                  optional: true
+            - name: GF_SMTP_PASSWORD
+              valueFrom:
+                secretKeyRef:
+                  key: password
+                  name: grafana-smtp
+                  optional: true
+            - name: GF_SMTP_USER
+              valueFrom:
+                secretKeyRef:
+                  key: user
+                  name: grafana-smtp
+                  optional: true
           image: grafana/grafana:9.2.5
           imagePullPolicy: IfNotPresent
           livenessProbe:
@@ -76,6 +100,15 @@ spec:
             - mountPath: /etc/grafana/provisioning/datasources/datasources.yaml
               name: config
               subPath: datasources.yaml
+            - mountPath: /etc/grafana/provisioning/alerting/contactpoints.yaml
+              name: config
+              subPath: contactpoints.yaml
+            - mountPath: /etc/grafana/provisioning/alerting/policies.yaml
+              name: config
+              subPath: policies.yaml
+            - mountPath: /etc/grafana/provisioning/alerting/rules.yaml
+              name: config
+              subPath: rules.yaml
             - mountPath: /etc/grafana/provisioning/dashboards/dashboardproviders.yaml
               name: config
               subPath: dashboardproviders.yaml

grafana/SealedSecret/grafana-smtp.yaml

@@ -0,0 +1,20 @@
+apiVersion: bitnami.com/v1alpha1
+kind: SealedSecret
+metadata:
+  name: grafana-smtp
+  namespace: grafana
+spec:
+  encryptedData:
+    from-address: >-
+      AgBMPexC9kIlK+TFolqa2Bbw5u5JvfQ91biez6qHky39Em5pm+Gfg5RU8X83NJQYIoGtNp2mjeAQyWRsc+W7Xyj/vw+OOuJ/1K6iCzGS29gaQAMRjyChCMLNpf2dVhm9LNQlR4/aN22UP8fFTf0OqT3YAAFkKRd5ViCKOkXX6Sipwmv6cfiKJUqMK0LtngAKp1RUTh2M6wJISDvIkV+tyFUqrteQD+LOHNg79p13B/+rFG9L3+yXhltm2gSiDJXW4byDMN+pZWmW9XC8x/gmIBh57TXwPFvtUfM89vpJwBeuoJPpW5Rg2pjYS7zyGmL7RqKJmztRKqp3ilPTP0GR+ZC6uLDRpJ9afkHCsrcOsULF6VkadX34aPtF60uYWlnm3CRG9fY0Au29wfxo5VWfWeUVf8obpAXWvXbKjiNEhPf3BwLGuk3ytoaQTeoLAbi83fBnTAWA3VXuN+NF4OYzQUXBdI0rsMmaiD24W7pKWgtVAsX+R147JhkZSpzTstfd9PhEE5v36uHd0q71Y6MU6FF813GtrzD7wSkmdjmSs+h4zpK20LUjErto7c6lXhj8Uf+X1bcS5cBs2X293PovrjSQ384euzdLFG2P34Q6ls0L4bQtwPja4hQJQT0YcxIBZ6IVbG1NQeKKsYIgJTJ40GrwJpZzxwFjk1feBE/vwrrh+fkOawFCeEjKkYIp8IxV7OhV6S2E1EqteF4a3sQfGx8t34n6jRGMTrvZ3Q==
+    password: >-
+      AgCZUNFtDA4l+wTpsrMsfCgDyMgEDfBBKNifH+doemPgONdxs6aUQBdD02x/T7qWQCE20hit6S9aJh5JnGsxi+/wJ86QticchM9WUKdtwLWrZr0Lg7mCyNZYLRZo5oX+G/rCdYC975Bjgw4L86AT5bR+Y6lQ9Gj2OZnm743QMnOqXxNncUjFRJZRCRfTzSy2JUfKU70ZMO3Ddgh1C+BD1efmXzbdLfV6AJ+QW/SJLbOWKVDjJttTywYiPIi/kKWiKoXykVzCUOCkIJBOYgv5kvAUVo/UrX2BLAgHh6yTc5a6C1SUyEwiqhv6Dp/ukXQD2ruDcoWJHEDX7AsmvQXiO0ryFIUFXOsmaeHd5OYjmRU5W40FHbkMqWOUS/gN1wFR3RyFFSXZDFTULRaC4/RWBHUycW93ClmeqZPv/m2udu4MMco19967xLh4n+IAc6iTwkCOSg+eOOrTgt5GL/a+GsNG3vXY0ptSMH/rweG6KAgF8fIRXZlHhxFwe5p6p8RW3N4IkcZY1sDewMhQvxU6ZY/QAu/VAiD49Pz3gALMARmjUsHMQEA/ORYn9xJrIDd7Y0nKEqllZB25uQP6LcvLHqNKXkleKdSkhKUQpelTtrj+axYk0RVXTpaXTgCeD8xNza7eJ2nuQ/GRgs1LwvGD6CHQil3VoAa4qWrwP5zxTTEpBHUNVuR6LXhF7MlnUfVaBID0hRh6HHeoLonZ0983deHnyO1yft6y/D7vaXJbFqGl9dfF1RY=
+    recipients: >-
+      AgAnBHf//CL7+82ZZ7/uQiHBcCmzNcIrxPBfK0PKVYP2e68SZXusEqaW2QBSGv5ZZY0m/NQfcaMzHojcHtciPRTNOu+RAVyEF93w6pull74PMfyNwmZQZuwr7Yo6kiDUwFz3iS+fDUacJ/4jzgtF0LGLFw+DAQPryXcbdhQWG4jvs69hZZxp90o2VN72Q8kmlgwcy48/iJYqRXiWx7fbFX02Nnep+SEuYhkau7ilk3rXAchFRJsIikZ9g0RjBk5hLK7pJPIJQNlk0cgRXYulzHO6R445+kv4CMzgV0pJz5ZxOcse9d6NRjn6fnPK5PYm/WgkPGyuKHcVaNlDcSdtUMb+eSbCmcKAplR0aGNoSgPi23Ozu4i7PwwVv09mNq2k8uiJaVFzsctIeG0SdFyueQQuGyzhiqY/D9Wy9Z0LI9NUSBgOCRwCzl54dy5JFEPYFIUGhXQGlWlNiGHaYowbvIaxqy+F2gN+MYmW+yL/N2oCLv7MzfBovHshzv4t6S5DwYbVf/P5EqbUJsqDp2eXsDfKdJk/hsLAmVAVROCB9ma62lloqQ4kK7A2WeZq/zJichIe0QfG5LdCJC/Q6kzQMQSs6+gV89+OvwCMEZBYv1+uX94jH/dHZ7gqbh27u+7+bNNHy/84QZbk8ZPb4HgScOTU9vfekOWN4fJXxHD7GyJt0koz37yhmCxFLcXUuGmz2hVvJE+2KwD3xU9cGpW8
+    user: >-
+      AgCSO/1AEYzE0FzKL66pNCdgsA6ILAbruTa9etC2rXEhSsX64krJ8cr08pBHQ4W3SMIDW4otTABGCnSt5Shi5XToRx4+p2xLoadLxlCAvF8TTsiayKA9Dj/h1r+eRJ5B/XlXxLR/cir+HI1WWE8cpMz1+7PSIJKoC6jPkc0woh5RUtecxDFU78zzPEWoZxItli0WrBdvZAcBWwX+P5ErNv0qlZZoiCApQocTGMHb/U9etO8W0r4bqkto0w71i94Z5grjg68M0jHVnyytUdM84Hf+VF0cpW+XhulEJT3fGQgAN+LiUTrUiRx9TqHC1iWbEwFRCKl6/yY/72i5D4MnbnfOGa5R52hl2ewzDg/DKr3Uulu1rsX06M31yq2RqTib2WZwkSEE/3vhOLUx7MbrRBfBOURoCPJP3sW48hBYGnZmowULbrHs1ZyKx1ClWRJEqZdx1rWWy6fBhh+5OTfXbNGv5JgFKMflCkeNf4y0jv+Ck1hVG7Gc0Yook/AVquhecN77uRBho3WJ7/opv93HvNw+2/2/1NpAd+9To6pzBS01qZj946yo+QNtLWKl/RTVkCESGayLIGeH6V6fbq/xUluuF+EOEJm7ylNHTOtaUmF+fN8zCwiHVI+xRjqHIpjyu5/M3M1uII7BSg/1RzJXjKq10dk+eNUwA4mrroooFvTumqJbDcVkrxeRTArooYdORTP+t1ckCgdNA1f7986/1eK5cJK4RKjhnXP8pRO7VtKbVK3bBWA=
+  template:
+    metadata:
+      name: grafana-smtp
+      namespace: grafana
+    type: Opaque