From 6002b003a0135fe169e273e15545d6585ff67916 Mon Sep 17 00:00:00 2001 From: Oleksandr Berezovskyi Date: Sat, 3 Oct 2026 23:08:04 +0300 Subject: [PATCH] feat(mon/alerting): route alerts to Telegram, add gateway and syslog rules --- .../apps/monitoring/release-logs.yaml | 3 ++ .../apps/monitoring/release.yaml | 43 ++++++++++++++++++- .../secret-alertmanager-telegram.sops.yaml | 24 +++++++++++ .../apps/monitoring/vmrule-homelab.yaml | 36 ++++++++++++++++ 4 files changed, 104 insertions(+), 2 deletions(-) create mode 100644 kubernetes-monitoring/apps/monitoring/secret-alertmanager-telegram.sops.yaml create mode 100644 kubernetes-monitoring/apps/monitoring/vmrule-homelab.yaml diff --git a/kubernetes-monitoring/apps/monitoring/release-logs.yaml b/kubernetes-monitoring/apps/monitoring/release-logs.yaml index 4fb1d18..54f5e00 100644 --- a/kubernetes-monitoring/apps/monitoring/release-logs.yaml +++ b/kubernetes-monitoring/apps/monitoring/release-logs.yaml @@ -41,6 +41,9 @@ spec: value: ":514" useRemoteIP: true useLocalTimestamp: true + # Scraped by the stack's vmagent so ingestion stalls can alert. + vmServiceScrape: + enabled: true resources: requests: cpu: 50m diff --git a/kubernetes-monitoring/apps/monitoring/release.yaml b/kubernetes-monitoring/apps/monitoring/release.yaml index 08f1f0a..0a1795e 100644 --- a/kubernetes-monitoring/apps/monitoring/release.yaml +++ b/kubernetes-monitoring/apps/monitoring/release.yaml @@ -59,10 +59,49 @@ spec: vmalert: enabled: true - # Telegram + email wired later (see monitoring-rpi5-setup.md step 11). - # Default config uses a blackhole receiver until then. + # k3s runs scheduler/controller-manager in one binary with no separate + # scrape endpoints, and uses kine instead of etcd — disabling these drops + # the empty scrape pools and their always-firing rules. + kubeScheduler: + enabled: false + kubeControllerManager: + enabled: false + kubeEtcd: + enabled: false + alertmanager: enabled: true + spec: + # Mounted at /etc/vm/secrets/alertmanager-telegram/ (bot_token, chat_id). + secrets: + - alertmanager-telegram + config: + route: + receiver: telegram + group_by: ["alertname", "job"] + group_wait: 30s + group_interval: 5m + repeat_interval: 12h + routes: + - receiver: blackhole + matchers: + - alertname=~"Watchdog|InfoInhibitor" + - receiver: blackhole + matchers: + - severity="info" + inhibit_rules: + - source_matchers: + - severity="critical" + target_matchers: + - severity="warning" + equal: ["alertname", "job"] + receivers: + - name: blackhole + - name: telegram + telegram_configs: + - bot_token_file: /etc/vm/secrets/alertmanager-telegram/bot_token + chat_id_file: /etc/vm/secrets/alertmanager-telegram/chat_id + send_resolved: true grafana: enabled: true diff --git a/kubernetes-monitoring/apps/monitoring/secret-alertmanager-telegram.sops.yaml b/kubernetes-monitoring/apps/monitoring/secret-alertmanager-telegram.sops.yaml new file mode 100644 index 0000000..b9e7061 --- /dev/null +++ b/kubernetes-monitoring/apps/monitoring/secret-alertmanager-telegram.sops.yaml @@ -0,0 +1,24 @@ +apiVersion: v1 +kind: Secret +metadata: + name: alertmanager-telegram + namespace: monitoring +type: Opaque +stringData: + bot_token: ENC[AES256_GCM,data:V/SDGyp/M7Qqa22fiKYMZ2CzkivNBl24hv9cVEFPl50NySKY0jqIPnVWLtm11g==,iv:UfHMfvu9b2CaXhImng5OLVej13zHrptKQOaPUbjXiw0=,tag:SFAGUaM0XYvbVc1CRrJepQ==,type:str] + chat_id: ENC[AES256_GCM,data:HRDHrCMMurk=,iv:hP7UB23qWZ+YukoP3B7QolUIfv2FgJtTfM1OLe/nMMY=,tag:JTwjQwDLQaG5LfKQGV73iw==,type:str] +sops: + age: + - enc: | + -----BEGIN AGE ENCRYPTED FILE----- + YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBwNXF4V1N3bmpxbWN5YUVR + UDNNdm1IL2hHWGgzRENvK0xWdkhFc1drRGhBCjdQc3hydG1RN3ZocUZnZGpKRG5i + K1RXUTdiZ29hWWdjLzZVMCtMb1NZbjQKLS0tIHpDcTNDeFpJUU5CeU5PSFNWWnBo + UmJNT1hVL0h3c3hWOGQxM21uWk14amsKIyFxmszp94FteSUXyAh4hrDNbPBja4dT + QL6wFw8GkYtFeucIgnvbjHWWi98QMwE1xEt930uMeNyRnLFqFfUtWw== + -----END AGE ENCRYPTED FILE----- + recipient: age1jetw4h63gq4uzyk08znynw4h852nexdmtkdz4kwxj5ew5st32vnspmtag2 + encrypted_regex: ^(data|stringData|email)$ + lastmodified: "2026-10-03T20:07:21Z" + mac: ENC[AES256_GCM,data:k7rBxhWrC2DFvFcoB6Wqufzk6vLtWFypITl/JK2qd6Vj9m7XKBZdiH5mCSDNfo4UK+ygFnGQ8oidkgcCrbGYqKFK8Q67+LQ9o1OMOXcZDkz6UzNRaLh3aO5NqbdleV36cICX6qpG1uO6GL6p5g121vRztQT+Qg1L0BkIMrF+sJQ=,iv:y8figgbzUg1gf0zug534Fvd9jdB00JiA7LoWXUm3/Lw=,tag:ccP3IKdWwMIJqP0s/U7NQA==,type:str] + version: 3.13.3 diff --git a/kubernetes-monitoring/apps/monitoring/vmrule-homelab.yaml b/kubernetes-monitoring/apps/monitoring/vmrule-homelab.yaml new file mode 100644 index 0000000..69d822d --- /dev/null +++ b/kubernetes-monitoring/apps/monitoring/vmrule-homelab.yaml @@ -0,0 +1,36 @@ +apiVersion: operator.victoriametrics.com/v1beta1 +kind: VMRule +metadata: + name: homelab + namespace: monitoring +spec: + groups: + - name: unifi-gateway + rules: + # Baseline load is 2–5; the 2026-09-26 memory-pressure incident sat at 15–25. + - alert: UnifiGatewayHighLoad + expr: avg_over_time(unpoller_device_load_average_1{type="udm"}[15m]) > 10 + for: 15m + labels: + severity: warning + annotations: + summary: "UniFi gateway load is {{ $value | humanize }} (15m avg)" + # Baseline is 84–92% and creeps up ~1%/3 days until something restarts. + - alert: UnifiGatewayMemoryHigh + expr: avg_over_time(unpoller_device_memory_utilization_ratio{type="udm"}[30m]) > 0.93 + for: 30m + labels: + severity: warning + annotations: + summary: "UniFi gateway memory at {{ $value | humanizePercentage }} (30m avg)" + - name: victoria-logs + rules: + # The gateway alone logs DHCP chatter every few minutes, so zero rows + # means the syslog path is broken (e.g. the 2026-09-22 IP change). + - alert: SyslogIngestionStalled + expr: sum(rate(vl_rows_ingested_total{type="syslog_udp"}[30m])) == 0 + for: 15m + labels: + severity: warning + annotations: + summary: "VictoriaLogs has received no syslog for 45m"