feat(mon/alerting): route alerts to Telegram, add gateway and syslog rules

kubernetes-monitoring/apps/monitoring/release-logs.yaml

@@ -41,6 +41,9 @@ spec:
             value: ":514"
             useRemoteIP: true
             useLocalTimestamp: true
+      # Scraped by the stack's vmagent so ingestion stalls can alert.
+      vmServiceScrape:
+        enabled: true
       resources:
         requests:
           cpu: 50m

kubernetes-monitoring/apps/monitoring/release.yaml

@@ -59,10 +59,49 @@ spec:
     vmalert:
       enabled: true
 
-    # Telegram + email wired later (see monitoring-rpi5-setup.md step 11).
-    # Default config uses a blackhole receiver until then.
+    # k3s runs scheduler/controller-manager in one binary with no separate
+    # scrape endpoints, and uses kine instead of etcd — disabling these drops
+    # the empty scrape pools and their always-firing rules.
+    kubeScheduler:
+      enabled: false
+    kubeControllerManager:
+      enabled: false
+    kubeEtcd:
+      enabled: false
+
     alertmanager:
       enabled: true
+      spec:
+        # Mounted at /etc/vm/secrets/alertmanager-telegram/ (bot_token, chat_id).
+        secrets:
+          - alertmanager-telegram
+      config:
+        route:
+          receiver: telegram
+          group_by: ["alertname", "job"]
+          group_wait: 30s
+          group_interval: 5m
+          repeat_interval: 12h
+          routes:
+            - receiver: blackhole
+              matchers:
+                - alertname=~"Watchdog|InfoInhibitor"
+            - receiver: blackhole
+              matchers:
+                - severity="info"
+        inhibit_rules:
+          - source_matchers:
+              - severity="critical"
+            target_matchers:
+              - severity="warning"
+            equal: ["alertname", "job"]
+        receivers:
+          - name: blackhole
+          - name: telegram
+            telegram_configs:
+              - bot_token_file: /etc/vm/secrets/alertmanager-telegram/bot_token
+                chat_id_file: /etc/vm/secrets/alertmanager-telegram/chat_id
+                send_resolved: true
 
     grafana:
       enabled: true

kubernetes-monitoring/apps/monitoring/secret-alertmanager-telegram.sops.yaml

@@ -0,0 +1,24 @@
+apiVersion: v1
+kind: Secret
+metadata:
+    name: alertmanager-telegram
+    namespace: monitoring
+type: Opaque
+stringData:
+    bot_token: ENC[AES256_GCM,data:V/SDGyp/M7Qqa22fiKYMZ2CzkivNBl24hv9cVEFPl50NySKY0jqIPnVWLtm11g==,iv:UfHMfvu9b2CaXhImng5OLVej13zHrptKQOaPUbjXiw0=,tag:SFAGUaM0XYvbVc1CRrJepQ==,type:str]
+    chat_id: ENC[AES256_GCM,data:HRDHrCMMurk=,iv:hP7UB23qWZ+YukoP3B7QolUIfv2FgJtTfM1OLe/nMMY=,tag:JTwjQwDLQaG5LfKQGV73iw==,type:str]
+sops:
+    age:
+        - enc: |
+            -----BEGIN AGE ENCRYPTED FILE-----
+            YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBwNXF4V1N3bmpxbWN5YUVR
+            UDNNdm1IL2hHWGgzRENvK0xWdkhFc1drRGhBCjdQc3hydG1RN3ZocUZnZGpKRG5i
+            K1RXUTdiZ29hWWdjLzZVMCtMb1NZbjQKLS0tIHpDcTNDeFpJUU5CeU5PSFNWWnBo
+            UmJNT1hVL0h3c3hWOGQxM21uWk14amsKIyFxmszp94FteSUXyAh4hrDNbPBja4dT
+            QL6wFw8GkYtFeucIgnvbjHWWi98QMwE1xEt930uMeNyRnLFqFfUtWw==
+            -----END AGE ENCRYPTED FILE-----
+          recipient: age1jetw4h63gq4uzyk08znynw4h852nexdmtkdz4kwxj5ew5st32vnspmtag2
+    encrypted_regex: ^(data|stringData|email)$
+    lastmodified: "2026-10-03T20:07:21Z"
+    mac: ENC[AES256_GCM,data:k7rBxhWrC2DFvFcoB6Wqufzk6vLtWFypITl/JK2qd6Vj9m7XKBZdiH5mCSDNfo4UK+ygFnGQ8oidkgcCrbGYqKFK8Q67+LQ9o1OMOXcZDkz6UzNRaLh3aO5NqbdleV36cICX6qpG1uO6GL6p5g121vRztQT+Qg1L0BkIMrF+sJQ=,iv:y8figgbzUg1gf0zug534Fvd9jdB00JiA7LoWXUm3/Lw=,tag:ccP3IKdWwMIJqP0s/U7NQA==,type:str]
+    version: 3.13.3

kubernetes-monitoring/apps/monitoring/vmrule-homelab.yaml

@@ -0,0 +1,36 @@
+apiVersion: operator.victoriametrics.com/v1beta1
+kind: VMRule
+metadata:
+  name: homelab
+  namespace: monitoring
+spec:
+  groups:
+    - name: unifi-gateway
+      rules:
+        # Baseline load is 2–5; the 2026-09-26 memory-pressure incident sat at 15–25.
+        - alert: UnifiGatewayHighLoad
+          expr: avg_over_time(unpoller_device_load_average_1{type="udm"}[15m]) > 10
+          for: 15m
+          labels:
+            severity: warning
+          annotations:
+            summary: "UniFi gateway load is {{ $value | humanize }} (15m avg)"
+        # Baseline is 84–92% and creeps up ~1%/3 days until something restarts.
+        - alert: UnifiGatewayMemoryHigh
+          expr: avg_over_time(unpoller_device_memory_utilization_ratio{type="udm"}[30m]) > 0.93
+          for: 30m
+          labels:
+            severity: warning
+          annotations:
+            summary: "UniFi gateway memory at {{ $value | humanizePercentage }} (30m avg)"
+    - name: victoria-logs
+      rules:
+        # The gateway alone logs DHCP chatter every few minutes, so zero rows
+        # means the syslog path is broken (e.g. the 2026-09-22 IP change).
+        - alert: SyslogIngestionStalled
+          expr: sum(rate(vl_rows_ingested_total{type="syslog_udp"}[30m])) == 0
+          for: 15m
+          labels:
+            severity: warning
+          annotations:
+            summary: "VictoriaLogs has received no syslog for 45m"