feat(mon/victoria-metrics): add metrics, alerting and Grafana stack

kubernetes-monitoring/.sops.yaml

@@ -0,0 +1,4 @@
+creation_rules:
+   - path_regex: .*\.sops\.yaml
+     encrypted_regex: "^(data|stringData|email)$"
+     age: age1jetw4h63gq4uzyk08znynw4h852nexdmtkdz4kwxj5ew5st32vnspmtag2

kubernetes-monitoring/apps/ks.yaml

@@ -0,0 +1,17 @@
+---
+apiVersion: kustomize.toolkit.fluxcd.io/v1
+kind: Kustomization
+metadata:
+  name: apps
+  namespace: flux-system
+spec:
+  interval: 10m
+  path: ./kubernetes-monitoring/apps
+  prune: true
+  sourceRef:
+    kind: GitRepository
+    name: flux-system
+  decryption:
+    provider: sops
+    secretRef:
+      name: sops-age

kubernetes-monitoring/apps/victoria-metrics/namespace.yaml

@@ -0,0 +1,4 @@
+apiVersion: v1
+kind: Namespace
+metadata:
+  name: monitoring

kubernetes-monitoring/apps/victoria-metrics/release.yaml

@@ -0,0 +1,82 @@
+apiVersion: helm.toolkit.fluxcd.io/v2
+kind: HelmRelease
+metadata:
+  name: victoria-metrics-k8s-stack
+  namespace: flux-system
+spec:
+  chart:
+    spec:
+      chart: victoria-metrics-k8s-stack
+      version: 0.84.0
+      reconcileStrategy: ChartVersion
+      sourceRef:
+        kind: HelmRepository
+        name: victoria-metrics
+        namespace: flux-system
+  targetNamespace: monitoring
+  interval: 30m
+  install:
+    remediation:
+      retries: 3
+  upgrade:
+    remediation:
+      retries: 3
+  values:
+    # Operator bundles the VictoriaMetrics CRDs (plain manifests, applied by Flux SSA).
+    victoria-metrics-operator:
+      enabled: true
+      crds:
+        plain: true
+
+    # Single-node TSDB. NodePort so the in-cluster Talos vmagent can remote_write
+    # and we can reach it by IP even when DNS (pihole on Talos) is down.
+    vmsingle:
+      enabled: true
+      spec:
+        retentionPeriod: "12"        # months — VM is very space-efficient
+        storage:
+          accessModes:
+            - ReadWriteOnce
+          storageClassName: local-path
+          resources:
+            requests:
+              storage: 30Gi
+        serviceSpec:
+          useAsDefault: true
+          spec:
+            type: NodePort
+            ports:
+              - name: http
+                port: 8428
+                targetPort: 8428
+                protocol: TCP
+                nodePort: 30428
+
+    # Local scrape of the Pi/k3s itself (node-exporter, kube-state-metrics, kubelet).
+    vmagent:
+      enabled: true
+
+    vmalert:
+      enabled: true
+
+    # Telegram + email wired later (see monitoring-rpi5-setup.md step 11).
+    # Default config uses a blackhole receiver until then.
+    alertmanager:
+      enabled: true
+
+    grafana:
+      enabled: true
+      # Native Grafana login — independent of Authelia (which lives in Talos).
+      admin:
+        existingSecret: grafana-admin
+        userKey: admin-user
+        passwordKey: admin-password
+      persistence:
+        enabled: true
+        type: pvc
+        storageClassName: local-path
+        size: 5Gi
+      # NodePort so Grafana is reachable by IP without depending on DNS.
+      service:
+        type: NodePort
+        nodePort: 30300

kubernetes-monitoring/apps/victoria-metrics/repository.yaml

@@ -0,0 +1,8 @@
+apiVersion: source.toolkit.fluxcd.io/v1
+kind: HelmRepository
+metadata:
+  name: victoria-metrics
+  namespace: flux-system
+spec:
+  interval: 1h
+  url: https://victoriametrics.github.io/helm-charts/

kubernetes-monitoring/apps/victoria-metrics/secret-grafana.sops.yaml

@@ -0,0 +1,23 @@
+apiVersion: v1
+kind: Secret
+metadata:
+    name: grafana-admin
+    namespace: monitoring
+stringData:
+    admin-user: ENC[AES256_GCM,data:KDBvi5I=,iv:tNHZ9IABD5l210PTjKhN+q4vMnmJLAP99fzzONpwwxg=,tag:YU62qttxO95+P1VlpVn0qA==,type:str]
+    admin-password: ENC[AES256_GCM,data:HjZYVYs2t6Oml52fjFS2PymFvp1GXRAJXCewxA==,iv:LK1Yf0teLPITo2pK4lAaeub3Ieyl90uRrZeBD3LXy2o=,tag:7I8fT/SSzU7JkcreNlI8rw==,type:str]
+sops:
+    age:
+        - enc: |
+            -----BEGIN AGE ENCRYPTED FILE-----
+            YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBobmoydk9FaHFkQXEzQU03
+            YnZ4RjlxVk40dkk5RGFya0Foeit1dEpWRkdZCmJ4M3pxcHl1S1FUTnlGbEtEeXJT
+            ZDN4c0dPYktwTXgveExQUWFsYlEwTmsKLS0tIElmTDNCM2dRR3VtRGRpeHVCSXVl
+            ckV1N3h1bmlOdjhnRXRZNExJank5Q2MKMsA6BkdzOrTQUN0uPSUQSa9aYLil2Chb
+            vjpFp1j6x9EwvKHLY7mry15ze7gjUtUt8e97zO9ZlturiPigl0jeoQ==
+            -----END AGE ENCRYPTED FILE-----
+          recipient: age1jetw4h63gq4uzyk08znynw4h852nexdmtkdz4kwxj5ew5st32vnspmtag2
+    encrypted_regex: ^(data|stringData|email)$
+    lastmodified: "2026-06-19T11:46:25Z"
+    mac: ENC[AES256_GCM,data:h7Bei7jgHb4dFIk6Vax8fL7uA0zlqLUtpQgYTNx7i+yssAXcKEr989WgbLqQY8Rx1BSj27rFHv2Y4P+K2ump+J930lZIe1NNBNiFnYbI2vN0TYrlVOAETk6c0DoqygG7Mmlq+kCGGml4A+4XuQcCA4ZYzBYE4QvGq/WS6bOCFXI=,iv:zE3GhfkvJl5Z4klGWHgw7iZmxOff+Mlf7aXtMWTKUEs=,tag:q0mxhLxK2SIujVa+mhiKVw==,type:str]
+    version: 3.13.1

kubernetes-monitoring/kustomization.yaml

@@ -0,0 +1,5 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+resources:
+- flux-system
+- apps/ks.yaml