monitoring: kube-state-metrics, node-exporter and Alloy shipping to LXC 112

Metrics go to Prometheus on 10.0.0.22:9090 by remote write (cluster=k3s),
pod logs to Loki on 10.0.0.22:3100.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
un-oeil 2026-10-25 23:20:23 +01:00
parent cceeb85af4
commit 8983fe295c

227
cluster/monitoring.yaml Normal file
View file

@ -0,0 +1,227 @@
# Cluster monitoring, shipped to the homelab Grafana stack on LXC 112 (10.0.0.22):
# Prometheus (remote write, :9090) and Loki (:3100). Nothing is stored in the cluster.
# - kube-state-metrics: object state (deployments, pods, PVCs, restarts…)
# - node-exporter: one per node, masters included
# - Alloy (1 replica): scrapes the two above + kubelet/cAdvisor through the API
# server, tails every pod's logs, and pushes everything to LXC 112.
apiVersion: argoproj.io/v1alpha1
kind: Application
metadata:
name: kube-state-metrics
namespace: argocd
spec:
project: default
source:
repoURL: https://prometheus-community.github.io/helm-charts
chart: kube-state-metrics
targetRevision: 8.7.1
helm:
valuesObject:
fullnameOverride: kube-state-metrics
destination:
server: https://kubernetes.default.svc
namespace: monitoring
syncPolicy:
automated:
prune: true
selfHeal: true
syncOptions:
- CreateNamespace=true
---
apiVersion: argoproj.io/v1alpha1
kind: Application
metadata:
name: node-exporter
namespace: argocd
spec:
project: default
source:
repoURL: https://prometheus-community.github.io/helm-charts
chart: prometheus-node-exporter
targetRevision: 4.60.1
helm:
valuesObject:
fullnameOverride: node-exporter
# Run on the tainted masters too.
tolerations:
- operator: Exists
destination:
server: https://kubernetes.default.svc
namespace: monitoring
syncPolicy:
automated:
prune: true
selfHeal: true
syncOptions:
- CreateNamespace=true
---
apiVersion: argoproj.io/v1alpha1
kind: Application
metadata:
name: alloy
namespace: argocd
spec:
project: default
source:
repoURL: https://grafana.github.io/helm-charts
chart: alloy
targetRevision: 1.14.3
helm:
valuesObject:
fullnameOverride: alloy
controller:
type: deployment
replicas: 1
alloy:
resources:
requests: { cpu: 100m, memory: 256Mi }
limits: { memory: 1Gi }
configMap:
content: |
// Every series gets cluster="k3s" so it can't be mixed up with the
// Proxmox node_exporters in Prometheus.
prometheus.remote_write "homelab" {
endpoint {
url = "http://10.0.0.22:9090/api/v1/write"
}
external_labels = { cluster = "k3s" }
}
loki.write "homelab" {
endpoint {
url = "http://10.0.0.22:3100/loki/api/v1/push"
}
external_labels = { cluster = "k3s", job = "k3s" }
}
discovery.kubernetes "nodes" {
role = "node"
}
discovery.kubernetes "pods" {
role = "pod"
}
// kubelet and cAdvisor, through the API server proxy
discovery.relabel "kubelet" {
targets = discovery.kubernetes.nodes.targets
rule {
target_label = "__address__"
replacement = "kubernetes.default.svc:443"
}
rule {
source_labels = ["__meta_kubernetes_node_name"]
target_label = "node"
}
rule {
source_labels = ["__meta_kubernetes_node_name"]
target_label = "__metrics_path__"
replacement = "/api/v1/nodes/$1/proxy/metrics"
}
}
discovery.relabel "cadvisor" {
targets = discovery.relabel.kubelet.output
rule {
source_labels = ["node"]
target_label = "__metrics_path__"
replacement = "/api/v1/nodes/$1/proxy/metrics/cadvisor"
}
}
prometheus.scrape "kubelet" {
job_name = "kubelet"
targets = discovery.relabel.kubelet.output
scheme = "https"
scrape_interval = "30s"
bearer_token_file = "/var/run/secrets/kubernetes.io/serviceaccount/token"
tls_config {
ca_file = "/var/run/secrets/kubernetes.io/serviceaccount/ca.crt"
}
forward_to = [prometheus.remote_write.homelab.receiver]
}
prometheus.scrape "cadvisor" {
job_name = "cadvisor"
targets = discovery.relabel.cadvisor.output
scheme = "https"
scrape_interval = "30s"
bearer_token_file = "/var/run/secrets/kubernetes.io/serviceaccount/token"
tls_config {
ca_file = "/var/run/secrets/kubernetes.io/serviceaccount/ca.crt"
}
forward_to = [prometheus.remote_write.homelab.receiver]
}
// node-exporter pods (hostNetwork, so the pod IP is the node IP)
discovery.relabel "node_exporter" {
targets = discovery.kubernetes.pods.targets
rule {
source_labels = ["__meta_kubernetes_pod_label_app_kubernetes_io_name"]
regex = "prometheus-node-exporter"
action = "keep"
}
rule {
source_labels = ["__meta_kubernetes_pod_container_port_number"]
regex = "9100"
action = "keep"
}
rule {
source_labels = ["__meta_kubernetes_pod_node_name"]
target_label = "instance"
}
}
prometheus.scrape "node_exporter" {
job_name = "k3s-node"
targets = discovery.relabel.node_exporter.output
scrape_interval = "30s"
forward_to = [prometheus.remote_write.homelab.receiver]
}
prometheus.scrape "kube_state_metrics" {
job_name = "kube-state-metrics"
targets = [{ __address__ = "kube-state-metrics.monitoring.svc:8080" }]
scrape_interval = "30s"
forward_to = [prometheus.remote_write.homelab.receiver]
}
// Pod logs, read through the API (no host mounts needed)
discovery.relabel "pod_logs" {
targets = discovery.kubernetes.pods.targets
rule {
source_labels = ["__meta_kubernetes_namespace"]
target_label = "namespace"
}
rule {
source_labels = ["__meta_kubernetes_pod_name"]
target_label = "pod"
}
rule {
source_labels = ["__meta_kubernetes_pod_container_name"]
target_label = "container"
}
rule {
source_labels = ["__meta_kubernetes_pod_node_name"]
target_label = "host"
}
rule {
source_labels = ["__meta_kubernetes_pod_label_app_kubernetes_io_name", "__meta_kubernetes_pod_label_app"]
regex = "^;*([^;]+)(;.*)?$"
target_label = "app"
}
}
loki.source.kubernetes "pods" {
targets = discovery.relabel.pod_logs.output
forward_to = [loki.write.homelab.receiver]
}
destination:
server: https://kubernetes.default.svc
namespace: monitoring
syncPolicy:
automated:
prune: true
selfHeal: true
syncOptions:
- CreateNamespace=true