# Cluster monitoring, shipped to the homelab Grafana stack on LXC 112 (10.0.0.22): # Prometheus (remote write, :9090) and Loki (:3100). Nothing is stored in the cluster. # - kube-state-metrics: object state (deployments, pods, PVCs, restarts…) # - node-exporter: one per node, masters included # - Alloy (1 replica): scrapes the two above + kubelet/cAdvisor through the API # server, tails every pod's logs, and pushes everything to LXC 112. apiVersion: argoproj.io/v1alpha1 kind: Application metadata: name: kube-state-metrics namespace: argocd spec: project: default source: repoURL: https://prometheus-community.github.io/helm-charts chart: kube-state-metrics targetRevision: 8.7.1 helm: valuesObject: fullnameOverride: kube-state-metrics destination: server: https://kubernetes.default.svc namespace: monitoring syncPolicy: automated: prune: true selfHeal: true syncOptions: - CreateNamespace=true --- apiVersion: argoproj.io/v1alpha1 kind: Application metadata: name: node-exporter namespace: argocd spec: project: default source: repoURL: https://prometheus-community.github.io/helm-charts chart: prometheus-node-exporter targetRevision: 4.60.1 helm: valuesObject: fullnameOverride: node-exporter # Run on the tainted masters too. tolerations: - operator: Exists destination: server: https://kubernetes.default.svc namespace: monitoring syncPolicy: automated: prune: true selfHeal: true syncOptions: - CreateNamespace=true --- apiVersion: argoproj.io/v1alpha1 kind: Application metadata: name: alloy namespace: argocd spec: project: default source: repoURL: https://grafana.github.io/helm-charts chart: alloy targetRevision: 1.14.3 helm: valuesObject: fullnameOverride: alloy controller: type: deployment replicas: 1 alloy: resources: requests: { cpu: 100m, memory: 256Mi } limits: { memory: 1Gi } configMap: content: | // Every series gets cluster="k3s" so it can't be mixed up with the // Proxmox node_exporters in Prometheus. prometheus.remote_write "homelab" { endpoint { url = "http://10.0.0.22:9090/api/v1/write" } external_labels = { cluster = "k3s" } } loki.write "homelab" { endpoint { url = "http://10.0.0.22:3100/loki/api/v1/push" } external_labels = { cluster = "k3s", job = "k3s" } } discovery.kubernetes "nodes" { role = "node" } discovery.kubernetes "pods" { role = "pod" } // kubelet and cAdvisor, through the API server proxy discovery.relabel "kubelet" { targets = discovery.kubernetes.nodes.targets rule { target_label = "__address__" replacement = "kubernetes.default.svc:443" } rule { source_labels = ["__meta_kubernetes_node_name"] target_label = "node" } rule { source_labels = ["__meta_kubernetes_node_name"] target_label = "__metrics_path__" replacement = "/api/v1/nodes/$1/proxy/metrics" } } discovery.relabel "cadvisor" { targets = discovery.relabel.kubelet.output rule { source_labels = ["node"] target_label = "__metrics_path__" replacement = "/api/v1/nodes/$1/proxy/metrics/cadvisor" } } prometheus.scrape "kubelet" { job_name = "kubelet" targets = discovery.relabel.kubelet.output scheme = "https" scrape_interval = "30s" bearer_token_file = "/var/run/secrets/kubernetes.io/serviceaccount/token" tls_config { ca_file = "/var/run/secrets/kubernetes.io/serviceaccount/ca.crt" } forward_to = [prometheus.remote_write.homelab.receiver] } prometheus.scrape "cadvisor" { job_name = "cadvisor" targets = discovery.relabel.cadvisor.output scheme = "https" scrape_interval = "30s" bearer_token_file = "/var/run/secrets/kubernetes.io/serviceaccount/token" tls_config { ca_file = "/var/run/secrets/kubernetes.io/serviceaccount/ca.crt" } forward_to = [prometheus.remote_write.homelab.receiver] } // node-exporter pods (hostNetwork, so the pod IP is the node IP) discovery.relabel "node_exporter" { targets = discovery.kubernetes.pods.targets rule { source_labels = ["__meta_kubernetes_pod_label_app_kubernetes_io_name"] regex = "prometheus-node-exporter" action = "keep" } rule { source_labels = ["__meta_kubernetes_pod_container_port_number"] regex = "9100" action = "keep" } rule { source_labels = ["__meta_kubernetes_pod_node_name"] target_label = "instance" } } prometheus.scrape "node_exporter" { job_name = "k3s-node" targets = discovery.relabel.node_exporter.output scrape_interval = "30s" forward_to = [prometheus.remote_write.homelab.receiver] } prometheus.scrape "kube_state_metrics" { job_name = "kube-state-metrics" targets = [{ __address__ = "kube-state-metrics.monitoring.svc:8080" }] scrape_interval = "30s" forward_to = [prometheus.remote_write.homelab.receiver] } // Pod logs, read through the API (no host mounts needed) discovery.relabel "pod_logs" { targets = discovery.kubernetes.pods.targets rule { source_labels = ["__meta_kubernetes_namespace"] target_label = "namespace" } rule { source_labels = ["__meta_kubernetes_pod_name"] target_label = "pod" } rule { source_labels = ["__meta_kubernetes_pod_container_name"] target_label = "container" } rule { source_labels = ["__meta_kubernetes_pod_node_name"] target_label = "host" } rule { source_labels = ["__meta_kubernetes_pod_label_app_kubernetes_io_name", "__meta_kubernetes_pod_label_app"] regex = "^;*([^;]+)(;.*)?$" target_label = "app" } } loki.source.kubernetes "pods" { targets = discovery.relabel.pod_logs.output forward_to = [loki.write.homelab.receiver] } destination: server: https://kubernetes.default.svc namespace: monitoring syncPolicy: automated: prune: true selfHeal: true syncOptions: - CreateNamespace=true