diff --git a/cluster/monitoring.yaml b/cluster/monitoring.yaml new file mode 100644 index 0000000..5cc5c8c --- /dev/null +++ b/cluster/monitoring.yaml @@ -0,0 +1,227 @@ +# Cluster monitoring, shipped to the homelab Grafana stack on LXC 112 (10.0.0.22): +# Prometheus (remote write, :9090) and Loki (:3100). Nothing is stored in the cluster. +# - kube-state-metrics: object state (deployments, pods, PVCs, restarts…) +# - node-exporter: one per node, masters included +# - Alloy (1 replica): scrapes the two above + kubelet/cAdvisor through the API +# server, tails every pod's logs, and pushes everything to LXC 112. +apiVersion: argoproj.io/v1alpha1 +kind: Application +metadata: + name: kube-state-metrics + namespace: argocd +spec: + project: default + source: + repoURL: https://prometheus-community.github.io/helm-charts + chart: kube-state-metrics + targetRevision: 8.7.1 + helm: + valuesObject: + fullnameOverride: kube-state-metrics + destination: + server: https://kubernetes.default.svc + namespace: monitoring + syncPolicy: + automated: + prune: true + selfHeal: true + syncOptions: + - CreateNamespace=true +--- +apiVersion: argoproj.io/v1alpha1 +kind: Application +metadata: + name: node-exporter + namespace: argocd +spec: + project: default + source: + repoURL: https://prometheus-community.github.io/helm-charts + chart: prometheus-node-exporter + targetRevision: 4.60.1 + helm: + valuesObject: + fullnameOverride: node-exporter + # Run on the tainted masters too. + tolerations: + - operator: Exists + destination: + server: https://kubernetes.default.svc + namespace: monitoring + syncPolicy: + automated: + prune: true + selfHeal: true + syncOptions: + - CreateNamespace=true +--- +apiVersion: argoproj.io/v1alpha1 +kind: Application +metadata: + name: alloy + namespace: argocd +spec: + project: default + source: + repoURL: https://grafana.github.io/helm-charts + chart: alloy + targetRevision: 1.14.3 + helm: + valuesObject: + fullnameOverride: alloy + controller: + type: deployment + replicas: 1 + alloy: + resources: + requests: { cpu: 100m, memory: 256Mi } + limits: { memory: 1Gi } + configMap: + content: | + // Every series gets cluster="k3s" so it can't be mixed up with the + // Proxmox node_exporters in Prometheus. + prometheus.remote_write "homelab" { + endpoint { + url = "http://10.0.0.22:9090/api/v1/write" + } + external_labels = { cluster = "k3s" } + } + + loki.write "homelab" { + endpoint { + url = "http://10.0.0.22:3100/loki/api/v1/push" + } + external_labels = { cluster = "k3s", job = "k3s" } + } + + discovery.kubernetes "nodes" { + role = "node" + } + + discovery.kubernetes "pods" { + role = "pod" + } + + // kubelet and cAdvisor, through the API server proxy + discovery.relabel "kubelet" { + targets = discovery.kubernetes.nodes.targets + rule { + target_label = "__address__" + replacement = "kubernetes.default.svc:443" + } + rule { + source_labels = ["__meta_kubernetes_node_name"] + target_label = "node" + } + rule { + source_labels = ["__meta_kubernetes_node_name"] + target_label = "__metrics_path__" + replacement = "/api/v1/nodes/$1/proxy/metrics" + } + } + + discovery.relabel "cadvisor" { + targets = discovery.relabel.kubelet.output + rule { + source_labels = ["node"] + target_label = "__metrics_path__" + replacement = "/api/v1/nodes/$1/proxy/metrics/cadvisor" + } + } + + prometheus.scrape "kubelet" { + job_name = "kubelet" + targets = discovery.relabel.kubelet.output + scheme = "https" + scrape_interval = "30s" + bearer_token_file = "/var/run/secrets/kubernetes.io/serviceaccount/token" + tls_config { + ca_file = "/var/run/secrets/kubernetes.io/serviceaccount/ca.crt" + } + forward_to = [prometheus.remote_write.homelab.receiver] + } + + prometheus.scrape "cadvisor" { + job_name = "cadvisor" + targets = discovery.relabel.cadvisor.output + scheme = "https" + scrape_interval = "30s" + bearer_token_file = "/var/run/secrets/kubernetes.io/serviceaccount/token" + tls_config { + ca_file = "/var/run/secrets/kubernetes.io/serviceaccount/ca.crt" + } + forward_to = [prometheus.remote_write.homelab.receiver] + } + + // node-exporter pods (hostNetwork, so the pod IP is the node IP) + discovery.relabel "node_exporter" { + targets = discovery.kubernetes.pods.targets + rule { + source_labels = ["__meta_kubernetes_pod_label_app_kubernetes_io_name"] + regex = "prometheus-node-exporter" + action = "keep" + } + rule { + source_labels = ["__meta_kubernetes_pod_container_port_number"] + regex = "9100" + action = "keep" + } + rule { + source_labels = ["__meta_kubernetes_pod_node_name"] + target_label = "instance" + } + } + + prometheus.scrape "node_exporter" { + job_name = "k3s-node" + targets = discovery.relabel.node_exporter.output + scrape_interval = "30s" + forward_to = [prometheus.remote_write.homelab.receiver] + } + + prometheus.scrape "kube_state_metrics" { + job_name = "kube-state-metrics" + targets = [{ __address__ = "kube-state-metrics.monitoring.svc:8080" }] + scrape_interval = "30s" + forward_to = [prometheus.remote_write.homelab.receiver] + } + + // Pod logs, read through the API (no host mounts needed) + discovery.relabel "pod_logs" { + targets = discovery.kubernetes.pods.targets + rule { + source_labels = ["__meta_kubernetes_namespace"] + target_label = "namespace" + } + rule { + source_labels = ["__meta_kubernetes_pod_name"] + target_label = "pod" + } + rule { + source_labels = ["__meta_kubernetes_pod_container_name"] + target_label = "container" + } + rule { + source_labels = ["__meta_kubernetes_pod_node_name"] + target_label = "host" + } + rule { + source_labels = ["__meta_kubernetes_pod_label_app_kubernetes_io_name", "__meta_kubernetes_pod_label_app"] + regex = "^;*([^;]+)(;.*)?$" + target_label = "app" + } + } + + loki.source.kubernetes "pods" { + targets = discovery.relabel.pod_logs.output + forward_to = [loki.write.homelab.receiver] + } + destination: + server: https://kubernetes.default.svc + namespace: monitoring + syncPolicy: + automated: + prune: true + selfHeal: true + syncOptions: + - CreateNamespace=true