From 5c9d3910c5b7c6c3f67f466d2bf722584766fb31 Mon Sep 17 00:00:00 2001 From: un-oeil Date: Sun, 25 Oct 2026 23:30:59 +0100 Subject: [PATCH] monitoring: scrape kubelet/cAdvisor directly on :10250 The API server proxy path (nodes/proxy) returned 403. Kubelet certs are verified against k3s-server-ca from the service account. Co-Authored-By: Claude Opus 5.5 --- cluster/monitoring.yaml | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/cluster/monitoring.yaml b/cluster/monitoring.yaml index 5cc5c8c..1797e5b 100644 --- a/cluster/monitoring.yaml +++ b/cluster/monitoring.yaml @@ -102,12 +102,16 @@ spec: role = "pod" } - // kubelet and cAdvisor, through the API server proxy + // kubelet and cAdvisor, straight from each kubelet (:10250). The API + // server proxy path (nodes/proxy) answers 403 on this k3s. The + // kubelet serving certs are signed by k3s-server-ca (the same CA as + // the service account's ca.crt) and carry the node IP as a SAN. discovery.relabel "kubelet" { targets = discovery.kubernetes.nodes.targets rule { - target_label = "__address__" - replacement = "kubernetes.default.svc:443" + source_labels = ["__meta_kubernetes_node_address_InternalIP"] + target_label = "__address__" + replacement = "$1:10250" } rule { source_labels = ["__meta_kubernetes_node_name"] @@ -115,17 +119,15 @@ spec: } rule { source_labels = ["__meta_kubernetes_node_name"] - target_label = "__metrics_path__" - replacement = "/api/v1/nodes/$1/proxy/metrics" + target_label = "instance" } } discovery.relabel "cadvisor" { targets = discovery.relabel.kubelet.output rule { - source_labels = ["node"] - target_label = "__metrics_path__" - replacement = "/api/v1/nodes/$1/proxy/metrics/cadvisor" + target_label = "__metrics_path__" + replacement = "/metrics/cadvisor" } }