argocd/cluster/monitoring.yaml

230 lines
8 KiB
YAML
Raw Permalink Normal View History

# Cluster monitoring, shipped to the homelab Grafana stack on LXC 112 (10.0.0.22):
# Prometheus (remote write, :9090) and Loki (:3100). Nothing is stored in the cluster.
# - kube-state-metrics: object state (deployments, pods, PVCs, restarts…)
# - node-exporter: one per node, masters included
# - Alloy (1 replica): scrapes the two above + kubelet/cAdvisor through the API
# server, tails every pod's logs, and pushes everything to LXC 112.
apiVersion: argoproj.io/v1alpha1
kind: Application
metadata:
name: kube-state-metrics
namespace: argocd
spec:
project: default
source:
repoURL: https://prometheus-community.github.io/helm-charts
chart: kube-state-metrics
targetRevision: 8.7.1
helm:
valuesObject:
fullnameOverride: kube-state-metrics
destination:
server: https://kubernetes.default.svc
namespace: monitoring
syncPolicy:
automated:
prune: true
selfHeal: true
syncOptions:
- CreateNamespace=true
---
apiVersion: argoproj.io/v1alpha1
kind: Application
metadata:
name: node-exporter
namespace: argocd
spec:
project: default
source:
repoURL: https://prometheus-community.github.io/helm-charts
chart: prometheus-node-exporter
targetRevision: 4.60.1
helm:
valuesObject:
fullnameOverride: node-exporter
# Run on the tainted masters too.
tolerations:
- operator: Exists
destination:
server: https://kubernetes.default.svc
namespace: monitoring
syncPolicy:
automated:
prune: true
selfHeal: true
syncOptions:
- CreateNamespace=true
---
apiVersion: argoproj.io/v1alpha1
kind: Application
metadata:
name: alloy
namespace: argocd
spec:
project: default
source:
repoURL: https://grafana.github.io/helm-charts
chart: alloy
targetRevision: 1.14.3
helm:
valuesObject:
fullnameOverride: alloy
controller:
type: deployment
replicas: 1
alloy:
resources:
requests: { cpu: 100m, memory: 256Mi }
limits: { memory: 1Gi }
configMap:
content: |
// Every series gets cluster="k3s" so it can't be mixed up with the
// Proxmox node_exporters in Prometheus.
prometheus.remote_write "homelab" {
endpoint {
url = "http://10.0.0.22:9090/api/v1/write"
}
external_labels = { cluster = "k3s" }
}
loki.write "homelab" {
endpoint {
url = "http://10.0.0.22:3100/loki/api/v1/push"
}
external_labels = { cluster = "k3s", job = "k3s" }
}
discovery.kubernetes "nodes" {
role = "node"
}
discovery.kubernetes "pods" {
role = "pod"
}
// kubelet and cAdvisor, straight from each kubelet (:10250). The API
// server proxy path (nodes/proxy) answers 403 on this k3s. The
// kubelet serving certs are signed by k3s-server-ca (the same CA as
// the service account's ca.crt) and carry the node IP as a SAN.
discovery.relabel "kubelet" {
targets = discovery.kubernetes.nodes.targets
rule {
source_labels = ["__meta_kubernetes_node_address_InternalIP"]
target_label = "__address__"
replacement = "$1:10250"
}
rule {
source_labels = ["__meta_kubernetes_node_name"]
target_label = "node"
}
rule {
source_labels = ["__meta_kubernetes_node_name"]
target_label = "instance"
}
}
discovery.relabel "cadvisor" {
targets = discovery.relabel.kubelet.output
rule {
target_label = "__metrics_path__"
replacement = "/metrics/cadvisor"
}
}
prometheus.scrape "kubelet" {
job_name = "kubelet"
targets = discovery.relabel.kubelet.output
scheme = "https"
scrape_interval = "30s"
bearer_token_file = "/var/run/secrets/kubernetes.io/serviceaccount/token"
tls_config {
ca_file = "/var/run/secrets/kubernetes.io/serviceaccount/ca.crt"
}
forward_to = [prometheus.remote_write.homelab.receiver]
}
prometheus.scrape "cadvisor" {
job_name = "cadvisor"
targets = discovery.relabel.cadvisor.output
scheme = "https"
scrape_interval = "30s"
bearer_token_file = "/var/run/secrets/kubernetes.io/serviceaccount/token"
tls_config {
ca_file = "/var/run/secrets/kubernetes.io/serviceaccount/ca.crt"
}
forward_to = [prometheus.remote_write.homelab.receiver]
}
// node-exporter pods (hostNetwork, so the pod IP is the node IP)
discovery.relabel "node_exporter" {
targets = discovery.kubernetes.pods.targets
rule {
source_labels = ["__meta_kubernetes_pod_label_app_kubernetes_io_name"]
regex = "prometheus-node-exporter"
action = "keep"
}
rule {
source_labels = ["__meta_kubernetes_pod_container_port_number"]
regex = "9100"
action = "keep"
}
rule {
source_labels = ["__meta_kubernetes_pod_node_name"]
target_label = "instance"
}
}
prometheus.scrape "node_exporter" {
job_name = "k3s-node"
targets = discovery.relabel.node_exporter.output
scrape_interval = "30s"
forward_to = [prometheus.remote_write.homelab.receiver]
}
prometheus.scrape "kube_state_metrics" {
job_name = "kube-state-metrics"
targets = [{ __address__ = "kube-state-metrics.monitoring.svc:8080" }]
scrape_interval = "30s"
forward_to = [prometheus.remote_write.homelab.receiver]
}
// Pod logs, read through the API (no host mounts needed)
discovery.relabel "pod_logs" {
targets = discovery.kubernetes.pods.targets
rule {
source_labels = ["__meta_kubernetes_namespace"]
target_label = "namespace"
}
rule {
source_labels = ["__meta_kubernetes_pod_name"]
target_label = "pod"
}
rule {
source_labels = ["__meta_kubernetes_pod_container_name"]
target_label = "container"
}
rule {
source_labels = ["__meta_kubernetes_pod_node_name"]
target_label = "host"
}
rule {
source_labels = ["__meta_kubernetes_pod_label_app_kubernetes_io_name", "__meta_kubernetes_pod_label_app"]
regex = "^;*([^;]+)(;.*)?$"
target_label = "app"
}
}
loki.source.kubernetes "pods" {
targets = discovery.relabel.pod_logs.output
forward_to = [loki.write.homelab.receiver]
}
destination:
server: https://kubernetes.default.svc
namespace: monitoring
syncPolicy:
automated:
prune: true
selfHeal: true
syncOptions:
- CreateNamespace=true