228 lines
7.8 KiB
YAML
228 lines
7.8 KiB
YAML
|
|
# Cluster monitoring, shipped to the homelab Grafana stack on LXC 112 (10.0.0.22):
|
||
|
|
# Prometheus (remote write, :9090) and Loki (:3100). Nothing is stored in the cluster.
|
||
|
|
# - kube-state-metrics: object state (deployments, pods, PVCs, restarts…)
|
||
|
|
# - node-exporter: one per node, masters included
|
||
|
|
# - Alloy (1 replica): scrapes the two above + kubelet/cAdvisor through the API
|
||
|
|
# server, tails every pod's logs, and pushes everything to LXC 112.
|
||
|
|
apiVersion: argoproj.io/v1alpha1
|
||
|
|
kind: Application
|
||
|
|
metadata:
|
||
|
|
name: kube-state-metrics
|
||
|
|
namespace: argocd
|
||
|
|
spec:
|
||
|
|
project: default
|
||
|
|
source:
|
||
|
|
repoURL: https://prometheus-community.github.io/helm-charts
|
||
|
|
chart: kube-state-metrics
|
||
|
|
targetRevision: 8.7.1
|
||
|
|
helm:
|
||
|
|
valuesObject:
|
||
|
|
fullnameOverride: kube-state-metrics
|
||
|
|
destination:
|
||
|
|
server: https://kubernetes.default.svc
|
||
|
|
namespace: monitoring
|
||
|
|
syncPolicy:
|
||
|
|
automated:
|
||
|
|
prune: true
|
||
|
|
selfHeal: true
|
||
|
|
syncOptions:
|
||
|
|
- CreateNamespace=true
|
||
|
|
---
|
||
|
|
apiVersion: argoproj.io/v1alpha1
|
||
|
|
kind: Application
|
||
|
|
metadata:
|
||
|
|
name: node-exporter
|
||
|
|
namespace: argocd
|
||
|
|
spec:
|
||
|
|
project: default
|
||
|
|
source:
|
||
|
|
repoURL: https://prometheus-community.github.io/helm-charts
|
||
|
|
chart: prometheus-node-exporter
|
||
|
|
targetRevision: 4.60.1
|
||
|
|
helm:
|
||
|
|
valuesObject:
|
||
|
|
fullnameOverride: node-exporter
|
||
|
|
# Run on the tainted masters too.
|
||
|
|
tolerations:
|
||
|
|
- operator: Exists
|
||
|
|
destination:
|
||
|
|
server: https://kubernetes.default.svc
|
||
|
|
namespace: monitoring
|
||
|
|
syncPolicy:
|
||
|
|
automated:
|
||
|
|
prune: true
|
||
|
|
selfHeal: true
|
||
|
|
syncOptions:
|
||
|
|
- CreateNamespace=true
|
||
|
|
---
|
||
|
|
apiVersion: argoproj.io/v1alpha1
|
||
|
|
kind: Application
|
||
|
|
metadata:
|
||
|
|
name: alloy
|
||
|
|
namespace: argocd
|
||
|
|
spec:
|
||
|
|
project: default
|
||
|
|
source:
|
||
|
|
repoURL: https://grafana.github.io/helm-charts
|
||
|
|
chart: alloy
|
||
|
|
targetRevision: 1.14.3
|
||
|
|
helm:
|
||
|
|
valuesObject:
|
||
|
|
fullnameOverride: alloy
|
||
|
|
controller:
|
||
|
|
type: deployment
|
||
|
|
replicas: 1
|
||
|
|
alloy:
|
||
|
|
resources:
|
||
|
|
requests: { cpu: 100m, memory: 256Mi }
|
||
|
|
limits: { memory: 1Gi }
|
||
|
|
configMap:
|
||
|
|
content: |
|
||
|
|
// Every series gets cluster="k3s" so it can't be mixed up with the
|
||
|
|
// Proxmox node_exporters in Prometheus.
|
||
|
|
prometheus.remote_write "homelab" {
|
||
|
|
endpoint {
|
||
|
|
url = "http://10.0.0.22:9090/api/v1/write"
|
||
|
|
}
|
||
|
|
external_labels = { cluster = "k3s" }
|
||
|
|
}
|
||
|
|
|
||
|
|
loki.write "homelab" {
|
||
|
|
endpoint {
|
||
|
|
url = "http://10.0.0.22:3100/loki/api/v1/push"
|
||
|
|
}
|
||
|
|
external_labels = { cluster = "k3s", job = "k3s" }
|
||
|
|
}
|
||
|
|
|
||
|
|
discovery.kubernetes "nodes" {
|
||
|
|
role = "node"
|
||
|
|
}
|
||
|
|
|
||
|
|
discovery.kubernetes "pods" {
|
||
|
|
role = "pod"
|
||
|
|
}
|
||
|
|
|
||
|
|
// kubelet and cAdvisor, through the API server proxy
|
||
|
|
discovery.relabel "kubelet" {
|
||
|
|
targets = discovery.kubernetes.nodes.targets
|
||
|
|
rule {
|
||
|
|
target_label = "__address__"
|
||
|
|
replacement = "kubernetes.default.svc:443"
|
||
|
|
}
|
||
|
|
rule {
|
||
|
|
source_labels = ["__meta_kubernetes_node_name"]
|
||
|
|
target_label = "node"
|
||
|
|
}
|
||
|
|
rule {
|
||
|
|
source_labels = ["__meta_kubernetes_node_name"]
|
||
|
|
target_label = "__metrics_path__"
|
||
|
|
replacement = "/api/v1/nodes/$1/proxy/metrics"
|
||
|
|
}
|
||
|
|
}
|
||
|
|
|
||
|
|
discovery.relabel "cadvisor" {
|
||
|
|
targets = discovery.relabel.kubelet.output
|
||
|
|
rule {
|
||
|
|
source_labels = ["node"]
|
||
|
|
target_label = "__metrics_path__"
|
||
|
|
replacement = "/api/v1/nodes/$1/proxy/metrics/cadvisor"
|
||
|
|
}
|
||
|
|
}
|
||
|
|
|
||
|
|
prometheus.scrape "kubelet" {
|
||
|
|
job_name = "kubelet"
|
||
|
|
targets = discovery.relabel.kubelet.output
|
||
|
|
scheme = "https"
|
||
|
|
scrape_interval = "30s"
|
||
|
|
bearer_token_file = "/var/run/secrets/kubernetes.io/serviceaccount/token"
|
||
|
|
tls_config {
|
||
|
|
ca_file = "/var/run/secrets/kubernetes.io/serviceaccount/ca.crt"
|
||
|
|
}
|
||
|
|
forward_to = [prometheus.remote_write.homelab.receiver]
|
||
|
|
}
|
||
|
|
|
||
|
|
prometheus.scrape "cadvisor" {
|
||
|
|
job_name = "cadvisor"
|
||
|
|
targets = discovery.relabel.cadvisor.output
|
||
|
|
scheme = "https"
|
||
|
|
scrape_interval = "30s"
|
||
|
|
bearer_token_file = "/var/run/secrets/kubernetes.io/serviceaccount/token"
|
||
|
|
tls_config {
|
||
|
|
ca_file = "/var/run/secrets/kubernetes.io/serviceaccount/ca.crt"
|
||
|
|
}
|
||
|
|
forward_to = [prometheus.remote_write.homelab.receiver]
|
||
|
|
}
|
||
|
|
|
||
|
|
// node-exporter pods (hostNetwork, so the pod IP is the node IP)
|
||
|
|
discovery.relabel "node_exporter" {
|
||
|
|
targets = discovery.kubernetes.pods.targets
|
||
|
|
rule {
|
||
|
|
source_labels = ["__meta_kubernetes_pod_label_app_kubernetes_io_name"]
|
||
|
|
regex = "prometheus-node-exporter"
|
||
|
|
action = "keep"
|
||
|
|
}
|
||
|
|
rule {
|
||
|
|
source_labels = ["__meta_kubernetes_pod_container_port_number"]
|
||
|
|
regex = "9100"
|
||
|
|
action = "keep"
|
||
|
|
}
|
||
|
|
rule {
|
||
|
|
source_labels = ["__meta_kubernetes_pod_node_name"]
|
||
|
|
target_label = "instance"
|
||
|
|
}
|
||
|
|
}
|
||
|
|
|
||
|
|
prometheus.scrape "node_exporter" {
|
||
|
|
job_name = "k3s-node"
|
||
|
|
targets = discovery.relabel.node_exporter.output
|
||
|
|
scrape_interval = "30s"
|
||
|
|
forward_to = [prometheus.remote_write.homelab.receiver]
|
||
|
|
}
|
||
|
|
|
||
|
|
prometheus.scrape "kube_state_metrics" {
|
||
|
|
job_name = "kube-state-metrics"
|
||
|
|
targets = [{ __address__ = "kube-state-metrics.monitoring.svc:8080" }]
|
||
|
|
scrape_interval = "30s"
|
||
|
|
forward_to = [prometheus.remote_write.homelab.receiver]
|
||
|
|
}
|
||
|
|
|
||
|
|
// Pod logs, read through the API (no host mounts needed)
|
||
|
|
discovery.relabel "pod_logs" {
|
||
|
|
targets = discovery.kubernetes.pods.targets
|
||
|
|
rule {
|
||
|
|
source_labels = ["__meta_kubernetes_namespace"]
|
||
|
|
target_label = "namespace"
|
||
|
|
}
|
||
|
|
rule {
|
||
|
|
source_labels = ["__meta_kubernetes_pod_name"]
|
||
|
|
target_label = "pod"
|
||
|
|
}
|
||
|
|
rule {
|
||
|
|
source_labels = ["__meta_kubernetes_pod_container_name"]
|
||
|
|
target_label = "container"
|
||
|
|
}
|
||
|
|
rule {
|
||
|
|
source_labels = ["__meta_kubernetes_pod_node_name"]
|
||
|
|
target_label = "host"
|
||
|
|
}
|
||
|
|
rule {
|
||
|
|
source_labels = ["__meta_kubernetes_pod_label_app_kubernetes_io_name", "__meta_kubernetes_pod_label_app"]
|
||
|
|
regex = "^;*([^;]+)(;.*)?$"
|
||
|
|
target_label = "app"
|
||
|
|
}
|
||
|
|
}
|
||
|
|
|
||
|
|
loki.source.kubernetes "pods" {
|
||
|
|
targets = discovery.relabel.pod_logs.output
|
||
|
|
forward_to = [loki.write.homelab.receiver]
|
||
|
|
}
|
||
|
|
destination:
|
||
|
|
server: https://kubernetes.default.svc
|
||
|
|
namespace: monitoring
|
||
|
|
syncPolicy:
|
||
|
|
automated:
|
||
|
|
prune: true
|
||
|
|
selfHeal: true
|
||
|
|
syncOptions:
|
||
|
|
- CreateNamespace=true
|