monitoring: silencia 4 alertas que llevaban meses sin significar nada
Tres eran falsos positivos de k3s: KubeControllerManagerDown, KubeSchedulerDown y KubeProxyDown llevaban 92 DÍAS disparadas en severidad critical. En k3s esos tres componentes van embebidos en el proceso del servidor y no exponen las métricas que espera el chart, así que sus ServiceMonitors nunca conectaban: la alerta no medía la salud de nada, solo la ausencia de un target imposible. La cuarta, PrometheusNotConnectedToAlertmanagers, llevaba disparada desde el reinicio del máster del 16-jul. La causa es de diseño: alertmanager.enabled es false y los avisos van por Grafana→Telegram y Kuma→Telegram. Ya se había desactivado el grupo de reglas `alertmanager`, pero esta alerta vive en el grupo `prometheus`, así que sobrevivió; se quita por nombre con defaultRules.disabled. Tres "critical" permanentes no son un aviso, son entrenamiento para ignorar el panel: cuando algo se rompa de verdad, se perderá entre el ruido. Hecho por Helm y no a mano (rev 1 → 2, mismo chart 83.2.0), que es lo único que sobrevive a un futuro upgrade. Verificado ANTES de aplicar renderizando el chart con y sin los valores nuevos: el diff quita exactamente esas 4 alertas (141 → 137) y ningún recurso nuevo aparece. Comprobado además que el render reproduce el manifiesto ya desplegado, o sea que no había ediciones a mano que el upgrade fuese a aplastar. Después: 1 sola alerta disparada (Watchdog, el latido intencionado del chart, severity none) y los 15 targets de scrape en up — antes 3 fallaban siempre. Grafana sano (api/health ok, 12.4.2) y su contraseña rotada intacta, porque GF_SECURITY_ADMIN_PASSWORD solo aplica en el primer arranque. Se añade values-kube-prometheus-stack.yaml: este namespace no está bajo ArgoCD y hasta hoy los valores del release solo vivían dentro de Helm, sin forma de saber cómo estaba configurado sin interrogarlo. Se refresca el export del ConfigMap de reglas y se borran 3 dashboards que el upgrade eliminó del cluster. Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
Claude Opus 4.8
@@ -1,60 +0,0 @@
|
||||
apiVersion: v1
|
||||
data:
|
||||
controller-manager.json: '{"editable":true,"links":[{"asDropdown":true,"includeVars":true,"keepTime":true,"tags":["kubernetes-mixin"],"targetBlank":false,"title":"Kubernetes","type":"dashboards"}],"panels":[{"datasource":{"type":"datasource","uid":"--
|
||||
Mixed --"},"fieldConfig":{"defaults":{"unit":"none"}},"gridPos":{"h":7,"w":4,"x":0,"y":0},"id":1,"interval":"1m","options":{"colorMode":"none"},"pluginVersion":"v11.4.0","targets":[{"datasource":{"type":"prometheus","uid":"${datasource}"},"expr":"sum(up{cluster=\"$cluster\",
|
||||
job=\"kube-controller-manager\"})","instant":true}],"title":"Up","type":"stat"},{"datasource":{"type":"datasource","uid":"--
|
||||
Mixed --"},"fieldConfig":{"defaults":{"custom":{"fillOpacity":10,"showPoints":"never","spanNulls":true},"unit":"ops"}},"gridPos":{"h":7,"w":20,"x":4,"y":0},"id":2,"interval":"1m","options":{"legend":{"asTable":true,"calcs":["lastNotNull"],"displayMode":"table","placement":"right","showLegend":true},"tooltip":{"mode":"single"}},"pluginVersion":"v11.4.0","targets":[{"datasource":{"type":"prometheus","uid":"${datasource}"},"expr":"sum(rate(workqueue_adds_total{cluster=\"$cluster\",
|
||||
job=\"kube-controller-manager\", instance=~\"$instance\"}[$__rate_interval]))
|
||||
by (cluster, instance, name)","legendFormat":"{{cluster}} {{instance}} {{name}}"}],"title":"Work
|
||||
Queue Add Rate","type":"timeseries"},{"datasource":{"type":"datasource","uid":"--
|
||||
Mixed --"},"fieldConfig":{"defaults":{"custom":{"fillOpacity":10,"showPoints":"never","spanNulls":true},"unit":"short"}},"gridPos":{"h":7,"w":24,"x":0,"y":7},"id":3,"interval":"1m","options":{"legend":{"asTable":true,"calcs":["lastNotNull"],"displayMode":"table","placement":"right","showLegend":true},"tooltip":{"mode":"single"}},"pluginVersion":"v11.4.0","targets":[{"datasource":{"type":"prometheus","uid":"${datasource}"},"expr":"sum(rate(workqueue_depth{cluster=\"$cluster\",
|
||||
job=\"kube-controller-manager\", instance=~\"$instance\"}[$__rate_interval]))
|
||||
by (cluster, instance, name)","legendFormat":"{{cluster}} {{instance}} {{name}}"}],"title":"Work
|
||||
Queue Depth","type":"timeseries"},{"datasource":{"type":"datasource","uid":"--
|
||||
Mixed --"},"fieldConfig":{"defaults":{"custom":{"fillOpacity":10,"showPoints":"never","spanNulls":true},"unit":"s"}},"gridPos":{"h":7,"w":24,"x":0,"y":14},"id":4,"interval":"1m","options":{"legend":{"asTable":true,"calcs":["lastNotNull"],"displayMode":"table","placement":"right","showLegend":true},"tooltip":{"mode":"single"}},"pluginVersion":"v11.4.0","targets":[{"datasource":{"type":"prometheus","uid":"${datasource}"},"expr":"histogram_quantile(0.99,
|
||||
sum(rate(workqueue_queue_duration_seconds_bucket{cluster=\"$cluster\", job=\"kube-controller-manager\",
|
||||
instance=~\"$instance\"}[$__rate_interval])) by (cluster, instance, name, le))","legendFormat":"{{cluster}}
|
||||
{{instance}} {{name}}"}],"title":"Work Queue Latency","type":"timeseries"},{"datasource":{"type":"datasource","uid":"--
|
||||
Mixed --"},"fieldConfig":{"defaults":{"custom":{"fillOpacity":10,"showPoints":"never","spanNulls":true},"unit":"ops"}},"gridPos":{"h":7,"w":8,"x":0,"y":21},"id":5,"interval":"1m","options":{"legend":{"asTable":true,"calcs":["lastNotNull"],"displayMode":"table","placement":"right","showLegend":true},"tooltip":{"mode":"single"}},"pluginVersion":"v11.4.0","targets":[{"datasource":{"type":"prometheus","uid":"${datasource}"},"expr":"sum(rate(rest_client_requests_total{job=\"kube-controller-manager\",
|
||||
instance=~\"$instance\",code=~\"2..\"}[$__rate_interval]))","legendFormat":"2xx"},{"datasource":{"type":"prometheus","uid":"${datasource}"},"expr":"sum(rate(rest_client_requests_total{job=\"kube-controller-manager\",
|
||||
instance=~\"$instance\",code=~\"3..\"}[$__rate_interval]))","legendFormat":"3xx"},{"datasource":{"type":"prometheus","uid":"${datasource}"},"expr":"sum(rate(rest_client_requests_total{job=\"kube-controller-manager\",
|
||||
instance=~\"$instance\",code=~\"4..\"}[$__rate_interval]))","legendFormat":"4xx"},{"datasource":{"type":"prometheus","uid":"${datasource}"},"expr":"sum(rate(rest_client_requests_total{job=\"kube-controller-manager\",
|
||||
instance=~\"$instance\",code=~\"5..\"}[$__rate_interval]))","legendFormat":"5xx"}],"title":"Kube
|
||||
API Request Rate","type":"timeseries"},{"datasource":{"type":"datasource","uid":"--
|
||||
Mixed --"},"fieldConfig":{"defaults":{"custom":{"fillOpacity":10,"showPoints":"never","spanNulls":true},"unit":"s"}},"gridPos":{"h":7,"w":16,"x":8,"y":21},"id":6,"interval":"1m","options":{"legend":{"asTable":true,"calcs":["lastNotNull"],"displayMode":"table","placement":"right","showLegend":true},"tooltip":{"mode":"single"}},"pluginVersion":"v11.4.0","targets":[{"datasource":{"type":"prometheus","uid":"${datasource}"},"expr":"histogram_quantile(0.99,
|
||||
sum(rate(rest_client_request_duration_seconds_bucket{cluster=\"$cluster\", job=\"kube-controller-manager\",
|
||||
instance=~\"$instance\", verb=\"POST\"}[$__rate_interval])) by (verb, le))","legendFormat":"{{verb}}"}],"title":"Post
|
||||
Request Latency 99th Quantile","type":"timeseries"},{"datasource":{"type":"datasource","uid":"--
|
||||
Mixed --"},"fieldConfig":{"defaults":{"custom":{"fillOpacity":10,"showPoints":"never","spanNulls":true},"unit":"s"}},"gridPos":{"h":7,"w":24,"x":0,"y":28},"id":7,"interval":"1m","options":{"legend":{"asTable":true,"calcs":["lastNotNull"],"displayMode":"table","placement":"right","showLegend":true},"tooltip":{"mode":"single"}},"pluginVersion":"v11.4.0","targets":[{"datasource":{"type":"prometheus","uid":"${datasource}"},"expr":"histogram_quantile(0.99,
|
||||
sum(rate(rest_client_request_duration_seconds_bucket{cluster=\"$cluster\", job=\"kube-controller-manager\",
|
||||
instance=~\"$instance\", verb=\"GET\"}[$__rate_interval])) by (verb, le))","legendFormat":"{{verb}}"}],"title":"Get
|
||||
Request Latency 99th Quantile","type":"timeseries"},{"datasource":{"type":"datasource","uid":"--
|
||||
Mixed --"},"fieldConfig":{"defaults":{"custom":{"fillOpacity":10,"showPoints":"never","spanNulls":true},"unit":"bytes"}},"gridPos":{"h":7,"w":8,"x":0,"y":35},"id":8,"interval":"1m","options":{"legend":{"asTable":true,"calcs":["lastNotNull"],"displayMode":"table","placement":"right","showLegend":true},"tooltip":{"mode":"single"}},"pluginVersion":"v11.4.0","targets":[{"datasource":{"type":"prometheus","uid":"${datasource}"},"expr":"process_resident_memory_bytes{cluster=\"$cluster\",
|
||||
job=\"kube-controller-manager\",instance=~\"$instance\"}","legendFormat":"{{instance}}"}],"title":"Memory","type":"timeseries"},{"datasource":{"type":"datasource","uid":"--
|
||||
Mixed --"},"fieldConfig":{"defaults":{"custom":{"fillOpacity":10,"showPoints":"never","spanNulls":true},"unit":"short"}},"gridPos":{"h":7,"w":8,"x":8,"y":35},"id":9,"interval":"1m","options":{"legend":{"asTable":true,"calcs":["lastNotNull"],"displayMode":"table","placement":"right","showLegend":true},"tooltip":{"mode":"single"}},"pluginVersion":"v11.4.0","targets":[{"datasource":{"type":"prometheus","uid":"${datasource}"},"expr":"rate(process_cpu_seconds_total{cluster=\"$cluster\",
|
||||
job=\"kube-controller-manager\",instance=~\"$instance\"}[$__rate_interval])","legendFormat":"{{instance}}"}],"title":"CPU
|
||||
usage","type":"timeseries"},{"datasource":{"type":"datasource","uid":"-- Mixed
|
||||
--"},"fieldConfig":{"defaults":{"custom":{"fillOpacity":10,"showPoints":"never","spanNulls":true},"unit":"short"}},"gridPos":{"h":7,"w":8,"x":16,"y":35},"id":10,"interval":"1m","options":{"legend":{"asTable":true,"calcs":["lastNotNull"],"displayMode":"table","placement":"right","showLegend":true},"tooltip":{"mode":"single"}},"pluginVersion":"v11.4.0","targets":[{"datasource":{"type":"prometheus","uid":"${datasource}"},"expr":"go_goroutines{cluster=\"$cluster\",
|
||||
job=\"kube-controller-manager\",instance=~\"$instance\"}","legendFormat":"{{instance}}"}],"title":"Goroutines","type":"timeseries"}],"refresh":"10s","schemaVersion":39,"tags":["kubernetes-mixin"],"templating":{"list":[{"current":{"selected":true,"text":"default","value":"default"},"hide":0,"label":"Data
|
||||
source","name":"datasource","query":"prometheus","regex":"","type":"datasource"},{"datasource":{"type":"prometheus","uid":"${datasource}"},"hide":2,"label":"cluster","name":"cluster","query":"label_values(up{job=\"kube-controller-manager\"},
|
||||
cluster)","refresh":2,"sort":1,"type":"query","allValue":".*"},{"datasource":{"type":"prometheus","uid":"${datasource}"},"hide":0,"includeAll":true,"label":"instance","name":"instance","query":"label_values(up{cluster=\"$cluster\",
|
||||
job=\"kube-controller-manager\"}, instance)","refresh":2,"sort":1,"type":"query"}]},"time":{"from":"now-1h","to":"now"},"timezone":
|
||||
"utc","title":"Kubernetes / Controller Manager","uid":"72e0e05bef5099e5f049b05fdc429ed4"}'
|
||||
kind: ConfigMap
|
||||
metadata:
|
||||
annotations:
|
||||
meta.helm.sh/release-name: kube-prometheus-stack
|
||||
meta.helm.sh/release-namespace: monitoring
|
||||
labels:
|
||||
app: kube-prometheus-stack-grafana
|
||||
app.kubernetes.io/instance: kube-prometheus-stack
|
||||
app.kubernetes.io/managed-by: Helm
|
||||
app.kubernetes.io/part-of: kube-prometheus-stack
|
||||
app.kubernetes.io/version: 83.2.0
|
||||
chart: kube-prometheus-stack-83.2.0
|
||||
grafana_dashboard: '1'
|
||||
heritage: Helm
|
||||
release: kube-prometheus-stack
|
||||
name: kube-prometheus-stack-controller-manager
|
||||
namespace: monitoring
|
||||
|
||||
@@ -1,62 +0,0 @@
|
||||
apiVersion: v1
|
||||
data:
|
||||
proxy.json: '{"editable":true,"links":[{"asDropdown":true,"includeVars":true,"keepTime":true,"tags":["kubernetes-mixin"],"targetBlank":false,"title":"Kubernetes","type":"dashboards"}],"panels":[{"datasource":{"type":"datasource","uid":"--
|
||||
Mixed --"},"fieldConfig":{"defaults":{"unit":"none"}},"gridPos":{"h":7,"w":4,"x":0,"y":0},"id":1,"interval":"1m","options":{"colorMode":"none"},"pluginVersion":"v11.4.0","targets":[{"datasource":{"type":"prometheus","uid":"${datasource}"},"expr":"sum(up{cluster=\"$cluster\",
|
||||
job=\"kube-proxy\"})","instant":true}],"title":"Up","type":"stat"},{"datasource":{"type":"datasource","uid":"--
|
||||
Mixed --"},"fieldConfig":{"defaults":{"custom":{"fillOpacity":10,"showPoints":"never","spanNulls":true},"unit":"ops"}},"gridPos":{"h":7,"w":10,"x":4,"y":0},"id":2,"interval":"1m","options":{"legend":{"asTable":true,"calcs":["lastNotNull"],"displayMode":"table","placement":"right","showLegend":true},"tooltip":{"mode":"single"}},"pluginVersion":"v11.4.0","targets":[{"datasource":{"type":"prometheus","uid":"${datasource}"},"expr":"sum(rate(kubeproxy_sync_proxy_rules_duration_seconds_count{cluster=\"$cluster\",
|
||||
job=\"kube-proxy\", instance=~\"$instance\"}[$__rate_interval]))","legendFormat":"rate"}],"title":"Rules
|
||||
Sync Rate","type":"timeseries"},{"datasource":{"type":"datasource","uid":"-- Mixed
|
||||
--"},"fieldConfig":{"defaults":{"custom":{"fillOpacity":10,"showPoints":"never","spanNulls":true},"unit":"s"}},"gridPos":{"h":7,"w":10,"x":14,"y":0},"id":3,"interval":"1m","options":{"legend":{"asTable":true,"calcs":["lastNotNull"],"displayMode":"table","placement":"right","showLegend":true},"tooltip":{"mode":"single"}},"pluginVersion":"v11.4.0","targets":[{"datasource":{"type":"prometheus","uid":"${datasource}"},"expr":"histogram_quantile(0.99,rate(kubeproxy_sync_proxy_rules_duration_seconds_bucket{cluster=\"$cluster\",
|
||||
job=\"kube-proxy\", instance=~\"$instance\"}[$__rate_interval]))","legendFormat":"{{instance}}"}],"title":"Rules
|
||||
Sync Latency 99th Quantile","type":"timeseries"},{"datasource":{"type":"datasource","uid":"--
|
||||
Mixed --"},"fieldConfig":{"defaults":{"custom":{"fillOpacity":10,"showPoints":"never","spanNulls":true},"unit":"ops"}},"gridPos":{"h":7,"w":12,"x":0,"y":7},"id":4,"interval":"1m","options":{"legend":{"asTable":true,"calcs":["lastNotNull"],"displayMode":"table","placement":"right","showLegend":true},"tooltip":{"mode":"single"}},"pluginVersion":"v11.4.0","targets":[{"datasource":{"type":"prometheus","uid":"${datasource}"},"expr":"sum(rate(kubeproxy_network_programming_duration_seconds_count{cluster=\"$cluster\",
|
||||
job=\"kube-proxy\", instance=~\"$instance\"}[$__rate_interval]))","legendFormat":"rate"}],"title":"Network
|
||||
Programming Rate","type":"timeseries"},{"datasource":{"type":"datasource","uid":"--
|
||||
Mixed --"},"fieldConfig":{"defaults":{"custom":{"fillOpacity":10,"showPoints":"never","spanNulls":true},"unit":"s"}},"gridPos":{"h":7,"w":12,"x":12,"y":7},"id":5,"interval":"1m","options":{"legend":{"asTable":true,"calcs":["lastNotNull"],"displayMode":"table","placement":"right","showLegend":true},"tooltip":{"mode":"single"}},"pluginVersion":"v11.4.0","targets":[{"datasource":{"type":"prometheus","uid":"${datasource}"},"expr":"histogram_quantile(0.99,
|
||||
sum(rate(kubeproxy_network_programming_duration_seconds_bucket{cluster=\"$cluster\",
|
||||
job=\"kube-proxy\", instance=~\"$instance\"}[$__rate_interval])) by (instance,
|
||||
le))","legendFormat":"{{instance}}"}],"title":"Network Programming Latency 99th
|
||||
Quantile","type":"timeseries"},{"datasource":{"type":"datasource","uid":"-- Mixed
|
||||
--"},"fieldConfig":{"defaults":{"custom":{"fillOpacity":10,"showPoints":"never","spanNulls":true},"unit":"ops"}},"gridPos":{"h":7,"w":8,"x":0,"y":14},"id":6,"interval":"1m","options":{"legend":{"asTable":true,"calcs":["lastNotNull"],"displayMode":"table","placement":"right","showLegend":true},"tooltip":{"mode":"single"}},"pluginVersion":"v11.4.0","targets":[{"datasource":{"type":"prometheus","uid":"${datasource}"},"expr":"sum(rate(rest_client_requests_total{cluster=\"$cluster\",job=\"kube-proxy\",
|
||||
instance=~\"$instance\",code=~\"2..\"}[$__rate_interval]))","legendFormat":"2xx"},{"datasource":{"type":"prometheus","uid":"${datasource}"},"expr":"sum(rate(rest_client_requests_total{cluster=\"$cluster\",job=\"kube-proxy\",
|
||||
instance=~\"$instance\",code=~\"3..\"}[$__rate_interval]))","legendFormat":"3xx"},{"datasource":{"type":"prometheus","uid":"${datasource}"},"expr":"sum(rate(rest_client_requests_total{cluster=\"$cluster\",job=\"kube-proxy\",
|
||||
instance=~\"$instance\",code=~\"4..\"}[$__rate_interval]))","legendFormat":"4xx"},{"datasource":{"type":"prometheus","uid":"${datasource}"},"expr":"sum(rate(rest_client_requests_total{cluster=\"$cluster\",job=\"kube-proxy\",
|
||||
instance=~\"$instance\",code=~\"5..\"}[$__rate_interval]))","legendFormat":"5xx"}],"title":"Kube
|
||||
API Request Rate","type":"timeseries"},{"datasource":{"type":"datasource","uid":"--
|
||||
Mixed --"},"fieldConfig":{"defaults":{"custom":{"fillOpacity":10,"showPoints":"never","spanNulls":true},"unit":"ops"}},"gridPos":{"h":7,"w":16,"x":8,"y":14},"id":7,"interval":"1m","options":{"legend":{"asTable":true,"calcs":["lastNotNull"],"displayMode":"table","placement":"right","showLegend":true},"tooltip":{"mode":"single"}},"pluginVersion":"v11.4.0","targets":[{"datasource":{"type":"prometheus","uid":"${datasource}"},"expr":"histogram_quantile(0.99,
|
||||
sum(rate(rest_client_request_duration_seconds_bucket{cluster=\"$cluster\", job=\"kube-proxy\",instance=~\"$instance\",verb=\"POST\"}[$__rate_interval]))
|
||||
by (verb, le))","legendFormat":"{{verb}}"}],"title":"Post Request Latency 99th
|
||||
Quantile","type":"timeseries"},{"datasource":{"type":"datasource","uid":"-- Mixed
|
||||
--"},"fieldConfig":{"defaults":{"custom":{"fillOpacity":10,"showPoints":"never","spanNulls":true},"unit":"s"}},"gridPos":{"h":7,"w":24,"x":0,"y":21},"id":8,"interval":"1m","options":{"legend":{"asTable":true,"calcs":["lastNotNull"],"displayMode":"table","placement":"right","showLegend":true},"tooltip":{"mode":"single"}},"pluginVersion":"v11.4.0","targets":[{"datasource":{"type":"prometheus","uid":"${datasource}"},"expr":"histogram_quantile(0.99,
|
||||
sum(rate(rest_client_request_duration_seconds_bucket{cluster=\"$cluster\", job=\"kube-proxy\",
|
||||
instance=~\"$instance\", verb=\"GET\"}[$__rate_interval])) by (verb, le))","legendFormat":"{{verb}}"}],"title":"Get
|
||||
Request Latency 99th Quantile","type":"timeseries"},{"datasource":{"type":"datasource","uid":"--
|
||||
Mixed --"},"fieldConfig":{"defaults":{"custom":{"fillOpacity":10,"showPoints":"never","spanNulls":true},"unit":"bytes"}},"gridPos":{"h":7,"w":8,"x":0,"y":28},"id":9,"interval":"1m","options":{"legend":{"asTable":true,"calcs":["lastNotNull"],"displayMode":"table","placement":"right","showLegend":true},"tooltip":{"mode":"single"}},"pluginVersion":"v11.4.0","targets":[{"datasource":{"type":"prometheus","uid":"${datasource}"},"expr":"process_resident_memory_bytes{cluster=\"$cluster\",
|
||||
job=\"kube-proxy\",instance=~\"$instance\"}","legendFormat":"{{instance}}"}],"title":"Memory","type":"timeseries"},{"datasource":{"type":"datasource","uid":"--
|
||||
Mixed --"},"fieldConfig":{"defaults":{"custom":{"fillOpacity":10,"showPoints":"never","spanNulls":true},"unit":"short"}},"gridPos":{"h":7,"w":8,"x":8,"y":28},"id":10,"interval":"1m","options":{"legend":{"asTable":true,"calcs":["lastNotNull"],"displayMode":"table","placement":"right","showLegend":true},"tooltip":{"mode":"single"}},"pluginVersion":"v11.4.0","targets":[{"datasource":{"type":"prometheus","uid":"${datasource}"},"expr":"rate(process_cpu_seconds_total{cluster=\"$cluster\",
|
||||
job=\"kube-proxy\",instance=~\"$instance\"}[$__rate_interval])","legendFormat":"{{instance}}"}],"title":"CPU
|
||||
usage","type":"timeseries"},{"datasource":{"type":"datasource","uid":"-- Mixed
|
||||
--"},"fieldConfig":{"defaults":{"custom":{"fillOpacity":10,"showPoints":"never","spanNulls":true},"unit":"short"}},"gridPos":{"h":7,"w":8,"x":16,"y":28},"id":11,"interval":"1m","options":{"legend":{"asTable":true,"calcs":["lastNotNull"],"displayMode":"table","placement":"right","showLegend":true},"tooltip":{"mode":"single"}},"pluginVersion":"v11.4.0","targets":[{"datasource":{"type":"prometheus","uid":"${datasource}"},"expr":"go_goroutines{cluster=\"$cluster\",
|
||||
job=\"kube-proxy\",instance=~\"$instance\"}","legendFormat":"{{instance}}"}],"title":"Goroutines","type":"timeseries"}],"refresh":"10s","schemaVersion":39,"tags":["kubernetes-mixin"],"templating":{"list":[{"current":{"selected":true,"text":"default","value":"default"},"hide":0,"label":"Data
|
||||
source","name":"datasource","query":"prometheus","regex":"","type":"datasource"},{"datasource":{"type":"prometheus","uid":"${datasource}"},"hide":2,"label":"cluster","name":"cluster","query":"label_values(up{job=\"kube-proxy\"},
|
||||
cluster)","refresh":2,"sort":1,"type":"query","allValue":".*"},{"allValue":".+","datasource":{"type":"prometheus","uid":"${datasource}"},"hide":0,"includeAll":true,"label":"instance","name":"instance","query":"label_values(up{job=\"kube-proxy\",
|
||||
cluster=\"$cluster\", job=\"kube-proxy\"}, instance)","refresh":2,"type":"query"}]},"time":{"from":"now-1h","to":"now"},"timezone":
|
||||
"utc","title":"Kubernetes / Proxy","uid":"632e265de029684c40b21cb76bca4f94"}'
|
||||
kind: ConfigMap
|
||||
metadata:
|
||||
annotations:
|
||||
meta.helm.sh/release-name: kube-prometheus-stack
|
||||
meta.helm.sh/release-namespace: monitoring
|
||||
labels:
|
||||
app: kube-prometheus-stack-grafana
|
||||
app.kubernetes.io/instance: kube-prometheus-stack
|
||||
app.kubernetes.io/managed-by: Helm
|
||||
app.kubernetes.io/part-of: kube-prometheus-stack
|
||||
app.kubernetes.io/version: 83.2.0
|
||||
chart: kube-prometheus-stack-83.2.0
|
||||
grafana_dashboard: '1'
|
||||
heritage: Helm
|
||||
release: kube-prometheus-stack
|
||||
name: kube-prometheus-stack-proxy
|
||||
namespace: monitoring
|
||||
|
||||
@@ -1,72 +0,0 @@
|
||||
apiVersion: v1
|
||||
data:
|
||||
scheduler.json: '{"editable":true,"links":[{"asDropdown":true,"includeVars":true,"keepTime":true,"tags":["kubernetes-mixin"],"targetBlank":false,"title":"Kubernetes","type":"dashboards"}],"panels":[{"datasource":{"type":"datasource","uid":"--
|
||||
Mixed --"},"fieldConfig":{"defaults":{"unit":"none"}},"gridPos":{"h":7,"w":4,"x":0,"y":0},"id":1,"interval":"1m","options":{"colorMode":"none"},"pluginVersion":"v11.4.0","targets":[{"datasource":{"type":"prometheus","uid":"${datasource}"},"expr":"sum(up{cluster=\"$cluster\",
|
||||
job=\"kube-scheduler\"})","instant":true}],"title":"Up","type":"stat"},{"datasource":{"type":"datasource","uid":"--
|
||||
Mixed --"},"fieldConfig":{"defaults":{"custom":{"fillOpacity":10,"showPoints":"never","spanNulls":true},"unit":"ops"}},"gridPos":{"h":7,"w":10,"x":4,"y":0},"id":2,"interval":"1m","options":{"legend":{"asTable":true,"calcs":["lastNotNull"],"displayMode":"table","placement":"right","showLegend":true},"tooltip":{"mode":"single"}},"pluginVersion":"v11.4.0","targets":[{"datasource":{"type":"prometheus","uid":"${datasource}"},"expr":"sum(rate(scheduler_scheduling_attempt_duration_seconds_count{cluster=\"$cluster\",
|
||||
job=\"kube-scheduler\", instance=~\"$instance\"}[$__rate_interval])) by (cluster,
|
||||
instance)","legendFormat":"{{cluster}} {{instance}} e2e"},{"datasource":{"type":"prometheus","uid":"${datasource}"},"expr":"sum(rate(scheduler_pod_scheduling_sli_duration_seconds_count{cluster=\"$cluster\",
|
||||
job=\"kube-scheduler\", instance=~\"$instance\"}[$__rate_interval])) by (cluster,
|
||||
instance)","legendFormat":"{{cluster}} {{instance}} binding"},{"datasource":{"type":"prometheus","uid":"${datasource}"},"expr":"sum(rate(scheduler_scheduling_algorithm_duration_seconds_count{cluster=\"$cluster\",
|
||||
job=\"kube-scheduler\", instance=~\"$instance\"}[$__rate_interval])) by (cluster,
|
||||
instance)","legendFormat":"{{cluster}} {{instance}} scheduling algorithm"},{"datasource":{"type":"prometheus","uid":"${datasource}"},"expr":"sum(rate(scheduler_volume_scheduling_duration_seconds_count{cluster=\"$cluster\",
|
||||
job=\"kube-scheduler\", instance=~\"$instance\"}[$__rate_interval])) by (cluster,
|
||||
instance)","legendFormat":"{{cluster}} {{instance}} volume"}],"title":"Scheduling
|
||||
Rate","type":"timeseries"},{"datasource":{"type":"datasource","uid":"-- Mixed
|
||||
--"},"fieldConfig":{"defaults":{"custom":{"fillOpacity":10,"showPoints":"never","spanNulls":true},"unit":"s"}},"gridPos":{"h":7,"w":10,"x":14,"y":0},"id":3,"interval":"1m","options":{"legend":{"asTable":true,"calcs":["lastNotNull"],"displayMode":"table","placement":"right","showLegend":true},"tooltip":{"mode":"single"}},"pluginVersion":"v11.4.0","targets":[{"datasource":{"type":"prometheus","uid":"${datasource}"},"expr":"histogram_quantile(0.99,
|
||||
sum(rate(scheduler_scheduling_attempt_duration_seconds_bucket{cluster=\"$cluster\",
|
||||
job=\"kube-scheduler\",instance=~\"$instance\"}[$__rate_interval])) by (cluster,
|
||||
instance, le))","legendFormat":"{{cluster}} {{instance}} e2e"},{"datasource":{"type":"prometheus","uid":"${datasource}"},"expr":"histogram_quantile(0.99,
|
||||
sum(rate(scheduler_pod_scheduling_sli_duration_seconds_bucket{cluster=\"$cluster\",
|
||||
job=\"kube-scheduler\",instance=~\"$instance\"}[$__rate_interval])) by (cluster,
|
||||
instance, le))","legendFormat":"{{cluster}} {{instance}} binding"},{"datasource":{"type":"prometheus","uid":"${datasource}"},"expr":"histogram_quantile(0.99,
|
||||
sum(rate(scheduler_scheduling_algorithm_duration_seconds_bucket{cluster=\"$cluster\",
|
||||
job=\"kube-scheduler\",instance=~\"$instance\"}[$__rate_interval])) by (cluster,
|
||||
instance, le))","legendFormat":"{{cluster}} {{instance}} scheduling algorithm"},{"datasource":{"type":"prometheus","uid":"${datasource}"},"expr":"histogram_quantile(0.99,
|
||||
sum(rate(scheduler_volume_scheduling_duration_seconds_bucket{cluster=\"$cluster\",
|
||||
job=\"kube-scheduler\",instance=~\"$instance\"}[$__rate_interval])) by (cluster,
|
||||
instance, le))","legendFormat":"{{cluster}} {{instance}} volume"}],"title":"Scheduling
|
||||
latency 99th Quantile","type":"timeseries"},{"datasource":{"type":"datasource","uid":"--
|
||||
Mixed --"},"fieldConfig":{"defaults":{"custom":{"fillOpacity":10,"showPoints":"never","spanNulls":true},"unit":"ops"}},"gridPos":{"h":7,"w":8,"x":0,"y":7},"id":4,"interval":"1m","options":{"legend":{"asTable":true,"calcs":["lastNotNull"],"displayMode":"table","placement":"right","showLegend":true},"tooltip":{"mode":"single"}},"pluginVersion":"v11.4.0","targets":[{"datasource":{"type":"prometheus","uid":"${datasource}"},"expr":"sum(rate(rest_client_requests_total{cluster=\"$cluster\",
|
||||
job=\"kube-scheduler\", instance=~\"$instance\",code=~\"2..\"}[$__rate_interval]))","legendFormat":"2xx"},{"datasource":{"type":"prometheus","uid":"${datasource}"},"expr":"sum(rate(rest_client_requests_total{cluster=\"$cluster\",
|
||||
job=\"kube-scheduler\", instance=~\"$instance\",code=~\"3..\"}[$__rate_interval]))","legendFormat":"3xx"},{"datasource":{"type":"prometheus","uid":"${datasource}"},"expr":"sum(rate(rest_client_requests_total{cluster=\"$cluster\",
|
||||
job=\"kube-scheduler\", instance=~\"$instance\",code=~\"4..\"}[$__rate_interval]))","legendFormat":"4xx"},{"datasource":{"type":"prometheus","uid":"${datasource}"},"expr":"sum(rate(rest_client_requests_total{cluster=\"$cluster\",
|
||||
job=\"kube-scheduler\", instance=~\"$instance\",code=~\"5..\"}[$__rate_interval]))","legendFormat":"5xx"}],"title":"Kube
|
||||
API Request Rate","type":"timeseries"},{"datasource":{"type":"datasource","uid":"--
|
||||
Mixed --"},"fieldConfig":{"defaults":{"custom":{"fillOpacity":10,"showPoints":"never","spanNulls":true},"unit":"ops"}},"gridPos":{"h":7,"w":16,"x":8,"y":7},"id":5,"interval":"1m","options":{"legend":{"asTable":true,"calcs":["lastNotNull"],"displayMode":"table","placement":"right","showLegend":true},"tooltip":{"mode":"single"}},"pluginVersion":"v11.4.0","targets":[{"datasource":{"type":"prometheus","uid":"${datasource}"},"expr":"histogram_quantile(0.99,
|
||||
sum(rate(rest_client_request_duration_seconds_bucket{cluster=\"$cluster\", job=\"kube-scheduler\",
|
||||
instance=~\"$instance\", verb=\"POST\"}[$__rate_interval])) by (verb, le))","legendFormat":"{{verb}}"}],"title":"Post
|
||||
Request Latency 99th Quantile","type":"timeseries"},{"datasource":{"type":"datasource","uid":"--
|
||||
Mixed --"},"fieldConfig":{"defaults":{"custom":{"fillOpacity":10,"showPoints":"never","spanNulls":true},"unit":"s"}},"gridPos":{"h":7,"w":24,"x":0,"y":14},"id":6,"interval":"1m","options":{"legend":{"asTable":true,"calcs":["lastNotNull"],"displayMode":"table","placement":"right","showLegend":true},"tooltip":{"mode":"single"}},"pluginVersion":"v11.4.0","targets":[{"datasource":{"type":"prometheus","uid":"${datasource}"},"expr":"histogram_quantile(0.99,
|
||||
sum(rate(rest_client_request_duration_seconds_bucket{cluster=\"$cluster\", job=\"kube-scheduler\",
|
||||
instance=~\"$instance\", verb=\"GET\"}[$__rate_interval])) by (verb, le))","legendFormat":"{{verb}}"}],"title":"Get
|
||||
Request Latency 99th Quantile","type":"timeseries"},{"datasource":{"type":"datasource","uid":"--
|
||||
Mixed --"},"fieldConfig":{"defaults":{"custom":{"fillOpacity":10,"showPoints":"never","spanNulls":true},"unit":"bytes"}},"gridPos":{"h":7,"w":8,"x":0,"y":21},"id":7,"interval":"1m","options":{"legend":{"asTable":true,"calcs":["lastNotNull"],"displayMode":"table","placement":"right","showLegend":true},"tooltip":{"mode":"single"}},"pluginVersion":"v11.4.0","targets":[{"datasource":{"type":"prometheus","uid":"${datasource}"},"expr":"process_resident_memory_bytes{cluster=\"$cluster\",
|
||||
job=\"kube-scheduler\", instance=~\"$instance\"}","legendFormat":"{{instance}}"}],"title":"Memory","type":"timeseries"},{"datasource":{"type":"datasource","uid":"--
|
||||
Mixed --"},"fieldConfig":{"defaults":{"custom":{"fillOpacity":10,"showPoints":"never","spanNulls":true},"unit":"short"}},"gridPos":{"h":7,"w":8,"x":8,"y":21},"id":8,"interval":"1m","options":{"legend":{"asTable":true,"calcs":["lastNotNull"],"displayMode":"table","placement":"right","showLegend":true},"tooltip":{"mode":"single"}},"pluginVersion":"v11.4.0","targets":[{"datasource":{"type":"prometheus","uid":"${datasource}"},"expr":"rate(process_cpu_seconds_total{cluster=\"$cluster\",
|
||||
job=\"kube-scheduler\", instance=~\"$instance\"}[$__rate_interval])","legendFormat":"{{instance}}"}],"title":"CPU
|
||||
usage","type":"timeseries"},{"datasource":{"type":"datasource","uid":"-- Mixed
|
||||
--"},"fieldConfig":{"defaults":{"custom":{"fillOpacity":10,"showPoints":"never","spanNulls":true},"unit":"short"}},"gridPos":{"h":7,"w":8,"x":16,"y":21},"id":9,"interval":"1m","options":{"legend":{"asTable":true,"calcs":["lastNotNull"],"displayMode":"table","placement":"right","showLegend":true},"tooltip":{"mode":"single"}},"pluginVersion":"v11.4.0","targets":[{"datasource":{"type":"prometheus","uid":"${datasource}"},"expr":"go_goroutines{cluster=\"$cluster\",
|
||||
job=\"kube-scheduler\",instance=~\"$instance\"}","legendFormat":"{{instance}}"}],"title":"Goroutines","type":"timeseries"}],"refresh":"10s","schemaVersion":39,"tags":["kubernetes-mixin"],"templating":{"list":[{"current":{"selected":true,"text":"default","value":"default"},"hide":0,"label":"Data
|
||||
source","name":"datasource","query":"prometheus","regex":"","type":"datasource"},{"datasource":{"type":"prometheus","uid":"${datasource}"},"hide":2,"label":"cluster","name":"cluster","query":"label_values(up{job=\"kube-scheduler\"},
|
||||
cluster)","refresh":2,"sort":1,"type":"query","allValue":".*"},{"allValue":".+","datasource":{"type":"prometheus","uid":"${datasource}"},"hide":0,"includeAll":true,"label":"instance","name":"instance","query":"label_values(up{job=\"kube-scheduler\",
|
||||
cluster=\"$cluster\"}, instance)","refresh":2,"type":"query"}]},"time":{"from":"now-1h","to":"now"},"timezone":
|
||||
"utc","title":"Kubernetes / Scheduler","uid":"2e6b6a3b4bddf1427b3a55aa1311c656"}'
|
||||
kind: ConfigMap
|
||||
metadata:
|
||||
annotations:
|
||||
meta.helm.sh/release-name: kube-prometheus-stack
|
||||
meta.helm.sh/release-namespace: monitoring
|
||||
labels:
|
||||
app: kube-prometheus-stack-grafana
|
||||
app.kubernetes.io/instance: kube-prometheus-stack
|
||||
app.kubernetes.io/managed-by: Helm
|
||||
app.kubernetes.io/part-of: kube-prometheus-stack
|
||||
app.kubernetes.io/version: 83.2.0
|
||||
chart: kube-prometheus-stack-83.2.0
|
||||
grafana_dashboard: '1'
|
||||
heritage: Helm
|
||||
release: kube-prometheus-stack
|
||||
name: kube-prometheus-stack-scheduler
|
||||
namespace: monitoring
|
||||
|
||||
+1224
-1561
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,60 @@
|
||||
# Valores del release Helm kube-prometheus-stack (chart 83.2.0, ns monitoring).
|
||||
#
|
||||
# Este namespace NO está bajo ArgoCD: se gestiona a mano. Hasta el 2026-07-21
|
||||
# estos valores solo vivían dentro del release, así que no había forma de saber
|
||||
# cómo estaba configurado sin interrogar a Helm. Este fichero es la fuente de
|
||||
# verdad; para aplicarlo:
|
||||
#
|
||||
# helm upgrade kube-prometheus-stack prometheus-community/kube-prometheus-stack \
|
||||
# --version 83.2.0 -n monitoring -f values-kube-prometheus-stack.yaml
|
||||
#
|
||||
# Revisión 2 (2026-07-21) silenció 4 alertas que llevaban meses disparadas sin
|
||||
# significar nada — ver el bloque kubeControllerManager/kubeScheduler/kubeProxy
|
||||
# y defaultRules.disabled al final.
|
||||
alertmanager:
|
||||
enabled: false
|
||||
defaultRules:
|
||||
create: true
|
||||
disabled:
|
||||
PrometheusNotConnectedToAlertmanagers: true
|
||||
rules:
|
||||
alertmanager: false
|
||||
grafana:
|
||||
adminPassword: admin
|
||||
grafana.ini:
|
||||
server:
|
||||
root_url: https://grafana.chemavx.xyz
|
||||
ingress:
|
||||
annotations:
|
||||
cert-manager.io/cluster-issuer: letsencrypt-prod
|
||||
traefik.ingress.kubernetes.io/router.entrypoints: websecure
|
||||
enabled: true
|
||||
hosts:
|
||||
- grafana.chemavx.xyz
|
||||
ingressClassName: traefik
|
||||
tls:
|
||||
- hosts:
|
||||
- grafana.chemavx.xyz
|
||||
secretName: grafana-tls
|
||||
persistence:
|
||||
enabled: true
|
||||
size: 5Gi
|
||||
storageClassName: local-path
|
||||
kubeControllerManager:
|
||||
enabled: false
|
||||
kubeProxy:
|
||||
enabled: false
|
||||
kubeScheduler:
|
||||
enabled: false
|
||||
prometheus:
|
||||
prometheusSpec:
|
||||
retention: 30d
|
||||
storageSpec:
|
||||
volumeClaimTemplate:
|
||||
spec:
|
||||
accessModes:
|
||||
- ReadWriteOnce
|
||||
resources:
|
||||
requests:
|
||||
storage: 20Gi
|
||||
storageClassName: local-path
|
||||
Reference in New Issue
Block a user