chemavxandClaude Opus 5
c33913e481
El incidente del 2026-07-25 (MEGA lleno -> dos CronJobs de backup caídos) solo se detectó por el síntoma. La causa llevaba meses creciendo sin que nada la mirase. - backup-system/mega-quota-exporter: Deployment que sondea `rclone about` y `rclone size` cada 15 min y los sirve a Prometheus vía ServiceMonitor. mega_trash_bytes (usado - visible) es la métrica que habría cazado esto. - monitoring: dos reglas de Grafana, cuota > 80% y papelera > 2GB. Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
339 lines
12 KiB
YAML
339 lines
12 KiB
YAML
apiVersion: v1
|
|
kind: ConfigMap
|
|
metadata:
|
|
name: grafana-alerting
|
|
namespace: monitoring
|
|
data:
|
|
contact-points.yaml: |
|
|
apiVersion: 1
|
|
contactPoints:
|
|
- orgId: 1
|
|
name: Telegram Homelab
|
|
receivers:
|
|
- uid: telegram-homelab
|
|
type: telegram
|
|
settings:
|
|
bottoken: "${TELEGRAM_BOT_TOKEN}"
|
|
chatid: "5138407666"
|
|
parseMode: HTML
|
|
message: "{{ range .Alerts }}{{ if eq .Status \"firing\" }}{{ .Annotations.summary }}\n{{ else }}✅ Resuelto: {{ .Labels.alertname }}\n{{ end }}{{ end }}"
|
|
disableResolveMessage: false
|
|
|
|
notification-policy.yaml: |
|
|
apiVersion: 1
|
|
policies:
|
|
- orgId: 1
|
|
receiver: Telegram Homelab
|
|
group_by:
|
|
- alertname
|
|
- namespace
|
|
- pod
|
|
group_wait: 30s
|
|
group_interval: 5m
|
|
repeat_interval: 4h
|
|
routes: []
|
|
|
|
alert-rules.yaml: |
|
|
apiVersion: 1
|
|
groups:
|
|
- orgId: 1
|
|
name: homelab-infra
|
|
folder: Homelab Alerts
|
|
interval: 1m
|
|
rules:
|
|
|
|
- uid: homelab-crashloop
|
|
title: "Pod CrashLoopBackOff"
|
|
condition: C
|
|
for: 1m
|
|
noDataState: OK
|
|
execErrState: Error
|
|
annotations:
|
|
summary: "🔄 CrashLoop: {{ $labels.pod }}\nNamespace: {{ $labels.namespace }}"
|
|
description: "Container {{ $labels.container }} ha entrado en CrashLoopBackOff."
|
|
labels:
|
|
severity: critical
|
|
isPaused: false
|
|
data:
|
|
- refId: A
|
|
relativeTimeRange: {from: 300, to: 0}
|
|
datasourceUid: prometheus
|
|
model:
|
|
editorMode: code
|
|
expr: kube_pod_container_status_waiting_reason{reason="CrashLoopBackOff"}
|
|
instant: true
|
|
refId: A
|
|
- refId: B
|
|
relativeTimeRange: {from: 0, to: 0}
|
|
datasourceUid: "-100"
|
|
model:
|
|
type: reduce
|
|
refId: B
|
|
expression: A
|
|
reducer: last
|
|
settings:
|
|
mode: ""
|
|
- refId: C
|
|
relativeTimeRange: {from: 0, to: 0}
|
|
datasourceUid: "-100"
|
|
model:
|
|
type: threshold
|
|
refId: C
|
|
expression: B
|
|
conditions:
|
|
- evaluator: {params: [0], type: gt}
|
|
|
|
- uid: homelab-backup-job-failed
|
|
title: "Backup job fallido"
|
|
condition: C
|
|
for: 1m
|
|
noDataState: OK
|
|
execErrState: Error
|
|
annotations:
|
|
summary: "🗄️ Backup FALLIDO: {{ $labels.job_name }}\nNamespace: {{ $labels.namespace }}"
|
|
description: "El job {{ $labels.job_name }} del sistema de backups ha fallado. Cubre los CronJobs backup, rclone-mega-backup y backup-verify."
|
|
labels:
|
|
severity: critical
|
|
isPaused: false
|
|
data:
|
|
- refId: A
|
|
relativeTimeRange: {from: 300, to: 0}
|
|
datasourceUid: prometheus
|
|
model:
|
|
editorMode: code
|
|
expr: "kube_job_status_failed{namespace=\"backup-system\"}"
|
|
instant: true
|
|
refId: A
|
|
- refId: B
|
|
relativeTimeRange: {from: 0, to: 0}
|
|
datasourceUid: "-100"
|
|
model:
|
|
type: reduce
|
|
refId: B
|
|
expression: A
|
|
reducer: last
|
|
settings:
|
|
mode: ""
|
|
- refId: C
|
|
relativeTimeRange: {from: 0, to: 0}
|
|
datasourceUid: "-100"
|
|
model:
|
|
type: threshold
|
|
refId: C
|
|
expression: B
|
|
conditions:
|
|
- evaluator: {params: [0], type: gt}
|
|
|
|
- uid: homelab-disk-high
|
|
title: "Disco > 80%"
|
|
condition: C
|
|
for: 5m
|
|
noDataState: NoData
|
|
execErrState: Error
|
|
annotations:
|
|
summary: "💾 Disco casi lleno: {{ $labels.mountpoint }}\nUso: {{ humanizePercentage $values.B.Value }}"
|
|
description: "Filesystem {{ $labels.mountpoint }} supera el 80% de uso."
|
|
labels:
|
|
severity: warning
|
|
isPaused: false
|
|
data:
|
|
- refId: A
|
|
relativeTimeRange: {from: 300, to: 0}
|
|
datasourceUid: prometheus
|
|
model:
|
|
editorMode: code
|
|
expr: "(node_filesystem_size_bytes{fstype!~\"tmpfs|overlay|squashfs|devtmpfs\"} - node_filesystem_avail_bytes{fstype!~\"tmpfs|overlay|squashfs|devtmpfs\"}) / node_filesystem_size_bytes{fstype!~\"tmpfs|overlay|squashfs|devtmpfs\"}"
|
|
instant: true
|
|
refId: A
|
|
- refId: B
|
|
relativeTimeRange: {from: 0, to: 0}
|
|
datasourceUid: "-100"
|
|
model:
|
|
type: reduce
|
|
refId: B
|
|
expression: A
|
|
reducer: last
|
|
settings:
|
|
mode: ""
|
|
- refId: C
|
|
relativeTimeRange: {from: 0, to: 0}
|
|
datasourceUid: "-100"
|
|
model:
|
|
type: threshold
|
|
refId: C
|
|
expression: B
|
|
conditions:
|
|
- evaluator: {params: [0.8], type: gt}
|
|
|
|
- uid: homelab-ram-high
|
|
title: "RAM > 85%"
|
|
condition: C
|
|
for: 5m
|
|
noDataState: NoData
|
|
execErrState: Error
|
|
annotations:
|
|
summary: "🧠 RAM alta: {{ $labels.instance }}\nUso: {{ humanizePercentage $values.B.Value }}"
|
|
description: "Uso de RAM supera el 85% en {{ $labels.instance }}."
|
|
labels:
|
|
severity: warning
|
|
isPaused: false
|
|
data:
|
|
- refId: A
|
|
relativeTimeRange: {from: 300, to: 0}
|
|
datasourceUid: prometheus
|
|
model:
|
|
editorMode: code
|
|
expr: "(node_memory_MemTotal_bytes - node_memory_MemAvailable_bytes) / node_memory_MemTotal_bytes"
|
|
instant: true
|
|
refId: A
|
|
- refId: B
|
|
relativeTimeRange: {from: 0, to: 0}
|
|
datasourceUid: "-100"
|
|
model:
|
|
type: reduce
|
|
refId: B
|
|
expression: A
|
|
reducer: last
|
|
settings:
|
|
mode: ""
|
|
- refId: C
|
|
relativeTimeRange: {from: 0, to: 0}
|
|
datasourceUid: "-100"
|
|
model:
|
|
type: threshold
|
|
refId: C
|
|
expression: B
|
|
conditions:
|
|
- evaluator: {params: [0.85], type: gt}
|
|
|
|
- uid: homelab-pod-failed
|
|
title: "Pod Failed/Unknown"
|
|
condition: C
|
|
for: 3m
|
|
noDataState: OK
|
|
execErrState: Error
|
|
annotations:
|
|
summary: "🚨 Pod caído: {{ $labels.pod }}\nNamespace: {{ $labels.namespace }}\nEstado: {{ $labels.phase }}"
|
|
description: "Pod {{ $labels.namespace }}/{{ $labels.pod }} lleva más de 3 minutos en estado {{ $labels.phase }}."
|
|
labels:
|
|
severity: warning
|
|
isPaused: false
|
|
data:
|
|
- refId: A
|
|
relativeTimeRange: {from: 300, to: 0}
|
|
datasourceUid: prometheus
|
|
model:
|
|
editorMode: code
|
|
expr: "kube_pod_status_phase{phase!~\"Running|Succeeded\"} == 1"
|
|
instant: true
|
|
refId: A
|
|
- refId: B
|
|
relativeTimeRange: {from: 0, to: 0}
|
|
datasourceUid: "-100"
|
|
model:
|
|
type: reduce
|
|
refId: B
|
|
expression: A
|
|
reducer: last
|
|
settings:
|
|
mode: ""
|
|
- refId: C
|
|
relativeTimeRange: {from: 0, to: 0}
|
|
datasourceUid: "-100"
|
|
model:
|
|
type: threshold
|
|
refId: C
|
|
expression: B
|
|
conditions:
|
|
- evaluator: {params: [0], type: gt}
|
|
|
|
# Las dos reglas siguientes leen las métricas de
|
|
# backup-system/mega-quota-exporter (sondea MEGA cada 15 min).
|
|
# El 2026-07-25 la cuota de MEGA se llenó y tumbó dos CronJobs de
|
|
# backup; no había ninguna alerta que lo viera venir.
|
|
- uid: homelab-mega-quota-high
|
|
title: "Cuota MEGA > 80%"
|
|
condition: C
|
|
for: 30m
|
|
noDataState: NoData
|
|
execErrState: Error
|
|
annotations:
|
|
summary: "☁️ Cuota de MEGA al {{ humanizePercentage $values.B.Value }}\nSi llega al 100% fallan los backups a MEGA."
|
|
description: "El uso de la cuenta de MEGA supera el 80% de los 20GB. Revisar retenciones y papelera (mega_trash_bytes)."
|
|
labels:
|
|
severity: warning
|
|
isPaused: false
|
|
data:
|
|
- refId: A
|
|
relativeTimeRange: {from: 3600, to: 0}
|
|
datasourceUid: prometheus
|
|
model:
|
|
editorMode: code
|
|
expr: "mega_quota_bytes_used / mega_quota_bytes_total"
|
|
instant: true
|
|
refId: A
|
|
- refId: B
|
|
relativeTimeRange: {from: 0, to: 0}
|
|
datasourceUid: "-100"
|
|
model:
|
|
type: reduce
|
|
refId: B
|
|
expression: A
|
|
reducer: last
|
|
settings:
|
|
mode: ""
|
|
- refId: C
|
|
relativeTimeRange: {from: 0, to: 0}
|
|
datasourceUid: "-100"
|
|
model:
|
|
type: threshold
|
|
refId: C
|
|
expression: B
|
|
conditions:
|
|
- evaluator: {params: [0.8], type: gt}
|
|
|
|
# Causa raíz del incidente del 2026-07-25: la papelera de MEGA cuenta
|
|
# cuota y las rotaciones la llenaron durante meses sin que se notara.
|
|
# Con --mega-hard-delete debería quedarse pegada a 0; si pasa de 2GB
|
|
# es que algún borrado se está yendo a la papelera otra vez.
|
|
- uid: homelab-mega-trash-growing
|
|
title: "Papelera de MEGA creciendo"
|
|
condition: C
|
|
for: 1h
|
|
noDataState: NoData
|
|
execErrState: Error
|
|
annotations:
|
|
summary: "🗑️ Papelera de MEGA: {{ humanize1024 $values.B.Value }}B\nAlgún borrado no está usando --mega-hard-delete."
|
|
description: "mega_trash_bytes (usado - visible) supera los 2GB. La papelera de MEGA consume cuota; revisar los scripts de backup-system."
|
|
labels:
|
|
severity: warning
|
|
isPaused: false
|
|
data:
|
|
- refId: A
|
|
relativeTimeRange: {from: 3600, to: 0}
|
|
datasourceUid: prometheus
|
|
model:
|
|
editorMode: code
|
|
expr: "mega_trash_bytes"
|
|
instant: true
|
|
refId: A
|
|
- refId: B
|
|
relativeTimeRange: {from: 0, to: 0}
|
|
datasourceUid: "-100"
|
|
model:
|
|
type: reduce
|
|
refId: B
|
|
expression: A
|
|
reducer: last
|
|
settings:
|
|
mode: ""
|
|
- refId: C
|
|
relativeTimeRange: {from: 0, to: 0}
|
|
datasourceUid: "-100"
|
|
model:
|
|
type: threshold
|
|
refId: C
|
|
expression: B
|
|
conditions:
|
|
- evaluator: {params: [2147483648], type: gt}
|