From c33913e481fd09af34d98b0b2862008c4096c84e Mon Sep 17 00:00:00 2001 From: chemavx Date: Sat, 25 Jul 2026 08:33:29 +0000 Subject: [PATCH] monitoring: alertar sobre la cuota y la papelera de MEGA MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit El incidente del 2026-07-25 (MEGA lleno -> dos CronJobs de backup caídos) solo se detectó por el síntoma. La causa llevaba meses creciendo sin que nada la mirase. - backup-system/mega-quota-exporter: Deployment que sondea `rclone about` y `rclone size` cada 15 min y los sirve a Prometheus vía ServiceMonitor. mega_trash_bytes (usado - visible) es la métrica que habría cazado esto. - monitoring: dos reglas de Grafana, cuota > 80% y papelera > 2GB. Co-Authored-By: Claude Opus 5 --- backup-system/mega-quota-exporter.yaml | 221 +++++++++++++++++++++ monitoring/configmap-grafana-alerting.yaml | 90 +++++++++ 2 files changed, 311 insertions(+) create mode 100644 backup-system/mega-quota-exporter.yaml diff --git a/backup-system/mega-quota-exporter.yaml b/backup-system/mega-quota-exporter.yaml new file mode 100644 index 0000000..cef7e32 --- /dev/null +++ b/backup-system/mega-quota-exporter.yaml @@ -0,0 +1,221 @@ +# Exporter de la cuota de MEGA. +# +# Contexto: el 2026-07-25 la papelera de MEGA (que cuenta cuota) llenó los 20GB +# de la cuenta y tumbó rclone-mega-backup y reddit-intel-backup con "Request +# over quota". El aviso llegó del síntoma (jobs fallidos), no de la causa, que +# llevaba meses engordando en silencio. Esto expone la cuota a Prometheus para +# poder alertar con días de margen. +# +# mega_trash_bytes = usado - visible: es la métrica que habría cazado aquello. +# Con --mega-hard-delete en las rotaciones debería quedarse cerca de 0; si +# vuelve a crecer, es que algún borrado se está yendo a la papelera otra vez. +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: mega-quota-exporter + namespace: backup-system +data: + export.sh: | + #!/bin/sh + # Imagen BusyBox (rclone) — sin GNU-ismos. + set -u + + CONF=/rclone/rclone.conf + OUT=/metrics/metrics + INTERVAL="${INTERVAL:-900}" + + # Extrae el entero que sigue a "clave". Hay que anclar en la clave y no + # limpiar la línea entera: `rclone size --json` saca {"count":N,"bytes":M} + # en una sola línea y un sed global concatenaría los dos números. + num() { sed -n 's/.*"'"$1"'"[^0-9]*\([0-9][0-9]*\).*/\1/p' "$2" 2>/dev/null | head -1; } + + # Últimos valores buenos: si un scrape falla preferimos republicarlos con + # success=0 antes que dejar la serie en NoData y disparar un falso positivo. + L_TOTAL=""; L_USED=""; L_FREE=""; L_VB=""; L_VC="" + + while true; do + OK=1 + rclone about mega: --json --config "$CONF" > /tmp/about.json 2>/tmp/about.err || OK=0 + rclone size mega: --json --config "$CONF" > /tmp/size.json 2>/tmp/size.err || OK=0 + + TOTAL=$(num total /tmp/about.json) + USED=$(num used /tmp/about.json) + FREE=$(num free /tmp/about.json) + VB=$(num bytes /tmp/size.json) + VC=$(num count /tmp/size.json) + + for v in "$TOTAL" "$USED" "$FREE" "$VB" "$VC"; do + [ -n "$v" ] || OK=0 + done + + if [ "$OK" = "1" ]; then + L_TOTAL="$TOTAL"; L_USED="$USED"; L_FREE="$FREE"; L_VB="$VB"; L_VC="$VC" + else + echo "[$(date '+%Y-%m-%d %H:%M:%S')] scrape fallido: $(cat /tmp/about.err /tmp/size.err 2>/dev/null | tr '\n' ' ')" + fi + + { + echo "# HELP mega_quota_scrape_success 1 si el ultimo sondeo a MEGA funciono." + echo "# TYPE mega_quota_scrape_success gauge" + echo "mega_quota_scrape_success $OK" + echo "# HELP mega_quota_last_scrape_timestamp_seconds Momento del ultimo sondeo." + echo "# TYPE mega_quota_last_scrape_timestamp_seconds gauge" + echo "mega_quota_last_scrape_timestamp_seconds $(date +%s)" + + if [ -n "$L_TOTAL" ]; then + TRASH=$((L_USED - L_VB)) + [ "$TRASH" -lt 0 ] && TRASH=0 + echo "# HELP mega_quota_bytes_total Cuota contratada en MEGA." + echo "# TYPE mega_quota_bytes_total gauge" + echo "mega_quota_bytes_total $L_TOTAL" + echo "# HELP mega_quota_bytes_used Espacio consumido (incluye la papelera)." + echo "# TYPE mega_quota_bytes_used gauge" + echo "mega_quota_bytes_used $L_USED" + echo "# HELP mega_quota_bytes_free Espacio libre segun MEGA." + echo "# TYPE mega_quota_bytes_free gauge" + echo "mega_quota_bytes_free $L_FREE" + echo "# HELP mega_visible_bytes Tamano de los ficheros visibles." + echo "# TYPE mega_visible_bytes gauge" + echo "mega_visible_bytes $L_VB" + echo "# HELP mega_visible_objects Numero de ficheros visibles." + echo "# TYPE mega_visible_objects gauge" + echo "mega_visible_objects $L_VC" + echo "# HELP mega_trash_bytes Estimacion de la papelera (usado - visible)." + echo "# TYPE mega_trash_bytes gauge" + echo "mega_trash_bytes $TRASH" + fi + } > "${OUT}.tmp" && mv "${OUT}.tmp" "$OUT" + + sleep "$INTERVAL" + done + nginx.conf: | + worker_processes 1; + error_log /dev/stderr warn; + pid /tmp/nginx.pid; + events { worker_connections 64; } + http { + access_log off; + client_body_temp_path /tmp/client_body; + proxy_temp_path /tmp/proxy; + fastcgi_temp_path /tmp/fastcgi; + uwsgi_temp_path /tmp/uwsgi; + scgi_temp_path /tmp/scgi; + server { + listen 8080; + # El fichero no tiene extension: sin esto nginx lo sirve como + # application/octet-stream y Prometheus se queja del formato. + default_type text/plain; + location = /metrics { alias /metrics/metrics; } + location / { return 404; } + } + } +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: mega-quota-exporter + namespace: backup-system +spec: + replicas: 1 + selector: + matchLabels: + app: mega-quota-exporter + template: + metadata: + labels: + app: mega-quota-exporter + spec: + containers: + - name: rclone + # OJO: imagen BusyBox — el script debe evitar GNU-ismos + image: docker.io/rclone/rclone:1.74.3 + command: ["/bin/sh", "/scripts/export.sh"] + env: + - name: INTERVAL + value: "900" + resources: + requests: + cpu: 10m + memory: 32Mi + limits: + cpu: 200m + memory: 128Mi + volumeMounts: + - mountPath: /rclone + name: rclone-conf + - mountPath: /scripts + name: scripts + - mountPath: /metrics + name: metrics + - name: nginx + image: docker.io/library/nginx:1.27-alpine + ports: + - name: metrics + containerPort: 8080 + resources: + requests: + cpu: 10m + memory: 16Mi + limits: + cpu: 100m + memory: 64Mi + volumeMounts: + - mountPath: /etc/nginx/nginx.conf + name: scripts + subPath: nginx.conf + - mountPath: /metrics + name: metrics + readOnly: true + readinessProbe: + httpGet: + path: /metrics + port: metrics + initialDelaySeconds: 10 + periodSeconds: 20 + volumes: + # RCLONE_CONF (Infisical /backup-system) contiene mega+b2 + - name: rclone-conf + secret: + secretName: rclone-conf-infisical + items: + - key: RCLONE_CONF + path: rclone.conf + - name: scripts + configMap: + name: mega-quota-exporter + defaultMode: 0755 + - name: metrics + emptyDir: {} +--- +apiVersion: v1 +kind: Service +metadata: + name: mega-quota-exporter + namespace: backup-system + labels: + app: mega-quota-exporter +spec: + selector: + app: mega-quota-exporter + ports: + - name: metrics + port: 8080 + targetPort: metrics +--- +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + name: mega-quota-exporter + namespace: backup-system + labels: + # Requerido por serviceMonitorSelector de kube-prometheus-stack + release: kube-prometheus-stack +spec: + selector: + matchLabels: + app: mega-quota-exporter + endpoints: + - port: metrics + path: /metrics + interval: 5m diff --git a/monitoring/configmap-grafana-alerting.yaml b/monitoring/configmap-grafana-alerting.yaml index f111299..d745ad4 100644 --- a/monitoring/configmap-grafana-alerting.yaml +++ b/monitoring/configmap-grafana-alerting.yaml @@ -246,3 +246,93 @@ data: expression: B conditions: - evaluator: {params: [0], type: gt} + + # Las dos reglas siguientes leen las métricas de + # backup-system/mega-quota-exporter (sondea MEGA cada 15 min). + # El 2026-07-25 la cuota de MEGA se llenó y tumbó dos CronJobs de + # backup; no había ninguna alerta que lo viera venir. + - uid: homelab-mega-quota-high + title: "Cuota MEGA > 80%" + condition: C + for: 30m + noDataState: NoData + execErrState: Error + annotations: + summary: "☁️ Cuota de MEGA al {{ humanizePercentage $values.B.Value }}\nSi llega al 100% fallan los backups a MEGA." + description: "El uso de la cuenta de MEGA supera el 80% de los 20GB. Revisar retenciones y papelera (mega_trash_bytes)." + labels: + severity: warning + isPaused: false + data: + - refId: A + relativeTimeRange: {from: 3600, to: 0} + datasourceUid: prometheus + model: + editorMode: code + expr: "mega_quota_bytes_used / mega_quota_bytes_total" + instant: true + refId: A + - refId: B + relativeTimeRange: {from: 0, to: 0} + datasourceUid: "-100" + model: + type: reduce + refId: B + expression: A + reducer: last + settings: + mode: "" + - refId: C + relativeTimeRange: {from: 0, to: 0} + datasourceUid: "-100" + model: + type: threshold + refId: C + expression: B + conditions: + - evaluator: {params: [0.8], type: gt} + + # Causa raíz del incidente del 2026-07-25: la papelera de MEGA cuenta + # cuota y las rotaciones la llenaron durante meses sin que se notara. + # Con --mega-hard-delete debería quedarse pegada a 0; si pasa de 2GB + # es que algún borrado se está yendo a la papelera otra vez. + - uid: homelab-mega-trash-growing + title: "Papelera de MEGA creciendo" + condition: C + for: 1h + noDataState: NoData + execErrState: Error + annotations: + summary: "🗑️ Papelera de MEGA: {{ humanize1024 $values.B.Value }}B\nAlgún borrado no está usando --mega-hard-delete." + description: "mega_trash_bytes (usado - visible) supera los 2GB. La papelera de MEGA consume cuota; revisar los scripts de backup-system." + labels: + severity: warning + isPaused: false + data: + - refId: A + relativeTimeRange: {from: 3600, to: 0} + datasourceUid: prometheus + model: + editorMode: code + expr: "mega_trash_bytes" + instant: true + refId: A + - refId: B + relativeTimeRange: {from: 0, to: 0} + datasourceUid: "-100" + model: + type: reduce + refId: B + expression: A + reducer: last + settings: + mode: "" + - refId: C + relativeTimeRange: {from: 0, to: 0} + datasourceUid: "-100" + model: + type: threshold + refId: C + expression: B + conditions: + - evaluator: {params: [2147483648], type: gt}