Files
k8s-manifests/backup-system/configmap-backup-scripts.yaml
T
chemavxandClaude Fable 5 c364d46ac6 feat(backup-system): segundo offsite Backblaze B2 (crown jewels)
- b2-crown-jewels.sh: sync diario 04:00 de las 2 copias mas recientes de
  cada servicio irremplazable (todo menos uptime-kuma) a
  b2:chemavx-k3s-backups/crown-jewels/ (~2GB, tier gratuito).
- reddit-intel.sh sube tambien a B2 (retencion 3d).
- verify.sh comprueba Mega (completo) + B2 (crown jewels).
- Config via Secret out-of-band rclone-conf-b2 (mega+b2, kubectl, NO git);
  consolidar en Infisical RCLONE_CONF cuando se anada [b2] por UI (la
  identity del operator es read-only por API).

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
2026-07-15 17:24:02 +00:00

388 lines
17 KiB
YAML

apiVersion: v1
data:
backup.sh: |
#!/bin/bash
# Backup diario del homelab. Imagen: bitnami/kubectl (bash GNU, corre en chemavx-k8).
# Regla: cada artefacto se valida (gzip -t + tamaño mínimo) — un tar vacío
# como el bug histórico de openclaw (104 bytes) cuenta como FALLO y el job sale 1
# (→ alerta Grafana homelab-backup-job-failed → Telegram).
set -u
# kubectl usa el ServiceAccount in-cluster (backup-sa + ClusterRole backup-role).
# NO usar el k3s.yaml del host: su server es https://127.0.0.1:6443, inalcanzable
# desde la red del pod (el script antiguo "funcionaba" porque apuntaba KUBECONFIG
# a un fichero inexistente y kubectl caía en fallback al ServiceAccount).
unset KUBECONFIG
BACKUP_DIR="/data/backups/backups"
DATE=$(date +%Y-%m-%d_%H-%M-%S)
STORAGE=/var/lib/rancher/k3s/storage
FAIL=0
mkdir -p "$BACKUP_DIR"
log() { echo "[$(date '+%Y-%m-%d %H:%M:%S')] $*"; }
fail() { log "ERROR: $1 FAILED"; FAIL=$((FAIL + 1)); }
# Valida artefacto: gzip íntegro y tamaño >= mínimo esperado
check() { # $1=fichero $2=bytes_minimos $3=servicio
if gzip -t "$1" 2>/dev/null && [ "$(stat -c%s "$1" 2>/dev/null || echo 0)" -ge "$2" ]; then
log "$3 OK ($(du -h "$1" | cut -f1))"
else
rm -f "$1"
fail "$3 (artefacto corrupto o menor de $2 bytes)"
fi
}
# Resuelve la ruta local-path de un PVC en el host
pvpath() { # $1=ns $2=pvc
local pv
pv=$(kubectl get pvc -n "$1" "$2" -o jsonpath='{.spec.volumeName}' 2>/dev/null) || return 1
echo "$STORAGE/${pv}_$1_$2"
}
# Copia consistente de la ghost.db (VACUUM INTO vía node del pod) + tar del content
backup_ghost() { # $1=ns $2=deploy $3=prefijo
local f="$BACKUP_DIR/${3}-db_${DATE}.db.gz"
kubectl exec -i -n "$1" "deploy/$2" -- sh -c \
'rm -f /tmp/ghost-backup.db && NODE_PATH=/var/lib/ghost/current/node_modules node -' \
< /scripts/ghost-db-backup.js >/dev/null 2>&1 \
&& kubectl exec -n "$1" "deploy/$2" -- cat /tmp/ghost-backup.db 2>/dev/null | gzip > "$f"
kubectl exec -n "$1" "deploy/$2" -- rm -f /tmp/ghost-backup.db 2>/dev/null || true
check "$f" 200000 "${3}-db"
local c="$BACKUP_DIR/${3}-content_${DATE}.tar.gz"
kubectl exec -n "$1" "deploy/$2" -- tar czf - -C /var/lib/ghost/content . > "$c" 2>/dev/null
check "$c" 5000000 "${3}-content"
}
# === n8n (hostPath /data/n8n — incluye .n8n/config con la encryptionKey) ===
log "Backing up n8n..."
tar -czf "$BACKUP_DIR/n8n_${DATE}.tar.gz" -C /data/n8n . 2>/dev/null
check "$BACKUP_DIR/n8n_${DATE}.tar.gz" 10000000 n8n
# === openclaw (PVC local-path; el hostPath /data/openclaw era un stub vacío) ===
log "Backing up openclaw..."
OC=$(pvpath openclaw openclaw-pvc)
tar -czf "$BACKUP_DIR/openclaw_${DATE}.tar.gz" -C "$OC" . 2>/dev/null
check "$BACKUP_DIR/openclaw_${DATE}.tar.gz" 50000 openclaw
# === homarr ===
log "Backing up homarr..."
HOMARR_DB=$(pvpath homarr homarr-db-pvc)
HOMARR_REDIS=$(pvpath homarr homarr-redis-pvc)
mkdir -p /tmp/backup-homarr/db /tmp/backup-homarr/redis
cp "$HOMARR_DB/db.sqlite" /tmp/backup-homarr/db/ 2>/dev/null || true
cp "$HOMARR_REDIS/dump.rdb" /tmp/backup-homarr/redis/ 2>/dev/null || true
tar -czf "$BACKUP_DIR/homarr_${DATE}.tar.gz" -C /tmp/backup-homarr . 2>/dev/null
check "$BACKUP_DIR/homarr_${DATE}.tar.gz" 10000 homarr
rm -rf /tmp/backup-homarr
# === authentik PostgreSQL ===
log "Backing up authentik postgres..."
kubectl exec -n authentik postgresql-0 -- pg_dump -U authentik authentik 2>/dev/null | \
gzip > "$BACKUP_DIR/authentik-pg_${DATE}.sql.gz"
check "$BACKUP_DIR/authentik-pg_${DATE}.sql.gz" 10000 authentik-pg
# === grafana ===
log "Backing up grafana..."
GRAFANA=$(pvpath monitoring kube-prometheus-stack-grafana)
mkdir -p /tmp/backup-grafana
cp "$GRAFANA/grafana.db" /tmp/backup-grafana/ 2>/dev/null || true
tar -czf "$BACKUP_DIR/grafana_${DATE}.tar.gz" -C /tmp/backup-grafana . 2>/dev/null
check "$BACKUP_DIR/grafana_${DATE}.tar.gz" 100000 grafana
rm -rf /tmp/backup-grafana
# === k3s: state.db + token del server + /etc/rancher/k3s ===
# Sin el token, state.db NO arranca en un k3s nuevo (cadena de restauración).
log "Backing up k3s (state.db + token + /etc/rancher/k3s)..."
tar -czf "$BACKUP_DIR/k3s-db_${DATE}.tar.gz" \
/data/k3s-db/state.db* /host/k3s-token /host/etc-rancher-k3s 2>/dev/null
check "$BACKUP_DIR/k3s-db_${DATE}.tar.gz" 1000000 k3s-db
# === Gitea: sqlite .backup consistente + repos + conf (SIN gitea/packages, 6.4G de registry reconstruible por CI) ===
log "Backing up gitea..."
kubectl exec -n gitea gitea-0 -- sh -c \
'rm -f /data/gitea/gitea-backup.db && sqlite3 /data/gitea/gitea.db ".backup /data/gitea/gitea-backup.db"' 2>/dev/null \
&& kubectl exec -n gitea gitea-0 -- tar czf - -C /data \
git ssh gitea/conf gitea/jwt gitea/avatars gitea/repo-avatars gitea/gitea-backup.db \
> "$BACKUP_DIR/gitea_${DATE}.tar.gz" 2>/dev/null
kubectl exec -n gitea gitea-0 -- rm -f /data/gitea/gitea-backup.db 2>/dev/null || true
check "$BACKUP_DIR/gitea_${DATE}.tar.gz" 5000000 gitea
# === Ghost EN + ES (db consistente + content con imágenes/temas/redirects) ===
log "Backing up ghost-en..."
backup_ghost ghost-en ghost-en ghost-en
log "Backing up zona-exclusion..."
backup_ghost zona-exclusion zona-exclusion zona-exclusion
# === ResearchOwl (sqlite .backup vía python3 del pod) ===
log "Backing up researchowl..."
kubectl exec -i -n researchowl deploy/researchowl -- sh -c \
'rm -f /tmp/ro-backup.db && python3 -' < /scripts/sqlite-stdin-backup.py >/dev/null 2>&1 \
&& kubectl exec -n researchowl deploy/researchowl -- cat /tmp/ro-backup.db 2>/dev/null | \
gzip > "$BACKUP_DIR/researchowl_${DATE}.db.gz"
kubectl exec -n researchowl deploy/researchowl -- rm -f /tmp/ro-backup.db 2>/dev/null || true
check "$BACKUP_DIR/researchowl_${DATE}.db.gz" 5000000 researchowl
# === Roswell (corpus) PostgreSQL ===
log "Backing up roswell postgres..."
kubectl exec -n roswell deploy/postgres -- sh -c \
'PGPASSWORD="${POSTGRES_PASSWORD:-}" pg_dump -U "$POSTGRES_USER" "${POSTGRES_DB:-$POSTGRES_USER}"' 2>/dev/null | \
gzip > "$BACKUP_DIR/roswell-pg_${DATE}.sql.gz"
check "$BACKUP_DIR/roswell-pg_${DATE}.sql.gz" 5000 roswell-pg
# === Umami PostgreSQL ===
log "Backing up umami postgres..."
kubectl exec -n umami deploy/postgres -- sh -c \
'PGPASSWORD="${POSTGRES_PASSWORD:-}" pg_dump -U "$POSTGRES_USER" "${POSTGRES_DB:-$POSTGRES_USER}"' 2>/dev/null | \
gzip > "$BACKUP_DIR/umami-pg_${DATE}.sql.gz"
check "$BACKUP_DIR/umami-pg_${DATE}.sql.gz" 5000 umami-pg
# === Infisical PostgreSQL (RESTAURAR REQUIERE infisical-secrets — está dentro de k3s-db/state.db, ver RESTORE.md) ===
log "Backing up infisical postgres..."
kubectl exec -n infisical postgresql-0 -- sh -c \
'PGPASSWORD="$POSTGRES_PASSWORD" pg_dump -U "$POSTGRES_USER" infisicalDB' 2>/dev/null | \
gzip > "$BACKUP_DIR/infisical-pg_${DATE}.sql.gz"
check "$BACKUP_DIR/infisical-pg_${DATE}.sql.gz" 10000 infisical-pg
# === Trilium ===
log "Backing up trilium..."
kubectl exec -n trilium deploy/trilium -- tar czf - -C /home/node/trilium-data . \
> "$BACKUP_DIR/trilium_${DATE}.tar.gz" 2>/dev/null
check "$BACKUP_DIR/trilium_${DATE}.tar.gz" 500000 trilium
# === Vaultwarden (db + attachments + claves RSA; copia en frío ~02:00) ===
log "Backing up vaultwarden..."
VW=$(pvpath vaultwarden vaultwarden-pvc)
tar -czf "$BACKUP_DIR/vaultwarden_${DATE}.tar.gz" -C "$VW" . 2>/dev/null
check "$BACKUP_DIR/vaultwarden_${DATE}.tar.gz" 100000 vaultwarden
# === Uptime-Kuma (sqlite .backup vía CLI del pod; retención corta, es el artefacto gordo) ===
log "Backing up uptime-kuma..."
kubectl exec -n monitoring deploy/uptime-kuma -- sh -c \
'rm -f /tmp/kuma-backup.db && sqlite3 /app/data/kuma.db ".backup /tmp/kuma-backup.db"' 2>/dev/null \
&& kubectl exec -n monitoring deploy/uptime-kuma -- cat /tmp/kuma-backup.db 2>/dev/null | \
gzip > "$BACKUP_DIR/uptime-kuma_${DATE}.db.gz"
kubectl exec -n monitoring deploy/uptime-kuma -- rm -f /tmp/kuma-backup.db 2>/dev/null || true
check "$BACKUP_DIR/uptime-kuma_${DATE}.db.gz" 1000000 uptime-kuma
# === Filebrowser (solo su BD; los ficheros servidos van en home-master/imagefinder) ===
log "Backing up filebrowser db..."
FB=$(pvpath filebrowser filebrowser-db)
tar -czf "$BACKUP_DIR/filebrowser-db_${DATE}.tar.gz" -C "$FB" . 2>/dev/null
check "$BACKUP_DIR/filebrowser-db_${DATE}.tar.gz" 1000 filebrowser-db
# === Home del master (~/.claude memoria+skills, imagefinder, proyectos sueltos, RUNBOOK...) ===
# Excluye ~/.ssh (claves privadas, decisión explícita), caches y binarios reproducibles.
log "Backing up home-master..."
tar -czf "$BACKUP_DIR/home-master_${DATE}.tar.gz" -C /host/home-chemavx \
--exclude='./.ssh' --exclude='./.cache' --exclude='./.venvs' --exclude='./.npm' \
--exclude='./.nvm' --exclude='./bin' --exclude='./.docker' --exclude='./snap' \
--exclude='*/node_modules' --exclude='./.local/share/pnpm' \
--exclude='./.warp' --exclude='./.local/share/claude' --exclude='./.local/lib' \
. 2>/dev/null
check "$BACKUP_DIR/home-master_${DATE}.tar.gz" 50000000 home-master
# === Retención (7 por defecto; 3 para los artefactos gordos) ===
log "Cleaning old backups..."
retain() { # $1=prefijo $2=copias
find "$BACKUP_DIR" -maxdepth 1 \
\( -name "${1}_*.tar.gz" -o -name "${1}_*.sql.gz" -o -name "${1}_*.db.gz" \) | \
sort | head -n -"$2" | xargs -r rm -f
}
for p in n8n openclaw homarr authentik-pg grafana k3s-db gitea \
ghost-en-db ghost-en-content zona-exclusion-db zona-exclusion-content \
researchowl roswell-pg umami-pg infisical-pg trilium vaultwarden filebrowser-db; do
retain "$p" 7
done
retain uptime-kuma 3
retain home-master 3
if [ "$FAIL" -gt 0 ]; then
log "Backup terminado con $FAIL fallo(s) — el job sale con error para disparar la alerta."
exit 1
fi
log "Backup complete!"
ghost-db-backup.js: |
// Copia consistente de la ghost.db usando el driver sqlite3 del propio Ghost.
// Se ejecuta DENTRO del pod vía: kubectl exec -i ... node - (script por stdin)
const sqlite3 = require('sqlite3');
const src = '/var/lib/ghost/content/data/ghost.db';
const dst = '/tmp/ghost-backup.db';
const db = new sqlite3.Database(src, sqlite3.OPEN_READONLY, (err) => {
if (err) { console.error(err.message); process.exit(1); }
});
db.run("VACUUM INTO '" + dst + "'", (err) => {
if (err) { console.error(err.message); process.exit(1); }
db.close(() => console.log('ghost db backup ok'));
});
sqlite-stdin-backup.py: |
# Copia consistente de researchowl.db (API backup de sqlite3, stdlib).
# Se ejecuta DENTRO del pod vía: kubectl exec -i ... python3 -
import sqlite3
src = sqlite3.connect('file:/data/researchowl.db?mode=ro', uri=True)
dst = sqlite3.connect('/tmp/ro-backup.db')
with dst:
src.backup(dst)
dst.close()
src.close()
print('researchowl db backup ok')
rclone-mega.sh: |
#!/bin/bash
set -e
log() { echo "[$(date '+%Y-%m-%d %H:%M:%S')] $*"; }
log "Iniciando sync a Mega..."
log "Origen: /data/backups/backups"
log "Destino: mega:k3s-backups/"
# --exclude reddit-intel: ese subdir remoto lo escribe el CronJob de n97
# directamente; un sync sin exclude lo borraría (sync = espejo exacto).
rclone sync /data/backups/backups mega:k3s-backups/ \
--exclude "/reddit-intel/**" \
--config /rclone/rclone.conf \
--log-level INFO \
--stats 30s
log "Sync completado."
log "Archivos en Mega:"
rclone ls mega:k3s-backups/ --config /rclone/rclone.conf
b2-crown-jewels.sh: |
#!/bin/sh
# Segunda copia offsite (Backblaze B2): "crown jewels" — las 2 copias más
# recientes de cada servicio irremplazable (~2GB, cabe en los 10GB gratis).
# Todo menos uptime-kuma (622M de historial de heartbeats, prescindible).
# Imagen BusyBox — sin GNU-ismos. Config: Secret out-of-band rclone-conf-b2
# (mega+b2; consolidar en Infisical RCLONE_CONF cuando se añada [b2] en la UI).
set -e
log() { echo "[$(date '+%Y-%m-%d %H:%M:%S')] $*"; }
BACKUP_DIR=/data/backups/backups
CROWN="n8n openclaw homarr authentik-pg grafana k3s-db gitea ghost-en-db ghost-en-content zona-exclusion-db zona-exclusion-content researchowl roswell-pg umami-pg infisical-pg trilium vaultwarden filebrowser-db home-master"
KEEP=/tmp/crown-jewels
rm -rf "$KEEP"; mkdir -p "$KEEP"
for p in $CROWN; do
for f in $(ls -1 "$BACKUP_DIR/${p}"_* 2>/dev/null | sort | tail -2); do
ln -s "$f" "$KEEP/$(basename "$f")"
done
done
N=$(ls -1 "$KEEP" | wc -l)
log "Sync de $N artefactos crown-jewels a B2..."
# -L: resuelve los symlinks; sync = espejo → la retención remota (2/servicio) es automática
rclone sync "$KEEP" b2:chemavx-k3s-backups/crown-jewels/ -L \
--config /rclone/rclone.conf --log-level INFO --stats 60s
rm -rf "$KEEP"
log "B2 crown-jewels completado ($N artefactos)."
reddit-intel.sh: |
#!/bin/sh
# Corre en chemavx-n97 (imagen rclone, BusyBox — sin GNU-ismos).
# Sube /opt/reddit-intel a Mega directamente: es el único dato del host n97
# y su cron local escribe en el MISMO disco (inútil contra fallo de disco).
set -e
log() { echo "[$(date '+%Y-%m-%d %H:%M:%S')] $*"; }
DATE=$(date +%Y-%m-%d_%H-%M-%S)
F="/tmp/reddit-intel_${DATE}.tar.gz"
log "Empaquetando /opt/reddit-intel..."
tar -czf "$F" -C /opt/reddit-intel .
gzip -t "$F"
log "Subiendo a mega:k3s-backups/reddit-intel/..."
rclone copy "$F" mega:k3s-backups/reddit-intel/ --config /rclone/rclone.conf --log-level INFO
log "Podando copias remotas de más de 8 días..."
rclone delete mega:k3s-backups/reddit-intel/ --min-age 8d --config /rclone/rclone.conf
log "Subiendo a B2 (crown jewels, retención 3 días)..."
rclone copy "$F" b2:chemavx-k3s-backups/reddit-intel/ --config /rclone/rclone.conf --log-level INFO
rclone delete b2:chemavx-k3s-backups/reddit-intel/ --min-age 3d --config /rclone/rclone.conf
rm -f "$F"
log "reddit-intel OK"
verify.sh: |
#!/bin/sh
# Imagen BusyBox (rclone) — evitar GNU-ismos (date -d 'yesterday' NO funciona).
set -e
log() { echo "[$(date '+%Y-%m-%d %H:%M:%S')] $*"; }
EXPECTED_SERVICES="n8n openclaw homarr authentik-pg grafana k3s-db gitea ghost-en-db ghost-en-content zona-exclusion-db zona-exclusion-content researchowl roswell-pg umami-pg infisical-pg trilium vaultwarden uptime-kuma filebrowser-db home-master"
ERRORS=0
# Ventana de frescura: backup y sync son diarios → un backup es "reciente"
# si su fecha está en los últimos 3 días (margen para jobs nocturnos largos).
NOW=$(date +%s)
ACCEPTED_DATES=""
for i in $(seq 0 2); do
ACCEPTED_DATES="$ACCEPTED_DATES $(date -d @$(( NOW - i * 86400 )) +%Y-%m-%d)"
done
log "=== Verificación de backups en Mega ==="
FILES=$(rclone ls mega:k3s-backups/ --config /rclone/rclone.conf 2>&1)
log "Fechas aceptadas (ventana de 3 días):$ACCEPTED_DATES"
for svc in $EXPECTED_SERVICES; do
FOUND=0
for d in $ACCEPTED_DATES; do
if echo "$FILES" | grep -q "${svc}_${d}"; then
FOUND=1
break
fi
done
if [ "$FOUND" -eq 1 ]; then
log "[OK] $svc"
else
log "[WARN] $svc — sin backup en los últimos 3 días"
ERRORS=$((ERRORS + 1))
fi
done
# reddit-intel vive en su subdir (lo sube el CronJob de n97)
FOUND=0
for d in $ACCEPTED_DATES; do
if echo "$FILES" | grep -q "reddit-intel/reddit-intel_${d}"; then
FOUND=1
break
fi
done
if [ "$FOUND" -eq 1 ]; then
log "[OK] reddit-intel"
else
log "[WARN] reddit-intel — sin backup en los últimos 3 días"
ERRORS=$((ERRORS + 1))
fi
log "---"
TOTAL=$(echo "$FILES" | grep -c "tar.gz\|sql.gz\|db.gz" || true)
log "Total archivos en Mega: $TOTAL"
# === Segundo offsite: B2 crown-jewels (todo menos uptime-kuma) ===
log "=== Verificación de crown-jewels en B2 ==="
CROWN="n8n openclaw homarr authentik-pg grafana k3s-db gitea ghost-en-db ghost-en-content zona-exclusion-db zona-exclusion-content researchowl roswell-pg umami-pg infisical-pg trilium vaultwarden filebrowser-db home-master"
B2FILES=$(rclone ls b2:chemavx-k3s-backups/ --config /rclone/rclone.conf 2>&1)
for svc in $CROWN; do
FOUND=0
for d in $ACCEPTED_DATES; do
if echo "$B2FILES" | grep -q "${svc}_${d}"; then
FOUND=1
break
fi
done
if [ "$FOUND" -eq 1 ]; then
log "[OK] b2/$svc"
else
log "[WARN] b2/$svc — sin backup en los últimos 3 días"
ERRORS=$((ERRORS + 1))
fi
done
if [ "$ERRORS" -gt 0 ]; then
log "RESULTADO: $ERRORS servicio(s) sin backup reciente. Revisar."
exit 1
fi
log "RESULTADO: Todos los backups verificados correctamente (Mega + B2)."
kind: ConfigMap
metadata:
name: backup-scripts
namespace: backup-system