apiVersion: v1 data: backup.sh: | #!/bin/bash # Backup diario del homelab. Imagen: bitnami/kubectl (bash GNU, corre en chemavx-k8). # Regla: cada artefacto se valida (gzip -t + tamaño mínimo) — un tar vacío # como el bug histórico de openclaw (104 bytes) cuenta como FALLO y el job sale 1 # (→ alerta Grafana homelab-backup-job-failed → Telegram). set -u # kubectl usa el ServiceAccount in-cluster (backup-sa + ClusterRole backup-role). # NO usar el k3s.yaml del host: su server es https://127.0.0.1:6443, inalcanzable # desde la red del pod (el script antiguo "funcionaba" porque apuntaba KUBECONFIG # a un fichero inexistente y kubectl caía en fallback al ServiceAccount). unset KUBECONFIG BACKUP_DIR="/data/backups/backups" DATE=$(date +%Y-%m-%d_%H-%M-%S) STORAGE=/var/lib/rancher/k3s/storage FAIL=0 mkdir -p "$BACKUP_DIR" log() { echo "[$(date '+%Y-%m-%d %H:%M:%S')] $*"; } fail() { log "ERROR: $1 FAILED"; FAIL=$((FAIL + 1)); } # Valida artefacto: gzip íntegro y tamaño >= mínimo esperado check() { # $1=fichero $2=bytes_minimos $3=servicio if gzip -t "$1" 2>/dev/null && [ "$(stat -c%s "$1" 2>/dev/null || echo 0)" -ge "$2" ]; then log "$3 OK ($(du -h "$1" | cut -f1))" else rm -f "$1" fail "$3 (artefacto corrupto o menor de $2 bytes)" fi } # Resuelve la ruta local-path de un PVC en el host pvpath() { # $1=ns $2=pvc local pv pv=$(kubectl get pvc -n "$1" "$2" -o jsonpath='{.spec.volumeName}' 2>/dev/null) || return 1 echo "$STORAGE/${pv}_$1_$2" } # Copia consistente de la ghost.db (VACUUM INTO vía node del pod) + tar del content backup_ghost() { # $1=ns $2=deploy $3=prefijo local f="$BACKUP_DIR/${3}-db_${DATE}.db.gz" kubectl exec -i -n "$1" "deploy/$2" -- sh -c \ 'rm -f /tmp/ghost-backup.db && NODE_PATH=/var/lib/ghost/current/node_modules node -' \ < /scripts/ghost-db-backup.js >/dev/null 2>&1 \ && kubectl exec -n "$1" "deploy/$2" -- cat /tmp/ghost-backup.db 2>/dev/null | gzip > "$f" kubectl exec -n "$1" "deploy/$2" -- rm -f /tmp/ghost-backup.db 2>/dev/null || true check "$f" 200000 "${3}-db" local c="$BACKUP_DIR/${3}-content_${DATE}.tar.gz" kubectl exec -n "$1" "deploy/$2" -- tar czf - -C /var/lib/ghost/content . > "$c" 2>/dev/null check "$c" 5000000 "${3}-content" } # === n8n (hostPath /data/n8n — incluye .n8n/config con la encryptionKey) === log "Backing up n8n..." tar -czf "$BACKUP_DIR/n8n_${DATE}.tar.gz" -C /data/n8n . 2>/dev/null check "$BACKUP_DIR/n8n_${DATE}.tar.gz" 10000000 n8n # === openclaw (PVC local-path; el hostPath /data/openclaw era un stub vacío) === log "Backing up openclaw..." OC=$(pvpath openclaw openclaw-pvc) tar -czf "$BACKUP_DIR/openclaw_${DATE}.tar.gz" -C "$OC" . 2>/dev/null check "$BACKUP_DIR/openclaw_${DATE}.tar.gz" 50000 openclaw # === homarr === log "Backing up homarr..." HOMARR_DB=$(pvpath homarr homarr-db-pvc) HOMARR_REDIS=$(pvpath homarr homarr-redis-pvc) mkdir -p /tmp/backup-homarr/db /tmp/backup-homarr/redis cp "$HOMARR_DB/db.sqlite" /tmp/backup-homarr/db/ 2>/dev/null || true cp "$HOMARR_REDIS/dump.rdb" /tmp/backup-homarr/redis/ 2>/dev/null || true tar -czf "$BACKUP_DIR/homarr_${DATE}.tar.gz" -C /tmp/backup-homarr . 2>/dev/null check "$BACKUP_DIR/homarr_${DATE}.tar.gz" 10000 homarr rm -rf /tmp/backup-homarr # === authentik PostgreSQL === log "Backing up authentik postgres..." kubectl exec -n authentik postgresql-0 -- pg_dump -U authentik authentik 2>/dev/null | \ gzip > "$BACKUP_DIR/authentik-pg_${DATE}.sql.gz" check "$BACKUP_DIR/authentik-pg_${DATE}.sql.gz" 10000 authentik-pg # === grafana === log "Backing up grafana..." GRAFANA=$(pvpath monitoring kube-prometheus-stack-grafana) mkdir -p /tmp/backup-grafana cp "$GRAFANA/grafana.db" /tmp/backup-grafana/ 2>/dev/null || true tar -czf "$BACKUP_DIR/grafana_${DATE}.tar.gz" -C /tmp/backup-grafana . 2>/dev/null check "$BACKUP_DIR/grafana_${DATE}.tar.gz" 100000 grafana rm -rf /tmp/backup-grafana # === k3s: state.db + token del server + /etc/rancher/k3s === # Sin el token, state.db NO arranca en un k3s nuevo (cadena de restauración). log "Backing up k3s (state.db + token + /etc/rancher/k3s)..." tar -czf "$BACKUP_DIR/k3s-db_${DATE}.tar.gz" \ /data/k3s-db/state.db* /host/k3s-token /host/etc-rancher-k3s 2>/dev/null check "$BACKUP_DIR/k3s-db_${DATE}.tar.gz" 1000000 k3s-db # === Gitea: sqlite .backup consistente + repos + conf (SIN gitea/packages, 6.4G de registry reconstruible por CI) === log "Backing up gitea..." kubectl exec -n gitea gitea-0 -- sh -c \ 'rm -f /data/gitea/gitea-backup.db && sqlite3 /data/gitea/gitea.db ".backup /data/gitea/gitea-backup.db"' 2>/dev/null \ && kubectl exec -n gitea gitea-0 -- tar czf - -C /data \ git ssh gitea/conf gitea/jwt gitea/avatars gitea/repo-avatars gitea/gitea-backup.db \ > "$BACKUP_DIR/gitea_${DATE}.tar.gz" 2>/dev/null kubectl exec -n gitea gitea-0 -- rm -f /data/gitea/gitea-backup.db 2>/dev/null || true check "$BACKUP_DIR/gitea_${DATE}.tar.gz" 5000000 gitea # === Ghost EN + ES (db consistente + content con imágenes/temas/redirects) === log "Backing up ghost-en..." backup_ghost ghost-en ghost-en ghost-en log "Backing up zona-exclusion..." backup_ghost zona-exclusion zona-exclusion zona-exclusion # === ResearchOwl (sqlite .backup vía python3 del pod) === log "Backing up researchowl..." kubectl exec -i -n researchowl deploy/researchowl -- sh -c \ 'rm -f /tmp/ro-backup.db && python3 -' < /scripts/sqlite-stdin-backup.py >/dev/null 2>&1 \ && kubectl exec -n researchowl deploy/researchowl -- cat /tmp/ro-backup.db 2>/dev/null | \ gzip > "$BACKUP_DIR/researchowl_${DATE}.db.gz" kubectl exec -n researchowl deploy/researchowl -- rm -f /tmp/ro-backup.db 2>/dev/null || true check "$BACKUP_DIR/researchowl_${DATE}.db.gz" 5000000 researchowl # === Roswell (corpus) PostgreSQL === log "Backing up roswell postgres..." kubectl exec -n roswell deploy/postgres -- sh -c \ 'PGPASSWORD="${POSTGRES_PASSWORD:-}" pg_dump -U "$POSTGRES_USER" "${POSTGRES_DB:-$POSTGRES_USER}"' 2>/dev/null | \ gzip > "$BACKUP_DIR/roswell-pg_${DATE}.sql.gz" check "$BACKUP_DIR/roswell-pg_${DATE}.sql.gz" 5000 roswell-pg # === Umami PostgreSQL === log "Backing up umami postgres..." kubectl exec -n umami deploy/postgres -- sh -c \ 'PGPASSWORD="${POSTGRES_PASSWORD:-}" pg_dump -U "$POSTGRES_USER" "${POSTGRES_DB:-$POSTGRES_USER}"' 2>/dev/null | \ gzip > "$BACKUP_DIR/umami-pg_${DATE}.sql.gz" check "$BACKUP_DIR/umami-pg_${DATE}.sql.gz" 5000 umami-pg # === Infisical PostgreSQL (RESTAURAR REQUIERE infisical-secrets — está dentro de k3s-db/state.db, ver RESTORE.md) === log "Backing up infisical postgres..." kubectl exec -n infisical postgresql-0 -- sh -c \ 'PGPASSWORD="$POSTGRES_PASSWORD" pg_dump -U "$POSTGRES_USER" infisicalDB' 2>/dev/null | \ gzip > "$BACKUP_DIR/infisical-pg_${DATE}.sql.gz" check "$BACKUP_DIR/infisical-pg_${DATE}.sql.gz" 10000 infisical-pg # === Trilium === log "Backing up trilium..." kubectl exec -n trilium deploy/trilium -- tar czf - -C /home/node/trilium-data . \ > "$BACKUP_DIR/trilium_${DATE}.tar.gz" 2>/dev/null check "$BACKUP_DIR/trilium_${DATE}.tar.gz" 500000 trilium # === Vaultwarden (db + attachments + claves RSA; copia en frío ~02:00) === log "Backing up vaultwarden..." VW=$(pvpath vaultwarden vaultwarden-pvc) tar -czf "$BACKUP_DIR/vaultwarden_${DATE}.tar.gz" -C "$VW" . 2>/dev/null check "$BACKUP_DIR/vaultwarden_${DATE}.tar.gz" 100000 vaultwarden # === Uptime-Kuma (sqlite .backup vía CLI del pod; retención corta, es el artefacto gordo) === log "Backing up uptime-kuma..." kubectl exec -n monitoring deploy/uptime-kuma -- sh -c \ 'rm -f /tmp/kuma-backup.db && sqlite3 /app/data/kuma.db ".backup /tmp/kuma-backup.db"' 2>/dev/null \ && kubectl exec -n monitoring deploy/uptime-kuma -- cat /tmp/kuma-backup.db 2>/dev/null | \ gzip > "$BACKUP_DIR/uptime-kuma_${DATE}.db.gz" kubectl exec -n monitoring deploy/uptime-kuma -- rm -f /tmp/kuma-backup.db 2>/dev/null || true check "$BACKUP_DIR/uptime-kuma_${DATE}.db.gz" 1000000 uptime-kuma # === Filebrowser (solo su BD; los ficheros servidos van en home-master/imagefinder) === log "Backing up filebrowser db..." FB=$(pvpath filebrowser filebrowser-db) tar -czf "$BACKUP_DIR/filebrowser-db_${DATE}.tar.gz" -C "$FB" . 2>/dev/null check "$BACKUP_DIR/filebrowser-db_${DATE}.tar.gz" 1000 filebrowser-db # === Home del master (~/.claude memoria+skills, imagefinder, proyectos sueltos, RUNBOOK...) === # Excluye ~/.ssh (claves privadas, decisión explícita), caches y binarios reproducibles. log "Backing up home-master..." tar -czf "$BACKUP_DIR/home-master_${DATE}.tar.gz" -C /host/home-chemavx \ --exclude='./.ssh' --exclude='./.cache' --exclude='./.venvs' --exclude='./.npm' \ --exclude='./.nvm' --exclude='./bin' --exclude='./.docker' --exclude='./snap' \ --exclude='*/node_modules' --exclude='./.local/share/pnpm' \ --exclude='./.warp' --exclude='./.local/share/claude' --exclude='./.local/lib' \ . 2>/dev/null check "$BACKUP_DIR/home-master_${DATE}.tar.gz" 50000000 home-master # === Retención (7 por defecto; 3 para los artefactos gordos) === log "Cleaning old backups..." retain() { # $1=prefijo $2=copias find "$BACKUP_DIR" -maxdepth 1 \ \( -name "${1}_*.tar.gz" -o -name "${1}_*.sql.gz" -o -name "${1}_*.db.gz" \) | \ sort | head -n -"$2" | xargs -r rm -f } for p in n8n openclaw homarr authentik-pg grafana k3s-db gitea \ ghost-en-db ghost-en-content zona-exclusion-db zona-exclusion-content \ researchowl roswell-pg umami-pg infisical-pg trilium vaultwarden filebrowser-db; do retain "$p" 7 done retain uptime-kuma 3 retain home-master 3 if [ "$FAIL" -gt 0 ]; then log "Backup terminado con $FAIL fallo(s) — el job sale con error para disparar la alerta." exit 1 fi log "Backup complete!" ghost-db-backup.js: | // Copia consistente de la ghost.db usando el driver sqlite3 del propio Ghost. // Se ejecuta DENTRO del pod vía: kubectl exec -i ... node - (script por stdin) const sqlite3 = require('sqlite3'); const src = '/var/lib/ghost/content/data/ghost.db'; const dst = '/tmp/ghost-backup.db'; const db = new sqlite3.Database(src, sqlite3.OPEN_READONLY, (err) => { if (err) { console.error(err.message); process.exit(1); } }); db.run("VACUUM INTO '" + dst + "'", (err) => { if (err) { console.error(err.message); process.exit(1); } db.close(() => console.log('ghost db backup ok')); }); sqlite-stdin-backup.py: | # Copia consistente de researchowl.db (API backup de sqlite3, stdlib). # Se ejecuta DENTRO del pod vía: kubectl exec -i ... python3 - import sqlite3 src = sqlite3.connect('file:/data/researchowl.db?mode=ro', uri=True) dst = sqlite3.connect('/tmp/ro-backup.db') with dst: src.backup(dst) dst.close() src.close() print('researchowl db backup ok') rclone-mega.sh: | #!/bin/bash set -e log() { echo "[$(date '+%Y-%m-%d %H:%M:%S')] $*"; } log "Iniciando sync a Mega..." log "Origen: /data/backups/backups" log "Destino: mega:k3s-backups/" # --exclude reddit-intel: ese subdir remoto lo escribe el CronJob de n97 # directamente; un sync sin exclude lo borraría (sync = espejo exacto). rclone sync /data/backups/backups mega:k3s-backups/ \ --exclude "/reddit-intel/**" \ --config /rclone/rclone.conf \ --log-level INFO \ --stats 30s log "Sync completado." log "Archivos en Mega:" rclone ls mega:k3s-backups/ --config /rclone/rclone.conf reddit-intel.sh: | #!/bin/sh # Corre en chemavx-n97 (imagen rclone, BusyBox — sin GNU-ismos). # Sube /opt/reddit-intel a Mega directamente: es el único dato del host n97 # y su cron local escribe en el MISMO disco (inútil contra fallo de disco). set -e log() { echo "[$(date '+%Y-%m-%d %H:%M:%S')] $*"; } DATE=$(date +%Y-%m-%d_%H-%M-%S) F="/tmp/reddit-intel_${DATE}.tar.gz" log "Empaquetando /opt/reddit-intel..." tar -czf "$F" -C /opt/reddit-intel . gzip -t "$F" log "Subiendo a mega:k3s-backups/reddit-intel/..." rclone copy "$F" mega:k3s-backups/reddit-intel/ --config /rclone/rclone.conf --log-level INFO log "Podando copias remotas de más de 8 días..." rclone delete mega:k3s-backups/reddit-intel/ --min-age 8d --config /rclone/rclone.conf rm -f "$F" log "reddit-intel OK" verify.sh: | #!/bin/sh # Imagen BusyBox (rclone) — evitar GNU-ismos (date -d 'yesterday' NO funciona). set -e log() { echo "[$(date '+%Y-%m-%d %H:%M:%S')] $*"; } EXPECTED_SERVICES="n8n openclaw homarr authentik-pg grafana k3s-db gitea ghost-en-db ghost-en-content zona-exclusion-db zona-exclusion-content researchowl roswell-pg umami-pg infisical-pg trilium vaultwarden uptime-kuma filebrowser-db home-master" ERRORS=0 # Ventana de frescura: backup y sync son diarios → un backup es "reciente" # si su fecha está en los últimos 3 días (margen para jobs nocturnos largos). NOW=$(date +%s) ACCEPTED_DATES="" for i in $(seq 0 2); do ACCEPTED_DATES="$ACCEPTED_DATES $(date -d @$(( NOW - i * 86400 )) +%Y-%m-%d)" done log "=== Verificación de backups en Mega ===" FILES=$(rclone ls mega:k3s-backups/ --config /rclone/rclone.conf 2>&1) log "Fechas aceptadas (ventana de 3 días):$ACCEPTED_DATES" for svc in $EXPECTED_SERVICES; do FOUND=0 for d in $ACCEPTED_DATES; do if echo "$FILES" | grep -q "${svc}_${d}"; then FOUND=1 break fi done if [ "$FOUND" -eq 1 ]; then log "[OK] $svc" else log "[WARN] $svc — sin backup en los últimos 3 días" ERRORS=$((ERRORS + 1)) fi done # reddit-intel vive en su subdir (lo sube el CronJob de n97) FOUND=0 for d in $ACCEPTED_DATES; do if echo "$FILES" | grep -q "reddit-intel/reddit-intel_${d}"; then FOUND=1 break fi done if [ "$FOUND" -eq 1 ]; then log "[OK] reddit-intel" else log "[WARN] reddit-intel — sin backup en los últimos 3 días" ERRORS=$((ERRORS + 1)) fi log "---" TOTAL=$(echo "$FILES" | grep -c "tar.gz\|sql.gz\|db.gz" || true) log "Total archivos en Mega: $TOTAL" if [ "$ERRORS" -gt 0 ]; then log "RESULTADO: $ERRORS servicio(s) sin backup reciente. Revisar." exit 1 fi log "RESULTADO: Todos los backups verificados correctamente." kind: ConfigMap metadata: name: backup-scripts namespace: backup-system