fd109ef892
The Qdrant snapshots directory is /qdrant/snapshots/{collection}/, not
/qdrant/storage/collections/{collection}/snapshots/ as the script assumed.
Verified against running mem0-qdrant container on beast.
97 lines
3.9 KiB
Bash
Executable file
97 lines
3.9 KiB
Bash
Executable file
#!/usr/bin/env bash
|
|
#
|
|
# Qdrant snapshot + off-host rotation.
|
|
#
|
|
# Snapshots both collections (mem0_v3 + mem0_v3_entities) back-to-back via the
|
|
# Qdrant REST API, downloads them to a date-stamped local directory, uploads to
|
|
# the configured rclone remote, prunes local copies older than 14 days, and
|
|
# emits a Prometheus textfile metric for future scrape.
|
|
#
|
|
# Env vars (override defaults):
|
|
# QDRANT_CONTAINER container name (default: mem0-qdrant)
|
|
# COLLECTIONS space-separated collection names
|
|
# (default: "mem0_v3 mem0_v3_entities")
|
|
# BACKUP_DIR local backup root
|
|
# (default: ~/aistuff/mem0/backups/qdrant)
|
|
# RCLONE_REMOTE rclone remote path (e.g. b2:mem0-backups/qdrant).
|
|
# If unset, off-host upload is skipped.
|
|
# LOCAL_RETENTION_DAYS how long to keep local copies (default: 14)
|
|
# TEXTFILE_DIR Prometheus node_exporter textfile collector dir
|
|
# (default: /var/lib/node_exporter/textfile_collector,
|
|
# skipped if the dir does not exist)
|
|
#
|
|
# Suggested cron (daily at 03:00 UTC):
|
|
# 0 3 * * * RCLONE_REMOTE=b2:mem0-backups/qdrant /home/ubuntu/aistuff/mem0/scripts/backup_qdrant.sh >> /home/ubuntu/aistuff/mem0/backups/backup.log 2>&1
|
|
#
|
|
# Exit codes:
|
|
# 0 success
|
|
# 1 snapshot/download failure
|
|
# 2 rclone failure (after local download succeeded)
|
|
|
|
set -euo pipefail
|
|
|
|
QDRANT_CONTAINER="${QDRANT_CONTAINER:-mem0-qdrant}"
|
|
COLLECTIONS="${COLLECTIONS:-mem0_v3 mem0_v3_entities}"
|
|
BACKUP_DIR="${BACKUP_DIR:-$HOME/aistuff/mem0/backups/qdrant}"
|
|
RCLONE_REMOTE="${RCLONE_REMOTE:-}"
|
|
LOCAL_RETENTION_DAYS="${LOCAL_RETENTION_DAYS:-14}"
|
|
TEXTFILE_DIR="${TEXTFILE_DIR:-/var/lib/node_exporter/textfile_collector}"
|
|
|
|
TS="$(date -u +%Y%m%dT%H%M%SZ)"
|
|
DAY="$(date -u +%Y-%m-%d)"
|
|
TARGET_DIR="$BACKUP_DIR/$DAY"
|
|
mkdir -p "$TARGET_DIR"
|
|
|
|
log() { printf '[%s] %s\n' "$(date -u +%FT%TZ)" "$*"; }
|
|
|
|
log "starting backup ts=$TS dir=$TARGET_DIR collections=$COLLECTIONS"
|
|
|
|
total_bytes=0
|
|
for col in $COLLECTIONS; do
|
|
log "snapshot create: $col"
|
|
resp=$(docker exec "$QDRANT_CONTAINER" curl -fsS -X POST \
|
|
"http://localhost:6333/collections/$col/snapshots?wait=true")
|
|
snap_name=$(printf '%s' "$resp" \
|
|
| python3 -c 'import sys,json; print(json.load(sys.stdin)["result"]["name"])')
|
|
|
|
out_file="$TARGET_DIR/${col}_${TS}_${snap_name}"
|
|
log "snapshot download: $col/$snap_name -> $out_file"
|
|
docker cp "$QDRANT_CONTAINER:/qdrant/snapshots/$col/$snap_name" "$out_file"
|
|
|
|
# Remove the in-container snapshot to avoid disk bloat on the volume.
|
|
docker exec "$QDRANT_CONTAINER" curl -fsS -X DELETE \
|
|
"http://localhost:6333/collections/$col/snapshots/$snap_name" >/dev/null
|
|
|
|
size=$(stat -c %s "$out_file" 2>/dev/null || stat -f %z "$out_file")
|
|
total_bytes=$((total_bytes + size))
|
|
log "downloaded: $out_file ($size bytes)"
|
|
done
|
|
|
|
if [ -n "$RCLONE_REMOTE" ]; then
|
|
log "rclone copy: $TARGET_DIR -> $RCLONE_REMOTE/$DAY"
|
|
if ! rclone copy "$TARGET_DIR" "$RCLONE_REMOTE/$DAY"; then
|
|
log "rclone failed (local copies retained)"
|
|
exit 2
|
|
fi
|
|
else
|
|
log "RCLONE_REMOTE unset; skipping off-host upload"
|
|
fi
|
|
|
|
log "pruning local copies older than $LOCAL_RETENTION_DAYS days"
|
|
find "$BACKUP_DIR" -mindepth 1 -maxdepth 1 -type d -mtime "+$LOCAL_RETENTION_DAYS" -exec rm -rf {} +
|
|
|
|
if [ -d "$TEXTFILE_DIR" ]; then
|
|
tmp="$(mktemp)"
|
|
{
|
|
echo "# HELP qdrant_last_backup_timestamp_seconds Unix timestamp of last successful Qdrant backup."
|
|
echo "# TYPE qdrant_last_backup_timestamp_seconds gauge"
|
|
echo "qdrant_last_backup_timestamp_seconds $(date -u +%s)"
|
|
echo "# HELP qdrant_last_backup_bytes Total bytes of last successful Qdrant backup."
|
|
echo "# TYPE qdrant_last_backup_bytes gauge"
|
|
echo "qdrant_last_backup_bytes $total_bytes"
|
|
} > "$tmp"
|
|
mv "$tmp" "$TEXTFILE_DIR/qdrant_backup.prom"
|
|
log "textfile metric written: $TEXTFILE_DIR/qdrant_backup.prom"
|
|
fi
|
|
|
|
log "backup complete: $total_bytes bytes across $(echo "$COLLECTIONS" | wc -w) collection(s)"
|