From 34c7b166751df7f4e9597ad9c3bab9410ed797fd Mon Sep 17 00:00:00 2001 From: Jim Lancaster <40281233+zoltan57@users.noreply.github.com> Date: Wed, 26 Aug 2026 17:32:20 -0500 Subject: [PATCH] Refine Synology backup workflow Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- deploy/backup/create_postgres_backup.sh | 78 ++++++++++++++++++------ deploy/backup/restore_postgres_backup.sh | 36 +++++++++-- docs/backup_restore.md | 25 +++++--- docs/production-runbook.md | 4 +- 4 files changed, 111 insertions(+), 32 deletions(-) diff --git a/deploy/backup/create_postgres_backup.sh b/deploy/backup/create_postgres_backup.sh index 6e7574e..c4d79cc 100644 --- a/deploy/backup/create_postgres_backup.sh +++ b/deploy/backup/create_postgres_backup.sh @@ -9,51 +9,93 @@ SYNOLOGY_BACKUP_DIR="${SYNOLOGY_BACKUP_DIR:-}" timestamp="$(date -u +%Y%m%d-%H%M%S)" postgres_file="postgres-${timestamp}.dump" -uploads_file="uploads-${timestamp}.tar.gz" config_file="config-${timestamp}.tar.gz" +logs_file="logs-${timestamp}.tar.gz" manifest_file="backup-${timestamp}.manifest" mkdir -p "${BACKUP_DIR}" +# If SYNOLOGY_BACKUP_DIR wasn't exported in the shell, read it from ENV_FILE. +if [ -z "${SYNOLOGY_BACKUP_DIR}" ] && [ -f "${ENV_FILE}" ]; then + SYNOLOGY_BACKUP_DIR="$( + sed -n 's/^SYNOLOGY_BACKUP_DIR=//p' "${ENV_FILE}" | tail -n 1 + )" +fi + docker compose --env-file "${ENV_FILE}" -f "${COMPOSE_FILE}" exec -T postgres sh -lc \ "PGPASSWORD=\"\$POSTGRES_PASSWORD\" pg_dump -U \"\$POSTGRES_USER\" -d \"\$POSTGRES_DB\" -Fc" \ > "${BACKUP_DIR}/${postgres_file}" -docker compose --env-file "${ENV_FILE}" -f "${COMPOSE_FILE}" run --rm --no-deps --entrypoint sh app -lc \ - "tar -C /app/uploads -czf - ." \ - > "${BACKUP_DIR}/${uploads_file}" - -tar -C . -czf "${BACKUP_DIR}/${config_file}" \ - "${ENV_FILE}" \ - "${COMPOSE_FILE}" \ - deploy/cloudflared/config.yml - cat > "${BACKUP_DIR}/${manifest_file}" < /backup/${logs_file}; fi" + + # 3) Sync uploads incrementally to Synology (copy only new files). + docker compose --env-file "${ENV_FILE}" -f "${COMPOSE_FILE}" run --rm --no-deps \ + -v "${SYNOLOGY_BACKUP_DIR}:/backup" \ + --entrypoint sh app -lc ' + mkdir -p /backup/uploads/documents /backup/uploads/photos; + if [ -f /app/uploads/homepage.md ] && [ ! -f /backup/uploads/homepage.md ]; then + cp /app/uploads/homepage.md /backup/uploads/homepage.md; + fi; + if [ -d /app/uploads/documents ]; then + cp -an /app/uploads/documents/. /backup/uploads/documents/; + fi; + if [ -d /app/uploads/photos ]; then + cp -an /app/uploads/photos/. /backup/uploads/photos/; + fi + ' + cp "${BACKUP_DIR}/${manifest_file}" "${SYNOLOGY_BACKUP_DIR}/${manifest_file}" + + # Keep rolling dump/config/log/manifest snapshots on Synology. + find "${SYNOLOGY_BACKUP_DIR}" -maxdepth 1 -type f \( \ + -name 'postgres-*.dump' -o \ + -name 'config-*.tar.gz' -o \ + -name 'logs-*.tar.gz' -o \ + -name 'backup-*.manifest' \ + \) -mtime +"${RETENTION_DAYS}" -delete fi echo "Created backup set:" echo " ${BACKUP_DIR}/${postgres_file}" -echo " ${BACKUP_DIR}/${uploads_file}" -echo " ${BACKUP_DIR}/${config_file}" echo " ${BACKUP_DIR}/${manifest_file}" +if [ -n "${SYNOLOGY_BACKUP_DIR}" ]; then + echo "Synology artifacts:" + echo " ${SYNOLOGY_BACKUP_DIR}/${postgres_file}" + echo " ${SYNOLOGY_BACKUP_DIR}/${config_file}" + echo " ${SYNOLOGY_BACKUP_DIR}/${logs_file}" + echo " ${SYNOLOGY_BACKUP_DIR}/uploads/{homepage.md,documents/,photos/} (incremental)" + echo " ${SYNOLOGY_BACKUP_DIR}/${manifest_file}" +else + echo "SYNOLOGY_BACKUP_DIR not set; skipped Synology copy." +fi diff --git a/deploy/backup/restore_postgres_backup.sh b/deploy/backup/restore_postgres_backup.sh index c4c3550..ddc4158 100644 --- a/deploy/backup/restore_postgres_backup.sh +++ b/deploy/backup/restore_postgres_backup.sh @@ -9,6 +9,7 @@ fi dump_file="$1" COMPOSE_FILE="${COMPOSE_FILE:-docker-compose.production.yml}" ENV_FILE="${ENV_FILE:-.env.production}" +SYNOLOGY_BACKUP_DIR="${SYNOLOGY_BACKUP_DIR:-}" if [ ! -f "${dump_file}" ]; then echo "Backup file not found: ${dump_file}" @@ -20,7 +21,16 @@ backup_name="$(basename "${dump_file}")" timestamp="$(printf '%s' "${backup_name}" | sed -n 's/^postgres-\([0-9]\{8\}-[0-9]\{6\}\)\.dump$/\1/p')" uploads_file="" config_file="" + +# If SYNOLOGY_BACKUP_DIR wasn't exported in the shell, read it from ENV_FILE. +if [ -z "${SYNOLOGY_BACKUP_DIR}" ] && [ -f "${ENV_FILE}" ]; then + SYNOLOGY_BACKUP_DIR="$( + sed -n 's/^SYNOLOGY_BACKUP_DIR=//p' "${ENV_FILE}" | tail -n 1 + )" +fi + if [ -n "${timestamp}" ]; then + # Legacy local full-archive naming. candidate_uploads="${backup_dir}/uploads-${timestamp}.tar.gz" candidate_config="${backup_dir}/config-${timestamp}.tar.gz" if [ -f "${candidate_uploads}" ]; then @@ -40,19 +50,37 @@ docker compose --env-file "${ENV_FILE}" -f "${COMPOSE_FILE}" exec -T postgres sh cat "${dump_file}" | docker compose --env-file "${ENV_FILE}" -f "${COMPOSE_FILE}" exec -T postgres sh -lc \ "PGPASSWORD=\"\$POSTGRES_PASSWORD\" pg_restore -U \"\$POSTGRES_USER\" -d \"\$POSTGRES_DB\" --clean --if-exists --no-owner --no-privileges" -if [ -n "${uploads_file}" ]; then +if [ -n "${SYNOLOGY_BACKUP_DIR}" ] && [ -d "${SYNOLOGY_BACKUP_DIR}/uploads" ]; then + docker compose --env-file "${ENV_FILE}" -f "${COMPOSE_FILE}" run --rm --no-deps \ + -v "${SYNOLOGY_BACKUP_DIR}:/backup" \ + --entrypoint sh app -lc \ + "mkdir -p /app/uploads/documents /app/uploads/photos && \ + find /app/uploads/documents -mindepth 1 -delete && \ + find /app/uploads/photos -mindepth 1 -delete && \ + if [ -d /backup/uploads/documents ]; then cp -a /backup/uploads/documents/. /app/uploads/documents/; fi && \ + if [ -d /backup/uploads/photos ]; then cp -a /backup/uploads/photos/. /app/uploads/photos/; fi && \ + if [ -f /backup/uploads/homepage.md ]; then cp /backup/uploads/homepage.md /app/uploads/homepage.md; else rm -f /app/uploads/homepage.md; fi" +elif [ -n "${uploads_file}" ]; then + # Legacy local full-archive restore. cat "${uploads_file}" | docker compose --env-file "${ENV_FILE}" -f "${COMPOSE_FILE}" run --rm --no-deps --entrypoint sh app -lc \ "mkdir -p /app/uploads && find /app/uploads -mindepth 1 -delete && tar -xzf - -C /app/uploads" fi -if [ -n "${config_file}" ]; then +if [ -n "${SYNOLOGY_BACKUP_DIR}" ] && [ -n "${timestamp}" ] && [ -f "${SYNOLOGY_BACKUP_DIR}/config-${timestamp}.tar.gz" ]; then + tar -xzf "${SYNOLOGY_BACKUP_DIR}/config-${timestamp}.tar.gz" -C . +elif [ -n "${config_file}" ]; then + # Legacy local full-archive restore. tar -xzf "${config_file}" -C . fi echo "Restore complete from: ${dump_file}" -if [ -n "${uploads_file}" ]; then +if [ -n "${SYNOLOGY_BACKUP_DIR}" ] && [ -d "${SYNOLOGY_BACKUP_DIR}/uploads" ]; then + echo "Restored uploads mirror from: ${SYNOLOGY_BACKUP_DIR}/uploads" +elif [ -n "${uploads_file}" ]; then echo "Restored uploads archive: ${uploads_file}" fi -if [ -n "${config_file}" ]; then +if [ -n "${SYNOLOGY_BACKUP_DIR}" ] && [ -n "${timestamp}" ] && [ -f "${SYNOLOGY_BACKUP_DIR}/config-${timestamp}.tar.gz" ]; then + echo "Restored config archive: ${SYNOLOGY_BACKUP_DIR}/config-${timestamp}.tar.gz" +elif [ -n "${config_file}" ]; then echo "Restored config archive: ${config_file}" fi diff --git a/docs/backup_restore.md b/docs/backup_restore.md index bd7181d..e2144ae 100644 --- a/docs/backup_restore.md +++ b/docs/backup_restore.md @@ -2,14 +2,21 @@ This guide defines operational backup/restore for clean-slate recovery of the Docker runtime and Synology replication. -## 1. Backup artifacts (full recovery set) +## 1. Backup artifacts (clean-slate recovery set) - Primary local backup location: `./data/backups` -- Files created per timestamp: +- Local files created per run: - `postgres-YYYYMMDD-HHMMSS.dump` (PostgreSQL custom dump via `pg_dump -Fc`) - - `uploads-YYYYMMDD-HHMMSS.tar.gz` (full `/app/uploads` volume; includes `homepage.md`, documents, and photos) - - `config-YYYYMMDD-HHMMSS.tar.gz` (deployment config snapshot: `.env.production`, `docker-compose.production.yml`, `deploy/cloudflared/config.yml`) - `backup-YYYYMMDD-HHMMSS.manifest` (artifact index) +- Synology files created per run (when `SYNOLOGY_BACKUP_DIR` is set): + - `postgres-YYYYMMDD-HHMMSS.dump` (copied from local) + - `config-YYYYMMDD-HHMMSS.tar.gz` (deployment config snapshot: `.env.production`, `docker-compose.production.yml`, `deploy/cloudflared/config.yml` when present) + - `logs-YYYYMMDD-HHMMSS.tar.gz` (snapshot of `/app/data/logs`) + - `backup-YYYYMMDD-HHMMSS.manifest` +- Synology incremental media mirror (copied only if missing): + - `uploads/homepage.md` + - `uploads/documents/**` + - `uploads/photos/**` ## 2. Creating backups @@ -23,14 +30,14 @@ Environment overrides: - `BACKUP_DIR` (default `./data/backups`) - `BACKUP_RETENTION_DAYS` (default `14`) -- `SYNOLOGY_BACKUP_DIR` (if set, backup is copied to this mounted path) +- `SYNOLOGY_BACKUP_DIR` (if set, dump/config/log/manifest are copied to this mounted path and media is synced incrementally) - `ENV_FILE` (default `.env.production`) - `COMPOSE_FILE` (default `docker-compose.production.yml`) Example with Synology mount: ```bash -SYNOLOGY_BACKUP_DIR=/mnt/synology/transcription-backups sh deploy/backup/create_postgres_backup.sh +SYNOLOGY_BACKUP_DIR=/mnt/synology-backups sh deploy/backup/create_postgres_backup.sh ``` ## 2.1 Persisting Synology mount (LXC) @@ -51,6 +58,7 @@ Recommended pattern: - Keep NAS credentials in a local file like `/etc/samba/credentials/transcription-synology` with `chmod 600`. - Keep `SYNOLOGY_BACKUP_DIR` in `.env.production` aligned to that mount point (for example `/mnt/synology-backups`). +- The script will also read `SYNOLOGY_BACKUP_DIR` from `ENV_FILE` when not exported in the shell. ## 3. Restoring from backup @@ -60,7 +68,8 @@ Restore requires downtime for app + worker writes. - `docker compose --env-file .env.production -f docker-compose.production.yml stop app worker` 2. Restore: - `sh deploy/backup/restore_postgres_backup.sh ./data/backups/postgres-YYYYMMDD-HHMMSS.dump` - - if same-timestamp `uploads-*.tar.gz` and `config-*.tar.gz` exist in the same directory, they are restored automatically. + - if `SYNOLOGY_BACKUP_DIR/uploads` exists, media is restored from the Synology mirror. + - if `SYNOLOGY_BACKUP_DIR/config-YYYYMMDD-HHMMSS.tar.gz` exists, config is restored from that archive. 3. Start app and worker: - `docker compose --env-file .env.production -f docker-compose.production.yml start app worker` 4. Validate `/healthz` and run one smoke workflow. @@ -68,7 +77,7 @@ Restore requires downtime for app + worker writes. Notes: - Config restore extracts the archived files back into the current repository path. -- If only the database dump is present, restore runs in database-only mode (legacy behavior). +- Legacy full-archive restores (`uploads-*.tar.gz`, `config-*.tar.gz` beside the dump) are still supported for older backups. ## 4. Retention and recovery targets diff --git a/docs/production-runbook.md b/docs/production-runbook.md index 8bd2615..2839672 100644 --- a/docs/production-runbook.md +++ b/docs/production-runbook.md @@ -36,7 +36,7 @@ This runbook is the operational checklist for releasing and monitoring the trans - stdout aggregation receives events - file logs are written under `./data/logs` 5. Create a fresh PostgreSQL backup after successful deployment: - - `sh deploy/backup/create_postgres_backup.sh` (creates DB dump + uploads + config recovery set) + - `sh deploy/backup/create_postgres_backup.sh` (creates local DB dump + Synology DB/config/log artifacts + incremental media mirror) ## 3. Rollback triggers and actions @@ -56,7 +56,7 @@ This runbook is the operational checklist for releasing and monitoring the trans - relevant DB rows (`job`, `job_source`, `execution_attempt`) 5. If persistence regression is confirmed, restore the latest valid DB dump: - `sh deploy/backup/restore_postgres_backup.sh ` - - paired uploads/config artifacts (same timestamp) are restored automatically when present. + - Synology media mirror and paired config snapshot (same timestamp) are restored automatically when present. ## 4. Post-release monitoring checklist