From c85cc6be20aae7c308fd361c1b813b49674578ca Mon Sep 17 00:00:00 2001
From: Jim Lancaster <40281233+zoltan57@users.noreply.github.com>
Date: Tue, 25 Aug 2026 13:09:14 -0500
Subject: [PATCH] V6 Phase 4 - Postgres backup to Synology

---
 .env.production.example                  |  7 ++++
 README.md                                |  4 ++
 deploy/backup/create_postgres_backup.sh  | 26 +++++++++++++
 deploy/backup/restore_postgres_backup.sh | 27 +++++++++++++
 docs/backup_restore.md                   | 49 ++++++++++++++++++++++++
 docs/production-runbook.md               | 16 +++++++-
 docs/v6_0_hosting_migration_plan.md      |  6 +++
 7 files changed, 133 insertions(+), 2 deletions(-)
 create mode 100644 deploy/backup/create_postgres_backup.sh
 create mode 100644 deploy/backup/restore_postgres_backup.sh
 create mode 100644 docs/backup_restore.md

diff --git a/.env.production.example b/.env.production.example
index 92f2c32..f249ad4 100644
--- a/.env.production.example
+++ b/.env.production.example
@@ -52,3 +52,10 @@ POSTGRES_PASSWORD=replace-with-strong-password
 # --- cloudflare tunnel ---
 # Required for token-based tunnel startup.
 CLOUDFLARE_TUNNEL_TOKEN=replace-with-cloudflare-tunnel-token
+
+# --- backup workflow helpers ---
+# Used by deploy/backup/create_postgres_backup.sh
+BACKUP_DIR=./data/backups
+BACKUP_RETENTION_DAYS=14
+# Optional mounted Synology destination path for replicated dumps
+# SYNOLOGY_BACKUP_DIR=/mnt/synology/transcription-backups
diff --git a/README.md b/README.md
index dc7fa52..5647899 100644
--- a/README.md
+++ b/README.md
@@ -175,6 +175,10 @@ The canonical MVP prompt is:
 Schema upgrades use an explicit export/import rebuild flow (no runtime legacy write compatibility).  
 See `docs/data_migration.md` for commands and cutover steps.
 
+## Backup and restore workflow
+
+Production PostgreSQL backup/restore steps are documented in `docs/backup_restore.md`.
+
 ## Destructive test procedure (with data backup)
 
 AI execution policy: before the first unit-test run in a test/fix cycle, create one backup of `./data`. Reuse that same backup for every subsequent test run in the cycle. After tests succeed, always pause and ask whether to restore now.
diff --git a/deploy/backup/create_postgres_backup.sh b/deploy/backup/create_postgres_backup.sh
new file mode 100644
index 0000000..13ae391
--- /dev/null
+++ b/deploy/backup/create_postgres_backup.sh
@@ -0,0 +1,26 @@
+#!/usr/bin/env sh
+set -eu
+
+COMPOSE_FILE="${COMPOSE_FILE:-docker-compose.production.yml}"
+ENV_FILE="${ENV_FILE:-.env.production}"
+BACKUP_DIR="${BACKUP_DIR:-./data/backups}"
+RETENTION_DAYS="${BACKUP_RETENTION_DAYS:-14}"
+SYNOLOGY_BACKUP_DIR="${SYNOLOGY_BACKUP_DIR:-}"
+
+timestamp="$(date -u +%Y%m%d-%H%M%S)"
+backup_file="postgres-${timestamp}.dump"
+
+mkdir -p "${BACKUP_DIR}"
+
+docker compose --env-file "${ENV_FILE}" -f "${COMPOSE_FILE}" exec -T postgres sh -lc \
+  "PGPASSWORD=\"\$POSTGRES_PASSWORD\" pg_dump -U \"\$POSTGRES_USER\" -d \"\$POSTGRES_DB\" -Fc" \
+  > "${BACKUP_DIR}/${backup_file}"
+
+find "${BACKUP_DIR}" -type f -name 'postgres-*.dump' -mtime +"${RETENTION_DAYS}" -delete
+
+if [ -n "${SYNOLOGY_BACKUP_DIR}" ]; then
+  mkdir -p "${SYNOLOGY_BACKUP_DIR}"
+  cp "${BACKUP_DIR}/${backup_file}" "${SYNOLOGY_BACKUP_DIR}/${backup_file}"
+fi
+
+echo "Created backup: ${BACKUP_DIR}/${backup_file}"
diff --git a/deploy/backup/restore_postgres_backup.sh b/deploy/backup/restore_postgres_backup.sh
new file mode 100644
index 0000000..2eb4e83
--- /dev/null
+++ b/deploy/backup/restore_postgres_backup.sh
@@ -0,0 +1,27 @@
+#!/usr/bin/env sh
+set -eu
+
+if [ "$#" -lt 1 ]; then
+  echo "Usage: $0 <path-to-postgres-dump>"
+  exit 1
+fi
+
+dump_file="$1"
+COMPOSE_FILE="${COMPOSE_FILE:-docker-compose.production.yml}"
+ENV_FILE="${ENV_FILE:-.env.production}"
+
+if [ ! -f "${dump_file}" ]; then
+  echo "Backup file not found: ${dump_file}"
+  exit 1
+fi
+
+docker compose --env-file "${ENV_FILE}" -f "${COMPOSE_FILE}" exec -T postgres sh -lc \
+  "PGPASSWORD=\"\$POSTGRES_PASSWORD\" psql -U \"\$POSTGRES_USER\" -d postgres -c \"DROP DATABASE IF EXISTS \\\"\$POSTGRES_DB\\\";\""
+
+docker compose --env-file "${ENV_FILE}" -f "${COMPOSE_FILE}" exec -T postgres sh -lc \
+  "PGPASSWORD=\"\$POSTGRES_PASSWORD\" psql -U \"\$POSTGRES_USER\" -d postgres -c \"CREATE DATABASE \\\"\$POSTGRES_DB\\\";\""
+
+cat "${dump_file}" | docker compose --env-file "${ENV_FILE}" -f "${COMPOSE_FILE}" exec -T postgres sh -lc \
+  "PGPASSWORD=\"\$POSTGRES_PASSWORD\" pg_restore -U \"\$POSTGRES_USER\" -d \"\$POSTGRES_DB\" --clean --if-exists --no-owner --no-privileges"
+
+echo "Restore complete from: ${dump_file}"
diff --git a/docs/backup_restore.md b/docs/backup_restore.md
new file mode 100644
index 0000000..c8eca69
--- /dev/null
+++ b/docs/backup_restore.md
@@ -0,0 +1,49 @@
+# Backup and Restore (V6.0 Phase 4)
+
+This guide defines operational backup/restore for the Docker PostgreSQL runtime and Synology replication.
+
+## 1. Backup artifacts
+
+- Primary local backup location: `./data/backups`
+- Backup format: PostgreSQL custom dump (`pg_dump -Fc`)
+- Naming: `postgres-YYYYMMDD-HHMMSS.dump` (UTC timestamp)
+
+## 2. Creating backups
+
+Use the scripted command:
+
+```bash
+sh deploy/backup/create_postgres_backup.sh
+```
+
+Optional environment overrides:
+
+- `BACKUP_DIR` (default `./data/backups`)
+- `BACKUP_RETENTION_DAYS` (default `14`)
+- `SYNOLOGY_BACKUP_DIR` (if set, backup is copied to this mounted path)
+- `ENV_FILE` (default `.env.production`)
+- `COMPOSE_FILE` (default `docker-compose.production.yml`)
+
+Example with Synology mount:
+
+```bash
+SYNOLOGY_BACKUP_DIR=/mnt/synology/transcription-backups sh deploy/backup/create_postgres_backup.sh
+```
+
+## 3. Restoring from backup
+
+Restore requires downtime for app + worker writes.
+
+1. Stop app and worker:
+   - `docker compose --env-file .env.production -f docker-compose.production.yml stop app worker`
+2. Restore:
+   - `sh deploy/backup/restore_postgres_backup.sh ./data/backups/postgres-YYYYMMDD-HHMMSS.dump`
+3. Start app and worker:
+   - `docker compose --env-file .env.production -f docker-compose.production.yml start app worker`
+4. Validate `/healthz` and run one smoke workflow.
+
+## 4. Retention and recovery targets
+
+- Retention baseline: keep at least 14 days of backups locally.
+- Synology copy: replicate each new backup to DS420j mounted path.
+- Periodic restore drill: run at least once per release cycle to verify recovery.
diff --git a/docs/production-runbook.md b/docs/production-runbook.md
index 4073e5b..3411f0c 100644
--- a/docs/production-runbook.md
+++ b/docs/production-runbook.md
@@ -12,6 +12,7 @@ This runbook is the operational checklist for releasing and monitoring the trans
    - `OPENROUTER_API_KEY`
    - `DATABASE__*`
    - filesystem paths for data/logs/backups.
+   - `CLOUDFLARE_TUNNEL_TOKEN`
 5. Confirm schema contract alignment is current:
    - `src/transcription/db/models.py`
    - `docs/schema.md`
@@ -33,6 +34,8 @@ This runbook is the operational checklist for releasing and monitoring the trans
 4. Verify log flow:
    - stdout aggregation receives events
    - file logs are written under `./data/logs`
+5. Create a fresh PostgreSQL backup after successful deployment:
+   - `sh deploy/backup/create_postgres_backup.sh`
 
 ## 3. Rollback triggers and actions
 
@@ -50,6 +53,8 @@ This runbook is the operational checklist for releasing and monitoring the trans
 4. Preserve incident evidence:
    - `./data/logs`
    - relevant DB rows (`job`, `job_source`, `execution_attempt`)
+5. If persistence regression is confirmed, restore the latest valid DB dump:
+   - `sh deploy/backup/restore_postgres_backup.sh <dump-file>`
 
 ## 4. Post-release monitoring checklist
 
@@ -90,10 +95,17 @@ This runbook is the operational checklist for releasing and monitoring the trans
 ### Cloudflare ingress/access failure
 
 1. Check `cloudflared` container logs for ingress parse, DNS, or auth failures.
-2. Confirm `deploy/cloudflared/config.yml` tunnel UUID and hostname mappings are correct.
-3. Confirm `deploy/cloudflared/credentials.json` matches the tunnel configured in Cloudflare.
+2. Confirm `deploy/cloudflared/config.yml` hostname mappings are correct.
+3. Confirm `CLOUDFLARE_TUNNEL_TOKEN` in `.env.production` matches the tunnel configured in Cloudflare.
 4. Confirm Cloudflare Access app policy includes the intended identity/group for that hostname.
 
+### Backup or restore failure
+
+1. Verify `postgres` container is healthy and accepting connections.
+2. Confirm dump file exists and is non-zero size.
+3. Re-run backup/restore scripts with explicit `ENV_FILE` and `COMPOSE_FILE` if using non-default paths.
+4. If Synology copy fails, keep local backup and resolve mount/network before next backup cycle.
+
 ## 6. Dependency upgrade policy
 
 Dependencies are declared in `pyproject.toml` and resolved through the committed
diff --git a/docs/v6_0_hosting_migration_plan.md b/docs/v6_0_hosting_migration_plan.md
index c98e664..150bdc5 100644
--- a/docs/v6_0_hosting_migration_plan.md
+++ b/docs/v6_0_hosting_migration_plan.md
@@ -60,6 +60,12 @@ Implemented workflow references:
 2. Validate restore drill from Synology-hosted dump artifacts.
 3. Document rollback procedure for deployment failure and migration failure scenarios.
 
+Implemented workflow references:
+
+- `deploy/backup/create_postgres_backup.sh`
+- `deploy/backup/restore_postgres_backup.sh`
+- `docs/backup_restore.md`
+
 ## 3.5 Validation and release gate
 
 1. `/healthz` confirms app and worker healthy in deployed environment.