diff --git a/Dockerfile b/Dockerfile index 2d35fda..b968824 100644 --- a/Dockerfile +++ b/Dockerfile @@ -29,6 +29,10 @@ ENV PYTHONDONTWRITEBYTECODE=1 \ WORKDIR /app +RUN apt-get update \ + && apt-get install -y --no-install-recommends postgresql-client \ + && rm -rf /var/lib/apt/lists/* + RUN groupadd --system --gid 1001 appgroup \ && useradd --system --uid 1001 --gid appgroup --create-home appuser diff --git a/deploy/backup/create_postgres_backup.sh b/deploy/backup/create_postgres_backup.sh index c4d79cc..7c81cbe 100644 --- a/deploy/backup/create_postgres_backup.sh +++ b/deploy/backup/create_postgres_backup.sh @@ -22,9 +22,34 @@ if [ -z "${SYNOLOGY_BACKUP_DIR}" ] && [ -f "${ENV_FILE}" ]; then )" fi -docker compose --env-file "${ENV_FILE}" -f "${COMPOSE_FILE}" exec -T postgres sh -lc \ - "PGPASSWORD=\"\$POSTGRES_PASSWORD\" pg_dump -U \"\$POSTGRES_USER\" -d \"\$POSTGRES_DB\" -Fc" \ - > "${BACKUP_DIR}/${postgres_file}" +use_compose_backup=0 +if command -v docker >/dev/null 2>&1 && [ -S /var/run/docker.sock ] && [ -f "${COMPOSE_FILE}" ]; then + use_compose_backup=1 +fi + +if [ "${use_compose_backup}" -eq 1 ]; then + docker compose --env-file "${ENV_FILE}" -f "${COMPOSE_FILE}" exec -T postgres sh -lc \ + "PGPASSWORD=\"\$POSTGRES_PASSWORD\" pg_dump -U \"\$POSTGRES_USER\" -d \"\$POSTGRES_DB\" -Fc" \ + > "${BACKUP_DIR}/${postgres_file}" +else + : "${POSTGRES_HOST:=postgres}" + : "${POSTGRES_PORT:=5432}" + if [ -z "${POSTGRES_DB:-}" ] || [ -z "${POSTGRES_USER:-}" ] || [ -z "${POSTGRES_PASSWORD:-}" ]; then + echo "POSTGRES_DB, POSTGRES_USER, and POSTGRES_PASSWORD must be set for in-container backup mode." >&2 + exit 1 + fi + if ! command -v pg_dump >/dev/null 2>&1; then + echo "pg_dump is not installed in this environment." >&2 + exit 1 + fi + PGPASSWORD="${POSTGRES_PASSWORD}" pg_dump \ + -h "${POSTGRES_HOST}" \ + -p "${POSTGRES_PORT}" \ + -U "${POSTGRES_USER}" \ + -d "${POSTGRES_DB}" \ + -Fc \ + > "${BACKUP_DIR}/${postgres_file}" +fi cat > "${BACKUP_DIR}/${manifest_file}" < "${SYNOLOGY_BACKUP_DIR}/${config_file}" fi - tar -C . -czf "${SYNOLOGY_BACKUP_DIR}/${config_file}" "$@" - docker compose --env-file "${ENV_FILE}" -f "${COMPOSE_FILE}" run --rm --no-deps \ - -v "${SYNOLOGY_BACKUP_DIR}:/backup" \ - --entrypoint sh app -lc \ - "if [ -d /app/data/logs ]; then tar -C /app/data/logs -czf /backup/${logs_file} .; else : > /backup/${logs_file}; fi" + if [ "${use_compose_backup}" -eq 1 ]; then + docker compose --env-file "${ENV_FILE}" -f "${COMPOSE_FILE}" run --rm --no-deps \ + -v "${SYNOLOGY_BACKUP_DIR}:/backup" \ + --entrypoint sh app -lc \ + "if [ -d /app/data/logs ]; then tar -C /app/data/logs -czf /backup/${logs_file} .; else : > /backup/${logs_file}; fi" - # 3) Sync uploads incrementally to Synology (copy only new files). - docker compose --env-file "${ENV_FILE}" -f "${COMPOSE_FILE}" run --rm --no-deps \ - -v "${SYNOLOGY_BACKUP_DIR}:/backup" \ - --entrypoint sh app -lc ' - mkdir -p /backup/uploads/documents /backup/uploads/photos; - if [ -f /app/uploads/homepage.md ] && [ ! -f /backup/uploads/homepage.md ]; then - cp /app/uploads/homepage.md /backup/uploads/homepage.md; - fi; - if [ -d /app/uploads/documents ]; then - cp -an /app/uploads/documents/. /backup/uploads/documents/; - fi; - if [ -d /app/uploads/photos ]; then - cp -an /app/uploads/photos/. /backup/uploads/photos/; - fi - ' + # 3) Sync uploads incrementally to Synology (copy only new files). + docker compose --env-file "${ENV_FILE}" -f "${COMPOSE_FILE}" run --rm --no-deps \ + -v "${SYNOLOGY_BACKUP_DIR}:/backup" \ + --entrypoint sh app -lc ' + mkdir -p /backup/uploads/documents /backup/uploads/photos; + if [ -f /app/uploads/homepage.md ] && [ ! -f /backup/uploads/homepage.md ]; then + cp /app/uploads/homepage.md /backup/uploads/homepage.md; + fi; + if [ -d /app/uploads/documents ]; then + cp -an /app/uploads/documents/. /backup/uploads/documents/; + fi; + if [ -d /app/uploads/photos ]; then + cp -an /app/uploads/photos/. /backup/uploads/photos/; + fi + ' + else + if [ -d /app/data/logs ]; then + tar -C /app/data/logs -czf "${SYNOLOGY_BACKUP_DIR}/${logs_file}" . + else + : > "${SYNOLOGY_BACKUP_DIR}/${logs_file}" + fi + + mkdir -p "${SYNOLOGY_BACKUP_DIR}/uploads/documents" "${SYNOLOGY_BACKUP_DIR}/uploads/photos" + if [ -f /app/uploads/homepage.md ] && [ ! -f "${SYNOLOGY_BACKUP_DIR}/uploads/homepage.md" ]; then + cp /app/uploads/homepage.md "${SYNOLOGY_BACKUP_DIR}/uploads/homepage.md" + fi + if [ -d /app/uploads/documents ]; then + cp -an /app/uploads/documents/. "${SYNOLOGY_BACKUP_DIR}/uploads/documents/" + fi + if [ -d /app/uploads/photos ]; then + cp -an /app/uploads/photos/. "${SYNOLOGY_BACKUP_DIR}/uploads/photos/" + fi + fi cp "${BACKUP_DIR}/${manifest_file}" "${SYNOLOGY_BACKUP_DIR}/${manifest_file}" diff --git a/docs/v6_0_hosting_migration_plan.md b/docs/v6_0_hosting_migration_plan.md deleted file mode 100644 index 5f96d9b..0000000 --- a/docs/v6_0_hosting_migration_plan.md +++ /dev/null @@ -1,94 +0,0 @@ -# V6.0 Hosting Migration Formal Plan - -This plan formalizes the V6.0 roadmap objective: move from local-only operation to secure, stable remote hosting. - -## 1. Confirmed Infrastructure Decisions - -- **Runtime host:** dedicated Debian/Ubuntu VM on Proxmox 8.4.x. -- **Remote ingress:** Cloudflare Tunnel + Cloudflare Access with public hostnames per internal service. -- **Backup target:** Synology DS420j for PostgreSQL dumps and restore points. -- **Out of scope for V6.0:** Synology as primary live upload/image storage. - -## 2. Target Runtime Topology - -The production stack is deployed with Docker Compose on the Proxmox VM: - -1. `app` service (FastAPI + NiceGUI runtime) -2. `worker` service (async transcription worker) -3. `postgres` service (primary datastore) -4. `cloudflared` service (tunnel ingress to app endpoints) - -Persistence: - -- PostgreSQL data: durable local volume on Proxmox VM. -- App data/log paths: durable local volume(s) on Proxmox VM. -- Backups: scheduled PostgreSQL dump artifacts replicated to Synology DS420j. - -## 3. V6.0 Workstreams - -## 3.1 Deployment and configuration - -1. Produce production-ready Docker/Compose definitions for `app`, `worker`, `postgres`, and `cloudflared`. -2. Move runtime settings to environment-based configuration (DB, uploads, prompts, logging, secrets). -3. Add operational defaults (health checks, restart policies, predictable service dependencies). - -## 3.2 Database migration (SQLite -> PostgreSQL) - -1. Define a deterministic migration method from SQLite to PostgreSQL. -2. Run migration in staging-like environment and validate entity counts and key relationships. -3. Execute cutover with rollback guardrails and preserved evidence/provenance history. - -Implemented workflow references: - -- `tools/export_import_migration.py` (`export`, `import`, `migrate`, `verify`) -- `docs/data_migration.md` for cutover and rollback procedure - -## 3.3 Cloudflare remote access - -1. Configure tunnel routing for service hostnames. -2. Apply Cloudflare Access policies for identity-gated remote access. -3. Keep non-required administrative/internal surfaces LAN-only unless explicitly approved. - -Implemented workflow references: - -- `deploy/cloudflared/config.yml.example` -- `docs/cloudflare_tunnel_access.md` -- `docker-compose.production.yml` (`cloudflared` forced to `--protocol http2` + explicit DNS resolvers for LXC reliability) - -## 3.4 Backup, restore, rollback - -1. Define backup schedule, retention, and artifact naming/versioning. -2. Validate restore drill from Synology-hosted dump artifacts. -3. Document rollback procedure for deployment failure and migration failure scenarios. - -Implemented workflow references: - -- `deploy/backup/create_postgres_backup.sh` -- `deploy/backup/restore_postgres_backup.sh` -- `docs/backup_restore.md` - -## 3.5 Validation and release gate - -1. `/healthz` confirms app and worker healthy in deployed environment. -2. One end-to-end document -> source -> job workflow succeeds through remote access. -3. Backup/restore drill completes and data integrity checks pass. - -## 4. Deliverables - -- Production-ready `docker-compose` deployment for app + worker + PostgreSQL + cloudflared. -- Environment configuration model suitable for production secrets and runtime overrides. -- Repeatable SQLite-to-PostgreSQL migration procedure with cutover checklist. -- Updated runbook content for deploy, rollback, backup, and restore. - -## 5. Exit Criteria (V6.0 Complete) - -- Health and worker liveness are green in deployed runtime. -- Remote transcription workflow is successful and stable. -- Backup and restore are tested and documented. -- Evidence/provenance guarantees remain intact (append-only attempt history and traceability preserved). - -## 6. Open Decisions to Finalize During Implementation - -- Hostname inventory and naming convention for each remotely exposed internal service. -- Cloudflare Access policy granularity (per-service policy shape and identity groups). -- Backup retention windows and RPO/RTO targets aligned with available Synology capacity.