generated from john/python-template
@@ -29,6 +29,10 @@ ENV PYTHONDONTWRITEBYTECODE=1 \
|
||||
|
||||
WORKDIR /app
|
||||
|
||||
RUN apt-get update \
|
||||
&& apt-get install -y --no-install-recommends postgresql-client \
|
||||
&& rm -rf /var/lib/apt/lists/*
|
||||
|
||||
RUN groupadd --system --gid 1001 appgroup \
|
||||
&& useradd --system --uid 1001 --gid appgroup --create-home appuser
|
||||
|
||||
|
||||
@@ -22,9 +22,34 @@ if [ -z "${SYNOLOGY_BACKUP_DIR}" ] && [ -f "${ENV_FILE}" ]; then
|
||||
)"
|
||||
fi
|
||||
|
||||
docker compose --env-file "${ENV_FILE}" -f "${COMPOSE_FILE}" exec -T postgres sh -lc \
|
||||
use_compose_backup=0
|
||||
if command -v docker >/dev/null 2>&1 && [ -S /var/run/docker.sock ] && [ -f "${COMPOSE_FILE}" ]; then
|
||||
use_compose_backup=1
|
||||
fi
|
||||
|
||||
if [ "${use_compose_backup}" -eq 1 ]; then
|
||||
docker compose --env-file "${ENV_FILE}" -f "${COMPOSE_FILE}" exec -T postgres sh -lc \
|
||||
"PGPASSWORD=\"\$POSTGRES_PASSWORD\" pg_dump -U \"\$POSTGRES_USER\" -d \"\$POSTGRES_DB\" -Fc" \
|
||||
> "${BACKUP_DIR}/${postgres_file}"
|
||||
else
|
||||
: "${POSTGRES_HOST:=postgres}"
|
||||
: "${POSTGRES_PORT:=5432}"
|
||||
if [ -z "${POSTGRES_DB:-}" ] || [ -z "${POSTGRES_USER:-}" ] || [ -z "${POSTGRES_PASSWORD:-}" ]; then
|
||||
echo "POSTGRES_DB, POSTGRES_USER, and POSTGRES_PASSWORD must be set for in-container backup mode." >&2
|
||||
exit 1
|
||||
fi
|
||||
if ! command -v pg_dump >/dev/null 2>&1; then
|
||||
echo "pg_dump is not installed in this environment." >&2
|
||||
exit 1
|
||||
fi
|
||||
PGPASSWORD="${POSTGRES_PASSWORD}" pg_dump \
|
||||
-h "${POSTGRES_HOST}" \
|
||||
-p "${POSTGRES_PORT}" \
|
||||
-U "${POSTGRES_USER}" \
|
||||
-d "${POSTGRES_DB}" \
|
||||
-Fc \
|
||||
> "${BACKUP_DIR}/${postgres_file}"
|
||||
fi
|
||||
|
||||
cat > "${BACKUP_DIR}/${manifest_file}" <<EOF
|
||||
created_at_utc=${timestamp}
|
||||
@@ -48,12 +73,17 @@ if [ -n "${SYNOLOGY_BACKUP_DIR}" ]; then
|
||||
cp "${BACKUP_DIR}/${postgres_file}" "${SYNOLOGY_BACKUP_DIR}/${postgres_file}"
|
||||
|
||||
# 2) Snapshot deployment config and logs to Synology each run.
|
||||
set -- "${ENV_FILE}" "${COMPOSE_FILE}"
|
||||
if [ -f deploy/cloudflared/config.yml ]; then
|
||||
set -- "$@" deploy/cloudflared/config.yml
|
||||
fi
|
||||
set --
|
||||
[ -f "${ENV_FILE}" ] && set -- "$@" "${ENV_FILE}"
|
||||
[ -f "${COMPOSE_FILE}" ] && set -- "$@" "${COMPOSE_FILE}"
|
||||
[ -f deploy/cloudflared/config.yml ] && set -- "$@" deploy/cloudflared/config.yml
|
||||
if [ "$#" -gt 0 ]; then
|
||||
tar -C . -czf "${SYNOLOGY_BACKUP_DIR}/${config_file}" "$@"
|
||||
else
|
||||
: > "${SYNOLOGY_BACKUP_DIR}/${config_file}"
|
||||
fi
|
||||
|
||||
if [ "${use_compose_backup}" -eq 1 ]; then
|
||||
docker compose --env-file "${ENV_FILE}" -f "${COMPOSE_FILE}" run --rm --no-deps \
|
||||
-v "${SYNOLOGY_BACKUP_DIR}:/backup" \
|
||||
--entrypoint sh app -lc \
|
||||
@@ -74,6 +104,24 @@ if [ -n "${SYNOLOGY_BACKUP_DIR}" ]; then
|
||||
cp -an /app/uploads/photos/. /backup/uploads/photos/;
|
||||
fi
|
||||
'
|
||||
else
|
||||
if [ -d /app/data/logs ]; then
|
||||
tar -C /app/data/logs -czf "${SYNOLOGY_BACKUP_DIR}/${logs_file}" .
|
||||
else
|
||||
: > "${SYNOLOGY_BACKUP_DIR}/${logs_file}"
|
||||
fi
|
||||
|
||||
mkdir -p "${SYNOLOGY_BACKUP_DIR}/uploads/documents" "${SYNOLOGY_BACKUP_DIR}/uploads/photos"
|
||||
if [ -f /app/uploads/homepage.md ] && [ ! -f "${SYNOLOGY_BACKUP_DIR}/uploads/homepage.md" ]; then
|
||||
cp /app/uploads/homepage.md "${SYNOLOGY_BACKUP_DIR}/uploads/homepage.md"
|
||||
fi
|
||||
if [ -d /app/uploads/documents ]; then
|
||||
cp -an /app/uploads/documents/. "${SYNOLOGY_BACKUP_DIR}/uploads/documents/"
|
||||
fi
|
||||
if [ -d /app/uploads/photos ]; then
|
||||
cp -an /app/uploads/photos/. "${SYNOLOGY_BACKUP_DIR}/uploads/photos/"
|
||||
fi
|
||||
fi
|
||||
|
||||
cp "${BACKUP_DIR}/${manifest_file}" "${SYNOLOGY_BACKUP_DIR}/${manifest_file}"
|
||||
|
||||
|
||||
@@ -1,94 +0,0 @@
|
||||
# V6.0 Hosting Migration Formal Plan
|
||||
|
||||
This plan formalizes the V6.0 roadmap objective: move from local-only operation to secure, stable remote hosting.
|
||||
|
||||
## 1. Confirmed Infrastructure Decisions
|
||||
|
||||
- **Runtime host:** dedicated Debian/Ubuntu VM on Proxmox 8.4.x.
|
||||
- **Remote ingress:** Cloudflare Tunnel + Cloudflare Access with public hostnames per internal service.
|
||||
- **Backup target:** Synology DS420j for PostgreSQL dumps and restore points.
|
||||
- **Out of scope for V6.0:** Synology as primary live upload/image storage.
|
||||
|
||||
## 2. Target Runtime Topology
|
||||
|
||||
The production stack is deployed with Docker Compose on the Proxmox VM:
|
||||
|
||||
1. `app` service (FastAPI + NiceGUI runtime)
|
||||
2. `worker` service (async transcription worker)
|
||||
3. `postgres` service (primary datastore)
|
||||
4. `cloudflared` service (tunnel ingress to app endpoints)
|
||||
|
||||
Persistence:
|
||||
|
||||
- PostgreSQL data: durable local volume on Proxmox VM.
|
||||
- App data/log paths: durable local volume(s) on Proxmox VM.
|
||||
- Backups: scheduled PostgreSQL dump artifacts replicated to Synology DS420j.
|
||||
|
||||
## 3. V6.0 Workstreams
|
||||
|
||||
## 3.1 Deployment and configuration
|
||||
|
||||
1. Produce production-ready Docker/Compose definitions for `app`, `worker`, `postgres`, and `cloudflared`.
|
||||
2. Move runtime settings to environment-based configuration (DB, uploads, prompts, logging, secrets).
|
||||
3. Add operational defaults (health checks, restart policies, predictable service dependencies).
|
||||
|
||||
## 3.2 Database migration (SQLite -> PostgreSQL)
|
||||
|
||||
1. Define a deterministic migration method from SQLite to PostgreSQL.
|
||||
2. Run migration in staging-like environment and validate entity counts and key relationships.
|
||||
3. Execute cutover with rollback guardrails and preserved evidence/provenance history.
|
||||
|
||||
Implemented workflow references:
|
||||
|
||||
- `tools/export_import_migration.py` (`export`, `import`, `migrate`, `verify`)
|
||||
- `docs/data_migration.md` for cutover and rollback procedure
|
||||
|
||||
## 3.3 Cloudflare remote access
|
||||
|
||||
1. Configure tunnel routing for service hostnames.
|
||||
2. Apply Cloudflare Access policies for identity-gated remote access.
|
||||
3. Keep non-required administrative/internal surfaces LAN-only unless explicitly approved.
|
||||
|
||||
Implemented workflow references:
|
||||
|
||||
- `deploy/cloudflared/config.yml.example`
|
||||
- `docs/cloudflare_tunnel_access.md`
|
||||
- `docker-compose.production.yml` (`cloudflared` forced to `--protocol http2` + explicit DNS resolvers for LXC reliability)
|
||||
|
||||
## 3.4 Backup, restore, rollback
|
||||
|
||||
1. Define backup schedule, retention, and artifact naming/versioning.
|
||||
2. Validate restore drill from Synology-hosted dump artifacts.
|
||||
3. Document rollback procedure for deployment failure and migration failure scenarios.
|
||||
|
||||
Implemented workflow references:
|
||||
|
||||
- `deploy/backup/create_postgres_backup.sh`
|
||||
- `deploy/backup/restore_postgres_backup.sh`
|
||||
- `docs/backup_restore.md`
|
||||
|
||||
## 3.5 Validation and release gate
|
||||
|
||||
1. `/healthz` confirms app and worker healthy in deployed environment.
|
||||
2. One end-to-end document -> source -> job workflow succeeds through remote access.
|
||||
3. Backup/restore drill completes and data integrity checks pass.
|
||||
|
||||
## 4. Deliverables
|
||||
|
||||
- Production-ready `docker-compose` deployment for app + worker + PostgreSQL + cloudflared.
|
||||
- Environment configuration model suitable for production secrets and runtime overrides.
|
||||
- Repeatable SQLite-to-PostgreSQL migration procedure with cutover checklist.
|
||||
- Updated runbook content for deploy, rollback, backup, and restore.
|
||||
|
||||
## 5. Exit Criteria (V6.0 Complete)
|
||||
|
||||
- Health and worker liveness are green in deployed runtime.
|
||||
- Remote transcription workflow is successful and stable.
|
||||
- Backup and restore are tested and documented.
|
||||
- Evidence/provenance guarantees remain intact (append-only attempt history and traceability preserved).
|
||||
|
||||
## 6. Open Decisions to Finalize During Implementation
|
||||
|
||||
- Hostname inventory and naming convention for each remotely exposed internal service.
|
||||
- Cloudflare Access policy granularity (per-service policy shape and identity groups).
|
||||
- Backup retention windows and RPO/RTO targets aligned with available Synology capacity.
|
||||
Reference in New Issue
Block a user