diff --git a/.env.example b/.env.example deleted file mode 100644 index d9357b7..0000000 --- a/.env.example +++ /dev/null @@ -1,68 +0,0 @@ -# Canonical settings mirror for src/transcription/config.py (Settings). -# Any value here overrides the in-code default. - -# --- NiceGUI Server --- -HOST=0.0.0.0 -PORT=8000 -# LOG_LEVEL: critical | error | warning | info | debug | trace -LOG_LEVEL=info -RELOAD=false -LOG_DIR=./data/logs -LOG_FILE_NAME=transcription.log -LOG_FILE_MAX_BYTES=10485760 -LOG_FILE_BACKUP_COUNT=5 - -# --- AI provider --- -# PROVIDER: openrouter -PROVIDER=openrouter -# Required. -OPENROUTER_API_KEY=your-api-key-goes-here -PROVIDER_MODEL=google/gemini-2.5-flash -# PROVIDER_MODELS default: derived from PROVIDER_MODEL when omitted. -# If provided, use a non-empty JSON array. -# PROVIDER_MODELS=["google/gemini-2.5-flash","anthropic/claude-sonnet-4"] -# OPENROUTER_HTTP_REFERER= -# OPENROUTER_APP_TITLE= -DEFAULT_PROMPT_NAME=transcribe_document.md -# TRANSCRIPTION_TEMPERATURE default: unset (optional range 0.0..2.0) -# TRANSCRIPTION_TEMPERATURE= -# TRANSCRIPTION_TOP_P default: unset (optional range 0.0..1.0) -# TRANSCRIPTION_TOP_P= - -# --- runtime environment --- -# ENVIRONMENT: development | test | production -ENVIRONMENT=development -# TRANSCRIPTION_COMMIT default: unset (optional build/commit identifier for provenance evidence) -# TRANSCRIPTION_COMMIT= -# RUN_EMBEDDED_WORKER default: true (set false when running a standalone worker process) -RUN_EMBEDDED_WORKER=true - -# --- persistence --- -# Use nested keys (env_nested_delimiter="__"). -DATABASE__DRIVER=sqlite -DATABASE__PATH=./data/transcription.db -# Postgres example: -# DATABASE__DRIVER=postgres -# DATABASE__HOST=localhost -# DATABASE__PORT=5432 -# DATABASE__DATABASE=transcription -# DATABASE__USER=postgres -# DATABASE__PASSWORD=change-me -BOOTSTRAP_SCHEMA_ON_STARTUP=false -SQLITE_CHECK_SAME_THREAD=false - -# --- filesystem paths --- -UPLOAD_DIR=./data -PROMPT_DIR=./prompts -DATABASE_BACKUP_DIR=./data/backups - -# --- worker reliability --- -WORKER_MAX_RETRIES=0 -WORKER_PROVIDER_TIMEOUT_SECONDS=30.0 -WORKER_STALE_JOB_SECONDS=30.0 -WORKER_RETRY_BACKOFF_SECONDS=1.0 -WORKER_SHUTDOWN_GRACE_SECONDS=5.0 -WORKER_POLL_INTERVAL_SECONDS=1.0 -WORKER_MIN_TRANSCRIPTION_CHARS=0 -WORKER_MIN_TRANSCRIPTION_LINES=0 -WORKER_FAIL_ON_FINISH_REASON_LENGTH=false diff --git a/.env.production.example b/.env.production.example index 327ea35..1da72a8 100644 --- a/.env.production.example +++ b/.env.production.example @@ -6,6 +6,7 @@ PORT=8000 LOG_LEVEL=info RELOAD=false ENVIRONMENT=production +# TRANSCRIPTION_COMMIT= RUN_EMBEDDED_WORKER=false LOG_DIR=/app/data/logs LOG_FILE_NAME=transcription.log @@ -16,10 +17,16 @@ LOG_FILE_BACKUP_COUNT=5 PROVIDER=openrouter OPENROUTER_API_KEY=replace-with-real-key PROVIDER_MODEL=google/gemini-2.5-flash +# PROVIDER_MODELS=["google/gemini-2.5-flash","anthropic/claude-sonnet-4"] +# OPENROUTER_HTTP_REFERER= +# OPENROUTER_APP_TITLE= DEFAULT_PROMPT_NAME=transcribe_document.md +# TRANSCRIPTION_TEMPERATURE= +# TRANSCRIPTION_TOP_P= # --- persistence --- DATABASE__DRIVER=postgres +# DATABASE__PATH=./data/transcription.db DATABASE__HOST=postgres DATABASE__PORT=5432 DATABASE__DATABASE=transcription @@ -31,14 +38,9 @@ SQLITE_CHECK_SAME_THREAD=false # --- filesystem paths --- UPLOAD_DIR=/app/uploads PROMPT_DIR=/app/prompts -DATABASE_BACKUP_DIR=/backup # --- backup workflow helpers (not Runtime Settings model fields) --- BACKUP_DIR=/backup -# APP_DATA_BACKUP_DIR=/backup/data -# UPLOADS_BACKUP_DIR=/backup/uploads BACKUP_RETENTION_DAYS=14 -# Optional mounted Synology destination path for replicated dumps -# SYNOLOGY_BACKUP_DIR=/mnt/synology/transcription-backups # --- worker reliability --- WORKER_MAX_RETRIES=0 diff --git a/.github/instructions/documentation-sync.instructions.md b/.github/instructions/documentation-sync.instructions.md index 6f38df4..a964522 100644 --- a/.github/instructions/documentation-sync.instructions.md +++ b/.github/instructions/documentation-sync.instructions.md @@ -15,7 +15,7 @@ Keep docs in sync in the same change whenever implementation alters a documented 2. **Configuration contract** - `Settings` keys, defaults, required/optional environment values. - - **Required doc update:** `.env.example` and any directly related setup docs. + - **Required doc update:** `.env.production.example` and any directly related setup docs. 3. **User-visible UI behavior** - Page flow, routes, button/action behavior, labels, status wording, empty/error states. diff --git a/.github/instructions/services.instructions.md b/.github/instructions/services.instructions.md index 99ff90f..a7b894a 100644 --- a/.github/instructions/services.instructions.md +++ b/.github/instructions/services.instructions.md @@ -145,7 +145,7 @@ durability is separately enforced by - `Source.raw_transcription` is a projection, not authoritative history. - Service/UI read paths that touch relationships must be eager-loaded for `lazy="raise"` compatibility. - If model fields, enums, constraints, indexes, or relationship-loading semantics change, update `docs/schema.md` in the same change. -- If `Settings` fields or defaults change in `src/transcription/config.py`, update `.env.example` in the same change so keys/defaults remain synchronized and no stale settings remain documented. +- If `Settings` fields or defaults change in `src/transcription/config.py`, update `.env.production.example` in the same change so keys/defaults remain synchronized and no stale settings remain documented. ## Schema Drift and Legacy Compatibility Policy diff --git a/.github/workflows/quality-gate.yml b/.github/workflows/quality-gate.yml index 642e5e4..c2f494d 100644 --- a/.github/workflows/quality-gate.yml +++ b/.github/workflows/quality-gate.yml @@ -27,13 +27,13 @@ jobs: - name: Write placeholder configuration # Settings requires openrouter_api_key and 115 tests cannot construct - # Settings without it. This is written to a .env file rather than exported + # Settings without it. This is written to .env.production rather than exported # as an environment variable on purpose: the external tests guard on # os.getenv("OPENROUTER_API_KEY"), which reads the process environment and # not the file, so writing the file reproduces the local result exactly - # the 4 external tests skip instead of running against a fake key and # failing. Exporting it instead produces 3 failures. - run: echo "OPENROUTER_API_KEY=ci-placeholder-not-a-real-key" > .env + run: echo "OPENROUTER_API_KEY=ci-placeholder-not-a-real-key" > .env.production - name: Lint and type check # Runs the hooks defined in .pre-commit-config.yaml instead of repeating diff --git a/README.md b/README.md index 7845ee9..084b848 100644 --- a/README.md +++ b/README.md @@ -22,13 +22,13 @@ uv sync ### 2) Configure environment -Create a `.env` file in the project root with the required OpenRouter API key: +Create a `.env.production` file in the project root with the required OpenRouter API key: ```env OPENROUTER_API_KEY=your_openrouter_api_key ``` -Settings are read from CLI arguments first, then environment variables, then `.env`, then the defaults below. +Settings are read from CLI arguments first, then environment variables, then `.env.production`, then the defaults below. ### Configuration Source Precedence @@ -37,13 +37,13 @@ When the same setting is provided in multiple places, the value is chosen in thi 1. CLI arguments (for example `--port 8000`) 2. Settings constructor arguments (used mainly in tests) 3. Environment variables -4. `.env` file values +4. `.env.production` file values 5. Model defaults in code Practical examples: -- `--port 8000` overrides both `PORT=8000` in the shell and `PORT=7000` in `.env`. -- `DATABASE__PATH=prod.db` in the shell overrides `DATABASE__PATH=dev.db` in `.env`. +- `--port 8000` overrides both `PORT=8000` in the shell and `PORT=7000` in `.env.production`. +- `DATABASE__PATH=prod.db` in the shell overrides `DATABASE__PATH=dev.db` in `.env.production`. #### Server and runtime @@ -93,7 +93,7 @@ DATABASE__USER=postgres DATABASE__PASSWORD=change-me ``` -This uses Pydantic nested settings (`env_nested_delimiter='__'`) and avoids JSON blobs in `.env`. A top-level `DATABASE={...}` JSON value is still supported as a fallback, and nested keys such as `DATABASE__PATH` take precedence over conflicting JSON keys. +This uses Pydantic nested settings (`env_nested_delimiter='__'`) and avoids JSON blobs in env files. A top-level `DATABASE={...}` JSON value is still supported as a fallback, and nested keys such as `DATABASE__PATH` take precedence over conflicting JSON keys. `BOOTSTRAP_SCHEMA_ON_STARTUP` creates missing tables when the app starts. When unset, it is enabled in `development` and `test`, and disabled in `production`; set it explicitly to override that policy. `SQLITE_CHECK_SAME_THREAD` defaults to `false`. diff --git a/deploy/backup/create_postgres_backup.sh b/deploy/backup/create_postgres_backup.sh index aa3a73e..673ce2d 100644 --- a/deploy/backup/create_postgres_backup.sh +++ b/deploy/backup/create_postgres_backup.sh @@ -1,68 +1,40 @@ #!/usr/bin/env sh set -eu -COMPOSE_FILE="${COMPOSE_FILE:-docker-compose.production.yml}" -ENV_FILE="${ENV_FILE:-.env.production}" -BACKUP_DIR="${BACKUP_DIR:-${DATABASE_BACKUP_DIR:-./data/backups}}" -APP_DATA_BACKUP_DIR="${APP_DATA_BACKUP_DIR:-${BACKUP_DIR}/data}" -UPLOADS_BACKUP_DIR="${UPLOADS_BACKUP_DIR:-${BACKUP_DIR}/uploads}" +BACKUP_DIR="${BACKUP_DIR:-./data/backups}" RETENTION_DAYS="${BACKUP_RETENTION_DAYS:-14}" -SYNOLOGY_BACKUP_DIR="${SYNOLOGY_BACKUP_DIR:-}" +UPLOAD_DIR="${UPLOAD_DIR:-/app/uploads}" +PROMPT_DIR="${PROMPT_DIR:-/app/prompts}" timestamp="$(date -u +%Y%m%d-%H%M%S)" postgres_file="postgres-${timestamp}.dump" -config_file="config-${timestamp}.tar.gz" -logs_file="logs-${timestamp}.tar.gz" manifest_file="backup-${timestamp}.manifest" mkdir -p "${BACKUP_DIR}" -mkdir -p "${APP_DATA_BACKUP_DIR}" -app_data_backup_abs="$(cd "${APP_DATA_BACKUP_DIR}" && pwd -P)" - -# If SYNOLOGY_BACKUP_DIR wasn't exported in the shell, read it from ENV_FILE. -if [ -z "${SYNOLOGY_BACKUP_DIR}" ] && [ -f "${ENV_FILE}" ]; then - SYNOLOGY_BACKUP_DIR="$( - sed -n 's/^SYNOLOGY_BACKUP_DIR=//p' "${ENV_FILE}" | tail -n 1 - )" +: "${POSTGRES_HOST:=postgres}" +: "${POSTGRES_PORT:=5432}" +if [ -z "${POSTGRES_DB:-}" ] || [ -z "${POSTGRES_USER:-}" ] || [ -z "${POSTGRES_PASSWORD:-}" ]; then + echo "POSTGRES_DB, POSTGRES_USER, and POSTGRES_PASSWORD must be set." >&2 + exit 1 fi - -use_compose_backup=0 -if command -v docker >/dev/null 2>&1 && [ -S /var/run/docker.sock ] && [ -f "${COMPOSE_FILE}" ]; then - use_compose_backup=1 -fi - -if [ "${use_compose_backup}" -eq 1 ]; then - docker compose --env-file "${ENV_FILE}" -f "${COMPOSE_FILE}" exec -T postgres sh -lc \ - "PGPASSWORD=\"\$POSTGRES_PASSWORD\" pg_dump -U \"\$POSTGRES_USER\" -d \"\$POSTGRES_DB\" -Fc" \ - > "${BACKUP_DIR}/${postgres_file}" -else - : "${POSTGRES_HOST:=postgres}" - : "${POSTGRES_PORT:=5432}" - if [ -z "${POSTGRES_DB:-}" ] || [ -z "${POSTGRES_USER:-}" ] || [ -z "${POSTGRES_PASSWORD:-}" ]; then - echo "POSTGRES_DB, POSTGRES_USER, and POSTGRES_PASSWORD must be set for in-container backup mode." >&2 - exit 1 - fi - if ! command -v pg_dump >/dev/null 2>&1; then - echo "pg_dump is not installed in this environment." >&2 - exit 1 - fi - PGPASSWORD="${POSTGRES_PASSWORD}" pg_dump \ - -h "${POSTGRES_HOST}" \ - -p "${POSTGRES_PORT}" \ - -U "${POSTGRES_USER}" \ - -d "${POSTGRES_DB}" \ - -Fc \ - > "${BACKUP_DIR}/${postgres_file}" +if ! command -v pg_dump >/dev/null 2>&1; then + echo "pg_dump is not installed in this environment." >&2 + exit 1 fi +PGPASSWORD="${POSTGRES_PASSWORD}" pg_dump \ + -h "${POSTGRES_HOST}" \ + -p "${POSTGRES_PORT}" \ + -U "${POSTGRES_USER}" \ + -d "${POSTGRES_DB}" \ + -Fc \ + > "${BACKUP_DIR}/${postgres_file}" cat > "${BACKUP_DIR}/${manifest_file}" < "${SYNOLOGY_BACKUP_DIR}/${config_file}" - fi - - if [ "${use_compose_backup}" -eq 1 ]; then - docker compose --env-file "${ENV_FILE}" -f "${COMPOSE_FILE}" run --rm --no-deps \ - -v "${SYNOLOGY_BACKUP_DIR}:/backup" \ - --entrypoint sh app -lc \ - "if [ -d /app/data/logs ]; then tar -C /app/data/logs -czf /backup/${logs_file} .; else : > /backup/${logs_file}; fi" - - else - if [ -d /app/data/logs ]; then - tar -C /app/data/logs -czf "${SYNOLOGY_BACKUP_DIR}/${logs_file}" . - else - : > "${SYNOLOGY_BACKUP_DIR}/${logs_file}" - fi - - mkdir -p "${SYNOLOGY_BACKUP_DIR}/uploads" - cp -an "${UPLOADS_BACKUP_DIR}/." "${SYNOLOGY_BACKUP_DIR}/uploads/" - fi - - cp "${BACKUP_DIR}/${manifest_file}" "${SYNOLOGY_BACKUP_DIR}/${manifest_file}" - - # Keep rolling dump/config/log/manifest snapshots on Synology. - find "${SYNOLOGY_BACKUP_DIR}" -maxdepth 1 -type f \( \ - -name 'postgres-*.dump' -o \ - -name 'config-*.tar.gz' -o \ - -name 'logs-*.tar.gz' -o \ - -name 'backup-*.manifest' \ - \) -mtime +"${RETENTION_DAYS}" -delete +prompts_backup_dir="${BACKUP_DIR}/prompts" +mkdir -p "${prompts_backup_dir}" +if [ -d "${PROMPT_DIR}" ]; then + cp -a "${PROMPT_DIR}/." "${prompts_backup_dir}/" fi echo "Created backup set:" echo " ${BACKUP_DIR}/${postgres_file}" echo " ${BACKUP_DIR}/${manifest_file}" -echo " ${APP_DATA_BACKUP_DIR}/ (incremental app data mirror)" -echo " ${UPLOADS_BACKUP_DIR}/ (incremental uploads mirror)" -if [ -n "${SYNOLOGY_BACKUP_DIR}" ]; then - echo "Synology artifacts:" - echo " ${SYNOLOGY_BACKUP_DIR}/${postgres_file}" - echo " ${SYNOLOGY_BACKUP_DIR}/${config_file}" - echo " ${SYNOLOGY_BACKUP_DIR}/${logs_file}" - echo " ${SYNOLOGY_BACKUP_DIR}/uploads/ (copied from local incremental mirror)" - echo " ${SYNOLOGY_BACKUP_DIR}/${manifest_file}" -else - echo "SYNOLOGY_BACKUP_DIR not set; skipped Synology copy." -fi +echo " ${uploads_backup_dir}/ (incremental uploads mirror)" +echo " ${prompts_backup_dir}/ (prompts mirror)" diff --git a/docker-compose.yml b/docker-compose.yml index 52e9314..6438e40 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -5,7 +5,7 @@ services: dockerfile: Dockerfile container_name: transcription-app env_file: - - .env + - .env.production environment: # Database configuration uses nested settings names (env_nested_delimiter="__"). # DATABASE_URL is NOT read by the application and must not be used here. diff --git a/docs/backup_restore.md b/docs/backup_restore.md index 548f7f2..fa3cbbf 100644 --- a/docs/backup_restore.md +++ b/docs/backup_restore.md @@ -1,75 +1,46 @@ -# Backup and Restore (V6.0 Phase 4) +# Backup and Restore (V6.1) -This guide defines operational backup/restore for clean-slate recovery of the Docker runtime with host-level backups that Synology Drive can replicate. +This guide defines operational backup/restore for clean-slate recovery of the Docker runtime using a host-visible backup folder. -## 1. Backup artifacts (clean-slate recovery set) +## 1. Backup artifacts -- Primary local backup location: `DATABASE_BACKUP_DIR` (recommended production value: `/backup`) -- Local files created per run: +- Backup target root: `BACKUP_DIR` (recommended production value: `/backup`) +- Database artifact per run: - `postgres-YYYYMMDD-HHMMSS.dump` (PostgreSQL custom dump via `pg_dump -Fc`) - - `backup-YYYYMMDD-HHMMSS.manifest` (artifact index) -- Optional Synology files created per run (when `SYNOLOGY_BACKUP_DIR` is set): - - `postgres-YYYYMMDD-HHMMSS.dump` (copied from local) - - `config-YYYYMMDD-HHMMSS.tar.gz` (deployment config snapshot: `.env.production`, `docker-compose.production.yml`, `deploy/cloudflared/config.yml` when present) - - `logs-YYYYMMDD-HHMMSS.tar.gz` (snapshot of `/app/data/logs`) - - `backup-YYYYMMDD-HHMMSS.manifest` -- Local app-data mirror (updated each run): - - `data/**` -- Local incremental media mirror (copied only if missing): - - `uploads/homepage.md` - - `uploads/documents/**` - - `uploads/photos/**` + - `backup-YYYYMMDD-HHMMSS.manifest` (run manifest) +- Media/config mirrors under `BACKUP_DIR`: + - `uploads/**` (incremental copy: new files only) + - `prompts/**` (prompt directory mirror) + +Retention: + +- `BACKUP_RETENTION_DAYS` applies to `postgres-*.dump` and `backup-*.manifest` files. ## 2. Creating backups -Use the scripted command from the repository root: +Run from repository root: ```bash sh deploy/backup/create_postgres_backup.sh ``` -Environment overrides: +Environment variables used by the backup script: -- `BACKUP_DIR` (default `DATABASE_BACKUP_DIR`, then `./data/backups`) -- `APP_DATA_BACKUP_DIR` (default `${BACKUP_DIR}/data`) -- `UPLOADS_BACKUP_DIR` (default `${BACKUP_DIR}/uploads`) +- `BACKUP_DIR` (default `./data/backups`) - `BACKUP_RETENTION_DAYS` (default `14`) -- `SYNOLOGY_BACKUP_DIR` (if set, dump/config/log/manifest are copied to this mounted path and media is synced incrementally) -- `ENV_FILE` (default `.env.production`) -- `COMPOSE_FILE` (default `docker-compose.production.yml`) +- `UPLOAD_DIR` (default `/app/uploads`) +- `PROMPT_DIR` (default `/app/prompts`) +- `POSTGRES_HOST` (default `postgres`) +- `POSTGRES_PORT` (default `5432`) +- `POSTGRES_DB` (required) +- `POSTGRES_USER` (required) +- `POSTGRES_PASSWORD` (required) Recommended production setup: -- Mount a host-visible folder into `/backup` for both `app` and `worker` services. -- Set `DATABASE_BACKUP_DIR=/backup` in `.env.production`. -- Leave `SYNOLOGY_BACKUP_DIR` empty when Synology Drive Client handles replication from the host folder. - -Example with Synology mount: - -```bash -SYNOLOGY_BACKUP_DIR=/mnt/synology-backups sh deploy/backup/create_postgres_backup.sh -``` - -## 2.1 Optional direct Synology mount (LXC) - -This section is only needed when using direct container-side Synology copy via `SYNOLOGY_BACKUP_DIR`. -Manual `mount` commands are lost after reboot unless persisted. - -Use the template (copy, edit placeholders, then run): - -```bash -cp deploy/backup/mount_synology_cifs.example.sh /root/mount_synology_cifs.sh -nano /root/mount_synology_cifs.sh -sh /root/mount_synology_cifs.sh -``` - -Then add the printed `/etc/fstab` line (with your real values) so the mount survives reboot. - -Recommended pattern: - -- Keep NAS credentials in a local file like `/etc/samba/credentials/transcription-synology` with `chmod 600`. -- Keep `SYNOLOGY_BACKUP_DIR` in `.env.production` aligned to that mount point (for example `/mnt/synology-backups`). -- The script will also read `SYNOLOGY_BACKUP_DIR` from `ENV_FILE` when not exported in the shell. +- Mount a host-visible folder into `/backup` for both `app` and `worker`. +- Set `BACKUP_DIR=/backup` in `.env.production`. +- Use host-level tooling (for example Synology Drive Client on the host) to replicate that folder externally. ## 3. Restoring from backup @@ -77,21 +48,12 @@ Restore requires downtime for app + worker writes. 1. Stop app and worker: - `docker compose --env-file .env.production -f docker-compose.production.yml stop app worker` -2. Restore: - - `sh deploy/backup/restore_postgres_backup.sh ./data/backups/postgres-YYYYMMDD-HHMMSS.dump` - - if `SYNOLOGY_BACKUP_DIR/uploads` exists, media is restored from the Synology mirror. - - if `SYNOLOGY_BACKUP_DIR/config-YYYYMMDD-HHMMSS.tar.gz` exists, config is restored from that archive. +2. Restore database: + - `sh deploy/backup/restore_postgres_backup.sh /backup/postgres-YYYYMMDD-HHMMSS.dump` 3. Start app and worker: - `docker compose --env-file .env.production -f docker-compose.production.yml start app worker` 4. Validate `/healthz` and run one smoke workflow. Notes: -- Config restore extracts the archived files back into the current repository path. -- Legacy full-archive restores (`uploads-*.tar.gz`, `config-*.tar.gz` beside the dump) are still supported for older backups. - -## 4. Retention and recovery targets - -- Retention baseline: keep at least 14 days of backups locally. -- Synology copy: replicate each backup artifact set to DS420j mounted path. -- Periodic restore drill: run at least once per release cycle to verify recovery. +- `restore_postgres_backup.sh` still supports legacy archive restore paths for older backup sets. diff --git a/docs/production-runbook.md b/docs/production-runbook.md index 9ca72f9..891e5d1 100644 --- a/docs/production-runbook.md +++ b/docs/production-runbook.md @@ -37,7 +37,7 @@ This runbook is the operational checklist for releasing and monitoring the trans - stdout aggregation receives events - file logs are written under `./data/logs` 5. Create a fresh PostgreSQL backup after successful deployment: - - `sh deploy/backup/create_postgres_backup.sh` (creates local DB dump + local incremental uploads mirror under `DATABASE_BACKUP_DIR`) + - `sh deploy/backup/create_postgres_backup.sh` (creates DB dump plus uploads/prompts backups under `BACKUP_DIR`) ## 3. Rollback triggers and actions diff --git a/docs/ui/pages/settings.md b/docs/ui/pages/settings.md index 3017565..fcca39b 100644 --- a/docs/ui/pages/settings.md +++ b/docs/ui/pages/settings.md @@ -30,7 +30,7 @@ Settings manages installation-local registries, safe runtime .env settings, and - Runtime Settings includes an explicit "Other settings not shown here" markdown table listing: - secrets (`OPENROUTER_API_KEY`, `DATABASE__PASSWORD`) - high-risk database connection settings (`DATABASE__DRIVER`, `DATABASE__PATH`, `DATABASE__HOST`, `DATABASE__PORT`, `DATABASE__DATABASE`, `DATABASE__USER`) - and deployment/helper keys (`POSTGRES_*`, `CLOUDFLARE_TUNNEL_TOKEN`, `BACKUP_DIR`, `APP_DATA_BACKUP_DIR`, `UPLOADS_BACKUP_DIR`, `BACKUP_RETENTION_DAYS`, `SYNOLOGY_BACKUP_DIR`, `RUNTIME_SETTINGS_ENV_FILE`, `ENV_FILE`, `COMPOSE_FILE`), and directs edits for those keys to the resolved runtime env file path. + and deployment/helper keys (`POSTGRES_*`, `CLOUDFLARE_TUNNEL_TOKEN`, `BACKUP_DIR`, `BACKUP_RETENTION_DAYS`, `RUNTIME_SETTINGS_ENV_FILE`, `ENV_FILE`, `COMPOSE_FILE`) plus legacy/deprecated keys (`DATABASE_BACKUP_DIR`, `APP_DATA_BACKUP_DIR`, `UPLOADS_BACKUP_DIR`, `SYNOLOGY_BACKUP_DIR`), and directs edits for those keys to the resolved runtime env file path. - Document Types, Person Roles, and Tags support Add/Edit/Delete with existing guardrails. - Prompts exposes only `transcribe_document.md` for editing and restore-from-backup. - Home Page Text edits the same Markdown content rendered on `/homepage`. diff --git a/src/transcription/app.py b/src/transcription/app.py index 461bcbf..88f452b 100644 --- a/src/transcription/app.py +++ b/src/transcription/app.py @@ -50,7 +50,6 @@ async def _lifespan(app: FastAPI): settings.upload_dir.mkdir(parents=True, exist_ok=True) settings.prompt_dir.mkdir(parents=True, exist_ok=True) settings.log_dir.mkdir(parents=True, exist_ok=True) - settings.database_backup_dir.mkdir(parents=True, exist_ok=True) await _recover_stale_processing_jobs(app) diff --git a/src/transcription/config.py b/src/transcription/config.py index 433165b..8d0bb14 100644 --- a/src/transcription/config.py +++ b/src/transcription/config.py @@ -1,6 +1,6 @@ """Centralized application configuration. -All settings are loaded from environment variables (or a .env file) +All settings are loaded from environment variables (or an env file) once at startup. Provider-specific defaults (model names, base URLs) are resolved by the provider adapters, not here. """ @@ -65,7 +65,7 @@ DatabaseSettings = Annotated[ class Settings(BaseSettings): model_config = SettingsConfigDict( - env_file=".env", + env_file=".env.production", env_file_encoding="utf-8", extra="ignore", env_nested_delimiter="__", @@ -108,7 +108,6 @@ class Settings(BaseSettings): # --- filesystem paths --- upload_dir: Path = Path("./data") prompt_dir: Path = Path("./prompts") - database_backup_dir: Path = Path("./data/backups") # --- worker reliability --- worker_max_retries: int = Field(default=0, ge=0) diff --git a/src/transcription/ui/runtime_settings_store.py b/src/transcription/ui/runtime_settings_store.py index a8f33f8..953d396 100644 --- a/src/transcription/ui/runtime_settings_store.py +++ b/src/transcription/ui/runtime_settings_store.py @@ -99,12 +99,13 @@ HIDDEN_SETTINGS_CATEGORIES: tuple[HiddenSettingsCategory, ...] = ( "CLOUDFLARE_TUNNEL_TOKEN", "RUNTIME_SETTINGS_ENV_FILE", "BACKUP_DIR", - "APP_DATA_BACKUP_DIR", - "UPLOADS_BACKUP_DIR", "BACKUP_RETENTION_DAYS", - "SYNOLOGY_BACKUP_DIR", "ENV_FILE", "COMPOSE_FILE", + "DATABASE_BACKUP_DIR", + "APP_DATA_BACKUP_DIR", + "UPLOADS_BACKUP_DIR", + "SYNOLOGY_BACKUP_DIR", ), ), ) @@ -250,13 +251,6 @@ RUNTIME_SETTINGS_CATALOG: tuple[RuntimeSettingDescriptor, ...] = ( "Directory containing editable markdown prompts.", "text", ), - RuntimeSettingDescriptor( - "database_backup_dir", - "DATABASE_BACKUP_DIR", - "Database backup directory", - "Directory for generated database backups.", - "text", - ), RuntimeSettingDescriptor( "worker_max_retries", "WORKER_MAX_RETRIES", @@ -404,7 +398,7 @@ def _resolve_env_file_path(*, settings: Settings, env_file_path: Path | None) -> configured = settings.model_config.get("env_file") if configured is None: - return Path(".env") + return Path(".env.production") if isinstance(configured, Path): return Path(configured) if isinstance(configured, str): @@ -415,7 +409,7 @@ def _resolve_env_file_path(*, settings: Settings, env_file_path: Path | None) -> return Path(first) if isinstance(first, str): return Path(first) - return Path(".env") + return Path(".env.production") def _display_value(value: object) -> str | bool: @@ -459,7 +453,7 @@ def _validate_candidate_env(*, env_map: dict[str, str], env_file_path: Path) -> raise AppError( "One or more settings values are invalid.", category=ErrorCategory.VALIDATION, - suggestion="Review the highlighted values and use the same formats shown in .env.example.", + suggestion="Review the highlighted values and use the same formats shown in .env.production.example.", detail=f"Invalid runtime settings for {env_file_path}: {exc}", ) from exc finally: diff --git a/tests/test_config.py b/tests/test_config.py index f299af0..7486d71 100644 --- a/tests/test_config.py +++ b/tests/test_config.py @@ -15,7 +15,7 @@ from transcription.config import parse_cli_settings def _make_settings(**overrides: Any) -> Settings: - """Build a Settings instance with a dummy API key, isolated from any local .env.""" + """Build a Settings instance with a dummy API key, isolated from local env files.""" defaults: dict[str, Any] = {"openrouter_api_key": "test-key-abc123", "provider_models": None} defaults.update(overrides) return Settings(_env_file=None, **defaults) @@ -145,7 +145,6 @@ class TestPathSettings: assert isinstance(settings.upload_dir, Path) assert isinstance(settings.prompt_dir, Path) assert isinstance(settings.log_dir, Path) - assert isinstance(settings.database_backup_dir, Path) def test_configure_logging_writes_rotating_file_logs_to_configured_directory(tmp_path): diff --git a/tests/test_meta_contract_guards.py b/tests/test_meta_contract_guards.py index a352e4d..2f35bd8 100644 --- a/tests/test_meta_contract_guards.py +++ b/tests/test_meta_contract_guards.py @@ -101,14 +101,14 @@ def test_canonical_authority_references_are_present(): assert missing == {} -def _declared_env_example_keys(*, include_commented: bool) -> set[str]: - text = _read(".env.example") +def _declared_env_production_example_keys(*, include_commented: bool) -> set[str]: + text = _read(".env.production.example") pattern = r"^\s*#?\s*([A-Z0-9_]+)\s*=" if include_commented else r"^\s*([A-Z0-9_]+)\s*=" return {match.group(1) for match in re.finditer(pattern, text, flags=re.MULTILINE)} -def _active_env_example_values() -> dict[str, str]: - text = _read(".env.example") +def _active_env_production_example_values() -> dict[str, str]: + text = _read(".env.production.example") return {key: value.strip() for key, value in re.findall(r"^\s*([A-Z0-9_]+)\s*=\s*(.*)$", text, flags=re.MULTILINE)} @@ -121,9 +121,9 @@ def _normalize_env_path_value(value: str | None) -> str | None: return normalized -def test_env_example_keys_match_runtime_settings_contract(): - """Guard against .env.example drift from Settings keys.""" - declared = _declared_env_example_keys(include_commented=True) +def test_env_production_example_keys_match_runtime_settings_contract(): + """Guard against .env.production.example drift from Settings keys.""" + declared = _declared_env_production_example_keys(include_commented=True) settings_keys = {field.upper() for field in Settings.model_fields if field != "database"} database_keys = { "DATABASE__DRIVER", @@ -134,7 +134,16 @@ def test_env_example_keys_match_runtime_settings_contract(): "DATABASE__USER", "DATABASE__PASSWORD", } - allowed = settings_keys | database_keys + deployment_helper_keys = { + "POSTGRES_DB", + "POSTGRES_USER", + "POSTGRES_PASSWORD", + "CLOUDFLARE_TUNNEL_TOKEN", + "RUNTIME_SETTINGS_ENV_FILE", + "BACKUP_DIR", + "BACKUP_RETENTION_DAYS", + } + allowed = settings_keys | database_keys | deployment_helper_keys missing = sorted(allowed - declared) unknown = sorted(declared - allowed) @@ -142,50 +151,11 @@ def test_env_example_keys_match_runtime_settings_contract(): assert unknown == [] -def test_env_example_default_values_match_settings_defaults(): - """Uncommented .env.example entries should mirror in-code defaults.""" - defaults = Settings(_env_file=None, openrouter_api_key="placeholder-key") - expected = { - "HOST": defaults.host, - "PORT": str(defaults.port), - "LOG_LEVEL": defaults.log_level, - "RELOAD": str(defaults.reload).lower(), - "LOG_DIR": str(defaults.log_dir), - "LOG_FILE_NAME": defaults.log_file_name, - "LOG_FILE_MAX_BYTES": str(defaults.log_file_max_bytes), - "LOG_FILE_BACKUP_COUNT": str(defaults.log_file_backup_count), - "PROVIDER": defaults.provider.value, - "PROVIDER_MODEL": defaults.provider_model or "", - "DEFAULT_PROMPT_NAME": defaults.default_prompt_name, - "ENVIRONMENT": defaults.environment, - "DATABASE__DRIVER": defaults.database.driver, - "DATABASE__PATH": getattr(defaults.database, "path", ""), - "BOOTSTRAP_SCHEMA_ON_STARTUP": str(defaults.bootstrap_schema_on_startup).lower(), - "SQLITE_CHECK_SAME_THREAD": str(defaults.sqlite_check_same_thread).lower(), - "UPLOAD_DIR": str(defaults.upload_dir), - "PROMPT_DIR": str(defaults.prompt_dir), - "DATABASE_BACKUP_DIR": str(defaults.database_backup_dir), - "WORKER_MAX_RETRIES": str(defaults.worker_max_retries), - "WORKER_PROVIDER_TIMEOUT_SECONDS": str(defaults.worker_provider_timeout_seconds), - "WORKER_STALE_JOB_SECONDS": str(defaults.worker_stale_job_seconds), - "WORKER_RETRY_BACKOFF_SECONDS": str(defaults.worker_retry_backoff_seconds), - "WORKER_SHUTDOWN_GRACE_SECONDS": str(defaults.worker_shutdown_grace_seconds), - "WORKER_POLL_INTERVAL_SECONDS": str(defaults.worker_poll_interval_seconds), - "WORKER_MIN_TRANSCRIPTION_CHARS": str(defaults.worker_min_transcription_chars), - "WORKER_MIN_TRANSCRIPTION_LINES": str(defaults.worker_min_transcription_lines), - "WORKER_FAIL_ON_FINISH_REASON_LENGTH": str(defaults.worker_fail_on_finish_reason_length).lower(), - } - active = _active_env_example_values() - path_like_keys = {"LOG_DIR", "UPLOAD_DIR", "PROMPT_DIR", "DATABASE_BACKUP_DIR"} - mismatches = { - key: { - "expected": _normalize_env_path_value(expected_value) if key in path_like_keys else expected_value, - "actual": _normalize_env_path_value(active.get(key)) if key in path_like_keys else active.get(key), - } - for key, expected_value in expected.items() - if ( - (_normalize_env_path_value(active.get(key)) if key in path_like_keys else active.get(key)) - != (_normalize_env_path_value(expected_value) if key in path_like_keys else expected_value) - ) - } - assert mismatches == {} +def test_env_production_example_builds_settings_and_keeps_production_defaults(): + """The production template must parse into Settings and preserve intended production posture.""" + active = _active_env_production_example_values() + Settings(_env_file=PROJECT_ROOT / ".env.production.example", _cli_parse_args=False) + + assert active["ENVIRONMENT"] == "production" + assert active["RUN_EMBEDDED_WORKER"] == "false" + assert active["DATABASE__DRIVER"] == "postgres" diff --git a/tests/ui/test_runtime_settings_store.py b/tests/ui/test_runtime_settings_store.py index e308e9a..e669434 100644 --- a/tests/ui/test_runtime_settings_store.py +++ b/tests/ui/test_runtime_settings_store.py @@ -41,7 +41,6 @@ def test_hidden_settings_categories_list_secret_and_high_risk_env_keys(): assert "POSTGRES_PASSWORD" in hidden_keys assert "CLOUDFLARE_TUNNEL_TOKEN" in hidden_keys assert "BACKUP_DIR" in hidden_keys - assert "SYNOLOGY_BACKUP_DIR" in hidden_keys def test_runtime_settings_snapshot_includes_catalog_fields(tmp_path: Path):