diff --git a/.env.example b/.env.example index ad29590..d9357b7 100644 --- a/.env.example +++ b/.env.example @@ -34,6 +34,8 @@ DEFAULT_PROMPT_NAME=transcribe_document.md ENVIRONMENT=development # TRANSCRIPTION_COMMIT default: unset (optional build/commit identifier for provenance evidence) # TRANSCRIPTION_COMMIT= +# RUN_EMBEDDED_WORKER default: true (set false when running a standalone worker process) +RUN_EMBEDDED_WORKER=true # --- persistence --- # Use nested keys (env_nested_delimiter="__"). diff --git a/.env.production.example b/.env.production.example new file mode 100644 index 0000000..c6a023b --- /dev/null +++ b/.env.production.example @@ -0,0 +1,53 @@ +# Production environment example for docker-compose.production.yml + +# --- NiceGUI Server --- +HOST=0.0.0.0 +PORT=8000 +LOG_LEVEL=info +RELOAD=false +ENVIRONMENT=production +RUN_EMBEDDED_WORKER=false +LOG_DIR=/app/data/logs +LOG_FILE_NAME=transcription.log +LOG_FILE_MAX_BYTES=10485760 +LOG_FILE_BACKUP_COUNT=5 + +# --- AI provider --- +PROVIDER=openrouter +OPENROUTER_API_KEY=replace-with-real-key +PROVIDER_MODEL=google/gemini-2.5-flash +DEFAULT_PROMPT_NAME=transcribe_document.md + +# --- persistence --- +DATABASE__DRIVER=postgres +DATABASE__HOST=postgres +DATABASE__PORT=5432 +DATABASE__DATABASE=transcription +DATABASE__USER=transcription +DATABASE__PASSWORD=replace-with-strong-password +BOOTSTRAP_SCHEMA_ON_STARTUP=false +SQLITE_CHECK_SAME_THREAD=false + +# --- filesystem paths --- +UPLOAD_DIR=/app/uploads +PROMPT_DIR=/app/prompts +DATABASE_BACKUP_DIR=/app/data/backups + +# --- worker reliability --- +WORKER_MAX_RETRIES=0 +WORKER_PROVIDER_TIMEOUT_SECONDS=30.0 +WORKER_STALE_JOB_SECONDS=30.0 +WORKER_RETRY_BACKOFF_SECONDS=1.0 +WORKER_SHUTDOWN_GRACE_SECONDS=5.0 +WORKER_POLL_INTERVAL_SECONDS=1.0 +WORKER_MIN_TRANSCRIPTION_CHARS=0 +WORKER_MIN_TRANSCRIPTION_LINES=0 +WORKER_FAIL_ON_FINISH_REASON_LENGTH=false + +# --- postgres container --- +POSTGRES_DB=transcription +POSTGRES_USER=transcription +POSTGRES_PASSWORD=replace-with-strong-password + +# --- cloudflare tunnel --- +CLOUDFLARE_TUNNEL_TOKEN=replace-with-cloudflare-tunnel-token diff --git a/README.md b/README.md index fd2ce20..0b69a38 100644 --- a/README.md +++ b/README.md @@ -54,6 +54,7 @@ Practical examples: | `LOG_LEVEL` | `info` | Uvicorn and application log level. | | `RELOAD` | `false` | Restart the development server when source files change. | | `ENVIRONMENT` | `development` | Runtime environment: `development`, `test`, or `production`. | +| `RUN_EMBEDDED_WORKER` | `true` | Run worker loop inside web app process. Set `false` when using a dedicated worker service. | #### Provider @@ -122,6 +123,22 @@ This starts the development server with SQLite, creates missing tables, and enab Replace `localhost` with the server's hostname or IP address when connecting from another machine. +## Production stack (Phase 1) + +Use the production compose profile for split app/worker deployment with PostgreSQL and Cloudflare Tunnel: + +```bash +copy .env.production.example .env.production +docker compose -f docker-compose.production.yml up -d --build +``` + +Services: + +- `app`: FastAPI + NiceGUI runtime (`RUN_EMBEDDED_WORKER=false`) +- `worker`: standalone queue processor (`python -m transcription.worker_service`) +- `postgres`: primary datastore +- `cloudflared`: tunnel client using `CLOUDFLARE_TUNNEL_TOKEN` + ## How to navigate the GUI - **Upload page** (`/ui`) diff --git a/docker-compose.production.yml b/docker-compose.production.yml new file mode 100644 index 0000000..ef79a31 --- /dev/null +++ b/docker-compose.production.yml @@ -0,0 +1,71 @@ +services: + app: + build: + context: . + dockerfile: Dockerfile + image: transcription:prod + env_file: + - .env.production + environment: + RUN_EMBEDDED_WORKER: "false" + depends_on: + postgres: + condition: service_healthy + volumes: + - app_uploads:/app/uploads + - app_data:/app/data + - ./prompts:/app/prompts:ro + restart: unless-stopped + healthcheck: + test: ["CMD", "python", "-c", "import urllib.request; urllib.request.urlopen('http://localhost:8000/healthz')"] + interval: 30s + timeout: 5s + retries: 5 + start_period: 20s + + worker: + image: transcription:prod + env_file: + - .env.production + command: ["python", "-m", "transcription.worker_service"] + depends_on: + postgres: + condition: service_healthy + volumes: + - app_uploads:/app/uploads + - app_data:/app/data + - ./prompts:/app/prompts:ro + restart: unless-stopped + + postgres: + image: postgres:16-alpine + env_file: + - .env.production + environment: + POSTGRES_DB: ${POSTGRES_DB} + POSTGRES_USER: ${POSTGRES_USER} + POSTGRES_PASSWORD: ${POSTGRES_PASSWORD} + volumes: + - postgres_data:/var/lib/postgresql/data + restart: unless-stopped + healthcheck: + test: ["CMD-SHELL", "pg_isready -U $$POSTGRES_USER -d $$POSTGRES_DB"] + interval: 10s + timeout: 5s + retries: 10 + start_period: 10s + + cloudflared: + image: cloudflare/cloudflared:2026.8.0 + env_file: + - .env.production + command: tunnel --no-autoupdate run --token ${CLOUDFLARE_TUNNEL_TOKEN} + depends_on: + app: + condition: service_healthy + restart: unless-stopped + +volumes: + postgres_data: + app_uploads: + app_data: diff --git a/docs/architecture.md b/docs/architecture.md index a049796..cf18584 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -30,6 +30,9 @@ W --> P[Provider Adapter] W --> DB ``` +For production split-process deployments, the worker may run as a dedicated service +while the app process runs with `RUN_EMBEDDED_WORKER=false`. + ## Layered Boundaries ### Interface Layer diff --git a/docs/production-runbook.md b/docs/production-runbook.md index 61f129d..19b05e9 100644 --- a/docs/production-runbook.md +++ b/docs/production-runbook.md @@ -19,9 +19,11 @@ This runbook is the operational checklist for releasing and monitoring the trans ## 2. Release execution steps 1. Deploy artifact/config to target environment. + - V6.0 Phase 1 production stack: `docker compose -f docker-compose.production.yml up -d --build` 2. Validate service startup: - `/healthz` responds `200` - - `worker.state` is `running` + - if `RUN_EMBEDDED_WORKER=true`, `worker.state` is `running` + - if `RUN_EMBEDDED_WORKER=false`, validate `worker` container is healthy/running in Compose 3. Execute one smoke workflow: - create a document/job with at least one source - verify terminal job outcome updates diff --git a/docs/roadmap_plan.md b/docs/roadmap_plan.md index 27ac909..0ed5523 100644 --- a/docs/roadmap_plan.md +++ b/docs/roadmap_plan.md @@ -6,6 +6,8 @@ This roadmap starts at **V6.0** and tracks forward-looking work only. Objective: move from local-only operation to secure, stable remote hosting. +Detailed plan: [`v6_0_hosting_migration_plan.md`](v6_0_hosting_migration_plan.md) + ### Scope 1. Containerize app runtime for production deployment. 2. Run PostgreSQL in Docker and migrate from SQLite. diff --git a/docs/v6_0_hosting_migration_plan.md b/docs/v6_0_hosting_migration_plan.md new file mode 100644 index 0000000..bfef8bc --- /dev/null +++ b/docs/v6_0_hosting_migration_plan.md @@ -0,0 +1,77 @@ +# V6.0 Hosting Migration Formal Plan + +This plan formalizes the V6.0 roadmap objective: move from local-only operation to secure, stable remote hosting. + +## 1. Confirmed Infrastructure Decisions + +- **Runtime host:** dedicated Debian/Ubuntu VM on Proxmox 8.4.x. +- **Remote ingress:** Cloudflare Tunnel + Cloudflare Access with public hostnames per internal service. +- **Backup target:** Synology DS420j for PostgreSQL dumps and restore points. +- **Out of scope for V6.0:** Synology as primary live upload/image storage. + +## 2. Target Runtime Topology + +The production stack is deployed with Docker Compose on the Proxmox VM: + +1. `app` service (FastAPI + NiceGUI runtime) +2. `worker` service (async transcription worker) +3. `postgres` service (primary datastore) +4. `cloudflared` service (tunnel ingress to app endpoints) + +Persistence: + +- PostgreSQL data: durable local volume on Proxmox VM. +- App data/log paths: durable local volume(s) on Proxmox VM. +- Backups: scheduled PostgreSQL dump artifacts replicated to Synology DS420j. + +## 3. V6.0 Workstreams + +## 3.1 Deployment and configuration + +1. Produce production-ready Docker/Compose definitions for `app`, `worker`, `postgres`, and `cloudflared`. +2. Move runtime settings to environment-based configuration (DB, uploads, prompts, logging, secrets). +3. Add operational defaults (health checks, restart policies, predictable service dependencies). + +## 3.2 Database migration (SQLite -> PostgreSQL) + +1. Define a deterministic migration method from SQLite to PostgreSQL. +2. Run migration in staging-like environment and validate entity counts and key relationships. +3. Execute cutover with rollback guardrails and preserved evidence/provenance history. + +## 3.3 Cloudflare remote access + +1. Configure tunnel routing for service hostnames. +2. Apply Cloudflare Access policies for identity-gated remote access. +3. Keep non-required administrative/internal surfaces LAN-only unless explicitly approved. + +## 3.4 Backup, restore, rollback + +1. Define backup schedule, retention, and artifact naming/versioning. +2. Validate restore drill from Synology-hosted dump artifacts. +3. Document rollback procedure for deployment failure and migration failure scenarios. + +## 3.5 Validation and release gate + +1. `/healthz` confirms app and worker healthy in deployed environment. +2. One end-to-end document -> source -> job workflow succeeds through remote access. +3. Backup/restore drill completes and data integrity checks pass. + +## 4. Deliverables + +- Production-ready `docker-compose` deployment for app + worker + PostgreSQL + cloudflared. +- Environment configuration model suitable for production secrets and runtime overrides. +- Repeatable SQLite-to-PostgreSQL migration procedure with cutover checklist. +- Updated runbook content for deploy, rollback, backup, and restore. + +## 5. Exit Criteria (V6.0 Complete) + +- Health and worker liveness are green in deployed runtime. +- Remote transcription workflow is successful and stable. +- Backup and restore are tested and documented. +- Evidence/provenance guarantees remain intact (append-only attempt history and traceability preserved). + +## 6. Open Decisions to Finalize During Implementation + +- Hostname inventory and naming convention for each remotely exposed internal service. +- Cloudflare Access policy granularity (per-service policy shape and identity groups). +- Backup retention windows and RPO/RTO targets aligned with available Synology capacity. diff --git a/src/transcription/app.py b/src/transcription/app.py index 4e9d264..461bcbf 100644 --- a/src/transcription/app.py +++ b/src/transcription/app.py @@ -56,18 +56,19 @@ async def _lifespan(app: FastAPI): async with AsyncExitStack() as stack: stack.push_async_callback(dispose_database_runtime) - stop_event, worker_notifier, worker_health = await stack.enter_async_context( - worker_consumer_lifespan( - session_factory=app.state.runtime.session_factory, - poll_interval_seconds=settings.worker_poll_interval_seconds, - shutdown_timeout_seconds=( - settings.worker_provider_timeout_seconds + settings.worker_shutdown_grace_seconds - ), + if settings.run_embedded_worker: + stop_event, worker_notifier, worker_health = await stack.enter_async_context( + worker_consumer_lifespan( + session_factory=app.state.runtime.session_factory, + poll_interval_seconds=settings.worker_poll_interval_seconds, + shutdown_timeout_seconds=( + settings.worker_provider_timeout_seconds + settings.worker_shutdown_grace_seconds + ), + ) ) - ) - app.state.worker_stop_event = stop_event - app.state.worker_notifier = worker_notifier - app.state.worker_health = worker_health + app.state.worker_stop_event = stop_event + app.state.worker_notifier = worker_notifier + app.state.worker_health = worker_health yield diff --git a/src/transcription/config.py b/src/transcription/config.py index bdac70a..433165b 100644 --- a/src/transcription/config.py +++ b/src/transcription/config.py @@ -98,6 +98,7 @@ class Settings(BaseSettings): # --- runtime environment --- environment: Literal["development", "test", "production"] = "development" transcription_commit: NonEmptyStr | None = None + run_embedded_worker: bool = True # --- persistence --- database: DatabaseSettings = Field(default_factory=SqliteSettings) diff --git a/src/transcription/worker_service.py b/src/transcription/worker_service.py new file mode 100644 index 0000000..3b5259f --- /dev/null +++ b/src/transcription/worker_service.py @@ -0,0 +1,43 @@ +"""Standalone worker process entrypoint for production deployments.""" + +from __future__ import annotations + +import asyncio +import logging + +from .config import configure_logging +from .config import parse_cli_settings +from .db import create_all +from .db import dispose_database_runtime +from .db import initialize_database_runtime +from .worker import run_worker_loop + +logger = logging.getLogger(__name__) + + +async def _run() -> None: + settings = parse_cli_settings() + configure_logging(settings) + runtime = initialize_database_runtime(settings=settings) + + if settings.should_bootstrap_schema: + await create_all(engine=runtime.engine) + + try: + await run_worker_loop( + session_factory=runtime.session_factory, + poll_interval_seconds=settings.worker_poll_interval_seconds, + ) + finally: + await dispose_database_runtime() + + +def main() -> None: + try: + asyncio.run(_run()) + except KeyboardInterrupt: + logger.info("Worker service received shutdown signal") + + +if __name__ == "__main__": + main() diff --git a/tests/test_app.py b/tests/test_app.py index 4c1fe81..94f74c6 100644 --- a/tests/test_app.py +++ b/tests/test_app.py @@ -136,3 +136,52 @@ class TestAppLifespan: pass assert calls == ["logging", "schema", "recover", "worker_start", "worker_stop", "dispose_db"] + + def test_startup_skips_embedded_worker_when_disabled(self, monkeypatch, tmp_path): + calls = [] + + monkeypatch.setattr("transcription.app.configure_logging", lambda _settings: calls.append("logging")) + monkeypatch.setattr("transcription.app.register_pages", lambda _app: None) + + async def _create_all(**_kwargs): + calls.append("schema") + + monkeypatch.setattr("transcription.app.create_all", _create_all) + monkeypatch.setattr( + "transcription.app.initialize_database_runtime", + lambda **_kwargs: type("_Runtime", (), {"engine": object(), "session_factory": object()})(), + ) + + async def _dispose_runtime(): + calls.append("dispose_db") + + monkeypatch.setattr("transcription.app.dispose_database_runtime", _dispose_runtime) + + async def _recover_stale(_app): + calls.append("recover") + + monkeypatch.setattr("transcription.app._recover_stale_processing_jobs", _recover_stale) + + @asynccontextmanager + async def _worker_lifespan(**_kwargs): + calls.append("worker_start") + yield object(), object(), object() + calls.append("worker_stop") + + monkeypatch.setattr("transcription.app.worker_consumer_lifespan", _worker_lifespan) + + settings = Settings( + openrouter_api_key="test-key", + environment="test", + bootstrap_schema_on_startup=True, + run_embedded_worker=False, + upload_dir=tmp_path / "uploads", + prompt_dir=tmp_path / "prompts", + ) + monkeypatch.setattr("transcription.app.get_settings", lambda: settings) + + app = create_app() + with TestClient(app): + pass + + assert calls == ["logging", "schema", "recover", "dispose_db"] diff --git a/tests/test_config.py b/tests/test_config.py index e9aa35b..f299af0 100644 --- a/tests/test_config.py +++ b/tests/test_config.py @@ -166,6 +166,7 @@ class TestWorkerReliabilitySettings: def test_worker_retry_defaults(self): """worker retry settings default to no retries.""" settings = _make_settings() + assert settings.run_embedded_worker is True assert settings.worker_max_retries == 0 assert settings.worker_stale_job_seconds == 30.0 assert settings.worker_retry_backoff_seconds == 1.0 diff --git a/tests/test_worker_service.py b/tests/test_worker_service.py new file mode 100644 index 0000000..0b20a78 --- /dev/null +++ b/tests/test_worker_service.py @@ -0,0 +1,34 @@ +"""Tests for the standalone worker service entrypoint.""" + +import pytest + +from transcription import worker_service + + +@pytest.mark.unit +def test_main_runs_async_worker_once(monkeypatch): + calls: list[str] = [] + + async def _fake_run() -> None: + calls.append("run") + + monkeypatch.setattr(worker_service, "_run", _fake_run) + worker_service.main() + + assert calls == ["run"] + + +@pytest.mark.unit +def test_main_handles_keyboard_interrupt(monkeypatch): + calls: list[str] = [] + + def _raise_keyboard_interrupt(coro): + coro.close() + raise KeyboardInterrupt + + monkeypatch.setattr(worker_service.asyncio, "run", _raise_keyboard_interrupt) + monkeypatch.setattr(worker_service.logger, "info", lambda _msg: calls.append("logged")) + + worker_service.main() + + assert calls == ["logged"]