V6 Phase 1 complete

.env.example

@@ -34,6 +34,8 @@ DEFAULT_PROMPT_NAME=transcribe_document.md
 ENVIRONMENT=development
 # TRANSCRIPTION_COMMIT default: unset (optional build/commit identifier for provenance evidence)
 # TRANSCRIPTION_COMMIT=
+# RUN_EMBEDDED_WORKER default: true (set false when running a standalone worker process)
+RUN_EMBEDDED_WORKER=true
 
 # --- persistence ---
 # Use nested keys (env_nested_delimiter="__").

.env.production.example

@@ -0,0 +1,53 @@
+# Production environment example for docker-compose.production.yml
+
+# --- NiceGUI Server ---
+HOST=0.0.0.0
+PORT=8000
+LOG_LEVEL=info
+RELOAD=false
+ENVIRONMENT=production
+RUN_EMBEDDED_WORKER=false
+LOG_DIR=/app/data/logs
+LOG_FILE_NAME=transcription.log
+LOG_FILE_MAX_BYTES=10485760
+LOG_FILE_BACKUP_COUNT=5
+
+# --- AI provider ---
+PROVIDER=openrouter
+OPENROUTER_API_KEY=replace-with-real-key
+PROVIDER_MODEL=google/gemini-2.5-flash
+DEFAULT_PROMPT_NAME=transcribe_document.md
+
+# --- persistence ---
+DATABASE__DRIVER=postgres
+DATABASE__HOST=postgres
+DATABASE__PORT=5432
+DATABASE__DATABASE=transcription
+DATABASE__USER=transcription
+DATABASE__PASSWORD=replace-with-strong-password
+BOOTSTRAP_SCHEMA_ON_STARTUP=false
+SQLITE_CHECK_SAME_THREAD=false
+
+# --- filesystem paths ---
+UPLOAD_DIR=/app/uploads
+PROMPT_DIR=/app/prompts
+DATABASE_BACKUP_DIR=/app/data/backups
+
+# --- worker reliability ---
+WORKER_MAX_RETRIES=0
+WORKER_PROVIDER_TIMEOUT_SECONDS=30.0
+WORKER_STALE_JOB_SECONDS=30.0
+WORKER_RETRY_BACKOFF_SECONDS=1.0
+WORKER_SHUTDOWN_GRACE_SECONDS=5.0
+WORKER_POLL_INTERVAL_SECONDS=1.0
+WORKER_MIN_TRANSCRIPTION_CHARS=0
+WORKER_MIN_TRANSCRIPTION_LINES=0
+WORKER_FAIL_ON_FINISH_REASON_LENGTH=false
+
+# --- postgres container ---
+POSTGRES_DB=transcription
+POSTGRES_USER=transcription
+POSTGRES_PASSWORD=replace-with-strong-password
+
+# --- cloudflare tunnel ---
+CLOUDFLARE_TUNNEL_TOKEN=replace-with-cloudflare-tunnel-token

README.md

@@ -54,6 +54,7 @@ Practical examples:
 | `LOG_LEVEL` | `info` | Uvicorn and application log level. |
 | `RELOAD` | `false` | Restart the development server when source files change. |
 | `ENVIRONMENT` | `development` | Runtime environment: `development`, `test`, or `production`. |
+| `RUN_EMBEDDED_WORKER` | `true` | Run worker loop inside web app process. Set `false` when using a dedicated worker service. |
 
 #### Provider
 
@@ -122,6 +123,22 @@ This starts the development server with SQLite, creates missing tables, and enab
 
 Replace `localhost` with the server's hostname or IP address when connecting from another machine.
 
+## Production stack (Phase 1)
+
+Use the production compose profile for split app/worker deployment with PostgreSQL and Cloudflare Tunnel:
+
+```bash
+copy .env.production.example .env.production
+docker compose -f docker-compose.production.yml up -d --build
+```
+
+Services:
+
+- `app`: FastAPI + NiceGUI runtime (`RUN_EMBEDDED_WORKER=false`)
+- `worker`: standalone queue processor (`python -m transcription.worker_service`)
+- `postgres`: primary datastore
+- `cloudflared`: tunnel client using `CLOUDFLARE_TUNNEL_TOKEN`
+
 ## How to navigate the GUI
 
 - **Upload page** (`/ui`)

docker-compose.production.yml

@@ -0,0 +1,71 @@
+services:
+  app:
+    build:
+      context: .
+      dockerfile: Dockerfile
+    image: transcription:prod
+    env_file:
+      - .env.production
+    environment:
+      RUN_EMBEDDED_WORKER: "false"
+    depends_on:
+      postgres:
+        condition: service_healthy
+    volumes:
+      - app_uploads:/app/uploads
+      - app_data:/app/data
+      - ./prompts:/app/prompts:ro
+    restart: unless-stopped
+    healthcheck:
+      test: ["CMD", "python", "-c", "import urllib.request; urllib.request.urlopen('http://localhost:8000/healthz')"]
+      interval: 30s
+      timeout: 5s
+      retries: 5
+      start_period: 20s
+
+  worker:
+    image: transcription:prod
+    env_file:
+      - .env.production
+    command: ["python", "-m", "transcription.worker_service"]
+    depends_on:
+      postgres:
+        condition: service_healthy
+    volumes:
+      - app_uploads:/app/uploads
+      - app_data:/app/data
+      - ./prompts:/app/prompts:ro
+    restart: unless-stopped
+
+  postgres:
+    image: postgres:16-alpine
+    env_file:
+      - .env.production
+    environment:
+      POSTGRES_DB: ${POSTGRES_DB}
+      POSTGRES_USER: ${POSTGRES_USER}
+      POSTGRES_PASSWORD: ${POSTGRES_PASSWORD}
+    volumes:
+      - postgres_data:/var/lib/postgresql/data
+    restart: unless-stopped
+    healthcheck:
+      test: ["CMD-SHELL", "pg_isready -U $$POSTGRES_USER -d $$POSTGRES_DB"]
+      interval: 10s
+      timeout: 5s
+      retries: 10
+      start_period: 10s
+
+  cloudflared:
+    image: cloudflare/cloudflared:2026.8.0
+    env_file:
+      - .env.production
+    command: tunnel --no-autoupdate run --token ${CLOUDFLARE_TUNNEL_TOKEN}
+    depends_on:
+      app:
+        condition: service_healthy
+    restart: unless-stopped
+
+volumes:
+  postgres_data:
+  app_uploads:
+  app_data:

docs/architecture.md

@@ -30,6 +30,9 @@ W --> P[Provider Adapter]
 W --> DB
 ```
 
+For production split-process deployments, the worker may run as a dedicated service
+while the app process runs with `RUN_EMBEDDED_WORKER=false`.
+
 ## Layered Boundaries
 
 ### Interface Layer

docs/production-runbook.md

@@ -19,9 +19,11 @@ This runbook is the operational checklist for releasing and monitoring the trans
 ## 2. Release execution steps
 
 1. Deploy artifact/config to target environment.
+   - V6.0 Phase 1 production stack: `docker compose -f docker-compose.production.yml up -d --build`
 2. Validate service startup:
    - `/healthz` responds `200`
-   - `worker.state` is `running`
+   - if `RUN_EMBEDDED_WORKER=true`, `worker.state` is `running`
+   - if `RUN_EMBEDDED_WORKER=false`, validate `worker` container is healthy/running in Compose
 3. Execute one smoke workflow:
    - create a document/job with at least one source
    - verify terminal job outcome updates

docs/roadmap_plan.md

@@ -6,6 +6,8 @@ This roadmap starts at **V6.0** and tracks forward-looking work only.
 
 Objective: move from local-only operation to secure, stable remote hosting.
 
+Detailed plan: [`v6_0_hosting_migration_plan.md`](v6_0_hosting_migration_plan.md)
+
 ### Scope
 1. Containerize app runtime for production deployment.
 2. Run PostgreSQL in Docker and migrate from SQLite.

docs/v6_0_hosting_migration_plan.md

@@ -0,0 +1,77 @@
+# V6.0 Hosting Migration Formal Plan
+
+This plan formalizes the V6.0 roadmap objective: move from local-only operation to secure, stable remote hosting.
+
+## 1. Confirmed Infrastructure Decisions
+
+- **Runtime host:** dedicated Debian/Ubuntu VM on Proxmox 8.4.x.
+- **Remote ingress:** Cloudflare Tunnel + Cloudflare Access with public hostnames per internal service.
+- **Backup target:** Synology DS420j for PostgreSQL dumps and restore points.
+- **Out of scope for V6.0:** Synology as primary live upload/image storage.
+
+## 2. Target Runtime Topology
+
+The production stack is deployed with Docker Compose on the Proxmox VM:
+
+1. `app` service (FastAPI + NiceGUI runtime)
+2. `worker` service (async transcription worker)
+3. `postgres` service (primary datastore)
+4. `cloudflared` service (tunnel ingress to app endpoints)
+
+Persistence:
+
+- PostgreSQL data: durable local volume on Proxmox VM.
+- App data/log paths: durable local volume(s) on Proxmox VM.
+- Backups: scheduled PostgreSQL dump artifacts replicated to Synology DS420j.
+
+## 3. V6.0 Workstreams
+
+## 3.1 Deployment and configuration
+
+1. Produce production-ready Docker/Compose definitions for `app`, `worker`, `postgres`, and `cloudflared`.
+2. Move runtime settings to environment-based configuration (DB, uploads, prompts, logging, secrets).
+3. Add operational defaults (health checks, restart policies, predictable service dependencies).
+
+## 3.2 Database migration (SQLite -> PostgreSQL)
+
+1. Define a deterministic migration method from SQLite to PostgreSQL.
+2. Run migration in staging-like environment and validate entity counts and key relationships.
+3. Execute cutover with rollback guardrails and preserved evidence/provenance history.
+
+## 3.3 Cloudflare remote access
+
+1. Configure tunnel routing for service hostnames.
+2. Apply Cloudflare Access policies for identity-gated remote access.
+3. Keep non-required administrative/internal surfaces LAN-only unless explicitly approved.
+
+## 3.4 Backup, restore, rollback
+
+1. Define backup schedule, retention, and artifact naming/versioning.
+2. Validate restore drill from Synology-hosted dump artifacts.
+3. Document rollback procedure for deployment failure and migration failure scenarios.
+
+## 3.5 Validation and release gate
+
+1. `/healthz` confirms app and worker healthy in deployed environment.
+2. One end-to-end document -> source -> job workflow succeeds through remote access.
+3. Backup/restore drill completes and data integrity checks pass.
+
+## 4. Deliverables
+
+- Production-ready `docker-compose` deployment for app + worker + PostgreSQL + cloudflared.
+- Environment configuration model suitable for production secrets and runtime overrides.
+- Repeatable SQLite-to-PostgreSQL migration procedure with cutover checklist.
+- Updated runbook content for deploy, rollback, backup, and restore.
+
+## 5. Exit Criteria (V6.0 Complete)
+
+- Health and worker liveness are green in deployed runtime.
+- Remote transcription workflow is successful and stable.
+- Backup and restore are tested and documented.
+- Evidence/provenance guarantees remain intact (append-only attempt history and traceability preserved).
+
+## 6. Open Decisions to Finalize During Implementation
+
+- Hostname inventory and naming convention for each remotely exposed internal service.
+- Cloudflare Access policy granularity (per-service policy shape and identity groups).
+- Backup retention windows and RPO/RTO targets aligned with available Synology capacity.

src/transcription/app.py

@@ -56,6 +56,7 @@ async def _lifespan(app: FastAPI):
 
     async with AsyncExitStack() as stack:
         stack.push_async_callback(dispose_database_runtime)
+        if settings.run_embedded_worker:
             stop_event, worker_notifier, worker_health = await stack.enter_async_context(
                 worker_consumer_lifespan(
                     session_factory=app.state.runtime.session_factory,

src/transcription/config.py

@@ -98,6 +98,7 @@ class Settings(BaseSettings):
     # --- runtime environment ---
     environment: Literal["development", "test", "production"] = "development"
     transcription_commit: NonEmptyStr | None = None
+    run_embedded_worker: bool = True
 
     # --- persistence ---
     database: DatabaseSettings = Field(default_factory=SqliteSettings)

src/transcription/worker_service.py

@@ -0,0 +1,43 @@
+"""Standalone worker process entrypoint for production deployments."""
+
+from __future__ import annotations
+
+import asyncio
+import logging
+
+from .config import configure_logging
+from .config import parse_cli_settings
+from .db import create_all
+from .db import dispose_database_runtime
+from .db import initialize_database_runtime
+from .worker import run_worker_loop
+
+logger = logging.getLogger(__name__)
+
+
+async def _run() -> None:
+    settings = parse_cli_settings()
+    configure_logging(settings)
+    runtime = initialize_database_runtime(settings=settings)
+
+    if settings.should_bootstrap_schema:
+        await create_all(engine=runtime.engine)
+
+    try:
+        await run_worker_loop(
+            session_factory=runtime.session_factory,
+            poll_interval_seconds=settings.worker_poll_interval_seconds,
+        )
+    finally:
+        await dispose_database_runtime()
+
+
+def main() -> None:
+    try:
+        asyncio.run(_run())
+    except KeyboardInterrupt:
+        logger.info("Worker service received shutdown signal")
+
+
+if __name__ == "__main__":
+    main()

tests/test_app.py

@@ -136,3 +136,52 @@ class TestAppLifespan:
             pass
 
         assert calls == ["logging", "schema", "recover", "worker_start", "worker_stop", "dispose_db"]
+
+    def test_startup_skips_embedded_worker_when_disabled(self, monkeypatch, tmp_path):
+        calls = []
+
+        monkeypatch.setattr("transcription.app.configure_logging", lambda _settings: calls.append("logging"))
+        monkeypatch.setattr("transcription.app.register_pages", lambda _app: None)
+
+        async def _create_all(**_kwargs):
+            calls.append("schema")
+
+        monkeypatch.setattr("transcription.app.create_all", _create_all)
+        monkeypatch.setattr(
+            "transcription.app.initialize_database_runtime",
+            lambda **_kwargs: type("_Runtime", (), {"engine": object(), "session_factory": object()})(),
+        )
+
+        async def _dispose_runtime():
+            calls.append("dispose_db")
+
+        monkeypatch.setattr("transcription.app.dispose_database_runtime", _dispose_runtime)
+
+        async def _recover_stale(_app):
+            calls.append("recover")
+
+        monkeypatch.setattr("transcription.app._recover_stale_processing_jobs", _recover_stale)
+
+        @asynccontextmanager
+        async def _worker_lifespan(**_kwargs):
+            calls.append("worker_start")
+            yield object(), object(), object()
+            calls.append("worker_stop")
+
+        monkeypatch.setattr("transcription.app.worker_consumer_lifespan", _worker_lifespan)
+
+        settings = Settings(
+            openrouter_api_key="test-key",
+            environment="test",
+            bootstrap_schema_on_startup=True,
+            run_embedded_worker=False,
+            upload_dir=tmp_path / "uploads",
+            prompt_dir=tmp_path / "prompts",
+        )
+        monkeypatch.setattr("transcription.app.get_settings", lambda: settings)
+
+        app = create_app()
+        with TestClient(app):
+            pass
+
+        assert calls == ["logging", "schema", "recover", "dispose_db"]

tests/test_config.py

@@ -166,6 +166,7 @@ class TestWorkerReliabilitySettings:
     def test_worker_retry_defaults(self):
         """worker retry settings default to no retries."""
         settings = _make_settings()
+        assert settings.run_embedded_worker is True
         assert settings.worker_max_retries == 0
         assert settings.worker_stale_job_seconds == 30.0
         assert settings.worker_retry_backoff_seconds == 1.0

tests/test_worker_service.py

@@ -0,0 +1,34 @@
+"""Tests for the standalone worker service entrypoint."""
+
+import pytest
+
+from transcription import worker_service
+
+
+@pytest.mark.unit
+def test_main_runs_async_worker_once(monkeypatch):
+    calls: list[str] = []
+
+    async def _fake_run() -> None:
+        calls.append("run")
+
+    monkeypatch.setattr(worker_service, "_run", _fake_run)
+    worker_service.main()
+
+    assert calls == ["run"]
+
+
+@pytest.mark.unit
+def test_main_handles_keyboard_interrupt(monkeypatch):
+    calls: list[str] = []
+
+    def _raise_keyboard_interrupt(coro):
+        coro.close()
+        raise KeyboardInterrupt
+
+    monkeypatch.setattr(worker_service.asyncio, "run", _raise_keyboard_interrupt)
+    monkeypatch.setattr(worker_service.logger, "info", lambda _msg: calls.append("logged"))
+
+    worker_service.main()
+
+    assert calls == ["logged"]