From 234476ba6d6a18e3e2f4e66ba028a2d3b80a2bff Mon Sep 17 00:00:00 2001 From: Jim Lancaster <40281233+zoltan57@users.noreply.github.com> Date: Tue, 25 Aug 2026 13:05:47 -0500 Subject: [PATCH] V6.0 Phase 3 - Add cloudflare tunnel. Remote access to app via the tunnel now working. --- .env.production.example | 1 + .gitignore | 3 ++ README.md | 8 +++- deploy/cloudflared/config.yml.example | 14 +++++++ docker-compose.production.yml | 6 +-- docs/cloudflare_tunnel_access.md | 60 +++++++++++++++++++++++++++ docs/production-runbook.md | 8 ++++ docs/v6_0_hosting_migration_plan.md | 5 +++ src/transcription/services/jobs.py | 25 ++++++++--- tests/services/test_job_service.py | 15 +++++++ 10 files changed, 136 insertions(+), 9 deletions(-) create mode 100644 deploy/cloudflared/config.yml.example create mode 100644 docs/cloudflare_tunnel_access.md diff --git a/.env.production.example b/.env.production.example index c6a023b..92f2c32 100644 --- a/.env.production.example +++ b/.env.production.example @@ -50,4 +50,5 @@ POSTGRES_USER=transcription POSTGRES_PASSWORD=replace-with-strong-password # --- cloudflare tunnel --- +# Required for token-based tunnel startup. CLOUDFLARE_TUNNEL_TOKEN=replace-with-cloudflare-tunnel-token diff --git a/.gitignore b/.gitignore index 875ec08..f06a874 100644 --- a/.gitignore +++ b/.gitignore @@ -27,3 +27,6 @@ data/photos/* data-migration-test/* .migration-bundle/* +# Cloudflare tunnel local runtime files +deploy/cloudflared/config.yml +deploy/cloudflared/credentials.json diff --git a/README.md b/README.md index 0b69a38..dc7fa52 100644 --- a/README.md +++ b/README.md @@ -137,7 +137,13 @@ Services: - `app`: FastAPI + NiceGUI runtime (`RUN_EMBEDDED_WORKER=false`) - `worker`: standalone queue processor (`python -m transcription.worker_service`) - `postgres`: primary datastore -- `cloudflared`: tunnel client using `CLOUDFLARE_TUNNEL_TOKEN` +- `cloudflared`: tunnel client using mounted ingress config + `CLOUDFLARE_TUNNEL_TOKEN` + +Cloudflare setup files: + +1. `copy deploy\cloudflared\config.yml.example deploy\cloudflared\config.yml` +2. set `CLOUDFLARE_TUNNEL_TOKEN` in `.env.production` +3. update ingress hostnames in `deploy\cloudflared\config.yml` ## How to navigate the GUI diff --git a/deploy/cloudflared/config.yml.example b/deploy/cloudflared/config.yml.example new file mode 100644 index 0000000..753f619 --- /dev/null +++ b/deploy/cloudflared/config.yml.example @@ -0,0 +1,14 @@ +ingress: + # Primary transcription app endpoint. + - hostname: transcription.example.com + service: http://app:8000 + + # Optional: generic remote access endpoints for other internal services. + # Replace hostnames and targets for your LAN. + - hostname: homeassistant.example.com + service: http://192.168.1.50:8123 + - hostname: pihole.example.com + service: http://192.168.1.60:80 + + # Required catch-all. + - service: http_status:404 diff --git a/docker-compose.production.yml b/docker-compose.production.yml index a8136a7..35c836a 100644 --- a/docker-compose.production.yml +++ b/docker-compose.production.yml @@ -54,17 +54,17 @@ services: timeout: 5s retries: 10 start_period: 10s - ports: - - "5432:5432" cloudflared: image: cloudflare/cloudflared:2026.8.0 env_file: - .env.production - command: tunnel --no-autoupdate run --token ${CLOUDFLARE_TUNNEL_TOKEN} + command: tunnel --no-autoupdate --config /etc/cloudflared/config.yml run --token ${CLOUDFLARE_TUNNEL_TOKEN} depends_on: app: condition: service_healthy + volumes: + - ./deploy/cloudflared/config.yml:/etc/cloudflared/config.yml:ro restart: unless-stopped volumes: diff --git a/docs/cloudflare_tunnel_access.md b/docs/cloudflare_tunnel_access.md new file mode 100644 index 0000000..57a8bea --- /dev/null +++ b/docs/cloudflare_tunnel_access.md @@ -0,0 +1,60 @@ +# Cloudflare Tunnel and Access Setup (V6.0 Phase 3) + +This guide defines the repository-supported setup for exposing app and selected LAN services through Cloudflare Tunnel with Cloudflare Access protection. + +## 1. Files used by this deployment + +1. `deploy/cloudflared/config.yml` (local copy from `config.yml.example`) +2. `.env.production` (`CLOUDFLARE_TUNNEL_TOKEN`) +3. `docker-compose.production.yml` (`cloudflared` service reads token + mounts config) + +Do not commit `config.yml` or `.env.production`. + +## 2. Configure cloudflared + +1. Copy `deploy/cloudflared/config.yml.example` to `deploy/cloudflared/config.yml`. +2. Update hostname -> service mappings in `ingress`. +3. Keep the final catch-all ingress `http_status:404`. +4. Set `CLOUDFLARE_TUNNEL_TOKEN` in `.env.production`. + +Example app route: + +- `transcription.example.com` -> `http://app:8000` + +Optional generic remote-access routes: + +- `homeassistant.example.com` -> `http://:8123` +- `pihole.example.com` -> `http://:80` + +## 3. Cloudflare Access policy baseline + +Create one Access app policy per exposed hostname: + +1. Include: your allowed identities/groups only. +2. Exclude: none by default. +3. Require: identity provider login (and MFA if available). + +Recommended baseline: + +- App endpoint (`transcription.*`): your admin identity set. +- Other internal endpoints (`homeassistant.*`, `pihole.*`, etc.): explicit least-privilege groups. + +## 4. Startup + +Start production stack: + +```bash +docker compose --env-file .env.production -f docker-compose.production.yml up -d --build +``` + +Validate tunnel container: + +```bash +docker compose --env-file .env.production -f docker-compose.production.yml logs cloudflared +``` + +## 5. Security notes + +- Keep `postgres` and other internal-only services off public hostnames unless required. +- Use distinct hostnames per service; avoid path-based multiplexing for unrelated admin surfaces. +- Rotate `CLOUDFLARE_TUNNEL_TOKEN` and Access policy memberships on a regular schedule. diff --git a/docs/production-runbook.md b/docs/production-runbook.md index 86d3028..4073e5b 100644 --- a/docs/production-runbook.md +++ b/docs/production-runbook.md @@ -25,6 +25,7 @@ This runbook is the operational checklist for releasing and monitoring the trans - `/healthz` responds `200` - if `RUN_EMBEDDED_WORKER=true`, `worker.state` is `running` - if `RUN_EMBEDDED_WORKER=false`, validate `worker` container is healthy/running in Compose + - validate `cloudflared` logs show active tunnel routes and no ingress errors 3. Execute one smoke workflow: - create a document/job with at least one source - verify terminal job outcome updates @@ -86,6 +87,13 @@ This runbook is the operational checklist for releasing and monitoring the trans 2. Confirm failures are category-aligned (`external`/`timeout`/`internal`). 3. Triage whether issue is source quality, provider, or runtime regression. +### Cloudflare ingress/access failure + +1. Check `cloudflared` container logs for ingress parse, DNS, or auth failures. +2. Confirm `deploy/cloudflared/config.yml` tunnel UUID and hostname mappings are correct. +3. Confirm `deploy/cloudflared/credentials.json` matches the tunnel configured in Cloudflare. +4. Confirm Cloudflare Access app policy includes the intended identity/group for that hostname. + ## 6. Dependency upgrade policy Dependencies are declared in `pyproject.toml` and resolved through the committed diff --git a/docs/v6_0_hosting_migration_plan.md b/docs/v6_0_hosting_migration_plan.md index 3c695b0..c98e664 100644 --- a/docs/v6_0_hosting_migration_plan.md +++ b/docs/v6_0_hosting_migration_plan.md @@ -49,6 +49,11 @@ Implemented workflow references: 2. Apply Cloudflare Access policies for identity-gated remote access. 3. Keep non-required administrative/internal surfaces LAN-only unless explicitly approved. +Implemented workflow references: + +- `deploy/cloudflared/config.yml.example` +- `docs/cloudflare_tunnel_access.md` + ## 3.4 Backup, restore, rollback 1. Define backup schedule, retention, and artifact naming/versioning. diff --git a/src/transcription/services/jobs.py b/src/transcription/services/jobs.py index ca6079d..822a236 100644 --- a/src/transcription/services/jobs.py +++ b/src/transcription/services/jobs.py @@ -26,6 +26,18 @@ from .base import ServiceBase logger = logging.getLogger(__name__) +def _utc_now_naive() -> datetime: + """Return current UTC as naive datetime for DB timestamp columns.""" + return datetime.now(UTC).replace(tzinfo=None) + + +def _as_naive_utc(value: datetime) -> datetime: + """Normalize datetimes to naive UTC for DB comparisons/binds.""" + if value.tzinfo is None: + return value + return value.astimezone(UTC).replace(tzinfo=None) + + class JobDeleteBlockedError(AppError): """Raised when a job delete operation is blocked by lifecycle policy.""" @@ -201,7 +213,7 @@ class JobService(ServiceBase): await self._finalize(session=_session, caller_session=session, refresh=(job,)) return job - now = datetime.now(UTC) + now = _utc_now_naive() queued_job_id = ( select(col(Job.id)) .where(col(Job.status) == JobStatus.QUEUED) @@ -239,12 +251,15 @@ class JobService(ServiceBase): ``stale_before`` are considered stale and re-queued. """ async with self._session_scope(session) as _session: - query = select(Job).where(Job.status == JobStatus.PROCESSING).where(Job.date_updated < stale_before) + normalized_stale_before = _as_naive_utc(stale_before) + query = ( + select(Job).where(Job.status == JobStatus.PROCESSING).where(Job.date_updated < normalized_stale_before) + ) stale_jobs = (await _session.exec(query)).all() if not stale_jobs: return 0 - now = datetime.now(UTC) + now = _utc_now_naive() for job in stale_jobs: job.status = JobStatus.QUEUED job.date_updated = now @@ -352,7 +367,7 @@ class JobService(ServiceBase): suggestion="Use resubmit for reprocessing needs, or leave the terminal job unchanged.", ) - now = datetime.now(UTC) + now = _utc_now_naive() job.status = JobStatus.FAILED job.date_updated = now @@ -398,7 +413,7 @@ class JobService(ServiceBase): suggestion="Only failed or cancelled sources can be resubmitted.", ) - now = datetime.now(UTC) + now = _utc_now_naive() for job_source in candidates: job_source.status = JobSourceStatus.PENDING diff --git a/tests/services/test_job_service.py b/tests/services/test_job_service.py index a34a0d7..a5e5567 100644 --- a/tests/services/test_job_service.py +++ b/tests/services/test_job_service.py @@ -20,9 +20,24 @@ from transcription.services.jobs import JobDeleteBlockedError from transcription.services.jobs import JobNotFoundError from transcription.services.jobs import JobResubmitBlockedError from transcription.services.jobs import JobService +from transcription.services.jobs import _as_naive_utc +from transcription.services.jobs import _utc_now_naive from transcription.services.sources import SourceService +def test_utc_now_naive_returns_naive_datetime(): + now = _utc_now_naive() + assert now.tzinfo is None + + +def test_as_naive_utc_normalizes_timezone_aware_datetime(): + aware = datetime.now(UTC) + normalized = _as_naive_utc(aware) + + assert normalized.tzinfo is None + assert normalized == aware.replace(tzinfo=None) + + class TestJobService: @pytest.mark.asyncio async def test_create_and_read_job(self, job_service: JobService, document_service: DocumentService):