bbchops/transcription
1
0
Fork 0
generated from john/python-template
Code Issues 4 Pull Requests Actions Packages Projects Releases Wiki Activity

Compare commits

base: bbchops/transcription:traumatized
Branches Tags
bbchops/transcription:traumatized bbchops/transcription:doc_update bbchops/transcription:going-back bbchops/transcription:main
..
compare: bbchops/transcription:a818d982a6e647eeddc211a1a442e996d4d9eeb7
Branches Tags
bbchops/transcription:traumatized bbchops/transcription:doc_update bbchops/transcription:going-back bbchops/transcription:main
1 Commits
traumatized .. a818d982a6
Author SHA1 Message Date
bbchops a818d982a6 Initial commit 2026-06-21 17:17:01 -05:00
239 changed files with 1038 additions and 39612 deletions
Expand all files Collapse all files
.dockerignore
-13
View File
@@ -1,13 +0,0 @@
.git
.gitignore
.vscode
.venv
.pytest_cache
.ruff_cache
__pycache__/
*.py[cod]
*.db
.env
tests/
docs/
uploads/
.env.production.example
-67
View File
@@ -1,67 +0,0 @@
# Production environment example for docker-compose.production.yml
# --- NiceGUI Server ---
HOST=0.0.0.0
PORT=8000
LOG_LEVEL=info
RELOAD=false
ENVIRONMENT=production
# TRANSCRIPTION_COMMIT=
RUN_EMBEDDED_WORKER=false
LOG_DIR=/app/data/logs
LOG_FILE_NAME=transcription.log
LOG_FILE_MAX_BYTES=10485760
LOG_FILE_BACKUP_COUNT=5
# --- AI provider ---
PROVIDER=openrouter
OPENROUTER_API_KEY=replace-with-real-key
PROVIDER_MODEL=google/gemini-2.5-flash
# PROVIDER_MODELS=["google/gemini-2.5-flash","anthropic/claude-sonnet-4"]
# OPENROUTER_HTTP_REFERER=
# OPENROUTER_APP_TITLE=
DEFAULT_PROMPT_NAME=transcribe_document.md
# TRANSCRIPTION_TEMPERATURE=
# TRANSCRIPTION_TOP_P=
# --- persistence ---
# Common database settings:
DATABASE__DRIVER=postgres
DATABASE__DATABASE=transcription
DATABASE__USER=transcription
DATABASE__PASSWORD=replace-with-strong-password
BOOTSTRAP_SCHEMA_ON_STARTUP=false
# SQLite-specific settings:
# DATABASE__PATH=./data/transcription.db
# SQLITE_CHECK_SAME_THREAD=false
# Postgres-specific settings:
DATABASE__HOST=postgres
DATABASE__PORT=5432
# --- filesystem paths ---
UPLOAD_DIR=/app/uploads
PROMPT_DIR=/app/prompts
# --- backup workflow helpers (not Runtime Settings model fields) ---
BACKUP_DIR=/backup
BACKUP_RETENTION_DAYS=14
# --- worker reliability ---
WORKER_MAX_RETRIES=0
WORKER_PROVIDER_TIMEOUT_SECONDS=30.0
WORKER_STALE_JOB_SECONDS=90.0
WORKER_RETRY_BACKOFF_SECONDS=1.0
WORKER_SHUTDOWN_GRACE_SECONDS=5.0
WORKER_POLL_INTERVAL_SECONDS=1.0
WORKER_MIN_TRANSCRIPTION_CHARS=0
WORKER_MIN_TRANSCRIPTION_LINES=0
WORKER_FAIL_ON_FINISH_REASON_LENGTH=false
# --- cloudflare tunnel ---
# Required for token-based tunnel startup.
CLOUDFLARE_TUNNEL_TOKEN=replace-with-cloudflare-tunnel-token
# --- deployment wiring helpers ---
# Runtime Settings writes target this file path inside the app container.
RUNTIME_SETTINGS_ENV_FILE=/app/.env.production
.env.production.postgres
-67
View File
@@ -1,67 +0,0 @@
# Production environment
# --- NiceGUI Server ---
HOST=0.0.0.0
PORT=8000
LOG_LEVEL=info
RELOAD=false
ENVIRONMENT=production
# TRANSCRIPTION_COMMIT=
RUN_EMBEDDED_WORKER=false
LOG_DIR=/app/data/logs
LOG_FILE_NAME=transcription.log
LOG_FILE_MAX_BYTES=10485760
LOG_FILE_BACKUP_COUNT=5
# --- AI provider ---
PROVIDER=openrouter
OPENROUTER_API_KEY=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
PROVIDER_MODEL=google/gemini-2.5-flash
PROVIDER_MODELS=["google/gemini-2.5-pro","google/gemini-2.5-flash","anthropic/claude-opus-5","anthropic/claude-sonnet-4","openai/gpt-5.6","openai/gpt-4o"]
# OPENROUTER_HTTP_REFERER=
# OPENROUTER_APP_TITLE=
DEFAULT_PROMPT_NAME=transcribe_document.md
# TRANSCRIPTION_TEMPERATURE=
# TRANSCRIPTION_TOP_P=
# --- persistence ---
# Common database settings:
DATABASE__DRIVER=postgres
DATABASE__DATABASE=transcription
DATABASE__USER=transcription
DATABASE__PASSWORD=<password>
BOOTSTRAP_SCHEMA_ON_STARTUP=false
# SQLite-specific settings:
# DATABASE__PATH=./data/transcription.db
# SQLITE_CHECK_SAME_THREAD=false
# Postgres-specific settings:
DATABASE__HOST=postgres
DATABASE__PORT=5432
# --- filesystem paths ---
UPLOAD_DIR=/app/uploads
PROMPT_DIR=/app/prompts
# --- backup workflow helpers (not Runtime Settings model fields) ---
BACKUP_DIR=/backup
BACKUP_RETENTION_DAYS=14
# --- worker reliability ---
WORKER_MAX_RETRIES=0
WORKER_PROVIDER_TIMEOUT_SECONDS=30.0
WORKER_STALE_JOB_SECONDS=30.0
WORKER_RETRY_BACKOFF_SECONDS=1.0
WORKER_SHUTDOWN_GRACE_SECONDS=5.0
WORKER_POLL_INTERVAL_SECONDS=1.0
WORKER_MIN_TRANSCRIPTION_CHARS=0
WORKER_MIN_TRANSCRIPTION_LINES=0
WORKER_FAIL_ON_FINISH_REASON_LENGTH=false
# --- cloudflare tunnel ---
# Required for token-based tunnel startup.
CLOUDFLARE_TUNNEL_TOKEN=CLOUDFLARE_TUNNEL_TOKEN=eyJhIjoiYTRhNjM0NzNhNzBiZjhhYmY3OWUyNjE4ZTcyNjgwZmMiLCJ0IjoiZWY1MjFkNWItYzY1ZS00ZGFmLTlmYTMtMzQyOGYzMGUyMDY4IiwicyI6IlpHWm1NVGRsTVRVdFpEYzNaaTAwWkRJeUxXRmhPRFV0TmpKallXRmhPRFJrWXpSaSJ9
# --- deployment wiring helpers ---
# Runtime Settings writes target this file path inside the app container.
RUNTIME_SETTINGS_ENV_FILE=/app/.env.production
.env.production.sqlite
-67
View File
@@ -1,67 +0,0 @@
# Production environment
# --- NiceGUI Server ---
HOST=0.0.0.0
PORT=8000
LOG_LEVEL=info
RELOAD=false
ENVIRONMENT=production
# TRANSCRIPTION_COMMIT=
RUN_EMBEDDED_WORKER=false
LOG_DIR=/app/data/logs
LOG_FILE_NAME=transcription.log
LOG_FILE_MAX_BYTES=10485760
LOG_FILE_BACKUP_COUNT=5
# --- AI provider ---
PROVIDER=openrouter
OPENROUTER_API_KEY=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
PROVIDER_MODEL=google/gemini-2.5-flash
PROVIDER_MODELS=["google/gemini-2.5-pro","google/gemini-2.5-flash","anthropic/claude-opus-5","anthropic/claude-sonnet-4","openai/gpt-5.6","openai/gpt-4o"]
# OPENROUTER_HTTP_REFERER=
# OPENROUTER_APP_TITLE=
DEFAULT_PROMPT_NAME=transcribe_document.md
# TRANSCRIPTION_TEMPERATURE=
# TRANSCRIPTION_TOP_P=
# --- persistence ---
# Common database settings:
DATABASE__DRIVER=sqlite
# DATABASE__DATABASE=transcription
# DATABASE__USER=transcription-local
# DATABASE__PASSWORD=My!3sons
# BOOTSTRAP_SCHEMA_ON_STARTUP=false
# SQLite-specific settings:
DATABASE__PATH=./data-local/transcription-local.db
SQLITE_CHECK_SAME_THREAD=false
# Postgres-specific settings:
# DATABASE__HOST=postgres
# DATABASE__PORT=5432
# --- filesystem paths ---
UPLOAD_DIR=./data-local
PROMPT_DIR=/data/prompts
# --- backup workflow helpers (not Runtime Settings model fields) ---
BACKUP_DIR=/backup
BACKUP_RETENTION_DAYS=14
# --- worker reliability ---
WORKER_MAX_RETRIES=0
WORKER_PROVIDER_TIMEOUT_SECONDS=30.0
WORKER_STALE_JOB_SECONDS=30.0
WORKER_RETRY_BACKOFF_SECONDS=1.0
WORKER_SHUTDOWN_GRACE_SECONDS=5.0
WORKER_POLL_INTERVAL_SECONDS=1.0
WORKER_MIN_TRANSCRIPTION_CHARS=0
WORKER_MIN_TRANSCRIPTION_LINES=0
WORKER_FAIL_ON_FINISH_REASON_LENGTH=false
# --- cloudflare tunnel ---
# Required for token-based tunnel startup.
CLOUDFLARE_TUNNEL_TOKEN=CLOUDFLARE_TUNNEL_TOKEN=eyJhIjoiYTRhNjM0NzNhNzBiZjhhYmY3OWUyNjE4ZTcyNjgwZmMiLCJ0IjoiZWY1MjFkNWItYzY1ZS00ZGFmLTlmYTMtMzQyOGYzMGUyMDY4IiwicyI6IlpHWm1NVGRsTVRVdFpEYzNaaTAwWkRJeUxXRmhPRFV0TmpKallXRmhPRFJrWXpSaSJ9
# --- deployment wiring helpers ---
# Runtime Settings writes target this file path inside the app container.
RUNTIME_SETTINGS_ENV_FILE=/app/.env.production
.gitattributes
-1
View File
@@ -1 +0,0 @@
* text=auto eol=lf
.github/agents/python-reviewer.agent.md
-23
View File
@@ -1,23 +0,0 @@
---
name: Python Architect Reviewer
description: Evidence-based senior architect reviewer for FastAPI, NiceGUI, and SQLModel codebases.
skills:
- python-code-reviewer
---
# Python Architect Reviewer
You are a Senior Python Architect performing an evidence-based, read-only code review.
> No `tools:` allowlist is declared here on purpose. Tool identifiers differ between the runtimes
> this agent is invoked from, so a hard-coded list silently under-tools the agent in one of them.
> Read-only discipline is enforced by the **Read-Only Scope** rule below, not by the frontmatter.
## Operating Principles
- **Stack Context:** Python 3.12+, FastAPI, NiceGUI, SQLModel, SQLAlchemy (SQLite/PostgreSQL), Pydantic V2, asyncio workers, and OpenRouter adapters.
- **Evidence-Based:** Always inspect real files. Every finding must reference concrete file paths and line numbers (e.g., `app/services/worker.py:45-78`). Do not speculate.
- **Tool Verification:** This is a `uv` project; the toolchain is not on `PATH`. Verify with `uv run ruff check .`, `uv run ty check`, and `uv run pytest -q -m "not external"`, and record the exact commands and outcomes. Never report a lint, type, or test claim you did not run.
- **Verify Recommendations, Not Just Findings:** Before recommending a change to a shared symbol, enumerate its consumers and confirm the fix is safe for each. See the skill's consumer-tracing step and `Blast Radius` field.
- **Skill Is Canonical:** The `python-code-reviewer` skill defines the review workflow, deterministic checks, severity and reachability rubrics, report location, and report template. Follow it exactly. Where this file and the skill disagree, the skill wins — do not restate its specifics here.
- **Read-Only Scope:** Do not modify source, tests, docs, instructions, or configuration. The review report is the only artifact you produce.
.github/instructions/documentation-sync.instructions.md
-41
View File
@@ -1,41 +0,0 @@
---
description: Require documentation updates whenever code changes alter contracts, behavior, or scope.
applyTo: 'src/transcription/**/*.py'
---
# Documentation Sync Requirements
Keep docs in sync in the same change whenever implementation alters a documented contract, behavior, or roadmap decision.
Documentation targets below always refer to the **current** baseline. `docs/index.md` states which
baseline that is; resolve any version-specific document from there. Never cite a superseded version
tree by name in this file or in the docs you update — retired revision trees are not authority, and
`tests/test_meta_contract_guards.py` fails active contract files that route authority through them.
## Update documentation when any of these change
1. **Schema/Data contract**
- Models, fields, enums, constraints, indexes, relationships, loading semantics.
- **Required doc update:** `docs/schema.md`.
2. **Configuration contract**
- `Settings` keys, defaults, required/optional environment values.
- **Required doc update:** `.env.production.example` and any directly related setup docs.
3. **User-visible UI behavior**
- Page flow, routes, button/action behavior, labels, status wording, empty/error states.
- **Required doc update:** relevant `docs/ui/pages/*.md` docs and feature docs when applicable.
4. **Error handling semantics**
- Error categories, retry behavior, envelope structure, translation boundaries.
- **Required doc update:** `docs/error_handling.md` and `docs/invariant/error_handling.md`.
5. **Roadmap/scope decisions**
- Version targets, sequencing, deferrals, and accepted alternatives.
- **Required doc update:** `docs/roadmap_plan.md`, plus any backlog or feature document for the
current baseline. Locate it through `docs/index.md` rather than assuming a version-named path.
## Working rule
If none of the categories above changed, documentation edits are optional.
If any category changed, update docs in the same PR/change set rather than deferring.
.github/instructions/error-handling.instructions.md
-121
View File
@@ -1,121 +0,0 @@
---
description: Cross-cutting error handling rules for services, API, and UI.
applyTo: 'src/transcription/**/*.py'
---
# Error Handling (Cross-cutting)
Primary references:
- `docs/error_handling.md`
- `docs/invariant/error_handling.md`
- `docs/requirements.md`
## Taxonomy and Categories
Use category-driven semantics aligned to canonical policy:
- `validation`
- `not_found`
- `conflict`
- `external`
- `timeout`
- `internal`
Do not invent ad hoc categories in user/API-facing envelopes unless canonical docs are updated.
Runtime/internal categories may be more specific for diagnostics and persistence, but they must map
deterministically to the canonical envelope categories through the centralized mapper in
`transcription.errors.canonical_error_category`.
Current internal categories:
- `validation_error`
- `user_input_error`
- `not_found_error`
- `conflict_error`
- `external_provider_error`
- `external_timeout_error`
- `processing_error`
- `infrastructure_transient_error`
- `infrastructure_persistent_error`
- `internal_unexpected_error`
Required internal -> canonical mapping:
- `validation_error`, `user_input_error` -> `validation`
- `not_found_error` -> `not_found`
- `conflict_error` -> `conflict`
- `external_provider_error` -> `external`
- `external_timeout_error`, `infrastructure_transient_error` -> `timeout`
- `processing_error`, `infrastructure_persistent_error`, `internal_unexpected_error` -> `internal`
## Translation Boundaries
- **Provider/adapters:** raise provider/domain exceptions; do not emit UI text.
- **Services:** map raw exceptions into internal categories and preserve causal chain (`raise ... from ...`).
- **UI/API:** map internal category -> canonical envelope category and emit user-safe, actionable messages.
## Retry Rules
- No auto-retry for `validation`, `not_found`, `conflict`.
- `external`/`timeout` may be retried when operation semantics are safe.
- Preserve each retry as new evidence where applicable (no history rewrite).
## Job/Page Failure Semantics
- Page-level (`JobSource`): `pending`, `transcribed`, `failed`, `cancelled`.
- Job terminals: `transcribed`, `partial_success`, `failed`.
- Cancellation must keep job-level and page-level semantics explicit and consistent.
- Do not emit legacy terminal state language such as `completed` in active user/API lifecycle contracts.
## User-Safe Messaging
- Never leak stack traces, credentials, auth headers, or local filesystem paths in user-facing output.
- Include actionable remediation guidance aligned to category.
- Keep envelope structure consistent across API endpoints.
### `AppError.message` vs `AppError.detail`
`AppError` carries two texts with different audiences, and they must not be collapsed. Getting this
wrong has already caused a real defect in this repository, in both directions.
| Attribute | Audience | Reaches | Rule |
| :--- | :--- | :--- | :--- |
| `message` | User and API clients | `ErrorEnvelope.message`, UI notifications | Stays generic. Never embed exception text, provider payloads, or filesystem paths. |
| `detail` | Internal only | Logs, and `format_error_detail` -> `ExecutionAttempt.error_detail` and `MaintenanceRun.error_detail` | Carries the root cause. Never rendered to users or serialized into an envelope without a sanitizing projection. |
- Putting root-cause data in `message` leaks infrastructure detail to users.
- Omitting it from `detail` silently degrades the provenance record this system exists to preserve —
a failed attempt whose `error_detail` says nothing is an attempt that cannot be diagnosed later.
- Any render boundary that displays persisted `error_detail` must apply the same no-local-path rule
as `message`: sanitize machine-local absolute paths before the text becomes user-visible.
- When you raise from a caught exception, populate **both**: a generic `message` and a `detail`
carrying `type(exc).__name__` and the exception text, with `raise ... from exc`.
- `detail` is optional (`None`). A read path that assumes it is populated must handle its absence.
- Before changing either attribute, or any helper that formats them, enumerate every consumer —
evidence writes, maintenance runs, logging, API envelopes, and UI presentation all read these
fields, and tests assert on the persisted text.
Canonical definitions live in `src/transcription/errors.py`; see also `docs/error_handling.md`.
## Logging and Diagnostics
- Log operation identifiers and error IDs where available.
- Preserve category + cause-chain context.
- Distinguish no-response timeout/network failures from returned provider error responses.
## Guardrails
- No broad catch-and-swallow patterns.
- No success-shaped fallback values after exceptions.
- Category mapping must remain deterministic and testable.
## Contract Sync Rule
If taxonomy, retries, or envelope semantics change:
1. Update canonical docs (`docs/error_handling.md`, and invariant docs if needed).
2. Update tests in the same change.
3. Update related instruction/skill references.
4. If change affects persisted status/category fields, update `docs/schema.md` when applicable.
.github/instructions/providers.instructions.md
-122
View File
@@ -1,122 +0,0 @@
---
description: Provider adapter rules for evidence capture, secret safety, and client lifecycle.
applyTo: 'src/transcription/providers/**/*.py'
---
# Provider Adapters
Primary references:
- `docs/invariant/ai_evidence_and_provenance.md` (canonical; provider adapters own provider-boundary evidence capture)
- `docs/architecture.md`
- `docs/schema.md`
The provider layer is where an external API becomes application data. It is also the only place
that can capture what actually crossed the wire — once a response reaches a service, the evidence
it did not preserve is gone permanently. Treat capture correctness as the primary job of this
layer and text extraction as secondary.
## Layer Boundary
- Adapters may depend on `transcription.config`, `transcription.providers.*`, the HTTP client, and
the provider SDK. They must not import `services`, `db`, `ui`, or `api`.
- Provider specifics — headers, model slugs, payload shapes, SDK types, error classes — stop here.
Callers receive `TranscriptionResult` and `ProviderError` subclasses only.
- Adapters raise provider/domain exceptions. They must not emit user-facing text, notifications,
or remediation wording; that translation belongs to services and UI. See
[error-handling instructions](./error-handling.instructions.md).
- Adapters do not persist. They return evidence; services decide what is written and when.
Enforced by `tests/test_provider_boundaries.py`.
## Contract Surface
- Every adapter satisfies the `TranscriptionProvider` protocol in `base.py`. Failed-call evidence is
returned through the caller-owned `ProviderCallEvidence` sink passed to `transcribe()`, so
evidence stays scoped to one invocation instead of living on mutable adapter instance state.
- `TranscriptionResult`, `RequestManifest`, and `TransportEvidence` are `extra="forbid"` and frozen.
Add a field to the contract rather than smuggling data through an untyped dict.
- Evidence contracts in `evidence.py` are versioned (`schema_name` + `schema_version`). A change to
the meaning or shape of a captured field requires a version bump, not a silent redefinition —
stored evidence must keep its original meaning.
## Transport Evidence
The rules below implement `docs/invariant/ai_evidence_and_provenance.md` §3.4-3.5. That document
wins if this file drifts from it.
- Capture the response body **at the HTTP boundary, before SDK parsing**, so fields the SDK does
not model are not lost. `_CapturingAsyncClient` exists for this; do not replace it with a
post-parse `model_dump()` and call the result transport evidence.
- Reset per-call capture state at the start of every call. Without it, a connection failure can
attach the *previous* call's response as evidence for this one. Guarded by
`tests/test_evidence_provenance.py::test_openrouter_does_not_reuse_prior_response_on_connection_failure`.
- Keep transport capture scoped to the call, not the adapter instance. Concurrent `transcribe()`
calls on one adapter must not be able to overwrite each other's response evidence.
- Handle the streamed-body case (`httpx.ResponseNotRead`) rather than assuming `response.content`
is always available.
- When no response arrives — timeout, DNS, connection reset — emit
`TransportEvidence(response_received=False)`. Absence of a response is itself evidence and must
be explicit, never an empty body or a missing record.
- Preserve safe response evidence for **unsuccessful** calls too, whenever a response was received.
- Never relabel an SDK snapshot or normalized metadata as transport evidence, and never backfill
it into an execution that predates capture.
## Secret Safety
- Persist response headers only through `filter_safe_response_headers` and the
`SAFE_RESPONSE_HEADERS` allowlist. Allowlist, never denylist: capture-then-redact is prohibited,
because an unknown header is unsafe by default.
- Adding a header to the allowlist is a deliberate evidence decision. Confirm it carries no
credential, cookie, or session material, and state why it is needed for correlation, content
interpretation, rate-limit diagnosis, or audit.
- API keys, `Authorization`, and cookies must never appear in a manifest, evidence record, log
line, or exception message.
- The request manifest references source content by identity (digest, size, media type, page).
Do not duplicate base64 source bytes into it — `_replace_embedded_media` exists for this.
## Execution Specification
The manifest must let a reader reconstruct what was asked, per invariant §3.3:
- Provider, requested model, full effective prompt text, and prompt digest.
- Every explicitly supplied parameter, and — separately — which optional parameters were
**omitted**. Omission is not the same as a null value or an assumed provider default; the
`optional_parameter_states` distinction between `omitted`, `null`, and `value` is deliberate.
- Timeout budget, retry policy, source reference, and `SoftwareContext` versions.
- Manifest digests use `canonical_json_bytes`. Do not hash a plain `json.dumps()`; key order and
separators must stay deterministic or digests become uncomparable.
## Client Lifecycle and Async Safety
- Reuse one pooled `AsyncClient` per adapter instance; do not construct a client per request.
- Accept an injected client so tests can drive the adapter without network access.
- Derive timeouts from `Settings` (`worker_provider_timeout_seconds`) rather than hard-coding, and
keep the client timeout aligned with the configured budget so the SDK cannot expire first and
hide the real failure.
- Implement `aclose()` and release pooled resources. An adapter that creates a client owns closing
it; one given a client must not close a caller-owned resource it did not create.
- Never block the event loop. Offload CPU-bound work (hashing large payloads, image encoding) with
`asyncio.to_thread`.
- Propagate `asyncio.CancelledError` untouched — do not convert cancellation into a provider error.
## Failure Handling
- Raise `ProviderAuthError` for authentication, `ProviderResponseError` for malformed or unusable
responses, and `ProviderError` otherwise.
- Always attach `request_manifest`, `transport_evidence`, and an accurate `failure_phase` to raised
errors. `failure_phase` must distinguish a received-but-failed response from a call that never
reached the provider.
- Validate responses with Pydantic rather than indexing into raw dicts.
- Invalid *optional* metadata (for example unparsable token counts) must not discard an otherwise
valid transcript. Degrade the metadata, not the result.
## Contract Sync Rule
If capture behavior, evidence schema, or the header allowlist changes:
1. Update `docs/invariant/ai_evidence_and_provenance.md` only if the durable preservation contract
itself is changing — that revision is deliberate and reviewed, not incidental.
2. Update `docs/schema.md` when persisted evidence fields change.
3. Update or add tests in the same change (`tests/providers/`, `tests/test_evidence_provenance.py`).
4. Bump the affected evidence `schema_version` when a field's meaning changes.
.github/instructions/services.instructions.md
-215
View File
@@ -1,215 +0,0 @@
---
description: Follow these guidelines when editing the services
applyTo: 'src/transcription/services/*.py'
---
# Services
## Structure
- Project core data models are defined in [models](../../src/transcription/db/models.py)
- One service class per **aggregate**, not per table. An aggregate is a root model plus
the models that have no independent lifecycle of their own. `DocumentType` has no
meaning without `Document`, so it belongs to `DocumentService`; it does not get its
own service. Splitting per table produces services that must reach across each other
for every real operation, which is what line 13 forbids.
- Only services interact with the database, and only through async methods.
- **A service module must not import another service module.** This is enforced by
[test_service_boundaries](../../tests/test_service_boundaries.py). Shared types go in a
neutral module that defines no service class (see [errors](../../src/transcription/services/errors.py)).
- Not every module in this package is a service. Modules fall into three kinds:
- **Aggregate services** own models and define a `*Service` class: `documents.py`, `sources.py`,
`jobs.py`, `people.py`, `photos.py`, `maintenance.py`, and `evidence.py` (read/projection only,
owns nothing).
- **Orchestration modules** define no service class and compose writes across aggregates:
`store.py`, `workflows.py`. They are the sanctioned place to create or delete rows owned by more
than one service — see [Service Composition](#service-composition).
- **Shared infrastructure and free-function helpers** are exempt from the service rules below:
`base.py` (`ServiceBase`), `registry.py` (`RegistryService`, a generic base for lookup tables —
not an aggregate owner itself), `unit_of_work.py`, `errors.py`, `normalization.py`, `prompts.py`,
`quality.py`, `media_storage.py`, `source_media.py`. `__init__.py` exposes `ServiceBundle`.
- Cross-cutting error behavior must follow
[error-handling instructions](./error-handling.instructions.md).
## Model Ownership
Every model has exactly one owning service. The owner defines that model's invariants and
is the only service that may **create or delete** its rows.
| Model | Owner |
| --- | --- |
| `Document`, `DocumentType`, `DocumentTag` | `DocumentService` |
| `Source`, `JobSource` | `SourceService` |
| `Job` | `JobService` |
| `Person`, `PersonRole`, `DocumentPerson`, `PersonTag` | `PeopleService` |
| `GenealogyPerson`, `GenealogyFamily`, `GenealogyFamilyChild`, `GenealogyCitation` | `MaintenanceService` |
| `Photo` | `PhotosService` |
| `MaintenanceRun` | `MaintenanceService` |
| `ExecutionAttempt` | `SourceService` |
| `Tag` | shared — see below |
Keep this table complete: every table in `src/transcription/db/models.py` appears exactly once,
except `Tag`. When you add a model, add its owner here in the same change.
### `Tag` is deliberately shared
`Tag` is one table reached through two `RegistryService[Tag]` facades that differ only in the
reference model they count usage through: `TagRegistry` (`documents.py`, via `DocumentTag`) and
`PersonTagRegistry` (`people.py`, via `PersonTag`). Both create and delete `Tag` rows through the
generic registry. This is the single sanctioned exception to one-owner-per-model — do not "fix" it by
assigning `Tag` to one service, because the other facade would then be creating rows it does not own.
Any change to `Tag` semantics, labels, or normalization must be validated against **both** facades
and the junction table each one counts.
`DocumentTag` and `PersonTag` follow the junction rule below: each is created and deleted only by the
service on its own side.
### Registries
`DocumentTypeRegistry`, `TagRegistry`, `PersonRoleRegistry`, and `PersonTagRegistry` are
`RegistryService` subclasses, not independent services. A registry belongs to the aggregate service
whose module declares it and shares that service's ownership. Registry CRUD uses `<operation>_entry`
naming (see [CRUD Methods](#crud-methods)).
### Junction tables
A junction table is owned by the service that **creates and deletes its rows** — its
lifecycle owner. The service on the other side may read through the junction (via
`selectinload`) but must not create rows in it.
- `document_person` -> `PeopleService`. Every write is there; `DocumentService` only
eager-loads through it.
- `job_source` -> `SourceService`, which creates the row, records each page's outcome,
and deletes it.
Two consequences follow, and both are deliberate:
- **Cascade deletion is not a violation.** A service deleting the aggregate root it owns
may delete rows referencing that root which cannot outlive it
(`JobService.delete_job_with_guardrails` deletes the job's `job_source` rows).
- **Evidence deletion is an explicit workflow, not a runtime path.** `JobService.delete_job_and_evidence`
deletes `ExecutionAttempt` rows owned by `SourceService`. That is sanctioned because it is the
named retention workflow that `delete_job_with_guardrails` refuses to perform implicitly — that
method *blocks* deletion when attempts exist. Append-only means runtime code never rewrites or
removes history to represent a new outcome; it does not forbid a deliberate, operator-invoked
retention operation. Do not add a second path that deletes attempts.
- **Ownership governs creation and deletion, not every state transition.** `job_source` is
both a link and the transcription work queue. `JobService.cancel_job` and
`resubmit_failed_sources` transition `job_source.status` across a whole job, because that
transition is a Job lifecycle event, not a per-page outcome. They create and delete
nothing.
`EvidenceService` is read-focused and projection-focused. It may coordinate selection
flows, but append-only attempt creation remains in `SourceService` write paths.
If a new operation cannot be expressed within one owner, it belongs in an orchestration
module, not in a cross-service import.
## Error Handling
- Errors used by a single service are defined at the top of that module and inherit from `AppError`.
- Errors shared by more than one service go in [errors](../../src/transcription/services/errors.py),
which defines no service class and is therefore importable by any of them.
- Use a context manager for large `try/except` blocks, like `handle_transcription_errors` in
[sources](../../src/transcription/services/sources.py).
- Category mapping, retry behavior, and translation boundaries are defined in
[error-handling instructions](./error-handling.instructions.md).
- Service-edge exception translation must be deterministic: map to canonical categories and preserve clear provider->service->API/UI boundaries.
## Checklist
- [ ] Uses `ServiceBase` for common logic
- [ ] Session kwarg for `AsyncSession` to pass a session object into each method
- [ ] Services use `self._session_scope` in their methods to pass the session through
- Multiple operations on the same object(s) require sharing a session between all the methods used
- [ ] Every model the module touches is either owned by it or reached read-only
- [ ] Evidence writes preserve append-only semantics
## CRUD Methods
- Name format `<operation>_<model>`, for example `create_document` or `update_job`.
- Where a service exposes create/read/update/delete for its root model, define them at the
top of the class in that order, before derived reads and workflow helpers.
- Not every aggregate needs all four. `ExecutionAttempt` is append-only evidence written by
`SourceService` workflow-facing methods, so `EvidenceService` deliberately exposes reads and
no create or delete.
Do not add unused CRUD methods to satisfy symmetry.
- `RegistryService` is generic across small lookup models and uses `<operation>_entry`
naming instead.
## Transaction Finalization
When a service method accepts an optional `session` kwarg, write methods must use `self._finalize` to finalize the transaction properly according to whether or not they are sharing a session.
- If `session` is `None`: the method owns the transaction and should `commit()`.
- If `session` is provided: the method must **not** commit; it should `flush()` so IDs and FK values are available to the caller's transaction.
- Use `refresh()` on returned ORM objects when the caller needs DB-populated values (defaults, triggers, merged state).
## Workflow Transaction Boundaries
For multi-step job lifecycles, orchestration functions must use explicit transaction phases.
- **Transaction A (claim):** transition `JobStatus.QUEUED -> JobStatus.PROCESSING` and commit immediately.
- Perform provider/network work **outside** database transactions.
- **Transaction B (terminal success):** write transcript content and set `JobStatus.TRANSCRIBED` in the same shared-session commit.
- **Transaction B (terminal failure):** write transcript error detail and set `JobStatus.FAILED` in the same shared-session commit.
- **Transaction C (retry path):** write transcript error detail, increment retry count, and set `JobStatus.QUEUED` in one shared-session commit.
Atomicity rules:
- Never commit transcript updates separately from the paired terminal/retry job status change.
- Terminal state (`TRANSCRIBED` or `FAILED`) and transcript row changes must succeed or roll back together.
- Retry persistence (`QUEUED` + retry increment + error detail) must succeed or roll back together.
### Multi-page batches
These two requirements are in tension for multi-page jobs: each page should be durable as
soon as its provider call returns, but the last page must commit together with the terminal
status. `process_queued_job` resolves it by committing every page except the last one
individually, then deferring the final page's write into `_finalize_batch_outcome` so it
shares the terminal transaction.
Both paths are shielded against cancellation, so the final page is no less durable than the
pages before it. Enforced by `tests/integration/test_pipeline_atomicity.py`; per-page
durability is separately enforced by
`tests/services/test_workflows_reliability.py::TestWorkflowReliability::test_transcribed_page_is_committed_before_next_provider_call_finishes`.
### Stale-reclaim safety
- `WORKER_STALE_JOB_SECONDS` must remain **greater than** `WORKER_PROVIDER_TIMEOUT_SECONDS`; stale
recovery must not be able to fire before one provider call can legitimately finish.
- Long-running multi-page orchestration must refresh job liveness explicitly between intermediate
page commits. Do not rely on incidental row updates or provider metadata writes to keep
`Job.date_updated` fresh.
- Enforced by `tests/test_config.py` and
`tests/services/test_workflows_reliability.py::TestWorkflowReliability::test_intermediate_page_commit_advances_job_liveness_timestamp`.
## Contract Alignment
- Treat `docs/` as the active architecture and requirements baseline.
- Legacy revision trees are out of scope for active implementation decisions and must not be referenced as authoritative service guidance.
- Treat `src/transcription/db/models.py` as runtime schema ground truth and `docs/schema.md` as the field-accurate contract mirror.
- `Job.status` success path is `TRANSCRIBED`.
- `JobSource.status` is queue/projection state only (`PENDING`, `TRANSCRIBED`, `FAILED`, `CANCELLED`).
- Source ingest may normalize media before persistence; persisted bytes/hash are canonical for processing and provenance.
- `ExecutionAttempt` is append-only evidence history; do not mutate historical attempt rows in runtime code.
- `Source.raw_transcription` is a projection, not authoritative history.
- Service/UI read paths that touch relationships must be eager-loaded for `lazy="raise"` compatibility.
- If model fields, enums, constraints, indexes, or relationship-loading semantics change, update `docs/schema.md` in the same change.
- If `Settings` fields or defaults change in `src/transcription/config.py`, update `.env.production.example` in the same change so keys/defaults remain synchronized and no stale settings remain documented.
## Schema Drift and Legacy Compatibility Policy
- Prefer schema migration over startup reconciliation or runtime compatibility paths in service writes.
- Do not add legacy read/write compatibility code in service workflows by default.
- If drift is discovered and a migration decision is ambiguous (for example, one-way destructive DDL, uncertain data retention impact, or unknown deployment sequence), pause and ask the user to choose migration vs compatibility before coding.
- If a temporary compatibility path is explicitly approved, document an expiration/removal plan in the same change.
# Service Composition
A service method may read across models it does not own, using eager loads from its own
aggregate root. What it may not do is import another service.
Operations that must **write** models owned by more than one service are composed in an orchestration module
([store](../../src/transcription/services/store.py),
[workflows](../../src/transcription/services/workflows.py)).
.github/instructions/tests.instructions.md
-119
View File
@@ -1,119 +0,0 @@
---
description: Authoring rules for the test suite, including markers, async discipline, and guard-test design.
applyTo: 'tests/**/*.py'
---
# Tests
Primary references:
- `AGENTS.md` (Change Protocol — failing test first)
- `docs/index.md` and `docs/invariant/*`
- `.github/skills/test-effectiveness-auditor/skill.md` (periodic audit of this suite)
The suite is not only regression protection here — it is where several architectural rules are
*defined*. `tests/test_service_boundaries.py`, `tests/test_ui_boundaries.py`,
`tests/test_provider_boundaries.py`, `tests/test_model_contract_guards.py`, and
`tests/test_meta_contract_guards.py` are the enforcement layer named in the `AGENTS.md` authority
order. A weak test in this repository does not merely fail to catch a bug; it can silently repeal a
documented invariant.
The baseline is green. `uv run pytest -q -m "not external"` must report zero failures and zero
errors, and there is no tolerated set of known-failing tests.
## Write the Failing Test First
For any behavioral fix, write the test before the fix and confirm it fails *for the reason you
expect*. A test that passes against the broken code proves nothing, and several defects in this
repository were subtle enough that a test written afterward would have done exactly that. If the
new test passes immediately, you have not reproduced the defect yet.
## Runner Configuration
Configured in `pyproject.toml`; do not work around these:
- `--strict-markers` — an unregistered marker is an error. Register new markers in
`[tool.pytest.ini_options] markers` with a description rather than inventing one at the call site.
- `asyncio_mode = "strict"` — every async test needs an explicit `@pytest.mark.asyncio`, and async
fixtures use `@pytest_asyncio.fixture`. There is no implicit promotion.
- `filterwarnings = ["error:coroutine .* was never awaited:RuntimeWarning"]` — an un-awaited
coroutine is an error, not a warning. This usually means a mock replaced an async callable with a
sync one, or an `await` was dropped. Fix the call; never silence the warning.
## Markers and Layout
- `unit` — pure logic, no framework or database.
- `integration` — touches framework, database, or multi-component contracts.
- `external` — calls live services; slow and credential-dependent.
`external` tests must also carry their own `skipif` so the suite stays green without credentials
(see `tests/services/test_transcription_external.py`). Local and documented runs use
`-m "not external"`; CI intentionally runs unfiltered, which is equivalent because those tests skip
themselves. Never let an unmarked test reach the network.
Place tests by the layer under test: `tests/services/`, `tests/ui/`, `tests/api/`,
`tests/providers/`, `tests/integration/`, with cross-cutting guards at the top level.
## Fixtures and Isolation
- Prefer the shared fixtures in `tests/conftest.py` (`default_settings`, `async_session`,
`default_session_factory`, and the per-aggregate service fixtures) over building settings or
engines by hand.
- `Settings` is isolated suite-wide by the session-scoped autouse fixture in `conftest.py`, because
`env_file` resolves against the working directory. Tests that need env-file loading pass
`_env_file=` explicitly; tests asserting declared defaults need nothing. Do not reintroduce
reliance on a developer's local env file. Guarded by `tests/test_config_isolation.py`.
- Database fixtures refuse to run against anything but the per-test path, and that refusal is
deliberate. Never relax it to point a destructive fixture at a real database.
- Tests must not leave artifacts outside `tmp_path`.
## Assertion Strength
Assert on the domain effect, not on the fact that code ran.
- Prefer persisted state, status transitions, error categories, and evidence records over
"no exception raised", "not None", or a bare status code.
- **Read committed state through a separate session.** Asserting against the same session that
performed the write can pass on unflushed in-memory state and prove nothing about durability.
This is how the atomicity guarantees in `tests/services/test_workflows_reliability.py` and
`tests/integration/test_pipeline_atomicity.py` are made real.
- Critical paths need negative-path coverage — timeouts, provider failures, validation errors,
cancellation. Happy-path-only coverage of a critical module is a gap, not a suite.
- Avoid count-threshold assertions as a proxy for correctness. A test asserting "at least N items
were discovered" passes indefinitely while the thing it was meant to protect rots; assert on a
specific known member instead.
## Guard Tests
Structural guards carry extra obligations, because they are cited as proof that a rule holds.
- **Guard the guard.** Every scanning guard needs a companion assertion that the scan actually found
something, following the existing `test_*_are_discovered` pattern. A guard that silently scans an
empty set passes forever.
- **Scope must match the claim.** A guard's name and docstring must describe only what it actually
verifies. A test covering one function while appearing to enforce a repo-wide rule is worse than
no test, because it stops anyone from writing the real one.
- **Prove non-vacuity by injected fault.** Temporarily introduce the violation, confirm the guard
fails with a comprehensible message, then revert. Do this whenever you add or materially change a
guard. Revert with an explicit edit if the file has uncommitted changes — `git checkout --` will
discard them.
- **Prefer structural analysis to substring matching.** AST inspection of imports and definitions is
resistant to false negatives; a bare-name search across the repository is not, since an unrelated
mention anywhere makes dead code look reachable.
- Failure messages should name the offending file, symbol, and the remedy. These fire for people who
did not write the guard.
- Any new file under `.github/**` must be added to `ACTIVE_CONTRACT_FILES` in
`tests/test_meta_contract_guards.py`, or the completeness guard fails by design.
## Redundancy
Duplicate coverage across layers costs runtime and dilutes signal. Pick the canonical layer for a
behavior — unit for logic, integration for wiring — and let the other layer assert only what is
unique to it. Retire tests superseded by a stronger guard instead of accumulating both, and record
deliberate retentions with a rationale rather than leaving them unexplained.
## Contract Sync Rule
When a test encodes or relaxes a documented rule, update the corresponding instruction file or
`docs/*` page in the same change. When a guard test is the enforcement for a rule stated in
`AGENTS.md` or an instruction file, cite the test by name there so the link survives refactoring.
.github/instructions/ui.instructions.md
-78
View File
@@ -1,78 +0,0 @@
---
description: "Use when modifying the NiceGUI application under src/transcription/ui. Defines ownership and dependency boundaries for UI registration, pages, components, services, persistence, state, and static assets."
applyTo: 'src/transcription/ui/**/*.py'
---
# UI Conceptual Boundaries
Keep dependencies flowing in this direction:
`ui/__init__.py` -> `pages` -> `components`
Pages may depend on application services and framework-provided dependencies. Components may depend on smaller components and shared presentation helpers. Services and domain modules must never depend on the UI.
Cross-cutting error behavior must follow
[error-handling instructions](./error-handling.instructions.md).
## Package Root
- Keep `ui/__init__.py` as the UI composition root: register global assets, register pages, and mount NiceGUI on FastAPI.
- Do not put feature rendering, service calls, persistence, or route-specific state in the package root.
## Pages
- Pages own route registration and route-level orchestration.
- Resolve request or application dependencies, call [services](../../src/transcription/services/), adapt returned data for presentation when needed, and coordinate refresh, navigation, and notifications here.
- Do not query, mutate, commit, or roll back the database from a page. Do not import database engines, sessions, operations, or query-building APIs. Persistence belongs to services or workflow functions.
- Framework dependency types may cross into page handlers only to construct or invoke services; do not pass sessions or session factories into components.
- Keep business rules, lifecycle transitions, transaction boundaries, and cross-service workflows out of page callbacks.
## Components
- Components own reusable rendering, widget-local state, input normalization, and presentation-only formatting.
- Expose user actions through typed callback parameters. The calling page decides which service or workflow runs and what refresh or navigation follows.
- Do not register routes, resolve request/app state, instantiate services, or access persistence from components.
- Components may accept ORM models returned by services as read-only snapshots. Only use fields and relationships that the service loaded eagerly; never mutate models, trigger lazy loading, or expose session behavior.
- A component may compose lower-level components, but it must not import from `pages`.
## Shared UI Infrastructure
- Keep app-wide navigation and layout primitives in `components/app_shell.py`.
- Keep generic table/event adaptation in `components/table/common.py`; feature-specific columns, row read models, and formatting belong in the feature table module.
- Keep exception normalization and user-facing error display in `components/error_presenter.py`; preserve `AppError` details and operation identifiers at page/component boundaries.
- Use `components/media_urls.py` for media URL generation; do not hand-build upload/static paths in page code.
## CSS Assets
- Keep all application CSS in `ui/static/theme.css`; do not add page- or component-specific stylesheets or embed style blocks in Python components.
- Load `theme.css` once from the composition root with `ui.add_css(..., shared=True)`.
- Read stylesheet text through `importlib.resources.files(...)` so loading works from installed packages and is independent of the working directory.
- Centralize CSS reading in one typed helper cached by resource path.
- Do not encode application behavior in CSS or other static assets.
## State and Side Effects
- Limit component state to ephemeral interaction state such as loading flags, form values, dialogs, and expansion state.
- Application and worker state must be resolved at the page or application boundary and passed through narrow interfaces.
- Keep filesystem, network, provider, and worker orchestration behind application services or dedicated adapters.
## Media Route Safety Rules
Two patterns are approved:
1. **Record-validated API routes** for print/export contexts.
2. **Controlled upload URL resolver** (`components/media_urls.py`) for general UI media.
Prohibited patterns:
- Direct `file://` links or exposing local filesystem paths.
- Manual URL construction from raw `Path` values in pages/components.
- User-facing payloads containing local absolute paths.
## Contract Alignment
- Treat `docs/` as the active baseline.
- Resolve lifecycle and status semantics against `src/transcription/db/models.py` and `docs/schema.md`; do not introduce alternate status labels or implied legacy states in UI behavior.
- Use status vocabulary exactly as modeled (`queued`, `processing`, `transcribed`, `partial_success`, `failed`; and `pending`, `transcribed`, `failed`, `cancelled`).
- Print/export media flows must use record-validated routes; direct local filesystem paths are prohibited.
- If lifecycle wording/behavior changes, update corresponding `docs/ui/pages/*.md` contracts in the same change.
.github/prompts/review-python-architecture.prompt.md
-23
View File
@@ -1,23 +0,0 @@
---
name: Review Python Architecture
description: Run an evidence-based architectural code review using the Python Architect Reviewer agent and python-code-reviewer skill.
agent: Python Architect Reviewer
---
# Instructions
Execute a comprehensive, evidence-based code review of the target codebase.
## Target Scope
- **Review Target:** the repository root, unless the invoker names a narrower path; review that path instead.
- **Source Root:** `src/`
- **Docs Root:** `docs/`
- **Focus Areas:** FastAPI endpoints, NiceGUI components, SQLModel persistence, asyncio workers, Pydantic V2 models, and OpenRouter provider adapters.
## Execution Rules
1. Map repository layout, dependency manifests, and configuration files from the project root before inspecting modules.
2. Read real code modules under `src/` (or the specified target path); cite exact file paths and line ranges for every finding.
3. Validate issues by running `uv run ruff check .`, `uv run ty check`, and `uv run pytest -q -m "not external"`. Nothing is on `PATH` in this `uv` project, so bare `ruff`/`ty`/`pytest` will fail.
4. Check for duplication, divergent implementations, and extractable helpers.
5. Format the entire review following the standardized 10-section template defined in the `python-code-reviewer` skill.
6. Write the final report to `./docs/reviews/<YYYY-MM-DD>-code-review.md`, using today's date. This path is defined by the skill; do not write the report anywhere else.
.github/skills/evidence-provenance-auditor/skill.md
-80
View File
@@ -1,80 +0,0 @@
---
name: evidence-provenance-auditor
description: Deterministic reviewer for transcription evidence/provenance guarantees. Use when changes touch execution attempts, source storage, retries, transport evidence, artifact provenance, or evidence exports.
---
# Evidence & Provenance Auditor
Perform focused, deterministic audits of evidence integrity and provenance behavior.
## When to Use
- Reviewing changes in:
- `src/transcription/services/sources.py`
- `src/transcription/services/store.py`
- `src/transcription/services/workflows.py`
- `src/transcription/services/evidence.py`
- `src/transcription/db/models.py`
- Auditing evidence exports/imports or evidence-display behavior.
- Verifying no drift from canonical provenance invariants.
## Normative References (must be used)
1. `docs/invariant/ai_evidence_and_provenance.md`
2. `docs/schema.md`
3. `docs/requirements.md`
4. `docs/error_handling.md`
## Deterministic Pass/Fail Checks
### A. Append-only history
- Every provider call results in a new `ExecutionAttempt`.
- Runtime paths do not mutate historical attempts to represent new outcomes.
- Retry behavior appends attempts rather than rewriting prior rows.
### B. Projection vs authority separation
- `Source.raw_transcription` and preferred pointers are mutable projection surfaces.
- Attempt rows remain authoritative historical evidence.
- Candidate promotion updates projection pointers without rewriting history.
### C. Transport evidence semantics
- Transport evidence is correctly labeled as application-boundary capture.
- SDK snapshots/normalized metadata are not mislabeled as native upstream payload.
- No-response timeout/network states are explicit.
### D. Canonical source identity
- Canonical stored bytes/hash/size are internally consistent.
- If ingest normalization is applied, code/docs consistently represent resulting canonical identity.
- Post-ingest derivatives do not overwrite canonical source bytes.
### E. Secret safety
- No credentials/auth headers/cookies/unrestricted headers persisted.
- Header persistence uses explicit allowlist semantics.
### F. Route/path safety
- Print/export source access is record-validated.
- UI/media path construction does not expose local filesystem paths.
### G. Schema/docs alignment
- Evidence-related model fields and semantics align with canonical docs.
- Evidence model changes require same-change doc updates.
### H. Canonical authority boundaries
- Active guidance resolves against `docs/*` and current instruction files.
## Review Workflow
1. Read normative references first.
2. Inspect model + service + workflow write paths.
3. Inspect evidence read/display/export paths.
4. Report high-confidence findings with concrete path/line evidence.
5. Classify each finding by invariant family (A-H).
## Output Format
Use this structure:
- Verdict by invariant family (A-H)
- Findings with `Location`, `Observed Behavior`, `Risk`, `Recommended Fix`
- Drift table (`Doc claim` vs `Code reality` vs `Action`)
- Regression guards needed
.github/skills/python-code-reviewer/skill.md
-276
View File
@@ -1,276 +0,0 @@
---
name: python-code-reviewer
description: Perform an evidence-based, senior architect code review for Python codebases using FastAPI, NiceGUI, SQLModel, SQLAlchemy, Pydantic V2, asyncio, and OpenRouter. Use when asked to review Python repositories, perform architectural or code audits, or evaluate code against Python 3.12+ best practices.
---
# Python Code Reviewer
Perform thorough, evidence-based code reviews for Python projects. Every finding must cite concrete file paths and line ranges, avoid speculation, and include recommended fixes.
## When to Use
- Performing an architectural or code quality review of a Python codebase.
- Auditing applications using FastAPI, NiceGUI, SQLModel/SQLAlchemy, Pydantic V2, or asyncio workers.
- Generating structured Markdown review reports in `./docs/reviews`.
## Technical Stack Scope
- **Runtime:** Python 3.12+
- **Web Application:** FastAPI and NiceGUI
- **Persistence:** SQLModel, SQLAlchemy (SQLite and PostgreSQL support)
- **Validation & Settings:** Pydantic V2 and pydantic-settings
- **Concurrency:** Python asyncio workers
- **Vision/LLM Integration:** OpenRouter / provider adapters
- **Image & Print Pipeline:** Pillow-backed media handling and print/export rendering
- **Quality & Testing:** pytest, pytest-asyncio, Ruff, and ty
NiceGUI is pinned to an exact version (`nicegui==3.13.0` in `pyproject.toml`); API guidance
must be correct for that release rather than for the latest published version. The exact pin is
a deliberate release-stability decision recorded in `docs/production-runbook.md` ("Dependency
upgrade policy") — do not report it as a defect or recommend widening it.
## Review Workflow
1. **Map the Repository First:** Inspect entry points, package layout, configurations, dependency manifests, and any project-specific rule files (`AGENTS.md`, `.github/instructions/`, `.github/skills/`). Project-specific conventions override generic advice.
2. **Establish Canonical Authority First:** Read architecture/contracts (`docs/*`, `docs/invariant/*`, UI docs) and active instructions/skills before evaluating source behavior.
3. **Read Representative Modules:** Sample across all layers (routes/pages, UI components, services, workers, persistence, provider adapters, settings, tests) before drawing conclusions.
4. **Run Drift Analysis:** Compare documented intended behavior versus repository ground truth; identify both implementation drift and undocumented-but-repeatable conventions that should be formalized.
5. **Run Dead-Code/Orphan Sweep:** Identify candidate orphan modules/functions/classes with zero inbound references, then verify expected exceptions (entrypoints, framework/plugin registration, dynamic imports/reflection, CLI hooks, test-only utilities) before marking as orphaned.
6. **Assess Boundary and Coupling Health:** Evaluate UI/service/persistence/provider dependency flow, identify circular dependencies, leaky abstractions, and transaction ownership ambiguity.
7. **Assess Invariant Placement:** For each hard rule, decide whether it belongs in docs (rationale), instructions (active steering), skills (periodic audit procedure), or deterministic tests (enforcement).
8. **Verify Claims:** This is a `uv` project (`uv.lock`, root `ruff.toml`). Run `uv run ruff check .`, `uv run ty check`, and `uv run pytest -q -m "not external"` rather than guessing, and record the exact commands and their outcomes in the report.
9. **Validate Recommendations Against Consumers:** A recommendation is a claim about the future and must be verified like any other. Before recommending a change to a shared symbol — a model field, an exception attribute, a helper's return value, a function signature — enumerate **every** consumer of that symbol (`grep` the whole repo, including tests) and confirm the fix is safe for each one. Record the consumers in the finding's **Blast Radius**. A fix that is correct for the path that produced the finding can silently break a second consumer, and evidence/provenance and logging paths are the usual casualties because they read the same fields the UI does.
10. **Prioritize Hot Paths:** Focus deeply on request handling, database sessions, background workers, and external API calls.
11. **Enforce Read-Only Safety:** Do not modify code unless explicitly instructed.
12. **Escalate Provenance Audits:** For evidence/provenance-heavy changes, apply invariant checks from `.github/skills/evidence-provenance-auditor/skill.md` and include pass/fail outcomes in the report.
13. **Escalate Test-Suite Audits:** When findings touch test coverage, redundancy, or assertion strength, apply `.github/skills/test-effectiveness-auditor/skill.md` and include its outcomes alongside the provenance results.
### Worked example: why step 9 exists
The 2026-08-23 review recommended fixing a filesystem-path leak in
`classify_unexpected_error` by making `AppError.message` generic and logging the exception
detail instead. The analysis of the leak was correct, and the fix was implemented as written.
It was wrong. `AppError.message` had a second consumer the review never traced:
`format_error_detail`, which writes `ExecutionAttempt.error_detail` — a **provenance record**.
The recommended fix closed a privacy leak by silently stripping root-cause data from the
evidence history this system exists to preserve. It was caught only because an unrelated
integration test asserted on the persisted error text.
The correct fix separated the audiences — a user-safe `message` and an internal-only `detail`
that still reaches evidence and logs. One `grep` for consumers of `.message` during the review
would have found this. Treat any recommendation that changes a widely-read field as unverified
until its consumers are enumerated.
## Repo-Specific Deterministic Checks (Transcription)
When reviewing this repository, always include explicit pass/fail checks for the following.
Where **Enforced by** reads *unenforced*, recommending a deterministic test is itself a finding.
| # | Check | Enforced by |
| :-- | :--- | :--- |
| 1 | **Service boundary rule:** no service-to-service imports | `tests/test_service_boundaries.py` |
| 2 | **UI boundary rule:** pages/components do not perform persistence access | `tests/test_ui_boundaries.py` |
| 3 | **Status vocabulary conformance:** `JobStatus`/`JobSourceStatus`/`JobPurpose` usage matches current enums in `src/transcription/db/models.py`; no stringly-typed status literals | `tests/test_model_contract_guards.py` |
| 4 | **Evidence ownership conformance:** append-only attempt history is preserved and projection writes are not mistaken for history mutation (`src/transcription/services/sources.py`, `src/transcription/services/evidence.py`) | `tests/test_evidence_provenance.py::test_attempts_are_append_only_and_exported_with_integrity` |
| 5 | **Canonical authority:** findings must resolve against `docs/*` first | `tests/test_meta_contract_guards.py::test_canonical_authority_references_are_present` |
| 6 | **Schema contract fidelity:** when model/persistence behavior changes, `docs/schema.md` remains field-accurate with `src/transcription/db/models.py` | `tests/test_model_contract_guards.py` (field names, ordering, enum members, table coverage), `tests/test_meta_contract_guards.py` (presence and references) |
| 7 | **Media boundary conformance:** print/export media is record-validated and UI media URL generation uses controlled resolver paths | `tests/test_media_path_safety.py`, `tests/ui/test_media_urls.py` |
| 8 | **Eager-loading conformance:** service/UI read paths satisfy `lazy="raise"` expectations | `tests/test_model_contract_guards.py` (declaration-side; documented `noload` exceptions must match `docs/schema.md`) |
| 9 | **Cross-cutting error conformance:** service/API/UI translation and retry behavior align with `.github/instructions/error-handling.instructions.md` | `tests/test_errors.py`, `tests/api/test_error_responses.py`, `tests/ui/test_error_presenter.py` |
| 10 | **Orphaned/dead-code conformance:** include a deterministic orphan sweep and report confirmed orphans removed/retained with rationale | `tests/test_orphan_sweep.py` (`KNOWN_ORPHANS` records each retained orphan and its rationale) |
## Core Review Areas
### 1. Python Best Practices (3.12+)
- **Type Annotations:** Ensure completeness, modern syntax (`X | None`, builtin generics, `Self`, `type` statements), and avoid unparameterized containers or bare `Any`.
- **Error Handling:** Identify bare/broad `except`, swallowed exceptions, missing `raise ... from`, and exceptions used for control flow.
- **Resource Management:** Verify context managers for files, DB sessions, HTTP clients, and locks. Check for leaked tasks or connections.
- **Data Modeling:** Check proper use of dataclasses vs. Pydantic models vs. dictionaries. Eliminate mutable default arguments and stringly-typed payloads.
- **Idioms & Clean Code:** Verify `pathlib` usage over `os.path`, comprehensions vs manual loops, removal of dead code, and elimination of magic numbers.
### 2. FastAPI
- **Dependency Injection:** Verify `Depends` is used for shared resources (DB sessions, settings, clients) rather than global singletons.
- **Route Design:** Validate HTTP verbs, status codes, path/query/body typing, `response_model`, and domain-based router organization.
- **Lifecycle & Concurrency:** Ensure lifespan handlers are used instead of deprecated `@app.on_event`. Flag blocking synchronous calls in `async def` endpoints.
### 3. NiceGUI
- **Separation of Concerns:** Ensure UI components delegate business logic and persistence to service layers.
- **Client State Handling:** Verify correct use of client-scoped state vs global state to avoid state leaks across sessions.
- **Async Execution:** Check for blocking operations on the UI event loop and unbounded timers/pollers.
### 4. Persistence (SQLModel / SQLAlchemy)
- **Session Lifecycle:** Enforce one session per request/unit of work with explicit commit/rollback/close boundaries.
- **Query Optimization:** Detect N+1 patterns, missing eager loads (`selectinload`/`joinedload`), queries inside loops, and unindexed filters.
- **Cross-Dialect Portability:** Check compatibility for both SQLite (WAL mode, pragmas) and PostgreSQL (JSONB, locking, autoincrement).
### 5. Pydantic V2 & Settings
- **V2 Migration:** Flag legacy V1 patterns (`@validator`, `Config` class, `.dict()`, `parse_obj`) and use V2 equivalents (`@field_validator`, `model_config = ConfigDict(...)`, `model_dump()`).
- **Settings Management:** Ensure `BaseSettings` is the single source of truth without scattered `os.getenv` calls or committed secrets.
### 6. Concurrency & Asyncio Workers
- **Task Lifecycle:** Flag unreferenced `create_task` calls that risk garbage collection, missing cancellation handling, and lack of graceful shutdown.
- **Backpressure & Synchronization:** Check for appropriate use of `asyncio.Queue`, `TaskGroup`, `Lock`, and backoff retries.
### 7. Provider Adapters (OpenRouter / APIs)
- **Adapter Encapsulation:** Verify provider-specific details (headers, model names, payload formats) do not leak into UI or business logic.
- **Client Lifecycle:** Reuse shared `AsyncClient` instances with proper connection pooling and timeouts. Validate API responses using Pydantic schemas.
### 8. Testing & Quality Tooling
- **Test Isolation:** Verify tests do not rely on live external services, real clocks, or shared global state.
- **Async Test Setup:** Check `pytest-asyncio` configuration and fixture lifecycle.
- **Project Test Contract (`pyproject.toml`):** `--strict-markers` is enabled, so every marker must be declared; `asyncio_mode = "strict"` requires explicit `@pytest.mark.asyncio`; declared markers are `unit`, `integration`, and `external`, and `external` must be excluded from default verification runs. `filterwarnings` promotes `coroutine ... was never awaited` to an **error** — treat any unawaited coroutine as a hard failure and a Critical/High finding, never a warning.
- **Suite Signal Quality:** For low-value, redundant, or tautological tests, escalate to `.github/skills/test-effectiveness-auditor/skill.md` and fold its outcomes into the report.
### 9. Duplication & Consolidation
- Identify repeated code blocks, candidate helper abstractions, divergent patterns for identical operations, and duplicated domain constants.
### 10. Orphaned/Dead Code Audit
- Find candidate orphan modules/functions/classes with no inbound references.
- Validate each candidate against dynamic wiring exceptions (entrypoints, plugin registration, reflection/dynamic imports, CLI hooks, test utilities).
- Report outcomes as: removed orphan, retained-with-justification, or uncertain-follow-up.
### 11. Architecture & Governance
- **Architectural Drift:** Compare intended architecture rules against implementation behavior and cite concrete drift points.
- **Systemic Health:** Evaluate domain cohesion, dependency direction, lifecycle consistency, and operational reliability seams.
- **Invariant Routing:** Recommend the correct enforcement layer per rule (docs vs instructions vs skills vs tests).
- **Meta-Tooling Alignment:** Recommend updates for instruction files and skills when repository patterns or contracts evolve.
## Severity Rubric
Severity reflects concrete consequence, never style preference or effort to fix.
- **Critical:** Data or evidence loss/corruption; provenance or append-only history violated; secret leakage; silent wrong output presented as authoritative.
- **High:** Architectural boundary violated (service/UI/persistence/provider); runtime failure or unhandled exception on a hot path (request handling, DB sessions, worker loop, external API calls); documented invariant contradicted by implementation.
- **Medium:** Correctness risk under load or edge conditions (N+1, missing eager load, leaked task, missing timeout); drift between docs and code with no immediate runtime impact.
- **Low:** Maintainability, typing completeness, duplication, naming, or dead code with no behavioral risk.
### Reachability
Severity states how bad the consequence is; **Reachability** states whether it can happen today.
They are independent, and a finding is not complete without both. Record one of:
- **Live:** reachable in the current configuration and deployment.
- **Latent:** the defective code is present but unreachable because of a current setting, single-
instance deployment, or absent caller. **State the exact condition that unblocks it.**
- **Theoretical:** requires a combination the project has explicitly ruled out.
Latent findings carry a scheduling constraint that severity alone cannot express: a latent defect
must usually be fixed *before* the change that makes it live, not after. Say so explicitly in the
finding and reflect the ordering in the §9 action plan — for example, "fix the retry-category gate
before raising `worker_max_retries` above 0," or "handle this `IntegrityError` before deploying a
second worker replica." Do not downgrade severity merely because a finding is latent.
### Conflicting invariants
When a fix sits between two invariants that pull in opposite directions, say so in the
**Recommendation** and name both, along with the test that guards each. Flag explicitly what the
over-correction would be, because the simplest-looking fix usually satisfies one invariant by
silently destroying the other. A recommendation that resolves one side without naming the other is
incomplete and will be implemented incorrectly.
## Output Report Structure & Template
Generate Markdown reports at `./docs/reviews/<YYYY-MM-DD>-code-review.md` following this exact
template structure. Reports are dated, non-canonical artifacts: `docs/reviews/**` is explicitly
**not** part of the canonical authority set that the canonical-authority check resolves against.
```markdown
# Architecture & Code Review Report
**Repository Target:** `project-root/`
**Target Stack:** Python 3.12+ | FastAPI | NiceGUI | SQLModel/SQLAlchemy | Pydantic V2 | asyncio | OpenRouter
---
## 1. Executive Summary
- 5-10 bullets on overall health, top risks, and high-leverage refactors.
---
## 2. Executive Architecture Assessment
- High-level verdict on domain cohesion, boundary clarity, and architecture fitness.
- Top 3-5 systemic risks or bottlenecks.
---
## 3. Findings by Severity
### Critical Severity
#### [CRIT-01] Title
- **Location:** `path/to/file.py:lines`
- **Reachability:** Live / Latent (state the exact condition that unblocks it) / Theoretical
- **Problem & Consequence:** Concrete consequence, not a style opinion.
- **Blast Radius:** Every consumer of the symbols the recommendation changes, each confirmed
safe. Write `None — change is local` only after actually searching. If the fix touches a
shared field or helper, list the call sites (including tests and evidence/logging paths).
- **Recommendation:** Fix with before/after sketch. If two invariants conflict here, name both,
name the test guarding each, and state what the over-correction would be.
- **Effort:** S / M / L
### High Severity
#### [HIGH-01] Title
...
### Medium Severity
#### [MED-01] Title
...
### Low Severity
#### [LOW-01] Title
...
---
## 4. Architectural Drift & Gap Analysis
`Direction` is `doc->code` (implementation must change to match documented intent) or
`code->doc` (an undocumented but repeatable convention that should be formalized).
| Area / Component | Direction | Documented / Intended Rule | Actual Implementation State | Severity | Recommended Resolution |
| :--- | :--- | :--- | :--- | :--- | :--- |
---
## 5. Invariant Inventory & Routing Recommendations
| Invariant / Constraint | Current Location | Recommended Target Layer | Rationale |
| :--- | :--- | :--- | :--- |
---
## 6. Stack-Specific Analysis
- Python 3.12+ Best Practices
- FastAPI
- NiceGUI
- SQLModel & SQLAlchemy
- Pydantic V2 & Settings
- Asyncio Workers
- OpenRouter / Adapter Boundary
- Testing & Quality Tooling
---
## 7. Duplication & Consolidation Report
| Pattern / Duplication | Locations | Proposed Canonical Home | Estimated Lines Removed |
| :--- | :--- | :--- | :--- |
### Proposed Canonical Abstractions
- Code signatures and implementation homes.
---
## 8. Meta-Tooling & Instruction Update Recommendations
- Required updates to docs, instructions, skills, or tests to keep enforcement current.
---
## 9. Prioritized Dependency-Ordered Action Plan
1. **Phase 1: Blocking fixes**
2. **Phase 2: Enforcement hardening**
3. **Phase 3: Reliability & concurrency**
4. **Phase 4: Consolidation & refactoring**
5. **Phase 5: Non-blocking governance/documentation depth**
---
## 10. Preserved Strengths
- Existing patterns worth maintaining.
.github/skills/test-effectiveness-auditor/skill.md
-98
View File
@@ -1,98 +0,0 @@
---
name: test-effectiveness-auditor
description: Periodic reviewer for test-suite signal quality. Detects low-value or redundant tests, validates contract coverage, and recommends pruning or strengthening actions.
---
# Test Effectiveness Auditor
Run a deterministic audit of test usefulness. Focus on whether tests catch real regressions, not whether they merely execute code.
## When to Use
- Monthly/quarterly test-health review.
- Pre-release hardening when test count grows quickly.
- After major AI-assisted test generation.
- When suite runtime is increasing without clear quality gains.
## Primary Objectives
1. Identify tests that are weak, redundant, or non-diagnostic.
2. Confirm critical contracts are guarded by meaningful assertions.
3. Produce a prune/strengthen backlog with explicit risk and effort.
## Normative References (Transcription Repo)
1. `docs/*`
2. `docs/invariant/*`
3. `.github/instructions/*.instructions.md`
4. `tests/test_meta_contract_guards.py`
5. Contract-specific guards (`tests/test_service_boundaries.py`, `tests/test_ui_boundaries.py`, worker/evidence/media/error suites)
## Deterministic Audit Checks
### A. Contract Traceability
- Each high-risk contract maps to at least one focused regression test file.
- Missing mapping is a gap.
### B. Assertion Strength
- Flag tests that only assert status code, non-null, or “no exception” without validating state transitions or persisted outcomes.
- Prefer assertions on domain effects: DB rows, status changes, error categories, evidence writes, or emitted payload shape.
### C. Failure-Path Coverage
- Critical paths must include negative-path tests (timeouts, provider errors, validation failures, cancellation paths, retries).
- Happy-path-only coverage on critical modules is a gap.
### D. Redundancy and Noise
- Detect near-duplicate tests asserting the same behavior at multiple layers with no extra signal.
- Recommend canonical location (unit/integration) and prune overlaps.
### E. Mutation/Change Sensitivity
- Prefer mutation testing for high-risk modules when practical.
- If not run, identify tests likely to survive meaningful code mutations (low sensitivity).
### F. Drift Guards
- Verify config/doc/instruction contracts have deterministic guards and are current.
- Ensure settings/docs synchronization checks remain active.
## Evidence Standards
- Every finding must include concrete file paths and line ranges.
- No speculative claims.
- Distinguish clearly between:
- **Confirmed ineffective tests**
- **Likely weak tests (needs mutation/probe confirmation)**
## Output Format
Produce a Markdown report at `docs/reviews/<YYYY-MM-DD>-test-effectiveness.md`, using today's date.
Like code review reports, it is a dated, non-canonical artifact: `docs/reviews/**` is not part of the
canonical authority set.
```markdown
# Test Effectiveness Audit Report
## 1. Executive Verdict
- Effective / Effective with Conditions / Needs Remediation
- Top risks to confidence
## 2. Contract Coverage Matrix
| Contract | Guarding Tests | Signal Quality | Gap | Action |
| :--- | :--- | :--- | :--- | :--- |
## 3. Weak/Redundant Test Findings
| Finding ID | Location | Why Low-Signal | Risk | Recommendation |
| :--- | :--- | :--- | :--- | :--- |
## 4. Prune/Strengthen Backlog
| Task ID | Goal | Files | Acceptance Criteria | Validation |
| :--- | :--- | :--- | :--- | :--- |
## 5. Confidence Recommendation
- Go / Go with Conditions / No-Go for release confidence
```
## Decision Rules
- Do not recommend deleting a test unless equivalent or stronger coverage is identified.
- Prefer strengthening assertions before adding more tests.
- Prioritize deterministic contract guards over broad snapshot-style tests.
.github/workflows/quality-gate.yml
-50
View File
@@ -1,50 +0,0 @@
name: Quality Gate
# Repository quality gate. Before this workflow existed, ruff, ty, and pytest were
# enforced only by .pre-commit-config.yaml for developers who had run
# `pre-commit install`.
on:
push:
pull_request:
jobs:
gate:
runs-on: ubuntu-latest
steps:
- name: Check out the commit under test
uses: actions/checkout@v4
- name: Install uv
run: |
curl -LsSf https://astral.sh/uv/install.sh | sh
echo "$HOME/.local/bin" >> "$GITHUB_PATH"
- name: Install dependencies from the lockfile
# --locked fails if uv.lock has drifted from pyproject.toml, so a stale
# lockfile is caught here rather than producing an untested dependency set.
run: uv sync --locked
- name: Write placeholder configuration
# Settings requires openrouter_api_key and 115 tests cannot construct
# Settings without it. This is written to .env.production rather than exported
# as an environment variable on purpose: the external tests guard on
# os.getenv("OPENROUTER_API_KEY"), which reads the process environment and
# not the file, so writing the file reproduces the local result exactly -
# the 4 external tests skip instead of running against a fake key and
# failing. Exporting it instead produces 3 failures.
run: echo "OPENROUTER_API_KEY=ci-placeholder-not-a-real-key" > .env.production
- name: Lint and type check
# Runs the hooks defined in .pre-commit-config.yaml instead of repeating
# "ruff check" and "ty check" here. The commands then have one definition,
# so the local and CI gates cannot drift apart.
run: uv run pre-commit run --all-files --show-diff-on-failure
- name: Tests
# Deliberately unfiltered, unlike the "-m 'not external'" form the guidance files
# use for local runs. Tests marked "external" skip themselves when live-service
# credentials are absent, so CI gets the same effective set plus a real run of any
# external test whose credentials are configured. Not drift -- do not "fix" this to
# match the local command without also giving those tests a way to run.
run: uv run pytest
.gitignore
-35
View File
@@ -1,35 +0,0 @@
# Python-generated files
__pycache__/
*.py[oc]
build/
dist/
wheels/
*.egg-info
# Virtual environments
.venv
# Environment secrets
.env
.env.production
# SQLite database
*.db
# All data including db, backups, document images, photos, and logs:
data/*
data/backups/*
data/documents/*
data/logs/*
data/photos/*
data-local/*
# Migration tests
data-migration-test/*
.migration-bundle/*
# Cloudflare tunnel local runtime files
deploy/cloudflared/config.yml
deploy/cloudflared/config.yaml
deploy/cloudflared/credentials.json
.pre-commit-config.yaml
-29
View File
@@ -1,29 +0,0 @@
# Quality gate: `ruff check`, `ruff format --check`, and `ty check`
# are blocking once known `ty` false positives are suppressed inline with rationale.
#
# Both tools are uv-managed dev dependencies and are not on PATH, so each entry must
# go through `uv run`.
repos:
- repo: local
hooks:
- id: ruff
name: ruff check
entry: uv run ruff check
language: system
types_or: [python, pyi]
require_serial: true
- id: ruff-format
name: ruff format check
entry: uv run ruff format --check .
language: system
types_or: [python, pyi]
pass_filenames: false
require_serial: true
- id: ty
name: ty check
entry: uv run ty check
language: system
types_or: [python, pyi]
pass_filenames: false
require_serial: true
verbose: true
.python-version
-1
View File
@@ -1 +0,0 @@
3.12
.vscode/launch.json
Vendored
-23
View File
@@ -1,23 +0,0 @@
{
"version": "0.2.0",
"configurations": [
{
"name": "Python: Debug transcription app",
"type": "debugpy",
"request": "launch",
"module": "debugpy",
"args": [
"-m",
"transcription",
"--host", "127.0.0.1",
"--port", "9999",
"--database.driver", "sqlite"
],
"justMyCode": true,
"console": "integratedTerminal",
"env": {
"PYTHONPATH": "${workspaceFolder}/src"
}
}
]
}
.vscode/settings.json
Vendored
-3
View File
@@ -1,3 +0,0 @@
{
"chat.sessionSync.enabled": true
}
AGENTS.md
-117
View File
@@ -1,117 +0,0 @@
# AGENTS.md
Orientation for AI agents working in this repository. This file is a **router**, not a spec:
it points at canonical authority and flags the traps that are expensive to discover by trial.
Where this file and `docs/*` disagree, `docs/*` wins.
## What This Is
A document transcription system that preserves durable archival records (Documents, Sources,
People) and executes page transcription asynchronously through vision/LLM providers. Its
defining constraint is **evidence**: every machine attempt is recorded append-only with
request/response provenance. Features that would lose, mutate, or obscure that history are
wrong regardless of how convenient they are.
Stack: Python 3.12+ · FastAPI + NiceGUI · SQLModel/SQLAlchemy (SQLite-first, PostgreSQL-
compatible) · Pydantic V2 · asyncio worker · OpenRouter adapter.
## Commands
This is a `uv` project. **Nothing is on `PATH`** — `ruff`, `ty`, and `pytest` all require
`uv run`. Bare invocations fail with command-not-found.
```bash
uv run ruff check . # lint (blocking in pre-commit)
uv run ruff format --check . # format (blocking in pre-commit)
uv run ty check # types (blocking in pre-commit)
uv run pytest -q -m "not external" # default verification run
```
`external` marks tests that hit live services; always exclude it unless explicitly asked.
All four commands are expected to pass clean — there is no tolerated baseline of failures.
If `ty` reports something, fix it or suppress it inline *with a rationale comment*; a bare
`ignore` will not survive review.
## Authority Order
Resolve every question in this order, and stop at the first that answers it:
1. **`docs/*`** — canonical. Start at [`docs/index.md`](docs/index.md), which defines the
reading order. `docs/invariant/*` holds cross-version rules that outlive any release.
2. **`.github/instructions/*.md`** — active steering, auto-attached when you edit matching
paths. Covers services, UI, providers, tests, error handling, and documentation sync.
3. **`.github/skills/*`** — periodic audit procedures (code review, provenance, test
effectiveness).
4. **`tests/`** — deterministic enforcement. A guard test is the ground truth for whatever
rule it encodes.
`.github/agents/` and `.github/prompts/` hold named workflows that are loaded only when
invoked explicitly, so they never override the order above. They are how a review or audit
is *started*, not a source of rules.
`docs/reviews/**` is **not** canonical. Those are dated, opinionated snapshots that were
accurate when written and may since have been fixed, superseded, or found wrong.
## Layout
| Path | Role |
| :--- | :--- |
| `src/transcription/ui/**`, `api/**` | Interface. No direct persistence access. |
| `src/transcription/services/**` | Domain logic and transaction ownership. |
| `src/transcription/db/**` | Models and persistence. |
| `src/transcription/providers/**` | Provider adapters; provider details stop here. |
| `src/transcription/worker.py` | Asyncio worker loop. |
| `tests/` | Includes boundary/contract guards, not just behavior tests. |
## Enforced Boundaries
These are not conventions — a test fails if you break them:
- **No service-to-service imports** (`test_service_boundaries.py`). Compose in the caller.
- **No persistence access from pages/components** (`test_ui_boundaries.py`, allowlist-based).
- **No hand-rolled error notifications in UI** — use the shared error presenter.
- **No stringly-typed status literals** — use the enums (`test_model_contract_guards.py`).
- **Attempt history is append-only** (`test_evidence_provenance.py`).
- **`docs/schema.md` stays field-accurate** with `db/models.py`.
- **Orphans are tracked, not tolerated** — `test_orphan_sweep.py` records each retained
orphan with rationale in `KNOWN_ORPHANS`.
## Traps
Non-obvious things that have already caused real bugs here:
- **`AppError.message` vs `AppError.detail`.** `message` is user/API-facing and must stay
generic — never put exception text or filesystem paths in it. `detail` is internal-only and
is what reaches logs and `ExecutionAttempt.error_detail`. Putting root-cause data in
`message` leaks; removing it from `detail` silently degrades provenance. See
`docs/error_handling.md`.
- **Two competing atomicity invariants in `services/workflows.py`.** Intermediate pages must
commit individually (durability across a long multi-page job); the *final* page must commit
atomically with the terminal job status. Collapsing the batch into one transaction satisfies
the second and destroys the first. Both are guarded — `test_workflows_reliability.py` and
`tests/integration/test_pipeline_atomicity.py`.
- **Shared symbols have more consumers than the obvious one.** Before changing a model field,
exception attribute, or helper return value, grep for every consumer including tests.
Evidence and logging paths frequently read the same fields the UI does.
- **`Tag` is owned by two services, on purpose.** Every other model has exactly one owning
service, so the ownership rule reads as absolute — it isn't. `Tag` is a single table reached
through two `RegistryService[Tag]` facades, `TagRegistry` (documents) and `PersonTagRegistry`
(people), which count usage through `DocumentTag` and `PersonTag` respectively. Changing tag
semantics through one facade silently changes the other. Consolidating them under one service
is not a cleanup; it makes the other side a cross-aggregate writer.
- **Import style:** ruff `isort` runs with `force-single-line = true`. One import per line.
- **Latent defects have ordering constraints.** Some code is unreachable only because of a
current setting or single-instance deployment. Fix it *before* the change that unblocks it,
not after.
## Change Protocol
- **Write the failing test first** for behavioral fixes, and confirm it actually fails for the
reason you think. Several bugs here were subtle enough that a test written afterward would
have passed against the broken code.
- **Update docs in the same change** when you alter a contract, behavior, or scope — see
`.github/instructions/documentation-sync.instructions.md`.
- **Do not commit unless asked.** Making a requested change is not consent to commit it.
- **Do not push or open PRs on your own initiative.**
- **Scope discipline:** fix what was asked plus what your change genuinely breaks. Pre-existing
unrelated issues are a separate conversation.
Dockerfile
-55
View File
@@ -1,55 +0,0 @@
FROM python:3.12-slim AS builder
ENV PYTHONDONTWRITEBYTECODE=1 \
PYTHONUNBUFFERED=1 \
UV_LINK_MODE=copy
WORKDIR /app
COPY --from=ghcr.io/astral-sh/uv:0.5.24 /uv /uvx /bin/
COPY pyproject.toml uv.lock README.md ./
RUN uv sync --frozen --no-dev --no-install-project
COPY src ./src
COPY prompts ./prompts
COPY tools ./tools
COPY deploy ./deploy
RUN uv sync --frozen --no-dev
FROM python:3.12-slim AS runtime
ENV PYTHONDONTWRITEBYTECODE=1 \
PYTHONUNBUFFERED=1 \
PATH="/app/.venv/bin:$PATH" \
PYTHONPATH="/app/src" \
UPLOAD_DIR="/app/uploads" \
PROMPT_DIR="/app/prompts"
WORKDIR /app
RUN apt-get update \
&& apt-get install -y --no-install-recommends postgresql-client \
&& rm -rf /var/lib/apt/lists/*
RUN groupadd --system --gid 1001 appgroup \
&& useradd --system --uid 1001 --gid appgroup --create-home appuser
COPY --from=builder /app/.venv /app/.venv
COPY --from=builder /app/src /app/src
COPY --from=builder /app/prompts /app/prompts
COPY --from=builder /app/tools /app/tools
COPY --from=builder /app/deploy /app/deploy
RUN mkdir -p /app/uploads /app/data \
&& chown -R appuser:appgroup /app
USER appuser
EXPOSE 8000
HEALTHCHECK --interval=30s --timeout=3s --start-period=3s --retries=3 \
CMD python -c "import urllib.request; urllib.request.urlopen('http://localhost:8000/healthz')"
CMD ["uvicorn", "transcription.app:create_app", "--factory", "--host", "0.0.0.0", "--port", "8000", "--proxy-headers"]
LICENSE
+232
View File
@@ -0,0 +1,232 @@
GNU GENERAL PUBLIC LICENSE
Version 3, 29 June 2007
Copyright © 2007 Free Software Foundation, Inc. <https://fsf.org/>
Everyone is permitted to copy and distribute verbatim copies of this license document, but changing it is not allowed.
Preamble
The GNU General Public License is a free, copyleft license for software and other kinds of works.
The licenses for most software and other practical works are designed to take away your freedom to share and change the works. By contrast, the GNU General Public License is intended to guarantee your freedom to share and change all versions of a program--to make sure it remains free software for all its users. We, the Free Software Foundation, use the GNU General Public License for most of our software; it applies also to any other work released this way by its authors. You can apply it to your programs, too.
When we speak of free software, we are referring to freedom, not price. Our General Public Licenses are designed to make sure that you have the freedom to distribute copies of free software (and charge for them if you wish), that you receive source code or can get it if you want it, that you can change the software or use pieces of it in new free programs, and that you know you can do these things.
To protect your rights, we need to prevent others from denying you these rights or asking you to surrender the rights. Therefore, you have certain responsibilities if you distribute copies of the software, or if you modify it: responsibilities to respect the freedom of others.
For example, if you distribute copies of such a program, whether gratis or for a fee, you must pass on to the recipients the same freedoms that you received. You must make sure that they, too, receive or can get the source code. And you must show them these terms so they know their rights.
Developers that use the GNU GPL protect your rights with two steps: (1) assert copyright on the software, and (2) offer you this License giving you legal permission to copy, distribute and/or modify it.
For the developers' and authors' protection, the GPL clearly explains that there is no warranty for this free software. For both users' and authors' sake, the GPL requires that modified versions be marked as changed, so that their problems will not be attributed erroneously to authors of previous versions.
Some devices are designed to deny users access to install or run modified versions of the software inside them, although the manufacturer can do so. This is fundamentally incompatible with the aim of protecting users' freedom to change the software. The systematic pattern of such abuse occurs in the area of products for individuals to use, which is precisely where it is most unacceptable. Therefore, we have designed this version of the GPL to prohibit the practice for those products. If such problems arise substantially in other domains, we stand ready to extend this provision to those domains in future versions of the GPL, as needed to protect the freedom of users.
Finally, every program is threatened constantly by software patents. States should not allow patents to restrict development and use of software on general-purpose computers, but in those that do, we wish to avoid the special danger that patents applied to a free program could make it effectively proprietary. To prevent this, the GPL assures that patents cannot be used to render the program non-free.
The precise terms and conditions for copying, distribution and modification follow.
TERMS AND CONDITIONS
0. Definitions.
“This License” refers to version 3 of the GNU General Public License.
“Copyright” also means copyright-like laws that apply to other kinds of works, such as semiconductor masks.
“The Program” refers to any copyrightable work licensed under this License. Each licensee is addressed as “you”. “Licensees” and “recipients” may be individuals or organizations.
To “modify” a work means to copy from or adapt all or part of the work in a fashion requiring copyright permission, other than the making of an exact copy. The resulting work is called a “modified version” of the earlier work or a work “based on” the earlier work.
A “covered work” means either the unmodified Program or a work based on the Program.
To “propagate” a work means to do anything with it that, without permission, would make you directly or secondarily liable for infringement under applicable copyright law, except executing it on a computer or modifying a private copy. Propagation includes copying, distribution (with or without modification), making available to the public, and in some countries other activities as well.
To “convey” a work means any kind of propagation that enables other parties to make or receive copies. Mere interaction with a user through a computer network, with no transfer of a copy, is not conveying.
An interactive user interface displays “Appropriate Legal Notices” to the extent that it includes a convenient and prominently visible feature that (1) displays an appropriate copyright notice, and (2) tells the user that there is no warranty for the work (except to the extent that warranties are provided), that licensees may convey the work under this License, and how to view a copy of this License. If the interface presents a list of user commands or options, such as a menu, a prominent item in the list meets this criterion.
1. Source Code.
The “source code” for a work means the preferred form of the work for making modifications to it. “Object code” means any non-source form of a work.
A “Standard Interface” means an interface that either is an official standard defined by a recognized standards body, or, in the case of interfaces specified for a particular programming language, one that is widely used among developers working in that language.
The “System Libraries” of an executable work include anything, other than the work as a whole, that (a) is included in the normal form of packaging a Major Component, but which is not part of that Major Component, and (b) serves only to enable use of the work with that Major Component, or to implement a Standard Interface for which an implementation is available to the public in source code form. A “Major Component”, in this context, means a major essential component (kernel, window system, and so on) of the specific operating system (if any) on which the executable work runs, or a compiler used to produce the work, or an object code interpreter used to run it.
The “Corresponding Source” for a work in object code form means all the source code needed to generate, install, and (for an executable work) run the object code and to modify the work, including scripts to control those activities. However, it does not include the work's System Libraries, or general-purpose tools or generally available free programs which are used unmodified in performing those activities but which are not part of the work. For example, Corresponding Source includes interface definition files associated with source files for the work, and the source code for shared libraries and dynamically linked subprograms that the work is specifically designed to require, such as by intimate data communication or control flow between those subprograms and other parts of the work.
The Corresponding Source need not include anything that users can regenerate automatically from other parts of the Corresponding Source.
The Corresponding Source for a work in source code form is that same work.
2. Basic Permissions.
All rights granted under this License are granted for the term of copyright on the Program, and are irrevocable provided the stated conditions are met. This License explicitly affirms your unlimited permission to run the unmodified Program. The output from running a covered work is covered by this License only if the output, given its content, constitutes a covered work. This License acknowledges your rights of fair use or other equivalent, as provided by copyright law.
You may make, run and propagate covered works that you do not convey, without conditions so long as your license otherwise remains in force. You may convey covered works to others for the sole purpose of having them make modifications exclusively for you, or provide you with facilities for running those works, provided that you comply with the terms of this License in conveying all material for which you do not control copyright. Those thus making or running the covered works for you must do so exclusively on your behalf, under your direction and control, on terms that prohibit them from making any copies of your copyrighted material outside their relationship with you.
Conveying under any other circumstances is permitted solely under the conditions stated below. Sublicensing is not allowed; section 10 makes it unnecessary.
3. Protecting Users' Legal Rights From Anti-Circumvention Law.
No covered work shall be deemed part of an effective technological measure under any applicable law fulfilling obligations under article 11 of the WIPO copyright treaty adopted on 20 December 1996, or similar laws prohibiting or restricting circumvention of such measures.
When you convey a covered work, you waive any legal power to forbid circumvention of technological measures to the extent such circumvention is effected by exercising rights under this License with respect to the covered work, and you disclaim any intention to limit operation or modification of the work as a means of enforcing, against the work's users, your or third parties' legal rights to forbid circumvention of technological measures.
4. Conveying Verbatim Copies.
You may convey verbatim copies of the Program's source code as you receive it, in any medium, provided that you conspicuously and appropriately publish on each copy an appropriate copyright notice; keep intact all notices stating that this License and any non-permissive terms added in accord with section 7 apply to the code; keep intact all notices of the absence of any warranty; and give all recipients a copy of this License along with the Program.
You may charge any price or no price for each copy that you convey, and you may offer support or warranty protection for a fee.
5. Conveying Modified Source Versions.
You may convey a work based on the Program, or the modifications to produce it from the Program, in the form of source code under the terms of section 4, provided that you also meet all of these conditions:
a) The work must carry prominent notices stating that you modified it, and giving a relevant date.
b) The work must carry prominent notices stating that it is released under this License and any conditions added under section 7. This requirement modifies the requirement in section 4 to “keep intact all notices”.
c) You must license the entire work, as a whole, under this License to anyone who comes into possession of a copy. This License will therefore apply, along with any applicable section 7 additional terms, to the whole of the work, and all its parts, regardless of how they are packaged. This License gives no permission to license the work in any other way, but it does not invalidate such permission if you have separately received it.
d) If the work has interactive user interfaces, each must display Appropriate Legal Notices; however, if the Program has interactive interfaces that do not display Appropriate Legal Notices, your work need not make them do so.
A compilation of a covered work with other separate and independent works, which are not by their nature extensions of the covered work, and which are not combined with it such as to form a larger program, in or on a volume of a storage or distribution medium, is called an “aggregate” if the compilation and its resulting copyright are not used to limit the access or legal rights of the compilation's users beyond what the individual works permit. Inclusion of a covered work in an aggregate does not cause this License to apply to the other parts of the aggregate.
6. Conveying Non-Source Forms.
You may convey a covered work in object code form under the terms of sections 4 and 5, provided that you also convey the machine-readable Corresponding Source under the terms of this License, in one of these ways:
a) Convey the object code in, or embodied in, a physical product (including a physical distribution medium), accompanied by the Corresponding Source fixed on a durable physical medium customarily used for software interchange.
b) Convey the object code in, or embodied in, a physical product (including a physical distribution medium), accompanied by a written offer, valid for at least three years and valid for as long as you offer spare parts or customer support for that product model, to give anyone who possesses the object code either (1) a copy of the Corresponding Source for all the software in the product that is covered by this License, on a durable physical medium customarily used for software interchange, for a price no more than your reasonable cost of physically performing this conveying of source, or (2) access to copy the Corresponding Source from a network server at no charge.
c) Convey individual copies of the object code with a copy of the written offer to provide the Corresponding Source. This alternative is allowed only occasionally and noncommercially, and only if you received the object code with such an offer, in accord with subsection 6b.
d) Convey the object code by offering access from a designated place (gratis or for a charge), and offer equivalent access to the Corresponding Source in the same way through the same place at no further charge. You need not require recipients to copy the Corresponding Source along with the object code. If the place to copy the object code is a network server, the Corresponding Source may be on a different server (operated by you or a third party) that supports equivalent copying facilities, provided you maintain clear directions next to the object code saying where to find the Corresponding Source. Regardless of what server hosts the Corresponding Source, you remain obligated to ensure that it is available for as long as needed to satisfy these requirements.
e) Convey the object code using peer-to-peer transmission, provided you inform other peers where the object code and Corresponding Source of the work are being offered to the general public at no charge under subsection 6d.
A separable portion of the object code, whose source code is excluded from the Corresponding Source as a System Library, need not be included in conveying the object code work.
A “User Product” is either (1) a “consumer product”, which means any tangible personal property which is normally used for personal, family, or household purposes, or (2) anything designed or sold for incorporation into a dwelling. In determining whether a product is a consumer product, doubtful cases shall be resolved in favor of coverage. For a particular product received by a particular user, “normally used” refers to a typical or common use of that class of product, regardless of the status of the particular user or of the way in which the particular user actually uses, or expects or is expected to use, the product. A product is a consumer product regardless of whether the product has substantial commercial, industrial or non-consumer uses, unless such uses represent the only significant mode of use of the product.
“Installation Information” for a User Product means any methods, procedures, authorization keys, or other information required to install and execute modified versions of a covered work in that User Product from a modified version of its Corresponding Source. The information must suffice to ensure that the continued functioning of the modified object code is in no case prevented or interfered with solely because modification has been made.
If you convey an object code work under this section in, or with, or specifically for use in, a User Product, and the conveying occurs as part of a transaction in which the right of possession and use of the User Product is transferred to the recipient in perpetuity or for a fixed term (regardless of how the transaction is characterized), the Corresponding Source conveyed under this section must be accompanied by the Installation Information. But this requirement does not apply if neither you nor any third party retains the ability to install modified object code on the User Product (for example, the work has been installed in ROM).
The requirement to provide Installation Information does not include a requirement to continue to provide support service, warranty, or updates for a work that has been modified or installed by the recipient, or for the User Product in which it has been modified or installed. Access to a network may be denied when the modification itself materially and adversely affects the operation of the network or violates the rules and protocols for communication across the network.
Corresponding Source conveyed, and Installation Information provided, in accord with this section must be in a format that is publicly documented (and with an implementation available to the public in source code form), and must require no special password or key for unpacking, reading or copying.
7. Additional Terms.
“Additional permissions” are terms that supplement the terms of this License by making exceptions from one or more of its conditions. Additional permissions that are applicable to the entire Program shall be treated as though they were included in this License, to the extent that they are valid under applicable law. If additional permissions apply only to part of the Program, that part may be used separately under those permissions, but the entire Program remains governed by this License without regard to the additional permissions.
When you convey a copy of a covered work, you may at your option remove any additional permissions from that copy, or from any part of it. (Additional permissions may be written to require their own removal in certain cases when you modify the work.) You may place additional permissions on material, added by you to a covered work, for which you have or can give appropriate copyright permission.
Notwithstanding any other provision of this License, for material you add to a covered work, you may (if authorized by the copyright holders of that material) supplement the terms of this License with terms:
a) Disclaiming warranty or limiting liability differently from the terms of sections 15 and 16 of this License; or
b) Requiring preservation of specified reasonable legal notices or author attributions in that material or in the Appropriate Legal Notices displayed by works containing it; or
c) Prohibiting misrepresentation of the origin of that material, or requiring that modified versions of such material be marked in reasonable ways as different from the original version; or
d) Limiting the use for publicity purposes of names of licensors or authors of the material; or
e) Declining to grant rights under trademark law for use of some trade names, trademarks, or service marks; or
f) Requiring indemnification of licensors and authors of that material by anyone who conveys the material (or modified versions of it) with contractual assumptions of liability to the recipient, for any liability that these contractual assumptions directly impose on those licensors and authors.
All other non-permissive additional terms are considered “further restrictions” within the meaning of section 10. If the Program as you received it, or any part of it, contains a notice stating that it is governed by this License along with a term that is a further restriction, you may remove that term. If a license document contains a further restriction but permits relicensing or conveying under this License, you may add to a covered work material governed by the terms of that license document, provided that the further restriction does not survive such relicensing or conveying.
If you add terms to a covered work in accord with this section, you must place, in the relevant source files, a statement of the additional terms that apply to those files, or a notice indicating where to find the applicable terms.
Additional terms, permissive or non-permissive, may be stated in the form of a separately written license, or stated as exceptions; the above requirements apply either way.
8. Termination.
You may not propagate or modify a covered work except as expressly provided under this License. Any attempt otherwise to propagate or modify it is void, and will automatically terminate your rights under this License (including any patent licenses granted under the third paragraph of section 11).
However, if you cease all violation of this License, then your license from a particular copyright holder is reinstated (a) provisionally, unless and until the copyright holder explicitly and finally terminates your license, and (b) permanently, if the copyright holder fails to notify you of the violation by some reasonable means prior to 60 days after the cessation.
Moreover, your license from a particular copyright holder is reinstated permanently if the copyright holder notifies you of the violation by some reasonable means, this is the first time you have received notice of violation of this License (for any work) from that copyright holder, and you cure the violation prior to 30 days after your receipt of the notice.
Termination of your rights under this section does not terminate the licenses of parties who have received copies or rights from you under this License. If your rights have been terminated and not permanently reinstated, you do not qualify to receive new licenses for the same material under section 10.
9. Acceptance Not Required for Having Copies.
You are not required to accept this License in order to receive or run a copy of the Program. Ancillary propagation of a covered work occurring solely as a consequence of using peer-to-peer transmission to receive a copy likewise does not require acceptance. However, nothing other than this License grants you permission to propagate or modify any covered work. These actions infringe copyright if you do not accept this License. Therefore, by modifying or propagating a covered work, you indicate your acceptance of this License to do so.
10. Automatic Licensing of Downstream Recipients.
Each time you convey a covered work, the recipient automatically receives a license from the original licensors, to run, modify and propagate that work, subject to this License. You are not responsible for enforcing compliance by third parties with this License.
An “entity transaction” is a transaction transferring control of an organization, or substantially all assets of one, or subdividing an organization, or merging organizations. If propagation of a covered work results from an entity transaction, each party to that transaction who receives a copy of the work also receives whatever licenses to the work the party's predecessor in interest had or could give under the previous paragraph, plus a right to possession of the Corresponding Source of the work from the predecessor in interest, if the predecessor has it or can get it with reasonable efforts.
You may not impose any further restrictions on the exercise of the rights granted or affirmed under this License. For example, you may not impose a license fee, royalty, or other charge for exercise of rights granted under this License, and you may not initiate litigation (including a cross-claim or counterclaim in a lawsuit) alleging that any patent claim is infringed by making, using, selling, offering for sale, or importing the Program or any portion of it.
11. Patents.
A “contributor” is a copyright holder who authorizes use under this License of the Program or a work on which the Program is based. The work thus licensed is called the contributor's “contributor version”.
A contributor's “essential patent claims” are all patent claims owned or controlled by the contributor, whether already acquired or hereafter acquired, that would be infringed by some manner, permitted by this License, of making, using, or selling its contributor version, but do not include claims that would be infringed only as a consequence of further modification of the contributor version. For purposes of this definition, “control” includes the right to grant patent sublicenses in a manner consistent with the requirements of this License.
Each contributor grants you a non-exclusive, worldwide, royalty-free patent license under the contributor's essential patent claims, to make, use, sell, offer for sale, import and otherwise run, modify and propagate the contents of its contributor version.
In the following three paragraphs, a “patent license” is any express agreement or commitment, however denominated, not to enforce a patent (such as an express permission to practice a patent or covenant not to sue for patent infringement). To “grant” such a patent license to a party means to make such an agreement or commitment not to enforce a patent against the party.
If you convey a covered work, knowingly relying on a patent license, and the Corresponding Source of the work is not available for anyone to copy, free of charge and under the terms of this License, through a publicly available network server or other readily accessible means, then you must either (1) cause the Corresponding Source to be so available, or (2) arrange to deprive yourself of the benefit of the patent license for this particular work, or (3) arrange, in a manner consistent with the requirements of this License, to extend the patent license to downstream recipients. “Knowingly relying” means you have actual knowledge that, but for the patent license, your conveying the covered work in a country, or your recipient's use of the covered work in a country, would infringe one or more identifiable patents in that country that you have reason to believe are valid.
If, pursuant to or in connection with a single transaction or arrangement, you convey, or propagate by procuring conveyance of, a covered work, and grant a patent license to some of the parties receiving the covered work authorizing them to use, propagate, modify or convey a specific copy of the covered work, then the patent license you grant is automatically extended to all recipients of the covered work and works based on it.
A patent license is “discriminatory” if it does not include within the scope of its coverage, prohibits the exercise of, or is conditioned on the non-exercise of one or more of the rights that are specifically granted under this License. You may not convey a covered work if you are a party to an arrangement with a third party that is in the business of distributing software, under which you make payment to the third party based on the extent of your activity of conveying the work, and under which the third party grants, to any of the parties who would receive the covered work from you, a discriminatory patent license (a) in connection with copies of the covered work conveyed by you (or copies made from those copies), or (b) primarily for and in connection with specific products or compilations that contain the covered work, unless you entered into that arrangement, or that patent license was granted, prior to 28 March 2007.
Nothing in this License shall be construed as excluding or limiting any implied license or other defenses to infringement that may otherwise be available to you under applicable patent law.
12. No Surrender of Others' Freedom.
If conditions are imposed on you (whether by court order, agreement or otherwise) that contradict the conditions of this License, they do not excuse you from the conditions of this License. If you cannot convey a covered work so as to satisfy simultaneously your obligations under this License and any other pertinent obligations, then as a consequence you may not convey it at all. For example, if you agree to terms that obligate you to collect a royalty for further conveying from those to whom you convey the Program, the only way you could satisfy both those terms and this License would be to refrain entirely from conveying the Program.
13. Use with the GNU Affero General Public License.
Notwithstanding any other provision of this License, you have permission to link or combine any covered work with a work licensed under version 3 of the GNU Affero General Public License into a single combined work, and to convey the resulting work. The terms of this License will continue to apply to the part which is the covered work, but the special requirements of the GNU Affero General Public License, section 13, concerning interaction through a network will apply to the combination as such.
14. Revised Versions of this License.
The Free Software Foundation may publish revised and/or new versions of the GNU General Public License from time to time. Such new versions will be similar in spirit to the present version, but may differ in detail to address new problems or concerns.
Each version is given a distinguishing version number. If the Program specifies that a certain numbered version of the GNU General Public License “or any later version” applies to it, you have the option of following the terms and conditions either of that numbered version or of any later version published by the Free Software Foundation. If the Program does not specify a version number of the GNU General Public License, you may choose any version ever published by the Free Software Foundation.
If the Program specifies that a proxy can decide which future versions of the GNU General Public License can be used, that proxy's public statement of acceptance of a version permanently authorizes you to choose that version for the Program.
Later license versions may give you additional or different permissions. However, no additional obligations are imposed on any author or copyright holder as a result of your choosing to follow a later version.
15. Disclaimer of Warranty.
THERE IS NO WARRANTY FOR THE PROGRAM, TO THE EXTENT PERMITTED BY APPLICABLE LAW. EXCEPT WHEN OTHERWISE STATED IN WRITING THE COPYRIGHT HOLDERS AND/OR OTHER PARTIES PROVIDE THE PROGRAM “AS IS” WITHOUT WARRANTY OF ANY KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE. THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE OF THE PROGRAM IS WITH YOU. SHOULD THE PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF ALL NECESSARY SERVICING, REPAIR OR CORRECTION.
16. Limitation of Liability.
IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MODIFIES AND/OR CONVEYS THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, INCLUDING ANY GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING OUT OF THE USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED TO LOSS OF DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY YOU OR THIRD PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER PROGRAMS), EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE POSSIBILITY OF SUCH DAMAGES.
17. Interpretation of Sections 15 and 16.
If the disclaimer of warranty and limitation of liability provided above cannot be given local legal effect according to their terms, reviewing courts shall apply local law that most closely approximates an absolute waiver of all civil liability in connection with the Program, unless a warranty or assumption of liability accompanies a copy of the Program in return for a fee.
END OF TERMS AND CONDITIONS
How to Apply These Terms to Your New Programs
If you develop a new program, and you want it to be of the greatest possible use to the public, the best way to achieve this is to make it free software which everyone can redistribute and change under these terms.
To do so, attach the following notices to the program. It is safest to attach them to the start of each source file to most effectively state the exclusion of warranty; and each file should have at least the “copyright” line and a pointer to where the full notice is found.
python-template
Copyright (C) 2026 john
This program is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation, either version 3 of the License, or (at your option) any later version.
This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details.
You should have received a copy of the GNU General Public License along with this program. If not, see <https://www.gnu.org/licenses/>.
Also add information on how to contact you by electronic and paper mail.
If the program does terminal interaction, make it output a short notice like this when it starts in an interactive mode:
python-template Copyright (C) 2026 john
This program comes with ABSOLUTELY NO WARRANTY; for details type `show w'.
This is free software, and you are welcome to redistribute it under certain conditions; type `show c' for details.
The hypothetical commands `show w' and `show c' should show the appropriate parts of the General Public License. Of course, your program's commands might be different; for a GUI interface, you would use an “about box”.
You should also get your employer (if you work as a programmer) or school, if any, to sign a “copyright disclaimer” for the program, if necessary. For more information on this, and how to apply and follow the GNU GPL, see <https://www.gnu.org/licenses/>.
The GNU General Public License does not permit incorporating your program into proprietary programs. If your program is a subroutine library, you may consider it more useful to permit linking proprietary applications with the library. If this is what you want to do, use the GNU Lesser General Public License instead of this License. But first, please read <https://www.gnu.org/philosophy/why-not-lgpl.html>.
README.md
+1 -242
View File
@@ -1,242 +1 @@
# Transcription
Historical document transcription system for family-history documents.
The app lets you upload a document image/PDF, queues a background transcription job, and then shows job status and results in a web UI.
## What the app does
- Upload document files (`.jpg`, `.jpeg`, `.png`, `.tif`, `.tiff`, `.pdf`)
- Persist document + job records in SQLite
- Process jobs in a background worker (`queued -> processing -> transcribed/failed`)
- Store transcript text (or failure detail)
- Show status and results in the NiceGUI interface
## Quick start
### 1) Install dependencies
```bash
uv sync
```
### 2) Configure environment
Create a `.env.production` file in the project root with the required OpenRouter API key:
```env
OPENROUTER_API_KEY=your_openrouter_api_key
```
Settings are read from CLI arguments first, then environment variables, then `.env.production`, then the defaults below.
### Configuration Source Precedence
When the same setting is provided in multiple places, the value is chosen in this order (highest priority first):
1. CLI arguments (for example `--port 8000`)
2. Settings constructor arguments (used mainly in tests)
3. Environment variables
4. `.env.production` file values
5. Model defaults in code
Practical examples:
- `--port 8000` overrides both `PORT=8000` in the shell and `PORT=7000` in `.env.production`.
- `DATABASE__PATH=prod.db` in the shell overrides `DATABASE__PATH=dev.db` in `.env.production`.
#### Server and runtime
| Environment variable | Default | Description |
| --- | --- | --- |
| `HOST` | `0.0.0.0` | Address on which the server listens. |
| `PORT` | `8000` | Server port. |
| `LOG_LEVEL` | `info` | Uvicorn and application log level. |
| `RELOAD` | `false` | Restart the development server when source files change. |
| `ENVIRONMENT` | `development` | Runtime environment: `development`, `test`, or `production`. |
| `RUN_EMBEDDED_WORKER` | `true` | Run worker loop inside web app process. Set `false` when using a dedicated worker service. |
#### Provider
| Environment variable | Default | Description |
| --- | --- | --- |
| `PROVIDER` | `openrouter` | Transcription provider. |
| `OPENROUTER_API_KEY` | Required | OpenRouter API key. |
| `PROVIDER_MODEL` | Provider default | Optional model override. |
| `OPENROUTER_HTTP_REFERER` | Unset | Optional OpenRouter attribution URL. |
| `OPENROUTER_APP_TITLE` | Unset | Optional OpenRouter attribution title. |
#### Database and files
Use nested env vars for database settings (recommended):
```env
DATABASE__DRIVER=sqlite
DATABASE__PATH=app.db
# BOOTSTRAP_SCHEMA_ON_STARTUP=true
SQLITE_CHECK_SAME_THREAD=false
UPLOAD_DIR=./uploads
PROMPT_DIR=./prompts
DEFAULT_PROMPT_NAME=transcribe_document.md
# TRANSCRIPTION_TEMPERATURE=0.2 # range: 0.0-2.0
# TRANSCRIPTION_TOP_P=0.9 # range: 0.0-1.0
```
For PostgreSQL:
```env
DATABASE__DRIVER=postgres
DATABASE__HOST=localhost
DATABASE__PORT=5432
DATABASE__DATABASE=transcription
DATABASE__USER=postgres
DATABASE__PASSWORD=change-me
```
This uses Pydantic nested settings (`env_nested_delimiter='__'`) and avoids JSON blobs in env files. A top-level `DATABASE={...}` JSON value is still supported as a fallback, and nested keys such as `DATABASE__PATH` take precedence over conflicting JSON keys.
`BOOTSTRAP_SCHEMA_ON_STARTUP` creates missing tables when the app starts. When unset, it is enabled in `development` and `test`, and disabled in `production`; set it explicitly to override that policy. `SQLITE_CHECK_SAME_THREAD` defaults to `false`.
#### Worker
```env
WORKER_MAX_RETRIES=0
WORKER_RETRY_BACKOFF_SECONDS=0
WORKER_PROVIDER_TIMEOUT_SECONDS=20
WORKER_MIN_TRANSCRIPTION_CHARS=0
WORKER_MIN_TRANSCRIPTION_LINES=0
WORKER_FAIL_ON_FINISH_REASON_LENGTH=false
```
### 3) Run the app
```bash
uv run python -m transcription --port 8000 --reload --database.driver sqlite --bootstrap-schema-on-startup
```
This starts the development server with SQLite, creates missing tables, and enables automatic reload. Run `uv run python -m transcription --help` for all CLI options; CLI names use kebab case and nested database options use dot notation, such as `--database.path ./data/transcription.db`.
### 4) Open in browser
- GUI: [http://localhost:8000/ui](http://localhost:8000/ui)
- Health check: [http://localhost:8000/healthz](http://localhost:8000/healthz)
Replace `localhost` with the server's hostname or IP address when connecting from another machine.
## Production stack (Phase 1)
Use the production compose profile for split app/worker deployment with PostgreSQL and Cloudflare Tunnel:
```bash
copy .env.production.example .env.production
docker compose -f docker-compose.production.yml up -d --build
```
Services:
- `app`: FastAPI + NiceGUI runtime (`RUN_EMBEDDED_WORKER=false`)
- `worker`: standalone queue processor (`python -m transcription.worker_service`)
- `postgres`: primary datastore
- `cloudflared`: tunnel client using mounted ingress config + `CLOUDFLARE_TUNNEL_TOKEN`
Operational defaults in the production compose file:
- worker healthcheck is disabled (the worker process has no HTTP `/healthz` endpoint)
- cloudflared is pinned to HTTP/2 with explicit DNS resolvers (`1.1.1.1`, `1.0.0.1`) for restricted LXC/container DNS environments
Cloudflare setup files:
1. `copy deploy\cloudflared\config.yml.example deploy\cloudflared\config.yml`
2. set `CLOUDFLARE_TUNNEL_TOKEN` in `.env.production`
3. update ingress hostnames in `deploy\cloudflared\config.yml`
## How to navigate the GUI
- **Upload page** (`/ui`)
- Select a supported file to upload.
- The app creates a queued transcription job.
- Use the **View jobs** link to inspect progress.
- **Jobs page** (`/ui/jobs`)
- See all jobs and their status.
- Use **Refresh** to reload current states.
- Open a specific job to see details.
- **Job detail page** (`/ui/jobs/{job_id}`)
- Shows job metadata and status.
- Displays transcript text when successful.
- Displays failure detail when transcription fails.
## Prompt artifacts
Prompt files are stored directly in `PROMPT_DIR` (default: `./prompts`). `DEFAULT_PROMPT_NAME` must be a filename,
not a path. Each job snapshots the validated prompt text, SHA-256 hash, and sampling values for reproducibility.
The canonical MVP prompt is:
- `prompts/transcribe_document.md`
## Database migration workflow
Schema upgrades use an explicit export/import rebuild flow (no runtime legacy write compatibility).
See `docs/data_migration.md` for commands and cutover steps.
## Backup and restore workflow
Production backup/restore (PostgreSQL + uploads + deployment config) steps are documented in `docs/backup_restore.md`.
## Destructive test procedure (with data backup)
AI execution policy: before the first unit-test run in a test/fix cycle, create one backup of `./data`. Reuse that same backup for every subsequent test run in the cycle. After tests succeed, always pause and ask whether to restore now.
Use the cross-platform Python wrapper below whenever an AI agent runs tests against this repository.
1. Create one backup of `./data` and mark it as the active test-cycle backup.
2. Run your test command.
3. On failure, fix the errors and run the wrapper again; it reuses the active backup and never backs up post-test data.
4. On success, always prompt whether to restore now (do not auto-restore unless explicitly approved).
5. Close the cycle only by restoring the active backup or explicitly accepting the current data.
Preflight behavior:
- Backup preflight is warning-only when `data/transcription.db` appears in use.
- Restore preflight is blocking: the script prompts you to close conflicting applications, then type `retry` to re-check or `cancel` to skip restore.
### Run with confirmation-gated restore (default)
```bash
uv run python tools/run_destructive_tests.py -- pytest tests/services/test_job_service.py tests/ui/test_jobs_page.py
```
After tests pass, the script asks whether to restore backup immediately.
This is the required default mode for AI-assisted test runs because it gives time to verify and accept code changes before any restoration happens.
### Run with automatic restore (non-interactive)
```bash
uv run python tools/run_destructive_tests.py --auto-restore -- pytest
```
### Run without terminal prompt (decide restore later)
```bash
uv run python tools/run_destructive_tests.py --skip-restore-prompt -- pytest
```
This keeps both the current post-test state and the backup, so restore can be decided explicitly later.
Repeated wrapper invocations reuse the backup recorded in `.test-backups/.active-backup`. If that backup is missing, the wrapper stops rather than creating a replacement from potentially destructive post-test data.
### Restore later from a saved backup
```bash
uv run python tools/run_destructive_tests.py --restore-from data-backup-YYYYMMDD-HHMMSS
```
To keep the current data and close the active cycle without restoring:
```bash
uv run python tools/run_destructive_tests.py --accept-current-data
```
Backups are stored in `.test-backups/` and ignored by git.
# Python Template
deploy/backup/create_postgres_backup.sh
-69
View File
@@ -1,69 +0,0 @@
#!/usr/bin/env sh
set -eu
BACKUP_DIR="${BACKUP_DIR:-./data/backups}"
RETENTION_DAYS="${BACKUP_RETENTION_DAYS:-14}"
UPLOAD_DIR="${UPLOAD_DIR:-/app/uploads}"
PROMPT_DIR="${PROMPT_DIR:-/app/prompts}"
DATABASE_DRIVER="${DATABASE__DRIVER:-postgres}"
DATABASE_HOST="${DATABASE__HOST:-postgres}"
DATABASE_PORT="${DATABASE__PORT:-5432}"
DATABASE_NAME="${DATABASE__DATABASE:-}"
DATABASE_USER="${DATABASE__USER:-}"
DATABASE_PASSWORD="${DATABASE__PASSWORD:-}"
timestamp="$(date -u +%Y%m%d-%H%M%S)"
postgres_file="postgres-${timestamp}.dump"
manifest_file="backup-${timestamp}.manifest"
mkdir -p "${BACKUP_DIR}"
if [ "${DATABASE_DRIVER}" != "postgres" ]; then
echo "create_postgres_backup.sh requires DATABASE__DRIVER=postgres." >&2
exit 1
fi
if [ -z "${DATABASE_NAME}" ] || [ -z "${DATABASE_USER}" ] || [ -z "${DATABASE_PASSWORD}" ]; then
echo "DATABASE__DATABASE, DATABASE__USER, and DATABASE__PASSWORD must be set." >&2
exit 1
fi
if ! command -v pg_dump >/dev/null 2>&1; then
echo "pg_dump is not installed in this environment." >&2
exit 1
fi
PGPASSWORD="${DATABASE_PASSWORD}" pg_dump \
-h "${DATABASE_HOST}" \
-p "${DATABASE_PORT}" \
-U "${DATABASE_USER}" \
-d "${DATABASE_NAME}" \
-Fc \
> "${BACKUP_DIR}/${postgres_file}"
cat > "${BACKUP_DIR}/${manifest_file}" <<EOF
created_at_utc=${timestamp}
postgres_dump=${postgres_file}
backup_dir=${BACKUP_DIR}
uploads_backup_dir=${BACKUP_DIR}/uploads
prompts_backup_dir=${BACKUP_DIR}/prompts
EOF
find "${BACKUP_DIR}" -type f \( \
-name 'postgres-*.dump' -o \
-name 'backup-*.manifest' \
\) -mtime +"${RETENTION_DAYS}" -delete
uploads_backup_dir="${BACKUP_DIR}/uploads"
mkdir -p "${uploads_backup_dir}"
if [ -d "${UPLOAD_DIR}" ]; then
cp -an "${UPLOAD_DIR}/." "${uploads_backup_dir}/"
fi
prompts_backup_dir="${BACKUP_DIR}/prompts"
mkdir -p "${prompts_backup_dir}"
if [ -d "${PROMPT_DIR}" ]; then
cp -a "${PROMPT_DIR}/." "${prompts_backup_dir}/"
fi
echo "Created backup set:"
echo " ${BACKUP_DIR}/${postgres_file}"
echo " ${BACKUP_DIR}/${manifest_file}"
echo " ${uploads_backup_dir}/ (incremental uploads mirror)"
echo " ${prompts_backup_dir}/ (prompts mirror)"
deploy/backup/mount_synology_cifs.example.sh
-36
View File
@@ -1,36 +0,0 @@
#!/usr/bin/env sh
set -eu
# Example only. Copy to a local script and replace placeholder values.
# Do NOT commit secrets.
SHARE="//nas-host-or-ip/share-name"
MOUNT_POINT="/mnt/nas-backups"
CREDENTIALS_FILE="/etc/samba/credentials/nas-share-credentials"
USERNAME="replace-with-nas-user"
PASSWORD="replace-with-nas-password"
if [ "${USERNAME}" = "replace-with-nas-user" ] || [ "${PASSWORD}" = "replace-with-nas-password" ]; then
echo "Edit USERNAME and PASSWORD placeholders before running this script."
exit 1
fi
mkdir -p "${MOUNT_POINT}"
mkdir -p "$(dirname "${CREDENTIALS_FILE}")"
cat > "${CREDENTIALS_FILE}" <<'EOF'
username=__USERNAME__
password=__PASSWORD__
EOF
sed -i "s|__USERNAME__|${USERNAME}|g" "${CREDENTIALS_FILE}"
sed -i "s|__PASSWORD__|${PASSWORD}|g" "${CREDENTIALS_FILE}"
chmod 600 "${CREDENTIALS_FILE}"
mount -t cifs "${SHARE}" "${MOUNT_POINT}" \
-o "credentials=${CREDENTIALS_FILE},vers=3.0,iocharset=utf8,uid=0,gid=0,file_mode=0600,dir_mode=0700"
echo ""
echo "Mounted ${SHARE} at ${MOUNT_POINT}"
echo ""
echo "To persist across reboot, add this line to /etc/fstab:"
echo "${SHARE} ${MOUNT_POINT} cifs credentials=${CREDENTIALS_FILE},vers=3.0,iocharset=utf8,uid=0,gid=0,file_mode=0600,dir_mode=0700,_netdev,nofail,x-systemd.automount 0 0"
deploy/backup/restore_postgres_backup.sh
-86
View File
@@ -1,86 +0,0 @@
#!/usr/bin/env sh
set -eu
if [ "$#" -lt 1 ]; then
echo "Usage: $0 <path-to-postgres-dump>"
exit 1
fi
dump_file="$1"
COMPOSE_FILE="${COMPOSE_FILE:-docker-compose.production.yml}"
ENV_FILE="${ENV_FILE:-.env.production}"
SYNOLOGY_BACKUP_DIR="${SYNOLOGY_BACKUP_DIR:-}"
if [ ! -f "${dump_file}" ]; then
echo "Backup file not found: ${dump_file}"
exit 1
fi
backup_dir="$(dirname "${dump_file}")"
backup_name="$(basename "${dump_file}")"
timestamp="$(printf '%s' "${backup_name}" | sed -n 's/^postgres-\([0-9]\{8\}-[0-9]\{6\}\)\.dump$/\1/p')"
uploads_file=""
config_file=""
# If SYNOLOGY_BACKUP_DIR wasn't exported in the shell, read it from ENV_FILE.
if [ -z "${SYNOLOGY_BACKUP_DIR}" ] && [ -f "${ENV_FILE}" ]; then
SYNOLOGY_BACKUP_DIR="$(
sed -n 's/^SYNOLOGY_BACKUP_DIR=//p' "${ENV_FILE}" | tail -n 1
)"
fi
if [ -n "${timestamp}" ]; then
# Legacy local full-archive naming.
candidate_uploads="${backup_dir}/uploads-${timestamp}.tar.gz"
candidate_config="${backup_dir}/config-${timestamp}.tar.gz"
if [ -f "${candidate_uploads}" ]; then
uploads_file="${candidate_uploads}"
fi
if [ -f "${candidate_config}" ]; then
config_file="${candidate_config}"
fi
fi
docker compose --env-file "${ENV_FILE}" -f "${COMPOSE_FILE}" exec -T postgres sh -lc \
"PGPASSWORD=\"\$POSTGRES_PASSWORD\" psql -U \"\$POSTGRES_USER\" -d postgres -c \"DROP DATABASE IF EXISTS \\\"\$POSTGRES_DB\\\";\""
docker compose --env-file "${ENV_FILE}" -f "${COMPOSE_FILE}" exec -T postgres sh -lc \
"PGPASSWORD=\"\$POSTGRES_PASSWORD\" psql -U \"\$POSTGRES_USER\" -d postgres -c \"CREATE DATABASE \\\"\$POSTGRES_DB\\\";\""
cat "${dump_file}" | docker compose --env-file "${ENV_FILE}" -f "${COMPOSE_FILE}" exec -T postgres sh -lc \
"PGPASSWORD=\"\$POSTGRES_PASSWORD\" pg_restore -U \"\$POSTGRES_USER\" -d \"\$POSTGRES_DB\" --clean --if-exists --no-owner --no-privileges"
if [ -n "${SYNOLOGY_BACKUP_DIR}" ] && [ -d "${SYNOLOGY_BACKUP_DIR}/uploads" ]; then
docker compose --env-file "${ENV_FILE}" -f "${COMPOSE_FILE}" run --rm --no-deps \
-v "${SYNOLOGY_BACKUP_DIR}:/backup" \
--entrypoint sh app -lc \
"mkdir -p /app/uploads/documents /app/uploads/photos && \
find /app/uploads/documents -mindepth 1 -delete && \
find /app/uploads/photos -mindepth 1 -delete && \
if [ -d /backup/uploads/documents ]; then cp -a /backup/uploads/documents/. /app/uploads/documents/; fi && \
if [ -d /backup/uploads/photos ]; then cp -a /backup/uploads/photos/. /app/uploads/photos/; fi && \
if [ -f /backup/uploads/homepage.md ]; then cp /backup/uploads/homepage.md /app/uploads/homepage.md; else rm -f /app/uploads/homepage.md; fi"
elif [ -n "${uploads_file}" ]; then
# Legacy local full-archive restore.
cat "${uploads_file}" | docker compose --env-file "${ENV_FILE}" -f "${COMPOSE_FILE}" run --rm --no-deps --entrypoint sh app -lc \
"mkdir -p /app/uploads && find /app/uploads -mindepth 1 -delete && tar -xzf - -C /app/uploads"
fi
if [ -n "${SYNOLOGY_BACKUP_DIR}" ] && [ -n "${timestamp}" ] && [ -f "${SYNOLOGY_BACKUP_DIR}/config-${timestamp}.tar.gz" ]; then
tar -xzf "${SYNOLOGY_BACKUP_DIR}/config-${timestamp}.tar.gz" -C .
elif [ -n "${config_file}" ]; then
# Legacy local full-archive restore.
tar -xzf "${config_file}" -C .
fi
echo "Restore complete from: ${dump_file}"
if [ -n "${SYNOLOGY_BACKUP_DIR}" ] && [ -d "${SYNOLOGY_BACKUP_DIR}/uploads" ]; then
echo "Restored uploads mirror from: ${SYNOLOGY_BACKUP_DIR}/uploads"
elif [ -n "${uploads_file}" ]; then
echo "Restored uploads archive: ${uploads_file}"
fi
if [ -n "${SYNOLOGY_BACKUP_DIR}" ] && [ -n "${timestamp}" ] && [ -f "${SYNOLOGY_BACKUP_DIR}/config-${timestamp}.tar.gz" ]; then
echo "Restored config archive: ${SYNOLOGY_BACKUP_DIR}/config-${timestamp}.tar.gz"
elif [ -n "${config_file}" ]; then
echo "Restored config archive: ${config_file}"
fi
deploy/cloudflared/config.yml.example
-14
View File
@@ -1,14 +0,0 @@
ingress:
# Primary transcription app endpoint.
- hostname: transcription.example.com
service: http://app:8000
# Optional: generic remote access endpoints for other internal services.
# Replace hostnames and targets for your LAN.
- hostname: homeassistant.example.com
service: http://192.168.1.50:8123
- hostname: pihole.example.com
service: http://192.168.1.60:80
# Required catch-all.
- service: http_status:404
docker-compose.production.yml
-70
View File
@@ -1,70 +0,0 @@
services:
app:
build:
context: .
dockerfile: Dockerfile
image: transcription:prod
env_file:
- .env.production
environment:
RUN_EMBEDDED_WORKER: "false"
RUNTIME_SETTINGS_ENV_FILE: "/app/.env.production"
depends_on:
postgres:
condition: service_healthy
ports:
- "8000:8000"
volumes:
- app_uploads:/app/uploads
- app_data:/app/data
- ./backup:/backup
- ./.env.production:/app/.env.production
- ./prompts:/app/prompts:ro
restart: unless-stopped
healthcheck:
test: ["CMD", "python", "-c", "import urllib.request; urllib.request.urlopen('http://localhost:8000/healthz')"]
interval: 30s
timeout: 5s
retries: 5
start_period: 20s
worker:
image: transcription:prod
env_file:
- .env.production
command: ["python", "-m", "transcription.worker_service"]
depends_on:
postgres:
condition: service_healthy
volumes:
- app_uploads:/app/uploads
- app_data:/app/data
- ./backup:/backup
- ./.env.production:/app/.env.production:ro
- ./prompts:/app/prompts:ro
healthcheck:
disable: true
restart: unless-stopped
postgres:
image: postgres:16-alpine
env_file:
- .env.production
environment:
POSTGRES_DB: ${DATABASE__DATABASE}
POSTGRES_USER: ${DATABASE__USER}
POSTGRES_PASSWORD: ${DATABASE__PASSWORD}
volumes:
- postgres_data:/var/lib/postgresql/data
restart: unless-stopped
healthcheck:
test: ["CMD-SHELL", "pg_isready -U $$POSTGRES_USER -d $$POSTGRES_DB"]
interval: 10s
timeout: 5s
retries: 10
start_period: 10s
volumes:
postgres_data:
app_uploads:
app_data:
docker-compose.yml
-24
View File
@@ -1,24 +0,0 @@
services:
transcription:
build:
context: .
dockerfile: Dockerfile
container_name: transcription-app
env_file:
- .env.production
environment:
# Database configuration uses nested settings names (env_nested_delimiter="__").
# DATABASE_URL is NOT read by the application and must not be used here.
DATABASE__DRIVER: sqlite
DATABASE__PATH: /app/data/transcription.db
UPLOAD_DIR: /app/uploads
PROMPT_DIR: /app/prompts
ports:
- "8002:8000"
volumes:
- ./uploads:/app/uploads
- transcription_data:/app/data
restart: unless-stopped
volumes:
transcription_data:
docs/architecture.md
-230
View File
@@ -1,230 +0,0 @@
# System Architecture (Current Baseline: V6.1)
This document defines the current V6.1 architecture baseline.
## Architecture Objectives
- Preserve durable archival records for Documents, Sources, People, and processing runs.
- Execute page transcription asynchronously with bounded worker behavior.
- Preserve append-only machine-attempt evidence with request/response provenance.
- Keep UI, API, service, persistence, and provider boundaries explicit and testable.
## Technical Stack
- **Runtime:** Python 3.12+
- **Web application:** FastAPI + NiceGUI
- **Persistence:** SQLModel / SQLAlchemy — PostgreSQL in production, SQLite for local development and tests
- **Validation and settings:** Pydantic V2 + pydantic-settings
- **Concurrency:** asyncio worker loop
- **Provider integration:** OpenRouter adapter behind provider interface
- **Deployment:** Docker Compose (app, worker, PostgreSQL, Cloudflare Tunnel)
- **Quality and tests:** Ruff, ty, pytest, pytest-asyncio
## Runtime Topology
```mermaid
flowchart LR
U[Browser User] --> A[FastAPI + NiceGUI App]
A --> W[Asyncio Worker]
A --> DB[(PostgreSQL / SQLite)]
W --> P[Provider Adapter]
W --> DB
```
The worker loop drains two queues in the same pass: queued transcription Jobs and queued
`MaintenanceRun` records. When neither has work, it idles.
### Production deployment
Production runs as a Docker Compose stack with the app and worker as separate services, so the app
process runs with `RUN_EMBEDDED_WORKER=false` and the worker process owns queue draining. Local
development runs a single process with the worker embedded.
```mermaid
flowchart LR
I[Internet] --> CF[cloudflared tunnel + Access]
CF --> APP[app service]
APP --> PG[(postgres service)]
WK[worker service] --> PG
WK --> PROV[OpenRouter]
```
Deployment, rollback, and recovery procedures are in [Production Runbook](production-runbook.md);
backup configuration and restore are in [Backup and Restore](backup_restore.md).
## Layered Boundaries
### Interface Layer
- `src/transcription/ui/**`
- `src/transcription/api/**`
Responsibilities:
- Route registration, page orchestration, presentation adapters.
- Structured user messaging through shared error presenter.
- No direct persistence access from pages/components.
### Service and Orchestration Layer
Aggregate services:
- `src/transcription/services/documents.py`
- `src/transcription/services/people.py`
- `src/transcription/services/jobs.py`
- `src/transcription/services/sources.py`
- `src/transcription/services/photos.py`
- `src/transcription/services/maintenance.py`
- `src/transcription/services/evidence.py` (read/projection only)
Orchestration modules:
- `src/transcription/services/store.py`
- `src/transcription/services/workflows.py`
Responsibilities:
- Aggregate ownership and invariants.
- Transaction-aware write helpers.
- Cross-service workflows in orchestration modules (`store.py`, `workflows.py`).
- Lookup-table CRUD through the generic `RegistryService` base (`registry.py`), which is not an
aggregate owner itself.
Module classification and per-model ownership are defined in
[services instructions](../.github/instructions/services.instructions.md).
### Persistence Layer
- `src/transcription/db/**`
Responsibilities:
- SQLModel definitions, async session/engine runtime, registry bootstrap.
- Loader helpers that enforce explicit eager loading with `lazy="raise"` relationships.
### Provider Layer
- `src/transcription/providers/**`
Responsibilities:
- Provider API encapsulation.
- Request manifest and transport evidence capture.
- Normalized transcription result contract.
## Core Domain Model
- `Document` owns archival metadata and links to `Source`, `Job`, and `DocumentPerson`.
- `Source` is a document page/file record with selected machine projection and human revision.
- `Job` is an aggregate processing run with status and frozen prompt/runtime settings.
- `JobSource` is queue/membership state for one `(job, source)` pair.
- `ExecutionAttempt` is append-only evidence for each provider call.
- `Photo` is person imagery owned by `PhotosService`.
- `MaintenanceRun` is one queued or executed operational maintenance run.
- `GenealogyPerson`, `GenealogyFamily`, `GenealogyFamilyChild`, and `GenealogyCitation` store
imported GEDCOM genealogy data and citation provenance.
- `DocumentType` and `PersonRole` are UUID-backed registries with optional protected `semantic_key`.
- `Tag` is a shared registry reached through both document and person tagging, linked by
`DocumentTag` and `PersonTag`.
## Processing and Evidence Workflow
1. User creates/updates Document metadata and linked People atomically through workflow orchestration.
2. User creates a Job by uploading one or more Source files or by retranscribing an existing Source.
3. Source files are validated and stored; orientation normalization may be applied at ingest, and stored bytes become the canonical processing bytes.
4. Worker claims queued Job, transitions to `processing`, and processes pending pages in deterministic order.
5. Each provider call writes one immutable `ExecutionAttempt` with:
- request manifest + hash
- transport evidence (when response exists)
- SDK snapshot and normalized metadata
- outcome, timing, and error details when applicable
6. `JobSource` status is updated as queue/projection state; `Source.raw_transcription` is set on first successful attempt and can be explicitly re-pointed by candidate promotion.
7. Job terminal status resolves to `transcribed`, `partial_success`, or `failed`.
## Status Semantics
- **Job statuses:** `queued`, `processing`, `transcribed`, `partial_success`, `failed`
- Operational success path resolves to `transcribed`.
- **JobSource statuses:** `pending`, `transcribed`, `failed`, `cancelled`
- **MaintenanceRun statuses:** `queued`, `processing`, `succeeded`, `failed`
- Maintenance uses `succeeded` rather than `transcribed`; the transcription vocabulary does not
apply to operational runs.
- **Maintenance job types:** `backup`, `storage_reconciliation`, `gedcom_import`
## Maintenance Execution
Operational maintenance is queue-backed rather than run inline from the UI, so it survives request
lifetime and is recorded:
1. Settings enqueues a `MaintenanceRun` with `status=queued` and a `triggered_by` marker.
2. The worker claims the oldest queued run with a conditional update, moving it to `processing`.
3. `backup` runs the deploy backup script; `storage_reconciliation` compares stored media against
`Document`/`Source` records; `gedcom_import` parses the latest uploaded `.ged` file and upserts
genealogy records.
4. The run finalizes to `succeeded` or `failed` with summary, timing, log path, and `error_detail`.
`MaintenanceRun` records operational history and is not evidence in the `ExecutionAttempt` sense;
append-only guarantees apply to transcription attempts.
## Security and Path Handling Boundaries
- Print media delivery uses record-validated API route:
- `src/transcription/api/print_api.py`
- General UI media links resolve through:
- `src/transcription/ui/components/media_urls.py`
- Local filesystem paths must never be accepted from user input as trusted media routes.
## Concurrency and Reliability Principles
- Worker loop reuses service bundle/provider resources for pooled calls.
- Provider-call timeout is explicit and bounded.
- Non-retriable worker-loop faults are surfaced and stop loop spin.
- Per-page outcomes are durably persisted before processing next page.
## Design Decisions and Rationale
### Why `transcribed` is the success terminal state
- The worker and job orchestration resolve successful completion to `JobStatus.TRANSCRIBED`, with mixed and failure outcomes represented by `partial_success` and `failed`.
- This keeps terminal status vocabulary aligned with what the pipeline actually produces: transcribed page content and evidence, not a generic completion marker.
### Why evidence history is append-only while page text is a projection
- `ExecutionAttempt` stores immutable per-call evidence and preserves full attempt history across retries.
- `Source.raw_transcription` is intentionally a mutable projection so UI and exports can show a selected current machine text without mutating historical evidence.
- This split keeps auditability and UX both first-class: history is durable, presentation is editable.
### Why orchestration modules own cross-service workflows
- Service modules do not import each other; aggregate ownership remains local to each service.
- Multi-aggregate writes are coordinated in orchestration modules (`store.py`, `workflows.py`) so transaction boundaries are explicit and testable.
- This avoids circular dependencies and keeps cross-cutting workflow logic centralized.
### Why explicit eager loading is required
- ORM relationships are configured with `lazy="raise"` in key paths, so code must request needed relationships up front.
- This prevents hidden query behavior in UI/service code and makes read shape deterministic and reviewable.
### Why canonical source bytes may be ingest-normalized
- Ingest normalization can correct orientation before persistence so provider calls, evidence hashes, and rendered processing source are consistent.
- The canonical stored bytes, digest, and size become the durable processing identity for that source.
### Why media access uses controlled routes/helpers
- Print/export media uses record-validated API endpoints to avoid direct filesystem path exposure.
- General UI media URLs are generated through shared resolver helpers to keep path handling consistent and centralized.
## Scope Boundary
Current architecture rules live in `docs/*`.
## Related References
- [System Requirements](requirements.md)
- [Data Model](schema.md)
- [Error Handling Policy](error_handling.md)
- [Production Runbook](production-runbook.md)
- [Backup and Restore](backup_restore.md)
- [Error Handling invariant](./invariant/error_handling.md)
- [AI evidence invariant](./invariant/ai_evidence_and_provenance.md)
docs/backup_restore.md
-60
View File
@@ -1,60 +0,0 @@
# Backup and Restore (V6.1)
This guide defines operational backup/restore for clean-slate recovery of the Docker runtime using a host-visible backup folder.
## 1. Backup artifacts
- Backup target root: `BACKUP_DIR` (recommended production value: `/backup`)
- Database artifact per run:
- `postgres-YYYYMMDD-HHMMSS.dump` (PostgreSQL custom dump via `pg_dump -Fc`)
- `backup-YYYYMMDD-HHMMSS.manifest` (run manifest)
- Media/config mirrors under `BACKUP_DIR`:
- `uploads/**` (incremental copy: new files only)
- `prompts/**` (prompt directory mirror)
Retention:
- `BACKUP_RETENTION_DAYS` applies to `postgres-*.dump` and `backup-*.manifest` files.
## 2. Creating backups
Run from repository root:
```bash
sh deploy/backup/create_postgres_backup.sh
```
Environment variables used by the backup script:
- `BACKUP_DIR` (default `./data/backups`)
- `BACKUP_RETENTION_DAYS` (default `14`)
- `UPLOAD_DIR` (default `/app/uploads`)
- `PROMPT_DIR` (default `/app/prompts`)
- `DATABASE__DRIVER` (must be `postgres`)
- `DATABASE__HOST` (default `postgres`)
- `DATABASE__PORT` (default `5432`)
- `DATABASE__DATABASE` (required)
- `DATABASE__USER` (required)
- `DATABASE__PASSWORD` (required)
Recommended production setup:
- Mount a host-visible folder into `/backup` for both `app` and `worker`.
- Set `BACKUP_DIR=/backup` in `.env.production`.
- Use host-level tooling (for example Synology Drive Client on the host) to replicate that folder externally.
## 3. Restoring from backup
Restore requires downtime for app + worker writes.
1. Stop app and worker:
- `docker compose --env-file .env.production -f docker-compose.production.yml stop app worker`
2. Restore database:
- `sh deploy/backup/restore_postgres_backup.sh /backup/postgres-YYYYMMDD-HHMMSS.dump`
3. Start app and worker:
- `docker compose --env-file .env.production -f docker-compose.production.yml start app worker`
4. Validate `/healthz` and run one smoke workflow.
Notes:
- `restore_postgres_backup.sh` still supports legacy archive restore paths for older backup sets.
docs/cloudflare_tunnel_access.md
-66
View File
@@ -1,66 +0,0 @@
# Cloudflare Tunnel and Access Setup
This guide defines the repository-supported setup for exposing app and selected LAN services through Cloudflare Tunnel with Cloudflare Access protection.
## 1. Files used by this deployment
1. `deploy/cloudflared/config.yml` (local copy from `config.yml.example`)
2. `.env.production` (`CLOUDFLARE_TUNNEL_TOKEN`)
3. `docker-compose.production.yml` (`cloudflared` service reads token + mounts config)
Do not commit `config.yml` or `.env.production`.
## 2. Configure cloudflared
1. Copy `deploy/cloudflared/config.yml.example` to `deploy/cloudflared/config.yml`.
2. Update hostname -> service mappings in `ingress`.
3. Keep the final catch-all ingress `http_status:404`.
4. Set `CLOUDFLARE_TUNNEL_TOKEN` in `.env.production`.
Example app route:
- `transcription.example.com` -> `http://app:8000`
Optional generic remote-access routes:
- `homeassistant.example.com` -> `http://<home-assistant-lan-ip>:8123`
- `pihole.example.com` -> `http://<pihole-lan-ip>:80`
## 3. Cloudflare Access policy baseline
Create one Access app policy per exposed hostname:
1. Include: your allowed identities/groups only.
2. Exclude: none by default.
3. Require: identity provider login (and MFA if available).
Recommended baseline:
- App endpoint (`transcription.*`): your admin identity set.
- Other internal endpoints (`homeassistant.*`, `pihole.*`, etc.): explicit least-privilege groups.
## 4. Startup
Start production stack:
```bash
docker compose --env-file .env.production -f docker-compose.production.yml up -d --build
```
Validate tunnel container:
```bash
docker compose --env-file .env.production -f docker-compose.production.yml logs cloudflared
```
LXC/proxied-network note:
- The `cloudflared` service is pinned to `--protocol http2` with explicit DNS resolvers (`1.1.1.1`, `1.0.0.1`) in `docker-compose.production.yml`.
- This avoids environments where Docker's embedded resolver (`127.0.0.11`) cannot resolve `region*.v2.argotunnel.com`, which causes connector precheck failure and tunnel shutdown.
- If tunnel status is still down, verify host/container egress for DNS and TCP 443 to `api.cloudflare.com` and `*.argotunnel.com`.
## 5. Security notes
- Keep `postgres` and other internal-only services off public hostnames unless required.
- Use distinct hostnames per service; avoid path-based multiplexing for unrelated admin surfaces.
- Rotate `CLOUDFLARE_TUNNEL_TOKEN` and Access policy memberships on a regular schedule.
docs/data_migration.md
-94
View File
@@ -1,94 +0,0 @@
# Database Rebuild Migration Workflow
This project uses an explicit **export/import rebuild workflow** for schema migration.
Policy:
- Do not add runtime legacy-compatibility write paths.
- Rebuild a fresh target database from current models.
- Export current data/media, then import into the fresh target.
## Commands
### 1) Export current DB + uploads into a bundle
```bash
uv run python tools/export_import_migration.py export --bundle-dir .migration-bundle
```
Optional source overrides:
- `--source-db <path-or-sqlalchemy-url>`
- `--source-upload-dir <path>`
### 2) Import bundle into a fresh target (SQLite or PostgreSQL)
```bash
uv run python tools/export_import_migration.py import --bundle-dir .migration-bundle --target-db .\data\transcription-new.db --target-upload-dir .\data-new
```
PostgreSQL target example:
```bash
uv run python tools/export_import_migration.py import --bundle-dir .migration-bundle --target-db postgresql://transcription:change-me@localhost:5432/transcription --target-upload-dir .\data-new
```
### 3) Verify migration parity and integrity
```bash
uv run python tools/export_import_migration.py verify --source-db .\data\transcription.db --target-db postgresql://transcription:change-me@localhost:5432/transcription
```
The verify command checks:
- row-count parity across migration tables
- orphan-reference checks for `source`, `job`, `job_source`, and `execution_attempt`
- duplicate `(job_id, source_id, attempt_number)` in `execution_attempt`
Exit code:
- `0` when counts and integrity checks pass
- `1` when mismatches or integrity violations are detected
### 4) One-shot export+import
```bash
uv run python tools/export_import_migration.py migrate --bundle-dir .migration-bundle --target-db .\data\transcription-new.db --target-upload-dir .\data-new
```
## What gets migrated
- Tables (in dependency order): `document_type`, `person_role`, `tag`, `document`, `person`, `photo`, `document_person`, `document_tag`, `person_tag`, `job`, `source`, `job_source`, `execution_attempt`.
- Media tree under `UPLOAD_DIR`.
The bundle contains:
- `database.json` (row export)
- `uploads/` (copied media files)
Path normalization during export/import:
- `source.file_path` is normalized to `documents/...` (upload-root-relative POSIX).
- `photo.path` is normalized to `photos/...` (upload-root-relative POSIX).
Legacy V4.x portrait/homepage backfill in the export step:
- If the source DB has no `photo` table, the exporter synthesizes `photo` rows from legacy `person.portrait_path` values and from legacy homepage image files under `UPLOAD_DIR/homepage`.
- Legacy portrait and homepage image files are copied into the unified `UPLOAD_DIR/photos/{photo_id}{suffix}` layout in the migration bundle.
- Legacy homepage markdown is relocated from `UPLOAD_DIR/homepage/homepage.md` to `UPLOAD_DIR/homepage.md`.
- Legacy `person.full_name` values are split into `given_names` + `last_name` for V5.1 schema compatibility.
## Cutover (SQLite -> PostgreSQL)
After importing to a fresh target:
1. Stop app and worker services to freeze writes.
2. Export a migration bundle from the last SQLite state.
3. Import bundle to PostgreSQL target.
4. Run `verify` against source and target before switching runtime.
5. Switch runtime config to PostgreSQL (`DATABASE__DRIVER=postgres` and related `DATABASE__*` values).
6. Start app and worker services.
7. Run smoke checks (`/healthz`, create/upload/process one job).
## Rollback
If verify or smoke checks fail:
1. Stop app and worker services.
2. Revert runtime config to SQLite.
3. Start app and worker against pre-cutover SQLite database.
4. Preserve failed migration bundle and logs for analysis.
docs/error_handling.md
-137
View File
@@ -1,137 +0,0 @@
# Error Handling Policy (Current Baseline: V6.1)
This policy defines the active V6.1 error taxonomy, translation boundaries, and retry semantics.
## Error Categories
| Category | Meaning | Typical Origin | User Treatment |
| :--- | :--- | :--- | :--- |
| `validation` | Input payload/selection is invalid | UI form parsing, service validators | Inline correction guidance |
| `not_found` | Target record is missing | ID lookup in service layer | Non-blocking warning or redirect |
| `conflict` | State prevents requested action | lifecycle transitions, duplicate semantic keys | Explain required precondition |
| `external` | Provider/network dependency failure | OpenRouter/provider adapter | Retry path and evidence retained |
| `timeout` | Provider call exceeded configured bound | worker/provider client timeout | Retry path and bounded messaging |
| `internal` | Unexpected local failure | unhandled service/runtime faults | Safe generic message + diagnostics capture |
## Runtime Taxonomy and Canonical Mapping
Runtime code uses a richer internal taxonomy for diagnostics and persisted evidence, then maps that
taxonomy to the six canonical categories at the API/UI envelope boundary.
### Internal runtime categories
- `validation_error`
- `user_input_error`
- `not_found_error`
- `conflict_error`
- `external_provider_error`
- `external_timeout_error`
- `processing_error`
- `infrastructure_transient_error`
- `infrastructure_persistent_error`
- `internal_unexpected_error`
### Internal -> Canonical mapping
| Internal category | Canonical envelope category |
| :--- | :--- |
| `validation_error` | `validation` |
| `user_input_error` | `validation` |
| `not_found_error` | `not_found` |
| `conflict_error` | `conflict` |
| `external_provider_error` | `external` |
| `external_timeout_error` | `timeout` |
| `infrastructure_transient_error` | `timeout` |
| `processing_error` | `internal` |
| `infrastructure_persistent_error` | `internal` |
| `internal_unexpected_error` | `internal` |
`ExecutionAttempt.error_category` stores the internal category value so diagnostics remain specific.
## Translation Boundaries
- **Provider layer:** raise provider-scoped exceptions with provider context; do not emit UI text.
- **Service layer:** map raw exceptions into internal categories and preserve causal chain.
- **UI/API layer:** convert internal categories to canonical categories using the centralized mapping.
## Decision Context
### Why taxonomy is category-based (not exception-class-based)
- Categories encode operator-facing recovery semantics (fix input, retry later, investigate internal failure) independent of low-level exception type.
- This keeps retry and messaging behavior consistent even when provider/client libraries change.
### Why page-level failure is isolated
- Multi-page archival documents often contain a mix of readable and degraded pages.
- Isolating failures to page scope preserves successful results and avoids all-or-nothing loss when one page fails.
- Aggregate job status then communicates overall outcome (`transcribed`, `partial_success`, `failed`) without hiding page detail.
### Why retries append evidence instead of mutating rows
- Retry operations are new observations, not corrections of history.
- Appending attempts preserves forensic traceability, timing history, and provider variability analysis.
- Projection updates remain explicit user/workflow decisions, separate from immutable evidence.
## Job and Page Failure Semantics
### Page-Level (`JobSource`)
- `pending` -> `transcribed` when attempt succeeds.
- `pending` -> `failed` when attempt fails terminally.
- `pending` -> `cancelled` on job cancellation before processing.
### Job-Level (`Job`)
- `transcribed` when all pages transcribe successfully.
- `partial_success` when mixed success/failure outcomes exist.
- `failed` when no page transcribes successfully.
## Retry and Retranscription Rules
1. Failed/cancelled pages may be re-queued through retranscription workflows.
2. Retry attempts must append new `ExecutionAttempt` rows; prior evidence remains immutable.
3. Selecting a better candidate must update projection pointers, not mutate historical attempt rows.
## Logging and Diagnostics Rules
1. Persist sufficient attempt error metadata (`error_category`, `error_message`, transport evidence) for post-hoc analysis.
2. Avoid leaking stack traces or local paths into user-facing message envelopes.
3. Preserve causal exception chains for internal diagnostics.
### Message vs detail split
Rules 1 and 2 pull in opposite directions: evidence records need the root cause, and
user-facing envelopes must not carry it. `AppError` therefore separates the two audiences:
| Field | Audience | Carries root cause | Surfaces |
| --- | --- | --- | --- |
| `message` | User-facing and API-facing | No | `show_error`, `build_error_envelope` |
| `detail` | Internal only | Yes | `format_error_detail` (evidence), logs, sanitized UI projection only |
`classify_unexpected_error` builds a generic `message` and puts the exception type and
text on `detail`. Anything rendered to a user or serialized into an API envelope must
read `message`; anything persisted as provenance or logged may read `detail`. When a UI
surface needs to show persisted `error_detail`, it must route through a sanitizing
projection that preserves the category, suggestion, and error reference while reducing
machine-local absolute paths to basenames only.
Enforced by `tests/test_errors.py::test_unexpected_error_does_not_leak_filesystem_paths`
and `tests/test_error_message_safety.py`.
## Operator Recovery Guidance
- **validation/conflict:** correct input or state and retry manually.
- **external/timeout:** allow bounded retries and keep prior attempt evidence visible.
- **internal:** stop automatic retries, surface a safe message, and inspect diagnostics with correlation context.
## UI Messaging Contract
- User-visible errors must be actionable, bounded, and category-consistent.
- Multi-page jobs must show partial outcomes instead of collapsing into a single opaque failure.
- Recovery actions (`retry`, `retranscribe`, `edit input`) must be offered where available.
## Cross-Reference
- [Error Handling invariant](./invariant/error_handling.md)
- [System Requirements](requirements.md)
- [Data Model](schema.md)
docs/index.md
-36
View File
@@ -1,36 +0,0 @@
# Document Transcription System Overview (Current Baseline: V6.1)
This directory is the single source of truth for current V6.1 behavior and architecture.
## Canonical Reading Order
1. [System Architecture](architecture.md) for runtime topology, boundaries, and lifecycle ownership.
2. [System Requirements](requirements.md) for verifiable current-state requirements.
3. [Data Model](schema.md) for entities, constraints, and evidence persistence rules.
4. [Error Handling Policy](error_handling.md) for category, translation, and retry behavior.
## Cross-Version Invariants
- [Historical Document Transcription Design Intent](./invariant/intent.md)
- [Transcription Methodology](./invariant/transcription_methodology.md)
- [Error Handling](./invariant/error_handling.md)
- [Digital Evidence and AI Processing Provenance](./invariant/ai_evidence_and_provenance.md)
- [UI Style Guide](./invariant/ui_style_guide.md)
## Deployment and Operations
- [Production Runbook](production-runbook.md) for deploy, rollback, and recovery.
- [Backup and Restore](backup_restore.md) for backup configuration and restore procedure.
- [Data Migration](data_migration.md) for the SQLite to PostgreSQL migration path.
- [Cloudflare Tunnel and Access](cloudflare_tunnel_access.md) for remote exposure and access control.
## Baseline Statement
The current V6.1 baseline includes the architectural cleanup, person-schema redesign,
containerized PostgreSQL deployment, and the navigation, Document Detail, and worker-backed
maintenance refinements reflected across this canonical document set.
Use this `docs/*` canonical set for active design and implementation decisions.
Every canonical document above states this same baseline; `tests/test_meta_contract_guards.py`
fails if one of them falls behind. Forward-looking work is tracked in
[`roadmap_plan.md`](roadmap_plan.md) and is not part of the baseline.
docs/invariant/ai_evidence_and_provenance.md
-145
View File
@@ -1,145 +0,0 @@
# Digital Evidence and AI Processing Provenance (Invariant)
## 1. Purpose
This document defines non-negotiable evidence and provenance rules for the transcription application.
The application exists to preserve historical source material and produce useful transcriptions without losing the ability to inspect, reinterpret, or reprocess the evidence later. Provider integrations, model names, schemas, and user interfaces may change; the principles below must remain true.
## 2. Evidence Model
The application distinguishes five kinds of information:
1. **Source evidence**: the canonical stored media used for processing and the facts needed to identify and verify it.
2. **Execution specification**: the frozen instructions, parameters, source identity, and software context for one processing attempt.
3. **Transport evidence**: the response received at the application/provider boundary, including safe protocol metadata.
4. **Normalized data**: selected fields extracted for search, display, accounting, and workflow behavior.
5. **Derived artifacts**: outputs produced from source evidence, such as transcription text, OCR geometry, confidence data, layout analysis, or entity extraction.
Normalized data and derived artifacts never replace source or transport evidence.
## 3. Core Invariants
### 3.1 Canonical Source Preservation
1. Each source must have one canonical stored byte stream used for processing and provenance.
2. Canonical storage may apply deterministic ingest normalization before persistence.
3. Canonical stored bytes must have a cryptographic content digest, byte size, and stable identity.
4. Post-ingest processing derivatives must not overwrite canonical stored bytes.
5. Moving or renaming a stored file must not change its evidence identity.
### 3.2 Append-Only Processing History
1. Every processing attempt must have a distinct execution record, whether it succeeds, partially succeeds, times out, or fails.
2. A later attempt must not overwrite the evidence from an earlier attempt.
3. A convenient “latest transcription” value may be maintained as a cache or projection, but it is not the authoritative execution history.
4. Human revisions must remain distinguishable from all machine-generated outputs.
5. Reprocessing a source must create new evidence rather than rewriting historical evidence.
### 3.3 Frozen Execution Specification
Each execution must preserve enough information to understand what the application asked the processor to do:
1. Requested provider, model, and provider-routing constraints.
2. Full effective system and user instructions.
3. Prompt asset name and content digest when a prompt asset is used.
4. Every explicitly supplied generation or processing parameter.
5. Whether an optional parameter was explicitly set or omitted.
6. Canonical source digest (and derivative digests when used), media type, dimensions or page geometry when known, and page identity.
7. A secret-safe representation of the request structure.
8. Application, provider-adapter, and client-library versions sufficient to interpret the execution.
The execution specification must not contain credentials, authorization headers, secret query values, or unnecessary duplicate source binaries.
### 3.4 Evidence-Layer Terminology
The following terms are not interchangeable:
- **Transport response**: the status, safe headers, and exact response body received by the application at its HTTP boundary.
- **Router-normalized response**: a response transformed by an intermediary into its common schema.
- **SDK-parsed response**: an object created when a client library validates or filters a response.
- **Normalized metadata**: application-selected fields derived from a response.
- **Native provider response**: the upstream provider's own response before any intermediary transformation.
The application and its documentation must identify which layer is stored. A response must not be described as “raw,” “complete,” or “native” without naming the boundary at which that claim is true.
### 3.5 Transport Evidence
1. Preserve the exact successful response body received at the application's transport boundary before SDK model parsing can discard unknown fields.
2. Preserve the response status and an allowlisted set of non-secret headers needed for correlation, content interpretation, rate-limit diagnosis, or audit.
3. Preserve provider/router request and generation identifiers when available.
4. Preserve safe response evidence for unsuccessful calls when a response was received.
5. Record explicitly when no response was received, such as a local timeout or connection failure.
6. Retain parsed and normalized forms only as additional representations of the preserved response.
Wire-level packet capture, TLS session data, credentials, and unrestricted headers are neither required nor permitted.
These requirements apply to executions performed after transport capture is implemented. For earlier executions, the absence of transport evidence must be represented explicitly. An SDK snapshot or normalized record must never be relabeled or backfilled as transport evidence.
### 3.6 Derived Artifact Provenance
1. Every derived artifact must identify its source evidence and producing execution.
2. Each artifact must declare its semantic type, media/serialization format, schema name and version, producer, producer version, and creation time.
3. Artifact content must be stored directly or referenced by a stable path or object identifier and protected by a cryptographic digest.
4. Coordinates must declare their coordinate system, units, origin, page/image dimensions, and transformation history.
5. Confidence values must identify the producer and scope to which they apply; values from different producers must not be treated as directly comparable without validation.
6. Provider-specific payloads may be retained, but durable application behavior must not depend on undocumented provider fields.
This model must accommodate future OCR text, word or line polygons, layout regions, confidence data, alternate transcriptions, and structured extraction without adding a dedicated column for every possible feature.
### 3.7 Integrity and Auditability
1. Stored evidence must be exportable with enough identifiers and metadata to verify relationships and digests outside the application.
2. Evidence mutation, deletion, and retention behavior must be explicit and testable.
3. Schema upgrades must preserve existing evidence and its original meaning.
4. Backfills must be identified as backfills; they must not imply that previously uncaptured evidence existed.
5. Integrity verification must distinguish a missing file, digest mismatch, unavailable external artifact, and malformed metadata.
### 3.8 Security and Privacy
1. API keys, authorization headers, cookies, and credentials must never be persisted as provenance.
2. Persist only headers and metadata fields that appear on an explicit allowlist of known-safe fields. Discard all other fields before storage; never persist an unrestricted capture and attempt to redact it afterward.
3. Request manifests should reference source content by identity instead of duplicating base64 source data.
4. Diagnostic displays and exports must avoid exposing secrets or machine-local details that are not necessary for evidence interpretation.
## 4. Reproducibility Limits
Provenance supports explanation, comparison, and best-effort reproduction; it does not guarantee identical output.
Identical requests may produce different results because of model updates, provider routing, nondeterministic computation, undocumented defaults, safety systems, or retired endpoints. The application must preserve whether a parameter was omitted rather than pretending to know the provider default used at that time.
Likewise, preserving a general vision-model response does not create OCR coordinates that were never returned. Future coordinate extraction remains possible because canonical source evidence is preserved and can be processed again by a suitable system.
## 5. Model Evaluation Policy
Model selection must be based on a representative sample of the actual archive rather than vendor claims alone.
Evaluation should:
1. Use manually reviewed reference transcriptions following the project's [Transcription Methodology](transcription_methodology.md).
2. Represent printed, typed, handwritten, degraded, tabular, multilingual, and spatially complex material present in the archive.
3. Measure character and word error rates where appropriate.
4. Separately record silent corrections, invented text, omitted text, uncertainty handling, layout fidelity, cost, and latency.
5. Preserve the exact model, endpoint or route, parameters, prompt, source digest, and scoring method for every comparison.
6. Treat model rankings as corpus- and version-specific, not permanent declarations of a universal “best” model.
The deterministic scorer for these comparisons lives in `src/transcription/benchmarking.py`; it is
retained as evaluation-policy infrastructure even though application runtime paths do not call it
directly.
Benchmark material containing family records remains private application data unless explicitly approved for publication.
## 6. Ownership and Change Policy
1. Canonical V6.1 architecture, schema, requirements, and error-policy documents define how current behavior satisfies this invariant.
2. Provider adapters own the capture of provider-boundary evidence.
3. Services own validation, persistence, retention, and export behavior.
4. UI pages may inspect evidence through service contracts but do not define evidence semantics.
5. If implementation conflicts with this invariant, either correct the implementation or explicitly revise this document before accepting the behavior.
6. Revisions to this document require deliberate review because they change the long-term preservation contract.
## 7. Related Invariants
- [Historical Document Transcription Design Intent](intent.md)
- [Transcription Methodology & Style Guide](transcription_methodology.md)
- [UI Style Guide](ui_style_guide.md)
docs/invariant/error_handling.md
-101
View File
@@ -1,101 +0,0 @@
# Error Handling (Invariant)
## 1. Purpose
This document defines the non-negotiable failure-handling principles for the transcription application.
Error categories, API envelopes, status codes, framework integrations, and persistence fields may change between versions. Failures must nevertheless remain visible, safe, diagnosable, and consistent across every application boundary.
## 2. Core Invariants
### 2.1 Failures Are Visible
1. An operation must not report success when all or part of the requested work failed.
2. Invalid input, unavailable dependencies, persistence failures, provider failures, and unexpected defects must be surfaced through the application's established error path.
3. Code must not silently discard an exception, provider response, invalid value, or failed state transition.
4. When work can partially succeed, the successful and failed portions must be identified separately.
### 2.2 Messages Are Actionable
1. Operator-facing errors must explain what failed in concise language.
2. When a safe corrective action is known, the error must state it.
3. Expected validation or conflict failures must not be presented as unexplained internal defects.
4. Internal diagnostics must not replace a usable operator-facing message.
### 2.3 Errors Have Stable Identity and Classification
1. Every surfaced failure must have a stable correlation identifier or equivalent trace identity.
2. Failures must be classified into a documented, machine-readable category.
3. Boundary-specific representations must preserve the original category and correlation identity.
4. Unknown exceptions must be converted at an explicit boundary, retain their causal chain for diagnostics, and be classified as unexpected rather than disguised as an expected failure.
### 2.4 Boundary Translation Is Consistent
1. UI, API, service, worker, persistence, and provider boundaries must use one shared error model or deterministic translations between documented models.
2. A boundary may simplify presentation, but it must not change the meaning, retryability, or identity of a failure.
3. Domain and service code must not depend on UI notifications or HTTP response types.
4. UI and API layers must not infer error categories by parsing message text.
### 2.5 State Changes Are Safe
1. A failed atomic operation must leave persisted state unchanged.
2. Batch operations may preserve successful independent items only when partial success is an explicit part of the workflow contract.
3. A failed item must retain enough state to identify what was attempted and whether retry is safe.
4. Error handling must not overwrite earlier successful results or historical execution evidence.
### 2.6 Retry Is Explicit and Bounded
1. Validation, authorization, policy, conflict, and other deterministic failures must not be retried automatically without a relevant input or state change.
2. Automatic retry is permitted only for failures classified as transient and only when the operation is idempotent or otherwise protected from duplicate effects.
3. Retry count, delay, and terminal behavior must be bounded and observable.
4. Exhausted retries must end in a visible terminal failure rather than an indefinitely pending state.
### 2.7 Diagnostics Are Preserved Safely
1. Logs and persisted diagnostic evidence must retain enough context to correlate the failure with the affected operation and record.
2. Provider and infrastructure failures must preserve safe diagnostic evidence at the boundary where it is available.
3. Credentials, authorization headers, cookies, secret values, and unnecessary personal data must not appear in errors, logs, notifications, or exports.
4. Diagnostic metadata capture must use explicit safe-field allowlists where unrestricted content could contain secrets.
5. User-facing messages must not expose stack traces, local filesystem details, database credentials, or raw internal exceptions.
AI execution failures also follow the evidence rules in [Digital Evidence and AI Processing Provenance](ai_evidence_and_provenance.md).
### 2.8 Cancellation and Timeout Are Distinct Outcomes
1. User cancellation, application shutdown, local timeout, remote timeout, and provider rejection must remain distinguishable.
2. Cancellation must not be converted into success or a generic unexpected error.
3. Timeout handling must identify whether a provider response was received when that fact is known.
4. Cleanup after cancellation or timeout must preserve consistency and must not conceal a completed side effect.
### 2.9 Logging Must Support Audit Without Becoming the Record
1. Structured logs must include correlation identity, operation, category, and relevant non-secret record identifiers.
2. Expected operator errors may be logged less severely than unexpected defects, but they must remain observable.
3. Logs are operational diagnostics and do not replace required database state or archival evidence.
4. Duplicate logging of the same failure at every layer should be avoided; ownership of the authoritative log event must be clear.
## 3. Verification Policy
Each version must verify:
1. Every documented error category reaches the intended UI and API representation.
2. Failed atomic writes roll back completely.
3. Partial-success workflows preserve successful independent results and identify failed items.
4. Retry behavior is bounded and restricted to eligible failures.
5. Unexpected exceptions retain correlation and causal information without exposing sensitive details.
6. Logs, persisted evidence, UI messages, and exports contain no credentials.
7. Cancellation, timeout, provider response failure, and no-response failure remain distinguishable.
## 4. Versioned Ownership
1. Version-specific error taxonomies, envelopes, HTTP mappings, model fields, and framework behavior belong in the applicable version documentation.
2. Each versioned error-handling document must state how it satisfies this invariant.
3. A version may add stricter safeguards but must not weaken these principles without first revising this invariant deliberately.
4. Implementation and tests must be updated together when a versioned error contract changes.
## 5. Related Invariants
- [Historical Document Transcription Design Intent](intent.md)
- [Digital Evidence and AI Processing Provenance](ai_evidence_and_provenance.md)
- [UI Style Guide](ui_style_guide.md)
docs/invariant/intent.md
-25
View File
@@ -1,25 +0,0 @@
# Historical Document Transcription Design Intent
I have several thousand pages of family history told through letters, postcards, books, and other documents that I want to transcribe to text.
---
## Goals
1. Preserve our family history
2. Unburden my family (and descendants) from having to store and care for the physical media. Once the documents have been transcribed and organized, they can be donated (or kept by a family member that wants to retain and preserve them).
3. Make the document text easily available and easily searchable.
4. Ability create timelines for individuals and/or families through document dates or the data contained in them. Perhaps even use AI to generate biographies or family histories.
---
## Source material
1. **letters, cards, diaries** - handwritten; mostly stored in boxes and tubs with little organization
2. **books** - typed or typeset; mostly self-published books 50-100 pages in length. This may be expanded to include selected pages from other publications.
3. **photos** - notes written on the backs of photos and the pages of photo albums
4. **other ephemera** - newspaper clippings, event programs, invitations, military records, immigration records, etc
---
## Methodology
1. Follow current best practices per **A Guide to Documentary Editing** by Mary-Jo Kline. (See [Transcription Methodology](transcription_methodology.md))
docs/invariant/transcription_methodology.md
-72
View File
@@ -1,72 +0,0 @@
# Transcription Methodology & Style Guide
## 1. Overview & Core Philosophy
This document defines the formal transcription standard for processing historical manuscripts, letters, diaries, and printed ephemera.
Following the principles established by Mary-Jo Kline in A Guide to Documentary Editing, this project adheres to a Strict Literal Transcription (Verbatim) model as its foundational layer. The primary goal is total textual fidelity—capturing what the author wrote, not what they intended to write—while ensuring the output remains machine-readable and indexable for downstream digital query and search systems.
## 2. Textual Policy
Transcribers (human or AI) must record the exact text of the source document without silent corrections, modernizations, or stylistic smoothing except where explicitly instructed in this guide.
* **Substantives:** Words, letter forms, structural layout, and semantic content must be recorded strictly as presented in the original document.
* **Accidentals:** Punctuation, capitalization, misspellings, and archaic character representations must be preserved unless an explicit rule below allows for standardization.
## 3. Standard Transcription Rules & Markup
The following rules map directly to editorial conventions for handling common manuscript anomalies and physical document features.
### 3.1 Textual Anomalies & Corrections
| Document Feature | Rule | Standard Markup Format | Output Example |
| --- | --- | --- | --- |
| **Misspellings & Errors** | Retain original spelling verbatim. Insert an italicized [sic] immediately following the error. Do not correct spelling silently. | [sic] | The weather was very cold and publick [sic] business delayed. |
| **Missing Words / Omissions** | Insert necessary words required to restore basic grammatical sense inside square brackets. | [word] | We went [to] the store to buy supplies. |
| **Uncertain / Conjectural** | Place best hypothesis followed by a question mark inside square brackets when handwriting is doubtful. | [word?] | He went to [Boston?] yesterday to meet the governor. |
| **Completely Illegible** | Use [illegible] for unreadable script. Use explicit damage descriptors when physical impairment prevents reading. | [illegible] or [reason] | The total cost was [illegible] dollars. or The letter ends here [remainder of page torn]. |
| **Canceled / Struck-through** | Wrap text removed by the author inside a [deleted: ...] tag to preserve authorial revisions. | [deleted: text] | We left at [deleted: noon] one o'clock instead. |
| **Interlineations / Additions** | Wrap text inserted above, below, or in margins into the narrative flow inside an [inserted: ...] tag. | [inserted: text] | The [inserted: red] house on the hill was abandoned. |
### 3.2 Typography, Characters & Layout
| Document Feature | Rule | Standard Markup Format | Output Example |
| --- | --- | --- | --- |
| **Superscripts & Abbreviations** | Bring raised letters down to the main line. Optionally expand abbreviations within square brackets based on project configuration. | [expanded] | Gen^l becomes Genl or Gen[era]l. |
| **Line-End Hyphenation** | Rejoin words split across a page or line boundary silently, dropping the soft hyphen. | Silently rejoin | Original: "estab- / lishment" becomes establishment |
| **Capitalization** | Preserve explicit capitalization. Default to modern capitalization rules only when authorial intent is ambiguous or archaic forms confuse sentence structure. | Literal / Contextual | If a standard noun like 'Farm' is clearly capitalized, record 'Farm'. If ambiguous, default to 'farm'. |
| **Hierarchical Outlines** | Preserve exact numbering characters (including lowercase Roman numerals or terminal 'j'). Replicate indentation levels using standard spacing. Do not correct sequence or mathematical errors. | Preserve syntax | I. Main Topic a. Sub-point b. Next pointIII. [sic] Third Topic |
### 3.3 Visual & Spatial Elements
| Document Feature | Rule | Standard Markup Format | Output Example |
| --- | --- | --- | --- |
| **Non-Textual Artifacts** | Record non-textual elements (seals, stamps, sketches, physical damage) using brief descriptive text inside square brackets. | [description] | [wax notary seal attached here] or [sketch of a fort layout] |
| **Marginalia & Addenda** | Explicitly indicate spatial transitions before transcribing content located in margins or non-standard orientations. | [location:] | [written in left margin:] Do not share this with anyone. |
### 3.4 Document-Body Medium
Every transcript must identify the predominant document-body medium exactly once at the beginning:
| Medium | Use | Standard Markup |
| --- | --- | --- |
| **Handwritten** | The main body was written by hand. | `[document body handwritten]` |
| **Typewritten** | The main body was produced with a typewriter. Uneven impressions, monospaced characters, and mechanical defects remain typewritten rather than handwritten. | `[document body typewritten]` |
| **Typeset** | The main body was composed for printing or produced as printed text rather than with a typewriter. | `[document body typeset]` |
| **Mixed** | No single medium predominates, or handwritten and printed/typewritten content are structurally interleaved. | `[document body mixed]` |
- Use exactly one document-body marker.
- Do not wrap each line in `[handwritten: ...]` after declaring the body handwritten.
- In typewritten or typeset documents, use localized handwriting markers only for genuinely handwritten annotations, insertions, or signatures.
- In mixed documents, identify handwritten portions locally while preserving their reading context.
- Preserve tables of contents, tables, forms, columns, captions, marginalia, page numbers, dotted leaders, and associated references in their logical reading order.
- Produce plain text characters rather than HTML entities for ordinary transcription content.
## 4. Prompt Asset Integration
When executing programmatic transcriptions via LLM APIs or local models, processing instructions must be packaged into single-purpose system prompts aligned with these rules.
1. **Isolation:** Each transcription prompt file exists as an independent Markdown asset in the repository.
2. **Deterministic Output:** Prompts must explicitly instruct models to follow the markup standards in Section 3 without introducing conversational wrappers, extra prose, or structural markdown outside the source document's native layout.
3. **Iterative Scoping:** Rule modifications or edge-case additions must be submitted as isolated delta commits to individual prompt files to maintain clean revision tracking.
docs/invariant/ui_style_guide.md
-110
View File
@@ -1,110 +0,0 @@
# UI Style Guide (Invariant)
## 1. Purpose
This guide defines non-negotiable UI styling rules for the transcription application.
The design system is token-first and class-driven:
1. Theme tokens are defined in [src/transcription/ui/static/theme.css](../../src/transcription/ui/static/theme.css).
2. Python UI code composes semantic classes instead of inline color values.
3. Pages and components should share a single visual language across Documents, Jobs, People, and Sources flows.
## 2. Source of Truth
Use these files as the style authority:
1. [src/transcription/ui/static/theme.css](../../src/transcription/ui/static/theme.css) for color tokens, semantic utility classes, table styles, and viewer surfaces.
2. [src/transcription/ui/theme.py](../../src/transcription/ui/theme.py) for runtime NiceGUI theme bridge and shared UI helpers.
If this document conflicts with implementation, update this document to match the code immediately after intentional style changes.
## 3. Core Design Invariants
1. Flat, high-density surfaces over decorative depth.
2. Strong content hierarchy with subdued backgrounds and border-based separation.
3. Viewer area remains the highest contrast region in image/transcription workflows.
4. Primary actions are consistent and visually recognizable.
5. Accessible focus rings are always visible for keyboard users.
## 4. Token System
### 4.1 Palette Tokens
Base palette variables live under :root in [src/transcription/ui/static/theme.css](../../src/transcription/ui/static/theme.css):
1. --palette-carbon-black: #1c2321
2. --palette-cool-steel: #7d98a1
3. --palette-blue-slate: #5e6572
4. --palette-powder-blue: #a9b4c2
5. --palette-platinum: #eef1ef
### 4.2 Semantic Theme Tokens
Do not style components directly with palette tokens when a semantic token exists.
Semantic tokens currently include:
1. --theme-text and --theme-text-muted
2. --theme-page, --theme-surface, --theme-surface-raised, --theme-surface-muted
3. --theme-border
4. --theme-primary and --theme-primary-hover
5. --theme-secondary and --theme-focus
6. --theme-inverse-text
7. --theme-viewer, --theme-viewer-border, --theme-viewer-muted
## 5. Approved Semantic Classes
### 5.1 Text and Background
1. ui-text-primary
2. ui-text-muted
3. ui-text-inverse
4. ui-bg-page
5. ui-bg-surface
6. ui-bg-surface-raised
7. ui-bg-surface-muted
8. ui-bg-viewer
9. ui-bg-viewer-overlay
10. ui-bg-viewer-overlay-soft
### 5.2 Borders and Surfaces
1. ui-border-subtle
2. ui-border-viewer
3. ui-header-divider
4. ui-card-surface
5. ui-row-surface
6. ui-note-box
7. ui-card-error
### 5.3 Interactive Elements
1. ui-btn-primary
2. ui-btn-secondary
3. ui-link-primary
4. ui-text-accent
5. ui-chip-primary
6. ui-badge-secondary
7. ui-status and ui-status--<status>
### 5.4 Table Patterns
1. ui-table
2. ui-table-header
3. ui-table-body
Use existing class combinations from [src/transcription/ui/components](../../src/transcription/ui/components) and [src/transcription/ui/pages](../../src/transcription/ui/pages) as reference implementations.
## 6. Legacy Class Policy
Legacy `vibe-` presentation classes are prohibited. Use `ui-` semantic classes from `theme.css`.
## 7. Prohibited Patterns
1. Inline hex colors in Python UI class strings or style blocks, except in isolated bridge code explicitly marked for migration.
2. Ad-hoc one-off class names that duplicate existing semantic class intent.
3. Page-specific palette forks that bypass theme tokens.
4. Hidden or low-contrast focus states on interactive controls.
5. Embedded `<style>` blocks or NiceGUI `.style(...)` calls in Python UI code.
6. Additional page- or component-specific stylesheets; `theme.css` is the single CSS source.
## 8. Implementation Rules For Contributors
1. Prefer composing existing semantic classes before creating new ones.
2. If a new class is required, add it to [src/transcription/ui/static/theme.css](../../src/transcription/ui/static/theme.css) with a semantic name, then reuse it.
3. Keep behavior ownership in Python and appearance ownership in CSS.
4. Update UI tests that assert exact text or labels when intentional copy changes are made.
5. Avoid introducing class churn unrelated to the feature being changed.
## 9. Verification Checklist
Before merging UI changes, verify:
1. No new inline hex colors were introduced in UI pages/components.
2. New styles are token-backed and added to [src/transcription/ui/static/theme.css](../../src/transcription/ui/static/theme.css).
3. Primary buttons, links, cards, and tables still render with consistent semantics.
4. Keyboard focus ring visibility is preserved.
5. Relevant UI and integration tests pass.
docs/production-runbook.md
-184
View File
@@ -1,184 +0,0 @@
# Production Runbook
This runbook is the operational checklist for releasing and monitoring the transcription system.
## 1. Pre-release gate checklist
1. Run the full suite: `uv run pytest`
2. Confirm contract guardrails are green:
- `uv run pytest tests/test_meta_contract_guards.py`
3. Confirm health endpoint includes worker liveness payload (`/healthz` returns `worker.state`).
4. Confirm required runtime settings are present in deployment environment:
- `OPENROUTER_API_KEY`
- `DATABASE__*`
- filesystem paths for data/logs/backups.
- `CLOUDFLARE_TUNNEL_TOKEN`
5. Confirm schema contract alignment is current:
- `src/transcription/db/models.py`
- `docs/schema.md`
## 2. Release execution steps
1. Deploy artifact/config to target environment.
- Production stack: `docker compose -f docker-compose.production.yml up -d --build`
- `Settings` loads from explicit `_env_file`, then `ENV_FILE`, then the repository-root `.env.production`; it does not resolve relative to the process working directory.
- For Runtime Settings writes in production, mount `.env.production` into the app container and set `RUNTIME_SETTINGS_ENV_FILE=/app/.env.production`.
- If deployment uses a non-default env-file location, set both `ENV_FILE` and `RUNTIME_SETTINGS_ENV_FILE` to that absolute path so startup reads and Settings-page writes stay aligned.
- For SQLite -> PostgreSQL cutover, run `uv run python tools/export_import_migration.py verify --source-db <sqlite-path-or-url> --target-db <postgres-url>` before switching runtime.
2. Validate service startup:
- `/healthz` responds `200`
- if `RUN_EMBEDDED_WORKER=true`, `worker.state` is `running`
- if `RUN_EMBEDDED_WORKER=false`, validate `worker` container is running in Compose
(worker healthcheck is intentionally disabled because it does not expose `/healthz`)
- validate `cloudflared` logs show active tunnel routes and no ingress errors
3. Execute one smoke workflow:
- create a document/job with at least one source
- verify terminal job outcome updates
- verify execution evidence row appended
4. Verify log flow:
- stdout aggregation receives events
- file logs are written under `./data/logs`
5. Create a fresh PostgreSQL backup after successful deployment:
- `sh deploy/backup/create_postgres_backup.sh` (creates DB dump plus uploads/prompts backups under `BACKUP_DIR`)
## 3. Rollback triggers and actions
### Trigger conditions
1. `/healthz` reports `worker.state=failed`
2. Repeated provider timeout/error spikes beyond normal baseline
3. Evidence write failures or DB persistence failures
### Actions
1. Roll back app artifact and config to previous release.
2. Restart service and re-check `/healthz`.
3. Re-run smoke workflow and confirm worker returns to `running`.
4. Preserve incident evidence:
- `./data/logs`
- relevant DB rows (`job`, `job_source`, `execution_attempt`)
5. If persistence regression is confirmed, restore the latest valid DB dump:
- `sh deploy/backup/restore_postgres_backup.sh <dump-file>`
- Synology media mirror and paired config snapshot (same timestamp) are restored automatically when present.
## 4. Post-release monitoring checklist
## First 24 hours
1. Monitor `/healthz` periodically for `worker.state`.
2. Track job terminal distribution (`transcribed`, `partial_success`, `failed`).
3. Sample timeout/error categories for abnormal increase.
4. Spot-check new `execution_attempt` records for append-only growth and timing metadata.
## First 72 hours
1. Re-check error/timeout trend versus 24h baseline.
2. Verify no recurring worker-failed states.
3. Verify storage growth and rotation behavior under `./data/logs`.
4. Confirm incident response notes are captured for any production anomalies.
## 5. Operator playbook for common incidents
### Worker failed
1. Check `/healthz` payload (`error_id`, `error_category`).
2. Locate matching error in logs.
3. If non-transient defect persists, roll back.
### Provider timeout spike
1. Confirm provider reachability and rate limits.
2. Review timeout frequency and impacted job volume.
3. If sustained, execute rollback criteria and notify stakeholders.
### Partial-success increase
1. Inspect affected `job_source` and `execution_attempt` records.
2. Confirm failures are category-aligned (`external`/`timeout`/`internal`).
3. Triage whether issue is source quality, provider, or runtime regression.
### Cloudflare ingress/access failure
1. Check `cloudflared` container logs for ingress parse, DNS, or auth failures.
2. Confirm `deploy/cloudflared/config.yml` hostname mappings are correct.
3. Confirm `CLOUDFLARE_TUNNEL_TOKEN` in `.env.production` matches the tunnel configured in Cloudflare.
4. Confirm Cloudflare Access app policy includes the intended identity/group for that hostname.
5. If logs show SRV/DNS failures via `127.0.0.11`, use the compose-defined resolver override
(`dns: 1.1.1.1, 1.0.0.1`) and ensure outbound TCP 443 is allowed.
### Backup or restore failure
1. Verify `postgres` container is healthy and accepting connections.
2. Confirm dump file exists and is non-zero size.
3. Re-run backup/restore scripts with explicit `ENV_FILE` and `COMPOSE_FILE` if using non-default paths.
4. If direct Synology copy fails, keep local backup and resolve mount/network before next backup cycle.
- For LXC setups, use `deploy/backup/mount_synology_cifs.example.sh` as the persistent mount template.
## 6. Dependency upgrade policy
Dependencies are declared in `pyproject.toml` and resolved through the committed
`uv.lock`. The lockfile guarantees reproducible installs; the version specifiers
control what a deliberate `uv lock --upgrade` is allowed to move.
### NiceGUI is pinned exactly (`nicegui==3.13.0`)
1. **Rationale.** NiceGUI bundles Quasar and Vue. Minor releases change component
props, slots, and styling, which surfaces as visual and interaction regressions
rather than import or type errors. The UI suite under `tests/ui/` asserts
structure and behavior, not rendered appearance, so a NiceGUI bump can pass the
full test suite and still degrade the interface.
2. **Scope of risk.** All NiceGUI usage is confined to `src/transcription/ui/` and
uses only the public `nicegui.ui` and `nicegui.events` surfaces. The coupling is
shallow, so the pin is about release stability, not about unpicking deep
framework entanglement.
3. **Current stance.** Hold the exact pin through release stabilization. Do not
widen it as incidental cleanup, and do not let automated dependency updates move
it. This includes forgoing patch releases, which is the accepted cost.
4. **Revisiting.** Treat a NiceGUI upgrade as scheduled work with its own change
window: bump the pin deliberately, run `uv run pytest -m "not external"`, then
manually verify each page contract in `docs/ui/pages/` before accepting.
### All other dependencies
Declared with `>=` floors and moved by explicit `uv lock --upgrade`. Verify with
`uv run ruff check .`, `uv run ty check`, and `uv run pytest -q -m "not external"`
before committing a changed lockfile.
## 7. Type-check suppression policy
`uv run ty check` is a blocking pre-commit gate. Suppressions are allowed only for
proven SQLAlchemy descriptor false positives where runtime behavior is correct and
the checker cannot represent the descriptor protocol at that call site.
Every suppression must be:
1. **Targeted** to a single rule (for example `# ty: ignore[unresolved-attribute]`).
2. **Inline** on the expression it suppresses (not file-wide).
3. Followed by a **one-line rationale** stating it is a SQLAlchemy descriptor false positive.
Do not use broad or rationale-free suppressions. If a diagnostic is not a known
false positive, fix the code instead of suppressing it.
## 8. Worker shutdown budget
Worker shutdown waits for at most:
`WORKER_PROVIDER_TIMEOUT_SECONDS + WORKER_SHUTDOWN_GRACE_SECONDS`
`WORKER_PROVIDER_TIMEOUT_SECONDS` covers an in-flight provider call, and
`WORKER_SHUTDOWN_GRACE_SECONDS` is extra time for the loop to persist outcomes
and exit cleanly after the call returns.
Set the container or service termination grace period **above this total**
budget. If termination grace is shorter, the process may be killed before
terminal status and evidence writes are finalized.
## 9. Horizontal scaling precondition
Multiple worker replicas can race on execution-attempt numbering for the same
`(job_id, source_id)` pair. The runtime now retries boundedly on unique-key
conflicts (`uq_execution_attempt_number`) and surfaces a conflict-domain error
if retries are exhausted.
Do not deploy additional worker replicas unless this conflict-retry path and its
tests are present and green in the target build.
docs/requirements.md
-118
View File
@@ -1,118 +0,0 @@
# System Requirements (Current Baseline: V6.1)
These requirements define the active V6.1 contract and align to current implementation.
Requirement IDs encode the baseline that introduced them (`REQ-4-*` from V4, `REQ-6-*` from V6) and
are stable. Never renumber an existing ID; retire it explicitly instead.
## Functional Requirements
### Domain and Record Management
- **REQ-4-001 Document Registry:** The system must create and update `Document` records with title, type, date metadata, optional location, optional archive identifier, and optional notes.
- **REQ-4-002 Source Registry:** The system must create and update `Source` records linked to exactly one `Document`.
- **REQ-4-003 People Registry:** The system must create and update `Person` records, support many-to-many links to `Document` with role, and support many-to-many Person tagging via the shared Tag registry.
- **REQ-4-004 Registry Semantics:** Document types and person roles must support optional immutable semantic keys and hard-delete only when unreferenced.
### Job and Workflow Behavior
- **REQ-4-010 Job Creation:** The system must create `Job` records from uploaded sources and from retranscription of existing sources.
- **REQ-4-011 Prompt Snapshotting:** Job creation must persist effective prompt and runtime settings as immutable per-job snapshots.
- **REQ-4-012 Queue Membership:** Each `(job, source)` pair must be represented by one `JobSource` row.
- **REQ-4-013 Job Status Lifecycle:** `Job.status` must use one of `queued`, `processing`, `transcribed`, `partial_success`, `failed`.
- **REQ-4-014 JobSource Status Lifecycle:** `JobSource.status` must use one of `pending`, `transcribed`, `failed`, `cancelled`.
- **REQ-4-015 Terminal Job Resolution:** Job terminal status must derive from page outcomes as `transcribed`, `partial_success`, or `failed`.
- **REQ-4-016 Cancellation Semantics:** Job cancellation must set remaining `pending` page entries to `cancelled`.
### Transcription and Evidence
- **REQ-4-020 Attempt Evidence:** Each provider call must emit one append-only `ExecutionAttempt` record.
- **REQ-4-021 Attempt Payload:** `ExecutionAttempt` must retain request manifest/hash, outcome, timing, model/provider fields, and error details when present.
- **REQ-4-022 Transport Evidence:** Provider response evidence must be attached to the attempt when a response is available.
- **REQ-4-023 Source Projection Rule:** `Source.raw_transcription` is a projection chosen from attempt outcomes and can be repointed by explicit promotion.
- **REQ-4-024 Candidate Visibility:** UI must expose candidate attempts with metadata needed for comparative review and selection.
### Media and Access
- **REQ-4-030 Ingest Canonicalization:** Stored source bytes may be normalized at ingest (for example orientation correction); stored bytes are the canonical processing source.
- **REQ-4-031 Path Safety:** Client-facing media URLs must be generated from controlled application paths only.
- **REQ-4-032 Print Media Validation:** Print/export source media must be served through record-validated API routes.
### Error and UX Contracts
- **REQ-4-040 Error Envelope:** Service/API errors must map to structured, user-safe error categories and messages.
- **REQ-4-041 Partial Failure Visibility:** Mixed page outcomes must be visible at job and page level.
- **REQ-4-042 Retry Support:** Failed and cancelled pages must support targeted retranscription without requiring full document recreation.
### Person Imagery
- **REQ-6-001 Photo Records:** The system must store reusable `Photo` records that are either owned by a `Person` or unowned for homepage gallery use.
- **REQ-6-002 Primary Photo:** At most one photo per owning `Person` may be marked `is_primary`; setting a new primary must clear the previous one.
- **REQ-6-003 Primary Reassignment:** Deleting an owner's primary photo must promote a remaining photo of that owner rather than leaving the owner without a primary.
### Operational Maintenance
- **REQ-6-010 Queued Maintenance Runs:** Settings-initiated maintenance must persist a `MaintenanceRun` and execute in the worker, not inline in the request that started it.
- **REQ-6-011 Maintenance Run Types:** `MaintenanceRun.job_type` must use one of `backup`, `storage_reconciliation`, `gedcom_import`.
- **REQ-6-012 Maintenance Status Lifecycle:** `MaintenanceRun.status` must use one of `queued`, `processing`, `succeeded`, `failed`.
- **REQ-6-013 Single Claim:** A queued run must be claimed by at most one worker, using a conditional status update rather than read-then-write.
- **REQ-6-014 Run History:** Completed runs must retain status, timing, summary, log reference, and error detail, and expose the log for viewing and download.
- **REQ-6-015 GEDCOM Upload Import:** Settings must support manual `.ged` upload and queue-backed import into genealogy tables.
- **REQ-6-016 GEDCOM Idempotent Upsert:** GEDCOM import must upsert `GenealogyPerson` and `GenealogyFamily` by FamilySearch IDs and avoid duplicate imported citations on re-run.
### Deployment and Runtime Configuration
- **REQ-6-020 Production Persistence:** Production must run against PostgreSQL; SQLite remains supported for local development and tests.
- **REQ-6-021 Split Worker Deployment:** Production must support running the worker as its own process with the app started at `RUN_EMBEDDED_WORKER=false`.
- **REQ-6-022 Runtime Settings Persistence:** Runtime settings edits must persist to the mounted production environment file and survive container restart.
- **REQ-6-023 Configuration Contract Sync:** `.env.production.example` must stay synchronized with `Settings` keys and production-safe defaults.
- **REQ-6-024 Health Reporting:** The deployed stack must report app and worker health through `/healthz`.
- **REQ-6-025 Backup and Restore:** Database and media/config backups must be produced on a host-visible path with a tested restore procedure.
## Non-Functional Requirements
- **REQ-4-100 Boundary Integrity:** UI pages/components must not access persistence directly and must call service APIs.
- **REQ-4-101 Service Ownership:** Aggregate writes must occur in owning service/workflow modules, not in UI handlers.
- **REQ-4-102 Deterministic Loading:** ORM relationship reads in service/UI code must use explicit eager loading compatible with `lazy="raise"`.
- **REQ-4-103 Async Safety:** Long-running provider calls must not block UI event handlers directly.
- **REQ-4-104 Evidence Durability:** Attempt evidence must survive process restart once the transaction commits.
- **REQ-4-105 Test Guardrails:** Architecture boundary tests must remain in place for services and UI boundaries.
## Requirement Interpretation Notes
### Status and lifecycle semantics
- `REQ-4-013` and `REQ-4-015` intentionally bind success to `transcribed`, not a generic `completed`, so docs, tests, and runtime transitions stay consistent.
- `REQ-4-016` and `REQ-4-042` distinguish cancellation from failure at page level (`cancelled` vs `failed`) while still allowing targeted retranscription.
### Evidence semantics
- `REQ-4-020` through `REQ-4-024` separate authoritative history (`ExecutionAttempt`) from operational projection (`Source.raw_transcription`).
- This supports immutable provenance while allowing explicit candidate promotion for operator workflows.
### Boundary and loading semantics
- `REQ-4-100` and `REQ-4-101` codify aggregate/service ownership and keep UI out of persistence concerns.
- `REQ-4-102` exists to enforce deterministic query shape under `lazy="raise"` and avoid hidden data access in rendering callbacks.
## Verification Anchors
- Service boundary enforcement: `tests/test_service_boundaries.py`
- UI boundary enforcement: `tests/test_ui_boundaries.py`
- Job lifecycle reliability and terminal status behavior: `tests/services/test_workflows_reliability.py`
- Evidence append-only and projection behavior: `tests/services/test_store.py`, `tests/services/test_transcription_service.py`
- Person photo ownership and primary selection: `tests/services/test_photo_service.py`
- Maintenance run lifecycle and worker execution: `tests/services/test_maintenance_service.py`
- Runtime settings persistence: `tests/ui/test_runtime_settings_store.py`, `tests/services/test_settings_services.py`
- Configuration contract synchronization: `tests/test_meta_contract_guards.py`
- Deployment health reporting: `tests/api/test_health.py`
## Traceability Notes
- Source of truth for status enums:
- `src/transcription/db/models.py`
- Source of truth for workflow transitions:
- `src/transcription/services/workflows.py`
- `src/transcription/services/jobs.py`
- Source of truth for attempt evidence writes:
- `src/transcription/services/sources.py`
docs/reviews/2026-08-23-code-review.md
-488
View File
@@ -1,488 +0,0 @@
# Architecture & Code Review Report
**Repository Target:** `transcription/`
**Target Stack:** Python 3.12+ | FastAPI | NiceGUI | SQLModel/SQLAlchemy | Pydantic V2 | asyncio | OpenRouter
**Review date:** 2026-08-23
**Governing procedure:** `.github/skills/python-code-reviewer/skill.md`
**Escalations applied:** `.github/skills/evidence-provenance-auditor/skill.md`, `.github/skills/test-effectiveness-auditor/skill.md`
**Scope:** 77 Python modules / ~13k LOC under `src/transcription`, 57 test files (377 collected non-external tests), 23 documents under `docs/`, 9 active rule files.
> **Status: closed.** Every finding below was remediated in the phases following this
> review. This document is retained as a record of the reasoning, **not** as a list of
> open work, and it is not canonical authority.
>
> Two recommendations were wrong on contact and were corrected during implementation:
> the HIGH-03 fix as written would have stripped root-cause data from `ExecutionAttempt`
> provenance, and the HIGH-01 fix needed to preserve per-page durability that the report
> did not mention. Where this text and the current code or guard tests disagree, the code
> and tests are correct.
### Verification commands and outcomes
| Command | Outcome |
| :--- | :--- |
| `uv run ruff check .` | **Pass** — `All checks passed!` |
| `uv run pytest -q -m "not external"` | **Pass** — 377 passed |
| `uv run ty check` | **10 diagnostics** — all SQLModel/SQLAlchemy column-descriptor false positives (`services/photos.py` ×8, `tests/test_storage_reconciliation.py` ×2). Advisory only; no suppression strategy exists. |
---
## 1. Executive Summary
- **Overall health is good.** The codebase has genuine architectural discipline: layered `ui → services → db`, a single Pydantic-V2 settings source, an atomic compare-and-swap job claim, append-only evidence history, and eleven deterministic guard tests that enforce structural rules rather than describing them.
- **No Critical findings.** The highest-risk category for this domain — secret leakage into stored provenance — was explicitly audited and **passes**: request headers are never persisted, response headers use an allowlist, and the API key is `SecretStr` end-to-end.
- **The top risk is a transaction-atomicity violation on the worker hot path.** Page evidence and terminal job status commit in two separate transactions (`workflows.py:549-598`), directly contradicting `services.instructions.md`. A crash between them leaves a transcript persisted against a job stuck in `PROCESSING`.
- **That violation is invisible to the test suite.** The test-effectiveness audit confirms no test can fail on a split commit — the pipeline tests assert the happy-path end state, which passes either way. The invariant is documented and steered but *not enforced*.
- **Stale-job recovery is startup-only** (`app.py:79`), with a 30-second staleness threshold. A job orphaned shortly before a fast restart is not recovered and remains `PROCESSING` indefinitely, because the worker only claims `QUEUED` rows.
- **The mandated error-presentation boundary is bypassed at 8 sites.** `home_page.py` and `people_page.py` hand-roll `ui.notify(str(exc), ...)`, discarding the `error_id`, category, and suggestion that `error_presenter.show_error` provides. `people_page.py` imports the correct helpers and still bypasses them.
- **User-facing output can leak filesystem paths.** `classify_unexpected_error` (`errors.py:94`) interpolates the raw exception into a message rendered in the UI; a SQLAlchemy `OperationalError` embeds the database file path. This contradicts an explicit rule in `error-handling.instructions.md`.
- **The retry gate ignores error category** (`workflows.py:185`), so non-retriable faults would be requeued. Currently latent because `worker_max_retries` defaults to `0`.
- **Highest-leverage work is enforcement, not refactoring.** Two atomicity tests, a `ty` suppression strategy that lets the pre-commit hook become blocking, and `ruff format --check` in the gate would convert three documented-but-unenforced invariants into deterministic ones.
---
## 2. Executive Architecture Assessment
**Verdict: architecturally sound with a concentrated reliability gap in the worker's commit boundary.**
Domain cohesion is strong. The `services/` layer owns transactions and business rules, `ui/` owns presentation, `db/` owns schema, and `providers/` isolates the OpenRouter adapter behind a `TranscriptionProvider` protocol. Dependency direction is correct and — unusually — *mechanically enforced*: `test_service_boundaries.py` AST-scans for service-to-service imports and `test_ui_boundaries.py` scans pages/components for persistence access. Provider details do not leak upward; `workflows.py` imports only the abstract `providers` types, never `openrouter`.
The evidence/provenance model is the strongest part of the system. `ExecutionAttempt` is genuinely append-only, retries append rather than rewrite, projection writes onto `JobSource` are clearly distinguished from history mutation, and all 14 provenance-auditor invariant checks pass.
**Top systemic risks:**
1. **Split commit boundary on the worker path (High).** Evidence durability and job terminal status are two transactions. This is the one place where the architecture's own written contract is contradicted by the implementation, on the hottest path in the system.
2. **Recovery is a startup-only, time-thresholded sweep (Medium).** There is no runtime reconciliation, so the self-healing property depends on restart cadence rather than on a bounded interval.
3. **Enforcement coverage has known holes (Medium).** Atomicity, error-presenter usage, and formatting are all documented rules with no deterministic test. The repo's own strength — routing invariants into tests — has not been applied to these three.
4. **Leaky transaction ownership (Medium).** `workflows.py` reaches into `services.jobs._session_scope()` and `services.sources._session_scope()` — private members of two different services — to open transactions. Session ownership is ambiguous exactly where it most needs to be explicit.
5. **A 10-diagnostic type-checker baseline with no suppression policy (Low).** The signal is currently ignorable, which means a real regression would blend into the noise.
---
## 3. Findings by Severity
### Critical Severity
**None identified.**
The secret-leakage check — the only plausible Critical for this system — passes explicitly. `OpenRouterProvider` stores an allowlisted subset of *response* headers only (`providers/evidence.py:130-134`, `SAFE_RESPONSE_HEADERS`); request headers containing `Authorization` are never captured into `TransportEvidence`; and the key is held as `SecretStr` from `config.py` through to the client. Append-only evidence history is likewise intact and test-enforced.
---
### High Severity
#### [HIGH-01] Page evidence and terminal job status commit in separate transactions
- **Location:** `src/transcription/services/workflows.py:549-565` (`_finalize_batch_outcome`), `src/transcription/services/workflows.py:584-598` (`_persist_page_outcome`)
- **Problem & Consequence:** `.github/instructions/services.instructions.md` states: *"Never commit transcript updates separately from the paired terminal/retry job status change."* The implementation does exactly that. `_persist_page_outcome` opens its own scope and commits page evidence (line 592-594); `_finalize_batch_outcome` later opens a *second* scope and commits the terminal `JobStatus` (line 558-560). For a single-page job these are two transactions with a window between them. A process crash, container eviction, or unhandled error in that window persists the transcript while the job remains `PROCESSING`. Because the worker only claims `QUEUED` rows, that job is not reprocessed; it is recoverable only by the startup sweep, and only if it has aged past the staleness threshold (see MED-01). The user sees a job that never completes despite the transcription having succeeded and been billed.
This is a deliberate design tension, not an oversight: `_persist_page_outcome_durably` (line 568-581) wraps the page write in `asyncio.shield` precisely so per-page evidence survives cancellation mid-batch. That goal is correct for *multi*-page jobs. The defect is that the single-page and final-page cases inherit the split unnecessarily.
- **Recommendation:** Keep per-page durability for intermediate pages, but commit the final page outcome and the terminal status in one transaction.
```python
# Before — two scopes, two commits
await _persist_page_outcome_durably(job=job, services=services, page=page, session=None)
...
await _finalize_batch_outcome(job=job, services=services, status=status, session=None)
# After — final page and terminal status share one transaction
async with services.jobs.session_scope() as tx:
for page in intermediate_pages:
await _persist_page_outcome_durably(job=job, services=services, page=page, session=None)
await _write_page_outcome(job=job, services=services, page=final_page, session=tx)
await services.jobs.mark_job_status(job.id, status, session=tx)
await tx.commit()
```
Pair this with the atomicity test in HIGH-04 so the boundary cannot silently regress.
- **Effort:** M
---
#### [HIGH-02] Mandated error-presentation boundary bypassed at 8 sites
- **Location:** `src/transcription/ui/pages/home_page.py:212,220,228,255`; `src/transcription/ui/pages/people_page.py:265,321,330,339`
- **Problem & Consequence:** `.github/instructions/ui.instructions.md:42` requires all user-facing error display to route through `components/error_presenter.py`. Seven of nine pages comply. These two hand-roll `ui.notify(str(exc), type="negative")`. The consequence is not cosmetic: `show_error` (`error_presenter.py:52-67`) surfaces the correlation `error_id`, the canonical error category, and the actionable `suggestion` field. Bypassing it means a user hitting a failure on the home or people page gets a bare exception string with **no error reference to report**, making these two pages unsupportable in production — precisely the pages most likely to be a user's entry point.
`people_page.py` already imports `run_ui_action` and `show_error` at lines 28-29 and uses them elsewhere in the same module, so the bypass is inconsistency rather than missing infrastructure.
- **Recommendation:** Replace each site with the canonical helper. The unused `summarize_error` helper in `error_presenter.py` (currently a retained orphan — see LOW-07) is the natural fit where a compact string is genuinely needed.
```python
# Before
except AppError as exc:
ui.notify(str(exc), type="negative")
# After
except AppError as exc:
show_error(exc)
```
Then close the hole permanently by extending `tests/test_ui_boundaries.py` with an AST check that no module under `PAGES_DIR` calls `ui.notify(...)` with `type="negative"`.
- **Effort:** S
---
#### [HIGH-03] Unexpected-error path leaks filesystem paths into user-facing output
- **Location:** `src/transcription/errors.py:91-98` (line 94), rendered via `src/transcription/ui/components/error_presenter.py:52-67`
- **Problem & Consequence:** `classify_unexpected_error` builds `f"Unexpected error during {operation}: {exc}"` and stores it as `AppError.message`. `show_error` renders `error.message` directly to the user. Any exception whose `str()` contains infrastructure detail is therefore displayed verbatim — a SQLAlchemy `OperationalError` embeds the absolute SQLite database path, and an `OSError` from the media layer embeds the storage root. `.github/instructions/error-handling.instructions.md:74` states: *"Never leak … local filesystem paths in user-facing output."* This is the generic catch-all path, so it applies to every unanticipated failure across the application.
- **Recommendation:** Split the diagnostic detail from the user-facing message. Log the full exception with the `error_id` as the correlation key; show the user a stable message plus that id.
```python
# Before
return AppError(
f"Unexpected error during {operation}: {exc}",
category=ErrorCategory.INTERNAL_UNEXPECTED,
...
)
# After
error = AppError(
f"Unexpected error during {operation}.",
category=ErrorCategory.INTERNAL_UNEXPECTED,
suggestion="Retry once. If it persists, report the error reference id.",
retriable=False,
)
logger.exception("error_id=%s operation=%s", error.error_id, operation)
return error
```
Add a case to `tests/ui/test_error_presenter.py` asserting that a raised `OperationalError` carrying a path does not surface that path in the rendered message.
- **Effort:** S
---
#### [HIGH-04] Transaction-atomicity invariants have no enforcing test
- **Location:** Contract at `.github/instructions/services.instructions.md` §"Workflow Transaction Boundaries"; gap confirmed across `tests/integration/test_pipeline_flow.py:66-160` and `tests/services/test_job_service.py:41-59`
- **Problem & Consequence:** The test-effectiveness audit establishes that **neither** Transaction B (transcript + `TRANSCRIBED`) nor Transaction C (retry: `error_detail` + `retry_count` + `QUEUED`) is enforced. The existing pipeline test asserts the final state after a successful run — which passes identically whether the writes shared one commit or used two. To fail on a split-commit regression a test must inject a fault *between* the writes; no such test exists.
The consequence is that HIGH-01 shipped undetected and any future refactor of `advance_job` can reintroduce it just as silently. This is a *governance* failure rather than a code defect: the repo's stated model is that hard rules belong in deterministic tests, and this rule is the most consequential one that never made the transition.
- **Recommendation:** Add `tests/integration/test_pipeline_atomicity.py` with two tests that patch the session to raise after `flush()` but before `commit()`, then assert that *neither* side of the pair is visible in a fresh session. These tests should **fail against the current implementation** and pass once HIGH-01 is fixed — write them first.
- **Effort:** M
---
### Medium Severity
#### [MED-01] Stale-job recovery runs only at startup, behind a 30-second threshold
- **Location:** `src/transcription/app.py:71-81` (`_recover_stale_processing_jobs`), sole caller at `app.py:79` inside `_lifespan`
- **Problem & Consequence:** `requeue_stale_processing_jobs` has exactly one call site, in the lifespan startup handler. There is no runtime re-check. The staleness predicate is `updated_at < now - worker_provider_timeout_seconds` (default **30.0s**, `config.py:116`). A job orphaned less than 30 seconds before a fast container restart therefore fails the predicate at the only moment recovery is attempted, and stays `PROCESSING` forever — the worker claims only `QUEUED` rows. It self-heals only on some *later, unrelated* restart. In a frequently-redeployed environment, restarts are exactly when orphans are created, so the recovery window is systematically misaligned with the failure it exists to handle.
- **Recommendation:** Move the sweep onto a periodic task in the worker loop (e.g. every `max(30, provider_timeout * 2)` seconds) in addition to the startup call, and derive the threshold from a dedicated `worker_stale_job_seconds` setting rather than reusing the provider timeout, so the two can be tuned independently.
- **Effort:** M
---
#### [MED-02] Retry gate ignores `error_category`, so non-retriable failures would be requeued
- **Location:** `src/transcription/services/workflows.py:184-194`
- **Problem & Consequence:** The `JobStatus.FAILED` branch gates solely on `job.retry_count < settings.worker_max_retries`. It does not consult `error_category` or the `AppError.retriable` flag. `.github/instructions/error-handling.instructions.md` classifies `validation`, `not_found`, and `conflict` as non-retriable; under this gate a malformed source or a missing record would be retried to exhaustion, consuming provider quota on calls that cannot succeed and delaying the terminal failure the user needs to see. There is also no backoff — retries requeue immediately.
Currently **latent**: `worker_max_retries` defaults to `0` (`config.py:113`) and is commented out in `.env`, so the branch always falls through to the max-retries log. It becomes live the moment anyone enables retries.
- **Recommendation:** Gate on retriability *and* count, and add exponential backoff before requeue.
```python
case JobStatus.FAILED:
if job.error_category in NON_RETRIABLE_CATEGORIES:
logger.error("Job %s failed non-retriably (%s).", job.id, job.error_category)
return
if job.retry_count < settings.worker_max_retries:
...
```
Cover with a test that a `validation`-category failure is not requeued even when `worker_max_retries > 0`.
- **Effort:** S
---
#### [MED-03] `IntegrityError` on the attempt-number flush is uncaught, risking evidence loss
- **Location:** `src/transcription/services/sources.py:540-546` (attempt-number computation), `sources.py:587` (unguarded `flush()`)
- **Problem & Consequence:** `attempt_number` is derived read-then-write as `MAX(attempt_number) + 1`, and `uq_execution_attempt_number` enforces uniqueness (`db/models.py:507`, documented at `docs/schema.md:273`). The sibling `JobSource` insert *does* catch `IntegrityError` (`sources.py:531-534`), but the `ExecutionAttempt` flush at line 587 does not. Two concurrent attempt writes for the same job source would raise an unhandled `IntegrityError` and lose an evidence row — the one class of data this system exists to preserve. Not currently reachable: the worker is single-instance and processes sources sequentially. It becomes reachable the moment a second worker replica is deployed.
- **Recommendation:** Mirror the `JobSource` handling — catch `IntegrityError`, recompute `MAX(attempt_number) + 1`, and retry the insert a bounded number of times, raising a domain error on exhaustion. Note this constraint as a horizontal-scaling precondition in `docs/production-runbook.md`.
- **Effort:** M
---
#### [MED-04] Shutdown timeout is shorter than the provider timeout
- **Location:** `src/transcription/worker.py:146` (`asyncio.wait_for(worker_task, timeout=2.0)`); provider timeout at `config.py:116` (default 30.0s)
- **Problem & Consequence:** Graceful shutdown waits 2 seconds for the worker task, but the stop event is only checked *between* jobs and an in-flight provider call may run for up to 30 seconds. Any shutdown during a provider call therefore cancels mid-flight. Combined with HIGH-01's split commit, a cancellation that lands between the evidence commit and the status commit produces exactly the stuck-`PROCESSING` state described there — so this finding materially raises HIGH-01's probability rather than being independent of it.
- **Recommendation:** Derive the shutdown budget from the provider timeout (`worker_provider_timeout_seconds + small_grace`) instead of hardcoding `2.0`, and ensure the container's termination grace period exceeds it. Document both in `docs/production-runbook.md`.
- **Effort:** S
---
#### [MED-05] `workflows.py` reaches into two services' private `_session_scope`
- **Location:** `src/transcription/services/workflows.py:558` (`services.jobs._session_scope()`), `workflows.py:592` (`services.sources._session_scope()`)
- **Problem & Consequence:** The orchestration module opens transactions by calling a private member on two different service objects. This is the concrete mechanism behind HIGH-01: because transaction ownership is expressed through a private back-door rather than a declared boundary, nothing in the design makes it obvious that two scopes are being opened for one logical unit of work. It also couples `workflows.py` to a service implementation detail that `test_service_boundaries.py` cannot see (it checks imports, not attribute access).
- **Recommendation:** Promote a single explicit transaction entry point — a `session_scope()` on `ServiceBundle`, or a module-level `unit_of_work(services)` helper — and make `workflows.py` use only that. Extend `test_service_boundaries.py` with an AST check forbidding `_session_scope` attribute access outside the owning service module.
- **Effort:** M
---
### Low Severity
#### [LOW-01] `hashlib.sha256` over full file bytes runs on the event loop
- **Location:** `src/transcription/services/store.py:401`
- **Problem & Consequence:** Digest computation is CPU-bound and synchronous inside an `async def`. For large uploads this blocks the loop, stalling both the NiceGUI UI and the worker. Every sibling I/O path in the codebase correctly uses `asyncio.to_thread` (`media_storage.py:43`, `normalization.py:117`, `photos.py:176`, `sources.py:740,753`), so this is an isolated deviation.
- **Recommendation:** `digest = await asyncio.to_thread(lambda: hashlib.sha256(file_bytes).hexdigest())`.
- **Effort:** S
#### [LOW-02] `homepage_store.py` performs synchronous file I/O from async callers
- **Location:** `src/transcription/ui/homepage_store.py:25,32`; called from `src/transcription/ui/pages/home_page.py:170`
- **Problem & Consequence:** Same class as LOW-01 — reads/writes the homepage JSON directly rather than via `asyncio.to_thread`. Impact is small (a tiny file), but it is a second deviation from an otherwise universal convention.
- **Recommendation:** Wrap both calls in `asyncio.to_thread`.
- **Effort:** S
#### [LOW-03] Worker poll interval is hardcoded outside `Settings`
- **Location:** `src/transcription/app.py:62` (`poll_interval_seconds=1.0`)
- **Problem & Consequence:** The single operational knob controlling worker latency-vs-load cannot be tuned without a code change, contradicting the otherwise-clean rule that all configuration lives in `config.py` (zero `os.getenv` calls exist outside it).
- **Recommendation:** Add `worker_poll_interval_seconds: float = 1.0` to `Settings` and read it at the call site.
- **Effort:** S
#### [LOW-04] `_build_request_manifest` returns `None` silently, producing incomplete evidence
- **Location:** `src/transcription/providers/openrouter.py:347`
- **Problem & Consequence:** When `source_reference is None` the manifest is skipped with no log line. The attempt is still recorded but its provenance is quietly incomplete, and there is no signal that it happened — the failure mode is undetectable after the fact.
- **Recommendation:** Log at `warning` with the job/source identifiers before returning `None`, so incomplete provenance is at least attributable.
- **Effort:** S
#### [LOW-05] Ten `ty` diagnostics with no suppression strategy
- **Location:** `src/transcription/services/photos.py` (8), `tests/test_storage_reconciliation.py` (2)
- **Problem & Consequence:** All ten are SQLModel/SQLAlchemy false positives — column descriptors are typed as their Python value type (`UUID`, `datetime`, `bool`), so `.is_()`, `.asc()`, `func.count()`, and `group_by()` appear invalid. Because there is no suppression policy, the pre-commit hook must run `ty` in advisory mode, which means a *genuine* new type error would print alongside the known ten and block nothing.
- **Recommendation:** Add targeted `# ty: ignore[...]` comments with a one-line rationale at each of the ten sites, then flip the pre-commit hook to blocking. This converts a permanently-ignored signal into a real gate.
- **Effort:** M
#### [LOW-06] `ruff format` is not enforced; 35 files have drifted
- **Location:** `.pre-commit-config.yaml`, `ruff.toml`
- **Problem & Consequence:** `ruff check` is blocking but `ruff format --check` is absent from the gate, so formatting drift accumulates silently and inflates unrelated diffs whenever anyone does run the formatter.
- **Recommendation:** Run `uv run ruff format .` once as a single isolated commit, then add `ruff format --check` to the pre-commit gate.
- **Effort:** S
#### [LOW-07] Four retained orphans, all recorded as "uncertain — follow-up"
- **Location:** `tests/test_orphan_sweep.py:33-52` (`KNOWN_ORPHANS`): `BenchmarkManifest`, `dispose_all_engines`, `refresh_engine`, `summarize_error`
- **Problem & Consequence:** Every entry carries the weakest possible justification. `summarize_error` is the notable one: it is an unused helper in `error_presenter.py` *while two pages hand-roll error display* (HIGH-02) — the orphan and the boundary violation are the same problem viewed from two directions. `dispose_all_engines` / `refresh_engine` are plausibly test-support utilities and should be classified as such rather than left uncertain.
- **Recommendation:** Resolve each to a definite outcome — `summarize_error` becomes used by the HIGH-02 fix; classify the engine helpers as test-support or delete them; decide on `BenchmarkManifest`.
- **Effort:** S
#### [LOW-08] Orphan sweep only scans module-level public definitions
- **Location:** `tests/test_orphan_sweep.py`
- **Problem & Consequence:** Methods and private functions are out of scope, so dead code inside classes — the most common kind in a service-oriented codebase — is structurally invisible to the sweep.
- **Recommendation:** Extend the AST walk to public methods on service classes, seeding `KNOWN_ORPHANS` with the current result set to keep the change non-breaking.
- **Effort:** M
#### [LOW-09] f-string interpolation in logging calls
- **Location:** `src/transcription/services/workflows.py:193` and similar sites
- **Problem & Consequence:** `logger.error(f"Job {job.id} has failed...")` formats eagerly regardless of level and prevents structured-logging backends from grouping by template. Ruff's `flake8-logging-format` (`G`) rules are not enabled, so this is unenforced.
- **Recommendation:** Use `logger.error("Job %s has failed and reached max retries.", job.id)` and enable ruff rule set `G`.
- **Effort:** S
#### [LOW-10] Low-signal and always-true assertions in the test suite
- **Location:** `tests/test_traceability.py:54-57`; `tests/integration/test_pipeline_flow.py:135-140,446-452`; `tests/test_orphan_sweep.py:119`; `tests/services/test_workflows_reliability.py:105,178,241,317,375`
- **Problem & Consequence:** Per the test-effectiveness audit: `test_traceability.py:54-57` asserts properties of dict literals defined in the same file (can only fail if the test itself is edited); `assert processed is True` in the pipeline tests is unfalsifiable because `read_job` raises rather than returning `None`; the `>= 200` orphan threshold is a historical snapshot that tolerates ±40 drift; and the `assert result is not None` guards are shadowed by the attribute assertions that follow. Together these overstate effective coverage.
- **Recommendation:** Apply the prune/strengthen backlog in §6 (Testing).
- **Effort:** S
#### [LOW-11] Wall-clock timing dependencies risk CI flakiness
- **Location:** `tests/services/test_workflows_reliability.py:157-196` (real `time.sleep(0.40)`, upper bound `< 540ms` with only 10% slack); `test_workflows_reliability.py:341` (`asyncio.wait_for(..., timeout=2)`)
- **Problem & Consequence:** On a loaded CI runner, a 200ms asyncio task plus 400ms blocking setup can exceed the 540ms bound, producing false failures that erode trust in the suite.
- **Recommendation:** Widen the slack factor to `0.8` or replace the blocking sleep with a controlled clock mock.
- **Effort:** S
---
## 4. Architectural Drift & Gap Analysis
`Direction` is `doc->code` (implementation must change to match documented intent) or `code->doc` (an undocumented but repeatable convention that should be formalized).
| Area / Component | Direction | Documented / Intended Rule | Actual Implementation State | Severity | Recommended Resolution |
| :--- | :--- | :--- | :--- | :--- | :--- |
| Worker commit boundary | `doc->code` | `services.instructions.md`: never commit transcript updates separately from the paired terminal status change | `workflows.py:549-598` commits page evidence and terminal status in two separate sessions | High | Fix per HIGH-01; enforce per HIGH-04 |
| UI error presentation | `doc->code` | `ui.instructions.md:42`: all user-facing error display routes through `error_presenter.py` | 8 hand-rolled `ui.notify` sites in `home_page.py` and `people_page.py` | High | Fix per HIGH-02; add AST guard to `test_ui_boundaries.py` |
| Unexpected-error messaging | `doc->code` | `error-handling.instructions.md:74`: never leak local filesystem paths in user-facing output | `errors.py:94` interpolates raw `exc` into the rendered message | High | Fix per HIGH-03 |
| Retry policy | `doc->code` | `error-handling.instructions.md`: validation / not_found / conflict are non-retriable | `workflows.py:185` gates on retry count only | Medium | Fix per MED-02 |
| Stale-job recovery | `code->doc` | Not documented as startup-only or time-thresholded | Single startup call site; 30s threshold reuses the provider timeout | Medium | Fix per MED-01, then document the recovery contract in `docs/production-runbook.md` |
| Transaction ownership | `code->doc` | `services.instructions.md` assigns transaction ownership to services | `workflows.py` opens scopes via two services' private `_session_scope` | Medium | Fix per MED-05; document the single unit-of-work entry point |
| Blocking-I/O convention | `code->doc` | Not stated as a rule; followed at 5 of 7 sites | `store.py:401` and `homepage_store.py:25,32` deviate | Low | Fix per LOW-01/LOW-02, then state the `asyncio.to_thread` rule in `services.instructions.md` |
| Configuration centralization | `code->doc` | Zero `os.getenv` outside `config.py` — a real, held convention | Held everywhere except the hardcoded `poll_interval_seconds` at `app.py:62` | Low | Fix per LOW-03, then formalize the rule and add a deterministic guard |
| Type-check baseline | `code->doc` | No documented policy for `ty` diagnostics | 10 tolerated false positives; hook is advisory-only | Low | Adopt the suppression strategy in LOW-05 and document it |
| Formatting | `code->doc` | `ruff.toml` configures the formatter | `ruff format --check` absent from the gate; 35 files drifted | Low | Fix per LOW-06 |
| Dependency pin | — | `docs/production-runbook.md` "Dependency upgrade policy" records the exact `nicegui==3.13.0` pin as a deliberate stability decision | Matches | — | **No action** — correctly documented, not a defect |
---
## 5. Invariant Inventory & Routing Recommendations
| Invariant / Constraint | Current Location | Recommended Target Layer | Rationale |
| :--- | :--- | :--- | :--- |
| Transcript + terminal status commit atomically | Instructions only | **Deterministic test** (`tests/integration/test_pipeline_atomicity.py`) | Highest-consequence rule in the system with zero enforcement; steering alone already failed to prevent HIGH-01 |
| Retry writes commit atomically | Instructions only | **Deterministic test** (same file) | Same class; a partial retry commit corrupts `retry_count` accounting |
| All UI errors route through `error_presenter` | Instructions (`ui.instructions.md:42`) | **Deterministic test** (extend `test_ui_boundaries.py`) | Mechanically checkable via AST; 8 live violations prove instructions are insufficient here |
| No filesystem paths in user-facing output | Instructions (`error-handling.instructions.md:74`) | **Deterministic test** (extend `tests/ui/test_error_presenter.py`) | Checkable by asserting a path-bearing exception does not surface its path |
| Non-retriable categories are never requeued | Instructions | **Deterministic test** (`tests/services/test_workflows_reliability.py`) | Latent today; a test freezes the correct behavior before retries are enabled |
| Blocking I/O runs via `asyncio.to_thread` | Convention only (5/7 sites) | **Instructions** (`services.instructions.md`) | Judgment-dependent (thresholds vary by payload size); steering fits better than a hard test |
| Transaction opened through one owned entry point | Convention, violated | **Instructions + test** | Document the entry point; AST-guard against `_session_scope` access outside its owning module |
| Append-only `ExecutionAttempt` history | Docs + 3 tests | **Keep as-is** | Correctly routed and genuinely mutation-sensitive; the model to imitate |
| Service/UI boundary rules | Instructions + 2 AST tests | **Keep as-is** | Working exactly as intended |
| Status vocabulary conformance | `docs/schema.md` + contract guards | **Keep as-is** | Enum drift would fail the suite |
| No secrets in stored evidence | Docs + provenance skill + allowlist in code | **Keep as-is** | Allowlist is the right mechanism — fails closed by construction |
| `ty` diagnostic suppression policy | Nonexistent | **Docs + blocking hook** | Needs a written rationale per suppression before the gate can be trusted |
| NiceGUI exact pin | `docs/production-runbook.md` | **Keep as-is** | Deliberate, documented, correctly excluded from review findings |
---
## 6. Stack-Specific Analysis
### Python 3.12+ Best Practices
Modern syntax is used consistently: `X | None` unions throughout, builtin generics, no `typing.List`/`Optional` legacy forms, `pathlib` over `os.path`. Type-annotation coverage is high, with no bare `Any` on public service signatures. Broad `except Exception` appears where it belongs — the per-page handler at `workflows.py:352` deliberately isolates one page's failure from the batch, which is correct. `# noqa: PLR0915` / `PLR1702` are used sparingly and consistently. Minor gaps: f-strings in logging (LOW-09), and two blocking-I/O deviations (LOW-01/LOW-02).
### FastAPI
Lifespan is handled correctly via an `asynccontextmanager` `_lifespan` (`app.py:36-68`) rather than deprecated `@app.on_event`. Routers are domain-organized with typed path/query parameters and `response_model` declarations. Error handling is centralized through `register_error_handlers`, and the full internal→canonical category mapping is round-trip tested at the HTTP layer (`tests/api/test_error_responses.py:59-95`). `print_api.py:42-49` performs correct `relative_to`-based path containment for media serving. No blocking calls found in `async def` route handlers.
### NiceGUI
Separation of concerns is good — pages delegate to services and `test_ui_boundaries.py` mechanically prevents persistence access from pages and components. Client state is client-scoped; no cross-session global-state leaks found. API usage is correct for the pinned 3.13.0 release. The two defects are the error-presenter bypass (HIGH-02) and synchronous file I/O in `homepage_store.py` (LOW-02).
### SQLModel & SQLAlchemy
The strongest layer. `lazy="raise"` is declared on relationships and correctly paired with `expire_on_commit=False`, which together make N+1 access a loud failure rather than a silent performance cost — no N+1 patterns found. The job claim is a genuine atomic compare-and-swap (`jobs.py:212-222`: conditional `UPDATE ... WHERE status = QUEUED ... RETURNING`), which is the correct primitive and correctly implemented. Hot-path indexes are declared and test-verified (`test_db.py:131`). Cross-dialect portability is handled for SQLite and PostgreSQL. Weaknesses are transaction *ownership* (MED-05, HIGH-01) rather than query construction, plus the uncaught `IntegrityError` at MED-03.
### Pydantic V2 & Settings
Fully migrated — no `@validator`, no `Config` class, no `.dict()` or `parse_obj` anywhere. `model_config = ConfigDict(...)` and `@field_validator` are used correctly. `config.py` is a clean single source of truth: **zero** `os.getenv` calls exist outside it, `.env` is untracked and gitignored, and the API key is `SecretStr` end-to-end. The only deviation is the hardcoded poll interval (LOW-03).
### Asyncio Workers
Task lifecycle is handled properly: task references are retained (no GC risk), `CancelledError` is re-raised rather than swallowed, the provider call happens outside any DB transaction, timeouts resolve to terminal states, and there is no tight polling spin. `_persist_page_outcome_durably`'s use of `asyncio.shield` (`workflows.py:568-581`) is a thoughtful durability mechanism. The defects are the split commit boundary (HIGH-01), the shutdown-vs-provider timeout mismatch (MED-04), and startup-only recovery (MED-01).
### OpenRouter / Adapter Boundary
Encapsulation is clean — `workflows.py` imports only abstract types from `providers`, never `openrouter` directly, so provider specifics do not leak into business logic. The `AsyncClient` is shared with configured timeouts and is properly closed: `worker.py:248,271` → `services.aclose()` → `sources.aclose()` (`sources.py:129-133`) → provider `aclose()` (`openrouter.py:86-87,233-235`). Responses are Pydantic-validated. **All 14 evidence-provenance-auditor invariant checks pass**, including the critical one: the API key is never persisted, request headers are never stored, and `TransportEvidence` captures response headers through an explicit allowlist (`evidence.py:130-134`). Only LOW-04 applies here.
### Testing & Quality Tooling
377 tests pass with `-m "not external"`. The project test contract is honored: `--strict-markers` with all three markers (`unit`, `integration`, `external`) declared, `asyncio_mode = "strict"` with **every** `async def test_` correctly decorated across all 17 async test files, `external` properly excluded from default runs, and **no unawaited-coroutine warnings** — the `filterwarnings` error promotion is clean.
Contract coverage is genuinely strong for structural rules. Confirmed *mutation-sensitive* enforcement exists for: append-only evidence history (3 independent tests, including full before/after field-tuple snapshots), stuck-in-`PROCESSING` prevention, the complete 10-category error mapping, and both boundary rules.
The critical gap is transaction atomicity (HIGH-04) — the audit verdict is **"Effective with Conditions / Go with Conditions"**, blocking on the two missing atomicity tests. Secondary items are the low-signal assertions (LOW-10) and wall-clock flakiness (LOW-11).
**Prune/strengthen backlog:**
| Priority | Task | Location |
| :--- | :--- | :--- |
| High | Add Transaction B atomicity test (fault injected between transcript and status writes) | new `tests/integration/test_pipeline_atomicity.py` |
| High | Add Transaction C atomicity test (retry: `error_detail` + `retry_count` + `QUEUED`) | same file |
| Medium | Delete tautological assertions on same-file dict literals | `tests/test_traceability.py:54-57` |
| Medium | Remove unfalsifiable `assert processed is True` | `tests/integration/test_pipeline_flow.py:135-140,446-452` |
| Medium | Replace `>= 200` snapshot threshold with set-membership assertion | `tests/test_orphan_sweep.py:119` |
| Medium | Assert mapped test files contain ≥1 test, not merely that they exist | `tests/test_traceability.py:59-60` |
| Low | Widen timing slack or mock the clock | `tests/services/test_workflows_reliability.py:157-196` |
| Low | Drop `assert result is not None` guards shadowed by following assertions | `tests/services/test_workflows_reliability.py:105,178,241,317,375` |
---
## 7. Duplication & Consolidation Report
| Pattern / Duplication | Locations | Proposed Canonical Home | Estimated Lines Removed |
| :--- | :--- | :--- | :--- |
| Hand-rolled `ui.notify(str(exc), type="negative")` | `home_page.py:212,220,228,255`; `people_page.py:265,321,330,339` | `ui/components/error_presenter.py::show_error` (already exists) | ~16 |
| Optional-session `if session is None: async with _session_scope()` preamble | `workflows.py:557-561`, `workflows.py:591-595`, and sibling service write paths | `services/base.py::unit_of_work(services, session)` context manager | ~30 |
| Synchronous I/O not wrapped in `asyncio.to_thread` | `store.py:401`, `homepage_store.py:25,32` | `services/base.py::run_blocking` helper | ~6 |
| Read-then-increment `MAX(n) + 1` with uniqueness retry | `sources.py:540-546` (uncaught) vs `sources.py:531-534` (caught) | `services/base.py::insert_with_sequence_retry` | ~20 |
### Proposed Canonical Abstractions
```python
# src/transcription/services/base.py
@asynccontextmanager
async def unit_of_work(
services: ServiceBundle,
session: AsyncSession | None = None,
) -> AsyncIterator[AsyncSession]:
"""Single transaction entry point. Yields a session and commits once on clean exit.
Replaces the `if session is None: async with X._session_scope()` preamble and the
private-member access at workflows.py:558,592. Makes the two-commit split of
HIGH-01 structurally hard to reintroduce.
"""
async def run_blocking[T](fn: Callable[[], T]) -> T:
"""Run a CPU- or disk-bound callable off the event loop."""
return await asyncio.to_thread(fn)
async def insert_with_sequence_retry(
session: AsyncSession,
*,
build: Callable[[int], SQLModel],
next_value: Callable[[], Awaitable[int]],
attempts: int = 3,
) -> SQLModel:
"""Insert a row carrying a derived sequence number, retrying on IntegrityError."""
```
---
## 8. Meta-Tooling & Instruction Update Recommendations
1. **Add `tests/integration/test_pipeline_atomicity.py`** (HIGH-04). The single highest-value enforcement change. Write it before fixing HIGH-01 so it demonstrably fails first.
2. **Extend `tests/test_ui_boundaries.py`** with an AST check forbidding `ui.notify(..., type="negative")` in `PAGES_DIR`, routing all error display through `error_presenter`. Converts `ui.instructions.md:42` from steering into enforcement.
3. **Extend `tests/ui/test_error_presenter.py`** with a case asserting that a path-bearing exception does not surface its path, enforcing `error-handling.instructions.md:74`.
4. **Adopt a `ty` suppression policy** — targeted `# ty: ignore[...]` with rationale at the 10 known sites, documented in `docs/` — then **flip the pre-commit `ty` hook from advisory to blocking**. Until this happens the type checker provides no gate.
5. **Add `ruff format --check` to the pre-commit gate**, preceded by one isolated formatting commit across the 35 drifted files.
6. **Enable ruff rule set `G`** (`flake8-logging-format`) to catch f-string logging (LOW-09).
7. **Extend `tests/test_orphan_sweep.py`** to public methods on service classes, seeding `KNOWN_ORPHANS` with current results (LOW-08). Then resolve all four existing "uncertain" entries to definite outcomes.
8. **Extend `tests/test_service_boundaries.py`** with an AST check forbidding `_session_scope` attribute access outside its owning service module (MED-05). Also address the noted classification gap: the test excludes orchestration modules by hardcoded stem name (`store`, `workflows`, `__init__`), so a new orchestration module under a different name would be misclassified as a service.
9. **Update `.github/instructions/services.instructions.md`** to state the `asyncio.to_thread` rule for blocking I/O and to name the single `unit_of_work` transaction entry point.
10. **Update `docs/production-runbook.md`** with the stale-job recovery contract (interval, threshold, and its relationship to the container termination grace period), and note single-worker as a current precondition until MED-03 is fixed.
11. **Note for `test_ui_boundaries.py`:** the forbidden-import lists are fixed string sets, so a future persistence helper under a new name would escape the check. Consider inverting to an allowlist of permitted imports for pages.
---
## 9. Prioritized Dependency-Ordered Action Plan
**Phase 1: Blocking fixes**
1. Write the two atomicity tests (HIGH-04) and confirm they **fail** against current `main`.
2. Fix the split commit boundary (HIGH-01) and confirm the tests now pass.
3. Fix the filesystem-path leak in `classify_unexpected_error` (HIGH-03).
4. Replace the 8 hand-rolled error notifications with `show_error` (HIGH-02).
**Phase 2: Enforcement hardening**
5. Add the `ui.notify` AST guard and the path-leak presenter test, locking in items 3-4.
6. Adopt the `ty` suppression policy and make the pre-commit hook blocking (LOW-05).
7. Run `ruff format .` as an isolated commit, then add `ruff format --check` to the gate (LOW-06).
8. Enable ruff rule set `G` and fix the resulting logging call sites (LOW-09).
**Phase 3: Reliability & concurrency**
9. Move stale-job recovery to a periodic worker task with a dedicated setting (MED-01).
10. Gate retries on `error_category` and add backoff (MED-02) — do this before ever raising `worker_max_retries` above 0.
11. Derive the shutdown budget from the provider timeout (MED-04).
12. Handle `IntegrityError` on the attempt-number flush (MED-03) — a hard precondition for running more than one worker replica.
13. Move `sha256` and homepage-store I/O off the event loop (LOW-01, LOW-02); move the poll interval into `Settings` (LOW-03).
**Phase 4: Consolidation & refactoring**
14. Introduce `unit_of_work` and migrate `workflows.py` off private `_session_scope` access (MED-05); add the corresponding boundary guard.
15. Extract `run_blocking` and `insert_with_sequence_retry` (§7).
16. Prune the low-signal assertions and reduce timing flakiness (LOW-10, LOW-11).
**Phase 5: Non-blocking governance/documentation depth**
17. Extend the orphan sweep to methods and resolve the four uncertain orphans (LOW-07, LOW-08).
18. Update `services.instructions.md` and `docs/production-runbook.md` per §8 items 9-10.
19. Log incomplete request manifests (LOW-04).
20. Consider inverting the UI boundary check to an allowlist.
---
## 10. Preserved Strengths
- **Evidence and provenance integrity is exemplary.** All 14 provenance-auditor invariants pass. `ExecutionAttempt` history is genuinely append-only, retries append rather than rewrite, and projection writes are cleanly distinguished from history mutation. Three independent tests — including full before/after field-tuple snapshots — make any mutation regression fail loudly.
- **Secret hygiene is correct by construction.** The response-header **allowlist** (`evidence.py:130-134`) fails closed: a newly-introduced sensitive header is excluded by default rather than requiring someone to remember to block it. Request headers are never captured, and `SecretStr` is used end-to-end.
- **Atomic job claiming.** `jobs.py:212-222` uses a conditional `UPDATE ... WHERE status = QUEUED ... RETURNING` — a true compare-and-swap that makes double-claiming impossible under concurrency, rather than the common read-then-write race.
- **`lazy="raise"` paired with `expire_on_commit=False`.** This combination turns accidental lazy loads into immediate errors instead of silent N+1 queries, and it is the reason no N+1 patterns exist in the codebase. Keep it.
- **Architectural rules are mechanically enforced, not merely documented.** AST-based boundary tests for service-to-service imports and UI persistence access are the right pattern; this review's main recommendation is simply to apply that same pattern to three more rules.
- **Configuration discipline.** Zero `os.getenv` calls outside `config.py`, `.env` untracked and gitignored, clean Pydantic V2 throughout with no V1 residue.
- **Path containment on media serving.** `print_api.py:42-49` uses proper `relative_to` validation rather than string prefix matching.
- **Async worker fundamentals.** Task references retained, `CancelledError` re-raised, provider calls outside DB transactions, timeouts resolving to terminal states, no tight polling loop. `asyncio.shield` in `_persist_page_outcome_durably` is a genuinely thoughtful durability mechanism — the fix in HIGH-01 should preserve it for intermediate pages.
- **Test contract rigor.** `--strict-markers`, `asyncio_mode = "strict"` honored across all 17 async test files with no missing decorators, and coroutine-never-awaited promoted to a hard error with a clean run.
docs/reviews/2026-09-02-code-review.md
-896
View File
@@ -1,896 +0,0 @@
# Architecture & Code Review Report
**Repository Target:** `C:\Github\transcription\`
**Target Stack:** Python 3.12+ | FastAPI | NiceGUI | SQLModel/SQLAlchemy | Pydantic V2 | asyncio | OpenRouter
**Review Date:** 2026-09-02
**Canonical Baseline:** V6.1 (`docs/index.md`)
---
## 0. Verification Commands and Outcomes
All four commands were executed in this checkout before any finding was written. This report
records the exact outcomes rather than assuming them.
| Command | Outcome |
| :--- | :--- |
| `uv run pytest -q -m "not external"` | **410 passed, 0 failed, 0 errors** (exit 0) |
| `uv run ruff check .` | **All checks passed!** |
| `uv run ruff format --check .` | **191 files already formatted** |
| `uv run ty check` | **All checks passed!** |
The stated green baseline is real. No finding below is a test failure; every finding is a
behavior, contract, or guard-coverage defect that the passing suite does not detect.
---
## 1. Executive Summary
- **The system's core evidence guarantees hold.** `ExecutionAttempt` is genuinely append-only,
attempt numbering is allocated with bounded conflict retry, transport evidence is captured at
the HTTP boundary before SDK parsing, and header persistence uses a true allowlist. Provenance
invariant families A–E and G pass.
- **Both competing atomicity invariants in `services/workflows.py` are real and both guards
genuinely enforce them.** I injected-fault-verified the tests rather than trusting the
docstrings: `test_pipeline_atomicity.py` fails on a split final-page commit, and
`test_workflows_reliability.py:318` reads intermediate attempts through a *separate session*,
so it would fail if intermediate pages stopped committing individually.
- **The most significant defect is a privacy leak that a prior review believed it had closed.**
The 2026-08-23 review moved root-cause text out of `AppError.message` into `AppError.detail`
to keep filesystem paths away from users. That text now reaches users anyway, because the UI
renders `ExecutionAttempt.error_detail` verbatim (HIGH-01). The leak was relocated, not closed.
- **A second, independent path leak exists in five explicit `raise` sites** that the existing
guard never covered — it tests only `classify_unexpected_error` (HIGH-02).
- **Provenance invariant family F (path safety) fails**, and it fails *inconsistently within one
file*: `sources_page.py:443` carefully sanitizes a stored path through
`public_media_path_label`, then `sources_page.py:484` dumps raw `error_detail` forty lines later.
- **The orphan sweep does not do what its docstring claims.** It matches definitions by bare name,
so an entirely dead *module* passes whenever its function names collide with live ones.
`ui/pages/tags_page.py` is the proof: 93 lines never imported by anything (MED-01/LOW-01).
- **On the three flagged open items:** the V4/V6.1 doc drift is confirmed (MED-02); the
`.env.production` coupling is real but currently correct and loud-failing, so Medium not High
(MED-03); and the Tags roadmap is **right** — the route is genuinely not registered, so the
module is dead code rather than a live retired route.
- **Two latent concurrency defects carry ordering constraints** and must be fixed *before* the
changes that would make them live (MED-04, MED-05), not after.
- **Guidance-file accuracy:** the recently revised `.github/instructions/*` files were verified
against code rather than trusted. They are accurate as written; the code is what diverges from
them. The one exception is that `error-handling.instructions.md` states a `detail` rule the UI
layer has never followed, which makes it an unenforced claim rather than a wrong one.
---
## 2. Executive Architecture Assessment
**Verdict: architecturally sound, with a concentrated failure in the *last mile* of error
presentation.**
Domain cohesion and dependency direction are good and, unusually, mechanically enforced.
`test_service_boundaries.py` and `test_ui_boundaries.py` AST-scan for violations using
*allowlists* rather than blocklists, which is the correct choice — a newly added persistence
helper cannot slip through under an unlisted name. `workflows.py` imports only the abstract
`providers` types and never `openrouter`, so provider details genuinely stop at the adapter.
Transaction ownership is explicit and well-reasoned: `ServiceBase._finalize` commits for
service-owned sessions and flushes for caller-owned ones, which is what lets orchestration
modules compose multi-aggregate writes without services importing each other.
The evidence layer is the strongest part of the system and shows real care. The distinction
between transport response, SDK-parsed response, and normalized metadata is maintained in code,
not just in prose — `_CapturingAsyncClient` exists specifically to retain the exact wire body
before the SDK can discard unknown fields, and `TransportEvidence(response_received=False)`
explicitly represents "no response was received" rather than conflating it with an empty one.
The weakness is at the boundary where internal diagnostic text becomes pixels. Every layer
*below* the UI respects the message/detail split; the UI layer reads the internal field directly
and renders it. The architecture defines the contract correctly and then has no enforcement at
the one layer that violates it.
**Top systemic risks:**
1. **Internal diagnostic text reaches users through the evidence display path** (HIGH-01). The
rule is documented in three places and enforced in none of them at the UI boundary.
2. **Path-safety discipline is applied per-call-site rather than structurally** (HIGH-02, HIGH-01).
It is correct wherever someone remembered; there is no guard that makes forgetting fail.
3. **Guard coverage is narrower than guard docstrings claim.** Two guards
(`test_orphan_sweep.py`, `test_errors.py`) assert something meaningfully weaker than the
invariant they are named for, which converts them into a false sense of enforcement.
4. **Worker safety currently rests on single-process sequential execution, not on configuration**
(MED-04, MED-05). Nothing is wrong today; two plausible future changes each make something wrong.
---
## 3. Findings by Severity
### Critical Severity
*None.* No evidence loss, append-only violation, secret leakage, or silent-wrong-output defect
was found. The candidates in this class (provider evidence mis-attribution, stale-job double
processing) are latent and are reported at High/Medium with their unblocking conditions.
---
### High Severity
#### [HIGH-01] Internal-only `error_detail` is rendered directly to users, reopening the leak the 2026-08-23 fix was meant to close
- **Location:**
- Write side: `src/transcription/errors.py:99-139` (`classify_unexpected_error` → `detail`, `format_error_detail` → persisted text)
- Persist: `src/transcription/services/workflows.py:720` (`error_detail=format_error_detail(page.error)`)
- **Render (Source Detail):** `src/transcription/ui/pages/sources_page.py:481-484`
- **Render (Sources list):** `src/transcription/ui/pages/sources_page.py:121` → `src/transcription/ui/components/table/sources.py:39,90-95` ("Error Detail" column)
- **Render (Maintenance):** `src/transcription/ui/pages/settings_page.py:562`, written by `src/transcription/services/maintenance.py:206`
- Contract violated: `docs/error_handling.md:107-114`; `.github/instructions/error-handling.instructions.md:86`; `docs/invariant/error_handling.md:59`; `docs/invariant/ai_evidence_and_provenance.md:103`
- **Reachability:** **Live.** Concrete path, no configuration required: a page fails with any
non-`AppError` exception → `workflows.py:388` calls `classify_unexpected_error(exc)` →
`errors.py:118` sets `detail=f"{type(exc).__name__}: {exc}"` → `format_error_detail`
(`errors.py:135-139`) emits `... | detail=OSError: [Errno 13] Permission denied: '/app/uploads/documents/<uuid>/page-1.jpg' | ...`
→ persisted to `ExecutionAttempt.error_detail` → rendered verbatim at
`sources_page.py:484` and in the `/sources` table column. A SQLAlchemy `OperationalError`
carries the database path by the same route.
- **Problem & Consequence:** `docs/error_handling.md:110` states `detail` is *"Internal only"* and
that its only surfaces are `format_error_detail` (evidence) and logs;
`error-handling.instructions.md:86` says *"Never rendered to users or serialized into an
envelope."* The UI reads it anyway. The consequence is not hypothetical drift — it is the
precise defect the previous review's fix existed to prevent. That fix made `message` generic and
moved the root cause to `detail` on the stated grounds that `detail` never reaches users. That
premise was never true: `error_detail` had a UI consumer the whole time. The result is that the
filesystem-path leak was relocated from the notification banner to the Source Detail card and
the Sources table, while the test suite records the leak as fixed
(`tests/test_errors.py:56-78`).
The inconsistency is visible inside a single file: `sources_page.py:443` deliberately routes a
stored path through `public_media_path_label` (`ui/components/media_urls.py:58-72`), which
correctly degrades an absolute path to its bare filename — and then `sources_page.py:484`
renders unsanitized text that may contain an absolute path.
- **Blast Radius:** Enumerated by grepping every reader of `.detail` and `error_detail`:
- `errors.py:137` — `format_error_detail`, the only reader of `AppError.detail`. **Must keep the root cause.**
- `services/workflows.py:720` — the only writer of `ExecutionAttempt.error_detail`.
- `services/maintenance.py:206` — the only writer of `MaintenanceRun.error_detail`.
- `services/evidence.py:195` — `build_evidence_export` emits `error_detail`. Export is an
operator-initiated evidence artifact; per invariant 3.7.1 it **must** retain it.
- `db/models.py:508-522` — `Source.latest_error_detail` projection, consumed only by `sources_page.py:121`.
- `ui/pages/sources_page.py:481-484`, `ui/components/table/sources.py`, `ui/pages/settings_page.py:562` — the three render sites.
- Tests asserting on persisted text: `tests/test_v42_evidence.py:284`,
`tests/services/test_workflows_reliability.py` (timeout detail),
`tests/services/test_maintenance_service.py`. A fix that changes *what is stored* breaks these;
a fix that changes *what is displayed* does not.
- **Recommendation — two invariants conflict here; both must be named.**
**Invariant 1 (evidence):** `ExecutionAttempt.error_detail` must retain the root cause.
`docs/requirements.md:30` (REQ-4-021) and `docs/invariant/ai_evidence_and_provenance.md:33`
require it; guarded by `tests/test_v42_evidence.py::test_attempts_are_append_only_and_exported_with_integrity`
and `tests/services/test_workflows_reliability.py`.
**Invariant 2 (privacy):** user-facing surfaces must not expose local filesystem details.
`docs/invariant/error_handling.md:59`; guarded (partially) by
`tests/test_errors.py::test_unexpected_error_does_not_leak_filesystem_paths`.
**The over-correction to avoid is stripping root-cause text out of `detail` or
`format_error_detail` to make the UI safe.** That is exactly the mistake documented in the
reviewer skill's worked example, and it would silently destroy the provenance record this
system exists to preserve while making every guard still pass.
Fix at the **render** boundary, not the write boundary. Add a presentation-layer projection and
route all three UI sites through it, leaving the persisted evidence untouched:
```python
# src/transcription/ui/components/error_presenter.py (new)
def display_failure_detail(error_detail: str | None) -> str | None:
"""Render persisted failure detail without machine-local paths.
`ExecutionAttempt.error_detail` is provenance and keeps the full root cause
(docs/error_handling.md). This projection is the only thing a page may show.
"""
```
It should preserve the `[category]`, `suggestion=`, and `error_id=` segments (which are what
make the display actionable) and reduce any absolute path inside `detail=` to its basename,
mirroring `public_media_path_label`. The operator keeps diagnosability — required by
`docs/ui/pages/sources.md:43` and `docs/requirements.md:59` (REQ-6-014) — without the container
filesystem layout being published to the browser.
Then decide and record which resolution was chosen: either the UI shows the sanitized
projection (recommended), or `docs/error_handling.md:107-114` and
`error-handling.instructions.md:86` are revised to state that operator-facing evidence displays
may render `error_detail` **and** that the guarantee moves to "no machine-local detail ever
enters `detail`" — which would be a much harder guarantee to keep. Do not leave the current
state, where the docs claim one thing and three pages do another.
- **Effort:** M
---
#### [HIGH-02] Absolute filesystem paths are embedded in user-facing `AppError.message` at five explicit raise sites
- **Location:**
- `src/transcription/services/sources.py:856` — `f"Prompt file not found: {prompt_path}"`
- `src/transcription/services/sources.py:864` — `f"Prompt file is empty: {prompt_path}"`
- `src/transcription/services/sources.py:914` — `f"Source file not found: {path}"`
- `src/transcription/services/prompts.py:99` — `f"Prompt directory is unavailable: {root}"`
- `src/transcription/services/prompts.py:186-191` — `_filesystem_error` builds `f"{message}: {exc}"`
- Contract violated: `.github/instructions/error-handling.instructions.md:74,85`; `docs/invariant/error_handling.md:59`
- **Reachability:** **Live**, on an ordinary user path. `sources.py:845` resolves
`prompt_root = runtime_settings.prompt_dir.resolve()`, so `prompt_path` is absolute
(`/app/prompts/transcribe_document.md` in the container). `load_prompt_text` is invoked by
`build_prompt_execution` (`sources.py:829-831`), which runs on **every document upload** via
`services/store.py:94` and `store.py:162`. The resulting `PromptLoadError` is an `AppError`
subclass, so it flows through `run_ui_action` → `show_error`
(`ui/components/error_presenter.py:51-66`), which renders `error.message` into both a
`ui.notify` banner and a card label, and through `build_error_envelope` (`errors.py:88-96`)
into API responses.
- **Problem & Consequence:** `error-handling.instructions.md:85` requires `message` to *"Stay
generic. Never embed exception text, provider payloads, or filesystem paths."* These five sites
embed exactly that. `prompts.py:186-191` violates the rule in **both** directions at once: it
puts `{exc}` — an `OSError` whose `str()` includes the offending filename — into `message`, and
it sets **no `detail=`**, so the internal field that is supposed to carry the root cause is
empty while the user-facing field carries all of it.
This is not a new regression; it is coverage that the existing guard never had.
`tests/test_errors.py:56-78` verifies only that `classify_unexpected_error` — the *catch-all*
path — does not leak. Every deliberate `raise SomeError(f"... {path}")` in the codebase is
outside its scope, so the suite reports the invariant as enforced while five live sites violate it.
- **Blast Radius:** Verified by grepping all consumers of these exception types.
`PromptLoadError`/`PromptStoreError`/`TranscriptionError` messages are consumed by:
`ui/components/error_presenter.py:55,63` (render), `errors.py:92` (API envelope),
`errors.py:135` (`format_error_detail` → evidence). Because the recommended change *adds* a
`detail` and *shortens* `message`, `format_error_detail` output still contains the path — so
evidence value is preserved, not reduced. Tests asserting on these messages:
`tests/test_prompts.py`, `tests/services/test_prompt_store.py`,
`tests/services/test_transcription_service.py`. These assert on message prefixes
(`"Prompt file not found"`), not on the interpolated path, and were checked to survive the change —
but re-run them, since `prompts.py:186` currently produces a message whose suffix some
assertion could depend on.
- **Recommendation:** Apply the pattern `errors.py:113-119` already establishes — generic
`message`, root cause on `detail`, `raise ... from exc`. Use `path.name` when a filename is
genuinely useful to the user.
```python
# sources.py:855 — before
raise PromptLoadError(f"Prompt file not found: {prompt_path}", ...)
# after
raise PromptLoadError(
f"Prompt file not found: {prompt_path.name}",
category=ErrorCategory.INFRA_PERSISTENT,
suggestion="Verify PROMPT_DIR and prompt file configuration, then retry.",
detail=f"Prompt file missing at {prompt_path}",
)
# prompts.py:186 — before
return PromptStoreError(f"{message}: {exc}", category=..., suggestion=...)
# after
return PromptStoreError(
message,
category=ErrorCategory.INFRA_PERSISTENT,
suggestion="Check prompt directory permissions and available disk space, then retry.",
detail=f"{type(exc).__name__}: {exc}",
)
```
Then widen the guard so this class cannot recur — see MED-07. Note the dependency: HIGH-02 and
HIGH-01 must be fixed **together**, because moving the path from `message` to `detail` while the
UI still renders `error_detail` relocates the leak instead of closing it. That is the same
mistake that produced HIGH-01.
- **Effort:** S (fix) / M (with the guard)
---
### Medium Severity
#### [MED-01] The orphan sweep matches by bare name and therefore cannot detect a dead module
- **Location:** `tests/test_orphan_sweep.py:101-163` (`_public_definitions`, `_orphans`)
- **Reachability:** **Live** — the guard is running now and reporting a clean sweep that is not clean.
- **Problem & Consequence:** `_public_definitions()` keys definitions by bare name
(`definitions[node.name]`, line 111) and `_orphans()` marks a definition referenced if that
bare name appears **anywhere** in `src/`, `tests/`, or `tools/` (lines 157-162). Two different
modules that define the same public name are therefore indistinguishable, and neither can ever
be reported as an orphan.
`src/transcription/ui/pages/tags_page.py` demonstrates the consequence. Its only public
definition is `register_page` (line 18). Seven live page modules define a function of the same
name and `ui/__init__.py:37-43` calls all seven — so `register_page` is heavily referenced and
`tags_page.register_page` is scored as reachable. In fact **nothing imports `tags_page` at all**
(verified: the only repo-wide references to the module are the file itself and
`tests/ui/test_tags_page.py`, which merely asserts the route 404s). 93 lines of code, including a
lazy-load-unsafe relationship traversal at `tags_page.py:71-74`, sit outside the sweep's reach.
The sweep also never asks whether a *module* is imported, only whether its definitions' names
appear somewhere — so this is a structural gap, not a one-off miss.
- **Blast Radius:** `tests/test_orphan_sweep.py` only; `KNOWN_ORPHANS` entries are keyed by the
same bare/dotted names and would need re-keying if qualification is added. Expect the stricter
sweep to surface additional true orphans on first run — triage them into `KNOWN_ORPHANS` with
rationales rather than weakening the check.
- **Recommendation:** Qualify definitions by module (`f"{module_path}:{name}"`) and add a separate,
cheap module-reachability pass: a module under `src/transcription/` is reachable if any other
module imports it, or it is a declared entrypoint (`app.py`, `__main__.py`, `worker_service.py`).
Report unreachable modules as orphans in their own right. Also fix
`test_public_definitions_are_discovered` (line 169), whose `>= 420` snapshot threshold is a
weak assertion that drifts upward silently — the 2026-08-23 review already flagged the same
pattern at the then-current `>= 200` and it was raised rather than replaced.
- **Effort:** M
---
#### [MED-02] Canonical invariant document declares a V4 baseline while the canonical baseline is V6.1
- **Location:** `docs/invariant/ai_evidence_and_provenance.md:130`
- **Reachability:** **Live** (documentation), no runtime impact.
- **Problem & Consequence:** Section 6.1 reads *"Canonical V4 architecture, schema, requirements,
and error-policy documents define how current behavior satisfies this invariant."*
`docs/index.md:1,29-32` establishes V6.1 as the baseline and states that every canonical document
asserts the same baseline. This is the **ownership clause of the invariant that governs the
entire evidence model** — the clause that tells a reader which documents are authoritative — and
it points at a superseded generation. A reader following it lands on stale authority precisely
when resolving an evidence question, which is the highest-stakes case.
A baseline-currency guard **does** exist —
`tests/test_meta_contract_guards.py::test_canonical_docs_declare_one_consistent_baseline`
(lines 89-112) — and `docs/invariant/ai_evidence_and_provenance.md` is **not** in
`BASELINE_SCAN_EXCLUSIONS` (lines 56-64), so the file is scanned. The claim escapes for two
independent reasons, either of which alone would be sufficient:
1. `_CURRENT_VERSION_CLAIM` (line 67) matches only the words `current` or `active` before a
version. This line says "**Canonical** V4", a third phrasing the pattern does not know.
2. Both patterns require `V(\d+\.\d+)` — a mandatory minor version. The bare token `V4` cannot
match either regex under any phrasing.
The guard is therefore not absent but *phrase-shaped*: it enforces currency only for the two
sentence forms someone thought of, against version strings that carry a minor. That is a weaker
property than its docstring implies ("Every canonical doc that names the current baseline must
name the same one").
- **Blast Radius:** Documentation only; no code reads this string. Widening the guard's patterns
will re-scan all canonical docs — expect it to surface further stale mentions on first run
(`docs/architecture.md`, `docs/schema.md`, `docs/requirements.md`, and `docs/error_handling.md`
each contain 2-3 version tokens), which should be triaged rather than excluded.
- **Recommendation:** Two parts, and the second matters more than the first.
1. Change "Canonical V4" to "Canonical V6.1" at line 130.
2. Fix the guard's shape rather than adding a third phrase to the list. Accept an optional minor
(`V(\d+)(?:\.(\d+))?`) and invert the matching: flag **every** `V<n>` token in a scanned
canonical doc that is not the declared baseline, rather than only those preceded by an
approved adjective. Phrase-list matching fails open — each new phrasing silently reopens the
hole — whereas token matching fails closed and forces an explicit exclusion.
See §8.1 for the alternative the maintainer is considering: dropping version labels from
canonical docs entirely, which removes the failure mode instead of guarding it.
- **Effort:** S
---
#### [MED-03] Settings resolve `.env.production` relative to the process working directory, and the isolation fix exists only in the test harness
- **Location:** `src/transcription/config.py:66-75` (`env_file=".env.production"`);
workaround at `tests/conftest.py:27-50`; guarded by `tests/test_config_isolation.py`;
depended on by `.github/workflows/quality-gate.yml` and `docker-compose.production.yml`
- **Reachability:** **Live but currently correct.** I verified the production path rather than
assuming it: `Dockerfile` sets `WORKDIR /app` in the runtime stage, and
`docker-compose.production.yml` mounts `./.env.production` to `/app/.env.production` for both the
`app` and `worker` services, so the relative path resolves correctly today.
- **Problem & Consequence:** Correct configuration loading depends on an **implicit, undocumented
contract between `config.py` and the process working directory.** Nothing in `config.py` states
it, and nothing tests it. The failure mode is not silent — `openrouter_api_key` is required with
no default, so a wrong cwd produces a `ValidationError` at startup rather than a partially
configured process — which is why this is Medium rather than High.
The more telling symptom is what the coupling forced on the test harness. `conftest.py:45-50`
cannot escape it by passing an argument; it must **mutate the Pydantic class-level
`model_config` dict at runtime** and restore it in a `finally`. That is a global, order-sensitive
side effect adopted because the module offers no seam. It also silently repairs a second
consumer: `ui/runtime_settings_store.py:402` reads the same `Settings.model_config["env_file"]`
to decide where the Settings page writes. Two subsystems are coupled through a mutable class
attribute.
- **Blast Radius:** Every `Settings` construction. Consumers of `model_config["env_file"]`:
`ui/runtime_settings_store.py:402` (write-target resolution, contract documented at
`docs/ui/pages/settings.md:27`) and `tests/conftest.py:45-50`. A change must preserve the
documented three-step resolution order — explicit override, `RUNTIME_SETTINGS_ENV_FILE`, then the
configured default — or `docs/ui/pages/settings.md:27` becomes wrong.
- **Recommendation:** Introduce one explicit resolution function that both `Settings` construction
and `runtime_settings_store` call, honoring an `ENV_FILE` environment variable and falling back
to a path anchored to a known root rather than to `os.getcwd()`. Tests then pass a path instead of
mutating class state, and `tests/test_config_isolation.py` can assert against the seam rather
than against the monkeypatch. If instead the cwd contract is accepted as deliberate, document it
in `config.py` and in `docs/production-runbook.md` and add a guard asserting `WORKDIR`/cwd
alignment — an implicit contract with a container image is exactly the kind of rule the invariant
routing table exists to place.
- **Effort:** M
---
#### [MED-04] Stale-job reclaim threshold is not derived from maximum job duration; safety currently comes from single-process sequencing
- **Location:** `src/transcription/config.py:116-117`
(`worker_provider_timeout_seconds=30.0`, `worker_stale_job_seconds=30.0`);
sweep at `src/transcription/worker.py:222-228`; reclaim at
`src/transcription/services/jobs.py:242-268`
- **Reachability:** **Latent.** Unblocked by *either* of: (a) running more than one worker replica
(adding `deploy.replicas > 1` to the `worker` service in `docker-compose.production.yml`), or
(b) setting `RUN_EMBEDDED_WORKER=true` on the `app` service while the standalone `worker`
container is also running. It is safe today only because
`docker-compose.production.yml` sets `RUN_EMBEDDED_WORKER: "false"` on `app` and defines exactly
one `worker`, and because within a single loop `run_worker_loop` awaits
`process_next_queued_job` to completion before returning to the stale sweep — so the sweep can
never observe a job that this same process is actively working.
- **Problem & Consequence:** The stale threshold (30s) **equals** the per-page provider timeout
(30s), leaving zero margin even for a single-page job. A multi-page document is legitimately
`PROCESSING` for up to N × 30s. `Job.date_updated` carries an `onupdate`
(`db/models.py:372-375`), but between the initial claim and the terminal write the only touch
is `sources.py:522-523` reassigning `job.provider`/`job.model` to values they usually already
hold, which SQLAlchemy resolves to no net change and therefore no `UPDATE`. I did not empirically
confirm the no-`UPDATE` behavior, so treat that specific step as unverified — but the finding does
not depend on it, because even a per-page refresh leaves only a 30s margin against a 30s timeout.
With a second concurrent worker, the sweep would requeue a job that is mid-provider-call. Both
workers then process the same job, producing duplicate `ExecutionAttempt` rows for the same
logical work and racing terminal status writes. Append-only history would be *preserved* but no
longer *faithful*: the evidence would show attempts that do not correspond to distinct
application decisions.
This is worth flagging because `jobs.py:191-197` explicitly implements and documents
`SKIP LOCKED` row locking "so concurrent workers never contend for the same job." The claim path
is built for multi-worker operation; the reclaim path is not. A reader who trusts the claim
docstring would reasonably scale the worker.
- **Blast Radius:** `requeue_stale_processing_jobs` has one production caller (`worker.py:226`) and
tests in `tests/test_worker.py` and `tests/services/test_job_service.py`. Changing the *default*
affects `tests/test_config.py` declared-defaults assertions — check those before editing the default.
- **Recommendation:** **Fix before adding a second worker replica, not after.** Two parts:
(1) Make the threshold a function of the real bound rather than a coincidental peer of the
page timeout — at minimum default `worker_stale_job_seconds` to a multiple of
`worker_provider_timeout_seconds` with headroom, and add a model validator rejecting a stale
threshold at or below the provider timeout.
(2) Preferably make reclaim heartbeat-based: have `_persist_page_outcome` bump `Job.date_updated`
explicitly so liveness reflects progress rather than elapsed time since claim.
Add a guard asserting a multi-page job in flight is not reclaimed by a concurrently-invoked sweep.
- **Effort:** M
---
#### [MED-05] Provider evidence capture is per-instance mutable state, making the adapter non-reentrant by contract
- **Location:** `src/transcription/providers/openrouter.py:197-199, 264-267, 274-275, 297-298, 397-412`;
`_CapturingAsyncClient.last_response`/`last_body` at `openrouter.py:66-94`;
contract at `src/transcription/providers/base.py:110-118`
(`current_request_manifest`, `current_transport_evidence`)
- **Reachability:** **Latent.** Unblocked by any concurrent `transcribe()` on a single adapter
instance — most plausibly by processing a job's pages in parallel (`workflows.py:274` is
currently a sequential `for` loop) or by any second consumer sharing one
`SourceService.provider`. Verified safe today: `workflows.py:272` resolves one provider for the
loop and awaits each page; the worker's `ServiceBundle` (`worker.py:206`) is distinct from
`app.state.services` (`app.py:43`), so the UI cannot share the worker's adapter instance, and
the UI only enqueues jobs (`ui/pages/jobs_page.py:186-208`).
- **Problem & Consequence:** The `TranscriptionProvider` protocol defines evidence retrieval as
"the most recent call" state read *after* the fact. `workflows.py:369-370` relies on this on the
timeout path, reading `provider.current_request_manifest` / `current_transport_evidence` when no
result object exists. Under concurrency, page B's response overwrites
`_CapturingAsyncClient.last_response` before page A's timeout handler reads it, and page A's
`ExecutionAttempt` is written with page B's transport evidence.
The consequence is **evidence mis-attribution** — a provenance-integrity failure, which this
project's own rubric treats as its most serious class. It would also be near-undetectable after
the fact: the attempt row would be well-formed, internally consistent, and wrong. The
application-level design that makes this safe (sequential pages) is not expressed in the
provider contract, so the constraint lives only in `workflows.py`'s loop structure.
- **Blast Radius:** Changing the protocol touches `providers/base.py:102-136`,
`providers/openrouter.py:221-231`, the two read sites at `workflows.py:369-370`, and the fakes in
`tests/providers/test_openrouter.py`, `tests/services/test_workflows_reliability.py`, and
`tests/test_provider_boundaries.py`, all of which implement or assert the current property-based
contract.
- **Recommendation:** **Fix before introducing any intra-job page concurrency.** The durable fix is
to stop returning evidence through instance state: attach `request_manifest` and
`transport_evidence` to the raised exception on every failure path — which `ProviderError`
already supports (`providers/base.py:18-29`) and which the timeout path cannot currently use
because `asyncio.wait_for` raises `TimeoutError` from outside the adapter. A narrower option is
to have `transcribe()` accept a caller-owned capture sink so evidence is scoped to the call
rather than to the adapter. As an immediate, near-zero-cost step, document the non-reentrancy on
the protocol in `providers/base.py` so the constraint is visible where it is depended upon.
- **Effort:** M
---
#### [MED-06] Provider error bodies reach user-facing text while three provider failure paths persist no `detail`
- **Location:** `src/transcription/services/sources.py:923-947` (`handle_transcription_errors`);
message construction at `src/transcription/providers/openrouter.py:414-431`
(`_transport_error_message`)
- **Reachability:** **Live** for the message half (any provider failure during a UI-initiated
transcription surfaces through `show_error`).
- **Problem & Consequence:** Two mirrored halves of the same rule are broken in one function.
- `sources.py:943` builds `f"Provider transcription failed: {exc}"`, and `exc` is a
`ProviderError` whose message may embed up to 500 characters of the provider's error body
(`openrouter.py:430`). That is a provider payload in `message`, which
`error-handling.instructions.md:85` explicitly forbids.
- None of the three handlers (lines 929, 935, 942) passes `detail=`. Per
`error-handling.instructions.md:89-92`, omitting it degrades the provenance record.
I checked whether the provenance half is actually harmful before reporting it, and it is
**substantially mitigated**: `workflows.py:391` calls `_find_provider_error`, which walks
`__cause__`/`__context__` (`workflows.py:806-813`) to recover the original `ProviderError` and
persists its `transport_evidence` — status code, safe headers, and the exact response body — onto
the attempt. So the root cause is preserved in transport evidence even though `error_detail` is
thin. This is why the finding is Medium rather than High. The residual cost is that the
human-readable failure summary is uninformative for the two paths (`ProviderAuthError`,
`ProviderResponseError`) whose messages are entirely generic.
- **Blast Radius:** `handle_transcription_errors` is used on the transcription path in
`sources.py`; `TranscriptionError.message` is consumed by `error_presenter.show_error`,
`build_error_envelope`, and `format_error_detail`. Assertions on these messages live in
`tests/services/test_transcription_service.py` and `tests/providers/test_openrouter.py`.
- **Recommendation:** Move the interpolated provider text from `message` to `detail` on all three
handlers, keeping the generic message the other two already use:
```python
except ProviderError as exc:
raise TranscriptionError(
"Provider transcription failed",
category=ErrorCategory.EXTERNAL_PROVIDER,
suggestion="Retry the transcription from jobs. If repeated, check provider availability.",
retriable=True,
detail=f"{type(exc).__name__}: {exc}",
) from exc
```
Apply the same `detail=` addition to the `ProviderAuthError` and `ProviderResponseError`
handlers. Note the interaction with HIGH-01: until the render boundary is sanitized, moving text
into `detail` still reaches users through the `error_detail` display. Sequence accordingly.
- **Effort:** S
---
#### [MED-07] No deterministic guard covers the `message`/`detail` split at explicit raise sites
- **Location:** `tests/test_errors.py:56-78`; rule at
`.github/instructions/error-handling.instructions.md:78-98`; canonical statement at
`docs/error_handling.md:102-115`
- **Reachability:** **Live** — this coverage gap is what allowed HIGH-02 and MED-06 to exist in a
fully green suite.
- **Problem & Consequence:** `docs/error_handling.md:115` names
`tests/test_errors.py::test_unexpected_error_does_not_leak_filesystem_paths` as the enforcement
for the message/detail split. That test exercises exactly one function,
`classify_unexpected_error`. Every direct `raise SomeAppError(...)` in `src/` — roughly 50 sites
by grep — is unenforced. The documentation therefore overstates the enforcement, which is worse
than having no guard: a contributor reading `error_handling.md:115` reasonably concludes the rule
is mechanically protected.
Per the reviewer skill, where a check is unenforced, recommending the deterministic test is
itself a finding.
- **Blast Radius:** Tests only.
- **Recommendation:** Add an AST guard, `tests/test_error_message_safety.py`, that scans `src/`
for `raise <AppError subclass>(...)` and fails when the first positional argument is an f-string
containing a formatted value whose name matches a path-like or exception-like identifier
(`path`, `_path`, `root`, `dir`, `exc`, `err`, `e`). Model it on the existing AST guards, which
are the established pattern here (`test_ui_boundaries.py`, `test_service_boundaries.py`,
`test_orphan_sweep.py`). Pair it with a second guard asserting that no UI module reads
`error_detail` without routing through the sanitizing projection from HIGH-01 — that one closes
the render side, which is where the real leak is.
- **Effort:** M
---
### Low Severity
#### [LOW-01] `ui/pages/tags_page.py` is dead code; the V6.1 roadmap is correct
- **Location:** `src/transcription/ui/pages/tags_page.py` (93 lines);
registration list at `src/transcription/ui/__init__.py:37-43`
- **Reachability:** **Not reachable.** This resolves the flagged open item: the route is genuinely
**not** registered. `register_pages` calls seven page registrars and `tags_page` is not among
them; nothing anywhere imports the module. `docs/roadmap_plan.md:47` ("Retire the Tags page") is
accurate, and `tests/ui/test_tags_page.py` correctly asserts `/ui/tags` returns 404 — though it
passes trivially, since an unimported module cannot register anything.
- **Problem & Consequence:** No runtime risk; purely stranded code. It is worth noting that if it
*were* ever re-registered, `tags_page.py:71-74` traverses `document.document_tags` and
`link.tag_ref` inside a page render, and those relationships are configured `lazy="raise"`
(`docs/architecture.md:200-203`) — so re-enabling this module without adding eager loads to
`list_documents` would raise on first render.
- **Recommendation:** Delete `src/transcription/ui/pages/tags_page.py`. Retain
`tests/ui/test_tags_page.py` as the retirement guard. Fixing MED-01 first would make this
finding reproducible by the suite rather than by manual inspection.
- **Effort:** S
#### [LOW-02] `benchmarking.py` ships in the runtime package but is referenced only by tests
- **Location:** `src/transcription/benchmarking.py` (69 lines); sole consumers
`tests/test_v42_evidence.py:15-16` (`EditorialAssessment`, `score_transcription`)
- **Reachability:** Live as importable API; never invoked by application code.
- **Problem & Consequence:** No defect. It supports the model-evaluation policy in
`docs/invariant/ai_evidence_and_provenance.md:113-126`, which is legitimate, but it currently has
no production caller and no tooling entrypoint, so it is indistinguishable from drift.
- **Recommendation:** Either move it under `tools/` alongside the other operator utilities, or add
a `KNOWN_ORPHANS`-style rationale recording that it is retained as the evaluation-policy
implementation. Do not silently keep it unlabeled.
- **Effort:** S
#### [LOW-03] Two overlapping prompt error types split across modules
- **Location:** `src/transcription/services/errors.py:15-16` (`PromptLoadError`) and
`src/transcription/services/prompts.py:19` (`PromptStoreError`)
- **Reachability:** Live; no misbehavior observed.
- **Problem & Consequence:** `services/errors.py:1-8` documents itself as the neutral home for
exceptions raised by more than one service, precisely so a caller's `except` clause does not
change when an operation moves. `PromptStoreError` is defined outside that module and covers an
overlapping domain (prompt file access), so a caller wanting to handle "any prompt failure" must
import from two modules and know which is which. `sources.py:855` raises `PromptLoadError` for a
missing prompt file while `prompts.py:132` raises `PromptStoreError` for the same condition
reached through the Settings page.
- **Recommendation:** Move `PromptStoreError` into `services/errors.py` next to `PromptLoadError`,
or make one a subclass of the other so a single `except` covers prompt failures. Low urgency; do
it opportunistically when HIGH-02 touches both files anyway.
- **Effort:** S
---
## 4. Architectural Drift & Gap Analysis
| Area / Component | Direction | Documented / Intended Rule | Actual Implementation State | Severity | Recommended Resolution |
| :--- | :--- | :--- | :--- | :--- | :--- |
| Error presentation | `doc->code` | `docs/error_handling.md:110` — `detail` is internal only, surfaced by `format_error_detail` and logs | `sources_page.py:484`, `table/sources.py:90`, `settings_page.py:562` render `error_detail` verbatim to users | High | Sanitizing render projection (HIGH-01); do **not** strip `detail` |
| User-facing messages | `doc->code` | `invariant/error_handling.md:59` — no local filesystem detail in user-facing messages | 5 live sites interpolate absolute paths into `AppError.message` | High | Generic `message`, path on `detail` (HIGH-02) |
| Evidence invariant ownership | `doc->doc` | `docs/index.md:1` — baseline is V6.1 | `invariant/ai_evidence_and_provenance.md:130` names "Canonical V4"; the currency guard scans the file but its regexes match neither the phrasing nor a minor-less `V4` | Medium | Update text; make the guard token-based, or drop version labels entirely (MED-02, §8.1) |
| Enforcement claim | `doc->code` | `docs/error_handling.md:115` — split "Enforced by `tests/test_errors.py::…`" | That test covers only `classify_unexpected_error`; explicit raises unguarded | Medium | Add AST guard (MED-07) |
| Orphan sweep | `doc->code` | `test_orphan_sweep.py:1-13` — sweep is "deterministic" and "conservative" | Bare-name matching; cannot see a dead module (`tags_page.py`) | Medium | Qualify by module + module-reachability pass (MED-01) |
| Worker scaling | `code->doc` | `jobs.py:191-197` — `SKIP LOCKED` so "concurrent workers never contend" | Claim path is multi-worker-safe; stale-reclaim path is not | Medium | Derive stale threshold from job duration; document single-worker constraint until fixed (MED-04) |
| Provider adapter contract | `code->doc` | `providers/base.py:110-118` — evidence read as "most recent call" state | Contract is silently non-reentrant; safety lives in `workflows.py`'s sequential loop | Medium | Scope evidence to the call; document non-reentrancy (MED-05) |
| Settings env file | `code->doc` | `config.py:66-75` — `env_file=".env.production"` | Correctness depends on an undocumented cwd contract with `Dockerfile` `WORKDIR /app` | Medium | Explicit resolver seam, or document + guard the contract (MED-03) |
| Tags page | *(no drift)* | `roadmap_plan.md:47` — Tags page retired | Route genuinely unregistered; module is stranded code | Low | Delete the module (LOW-01) |
---
## 5. Invariant Inventory & Routing Recommendations
| Invariant / Constraint | Current Location | Recommended Target Layer | Rationale |
| :--- | :--- | :--- | :--- |
| `detail`/`error_detail` never rendered to users | docs + instructions | **Deterministic test** + sanitizing projection | Stated in three documents and violated in three files; prose has demonstrably failed to hold it |
| `message` carries no paths or exception text | instructions; partial test | **Deterministic test** (AST, all raise sites) | Existing guard covers one function; the gap produced HIGH-02 |
| `ExecutionAttempt.error_detail` retains root cause | docs + `test_v42_evidence.py` | **Keep in tests** — already correct | Counterweight to the above; must be named in any fix so it is not over-corrected |
| Intermediate pages commit individually | `workflows.py` docstring + `test_workflows_reliability.py:318` | **Keep in tests** — verified genuine | Cross-session read makes it a real durability assertion |
| Final page atomic with terminal status | `services.instructions.md` + `test_pipeline_atomicity.py` | **Keep in tests** — verified genuine | Fault injection makes a split commit fail |
| Canonical baseline version consistency | `docs/index.md` + `test_meta_contract_guards.py:89` | **Repair existing test, or remove the labels** | Guard exists but matches by approved phrase and requires a minor version, so it fails open on new phrasings (MED-02) |
| Module-level reachability / dead modules | `test_orphan_sweep.py` (ineffective) | **Deterministic test** (repair existing) | Guard exists but cannot detect the case (MED-01) |
| Stale threshold > max job duration | *(unenforced)* | **Config validator + test** | Currently a coincidence of two equal defaults (MED-04) |
| Provider adapter non-reentrancy | *(unenforced, implicit)* | **Instructions** + protocol docstring | A design constraint callers must know before adding concurrency (MED-05) |
| Env-file resolution independent of cwd | `tests/conftest.py` monkeypatch | **Code seam** + `docs/production-runbook.md` | A test-only fix for a production coupling is misrouted enforcement (MED-03) |
---
## 6. Stack-Specific Analysis
**Python 3.12+.** Modern and consistent. PEP 695 generics are used correctly and non-trivially
(`RegistryService[ModelT: RegistryEntry]` in `services/registry.py:58`, `UiActionOutcome[T]`,
`_get_or_raise[ModelT]`), `type` statements appear in `db/session.py:15,50`, and `X | None` is
used throughout. `structural Protocol` bounds (`RegistryEntry`, `WorkerNotifier`,
`TranscriptionProvider`) are used to avoid type suppressions rather than to decorate. `ty` passes
clean with no suppressions found. The two `# noqa` uses (`workflows.py:228` `PLR0915`,
`workflows.py:383` `BLE001`) are both justified in context — the broad catch is a deliberate
per-page containment boundary that immediately classifies and re-records.
**FastAPI.** Lifespan is handled via `@asynccontextmanager` (`app.py:36`), not the deprecated
`@app.on_event`. Session factories are injected through `Depends` (`SessionFactoryDep`,
`db/session.py:50`) rather than reached as globals from routes. `api/errors.py` centralizes
envelope translation. One residual: `get_settings` is `@cache`d and read as a module-level
fallback in ~10 modules; this is acceptable given the documented restart-to-apply contract
(`docs/ui/pages/settings.md:28`) but means the cache is process-lifetime and unclearable.
**NiceGUI (pinned `3.13.0`).** The pin is a recorded release-stability decision and is not
reported as a defect. Boundaries are enforced structurally: `test_ui_boundaries.py` uses an
import **allowlist**, which is the right polarity. Blocking work is dispatched off the event loop
via `run_blocking` (`settings_page.py:818,822`). The one boundary that is *not* enforced is
presentation of internal fields (HIGH-01) — pages are prevented from touching persistence but not
from rendering internal-only text.
**SQLModel / SQLAlchemy.** Strong. `lazy="raise"` on relationships forces explicit eager loading;
read paths declare `selectinload` chains with comments explaining *why* each is needed
(`sources.py:309-316` is a good example). `expire_on_commit=False` (`db/session.py:28`) is set
deliberately, which is what makes post-commit attribute access in `evidence.py:159-202` safe.
`claim_next_queued_job` (`jobs.py:186-241`) branches correctly on dialect — `SKIP LOCKED` on
PostgreSQL, conditional `UPDATE ... RETURNING` on SQLite — rather than assuming one engine.
Attempt-number allocation uses `begin_nested()` with bounded retry (`sources.py:598-616`), the
right pattern for a monotonic per-parent sequence. No N+1 patterns were found in the read paths
sampled.
**Pydantic V2 & Settings.** Fully V2; no `@validator`, `class Config`, `.dict()`, or `parse_obj`
anywhere. Evidence contracts use `ConfigDict(extra="forbid", frozen=True)` (`providers/evidence.py:47`),
which is exactly right for persisted provenance — an unexpected field fails loudly rather than
being silently dropped. `SecretStr` guards the API key. The discriminated
`SqliteSettings | PostgresSettings` union is clean. `normalize_provider_models` correctly runs
`mode="before"` so the derived tuple is produced by construction rather than by mutating a frozen
model — a subtlety that is easy to get wrong. Sole issue: the cwd-coupled `env_file` (MED-03).
**Asyncio Workers.** Notably careful. `asyncio.shield` wraps both the per-page commit and the
terminal commit (`workflows.py:601-614`, `650-663`), with the `except CancelledError: await task;
raise` pattern that actually completes the shielded work rather than merely deferring cancellation —
a detail most implementations get wrong. `handle_worker_exceptions` (`worker.py:157-182`)
distinguishes retriable from non-retriable faults and stops the loop rather than spinning.
`_advance_job_with_containment` (`worker.py`/`workflows.py:507-540`) guarantees a claimed job
cannot strand in `PROCESSING`. `worker_consumer_lifespan` has a bounded shutdown with escalation to
`cancel()`. Gaps are MED-04 and MED-05, both latent and both with stated unblocking conditions.
**OpenRouter / Adapter Boundary.** Encapsulation holds: `test_provider_boundaries.py` enforces it,
and `workflows.py` imports only `providers` abstractions. `_CapturingAsyncClient` is a
well-judged design — it captures the exact transport body before SDK parsing without altering what
the SDK consumes, including the streamed case. Timeout construction (`openrouter.py:200-206`)
correctly overrides httpx's 5s per-phase default that would otherwise silently cap the configured
budget. `SAFE_RESPONSE_HEADERS` (`providers/evidence.py:29-41`) was reviewed field-by-field:
all nine entries are non-secret correlation, content, or rate-limit headers, and
`filter_safe_response_headers` is a true allowlist filter with no redaction-after-capture — this
satisfies invariant 3.8.2 exactly. `_replace_embedded_media` correctly substitutes a source
reference for base64 payloads, satisfying 3.8.3. The one structural weakness is MED-05.
**Testing & Quality Tooling.** 410 tests, all green, with genuinely strong contract guards
(boundaries, model contract, media path safety, evidence append-only, atomicity). Marker strictness
and `asyncio_mode = "strict"` are configured, and no unawaited-coroutine warnings appeared. Two
guards, however, assert meaningfully less than their names and docstrings claim
(`test_orphan_sweep.py` — MED-01; `test_errors.py` path-leak coverage — MED-07), and the
`>= 420` snapshot threshold at `test_orphan_sweep.py:169` repeats a weak-assertion pattern the
2026-08-23 review already flagged at `>= 200`; it was raised rather than replaced with
set-membership.
---
## 7. Duplication & Consolidation Report
| Pattern / Duplication | Locations | Proposed Canonical Home | Estimated Lines Removed |
| :--- | :--- | :--- | :--- |
| `f"{type(exc).__name__}: {exc}"` detail construction | `errors.py:118`, `maintenance.py:71,95,135,206`, `runtime_settings_store.py:388,476,554` | `errors.py::exception_detail(exc)` | ~8 (consistency > line count) |
| Filesystem `AppError` construction from `OSError` | `prompts.py:186-191`, `runtime_settings_store.py:384-389,472-477,550-555` | `errors.py::filesystem_error(message, exc, *, suggestion)` | ~20 |
| Overlapping prompt error types | `services/errors.py:15`, `services/prompts.py:19` | `services/errors.py` (LOW-03) | ~5 |
| Duplicated `provider_duration_ms` / `processing_duration_ms` max-clamp arithmetic | `workflows.py:341-345, 361-368, 397-404` | `workflows.py::_page_durations(started_at, finished_at, monotonic_started_at)` | ~20 |
| `_utc_now_naive` defined per module | `workflows.py:51`, `jobs.py:29`, `db/models.py`, `sources.py` | Single helper in `db/models.py`, imported | ~12 |
### Proposed Canonical Abstractions
```python
# src/transcription/errors.py
def exception_detail(exc: BaseException) -> str:
"""Internal-only root-cause text for AppError.detail. Never user-facing."""
def filesystem_error[E: AppError](error_type: type[E], message: str, exc: OSError, *, suggestion: str) -> E:
"""Build a filesystem AppError with a generic message and the path on detail."""
# src/transcription/ui/components/error_presenter.py
def display_failure_detail(error_detail: str | None) -> str | None:
"""Sanitize persisted failure detail for UI rendering (HIGH-01)."""
```
---
## 8. Meta-Tooling & Instruction Update Recommendations
1. **`docs/invariant/ai_evidence_and_provenance.md:130`** — resolve the V4 label. Two viable
routes, and the maintainer has proposed the second:
- **(a) Repair the guard.** Fix the text to V6.1 and make
`test_canonical_docs_declare_one_consistent_baseline` token-based rather than phrase-based
(MED-02). Keeps version labels as navigational anchors.
- **(b) Remove version labels from canonical docs.** While the project has a single principal
user and no released versions to support, "canonical" and "current" are the same thing, so the
label carries no information a reader can act on — it only creates a second thing to keep in
sync. Retain the baseline declaration in `docs/index.md` alone as the release marker, keep
version language in `docs/roadmap_plan.md` and the migration/deployment docs (already
excluded from the scan for exactly this reason), and replace in-body references with
unversioned phrasing ("the canonical architecture, schema, requirements, and error-policy
documents"). The guard then inverts: assert that no canonical doc outside the exclusion set
contains a version token at all, which is a stricter and much cheaper property to hold than
agreement between many labels. Requirement IDs (`REQ-4-021`, `REQ-6-014`) are stable
identifiers, not currency claims, and should be left alone.
2. **`docs/error_handling.md:107-115`** — either add the sanitizing-projection rule for UI display
of `error_detail`, or revise the `detail` "Surfaces" row to admit operator-facing evidence
displays. Update the "Enforced by" line once MED-07's guard lands, since it currently overstates
coverage.
3. **`.github/instructions/error-handling.instructions.md`** — add an explicit clause under
"User-Safe Messaging" stating that *persisted* `error_detail` is subject to the same no-paths
rule at any render boundary. The current table (line 86) states the rule for `AppError.detail`
and stops there, so the persisted-then-rendered path falls between the lines.
4. **`.github/instructions/providers.instructions.md`** — record the adapter non-reentrancy
constraint (MED-05); it is currently an undocumented precondition of `workflows.py`.
5. **`.github/instructions/services.instructions.md`** — the two competing atomicity invariants are
well described and both guards verified; no change needed. Worth adding the stale-reclaim
threshold constraint (MED-04) alongside them, since it is a third worker-lifecycle rule with no
documented home.
6. **`tests/test_orphan_sweep.py`** — repair per MED-01 and replace the `>= 420` threshold with
set-membership assertions.
7. **New `tests/test_error_message_safety.py`** — AST guard per MED-07, covering both the raise
sites and the UI render sites.
8. **`docs/production-runbook.md`** — document the cwd/`WORKDIR` contract for `.env.production`
resolution if MED-03 is resolved by documentation rather than by a code seam.
---
## 9. Prioritized Dependency-Ordered Action Plan
**Phase 1 — Blocking fixes (privacy; ordered, HIGH-01 first)**
1. **HIGH-01** — add `display_failure_detail` and route `sources_page.py:484`,
`table/sources.py:90-95`, and `settings_page.py:562` through it. Do this **first**: it closes
the render boundary, so the Phase-1.2 fix cannot relocate a leak again.
2. **HIGH-02** — move paths from `message` to `detail` at the five sites, including the
`prompts.py:186` double violation.
3. **MED-06** — move provider payload text to `detail`; add `detail=` to all three
`handle_transcription_errors` handlers.
**Phase 2 — Enforcement hardening (make Phase 1 permanent)**
4. **MED-07** — AST guard for raise-site `message` safety **and** for UI reads of `error_detail`.
5. **MED-01** — qualify orphan definitions by module; add module-reachability; replace the
snapshot threshold.
6. **MED-02** — fix the V4/V6.1 text and extend the meta-contract guard to baseline-version currency.
**Phase 3 — Reliability & concurrency (latent; each must precede its unblocking change)**
7. **MED-04** — derive `worker_stale_job_seconds` from `worker_provider_timeout_seconds` with a
rejecting validator, ideally plus a progress heartbeat. **Must land before any second worker
replica.**
8. **MED-05** — scope provider evidence to the call rather than the instance. **Must land before
any intra-job page concurrency.** Document non-reentrancy immediately as an interim step.
**Phase 4 — Consolidation & refactoring**
9. **MED-03** — explicit env-file resolution seam shared by `Settings` and `runtime_settings_store`;
remove the `model_config` monkeypatch from `conftest.py`.
10. **LOW-01** — delete `tags_page.py` (after MED-01, so the suite reproduces the finding).
11. **LOW-03** and the §7 consolidations — fold in opportunistically while Phase 1 touches these files.
**Phase 5 — Non-blocking governance/documentation depth**
12. **LOW-02** — relocate or annotate `benchmarking.py`.
13. Instruction/doc updates §8.3–§8.5, §8.8.
---
## 10. Preserved Strengths
- **Append-only evidence is real, not aspirational.** Every provider call produces a distinct
`ExecutionAttempt`; no runtime path mutates a historical row. Projection writes onto
`Source.raw_transcription` are clearly separated from history, and `promote_machine_attempt`
(`evidence.py:117-146`) repoints the projection without rewriting evidence — with a docstring
that explains exactly why that one write lives in a read-oriented service.
- **Transport-layer terminology is honored in code.** `_CapturingAsyncClient` exists specifically so
the stored body is the application-boundary capture rather than an SDK-parsed object, and
`TransportEvidence(response_received=False)` explicitly represents "no response" instead of
conflating it with an empty one. This is invariant 3.4/3.5 implemented rather than asserted.
- **Header allowlisting is done the hard, correct way** — filter-before-store with an explicit
frozenset, never capture-then-redact (`providers/evidence.py:29-41,130-134`).
- **Boundaries are enforced by allowlist, not blocklist.** `test_ui_boundaries.py:20-25` states the
reasoning explicitly; it means a newly added persistence helper cannot slip through under an
unlisted name.
- **The two competing atomicity invariants are both correctly implemented and both genuinely
guarded**, with the tests structured so that the naive over-correction fails.
- **Cancellation safety in the worker is unusually well handled** — `asyncio.shield` plus
`await task` on `CancelledError` actually completes the commit rather than merely deferring
cancellation.
- **Comments explain rationale, not mechanics.** `workflows.py:269-271`, `openrouter.py:200-202`,
`config.py:114-115`, and `jobs.py:191-197` each record *why* a non-obvious choice was made,
several citing the review log entry that motivated it. This is what made verifying the atomicity
and timeout invariants tractable in this review.
- **Documentation-to-code traceability is strong overall.** Page contracts, schema field tables,
and requirement IDs are maintained and guarded; the drift found in this review is narrow and
specific rather than systemic.
---
## Appendix A — Repo-Specific Deterministic Checks
| # | Check | Result | Evidence |
| :-- | :--- | :--- | :--- |
| 1 | Service boundary rule: no service-to-service imports | **Pass** | `tests/test_service_boundaries.py` green; AST scan, allowlist-based; `workflows.py` composes via `ServiceBundle` |
| 2 | UI boundary rule: no persistence access from pages/components | **Pass (structurally)** | `tests/test_ui_boundaries.py` green. Caveat: it guards *data access*, not presentation of internal-only fields — see HIGH-01 |
| 3 | Status vocabulary conformance; no stringly-typed literals | **Pass** | `tests/test_model_contract_guards.py` green; enum members verified against `db/models.py` |
| 4 | Evidence ownership: append-only history, projections not history mutation | **Pass** | `test_v42_evidence.py::test_attempts_are_append_only_and_exported_with_integrity` verified non-vacuous (asserts both retained attempts and export integrity at lines 281-290) |
| 5 | Canonical authority: findings resolve against `docs/*` first | **Pass with defect** | `test_canonical_authority_references_are_present` green. The companion baseline-currency guard (`test_canonical_docs_declare_one_consistent_baseline`) scans the offending file but fails open on its phrasing and on minor-less version tokens — MED-02 |
| 6 | Schema contract fidelity: `docs/schema.md` field-accurate | **Pass** | `test_model_contract_guards.py` + `test_meta_contract_guards.py` green |
| 7 | Media boundary: record-validated media, controlled URL resolver | **Pass** | `test_media_path_safety.py`, `tests/ui/test_media_urls.py` green; `public_media_path_label` verified path-safe |
| 8 | Eager-loading conformance vs `lazy="raise"` | **Pass** | Declaration-side guard green; sampled read paths declare explicit `selectinload` chains. Note: dead `tags_page.py:71-74` would violate it if re-registered (LOW-01) |
| 9 | Cross-cutting error conformance | **FAIL** | Guards green but coverage is narrower than documented: HIGH-01, HIGH-02, MED-06, MED-07 |
| 10 | Orphan/dead-code conformance | **FAIL** | Guard green but structurally unable to detect a dead module: MED-01, proven by LOW-01 |
## Appendix B — Evidence & Provenance Auditor Families
| Family | Subject | Result | Evidence |
| :--- | :--- | :--- | :--- |
| A | Attempt history append-only | **Pass** | No update/delete path to `ExecutionAttempt`; insert-only with `begin_nested` + bounded sequence retry (`sources.py:556-616`) |
| B | Attempt numbering monotonic per source | **Pass** | `insert_with_sequence_retry`; uniqueness constraint plus retry on conflict |
| C | Transport evidence captured at the transport boundary | **Pass** | `_CapturingAsyncClient` retains the exact wire body pre-SDK-parse (`openrouter.py:66-94`) |
| D | Absent response distinguished from empty response | **Pass** | `TransportEvidence.response_received` is explicit, not inferred |
| E | Response header persistence is allowlist-based | **Pass** | `SAFE_RESPONSE_HEADERS` (`providers/evidence.py:29-41`) — all nine entries verified non-secret; filter-before-store |
| F | No machine-local detail on user-facing surfaces | **FAIL** | `error_detail` rendered verbatim at three UI sites (HIGH-01); paths in `message` at five sites (HIGH-02) |
| G | Request manifest excludes embedded media payloads | **Pass** | `_replace_embedded_media` (`openrouter.py:377-395`) substitutes a source reference for base64 data |
| H | Evidence attribution is correct under concurrency | **Pass today / at risk** | Correct in the current sequential single-worker deployment; the contract itself is non-reentrant (MED-05) and reclaim has no margin (MED-04) |
docs/reviews/README.md
-21
View File
@@ -1,21 +0,0 @@
# Review Reports
Dated architecture and code review reports generated by
`.github/skills/python-code-reviewer/skill.md`.
**These files are not canonical authority.** Everything in `docs/reviews/**` is a
point-in-time observation, not a contract. Canonical intent lives in `docs/index.md`,
`docs/architecture.md`, `docs/requirements.md`, `docs/schema.md`,
`docs/error_handling.md`, and `docs/invariant/**`. When a report and a canonical
document disagree, the canonical document wins until it is deliberately updated.
Naming: `<YYYY-MM-DD>-code-review.md` for review reports, and
`<YYYY-MM-DD>-remediation-handoff.md` for the implementation plan derived from one.
## Current
- [`2026-08-23-code-review.md`](./2026-08-23-code-review.md) — full review. 0 critical,
4 high, 5 medium, 11 low. **All findings remediated.** Retained as a record of the
reasoning, not as a list of open work. Note that a few of its recommendations were
wrong on contact and were corrected during implementation; the code and the guard
tests are authoritative over the report text.
docs/roadmap_plan.md
-210
View File
@@ -1,210 +0,0 @@
# Roadmap Plan (Starting at V6.0)
This roadmap starts at **V6.0** and tracks forward-looking work only.
## V6.0 - Hosting Migration
Objective: move from local-only operation to secure, stable remote hosting.
Status: **Completed**
Detailed plan: [`v6_0_hosting_migration_plan.md`](v6_0_hosting_migration_plan.md)
### Scope
1. Containerize app runtime for production deployment.
2. Run PostgreSQL in Docker and migrate from SQLite.
3. Add Cloudflare Tunnel exposure with Access protection.
4. Add operational safeguards (health checks, restart policies, backups).
### Deliverables
- Production-ready `docker-compose` deployment for app + database + tunnel.
- Environment-based configuration for DB, uploads, prompts, and logging.
- Verified data migration path into PostgreSQL.
- Runbook updates for deploy, rollback, and backup/restore.
### Exit Criteria
- `/healthz` reports healthy app and worker in deployed environment.
- One end-to-end document -> source -> job workflow succeeds remotely.
- Backup and restore procedure is tested.
### Accomplished
1. Delivered production Docker deployment with split `app`/`worker`, `postgres`, and `cloudflared`.
2. Landed SQLite -> PostgreSQL migration tooling and runbook coverage.
3. Added production health/reliability wiring and operational runbooks for deploy/rollback/recovery.
4. Established host-visible backup workflow and restore path for PostgreSQL plus media/config assets.
## V6.1 - Testing and Refinement
Objective: improve navigation and operational workflows after user feedback.
Status: **Completed**
### Scope
1. Make Document Detail the primary source-page workspace:
- Use Source-style pan/zoom + previous/next page controls.
- Move editable revision controls into Document Detail.
- Move archival/system metadata to dedicated Document Info route.
2. Simplify top navigation:
- Remove top-level Tags and Sources entries.
- Retire the Tags page and the global Source Asset Records entry flow.
3. Improve list/detail clarity:
- Add Document transcription status to Archival Documents list.
- Add Document Date in People Detail -> Linked Documents table.
4. Add worker-backed Settings maintenance runs:
- Add `maintenance_run` persistence (`id`, `job_type`, `status`, `started_at`, `finished_at`, `triggered_by`, `summary`, `log_path`, `error_detail`).
- Add Run Backup and Run Storage Reconciliation actions that enqueue runs and execute in the worker.
- Add run history with status, duration, summary, and log view/download.
- Defer daily/weekly scheduling controls to V6.2.
### Accomplished
1. Refactored Document Detail into the primary source-page workspace (pan/zoom viewer, previous/next page navigation, editable revision flow) and moved archival/system metadata to Document Info.
2. Simplified top navigation by removing Tags/Sources entries and retiring the Tags page/global Source Asset Records flow.
3. Improved data clarity with document transcription status in Archival Documents and Document Date in People Detail linked documents.
4. Implemented queue-backed maintenance operations (`maintenance_run` model/service/worker/UI) with run history and log view/download.
5. Hardened runtime settings operations in production:
- runtime settings writes target mounted `.env.production`,
- fallback write path for single-file bind mounts,
- explicit hidden/deployment-key disclosure in Settings UI.
6. Simplified backup configuration and behavior:
- standardized on `BACKUP_DIR` + `BACKUP_RETENTION_DAYS`,
- backup script uses `DATABASE__*` persistence keys,
- compose maps Postgres container init values from `DATABASE__*`,
- env contract drift tests now guard `.env.production.example`.
## V6.2 - GEDCOM Data Layer
Objective: introduce a genealogical data layer sourced from GEDCOM exports, bridged to
existing `Person` records via FamilySearch ID, without disrupting document-focused Person
workflows.
### Scope
1. Manual `.ged` file upload only. No FamilySearch credentials are stored or used by the
app; the user runs the third-party `getmyancestors` tool themselves and uploads the
resulting export.
2. Four new tables: `genealogy_person`, `genealogy_family`, `genealogy_family_child`, and
`genealogy_citation` (raw GEDCOM `SOUR` citations, reusable in a later version to record
when a transcribed document itself becomes citation evidence for FamilySearch).
3. Upsert-based import keyed on FamilySearch ID (`fs_id`) so repeat imports update existing
records in place without breaking existing `Person.family_search_id` links or duplicating
surrogate keys.
4. Reuse the existing V6.1 worker-backed `maintenance_run` pattern for import runs (run
history, status, summary, log view/download) rather than new infrastructure.
### Deliverables
- GEDCOM parser/importer producing the four genealogy tables.
- `MaintenanceJobType` entry for GEDCOM import with upsert semantics and a run summary
(records added/updated).
- Settings UI entry to upload a `.ged` file, trigger an import run, and view history.
### Exit Criteria
- Importing the same `.ged` file twice does not duplicate or orphan data.
- Existing `Person.family_search_id` values continue to resolve to the correct
`genealogy_person` row after import.
- Import run history is visible with status, duration, and summary, consistent with other
maintenance runs.
## V6.3 - Reporting and Genealogy-Enriched Features
Objective: improve research value with person-centric outputs, grounded in both archival
documents and the V6.2 genealogical data layer.
This version is broken into five sequential sub-versions because of real dependency
ordering: entity linking must exist before GEDCOM data can be targeted per-person; the
Facts/Events mechanism must exist before timelines or reconciliation have anything
meaningful to consume.
### V6.3.1 - Manual Entity Linking
- Search/browse UI over `genealogy_person` to find and link a candidate match to an
application `Person`, setting `family_search_id`. Linking is reversible (unlink).
- Once linked, GEDCOM vitals display alongside the `Person` record without requiring any
schema change to `Person`.
### V6.3.2 - Person Facts and Events
- New fact/event table capturing: person, fact type (birth/death/event/free-form), date
(+raw), place, free-text description, and a link to the source document as evidence.
- Manual tagging UI while reviewing a transcribed document: select a passage, choose the
person and fact type, record the date/description.
- One-time migration of existing `Person.birth_date`/`birth_date_raw`/`birth_place`/
`death_date`/`death_date_raw`/`death_place` values into fact/event rows (tagged as
legacy/no-document-evidence where no source document is known), followed by retiring those
six columns from `Person`. Birth/death become Facts/Events like any other locally-known
fact, for both linked and unlinked people. `Person` permanently keeps `last_name`,
`given_names`, `biography`, `family_search_id`, `metadata_`, tags, photos, and document
associations.
### V6.3.3 - Person Timelines
- Timeline query merging GEDCOM milestones (birth, marriage, children's births, death) for
linked persons with locally recorded Facts/Events.
- Timeline UI on Person Detail with clear ordering/filters; entries link back to their
originating document or GEDCOM record.
### V6.3.4 - Reconciliation
- Compares Facts/Events (the real, document-evidenced local signal) against corresponding
`genealogy_person` fields for linked persons.
- Persisted reconciliation record: person, field, local value with evidence-document link,
GEDCOM value, and status (open / submitted / dismissed).
- Re-evaluated automatically as part of each GEDCOM import maintenance run: opens new
discrepancies, auto-resolves ones where GEDCOM now matches, leaves others unchanged.
- Reconciliation review UI functions as a manual to-do list for updating FamilySearch; the
app does not write back to FamilySearch itself.
### V6.3.5 - AI-Assisted Biography Generation
- Prompted narrative generation grounded in GEDCOM facts, Facts/Events, and relevant
document snippets as structured input, using existing evidence-safe prompting patterns.
- Output cites back to source documents and FamilySearch records.
- Saved/printable report presentation for review; reports do not modify archival source
data.
### Exit Criteria (applies across V6.3.1-V6.3.5)
- Entity links are reversible and do not alter document associations.
- Timelines are reproducible from persisted records.
- Reconciliation items always carry a link to the document evidence justifying the local
value, and re-running GEDCOM import correctly opens, resolves, or leaves items unchanged.
- Narrative generation is traceable to source records and prompts.
- Reports can be reviewed without modifying archival source data.
## V6.4 - Access Control and Multi-User Readiness
[ *More thoughts on user accounts:*
* *Create a generic "view only" user that does not have the rights to alter any of the data*
* *Limit user accounts access to data by Tag. I have distant family members that I would want to share the transcribed data with, but they would only be interested in a subset of it. For example my Cochran cousins would have no interest in Lancaster documents, so limit the Cochra Clan cousins to view-only access to documents tagged "cochran clan"* ]
Objective: prepare for managed collaboration beyond single-user operation.
### Scope
1. Introduce application-level authentication.
2. Add role-based authorization (admin/editor/contributor/viewer).
3. Add audit visibility for user-attributed write actions.
### Deliverables
- User identity model and login/session flow.
- Route/page/service authorization enforcement.
- Audit metadata for sensitive create/update/delete workflows.
### Exit Criteria
- Unauthorized operations are blocked consistently across UI/API.
- Role policies are enforced by deterministic tests.
- User-attributed changes are visible for audit/review.
## Deferred / Future Ideas (not committed scope)
Captured for later consideration, not yet scheduled to a version:
* AI-assisted entity disambiguation (kinship co-occurrence, chronological plausibility
filtering) when linking document mentions to people.
* Kinship-aware `@mention` tagging while transcribing.
* Relationship-calculator badges (e.g., "3rd Great-Grandmother") in the document viewer.
* Interactive migration/geography mapping from GEDCOM and document place mentions.
* AI-suggested document discovery by date/location overlap with known persons.
* Ability to search within a document to find potential people to add to the People table.
## Planning Notes
- Keep architecture, schema, and UI contracts synchronized in `docs/` as each version lands.
- Prefer explicit schema migration over runtime compatibility write paths.
- Preserve evidence/provenance guarantees when adding new AI-powered features.
- GEDCOM/FamilySearch data is external, collaborative, and mutable; treat it as a managed
cache bridged via `fs_id`, never as a replacement for archival evidence recorded from
transcribed documents.
docs/schema.md
-391
View File
@@ -1,391 +0,0 @@
# Data Model and Persistence Schema (Current Baseline: V6.1)
This document is the field-accurate V6.1 schema contract aligned to `src/transcription/db/models.py`.
## Source of Truth Anchors
- `src/transcription/db/models.py` (status and purpose enums, including maintenance lifecycle enums)
- `src/transcription/db/models.py:80-120` (`DocumentType`, `PersonRole`)
- `src/transcription/db/models.py:122-172` (`Tag`, `Document`)
- `src/transcription/db/models.py` (`Person`, `GenealogyPerson`, `GenealogyFamily`, `GenealogyFamilyChild`, `GenealogyCitation`)
- `src/transcription/db/models.py` (`Photo`, `DocumentPerson`, `DocumentTag`)
- `src/transcription/db/models.py:285-347` (`Job`)
- `src/transcription/db/models.py` (`MaintenanceRun`)
- `src/transcription/db/models.py:350-462` (`Source`, `JobSource`)
- `src/transcription/db/models.py:465-522` (`ExecutionAttempt`)
## Entity Relationship Overview
```mermaid
erDiagram
DocumentType ||--o{ Document : classifies
Document ||--o{ Job : has
Document ||--o{ Source : has
Document ||--o{ DocumentPerson : links
Document ||--o{ DocumentTag : tagged
Person ||--o{ DocumentPerson : links
Person ||--o{ PersonTag : tagged
Person ||--o{ Photo : owns
GenealogyPerson ||--o{ GenealogyFamily : husband
GenealogyPerson ||--o{ GenealogyFamily : wife
GenealogyPerson ||--o{ GenealogyFamilyChild : child
GenealogyFamily ||--o{ GenealogyFamilyChild : includes
GenealogyPerson ||--o{ GenealogyCitation : cited
GenealogyFamily ||--o{ GenealogyCitation : cited
Document ||--o{ GenealogyCitation : evidence
PersonRole ||--o{ DocumentPerson : labels
Tag ||--o{ DocumentTag : labels
Tag ||--o{ PersonTag : labels
Job ||--o{ JobSource : includes
Source ||--o{ JobSource : participates
JobSource ||--o{ ExecutionAttempt : attempts
MaintenanceRun {
uuid id PK
}
```
## Authoritative Enumerations
### JobStatus
- `queued`
- `processing`
- `transcribed`
- `partial_success`
- `failed`
### JobSourceStatus
- `pending`
- `transcribed`
- `failed`
- `cancelled`
### JobPurpose
- `transcription`
- `retranscription`
### MaintenanceJobType
- `backup`
- `storage_reconciliation`
- `gedcom_import`
### MaintenanceRunStatus
- `queued`
- `processing`
- `succeeded`
- `failed`
## Field-Accurate Table Contracts
### `DocumentType`
| Field | Type | Notes |
| :--- | :--- | :--- |
| `id` | `UUID` | PK |
| `semantic_key` | `str \| None` | nullable unique, indexed |
| `label` | `str` | required |
| `normalized_label` | `str` | unique, indexed |
| `is_active` | `bool` | default `True` |
| `created_at` | `datetime` | default now |
| `updated_at` | `datetime` | default now, onupdate |
### `PersonRole`
| Field | Type | Notes |
| :--- | :--- | :--- |
| `id` | `UUID` | PK |
| `semantic_key` | `str \| None` | nullable unique, indexed |
| `label` | `str` | required |
| `normalized_label` | `str` | unique, indexed |
| `is_active` | `bool` | default `True` |
| `created_at` | `datetime` | default now |
| `updated_at` | `datetime` | default now, onupdate |
### `Tag`
| Field | Type | Notes |
| :--- | :--- | :--- |
| `id` | `UUID` | PK |
| `semantic_key` | `str \| None` | nullable unique, indexed |
| `label` | `str` | required |
| `normalized_label` | `str` | unique, indexed |
| `is_active` | `bool` | default `True` |
| `created_at` | `datetime` | default now |
| `updated_at` | `datetime` | default now, onupdate |
### `Document`
| Field | Type | Notes |
| :--- | :--- | :--- |
| `id` | `UUID` | PK |
| `name` | `str` | required |
| `document_type_id` | `UUID \| None` | FK -> `document_type.id`, indexed |
| `document_date` | `date \| None` | optional |
| `document_date_raw` | `str \| None` | optional |
| `location_created` | `str \| None` | optional |
| `notes` | `str \| None` | optional |
| `archive_identifier` | `str \| None` | optional |
| `created_at` | `datetime` | default now |
| `updated_at` | `datetime` | default now, onupdate |
### `Person`
| Field | Type | Notes |
| :--- | :--- | :--- |
| `id` | `UUID` | PK |
| `last_name` | `str` | required |
| `given_names` | `str` | required |
| `birth_date` | `date \| None` | optional |
| `birth_date_raw` | `str \| None` | optional |
| `birth_place` | `str \| None` | optional |
| `death_date` | `date \| None` | optional |
| `death_date_raw` | `str \| None` | optional |
| `death_place` | `str \| None` | optional |
| `biography` | `str \| None` | optional |
| `family_search_id` | `str \| None` | nullable unique |
| `metadata_` | `dict[str, JsonValue] \| None` | stored as DB column `metadata` (`JSONBCompat`) |
| `created_at` | `datetime` | default now |
| `updated_at` | `datetime` | default now, onupdate |
### `GenealogyPerson`
| Field | Type | Notes |
| :--- | :--- | :--- |
| `id` | `UUID` | PK |
| `fs_id` | `str` | unique, indexed FamilySearch identifier |
| `full_name` | `str` | required |
| `birth_date` | `date \| None` | optional |
| `birth_date_raw` | `str \| None` | optional |
| `birth_place` | `str \| None` | optional |
| `death_date` | `date \| None` | optional |
| `death_date_raw` | `str \| None` | optional |
| `death_place` | `str \| None` | optional |
| `created_at` | `datetime` | default now |
| `updated_at` | `datetime` | default now, onupdate |
### `GenealogyFamily`
| Field | Type | Notes |
| :--- | :--- | :--- |
| `id` | `UUID` | PK |
| `fs_family_id` | `str` | unique, indexed FamilySearch family identifier |
| `husband_id` | `UUID \| None` | nullable FK -> `genealogy_person.id`, indexed |
| `wife_id` | `UUID \| None` | nullable FK -> `genealogy_person.id`, indexed |
| `marriage_date` | `date \| None` | optional |
| `marriage_date_raw` | `str \| None` | optional |
| `marriage_place` | `str \| None` | optional |
| `created_at` | `datetime` | default now |
| `updated_at` | `datetime` | default now, onupdate |
### `GenealogyFamilyChild`
| Field | Type | Notes |
| :--- | :--- | :--- |
| `id` | `UUID` | PK |
| `family_id` | `UUID` | FK -> `genealogy_family.id`, indexed |
| `child_id` | `UUID` | FK -> `genealogy_person.id`, indexed |
| `relationship_type` | `str \| None` | optional |
| `created_at` | `datetime` | default now |
Constraint:
- `UniqueConstraint(family_id, child_id)` named `uq_genealogy_family_child`
### `GenealogyCitation`
| Field | Type | Notes |
| :--- | :--- | :--- |
| `id` | `UUID` | PK |
| `genealogy_person_id` | `UUID \| None` | nullable FK -> `genealogy_person.id`, indexed |
| `genealogy_family_id` | `UUID \| None` | nullable FK -> `genealogy_family.id`, indexed |
| `fact_type` | `GenealogyCitationFactType` | enum: `birth`, `death`, `marriage`, `other` |
| `raw_citation_text` | `str` | required raw GEDCOM citation text |
| `source_kind` | `GenealogyCitationSourceKind` | enum: `familysearch_imported`, `transcription_evidence` |
| `document_id` | `UUID \| None` | nullable FK -> `document.id`, indexed |
| `created_at` | `datetime` | default now |
### `Photo`
| Field | Type | Notes |
| :--- | :--- | :--- |
| `id` | `UUID` | PK |
| `person_id` | `UUID \| None` | nullable FK -> `person.id`, indexed (`NULL` = homepage photo) |
| `path` | `str` | required upload-root-relative POSIX path (`photos/...`) |
| `description` | `str \| None` | optional |
| `is_primary` | `bool` | default `False`; owner-level "featured/primary" marker |
| `created_at` | `datetime` | default now |
| `updated_at` | `datetime` | default now, onupdate |
### `DocumentPerson`
| Field | Type | Notes |
| :--- | :--- | :--- |
| `id` | `UUID` | PK |
| `document_id` | `UUID` | FK -> `document.id`, indexed |
| `person_id` | `UUID` | FK -> `person.id`, indexed |
| `role_id` | `UUID` | FK -> `person_role.id`, indexed |
| `created_at` | `datetime` | default now |
| `updated_at` | `datetime` | default now, onupdate |
Constraint:
- `UniqueConstraint(document_id, person_id)` named `uq_document_person`
### `DocumentTag`
| Field | Type | Notes |
| :--- | :--- | :--- |
| `id` | `UUID` | PK |
| `document_id` | `UUID` | FK -> `document.id`, indexed |
| `tag_id` | `UUID` | FK -> `tag.id`, indexed |
| `created_at` | `datetime` | default now |
| `updated_at` | `datetime` | default now, onupdate |
Constraint:
- `UniqueConstraint(document_id, tag_id)` named `uq_document_tag`
### `PersonTag`
| Field | Type | Notes |
| :--- | :--- | :--- |
| `id` | `UUID` | PK |
| `person_id` | `UUID` | FK -> `person.id`, indexed |
| `tag_id` | `UUID` | FK -> `tag.id`, indexed |
| `created_at` | `datetime` | default now |
| `updated_at` | `datetime` | default now, onupdate |
Constraint:
- `UniqueConstraint(person_id, tag_id)` named `uq_person_tag`
### `Job`
| Field | Type | Notes |
| :--- | :--- | :--- |
| `id` | `UUID` | PK |
| `document_id` | `UUID` | FK -> `document.id`, indexed |
| `status` | `JobStatus` | non-null enum (stored as enum values) |
| `retry_count` | `int` | default `0`, `ge=0` |
| `purpose` | `JobPurpose` | non-null enum, default `transcription` |
| `date_created` | `datetime` | default now |
| `date_updated` | `datetime` | default now, onupdate |
| `provider` | `str \| None` | optional |
| `model` | `str \| None` | optional |
| `prompt_name` | `str \| None` | optional |
| `prompt_hash` | `str \| None` | optional |
| `system_prompt` | `str \| None` | optional |
| `user_prompt` | `str \| None` | optional |
| `temperature` | `float \| None` | optional |
| `top_p` | `float \| None` | optional |
Index:
- `Index("ix_job_status_date_created", "status", "date_created")`
### `MaintenanceRun`
| Field | Type | Notes |
| :--- | :--- | :--- |
| `id` | `UUID` | PK |
| `job_type` | `MaintenanceJobType` | non-null enum |
| `status` | `MaintenanceRunStatus` | non-null enum, default `queued` |
| `started_at` | `datetime \| None` | optional |
| `finished_at` | `datetime \| None` | optional |
| `triggered_by` | `str \| None` | optional |
| `summary` | `str \| None` | optional |
| `log_path` | `str \| None` | optional, log-root-relative POSIX path |
| `error_detail` | `str \| None` | optional internal failure detail |
| `created_at` | `datetime` | default now |
| `updated_at` | `datetime` | default now, onupdate |
### `Source`
| Field | Type | Notes |
| :--- | :--- | :--- |
| `id` | `UUID` | PK |
| `document_id` | `UUID` | FK -> `document.id`, indexed |
| `page_number` | `int` | default `1`, `ge=1` |
| `upload_name` | `str` | required |
| `filename` | `str` | required |
| `file_path` | `str` | required upload-root-relative POSIX path (`documents/...`) |
| `file_hash` | `str` | required |
| `file_size_bytes` | `int` | `BigInteger`, non-null |
| `raw_transcription` | `str \| None` | projection field |
| `preferred_execution_attempt_id` | `UUID \| None` | nullable FK -> `execution_attempt.id`, indexed (`use_alter`) |
| `revised_text` | `str \| None` | optional human revision |
| `date_uploaded` | `datetime` | default now |
| `date_revised` | `datetime \| None` | optional |
### `JobSource`
| Field | Type | Notes |
| :--- | :--- | :--- |
| `id` | `UUID` | PK |
| `job_id` | `UUID` | FK -> `job.id`, indexed |
| `source_id` | `UUID` | FK -> `source.id`, indexed |
| `status` | `JobSourceStatus` | non-null enum, default `pending` |
Constraint:
- `UniqueConstraint(job_id, source_id)` named `uq_job_source_job_source`
Runtime reconciliation:
- Startup database operations remove retired V4.6 `job_source` evidence columns (`raw_transcription`, `ai_metadata`, `raw_api_response`, `error_detail`, `executed_at`) when present so persisted schema matches this contract.
### `ExecutionAttempt`
| Field | Type | Notes |
| :--- | :--- | :--- |
| `id` | `UUID` | PK |
| `job_source_id` | `UUID` | FK -> `job_source.id`, indexed |
| `job_id` | `UUID` | FK -> `job.id`, indexed |
| `source_id` | `UUID` | FK -> `source.id`, indexed |
| `attempt_number` | `int` | `ge=1` |
| `status` | `JobSourceStatus` | non-null enum, value-stable with `JobSource.status` |
| `provider` | `str` | required |
| `model` | `str \| None` | optional |
| `request_manifest` | `dict[str, JsonValue] \| None` | JSONBCompat |
| `request_manifest_sha256` | `str \| None` | optional |
| `request_manifest_schema_version` | `str \| None` | optional |
| `response_received` | `bool` | default `False` |
| `transport_status_code` | `int \| None` | optional |
| `transport_body` | `bytes \| None` | LargeBinary |
| `transport_content_type` | `str \| None` | optional |
| `transport_content_encoding` | `str \| None` | optional |
| `transport_safe_headers` | `dict[str, JsonValue] \| None` | JSONBCompat |
| `router_request_id` | `str \| None` | optional |
| `router_generation_id` | `str \| None` | optional |
| `sdk_response_snapshot` | `dict[str, JsonValue] \| None` | JSONBCompat |
| `normalized_metadata` | `dict[str, JsonValue] \| None` | JSONBCompat; may include app-namespaced `processing_timing` (`provider_call_duration_ms`, `processing_duration_ms`) |
| `software_context` | `dict[str, JsonValue] \| None` | JSONBCompat |
| `raw_transcription` | `str \| None` | optional |
| `error_category` | `str \| None` | optional |
| `error_detail` | `str \| None` | optional |
| `failure_phase` | `str \| None` | optional |
| `started_at` | `datetime` | required |
| `finished_at` | `datetime` | required |
| `duration_ms` | `int` | `ge=0` |
| `created_at` | `datetime` | default now |
Constraint:
- `UniqueConstraint(job_id, source_id, attempt_number)` named `uq_execution_attempt_number`
## Relationship Loading Contract
- Most ORM relationships are configured with `lazy="raise"`.
- `JobSource.execution_attempts` is intentionally `lazy="noload"` with ordered attempts.
- Service/UI read paths must explicitly eager-load required relationships before access.
## Persistence Invariants (Ground Truth)
1. `ExecutionAttempt` is append-only runtime evidence.
2. `JobSource.status` represents queue/projection execution state and is not a full evidence container.
3. `Source.raw_transcription` is a mutable projection and not authoritative attempt history.
4. `Job` terminal status derives from page outcomes (`JobSource` state), not from a separate summary table.
5. `DocumentType.semantic_key` and `PersonRole.semantic_key` are nullable-unique semantic identifiers.
## Cross-Reference
- [System Architecture](architecture.md)
- [System Requirements](requirements.md)
- [Error Handling Policy](error_handling.md)
- [AI Evidence and Provenance Invariant](./invariant/ai_evidence_and_provenance.md)
docs/ui/README.md
-60
View File
@@ -1,60 +0,0 @@
# UI Behavioral Contracts
## Purpose
This directory defines the current user-facing behavior of the NiceGUI application. It records what each page is for, which routes and actions it exposes, what information it presents, and how success, empty, validation, and failure states behave.
These documents are written for maintainers and AI contributors. They are behavioral contracts, not historical implementation notes and not substitutes for the database schema.
## Current Page Contracts
- [Home](pages/home.md)
- [Documents](pages/documents.md)
- [People](pages/people.md)
- [Jobs](pages/jobs.md)
- [Sources](pages/sources.md)
- [Settings](pages/settings.md)
NiceGUI registers the routes shown in each contract without the `/ui` prefix. The application mounts NiceGUI under `/ui`, so `/documents` in page code is served to a browser as `/ui/documents`.
## Authority Hierarchy
When documents disagree, use this order:
1. User-facing page intent and accepted behavior: the page contracts in this directory.
2. Visual and interaction styling: [UI Style Guide](../invariant/ui_style_guide.md).
3. UI dependency and ownership boundaries: [UI contributor instructions](../../.github/instructions/ui.instructions.md).
4. Durable failure behavior: [Error Handling invariant](../invariant/error_handling.md).
5. Durable AI evidence behavior: [Digital Evidence and AI Processing Provenance](../invariant/ai_evidence_and_provenance.md).
6. Data definitions and relationships: current models plus the [schema contract](../schema.md).
7. Implementation truth: current code and tests.
If code intentionally changes accepted page behavior, update the corresponding page contract in the same change. If code accidentally differs, correct the implementation rather than rewriting intent to match a defect.
## Contract Contents
Each page contract contains:
1. Purpose and user goals.
2. Registered routes and navigation context.
3. List, detail, and form behavior.
4. Editable and system-managed information.
5. Validation, empty, loading, and failure states.
6. A concise acceptance checklist.
7. Current implementation and test anchors.
8. Known limitations and deferred work.
## Maintenance Rules
- Describe current accepted behavior in present tense.
- Do not mix an obsolete “first release” design with current behavior.
- Keep future changes in versioned scope documents and link to them from a Deferred Work section.
- Do not reproduce the complete database field inventory here; include only fields that affect page behavior.
- Keep service, file, and test anchors current.
- Do not create separate current-state, target-state, and traceability copies of the same contract.
- Keep cross-page visual rules in the UI Style Guide instead of repeating them on each page.
- Keep database joins such as `DocumentPerson` and `JobSource` in schema/architecture documentation unless they directly affect a page interaction.
## Current Baseline
These contracts describe the current V6.1 baseline.
docs/ui/pages/documents.md
-150
View File
@@ -1,150 +0,0 @@
# Documents Page Contract
## Purpose
Documents manages the archival record for each historical artifact independently of its source files and transcription jobs. A Document can be created first, linked to people in one or more roles, and used later as the parent for Sources and Jobs.
## Routes
| Route | Purpose |
| --- | --- |
| `/documents` | Searchable archival Document list. |
| `/documents/new` | Create a Document. |
| `/documents/{document_id}` | View one Document and its related records. |
| `/documents/{document_id}/info` | View archival metadata and system logistics for one Document. |
| `/documents/{document_id}/edit` | Edit metadata and the complete Linked People set. |
| `/documents/{document_id}/delete` | Confirm or block deletion. |
| `/documents/{document_id}/jobs` | Show Jobs belonging to the Document. |
| `/documents/{document_id}/sources` | Source-image gallery for the Document. |
| `/documents/{document_id}/print` | Preview and browser-print the persisted Document. |
## List Behavior
- The title is **Archival Documents**.
- **Create new document** opens the create route.
- The table defaults to Document Title order and supports search and column sorting.
- Columns are Document Title, Author, Tags, Document Date, Type, # Sources, and Transcription Status.
- Document Title is left-aligned; the remaining columns are centered.
- Author lists all linked people in the `author` role.
- # Sources reflects the count of linked Source rows for each Document.
- Transcription Status reflects the most recent Job status for that Document; documents with no Jobs show a blank marker.
- Date display prefers exact date, then approximate date, then `Unknown`.
- Selecting a row opens Document Detail.
- Row navigation includes list context so Document Detail provides **Back to Documents**.
- No records displays `No documents found in repository.`
## Create and Edit Behavior
Required:
- Document name.
- Document type selected from the Document Type registry.
Optional:
- Exact date.
- Approximate date.
- Document location.
- Archive identifier.
- Notes.
- Tags.
- Linked People, with exactly one Person Role per linked Person.
Rules:
- Exact date must parse as `YYYY-MM-DD`; browser presentation may follow locale.
- The exact-date input is labeled **Document date**.
- Existing people appear with disambiguating labels.
- Tag assignment supports selecting existing tags and adding new labels inline.
- **Create new person** opens Person creation.
- `person_id` may preselect that Person in the author role on Document creation.
- An invalid requested Person produces a warning rather than a broken form.
- `return_to=jobs_new` returns a successful create to Job creation with the new Document selected.
- Edit includes active and inactive Document Types so historical values remain maintainable.
- One Linked People table contains Select, Person, and Role columns.
- Add and Edit use an inline Person/Role editor; Save, Cancel, and Delete change staged UI state only.
- A Person may appear once per Document regardless of role.
- Existing inactive-role links remain visible; only active roles may be newly assigned.
- Document fields and the complete staged link set commit atomically on the main save.
- Save success returns to Document Detail.
## Detail Behavior
- The heading shows name, type, and internal ID.
- The header includes a contextual back action: **Back to Documents** by default, **Back to Person** when opened from Person Detail, and **Back to Job** when opened from Job Detail.
- The detail workspace shows a Source-style pan/zoom media viewer with **Previous Page** / **Next Page** navigation for document source pages.
- The center column is **Editable Revision** for the active source page.
- Related People are grouped by role and link to Person Detail.
- **Source Pages & Transcriptions** shows source/job counts and actions for source-image gallery, document jobs, and adding a Job.
- **Edit Document**, **Print**, **Document Details**, **View Source Detail**, and **Delete** are available from the header.
- Invalid IDs and missing Documents produce explicit states without rendering a partial page.
## Document Source Images Behavior
- `/documents/{document_id}/sources` shows the current Document's source pages in a thumbnail gallery.
- Each card shows the page number, stored filename, and an **Open Source Detail** action.
- The page includes a **Back to Document** action.
- No source pages displays an explicit empty state.
## Document Info Behavior
- `/documents/{document_id}/info` contains **Archival Metadata** and **System Logistics**.
- It includes a **Back to Document** action.
- Archival metadata includes authors, document type, tags, document date, location (linked when present), archive identifier, and notes.
## Print Behavior
- Print opens a dedicated preview for persisted Document data.
- **Facsimile** places each Source image beside its current transcription and starts every Source on a new printed sheet.
- **Text only** omits images, joins single line breaks inside paragraphs, and preserves blank-line paragraph boundaries.
- Non-null revised text takes precedence over raw transcription, including an intentionally empty revision.
- Archival metadata resolves Author through the hidden built-in semantic identity, not its mutable label.
- Archival metadata includes the Document Type label.
- Metadata tables use a narrow non-wrapping label column and wider wrapping data columns rather than stretching across the page.
- Job metadata uses one oldest-to-newest column per Job and ends with Status.
- Stored text is escaped and Source media uses record-validated application URLs rather than local file paths.
- Printing uses the browser print dialog; server-generated PDFs are not provided.
## Document Jobs Behavior
- The page lists the Document's Jobs newest first with status and Job ID.
- **Open Job** navigates to Job Detail.
- **Create Job** opens Job creation with the Document selected.
- No jobs displays an explicit empty state.
## Delete Behavior
- Deletion is blocked while any Source or Job belongs to the Document.
- The blocked state names the dependency categories and provides navigation back and to Jobs.
- An unlinked Document requires an explicit permanent-delete action.
- Success returns to the Documents list.
## Acceptance Checklist
- List columns, alignment, search, sorting, date fallback, and row navigation match this contract.
- Create/edit enforce name, registered type, and valid exact-date input.
- Linked People staging enforces one role and one row per Person.
- Document and Linked People writes never partially commit.
- Person-first Document creation preselects the requested Person as author.
- Detail links people, Sources, and Jobs to the correct records.
- Delete never removes a Document with Source or Job dependencies.
- Both print formats preserve the frozen content, ordering, text-precedence, and safety contracts.
- Service failures use the shared error presenter and never report false success.
## Implementation Anchors
- `src/transcription/ui/pages/documents_page.py`
- `src/transcription/ui/components/table/documents.py`
- `src/transcription/services/documents.py`
- `src/transcription/services/people.py`
- `src/transcription/services/workflows.py`
- `src/transcription/ui/components/linked_people.py`
- `src/transcription/ui/pages/print_preview_page.py`
- `src/transcription/api/print_api.py`
- `tests/ui/test_documents_page.py`
- `tests/services/test_document_service.py`
## Known Limitations and Deferred Work
- Source page ordering remains read-only.
- Printing other entities, batch printing, and server-side export formats are deferred.
docs/ui/pages/home.md
-64
View File
@@ -1,64 +0,0 @@
# Home Page Contract
## Purpose
Home provides a user-maintained landing page for the local archive. It combines a database-backed image gallery with Markdown text and lets the operator edit both without changing application source or prompt assets.
## Routes
| Route | Browser path | Purpose |
| --- | --- | --- |
| `/homepage` | `/ui/homepage` | View homepage gallery and Markdown. |
| `/homepage/edit` | `/ui/homepage/edit` | Upload images, manage image metadata, and edit Markdown. |
The application root and `/ui` redirect to `/ui/homepage`.
## View Behavior
- The visible page heading is **Home**; the browser tab title is **VibeScribe Home**.
- The featured homepage image (`photo.is_primary`) is shown first; remaining images are shown in random order.
- The current image appears in the shared dark-room viewer with its description.
- Saved Markdown is rendered in the **Home Text** card.
- Missing text displays `No homepage text saved yet.`
- Missing image displays the viewer's empty state.
- **Edit Home Page** opens the edit route.
- The same Home Text content is also editable from **Settings → Home Page Text**.
## Edit Behavior
- The image upload accepts JPEG, PNG, GIF, WebP, BMP, and TIFF files and supports multi-file uploads.
- A successful upload immediately stores files in the shared `photo` table/media layout and displays a positive notification.
- The editor supports per-image description edits, setting a featured image, and deleting the current image.
- The Markdown textarea is initialized from the currently stored homepage text.
- **Save** writes the textarea content, displays `Homepage saved`, and returns to Home.
- **Cancel** returns to Home without saving textarea changes. An image already uploaded during the edit session remains stored.
## Storage Contract
- Homepage markdown text is mutable application data at `UPLOAD_DIR/homepage.md`.
- Homepage images are stored as `photo` rows (`person_id = NULL`) with files under `UPLOAD_DIR/photos/`.
- Uploaded images are renamed to `{photo_id}{suffix}`.
- Homepage images are database records; markdown remains file-backed.
## Acceptance Checklist
- `/`, `/ui`, and the application brand reach Home.
- Home renders with or without stored Markdown and image content.
- Edit loads existing Markdown.
- Supported image upload stores one or more images and makes the first image featured when no featured image exists yet.
- Save persists Markdown and returns to Home.
- Cancel does not save changed Markdown.
## Implementation Anchors
- `src/transcription/ui/pages/home_page.py`
- `src/transcription/ui/homepage_store.py`
- `src/transcription/ui/components/app_shell.py`
- `tests/ui/test_upload_page.py`
- `tests/ui/test_navigation_and_mounts.py`
- `tests/ui/test_pages_registration.py`
## Known Limitations
- Homepage markdown storage location is `UPLOAD_DIR/homepage.md` and must remain writable in the active runtime environment.
- Uploading an image is immediate and is not rolled back by Cancel.
docs/ui/pages/jobs.md
-99
View File
@@ -1,99 +0,0 @@
# Jobs Page Contract
## Purpose
Jobs manages transcription processing runs. A Job belongs to one Document, links one or more Source pages, records processing provenance, and exposes lifecycle actions without making lifecycle fields directly editable.
## Routes
| Route | Purpose |
| --- | --- |
| `/jobs` | Searchable processing Job list. |
| `/jobs/new` | Create and queue a Job. |
| `/jobs/{job_id}` | View status, execution logistics, and related records. |
| `/jobs/{job_id}/cancel` | Confirm cancellation. |
| `/jobs/{job_id}/resubmit` | Confirm resubmission of failed Sources. |
| `/jobs/{job_id}/delete` | Confirm or block deletion. |
## List Behavior
- The title is **Transcription Pipeline Jobs**.
- **Create job** opens Job creation and **Refresh** reloads the table.
- Columns are Job ID, Status, Document Name, # Sources, Retries, and Updated.
- Updated is the primary date/sort field.
- Search covers Job ID, document name, and status.
- Status is displayed as a semantic status chip.
- Selecting a row opens Job Detail.
- Global Job-list row navigation includes list context so Job Detail provides **Back to Jobs**.
- No records displays `No job records found in repository.`
## Create Behavior
- A Target Document and at least one source file are required.
- `document_id` may preselect a Target Document.
- If no Documents exist, the page explains the prerequisite and links to Document creation with a return path.
- Provider and Model are selectable when creating a new Job.
- Upload accepts JPEG, PNG, TIFF, and PDF files and supports multiple/folder selection.
- The visible upload queue is sorted alphabetically by original filename.
- Files can be removed individually or cleared before submission.
- Helper text explains numeric filename prefixes for page ordering.
- Submission creates the Job, Source records, and JobSource links, notifies the worker, and opens Job Detail.
- When opened with `source_id`, creation becomes a retranscription flow: Source and Document are locked, Provider is
read-only, Model is restricted to `PROVIDER_MODELS`, no upload is accepted, and one existing Source is linked.
## Detail and Lifecycle Behavior
- The heading shows Job ID and a status badge.
- Job Detail includes a contextual back action: **Back to Jobs** by default and **Back to Document** when opened from a Document-filtered Job list.
- Execution Logistics shows provider, model, prompt, retry count, and last update.
- Document Links show a clickable Document Name (with Job context), Sources count, and a single **View Sources** action using job filtering.
- Queued and processing Jobs show an auto-refresh notice and reload every four seconds.
- Polling stops when the Job becomes terminal or a refresh fails.
- Queued and processing Jobs expose **Cancel**.
- Jobs other than `transcribed` expose **Resubmit** under the current UI rule. The service blocks resubmission while processing is active or when no failed Sources exist.
- All Jobs expose **Delete Job**, subject to explicit evidence-deletion guardrails.
- Invalid and missing IDs produce explicit states.
## Cancel Behavior
- The confirmation explains that processing stops and remaining pending Sources become cancelled.
- The service decides whether the current state permits cancellation.
- Success updates the Job, notifies the worker, and returns to Job Detail.
## Resubmit Behavior
- The page shows current status and failed Source count.
- The page explains that resubmission queues failed linked Sources while preserving immutable prior attempt evidence.
- The service blocks submission while processing is active or when no failed Sources exist.
- `JobSource` remains the latest compatibility projection, while every provider call appends an `ExecutionAttempt`.
- The selected `Source.raw_transcription` projection remains available while a retry is pending or fails.
- Success reports the number of resubmitted Sources and returns to Job Detail.
## Delete Behavior
- Deletion is blocked while status is `processing`.
- Allowed deletion explicitly warns that related `JobSource` projections,
immutable execution attempts, captured transport responses, and attempt-owned
artifacts are permanently removed.
- Source records and source files remain available for separate deletion.
- Success returns to the Jobs list.
## Acceptance Checklist
- Job creation cannot proceed without a valid Document and at least one Source.
- Upload ordering and removal controls match the displayed queue.
- Detail shows current status and provenance summary with correct related links.
- Active Jobs refresh without overlapping permanent polling after terminal state.
- Cancel, resubmit, and delete honor service guardrails and show actionable failures.
- Lifecycle fields cannot be edited directly.
## Implementation Anchors
- `src/transcription/ui/pages/jobs_page.py`
- `src/transcription/ui/components/table/jobs.py`
- `src/transcription/services/jobs.py`
- `src/transcription/services/store.py`
- `src/transcription/services/workflows.py`
- `tests/ui/test_jobs_page.py`
- `tests/services/test_job_service.py`
- `tests/services/test_store.py`
docs/ui/pages/people.md
-106
View File
@@ -1,106 +0,0 @@
# People Page Contract
## Purpose
People manages reusable historical-person records. A Person may appear in many Documents under different relationship roles and may optionally carry one or more photos plus a FamilySearch identifier.
## Routes
| Route | Purpose |
| --- | --- |
| `/people` | Searchable People list. |
| `/people/new` | Create a Person. |
| `/people/{person_id}` | View one Person and linked Documents. |
| `/people/{person_id}/photos` | Manage Person photos. |
| `/people/{person_id}/edit` | Edit the Person. |
| `/people/{person_id}/delete` | Confirm permanent deletion. |
## List Behavior
- The title is **Archival Entities: People**.
- **Create new person** opens the create route.
- The table defaults to Name order (`Last Name, First & Middle`) and supports search and column sorting.
- Columns are Last Name, First & Middle; Tags; FamilySearch ID; Birth Date; Death Date; and # Documents.
- Name and Tags are left-aligned; FamilySearch ID, date columns, and # Documents are centered.
- # Documents reflects how many linked Documents each Person is connected to.
- Birth and death values independently prefer exact date, then approximate date, then `Unknown`.
- Selecting a row opens Person Detail.
- Row navigation includes list context so Person Detail provides **Back to People**.
- No records displays `No person records found in repository.`
## Create and Edit Behavior
Required:
- Last name.
- First & middle names.
Optional:
- Exact and approximate birth/death dates.
- Birth/death places.
- Biography.
- FamilySearch ID.
- Tags.
Rules:
- Missing last name or first/middle names blocks save with a warning.
- Exact date inputs are native browser date inputs.
- FamilySearch IDs are normalized and validated by `PeopleService`.
- Tags use the shared Tag registry and support inline add/select behavior.
- Photos are managed from Person Detail via `/people/{person_id}/photos` (not in create/edit form fields).
- Metadata JSON remains hidden.
- Save success returns to Person Detail.
## Detail Behavior
- The header provides **New Document**, **Edit Person**, **Edit Photo(s)**, and **Delete**.
- The header includes a contextual back action: **Back to People** by default, and **Back to Document** when opened from Document Detail.
- **New Document** opens Document creation with this Person requested for author preselection.
- Person Detail shows a single-photo viewer with **Previous/Next** navigation; the page-level **Edit Photo(s)** header action opens photo management.
- Photo management (upload, description edit, set-primary, delete) is intentionally moved to `/people/{person_id}/photos`.
- Biographical Record shows split names, computed full name, tags, compact birth/death dates, and places.
- Birth and death place values are clickable links to Google Maps when present.
- FamilySearch ID is shown as a metadata value and is clickable to the FamilySearch person details route when present.
- Biography has an explicit empty value.
- Linked Documents render as a table with **Document Name**, **Document Date**, **Role**, and **Number of Pages**; selecting a row opens Document Detail.
- No links shows both an empty state and guidance to link from a Document workflow.
- System Logistics shows created and updated timestamps.
## Delete Behavior
- The page warns when linked Document relationships exist.
- Delete is blocked when related Photos exist.
- Confirmed deletion removes the Person and its relationship links; it does not delete Documents.
- Success returns to the People list.
- Missing or already-deleted records return to a safe list state.
## Photo Gallery Behavior (`/people/{person_id}/photos`)
- Upload is triggered from a header-level **Upload Photo(s)** control beside **Back to Person**.
- The gallery renders all photos in a responsive grid (3-4 tiles wide on larger screens).
- Description text is shown as an overlay at the bottom of each image for quick context.
- The editor provides **Save Description**, **Set Primary** (when applicable), and **Delete Photo** actions.
## Acceptance Checklist
- List fields, alignment, date fallback, search, sorting, and navigation match this contract.
- Last name and first/middle names are enforced on create and edit.
- FamilySearch ID validation and link generation use the fixed supported identifier format.
- Photo upload and rendering remain constrained to supported media paths.
- New Document carries the Person context.
- Linked Documents show the correct role and target.
- Delete wording distinguishes removal of relationship links from deletion of Documents.
## Implementation Anchors
- `src/transcription/ui/pages/people_page.py`
- `src/transcription/ui/components/table/people.py`
- `src/transcription/services/people.py`
- `tests/ui/test_people_page.py`
- `tests/services/test_v2_crud.py`
## Deferred Work
- Structured name fields, merge/deduplication, advanced metadata editing, and Person-side relationship editing are not current behavior.
docs/ui/pages/settings.md
-55
View File
@@ -1,55 +0,0 @@
# Settings Page Contract
## Purpose
Settings manages installation-local registries, safe runtime .env settings, and editable text assets from one route.
## Route
| Route | Purpose |
| --- | --- |
| `/settings` | Manage Runtime Settings, Document Types, Person Roles, Tags, Prompts, Home Page Text, and Maintenance runs. |
## Behavior
- The page title is **Settings**.
- Configuration surfaces are grouped as tabs:
- **Document Types**
- **Person Roles**
- **Tags**
- **Prompts**
- **Home Page Text**
- **Maintenance**
- **Runtime Settings**
- Runtime Settings exposes an allowlisted set of non-secret fields synchronized with `Settings` model fields except excluded secret/unsafe fields.
- Runtime Settings is rendered as a compact two-column editor (**Setting**, **Value**) in a centered, narrower responsive container.
- Runtime Settings persists changes to the resolved runtime env file, validates by constructing a `Settings` instance, and reports validation failures through the shared UI error presenter.
- `Settings` resolves its env file in this order: explicit `_env_file`, `ENV_FILE`, then the repository-root `.env.production`.
- Runtime Settings resolves its write target in this order: explicit function override (tests/tools), `RUNTIME_SETTINGS_ENV_FILE` environment variable (deployment override), `ENV_FILE`, then the repository-root `.env.production`.
- Runtime Settings changes require application restart to take effect.
- Runtime Settings renders a host-side restart command (`docker compose -f docker-compose.production.yml up -d --force-recreate app worker`) so operators can apply saved values without granting Docker control to the app container.
- Runtime Settings includes an explicit "Other settings not shown here" markdown table listing:
- secrets (`OPENROUTER_API_KEY`, `DATABASE__PASSWORD`)
- high-risk database connection settings (`DATABASE__DRIVER`, `DATABASE__PATH`, `DATABASE__HOST`, `DATABASE__PORT`, `DATABASE__DATABASE`, `DATABASE__USER`)
and deployment/helper keys (`CLOUDFLARE_TUNNEL_TOKEN`, `BACKUP_DIR`, `BACKUP_RETENTION_DAYS`, `RUNTIME_SETTINGS_ENV_FILE`, `ENV_FILE`, `COMPOSE_FILE`) plus legacy/deprecated keys (`POSTGRES_*`, `DATABASE_BACKUP_DIR`, `APP_DATA_BACKUP_DIR`, `UPLOADS_BACKUP_DIR`, `SYNOLOGY_BACKUP_DIR`), and directs edits for those keys to the resolved runtime env file path.
- Document Types, Person Roles, and Tags support Add/Edit/Delete with existing guardrails.
- Prompts exposes only `transcribe_document.md` for editing and restore-from-backup.
- Home Page Text edits the same Markdown content rendered on `/homepage`.
- Maintenance provides queue-backed **Run Backup** and **Run Storage Reconciliation** actions.
- Maintenance also provides GEDCOM upload and **Run GEDCOM Import** actions, using the same queue-backed `MaintenanceRun` history/log flow.
- Maintenance run history shows job type, status, started/finished timestamps, duration, summary, and log view/download actions.
- Maintenance actions enqueue work and signal the worker; the page itself does not execute shell commands directly.
## Acceptance Checklist
- `/ui/settings` renders all seven tabs.
- Registry and prompt workflows keep existing validation and error handling.
- Runtime Settings excludes secret fields and rejects invalid values.
- Saving Home Page Text persists content for the homepage view.
## Implementation Anchors
- `src/transcription/ui/pages/settings_page.py`
- `src/transcription/ui/runtime_settings_store.py`
- `src/transcription/ui/homepage_store.py`
- `tests/ui/test_pages_registration.py`
docs/ui/pages/sources.md
-99
View File
@@ -1,99 +0,0 @@
# Sources Page Contract
## Purpose
Sources manages individual archived page/file records. It provides source-media viewing, current processing context, provider evidence inspection, previous/next page navigation, and human revision without allowing machine output to be edited.
## Routes
| Route | Purpose |
| --- | --- |
| `/sources` | Document-filtered or Job-filtered Source list; global route redirects to Documents. |
| `/sources/{source_id}` | View media, transcription, revision, metadata, and evidence. |
| `/sources/{source_id}/delete` | Confirm or block deletion. |
The list accepts optional `document_id` and `job_id` query parameters. Document context takes precedence if both parse successfully.
## List Behavior
- The global `/sources` route redirects to `/documents`.
- Filtered list titles are **Sources for Document** and **Sources for Job**.
- Filtered context provides **Back to Document** or **Back to Job**.
- Rows are ordered by page number and then upload name.
- Columns are Upload Title, Page Number, Document Name, Status, and Error Detail.
- Document Name, Upload Title, and Error Detail are left-aligned; Status is centered.
- Status labels are presented in uppercase for consistency with Jobs.
- Stored Filename is intentionally absent from the list.
- Selecting a row opens Source Detail.
- No records displays `No source asset records found in repository.`
## Detail Behavior
- The heading shows page number, upload name, and Source ID.
- **Back to Document** returns to Document Detail for the active source page.
- **Retranscribe Source** opens Create Processing Job with this Source and its Document locked.
- **Delete Source** opens the guarded delete route.
- Previous and Next navigate only among Sources belonging to the same Document in page order; unavailable boundary actions are disabled.
- The media viewer resolves the stored Source path through the configured upload root.
- The top layout is adaptive:
- Standard pages use three columns with a wider Editable Revision column than the image column.
- Wide+narrow landscape images switch to a stacked left layout (image above Editable Revision) with metadata on the right.
- Editable Revision is seeded from an existing revision or the preferred machine transcription.
- Source Metadata shows upload name, stored filename, page number, Document Name, Document ID, and stored path. Source ID appears in the page-header subtitle.
- SourceJob Metadata shows latest status (uppercase display), Job ID, execution time, provider, model, prompt, and failure detail.
- Revision Logistics shows revised state, last-revised time, and upload time.
- Candidate Machine Transcriptions appears below the image/revision area, remains compact until expanded, then compares it with the preferred
machine result and requires confirmation before **Use this transcription**.
- Candidate promotion does not alter a human revision. Empty states distinguish no machine result from no candidates.
- An orientation-normalized artifact appears in evidence only when recognized metadata required a physical rotation.
## Provider Evidence
- Provider Evidence is associated with the latest JobSource execution.
- New attempts display separate expandable Request Manifest, Transport Response, OpenRouter SDK Response Snapshot,
Normalized Metadata, Software Context, and Derived Artifacts sections.
- Historical `raw_api_response` values are labeled as OpenRouter SDK response snapshots.
- Missing evidence has an explicit empty state.
- Historical executions explicitly state that exact transport evidence was not captured.
- Quality warning artifacts remain attached to their machine attempt and are not recomputed during page rendering.
- **Export Evidence** downloads a versioned package containing source identity, attempts, artifacts, relationships,
schema versions, and integrity digests without source binaries, credentials, or machine-local source paths.
## Revision Behavior
- Machine transcription is never edited directly.
- A revision must contain non-whitespace text.
- Save persists revised text and updates the saved timestamp without leaving the page.
- Reset restores the in-memory revision from page load or the most recent successful save. When no revision exists, it restores the machine transcription; it does not re-read the database.
- A failed latest execution displays guidance that a human revision can preserve corrected text.
## Delete Behavior
- Deletion is allowed only when the Source has no JobSource links.
- A linked Source shows cleanup guidance and navigation to Jobs.
- An unlinked Source requires explicit permanent deletion.
- Success returns to the Sources list.
## Acceptance Checklist
- Global, Document-filtered, and Job-filtered lists show the correct context and return action.
- List columns and alignments match this contract and omit Stored Filename.
- Previous/next navigation never crosses Document boundaries.
- Detail keeps machine output read-only and human revision separately editable.
- Retranscription, candidate comparison, warnings, and explicit promotion preserve every prior attempt.
- Empty, failed, and missing-evidence states remain explicit.
- JSON evidence is readable without being mislabeled as native transport evidence.
- Delete cannot remove a Source with processing-history links.
## Implementation Anchors
- `src/transcription/ui/pages/sources_page.py`
- `src/transcription/ui/components/table/sources.py`
- `src/transcription/services/sources.py`
- `tests/ui/test_sources_page.py`
- `tests/services/test_transcription_service.py`
- `tests/services/test_v2_crud.py`
## Planned Changes
- Source page reordering remains deferred unless a demonstrated workflow need emerges.
project/.gitignore
+176
View File
@@ -0,0 +1,176 @@
# ---> Python
# Byte-compiled / optimized / DLL files
__pycache__/
*.py[cod]
*$py.class
# C extensions
*.so
# Distribution / packaging
.Python
build/
develop-eggs/
dist/
downloads/
eggs/
.eggs/
lib/
lib64/
parts/
sdist/
var/
wheels/
share/python-wheels/
*.egg-info/
.installed.cfg
*.egg
MANIFEST
# PyInstaller
# Usually these files are written by a python script from a template
# before PyInstaller builds the exe, so as to inject date/other infos into it.
*.manifest
*.spec
# Installer logs
pip-log.txt
pip-delete-this-directory.txt
# Unit test / coverage reports
htmlcov/
.tox/
.nox/
.coverage
.coverage.*
.cache
nosetests.xml
coverage.xml
*.cover
*.py,cover
.hypothesis/
.pytest_cache/
cover/
# Translations
*.mo
*.pot
# Django stuff:
*.log
local_settings.py
db.sqlite3
db.sqlite3-journal
# Flask stuff:
instance/
.webassets-cache
# Scrapy stuff:
.scrapy
# Sphinx documentation
docs/_build/
# PyBuilder
.pybuilder/
target/
# Jupyter Notebook
.ipynb_checkpoints
# IPython
profile_default/
ipython_config.py
# pyenv
# For a library or package, you might want to ignore these files since the code is
# intended to run in multiple environments; otherwise, check them in:
# .python-version
# pipenv
# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
# However, in case of collaboration, if having platform-specific dependencies or dependencies
# having no cross-platform support, pipenv may install dependencies that don't work, or not
# install all needed dependencies.
#Pipfile.lock
# UV
# Similar to Pipfile.lock, it is generally recommended to include uv.lock in version control.
# This is especially recommended for binary packages to ensure reproducibility, and is more
# commonly ignored for libraries.
#uv.lock
# poetry
# Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control.
# This is especially recommended for binary packages to ensure reproducibility, and is more
# commonly ignored for libraries.
# https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control
#poetry.lock
# pdm
# Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control.
#pdm.lock
# pdm stores project-wide configurations in .pdm.toml, but it is recommended to not include it
# in version control.
# https://pdm.fming.dev/latest/usage/project/#working-with-version-control
.pdm.toml
.pdm-python
.pdm-build/
# PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm
__pypackages__/
# Celery stuff
celerybeat-schedule
celerybeat.pid
# SageMath parsed files
*.sage.py
# Environments
.env
.venv
env/
venv/
ENV/
env.bak/
venv.bak/
# Spyder project settings
.spyderproject
.spyproject
# Rope project settings
.ropeproject
# mkdocs documentation
/site
# mypy
.mypy_cache/
.dmypy.json
dmypy.json
# Pyre type checker
.pyre/
# pytype static type analyzer
.pytype/
# Cython debug symbols
cython_debug/
# PyCharm
# JetBrains specific template is maintained in a separate JetBrains.gitignore that can
# be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore
# and can be added to the global gitignore or merged into this file. For a more nuclear
# option (not recommended) you can uncomment the following to ignore the entire idea folder.
#.idea/
# Ruff stuff:
.ruff_cache/
# PyPI configuration file
.pypirc
project/README.md.jinja
+1
View File
@@ -0,0 +1 @@
# {{project_name}}
project/copier.yml
+11
View File
@@ -0,0 +1,11 @@
project_name:
type: str
help: What is the project name?
repo_name:
type: str
help: What is the repo name?
module_name:
type: str
help: What is your Python module name?
project/docs/index.md.jinja
+1
View File
@@ -0,0 +1 @@
# {{project_name}}
project/pyproject.toml.jinja
+11
View File
@@ -0,0 +1,11 @@
[project]
name = "{{repo_name}}"
version = "0.1.0"
requires-python = ">=3.12"
[dependency-groups]
dev = [
"pre-commit>=4.6.0",
"ruff>=0.15.15",
"ipykernel>=7.2.0",
]
ruff.toml → project/ruff.toml
+1 -4
View File
@@ -10,11 +10,10 @@ exclude = [
"build",
"site",
"__pycache__",
]
[lint]
preview = true
extend-select = [
"ARG", # https://docs.astral.sh/ruff/rules/#flake8-unused-arguments-arg
"B", # https://docs.astral.sh/ruff/rules/#flake8-bugbear-b
@@ -26,7 +25,6 @@ extend-select = [
"E", "W", # https://docs.astral.sh/ruff/rules/#pycodestyle-e-w
"F", # https://docs.astral.sh/ruff/rules/#pyflakes-f
"FURB", # https://docs.astral.sh/ruff/rules/#refurb-furb
"G", # https://docs.astral.sh/ruff/rules/#flake8-logging-format-g
"I", # https://docs.astral.sh/ruff/rules/#isort-i
"N", # https://docs.astral.sh/ruff/rules/#pep8-naming-n
"PD", # https://docs.astral.sh/ruff/rules/#pandas-vet-pd
@@ -50,7 +48,6 @@ ignore = [
"*.ipynb" = [
"F401", # unused imports
"F841", # unused local variable
"F821", # undefined name in exploratory notebook cells
]
[lint.isort]
src/transcription/__init__.py → project/src/{{module_name}}/__init__.py
View File
project/zensical.toml.jinja
+350
View File
@@ -0,0 +1,350 @@
# ============================================================================
#
# The configuration produced by default is meant to highlight the features
# that Zensical provides and to serve as a starting point for your own
# projects.
#
# ============================================================================
[project]
# The site_name is shown in the page header and the browser window title
#
# Read more: https://zensical.org/docs/setup/basics/#site_name
site_name = "{{project_name}} Documentation"
# The site_description is included in the HTML head and should contain a
# meaningful description of the site content for use by search engines.
#
# Read more: https://zensical.org/docs/setup/basics/#site_description
site_description = "Documentation site for {{project_name}}"
# The site_author attribute. This is used in the HTML head element.
#
# Read more: https://zensical.org/docs/setup/basics/#site_author
site_author = "John Lancaster"
# The site_url is the canonical URL for your site. When building online
# documentation you should set this.
# Read more: https://zensical.org/docs/setup/basics/#site_url
#site_url = "https://www.example.com/"
# The copyright notice appears in the page footer and can contain an HTML
# fragment.
#
# Read more: https://zensical.org/docs/setup/basics/#copyright
copyright = """
Copyright &copy; 2026 The authors
"""
# Zensical supports both implicit navigation and explicitly defined navigation.
# If you decide not to define a navigation here then Zensical will simply
# derive the navigation structure from the directory structure of your
# "docs_dir". The definition below demonstrates how a navigation structure
# can be defined using TOML syntax.
#
# Read more: https://zensical.org/docs/setup/navigation/
# nav = [
# { "Get started" = "index.md" },
# { "Markdown in 5min" = "markdown.md" },
# ]
# With the "extra_css" option you can add your own CSS styling to customize
# your Zensical project according to your needs. You can add any number of
# CSS files.
#
# The path provided should be relative to the "docs_dir".
#
# Read more: https://zensical.org/docs/customization/#additional-css
#
#extra_css = ["stylesheets/extra.css"]
# With the `extra_javascript` option you can add your own JavaScript to your
# project to customize the behavior according to your needs.
#
# The path provided should be relative to the "docs_dir".
#
# Read more: https://zensical.org/docs/customization/#additional-javascript
#extra_javascript = ["javascripts/extra.js"]
# ----------------------------------------------------------------------------
# Section for configuring theme options
# ----------------------------------------------------------------------------
[project.theme]
# change this to "classic" to use the traditional Material for MkDocs look.
#variant = "classic"
# Zensical allows you to override specific blocks, partials, or whole
# templates as well as to define your own templates. To do this, uncomment
# the custom_dir setting below and set it to a directory in which you
# keep your template overrides.
#
# Read more:
# - https://zensical.org/docs/customization/#extending-the-theme
#
#custom_dir = "overrides"
# With the "favicon" option you can set your own image to use as the icon
# browsers will use in the browser title bar or tab bar. The path provided
# must be relative to the "docs_dir".
#
# Read more:
# - https://zensical.org/docs/setup/logo-and-icons/#favicon
# - https://developer.mozilla.org/en-US/docs/Glossary/Favicon
#
#favicon = "images/favicon.png"
# Zensical supports more than 60 different languages. This means that the
# labels and tooltips that Zensical's templates produce are translated.
# The "language" option allows you to set the language used. This language
# is also indicated in the HTML head element to help with accessibility
# and guide search engines and translation tools.
#
# The default language is "en" (English). It is possible to create
# sites with multiple languages and configure a language selector. See
# the documentation for details.
#
# Read more:
# - https://zensical.org/docs/setup/language/
#
language = "en"
# Zensical provides a number of feature toggles that change the behavior
# of the documentation site.
features = [
# Zensical includes an announcement bar. This feature allows users to
# dismiss it when they have read the announcement.
# https://zensical.org/docs/setup/header/#announcement-bar
"announce.dismiss",
# If you have a repository configured and turn on this feature, Zensical
# will generate an edit button for the page. This works for common
# repository hosting services.
# https://zensical.org/docs/setup/repository/#content-actions
#"content.action.edit",
# If you have a repository configured and turn on this feature, Zensical
# will generate a button that allows the user to view the Markdown
# code for the current page.
# https://zensical.org/docs/setup/repository/#content-actions
#"content.action.view",
# Code annotations allow you to add an icon with a tooltip to your
# code blocks to provide explanations at crucial points.
# https://zensical.org/docs/authoring/code-blocks/#code-annotations
"content.code.annotate",
# This feature turns on a button in code blocks that allow users to
# copy the content to their clipboard without first selecting it.
# https://zensical.org/docs/authoring/code-blocks/#code-copy-button
"content.code.copy",
# Code blocks can include a button to allow for the selection of line
# ranges by the user.
# https://zensical.org/docs/authoring/code-blocks/#code-selection-button
"content.code.select",
# Zensical can render footnotes as inline tooltips, so the user can read
# the footnote without leaving the context of the document.
# https://zensical.org/docs/authoring/footnotes/#footnote-tooltips
"content.footnote.tooltips",
# If you have many content tabs that have the same titles (e.g., "Python",
# "JavaScript", "Cobol"), this feature causes all of them to switch to
# at the same time when the user chooses their language in one.
# https://zensical.org/docs/authoring/content-tabs/#linked-content-tabs
"content.tabs.link",
# With this feature enabled users can add tooltips to links that will be
# displayed when the mouse pointer hovers the link.
# https://zensical.org/docs/authoring/tooltips/#improved-tooltips
"content.tooltips",
# With this feature enabled, Zensical will automatically hide parts
# of the header when the user scrolls past a certain point.
# https://zensical.org/docs/setup/header/#automatic-hiding
# "header.autohide",
# Turn on this feature to expand all collapsible sections in the
# navigation sidebar by default.
# https://zensical.org/docs/setup/navigation/#navigation-expansion
# "navigation.expand",
# This feature turns on navigation elements in the footer that allow the
# user to navigate to a next or previous page.
# https://zensical.org/docs/setup/footer/#navigation
"navigation.footer",
# When section index pages are enabled, documents can be directly attached
# to sections, which is particularly useful for providing overview pages.
# https://zensical.org/docs/setup/navigation/#section-index-pages
"navigation.indexes",
# When instant navigation is enabled, clicks on all internal links will be
# intercepted and dispatched via XHR without fully reloading the page.
# https://zensical.org/docs/setup/navigation/#instant-navigation
"navigation.instant",
# With instant prefetching, your site will start to fetch a page once the
# user hovers over a link. This will reduce the perceived loading time
# for the user.
# https://zensical.org/docs/setup/navigation/#instant-prefetching
"navigation.instant.prefetch",
# In order to provide a better user experience on slow connections when
# using instant navigation, a progress indicator can be enabled.
# https://zensical.org/docs/setup/navigation/#progress-indicator
#"navigation.instant.progress",
# When navigation paths are activated, a breadcrumb navigation is rendered
# above the title of each page
# https://zensical.org/docs/setup/navigation/#navigation-path
"navigation.path",
# When pruning is enabled, only the visible navigation items are included
# in the rendered HTML, reducing the size of the built site by 33% or more.
# https://zensical.org/docs/setup/navigation/#navigation-pruning
#"navigation.prune",
# When sections are enabled, top-level sections are rendered as groups in
# the sidebar for viewports above 1220px, but remain as-is on mobile.
# https://zensical.org/docs/setup/navigation/#navigation-sections
"navigation.sections",
# When tabs are enabled, top-level sections are rendered in a menu layer
# below the header for viewports above 1220px, but remain as-is on mobile.
# https://zensical.org/docs/setup/navigation/#navigation-tabs
#"navigation.tabs",
# When sticky tabs are enabled, navigation tabs will lock below the header
# and always remain visible when scrolling down.
# https://zensical.org/docs/setup/navigation/#sticky-navigation-tabs
#"navigation.tabs.sticky",
# A back-to-top button can be shown when the user, after scrolling down,
# starts to scroll up again.
# https://zensical.org/docs/setup/navigation/#back-to-top-button
"navigation.top",
# When anchor tracking is enabled, the URL in the address bar is
# automatically updated with the active anchor as highlighted in the table
# of contents.
# https://zensical.org/docs/setup/navigation/#anchor-tracking
"navigation.tracking",
# When search highlighting is enabled and a user clicks on a search result,
# Zensical will highlight all occurrences after following the link.
# https://zensical.org/docs/setup/search/#search-highlighting
"search.highlight",
# When anchor following for the table of contents is enabled, the sidebar
# is automatically scrolled so that the active anchor is always visible.
# https://zensical.org/docs/setup/navigation/#anchor-following
# "toc.follow",
# When navigation integration for the table of contents is enabled, it is
# always rendered as part of the navigation sidebar on the left.
# https://zensical.org/docs/setup/navigation/#navigation-integration
#"toc.integrate",
]
# ----------------------------------------------------------------------------
# You can configure your own logo to be shown in the header using the "logo"
# option in the "theme" subsection. The logo must be a relative path to a file
# in your "docs_dir", e.g., to use `docs/assets/logo.png` you would set:
# ----------------------------------------------------------------------------
#logo = "assets/logo.png"
# ----------------------------------------------------------------------------
# If you don't have a dedicated project logo, you can use a built-in icon from
# the icon sets shipped in Zensical. Please note that the setting lives in a
# different subsection, and that the above take precedence over the icon.
#
# Read more:
# - https://zensical.org/docs/setup/logo-and-icons
# - https://github.com/zensical/ui/tree/master/dist/.icons
# ----------------------------------------------------------------------------
#[project.theme.icon]
#logo = "lucide/smile"
# ----------------------------------------------------------------------------
# In the "font" subsection you can configure the fonts used. By default, fonts
# are loaded from Google Fonts, giving you a wide range of choices from a set
# of suitably licensed fonts. There are options for a normal text font and for
# a monospaced font used in code blocks.
# ----------------------------------------------------------------------------
#[project.theme.font]
#text = "Inter"
#code = "Jetbrains Mono"
# ----------------------------------------------------------------------------
# In the "palette" subsection you can configure options for the color scheme.
# You can configure different color schemes, e.g., to turn on dark mode,
# that the user can switch between. Each color scheme can be further
# customized.
#
# Read more:
# - https://zensical.org/docs/setup/colors/
# ----------------------------------------------------------------------------
[[project.theme.palette]]
scheme = "default"
toggle.icon = "lucide/sun"
toggle.name = "Switch to dark mode"
[[project.theme.palette]]
scheme = "slate"
toggle.icon = "lucide/moon"
toggle.name = "Switch to light mode"
# ----------------------------------------------------------------------------
# The "extra" section contains miscellaneous settings.
# ----------------------------------------------------------------------------
#[[project.extra.social]]
#icon = "fontawesome/brands/github"
#link = "https://github.com/user/repo"
# ----------------------------------------------------------------------------
# In this section you can configure the Markdown extensions that are used when
# rendering your documentation. We enable the most useful extensions by default,
# but you can customize this list to your needs.
#
# Read more:
# - https://zensical.org/docs/setup/extensions/
# ----------------------------------------------------------------------------
[project.markdown_extensions.abbr]
[project.markdown_extensions.admonition]
[project.markdown_extensions.attr_list]
[project.markdown_extensions.def_list]
[project.markdown_extensions.footnotes]
[project.markdown_extensions.md_in_html]
[project.markdown_extensions.toc]
permalink = true
[project.markdown_extensions.pymdownx.arithmatex]
generic = true
[project.markdown_extensions.pymdownx.betterem]
[project.markdown_extensions.pymdownx.caret]
[project.markdown_extensions.pymdownx.details]
[project.markdown_extensions.pymdownx.emoji]
emoji_generator = "zensical.extensions.emoji.to_svg"
emoji_index = "zensical.extensions.emoji.twemoji"
[project.markdown_extensions.pymdownx.highlight]
anchor_linenums = true
line_spans = "__span"
pygments_lang_class = true
[project.markdown_extensions.pymdownx.inlinehilite]
[project.markdown_extensions.pymdownx.keys]
[project.markdown_extensions.pymdownx.magiclink]
[project.markdown_extensions.pymdownx.mark]
[project.markdown_extensions.pymdownx.smartsymbols]
[project.markdown_extensions.pymdownx.snippets]
[project.markdown_extensions.pymdownx.superfences]
custom_fences = [
{ name = "mermaid", class = "mermaid", format = "pymdownx.superfences.fence_code_format" }
]
[project.markdown_extensions.pymdownx.tabbed]
alternate_style = true
combine_header_slug = true
[project.markdown_extensions.pymdownx.tasklist]
custom_checkbox = true
[project.markdown_extensions.pymdownx.tilde]
project/{{_copier_conf.answers_file}}.jinja
+2
View File
@@ -0,0 +1,2 @@
# Changes here will be overwritten by Copier
{{ _copier_answers|to_nice_yaml -}}
prompts/README.md
-15
View File
@@ -1,15 +0,0 @@
# Prompt Artifacts
This directory stores transcription prompts as individual Markdown artifacts.
## Conventions
- Keep one prompt per file.
- Use stable, descriptive snake_case file names.
- Store prompt files directly in this directory; nested paths are rejected.
- Prefer incremental edits to a single prompt per change for clean history.
- Keep prompts human-readable and policy-focused.
- Do not store secrets in prompt files.
- Runtime jobs snapshot prompt text, SHA-256 provenance, and sampling configuration.
## Current Prompt
- `transcribe_document.md`: baseline verbatim transcription policy for historical documents.
prompts/transcribe_document.md
-105
View File
@@ -1,105 +0,0 @@
# Historical Document Verbatim Transcription Prompt
## Purpose
Transcribe the provided historical document image as a faithful **verbatim** transcript.
Do not summarize. Do not paraphrase. Do not modernize style.
## Output Contract
- Return only the transcription text.
- Begin with exactly one applicable body marker:
- `[document body handwritten]`
- `[document body typewritten]`
- `[document body typeset]`
- `[document body mixed]`
- Preserve original wording, punctuation, and meaningful structure.
- Keep line/section flow readable while preserving intent and document organization.
- Never invent missing content.
- Use ordinary plain-text characters rather than HTML entities.
## Rules for Ambiguous or Damaged Text
### Misspellings and original errors
- Preserve original spelling.
- Add `[sic]` immediately after an evident original error.
### Missing words or clear omissions
- If a single missing word is obvious from context, insert it in square brackets.
- Example form: `[to]`
### Uncertain readings
- If best-effort interpretation is uncertain, use bracketed guess with question mark.
- Example form: `[Boston?]`
### Completely illegible text
- Use a clear bracketed label.
- Preferred forms: `[illegible]`, `[torn]`, `[ink blot]`, `[remainder of page torn]`
### Crossed-out text
- Preserve it using: `[deleted: ...]`
### Squeezed-in or above-line insertions
- Preserve it using: `[inserted: ...]`
### Superscripts and abbreviations
- Bring superscript letters down to baseline text.
- Expand only when clearly intended; if expanded, place added letters in brackets.
### Non-text visual elements
- Describe briefly in square brackets.
- Example forms: `[wax notary seal attached here]`, `[sketch of a fort layout]`
### Marginalia and side notes
- Signal location before the note text.
- Example form: `[written in left margin: ...]`
### Document body medium
- Use `[document body handwritten]` when the main body is written by hand.
- Use `[document body typewritten]` for mechanically typewritten pages. Uneven impressions,
monospaced characters, worn type, and other typewriter defects are not handwriting.
- Use `[document body typeset]` for printed pages composed with movable type or comparable
typesetting.
- Use `[document body mixed]` when substantial body content uses more than one medium, such
as a completed printed form.
- Preserve printed and handwritten text together in their original reading context.
- On mixed documents, leave printed labels and instructions unmarked and wrap only actual
handwritten entries in `[handwritten: ...]`.
- Mark handwritten signatures as `[handwritten signature: ...]`.
- If the main body is entirely handwritten, use its one body marker rather than wrapping
each line in `[handwritten: ...]`.
- Mark later notes or uncertain additions as `[handwritten annotation: ...]`.
- When authorship is unclear, use `[handwritten annotation, author uncertain: ...]`.
- Do not infer authorship, writing date, or whether different handwriting belongs to different people unless explicitly evident.
### Structured layouts
- Preserve tables of contents as associated title, dotted-leader, and page-reference rows.
- Preserve tables and forms in reading order while keeping labels associated with their values.
- Preserve columns in their evident reading order; do not interleave unrelated rows.
- Preserve captions with the visual element they describe.
- Preserve marginalia with its location marker and page numbers in their evident position.
### Line-break hyphenation
- Rejoin words split across line breaks when they are clearly one word.
- Remove only line-break hyphens used for wrapping.
### Ambiguous capitalization
- Prefer modern capitalization only when uncertainty is high.
- Preserve clearly intentional archaic capitalization.
### Hierarchical outlines and numbering
- Preserve original numbering characters exactly (including roman numerals and unusual suffixes).
- Preserve indentation levels.
- Do not silently correct sequence mistakes; if clearly erroneous, preserve and use `[sic]` where appropriate.
## Confidence and Integrity Policy
- When uncertain, mark uncertainty explicitly rather than guessing silently.
- If text cannot be read, use a bracketed illegibility label instead of fabrication.
- Do not add commentary outside the transcription.
## Final Self-Check
Before finalizing, ensure:
1. The transcript is verbatim and not summarized.
2. Uncertain/illegible areas are explicitly marked.
3. Crossed-out and inserted text are preserved with required tags.
4. Structure/ordering is preserved as faithfully as possible.
5. Exactly one document-body marker appears, and localized handwriting markers are used only where applicable.
6. Tables, forms, columns, captions, marginalia, dotted leaders, and page references retain their associations.
pyproject.toml
+2 -52
View File
@@ -1,57 +1,7 @@
[build-system]
requires = ["hatchling"]
build-backend = "hatchling.build"
[tool.hatch.build.targets.wheel]
packages = ["src/transcription"]
[project]
name = "transcription"
name = "python-template"
version = "0.1.0"
description = "Historical document transcription system"
readme = "README.md"
requires-python = ">=3.12"
dependencies = [
"aiosqlite>=0.21.0",
"asyncpg>=0.31.0",
"fastapi>=0.138.0",
# Exact pin, deliberate. NiceGUI 3.x minor releases ship Quasar/Vue changes that
# break component props and styling, and tests/ui/ cannot detect visual regressions.
# Hold through the current release stabilization; revisit as a scheduled upgrade.
# See docs/production-runbook.md, "Dependency upgrade policy".
"nicegui==3.13.0",
"openrouter>=0.7.0",
"pillow>=10.0.0",
"psycopg2-binary>=2.9.12",
"pydantic>=2.13.4",
"pydantic-settings>=2.9.1",
"python-gedcom>=1.1.0",
"sqlmodel>=0.0.25",
]
[dependency-groups]
dev = [
"pytest>=8.0",
"pytest-asyncio>=0.25",
"httpx2>=2.5.0",
"ipykernel>=7.3.0",
"ipywidgets>=8.1.8",
"pre-commit>=4.6.0",
"rich>=15.0.0",
"ruff>=0.15.20",
"ty>=0.0.54",
]
[tool.pytest.ini_options]
addopts = "--strict-markers -q"
asyncio_mode = "strict"
asyncio_default_fixture_loop_scope = "function"
filterwarnings = [
"error:coroutine .* was never awaited:RuntimeWarning",
]
markers = [
"unit: pure logic tests with no external dependencies",
"integration: tests that touch framework or database contracts",
"external: tests that call external services (slow, requires credentials)",
"copier>=9.15.1",
]
src/transcription/__main__.py
-27
View File
@@ -1,27 +0,0 @@
import uvicorn
from fastapi import FastAPI
from .app import create_app
from .config import parse_cli_settings
def create_cli_app() -> FastAPI:
"""Create an app from CLI settings for Uvicorn's reload process."""
return create_app(settings=parse_cli_settings())
def main() -> None:
settings = parse_cli_settings()
application = "transcription.__main__:create_cli_app" if settings.reload else create_app(settings=settings)
uvicorn.run(
application,
factory=settings.reload,
host=settings.host,
port=settings.port,
log_level=settings.log_level,
reload=settings.reload,
)
if __name__ == "__main__":
main()
src/transcription/api/__init__.py
-1
View File
@@ -1 +0,0 @@
"""API route modules for the transcription app."""
src/transcription/api/documents_api.py
-202
View File
@@ -1,202 +0,0 @@
"""API routes for relationship and classification registries."""
from __future__ import annotations
from typing import Annotated
from uuid import UUID
from fastapi import APIRouter
from fastapi import Depends
from fastapi import Request
from fastapi import Response
from pydantic import BaseModel
from pydantic import ConfigDict
from pydantic import Field
from transcription.db.models import Document
from transcription.db.models import DocumentPerson
from transcription.db.models import DocumentType
from transcription.db.models import PersonRole
from transcription.services import DocumentService
from transcription.services import PeopleService
router = APIRouter(prefix="/api", tags=["documents"])
class ApiModel(BaseModel):
model_config = ConfigDict(extra="forbid", frozen=True, str_strip_whitespace=True)
class DocumentTypeRead(ApiModel):
id: UUID
label: str
is_active: bool
class PersonRoleRead(ApiModel):
id: UUID
label: str
is_active: bool
class DocumentTypeWriteRequest(ApiModel):
document_type_id: UUID
class DocumentTypeWriteResponse(ApiModel):
document_id: UUID
document_type_id: UUID
class DocumentPersonWriteRequest(ApiModel):
person_id: UUID
role_id: UUID
class DocumentPersonRoleUpdateRequest(ApiModel):
role_id: UUID
class DocumentPersonRead(ApiModel):
id: UUID
document_id: UUID
person_id: UUID
role_id: UUID
role_label: str | None = None
person_name: str | None = None
class DocumentPeopleResponse(ApiModel):
document_id: UUID
links: list[DocumentPersonRead] = Field(default_factory=list)
def _document_type_to_read(item: DocumentType) -> DocumentTypeRead:
item_id, label, is_active = _registry_read_values(item)
return DocumentTypeRead(id=item_id, label=label, is_active=is_active)
def _person_role_to_read(item: PersonRole) -> PersonRoleRead:
item_id, label, is_active = _registry_read_values(item)
return PersonRoleRead(id=item_id, label=label, is_active=is_active)
def _registry_read_values(item: DocumentType | PersonRole) -> tuple[UUID, str, bool]:
return item.id, item.label, item.is_active
def _document_person_to_read(item: DocumentPerson) -> DocumentPersonRead:
person_name = item.person.full_name if item.person is not None else None
return DocumentPersonRead(
id=item.id,
document_id=item.document_id,
person_id=item.person_id,
role_id=item.role_id,
role_label=item.role_ref.label if item.role_ref is not None else None,
person_name=person_name,
)
def _document_to_type_response(item: Document) -> DocumentTypeWriteResponse:
if item.document_type_id is None:
raise ValueError("Document Type assignment did not persist")
return DocumentTypeWriteResponse(
document_id=item.id,
document_type_id=item.document_type_id,
)
def get_document_service(request: Request) -> DocumentService:
"""Resolve the document service from app lifespan state when available."""
services = getattr(request.app.state, "services", None)
if services is not None:
return services.documents
return DocumentService()
def get_people_service(request: Request) -> PeopleService:
"""Resolve the People service from app lifespan state when available."""
services = getattr(request.app.state, "services", None)
if services is not None:
return services.people
return PeopleService()
DocumentServiceDependency = Annotated[DocumentService, Depends(get_document_service)]
PeopleServiceDependency = Annotated[PeopleService, Depends(get_people_service)]
@router.get("/document-types", response_model=list[DocumentTypeRead])
async def list_document_types(
service: DocumentServiceDependency,
active_only: bool = True,
) -> list[DocumentTypeRead]:
items = await service.list_document_types(active_only=active_only)
return [_document_type_to_read(item) for item in items]
@router.get("/person-roles", response_model=list[PersonRoleRead])
async def list_person_roles(
service: PeopleServiceDependency,
active_only: bool = True,
) -> list[PersonRoleRead]:
items = await service.list_person_roles(active_only=active_only)
return [_person_role_to_read(item) for item in items]
@router.put("/documents/{document_id}/type", response_model=DocumentTypeWriteResponse)
async def set_document_type(
document_id: UUID,
payload: DocumentTypeWriteRequest,
service: DocumentServiceDependency,
) -> DocumentTypeWriteResponse:
document = await service.set_document_type(
document_id=document_id,
document_type_id=payload.document_type_id,
)
return _document_to_type_response(document)
@router.get("/documents/{document_id}/people", response_model=DocumentPeopleResponse)
async def list_document_people(
document_id: UUID,
service: PeopleServiceDependency,
) -> DocumentPeopleResponse:
links = await service.list_document_people(document_id=document_id)
return DocumentPeopleResponse(document_id=document_id, links=[_document_person_to_read(item) for item in links])
@router.post("/documents/{document_id}/people", response_model=DocumentPersonRead)
async def add_document_person_link(
document_id: UUID,
payload: DocumentPersonWriteRequest,
service: PeopleServiceDependency,
) -> DocumentPersonRead:
link = await service.add_document_person_link(
document_id=document_id,
person_id=payload.person_id,
role_id=payload.role_id,
)
return _document_person_to_read(link)
@router.patch("/document-people/{document_person_id}", response_model=DocumentPersonRead)
async def set_document_person_role(
document_person_id: UUID,
payload: DocumentPersonRoleUpdateRequest,
service: PeopleServiceDependency,
) -> DocumentPersonRead:
link = await service.set_document_person_role(
document_person_id=document_person_id,
role_id=payload.role_id,
)
return _document_person_to_read(link)
@router.delete("/document-people/{document_person_id}", status_code=204)
async def delete_document_person_link(
document_person_id: UUID,
service: PeopleServiceDependency,
) -> Response:
await service.remove_document_person_link(document_person_id=document_person_id)
return Response(status_code=204)
src/transcription/api/errors.py
-58
View File
@@ -1,58 +0,0 @@
"""Centralized API exception handlers."""
from __future__ import annotations
import logging
from fastapi import FastAPI
from fastapi import Request
from fastapi.responses import JSONResponse
from transcription.errors import AppError
from transcription.errors import ErrorCategory
from transcription.errors import build_error_envelope
logger = logging.getLogger(__name__)
_STATUS_BY_CATEGORY: dict[ErrorCategory, int] = {
ErrorCategory.VALIDATION: 400,
ErrorCategory.USER_INPUT: 400,
ErrorCategory.NOT_FOUND: 404,
ErrorCategory.CONFLICT: 409,
ErrorCategory.EXTERNAL_PROVIDER: 503,
ErrorCategory.EXTERNAL_TIMEOUT: 503,
ErrorCategory.INFRA_TRANSIENT: 503,
ErrorCategory.PROCESSING: 500,
ErrorCategory.INFRA_PERSISTENT: 500,
ErrorCategory.INTERNAL_UNEXPECTED: 500,
}
def _status_for(error: AppError) -> int:
return _STATUS_BY_CATEGORY.get(error.category, 500)
def register_error_handlers(app: FastAPI) -> None:
"""Register API exception handlers on the app."""
@app.exception_handler(AppError)
async def app_error_handler(_request: Request, exc: AppError) -> JSONResponse:
envelope = build_error_envelope(exc)
return JSONResponse(status_code=_status_for(exc), content=envelope.__dict__)
@app.exception_handler(Exception)
async def fallback_error_handler(_request: Request, exc: Exception) -> JSONResponse:
normalized = AppError(
"Unexpected error while handling request",
category=ErrorCategory.INTERNAL_UNEXPECTED,
suggestion="Retry once. If it persists, report the error reference id.",
)
logger.exception(
"Unhandled API exception operation=api.request error_id=%s category=%s exception_type=%s",
normalized.error_id,
normalized.category.value,
type(exc).__name__,
)
envelope = build_error_envelope(normalized)
return JSONResponse(status_code=_status_for(normalized), content=envelope.__dict__)
src/transcription/api/health.py
-44
View File
@@ -1,44 +0,0 @@
"""Health endpoint routes."""
from typing import NotRequired
from typing import TypedDict
from fastapi import APIRouter
from fastapi import Request
from transcription.worker import resolve_worker_health
router = APIRouter()
class WorkerHealthPayload(TypedDict):
state: str
error_id: NotRequired[str]
error_category: NotRequired[str]
class HealthPayload(TypedDict):
status: str
worker: WorkerHealthPayload
def healthz(request: Request) -> HealthPayload:
"""Return health status with worker-liveness signal."""
worker = resolve_worker_health(request.app.state)
payload: HealthPayload = {
"status": "ok",
"worker": {
"state": worker.state,
},
}
if worker.error_id is not None:
payload["worker"]["error_id"] = worker.error_id
if worker.error_category is not None:
payload["worker"]["error_category"] = worker.error_category
return payload
@router.get("/healthz")
def healthz_route(request: Request) -> HealthPayload:
"""Route wrapper for health status payload."""
return healthz(request)
src/transcription/api/print_api.py
-54
View File
@@ -1,54 +0,0 @@
"""Safe media route for Document print previews."""
from __future__ import annotations
from pathlib import Path
from typing import Annotated
from uuid import UUID
from fastapi import APIRouter
from fastapi import Depends
from fastapi import HTTPException
from fastapi import Request
from fastapi.responses import FileResponse
from transcription.services.source_media import SOURCE_MIME_TYPES
from transcription.services.sources import SourceService
router = APIRouter(prefix="/api", tags=["print"])
def get_source_service(request: Request) -> SourceService:
services = getattr(request.app.state, "services", None)
if services is not None:
return services.sources
return SourceService()
SourceServiceDependency = Annotated[SourceService, Depends(get_source_service)]
@router.get("/documents/{document_id}/sources/{source_id}/media", response_class=FileResponse)
async def read_document_source_media(
document_id: UUID,
source_id: UUID,
service: SourceServiceDependency,
) -> FileResponse:
"""Serve one validated Source through record identifiers, never a supplied path."""
source = await service.read_source(source_id)
if source.document_id != document_id:
raise HTTPException(status_code=404, detail="Source not found for Document")
upload_root = service.settings.upload_dir.resolve()
path = (upload_root / Path(source.file_path)).resolve()
try:
path.relative_to(upload_root)
except ValueError as exc:
raise HTTPException(status_code=404, detail="Source media is outside managed storage") from exc
if not path.is_file():
raise HTTPException(status_code=404, detail="Source media is unavailable")
media_type = SOURCE_MIME_TYPES.get(path.suffix.lower())
if media_type is None:
raise HTTPException(status_code=415, detail="Unsupported Source media type")
return FileResponse(path, media_type=media_type)
src/transcription/app.py
-111
View File
@@ -1,111 +0,0 @@
"""Application factory and lifespan wiring for the transcription app."""
from __future__ import annotations
import logging
from contextlib import AsyncExitStack
from contextlib import asynccontextmanager
from datetime import UTC
from datetime import datetime
from datetime import timedelta
from fastapi import FastAPI
from fastapi import status
from fastapi.responses import RedirectResponse
from fastapi.staticfiles import StaticFiles
from .api.documents_api import router as documents_router
from .api.errors import register_error_handlers
from .api.health import router as health_router
from .api.print_api import router as print_router
from .config import Settings
from .config import configure_logging
from .config import get_settings
from .db import create_all
from .db import dispose_database_runtime
from .db import initialize_database_runtime
from .db import reconcile_canonical_media_paths
from .db import reconcile_legacy_job_source_columns
from .services import ServiceBundle
from .ui import register_pages
from .worker import worker_consumer_lifespan
logger = logging.getLogger(__name__)
@asynccontextmanager
async def _lifespan(app: FastAPI):
settings = getattr(app.state, "settings", None) or get_settings()
configure_logging(settings)
app.state.settings = settings
app.state.runtime = initialize_database_runtime(settings=settings)
session_factory = app.state.runtime.session_factory
app.state.services = ServiceBundle.from_session_factory(session_factory, settings=settings)
if settings.should_bootstrap_schema:
await create_all(engine=app.state.runtime.engine)
await reconcile_legacy_job_source_columns(engine=app.state.runtime.engine)
await reconcile_canonical_media_paths(engine=app.state.runtime.engine)
settings.upload_dir.mkdir(parents=True, exist_ok=True)
settings.prompt_dir.mkdir(parents=True, exist_ok=True)
settings.log_dir.mkdir(parents=True, exist_ok=True)
await _recover_stale_processing_jobs(app)
async with AsyncExitStack() as stack:
stack.push_async_callback(dispose_database_runtime)
if settings.run_embedded_worker:
stop_event, worker_notifier, worker_health = await stack.enter_async_context(
worker_consumer_lifespan(
session_factory=app.state.runtime.session_factory,
poll_interval_seconds=settings.worker_poll_interval_seconds,
shutdown_timeout_seconds=(
settings.worker_provider_timeout_seconds + settings.worker_shutdown_grace_seconds
),
)
)
app.state.worker_stop_event = stop_event
app.state.worker_notifier = worker_notifier
app.state.worker_health = worker_health
yield
async def _recover_stale_processing_jobs(app: FastAPI) -> None:
"""Re-queue stale processing jobs at startup.
Any job left in PROCESSING longer than the stale-job threshold is assumed
orphaned and moved back to QUEUED before the worker starts.
"""
settings = app.state.settings
stale_before = datetime.now(UTC) - timedelta(seconds=settings.worker_stale_job_seconds)
recovered = await app.state.services.jobs.requeue_stale_processing_jobs(stale_before=stale_before)
if recovered > 0:
logger.warning("Recovered %s stale processing job(s) at startup", recovered)
def create_app(settings: Settings | None = None) -> FastAPI:
"""Create and configure the FastAPI application."""
app = FastAPI(title="Transcription", lifespan=_lifespan)
active_settings = settings or get_settings()
app.state.settings = active_settings
app.mount(
"/uploads",
StaticFiles(directory=active_settings.upload_dir, check_dir=False),
name="uploads",
)
@app.get("/", include_in_schema=False)
async def root_redirect() -> RedirectResponse:
return RedirectResponse(url="/ui/homepage", status_code=status.HTTP_307_TEMPORARY_REDIRECT)
@app.get("/ui", include_in_schema=False)
async def ui_redirect() -> RedirectResponse:
return RedirectResponse(url="/ui/homepage", status_code=status.HTTP_307_TEMPORARY_REDIRECT)
register_error_handlers(app)
app.include_router(health_router)
app.include_router(documents_router)
app.include_router(print_router)
register_pages(app)
return app
src/transcription/benchmarking.py
-91
View File
@@ -1,91 +0,0 @@
"""Private-corpus benchmark contracts and deterministic text scoring.
This module is retained as the implementation of the evaluation policy in
`docs/invariant/ai_evidence_and_provenance.md` §5. Application runtime paths do
not call it directly, but preserved execution attempts and manually reviewed
references need a deterministic scorer that remains importable for tests and
operator tooling.
"""
from __future__ import annotations
from uuid import UUID
from pydantic import BaseModel
from pydantic import ConfigDict
from pydantic import Field
class BenchmarkModel(BaseModel):
model_config = ConfigDict(extra="forbid", frozen=True)
class EditorialAssessment(BenchmarkModel):
"""Manually reviewed errors not represented adequately by CER or WER."""
omissions: int = Field(default=0, ge=0)
inventions: int = Field(default=0, ge=0)
silent_normalizations: int = Field(default=0, ge=0)
uncertainty_errors: int = Field(default=0, ge=0)
layout_errors: int = Field(default=0, ge=0)
class BenchmarkScore(BenchmarkModel):
"""Measured score for one preserved execution attempt."""
execution_attempt_id: UUID
character_error_rate: float = Field(ge=0)
word_error_rate: float = Field(ge=0)
character_edits: int = Field(ge=0)
word_edits: int = Field(ge=0)
reference_characters: int = Field(ge=0)
reference_words: int = Field(ge=0)
assessment: EditorialAssessment
latency_ms: int = Field(ge=0)
cost_usd: float | None = Field(default=None, ge=0)
def score_transcription(
*,
execution_attempt_id: UUID,
reference: str,
candidate: str,
assessment: EditorialAssessment,
latency_ms: int,
cost_usd: float | None = None,
) -> BenchmarkScore:
"""Score literal text without case-folding or silent normalization."""
reference_words = reference.split()
candidate_words = candidate.split()
character_edits = _levenshtein(list(reference), list(candidate))
word_edits = _levenshtein(reference_words, candidate_words)
return BenchmarkScore(
execution_attempt_id=execution_attempt_id,
character_error_rate=character_edits / max(1, len(reference)),
word_error_rate=word_edits / max(1, len(reference_words)),
character_edits=character_edits,
word_edits=word_edits,
reference_characters=len(reference),
reference_words=len(reference_words),
assessment=assessment,
latency_ms=latency_ms,
cost_usd=cost_usd,
)
def _levenshtein(reference: list[str], candidate: list[str]) -> int:
if len(reference) < len(candidate):
reference, candidate = candidate, reference
previous = list(range(len(candidate) + 1))
for reference_index, reference_value in enumerate(reference, start=1):
current = [reference_index]
for candidate_index, candidate_value in enumerate(candidate, start=1):
current.append(
min(
current[-1] + 1,
previous[candidate_index] + 1,
previous[candidate_index - 1] + (reference_value != candidate_value),
)
)
previous = current
return previous[-1]
src/transcription/config.py
-286
View File
@@ -1,286 +0,0 @@
"""Centralized application configuration.
All settings are loaded from environment variables (or an env file)
once at startup. Provider-specific defaults (model names, base URLs)
are resolved by the provider adapters, not here.
"""
import copy
import logging.config
import os
from collections.abc import Sequence
from enum import StrEnum
from functools import cache
from pathlib import Path
from typing import Annotated
from typing import Any
from typing import Literal
from pydantic import BaseModel
from pydantic import ConfigDict
from pydantic import Field
from pydantic import SecretStr
from pydantic import StringConstraints
from pydantic import field_validator
from pydantic import model_validator
from pydantic_settings import BaseSettings
from pydantic_settings import SettingsConfigDict
logger = logging.getLogger(__name__)
PROJECT_ROOT = Path(__file__).resolve().parents[2]
DEFAULT_ENV_FILE_NAME = ".env.production"
def resolve_settings_env_file_path() -> Path:
"""Resolve the runtime env file independent of the process working directory."""
override = os.getenv("ENV_FILE", "").strip()
if override:
return Path(override)
return PROJECT_ROOT / DEFAULT_ENV_FILE_NAME
class Provider(StrEnum):
OPENROUTER = "openrouter"
NonEmptyStr = Annotated[str, StringConstraints(strip_whitespace=True, min_length=1)]
PromptFilename = Annotated[str, StringConstraints(strip_whitespace=True, min_length=1, pattern=r"^[^/\\]+$")]
Probability = Annotated[float, Field(ge=0.0, le=1.0)]
Temperature = Annotated[float, Field(ge=0.0, le=2.0)]
DEFAULT_PROVIDER_MODEL = "google/gemini-2.5-flash"
WORKER_STALE_TIMEOUT_MULTIPLIER = 3.0
class SqliteSettings(BaseModel):
model_config = ConfigDict(extra="forbid", frozen=True)
driver: Literal["sqlite"] = "sqlite"
path: NonEmptyStr = "./data/transcription.db"
class PostgresSettings(BaseModel):
model_config = ConfigDict(extra="forbid", frozen=True)
driver: Literal["postgres"] = "postgres"
host: NonEmptyStr
port: int = Field(default=5432, ge=1, le=65535)
database: NonEmptyStr
user: NonEmptyStr
password: SecretStr
DatabaseSettings = Annotated[
SqliteSettings | PostgresSettings,
Field(discriminator="driver"),
]
class Settings(BaseSettings):
model_config = SettingsConfigDict(
env_file=None,
env_file_encoding="utf-8",
extra="ignore",
env_nested_delimiter="__",
cli_implicit_flags=True,
cli_kebab_case=True,
frozen=True,
)
def __init__(self, /, **values: Any) -> None:
if "_env_file" not in values:
values["_env_file"] = resolve_settings_env_file_path()
super().__init__(**values)
# --- NiceGUI Server ---
host: str = "0.0.0.0"
port: int = 8000
log_level: Literal["critical", "error", "warning", "info", "debug", "trace"] = "info"
reload: bool = False
log_dir: Path = Path("./data/logs")
log_file_name: NonEmptyStr = "transcription.log"
log_file_max_bytes: int = Field(default=10 * 1024 * 1024, gt=0)
log_file_backup_count: int = Field(default=5, ge=1)
# --- AI provider ---
provider: Provider = Provider.OPENROUTER
openrouter_api_key: SecretStr
provider_model: NonEmptyStr | None = DEFAULT_PROVIDER_MODEL
provider_models: tuple[NonEmptyStr, ...] = ()
openrouter_http_referer: NonEmptyStr | None = None
openrouter_app_title: NonEmptyStr | None = None
default_prompt_name: PromptFilename = "transcribe_document.md"
transcription_temperature: Temperature | None = None
transcription_top_p: Probability | None = None
# --- runtime environment ---
environment: Literal["development", "test", "production"] = "development"
transcription_commit: NonEmptyStr | None = None
run_embedded_worker: bool = True
# --- persistence ---
database: DatabaseSettings = Field(default_factory=SqliteSettings)
bootstrap_schema_on_startup: bool = False
sqlite_check_same_thread: bool = False
# --- filesystem paths ---
upload_dir: Path = Path("./data")
prompt_dir: Path = Path("./prompts")
# --- worker reliability ---
worker_max_retries: int = Field(default=0, ge=0)
# Bounded only from below. Vision transcription of a dense page routinely runs
# well past twenty seconds, so an upper cap here would silently fail real work.
worker_provider_timeout_seconds: float = Field(default=30.0, gt=0.0)
worker_stale_job_seconds: float = Field(default=90.0, gt=0.0)
worker_retry_backoff_seconds: float = Field(default=1.0, ge=0.0)
worker_shutdown_grace_seconds: float = Field(default=5.0, ge=0.0)
worker_poll_interval_seconds: float = Field(default=1.0, gt=0.0)
worker_min_transcription_chars: int = Field(default=0, ge=0)
worker_min_transcription_lines: int = Field(default=0, ge=0)
worker_fail_on_finish_reason_length: bool = False
@field_validator("provider_models", mode="before")
@classmethod
def validate_provider_models_input(cls, value: object) -> object:
if value is None:
return ()
if isinstance(value, (list, tuple)) and not value:
raise ValueError("PROVIDER_MODELS must contain at least one model")
return value
@model_validator(mode="before")
@classmethod
def normalize_provider_models(cls, data: object) -> object:
"""Build the immutable model selector with the configured default first.
This runs before field validation so the derived value is produced by
normal construction rather than by mutating a frozen instance.
"""
if not isinstance(data, dict):
return data
default_model = data.get("provider_model") or DEFAULT_PROVIDER_MODEL
if not isinstance(default_model, str):
return data
default_model = default_model.strip()
configured = data.get("provider_models")
if configured is None:
configured = ()
elif isinstance(configured, str):
# Left as-is so the field validator can report the malformed value.
return {**data, "provider_model": default_model}
elif not isinstance(configured, (list, tuple)):
return {**data, "provider_model": default_model}
elif not configured:
# Preserved so validate_provider_models_input can reject it.
return {**data, "provider_model": default_model}
deduplicated: list[str] = []
for model in (default_model, *configured):
if not isinstance(model, str):
return {**data, "provider_model": default_model}
normalized = model.strip()
if normalized not in deduplicated:
deduplicated.append(normalized)
return {**data, "provider_model": default_model, "provider_models": tuple(deduplicated)}
@model_validator(mode="before")
@classmethod
def _derive_worker_stale_job_seconds(cls, data: object) -> object:
"""Default stale-job recovery with margin over one provider timeout."""
if not isinstance(data, dict):
return data
if data.get("worker_stale_job_seconds") is not None:
return data
timeout = data.get("worker_provider_timeout_seconds", 30.0)
if not isinstance(timeout, (str, int, float)):
return data
try:
timeout_seconds = float(timeout)
except ValueError:
return data
return {
**data,
"worker_stale_job_seconds": timeout_seconds * WORKER_STALE_TIMEOUT_MULTIPLIER,
}
@model_validator(mode="after")
def _validate_worker_stale_job_seconds(self) -> "Settings":
"""Reject stale recovery that can fire before one provider timeout expires."""
if self.worker_stale_job_seconds <= self.worker_provider_timeout_seconds:
raise ValueError("WORKER_STALE_JOB_SECONDS must exceed WORKER_PROVIDER_TIMEOUT_SECONDS")
return self
@property
def should_bootstrap_schema(self) -> bool:
"""Return whether startup should auto-create schema for this environment."""
if "bootstrap_schema_on_startup" in self.model_fields_set:
return self.bootstrap_schema_on_startup
return self.environment in {"development", "test"}
@cache
def get_settings(**kwargs: Any) -> Settings:
"""Load cached settings without reading process CLI arguments."""
return Settings(_cli_parse_args=False, **kwargs)
def parse_cli_settings(args: Sequence[str] | None = None) -> Settings:
"""Load settings with CLI arguments at the executable boundary."""
cli_args = True if args is None else list(args)
return Settings(_cli_parse_args=cli_args)
LOGGING_CONFIG: dict[str, Any] = {
"version": 1,
"disable_existing_loggers": False,
"formatters": {
"standard": {
"format": "%(asctime)s %(levelname)-8s | %(message)s",
"datefmt": "%Y-%m-%d %H:%M:%S",
}
},
"handlers": {
"console": {
"class": "logging.StreamHandler",
"formatter": "standard",
"stream": "ext://sys.stdout",
},
"file": {
"class": "logging.handlers.RotatingFileHandler",
"formatter": "standard",
"filename": str(Path("./data/logs") / "transcription.log"),
"maxBytes": 10 * 1024 * 1024,
"backupCount": 5,
"encoding": "utf-8",
},
},
"root": {
"level": "INFO",
"handlers": ["console", "file"],
},
"loggers": {
"transcription": {
"level": "DEBUG",
"handlers": ["console", "file"],
"propagate": False,
}
},
}
def configure_logging(settings: Settings | None = None) -> None:
"""Configure root logging once at startup."""
cfg = copy.deepcopy(LOGGING_CONFIG)
active_settings = settings or get_settings()
active_settings.log_dir.mkdir(parents=True, exist_ok=True)
file_handler = cfg["handlers"]["file"]
file_handler["filename"] = str(active_settings.log_dir / active_settings.log_file_name)
file_handler["maxBytes"] = active_settings.log_file_max_bytes
file_handler["backupCount"] = active_settings.log_file_backup_count
cfg["loggers"]["transcription"]["level"] = active_settings.log_level.upper()
logging.config.dictConfig(cfg)
logger.debug("Logging configured")
src/transcription/db/__init__.py
-19
View File
@@ -1,19 +0,0 @@
from .operations import create_all
from .operations import reconcile_canonical_media_paths
from .operations import reconcile_legacy_job_source_columns
from .operations import reconcile_person_name_columns
from .runtime import dispose_database_runtime
from .runtime import initialize_database_runtime
from .session import session_scope
from .session import transaction_scope
__all__ = [
"create_all",
"dispose_database_runtime",
"initialize_database_runtime",
"reconcile_canonical_media_paths",
"reconcile_legacy_job_source_columns",
"reconcile_person_name_columns",
"session_scope",
"transaction_scope",
]
src/transcription/db/engine.py
-75
View File
@@ -1,75 +0,0 @@
from typing import Any
from sqlalchemy import URL
from sqlalchemy import StaticPool
from sqlalchemy.ext.asyncio import AsyncEngine
from sqlalchemy.ext.asyncio import create_async_engine
from ..config import PostgresSettings
from ..config import Settings
from ..config import SqliteSettings
from ..config import get_settings
def get_database_url(settings: Settings) -> str:
match settings.database:
case SqliteSettings(path=path):
url = URL.create(
drivername="sqlite+aiosqlite",
database=path,
)
case PostgresSettings() as database:
url = URL.create(
drivername="postgresql+asyncpg",
host=database.host,
port=database.port,
database=database.database,
username=database.user,
password=database.password.get_secret_value(),
)
return url.render_as_string(hide_password=False)
def resolve_engine(settings: Settings | None = None) -> AsyncEngine:
active_settings = settings or get_settings()
return get_engine(
get_database_url(active_settings),
sqlite_check_same_thread=active_settings.sqlite_check_same_thread,
)
_ENGINES: dict[str, AsyncEngine] = {}
def _create_engine(database_url: str, *, sqlite_check_same_thread: bool) -> AsyncEngine:
kwargs: dict[str, Any] = {"echo": False, "pool_pre_ping": True}
if database_url.startswith("sqlite"):
kwargs["connect_args"] = {"check_same_thread": sqlite_check_same_thread}
if ":memory:" in database_url:
kwargs["poolclass"] = StaticPool
return create_async_engine(database_url, **kwargs)
def get_engine(database_url: str, *, sqlite_check_same_thread: bool = False) -> AsyncEngine:
"""Return the process-wide engine for ``database_url``, creating it on first use.
Engines are registered per URL so that disposing one leaves every other
database untouched.
"""
engine = _ENGINES.get(database_url)
if engine is None:
engine = _create_engine(database_url, sqlite_check_same_thread=sqlite_check_same_thread)
_ENGINES[database_url] = engine
return engine
async def dispose_engine(database_url: str) -> None:
"""Dispose and unregister the engine for ``database_url`` only.
Unknown URLs are a no-op rather than provoking the creation of an engine
purely so that it can be thrown away.
"""
engine = _ENGINES.pop(database_url, None)
if engine is not None:
await engine.dispose()
src/transcription/db/loading.py
-45
View File
@@ -1,45 +0,0 @@
"""Typed loader-option wrappers for SQLModel relationship attributes.
SQLModel declares relationships with their runtime Python type, so
``Document.jobs`` is annotated ``list[Job]`` even though at runtime it is an
``InstrumentedAttribute``. SQLAlchemy's loader options are typed against
``QueryableAttribute``, so every eager-load call site reads as a type error to a
static checker even though the code is correct.
These wrappers put that reinterpretation in one documented place instead of
scattering a suppression comment across every eager-load call. Import
``selectinload`` and ``defer`` from here rather than from ``sqlalchemy.orm``.
Multi-level eager loads must keep using the chained form --
``selectinload(A.b).selectinload(orm_attribute(B.c))`` -- and not the varargs
form ``selectinload(A.b, B.c)``. The two produce the same loader path, but
varargs applies the selectin strategy only to the last element while the
intermediate falls back to its default strategy. Every relationship here
declares ``lazy="raise"``, so the varargs form raises at render time.
"""
from __future__ import annotations
from typing import Any
from typing import cast
from sqlalchemy.orm import defer as _defer
from sqlalchemy.orm import selectinload as _selectinload
from sqlalchemy.orm.attributes import QueryableAttribute
from sqlalchemy.orm.strategy_options import _AbstractLoad
def orm_attribute(attribute: object) -> QueryableAttribute[Any]:
"""Reinterpret a SQLModel relationship or field as its ORM descriptor."""
return cast("QueryableAttribute[Any]", attribute)
def selectinload(*keys: object) -> _AbstractLoad:
"""``sqlalchemy.orm.selectinload`` accepting SQLModel-annotated attributes."""
return _selectinload(*(orm_attribute(key) for key in keys))
def defer(*keys: object, raiseload: bool = False) -> _AbstractLoad:
"""``sqlalchemy.orm.defer`` accepting SQLModel-annotated attributes."""
first, *rest = (orm_attribute(key) for key in keys)
return _defer(first, *rest, raiseload=raiseload)
src/transcription/db/migration.py
-610
View File
@@ -1,610 +0,0 @@
from __future__ import annotations
import base64
import json
import shutil
from collections.abc import Sequence
from dataclasses import dataclass
from datetime import UTC
from datetime import date
from datetime import datetime
from pathlib import Path
from typing import Any
from uuid import UUID
from uuid import uuid4
from sqlalchemy import URL
from sqlalchemy import MetaData
from sqlalchemy import Table
from sqlalchemy import bindparam
from sqlalchemy import create_engine
from sqlalchemy import func
from sqlalchemy import inspect as sqlalchemy_inspect
from sqlalchemy import select
from sqlalchemy import text
from sqlalchemy.engine import RowMapping
from sqlalchemy.engine import make_url
from sqlmodel import SQLModel
from transcription.config import Settings
from transcription.config import get_settings
# Register table metadata.
from transcription.db import models as _models # noqa: F401
from transcription.db.engine import get_database_url
EXPORT_TABLE_ORDER = (
"document_type",
"person_role",
"tag",
"document",
"person",
"genealogy_person",
"genealogy_family",
"photo",
"document_person",
"document_tag",
"person_tag",
"job",
"source",
"job_source",
"execution_attempt",
"genealogy_family_child",
"genealogy_citation",
)
BYTES_FIELDS = {"transport_body"}
VERIFICATION_TABLES = EXPORT_TABLE_ORDER
@dataclass(frozen=True)
class MigrationPaths:
source_db_url: str
target_db_url: str
source_upload_dir: Path
target_upload_dir: Path
bundle_dir: Path
def export_bundle(*, source_db_url: str, source_upload_dir: Path, bundle_dir: Path) -> None:
bundle_dir.mkdir(parents=True, exist_ok=True)
export_json = bundle_dir / "database.json"
uploads_bundle_dir = bundle_dir / "uploads"
payload: dict[str, Any] = {
"schema_name": "transcription.export-import",
"schema_version": "1",
"created_at": datetime.now(UTC).isoformat(),
"tables": {},
}
engine = create_engine(source_db_url)
legacy_portrait_rows: Sequence[RowMapping] = ()
try: # noqa: PLR1702
inspector = sqlalchemy_inspect(engine)
source_tables = set(inspector.get_table_names())
metadata = MetaData()
metadata.reflect(bind=engine)
current_metadata = SQLModel.metadata
with engine.connect() as connection:
for table_name in EXPORT_TABLE_ORDER:
if table_name not in source_tables:
payload["tables"][table_name] = []
continue
source_table = metadata.tables[table_name]
target_table = current_metadata.tables[table_name]
export_columns = [column.name for column in target_table.columns if column.name in source_table.columns]
if table_name == "person" and "full_name" in source_table.columns:
for legacy_column in ("full_name",):
if legacy_column not in export_columns:
export_columns.append(legacy_column)
if table_name == "person" and "portrait_path" in source_table.columns:
legacy_portrait_rows = (
connection.execute(
select(source_table.c["id"], source_table.c["portrait_path"]).where(
source_table.c["portrait_path"].is_not(None)
)
)
.mappings()
.all()
)
rows = connection.execute(select(*(source_table.c[name] for name in export_columns))).mappings().all()
payload["tables"][table_name] = [
_serialize_row(row, table_name=table_name, source_upload_dir=source_upload_dir) for row in rows
]
finally:
engine.dispose()
if uploads_bundle_dir.exists():
shutil.rmtree(uploads_bundle_dir)
if source_upload_dir.exists():
shutil.copytree(source_upload_dir, uploads_bundle_dir)
else:
uploads_bundle_dir.mkdir(parents=True, exist_ok=True)
_prepare_photo_payload_and_uploads(
payload=payload,
uploads_bundle_dir=uploads_bundle_dir,
legacy_portrait_rows=legacy_portrait_rows,
)
_relocate_homepage_markdown(uploads_bundle_dir=uploads_bundle_dir)
export_json.write_text(json.dumps(payload, indent=2), encoding="utf-8")
def import_bundle(*, target_db_url: str, target_upload_dir: Path, bundle_dir: Path) -> None:
export_json = bundle_dir / "database.json"
uploads_bundle_dir = bundle_dir / "uploads"
payload = json.loads(export_json.read_text(encoding="utf-8"))
if target_upload_dir.exists():
shutil.rmtree(target_upload_dir)
target_upload_dir.mkdir(parents=True, exist_ok=True)
if uploads_bundle_dir.exists():
shutil.copytree(uploads_bundle_dir, target_upload_dir, dirs_exist_ok=True)
_reset_sqlite_target_file(target_db_url)
_ensure_sqlite_target_parent_exists(target_db_url)
engine = create_engine(target_db_url)
try:
SQLModel.metadata.create_all(engine)
execution_attempt_ids = _collect_execution_attempt_ids(payload)
with engine.begin() as connection:
for table_name in reversed(EXPORT_TABLE_ORDER):
table = SQLModel.metadata.tables[table_name]
connection.execute(table.delete())
deferred_source_preferred_attempt_updates: list[dict[str, Any]] = []
for table_name in EXPORT_TABLE_ORDER:
rows = payload.get("tables", {}).get(table_name, [])
if not rows:
continue
table = SQLModel.metadata.tables[table_name]
if table_name == "source":
prepared_source_rows, updates = _prepare_source_rows_for_import(
rows=rows,
source_table=table,
execution_attempt_ids=execution_attempt_ids,
)
deferred_source_preferred_attempt_updates.extend(updates)
connection.execute(table.insert(), prepared_source_rows)
continue
connection.execute(table.insert(), [_deserialize_row(row, table) for row in rows])
if deferred_source_preferred_attempt_updates:
source_table = SQLModel.metadata.tables["source"]
connection.execute(
source_table.update()
.where(source_table.c.id == bindparam("source_id"))
.values(preferred_execution_attempt_id=bindparam("preferred_execution_attempt_id")),
deferred_source_preferred_attempt_updates,
)
finally:
engine.dispose()
def _collect_execution_attempt_ids(payload: dict[str, Any]) -> set[str]:
execution_attempt_rows = payload.get("tables", {}).get("execution_attempt", [])
return {_normalize_uuid_like(row.get("id")) for row in execution_attempt_rows if row.get("id") is not None}
def _prepare_source_rows_for_import(
*,
rows: list[dict[str, Any]],
source_table: Table,
execution_attempt_ids: set[str],
) -> tuple[list[dict[str, Any]], list[dict[str, Any]]]:
prepared_source_rows: list[dict[str, Any]] = []
updates: list[dict[str, Any]] = []
for row in rows:
source_row = _deserialize_row(row, source_table)
source_id = source_row.get("id")
preferred_attempt_id = source_row.get("preferred_execution_attempt_id")
if (
source_id is not None
and preferred_attempt_id is not None
and _normalize_uuid_like(preferred_attempt_id) in execution_attempt_ids
):
updates.append(
{
"source_id": source_id,
"preferred_execution_attempt_id": preferred_attempt_id,
}
)
source_row["preferred_execution_attempt_id"] = None
prepared_source_rows.append(source_row)
return prepared_source_rows, updates
def _ensure_sqlite_target_parent_exists(target_db_url: str) -> None:
parsed = make_url(target_db_url)
if not parsed.drivername.startswith("sqlite"):
return
database = parsed.database
if not database or database == ":memory:":
return
Path(database).parent.mkdir(parents=True, exist_ok=True)
def _reset_sqlite_target_file(target_db_url: str) -> None:
parsed = make_url(target_db_url)
if not parsed.drivername.startswith("sqlite"):
return
database = parsed.database
if not database or database == ":memory:":
return
target = Path(database)
if target.exists():
target.unlink()
def migrate_via_bundle(paths: MigrationPaths) -> None:
export_bundle(
source_db_url=paths.source_db_url,
source_upload_dir=paths.source_upload_dir,
bundle_dir=paths.bundle_dir,
)
import_bundle(
target_db_url=paths.target_db_url,
target_upload_dir=paths.target_upload_dir,
bundle_dir=paths.bundle_dir,
)
@dataclass(frozen=True)
class MigrationVerificationReport:
source_counts: dict[str, int]
target_counts: dict[str, int]
mismatched_tables: dict[str, dict[str, int]]
integrity_violations: dict[str, int]
success: bool
def to_dict(self) -> dict[str, Any]:
return {
"success": self.success,
"source_counts": self.source_counts,
"target_counts": self.target_counts,
"mismatched_tables": self.mismatched_tables,
"integrity_violations": self.integrity_violations,
}
def sqlite_url_from_path(path: Path) -> str:
return URL.create(drivername="sqlite", database=str(path)).render_as_string(hide_password=False)
def default_sync_db_url(settings: Settings | None = None) -> str:
runtime_settings = settings or get_settings()
return get_database_url(runtime_settings).replace("+aiosqlite", "").replace("+asyncpg", "").replace("+psycopg", "")
def verify_migration(*, source_db_url: str, target_db_url: str) -> MigrationVerificationReport:
source_counts = _table_counts(source_db_url)
target_counts = _table_counts(target_db_url)
mismatched_tables = {
table_name: {"source": source_counts[table_name], "target": target_counts[table_name]}
for table_name in VERIFICATION_TABLES
if source_counts[table_name] != target_counts[table_name]
}
integrity_violations = _integrity_violations(target_db_url)
success = not mismatched_tables and all(count == 0 for count in integrity_violations.values())
return MigrationVerificationReport(
source_counts=source_counts,
target_counts=target_counts,
mismatched_tables=mismatched_tables,
integrity_violations=integrity_violations,
success=success,
)
def _table_counts(db_url: str) -> dict[str, int]:
engine = create_engine(db_url)
try:
metadata = MetaData()
metadata.reflect(bind=engine)
counts: dict[str, int] = {}
with engine.connect() as connection:
for table_name in VERIFICATION_TABLES:
table = metadata.tables.get(table_name)
if table is None:
counts[table_name] = 0
continue
counts[table_name] = int(connection.execute(select(func.count()).select_from(table)).scalar_one())
return counts
finally:
engine.dispose()
def _integrity_violations(db_url: str) -> dict[str, int]:
checks = {
"orphan_source_document": (
"select count(*) from source s left join document d on d.id = s.document_id where d.id is null"
),
"orphan_job_document": (
"select count(*) from job j left join document d on d.id = j.document_id where d.id is null"
),
"orphan_job_source_job": (
"select count(*) from job_source js left join job j on j.id = js.job_id where j.id is null"
),
"orphan_job_source_source": (
"select count(*) from job_source js left join source s on s.id = js.source_id where s.id is null"
),
"orphan_attempt_job_source": (
"select count(*) from execution_attempt ea "
"left join job_source js on js.id = ea.job_source_id "
"where js.id is null"
),
"orphan_attempt_job": (
"select count(*) from execution_attempt ea left join job j on j.id = ea.job_id where j.id is null"
),
"orphan_attempt_source": (
"select count(*) from execution_attempt ea left join source s on s.id = ea.source_id where s.id is null"
),
"duplicate_attempt_numbers": (
"select count(*) from ("
" select job_id, source_id, attempt_number, count(*) as c"
" from execution_attempt"
" group by job_id, source_id, attempt_number"
" having count(*) > 1"
") x"
),
}
engine = create_engine(db_url)
try:
with engine.connect() as connection:
return {
check_name: int(connection.execute(text(query)).scalar_one()) for check_name, query in checks.items()
}
finally:
engine.dispose()
def _serialize_row(row: RowMapping, *, table_name: str, source_upload_dir: Path) -> dict[str, Any]:
serialized: dict[str, Any] = {}
for raw_key, value in row.items():
key = str(raw_key)
serialized_value = _serialize_value(value)
if table_name == "source" and key == "file_path" and isinstance(serialized_value, str):
serialized[key] = _canonical_media_relative_path(
serialized_value,
source_upload_dir=source_upload_dir,
preferred_prefix="documents/",
)
continue
if table_name == "photo" and key == "path" and isinstance(serialized_value, str):
serialized[key] = _canonical_media_relative_path(
serialized_value,
source_upload_dir=source_upload_dir,
preferred_prefix="photos/",
)
continue
if table_name == "person" and key == "full_name" and isinstance(serialized_value, str):
given_names, last_name = _split_legacy_full_name(serialized_value)
serialized["given_names"] = given_names
serialized["last_name"] = last_name
continue
serialized[key] = serialized_value
if table_name == "person":
serialized["given_names"] = str(serialized.get("given_names") or "").strip()
serialized["last_name"] = str(serialized.get("last_name") or "").strip()
return serialized
def _split_legacy_full_name(full_name: str) -> tuple[str, str]:
tokens = [token for token in full_name.strip().split() if token]
if len(tokens) >= 2:
return (" ".join(tokens[:-1]), tokens[-1])
if len(tokens) == 1:
return (tokens[0], tokens[0])
return ("Unknown", "Unknown")
def _serialize_value(value: Any) -> Any:
if isinstance(value, UUID):
return str(value)
if isinstance(value, (datetime, date)):
return value.isoformat()
if isinstance(value, bytes):
return {"encoding": "base64", "data": base64.b64encode(value).decode("ascii")}
if isinstance(value, dict):
return {str(k): _serialize_value(v) for k, v in value.items()}
if isinstance(value, list):
return [_serialize_value(item) for item in value]
return value
def _deserialize_row(row: dict[str, Any], table: Table) -> dict[str, Any]:
deserialized: dict[str, Any] = {}
for key, value in row.items():
if key in BYTES_FIELDS and isinstance(value, dict) and value.get("encoding") == "base64":
deserialized[key] = base64.b64decode(value["data"])
continue
if key in table.columns:
try:
python_type: type[Any] = table.columns[key].type.python_type
except NotImplementedError:
deserialized[key] = value
continue
deserialized[key] = _deserialize_value(python_type, value)
return deserialized
def _deserialize_value(python_type: type[Any], value: Any) -> Any:
if value is None:
return None
if python_type is UUID and isinstance(value, str):
return UUID(value)
if python_type is datetime and isinstance(value, str):
return datetime.fromisoformat(value)
if python_type is date and isinstance(value, str):
return date.fromisoformat(value)
return value
def _normalize_uuid_like(value: Any) -> str:
if isinstance(value, UUID):
return str(value)
if isinstance(value, str):
text_value = value.strip()
try:
return str(UUID(text_value))
except ValueError:
return text_value
return str(value)
def _canonical_media_relative_path(value: str, *, source_upload_dir: Path, preferred_prefix: str) -> str:
normalized = value.strip().replace("\\", "/")
lowered = normalized.casefold()
upload_root = source_upload_dir.resolve().as_posix().casefold().rstrip("/")
if lowered.startswith(upload_root + "/"):
normalized = normalized[len(source_upload_dir.resolve().as_posix()) + 1 :]
lowered = normalized.casefold()
if lowered.startswith("/uploads/"):
normalized = normalized[len("/uploads/") :]
lowered = normalized.casefold()
elif lowered.startswith("uploads/"):
normalized = normalized[len("uploads/") :]
lowered = normalized.casefold()
elif lowered.startswith("data/"):
normalized = normalized[len("data/") :]
lowered = normalized.casefold()
if preferred_prefix == "persons/" and lowered.startswith("portraits/"):
normalized = "persons/" + normalized[len("portraits/") :]
lowered = normalized.casefold()
for prefix in ("documents/", "photos/", "persons/"):
marker = f"/{prefix}"
index = lowered.find(marker)
if index >= 0:
normalized = normalized[index + 1 :]
lowered = normalized.casefold()
break
if not lowered.startswith(preferred_prefix):
return normalized
return Path(normalized).as_posix()
def _prepare_photo_payload_and_uploads( # noqa: PLR0915
*,
payload: dict[str, Any],
uploads_bundle_dir: Path,
legacy_portrait_rows: Sequence[RowMapping],
) -> None:
photo_rows = payload.setdefault("tables", {}).setdefault("photo", [])
photos_dir = uploads_bundle_dir / "photos"
photos_dir.mkdir(parents=True, exist_ok=True)
# Keep only photo rows whose referenced media exists inside the uploads tree.
# This prevents stale/injected rows from blocking legacy backfill.
retained_rows: list[dict[str, Any]] = []
for row in photo_rows:
path_value = row.get("path")
if not isinstance(path_value, str) or not path_value.strip():
continue
canonical_path = _canonical_media_relative_path(
path_value,
source_upload_dir=uploads_bundle_dir,
preferred_prefix="photos/",
)
candidate = uploads_bundle_dir / canonical_path
if not candidate.exists():
continue
row["path"] = canonical_path
retained_rows.append(row)
photo_rows[:] = retained_rows
existing_homepage_rows = [row for row in photo_rows if row.get("person_id") is None]
existing_person_ids = {str(row["person_id"]) for row in photo_rows if row.get("person_id") is not None}
existing_primary_person_ids = {
str(row["person_id"]) for row in photo_rows if row.get("person_id") is not None and bool(row.get("is_primary"))
}
has_homepage_primary = any(bool(row.get("is_primary")) for row in existing_homepage_rows)
now_iso = datetime.now(UTC).isoformat()
for row in legacy_portrait_rows:
portrait_path = row.get("portrait_path")
person_id = row.get("id")
if not isinstance(portrait_path, str) or not portrait_path.strip():
continue
if person_id is None:
continue
canonical = _canonical_media_relative_path(
portrait_path,
source_upload_dir=uploads_bundle_dir,
preferred_prefix="persons/",
)
source_file = uploads_bundle_dir / canonical
if not source_file.exists():
continue
person_key = str(person_id)
if person_key in existing_person_ids:
continue
suffix = Path(canonical).suffix.lower() or ".jpg"
photo_id = str(uuid4())
relative_path = f"photos/{photo_id}{suffix}"
target_file = uploads_bundle_dir / relative_path
target_file.parent.mkdir(parents=True, exist_ok=True)
shutil.copy2(source_file, target_file)
is_primary = person_key not in existing_primary_person_ids
photo_rows.append(
{
"id": photo_id,
"person_id": person_key,
"path": relative_path,
"description": None,
"is_primary": is_primary,
"created_at": now_iso,
"updated_at": now_iso,
}
)
existing_person_ids.add(person_key)
if is_primary:
existing_primary_person_ids.add(person_key)
legacy_homepage_dir = uploads_bundle_dir / "homepage"
if not legacy_homepage_dir.exists():
return
homepage_images = sorted(
[
path
for path in legacy_homepage_dir.iterdir()
if path.is_file()
and path.suffix.lower() in {".jpg", ".jpeg", ".png", ".gif", ".webp", ".bmp", ".tif", ".tiff"}
],
key=lambda path: (path.stat().st_mtime, path.name),
)
if existing_homepage_rows:
return
for index, image_path in enumerate(homepage_images):
photo_id = str(uuid4())
relative_path = f"photos/{photo_id}{image_path.suffix.lower()}"
target_file = uploads_bundle_dir / relative_path
target_file.parent.mkdir(parents=True, exist_ok=True)
shutil.copy2(image_path, target_file)
photo_rows.append(
{
"id": photo_id,
"person_id": None,
"path": relative_path,
"description": None,
"is_primary": (not has_homepage_primary) and index == 0,
"created_at": now_iso,
"updated_at": now_iso,
}
)
def _relocate_homepage_markdown(*, uploads_bundle_dir: Path) -> None:
legacy_markdown = uploads_bundle_dir / "homepage" / "homepage.md"
target_markdown = uploads_bundle_dir / "homepage.md"
if not legacy_markdown.exists() or target_markdown.exists():
return
target_markdown.parent.mkdir(parents=True, exist_ok=True)
shutil.copy2(legacy_markdown, target_markdown)
src/transcription/db/models.py
-769
View File
@@ -1,769 +0,0 @@
"""SQLModel domain models for the V3 transcription system."""
from datetime import UTC
from datetime import date
from datetime import datetime
from enum import StrEnum
from typing import Any
from typing import Optional
from uuid import UUID
from uuid import uuid4
from pydantic import JsonValue
from sqlalchemy import JSON
from sqlalchemy import BigInteger
from sqlalchemy import Column
from sqlalchemy import Enum as SAEnum
from sqlalchemy import ForeignKey
from sqlalchemy import Index
from sqlalchemy import LargeBinary
from sqlalchemy import UniqueConstraint
from sqlalchemy import Uuid
from sqlalchemy import inspect as sqlalchemy_inspect
from sqlalchemy.dialects.postgresql import JSONB
from sqlalchemy.exc import NoInspectionAvailable
from sqlalchemy.orm.state import InstanceState
from sqlalchemy.types import TypeDecorator
from sqlmodel import Field
from sqlmodel import Relationship
from sqlmodel import SQLModel
def _loaded_attribute(instance: object, attribute: str) -> Any | None:
"""Return ``attribute`` only when it is already loaded on ``instance``.
Relationships in this module declare ``lazy="raise"``, so reading an
unloaded attribute is an error rather than a silent query. Callers that
render optional detail use this to distinguish "not loaded" from "absent"
without catching exceptions indiscriminately.
"""
try:
state: InstanceState[Any] = sqlalchemy_inspect(instance, raiseerr=True)
except NoInspectionAvailable:
return None
if attribute in state.unloaded:
return None
return state.dict.get(attribute)
def _utc_now_naive() -> datetime:
"""Return a UTC timestamp stored as a naive datetime."""
return datetime.now(UTC).replace(tzinfo=None)
class JSONBCompat(TypeDecorator):
"""JSONB for PostgreSQL and JSON for SQLite/testing backends."""
impl = JSON(none_as_null=True)
def load_dialect_impl(self, dialect):
if dialect.name == "postgresql":
return dialect.type_descriptor(JSONB(none_as_null=True))
return dialect.type_descriptor(JSON(none_as_null=True))
class JobStatus(StrEnum):
QUEUED = "queued"
PROCESSING = "processing"
TRANSCRIBED = "transcribed"
PARTIAL_SUCCESS = "partial_success"
FAILED = "failed"
class JobSourceStatus(StrEnum):
PENDING = "pending"
TRANSCRIBED = "transcribed"
FAILED = "failed"
CANCELLED = "cancelled"
class JobPurpose(StrEnum):
TRANSCRIPTION = "transcription"
RETRANSCRIPTION = "retranscription"
class MaintenanceJobType(StrEnum):
BACKUP = "backup"
STORAGE_RECONCILIATION = "storage_reconciliation"
GEDCOM_IMPORT = "gedcom_import"
class MaintenanceRunStatus(StrEnum):
QUEUED = "queued"
PROCESSING = "processing"
SUCCEEDED = "succeeded"
FAILED = "failed"
class GenealogyCitationFactType(StrEnum):
BIRTH = "birth"
DEATH = "death"
MARRIAGE = "marriage"
OTHER = "other"
class GenealogyCitationSourceKind(StrEnum):
FAMILYSEARCH_IMPORTED = "familysearch_imported"
TRANSCRIPTION_EVIDENCE = "transcription_evidence"
class DocumentType(SQLModel, table=True):
"""Registry of allowed document types."""
__tablename__ = "document_type"
id: UUID = Field(default_factory=uuid4, primary_key=True)
semantic_key: str | None = Field(default=None, index=True, unique=True)
label: str
normalized_label: str = Field(index=True, unique=True)
is_active: bool = True
created_at: datetime = Field(default_factory=_utc_now_naive)
updated_at: datetime = Field(
default_factory=_utc_now_naive,
sa_column_kwargs={"onupdate": _utc_now_naive},
)
documents: list["Document"] = Relationship(
back_populates="document_type_ref", sa_relationship_kwargs={"lazy": "raise"}
)
class PersonRole(SQLModel, table=True):
"""Registry of allowed document-person relationship roles."""
__tablename__ = "person_role"
id: UUID = Field(default_factory=uuid4, primary_key=True)
semantic_key: str | None = Field(default=None, index=True, unique=True)
label: str
normalized_label: str = Field(index=True, unique=True)
is_active: bool = True
created_at: datetime = Field(default_factory=_utc_now_naive)
updated_at: datetime = Field(
default_factory=_utc_now_naive,
sa_column_kwargs={"onupdate": _utc_now_naive},
)
document_people: list["DocumentPerson"] = Relationship(
back_populates="role_ref", sa_relationship_kwargs={"lazy": "raise"}
)
class Tag(SQLModel, table=True):
"""Registry of labels that can be attached to Documents."""
__tablename__ = "tag"
id: UUID = Field(default_factory=uuid4, primary_key=True)
semantic_key: str | None = Field(default=None, index=True, unique=True)
label: str
normalized_label: str = Field(index=True, unique=True)
is_active: bool = True
created_at: datetime = Field(default_factory=_utc_now_naive)
updated_at: datetime = Field(
default_factory=_utc_now_naive,
sa_column_kwargs={"onupdate": _utc_now_naive},
)
document_tags: list["DocumentTag"] = Relationship(
back_populates="tag_ref",
sa_relationship_kwargs={"lazy": "raise"},
)
person_tags: list["PersonTag"] = Relationship(
back_populates="tag_ref",
sa_relationship_kwargs={"lazy": "raise"},
)
class Document(SQLModel, table=True):
"""An historical document."""
id: UUID = Field(default_factory=uuid4, primary_key=True)
name: str
document_type_id: UUID | None = Field(default=None, foreign_key="document_type.id", index=True)
document_date: date | None = None
document_date_raw: str | None = None
location_created: str | None = None
notes: str | None = None
archive_identifier: str | None = None
created_at: datetime = Field(default_factory=_utc_now_naive)
updated_at: datetime = Field(
default_factory=_utc_now_naive,
sa_column_kwargs={"onupdate": _utc_now_naive},
)
jobs: list["Job"] = Relationship(back_populates="document", sa_relationship_kwargs={"lazy": "raise"})
sources: list["Source"] = Relationship(back_populates="document", sa_relationship_kwargs={"lazy": "raise"})
document_people: list["DocumentPerson"] = Relationship(
back_populates="document", sa_relationship_kwargs={"lazy": "raise"}
)
document_tags: list["DocumentTag"] = Relationship(
back_populates="document",
sa_relationship_kwargs={"lazy": "raise"},
)
document_type_ref: Optional["DocumentType"] = Relationship(
back_populates="documents", sa_relationship_kwargs={"lazy": "raise"}
)
class Person(SQLModel, table=True):
"""A historical person linked to one or more documents."""
id: UUID = Field(default_factory=uuid4, primary_key=True)
last_name: str
given_names: str
birth_date: date | None = None
birth_date_raw: str | None = None
birth_place: str | None = None
death_date: date | None = None
death_date_raw: str | None = None
death_place: str | None = None
biography: str | None = None
family_search_id: str | None = Field(default=None, unique=True)
metadata_: dict[str, JsonValue] | None = Field(
default=None,
sa_column=Column("metadata", JSONBCompat(), nullable=True),
)
created_at: datetime = Field(default_factory=_utc_now_naive)
updated_at: datetime = Field(
default_factory=_utc_now_naive,
sa_column_kwargs={"onupdate": _utc_now_naive},
)
document_people: list["DocumentPerson"] = Relationship(
back_populates="person", sa_relationship_kwargs={"lazy": "raise"}
)
person_tags: list["PersonTag"] = Relationship(
back_populates="person",
sa_relationship_kwargs={"lazy": "raise"},
)
photos: list["Photo"] = Relationship(
back_populates="person",
sa_relationship_kwargs={"lazy": "raise"},
)
@property
def full_name(self) -> str:
"""Presentation-friendly combined name."""
return f"{self.given_names} {self.last_name}".strip()
class GenealogyPerson(SQLModel, table=True):
"""An individual imported from a GEDCOM export."""
__tablename__ = "genealogy_person"
id: UUID = Field(default_factory=uuid4, primary_key=True)
fs_id: str = Field(index=True, unique=True)
full_name: str
birth_date: date | None = None
birth_date_raw: str | None = None
birth_place: str | None = None
death_date: date | None = None
death_date_raw: str | None = None
death_place: str | None = None
created_at: datetime = Field(default_factory=_utc_now_naive)
updated_at: datetime = Field(
default_factory=_utc_now_naive,
sa_column_kwargs={"onupdate": _utc_now_naive},
)
husband_families: list["GenealogyFamily"] = Relationship(
back_populates="husband",
sa_relationship_kwargs={"lazy": "raise", "foreign_keys": "[GenealogyFamily.husband_id]"},
)
wife_families: list["GenealogyFamily"] = Relationship(
back_populates="wife",
sa_relationship_kwargs={"lazy": "raise", "foreign_keys": "[GenealogyFamily.wife_id]"},
)
child_family_memberships: list["GenealogyFamilyChild"] = Relationship(
back_populates="child",
sa_relationship_kwargs={"lazy": "raise"},
)
citations: list["GenealogyCitation"] = Relationship(
back_populates="genealogy_person",
sa_relationship_kwargs={"lazy": "raise"},
)
class GenealogyFamily(SQLModel, table=True):
"""A family linking two GenealogyPerson records."""
__tablename__ = "genealogy_family"
id: UUID = Field(default_factory=uuid4, primary_key=True)
fs_family_id: str = Field(index=True, unique=True)
husband_id: UUID | None = Field(default=None, foreign_key="genealogy_person.id", index=True)
wife_id: UUID | None = Field(default=None, foreign_key="genealogy_person.id", index=True)
marriage_date: date | None = None
marriage_date_raw: str | None = None
marriage_place: str | None = None
created_at: datetime = Field(default_factory=_utc_now_naive)
updated_at: datetime = Field(
default_factory=_utc_now_naive,
sa_column_kwargs={"onupdate": _utc_now_naive},
)
husband: Optional["GenealogyPerson"] = Relationship(
back_populates="husband_families",
sa_relationship_kwargs={"lazy": "raise", "foreign_keys": "[GenealogyFamily.husband_id]"},
)
wife: Optional["GenealogyPerson"] = Relationship(
back_populates="wife_families",
sa_relationship_kwargs={"lazy": "raise", "foreign_keys": "[GenealogyFamily.wife_id]"},
)
children: list["GenealogyFamilyChild"] = Relationship(
back_populates="family",
sa_relationship_kwargs={"lazy": "raise"},
)
citations: list["GenealogyCitation"] = Relationship(
back_populates="genealogy_family",
sa_relationship_kwargs={"lazy": "raise"},
)
class GenealogyFamilyChild(SQLModel, table=True):
"""Junction table for child membership within a genealogy family."""
__tablename__ = "genealogy_family_child"
id: UUID = Field(default_factory=uuid4, primary_key=True)
family_id: UUID = Field(foreign_key="genealogy_family.id", index=True)
child_id: UUID = Field(foreign_key="genealogy_person.id", index=True)
relationship_type: str | None = None
created_at: datetime = Field(default_factory=_utc_now_naive)
__table_args__ = (UniqueConstraint("family_id", "child_id", name="uq_genealogy_family_child"),)
family: Optional["GenealogyFamily"] = Relationship(
back_populates="children",
sa_relationship_kwargs={"lazy": "raise"},
)
child: Optional["GenealogyPerson"] = Relationship(
back_populates="child_family_memberships",
sa_relationship_kwargs={"lazy": "raise"},
)
class GenealogyCitation(SQLModel, table=True):
"""A source citation attached to a genealogical fact."""
__tablename__ = "genealogy_citation"
id: UUID = Field(default_factory=uuid4, primary_key=True)
genealogy_person_id: UUID | None = Field(default=None, foreign_key="genealogy_person.id", index=True)
genealogy_family_id: UUID | None = Field(default=None, foreign_key="genealogy_family.id", index=True)
fact_type: GenealogyCitationFactType = Field(
sa_column=Column(
SAEnum(
GenealogyCitationFactType,
values_callable=lambda enum_cls: [item.value for item in enum_cls],
native_enum=False,
),
nullable=False,
)
)
raw_citation_text: str
source_kind: GenealogyCitationSourceKind = Field(
sa_column=Column(
SAEnum(
GenealogyCitationSourceKind,
values_callable=lambda enum_cls: [item.value for item in enum_cls],
native_enum=False,
),
nullable=False,
)
)
document_id: UUID | None = Field(default=None, foreign_key="document.id", index=True)
created_at: datetime = Field(default_factory=_utc_now_naive)
genealogy_person: Optional["GenealogyPerson"] = Relationship(
back_populates="citations",
sa_relationship_kwargs={"lazy": "raise"},
)
genealogy_family: Optional["GenealogyFamily"] = Relationship(
back_populates="citations",
sa_relationship_kwargs={"lazy": "raise"},
)
document: Optional["Document"] = Relationship(sa_relationship_kwargs={"lazy": "raise"})
class Photo(SQLModel, table=True):
"""A reusable image record for Person and homepage galleries."""
__tablename__ = "photo"
id: UUID = Field(default_factory=uuid4, primary_key=True)
person_id: UUID | None = Field(default=None, foreign_key="person.id", index=True)
path: str
description: str | None = None
is_primary: bool = False
created_at: datetime = Field(default_factory=_utc_now_naive)
updated_at: datetime = Field(
default_factory=_utc_now_naive,
sa_column_kwargs={"onupdate": _utc_now_naive},
)
person: Optional["Person"] = Relationship(
back_populates="photos",
sa_relationship_kwargs={"lazy": "raise"},
)
class DocumentPerson(SQLModel, table=True):
"""Associates documents with people in a given role."""
__tablename__ = "document_person"
id: UUID = Field(default_factory=uuid4, primary_key=True)
document_id: UUID = Field(foreign_key="document.id", index=True)
person_id: UUID = Field(foreign_key="person.id", index=True)
role_id: UUID = Field(foreign_key="person_role.id", index=True)
created_at: datetime = Field(default_factory=_utc_now_naive)
updated_at: datetime = Field(
default_factory=_utc_now_naive,
sa_column_kwargs={"onupdate": _utc_now_naive},
)
__table_args__ = (UniqueConstraint("document_id", "person_id", name="uq_document_person"),)
document: Optional["Document"] = Relationship(
back_populates="document_people", sa_relationship_kwargs={"lazy": "raise"}
)
person: Optional["Person"] = Relationship(
back_populates="document_people", sa_relationship_kwargs={"lazy": "raise"}
)
role_ref: Optional["PersonRole"] = Relationship(
back_populates="document_people", sa_relationship_kwargs={"lazy": "raise"}
)
class DocumentTag(SQLModel, table=True):
"""Associates Documents with Tags."""
__tablename__ = "document_tag"
id: UUID = Field(default_factory=uuid4, primary_key=True)
document_id: UUID = Field(foreign_key="document.id", index=True)
tag_id: UUID = Field(foreign_key="tag.id", index=True)
created_at: datetime = Field(default_factory=_utc_now_naive)
updated_at: datetime = Field(
default_factory=_utc_now_naive,
sa_column_kwargs={"onupdate": _utc_now_naive},
)
__table_args__ = (UniqueConstraint("document_id", "tag_id", name="uq_document_tag"),)
document: Optional["Document"] = Relationship(
back_populates="document_tags",
sa_relationship_kwargs={"lazy": "raise"},
)
tag_ref: Optional["Tag"] = Relationship(
back_populates="document_tags",
sa_relationship_kwargs={"lazy": "raise"},
)
class PersonTag(SQLModel, table=True):
"""Associates People with Tags."""
__tablename__ = "person_tag"
id: UUID = Field(default_factory=uuid4, primary_key=True)
person_id: UUID = Field(foreign_key="person.id", index=True)
tag_id: UUID = Field(foreign_key="tag.id", index=True)
created_at: datetime = Field(default_factory=_utc_now_naive)
updated_at: datetime = Field(
default_factory=_utc_now_naive,
sa_column_kwargs={"onupdate": _utc_now_naive},
)
__table_args__ = (UniqueConstraint("person_id", "tag_id", name="uq_person_tag"),)
person: Optional["Person"] = Relationship(
back_populates="person_tags",
sa_relationship_kwargs={"lazy": "raise"},
)
tag_ref: Optional["Tag"] = Relationship(
back_populates="person_tags",
sa_relationship_kwargs={"lazy": "raise"},
)
class Job(SQLModel, table=True):
"""A transcription job tied to a single document."""
__table_args__ = (Index("ix_job_status_date_created", "status", "date_created"),)
id: UUID = Field(default_factory=uuid4, primary_key=True)
document_id: UUID = Field(foreign_key="document.id", index=True)
status: JobStatus = Field(
default=JobStatus.QUEUED,
sa_column=Column(
SAEnum(
JobStatus,
values_callable=lambda enum_cls: [item.value for item in enum_cls],
native_enum=False,
),
nullable=False,
),
)
retry_count: int = Field(default=0, ge=0)
purpose: JobPurpose = Field(
default=JobPurpose.TRANSCRIPTION,
sa_column=Column(
SAEnum(
JobPurpose,
values_callable=lambda enum_cls: [item.value for item in enum_cls],
native_enum=False,
),
nullable=False,
default=JobPurpose.TRANSCRIPTION.value,
),
)
date_created: datetime = Field(default_factory=_utc_now_naive)
date_updated: datetime = Field(
default_factory=_utc_now_naive,
sa_column_kwargs={"onupdate": _utc_now_naive},
)
provider: str | None = None
model: str | None = None
prompt_name: str | None = None
prompt_hash: str | None = None
system_prompt: str | None = None
user_prompt: str | None = None
temperature: float | None = None
top_p: float | None = None
document: Optional["Document"] = Relationship(back_populates="jobs", sa_relationship_kwargs={"lazy": "raise"})
job_sources: list["JobSource"] = Relationship(back_populates="job", sa_relationship_kwargs={"lazy": "raise"})
@property
def filename(self) -> str:
"""Return the filename of the first loaded source, when available.
Relationships on this model use ``lazy="raise"``, so this deliberately
inspects load state rather than triggering (or swallowing) a lazy load:
a read model that did not eager-load its sources gets "unknown" instead
of an unhandled error, and genuine errors are no longer hidden.
"""
for job_source in _loaded_attribute(self, "job_sources") or ():
source = _loaded_attribute(job_source, "source")
if source is not None:
return source.filename
return "unknown"
class MaintenanceRun(SQLModel, table=True):
"""A queued/processed maintenance task execution record."""
__tablename__ = "maintenance_run"
id: UUID = Field(default_factory=uuid4, primary_key=True)
job_type: MaintenanceJobType = Field(
sa_column=Column(
SAEnum(
MaintenanceJobType,
values_callable=lambda enum_cls: [item.value for item in enum_cls],
native_enum=False,
),
nullable=False,
)
)
status: MaintenanceRunStatus = Field(
default=MaintenanceRunStatus.QUEUED,
sa_column=Column(
SAEnum(
MaintenanceRunStatus,
values_callable=lambda enum_cls: [item.value for item in enum_cls],
native_enum=False,
),
nullable=False,
default=MaintenanceRunStatus.QUEUED.value,
),
)
started_at: datetime | None = None
finished_at: datetime | None = None
triggered_by: str | None = None
summary: str | None = None
log_path: str | None = None
error_detail: str | None = None
created_at: datetime = Field(default_factory=_utc_now_naive)
updated_at: datetime = Field(
default_factory=_utc_now_naive,
sa_column_kwargs={"onupdate": _utc_now_naive},
)
class Source(SQLModel, table=True):
"""A document source image or PDF page."""
id: UUID = Field(default_factory=uuid4, primary_key=True)
document_id: UUID = Field(foreign_key="document.id", index=True)
page_number: int = Field(default=1, ge=1)
upload_name: str
filename: str
file_path: str
file_hash: str
file_size_bytes: int = Field(sa_column=Column(BigInteger(), nullable=False))
raw_transcription: str | None = None
preferred_execution_attempt_id: UUID | None = Field(
default=None,
sa_column=Column(
Uuid(),
# use_alter breaks the source / job_source / execution_attempt cycle so
# metadata.create_all can order table creation on every dialect.
ForeignKey(
"execution_attempt.id",
use_alter=True,
name="fk_source_preferred_execution_attempt_id",
),
nullable=True,
index=True,
),
)
revised_text: str | None = None
date_uploaded: datetime = Field(default_factory=_utc_now_naive)
date_revised: datetime | None = None
document: Optional["Document"] = Relationship(
back_populates="sources",
sa_relationship_kwargs={"lazy": "raise"},
)
job_sources: list["JobSource"] = Relationship(
back_populates="source",
sa_relationship_kwargs={"lazy": "raise"},
)
@property
def latest_job_source(self) -> Optional["JobSource"]:
"""Return the most recent job execution record for this source.
``JobSource`` carries no timestamp of its own, so recency is the parent
job's creation time. ``(job_id, source_id)`` is unique per source, so
this is exactly "the most recent job that included this page".
"""
job_sources = _loaded_attribute(self, "job_sources") or ()
dated = [
(job, job_source) for job_source in job_sources if (job := _loaded_attribute(job_source, "job")) is not None
]
if dated:
return max(dated, key=lambda pair: pair[0].date_created)[1]
return job_sources[0] if job_sources else None
@property
def latest_status(self) -> JobSourceStatus | None:
"""Return the execution status of the latest job run."""
latest = self.latest_job_source
return latest.status if latest else None
@property
def latest_error_detail(self) -> str | None:
"""Return the error detail of the latest attempt on the latest job run.
Failure detail lives on ``ExecutionAttempt``; ``JobSource`` records only
which page a job is working on and how far it got.
"""
latest = self.latest_job_source
if latest is None:
return None
attempts = _loaded_attribute(latest, "execution_attempts") or ()
if not attempts:
return None
latest_attempt = max(attempts, key=lambda item: item.attempt_number)
return latest_attempt.error_detail
@property
def document_name(self) -> str | None:
"""Return the parent document name if loaded."""
return self.document.name if self.document else None
class JobSource(SQLModel, table=True):
"""A single AI execution record for one source page."""
__tablename__ = "job_source"
__table_args__ = (UniqueConstraint("job_id", "source_id", name="uq_job_source_job_source"),)
id: UUID = Field(default_factory=uuid4, primary_key=True)
job_id: UUID = Field(foreign_key="job.id", index=True)
source_id: UUID = Field(foreign_key="source.id", index=True)
status: JobSourceStatus = Field(
default=JobSourceStatus.PENDING,
sa_column=Column(
SAEnum(
JobSourceStatus,
values_callable=lambda enum_cls: [item.value for item in enum_cls],
native_enum=False,
),
nullable=False,
),
)
job: Optional["Job"] = Relationship(back_populates="job_sources", sa_relationship_kwargs={"lazy": "raise"})
source: Optional["Source"] = Relationship(back_populates="job_sources", sa_relationship_kwargs={"lazy": "raise"})
execution_attempts: list["ExecutionAttempt"] = Relationship(
back_populates="job_source",
sa_relationship_kwargs={"lazy": "noload", "order_by": "ExecutionAttempt.attempt_number"},
)
class ExecutionAttempt(SQLModel, table=True):
"""Immutable evidence for one provider call attempt."""
__tablename__ = "execution_attempt"
__table_args__ = (UniqueConstraint("job_id", "source_id", "attempt_number", name="uq_execution_attempt_number"),)
id: UUID = Field(default_factory=uuid4, primary_key=True)
job_source_id: UUID = Field(foreign_key="job_source.id", index=True)
job_id: UUID = Field(foreign_key="job.id", index=True)
source_id: UUID = Field(foreign_key="source.id", index=True)
attempt_number: int = Field(ge=1)
status: JobSourceStatus = Field(
sa_column=Column(
# Declared identically to job_source.status. Without values_callable
# SQLAlchemy persists enum *names*, which is defect [45]: the two
# columns spelled the same status differently and never compared equal.
SAEnum(
JobSourceStatus,
values_callable=lambda enum_cls: [item.value for item in enum_cls],
native_enum=False,
),
nullable=False,
)
)
provider: str
model: str | None = None
request_manifest: dict[str, JsonValue] | None = Field(default=None, sa_column=Column(JSONBCompat(), nullable=True))
request_manifest_sha256: str | None = None
request_manifest_schema_version: str | None = None
response_received: bool = False
transport_status_code: int | None = None
transport_body: bytes | None = Field(default=None, sa_column=Column(LargeBinary(), nullable=True))
transport_content_type: str | None = None
transport_content_encoding: str | None = None
transport_safe_headers: dict[str, JsonValue] | None = Field(
default=None, sa_column=Column(JSONBCompat(), nullable=True)
)
router_request_id: str | None = None
router_generation_id: str | None = None
sdk_response_snapshot: dict[str, JsonValue] | None = Field(
default=None, sa_column=Column(JSONBCompat(), nullable=True)
)
normalized_metadata: dict[str, JsonValue] | None = Field(
default=None, sa_column=Column(JSONBCompat(), nullable=True)
)
software_context: dict[str, JsonValue] | None = Field(default=None, sa_column=Column(JSONBCompat(), nullable=True))
raw_transcription: str | None = None
error_category: str | None = None
error_detail: str | None = None
failure_phase: str | None = None
started_at: datetime
finished_at: datetime
duration_ms: int = Field(ge=0)
created_at: datetime = Field(default_factory=_utc_now_naive)
job_source: Optional["JobSource"] = Relationship(
back_populates="execution_attempts", sa_relationship_kwargs={"lazy": "raise"}
)
src/transcription/db/operations.py
-248
View File
@@ -1,248 +0,0 @@
from __future__ import annotations
import logging
from pathlib import Path
from sqlalchemy import inspect as sqlalchemy_inspect
from sqlalchemy import text
from sqlalchemy.ext.asyncio import AsyncEngine
from sqlalchemy.ext.asyncio import async_sessionmaker
from sqlmodel import SQLModel
from sqlmodel import select
from sqlmodel.ext.asyncio.session import AsyncSession
from .engine import resolve_engine
from .models import DocumentType
from .models import PersonRole
from .registries import BUILT_IN_DOCUMENT_TYPES
from .registries import BUILT_IN_PERSON_ROLES
logger = logging.getLogger(__name__)
async def create_all(*, engine: AsyncEngine | None = None) -> None:
"""Create any missing tables on the selected engine."""
# Import models so SQLModel metadata is fully registered before bootstrap.
from transcription.db import models as _models # noqa: F401
active_engine = engine or resolve_engine()
async with active_engine.begin() as connection:
await connection.run_sync(SQLModel.metadata.create_all)
await seed_registry_defaults(engine=active_engine)
logger.debug("Database schema bootstrap complete for database_url=%s", active_engine.url)
async def reconcile_legacy_job_source_columns(*, engine: AsyncEngine | None = None) -> int:
"""Remove stale V4.6 ``job_source`` evidence columns from existing databases.
Runtime models define ``job_source`` as a queue/projection table only. If an
older database still carries the retired evidence columns, writes can fail
on stale constraints (for example ``executed_at NOT NULL``).
"""
active_engine = engine or resolve_engine()
if not hasattr(active_engine, "begin"):
return 0
def _reconcile(sync_connection) -> int:
inspector = sqlalchemy_inspect(sync_connection)
table_names = set(inspector.get_table_names())
if "job_source" not in table_names:
return 0
present_columns = {column["name"] for column in inspector.get_columns("job_source")}
dropped = 0
for column_name in (
"raw_transcription",
"ai_metadata",
"raw_api_response",
"error_detail",
"executed_at",
):
if column_name not in present_columns:
continue
sync_connection.execute(text(f'alter table "job_source" drop column "{column_name}"'))
dropped += 1
return dropped
async with active_engine.begin() as connection:
dropped_columns = await connection.run_sync(_reconcile)
if dropped_columns:
logger.warning("Dropped %s legacy job_source column(s) during startup reconciliation", dropped_columns)
return dropped_columns
async def reconcile_canonical_media_paths(*, engine: AsyncEngine | None = None) -> int:
"""Normalize stored media paths to upload-root-relative POSIX form."""
active_engine = engine or resolve_engine()
if not hasattr(active_engine, "begin"):
return 0
def _reconcile(sync_connection) -> int:
rows_changed = 0
inspector = sqlalchemy_inspect(sync_connection)
table_names = set(inspector.get_table_names())
if "source" in table_names:
rows = (
sync_connection.execute(text('select id, file_path from "source" where file_path is not null'))
.mappings()
.all()
)
for row in rows:
original = str(row["file_path"])
normalized = _canonical_relative_path(original, preferred_prefix="documents/")
if normalized is None or normalized == original:
continue
sync_connection.execute(
text('update "source" set file_path = :file_path where id = :id'),
{"id": row["id"], "file_path": normalized},
)
rows_changed += 1
if "photo" in table_names:
rows = sync_connection.execute(text('select id, path from "photo" where path is not null')).mappings().all()
for row in rows:
original = str(row["path"])
normalized = _canonical_relative_path(original, preferred_prefix="photos/")
if normalized is None or normalized == original:
continue
sync_connection.execute(
text('update "photo" set path = :path where id = :id'),
{"id": row["id"], "path": normalized},
)
rows_changed += 1
return rows_changed
async with active_engine.begin() as connection:
rows_changed = await connection.run_sync(_reconcile)
if rows_changed:
logger.warning("Normalized %s media-path row(s) to canonical relative format", rows_changed)
return rows_changed
async def reconcile_person_name_columns(*, engine: AsyncEngine | None = None) -> int:
"""Backfill V5.1 Person name columns on existing databases."""
active_engine = engine or resolve_engine()
if not hasattr(active_engine, "begin"):
return 0
def _reconcile(sync_connection) -> int:
rows_changed = 0
inspector = sqlalchemy_inspect(sync_connection)
table_names = set(inspector.get_table_names())
if "person" not in table_names:
return 0
present_columns = {column["name"] for column in inspector.get_columns("person")}
if "last_name" not in present_columns:
sync_connection.execute(text('alter table "person" add column "last_name" varchar'))
if "given_names" not in present_columns:
sync_connection.execute(text('alter table "person" add column "given_names" varchar'))
query = (
text('select id, full_name, given_names, last_name from "person"')
if "full_name" in present_columns
else text('select id, null as full_name, given_names, last_name from "person"')
)
rows = sync_connection.execute(query).mappings().all()
for row in rows:
given_names = (str(row.get("given_names") or "")).strip()
last_name = (str(row.get("last_name") or "")).strip()
if given_names and last_name:
continue
tokens = [token for token in str(row.get("full_name") or "").split() if token]
if len(tokens) >= 2:
given_names, last_name = (" ".join(tokens[:-1]), tokens[-1])
elif len(tokens) == 1:
given_names = tokens[0]
last_name = tokens[0]
else:
given_names = "Unknown"
last_name = "Unknown"
sync_connection.execute(
text('update "person" set given_names = :given_names, last_name = :last_name where id = :id'),
{
"id": row["id"],
"given_names": given_names,
"last_name": last_name,
},
)
rows_changed += 1
return rows_changed
async with active_engine.begin() as connection:
rows_changed = await connection.run_sync(_reconcile)
if rows_changed:
logger.warning("Backfilled V5.1 name columns for %s person row(s)", rows_changed)
return rows_changed
def _canonical_relative_path(value: str, *, preferred_prefix: str) -> str | None:
normalized = value.strip().replace("\\", "/")
if not normalized:
return None
lowered = normalized.casefold()
if lowered.startswith(("http://", "https://", "data:")):
return None
if lowered.startswith("/uploads/"):
normalized = normalized[len("/uploads/") :]
lowered = normalized.casefold()
elif lowered.startswith("uploads/"):
normalized = normalized[len("uploads/") :]
lowered = normalized.casefold()
elif lowered.startswith("data/"):
normalized = normalized[len("data/") :]
lowered = normalized.casefold()
for prefix in ("documents/", "photos/", "persons/", "portraits/"):
marker = f"/{prefix}"
index = lowered.find(marker)
if index >= 0:
normalized = normalized[index + 1 :]
lowered = normalized.casefold()
break
if lowered.startswith(prefix):
break
if preferred_prefix == "persons/" and lowered.startswith("portraits/"):
normalized = "persons/" + normalized[len("portraits/") :]
lowered = normalized.casefold()
if not lowered.startswith(preferred_prefix):
return None
# Collapse any accidental "." segments while preserving relative semantics.
collapsed = Path(normalized).as_posix()
if collapsed.startswith("../") or collapsed == "..":
return None
return collapsed
async def seed_registry_defaults(*, engine: AsyncEngine | None = None) -> None:
"""Seed default registry rows for role and document type taxonomies."""
active_engine = engine or resolve_engine()
session_factory = async_sessionmaker(active_engine, class_=AsyncSession, expire_on_commit=False)
async with session_factory() as session:
role_keys = set((await session.exec(select(PersonRole.semantic_key))).all())
for semantic_key, label in BUILT_IN_PERSON_ROLES:
if semantic_key not in role_keys:
session.add(
PersonRole(
semantic_key=semantic_key,
label=label,
normalized_label=label.casefold(),
)
)
type_keys = set((await session.exec(select(DocumentType.semantic_key))).all())
for semantic_key, label in BUILT_IN_DOCUMENT_TYPES:
if semantic_key not in type_keys:
session.add(
DocumentType(
semantic_key=semantic_key,
label=label,
normalized_label=label.casefold(),
)
)
await session.commit()
src/transcription/db/registries.py
-20
View File
@@ -1,20 +0,0 @@
"""Application-defined semantic registry entries."""
from __future__ import annotations
BUILT_IN_DOCUMENT_TYPES: tuple[tuple[str, str], ...] = (
("book", "Book"),
("letter", "Letter"),
("postcard", "Postcard"),
("photo", "Photo"),
("journal", "Journal"),
("form", "Form"),
)
BUILT_IN_PERSON_ROLES: tuple[tuple[str, str], ...] = (
("author", "Author"),
("recipient", "Recipient"),
("mentioned", "Mentioned"),
)
AUTHOR_ROLE_SEMANTIC_KEY = "author"
src/transcription/db/runtime.py
-62
View File
@@ -1,62 +0,0 @@
import logging
from dataclasses import dataclass
from sqlalchemy.ext.asyncio import AsyncEngine
from sqlalchemy.ext.asyncio import async_sessionmaker
from sqlmodel.ext.asyncio.session import AsyncSession
from ..config import Settings
from ..config import get_settings
from .engine import get_database_url
from .engine import get_engine
from .session import get_session_factory
logger = logging.getLogger(__name__)
@dataclass(frozen=True)
class DatabaseRuntime:
"""Database runtime resources owned by app lifespan."""
engine: AsyncEngine
session_factory: async_sessionmaker[AsyncSession]
_runtime: DatabaseRuntime | None = None
def get_database_runtime() -> DatabaseRuntime | None:
"""Return the process-owned database runtime."""
return _runtime
async def dispose_database_runtime() -> None:
"""Dispose lifespan-owned async database resources."""
global _runtime
runtime = _runtime
if runtime is None:
return
await runtime.engine.dispose()
_runtime = None
def initialize_database_runtime(*, settings: Settings | None = None) -> DatabaseRuntime:
"""Initialize lifespan-owned async DB resources once per process."""
global _runtime
active_settings = settings or get_settings()
database_url = get_database_url(active_settings)
runtime = _runtime
if runtime is not None:
runtime_url = runtime.engine.url.render_as_string(hide_password=False)
if runtime_url != database_url:
raise RuntimeError(
f"Database runtime is already initialized for a different database: {runtime_url!r} != {database_url!r}"
)
return runtime
engine = get_engine(database_url)
session_factory = get_session_factory(database_url)
runtime = DatabaseRuntime(engine=engine, session_factory=session_factory)
_runtime = runtime
logger.debug("Initialized async database runtime for database_url=%s", engine.url)
return runtime
src/transcription/db/session.py
-104
View File
@@ -1,104 +0,0 @@
from collections.abc import AsyncGenerator
from contextlib import asynccontextmanager
from typing import Annotated
from fastapi import Depends
from sqlalchemy.ext.asyncio import async_sessionmaker
from sqlmodel.ext.asyncio.session import AsyncSession
from ..config import Settings
from ..config import get_settings
from .engine import dispose_engine
from .engine import get_database_url
from .engine import get_engine
type SessionFactory = async_sessionmaker[AsyncSession]
_SESSION_FACTORIES: dict[str, SessionFactory] = {}
def get_session_factory(database_url: str) -> SessionFactory:
"""Return the process-wide session factory for ``database_url``."""
factory = _SESSION_FACTORIES.get(database_url)
if factory is None:
factory = async_sessionmaker(
bind=get_engine(database_url),
class_=AsyncSession,
expire_on_commit=False,
)
_SESSION_FACTORIES[database_url] = factory
return factory
def resolve_session_factory(
database_url: str | None = None,
*,
settings: Settings | None = None,
) -> SessionFactory:
if database_url is not None:
return get_session_factory(database_url)
if settings is None:
from .runtime import get_database_runtime
runtime = get_database_runtime()
if runtime is not None:
return runtime.session_factory
return get_session_factory(get_database_url(settings or get_settings()))
type SessionFactoryDep = Annotated[SessionFactory, Depends(resolve_session_factory)]
async def dispose_session_factory(database_url: str) -> None:
"""Drop the session factory and engine for ``database_url`` only."""
_SESSION_FACTORIES.pop(database_url, None)
await dispose_engine(database_url)
@asynccontextmanager
async def session_scope(
*,
settings: Settings | None = None,
database_url: str | None = None,
session_factory: SessionFactory | None = None,
session: AsyncSession | None = None,
) -> AsyncGenerator[AsyncSession]:
if session is not None:
yield session
return
active_session_factory = session_factory or resolve_session_factory(
database_url,
settings=settings,
)
async with active_session_factory() as owned_session:
yield owned_session
type SessionScopeDep = Annotated[AsyncSession, Depends(session_scope)]
@asynccontextmanager
async def transaction_scope(
*,
settings: Settings | None = None,
database_url: str | None = None,
session_factory: SessionFactory | None = None,
session: AsyncSession | None = None,
) -> AsyncGenerator[AsyncSession]:
if session is not None:
if not session.in_transaction():
raise RuntimeError("A supplied session must have an active transaction")
yield session
return
active_session_factory = session_factory or resolve_session_factory(
database_url,
settings=settings,
)
async with active_session_factory.begin() as owned_session:
yield owned_session
type TransactionScopeDep = Annotated[AsyncSession, Depends(transaction_scope)]
src/transcription/errors.py
-144
View File
@@ -1,144 +0,0 @@
"""Shared error taxonomy and helpers for runtime boundaries."""
from __future__ import annotations
import logging
from dataclasses import dataclass
from datetime import UTC
from datetime import datetime
from enum import StrEnum
from uuid import uuid4
logger = logging.getLogger(__name__)
class ErrorCategory(StrEnum):
"""Stable error categories defined by docs/error_handling.md."""
VALIDATION = "validation_error"
USER_INPUT = "user_input_error"
NOT_FOUND = "not_found_error"
CONFLICT = "conflict_error"
EXTERNAL_PROVIDER = "external_provider_error"
EXTERNAL_TIMEOUT = "external_timeout_error"
PROCESSING = "processing_error"
INFRA_TRANSIENT = "infrastructure_transient_error"
INFRA_PERSISTENT = "infrastructure_persistent_error"
INTERNAL_UNEXPECTED = "internal_unexpected_error"
def new_error_id() -> str:
"""Return a short, user-shareable error reference id."""
return uuid4().hex[:8]
def exception_detail(exc: BaseException) -> str:
"""Return internal-only root-cause text for persisted diagnostics."""
return f"{type(exc).__name__}: {exc}"
class AppError(RuntimeError):
"""Base application error carrying user-safe handling metadata."""
def __init__(
self,
message: str,
*,
category: ErrorCategory = ErrorCategory.INTERNAL_UNEXPECTED,
suggestion: str = "Retry once. If it persists, review logs and report the error reference id.",
retriable: bool = False,
error_id: str | None = None,
detail: str | None = None,
) -> None:
super().__init__(message)
self.message = message
self.category = category
self.suggestion = suggestion
self.retriable = retriable
self.error_id = error_id or new_error_id()
# Internal-only diagnostic text. Persisted to evidence and logs, never rendered
# to users or serialized into API envelopes, because it may embed local
# filesystem paths and other infrastructure detail.
self.detail = detail
@dataclass(frozen=True)
class ErrorEnvelope:
"""Serializable API/UI error payload."""
error_id: str
category: str
message: str
suggestion: str
timestamp: str
def canonical_error_category(error: AppError) -> str:
"""Map internal categories to canonical API/UI envelope categories."""
mapping: dict[ErrorCategory, str] = {
ErrorCategory.VALIDATION: "validation",
ErrorCategory.USER_INPUT: "validation",
ErrorCategory.NOT_FOUND: "not_found",
ErrorCategory.CONFLICT: "conflict",
ErrorCategory.EXTERNAL_PROVIDER: "external",
ErrorCategory.EXTERNAL_TIMEOUT: "timeout",
ErrorCategory.INFRA_TRANSIENT: "timeout",
ErrorCategory.PROCESSING: "internal",
ErrorCategory.INFRA_PERSISTENT: "internal",
ErrorCategory.INTERNAL_UNEXPECTED: "internal",
}
return mapping.get(error.category, "internal")
def build_error_envelope(error: AppError) -> ErrorEnvelope:
"""Build an API-safe response envelope from an AppError."""
return ErrorEnvelope(
error_id=error.error_id,
category=canonical_error_category(error),
message=error.message,
suggestion=error.suggestion,
timestamp=datetime.now(UTC).isoformat(),
)
def classify_unexpected_error(exc: Exception, *, operation: str) -> AppError:
"""Normalize unknown exceptions into internal_unexpected_error.
The exception text is deliberately excluded from ``message``. ``AppError.message``
is rendered directly to users by the UI error presenter and is serialized into API
responses by :func:`build_error_envelope`, and unexpected exceptions routinely embed
local filesystem paths (SQLAlchemy ``OperationalError`` carries the database path,
``OSError`` carries the storage root). Leaking those is forbidden by
``.github/instructions/error-handling.instructions.md``.
The detail is preserved on ``AppError.detail`` and logged against ``error_id``. That
keeps the root cause in evidence records and operator logs, which are internal, while
keeping it out of user-facing and API-facing text.
"""
error = AppError(
f"Unexpected error during {operation}.",
category=ErrorCategory.INTERNAL_UNEXPECTED,
suggestion="Retry once. If it persists, review logs and report the error reference id.",
retriable=False,
detail=exception_detail(exc),
)
logger.error(
"Unexpected error operation=%s error_id=%s",
operation,
error.error_id,
exc_info=exc,
)
return error
def format_error_detail(error: AppError) -> str:
"""Return a compact persisted failure string for transcript.error_detail.
This is internal provenance, not user-facing output, so it carries
``AppError.detail`` (the root cause) in addition to the user-safe message.
"""
parts = [f"[{error.category.value}] {error.message}"]
if error.detail:
parts.append(f"detail={error.detail}")
parts.extend((f"suggestion={error.suggestion}", f"error_id={error.error_id}"))
return " | ".join(parts)
src/transcription/providers/__init__.py
-41
View File
@@ -1,41 +0,0 @@
"""Provider exports and factory for transcription adapters."""
from transcription.config import Provider
from transcription.config import Settings
from transcription.config import get_settings
from transcription.providers.base import ProviderAuthError
from transcription.providers.base import ProviderCallEvidence
from transcription.providers.base import ProviderError
from transcription.providers.base import ProviderResponseError
from transcription.providers.base import TranscriptionMetadata
from transcription.providers.base import TranscriptionProvider
from transcription.providers.base import TranscriptionResult
from transcription.providers.evidence import RequestManifest
from transcription.providers.evidence import SourceEvidenceReference
from transcription.providers.evidence import TransportEvidence
from transcription.providers.openrouter import OpenRouterTranscriptionProvider
def get_transcription_provider(*, settings: Settings | None = None) -> TranscriptionProvider:
"""Return the configured transcription provider adapter."""
runtime_settings = settings or get_settings()
if runtime_settings.provider == Provider.OPENROUTER:
return OpenRouterTranscriptionProvider(settings=runtime_settings)
raise ProviderError(f"Unsupported transcription provider: {runtime_settings.provider}")
__all__ = [
"OpenRouterTranscriptionProvider",
"ProviderAuthError",
"ProviderCallEvidence",
"ProviderError",
"ProviderResponseError",
"RequestManifest",
"SourceEvidenceReference",
"TranscriptionMetadata",
"TranscriptionProvider",
"TranscriptionResult",
"TransportEvidence",
"get_transcription_provider",
]
src/transcription/providers/base.py
-136
View File
@@ -1,136 +0,0 @@
"""Provider interfaces and validated shared contracts for transcription adapters."""
from dataclasses import dataclass
from typing import Protocol
from pydantic import BaseModel
from pydantic import ConfigDict
from pydantic import Field
from pydantic import JsonValue
from transcription.providers.evidence import RequestManifest
from transcription.providers.evidence import SourceEvidenceReference
from transcription.providers.evidence import TransportEvidence
class ProviderError(RuntimeError):
"""Base error for provider failures."""
def __init__(
self,
message: str,
*,
request_manifest: RequestManifest | None = None,
transport_evidence: TransportEvidence | None = None,
failure_phase: str = "provider_request",
) -> None:
super().__init__(message)
self.request_manifest = request_manifest
self.transport_evidence = transport_evidence
self.failure_phase = failure_phase
class ProviderAuthError(ProviderError):
"""Raised when provider authentication fails."""
class ProviderResponseError(ProviderError):
"""Raised when provider responses are malformed or unusable."""
@dataclass(slots=True)
class ProviderCallEvidence:
"""Caller-owned evidence sink for one provider invocation."""
request_manifest: RequestManifest | None = None
transport_evidence: TransportEvidence | None = None
class ProviderUsage(BaseModel):
"""Normalized provider token accounting."""
model_config = ConfigDict(extra="forbid", frozen=True)
input_tokens: int | None = Field(default=None, ge=0)
output_tokens: int | None = Field(default=None, ge=0)
total_tokens: int | None = Field(default=None, ge=0)
class TranscriptionMetadata(BaseModel):
"""Stable structured metadata persisted for one provider execution."""
model_config = ConfigDict(extra="forbid", frozen=True)
finish_reason: str | None = Field(default=None, min_length=1)
usage: ProviderUsage | None = None
def as_json_object(self) -> dict[str, JsonValue] | None:
payload = self.model_dump(mode="json", exclude_none=True)
return payload or None
class TranscriptionResult(BaseModel):
"""Normalized output returned by any transcription provider."""
model_config = ConfigDict(extra="forbid", frozen=True, str_strip_whitespace=True)
text: str = Field(min_length=1)
provider: str = Field(min_length=1)
model: str = Field(min_length=1)
prompt_name: str | None = None
prompt_hash: str | None = Field(default=None, pattern=r"^[0-9a-f]{64}$")
system_prompt: str | None = None
user_prompt: str | None = None
temperature: float | None = Field(default=None, ge=0.0, le=2.0)
top_p: float | None = Field(default=None, ge=0.0, le=1.0)
metadata: TranscriptionMetadata = Field(default_factory=TranscriptionMetadata)
raw_api_response: dict[str, JsonValue] | None = None
request_manifest: RequestManifest | None = None
transport_evidence: TransportEvidence | None = None
@property
def finish_reason(self) -> str | None:
return self.metadata.finish_reason
@property
def usage_input_tokens(self) -> int | None:
return self.metadata.usage.input_tokens if self.metadata.usage else None
@property
def usage_output_tokens(self) -> int | None:
return self.metadata.usage.output_tokens if self.metadata.usage else None
@property
def usage_total_tokens(self) -> int | None:
return self.metadata.usage.total_tokens if self.metadata.usage else None
def metadata_payload(self) -> dict[str, JsonValue] | None:
return self.metadata.as_json_object()
class TranscriptionProvider(Protocol):
"""Contract every transcription provider adapter must satisfy."""
@property
def model(self) -> str:
"""Return the resolved model slug this adapter will call."""
...
async def transcribe(
self,
*,
prompt_text: str,
image_bytes: bytes,
mime_type: str,
temperature: float | None = None,
top_p: float | None = None,
source_reference: SourceEvidenceReference | None = None,
requested_model: str | None = None,
evidence_capture: ProviderCallEvidence | None = None,
) -> TranscriptionResult:
"""Transcribe one source and write failure evidence into the provided capture sink."""
...
async def aclose(self) -> None:
"""Release any pooled network resources held by the adapter."""
...
src/transcription/providers/evidence.py
-161
View File
@@ -1,161 +0,0 @@
"""Versioned, provider-neutral contracts for processing evidence."""
from __future__ import annotations
import hashlib
import json
import platform
from importlib.metadata import PackageNotFoundError
from importlib.metadata import version
from typing import Any
from typing import Literal
from uuid import UUID
from pydantic import BaseModel
from pydantic import ConfigDict
from pydantic import Field
from pydantic import JsonValue
from transcription.config import Settings
REQUEST_MANIFEST_SCHEMA = "transcription.request-manifest"
REQUEST_MANIFEST_VERSION = "1"
SOFTWARE_CONTEXT_SCHEMA = "transcription.software-context"
SOFTWARE_CONTEXT_VERSION = "1"
TRANSPORT_EVIDENCE_SCHEMA = "transcription.transport-evidence"
TRANSPORT_EVIDENCE_VERSION = "1"
CANONICAL_JSON_ALGORITHM = "transcription-canonical-json-v1"
SAFE_RESPONSE_HEADERS = frozenset(
{
"content-type",
"content-encoding",
"date",
"retry-after",
"x-request-id",
"x-openrouter-generation-id",
"x-ratelimit-limit",
"x-ratelimit-remaining",
"x-ratelimit-reset",
}
)
class EvidenceModel(BaseModel):
"""Strict immutable base for persisted evidence contracts."""
model_config = ConfigDict(extra="forbid", frozen=True)
class SourceEvidenceReference(EvidenceModel):
"""Secret-safe identity for source content used by one execution."""
source_id: UUID
digest_sha256: str = Field(pattern=r"^[0-9a-f]{64}$")
byte_size: int = Field(ge=0)
media_type: str = Field(min_length=1)
page_number: int = Field(ge=1)
width: int | None = Field(default=None, ge=1)
height: int | None = Field(default=None, ge=1)
derivative_id: UUID | None = None
transformation: str | None = None
class SoftwareContext(EvidenceModel):
"""Versions needed to interpret a provider execution."""
schema_name: Literal["transcription.software-context"] = SOFTWARE_CONTEXT_SCHEMA
schema_version: Literal["1"] = SOFTWARE_CONTEXT_VERSION
application_version: str
application_commit: str | None = None
adapter_name: str
adapter_version: str
client_library: str
client_library_version: str
python_version: str
class RequestManifest(EvidenceModel):
"""Frozen, secret-safe representation of one concrete provider request."""
schema_name: Literal["transcription.request-manifest"] = REQUEST_MANIFEST_SCHEMA
schema_version: Literal["1"] = REQUEST_MANIFEST_VERSION
provider: str = Field(min_length=1)
requested_model: str = Field(min_length=1)
request: dict[str, JsonValue]
source: SourceEvidenceReference
explicitly_supplied_parameters: tuple[str, ...] = ()
omitted_optional_parameters: tuple[str, ...] = ()
optional_parameter_states: dict[str, Literal["omitted", "null", "value"]]
prompt_content: str = Field(min_length=1)
prompt_sha256: str = Field(pattern=r"^[0-9a-f]{64}$")
timeout_seconds: float = Field(gt=0)
retry_policy: str = Field(min_length=1)
software: SoftwareContext
canonicalization: Literal["transcription-canonical-json-v1"] = CANONICAL_JSON_ALGORITHM
def canonical_bytes(self) -> bytes:
return canonical_json_bytes(self.model_dump(mode="json"))
def digest(self) -> str:
return hashlib.sha256(self.canonical_bytes()).hexdigest()
class TransportEvidence(EvidenceModel):
"""Exact response captured at the application/router HTTP boundary."""
schema_name: Literal["transcription.transport-evidence"] = TRANSPORT_EVIDENCE_SCHEMA
schema_version: Literal["1"] = TRANSPORT_EVIDENCE_VERSION
response_received: bool
status_code: int | None = Field(default=None, ge=100, le=599)
body: bytes | None = None
safe_headers: dict[str, str] = Field(default_factory=dict)
content_type: str | None = None
content_encoding: str | None = None
request_id: str | None = None
generation_id: str | None = None
def canonical_json_bytes(value: Any) -> bytes:
"""Serialize JSON deterministically for evidence integrity hashes."""
return json.dumps(
value,
ensure_ascii=False,
allow_nan=False,
sort_keys=True,
separators=(",", ":"),
).encode("utf-8")
def filter_safe_response_headers(headers: Any) -> dict[str, str]:
"""Return only explicitly allowlisted response headers."""
return {
str(name).lower(): str(value) for name, value in headers.items() if str(name).lower() in SAFE_RESPONSE_HEADERS
}
def package_version(package: str) -> str:
"""Return an installed package version without failing evidence capture."""
try:
return version(package)
except PackageNotFoundError:
return "unknown"
def build_software_context(
*,
adapter_name: str,
adapter_version: str,
client_library: str,
settings: Settings,
) -> SoftwareContext:
"""Build the runtime software identity for an execution."""
return SoftwareContext(
application_version=package_version("transcription"),
application_commit=settings.transcription_commit,
adapter_name=adapter_name,
adapter_version=adapter_version,
client_library=client_library,
client_library_version=package_version(client_library),
python_version=platform.python_version(),
)
src/transcription/providers/openrouter.py
-544
View File
@@ -1,544 +0,0 @@
"""OpenRouter transcription provider adapter."""
from __future__ import annotations
import base64
import contextvars
import hashlib
import json
import logging
from collections.abc import AsyncIterator
from collections.abc import Callable
from dataclasses import dataclass
from typing import Annotated
from typing import Any
from typing import Literal
import httpx
from openrouter import OpenRouter
from openrouter import errors as openrouter_errors
from pydantic import BaseModel
from pydantic import ConfigDict
from pydantic import Field
from pydantic import JsonValue
from pydantic import TypeAdapter
from pydantic import ValidationError
from transcription.config import Settings
from transcription.config import get_settings
from transcription.providers.base import ProviderAuthError
from transcription.providers.base import ProviderCallEvidence
from transcription.providers.base import ProviderError
from transcription.providers.base import ProviderResponseError
from transcription.providers.base import ProviderUsage
from transcription.providers.base import TranscriptionMetadata
from transcription.providers.base import TranscriptionResult
from transcription.providers.evidence import RequestManifest
from transcription.providers.evidence import SourceEvidenceReference
from transcription.providers.evidence import TransportEvidence
from transcription.providers.evidence import build_software_context
from transcription.providers.evidence import filter_safe_response_headers
logger = logging.getLogger(__name__)
DEFAULT_OPENROUTER_MODEL = "google/gemini-2.5-flash"
OPENROUTER_ADAPTER_VERSION = "2"
class _CapturingAsyncByteStream(httpx.AsyncByteStream):
"""Copy streamed response bytes without changing what the SDK consumes."""
def __init__(self, stream: httpx.AsyncByteStream, on_complete: Callable[[bytes], None]):
self._stream = stream
self._on_complete = on_complete
async def __aiter__(self) -> AsyncIterator[bytes]:
content = bytearray()
async for chunk in self._stream:
content.extend(chunk)
yield chunk
self._on_complete(bytes(content))
async def aclose(self) -> None:
await self._stream.aclose()
class _CapturingAsyncClient:
"""Delegate SDK HTTP calls while retaining the response before SDK parsing."""
def __init__(self, client: httpx.AsyncClient):
self._client = client
self._active_capture: contextvars.ContextVar[_TransportCapture | None] = contextvars.ContextVar(
"openrouter_transport_capture",
default=None,
)
async def send(self, request: httpx.Request, **kwargs: Any) -> httpx.Response:
capture = self._active_capture.get()
response = await self._client.send(request, **kwargs)
if capture is None:
return response
capture.response = response
try:
capture.body = response.content
except httpx.ResponseNotRead:
stream = response.stream
if not isinstance(stream, httpx.AsyncByteStream):
raise
response.stream = _CapturingAsyncByteStream(stream, lambda body: self._capture_body(capture, body))
return response
def build_request(self, *args: Any, **kwargs: Any) -> httpx.Request:
return self._client.build_request(*args, **kwargs)
async def aclose(self) -> None:
await self._client.aclose()
def begin_capture(self, capture: _TransportCapture) -> contextvars.Token[_TransportCapture | None]:
return self._active_capture.set(capture)
def end_capture(self, token: contextvars.Token[_TransportCapture | None]) -> None:
self._active_capture.reset(token)
@staticmethod
def _capture_body(capture: _TransportCapture, body: bytes) -> None:
capture.body = body
@dataclass(slots=True)
class _TransportCapture:
response: httpx.Response | None = None
body: bytes | None = None
class _ProviderModel(BaseModel):
model_config = ConfigDict(extra="forbid", frozen=True)
class TextContent(_ProviderModel):
type: Literal["text"] = "text"
text: str = Field(min_length=1)
class ImageUrl(_ProviderModel):
url: str = Field(min_length=1)
class ImageContent(_ProviderModel):
type: Literal["image_url"] = "image_url"
image_url: ImageUrl
class FileData(_ProviderModel):
filename: str = Field(min_length=1)
file_data: str = Field(min_length=1)
class FileContent(_ProviderModel):
type: Literal["file"] = "file"
file: FileData
MessageContent = Annotated[TextContent | ImageContent | FileContent, Field(discriminator="type")]
class UserMessage(_ProviderModel):
role: Literal["user"] = "user"
content: tuple[MessageContent, ...] = Field(min_length=2)
class OpenRouterRequest(_ProviderModel):
"""Normalized request payload fields for OpenRouter calls."""
model: str = Field(min_length=1)
messages: tuple[UserMessage, ...] = Field(min_length=1)
http_referer: str | None
x_open_router_title: str | None
temperature: float | None = Field(ge=0.0, le=2.0)
top_p: float | None = Field(ge=0.0, le=1.0)
class ResponseContentPart(BaseModel):
model_config = ConfigDict(extra="allow", frozen=True)
text: str | None = None
class ResponseMessage(BaseModel):
model_config = ConfigDict(extra="allow", frozen=True)
content: str | tuple[ResponseContentPart, ...] | None = None
class ResponseChoice(BaseModel):
model_config = ConfigDict(extra="allow", frozen=True)
message: ResponseMessage
finish_reason: str | None = None
class ResponseUsage(BaseModel):
model_config = ConfigDict(extra="allow", frozen=True)
prompt_tokens: int | None = Field(default=None, ge=0)
completion_tokens: int | None = Field(default=None, ge=0)
total_tokens: int | None = Field(default=None, ge=0)
input_tokens: int | None = Field(default=None, ge=0)
output_tokens: int | None = Field(default=None, ge=0)
total: int | None = Field(default=None, ge=0)
class OpenRouterResponse(BaseModel):
model_config = ConfigDict(extra="allow", frozen=True)
model: str | None = None
choices: tuple[ResponseChoice, ...] = Field(min_length=1)
usage: dict[str, JsonValue] | None = None
JSON_OBJECT_ADAPTER = TypeAdapter(dict[str, JsonValue])
class OpenRouterTranscriptionProvider:
"""Adapter that performs image transcription through OpenRouter."""
def __init__(
self,
*,
settings: Settings | None = None,
client: OpenRouter | None = None,
async_client: httpx.AsyncClient | None = None,
):
self._settings = settings or get_settings()
self._model = self._settings.provider_model or DEFAULT_OPENROUTER_MODEL
self._capturing_client: _CapturingAsyncClient | None = None
if client is None:
# httpx defaults every phase to 5s, which silently caps provider calls far
# below worker_provider_timeout_seconds. Track the configured budget instead.
timeout = httpx.Timeout(
self._settings.worker_provider_timeout_seconds,
connect=10.0,
)
self._capturing_client = _CapturingAsyncClient(
async_client or httpx.AsyncClient(follow_redirects=True, timeout=timeout)
)
client = OpenRouter(
api_key=self._settings.openrouter_api_key.get_secret_value(),
async_client=self._capturing_client,
)
self._client = client
@property
def model(self) -> str:
"""Return the resolved OpenRouter model slug."""
return self._model
async def aclose(self) -> None:
if self._capturing_client is not None:
await self._capturing_client.aclose()
async def transcribe(
self,
*,
prompt_text: str,
image_bytes: bytes,
mime_type: str,
temperature: float | None = None,
top_p: float | None = None,
source_reference: SourceEvidenceReference | None = None,
requested_model: str | None = None,
evidence_capture: ProviderCallEvidence | None = None,
) -> TranscriptionResult:
"""Send prompt + image to OpenRouter and return normalized text output."""
request = self._build_request(
prompt_text=prompt_text,
image_bytes=image_bytes,
mime_type=mime_type,
temperature=temperature,
top_p=top_p,
requested_model=requested_model,
)
manifest = self._build_request_manifest(
request=request,
prompt_text=prompt_text,
source_reference=source_reference,
temperature=temperature,
top_p=top_p,
)
if evidence_capture is not None:
evidence_capture.request_manifest = manifest
evidence_capture.transport_evidence = None
transport_capture = _TransportCapture()
token = self._capturing_client.begin_capture(transport_capture) if self._capturing_client is not None else None
try:
try:
response = await self._client.chat.send_async(
**request.model_dump(mode="json", exclude_none=True),
retries=None,
)
except Exception as exc:
transport = self._captured_transport_evidence(transport_capture)
if evidence_capture is not None:
evidence_capture.transport_evidence = transport
if isinstance(exc, openrouter_errors.UnauthorizedResponseError):
raise ProviderAuthError(
"OpenRouter authentication failed",
request_manifest=manifest,
transport_evidence=transport,
failure_phase="http_response" if transport.response_received else "connection",
) from exc
failure_phase = (
"response_validation"
if isinstance(exc, openrouter_errors.ResponseValidationError)
else "http_response"
if transport.response_received
else "connection"
)
raise ProviderError(
self._transport_error_message(transport),
request_manifest=manifest,
transport_evidence=transport,
failure_phase=failure_phase,
) from exc
transport = self._captured_transport_evidence(transport_capture)
if evidence_capture is not None:
evidence_capture.transport_evidence = transport
raw_api_response = self._coerce_raw_response(response)
try:
validated_response = OpenRouterResponse.model_validate(raw_api_response)
except ValidationError as exc:
raise ProviderResponseError(
"OpenRouter response failed schema validation",
request_manifest=manifest,
transport_evidence=transport,
failure_phase="response_validation",
) from exc
try:
text = self._extract_text(validated_response)
except ProviderResponseError as exc:
raise ProviderResponseError(
str(exc),
request_manifest=manifest,
transport_evidence=transport,
failure_phase="response_validation",
) from exc
model = validated_response.model or requested_model or self.model
metadata = self._build_metadata(validated_response)
logger.info("OpenRouter transcription completed using model=%s", model)
return TranscriptionResult(
text=text,
provider="openrouter",
prompt_name=None,
prompt_hash=None,
system_prompt=None,
user_prompt=prompt_text,
temperature=temperature,
top_p=top_p,
model=model,
metadata=metadata,
raw_api_response=raw_api_response,
request_manifest=manifest,
transport_evidence=transport,
)
finally:
client = self._capturing_client
if token is not None and client is not None:
client.end_capture(token)
def _build_request_manifest(
self,
*,
request: OpenRouterRequest,
prompt_text: str,
source_reference: SourceEvidenceReference | None,
temperature: float | None,
top_p: float | None,
) -> RequestManifest | None:
if source_reference is None:
logger.warning("OpenRouter request manifest omitted because source evidence reference is missing.")
return None
request_payload = request.model_dump(mode="json", exclude_none=True)
sanitized_request = self._replace_embedded_media(request_payload, source_reference=source_reference)
explicit = tuple(name for name, value in (("temperature", temperature), ("top_p", top_p)) if value is not None)
omitted = tuple(name for name in ("temperature", "top_p") if name not in explicit)
return RequestManifest(
provider="openrouter",
requested_model=request.model,
request=JSON_OBJECT_ADAPTER.validate_python(sanitized_request),
source=source_reference,
explicitly_supplied_parameters=explicit,
omitted_optional_parameters=omitted,
optional_parameter_states={
"temperature": "value" if temperature is not None else "omitted",
"top_p": "value" if top_p is not None else "omitted",
},
prompt_content=prompt_text,
prompt_sha256=hashlib.sha256(prompt_text.encode("utf-8")).hexdigest(),
timeout_seconds=self._settings.worker_provider_timeout_seconds,
retry_policy="application-bounded; sdk-retries=0",
software=build_software_context(
adapter_name="openrouter",
adapter_version=OPENROUTER_ADAPTER_VERSION,
client_library="openrouter",
settings=self._settings,
),
)
def _replace_embedded_media(
self,
value: Any,
*,
source_reference: SourceEvidenceReference,
) -> Any:
if isinstance(value, str) and value.startswith("data:") and ";base64," in value:
return {
"source_reference": source_reference.model_dump(mode="json"),
"embedded_media_omitted": True,
}
if isinstance(value, dict):
return {
str(key): self._replace_embedded_media(item, source_reference=source_reference)
for key, item in value.items()
}
if isinstance(value, list | tuple):
return [self._replace_embedded_media(item, source_reference=source_reference) for item in value]
return value
def _captured_transport_evidence(self, capture: _TransportCapture) -> TransportEvidence:
response = capture.response
if response is None:
return TransportEvidence(response_received=False)
headers = filter_safe_response_headers(response.headers)
return TransportEvidence(
response_received=True,
status_code=response.status_code,
body=capture.body,
safe_headers=headers,
content_type=headers.get("content-type"),
content_encoding=headers.get("content-encoding"),
request_id=headers.get("x-request-id"),
generation_id=headers.get("x-openrouter-generation-id"),
)
@staticmethod
def _transport_error_message(transport: TransportEvidence) -> str:
message = "OpenRouter request failed"
if transport.status_code is not None:
message += f" with HTTP {transport.status_code}"
if transport.body is None:
return message
try:
payload = json.loads(transport.body)
except (UnicodeDecodeError, json.JSONDecodeError):
return message
if not isinstance(payload, dict):
return message
error = payload.get("error")
detail = error.get("message") if isinstance(error, dict) else None
if isinstance(detail, str) and detail.strip():
return f"{message}: {detail.strip()[:500]}"
return message
def _build_metadata(self, response: OpenRouterResponse) -> TranscriptionMetadata:
choice = response.choices[0]
finish_reason = choice.finish_reason.strip() if choice.finish_reason and choice.finish_reason.strip() else None
normalized_usage = None
if response.usage is not None:
try:
usage = ResponseUsage.model_validate(response.usage)
except ValidationError as exc:
logger.warning("Ignoring invalid OpenRouter usage metadata: %s", exc)
else:
normalized_usage = ProviderUsage(
input_tokens=usage.prompt_tokens if usage.prompt_tokens is not None else usage.input_tokens,
output_tokens=usage.completion_tokens
if usage.completion_tokens is not None
else usage.output_tokens,
total_tokens=usage.total_tokens if usage.total_tokens is not None else usage.total,
)
if normalized_usage.model_dump(exclude_none=True) == {}:
normalized_usage = None
return TranscriptionMetadata(finish_reason=finish_reason, usage=normalized_usage)
def _coerce_raw_response(self, response: Any) -> dict[str, JsonValue]:
payload = self._to_json_compatible(response)
try:
return JSON_OBJECT_ADAPTER.validate_python(payload)
except ValidationError as exc:
raise ProviderResponseError("OpenRouter response is not a JSON object") from exc
def _to_json_compatible(self, value: Any) -> Any:
if value is None or isinstance(value, str | int | float | bool):
return value
if isinstance(value, dict):
return {str(key): self._to_json_compatible(item) for key, item in value.items()}
if isinstance(value, list | tuple | set):
return [self._to_json_compatible(item) for item in value]
for method_name in ("model_dump", "to_dict"):
serializer = getattr(value, method_name, None)
if callable(serializer):
try:
serialized = serializer(mode="json") if method_name == "model_dump" else serializer()
return self._to_json_compatible(serialized)
except (TypeError, ValueError) as exc:
logger.debug("OpenRouter response serializer %s failed: %s", method_name, exc)
continue
object_dict = getattr(value, "__dict__", None)
if isinstance(object_dict, dict):
return {
str(key): self._to_json_compatible(item)
for key, item in object_dict.items()
if not str(key).startswith("_")
}
raise ProviderResponseError(f"OpenRouter response contains unsupported value type: {type(value).__name__}")
def _build_request(
self,
*,
prompt_text: str,
image_bytes: bytes,
mime_type: str,
temperature: float | None,
top_p: float | None,
requested_model: str | None = None,
) -> OpenRouterRequest:
image_b64 = base64.b64encode(image_bytes).decode("ascii")
data_url = f"data:{mime_type};base64,{image_b64}"
media_content: ImageContent | FileContent
if mime_type == "application/pdf":
media_content = FileContent(file=FileData(filename="source.pdf", file_data=data_url))
else:
media_content = ImageContent(image_url=ImageUrl(url=data_url))
return OpenRouterRequest(
model=requested_model or self.model,
messages=(UserMessage(content=(TextContent(text=prompt_text), media_content)),),
http_referer=self._settings.openrouter_http_referer,
x_open_router_title=self._settings.openrouter_app_title,
temperature=temperature,
top_p=top_p,
)
def _extract_text(self, response: OpenRouterResponse) -> str:
content = response.choices[0].message.content
text = self._normalize_content(content)
if not text:
raise ProviderResponseError("OpenRouter response contained no transcription text")
return text
def _normalize_content(self, content: str | tuple[ResponseContentPart, ...] | None) -> str:
if isinstance(content, str):
return content.strip()
if isinstance(content, tuple):
parts = [item.text.strip() for item in content if item.text and item.text.strip()]
return "\n".join(parts).strip()
return ""
src/transcription/runtime_helpers.py
-37
View File
@@ -1,37 +0,0 @@
from __future__ import annotations
import asyncio
from collections.abc import Awaitable
from collections.abc import Callable
from typing import TypeVar
from sqlalchemy.exc import IntegrityError
ResultT = TypeVar("ResultT")
async def run_blocking(func: Callable[..., ResultT], /, *args, **kwargs) -> ResultT:
"""Run blocking CPU/filesystem work on a worker thread."""
return await asyncio.to_thread(func, *args, **kwargs)
async def insert_with_sequence_retry(
*,
max_retries: int,
operation: Callable[[int], Awaitable[ResultT]],
on_conflict: Callable[[int, IntegrityError], None] | None = None,
) -> ResultT:
"""Retry a sequence-based insert operation on unique-key conflicts."""
if max_retries < 1:
raise ValueError("max_retries must be at least 1")
for retry in range(1, max_retries + 1):
try:
return await operation(retry)
except IntegrityError as exc:
if on_conflict is not None:
on_conflict(retry, exc)
if retry == max_retries:
raise
raise RuntimeError("insert_with_sequence_retry exhausted retries without returning or raising")
src/transcription/services/__init__.py
-67
View File
@@ -1,67 +0,0 @@
"""Service layer exports."""
from dataclasses import dataclass
from dataclasses import field
from typing import Self
from sqlalchemy.ext.asyncio import async_sessionmaker
from sqlmodel.ext.asyncio.session import AsyncSession
from ..config import Settings
from .documents import DocumentService
from .evidence import EvidenceService
from .jobs import JobService
from .maintenance import MaintenanceService
from .people import PeopleService
from .photos import PhotosService
from .prompts import PromptStore
from .sources import SourceService
__all__ = [
"DocumentService",
"EvidenceService",
"JobService",
"MaintenanceService",
"PeopleService",
"PhotosService",
"PromptStore",
"ServiceBundle",
"SourceService",
]
@dataclass(frozen=True, slots=True)
class ServiceBundle:
"""Container for all service instances."""
documents: DocumentService = field(default_factory=DocumentService)
sources: SourceService = field(default_factory=SourceService)
jobs: JobService = field(default_factory=JobService)
maintenance: MaintenanceService = field(default_factory=MaintenanceService)
people: PeopleService = field(default_factory=PeopleService)
photos: PhotosService = field(default_factory=PhotosService)
evidence: EvidenceService = field(default_factory=EvidenceService)
@classmethod
def from_session_factory(
cls,
session_factory: async_sessionmaker[AsyncSession] | None = None,
*,
settings: Settings | None = None,
) -> Self:
"""Build a bundle whose services all share one session factory and settings."""
if session_factory is None:
return cls()
return cls(
documents=DocumentService(session_factory=session_factory, settings=settings),
sources=SourceService(session_factory=session_factory, settings=settings),
jobs=JobService(session_factory=session_factory, settings=settings),
maintenance=MaintenanceService(session_factory=session_factory, settings=settings),
people=PeopleService(session_factory=session_factory, settings=settings),
photos=PhotosService(session_factory=session_factory, settings=settings),
evidence=EvidenceService(session_factory=session_factory, settings=settings),
)
async def aclose(self) -> None:
"""Release provider resources held by the bundle."""
await self.sources.aclose()
src/transcription/services/base.py
-84
View File
@@ -1,84 +0,0 @@
from abc import ABC
from collections.abc import Sequence
from contextlib import asynccontextmanager
from typing import Any
from sqlalchemy.ext.asyncio import async_sessionmaker
from sqlmodel.ext.asyncio.session import AsyncSession
from ..config import Settings
from ..config import get_settings
from ..db.session import resolve_session_factory
from ..db.session import session_scope
from ..errors import AppError
from ..errors import ErrorCategory
class ServiceBase(ABC):
"""Thin service class for managing documents in the database."""
settings: Settings
session_factory: async_sessionmaker[AsyncSession]
def __init__(
self,
session_factory: async_sessionmaker[AsyncSession] | None = None,
settings: Settings | None = None,
):
self.settings = settings or get_settings()
self.session_factory = session_factory or resolve_session_factory(settings=self.settings)
@asynccontextmanager
async def _session_scope(self, session: AsyncSession | None = None):
"""Provide a transactional scope around a series of operations."""
async with session_scope(
session_factory=self.session_factory,
session=session,
) as active_session:
yield active_session
async def _finalize(
self,
*,
session: AsyncSession,
caller_session: AsyncSession | None,
refresh: Sequence[object] = (),
) -> None:
"""Finalize a write based on transaction ownership.
Service-owned sessions commit immediately. Caller-owned sessions flush so
orchestration code can commit once at a larger transaction boundary.
"""
should_commit = caller_session is None
if should_commit:
await session.commit()
else:
await session.flush()
for obj in refresh:
await session.refresh(obj)
async def _get_or_raise[ModelT](
self,
model: type[ModelT],
entity_id: object,
*,
session: AsyncSession,
error: type[AppError],
noun: str,
suggestion: str,
options: Sequence[Any] = (),
) -> ModelT:
"""Load an entity by primary key or raise a not-found service error.
``noun`` and ``suggestion`` are supplied by the caller so each domain
keeps its own user-facing wording.
"""
entity = await session.get(model, entity_id, options=list(options) or None)
if entity is None:
raise error(
f"{noun} with id {entity_id} not found",
category=ErrorCategory.NOT_FOUND,
suggestion=suggestion,
)
return entity
src/transcription/services/documents.py
-628
View File
@@ -1,628 +0,0 @@
import logging
import shutil
from collections.abc import Sequence
from dataclasses import dataclass
from datetime import date
from datetime import datetime
from pathlib import Path
from typing import Any
from uuid import UUID
from sqlalchemy.exc import IntegrityError
from sqlalchemy.ext.asyncio import async_sessionmaker
from sqlmodel import SQLModel
from sqlmodel import col
from sqlmodel import select
from sqlmodel.ext.asyncio.session import AsyncSession
from ..config import Settings
from ..db.loading import orm_attribute
from ..db.loading import selectinload
from ..db.models import Document
from ..db.models import DocumentPerson
from ..db.models import DocumentTag
from ..db.models import DocumentType
from ..db.models import Tag
from ..db.registries import AUTHOR_ROLE_SEMANTIC_KEY
from ..errors import AppError
from ..errors import ErrorCategory
from .base import ServiceBase
from .registry import RegistryService
from .registry import RegistrySummary
from .source_media import lookup_source_mime_type
from .source_media import supported_source_formats
logger = logging.getLogger(__name__)
class DocumentError(AppError):
"""Raised when document operations fail."""
class MissingSourceError(DocumentError):
"""Raised when a document has no associated sources."""
class DocumentAlreadyExistsError(DocumentError):
"""Raised when a document with the same name already exists in the database."""
class DocumentDeleteBlockedError(DocumentError):
"""Raised when a document delete is blocked by dependent records."""
class DocumentTypeError(DocumentError):
"""Raised when Document Type maintenance fails."""
class TagError(DocumentError):
"""Raised when Tag maintenance fails."""
class DocumentTypeRegistry(RegistryService[DocumentType]):
"""Document Type registry maintenance."""
model = DocumentType
error = DocumentTypeError
noun = "Document Type"
short_noun = "type"
referenced_retainer = "historical Documents"
def reference_model(self) -> type[SQLModel]:
return Document
def reference_id_column(self) -> Any:
return col(Document.id)
def reference_key_column(self) -> Any:
return col(Document.document_type_id)
type DocumentTypeSummary = RegistrySummary
class TagRegistry(RegistryService[Tag]):
"""Tag registry maintenance."""
model = Tag
error = TagError
noun = "Tag"
short_noun = "tag"
referenced_retainer = "historical Documents"
def reference_model(self) -> type[SQLModel]:
return DocumentTag
def reference_id_column(self) -> Any:
return col(DocumentTag.id)
def reference_key_column(self) -> Any:
return col(DocumentTag.tag_id)
type TagSummary = RegistrySummary
@dataclass(frozen=True, slots=True)
class DocumentPrintSource:
id: UUID
page_number: int
media_type: str
current_text: str | None
@dataclass(frozen=True, slots=True)
class DocumentPrintJob:
id: UUID
date_created: datetime
provider: str | None
model: str | None
prompt_name: str | None
retry_count: int
status: str
@dataclass(frozen=True, slots=True)
class DocumentPrintProjection:
id: UUID
title: str
document_type: str | None
authors: tuple[str, ...]
document_date: date | None
document_date_raw: str | None
location_created: str | None
archive_identifier: str | None
notes: str | None
sources: tuple[DocumentPrintSource, ...]
jobs: tuple[DocumentPrintJob, ...]
class DocumentService(ServiceBase):
"""Thin service class for managing documents in the database."""
def __init__(
self,
session_factory: async_sessionmaker[AsyncSession] | None = None,
settings: Settings | None = None,
) -> None:
super().__init__(session_factory, settings)
self._document_types = DocumentTypeRegistry(self.session_factory, self.settings)
self._tags = TagRegistry(self.session_factory, self.settings)
async def _validate_document_type(self, *, session: AsyncSession, document: Document) -> None:
"""Validate the UUID-backed Document Type reference."""
if document.document_type_id is None:
return
if await session.get(DocumentType, document.document_type_id) is None:
raise DocumentError(
f"Document type with id {document.document_type_id} not found",
category=ErrorCategory.VALIDATION,
suggestion="Select a valid document type and retry.",
)
async def _read_document(
self,
*,
session: AsyncSession,
document_id: UUID,
options: Sequence[Any] = (),
suggestion: str = "Verify the document id and retry.",
) -> Document:
return await self._get_or_raise(
Document,
document_id,
session=session,
error=DocumentError,
noun="Document",
suggestion=suggestion,
options=options,
)
#
# CRUD Operations
#
async def create_document(
self,
document: Document,
*,
session: AsyncSession | None = None,
) -> Document:
"""Create a new document in the database."""
async with self._session_scope(session) as _session:
await self._validate_document_type(session=_session, document=document)
_session.add(document)
try:
await self._finalize(session=_session, caller_session=session, refresh=(document,))
except IntegrityError as exc:
raise DocumentAlreadyExistsError(
f"Document with id {document.id} already exists",
category=ErrorCategory.VALIDATION,
suggestion="Rename the file and try again.",
) from exc
return document
async def read_document(self, document_id: UUID, *, session: AsyncSession | None = None) -> Document:
"""Read an existing document from the database.
The selectinload option is used to eagerly load related jobs and sources.
"""
async with self._session_scope(session) as _session:
document = await self._read_document(
session=_session,
document_id=document_id,
options=(
selectinload(Document.jobs),
selectinload(Document.sources),
),
suggestion="Re-upload the source document and retry.",
)
if not document.sources:
raise MissingSourceError(
f"Document with id {document_id} has no associated source records",
category=ErrorCategory.NOT_FOUND,
suggestion="Upload at least one source for this document and retry.",
)
return document
async def update_document(self, document: Document, *, session: AsyncSession | None = None) -> Document:
"""Update an existing document in the database."""
async with self._session_scope(session) as _session:
await self._validate_document_type(session=_session, document=document)
merged = await _session.merge(document)
await self._finalize(session=_session, caller_session=session, refresh=(merged,))
return merged
async def delete_document(self, document: Document, *, session: AsyncSession | None = None) -> None:
"""Delete a document from the database."""
document_id = document.id
async with self._session_scope(session) as _session:
existing = await self._read_document(
session=_session,
document_id=document.id,
options=(
selectinload(Document.jobs),
selectinload(Document.sources),
selectinload(Document.document_people),
),
)
has_jobs = bool(existing.jobs)
has_sources = bool(existing.sources)
if has_jobs or has_sources:
blocked_by: list[str] = []
if has_sources:
blocked_by.append("Sources")
if has_jobs:
blocked_by.append("Jobs")
raise DocumentDeleteBlockedError(
f"Document delete blocked by related records: {', '.join(blocked_by)}",
category=ErrorCategory.VALIDATION,
suggestion="Remove related Sources and Jobs first, then retry deletion.",
)
for link in list(existing.document_people):
await _session.delete(link)
await _session.delete(existing)
await self._finalize(session=_session, caller_session=session)
self._delete_document_storage_folder(document_id=document_id)
def _delete_document_storage_folder(self, *, document_id: UUID) -> None:
"""Best-effort cleanup for document-scoped source storage."""
document_dir = self.settings.upload_dir / "documents" / str(document_id)
if not document_dir.exists():
return
try:
shutil.rmtree(document_dir)
logger.info("Deleted document storage folder: %s", document_dir)
except OSError:
logger.warning("Failed to delete document storage folder: %s", document_dir)
# Query Operations
async def query_documents(
self, *, name: str | None = None, session: AsyncSession | None = None
) -> Sequence[Document]:
"""Query documents from the database based on provided filters."""
async with self._session_scope(session) as _session:
query = select(Document)
if name is not None:
query = query.where(Document.name == name)
result = await _session.exec(query)
return result.all()
async def list_documents(self, *, session: AsyncSession | None = None) -> Sequence[Document]:
"""List documents with relations needed by the archival table."""
async with self._session_scope(session) as _session:
query = select(Document).options(
selectinload(Document.document_people).selectinload(orm_attribute(DocumentPerson.person)),
selectinload(Document.document_people).selectinload(orm_attribute(DocumentPerson.role_ref)),
selectinload(Document.document_type_ref),
selectinload(Document.document_tags).selectinload(orm_attribute(DocumentTag.tag_ref)),
selectinload(Document.sources),
selectinload(Document.jobs),
)
result = await _session.exec(query)
return result.all()
async def read_document_detail(self, document_id: UUID, *, session: AsyncSession | None = None) -> Document:
"""Read a document with eagerly loaded relations for UI detail rendering."""
async with self._session_scope(session) as _session:
query = (
select(Document)
.options(
selectinload(Document.jobs),
selectinload(Document.sources),
selectinload(Document.document_people).selectinload(orm_attribute(DocumentPerson.person)),
selectinload(Document.document_people).selectinload(orm_attribute(DocumentPerson.role_ref)),
selectinload(Document.document_type_ref),
selectinload(Document.document_tags).selectinload(orm_attribute(DocumentTag.tag_ref)),
)
.where(Document.id == document_id)
.execution_options(populate_existing=True)
)
document = (await _session.exec(query)).first()
if document is None:
raise DocumentError(
f"Document with id {document_id} not found",
category=ErrorCategory.NOT_FOUND,
suggestion="Verify the document id and retry.",
)
return document
async def read_document_print_projection(
self,
document_id: UUID,
*,
session: AsyncSession | None = None,
) -> DocumentPrintProjection:
"""Build the safe, deterministic read model used by print previews."""
document = await self.read_document_detail(document_id, session=session)
authors = sorted(
(
link.person.full_name
for link in document.document_people
if link.person is not None
and link.role_ref is not None
and link.role_ref.semantic_key == AUTHOR_ROLE_SEMANTIC_KEY
),
key=str.casefold,
)
sources = tuple(
DocumentPrintSource(
id=source.id,
page_number=source.page_number,
media_type=_print_media_type(source.filename),
current_text=_current_print_text(source.revised_text, source.raw_transcription),
)
for source in sorted(document.sources, key=lambda item: (item.page_number, item.id))
)
jobs = tuple(
DocumentPrintJob(
id=job.id,
date_created=job.date_created,
provider=job.provider,
model=job.model,
prompt_name=job.prompt_name,
retry_count=job.retry_count,
status=getattr(job.status, "value", str(job.status)),
)
for job in sorted(document.jobs, key=lambda item: (item.date_created, item.id))
)
return DocumentPrintProjection(
id=document.id,
title=document.name,
document_type=document.document_type_ref.label if document.document_type_ref is not None else None,
authors=tuple(authors),
document_date=document.document_date,
document_date_raw=document.document_date_raw,
location_created=document.location_created,
archive_identifier=document.archive_identifier,
notes=document.notes,
sources=sources,
jobs=jobs,
)
async def list_document_types(
self,
*,
active_only: bool = True,
session: AsyncSession | None = None,
) -> Sequence[DocumentType]:
"""List configured document types."""
return await self._document_types.list_entries(active_only=active_only, session=session)
async def list_tags(
self,
*,
active_only: bool = True,
session: AsyncSession | None = None,
) -> Sequence[Tag]:
"""List configured tags."""
return await self._tags.list_entries(active_only=active_only, session=session)
async def list_document_type_summaries(
self,
*,
session: AsyncSession | None = None,
) -> Sequence[DocumentTypeSummary]:
"""List Document Types alphabetically with current usage counts."""
rows = await self._document_types.list_entries_with_counts(session=session)
return [
RegistrySummary(
id=document_type.id,
label=document_type.label,
is_active=document_type.is_active,
is_built_in=document_type.semantic_key is not None,
reference_count=document_count,
)
for document_type, document_count in rows
]
async def list_tag_summaries(
self,
*,
session: AsyncSession | None = None,
) -> Sequence[TagSummary]:
"""List Tags alphabetically with current usage counts."""
rows = await self._tags.list_entries_with_counts(session=session)
return [
RegistrySummary(
id=tag.id,
label=tag.label,
is_active=tag.is_active,
is_built_in=tag.semantic_key is not None,
reference_count=document_count,
)
for tag, document_count in rows
]
async def create_document_type(
self,
*,
label: str,
is_active: bool = True,
session: AsyncSession | None = None,
) -> DocumentType:
"""Create a UUID-identified Document Type with a unique label."""
return await self._document_types.create_entry(label=label, is_active=is_active, session=session)
async def create_tag(
self,
*,
label: str,
is_active: bool = True,
session: AsyncSession | None = None,
) -> Tag:
"""Create a UUID-identified Tag with a unique label."""
return await self._tags.create_entry(label=label, is_active=is_active, session=session)
async def read_document_type(
self,
document_type_id: UUID,
*,
session: AsyncSession | None = None,
) -> DocumentType:
"""Read a Document Type by id."""
return await self._document_types.read_entry(document_type_id, session=session)
async def read_tag(
self,
tag_id: UUID,
*,
session: AsyncSession | None = None,
) -> Tag:
"""Read a Tag by id."""
return await self._tags.read_entry(tag_id, session=session)
async def update_document_type(
self,
document_type_id: UUID,
*,
label: str,
is_active: bool,
session: AsyncSession | None = None,
) -> DocumentType:
"""Update a Document Type label and active state."""
return await self._document_types.update_entry(
document_type_id,
label=label,
is_active=is_active,
session=session,
)
async def update_tag(
self,
tag_id: UUID,
*,
label: str,
is_active: bool,
session: AsyncSession | None = None,
) -> Tag:
"""Update a Tag label and active state."""
return await self._tags.update_entry(
tag_id,
label=label,
is_active=is_active,
session=session,
)
async def delete_document_type(
self,
document_type_id: UUID,
*,
session: AsyncSession | None = None,
) -> None:
"""Delete an unreferenced Document Type without cascade behavior."""
await self._document_types.delete_entry(document_type_id, session=session)
async def delete_tag(
self,
tag_id: UUID,
*,
session: AsyncSession | None = None,
) -> None:
"""Delete an unreferenced Tag without cascade behavior."""
await self._tags.delete_entry(tag_id, session=session)
async def is_document_type_referenced(
self,
document_type_id: UUID,
*,
session: AsyncSession | None = None,
) -> bool:
"""Return whether a Document references a Document Type."""
return await self._document_types.is_referenced(document_type_id, session=session)
async def is_tag_referenced(
self,
tag_id: UUID,
*,
session: AsyncSession | None = None,
) -> bool:
"""Return whether a Document references a Tag."""
return await self._tags.is_referenced(tag_id, session=session)
async def set_document_type(
self,
*,
document_id: UUID,
document_type_id: UUID,
session: AsyncSession | None = None,
) -> Document:
"""Set a Document Type by UUID."""
async with self._session_scope(session) as _session:
document = await self._read_document(session=_session, document_id=document_id)
document.document_type_id = document_type_id
await self._validate_document_type(session=_session, document=document)
await self._finalize(session=_session, caller_session=session, refresh=(document,))
return document
async def sync_document_tags_by_labels(
self,
*,
document_id: UUID,
labels: Sequence[str],
session: AsyncSession | None = None,
) -> None:
"""Replace a Document's tag set using label-based assignment."""
normalized_labels = [self._tags.normalize_label(label) for label in labels]
deduplicated_labels = list(dict.fromkeys(normalized_labels))
label_keys = [self._tags.label_key(label) for label in deduplicated_labels]
async with self._session_scope(session) as _session:
existing_document = await _session.get(Document, document_id)
if existing_document is None:
raise DocumentError(
f"Document with id {document_id} not found",
category=ErrorCategory.NOT_FOUND,
suggestion="Refresh and select an existing document.",
)
existing_tags = (
(await _session.exec(select(Tag).where(col(Tag.normalized_label).in_(label_keys)))).all()
if label_keys
else []
)
tags_by_key = {tag.normalized_label: tag for tag in existing_tags}
selected_tag_ids: set[UUID] = set()
for label in deduplicated_labels:
key = self._tags.label_key(label)
tag = tags_by_key.get(key)
if tag is None:
tag = await self._tags.create_entry(label=label, is_active=True, session=_session)
tags_by_key[key] = tag
selected_tag_ids.add(tag.id)
links = (await _session.exec(select(DocumentTag).where(DocumentTag.document_id == document_id))).all()
existing_ids = {link.tag_id for link in links}
for link in links:
if link.tag_id not in selected_tag_ids:
await _session.delete(link)
for tag_id in selected_tag_ids - existing_ids:
_session.add(DocumentTag(document_id=document_id, tag_id=tag_id))
await self._finalize(session=_session, caller_session=session)
def _print_media_type(filename: str) -> str:
"""Resolve a stored Source filename to its MIME type for print rendering."""
mime_type = lookup_source_mime_type(filename)
if mime_type is None:
raise DocumentError(
f"Unsupported Source format: {Path(filename).suffix.lower() or '<none>'}",
category=ErrorCategory.USER_INPUT,
suggestion=f"Use one of the supported Source formats: {supported_source_formats()}.",
)
return mime_type
def _current_print_text(revised_text: str | None, raw_transcription: str | None) -> str | None:
selected = revised_text if revised_text is not None else raw_transcription
return selected if selected is not None and selected.strip() else None
src/transcription/services/errors.py
-36
View File
@@ -1,36 +0,0 @@
"""Error vocabulary shared across the source, evidence, and prompt services.
These live in a neutral module rather than in the service that raises them
because more than one service raises them, and ``services.instructions.md``
forbids a service module from importing a sibling. Orchestration modules and
the UI import from here, so the exception a caller catches does not change when
an operation moves between services.
"""
from __future__ import annotations
from transcription.errors import AppError
class PromptLoadError(AppError):
"""Raised when prompt artifacts cannot be loaded safely."""
class PromptStoreError(PromptLoadError):
"""Raised when prompt storage validation or persistence fails."""
class TranscriptionError(AppError):
"""Raised when transcription execution fails."""
class TranscriptionNotFoundError(TranscriptionError):
"""Raised when a transcription-related resource is not found."""
class SourceDeleteBlockedError(TranscriptionError):
"""Raised when source deletion is blocked by dependency policy."""
class CandidatePromotionError(TranscriptionError):
"""Raised when a machine attempt cannot be selected for its Source."""
src/transcription/services/evidence.py
-230
View File
@@ -1,230 +0,0 @@
"""Read and export the immutable execution evidence trail.
``ExecutionAttempt`` is append-only: one row per provider call, written once by
the transcription workflow and never updated. Everything here is therefore a
read, a projection, or an export, with one exception - ``promote_machine_attempt``
selects which attempt a ``Source`` presents, which is an evidence decision even
though the write lands on ``Source``.
"""
from __future__ import annotations
import base64
import hashlib
from collections.abc import Sequence
from dataclasses import dataclass
from uuid import UUID
from pydantic import JsonValue
from sqlalchemy import inspect as sqlalchemy_inspect
from sqlmodel import col
from sqlmodel import select
from sqlmodel.ext.asyncio.session import AsyncSession
from transcription.db.models import ExecutionAttempt
from transcription.db.models import JobSourceStatus
from transcription.db.models import Source
from transcription.errors import ErrorCategory
from ..db.loading import defer
from .base import ServiceBase
from .errors import CandidatePromotionError
from .errors import TranscriptionNotFoundError
@dataclass(frozen=True, slots=True)
class LatestExecutionAttempt:
"""One execution attempt plus the loader facts a caller needs to render it."""
attempt: ExecutionAttempt
transport_body_deferred: bool
class EvidenceService(ServiceBase):
"""Read, project, and export execution attempt evidence."""
async def read_latest_job_error_category(
self,
*,
job_id: UUID,
session: AsyncSession | None = None,
) -> str | None:
"""Read the latest persisted execution-attempt error category for a job."""
async with self._session_scope(session) as _session:
query = (
select(ExecutionAttempt.error_category)
.where(ExecutionAttempt.job_id == job_id)
.where(col(ExecutionAttempt.error_category).is_not(None))
.order_by(
col(ExecutionAttempt.created_at).desc(),
col(ExecutionAttempt.id).desc(),
)
.limit(1)
)
return (await _session.exec(query)).first()
async def read_latest_execution_attempt(
self,
*,
job_source_id: UUID,
session: AsyncSession | None = None,
) -> LatestExecutionAttempt | None:
"""Read only the latest immutable attempt for one compatibility projection.
The transport body is deferred because it can be arbitrarily large; the
returned read model reports that as a plain flag so callers never have to
inspect ORM loader state.
"""
async with self._session_scope(session) as _session:
query = (
select(ExecutionAttempt)
.options(defer(ExecutionAttempt.transport_body))
.where(ExecutionAttempt.job_source_id == job_source_id)
.order_by(
col(ExecutionAttempt.attempt_number).desc(),
col(ExecutionAttempt.id).desc(),
)
.limit(1)
)
attempt = (await _session.exec(query)).first()
if attempt is None:
return None
deferred = "transport_body" in sqlalchemy_inspect(attempt).unloaded
return LatestExecutionAttempt(attempt=attempt, transport_body_deferred=deferred)
async def list_execution_attempts(
self,
*,
source_id: UUID | None = None,
job_id: UUID | None = None,
session: AsyncSession | None = None,
) -> Sequence[ExecutionAttempt]:
"""List immutable execution evidence in stable attempt order."""
async with self._session_scope(session) as _session:
query = select(ExecutionAttempt)
if source_id is not None:
query = query.where(ExecutionAttempt.source_id == source_id)
if job_id is not None:
query = query.where(ExecutionAttempt.job_id == job_id)
query = query.order_by(
col(ExecutionAttempt.job_id),
col(ExecutionAttempt.source_id),
col(ExecutionAttempt.attempt_number),
col(ExecutionAttempt.id),
)
return (await _session.exec(query)).all()
async def promote_machine_attempt(
self,
*,
source_id: UUID,
execution_attempt_id: UUID,
session: AsyncSession | None = None,
) -> Source:
"""Atomically select one successful machine attempt as the Source projection."""
async with self._session_scope(session) as _session:
source = await self._read_source(
session=_session,
source_id=source_id,
suggestion="Refresh Source Detail and retry.",
)
attempt = await _session.get(ExecutionAttempt, execution_attempt_id)
if (
attempt is None
or attempt.source_id != source_id
or attempt.status != JobSourceStatus.TRANSCRIBED
or not attempt.raw_transcription
):
raise CandidatePromotionError(
"Only a successful transcription attempt belonging to this Source can be selected",
category=ErrorCategory.VALIDATION,
suggestion="Select an available successful candidate from Source Detail.",
)
source.preferred_execution_attempt_id = attempt.id
source.raw_transcription = attempt.raw_transcription
await self._finalize(session=_session, caller_session=session, refresh=(source,))
return source
async def build_evidence_export(
self,
*,
source_id: UUID,
session: AsyncSession | None = None,
) -> dict[str, JsonValue]:
"""Build a versioned, source-reference-only evidence export."""
async with self._session_scope(session) as _session:
source = await self._read_source(session=_session, source_id=source_id)
attempts = list(await self.list_execution_attempts(source_id=source_id, session=_session))
attempt_payloads = [
{
"id": str(attempt.id),
"job_id": str(attempt.job_id),
"source_id": str(attempt.source_id),
"attempt_number": attempt.attempt_number,
"status": attempt.status.value,
"provider": attempt.provider,
"model": attempt.model,
"request_manifest": attempt.request_manifest,
"request_manifest_sha256": attempt.request_manifest_sha256,
"request_manifest_schema_version": attempt.request_manifest_schema_version,
"transport": {
"response_received": attempt.response_received,
"status_code": attempt.transport_status_code,
"body_base64": (
base64.b64encode(attempt.transport_body).decode("ascii")
if attempt.transport_body is not None
else None
),
"body_sha256": (
hashlib.sha256(attempt.transport_body).hexdigest()
if attempt.transport_body is not None
else None
),
"content_type": attempt.transport_content_type,
"content_encoding": attempt.transport_content_encoding,
"safe_headers": attempt.transport_safe_headers,
"request_id": attempt.router_request_id,
"generation_id": attempt.router_generation_id,
},
"sdk_response_snapshot": attempt.sdk_response_snapshot,
"normalized_metadata": attempt.normalized_metadata,
"software_context": attempt.software_context,
"raw_transcription": attempt.raw_transcription,
"error_category": attempt.error_category,
"error_detail": attempt.error_detail,
"failure_phase": attempt.failure_phase,
"started_at": attempt.started_at.isoformat(),
"finished_at": attempt.finished_at.isoformat(),
"duration_ms": attempt.duration_ms,
}
for attempt in attempts
]
return {
"schema_name": "transcription.evidence-export",
"schema_version": "1",
"source": {
"id": str(source.id),
"digest_sha256": source.file_hash,
"byte_size": source.file_size_bytes,
"page_number": source.page_number,
"upload_name": source.upload_name,
},
"attempts": attempt_payloads,
}
async def _read_source(
self,
*,
session: AsyncSession,
source_id: UUID,
suggestion: str = "Verify the source id and retry.",
) -> Source:
return await self._get_or_raise(
Source,
source_id,
session=session,
error=TranscriptionNotFoundError,
noun="Source",
suggestion=suggestion,
)
src/transcription/services/gedcom_import.py
-599
View File
@@ -1,599 +0,0 @@
"""GEDCOM parsing and import helpers for maintenance-driven genealogy sync."""
from __future__ import annotations
from collections.abc import Mapping
from dataclasses import dataclass
from datetime import date
from pathlib import Path
from uuid import UUID
from uuid import uuid4
from gedcom.element.element import Element
from gedcom.parser import Parser
from sqlalchemy import or_
from sqlmodel import col
from sqlmodel import delete
from sqlmodel import select
from sqlmodel.ext.asyncio.session import AsyncSession
from transcription.db.models import GenealogyCitation
from transcription.db.models import GenealogyCitationFactType
from transcription.db.models import GenealogyCitationSourceKind
from transcription.db.models import GenealogyFamily
from transcription.db.models import GenealogyFamilyChild
from transcription.db.models import GenealogyPerson
from transcription.errors import AppError
from transcription.errors import ErrorCategory
_MONTHS = {
"JAN": 1,
"FEB": 2,
"MAR": 3,
"APR": 4,
"MAY": 5,
"JUN": 6,
"JUL": 7,
"AUG": 8,
"SEP": 9,
"OCT": 10,
"NOV": 11,
"DEC": 12,
}
@dataclass(frozen=True, slots=True)
class ParsedCitation:
fact_type: GenealogyCitationFactType
raw_citation_text: str
@dataclass(frozen=True, slots=True)
class ParsedPerson:
pointer: str
fs_id: str | None
full_name: str
birth_date: date | None
birth_date_raw: str | None
birth_place: str | None
death_date: date | None
death_date_raw: str | None
death_place: str | None
citations: tuple[ParsedCitation, ...]
@dataclass(frozen=True, slots=True)
class ParsedFamilyChild:
child_pointer: str
relationship_type: str | None
@dataclass(frozen=True, slots=True)
class ParsedFamily:
fs_family_id: str | None
husband_pointer: str | None
wife_pointer: str | None
marriage_date: date | None
marriage_date_raw: str | None
marriage_place: str | None
children: tuple[ParsedFamilyChild, ...]
citations: tuple[ParsedCitation, ...]
@dataclass(frozen=True, slots=True)
class ParsedGedcom:
people: tuple[ParsedPerson, ...]
families: tuple[ParsedFamily, ...]
@dataclass(frozen=True, slots=True)
class GedcomImportResult:
new_people: int
updated_people: int
skipped_people_without_fs_id: int
new_families: int
updated_families: int
skipped_families_without_fs_id: int
family_children: int
citations: int
class GedcomImportError(AppError):
"""Raised when GEDCOM content cannot be parsed or imported."""
def parse_gedcom(*, file_path: Path) -> ParsedGedcom:
parser = Parser()
try:
parser.parse_file(str(file_path))
except Exception as exc:
raise GedcomImportError(
"GEDCOM file could not be parsed.",
category=ErrorCategory.VALIDATION,
suggestion="Upload a GEDCOM 5.5.1-compatible export and retry.",
detail=f"{type(exc).__name__}: {exc}",
) from exc
people: list[ParsedPerson] = []
families: list[ParsedFamily] = []
for element in parser.get_root_child_elements():
tag = element.get_tag()
if tag == "INDI":
people.append(_parse_person(element))
elif tag == "FAM":
families.append(_parse_family(element))
return ParsedGedcom(people=tuple(people), families=tuple(families))
async def import_gedcom_file(*, session: AsyncSession, file_path: Path) -> GedcomImportResult:
parsed = parse_gedcom(file_path=file_path)
pointer_to_fs_id = _pointer_to_fs_id_map(parsed=parsed)
people_by_fs_id, new_people, updated_people, skipped_people_without_fs_id = await _upsert_people(
session=session,
parsed=parsed,
)
(
families_by_fs_id,
new_families,
updated_families,
skipped_families_without_fs_id,
family_children,
) = await _upsert_families(
session=session,
parsed=parsed,
pointer_to_fs_id=pointer_to_fs_id,
people_by_fs_id=people_by_fs_id,
)
citation_rows = _citation_rows(
parsed=parsed,
pointer_to_fs_id=pointer_to_fs_id,
people=people_by_fs_id,
families=families_by_fs_id,
)
await _replace_imported_citations(
session=session,
person_ids={item.id for item in people_by_fs_id.values()},
family_ids={item.id for item in families_by_fs_id.values()},
citations=citation_rows,
)
await session.commit()
return GedcomImportResult(
new_people=new_people,
updated_people=updated_people,
skipped_people_without_fs_id=skipped_people_without_fs_id,
new_families=new_families,
updated_families=updated_families,
skipped_families_without_fs_id=skipped_families_without_fs_id,
family_children=family_children,
citations=len(citation_rows),
)
async def _load_people_by_fs_id(*, session: AsyncSession, fs_ids: set[str]) -> dict[str, GenealogyPerson]:
if not fs_ids:
return {}
query = select(GenealogyPerson).where(col(GenealogyPerson.fs_id).in_(fs_ids))
return {person.fs_id: person for person in (await session.exec(query)).all()}
async def _load_families_by_fs_id(*, session: AsyncSession, fs_family_ids: set[str]) -> dict[str, GenealogyFamily]:
if not fs_family_ids:
return {}
query = select(GenealogyFamily).where(col(GenealogyFamily.fs_family_id).in_(fs_family_ids))
return {family.fs_family_id: family for family in (await session.exec(query)).all()}
def _pointer_to_fs_id_map(*, parsed: ParsedGedcom) -> dict[str, str]:
return {person.pointer: person.fs_id for person in parsed.people if person.pointer and person.fs_id is not None}
async def _upsert_people(
*,
session: AsyncSession,
parsed: ParsedGedcom,
) -> tuple[dict[str, GenealogyPerson], int, int, int]:
people_with_fs_id = [person for person in parsed.people if person.fs_id is not None]
fs_ids = {person.fs_id for person in people_with_fs_id if person.fs_id is not None}
people_by_fs_id = await _load_people_by_fs_id(session=session, fs_ids=fs_ids)
new_people = 0
updated_people = 0
for person in people_with_fs_id:
assert person.fs_id is not None
existing = people_by_fs_id.get(person.fs_id)
if existing is None:
existing = GenealogyPerson(
fs_id=person.fs_id,
full_name=person.full_name,
birth_date=person.birth_date,
birth_date_raw=person.birth_date_raw,
birth_place=person.birth_place,
death_date=person.death_date,
death_date_raw=person.death_date_raw,
death_place=person.death_place,
)
session.add(existing)
await session.flush()
people_by_fs_id[person.fs_id] = existing
new_people += 1
continue
if _apply_person_updates(existing=existing, person=person):
updated_people += 1
skipped = len(parsed.people) - len(people_with_fs_id)
return people_by_fs_id, new_people, updated_people, skipped
def _apply_person_updates(*, existing: GenealogyPerson, person: ParsedPerson) -> bool:
changed = False
fields = (
("full_name", person.full_name),
("birth_date", person.birth_date),
("birth_date_raw", person.birth_date_raw),
("birth_place", person.birth_place),
("death_date", person.death_date),
("death_date_raw", person.death_date_raw),
("death_place", person.death_place),
)
for name, value in fields:
if getattr(existing, name) != value:
setattr(existing, name, value)
changed = True
return changed
async def _upsert_families(
*,
session: AsyncSession,
parsed: ParsedGedcom,
pointer_to_fs_id: Mapping[str, str],
people_by_fs_id: dict[str, GenealogyPerson],
) -> tuple[dict[str, GenealogyFamily], int, int, int, int]:
families_with_fs_id = [family for family in parsed.families if family.fs_family_id is not None]
fs_family_ids = {family.fs_family_id for family in families_with_fs_id if family.fs_family_id is not None}
families_by_fs_id = await _load_families_by_fs_id(session=session, fs_family_ids=fs_family_ids)
new_families = 0
updated_families = 0
family_children = 0
for family in families_with_fs_id:
assert family.fs_family_id is not None
husband_id = _person_id_from_pointer(
pointer=family.husband_pointer,
pointer_to_fs_id=pointer_to_fs_id,
people=people_by_fs_id,
)
wife_id = _person_id_from_pointer(
pointer=family.wife_pointer,
pointer_to_fs_id=pointer_to_fs_id,
people=people_by_fs_id,
)
target = families_by_fs_id.get(family.fs_family_id)
if target is None:
target = GenealogyFamily(
fs_family_id=family.fs_family_id,
husband_id=husband_id,
wife_id=wife_id,
marriage_date=family.marriage_date,
marriage_date_raw=family.marriage_date_raw,
marriage_place=family.marriage_place,
)
session.add(target)
await session.flush()
families_by_fs_id[family.fs_family_id] = target
new_families += 1
elif _apply_family_updates(existing=target, family=family, husband_id=husband_id, wife_id=wife_id):
updated_families += 1
family_children += await _replace_family_children(
session=session,
family=family,
family_id=target.id,
pointer_to_fs_id=pointer_to_fs_id,
people=people_by_fs_id,
)
skipped = len(parsed.families) - len(families_with_fs_id)
return families_by_fs_id, new_families, updated_families, skipped, family_children
def _apply_family_updates(
*,
existing: GenealogyFamily,
family: ParsedFamily,
husband_id: UUID | None,
wife_id: UUID | None,
) -> bool:
changed = False
fields = (
("husband_id", husband_id),
("wife_id", wife_id),
("marriage_date", family.marriage_date),
("marriage_date_raw", family.marriage_date_raw),
("marriage_place", family.marriage_place),
)
for name, value in fields:
if getattr(existing, name) != value:
setattr(existing, name, value)
changed = True
return changed
async def _replace_family_children(
*,
session: AsyncSession,
family: ParsedFamily,
family_id: UUID,
pointer_to_fs_id: Mapping[str, str],
people: dict[str, GenealogyPerson],
) -> int:
await session.exec(delete(GenealogyFamilyChild).where(col(GenealogyFamilyChild.family_id) == family_id))
child_rows = _family_child_rows(
family=family,
family_id=family_id,
pointer_to_fs_id=pointer_to_fs_id,
people=people,
)
for child_row in child_rows:
session.add(child_row)
return len(child_rows)
def _person_id_from_pointer(
*,
pointer: str | None,
pointer_to_fs_id: Mapping[str, str],
people: dict[str, GenealogyPerson],
) -> UUID | None:
if pointer is None:
return None
fs_id = pointer_to_fs_id.get(pointer)
if fs_id is None:
return None
person = people.get(fs_id)
return person.id if person is not None else None
def _family_child_rows(
*,
family: ParsedFamily,
family_id: UUID,
pointer_to_fs_id: Mapping[str, str],
people: dict[str, GenealogyPerson],
) -> list[GenealogyFamilyChild]:
rows: list[GenealogyFamilyChild] = []
seen_child_ids: set[UUID] = set()
for child in family.children:
child_id = _person_id_from_pointer(
pointer=child.child_pointer,
pointer_to_fs_id=pointer_to_fs_id,
people=people,
)
if child_id is None or child_id in seen_child_ids:
continue
seen_child_ids.add(child_id)
rows.append(
GenealogyFamilyChild(
id=uuid4(),
family_id=family_id,
child_id=child_id,
relationship_type=child.relationship_type,
)
)
return rows
def _citation_rows(
*,
parsed: ParsedGedcom,
pointer_to_fs_id: Mapping[str, str],
people: dict[str, GenealogyPerson],
families: dict[str, GenealogyFamily],
) -> list[GenealogyCitation]:
rows: list[GenealogyCitation] = []
seen: set[tuple[UUID | None, UUID | None, str, str]] = set()
for person in parsed.people:
fs_id = pointer_to_fs_id.get(person.pointer)
if fs_id is None:
continue
person_row = people.get(fs_id)
if person_row is None:
continue
for citation in person.citations:
key = (person_row.id, None, citation.fact_type.value, citation.raw_citation_text)
if key in seen:
continue
seen.add(key)
rows.append(
GenealogyCitation(
id=uuid4(),
genealogy_person_id=person_row.id,
genealogy_family_id=None,
fact_type=citation.fact_type,
raw_citation_text=citation.raw_citation_text,
source_kind=GenealogyCitationSourceKind.FAMILYSEARCH_IMPORTED,
document_id=None,
)
)
for family in parsed.families:
if family.fs_family_id is None:
continue
family_row = families.get(family.fs_family_id)
if family_row is None:
continue
for citation in family.citations:
key = (None, family_row.id, citation.fact_type.value, citation.raw_citation_text)
if key in seen:
continue
seen.add(key)
rows.append(
GenealogyCitation(
id=uuid4(),
genealogy_person_id=None,
genealogy_family_id=family_row.id,
fact_type=citation.fact_type,
raw_citation_text=citation.raw_citation_text,
source_kind=GenealogyCitationSourceKind.FAMILYSEARCH_IMPORTED,
document_id=None,
)
)
return rows
async def _replace_imported_citations(
*,
session: AsyncSession,
person_ids: set[UUID],
family_ids: set[UUID],
citations: list[GenealogyCitation],
) -> None:
where_clauses = []
if person_ids:
where_clauses.append(col(GenealogyCitation.genealogy_person_id).in_(person_ids))
if family_ids:
where_clauses.append(col(GenealogyCitation.genealogy_family_id).in_(family_ids))
if where_clauses:
target_scope = where_clauses[0] if len(where_clauses) == 1 else or_(*where_clauses)
await session.exec(
delete(GenealogyCitation).where(
col(GenealogyCitation.source_kind) == GenealogyCitationSourceKind.FAMILYSEARCH_IMPORTED,
target_scope,
)
)
for citation in citations:
session.add(citation)
def _parse_person(element: Element) -> ParsedPerson:
birth_event = _first_child(element, "BIRT")
death_event = _first_child(element, "DEAT")
birth_date_raw = _child_value(birth_event, "DATE")
death_date_raw = _child_value(death_event, "DATE")
return ParsedPerson(
pointer=element.get_pointer() or "",
fs_id=_extract_fs_identifier(element),
full_name=_person_name(element),
birth_date=_parse_exact_date(birth_date_raw),
birth_date_raw=birth_date_raw,
birth_place=_child_value(birth_event, "PLAC"),
death_date=_parse_exact_date(death_date_raw),
death_date_raw=death_date_raw,
death_place=_child_value(death_event, "PLAC"),
citations=(
*_fact_citations(fact_element=birth_event, fact_type=GenealogyCitationFactType.BIRTH),
*_fact_citations(fact_element=death_event, fact_type=GenealogyCitationFactType.DEATH),
),
)
def _parse_family(element: Element) -> ParsedFamily:
marriage_event = _first_child(element, "MARR")
marriage_date_raw = _child_value(marriage_event, "DATE")
children = tuple(
ParsedFamilyChild(
child_pointer=(child.get_value() or "").strip(),
relationship_type=_child_value(child, "PEDI"),
)
for child in _children(element, "CHIL")
if (child.get_value() or "").strip()
)
return ParsedFamily(
fs_family_id=_extract_fs_identifier(element),
husband_pointer=(_child_value(element, "HUSB") or "").strip() or None,
wife_pointer=(_child_value(element, "WIFE") or "").strip() or None,
marriage_date=_parse_exact_date(marriage_date_raw),
marriage_date_raw=marriage_date_raw,
marriage_place=_child_value(marriage_event, "PLAC"),
children=children,
citations=_fact_citations(fact_element=marriage_event, fact_type=GenealogyCitationFactType.MARRIAGE),
)
def _fact_citations(
*, fact_element: Element | None, fact_type: GenealogyCitationFactType
) -> tuple[ParsedCitation, ...]:
if fact_element is None:
return ()
citations: list[ParsedCitation] = []
for source in _children(fact_element, "SOUR"):
raw_citation = _flatten_tag_values(source)
if raw_citation is None:
continue
citations.append(ParsedCitation(fact_type=fact_type, raw_citation_text=raw_citation))
return tuple(citations)
def _extract_fs_identifier(element: Element) -> str | None:
for tag in ("_FSFTID", "FSFTID"):
value = _child_value(element, tag)
if value:
return value
for refn in _children(element, "REFN"):
refn_value = (refn.get_value() or "").strip()
refn_type = (_child_value(refn, "TYPE") or "").strip().casefold()
if refn_value and ("fsftid" in refn_type or "familysearch" in refn_type):
return refn_value
return None
def _person_name(element: Element) -> str:
raw_name = _child_value(element, "NAME")
if raw_name is None:
return "Unknown"
cleaned = raw_name.replace("/", " ").strip()
return " ".join(part for part in cleaned.split() if part) or "Unknown"
def _parse_exact_date(raw: str | None) -> date | None:
if raw is None:
return None
tokens = [token for token in raw.strip().upper().split() if token]
if len(tokens) != 3:
return None
day_token, month_token, year_token = tokens
if month_token not in _MONTHS:
return None
try:
return date(year=int(year_token), month=_MONTHS[month_token], day=int(day_token))
except ValueError:
return None
def _first_child(element: Element | None, tag: str) -> Element | None:
if element is None:
return None
for child in element.get_child_elements():
if child.get_tag() == tag:
return child
return None
def _children(element: Element | None, tag: str) -> list[Element]:
if element is None:
return []
return [child for child in element.get_child_elements() if child.get_tag() == tag]
def _child_value(element: Element | None, tag: str) -> str | None:
child = _first_child(element, tag)
if child is None:
return None
value = (child.get_value() or "").strip()
return value or None
def _flatten_tag_values(element: Element) -> str | None:
lines: list[str] = []
def walk(node: Element) -> None:
value = (node.get_value() or "").strip()
if value:
lines.append(f"{node.get_tag()}: {value}")
for child in node.get_child_elements():
walk(child)
walk(element)
return " | ".join(lines) if lines else None
src/transcription/services/jobs.py
-442
View File
@@ -1,442 +0,0 @@
import logging
from collections.abc import Sequence
from datetime import UTC
from datetime import datetime
from uuid import UUID
from sqlalchemy import func
from sqlalchemy import update
from sqlmodel import col
from sqlmodel import select
from sqlmodel.ext.asyncio.session import AsyncSession
from ..db.loading import orm_attribute
from ..db.loading import selectinload
from ..db.models import Document
from ..db.models import ExecutionAttempt
from ..db.models import Job
from ..db.models import JobSource
from ..db.models import JobSourceStatus
from ..db.models import JobStatus
from ..db.models import Source
from ..errors import AppError
from ..errors import ErrorCategory
from .base import ServiceBase
logger = logging.getLogger(__name__)
def _utc_now_naive() -> datetime:
"""Return current UTC as naive datetime for DB timestamp columns."""
return datetime.now(UTC).replace(tzinfo=None)
def _as_naive_utc(value: datetime) -> datetime:
"""Normalize datetimes to naive UTC for DB comparisons/binds."""
if value.tzinfo is None:
return value
return value.astimezone(UTC).replace(tzinfo=None)
class JobDeleteBlockedError(AppError):
"""Raised when a job delete operation is blocked by lifecycle policy."""
class JobCancelBlockedError(AppError):
"""Raised when a job cancel operation is blocked by lifecycle policy."""
class JobResubmitBlockedError(AppError):
"""Raised when a job resubmit operation is blocked by lifecycle policy."""
class JobNotFoundError(AppError):
"""Raised when a requested Job does not exist."""
class JobService(ServiceBase):
"""Thin service class for managing jobs in the database."""
#
# CRUD Operations
#
async def create_job(self, job: Job, session: AsyncSession | None = None) -> Job:
"""Create a new job in the database."""
async with self._session_scope(session) as _session:
_session.add(job)
await self._finalize(session=_session, caller_session=session, refresh=(job,))
return job
async def read_job(self, job_id: UUID, session: AsyncSession | None = None) -> Job:
"""Read an existing job from the database.
The related document is always eagerly loaded so callers can safely
access ``job.document`` in async contexts without triggering lazy-load IO.
"""
async with self._session_scope(session) as _session:
query = (
select(Job)
.options(
selectinload(Job.document).selectinload(orm_attribute(Document.sources)),
selectinload(Job.job_sources).selectinload(orm_attribute(JobSource.source)),
)
.where(Job.id == job_id)
.execution_options(populate_existing=True)
)
job = (await _session.exec(query)).first()
if job is None:
raise self._not_found(job_id)
return job
async def update_job(self, job: Job, session: AsyncSession | None = None) -> Job:
"""Update an existing job in the database."""
async with self._session_scope(session) as _session:
merged = await _session.merge(job)
await self._finalize(session=_session, caller_session=session, refresh=(merged,))
return merged
async def delete_job(self, job: Job, session: AsyncSession | None = None) -> None:
"""Delete a job from the database."""
async with self._session_scope(session) as _session:
await _session.delete(job)
await self._finalize(session=_session, caller_session=session)
# Query Operations
async def query_jobs(
self,
*,
status: JobStatus | None = None,
filename: str | None = None,
session: AsyncSession | None = None,
) -> Sequence[Job]:
"""Query jobs from the database based on provided filters."""
async with self._session_scope(session) as _session:
query = select(Job).options(
selectinload(Job.document),
selectinload(Job.job_sources).selectinload(orm_attribute(JobSource.source)),
)
if status is not None:
query = query.where(Job.status == status)
if filename is not None:
query = query.where(
col(Job.job_sources).any(col(JobSource.source).has(col(Source.filename) == filename))
)
result = await _session.exec(query)
return result.all()
async def list_jobs(
self,
*,
session: AsyncSession | None = None,
) -> Sequence[Job]:
"""List all jobs in the database with eagerly loaded documents."""
async with self._session_scope(session) as _session:
query = select(Job).options(
selectinload(Job.document),
selectinload(Job.job_sources).selectinload(orm_attribute(JobSource.source)),
)
result = await _session.exec(query)
return result.all()
# Other Operations
async def mark_job_status(
self,
job_id: UUID,
status: JobStatus,
session: AsyncSession | None = None,
) -> Job:
"""Mark a job with a new status."""
return await self.update_job_state(job_id=job_id, status=status, session=session)
async def update_job_state(
self,
*,
job_id: UUID,
status: JobStatus,
retry_count_increment: int = 0,
session: AsyncSession | None = None,
) -> Job:
"""Update a job's lifecycle fields.
When ``session`` is provided, this method flushes so callers can commit
once at an orchestration boundary.
"""
async with self._session_scope(session) as _session:
query = (
select(Job)
.options(
selectinload(Job.document),
selectinload(Job.job_sources).selectinload(orm_attribute(JobSource.source)),
)
.where(Job.id == job_id)
.execution_options(populate_existing=True)
)
job = (await _session.exec(query)).first()
if job is None:
raise self._not_found(job_id)
job.status = status
if retry_count_increment:
job.retry_count += retry_count_increment
await self._finalize(session=_session, caller_session=session, refresh=(job,))
return job
async def note_processing_progress(self, *, job_id: UUID, session: AsyncSession | None = None) -> Job:
"""Refresh job liveness while a multi-page batch is still in progress."""
async with self._session_scope(session) as _session:
job = await _session.get(Job, job_id)
if job is None:
raise self._not_found(job_id)
job.date_updated = _utc_now_naive()
await self._finalize(session=_session, caller_session=session, refresh=(job,))
return job
async def claim_next_queued_job(
self,
*,
session: AsyncSession | None = None,
) -> Job | None:
"""Atomically claim the oldest queued job by transitioning it to PROCESSING.
The selection is deliberately unadorned: no eager loads are applied to the
hot poll, because callers re-read the claimed job with the relationships
they actually need. On PostgreSQL the row is locked with ``SKIP LOCKED`` so
concurrent workers never contend for the same job.
"""
async with self._session_scope(session) as _session:
dialect = _session.get_bind().dialect.name
if dialect == "postgresql":
query = (
select(Job)
.where(Job.status == JobStatus.QUEUED)
# Break ties by id so "next" is stable when two rows share close timestamps.
.order_by(col(Job.date_created), col(Job.id))
.limit(1)
.with_for_update(skip_locked=True)
)
job = (await _session.exec(query)).first()
if job is None:
return None
job.status = JobStatus.PROCESSING
await self._finalize(session=_session, caller_session=session, refresh=(job,))
return job
now = _utc_now_naive()
queued_job_id = (
select(col(Job.id))
.where(col(Job.status) == JobStatus.QUEUED)
.order_by(col(Job.date_created), col(Job.id))
.limit(1)
.scalar_subquery()
)
claim_statement = (
update(Job)
.where(col(Job.id) == queued_job_id)
.where(col(Job.status) == JobStatus.QUEUED)
.values(status=JobStatus.PROCESSING, date_updated=now)
.returning(col(Job.id))
)
claimed_row = (await _session.exec(claim_statement)).first()
if claimed_row is None:
return None
claimed_job_id = claimed_row if isinstance(claimed_row, UUID) else claimed_row[0]
job = (await _session.exec(select(Job).where(Job.id == claimed_job_id))).first()
if job is None:
return None
await self._finalize(session=_session, caller_session=session, refresh=(job,))
return job
async def requeue_stale_processing_jobs(
self,
*,
stale_before: datetime,
session: AsyncSession | None = None,
) -> int:
"""Move stale processing jobs back to queued state.
Jobs with ``status=PROCESSING`` and ``date_updated`` older than
``stale_before`` are considered stale and re-queued.
"""
async with self._session_scope(session) as _session:
normalized_stale_before = _as_naive_utc(stale_before)
query = (
select(Job).where(Job.status == JobStatus.PROCESSING).where(Job.date_updated < normalized_stale_before)
)
stale_jobs = (await _session.exec(query)).all()
if not stale_jobs:
return 0
now = _utc_now_naive()
for job in stale_jobs:
job.status = JobStatus.QUEUED
job.date_updated = now
await self._finalize(session=_session, caller_session=session, refresh=stale_jobs)
return len(stale_jobs)
async def delete_job_with_guardrails(self, *, job_id: UUID, session: AsyncSession | None = None) -> None:
"""Delete a job with lifecycle guardrails and dependent cleanup policy.
Policy:
- Block when the job is actively processing.
- Otherwise remove related JobSource rows, then delete the job.
"""
async with self._session_scope(session) as _session:
query = (
select(Job)
.options(selectinload(Job.job_sources))
.where(Job.id == job_id)
.execution_options(populate_existing=True)
)
job = (await _session.exec(query)).first()
if job is None:
raise self._not_found(job_id)
if job.status == JobStatus.PROCESSING:
raise JobDeleteBlockedError(
"Job delete blocked while status is processing",
category=ErrorCategory.VALIDATION,
suggestion="Wait for processing to complete, or move the job out of processing before deleting.",
)
attempt_count = (
await _session.exec(
select(func.count()).select_from(ExecutionAttempt).where(ExecutionAttempt.job_id == job_id)
)
).one()
if attempt_count:
raise JobDeleteBlockedError(
"Job delete blocked because immutable execution evidence exists",
category=ErrorCategory.CONFLICT,
suggestion=(
"Retain the Job as processing history. Evidence deletion requires "
"an explicit retention workflow."
),
)
for job_source in list(job.job_sources):
await _session.delete(job_source)
await _session.delete(job)
await self._finalize(session=_session, caller_session=session)
async def delete_job_and_evidence(self, *, job_id: UUID) -> None:
"""Explicitly delete a terminal job and all evidence owned by its attempts."""
async with self._session_scope() as session:
job = (
await session.exec(
select(Job)
.options(selectinload(Job.job_sources))
.where(Job.id == job_id)
.execution_options(populate_existing=True)
)
).first()
if job is None:
raise self._not_found(job_id)
if job.status == JobStatus.PROCESSING:
raise JobDeleteBlockedError(
"Job delete blocked while status is processing",
category=ErrorCategory.VALIDATION,
suggestion="Wait for processing to complete, or cancel it before deleting evidence.",
)
attempts = list(
(await session.exec(select(ExecutionAttempt).where(ExecutionAttempt.job_id == job_id))).all()
)
for attempt in attempts:
await session.delete(attempt)
await session.flush()
for job_source in list(job.job_sources):
await session.delete(job_source)
await session.flush()
await session.delete(job)
await self._finalize(session=session, caller_session=None)
async def cancel_job(self, *, job_id: UUID, session: AsyncSession | None = None) -> Job:
"""Cancel a queued/processing job and stop remaining source work."""
async with self._session_scope(session) as _session:
query = (
select(Job)
.options(
selectinload(Job.job_sources).selectinload(orm_attribute(JobSource.source)),
)
.where(Job.id == job_id)
.execution_options(populate_existing=True)
)
job = (await _session.exec(query)).first()
if job is None:
raise self._not_found(job_id)
if job.status == JobStatus.TRANSCRIBED:
raise JobCancelBlockedError(
"Job cancel is not allowed for transcribed jobs",
category=ErrorCategory.VALIDATION,
suggestion="Use resubmit for reprocessing needs, or leave the terminal job unchanged.",
)
now = _utc_now_naive()
job.status = JobStatus.FAILED
job.date_updated = now
for job_source in job.job_sources:
if job_source.status == JobSourceStatus.TRANSCRIBED:
continue
job_source.status = JobSourceStatus.CANCELLED
await self._finalize(session=_session, caller_session=session, refresh=(job,))
return job
async def resubmit_failed_sources(self, *, job_id: UUID, session: AsyncSession | None = None) -> int:
"""Reset failed source executions and queue the job for reprocessing."""
async with self._session_scope(session) as _session:
query = (
select(Job)
.options(
selectinload(Job.job_sources).selectinload(orm_attribute(JobSource.source)),
)
.where(Job.id == job_id)
.execution_options(populate_existing=True)
)
job = (await _session.exec(query)).first()
if job is None:
raise self._not_found(job_id)
if job.status == JobStatus.PROCESSING:
raise JobResubmitBlockedError(
"Job resubmit is blocked while processing is active",
category=ErrorCategory.VALIDATION,
suggestion="Cancel processing first, then resubmit remaining sources.",
)
# Cancelled pages are re-attemptable: older flows wrote FAILED,
# so resubmit already reset them. Excluding CANCELLED here would make
# cancelled work permanently unrecoverable.
resubmittable = {JobSourceStatus.FAILED, JobSourceStatus.CANCELLED}
candidates = [job_source for job_source in job.job_sources if job_source.status in resubmittable]
if not candidates:
raise JobResubmitBlockedError(
"Job has no failed or cancelled sources to resubmit",
category=ErrorCategory.VALIDATION,
suggestion="Only failed or cancelled sources can be resubmitted.",
)
now = _utc_now_naive()
for job_source in candidates:
job_source.status = JobSourceStatus.PENDING
job.status = JobStatus.QUEUED
job.date_updated = now
await self._finalize(session=_session, caller_session=session, refresh=(job,))
return len(candidates)
@staticmethod
def _not_found(job_id: UUID) -> JobNotFoundError:
return JobNotFoundError(
f"Job with id {job_id} not found",
category=ErrorCategory.NOT_FOUND,
suggestion="Verify the Job id and retry.",
)

Some files were not shown because too many files have changed in this diff Show More