From 065acad125cb46d98d002fe62d4844d55c50ecb7 Mon Sep 17 00:00:00 2001 From: Jim Lancaster <40281233+zoltan57@users.noreply.github.com> Date: Wed, 2 Sep 2026 17:02:48 -0500 Subject: [PATCH] Scrub references to older versions --- .../instructions/providers.instructions.md | 4 +- .github/skills/python-code-reviewer/skill.md | 2 +- .github/workflows/quality-gate.yml | 4 +- .pre-commit-config.yaml | 2 +- AGENTS.md | 2 +- docs/cloudflare_tunnel_access.md | 2 +- docs/index.md | 6 +- docs/production-runbook.md | 2 +- docs/ui/pages/documents.md | 2 +- docs/ui/pages/sources.md | 2 +- tests/integration/test_pipeline_flow.py | 2 +- ...didates.py => test_candidate_promotion.py} | 4 +- tests/services/test_quality.py | 4 +- ...vidence.py => test_evidence_provenance.py} | 2 +- tests/test_meta_contract_guards.py | 47 +++ tests/test_models.py | 2 +- tests/ui/conftest.py | 2 +- tests/ui/test_sources_page.py | 2 +- tools/migrate_v46_to_v47.py | 337 ------------------ tools/migrate_v47_to_v48.py | 185 ---------- 20 files changed, 70 insertions(+), 545 deletions(-) rename tests/services/{test_v45_candidates.py => test_candidate_promotion.py} (99%) rename tests/{test_v42_evidence.py => test_evidence_provenance.py} (99%) delete mode 100644 tools/migrate_v46_to_v47.py delete mode 100644 tools/migrate_v47_to_v48.py diff --git a/.github/instructions/providers.instructions.md b/.github/instructions/providers.instructions.md index 02d4b70..1816446 100644 --- a/.github/instructions/providers.instructions.md +++ b/.github/instructions/providers.instructions.md @@ -50,7 +50,7 @@ wins if this file drifts from it. post-parse `model_dump()` and call the result transport evidence. - Reset per-call capture state at the start of every call. Without it, a connection failure can attach the *previous* call's response as evidence for this one. Guarded by - `tests/test_v42_evidence.py::test_openrouter_does_not_reuse_prior_response_on_connection_failure`. + `tests/test_evidence_provenance.py::test_openrouter_does_not_reuse_prior_response_on_connection_failure`. - Keep transport capture scoped to the call, not the adapter instance. Concurrent `transcribe()` calls on one adapter must not be able to overwrite each other's response evidence. - Handle the streamed-body case (`httpx.ResponseNotRead`) rather than assuming `response.content` @@ -118,5 +118,5 @@ If capture behavior, evidence schema, or the header allowlist changes: 1. Update `docs/invariant/ai_evidence_and_provenance.md` only if the durable preservation contract itself is changing — that revision is deliberate and reviewed, not incidental. 2. Update `docs/schema.md` when persisted evidence fields change. -3. Update or add tests in the same change (`tests/providers/`, `tests/test_v42_evidence.py`). +3. Update or add tests in the same change (`tests/providers/`, `tests/test_evidence_provenance.py`). 4. Bump the affected evidence `schema_version` when a field's meaning changes. diff --git a/.github/skills/python-code-reviewer/skill.md b/.github/skills/python-code-reviewer/skill.md index 02f857c..9fb3684 100644 --- a/.github/skills/python-code-reviewer/skill.md +++ b/.github/skills/python-code-reviewer/skill.md @@ -71,7 +71,7 @@ Where **Enforced by** reads *unenforced*, recommending a deterministic test is i | 1 | **Service boundary rule:** no service-to-service imports | `tests/test_service_boundaries.py` | | 2 | **UI boundary rule:** pages/components do not perform persistence access | `tests/test_ui_boundaries.py` | | 3 | **Status vocabulary conformance:** `JobStatus`/`JobSourceStatus`/`JobPurpose` usage matches current enums in `src/transcription/db/models.py`; no stringly-typed status literals | `tests/test_model_contract_guards.py` | -| 4 | **Evidence ownership conformance:** append-only attempt history is preserved and projection writes are not mistaken for history mutation (`src/transcription/services/sources.py`, `src/transcription/services/evidence.py`) | `tests/test_v42_evidence.py::test_attempts_are_append_only_and_exported_with_integrity` | +| 4 | **Evidence ownership conformance:** append-only attempt history is preserved and projection writes are not mistaken for history mutation (`src/transcription/services/sources.py`, `src/transcription/services/evidence.py`) | `tests/test_evidence_provenance.py::test_attempts_are_append_only_and_exported_with_integrity` | | 5 | **Canonical authority:** findings must resolve against `docs/*` first | `tests/test_meta_contract_guards.py::test_canonical_authority_references_are_present` | | 6 | **Schema contract fidelity:** when model/persistence behavior changes, `docs/schema.md` remains field-accurate with `src/transcription/db/models.py` | `tests/test_model_contract_guards.py` (field names, ordering, enum members, table coverage), `tests/test_meta_contract_guards.py` (presence and references) | | 7 | **Media boundary conformance:** print/export media is record-validated and UI media URL generation uses controlled resolver paths | `tests/test_media_path_safety.py`, `tests/ui/test_media_urls.py` | diff --git a/.github/workflows/quality-gate.yml b/.github/workflows/quality-gate.yml index 639112e..27243dc 100644 --- a/.github/workflows/quality-gate.yml +++ b/.github/workflows/quality-gate.yml @@ -1,7 +1,7 @@ name: Quality Gate -# V4.7 Phase 6 / review log [40]. Before this, ruff, ty and pytest were enforced -# only by .pre-commit-config.yaml, and only for developers who had actually run +# Repository quality gate. Before this workflow existed, ruff, ty, and pytest were +# enforced only by .pre-commit-config.yaml for developers who had run # `pre-commit install`. on: diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 6545baa..5c91e1f 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,4 +1,4 @@ -# Quality gate for V4.6 [HIGH-06]. `ruff check`, `ruff format --check`, and `ty check` +# Quality gate: `ruff check`, `ruff format --check`, and `ty check` # are blocking once known `ty` false positives are suppressed inline with rationale. # # Both tools are uv-managed dev dependencies and are not on PATH, so each entry must diff --git a/AGENTS.md b/AGENTS.md index c3d08a4..a51b07d 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -71,7 +71,7 @@ These are not conventions — a test fails if you break them: - **No persistence access from pages/components** (`test_ui_boundaries.py`, allowlist-based). - **No hand-rolled error notifications in UI** — use the shared error presenter. - **No stringly-typed status literals** — use the enums (`test_model_contract_guards.py`). -- **Attempt history is append-only** (`test_v42_evidence.py`). +- **Attempt history is append-only** (`test_evidence_provenance.py`). - **`docs/schema.md` stays field-accurate** with `db/models.py`. - **Orphans are tracked, not tolerated** — `test_orphan_sweep.py` records each retained orphan with rationale in `KNOWN_ORPHANS`. diff --git a/docs/cloudflare_tunnel_access.md b/docs/cloudflare_tunnel_access.md index 654265f..328ff40 100644 --- a/docs/cloudflare_tunnel_access.md +++ b/docs/cloudflare_tunnel_access.md @@ -1,4 +1,4 @@ -# Cloudflare Tunnel and Access Setup (V6.0 Phase 3) +# Cloudflare Tunnel and Access Setup This guide defines the repository-supported setup for exposing app and selected LAN services through Cloudflare Tunnel with Cloudflare Access protection. diff --git a/docs/index.md b/docs/index.md index 47fc89e..e0752be 100644 --- a/docs/index.md +++ b/docs/index.md @@ -26,9 +26,9 @@ This directory is the single source of truth for current V6.1 behavior and archi ## Baseline Statement -The current V6.1 baseline includes behavior delivered through the architectural cleanup phases and -person-schema redesign (through V5.1), the V6.0 hosting migration to a containerized PostgreSQL -deployment, and the V6.1 navigation, Document Detail, and worker-backed maintenance refinements. +The current V6.1 baseline includes the architectural cleanup, person-schema redesign, +containerized PostgreSQL deployment, and the navigation, Document Detail, and worker-backed +maintenance refinements reflected across this canonical document set. Use this `docs/*` canonical set for active design and implementation decisions. Every canonical document above states this same baseline; `tests/test_meta_contract_guards.py` diff --git a/docs/production-runbook.md b/docs/production-runbook.md index a77afc1..2fbc2fa 100644 --- a/docs/production-runbook.md +++ b/docs/production-runbook.md @@ -20,7 +20,7 @@ This runbook is the operational checklist for releasing and monitoring the trans ## 2. Release execution steps 1. Deploy artifact/config to target environment. - - V6.0 Phase 1 production stack: `docker compose -f docker-compose.production.yml up -d --build` + - Production stack: `docker compose -f docker-compose.production.yml up -d --build` - `Settings` loads from explicit `_env_file`, then `ENV_FILE`, then the repository-root `.env.production`; it does not resolve relative to the process working directory. - For Runtime Settings writes in production, mount `.env.production` into the app container and set `RUNTIME_SETTINGS_ENV_FILE=/app/.env.production`. - If deployment uses a non-default env-file location, set both `ENV_FILE` and `RUNTIME_SETTINGS_ENV_FILE` to that absolute path so startup reads and Settings-page writes stay aligned. diff --git a/docs/ui/pages/documents.md b/docs/ui/pages/documents.md index f35b5b2..97a46c4 100644 --- a/docs/ui/pages/documents.md +++ b/docs/ui/pages/documents.md @@ -146,5 +146,5 @@ Rules: ## Known Limitations and Deferred Work -- Source page ordering remains read-only in V4.4. +- Source page ordering remains read-only. - Printing other entities, batch printing, and server-side export formats are deferred. diff --git a/docs/ui/pages/sources.md b/docs/ui/pages/sources.md index d22d4ef..e1d033c 100644 --- a/docs/ui/pages/sources.md +++ b/docs/ui/pages/sources.md @@ -96,4 +96,4 @@ The list accepts optional `document_id` and `job_id` query parameters. Document ## Planned Changes -- Source page reordering is deferred beyond V4.3 and may be reconsidered if a demonstrated workflow need emerges. +- Source page reordering remains deferred unless a demonstrated workflow need emerges. diff --git a/tests/integration/test_pipeline_flow.py b/tests/integration/test_pipeline_flow.py index a4a7f9a..f148f8b 100644 --- a/tests/integration/test_pipeline_flow.py +++ b/tests/integration/test_pipeline_flow.py @@ -24,7 +24,7 @@ from transcription.services.workflows import advance_job async def _attempts_for_job(session, job) -> list[ExecutionAttempt]: - """Load execution attempts for a job; V4.7 moved evidence off JobSource.""" + """Load execution attempts for a job; evidence lives on ExecutionAttempt.""" job_source_ids = [job_source.id for job_source in job.job_sources] result = await session.exec(select(ExecutionAttempt).where(col(ExecutionAttempt.job_source_id).in_(job_source_ids))) return list(result.all()) diff --git a/tests/services/test_v45_candidates.py b/tests/services/test_candidate_promotion.py similarity index 99% rename from tests/services/test_v45_candidates.py rename to tests/services/test_candidate_promotion.py index 372dba0..7694d78 100644 --- a/tests/services/test_v45_candidates.py +++ b/tests/services/test_candidate_promotion.py @@ -1,4 +1,4 @@ -"""V4.5 retranscription candidate and promotion tests.""" +"""Retranscription candidate and promotion tests.""" from uuid import uuid4 @@ -20,7 +20,7 @@ def _services(default_session_factory, settings: Settings) -> ServiceBundle: async def _seed_source(services: ServiceBundle) -> Source: - document = await services.documents.create_document(Document(name="V4.5 source")) + document = await services.documents.create_document(Document(name="candidate source")) source = Source( document_id=document.id, page_number=1, diff --git a/tests/services/test_quality.py b/tests/services/test_quality.py index 89035e6..d31693f 100644 --- a/tests/services/test_quality.py +++ b/tests/services/test_quality.py @@ -1,4 +1,4 @@ -"""Tests for deterministic V4.5 transcription warnings.""" +"""Tests for deterministic transcription warnings.""" import pytest @@ -8,7 +8,7 @@ from transcription.services.quality import quality_warning_payload @pytest.mark.unit -def test_quality_analysis_reports_each_v45_warning_without_mutating_text(): +def test_quality_analysis_reports_each_quality_warning_without_mutating_text(): text = ( "[document body handwritten]\n" "[document body typewritten]\n" diff --git a/tests/test_v42_evidence.py b/tests/test_evidence_provenance.py similarity index 99% rename from tests/test_v42_evidence.py rename to tests/test_evidence_provenance.py index 5c3f35a..78b1b20 100644 --- a/tests/test_v42_evidence.py +++ b/tests/test_evidence_provenance.py @@ -1,4 +1,4 @@ -"""Focused V4.2 evidence, integrity, and benchmark tests.""" +"""Focused evidence, integrity, and benchmark tests.""" from __future__ import annotations diff --git a/tests/test_meta_contract_guards.py b/tests/test_meta_contract_guards.py index c3a4239..a7dd31d 100644 --- a/tests/test_meta_contract_guards.py +++ b/tests/test_meta_contract_guards.py @@ -69,6 +69,19 @@ _BASELINE_CLAIM = re.compile( r"\b(?:Current Baseline:\s*|current\s+|active\s+|canonical\s+)V(\d+(?:\.\d+)?)", re.IGNORECASE, ) +_LEGACY_VERSION_REFERENCE = re.compile(r"\b[Vv](4(?:\.\d+)?|5(?:\.\d+)?|6\.0)\b|test_v42_evidence|test_v45_candidates") +LEGACY_VERSION_REFERENCE_EXCLUSIONS = frozenset( + { + ".github/workflows/quality-gate.yml", + "docs/data_migration.md", + "docs/requirements.md", + "docs/reviews", + "docs/roadmap_plan.md", + "docs/schema.md", + "src/transcription/db/operations.py", + "tests/test_meta_contract_guards.py", + } +) def _declared_baseline() -> str: @@ -112,6 +125,40 @@ def test_canonical_docs_declare_one_consistent_baseline(): ) +def test_legacy_version_markers_are_scoped_to_allowed_historical_files(): + """Version labels from older project baselines should not linger in active files. + + Historical references are allowed only where the old version is the subject matter: + roadmap planning, dated reviews, stable requirement IDs, and explicit migration or + compatibility notes. + """ + scanned: list[Path] = [] + for root in ( + PROJECT_ROOT / ".github", + PROJECT_ROOT / "docs", + PROJECT_ROOT / "src", + PROJECT_ROOT / "tests", + ): + scanned.extend(path for path in root.rglob("*") if path.suffix in {".md", ".py", ".yml", ".yaml"}) + scanned.extend((PROJECT_ROOT / "AGENTS.md", PROJECT_ROOT / ".pre-commit-config.yaml")) + + violations: dict[str, list[str]] = {} + for path in sorted(set(scanned)): + relative = path.relative_to(PROJECT_ROOT).as_posix() + if any( + relative == excluded or relative.startswith(f"{excluded}/") + for excluded in LEGACY_VERSION_REFERENCE_EXCLUSIONS + ): + continue + matches = sorted( + {match.group(0) for match in _LEGACY_VERSION_REFERENCE.finditer(path.read_text(encoding="utf-8"))} + ) + if matches: + violations[relative] = matches + + assert violations == {}, f"Legacy version markers remain in active files: {violations}" + + def test_active_contract_files_are_present(): """Guard the guard: ensure all expected authority files are scanned.""" missing = [path for path in ACTIVE_CONTRACT_FILES if not (PROJECT_ROOT / path).exists()] diff --git a/tests/test_models.py b/tests/test_models.py index 07a4491..cc6f65c 100644 --- a/tests/test_models.py +++ b/tests/test_models.py @@ -232,7 +232,7 @@ class TestPersonAndDocumentPersonModel: class TestJobSourceModel: def test_job_source_is_a_queue_row_not_an_evidence_row(self, session): - """V4.7: job_source carries only queue state; evidence lives on execution_attempt.""" + """job_source carries only queue state; evidence lives on execution_attempt.""" document = _persist_document(session) job = _persist_job(session, document) source = _persist_source(session, document) diff --git a/tests/ui/conftest.py b/tests/ui/conftest.py index ac2d4d9..df4fe7c 100644 --- a/tests/ui/conftest.py +++ b/tests/ui/conftest.py @@ -142,7 +142,7 @@ async def seed_job(app_client: tuple[FastAPI, TestClient]) -> Callable[..., Awai session.add(job_source) await session.flush() - # V4.7: evidence lives on execution_attempt, not job_source. + # Evidence lives on execution_attempt, not job_source. executed_at = datetime.now(UTC) session.add( ExecutionAttempt( diff --git a/tests/ui/test_sources_page.py b/tests/ui/test_sources_page.py index c69f028..04d4e1b 100644 --- a/tests/ui/test_sources_page.py +++ b/tests/ui/test_sources_page.py @@ -357,7 +357,7 @@ class TestSourcesPageRendering: await service.update_job_source_transcription( job_id=job_id, source_id=source_id, - text="V4.2 transcription", + text="Machine transcription", raw_api_response={"id": "sdk-snapshot"}, ai_metadata={"finish_reason": "stop"}, provider="openrouter", diff --git a/tools/migrate_v46_to_v47.py b/tools/migrate_v46_to_v47.py deleted file mode 100644 index 32b9fc1..0000000 --- a/tools/migrate_v46_to_v47.py +++ /dev/null @@ -1,337 +0,0 @@ -"""One-time migration of a V4.6 database into the V4.7 schema. - -V4.7 is an architectural cleanup: no new user-facing behaviour, but three -structural changes plus a one-time image backfill. This tool carries all of -them, and is built up phase by phase so the live database stays usable at -every phase boundary. - -Steps, in execution order: - -1. Rotate every stored Source image that still carries a supported EXIF - orientation, in place, and update ``source.file_hash`` and - ``source.file_size_bytes`` to describe the rewritten file. -2. Drop the ``processing_artifact`` table and delete its external files. -3. Rewrite ``execution_attempt.status`` from enum *names* to enum *values*, so - it compares equal to ``job_source.status`` (defect [45]). -4. Drop the five evidence columns from ``job_source``, leaving it a pure work - queue of ``id``, ``job_id``, ``source_id`` and ``status``. - -Design notes: - -- The image rewrite reuses the application's own - :func:`~transcription.services.normalization.normalize_orientation`, so the - backfilled bytes are byte-identical to what ingest would now produce. It - reuses the source quantization tables and subsampling rather than - re-quantizing, which is both smaller and higher fidelity than a fixed - quality setting. -- The hash and size are rewritten alongside the file. After V4.7 the evidence - digest is derived straight from ``source.file_hash``, so leaving it - describing the pre-rotation bytes would silently invalidate every future - export. -- The database is read and written through SQLAlchemy Core against the live - metadata, so the same script works against PostgreSQL when that cutover - happens. Raw DDL is used only for the table drop, which has no Core - equivalent that is safe to express against deleted metadata. -- The script is idempotent, keyed on state rather than on a version marker: - an image with no supported orientation tag is skipped, and a table that is - already absent is skipped. It is never invoked from application startup and - never runs in the test suite. -- **The application must not be running.** The image rewrite is not atomic - with the row update, and SQLite will refuse the schema change while another - connection holds the database. - -Usage:: - - python tools/migrate_v46_to_v47.py --dry-run - python tools/migrate_v46_to_v47.py -""" - -from __future__ import annotations - -import argparse -import hashlib -import sys -from collections.abc import Sequence -from pathlib import Path - -from sqlalchemy import bindparam -from sqlalchemy import create_engine -from sqlalchemy import inspect as sqlalchemy_inspect -from sqlalchemy import select -from sqlalchemy import text -from sqlalchemy import update -from sqlalchemy.engine import Connection -from sqlmodel import SQLModel - -from transcription.config import Settings -from transcription.config import get_settings -from transcription.db import models as _models # noqa: F401 (registers every table) -from transcription.db.engine import get_database_url -from transcription.db.models import JobSourceStatus -from transcription.services.normalization import normalize_orientation -from transcription.services.sources import source_mime_type - -#: Row counts the V4.6 database is expected to carry, used as a pre-flight -#: guard so the script cannot silently run against the wrong file. -EXPECTED_ROW_COUNTS = { - "document": 8, - "document_person": 11, - "document_type": 7, - "execution_attempt": 80, - "job": 11, - "job_source": 79, - "person": 5, - "person_role": 3, - "source": 76, -} - -ARTIFACT_TABLE = "processing_artifact" - -#: Evidence columns removed from ``job_source`` in V4.7. Every one of them is -#: duplicated byte-for-byte by ``execution_attempt`` across all 77 rows that -#: carry evidence, so no information is lost by dropping them. -JOB_SOURCE_DROPPED_COLUMNS = ( - "raw_transcription", - "ai_metadata", - "raw_api_response", - "error_detail", - "executed_at", -) - -#: The V4.6 default for the deleted ``Settings.artifact_dir``. The setting no -#: longer exists, so the historical location is recorded here instead. -DEFAULT_ARTIFACT_DIR = Path("data/artifacts") - - -def _sync_url(settings: Settings) -> str: - """Return the target database URL with any async driver stripped.""" - url = get_database_url(settings) - return url.replace("+aiosqlite", "").replace("+asyncpg", "").replace("+psycopg", "") - - -def _preflight(connection: Connection, *, strict: bool) -> None: - inspector = sqlalchemy_inspect(connection) - present = set(inspector.get_table_names()) - mismatched: dict[str, tuple[object, int]] = {} - for name, expected in EXPECTED_ROW_COUNTS.items(): - if name not in present: - mismatched[name] = ("missing", expected) - continue - actual = connection.execute(text(f'select count(*) from "{name}"')).scalar_one() - if actual != expected: - mismatched[name] = (actual, expected) - if not mismatched: - return - detail = ", ".join(f"{name}: found {found}, expected {want}" for name, (found, want) in sorted(mismatched.items())) - message = f"Database row counts do not match the recorded V4.6 snapshot ({detail})" - if strict: - raise RuntimeError(message) - print(f"WARNING: {message}", file=sys.stderr) - - -def rotate_stored_images(connection: Connection, *, dry_run: bool) -> int: - """Step 1: rewrite every mis-oriented stored image and its recorded digest.""" - source = SQLModel.metadata.tables["source"] - rows = connection.execute(select(source.c.id, source.c.file_path, source.c.filename)).all() - - rotated = 0 - missing = 0 - for source_id, file_path, filename in rows: - path = Path(str(file_path)) - if not path.is_file(): - print(f" WARNING: source file not found, skipped: {path}", file=sys.stderr) - missing += 1 - continue - - content = path.read_bytes() - normalized = normalize_orientation(content, media_type=source_mime_type(str(filename))) - if normalized is None: - continue - - rotated += 1 - print( - f" {path.name} orientation={normalized.original_orientation} " - f"rotation={normalized.applied_rotation_degrees} " - f"{len(content)} -> {len(normalized.content)} bytes" - ) - if dry_run: - continue - - path.write_bytes(normalized.content) - connection.execute( - update(source) - .where(source.c.id == source_id) - .values( - file_hash=hashlib.sha256(normalized.content).hexdigest(), - file_size_bytes=len(normalized.content), - ) - ) - - print(f" rotated={rotated} upright={len(rows) - rotated - missing} missing={missing}") - return rotated - - -def drop_processing_artifacts(connection: Connection, artifact_dir: Path, *, dry_run: bool) -> int: - """Step 2: drop the artifact table and delete the files it referenced.""" - inspector = sqlalchemy_inspect(connection) - if ARTIFACT_TABLE not in set(inspector.get_table_names()): - print(f" {ARTIFACT_TABLE} already absent") - return 0 - - references = [ - str(row[0]) - for row in connection.execute( - text(f'select external_reference from "{ARTIFACT_TABLE}" where external_reference is not null') - ) - ] - count = connection.execute(text(f'select count(*) from "{ARTIFACT_TABLE}"')).scalar_one() - print(f" dropping {ARTIFACT_TABLE} ({count} row(s), {len(references)} external file(s))") - - if dry_run: - return count - - connection.execute(text(f'drop table "{ARTIFACT_TABLE}"')) - - artifact_root = artifact_dir.resolve() - for reference in references: - relative = Path(reference) - if relative.is_absolute() or ".." in relative.parts: - print(f" WARNING: skipped unsafe artifact reference: {reference}", file=sys.stderr) - continue - artifact_path = (artifact_root / relative).resolve() - if artifact_root not in artifact_path.parents: - print(f" WARNING: skipped artifact outside root: {reference}", file=sys.stderr) - continue - artifact_path.unlink(missing_ok=True) - parent = artifact_path.parent - if parent != artifact_root and parent.is_dir() and not any(parent.iterdir()): - parent.rmdir() - - return count - - -def normalize_attempt_status(connection: Connection, *, dry_run: bool) -> int: - """Step 3: rewrite ``execution_attempt.status`` from enum names to values. - - Defect [45]: ``execution_attempt.status`` was declared without - ``values_callable``, so SQLAlchemy persisted enum *names* ('TRANSCRIBED') - while ``job_source.status`` persisted *values* ('transcribed'). The two - columns never compared equal on a single one of the 79 rows. The model - declaration is fixed in V4.7; the stored rows are fixed here. - """ - name_to_value = {member.name: member.value for member in JobSourceStatus} - recognised = sorted(set(name_to_value) | set(name_to_value.values())) - unknown = ( - connection.execute( - text("select distinct status from execution_attempt where status not in :values").bindparams( - bindparam("values", recognised, expanding=True) - ) - ) - .scalars() - .all() - ) - if unknown: - raise RuntimeError(f"execution_attempt.status carries unrecognised spellings: {sorted(unknown)}") - - rewritten = 0 - for name, value in sorted(name_to_value.items()): - if name == value: - continue - count = connection.execute( - text("select count(*) from execution_attempt where status = :name"), - {"name": name}, - ).scalar_one() - if not count: - continue - print(f" {name} -> {value}: {count} row(s)") - rewritten += count - if dry_run: - continue - connection.execute( - text("update execution_attempt set status = :value where status = :name"), - {"name": name, "value": value}, - ) - - print(f" rewritten={rewritten}") - return rewritten - - -def strip_job_source_columns(connection: Connection, *, dry_run: bool) -> int: - """Step 4: drop the evidence columns from ``job_source``. - - Uses ``ALTER TABLE ... DROP COLUMN``, supported by SQLite 3.35+ and by - PostgreSQL. Idempotent: a column that is already gone is skipped. - """ - inspector = sqlalchemy_inspect(connection) - present = {column["name"] for column in inspector.get_columns("job_source")} - targets = [name for name in JOB_SOURCE_DROPPED_COLUMNS if name in present] - if not targets: - print(" all evidence columns already dropped") - return 0 - - print(f" dropping {len(targets)} column(s): {', '.join(targets)}") - if dry_run: - return len(targets) - - for name in targets: - connection.execute(text(f'alter table "job_source" drop column "{name}"')) - return len(targets) - - -def migrate(*, settings: Settings, artifact_dir: Path, dry_run: bool, strict_counts: bool) -> None: - """Apply every V4.7 migration step in order.""" - engine = create_engine(_sync_url(settings)) - try: - with engine.begin() as connection: - _preflight(connection, strict=strict_counts) - - print("\nStep 1: rotate stored images") - rotate_stored_images(connection, dry_run=dry_run) - - print(f"\nStep 2: drop {ARTIFACT_TABLE}") - drop_processing_artifacts(connection, artifact_dir, dry_run=dry_run) - - print("\nStep 3: normalize execution_attempt.status spelling") - normalize_attempt_status(connection, dry_run=dry_run) - - print("\nStep 4: strip evidence columns from job_source") - strip_job_source_columns(connection, dry_run=dry_run) - finally: - engine.dispose() - - if dry_run: - print("\nDry run: nothing was written.") - else: - print("\nDone.") - - -def main(argv: Sequence[str] | None = None) -> int: - parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) - parser.add_argument("--dry-run", action="store_true", help="Report what would change without writing") - parser.add_argument( - "--allow-count-mismatch", - action="store_true", - help="Warn instead of aborting when row counts differ from the recorded V4.6 snapshot", - ) - parser.add_argument( - "--artifact-dir", - type=Path, - default=DEFAULT_ARTIFACT_DIR, - help="Directory that held external artifact files before V4.7", - ) - args = parser.parse_args(argv) - - settings = get_settings() - print(f"Target: {_sync_url(settings)}") - - migrate( - settings=settings, - artifact_dir=args.artifact_dir, - dry_run=args.dry_run, - strict_counts=not args.allow_count_mismatch, - ) - return 0 - - -if __name__ == "__main__": - raise SystemExit(main()) diff --git a/tools/migrate_v47_to_v48.py b/tools/migrate_v47_to_v48.py deleted file mode 100644 index cc06d62..0000000 --- a/tools/migrate_v47_to_v48.py +++ /dev/null @@ -1,185 +0,0 @@ -"""One-time migration for V4.8 queue-membership uniqueness. - -This migration enforces the V4 requirement that each ``(job_id, source_id)`` -pair appears at most once in ``job_source``. - -Steps: -1. Detect duplicate ``job_source`` rows per ``(job_id, source_id)``. -2. Keep one row per pair (prefer the row with the latest attempt evidence). -3. Re-point ``execution_attempt.job_source_id`` from removed duplicates to the - kept row. -4. Delete duplicate ``job_source`` rows. -5. Add uniqueness enforcement for ``(job_id, source_id)``. - -Usage:: - - python tools/migrate_v47_to_v48.py --dry-run - python tools/migrate_v47_to_v48.py -""" - -from __future__ import annotations - -import argparse -from collections import defaultdict -from collections.abc import Sequence -from datetime import UTC -from datetime import datetime -from uuid import UUID - -from sqlalchemy import bindparam -from sqlalchemy import create_engine -from sqlalchemy import func -from sqlalchemy import inspect as sqlalchemy_inspect -from sqlalchemy import select -from sqlalchemy import text -from sqlalchemy import update -from sqlalchemy.engine import Connection -from sqlmodel import SQLModel - -from transcription.config import Settings -from transcription.config import get_settings -from transcription.db import models as _models # noqa: F401 (registers metadata tables) -from transcription.db.engine import get_database_url - -UNIQUE_NAME = "uq_job_source_job_source" - - -def _sync_url(settings: Settings) -> str: - """Return a sync URL for direct SQLAlchemy Core access.""" - return get_database_url(settings).replace("+aiosqlite", "").replace("+asyncpg", "").replace("+psycopg", "") - - -def _uniqueness_already_enforced(connection: Connection) -> bool: - inspector = sqlalchemy_inspect(connection) - unique_constraints = inspector.get_unique_constraints("job_source") - if any(constraint.get("name") == UNIQUE_NAME for constraint in unique_constraints): - return True - indexes = inspector.get_indexes("job_source") - return any(index.get("name") == UNIQUE_NAME and index.get("unique") is True for index in indexes) - - -def _choose_keeper( - candidates: list[dict[str, object]], -) -> dict[str, object]: - """Keep the row with latest attempt activity, then lexicographically greatest UUID.""" - - def key(item: dict[str, object]) -> tuple[datetime, str]: - latest_attempt = item["latest_attempt_at"] - attempt_key = latest_attempt if isinstance(latest_attempt, datetime) else datetime.min.replace(tzinfo=UTC) - return (attempt_key, str(item["job_source_id"])) - - return max(candidates, key=key) - - -def deduplicate_job_source_membership(connection: Connection, *, dry_run: bool) -> tuple[int, int]: - """Return ``(pairs_deduplicated, rows_deleted)``.""" - job_source = SQLModel.metadata.tables["job_source"] - execution_attempt = SQLModel.metadata.tables["execution_attempt"] - - duplicate_pairs = ( - connection.execute( - select(job_source.c.job_id, job_source.c.source_id) - .group_by(job_source.c.job_id, job_source.c.source_id) - .having(func.count(job_source.c.id) > 1) - ) - .mappings() - .all() - ) - if not duplicate_pairs: - return (0, 0) - - rows_by_pair: dict[tuple[UUID, UUID], list[dict[str, object]]] = defaultdict(list) - duplicate_memberships = ( - connection.execute( - select( - job_source.c.id.label("job_source_id"), - job_source.c.job_id, - job_source.c.source_id, - func.max(execution_attempt.c.created_at).label("latest_attempt_at"), - ) - .select_from(job_source.outerjoin(execution_attempt, execution_attempt.c.job_source_id == job_source.c.id)) - .group_by(job_source.c.id, job_source.c.job_id, job_source.c.source_id) - ) - .mappings() - .all() - ) - duplicate_key_set = {(row["job_id"], row["source_id"]) for row in duplicate_pairs} - for row in duplicate_memberships: - key = (row["job_id"], row["source_id"]) - if key in duplicate_key_set: - rows_by_pair[key].append(dict(row)) - - rows_deleted = 0 - for key, rows in sorted(rows_by_pair.items(), key=lambda item: (str(item[0][0]), str(item[0][1]))): - keeper = _choose_keeper(rows) - keeper_id = keeper["job_source_id"] - duplicate_ids = [row["job_source_id"] for row in rows if row["job_source_id"] != keeper_id] - rows_deleted += len(duplicate_ids) - print( - f" dedupe pair job_id={key[0]} source_id={key[1]} " - f"keep={keeper_id} drop={','.join(str(item) for item in duplicate_ids)}" - ) - if dry_run or not duplicate_ids: - continue - connection.execute( - update(execution_attempt) - .where(execution_attempt.c.job_source_id.in_(bindparam("duplicate_ids", expanding=True))) - .values(job_source_id=keeper_id), - {"duplicate_ids": duplicate_ids}, - ) - connection.execute( - job_source.delete().where(job_source.c.id.in_(bindparam("duplicate_ids", expanding=True))), - {"duplicate_ids": duplicate_ids}, - ) - - return (len(duplicate_pairs), rows_deleted) - - -def add_job_source_uniqueness(connection: Connection, *, dry_run: bool) -> None: - """Create uniqueness enforcement for ``(job_id, source_id)``.""" - if _uniqueness_already_enforced(connection): - print(f" uniqueness already enforced ({UNIQUE_NAME})") - return - - dialect = connection.dialect.name - if dialect == "postgresql": - statement = text(f'alter table "job_source" add constraint "{UNIQUE_NAME}" unique ("job_id", "source_id")') - else: - statement = text(f'create unique index "{UNIQUE_NAME}" on "job_source" ("job_id", "source_id")') - print(f" applying {UNIQUE_NAME}") - if not dry_run: - connection.execute(statement) - - -def migrate(*, settings: Settings, dry_run: bool) -> None: - engine = create_engine(_sync_url(settings)) - try: - with engine.begin() as connection: - print("Step 1: deduplicate job_source membership") - pair_count, rows_deleted = deduplicate_job_source_membership(connection, dry_run=dry_run) - print(f" duplicate pairs={pair_count} rows_deleted={rows_deleted}") - - print("Step 2: enforce unique (job_id, source_id)") - add_job_source_uniqueness(connection, dry_run=dry_run) - finally: - engine.dispose() - - if dry_run: - print("\nDry run: nothing was written.") - else: - print("\nDone.") - - -def main(argv: Sequence[str] | None = None) -> int: - parser = argparse.ArgumentParser(description=__doc__) - parser.add_argument("--dry-run", action="store_true", help="Report planned changes without writing") - args = parser.parse_args(argv) - - settings = get_settings() - print(f"Target: {_sync_url(settings)}") - migrate(settings=settings, dry_run=args.dry_run) - return 0 - - -if __name__ == "__main__": - raise SystemExit(main())