V4.7 Phase 2: Evidence Model Simplification (part 2)

tools/migrate_v46_to_v47.py

@@ -11,6 +11,10 @@ Steps, in execution order:
    orientation, in place, and update ``source.file_hash`` and
    ``source.file_size_bytes`` to describe the rewritten file.
 2. Drop the ``processing_artifact`` table and delete its external files.
+3. Rewrite ``execution_attempt.status`` from enum *names* to enum *values*, so
+   it compares equal to ``job_source.status`` (defect [45]).
+4. Drop the five evidence columns from ``job_source``, leaving it a pure work
+   queue of ``id``, ``job_id``, ``source_id`` and ``status``.
 
 Design notes:
 
@@ -50,6 +54,7 @@ import sys
 from collections.abc import Sequence
 from pathlib import Path
 
+from sqlalchemy import bindparam
 from sqlalchemy import create_engine
 from sqlalchemy import inspect as sqlalchemy_inspect
 from sqlalchemy import select
@@ -62,6 +67,7 @@ from transcription.config import Settings
 from transcription.config import get_settings
 from transcription.db import models as _models  # noqa: F401  (registers every table)
 from transcription.db.engine import get_database_url
+from transcription.db.models import JobSourceStatus
 from transcription.services.normalization import normalize_orientation
 from transcription.services.sources import source_mime_type
 
@@ -81,6 +87,17 @@ EXPECTED_ROW_COUNTS = {
 
 ARTIFACT_TABLE = "processing_artifact"
 
+#: Evidence columns removed from ``job_source`` in V4.7. Every one of them is
+#: duplicated byte-for-byte by ``execution_attempt`` across all 77 rows that
+#: carry evidence, so no information is lost by dropping them.
+JOB_SOURCE_DROPPED_COLUMNS = (
+    "raw_transcription",
+    "ai_metadata",
+    "raw_api_response",
+    "error_detail",
+    "executed_at",
+)
+
 #: The V4.6 default for the deleted ``Settings.artifact_dir``. The setting no
 #: longer exists, so the historical location is recorded here instead.
 DEFAULT_ARTIFACT_DIR = Path("data/artifacts")
@@ -195,6 +212,74 @@ def drop_processing_artifacts(connection: Connection, artifact_dir: Path, *, dry
     return count
 
 
+def normalize_attempt_status(connection: Connection, *, dry_run: bool) -> int:
+    """Step 3: rewrite ``execution_attempt.status`` from enum names to values.
+
+    Defect [45]: ``execution_attempt.status`` was declared without
+    ``values_callable``, so SQLAlchemy persisted enum *names* ('TRANSCRIBED')
+    while ``job_source.status`` persisted *values* ('transcribed'). The two
+    columns never compared equal on a single one of the 79 rows. The model
+    declaration is fixed in V4.7; the stored rows are fixed here.
+    """
+    name_to_value = {member.name: member.value for member in JobSourceStatus}
+    recognised = sorted(set(name_to_value) | set(name_to_value.values()))
+    unknown = (
+        connection.execute(
+            text("select distinct status from execution_attempt where status not in :values").bindparams(
+                bindparam("values", recognised, expanding=True)
+            )
+        )
+        .scalars()
+        .all()
+    )
+    if unknown:
+        raise RuntimeError(f"execution_attempt.status carries unrecognised spellings: {sorted(unknown)}")
+
+    rewritten = 0
+    for name, value in sorted(name_to_value.items()):
+        if name == value:
+            continue
+        count = connection.execute(
+            text("select count(*) from execution_attempt where status = :name"),
+            {"name": name},
+        ).scalar_one()
+        if not count:
+            continue
+        print(f"  {name} -> {value}: {count} row(s)")
+        rewritten += count
+        if dry_run:
+            continue
+        connection.execute(
+            text("update execution_attempt set status = :value where status = :name"),
+            {"name": name, "value": value},
+        )
+
+    print(f"  rewritten={rewritten}")
+    return rewritten
+
+
+def strip_job_source_columns(connection: Connection, *, dry_run: bool) -> int:
+    """Step 4: drop the evidence columns from ``job_source``.
+
+    Uses ``ALTER TABLE ... DROP COLUMN``, supported by SQLite 3.35+ and by
+    PostgreSQL. Idempotent: a column that is already gone is skipped.
+    """
+    inspector = sqlalchemy_inspect(connection)
+    present = {column["name"] for column in inspector.get_columns("job_source")}
+    targets = [name for name in JOB_SOURCE_DROPPED_COLUMNS if name in present]
+    if not targets:
+        print("  all evidence columns already dropped")
+        return 0
+
+    print(f"  dropping {len(targets)} column(s): {', '.join(targets)}")
+    if dry_run:
+        return len(targets)
+
+    for name in targets:
+        connection.execute(text(f'alter table "job_source" drop column "{name}"'))
+    return len(targets)
+
+
 def migrate(*, settings: Settings, artifact_dir: Path, dry_run: bool, strict_counts: bool) -> None:
     """Apply every V4.7 migration step in order."""
     engine = create_engine(_sync_url(settings))
@@ -207,6 +292,12 @@ def migrate(*, settings: Settings, artifact_dir: Path, dry_run: bool, strict_cou
 
             print(f"\nStep 2: drop {ARTIFACT_TABLE}")
             drop_processing_artifacts(connection, artifact_dir, dry_run=dry_run)
+
+            print("\nStep 3: normalize execution_attempt.status spelling")
+            normalize_attempt_status(connection, dry_run=dry_run)
+
+            print("\nStep 4: strip evidence columns from job_source")
+            strip_job_source_columns(connection, dry_run=dry_run)
     finally:
         engine.dispose()