generated from john/python-template
V4.7 Phase 1: ingest orientation normalization, ProcessingArtifact removal
Move orientation normalization to the Source-ingest boundary and delete the ProcessingArtifact subsystem it was built to serve. Stored pages are now already upright, so nothing downstream derives a rotated copy: every stored byte is the byte a provider is later sent. Rotation runs in store_source_file ahead of hashing, so source.file_hash and file_size_bytes describe exactly what is on disk. normalize_orientation becomes bytes-in / bytes-out, and JPEG output reuses the source quantization tables and chroma subsampling instead of re-quantizing at a fixed quality - measured at 50.3-56.1 dB PSNR at -6% size, against 50.0-53.5 dB at +38% for quality=95. ProcessingArtifact held 2 rows against 77 successful transcriptions; the subsystem effectively never ran. Deleting it removes the artifact cluster from sources.py, the derivative resolution in workflows.py, the pre-provider commit that only existed to make an artifact row durable, and the artifact evidence dump from the Source detail page. The transcription_quality_warnings payload folds into execution_attempt.normalized_metadata, so that feature keeps working without the table. tools/migrate_v46_to_v47.py carries steps 1 and 2: it rotated the 58 stored images carrying EXIF orientation 3 in place, updated their recorded hash and size, dropped processing_artifact and removed its one external file. It is idempotent, keyed on state rather than a version marker. tools/migrate_v45_to_v46.py is deleted. That migration is complete, and after V4.7 it would restore a V4.5 backup into a schema that no longer matches. Also fixes tests/test_config.py, which read the developer's local .env and failed whenever WORKER_MAX_RETRIES was set. Co-authored-by: Copilot App <[email protected]>
This commit is contained in:
zoltan57
Copilot App
@@ -0,0 +1,248 @@
|
||||
"""One-time migration of a V4.6 database into the V4.7 schema.
|
||||
|
||||
V4.7 is an architectural cleanup: no new user-facing behaviour, but three
|
||||
structural changes plus a one-time image backfill. This tool carries all of
|
||||
them, and is built up phase by phase so the live database stays usable at
|
||||
every phase boundary.
|
||||
|
||||
Steps, in execution order:
|
||||
|
||||
1. Rotate every stored Source image that still carries a supported EXIF
|
||||
orientation, in place, and update ``source.file_hash`` and
|
||||
``source.file_size_bytes`` to describe the rewritten file.
|
||||
2. Drop the ``processing_artifact`` table and delete its external files.
|
||||
|
||||
Design notes:
|
||||
|
||||
- The image rewrite reuses the application's own
|
||||
:func:`~transcription.services.normalization.normalize_orientation`, so the
|
||||
backfilled bytes are byte-identical to what ingest would now produce. It
|
||||
reuses the source quantization tables and subsampling rather than
|
||||
re-quantizing, which is both smaller and higher fidelity than a fixed
|
||||
quality setting.
|
||||
- The hash and size are rewritten alongside the file. After V4.7 the evidence
|
||||
digest is derived straight from ``source.file_hash``, so leaving it
|
||||
describing the pre-rotation bytes would silently invalidate every future
|
||||
export.
|
||||
- The database is read and written through SQLAlchemy Core against the live
|
||||
metadata, so the same script works against PostgreSQL when that cutover
|
||||
happens. Raw DDL is used only for the table drop, which has no Core
|
||||
equivalent that is safe to express against deleted metadata.
|
||||
- The script is idempotent, keyed on state rather than on a version marker:
|
||||
an image with no supported orientation tag is skipped, and a table that is
|
||||
already absent is skipped. It is never invoked from application startup and
|
||||
never runs in the test suite.
|
||||
- **The application must not be running.** The image rewrite is not atomic
|
||||
with the row update, and SQLite will refuse the schema change while another
|
||||
connection holds the database.
|
||||
|
||||
Usage::
|
||||
|
||||
python tools/migrate_v46_to_v47.py --dry-run
|
||||
python tools/migrate_v46_to_v47.py
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import hashlib
|
||||
import sys
|
||||
from collections.abc import Sequence
|
||||
from pathlib import Path
|
||||
|
||||
from sqlalchemy import create_engine
|
||||
from sqlalchemy import inspect as sqlalchemy_inspect
|
||||
from sqlalchemy import select
|
||||
from sqlalchemy import text
|
||||
from sqlalchemy import update
|
||||
from sqlalchemy.engine import Connection
|
||||
from sqlmodel import SQLModel
|
||||
|
||||
from transcription.config import Settings
|
||||
from transcription.config import get_settings
|
||||
from transcription.db import models as _models # noqa: F401 (registers every table)
|
||||
from transcription.db.engine import get_database_url
|
||||
from transcription.services.normalization import normalize_orientation
|
||||
from transcription.services.sources import source_mime_type
|
||||
|
||||
#: Row counts the V4.6 database is expected to carry, used as a pre-flight
|
||||
#: guard so the script cannot silently run against the wrong file.
|
||||
EXPECTED_ROW_COUNTS = {
|
||||
"document": 8,
|
||||
"document_person": 11,
|
||||
"document_type": 7,
|
||||
"execution_attempt": 80,
|
||||
"job": 11,
|
||||
"job_source": 79,
|
||||
"person": 5,
|
||||
"person_role": 3,
|
||||
"source": 76,
|
||||
}
|
||||
|
||||
ARTIFACT_TABLE = "processing_artifact"
|
||||
|
||||
#: The V4.6 default for the deleted ``Settings.artifact_dir``. The setting no
|
||||
#: longer exists, so the historical location is recorded here instead.
|
||||
DEFAULT_ARTIFACT_DIR = Path("data/artifacts")
|
||||
|
||||
|
||||
def _sync_url(settings: Settings) -> str:
|
||||
"""Return the target database URL with any async driver stripped."""
|
||||
url = get_database_url(settings)
|
||||
return url.replace("+aiosqlite", "").replace("+asyncpg", "").replace("+psycopg", "")
|
||||
|
||||
|
||||
def _preflight(connection: Connection, *, strict: bool) -> None:
|
||||
inspector = sqlalchemy_inspect(connection)
|
||||
present = set(inspector.get_table_names())
|
||||
mismatched: dict[str, tuple[object, int]] = {}
|
||||
for name, expected in EXPECTED_ROW_COUNTS.items():
|
||||
if name not in present:
|
||||
mismatched[name] = ("missing", expected)
|
||||
continue
|
||||
actual = connection.execute(text(f'select count(*) from "{name}"')).scalar_one()
|
||||
if actual != expected:
|
||||
mismatched[name] = (actual, expected)
|
||||
if not mismatched:
|
||||
return
|
||||
detail = ", ".join(f"{name}: found {found}, expected {want}" for name, (found, want) in sorted(mismatched.items()))
|
||||
message = f"Database row counts do not match the recorded V4.6 snapshot ({detail})"
|
||||
if strict:
|
||||
raise RuntimeError(message)
|
||||
print(f"WARNING: {message}", file=sys.stderr)
|
||||
|
||||
|
||||
def rotate_stored_images(connection: Connection, *, dry_run: bool) -> int:
|
||||
"""Step 1: rewrite every mis-oriented stored image and its recorded digest."""
|
||||
source = SQLModel.metadata.tables["source"]
|
||||
rows = connection.execute(
|
||||
select(source.c.id, source.c.file_path, source.c.filename)
|
||||
).all()
|
||||
|
||||
rotated = 0
|
||||
missing = 0
|
||||
for source_id, file_path, filename in rows:
|
||||
path = Path(str(file_path))
|
||||
if not path.is_file():
|
||||
print(f" WARNING: source file not found, skipped: {path}", file=sys.stderr)
|
||||
missing += 1
|
||||
continue
|
||||
|
||||
content = path.read_bytes()
|
||||
normalized = normalize_orientation(content, media_type=source_mime_type(str(filename)))
|
||||
if normalized is None:
|
||||
continue
|
||||
|
||||
rotated += 1
|
||||
print(
|
||||
f" {path.name} orientation={normalized.original_orientation} "
|
||||
f"rotation={normalized.applied_rotation_degrees} "
|
||||
f"{len(content)} -> {len(normalized.content)} bytes"
|
||||
)
|
||||
if dry_run:
|
||||
continue
|
||||
|
||||
path.write_bytes(normalized.content)
|
||||
connection.execute(
|
||||
update(source)
|
||||
.where(source.c.id == source_id)
|
||||
.values(
|
||||
file_hash=hashlib.sha256(normalized.content).hexdigest(),
|
||||
file_size_bytes=len(normalized.content),
|
||||
)
|
||||
)
|
||||
|
||||
print(f" rotated={rotated} upright={len(rows) - rotated - missing} missing={missing}")
|
||||
return rotated
|
||||
|
||||
|
||||
def drop_processing_artifacts(connection: Connection, artifact_dir: Path, *, dry_run: bool) -> int:
|
||||
"""Step 2: drop the artifact table and delete the files it referenced."""
|
||||
inspector = sqlalchemy_inspect(connection)
|
||||
if ARTIFACT_TABLE not in set(inspector.get_table_names()):
|
||||
print(f" {ARTIFACT_TABLE} already absent")
|
||||
return 0
|
||||
|
||||
references = [
|
||||
str(row[0])
|
||||
for row in connection.execute(
|
||||
text(f'select external_reference from "{ARTIFACT_TABLE}" where external_reference is not null')
|
||||
)
|
||||
]
|
||||
count = connection.execute(text(f'select count(*) from "{ARTIFACT_TABLE}"')).scalar_one()
|
||||
print(f" dropping {ARTIFACT_TABLE} ({count} row(s), {len(references)} external file(s))")
|
||||
|
||||
if dry_run:
|
||||
return count
|
||||
|
||||
connection.execute(text(f'drop table "{ARTIFACT_TABLE}"'))
|
||||
|
||||
artifact_root = artifact_dir.resolve()
|
||||
for reference in references:
|
||||
relative = Path(reference)
|
||||
if relative.is_absolute() or ".." in relative.parts:
|
||||
print(f" WARNING: skipped unsafe artifact reference: {reference}", file=sys.stderr)
|
||||
continue
|
||||
artifact_path = (artifact_root / relative).resolve()
|
||||
if artifact_root not in artifact_path.parents:
|
||||
print(f" WARNING: skipped artifact outside root: {reference}", file=sys.stderr)
|
||||
continue
|
||||
artifact_path.unlink(missing_ok=True)
|
||||
parent = artifact_path.parent
|
||||
if parent != artifact_root and parent.is_dir() and not any(parent.iterdir()):
|
||||
parent.rmdir()
|
||||
|
||||
return count
|
||||
|
||||
|
||||
def migrate(*, settings: Settings, artifact_dir: Path, dry_run: bool, strict_counts: bool) -> None:
|
||||
"""Apply every V4.7 migration step in order."""
|
||||
engine = create_engine(_sync_url(settings))
|
||||
try:
|
||||
with engine.begin() as connection:
|
||||
_preflight(connection, strict=strict_counts)
|
||||
|
||||
print("\nStep 1: rotate stored images")
|
||||
rotate_stored_images(connection, dry_run=dry_run)
|
||||
|
||||
print(f"\nStep 2: drop {ARTIFACT_TABLE}")
|
||||
drop_processing_artifacts(connection, artifact_dir, dry_run=dry_run)
|
||||
finally:
|
||||
engine.dispose()
|
||||
|
||||
if dry_run:
|
||||
print("\nDry run: nothing was written.")
|
||||
else:
|
||||
print("\nDone.")
|
||||
|
||||
|
||||
def main(argv: Sequence[str] | None = None) -> int:
|
||||
parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
|
||||
parser.add_argument("--dry-run", action="store_true", help="Report what would change without writing")
|
||||
parser.add_argument(
|
||||
"--allow-count-mismatch",
|
||||
action="store_true",
|
||||
help="Warn instead of aborting when row counts differ from the recorded V4.6 snapshot",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--artifact-dir",
|
||||
type=Path,
|
||||
default=DEFAULT_ARTIFACT_DIR,
|
||||
help="Directory that held external artifact files before V4.7",
|
||||
)
|
||||
args = parser.parse_args(argv)
|
||||
|
||||
settings = get_settings()
|
||||
print(f"Target: {_sync_url(settings)}")
|
||||
|
||||
migrate(
|
||||
settings=settings,
|
||||
artifact_dir=args.artifact_dir,
|
||||
dry_run=args.dry_run,
|
||||
strict_counts=not args.allow_count_mismatch,
|
||||
)
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
Reference in New Issue
Block a user