generated from john/python-template
zoltan57andCopilot App
3e418a0889
These changes all regenerate the same schema, so they land together and revert
together. A partially applied schema pass is not a valid state.
Remove hand-rolled migrations [HIGH-05]
- Delete upgrade_schema and the _upgrade_person_family_search_id /
_upgrade_v42_evidence_tables / _upgrade_v45_selection_columns chain, plus the
two tests that exercised them. The DDL was SQLite-shaped raw SQL that would
not have run on PostgreSQL. create_all now derives everything from metadata
and remains gated by Settings.should_bootstrap_schema. No raw ALTER TABLE or
CREATE INDEX string remains in src.
Break the foreign key cycle [HIGH-08]
- Declare Source.preferred_execution_attempt_id with use_alter=True and an
explicit constraint name. source / job_source / execution_attempt formed an
unresolvable cycle that made metadata.sorted_tables emit an SAWarning and
order execution_attempt before source, which would have been a hard
create_all failure on PostgreSQL and was invisible on SQLite.
- As a side effect the column is now a dialect-aware Uuid rather than the
hardcoded CHAR(32) the raw upgrade DDL produced, so it emits native UUID on
PostgreSQL.
Index the hot filters [HIGH-04]
- Add composite Index("ix_job_status_date_created", "status", "date_created")
for the worker poll, and index the foreign keys the worker and detail pages
filter on: job.document_id, source.document_id, job_source.job_id,
job_source.source_id, document.document_type_id, and the three
document_person foreign keys.
Stop preloading by default [CRIT-02]
- Flip 16 relationships from lazy="selectin" to lazy="raise". The bidirectional
selectin defaults meant loading one Job pulled a large connected subgraph.
- Three further relationships (ExecutionAttempt.job_source,
ProcessingArtifact.execution_attempt, ProcessingArtifact.source) declared no
lazy at all and defaulted to "select", which raises MissingGreenlet under
async. These are now "raise" as well.
- Only 5 of 262 tests failed under the flip; the service layer already carried
explicit eager loads. Fixes went into the service queries, never back into
the models:
- PeopleService._finalize_link refreshes document, person, and role_ref so
the DocumentPerson write endpoints can still project them.
- JobService.update_job_state loads job_sources -> source so the Job it
returns still answers .error_detail and .filename.
- Two tests that bypassed the service layer now load explicitly.
- Audited every UI relationship access against its feeding service method; all
resolve to *_detail / list_*_detail variants with complete eager loads.
Tests
- Assert the composite and hot foreign key indexes exist in a fresh schema.
- Assert metadata.sorted_tables raises no SAWarning and orders source before
execution_attempt.
- Assert preferred_execution_attempt_id is a Uuid that compiles to UUID on
PostgreSQL and that its foreign key carries use_alter.
- Guard CRIT-02 from regression: no mapped relationship may declare a lazy
strategy outside {raise, noload}.
The development database was rebuilt from metadata rather than upgraded; the
previous file is retained out of tree as the Phase 8 migration source.
Verified: 266 passed, 4 skipped; ruff check clean.
Co-authored-by: Copilot App <[email protected]>
282 lines
9.2 KiB
Python
282 lines
9.2 KiB
Python
"""Tests for the V2 SQLModel persistence layer and relationships."""
|
|
|
|
from uuid import UUID
|
|
|
|
import pytest
|
|
from sqlalchemy.exc import IntegrityError
|
|
|
|
from transcription.db.models import Document
|
|
from transcription.db.models import DocumentPerson
|
|
from transcription.db.models import DocumentType
|
|
from transcription.db.models import Job
|
|
from transcription.db.models import JobSource
|
|
from transcription.db.models import JobSourceStatus
|
|
from transcription.db.models import JobStatus
|
|
from transcription.db.models import Person
|
|
from transcription.db.models import PersonRole
|
|
from transcription.db.models import Source
|
|
|
|
|
|
def _make_document(**overrides) -> Document:
|
|
defaults = {
|
|
"name": "letter bundle",
|
|
"notes": "Family correspondence",
|
|
}
|
|
defaults.update(overrides)
|
|
return Document(**defaults)
|
|
|
|
|
|
def _persist_document_type(session, *, label: str = "Letter") -> DocumentType:
|
|
document_type = DocumentType(label=label, normalized_label=label.strip().casefold())
|
|
session.add(document_type)
|
|
session.commit()
|
|
session.refresh(document_type)
|
|
return document_type
|
|
|
|
|
|
def _persist_person_role(session, *, label: str = "Author") -> PersonRole:
|
|
role = PersonRole(label=label, normalized_label=label.strip().casefold())
|
|
session.add(role)
|
|
session.commit()
|
|
session.refresh(role)
|
|
return role
|
|
|
|
|
|
def _persist_document(session) -> Document:
|
|
document_type = _persist_document_type(session)
|
|
document = _make_document(document_type_id=document_type.id)
|
|
session.add(document)
|
|
session.commit()
|
|
session.refresh(document)
|
|
return document
|
|
|
|
|
|
def _persist_person(session, **overrides) -> Person:
|
|
defaults = {"full_name": "Ada Lovelace"}
|
|
defaults.update(overrides)
|
|
person = Person(**defaults)
|
|
session.add(person)
|
|
session.commit()
|
|
session.refresh(person)
|
|
return person
|
|
|
|
|
|
def _persist_job(session, document: Document) -> Job:
|
|
job = Job(document_id=document.id)
|
|
session.add(job)
|
|
session.commit()
|
|
session.refresh(job)
|
|
return job
|
|
|
|
|
|
def _persist_source(session, document: Document, *, page_number: int = 1, **overrides) -> Source:
|
|
defaults = {
|
|
"document_id": document.id,
|
|
"page_number": page_number,
|
|
"upload_name": "letter.jpg",
|
|
"filename": "stored-letter.jpg",
|
|
"file_path": "/uploads/stored-letter.jpg",
|
|
"raw_transcription": "Original machine text",
|
|
}
|
|
defaults.update(overrides)
|
|
defaults["file_hash"] = "a" * 64
|
|
defaults["file_size_bytes"] = 123
|
|
source = Source(**defaults)
|
|
session.add(source)
|
|
session.commit()
|
|
session.refresh(source)
|
|
return source
|
|
|
|
|
|
def _persist_job_source(session, job: Job, source: Source, **overrides) -> JobSource:
|
|
defaults = {
|
|
"job_id": job.id,
|
|
"source_id": source.id,
|
|
"status": JobSourceStatus.PENDING,
|
|
}
|
|
defaults.update(overrides)
|
|
job_source = JobSource(**defaults)
|
|
session.add(job_source)
|
|
session.commit()
|
|
session.refresh(job_source)
|
|
return job_source
|
|
|
|
|
|
class TestDocumentModel:
|
|
def test_can_be_persisted(self, session):
|
|
document = _persist_document(session)
|
|
fetched = session.get(Document, document.id)
|
|
assert fetched is not None
|
|
assert fetched.name == "letter bundle"
|
|
|
|
def test_defaults_are_populated(self, session):
|
|
document = _persist_document(session)
|
|
assert isinstance(document.id, UUID)
|
|
assert document.created_at is not None
|
|
assert document.updated_at is not None
|
|
|
|
def test_can_reference_document_type_registry(self, session):
|
|
document_type = _persist_document_type(session, label="Record")
|
|
document = _make_document(document_type_id=document_type.id)
|
|
session.add(document)
|
|
session.commit()
|
|
session.refresh(document)
|
|
assert document.document_type_id == document_type.id
|
|
|
|
|
|
class TestJobModel:
|
|
def test_can_be_created_for_document(self, session):
|
|
document = _persist_document(session)
|
|
job = _persist_job(session, document)
|
|
fetched = session.get(Job, job.id)
|
|
assert fetched is not None
|
|
assert fetched.document_id == document.id
|
|
|
|
def test_defaults_are_populated(self, session):
|
|
document = _persist_document(session)
|
|
job = _persist_job(session, document)
|
|
assert job.status == JobStatus.QUEUED
|
|
assert job.retry_count == 0
|
|
assert job.date_created is not None
|
|
assert job.date_updated is not None
|
|
|
|
def test_transitions_to_completed(self, session):
|
|
document = _persist_document(session)
|
|
job = _persist_job(session, document)
|
|
|
|
job.status = JobStatus.PROCESSING
|
|
session.add(job)
|
|
session.commit()
|
|
session.refresh(job)
|
|
|
|
job.status = JobStatus.COMPLETED
|
|
session.add(job)
|
|
session.commit()
|
|
session.refresh(job)
|
|
|
|
assert job.status == JobStatus.COMPLETED
|
|
|
|
|
|
class TestSourceModel:
|
|
def test_can_be_created_for_document(self, session):
|
|
document = _persist_document(session)
|
|
source = _persist_source(session, document)
|
|
|
|
fetched = session.get(Source, source.id)
|
|
assert fetched is not None
|
|
assert fetched.document_id == document.id
|
|
assert fetched.page_number == 1
|
|
assert fetched.date_uploaded is not None
|
|
|
|
def test_revised_text_is_supported(self, session):
|
|
document = _persist_document(session)
|
|
source = _persist_source(session, document, revised_text="Edited output")
|
|
|
|
fetched = session.get(Source, source.id)
|
|
assert fetched is not None
|
|
assert fetched.revised_text == "Edited output"
|
|
|
|
|
|
class TestPersonAndDocumentPersonModel:
|
|
def test_family_search_id_is_unique_when_present(self, session):
|
|
session.add(Person(full_name="First Person", family_search_id="G8T4-MDQ"))
|
|
session.commit()
|
|
|
|
session.add(Person(full_name="Second Person", family_search_id="G8T4-MDQ"))
|
|
with pytest.raises(IntegrityError):
|
|
session.commit()
|
|
|
|
def test_document_person_role_is_unique_per_document_person(self, session):
|
|
document = _persist_document(session)
|
|
person = _persist_person(session)
|
|
person_role = _persist_person_role(session)
|
|
|
|
first = DocumentPerson(
|
|
document_id=document.id,
|
|
person_id=person.id,
|
|
role_id=person_role.id,
|
|
)
|
|
session.add(first)
|
|
session.commit()
|
|
|
|
duplicate = DocumentPerson(
|
|
document_id=document.id,
|
|
person_id=person.id,
|
|
role_id=person_role.id,
|
|
)
|
|
session.add(duplicate)
|
|
with pytest.raises(IntegrityError):
|
|
session.commit()
|
|
|
|
|
|
class TestJobSourceModel:
|
|
def test_job_source_persists_json_payloads(self, session):
|
|
document = _persist_document(session)
|
|
job = _persist_job(session, document)
|
|
source = _persist_source(session, document)
|
|
job_source = _persist_job_source(
|
|
session,
|
|
job,
|
|
source,
|
|
raw_transcription="Page transcript",
|
|
ai_metadata={"confidence": 0.91, "boxes": [{"x": 1, "y": 2}]},
|
|
raw_api_response={"provider": "test"},
|
|
)
|
|
|
|
fetched = session.get(JobSource, job_source.id)
|
|
assert fetched is not None
|
|
assert fetched.status == JobSourceStatus.PENDING
|
|
assert fetched.ai_metadata == {"confidence": 0.91, "boxes": [{"x": 1, "y": 2}]}
|
|
assert fetched.raw_api_response == {"provider": "test"}
|
|
|
|
|
|
class TestRelationships:
|
|
def test_document_exposes_jobs_sources_and_people(self, session):
|
|
document = _persist_document(session)
|
|
_persist_job(session, document)
|
|
_persist_source(session, document)
|
|
person = _persist_person(session)
|
|
person_role = _persist_person_role(session)
|
|
|
|
link = DocumentPerson(
|
|
document_id=document.id,
|
|
person_id=person.id,
|
|
role_id=person_role.id,
|
|
)
|
|
session.add(link)
|
|
session.commit()
|
|
|
|
session.refresh(document, attribute_names=["jobs", "sources", "document_people"])
|
|
assert len(document.jobs) == 1
|
|
assert len(document.sources) == 1
|
|
assert len(document.document_people) == 1
|
|
|
|
|
|
class TestRegistryModels:
|
|
def test_person_role_normalized_label_is_unique(self, session):
|
|
_persist_person_role(session, label="Mentioned")
|
|
duplicate = PersonRole(label=" mentioned ", normalized_label="mentioned")
|
|
session.add(duplicate)
|
|
with pytest.raises(IntegrityError):
|
|
session.commit()
|
|
|
|
def test_document_type_normalized_label_is_unique(self, session):
|
|
_persist_document_type(session, label="Journal")
|
|
duplicate = DocumentType(label=" journal ", normalized_label="journal")
|
|
session.add(duplicate)
|
|
with pytest.raises(IntegrityError):
|
|
session.commit()
|
|
|
|
|
|
def test_no_relationship_declares_an_implicit_eager_load():
|
|
"""CRIT-02 guard: eager loading is a per-query decision, never a model default."""
|
|
from sqlmodel import SQLModel
|
|
|
|
offenders = {
|
|
f"{mapper.class_.__name__}.{relationship.key}": relationship.lazy
|
|
for mapper in SQLModel._sa_registry.mappers
|
|
for relationship in mapper.relationships
|
|
if relationship.lazy not in {"raise", "noload"}
|
|
}
|
|
assert offenders == {}, f"Relationships must not preload by default: {offenders}"
|