diff --git a/src/transcription/db/models.py b/src/transcription/db/models.py index f34f38e..d8855ab 100644 --- a/src/transcription/db/models.py +++ b/src/transcription/db/models.py @@ -1,27 +1,50 @@ -"""SQLModel domain models for the transcription system. - -Core V1 lifecycle: - Document -> one-to-many -> Source - Document -> one-to-many -> Job - Source -> one-to-one? -> Revision (optional) -""" +"""SQLModel domain models for the V2 transcription system.""" from datetime import UTC +from datetime import date from datetime import datetime from enum import StrEnum +from typing import Any from typing import Optional from uuid import UUID from uuid import uuid4 +from sqlalchemy import Column +from sqlalchemy import JSON from sqlalchemy import UniqueConstraint +from sqlalchemy.dialects.postgresql import JSONB +from sqlalchemy.types import TypeDecorator from sqlmodel import Field from sqlmodel import Relationship from sqlmodel import SQLModel +class JSONBCompat(TypeDecorator): + """JSONB for PostgreSQL and JSON for SQLite/testing backends.""" + + impl = JSON + + def load_dialect_impl(self, dialect): + if dialect.name == "postgresql": + return dialect.type_descriptor(JSONB()) + return dialect.type_descriptor(JSON()) + + class JobStatus(StrEnum): QUEUED = "queued" PROCESSING = "processing" + COMPLETED = "completed" + PARTIAL_SUCCESS = "partial_success" + FAILED = "failed" + + +class DocumentPersonRole(StrEnum): + AUTHOR = "author" + RECIPIENT = "recipient" + + +class JobSourceStatus(StrEnum): + PENDING = "pending" TRANSCRIBED = "transcribed" FAILED = "failed" @@ -31,34 +54,63 @@ class Document(SQLModel, table=True): id: UUID = Field(default_factory=uuid4, primary_key=True) name: str + document_type: str | None = None + document_date: date | None = None + document_date_raw: str | None = None + location_created: str | None = None + notes: str | None = None + archive_identifier: str | None = None + created_at: datetime = Field(default_factory=lambda: datetime.now(UTC)) + updated_at: datetime = Field(default_factory=lambda: datetime.now(UTC)) - # Relationships jobs: list["Job"] = Relationship(back_populates="document") sources: list["Source"] = Relationship(back_populates="document") + document_people: list["DocumentPerson"] = Relationship(back_populates="document") -class Source(SQLModel, table=True): - """A document source (image or PDF).""" +class Person(SQLModel, table=True): + """A historical person linked to one or more documents.""" + + id: UUID = Field(default_factory=uuid4, primary_key=True) + full_name: str + display_name: str | None = None + maiden_name: str | None = None + birth_date: date | None = None + birth_date_raw: str | None = None + birth_place: str | None = None + death_date: date | None = None + death_date_raw: str | None = None + death_place: str | None = None + biography: str | None = None + portrait_path: str | None = None + metadata_: dict[str, Any] | None = Field( + default=None, + sa_column=Column("metadata", JSONBCompat(), nullable=True), + ) + created_at: datetime = Field(default_factory=lambda: datetime.now(UTC)) + updated_at: datetime = Field(default_factory=lambda: datetime.now(UTC)) + + document_people: list["DocumentPerson"] = Relationship(back_populates="person") + + +class DocumentPerson(SQLModel, table=True): + """Associates documents with people in a given role.""" + + __tablename__ = "document_person" id: UUID = Field(default_factory=uuid4, primary_key=True) document_id: UUID = Field(foreign_key="document.id") - job_id: UUID = Field(foreign_key="job.id") - upload_name: str - """The filename of the source that was uploaded for transcription.""" - filename: str - """The system generated unique source name.""" - file_path: str - """The location where the sources are stored on the local filesystem.""" - date_uploaded: datetime = Field(default_factory=lambda: datetime.now(UTC)) + person_id: UUID = Field(foreign_key="person.id") + role: DocumentPersonRole = Field(default=DocumentPersonRole.AUTHOR) + created_at: datetime = Field(default_factory=lambda: datetime.now(UTC)) - # Relationships - document: Optional["Document"] = Relationship(back_populates="sources") - job: Optional["Job"] = Relationship(back_populates="sources") - revision: Optional["Revision"] = Relationship( - back_populates="source", - sa_relationship_kwargs={"uselist": False}, + __table_args__ = ( + UniqueConstraint("document_id", "person_id", "role", name="uq_document_person_role"), ) + document: Optional["Document"] = Relationship(back_populates="document_people") + person: Optional["Person"] = Relationship(back_populates="document_people") + class Job(SQLModel, table=True): """A transcription job tied to a single document.""" @@ -70,41 +122,61 @@ class Job(SQLModel, table=True): date_created: datetime = Field(default_factory=lambda: datetime.now(UTC)) date_updated: datetime = Field(default_factory=lambda: datetime.now(UTC)) provider: str | None = None - """Name of the transcription provider used to generate this transcript.""" model: str | None = None - """Model identifier used to generate this transcript.""" prompt_name: str | None = None - """Name of the prompt used to generate this transcript.""" - text: str | None = None - """The transcribed text. This may be None if the job failed or is still in progress.""" - error_detail: str | None = None - """Details of any error that occurred during transcription.""" - # Relationships document: Optional["Document"] = Relationship(back_populates="jobs") - sources: list["Source"] = Relationship(back_populates="job") + job_sources: list["JobSource"] = Relationship(back_populates="job") @property def filename(self) -> str: """Return the filename of the associated source, when available.""" - if not self.sources: + if not self.job_sources: return "unknown" - return self.sources[0].filename + return self.job_sources[0].source.filename if self.job_sources[0].source is not None else "unknown" -class Revision(SQLModel, table=True): - """A revision of a transcription text.""" +class Source(SQLModel, table=True): + """A document source image or PDF page.""" id: UUID = Field(default_factory=uuid4, primary_key=True) + document_id: UUID = Field(foreign_key="document.id") + page_number: int = Field(ge=1) + upload_name: str + filename: str + file_path: str + raw_transcription: str | None = None + revised_text: str | None = None + date_uploaded: datetime = Field(default_factory=lambda: datetime.now(UTC)) + date_revised: datetime | None = None + + document: Optional["Document"] = Relationship(back_populates="sources") + job_sources: list["JobSource"] = Relationship(back_populates="source") + + +class JobSource(SQLModel, table=True): + """A single AI execution record for one source page.""" + + __tablename__ = "job_source" + + id: UUID = Field(default_factory=uuid4, primary_key=True) + job_id: UUID = Field(foreign_key="job.id") source_id: UUID = Field(foreign_key="source.id") - """ID for the associated source.""" - revision: int = Field(default=1, ge=1) - """Revision number of this transcription revision, starting at 1.""" - text: str - """The revised text.""" + status: JobSourceStatus = Field(default=JobSourceStatus.PENDING) + raw_transcription: str | None = None + ai_metadata: dict[str, Any] | None = Field(default=None, sa_column=Column(JSONBCompat(), nullable=True)) + raw_api_response: dict[str, Any] | None = Field(default=None, sa_column=Column(JSONBCompat(), nullable=True)) + error_detail: str | None = None + executed_at: datetime = Field(default_factory=lambda: datetime.now(UTC)) + + job: Optional["Job"] = Relationship(back_populates="job_sources") + source: Optional["Source"] = Relationship(back_populates="job_sources") + + +class Revision(SQLModel): + """Temporary compatibility shim for older service imports during the V2 migration.""" + + id: UUID = Field(default_factory=uuid4, primary_key=True) + source_id: UUID = Field(default_factory=uuid4) + text: str | None = None date_created: datetime = Field(default_factory=lambda: datetime.now(UTC)) - - __table_args__ = (UniqueConstraint("source_id", name="uq_revision_source_id"),) - - # Relationships - source: Optional["Source"] = Relationship(back_populates="revision") diff --git a/src/transcription/db/operations.py b/src/transcription/db/operations.py index 6562448..7663745 100644 --- a/src/transcription/db/operations.py +++ b/src/transcription/db/operations.py @@ -2,9 +2,6 @@ from __future__ import annotations import logging -from sqlalchemy import inspect -from sqlalchemy import text -from sqlalchemy.engine import Connection from sqlalchemy.ext.asyncio import AsyncEngine from sqlmodel import SQLModel from sqlmodel import select @@ -25,7 +22,6 @@ async def create_all(*, engine: AsyncEngine | None = None) -> None: active_engine = engine or resolve_engine() async with active_engine.begin() as connection: await connection.run_sync(SQLModel.metadata.create_all) - await connection.run_sync(_ensure_sqlite_compat_columns) logger.debug("Database schema bootstrap complete for database_url=%s", active_engine.url) @@ -38,38 +34,3 @@ async def get_next_queued_job(*, session: AsyncSession) -> Job | None: .limit(1) ) # fmt: skip return result.first() - - -def _ensure_sqlite_compat_columns(connection: Connection) -> None: - """Apply lightweight dev/test SQLite compatibility column patches. - - This keeps local bootstrap resilient when models evolve but no full - migration tooling is in place yet. - """ - if connection.engine.url.get_backend_name() != "sqlite": - return - - inspector = inspect(connection) - table_names = set(inspector.get_table_names()) - - if "job" in table_names: - job_columns = {column["name"] for column in inspector.get_columns("job")} - if "retry_count" not in job_columns: - connection.execute(text("ALTER TABLE job ADD COLUMN retry_count INTEGER NOT NULL DEFAULT 0")) - logger.warning("Applied SQLite compatibility schema patch table=job column=retry_count default=0") - - if "revision" in table_names: - revision_columns = {column["name"] for column in inspector.get_columns("revision")} - if "source_id" in revision_columns: - has_unique_source = False - for index in inspector.get_indexes("revision"): - if index.get("unique") and index.get("column_names") == ["source_id"]: - has_unique_source = True - break - if not has_unique_source: - connection.execute( - text("CREATE UNIQUE INDEX IF NOT EXISTS ux_revision_source_id ON revision(source_id)") - ) - logger.warning( - "Applied SQLite compatibility schema patch table=revision unique_index=ux_revision_source_id" - ) diff --git a/tests/test_db.py b/tests/test_db.py index 41c1f42..eb0079e 100644 --- a/tests/test_db.py +++ b/tests/test_db.py @@ -1,4 +1,4 @@ -"""Tests for transcription.db runtime and schema bootstrap behavior.""" +"""Tests for the database runtime and V2 schema bootstrap behavior.""" import pytest from sqlalchemy import inspect @@ -26,9 +26,12 @@ async def test_create_all_creates_expected_tables(tmp_path): table_names = set(await conn.run_sync(lambda c: inspect(c).get_table_names())) assert "document" in table_names + assert "person" in table_names + assert "document_person" in table_names assert "job" in table_names assert "source" in table_names - assert "revision" in table_names + assert "job_source" in table_names + assert "revision" not in table_names finally: await dispose_database_runtime() diff --git a/tests/test_models.py b/tests/test_models.py index 480649e..40c53e0 100644 --- a/tests/test_models.py +++ b/tests/test_models.py @@ -1,19 +1,29 @@ -"""Tests for transcription.models — Document, Source, Job, Revision persistence and relationships.""" +"""Tests for the V2 SQLModel persistence layer and relationships.""" +from datetime import UTC +from datetime import datetime from uuid import UUID import pytest from sqlalchemy.exc import IntegrityError from transcription.db.models import Document +from transcription.db.models import DocumentPerson +from transcription.db.models import DocumentPersonRole from transcription.db.models import Job +from transcription.db.models import JobSource +from transcription.db.models import JobSourceStatus from transcription.db.models import JobStatus -from transcription.db.models import Revision +from transcription.db.models import Person from transcription.db.models import Source def _make_document(**overrides) -> Document: - defaults = {"name": "letter bundle"} + defaults = { + "name": "letter bundle", + "document_type": "letter", + "notes": "Family correspondence", + } defaults.update(overrides) return Document(**defaults) @@ -26,6 +36,16 @@ def _persist_document(session) -> Document: return document +def _persist_person(session, **overrides) -> Person: + defaults = {"full_name": "Ada Lovelace"} + defaults.update(overrides) + person = Person(**defaults) + session.add(person) + session.commit() + session.refresh(person) + return person + + def _persist_job(session, document: Document) -> Job: job = Job(document_id=document.id) session.add(job) @@ -34,13 +54,14 @@ def _persist_job(session, document: Document) -> Job: return job -def _persist_source(session, document: Document, job: Job, **overrides) -> Source: +def _persist_source(session, document: Document, *, page_number: int = 1, **overrides) -> Source: defaults = { "document_id": document.id, - "job_id": job.id, + "page_number": page_number, "upload_name": "letter.jpg", "filename": "stored-letter.jpg", "file_path": "/uploads/stored-letter.jpg", + "raw_transcription": "Original machine text", } defaults.update(overrides) source = Source(**defaults) @@ -50,6 +71,20 @@ def _persist_source(session, document: Document, job: Job, **overrides) -> Sourc return source +def _persist_job_source(session, job: Job, source: Source, **overrides) -> JobSource: + defaults = { + "job_id": job.id, + "source_id": source.id, + "status": JobSourceStatus.PENDING, + } + defaults.update(overrides) + job_source = JobSource(**defaults) + session.add(job_source) + session.commit() + session.refresh(job_source) + return job_source + + class TestDocumentModel: def test_can_be_persisted(self, session): document = _persist_document(session) @@ -60,6 +95,8 @@ class TestDocumentModel: def test_defaults_are_populated(self, session): document = _persist_document(session) assert isinstance(document.id, UUID) + assert document.created_at is not None + assert document.updated_at is not None class TestJobModel: @@ -78,7 +115,7 @@ class TestJobModel: assert job.date_created is not None assert job.date_updated is not None - def test_transitions_to_transcribed(self, session): + def test_transitions_to_completed(self, session): document = _persist_document(session) job = _persist_job(session, document) @@ -87,79 +124,82 @@ class TestJobModel: session.commit() session.refresh(job) - job.status = JobStatus.TRANSCRIBED + job.status = JobStatus.COMPLETED session.add(job) session.commit() session.refresh(job) - assert job.status == JobStatus.TRANSCRIBED + assert job.status == JobStatus.COMPLETED class TestSourceModel: - def test_can_be_created_for_document_and_job(self, session): + def test_can_be_created_for_document(self, session): document = _persist_document(session) - job = _persist_job(session, document) - source = _persist_source(session, document, job) + source = _persist_source(session, document) fetched = session.get(Source, source.id) assert fetched is not None assert fetched.document_id == document.id - assert fetched.job_id == job.id + assert fetched.page_number == 1 assert fetched.date_uploaded is not None - -class TestRevisionModel: - def test_revision_persists_for_source(self, session): + def test_revised_text_is_supported(self, session): document = _persist_document(session) - job = _persist_job(session, document) - source = _persist_source(session, document, job) + source = _persist_source(session, document, revised_text="Edited output") - revision = Revision(source_id=source.id, text="Edited revision text") - session.add(revision) - session.commit() - session.refresh(revision) - - fetched = session.get(Revision, revision.id) + fetched = session.get(Source, source.id) assert fetched is not None - assert fetched.text == "Edited revision text" - assert fetched.date_created is not None + assert fetched.revised_text == "Edited output" - def test_source_id_is_unique(self, session): + +class TestPersonAndDocumentPersonModel: + def test_document_person_role_is_unique_per_document_person(self, session): document = _persist_document(session) - job = _persist_job(session, document) - source = _persist_source(session, document, job) + person = _persist_person(session) - first = Revision(source_id=source.id, text="First") + first = DocumentPerson(document_id=document.id, person_id=person.id, role=DocumentPersonRole.AUTHOR) session.add(first) session.commit() - duplicate = Revision(source_id=source.id, text="Duplicate") + duplicate = DocumentPerson(document_id=document.id, person_id=person.id, role=DocumentPersonRole.AUTHOR) session.add(duplicate) with pytest.raises(IntegrityError): session.commit() -class TestRelationships: - def test_document_exposes_jobs_and_sources(self, session): +class TestJobSourceModel: + def test_job_source_persists_json_payloads(self, session): document = _persist_document(session) job = _persist_job(session, document) - _persist_source(session, document, job) + source = _persist_source(session, document) + job_source = _persist_job_source( + session, + job, + source, + raw_transcription="Page transcript", + ai_metadata={"confidence": 0.91, "boxes": [{"x": 1, "y": 2}]}, + raw_api_response={"provider": "test"}, + ) + + fetched = session.get(JobSource, job_source.id) + assert fetched is not None + assert fetched.status == JobSourceStatus.PENDING + assert fetched.ai_metadata == {"confidence": 0.91, "boxes": [{"x": 1, "y": 2}]} + assert fetched.raw_api_response == {"provider": "test"} + + +class TestRelationships: + def test_document_exposes_jobs_sources_and_people(self, session): + document = _persist_document(session) + _persist_job(session, document) + _persist_source(session, document) + person = _persist_person(session) + + link = DocumentPerson(document_id=document.id, person_id=person.id, role=DocumentPersonRole.AUTHOR) + session.add(link) + session.commit() session.refresh(document) assert len(document.jobs) == 1 assert len(document.sources) == 1 - - def test_source_exposes_optional_single_revision(self, session): - document = _persist_document(session) - job = _persist_job(session, document) - source = _persist_source(session, document, job) - - assert source.revision is None - - revision = Revision(source_id=source.id, text="Edited") - session.add(revision) - session.commit() - - session.refresh(source) - assert source.revision is not None - assert source.revision.text == "Edited" + assert len(document.document_people) == 1 diff --git a/uploads/7704f9cc-4b04-494b-8c55-0ea720249a54_Biography of Omie Rachel Cochran McCarthy_ 1892-1961.pdf b/uploads/7704f9cc-4b04-494b-8c55-0ea720249a54_Biography of Omie Rachel Cochran McCarthy_ 1892-1961.pdf deleted file mode 100644 index 0ac4810..0000000 Binary files a/uploads/7704f9cc-4b04-494b-8c55-0ea720249a54_Biography of Omie Rachel Cochran McCarthy_ 1892-1961.pdf and /dev/null differ diff --git a/uploads/ca87e22d-21bf-4af8-8cff-312e9384bd24_Effie Ruth Cochran Montague.pdf b/uploads/ca87e22d-21bf-4af8-8cff-312e9384bd24_Effie Ruth Cochran Montague.pdf deleted file mode 100644 index 4264f8f..0000000 Binary files a/uploads/ca87e22d-21bf-4af8-8cff-312e9384bd24_Effie Ruth Cochran Montague.pdf and /dev/null differ