V2 implementation step 1

src/transcription/db/models.py

@@ -1,27 +1,50 @@
-"""SQLModel domain models for the transcription system.
+"""SQLModel domain models for the V2 transcription system."""
-
-Core V1 lifecycle:
-    Document  ->  one-to-many  ->  Source
-    Document  ->  one-to-many  ->  Job
-    Source    ->  one-to-one?  ->  Revision (optional)
-"""
 
 from datetime import UTC
+from datetime import date
 from datetime import datetime
 from enum import StrEnum
+from typing import Any
 from typing import Optional
 from uuid import UUID
 from uuid import uuid4
 
+from sqlalchemy import Column
+from sqlalchemy import JSON
 from sqlalchemy import UniqueConstraint
+from sqlalchemy.dialects.postgresql import JSONB
+from sqlalchemy.types import TypeDecorator
 from sqlmodel import Field
 from sqlmodel import Relationship
 from sqlmodel import SQLModel
 
 
+class JSONBCompat(TypeDecorator):
+    """JSONB for PostgreSQL and JSON for SQLite/testing backends."""
+
+    impl = JSON
+
+    def load_dialect_impl(self, dialect):
+        if dialect.name == "postgresql":
+            return dialect.type_descriptor(JSONB())
+        return dialect.type_descriptor(JSON())
+
+
 class JobStatus(StrEnum):
     QUEUED = "queued"
     PROCESSING = "processing"
+    COMPLETED = "completed"
+    PARTIAL_SUCCESS = "partial_success"
+    FAILED = "failed"
+
+
+class DocumentPersonRole(StrEnum):
+    AUTHOR = "author"
+    RECIPIENT = "recipient"
+
+
+class JobSourceStatus(StrEnum):
+    PENDING = "pending"
     TRANSCRIBED = "transcribed"
     FAILED = "failed"
 
@@ -31,34 +54,63 @@ class Document(SQLModel, table=True):
 
     id: UUID = Field(default_factory=uuid4, primary_key=True)
     name: str
+    document_type: str | None = None
+    document_date: date | None = None
+    document_date_raw: str | None = None
+    location_created: str | None = None
+    notes: str | None = None
+    archive_identifier: str | None = None
+    created_at: datetime = Field(default_factory=lambda: datetime.now(UTC))
+    updated_at: datetime = Field(default_factory=lambda: datetime.now(UTC))
 
-    # Relationships
     jobs: list["Job"] = Relationship(back_populates="document")
     sources: list["Source"] = Relationship(back_populates="document")
+    document_people: list["DocumentPerson"] = Relationship(back_populates="document")
 
 
-class Source(SQLModel, table=True):
+class Person(SQLModel, table=True):
-    """A document source (image or PDF)."""
+    """A historical person linked to one or more documents."""
+
+    id: UUID = Field(default_factory=uuid4, primary_key=True)
+    full_name: str
+    display_name: str | None = None
+    maiden_name: str | None = None
+    birth_date: date | None = None
+    birth_date_raw: str | None = None
+    birth_place: str | None = None
+    death_date: date | None = None
+    death_date_raw: str | None = None
+    death_place: str | None = None
+    biography: str | None = None
+    portrait_path: str | None = None
+    metadata_: dict[str, Any] | None = Field(
+        default=None,
+        sa_column=Column("metadata", JSONBCompat(), nullable=True),
+    )
+    created_at: datetime = Field(default_factory=lambda: datetime.now(UTC))
+    updated_at: datetime = Field(default_factory=lambda: datetime.now(UTC))
+
+    document_people: list["DocumentPerson"] = Relationship(back_populates="person")
+
+
+class DocumentPerson(SQLModel, table=True):
+    """Associates documents with people in a given role."""
+
+    __tablename__ = "document_person"
 
     id: UUID = Field(default_factory=uuid4, primary_key=True)
     document_id: UUID = Field(foreign_key="document.id")
-    job_id: UUID = Field(foreign_key="job.id")
+    person_id: UUID = Field(foreign_key="person.id")
-    upload_name: str
+    role: DocumentPersonRole = Field(default=DocumentPersonRole.AUTHOR)
-    """The filename of the source that was uploaded for transcription."""
+    created_at: datetime = Field(default_factory=lambda: datetime.now(UTC))
-    filename: str
-    """The system generated unique source name."""
-    file_path: str
-    """The location where the sources are stored on the local filesystem."""
-    date_uploaded: datetime = Field(default_factory=lambda: datetime.now(UTC))
 
-    # Relationships
+    __table_args__ = (
-    document: Optional["Document"] = Relationship(back_populates="sources")
+        UniqueConstraint("document_id", "person_id", "role", name="uq_document_person_role"),
-    job: Optional["Job"] = Relationship(back_populates="sources")
-    revision: Optional["Revision"] = Relationship(
-        back_populates="source",
-        sa_relationship_kwargs={"uselist": False},
     )
 
+    document: Optional["Document"] = Relationship(back_populates="document_people")
+    person: Optional["Person"] = Relationship(back_populates="document_people")
+
 
 class Job(SQLModel, table=True):
     """A transcription job tied to a single document."""
@@ -70,41 +122,61 @@ class Job(SQLModel, table=True):
     date_created: datetime = Field(default_factory=lambda: datetime.now(UTC))
     date_updated: datetime = Field(default_factory=lambda: datetime.now(UTC))
     provider: str | None = None
-    """Name of the transcription provider used to generate this transcript."""
     model: str | None = None
-    """Model identifier used to generate this transcript."""
     prompt_name: str | None = None
-    """Name of the prompt used to generate this transcript."""
-    text: str | None = None
-    """The transcribed text. This may be None if the job failed or is still in progress."""
-    error_detail: str | None = None
-    """Details of any error that occurred during transcription."""
 
-    # Relationships
     document: Optional["Document"] = Relationship(back_populates="jobs")
-    sources: list["Source"] = Relationship(back_populates="job")
+    job_sources: list["JobSource"] = Relationship(back_populates="job")
 
     @property
     def filename(self) -> str:
         """Return the filename of the associated source, when available."""
-        if not self.sources:
+        if not self.job_sources:
             return "unknown"
-        return self.sources[0].filename
+        return self.job_sources[0].source.filename if self.job_sources[0].source is not None else "unknown"
 
 
-class Revision(SQLModel, table=True):
+class Source(SQLModel, table=True):
-    """A revision of a transcription text."""
+    """A document source image or PDF page."""
 
     id: UUID = Field(default_factory=uuid4, primary_key=True)
+    document_id: UUID = Field(foreign_key="document.id")
+    page_number: int = Field(ge=1)
+    upload_name: str
+    filename: str
+    file_path: str
+    raw_transcription: str | None = None
+    revised_text: str | None = None
+    date_uploaded: datetime = Field(default_factory=lambda: datetime.now(UTC))
+    date_revised: datetime | None = None
+
+    document: Optional["Document"] = Relationship(back_populates="sources")
+    job_sources: list["JobSource"] = Relationship(back_populates="source")
+
+
+class JobSource(SQLModel, table=True):
+    """A single AI execution record for one source page."""
+
+    __tablename__ = "job_source"
+
+    id: UUID = Field(default_factory=uuid4, primary_key=True)
+    job_id: UUID = Field(foreign_key="job.id")
     source_id: UUID = Field(foreign_key="source.id")
-    """ID for the associated source."""
+    status: JobSourceStatus = Field(default=JobSourceStatus.PENDING)
-    revision: int = Field(default=1, ge=1)
+    raw_transcription: str | None = None
-    """Revision number of this transcription revision, starting at 1."""
+    ai_metadata: dict[str, Any] | None = Field(default=None, sa_column=Column(JSONBCompat(), nullable=True))
-    text: str
+    raw_api_response: dict[str, Any] | None = Field(default=None, sa_column=Column(JSONBCompat(), nullable=True))
-    """The revised text."""
+    error_detail: str | None = None
+    executed_at: datetime = Field(default_factory=lambda: datetime.now(UTC))
+
+    job: Optional["Job"] = Relationship(back_populates="job_sources")
+    source: Optional["Source"] = Relationship(back_populates="job_sources")
+
+
+class Revision(SQLModel):
+    """Temporary compatibility shim for older service imports during the V2 migration."""
+
+    id: UUID = Field(default_factory=uuid4, primary_key=True)
+    source_id: UUID = Field(default_factory=uuid4)
+    text: str | None = None
     date_created: datetime = Field(default_factory=lambda: datetime.now(UTC))
-
-    __table_args__ = (UniqueConstraint("source_id", name="uq_revision_source_id"),)
-
-    # Relationships
-    source: Optional["Source"] = Relationship(back_populates="revision")

src/transcription/db/operations.py

@@ -2,9 +2,6 @@ from __future__ import annotations
 
 import logging
 
-from sqlalchemy import inspect
-from sqlalchemy import text
-from sqlalchemy.engine import Connection
 from sqlalchemy.ext.asyncio import AsyncEngine
 from sqlmodel import SQLModel
 from sqlmodel import select
@@ -25,7 +22,6 @@ async def create_all(*, engine: AsyncEngine | None = None) -> None:
     active_engine = engine or resolve_engine()
     async with active_engine.begin() as connection:
         await connection.run_sync(SQLModel.metadata.create_all)
-        await connection.run_sync(_ensure_sqlite_compat_columns)
     logger.debug("Database schema bootstrap complete for database_url=%s", active_engine.url)
 
 
@@ -38,38 +34,3 @@ async def get_next_queued_job(*, session: AsyncSession) -> Job | None:
         .limit(1)
     )  # fmt: skip
     return result.first()
-
-
-def _ensure_sqlite_compat_columns(connection: Connection) -> None:
-    """Apply lightweight dev/test SQLite compatibility column patches.
-
-    This keeps local bootstrap resilient when models evolve but no full
-    migration tooling is in place yet.
-    """
-    if connection.engine.url.get_backend_name() != "sqlite":
-        return
-
-    inspector = inspect(connection)
-    table_names = set(inspector.get_table_names())
-
-    if "job" in table_names:
-        job_columns = {column["name"] for column in inspector.get_columns("job")}
-        if "retry_count" not in job_columns:
-            connection.execute(text("ALTER TABLE job ADD COLUMN retry_count INTEGER NOT NULL DEFAULT 0"))
-            logger.warning("Applied SQLite compatibility schema patch table=job column=retry_count default=0")
-
-    if "revision" in table_names:
-        revision_columns = {column["name"] for column in inspector.get_columns("revision")}
-        if "source_id" in revision_columns:
-            has_unique_source = False
-            for index in inspector.get_indexes("revision"):
-                if index.get("unique") and index.get("column_names") == ["source_id"]:
-                    has_unique_source = True
-                    break
-            if not has_unique_source:
-                connection.execute(
-                    text("CREATE UNIQUE INDEX IF NOT EXISTS ux_revision_source_id ON revision(source_id)")
-                )
-                logger.warning(
-                    "Applied SQLite compatibility schema patch table=revision unique_index=ux_revision_source_id"
-                )

tests/test_db.py

@@ -1,4 +1,4 @@
-"""Tests for transcription.db runtime and schema bootstrap behavior."""
+"""Tests for the database runtime and V2 schema bootstrap behavior."""
 
 import pytest
 from sqlalchemy import inspect
@@ -26,9 +26,12 @@ async def test_create_all_creates_expected_tables(tmp_path):
             table_names = set(await conn.run_sync(lambda c: inspect(c).get_table_names()))
 
         assert "document" in table_names
+        assert "person" in table_names
+        assert "document_person" in table_names
         assert "job" in table_names
         assert "source" in table_names
-        assert "revision" in table_names
+        assert "job_source" in table_names
+        assert "revision" not in table_names
     finally:
         await dispose_database_runtime()
 

tests/test_models.py

@@ -1,19 +1,29 @@
-"""Tests for transcription.models — Document, Source, Job, Revision persistence and relationships."""
+"""Tests for the V2 SQLModel persistence layer and relationships."""
 
+from datetime import UTC
+from datetime import datetime
 from uuid import UUID
 
 import pytest
 from sqlalchemy.exc import IntegrityError
 
 from transcription.db.models import Document
+from transcription.db.models import DocumentPerson
+from transcription.db.models import DocumentPersonRole
 from transcription.db.models import Job
+from transcription.db.models import JobSource
+from transcription.db.models import JobSourceStatus
 from transcription.db.models import JobStatus
-from transcription.db.models import Revision
+from transcription.db.models import Person
 from transcription.db.models import Source
 
 
 def _make_document(**overrides) -> Document:
-    defaults = {"name": "letter bundle"}
+    defaults = {
+        "name": "letter bundle",
+        "document_type": "letter",
+        "notes": "Family correspondence",
+    }
     defaults.update(overrides)
     return Document(**defaults)
 
@@ -26,6 +36,16 @@ def _persist_document(session) -> Document:
     return document
 
 
+def _persist_person(session, **overrides) -> Person:
+    defaults = {"full_name": "Ada Lovelace"}
+    defaults.update(overrides)
+    person = Person(**defaults)
+    session.add(person)
+    session.commit()
+    session.refresh(person)
+    return person
+
+
 def _persist_job(session, document: Document) -> Job:
     job = Job(document_id=document.id)
     session.add(job)
@@ -34,13 +54,14 @@ def _persist_job(session, document: Document) -> Job:
     return job
 
 
-def _persist_source(session, document: Document, job: Job, **overrides) -> Source:
+def _persist_source(session, document: Document, *, page_number: int = 1, **overrides) -> Source:
     defaults = {
         "document_id": document.id,
-        "job_id": job.id,
+        "page_number": page_number,
         "upload_name": "letter.jpg",
         "filename": "stored-letter.jpg",
         "file_path": "/uploads/stored-letter.jpg",
+        "raw_transcription": "Original machine text",
     }
     defaults.update(overrides)
     source = Source(**defaults)
@@ -50,6 +71,20 @@ def _persist_source(session, document: Document, job: Job, **overrides) -> Sourc
     return source
 
 
+def _persist_job_source(session, job: Job, source: Source, **overrides) -> JobSource:
+    defaults = {
+        "job_id": job.id,
+        "source_id": source.id,
+        "status": JobSourceStatus.PENDING,
+    }
+    defaults.update(overrides)
+    job_source = JobSource(**defaults)
+    session.add(job_source)
+    session.commit()
+    session.refresh(job_source)
+    return job_source
+
+
 class TestDocumentModel:
     def test_can_be_persisted(self, session):
         document = _persist_document(session)
@@ -60,6 +95,8 @@ class TestDocumentModel:
     def test_defaults_are_populated(self, session):
         document = _persist_document(session)
         assert isinstance(document.id, UUID)
+        assert document.created_at is not None
+        assert document.updated_at is not None
 
 
 class TestJobModel:
@@ -78,7 +115,7 @@ class TestJobModel:
         assert job.date_created is not None
         assert job.date_updated is not None
 
-    def test_transitions_to_transcribed(self, session):
+    def test_transitions_to_completed(self, session):
         document = _persist_document(session)
         job = _persist_job(session, document)
 
@@ -87,79 +124,82 @@ class TestJobModel:
         session.commit()
         session.refresh(job)
 
-        job.status = JobStatus.TRANSCRIBED
+        job.status = JobStatus.COMPLETED
         session.add(job)
         session.commit()
         session.refresh(job)
 
-        assert job.status == JobStatus.TRANSCRIBED
+        assert job.status == JobStatus.COMPLETED
 
 
 class TestSourceModel:
-    def test_can_be_created_for_document_and_job(self, session):
+    def test_can_be_created_for_document(self, session):
         document = _persist_document(session)
-        job = _persist_job(session, document)
+        source = _persist_source(session, document)
-        source = _persist_source(session, document, job)
 
         fetched = session.get(Source, source.id)
         assert fetched is not None
         assert fetched.document_id == document.id
-        assert fetched.job_id == job.id
+        assert fetched.page_number == 1
         assert fetched.date_uploaded is not None
 
-
+    def test_revised_text_is_supported(self, session):
-class TestRevisionModel:
-    def test_revision_persists_for_source(self, session):
         document = _persist_document(session)
-        job = _persist_job(session, document)
+        source = _persist_source(session, document, revised_text="Edited output")
-        source = _persist_source(session, document, job)
 
-        revision = Revision(source_id=source.id, text="Edited revision text")
+        fetched = session.get(Source, source.id)
-        session.add(revision)
-        session.commit()
-        session.refresh(revision)
-
-        fetched = session.get(Revision, revision.id)
         assert fetched is not None
-        assert fetched.text == "Edited revision text"
+        assert fetched.revised_text == "Edited output"
-        assert fetched.date_created is not None
 
-    def test_source_id_is_unique(self, session):
+
+class TestPersonAndDocumentPersonModel:
+    def test_document_person_role_is_unique_per_document_person(self, session):
         document = _persist_document(session)
-        job = _persist_job(session, document)
+        person = _persist_person(session)
-        source = _persist_source(session, document, job)
 
-        first = Revision(source_id=source.id, text="First")
+        first = DocumentPerson(document_id=document.id, person_id=person.id, role=DocumentPersonRole.AUTHOR)
         session.add(first)
         session.commit()
 
-        duplicate = Revision(source_id=source.id, text="Duplicate")
+        duplicate = DocumentPerson(document_id=document.id, person_id=person.id, role=DocumentPersonRole.AUTHOR)
         session.add(duplicate)
         with pytest.raises(IntegrityError):
             session.commit()
 
 
-class TestRelationships:
+class TestJobSourceModel:
-    def test_document_exposes_jobs_and_sources(self, session):
+    def test_job_source_persists_json_payloads(self, session):
         document = _persist_document(session)
         job = _persist_job(session, document)
-        _persist_source(session, document, job)
+        source = _persist_source(session, document)
+        job_source = _persist_job_source(
+            session,
+            job,
+            source,
+            raw_transcription="Page transcript",
+            ai_metadata={"confidence": 0.91, "boxes": [{"x": 1, "y": 2}]},
+            raw_api_response={"provider": "test"},
+        )
+
+        fetched = session.get(JobSource, job_source.id)
+        assert fetched is not None
+        assert fetched.status == JobSourceStatus.PENDING
+        assert fetched.ai_metadata == {"confidence": 0.91, "boxes": [{"x": 1, "y": 2}]}
+        assert fetched.raw_api_response == {"provider": "test"}
+
+
+class TestRelationships:
+    def test_document_exposes_jobs_sources_and_people(self, session):
+        document = _persist_document(session)
+        _persist_job(session, document)
+        _persist_source(session, document)
+        person = _persist_person(session)
+
+        link = DocumentPerson(document_id=document.id, person_id=person.id, role=DocumentPersonRole.AUTHOR)
+        session.add(link)
+        session.commit()
 
         session.refresh(document)
         assert len(document.jobs) == 1
         assert len(document.sources) == 1
-
+        assert len(document.document_people) == 1
-    def test_source_exposes_optional_single_revision(self, session):
-        document = _persist_document(session)
-        job = _persist_job(session, document)
-        source = _persist_source(session, document, job)
-
-        assert source.revision is None
-
-        revision = Revision(source_id=source.id, text="Edited")
-        session.add(revision)
-        session.commit()
-
-        session.refresh(source)
-        assert source.revision is not None
-        assert source.revision.text == "Edited"