V2 implementation step 1

2026-08-01 15:34:06 -05:00
parent 00ed176ac1
commit c46d1bd0bc
6 changed files with 212 additions and 136 deletions
@@ -1,27 +1,50 @@
-"""SQLModel domain models for the transcription system.
-
-Core V1 lifecycle:
-    Document  ->  one-to-many  ->  Source
-    Document  ->  one-to-many  ->  Job
-    Source    ->  one-to-one?  ->  Revision (optional)
-"""
+"""SQLModel domain models for the V2 transcription system."""

 from datetime import UTC
+from datetime import date
 from datetime import datetime
 from enum import StrEnum
+from typing import Any
 from typing import Optional
 from uuid import UUID
 from uuid import uuid4

+from sqlalchemy import Column
+from sqlalchemy import JSON
 from sqlalchemy import UniqueConstraint
+from sqlalchemy.dialects.postgresql import JSONB
+from sqlalchemy.types import TypeDecorator
 from sqlmodel import Field
 from sqlmodel import Relationship
 from sqlmodel import SQLModel


+class JSONBCompat(TypeDecorator):
+    """JSONB for PostgreSQL and JSON for SQLite/testing backends."""
+
+    impl = JSON
+
+    def load_dialect_impl(self, dialect):
+        if dialect.name == "postgresql":
+            return dialect.type_descriptor(JSONB())
+        return dialect.type_descriptor(JSON())
+
+
 class JobStatus(StrEnum):
    QUEUED = "queued"
    PROCESSING = "processing"
+    COMPLETED = "completed"
+    PARTIAL_SUCCESS = "partial_success"
+    FAILED = "failed"
+
+
+class DocumentPersonRole(StrEnum):
+    AUTHOR = "author"
+    RECIPIENT = "recipient"
+
+
+class JobSourceStatus(StrEnum):
+    PENDING = "pending"
    TRANSCRIBED = "transcribed"
    FAILED = "failed"

@@ -31,34 +54,63 @@ class Document(SQLModel, table=True):

    id: UUID = Field(default_factory=uuid4, primary_key=True)
    name: str
+    document_type: str | None = None
+    document_date: date | None = None
+    document_date_raw: str | None = None
+    location_created: str | None = None
+    notes: str | None = None
+    archive_identifier: str | None = None
+    created_at: datetime = Field(default_factory=lambda: datetime.now(UTC))
+    updated_at: datetime = Field(default_factory=lambda: datetime.now(UTC))

-    # Relationships
    jobs: list["Job"] = Relationship(back_populates="document")
    sources: list["Source"] = Relationship(back_populates="document")
+    document_people: list["DocumentPerson"] = Relationship(back_populates="document")


-class Source(SQLModel, table=True):
-    """A document source (image or PDF)."""
+class Person(SQLModel, table=True):
+    """A historical person linked to one or more documents."""
+
+    id: UUID = Field(default_factory=uuid4, primary_key=True)
+    full_name: str
+    display_name: str | None = None
+    maiden_name: str | None = None
+    birth_date: date | None = None
+    birth_date_raw: str | None = None
+    birth_place: str | None = None
+    death_date: date | None = None
+    death_date_raw: str | None = None
+    death_place: str | None = None
+    biography: str | None = None
+    portrait_path: str | None = None
+    metadata_: dict[str, Any] | None = Field(
+        default=None,
+        sa_column=Column("metadata", JSONBCompat(), nullable=True),
+    )
+    created_at: datetime = Field(default_factory=lambda: datetime.now(UTC))
+    updated_at: datetime = Field(default_factory=lambda: datetime.now(UTC))
+
+    document_people: list["DocumentPerson"] = Relationship(back_populates="person")
+
+
+class DocumentPerson(SQLModel, table=True):
+    """Associates documents with people in a given role."""
+
+    __tablename__ = "document_person"

    id: UUID = Field(default_factory=uuid4, primary_key=True)
    document_id: UUID = Field(foreign_key="document.id")
-    job_id: UUID = Field(foreign_key="job.id")
-    upload_name: str
-    """The filename of the source that was uploaded for transcription."""
-    filename: str
-    """The system generated unique source name."""
-    file_path: str
-    """The location where the sources are stored on the local filesystem."""
-    date_uploaded: datetime = Field(default_factory=lambda: datetime.now(UTC))
+    person_id: UUID = Field(foreign_key="person.id")
+    role: DocumentPersonRole = Field(default=DocumentPersonRole.AUTHOR)
+    created_at: datetime = Field(default_factory=lambda: datetime.now(UTC))

-    # Relationships
-    document: Optional["Document"] = Relationship(back_populates="sources")
-    job: Optional["Job"] = Relationship(back_populates="sources")
-    revision: Optional["Revision"] = Relationship(
-        back_populates="source",
-        sa_relationship_kwargs={"uselist": False},
+    __table_args__ = (
+        UniqueConstraint("document_id", "person_id", "role", name="uq_document_person_role"),
    )

+    document: Optional["Document"] = Relationship(back_populates="document_people")
+    person: Optional["Person"] = Relationship(back_populates="document_people")
+

 class Job(SQLModel, table=True):
    """A transcription job tied to a single document."""
@@ -70,41 +122,61 @@ class Job(SQLModel, table=True):
    date_created: datetime = Field(default_factory=lambda: datetime.now(UTC))
    date_updated: datetime = Field(default_factory=lambda: datetime.now(UTC))
    provider: str | None = None
-    """Name of the transcription provider used to generate this transcript."""
    model: str | None = None
-    """Model identifier used to generate this transcript."""
    prompt_name: str | None = None
-    """Name of the prompt used to generate this transcript."""
-    text: str | None = None
-    """The transcribed text. This may be None if the job failed or is still in progress."""
-    error_detail: str | None = None
-    """Details of any error that occurred during transcription."""

-    # Relationships
    document: Optional["Document"] = Relationship(back_populates="jobs")
-    sources: list["Source"] = Relationship(back_populates="job")
+    job_sources: list["JobSource"] = Relationship(back_populates="job")

    @property
    def filename(self) -> str:
        """Return the filename of the associated source, when available."""
-        if not self.sources:
+        if not self.job_sources:
            return "unknown"
-        return self.sources[0].filename
+        return self.job_sources[0].source.filename if self.job_sources[0].source is not None else "unknown"


-class Revision(SQLModel, table=True):
-    """A revision of a transcription text."""
+class Source(SQLModel, table=True):
+    """A document source image or PDF page."""

    id: UUID = Field(default_factory=uuid4, primary_key=True)
+    document_id: UUID = Field(foreign_key="document.id")
+    page_number: int = Field(ge=1)
+    upload_name: str
+    filename: str
+    file_path: str
+    raw_transcription: str | None = None
+    revised_text: str | None = None
+    date_uploaded: datetime = Field(default_factory=lambda: datetime.now(UTC))
+    date_revised: datetime | None = None
+
+    document: Optional["Document"] = Relationship(back_populates="sources")
+    job_sources: list["JobSource"] = Relationship(back_populates="source")
+
+
+class JobSource(SQLModel, table=True):
+    """A single AI execution record for one source page."""
+
+    __tablename__ = "job_source"
+
+    id: UUID = Field(default_factory=uuid4, primary_key=True)
+    job_id: UUID = Field(foreign_key="job.id")
    source_id: UUID = Field(foreign_key="source.id")
-    """ID for the associated source."""
-    revision: int = Field(default=1, ge=1)
-    """Revision number of this transcription revision, starting at 1."""
-    text: str
-    """The revised text."""
+    status: JobSourceStatus = Field(default=JobSourceStatus.PENDING)
+    raw_transcription: str | None = None
+    ai_metadata: dict[str, Any] | None = Field(default=None, sa_column=Column(JSONBCompat(), nullable=True))
+    raw_api_response: dict[str, Any] | None = Field(default=None, sa_column=Column(JSONBCompat(), nullable=True))
+    error_detail: str | None = None
+    executed_at: datetime = Field(default_factory=lambda: datetime.now(UTC))
+
+    job: Optional["Job"] = Relationship(back_populates="job_sources")
+    source: Optional["Source"] = Relationship(back_populates="job_sources")
+
+
+class Revision(SQLModel):
+    """Temporary compatibility shim for older service imports during the V2 migration."""
+
+    id: UUID = Field(default_factory=uuid4, primary_key=True)
+    source_id: UUID = Field(default_factory=uuid4)
+    text: str | None = None
    date_created: datetime = Field(default_factory=lambda: datetime.now(UTC))
-
-    __table_args__ = (UniqueConstraint("source_id", name="uq_revision_source_id"),)
-
-    # Relationships
-    source: Optional["Source"] = Relationship(back_populates="revision")
@@ -2,9 +2,6 @@ from __future__ import annotations

 import logging

-from sqlalchemy import inspect
-from sqlalchemy import text
-from sqlalchemy.engine import Connection
 from sqlalchemy.ext.asyncio import AsyncEngine
 from sqlmodel import SQLModel
 from sqlmodel import select
@@ -25,7 +22,6 @@ async def create_all(*, engine: AsyncEngine | None = None) -> None:
    active_engine = engine or resolve_engine()
    async with active_engine.begin() as connection:
        await connection.run_sync(SQLModel.metadata.create_all)
-        await connection.run_sync(_ensure_sqlite_compat_columns)
    logger.debug("Database schema bootstrap complete for database_url=%s", active_engine.url)


@@ -38,38 +34,3 @@ async def get_next_queued_job(*, session: AsyncSession) -> Job | None:
        .limit(1)
    )  # fmt: skip
    return result.first()
-
-
-def _ensure_sqlite_compat_columns(connection: Connection) -> None:
-    """Apply lightweight dev/test SQLite compatibility column patches.
-
-    This keeps local bootstrap resilient when models evolve but no full
-    migration tooling is in place yet.
-    """
-    if connection.engine.url.get_backend_name() != "sqlite":
-        return
-
-    inspector = inspect(connection)
-    table_names = set(inspector.get_table_names())
-
-    if "job" in table_names:
-        job_columns = {column["name"] for column in inspector.get_columns("job")}
-        if "retry_count" not in job_columns:
-            connection.execute(text("ALTER TABLE job ADD COLUMN retry_count INTEGER NOT NULL DEFAULT 0"))
-            logger.warning("Applied SQLite compatibility schema patch table=job column=retry_count default=0")
-
-    if "revision" in table_names:
-        revision_columns = {column["name"] for column in inspector.get_columns("revision")}
-        if "source_id" in revision_columns:
-            has_unique_source = False
-            for index in inspector.get_indexes("revision"):
-                if index.get("unique") and index.get("column_names") == ["source_id"]:
-                    has_unique_source = True
-                    break
-            if not has_unique_source:
-                connection.execute(
-                    text("CREATE UNIQUE INDEX IF NOT EXISTS ux_revision_source_id ON revision(source_id)")
-                )
-                logger.warning(
-                    "Applied SQLite compatibility schema patch table=revision unique_index=ux_revision_source_id"
-                )