From bbf7fe28c2ac8d9714fbfb7d3fd9a3da295ff394 Mon Sep 17 00:00:00 2001 From: John Lancaster <32917998+jsl12@users.noreply.github.com> Date: Fri, 31 Jul 2026 10:20:02 -0500 Subject: [PATCH] cleanup --- docs/V2 Python Pydantic Models.md | 408 ------------------------- docs/ver2/V2 Python Pydantic Models.md | 12 +- 2 files changed, 10 insertions(+), 410 deletions(-) delete mode 100644 docs/V2 Python Pydantic Models.md diff --git a/docs/V2 Python Pydantic Models.md b/docs/V2 Python Pydantic Models.md deleted file mode 100644 index 816f1d8..0000000 --- a/docs/V2 Python Pydantic Models.md +++ /dev/null @@ -1,408 +0,0 @@ -# SQLModel Table Models - -Each V2 table is represented by one `SQLModel` class. Because `SQLModel` is built on Pydantic and SQLAlchemy, these classes provide application validation and PostgreSQL mappings without parallel row and create models. - -Database-generated UUIDs and timestamps are `None` until PostgreSQL supplies their values during insert. The database columns remain non-nullable. `Person.metadata_` maps to the `metadata` column because `metadata` is reserved by SQLAlchemy's declarative API. - -```python -from datetime import date -from datetime import datetime -from enum import StrEnum -from uuid import UUID - -from pydantic import JsonValue -from sqlalchemy import Column -from sqlalchemy import Date -from sqlalchemy import DateTime -from sqlalchemy import ForeignKey -from sqlalchemy import Index -from sqlalchemy import Integer -from sqlalchemy import String -from sqlalchemy import Text -from sqlalchemy import UniqueConstraint -from sqlalchemy import text -from sqlalchemy.dialects.postgresql import JSONB -from sqlalchemy.dialects.postgresql import UUID as PostgreSQLUUID -from sqlmodel import Field -from sqlmodel import Relationship -from sqlmodel import SQLModel - - -class PersonRole(StrEnum): - AUTHOR = "author" - RECIPIENT = "recipient" - - -class JobStatus(StrEnum): - QUEUED = "queued" - PROCESSING = "processing" - COMPLETED = "completed" - PARTIAL_SUCCESS = "partial_success" - FAILED = "failed" - - -class JobSourceStatus(StrEnum): - PENDING = "pending" - TRANSCRIBED = "transcribed" - FAILED = "failed" - - -class Person(SQLModel, table=True): - __tablename__ = "person" - __table_args__ = (Index("idx_person_full_name", "full_name"),) - - id: UUID | None = Field( - default=None, - sa_column=Column( - PostgreSQLUUID(as_uuid=True), - primary_key=True, - server_default=text("gen_random_uuid()"), - ), - ) - full_name: str = Field(sa_column=Column(Text, nullable=False)) - display_name: str | None = Field(default=None, sa_column=Column(Text)) - maiden_name: str | None = Field(default=None, sa_column=Column(Text)) - birth_date: date | None = Field(default=None, sa_column=Column(Date)) - birth_date_raw: str | None = Field(default=None, sa_column=Column(Text)) - birth_place: str | None = Field(default=None, sa_column=Column(Text)) - death_date: date | None = Field(default=None, sa_column=Column(Date)) - death_date_raw: str | None = Field(default=None, sa_column=Column(Text)) - death_place: str | None = Field(default=None, sa_column=Column(Text)) - biography: str | None = Field(default=None, sa_column=Column(Text)) - portrait_path: str | None = Field(default=None, sa_column=Column(Text)) - metadata_: JsonValue | None = Field( - default_factory=dict, - sa_column=Column( - "metadata", - JSONB, - server_default=text("'{}'::jsonb"), - ), - ) - created_at: datetime | None = Field( - default=None, - sa_column=Column( - DateTime(timezone=True), - nullable=False, - server_default=text("now()"), - ), - ) - updated_at: datetime | None = Field( - default=None, - sa_column=Column( - DateTime(timezone=True), - nullable=False, - server_default=text("now()"), - ), - ) - - document_people: list["DocumentPerson"] = Relationship( - back_populates="person", - sa_relationship_kwargs={"lazy": "raise", "passive_deletes": True}, - ) - - -class Document(SQLModel, table=True): - __tablename__ = "document" - __table_args__ = (Index("idx_document_date", "document_date"),) - - id: UUID | None = Field( - default=None, - sa_column=Column( - PostgreSQLUUID(as_uuid=True), - primary_key=True, - server_default=text("gen_random_uuid()"), - ), - ) - name: str = Field(sa_column=Column(Text, nullable=False)) - document_type: str | None = Field(default=None, sa_column=Column(Text)) - document_date: date | None = Field(default=None, sa_column=Column(Date)) - document_date_raw: str | None = Field(default=None, sa_column=Column(Text)) - location_created: str | None = Field(default=None, sa_column=Column(Text)) - notes: str | None = Field(default=None, sa_column=Column(Text)) - archive_identifier: str | None = Field(default=None, sa_column=Column(Text)) - created_at: datetime | None = Field( - default=None, - sa_column=Column( - DateTime(timezone=True), - nullable=False, - server_default=text("now()"), - ), - ) - updated_at: datetime | None = Field( - default=None, - sa_column=Column( - DateTime(timezone=True), - nullable=False, - server_default=text("now()"), - ), - ) - - document_people: list["DocumentPerson"] = Relationship( - back_populates="document", - sa_relationship_kwargs={"lazy": "raise", "passive_deletes": True}, - ) - jobs: list["Job"] = Relationship( - back_populates="document", - sa_relationship_kwargs={"lazy": "raise", "passive_deletes": True}, - ) - sources: list["Source"] = Relationship( - back_populates="document", - sa_relationship_kwargs={"lazy": "raise", "passive_deletes": True}, - ) - - -class DocumentPerson(SQLModel, table=True): - __tablename__ = "document_person" - __table_args__ = ( - UniqueConstraint( - "document_id", - "person_id", - "role", - name="unique_document_person_role", - ), - Index("idx_document_person_doc", "document_id"), - Index("idx_document_person_per", "person_id"), - ) - - id: UUID | None = Field( - default=None, - sa_column=Column( - PostgreSQLUUID(as_uuid=True), - primary_key=True, - server_default=text("gen_random_uuid()"), - ), - ) - document_id: UUID = Field( - sa_column=Column( - PostgreSQLUUID(as_uuid=True), - ForeignKey("document.id", ondelete="CASCADE"), - nullable=False, - ), - ) - person_id: UUID = Field( - sa_column=Column( - PostgreSQLUUID(as_uuid=True), - ForeignKey("person.id", ondelete="CASCADE"), - nullable=False, - ), - ) - role: PersonRole = Field(sa_column=Column(String(20), nullable=False)) - created_at: datetime | None = Field( - default=None, - sa_column=Column( - DateTime(timezone=True), - nullable=False, - server_default=text("now()"), - ), - ) - - document: Document | None = Relationship( - back_populates="document_people", - sa_relationship_kwargs={"lazy": "raise"}, - ) - person: Person | None = Relationship( - back_populates="document_people", - sa_relationship_kwargs={"lazy": "raise"}, - ) - - -class Job(SQLModel, table=True): - __tablename__ = "job" - __table_args__ = (Index("idx_job_document", "document_id"),) - - id: UUID | None = Field( - default=None, - sa_column=Column( - PostgreSQLUUID(as_uuid=True), - primary_key=True, - server_default=text("gen_random_uuid()"), - ), - ) - document_id: UUID = Field( - sa_column=Column( - PostgreSQLUUID(as_uuid=True), - ForeignKey("document.id", ondelete="CASCADE"), - nullable=False, - ), - ) - status: JobStatus = Field( - default=JobStatus.QUEUED, - sa_column=Column( - String(50), - nullable=False, - server_default=text("'queued'"), - ), - ) - retry_count: int = Field( - default=0, - sa_column=Column( - Integer, - nullable=False, - server_default=text("0"), - ), - ) - provider: str = Field(sa_column=Column(Text, nullable=False)) - model: str = Field(sa_column=Column(Text, nullable=False)) - prompt_name: str | None = Field(default=None, sa_column=Column(Text)) - date_created: datetime | None = Field( - default=None, - sa_column=Column( - DateTime(timezone=True), - nullable=False, - server_default=text("now()"), - ), - ) - date_updated: datetime | None = Field( - default=None, - sa_column=Column( - DateTime(timezone=True), - nullable=False, - server_default=text("now()"), - ), - ) - - document: Document | None = Relationship( - back_populates="jobs", - sa_relationship_kwargs={"lazy": "raise"}, - ) - job_sources: list["JobSource"] = Relationship( - back_populates="job", - sa_relationship_kwargs={"lazy": "raise", "passive_deletes": True}, - ) - - -class Source(SQLModel, table=True): - __tablename__ = "source" - __table_args__ = ( - Index("idx_source_document", "document_id"), - Index("idx_source_page_order", "document_id", "page_number"), - ) - - id: UUID | None = Field( - default=None, - sa_column=Column( - PostgreSQLUUID(as_uuid=True), - primary_key=True, - server_default=text("gen_random_uuid()"), - ), - ) - document_id: UUID = Field( - sa_column=Column( - PostgreSQLUUID(as_uuid=True), - ForeignKey("document.id", ondelete="CASCADE"), - nullable=False, - ), - ) - page_number: int = Field( - default=1, - sa_column=Column( - Integer, - nullable=False, - server_default=text("1"), - ), - ) - upload_name: str = Field(sa_column=Column(Text, nullable=False)) - filename: str = Field(sa_column=Column(Text, nullable=False)) - file_path: str = Field(sa_column=Column(Text, nullable=False)) - raw_transcription: str | None = Field(default=None, sa_column=Column(Text)) - revised_text: str | None = Field(default=None, sa_column=Column(Text)) - date_uploaded: datetime | None = Field( - default=None, - sa_column=Column( - DateTime(timezone=True), - nullable=False, - server_default=text("now()"), - ), - ) - date_revised: datetime | None = Field( - default=None, - sa_column=Column(DateTime(timezone=True)), - ) - - document: Document | None = Relationship( - back_populates="sources", - sa_relationship_kwargs={"lazy": "raise"}, - ) - job_sources: list["JobSource"] = Relationship( - back_populates="source", - sa_relationship_kwargs={"lazy": "raise", "passive_deletes": True}, - ) - - -class JobSource(SQLModel, table=True): - __tablename__ = "job_source" - __table_args__ = ( - UniqueConstraint("job_id", "source_id", name="unique_job_source"), - Index("idx_job_source_job", "job_id"), - Index("idx_job_source_source", "source_id"), - Index( - "idx_job_source_ai_metadata", - "ai_metadata", - postgresql_using="gin", - ), - ) - - id: UUID | None = Field( - default=None, - sa_column=Column( - PostgreSQLUUID(as_uuid=True), - primary_key=True, - server_default=text("gen_random_uuid()"), - ), - ) - job_id: UUID = Field( - sa_column=Column( - PostgreSQLUUID(as_uuid=True), - ForeignKey("job.id", ondelete="CASCADE"), - nullable=False, - ), - ) - source_id: UUID = Field( - sa_column=Column( - PostgreSQLUUID(as_uuid=True), - ForeignKey("source.id", ondelete="CASCADE"), - nullable=False, - ), - ) - status: JobSourceStatus = Field( - default=JobSourceStatus.PENDING, - sa_column=Column( - String(50), - nullable=False, - server_default=text("'pending'"), - ), - ) - raw_transcription: str | None = Field(default=None, sa_column=Column(Text)) - ai_metadata: JsonValue | None = Field( - default=None, - sa_column=Column(JSONB), - ) - raw_api_response: JsonValue | None = Field( - default=None, - sa_column=Column(JSONB), - ) - error_detail: str | None = Field(default=None, sa_column=Column(Text)) - executed_at: datetime | None = Field( - default=None, - sa_column=Column( - DateTime(timezone=True), - nullable=False, - server_default=text("now()"), - ), - ) - - job: Job | None = Relationship( - back_populates="job_sources", - sa_relationship_kwargs={"lazy": "raise"}, - ) - source: Source | None = Relationship( - back_populates="job_sources", - sa_relationship_kwargs={"lazy": "raise"}, - ) -``` - -The enum annotations validate application values while the mapped columns retain the `VARCHAR` types specified by the DDL. PostgreSQL owns generated UUIDs and timestamps through `server_default`; call `session.refresh(instance)` after a flush or commit when those generated values are needed immediately. - -`ai_metadata`, `raw_api_response`, and `metadata_` accept any JSON value supported by `JSONB`. Validate provider-specific payload structure before assigning it to these fields, while preserving the complete raw response in `raw_api_response`. - -Relationships use `lazy="raise"` to prevent implicit database I/O in async code. Queries must explicitly load relationships they need, for example with `selectinload()`. \ No newline at end of file diff --git a/docs/ver2/V2 Python Pydantic Models.md b/docs/ver2/V2 Python Pydantic Models.md index 816f1d8..ea869f1 100644 --- a/docs/ver2/V2 Python Pydantic Models.md +++ b/docs/ver2/V2 Python Pydantic Models.md @@ -1,6 +1,6 @@ # SQLModel Table Models -Each V2 table is represented by one `SQLModel` class. Because `SQLModel` is built on Pydantic and SQLAlchemy, these classes provide application validation and PostgreSQL mappings without parallel row and create models. +These models implement the canonical [Version 2 database schema](../schema_v2.md). Each schema entity is represented by exactly one `SQLModel` table class. Because `SQLModel` is built on Pydantic and SQLAlchemy, these classes provide application validation and PostgreSQL mappings without parallel row and create models. Database-generated UUIDs and timestamps are `None` until PostgreSQL supplies their values during insert. The database columns remain non-nullable. `Person.metadata_` maps to the `metadata` column because `metadata` is reserved by SQLAlchemy's declarative API. @@ -405,4 +405,12 @@ The enum annotations validate application values while the mapped columns retain `ai_metadata`, `raw_api_response`, and `metadata_` accept any JSON value supported by `JSONB`. Validate provider-specific payload structure before assigning it to these fields, while preserving the complete raw response in `raw_api_response`. -Relationships use `lazy="raise"` to prevent implicit database I/O in async code. Queries must explicitly load relationships they need, for example with `selectinload()`. \ No newline at end of file +Relationships use `lazy="raise"` to prevent implicit database I/O in async code. Queries must explicitly load relationships they need, for example with `selectinload()`. + +The schema's behavioral invariants are enforced outside the table shape where appropriate: + +- `PersonRole`, `JobStatus`, and `JobSourceStatus` define the exact values listed by the schema. +- `unique_document_person_role` enforces role uniqueness for `(document_id, person_id, role)`. +- Services order document sources by `Source.document_id` and `Source.page_number`. +- Services derive aggregate `Job.status` from related `JobSource.status` values. +- Services preserve `JobSource.raw_transcription` and `JobSource.raw_api_response` as point-in-time outputs while updating the active text on `Source`. \ No newline at end of file