From 1fa5eb11274fc2fed4c468eeda4a029b0189b150 Mon Sep 17 00:00:00 2001 From: John Lancaster <32917998+jsl12@users.noreply.github.com> Date: Fri, 31 Jul 2026 10:12:34 -0500 Subject: [PATCH] doc updates for pydantic --- docs/ver2/V2 Context for DB Updates.md | 4 +- docs/ver2/V2 Python Pydantic Models.md | 408 +++++++++++++++++++++++++ docs/ver2/V2 TypeScript Zod Schemas.md | 169 ---------- docs/ver2/ver2.md | 241 ++++++++------- 4 files changed, 547 insertions(+), 275 deletions(-) create mode 100644 docs/ver2/V2 Python Pydantic Models.md delete mode 100644 docs/ver2/V2 TypeScript Zod Schemas.md diff --git a/docs/ver2/V2 Context for DB Updates.md b/docs/ver2/V2 Context for DB Updates.md index 88c20e2..e63c941 100644 --- a/docs/ver2/V2 Context for DB Updates.md +++ b/docs/ver2/V2 Context for DB Updates.md @@ -8,8 +8,8 @@ This application is a family history archival and transcription platform. Its pr ## Technical Stack & Architecture * **Database:** PostgreSQL 13+ with native `UUID` (`gen_random_uuid()`) and `JSONB` columns. * **Backend Runtime / Concurrency:** Python utilizing `asyncio` for concurrent HTTP API calls to AI providers, with strict rate-limiting via `asyncio.Semaphore`. -* **Validation & Types:** TypeScript with **Zod** schema definitions. Incoming AI responses must be parsed and validated with Zod schemas *before* database insertion. -* **ORM / Database Access:** Raw parameterized SQL queries or lightweight query builders (e.g., Kysely/Prisma) respecting PostgreSQL native types. +* **Validation & Types:** Python with **Pydantic** model definitions. Incoming AI responses must be parsed and validated with Pydantic models *before* database insertion. +* **ORM / Database Access:** SQLModel and SQLAlchemy, using parameterized statements and PostgreSQL-native types. --- diff --git a/docs/ver2/V2 Python Pydantic Models.md b/docs/ver2/V2 Python Pydantic Models.md new file mode 100644 index 0000000..816f1d8 --- /dev/null +++ b/docs/ver2/V2 Python Pydantic Models.md @@ -0,0 +1,408 @@ +# SQLModel Table Models + +Each V2 table is represented by one `SQLModel` class. Because `SQLModel` is built on Pydantic and SQLAlchemy, these classes provide application validation and PostgreSQL mappings without parallel row and create models. + +Database-generated UUIDs and timestamps are `None` until PostgreSQL supplies their values during insert. The database columns remain non-nullable. `Person.metadata_` maps to the `metadata` column because `metadata` is reserved by SQLAlchemy's declarative API. + +```python +from datetime import date +from datetime import datetime +from enum import StrEnum +from uuid import UUID + +from pydantic import JsonValue +from sqlalchemy import Column +from sqlalchemy import Date +from sqlalchemy import DateTime +from sqlalchemy import ForeignKey +from sqlalchemy import Index +from sqlalchemy import Integer +from sqlalchemy import String +from sqlalchemy import Text +from sqlalchemy import UniqueConstraint +from sqlalchemy import text +from sqlalchemy.dialects.postgresql import JSONB +from sqlalchemy.dialects.postgresql import UUID as PostgreSQLUUID +from sqlmodel import Field +from sqlmodel import Relationship +from sqlmodel import SQLModel + + +class PersonRole(StrEnum): + AUTHOR = "author" + RECIPIENT = "recipient" + + +class JobStatus(StrEnum): + QUEUED = "queued" + PROCESSING = "processing" + COMPLETED = "completed" + PARTIAL_SUCCESS = "partial_success" + FAILED = "failed" + + +class JobSourceStatus(StrEnum): + PENDING = "pending" + TRANSCRIBED = "transcribed" + FAILED = "failed" + + +class Person(SQLModel, table=True): + __tablename__ = "person" + __table_args__ = (Index("idx_person_full_name", "full_name"),) + + id: UUID | None = Field( + default=None, + sa_column=Column( + PostgreSQLUUID(as_uuid=True), + primary_key=True, + server_default=text("gen_random_uuid()"), + ), + ) + full_name: str = Field(sa_column=Column(Text, nullable=False)) + display_name: str | None = Field(default=None, sa_column=Column(Text)) + maiden_name: str | None = Field(default=None, sa_column=Column(Text)) + birth_date: date | None = Field(default=None, sa_column=Column(Date)) + birth_date_raw: str | None = Field(default=None, sa_column=Column(Text)) + birth_place: str | None = Field(default=None, sa_column=Column(Text)) + death_date: date | None = Field(default=None, sa_column=Column(Date)) + death_date_raw: str | None = Field(default=None, sa_column=Column(Text)) + death_place: str | None = Field(default=None, sa_column=Column(Text)) + biography: str | None = Field(default=None, sa_column=Column(Text)) + portrait_path: str | None = Field(default=None, sa_column=Column(Text)) + metadata_: JsonValue | None = Field( + default_factory=dict, + sa_column=Column( + "metadata", + JSONB, + server_default=text("'{}'::jsonb"), + ), + ) + created_at: datetime | None = Field( + default=None, + sa_column=Column( + DateTime(timezone=True), + nullable=False, + server_default=text("now()"), + ), + ) + updated_at: datetime | None = Field( + default=None, + sa_column=Column( + DateTime(timezone=True), + nullable=False, + server_default=text("now()"), + ), + ) + + document_people: list["DocumentPerson"] = Relationship( + back_populates="person", + sa_relationship_kwargs={"lazy": "raise", "passive_deletes": True}, + ) + + +class Document(SQLModel, table=True): + __tablename__ = "document" + __table_args__ = (Index("idx_document_date", "document_date"),) + + id: UUID | None = Field( + default=None, + sa_column=Column( + PostgreSQLUUID(as_uuid=True), + primary_key=True, + server_default=text("gen_random_uuid()"), + ), + ) + name: str = Field(sa_column=Column(Text, nullable=False)) + document_type: str | None = Field(default=None, sa_column=Column(Text)) + document_date: date | None = Field(default=None, sa_column=Column(Date)) + document_date_raw: str | None = Field(default=None, sa_column=Column(Text)) + location_created: str | None = Field(default=None, sa_column=Column(Text)) + notes: str | None = Field(default=None, sa_column=Column(Text)) + archive_identifier: str | None = Field(default=None, sa_column=Column(Text)) + created_at: datetime | None = Field( + default=None, + sa_column=Column( + DateTime(timezone=True), + nullable=False, + server_default=text("now()"), + ), + ) + updated_at: datetime | None = Field( + default=None, + sa_column=Column( + DateTime(timezone=True), + nullable=False, + server_default=text("now()"), + ), + ) + + document_people: list["DocumentPerson"] = Relationship( + back_populates="document", + sa_relationship_kwargs={"lazy": "raise", "passive_deletes": True}, + ) + jobs: list["Job"] = Relationship( + back_populates="document", + sa_relationship_kwargs={"lazy": "raise", "passive_deletes": True}, + ) + sources: list["Source"] = Relationship( + back_populates="document", + sa_relationship_kwargs={"lazy": "raise", "passive_deletes": True}, + ) + + +class DocumentPerson(SQLModel, table=True): + __tablename__ = "document_person" + __table_args__ = ( + UniqueConstraint( + "document_id", + "person_id", + "role", + name="unique_document_person_role", + ), + Index("idx_document_person_doc", "document_id"), + Index("idx_document_person_per", "person_id"), + ) + + id: UUID | None = Field( + default=None, + sa_column=Column( + PostgreSQLUUID(as_uuid=True), + primary_key=True, + server_default=text("gen_random_uuid()"), + ), + ) + document_id: UUID = Field( + sa_column=Column( + PostgreSQLUUID(as_uuid=True), + ForeignKey("document.id", ondelete="CASCADE"), + nullable=False, + ), + ) + person_id: UUID = Field( + sa_column=Column( + PostgreSQLUUID(as_uuid=True), + ForeignKey("person.id", ondelete="CASCADE"), + nullable=False, + ), + ) + role: PersonRole = Field(sa_column=Column(String(20), nullable=False)) + created_at: datetime | None = Field( + default=None, + sa_column=Column( + DateTime(timezone=True), + nullable=False, + server_default=text("now()"), + ), + ) + + document: Document | None = Relationship( + back_populates="document_people", + sa_relationship_kwargs={"lazy": "raise"}, + ) + person: Person | None = Relationship( + back_populates="document_people", + sa_relationship_kwargs={"lazy": "raise"}, + ) + + +class Job(SQLModel, table=True): + __tablename__ = "job" + __table_args__ = (Index("idx_job_document", "document_id"),) + + id: UUID | None = Field( + default=None, + sa_column=Column( + PostgreSQLUUID(as_uuid=True), + primary_key=True, + server_default=text("gen_random_uuid()"), + ), + ) + document_id: UUID = Field( + sa_column=Column( + PostgreSQLUUID(as_uuid=True), + ForeignKey("document.id", ondelete="CASCADE"), + nullable=False, + ), + ) + status: JobStatus = Field( + default=JobStatus.QUEUED, + sa_column=Column( + String(50), + nullable=False, + server_default=text("'queued'"), + ), + ) + retry_count: int = Field( + default=0, + sa_column=Column( + Integer, + nullable=False, + server_default=text("0"), + ), + ) + provider: str = Field(sa_column=Column(Text, nullable=False)) + model: str = Field(sa_column=Column(Text, nullable=False)) + prompt_name: str | None = Field(default=None, sa_column=Column(Text)) + date_created: datetime | None = Field( + default=None, + sa_column=Column( + DateTime(timezone=True), + nullable=False, + server_default=text("now()"), + ), + ) + date_updated: datetime | None = Field( + default=None, + sa_column=Column( + DateTime(timezone=True), + nullable=False, + server_default=text("now()"), + ), + ) + + document: Document | None = Relationship( + back_populates="jobs", + sa_relationship_kwargs={"lazy": "raise"}, + ) + job_sources: list["JobSource"] = Relationship( + back_populates="job", + sa_relationship_kwargs={"lazy": "raise", "passive_deletes": True}, + ) + + +class Source(SQLModel, table=True): + __tablename__ = "source" + __table_args__ = ( + Index("idx_source_document", "document_id"), + Index("idx_source_page_order", "document_id", "page_number"), + ) + + id: UUID | None = Field( + default=None, + sa_column=Column( + PostgreSQLUUID(as_uuid=True), + primary_key=True, + server_default=text("gen_random_uuid()"), + ), + ) + document_id: UUID = Field( + sa_column=Column( + PostgreSQLUUID(as_uuid=True), + ForeignKey("document.id", ondelete="CASCADE"), + nullable=False, + ), + ) + page_number: int = Field( + default=1, + sa_column=Column( + Integer, + nullable=False, + server_default=text("1"), + ), + ) + upload_name: str = Field(sa_column=Column(Text, nullable=False)) + filename: str = Field(sa_column=Column(Text, nullable=False)) + file_path: str = Field(sa_column=Column(Text, nullable=False)) + raw_transcription: str | None = Field(default=None, sa_column=Column(Text)) + revised_text: str | None = Field(default=None, sa_column=Column(Text)) + date_uploaded: datetime | None = Field( + default=None, + sa_column=Column( + DateTime(timezone=True), + nullable=False, + server_default=text("now()"), + ), + ) + date_revised: datetime | None = Field( + default=None, + sa_column=Column(DateTime(timezone=True)), + ) + + document: Document | None = Relationship( + back_populates="sources", + sa_relationship_kwargs={"lazy": "raise"}, + ) + job_sources: list["JobSource"] = Relationship( + back_populates="source", + sa_relationship_kwargs={"lazy": "raise", "passive_deletes": True}, + ) + + +class JobSource(SQLModel, table=True): + __tablename__ = "job_source" + __table_args__ = ( + UniqueConstraint("job_id", "source_id", name="unique_job_source"), + Index("idx_job_source_job", "job_id"), + Index("idx_job_source_source", "source_id"), + Index( + "idx_job_source_ai_metadata", + "ai_metadata", + postgresql_using="gin", + ), + ) + + id: UUID | None = Field( + default=None, + sa_column=Column( + PostgreSQLUUID(as_uuid=True), + primary_key=True, + server_default=text("gen_random_uuid()"), + ), + ) + job_id: UUID = Field( + sa_column=Column( + PostgreSQLUUID(as_uuid=True), + ForeignKey("job.id", ondelete="CASCADE"), + nullable=False, + ), + ) + source_id: UUID = Field( + sa_column=Column( + PostgreSQLUUID(as_uuid=True), + ForeignKey("source.id", ondelete="CASCADE"), + nullable=False, + ), + ) + status: JobSourceStatus = Field( + default=JobSourceStatus.PENDING, + sa_column=Column( + String(50), + nullable=False, + server_default=text("'pending'"), + ), + ) + raw_transcription: str | None = Field(default=None, sa_column=Column(Text)) + ai_metadata: JsonValue | None = Field( + default=None, + sa_column=Column(JSONB), + ) + raw_api_response: JsonValue | None = Field( + default=None, + sa_column=Column(JSONB), + ) + error_detail: str | None = Field(default=None, sa_column=Column(Text)) + executed_at: datetime | None = Field( + default=None, + sa_column=Column( + DateTime(timezone=True), + nullable=False, + server_default=text("now()"), + ), + ) + + job: Job | None = Relationship( + back_populates="job_sources", + sa_relationship_kwargs={"lazy": "raise"}, + ) + source: Source | None = Relationship( + back_populates="job_sources", + sa_relationship_kwargs={"lazy": "raise"}, + ) +``` + +The enum annotations validate application values while the mapped columns retain the `VARCHAR` types specified by the DDL. PostgreSQL owns generated UUIDs and timestamps through `server_default`; call `session.refresh(instance)` after a flush or commit when those generated values are needed immediately. + +`ai_metadata`, `raw_api_response`, and `metadata_` accept any JSON value supported by `JSONB`. Validate provider-specific payload structure before assigning it to these fields, while preserving the complete raw response in `raw_api_response`. + +Relationships use `lazy="raise"` to prevent implicit database I/O in async code. Queries must explicitly load relationships they need, for example with `selectinload()`. \ No newline at end of file diff --git a/docs/ver2/V2 TypeScript Zod Schemas.md b/docs/ver2/V2 TypeScript Zod Schemas.md deleted file mode 100644 index 8620c84..0000000 --- a/docs/ver2/V2 TypeScript Zod Schemas.md +++ /dev/null @@ -1,169 +0,0 @@ -## TypeScript Zod schemas - -Here are the TypeScript Zod schemas matching your V2 PostgreSQL database definition. - -These schemas cover: -1. Database Entities: Pure runtime validators representing rows fetched directly from PostgreSQL. -2. AI Payload Extensions: The structured document output stored inside job.ai_metadata. -3. Insert/Create Schemas: Utility types derived with .omit() for creating new records where auto-generated columns (id, created_at, updated_at, etc.) are handled by PostgreSQL defaults. - ---- - -``` -import { z } from "zod"; - -// ========================================== -// 1. ATOMIC & REUSABLE SCHEMAS -// ========================================== - -export const UUIDSchema = z.string().uuid(); -export const ISODateTimeSchema = z.coerce.date(); - -export const BoundingBoxSchema = z.object({ - ymin: z.number().min(0).max(1000), - xmin: z.number().min(0).max(1000), - ymax: z.number().min(0).max(1000), - xmax: z.number().min(0).max(1000), -}); - -export const BlockTypeSchema = z.enum([ - "heading", - "paragraph", - "table", - "margin_note", - "signature", - "footnote", - "header", -]); - -// ========================================== -// 2. PAGE-LEVEL AI METADATA SCHEMA (job_source.ai_metadata) -// ========================================== - -export const TranscribedBlockSchema = z.object({ - text: z.string(), - confidence: z.number().min(0).max(1), - blockType: BlockTypeSchema, - boundingBox: BoundingBoxSchema.optional(), -}); - -export const PageAIMetadataSchema = z.object({ - detectedLanguage: z.string().optional(), - overallConfidence: z.number().min(0).max(1), - blocks: z.array(TranscribedBlockSchema), - inputTokens: z.number().optional(), - outputTokens: z.number().optional(), - extractedEntities: z.record(z.string(), z.unknown()).optional(), -}); - -export type PageAIMetadata = z.infer; - -// ========================================== -// 3. TABLE ENTITY SCHEMAS -// ========================================== - -// --- PERSON TABLE --- -export const PersonSchema = z.object({ - id: UUIDSchema, - fullName: z.string().min(1), - displayName: z.string().nullable().optional(), - maidenName: z.string().nullable().optional(), - birthDate: z.string().nullable().optional(), - birthDateRaw: z.string().nullable().optional(), - birthPlace: z.string().nullable().optional(), - deathDate: z.string().nullable().optional(), - deathDateRaw: z.string().nullable().optional(), - deathPlace: z.string().nullable().optional(), - biography: z.string().nullable().optional(), - portraitPath: z.string().nullable().optional(), - metadata: z.record(z.string(), z.unknown()).default({}), - createdAt: ISODateTimeSchema, - updatedAt: ISODateTimeSchema, -}); - -// --- DOCUMENT TABLE --- -export const DocumentSchema = z.object({ - id: UUIDSchema, - name: z.string().min(1), - documentType: z.string().nullable().optional(), - documentDate: z.string().nullable().optional(), - documentDateRaw: z.string().nullable().optional(), - locationCreated: z.string().nullable().optional(), - notes: z.string().nullable().optional(), - archiveIdentifier: z.string().nullable().optional(), - createdAt: ISODateTimeSchema, - updatedAt: ISODateTimeSchema, -}); - -// --- DOCUMENT_PERSON JUNCTION --- -export const PersonRoleSchema = z.enum(["author", "recipient"]); - -export const DocumentPersonSchema = z.object({ - id: UUIDSchema, - documentId: UUIDSchema, - personId: UUIDSchema, - role: PersonRoleSchema, - createdAt: ISODateTimeSchema, -}); - -// --- JOB TABLE --- -export const JobStatusSchema = z.enum([ - "queued", - "processing", - "completed", - "partial_success", - "failed", -]); - -export const JobSchema = z.object({ - id: UUIDSchema, - documentId: UUIDSchema, - status: JobStatusSchema.default("queued"), - retryCount: z.number().int().nonnegative().default(0), - provider: z.string(), - model: z.string(), - promptName: z.string().nullable().optional(), - dateCreated: ISODateTimeSchema, - dateUpdated: ISODateTimeSchema, -}); - -// --- SOURCE TABLE --- -export const SourceSchema = z.object({ - id: UUIDSchema, - documentId: UUIDSchema, - pageNumber: z.number().int().positive().default(1), - uploadName: z.string(), - filename: z.string(), - filePath: z.string(), - rawTranscription: z.string().nullable().optional(), - revisedText: z.string().nullable().optional(), - dateUploaded: ISODateTimeSchema, - dateRevised: ISODateTimeSchema.nullable().optional(), -}); - -// --- JOB_SOURCE JUNCTION (Page Execution Output) --- -export const JobSourceStatusSchema = z.enum([ - "pending", - "transcribed", - "failed", -]); - -export const JobSourceSchema = z.object({ - id: UUIDSchema, - jobId: UUIDSchema, - sourceId: UUIDSchema, - status: JobSourceStatusSchema.default("pending"), - rawTranscription: z.string().nullable().optional(), - aiMetadata: PageAIMetadataSchema.nullable().optional(), - rawApiResponse: z.record(z.string(), z.unknown()).nullable().optional(), - errorDetail: z.string().nullable().optional(), - executedAt: ISODateTimeSchema, -}); - -export type Person = z.infer; -export type Document = z.infer; -export type DocumentPerson = z.infer; -export type Job = z.infer; -export type Source = z.infer; -export type JobSource = z.infer; -``` diff --git a/docs/ver2/ver2.md b/docs/ver2/ver2.md index 09e558f..a1dacfa 100644 --- a/docs/ver2/ver2.md +++ b/docs/ver2/ver2.md @@ -1,150 +1,183 @@ # Version 2 Plan -Desired Enhancements: -1. Data store - * Upgrade db to PostgresSQL - * Begin capturing JSONB data (which will allow future migration to MongoDB if desired) - * Relocate db and uploaded images to a location outside of the project folder that can be backed up. (This needs to be done for all projects.) (c:/github/data/transcription?) -2. Add ability to upload multiple images (or a folder of images) - * How many is too many? - * If there is a practical max image count, can I break a block of images up into smaller batches automatically? -3. UI - * Introduce the concept of "documents" to the UI. - * Before an image can be uploaded a "document" needs to be created/defined. - * As part of the upload process, document images need to be associated with a document. - * Multiple image upload - * Refine the job detail/log screen - * Is document id + original filename the best name for uploaded images? - * How to present multiple images within one job? - * Add document name, original filename to job detail. - - - - - - - - ## Purpose -Version 2 extends the V1 baseline by introducing a production-oriented persistence architecture while preserving current user workflows. +Version 2 updates the existing SQLModel domain schema to support multi-page documents, page-level transcription results, richer document metadata, and author/recipient attribution. -Primary target changes: +PostgreSQL support is already present in the database runtime. V2 does not require a database-layer rewrite or a general SQLite-to-PostgreSQL migration system. PostgreSQL adoption consists primarily of selecting the existing PostgreSQL settings, provisioning the database, creating the V2 schema, and verifying the application against it. -- Migrate relational persistence from SQLite to PostgreSQL -- Introduce optional MongoDB for document-oriented adjunct data (non-canonical) - -V1 behavior remains the functional baseline unless explicitly superseded by approved V2 requirements. +The main implementation effort is the schema update and the application changes that depend on it. --- -## V2 Goals +## Current State -1. **Relational migration complete** - - PostgreSQL becomes the default system of record for `Document`, `Source`, `Job`, and `Revision`. -1. **Operational maturity** - - Repeatable migrations, rollback paths, and environment-specific deployment procedures are documented and tested. -1. **Optional document store integration** - - MongoDB is introduced only for clearly scoped use cases that do not replace canonical relational ownership. -1. **No regression of V1 workflows** - - Upload, queue/worker processing, status inspection, original transcription, and optional single revision remain stable. +- The application uses Python 3.12, Pydantic v2, SQLModel, and async SQLAlchemy sessions. +- The database engine already supports both SQLite and PostgreSQL through `SqliteSettings` and `PostgresSettings`. +- The PostgreSQL async driver is installed and the engine already builds `postgresql+asyncpg` connections. +- Schema bootstrap currently uses `SQLModel.metadata.create_all()`. +- SQLite remains the default local configuration and the current Compose configuration still selects SQLite. +- The current V1 domain contains `Document`, `Source`, `Job`, and `Revision` tables. +- The V2 target is defined in [V2 DB Schema](V2%20DB%20Schema.md) and [V2 PostgreSQL DDL Specification](V2%20PostgreSQL%20DDL%20Specification.md). --- -## Non-Goals (V2) +## V2 Outcomes -- Replacing SQLModel domain ownership with MongoDB -- Introducing breaking UI behavior for existing V1 flows -- Expanding revision cardinality beyond current `0..1` without explicit requirements update +1. **V2 schema implemented** + - SQLModel models, relationships, enums, constraints, and indexes match the approved V2 schema. +1. **Page-level batch processing supported** + - A job can process multiple sources and retain an independent result for each source through `JobSource`. +1. **Document metadata expanded** + - Documents support ordered pages, descriptive metadata, and multiple authors and recipients. +1. **Raw provider data retained** + - Complete provider payloads are stored in PostgreSQL `JSONB` without flattening or discarding fields. + - Stored documents remain suitable for a future MongoDB import if one is ever needed. +1. **PostgreSQL enabled through configuration** + - The application starts against a provisioned PostgreSQL database using the existing runtime path. +1. **Existing workflows remain reliable** + - Upload, worker execution, status inspection, and transcription revision work with the new schema. --- -## Proposed Scope +## Non-Goals -### A) PostgreSQL migration (required) +- Rewriting the database engine or session layer +- Building a general-purpose SQLite-to-PostgreSQL migration utility +- Rehearsing a production database cutover when no production dataset requires preservation +- Running or integrating MongoDB in V2 +- Building MongoDB projections, synchronization, or fallback behavior +- Replacing Python, Pydantic, SQLModel, or SQLAlchemy +- Supporting more than one active human revision per source -- Add PostgreSQL runtime profile for local/dev/prod -- Introduce migration toolchain and migration history -- Convert bootstrap strategy from compatibility patching to explicit migrations -- Validate model constraints and indexes against PostgreSQL -- Add operational checks (connectivity, pool, transaction behavior) +If an existing SQLite dataset must be retained, define a small one-time import task separately. It is not part of the default V2 implementation path. -### B) MongoDB integration (optional, gated) +--- -- Define approved use cases (for example: denormalized read models, audit/event projections, or search-oriented materializations) -- Keep canonical write path in relational store -- Add feature flag/config gate to enable or disable Mongo features -- Document consistency model and failure behavior +## Scope + +### A) SQLModel schema update (primary) + +- Add `Person`, `DocumentPerson`, and `JobSource` models. +- Expand `Document` with type, date, location, notes, archive identifier, and timestamps. +- Update `Source` with page ordering, active raw transcription, revised text, and revision timestamp. +- Update `Job` for batch execution and the `partial_success` terminal state. +- Replace the standalone `Revision` table with revision fields on `Source`. +- Remove the direct `Source.job_id` relationship; connect sources to jobs through `JobSource`. +- Add role, job status, and job-source status enums. +- Add required uniqueness constraints, foreign-key delete behavior, lookup indexes, and PostgreSQL JSON indexes. +- Keep model definitions aligned with [V2 Python Pydantic Models](V2%20Python%20Pydantic%20Models.md). + +### B) Raw document storage + +- Store structured AI metadata in `job_source.ai_metadata` as `JSONB`. +- Store the complete raw provider response in `job_source.raw_api_response` as `JSONB`. +- Preserve the original document structure, field names, nested values, and unknown fields in the raw response. +- Keep validation of extracted application fields separate from retention of the raw response. +- Serialize UUIDs and datetimes using portable string representations. +- Use MongoDB Extended JSON representations only if a provider value cannot be represented faithfully in standard JSON. +- Do not store opaque BSON bytes in PostgreSQL unless a future payload contains BSON-only values that cannot be preserved in `JSONB`. + +### C) Schema creation and verification + +- Use a fresh V2 database during development unless preservation of existing data becomes a requirement. +- Create the schema from SQLModel metadata and verify it against the approved DDL. +- Keep SQLite available for fast unit tests where its behavior is equivalent. +- Add focused PostgreSQL integration tests for native UUIDs, JSON storage, constraints, indexes, and transactions. +- Introduce migration tooling only if V2 must update a populated deployed database in place. + +### D) Service and worker alignment + +- Update document, source, job, and store operations for the new relationships. +- Create one `JobSource` row per source included in a job. +- Persist page-level status, transcription, AI metadata, raw provider response, and errors on `JobSource`. +- Derive the parent job status from its page results: + - `completed` when all pages succeed + - `partial_success` when successful and failed pages are mixed + - `failed` when all pages fail or a job-level failure prevents execution +- Update `Source.raw_transcription` after a successful page result while keeping the original `JobSource.raw_transcription` immutable. +- Read `Source.revised_text` in preference to `Source.raw_transcription` when presenting active text. + +### E) Document and multi-image workflows + +- Require a document before associating uploaded sources. +- Support uploading multiple images into one document. +- Preserve page order through `Source.page_number`. +- Allow a job to include one or more sources from the same document. +- Update job details to show the document, each source filename, page order, page status, and page-level errors. +- Define a practical upload limit and split oversized selections into manageable batches if needed. + +### F) PostgreSQL configuration + +- Provision PostgreSQL for local and deployed environments. +- Configure the existing `Settings.database` field with PostgreSQL host, port, database, user, and password values. +- Update Compose and environment configuration to stop selecting SQLite. +- Decide whether schema bootstrap is enabled for local development or performed as a separate deployment step. +- Run a connectivity and schema smoke test against PostgreSQL. +- Keep uploaded files on a persistent, backup-capable path outside the application image. --- ## Milestones -## M1 — Requirements and architecture baseline +### M1 - Schema models -- Create V2 requirements delta from V1 baseline -- Define relational/document ownership boundaries -- Approve migration strategy and cutover approach +- Implement the V2 SQLModel models and enums. +- Implement relationships, constraints, indexes, and JSON column types. +- Update the Pydantic data contracts where model decisions change. +- Add schema-focused tests. -**Exit criteria:** signed architecture decision and updated traceability map. +**Exit criteria:** SQLModel metadata represents the approved V2 schema and schema tests pass. -## M2 — PostgreSQL foundation +### M2 - Persistence and worker behavior -- Add PostgreSQL environment wiring and secrets strategy -- Add migration framework and initial schema migration -- Add CI path using PostgreSQL service container +- Update database operations and services for the V2 entities. +- Implement page-level `JobSource` execution records. +- Preserve complete raw provider responses in `JSONB`. +- Implement aggregate job status calculation. +- Add transaction, partial-success, and failure-isolation tests. -**Exit criteria:** test suite green on PostgreSQL in CI. +**Exit criteria:** single-page and multi-page jobs persist correct page, raw payload, and aggregate states. -## M3 — Data migration and cutover rehearsal +### M3 - Document and upload workflows -- Build SQLite -> PostgreSQL migration utility/playbook -- Rehearse migration on representative datasets -- Validate rollback/recovery procedures +- Update document creation and source association flows. +- Add ordered multi-image upload. +- Update job and document detail views for page-level results. +- Add focused UI and service tests. -**Exit criteria:** successful dry-run migration with measured rollback test. +**Exit criteria:** a user can create a document, upload ordered pages, run a job, and inspect each result. -## M4 — MongoDB optional integration +### M4 - PostgreSQL verification and release -- Implement scoped Mongo use case(s) -- Add fallback behavior when Mongo unavailable -- Add tests and operational runbook updates +- Switch local or test configuration to the existing PostgreSQL runtime path. +- Create the V2 schema in a fresh PostgreSQL database. +- Run PostgreSQL-specific schema and workflow tests. +- Document startup, backup, and recovery settings. +- Run the final regression suite. -**Exit criteria:** feature-gated Mongo behavior validated with no V1 flow regressions. - -## M5 — Release readiness - -- Final regression suite (functional + reliability) -- Performance and failure-mode checks -- Production release checklist and sign-off - -**Exit criteria:** V2 release approval. +**Exit criteria:** V2 workflows pass against PostgreSQL and release checks are complete. --- ## Risks and Mitigations -- **Schema drift risk** -> enforce migration-first policy and CI migration checks. -- **Dual-store consistency risk** -> keep relational source of truth and explicit projection contracts. -- **Operational complexity** -> staged rollout, runbooks, and feature flags. -- **Regression risk in worker lifecycle** -> keep dedicated reliability tests around terminal-state guarantees. +- **Model and DDL drift** -> compare generated metadata with the approved schema and test named constraints and indexes. +- **Raw payload loss** -> retain the complete provider response separately from validated and extracted fields. +- **Cross-database differences** -> retain fast SQLite tests but verify PostgreSQL-native UUID, JSON, and index behavior in integration tests. +- **Batch state errors** -> test all-success, mixed-result, and all-failed jobs explicitly. +- **Page ordering errors** -> enforce uniqueness and ordering rules for document pages. +- **Unexpected data-preservation need** -> confirm whether existing SQLite data matters before implementation; add a one-time importer only when required. +- **Worker regressions** -> preserve terminal-state and retry reliability tests while changing persistence ownership. --- -## Traceability and Evidence +## Suggested First Tasks -Maintain a V2 table with: - -- requirement/change ID -- status (`not started` / `in progress` / `done`) -- implementation PR -- validation evidence (test names, migration rehearsal logs, runbook references) - ---- - -## Suggested first implementation tasks - -1. Create `docs/ver2/adr/` and draft ADR for persistence ownership boundaries. -2. Add PostgreSQL compose profile and env contract. -3. Introduce migration tooling and generate initial migration from current schema. -4. Add CI job for PostgreSQL-backed `pytest -m "not external"`. +1. Update `src/transcription/db/models.py` to represent the approved V2 schema. +2. Add schema tests for tables, columns, relationships, constraints, indexes, and enums. +3. Define and test lossless raw provider response storage in `job_source.raw_api_response`. +4. Update database operations and services to use `JobSource` and source-level revisions. +5. Add page-result aggregation tests before changing the worker workflow. +6. Update document and multi-image upload flows. +7. Select PostgreSQL in configuration and run the integration suite against a fresh V2 database.