8 Commits
Author SHA1 Message Date
John Lancaster c4d25c1be8 Merge remote-tracking branch 'origin/doc_update' into session-engine 2026-07-31 10:16:17 -05:00
John Lancaster 1fa5eb1127 doc updates for pydantic 2026-07-31 10:12:34 -05:00
John Lancaster ec6617a1c4 updates 2026-07-30 23:16:17 -05:00
John Lancaster 9eb0f40c08 session and engine 2026-07-30 22:33:49 -05:00
John Lancaster f769d29da1 uv.lock update 2026-07-30 21:17:09 -05:00
John Lancaster 8afc462a6d startup 2026-07-30 21:16:48 -05:00
John Lancaster 3d6daec561 moved models to db pkg 2026-07-30 21:10:13 -05:00
John Lancaster 1cc2f319d5 uvicorn startup 2026-07-30 21:09:51 -05:00
38 changed files with 1830 additions and 727 deletions
+408
View File
@@ -0,0 +1,408 @@
# SQLModel Table Models
Each V2 table is represented by one `SQLModel` class. Because `SQLModel` is built on Pydantic and SQLAlchemy, these classes provide application validation and PostgreSQL mappings without parallel row and create models.
Database-generated UUIDs and timestamps are `None` until PostgreSQL supplies their values during insert. The database columns remain non-nullable. `Person.metadata_` maps to the `metadata` column because `metadata` is reserved by SQLAlchemy's declarative API.
```python
from datetime import date
from datetime import datetime
from enum import StrEnum
from uuid import UUID
from pydantic import JsonValue
from sqlalchemy import Column
from sqlalchemy import Date
from sqlalchemy import DateTime
from sqlalchemy import ForeignKey
from sqlalchemy import Index
from sqlalchemy import Integer
from sqlalchemy import String
from sqlalchemy import Text
from sqlalchemy import UniqueConstraint
from sqlalchemy import text
from sqlalchemy.dialects.postgresql import JSONB
from sqlalchemy.dialects.postgresql import UUID as PostgreSQLUUID
from sqlmodel import Field
from sqlmodel import Relationship
from sqlmodel import SQLModel
class PersonRole(StrEnum):
AUTHOR = "author"
RECIPIENT = "recipient"
class JobStatus(StrEnum):
QUEUED = "queued"
PROCESSING = "processing"
COMPLETED = "completed"
PARTIAL_SUCCESS = "partial_success"
FAILED = "failed"
class JobSourceStatus(StrEnum):
PENDING = "pending"
TRANSCRIBED = "transcribed"
FAILED = "failed"
class Person(SQLModel, table=True):
__tablename__ = "person"
__table_args__ = (Index("idx_person_full_name", "full_name"),)
id: UUID | None = Field(
default=None,
sa_column=Column(
PostgreSQLUUID(as_uuid=True),
primary_key=True,
server_default=text("gen_random_uuid()"),
),
)
full_name: str = Field(sa_column=Column(Text, nullable=False))
display_name: str | None = Field(default=None, sa_column=Column(Text))
maiden_name: str | None = Field(default=None, sa_column=Column(Text))
birth_date: date | None = Field(default=None, sa_column=Column(Date))
birth_date_raw: str | None = Field(default=None, sa_column=Column(Text))
birth_place: str | None = Field(default=None, sa_column=Column(Text))
death_date: date | None = Field(default=None, sa_column=Column(Date))
death_date_raw: str | None = Field(default=None, sa_column=Column(Text))
death_place: str | None = Field(default=None, sa_column=Column(Text))
biography: str | None = Field(default=None, sa_column=Column(Text))
portrait_path: str | None = Field(default=None, sa_column=Column(Text))
metadata_: JsonValue | None = Field(
default_factory=dict,
sa_column=Column(
"metadata",
JSONB,
server_default=text("'{}'::jsonb"),
),
)
created_at: datetime | None = Field(
default=None,
sa_column=Column(
DateTime(timezone=True),
nullable=False,
server_default=text("now()"),
),
)
updated_at: datetime | None = Field(
default=None,
sa_column=Column(
DateTime(timezone=True),
nullable=False,
server_default=text("now()"),
),
)
document_people: list["DocumentPerson"] = Relationship(
back_populates="person",
sa_relationship_kwargs={"lazy": "raise", "passive_deletes": True},
)
class Document(SQLModel, table=True):
__tablename__ = "document"
__table_args__ = (Index("idx_document_date", "document_date"),)
id: UUID | None = Field(
default=None,
sa_column=Column(
PostgreSQLUUID(as_uuid=True),
primary_key=True,
server_default=text("gen_random_uuid()"),
),
)
name: str = Field(sa_column=Column(Text, nullable=False))
document_type: str | None = Field(default=None, sa_column=Column(Text))
document_date: date | None = Field(default=None, sa_column=Column(Date))
document_date_raw: str | None = Field(default=None, sa_column=Column(Text))
location_created: str | None = Field(default=None, sa_column=Column(Text))
notes: str | None = Field(default=None, sa_column=Column(Text))
archive_identifier: str | None = Field(default=None, sa_column=Column(Text))
created_at: datetime | None = Field(
default=None,
sa_column=Column(
DateTime(timezone=True),
nullable=False,
server_default=text("now()"),
),
)
updated_at: datetime | None = Field(
default=None,
sa_column=Column(
DateTime(timezone=True),
nullable=False,
server_default=text("now()"),
),
)
document_people: list["DocumentPerson"] = Relationship(
back_populates="document",
sa_relationship_kwargs={"lazy": "raise", "passive_deletes": True},
)
jobs: list["Job"] = Relationship(
back_populates="document",
sa_relationship_kwargs={"lazy": "raise", "passive_deletes": True},
)
sources: list["Source"] = Relationship(
back_populates="document",
sa_relationship_kwargs={"lazy": "raise", "passive_deletes": True},
)
class DocumentPerson(SQLModel, table=True):
__tablename__ = "document_person"
__table_args__ = (
UniqueConstraint(
"document_id",
"person_id",
"role",
name="unique_document_person_role",
),
Index("idx_document_person_doc", "document_id"),
Index("idx_document_person_per", "person_id"),
)
id: UUID | None = Field(
default=None,
sa_column=Column(
PostgreSQLUUID(as_uuid=True),
primary_key=True,
server_default=text("gen_random_uuid()"),
),
)
document_id: UUID = Field(
sa_column=Column(
PostgreSQLUUID(as_uuid=True),
ForeignKey("document.id", ondelete="CASCADE"),
nullable=False,
),
)
person_id: UUID = Field(
sa_column=Column(
PostgreSQLUUID(as_uuid=True),
ForeignKey("person.id", ondelete="CASCADE"),
nullable=False,
),
)
role: PersonRole = Field(sa_column=Column(String(20), nullable=False))
created_at: datetime | None = Field(
default=None,
sa_column=Column(
DateTime(timezone=True),
nullable=False,
server_default=text("now()"),
),
)
document: Document | None = Relationship(
back_populates="document_people",
sa_relationship_kwargs={"lazy": "raise"},
)
person: Person | None = Relationship(
back_populates="document_people",
sa_relationship_kwargs={"lazy": "raise"},
)
class Job(SQLModel, table=True):
__tablename__ = "job"
__table_args__ = (Index("idx_job_document", "document_id"),)
id: UUID | None = Field(
default=None,
sa_column=Column(
PostgreSQLUUID(as_uuid=True),
primary_key=True,
server_default=text("gen_random_uuid()"),
),
)
document_id: UUID = Field(
sa_column=Column(
PostgreSQLUUID(as_uuid=True),
ForeignKey("document.id", ondelete="CASCADE"),
nullable=False,
),
)
status: JobStatus = Field(
default=JobStatus.QUEUED,
sa_column=Column(
String(50),
nullable=False,
server_default=text("'queued'"),
),
)
retry_count: int = Field(
default=0,
sa_column=Column(
Integer,
nullable=False,
server_default=text("0"),
),
)
provider: str = Field(sa_column=Column(Text, nullable=False))
model: str = Field(sa_column=Column(Text, nullable=False))
prompt_name: str | None = Field(default=None, sa_column=Column(Text))
date_created: datetime | None = Field(
default=None,
sa_column=Column(
DateTime(timezone=True),
nullable=False,
server_default=text("now()"),
),
)
date_updated: datetime | None = Field(
default=None,
sa_column=Column(
DateTime(timezone=True),
nullable=False,
server_default=text("now()"),
),
)
document: Document | None = Relationship(
back_populates="jobs",
sa_relationship_kwargs={"lazy": "raise"},
)
job_sources: list["JobSource"] = Relationship(
back_populates="job",
sa_relationship_kwargs={"lazy": "raise", "passive_deletes": True},
)
class Source(SQLModel, table=True):
__tablename__ = "source"
__table_args__ = (
Index("idx_source_document", "document_id"),
Index("idx_source_page_order", "document_id", "page_number"),
)
id: UUID | None = Field(
default=None,
sa_column=Column(
PostgreSQLUUID(as_uuid=True),
primary_key=True,
server_default=text("gen_random_uuid()"),
),
)
document_id: UUID = Field(
sa_column=Column(
PostgreSQLUUID(as_uuid=True),
ForeignKey("document.id", ondelete="CASCADE"),
nullable=False,
),
)
page_number: int = Field(
default=1,
sa_column=Column(
Integer,
nullable=False,
server_default=text("1"),
),
)
upload_name: str = Field(sa_column=Column(Text, nullable=False))
filename: str = Field(sa_column=Column(Text, nullable=False))
file_path: str = Field(sa_column=Column(Text, nullable=False))
raw_transcription: str | None = Field(default=None, sa_column=Column(Text))
revised_text: str | None = Field(default=None, sa_column=Column(Text))
date_uploaded: datetime | None = Field(
default=None,
sa_column=Column(
DateTime(timezone=True),
nullable=False,
server_default=text("now()"),
),
)
date_revised: datetime | None = Field(
default=None,
sa_column=Column(DateTime(timezone=True)),
)
document: Document | None = Relationship(
back_populates="sources",
sa_relationship_kwargs={"lazy": "raise"},
)
job_sources: list["JobSource"] = Relationship(
back_populates="source",
sa_relationship_kwargs={"lazy": "raise", "passive_deletes": True},
)
class JobSource(SQLModel, table=True):
__tablename__ = "job_source"
__table_args__ = (
UniqueConstraint("job_id", "source_id", name="unique_job_source"),
Index("idx_job_source_job", "job_id"),
Index("idx_job_source_source", "source_id"),
Index(
"idx_job_source_ai_metadata",
"ai_metadata",
postgresql_using="gin",
),
)
id: UUID | None = Field(
default=None,
sa_column=Column(
PostgreSQLUUID(as_uuid=True),
primary_key=True,
server_default=text("gen_random_uuid()"),
),
)
job_id: UUID = Field(
sa_column=Column(
PostgreSQLUUID(as_uuid=True),
ForeignKey("job.id", ondelete="CASCADE"),
nullable=False,
),
)
source_id: UUID = Field(
sa_column=Column(
PostgreSQLUUID(as_uuid=True),
ForeignKey("source.id", ondelete="CASCADE"),
nullable=False,
),
)
status: JobSourceStatus = Field(
default=JobSourceStatus.PENDING,
sa_column=Column(
String(50),
nullable=False,
server_default=text("'pending'"),
),
)
raw_transcription: str | None = Field(default=None, sa_column=Column(Text))
ai_metadata: JsonValue | None = Field(
default=None,
sa_column=Column(JSONB),
)
raw_api_response: JsonValue | None = Field(
default=None,
sa_column=Column(JSONB),
)
error_detail: str | None = Field(default=None, sa_column=Column(Text))
executed_at: datetime | None = Field(
default=None,
sa_column=Column(
DateTime(timezone=True),
nullable=False,
server_default=text("now()"),
),
)
job: Job | None = Relationship(
back_populates="job_sources",
sa_relationship_kwargs={"lazy": "raise"},
)
source: Source | None = Relationship(
back_populates="job_sources",
sa_relationship_kwargs={"lazy": "raise"},
)
```
The enum annotations validate application values while the mapped columns retain the `VARCHAR` types specified by the DDL. PostgreSQL owns generated UUIDs and timestamps through `server_default`; call `session.refresh(instance)` after a flush or commit when those generated values are needed immediately.
`ai_metadata`, `raw_api_response`, and `metadata_` accept any JSON value supported by `JSONB`. Validate provider-specific payload structure before assigning it to these fields, while preserving the complete raw response in `raw_api_response`.
Relationships use `lazy="raise"` to prevent implicit database I/O in async code. Queries must explicitly load relationships they need, for example with `selectinload()`.
-92
View File
@@ -1,92 +0,0 @@
```mermaid
block-beta
columns 3
%% UI Component Column
block:UI["UI COMPONENTS / WIREFRAME"]:1
columns 1
block:HeaderUI["Header & Nav"]:1
columns 1
h_title["[Text] Document Name & Type"]
h_date["[Text] Date & Origin Location"]
end
block:EditorUI["Page Transcription Editor"]:1
columns 1
ed_img["[Image Viewer] Source Image"]
ed_page["[Badge] Page Number"]
ed_raw["[Read-Only] AI Raw Output"]
ed_rev["[Textarea] Human Revised Text"]
end
block:PeopleUI["Attribution Sidebar"]:1
columns 1
p_author["[List] Authors (Full Name)"]
p_recip["[List] Recipients (Full Name)"]
p_bio["[Card] Person Biography & Dates"]
end
block:JobUI["AI Processing Drawer"]:1
columns 1
j_status["[Badge] Job Status"]
j_model["[Text] Provider & Model"]
j_tokens["[JSON View] AI Token Usage"]
end
end
%% Directional Mapping / Connectors
block:FLOW["MAPPING / FLOW"]:1
columns 1
f1["Reads / Updates -->"]
f2["Renders Active Page -->"]
f3["Joins via Role -->"]
f4["Executes & Logs -->"]
end
%% Postgres Schema Column
block:DB["POSTGRES SQL SCHEMA"]:1
columns 1
block:DocTbl["Table: document"]:1
columns 1
d_id["id : UUID (PK)"]
d_name["name : TEXT"]
d_type["document_type : TEXT"]
d_date["document_date : DATE"]
end
block:SrcTbl["Table: source"]:1
columns 1
s_id["id : UUID (PK)"]
s_page["page_number : INT"]
s_path["file_path : TEXT"]
s_raw["raw_transcription : TEXT"]
s_rev["revised_text : TEXT"]
end
block:PersonTbl["Table: person & document_person"]:1
columns 1
p_id["id : UUID (PK)"]
p_name["full_name : TEXT"]
p_role["role : 'author' | 'recipient'"]
end
block:JobTbl["Table: job & job_source"]:1
columns 1
j_id["id : UUID (PK)"]
j_stat["status : VARCHAR"]
j_prov["provider / model : TEXT"]
j_meta["ai_metadata : JSONB"]
end
end
%% Connections
HeaderUI --> DocTbl
ed_img --> s_path
ed_page --> s_page
ed_raw --> s_raw
ed_rev --> s_rev
PeopleUI --> PersonTbl
JobUI --> JobTbl
```
-53
View File
@@ -1,53 +0,0 @@
```mermaid
flowchart LR
subgraph UI["UI Components / Wireframe"]
direction TB
subgraph HeaderUI["Header & Nav"]
h_title["[Text] Document Name & Type"]
h_date["[Text] Date & Origin Location"]
end
subgraph EditorUI["Page Transcription Editor"]
ed_img["[Image Viewer] Source Image"]
ed_page["[Badge] Page Number"]
ed_raw["[Read-Only] AI Raw Output"]
ed_rev["[Textarea] Human Revised Text"]
end
subgraph PeopleUI["Attribution Sidebar"]
p_author["[List] Authors / Recipients"]
end
subgraph JobUI["AI Processing Drawer"]
j_status["[Badge] Job Status"]
end
end
subgraph DB["Postgres SQL Schema"]
direction TB
subgraph DocTbl["Table: document"]
d_name["name : TEXT"]
d_type["document_type : TEXT"]
end
subgraph SrcTbl["Table: source"]
s_path["file_path : TEXT"]
s_page["page_number : INT"]
s_raw["raw_transcription : TEXT"]
s_rev["revised_text : TEXT"]
end
subgraph PersonTbl["Table: person & document_person"]
p_name["full_name : TEXT"]
p_role["role : author | recipient"]
end
subgraph JobTbl["Table: job & job_source"]
j_stat["status : VARCHAR"]
j_meta["ai_metadata : JSONB"]
end
end
%% Mappings
HeaderUI --> DocTbl
ed_img --> s_path
ed_page --> s_page
ed_raw --> s_raw
ed_rev --> s_rev
PeopleUI --> PersonTbl
JobUI --> JobTbl
```
+35
View File
@@ -0,0 +1,35 @@
# AI Coding Assistant Project Briefing & Context
## Project Mission
This application is a family history archival and transcription platform. Its primary goal is to accept scanned document images (letters, postcards, logbooks, diaries), execute OCR and structured transcription via AI vision models (OpenAI GPT-4o, Anthropic Claude 3.5 Sonnet), and manage historical metadata (authors, recipients, dates, and locations).
---
## Technical Stack & Architecture
* **Database:** PostgreSQL 13+ with native `UUID` (`gen_random_uuid()`) and `JSONB` columns.
* **Backend Runtime / Concurrency:** Python utilizing `asyncio` for concurrent HTTP API calls to AI providers, with strict rate-limiting via `asyncio.Semaphore`.
* **Validation & Types:** Python with **Pydantic** model definitions. Incoming AI responses must be parsed and validated with Pydantic models *before* database insertion.
* **ORM / Database Access:** SQLModel and SQLAlchemy, using parameterized statements and PostgreSQL-native types.
---
## Core System Directives for AI Code Generation
### 1. Data Immutability vs. Human Corrections
* `job_source.raw_transcription` and `source.raw_transcription` represent original, point-in-time machine outputs and are **immutable**.
* Human corrections occur on `source.revised_text`.
* When fetching text for the UI, always display `COALESCE(source.revised_text, source.raw_transcription)`.
### 2. Async Execution & Batching Rules
* A `job` represents an overarching execution run for a folder/group of images belonging to a single `document`.
* Images are submitted to AI APIs **one at a time in rapid succession** using `asyncio` worker pools.
* Each single-image API call populates a row in `job_source` with its own `status`, `raw_transcription`, `ai_metadata`, and `raw_api_response`.
* If 9 of 10 pages succeed and 1 fails, `job_source.status` for the failed image becomes `'failed'`, while `job.status` becomes `'partial_success'`. Do not mark the entire batch as failed if partial results exist.
### 3. Entity Relationships
* **Authors/Recipients:** A `document` can have multiple authors and recipients. Do NOT put direct `author_id` foreign keys on `document`. Query authors/recipients via `document_person` where `role = 'author'` or `role = 'recipient'`.
* **Page Ordering:** Multi-page documents must always be queried using `ORDER BY page_number ASC`.
### 4. Database Mutations
* Always use parameterized SQL queries (`$1`, `$2`) to prevent SQL injection.
* Store datetimes using UTC ISO 8601 strings or native PostgreSQL `TIMESTAMPTZ`.
+408
View File
@@ -0,0 +1,408 @@
# SQLModel Table Models
Each V2 table is represented by one `SQLModel` class. Because `SQLModel` is built on Pydantic and SQLAlchemy, these classes provide application validation and PostgreSQL mappings without parallel row and create models.
Database-generated UUIDs and timestamps are `None` until PostgreSQL supplies their values during insert. The database columns remain non-nullable. `Person.metadata_` maps to the `metadata` column because `metadata` is reserved by SQLAlchemy's declarative API.
```python
from datetime import date
from datetime import datetime
from enum import StrEnum
from uuid import UUID
from pydantic import JsonValue
from sqlalchemy import Column
from sqlalchemy import Date
from sqlalchemy import DateTime
from sqlalchemy import ForeignKey
from sqlalchemy import Index
from sqlalchemy import Integer
from sqlalchemy import String
from sqlalchemy import Text
from sqlalchemy import UniqueConstraint
from sqlalchemy import text
from sqlalchemy.dialects.postgresql import JSONB
from sqlalchemy.dialects.postgresql import UUID as PostgreSQLUUID
from sqlmodel import Field
from sqlmodel import Relationship
from sqlmodel import SQLModel
class PersonRole(StrEnum):
AUTHOR = "author"
RECIPIENT = "recipient"
class JobStatus(StrEnum):
QUEUED = "queued"
PROCESSING = "processing"
COMPLETED = "completed"
PARTIAL_SUCCESS = "partial_success"
FAILED = "failed"
class JobSourceStatus(StrEnum):
PENDING = "pending"
TRANSCRIBED = "transcribed"
FAILED = "failed"
class Person(SQLModel, table=True):
__tablename__ = "person"
__table_args__ = (Index("idx_person_full_name", "full_name"),)
id: UUID | None = Field(
default=None,
sa_column=Column(
PostgreSQLUUID(as_uuid=True),
primary_key=True,
server_default=text("gen_random_uuid()"),
),
)
full_name: str = Field(sa_column=Column(Text, nullable=False))
display_name: str | None = Field(default=None, sa_column=Column(Text))
maiden_name: str | None = Field(default=None, sa_column=Column(Text))
birth_date: date | None = Field(default=None, sa_column=Column(Date))
birth_date_raw: str | None = Field(default=None, sa_column=Column(Text))
birth_place: str | None = Field(default=None, sa_column=Column(Text))
death_date: date | None = Field(default=None, sa_column=Column(Date))
death_date_raw: str | None = Field(default=None, sa_column=Column(Text))
death_place: str | None = Field(default=None, sa_column=Column(Text))
biography: str | None = Field(default=None, sa_column=Column(Text))
portrait_path: str | None = Field(default=None, sa_column=Column(Text))
metadata_: JsonValue | None = Field(
default_factory=dict,
sa_column=Column(
"metadata",
JSONB,
server_default=text("'{}'::jsonb"),
),
)
created_at: datetime | None = Field(
default=None,
sa_column=Column(
DateTime(timezone=True),
nullable=False,
server_default=text("now()"),
),
)
updated_at: datetime | None = Field(
default=None,
sa_column=Column(
DateTime(timezone=True),
nullable=False,
server_default=text("now()"),
),
)
document_people: list["DocumentPerson"] = Relationship(
back_populates="person",
sa_relationship_kwargs={"lazy": "raise", "passive_deletes": True},
)
class Document(SQLModel, table=True):
__tablename__ = "document"
__table_args__ = (Index("idx_document_date", "document_date"),)
id: UUID | None = Field(
default=None,
sa_column=Column(
PostgreSQLUUID(as_uuid=True),
primary_key=True,
server_default=text("gen_random_uuid()"),
),
)
name: str = Field(sa_column=Column(Text, nullable=False))
document_type: str | None = Field(default=None, sa_column=Column(Text))
document_date: date | None = Field(default=None, sa_column=Column(Date))
document_date_raw: str | None = Field(default=None, sa_column=Column(Text))
location_created: str | None = Field(default=None, sa_column=Column(Text))
notes: str | None = Field(default=None, sa_column=Column(Text))
archive_identifier: str | None = Field(default=None, sa_column=Column(Text))
created_at: datetime | None = Field(
default=None,
sa_column=Column(
DateTime(timezone=True),
nullable=False,
server_default=text("now()"),
),
)
updated_at: datetime | None = Field(
default=None,
sa_column=Column(
DateTime(timezone=True),
nullable=False,
server_default=text("now()"),
),
)
document_people: list["DocumentPerson"] = Relationship(
back_populates="document",
sa_relationship_kwargs={"lazy": "raise", "passive_deletes": True},
)
jobs: list["Job"] = Relationship(
back_populates="document",
sa_relationship_kwargs={"lazy": "raise", "passive_deletes": True},
)
sources: list["Source"] = Relationship(
back_populates="document",
sa_relationship_kwargs={"lazy": "raise", "passive_deletes": True},
)
class DocumentPerson(SQLModel, table=True):
__tablename__ = "document_person"
__table_args__ = (
UniqueConstraint(
"document_id",
"person_id",
"role",
name="unique_document_person_role",
),
Index("idx_document_person_doc", "document_id"),
Index("idx_document_person_per", "person_id"),
)
id: UUID | None = Field(
default=None,
sa_column=Column(
PostgreSQLUUID(as_uuid=True),
primary_key=True,
server_default=text("gen_random_uuid()"),
),
)
document_id: UUID = Field(
sa_column=Column(
PostgreSQLUUID(as_uuid=True),
ForeignKey("document.id", ondelete="CASCADE"),
nullable=False,
),
)
person_id: UUID = Field(
sa_column=Column(
PostgreSQLUUID(as_uuid=True),
ForeignKey("person.id", ondelete="CASCADE"),
nullable=False,
),
)
role: PersonRole = Field(sa_column=Column(String(20), nullable=False))
created_at: datetime | None = Field(
default=None,
sa_column=Column(
DateTime(timezone=True),
nullable=False,
server_default=text("now()"),
),
)
document: Document | None = Relationship(
back_populates="document_people",
sa_relationship_kwargs={"lazy": "raise"},
)
person: Person | None = Relationship(
back_populates="document_people",
sa_relationship_kwargs={"lazy": "raise"},
)
class Job(SQLModel, table=True):
__tablename__ = "job"
__table_args__ = (Index("idx_job_document", "document_id"),)
id: UUID | None = Field(
default=None,
sa_column=Column(
PostgreSQLUUID(as_uuid=True),
primary_key=True,
server_default=text("gen_random_uuid()"),
),
)
document_id: UUID = Field(
sa_column=Column(
PostgreSQLUUID(as_uuid=True),
ForeignKey("document.id", ondelete="CASCADE"),
nullable=False,
),
)
status: JobStatus = Field(
default=JobStatus.QUEUED,
sa_column=Column(
String(50),
nullable=False,
server_default=text("'queued'"),
),
)
retry_count: int = Field(
default=0,
sa_column=Column(
Integer,
nullable=False,
server_default=text("0"),
),
)
provider: str = Field(sa_column=Column(Text, nullable=False))
model: str = Field(sa_column=Column(Text, nullable=False))
prompt_name: str | None = Field(default=None, sa_column=Column(Text))
date_created: datetime | None = Field(
default=None,
sa_column=Column(
DateTime(timezone=True),
nullable=False,
server_default=text("now()"),
),
)
date_updated: datetime | None = Field(
default=None,
sa_column=Column(
DateTime(timezone=True),
nullable=False,
server_default=text("now()"),
),
)
document: Document | None = Relationship(
back_populates="jobs",
sa_relationship_kwargs={"lazy": "raise"},
)
job_sources: list["JobSource"] = Relationship(
back_populates="job",
sa_relationship_kwargs={"lazy": "raise", "passive_deletes": True},
)
class Source(SQLModel, table=True):
__tablename__ = "source"
__table_args__ = (
Index("idx_source_document", "document_id"),
Index("idx_source_page_order", "document_id", "page_number"),
)
id: UUID | None = Field(
default=None,
sa_column=Column(
PostgreSQLUUID(as_uuid=True),
primary_key=True,
server_default=text("gen_random_uuid()"),
),
)
document_id: UUID = Field(
sa_column=Column(
PostgreSQLUUID(as_uuid=True),
ForeignKey("document.id", ondelete="CASCADE"),
nullable=False,
),
)
page_number: int = Field(
default=1,
sa_column=Column(
Integer,
nullable=False,
server_default=text("1"),
),
)
upload_name: str = Field(sa_column=Column(Text, nullable=False))
filename: str = Field(sa_column=Column(Text, nullable=False))
file_path: str = Field(sa_column=Column(Text, nullable=False))
raw_transcription: str | None = Field(default=None, sa_column=Column(Text))
revised_text: str | None = Field(default=None, sa_column=Column(Text))
date_uploaded: datetime | None = Field(
default=None,
sa_column=Column(
DateTime(timezone=True),
nullable=False,
server_default=text("now()"),
),
)
date_revised: datetime | None = Field(
default=None,
sa_column=Column(DateTime(timezone=True)),
)
document: Document | None = Relationship(
back_populates="sources",
sa_relationship_kwargs={"lazy": "raise"},
)
job_sources: list["JobSource"] = Relationship(
back_populates="source",
sa_relationship_kwargs={"lazy": "raise", "passive_deletes": True},
)
class JobSource(SQLModel, table=True):
__tablename__ = "job_source"
__table_args__ = (
UniqueConstraint("job_id", "source_id", name="unique_job_source"),
Index("idx_job_source_job", "job_id"),
Index("idx_job_source_source", "source_id"),
Index(
"idx_job_source_ai_metadata",
"ai_metadata",
postgresql_using="gin",
),
)
id: UUID | None = Field(
default=None,
sa_column=Column(
PostgreSQLUUID(as_uuid=True),
primary_key=True,
server_default=text("gen_random_uuid()"),
),
)
job_id: UUID = Field(
sa_column=Column(
PostgreSQLUUID(as_uuid=True),
ForeignKey("job.id", ondelete="CASCADE"),
nullable=False,
),
)
source_id: UUID = Field(
sa_column=Column(
PostgreSQLUUID(as_uuid=True),
ForeignKey("source.id", ondelete="CASCADE"),
nullable=False,
),
)
status: JobSourceStatus = Field(
default=JobSourceStatus.PENDING,
sa_column=Column(
String(50),
nullable=False,
server_default=text("'pending'"),
),
)
raw_transcription: str | None = Field(default=None, sa_column=Column(Text))
ai_metadata: JsonValue | None = Field(
default=None,
sa_column=Column(JSONB),
)
raw_api_response: JsonValue | None = Field(
default=None,
sa_column=Column(JSONB),
)
error_detail: str | None = Field(default=None, sa_column=Column(Text))
executed_at: datetime | None = Field(
default=None,
sa_column=Column(
DateTime(timezone=True),
nullable=False,
server_default=text("now()"),
),
)
job: Job | None = Relationship(
back_populates="job_sources",
sa_relationship_kwargs={"lazy": "raise"},
)
source: Source | None = Relationship(
back_populates="job_sources",
sa_relationship_kwargs={"lazy": "raise"},
)
```
The enum annotations validate application values while the mapped columns retain the `VARCHAR` types specified by the DDL. PostgreSQL owns generated UUIDs and timestamps through `server_default`; call `session.refresh(instance)` after a flush or commit when those generated values are needed immediately.
`ai_metadata`, `raw_api_response`, and `metadata_` accept any JSON value supported by `JSONB`. Validate provider-specific payload structure before assigning it to these fields, while preserving the complete raw response in `raw_api_response`.
Relationships use `lazy="raise"` to prevent implicit database I/O in async code. Queries must explicitly load relationships they need, for example with `selectinload()`.
+183
View File
@@ -0,0 +1,183 @@
# Version 2 Plan
## Purpose
Version 2 updates the existing SQLModel domain schema to support multi-page documents, page-level transcription results, richer document metadata, and author/recipient attribution.
PostgreSQL support is already present in the database runtime. V2 does not require a database-layer rewrite or a general SQLite-to-PostgreSQL migration system. PostgreSQL adoption consists primarily of selecting the existing PostgreSQL settings, provisioning the database, creating the V2 schema, and verifying the application against it.
The main implementation effort is the schema update and the application changes that depend on it.
---
## Current State
- The application uses Python 3.12, Pydantic v2, SQLModel, and async SQLAlchemy sessions.
- The database engine already supports both SQLite and PostgreSQL through `SqliteSettings` and `PostgresSettings`.
- The PostgreSQL async driver is installed and the engine already builds `postgresql+asyncpg` connections.
- Schema bootstrap currently uses `SQLModel.metadata.create_all()`.
- SQLite remains the default local configuration and the current Compose configuration still selects SQLite.
- The current V1 domain contains `Document`, `Source`, `Job`, and `Revision` tables.
- The V2 target is defined in [V2 DB Schema](V2%20DB%20Schema.md) and [V2 PostgreSQL DDL Specification](V2%20PostgreSQL%20DDL%20Specification.md).
---
## V2 Outcomes
1. **V2 schema implemented**
- SQLModel models, relationships, enums, constraints, and indexes match the approved V2 schema.
1. **Page-level batch processing supported**
- A job can process multiple sources and retain an independent result for each source through `JobSource`.
1. **Document metadata expanded**
- Documents support ordered pages, descriptive metadata, and multiple authors and recipients.
1. **Raw provider data retained**
- Complete provider payloads are stored in PostgreSQL `JSONB` without flattening or discarding fields.
- Stored documents remain suitable for a future MongoDB import if one is ever needed.
1. **PostgreSQL enabled through configuration**
- The application starts against a provisioned PostgreSQL database using the existing runtime path.
1. **Existing workflows remain reliable**
- Upload, worker execution, status inspection, and transcription revision work with the new schema.
---
## Non-Goals
- Rewriting the database engine or session layer
- Building a general-purpose SQLite-to-PostgreSQL migration utility
- Rehearsing a production database cutover when no production dataset requires preservation
- Running or integrating MongoDB in V2
- Building MongoDB projections, synchronization, or fallback behavior
- Replacing Python, Pydantic, SQLModel, or SQLAlchemy
- Supporting more than one active human revision per source
If an existing SQLite dataset must be retained, define a small one-time import task separately. It is not part of the default V2 implementation path.
---
## Scope
### A) SQLModel schema update (primary)
- Add `Person`, `DocumentPerson`, and `JobSource` models.
- Expand `Document` with type, date, location, notes, archive identifier, and timestamps.
- Update `Source` with page ordering, active raw transcription, revised text, and revision timestamp.
- Update `Job` for batch execution and the `partial_success` terminal state.
- Replace the standalone `Revision` table with revision fields on `Source`.
- Remove the direct `Source.job_id` relationship; connect sources to jobs through `JobSource`.
- Add role, job status, and job-source status enums.
- Add required uniqueness constraints, foreign-key delete behavior, lookup indexes, and PostgreSQL JSON indexes.
- Keep model definitions aligned with [V2 Python Pydantic Models](V2%20Python%20Pydantic%20Models.md).
### B) Raw document storage
- Store structured AI metadata in `job_source.ai_metadata` as `JSONB`.
- Store the complete raw provider response in `job_source.raw_api_response` as `JSONB`.
- Preserve the original document structure, field names, nested values, and unknown fields in the raw response.
- Keep validation of extracted application fields separate from retention of the raw response.
- Serialize UUIDs and datetimes using portable string representations.
- Use MongoDB Extended JSON representations only if a provider value cannot be represented faithfully in standard JSON.
- Do not store opaque BSON bytes in PostgreSQL unless a future payload contains BSON-only values that cannot be preserved in `JSONB`.
### C) Schema creation and verification
- Use a fresh V2 database during development unless preservation of existing data becomes a requirement.
- Create the schema from SQLModel metadata and verify it against the approved DDL.
- Keep SQLite available for fast unit tests where its behavior is equivalent.
- Add focused PostgreSQL integration tests for native UUIDs, JSON storage, constraints, indexes, and transactions.
- Introduce migration tooling only if V2 must update a populated deployed database in place.
### D) Service and worker alignment
- Update document, source, job, and store operations for the new relationships.
- Create one `JobSource` row per source included in a job.
- Persist page-level status, transcription, AI metadata, raw provider response, and errors on `JobSource`.
- Derive the parent job status from its page results:
- `completed` when all pages succeed
- `partial_success` when successful and failed pages are mixed
- `failed` when all pages fail or a job-level failure prevents execution
- Update `Source.raw_transcription` after a successful page result while keeping the original `JobSource.raw_transcription` immutable.
- Read `Source.revised_text` in preference to `Source.raw_transcription` when presenting active text.
### E) Document and multi-image workflows
- Require a document before associating uploaded sources.
- Support uploading multiple images into one document.
- Preserve page order through `Source.page_number`.
- Allow a job to include one or more sources from the same document.
- Update job details to show the document, each source filename, page order, page status, and page-level errors.
- Define a practical upload limit and split oversized selections into manageable batches if needed.
### F) PostgreSQL configuration
- Provision PostgreSQL for local and deployed environments.
- Configure the existing `Settings.database` field with PostgreSQL host, port, database, user, and password values.
- Update Compose and environment configuration to stop selecting SQLite.
- Decide whether schema bootstrap is enabled for local development or performed as a separate deployment step.
- Run a connectivity and schema smoke test against PostgreSQL.
- Keep uploaded files on a persistent, backup-capable path outside the application image.
---
## Milestones
### M1 - Schema models
- Implement the V2 SQLModel models and enums.
- Implement relationships, constraints, indexes, and JSON column types.
- Update the Pydantic data contracts where model decisions change.
- Add schema-focused tests.
**Exit criteria:** SQLModel metadata represents the approved V2 schema and schema tests pass.
### M2 - Persistence and worker behavior
- Update database operations and services for the V2 entities.
- Implement page-level `JobSource` execution records.
- Preserve complete raw provider responses in `JSONB`.
- Implement aggregate job status calculation.
- Add transaction, partial-success, and failure-isolation tests.
**Exit criteria:** single-page and multi-page jobs persist correct page, raw payload, and aggregate states.
### M3 - Document and upload workflows
- Update document creation and source association flows.
- Add ordered multi-image upload.
- Update job and document detail views for page-level results.
- Add focused UI and service tests.
**Exit criteria:** a user can create a document, upload ordered pages, run a job, and inspect each result.
### M4 - PostgreSQL verification and release
- Switch local or test configuration to the existing PostgreSQL runtime path.
- Create the V2 schema in a fresh PostgreSQL database.
- Run PostgreSQL-specific schema and workflow tests.
- Document startup, backup, and recovery settings.
- Run the final regression suite.
**Exit criteria:** V2 workflows pass against PostgreSQL and release checks are complete.
---
## Risks and Mitigations
- **Model and DDL drift** -> compare generated metadata with the approved schema and test named constraints and indexes.
- **Raw payload loss** -> retain the complete provider response separately from validated and extracted fields.
- **Cross-database differences** -> retain fast SQLite tests but verify PostgreSQL-native UUID, JSON, and index behavior in integration tests.
- **Batch state errors** -> test all-success, mixed-result, and all-failed jobs explicitly.
- **Page ordering errors** -> enforce uniqueness and ordering rules for document pages.
- **Unexpected data-preservation need** -> confirm whether existing SQLite data matters before implementation; add a one-time importer only when required.
- **Worker regressions** -> preserve terminal-state and retry reliability tests while changing persistence ownership.
---
## Suggested First Tasks
1. Update `src/transcription/db/models.py` to represent the approved V2 schema.
2. Add schema tests for tables, columns, relationships, constraints, indexes, and enums.
3. Define and test lossless raw provider response storage in `job_source.raw_api_response`.
4. Update database operations and services to use `JobSource` and source-level revisions.
5. Add page-result aggregation tests before changing the worker workflow.
6. Update document and multi-image upload flows.
7. Select PostgreSQL in configuration and run the integration suite against a fresh V2 database.
+20
View File
@@ -0,0 +1,20 @@
import uvicorn
from .config import LOGGING_CONFIG
from .config import get_settings
def main() -> None:
settings = get_settings()
uvicorn.run(
"transcription.app:create_app",
factory=True,
host=settings.host,
port=settings.port,
log_level=LOGGING_CONFIG.get("root", {}).get("level", "info").lower(),
reload=settings.reload,
)
if __name__ == "__main__":
main()
+17 -7
View File
@@ -16,11 +16,14 @@ from fastapi.staticfiles import StaticFiles
from .api.errors import register_error_handlers from .api.errors import register_error_handlers
from .api.health import router as health_router from .api.health import router as health_router
from .config import Settings
from .config import configure_logging from .config import configure_logging
from .config import get_settings from .config import get_settings
from .db import create_all from .db import create_all
from .db import dispose_database_runtime
from .db import initialize_database_runtime from .db import initialize_database_runtime
from .db.engine import get_database_url
from .db.engine import resolve_engine
from .db.session import dispose_session_factory
from .services import ServiceBundle from .services import ServiceBundle
from .services.jobs import JobService from .services.jobs import JobService
from .ui import register_pages from .ui import register_pages
@@ -39,7 +42,7 @@ async def _lifespan(app: FastAPI):
app.state.runtime = initialize_database_runtime(settings=settings) app.state.runtime = initialize_database_runtime(settings=settings)
if settings.should_bootstrap_schema: if settings.should_bootstrap_schema:
await create_all(engine=app.state.runtime.engine) await create_all(engine=resolve_engine(settings=settings))
settings.upload_dir.mkdir(parents=True, exist_ok=True) settings.upload_dir.mkdir(parents=True, exist_ok=True)
settings.prompt_dir.mkdir(parents=True, exist_ok=True) settings.prompt_dir.mkdir(parents=True, exist_ok=True)
@@ -47,7 +50,10 @@ async def _lifespan(app: FastAPI):
await _recover_stale_processing_jobs(app) await _recover_stale_processing_jobs(app)
async with AsyncExitStack() as stack: async with AsyncExitStack() as stack:
stack.push_async_callback(dispose_database_runtime) stack.push_async_callback(
dispose_session_factory,
database_url=get_database_url(settings),
)
stop_event, worker_notifier = await stack.enter_async_context( stop_event, worker_notifier = await stack.enter_async_context(
worker_consumer_lifespan( worker_consumer_lifespan(
session_factory=app.state.runtime.session_factory, session_factory=app.state.runtime.session_factory,
@@ -73,14 +79,14 @@ async def _recover_stale_processing_jobs(app: FastAPI) -> None:
logger.warning("Recovered %s stale processing job(s) at startup", recovered) logger.warning("Recovered %s stale processing job(s) at startup", recovered)
def create_app() -> FastAPI: def create_app(settings: Settings | None = None) -> FastAPI:
"""Create and configure the FastAPI application.""" """Create and configure the FastAPI application."""
app = FastAPI(title="Transcription", lifespan=_lifespan) app = FastAPI(title="Transcription", lifespan=_lifespan)
settings = get_settings() active_settings = settings or get_settings()
app.state.settings = settings app.state.settings = active_settings
app.mount( app.mount(
"/uploads", "/uploads",
StaticFiles(directory=settings.upload_dir, check_dir=False), StaticFiles(directory=active_settings.upload_dir, check_dir=False),
name="uploads", name="uploads",
) )
@@ -92,6 +98,10 @@ def create_app() -> FastAPI:
async def ui_redirect() -> RedirectResponse: async def ui_redirect() -> RedirectResponse:
return RedirectResponse(url="/ui/upload", status_code=status.HTTP_307_TEMPORARY_REDIRECT) return RedirectResponse(url="/ui/upload", status_code=status.HTTP_307_TEMPORARY_REDIRECT)
@app.get("/healthz")
def health() -> dict[str, str]:
return {"status": "ok"}
register_error_handlers(app) register_error_handlers(app)
register_pages(app) register_pages(app)
app.include_router(health_router) app.include_router(health_router)
+1 -1
View File
@@ -7,7 +7,7 @@ from sqlalchemy.ext.asyncio import async_sessionmaker
from sqlmodel.ext.asyncio.session import AsyncSession from sqlmodel.ext.asyncio.session import AsyncSession
from transcription.db.runtime import DatabaseRuntime from transcription.db.runtime import DatabaseRuntime
from transcription.db.runtime import get_session_factory from transcription.db.session import get_session_factory
from transcription.worker import WorkerNotifier from transcription.worker import WorkerNotifier
from transcription.worker import resolve_worker_notifier from transcription.worker import resolve_worker_notifier
+44 -13
View File
@@ -6,12 +6,16 @@ are resolved by the provider adapters, not here.
""" """
import logging.config import logging.config
from contextvars import ContextVar
from enum import StrEnum from enum import StrEnum
from functools import cache
from pathlib import Path from pathlib import Path
from typing import Annotated
from typing import Any
from typing import Literal from typing import Literal
from pydantic import BaseModel
from pydantic import Field from pydantic import Field
from pydantic import SecretStr
from pydantic_settings import BaseSettings from pydantic_settings import BaseSettings
from pydantic_settings import SettingsConfigDict from pydantic_settings import SettingsConfigDict
@@ -22,13 +26,42 @@ class Provider(StrEnum):
OPENROUTER = "openrouter" OPENROUTER = "openrouter"
class SqliteSettings(BaseModel):
driver: Literal["sqlite"] = "sqlite"
path: str = "app.db"
class PostgresSettings(BaseModel):
driver: Literal["postgres"] = "postgres"
host: str
port: int = 5432
database: str
user: str
password: SecretStr
DatabaseSettings = Annotated[
SqliteSettings | PostgresSettings,
Field(discriminator="driver"),
]
class Settings(BaseSettings): class Settings(BaseSettings):
model_config = SettingsConfigDict( model_config = SettingsConfigDict(
env_file=".env", env_file=".env",
env_file_encoding="utf-8", env_file_encoding="utf-8",
extra="ignore", extra="ignore",
cli_parse_args=True,
cli_implicit_flags=True,
cli_kebab_case=True,
) )
# --- NiceGUI Server ---
host: str = "0.0.0.0"
port: int = 8000
log_level: Literal["critical", "error", "warning", "info", "debug", "trace"] = "info"
reload: bool = False
# --- AI provider --- # --- AI provider ---
provider: Provider = Provider.OPENROUTER provider: Provider = Provider.OPENROUTER
openrouter_api_key: str openrouter_api_key: str
@@ -40,8 +73,9 @@ class Settings(BaseSettings):
environment: Literal["development", "test", "production"] = "development" environment: Literal["development", "test", "production"] = "development"
# --- persistence --- # --- persistence ---
database: DatabaseSettings = Field(default_factory=SqliteSettings)
database_url: str = "sqlite:///./transcription.db" database_url: str = "sqlite:///./transcription.db"
bootstrap_schema_on_startup: bool | None = None bootstrap_schema_on_startup: bool = False
sqlite_check_same_thread: bool = False sqlite_check_same_thread: bool = False
# --- filesystem paths --- # --- filesystem paths ---
@@ -64,18 +98,12 @@ class Settings(BaseSettings):
return self.environment in {"development", "test"} return self.environment in {"development", "test"}
_settings: ContextVar[Settings | None] = ContextVar("settings", default=None) @cache
def get_settings(**kwargs) -> Settings: def get_settings(**kwargs) -> Settings:
settings = _settings.get() return Settings(**kwargs)
if settings is None:
settings = Settings(**kwargs) # pyright: ignore[reportCallIssue]
_settings.set(settings)
return settings
LOGGING_CONFIG: dict[str, object] = { LOGGING_CONFIG: dict[str, Any] = {
"version": 1, "version": 1,
"disable_existing_loggers": False, "disable_existing_loggers": False,
"formatters": { "formatters": {
@@ -105,7 +133,10 @@ LOGGING_CONFIG: dict[str, object] = {
} }
def configure_logging() -> None: def configure_logging(settings: Settings | None = None) -> None:
"""Configure root logging once at startup.""" """Configure root logging once at startup."""
logging.config.dictConfig(LOGGING_CONFIG) cfg = LOGGING_CONFIG.copy()
active_settings = settings or get_settings()
cfg["loggers"]["transcription"]["level"] = active_settings.log_level.upper()
logging.config.dictConfig(cfg)
logger.debug("Logging configured") logger.debug("Logging configured")
+9 -2
View File
@@ -1,6 +1,13 @@
from .operations import create_all from .operations import create_all
from .runtime import dispose_database_runtime from .runtime import dispose_database_runtime
from .runtime import get_session
from .runtime import initialize_database_runtime from .runtime import initialize_database_runtime
from .session import session_scope
from .session import transaction_scope
__all__ = ["create_all", "dispose_database_runtime", "get_session", "initialize_database_runtime"] __all__ = [
"create_all",
"dispose_database_runtime",
"initialize_database_runtime",
"session_scope",
"transaction_scope",
]
+60
View File
@@ -0,0 +1,60 @@
from functools import cache
from typing import Any
from sqlalchemy import URL
from sqlalchemy import StaticPool
from sqlalchemy.ext.asyncio import AsyncEngine
from sqlalchemy.ext.asyncio import create_async_engine
from ..config import PostgresSettings
from ..config import Settings
from ..config import SqliteSettings
from ..config import get_settings
def get_database_url(settings: Settings) -> str:
match settings.database:
case SqliteSettings(path=path):
url = URL.create(
drivername="sqlite+aiosqlite",
database=path,
)
case PostgresSettings() as database:
url = URL.create(
drivername="postgresql+asyncpg",
host=database.host,
port=database.port,
database=database.database,
username=database.user,
password=database.password.get_secret_value(),
)
return url.render_as_string(hide_password=False)
def resolve_engine(settings: Settings | None = None) -> AsyncEngine:
active_settings = settings or get_settings()
return get_engine(get_database_url(active_settings))
@cache
def get_engine(database_url: str) -> AsyncEngine:
kwargs: dict[str, Any] = {"echo": False, "pool_pre_ping": True}
if database_url.startswith("sqlite"):
kwargs["connect_args"] = {"check_same_thread": False}
if ":memory:" in database_url:
kwargs["poolclass"] = StaticPool
return create_async_engine(database_url, **kwargs)
async def dispose_engine(database_url: str) -> None:
engine = get_engine(database_url)
try:
await engine.dispose()
finally:
get_engine.cache_clear()
async def refresh_engine(database_url: str) -> AsyncEngine:
await dispose_engine(database_url)
return get_engine(database_url)
+17 -21
View File
@@ -10,13 +10,25 @@ from sqlmodel import SQLModel
from sqlmodel import select from sqlmodel import select
from sqlmodel.ext.asyncio.session import AsyncSession from sqlmodel.ext.asyncio.session import AsyncSession
from ..models import Job from .engine import resolve_engine
from ..models import JobStatus from .models import Job
from .runtime import get_engine from .models import JobStatus
logger = logging.getLogger(__name__) logger = logging.getLogger(__name__)
async def create_all(*, engine: AsyncEngine | None = None) -> None:
"""Create all tables on the selected engine."""
# Import models so SQLModel metadata is fully registered before bootstrap.
from transcription.db import models as _models # noqa: F401
active_engine = engine or resolve_engine()
async with active_engine.begin() as connection:
await connection.run_sync(SQLModel.metadata.create_all)
await connection.run_sync(_ensure_sqlite_compat_columns)
logger.debug("Database schema bootstrap complete for database_url=%s", active_engine.url)
async def get_next_queued_job(*, session: AsyncSession) -> Job | None: async def get_next_queued_job(*, session: AsyncSession) -> Job | None:
"""Get the next queued job, if any.""" """Get the next queued job, if any."""
result = await session.exec( result = await session.exec(
@@ -28,18 +40,6 @@ async def get_next_queued_job(*, session: AsyncSession) -> Job | None:
return result.first() return result.first()
async def create_all(*, engine: AsyncEngine | None = None) -> None:
"""Create all tables on the selected engine."""
# Import models so SQLModel metadata is fully registered before bootstrap.
from transcription import models as _models # noqa: F401
active_engine = engine or get_engine()
async with active_engine.begin() as connection:
await connection.run_sync(SQLModel.metadata.create_all)
await connection.run_sync(_ensure_sqlite_compat_columns)
logger.debug("Database schema bootstrap complete for database_url=%s", active_engine.url)
def _ensure_sqlite_compat_columns(connection: Connection) -> None: def _ensure_sqlite_compat_columns(connection: Connection) -> None:
"""Apply lightweight dev/test SQLite compatibility column patches. """Apply lightweight dev/test SQLite compatibility column patches.
@@ -68,12 +68,8 @@ def _ensure_sqlite_compat_columns(connection: Connection) -> None:
break break
if not has_unique_source: if not has_unique_source:
connection.execute( connection.execute(
text( text("CREATE UNIQUE INDEX IF NOT EXISTS ux_revision_source_id ON revision(source_id)")
"CREATE UNIQUE INDEX IF NOT EXISTS "
"ux_revision_source_id ON revision(source_id)"
)
) )
logger.warning( logger.warning(
"Applied SQLite compatibility schema patch " "Applied SQLite compatibility schema patch table=revision unique_index=ux_revision_source_id"
"table=revision unique_index=ux_revision_source_id"
) )
+6 -58
View File
@@ -1,18 +1,16 @@
import logging import logging
from collections.abc import AsyncGenerator
from contextlib import asynccontextmanager
from contextvars import ContextVar from contextvars import ContextVar
from dataclasses import dataclass from dataclasses import dataclass
from functools import partial
from sqlalchemy.ext.asyncio import AsyncEngine from sqlalchemy.ext.asyncio import AsyncEngine
from sqlalchemy.ext.asyncio import async_sessionmaker from sqlalchemy.ext.asyncio import async_sessionmaker
from sqlalchemy.ext.asyncio import create_async_engine
from sqlmodel.ext.asyncio.session import AsyncSession from sqlmodel.ext.asyncio.session import AsyncSession
from sqlmodel.pool import StaticPool
from ..config import Settings from ..config import Settings
from ..config import get_settings from ..config import get_settings
from .engine import get_database_url
from .engine import get_engine
from .session import get_session_factory
logger = logging.getLogger(__name__) logger = logging.getLogger(__name__)
@@ -37,33 +35,6 @@ async def dispose_database_runtime() -> None:
_runtime.set(None) _runtime.set(None)
def _to_async_database_url(database_url: str) -> str:
"""Normalize configured database URL to an async SQLAlchemy driver URL."""
if database_url.startswith("sqlite://") and not database_url.startswith("sqlite+aiosqlite://"):
return database_url.replace("sqlite://", "sqlite+aiosqlite://", 1)
if database_url.startswith("postgresql://") and not database_url.startswith("postgresql+asyncpg://"):
return database_url.replace("postgresql://", "postgresql+asyncpg://", 1)
return database_url
def _build_engine(settings: Settings) -> AsyncEngine:
database_url = _to_async_database_url(settings.database_url)
engine_factory = partial(
create_async_engine,
url=database_url,
echo=False,
pool_pre_ping=True,
)
if database_url.startswith("sqlite"):
sqlite_connect_settings = {"check_same_thread": settings.sqlite_check_same_thread}
engine_factory = partial(engine_factory, connect_args=sqlite_connect_settings)
if ":memory:" in database_url:
engine_factory = partial(engine_factory, poolclass=StaticPool)
return engine_factory()
def initialize_database_runtime(*, settings: Settings | None = None) -> DatabaseRuntime: def initialize_database_runtime(*, settings: Settings | None = None) -> DatabaseRuntime:
"""Initialize lifespan-owned async DB resources once per process.""" """Initialize lifespan-owned async DB resources once per process."""
runtime = _runtime.get() runtime = _runtime.get()
@@ -71,33 +42,10 @@ def initialize_database_runtime(*, settings: Settings | None = None) -> Database
return runtime return runtime
active_settings = settings or get_settings() active_settings = settings or get_settings()
engine = _build_engine(active_settings) database_url = get_database_url(active_settings)
session_factory = async_sessionmaker(engine, class_=AsyncSession, expire_on_commit=False) engine = get_engine(database_url)
session_factory = get_session_factory(database_url)
runtime = DatabaseRuntime(engine=engine, session_factory=session_factory) runtime = DatabaseRuntime(engine=engine, session_factory=session_factory)
_runtime.set(runtime) _runtime.set(runtime)
logger.debug("Initialized async database runtime for database_url=%s", engine.url) logger.debug("Initialized async database runtime for database_url=%s", engine.url)
return runtime return runtime
def get_engine(settings: Settings | None = None) -> AsyncEngine:
"""Return the current async SQLAlchemy engine."""
runtime = _runtime.get() or initialize_database_runtime(settings=settings)
return runtime.engine
def get_session_factory(settings: Settings | None = None) -> async_sessionmaker[AsyncSession]:
"""Return the shared async session factory."""
runtime = _runtime.get() or initialize_database_runtime(settings=settings)
return runtime.session_factory
@asynccontextmanager
async def get_session(
*,
settings: Settings | None = None,
session_factory: async_sessionmaker[AsyncSession] | None = None,
) -> AsyncGenerator[AsyncSession]:
"""Yield a database session and ensure cleanup."""
active_session_factory = session_factory or get_session_factory(settings)
async with active_session_factory() as session:
yield session
+79
View File
@@ -0,0 +1,79 @@
from collections.abc import AsyncGenerator
from contextlib import asynccontextmanager
from functools import cache
from typing import Annotated
from fastapi import Depends
from sqlalchemy.ext.asyncio import AsyncSessionTransaction
from sqlalchemy.ext.asyncio import async_sessionmaker
from sqlmodel.ext.asyncio.session import AsyncSession
from ..config import get_settings
from .engine import dispose_engine
from .engine import get_database_url
from .engine import get_engine
type SessionFactory = async_sessionmaker[AsyncSession]
@cache
def get_session_factory(database_url: str) -> SessionFactory:
return async_sessionmaker(
bind=get_engine(database_url),
class_=AsyncSession,
expire_on_commit=False,
)
def resolve_session_factory(database_url: str | None = None) -> SessionFactory:
return get_session_factory(database_url or get_database_url(get_settings()))
type SessionFactoryDep = Annotated[SessionFactory, Depends(resolve_session_factory)]
async def dispose_session_factory(database_url: str) -> None:
get_session_factory.cache_clear()
await dispose_engine(database_url)
@asynccontextmanager
async def session_scope(
*,
database_url: str | None = None,
session: AsyncSession | None = None,
) -> AsyncGenerator[AsyncSession]:
if session is not None:
yield session
return
session_factory = resolve_session_factory(database_url)
async with session_factory() as owned_session:
yield owned_session
type SessionScopeDep = Annotated[AsyncSession, Depends(session_scope)]
@asynccontextmanager
async def transaction_scope(
*,
database_url: str | None = None,
session: AsyncSessionTransaction | None = None,
) -> AsyncGenerator[AsyncSessionTransaction]:
match session:
case AsyncSession() as async_session:
if not async_session.in_transaction():
raise RuntimeError("A supplied session must have an active transaction")
yield async_session
return
case AsyncSessionTransaction() as async_transaction:
yield async_transaction
return
session_factory = resolve_session_factory(database_url)
async with session_factory().begin() as owned_session:
yield owned_session
type TransactionScopeDep = Annotated[AsyncSessionTransaction, Depends(transaction_scope)]
+5 -9
View File
@@ -8,7 +8,8 @@ from sqlmodel.ext.asyncio.session import AsyncSession
from ..config import Settings from ..config import Settings
from ..config import get_settings from ..config import get_settings
from ..db.runtime import get_session_factory from ..db.session import resolve_session_factory
from ..db.session import session_scope
class ServiceBase(ABC): class ServiceBase(ABC):
@@ -24,19 +25,14 @@ class ServiceBase(ABC):
queue: asyncio.Queue | None = None, queue: asyncio.Queue | None = None,
): ):
self.settings = get_settings() self.settings = get_settings()
self.session_factory = session_factory or get_session_factory() self.session_factory = session_factory or resolve_session_factory()
self.queue = queue or asyncio.Queue() self.queue = queue or asyncio.Queue()
@asynccontextmanager @asynccontextmanager
async def _session_scope(self, session: AsyncSession | None = None): async def _session_scope(self, session: AsyncSession | None = None):
"""Provide a transactional scope around a series of operations.""" """Provide a transactional scope around a series of operations."""
if session is not None: async with session_scope(session=session) as active_session:
# Reuse the provided session if one is passed in yield active_session
yield session
else:
# Otherwise, create a new session for this scope
async with self.session_factory() as new_session:
yield new_session
async def _finalize( async def _finalize(
self, self,
+1 -1
View File
@@ -9,9 +9,9 @@ from sqlalchemy.orm import selectinload
from sqlmodel import select from sqlmodel import select
from sqlmodel.ext.asyncio.session import AsyncSession from sqlmodel.ext.asyncio.session import AsyncSession
from ..db.models import Document
from ..errors import AppError from ..errors import AppError
from ..errors import ErrorCategory from ..errors import ErrorCategory
from ..models import Document
from .base import ServiceBase from .base import ServiceBase
logger = logging.getLogger(__name__) logger = logging.getLogger(__name__)
+4 -8
View File
@@ -7,9 +7,9 @@ from sqlalchemy.orm import selectinload
from sqlmodel import select from sqlmodel import select
from sqlmodel.ext.asyncio.session import AsyncSession from sqlmodel.ext.asyncio.session import AsyncSession
from ..models import Job from ..db.models import Job
from ..models import JobStatus from ..db.models import JobStatus
from ..models import Source from ..db.models import Source
from .base import ServiceBase from .base import ServiceBase
@@ -170,11 +170,7 @@ class JobService(ServiceBase):
``stale_before`` are considered stale and re-queued. ``stale_before`` are considered stale and re-queued.
""" """
async with self._session_scope(session) as _session: async with self._session_scope(session) as _session:
query = ( query = select(Job).where(Job.status == JobStatus.PROCESSING).where(Job.date_updated < stale_before)
select(Job)
.where(Job.status == JobStatus.PROCESSING)
.where(Job.date_updated < stale_before)
)
stale_jobs = (await _session.exec(query)).all() stale_jobs = (await _session.exec(query)).all()
if not stale_jobs: if not stale_jobs:
return 0 return 0
+3 -3
View File
@@ -11,9 +11,9 @@ from transcription.config import get_settings
from transcription.errors import AppError from transcription.errors import AppError
from transcription.errors import ErrorCategory from transcription.errors import ErrorCategory
from ..models import Document from ..db.models import Document
from ..models import Job from ..db.models import Job
from ..models import Source from ..db.models import Source
from .documents import UploadJobResult from .documents import UploadJobResult
logger = logging.getLogger(__name__) logger = logging.getLogger(__name__)
+3 -3
View File
@@ -18,11 +18,11 @@ from sqlmodel.ext.asyncio.session import AsyncSession
from transcription.config import Settings from transcription.config import Settings
from transcription.config import get_settings from transcription.config import get_settings
from transcription.db.models import Job
from transcription.db.models import Revision
from transcription.db.models import Source
from transcription.errors import AppError from transcription.errors import AppError
from transcription.errors import ErrorCategory from transcription.errors import ErrorCategory
from transcription.models import Job
from transcription.models import Revision
from transcription.models import Source
from transcription.providers import ProviderAuthError from transcription.providers import ProviderAuthError
from transcription.providers import ProviderError from transcription.providers import ProviderError
from transcription.providers import ProviderResponseError from transcription.providers import ProviderResponseError
+3 -3
View File
@@ -5,13 +5,13 @@ from sqlmodel.ext.asyncio.session import AsyncSession
from ..config import Settings from ..config import Settings
from ..config import get_settings from ..config import get_settings
from ..db.models import Job
from ..db.models import JobStatus
from ..db.models import Source
from ..errors import AppError from ..errors import AppError
from ..errors import ErrorCategory from ..errors import ErrorCategory
from ..errors import classify_unexpected_error from ..errors import classify_unexpected_error
from ..errors import format_error_detail from ..errors import format_error_detail
from ..models import Job
from ..models import JobStatus
from ..models import Source
from ..providers import TranscriptionResult from ..providers import TranscriptionResult
from . import ServiceBundle from . import ServiceBundle
from .transcription import DEFAULT_PROMPT_FILE from .transcription import DEFAULT_PROMPT_FILE
@@ -10,7 +10,7 @@ from uuid import uuid4
from nicegui import ui from nicegui import ui
from transcription.config import get_settings from transcription.config import get_settings
from transcription.models import Source from transcription.db.models import Source
PANGOZOOM_CDN_URL = "https://unpkg.com/@panzoom/[email protected]/dist/panzoom.min.js" PANGOZOOM_CDN_URL = "https://unpkg.com/@panzoom/[email protected]/dist/panzoom.min.js"
UPLOADS_URL_PREFIX = "/uploads" UPLOADS_URL_PREFIX = "/uploads"
@@ -6,9 +6,9 @@ import logging
from nicegui import ui from nicegui import ui
from transcription.models import Job from transcription.db.models import Job
from transcription.models import Revision from transcription.db.models import Revision
from transcription.models import Source from transcription.db.models import Source
from transcription.ui.components.document_panzoom import render_document_panzoom from transcription.ui.components.document_panzoom import render_document_panzoom
from transcription.ui.components.transcript import render_original_transcription_card from transcription.ui.components.transcript import render_original_transcription_card
from transcription.ui.components.transcript import render_revision_row from transcription.ui.components.transcript import render_revision_row
@@ -9,8 +9,8 @@ from typing import Any
from nicegui import ui from nicegui import ui
from transcription.models import Job from transcription.db.models import Job
from transcription.models import Revision from transcription.db.models import Revision
type RevisionAction = Callable[[Revision], Awaitable[None] | None] type RevisionAction = Callable[[Revision], Awaitable[None] | None]
+6 -9
View File
@@ -4,19 +4,18 @@ from __future__ import annotations
from uuid import UUID from uuid import UUID
from fastapi import Request
from nicegui import ui from nicegui import ui
from transcription.app_state import resolve_session_factory from transcription.db.models import Job
from transcription.models import Job from transcription.db.models import JobStatus
from transcription.models import JobStatus from transcription.db.models import Source
from transcription.models import Source
from transcription.services.jobs import JobService from transcription.services.jobs import JobService
from transcription.services.transcription import TranscriptionService from transcription.services.transcription import TranscriptionService
from transcription.ui.components.app_shell import render_navigation_header from transcription.ui.components.app_shell import render_navigation_header
from transcription.ui.components.error_presenter import show_error from transcription.ui.components.error_presenter import show_error
from transcription.ui.components.table.jobs import render_jobs_table from transcription.ui.components.table.jobs import render_jobs_table
from ...db.session import SessionFactoryDep
from ..components.document_panzoom import render_document_panzoom from ..components.document_panzoom import render_document_panzoom
from ..components.table.jobs import JobTableRow from ..components.table.jobs import JobTableRow
from ..components.transcript import render_original_transcription_card from ..components.transcript import render_original_transcription_card
@@ -27,8 +26,7 @@ def register_page() -> None: # noqa: PLR0915
"""Register jobs list and detail routes.""" """Register jobs list and detail routes."""
@ui.page("/jobs") @ui.page("/jobs")
async def jobs_page(request: Request) -> None: async def jobs_page(session_factory: SessionFactoryDep) -> None:
session_factory = resolve_session_factory(request.app.state)
jobs_service = JobService(session_factory=session_factory) jobs_service = JobService(session_factory=session_factory)
render_navigation_header(current_path="/jobs") render_navigation_header(current_path="/jobs")
@@ -51,8 +49,7 @@ def register_page() -> None: # noqa: PLR0915
await render_table() await render_table()
@ui.page("/jobs/{job_id}") @ui.page("/jobs/{job_id}")
async def job_detail_page(job_id: str, request: Request) -> None: # noqa: PLR0915 async def job_detail_page(job_id: str, session_factory: SessionFactoryDep) -> None: # noqa: PLR0915
session_factory = resolve_session_factory(request.app.state)
jobs_service = JobService(session_factory=session_factory) jobs_service = JobService(session_factory=session_factory)
transcription_service = TranscriptionService(session_factory=session_factory) transcription_service = TranscriptionService(session_factory=session_factory)
render_navigation_header(current_path="/jobs") render_navigation_header(current_path="/jobs")
+2 -4
View File
@@ -5,8 +5,7 @@ from __future__ import annotations
from fastapi import Request from fastapi import Request
from nicegui import ui from nicegui import ui
from transcription.app_state import resolve_session_factory from transcription.db import session_scope
from transcription.db import get_session
from transcription.services.store import create_upload_job from transcription.services.store import create_upload_job
from transcription.ui.components.app_shell import render_navigation_header from transcription.ui.components.app_shell import render_navigation_header
from transcription.ui.components.upload import render_upload_widget from transcription.ui.components.upload import render_upload_widget
@@ -19,10 +18,9 @@ def register_page() -> None:
@ui.page("/upload", title="Upload Document") @ui.page("/upload", title="Upload Document")
def upload_page(request: Request) -> None: def upload_page(request: Request) -> None:
render_navigation_header(current_path="/upload") render_navigation_header(current_path="/upload")
session_factory = resolve_session_factory(request.app.state)
async def submit_upload(filename: str, file_bytes: bytes): async def submit_upload(filename: str, file_bytes: bytes):
async with get_session(session_factory=session_factory) as session: async with session_scope() as session:
return await create_upload_job( return await create_upload_job(
filename=filename, filename=filename,
file_bytes=file_bytes, file_bytes=file_bytes,
+2 -2
View File
@@ -14,7 +14,7 @@ from uuid import UUID
from sqlalchemy.ext.asyncio import async_sessionmaker from sqlalchemy.ext.asyncio import async_sessionmaker
from sqlmodel.ext.asyncio.session import AsyncSession from sqlmodel.ext.asyncio.session import AsyncSession
from transcription.db import get_session from transcription.db import session_scope
from transcription.errors import AppError from transcription.errors import AppError
from transcription.errors import classify_unexpected_error from transcription.errors import classify_unexpected_error
@@ -178,7 +178,7 @@ async def process_next_queued_job(
) )
if session is None: if session is None:
async with get_session(session_factory=session_factory) as local_session: async with session_scope(session_factory=session_factory) as local_session:
return await process_next_queued_job_workflow(services=services, session=local_session) return await process_next_queued_job_workflow(services=services, session=local_session)
return await process_next_queued_job_workflow(services=services, session=session) return await process_next_queued_job_workflow(services=services, session=session)
+12 -8
View File
@@ -13,11 +13,12 @@ from sqlmodel.pool import StaticPool
from transcription.config import Settings from transcription.config import Settings
from transcription.config import get_settings from transcription.config import get_settings
from transcription.db.engine import get_database_url
from transcription.db.engine import get_engine
from transcription.db.operations import create_all from transcription.db.operations import create_all
from transcription.db.runtime import dispose_database_runtime from transcription.db.session import dispose_session_factory
from transcription.db.runtime import get_engine from transcription.db.session import get_session_factory
from transcription.db.runtime import get_session from transcription.db.session import session_scope
from transcription.db.runtime import get_session_factory
from transcription.services.documents import DocumentService from transcription.services.documents import DocumentService
from transcription.services.jobs import JobService from transcription.services.jobs import JobService
@@ -39,23 +40,26 @@ def session():
async def default_settings(): async def default_settings():
"""Provide default settings for tests.""" """Provide default settings for tests."""
settings = get_settings(database_url="sqlite:///:memory:") settings = get_settings(database_url="sqlite:///:memory:")
await create_all(engine=get_engine(settings=settings)) db_url = get_database_url(settings)
await create_all(engine=get_engine(database_url=db_url))
return settings return settings
@pytest_asyncio.fixture @pytest_asyncio.fixture
async def async_session(default_settings: Settings): async def async_session(default_settings: Settings):
"""Provide a clean asynchronous database session for async tests.""" """Provide a clean asynchronous database session for async tests."""
async with get_session(settings=default_settings) as async_session: db_url = get_database_url(default_settings)
async with session_scope(database_url=db_url) as async_session:
yield async_session yield async_session
await dispose_database_runtime() await dispose_session_factory(db_url)
@pytest.fixture @pytest.fixture
def default_session_factory(default_settings: Settings): def default_session_factory(default_settings: Settings):
"""Provide a base fixture for tests that require database access.""" """Provide a base fixture for tests that require database access."""
session_factory = get_session_factory(settings=default_settings) db_url = get_database_url(default_settings)
session_factory = get_session_factory(database_url=db_url)
return session_factory return session_factory
+2 -2
View File
@@ -5,8 +5,8 @@ from pathlib import Path
import pytest import pytest
from transcription.config import Settings from transcription.config import Settings
from transcription.models import Job from transcription.db.models import Job
from transcription.models import JobStatus from transcription.db.models import JobStatus
from transcription.providers.base import TranscriptionResult from transcription.providers.base import TranscriptionResult
from transcription.services.store import create_upload_job from transcription.services.store import create_upload_job
from transcription.worker import process_next_queued_job from transcription.worker import process_next_queued_job
+4 -4
View File
@@ -2,10 +2,10 @@ from uuid import uuid4
import pytest import pytest
from transcription.models import Document from transcription.db.models import Document
from transcription.models import Job from transcription.db.models import Job
from transcription.models import JobStatus from transcription.db.models import JobStatus
from transcription.models import Source from transcription.db.models import Source
from transcription.services.documents import DocumentService from transcription.services.documents import DocumentService
from transcription.services.jobs import JobService from transcription.services.jobs import JobService
+4 -4
View File
@@ -4,10 +4,10 @@ from uuid import uuid4
import pytest import pytest
from transcription.models import Document from transcription.db.models import Document
from transcription.models import Job from transcription.db.models import Job
from transcription.models import JobStatus from transcription.db.models import JobStatus
from transcription.models import Source from transcription.db.models import Source
from transcription.services.documents import DocumentService from transcription.services.documents import DocumentService
from transcription.services.jobs import JobService from transcription.services.jobs import JobService
from transcription.services.transcription import TranscriptionService from transcription.services.transcription import TranscriptionService
+4 -4
View File
@@ -6,10 +6,10 @@ from uuid import uuid4
import pytest import pytest
from transcription.config import Settings from transcription.config import Settings
from transcription.models import Document from transcription.db.models import Document
from transcription.models import Job from transcription.db.models import Job
from transcription.models import JobStatus from transcription.db.models import JobStatus
from transcription.models import Source from transcription.db.models import Source
from transcription.services import ServiceBundle from transcription.services import ServiceBundle
from transcription.services.workflows import process_queued_job from transcription.services.workflows import process_queued_job
+5 -4
View File
@@ -4,17 +4,18 @@ import pytest
from sqlalchemy import inspect from sqlalchemy import inspect
from transcription.config import Settings from transcription.config import Settings
from transcription.config import SqliteSettings
from transcription.db import create_all from transcription.db import create_all
from transcription.db import dispose_database_runtime from transcription.db import dispose_database_runtime
from transcription.db import get_session
from transcription.db import initialize_database_runtime from transcription.db import initialize_database_runtime
from transcription.db import session_scope
@pytest.mark.asyncio @pytest.mark.asyncio
async def test_create_all_creates_expected_tables(tmp_path): async def test_create_all_creates_expected_tables(tmp_path):
settings = Settings( settings = Settings(
openrouter_api_key="test-key", openrouter_api_key="test-key",
database_url=f"sqlite:///{tmp_path / 'schema.db'}", database=SqliteSettings(path=str(tmp_path / "schema.db")),
environment="test", environment="test",
) )
runtime = initialize_database_runtime(settings=settings) runtime = initialize_database_runtime(settings=settings)
@@ -36,13 +37,13 @@ async def test_create_all_creates_expected_tables(tmp_path):
async def test_get_session_yields_async_session(tmp_path): async def test_get_session_yields_async_session(tmp_path):
settings = Settings( settings = Settings(
openrouter_api_key="test-key", openrouter_api_key="test-key",
database_url=f"sqlite:///{tmp_path / 'session.db'}", database=SqliteSettings(path=str(tmp_path / "session.db")),
environment="test", environment="test",
) )
initialize_database_runtime(settings=settings) initialize_database_runtime(settings=settings)
try: try:
async with get_session(settings=settings) as session: async with session_scope(settings=settings) as session:
assert session is not None assert session is not None
finally: finally:
await dispose_database_runtime() await dispose_database_runtime()
+5 -5
View File
@@ -5,11 +5,11 @@ from uuid import UUID
import pytest import pytest
from sqlalchemy.exc import IntegrityError from sqlalchemy.exc import IntegrityError
from transcription.models import Document from transcription.db.models import Document
from transcription.models import Job from transcription.db.models import Job
from transcription.models import JobStatus from transcription.db.models import JobStatus
from transcription.models import Revision from transcription.db.models import Revision
from transcription.models import Source from transcription.db.models import Source
def _make_document(**overrides) -> Document: def _make_document(**overrides) -> Document:
+12 -12
View File
@@ -4,6 +4,7 @@ from __future__ import annotations
import asyncio import asyncio
from collections.abc import Callable from collections.abc import Callable
from collections.abc import Generator
from pathlib import Path from pathlib import Path
from uuid import UUID from uuid import UUID
@@ -14,32 +15,31 @@ from sqlmodel import delete
from transcription.app import create_app from transcription.app import create_app
from transcription.config import Settings from transcription.config import Settings
from transcription.config import _settings from transcription.config import SqliteSettings
from transcription.db import create_all from transcription.db import create_all
from transcription.db import get_session
from transcription.db import initialize_database_runtime from transcription.db import initialize_database_runtime
from transcription.models import Document from transcription.db import session_scope
from transcription.models import Job from transcription.db.models import Document
from transcription.models import JobStatus from transcription.db.models import Job
from transcription.models import Revision from transcription.db.models import JobStatus
from transcription.models import Source from transcription.db.models import Revision
from transcription.db.models import Source
RevisionSeed = str RevisionSeed = str
@pytest.fixture(scope="session") @pytest.fixture(scope="session")
def app_client(tmp_path_factory: pytest.TempPathFactory) -> tuple[FastAPI, TestClient]: def app_client(tmp_path_factory: pytest.TempPathFactory) -> Generator[tuple[FastAPI, TestClient]]:
"""Provide a real application and test client backed by in-memory SQLite.""" """Provide a real application and test client backed by in-memory SQLite."""
tmp_path = tmp_path_factory.mktemp("ui") tmp_path = tmp_path_factory.mktemp("ui")
settings = Settings( settings = Settings(
openrouter_api_key="test-key", openrouter_api_key="test-key",
database_url="sqlite:///:memory:", database=SqliteSettings(path=":memory:"),
environment="test", environment="test",
bootstrap_schema_on_startup=True, bootstrap_schema_on_startup=True,
upload_dir=tmp_path / "uploads", upload_dir=tmp_path / "uploads",
prompt_dir=tmp_path / "prompts", prompt_dir=tmp_path / "prompts",
) )
_settings.set(settings)
app = create_app() app = create_app()
app.state.runtime = initialize_database_runtime(settings=settings) app.state.runtime = initialize_database_runtime(settings=settings)
@@ -54,7 +54,7 @@ def clear_ui_database(app_client: tuple[FastAPI, TestClient]) -> None:
app, _ = app_client app, _ = app_client
async def _clear() -> None: async def _clear() -> None:
async with get_session(session_factory=app.state.runtime.session_factory) as session: async with session_scope() as session:
await session.exec(delete(Revision)) await session.exec(delete(Revision))
await session.exec(delete(Source)) await session.exec(delete(Source))
await session.exec(delete(Job)) await session.exec(delete(Job))
@@ -80,7 +80,7 @@ def seed_job(app_client: tuple[FastAPI, TestClient]) -> Callable[..., UUID]:
source_file: Path | None = None, source_file: Path | None = None,
) -> UUID: ) -> UUID:
async def _insert() -> UUID: async def _insert() -> UUID:
async with get_session(session_factory=app.state.runtime.session_factory) as session: async with session_scope() as session:
stored_path = app.state.settings.upload_dir / filename stored_path = app.state.settings.upload_dir / filename
stored_path.parent.mkdir(parents=True, exist_ok=True) stored_path.parent.mkdir(parents=True, exist_ok=True)
source_path = source_file or fixtures_dir / "small_png.png" source_path = source_file or fixtures_dir / "small_png.png"
+1 -1
View File
@@ -5,7 +5,7 @@ from uuid import uuid4
import pytest import pytest
from transcription.models import JobStatus from transcription.db.models import JobStatus
@pytest.mark.integration @pytest.mark.integration
Generated
+459 -388
View File
File diff suppressed because it is too large Load Diff